2115 files changed, 191668 insertions, 57222 deletions

diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 047281a6ae11..c570b1d9f094 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -18,6 +18,7 @@
 #include <linux/mutex.h>
 #include <linux/platform_device.h>
 #include <linux/platform_data/clk-lpss.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 #include <linux/delay.h>
 
@@ -875,13 +876,14 @@ static int acpi_lpss_platform_notify(struct notifier_block *nb,
 
 	switch (action) {
 	case BUS_NOTIFY_BIND_DRIVER:
-		pdev->dev.pm_domain = &acpi_lpss_pm_domain;
+		dev_pm_domain_set(&pdev->dev, &acpi_lpss_pm_domain);
 		break;
 	case BUS_NOTIFY_DRIVER_NOT_BOUND:
 	case BUS_NOTIFY_UNBOUND_DRIVER:
 		pdev->dev.pm_domain = NULL;
 		break;
 	case BUS_NOTIFY_ADD_DEVICE:
+		dev_pm_domain_set(&pdev->dev, &acpi_lpss_pm_domain);
 		if (pdata->dev_desc->flags & LPSS_LTR)
 			return sysfs_create_group(&pdev->dev.kobj,
 						  &lpss_attr_group);
@@ -889,6 +891,7 @@ static int acpi_lpss_platform_notify(struct notifier_block *nb,
 	case BUS_NOTIFY_DEL_DEVICE:
 		if (pdata->dev_desc->flags & LPSS_LTR)
 			sysfs_remove_group(&pdev->dev.kobj, &lpss_attr_group);
+		dev_pm_domain_set(&pdev->dev, NULL);
 		break;
 	default:
 		break;
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 06a006ff89b0..a76f8be1bfe7 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -90,10 +90,10 @@ module_param(device_id_scheme, bool, 0444);
 static bool only_lcd = false;
 module_param(only_lcd, bool, 0444);
 
-static DECLARE_COMPLETION(register_done);
-static DEFINE_MUTEX(register_done_mutex);
-static struct mutex video_list_lock;
-static struct list_head video_bus_head;
+static int register_count;
+static DEFINE_MUTEX(register_count_mutex);
+static DEFINE_MUTEX(video_list_lock);
+static LIST_HEAD(video_bus_head);
 static int acpi_video_bus_add(struct acpi_device *device);
 static int acpi_video_bus_remove(struct acpi_device *device);
 static void acpi_video_bus_notify(struct acpi_device *device, u32 event);
@@ -479,6 +479,15 @@ static struct dmi_system_id video_dmi_table[] = {
 	 * as brightness control does not work.
 	 */
 	{
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .callback = video_disable_backlight_sysfs_if,
+	 .ident = "Toshiba Portege R700",
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"),
+		},
+	},
+	{
 	 /* https://bugs.freedesktop.org/show_bug.cgi?id=82634 */
 	 .callback = video_disable_backlight_sysfs_if,
 	 .ident = "Toshiba Portege R830",
@@ -487,6 +496,15 @@ static struct dmi_system_id video_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R830"),
 		},
 	},
+	{
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .callback = video_disable_backlight_sysfs_if,
+	 .ident = "Toshiba Satellite R830",
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE R830"),
+		},
+	},
 	/*
 	 * Some machine's _DOD IDs don't have bit 31(Device ID Scheme) set
 	 * but the IDs actually follow the Device ID Scheme.
@@ -2049,8 +2067,8 @@ int acpi_video_register(void)
 {
 	int ret = 0;
 
-	mutex_lock(&register_done_mutex);
-	if (completion_done(&register_done)) {
+	mutex_lock(&register_count_mutex);
+	if (register_count) {
 		/*
 		 * if the function of acpi_video_register is already called,
 		 * don't register the acpi_vide_bus again and return no error.
@@ -2058,9 +2076,6 @@ int acpi_video_register(void)
 		goto leave;
 	}
 
-	mutex_init(&video_list_lock);
-	INIT_LIST_HEAD(&video_bus_head);
-
 	dmi_check_system(video_dmi_table);
 
 	ret = acpi_bus_register_driver(&acpi_video_bus);
@@ -2071,22 +2086,22 @@ int acpi_video_register(void)
 	 * When the acpi_video_bus is loaded successfully, increase
 	 * the counter reference.
 	 */
-	complete(&register_done);
+	register_count = 1;
 
 leave:
-	mutex_unlock(&register_done_mutex);
+	mutex_unlock(&register_count_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(acpi_video_register);
 
 void acpi_video_unregister(void)
 {
-	mutex_lock(&register_done_mutex);
-	if (completion_done(&register_done)) {
+	mutex_lock(&register_count_mutex);
+	if (register_count) {
 		acpi_bus_unregister_driver(&acpi_video_bus);
-		reinit_completion(&register_done);
+		register_count = 0;
 	}
-	mutex_unlock(&register_done_mutex);
+	mutex_unlock(&register_count_mutex);
 }
 EXPORT_SYMBOL(acpi_video_unregister);
 
@@ -2094,21 +2109,20 @@ void acpi_video_unregister_backlight(void)
 {
 	struct acpi_video_bus *video;
 
-	mutex_lock(&register_done_mutex);
-	if (completion_done(&register_done)) {
+	mutex_lock(&register_count_mutex);
+	if (register_count) {
 		mutex_lock(&video_list_lock);
 		list_for_each_entry(video, &video_bus_head, entry)
 			acpi_video_bus_unregister_backlight(video);
 		mutex_unlock(&video_list_lock);
 	}
-	mutex_unlock(&register_done_mutex);
+	mutex_unlock(&register_count_mutex);
 }
 
 bool acpi_video_handles_brightness_key_presses(void)
 {
 	bool have_video_busses;
 
-	wait_for_completion(&register_done);
 	mutex_lock(&video_list_lock);
 	have_video_busses = !list_empty(&video_bus_head);
 	mutex_unlock(&video_list_lock);
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index 8b4ff40a294c..ca2c0607104b 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -49,7 +49,7 @@
 /* Common info for tool signons */
 
 #define ACPICA_NAME                 "Intel ACPI Component Architecture"
-#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2015 Intel Corporation"
+#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2016 Intel Corporation"
 
 #if ACPI_MACHINE_WIDTH == 64
 #define ACPI_WIDTH          "-64"
diff --git a/drivers/acpi/acpica/accommon.h b/drivers/acpi/acpica/accommon.h
index a8d8092ee391..19d6ec815d12 100644
--- a/drivers/acpi/acpica/accommon.h
+++ b/drivers/acpi/acpica/accommon.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h
index ecb05f1c1d5c..993af9eb007a 100644
--- a/drivers/acpi/acpica/acdebug.h
+++ b/drivers/acpi/acpica/acdebug.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acdispat.h b/drivers/acpi/acpica/acdispat.h
index 7094dc89eb81..dcd48bfedb4d 100644
--- a/drivers/acpi/acpica/acdispat.h
+++ b/drivers/acpi/acpica/acdispat.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index d18f18409071..010cf81bada9 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index 73462cac41d2..55c8197036f3 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h
index 196a55244559..27addcf50c37 100644
--- a/drivers/acpi/acpica/achware.h
+++ b/drivers/acpi/acpica/achware.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acinterp.h b/drivers/acpi/acpica/acinterp.h
index e9e936e78154..bae1a35c345f 100644
--- a/drivers/acpi/acpica/acinterp.h
+++ b/drivers/acpi/acpica/acinterp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 24928ec444de..e4977fac9c1d 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acmacros.h b/drivers/acpi/acpica/acmacros.h
index bad5bca03acc..411c18b7d541 100644
--- a/drivers/acpi/acpica/acmacros.h
+++ b/drivers/acpi/acpica/acmacros.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index d082e62d7308..9684ed61284d 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h
index 2b154cfbe136..094b042678f7 100644
--- a/drivers/acpi/acpica/acobject.h
+++ b/drivers/acpi/acpica/acobject.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acopcode.h b/drivers/acpi/acpica/acopcode.h
index 324512db62bf..ca4bda1a60be 100644
--- a/drivers/acpi/acpica/acopcode.h
+++ b/drivers/acpi/acpica/acopcode.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acparser.h b/drivers/acpi/acpica/acparser.h
index 96d510a7feba..7da639d62416 100644
--- a/drivers/acpi/acpica/acparser.h
+++ b/drivers/acpi/acpica/acparser.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h
index b9474b529fcb..52f6bee52d47 100644
--- a/drivers/acpi/acpica/acpredef.h
+++ b/drivers/acpi/acpica/acpredef.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acresrc.h b/drivers/acpi/acpica/acresrc.h
index 6357efb01b93..5dd58beafa5c 100644
--- a/drivers/acpi/acpica/acresrc.h
+++ b/drivers/acpi/acpica/acresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acstruct.h b/drivers/acpi/acpica/acstruct.h
index f9992dced1f9..b3b386e0b119 100644
--- a/drivers/acpi/acpica/acstruct.h
+++ b/drivers/acpi/acpica/acstruct.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index 591ea95319e2..848ad3ac938f 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 9e84c05c0b91..e43ab6f2ad7e 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/amlcode.h b/drivers/acpi/acpica/amlcode.h
index ab9f3f1fbb0f..ceb4f7365f7f 100644
--- a/drivers/acpi/acpica/amlcode.h
+++ b/drivers/acpi/acpica/amlcode.h
@@ -7,7 +7,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/amlresrc.h b/drivers/acpi/acpica/amlresrc.h
index ee0cdd60b93d..dee6c7ea4773 100644
--- a/drivers/acpi/acpica/amlresrc.h
+++ b/drivers/acpi/acpica/amlresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbcmds.c b/drivers/acpi/acpica/dbcmds.c
index 328c35b323d5..7ec62c461280 100644
--- a/drivers/acpi/acpica/dbcmds.c
+++ b/drivers/acpi/acpica/dbcmds.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbconvert.c b/drivers/acpi/acpica/dbconvert.c
index a71632ca8a81..9fee88f1c654 100644
--- a/drivers/acpi/acpica/dbconvert.c
+++ b/drivers/acpi/acpica/dbconvert.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbdisply.c b/drivers/acpi/acpica/dbdisply.c
index 1965b48d8e83..502bb587f112 100644
--- a/drivers/acpi/acpica/dbdisply.c
+++ b/drivers/acpi/acpica/dbdisply.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -599,12 +599,14 @@ void acpi_db_display_calling_tree(void)
 
 void acpi_db_display_object_type(char *object_arg)
 {
+	acpi_size arg;
 	acpi_handle handle;
 	struct acpi_device_info *info;
 	acpi_status status;
 	u32 i;
 
-	handle = ACPI_TO_POINTER(strtoul(object_arg, NULL, 16));
+	arg = strtoul(object_arg, NULL, 16);
+	handle = ACPI_TO_POINTER(arg);
 
 	status = acpi_get_object_info(handle, &info);
 	if (ACPI_FAILURE(status)) {
diff --git a/drivers/acpi/acpica/dbexec.c b/drivers/acpi/acpica/dbexec.c
index d713e2df65b9..c814855376e2 100644
--- a/drivers/acpi/acpica/dbexec.c
+++ b/drivers/acpi/acpica/dbexec.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbfileio.c b/drivers/acpi/acpica/dbfileio.c
index 31f54d71c51a..483287942372 100644
--- a/drivers/acpi/acpica/dbfileio.c
+++ b/drivers/acpi/acpica/dbfileio.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbhistry.c b/drivers/acpi/acpica/dbhistry.c
index 9c66a9eadd38..46bd65d38df9 100644
--- a/drivers/acpi/acpica/dbhistry.c
+++ b/drivers/acpi/acpica/dbhistry.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c
index 6203001baa30..417c02a89915 100644
--- a/drivers/acpi/acpica/dbinput.c
+++ b/drivers/acpi/acpica/dbinput.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbmethod.c b/drivers/acpi/acpica/dbmethod.c
index 01e5a71147fd..f17a86f6b16b 100644
--- a/drivers/acpi/acpica/dbmethod.c
+++ b/drivers/acpi/acpica/dbmethod.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbnames.c b/drivers/acpi/acpica/dbnames.c
index 4f68dfc6ea55..3c23b5a1079b 100644
--- a/drivers/acpi/acpica/dbnames.c
+++ b/drivers/acpi/acpica/dbnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbobject.c b/drivers/acpi/acpica/dbobject.c
index 116f6db8c2ed..1d59e8b6f859 100644
--- a/drivers/acpi/acpica/dbobject.c
+++ b/drivers/acpi/acpica/dbobject.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbstats.c b/drivers/acpi/acpica/dbstats.c
index de255d975941..a414e1fa6f9d 100644
--- a/drivers/acpi/acpica/dbstats.c
+++ b/drivers/acpi/acpica/dbstats.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbtest.c b/drivers/acpi/acpica/dbtest.c
index 68b4e8d9e1d6..74aa38156cdc 100644
--- a/drivers/acpi/acpica/dbtest.c
+++ b/drivers/acpi/acpica/dbtest.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbutils.c b/drivers/acpi/acpica/dbutils.c
index 8c85d85a9cb2..b37a2c77b86b 100644
--- a/drivers/acpi/acpica/dbutils.c
+++ b/drivers/acpi/acpica/dbutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbxface.c b/drivers/acpi/acpica/dbxface.c
index d7ff58e8c233..e94e0d80bc7b 100644
--- a/drivers/acpi/acpica/dbxface.c
+++ b/drivers/acpi/acpica/dbxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsargs.c b/drivers/acpi/acpica/dsargs.c
index 76cfced31f9f..ad0413beeeae 100644
--- a/drivers/acpi/acpica/dsargs.c
+++ b/drivers/acpi/acpica/dsargs.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dscontrol.c b/drivers/acpi/acpica/dscontrol.c
index 06a6f7f3af52..c9a663f21ac8 100644
--- a/drivers/acpi/acpica/dscontrol.c
+++ b/drivers/acpi/acpica/dscontrol.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsdebug.c b/drivers/acpi/acpica/dsdebug.c
index 1eb82bd7ee16..56c3aadb4cba 100644
--- a/drivers/acpi/acpica/dsdebug.c
+++ b/drivers/acpi/acpica/dsdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsfield.c b/drivers/acpi/acpica/dsfield.c
index 6bca0ec42dbd..6a4b603d0e83 100644
--- a/drivers/acpi/acpica/dsfield.c
+++ b/drivers/acpi/acpica/dsfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c
index c1d8af8a8aaf..5aa1c5feee50 100644
--- a/drivers/acpi/acpica/dsinit.c
+++ b/drivers/acpi/acpica/dsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c
index 6585e8e37c8e..6a72047aae1c 100644
--- a/drivers/acpi/acpica/dsmethod.c
+++ b/drivers/acpi/acpica/dsmethod.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsmthdat.c b/drivers/acpi/acpica/dsmthdat.c
index 03c44f2ac7b7..45cbebaa32c0 100644
--- a/drivers/acpi/acpica/dsmthdat.c
+++ b/drivers/acpi/acpica/dsmthdat.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsobject.c b/drivers/acpi/acpica/dsobject.c
index 302c91f5377b..c303e9d9266f 100644
--- a/drivers/acpi/acpica/dsobject.c
+++ b/drivers/acpi/acpica/dsobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c
index 1edd66f18907..4cc9d989a114 100644
--- a/drivers/acpi/acpica/dsopcode.c
+++ b/drivers/acpi/acpica/dsopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsutils.c b/drivers/acpi/acpica/dsutils.c
index fa8e2920a3ef..8ca9416320e0 100644
--- a/drivers/acpi/acpica/dsutils.c
+++ b/drivers/acpi/acpica/dsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswexec.c b/drivers/acpi/acpica/dswexec.c
index ed2f1d362092..402ecc590c56 100644
--- a/drivers/acpi/acpica/dswexec.c
+++ b/drivers/acpi/acpica/dswexec.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswload.c b/drivers/acpi/acpica/dswload.c
index b3254742aaf6..d1cedcfda1d2 100644
--- a/drivers/acpi/acpica/dswload.c
+++ b/drivers/acpi/acpica/dswload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c
index 8a32153a111b..0bac6e14170e 100644
--- a/drivers/acpi/acpica/dswload2.c
+++ b/drivers/acpi/acpica/dswload2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswscope.c b/drivers/acpi/acpica/dswscope.c
index 2d7a04493469..9f32e08a07d9 100644
--- a/drivers/acpi/acpica/dswscope.c
+++ b/drivers/acpi/acpica/dswscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswstate.c b/drivers/acpi/acpica/dswstate.c
index 89ac2022465e..3a26ddbaed6d 100644
--- a/drivers/acpi/acpica/dswstate.c
+++ b/drivers/acpi/acpica/dswstate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evevent.c b/drivers/acpi/acpica/evevent.c
index bf6873f95e72..80fc0b9b11e5 100644
--- a/drivers/acpi/acpica/evevent.c
+++ b/drivers/acpi/acpica/evevent.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evglock.c b/drivers/acpi/acpica/evglock.c
index b78dc7c6d5d7..9f015782cdd3 100644
--- a/drivers/acpi/acpica/evglock.c
+++ b/drivers/acpi/acpica/evglock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index 112e821a1cec..b47e62aaf654 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c
index c00a9f2f82d5..9275e626ed8d 100644
--- a/drivers/acpi/acpica/evgpeblk.c
+++ b/drivers/acpi/acpica/evgpeblk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeinit.c b/drivers/acpi/acpica/evgpeinit.c
index ea4c0d3fca2d..9fdd8d09141b 100644
--- a/drivers/acpi/acpica/evgpeinit.c
+++ b/drivers/acpi/acpica/evgpeinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeutil.c b/drivers/acpi/acpica/evgpeutil.c
index fd5ab9012238..66c4b5b7cd64 100644
--- a/drivers/acpi/acpica/evgpeutil.c
+++ b/drivers/acpi/acpica/evgpeutil.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c
index 709419c7cde4..0f6be8956a99 100644
--- a/drivers/acpi/acpica/evhandler.c
+++ b/drivers/acpi/acpica/evhandler.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evmisc.c b/drivers/acpi/acpica/evmisc.c
index 8866f50d38f7..c67d78c5995f 100644
--- a/drivers/acpi/acpica/evmisc.c
+++ b/drivers/acpi/acpica/evmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c
index a43178f20c59..47092b4d633c 100644
--- a/drivers/acpi/acpica/evregion.c
+++ b/drivers/acpi/acpica/evregion.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c
index bb2e529249c7..fda869c9ad0b 100644
--- a/drivers/acpi/acpica/evrgnini.c
+++ b/drivers/acpi/acpica/evrgnini.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evsci.c b/drivers/acpi/acpica/evsci.c
index 0366703d2970..3b7757c9c916 100644
--- a/drivers/acpi/acpica/evsci.c
+++ b/drivers/acpi/acpica/evsci.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxface.c b/drivers/acpi/acpica/evxface.c
index 012b9dedfa79..e4e9260cdc57 100644
--- a/drivers/acpi/acpica/evxface.c
+++ b/drivers/acpi/acpica/evxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfevnt.c b/drivers/acpi/acpica/evxfevnt.c
index 10ce48e16ebf..9179e9abe3db 100644
--- a/drivers/acpi/acpica/evxfevnt.c
+++ b/drivers/acpi/acpica/evxfevnt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 70eb47e3d724..90456714821f 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c
index 35f9e60ce2b7..d2743067126a 100644
--- a/drivers/acpi/acpica/evxfregn.c
+++ b/drivers/acpi/acpica/evxfregn.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index adcb9c7029c4..011df210b7b2 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exconvrt.c b/drivers/acpi/acpica/exconvrt.c
index 73c2e823488d..0b9f2c13b98a 100644
--- a/drivers/acpi/acpica/exconvrt.c
+++ b/drivers/acpi/acpica/exconvrt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/excreate.c b/drivers/acpi/acpica/excreate.c
index 46be5a276863..bea9612e4720 100644
--- a/drivers/acpi/acpica/excreate.c
+++ b/drivers/acpi/acpica/excreate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exdebug.c b/drivers/acpi/acpica/exdebug.c
index b22309094c5f..37a509d016da 100644
--- a/drivers/acpi/acpica/exdebug.c
+++ b/drivers/acpi/acpica/exdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c
index ff976c43b992..ee30974b245a 100644
--- a/drivers/acpi/acpica/exdump.c
+++ b/drivers/acpi/acpica/exdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exfield.c b/drivers/acpi/acpica/exfield.c
index ad7080ba65e2..d5d8020a8523 100644
--- a/drivers/acpi/acpica/exfield.c
+++ b/drivers/acpi/acpica/exfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exfldio.c b/drivers/acpi/acpica/exfldio.c
index 0337191dbf3d..f0c5ed0b7db8 100644
--- a/drivers/acpi/acpica/exfldio.c
+++ b/drivers/acpi/acpica/exfldio.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exmisc.c b/drivers/acpi/acpica/exmisc.c
index f598b3948c17..db30ae43ddd8 100644
--- a/drivers/acpi/acpica/exmisc.c
+++ b/drivers/acpi/acpica/exmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exmutex.c b/drivers/acpi/acpica/exmutex.c
index 843c60ae91f6..26faa91e930c 100644
--- a/drivers/acpi/acpica/exmutex.c
+++ b/drivers/acpi/acpica/exmutex.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exnames.c b/drivers/acpi/acpica/exnames.c
index b2e911a35866..27c11ab5eb04 100644
--- a/drivers/acpi/acpica/exnames.c
+++ b/drivers/acpi/acpica/exnames.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg1.c b/drivers/acpi/acpica/exoparg1.c
index efe7ac319f65..4e17506a7384 100644
--- a/drivers/acpi/acpica/exoparg1.c
+++ b/drivers/acpi/acpica/exoparg1.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg2.c b/drivers/acpi/acpica/exoparg2.c
index 6dad2ca1c8c9..79ef3b6811a9 100644
--- a/drivers/acpi/acpica/exoparg2.c
+++ b/drivers/acpi/acpica/exoparg2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c
index 27fb0172fca2..28eb861c44eb 100644
--- a/drivers/acpi/acpica/exoparg3.c
+++ b/drivers/acpi/acpica/exoparg3.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg6.c b/drivers/acpi/acpica/exoparg6.c
index 7efc9f47ffb9..e2b63483857f 100644
--- a/drivers/acpi/acpica/exoparg6.c
+++ b/drivers/acpi/acpica/exoparg6.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exprep.c b/drivers/acpi/acpica/exprep.c
index 1f111cc94c00..aed8d3459220 100644
--- a/drivers/acpi/acpica/exprep.c
+++ b/drivers/acpi/acpica/exprep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exregion.c b/drivers/acpi/acpica/exregion.c
index 1851a307544a..076074daf2b6 100644
--- a/drivers/acpi/acpica/exregion.c
+++ b/drivers/acpi/acpica/exregion.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresnte.c b/drivers/acpi/acpica/exresnte.c
index 6793dcc8a946..c1e8bfb0f7f4 100644
--- a/drivers/acpi/acpica/exresnte.c
+++ b/drivers/acpi/acpica/exresnte.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresolv.c b/drivers/acpi/acpica/exresolv.c
index 7f9260b129fc..fedacf13dc36 100644
--- a/drivers/acpi/acpica/exresolv.c
+++ b/drivers/acpi/acpica/exresolv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresop.c b/drivers/acpi/acpica/exresop.c
index 861453e58555..cc2c26c46a6d 100644
--- a/drivers/acpi/acpica/exresop.c
+++ b/drivers/acpi/acpica/exresop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstore.c b/drivers/acpi/acpica/exstore.c
index d3afbcbe7886..cd70cbcf6de6 100644
--- a/drivers/acpi/acpica/exstore.c
+++ b/drivers/acpi/acpica/exstore.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstoren.c b/drivers/acpi/acpica/exstoren.c
index d1841defa669..13bbb2b241a3 100644
--- a/drivers/acpi/acpica/exstoren.c
+++ b/drivers/acpi/acpica/exstoren.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstorob.c b/drivers/acpi/acpica/exstorob.c
index ad3bc92af2e6..28b724827f0f 100644
--- a/drivers/acpi/acpica/exstorob.c
+++ b/drivers/acpi/acpica/exstorob.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exsystem.c b/drivers/acpi/acpica/exsystem.c
index 7c91c1f799a5..ac09c31cc70e 100644
--- a/drivers/acpi/acpica/exsystem.c
+++ b/drivers/acpi/acpica/exsystem.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/extrace.c b/drivers/acpi/acpica/extrace.c
index e4a185eece8a..b52e84841c1a 100644
--- a/drivers/acpi/acpica/extrace.c
+++ b/drivers/acpi/acpica/extrace.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exutils.c b/drivers/acpi/acpica/exutils.c
index 8ae7634bd7d2..4d44bc1cb2ca 100644
--- a/drivers/acpi/acpica/exutils.c
+++ b/drivers/acpi/acpica/exutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwacpi.c b/drivers/acpi/acpica/hwacpi.c
index e5c5949f9081..3ebbb09030b4 100644
--- a/drivers/acpi/acpica/hwacpi.c
+++ b/drivers/acpi/acpica/hwacpi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwesleep.c b/drivers/acpi/acpica/hwesleep.c
index d0319a228ef7..3f2fb4b31fdc 100644
--- a/drivers/acpi/acpica/hwesleep.c
+++ b/drivers/acpi/acpica/hwesleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c
index 8272f966382a..1c4f4518611a 100644
--- a/drivers/acpi/acpica/hwgpe.c
+++ b/drivers/acpi/acpica/hwgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwpci.c b/drivers/acpi/acpica/hwpci.c
index f785ea788356..3dd60c96aa07 100644
--- a/drivers/acpi/acpica/hwpci.c
+++ b/drivers/acpi/acpica/hwpci.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwregs.c b/drivers/acpi/acpica/hwregs.c
index 3cf77afd142c..5ba0498412fd 100644
--- a/drivers/acpi/acpica/hwregs.c
+++ b/drivers/acpi/acpica/hwregs.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwsleep.c b/drivers/acpi/acpica/hwsleep.c
index ac5b7f768d4b..d00c9810845b 100644
--- a/drivers/acpi/acpica/hwsleep.c
+++ b/drivers/acpi/acpica/hwsleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwtimer.c b/drivers/acpi/acpica/hwtimer.c
index 675c709a300b..04cc9406c7d8 100644
--- a/drivers/acpi/acpica/hwtimer.c
+++ b/drivers/acpi/acpica/hwtimer.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwvalid.c b/drivers/acpi/acpica/hwvalid.c
index 29033d71417b..ad0a745712a9 100644
--- a/drivers/acpi/acpica/hwvalid.c
+++ b/drivers/acpi/acpica/hwvalid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwxface.c b/drivers/acpi/acpica/hwxface.c
index b2e50d8007fe..a01ddb393a55 100644
--- a/drivers/acpi/acpica/hwxface.c
+++ b/drivers/acpi/acpica/hwxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c
index 1ce4efa1a2bd..f76e0eab32b8 100644
--- a/drivers/acpi/acpica/hwxfsleep.c
+++ b/drivers/acpi/acpica/hwxfsleep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsaccess.c b/drivers/acpi/acpica/nsaccess.c
index c687b9979fb2..697af810e5ad 100644
--- a/drivers/acpi/acpica/nsaccess.c
+++ b/drivers/acpi/acpica/nsaccess.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsalloc.c b/drivers/acpi/acpica/nsalloc.c
index e107f929d9cf..c2cf73fd3918 100644
--- a/drivers/acpi/acpica/nsalloc.c
+++ b/drivers/acpi/acpica/nsalloc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsarguments.c b/drivers/acpi/acpica/nsarguments.c
index 5d347a71bd0b..f45bff632692 100644
--- a/drivers/acpi/acpica/nsarguments.c
+++ b/drivers/acpi/acpica/nsarguments.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsconvert.c b/drivers/acpi/acpica/nsconvert.c
index f21568ba325b..878e8fb6a64c 100644
--- a/drivers/acpi/acpica/nsconvert.c
+++ b/drivers/acpi/acpica/nsconvert.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsdump.c b/drivers/acpi/acpica/nsdump.c
index bc5ff358b2a7..af236e348294 100644
--- a/drivers/acpi/acpica/nsdump.c
+++ b/drivers/acpi/acpica/nsdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsdumpdv.c b/drivers/acpi/acpica/nsdumpdv.c
index 7dc367e6fe09..7060a5668989 100644
--- a/drivers/acpi/acpica/nsdumpdv.c
+++ b/drivers/acpi/acpica/nsdumpdv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nseval.c b/drivers/acpi/acpica/nseval.c
index 15e0b2ec5d65..65d58bea4320 100644
--- a/drivers/acpi/acpica/nseval.c
+++ b/drivers/acpi/acpica/nseval.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -418,7 +418,8 @@ acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
 	 * Get the parent node. We cheat by using the next_object field
 	 * of the method object descriptor.
 	 */
-	parent_node = ACPI_CAST_PTR(struct acpi_namespace_node,
+	parent_node =
+	    ACPI_CAST_PTR(struct acpi_namespace_node,
 				    method_obj->method.next_object);
 	type = acpi_ns_get_type(parent_node);
 
@@ -444,9 +445,9 @@ acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
 	info->prefix_node = parent_node;
 
 	/*
-	 * Get the currently attached parent object. Add a reference, because the
-	 * ref count will be decreased when the method object is installed to
-	 * the parent node.
+	 * Get the currently attached parent object. Add a reference,
+	 * because the ref count will be decreased when the method object
+	 * is installed to the parent node.
 	 */
 	parent_obj = acpi_ns_get_attached_object(parent_node);
 	if (parent_obj) {
@@ -455,8 +456,8 @@ acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
 
 	/* Install the method (module-level code) in the parent node */
 
-	status = acpi_ns_attach_object(parent_node, method_obj,
-				       ACPI_TYPE_METHOD);
+	status =
+	    acpi_ns_attach_object(parent_node, method_obj, ACPI_TYPE_METHOD);
 	if (ACPI_FAILURE(status)) {
 		goto exit;
 	}
diff --git a/drivers/acpi/acpica/nsinit.c b/drivers/acpi/acpica/nsinit.c
index ac59929c3ee9..bd75d46234a4 100644
--- a/drivers/acpi/acpica/nsinit.c
+++ b/drivers/acpi/acpica/nsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c
index 14c953e6fe9e..75cdb8790d49 100644
--- a/drivers/acpi/acpica/nsload.c
+++ b/drivers/acpi/acpica/nsload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c
index 521031f9b6c6..eb6e1b88a51d 100644
--- a/drivers/acpi/acpica/nsnames.c
+++ b/drivers/acpi/acpica/nsnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsobject.c b/drivers/acpi/acpica/nsobject.c
index 677bc9330e64..051306f0d0d6 100644
--- a/drivers/acpi/acpica/nsobject.c
+++ b/drivers/acpi/acpica/nsobject.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsparse.c b/drivers/acpi/acpica/nsparse.c
index 43b45a8c2fe4..f631a47724f0 100644
--- a/drivers/acpi/acpica/nsparse.c
+++ b/drivers/acpi/acpica/nsparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nspredef.c b/drivers/acpi/acpica/nspredef.c
index 0c20980bbcf3..6d7844580b2a 100644
--- a/drivers/acpi/acpica/nspredef.c
+++ b/drivers/acpi/acpica/nspredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsprepkg.c b/drivers/acpi/acpica/nsprepkg.c
index c05a83be5c11..9047f2808d5b 100644
--- a/drivers/acpi/acpica/nsprepkg.c
+++ b/drivers/acpi/acpica/nsprepkg.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsrepair.c b/drivers/acpi/acpica/nsrepair.c
index 6418863f93d5..805e36de8707 100644
--- a/drivers/acpi/acpica/nsrepair.c
+++ b/drivers/acpi/acpica/nsrepair.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsrepair2.c b/drivers/acpi/acpica/nsrepair2.c
index f6dd2a83ea63..63edbbbf9ae4 100644
--- a/drivers/acpi/acpica/nsrepair2.c
+++ b/drivers/acpi/acpica/nsrepair2.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nssearch.c b/drivers/acpi/acpica/nssearch.c
index 9cc3564de37e..61036d210274 100644
--- a/drivers/acpi/acpica/nssearch.c
+++ b/drivers/acpi/acpica/nssearch.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c
index 32f1d956eb7f..c72cc62b92d0 100644
--- a/drivers/acpi/acpica/nsutils.c
+++ b/drivers/acpi/acpica/nsutils.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c
index c68609a2bc1b..ebd731fe8e45 100644
--- a/drivers/acpi/acpica/nswalk.c
+++ b/drivers/acpi/acpica/nswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsxfeval.c b/drivers/acpi/acpica/nsxfeval.c
index 429f0d27bef0..a7deeaa8eddc 100644
--- a/drivers/acpi/acpica/nsxfeval.c
+++ b/drivers/acpi/acpica/nsxfeval.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c
index 669e0f1b0967..285b82044e7b 100644
--- a/drivers/acpi/acpica/nsxfname.c
+++ b/drivers/acpi/acpica/nsxfname.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsxfobj.c b/drivers/acpi/acpica/nsxfobj.c
index 6e1389babb47..c312cd490450 100644
--- a/drivers/acpi/acpica/nsxfobj.c
+++ b/drivers/acpi/acpica/nsxfobj.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psargs.c b/drivers/acpi/acpica/psargs.c
index f3bcfa20b0ae..305218539df2 100644
--- a/drivers/acpi/acpica/psargs.c
+++ b/drivers/acpi/acpica/psargs.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c
index a57f473bac83..6a9f5059f682 100644
--- a/drivers/acpi/acpica/psloop.c
+++ b/drivers/acpi/acpica/psloop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c
index e54bc2aa7a88..db0e90342e82 100644
--- a/drivers/acpi/acpica/psobject.c
+++ b/drivers/acpi/acpica/psobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psopcode.c b/drivers/acpi/acpica/psopcode.c
index 40909ddeebb3..8e0c97dca01f 100644
--- a/drivers/acpi/acpica/psopcode.c
+++ b/drivers/acpi/acpica/psopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psopinfo.c b/drivers/acpi/acpica/psopinfo.c
index 58310907fa7b..cfd17a4f2e91 100644
--- a/drivers/acpi/acpica/psopinfo.c
+++ b/drivers/acpi/acpica/psopinfo.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c
index b729d9b291d0..8038ed2aca05 100644
--- a/drivers/acpi/acpica/psparse.c
+++ b/drivers/acpi/acpica/psparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psscope.c b/drivers/acpi/acpica/psscope.c
index 9d669cc6cb62..560c3684ef43 100644
--- a/drivers/acpi/acpica/psscope.c
+++ b/drivers/acpi/acpica/psscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/pstree.c b/drivers/acpi/acpica/pstree.c
index cf2f2faf4f92..0288cdbda88e 100644
--- a/drivers/acpi/acpica/pstree.c
+++ b/drivers/acpi/acpica/pstree.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psutils.c b/drivers/acpi/acpica/psutils.c
index 6cb02a2a1468..b28b0da171b6 100644
--- a/drivers/acpi/acpica/psutils.c
+++ b/drivers/acpi/acpica/psutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/pswalk.c b/drivers/acpi/acpica/pswalk.c
index f620d4395b66..04f98c0a7684 100644
--- a/drivers/acpi/acpica/pswalk.c
+++ b/drivers/acpi/acpica/pswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psxface.c b/drivers/acpi/acpica/psxface.c
index 4254805dd319..04b37fcca684 100644
--- a/drivers/acpi/acpica/psxface.c
+++ b/drivers/acpi/acpica/psxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsaddr.c b/drivers/acpi/acpica/rsaddr.c
index bdb7e73cdf4a..492d5b011f33 100644
--- a/drivers/acpi/acpica/rsaddr.c
+++ b/drivers/acpi/acpica/rsaddr.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rscalc.c b/drivers/acpi/acpica/rscalc.c
index 88fce58cc545..2b1209d73e44 100644
--- a/drivers/acpi/acpica/rscalc.c
+++ b/drivers/acpi/acpica/rscalc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rscreate.c b/drivers/acpi/acpica/rscreate.c
index 603e544e3f64..12978891e842 100644
--- a/drivers/acpi/acpica/rscreate.c
+++ b/drivers/acpi/acpica/rscreate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsdump.c b/drivers/acpi/acpica/rsdump.c
index 05cc560699e1..23a17c86d5a9 100644
--- a/drivers/acpi/acpica/rsdump.c
+++ b/drivers/acpi/acpica/rsdump.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsdumpinfo.c b/drivers/acpi/acpica/rsdumpinfo.c
index b29d9ec63d1b..5c3491387f9f 100644
--- a/drivers/acpi/acpica/rsdumpinfo.c
+++ b/drivers/acpi/acpica/rsdumpinfo.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsinfo.c b/drivers/acpi/acpica/rsinfo.c
index edecfc675979..8e067cb73973 100644
--- a/drivers/acpi/acpica/rsinfo.c
+++ b/drivers/acpi/acpica/rsinfo.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsio.c b/drivers/acpi/acpica/rsio.c
index 5adba018bab0..07dfbed10d55 100644
--- a/drivers/acpi/acpica/rsio.c
+++ b/drivers/acpi/acpica/rsio.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsirq.c b/drivers/acpi/acpica/rsirq.c
index 07cfa70a475b..bc8f34590d95 100644
--- a/drivers/acpi/acpica/rsirq.c
+++ b/drivers/acpi/acpica/rsirq.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rslist.c b/drivers/acpi/acpica/rslist.c
index 286ccb461a20..8c42dd734559 100644
--- a/drivers/acpi/acpica/rslist.c
+++ b/drivers/acpi/acpica/rslist.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsmemory.c b/drivers/acpi/acpica/rsmemory.c
index c6b80862030e..88b53ef9105d 100644
--- a/drivers/acpi/acpica/rsmemory.c
+++ b/drivers/acpi/acpica/rsmemory.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsmisc.c b/drivers/acpi/acpica/rsmisc.c
index b112c7b1abbf..ce3d0b77ec89 100644
--- a/drivers/acpi/acpica/rsmisc.c
+++ b/drivers/acpi/acpica/rsmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsserial.c b/drivers/acpi/acpica/rsserial.c
index 4c8c6fe6ea74..8a01296ac7cf 100644
--- a/drivers/acpi/acpica/rsserial.c
+++ b/drivers/acpi/acpica/rsserial.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsutils.c b/drivers/acpi/acpica/rsutils.c
index 33e558c9434f..cf06e49cd91c 100644
--- a/drivers/acpi/acpica/rsutils.c
+++ b/drivers/acpi/acpica/rsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsxface.c b/drivers/acpi/acpica/rsxface.c
index 308bfd6bff5f..900933be9909 100644
--- a/drivers/acpi/acpica/rsxface.c
+++ b/drivers/acpi/acpica/rsxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index 4a8152777767..7da79ce74080 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c
index a6454f4a6fb3..a79e4f30b530 100644
--- a/drivers/acpi/acpica/tbfadt.c
+++ b/drivers/acpi/acpica/tbfadt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbfind.c b/drivers/acpi/acpica/tbfind.c
index 405529d49a1a..f2d08034630e 100644
--- a/drivers/acpi/acpica/tbfind.c
+++ b/drivers/acpi/acpica/tbfind.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index bd87801acedf..b661a1e013fb 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbprint.c b/drivers/acpi/acpica/tbprint.c
index d0d12596cfc9..fd4146d4ff49 100644
--- a/drivers/acpi/acpica/tbprint.c
+++ b/drivers/acpi/acpica/tbprint.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c
index 7c1b5f8a5cbf..3269bef371d7 100644
--- a/drivers/acpi/acpica/tbutils.c
+++ b/drivers/acpi/acpica/tbutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 5559e2c70b15..326df65decef 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index ca2f1366b498..278666e39563 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxfroot.c b/drivers/acpi/acpica/tbxfroot.c
index fa76a3603aa1..b9a78e457d19 100644
--- a/drivers/acpi/acpica/tbxfroot.c
+++ b/drivers/acpi/acpica/tbxfroot.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utaddress.c b/drivers/acpi/acpica/utaddress.c
index 38a29e235b74..c986ec66a118 100644
--- a/drivers/acpi/acpica/utaddress.c
+++ b/drivers/acpi/acpica/utaddress.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utalloc.c b/drivers/acpi/acpica/utalloc.c
index 7a4101f0685e..3dbdc3ab8b78 100644
--- a/drivers/acpi/acpica/utalloc.c
+++ b/drivers/acpi/acpica/utalloc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utbuffer.c b/drivers/acpi/acpica/utbuffer.c
index 01c8709ca586..0cfb2b8edad5 100644
--- a/drivers/acpi/acpica/utbuffer.c
+++ b/drivers/acpi/acpica/utbuffer.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utcache.c b/drivers/acpi/acpica/utcache.c
index 0d21fbd99363..c9a720f2274a 100644
--- a/drivers/acpi/acpica/utcache.c
+++ b/drivers/acpi/acpica/utcache.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utcopy.c b/drivers/acpi/acpica/utcopy.c
index ade8acf3f3a5..98d53e59ce55 100644
--- a/drivers/acpi/acpica/utcopy.c
+++ b/drivers/acpi/acpica/utcopy.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c
index 4146229103c8..1cfc5f69b033 100644
--- a/drivers/acpi/acpica/utdebug.c
+++ b/drivers/acpi/acpica/utdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index 3533135dbd4d..6ba65b02550c 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utdelete.c b/drivers/acpi/acpica/utdelete.c
index 1afd7427a90c..529d6c38ea7c 100644
--- a/drivers/acpi/acpica/utdelete.c
+++ b/drivers/acpi/acpica/utdelete.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uterror.c b/drivers/acpi/acpica/uterror.c
index f93bb90ea72a..475932cecf1a 100644
--- a/drivers/acpi/acpica/uterror.c
+++ b/drivers/acpi/acpica/uterror.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uteval.c b/drivers/acpi/acpica/uteval.c
index 6c738fa0cd42..17b9f3e6e1e1 100644
--- a/drivers/acpi/acpica/uteval.c
+++ b/drivers/acpi/acpica/uteval.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utexcep.c b/drivers/acpi/acpica/utexcep.c
index 743a0ae9fb17..695240338e00 100644
--- a/drivers/acpi/acpica/utexcep.c
+++ b/drivers/acpi/acpica/utexcep.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utglobal.c b/drivers/acpi/acpica/utglobal.c
index a72685c1e819..48fffcfe9911 100644
--- a/drivers/acpi/acpica/utglobal.c
+++ b/drivers/acpi/acpica/utglobal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uthex.c b/drivers/acpi/acpica/uthex.c
index 8ad086ed1a06..4354fb800fe4 100644
--- a/drivers/acpi/acpica/uthex.c
+++ b/drivers/acpi/acpica/uthex.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utids.c b/drivers/acpi/acpica/utids.c
index 05ee76eec314..6fb4ec365272 100644
--- a/drivers/acpi/acpica/utids.c
+++ b/drivers/acpi/acpica/utids.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c
index fd82a122785e..f91f724c487c 100644
--- a/drivers/acpi/acpica/utinit.c
+++ b/drivers/acpi/acpica/utinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utlock.c b/drivers/acpi/acpica/utlock.c
index 089f78bbd59b..3cd0978925ef 100644
--- a/drivers/acpi/acpica/utlock.c
+++ b/drivers/acpi/acpica/utlock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utmath.c b/drivers/acpi/acpica/utmath.c
index 58b5d4236429..667372093de1 100644
--- a/drivers/acpi/acpica/utmath.c
+++ b/drivers/acpi/acpica/utmath.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utmisc.c b/drivers/acpi/acpica/utmisc.c
index eab1cfeb52cc..d938c27cc6cf 100644
--- a/drivers/acpi/acpica/utmisc.c
+++ b/drivers/acpi/acpica/utmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utmutex.c b/drivers/acpi/acpica/utmutex.c
index 038ff849ad20..15073375bd00 100644
--- a/drivers/acpi/acpica/utmutex.c
+++ b/drivers/acpi/acpica/utmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utnonansi.c b/drivers/acpi/acpica/utnonansi.c
index 9c3cadc27fb8..c427a5cda465 100644
--- a/drivers/acpi/acpica/utnonansi.c
+++ b/drivers/acpi/acpica/utnonansi.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utobject.c b/drivers/acpi/acpica/utobject.c
index 787eccf6a1d5..edad3f043ab9 100644
--- a/drivers/acpi/acpica/utobject.c
+++ b/drivers/acpi/acpica/utobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utosi.c b/drivers/acpi/acpica/utosi.c
index 0809d73193e1..b5cfe577fabf 100644
--- a/drivers/acpi/acpica/utosi.c
+++ b/drivers/acpi/acpica/utosi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utownerid.c b/drivers/acpi/acpica/utownerid.c
index ebb811c43c89..813520ab8ca4 100644
--- a/drivers/acpi/acpica/utownerid.c
+++ b/drivers/acpi/acpica/utownerid.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utpredef.c b/drivers/acpi/acpica/utpredef.c
index 9f8e415bf0af..770a1775b264 100644
--- a/drivers/acpi/acpica/utpredef.c
+++ b/drivers/acpi/acpica/utpredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index 01f04da779c5..8c218ad787cd 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utresrc.c b/drivers/acpi/acpica/utresrc.c
index d50b41c4daa7..1de3376da66a 100644
--- a/drivers/acpi/acpica/utresrc.c
+++ b/drivers/acpi/acpica/utresrc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utstate.c b/drivers/acpi/acpica/utstate.c
index 0050e00997ed..f3d4dbd5fac0 100644
--- a/drivers/acpi/acpica/utstate.c
+++ b/drivers/acpi/acpica/utstate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utstring.c b/drivers/acpi/acpica/utstring.c
index 958b2f7b552d..0b005728db4e 100644
--- a/drivers/acpi/acpica/utstring.c
+++ b/drivers/acpi/acpica/utstring.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uttrack.c b/drivers/acpi/acpica/uttrack.c
index ea698e98442e..c7c2bb8f3559 100644
--- a/drivers/acpi/acpica/uttrack.c
+++ b/drivers/acpi/acpica/uttrack.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utuuid.c b/drivers/acpi/acpica/utuuid.c
index e6cab669bd9c..81088ff9d67b 100644
--- a/drivers/acpi/acpica/utuuid.c
+++ b/drivers/acpi/acpica/utuuid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxface.c b/drivers/acpi/acpica/utxface.c
index 9f3f0a1591f6..68d4673f62e6 100644
--- a/drivers/acpi/acpica/utxface.c
+++ b/drivers/acpi/acpica/utxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxferror.c b/drivers/acpi/acpica/utxferror.c
index f6cbaf451dbf..6fe59597b599 100644
--- a/drivers/acpi/acpica/utxferror.c
+++ b/drivers/acpi/acpica/utxferror.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c
index e38facd3e32f..721b87cce908 100644
--- a/drivers/acpi/acpica/utxfinit.c
+++ b/drivers/acpi/acpica/utxfinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxfmutex.c b/drivers/acpi/acpica/utxfmutex.c
index 95d6123a7010..850de0155528 100644
--- a/drivers/acpi/acpica/utxfmutex.c
+++ b/drivers/acpi/acpica/utxfmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2015, Intel Corp.
+ * Copyright (C) 2000 - 2016, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c
index 6682c5daf742..6e6bc1059301 100644
--- a/drivers/acpi/apei/erst.c
+++ b/drivers/acpi/apei/erst.c
@@ -32,6 +32,7 @@
 #include <linux/hardirq.h>
 #include <linux/pstore.h>
 #include <linux/vmalloc.h>
+#include <linux/mm.h> /* kvfree() */
 #include <acpi/apei.h>
 
 #include "apei-internal.h"
@@ -532,10 +533,7 @@ retry:
 			return -ENOMEM;
 		memcpy(new_entries, entries,
 		       erst_record_id_cache.len * sizeof(entries[0]));
-		if (erst_record_id_cache.size < PAGE_SIZE)
-			kfree(entries);
-		else
-			vfree(entries);
+		kvfree(entries);
 		erst_record_id_cache.entries = entries = new_entries;
 		erst_record_id_cache.size = new_size;
 	}
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index 08a02cdc737c..cd2c3d6d40e0 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -22,6 +22,7 @@
 #include <linux/export.h>
 #include <linux/mutex.h>
 #include <linux/pm_qos.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 
 #include "internal.h"
@@ -1059,7 +1060,7 @@ static void acpi_dev_pm_detach(struct device *dev, bool power_off)
 	struct acpi_device *adev = ACPI_COMPANION(dev);
 
 	if (adev && dev->pm_domain == &acpi_general_pm_domain) {
-		dev->pm_domain = NULL;
+		dev_pm_domain_set(dev, NULL);
 		acpi_remove_pm_notifier(adev);
 		if (power_off) {
 			/*
@@ -1111,7 +1112,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on)
 		return -EBUSY;
 
 	acpi_add_pm_notifier(adev, dev, acpi_pm_notify_work_func);
-	dev->pm_domain = &acpi_general_pm_domain;
+	dev_pm_domain_set(dev, &acpi_general_pm_domain);
 	if (power_on) {
 		acpi_dev_pm_full_power(adev);
 		acpi_device_wakeup(adev, ACPI_STATE_S0, false);
diff --git a/drivers/acpi/fan.c b/drivers/acpi/fan.c
index e297a480e135..6322db64b4a4 100644
--- a/drivers/acpi/fan.c
+++ b/drivers/acpi/fan.c
@@ -339,7 +339,7 @@ static int acpi_fan_probe(struct platform_device *pdev)
 	} else {
 		result = acpi_device_update_power(device, NULL);
 		if (result) {
-			dev_err(&device->dev, "Setting initial power state\n");
+			dev_err(&device->dev, "Failed to set initial power state\n");
 			goto end;
 		}
 	}
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index 90e2d54be526..1316ddd92fac 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -135,14 +135,6 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "UL30A"),
 		},
 	},
-	{
-	.callback = video_detect_force_vendor,
-	.ident = "Dell Inspiron 5737",
-	.matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 5737"),
-		},
-	},
 
 	/*
 	 * These models have a working acpi_video backlight control, and using
diff --git a/drivers/amba/Kconfig b/drivers/amba/Kconfig
index 4a5c9d279059..294ba6f36396 100644
--- a/drivers/amba/Kconfig
+++ b/drivers/amba/Kconfig
@@ -4,7 +4,7 @@ config ARM_AMBA
 if ARM_AMBA
 
 config TEGRA_AHB
-	bool "Enable AHB driver for NVIDIA Tegra SoCs"
+	bool
 	default y if ARCH_TEGRA
 	help
 	  Adds AHB configuration functionality for NVIDIA Tegra SoCs,
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 91bbb1959d8d..691eeea2f19a 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -200,7 +200,7 @@ static const struct attribute_group *hotplugable_cpu_attr_groups[] = {
 
 struct cpu_attr {
 	struct device_attribute attr;
-	const struct cpumask *const * const map;
+	const struct cpumask *const map;
 };
 
 static ssize_t show_cpus_attr(struct device *dev,
@@ -209,7 +209,7 @@ static ssize_t show_cpus_attr(struct device *dev,
 {
 	struct cpu_attr *ca = container_of(attr, struct cpu_attr, attr);
 
-	return cpumap_print_to_pagebuf(true, buf, *ca->map);
+	return cpumap_print_to_pagebuf(true, buf, ca->map);
 }
 
 #define _CPU_ATTR(name, map) \
@@ -217,9 +217,9 @@ static ssize_t show_cpus_attr(struct device *dev,
 
 /* Keep in sync with cpu_subsys_attrs */
 static struct cpu_attr cpu_attrs[] = {
-	_CPU_ATTR(online, &cpu_online_mask),
-	_CPU_ATTR(possible, &cpu_possible_mask),
-	_CPU_ATTR(present, &cpu_present_mask),
+	_CPU_ATTR(online, &__cpu_online_mask),
+	_CPU_ATTR(possible, &__cpu_possible_mask),
+	_CPU_ATTR(present, &__cpu_present_mask),
 };
 
 /*
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 7399be790b5d..c4da2df62e02 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -223,9 +223,23 @@ static int deferred_probe_initcall(void)
 }
 late_initcall(deferred_probe_initcall);
 
+/**
+ * device_is_bound() - Check if device is bound to a driver
+ * @dev: device to check
+ *
+ * Returns true if passed device has already finished probing successfully
+ * against a driver.
+ *
+ * This function must be called with the device lock held.
+ */
+bool device_is_bound(struct device *dev)
+{
+	return dev->p && klist_node_attached(&dev->p->knode_driver);
+}
+
 static void driver_bound(struct device *dev)
 {
-	if (klist_node_attached(&dev->p->knode_driver)) {
+	if (device_is_bound(dev)) {
 		printk(KERN_WARNING "%s: device %s already bound\n",
 			__func__, kobject_name(&dev->kobj));
 		return;
@@ -236,6 +250,8 @@ static void driver_bound(struct device *dev)
 
 	klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices);
 
+	device_pm_check_callbacks(dev);
+
 	/*
 	 * Make sure the device is no longer in one of the deferred lists and
 	 * kick off retrying all pending devices
@@ -601,7 +617,7 @@ static int __device_attach(struct device *dev, bool allow_async)
 
 	device_lock(dev);
 	if (dev->driver) {
-		if (klist_node_attached(&dev->p->knode_driver)) {
+		if (device_is_bound(dev)) {
 			ret = 1;
 			goto out_unlock;
 		}
@@ -752,6 +768,7 @@ static void __device_release_driver(struct device *dev)
 		pm_runtime_reinit(dev);
 
 		klist_remove(&dev->p->knode_driver);
+		device_pm_check_callbacks(dev);
 		if (dev->bus)
 			blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 						     BUS_NOTIFY_UNBOUND_DRIVER,
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index 68f03141e432..44a74cf1372c 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -215,9 +215,9 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid,
 		newattrs.ia_uid = uid;
 		newattrs.ia_gid = gid;
 		newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID;
-		mutex_lock(&d_inode(dentry)->i_mutex);
+		inode_lock(d_inode(dentry));
 		notify_change(dentry, &newattrs, NULL);
-		mutex_unlock(&d_inode(dentry)->i_mutex);
+		inode_unlock(d_inode(dentry));
 
 		/* mark as kernel-created inode */
 		d_inode(dentry)->i_private = &thread;
@@ -244,7 +244,7 @@ static int dev_rmdir(const char *name)
 		err = -ENOENT;
 	}
 	dput(dentry);
-	mutex_unlock(&d_inode(parent.dentry)->i_mutex);
+	inode_unlock(d_inode(parent.dentry));
 	path_put(&parent);
 	return err;
 }
@@ -321,9 +321,9 @@ static int handle_remove(const char *nodename, struct device *dev)
 			newattrs.ia_mode = stat.mode & ~0777;
 			newattrs.ia_valid =
 				ATTR_UID|ATTR_GID|ATTR_MODE;
-			mutex_lock(&d_inode(dentry)->i_mutex);
+			inode_lock(d_inode(dentry));
 			notify_change(dentry, &newattrs, NULL);
-			mutex_unlock(&d_inode(dentry)->i_mutex);
+			inode_unlock(d_inode(dentry));
 			err = vfs_unlink(d_inode(parent.dentry), dentry, NULL);
 			if (!err || err == -ENOENT)
 				deleted = 1;
@@ -332,7 +332,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		err = -ENOENT;
 	}
 	dput(dentry);
-	mutex_unlock(&d_inode(parent.dentry)->i_mutex);
+	inode_unlock(d_inode(parent.dentry));
 
 	path_put(&parent);
 	if (deleted && strchr(nodename, '/'))
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index d95c5971c225..d799662f19eb 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -12,7 +12,6 @@
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
-#include <asm-generic/dma-coherent.h>
 
 /*
  * Managed DMA API
@@ -167,7 +166,7 @@ void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr,
 }
 EXPORT_SYMBOL(dmam_free_noncoherent);
 
-#ifdef ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY
+#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
 
 static void dmam_coherent_decl_release(struct device *dev, void *res)
 {
@@ -247,7 +246,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 		    void *cpu_addr, dma_addr_t dma_addr, size_t size)
 {
 	int ret = -ENXIO;
-#ifdef CONFIG_MMU
+#if defined(CONFIG_MMU) && !defined(CONFIG_ARCH_NO_COHERENT_DMA_MMAP)
 	unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
@@ -264,7 +263,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 				      user_count << PAGE_SHIFT,
 				      vma->vm_page_prot);
 	}
-#endif	/* CONFIG_MMU */
+#endif	/* CONFIG_MMU && !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
 
 	return ret;
 }
diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c
index 8524450e75bd..b9250e564ebf 100644
--- a/drivers/base/firmware_class.c
+++ b/drivers/base/firmware_class.c
@@ -1118,15 +1118,17 @@ static int
 _request_firmware(const struct firmware **firmware_p, const char *name,
 		  struct device *device, unsigned int opt_flags)
 {
-	struct firmware *fw;
+	struct firmware *fw = NULL;
 	long timeout;
 	int ret;
 
 	if (!firmware_p)
 		return -EINVAL;
 
-	if (!name || name[0] == '\0')
-		return -EINVAL;
+	if (!name || name[0] == '\0') {
+		ret = -EINVAL;
+		goto out;
+	}
 
 	ret = _request_firmware_prepare(&fw, name, device);
 	if (ret <= 0) /* error or already assigned */
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 25425d3f2575..213456c2b123 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -450,8 +450,7 @@ memory_probe_store(struct device *dev, struct device_attribute *attr,
 		   const char *buf, size_t count)
 {
 	u64 phys_addr;
-	int nid;
-	int i, ret;
+	int nid, ret;
 	unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
 
 	ret = kstrtoull(buf, 0, &phys_addr);
@@ -461,15 +460,12 @@ memory_probe_store(struct device *dev, struct device_attribute *attr,
 	if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
 		return -EINVAL;
 
-	for (i = 0; i < sections_per_block; i++) {
-		nid = memory_add_physaddr_to_nid(phys_addr);
-		ret = add_memory(nid, phys_addr,
-				 PAGES_PER_SECTION << PAGE_SHIFT);
-		if (ret)
-			goto out;
+	nid = memory_add_physaddr_to_nid(phys_addr);
+	ret = add_memory(nid, phys_addr,
+			 MIN_MEMORY_BLOCK_SIZE * sections_per_block);
 
-		phys_addr += MIN_MEMORY_BLOCK_SIZE;
-	}
+	if (ret)
+		goto out;
 
 	ret = count;
 out:
@@ -618,7 +614,6 @@ static int init_memory_block(struct memory_block **memory,
 			base_memory_block_id(scn_nr) * sections_per_block;
 	mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
 	mem->state = state;
-	mem->section_count++;
 	start_pfn = section_nr_to_pfn(mem->start_section_nr);
 	mem->phys_device = arch_get_memory_phys_device(start_pfn);
 
@@ -652,6 +647,13 @@ static int add_memory_block(int base_section_nr)
 	return 0;
 }
 
+static bool is_zone_device_section(struct mem_section *ms)
+{
+	struct page *page;
+
+	page = sparse_decode_mem_map(ms->section_mem_map, __section_nr(ms));
+	return is_zone_device_page(page);
+}
 
 /*
  * need an interface for the VM to add new memory regions,
@@ -662,6 +664,9 @@ int register_new_memory(int nid, struct mem_section *section)
 	int ret = 0;
 	struct memory_block *mem;
 
+	if (is_zone_device_section(section))
+		return 0;
+
 	mutex_lock(&mem_sysfs_mutex);
 
 	mem = find_memory_block(section);
@@ -672,6 +677,7 @@ int register_new_memory(int nid, struct mem_section *section)
 		ret = init_memory_block(&mem, section, MEM_OFFLINE);
 		if (ret)
 			goto out;
+		mem->section_count++;
 	}
 
 	if (mem->section_count == sections_per_block)
@@ -692,11 +698,14 @@ unregister_memory(struct memory_block *memory)
 	device_unregister(&memory->dev);
 }
 
-static int remove_memory_block(unsigned long node_id,
+static int remove_memory_section(unsigned long node_id,
 			       struct mem_section *section, int phys_device)
 {
 	struct memory_block *mem;
 
+	if (is_zone_device_section(section))
+		return 0;
+
 	mutex_lock(&mem_sysfs_mutex);
 	mem = find_memory_block(section);
 	unregister_mem_sect_under_nodes(mem, __section_nr(section));
@@ -716,7 +725,7 @@ int unregister_memory_section(struct mem_section *section)
 	if (!present_section(section))
 		return -EINVAL;
 
-	return remove_memory_block(0, section, 0);
+	return remove_memory_section(0, section, 0);
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 47c43386786b..279e53989374 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -284,6 +284,7 @@ out_free_priv_data:
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs);
 
 /**
  * platform_msi_domain_free_irqs - Free MSI interrupts for @dev
@@ -301,6 +302,7 @@ void platform_msi_domain_free_irqs(struct device *dev)
 	msi_domain_free_irqs(dev->msi_domain, dev);
 	platform_msi_free_descs(dev, 0, MAX_DEV_MSIS);
 }
+EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs);
 
 /**
  * platform_msi_get_host_data - Query the private data associated with
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 8dcbb266643b..73d6e5d39e33 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -597,7 +597,6 @@ static void platform_drv_shutdown(struct device *_dev)
 
 	if (drv->shutdown)
 		drv->shutdown(dev);
-	dev_pm_domain_detach(_dev, true);
 }
 
 /**
diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c
index c39b8617280f..272a52ebafc0 100644
--- a/drivers/base/power/clock_ops.c
+++ b/drivers/base/power/clock_ops.c
@@ -15,6 +15,7 @@
 #include <linux/clkdev.h>
 #include <linux/slab.h>
 #include <linux/err.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 
 #ifdef CONFIG_PM_CLK
@@ -348,7 +349,7 @@ static int pm_clk_notify(struct notifier_block *nb,
 		if (error)
 			break;
 
-		dev->pm_domain = clknb->pm_domain;
+		dev_pm_domain_set(dev, clknb->pm_domain);
 		if (clknb->con_ids[0]) {
 			for (con_id = clknb->con_ids; *con_id; con_id++)
 				pm_clk_add(dev, *con_id);
@@ -361,7 +362,7 @@ static int pm_clk_notify(struct notifier_block *nb,
 		if (dev->pm_domain != clknb->pm_domain)
 			break;
 
-		dev->pm_domain = NULL;
+		dev_pm_domain_set(dev, NULL);
 		pm_clk_destroy(dev);
 		break;
 	}
diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c
index f48e33385b3e..93ed14cc2252 100644
--- a/drivers/base/power/common.c
+++ b/drivers/base/power/common.c
@@ -14,6 +14,8 @@
 #include <linux/acpi.h>
 #include <linux/pm_domain.h>
 
+#include "power.h"
+
 /**
  * dev_pm_get_subsys_data - Create or refcount power.subsys_data for device.
  * @dev: Device to handle.
@@ -128,3 +130,25 @@ void dev_pm_domain_detach(struct device *dev, bool power_off)
 		dev->pm_domain->detach(dev, power_off);
 }
 EXPORT_SYMBOL_GPL(dev_pm_domain_detach);
+
+/**
+ * dev_pm_domain_set - Set PM domain of a device.
+ * @dev: Device whose PM domain is to be set.
+ * @pd: PM domain to be set, or NULL.
+ *
+ * Sets the PM domain the device belongs to. The PM domain of a device needs
+ * to be set before its probe finishes (it's bound to a driver).
+ *
+ * This function must be called with the device lock held.
+ */
+void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd)
+{
+	if (dev->pm_domain == pd)
+		return;
+
+	WARN(device_is_bound(dev),
+	     "PM domains can only be changed for unbound devices\n");
+	dev->pm_domain = pd;
+	device_pm_check_callbacks(dev);
+}
+EXPORT_SYMBOL_GPL(dev_pm_domain_set);
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index b80379012840..784dbe897a5e 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -20,6 +20,8 @@
 #include <linux/suspend.h>
 #include <linux/export.h>
 
+#include "power.h"
+
 #define GENPD_RETRY_MAX_MS	250		/* Approximate */
 
 #define GENPD_DEV_CALLBACK(genpd, type, callback, dev)		\
@@ -160,7 +162,7 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool timed)
 
 /**
  * genpd_queue_power_off_work - Queue up the execution of genpd_poweroff().
- * @genpd: PM domait to power off.
+ * @genpd: PM domain to power off.
  *
  * Queue up the execution of genpd_poweroff() unless it's already been done
  * before.
@@ -170,16 +172,15 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
 	queue_work(pm_wq, &genpd->power_off_work);
 }
 
-static int genpd_poweron(struct generic_pm_domain *genpd);
-
 /**
  * __genpd_poweron - Restore power to a given PM domain and its masters.
  * @genpd: PM domain to power up.
+ * @depth: nesting count for lockdep.
  *
  * Restore power to @genpd and all of its masters so that it is possible to
  * resume a device belonging to it.
  */
-static int __genpd_poweron(struct generic_pm_domain *genpd)
+static int __genpd_poweron(struct generic_pm_domain *genpd, unsigned int depth)
 {
 	struct gpd_link *link;
 	int ret = 0;
@@ -194,11 +195,16 @@ static int __genpd_poweron(struct generic_pm_domain *genpd)
 	 * with it.
 	 */
 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
-		genpd_sd_counter_inc(link->master);
+		struct generic_pm_domain *master = link->master;
+
+		genpd_sd_counter_inc(master);
+
+		mutex_lock_nested(&master->lock, depth + 1);
+		ret = __genpd_poweron(master, depth + 1);
+		mutex_unlock(&master->lock);
 
-		ret = genpd_poweron(link->master);
 		if (ret) {
-			genpd_sd_counter_dec(link->master);
+			genpd_sd_counter_dec(master);
 			goto err;
 		}
 	}
@@ -230,11 +236,12 @@ static int genpd_poweron(struct generic_pm_domain *genpd)
 	int ret;
 
 	mutex_lock(&genpd->lock);
-	ret = __genpd_poweron(genpd);
+	ret = __genpd_poweron(genpd, 0);
 	mutex_unlock(&genpd->lock);
 	return ret;
 }
 
+
 static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)
 {
 	return GENPD_DEV_CALLBACK(genpd, int, save_state, dev);
@@ -482,7 +489,7 @@ static int pm_genpd_runtime_resume(struct device *dev)
 	}
 
 	mutex_lock(&genpd->lock);
-	ret = __genpd_poweron(genpd);
+	ret = __genpd_poweron(genpd, 0);
 	mutex_unlock(&genpd->lock);
 
 	if (ret)
@@ -1188,10 +1195,11 @@ static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
 	}
 
 	dev->power.subsys_data->domain_data = &gpd_data->base;
-	dev->pm_domain = &genpd->domain;
 
 	spin_unlock_irq(&dev->power.lock);
 
+	dev_pm_domain_set(dev, &genpd->domain);
+
 	return gpd_data;
 
  err_free:
@@ -1205,9 +1213,10 @@ static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
 static void genpd_free_dev_data(struct device *dev,
 				struct generic_pm_domain_data *gpd_data)
 {
+	dev_pm_domain_set(dev, NULL);
+
 	spin_lock_irq(&dev->power.lock);
 
-	dev->pm_domain = NULL;
 	dev->power.subsys_data->domain_data = NULL;
 
 	spin_unlock_irq(&dev->power.lock);
@@ -1335,8 +1344,8 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 	if (!link)
 		return -ENOMEM;
 
-	mutex_lock(&genpd->lock);
-	mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
+	mutex_lock(&subdomain->lock);
+	mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING);
 
 	if (genpd->status == GPD_STATE_POWER_OFF
 	    &&  subdomain->status != GPD_STATE_POWER_OFF) {
@@ -1359,8 +1368,8 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 		genpd_sd_counter_inc(genpd);
 
  out:
-	mutex_unlock(&subdomain->lock);
 	mutex_unlock(&genpd->lock);
+	mutex_unlock(&subdomain->lock);
 	if (ret)
 		kfree(link);
 	return ret;
@@ -1381,7 +1390,8 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
 		return -EINVAL;
 
-	mutex_lock(&genpd->lock);
+	mutex_lock(&subdomain->lock);
+	mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING);
 
 	if (!list_empty(&subdomain->slave_links) || subdomain->device_count) {
 		pr_warn("%s: unable to remove subdomain %s\n", genpd->name,
@@ -1394,22 +1404,19 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 		if (link->slave != subdomain)
 			continue;
 
-		mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
-
 		list_del(&link->master_node);
 		list_del(&link->slave_node);
 		kfree(link);
 		if (subdomain->status != GPD_STATE_POWER_OFF)
 			genpd_sd_counter_dec(genpd);
 
-		mutex_unlock(&subdomain->lock);
-
 		ret = 0;
 		break;
 	}
 
 out:
 	mutex_unlock(&genpd->lock);
+	mutex_unlock(&subdomain->lock);
 
 	return ret;
 }
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 9d626ac08d9c..6e7c3ccea24b 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -125,6 +125,7 @@ void device_pm_add(struct device *dev)
 {
 	pr_debug("PM: Adding info for %s:%s\n",
 		 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
+	device_pm_check_callbacks(dev);
 	mutex_lock(&dpm_list_mtx);
 	if (dev->parent && dev->parent->power.is_prepared)
 		dev_warn(dev, "parent %s should not be sleeping\n",
@@ -147,6 +148,7 @@ void device_pm_remove(struct device *dev)
 	mutex_unlock(&dpm_list_mtx);
 	device_wakeup_disable(dev);
 	pm_runtime_remove(dev);
+	device_pm_check_callbacks(dev);
 }
 
 /**
@@ -1572,6 +1574,11 @@ static int device_prepare(struct device *dev, pm_message_t state)
 
 	dev->power.wakeup_path = device_may_wakeup(dev);
 
+	if (dev->power.no_pm_callbacks) {
+		ret = 1;	/* Let device go direct_complete */
+		goto unlock;
+	}
+
 	if (dev->pm_domain) {
 		info = "preparing power domain ";
 		callback = dev->pm_domain->ops.prepare;
@@ -1594,6 +1601,7 @@ static int device_prepare(struct device *dev, pm_message_t state)
 	if (callback)
 		ret = callback(dev);
 
+unlock:
 	device_unlock(dev);
 
 	if (ret < 0) {
@@ -1736,3 +1744,30 @@ void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
 	device_pm_unlock();
 }
 EXPORT_SYMBOL_GPL(dpm_for_each_dev);
+
+static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
+{
+	if (!ops)
+		return true;
+
+	return !ops->prepare &&
+	       !ops->suspend &&
+	       !ops->suspend_late &&
+	       !ops->suspend_noirq &&
+	       !ops->resume_noirq &&
+	       !ops->resume_early &&
+	       !ops->resume &&
+	       !ops->complete;
+}
+
+void device_pm_check_callbacks(struct device *dev)
+{
+	spin_lock_irq(&dev->power.lock);
+	dev->power.no_pm_callbacks =
+		(!dev->bus || pm_ops_is_empty(dev->bus->pm)) &&
+		(!dev->class || pm_ops_is_empty(dev->class->pm)) &&
+		(!dev->type || pm_ops_is_empty(dev->type->pm)) &&
+		(!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
+		(!dev->driver || pm_ops_is_empty(dev->driver->pm));
+	spin_unlock_irq(&dev->power.lock);
+}
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index 8b06193d4a5e..50e30e7b059d 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -125,6 +125,7 @@ extern void device_pm_remove(struct device *);
 extern void device_pm_move_before(struct device *, struct device *);
 extern void device_pm_move_after(struct device *, struct device *);
 extern void device_pm_move_last(struct device *);
+extern void device_pm_check_callbacks(struct device *dev);
 
 #else /* !CONFIG_PM_SLEEP */
 
@@ -143,6 +144,8 @@ static inline void device_pm_move_after(struct device *deva,
 					struct device *devb) {}
 static inline void device_pm_move_last(struct device *dev) {}
 
+static inline void device_pm_check_callbacks(struct device *dev) {}
+
 #endif /* !CONFIG_PM_SLEEP */
 
 static inline void device_pm_init(struct device *dev)
diff --git a/drivers/bcma/driver_gpio.c b/drivers/bcma/driver_gpio.c
index 504899a72966..98067f757fb0 100644
--- a/drivers/bcma/driver_gpio.c
+++ b/drivers/bcma/driver_gpio.c
@@ -17,14 +17,9 @@
 
 #define BCMA_GPIO_MAX_PINS	32
 
-static inline struct bcma_drv_cc *bcma_gpio_get_cc(struct gpio_chip *chip)
-{
-	return container_of(chip, struct bcma_drv_cc, gpio);
-}
-
 static int bcma_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	return !!bcma_chipco_gpio_in(cc, 1 << gpio);
 }
@@ -32,14 +27,14 @@ static int bcma_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 static void bcma_gpio_set_value(struct gpio_chip *chip, unsigned gpio,
 				int value)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	bcma_chipco_gpio_out(cc, 1 << gpio, value ? 1 << gpio : 0);
 }
 
 static int bcma_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	bcma_chipco_gpio_outen(cc, 1 << gpio, 0);
 	return 0;
@@ -48,7 +43,7 @@ static int bcma_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 static int bcma_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 				      int value)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	bcma_chipco_gpio_outen(cc, 1 << gpio, 1 << gpio);
 	bcma_chipco_gpio_out(cc, 1 << gpio, value ? 1 << gpio : 0);
@@ -57,7 +52,7 @@ static int bcma_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 
 static int bcma_gpio_request(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	bcma_chipco_gpio_control(cc, 1 << gpio, 0);
 	/* clear pulldown */
@@ -70,7 +65,7 @@ static int bcma_gpio_request(struct gpio_chip *chip, unsigned gpio)
 
 static void bcma_gpio_free(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip);
+	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
 
 	/* clear pullup */
 	bcma_chipco_gpio_pullup(cc, 1 << gpio, 0);
@@ -81,7 +76,7 @@ static void bcma_gpio_free(struct gpio_chip *chip, unsigned gpio)
 static void bcma_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(gc);
+	struct bcma_drv_cc *cc = gpiochip_get_data(gc);
 	int gpio = irqd_to_hwirq(d);
 	u32 val = bcma_chipco_gpio_in(cc, BIT(gpio));
 
@@ -92,7 +87,7 @@ static void bcma_gpio_irq_unmask(struct irq_data *d)
 static void bcma_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct bcma_drv_cc *cc = bcma_gpio_get_cc(gc);
+	struct bcma_drv_cc *cc = gpiochip_get_data(gc);
 	int gpio = irqd_to_hwirq(d);
 
 	bcma_chipco_gpio_intmask(cc, BIT(gpio), 0);
@@ -188,7 +183,7 @@ int bcma_gpio_init(struct bcma_drv_cc *cc)
 	chip->direction_input	= bcma_gpio_direction_input;
 	chip->direction_output	= bcma_gpio_direction_output;
 	chip->owner		= THIS_MODULE;
-	chip->dev		= bcma_bus_get_host_dev(bus);
+	chip->parent		= bcma_bus_get_host_dev(bus);
 #if IS_BUILTIN(CONFIG_OF)
 	if (cc->core->bus->hosttype == BCMA_HOSTTYPE_SOC)
 		chip->of_node	= cc->core->dev.of_node;
@@ -216,7 +211,7 @@ int bcma_gpio_init(struct bcma_drv_cc *cc)
 	else
 		chip->base		= -1;
 
-	err = gpiochip_add(chip);
+	err = gpiochip_add_data(chip, cc);
 	if (err)
 		return err;
 
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index ad80c85e0857..d048d2009e89 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -964,9 +964,9 @@ aoecmd_sleepwork(struct work_struct *work)
 		ssize = get_capacity(d->gd);
 		bd = bdget_disk(d->gd, 0);
 		if (bd) {
-			mutex_lock(&bd->bd_inode->i_mutex);
+			inode_lock(bd->bd_inode);
 			i_size_write(bd->bd_inode, (loff_t)ssize<<9);
-			mutex_unlock(&bd->bd_inode->i_mutex);
+			inode_unlock(bd->bd_inode);
 			bdput(bd);
 		}
 		spin_lock_irq(&d->lock);
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index a5880f4ab40e..cb27190e9f39 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -19,6 +19,9 @@
 #include <linux/radix-tree.h>
 #include <linux/fs.h>
 #include <linux/slab.h>
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+#include <linux/pfn_t.h>
+#endif
 
 #include <asm/uaccess.h>
 
@@ -378,7 +381,7 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector,
 
 #ifdef CONFIG_BLK_DEV_RAM_DAX
 static long brd_direct_access(struct block_device *bdev, sector_t sector,
-			void __pmem **kaddr, unsigned long *pfn)
+			void __pmem **kaddr, pfn_t *pfn)
 {
 	struct brd_device *brd = bdev->bd_disk->private_data;
 	struct page *page;
@@ -389,7 +392,7 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector,
 	if (!page)
 		return -ENOSPC;
 	*kaddr = (void __pmem *)page_address(page);
-	*pfn = page_to_pfn(page);
+	*pfn = page_to_pfn_t(page);
 
 	return PAGE_SIZE;
 }
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index b3868e7a1ffd..10459a145062 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -288,7 +288,162 @@ bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *
 	return need_transaction;
 }
 
-static int al_write_transaction(struct drbd_device *device);
+#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
+/* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
+ * are still coupled, or assume too much about their relation.
+ * Code below will not work if this is violated.
+ * Will be cleaned up with some followup patch.
+ */
+# error FIXME
+#endif
+
+static unsigned int al_extent_to_bm_page(unsigned int al_enr)
+{
+	return al_enr >>
+		/* bit to page */
+		((PAGE_SHIFT + 3) -
+		/* al extent number to bit */
+		 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
+}
+
+static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
+{
+	const unsigned int stripes = device->ldev->md.al_stripes;
+	const unsigned int stripe_size_4kB = device->ldev->md.al_stripe_size_4k;
+
+	/* transaction number, modulo on-disk ring buffer wrap around */
+	unsigned int t = device->al_tr_number % (device->ldev->md.al_size_4k);
+
+	/* ... to aligned 4k on disk block */
+	t = ((t % stripes) * stripe_size_4kB) + t/stripes;
+
+	/* ... to 512 byte sector in activity log */
+	t *= 8;
+
+	/* ... plus offset to the on disk position */
+	return device->ldev->md.md_offset + device->ldev->md.al_offset + t;
+}
+
+static int __al_write_transaction(struct drbd_device *device, struct al_transaction_on_disk *buffer)
+{
+	struct lc_element *e;
+	sector_t sector;
+	int i, mx;
+	unsigned extent_nr;
+	unsigned crc = 0;
+	int err = 0;
+
+	memset(buffer, 0, sizeof(*buffer));
+	buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
+	buffer->tr_number = cpu_to_be32(device->al_tr_number);
+
+	i = 0;
+
+	/* Even though no one can start to change this list
+	 * once we set the LC_LOCKED -- from drbd_al_begin_io(),
+	 * lc_try_lock_for_transaction() --, someone may still
+	 * be in the process of changing it. */
+	spin_lock_irq(&device->al_lock);
+	list_for_each_entry(e, &device->act_log->to_be_changed, list) {
+		if (i == AL_UPDATES_PER_TRANSACTION) {
+			i++;
+			break;
+		}
+		buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
+		buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
+		if (e->lc_number != LC_FREE)
+			drbd_bm_mark_for_writeout(device,
+					al_extent_to_bm_page(e->lc_number));
+		i++;
+	}
+	spin_unlock_irq(&device->al_lock);
+	BUG_ON(i > AL_UPDATES_PER_TRANSACTION);
+
+	buffer->n_updates = cpu_to_be16(i);
+	for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) {
+		buffer->update_slot_nr[i] = cpu_to_be16(-1);
+		buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE);
+	}
+
+	buffer->context_size = cpu_to_be16(device->act_log->nr_elements);
+	buffer->context_start_slot_nr = cpu_to_be16(device->al_tr_cycle);
+
+	mx = min_t(int, AL_CONTEXT_PER_TRANSACTION,
+		   device->act_log->nr_elements - device->al_tr_cycle);
+	for (i = 0; i < mx; i++) {
+		unsigned idx = device->al_tr_cycle + i;
+		extent_nr = lc_element_by_index(device->act_log, idx)->lc_number;
+		buffer->context[i] = cpu_to_be32(extent_nr);
+	}
+	for (; i < AL_CONTEXT_PER_TRANSACTION; i++)
+		buffer->context[i] = cpu_to_be32(LC_FREE);
+
+	device->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION;
+	if (device->al_tr_cycle >= device->act_log->nr_elements)
+		device->al_tr_cycle = 0;
+
+	sector = al_tr_number_to_on_disk_sector(device);
+
+	crc = crc32c(0, buffer, 4096);
+	buffer->crc32c = cpu_to_be32(crc);
+
+	if (drbd_bm_write_hinted(device))
+		err = -EIO;
+	else {
+		bool write_al_updates;
+		rcu_read_lock();
+		write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
+		rcu_read_unlock();
+		if (write_al_updates) {
+			if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
+				err = -EIO;
+				drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
+			} else {
+				device->al_tr_number++;
+				device->al_writ_cnt++;
+			}
+		}
+	}
+
+	return err;
+}
+
+static int al_write_transaction(struct drbd_device *device)
+{
+	struct al_transaction_on_disk *buffer;
+	int err;
+
+	if (!get_ldev(device)) {
+		drbd_err(device, "disk is %s, cannot start al transaction\n",
+			drbd_disk_str(device->state.disk));
+		return -EIO;
+	}
+
+	/* The bitmap write may have failed, causing a state change. */
+	if (device->state.disk < D_INCONSISTENT) {
+		drbd_err(device,
+			"disk is %s, cannot write al transaction\n",
+			drbd_disk_str(device->state.disk));
+		put_ldev(device);
+		return -EIO;
+	}
+
+	/* protects md_io_buffer, al_tr_cycle, ... */
+	buffer = drbd_md_get_buffer(device, __func__);
+	if (!buffer) {
+		drbd_err(device, "disk failed while waiting for md_io buffer\n");
+		put_ldev(device);
+		return -ENODEV;
+	}
+
+	err = __al_write_transaction(device, buffer);
+
+	drbd_md_put_buffer(device);
+	put_ldev(device);
+
+	return err;
+}
+
 
 void drbd_al_begin_io_commit(struct drbd_device *device)
 {
@@ -420,153 +575,6 @@ void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i)
 	wake_up(&device->al_wait);
 }
 
-#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
-/* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
- * are still coupled, or assume too much about their relation.
- * Code below will not work if this is violated.
- * Will be cleaned up with some followup patch.
- */
-# error FIXME
-#endif
-
-static unsigned int al_extent_to_bm_page(unsigned int al_enr)
-{
-	return al_enr >>
-		/* bit to page */
-		((PAGE_SHIFT + 3) -
-		/* al extent number to bit */
-		 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
-}
-
-static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
-{
-	const unsigned int stripes = device->ldev->md.al_stripes;
-	const unsigned int stripe_size_4kB = device->ldev->md.al_stripe_size_4k;
-
-	/* transaction number, modulo on-disk ring buffer wrap around */
-	unsigned int t = device->al_tr_number % (device->ldev->md.al_size_4k);
-
-	/* ... to aligned 4k on disk block */
-	t = ((t % stripes) * stripe_size_4kB) + t/stripes;
-
-	/* ... to 512 byte sector in activity log */
-	t *= 8;
-
-	/* ... plus offset to the on disk position */
-	return device->ldev->md.md_offset + device->ldev->md.al_offset + t;
-}
-
-int al_write_transaction(struct drbd_device *device)
-{
-	struct al_transaction_on_disk *buffer;
-	struct lc_element *e;
-	sector_t sector;
-	int i, mx;
-	unsigned extent_nr;
-	unsigned crc = 0;
-	int err = 0;
-
-	if (!get_ldev(device)) {
-		drbd_err(device, "disk is %s, cannot start al transaction\n",
-			drbd_disk_str(device->state.disk));
-		return -EIO;
-	}
-
-	/* The bitmap write may have failed, causing a state change. */
-	if (device->state.disk < D_INCONSISTENT) {
-		drbd_err(device,
-			"disk is %s, cannot write al transaction\n",
-			drbd_disk_str(device->state.disk));
-		put_ldev(device);
-		return -EIO;
-	}
-
-	/* protects md_io_buffer, al_tr_cycle, ... */
-	buffer = drbd_md_get_buffer(device, __func__);
-	if (!buffer) {
-		drbd_err(device, "disk failed while waiting for md_io buffer\n");
-		put_ldev(device);
-		return -ENODEV;
-	}
-
-	memset(buffer, 0, sizeof(*buffer));
-	buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
-	buffer->tr_number = cpu_to_be32(device->al_tr_number);
-
-	i = 0;
-
-	/* Even though no one can start to change this list
-	 * once we set the LC_LOCKED -- from drbd_al_begin_io(),
-	 * lc_try_lock_for_transaction() --, someone may still
-	 * be in the process of changing it. */
-	spin_lock_irq(&device->al_lock);
-	list_for_each_entry(e, &device->act_log->to_be_changed, list) {
-		if (i == AL_UPDATES_PER_TRANSACTION) {
-			i++;
-			break;
-		}
-		buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
-		buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
-		if (e->lc_number != LC_FREE)
-			drbd_bm_mark_for_writeout(device,
-					al_extent_to_bm_page(e->lc_number));
-		i++;
-	}
-	spin_unlock_irq(&device->al_lock);
-	BUG_ON(i > AL_UPDATES_PER_TRANSACTION);
-
-	buffer->n_updates = cpu_to_be16(i);
-	for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) {
-		buffer->update_slot_nr[i] = cpu_to_be16(-1);
-		buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE);
-	}
-
-	buffer->context_size = cpu_to_be16(device->act_log->nr_elements);
-	buffer->context_start_slot_nr = cpu_to_be16(device->al_tr_cycle);
-
-	mx = min_t(int, AL_CONTEXT_PER_TRANSACTION,
-		   device->act_log->nr_elements - device->al_tr_cycle);
-	for (i = 0; i < mx; i++) {
-		unsigned idx = device->al_tr_cycle + i;
-		extent_nr = lc_element_by_index(device->act_log, idx)->lc_number;
-		buffer->context[i] = cpu_to_be32(extent_nr);
-	}
-	for (; i < AL_CONTEXT_PER_TRANSACTION; i++)
-		buffer->context[i] = cpu_to_be32(LC_FREE);
-
-	device->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION;
-	if (device->al_tr_cycle >= device->act_log->nr_elements)
-		device->al_tr_cycle = 0;
-
-	sector = al_tr_number_to_on_disk_sector(device);
-
-	crc = crc32c(0, buffer, 4096);
-	buffer->crc32c = cpu_to_be32(crc);
-
-	if (drbd_bm_write_hinted(device))
-		err = -EIO;
-	else {
-		bool write_al_updates;
-		rcu_read_lock();
-		write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
-		rcu_read_unlock();
-		if (write_al_updates) {
-			if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
-				err = -EIO;
-				drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
-			} else {
-				device->al_tr_number++;
-				device->al_writ_cnt++;
-			}
-		}
-	}
-
-	drbd_md_put_buffer(device);
-	put_ldev(device);
-
-	return err;
-}
-
 static int _try_lc_del(struct drbd_device *device, struct lc_element *al_ext)
 {
 	int rv;
@@ -606,21 +614,24 @@ void drbd_al_shrink(struct drbd_device *device)
 	wake_up(&device->al_wait);
 }
 
-int drbd_initialize_al(struct drbd_device *device, void *buffer)
+int drbd_al_initialize(struct drbd_device *device, void *buffer)
 {
 	struct al_transaction_on_disk *al = buffer;
 	struct drbd_md *md = &device->ldev->md;
-	sector_t al_base = md->md_offset + md->al_offset;
 	int al_size_4k = md->al_stripes * md->al_stripe_size_4k;
 	int i;
 
-	memset(al, 0, 4096);
-	al->magic = cpu_to_be32(DRBD_AL_MAGIC);
-	al->transaction_type = cpu_to_be16(AL_TR_INITIALIZED);
-	al->crc32c = cpu_to_be32(crc32c(0, al, 4096));
+	__al_write_transaction(device, al);
+	/* There may or may not have been a pending transaction. */
+	spin_lock_irq(&device->al_lock);
+	lc_committed(device->act_log);
+	spin_unlock_irq(&device->al_lock);
 
-	for (i = 0; i < al_size_4k; i++) {
-		int err = drbd_md_sync_page_io(device, device->ldev, al_base + i * 8, WRITE);
+	/* The rest of the transactions will have an empty "updates" list, and
+	 * are written out only to provide the context, and to initialize the
+	 * on-disk ring buffer. */
+	for (i = 1; i < al_size_4k; i++) {
+		int err = __al_write_transaction(device, al);
 		if (err)
 			return err;
 	}
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 9462d2752850..92d6fc020a65 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -24,7 +24,7 @@
 
 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
-#include <linux/bitops.h>
+#include <linux/bitmap.h>
 #include <linux/vmalloc.h>
 #include <linux/string.h>
 #include <linux/drbd.h>
@@ -364,12 +364,9 @@ static void bm_free_pages(struct page **pages, unsigned long number)
 	}
 }
 
-static void bm_vk_free(void *ptr, int v)
+static inline void bm_vk_free(void *ptr)
 {
-	if (v)
-		vfree(ptr);
-	else
-		kfree(ptr);
+	kvfree(ptr);
 }
 
 /*
@@ -379,7 +376,7 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 {
 	struct page **old_pages = b->bm_pages;
 	struct page **new_pages, *page;
-	unsigned int i, bytes, vmalloced = 0;
+	unsigned int i, bytes;
 	unsigned long have = b->bm_number_of_pages;
 
 	BUG_ON(have == 0 && old_pages != NULL);
@@ -401,7 +398,6 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 				PAGE_KERNEL);
 		if (!new_pages)
 			return NULL;
-		vmalloced = 1;
 	}
 
 	if (want >= have) {
@@ -411,7 +407,7 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 			page = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
 			if (!page) {
 				bm_free_pages(new_pages + have, i - have);
-				bm_vk_free(new_pages, vmalloced);
+				bm_vk_free(new_pages);
 				return NULL;
 			}
 			/* we want to know which page it is
@@ -427,11 +423,6 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 		*/
 	}
 
-	if (vmalloced)
-		b->bm_flags |= BM_P_VMALLOCED;
-	else
-		b->bm_flags &= ~BM_P_VMALLOCED;
-
 	return new_pages;
 }
 
@@ -469,7 +460,7 @@ void drbd_bm_cleanup(struct drbd_device *device)
 	if (!expect(device->bitmap))
 		return;
 	bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);
-	bm_vk_free(device->bitmap->bm_pages, (BM_P_VMALLOCED & device->bitmap->bm_flags));
+	bm_vk_free(device->bitmap->bm_pages);
 	kfree(device->bitmap);
 	device->bitmap = NULL;
 }
@@ -479,8 +470,14 @@ void drbd_bm_cleanup(struct drbd_device *device)
  * this masks out the remaining bits.
  * Returns the number of bits cleared.
  */
+#ifndef BITS_PER_PAGE
 #define BITS_PER_PAGE		(1UL << (PAGE_SHIFT + 3))
 #define BITS_PER_PAGE_MASK	(BITS_PER_PAGE - 1)
+#else
+# if BITS_PER_PAGE != (1UL << (PAGE_SHIFT + 3))
+#  error "ambiguous BITS_PER_PAGE"
+# endif
+#endif
 #define BITS_PER_LONG_MASK	(BITS_PER_LONG - 1)
 static int bm_clear_surplus(struct drbd_bitmap *b)
 {
@@ -559,21 +556,19 @@ static unsigned long bm_count_bits(struct drbd_bitmap *b)
 	unsigned long *p_addr;
 	unsigned long bits = 0;
 	unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;
-	int idx, i, last_word;
+	int idx, last_word;
 
 	/* all but last page */
 	for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) {
 		p_addr = __bm_map_pidx(b, idx);
-		for (i = 0; i < LWPP; i++)
-			bits += hweight_long(p_addr[i]);
+		bits += bitmap_weight(p_addr, BITS_PER_PAGE);
 		__bm_unmap(p_addr);
 		cond_resched();
 	}
 	/* last (or only) page */
 	last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;
 	p_addr = __bm_map_pidx(b, idx);
-	for (i = 0; i < last_word; i++)
-		bits += hweight_long(p_addr[i]);
+	bits += bitmap_weight(p_addr, last_word * BITS_PER_LONG);
 	p_addr[last_word] &= cpu_to_lel(mask);
 	bits += hweight_long(p_addr[last_word]);
 	/* 32bit arch, may have an unused padding long */
@@ -639,7 +634,6 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
 	unsigned long want, have, onpages; /* number of pages */
 	struct page **npages, **opages = NULL;
 	int err = 0, growing;
-	int opages_vmalloced;
 
 	if (!expect(b))
 		return -ENOMEM;
@@ -652,8 +646,6 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
 	if (capacity == b->bm_dev_capacity)
 		goto out;
 
-	opages_vmalloced = (BM_P_VMALLOCED & b->bm_flags);
-
 	if (capacity == 0) {
 		spin_lock_irq(&b->bm_lock);
 		opages = b->bm_pages;
@@ -667,7 +659,7 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
 		b->bm_dev_capacity = 0;
 		spin_unlock_irq(&b->bm_lock);
 		bm_free_pages(opages, onpages);
-		bm_vk_free(opages, opages_vmalloced);
+		bm_vk_free(opages);
 		goto out;
 	}
 	bits  = BM_SECT_TO_BIT(ALIGN(capacity, BM_SECT_PER_BIT));
@@ -740,7 +732,7 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
 
 	spin_unlock_irq(&b->bm_lock);
 	if (opages != npages)
-		bm_vk_free(opages, opages_vmalloced);
+		bm_vk_free(opages);
 	if (!growing)
 		b->bm_set = bm_count_bits(b);
 	drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want);
@@ -1419,6 +1411,9 @@ static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,
 	int bits;
 	int changed = 0;
 	unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]);
+
+	/* I think it is more cache line friendly to hweight_long then set to ~0UL,
+	 * than to first bitmap_weight() all words, then bitmap_fill() all words */
 	for (i = first_word; i < last_word; i++) {
 		bits = hweight_long(paddr[i]);
 		paddr[i] = ~0UL;
@@ -1628,8 +1623,7 @@ int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr)
 		int n = e-s;
 		p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s));
 		bm = p_addr + MLPP(s);
-		while (n--)
-			count += hweight_long(*bm++);
+		count += bitmap_weight(bm, n * BITS_PER_LONG);
 		bm_unmap(p_addr);
 	} else {
 		drbd_err(device, "start offset (%d) too large in drbd_bm_e_weight\n", s);
diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c
index 6b88a35fb048..4de95bbff486 100644
--- a/drivers/block/drbd/drbd_debugfs.c
+++ b/drivers/block/drbd/drbd_debugfs.c
@@ -434,12 +434,12 @@ static int drbd_single_open(struct file *file, int (*show)(struct seq_file *, vo
 	if (!parent || d_really_is_negative(parent))
 		goto out;
 	/* serialize with d_delete() */
-	mutex_lock(&d_inode(parent)->i_mutex);
+	inode_lock(d_inode(parent));
 	/* Make sure the object is still alive */
 	if (simple_positive(file->f_path.dentry)
 	&& kref_get_unless_zero(kref))
 		ret = 0;
-	mutex_unlock(&d_inode(parent)->i_mutex);
+	inode_unlock(d_inode(parent));
 	if (!ret) {
 		ret = single_open(file, show, data);
 		if (ret)
@@ -771,6 +771,13 @@ static int device_data_gen_id_show(struct seq_file *m, void *ignored)
 	return 0;
 }
 
+static int device_ed_gen_id_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+	seq_printf(m, "0x%016llX\n", (unsigned long long)device->ed_uuid);
+	return 0;
+}
+
 #define drbd_debugfs_device_attr(name)						\
 static int device_ ## name ## _open(struct inode *inode, struct file *file)	\
 {										\
@@ -796,6 +803,7 @@ drbd_debugfs_device_attr(oldest_requests)
 drbd_debugfs_device_attr(act_log_extents)
 drbd_debugfs_device_attr(resync_extents)
 drbd_debugfs_device_attr(data_gen_id)
+drbd_debugfs_device_attr(ed_gen_id)
 
 void drbd_debugfs_device_add(struct drbd_device *device)
 {
@@ -839,6 +847,7 @@ void drbd_debugfs_device_add(struct drbd_device *device)
 	DCF(act_log_extents);
 	DCF(resync_extents);
 	DCF(data_gen_id);
+	DCF(ed_gen_id);
 #undef DCF
 	return;
 
@@ -854,6 +863,7 @@ void drbd_debugfs_device_cleanup(struct drbd_device *device)
 	drbd_debugfs_remove(&device->debugfs_vol_act_log_extents);
 	drbd_debugfs_remove(&device->debugfs_vol_resync_extents);
 	drbd_debugfs_remove(&device->debugfs_vol_data_gen_id);
+	drbd_debugfs_remove(&device->debugfs_vol_ed_gen_id);
 	drbd_debugfs_remove(&device->debugfs_vol);
 }
 
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index e66d453a5f2b..34bc84efc29e 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -77,13 +77,6 @@ extern int fault_devs;
 extern char usermode_helper[];
 
 
-/* I don't remember why XCPU ...
- * This is used to wake the asender,
- * and to interrupt sending the sending task
- * on disconnect.
- */
-#define DRBD_SIG SIGXCPU
-
 /* This is used to stop/restart our threads.
  * Cannot use SIGTERM nor SIGKILL, since these
  * are sent out by init on runlevel changes
@@ -292,6 +285,9 @@ struct drbd_device_work {
 
 extern int drbd_wait_misc(struct drbd_device *, struct drbd_interval *);
 
+extern void lock_all_resources(void);
+extern void unlock_all_resources(void);
+
 struct drbd_request {
 	struct drbd_work w;
 	struct drbd_device *device;
@@ -504,7 +500,6 @@ enum {
 
 	MD_NO_FUA,		/* Users wants us to not use FUA/FLUSH on meta data dev */
 
-	SUSPEND_IO,		/* suspend application io */
 	BITMAP_IO,		/* suspend application io;
 				   once no more io in flight, start bitmap io */
 	BITMAP_IO_QUEUED,       /* Started bitmap IO */
@@ -541,9 +536,6 @@ struct drbd_bitmap; /* opaque for drbd_device */
 /* definition of bits in bm_flags to be used in drbd_bm_lock
  * and drbd_bitmap_io and friends. */
 enum bm_flag {
-	/* do we need to kfree, or vfree bm_pages? */
-	BM_P_VMALLOCED = 0x10000, /* internal use only, will be masked out */
-
 	/* currently locked for bulk operation */
 	BM_LOCKED_MASK = 0xf,
 
@@ -632,12 +624,6 @@ struct bm_io_work {
 	void (*done)(struct drbd_device *device, int rv);
 };
 
-enum write_ordering_e {
-	WO_none,
-	WO_drain_io,
-	WO_bdev_flush,
-};
-
 struct fifo_buffer {
 	unsigned int head_index;
 	unsigned int size;
@@ -650,8 +636,7 @@ extern struct fifo_buffer *fifo_alloc(int fifo_size);
 enum {
 	NET_CONGESTED,		/* The data socket is congested */
 	RESOLVE_CONFLICTS,	/* Set on one node, cleared on the peer! */
-	SEND_PING,		/* whether asender should send a ping asap */
-	SIGNAL_ASENDER,		/* whether asender wants to be interrupted */
+	SEND_PING,
 	GOT_PING_ACK,		/* set when we receive a ping_ack packet, ping_wait gets woken */
 	CONN_WD_ST_CHG_REQ,	/* A cluster wide state change on the connection is active */
 	CONN_WD_ST_CHG_OKAY,
@@ -670,6 +655,8 @@ enum {
 	DEVICE_WORK_PENDING,	/* tell worker that some device has pending work */
 };
 
+enum which_state { NOW, OLD = NOW, NEW };
+
 struct drbd_resource {
 	char *name;
 #ifdef CONFIG_DEBUG_FS
@@ -755,7 +742,8 @@ struct drbd_connection {
 	unsigned long last_reconnect_jif;
 	struct drbd_thread receiver;
 	struct drbd_thread worker;
-	struct drbd_thread asender;
+	struct drbd_thread ack_receiver;
+	struct workqueue_struct *ack_sender;
 
 	/* cached pointers,
 	 * so we can look up the oldest pending requests more quickly.
@@ -774,6 +762,8 @@ struct drbd_connection {
 	struct drbd_thread_timing_details r_timing_details[DRBD_THREAD_DETAILS_HIST];
 
 	struct {
+		unsigned long last_sent_barrier_jif;
+
 		/* whether this sender thread
 		 * has processed a single write yet. */
 		bool seen_any_write_yet;
@@ -788,6 +778,17 @@ struct drbd_connection {
 	} send;
 };
 
+static inline bool has_net_conf(struct drbd_connection *connection)
+{
+	bool has_net_conf;
+
+	rcu_read_lock();
+	has_net_conf = rcu_dereference(connection->net_conf);
+	rcu_read_unlock();
+
+	return has_net_conf;
+}
+
 void __update_timing_details(
 		struct drbd_thread_timing_details *tdp,
 		unsigned int *cb_nr,
@@ -811,6 +812,7 @@ struct drbd_peer_device {
 	struct list_head peer_devices;
 	struct drbd_device *device;
 	struct drbd_connection *connection;
+	struct work_struct send_acks_work;
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *debugfs_peer_dev;
 #endif
@@ -829,6 +831,7 @@ struct drbd_device {
 	struct dentry *debugfs_vol_act_log_extents;
 	struct dentry *debugfs_vol_resync_extents;
 	struct dentry *debugfs_vol_data_gen_id;
+	struct dentry *debugfs_vol_ed_gen_id;
 #endif
 
 	unsigned int vnr;	/* volume number within the connection */
@@ -873,6 +876,7 @@ struct drbd_device {
 	atomic_t rs_pending_cnt; /* RS request/data packets on the wire */
 	atomic_t unacked_cnt;	 /* Need to send replies for */
 	atomic_t local_cnt;	 /* Waiting for local completion */
+	atomic_t suspend_cnt;
 
 	/* Interval tree of pending local requests */
 	struct rb_root read_requests;
@@ -1020,6 +1024,12 @@ static inline struct drbd_peer_device *first_peer_device(struct drbd_device *dev
 	return list_first_entry_or_null(&device->peer_devices, struct drbd_peer_device, peer_devices);
 }
 
+static inline struct drbd_peer_device *
+conn_peer_device(struct drbd_connection *connection, int volume_number)
+{
+	return idr_find(&connection->peer_devices, volume_number);
+}
+
 #define for_each_resource(resource, _resources) \
 	list_for_each_entry(resource, _resources, resources)
 
@@ -1113,7 +1123,7 @@ extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int
 extern int drbd_send_bitmap(struct drbd_device *device);
 extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode);
 extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode);
-extern void drbd_free_ldev(struct drbd_backing_dev *ldev);
+extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev);
 extern void drbd_device_cleanup(struct drbd_device *device);
 void drbd_print_uuids(struct drbd_device *device, const char *text);
 
@@ -1424,7 +1434,7 @@ extern struct bio_set *drbd_md_io_bio_set;
 /* to allocate from that set */
 extern struct bio *bio_alloc_drbd(gfp_t gfp_mask);
 
-extern rwlock_t global_state_lock;
+extern struct mutex resources_mutex;
 
 extern int conn_lowest_minor(struct drbd_connection *connection);
 extern enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor);
@@ -1454,6 +1464,9 @@ extern int is_valid_ar_handle(struct drbd_request *, sector_t);
 
 
 /* drbd_nl.c */
+
+extern struct mutex notification_mutex;
+
 extern void drbd_suspend_io(struct drbd_device *device);
 extern void drbd_resume_io(struct drbd_device *device);
 extern char *ppsize(char *buf, unsigned long long size);
@@ -1536,7 +1549,9 @@ extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req);
 
 /* drbd_receiver.c */
 extern int drbd_receiver(struct drbd_thread *thi);
-extern int drbd_asender(struct drbd_thread *thi);
+extern int drbd_ack_receiver(struct drbd_thread *thi);
+extern void drbd_send_ping_wf(struct work_struct *ws);
+extern void drbd_send_acks_wf(struct work_struct *ws);
 extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device);
 extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,
 		bool throttle_if_app_is_waiting);
@@ -1649,7 +1664,7 @@ extern int __drbd_change_sync(struct drbd_device *device, sector_t sector, int s
 #define drbd_rs_failed_io(device, sector, size) \
 	__drbd_change_sync(device, sector, size, RECORD_RS_FAILED)
 extern void drbd_al_shrink(struct drbd_device *device);
-extern int drbd_initialize_al(struct drbd_device *, void *);
+extern int drbd_al_initialize(struct drbd_device *, void *);
 
 /* drbd_nl.c */
 /* state info broadcast */
@@ -1668,6 +1683,29 @@ struct sib_info {
 };
 void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib);
 
+extern void notify_resource_state(struct sk_buff *,
+				  unsigned int,
+				  struct drbd_resource *,
+				  struct resource_info *,
+				  enum drbd_notification_type);
+extern void notify_device_state(struct sk_buff *,
+				unsigned int,
+				struct drbd_device *,
+				struct device_info *,
+				enum drbd_notification_type);
+extern void notify_connection_state(struct sk_buff *,
+				    unsigned int,
+				    struct drbd_connection *,
+				    struct connection_info *,
+				    enum drbd_notification_type);
+extern void notify_peer_device_state(struct sk_buff *,
+				     unsigned int,
+				     struct drbd_peer_device *,
+				     struct peer_device_info *,
+				     enum drbd_notification_type);
+extern void notify_helper(enum drbd_notification_type, struct drbd_device *,
+			  struct drbd_connection *, const char *, int);
+
 /*
  * inline helper functions
  *************************/
@@ -1694,19 +1732,6 @@ static inline int drbd_peer_req_has_active_page(struct drbd_peer_request *peer_r
 	return 0;
 }
 
-static inline enum drbd_state_rv
-_drbd_set_state(struct drbd_device *device, union drbd_state ns,
-		enum chg_state_flags flags, struct completion *done)
-{
-	enum drbd_state_rv rv;
-
-	read_lock(&global_state_lock);
-	rv = __drbd_set_state(device, ns, flags, done);
-	read_unlock(&global_state_lock);
-
-	return rv;
-}
-
 static inline union drbd_state drbd_read_state(struct drbd_device *device)
 {
 	struct drbd_resource *resource = device->resource;
@@ -1937,16 +1962,21 @@ drbd_device_post_work(struct drbd_device *device, int work_bit)
 
 extern void drbd_flush_workqueue(struct drbd_work_queue *work_queue);
 
-static inline void wake_asender(struct drbd_connection *connection)
+/* To get the ack_receiver out of the blocking network stack,
+ * so it can change its sk_rcvtimeo from idle- to ping-timeout,
+ * and send a ping, we need to send a signal.
+ * Which signal we send is irrelevant. */
+static inline void wake_ack_receiver(struct drbd_connection *connection)
 {
-	if (test_bit(SIGNAL_ASENDER, &connection->flags))
-		force_sig(DRBD_SIG, connection->asender.task);
+	struct task_struct *task = connection->ack_receiver.task;
+	if (task && get_t_state(&connection->ack_receiver) == RUNNING)
+		force_sig(SIGXCPU, task);
 }
 
 static inline void request_ping(struct drbd_connection *connection)
 {
 	set_bit(SEND_PING, &connection->flags);
-	wake_asender(connection);
+	wake_ack_receiver(connection);
 }
 
 extern void *conn_prepare_command(struct drbd_connection *, struct drbd_socket *);
@@ -2230,7 +2260,7 @@ static inline bool may_inc_ap_bio(struct drbd_device *device)
 
 	if (drbd_suspended(device))
 		return false;
-	if (test_bit(SUSPEND_IO, &device->flags))
+	if (atomic_read(&device->suspend_cnt))
 		return false;
 
 	/* to avoid potential deadlock or bitmap corruption,
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 74d97f4bac34..5b43dfb79819 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -117,6 +117,7 @@ module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0
  */
 struct idr drbd_devices;
 struct list_head drbd_resources;
+struct mutex resources_mutex;
 
 struct kmem_cache *drbd_request_cache;
 struct kmem_cache *drbd_ee_cache;	/* peer requests */
@@ -1435,8 +1436,8 @@ static int we_should_drop_the_connection(struct drbd_connection *connection, str
 	/* long elapsed = (long)(jiffies - device->last_received); */
 
 	drop_it =   connection->meta.socket == sock
-		|| !connection->asender.task
-		|| get_t_state(&connection->asender) != RUNNING
+		|| !connection->ack_receiver.task
+		|| get_t_state(&connection->ack_receiver) != RUNNING
 		|| connection->cstate < C_WF_REPORT_PARAMS;
 
 	if (drop_it)
@@ -1793,15 +1794,6 @@ int drbd_send(struct drbd_connection *connection, struct socket *sock,
 		drbd_update_congested(connection);
 	}
 	do {
-		/* STRANGE
-		 * tcp_sendmsg does _not_ use its size parameter at all ?
-		 *
-		 * -EAGAIN on timeout, -EINTR on signal.
-		 */
-/* THINK
- * do we need to block DRBD_SIG if sock == &meta.socket ??
- * otherwise wake_asender() might interrupt some send_*Ack !
- */
 		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
 		if (rv == -EAGAIN) {
 			if (we_should_drop_the_connection(connection, sock))
@@ -2000,7 +1992,7 @@ void drbd_device_cleanup(struct drbd_device *device)
 		drbd_bm_cleanup(device);
 	}
 
-	drbd_free_ldev(device->ldev);
+	drbd_backing_dev_free(device, device->ldev);
 	device->ldev = NULL;
 
 	clear_bit(AL_SUSPENDED, &device->flags);
@@ -2179,7 +2171,7 @@ void drbd_destroy_device(struct kref *kref)
 	if (device->this_bdev)
 		bdput(device->this_bdev);
 
-	drbd_free_ldev(device->ldev);
+	drbd_backing_dev_free(device, device->ldev);
 	device->ldev = NULL;
 
 	drbd_release_all_peer_reqs(device);
@@ -2563,7 +2555,7 @@ int set_resource_options(struct drbd_resource *resource, struct res_opts *res_op
 		cpumask_copy(resource->cpu_mask, new_cpu_mask);
 		for_each_connection_rcu(connection, resource) {
 			connection->receiver.reset_cpu_mask = 1;
-			connection->asender.reset_cpu_mask = 1;
+			connection->ack_receiver.reset_cpu_mask = 1;
 			connection->worker.reset_cpu_mask = 1;
 		}
 	}
@@ -2590,7 +2582,7 @@ struct drbd_resource *drbd_create_resource(const char *name)
 	kref_init(&resource->kref);
 	idr_init(&resource->devices);
 	INIT_LIST_HEAD(&resource->connections);
-	resource->write_ordering = WO_bdev_flush;
+	resource->write_ordering = WO_BDEV_FLUSH;
 	list_add_tail_rcu(&resource->resources, &drbd_resources);
 	mutex_init(&resource->conf_update);
 	mutex_init(&resource->adm_mutex);
@@ -2652,8 +2644,8 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
 	connection->receiver.connection = connection;
 	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
 	connection->worker.connection = connection;
-	drbd_thread_init(resource, &connection->asender, drbd_asender, "asender");
-	connection->asender.connection = connection;
+	drbd_thread_init(resource, &connection->ack_receiver, drbd_ack_receiver, "ack_recv");
+	connection->ack_receiver.connection = connection;
 
 	kref_init(&connection->kref);
 
@@ -2702,8 +2694,8 @@ static int init_submitter(struct drbd_device *device)
 {
 	/* opencoded create_singlethread_workqueue(),
 	 * to be able to say "drbd%d", ..., minor */
-	device->submit.wq = alloc_workqueue("drbd%u_submit",
-			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
+	device->submit.wq =
+		alloc_ordered_workqueue("drbd%u_submit", WQ_MEM_RECLAIM, device->minor);
 	if (!device->submit.wq)
 		return -ENOMEM;
 
@@ -2820,6 +2812,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
 			goto out_idr_remove_from_resource;
 		}
 		kref_get(&connection->kref);
+		INIT_WORK(&peer_device->send_acks_work, drbd_send_acks_wf);
 	}
 
 	if (init_submitter(device)) {
@@ -2923,7 +2916,7 @@ static int __init drbd_init(void)
 	drbd_proc = NULL; /* play safe for drbd_cleanup */
 	idr_init(&drbd_devices);
 
-	rwlock_init(&global_state_lock);
+	mutex_init(&resources_mutex);
 	INIT_LIST_HEAD(&drbd_resources);
 
 	err = drbd_genl_register();
@@ -2971,18 +2964,6 @@ fail:
 	return err;
 }
 
-void drbd_free_ldev(struct drbd_backing_dev *ldev)
-{
-	if (ldev == NULL)
-		return;
-
-	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
-	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
-
-	kfree(ldev->disk_conf);
-	kfree(ldev);
-}
-
 static void drbd_free_one_sock(struct drbd_socket *ds)
 {
 	struct socket *s;
@@ -3277,6 +3258,10 @@ int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
 	 * and read it. */
 	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
 	bdev->md.md_offset = drbd_md_ss(bdev);
+	/* Even for (flexible or indexed) external meta data,
+	 * initially restrict us to the 4k superblock for now.
+	 * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */
+	bdev->md.md_size_sect = 8;
 
 	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
 		/* NOTE: can't do normal error processing here as this is
@@ -3578,7 +3563,9 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
 
 	spin_lock_irq(&device->resource->req_lock);
 	set_bit(BITMAP_IO, &device->flags);
-	if (atomic_read(&device->ap_bio_cnt) == 0) {
+	/* don't wait for pending application IO if the caller indicates that
+	 * application IO does not conflict anyways. */
+	if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
 		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
 			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
 					&device->bm_io_work.w);
@@ -3746,6 +3733,27 @@ int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
 	return 0;
 }
 
+void lock_all_resources(void)
+{
+	struct drbd_resource *resource;
+	int __maybe_unused i = 0;
+
+	mutex_lock(&resources_mutex);
+	local_irq_disable();
+	for_each_resource(resource, &drbd_resources)
+		spin_lock_nested(&resource->req_lock, i++);
+}
+
+void unlock_all_resources(void)
+{
+	struct drbd_resource *resource;
+
+	for_each_resource(resource, &drbd_resources)
+		spin_unlock(&resource->req_lock);
+	local_irq_enable();
+	mutex_unlock(&resources_mutex);
+}
+
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 /* Fault insertion support including random number generator shamelessly
  * stolen from kernel/rcutorture.c */
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index e80cbefbc2b5..c055c5e12f24 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -36,6 +36,7 @@
 #include "drbd_int.h"
 #include "drbd_protocol.h"
 #include "drbd_req.h"
+#include "drbd_state_change.h"
 #include <asm/unaligned.h>
 #include <linux/drbd_limits.h>
 #include <linux/kthread.h>
@@ -75,11 +76,24 @@ int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info);
 int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info);
 /* .dumpit */
 int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_adm_dump_devices_done(struct netlink_callback *cb);
+int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_adm_dump_connections_done(struct netlink_callback *cb);
+int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb);
+int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb);
 
 #include <linux/drbd_genl_api.h>
 #include "drbd_nla.h"
 #include <linux/genl_magic_func.h>
 
+static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
+static atomic_t notify_genl_seq = ATOMIC_INIT(2); /* two. */
+
+DEFINE_MUTEX(notification_mutex);
+
 /* used blkdev_get_by_path, to claim our meta data device(s) */
 static char *drbd_m_holder = "Hands off! this is DRBD's meta data device.";
 
@@ -349,6 +363,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd)
 	sib.sib_reason = SIB_HELPER_PRE;
 	sib.helper_name = cmd;
 	drbd_bcast_event(device, &sib);
+	notify_helper(NOTIFY_CALL, device, connection, cmd, 0);
 	ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC);
 	if (ret)
 		drbd_warn(device, "helper command: %s %s %s exit code %u (0x%x)\n",
@@ -361,6 +376,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd)
 	sib.sib_reason = SIB_HELPER_POST;
 	sib.helper_exit_code = ret;
 	drbd_bcast_event(device, &sib);
+	notify_helper(NOTIFY_RESPONSE, device, connection, cmd, ret);
 
 	if (current == connection->worker.task)
 		clear_bit(CALLBACK_PENDING, &connection->flags);
@@ -388,6 +404,7 @@ static int conn_khelper(struct drbd_connection *connection, char *cmd)
 
 	drbd_info(connection, "helper command: %s %s %s\n", usermode_helper, cmd, resource_name);
 	/* TODO: conn_bcast_event() ?? */
+	notify_helper(NOTIFY_CALL, NULL, connection, cmd, 0);
 
 	ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC);
 	if (ret)
@@ -399,6 +416,7 @@ static int conn_khelper(struct drbd_connection *connection, char *cmd)
 			  usermode_helper, cmd, resource_name,
 			  (ret >> 8) & 0xff, ret);
 	/* TODO: conn_bcast_event() ?? */
+	notify_helper(NOTIFY_RESPONSE, NULL, connection, cmd, ret);
 
 	if (ret < 0) /* Ignore any ERRNOs we got. */
 		ret = 0;
@@ -847,9 +865,11 @@ char *ppsize(char *buf, unsigned long long size)
  * and can be long lived.
  * This changes an device->flag, is triggered by drbd internals,
  * and should be short-lived. */
+/* It needs to be a counter, since multiple threads might
+   independently suspend and resume IO. */
 void drbd_suspend_io(struct drbd_device *device)
 {
-	set_bit(SUSPEND_IO, &device->flags);
+	atomic_inc(&device->suspend_cnt);
 	if (drbd_suspended(device))
 		return;
 	wait_event(device->misc_wait, !atomic_read(&device->ap_bio_cnt));
@@ -857,8 +877,8 @@ void drbd_suspend_io(struct drbd_device *device)
 
 void drbd_resume_io(struct drbd_device *device)
 {
-	clear_bit(SUSPEND_IO, &device->flags);
-	wake_up(&device->misc_wait);
+	if (atomic_dec_and_test(&device->suspend_cnt))
+		wake_up(&device->misc_wait);
 }
 
 /**
@@ -871,27 +891,32 @@ void drbd_resume_io(struct drbd_device *device)
 enum determine_dev_size
 drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct resize_parms *rs) __must_hold(local)
 {
-	sector_t prev_first_sect, prev_size; /* previous meta location */
-	sector_t la_size_sect, u_size;
+	struct md_offsets_and_sizes {
+		u64 last_agreed_sect;
+		u64 md_offset;
+		s32 al_offset;
+		s32 bm_offset;
+		u32 md_size_sect;
+
+		u32 al_stripes;
+		u32 al_stripe_size_4k;
+	} prev;
+	sector_t u_size, size;
 	struct drbd_md *md = &device->ldev->md;
-	u32 prev_al_stripe_size_4k;
-	u32 prev_al_stripes;
-	sector_t size;
 	char ppb[10];
 	void *buffer;
 
 	int md_moved, la_size_changed;
 	enum determine_dev_size rv = DS_UNCHANGED;
 
-	/* race:
-	 * application request passes inc_ap_bio,
-	 * but then cannot get an AL-reference.
-	 * this function later may wait on ap_bio_cnt == 0. -> deadlock.
+	/* We may change the on-disk offsets of our meta data below.  Lock out
+	 * anything that may cause meta data IO, to avoid acting on incomplete
+	 * layout changes or scribbling over meta data that is in the process
+	 * of being moved.
 	 *
-	 * to avoid that:
-	 * Suspend IO right here.
-	 * still lock the act_log to not trigger ASSERTs there.
-	 */
+	 * Move is not exactly correct, btw, currently we have all our meta
+	 * data in core memory, to "move" it we just write it all out, there
+	 * are no reads. */
 	drbd_suspend_io(device);
 	buffer = drbd_md_get_buffer(device, __func__); /* Lock meta-data IO */
 	if (!buffer) {
@@ -899,19 +924,17 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		return DS_ERROR;
 	}
 
-	/* no wait necessary anymore, actually we could assert that */
-	wait_event(device->al_wait, lc_try_lock(device->act_log));
-
-	prev_first_sect = drbd_md_first_sector(device->ldev);
-	prev_size = device->ldev->md.md_size_sect;
-	la_size_sect = device->ldev->md.la_size_sect;
+	/* remember current offset and sizes */
+	prev.last_agreed_sect = md->la_size_sect;
+	prev.md_offset = md->md_offset;
+	prev.al_offset = md->al_offset;
+	prev.bm_offset = md->bm_offset;
+	prev.md_size_sect = md->md_size_sect;
+	prev.al_stripes = md->al_stripes;
+	prev.al_stripe_size_4k = md->al_stripe_size_4k;
 
 	if (rs) {
 		/* rs is non NULL if we should change the AL layout only */
-
-		prev_al_stripes = md->al_stripes;
-		prev_al_stripe_size_4k = md->al_stripe_size_4k;
-
 		md->al_stripes = rs->al_stripes;
 		md->al_stripe_size_4k = rs->al_stripe_size / 4;
 		md->al_size_4k = (u64)rs->al_stripes * rs->al_stripe_size / 4;
@@ -924,7 +947,7 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 	rcu_read_unlock();
 	size = drbd_new_dev_size(device, device->ldev, u_size, flags & DDSF_FORCED);
 
-	if (size < la_size_sect) {
+	if (size < prev.last_agreed_sect) {
 		if (rs && u_size == 0) {
 			/* Remove "rs &&" later. This check should always be active, but
 			   right now the receiver expects the permissive behavior */
@@ -945,30 +968,29 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		err = drbd_bm_resize(device, size, !(flags & DDSF_NO_RESYNC));
 		if (unlikely(err)) {
 			/* currently there is only one error: ENOMEM! */
-			size = drbd_bm_capacity(device)>>1;
+			size = drbd_bm_capacity(device);
 			if (size == 0) {
 				drbd_err(device, "OUT OF MEMORY! "
 				    "Could not allocate bitmap!\n");
 			} else {
 				drbd_err(device, "BM resizing failed. "
-				    "Leaving size unchanged at size = %lu KB\n",
-				    (unsigned long)size);
+				    "Leaving size unchanged\n");
 			}
 			rv = DS_ERROR;
 		}
 		/* racy, see comments above. */
 		drbd_set_my_capacity(device, size);
-		device->ldev->md.la_size_sect = size;
+		md->la_size_sect = size;
 		drbd_info(device, "size = %s (%llu KB)\n", ppsize(ppb, size>>1),
 		     (unsigned long long)size>>1);
 	}
 	if (rv <= DS_ERROR)
 		goto err_out;
 
-	la_size_changed = (la_size_sect != device->ldev->md.la_size_sect);
+	la_size_changed = (prev.last_agreed_sect != md->la_size_sect);
 
-	md_moved = prev_first_sect != drbd_md_first_sector(device->ldev)
-		|| prev_size	   != device->ldev->md.md_size_sect;
+	md_moved = prev.md_offset    != md->md_offset
+		|| prev.md_size_sect != md->md_size_sect;
 
 	if (la_size_changed || md_moved || rs) {
 		u32 prev_flags;
@@ -977,20 +999,29 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		 * Clear the timer, to avoid scary "timer expired!" messages,
 		 * "Superblock" is written out at least twice below, anyways. */
 		del_timer(&device->md_sync_timer);
-		drbd_al_shrink(device); /* All extents inactive. */
 
+		/* We won't change the "al-extents" setting, we just may need
+		 * to move the on-disk location of the activity log ringbuffer.
+		 * Lock for transaction is good enough, it may well be "dirty"
+		 * or even "starving". */
+		wait_event(device->al_wait, lc_try_lock_for_transaction(device->act_log));
+
+		/* mark current on-disk bitmap and activity log as unreliable */
 		prev_flags = md->flags;
-		md->flags &= ~MDF_PRIMARY_IND;
+		md->flags |= MDF_FULL_SYNC | MDF_AL_DISABLED;
 		drbd_md_write(device, buffer);
 
+		drbd_al_initialize(device, buffer);
+
 		drbd_info(device, "Writing the whole bitmap, %s\n",
 			 la_size_changed && md_moved ? "size changed and md moved" :
 			 la_size_changed ? "size changed" : "md moved");
 		/* next line implicitly does drbd_suspend_io()+drbd_resume_io() */
 		drbd_bitmap_io(device, md_moved ? &drbd_bm_write_all : &drbd_bm_write,
 			       "size changed", BM_LOCKED_MASK);
-		drbd_initialize_al(device, buffer);
 
+		/* on-disk bitmap and activity log is authoritative again
+		 * (unless there was an IO error meanwhile...) */
 		md->flags = prev_flags;
 		drbd_md_write(device, buffer);
 
@@ -999,20 +1030,22 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 				  md->al_stripes, md->al_stripe_size_4k * 4);
 	}
 
-	if (size > la_size_sect)
-		rv = la_size_sect ? DS_GREW : DS_GREW_FROM_ZERO;
-	if (size < la_size_sect)
+	if (size > prev.last_agreed_sect)
+		rv = prev.last_agreed_sect ? DS_GREW : DS_GREW_FROM_ZERO;
+	if (size < prev.last_agreed_sect)
 		rv = DS_SHRUNK;
 
 	if (0) {
 	err_out:
-		if (rs) {
-			md->al_stripes = prev_al_stripes;
-			md->al_stripe_size_4k = prev_al_stripe_size_4k;
-			md->al_size_4k = (u64)prev_al_stripes * prev_al_stripe_size_4k;
-
-			drbd_md_set_sector_offsets(device, device->ldev);
-		}
+		/* restore previous offset and sizes */
+		md->la_size_sect = prev.last_agreed_sect;
+		md->md_offset = prev.md_offset;
+		md->al_offset = prev.al_offset;
+		md->bm_offset = prev.bm_offset;
+		md->md_size_sect = prev.md_size_sect;
+		md->al_stripes = prev.al_stripes;
+		md->al_stripe_size_4k = prev.al_stripe_size_4k;
+		md->al_size_4k = (u64)prev.al_stripes * prev.al_stripe_size_4k;
 	}
 	lc_unlock(device->act_log);
 	wake_up(&device->al_wait);
@@ -1115,8 +1148,7 @@ static int drbd_check_al_size(struct drbd_device *device, struct disk_conf *dc)
 		lc_destroy(n);
 		return -EBUSY;
 	} else {
-		if (t)
-			lc_destroy(t);
+		lc_destroy(t);
 	}
 	drbd_md_mark_dirty(device); /* we changed device->act_log->nr_elemens */
 	return 0;
@@ -1151,21 +1183,20 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
 	if (b) {
 		struct drbd_connection *connection = first_peer_device(device)->connection;
 
+		blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS);
+
 		if (blk_queue_discard(b) &&
 		    (connection->cstate < C_CONNECTED || connection->agreed_features & FF_TRIM)) {
-			/* For now, don't allow more than one activity log extent worth of data
-			 * to be discarded in one go. We may need to rework drbd_al_begin_io()
-			 * to allow for even larger discard ranges */
-			blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS);
-
+			/* We don't care, stacking below should fix it for the local device.
+			 * Whether or not it is a suitable granularity on the remote device
+			 * is not our problem, really. If you care, you need to
+			 * use devices with similar topology on all peers. */
+			q->limits.discard_granularity = 512;
 			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
-			/* REALLY? Is stacking secdiscard "legal"? */
-			if (blk_queue_secdiscard(b))
-				queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q);
 		} else {
 			blk_queue_max_discard_sectors(q, 0);
 			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
-			queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q);
+			q->limits.discard_granularity = 0;
 		}
 
 		blk_queue_stack_limits(q, b);
@@ -1177,6 +1208,12 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
 			q->backing_dev_info.ra_pages = b->backing_dev_info.ra_pages;
 		}
 	}
+	/* To avoid confusion, if this queue does not support discard, clear
+	 * max_discard_sectors, which is what lsblk -D reports to the user.  */
+	if (!blk_queue_discard(q)) {
+		blk_queue_max_discard_sectors(q, 0);
+		q->limits.discard_granularity = 0;
+	}
 }
 
 void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev)
@@ -1241,8 +1278,8 @@ static void conn_reconfig_done(struct drbd_connection *connection)
 		connection->cstate == C_STANDALONE;
 	spin_unlock_irq(&connection->resource->req_lock);
 	if (stop_threads) {
-		/* asender is implicitly stopped by receiver
-		 * in conn_disconnect() */
+		/* ack_receiver thread and ack_sender workqueue are implicitly
+		 * stopped by receiver in conn_disconnect() */
 		drbd_thread_stop(&connection->receiver);
 		drbd_thread_stop(&connection->worker);
 	}
@@ -1389,13 +1426,13 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		goto fail_unlock;
 	}
 
-	write_lock_irq(&global_state_lock);
+	lock_all_resources();
 	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
 	if (retcode == NO_ERROR) {
 		rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
 		drbd_resync_after_changed(device);
 	}
-	write_unlock_irq(&global_state_lock);
+	unlock_all_resources();
 
 	if (retcode != NO_ERROR)
 		goto fail_unlock;
@@ -1418,7 +1455,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		set_bit(MD_NO_FUA, &device->flags);
 
 	if (write_ordering_changed(old_disk_conf, new_disk_conf))
-		drbd_bump_write_ordering(device->resource, NULL, WO_bdev_flush);
+		drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH);
 
 	drbd_md_sync(device);
 
@@ -1449,6 +1486,88 @@ success:
 	return 0;
 }
 
+static struct block_device *open_backing_dev(struct drbd_device *device,
+		const char *bdev_path, void *claim_ptr, bool do_bd_link)
+{
+	struct block_device *bdev;
+	int err = 0;
+
+	bdev = blkdev_get_by_path(bdev_path,
+				  FMODE_READ | FMODE_WRITE | FMODE_EXCL, claim_ptr);
+	if (IS_ERR(bdev)) {
+		drbd_err(device, "open(\"%s\") failed with %ld\n",
+				bdev_path, PTR_ERR(bdev));
+		return bdev;
+	}
+
+	if (!do_bd_link)
+		return bdev;
+
+	err = bd_link_disk_holder(bdev, device->vdisk);
+	if (err) {
+		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
+		drbd_err(device, "bd_link_disk_holder(\"%s\", ...) failed with %d\n",
+				bdev_path, err);
+		bdev = ERR_PTR(err);
+	}
+	return bdev;
+}
+
+static int open_backing_devices(struct drbd_device *device,
+		struct disk_conf *new_disk_conf,
+		struct drbd_backing_dev *nbc)
+{
+	struct block_device *bdev;
+
+	bdev = open_backing_dev(device, new_disk_conf->backing_dev, device, true);
+	if (IS_ERR(bdev))
+		return ERR_OPEN_DISK;
+	nbc->backing_bdev = bdev;
+
+	/*
+	 * meta_dev_idx >= 0: external fixed size, possibly multiple
+	 * drbd sharing one meta device.  TODO in that case, paranoia
+	 * check that [md_bdev, meta_dev_idx] is not yet used by some
+	 * other drbd minor!  (if you use drbd.conf + drbdadm, that
+	 * should check it for you already; but if you don't, or
+	 * someone fooled it, we need to double check here)
+	 */
+	bdev = open_backing_dev(device, new_disk_conf->meta_dev,
+		/* claim ptr: device, if claimed exclusively; shared drbd_m_holder,
+		 * if potentially shared with other drbd minors */
+			(new_disk_conf->meta_dev_idx < 0) ? (void*)device : (void*)drbd_m_holder,
+		/* avoid double bd_claim_by_disk() for the same (source,target) tuple,
+		 * as would happen with internal metadata. */
+			(new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_FLEX_INT &&
+			 new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_INTERNAL));
+	if (IS_ERR(bdev))
+		return ERR_OPEN_MD_DISK;
+	nbc->md_bdev = bdev;
+	return NO_ERROR;
+}
+
+static void close_backing_dev(struct drbd_device *device, struct block_device *bdev,
+	bool do_bd_unlink)
+{
+	if (!bdev)
+		return;
+	if (do_bd_unlink)
+		bd_unlink_disk_holder(bdev, device->vdisk);
+	blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
+}
+
+void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev)
+{
+	if (ldev == NULL)
+		return;
+
+	close_backing_dev(device, ldev->md_bdev, ldev->md_bdev != ldev->backing_bdev);
+	close_backing_dev(device, ldev->backing_bdev, true);
+
+	kfree(ldev->disk_conf);
+	kfree(ldev);
+}
+
 int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
@@ -1462,7 +1581,6 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	sector_t min_md_device_sectors;
 	struct drbd_backing_dev *nbc = NULL; /* new_backing_conf */
 	struct disk_conf *new_disk_conf = NULL;
-	struct block_device *bdev;
 	struct lru_cache *resync_lru = NULL;
 	struct fifo_buffer *new_plan = NULL;
 	union drbd_state ns, os;
@@ -1478,7 +1596,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	device = adm_ctx.device;
 	mutex_lock(&adm_ctx.resource->adm_mutex);
 	peer_device = first_peer_device(device);
-	connection = peer_device ? peer_device->connection : NULL;
+	connection = peer_device->connection;
 	conn_reconfig_start(connection);
 
 	/* if you want to reconfigure, please tear down first */
@@ -1539,12 +1657,6 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		goto fail;
 	}
 
-	write_lock_irq(&global_state_lock);
-	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
-	write_unlock_irq(&global_state_lock);
-	if (retcode != NO_ERROR)
-		goto fail;
-
 	rcu_read_lock();
 	nc = rcu_dereference(connection->net_conf);
 	if (nc) {
@@ -1556,35 +1668,9 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	}
 	rcu_read_unlock();
 
-	bdev = blkdev_get_by_path(new_disk_conf->backing_dev,
-				  FMODE_READ | FMODE_WRITE | FMODE_EXCL, device);
-	if (IS_ERR(bdev)) {
-		drbd_err(device, "open(\"%s\") failed with %ld\n", new_disk_conf->backing_dev,
-			PTR_ERR(bdev));
-		retcode = ERR_OPEN_DISK;
-		goto fail;
-	}
-	nbc->backing_bdev = bdev;
-
-	/*
-	 * meta_dev_idx >= 0: external fixed size, possibly multiple
-	 * drbd sharing one meta device.  TODO in that case, paranoia
-	 * check that [md_bdev, meta_dev_idx] is not yet used by some
-	 * other drbd minor!  (if you use drbd.conf + drbdadm, that
-	 * should check it for you already; but if you don't, or
-	 * someone fooled it, we need to double check here)
-	 */
-	bdev = blkdev_get_by_path(new_disk_conf->meta_dev,
-				  FMODE_READ | FMODE_WRITE | FMODE_EXCL,
-				  (new_disk_conf->meta_dev_idx < 0) ?
-				  (void *)device : (void *)drbd_m_holder);
-	if (IS_ERR(bdev)) {
-		drbd_err(device, "open(\"%s\") failed with %ld\n", new_disk_conf->meta_dev,
-			PTR_ERR(bdev));
-		retcode = ERR_OPEN_MD_DISK;
+	retcode = open_backing_devices(device, new_disk_conf, nbc);
+	if (retcode != NO_ERROR)
 		goto fail;
-	}
-	nbc->md_bdev = bdev;
 
 	if ((nbc->backing_bdev == nbc->md_bdev) !=
 	    (new_disk_conf->meta_dev_idx == DRBD_MD_INDEX_INTERNAL ||
@@ -1707,6 +1793,13 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		goto force_diskless_dec;
 	}
 
+	lock_all_resources();
+	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
+	if (retcode != NO_ERROR) {
+		unlock_all_resources();
+		goto force_diskless_dec;
+	}
+
 	/* Reset the "barriers don't work" bits here, then force meta data to
 	 * be written, to ensure we determine if barriers are supported. */
 	if (new_disk_conf->md_flushes)
@@ -1727,7 +1820,9 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	new_disk_conf = NULL;
 	new_plan = NULL;
 
-	drbd_bump_write_ordering(device->resource, device->ldev, WO_bdev_flush);
+	drbd_resync_after_changed(device);
+	drbd_bump_write_ordering(device->resource, device->ldev, WO_BDEV_FLUSH);
+	unlock_all_resources();
 
 	if (drbd_md_test_flag(device->ldev, MDF_CRASHED_PRIMARY))
 		set_bit(CRASHED_PRIMARY, &device->flags);
@@ -1875,12 +1970,8 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
  fail:
 	conn_reconfig_done(connection);
 	if (nbc) {
-		if (nbc->backing_bdev)
-			blkdev_put(nbc->backing_bdev,
-				   FMODE_READ | FMODE_WRITE | FMODE_EXCL);
-		if (nbc->md_bdev)
-			blkdev_put(nbc->md_bdev,
-				   FMODE_READ | FMODE_WRITE | FMODE_EXCL);
+		close_backing_dev(device, nbc->md_bdev, nbc->md_bdev != nbc->backing_bdev);
+		close_backing_dev(device, nbc->backing_bdev, true);
 		kfree(nbc);
 	}
 	kfree(new_disk_conf);
@@ -1895,6 +1986,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 static int adm_detach(struct drbd_device *device, int force)
 {
 	enum drbd_state_rv retcode;
+	void *buffer;
 	int ret;
 
 	if (force) {
@@ -1905,13 +1997,16 @@ static int adm_detach(struct drbd_device *device, int force)
 	}
 
 	drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */
-	drbd_md_get_buffer(device, __func__); /* make sure there is no in-flight meta-data IO */
-	retcode = drbd_request_state(device, NS(disk, D_FAILED));
-	drbd_md_put_buffer(device);
+	buffer = drbd_md_get_buffer(device, __func__); /* make sure there is no in-flight meta-data IO */
+	if (buffer) {
+		retcode = drbd_request_state(device, NS(disk, D_FAILED));
+		drbd_md_put_buffer(device);
+	} else /* already <= D_FAILED */
+		retcode = SS_NOTHING_TO_DO;
 	/* D_FAILED will transition to DISKLESS. */
+	drbd_resume_io(device);
 	ret = wait_event_interruptible(device->misc_wait,
 			device->state.disk != D_FAILED);
-	drbd_resume_io(device);
 	if ((int)retcode == (int)SS_IS_DISKLESS)
 		retcode = SS_NOTHING_TO_DO;
 	if (ret)
@@ -2245,8 +2340,31 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
+static void connection_to_info(struct connection_info *info,
+			       struct drbd_connection *connection)
+{
+	info->conn_connection_state = connection->cstate;
+	info->conn_role = conn_highest_peer(connection);
+}
+
+static void peer_device_to_info(struct peer_device_info *info,
+				struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+
+	info->peer_repl_state =
+		max_t(enum drbd_conns, C_WF_REPORT_PARAMS, device->state.conn);
+	info->peer_disk_state = device->state.pdsk;
+	info->peer_resync_susp_user = device->state.user_isp;
+	info->peer_resync_susp_peer = device->state.peer_isp;
+	info->peer_resync_susp_dependency = device->state.aftr_isp;
+}
+
 int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
 {
+	struct connection_info connection_info;
+	enum drbd_notification_type flags;
+	unsigned int peer_devices = 0;
 	struct drbd_config_context adm_ctx;
 	struct drbd_peer_device *peer_device;
 	struct net_conf *old_net_conf, *new_net_conf = NULL;
@@ -2347,6 +2465,22 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
 	connection->peer_addr_len = nla_len(adm_ctx.peer_addr);
 	memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len);
 
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		peer_devices++;
+	}
+
+	connection_to_info(&connection_info, connection);
+	flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+	mutex_lock(&notification_mutex);
+	notify_connection_state(NULL, 0, connection, &connection_info, NOTIFY_CREATE | flags);
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		struct peer_device_info peer_device_info;
+
+		peer_device_to_info(&peer_device_info, peer_device);
+		flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+		notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CREATE | flags);
+	}
+	mutex_unlock(&notification_mutex);
 	mutex_unlock(&adm_ctx.resource->conf_update);
 
 	rcu_read_lock();
@@ -2428,6 +2562,8 @@ static enum drbd_state_rv conn_try_disconnect(struct drbd_connection *connection
 			drbd_err(connection,
 				"unexpected rv2=%d in conn_try_disconnect()\n",
 				rv2);
+		/* Unlike in DRBD 9, the state engine has generated
+		 * NOTIFY_DESTROY events before clearing connection->net_conf. */
 	}
 	return rv;
 }
@@ -2585,6 +2721,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 		mutex_unlock(&device->resource->conf_update);
 		synchronize_rcu();
 		kfree(old_disk_conf);
+		new_disk_conf = NULL;
 	}
 
 	ddsf = (rs.resize_force ? DDSF_FORCED : 0) | (rs.no_resync ? DDSF_NO_RESYNC : 0);
@@ -2618,6 +2755,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 
  fail_ldev:
 	put_ldev(device);
+	kfree(new_disk_conf);
 	goto fail;
 }
 
@@ -2855,7 +2993,30 @@ int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info)
 	mutex_lock(&adm_ctx.resource->adm_mutex);
 	device = adm_ctx.device;
 	if (test_bit(NEW_CUR_UUID, &device->flags)) {
-		drbd_uuid_new_current(device);
+		if (get_ldev_if_state(device, D_ATTACHING)) {
+			drbd_uuid_new_current(device);
+			put_ldev(device);
+		} else {
+			/* This is effectively a multi-stage "forced down".
+			 * The NEW_CUR_UUID bit is supposedly only set, if we
+			 * lost the replication connection, and are configured
+			 * to freeze IO and wait for some fence-peer handler.
+			 * So we still don't have a replication connection.
+			 * And now we don't have a local disk either.  After
+			 * resume, we will fail all pending and new IO, because
+			 * we don't have any data anymore.  Which means we will
+			 * eventually be able to terminate all users of this
+			 * device, and then take it down.  By bumping the
+			 * "effective" data uuid, we make sure that you really
+			 * need to tear down before you reconfigure, we will
+			 * the refuse to re-connect or re-attach (because no
+			 * matching real data uuid exists).
+			 */
+			u64 val;
+			get_random_bytes(&val, sizeof(u64));
+			drbd_set_ed_uuid(device, val);
+			drbd_warn(device, "Resumed without access to data; please tear down before attempting to re-configure.\n");
+		}
 		clear_bit(NEW_CUR_UUID, &device->flags);
 	}
 	drbd_suspend_io(device);
@@ -2910,6 +3071,486 @@ nla_put_failure:
 }
 
 /*
+ * The generic netlink dump callbacks are called outside the genl_lock(), so
+ * they cannot use the simple attribute parsing code which uses global
+ * attribute tables.
+ */
+static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr)
+{
+	const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ;
+	const int maxtype = ARRAY_SIZE(drbd_cfg_context_nl_policy) - 1;
+	struct nlattr *nla;
+
+	nla = nla_find(nlmsg_attrdata(nlh, hdrlen), nlmsg_attrlen(nlh, hdrlen),
+		       DRBD_NLA_CFG_CONTEXT);
+	if (!nla)
+		return NULL;
+	return drbd_nla_find_nested(maxtype, nla, __nla_type(attr));
+}
+
+static void resource_to_info(struct resource_info *, struct drbd_resource *);
+
+int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct drbd_genlmsghdr *dh;
+	struct drbd_resource *resource;
+	struct resource_info resource_info;
+	struct resource_statistics resource_statistics;
+	int err;
+
+	rcu_read_lock();
+	if (cb->args[0]) {
+		for_each_resource_rcu(resource, &drbd_resources)
+			if (resource == (struct drbd_resource *)cb->args[0])
+				goto found_resource;
+		err = 0;  /* resource was probably deleted */
+		goto out;
+	}
+	resource = list_entry(&drbd_resources,
+			      struct drbd_resource, resources);
+
+found_resource:
+	list_for_each_entry_continue_rcu(resource, &drbd_resources, resources) {
+		goto put_result;
+	}
+	err = 0;
+	goto out;
+
+put_result:
+	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			NLM_F_MULTI, DRBD_ADM_GET_RESOURCES);
+	err = -ENOMEM;
+	if (!dh)
+		goto out;
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL);
+	if (err)
+		goto out;
+	err = res_opts_to_skb(skb, &resource->res_opts, !capable(CAP_SYS_ADMIN));
+	if (err)
+		goto out;
+	resource_to_info(&resource_info, resource);
+	err = resource_info_to_skb(skb, &resource_info, !capable(CAP_SYS_ADMIN));
+	if (err)
+		goto out;
+	resource_statistics.res_stat_write_ordering = resource->write_ordering;
+	err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN));
+	if (err)
+		goto out;
+	cb->args[0] = (long)resource;
+	genlmsg_end(skb, dh);
+	err = 0;
+
+out:
+	rcu_read_unlock();
+	if (err)
+		return err;
+	return skb->len;
+}
+
+static void device_to_statistics(struct device_statistics *s,
+				 struct drbd_device *device)
+{
+	memset(s, 0, sizeof(*s));
+	s->dev_upper_blocked = !may_inc_ap_bio(device);
+	if (get_ldev(device)) {
+		struct drbd_md *md = &device->ldev->md;
+		u64 *history_uuids = (u64 *)s->history_uuids;
+		struct request_queue *q;
+		int n;
+
+		spin_lock_irq(&md->uuid_lock);
+		s->dev_current_uuid = md->uuid[UI_CURRENT];
+		BUILD_BUG_ON(sizeof(s->history_uuids) < UI_HISTORY_END - UI_HISTORY_START + 1);
+		for (n = 0; n < UI_HISTORY_END - UI_HISTORY_START + 1; n++)
+			history_uuids[n] = md->uuid[UI_HISTORY_START + n];
+		for (; n < HISTORY_UUIDS; n++)
+			history_uuids[n] = 0;
+		s->history_uuids_len = HISTORY_UUIDS;
+		spin_unlock_irq(&md->uuid_lock);
+
+		s->dev_disk_flags = md->flags;
+		q = bdev_get_queue(device->ldev->backing_bdev);
+		s->dev_lower_blocked =
+			bdi_congested(&q->backing_dev_info,
+				      (1 << WB_async_congested) |
+				      (1 << WB_sync_congested));
+		put_ldev(device);
+	}
+	s->dev_size = drbd_get_capacity(device->this_bdev);
+	s->dev_read = device->read_cnt;
+	s->dev_write = device->writ_cnt;
+	s->dev_al_writes = device->al_writ_cnt;
+	s->dev_bm_writes = device->bm_writ_cnt;
+	s->dev_upper_pending = atomic_read(&device->ap_bio_cnt);
+	s->dev_lower_pending = atomic_read(&device->local_cnt);
+	s->dev_al_suspended = test_bit(AL_SUSPENDED, &device->flags);
+	s->dev_exposed_data_uuid = device->ed_uuid;
+}
+
+static int put_resource_in_arg0(struct netlink_callback *cb, int holder_nr)
+{
+	if (cb->args[0]) {
+		struct drbd_resource *resource =
+			(struct drbd_resource *)cb->args[0];
+		kref_put(&resource->kref, drbd_destroy_resource);
+	}
+
+	return 0;
+}
+
+int drbd_adm_dump_devices_done(struct netlink_callback *cb) {
+	return put_resource_in_arg0(cb, 7);
+}
+
+static void device_to_info(struct device_info *, struct drbd_device *);
+
+int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct nlattr *resource_filter;
+	struct drbd_resource *resource;
+	struct drbd_device *uninitialized_var(device);
+	int minor, err, retcode;
+	struct drbd_genlmsghdr *dh;
+	struct device_info device_info;
+	struct device_statistics device_statistics;
+	struct idr *idr_to_search;
+
+	resource = (struct drbd_resource *)cb->args[0];
+	if (!cb->args[0] && !cb->args[1]) {
+		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		if (resource_filter) {
+			retcode = ERR_RES_NOT_KNOWN;
+			resource = drbd_find_resource(nla_data(resource_filter));
+			if (!resource)
+				goto put_result;
+			cb->args[0] = (long)resource;
+		}
+	}
+
+	rcu_read_lock();
+	minor = cb->args[1];
+	idr_to_search = resource ? &resource->devices : &drbd_devices;
+	device = idr_get_next(idr_to_search, &minor);
+	if (!device) {
+		err = 0;
+		goto out;
+	}
+	idr_for_each_entry_continue(idr_to_search, device, minor) {
+		retcode = NO_ERROR;
+		goto put_result;  /* only one iteration */
+	}
+	err = 0;
+	goto out;  /* no more devices */
+
+put_result:
+	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			NLM_F_MULTI, DRBD_ADM_GET_DEVICES);
+	err = -ENOMEM;
+	if (!dh)
+		goto out;
+	dh->ret_code = retcode;
+	dh->minor = -1U;
+	if (retcode == NO_ERROR) {
+		dh->minor = device->minor;
+		err = nla_put_drbd_cfg_context(skb, device->resource, NULL, device);
+		if (err)
+			goto out;
+		if (get_ldev(device)) {
+			struct disk_conf *disk_conf =
+				rcu_dereference(device->ldev->disk_conf);
+
+			err = disk_conf_to_skb(skb, disk_conf, !capable(CAP_SYS_ADMIN));
+			put_ldev(device);
+			if (err)
+				goto out;
+		}
+		device_to_info(&device_info, device);
+		err = device_info_to_skb(skb, &device_info, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+
+		device_to_statistics(&device_statistics, device);
+		err = device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		cb->args[1] = minor + 1;
+	}
+	genlmsg_end(skb, dh);
+	err = 0;
+
+out:
+	rcu_read_unlock();
+	if (err)
+		return err;
+	return skb->len;
+}
+
+int drbd_adm_dump_connections_done(struct netlink_callback *cb)
+{
+	return put_resource_in_arg0(cb, 6);
+}
+
+enum { SINGLE_RESOURCE, ITERATE_RESOURCES };
+
+int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct nlattr *resource_filter;
+	struct drbd_resource *resource = NULL, *next_resource;
+	struct drbd_connection *uninitialized_var(connection);
+	int err = 0, retcode;
+	struct drbd_genlmsghdr *dh;
+	struct connection_info connection_info;
+	struct connection_statistics connection_statistics;
+
+	rcu_read_lock();
+	resource = (struct drbd_resource *)cb->args[0];
+	if (!cb->args[0]) {
+		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		if (resource_filter) {
+			retcode = ERR_RES_NOT_KNOWN;
+			resource = drbd_find_resource(nla_data(resource_filter));
+			if (!resource)
+				goto put_result;
+			cb->args[0] = (long)resource;
+			cb->args[1] = SINGLE_RESOURCE;
+		}
+	}
+	if (!resource) {
+		if (list_empty(&drbd_resources))
+			goto out;
+		resource = list_first_entry(&drbd_resources, struct drbd_resource, resources);
+		kref_get(&resource->kref);
+		cb->args[0] = (long)resource;
+		cb->args[1] = ITERATE_RESOURCES;
+	}
+
+    next_resource:
+	rcu_read_unlock();
+	mutex_lock(&resource->conf_update);
+	rcu_read_lock();
+	if (cb->args[2]) {
+		for_each_connection_rcu(connection, resource)
+			if (connection == (struct drbd_connection *)cb->args[2])
+				goto found_connection;
+		/* connection was probably deleted */
+		goto no_more_connections;
+	}
+	connection = list_entry(&resource->connections, struct drbd_connection, connections);
+
+found_connection:
+	list_for_each_entry_continue_rcu(connection, &resource->connections, connections) {
+		if (!has_net_conf(connection))
+			continue;
+		retcode = NO_ERROR;
+		goto put_result;  /* only one iteration */
+	}
+
+no_more_connections:
+	if (cb->args[1] == ITERATE_RESOURCES) {
+		for_each_resource_rcu(next_resource, &drbd_resources) {
+			if (next_resource == resource)
+				goto found_resource;
+		}
+		/* resource was probably deleted */
+	}
+	goto out;
+
+found_resource:
+	list_for_each_entry_continue_rcu(next_resource, &drbd_resources, resources) {
+		mutex_unlock(&resource->conf_update);
+		kref_put(&resource->kref, drbd_destroy_resource);
+		resource = next_resource;
+		kref_get(&resource->kref);
+		cb->args[0] = (long)resource;
+		cb->args[2] = 0;
+		goto next_resource;
+	}
+	goto out;  /* no more resources */
+
+put_result:
+	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			NLM_F_MULTI, DRBD_ADM_GET_CONNECTIONS);
+	err = -ENOMEM;
+	if (!dh)
+		goto out;
+	dh->ret_code = retcode;
+	dh->minor = -1U;
+	if (retcode == NO_ERROR) {
+		struct net_conf *net_conf;
+
+		err = nla_put_drbd_cfg_context(skb, resource, connection, NULL);
+		if (err)
+			goto out;
+		net_conf = rcu_dereference(connection->net_conf);
+		if (net_conf) {
+			err = net_conf_to_skb(skb, net_conf, !capable(CAP_SYS_ADMIN));
+			if (err)
+				goto out;
+		}
+		connection_to_info(&connection_info, connection);
+		err = connection_info_to_skb(skb, &connection_info, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
+		err = connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		cb->args[2] = (long)connection;
+	}
+	genlmsg_end(skb, dh);
+	err = 0;
+
+out:
+	rcu_read_unlock();
+	if (resource)
+		mutex_unlock(&resource->conf_update);
+	if (err)
+		return err;
+	return skb->len;
+}
+
+enum mdf_peer_flag {
+	MDF_PEER_CONNECTED =	1 << 0,
+	MDF_PEER_OUTDATED =	1 << 1,
+	MDF_PEER_FENCING =	1 << 2,
+	MDF_PEER_FULL_SYNC =	1 << 3,
+};
+
+static void peer_device_to_statistics(struct peer_device_statistics *s,
+				      struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+
+	memset(s, 0, sizeof(*s));
+	s->peer_dev_received = device->recv_cnt;
+	s->peer_dev_sent = device->send_cnt;
+	s->peer_dev_pending = atomic_read(&device->ap_pending_cnt) +
+			      atomic_read(&device->rs_pending_cnt);
+	s->peer_dev_unacked = atomic_read(&device->unacked_cnt);
+	s->peer_dev_out_of_sync = drbd_bm_total_weight(device) << (BM_BLOCK_SHIFT - 9);
+	s->peer_dev_resync_failed = device->rs_failed << (BM_BLOCK_SHIFT - 9);
+	if (get_ldev(device)) {
+		struct drbd_md *md = &device->ldev->md;
+
+		spin_lock_irq(&md->uuid_lock);
+		s->peer_dev_bitmap_uuid = md->uuid[UI_BITMAP];
+		spin_unlock_irq(&md->uuid_lock);
+		s->peer_dev_flags =
+			(drbd_md_test_flag(device->ldev, MDF_CONNECTED_IND) ?
+				MDF_PEER_CONNECTED : 0) +
+			(drbd_md_test_flag(device->ldev, MDF_CONSISTENT) &&
+			 !drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE) ?
+				MDF_PEER_OUTDATED : 0) +
+			/* FIXME: MDF_PEER_FENCING? */
+			(drbd_md_test_flag(device->ldev, MDF_FULL_SYNC) ?
+				MDF_PEER_FULL_SYNC : 0);
+		put_ldev(device);
+	}
+}
+
+int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb)
+{
+	return put_resource_in_arg0(cb, 9);
+}
+
+int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct nlattr *resource_filter;
+	struct drbd_resource *resource;
+	struct drbd_device *uninitialized_var(device);
+	struct drbd_peer_device *peer_device = NULL;
+	int minor, err, retcode;
+	struct drbd_genlmsghdr *dh;
+	struct idr *idr_to_search;
+
+	resource = (struct drbd_resource *)cb->args[0];
+	if (!cb->args[0] && !cb->args[1]) {
+		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		if (resource_filter) {
+			retcode = ERR_RES_NOT_KNOWN;
+			resource = drbd_find_resource(nla_data(resource_filter));
+			if (!resource)
+				goto put_result;
+		}
+		cb->args[0] = (long)resource;
+	}
+
+	rcu_read_lock();
+	minor = cb->args[1];
+	idr_to_search = resource ? &resource->devices : &drbd_devices;
+	device = idr_find(idr_to_search, minor);
+	if (!device) {
+next_device:
+		minor++;
+		cb->args[2] = 0;
+		device = idr_get_next(idr_to_search, &minor);
+		if (!device) {
+			err = 0;
+			goto out;
+		}
+	}
+	if (cb->args[2]) {
+		for_each_peer_device(peer_device, device)
+			if (peer_device == (struct drbd_peer_device *)cb->args[2])
+				goto found_peer_device;
+		/* peer device was probably deleted */
+		goto next_device;
+	}
+	/* Make peer_device point to the list head (not the first entry). */
+	peer_device = list_entry(&device->peer_devices, struct drbd_peer_device, peer_devices);
+
+found_peer_device:
+	list_for_each_entry_continue_rcu(peer_device, &device->peer_devices, peer_devices) {
+		if (!has_net_conf(peer_device->connection))
+			continue;
+		retcode = NO_ERROR;
+		goto put_result;  /* only one iteration */
+	}
+	goto next_device;
+
+put_result:
+	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			NLM_F_MULTI, DRBD_ADM_GET_PEER_DEVICES);
+	err = -ENOMEM;
+	if (!dh)
+		goto out;
+	dh->ret_code = retcode;
+	dh->minor = -1U;
+	if (retcode == NO_ERROR) {
+		struct peer_device_info peer_device_info;
+		struct peer_device_statistics peer_device_statistics;
+
+		dh->minor = minor;
+		err = nla_put_drbd_cfg_context(skb, device->resource, peer_device->connection, device);
+		if (err)
+			goto out;
+		peer_device_to_info(&peer_device_info, peer_device);
+		err = peer_device_info_to_skb(skb, &peer_device_info, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		peer_device_to_statistics(&peer_device_statistics, peer_device);
+		err = peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		cb->args[1] = minor;
+		cb->args[2] = (long)peer_device;
+	}
+	genlmsg_end(skb, dh);
+	err = 0;
+
+out:
+	rcu_read_unlock();
+	if (err)
+		return err;
+	return skb->len;
+}
+/*
  * Return the connection of @resource if @resource has exactly one connection.
  */
 static struct drbd_connection *the_only_connection(struct drbd_resource *resource)
@@ -3414,8 +4055,18 @@ drbd_check_resource_name(struct drbd_config_context *adm_ctx)
 	return NO_ERROR;
 }
 
+static void resource_to_info(struct resource_info *info,
+			     struct drbd_resource *resource)
+{
+	info->res_role = conn_highest_role(first_connection(resource));
+	info->res_susp = resource->susp;
+	info->res_susp_nod = resource->susp_nod;
+	info->res_susp_fen = resource->susp_fen;
+}
+
 int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info)
 {
+	struct drbd_connection *connection;
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
 	struct res_opts res_opts;
@@ -3449,13 +4100,33 @@ int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	/* not yet safe for genl_family.parallel_ops */
-	if (!conn_create(adm_ctx.resource_name, &res_opts))
+	mutex_lock(&resources_mutex);
+	connection = conn_create(adm_ctx.resource_name, &res_opts);
+	mutex_unlock(&resources_mutex);
+
+	if (connection) {
+		struct resource_info resource_info;
+
+		mutex_lock(&notification_mutex);
+		resource_to_info(&resource_info, connection->resource);
+		notify_resource_state(NULL, 0, connection->resource,
+				      &resource_info, NOTIFY_CREATE);
+		mutex_unlock(&notification_mutex);
+	} else
 		retcode = ERR_NOMEM;
+
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
+static void device_to_info(struct device_info *info,
+			   struct drbd_device *device)
+{
+	info->dev_disk_state = device->state.disk;
+}
+
+
 int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
@@ -3490,6 +4161,36 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 
 	mutex_lock(&adm_ctx.resource->adm_mutex);
 	retcode = drbd_create_device(&adm_ctx, dh->minor);
+	if (retcode == NO_ERROR) {
+		struct drbd_device *device;
+		struct drbd_peer_device *peer_device;
+		struct device_info info;
+		unsigned int peer_devices = 0;
+		enum drbd_notification_type flags;
+
+		device = minor_to_device(dh->minor);
+		for_each_peer_device(peer_device, device) {
+			if (!has_net_conf(peer_device->connection))
+				continue;
+			peer_devices++;
+		}
+
+		device_to_info(&info, device);
+		mutex_lock(&notification_mutex);
+		flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+		notify_device_state(NULL, 0, device, &info, NOTIFY_CREATE | flags);
+		for_each_peer_device(peer_device, device) {
+			struct peer_device_info peer_device_info;
+
+			if (!has_net_conf(peer_device->connection))
+				continue;
+			peer_device_to_info(&peer_device_info, peer_device);
+			flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+			notify_peer_device_state(NULL, 0, peer_device, &peer_device_info,
+						 NOTIFY_CREATE | flags);
+		}
+		mutex_unlock(&notification_mutex);
+	}
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
@@ -3498,13 +4199,35 @@ out:
 
 static enum drbd_ret_code adm_del_minor(struct drbd_device *device)
 {
+	struct drbd_peer_device *peer_device;
+
 	if (device->state.disk == D_DISKLESS &&
 	    /* no need to be device->state.conn == C_STANDALONE &&
 	     * we may want to delete a minor from a live replication group.
 	     */
 	    device->state.role == R_SECONDARY) {
+		struct drbd_connection *connection =
+			first_connection(device->resource);
+
 		_drbd_request_state(device, NS(conn, C_WF_REPORT_PARAMS),
 				    CS_VERBOSE + CS_WAIT_COMPLETE);
+
+		/* If the state engine hasn't stopped the sender thread yet, we
+		 * need to flush the sender work queue before generating the
+		 * DESTROY events here. */
+		if (get_t_state(&connection->worker) == RUNNING)
+			drbd_flush_workqueue(&connection->sender_work);
+
+		mutex_lock(&notification_mutex);
+		for_each_peer_device(peer_device, device) {
+			if (!has_net_conf(peer_device->connection))
+				continue;
+			notify_peer_device_state(NULL, 0, peer_device, NULL,
+						 NOTIFY_DESTROY | NOTIFY_CONTINUES);
+		}
+		notify_device_state(NULL, 0, device, NULL, NOTIFY_DESTROY);
+		mutex_unlock(&notification_mutex);
+
 		drbd_delete_device(device);
 		return NO_ERROR;
 	} else
@@ -3541,7 +4264,16 @@ static int adm_del_resource(struct drbd_resource *resource)
 	if (!idr_is_empty(&resource->devices))
 		return ERR_RES_IN_USE;
 
+	/* The state engine has stopped the sender thread, so we don't
+	 * need to flush the sender work queue before generating the
+	 * DESTROY event here. */
+	mutex_lock(&notification_mutex);
+	notify_resource_state(NULL, 0, resource, NULL, NOTIFY_DESTROY);
+	mutex_unlock(&notification_mutex);
+
+	mutex_lock(&resources_mutex);
 	list_del_rcu(&resource->resources);
+	mutex_unlock(&resources_mutex);
 	/* Make sure all threads have actually stopped: state handling only
 	 * does drbd_thread_stop_nowait(). */
 	list_for_each_entry(connection, &resource->connections, connections)
@@ -3637,7 +4369,6 @@ finish:
 
 void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
 {
-	static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
 	struct sk_buff *msg;
 	struct drbd_genlmsghdr *d_out;
 	unsigned seq;
@@ -3658,7 +4389,7 @@ void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
 	if (nla_put_status_info(msg, device, sib))
 		goto nla_put_failure;
 	genlmsg_end(msg, d_out);
-	err = drbd_genl_multicast_events(msg, 0);
+	err = drbd_genl_multicast_events(msg, GFP_NOWAIT);
 	/* msg has been consumed or freed in netlink_broadcast() */
 	if (err && err != -ESRCH)
 		goto failed;
@@ -3672,3 +4403,405 @@ failed:
 			"Event seq:%u sib_reason:%u\n",
 			err, seq, sib->sib_reason);
 }
+
+static int nla_put_notification_header(struct sk_buff *msg,
+				       enum drbd_notification_type type)
+{
+	struct drbd_notification_header nh = {
+		.nh_type = type,
+	};
+
+	return drbd_notification_header_to_skb(msg, &nh, true);
+}
+
+void notify_resource_state(struct sk_buff *skb,
+			   unsigned int seq,
+			   struct drbd_resource *resource,
+			   struct resource_info *resource_info,
+			   enum drbd_notification_type type)
+{
+	struct resource_statistics resource_statistics;
+	struct drbd_genlmsghdr *dh;
+	bool multicast = false;
+	int err;
+
+	if (!skb) {
+		seq = atomic_inc_return(&notify_genl_seq);
+		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+		err = -ENOMEM;
+		if (!skb)
+			goto failed;
+		multicast = true;
+	}
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_RESOURCE_STATE);
+	if (!dh)
+		goto nla_put_failure;
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL) ||
+	    nla_put_notification_header(skb, type) ||
+	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
+	     resource_info_to_skb(skb, resource_info, true)))
+		goto nla_put_failure;
+	resource_statistics.res_stat_write_ordering = resource->write_ordering;
+	err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN));
+	if (err)
+		goto nla_put_failure;
+	genlmsg_end(skb, dh);
+	if (multicast) {
+		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		/* skb has been consumed or freed in netlink_broadcast() */
+		if (err && err != -ESRCH)
+			goto failed;
+	}
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+failed:
+	drbd_err(resource, "Error %d while broadcasting event. Event seq:%u\n",
+			err, seq);
+}
+
+void notify_device_state(struct sk_buff *skb,
+			 unsigned int seq,
+			 struct drbd_device *device,
+			 struct device_info *device_info,
+			 enum drbd_notification_type type)
+{
+	struct device_statistics device_statistics;
+	struct drbd_genlmsghdr *dh;
+	bool multicast = false;
+	int err;
+
+	if (!skb) {
+		seq = atomic_inc_return(&notify_genl_seq);
+		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+		err = -ENOMEM;
+		if (!skb)
+			goto failed;
+		multicast = true;
+	}
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_DEVICE_STATE);
+	if (!dh)
+		goto nla_put_failure;
+	dh->minor = device->minor;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device) ||
+	    nla_put_notification_header(skb, type) ||
+	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
+	     device_info_to_skb(skb, device_info, true)))
+		goto nla_put_failure;
+	device_to_statistics(&device_statistics, device);
+	device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN));
+	genlmsg_end(skb, dh);
+	if (multicast) {
+		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		/* skb has been consumed or freed in netlink_broadcast() */
+		if (err && err != -ESRCH)
+			goto failed;
+	}
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+failed:
+	drbd_err(device, "Error %d while broadcasting event. Event seq:%u\n",
+		 err, seq);
+}
+
+void notify_connection_state(struct sk_buff *skb,
+			     unsigned int seq,
+			     struct drbd_connection *connection,
+			     struct connection_info *connection_info,
+			     enum drbd_notification_type type)
+{
+	struct connection_statistics connection_statistics;
+	struct drbd_genlmsghdr *dh;
+	bool multicast = false;
+	int err;
+
+	if (!skb) {
+		seq = atomic_inc_return(&notify_genl_seq);
+		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+		err = -ENOMEM;
+		if (!skb)
+			goto failed;
+		multicast = true;
+	}
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_CONNECTION_STATE);
+	if (!dh)
+		goto nla_put_failure;
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL) ||
+	    nla_put_notification_header(skb, type) ||
+	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
+	     connection_info_to_skb(skb, connection_info, true)))
+		goto nla_put_failure;
+	connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
+	connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
+	genlmsg_end(skb, dh);
+	if (multicast) {
+		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		/* skb has been consumed or freed in netlink_broadcast() */
+		if (err && err != -ESRCH)
+			goto failed;
+	}
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+failed:
+	drbd_err(connection, "Error %d while broadcasting event. Event seq:%u\n",
+		 err, seq);
+}
+
+void notify_peer_device_state(struct sk_buff *skb,
+			      unsigned int seq,
+			      struct drbd_peer_device *peer_device,
+			      struct peer_device_info *peer_device_info,
+			      enum drbd_notification_type type)
+{
+	struct peer_device_statistics peer_device_statistics;
+	struct drbd_resource *resource = peer_device->device->resource;
+	struct drbd_genlmsghdr *dh;
+	bool multicast = false;
+	int err;
+
+	if (!skb) {
+		seq = atomic_inc_return(&notify_genl_seq);
+		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+		err = -ENOMEM;
+		if (!skb)
+			goto failed;
+		multicast = true;
+	}
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_PEER_DEVICE_STATE);
+	if (!dh)
+		goto nla_put_failure;
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device) ||
+	    nla_put_notification_header(skb, type) ||
+	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
+	     peer_device_info_to_skb(skb, peer_device_info, true)))
+		goto nla_put_failure;
+	peer_device_to_statistics(&peer_device_statistics, peer_device);
+	peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
+	genlmsg_end(skb, dh);
+	if (multicast) {
+		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		/* skb has been consumed or freed in netlink_broadcast() */
+		if (err && err != -ESRCH)
+			goto failed;
+	}
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+failed:
+	drbd_err(peer_device, "Error %d while broadcasting event. Event seq:%u\n",
+		 err, seq);
+}
+
+void notify_helper(enum drbd_notification_type type,
+		   struct drbd_device *device, struct drbd_connection *connection,
+		   const char *name, int status)
+{
+	struct drbd_resource *resource = device ? device->resource : connection->resource;
+	struct drbd_helper_info helper_info;
+	unsigned int seq = atomic_inc_return(&notify_genl_seq);
+	struct sk_buff *skb = NULL;
+	struct drbd_genlmsghdr *dh;
+	int err;
+
+	strlcpy(helper_info.helper_name, name, sizeof(helper_info.helper_name));
+	helper_info.helper_name_len = min(strlen(name), sizeof(helper_info.helper_name));
+	helper_info.helper_status = status;
+
+	skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+	err = -ENOMEM;
+	if (!skb)
+		goto fail;
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_HELPER);
+	if (!dh)
+		goto fail;
+	dh->minor = device ? device->minor : -1;
+	dh->ret_code = NO_ERROR;
+	mutex_lock(&notification_mutex);
+	if (nla_put_drbd_cfg_context(skb, resource, connection, device) ||
+	    nla_put_notification_header(skb, type) ||
+	    drbd_helper_info_to_skb(skb, &helper_info, true))
+		goto unlock_fail;
+	genlmsg_end(skb, dh);
+	err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+	skb = NULL;
+	/* skb has been consumed or freed in netlink_broadcast() */
+	if (err && err != -ESRCH)
+		goto unlock_fail;
+	mutex_unlock(&notification_mutex);
+	return;
+
+unlock_fail:
+	mutex_unlock(&notification_mutex);
+fail:
+	nlmsg_free(skb);
+	drbd_err(resource, "Error %d while broadcasting event. Event seq:%u\n",
+		 err, seq);
+}
+
+static void notify_initial_state_done(struct sk_buff *skb, unsigned int seq)
+{
+	struct drbd_genlmsghdr *dh;
+	int err;
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_INITIAL_STATE_DONE);
+	if (!dh)
+		goto nla_put_failure;
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_notification_header(skb, NOTIFY_EXISTS))
+		goto nla_put_failure;
+	genlmsg_end(skb, dh);
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+	pr_err("Error %d sending event. Event seq:%u\n", err, seq);
+}
+
+static void free_state_changes(struct list_head *list)
+{
+	while (!list_empty(list)) {
+		struct drbd_state_change *state_change =
+			list_first_entry(list, struct drbd_state_change, list);
+		list_del(&state_change->list);
+		forget_state_change(state_change);
+	}
+}
+
+static unsigned int notifications_for_state_change(struct drbd_state_change *state_change)
+{
+	return 1 +
+	       state_change->n_connections +
+	       state_change->n_devices +
+	       state_change->n_devices * state_change->n_connections;
+}
+
+static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct drbd_state_change *state_change = (struct drbd_state_change *)cb->args[0];
+	unsigned int seq = cb->args[2];
+	unsigned int n;
+	enum drbd_notification_type flags = 0;
+
+	/* There is no need for taking notification_mutex here: it doesn't
+	   matter if the initial state events mix with later state chage
+	   events; we can always tell the events apart by the NOTIFY_EXISTS
+	   flag. */
+
+	cb->args[5]--;
+	if (cb->args[5] == 1) {
+		notify_initial_state_done(skb, seq);
+		goto out;
+	}
+	n = cb->args[4]++;
+	if (cb->args[4] < cb->args[3])
+		flags |= NOTIFY_CONTINUES;
+	if (n < 1) {
+		notify_resource_state_change(skb, seq, state_change->resource,
+					     NOTIFY_EXISTS | flags);
+		goto next;
+	}
+	n--;
+	if (n < state_change->n_connections) {
+		notify_connection_state_change(skb, seq, &state_change->connections[n],
+					       NOTIFY_EXISTS | flags);
+		goto next;
+	}
+	n -= state_change->n_connections;
+	if (n < state_change->n_devices) {
+		notify_device_state_change(skb, seq, &state_change->devices[n],
+					   NOTIFY_EXISTS | flags);
+		goto next;
+	}
+	n -= state_change->n_devices;
+	if (n < state_change->n_devices * state_change->n_connections) {
+		notify_peer_device_state_change(skb, seq, &state_change->peer_devices[n],
+						NOTIFY_EXISTS | flags);
+		goto next;
+	}
+
+next:
+	if (cb->args[4] == cb->args[3]) {
+		struct drbd_state_change *next_state_change =
+			list_entry(state_change->list.next,
+				   struct drbd_state_change, list);
+		cb->args[0] = (long)next_state_change;
+		cb->args[3] = notifications_for_state_change(next_state_change);
+		cb->args[4] = 0;
+	}
+out:
+	return skb->len;
+}
+
+int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct drbd_resource *resource;
+	LIST_HEAD(head);
+
+	if (cb->args[5] >= 1) {
+		if (cb->args[5] > 1)
+			return get_initial_state(skb, cb);
+		if (cb->args[0]) {
+			struct drbd_state_change *state_change =
+				(struct drbd_state_change *)cb->args[0];
+
+			/* connect list to head */
+			list_add(&head, &state_change->list);
+			free_state_changes(&head);
+		}
+		return 0;
+	}
+
+	cb->args[5] = 2;  /* number of iterations */
+	mutex_lock(&resources_mutex);
+	for_each_resource(resource, &drbd_resources) {
+		struct drbd_state_change *state_change;
+
+		state_change = remember_old_state(resource, GFP_KERNEL);
+		if (!state_change) {
+			if (!list_empty(&head))
+				free_state_changes(&head);
+			mutex_unlock(&resources_mutex);
+			return -ENOMEM;
+		}
+		copy_old_to_new_state_change(state_change);
+		list_add_tail(&state_change->list, &head);
+		cb->args[5] += notifications_for_state_change(state_change);
+	}
+	mutex_unlock(&resources_mutex);
+
+	if (!list_empty(&head)) {
+		struct drbd_state_change *state_change =
+			list_entry(head.next, struct drbd_state_change, list);
+		cb->args[0] = (long)state_change;
+		cb->args[3] = notifications_for_state_change(state_change);
+		list_del(&head);  /* detach list from head */
+	}
+
+	cb->args[2] = cb->nlh->nlmsg_seq;
+	return get_initial_state(skb, cb);
+}
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 3b10fa6cb039..6537b25db9c1 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -245,9 +245,9 @@ static int drbd_seq_show(struct seq_file *seq, void *v)
 	char wp;
 
 	static char write_ordering_chars[] = {
-		[WO_none] = 'n',
-		[WO_drain_io] = 'd',
-		[WO_bdev_flush] = 'f',
+		[WO_NONE] = 'n',
+		[WO_DRAIN_IO] = 'd',
+		[WO_BDEV_FLUSH] = 'f',
 	};
 
 	seq_printf(seq, "version: " REL_VERSION " (api:%d/proto:%d-%d)\n%s\n",
diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h
index 2da9104a3851..ef9245363dcc 100644
--- a/drivers/block/drbd/drbd_protocol.h
+++ b/drivers/block/drbd/drbd_protocol.h
@@ -23,7 +23,7 @@ enum drbd_packet {
 	P_AUTH_RESPONSE	      = 0x11,
 	P_STATE_CHG_REQ	      = 0x12,
 
-	/* asender (meta socket */
+	/* (meta socket) */
 	P_PING		      = 0x13,
 	P_PING_ACK	      = 0x14,
 	P_RECV_ACK	      = 0x15, /* Used in protocol B */
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index b4b5680ac6ad..1957fe8601dc 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -215,7 +215,7 @@ static void reclaim_finished_net_peer_reqs(struct drbd_device *device,
 	}
 }
 
-static void drbd_kick_lo_and_reclaim_net(struct drbd_device *device)
+static void drbd_reclaim_net_peer_reqs(struct drbd_device *device)
 {
 	LIST_HEAD(reclaimed);
 	struct drbd_peer_request *peer_req, *t;
@@ -223,11 +223,30 @@ static void drbd_kick_lo_and_reclaim_net(struct drbd_device *device)
 	spin_lock_irq(&device->resource->req_lock);
 	reclaim_finished_net_peer_reqs(device, &reclaimed);
 	spin_unlock_irq(&device->resource->req_lock);
-
 	list_for_each_entry_safe(peer_req, t, &reclaimed, w.list)
 		drbd_free_net_peer_req(device, peer_req);
 }
 
+static void conn_reclaim_net_peer_reqs(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		if (!atomic_read(&device->pp_in_use_by_net))
+			continue;
+
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		drbd_reclaim_net_peer_reqs(device);
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+}
+
 /**
  * drbd_alloc_pages() - Returns @number pages, retries forever (or until signalled)
  * @device:	DRBD device.
@@ -265,10 +284,15 @@ struct page *drbd_alloc_pages(struct drbd_peer_device *peer_device, unsigned int
 	if (atomic_read(&device->pp_in_use) < mxb)
 		page = __drbd_alloc_pages(device, number);
 
+	/* Try to keep the fast path fast, but occasionally we need
+	 * to reclaim the pages we lended to the network stack. */
+	if (page && atomic_read(&device->pp_in_use_by_net) > 512)
+		drbd_reclaim_net_peer_reqs(device);
+
 	while (page == NULL) {
 		prepare_to_wait(&drbd_pp_wait, &wait, TASK_INTERRUPTIBLE);
 
-		drbd_kick_lo_and_reclaim_net(device);
+		drbd_reclaim_net_peer_reqs(device);
 
 		if (atomic_read(&device->pp_in_use) < mxb) {
 			page = __drbd_alloc_pages(device, number);
@@ -1099,7 +1123,15 @@ randomize:
 		return 0;
 	}
 
-	drbd_thread_start(&connection->asender);
+	drbd_thread_start(&connection->ack_receiver);
+	/* opencoded create_singlethread_workqueue(),
+	 * to be able to use format string arguments */
+	connection->ack_sender =
+		alloc_ordered_workqueue("drbd_as_%s", WQ_MEM_RECLAIM, connection->resource->name);
+	if (!connection->ack_sender) {
+		drbd_err(connection, "Failed to create workqueue ack_sender\n");
+		return 0;
+	}
 
 	mutex_lock(&connection->resource->conf_update);
 	/* The discard_my_data flag is a single-shot modifier to the next
@@ -1178,7 +1210,7 @@ static void drbd_flush(struct drbd_connection *connection)
 	struct drbd_peer_device *peer_device;
 	int vnr;
 
-	if (connection->resource->write_ordering >= WO_bdev_flush) {
+	if (connection->resource->write_ordering >= WO_BDEV_FLUSH) {
 		rcu_read_lock();
 		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
 			struct drbd_device *device = peer_device->device;
@@ -1203,7 +1235,7 @@ static void drbd_flush(struct drbd_connection *connection)
 				/* would rather check on EOPNOTSUPP, but that is not reliable.
 				 * don't try again for ANY return value != 0
 				 * if (rv == -EOPNOTSUPP) */
-				drbd_bump_write_ordering(connection->resource, NULL, WO_drain_io);
+				drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO);
 			}
 			put_ldev(device);
 			kref_put(&device->kref, drbd_destroy_device);
@@ -1299,10 +1331,10 @@ max_allowed_wo(struct drbd_backing_dev *bdev, enum write_ordering_e wo)
 
 	dc = rcu_dereference(bdev->disk_conf);
 
-	if (wo == WO_bdev_flush && !dc->disk_flushes)
-		wo = WO_drain_io;
-	if (wo == WO_drain_io && !dc->disk_drain)
-		wo = WO_none;
+	if (wo == WO_BDEV_FLUSH && !dc->disk_flushes)
+		wo = WO_DRAIN_IO;
+	if (wo == WO_DRAIN_IO && !dc->disk_drain)
+		wo = WO_NONE;
 
 	return wo;
 }
@@ -1319,13 +1351,13 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin
 	enum write_ordering_e pwo;
 	int vnr;
 	static char *write_ordering_str[] = {
-		[WO_none] = "none",
-		[WO_drain_io] = "drain",
-		[WO_bdev_flush] = "flush",
+		[WO_NONE] = "none",
+		[WO_DRAIN_IO] = "drain",
+		[WO_BDEV_FLUSH] = "flush",
 	};
 
 	pwo = resource->write_ordering;
-	if (wo != WO_bdev_flush)
+	if (wo != WO_BDEV_FLUSH)
 		wo = min(pwo, wo);
 	rcu_read_lock();
 	idr_for_each_entry(&resource->devices, device, vnr) {
@@ -1343,7 +1375,7 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin
 	rcu_read_unlock();
 
 	resource->write_ordering = wo;
-	if (pwo != resource->write_ordering || wo == WO_bdev_flush)
+	if (pwo != resource->write_ordering || wo == WO_BDEV_FLUSH)
 		drbd_info(resource, "Method to ensure write ordering: %s\n", write_ordering_str[resource->write_ordering]);
 }
 
@@ -1380,7 +1412,7 @@ int drbd_submit_peer_request(struct drbd_device *device,
 	if (peer_req->flags & EE_IS_TRIM_USE_ZEROOUT) {
 		/* wait for all pending IO completions, before we start
 		 * zeroing things out. */
-		conn_wait_active_ee_empty(first_peer_device(device)->connection);
+		conn_wait_active_ee_empty(peer_req->peer_device->connection);
 		/* add it to the active list now,
 		 * so we can find it to present it in debugfs */
 		peer_req->submit_jif = jiffies;
@@ -1508,12 +1540,6 @@ static void conn_wait_active_ee_empty(struct drbd_connection *connection)
 	rcu_read_unlock();
 }
 
-static struct drbd_peer_device *
-conn_peer_device(struct drbd_connection *connection, int volume_number)
-{
-	return idr_find(&connection->peer_devices, volume_number);
-}
-
 static int receive_Barrier(struct drbd_connection *connection, struct packet_info *pi)
 {
 	int rv;
@@ -1533,7 +1559,7 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
 	 * Therefore we must send the barrier_ack after the barrier request was
 	 * completed. */
 	switch (connection->resource->write_ordering) {
-	case WO_none:
+	case WO_NONE:
 		if (rv == FE_RECYCLED)
 			return 0;
 
@@ -1546,8 +1572,8 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
 			drbd_warn(connection, "Allocation of an epoch failed, slowing down\n");
 			/* Fall through */
 
-	case WO_bdev_flush:
-	case WO_drain_io:
+	case WO_BDEV_FLUSH:
+	case WO_DRAIN_IO:
 		conn_wait_active_ee_empty(connection);
 		drbd_flush(connection);
 
@@ -1752,7 +1778,7 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req
 }
 
 /*
- * e_end_resync_block() is called in asender context via
+ * e_end_resync_block() is called in ack_sender context via
  * drbd_finish_peer_reqs().
  */
 static int e_end_resync_block(struct drbd_work *w, int unused)
@@ -1926,7 +1952,7 @@ static void restart_conflicting_writes(struct drbd_device *device,
 }
 
 /*
- * e_end_block() is called in asender context via drbd_finish_peer_reqs().
+ * e_end_block() is called in ack_sender context via drbd_finish_peer_reqs().
  */
 static int e_end_block(struct drbd_work *w, int cancel)
 {
@@ -1966,7 +1992,7 @@ static int e_end_block(struct drbd_work *w, int cancel)
 	} else
 		D_ASSERT(device, drbd_interval_empty(&peer_req->i));
 
-	drbd_may_finish_epoch(first_peer_device(device)->connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0));
+	drbd_may_finish_epoch(peer_device->connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0));
 
 	return err;
 }
@@ -2098,7 +2124,7 @@ static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, co
 		}
 
 		rcu_read_lock();
-		tp = rcu_dereference(first_peer_device(device)->connection->net_conf)->two_primaries;
+		tp = rcu_dereference(peer_device->connection->net_conf)->two_primaries;
 		rcu_read_unlock();
 
 		if (!tp)
@@ -2217,7 +2243,7 @@ static int handle_write_conflicts(struct drbd_device *device,
 			peer_req->w.cb = superseded ? e_send_superseded :
 						   e_send_retry_write;
 			list_add_tail(&peer_req->w.list, &device->done_ee);
-			wake_asender(connection);
+			queue_work(connection->ack_sender, &peer_req->peer_device->send_acks_work);
 
 			err = -ENOENT;
 			goto out;
@@ -2364,7 +2390,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
 	if (dp_flags & DP_SEND_RECEIVE_ACK) {
 		/* I really don't like it that the receiver thread
 		 * sends on the msock, but anyways */
-		drbd_send_ack(first_peer_device(device), P_RECV_ACK, peer_req);
+		drbd_send_ack(peer_device, P_RECV_ACK, peer_req);
 	}
 
 	if (tp) {
@@ -4056,7 +4082,7 @@ static int receive_state(struct drbd_connection *connection, struct packet_info
 	os = ns = drbd_read_state(device);
 	spin_unlock_irq(&device->resource->req_lock);
 
-	/* If some other part of the code (asender thread, timeout)
+	/* If some other part of the code (ack_receiver thread, timeout)
 	 * already decided to close the connection again,
 	 * we must not "re-establish" it here. */
 	if (os.conn <= C_TEAR_DOWN)
@@ -4661,8 +4687,12 @@ static void conn_disconnect(struct drbd_connection *connection)
 	 */
 	conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
 
-	/* asender does not clean up anything. it must not interfere, either */
-	drbd_thread_stop(&connection->asender);
+	/* ack_receiver does not clean up anything. it must not interfere, either */
+	drbd_thread_stop(&connection->ack_receiver);
+	if (connection->ack_sender) {
+		destroy_workqueue(connection->ack_sender);
+		connection->ack_sender = NULL;
+	}
 	drbd_free_sock(connection);
 
 	rcu_read_lock();
@@ -5431,49 +5461,39 @@ static int got_skip(struct drbd_connection *connection, struct packet_info *pi)
 	return 0;
 }
 
-static int connection_finish_peer_reqs(struct drbd_connection *connection)
+struct meta_sock_cmd {
+	size_t pkt_size;
+	int (*fn)(struct drbd_connection *connection, struct packet_info *);
+};
+
+static void set_rcvtimeo(struct drbd_connection *connection, bool ping_timeout)
 {
-	struct drbd_peer_device *peer_device;
-	int vnr, not_empty = 0;
+	long t;
+	struct net_conf *nc;
 
-	do {
-		clear_bit(SIGNAL_ASENDER, &connection->flags);
-		flush_signals(current);
+	rcu_read_lock();
+	nc = rcu_dereference(connection->net_conf);
+	t = ping_timeout ? nc->ping_timeo : nc->ping_int;
+	rcu_read_unlock();
 
-		rcu_read_lock();
-		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-			struct drbd_device *device = peer_device->device;
-			kref_get(&device->kref);
-			rcu_read_unlock();
-			if (drbd_finish_peer_reqs(device)) {
-				kref_put(&device->kref, drbd_destroy_device);
-				return 1;
-			}
-			kref_put(&device->kref, drbd_destroy_device);
-			rcu_read_lock();
-		}
-		set_bit(SIGNAL_ASENDER, &connection->flags);
+	t *= HZ;
+	if (ping_timeout)
+		t /= 10;
 
-		spin_lock_irq(&connection->resource->req_lock);
-		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-			struct drbd_device *device = peer_device->device;
-			not_empty = !list_empty(&device->done_ee);
-			if (not_empty)
-				break;
-		}
-		spin_unlock_irq(&connection->resource->req_lock);
-		rcu_read_unlock();
-	} while (not_empty);
+	connection->meta.socket->sk->sk_rcvtimeo = t;
+}
 
-	return 0;
+static void set_ping_timeout(struct drbd_connection *connection)
+{
+	set_rcvtimeo(connection, 1);
 }
 
-struct asender_cmd {
-	size_t pkt_size;
-	int (*fn)(struct drbd_connection *connection, struct packet_info *);
-};
+static void set_idle_timeout(struct drbd_connection *connection)
+{
+	set_rcvtimeo(connection, 0);
+}
 
-static struct asender_cmd asender_tbl[] = {
+static struct meta_sock_cmd ack_receiver_tbl[] = {
 	[P_PING]	    = { 0, got_Ping },
 	[P_PING_ACK]	    = { 0, got_PingAck },
 	[P_RECV_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
@@ -5493,64 +5513,40 @@ static struct asender_cmd asender_tbl[] = {
 	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
 };
 
-int drbd_asender(struct drbd_thread *thi)
+int drbd_ack_receiver(struct drbd_thread *thi)
 {
 	struct drbd_connection *connection = thi->connection;
-	struct asender_cmd *cmd = NULL;
+	struct meta_sock_cmd *cmd = NULL;
 	struct packet_info pi;
+	unsigned long pre_recv_jif;
 	int rv;
 	void *buf    = connection->meta.rbuf;
 	int received = 0;
 	unsigned int header_size = drbd_header_size(connection);
 	int expect   = header_size;
 	bool ping_timeout_active = false;
-	struct net_conf *nc;
-	int ping_timeo, tcp_cork, ping_int;
 	struct sched_param param = { .sched_priority = 2 };
 
 	rv = sched_setscheduler(current, SCHED_RR, &param);
 	if (rv < 0)
-		drbd_err(connection, "drbd_asender: ERROR set priority, ret=%d\n", rv);
+		drbd_err(connection, "drbd_ack_receiver: ERROR set priority, ret=%d\n", rv);
 
 	while (get_t_state(thi) == RUNNING) {
 		drbd_thread_current_set_cpu(thi);
 
-		rcu_read_lock();
-		nc = rcu_dereference(connection->net_conf);
-		ping_timeo = nc->ping_timeo;
-		tcp_cork = nc->tcp_cork;
-		ping_int = nc->ping_int;
-		rcu_read_unlock();
+		conn_reclaim_net_peer_reqs(connection);
 
 		if (test_and_clear_bit(SEND_PING, &connection->flags)) {
 			if (drbd_send_ping(connection)) {
 				drbd_err(connection, "drbd_send_ping has failed\n");
 				goto reconnect;
 			}
-			connection->meta.socket->sk->sk_rcvtimeo = ping_timeo * HZ / 10;
+			set_ping_timeout(connection);
 			ping_timeout_active = true;
 		}
 
-		/* TODO: conditionally cork; it may hurt latency if we cork without
-		   much to send */
-		if (tcp_cork)
-			drbd_tcp_cork(connection->meta.socket);
-		if (connection_finish_peer_reqs(connection)) {
-			drbd_err(connection, "connection_finish_peer_reqs() failed\n");
-			goto reconnect;
-		}
-		/* but unconditionally uncork unless disabled */
-		if (tcp_cork)
-			drbd_tcp_uncork(connection->meta.socket);
-
-		/* short circuit, recv_msg would return EINTR anyways. */
-		if (signal_pending(current))
-			continue;
-
+		pre_recv_jif = jiffies;
 		rv = drbd_recv_short(connection->meta.socket, buf, expect-received, 0);
-		clear_bit(SIGNAL_ASENDER, &connection->flags);
-
-		flush_signals(current);
 
 		/* Note:
 		 * -EINTR	 (on meta) we got a signal
@@ -5562,7 +5558,6 @@ int drbd_asender(struct drbd_thread *thi)
 		 * rv <  expected: "woken" by signal during receive
 		 * rv == 0	 : "connection shut down by peer"
 		 */
-received_more:
 		if (likely(rv > 0)) {
 			received += rv;
 			buf	 += rv;
@@ -5584,8 +5579,7 @@ received_more:
 		} else if (rv == -EAGAIN) {
 			/* If the data socket received something meanwhile,
 			 * that is good enough: peer is still alive. */
-			if (time_after(connection->last_received,
-				jiffies - connection->meta.socket->sk->sk_rcvtimeo))
+			if (time_after(connection->last_received, pre_recv_jif))
 				continue;
 			if (ping_timeout_active) {
 				drbd_err(connection, "PingAck did not arrive in time.\n");
@@ -5594,6 +5588,10 @@ received_more:
 			set_bit(SEND_PING, &connection->flags);
 			continue;
 		} else if (rv == -EINTR) {
+			/* maybe drbd_thread_stop(): the while condition will notice.
+			 * maybe woken for send_ping: we'll send a ping above,
+			 * and change the rcvtimeo */
+			flush_signals(current);
 			continue;
 		} else {
 			drbd_err(connection, "sock_recvmsg returned %d\n", rv);
@@ -5603,8 +5601,8 @@ received_more:
 		if (received == expect && cmd == NULL) {
 			if (decode_header(connection, connection->meta.rbuf, &pi))
 				goto reconnect;
-			cmd = &asender_tbl[pi.cmd];
-			if (pi.cmd >= ARRAY_SIZE(asender_tbl) || !cmd->fn) {
+			cmd = &ack_receiver_tbl[pi.cmd];
+			if (pi.cmd >= ARRAY_SIZE(ack_receiver_tbl) || !cmd->fn) {
 				drbd_err(connection, "Unexpected meta packet %s (0x%04x)\n",
 					 cmdname(pi.cmd), pi.cmd);
 				goto disconnect;
@@ -5627,9 +5625,8 @@ received_more:
 
 			connection->last_received = jiffies;
 
-			if (cmd == &asender_tbl[P_PING_ACK]) {
-				/* restore idle timeout */
-				connection->meta.socket->sk->sk_rcvtimeo = ping_int * HZ;
+			if (cmd == &ack_receiver_tbl[P_PING_ACK]) {
+				set_idle_timeout(connection);
 				ping_timeout_active = false;
 			}
 
@@ -5638,11 +5635,6 @@ received_more:
 			expect	 = header_size;
 			cmd	 = NULL;
 		}
-		if (test_bit(SEND_PING, &connection->flags))
-			continue;
-		rv = drbd_recv_short(connection->meta.socket, buf, expect-received, MSG_DONTWAIT);
-		if (rv > 0)
-			goto received_more;
 	}
 
 	if (0) {
@@ -5654,9 +5646,41 @@ reconnect:
 disconnect:
 		conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
 	}
-	clear_bit(SIGNAL_ASENDER, &connection->flags);
 
-	drbd_info(connection, "asender terminated\n");
+	drbd_info(connection, "ack_receiver terminated\n");
 
 	return 0;
 }
+
+void drbd_send_acks_wf(struct work_struct *ws)
+{
+	struct drbd_peer_device *peer_device =
+		container_of(ws, struct drbd_peer_device, send_acks_work);
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct net_conf *nc;
+	int tcp_cork, err;
+
+	rcu_read_lock();
+	nc = rcu_dereference(connection->net_conf);
+	tcp_cork = nc->tcp_cork;
+	rcu_read_unlock();
+
+	if (tcp_cork)
+		drbd_tcp_cork(connection->meta.socket);
+
+	err = drbd_finish_peer_reqs(device);
+	kref_put(&device->kref, drbd_destroy_device);
+	/* get is in drbd_endio_write_sec_final(). That is necessary to keep the
+	   struct work_struct send_acks_work alive, which is in the peer_device object */
+
+	if (err) {
+		conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
+		return;
+	}
+
+	if (tcp_cork)
+		drbd_tcp_uncork(connection->meta.socket);
+
+	return;
+}
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 3ae2c0086563..2255dcfebd2b 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -453,12 +453,12 @@ static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
 		kref_get(&req->kref); /* wait for the DONE */
 
 	if (!(s & RQ_NET_SENT) && (set & RQ_NET_SENT)) {
-		/* potentially already completed in the asender thread */
+		/* potentially already completed in the ack_receiver thread */
 		if (!(s & RQ_NET_DONE)) {
 			atomic_add(req->i.size >> 9, &device->ap_in_flight);
 			set_if_null_req_not_net_done(peer_device, req);
 		}
-		if (s & RQ_NET_PENDING)
+		if (req->rq_state & RQ_NET_PENDING)
 			set_if_null_req_ack_pending(peer_device, req);
 	}
 
@@ -1095,6 +1095,24 @@ static bool do_remote_read(struct drbd_request *req)
 	return false;
 }
 
+bool drbd_should_do_remote(union drbd_dev_state s)
+{
+	return s.pdsk == D_UP_TO_DATE ||
+		(s.pdsk >= D_INCONSISTENT &&
+		 s.conn >= C_WF_BITMAP_T &&
+		 s.conn < C_AHEAD);
+	/* Before proto 96 that was >= CONNECTED instead of >= C_WF_BITMAP_T.
+	   That is equivalent since before 96 IO was frozen in the C_WF_BITMAP*
+	   states. */
+}
+
+static bool drbd_should_send_out_of_sync(union drbd_dev_state s)
+{
+	return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
+	/* pdsk = D_INCONSISTENT as a consequence. Protocol 96 check not necessary
+	   since we enter state C_AHEAD only if proto >= 96 */
+}
+
 /* returns number of connections (== 1, for drbd 8.4)
  * expected to actually write this data,
  * which does NOT include those that we are L_AHEAD for. */
@@ -1149,7 +1167,6 @@ drbd_submit_req_private_bio(struct drbd_request *req)
 	 * stable storage, and this is a WRITE, we may not even submit
 	 * this bio. */
 	if (get_ldev(device)) {
-		req->pre_submit_jif = jiffies;
 		if (drbd_insert_fault(device,
 				      rw == WRITE ? DRBD_FAULT_DT_WR
 				    : rw == READ  ? DRBD_FAULT_DT_RD
@@ -1293,6 +1310,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 			&device->pending_master_completion[rw == WRITE]);
 	if (req->private_bio) {
 		/* needs to be marked within the same spinlock */
+		req->pre_submit_jif = jiffies;
 		list_add_tail(&req->req_pending_local,
 			&device->pending_completion[rw == WRITE]);
 		_req_mod(req, TO_BE_SUBMITTED);
@@ -1513,6 +1531,78 @@ blk_qc_t drbd_make_request(struct request_queue *q, struct bio *bio)
 	return BLK_QC_T_NONE;
 }
 
+static bool net_timeout_reached(struct drbd_request *net_req,
+		struct drbd_connection *connection,
+		unsigned long now, unsigned long ent,
+		unsigned int ko_count, unsigned int timeout)
+{
+	struct drbd_device *device = net_req->device;
+
+	if (!time_after(now, net_req->pre_send_jif + ent))
+		return false;
+
+	if (time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent))
+		return false;
+
+	if (net_req->rq_state & RQ_NET_PENDING) {
+		drbd_warn(device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
+			jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
+		return true;
+	}
+
+	/* We received an ACK already (or are using protocol A),
+	 * but are waiting for the epoch closing barrier ack.
+	 * Check if we sent the barrier already.  We should not blame the peer
+	 * for being unresponsive, if we did not even ask it yet. */
+	if (net_req->epoch == connection->send.current_epoch_nr) {
+		drbd_warn(device,
+			"We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread blocked?\n",
+			jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
+		return false;
+	}
+
+	/* Worst case: we may have been blocked for whatever reason, then
+	 * suddenly are able to send a lot of requests (and epoch separating
+	 * barriers) in quick succession.
+	 * The timestamp of the net_req may be much too old and not correspond
+	 * to the sending time of the relevant unack'ed barrier packet, so
+	 * would trigger a spurious timeout.  The latest barrier packet may
+	 * have a too recent timestamp to trigger the timeout, potentially miss
+	 * a timeout.  Right now we don't have a place to conveniently store
+	 * these timestamps.
+	 * But in this particular situation, the application requests are still
+	 * completed to upper layers, DRBD should still "feel" responsive.
+	 * No need yet to kill this connection, it may still recover.
+	 * If not, eventually we will have queued enough into the network for
+	 * us to block. From that point of view, the timestamp of the last sent
+	 * barrier packet is relevant enough.
+	 */
+	if (time_after(now, connection->send.last_sent_barrier_jif + ent)) {
+		drbd_warn(device, "Remote failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
+			connection->send.last_sent_barrier_jif, now,
+			jiffies_to_msecs(now - connection->send.last_sent_barrier_jif), ko_count, timeout);
+		return true;
+	}
+	return false;
+}
+
+/* A request is considered timed out, if
+ * - we have some effective timeout from the configuration,
+ *   with some state restrictions applied,
+ * - the oldest request is waiting for a response from the network
+ *   resp. the local disk,
+ * - the oldest request is in fact older than the effective timeout,
+ * - the connection was established (resp. disk was attached)
+ *   for longer than the timeout already.
+ * Note that for 32bit jiffies and very stable connections/disks,
+ * we may have a wrap around, which is catched by
+ *   !time_in_range(now, last_..._jif, last_..._jif + timeout).
+ *
+ * Side effect: once per 32bit wrap-around interval, which means every
+ * ~198 days with 250 HZ, we have a window where the timeout would need
+ * to expire twice (worst case) to become effective. Good enough.
+ */
+
 void request_timer_fn(unsigned long data)
 {
 	struct drbd_device *device = (struct drbd_device *) data;
@@ -1522,11 +1612,14 @@ void request_timer_fn(unsigned long data)
 	unsigned long oldest_submit_jif;
 	unsigned long ent = 0, dt = 0, et, nt; /* effective timeout = ko_count * timeout */
 	unsigned long now;
+	unsigned int ko_count = 0, timeout = 0;
 
 	rcu_read_lock();
 	nc = rcu_dereference(connection->net_conf);
-	if (nc && device->state.conn >= C_WF_REPORT_PARAMS)
-		ent = nc->timeout * HZ/10 * nc->ko_count;
+	if (nc && device->state.conn >= C_WF_REPORT_PARAMS) {
+		ko_count = nc->ko_count;
+		timeout = nc->timeout;
+	}
 
 	if (get_ldev(device)) { /* implicit state.disk >= D_INCONSISTENT */
 		dt = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10;
@@ -1534,6 +1627,8 @@ void request_timer_fn(unsigned long data)
 	}
 	rcu_read_unlock();
 
+
+	ent = timeout * HZ/10 * ko_count;
 	et = min_not_zero(dt, ent);
 
 	if (!et)
@@ -1545,11 +1640,22 @@ void request_timer_fn(unsigned long data)
 	spin_lock_irq(&device->resource->req_lock);
 	req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pending_local);
 	req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pending_local);
-	req_peer = connection->req_not_net_done;
+
 	/* maybe the oldest request waiting for the peer is in fact still
-	 * blocking in tcp sendmsg */
-	if (!req_peer && connection->req_next && connection->req_next->pre_send_jif)
-		req_peer = connection->req_next;
+	 * blocking in tcp sendmsg.  That's ok, though, that's handled via the
+	 * socket send timeout, requesting a ping, and bumping ko-count in
+	 * we_should_drop_the_connection().
+	 */
+
+	/* check the oldest request we did successfully sent,
+	 * but which is still waiting for an ACK. */
+	req_peer = connection->req_ack_pending;
+
+	/* if we don't have such request (e.g. protocoll A)
+	 * check the oldest requests which is still waiting on its epoch
+	 * closing barrier ack. */
+	if (!req_peer)
+		req_peer = connection->req_not_net_done;
 
 	/* evaluate the oldest peer request only in one timer! */
 	if (req_peer && req_peer->device != device)
@@ -1566,28 +1672,9 @@ void request_timer_fn(unsigned long data)
 		: req_write ? req_write->pre_submit_jif
 		: req_read ? req_read->pre_submit_jif : now;
 
-	/* The request is considered timed out, if
-	 * - we have some effective timeout from the configuration,
-	 *   with above state restrictions applied,
-	 * - the oldest request is waiting for a response from the network
-	 *   resp. the local disk,
-	 * - the oldest request is in fact older than the effective timeout,
-	 * - the connection was established (resp. disk was attached)
-	 *   for longer than the timeout already.
-	 * Note that for 32bit jiffies and very stable connections/disks,
-	 * we may have a wrap around, which is catched by
-	 *   !time_in_range(now, last_..._jif, last_..._jif + timeout).
-	 *
-	 * Side effect: once per 32bit wrap-around interval, which means every
-	 * ~198 days with 250 HZ, we have a window where the timeout would need
-	 * to expire twice (worst case) to become effective. Good enough.
-	 */
-	if (ent && req_peer &&
-		 time_after(now, req_peer->pre_send_jif + ent) &&
-		!time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent)) {
-		drbd_warn(device, "Remote failed to finish a request within ko-count * timeout\n");
+	if (ent && req_peer && net_timeout_reached(req_peer, connection, now, ent, ko_count, timeout))
 		_conn_request_state(connection, NS(conn, C_TIMEOUT), CS_VERBOSE | CS_HARD);
-	}
+
 	if (dt && oldest_submit_jif != now &&
 		 time_after(now, oldest_submit_jif + dt) &&
 		!time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + dt)) {
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index 9f6a04080e9f..bb2ef78165e5 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -331,21 +331,6 @@ static inline int req_mod(struct drbd_request *req,
 	return rv;
 }
 
-static inline bool drbd_should_do_remote(union drbd_dev_state s)
-{
-	return s.pdsk == D_UP_TO_DATE ||
-		(s.pdsk >= D_INCONSISTENT &&
-		 s.conn >= C_WF_BITMAP_T &&
-		 s.conn < C_AHEAD);
-	/* Before proto 96 that was >= CONNECTED instead of >= C_WF_BITMAP_T.
-	   That is equivalent since before 96 IO was frozen in the C_WF_BITMAP*
-	   states. */
-}
-static inline bool drbd_should_send_out_of_sync(union drbd_dev_state s)
-{
-	return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
-	/* pdsk = D_INCONSISTENT as a consequence. Protocol 96 check not necessary
-	   since we enter state C_AHEAD only if proto >= 96 */
-}
+extern bool drbd_should_do_remote(union drbd_dev_state);
 
 #endif
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 2d7dd269b6a8..5a7ef7873b67 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -29,6 +29,7 @@
 #include "drbd_int.h"
 #include "drbd_protocol.h"
 #include "drbd_req.h"
+#include "drbd_state_change.h"
 
 struct after_state_chg_work {
 	struct drbd_work w;
@@ -37,6 +38,7 @@ struct after_state_chg_work {
 	union drbd_state ns;
 	enum chg_state_flags flags;
 	struct completion *done;
+	struct drbd_state_change *state_change;
 };
 
 enum sanitize_state_warnings {
@@ -48,9 +50,248 @@ enum sanitize_state_warnings {
 	IMPLICITLY_UPGRADED_PDSK,
 };
 
+static void count_objects(struct drbd_resource *resource,
+			  unsigned int *n_devices,
+			  unsigned int *n_connections)
+{
+	struct drbd_device *device;
+	struct drbd_connection *connection;
+	int vnr;
+
+	*n_devices = 0;
+	*n_connections = 0;
+
+	idr_for_each_entry(&resource->devices, device, vnr)
+		(*n_devices)++;
+	for_each_connection(connection, resource)
+		(*n_connections)++;
+}
+
+static struct drbd_state_change *alloc_state_change(unsigned int n_devices, unsigned int n_connections, gfp_t gfp)
+{
+	struct drbd_state_change *state_change;
+	unsigned int size, n;
+
+	size = sizeof(struct drbd_state_change) +
+	       n_devices * sizeof(struct drbd_device_state_change) +
+	       n_connections * sizeof(struct drbd_connection_state_change) +
+	       n_devices * n_connections * sizeof(struct drbd_peer_device_state_change);
+	state_change = kmalloc(size, gfp);
+	if (!state_change)
+		return NULL;
+	state_change->n_devices = n_devices;
+	state_change->n_connections = n_connections;
+	state_change->devices = (void *)(state_change + 1);
+	state_change->connections = (void *)&state_change->devices[n_devices];
+	state_change->peer_devices = (void *)&state_change->connections[n_connections];
+	state_change->resource->resource = NULL;
+	for (n = 0; n < n_devices; n++)
+		state_change->devices[n].device = NULL;
+	for (n = 0; n < n_connections; n++)
+		state_change->connections[n].connection = NULL;
+	return state_change;
+}
+
+struct drbd_state_change *remember_old_state(struct drbd_resource *resource, gfp_t gfp)
+{
+	struct drbd_state_change *state_change;
+	struct drbd_device *device;
+	unsigned int n_devices;
+	struct drbd_connection *connection;
+	unsigned int n_connections;
+	int vnr;
+
+	struct drbd_device_state_change *device_state_change;
+	struct drbd_peer_device_state_change *peer_device_state_change;
+	struct drbd_connection_state_change *connection_state_change;
+
+	/* Caller holds req_lock spinlock.
+	 * No state, no device IDR, no connections lists can change. */
+	count_objects(resource, &n_devices, &n_connections);
+	state_change = alloc_state_change(n_devices, n_connections, gfp);
+	if (!state_change)
+		return NULL;
+
+	kref_get(&resource->kref);
+	state_change->resource->resource = resource;
+	state_change->resource->role[OLD] =
+		conn_highest_role(first_connection(resource));
+	state_change->resource->susp[OLD] = resource->susp;
+	state_change->resource->susp_nod[OLD] = resource->susp_nod;
+	state_change->resource->susp_fen[OLD] = resource->susp_fen;
+
+	connection_state_change = state_change->connections;
+	for_each_connection(connection, resource) {
+		kref_get(&connection->kref);
+		connection_state_change->connection = connection;
+		connection_state_change->cstate[OLD] =
+			connection->cstate;
+		connection_state_change->peer_role[OLD] =
+			conn_highest_peer(connection);
+		connection_state_change++;
+	}
+
+	device_state_change = state_change->devices;
+	peer_device_state_change = state_change->peer_devices;
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		kref_get(&device->kref);
+		device_state_change->device = device;
+		device_state_change->disk_state[OLD] = device->state.disk;
+
+		/* The peer_devices for each device have to be enumerated in
+		   the order of the connections. We may not use for_each_peer_device() here. */
+		for_each_connection(connection, resource) {
+			struct drbd_peer_device *peer_device;
+
+			peer_device = conn_peer_device(connection, device->vnr);
+			peer_device_state_change->peer_device = peer_device;
+			peer_device_state_change->disk_state[OLD] =
+				device->state.pdsk;
+			peer_device_state_change->repl_state[OLD] =
+				max_t(enum drbd_conns,
+				      C_WF_REPORT_PARAMS, device->state.conn);
+			peer_device_state_change->resync_susp_user[OLD] =
+				device->state.user_isp;
+			peer_device_state_change->resync_susp_peer[OLD] =
+				device->state.peer_isp;
+			peer_device_state_change->resync_susp_dependency[OLD] =
+				device->state.aftr_isp;
+			peer_device_state_change++;
+		}
+		device_state_change++;
+	}
+
+	return state_change;
+}
+
+static void remember_new_state(struct drbd_state_change *state_change)
+{
+	struct drbd_resource_state_change *resource_state_change;
+	struct drbd_resource *resource;
+	unsigned int n;
+
+	if (!state_change)
+		return;
+
+	resource_state_change = &state_change->resource[0];
+	resource = resource_state_change->resource;
+
+	resource_state_change->role[NEW] =
+		conn_highest_role(first_connection(resource));
+	resource_state_change->susp[NEW] = resource->susp;
+	resource_state_change->susp_nod[NEW] = resource->susp_nod;
+	resource_state_change->susp_fen[NEW] = resource->susp_fen;
+
+	for (n = 0; n < state_change->n_devices; n++) {
+		struct drbd_device_state_change *device_state_change =
+			&state_change->devices[n];
+		struct drbd_device *device = device_state_change->device;
+
+		device_state_change->disk_state[NEW] = device->state.disk;
+	}
+
+	for (n = 0; n < state_change->n_connections; n++) {
+		struct drbd_connection_state_change *connection_state_change =
+			&state_change->connections[n];
+		struct drbd_connection *connection =
+			connection_state_change->connection;
+
+		connection_state_change->cstate[NEW] = connection->cstate;
+		connection_state_change->peer_role[NEW] =
+			conn_highest_peer(connection);
+	}
+
+	for (n = 0; n < state_change->n_devices * state_change->n_connections; n++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[n];
+		struct drbd_device *device =
+			peer_device_state_change->peer_device->device;
+		union drbd_dev_state state = device->state;
+
+		peer_device_state_change->disk_state[NEW] = state.pdsk;
+		peer_device_state_change->repl_state[NEW] =
+			max_t(enum drbd_conns, C_WF_REPORT_PARAMS, state.conn);
+		peer_device_state_change->resync_susp_user[NEW] =
+			state.user_isp;
+		peer_device_state_change->resync_susp_peer[NEW] =
+			state.peer_isp;
+		peer_device_state_change->resync_susp_dependency[NEW] =
+			state.aftr_isp;
+	}
+}
+
+void copy_old_to_new_state_change(struct drbd_state_change *state_change)
+{
+	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+	unsigned int n_device, n_connection, n_peer_device, n_peer_devices;
+
+#define OLD_TO_NEW(x) \
+	(x[NEW] = x[OLD])
+
+	OLD_TO_NEW(resource_state_change->role);
+	OLD_TO_NEW(resource_state_change->susp);
+	OLD_TO_NEW(resource_state_change->susp_nod);
+	OLD_TO_NEW(resource_state_change->susp_fen);
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+				&state_change->connections[n_connection];
+
+		OLD_TO_NEW(connection_state_change->peer_role);
+		OLD_TO_NEW(connection_state_change->cstate);
+	}
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change =
+			&state_change->devices[n_device];
+
+		OLD_TO_NEW(device_state_change->disk_state);
+	}
+
+	n_peer_devices = state_change->n_devices * state_change->n_connections;
+	for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) {
+		struct drbd_peer_device_state_change *p =
+			&state_change->peer_devices[n_peer_device];
+
+		OLD_TO_NEW(p->disk_state);
+		OLD_TO_NEW(p->repl_state);
+		OLD_TO_NEW(p->resync_susp_user);
+		OLD_TO_NEW(p->resync_susp_peer);
+		OLD_TO_NEW(p->resync_susp_dependency);
+	}
+
+#undef OLD_TO_NEW
+}
+
+void forget_state_change(struct drbd_state_change *state_change)
+{
+	unsigned int n;
+
+	if (!state_change)
+		return;
+
+	if (state_change->resource->resource)
+		kref_put(&state_change->resource->resource->kref, drbd_destroy_resource);
+	for (n = 0; n < state_change->n_devices; n++) {
+		struct drbd_device *device = state_change->devices[n].device;
+
+		if (device)
+			kref_put(&device->kref, drbd_destroy_device);
+	}
+	for (n = 0; n < state_change->n_connections; n++) {
+		struct drbd_connection *connection =
+			state_change->connections[n].connection;
+
+		if (connection)
+			kref_put(&connection->kref, drbd_destroy_connection);
+	}
+	kfree(state_change);
+}
+
 static int w_after_state_ch(struct drbd_work *w, int unused);
 static void after_state_ch(struct drbd_device *device, union drbd_state os,
-			   union drbd_state ns, enum chg_state_flags flags);
+			   union drbd_state ns, enum chg_state_flags flags,
+			   struct drbd_state_change *);
 static enum drbd_state_rv is_valid_state(struct drbd_device *, union drbd_state);
 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_connection *);
 static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
@@ -93,6 +334,7 @@ static enum drbd_role max_role(enum drbd_role role1, enum drbd_role role2)
 		return R_SECONDARY;
 	return R_UNKNOWN;
 }
+
 static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2)
 {
 	if (role1 == R_UNKNOWN || role2 == R_UNKNOWN)
@@ -937,7 +1179,7 @@ void drbd_resume_al(struct drbd_device *device)
 		drbd_info(device, "Resumed AL updates\n");
 }
 
-/* helper for __drbd_set_state */
+/* helper for _drbd_set_state */
 static void set_ov_position(struct drbd_device *device, enum drbd_conns cs)
 {
 	if (first_peer_device(device)->connection->agreed_pro_version < 90)
@@ -965,17 +1207,17 @@ static void set_ov_position(struct drbd_device *device, enum drbd_conns cs)
 }
 
 /**
- * __drbd_set_state() - Set a new DRBD state
+ * _drbd_set_state() - Set a new DRBD state
  * @device:	DRBD device.
  * @ns:		new state.
  * @flags:	Flags
  * @done:	Optional completion, that will get completed after the after_state_ch() finished
  *
- * Caller needs to hold req_lock, and global_state_lock. Do not call directly.
+ * Caller needs to hold req_lock. Do not call directly.
  */
 enum drbd_state_rv
-__drbd_set_state(struct drbd_device *device, union drbd_state ns,
-	         enum chg_state_flags flags, struct completion *done)
+_drbd_set_state(struct drbd_device *device, union drbd_state ns,
+	        enum chg_state_flags flags, struct completion *done)
 {
 	struct drbd_peer_device *peer_device = first_peer_device(device);
 	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
@@ -983,6 +1225,7 @@ __drbd_set_state(struct drbd_device *device, union drbd_state ns,
 	enum drbd_state_rv rv = SS_SUCCESS;
 	enum sanitize_state_warnings ssw;
 	struct after_state_chg_work *ascw;
+	struct drbd_state_change *state_change;
 
 	os = drbd_read_state(device);
 
@@ -1037,6 +1280,9 @@ __drbd_set_state(struct drbd_device *device, union drbd_state ns,
 	if (!is_sync_state(os.conn) && is_sync_state(ns.conn))
 		clear_bit(RS_DONE, &device->flags);
 
+	/* FIXME: Have any flags been set earlier in this function already? */
+	state_change = remember_old_state(device->resource, GFP_ATOMIC);
+
 	/* changes to local_cnt and device flags should be visible before
 	 * changes to state, which again should be visible before anything else
 	 * depending on that change happens. */
@@ -1047,6 +1293,8 @@ __drbd_set_state(struct drbd_device *device, union drbd_state ns,
 	device->resource->susp_fen = ns.susp_fen;
 	smp_wmb();
 
+	remember_new_state(state_change);
+
 	/* put replicated vs not-replicated requests in seperate epochs */
 	if (drbd_should_do_remote((union drbd_dev_state)os.i) !=
 	    drbd_should_do_remote((union drbd_dev_state)ns.i))
@@ -1184,6 +1432,7 @@ __drbd_set_state(struct drbd_device *device, union drbd_state ns,
 		ascw->w.cb = w_after_state_ch;
 		ascw->device = device;
 		ascw->done = done;
+		ascw->state_change = state_change;
 		drbd_queue_work(&connection->sender_work,
 				&ascw->w);
 	} else {
@@ -1199,7 +1448,8 @@ static int w_after_state_ch(struct drbd_work *w, int unused)
 		container_of(w, struct after_state_chg_work, w);
 	struct drbd_device *device = ascw->device;
 
-	after_state_ch(device, ascw->os, ascw->ns, ascw->flags);
+	after_state_ch(device, ascw->os, ascw->ns, ascw->flags, ascw->state_change);
+	forget_state_change(ascw->state_change);
 	if (ascw->flags & CS_WAIT_COMPLETE)
 		complete(ascw->done);
 	kfree(ascw);
@@ -1234,7 +1484,7 @@ int drbd_bitmap_io_from_worker(struct drbd_device *device,
 	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
 
 	/* open coded non-blocking drbd_suspend_io(device); */
-	set_bit(SUSPEND_IO, &device->flags);
+	atomic_inc(&device->suspend_cnt);
 
 	drbd_bm_lock(device, why, flags);
 	rv = io_fn(device);
@@ -1245,6 +1495,139 @@ int drbd_bitmap_io_from_worker(struct drbd_device *device,
 	return rv;
 }
 
+void notify_resource_state_change(struct sk_buff *skb,
+				  unsigned int seq,
+				  struct drbd_resource_state_change *resource_state_change,
+				  enum drbd_notification_type type)
+{
+	struct drbd_resource *resource = resource_state_change->resource;
+	struct resource_info resource_info = {
+		.res_role = resource_state_change->role[NEW],
+		.res_susp = resource_state_change->susp[NEW],
+		.res_susp_nod = resource_state_change->susp_nod[NEW],
+		.res_susp_fen = resource_state_change->susp_fen[NEW],
+	};
+
+	notify_resource_state(skb, seq, resource, &resource_info, type);
+}
+
+void notify_connection_state_change(struct sk_buff *skb,
+				    unsigned int seq,
+				    struct drbd_connection_state_change *connection_state_change,
+				    enum drbd_notification_type type)
+{
+	struct drbd_connection *connection = connection_state_change->connection;
+	struct connection_info connection_info = {
+		.conn_connection_state = connection_state_change->cstate[NEW],
+		.conn_role = connection_state_change->peer_role[NEW],
+	};
+
+	notify_connection_state(skb, seq, connection, &connection_info, type);
+}
+
+void notify_device_state_change(struct sk_buff *skb,
+				unsigned int seq,
+				struct drbd_device_state_change *device_state_change,
+				enum drbd_notification_type type)
+{
+	struct drbd_device *device = device_state_change->device;
+	struct device_info device_info = {
+		.dev_disk_state = device_state_change->disk_state[NEW],
+	};
+
+	notify_device_state(skb, seq, device, &device_info, type);
+}
+
+void notify_peer_device_state_change(struct sk_buff *skb,
+				     unsigned int seq,
+				     struct drbd_peer_device_state_change *p,
+				     enum drbd_notification_type type)
+{
+	struct drbd_peer_device *peer_device = p->peer_device;
+	struct peer_device_info peer_device_info = {
+		.peer_repl_state = p->repl_state[NEW],
+		.peer_disk_state = p->disk_state[NEW],
+		.peer_resync_susp_user = p->resync_susp_user[NEW],
+		.peer_resync_susp_peer = p->resync_susp_peer[NEW],
+		.peer_resync_susp_dependency = p->resync_susp_dependency[NEW],
+	};
+
+	notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type);
+}
+
+static void broadcast_state_change(struct drbd_state_change *state_change)
+{
+	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+	bool resource_state_has_changed;
+	unsigned int n_device, n_connection, n_peer_device, n_peer_devices;
+	void (*last_func)(struct sk_buff *, unsigned int, void *,
+			  enum drbd_notification_type) = NULL;
+	void *uninitialized_var(last_arg);
+
+#define HAS_CHANGED(state) ((state)[OLD] != (state)[NEW])
+#define FINAL_STATE_CHANGE(type) \
+	({ if (last_func) \
+		last_func(NULL, 0, last_arg, type); \
+	})
+#define REMEMBER_STATE_CHANGE(func, arg, type) \
+	({ FINAL_STATE_CHANGE(type | NOTIFY_CONTINUES); \
+	   last_func = (typeof(last_func))func; \
+	   last_arg = arg; \
+	 })
+
+	mutex_lock(&notification_mutex);
+
+	resource_state_has_changed =
+	    HAS_CHANGED(resource_state_change->role) ||
+	    HAS_CHANGED(resource_state_change->susp) ||
+	    HAS_CHANGED(resource_state_change->susp_nod) ||
+	    HAS_CHANGED(resource_state_change->susp_fen);
+
+	if (resource_state_has_changed)
+		REMEMBER_STATE_CHANGE(notify_resource_state_change,
+				      resource_state_change, NOTIFY_CHANGE);
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+				&state_change->connections[n_connection];
+
+		if (HAS_CHANGED(connection_state_change->peer_role) ||
+		    HAS_CHANGED(connection_state_change->cstate))
+			REMEMBER_STATE_CHANGE(notify_connection_state_change,
+					      connection_state_change, NOTIFY_CHANGE);
+	}
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change =
+			&state_change->devices[n_device];
+
+		if (HAS_CHANGED(device_state_change->disk_state))
+			REMEMBER_STATE_CHANGE(notify_device_state_change,
+					      device_state_change, NOTIFY_CHANGE);
+	}
+
+	n_peer_devices = state_change->n_devices * state_change->n_connections;
+	for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) {
+		struct drbd_peer_device_state_change *p =
+			&state_change->peer_devices[n_peer_device];
+
+		if (HAS_CHANGED(p->disk_state) ||
+		    HAS_CHANGED(p->repl_state) ||
+		    HAS_CHANGED(p->resync_susp_user) ||
+		    HAS_CHANGED(p->resync_susp_peer) ||
+		    HAS_CHANGED(p->resync_susp_dependency))
+			REMEMBER_STATE_CHANGE(notify_peer_device_state_change,
+					      p, NOTIFY_CHANGE);
+	}
+
+	FINAL_STATE_CHANGE(NOTIFY_CHANGE);
+	mutex_unlock(&notification_mutex);
+
+#undef HAS_CHANGED
+#undef FINAL_STATE_CHANGE
+#undef REMEMBER_STATE_CHANGE
+}
+
 /**
  * after_state_ch() - Perform after state change actions that may sleep
  * @device:	DRBD device.
@@ -1253,13 +1636,16 @@ int drbd_bitmap_io_from_worker(struct drbd_device *device,
  * @flags:	Flags
  */
 static void after_state_ch(struct drbd_device *device, union drbd_state os,
-			   union drbd_state ns, enum chg_state_flags flags)
+			   union drbd_state ns, enum chg_state_flags flags,
+			   struct drbd_state_change *state_change)
 {
 	struct drbd_resource *resource = device->resource;
 	struct drbd_peer_device *peer_device = first_peer_device(device);
 	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
 	struct sib_info sib;
 
+	broadcast_state_change(state_change);
+
 	sib.sib_reason = SIB_STATE_CHANGE;
 	sib.os = os;
 	sib.ns = ns;
@@ -1377,7 +1763,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 	}
 
 	if (ns.pdsk < D_INCONSISTENT && get_ldev(device)) {
-		if (os.peer == R_SECONDARY && ns.peer == R_PRIMARY &&
+		if (os.peer != R_PRIMARY && ns.peer == R_PRIMARY &&
 		    device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
 			drbd_uuid_new_current(device);
 			drbd_send_uuids(peer_device);
@@ -1444,7 +1830,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 	if (os.disk != D_FAILED && ns.disk == D_FAILED) {
 		enum drbd_io_error_p eh = EP_PASS_ON;
 		int was_io_error = 0;
-		/* corresponding get_ldev was in __drbd_set_state, to serialize
+		/* corresponding get_ldev was in _drbd_set_state, to serialize
 		 * our cleanup here with the transition to D_DISKLESS.
 		 * But is is still not save to dreference ldev here, since
 		 * we might come from an failed Attach before ldev was set. */
@@ -1455,6 +1841,10 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 
 			was_io_error = test_and_clear_bit(WAS_IO_ERROR, &device->flags);
 
+			/* Intentionally call this handler first, before drbd_send_state().
+			 * See: 2932204 drbd: call local-io-error handler early
+			 * People may chose to hard-reset the box from this handler.
+			 * It is useful if this looks like a "regular node crash". */
 			if (was_io_error && eh == EP_CALL_HELPER)
 				drbd_khelper(device, "local-io-error");
 
@@ -1572,6 +1962,7 @@ struct after_conn_state_chg_work {
 	union drbd_state ns_max; /* new, max state, over all devices */
 	enum chg_state_flags flags;
 	struct drbd_connection *connection;
+	struct drbd_state_change *state_change;
 };
 
 static int w_after_conn_state_ch(struct drbd_work *w, int unused)
@@ -1584,6 +1975,8 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 	struct drbd_peer_device *peer_device;
 	int vnr;
 
+	broadcast_state_change(acscw->state_change);
+	forget_state_change(acscw->state_change);
 	kfree(acscw);
 
 	/* Upon network configuration, we need to start the receiver */
@@ -1593,6 +1986,13 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 	if (oc == C_DISCONNECTING && ns_max.conn == C_STANDALONE) {
 		struct net_conf *old_conf;
 
+		mutex_lock(&notification_mutex);
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+			notify_peer_device_state(NULL, 0, peer_device, NULL,
+						 NOTIFY_DESTROY | NOTIFY_CONTINUES);
+		notify_connection_state(NULL, 0, connection, NULL, NOTIFY_DESTROY);
+		mutex_unlock(&notification_mutex);
+
 		mutex_lock(&connection->resource->conf_update);
 		old_conf = connection->net_conf;
 		connection->my_addr_len = 0;
@@ -1759,7 +2159,7 @@ conn_set_state(struct drbd_connection *connection, union drbd_state mask, union
 		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
 			ns.disk = os.disk;
 
-		rv = __drbd_set_state(device, ns, flags, NULL);
+		rv = _drbd_set_state(device, ns, flags, NULL);
 		if (rv < SS_SUCCESS)
 			BUG();
 
@@ -1823,6 +2223,7 @@ _conn_request_state(struct drbd_connection *connection, union drbd_state mask, u
 	enum drbd_conns oc = connection->cstate;
 	union drbd_state ns_max, ns_min, os;
 	bool have_mutex = false;
+	struct drbd_state_change *state_change;
 
 	if (mask.conn) {
 		rv = is_valid_conn_transition(oc, val.conn);
@@ -1868,10 +2269,12 @@ _conn_request_state(struct drbd_connection *connection, union drbd_state mask, u
 			goto abort;
 	}
 
+	state_change = remember_old_state(connection->resource, GFP_ATOMIC);
 	conn_old_common_state(connection, &os, &flags);
 	flags |= CS_DC_SUSP;
 	conn_set_state(connection, mask, val, &ns_min, &ns_max, flags);
 	conn_pr_state_change(connection, os, ns_max, flags);
+	remember_new_state(state_change);
 
 	acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC);
 	if (acscw) {
@@ -1882,6 +2285,7 @@ _conn_request_state(struct drbd_connection *connection, union drbd_state mask, u
 		acscw->w.cb = w_after_conn_state_ch;
 		kref_get(&connection->kref);
 		acscw->connection = connection;
+		acscw->state_change = state_change;
 		drbd_queue_work(&connection->sender_work, &acscw->w);
 	} else {
 		drbd_err(connection, "Could not kmalloc an acscw\n");
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 7f53c40823cd..bd989536f888 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -122,9 +122,9 @@ extern enum drbd_state_rv
 _drbd_request_state_holding_state_mutex(struct drbd_device *, union drbd_state,
 					union drbd_state, enum chg_state_flags);
 
-extern enum drbd_state_rv __drbd_set_state(struct drbd_device *, union drbd_state,
-					   enum chg_state_flags,
-					   struct completion *done);
+extern enum drbd_state_rv _drbd_set_state(struct drbd_device *, union drbd_state,
+					  enum chg_state_flags,
+					  struct completion *done);
 extern void print_st_err(struct drbd_device *, union drbd_state,
 			union drbd_state, int);
 
diff --git a/drivers/block/drbd/drbd_state_change.h b/drivers/block/drbd/drbd_state_change.h
new file mode 100644
index 000000000000..9e503a1a0bfb
--- /dev/null
+++ b/drivers/block/drbd/drbd_state_change.h
@@ -0,0 +1,63 @@
+#ifndef DRBD_STATE_CHANGE_H
+#define DRBD_STATE_CHANGE_H
+
+struct drbd_resource_state_change {
+	struct drbd_resource *resource;
+	enum drbd_role role[2];
+	bool susp[2];
+	bool susp_nod[2];
+	bool susp_fen[2];
+};
+
+struct drbd_device_state_change {
+	struct drbd_device *device;
+	enum drbd_disk_state disk_state[2];
+};
+
+struct drbd_connection_state_change {
+	struct drbd_connection *connection;
+	enum drbd_conns cstate[2];  /* drbd9: enum drbd_conn_state */
+	enum drbd_role peer_role[2];
+};
+
+struct drbd_peer_device_state_change {
+	struct drbd_peer_device *peer_device;
+	enum drbd_disk_state disk_state[2];
+	enum drbd_conns repl_state[2];  /* drbd9: enum drbd_repl_state */
+	bool resync_susp_user[2];
+	bool resync_susp_peer[2];
+	bool resync_susp_dependency[2];
+};
+
+struct drbd_state_change {
+	struct list_head list;
+	unsigned int n_devices;
+	unsigned int n_connections;
+	struct drbd_resource_state_change resource[1];
+	struct drbd_device_state_change *devices;
+	struct drbd_connection_state_change *connections;
+	struct drbd_peer_device_state_change *peer_devices;
+};
+
+extern struct drbd_state_change *remember_old_state(struct drbd_resource *, gfp_t);
+extern void copy_old_to_new_state_change(struct drbd_state_change *);
+extern void forget_state_change(struct drbd_state_change *);
+
+extern void notify_resource_state_change(struct sk_buff *,
+					 unsigned int,
+					 struct drbd_resource_state_change *,
+					 enum drbd_notification_type type);
+extern void notify_connection_state_change(struct sk_buff *,
+					   unsigned int,
+					   struct drbd_connection_state_change *,
+					   enum drbd_notification_type type);
+extern void notify_device_state_change(struct sk_buff *,
+				       unsigned int,
+				       struct drbd_device_state_change *,
+				       enum drbd_notification_type type);
+extern void notify_peer_device_state_change(struct sk_buff *,
+					    unsigned int,
+					    struct drbd_peer_device_state_change *,
+					    enum drbd_notification_type type);
+
+#endif  /* DRBD_STATE_CHANGE_H */
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 5578c1477ba6..eff716c27b1f 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -55,13 +55,6 @@ static int make_resync_request(struct drbd_device *, int);
  *
  */
 
-
-/* About the global_state_lock
-   Each state transition on an device holds a read lock. In case we have
-   to evaluate the resync after dependencies, we grab a write lock, because
-   we need stable states on all devices for that.  */
-rwlock_t global_state_lock;
-
 /* used for synchronous meta data and bitmap IO
  * submitted by drbd_md_sync_page_io()
  */
@@ -120,6 +113,7 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
 	unsigned long flags = 0;
 	struct drbd_peer_device *peer_device = peer_req->peer_device;
 	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
 	struct drbd_interval i;
 	int do_wake;
 	u64 block_id;
@@ -152,6 +146,12 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
 	 * ((peer_req->flags & (EE_WAS_ERROR|EE_IS_TRIM)) == EE_WAS_ERROR) */
 	if (peer_req->flags & EE_WAS_ERROR)
 		__drbd_chk_io_error(device, DRBD_WRITE_ERROR);
+
+	if (connection->cstate >= C_WF_REPORT_PARAMS) {
+		kref_get(&device->kref); /* put is in drbd_send_acks_wf() */
+		if (!queue_work(connection->ack_sender, &peer_device->send_acks_work))
+			kref_put(&device->kref, drbd_destroy_device);
+	}
 	spin_unlock_irqrestore(&device->resource->req_lock, flags);
 
 	if (block_id == ID_SYNCER)
@@ -163,7 +163,6 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
 	if (do_al_complete_io)
 		drbd_al_complete_io(device, &i);
 
-	wake_asender(peer_device->connection);
 	put_ldev(device);
 }
 
@@ -195,6 +194,12 @@ void drbd_peer_request_endio(struct bio *bio)
 	}
 }
 
+void drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
+{
+	panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n",
+		device->minor, device->resource->name, device->vnr);
+}
+
 /* read, readA or write requests on R_PRIMARY coming from drbd_make_request
  */
 void drbd_request_endio(struct bio *bio)
@@ -238,7 +243,7 @@ void drbd_request_endio(struct bio *bio)
 			drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n");
 
 		if (!bio->bi_error)
-			panic("possible random memory corruption caused by delayed completion of aborted local request\n");
+			drbd_panic_after_delayed_completion_of_aborted_request(device);
 	}
 
 	/* to avoid recursion in __req_mod */
@@ -1291,6 +1296,7 @@ static int drbd_send_barrier(struct drbd_connection *connection)
 	p->barrier = connection->send.current_epoch_nr;
 	p->pad = 0;
 	connection->send.current_epoch_writes = 0;
+	connection->send.last_sent_barrier_jif = jiffies;
 
 	return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0);
 }
@@ -1315,6 +1321,7 @@ static void re_init_if_first_write(struct drbd_connection *connection, unsigned
 		connection->send.seen_any_write_yet = true;
 		connection->send.current_epoch_nr = epoch;
 		connection->send.current_epoch_writes = 0;
+		connection->send.last_sent_barrier_jif = jiffies;
 	}
 }
 
@@ -1456,70 +1463,73 @@ static int _drbd_may_sync_now(struct drbd_device *device)
 }
 
 /**
- * _drbd_pause_after() - Pause resync on all devices that may not resync now
+ * drbd_pause_after() - Pause resync on all devices that may not resync now
  * @device:	DRBD device.
  *
  * Called from process context only (admin command and after_state_ch).
  */
-static int _drbd_pause_after(struct drbd_device *device)
+static bool drbd_pause_after(struct drbd_device *device)
 {
+	bool changed = false;
 	struct drbd_device *odev;
-	int i, rv = 0;
+	int i;
 
 	rcu_read_lock();
 	idr_for_each_entry(&drbd_devices, odev, i) {
 		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
 			continue;
-		if (!_drbd_may_sync_now(odev))
-			rv |= (__drbd_set_state(_NS(odev, aftr_isp, 1), CS_HARD, NULL)
-			       != SS_NOTHING_TO_DO);
+		if (!_drbd_may_sync_now(odev) &&
+		    _drbd_set_state(_NS(odev, aftr_isp, 1),
+				    CS_HARD, NULL) != SS_NOTHING_TO_DO)
+			changed = true;
 	}
 	rcu_read_unlock();
 
-	return rv;
+	return changed;
 }
 
 /**
- * _drbd_resume_next() - Resume resync on all devices that may resync now
+ * drbd_resume_next() - Resume resync on all devices that may resync now
  * @device:	DRBD device.
  *
  * Called from process context only (admin command and worker).
  */
-static int _drbd_resume_next(struct drbd_device *device)
+static bool drbd_resume_next(struct drbd_device *device)
 {
+	bool changed = false;
 	struct drbd_device *odev;
-	int i, rv = 0;
+	int i;
 
 	rcu_read_lock();
 	idr_for_each_entry(&drbd_devices, odev, i) {
 		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
 			continue;
 		if (odev->state.aftr_isp) {
-			if (_drbd_may_sync_now(odev))
-				rv |= (__drbd_set_state(_NS(odev, aftr_isp, 0),
-							CS_HARD, NULL)
-				       != SS_NOTHING_TO_DO) ;
+			if (_drbd_may_sync_now(odev) &&
+			    _drbd_set_state(_NS(odev, aftr_isp, 0),
+					    CS_HARD, NULL) != SS_NOTHING_TO_DO)
+				changed = true;
 		}
 	}
 	rcu_read_unlock();
-	return rv;
+	return changed;
 }
 
 void resume_next_sg(struct drbd_device *device)
 {
-	write_lock_irq(&global_state_lock);
-	_drbd_resume_next(device);
-	write_unlock_irq(&global_state_lock);
+	lock_all_resources();
+	drbd_resume_next(device);
+	unlock_all_resources();
 }
 
 void suspend_other_sg(struct drbd_device *device)
 {
-	write_lock_irq(&global_state_lock);
-	_drbd_pause_after(device);
-	write_unlock_irq(&global_state_lock);
+	lock_all_resources();
+	drbd_pause_after(device);
+	unlock_all_resources();
 }
 
-/* caller must hold global_state_lock */
+/* caller must lock_all_resources() */
 enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor)
 {
 	struct drbd_device *odev;
@@ -1557,15 +1567,15 @@ enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_min
 	}
 }
 
-/* caller must hold global_state_lock */
+/* caller must lock_all_resources() */
 void drbd_resync_after_changed(struct drbd_device *device)
 {
-	int changes;
+	int changed;
 
 	do {
-		changes  = _drbd_pause_after(device);
-		changes |= _drbd_resume_next(device);
-	} while (changes);
+		changed  = drbd_pause_after(device);
+		changed |= drbd_resume_next(device);
+	} while (changed);
 }
 
 void drbd_rs_controller_reset(struct drbd_device *device)
@@ -1685,19 +1695,14 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 	} else {
 		mutex_lock(device->state_mutex);
 	}
-	clear_bit(B_RS_H_DONE, &device->flags);
 
-	/* req_lock: serialize with drbd_send_and_submit() and others
-	 * global_state_lock: for stable sync-after dependencies */
-	spin_lock_irq(&device->resource->req_lock);
-	write_lock(&global_state_lock);
+	lock_all_resources();
+	clear_bit(B_RS_H_DONE, &device->flags);
 	/* Did some connection breakage or IO error race with us? */
 	if (device->state.conn < C_CONNECTED
 	|| !get_ldev_if_state(device, D_NEGOTIATING)) {
-		write_unlock(&global_state_lock);
-		spin_unlock_irq(&device->resource->req_lock);
-		mutex_unlock(device->state_mutex);
-		return;
+		unlock_all_resources();
+		goto out;
 	}
 
 	ns = drbd_read_state(device);
@@ -1711,7 +1716,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 	else /* side == C_SYNC_SOURCE */
 		ns.pdsk = D_INCONSISTENT;
 
-	r = __drbd_set_state(device, ns, CS_VERBOSE, NULL);
+	r = _drbd_set_state(device, ns, CS_VERBOSE, NULL);
 	ns = drbd_read_state(device);
 
 	if (ns.conn < C_CONNECTED)
@@ -1732,7 +1737,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 			device->rs_mark_left[i] = tw;
 			device->rs_mark_time[i] = now;
 		}
-		_drbd_pause_after(device);
+		drbd_pause_after(device);
 		/* Forget potentially stale cached per resync extent bit-counts.
 		 * Open coded drbd_rs_cancel_all(device), we already have IRQs
 		 * disabled, and know the disk state is ok. */
@@ -1742,8 +1747,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 		device->resync_wenr = LC_FREE;
 		spin_unlock(&device->al_lock);
 	}
-	write_unlock(&global_state_lock);
-	spin_unlock_irq(&device->resource->req_lock);
+	unlock_all_resources();
 
 	if (r == SS_SUCCESS) {
 		wake_up(&device->al_wait); /* for lc_reset() above */
@@ -1807,6 +1811,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 		drbd_md_sync(device);
 	}
 	put_ldev(device);
+out:
 	mutex_unlock(device->state_mutex);
 }
 
@@ -1836,7 +1841,7 @@ static void drbd_ldev_destroy(struct drbd_device *device)
 	device->act_log = NULL;
 
 	__acquire(local);
-	drbd_free_ldev(device->ldev);
+	drbd_backing_dev_free(device, device->ldev);
 	device->ldev = NULL;
 	__release(local);
 
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index 331363e7de0f..9e251201dd48 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -3585,7 +3585,7 @@ static void __init config_types(void)
 		unsigned int type = UDP->cmos;
 		struct floppy_drive_params *params;
 		const char *name = NULL;
-		static char temparea[32];
+		char temparea[32];
 
 		if (type < ARRAY_SIZE(default_drive_params)) {
 			params = &default_drive_params[type].params;
@@ -3596,7 +3596,8 @@ static void __init config_types(void)
 				allowed_drive_mask &= ~(1 << drive);
 		} else {
 			params = &default_drive_params[0].params;
-			sprintf(temparea, "unknown type %d (usb?)", type);
+			snprintf(temparea, sizeof(temparea),
+				 "unknown type %d (usb?)", type);
 			name = temparea;
 		}
 		if (name) {
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index 34997d8ecd64..9b180dbbd03c 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -104,9 +104,9 @@
 /* Device instance number, incremented each time a device is probed. */
 static int instance;
 
-struct list_head online_list;
-struct list_head removing_list;
-spinlock_t dev_lock;
+static struct list_head online_list;
+static struct list_head removing_list;
+static spinlock_t dev_lock;
 
 /*
  * Global variable used to hold the major block device number
@@ -173,7 +173,7 @@ static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd)
 {
 	struct request *rq;
 
-	rq = blk_mq_alloc_request(dd->queue, 0, __GFP_RECLAIM, true);
+	rq = blk_mq_alloc_request(dd->queue, 0, BLK_MQ_REQ_RESERVED);
 	return blk_mq_rq_to_pdu(rq);
 }
 
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index 09e3c0d87ecc..8ba1e97d573c 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -436,9 +436,8 @@ static void null_del_dev(struct nullb *nullb)
 static void null_lnvm_end_io(struct request *rq, int error)
 {
 	struct nvm_rq *rqd = rq->end_io_data;
-	struct nvm_dev *dev = rqd->dev;
 
-	dev->mt->end_io(rqd, error);
+	nvm_end_io(rqd, error);
 
 	blk_put_request(rq);
 }
@@ -449,7 +448,7 @@ static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 	struct request *rq;
 	struct bio *bio = rqd->bio;
 
-	rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0);
+	rq = blk_mq_alloc_request(q, bio_rw(bio), 0);
 	if (IS_ERR(rq))
 		return -ENOMEM;
 
@@ -495,17 +494,17 @@ static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 	id->ppaf.ch_offset = 56;
 	id->ppaf.ch_len = 8;
 
-	do_div(size, bs); /* convert size to pages */
-	do_div(size, 256); /* concert size to pgs pr blk */
+	sector_div(size, bs); /* convert size to pages */
+	size >>= 8; /* concert size to pgs pr blk */
 	grp = &id->groups[0];
 	grp->mtype = 0;
 	grp->fmtype = 0;
 	grp->num_ch = 1;
 	grp->num_pg = 256;
 	blksize = size;
-	do_div(size, (1 << 16));
+	size >>= 16;
 	grp->num_lun = size + 1;
-	do_div(blksize, grp->num_lun);
+	sector_div(blksize, grp->num_lun);
 	grp->num_blk = blksize;
 	grp->num_pln = 1;
 
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 81ea69fee7ca..4a876785b68c 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -5185,8 +5185,7 @@ static int rbd_dev_probe_parent(struct rbd_device *rbd_dev, int depth)
 
 out_err:
 	rbd_dev_unparent(rbd_dev);
-	if (parent)
-		rbd_dev_destroy(parent);
+	rbd_dev_destroy(parent);
 	return ret;
 }
 
diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c
index 59c91d49b14b..ba4bfe933276 100644
--- a/drivers/block/sx8.c
+++ b/drivers/block/sx8.c
@@ -23,7 +23,7 @@
 #include <linux/workqueue.h>
 #include <linux/bitops.h>
 #include <linux/delay.h>
-#include <linux/time.h>
+#include <linux/ktime.h>
 #include <linux/hdreg.h>
 #include <linux/dma-mapping.h>
 #include <linux/completion.h>
@@ -671,16 +671,15 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func)
 static unsigned int carm_fill_sync_time(struct carm_host *host,
 					unsigned int idx, void *mem)
 {
-	struct timeval tv;
 	struct carm_msg_sync_time *st = mem;
 
-	do_gettimeofday(&tv);
+	time64_t tv = ktime_get_real_seconds();
 
 	memset(st, 0, sizeof(*st));
 	st->type	= CARM_MSG_MISC;
 	st->subtype	= MISC_SET_TIME;
 	st->handle	= cpu_to_le32(TAG_ENCODE(idx));
-	st->timestamp	= cpu_to_le32(tv.tv_sec);
+	st->timestamp	= cpu_to_le32(tv);
 
 	return sizeof(struct carm_msg_sync_time);
 }
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index 41fb1a917b17..4809c1501d7e 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -84,6 +84,16 @@ MODULE_PARM_DESC(max_persistent_grants,
                  "Maximum number of grants to map persistently");
 
 /*
+ * Maximum number of rings/queues blkback supports, allow as many queues as there
+ * are CPUs if user has not specified a value.
+ */
+unsigned int xenblk_max_queues;
+module_param_named(max_queues, xenblk_max_queues, uint, 0644);
+MODULE_PARM_DESC(max_queues,
+		 "Maximum number of hardware queues per virtual disk." \
+		 "By default it is the number of online CPUs.");
+
+/*
  * Maximum order of pages to be used for the shared ring between front and
  * backend, 4KB page granularity is used.
  */
@@ -113,71 +123,71 @@ module_param(log_stats, int, 0644);
 /* Number of free pages to remove on each call to gnttab_free_pages */
 #define NUM_BATCH_FREE_PAGES 10
 
-static inline int get_free_page(struct xen_blkif *blkif, struct page **page)
+static inline int get_free_page(struct xen_blkif_ring *ring, struct page **page)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&blkif->free_pages_lock, flags);
-	if (list_empty(&blkif->free_pages)) {
-		BUG_ON(blkif->free_pages_num != 0);
-		spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
+	spin_lock_irqsave(&ring->free_pages_lock, flags);
+	if (list_empty(&ring->free_pages)) {
+		BUG_ON(ring->free_pages_num != 0);
+		spin_unlock_irqrestore(&ring->free_pages_lock, flags);
 		return gnttab_alloc_pages(1, page);
 	}
-	BUG_ON(blkif->free_pages_num == 0);
-	page[0] = list_first_entry(&blkif->free_pages, struct page, lru);
+	BUG_ON(ring->free_pages_num == 0);
+	page[0] = list_first_entry(&ring->free_pages, struct page, lru);
 	list_del(&page[0]->lru);
-	blkif->free_pages_num--;
-	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
+	ring->free_pages_num--;
+	spin_unlock_irqrestore(&ring->free_pages_lock, flags);
 
 	return 0;
 }
 
-static inline void put_free_pages(struct xen_blkif *blkif, struct page **page,
+static inline void put_free_pages(struct xen_blkif_ring *ring, struct page **page,
                                   int num)
 {
 	unsigned long flags;
 	int i;
 
-	spin_lock_irqsave(&blkif->free_pages_lock, flags);
+	spin_lock_irqsave(&ring->free_pages_lock, flags);
 	for (i = 0; i < num; i++)
-		list_add(&page[i]->lru, &blkif->free_pages);
-	blkif->free_pages_num += num;
-	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
+		list_add(&page[i]->lru, &ring->free_pages);
+	ring->free_pages_num += num;
+	spin_unlock_irqrestore(&ring->free_pages_lock, flags);
 }
 
-static inline void shrink_free_pagepool(struct xen_blkif *blkif, int num)
+static inline void shrink_free_pagepool(struct xen_blkif_ring *ring, int num)
 {
 	/* Remove requested pages in batches of NUM_BATCH_FREE_PAGES */
 	struct page *page[NUM_BATCH_FREE_PAGES];
 	unsigned int num_pages = 0;
 	unsigned long flags;
 
-	spin_lock_irqsave(&blkif->free_pages_lock, flags);
-	while (blkif->free_pages_num > num) {
-		BUG_ON(list_empty(&blkif->free_pages));
-		page[num_pages] = list_first_entry(&blkif->free_pages,
+	spin_lock_irqsave(&ring->free_pages_lock, flags);
+	while (ring->free_pages_num > num) {
+		BUG_ON(list_empty(&ring->free_pages));
+		page[num_pages] = list_first_entry(&ring->free_pages,
 		                                   struct page, lru);
 		list_del(&page[num_pages]->lru);
-		blkif->free_pages_num--;
+		ring->free_pages_num--;
 		if (++num_pages == NUM_BATCH_FREE_PAGES) {
-			spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
+			spin_unlock_irqrestore(&ring->free_pages_lock, flags);
 			gnttab_free_pages(num_pages, page);
-			spin_lock_irqsave(&blkif->free_pages_lock, flags);
+			spin_lock_irqsave(&ring->free_pages_lock, flags);
 			num_pages = 0;
 		}
 	}
-	spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
+	spin_unlock_irqrestore(&ring->free_pages_lock, flags);
 	if (num_pages != 0)
 		gnttab_free_pages(num_pages, page);
 }
 
 #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
 
-static int do_block_io_op(struct xen_blkif *blkif);
-static int dispatch_rw_block_io(struct xen_blkif *blkif,
+static int do_block_io_op(struct xen_blkif_ring *ring);
+static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 				struct blkif_request *req,
 				struct pending_req *pending_req);
-static void make_response(struct xen_blkif *blkif, u64 id,
+static void make_response(struct xen_blkif_ring *ring, u64 id,
 			  unsigned short op, int st);
 
 #define foreach_grant_safe(pos, n, rbtree, node) \
@@ -190,7 +200,7 @@ static void make_response(struct xen_blkif *blkif, u64 id,
 
 /*
  * We don't need locking around the persistent grant helpers
- * because blkback uses a single-thread for each backed, so we
+ * because blkback uses a single-thread for each backend, so we
  * can be sure that this functions will never be called recursively.
  *
  * The only exception to that is put_persistent_grant, that can be called
@@ -198,19 +208,20 @@ static void make_response(struct xen_blkif *blkif, u64 id,
  * bit operations to modify the flags of a persistent grant and to count
  * the number of used grants.
  */
-static int add_persistent_gnt(struct xen_blkif *blkif,
+static int add_persistent_gnt(struct xen_blkif_ring *ring,
 			       struct persistent_gnt *persistent_gnt)
 {
 	struct rb_node **new = NULL, *parent = NULL;
 	struct persistent_gnt *this;
+	struct xen_blkif *blkif = ring->blkif;
 
-	if (blkif->persistent_gnt_c >= xen_blkif_max_pgrants) {
+	if (ring->persistent_gnt_c >= xen_blkif_max_pgrants) {
 		if (!blkif->vbd.overflow_max_grants)
 			blkif->vbd.overflow_max_grants = 1;
 		return -EBUSY;
 	}
 	/* Figure out where to put new node */
-	new = &blkif->persistent_gnts.rb_node;
+	new = &ring->persistent_gnts.rb_node;
 	while (*new) {
 		this = container_of(*new, struct persistent_gnt, node);
 
@@ -229,19 +240,19 @@ static int add_persistent_gnt(struct xen_blkif *blkif,
 	set_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
 	/* Add new node and rebalance tree. */
 	rb_link_node(&(persistent_gnt->node), parent, new);
-	rb_insert_color(&(persistent_gnt->node), &blkif->persistent_gnts);
-	blkif->persistent_gnt_c++;
-	atomic_inc(&blkif->persistent_gnt_in_use);
+	rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts);
+	ring->persistent_gnt_c++;
+	atomic_inc(&ring->persistent_gnt_in_use);
 	return 0;
 }
 
-static struct persistent_gnt *get_persistent_gnt(struct xen_blkif *blkif,
+static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring,
 						 grant_ref_t gref)
 {
 	struct persistent_gnt *data;
 	struct rb_node *node = NULL;
 
-	node = blkif->persistent_gnts.rb_node;
+	node = ring->persistent_gnts.rb_node;
 	while (node) {
 		data = container_of(node, struct persistent_gnt, node);
 
@@ -255,24 +266,24 @@ static struct persistent_gnt *get_persistent_gnt(struct xen_blkif *blkif,
 				return NULL;
 			}
 			set_bit(PERSISTENT_GNT_ACTIVE, data->flags);
-			atomic_inc(&blkif->persistent_gnt_in_use);
+			atomic_inc(&ring->persistent_gnt_in_use);
 			return data;
 		}
 	}
 	return NULL;
 }
 
-static void put_persistent_gnt(struct xen_blkif *blkif,
+static void put_persistent_gnt(struct xen_blkif_ring *ring,
                                struct persistent_gnt *persistent_gnt)
 {
 	if(!test_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags))
 		pr_alert_ratelimited("freeing a grant already unused\n");
 	set_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags);
 	clear_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
-	atomic_dec(&blkif->persistent_gnt_in_use);
+	atomic_dec(&ring->persistent_gnt_in_use);
 }
 
-static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
+static void free_persistent_gnts(struct xen_blkif_ring *ring, struct rb_root *root,
                                  unsigned int num)
 {
 	struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
@@ -303,7 +314,7 @@ static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
 			unmap_data.count = segs_to_unmap;
 			BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
 
-			put_free_pages(blkif, pages, segs_to_unmap);
+			put_free_pages(ring, pages, segs_to_unmap);
 			segs_to_unmap = 0;
 		}
 
@@ -320,15 +331,15 @@ void xen_blkbk_unmap_purged_grants(struct work_struct *work)
 	struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 	struct persistent_gnt *persistent_gnt;
 	int segs_to_unmap = 0;
-	struct xen_blkif *blkif = container_of(work, typeof(*blkif), persistent_purge_work);
+	struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work);
 	struct gntab_unmap_queue_data unmap_data;
 
 	unmap_data.pages = pages;
 	unmap_data.unmap_ops = unmap;
 	unmap_data.kunmap_ops = NULL;
 
-	while(!list_empty(&blkif->persistent_purge_list)) {
-		persistent_gnt = list_first_entry(&blkif->persistent_purge_list,
+	while(!list_empty(&ring->persistent_purge_list)) {
+		persistent_gnt = list_first_entry(&ring->persistent_purge_list,
 		                                  struct persistent_gnt,
 		                                  remove_node);
 		list_del(&persistent_gnt->remove_node);
@@ -343,7 +354,7 @@ void xen_blkbk_unmap_purged_grants(struct work_struct *work)
 		if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
 			unmap_data.count = segs_to_unmap;
 			BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
-			put_free_pages(blkif, pages, segs_to_unmap);
+			put_free_pages(ring, pages, segs_to_unmap);
 			segs_to_unmap = 0;
 		}
 		kfree(persistent_gnt);
@@ -351,11 +362,11 @@ void xen_blkbk_unmap_purged_grants(struct work_struct *work)
 	if (segs_to_unmap > 0) {
 		unmap_data.count = segs_to_unmap;
 		BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
-		put_free_pages(blkif, pages, segs_to_unmap);
+		put_free_pages(ring, pages, segs_to_unmap);
 	}
 }
 
-static void purge_persistent_gnt(struct xen_blkif *blkif)
+static void purge_persistent_gnt(struct xen_blkif_ring *ring)
 {
 	struct persistent_gnt *persistent_gnt;
 	struct rb_node *n;
@@ -363,23 +374,23 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 	bool scan_used = false, clean_used = false;
 	struct rb_root *root;
 
-	if (blkif->persistent_gnt_c < xen_blkif_max_pgrants ||
-	    (blkif->persistent_gnt_c == xen_blkif_max_pgrants &&
-	    !blkif->vbd.overflow_max_grants)) {
-		return;
+	if (ring->persistent_gnt_c < xen_blkif_max_pgrants ||
+	    (ring->persistent_gnt_c == xen_blkif_max_pgrants &&
+	    !ring->blkif->vbd.overflow_max_grants)) {
+		goto out;
 	}
 
-	if (work_busy(&blkif->persistent_purge_work)) {
+	if (work_busy(&ring->persistent_purge_work)) {
 		pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n");
-		return;
+		goto out;
 	}
 
 	num_clean = (xen_blkif_max_pgrants / 100) * LRU_PERCENT_CLEAN;
-	num_clean = blkif->persistent_gnt_c - xen_blkif_max_pgrants + num_clean;
-	num_clean = min(blkif->persistent_gnt_c, num_clean);
+	num_clean = ring->persistent_gnt_c - xen_blkif_max_pgrants + num_clean;
+	num_clean = min(ring->persistent_gnt_c, num_clean);
 	if ((num_clean == 0) ||
-	    (num_clean > (blkif->persistent_gnt_c - atomic_read(&blkif->persistent_gnt_in_use))))
-		return;
+	    (num_clean > (ring->persistent_gnt_c - atomic_read(&ring->persistent_gnt_in_use))))
+		goto out;
 
 	/*
 	 * At this point, we can assure that there will be no calls
@@ -394,8 +405,8 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
-	root = &blkif->persistent_gnts;
+	BUG_ON(!list_empty(&ring->persistent_purge_list));
+	root = &ring->persistent_gnts;
 purge_list:
 	foreach_grant_safe(persistent_gnt, n, root, node) {
 		BUG_ON(persistent_gnt->handle ==
@@ -414,7 +425,7 @@ purge_list:
 
 		rb_erase(&persistent_gnt->node, root);
 		list_add(&persistent_gnt->remove_node,
-		         &blkif->persistent_purge_list);
+			 &ring->persistent_purge_list);
 		if (--num_clean == 0)
 			goto finished;
 	}
@@ -435,30 +446,32 @@ finished:
 		goto purge_list;
 	}
 
-	blkif->persistent_gnt_c -= (total - num_clean);
-	blkif->vbd.overflow_max_grants = 0;
+	ring->persistent_gnt_c -= (total - num_clean);
+	ring->blkif->vbd.overflow_max_grants = 0;
 
 	/* We can defer this work */
-	schedule_work(&blkif->persistent_purge_work);
+	schedule_work(&ring->persistent_purge_work);
 	pr_debug("Purged %u/%u\n", (total - num_clean), total);
+
+out:
 	return;
 }
 
 /*
  * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
  */
-static struct pending_req *alloc_req(struct xen_blkif *blkif)
+static struct pending_req *alloc_req(struct xen_blkif_ring *ring)
 {
 	struct pending_req *req = NULL;
 	unsigned long flags;
 
-	spin_lock_irqsave(&blkif->pending_free_lock, flags);
-	if (!list_empty(&blkif->pending_free)) {
-		req = list_entry(blkif->pending_free.next, struct pending_req,
+	spin_lock_irqsave(&ring->pending_free_lock, flags);
+	if (!list_empty(&ring->pending_free)) {
+		req = list_entry(ring->pending_free.next, struct pending_req,
 				 free_list);
 		list_del(&req->free_list);
 	}
-	spin_unlock_irqrestore(&blkif->pending_free_lock, flags);
+	spin_unlock_irqrestore(&ring->pending_free_lock, flags);
 	return req;
 }
 
@@ -466,17 +479,17 @@ static struct pending_req *alloc_req(struct xen_blkif *blkif)
  * Return the 'pending_req' structure back to the freepool. We also
  * wake up the thread if it was waiting for a free page.
  */
-static void free_req(struct xen_blkif *blkif, struct pending_req *req)
+static void free_req(struct xen_blkif_ring *ring, struct pending_req *req)
 {
 	unsigned long flags;
 	int was_empty;
 
-	spin_lock_irqsave(&blkif->pending_free_lock, flags);
-	was_empty = list_empty(&blkif->pending_free);
-	list_add(&req->free_list, &blkif->pending_free);
-	spin_unlock_irqrestore(&blkif->pending_free_lock, flags);
+	spin_lock_irqsave(&ring->pending_free_lock, flags);
+	was_empty = list_empty(&ring->pending_free);
+	list_add(&req->free_list, &ring->pending_free);
+	spin_unlock_irqrestore(&ring->pending_free_lock, flags);
 	if (was_empty)
-		wake_up(&blkif->pending_free_wq);
+		wake_up(&ring->pending_free_wq);
 }
 
 /*
@@ -556,10 +569,10 @@ abort:
 /*
  * Notification from the guest OS.
  */
-static void blkif_notify_work(struct xen_blkif *blkif)
+static void blkif_notify_work(struct xen_blkif_ring *ring)
 {
-	blkif->waiting_reqs = 1;
-	wake_up(&blkif->wq);
+	ring->waiting_reqs = 1;
+	wake_up(&ring->wq);
 }
 
 irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
@@ -572,31 +585,33 @@ irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
  * SCHEDULER FUNCTIONS
  */
 
-static void print_stats(struct xen_blkif *blkif)
+static void print_stats(struct xen_blkif_ring *ring)
 {
 	pr_info("(%s): oo %3llu  |  rd %4llu  |  wr %4llu  |  f %4llu"
 		 "  |  ds %4llu | pg: %4u/%4d\n",
-		 current->comm, blkif->st_oo_req,
-		 blkif->st_rd_req, blkif->st_wr_req,
-		 blkif->st_f_req, blkif->st_ds_req,
-		 blkif->persistent_gnt_c,
+		 current->comm, ring->st_oo_req,
+		 ring->st_rd_req, ring->st_wr_req,
+		 ring->st_f_req, ring->st_ds_req,
+		 ring->persistent_gnt_c,
 		 xen_blkif_max_pgrants);
-	blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000);
-	blkif->st_rd_req = 0;
-	blkif->st_wr_req = 0;
-	blkif->st_oo_req = 0;
-	blkif->st_ds_req = 0;
+	ring->st_print = jiffies + msecs_to_jiffies(10 * 1000);
+	ring->st_rd_req = 0;
+	ring->st_wr_req = 0;
+	ring->st_oo_req = 0;
+	ring->st_ds_req = 0;
 }
 
 int xen_blkif_schedule(void *arg)
 {
-	struct xen_blkif *blkif = arg;
+	struct xen_blkif_ring *ring = arg;
+	struct xen_blkif *blkif = ring->blkif;
 	struct xen_vbd *vbd = &blkif->vbd;
 	unsigned long timeout;
 	int ret;
 
 	xen_blkif_get(blkif);
 
+	set_freezable();
 	while (!kthread_should_stop()) {
 		if (try_to_freeze())
 			continue;
@@ -606,50 +621,50 @@ int xen_blkif_schedule(void *arg)
 		timeout = msecs_to_jiffies(LRU_INTERVAL);
 
 		timeout = wait_event_interruptible_timeout(
-			blkif->wq,
-			blkif->waiting_reqs || kthread_should_stop(),
+			ring->wq,
+			ring->waiting_reqs || kthread_should_stop(),
 			timeout);
 		if (timeout == 0)
 			goto purge_gnt_list;
 		timeout = wait_event_interruptible_timeout(
-			blkif->pending_free_wq,
-			!list_empty(&blkif->pending_free) ||
+			ring->pending_free_wq,
+			!list_empty(&ring->pending_free) ||
 			kthread_should_stop(),
 			timeout);
 		if (timeout == 0)
 			goto purge_gnt_list;
 
-		blkif->waiting_reqs = 0;
+		ring->waiting_reqs = 0;
 		smp_mb(); /* clear flag *before* checking for work */
 
-		ret = do_block_io_op(blkif);
+		ret = do_block_io_op(ring);
 		if (ret > 0)
-			blkif->waiting_reqs = 1;
+			ring->waiting_reqs = 1;
 		if (ret == -EACCES)
-			wait_event_interruptible(blkif->shutdown_wq,
+			wait_event_interruptible(ring->shutdown_wq,
 						 kthread_should_stop());
 
 purge_gnt_list:
 		if (blkif->vbd.feature_gnt_persistent &&
-		    time_after(jiffies, blkif->next_lru)) {
-			purge_persistent_gnt(blkif);
-			blkif->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
+		    time_after(jiffies, ring->next_lru)) {
+			purge_persistent_gnt(ring);
+			ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
 		}
 
 		/* Shrink if we have more than xen_blkif_max_buffer_pages */
-		shrink_free_pagepool(blkif, xen_blkif_max_buffer_pages);
+		shrink_free_pagepool(ring, xen_blkif_max_buffer_pages);
 
-		if (log_stats && time_after(jiffies, blkif->st_print))
-			print_stats(blkif);
+		if (log_stats && time_after(jiffies, ring->st_print))
+			print_stats(ring);
 	}
 
 	/* Drain pending purge work */
-	flush_work(&blkif->persistent_purge_work);
+	flush_work(&ring->persistent_purge_work);
 
 	if (log_stats)
-		print_stats(blkif);
+		print_stats(ring);
 
-	blkif->xenblkd = NULL;
+	ring->xenblkd = NULL;
 	xen_blkif_put(blkif);
 
 	return 0;
@@ -658,22 +673,22 @@ purge_gnt_list:
 /*
  * Remove persistent grants and empty the pool of free pages
  */
-void xen_blkbk_free_caches(struct xen_blkif *blkif)
+void xen_blkbk_free_caches(struct xen_blkif_ring *ring)
 {
 	/* Free all persistent grant pages */
-	if (!RB_EMPTY_ROOT(&blkif->persistent_gnts))
-		free_persistent_gnts(blkif, &blkif->persistent_gnts,
-			blkif->persistent_gnt_c);
+	if (!RB_EMPTY_ROOT(&ring->persistent_gnts))
+		free_persistent_gnts(ring, &ring->persistent_gnts,
+			ring->persistent_gnt_c);
 
-	BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts));
-	blkif->persistent_gnt_c = 0;
+	BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
+	ring->persistent_gnt_c = 0;
 
 	/* Since we are shutting down remove all pages from the buffer */
-	shrink_free_pagepool(blkif, 0 /* All */);
+	shrink_free_pagepool(ring, 0 /* All */);
 }
 
 static unsigned int xen_blkbk_unmap_prepare(
-	struct xen_blkif *blkif,
+	struct xen_blkif_ring *ring,
 	struct grant_page **pages,
 	unsigned int num,
 	struct gnttab_unmap_grant_ref *unmap_ops,
@@ -683,7 +698,7 @@ static unsigned int xen_blkbk_unmap_prepare(
 
 	for (i = 0; i < num; i++) {
 		if (pages[i]->persistent_gnt != NULL) {
-			put_persistent_gnt(blkif, pages[i]->persistent_gnt);
+			put_persistent_gnt(ring, pages[i]->persistent_gnt);
 			continue;
 		}
 		if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
@@ -700,17 +715,18 @@ static unsigned int xen_blkbk_unmap_prepare(
 
 static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
 {
-	struct pending_req* pending_req = (struct pending_req*) (data->data);
-	struct xen_blkif *blkif = pending_req->blkif;
+	struct pending_req *pending_req = (struct pending_req *)(data->data);
+	struct xen_blkif_ring *ring = pending_req->ring;
+	struct xen_blkif *blkif = ring->blkif;
 
 	/* BUG_ON used to reproduce existing behaviour,
 	   but is this the best way to deal with this? */
 	BUG_ON(result);
 
-	put_free_pages(blkif, data->pages, data->count);
-	make_response(blkif, pending_req->id,
+	put_free_pages(ring, data->pages, data->count);
+	make_response(ring, pending_req->id,
 		      pending_req->operation, pending_req->status);
-	free_req(blkif, pending_req);
+	free_req(ring, pending_req);
 	/*
 	 * Make sure the request is freed before releasing blkif,
 	 * or there could be a race between free_req and the
@@ -723,7 +739,7 @@ static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_
 	 * pending_free_wq if there's a drain going on, but it has
 	 * to be taken into account if the current model is changed.
 	 */
-	if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) {
+	if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) {
 		complete(&blkif->drain_complete);
 	}
 	xen_blkif_put(blkif);
@@ -732,11 +748,11 @@ static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_
 static void xen_blkbk_unmap_and_respond(struct pending_req *req)
 {
 	struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
-	struct xen_blkif *blkif = req->blkif;
+	struct xen_blkif_ring *ring = req->ring;
 	struct grant_page **pages = req->segments;
 	unsigned int invcount;
 
-	invcount = xen_blkbk_unmap_prepare(blkif, pages, req->nr_segs,
+	invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs,
 					   req->unmap, req->unmap_pages);
 
 	work->data = req;
@@ -757,7 +773,7 @@ static void xen_blkbk_unmap_and_respond(struct pending_req *req)
  * of hypercalls, but since this is only used in error paths there's
  * no real need.
  */
-static void xen_blkbk_unmap(struct xen_blkif *blkif,
+static void xen_blkbk_unmap(struct xen_blkif_ring *ring,
                             struct grant_page *pages[],
                             int num)
 {
@@ -768,20 +784,20 @@ static void xen_blkbk_unmap(struct xen_blkif *blkif,
 
 	while (num) {
 		unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
-		
-		invcount = xen_blkbk_unmap_prepare(blkif, pages, batch,
+
+		invcount = xen_blkbk_unmap_prepare(ring, pages, batch,
 						   unmap, unmap_pages);
 		if (invcount) {
 			ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
 			BUG_ON(ret);
-			put_free_pages(blkif, unmap_pages, invcount);
+			put_free_pages(ring, unmap_pages, invcount);
 		}
 		pages += batch;
 		num -= batch;
 	}
 }
 
-static int xen_blkbk_map(struct xen_blkif *blkif,
+static int xen_blkbk_map(struct xen_blkif_ring *ring,
 			 struct grant_page *pages[],
 			 int num, bool ro)
 {
@@ -794,6 +810,7 @@ static int xen_blkbk_map(struct xen_blkif *blkif,
 	int ret = 0;
 	int last_map = 0, map_until = 0;
 	int use_persistent_gnts;
+	struct xen_blkif *blkif = ring->blkif;
 
 	use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
 
@@ -806,10 +823,11 @@ again:
 	for (i = map_until; i < num; i++) {
 		uint32_t flags;
 
-		if (use_persistent_gnts)
+		if (use_persistent_gnts) {
 			persistent_gnt = get_persistent_gnt(
-				blkif,
+				ring,
 				pages[i]->gref);
+		}
 
 		if (persistent_gnt) {
 			/*
@@ -819,7 +837,7 @@ again:
 			pages[i]->page = persistent_gnt->page;
 			pages[i]->persistent_gnt = persistent_gnt;
 		} else {
-			if (get_free_page(blkif, &pages[i]->page))
+			if (get_free_page(ring, &pages[i]->page))
 				goto out_of_memory;
 			addr = vaddr(pages[i]->page);
 			pages_to_gnt[segs_to_map] = pages[i]->page;
@@ -852,7 +870,7 @@ again:
 			BUG_ON(new_map_idx >= segs_to_map);
 			if (unlikely(map[new_map_idx].status != 0)) {
 				pr_debug("invalid buffer -- could not remap it\n");
-				put_free_pages(blkif, &pages[seg_idx]->page, 1);
+				put_free_pages(ring, &pages[seg_idx]->page, 1);
 				pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
 				ret |= 1;
 				goto next;
@@ -862,7 +880,7 @@ again:
 			continue;
 		}
 		if (use_persistent_gnts &&
-		    blkif->persistent_gnt_c < xen_blkif_max_pgrants) {
+		    ring->persistent_gnt_c < xen_blkif_max_pgrants) {
 			/*
 			 * We are using persistent grants, the grant is
 			 * not mapped but we might have room for it.
@@ -880,7 +898,7 @@ again:
 			persistent_gnt->gnt = map[new_map_idx].ref;
 			persistent_gnt->handle = map[new_map_idx].handle;
 			persistent_gnt->page = pages[seg_idx]->page;
-			if (add_persistent_gnt(blkif,
+			if (add_persistent_gnt(ring,
 			                       persistent_gnt)) {
 				kfree(persistent_gnt);
 				persistent_gnt = NULL;
@@ -888,7 +906,7 @@ again:
 			}
 			pages[seg_idx]->persistent_gnt = persistent_gnt;
 			pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n",
-				 persistent_gnt->gnt, blkif->persistent_gnt_c,
+				 persistent_gnt->gnt, ring->persistent_gnt_c,
 				 xen_blkif_max_pgrants);
 			goto next;
 		}
@@ -913,7 +931,7 @@ next:
 
 out_of_memory:
 	pr_alert("%s: out of memory\n", __func__);
-	put_free_pages(blkif, pages_to_gnt, segs_to_map);
+	put_free_pages(ring, pages_to_gnt, segs_to_map);
 	return -ENOMEM;
 }
 
@@ -921,7 +939,7 @@ static int xen_blkbk_map_seg(struct pending_req *pending_req)
 {
 	int rc;
 
-	rc = xen_blkbk_map(pending_req->blkif, pending_req->segments,
+	rc = xen_blkbk_map(pending_req->ring, pending_req->segments,
 			   pending_req->nr_segs,
 	                   (pending_req->operation != BLKIF_OP_READ));
 
@@ -934,7 +952,7 @@ static int xen_blkbk_parse_indirect(struct blkif_request *req,
 				    struct phys_req *preq)
 {
 	struct grant_page **pages = pending_req->indirect_pages;
-	struct xen_blkif *blkif = pending_req->blkif;
+	struct xen_blkif_ring *ring = pending_req->ring;
 	int indirect_grefs, rc, n, nseg, i;
 	struct blkif_request_segment *segments = NULL;
 
@@ -945,7 +963,7 @@ static int xen_blkbk_parse_indirect(struct blkif_request *req,
 	for (i = 0; i < indirect_grefs; i++)
 		pages[i]->gref = req->u.indirect.indirect_grefs[i];
 
-	rc = xen_blkbk_map(blkif, pages, indirect_grefs, true);
+	rc = xen_blkbk_map(ring, pages, indirect_grefs, true);
 	if (rc)
 		goto unmap;
 
@@ -977,15 +995,16 @@ static int xen_blkbk_parse_indirect(struct blkif_request *req,
 unmap:
 	if (segments)
 		kunmap_atomic(segments);
-	xen_blkbk_unmap(blkif, pages, indirect_grefs);
+	xen_blkbk_unmap(ring, pages, indirect_grefs);
 	return rc;
 }
 
-static int dispatch_discard_io(struct xen_blkif *blkif,
+static int dispatch_discard_io(struct xen_blkif_ring *ring,
 				struct blkif_request *req)
 {
 	int err = 0;
 	int status = BLKIF_RSP_OKAY;
+	struct xen_blkif *blkif = ring->blkif;
 	struct block_device *bdev = blkif->vbd.bdev;
 	unsigned long secure;
 	struct phys_req preq;
@@ -1002,7 +1021,7 @@ static int dispatch_discard_io(struct xen_blkif *blkif,
 			preq.sector_number + preq.nr_sects, blkif->vbd.pdevice);
 		goto fail_response;
 	}
-	blkif->st_ds_req++;
+	ring->st_ds_req++;
 
 	secure = (blkif->vbd.discard_secure &&
 		 (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
@@ -1018,26 +1037,28 @@ fail_response:
 	} else if (err)
 		status = BLKIF_RSP_ERROR;
 
-	make_response(blkif, req->u.discard.id, req->operation, status);
+	make_response(ring, req->u.discard.id, req->operation, status);
 	xen_blkif_put(blkif);
 	return err;
 }
 
-static int dispatch_other_io(struct xen_blkif *blkif,
+static int dispatch_other_io(struct xen_blkif_ring *ring,
 			     struct blkif_request *req,
 			     struct pending_req *pending_req)
 {
-	free_req(blkif, pending_req);
-	make_response(blkif, req->u.other.id, req->operation,
+	free_req(ring, pending_req);
+	make_response(ring, req->u.other.id, req->operation,
 		      BLKIF_RSP_EOPNOTSUPP);
 	return -EIO;
 }
 
-static void xen_blk_drain_io(struct xen_blkif *blkif)
+static void xen_blk_drain_io(struct xen_blkif_ring *ring)
 {
+	struct xen_blkif *blkif = ring->blkif;
+
 	atomic_set(&blkif->drain, 1);
 	do {
-		if (atomic_read(&blkif->inflight) == 0)
+		if (atomic_read(&ring->inflight) == 0)
 			break;
 		wait_for_completion_interruptible_timeout(
 				&blkif->drain_complete, HZ);
@@ -1058,12 +1079,12 @@ static void __end_block_io_op(struct pending_req *pending_req, int error)
 	if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&
 	    (error == -EOPNOTSUPP)) {
 		pr_debug("flush diskcache op failed, not supported\n");
-		xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0);
+		xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0);
 		pending_req->status = BLKIF_RSP_EOPNOTSUPP;
 	} else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) &&
 		    (error == -EOPNOTSUPP)) {
 		pr_debug("write barrier op failed, not supported\n");
-		xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0);
+		xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0);
 		pending_req->status = BLKIF_RSP_EOPNOTSUPP;
 	} else if (error) {
 		pr_debug("Buffer not up-to-date at end of operation,"
@@ -1097,9 +1118,9 @@ static void end_block_io_op(struct bio *bio)
  * and transmute  it to the block API to hand it over to the proper block disk.
  */
 static int
-__do_block_io_op(struct xen_blkif *blkif)
+__do_block_io_op(struct xen_blkif_ring *ring)
 {
-	union blkif_back_rings *blk_rings = &blkif->blk_rings;
+	union blkif_back_rings *blk_rings = &ring->blk_rings;
 	struct blkif_request req;
 	struct pending_req *pending_req;
 	RING_IDX rc, rp;
@@ -1112,7 +1133,7 @@ __do_block_io_op(struct xen_blkif *blkif)
 	if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) {
 		rc = blk_rings->common.rsp_prod_pvt;
 		pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n",
-			rp, rc, rp - rc, blkif->vbd.pdevice);
+			rp, rc, rp - rc, ring->blkif->vbd.pdevice);
 		return -EACCES;
 	}
 	while (rc != rp) {
@@ -1125,14 +1146,14 @@ __do_block_io_op(struct xen_blkif *blkif)
 			break;
 		}
 
-		pending_req = alloc_req(blkif);
+		pending_req = alloc_req(ring);
 		if (NULL == pending_req) {
-			blkif->st_oo_req++;
+			ring->st_oo_req++;
 			more_to_do = 1;
 			break;
 		}
 
-		switch (blkif->blk_protocol) {
+		switch (ring->blkif->blk_protocol) {
 		case BLKIF_PROTOCOL_NATIVE:
 			memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
 			break;
@@ -1156,16 +1177,16 @@ __do_block_io_op(struct xen_blkif *blkif)
 		case BLKIF_OP_WRITE_BARRIER:
 		case BLKIF_OP_FLUSH_DISKCACHE:
 		case BLKIF_OP_INDIRECT:
-			if (dispatch_rw_block_io(blkif, &req, pending_req))
+			if (dispatch_rw_block_io(ring, &req, pending_req))
 				goto done;
 			break;
 		case BLKIF_OP_DISCARD:
-			free_req(blkif, pending_req);
-			if (dispatch_discard_io(blkif, &req))
+			free_req(ring, pending_req);
+			if (dispatch_discard_io(ring, &req))
 				goto done;
 			break;
 		default:
-			if (dispatch_other_io(blkif, &req, pending_req))
+			if (dispatch_other_io(ring, &req, pending_req))
 				goto done;
 			break;
 		}
@@ -1178,13 +1199,13 @@ done:
 }
 
 static int
-do_block_io_op(struct xen_blkif *blkif)
+do_block_io_op(struct xen_blkif_ring *ring)
 {
-	union blkif_back_rings *blk_rings = &blkif->blk_rings;
+	union blkif_back_rings *blk_rings = &ring->blk_rings;
 	int more_to_do;
 
 	do {
-		more_to_do = __do_block_io_op(blkif);
+		more_to_do = __do_block_io_op(ring);
 		if (more_to_do)
 			break;
 
@@ -1197,7 +1218,7 @@ do_block_io_op(struct xen_blkif *blkif)
  * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
  * and call the 'submit_bio' to pass it to the underlying storage.
  */
-static int dispatch_rw_block_io(struct xen_blkif *blkif,
+static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 				struct blkif_request *req,
 				struct pending_req *pending_req)
 {
@@ -1225,17 +1246,17 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 
 	switch (req_operation) {
 	case BLKIF_OP_READ:
-		blkif->st_rd_req++;
+		ring->st_rd_req++;
 		operation = READ;
 		break;
 	case BLKIF_OP_WRITE:
-		blkif->st_wr_req++;
+		ring->st_wr_req++;
 		operation = WRITE_ODIRECT;
 		break;
 	case BLKIF_OP_WRITE_BARRIER:
 		drain = true;
 	case BLKIF_OP_FLUSH_DISKCACHE:
-		blkif->st_f_req++;
+		ring->st_f_req++;
 		operation = WRITE_FLUSH;
 		break;
 	default:
@@ -1260,7 +1281,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 
 	preq.nr_sects      = 0;
 
-	pending_req->blkif     = blkif;
+	pending_req->ring      = ring;
 	pending_req->id        = req->u.rw.id;
 	pending_req->operation = req_operation;
 	pending_req->status    = BLKIF_RSP_OKAY;
@@ -1287,12 +1308,12 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 			goto fail_response;
 	}
 
-	if (xen_vbd_translate(&preq, blkif, operation) != 0) {
+	if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) {
 		pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n",
 			 operation == READ ? "read" : "write",
 			 preq.sector_number,
 			 preq.sector_number + preq.nr_sects,
-			 blkif->vbd.pdevice);
+			 ring->blkif->vbd.pdevice);
 		goto fail_response;
 	}
 
@@ -1304,7 +1325,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 		if (((int)preq.sector_number|(int)seg[i].nsec) &
 		    ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
 			pr_debug("Misaligned I/O request from domain %d\n",
-				 blkif->domid);
+				 ring->blkif->domid);
 			goto fail_response;
 		}
 	}
@@ -1313,7 +1334,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 	 * issue the WRITE_FLUSH.
 	 */
 	if (drain)
-		xen_blk_drain_io(pending_req->blkif);
+		xen_blk_drain_io(pending_req->ring);
 
 	/*
 	 * If we have failed at this point, we need to undo the M2P override,
@@ -1328,8 +1349,8 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 	 * This corresponding xen_blkif_put is done in __end_block_io_op, or
 	 * below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
 	 */
-	xen_blkif_get(blkif);
-	atomic_inc(&blkif->inflight);
+	xen_blkif_get(ring->blkif);
+	atomic_inc(&ring->inflight);
 
 	for (i = 0; i < nseg; i++) {
 		while ((bio == NULL) ||
@@ -1377,19 +1398,19 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 	blk_finish_plug(&plug);
 
 	if (operation == READ)
-		blkif->st_rd_sect += preq.nr_sects;
+		ring->st_rd_sect += preq.nr_sects;
 	else if (operation & WRITE)
-		blkif->st_wr_sect += preq.nr_sects;
+		ring->st_wr_sect += preq.nr_sects;
 
 	return 0;
 
  fail_flush:
-	xen_blkbk_unmap(blkif, pending_req->segments,
+	xen_blkbk_unmap(ring, pending_req->segments,
 	                pending_req->nr_segs);
  fail_response:
 	/* Haven't submitted any bio's yet. */
-	make_response(blkif, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
-	free_req(blkif, pending_req);
+	make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
+	free_req(ring, pending_req);
 	msleep(1); /* back off a bit */
 	return -EIO;
 
@@ -1407,21 +1428,22 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
 /*
  * Put a response on the ring on how the operation fared.
  */
-static void make_response(struct xen_blkif *blkif, u64 id,
+static void make_response(struct xen_blkif_ring *ring, u64 id,
 			  unsigned short op, int st)
 {
 	struct blkif_response  resp;
 	unsigned long     flags;
-	union blkif_back_rings *blk_rings = &blkif->blk_rings;
+	union blkif_back_rings *blk_rings;
 	int notify;
 
 	resp.id        = id;
 	resp.operation = op;
 	resp.status    = st;
 
-	spin_lock_irqsave(&blkif->blk_ring_lock, flags);
+	spin_lock_irqsave(&ring->blk_ring_lock, flags);
+	blk_rings = &ring->blk_rings;
 	/* Place on the response ring for the relevant domain. */
-	switch (blkif->blk_protocol) {
+	switch (ring->blkif->blk_protocol) {
 	case BLKIF_PROTOCOL_NATIVE:
 		memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),
 		       &resp, sizeof(resp));
@@ -1439,9 +1461,9 @@ static void make_response(struct xen_blkif *blkif, u64 id,
 	}
 	blk_rings->common.rsp_prod_pvt++;
 	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
-	spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);
+	spin_unlock_irqrestore(&ring->blk_ring_lock, flags);
 	if (notify)
-		notify_remote_via_irq(blkif->irq);
+		notify_remote_via_irq(ring->irq);
 }
 
 static int __init xen_blkif_init(void)
@@ -1457,6 +1479,9 @@ static int __init xen_blkif_init(void)
 		xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
 	}
 
+	if (xenblk_max_queues == 0)
+		xenblk_max_queues = num_online_cpus();
+
 	rc = xen_blkif_interface_init();
 	if (rc)
 		goto failed_init;
diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h
index c929ae22764c..dea61f6ab8cb 100644
--- a/drivers/block/xen-blkback/common.h
+++ b/drivers/block/xen-blkback/common.h
@@ -46,6 +46,7 @@
 #include <xen/interface/io/protocols.h>
 
 extern unsigned int xen_blkif_max_ring_order;
+extern unsigned int xenblk_max_queues;
 /*
  * This is the maximum number of segments that would be allowed in indirect
  * requests. This value will also be passed to the frontend.
@@ -269,68 +270,79 @@ struct persistent_gnt {
 	struct list_head remove_node;
 };
 
-struct xen_blkif {
-	/* Unique identifier for this interface. */
-	domid_t			domid;
-	unsigned int		handle;
+/* Per-ring information. */
+struct xen_blkif_ring {
 	/* Physical parameters of the comms window. */
 	unsigned int		irq;
-	/* Comms information. */
-	enum blkif_protocol	blk_protocol;
 	union blkif_back_rings	blk_rings;
 	void			*blk_ring;
-	/* The VBD attached to this interface. */
-	struct xen_vbd		vbd;
-	/* Back pointer to the backend_info. */
-	struct backend_info	*be;
 	/* Private fields. */
 	spinlock_t		blk_ring_lock;
-	atomic_t		refcnt;
 
 	wait_queue_head_t	wq;
-	/* for barrier (drain) requests */
-	struct completion	drain_complete;
-	atomic_t		drain;
 	atomic_t		inflight;
-	/* One thread per one blkif. */
+	/* One thread per blkif ring. */
 	struct task_struct	*xenblkd;
 	unsigned int		waiting_reqs;
 
-	/* tree to store persistent grants */
+	/* List of all 'pending_req' available */
+	struct list_head	pending_free;
+	/* And its spinlock. */
+	spinlock_t		pending_free_lock;
+	wait_queue_head_t	pending_free_wq;
+
+	/* Tree to store persistent grants. */
+	spinlock_t		pers_gnts_lock;
 	struct rb_root		persistent_gnts;
 	unsigned int		persistent_gnt_c;
 	atomic_t		persistent_gnt_in_use;
 	unsigned long           next_lru;
 
-	/* used by the kworker that offload work from the persistent purge */
+	/* Statistics. */
+	unsigned long		st_print;
+	unsigned long long	st_rd_req;
+	unsigned long long	st_wr_req;
+	unsigned long long	st_oo_req;
+	unsigned long long	st_f_req;
+	unsigned long long	st_ds_req;
+	unsigned long long	st_rd_sect;
+	unsigned long long	st_wr_sect;
+
+	/* Used by the kworker that offload work from the persistent purge. */
 	struct list_head	persistent_purge_list;
 	struct work_struct	persistent_purge_work;
 
-	/* buffer of free pages to map grant refs */
+	/* Buffer of free pages to map grant refs. */
 	spinlock_t		free_pages_lock;
 	int			free_pages_num;
 	struct list_head	free_pages;
 
-	/* List of all 'pending_req' available */
-	struct list_head	pending_free;
-	/* And its spinlock. */
-	spinlock_t		pending_free_lock;
-	wait_queue_head_t	pending_free_wq;
-
-	/* statistics */
-	unsigned long		st_print;
-	unsigned long long			st_rd_req;
-	unsigned long long			st_wr_req;
-	unsigned long long			st_oo_req;
-	unsigned long long			st_f_req;
-	unsigned long long			st_ds_req;
-	unsigned long long			st_rd_sect;
-	unsigned long long			st_wr_sect;
-
 	struct work_struct	free_work;
 	/* Thread shutdown wait queue. */
 	wait_queue_head_t	shutdown_wq;
-	unsigned int nr_ring_pages;
+	struct xen_blkif 	*blkif;
+};
+
+struct xen_blkif {
+	/* Unique identifier for this interface. */
+	domid_t			domid;
+	unsigned int		handle;
+	/* Comms information. */
+	enum blkif_protocol	blk_protocol;
+	/* The VBD attached to this interface. */
+	struct xen_vbd		vbd;
+	/* Back pointer to the backend_info. */
+	struct backend_info	*be;
+	atomic_t		refcnt;
+	/* for barrier (drain) requests */
+	struct completion	drain_complete;
+	atomic_t		drain;
+
+	struct work_struct	free_work;
+	unsigned int 		nr_ring_pages;
+	/* All rings for this device. */
+	struct xen_blkif_ring	*rings;
+	unsigned int		nr_rings;
 };
 
 struct seg_buf {
@@ -352,7 +364,7 @@ struct grant_page {
  * response queued for it, with the saved 'id' passed back.
  */
 struct pending_req {
-	struct xen_blkif	*blkif;
+	struct xen_blkif_ring   *ring;
 	u64			id;
 	int			nr_segs;
 	atomic_t		pendcnt;
@@ -394,7 +406,7 @@ int xen_blkif_xenbus_init(void);
 irqreturn_t xen_blkif_be_int(int irq, void *dev_id);
 int xen_blkif_schedule(void *arg);
 int xen_blkif_purge_persistent(void *arg);
-void xen_blkbk_free_caches(struct xen_blkif *blkif);
+void xen_blkbk_free_caches(struct xen_blkif_ring *ring);
 
 int xen_blkbk_flush_diskcache(struct xenbus_transaction xbt,
 			      struct backend_info *be, int state);
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index f53cff42f8da..876763f7f13e 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -86,9 +86,11 @@ static void xen_update_blkif_status(struct xen_blkif *blkif)
 {
 	int err;
 	char name[BLKBACK_NAME_LEN];
+	struct xen_blkif_ring *ring;
+	int i;
 
 	/* Not ready to connect? */
-	if (!blkif->irq || !blkif->vbd.bdev)
+	if (!blkif->rings || !blkif->rings[0].irq || !blkif->vbd.bdev)
 		return;
 
 	/* Already connected? */
@@ -113,13 +115,55 @@ static void xen_update_blkif_status(struct xen_blkif *blkif)
 	}
 	invalidate_inode_pages2(blkif->vbd.bdev->bd_inode->i_mapping);
 
-	blkif->xenblkd = kthread_run(xen_blkif_schedule, blkif, "%s", name);
-	if (IS_ERR(blkif->xenblkd)) {
-		err = PTR_ERR(blkif->xenblkd);
-		blkif->xenblkd = NULL;
-		xenbus_dev_error(blkif->be->dev, err, "start xenblkd");
-		return;
+	for (i = 0; i < blkif->nr_rings; i++) {
+		ring = &blkif->rings[i];
+		ring->xenblkd = kthread_run(xen_blkif_schedule, ring, "%s-%d", name, i);
+		if (IS_ERR(ring->xenblkd)) {
+			err = PTR_ERR(ring->xenblkd);
+			ring->xenblkd = NULL;
+			xenbus_dev_fatal(blkif->be->dev, err,
+					"start %s-%d xenblkd", name, i);
+			goto out;
+		}
+	}
+	return;
+
+out:
+	while (--i >= 0) {
+		ring = &blkif->rings[i];
+		kthread_stop(ring->xenblkd);
+	}
+	return;
+}
+
+static int xen_blkif_alloc_rings(struct xen_blkif *blkif)
+{
+	unsigned int r;
+
+	blkif->rings = kzalloc(blkif->nr_rings * sizeof(struct xen_blkif_ring), GFP_KERNEL);
+	if (!blkif->rings)
+		return -ENOMEM;
+
+	for (r = 0; r < blkif->nr_rings; r++) {
+		struct xen_blkif_ring *ring = &blkif->rings[r];
+
+		spin_lock_init(&ring->blk_ring_lock);
+		init_waitqueue_head(&ring->wq);
+		INIT_LIST_HEAD(&ring->pending_free);
+		INIT_LIST_HEAD(&ring->persistent_purge_list);
+		INIT_WORK(&ring->persistent_purge_work, xen_blkbk_unmap_purged_grants);
+		spin_lock_init(&ring->free_pages_lock);
+		INIT_LIST_HEAD(&ring->free_pages);
+
+		spin_lock_init(&ring->pending_free_lock);
+		init_waitqueue_head(&ring->pending_free_wq);
+		init_waitqueue_head(&ring->shutdown_wq);
+		ring->blkif = blkif;
+		ring->st_print = jiffies;
+		xen_blkif_get(blkif);
 	}
+
+	return 0;
 }
 
 static struct xen_blkif *xen_blkif_alloc(domid_t domid)
@@ -133,41 +177,25 @@ static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 		return ERR_PTR(-ENOMEM);
 
 	blkif->domid = domid;
-	spin_lock_init(&blkif->blk_ring_lock);
 	atomic_set(&blkif->refcnt, 1);
-	init_waitqueue_head(&blkif->wq);
 	init_completion(&blkif->drain_complete);
-	atomic_set(&blkif->drain, 0);
-	blkif->st_print = jiffies;
-	blkif->persistent_gnts.rb_node = NULL;
-	spin_lock_init(&blkif->free_pages_lock);
-	INIT_LIST_HEAD(&blkif->free_pages);
-	INIT_LIST_HEAD(&blkif->persistent_purge_list);
-	blkif->free_pages_num = 0;
-	atomic_set(&blkif->persistent_gnt_in_use, 0);
-	atomic_set(&blkif->inflight, 0);
-	INIT_WORK(&blkif->persistent_purge_work, xen_blkbk_unmap_purged_grants);
-
-	INIT_LIST_HEAD(&blkif->pending_free);
 	INIT_WORK(&blkif->free_work, xen_blkif_deferred_free);
-	spin_lock_init(&blkif->pending_free_lock);
-	init_waitqueue_head(&blkif->pending_free_wq);
-	init_waitqueue_head(&blkif->shutdown_wq);
 
 	return blkif;
 }
 
-static int xen_blkif_map(struct xen_blkif *blkif, grant_ref_t *gref,
+static int xen_blkif_map(struct xen_blkif_ring *ring, grant_ref_t *gref,
 			 unsigned int nr_grefs, unsigned int evtchn)
 {
 	int err;
+	struct xen_blkif *blkif = ring->blkif;
 
 	/* Already connected through? */
-	if (blkif->irq)
+	if (ring->irq)
 		return 0;
 
 	err = xenbus_map_ring_valloc(blkif->be->dev, gref, nr_grefs,
-				     &blkif->blk_ring);
+				     &ring->blk_ring);
 	if (err < 0)
 		return err;
 
@@ -175,24 +203,24 @@ static int xen_blkif_map(struct xen_blkif *blkif, grant_ref_t *gref,
 	case BLKIF_PROTOCOL_NATIVE:
 	{
 		struct blkif_sring *sring;
-		sring = (struct blkif_sring *)blkif->blk_ring;
-		BACK_RING_INIT(&blkif->blk_rings.native, sring,
+		sring = (struct blkif_sring *)ring->blk_ring;
+		BACK_RING_INIT(&ring->blk_rings.native, sring,
 			       XEN_PAGE_SIZE * nr_grefs);
 		break;
 	}
 	case BLKIF_PROTOCOL_X86_32:
 	{
 		struct blkif_x86_32_sring *sring_x86_32;
-		sring_x86_32 = (struct blkif_x86_32_sring *)blkif->blk_ring;
-		BACK_RING_INIT(&blkif->blk_rings.x86_32, sring_x86_32,
+		sring_x86_32 = (struct blkif_x86_32_sring *)ring->blk_ring;
+		BACK_RING_INIT(&ring->blk_rings.x86_32, sring_x86_32,
 			       XEN_PAGE_SIZE * nr_grefs);
 		break;
 	}
 	case BLKIF_PROTOCOL_X86_64:
 	{
 		struct blkif_x86_64_sring *sring_x86_64;
-		sring_x86_64 = (struct blkif_x86_64_sring *)blkif->blk_ring;
-		BACK_RING_INIT(&blkif->blk_rings.x86_64, sring_x86_64,
+		sring_x86_64 = (struct blkif_x86_64_sring *)ring->blk_ring;
+		BACK_RING_INIT(&ring->blk_rings.x86_64, sring_x86_64,
 			       XEN_PAGE_SIZE * nr_grefs);
 		break;
 	}
@@ -202,13 +230,13 @@ static int xen_blkif_map(struct xen_blkif *blkif, grant_ref_t *gref,
 
 	err = bind_interdomain_evtchn_to_irqhandler(blkif->domid, evtchn,
 						    xen_blkif_be_int, 0,
-						    "blkif-backend", blkif);
+						    "blkif-backend", ring);
 	if (err < 0) {
-		xenbus_unmap_ring_vfree(blkif->be->dev, blkif->blk_ring);
-		blkif->blk_rings.common.sring = NULL;
+		xenbus_unmap_ring_vfree(blkif->be->dev, ring->blk_ring);
+		ring->blk_rings.common.sring = NULL;
 		return err;
 	}
-	blkif->irq = err;
+	ring->irq = err;
 
 	return 0;
 }
@@ -216,50 +244,69 @@ static int xen_blkif_map(struct xen_blkif *blkif, grant_ref_t *gref,
 static int xen_blkif_disconnect(struct xen_blkif *blkif)
 {
 	struct pending_req *req, *n;
-	int i = 0, j;
+	unsigned int j, r;
 
-	if (blkif->xenblkd) {
-		kthread_stop(blkif->xenblkd);
-		wake_up(&blkif->shutdown_wq);
-		blkif->xenblkd = NULL;
-	}
+	for (r = 0; r < blkif->nr_rings; r++) {
+		struct xen_blkif_ring *ring = &blkif->rings[r];
+		unsigned int i = 0;
 
-	/* The above kthread_stop() guarantees that at this point we
-	 * don't have any discard_io or other_io requests. So, checking
-	 * for inflight IO is enough.
-	 */
-	if (atomic_read(&blkif->inflight) > 0)
-		return -EBUSY;
+		if (ring->xenblkd) {
+			kthread_stop(ring->xenblkd);
+			wake_up(&ring->shutdown_wq);
+			ring->xenblkd = NULL;
+		}
 
-	if (blkif->irq) {
-		unbind_from_irqhandler(blkif->irq, blkif);
-		blkif->irq = 0;
-	}
+		/* The above kthread_stop() guarantees that at this point we
+		 * don't have any discard_io or other_io requests. So, checking
+		 * for inflight IO is enough.
+		 */
+		if (atomic_read(&ring->inflight) > 0)
+			return -EBUSY;
 
-	if (blkif->blk_rings.common.sring) {
-		xenbus_unmap_ring_vfree(blkif->be->dev, blkif->blk_ring);
-		blkif->blk_rings.common.sring = NULL;
-	}
+		if (ring->irq) {
+			unbind_from_irqhandler(ring->irq, ring);
+			ring->irq = 0;
+		}
 
-	/* Remove all persistent grants and the cache of ballooned pages. */
-	xen_blkbk_free_caches(blkif);
+		if (ring->blk_rings.common.sring) {
+			xenbus_unmap_ring_vfree(blkif->be->dev, ring->blk_ring);
+			ring->blk_rings.common.sring = NULL;
+		}
 
-	/* Check that there is no request in use */
-	list_for_each_entry_safe(req, n, &blkif->pending_free, free_list) {
-		list_del(&req->free_list);
+		/* Remove all persistent grants and the cache of ballooned pages. */
+		xen_blkbk_free_caches(ring);
 
-		for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++)
-			kfree(req->segments[j]);
+		/* Check that there is no request in use */
+		list_for_each_entry_safe(req, n, &ring->pending_free, free_list) {
+			list_del(&req->free_list);
 
-		for (j = 0; j < MAX_INDIRECT_PAGES; j++)
-			kfree(req->indirect_pages[j]);
+			for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++)
+				kfree(req->segments[j]);
 
-		kfree(req);
-		i++;
-	}
+			for (j = 0; j < MAX_INDIRECT_PAGES; j++)
+				kfree(req->indirect_pages[j]);
+
+			kfree(req);
+			i++;
+		}
 
-	WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
+		BUG_ON(atomic_read(&ring->persistent_gnt_in_use) != 0);
+		BUG_ON(!list_empty(&ring->persistent_purge_list));
+		BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
+		BUG_ON(!list_empty(&ring->free_pages));
+		BUG_ON(ring->free_pages_num != 0);
+		BUG_ON(ring->persistent_gnt_c != 0);
+		WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
+		xen_blkif_put(blkif);
+	}
 	blkif->nr_ring_pages = 0;
+	/*
+	 * blkif->rings was allocated in connect_ring, so we should free it in
+	 * here.
+	 */
+	kfree(blkif->rings);
+	blkif->rings = NULL;
+	blkif->nr_rings = 0;
 
 	return 0;
 }
@@ -271,13 +318,6 @@ static void xen_blkif_free(struct xen_blkif *blkif)
 	xen_vbd_free(&blkif->vbd);
 
 	/* Make sure everything is drained before shutting down */
-	BUG_ON(blkif->persistent_gnt_c != 0);
-	BUG_ON(atomic_read(&blkif->persistent_gnt_in_use) != 0);
-	BUG_ON(blkif->free_pages_num != 0);
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
-	BUG_ON(!list_empty(&blkif->free_pages));
-	BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts));
-
 	kmem_cache_free(xen_blkif_cachep, blkif);
 }
 
@@ -296,25 +336,38 @@ int __init xen_blkif_interface_init(void)
  *  sysfs interface for VBD I/O requests
  */
 
-#define VBD_SHOW(name, format, args...)					\
+#define VBD_SHOW_ALLRING(name, format)					\
 	static ssize_t show_##name(struct device *_dev,			\
 				   struct device_attribute *attr,	\
 				   char *buf)				\
 	{								\
 		struct xenbus_device *dev = to_xenbus_device(_dev);	\
 		struct backend_info *be = dev_get_drvdata(&dev->dev);	\
+		struct xen_blkif *blkif = be->blkif;			\
+		unsigned int i;						\
+		unsigned long long result = 0;				\
 									\
-		return sprintf(buf, format, ##args);			\
+		if (!blkif->rings)				\
+			goto out;					\
+									\
+		for (i = 0; i < blkif->nr_rings; i++) {		\
+			struct xen_blkif_ring *ring = &blkif->rings[i];	\
+									\
+			result += ring->st_##name;			\
+		}							\
+									\
+out:									\
+		return sprintf(buf, format, result);			\
 	}								\
 	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 
-VBD_SHOW(oo_req,  "%llu\n", be->blkif->st_oo_req);
-VBD_SHOW(rd_req,  "%llu\n", be->blkif->st_rd_req);
-VBD_SHOW(wr_req,  "%llu\n", be->blkif->st_wr_req);
-VBD_SHOW(f_req,  "%llu\n", be->blkif->st_f_req);
-VBD_SHOW(ds_req,  "%llu\n", be->blkif->st_ds_req);
-VBD_SHOW(rd_sect, "%llu\n", be->blkif->st_rd_sect);
-VBD_SHOW(wr_sect, "%llu\n", be->blkif->st_wr_sect);
+VBD_SHOW_ALLRING(oo_req,  "%llu\n");
+VBD_SHOW_ALLRING(rd_req,  "%llu\n");
+VBD_SHOW_ALLRING(wr_req,  "%llu\n");
+VBD_SHOW_ALLRING(f_req,  "%llu\n");
+VBD_SHOW_ALLRING(ds_req,  "%llu\n");
+VBD_SHOW_ALLRING(rd_sect, "%llu\n");
+VBD_SHOW_ALLRING(wr_sect, "%llu\n");
 
 static struct attribute *xen_vbdstat_attrs[] = {
 	&dev_attr_oo_req.attr,
@@ -332,6 +385,18 @@ static struct attribute_group xen_vbdstat_group = {
 	.attrs = xen_vbdstat_attrs,
 };
 
+#define VBD_SHOW(name, format, args...)					\
+	static ssize_t show_##name(struct device *_dev,			\
+				   struct device_attribute *attr,	\
+				   char *buf)				\
+	{								\
+		struct xenbus_device *dev = to_xenbus_device(_dev);	\
+		struct backend_info *be = dev_get_drvdata(&dev->dev);	\
+									\
+		return sprintf(buf, format, ##args);			\
+	}								\
+	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
+
 VBD_SHOW(physical_device, "%x:%x\n", be->major, be->minor);
 VBD_SHOW(mode, "%s\n", be->mode);
 
@@ -440,11 +505,11 @@ static int xen_blkbk_remove(struct xenbus_device *dev)
 
 	dev_set_drvdata(&dev->dev, NULL);
 
-	if (be->blkif) {
+	if (be->blkif)
 		xen_blkif_disconnect(be->blkif);
-		xen_blkif_put(be->blkif);
-	}
 
+	/* Put the reference we set in xen_blkif_alloc(). */
+	xen_blkif_put(be->blkif);
 	kfree(be->mode);
 	kfree(be);
 	return 0;
@@ -553,6 +618,12 @@ static int xen_blkbk_probe(struct xenbus_device *dev,
 		goto fail;
 	}
 
+	/* Multi-queue: advertise how many queues are supported by us.*/
+	err = xenbus_printf(XBT_NIL, dev->nodename,
+			    "multi-queue-max-queues", "%u", xenblk_max_queues);
+	if (err)
+		pr_warn("Error writing multi-queue-max-queues\n");
+
 	/* setup back pointer */
 	be->blkif->be = be;
 
@@ -708,8 +779,14 @@ static void frontend_changed(struct xenbus_device *dev,
 		}
 
 		err = connect_ring(be);
-		if (err)
+		if (err) {
+			/*
+			 * Clean up so that memory resources can be used by
+			 * other devices. connect_ring reported already error.
+			 */
+			xen_blkif_disconnect(be->blkif);
 			break;
+		}
 		xen_update_blkif_status(be->blkif);
 		break;
 
@@ -825,50 +902,43 @@ again:
 	xenbus_transaction_end(xbt, 1);
 }
 
-
-static int connect_ring(struct backend_info *be)
+/*
+ * Each ring may have multi pages, depends on "ring-page-order".
+ */
+static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir)
 {
-	struct xenbus_device *dev = be->dev;
 	unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
-	unsigned int evtchn, nr_grefs, ring_page_order;
-	unsigned int pers_grants;
-	char protocol[64] = "";
 	struct pending_req *req, *n;
 	int err, i, j;
+	struct xen_blkif *blkif = ring->blkif;
+	struct xenbus_device *dev = blkif->be->dev;
+	unsigned int ring_page_order, nr_grefs, evtchn;
 
-	pr_debug("%s %s\n", __func__, dev->otherend);
-
-	err = xenbus_scanf(XBT_NIL, dev->otherend, "event-channel", "%u",
+	err = xenbus_scanf(XBT_NIL, dir, "event-channel", "%u",
 			  &evtchn);
 	if (err != 1) {
 		err = -EINVAL;
-		xenbus_dev_fatal(dev, err, "reading %s/event-channel",
-				 dev->otherend);
+		xenbus_dev_fatal(dev, err, "reading %s/event-channel", dir);
 		return err;
 	}
-	pr_info("event-channel %u\n", evtchn);
 
 	err = xenbus_scanf(XBT_NIL, dev->otherend, "ring-page-order", "%u",
 			  &ring_page_order);
 	if (err != 1) {
-		err = xenbus_scanf(XBT_NIL, dev->otherend, "ring-ref",
-				  "%u", &ring_ref[0]);
+		err = xenbus_scanf(XBT_NIL, dir, "ring-ref", "%u", &ring_ref[0]);
 		if (err != 1) {
 			err = -EINVAL;
-			xenbus_dev_fatal(dev, err, "reading %s/ring-ref",
-					 dev->otherend);
+			xenbus_dev_fatal(dev, err, "reading %s/ring-ref", dir);
 			return err;
 		}
 		nr_grefs = 1;
-		pr_info("%s:using single page: ring-ref %d\n", dev->otherend,
-			ring_ref[0]);
 	} else {
 		unsigned int i;
 
 		if (ring_page_order > xen_blkif_max_ring_order) {
 			err = -EINVAL;
 			xenbus_dev_fatal(dev, err, "%s/request %d ring page order exceed max:%d",
-					 dev->otherend, ring_page_order,
+					 dir, ring_page_order,
 					 xen_blkif_max_ring_order);
 			return err;
 		}
@@ -878,52 +948,23 @@ static int connect_ring(struct backend_info *be)
 			char ring_ref_name[RINGREF_NAME_LEN];
 
 			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
-			err = xenbus_scanf(XBT_NIL, dev->otherend, ring_ref_name,
+			err = xenbus_scanf(XBT_NIL, dir, ring_ref_name,
 					   "%u", &ring_ref[i]);
 			if (err != 1) {
 				err = -EINVAL;
 				xenbus_dev_fatal(dev, err, "reading %s/%s",
-						 dev->otherend, ring_ref_name);
+						 dir, ring_ref_name);
 				return err;
 			}
-			pr_info("ring-ref%u: %u\n", i, ring_ref[i]);
 		}
 	}
-
-	be->blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT;
-	err = xenbus_gather(XBT_NIL, dev->otherend, "protocol",
-			    "%63s", protocol, NULL);
-	if (err)
-		strcpy(protocol, "unspecified, assuming default");
-	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE))
-		be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE;
-	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_32))
-		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_32;
-	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_64))
-		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_64;
-	else {
-		xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol);
-		return -1;
-	}
-	err = xenbus_gather(XBT_NIL, dev->otherend,
-			    "feature-persistent", "%u",
-			    &pers_grants, NULL);
-	if (err)
-		pers_grants = 0;
-
-	be->blkif->vbd.feature_gnt_persistent = pers_grants;
-	be->blkif->vbd.overflow_max_grants = 0;
-	be->blkif->nr_ring_pages = nr_grefs;
-
-	pr_info("ring-pages:%d, event-channel %d, protocol %d (%s) %s\n",
-		nr_grefs, evtchn, be->blkif->blk_protocol, protocol,
-		pers_grants ? "persistent grants" : "");
+	blkif->nr_ring_pages = nr_grefs;
 
 	for (i = 0; i < nr_grefs * XEN_BLKIF_REQS_PER_PAGE; i++) {
 		req = kzalloc(sizeof(*req), GFP_KERNEL);
 		if (!req)
 			goto fail;
-		list_add_tail(&req->free_list, &be->blkif->pending_free);
+		list_add_tail(&req->free_list, &ring->pending_free);
 		for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++) {
 			req->segments[j] = kzalloc(sizeof(*req->segments[0]), GFP_KERNEL);
 			if (!req->segments[j])
@@ -938,7 +979,7 @@ static int connect_ring(struct backend_info *be)
 	}
 
 	/* Map the shared frame, irq etc. */
-	err = xen_blkif_map(be->blkif, ring_ref, nr_grefs, evtchn);
+	err = xen_blkif_map(ring, ring_ref, nr_grefs, evtchn);
 	if (err) {
 		xenbus_dev_fatal(dev, err, "mapping ring-ref port %u", evtchn);
 		return err;
@@ -947,7 +988,7 @@ static int connect_ring(struct backend_info *be)
 	return 0;
 
 fail:
-	list_for_each_entry_safe(req, n, &be->blkif->pending_free, free_list) {
+	list_for_each_entry_safe(req, n, &ring->pending_free, free_list) {
 		list_del(&req->free_list);
 		for (j = 0; j < MAX_INDIRECT_SEGMENTS; j++) {
 			if (!req->segments[j])
@@ -962,6 +1003,93 @@ fail:
 		kfree(req);
 	}
 	return -ENOMEM;
+
+}
+
+static int connect_ring(struct backend_info *be)
+{
+	struct xenbus_device *dev = be->dev;
+	unsigned int pers_grants;
+	char protocol[64] = "";
+	int err, i;
+	char *xspath;
+	size_t xspathsize;
+	const size_t xenstore_path_ext_size = 11; /* sufficient for "/queue-NNN" */
+	unsigned int requested_num_queues = 0;
+
+	pr_debug("%s %s\n", __func__, dev->otherend);
+
+	be->blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT;
+	err = xenbus_gather(XBT_NIL, dev->otherend, "protocol",
+			    "%63s", protocol, NULL);
+	if (err)
+		strcpy(protocol, "unspecified, assuming default");
+	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE))
+		be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE;
+	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_32))
+		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_32;
+	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_64))
+		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_64;
+	else {
+		xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol);
+		return -ENOSYS;
+	}
+	err = xenbus_gather(XBT_NIL, dev->otherend,
+			    "feature-persistent", "%u",
+			    &pers_grants, NULL);
+	if (err)
+		pers_grants = 0;
+
+	be->blkif->vbd.feature_gnt_persistent = pers_grants;
+	be->blkif->vbd.overflow_max_grants = 0;
+
+	/*
+	 * Read the number of hardware queues from frontend.
+	 */
+	err = xenbus_scanf(XBT_NIL, dev->otherend, "multi-queue-num-queues",
+			   "%u", &requested_num_queues);
+	if (err < 0) {
+		requested_num_queues = 1;
+	} else {
+		if (requested_num_queues > xenblk_max_queues
+		    || requested_num_queues == 0) {
+			/* Buggy or malicious guest. */
+			xenbus_dev_fatal(dev, err,
+					"guest requested %u queues, exceeding the maximum of %u.",
+					requested_num_queues, xenblk_max_queues);
+			return -ENOSYS;
+		}
+	}
+	be->blkif->nr_rings = requested_num_queues;
+	if (xen_blkif_alloc_rings(be->blkif))
+		return -ENOMEM;
+
+	pr_info("%s: using %d queues, protocol %d (%s) %s\n", dev->nodename,
+		 be->blkif->nr_rings, be->blkif->blk_protocol, protocol,
+		 pers_grants ? "persistent grants" : "");
+
+	if (be->blkif->nr_rings == 1)
+		return read_per_ring_refs(&be->blkif->rings[0], dev->otherend);
+	else {
+		xspathsize = strlen(dev->otherend) + xenstore_path_ext_size;
+		xspath = kmalloc(xspathsize, GFP_KERNEL);
+		if (!xspath) {
+			xenbus_dev_fatal(dev, -ENOMEM, "reading ring references");
+			return -ENOMEM;
+		}
+
+		for (i = 0; i < be->blkif->nr_rings; i++) {
+			memset(xspath, 0, xspathsize);
+			snprintf(xspath, xspathsize, "%s/queue-%u", dev->otherend, i);
+			err = read_per_ring_refs(&be->blkif->rings[i], xspath);
+			if (err) {
+				kfree(xspath);
+				return err;
+			}
+		}
+		kfree(xspath);
+	}
+	return 0;
 }
 
 static const struct xenbus_device_id xen_blkbk_ids[] = {
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 2fee2eef988d..8a8dc91c39f7 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -60,6 +60,20 @@
 
 #include <asm/xen/hypervisor.h>
 
+/*
+ * The minimal size of segment supported by the block framework is PAGE_SIZE.
+ * When Linux is using a different page size than Xen, it may not be possible
+ * to put all the data in a single segment.
+ * This can happen when the backend doesn't support indirect descriptor and
+ * therefore the maximum amount of data that a request can carry is
+ * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE = 44KB
+ *
+ * Note that we only support one extra request. So the Linux page size
+ * should be <= ( 2 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) =
+ * 88KB.
+ */
+#define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE)
+
 enum blkif_state {
 	BLKIF_STATE_DISCONNECTED,
 	BLKIF_STATE_CONNECTED,
@@ -72,6 +86,13 @@ struct grant {
 	struct list_head node;
 };
 
+enum blk_req_status {
+	REQ_WAITING,
+	REQ_DONE,
+	REQ_ERROR,
+	REQ_EOPNOTSUPP,
+};
+
 struct blk_shadow {
 	struct blkif_request req;
 	struct request *request;
@@ -79,6 +100,14 @@ struct blk_shadow {
 	struct grant **indirect_grants;
 	struct scatterlist *sg;
 	unsigned int num_sg;
+	enum blk_req_status status;
+
+	#define NO_ASSOCIATED_ID ~0UL
+	/*
+	 * Id of the sibling if we ever need 2 requests when handling a
+	 * block I/O request
+	 */
+	unsigned long associated_id;
 };
 
 struct split_bio {
@@ -99,6 +128,10 @@ static unsigned int xen_blkif_max_segments = 32;
 module_param_named(max, xen_blkif_max_segments, int, S_IRUGO);
 MODULE_PARM_DESC(max, "Maximum amount of segments in indirect requests (default is 32)");
 
+static unsigned int xen_blkif_max_queues = 4;
+module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
+MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");
+
 /*
  * Maximum order of pages to be used for the shared ring between front and
  * backend, 4KB page granularity is used.
@@ -114,10 +147,35 @@ MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the
 	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS)
 
 /*
- * ring-ref%i i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19
- * characters are enough. Define to 20 to keep consist with backend.
+ * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19
+ * characters are enough. Define to 20 to keep consistent with backend.
  */
 #define RINGREF_NAME_LEN (20)
+/*
+ * queue-%u would take 7 + 10(UINT_MAX) = 17 characters.
+ */
+#define QUEUE_NAME_LEN (17)
+
+/*
+ *  Per-ring info.
+ *  Every blkfront device can associate with one or more blkfront_ring_info,
+ *  depending on how many hardware queues/rings to be used.
+ */
+struct blkfront_ring_info {
+	/* Lock to protect data in every ring buffer. */
+	spinlock_t ring_lock;
+	struct blkif_front_ring ring;
+	unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
+	unsigned int evtchn, irq;
+	struct work_struct work;
+	struct gnttab_free_callback callback;
+	struct blk_shadow shadow[BLK_MAX_RING_SIZE];
+	struct list_head indirect_pages;
+	struct list_head grants;
+	unsigned int persistent_gnts_c;
+	unsigned long shadow_free;
+	struct blkfront_info *dev_info;
+};
 
 /*
  * We have one of these per vbd, whether ide, scsi or 'other'.  They
@@ -126,25 +184,15 @@ MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the
  */
 struct blkfront_info
 {
-	spinlock_t io_lock;
 	struct mutex mutex;
 	struct xenbus_device *xbdev;
 	struct gendisk *gd;
 	int vdevice;
 	blkif_vdev_t handle;
 	enum blkif_state connected;
-	int ring_ref[XENBUS_MAX_RING_GRANTS];
+	/* Number of pages per ring buffer. */
 	unsigned int nr_ring_pages;
-	struct blkif_front_ring ring;
-	unsigned int evtchn, irq;
 	struct request_queue *rq;
-	struct work_struct work;
-	struct gnttab_free_callback callback;
-	struct blk_shadow shadow[BLK_MAX_RING_SIZE];
-	struct list_head grants;
-	struct list_head indirect_pages;
-	unsigned int persistent_gnts_c;
-	unsigned long shadow_free;
 	unsigned int feature_flush;
 	unsigned int feature_discard:1;
 	unsigned int feature_secdiscard:1;
@@ -155,6 +203,8 @@ struct blkfront_info
 	unsigned int max_indirect_segments;
 	int is_ready;
 	struct blk_mq_tag_set tag_set;
+	struct blkfront_ring_info *rinfo;
+	unsigned int nr_rings;
 };
 
 static unsigned int nr_minors;
@@ -198,38 +248,40 @@ static DEFINE_SPINLOCK(minor_lock);
 
 #define GREFS(_psegs)	((_psegs) * GRANTS_PER_PSEG)
 
-static int blkfront_setup_indirect(struct blkfront_info *info);
-static int blkfront_gather_backend_features(struct blkfront_info *info);
+static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
+static void blkfront_gather_backend_features(struct blkfront_info *info);
 
-static int get_id_from_freelist(struct blkfront_info *info)
+static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
 {
-	unsigned long free = info->shadow_free;
-	BUG_ON(free >= BLK_RING_SIZE(info));
-	info->shadow_free = info->shadow[free].req.u.rw.id;
-	info->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
+	unsigned long free = rinfo->shadow_free;
+
+	BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info));
+	rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id;
+	rinfo->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
 	return free;
 }
 
-static int add_id_to_freelist(struct blkfront_info *info,
-			       unsigned long id)
+static int add_id_to_freelist(struct blkfront_ring_info *rinfo,
+			      unsigned long id)
 {
-	if (info->shadow[id].req.u.rw.id != id)
+	if (rinfo->shadow[id].req.u.rw.id != id)
 		return -EINVAL;
-	if (info->shadow[id].request == NULL)
+	if (rinfo->shadow[id].request == NULL)
 		return -EINVAL;
-	info->shadow[id].req.u.rw.id  = info->shadow_free;
-	info->shadow[id].request = NULL;
-	info->shadow_free = id;
+	rinfo->shadow[id].req.u.rw.id  = rinfo->shadow_free;
+	rinfo->shadow[id].request = NULL;
+	rinfo->shadow_free = id;
 	return 0;
 }
 
-static int fill_grant_buffer(struct blkfront_info *info, int num)
+static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
 {
+	struct blkfront_info *info = rinfo->dev_info;
 	struct page *granted_page;
 	struct grant *gnt_list_entry, *n;
 	int i = 0;
 
-	while(i < num) {
+	while (i < num) {
 		gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
 		if (!gnt_list_entry)
 			goto out_of_memory;
@@ -244,7 +296,7 @@ static int fill_grant_buffer(struct blkfront_info *info, int num)
 		}
 
 		gnt_list_entry->gref = GRANT_INVALID_REF;
-		list_add(&gnt_list_entry->node, &info->grants);
+		list_add(&gnt_list_entry->node, &rinfo->grants);
 		i++;
 	}
 
@@ -252,7 +304,7 @@ static int fill_grant_buffer(struct blkfront_info *info, int num)
 
 out_of_memory:
 	list_for_each_entry_safe(gnt_list_entry, n,
-	                         &info->grants, node) {
+	                         &rinfo->grants, node) {
 		list_del(&gnt_list_entry->node);
 		if (info->feature_persistent)
 			__free_page(gnt_list_entry->page);
@@ -263,17 +315,17 @@ out_of_memory:
 	return -ENOMEM;
 }
 
-static struct grant *get_free_grant(struct blkfront_info *info)
+static struct grant *get_free_grant(struct blkfront_ring_info *rinfo)
 {
 	struct grant *gnt_list_entry;
 
-	BUG_ON(list_empty(&info->grants));
-	gnt_list_entry = list_first_entry(&info->grants, struct grant,
+	BUG_ON(list_empty(&rinfo->grants));
+	gnt_list_entry = list_first_entry(&rinfo->grants, struct grant,
 					  node);
 	list_del(&gnt_list_entry->node);
 
 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
-		info->persistent_gnts_c--;
+		rinfo->persistent_gnts_c--;
 
 	return gnt_list_entry;
 }
@@ -289,9 +341,10 @@ static inline void grant_foreign_access(const struct grant *gnt_list_entry,
 
 static struct grant *get_grant(grant_ref_t *gref_head,
 			       unsigned long gfn,
-			       struct blkfront_info *info)
+			       struct blkfront_ring_info *rinfo)
 {
-	struct grant *gnt_list_entry = get_free_grant(info);
+	struct grant *gnt_list_entry = get_free_grant(rinfo);
+	struct blkfront_info *info = rinfo->dev_info;
 
 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
 		return gnt_list_entry;
@@ -312,9 +365,10 @@ static struct grant *get_grant(grant_ref_t *gref_head,
 }
 
 static struct grant *get_indirect_grant(grant_ref_t *gref_head,
-					struct blkfront_info *info)
+					struct blkfront_ring_info *rinfo)
 {
-	struct grant *gnt_list_entry = get_free_grant(info);
+	struct grant *gnt_list_entry = get_free_grant(rinfo);
+	struct blkfront_info *info = rinfo->dev_info;
 
 	if (gnt_list_entry->gref != GRANT_INVALID_REF)
 		return gnt_list_entry;
@@ -326,8 +380,8 @@ static struct grant *get_indirect_grant(grant_ref_t *gref_head,
 		struct page *indirect_page;
 
 		/* Fetch a pre-allocated page to use for indirect grefs */
-		BUG_ON(list_empty(&info->indirect_pages));
-		indirect_page = list_first_entry(&info->indirect_pages,
+		BUG_ON(list_empty(&rinfo->indirect_pages));
+		indirect_page = list_first_entry(&rinfo->indirect_pages,
 						 struct page, lru);
 		list_del(&indirect_page->lru);
 		gnt_list_entry->page = indirect_page;
@@ -403,8 +457,8 @@ static void xlbd_release_minors(unsigned int minor, unsigned int nr)
 
 static void blkif_restart_queue_callback(void *arg)
 {
-	struct blkfront_info *info = (struct blkfront_info *)arg;
-	schedule_work(&info->work);
+	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg;
+	schedule_work(&rinfo->work);
 }
 
 static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
@@ -456,16 +510,33 @@ static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
 	return 0;
 }
 
-static int blkif_queue_discard_req(struct request *req)
+static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo,
+					    struct request *req,
+					    struct blkif_request **ring_req)
 {
-	struct blkfront_info *info = req->rq_disk->private_data;
+	unsigned long id;
+
+	*ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt);
+	rinfo->ring.req_prod_pvt++;
+
+	id = get_id_from_freelist(rinfo);
+	rinfo->shadow[id].request = req;
+	rinfo->shadow[id].status = REQ_WAITING;
+	rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID;
+
+	(*ring_req)->u.rw.id = id;
+
+	return id;
+}
+
+static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo)
+{
+	struct blkfront_info *info = rinfo->dev_info;
 	struct blkif_request *ring_req;
 	unsigned long id;
 
 	/* Fill out a communications ring structure. */
-	ring_req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt);
-	id = get_id_from_freelist(info);
-	info->shadow[id].request = req;
+	id = blkif_ring_get_request(rinfo, req, &ring_req);
 
 	ring_req->operation = BLKIF_OP_DISCARD;
 	ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
@@ -476,10 +547,8 @@ static int blkif_queue_discard_req(struct request *req)
 	else
 		ring_req->u.discard.flag = 0;
 
-	info->ring.req_prod_pvt++;
-
 	/* Keep a private copy so we can reissue requests when recovering. */
-	info->shadow[id].req = *ring_req;
+	rinfo->shadow[id].req = *ring_req;
 
 	return 0;
 }
@@ -487,7 +556,7 @@ static int blkif_queue_discard_req(struct request *req)
 struct setup_rw_req {
 	unsigned int grant_idx;
 	struct blkif_request_segment *segments;
-	struct blkfront_info *info;
+	struct blkfront_ring_info *rinfo;
 	struct blkif_request *ring_req;
 	grant_ref_t gref_head;
 	unsigned int id;
@@ -495,6 +564,9 @@ struct setup_rw_req {
 	bool need_copy;
 	unsigned int bvec_off;
 	char *bvec_data;
+
+	bool require_extra_req;
+	struct blkif_request *extra_ring_req;
 };
 
 static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
@@ -507,8 +579,24 @@ static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
 	/* Convenient aliases */
 	unsigned int grant_idx = setup->grant_idx;
 	struct blkif_request *ring_req = setup->ring_req;
-	struct blkfront_info *info = setup->info;
-	struct blk_shadow *shadow = &info->shadow[setup->id];
+	struct blkfront_ring_info *rinfo = setup->rinfo;
+	/*
+	 * We always use the shadow of the first request to store the list
+	 * of grant associated to the block I/O request. This made the
+	 * completion more easy to handle even if the block I/O request is
+	 * split.
+	 */
+	struct blk_shadow *shadow = &rinfo->shadow[setup->id];
+
+	if (unlikely(setup->require_extra_req &&
+		     grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
+		/*
+		 * We are using the second request, setup grant_idx
+		 * to be the index of the segment array.
+		 */
+		grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST;
+		ring_req = setup->extra_ring_req;
+	}
 
 	if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
 	    (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) {
@@ -516,15 +604,19 @@ static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
 			kunmap_atomic(setup->segments);
 
 		n = grant_idx / GRANTS_PER_INDIRECT_FRAME;
-		gnt_list_entry = get_indirect_grant(&setup->gref_head, info);
+		gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo);
 		shadow->indirect_grants[n] = gnt_list_entry;
 		setup->segments = kmap_atomic(gnt_list_entry->page);
 		ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
 	}
 
-	gnt_list_entry = get_grant(&setup->gref_head, gfn, info);
+	gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo);
 	ref = gnt_list_entry->gref;
-	shadow->grants_used[grant_idx] = gnt_list_entry;
+	/*
+	 * All the grants are stored in the shadow of the first
+	 * request. Therefore we have to use the global index.
+	 */
+	shadow->grants_used[setup->grant_idx] = gnt_list_entry;
 
 	if (setup->need_copy) {
 		void *shared_data;
@@ -566,16 +658,36 @@ static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
 	(setup->grant_idx)++;
 }
 
-static int blkif_queue_rw_req(struct request *req)
+static void blkif_setup_extra_req(struct blkif_request *first,
+				  struct blkif_request *second)
 {
-	struct blkfront_info *info = req->rq_disk->private_data;
-	struct blkif_request *ring_req;
-	unsigned long id;
+	uint16_t nr_segments = first->u.rw.nr_segments;
+
+	/*
+	 * The second request is only present when the first request uses
+	 * all its segments. It's always the continuity of the first one.
+	 */
+	first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;
+
+	second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST;
+	second->u.rw.sector_number = first->u.rw.sector_number +
+		(BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512;
+
+	second->u.rw.handle = first->u.rw.handle;
+	second->operation = first->operation;
+}
+
+static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo)
+{
+	struct blkfront_info *info = rinfo->dev_info;
+	struct blkif_request *ring_req, *extra_ring_req = NULL;
+	unsigned long id, extra_id = NO_ASSOCIATED_ID;
+	bool require_extra_req = false;
 	int i;
 	struct setup_rw_req setup = {
 		.grant_idx = 0,
 		.segments = NULL,
-		.info = info,
+		.rinfo = rinfo,
 		.need_copy = rq_data_dir(req) && info->feature_persistent,
 	};
 
@@ -584,7 +696,6 @@ static int blkif_queue_rw_req(struct request *req)
 	 * existing persistent grants, or if we have to get new grants,
 	 * as there are not sufficiently many free.
 	 */
-	bool new_persistent_gnts;
 	struct scatterlist *sg;
 	int num_sg, max_grefs, num_grant;
 
@@ -596,41 +707,36 @@ static int blkif_queue_rw_req(struct request *req)
 		 */
 		max_grefs += INDIRECT_GREFS(max_grefs);
 
-	/* Check if we have enough grants to allocate a requests */
-	if (info->persistent_gnts_c < max_grefs) {
-		new_persistent_gnts = 1;
-		if (gnttab_alloc_grant_references(
-		    max_grefs - info->persistent_gnts_c,
-		    &setup.gref_head) < 0) {
+	/*
+	 * We have to reserve 'max_grefs' grants because persistent
+	 * grants are shared by all rings.
+	 */
+	if (max_grefs > 0)
+		if (gnttab_alloc_grant_references(max_grefs, &setup.gref_head) < 0) {
 			gnttab_request_free_callback(
-				&info->callback,
+				&rinfo->callback,
 				blkif_restart_queue_callback,
-				info,
+				rinfo,
 				max_grefs);
 			return 1;
 		}
-	} else
-		new_persistent_gnts = 0;
 
 	/* Fill out a communications ring structure. */
-	ring_req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt);
-	id = get_id_from_freelist(info);
-	info->shadow[id].request = req;
-
-	BUG_ON(info->max_indirect_segments == 0 &&
-	       GREFS(req->nr_phys_segments) > BLKIF_MAX_SEGMENTS_PER_REQUEST);
-	BUG_ON(info->max_indirect_segments &&
-	       GREFS(req->nr_phys_segments) > info->max_indirect_segments);
+	id = blkif_ring_get_request(rinfo, req, &ring_req);
 
-	num_sg = blk_rq_map_sg(req->q, req, info->shadow[id].sg);
+	num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg);
 	num_grant = 0;
 	/* Calculate the number of grant used */
-	for_each_sg(info->shadow[id].sg, sg, num_sg, i)
+	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i)
 	       num_grant += gnttab_count_grant(sg->offset, sg->length);
 
-	ring_req->u.rw.id = id;
-	info->shadow[id].num_sg = num_sg;
-	if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+	require_extra_req = info->max_indirect_segments == 0 &&
+		num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST;
+	BUG_ON(!HAS_EXTRA_REQ && require_extra_req);
+
+	rinfo->shadow[id].num_sg = num_sg;
+	if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST &&
+	    likely(!require_extra_req)) {
 		/*
 		 * The indirect operation can only be a BLKIF_OP_READ or
 		 * BLKIF_OP_WRITE
@@ -670,11 +776,31 @@ static int blkif_queue_rw_req(struct request *req)
 			}
 		}
 		ring_req->u.rw.nr_segments = num_grant;
+		if (unlikely(require_extra_req)) {
+			extra_id = blkif_ring_get_request(rinfo, req,
+							  &extra_ring_req);
+			/*
+			 * Only the first request contains the scatter-gather
+			 * list.
+			 */
+			rinfo->shadow[extra_id].num_sg = 0;
+
+			blkif_setup_extra_req(ring_req, extra_ring_req);
+
+			/* Link the 2 requests together */
+			rinfo->shadow[extra_id].associated_id = id;
+			rinfo->shadow[id].associated_id = extra_id;
+		}
 	}
 
 	setup.ring_req = ring_req;
 	setup.id = id;
-	for_each_sg(info->shadow[id].sg, sg, num_sg, i) {
+
+	setup.require_extra_req = require_extra_req;
+	if (unlikely(require_extra_req))
+		setup.extra_ring_req = extra_ring_req;
+
+	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) {
 		BUG_ON(sg->offset + sg->length > PAGE_SIZE);
 
 		if (setup.need_copy) {
@@ -694,12 +820,12 @@ static int blkif_queue_rw_req(struct request *req)
 	if (setup.segments)
 		kunmap_atomic(setup.segments);
 
-	info->ring.req_prod_pvt++;
-
 	/* Keep a private copy so we can reissue requests when recovering. */
-	info->shadow[id].req = *ring_req;
+	rinfo->shadow[id].req = *ring_req;
+	if (unlikely(require_extra_req))
+		rinfo->shadow[extra_id].req = *extra_ring_req;
 
-	if (new_persistent_gnts)
+	if (max_grefs > 0)
 		gnttab_free_grant_references(setup.gref_head);
 
 	return 0;
@@ -711,27 +837,25 @@ static int blkif_queue_rw_req(struct request *req)
  *
  * @req: a request struct
  */
-static int blkif_queue_request(struct request *req)
+static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo)
 {
-	struct blkfront_info *info = req->rq_disk->private_data;
-
-	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
+	if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED))
 		return 1;
 
 	if (unlikely(req->cmd_flags & (REQ_DISCARD | REQ_SECURE)))
-		return blkif_queue_discard_req(req);
+		return blkif_queue_discard_req(req, rinfo);
 	else
-		return blkif_queue_rw_req(req);
+		return blkif_queue_rw_req(req, rinfo);
 }
 
-static inline void flush_requests(struct blkfront_info *info)
+static inline void flush_requests(struct blkfront_ring_info *rinfo)
 {
 	int notify;
 
-	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&info->ring, notify);
+	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify);
 
 	if (notify)
-		notify_remote_via_irq(info->irq);
+		notify_remote_via_irq(rinfo->irq);
 }
 
 static inline bool blkif_request_flush_invalid(struct request *req,
@@ -745,38 +869,50 @@ static inline bool blkif_request_flush_invalid(struct request *req,
 }
 
 static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
-			   const struct blk_mq_queue_data *qd)
+			  const struct blk_mq_queue_data *qd)
 {
-	struct blkfront_info *info = qd->rq->rq_disk->private_data;
+	unsigned long flags;
+	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)hctx->driver_data;
 
 	blk_mq_start_request(qd->rq);
-	spin_lock_irq(&info->io_lock);
-	if (RING_FULL(&info->ring))
+	spin_lock_irqsave(&rinfo->ring_lock, flags);
+	if (RING_FULL(&rinfo->ring))
 		goto out_busy;
 
-	if (blkif_request_flush_invalid(qd->rq, info))
+	if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info))
 		goto out_err;
 
-	if (blkif_queue_request(qd->rq))
+	if (blkif_queue_request(qd->rq, rinfo))
 		goto out_busy;
 
-	flush_requests(info);
-	spin_unlock_irq(&info->io_lock);
+	flush_requests(rinfo);
+	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
 	return BLK_MQ_RQ_QUEUE_OK;
 
 out_err:
-	spin_unlock_irq(&info->io_lock);
+	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
 	return BLK_MQ_RQ_QUEUE_ERROR;
 
 out_busy:
-	spin_unlock_irq(&info->io_lock);
+	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
 	blk_mq_stop_hw_queue(hctx);
 	return BLK_MQ_RQ_QUEUE_BUSY;
 }
 
+static int blk_mq_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+			    unsigned int index)
+{
+	struct blkfront_info *info = (struct blkfront_info *)data;
+
+	BUG_ON(info->nr_rings <= index);
+	hctx->driver_data = &info->rinfo[index];
+	return 0;
+}
+
 static struct blk_mq_ops blkfront_mq_ops = {
 	.queue_rq = blkif_queue_rq,
 	.map_queue = blk_mq_map_queue,
+	.init_hctx = blk_mq_init_hctx,
 };
 
 static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
@@ -788,19 +924,28 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
 
 	memset(&info->tag_set, 0, sizeof(info->tag_set));
 	info->tag_set.ops = &blkfront_mq_ops;
-	info->tag_set.nr_hw_queues = 1;
-	info->tag_set.queue_depth =  BLK_RING_SIZE(info);
+	info->tag_set.nr_hw_queues = info->nr_rings;
+	if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) {
+		/*
+		 * When indirect descriptior is not supported, the I/O request
+		 * will be split between multiple request in the ring.
+		 * To avoid problems when sending the request, divide by
+		 * 2 the depth of the queue.
+		 */
+		info->tag_set.queue_depth =  BLK_RING_SIZE(info) / 2;
+	} else
+		info->tag_set.queue_depth = BLK_RING_SIZE(info);
 	info->tag_set.numa_node = NUMA_NO_NODE;
 	info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
 	info->tag_set.cmd_size = 0;
 	info->tag_set.driver_data = info;
 
 	if (blk_mq_alloc_tag_set(&info->tag_set))
-		return -1;
+		return -EINVAL;
 	rq = blk_mq_init_queue(&info->tag_set);
 	if (IS_ERR(rq)) {
 		blk_mq_free_tag_set(&info->tag_set);
-		return -1;
+		return PTR_ERR(rq);
 	}
 
 	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
@@ -1028,7 +1173,7 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
 
 static void xlvbd_release_gendisk(struct blkfront_info *info)
 {
-	unsigned int minor, nr_minors;
+	unsigned int minor, nr_minors, i;
 
 	if (info->rq == NULL)
 		return;
@@ -1036,11 +1181,15 @@ static void xlvbd_release_gendisk(struct blkfront_info *info)
 	/* No more blkif_request(). */
 	blk_mq_stop_hw_queues(info->rq);
 
-	/* No more gnttab callback work. */
-	gnttab_cancel_free_callback(&info->callback);
+	for (i = 0; i < info->nr_rings; i++) {
+		struct blkfront_ring_info *rinfo = &info->rinfo[i];
 
-	/* Flush gnttab callback work. Must be done with no locks held. */
-	flush_work(&info->work);
+		/* No more gnttab callback work. */
+		gnttab_cancel_free_callback(&rinfo->callback);
+
+		/* Flush gnttab callback work. Must be done with no locks held. */
+		flush_work(&rinfo->work);
+	}
 
 	del_gendisk(info->gd);
 
@@ -1056,88 +1205,87 @@ static void xlvbd_release_gendisk(struct blkfront_info *info)
 	info->gd = NULL;
 }
 
-/* Must be called with io_lock holded */
-static void kick_pending_request_queues(struct blkfront_info *info)
+/* Already hold rinfo->ring_lock. */
+static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
 {
-	if (!RING_FULL(&info->ring))
-		blk_mq_start_stopped_hw_queues(info->rq, true);
+	if (!RING_FULL(&rinfo->ring))
+		blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
 }
 
-static void blkif_restart_queue(struct work_struct *work)
+static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
 {
-	struct blkfront_info *info = container_of(work, struct blkfront_info, work);
+	unsigned long flags;
 
-	spin_lock_irq(&info->io_lock);
-	if (info->connected == BLKIF_STATE_CONNECTED)
-		kick_pending_request_queues(info);
-	spin_unlock_irq(&info->io_lock);
+	spin_lock_irqsave(&rinfo->ring_lock, flags);
+	kick_pending_request_queues_locked(rinfo);
+	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
 }
 
-static void blkif_free(struct blkfront_info *info, int suspend)
+static void blkif_restart_queue(struct work_struct *work)
 {
-	struct grant *persistent_gnt;
-	struct grant *n;
-	int i, j, segs;
+	struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
 
-	/* Prevent new requests being issued until we fix things up. */
-	spin_lock_irq(&info->io_lock);
-	info->connected = suspend ?
-		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
-	/* No more blkif_request(). */
-	if (info->rq)
-		blk_mq_stop_hw_queues(info->rq);
+	if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
+		kick_pending_request_queues(rinfo);
+}
 
-	/* Remove all persistent grants */
-	if (!list_empty(&info->grants)) {
-		list_for_each_entry_safe(persistent_gnt, n,
-		                         &info->grants, node) {
-			list_del(&persistent_gnt->node);
-			if (persistent_gnt->gref != GRANT_INVALID_REF) {
-				gnttab_end_foreign_access(persistent_gnt->gref,
-				                          0, 0UL);
-				info->persistent_gnts_c--;
-			}
-			if (info->feature_persistent)
-				__free_page(persistent_gnt->page);
-			kfree(persistent_gnt);
-		}
-	}
-	BUG_ON(info->persistent_gnts_c != 0);
+static void blkif_free_ring(struct blkfront_ring_info *rinfo)
+{
+	struct grant *persistent_gnt, *n;
+	struct blkfront_info *info = rinfo->dev_info;
+	int i, j, segs;
 
 	/*
 	 * Remove indirect pages, this only happens when using indirect
 	 * descriptors but not persistent grants
 	 */
-	if (!list_empty(&info->indirect_pages)) {
+	if (!list_empty(&rinfo->indirect_pages)) {
 		struct page *indirect_page, *n;
 
 		BUG_ON(info->feature_persistent);
-		list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
+		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
 			list_del(&indirect_page->lru);
 			__free_page(indirect_page);
 		}
 	}
 
+	/* Remove all persistent grants. */
+	if (!list_empty(&rinfo->grants)) {
+		list_for_each_entry_safe(persistent_gnt, n,
+					 &rinfo->grants, node) {
+			list_del(&persistent_gnt->node);
+			if (persistent_gnt->gref != GRANT_INVALID_REF) {
+				gnttab_end_foreign_access(persistent_gnt->gref,
+							  0, 0UL);
+				rinfo->persistent_gnts_c--;
+			}
+			if (info->feature_persistent)
+				__free_page(persistent_gnt->page);
+			kfree(persistent_gnt);
+		}
+	}
+	BUG_ON(rinfo->persistent_gnts_c != 0);
+
 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
 		/*
 		 * Clear persistent grants present in requests already
 		 * on the shared ring
 		 */
-		if (!info->shadow[i].request)
+		if (!rinfo->shadow[i].request)
 			goto free_shadow;
 
-		segs = info->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
-		       info->shadow[i].req.u.indirect.nr_segments :
-		       info->shadow[i].req.u.rw.nr_segments;
+		segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
+		       rinfo->shadow[i].req.u.indirect.nr_segments :
+		       rinfo->shadow[i].req.u.rw.nr_segments;
 		for (j = 0; j < segs; j++) {
-			persistent_gnt = info->shadow[i].grants_used[j];
+			persistent_gnt = rinfo->shadow[i].grants_used[j];
 			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
 			if (info->feature_persistent)
 				__free_page(persistent_gnt->page);
 			kfree(persistent_gnt);
 		}
 
-		if (info->shadow[i].req.operation != BLKIF_OP_INDIRECT)
+		if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT)
 			/*
 			 * If this is not an indirect operation don't try to
 			 * free indirect segments
@@ -1145,42 +1293,59 @@ static void blkif_free(struct blkfront_info *info, int suspend)
 			goto free_shadow;
 
 		for (j = 0; j < INDIRECT_GREFS(segs); j++) {
-			persistent_gnt = info->shadow[i].indirect_grants[j];
+			persistent_gnt = rinfo->shadow[i].indirect_grants[j];
 			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
 			__free_page(persistent_gnt->page);
 			kfree(persistent_gnt);
 		}
 
 free_shadow:
-		kfree(info->shadow[i].grants_used);
-		info->shadow[i].grants_used = NULL;
-		kfree(info->shadow[i].indirect_grants);
-		info->shadow[i].indirect_grants = NULL;
-		kfree(info->shadow[i].sg);
-		info->shadow[i].sg = NULL;
+		kfree(rinfo->shadow[i].grants_used);
+		rinfo->shadow[i].grants_used = NULL;
+		kfree(rinfo->shadow[i].indirect_grants);
+		rinfo->shadow[i].indirect_grants = NULL;
+		kfree(rinfo->shadow[i].sg);
+		rinfo->shadow[i].sg = NULL;
 	}
 
 	/* No more gnttab callback work. */
-	gnttab_cancel_free_callback(&info->callback);
-	spin_unlock_irq(&info->io_lock);
+	gnttab_cancel_free_callback(&rinfo->callback);
 
 	/* Flush gnttab callback work. Must be done with no locks held. */
-	flush_work(&info->work);
+	flush_work(&rinfo->work);
 
 	/* Free resources associated with old device channel. */
 	for (i = 0; i < info->nr_ring_pages; i++) {
-		if (info->ring_ref[i] != GRANT_INVALID_REF) {
-			gnttab_end_foreign_access(info->ring_ref[i], 0, 0);
-			info->ring_ref[i] = GRANT_INVALID_REF;
+		if (rinfo->ring_ref[i] != GRANT_INVALID_REF) {
+			gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0);
+			rinfo->ring_ref[i] = GRANT_INVALID_REF;
 		}
 	}
-	free_pages((unsigned long)info->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE));
-	info->ring.sring = NULL;
+	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE));
+	rinfo->ring.sring = NULL;
 
-	if (info->irq)
-		unbind_from_irqhandler(info->irq, info);
-	info->evtchn = info->irq = 0;
+	if (rinfo->irq)
+		unbind_from_irqhandler(rinfo->irq, rinfo);
+	rinfo->evtchn = rinfo->irq = 0;
+}
 
+static void blkif_free(struct blkfront_info *info, int suspend)
+{
+	unsigned int i;
+
+	/* Prevent new requests being issued until we fix things up. */
+	info->connected = suspend ?
+		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
+	/* No more blkif_request(). */
+	if (info->rq)
+		blk_mq_stop_hw_queues(info->rq);
+
+	for (i = 0; i < info->nr_rings; i++)
+		blkif_free_ring(&info->rinfo[i]);
+
+	kfree(info->rinfo);
+	info->rinfo = NULL;
+	info->nr_rings = 0;
 }
 
 struct copy_from_grant {
@@ -1209,19 +1374,93 @@ static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset,
 	kunmap_atomic(shared_data);
 }
 
-static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
+static enum blk_req_status blkif_rsp_to_req_status(int rsp)
+{
+	switch (rsp)
+	{
+	case BLKIF_RSP_OKAY:
+		return REQ_DONE;
+	case BLKIF_RSP_EOPNOTSUPP:
+		return REQ_EOPNOTSUPP;
+	case BLKIF_RSP_ERROR:
+		/* Fallthrough. */
+	default:
+		return REQ_ERROR;
+	}
+}
+
+/*
+ * Get the final status of the block request based on two ring response
+ */
+static int blkif_get_final_status(enum blk_req_status s1,
+				  enum blk_req_status s2)
+{
+	BUG_ON(s1 == REQ_WAITING);
+	BUG_ON(s2 == REQ_WAITING);
+
+	if (s1 == REQ_ERROR || s2 == REQ_ERROR)
+		return BLKIF_RSP_ERROR;
+	else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP)
+		return BLKIF_RSP_EOPNOTSUPP;
+	return BLKIF_RSP_OKAY;
+}
+
+static bool blkif_completion(unsigned long *id,
+			     struct blkfront_ring_info *rinfo,
 			     struct blkif_response *bret)
 {
 	int i = 0;
 	struct scatterlist *sg;
 	int num_sg, num_grant;
+	struct blkfront_info *info = rinfo->dev_info;
+	struct blk_shadow *s = &rinfo->shadow[*id];
 	struct copy_from_grant data = {
-		.s = s,
 		.grant_idx = 0,
 	};
 
 	num_grant = s->req.operation == BLKIF_OP_INDIRECT ?
 		s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
+
+	/* The I/O request may be split in two. */
+	if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) {
+		struct blk_shadow *s2 = &rinfo->shadow[s->associated_id];
+
+		/* Keep the status of the current response in shadow. */
+		s->status = blkif_rsp_to_req_status(bret->status);
+
+		/* Wait the second response if not yet here. */
+		if (s2->status == REQ_WAITING)
+			return 0;
+
+		bret->status = blkif_get_final_status(s->status,
+						      s2->status);
+
+		/*
+		 * All the grants is stored in the first shadow in order
+		 * to make the completion code simpler.
+		 */
+		num_grant += s2->req.u.rw.nr_segments;
+
+		/*
+		 * The two responses may not come in order. Only the
+		 * first request will store the scatter-gather list.
+		 */
+		if (s2->num_sg != 0) {
+			/* Update "id" with the ID of the first response. */
+			*id = s->associated_id;
+			s = s2;
+		}
+
+		/*
+		 * We don't need anymore the second request, so recycling
+		 * it now.
+		 */
+		if (add_id_to_freelist(rinfo, s->associated_id))
+			WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n",
+			     info->gd->disk_name, s->associated_id);
+	}
+
+	data.s = s;
 	num_sg = s->num_sg;
 
 	if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
@@ -1252,8 +1491,8 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
 			if (!info->feature_persistent)
 				pr_alert_ratelimited("backed has not unmapped grant: %u\n",
 						     s->grants_used[i]->gref);
-			list_add(&s->grants_used[i]->node, &info->grants);
-			info->persistent_gnts_c++;
+			list_add(&s->grants_used[i]->node, &rinfo->grants);
+			rinfo->persistent_gnts_c++;
 		} else {
 			/*
 			 * If the grant is not mapped by the backend we end the
@@ -1263,7 +1502,7 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
 			 */
 			gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
 			s->grants_used[i]->gref = GRANT_INVALID_REF;
-			list_add_tail(&s->grants_used[i]->node, &info->grants);
+			list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
 		}
 	}
 	if (s->req.operation == BLKIF_OP_INDIRECT) {
@@ -1272,8 +1511,8 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
 				if (!info->feature_persistent)
 					pr_alert_ratelimited("backed has not unmapped grant: %u\n",
 							     s->indirect_grants[i]->gref);
-				list_add(&s->indirect_grants[i]->node, &info->grants);
-				info->persistent_gnts_c++;
+				list_add(&s->indirect_grants[i]->node, &rinfo->grants);
+				rinfo->persistent_gnts_c++;
 			} else {
 				struct page *indirect_page;
 
@@ -1284,13 +1523,15 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
 				 */
 				if (!info->feature_persistent) {
 					indirect_page = s->indirect_grants[i]->page;
-					list_add(&indirect_page->lru, &info->indirect_pages);
+					list_add(&indirect_page->lru, &rinfo->indirect_pages);
 				}
 				s->indirect_grants[i]->gref = GRANT_INVALID_REF;
-				list_add_tail(&s->indirect_grants[i]->node, &info->grants);
+				list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
 			}
 		}
 	}
+
+	return 1;
 }
 
 static irqreturn_t blkif_interrupt(int irq, void *dev_id)
@@ -1299,24 +1540,22 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
 	struct blkif_response *bret;
 	RING_IDX i, rp;
 	unsigned long flags;
-	struct blkfront_info *info = (struct blkfront_info *)dev_id;
+	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
+	struct blkfront_info *info = rinfo->dev_info;
 	int error;
 
-	spin_lock_irqsave(&info->io_lock, flags);
-
-	if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) {
-		spin_unlock_irqrestore(&info->io_lock, flags);
+	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
 		return IRQ_HANDLED;
-	}
 
+	spin_lock_irqsave(&rinfo->ring_lock, flags);
  again:
-	rp = info->ring.sring->rsp_prod;
+	rp = rinfo->ring.sring->rsp_prod;
 	rmb(); /* Ensure we see queued responses up to 'rp'. */
 
-	for (i = info->ring.rsp_cons; i != rp; i++) {
+	for (i = rinfo->ring.rsp_cons; i != rp; i++) {
 		unsigned long id;
 
-		bret = RING_GET_RESPONSE(&info->ring, i);
+		bret = RING_GET_RESPONSE(&rinfo->ring, i);
 		id   = bret->id;
 		/*
 		 * The backend has messed up and given us an id that we would
@@ -1330,12 +1569,18 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
 			 * the id is busted. */
 			continue;
 		}
-		req  = info->shadow[id].request;
+		req  = rinfo->shadow[id].request;
 
-		if (bret->operation != BLKIF_OP_DISCARD)
-			blkif_completion(&info->shadow[id], info, bret);
+		if (bret->operation != BLKIF_OP_DISCARD) {
+			/*
+			 * We may need to wait for an extra response if the
+			 * I/O request is split in 2
+			 */
+			if (!blkif_completion(&id, rinfo, bret))
+				continue;
+		}
 
-		if (add_id_to_freelist(info, id)) {
+		if (add_id_to_freelist(rinfo, id)) {
 			WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
 			     info->gd->disk_name, op_name(bret->operation), id);
 			continue;
@@ -1364,7 +1609,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
 				error = -EOPNOTSUPP;
 			}
 			if (unlikely(bret->status == BLKIF_RSP_ERROR &&
-				     info->shadow[id].req.u.rw.nr_segments == 0)) {
+				     rinfo->shadow[id].req.u.rw.nr_segments == 0)) {
 				printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
 				       info->gd->disk_name, op_name(bret->operation));
 				error = -EOPNOTSUPP;
@@ -1389,34 +1634,35 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
 		}
 	}
 
-	info->ring.rsp_cons = i;
+	rinfo->ring.rsp_cons = i;
 
-	if (i != info->ring.req_prod_pvt) {
+	if (i != rinfo->ring.req_prod_pvt) {
 		int more_to_do;
-		RING_FINAL_CHECK_FOR_RESPONSES(&info->ring, more_to_do);
+		RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do);
 		if (more_to_do)
 			goto again;
 	} else
-		info->ring.sring->rsp_event = i + 1;
+		rinfo->ring.sring->rsp_event = i + 1;
 
-	kick_pending_request_queues(info);
+	kick_pending_request_queues_locked(rinfo);
 
-	spin_unlock_irqrestore(&info->io_lock, flags);
+	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
 
 	return IRQ_HANDLED;
 }
 
 
 static int setup_blkring(struct xenbus_device *dev,
-			 struct blkfront_info *info)
+			 struct blkfront_ring_info *rinfo)
 {
 	struct blkif_sring *sring;
 	int err, i;
+	struct blkfront_info *info = rinfo->dev_info;
 	unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE;
 	grant_ref_t gref[XENBUS_MAX_RING_GRANTS];
 
 	for (i = 0; i < info->nr_ring_pages; i++)
-		info->ring_ref[i] = GRANT_INVALID_REF;
+		rinfo->ring_ref[i] = GRANT_INVALID_REF;
 
 	sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
 						       get_order(ring_size));
@@ -1425,29 +1671,29 @@ static int setup_blkring(struct xenbus_device *dev,
 		return -ENOMEM;
 	}
 	SHARED_RING_INIT(sring);
-	FRONT_RING_INIT(&info->ring, sring, ring_size);
+	FRONT_RING_INIT(&rinfo->ring, sring, ring_size);
 
-	err = xenbus_grant_ring(dev, info->ring.sring, info->nr_ring_pages, gref);
+	err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
 	if (err < 0) {
 		free_pages((unsigned long)sring, get_order(ring_size));
-		info->ring.sring = NULL;
+		rinfo->ring.sring = NULL;
 		goto fail;
 	}
 	for (i = 0; i < info->nr_ring_pages; i++)
-		info->ring_ref[i] = gref[i];
+		rinfo->ring_ref[i] = gref[i];
 
-	err = xenbus_alloc_evtchn(dev, &info->evtchn);
+	err = xenbus_alloc_evtchn(dev, &rinfo->evtchn);
 	if (err)
 		goto fail;
 
-	err = bind_evtchn_to_irqhandler(info->evtchn, blkif_interrupt, 0,
-					"blkif", info);
+	err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
+					"blkif", rinfo);
 	if (err <= 0) {
 		xenbus_dev_fatal(dev, err,
 				 "bind_evtchn_to_irqhandler failed");
 		goto fail;
 	}
-	info->irq = err;
+	rinfo->irq = err;
 
 	return 0;
 fail:
@@ -1455,6 +1701,53 @@ fail:
 	return err;
 }
 
+/*
+ * Write out per-ring/queue nodes including ring-ref and event-channel, and each
+ * ring buffer may have multi pages depending on ->nr_ring_pages.
+ */
+static int write_per_ring_nodes(struct xenbus_transaction xbt,
+				struct blkfront_ring_info *rinfo, const char *dir)
+{
+	int err;
+	unsigned int i;
+	const char *message = NULL;
+	struct blkfront_info *info = rinfo->dev_info;
+
+	if (info->nr_ring_pages == 1) {
+		err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]);
+		if (err) {
+			message = "writing ring-ref";
+			goto abort_transaction;
+		}
+	} else {
+		for (i = 0; i < info->nr_ring_pages; i++) {
+			char ring_ref_name[RINGREF_NAME_LEN];
+
+			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
+			err = xenbus_printf(xbt, dir, ring_ref_name,
+					    "%u", rinfo->ring_ref[i]);
+			if (err) {
+				message = "writing ring-ref";
+				goto abort_transaction;
+			}
+		}
+	}
+
+	err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn);
+	if (err) {
+		message = "writing event-channel";
+		goto abort_transaction;
+	}
+
+	return 0;
+
+abort_transaction:
+	xenbus_transaction_end(xbt, 1);
+	if (message)
+		xenbus_dev_fatal(info->xbdev, err, "%s", message);
+
+	return err;
+}
 
 /* Common code used when first setting up, and when resuming. */
 static int talk_to_blkback(struct xenbus_device *dev,
@@ -1462,8 +1755,8 @@ static int talk_to_blkback(struct xenbus_device *dev,
 {
 	const char *message = NULL;
 	struct xenbus_transaction xbt;
-	int err, i;
-	unsigned int max_page_order = 0;
+	int err;
+	unsigned int i, max_page_order = 0;
 	unsigned int ring_page_order = 0;
 
 	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
@@ -1475,10 +1768,14 @@ static int talk_to_blkback(struct xenbus_device *dev,
 		info->nr_ring_pages = 1 << ring_page_order;
 	}
 
-	/* Create shared ring, alloc event channel. */
-	err = setup_blkring(dev, info);
-	if (err)
-		goto out;
+	for (i = 0; i < info->nr_rings; i++) {
+		struct blkfront_ring_info *rinfo = &info->rinfo[i];
+
+		/* Create shared ring, alloc event channel. */
+		err = setup_blkring(dev, rinfo);
+		if (err)
+			goto destroy_blkring;
+	}
 
 again:
 	err = xenbus_transaction_start(&xbt);
@@ -1487,38 +1784,49 @@ again:
 		goto destroy_blkring;
 	}
 
-	if (info->nr_ring_pages == 1) {
-		err = xenbus_printf(xbt, dev->nodename,
-				    "ring-ref", "%u", info->ring_ref[0]);
+	if (info->nr_ring_pages > 1) {
+		err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u",
+				    ring_page_order);
 		if (err) {
-			message = "writing ring-ref";
+			message = "writing ring-page-order";
 			goto abort_transaction;
 		}
+	}
+
+	/* We already got the number of queues/rings in _probe */
+	if (info->nr_rings == 1) {
+		err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename);
+		if (err)
+			goto destroy_blkring;
 	} else {
-		err = xenbus_printf(xbt, dev->nodename,
-				    "ring-page-order", "%u", ring_page_order);
+		char *path;
+		size_t pathsize;
+
+		err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u",
+				    info->nr_rings);
 		if (err) {
-			message = "writing ring-page-order";
+			message = "writing multi-queue-num-queues";
 			goto abort_transaction;
 		}
 
-		for (i = 0; i < info->nr_ring_pages; i++) {
-			char ring_ref_name[RINGREF_NAME_LEN];
+		pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN;
+		path = kmalloc(pathsize, GFP_KERNEL);
+		if (!path) {
+			err = -ENOMEM;
+			message = "ENOMEM while writing ring references";
+			goto abort_transaction;
+		}
 
-			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
-			err = xenbus_printf(xbt, dev->nodename, ring_ref_name,
-					    "%u", info->ring_ref[i]);
+		for (i = 0; i < info->nr_rings; i++) {
+			memset(path, 0, pathsize);
+			snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i);
+			err = write_per_ring_nodes(xbt, &info->rinfo[i], path);
 			if (err) {
-				message = "writing ring-ref";
-				goto abort_transaction;
+				kfree(path);
+				goto destroy_blkring;
 			}
 		}
-	}
-	err = xenbus_printf(xbt, dev->nodename,
-			    "event-channel", "%u", info->evtchn);
-	if (err) {
-		message = "writing event-channel";
-		goto abort_transaction;
+		kfree(path);
 	}
 	err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
 			    XEN_IO_PROTO_ABI_NATIVE);
@@ -1540,9 +1848,14 @@ again:
 		goto destroy_blkring;
 	}
 
-	for (i = 0; i < BLK_RING_SIZE(info); i++)
-		info->shadow[i].req.u.rw.id = i+1;
-	info->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
+	for (i = 0; i < info->nr_rings; i++) {
+		unsigned int j;
+		struct blkfront_ring_info *rinfo = &info->rinfo[i];
+
+		for (j = 0; j < BLK_RING_SIZE(info); j++)
+			rinfo->shadow[j].req.u.rw.id = j + 1;
+		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
+	}
 	xenbus_switch_state(dev, XenbusStateInitialised);
 
 	return 0;
@@ -1553,7 +1866,10 @@ again:
 		xenbus_dev_fatal(dev, err, "%s", message);
  destroy_blkring:
 	blkif_free(info, 0);
- out:
+
+	kfree(info);
+	dev_set_drvdata(&dev->dev, NULL);
+
 	return err;
 }
 
@@ -1567,7 +1883,9 @@ static int blkfront_probe(struct xenbus_device *dev,
 			  const struct xenbus_device_id *id)
 {
 	int err, vdevice;
+	unsigned int r_index;
 	struct blkfront_info *info;
+	unsigned int backend_max_queues = 0;
 
 	/* FIXME: Use dynamic device id if this is not set. */
 	err = xenbus_scanf(XBT_NIL, dev->nodename,
@@ -1617,15 +1935,39 @@ static int blkfront_probe(struct xenbus_device *dev,
 		return -ENOMEM;
 	}
 
-	mutex_init(&info->mutex);
-	spin_lock_init(&info->io_lock);
 	info->xbdev = dev;
+	/* Check if backend supports multiple queues. */
+	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
+			   "multi-queue-max-queues", "%u", &backend_max_queues);
+	if (err < 0)
+		backend_max_queues = 1;
+
+	info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
+	/* We need at least one ring. */
+	if (!info->nr_rings)
+		info->nr_rings = 1;
+
+	info->rinfo = kzalloc(sizeof(struct blkfront_ring_info) * info->nr_rings, GFP_KERNEL);
+	if (!info->rinfo) {
+		xenbus_dev_fatal(dev, -ENOMEM, "allocating ring_info structure");
+		kfree(info);
+		return -ENOMEM;
+	}
+
+	for (r_index = 0; r_index < info->nr_rings; r_index++) {
+		struct blkfront_ring_info *rinfo;
+
+		rinfo = &info->rinfo[r_index];
+		INIT_LIST_HEAD(&rinfo->indirect_pages);
+		INIT_LIST_HEAD(&rinfo->grants);
+		rinfo->dev_info = info;
+		INIT_WORK(&rinfo->work, blkif_restart_queue);
+		spin_lock_init(&rinfo->ring_lock);
+	}
+
+	mutex_init(&info->mutex);
 	info->vdevice = vdevice;
-	INIT_LIST_HEAD(&info->grants);
-	INIT_LIST_HEAD(&info->indirect_pages);
-	info->persistent_gnts_c = 0;
 	info->connected = BLKIF_STATE_DISCONNECTED;
-	INIT_WORK(&info->work, blkif_restart_queue);
 
 	/* Front end dir is a number, which is used as the id. */
 	info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
@@ -1649,7 +1991,7 @@ static void split_bio_end(struct bio *bio)
 
 static int blkif_recover(struct blkfront_info *info)
 {
-	int i;
+	unsigned int i, r_index;
 	struct request *req, *n;
 	struct blk_shadow *copy;
 	int rc;
@@ -1660,64 +2002,73 @@ static int blkif_recover(struct blkfront_info *info)
 	struct split_bio *split_bio;
 	struct list_head requests;
 
-	/* Stage 1: Make a safe copy of the shadow state. */
-	copy = kmemdup(info->shadow, sizeof(info->shadow),
-		       GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
-	if (!copy)
-		return -ENOMEM;
-
-	/* Stage 2: Set up free list. */
-	memset(&info->shadow, 0, sizeof(info->shadow));
-	for (i = 0; i < BLK_RING_SIZE(info); i++)
-		info->shadow[i].req.u.rw.id = i+1;
-	info->shadow_free = info->ring.req_prod_pvt;
-	info->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
-
-	rc = blkfront_gather_backend_features(info);
-	if (rc) {
-		kfree(copy);
-		return rc;
-	}
-
+	blkfront_gather_backend_features(info);
 	segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
 	blk_queue_max_segments(info->rq, segs);
 	bio_list_init(&bio_list);
 	INIT_LIST_HEAD(&requests);
-	for (i = 0; i < BLK_RING_SIZE(info); i++) {
-		/* Not in use? */
-		if (!copy[i].request)
-			continue;
 
-		/*
-		 * Get the bios in the request so we can re-queue them.
-		 */
-		if (copy[i].request->cmd_flags &
-		    (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) {
+	for (r_index = 0; r_index < info->nr_rings; r_index++) {
+		struct blkfront_ring_info *rinfo;
+
+		rinfo = &info->rinfo[r_index];
+		/* Stage 1: Make a safe copy of the shadow state. */
+		copy = kmemdup(rinfo->shadow, sizeof(rinfo->shadow),
+			       GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
+		if (!copy)
+			return -ENOMEM;
+
+		/* Stage 2: Set up free list. */
+		memset(&rinfo->shadow, 0, sizeof(rinfo->shadow));
+		for (i = 0; i < BLK_RING_SIZE(info); i++)
+			rinfo->shadow[i].req.u.rw.id = i+1;
+		rinfo->shadow_free = rinfo->ring.req_prod_pvt;
+		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
+
+		rc = blkfront_setup_indirect(rinfo);
+		if (rc) {
+			kfree(copy);
+			return rc;
+		}
+
+		for (i = 0; i < BLK_RING_SIZE(info); i++) {
+			/* Not in use? */
+			if (!copy[i].request)
+				continue;
+
 			/*
-			 * Flush operations don't contain bios, so
-			 * we need to requeue the whole request
+			 * Get the bios in the request so we can re-queue them.
 			 */
-			list_add(&copy[i].request->queuelist, &requests);
-			continue;
+			if (copy[i].request->cmd_flags &
+			    (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) {
+				/*
+				 * Flush operations don't contain bios, so
+				 * we need to requeue the whole request
+				 */
+				list_add(&copy[i].request->queuelist, &requests);
+				continue;
+			}
+			merge_bio.head = copy[i].request->bio;
+			merge_bio.tail = copy[i].request->biotail;
+			bio_list_merge(&bio_list, &merge_bio);
+			copy[i].request->bio = NULL;
+			blk_end_request_all(copy[i].request, 0);
 		}
-		merge_bio.head = copy[i].request->bio;
-		merge_bio.tail = copy[i].request->biotail;
-		bio_list_merge(&bio_list, &merge_bio);
-		copy[i].request->bio = NULL;
-		blk_end_request_all(copy[i].request, 0);
-	}
-
-	kfree(copy);
 
+		kfree(copy);
+	}
 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
 
-	spin_lock_irq(&info->io_lock);
-
 	/* Now safe for us to use the shared ring */
 	info->connected = BLKIF_STATE_CONNECTED;
 
-	/* Kick any other new requests queued since we resumed */
-	kick_pending_request_queues(info);
+	for (r_index = 0; r_index < info->nr_rings; r_index++) {
+		struct blkfront_ring_info *rinfo;
+
+		rinfo = &info->rinfo[r_index];
+		/* Kick any other new requests queued since we resumed */
+		kick_pending_request_queues(rinfo);
+	}
 
 	list_for_each_entry_safe(req, n, &requests, queuelist) {
 		/* Requeue pending requests (flush or discard) */
@@ -1725,7 +2076,6 @@ static int blkif_recover(struct blkfront_info *info)
 		BUG_ON(req->nr_phys_segments > segs);
 		blk_mq_requeue_request(req);
 	}
-	spin_unlock_irq(&info->io_lock);
 	blk_mq_kick_requeue_list(info->rq);
 
 	while ((bio = bio_list_pop(&bio_list)) != NULL) {
@@ -1790,8 +2140,7 @@ static int blkfront_resume(struct xenbus_device *dev)
 	return err;
 }
 
-static void
-blkfront_closing(struct blkfront_info *info)
+static void blkfront_closing(struct blkfront_info *info)
 {
 	struct xenbus_device *xbdev = info->xbdev;
 	struct block_device *bdev = NULL;
@@ -1851,18 +2200,29 @@ static void blkfront_setup_discard(struct blkfront_info *info)
 		info->feature_secdiscard = !!discard_secure;
 }
 
-static int blkfront_setup_indirect(struct blkfront_info *info)
+static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
 {
 	unsigned int psegs, grants;
 	int err, i;
+	struct blkfront_info *info = rinfo->dev_info;
 
-	if (info->max_indirect_segments == 0)
-		grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
+	if (info->max_indirect_segments == 0) {
+		if (!HAS_EXTRA_REQ)
+			grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
+		else {
+			/*
+			 * When an extra req is required, the maximum
+			 * grants supported is related to the size of the
+			 * Linux block segment.
+			 */
+			grants = GRANTS_PER_PSEG;
+		}
+	}
 	else
 		grants = info->max_indirect_segments;
 	psegs = grants / GRANTS_PER_PSEG;
 
-	err = fill_grant_buffer(info,
+	err = fill_grant_buffer(rinfo,
 				(grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info));
 	if (err)
 		goto out_of_memory;
@@ -1875,31 +2235,31 @@ static int blkfront_setup_indirect(struct blkfront_info *info)
 		 */
 		int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
 
-		BUG_ON(!list_empty(&info->indirect_pages));
+		BUG_ON(!list_empty(&rinfo->indirect_pages));
 		for (i = 0; i < num; i++) {
 			struct page *indirect_page = alloc_page(GFP_NOIO);
 			if (!indirect_page)
 				goto out_of_memory;
-			list_add(&indirect_page->lru, &info->indirect_pages);
+			list_add(&indirect_page->lru, &rinfo->indirect_pages);
 		}
 	}
 
 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
-		info->shadow[i].grants_used = kzalloc(
-			sizeof(info->shadow[i].grants_used[0]) * grants,
+		rinfo->shadow[i].grants_used = kzalloc(
+			sizeof(rinfo->shadow[i].grants_used[0]) * grants,
 			GFP_NOIO);
-		info->shadow[i].sg = kzalloc(sizeof(info->shadow[i].sg[0]) * psegs, GFP_NOIO);
+		rinfo->shadow[i].sg = kzalloc(sizeof(rinfo->shadow[i].sg[0]) * psegs, GFP_NOIO);
 		if (info->max_indirect_segments)
-			info->shadow[i].indirect_grants = kzalloc(
-				sizeof(info->shadow[i].indirect_grants[0]) *
+			rinfo->shadow[i].indirect_grants = kzalloc(
+				sizeof(rinfo->shadow[i].indirect_grants[0]) *
 				INDIRECT_GREFS(grants),
 				GFP_NOIO);
-		if ((info->shadow[i].grants_used == NULL) ||
-			(info->shadow[i].sg == NULL) ||
+		if ((rinfo->shadow[i].grants_used == NULL) ||
+			(rinfo->shadow[i].sg == NULL) ||
 		     (info->max_indirect_segments &&
-		     (info->shadow[i].indirect_grants == NULL)))
+		     (rinfo->shadow[i].indirect_grants == NULL)))
 			goto out_of_memory;
-		sg_init_table(info->shadow[i].sg, psegs);
+		sg_init_table(rinfo->shadow[i].sg, psegs);
 	}
 
 
@@ -1907,16 +2267,16 @@ static int blkfront_setup_indirect(struct blkfront_info *info)
 
 out_of_memory:
 	for (i = 0; i < BLK_RING_SIZE(info); i++) {
-		kfree(info->shadow[i].grants_used);
-		info->shadow[i].grants_used = NULL;
-		kfree(info->shadow[i].sg);
-		info->shadow[i].sg = NULL;
-		kfree(info->shadow[i].indirect_grants);
-		info->shadow[i].indirect_grants = NULL;
-	}
-	if (!list_empty(&info->indirect_pages)) {
+		kfree(rinfo->shadow[i].grants_used);
+		rinfo->shadow[i].grants_used = NULL;
+		kfree(rinfo->shadow[i].sg);
+		rinfo->shadow[i].sg = NULL;
+		kfree(rinfo->shadow[i].indirect_grants);
+		rinfo->shadow[i].indirect_grants = NULL;
+	}
+	if (!list_empty(&rinfo->indirect_pages)) {
 		struct page *indirect_page, *n;
-		list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
+		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
 			list_del(&indirect_page->lru);
 			__free_page(indirect_page);
 		}
@@ -1927,7 +2287,7 @@ out_of_memory:
 /*
  * Gather all backend feature-*
  */
-static int blkfront_gather_backend_features(struct blkfront_info *info)
+static void blkfront_gather_backend_features(struct blkfront_info *info)
 {
 	int err;
 	int barrier, flush, discard, persistent;
@@ -1982,8 +2342,6 @@ static int blkfront_gather_backend_features(struct blkfront_info *info)
 	else
 		info->max_indirect_segments = min(indirect_segments,
 						  xen_blkif_max_segments);
-
-	return blkfront_setup_indirect(info);
 }
 
 /*
@@ -1996,7 +2354,7 @@ static void blkfront_connect(struct blkfront_info *info)
 	unsigned long sector_size;
 	unsigned int physical_sector_size;
 	unsigned int binfo;
-	int err;
+	int err, i;
 
 	switch (info->connected) {
 	case BLKIF_STATE_CONNECTED:
@@ -2053,11 +2411,15 @@ static void blkfront_connect(struct blkfront_info *info)
 	if (err != 1)
 		physical_sector_size = sector_size;
 
-	err = blkfront_gather_backend_features(info);
-	if (err) {
-		xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
-				 info->xbdev->otherend);
-		return;
+	blkfront_gather_backend_features(info);
+	for (i = 0; i < info->nr_rings; i++) {
+		err = blkfront_setup_indirect(&info->rinfo[i]);
+		if (err) {
+			xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
+					 info->xbdev->otherend);
+			blkif_free(info, 0);
+			break;
+		}
 	}
 
 	err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size,
@@ -2071,10 +2433,9 @@ static void blkfront_connect(struct blkfront_info *info)
 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
 
 	/* Kick pending requests. */
-	spin_lock_irq(&info->io_lock);
 	info->connected = BLKIF_STATE_CONNECTED;
-	kick_pending_request_queues(info);
-	spin_unlock_irq(&info->io_lock);
+	for (i = 0; i < info->nr_rings; i++)
+		kick_pending_request_queues(&info->rinfo[i]);
 
 	add_disk(info->gd);
 
@@ -2095,11 +2456,8 @@ static void blkback_changed(struct xenbus_device *dev,
 	case XenbusStateInitWait:
 		if (dev->state != XenbusStateInitialising)
 			break;
-		if (talk_to_blkback(dev, info)) {
-			kfree(info);
-			dev_set_drvdata(&dev->dev, NULL);
+		if (talk_to_blkback(dev, info))
 			break;
-		}
 	case XenbusStateInitialising:
 	case XenbusStateInitialised:
 	case XenbusStateReconfiguring:
@@ -2108,6 +2466,10 @@ static void blkback_changed(struct xenbus_device *dev,
 		break;
 
 	case XenbusStateConnected:
+		if (dev->state != XenbusStateInitialised) {
+			if (talk_to_blkback(dev, info))
+				break;
+		}
 		blkfront_connect(info);
 		break;
 
@@ -2281,6 +2643,7 @@ static struct xenbus_driver blkfront_driver = {
 static int __init xlblk_init(void)
 {
 	int ret;
+	int nr_cpus = num_online_cpus();
 
 	if (!xen_domain())
 		return -ENODEV;
@@ -2288,7 +2651,13 @@ static int __init xlblk_init(void)
 	if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
 		pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
 			xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
-		xen_blkif_max_ring_order = 0;
+		xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
+	}
+
+	if (xen_blkif_max_queues > nr_cpus) {
+		pr_info("Invalid max_queues (%d), will use default max: %d.\n",
+			xen_blkif_max_queues, nr_cpus);
+		xen_blkif_max_queues = nr_cpus;
 	}
 
 	if (!xen_has_pv_disk_devices())
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
index 5cb13ca3a3ac..3ef42e563bb5 100644
--- a/drivers/block/zram/zcomp.c
+++ b/drivers/block/zram/zcomp.c
@@ -74,18 +74,18 @@ static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm)
  * allocate new zcomp_strm structure with ->private initialized by
  * backend, return NULL on error
  */
-static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp)
+static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags)
 {
-	struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL);
+	struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags);
 	if (!zstrm)
 		return NULL;
 
-	zstrm->private = comp->backend->create();
+	zstrm->private = comp->backend->create(flags);
 	/*
 	 * allocate 2 pages. 1 for compressed data, plus 1 extra for the
 	 * case when compressed size is larger than the original one
 	 */
-	zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
+	zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1);
 	if (!zstrm->private || !zstrm->buffer) {
 		zcomp_strm_free(comp, zstrm);
 		zstrm = NULL;
@@ -120,8 +120,16 @@ static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp)
 		/* allocate new zstrm stream */
 		zs->avail_strm++;
 		spin_unlock(&zs->strm_lock);
-
-		zstrm = zcomp_strm_alloc(comp);
+		/*
+		 * This function can be called in swapout/fs write path
+		 * so we can't use GFP_FS|IO. And it assumes we already
+		 * have at least one stream in zram initialization so we
+		 * don't do best effort to allocate more stream in here.
+		 * A default stream will work well without further multiple
+		 * streams. That's why we use NORETRY | NOWARN.
+		 */
+		zstrm = zcomp_strm_alloc(comp, GFP_NOIO | __GFP_NORETRY |
+					__GFP_NOWARN);
 		if (!zstrm) {
 			spin_lock(&zs->strm_lock);
 			zs->avail_strm--;
@@ -209,7 +217,7 @@ static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm)
 	zs->max_strm = max_strm;
 	zs->avail_strm = 1;
 
-	zstrm = zcomp_strm_alloc(comp);
+	zstrm = zcomp_strm_alloc(comp, GFP_KERNEL);
 	if (!zstrm) {
 		kfree(zs);
 		return -ENOMEM;
@@ -259,7 +267,7 @@ static int zcomp_strm_single_create(struct zcomp *comp)
 
 	comp->stream = zs;
 	mutex_init(&zs->strm_lock);
-	zs->zstrm = zcomp_strm_alloc(comp);
+	zs->zstrm = zcomp_strm_alloc(comp, GFP_KERNEL);
 	if (!zs->zstrm) {
 		kfree(zs);
 		return -ENOMEM;
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h
index 46e2b9f8f1f0..b7d2a4bcae54 100644
--- a/drivers/block/zram/zcomp.h
+++ b/drivers/block/zram/zcomp.h
@@ -33,7 +33,7 @@ struct zcomp_backend {
 	int (*decompress)(const unsigned char *src, size_t src_len,
 			unsigned char *dst);
 
-	void *(*create)(void);
+	void *(*create)(gfp_t flags);
 	void (*destroy)(void *private);
 
 	const char *name;
diff --git a/drivers/block/zram/zcomp_lz4.c b/drivers/block/zram/zcomp_lz4.c
index f2afb7e988c3..0110086accba 100644
--- a/drivers/block/zram/zcomp_lz4.c
+++ b/drivers/block/zram/zcomp_lz4.c
@@ -10,17 +10,26 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/lz4.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
 
 #include "zcomp_lz4.h"
 
-static void *zcomp_lz4_create(void)
+static void *zcomp_lz4_create(gfp_t flags)
 {
-	return kzalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
+	void *ret;
+
+	ret = kmalloc(LZ4_MEM_COMPRESS, flags);
+	if (!ret)
+		ret = __vmalloc(LZ4_MEM_COMPRESS,
+				flags | __GFP_HIGHMEM,
+				PAGE_KERNEL);
+	return ret;
 }
 
 static void zcomp_lz4_destroy(void *private)
 {
-	kfree(private);
+	kvfree(private);
 }
 
 static int zcomp_lz4_compress(const unsigned char *src, unsigned char *dst,
diff --git a/drivers/block/zram/zcomp_lzo.c b/drivers/block/zram/zcomp_lzo.c
index da1bc47d588e..ed7a1f0549ec 100644
--- a/drivers/block/zram/zcomp_lzo.c
+++ b/drivers/block/zram/zcomp_lzo.c
@@ -10,17 +10,26 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/lzo.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
 
 #include "zcomp_lzo.h"
 
-static void *lzo_create(void)
+static void *lzo_create(gfp_t flags)
 {
-	return kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
+	void *ret;
+
+	ret = kmalloc(LZO1X_MEM_COMPRESS, flags);
+	if (!ret)
+		ret = __vmalloc(LZO1X_MEM_COMPRESS,
+				flags | __GFP_HIGHMEM,
+				PAGE_KERNEL);
+	return ret;
 }
 
 static void lzo_destroy(void *private)
 {
-	kfree(private);
+	kvfree(private);
 }
 
 static int lzo_compress(const unsigned char *src, unsigned char *dst,
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 47915d736f8d..370c2f76016d 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -1325,7 +1325,6 @@ static int zram_remove(struct zram *zram)
 
 	pr_info("Removed device: %s\n", zram->disk->disk_name);
 
-	idr_remove(&zram_index_idr, zram->disk->first_minor);
 	blk_cleanup_queue(zram->disk->queue);
 	del_gendisk(zram->disk);
 	put_disk(zram->disk);
@@ -1367,10 +1366,12 @@ static ssize_t hot_remove_store(struct class *class,
 	mutex_lock(&zram_index_mutex);
 
 	zram = idr_find(&zram_index_idr, dev_id);
-	if (zram)
+	if (zram) {
 		ret = zram_remove(zram);
-	else
+		idr_remove(&zram_index_idr, dev_id);
+	} else {
 		ret = -ENODEV;
+	}
 
 	mutex_unlock(&zram_index_mutex);
 	return ret ? ret : count;
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 116b363b7987..129d47bcc5fc 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -131,6 +131,14 @@ config SUNXI_RSB
 	  with various RSB based devices, such as AXP223, AXP8XX PMICs,
 	  and AC100/AC200 ICs.
 
+config UNIPHIER_SYSTEM_BUS
+	bool "UniPhier System Bus driver"
+	depends on ARCH_UNIPHIER && OF
+	default y
+	help
+	  Support for UniPhier System Bus, a simple external bus.  This is
+	  needed to use on-board devices connected to UniPhier SoCs.
+
 config VEXPRESS_CONFIG
 	bool "Versatile Express configuration bus"
 	default y if ARCH_VEXPRESS
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index fcb9f9794a1f..ccff007ee7e8 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -17,4 +17,5 @@ obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 obj-$(CONFIG_OMAP_OCP2SCP)	+= omap-ocp2scp.o
 obj-$(CONFIG_SUNXI_RSB)		+= sunxi-rsb.o
 obj-$(CONFIG_SIMPLE_PM_BUS)	+= simple-pm-bus.o
+obj-$(CONFIG_UNIPHIER_SYSTEM_BUS)	+= uniphier-system-bus.o
 obj-$(CONFIG_VEXPRESS_CONFIG)	+= vexpress-config.o
diff --git a/drivers/bus/uniphier-system-bus.c b/drivers/bus/uniphier-system-bus.c
new file mode 100644
index 000000000000..834a2aeaf27a
--- /dev/null
+++ b/drivers/bus/uniphier-system-bus.c
@@ -0,0 +1,281 @@
+/*
+ * Copyright (C) 2015 Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/io.h>
+#include <linux/log2.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+/* System Bus Controller registers */
+#define UNIPHIER_SBC_BASE	0x100	/* base address of bank0 space */
+#define    UNIPHIER_SBC_BASE_BE		BIT(0)	/* bank_enable */
+#define UNIPHIER_SBC_CTRL0	0x200	/* timing parameter 0 of bank0 */
+#define UNIPHIER_SBC_CTRL1	0x204	/* timing parameter 1 of bank0 */
+#define UNIPHIER_SBC_CTRL2	0x208	/* timing parameter 2 of bank0 */
+#define UNIPHIER_SBC_CTRL3	0x20c	/* timing parameter 3 of bank0 */
+#define UNIPHIER_SBC_CTRL4	0x300	/* timing parameter 4 of bank0 */
+
+#define UNIPHIER_SBC_STRIDE	0x10	/* register stride to next bank */
+#define UNIPHIER_SBC_NR_BANKS	8	/* number of banks (chip select) */
+#define UNIPHIER_SBC_BASE_DUMMY	0xffffffff	/* data to squash bank 0, 1 */
+
+struct uniphier_system_bus_bank {
+	u32 base;
+	u32 end;
+};
+
+struct uniphier_system_bus_priv {
+	struct device *dev;
+	void __iomem *membase;
+	struct uniphier_system_bus_bank bank[UNIPHIER_SBC_NR_BANKS];
+};
+
+static int uniphier_system_bus_add_bank(struct uniphier_system_bus_priv *priv,
+					int bank, u32 addr, u64 paddr, u32 size)
+{
+	u64 end, mask;
+
+	dev_dbg(priv->dev,
+		"range found: bank = %d, addr = %08x, paddr = %08llx, size = %08x\n",
+		bank, addr, paddr, size);
+
+	if (bank >= ARRAY_SIZE(priv->bank)) {
+		dev_err(priv->dev, "unsupported bank number %d\n", bank);
+		return -EINVAL;
+	}
+
+	if (priv->bank[bank].base || priv->bank[bank].end) {
+		dev_err(priv->dev,
+			"range for bank %d has already been specified\n", bank);
+		return -EINVAL;
+	}
+
+	if (paddr > U32_MAX) {
+		dev_err(priv->dev, "base address %llx is too high\n", paddr);
+		return -EINVAL;
+	}
+
+	end = paddr + size;
+
+	if (addr > paddr) {
+		dev_err(priv->dev,
+			"base %08x cannot be mapped to %08llx of parent\n",
+			addr, paddr);
+		return -EINVAL;
+	}
+	paddr -= addr;
+
+	paddr = round_down(paddr, 0x00020000);
+	end = round_up(end, 0x00020000);
+
+	if (end > U32_MAX) {
+		dev_err(priv->dev, "end address %08llx is too high\n", end);
+		return -EINVAL;
+	}
+	mask = paddr ^ (end - 1);
+	mask = roundup_pow_of_two(mask);
+
+	paddr = round_down(paddr, mask);
+	end = round_up(end, mask);
+
+	priv->bank[bank].base = paddr;
+	priv->bank[bank].end = end;
+
+	dev_dbg(priv->dev, "range added: bank = %d, addr = %08x, end = %08x\n",
+		bank, priv->bank[bank].base, priv->bank[bank].end);
+
+	return 0;
+}
+
+static int uniphier_system_bus_check_overlap(
+				const struct uniphier_system_bus_priv *priv)
+{
+	int i, j;
+
+	for (i = 0; i < ARRAY_SIZE(priv->bank); i++) {
+		for (j = i + 1; j < ARRAY_SIZE(priv->bank); j++) {
+			if (priv->bank[i].end > priv->bank[j].base ||
+			    priv->bank[i].base < priv->bank[j].end) {
+				dev_err(priv->dev,
+					"region overlap between bank%d and bank%d\n",
+					i, j);
+				return -EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static void uniphier_system_bus_check_boot_swap(
+					struct uniphier_system_bus_priv *priv)
+{
+	void __iomem *base_reg = priv->membase + UNIPHIER_SBC_BASE;
+	int is_swapped;
+
+	is_swapped = !(readl(base_reg) & UNIPHIER_SBC_BASE_BE);
+
+	dev_dbg(priv->dev, "Boot Swap: %s\n", is_swapped ? "on" : "off");
+
+	/*
+	 * If BOOT_SWAP was asserted on power-on-reset, the CS0 and CS1 are
+	 * swapped.  In this case, bank0 and bank1 should be swapped as well.
+	 */
+	if (is_swapped)
+		swap(priv->bank[0], priv->bank[1]);
+}
+
+static void uniphier_system_bus_set_reg(
+				const struct uniphier_system_bus_priv *priv)
+{
+	void __iomem *base_reg = priv->membase + UNIPHIER_SBC_BASE;
+	u32 base, end, mask, val;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->bank); i++) {
+		base = priv->bank[i].base;
+		end = priv->bank[i].end;
+
+		if (base == end) {
+			/*
+			 * If SBC_BASE0 or SBC_BASE1 is set to zero, the access
+			 * to anywhere in the system bus space is routed to
+			 * bank 0 (if boot swap if off) or bank 1 (if boot swap
+			 * if on).  It means that CPUs cannot get access to
+			 * bank 2 or later.  In other words, bank 0/1 cannot
+			 * be disabled even if its bank_enable bits is cleared.
+			 * This seems odd, but it is how this hardware goes.
+			 * As a workaround, dummy data (0xffffffff) should be
+			 * set when the bank 0/1 is unused.  As for bank 2 and
+			 * later, they can be simply disable by clearing the
+			 * bank_enable bit.
+			 */
+			if (i < 2)
+				val = UNIPHIER_SBC_BASE_DUMMY;
+			else
+				val = 0;
+		} else {
+			mask = base ^ (end - 1);
+
+			val = base & 0xfffe0000;
+			val |= (~mask >> 16) & 0xfffe;
+			val |= UNIPHIER_SBC_BASE_BE;
+		}
+		dev_dbg(priv->dev, "SBC_BASE[%d] = 0x%08x\n", i, val);
+
+		writel(val, base_reg + UNIPHIER_SBC_STRIDE * i);
+	}
+}
+
+static int uniphier_system_bus_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct uniphier_system_bus_priv *priv;
+	struct resource *regs;
+	const __be32 *ranges;
+	u32 cells, addr, size;
+	u64 paddr;
+	int pna, bank, rlen, rone, ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->membase = devm_ioremap_resource(dev, regs);
+	if (IS_ERR(priv->membase))
+		return PTR_ERR(priv->membase);
+
+	priv->dev = dev;
+
+	pna = of_n_addr_cells(dev->of_node);
+
+	ret = of_property_read_u32(dev->of_node, "#address-cells", &cells);
+	if (ret) {
+		dev_err(dev, "failed to get #address-cells\n");
+		return ret;
+	}
+	if (cells != 2) {
+		dev_err(dev, "#address-cells must be 2\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "#size-cells", &cells);
+	if (ret) {
+		dev_err(dev, "failed to get #size-cells\n");
+		return ret;
+	}
+	if (cells != 1) {
+		dev_err(dev, "#size-cells must be 1\n");
+		return -EINVAL;
+	}
+
+	ranges = of_get_property(dev->of_node, "ranges", &rlen);
+	if (!ranges) {
+		dev_err(dev, "failed to get ranges property\n");
+		return -ENOENT;
+	}
+
+	rlen /= sizeof(*ranges);
+	rone = pna + 2;
+
+	for (; rlen >= rone; rlen -= rone) {
+		bank = be32_to_cpup(ranges++);
+		addr = be32_to_cpup(ranges++);
+		paddr = of_translate_address(dev->of_node, ranges);
+		if (paddr == OF_BAD_ADDR)
+			return -EINVAL;
+		ranges += pna;
+		size = be32_to_cpup(ranges++);
+
+		ret = uniphier_system_bus_add_bank(priv, bank, addr,
+						   paddr, size);
+		if (ret)
+			return ret;
+	}
+
+	ret = uniphier_system_bus_check_overlap(priv);
+	if (ret)
+		return ret;
+
+	uniphier_system_bus_check_boot_swap(priv);
+
+	uniphier_system_bus_set_reg(priv);
+
+	/* Now, the bus is configured.  Populate platform_devices below it */
+	return of_platform_populate(dev->of_node, of_default_bus_match_table,
+				    NULL, dev);
+}
+
+static const struct of_device_id uniphier_system_bus_match[] = {
+	{ .compatible = "socionext,uniphier-system-bus" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, uniphier_system_bus_match);
+
+static struct platform_driver uniphier_system_bus_driver = {
+	.probe		= uniphier_system_bus_probe,
+	.driver = {
+		.name	= "uniphier-system-bus",
+		.of_match_table = uniphier_system_bus_match,
+	},
+};
+module_platform_driver(uniphier_system_bus_driver);
+
+MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
+MODULE_DESCRIPTION("UniPhier System Bus driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 6b1721f978c2..4f6f94c43412 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -689,7 +689,7 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
 {
 	loff_t ret;
 
-	mutex_lock(&file_inode(file)->i_mutex);
+	inode_lock(file_inode(file));
 	switch (orig) {
 	case SEEK_CUR:
 		offset += file->f_pos;
@@ -706,7 +706,7 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
 	default:
 		ret = -EINVAL;
 	}
-	mutex_unlock(&file_inode(file)->i_mutex);
+	inode_unlock(file_inode(file));
 	return ret;
 }
 
diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c
index f1d7fa45c275..f3f92d5fcda0 100644
--- a/drivers/char/mspec.c
+++ b/drivers/char/mspec.c
@@ -93,14 +93,11 @@ struct vma_data {
 	spinlock_t lock;	/* Serialize access to this structure. */
 	int count;		/* Number of pages allocated. */
 	enum mspec_page_type type; /* Type of pages allocated. */
-	int flags;		/* See VMD_xxx below. */
 	unsigned long vm_start;	/* Original (unsplit) base. */
 	unsigned long vm_end;	/* Original (unsplit) end. */
 	unsigned long maddr[0];	/* Array of MSPEC addresses. */
 };
 
-#define VMD_VMALLOCED 0x1	/* vmalloc'd rather than kmalloc'd */
-
 /* used on shub2 to clear FOP cache in the HUB */
 static unsigned long scratch_page[MAX_NUMNODES];
 #define SH2_AMO_CACHE_ENTRIES	4
@@ -185,10 +182,7 @@ mspec_close(struct vm_area_struct *vma)
 			       "failed to zero page %ld\n", my_page);
 	}
 
-	if (vdata->flags & VMD_VMALLOCED)
-		vfree(vdata);
-	else
-		kfree(vdata);
+	kvfree(vdata);
 }
 
 /*
@@ -256,7 +250,7 @@ mspec_mmap(struct file *file, struct vm_area_struct *vma,
 					enum mspec_page_type type)
 {
 	struct vma_data *vdata;
-	int pages, vdata_size, flags = 0;
+	int pages, vdata_size;
 
 	if (vma->vm_pgoff != 0)
 		return -EINVAL;
@@ -271,16 +265,13 @@ mspec_mmap(struct file *file, struct vm_area_struct *vma,
 	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
 	if (vdata_size <= PAGE_SIZE)
 		vdata = kzalloc(vdata_size, GFP_KERNEL);
-	else {
+	else
 		vdata = vzalloc(vdata_size);
-		flags = VMD_VMALLOCED;
-	}
 	if (!vdata)
 		return -ENOMEM;
 
 	vdata->vm_start = vma->vm_start;
 	vdata->vm_end = vma->vm_end;
-	vdata->flags = flags;
 	vdata->type = type;
 	spin_lock_init(&vdata->lock);
 	atomic_set(&vdata->refcnt, 1);
diff --git a/drivers/char/ps3flash.c b/drivers/char/ps3flash.c
index 0b311fa277ef..b526dc15c271 100644
--- a/drivers/char/ps3flash.c
+++ b/drivers/char/ps3flash.c
@@ -290,9 +290,9 @@ static int ps3flash_fsync(struct file *file, loff_t start, loff_t end, int datas
 {
 	struct inode *inode = file_inode(file);
 	int err;
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	err = ps3flash_writeback(ps3flash_dev);
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 	return err;
 }
 
diff --git a/drivers/char/raw.c b/drivers/char/raw.c
index 60316fbaf295..9b9809b709a5 100644
--- a/drivers/char/raw.c
+++ b/drivers/char/raw.c
@@ -71,7 +71,7 @@ static int raw_open(struct inode *inode, struct file *filp)
 	err = -ENODEV;
 	if (!bdev)
 		goto out;
-	igrab(bdev->bd_inode);
+	bdgrab(bdev);
 	err = blkdev_get(bdev, filp->f_mode | FMODE_EXCL, raw_open);
 	if (err)
 		goto out;
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index c50637db3a8a..e2fa89c88304 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -310,10 +310,12 @@ unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip,
 {
 	int duration_idx = TPM_UNDEFINED;
 	int duration = 0;
-	u8 category = (ordinal >> 24) & 0xFF;
 
-	if ((category == TPM_PROTECTED_COMMAND && ordinal < TPM_MAX_ORDINAL) ||
-	    (category == TPM_CONNECTION_COMMAND && ordinal < TSC_MAX_ORDINAL))
+	/*
+	 * We only have a duration table for protected commands, where the upper
+	 * 16 bits are 0. For the few other ordinals the fallback will be used.
+	 */
+	if (ordinal < TPM_MAX_ORDINAL)
 		duration_idx = tpm_ordinal_duration[ordinal];
 
 	if (duration_idx != TPM_UNDEFINED)
@@ -501,6 +503,21 @@ int tpm_get_timeouts(struct tpm_chip *chip)
 	struct duration_t *duration_cap;
 	ssize_t rc;
 
+	if (chip->flags & TPM_CHIP_FLAG_TPM2) {
+		/* Fixed timeouts for TPM2 */
+		chip->vendor.timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A);
+		chip->vendor.timeout_b = msecs_to_jiffies(TPM2_TIMEOUT_B);
+		chip->vendor.timeout_c = msecs_to_jiffies(TPM2_TIMEOUT_C);
+		chip->vendor.timeout_d = msecs_to_jiffies(TPM2_TIMEOUT_D);
+		chip->vendor.duration[TPM_SHORT] =
+		    msecs_to_jiffies(TPM2_DURATION_SHORT);
+		chip->vendor.duration[TPM_MEDIUM] =
+		    msecs_to_jiffies(TPM2_DURATION_MEDIUM);
+		chip->vendor.duration[TPM_LONG] =
+		    msecs_to_jiffies(TPM2_DURATION_LONG);
+		return 0;
+	}
+
 	tpm_cmd.header.in = tpm_getcap_header;
 	tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
 	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index a4257a32964f..542a80cbfd9c 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -83,16 +83,20 @@ enum tpm2_structures {
 };
 
 enum tpm2_return_codes {
-	TPM2_RC_INITIALIZE	= 0x0100,
-	TPM2_RC_TESTING		= 0x090A,
+	TPM2_RC_HASH		= 0x0083, /* RC_FMT1 */
+	TPM2_RC_INITIALIZE	= 0x0100, /* RC_VER1 */
 	TPM2_RC_DISABLED	= 0x0120,
+	TPM2_RC_TESTING		= 0x090A, /* RC_WARN */
 };
 
 enum tpm2_algorithms {
 	TPM2_ALG_SHA1		= 0x0004,
 	TPM2_ALG_KEYEDHASH	= 0x0008,
 	TPM2_ALG_SHA256		= 0x000B,
-	TPM2_ALG_NULL		= 0x0010
+	TPM2_ALG_SHA384		= 0x000C,
+	TPM2_ALG_SHA512		= 0x000D,
+	TPM2_ALG_NULL		= 0x0010,
+	TPM2_ALG_SM3_256	= 0x0012,
 };
 
 enum tpm2_command_codes {
@@ -138,7 +142,6 @@ struct tpm_vendor_specific {
 	unsigned long base;		/* TPM base address */
 
 	int irq;
-	int probed_irq;
 
 	int region_size;
 	int have_region;
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index c12130485fc1..45a634016f95 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -16,6 +16,7 @@
  */
 
 #include "tpm.h"
+#include <crypto/hash_info.h>
 #include <keys/trusted-type.h>
 
 enum tpm2_object_attributes {
@@ -104,6 +105,19 @@ struct tpm2_cmd {
 	union tpm2_cmd_params	params;
 } __packed;
 
+struct tpm2_hash {
+	unsigned int crypto_id;
+	unsigned int tpm_id;
+};
+
+static struct tpm2_hash tpm2_hash_map[] = {
+	{HASH_ALGO_SHA1, TPM2_ALG_SHA1},
+	{HASH_ALGO_SHA256, TPM2_ALG_SHA256},
+	{HASH_ALGO_SHA384, TPM2_ALG_SHA384},
+	{HASH_ALGO_SHA512, TPM2_ALG_SHA512},
+	{HASH_ALGO_SM3_256, TPM2_ALG_SM3_256},
+};
+
 /*
  * Array with one entry per ordinal defining the maximum amount
  * of time the chip could take to return the result. The values
@@ -429,8 +443,20 @@ int tpm2_seal_trusted(struct tpm_chip *chip,
 {
 	unsigned int blob_len;
 	struct tpm_buf buf;
+	u32 hash;
+	int i;
 	int rc;
 
+	for (i = 0; i < ARRAY_SIZE(tpm2_hash_map); i++) {
+		if (options->hash == tpm2_hash_map[i].crypto_id) {
+			hash = tpm2_hash_map[i].tpm_id;
+			break;
+		}
+	}
+
+	if (i == ARRAY_SIZE(tpm2_hash_map))
+		return -EINVAL;
+
 	rc = tpm_buf_init(&buf, TPM2_ST_SESSIONS, TPM2_CC_CREATE);
 	if (rc)
 		return rc;
@@ -452,12 +478,26 @@ int tpm2_seal_trusted(struct tpm_chip *chip,
 	tpm_buf_append_u8(&buf, payload->migratable);
 
 	/* public */
-	tpm_buf_append_u16(&buf, 14);
+	if (options->policydigest)
+		tpm_buf_append_u16(&buf, 14 + options->digest_len);
+	else
+		tpm_buf_append_u16(&buf, 14);
 
 	tpm_buf_append_u16(&buf, TPM2_ALG_KEYEDHASH);
-	tpm_buf_append_u16(&buf, TPM2_ALG_SHA256);
-	tpm_buf_append_u32(&buf, TPM2_ATTR_USER_WITH_AUTH);
-	tpm_buf_append_u16(&buf, 0); /* policy digest size */
+	tpm_buf_append_u16(&buf, hash);
+
+	/* policy */
+	if (options->policydigest) {
+		tpm_buf_append_u32(&buf, 0);
+		tpm_buf_append_u16(&buf, options->digest_len);
+		tpm_buf_append(&buf, options->policydigest,
+			       options->digest_len);
+	} else {
+		tpm_buf_append_u32(&buf, TPM2_ATTR_USER_WITH_AUTH);
+		tpm_buf_append_u16(&buf, 0);
+	}
+
+	/* public parameters */
 	tpm_buf_append_u16(&buf, TPM2_ALG_NULL);
 	tpm_buf_append_u16(&buf, 0);
 
@@ -488,8 +528,12 @@ int tpm2_seal_trusted(struct tpm_chip *chip,
 out:
 	tpm_buf_destroy(&buf);
 
-	if (rc > 0)
-		rc = -EPERM;
+	if (rc > 0) {
+		if ((rc & TPM2_RC_HASH) == TPM2_RC_HASH)
+			rc = -EINVAL;
+		else
+			rc = -EPERM;
+	}
 
 	return rc;
 }
@@ -583,7 +627,9 @@ static int tpm2_unseal(struct tpm_chip *chip,
 		return rc;
 
 	tpm_buf_append_u32(&buf, blob_handle);
-	tpm2_buf_append_auth(&buf, TPM2_RS_PW,
+	tpm2_buf_append_auth(&buf,
+			     options->policyhandle ?
+			     options->policyhandle : TPM2_RS_PW,
 			     NULL /* nonce */, 0,
 			     0 /* session_attributes */,
 			     options->blobauth /* hmac */,
diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c
index 4bb9727c1047..8342cf51ffdc 100644
--- a/drivers/char/tpm/tpm_crb.c
+++ b/drivers/char/tpm/tpm_crb.c
@@ -284,17 +284,9 @@ static int crb_acpi_add(struct acpi_device *device)
 
 	chip->vendor.priv = priv;
 
-	/* Default timeouts and durations */
-	chip->vendor.timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A);
-	chip->vendor.timeout_b = msecs_to_jiffies(TPM2_TIMEOUT_B);
-	chip->vendor.timeout_c = msecs_to_jiffies(TPM2_TIMEOUT_C);
-	chip->vendor.timeout_d = msecs_to_jiffies(TPM2_TIMEOUT_D);
-	chip->vendor.duration[TPM_SHORT] =
-		msecs_to_jiffies(TPM2_DURATION_SHORT);
-	chip->vendor.duration[TPM_MEDIUM] =
-		msecs_to_jiffies(TPM2_DURATION_MEDIUM);
-	chip->vendor.duration[TPM_LONG] =
-		msecs_to_jiffies(TPM2_DURATION_LONG);
+	rc = tpm_get_timeouts(chip);
+	if (rc)
+		return rc;
 
 	chip->acpi_dev_handle = device->handle;
 
diff --git a/drivers/char/tpm/tpm_ibmvtpm.c b/drivers/char/tpm/tpm_ibmvtpm.c
index 3e6a22658b63..b0a9a9e34241 100644
--- a/drivers/char/tpm/tpm_ibmvtpm.c
+++ b/drivers/char/tpm/tpm_ibmvtpm.c
@@ -90,7 +90,7 @@ static int tpm_ibmvtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 		return 0;
 	}
 
-	sig = wait_event_interruptible(ibmvtpm->wq, ibmvtpm->res_len != 0);
+	sig = wait_event_interruptible(ibmvtpm->wq, !ibmvtpm->tpm_processing_cmd);
 	if (sig)
 		return -EINTR;
 
@@ -125,7 +125,7 @@ static int tpm_ibmvtpm_send(struct tpm_chip *chip, u8 *buf, size_t count)
 	struct ibmvtpm_dev *ibmvtpm;
 	struct ibmvtpm_crq crq;
 	__be64 *word = (__be64 *)&crq;
-	int rc;
+	int rc, sig;
 
 	ibmvtpm = (struct ibmvtpm_dev *)TPM_VPRIV(chip);
 
@@ -141,18 +141,35 @@ static int tpm_ibmvtpm_send(struct tpm_chip *chip, u8 *buf, size_t count)
 		return -EIO;
 	}
 
+	if (ibmvtpm->tpm_processing_cmd) {
+		dev_info(ibmvtpm->dev,
+		         "Need to wait for TPM to finish\n");
+		/* wait for previous command to finish */
+		sig = wait_event_interruptible(ibmvtpm->wq, !ibmvtpm->tpm_processing_cmd);
+		if (sig)
+			return -EINTR;
+	}
+
 	spin_lock(&ibmvtpm->rtce_lock);
+	ibmvtpm->res_len = 0;
 	memcpy((void *)ibmvtpm->rtce_buf, (void *)buf, count);
 	crq.valid = (u8)IBMVTPM_VALID_CMD;
 	crq.msg = (u8)VTPM_TPM_COMMAND;
 	crq.len = cpu_to_be16(count);
 	crq.data = cpu_to_be32(ibmvtpm->rtce_dma_handle);
 
+	/*
+	 * set the processing flag before the Hcall, since we may get the
+	 * result (interrupt) before even being able to check rc.
+	 */
+	ibmvtpm->tpm_processing_cmd = true;
+
 	rc = ibmvtpm_send_crq(ibmvtpm->vdev, be64_to_cpu(word[0]),
 			      be64_to_cpu(word[1]));
 	if (rc != H_SUCCESS) {
 		dev_err(ibmvtpm->dev, "tpm_ibmvtpm_send failed rc=%d\n", rc);
 		rc = 0;
+		ibmvtpm->tpm_processing_cmd = false;
 	} else
 		rc = count;
 
@@ -515,6 +532,7 @@ static void ibmvtpm_crq_process(struct ibmvtpm_crq *crq,
 		case VTPM_TPM_COMMAND_RES:
 			/* len of the data in rtce buffer */
 			ibmvtpm->res_len = be16_to_cpu(crq->len);
+			ibmvtpm->tpm_processing_cmd = false;
 			wake_up_interruptible(&ibmvtpm->wq);
 			return;
 		default:
diff --git a/drivers/char/tpm/tpm_ibmvtpm.h b/drivers/char/tpm/tpm_ibmvtpm.h
index 6af92890518f..91dfe766d080 100644
--- a/drivers/char/tpm/tpm_ibmvtpm.h
+++ b/drivers/char/tpm/tpm_ibmvtpm.h
@@ -45,6 +45,7 @@ struct ibmvtpm_dev {
 	wait_queue_head_t wq;
 	u16 res_len;
 	u32 vtpm_version;
+	bool tpm_processing_cmd;
 };
 
 #define CRQ_RES_BUF_SIZE	PAGE_SIZE
diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c
index 65f7eecc45b0..8a3509cb10da 100644
--- a/drivers/char/tpm/tpm_tis.c
+++ b/drivers/char/tpm/tpm_tis.c
@@ -401,7 +401,7 @@ static void disable_interrupts(struct tpm_chip *chip)
 	iowrite32(intmask,
 		  chip->vendor.iobase +
 		  TPM_INT_ENABLE(chip->vendor.locality));
-	free_irq(chip->vendor.irq, chip);
+	devm_free_irq(chip->pdev, chip->vendor.irq, chip);
 	chip->vendor.irq = 0;
 }
 
@@ -461,11 +461,8 @@ static int tpm_tis_send(struct tpm_chip *chip, u8 *buf, size_t len)
 	chip->vendor.irq = irq;
 	if (!priv->irq_tested)
 		msleep(1);
-	if (!priv->irq_tested) {
+	if (!priv->irq_tested)
 		disable_interrupts(chip);
-		dev_err(chip->pdev,
-			FW_BUG "TPM interrupt not working, polling instead\n");
-	}
 	priv->irq_tested = true;
 	return rc;
 }
@@ -570,26 +567,6 @@ static const struct tpm_class_ops tpm_tis = {
 	.req_canceled = tpm_tis_req_canceled,
 };
 
-static irqreturn_t tis_int_probe(int irq, void *dev_id)
-{
-	struct tpm_chip *chip = dev_id;
-	u32 interrupt;
-
-	interrupt = ioread32(chip->vendor.iobase +
-			     TPM_INT_STATUS(chip->vendor.locality));
-
-	if (interrupt == 0)
-		return IRQ_NONE;
-
-	chip->vendor.probed_irq = irq;
-
-	/* Clear interrupts handled with TPM_EOI */
-	iowrite32(interrupt,
-		  chip->vendor.iobase +
-		  TPM_INT_STATUS(chip->vendor.locality));
-	return IRQ_HANDLED;
-}
-
 static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 {
 	struct tpm_chip *chip = dev_id;
@@ -622,6 +599,84 @@ static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+/* Register the IRQ and issue a command that will cause an interrupt. If an
+ * irq is seen then leave the chip setup for IRQ operation, otherwise reverse
+ * everything and leave in polling mode. Returns 0 on success.
+ */
+static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask,
+				    int flags, int irq)
+{
+	struct priv_data *priv = chip->vendor.priv;
+	u8 original_int_vec;
+
+	if (devm_request_irq(chip->pdev, irq, tis_int_handler, flags,
+			     chip->devname, chip) != 0) {
+		dev_info(chip->pdev, "Unable to request irq: %d for probe\n",
+			 irq);
+		return -1;
+	}
+	chip->vendor.irq = irq;
+
+	original_int_vec = ioread8(chip->vendor.iobase +
+				   TPM_INT_VECTOR(chip->vendor.locality));
+	iowrite8(irq,
+		 chip->vendor.iobase + TPM_INT_VECTOR(chip->vendor.locality));
+
+	/* Clear all existing */
+	iowrite32(ioread32(chip->vendor.iobase +
+			   TPM_INT_STATUS(chip->vendor.locality)),
+		  chip->vendor.iobase + TPM_INT_STATUS(chip->vendor.locality));
+
+	/* Turn on */
+	iowrite32(intmask | TPM_GLOBAL_INT_ENABLE,
+		  chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality));
+
+	priv->irq_tested = false;
+
+	/* Generate an interrupt by having the core call through to
+	 * tpm_tis_send
+	 */
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		tpm2_gen_interrupt(chip);
+	else
+		tpm_gen_interrupt(chip);
+
+	/* tpm_tis_send will either confirm the interrupt is working or it
+	 * will call disable_irq which undoes all of the above.
+	 */
+	if (!chip->vendor.irq) {
+		iowrite8(original_int_vec,
+			 chip->vendor.iobase +
+			     TPM_INT_VECTOR(chip->vendor.locality));
+		return 1;
+	}
+
+	return 0;
+}
+
+/* Try to find the IRQ the TPM is using. This is for legacy x86 systems that
+ * do not have ACPI/etc. We typically expect the interrupt to be declared if
+ * present.
+ */
+static void tpm_tis_probe_irq(struct tpm_chip *chip, u32 intmask)
+{
+	u8 original_int_vec;
+	int i;
+
+	original_int_vec = ioread8(chip->vendor.iobase +
+				   TPM_INT_VECTOR(chip->vendor.locality));
+
+	if (!original_int_vec) {
+		if (IS_ENABLED(CONFIG_X86))
+			for (i = 3; i <= 15; i++)
+				if (!tpm_tis_probe_irq_single(chip, intmask, 0,
+							      i))
+					return;
+	} else if (!tpm_tis_probe_irq_single(chip, intmask, 0,
+					     original_int_vec))
+		return;
+}
+
 static bool interrupts = true;
 module_param(interrupts, bool, 0444);
 MODULE_PARM_DESC(interrupts, "Enable interrupts");
@@ -644,8 +699,7 @@ static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info,
 			acpi_handle acpi_dev_handle)
 {
 	u32 vendor, intfcaps, intmask;
-	int rc, i, irq_s, irq_e, probe;
-	int irq_r = -1;
+	int rc, probe;
 	struct tpm_chip *chip;
 	struct priv_data *priv;
 
@@ -677,6 +731,15 @@ static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info,
 		goto out_err;
 	}
 
+	/* Take control of the TPM's interrupt hardware and shut it off */
+	intmask = ioread32(chip->vendor.iobase +
+			   TPM_INT_ENABLE(chip->vendor.locality));
+	intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT |
+		   TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT;
+	intmask &= ~TPM_GLOBAL_INT_ENABLE;
+	iowrite32(intmask,
+		  chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality));
+
 	if (request_locality(chip, 0) != 0) {
 		rc = -ENODEV;
 		goto out_err;
@@ -731,126 +794,31 @@ static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info,
 	if (intfcaps & TPM_INTF_DATA_AVAIL_INT)
 		dev_dbg(dev, "\tData Avail Int Support\n");
 
+	/* Very early on issue a command to the TPM in polling mode to make
+	 * sure it works. May as well use that command to set the proper
+	 *  timeouts for the driver.
+	 */
+	if (tpm_get_timeouts(chip)) {
+		dev_err(dev, "Could not get TPM timeouts and durations\n");
+		rc = -ENODEV;
+		goto out_err;
+	}
+
 	/* INTERRUPT Setup */
 	init_waitqueue_head(&chip->vendor.read_queue);
 	init_waitqueue_head(&chip->vendor.int_queue);
-
-	intmask =
-	    ioread32(chip->vendor.iobase +
-		     TPM_INT_ENABLE(chip->vendor.locality));
-
-	intmask |= TPM_INTF_CMD_READY_INT
-	    | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT
-	    | TPM_INTF_STS_VALID_INT;
-
-	iowrite32(intmask,
-		  chip->vendor.iobase +
-		  TPM_INT_ENABLE(chip->vendor.locality));
-	if (interrupts)
-		chip->vendor.irq = tpm_info->irq;
-	if (interrupts && !chip->vendor.irq) {
-		irq_s =
-		    ioread8(chip->vendor.iobase +
-			    TPM_INT_VECTOR(chip->vendor.locality));
-		irq_r = irq_s;
-		if (irq_s) {
-			irq_e = irq_s;
-		} else {
-			irq_s = 3;
-			irq_e = 15;
-		}
-
-		for (i = irq_s; i <= irq_e && chip->vendor.irq == 0; i++) {
-			iowrite8(i, chip->vendor.iobase +
-				 TPM_INT_VECTOR(chip->vendor.locality));
-			if (devm_request_irq
-			    (dev, i, tis_int_probe, IRQF_SHARED,
-			     chip->devname, chip) != 0) {
-				dev_info(chip->pdev,
-					 "Unable to request irq: %d for probe\n",
-					 i);
-				continue;
-			}
-
-			/* Clear all existing */
-			iowrite32(ioread32
-				  (chip->vendor.iobase +
-				   TPM_INT_STATUS(chip->vendor.locality)),
-				  chip->vendor.iobase +
-				  TPM_INT_STATUS(chip->vendor.locality));
-
-			/* Turn on */
-			iowrite32(intmask | TPM_GLOBAL_INT_ENABLE,
-				  chip->vendor.iobase +
-				  TPM_INT_ENABLE(chip->vendor.locality));
-
-			chip->vendor.probed_irq = 0;
-
-			/* Generate Interrupts */
-			if (chip->flags & TPM_CHIP_FLAG_TPM2)
-				tpm2_gen_interrupt(chip);
-			else
-				tpm_gen_interrupt(chip);
-
-			chip->vendor.irq = chip->vendor.probed_irq;
-
-			/* free_irq will call into tis_int_probe;
-			   clear all irqs we haven't seen while doing
-			   tpm_gen_interrupt */
-			iowrite32(ioread32
-				  (chip->vendor.iobase +
-				   TPM_INT_STATUS(chip->vendor.locality)),
-				  chip->vendor.iobase +
-				  TPM_INT_STATUS(chip->vendor.locality));
-
-			/* Turn off */
-			iowrite32(intmask,
-				  chip->vendor.iobase +
-				  TPM_INT_ENABLE(chip->vendor.locality));
-
-			devm_free_irq(dev, i, chip);
-		}
+	if (interrupts) {
+		if (tpm_info->irq) {
+			tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED,
+						 tpm_info->irq);
+			if (!chip->vendor.irq)
+				dev_err(chip->pdev, FW_BUG
+					"TPM interrupt not working, polling instead\n");
+		} else
+			tpm_tis_probe_irq(chip, intmask);
 	}
-	if (chip->vendor.irq) {
-		iowrite8(chip->vendor.irq,
-			 chip->vendor.iobase +
-			 TPM_INT_VECTOR(chip->vendor.locality));
-		if (devm_request_irq
-		    (dev, chip->vendor.irq, tis_int_handler, IRQF_SHARED,
-		     chip->devname, chip) != 0) {
-			dev_info(chip->pdev,
-				 "Unable to request irq: %d for use\n",
-				 chip->vendor.irq);
-			chip->vendor.irq = 0;
-		} else {
-			/* Clear all existing */
-			iowrite32(ioread32
-				  (chip->vendor.iobase +
-				   TPM_INT_STATUS(chip->vendor.locality)),
-				  chip->vendor.iobase +
-				  TPM_INT_STATUS(chip->vendor.locality));
-
-			/* Turn on */
-			iowrite32(intmask | TPM_GLOBAL_INT_ENABLE,
-				  chip->vendor.iobase +
-				  TPM_INT_ENABLE(chip->vendor.locality));
-		}
-	} else if (irq_r != -1)
-		iowrite8(irq_r, chip->vendor.iobase +
-			 TPM_INT_VECTOR(chip->vendor.locality));
 
 	if (chip->flags & TPM_CHIP_FLAG_TPM2) {
-		chip->vendor.timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A);
-		chip->vendor.timeout_b = msecs_to_jiffies(TPM2_TIMEOUT_B);
-		chip->vendor.timeout_c = msecs_to_jiffies(TPM2_TIMEOUT_C);
-		chip->vendor.timeout_d = msecs_to_jiffies(TPM2_TIMEOUT_D);
-		chip->vendor.duration[TPM_SHORT] =
-			msecs_to_jiffies(TPM2_DURATION_SHORT);
-		chip->vendor.duration[TPM_MEDIUM] =
-			msecs_to_jiffies(TPM2_DURATION_MEDIUM);
-		chip->vendor.duration[TPM_LONG] =
-			msecs_to_jiffies(TPM2_DURATION_LONG);
-
 		rc = tpm2_do_selftest(chip);
 		if (rc == TPM2_RC_INITIALIZE) {
 			dev_warn(dev, "Firmware has not started TPM\n");
@@ -866,12 +834,6 @@ static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info,
 			goto out_err;
 		}
 	} else {
-		if (tpm_get_timeouts(chip)) {
-			dev_err(dev, "Could not get TPM timeouts and durations\n");
-			rc = -ENODEV;
-			goto out_err;
-		}
-
 		if (tpm_do_selftest(chip)) {
 			dev_err(dev, "TPM self test failed\n");
 			rc = -ENODEV;
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index c3e3a02f7f1f..eca8e019e005 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -116,6 +116,12 @@ config COMMON_CLK_CDCE925
 	  Given a target output frequency, the driver will set the PLL and
 	  divider to best approximate the desired output.
 
+config COMMON_CLK_CS2000_CP
+	tristate "Clock driver for CS2000 Fractional-N Clock Synthesizer & Clock Multiplier"
+	depends on I2C
+	help
+	  If you say yes here you get support for the CS2000 clock multiplier.
+
 config COMMON_CLK_S2MPS11
 	tristate "Clock driver for S2MPS1X/S5M8767 MFD"
 	depends on MFD_SEC_CORE
@@ -161,6 +167,12 @@ config COMMON_CLK_KEYSTONE
           Supports clock drivers for Keystone based SOCs. These SOCs have local
 	  a power sleep control module that gate the clock to the IPs and PLLs.
 
+config COMMON_CLK_NXP
+	def_bool COMMON_CLK && (ARCH_LPC18XX || ARCH_LPC32XX)
+	select REGMAP_MMIO if ARCH_LPC32XX
+	---help---
+	  Support for clock providers on NXP platforms.
+
 config COMMON_CLK_PALMAS
 	tristate "Clock driver for TI Palmas devices"
 	depends on MFD_PALMAS
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 820714c72d36..b038e3666058 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -21,6 +21,7 @@ obj-$(CONFIG_MACH_ASM9260)		+= clk-asm9260.o
 obj-$(CONFIG_COMMON_CLK_AXI_CLKGEN)	+= clk-axi-clkgen.o
 obj-$(CONFIG_ARCH_AXXIA)		+= clk-axm5516.o
 obj-$(CONFIG_COMMON_CLK_CDCE706)	+= clk-cdce706.o
+obj-$(CONFIG_COMMON_CLK_CS2000_CP)	+= clk-cs2000-cp.o
 obj-$(CONFIG_ARCH_CLPS711X)		+= clk-clps711x.o
 obj-$(CONFIG_ARCH_EFM32)		+= clk-efm32gg.o
 obj-$(CONFIG_ARCH_HIGHBANK)		+= clk-highbank.o
@@ -42,6 +43,7 @@ obj-$(CONFIG_COMMON_CLK_SI514)		+= clk-si514.o
 obj-$(CONFIG_COMMON_CLK_SI570)		+= clk-si570.o
 obj-$(CONFIG_COMMON_CLK_CDCE925)	+= clk-cdce925.o
 obj-$(CONFIG_ARCH_STM32)		+= clk-stm32f4.o
+obj-$(CONFIG_ARCH_TANGOX)		+= clk-tango4.o
 obj-$(CONFIG_CLK_TWL6040)		+= clk-twl6040.o
 obj-$(CONFIG_ARCH_U300)			+= clk-u300.o
 obj-$(CONFIG_ARCH_VT8500)		+= clk-vt8500.o
@@ -62,13 +64,14 @@ endif
 obj-$(CONFIG_PLAT_ORION)		+= mvebu/
 obj-$(CONFIG_ARCH_MESON)		+= meson/
 obj-$(CONFIG_ARCH_MXS)			+= mxs/
-obj-$(CONFIG_ARCH_LPC18XX)		+= nxp/
 obj-$(CONFIG_MACH_PISTACHIO)		+= pistachio/
+obj-$(CONFIG_COMMON_CLK_NXP)		+= nxp/
 obj-$(CONFIG_COMMON_CLK_PXA)		+= pxa/
 obj-$(CONFIG_COMMON_CLK_QCOM)		+= qcom/
 obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
 obj-$(CONFIG_COMMON_CLK_SAMSUNG)	+= samsung/
 obj-$(CONFIG_ARCH_SHMOBILE_MULTI)	+= shmobile/
+obj-$(CONFIG_ARCH_RENESAS)		+= shmobile/
 obj-$(CONFIG_ARCH_SIRF)			+= sirf/
 obj-$(CONFIG_ARCH_SOCFPGA)		+= socfpga/
 obj-$(CONFIG_PLAT_SPEAR)		+= spear/
diff --git a/drivers/clk/at91/clk-slow.c b/drivers/clk/at91/clk-slow.c
index d0d5076a9b94..6f99a530ead6 100644
--- a/drivers/clk/at91/clk-slow.c
+++ b/drivers/clk/at91/clk-slow.c
@@ -10,7 +10,6 @@
  *
  */
 
-#include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
 #include <linux/slab.h>
@@ -72,8 +71,6 @@ struct clk_sam9x5_slow {
 
 #define to_clk_sam9x5_slow(hw) container_of(hw, struct clk_sam9x5_slow, hw)
 
-static struct clk *slow_clk;
-
 static int clk_slow_osc_prepare(struct clk_hw *hw)
 {
 	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
@@ -360,8 +357,6 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr,
 	clk = clk_register(NULL, &slowck->hw);
 	if (IS_ERR(clk))
 		kfree(slowck);
-	else
-		slow_clk = clk;
 
 	return clk;
 }
@@ -433,8 +428,6 @@ at91_clk_register_sam9260_slow(struct at91_pmc *pmc,
 	clk = clk_register(NULL, &slowck->hw);
 	if (IS_ERR(clk))
 		kfree(slowck);
-	else
-		slow_clk = clk;
 
 	return clk;
 }
@@ -462,25 +455,3 @@ void __init of_at91sam9260_clk_slow_setup(struct device_node *np,
 
 	of_clk_add_provider(np, of_clk_src_simple_get, clk);
 }
-
-/*
- * FIXME: All slow clk users are not properly claiming it (get + prepare +
- * enable) before using it.
- * If all users properly claiming this clock decide that they don't need it
- * anymore (or are removed), it is disabled while faulty users are still
- * requiring it, and the system hangs.
- * Prevent this clock from being disabled until all users are properly
- * requesting it.
- * Once this is done we should remove this function and the slow_clk variable.
- */
-static int __init of_at91_clk_slow_retain(void)
-{
-	if (!slow_clk)
-		return 0;
-
-	__clk_get(slow_clk);
-	clk_prepare_enable(slow_clk);
-
-	return 0;
-}
-arch_initcall(of_at91_clk_slow_retain);
diff --git a/drivers/clk/bcm/Kconfig b/drivers/clk/bcm/Kconfig
index 85260fb96b36..f2878459199a 100644
--- a/drivers/clk/bcm/Kconfig
+++ b/drivers/clk/bcm/Kconfig
@@ -1,3 +1,13 @@
+config CLK_BCM_63XX
+	bool "Broadcom BCM63xx clock support"
+	depends on ARCH_BCM_63XX || COMPILE_TEST
+	depends on COMMON_CLK
+	select COMMON_CLK_IPROC
+	default ARCH_BCM_63XX
+	help
+	  Enable common clock framework support for Broadcom BCM63xx DSL SoCs
+	  based on the ARM architecture
+
 config CLK_BCM_KONA
 	bool "Broadcom Kona CCU clock support"
 	depends on ARCH_BCM_MOBILE || COMPILE_TEST
diff --git a/drivers/clk/bcm/Makefile b/drivers/clk/bcm/Makefile
index 3fc95060d875..1d79bd2c36f0 100644
--- a/drivers/clk/bcm/Makefile
+++ b/drivers/clk/bcm/Makefile
@@ -1,9 +1,11 @@
+obj-$(CONFIG_CLK_BCM_63XX)	+= clk-bcm63xx.o
 obj-$(CONFIG_CLK_BCM_KONA)	+= clk-kona.o
 obj-$(CONFIG_CLK_BCM_KONA)	+= clk-kona-setup.o
 obj-$(CONFIG_CLK_BCM_KONA)	+= clk-bcm281xx.o
 obj-$(CONFIG_CLK_BCM_KONA)	+= clk-bcm21664.o
 obj-$(CONFIG_COMMON_CLK_IPROC)	+= clk-iproc-armpll.o clk-iproc-pll.o clk-iproc-asiu.o
 obj-$(CONFIG_ARCH_BCM2835)	+= clk-bcm2835.o
+obj-$(CONFIG_ARCH_BCM2835)	+= clk-bcm2835-aux.o
 obj-$(CONFIG_COMMON_CLK_IPROC)	+= clk-ns2.o
 obj-$(CONFIG_ARCH_BCM_CYGNUS)	+= clk-cygnus.o
 obj-$(CONFIG_ARCH_BCM_NSP)	+= clk-nsp.o
diff --git a/drivers/clk/bcm/clk-bcm2835-aux.c b/drivers/clk/bcm/clk-bcm2835-aux.c
new file mode 100644
index 000000000000..e4f89e28b5ec
--- /dev/null
+++ b/drivers/clk/bcm/clk-bcm2835-aux.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/clk/bcm2835.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/clock/bcm2835-aux.h>
+
+#define BCM2835_AUXIRQ		0x00
+#define BCM2835_AUXENB		0x04
+
+static int bcm2835_aux_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct clk_onecell_data *onecell;
+	const char *parent;
+	struct clk *parent_clk;
+	struct resource *res;
+	void __iomem *reg, *gate;
+
+	parent_clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(parent_clk))
+		return PTR_ERR(parent_clk);
+	parent = __clk_get_name(parent_clk);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(dev, res);
+	if (!reg)
+		return -ENODEV;
+
+	onecell = devm_kmalloc(dev, sizeof(*onecell), GFP_KERNEL);
+	if (!onecell)
+		return -ENOMEM;
+	onecell->clk_num = BCM2835_AUX_CLOCK_COUNT;
+	onecell->clks = devm_kcalloc(dev, BCM2835_AUX_CLOCK_COUNT,
+				     sizeof(*onecell->clks), GFP_KERNEL);
+	if (!onecell->clks)
+		return -ENOMEM;
+
+	gate = reg + BCM2835_AUXENB;
+	onecell->clks[BCM2835_AUX_CLOCK_UART] =
+		clk_register_gate(dev, "aux_uart", parent, 0, gate, 0, 0, NULL);
+
+	onecell->clks[BCM2835_AUX_CLOCK_SPI1] =
+		clk_register_gate(dev, "aux_spi1", parent, 0, gate, 1, 0, NULL);
+
+	onecell->clks[BCM2835_AUX_CLOCK_SPI2] =
+		clk_register_gate(dev, "aux_spi2", parent, 0, gate, 2, 0, NULL);
+
+	of_clk_add_provider(pdev->dev.of_node, of_clk_src_onecell_get, onecell);
+
+	return 0;
+}
+
+static const struct of_device_id bcm2835_aux_clk_of_match[] = {
+	{ .compatible = "brcm,bcm2835-aux", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm2835_aux_clk_of_match);
+
+static struct platform_driver bcm2835_aux_clk_driver = {
+	.driver = {
+		.name = "bcm2835-aux-clk",
+		.of_match_table = bcm2835_aux_clk_of_match,
+	},
+	.probe          = bcm2835_aux_clk_probe,
+};
+builtin_platform_driver(bcm2835_aux_clk_driver);
+
+MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
+MODULE_DESCRIPTION("BCM2835 auxiliary peripheral clock driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index 39bf5820297e..015e687ffabe 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -807,6 +807,16 @@ static const struct bcm2835_clock_data bcm2835_clock_emmc_data = {
 	.frac_bits = 8,
 };
 
+static const struct bcm2835_clock_data bcm2835_clock_pwm_data = {
+	.name = "pwm",
+	.num_mux_parents = ARRAY_SIZE(bcm2835_clock_per_parents),
+	.parents = bcm2835_clock_per_parents,
+	.ctl_reg = CM_PWMCTL,
+	.div_reg = CM_PWMDIV,
+	.int_bits = 12,
+	.frac_bits = 12,
+};
+
 struct bcm2835_pll {
 	struct clk_hw hw;
 	struct bcm2835_cprman *cprman;
@@ -1148,22 +1158,24 @@ static int bcm2835_clock_is_on(struct clk_hw *hw)
 
 static u32 bcm2835_clock_choose_div(struct clk_hw *hw,
 				    unsigned long rate,
-				    unsigned long parent_rate)
+				    unsigned long parent_rate,
+				    bool round_up)
 {
 	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
 	const struct bcm2835_clock_data *data = clock->data;
-	u32 unused_frac_mask = GENMASK(CM_DIV_FRAC_BITS - data->frac_bits, 0);
+	u32 unused_frac_mask =
+		GENMASK(CM_DIV_FRAC_BITS - data->frac_bits, 0) >> 1;
 	u64 temp = (u64)parent_rate << CM_DIV_FRAC_BITS;
+	u64 rem;
 	u32 div;
 
-	do_div(temp, rate);
+	rem = do_div(temp, rate);
 	div = temp;
 
-	/* Round and mask off the unused bits */
-	if (unused_frac_mask != 0) {
-		div += unused_frac_mask >> 1;
-		div &= ~unused_frac_mask;
-	}
+	/* Round up and mask off the unused bits */
+	if (round_up && ((div & unused_frac_mask) != 0 || rem != 0))
+		div += unused_frac_mask + 1;
+	div &= ~unused_frac_mask;
 
 	/* Clamp to the limits. */
 	div = max(div, unused_frac_mask + 1);
@@ -1197,16 +1209,6 @@ static long bcm2835_clock_rate_from_divisor(struct bcm2835_clock *clock,
 	return temp;
 }
 
-static long bcm2835_clock_round_rate(struct clk_hw *hw,
-				     unsigned long rate,
-				     unsigned long *parent_rate)
-{
-	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
-	u32 div = bcm2835_clock_choose_div(hw, rate, *parent_rate);
-
-	return bcm2835_clock_rate_from_divisor(clock, *parent_rate, div);
-}
-
 static unsigned long bcm2835_clock_get_rate(struct clk_hw *hw,
 					    unsigned long parent_rate)
 {
@@ -1271,20 +1273,82 @@ static int bcm2835_clock_set_rate(struct clk_hw *hw,
 	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
 	struct bcm2835_cprman *cprman = clock->cprman;
 	const struct bcm2835_clock_data *data = clock->data;
-	u32 div = bcm2835_clock_choose_div(hw, rate, parent_rate);
+	u32 div = bcm2835_clock_choose_div(hw, rate, parent_rate, false);
 
 	cprman_write(cprman, data->div_reg, div);
 
 	return 0;
 }
 
+static int bcm2835_clock_determine_rate(struct clk_hw *hw,
+		struct clk_rate_request *req)
+{
+	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+	struct clk_hw *parent, *best_parent = NULL;
+	unsigned long rate, best_rate = 0;
+	unsigned long prate, best_prate = 0;
+	size_t i;
+	u32 div;
+
+	/*
+	 * Select parent clock that results in the closest but lower rate
+	 */
+	for (i = 0; i < clk_hw_get_num_parents(hw); ++i) {
+		parent = clk_hw_get_parent_by_index(hw, i);
+		if (!parent)
+			continue;
+		prate = clk_hw_get_rate(parent);
+		div = bcm2835_clock_choose_div(hw, req->rate, prate, true);
+		rate = bcm2835_clock_rate_from_divisor(clock, prate, div);
+		if (rate > best_rate && rate <= req->rate) {
+			best_parent = parent;
+			best_prate = prate;
+			best_rate = rate;
+		}
+	}
+
+	if (!best_parent)
+		return -EINVAL;
+
+	req->best_parent_hw = best_parent;
+	req->best_parent_rate = best_prate;
+
+	req->rate = best_rate;
+
+	return 0;
+}
+
+static int bcm2835_clock_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+	struct bcm2835_cprman *cprman = clock->cprman;
+	const struct bcm2835_clock_data *data = clock->data;
+	u8 src = (index << CM_SRC_SHIFT) & CM_SRC_MASK;
+
+	cprman_write(cprman, data->ctl_reg, src);
+	return 0;
+}
+
+static u8 bcm2835_clock_get_parent(struct clk_hw *hw)
+{
+	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+	struct bcm2835_cprman *cprman = clock->cprman;
+	const struct bcm2835_clock_data *data = clock->data;
+	u32 src = cprman_read(cprman, data->ctl_reg);
+
+	return (src & CM_SRC_MASK) >> CM_SRC_SHIFT;
+}
+
+
 static const struct clk_ops bcm2835_clock_clk_ops = {
 	.is_prepared = bcm2835_clock_is_on,
 	.prepare = bcm2835_clock_on,
 	.unprepare = bcm2835_clock_off,
 	.recalc_rate = bcm2835_clock_get_rate,
 	.set_rate = bcm2835_clock_set_rate,
-	.round_rate = bcm2835_clock_round_rate,
+	.determine_rate = bcm2835_clock_determine_rate,
+	.set_parent = bcm2835_clock_set_parent,
+	.get_parent = bcm2835_clock_get_parent,
 };
 
 static int bcm2835_vpu_clock_is_on(struct clk_hw *hw)
@@ -1300,7 +1364,9 @@ static const struct clk_ops bcm2835_vpu_clock_clk_ops = {
 	.is_prepared = bcm2835_vpu_clock_is_on,
 	.recalc_rate = bcm2835_clock_get_rate,
 	.set_rate = bcm2835_clock_set_rate,
-	.round_rate = bcm2835_clock_round_rate,
+	.determine_rate = bcm2835_clock_determine_rate,
+	.set_parent = bcm2835_clock_set_parent,
+	.get_parent = bcm2835_clock_get_parent,
 };
 
 static struct clk *bcm2835_register_pll(struct bcm2835_cprman *cprman,
@@ -1394,45 +1460,23 @@ static struct clk *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 {
 	struct bcm2835_clock *clock;
 	struct clk_init_data init;
-	const char *parent;
+	const char *parents[1 << CM_SRC_BITS];
+	size_t i;
 
 	/*
-	 * Most of the clock generators have a mux field, so we
-	 * instantiate a generic mux as our parent to handle it.
+	 * Replace our "xosc" references with the oscillator's
+	 * actual name.
 	 */
-	if (data->num_mux_parents) {
-		const char *parents[1 << CM_SRC_BITS];
-		int i;
-
-		parent = devm_kasprintf(cprman->dev, GFP_KERNEL,
-					"mux_%s", data->name);
-		if (!parent)
-			return NULL;
-
-		/*
-		 * Replace our "xosc" references with the oscillator's
-		 * actual name.
-		 */
-		for (i = 0; i < data->num_mux_parents; i++) {
-			if (strcmp(data->parents[i], "xosc") == 0)
-				parents[i] = cprman->osc_name;
-			else
-				parents[i] = data->parents[i];
-		}
-
-		clk_register_mux(cprman->dev, parent,
-				 parents, data->num_mux_parents,
-				 CLK_SET_RATE_PARENT,
-				 cprman->regs + data->ctl_reg,
-				 CM_SRC_SHIFT, CM_SRC_BITS,
-				 0, &cprman->regs_lock);
-	} else {
-		parent = data->parents[0];
+	for (i = 0; i < data->num_mux_parents; i++) {
+		if (strcmp(data->parents[i], "xosc") == 0)
+			parents[i] = cprman->osc_name;
+		else
+			parents[i] = data->parents[i];
 	}
 
 	memset(&init, 0, sizeof(init));
-	init.parent_names = &parent;
-	init.num_parents = 1;
+	init.parent_names = parents;
+	init.num_parents = data->num_mux_parents;
 	init.name = data->name;
 	init.flags = CLK_IGNORE_UNUSED;
 
@@ -1550,6 +1594,9 @@ static int bcm2835_clk_probe(struct platform_device *pdev)
 				  cprman->regs + CM_PERIICTL, CM_GATE_BIT,
 				  0, &cprman->regs_lock);
 
+	clks[BCM2835_CLOCK_PWM] =
+		bcm2835_register_clock(cprman, &bcm2835_clock_pwm_data);
+
 	return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get,
 				   &cprman->onecell);
 }
diff --git a/drivers/clk/bcm/clk-bcm63xx.c b/drivers/clk/bcm/clk-bcm63xx.c
new file mode 100644
index 000000000000..fbc17ae5ff2b
--- /dev/null
+++ b/drivers/clk/bcm/clk-bcm63xx.c
@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/init.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include "clk-iproc.h"
+
+static void __init bcm63138_armpll_init(struct device_node *node)
+{
+	iproc_armpll_setup(node);
+}
+CLK_OF_DECLARE(bcm63138_armpll, "brcm,bcm63138-armpll", bcm63138_armpll_init);
diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c
new file mode 100644
index 000000000000..7379de8dc894
--- /dev/null
+++ b/drivers/clk/clk-cs2000-cp.c
@@ -0,0 +1,510 @@
+/*
+ * CS2000  --  CIRRUS LOGIC Fractional-N Clock Synthesizer & Clock Multiplier
+ *
+ * Copyright (C) 2015 Renesas Electronics Corporation
+ * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/clk.h>
+#include <linux/i2c.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+
+#define CH_MAX 4
+#define RATIO_REG_SIZE 4
+
+#define DEVICE_ID	0x1
+#define DEVICE_CTRL	0x2
+#define DEVICE_CFG1	0x3
+#define DEVICE_CFG2	0x4
+#define GLOBAL_CFG	0x5
+#define Ratio_Add(x, nth)	(6 + (x * 4) + (nth))
+#define Ratio_Val(x, nth)	((x >> (24 - (8 * nth))) & 0xFF)
+#define Val_Ratio(x, nth)	((x & 0xFF) << (24 - (8 * nth)))
+#define FUNC_CFG1	0x16
+#define FUNC_CFG2	0x17
+
+/* DEVICE_ID */
+#define REVISION_MASK	(0x7)
+#define REVISION_B2_B3	(0x4)
+#define REVISION_C1	(0x6)
+
+/* DEVICE_CTRL */
+#define PLL_UNLOCK	(1 << 7)
+
+/* DEVICE_CFG1 */
+#define RSEL(x)		(((x) & 0x3) << 3)
+#define RSEL_MASK	RSEL(0x3)
+#define ENDEV1		(0x1)
+
+/* GLOBAL_CFG */
+#define ENDEV2		(0x1)
+
+#define CH_SIZE_ERR(ch)		((ch < 0) || (ch >= CH_MAX))
+#define hw_to_priv(_hw)		container_of(_hw, struct cs2000_priv, hw)
+#define priv_to_client(priv)	(priv->client)
+#define priv_to_dev(priv)	(&(priv_to_client(priv)->dev))
+
+#define CLK_IN	0
+#define REF_CLK	1
+#define CLK_MAX 2
+
+struct cs2000_priv {
+	struct clk_hw hw;
+	struct i2c_client *client;
+	struct clk *clk_in;
+	struct clk *ref_clk;
+	struct clk *clk_out;
+};
+
+static const struct of_device_id cs2000_of_match[] = {
+	{ .compatible = "cirrus,cs2000-cp", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, cs2000_of_match);
+
+static const struct i2c_device_id cs2000_id[] = {
+	{ "cs2000-cp", },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, cs2000_id);
+
+#define cs2000_read(priv, addr) \
+	i2c_smbus_read_byte_data(priv_to_client(priv), addr)
+#define cs2000_write(priv, addr, val) \
+	i2c_smbus_write_byte_data(priv_to_client(priv), addr, val)
+
+static int cs2000_bset(struct cs2000_priv *priv, u8 addr, u8 mask, u8 val)
+{
+	s32 data;
+
+	data = cs2000_read(priv, addr);
+	if (data < 0)
+		return data;
+
+	data &= ~mask;
+	data |= (val & mask);
+
+	return cs2000_write(priv, addr, data);
+}
+
+static int cs2000_enable_dev_config(struct cs2000_priv *priv, bool enable)
+{
+	int ret;
+
+	ret = cs2000_bset(priv, DEVICE_CFG1, ENDEV1,
+			  enable ? ENDEV1 : 0);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_bset(priv, GLOBAL_CFG,  ENDEV2,
+			  enable ? ENDEV2 : 0);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int cs2000_clk_in_bound_rate(struct cs2000_priv *priv,
+				    u32 rate_in)
+{
+	u32 val;
+
+	if (rate_in >= 32000000 && rate_in < 56000000)
+		val = 0x0;
+	else if (rate_in >= 16000000 && rate_in < 28000000)
+		val = 0x1;
+	else if (rate_in >= 8000000 && rate_in < 14000000)
+		val = 0x2;
+	else
+		return -EINVAL;
+
+	return cs2000_bset(priv, FUNC_CFG1, 0x3 << 3, val << 3);
+}
+
+static int cs2000_wait_pll_lock(struct cs2000_priv *priv)
+{
+	struct device *dev = priv_to_dev(priv);
+	s32 val;
+	unsigned int i;
+
+	for (i = 0; i < 256; i++) {
+		val = cs2000_read(priv, DEVICE_CTRL);
+		if (val < 0)
+			return val;
+		if (!(val & PLL_UNLOCK))
+			return 0;
+		udelay(1);
+	}
+
+	dev_err(dev, "pll lock failed\n");
+
+	return -ETIMEDOUT;
+}
+
+static int cs2000_clk_out_enable(struct cs2000_priv *priv, bool enable)
+{
+	/* enable both AUX_OUT, CLK_OUT */
+	return cs2000_write(priv, DEVICE_CTRL, enable ? 0 : 0x3);
+}
+
+static u32 cs2000_rate_to_ratio(u32 rate_in, u32 rate_out)
+{
+	u64 ratio;
+
+	/*
+	 * ratio = rate_out / rate_in * 2^20
+	 *
+	 * To avoid over flow, rate_out is u64.
+	 * The result should be u32.
+	 */
+	ratio = (u64)rate_out << 20;
+	do_div(ratio, rate_in);
+
+	return ratio;
+}
+
+static unsigned long cs2000_ratio_to_rate(u32 ratio, u32 rate_in)
+{
+	u64 rate_out;
+
+	/*
+	 * ratio = rate_out / rate_in * 2^20
+	 *
+	 * To avoid over flow, rate_out is u64.
+	 * The result should be u32 or unsigned long.
+	 */
+
+	rate_out = (u64)ratio * rate_in;
+	return rate_out >> 20;
+}
+
+static int cs2000_ratio_set(struct cs2000_priv *priv,
+			    int ch, u32 rate_in, u32 rate_out)
+{
+	u32 val;
+	unsigned int i;
+	int ret;
+
+	if (CH_SIZE_ERR(ch))
+		return -EINVAL;
+
+	val = cs2000_rate_to_ratio(rate_in, rate_out);
+	for (i = 0; i < RATIO_REG_SIZE; i++) {
+		ret = cs2000_write(priv,
+				   Ratio_Add(ch, i),
+				   Ratio_Val(val, i));
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static u32 cs2000_ratio_get(struct cs2000_priv *priv, int ch)
+{
+	s32 tmp;
+	u32 val;
+	unsigned int i;
+
+	val = 0;
+	for (i = 0; i < RATIO_REG_SIZE; i++) {
+		tmp = cs2000_read(priv, Ratio_Add(ch, i));
+		if (tmp < 0)
+			return 0;
+
+		val |= Val_Ratio(tmp, i);
+	}
+
+	return val;
+}
+
+static int cs2000_ratio_select(struct cs2000_priv *priv, int ch)
+{
+	int ret;
+
+	if (CH_SIZE_ERR(ch))
+		return -EINVAL;
+
+	/*
+	 * FIXME
+	 *
+	 * this driver supports static ratio mode only at this point.
+	 */
+	ret = cs2000_bset(priv, DEVICE_CFG1, RSEL_MASK, RSEL(ch));
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_write(priv, DEVICE_CFG2, 0x0);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static unsigned long cs2000_recalc_rate(struct clk_hw *hw,
+					unsigned long parent_rate)
+{
+	struct cs2000_priv *priv = hw_to_priv(hw);
+	int ch = 0; /* it uses ch0 only at this point */
+	u32 ratio;
+
+	ratio = cs2000_ratio_get(priv, ch);
+
+	return cs2000_ratio_to_rate(ratio, parent_rate);
+}
+
+static long cs2000_round_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long *parent_rate)
+{
+	u32 ratio;
+
+	ratio = cs2000_rate_to_ratio(*parent_rate, rate);
+
+	return cs2000_ratio_to_rate(ratio, *parent_rate);
+}
+
+static int __cs2000_set_rate(struct cs2000_priv *priv, int ch,
+			     unsigned long rate, unsigned long parent_rate)
+
+{
+	int ret;
+
+	ret = cs2000_clk_in_bound_rate(priv, parent_rate);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_ratio_set(priv, ch, parent_rate, rate);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_ratio_select(priv, ch);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int cs2000_set_rate(struct clk_hw *hw,
+			   unsigned long rate, unsigned long parent_rate)
+{
+	struct cs2000_priv *priv = hw_to_priv(hw);
+	int ch = 0; /* it uses ch0 only at this point */
+
+	return __cs2000_set_rate(priv, ch, rate, parent_rate);
+}
+
+static int cs2000_enable(struct clk_hw *hw)
+{
+	struct cs2000_priv *priv = hw_to_priv(hw);
+	int ret;
+
+	ret = cs2000_enable_dev_config(priv, true);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_clk_out_enable(priv, true);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_wait_pll_lock(priv);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static void cs2000_disable(struct clk_hw *hw)
+{
+	struct cs2000_priv *priv = hw_to_priv(hw);
+
+	cs2000_enable_dev_config(priv, false);
+
+	cs2000_clk_out_enable(priv, false);
+}
+
+static u8 cs2000_get_parent(struct clk_hw *hw)
+{
+	/* always return REF_CLK */
+	return REF_CLK;
+}
+
+static const struct clk_ops cs2000_ops = {
+	.get_parent	= cs2000_get_parent,
+	.recalc_rate	= cs2000_recalc_rate,
+	.round_rate	= cs2000_round_rate,
+	.set_rate	= cs2000_set_rate,
+	.prepare	= cs2000_enable,
+	.unprepare	= cs2000_disable,
+};
+
+static int cs2000_clk_get(struct cs2000_priv *priv)
+{
+	struct i2c_client *client = priv_to_client(priv);
+	struct device *dev = &client->dev;
+	struct clk *clk_in, *ref_clk;
+
+	clk_in = devm_clk_get(dev, "clk_in");
+	/* not yet provided */
+	if (IS_ERR(clk_in))
+		return -EPROBE_DEFER;
+
+	ref_clk = devm_clk_get(dev, "ref_clk");
+	/* not yet provided */
+	if (IS_ERR(ref_clk))
+		return -EPROBE_DEFER;
+
+	priv->clk_in	= clk_in;
+	priv->ref_clk	= ref_clk;
+
+	return 0;
+}
+
+static int cs2000_clk_register(struct cs2000_priv *priv)
+{
+	struct device *dev = priv_to_dev(priv);
+	struct device_node *np = dev->of_node;
+	struct clk_init_data init;
+	const char *name = np->name;
+	struct clk *clk;
+	static const char *parent_names[CLK_MAX];
+	int ch = 0; /* it uses ch0 only at this point */
+	int rate;
+	int ret;
+
+	of_property_read_string(np, "clock-output-names", &name);
+
+	/*
+	 * set default rate as 1/1.
+	 * otherwise .set_rate which setup ratio
+	 * is never called if user requests 1/1 rate
+	 */
+	rate = clk_get_rate(priv->ref_clk);
+	ret = __cs2000_set_rate(priv, ch, rate, rate);
+	if (ret < 0)
+		return ret;
+
+	parent_names[CLK_IN]	= __clk_get_name(priv->clk_in);
+	parent_names[REF_CLK]	= __clk_get_name(priv->ref_clk);
+
+	init.name		= name;
+	init.ops		= &cs2000_ops;
+	init.flags		= CLK_SET_RATE_GATE;
+	init.parent_names	= parent_names;
+	init.num_parents	= ARRAY_SIZE(parent_names);
+
+	priv->hw.init = &init;
+
+	clk = clk_register(dev, &priv->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	ret = of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	if (ret < 0) {
+		clk_unregister(clk);
+		return ret;
+	}
+
+	priv->clk_out = clk;
+
+	return 0;
+}
+
+static int cs2000_version_print(struct cs2000_priv *priv)
+{
+	struct i2c_client *client = priv_to_client(priv);
+	struct device *dev = &client->dev;
+	s32 val;
+	const char *revision;
+
+	val = cs2000_read(priv, DEVICE_ID);
+	if (val < 0)
+		return val;
+
+	/* CS2000 should be 0x0 */
+	if (val >> 3)
+		return -EIO;
+
+	switch (val & REVISION_MASK) {
+	case REVISION_B2_B3:
+		revision = "B2 / B3";
+		break;
+	case REVISION_C1:
+		revision = "C1";
+		break;
+	default:
+		return -EIO;
+	}
+
+	dev_info(dev, "revision - %s\n", revision);
+
+	return 0;
+}
+
+static int cs2000_remove(struct i2c_client *client)
+{
+	struct cs2000_priv *priv = i2c_get_clientdata(client);
+	struct device *dev = &client->dev;
+	struct device_node *np = dev->of_node;
+
+	of_clk_del_provider(np);
+
+	clk_unregister(priv->clk_out);
+
+	return 0;
+}
+
+static int cs2000_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct cs2000_priv *priv;
+	struct device *dev = &client->dev;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->client = client;
+	i2c_set_clientdata(client, priv);
+
+	ret = cs2000_clk_get(priv);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_clk_register(priv);
+	if (ret < 0)
+		return ret;
+
+	ret = cs2000_version_print(priv);
+	if (ret < 0)
+		goto probe_err;
+
+	return 0;
+
+probe_err:
+	cs2000_remove(client);
+
+	return ret;
+}
+
+static struct i2c_driver cs2000_driver = {
+	.driver = {
+		.name = "cs2000-cp",
+		.of_match_table = cs2000_of_match,
+	},
+	.probe		= cs2000_probe,
+	.remove		= cs2000_remove,
+	.id_table	= cs2000_id,
+};
+
+module_i2c_driver(cs2000_driver);
+
+MODULE_DESCRIPTION("CS2000-CP driver");
+MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c
index 3ace102a2a0a..ded3ff4b91b9 100644
--- a/drivers/clk/clk-divider.c
+++ b/drivers/clk/clk-divider.c
@@ -32,13 +32,14 @@
 
 #define div_mask(width)	((1 << (width)) - 1)
 
-static unsigned int _get_table_maxdiv(const struct clk_div_table *table)
+static unsigned int _get_table_maxdiv(const struct clk_div_table *table,
+				      u8 width)
 {
-	unsigned int maxdiv = 0;
+	unsigned int maxdiv = 0, mask = div_mask(width);
 	const struct clk_div_table *clkt;
 
 	for (clkt = table; clkt->div; clkt++)
-		if (clkt->div > maxdiv)
+		if (clkt->div > maxdiv && clkt->val <= mask)
 			maxdiv = clkt->div;
 	return maxdiv;
 }
@@ -62,7 +63,7 @@ static unsigned int _get_maxdiv(const struct clk_div_table *table, u8 width,
 	if (flags & CLK_DIVIDER_POWER_OF_TWO)
 		return 1 << div_mask(width);
 	if (table)
-		return _get_table_maxdiv(table);
+		return _get_table_maxdiv(table, width);
 	return div_mask(width) + 1;
 }
 
diff --git a/drivers/clk/clk-gpio.c b/drivers/clk/clk-gpio.c
index 335322dc403f..19fed65587e8 100644
--- a/drivers/clk/clk-gpio.c
+++ b/drivers/clk/clk-gpio.c
@@ -264,8 +264,8 @@ static struct clk *of_clk_gpio_gate_delayed_register_get(const char *name,
 		const char * const *parent_names, u8 num_parents,
 		unsigned gpio, bool active_low)
 {
-	return clk_register_gpio_gate(NULL, name, parent_names[0],
-			gpio, active_low, 0);
+	return clk_register_gpio_gate(NULL, name, parent_names ?
+			parent_names[0] : NULL, gpio, active_low, 0);
 }
 
 static struct clk *of_clk_gpio_mux_delayed_register_get(const char *name,
@@ -287,18 +287,26 @@ static void __init of_gpio_clk_setup(struct device_node *node,
 	const char **parent_names;
 	int i, num_parents;
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
-	if (!data)
-		return;
-
 	num_parents = of_clk_get_parent_count(node);
+	if (num_parents < 0)
+		return;
 
-	parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL);
-	if (!parent_names)
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
 		return;
 
-	for (i = 0; i < num_parents; i++)
-		parent_names[i] = of_clk_get_parent_name(node, i);
+	if (num_parents) {
+		parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL);
+		if (!parent_names) {
+			kfree(data);
+			return;
+		}
+
+		for (i = 0; i < num_parents; i++)
+			parent_names[i] = of_clk_get_parent_name(node, i);
+	} else {
+		parent_names = NULL;
+	}
 
 	data->num_parents = num_parents;
 	data->parent_names = parent_names;
diff --git a/drivers/clk/clk-mux.c b/drivers/clk/clk-mux.c
index 7129c86a79db..5ed03c8a8df9 100644
--- a/drivers/clk/clk-mux.c
+++ b/drivers/clk/clk-mux.c
@@ -71,10 +71,9 @@ static int clk_mux_set_parent(struct clk_hw *hw, u8 index)
 	u32 val;
 	unsigned long flags = 0;
 
-	if (mux->table)
+	if (mux->table) {
 		index = mux->table[index];
-
-	else {
+	} else {
 		if (mux->flags & CLK_MUX_INDEX_BIT)
 			index = 1 << index;
 
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index e346b223199d..850316ac8831 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -1091,6 +1091,13 @@ static int si5351_clkout_set_rate(struct clk_hw *hw, unsigned long rate,
 	si5351_set_bits(hwdata->drvdata, SI5351_CLK0_CTRL + hwdata->num,
 			SI5351_CLK_POWERDOWN, 0);
 
+	/*
+	 * Do a pll soft reset on both plls, needed in some cases to get
+	 * all outputs running.
+	 */
+	si5351_reg_write(hwdata->drvdata, SI5351_PLL_RESET,
+			 SI5351_PLL_RESET_A | SI5351_PLL_RESET_B);
+
 	dev_dbg(&hwdata->drvdata->client->dev,
 		"%s - %s: rdiv = %u, parent_rate = %lu, rate = %lu\n",
 		__func__, clk_hw_get_name(hw), (1 << rdiv),
diff --git a/drivers/clk/clk-tango4.c b/drivers/clk/clk-tango4.c
new file mode 100644
index 000000000000..004ab7dfcfe3
--- /dev/null
+++ b/drivers/clk/clk-tango4.c
@@ -0,0 +1,61 @@
+#include <linux/kernel.h>
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/init.h>
+#include <linux/io.h>
+
+static struct clk *out[2];
+static struct clk_onecell_data clk_data = { out, 2 };
+
+#define SYSCLK_CTRL	0x20
+#define CPUCLK_CTRL	0x24
+#define LEGACY_DIV	0x3c
+
+#define PLL_N(val)	(((val) >>  0) & 0x7f)
+#define PLL_K(val)	(((val) >> 13) & 0x7)
+#define PLL_M(val)	(((val) >> 16) & 0x7)
+#define DIV_INDEX(val)	(((val) >>  8) & 0xf)
+
+static void __init make_pll(int idx, const char *parent, void __iomem *base)
+{
+	char name[8];
+	u32 val, mul, div;
+
+	sprintf(name, "pll%d", idx);
+	val = readl_relaxed(base + idx*8);
+	mul =  PLL_N(val) + 1;
+	div = (PLL_M(val) + 1) << PLL_K(val);
+	clk_register_fixed_factor(NULL, name, parent, 0, mul, div);
+}
+
+static int __init get_div(void __iomem *base)
+{
+	u8 sysclk_tab[16] = { 2, 4, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4 };
+	int idx = DIV_INDEX(readl_relaxed(base + LEGACY_DIV));
+
+	return sysclk_tab[idx];
+}
+
+static void __init tango4_clkgen_setup(struct device_node *np)
+{
+	int div, ret;
+	void __iomem *base = of_iomap(np, 0);
+	const char *parent = of_clk_get_parent_name(np, 0);
+
+	if (!base)
+		panic("%s: invalid address\n", np->full_name);
+
+	make_pll(0, parent, base);
+	make_pll(1, parent, base);
+
+	out[0] = clk_register_divider(NULL, "cpuclk", "pll0", 0,
+			base + CPUCLK_CTRL, 8, 8, CLK_DIVIDER_ONE_BASED, NULL);
+
+	div = readl_relaxed(base + SYSCLK_CTRL) & BIT(23) ? get_div(base) : 4;
+	out[1] = clk_register_fixed_factor(NULL, "sysclk", "pll1", 0, 1, div);
+
+	ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	if (IS_ERR(out[0]) || IS_ERR(out[1]) || ret < 0)
+		panic("%s: clk registration failed\n", np->full_name);
+}
+CLK_OF_DECLARE(tango4_clkgen, "sigma,tango4-clkgen", tango4_clkgen_setup);
diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c
index 27c0da29eca3..10224b01b97c 100644
--- a/drivers/clk/clk-xgene.c
+++ b/drivers/clk/clk-xgene.c
@@ -351,7 +351,8 @@ static int xgene_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 		/* Set new divider */
 		data = xgene_clk_read(pclk->param.divider_reg +
 				pclk->param.reg_divider_offset);
-		data &= ~((1 << pclk->param.reg_divider_width) - 1);
+		data &= ~((1 << pclk->param.reg_divider_width) - 1)
+				<< pclk->param.reg_divider_shift;
 		data |= divider;
 		xgene_clk_write(data, pclk->param.divider_reg +
 					pclk->param.reg_divider_offset);
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index f13c3f4228d4..b4db67a446c8 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -1443,6 +1443,15 @@ static void clk_change_rate(struct clk_core *core)
 	else if (core->parent)
 		best_parent_rate = core->parent->rate;
 
+	if (core->flags & CLK_SET_RATE_UNGATE) {
+		unsigned long flags;
+
+		clk_core_prepare(core);
+		flags = clk_enable_lock();
+		clk_core_enable(core);
+		clk_enable_unlock(flags);
+	}
+
 	if (core->new_parent && core->new_parent != core->parent) {
 		old_parent = __clk_set_parent_before(core, core->new_parent);
 		trace_clk_set_parent(core, core->new_parent);
@@ -1469,6 +1478,15 @@ static void clk_change_rate(struct clk_core *core)
 
 	core->rate = clk_recalc(core, best_parent_rate);
 
+	if (core->flags & CLK_SET_RATE_UNGATE) {
+		unsigned long flags;
+
+		flags = clk_enable_lock();
+		clk_core_disable(core);
+		clk_enable_unlock(flags);
+		clk_core_unprepare(core);
+	}
+
 	if (core->notifier_count && old_rate != core->rate)
 		__clk_notify(core, POST_RATE_CHANGE, old_rate, core->rate);
 
@@ -1944,7 +1962,7 @@ bool clk_is_match(const struct clk *p, const struct clk *q)
 	if (p == q)
 		return true;
 
-	/* true if clk->core pointers match. Avoid derefing garbage */
+	/* true if clk->core pointers match. Avoid dereferencing garbage */
 	if (!IS_ERR_OR_NULL(p) && !IS_ERR_OR_NULL(q))
 		if (p->core == q->core)
 			return true;
@@ -2482,7 +2500,7 @@ struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id,
 	struct clk *clk;
 
 	/* This is to allow this function to be chained to others */
-	if (!hw || IS_ERR(hw))
+	if (IS_ERR_OR_NULL(hw))
 		return (struct clk *) hw;
 
 	clk = kzalloc(sizeof(*clk), GFP_KERNEL);
@@ -2806,10 +2824,9 @@ void __clk_put(struct clk *clk)
  * re-enter into the clk framework by calling any top-level clk APIs;
  * this will cause a nested prepare_lock mutex.
  *
- * In all notification cases cases (pre, post and abort rate change) the
- * original clock rate is passed to the callback via struct
- * clk_notifier_data.old_rate and the new frequency is passed via struct
- * clk_notifier_data.new_rate.
+ * In all notification cases (pre, post and abort rate change) the original
+ * clock rate is passed to the callback via struct clk_notifier_data.old_rate
+ * and the new frequency is passed via struct clk_notifier_data.new_rate.
  *
  * clk_notifier_register() must be called from non-atomic context.
  * Returns -EINVAL if called with null arguments, -ENOMEM upon
@@ -3062,9 +3079,6 @@ const char *of_clk_get_parent_name(struct device_node *np, int index)
 	int count;
 	struct clk *clk;
 
-	if (index < 0)
-		return NULL;
-
 	rc = of_parse_phandle_with_args(np, "clocks", "#clock-cells", index,
 					&clkspec);
 	if (rc)
@@ -3083,6 +3097,9 @@ const char *of_clk_get_parent_name(struct device_node *np, int index)
 		}
 		count++;
 	}
+	/* We went off the end of 'clock-indices' without finding it */
+	if (prop && !vp)
+		return NULL;
 
 	if (of_property_read_string_index(clkspec.np, "clock-output-names",
 					  index,
diff --git a/drivers/clk/imx/clk-imx25.c b/drivers/clk/imx/clk-imx25.c
index c4c141cab444..23686f756b5e 100644
--- a/drivers/clk/imx/clk-imx25.c
+++ b/drivers/clk/imx/clk-imx25.c
@@ -96,13 +96,11 @@ static struct clk ** const uart_clks[] __initconst = {
 	NULL
 };
 
-static int __init __mx25_clocks_init(unsigned long osc_rate,
-				     void __iomem *ccm_base)
+static int __init __mx25_clocks_init(void __iomem *ccm_base)
 {
 	BUG_ON(!ccm_base);
 
 	clk[dummy] = imx_clk_fixed("dummy", 0);
-	clk[osc] = imx_clk_fixed("osc", osc_rate);
 	clk[mpll] = imx_clk_pllv1(IMX_PLLV1_IMX25, "mpll", "osc", ccm(CCM_MPCTL));
 	clk[upll] = imx_clk_pllv1(IMX_PLLV1_IMX25, "upll", "osc", ccm(CCM_UPCTL));
 	clk[mpll_cpu_3_4] = imx_clk_fixed_factor("mpll_cpu_3_4", "mpll", 3, 4);
@@ -250,22 +248,10 @@ static int __init __mx25_clocks_init(unsigned long osc_rate,
 
 static void __init mx25_clocks_init_dt(struct device_node *np)
 {
-	struct device_node *refnp;
-	unsigned long osc_rate = 24000000;
 	void __iomem *ccm;
 
-	/* retrieve the freqency of fixed clocks from device tree */
-	for_each_compatible_node(refnp, NULL, "fixed-clock") {
-		u32 rate;
-		if (of_property_read_u32(refnp, "clock-frequency", &rate))
-			continue;
-
-		if (of_device_is_compatible(refnp, "fsl,imx-osc"))
-			osc_rate = rate;
-	}
-
 	ccm = of_iomap(np, 0);
-	__mx25_clocks_init(osc_rate, ccm);
+	__mx25_clocks_init(ccm);
 
 	clk_data.clks = clk;
 	clk_data.clk_num = ARRAY_SIZE(clk);
diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx51-imx53.c
index c6770348d2ab..29d4c44ef356 100644
--- a/drivers/clk/imx/clk-imx51-imx53.c
+++ b/drivers/clk/imx/clk-imx51-imx53.c
@@ -519,10 +519,10 @@ static void __init mx53_clocks_init(struct device_node *np)
 						mx53_ldb_di0_sel, ARRAY_SIZE(mx53_ldb_di0_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_LDB_DI0_GATE]	= imx_clk_gate2("ldb_di0_gate", "ldb_di0_div", MXC_CCM_CCGR6, 28);
 	clk[IMX5_CLK_LDB_DI1_GATE]	= imx_clk_gate2("ldb_di1_gate", "ldb_di1_div", MXC_CCM_CCGR6, 30);
-	clk[IMX5_CLK_IPU_DI0_SEL]	= imx_clk_mux("ipu_di0_sel", MXC_CCM_CSCMR2, 26, 3,
-						mx53_ipu_di0_sel, ARRAY_SIZE(mx53_ipu_di0_sel));
-	clk[IMX5_CLK_IPU_DI1_SEL]	= imx_clk_mux("ipu_di1_sel", MXC_CCM_CSCMR2, 29, 3,
-						mx53_ipu_di1_sel, ARRAY_SIZE(mx53_ipu_di1_sel));
+	clk[IMX5_CLK_IPU_DI0_SEL]	= imx_clk_mux_flags("ipu_di0_sel", MXC_CCM_CSCMR2, 26, 3,
+						mx53_ipu_di0_sel, ARRAY_SIZE(mx53_ipu_di0_sel), CLK_SET_RATE_PARENT);
+	clk[IMX5_CLK_IPU_DI1_SEL]	= imx_clk_mux_flags("ipu_di1_sel", MXC_CCM_CSCMR2, 29, 3,
+						mx53_ipu_di1_sel, ARRAY_SIZE(mx53_ipu_di1_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_TVE_EXT_SEL]	= imx_clk_mux_flags("tve_ext_sel", MXC_CCM_CSCMR1, 6, 1,
 						mx53_tve_ext_sel, ARRAY_SIZE(mx53_tve_ext_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_TVE_GATE]		= imx_clk_gate2("tve_gate", "tve_pred", MXC_CCM_CCGR2, 30);
diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c
index c1935081d34a..f0efc6feeec2 100644
--- a/drivers/clk/imx/clk-imx6q.c
+++ b/drivers/clk/imx/clk-imx6q.c
@@ -70,7 +70,8 @@ static const char *cko_sels[] = { "cko1", "cko2", };
 static const char *lvds_sels[] = {
 	"dummy", "dummy", "dummy", "dummy", "dummy", "dummy",
 	"pll4_audio", "pll5_video", "pll8_mlb", "enet_ref",
-	"pcie_ref_125m", "sata_ref_100m",
+	"pcie_ref_125m", "sata_ref_100m",  "usbphy1", "usbphy2",
+	"dummy", "dummy", "dummy", "dummy", "osc",
 };
 static const char *pll_bypass_src_sels[] = { "osc", "lvds1_in", "lvds2_in", "dummy", };
 static const char *pll1_bypass_sels[] = { "pll1", "pll1_bypass_src", };
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index 01718d05e952..08692d74b884 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -399,9 +399,7 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	/* mask handshake of mmdc */
 	writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR);
 
-	for (i = 0; i < ARRAY_SIZE(clks); i++)
-		if (IS_ERR(clks[i]))
-			pr_err("i.MX6UL clk %d: register failed with %ld\n", i, PTR_ERR(clks[i]));
+	imx_check_clocks(clks, ARRAY_SIZE(clks));
 
 	clk_data.clks = clks;
 	clk_data.clk_num = ARRAY_SIZE(clks);
diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
index 448ef321948b..fbb6a8c8653d 100644
--- a/drivers/clk/imx/clk-imx7d.c
+++ b/drivers/clk/imx/clk-imx7d.c
@@ -833,10 +833,13 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 
 	clks[IMX7D_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8);
 
-	for (i = 0; i < ARRAY_SIZE(clks); i++)
-		if (IS_ERR(clks[i]))
-			pr_err("i.MX7D clk %d: register failed with %ld\n",
-					i, PTR_ERR(clks[i]));
+	clks[IMX7D_CLK_ARM] = imx_clk_cpu("arm", "arm_a7_root_clk",
+					 clks[IMX7D_ARM_A7_ROOT_CLK],
+					 clks[IMX7D_ARM_A7_ROOT_SRC],
+					 clks[IMX7D_PLL_ARM_MAIN_CLK],
+					 clks[IMX7D_PLL_SYS_MAIN_CLK]);
+
+	imx_check_clocks(clks, ARRAY_SIZE(clks));
 
 	clk_data.clks = clks;
 	clk_data.clk_num = ARRAY_SIZE(clks);
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index 6addf8f58b97..c05c43d56a94 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -97,6 +97,16 @@ static void clk_pllv3_unprepare(struct clk_hw *hw)
 	writel_relaxed(val, pll->base);
 }
 
+static int clk_pllv3_is_prepared(struct clk_hw *hw)
+{
+	struct clk_pllv3 *pll = to_clk_pllv3(hw);
+
+	if (readl_relaxed(pll->base) & BM_PLL_LOCK)
+		return 1;
+
+	return 0;
+}
+
 static unsigned long clk_pllv3_recalc_rate(struct clk_hw *hw,
 					   unsigned long parent_rate)
 {
@@ -139,6 +149,7 @@ static int clk_pllv3_set_rate(struct clk_hw *hw, unsigned long rate,
 static const struct clk_ops clk_pllv3_ops = {
 	.prepare	= clk_pllv3_prepare,
 	.unprepare	= clk_pllv3_unprepare,
+	.is_prepared	= clk_pllv3_is_prepared,
 	.recalc_rate	= clk_pllv3_recalc_rate,
 	.round_rate	= clk_pllv3_round_rate,
 	.set_rate	= clk_pllv3_set_rate,
@@ -193,6 +204,7 @@ static int clk_pllv3_sys_set_rate(struct clk_hw *hw, unsigned long rate,
 static const struct clk_ops clk_pllv3_sys_ops = {
 	.prepare	= clk_pllv3_prepare,
 	.unprepare	= clk_pllv3_unprepare,
+	.is_prepared	= clk_pllv3_is_prepared,
 	.recalc_rate	= clk_pllv3_sys_recalc_rate,
 	.round_rate	= clk_pllv3_sys_round_rate,
 	.set_rate	= clk_pllv3_sys_set_rate,
@@ -265,6 +277,7 @@ static int clk_pllv3_av_set_rate(struct clk_hw *hw, unsigned long rate,
 static const struct clk_ops clk_pllv3_av_ops = {
 	.prepare	= clk_pllv3_prepare,
 	.unprepare	= clk_pllv3_unprepare,
+	.is_prepared	= clk_pllv3_is_prepared,
 	.recalc_rate	= clk_pllv3_av_recalc_rate,
 	.round_rate	= clk_pllv3_av_round_rate,
 	.set_rate	= clk_pllv3_av_set_rate,
@@ -279,6 +292,7 @@ static unsigned long clk_pllv3_enet_recalc_rate(struct clk_hw *hw,
 static const struct clk_ops clk_pllv3_enet_ops = {
 	.prepare	= clk_pllv3_prepare,
 	.unprepare	= clk_pllv3_unprepare,
+	.is_prepared	= clk_pllv3_is_prepared,
 	.recalc_rate	= clk_pllv3_enet_recalc_rate,
 };
 
diff --git a/drivers/clk/mmp/clk-mmp2.c b/drivers/clk/mmp/clk-mmp2.c
index 71fd29348f28..38931dbd1eff 100644
--- a/drivers/clk/mmp/clk-mmp2.c
+++ b/drivers/clk/mmp/clk-mmp2.c
@@ -17,8 +17,6 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 
-#include <mach/addr-map.h>
-
 #include "clk.h"
 
 #define APBC_RTC	0x0
@@ -74,7 +72,8 @@ static const char *sdh_parent[] = {"pll1_4", "pll2", "usb_pll", "pll1"};
 static const char *disp_parent[] = {"pll1", "pll1_16", "pll2", "vctcxo"};
 static const char *ccic_parent[] = {"pll1_2", "pll1_16", "vctcxo"};
 
-void __init mmp2_clk_init(void)
+void __init mmp2_clk_init(phys_addr_t mpmu_phys, phys_addr_t apmu_phys,
+			  phys_addr_t apbc_phys)
 {
 	struct clk *clk;
 	struct clk *vctcxo;
@@ -82,19 +81,19 @@ void __init mmp2_clk_init(void)
 	void __iomem *apmu_base;
 	void __iomem *apbc_base;
 
-	mpmu_base = ioremap(APB_PHYS_BASE + 0x50000, SZ_4K);
+	mpmu_base = ioremap(mpmu_phys, SZ_4K);
 	if (mpmu_base == NULL) {
 		pr_err("error to ioremap MPMU base\n");
 		return;
 	}
 
-	apmu_base = ioremap(AXI_PHYS_BASE + 0x82800, SZ_4K);
+	apmu_base = ioremap(apmu_phys, SZ_4K);
 	if (apmu_base == NULL) {
 		pr_err("error to ioremap APMU base\n");
 		return;
 	}
 
-	apbc_base = ioremap(APB_PHYS_BASE + 0x15000, SZ_4K);
+	apbc_base = ioremap(apbc_phys, SZ_4K);
 	if (apbc_base == NULL) {
 		pr_err("error to ioremap APBC base\n");
 		return;
diff --git a/drivers/clk/mmp/clk-pxa168.c b/drivers/clk/mmp/clk-pxa168.c
index 75244915df05..0dd83fb950c9 100644
--- a/drivers/clk/mmp/clk-pxa168.c
+++ b/drivers/clk/mmp/clk-pxa168.c
@@ -17,8 +17,6 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 
-#include <mach/addr-map.h>
-
 #include "clk.h"
 
 #define APBC_RTC	0x28
@@ -67,7 +65,8 @@ static const char *disp_parent[] = {"pll1_2", "pll1_12"};
 static const char *ccic_parent[] = {"pll1_2", "pll1_12"};
 static const char *ccic_phy_parent[] = {"pll1_6", "pll1_12"};
 
-void __init pxa168_clk_init(void)
+void __init pxa168_clk_init(phys_addr_t mpmu_phys, phys_addr_t apmu_phys,
+			    phys_addr_t apbc_phys)
 {
 	struct clk *clk;
 	struct clk *uart_pll;
@@ -75,19 +74,19 @@ void __init pxa168_clk_init(void)
 	void __iomem *apmu_base;
 	void __iomem *apbc_base;
 
-	mpmu_base = ioremap(APB_PHYS_BASE + 0x50000, SZ_4K);
+	mpmu_base = ioremap(mpmu_phys, SZ_4K);
 	if (mpmu_base == NULL) {
 		pr_err("error to ioremap MPMU base\n");
 		return;
 	}
 
-	apmu_base = ioremap(AXI_PHYS_BASE + 0x82800, SZ_4K);
+	apmu_base = ioremap(apmu_phys, SZ_4K);
 	if (apmu_base == NULL) {
 		pr_err("error to ioremap APMU base\n");
 		return;
 	}
 
-	apbc_base = ioremap(APB_PHYS_BASE + 0x15000, SZ_4K);
+	apbc_base = ioremap(apbc_phys, SZ_4K);
 	if (apbc_base == NULL) {
 		pr_err("error to ioremap APBC base\n");
 		return;
diff --git a/drivers/clk/mmp/clk-pxa910.c b/drivers/clk/mmp/clk-pxa910.c
index 37ba04ba1368..e1d2ce22cdf1 100644
--- a/drivers/clk/mmp/clk-pxa910.c
+++ b/drivers/clk/mmp/clk-pxa910.c
@@ -17,8 +17,6 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 
-#include <mach/addr-map.h>
-
 #include "clk.h"
 
 #define APBC_RTC	0x28
@@ -65,7 +63,8 @@ static const char *disp_parent[] = {"pll1_2", "pll1_12"};
 static const char *ccic_parent[] = {"pll1_2", "pll1_12"};
 static const char *ccic_phy_parent[] = {"pll1_6", "pll1_12"};
 
-void __init pxa910_clk_init(void)
+void __init pxa910_clk_init(phys_addr_t mpmu_phys, phys_addr_t apmu_phys,
+			    phys_addr_t apbc_phys, phys_addr_t apbcp_phys)
 {
 	struct clk *clk;
 	struct clk *uart_pll;
@@ -74,25 +73,25 @@ void __init pxa910_clk_init(void)
 	void __iomem *apbcp_base;
 	void __iomem *apbc_base;
 
-	mpmu_base = ioremap(APB_PHYS_BASE + 0x50000, SZ_4K);
+	mpmu_base = ioremap(mpmu_phys, SZ_4K);
 	if (mpmu_base == NULL) {
 		pr_err("error to ioremap MPMU base\n");
 		return;
 	}
 
-	apmu_base = ioremap(AXI_PHYS_BASE + 0x82800, SZ_4K);
+	apmu_base = ioremap(apmu_phys, SZ_4K);
 	if (apmu_base == NULL) {
 		pr_err("error to ioremap APMU base\n");
 		return;
 	}
 
-	apbcp_base = ioremap(APB_PHYS_BASE + 0x3b000, SZ_4K);
+	apbcp_base = ioremap(apbcp_phys, SZ_4K);
 	if (apbcp_base == NULL) {
 		pr_err("error to ioremap APBC extension base\n");
 		return;
 	}
 
-	apbc_base = ioremap(APB_PHYS_BASE + 0x15000, SZ_4K);
+	apbc_base = ioremap(apbc_phys, SZ_4K);
 	if (apbc_base == NULL) {
 		pr_err("error to ioremap APBC base\n");
 		return;
diff --git a/drivers/clk/mvebu/Makefile b/drivers/clk/mvebu/Makefile
index 645ac7ea3565..8866115486f7 100644
--- a/drivers/clk/mvebu/Makefile
+++ b/drivers/clk/mvebu/Makefile
@@ -7,6 +7,6 @@ obj-$(CONFIG_ARMADA_375_CLK)	+= armada-375.o
 obj-$(CONFIG_ARMADA_38X_CLK)	+= armada-38x.o
 obj-$(CONFIG_ARMADA_39X_CLK)	+= armada-39x.o
 obj-$(CONFIG_ARMADA_XP_CLK)	+= armada-xp.o
-obj-$(CONFIG_DOVE_CLK)		+= dove.o
+obj-$(CONFIG_DOVE_CLK)		+= dove.o dove-divider.o
 obj-$(CONFIG_KIRKWOOD_CLK)	+= kirkwood.o
 obj-$(CONFIG_ORION_CLK)		+= orion.o
diff --git a/drivers/clk/mvebu/dove-divider.c b/drivers/clk/mvebu/dove-divider.c
new file mode 100644
index 000000000000..d5c5bfa35a5a
--- /dev/null
+++ b/drivers/clk/mvebu/dove-divider.c
@@ -0,0 +1,262 @@
+/*
+ * Marvell Dove PMU Core PLL divider driver
+ *
+ * Cleaned up by substantially rewriting, and converted to DT by
+ * Russell King.  Origin is not known.
+ */
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+
+#include "dove-divider.h"
+
+struct dove_clk {
+	const char *name;
+	struct clk_hw hw;
+	void __iomem *base;
+	spinlock_t *lock;
+	u8 div_bit_start;
+	u8 div_bit_end;
+	u8 div_bit_load;
+	u8 div_bit_size;
+	u32 *divider_table;
+};
+
+enum {
+	DIV_CTRL0 = 0,
+	DIV_CTRL1 = 4,
+	DIV_CTRL1_N_RESET_MASK = BIT(10),
+};
+
+#define to_dove_clk(hw) container_of(hw, struct dove_clk, hw)
+
+static void dove_load_divider(void __iomem *base, u32 val, u32 mask, u32 load)
+{
+	u32 v;
+
+	v = readl_relaxed(base + DIV_CTRL1) | DIV_CTRL1_N_RESET_MASK;
+	writel_relaxed(v, base + DIV_CTRL1);
+
+	v = (readl_relaxed(base + DIV_CTRL0) & ~(mask | load)) | val;
+	writel_relaxed(v, base + DIV_CTRL0);
+	writel_relaxed(v | load, base + DIV_CTRL0);
+	ndelay(250);
+	writel_relaxed(v, base + DIV_CTRL0);
+}
+
+static unsigned int dove_get_divider(struct dove_clk *dc)
+{
+	unsigned int divider;
+	u32 val;
+
+	val = readl_relaxed(dc->base + DIV_CTRL0);
+	val >>= dc->div_bit_start;
+
+	divider = val & ~(~0 << dc->div_bit_size);
+
+	if (dc->divider_table)
+		divider = dc->divider_table[divider];
+
+	return divider;
+}
+
+static int dove_calc_divider(const struct dove_clk *dc, unsigned long rate,
+			     unsigned long parent_rate, bool set)
+{
+	unsigned int divider, max;
+
+	divider = DIV_ROUND_CLOSEST(parent_rate, rate);
+
+	if (dc->divider_table) {
+		unsigned int i;
+
+		for (i = 0; dc->divider_table[i]; i++)
+			if (divider == dc->divider_table[i]) {
+				divider = i;
+				break;
+			}
+
+		if (!dc->divider_table[i])
+			return -EINVAL;
+	} else {
+		max = 1 << dc->div_bit_size;
+
+		if (set && (divider == 0 || divider >= max))
+			return -EINVAL;
+		if (divider >= max)
+			divider = max - 1;
+		else if (divider == 0)
+			divider = 1;
+	}
+
+	return divider;
+}
+
+static unsigned long dove_recalc_rate(struct clk_hw *hw, unsigned long parent)
+{
+	struct dove_clk *dc = to_dove_clk(hw);
+	unsigned int divider = dove_get_divider(dc);
+	unsigned long rate = DIV_ROUND_CLOSEST(parent, divider);
+
+	pr_debug("%s(): %s divider=%u parent=%lu rate=%lu\n",
+		 __func__, dc->name, divider, parent, rate);
+
+	return rate;
+}
+
+static long dove_round_rate(struct clk_hw *hw, unsigned long rate,
+			    unsigned long *parent)
+{
+	struct dove_clk *dc = to_dove_clk(hw);
+	unsigned long parent_rate = *parent;
+	int divider;
+
+	divider = dove_calc_divider(dc, rate, parent_rate, false);
+	if (divider < 0)
+		return divider;
+
+	rate = DIV_ROUND_CLOSEST(parent_rate, divider);
+
+	pr_debug("%s(): %s divider=%u parent=%lu rate=%lu\n",
+		 __func__, dc->name, divider, parent_rate, rate);
+
+	return rate;
+}
+
+static int dove_set_clock(struct clk_hw *hw, unsigned long rate,
+			  unsigned long parent_rate)
+{
+	struct dove_clk *dc = to_dove_clk(hw);
+	u32 mask, load, div;
+	int divider;
+
+	divider = dove_calc_divider(dc, rate, parent_rate, true);
+	if (divider < 0)
+		return divider;
+
+	pr_debug("%s(): %s divider=%u parent=%lu rate=%lu\n",
+		 __func__, dc->name, divider, parent_rate, rate);
+
+	div = (u32)divider << dc->div_bit_start;
+	mask = ~(~0 << dc->div_bit_size) << dc->div_bit_start;
+	load = BIT(dc->div_bit_load);
+
+	spin_lock(dc->lock);
+	dove_load_divider(dc->base, div, mask, load);
+	spin_unlock(dc->lock);
+
+	return 0;
+}
+
+static const struct clk_ops dove_divider_ops = {
+	.set_rate	= dove_set_clock,
+	.round_rate	= dove_round_rate,
+	.recalc_rate	= dove_recalc_rate,
+};
+
+static struct clk *clk_register_dove_divider(struct device *dev,
+	struct dove_clk *dc, const char **parent_names, size_t num_parents,
+	void __iomem *base)
+{
+	char name[32];
+	struct clk_init_data init = {
+		.name = name,
+		.ops = &dove_divider_ops,
+		.parent_names = parent_names,
+		.num_parents = num_parents,
+	};
+
+	strlcpy(name, dc->name, sizeof(name));
+
+	dc->hw.init = &init;
+	dc->base = base;
+	dc->div_bit_size = dc->div_bit_end - dc->div_bit_start + 1;
+
+	return clk_register(dev, &dc->hw);
+}
+
+static DEFINE_SPINLOCK(dove_divider_lock);
+
+static u32 axi_divider[] = {-1, 2, 1, 3, 4, 6, 5, 7, 8, 10, 9, 0};
+
+static struct dove_clk dove_hw_clocks[4] = {
+	{
+		.name = "axi",
+		.lock = &dove_divider_lock,
+		.div_bit_start = 1,
+		.div_bit_end = 6,
+		.div_bit_load = 7,
+		.divider_table = axi_divider,
+	}, {
+		.name = "gpu",
+		.lock = &dove_divider_lock,
+		.div_bit_start = 8,
+		.div_bit_end = 13,
+		.div_bit_load = 14,
+	}, {
+		.name = "vmeta",
+		.lock = &dove_divider_lock,
+		.div_bit_start = 15,
+		.div_bit_end = 20,
+		.div_bit_load = 21,
+	}, {
+		.name = "lcd",
+		.lock = &dove_divider_lock,
+		.div_bit_start = 22,
+		.div_bit_end = 27,
+		.div_bit_load = 28,
+	},
+};
+
+static const char *core_pll[] = {
+	"core-pll",
+};
+
+static int dove_divider_init(struct device *dev, void __iomem *base,
+	struct clk **clks)
+{
+	struct clk *clk;
+	int i;
+
+	/*
+	 * Create the core PLL clock.  We treat this as a fixed rate
+	 * clock as we don't know any better, and documentation is sparse.
+	 */
+	clk = clk_register_fixed_rate(dev, core_pll[0], NULL, CLK_IS_ROOT,
+				      2000000000UL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	for (i = 0; i < ARRAY_SIZE(dove_hw_clocks); i++)
+		clks[i] = clk_register_dove_divider(dev, &dove_hw_clocks[i],
+						    core_pll,
+						    ARRAY_SIZE(core_pll), base);
+
+	return 0;
+}
+
+static struct clk *dove_divider_clocks[4];
+
+static struct clk_onecell_data dove_divider_data = {
+	.clks = dove_divider_clocks,
+	.clk_num = ARRAY_SIZE(dove_divider_clocks),
+};
+
+void __init dove_divider_clk_init(struct device_node *np)
+{
+	void *base;
+
+	base = of_iomap(np, 0);
+	if (WARN_ON(!base))
+		return;
+
+	if (WARN_ON(dove_divider_init(NULL, base, dove_divider_clocks))) {
+		iounmap(base);
+		return;
+	}
+
+	of_clk_add_provider(np, of_clk_src_onecell_get, &dove_divider_data);
+}
diff --git a/drivers/clk/mvebu/dove-divider.h b/drivers/clk/mvebu/dove-divider.h
new file mode 100644
index 000000000000..4f2f718deb8e
--- /dev/null
+++ b/drivers/clk/mvebu/dove-divider.h
@@ -0,0 +1,6 @@
+#ifndef DOVE_DIVIDER_H
+#define DOVE_DIVIDER_H
+
+void __init dove_divider_clk_init(struct device_node *np);
+
+#endif
diff --git a/drivers/clk/mvebu/dove.c b/drivers/clk/mvebu/dove.c
index b8c2424ac926..59fad9546c84 100644
--- a/drivers/clk/mvebu/dove.c
+++ b/drivers/clk/mvebu/dove.c
@@ -17,6 +17,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include "common.h"
+#include "dove-divider.h"
 
 /*
  * Core Clocks
@@ -184,9 +185,14 @@ static void __init dove_clk_init(struct device_node *np)
 {
 	struct device_node *cgnp =
 		of_find_compatible_node(NULL, NULL, "marvell,dove-gating-clock");
+	struct device_node *ddnp =
+		of_find_compatible_node(NULL, NULL, "marvell,dove-divider-clock");
 
 	mvebu_coreclk_setup(np, &dove_coreclks);
 
+	if (ddnp)
+		dove_divider_clk_init(ddnp);
+
 	if (cgnp)
 		mvebu_clk_gating_setup(cgnp, dove_gating_desc);
 }
diff --git a/drivers/clk/nxp/Makefile b/drivers/clk/nxp/Makefile
index 7f608b0ad7b4..607bd48c6563 100644
--- a/drivers/clk/nxp/Makefile
+++ b/drivers/clk/nxp/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_ARCH_LPC18XX)	+= clk-lpc18xx-cgu.o
 obj-$(CONFIG_ARCH_LPC18XX)	+= clk-lpc18xx-ccu.o
+obj-$(CONFIG_ARCH_LPC32XX)	+= clk-lpc32xx.o
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
new file mode 100644
index 000000000000..10dd0fdaa474
--- /dev/null
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -0,0 +1,1569 @@
+/*
+ * Copyright 2015 Vladimir Zapolskiy <vz@mleia.com>
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/lpc32xx-clock.h>
+
+#undef pr_fmt
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+/* Common bitfield definitions for x397 PLL (lock), USB PLL and HCLK PLL */
+#define PLL_CTRL_ENABLE			BIT(16)
+#define PLL_CTRL_BYPASS			BIT(15)
+#define PLL_CTRL_DIRECT			BIT(14)
+#define PLL_CTRL_FEEDBACK		BIT(13)
+#define PLL_CTRL_POSTDIV		(BIT(12)|BIT(11))
+#define PLL_CTRL_PREDIV			(BIT(10)|BIT(9))
+#define PLL_CTRL_FEEDDIV		(0xFF << 1)
+#define PLL_CTRL_LOCK			BIT(0)
+
+/* Clock registers on System Control Block */
+#define LPC32XX_CLKPWR_DEBUG_CTRL	0x00
+#define LPC32XX_CLKPWR_USB_DIV		0x1C
+#define LPC32XX_CLKPWR_HCLKDIV_CTRL	0x40
+#define LPC32XX_CLKPWR_PWR_CTRL		0x44
+#define LPC32XX_CLKPWR_PLL397_CTRL	0x48
+#define LPC32XX_CLKPWR_OSC_CTRL		0x4C
+#define LPC32XX_CLKPWR_SYSCLK_CTRL	0x50
+#define LPC32XX_CLKPWR_LCDCLK_CTRL	0x54
+#define LPC32XX_CLKPWR_HCLKPLL_CTRL	0x58
+#define LPC32XX_CLKPWR_ADCCLK_CTRL1	0x60
+#define LPC32XX_CLKPWR_USB_CTRL		0x64
+#define LPC32XX_CLKPWR_SSP_CTRL		0x78
+#define LPC32XX_CLKPWR_I2S_CTRL		0x7C
+#define LPC32XX_CLKPWR_MS_CTRL		0x80
+#define LPC32XX_CLKPWR_MACCLK_CTRL	0x90
+#define LPC32XX_CLKPWR_TEST_CLK_CTRL	0xA4
+#define LPC32XX_CLKPWR_I2CCLK_CTRL	0xAC
+#define LPC32XX_CLKPWR_KEYCLK_CTRL	0xB0
+#define LPC32XX_CLKPWR_ADCCLK_CTRL	0xB4
+#define LPC32XX_CLKPWR_PWMCLK_CTRL	0xB8
+#define LPC32XX_CLKPWR_TIMCLK_CTRL	0xBC
+#define LPC32XX_CLKPWR_TIMCLK_CTRL1	0xC0
+#define LPC32XX_CLKPWR_SPI_CTRL		0xC4
+#define LPC32XX_CLKPWR_FLASHCLK_CTRL	0xC8
+#define LPC32XX_CLKPWR_UART3_CLK_CTRL	0xD0
+#define LPC32XX_CLKPWR_UART4_CLK_CTRL	0xD4
+#define LPC32XX_CLKPWR_UART5_CLK_CTRL	0xD8
+#define LPC32XX_CLKPWR_UART6_CLK_CTRL	0xDC
+#define LPC32XX_CLKPWR_IRDA_CLK_CTRL	0xE0
+#define LPC32XX_CLKPWR_UART_CLK_CTRL	0xE4
+#define LPC32XX_CLKPWR_DMA_CLK_CTRL	0xE8
+
+/* Clock registers on USB controller */
+#define LPC32XX_USB_CLK_CTRL		0xF4
+#define LPC32XX_USB_CLK_STS		0xF8
+
+static struct regmap_config lpc32xx_scb_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+	.max_register = 0x114,
+	.fast_io = true,
+};
+
+static struct regmap *clk_regmap;
+static void __iomem *usb_clk_vbase;
+
+enum {
+	LPC32XX_USB_CLK_OTG = LPC32XX_USB_CLK_HOST + 1,
+	LPC32XX_USB_CLK_AHB,
+
+	LPC32XX_USB_CLK_MAX = LPC32XX_USB_CLK_AHB + 1,
+};
+
+enum {
+	/* Start from the last defined clock in dt bindings */
+	LPC32XX_CLK_ADC_DIV = LPC32XX_CLK_ADC + 1,
+	LPC32XX_CLK_ADC_RTC,
+	LPC32XX_CLK_TEST1,
+	LPC32XX_CLK_TEST2,
+
+	/* System clocks, PLL 397x and HCLK PLL clocks */
+	LPC32XX_CLK_OSC,
+	LPC32XX_CLK_SYS,
+	LPC32XX_CLK_PLL397X,
+	LPC32XX_CLK_HCLK_PLL,
+	LPC32XX_CLK_HCLK_DIV_PERIPH,
+	LPC32XX_CLK_HCLK_DIV,
+	LPC32XX_CLK_HCLK,
+	LPC32XX_CLK_PERIPH,
+	LPC32XX_CLK_ARM,
+	LPC32XX_CLK_ARM_VFP,
+
+	/* USB clocks */
+	LPC32XX_CLK_USB_PLL,
+	LPC32XX_CLK_USB_DIV,
+	LPC32XX_CLK_USB,
+
+	/* Only one control PWR_CTRL[10] for both muxes */
+	LPC32XX_CLK_PERIPH_HCLK_MUX,
+	LPC32XX_CLK_PERIPH_ARM_MUX,
+
+	/* Only one control PWR_CTRL[2] for all three muxes */
+	LPC32XX_CLK_SYSCLK_PERIPH_MUX,
+	LPC32XX_CLK_SYSCLK_HCLK_MUX,
+	LPC32XX_CLK_SYSCLK_ARM_MUX,
+
+	/* Two clock sources external to the driver */
+	LPC32XX_CLK_XTAL_32K,
+	LPC32XX_CLK_XTAL,
+
+	/* Renumbered USB clocks, may have a parent from SCB table */
+	LPC32XX_CLK_USB_OFFSET,
+	LPC32XX_CLK_USB_I2C = LPC32XX_USB_CLK_I2C + LPC32XX_CLK_USB_OFFSET,
+	LPC32XX_CLK_USB_DEV = LPC32XX_USB_CLK_DEVICE + LPC32XX_CLK_USB_OFFSET,
+	LPC32XX_CLK_USB_HOST = LPC32XX_USB_CLK_HOST + LPC32XX_CLK_USB_OFFSET,
+	LPC32XX_CLK_USB_OTG = LPC32XX_USB_CLK_OTG + LPC32XX_CLK_USB_OFFSET,
+	LPC32XX_CLK_USB_AHB = LPC32XX_USB_CLK_AHB + LPC32XX_CLK_USB_OFFSET,
+
+	/* Stub for composite clocks */
+	LPC32XX_CLK__NULL,
+
+	/* Subclocks of composite clocks, clocks above are for CCF */
+	LPC32XX_CLK_PWM1_MUX,
+	LPC32XX_CLK_PWM1_DIV,
+	LPC32XX_CLK_PWM1_GATE,
+	LPC32XX_CLK_PWM2_MUX,
+	LPC32XX_CLK_PWM2_DIV,
+	LPC32XX_CLK_PWM2_GATE,
+	LPC32XX_CLK_UART3_MUX,
+	LPC32XX_CLK_UART3_DIV,
+	LPC32XX_CLK_UART3_GATE,
+	LPC32XX_CLK_UART4_MUX,
+	LPC32XX_CLK_UART4_DIV,
+	LPC32XX_CLK_UART4_GATE,
+	LPC32XX_CLK_UART5_MUX,
+	LPC32XX_CLK_UART5_DIV,
+	LPC32XX_CLK_UART5_GATE,
+	LPC32XX_CLK_UART6_MUX,
+	LPC32XX_CLK_UART6_DIV,
+	LPC32XX_CLK_UART6_GATE,
+	LPC32XX_CLK_TEST1_MUX,
+	LPC32XX_CLK_TEST1_GATE,
+	LPC32XX_CLK_TEST2_MUX,
+	LPC32XX_CLK_TEST2_GATE,
+	LPC32XX_CLK_USB_DIV_DIV,
+	LPC32XX_CLK_USB_DIV_GATE,
+	LPC32XX_CLK_SD_DIV,
+	LPC32XX_CLK_SD_GATE,
+	LPC32XX_CLK_LCD_DIV,
+	LPC32XX_CLK_LCD_GATE,
+
+	LPC32XX_CLK_HW_MAX,
+	LPC32XX_CLK_MAX = LPC32XX_CLK_SYSCLK_ARM_MUX + 1,
+	LPC32XX_CLK_CCF_MAX = LPC32XX_CLK_USB_AHB + 1,
+};
+
+static struct clk *clk[LPC32XX_CLK_MAX];
+static struct clk_onecell_data clk_data = {
+	.clks = clk,
+	.clk_num = LPC32XX_CLK_MAX,
+};
+
+static struct clk *usb_clk[LPC32XX_USB_CLK_MAX];
+static struct clk_onecell_data usb_clk_data = {
+	.clks = usb_clk,
+	.clk_num = LPC32XX_USB_CLK_MAX,
+};
+
+#define LPC32XX_CLK_PARENTS_MAX			5
+
+struct clk_proto_t {
+	const char *name;
+	const u8 parents[LPC32XX_CLK_PARENTS_MAX];
+	u8 num_parents;
+	unsigned long flags;
+};
+
+#define CLK_PREFIX(LITERAL)		LPC32XX_CLK_ ## LITERAL
+#define NUMARGS(...)	(sizeof((int[]){__VA_ARGS__})/sizeof(int))
+
+#define LPC32XX_CLK_DEFINE(_idx, _name, _flags, ...)		\
+	[CLK_PREFIX(_idx)] = {					\
+		.name = _name,					\
+		.flags = _flags,				\
+		.parents = { __VA_ARGS__ },			\
+		.num_parents = NUMARGS(__VA_ARGS__),		\
+	 }
+
+static const struct clk_proto_t clk_proto[LPC32XX_CLK_CCF_MAX] __initconst = {
+	LPC32XX_CLK_DEFINE(XTAL, "xtal", 0x0),
+	LPC32XX_CLK_DEFINE(XTAL_32K, "xtal_32k", 0x0),
+
+	LPC32XX_CLK_DEFINE(RTC, "rtc", 0x0, LPC32XX_CLK_XTAL_32K),
+	LPC32XX_CLK_DEFINE(OSC, "osc", CLK_IGNORE_UNUSED, LPC32XX_CLK_XTAL),
+	LPC32XX_CLK_DEFINE(SYS, "sys", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_OSC, LPC32XX_CLK_PLL397X),
+	LPC32XX_CLK_DEFINE(PLL397X, "pll_397x", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_RTC),
+	LPC32XX_CLK_DEFINE(HCLK_PLL, "hclk_pll", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYS),
+	LPC32XX_CLK_DEFINE(HCLK_DIV_PERIPH, "hclk_div_periph",
+		CLK_IGNORE_UNUSED, LPC32XX_CLK_HCLK_PLL),
+	LPC32XX_CLK_DEFINE(HCLK_DIV, "hclk_div", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_HCLK_PLL),
+	LPC32XX_CLK_DEFINE(HCLK, "hclk", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_PERIPH_HCLK_MUX),
+	LPC32XX_CLK_DEFINE(PERIPH, "pclk", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYSCLK_PERIPH_MUX),
+	LPC32XX_CLK_DEFINE(ARM, "arm", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_PERIPH_ARM_MUX),
+
+	LPC32XX_CLK_DEFINE(PERIPH_HCLK_MUX, "periph_hclk_mux",
+		CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYSCLK_HCLK_MUX, LPC32XX_CLK_SYSCLK_PERIPH_MUX),
+	LPC32XX_CLK_DEFINE(PERIPH_ARM_MUX, "periph_arm_mux", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYSCLK_ARM_MUX, LPC32XX_CLK_SYSCLK_PERIPH_MUX),
+	LPC32XX_CLK_DEFINE(SYSCLK_PERIPH_MUX, "sysclk_periph_mux",
+		CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYS, LPC32XX_CLK_HCLK_DIV_PERIPH),
+	LPC32XX_CLK_DEFINE(SYSCLK_HCLK_MUX, "sysclk_hclk_mux",
+		CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYS, LPC32XX_CLK_HCLK_DIV),
+	LPC32XX_CLK_DEFINE(SYSCLK_ARM_MUX, "sysclk_arm_mux", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_SYS, LPC32XX_CLK_HCLK_PLL),
+
+	LPC32XX_CLK_DEFINE(ARM_VFP, "vfp9", CLK_IGNORE_UNUSED,
+		LPC32XX_CLK_ARM),
+	LPC32XX_CLK_DEFINE(USB_PLL, "usb_pll",
+		CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT, LPC32XX_CLK_USB_DIV),
+	LPC32XX_CLK_DEFINE(USB_DIV, "usb_div", 0x0, LPC32XX_CLK_OSC),
+	LPC32XX_CLK_DEFINE(USB, "usb", 0x0, LPC32XX_CLK_USB_PLL),
+	LPC32XX_CLK_DEFINE(DMA, "dma", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(MLC, "mlc", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(SLC, "slc", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(LCD, "lcd", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(MAC, "mac", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(SD, "sd", 0x0, LPC32XX_CLK_ARM),
+	LPC32XX_CLK_DEFINE(DDRAM, "ddram", CLK_GET_RATE_NOCACHE,
+		LPC32XX_CLK_SYSCLK_ARM_MUX),
+	LPC32XX_CLK_DEFINE(SSP0, "ssp0", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(SSP1, "ssp1", 0x0, LPC32XX_CLK_HCLK),
+
+	/*
+	 * CLK_GET_RATE_NOCACHE is needed, if UART clock is disabled, its
+	 * divider register does not contain information about selected rate.
+	 */
+	LPC32XX_CLK_DEFINE(UART3, "uart3", CLK_GET_RATE_NOCACHE,
+		LPC32XX_CLK_PERIPH, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(UART4, "uart4", CLK_GET_RATE_NOCACHE,
+		LPC32XX_CLK_PERIPH, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(UART5, "uart5", CLK_GET_RATE_NOCACHE,
+		LPC32XX_CLK_PERIPH, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(UART6, "uart6", CLK_GET_RATE_NOCACHE,
+		LPC32XX_CLK_PERIPH, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(IRDA, "irda", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(I2C1, "i2c1", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(I2C2, "i2c2", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(TIMER0, "timer0", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(TIMER1, "timer1", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(TIMER2, "timer2", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(TIMER3, "timer3", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(TIMER4, "timer4", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(TIMER5, "timer5", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(WDOG, "watchdog", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(I2S0, "i2s0", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(I2S1, "i2s1", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(SPI1, "spi1", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(SPI2, "spi2", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(MCPWM, "mcpwm", 0x0, LPC32XX_CLK_HCLK),
+	LPC32XX_CLK_DEFINE(HSTIMER, "hstimer", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(KEY, "key", 0x0, LPC32XX_CLK_RTC),
+	LPC32XX_CLK_DEFINE(PWM1, "pwm1", 0x0,
+		LPC32XX_CLK_RTC, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(PWM2, "pwm2", 0x0,
+		LPC32XX_CLK_RTC, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(ADC, "adc", 0x0,
+		LPC32XX_CLK_ADC_RTC, LPC32XX_CLK_ADC_DIV),
+	LPC32XX_CLK_DEFINE(ADC_DIV, "adc_div", 0x0, LPC32XX_CLK_PERIPH),
+	LPC32XX_CLK_DEFINE(ADC_RTC, "adc_rtc", 0x0, LPC32XX_CLK_RTC),
+	LPC32XX_CLK_DEFINE(TEST1, "test1", 0x0,
+		LPC32XX_CLK_PERIPH, LPC32XX_CLK_RTC, LPC32XX_CLK_OSC),
+	LPC32XX_CLK_DEFINE(TEST2, "test2", 0x0,
+		LPC32XX_CLK_HCLK, LPC32XX_CLK_PERIPH, LPC32XX_CLK_USB,
+		LPC32XX_CLK_OSC, LPC32XX_CLK_PLL397X),
+
+	/* USB controller clocks */
+	LPC32XX_CLK_DEFINE(USB_AHB, "usb_ahb", 0x0, LPC32XX_CLK_USB),
+	LPC32XX_CLK_DEFINE(USB_OTG, "usb_otg", 0x0, LPC32XX_CLK_USB_AHB),
+	LPC32XX_CLK_DEFINE(USB_I2C, "usb_i2c", 0x0, LPC32XX_CLK_USB_AHB),
+	LPC32XX_CLK_DEFINE(USB_DEV, "usb_dev", 0x0, LPC32XX_CLK_USB_OTG),
+	LPC32XX_CLK_DEFINE(USB_HOST, "usb_host", 0x0, LPC32XX_CLK_USB_OTG),
+};
+
+struct lpc32xx_clk {
+	struct clk_hw hw;
+	u32 reg;
+	u32 enable;
+	u32 enable_mask;
+	u32 disable;
+	u32 disable_mask;
+	u32 busy;
+	u32 busy_mask;
+};
+
+enum clk_pll_mode {
+	PLL_UNKNOWN,
+	PLL_DIRECT,
+	PLL_BYPASS,
+	PLL_DIRECT_BYPASS,
+	PLL_INTEGER,
+	PLL_NON_INTEGER,
+};
+
+struct lpc32xx_pll_clk {
+	struct clk_hw hw;
+	u32 reg;
+	u32 enable;
+	unsigned long m_div;
+	unsigned long n_div;
+	unsigned long p_div;
+	enum clk_pll_mode mode;
+};
+
+struct lpc32xx_usb_clk {
+	struct clk_hw hw;
+	u32 ctrl_enable;
+	u32 ctrl_disable;
+	u32 ctrl_mask;
+	u32 enable;
+	u32 busy;
+};
+
+struct lpc32xx_clk_mux {
+	struct clk_hw	hw;
+	u32		reg;
+	u32		mask;
+	u8		shift;
+	u32		*table;
+	u8		flags;
+};
+
+struct lpc32xx_clk_div {
+	struct clk_hw	hw;
+	u32		reg;
+	u8		shift;
+	u8		width;
+	const struct clk_div_table	*table;
+	u8		flags;
+};
+
+struct lpc32xx_clk_gate {
+	struct clk_hw	hw;
+	u32		reg;
+	u8		bit_idx;
+	u8		flags;
+};
+
+#define to_lpc32xx_clk(_hw)	container_of(_hw, struct lpc32xx_clk, hw)
+#define to_lpc32xx_pll_clk(_hw)	container_of(_hw, struct lpc32xx_pll_clk, hw)
+#define to_lpc32xx_usb_clk(_hw)	container_of(_hw, struct lpc32xx_usb_clk, hw)
+#define to_lpc32xx_mux(_hw)	container_of(_hw, struct lpc32xx_clk_mux, hw)
+#define to_lpc32xx_div(_hw)	container_of(_hw, struct lpc32xx_clk_div, hw)
+#define to_lpc32xx_gate(_hw)	container_of(_hw, struct lpc32xx_clk_gate, hw)
+
+static inline bool pll_is_valid(u64 val0, u64 val1, u64 min, u64 max)
+{
+	return (val0 >= (val1 * min) && val0 <= (val1 * max));
+}
+
+static inline u32 lpc32xx_usb_clk_read(struct lpc32xx_usb_clk *clk)
+{
+	return readl(usb_clk_vbase + LPC32XX_USB_CLK_STS);
+}
+
+static inline void lpc32xx_usb_clk_write(struct lpc32xx_usb_clk *clk, u32 val)
+{
+	writel(val, usb_clk_vbase + LPC32XX_USB_CLK_CTRL);
+}
+
+static int clk_mask_enable(struct clk_hw *hw)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+
+	if (clk->busy_mask && (val & clk->busy_mask) == clk->busy)
+		return -EBUSY;
+
+	return regmap_update_bits(clk_regmap, clk->reg,
+				  clk->enable_mask, clk->enable);
+}
+
+static void clk_mask_disable(struct clk_hw *hw)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+
+	regmap_update_bits(clk_regmap, clk->reg,
+			   clk->disable_mask, clk->disable);
+}
+
+static int clk_mask_is_enabled(struct clk_hw *hw)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+
+	return ((val & clk->enable_mask) == clk->enable);
+}
+
+static const struct clk_ops clk_mask_ops = {
+	.enable = clk_mask_enable,
+	.disable = clk_mask_disable,
+	.is_enabled = clk_mask_is_enabled,
+};
+
+static int clk_pll_enable(struct clk_hw *hw)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	u32 val, count;
+
+	regmap_update_bits(clk_regmap, clk->reg, clk->enable, clk->enable);
+
+	for (count = 0; count < 1000; count++) {
+		regmap_read(clk_regmap, clk->reg, &val);
+		if (val & PLL_CTRL_LOCK)
+			break;
+	}
+
+	if (val & PLL_CTRL_LOCK)
+		return 0;
+
+	return -ETIMEDOUT;
+}
+
+static void clk_pll_disable(struct clk_hw *hw)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+
+	regmap_update_bits(clk_regmap, clk->reg, clk->enable, 0x0);
+}
+
+static int clk_pll_is_enabled(struct clk_hw *hw)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	u32 val;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+
+	val &= clk->enable | PLL_CTRL_LOCK;
+	if (val == (clk->enable | PLL_CTRL_LOCK))
+		return 1;
+
+	return 0;
+}
+
+static unsigned long clk_pll_397x_recalc_rate(struct clk_hw *hw,
+					      unsigned long parent_rate)
+{
+	return parent_rate * 397;
+}
+
+static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
+					 unsigned long parent_rate)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	bool is_direct, is_bypass, is_feedback;
+	unsigned long rate, cco_rate, ref_rate;
+	u32 val;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	is_direct = val & PLL_CTRL_DIRECT;
+	is_bypass = val & PLL_CTRL_BYPASS;
+	is_feedback = val & PLL_CTRL_FEEDBACK;
+
+	clk->m_div = ((val & PLL_CTRL_FEEDDIV) >> 1) + 1;
+	clk->n_div = ((val & PLL_CTRL_PREDIV) >> 9) + 1;
+	clk->p_div = ((val & PLL_CTRL_POSTDIV) >> 11) + 1;
+
+	if (is_direct && is_bypass) {
+		clk->p_div = 0;
+		clk->mode = PLL_DIRECT_BYPASS;
+		return parent_rate;
+	}
+	if (is_bypass) {
+		clk->mode = PLL_BYPASS;
+		return parent_rate / (1 << clk->p_div);
+	}
+	if (is_direct) {
+		clk->p_div = 0;
+		clk->mode = PLL_DIRECT;
+	}
+
+	ref_rate = parent_rate / clk->n_div;
+	rate = cco_rate = ref_rate * clk->m_div;
+
+	if (!is_direct) {
+		if (is_feedback) {
+			cco_rate *= (1 << clk->p_div);
+			clk->mode = PLL_INTEGER;
+		} else {
+			rate /= (1 << clk->p_div);
+			clk->mode = PLL_NON_INTEGER;
+		}
+	}
+
+	pr_debug("%s: %lu: 0x%x: %d/%d/%d, %lu/%lu/%d => %lu\n",
+		 clk_hw_get_name(hw),
+		 parent_rate, val, is_direct, is_bypass, is_feedback,
+		 clk->n_div, clk->m_div, (1 << clk->p_div), rate);
+
+	if (clk_pll_is_enabled(hw) &&
+	    !(pll_is_valid(parent_rate, 1, 1000000, 20000000)
+	      && pll_is_valid(cco_rate, 1, 156000000, 320000000)
+	      && pll_is_valid(ref_rate, 1, 1000000, 27000000)))
+		pr_err("%s: PLL clocks are not in valid ranges: %lu/%lu/%lu",
+		       clk_hw_get_name(hw),
+		       parent_rate, cco_rate, ref_rate);
+
+	return rate;
+}
+
+static int clk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			    unsigned long parent_rate)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	u32 val;
+	unsigned long new_rate;
+
+	/* Validate PLL clock parameters computed on round rate stage */
+	switch (clk->mode) {
+	case PLL_DIRECT:
+		val = PLL_CTRL_DIRECT;
+		val |= (clk->m_div - 1) << 1;
+		val |= (clk->n_div - 1) << 9;
+		new_rate = (parent_rate * clk->m_div) / clk->n_div;
+		break;
+	case PLL_BYPASS:
+		val = PLL_CTRL_BYPASS;
+		val |= (clk->p_div - 1) << 11;
+		new_rate = parent_rate / (1 << (clk->p_div));
+		break;
+	case PLL_DIRECT_BYPASS:
+		val = PLL_CTRL_DIRECT | PLL_CTRL_BYPASS;
+		new_rate = parent_rate;
+		break;
+	case PLL_INTEGER:
+		val = PLL_CTRL_FEEDBACK;
+		val |= (clk->m_div - 1) << 1;
+		val |= (clk->n_div - 1) << 9;
+		val |= (clk->p_div - 1) << 11;
+		new_rate = (parent_rate * clk->m_div) / clk->n_div;
+		break;
+	case PLL_NON_INTEGER:
+		val = 0x0;
+		val |= (clk->m_div - 1) << 1;
+		val |= (clk->n_div - 1) << 9;
+		val |= (clk->p_div - 1) << 11;
+		new_rate = (parent_rate * clk->m_div) /
+				(clk->n_div * (1 << clk->p_div));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Sanity check that round rate is equal to the requested one */
+	if (new_rate != rate)
+		return -EINVAL;
+
+	return regmap_update_bits(clk_regmap, clk->reg, 0x1FFFF, val);
+}
+
+static long clk_hclk_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long *parent_rate)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	u64 m_i, m, n, p, o = rate, i = *parent_rate, d = (u64)rate << 6;
+	int p_i, n_i;
+
+	pr_debug("%s: %lu/%lu\n", clk_hw_get_name(hw), *parent_rate, rate);
+
+	if (rate > 266500000)
+		return -EINVAL;
+
+	/* Have to check all 20 possibilities to find the minimal M */
+	for (p_i = 4; p_i >= 0; p_i--) {
+		for (n_i = 4; n_i > 0; n_i--) {
+			m_i = div64_u64(o * n_i * (1 << p_i), i);
+
+			/* Check for valid PLL parameter constraints */
+			if (!(m_i && m_i <= 256
+			      && pll_is_valid(i, n_i, 1000000, 27000000)
+			      && pll_is_valid(i * m_i * (1 << p_i), n_i,
+					      156000000, 320000000)))
+				continue;
+
+			/* Store some intermediate valid parameters */
+			if (o * n_i * (1 << p_i) - i * m_i <= d) {
+				m = m_i;
+				n = n_i;
+				p = p_i;
+				d = o * n_i * (1 << p_i) - i * m_i;
+			}
+		}
+	}
+
+	if (d == (u64)rate << 6) {
+		pr_err("%s: %lu: no valid PLL parameters are found\n",
+		       clk_hw_get_name(hw), rate);
+		return -EINVAL;
+	}
+
+	clk->m_div = m;
+	clk->n_div = n;
+	clk->p_div = p;
+
+	/* Set only direct or non-integer mode of PLL */
+	if (!p)
+		clk->mode = PLL_DIRECT;
+	else
+		clk->mode = PLL_NON_INTEGER;
+
+	o = div64_u64(i * m, n * (1 << p));
+
+	if (!d)
+		pr_debug("%s: %lu: found exact match: %llu/%llu/%llu\n",
+			 clk_hw_get_name(hw), rate, m, n, p);
+	else
+		pr_debug("%s: %lu: found closest: %llu/%llu/%llu - %llu\n",
+			 clk_hw_get_name(hw), rate, m, n, p, o);
+
+	return o;
+}
+
+static long clk_usb_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	struct lpc32xx_pll_clk *clk = to_lpc32xx_pll_clk(hw);
+	struct clk_hw *usb_div_hw, *osc_hw;
+	u64 d_i, n_i, m, o;
+
+	pr_debug("%s: %lu/%lu\n", clk_hw_get_name(hw), *parent_rate, rate);
+
+	/*
+	 * The only supported USB clock is 48MHz, with PLL internal constraints
+	 * on Fclkin, Fcco and Fref this implies that Fcco must be 192MHz
+	 * and post-divider must be 4, this slightly simplifies calculation of
+	 * USB divider, USB PLL N and M parameters.
+	 */
+	if (rate != 48000000)
+		return -EINVAL;
+
+	/* USB divider clock */
+	usb_div_hw = clk_hw_get_parent_by_index(hw, 0);
+	if (!usb_div_hw)
+		return -EINVAL;
+
+	/* Main oscillator clock */
+	osc_hw = clk_hw_get_parent_by_index(usb_div_hw, 0);
+	if (!osc_hw)
+		return -EINVAL;
+	o = clk_hw_get_rate(osc_hw);	/* must be in range 1..20 MHz */
+
+	/* Check if valid USB divider and USB PLL parameters exists */
+	for (d_i = 16; d_i >= 1; d_i--) {
+		for (n_i = 1; n_i <= 4; n_i++) {
+			m = div64_u64(192000000 * d_i * n_i, o);
+			if (!(m && m <= 256
+			      && m * o == 192000000 * d_i * n_i
+			      && pll_is_valid(o, d_i, 1000000, 20000000)
+			      && pll_is_valid(o, d_i * n_i, 1000000, 27000000)))
+				continue;
+
+			clk->n_div = n_i;
+			clk->m_div = m;
+			clk->p_div = 2;
+			clk->mode = PLL_NON_INTEGER;
+			*parent_rate = div64_u64(o, d_i);
+
+			return rate;
+		}
+	}
+
+	return -EINVAL;
+}
+
+#define LPC32XX_DEFINE_PLL_OPS(_name, _rc, _sr, _rr)			\
+	static const struct clk_ops clk_ ##_name ## _ops = {		\
+		.enable = clk_pll_enable,				\
+		.disable = clk_pll_disable,				\
+		.is_enabled = clk_pll_is_enabled,			\
+		.recalc_rate = _rc,					\
+		.set_rate = _sr,					\
+		.round_rate = _rr,					\
+	}
+
+LPC32XX_DEFINE_PLL_OPS(pll_397x, clk_pll_397x_recalc_rate, NULL, NULL);
+LPC32XX_DEFINE_PLL_OPS(hclk_pll, clk_pll_recalc_rate,
+		       clk_pll_set_rate, clk_hclk_pll_round_rate);
+LPC32XX_DEFINE_PLL_OPS(usb_pll,  clk_pll_recalc_rate,
+		       clk_pll_set_rate, clk_usb_pll_round_rate);
+
+static int clk_ddram_is_enabled(struct clk_hw *hw)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	val &= clk->enable_mask | clk->busy_mask;
+
+	return (val == (BIT(7) | BIT(0)) ||
+		val == (BIT(8) | BIT(1)));
+}
+
+static int clk_ddram_enable(struct clk_hw *hw)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val, hclk_div;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	hclk_div = val & clk->busy_mask;
+
+	/*
+	 * DDRAM clock must be 2 times higher than HCLK,
+	 * this implies DDRAM clock can not be enabled,
+	 * if HCLK clock rate is equal to ARM clock rate
+	 */
+	if (hclk_div == 0x0 || hclk_div == (BIT(1) | BIT(0)))
+		return -EINVAL;
+
+	return regmap_update_bits(clk_regmap, clk->reg,
+				  clk->enable_mask, hclk_div << 7);
+}
+
+static unsigned long clk_ddram_recalc_rate(struct clk_hw *hw,
+					   unsigned long parent_rate)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val;
+
+	if (!clk_ddram_is_enabled(hw))
+		return 0;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	val &= clk->enable_mask;
+
+	return parent_rate / (val >> 7);
+}
+
+static const struct clk_ops clk_ddram_ops = {
+	.enable = clk_ddram_enable,
+	.disable = clk_mask_disable,
+	.is_enabled = clk_ddram_is_enabled,
+	.recalc_rate = clk_ddram_recalc_rate,
+};
+
+static unsigned long lpc32xx_clk_uart_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct lpc32xx_clk *clk = to_lpc32xx_clk(hw);
+	u32 val, x, y;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	x = (val & 0xFF00) >> 8;
+	y = val & 0xFF;
+
+	if (x && y)
+		return (parent_rate * x) / y;
+	else
+		return 0;
+}
+
+static const struct clk_ops lpc32xx_uart_div_ops = {
+	.recalc_rate = lpc32xx_clk_uart_recalc_rate,
+};
+
+static const struct clk_div_table clk_hclk_div_table[] = {
+	{ .val = 0, .div = 1 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ },
+};
+
+static u32 test1_mux_table[] = { 0, 1, 2, };
+static u32 test2_mux_table[] = { 0, 1, 2, 5, 7, };
+
+static int clk_usb_enable(struct clk_hw *hw)
+{
+	struct lpc32xx_usb_clk *clk = to_lpc32xx_usb_clk(hw);
+	u32 val, ctrl_val, count;
+
+	pr_debug("%s: 0x%x\n", clk_hw_get_name(hw), clk->enable);
+
+	if (clk->ctrl_mask) {
+		regmap_read(clk_regmap, LPC32XX_CLKPWR_USB_CTRL, &ctrl_val);
+		regmap_update_bits(clk_regmap, LPC32XX_CLKPWR_USB_CTRL,
+				   clk->ctrl_mask, clk->ctrl_enable);
+	}
+
+	val = lpc32xx_usb_clk_read(clk);
+	if (clk->busy && (val & clk->busy) == clk->busy) {
+		if (clk->ctrl_mask)
+			regmap_write(clk_regmap, LPC32XX_CLKPWR_USB_CTRL,
+				     ctrl_val);
+		return -EBUSY;
+	}
+
+	val |= clk->enable;
+	lpc32xx_usb_clk_write(clk, val);
+
+	for (count = 0; count < 1000; count++) {
+		val = lpc32xx_usb_clk_read(clk);
+		if ((val & clk->enable) == clk->enable)
+			break;
+	}
+
+	if ((val & clk->enable) == clk->enable)
+		return 0;
+
+	if (clk->ctrl_mask)
+		regmap_write(clk_regmap, LPC32XX_CLKPWR_USB_CTRL, ctrl_val);
+
+	return -ETIMEDOUT;
+}
+
+static void clk_usb_disable(struct clk_hw *hw)
+{
+	struct lpc32xx_usb_clk *clk = to_lpc32xx_usb_clk(hw);
+	u32 val = lpc32xx_usb_clk_read(clk);
+
+	val &= ~clk->enable;
+	lpc32xx_usb_clk_write(clk, val);
+
+	if (clk->ctrl_mask)
+		regmap_update_bits(clk_regmap, LPC32XX_CLKPWR_USB_CTRL,
+				   clk->ctrl_mask, clk->ctrl_disable);
+}
+
+static int clk_usb_is_enabled(struct clk_hw *hw)
+{
+	struct lpc32xx_usb_clk *clk = to_lpc32xx_usb_clk(hw);
+	u32 ctrl_val, val;
+
+	if (clk->ctrl_mask) {
+		regmap_read(clk_regmap, LPC32XX_CLKPWR_USB_CTRL, &ctrl_val);
+		if ((ctrl_val & clk->ctrl_mask) != clk->ctrl_enable)
+			return 0;
+	}
+
+	val = lpc32xx_usb_clk_read(clk);
+
+	return ((val & clk->enable) == clk->enable);
+}
+
+static unsigned long clk_usb_i2c_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	return clk_get_rate(clk[LPC32XX_CLK_PERIPH]);
+}
+
+static const struct clk_ops clk_usb_ops = {
+	.enable = clk_usb_enable,
+	.disable = clk_usb_disable,
+	.is_enabled = clk_usb_is_enabled,
+};
+
+static const struct clk_ops clk_usb_i2c_ops = {
+	.enable = clk_usb_enable,
+	.disable = clk_usb_disable,
+	.is_enabled = clk_usb_is_enabled,
+	.recalc_rate = clk_usb_i2c_recalc_rate,
+};
+
+static int clk_gate_enable(struct clk_hw *hw)
+{
+	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
+	u32 mask = BIT(clk->bit_idx);
+	u32 val = (clk->flags & CLK_GATE_SET_TO_DISABLE ? 0x0 : mask);
+
+	return regmap_update_bits(clk_regmap, clk->reg, mask, val);
+}
+
+static void clk_gate_disable(struct clk_hw *hw)
+{
+	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
+	u32 mask = BIT(clk->bit_idx);
+	u32 val = (clk->flags & CLK_GATE_SET_TO_DISABLE ? mask : 0x0);
+
+	regmap_update_bits(clk_regmap, clk->reg, mask, val);
+}
+
+static int clk_gate_is_enabled(struct clk_hw *hw)
+{
+	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
+	u32 val;
+	bool is_set;
+
+	regmap_read(clk_regmap, clk->reg, &val);
+	is_set = val & BIT(clk->bit_idx);
+
+	return (clk->flags & CLK_GATE_SET_TO_DISABLE ? !is_set : is_set);
+}
+
+static const struct clk_ops lpc32xx_clk_gate_ops = {
+	.enable = clk_gate_enable,
+	.disable = clk_gate_disable,
+	.is_enabled = clk_gate_is_enabled,
+};
+
+#define div_mask(width)	((1 << (width)) - 1)
+
+static unsigned int _get_table_div(const struct clk_div_table *table,
+							unsigned int val)
+{
+	const struct clk_div_table *clkt;
+
+	for (clkt = table; clkt->div; clkt++)
+		if (clkt->val == val)
+			return clkt->div;
+	return 0;
+}
+
+static unsigned int _get_div(const struct clk_div_table *table,
+			     unsigned int val, unsigned long flags, u8 width)
+{
+	if (flags & CLK_DIVIDER_ONE_BASED)
+		return val;
+	if (table)
+		return _get_table_div(table, val);
+	return val + 1;
+}
+
+static unsigned long clk_divider_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct lpc32xx_clk_div *divider = to_lpc32xx_div(hw);
+	unsigned int val;
+
+	regmap_read(clk_regmap, divider->reg, &val);
+
+	val >>= divider->shift;
+	val &= div_mask(divider->width);
+
+	return divider_recalc_rate(hw, parent_rate, val, divider->table,
+				   divider->flags);
+}
+
+static long clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *prate)
+{
+	struct lpc32xx_clk_div *divider = to_lpc32xx_div(hw);
+	unsigned int bestdiv;
+
+	/* if read only, just return current value */
+	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
+		regmap_read(clk_regmap, divider->reg, &bestdiv);
+		bestdiv >>= divider->shift;
+		bestdiv &= div_mask(divider->width);
+		bestdiv = _get_div(divider->table, bestdiv, divider->flags,
+			divider->width);
+		return DIV_ROUND_UP(*prate, bestdiv);
+	}
+
+	return divider_round_rate(hw, rate, prate, divider->table,
+				  divider->width, divider->flags);
+}
+
+static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct lpc32xx_clk_div *divider = to_lpc32xx_div(hw);
+	unsigned int value;
+
+	value = divider_get_val(rate, parent_rate, divider->table,
+				divider->width, divider->flags);
+
+	return regmap_update_bits(clk_regmap, divider->reg,
+				  div_mask(divider->width) << divider->shift,
+				  value << divider->shift);
+}
+
+static const struct clk_ops lpc32xx_clk_divider_ops = {
+	.recalc_rate = clk_divider_recalc_rate,
+	.round_rate = clk_divider_round_rate,
+	.set_rate = clk_divider_set_rate,
+};
+
+static u8 clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct lpc32xx_clk_mux *mux = to_lpc32xx_mux(hw);
+	u32 num_parents = clk_hw_get_num_parents(hw);
+	u32 val;
+
+	regmap_read(clk_regmap, mux->reg, &val);
+	val >>= mux->shift;
+	val &= mux->mask;
+
+	if (mux->table) {
+		u32 i;
+
+		for (i = 0; i < num_parents; i++)
+			if (mux->table[i] == val)
+				return i;
+		return -EINVAL;
+	}
+
+	if (val >= num_parents)
+		return -EINVAL;
+
+	return val;
+}
+
+static int clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct lpc32xx_clk_mux *mux = to_lpc32xx_mux(hw);
+
+	if (mux->table)
+		index = mux->table[index];
+
+	return regmap_update_bits(clk_regmap, mux->reg,
+			  mux->mask << mux->shift, index << mux->shift);
+}
+
+static const struct clk_ops lpc32xx_clk_mux_ro_ops = {
+	.get_parent = clk_mux_get_parent,
+};
+
+static const struct clk_ops lpc32xx_clk_mux_ops = {
+	.get_parent = clk_mux_get_parent,
+	.set_parent = clk_mux_set_parent,
+	.determine_rate = __clk_mux_determine_rate,
+};
+
+enum lpc32xx_clk_type {
+	CLK_FIXED,
+	CLK_MUX,
+	CLK_DIV,
+	CLK_GATE,
+	CLK_COMPOSITE,
+	CLK_LPC32XX,
+	CLK_LPC32XX_PLL,
+	CLK_LPC32XX_USB,
+};
+
+struct clk_hw_proto0 {
+	const struct clk_ops *ops;
+	union {
+		struct lpc32xx_pll_clk pll;
+		struct lpc32xx_clk clk;
+		struct lpc32xx_usb_clk usb_clk;
+		struct lpc32xx_clk_mux mux;
+		struct lpc32xx_clk_div div;
+		struct lpc32xx_clk_gate gate;
+	};
+};
+
+struct clk_hw_proto1 {
+	struct clk_hw_proto0 *mux;
+	struct clk_hw_proto0 *div;
+	struct clk_hw_proto0 *gate;
+};
+
+struct clk_hw_proto {
+	enum lpc32xx_clk_type type;
+
+	union {
+		struct clk_fixed_rate f;
+		struct clk_hw_proto0 hw0;
+		struct clk_hw_proto1 hw1;
+	};
+};
+
+#define LPC32XX_DEFINE_FIXED(_idx, _rate, _flags)			\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_FIXED,						\
+	{								\
+		.f = {							\
+			.fixed_rate = (_rate),				\
+			.flags = (_flags),				\
+		},							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_PLL(_idx, _name, _reg, _enable)			\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_LPC32XX_PLL,					\
+	{								\
+		.hw0 = {						\
+			.ops = &clk_ ##_name ## _ops,			\
+			{						\
+				.pll = {				\
+					.reg = LPC32XX_CLKPWR_ ## _reg,	\
+					.enable = (_enable),		\
+				},					\
+			},						\
+		},							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_MUX(_idx, _reg, _shift, _mask, _table, _flags)	\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_MUX,						\
+	{								\
+		.hw0 = {						\
+			.ops = (_flags & CLK_MUX_READ_ONLY ?		\
+				&lpc32xx_clk_mux_ro_ops :		\
+				&lpc32xx_clk_mux_ops),			\
+			{						\
+				.mux = {				\
+					.reg = LPC32XX_CLKPWR_ ## _reg,	\
+					.mask = (_mask),		\
+					.shift = (_shift),		\
+					.table = (_table),		\
+					.flags = (_flags),		\
+				},					\
+			},						\
+		},							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_DIV(_idx, _reg, _shift, _width, _table, _flags)	\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_DIV,						\
+	{								\
+		.hw0 = {						\
+			.ops = &lpc32xx_clk_divider_ops,		\
+			{						\
+				.div = {				\
+					.reg = LPC32XX_CLKPWR_ ## _reg,	\
+					.shift = (_shift),		\
+					.width = (_width),		\
+					.table = (_table),		\
+					.flags = (_flags),		\
+				 },					\
+			},						\
+		 },							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_GATE(_idx, _reg, _bit, _flags)			\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_GATE,						\
+	{								\
+		.hw0 = {						\
+			.ops = &lpc32xx_clk_gate_ops,			\
+			{						\
+				.gate = {				\
+					.reg = LPC32XX_CLKPWR_ ## _reg,	\
+					.bit_idx = (_bit),		\
+					.flags = (_flags),		\
+				},					\
+			},						\
+		},							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_CLK(_idx, _reg, _e, _em, _d, _dm, _b, _bm, _ops)	\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_LPC32XX,						\
+	{								\
+		.hw0 = {						\
+			.ops = &(_ops),					\
+			{						\
+				.clk = {				\
+					.reg = LPC32XX_CLKPWR_ ## _reg,	\
+					.enable = (_e),			\
+					.enable_mask = (_em),		\
+					.disable = (_d),		\
+					.disable_mask = (_dm),		\
+					.busy = (_b),			\
+					.busy_mask = (_bm),		\
+				},					\
+			},						\
+		},							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_USB(_idx, _ce, _cd, _cm, _e, _b, _ops)		\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_LPC32XX_USB,					\
+	{								\
+		.hw0 = {						\
+			.ops = &(_ops),					\
+			{						\
+				.usb_clk = {				\
+					.ctrl_enable = (_ce),		\
+					.ctrl_disable = (_cd),		\
+					.ctrl_mask = (_cm),		\
+					.enable = (_e),			\
+					.busy = (_b),			\
+				}					\
+			},						\
+		}							\
+	},								\
+}
+
+#define LPC32XX_DEFINE_COMPOSITE(_idx, _mux, _div, _gate)		\
+[CLK_PREFIX(_idx)] = {							\
+	.type = CLK_COMPOSITE,						\
+	{								\
+		.hw1 = {						\
+		.mux = (CLK_PREFIX(_mux) == LPC32XX_CLK__NULL ? NULL :	\
+			&clk_hw_proto[CLK_PREFIX(_mux)].hw0),		\
+		.div = (CLK_PREFIX(_div) == LPC32XX_CLK__NULL ? NULL :	\
+			&clk_hw_proto[CLK_PREFIX(_div)].hw0),		\
+		.gate = (CLK_PREFIX(_gate) == LPC32XX_CLK__NULL ? NULL :\
+			 &clk_hw_proto[CLK_PREFIX(_gate)].hw0),		\
+		},							\
+	},								\
+}
+
+static struct clk_hw_proto clk_hw_proto[LPC32XX_CLK_HW_MAX] = {
+	LPC32XX_DEFINE_FIXED(RTC, 32768, 0),
+	LPC32XX_DEFINE_PLL(PLL397X, pll_397x, HCLKPLL_CTRL, BIT(1)),
+	LPC32XX_DEFINE_PLL(HCLK_PLL, hclk_pll, HCLKPLL_CTRL, PLL_CTRL_ENABLE),
+	LPC32XX_DEFINE_PLL(USB_PLL, usb_pll, USB_CTRL, PLL_CTRL_ENABLE),
+	LPC32XX_DEFINE_GATE(OSC, OSC_CTRL, 0, CLK_GATE_SET_TO_DISABLE),
+	LPC32XX_DEFINE_GATE(USB, USB_CTRL, 18, 0),
+
+	LPC32XX_DEFINE_DIV(HCLK_DIV_PERIPH, HCLKDIV_CTRL, 2, 5, NULL,
+			   CLK_DIVIDER_READ_ONLY),
+	LPC32XX_DEFINE_DIV(HCLK_DIV, HCLKDIV_CTRL, 0, 2, clk_hclk_div_table,
+			   CLK_DIVIDER_READ_ONLY),
+
+	/* Register 3 read-only muxes with a single control PWR_CTRL[2] */
+	LPC32XX_DEFINE_MUX(SYSCLK_PERIPH_MUX, PWR_CTRL, 2, 0x1, NULL,
+			   CLK_MUX_READ_ONLY),
+	LPC32XX_DEFINE_MUX(SYSCLK_HCLK_MUX, PWR_CTRL, 2, 0x1, NULL,
+			   CLK_MUX_READ_ONLY),
+	LPC32XX_DEFINE_MUX(SYSCLK_ARM_MUX, PWR_CTRL, 2, 0x1, NULL,
+			   CLK_MUX_READ_ONLY),
+	/* Register 2 read-only muxes with a single control PWR_CTRL[10] */
+	LPC32XX_DEFINE_MUX(PERIPH_HCLK_MUX, PWR_CTRL, 10, 0x1, NULL,
+			   CLK_MUX_READ_ONLY),
+	LPC32XX_DEFINE_MUX(PERIPH_ARM_MUX, PWR_CTRL, 10, 0x1, NULL,
+			   CLK_MUX_READ_ONLY),
+
+	/* 3 always on gates with a single control PWR_CTRL[0] same as OSC */
+	LPC32XX_DEFINE_GATE(PERIPH, PWR_CTRL, 0, CLK_GATE_SET_TO_DISABLE),
+	LPC32XX_DEFINE_GATE(HCLK, PWR_CTRL, 0, CLK_GATE_SET_TO_DISABLE),
+	LPC32XX_DEFINE_GATE(ARM, PWR_CTRL, 0, CLK_GATE_SET_TO_DISABLE),
+
+	LPC32XX_DEFINE_GATE(ARM_VFP, DEBUG_CTRL, 4, 0),
+	LPC32XX_DEFINE_GATE(DMA, DMA_CLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_CLK(DDRAM, HCLKDIV_CTRL, 0x0, BIT(8) | BIT(7),
+		   0x0, BIT(8) | BIT(7), 0x0, BIT(1) | BIT(0), clk_ddram_ops),
+
+	LPC32XX_DEFINE_GATE(TIMER0, TIMCLK_CTRL1, 2, 0),
+	LPC32XX_DEFINE_GATE(TIMER1, TIMCLK_CTRL1, 3, 0),
+	LPC32XX_DEFINE_GATE(TIMER2, TIMCLK_CTRL1, 4, 0),
+	LPC32XX_DEFINE_GATE(TIMER3, TIMCLK_CTRL1, 5, 0),
+	LPC32XX_DEFINE_GATE(TIMER4, TIMCLK_CTRL1, 0, 0),
+	LPC32XX_DEFINE_GATE(TIMER5, TIMCLK_CTRL1, 1, 0),
+
+	LPC32XX_DEFINE_GATE(SSP0, SSP_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(SSP1, SSP_CTRL, 1, 0),
+	LPC32XX_DEFINE_GATE(SPI1, SPI_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(SPI2, SPI_CTRL, 4, 0),
+	LPC32XX_DEFINE_GATE(I2S0, I2S_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(I2S1, I2S_CTRL, 1, 0),
+	LPC32XX_DEFINE_GATE(I2C1, I2CCLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(I2C2, I2CCLK_CTRL, 1, 0),
+	LPC32XX_DEFINE_GATE(WDOG, TIMCLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(HSTIMER, TIMCLK_CTRL, 1, 0),
+
+	LPC32XX_DEFINE_GATE(KEY, KEYCLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_GATE(MCPWM, TIMCLK_CTRL1, 6, 0),
+
+	LPC32XX_DEFINE_MUX(PWM1_MUX, PWMCLK_CTRL, 1, 0x1, NULL, 0),
+	LPC32XX_DEFINE_DIV(PWM1_DIV, PWMCLK_CTRL, 4, 4, NULL,
+			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+	LPC32XX_DEFINE_GATE(PWM1_GATE, PWMCLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_COMPOSITE(PWM1, PWM1_MUX, PWM1_DIV, PWM1_GATE),
+
+	LPC32XX_DEFINE_MUX(PWM2_MUX, PWMCLK_CTRL, 3, 0x1, NULL, 0),
+	LPC32XX_DEFINE_DIV(PWM2_DIV, PWMCLK_CTRL, 8, 4, NULL,
+			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+	LPC32XX_DEFINE_GATE(PWM2_GATE, PWMCLK_CTRL, 2, 0),
+	LPC32XX_DEFINE_COMPOSITE(PWM2, PWM2_MUX, PWM2_DIV, PWM2_GATE),
+
+	LPC32XX_DEFINE_MUX(UART3_MUX, UART3_CLK_CTRL, 16, 0x1, NULL, 0),
+	LPC32XX_DEFINE_CLK(UART3_DIV, UART3_CLK_CTRL,
+			   0, 0, 0, 0, 0, 0, lpc32xx_uart_div_ops),
+	LPC32XX_DEFINE_GATE(UART3_GATE, UART_CLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_COMPOSITE(UART3, UART3_MUX, UART3_DIV, UART3_GATE),
+
+	LPC32XX_DEFINE_MUX(UART4_MUX, UART4_CLK_CTRL, 16, 0x1, NULL, 0),
+	LPC32XX_DEFINE_CLK(UART4_DIV, UART4_CLK_CTRL,
+			   0, 0, 0, 0, 0, 0, lpc32xx_uart_div_ops),
+	LPC32XX_DEFINE_GATE(UART4_GATE, UART_CLK_CTRL, 1, 0),
+	LPC32XX_DEFINE_COMPOSITE(UART4, UART4_MUX, UART4_DIV, UART4_GATE),
+
+	LPC32XX_DEFINE_MUX(UART5_MUX, UART5_CLK_CTRL, 16, 0x1, NULL, 0),
+	LPC32XX_DEFINE_CLK(UART5_DIV, UART5_CLK_CTRL,
+			   0, 0, 0, 0, 0, 0, lpc32xx_uart_div_ops),
+	LPC32XX_DEFINE_GATE(UART5_GATE, UART_CLK_CTRL, 2, 0),
+	LPC32XX_DEFINE_COMPOSITE(UART5, UART5_MUX, UART5_DIV, UART5_GATE),
+
+	LPC32XX_DEFINE_MUX(UART6_MUX, UART6_CLK_CTRL, 16, 0x1, NULL, 0),
+	LPC32XX_DEFINE_CLK(UART6_DIV, UART6_CLK_CTRL,
+			   0, 0, 0, 0, 0, 0, lpc32xx_uart_div_ops),
+	LPC32XX_DEFINE_GATE(UART6_GATE, UART_CLK_CTRL, 3, 0),
+	LPC32XX_DEFINE_COMPOSITE(UART6, UART6_MUX, UART6_DIV, UART6_GATE),
+
+	LPC32XX_DEFINE_CLK(IRDA, IRDA_CLK_CTRL,
+			   0, 0, 0, 0, 0, 0, lpc32xx_uart_div_ops),
+
+	LPC32XX_DEFINE_MUX(TEST1_MUX, TEST_CLK_CTRL, 5, 0x3,
+			   test1_mux_table, 0),
+	LPC32XX_DEFINE_GATE(TEST1_GATE, TEST_CLK_CTRL, 4, 0),
+	LPC32XX_DEFINE_COMPOSITE(TEST1, TEST1_MUX, _NULL, TEST1_GATE),
+
+	LPC32XX_DEFINE_MUX(TEST2_MUX, TEST_CLK_CTRL, 1, 0x7,
+			   test2_mux_table, 0),
+	LPC32XX_DEFINE_GATE(TEST2_GATE, TEST_CLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_COMPOSITE(TEST2, TEST2_MUX, _NULL, TEST2_GATE),
+
+	LPC32XX_DEFINE_MUX(SYS, SYSCLK_CTRL, 0, 0x1, NULL, CLK_MUX_READ_ONLY),
+
+	LPC32XX_DEFINE_DIV(USB_DIV_DIV, USB_DIV, 0, 4, NULL, 0),
+	LPC32XX_DEFINE_GATE(USB_DIV_GATE, USB_CTRL, 17, 0),
+	LPC32XX_DEFINE_COMPOSITE(USB_DIV, _NULL, USB_DIV_DIV, USB_DIV_GATE),
+
+	LPC32XX_DEFINE_DIV(SD_DIV, MS_CTRL, 0, 4, NULL,
+			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+	LPC32XX_DEFINE_CLK(SD_GATE, MS_CTRL, BIT(5) | BIT(9), BIT(5) | BIT(9),
+			   0x0, BIT(5) | BIT(9), 0x0, 0x0, clk_mask_ops),
+	LPC32XX_DEFINE_COMPOSITE(SD, _NULL, SD_DIV, SD_GATE),
+
+	LPC32XX_DEFINE_DIV(LCD_DIV, LCDCLK_CTRL, 0, 5, NULL, 0),
+	LPC32XX_DEFINE_GATE(LCD_GATE, LCDCLK_CTRL, 5, 0),
+	LPC32XX_DEFINE_COMPOSITE(LCD, _NULL, LCD_DIV, LCD_GATE),
+
+	LPC32XX_DEFINE_CLK(MAC, MACCLK_CTRL,
+			   BIT(2) | BIT(1) | BIT(0), BIT(2) | BIT(1) | BIT(0),
+			   BIT(2) | BIT(1) | BIT(0), BIT(2) | BIT(1) | BIT(0),
+			   0x0, 0x0, clk_mask_ops),
+	LPC32XX_DEFINE_CLK(SLC, FLASHCLK_CTRL,
+			   BIT(2) | BIT(0), BIT(2) | BIT(0), 0x0,
+			   BIT(0), BIT(1), BIT(2) | BIT(1), clk_mask_ops),
+	LPC32XX_DEFINE_CLK(MLC, FLASHCLK_CTRL,
+			   BIT(1), BIT(2) | BIT(1), 0x0, BIT(1),
+			   BIT(2) | BIT(0), BIT(2) | BIT(0), clk_mask_ops),
+	/*
+	 * ADC/TS clock unfortunately cannot be registered as a composite one
+	 * due to a different connection of gate, div and mux, e.g. gating it
+	 * won't mean that the clock is off, if peripheral clock is its parent:
+	 *
+	 * rtc-->[gate]-->|     |
+	 *                | mux |--> adc/ts
+	 * pclk-->[div]-->|     |
+	 *
+	 * Constraints:
+	 * ADC --- resulting clock must be <= 4.5 MHz
+	 * TS  --- resulting clock must be <= 400 KHz
+	 */
+	LPC32XX_DEFINE_DIV(ADC_DIV, ADCCLK_CTRL1, 0, 8, NULL, 0),
+	LPC32XX_DEFINE_GATE(ADC_RTC, ADCCLK_CTRL, 0, 0),
+	LPC32XX_DEFINE_MUX(ADC, ADCCLK_CTRL1, 8, 0x1, NULL, 0),
+
+	/* USB controller clocks */
+	LPC32XX_DEFINE_USB(USB_AHB,
+			   BIT(24), 0x0, BIT(24), BIT(4), 0, clk_usb_ops),
+	LPC32XX_DEFINE_USB(USB_OTG,
+			   0x0, 0x0, 0x0, BIT(3), 0, clk_usb_ops),
+	LPC32XX_DEFINE_USB(USB_I2C,
+			   0x0, BIT(23), BIT(23), BIT(2), 0, clk_usb_i2c_ops),
+	LPC32XX_DEFINE_USB(USB_DEV,
+			   BIT(22), 0x0, BIT(22), BIT(1), BIT(0), clk_usb_ops),
+	LPC32XX_DEFINE_USB(USB_HOST,
+			   BIT(21), 0x0, BIT(21), BIT(0), BIT(1), clk_usb_ops),
+};
+
+static struct clk * __init lpc32xx_clk_register(u32 id)
+{
+	const struct clk_proto_t *lpc32xx_clk = &clk_proto[id];
+	struct clk_hw_proto *clk_hw = &clk_hw_proto[id];
+	const char *parents[LPC32XX_CLK_PARENTS_MAX];
+	struct clk *clk;
+	unsigned int i;
+
+	for (i = 0; i < lpc32xx_clk->num_parents; i++)
+		parents[i] = clk_proto[lpc32xx_clk->parents[i]].name;
+
+	pr_debug("%s: derived from '%s', clock type %d\n", lpc32xx_clk->name,
+		 parents[0], clk_hw->type);
+
+	switch (clk_hw->type) {
+	case CLK_LPC32XX:
+	case CLK_LPC32XX_PLL:
+	case CLK_LPC32XX_USB:
+	case CLK_MUX:
+	case CLK_DIV:
+	case CLK_GATE:
+	{
+		struct clk_init_data clk_init = {
+			.name = lpc32xx_clk->name,
+			.parent_names = parents,
+			.num_parents = lpc32xx_clk->num_parents,
+			.flags = lpc32xx_clk->flags,
+			.ops = clk_hw->hw0.ops,
+		};
+		struct clk_hw *hw;
+
+		if (clk_hw->type == CLK_LPC32XX)
+			hw = &clk_hw->hw0.clk.hw;
+		else if (clk_hw->type == CLK_LPC32XX_PLL)
+			hw = &clk_hw->hw0.pll.hw;
+		else if (clk_hw->type == CLK_LPC32XX_USB)
+			hw = &clk_hw->hw0.usb_clk.hw;
+		else if (clk_hw->type == CLK_MUX)
+			hw = &clk_hw->hw0.mux.hw;
+		else if (clk_hw->type == CLK_DIV)
+			hw = &clk_hw->hw0.div.hw;
+		else if (clk_hw->type == CLK_GATE)
+			hw = &clk_hw->hw0.gate.hw;
+
+		hw->init = &clk_init;
+		clk = clk_register(NULL, hw);
+		break;
+	}
+	case CLK_COMPOSITE:
+	{
+		struct clk_hw *mux_hw = NULL, *div_hw = NULL, *gate_hw = NULL;
+		const struct clk_ops *mops = NULL, *dops = NULL, *gops = NULL;
+		struct clk_hw_proto0 *mux0, *div0, *gate0;
+
+		mux0 = clk_hw->hw1.mux;
+		div0 = clk_hw->hw1.div;
+		gate0 = clk_hw->hw1.gate;
+		if (mux0) {
+			mops = mux0->ops;
+			mux_hw = &mux0->clk.hw;
+		}
+		if (div0) {
+			dops = div0->ops;
+			div_hw = &div0->clk.hw;
+		}
+		if (gate0) {
+			gops = gate0->ops;
+			gate_hw = &gate0->clk.hw;
+		}
+
+		clk = clk_register_composite(NULL, lpc32xx_clk->name,
+				parents, lpc32xx_clk->num_parents,
+				mux_hw, mops, div_hw, dops,
+				gate_hw, gops, lpc32xx_clk->flags);
+		break;
+	}
+	case CLK_FIXED:
+	{
+		struct clk_fixed_rate *fixed = &clk_hw->f;
+
+		clk = clk_register_fixed_rate(NULL, lpc32xx_clk->name,
+			parents[0], fixed->flags, fixed->fixed_rate);
+		break;
+	}
+	default:
+		clk = ERR_PTR(-EINVAL);
+	}
+
+	return clk;
+}
+
+static void __init lpc32xx_clk_init(struct device_node *np)
+{
+	unsigned int i;
+	struct clk *clk_osc, *clk_32k;
+	void __iomem *base = NULL;
+
+	/* Ensure that parent clocks are available and valid */
+	clk_32k = of_clk_get_by_name(np, clk_proto[LPC32XX_CLK_XTAL_32K].name);
+	if (IS_ERR(clk_32k)) {
+		pr_err("failed to find external 32KHz clock: %ld\n",
+		       PTR_ERR(clk_32k));
+		return;
+	}
+	if (clk_get_rate(clk_32k) != 32768) {
+		pr_err("invalid clock rate of external 32KHz oscillator");
+		return;
+	}
+
+	clk_osc = of_clk_get_by_name(np, clk_proto[LPC32XX_CLK_XTAL].name);
+	if (IS_ERR(clk_osc)) {
+		pr_err("failed to find external main oscillator clock: %ld\n",
+		       PTR_ERR(clk_osc));
+		return;
+	}
+
+	base = of_iomap(np, 0);
+	if (!base) {
+		pr_err("failed to map system control block registers\n");
+		return;
+	}
+
+	clk_regmap = regmap_init_mmio(NULL, base, &lpc32xx_scb_regmap_config);
+	if (IS_ERR(clk_regmap)) {
+		pr_err("failed to regmap system control block: %ld\n",
+			PTR_ERR(clk_regmap));
+		return;
+	}
+
+	for (i = 0; i < LPC32XX_CLK_MAX; i++) {
+		clk[i] = lpc32xx_clk_register(i);
+		if (IS_ERR(clk[i])) {
+			pr_err("failed to register %s clock: %ld\n",
+				clk_proto[i].name, PTR_ERR(clk[i]));
+			clk[i] = NULL;
+		}
+	}
+
+	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+
+	/* For 13MHz osc valid output range of PLL is from 156MHz to 266.5MHz */
+	clk_set_rate(clk[LPC32XX_CLK_HCLK_PLL], 208000000);
+
+	/* Set 48MHz rate of USB PLL clock */
+	clk_set_rate(clk[LPC32XX_CLK_USB_PLL], 48000000);
+
+	/* These two clocks must be always on independently on consumers */
+	clk_prepare_enable(clk[LPC32XX_CLK_ARM]);
+	clk_prepare_enable(clk[LPC32XX_CLK_HCLK]);
+
+	/* Enable ARM VFP by default */
+	clk_prepare_enable(clk[LPC32XX_CLK_ARM_VFP]);
+
+	/* Disable enabled by default clocks for NAND MLC and SLC */
+	clk_mask_disable(&clk_hw_proto[LPC32XX_CLK_SLC].hw0.clk.hw);
+	clk_mask_disable(&clk_hw_proto[LPC32XX_CLK_MLC].hw0.clk.hw);
+}
+CLK_OF_DECLARE(lpc32xx_clk, "nxp,lpc3220-clk", lpc32xx_clk_init);
+
+static void __init lpc32xx_usb_clk_init(struct device_node *np)
+{
+	unsigned int i;
+
+	usb_clk_vbase = of_iomap(np, 0);
+	if (!usb_clk_vbase) {
+		pr_err("failed to map address range\n");
+		return;
+	}
+
+	for (i = 0; i < LPC32XX_USB_CLK_MAX; i++) {
+		usb_clk[i] = lpc32xx_clk_register(i + LPC32XX_CLK_USB_OFFSET);
+		if (IS_ERR(usb_clk[i])) {
+			pr_err("failed to register %s clock: %ld\n",
+				clk_proto[i].name, PTR_ERR(usb_clk[i]));
+			usb_clk[i] = NULL;
+		}
+	}
+
+	of_clk_add_provider(np, of_clk_src_onecell_get, &usb_clk_data);
+}
+CLK_OF_DECLARE(lpc32xx_usb_clk, "nxp,lpc3220-usb-clk", lpc32xx_usb_clk_init);
diff --git a/drivers/clk/pxa/clk-pxa25x.c b/drivers/clk/pxa/clk-pxa25x.c
index 542e45ef5087..b7747229db9a 100644
--- a/drivers/clk/pxa/clk-pxa25x.c
+++ b/drivers/clk/pxa/clk-pxa25x.c
@@ -17,7 +17,6 @@
 #include <linux/clkdev.h>
 #include <linux/io.h>
 #include <linux/of.h>
-#include <mach/pxa25x.h>
 #include <mach/pxa2xx-regs.h>
 
 #include <dt-bindings/clock/pxa-clock.h>
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index ee4c83aab4f4..b552eceec2be 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -106,3 +106,20 @@ config MSM_MMCC_8974
 	  Support for the multimedia clock controller on msm8974 devices.
 	  Say Y if you want to support multimedia devices such as display,
 	  graphics, video encode/decode, camera, etc.
+
+config MSM_GCC_8996
+	tristate "MSM8996 Global Clock Controller"
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the global clock controller on msm8996 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, UFS, SD/eMMC, PCIe, etc.
+
+config MSM_MMCC_8996
+	tristate "MSM8996 Multimedia Clock Controller"
+	select MSM_GCC_8996
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the multimedia clock controller on msm8996 devices.
+	  Say Y if you want to support multimedia devices such as display,
+	  graphics, video encode/decode, camera, etc.
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index fe6252349e55..dc4280b85db1 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -2,6 +2,7 @@ obj-$(CONFIG_COMMON_CLK_QCOM) += clk-qcom.o
 
 clk-qcom-y += common.o
 clk-qcom-y += clk-regmap.o
+clk-qcom-y += clk-alpha-pll.o
 clk-qcom-y += clk-pll.o
 clk-qcom-y += clk-rcg.o
 clk-qcom-y += clk-rcg2.o
@@ -20,5 +21,7 @@ obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o
 obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o
 obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
 obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
+obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o
 obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o
+obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o
diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
new file mode 100644
index 000000000000..e6a03eaf7a93
--- /dev/null
+++ b/drivers/clk/qcom/clk-alpha-pll.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/export.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/delay.h>
+
+#include "clk-alpha-pll.h"
+
+#define PLL_MODE		0x00
+# define PLL_OUTCTRL		BIT(0)
+# define PLL_BYPASSNL		BIT(1)
+# define PLL_RESET_N		BIT(2)
+# define PLL_LOCK_COUNT_SHIFT	8
+# define PLL_LOCK_COUNT_MASK	0x3f
+# define PLL_BIAS_COUNT_SHIFT	14
+# define PLL_BIAS_COUNT_MASK	0x3f
+# define PLL_VOTE_FSM_ENA	BIT(20)
+# define PLL_VOTE_FSM_RESET	BIT(21)
+# define PLL_ACTIVE_FLAG	BIT(30)
+# define PLL_LOCK_DET		BIT(31)
+
+#define PLL_L_VAL		0x04
+#define PLL_ALPHA_VAL		0x08
+#define PLL_ALPHA_VAL_U		0x0c
+
+#define PLL_USER_CTL		0x10
+# define PLL_POST_DIV_SHIFT	8
+# define PLL_POST_DIV_MASK	0xf
+# define PLL_ALPHA_EN		BIT(24)
+# define PLL_VCO_SHIFT		20
+# define PLL_VCO_MASK		0x3
+
+#define PLL_USER_CTL_U		0x14
+
+#define PLL_CONFIG_CTL		0x18
+#define PLL_TEST_CTL		0x1c
+#define PLL_TEST_CTL_U		0x20
+#define PLL_STATUS		0x24
+
+/*
+ * Even though 40 bits are present, use only 32 for ease of calculation.
+ */
+#define ALPHA_REG_BITWIDTH	40
+#define ALPHA_BITWIDTH		32
+
+#define to_clk_alpha_pll(_hw) container_of(to_clk_regmap(_hw), \
+					   struct clk_alpha_pll, clkr)
+
+#define to_clk_alpha_pll_postdiv(_hw) container_of(to_clk_regmap(_hw), \
+					   struct clk_alpha_pll_postdiv, clkr)
+
+static int wait_for_pll(struct clk_alpha_pll *pll)
+{
+	u32 val, mask, off;
+	int count;
+	int ret;
+	const char *name = clk_hw_get_name(&pll->clkr.hw);
+
+	off = pll->offset;
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return ret;
+
+	if (val & PLL_VOTE_FSM_ENA)
+		mask = PLL_ACTIVE_FLAG;
+	else
+		mask = PLL_LOCK_DET;
+
+	/* Wait for pll to enable. */
+	for (count = 100; count > 0; count--) {
+		ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+		if (ret)
+			return ret;
+		if ((val & mask) == mask)
+			return 0;
+
+		udelay(1);
+	}
+
+	WARN(1, "%s didn't enable after voting for it!\n", name);
+	return -ETIMEDOUT;
+}
+
+static int clk_alpha_pll_enable(struct clk_hw *hw)
+{
+	int ret;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val, mask, off;
+
+	off = pll->offset;
+
+	mask = PLL_OUTCTRL | PLL_RESET_N | PLL_BYPASSNL;
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return ret;
+
+	/* If in FSM mode, just vote for it */
+	if (val & PLL_VOTE_FSM_ENA) {
+		ret = clk_enable_regmap(hw);
+		if (ret)
+			return ret;
+		return wait_for_pll(pll);
+	}
+
+	/* Skip if already enabled */
+	if ((val & mask) == mask)
+		return 0;
+
+	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+				 PLL_BYPASSNL, PLL_BYPASSNL);
+	if (ret)
+		return ret;
+
+	/*
+	 * H/W requires a 5us delay between disabling the bypass and
+	 * de-asserting the reset.
+	 */
+	mb();
+	udelay(5);
+
+	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+				 PLL_RESET_N, PLL_RESET_N);
+	if (ret)
+		return ret;
+
+	ret = wait_for_pll(pll);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+				 PLL_OUTCTRL, PLL_OUTCTRL);
+
+	/* Ensure that the write above goes through before returning. */
+	mb();
+	return ret;
+}
+
+static void clk_alpha_pll_disable(struct clk_hw *hw)
+{
+	int ret;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val, mask, off;
+
+	off = pll->offset;
+
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return;
+
+	/* If in FSM mode, just unvote it */
+	if (val & PLL_VOTE_FSM_ENA) {
+		clk_disable_regmap(hw);
+		return;
+	}
+
+	mask = PLL_OUTCTRL;
+	regmap_update_bits(pll->clkr.regmap, off + PLL_MODE, mask, 0);
+
+	/* Delay of 2 output clock ticks required until output is disabled */
+	mb();
+	udelay(1);
+
+	mask = PLL_RESET_N | PLL_BYPASSNL;
+	regmap_update_bits(pll->clkr.regmap, off + PLL_MODE, mask, 0);
+}
+
+static unsigned long alpha_pll_calc_rate(u64 prate, u32 l, u32 a)
+{
+	return (prate * l) + ((prate * a) >> ALPHA_BITWIDTH);
+}
+
+static unsigned long
+alpha_pll_round_rate(unsigned long rate, unsigned long prate, u32 *l, u64 *a)
+{
+	u64 remainder;
+	u64 quotient;
+
+	quotient = rate;
+	remainder = do_div(quotient, prate);
+	*l = quotient;
+
+	if (!remainder) {
+		*a = 0;
+		return rate;
+	}
+
+	/* Upper ALPHA_BITWIDTH bits of Alpha */
+	quotient = remainder << ALPHA_BITWIDTH;
+	remainder = do_div(quotient, prate);
+
+	if (remainder)
+		quotient++;
+
+	*a = quotient;
+	return alpha_pll_calc_rate(prate, *l, *a);
+}
+
+static const struct pll_vco *
+alpha_pll_find_vco(const struct clk_alpha_pll *pll, unsigned long rate)
+{
+	const struct pll_vco *v = pll->vco_table;
+	const struct pll_vco *end = v + pll->num_vco;
+
+	for (; v < end; v++)
+		if (rate >= v->min_freq && rate <= v->max_freq)
+			return v;
+
+	return NULL;
+}
+
+static unsigned long
+clk_alpha_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	u32 l, low, high, ctl;
+	u64 a = 0, prate = parent_rate;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 off = pll->offset;
+
+	regmap_read(pll->clkr.regmap, off + PLL_L_VAL, &l);
+
+	regmap_read(pll->clkr.regmap, off + PLL_USER_CTL, &ctl);
+	if (ctl & PLL_ALPHA_EN) {
+		regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL, &low);
+		regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, &high);
+		a = (u64)high << 32 | low;
+		a >>= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH;
+	}
+
+	return alpha_pll_calc_rate(prate, l, a);
+}
+
+static int clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long prate)
+{
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	const struct pll_vco *vco;
+	u32 l, off = pll->offset;
+	u64 a;
+
+	rate = alpha_pll_round_rate(rate, prate, &l, &a);
+	vco = alpha_pll_find_vco(pll, rate);
+	if (!vco) {
+		pr_err("alpha pll not in a valid vco range\n");
+		return -EINVAL;
+	}
+
+	a <<= (ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH);
+
+	regmap_write(pll->clkr.regmap, off + PLL_L_VAL, l);
+	regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL, a);
+	regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, a >> 32);
+
+	regmap_update_bits(pll->clkr.regmap, off + PLL_USER_CTL,
+			   PLL_VCO_MASK << PLL_VCO_SHIFT,
+			   vco->val << PLL_VCO_SHIFT);
+
+	regmap_update_bits(pll->clkr.regmap, off + PLL_USER_CTL, PLL_ALPHA_EN,
+			   PLL_ALPHA_EN);
+
+	return 0;
+}
+
+static long clk_alpha_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long *prate)
+{
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 l;
+	u64 a;
+	unsigned long min_freq, max_freq;
+
+	rate = alpha_pll_round_rate(rate, *prate, &l, &a);
+	if (alpha_pll_find_vco(pll, rate))
+		return rate;
+
+	min_freq = pll->vco_table[0].min_freq;
+	max_freq = pll->vco_table[pll->num_vco - 1].max_freq;
+
+	return clamp(rate, min_freq, max_freq);
+}
+
+const struct clk_ops clk_alpha_pll_ops = {
+	.enable = clk_alpha_pll_enable,
+	.disable = clk_alpha_pll_disable,
+	.recalc_rate = clk_alpha_pll_recalc_rate,
+	.round_rate = clk_alpha_pll_round_rate,
+	.set_rate = clk_alpha_pll_set_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_ops);
+
+static unsigned long
+clk_alpha_pll_postdiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	u32 ctl;
+
+	regmap_read(pll->clkr.regmap, pll->offset + PLL_USER_CTL, &ctl);
+
+	ctl >>= PLL_POST_DIV_SHIFT;
+	ctl &= PLL_POST_DIV_MASK;
+
+	return parent_rate >> fls(ctl);
+}
+
+static const struct clk_div_table clk_alpha_div_table[] = {
+	{ 0x0, 1 },
+	{ 0x1, 2 },
+	{ 0x3, 4 },
+	{ 0x7, 8 },
+	{ 0xf, 16 },
+	{ }
+};
+
+static long
+clk_alpha_pll_postdiv_round_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long *prate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+
+	return divider_round_rate(hw, rate, prate, clk_alpha_div_table,
+				  pll->width, CLK_DIVIDER_POWER_OF_TWO);
+}
+
+static int clk_alpha_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate,
+					  unsigned long parent_rate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	int div;
+
+	/* 16 -> 0xf, 8 -> 0x7, 4 -> 0x3, 2 -> 0x1, 1 -> 0x0 */
+	div = DIV_ROUND_UP_ULL((u64)parent_rate, rate) - 1;
+
+	return regmap_update_bits(pll->clkr.regmap, pll->offset + PLL_USER_CTL,
+				  PLL_POST_DIV_MASK << PLL_POST_DIV_SHIFT,
+				  div << PLL_POST_DIV_SHIFT);
+}
+
+const struct clk_ops clk_alpha_pll_postdiv_ops = {
+	.recalc_rate = clk_alpha_pll_postdiv_recalc_rate,
+	.round_rate = clk_alpha_pll_postdiv_round_rate,
+	.set_rate = clk_alpha_pll_postdiv_set_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_ops);
diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h
new file mode 100644
index 000000000000..90ce2016e1a0
--- /dev/null
+++ b/drivers/clk/qcom/clk-alpha-pll.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __QCOM_CLK_ALPHA_PLL_H__
+#define __QCOM_CLK_ALPHA_PLL_H__
+
+#include <linux/clk-provider.h>
+#include "clk-regmap.h"
+
+struct pll_vco {
+	unsigned long min_freq;
+	unsigned long max_freq;
+	u32 val;
+};
+
+/**
+ * struct clk_alpha_pll - phase locked loop (PLL)
+ * @offset: base address of registers
+ * @vco_table: array of VCO settings
+ * @clkr: regmap clock handle
+ */
+struct clk_alpha_pll {
+	u32 offset;
+
+	const struct pll_vco *vco_table;
+	size_t num_vco;
+
+	struct clk_regmap clkr;
+};
+
+/**
+ * struct clk_alpha_pll_postdiv - phase locked loop (PLL) post-divider
+ * @offset: base address of registers
+ * @width: width of post-divider
+ * @clkr: regmap clock handle
+ */
+struct clk_alpha_pll_postdiv {
+	u32 offset;
+	u8 width;
+
+	struct clk_regmap clkr;
+};
+
+extern const struct clk_ops clk_alpha_pll_ops;
+extern const struct clk_ops clk_alpha_pll_postdiv_ops;
+
+#endif
diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h
index 4b1e94bdf29e..b904c335cda4 100644
--- a/drivers/clk/qcom/clk-rcg.h
+++ b/drivers/clk/qcom/clk-rcg.h
@@ -178,5 +178,6 @@ extern const struct clk_ops clk_edp_pixel_ops;
 extern const struct clk_ops clk_byte_ops;
 extern const struct clk_ops clk_byte2_ops;
 extern const struct clk_ops clk_pixel_ops;
+extern const struct clk_ops clk_gfx3d_ops;
 
 #endif
diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c
index b544bb302f79..a071bba8018c 100644
--- a/drivers/clk/qcom/clk-rcg2.c
+++ b/drivers/clk/qcom/clk-rcg2.c
@@ -723,3 +723,90 @@ const struct clk_ops clk_pixel_ops = {
 	.determine_rate = clk_pixel_determine_rate,
 };
 EXPORT_SYMBOL_GPL(clk_pixel_ops);
+
+static int clk_gfx3d_determine_rate(struct clk_hw *hw,
+				    struct clk_rate_request *req)
+{
+	struct clk_rate_request parent_req = { };
+	struct clk_hw *p2, *p8, *p9, *xo;
+	unsigned long p9_rate;
+	int ret;
+
+	xo = clk_hw_get_parent_by_index(hw, 0);
+	if (req->rate == clk_hw_get_rate(xo)) {
+		req->best_parent_hw = xo;
+		return 0;
+	}
+
+	p9 = clk_hw_get_parent_by_index(hw, 2);
+	p2 = clk_hw_get_parent_by_index(hw, 3);
+	p8 = clk_hw_get_parent_by_index(hw, 4);
+
+	/* PLL9 is a fixed rate PLL */
+	p9_rate = clk_hw_get_rate(p9);
+
+	parent_req.rate = req->rate = min(req->rate, p9_rate);
+	if (req->rate == p9_rate) {
+		req->rate = req->best_parent_rate = p9_rate;
+		req->best_parent_hw = p9;
+		return 0;
+	}
+
+	if (req->best_parent_hw == p9) {
+		/* Are we going back to a previously used rate? */
+		if (clk_hw_get_rate(p8) == req->rate)
+			req->best_parent_hw = p8;
+		else
+			req->best_parent_hw = p2;
+	} else if (req->best_parent_hw == p8) {
+		req->best_parent_hw = p2;
+	} else {
+		req->best_parent_hw = p8;
+	}
+
+	ret = __clk_determine_rate(req->best_parent_hw, &parent_req);
+	if (ret)
+		return ret;
+
+	req->rate = req->best_parent_rate = parent_req.rate;
+
+	return 0;
+}
+
+static int clk_gfx3d_set_rate_and_parent(struct clk_hw *hw, unsigned long rate,
+		unsigned long parent_rate, u8 index)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	u32 cfg;
+	int ret;
+
+	/* Just mux it, we don't use the division or m/n hardware */
+	cfg = rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT;
+	ret = regmap_write(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, cfg);
+	if (ret)
+		return ret;
+
+	return update_config(rcg);
+}
+
+static int clk_gfx3d_set_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long parent_rate)
+{
+	/*
+	 * We should never get here; clk_gfx3d_determine_rate() should always
+	 * make us use a different parent than what we're currently using, so
+	 * clk_gfx3d_set_rate_and_parent() should always be called.
+	 */
+	return 0;
+}
+
+const struct clk_ops clk_gfx3d_ops = {
+	.is_enabled = clk_rcg2_is_enabled,
+	.get_parent = clk_rcg2_get_parent,
+	.set_parent = clk_rcg2_set_parent,
+	.recalc_rate = clk_rcg2_recalc_rate,
+	.set_rate = clk_gfx3d_set_rate,
+	.set_rate_and_parent = clk_gfx3d_set_rate_and_parent,
+	.determine_rate = clk_gfx3d_determine_rate,
+};
+EXPORT_SYMBOL_GPL(clk_gfx3d_ops);
diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c
index 8fa477293ae0..c112ebaba70d 100644
--- a/drivers/clk/qcom/common.c
+++ b/drivers/clk/qcom/common.c
@@ -17,6 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/clk-provider.h>
 #include <linux/reset-controller.h>
+#include <linux/of.h>
 
 #include "common.h"
 #include "clk-rcg.h"
@@ -88,6 +89,92 @@ static void qcom_cc_gdsc_unregister(void *data)
 	gdsc_unregister(data);
 }
 
+/*
+ * Backwards compatibility with old DTs. Register a pass-through factor 1/1
+ * clock to translate 'path' clk into 'name' clk and regsiter the 'path'
+ * clk as a fixed rate clock if it isn't present.
+ */
+static int _qcom_cc_register_board_clk(struct device *dev, const char *path,
+				       const char *name, unsigned long rate,
+				       bool add_factor)
+{
+	struct device_node *node = NULL;
+	struct device_node *clocks_node;
+	struct clk_fixed_factor *factor;
+	struct clk_fixed_rate *fixed;
+	struct clk *clk;
+	struct clk_init_data init_data = { };
+
+	clocks_node = of_find_node_by_path("/clocks");
+	if (clocks_node)
+		node = of_find_node_by_name(clocks_node, path);
+	of_node_put(clocks_node);
+
+	if (!node) {
+		fixed = devm_kzalloc(dev, sizeof(*fixed), GFP_KERNEL);
+		if (!fixed)
+			return -EINVAL;
+
+		fixed->fixed_rate = rate;
+		fixed->hw.init = &init_data;
+
+		init_data.name = path;
+		init_data.flags = CLK_IS_ROOT;
+		init_data.ops = &clk_fixed_rate_ops;
+
+		clk = devm_clk_register(dev, &fixed->hw);
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+	}
+	of_node_put(node);
+
+	if (add_factor) {
+		factor = devm_kzalloc(dev, sizeof(*factor), GFP_KERNEL);
+		if (!factor)
+			return -EINVAL;
+
+		factor->mult = factor->div = 1;
+		factor->hw.init = &init_data;
+
+		init_data.name = name;
+		init_data.parent_names = &path;
+		init_data.num_parents = 1;
+		init_data.flags = 0;
+		init_data.ops = &clk_fixed_factor_ops;
+
+		clk = devm_clk_register(dev, &factor->hw);
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+	}
+
+	return 0;
+}
+
+int qcom_cc_register_board_clk(struct device *dev, const char *path,
+			       const char *name, unsigned long rate)
+{
+	bool add_factor = true;
+	struct device_node *node;
+
+	/* The RPM clock driver will add the factor clock if present */
+	if (IS_ENABLED(CONFIG_QCOM_RPMCC)) {
+		node = of_find_compatible_node(NULL, NULL, "qcom,rpmcc");
+		if (of_device_is_available(node))
+			add_factor = false;
+		of_node_put(node);
+	}
+
+	return _qcom_cc_register_board_clk(dev, path, name, rate, add_factor);
+}
+EXPORT_SYMBOL_GPL(qcom_cc_register_board_clk);
+
+int qcom_cc_register_sleep_clk(struct device *dev)
+{
+	return _qcom_cc_register_board_clk(dev, "sleep_clk", "sleep_clk_src",
+					   32768, true);
+}
+EXPORT_SYMBOL_GPL(qcom_cc_register_sleep_clk);
+
 int qcom_cc_really_probe(struct platform_device *pdev,
 			 const struct qcom_cc_desc *desc, struct regmap *regmap)
 {
diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h
index 7c1fba3ebc03..ae9bdeb21f29 100644
--- a/drivers/clk/qcom/common.h
+++ b/drivers/clk/qcom/common.h
@@ -37,6 +37,10 @@ extern const struct freq_tbl *qcom_find_freq(const struct freq_tbl *f,
 extern int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map,
 			       u8 src);
 
+extern int qcom_cc_register_board_clk(struct device *dev, const char *path,
+				      const char *name, unsigned long rate);
+extern int qcom_cc_register_sleep_clk(struct device *dev);
+
 extern struct regmap *qcom_cc_map(struct platform_device *pdev,
 				  const struct qcom_cc_desc *desc);
 extern int qcom_cc_really_probe(struct platform_device *pdev,
diff --git a/drivers/clk/qcom/gcc-apq8084.c b/drivers/clk/qcom/gcc-apq8084.c
index 1567c3a79534..cf73e539e9f6 100644
--- a/drivers/clk/qcom/gcc-apq8084.c
+++ b/drivers/clk/qcom/gcc-apq8084.c
@@ -3587,6 +3587,7 @@ static const struct regmap_config gcc_apq8084_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x1fc0,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_apq8084_desc = {
@@ -3607,18 +3608,16 @@ MODULE_DEVICE_TABLE(of, gcc_apq8084_match_table);
 
 static int gcc_apq8084_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
+	int ret;
 	struct device *dev = &pdev->dev;
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "xo", NULL, CLK_IS_ROOT, 19200000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "xo_board", "xo", 19200000);
+	if (ret)
+		return ret;
 
-	clk = clk_register_fixed_rate(dev, "sleep_clk_src", NULL,
-				      CLK_IS_ROOT, 32768);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_sleep_clk(dev);
+	if (ret)
+		return ret;
 
 	return qcom_cc_probe(pdev, &gcc_apq8084_desc);
 }
diff --git a/drivers/clk/qcom/gcc-ipq806x.c b/drivers/clk/qcom/gcc-ipq806x.c
index 16fc64c082a5..b692ae881d6a 100644
--- a/drivers/clk/qcom/gcc-ipq806x.c
+++ b/drivers/clk/qcom/gcc-ipq806x.c
@@ -3005,6 +3005,7 @@ static const struct regmap_config gcc_ipq806x_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x3e40,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_ipq806x_desc = {
@@ -3023,19 +3024,17 @@ MODULE_DEVICE_TABLE(of, gcc_ipq806x_match_table);
 
 static int gcc_ipq806x_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
 	struct device *dev = &pdev->dev;
 	struct regmap *regmap;
 	int ret;
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "cxo", NULL, CLK_IS_ROOT, 25000000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000);
+	if (ret)
+		return ret;
 
-	clk = clk_register_fixed_rate(dev, "pxo", NULL, CLK_IS_ROOT, 25000000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000);
+	if (ret)
+		return ret;
 
 	ret = qcom_cc_probe(pdev, &gcc_ipq806x_desc);
 	if (ret)
diff --git a/drivers/clk/qcom/gcc-msm8660.c b/drivers/clk/qcom/gcc-msm8660.c
index f110bb5a1df3..f6a2b14dfec4 100644
--- a/drivers/clk/qcom/gcc-msm8660.c
+++ b/drivers/clk/qcom/gcc-msm8660.c
@@ -2702,6 +2702,7 @@ static const struct regmap_config gcc_msm8660_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x363c,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_msm8660_desc = {
@@ -2720,17 +2721,16 @@ MODULE_DEVICE_TABLE(of, gcc_msm8660_match_table);
 
 static int gcc_msm8660_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
+	int ret;
 	struct device *dev = &pdev->dev;
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "cxo", NULL, CLK_IS_ROOT, 19200000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000);
+	if (ret)
+		return ret;
 
-	clk = clk_register_fixed_rate(dev, "pxo", NULL, CLK_IS_ROOT, 27000000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000);
+	if (ret)
+		return ret;
 
 	return qcom_cc_probe(pdev, &gcc_msm8660_desc);
 }
diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c
index d0a0313d6bef..e3bf09d7d0ef 100644
--- a/drivers/clk/qcom/gcc-msm8916.c
+++ b/drivers/clk/qcom/gcc-msm8916.c
@@ -3336,6 +3336,7 @@ static const struct regmap_config gcc_msm8916_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x80000,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_msm8916_desc = {
@@ -3356,18 +3357,16 @@ MODULE_DEVICE_TABLE(of, gcc_msm8916_match_table);
 
 static int gcc_msm8916_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
+	int ret;
 	struct device *dev = &pdev->dev;
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "xo", NULL, CLK_IS_ROOT, 19200000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "xo_board", "xo", 19200000);
+	if (ret)
+		return ret;
 
-	clk = clk_register_fixed_rate(dev, "sleep_clk_src", NULL,
-				      CLK_IS_ROOT, 32768);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_sleep_clk(dev);
+	if (ret)
+		return ret;
 
 	return qcom_cc_probe(pdev, &gcc_msm8916_desc);
 }
diff --git a/drivers/clk/qcom/gcc-msm8960.c b/drivers/clk/qcom/gcc-msm8960.c
index 66c18bc97857..f31111e32d44 100644
--- a/drivers/clk/qcom/gcc-msm8960.c
+++ b/drivers/clk/qcom/gcc-msm8960.c
@@ -3468,6 +3468,7 @@ static const struct regmap_config gcc_msm8960_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x3660,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct regmap_config gcc_apq8064_regmap_config = {
@@ -3476,6 +3477,7 @@ static const struct regmap_config gcc_apq8064_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x3880,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_msm8960_desc = {
@@ -3503,7 +3505,6 @@ MODULE_DEVICE_TABLE(of, gcc_msm8960_match_table);
 
 static int gcc_msm8960_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
 	struct device *dev = &pdev->dev;
 	const struct of_device_id *match;
 	struct platform_device *tsens;
@@ -3513,14 +3514,13 @@ static int gcc_msm8960_probe(struct platform_device *pdev)
 	if (!match)
 		return -EINVAL;
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "cxo", NULL, CLK_IS_ROOT, 19200000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000);
+	if (ret)
+		return ret;
 
-	clk = clk_register_fixed_rate(dev, "pxo", NULL, CLK_IS_ROOT, 27000000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000);
+	if (ret)
+		return ret;
 
 	ret = qcom_cc_probe(pdev, match->data);
 	if (ret)
diff --git a/drivers/clk/qcom/gcc-msm8974.c b/drivers/clk/qcom/gcc-msm8974.c
index 28abb8f8f293..df164d618e34 100644
--- a/drivers/clk/qcom/gcc-msm8974.c
+++ b/drivers/clk/qcom/gcc-msm8974.c
@@ -2680,6 +2680,7 @@ static const struct regmap_config gcc_msm8974_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x1fc0,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc gcc_msm8974_desc = {
@@ -2717,7 +2718,7 @@ static void msm8974_pro_clock_override(void)
 
 static int gcc_msm8974_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
+	int ret;
 	struct device *dev = &pdev->dev;
 	bool pro;
 	const struct of_device_id *id;
@@ -2730,16 +2731,13 @@ static int gcc_msm8974_probe(struct platform_device *pdev)
 	if (pro)
 		msm8974_pro_clock_override();
 
-	/* Temporary until RPM clocks supported */
-	clk = clk_register_fixed_rate(dev, "xo", NULL, CLK_IS_ROOT, 19200000);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_board_clk(dev, "xo_board", "xo", 19200000);
+	if (ret)
+		return ret;
 
-	/* Should move to DT node? */
-	clk = clk_register_fixed_rate(dev, "sleep_clk_src", NULL,
-				      CLK_IS_ROOT, 32768);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = qcom_cc_register_sleep_clk(dev);
+	if (ret)
+		return ret;
 
 	return qcom_cc_probe(pdev, &gcc_msm8974_desc);
 }
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
new file mode 100644
index 000000000000..16d7c323db49
--- /dev/null
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -0,0 +1,3422 @@
+/*
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-msm8996.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-alpha-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+
+#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
+
+enum {
+	P_XO,
+	P_GPLL0,
+	P_GPLL2,
+	P_GPLL3,
+	P_GPLL1,
+	P_GPLL2_EARLY,
+	P_GPLL0_EARLY_DIV,
+	P_SLEEP_CLK,
+	P_GPLL4,
+	P_AUD_REF_CLK,
+	P_GPLL1_EARLY_DIV
+};
+
+static const struct parent_map gcc_sleep_clk_map[] = {
+	{ P_SLEEP_CLK, 5 }
+};
+
+static const char * const gcc_sleep_clk[] = {
+	"sleep_clk"
+};
+
+static const struct parent_map gcc_xo_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 }
+};
+
+static const char * const gcc_xo_gpll0[] = {
+	"xo",
+	"gpll0"
+};
+
+static const struct parent_map gcc_xo_sleep_clk_map[] = {
+	{ P_XO, 0 },
+	{ P_SLEEP_CLK, 5 }
+};
+
+static const char * const gcc_xo_sleep_clk[] = {
+	"xo",
+	"sleep_clk"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll4_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL4, 5 }
+};
+
+static const char * const gcc_xo_gpll0_gpll4[] = {
+	"xo",
+	"gpll0",
+	"gpll4"
+};
+
+static const struct parent_map gcc_xo_gpll0_aud_ref_clk_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_AUD_REF_CLK, 2 }
+};
+
+static const char * const gcc_xo_gpll0_aud_ref_clk[] = {
+	"xo",
+	"gpll0",
+	"aud_ref_clk"
+};
+
+static const struct parent_map gcc_xo_gpll0_sleep_clk_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_sleep_clk_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"sleep_clk",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll4_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL4, 5 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll4_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll4",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 2 },
+	{ P_GPLL3, 3 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll2",
+	"gpll3",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL1_EARLY_DIV, 3 },
+	{ P_GPLL1, 4 },
+	{ P_GPLL4, 5 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll1_early_div",
+	"gpll1",
+	"gpll4",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 2 },
+	{ P_GPLL3, 3 },
+	{ P_GPLL1, 4 },
+	{ P_GPLL2_EARLY, 5 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll2",
+	"gpll3",
+	"gpll1",
+	"gpll2_early",
+	"gpll0_early_div"
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 2 },
+	{ P_GPLL3, 3 },
+	{ P_GPLL1, 4 },
+	{ P_GPLL4, 5 },
+	{ P_GPLL0_EARLY_DIV, 6 }
+};
+
+static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div[] = {
+	"xo",
+	"gpll0",
+	"gpll2",
+	"gpll3",
+	"gpll1",
+	"gpll4",
+	"gpll0_early_div"
+};
+
+static struct clk_fixed_factor xo = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data){
+		.name = "xo",
+		.parent_names = (const char *[]){ "xo_board" },
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll0_early = {
+	.offset = 0x00000,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll0_early",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor gpll0_early_div = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll0_early_div",
+		.parent_names = (const char *[]){ "gpll0_early" },
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0 = {
+	.offset = 0x00000,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0",
+		.parent_names = (const char *[]){ "gpll0_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4_early = {
+	.offset = 0x77000,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll4_early",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4 = {
+	.offset = 0x77000,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4",
+		.parent_names = (const char *[]){ "gpll4_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_system_noc_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0_EARLY_DIV, 6, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(240000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 system_noc_clk_src = {
+	.cmd_rcgr = 0x0401c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early_div_map,
+	.freq_tbl = ftbl_system_noc_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "system_noc_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_config_noc_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(37500000, P_GPLL0, 16, 0, 0),
+	F(75000000, P_GPLL0, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 config_noc_clk_src = {
+	.cmd_rcgr = 0x0500c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_config_noc_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "config_noc_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_periph_noc_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(37500000, P_GPLL0, 16, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(75000000, P_GPLL0, 8, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 periph_noc_clk_src = {
+	.cmd_rcgr = 0x06014,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_periph_noc_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "periph_noc_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb30_master_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(120000000, P_GPLL0, 5, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb30_master_clk_src = {
+	.cmd_rcgr = 0x0f014,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_early_div_map,
+	.freq_tbl = ftbl_usb30_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb30_master_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_early_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb30_mock_utmi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb30_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x0f028,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_early_div_map,
+	.freq_tbl = ftbl_usb30_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb30_mock_utmi_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_early_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb3_phy_aux_clk_src[] = {
+	F(1200000, P_XO, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb3_phy_aux_clk_src = {
+	.cmd_rcgr = 0x5000c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_sleep_clk_map,
+	.freq_tbl = ftbl_usb3_phy_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb3_phy_aux_clk_src",
+		.parent_names = gcc_xo_sleep_clk,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb20_master_clk_src[] = {
+	F(120000000, P_GPLL0, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb20_master_clk_src = {
+	.cmd_rcgr = 0x12010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_early_div_map,
+	.freq_tbl = ftbl_usb20_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb20_master_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_early_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 usb20_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x12024,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_early_div_map,
+	.freq_tbl = ftbl_usb30_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb20_mock_utmi_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_early_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc1_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 15, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(96000000, P_GPLL4, 4, 0, 0),
+	F(192000000, P_GPLL4, 2, 0, 0),
+	F(384000000, P_GPLL4, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x13010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_gpll0_early_div_map,
+	.freq_tbl = ftbl_sdcc1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4_gpll0_early_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x13024,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_gpll0_early_div_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_ice_core_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4_gpll0_early_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc2_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 15, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x14010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_map,
+	.freq_tbl = ftbl_sdcc2_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc3_apps_clk_src = {
+	.cmd_rcgr = 0x15010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_map,
+	.freq_tbl = ftbl_sdcc2_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc4_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 15, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x16010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_sdcc4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup1_spi_apps_clk_src[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(15000000, P_GPLL0, 10, 1, 4),
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1900c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup1_i2c_apps_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x19020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_uart1_apps_clk_src[] = {
+	F(3686400, P_GPLL0, 1, 96, 15625),
+	F(7372800, P_GPLL0, 1, 192, 15625),
+	F(14745600, P_GPLL0, 1, 384, 15625),
+	F(16000000, P_GPLL0, 5, 2, 15),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0, 5, 1, 5),
+	F(32000000, P_GPLL0, 1, 4, 75),
+	F(40000000, P_GPLL0, 15, 0, 0),
+	F(46400000, P_GPLL0, 1, 29, 375),
+	F(48000000, P_GPLL0, 12.5, 0, 0),
+	F(51200000, P_GPLL0, 1, 32, 375),
+	F(56000000, P_GPLL0, 1, 7, 75),
+	F(58982400, P_GPLL0, 1, 1536, 15625),
+	F(60000000, P_GPLL0, 10, 0, 0),
+	F(63157895, P_GPLL0, 9.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x1a00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1b00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1b020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x1c00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1d00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1d020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x1e00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1f00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1f020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart4_apps_clk_src = {
+	.cmd_rcgr = 0x2000c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2100c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x21020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart5_apps_clk_src = {
+	.cmd_rcgr = 0x2200c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart5_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2300c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x23020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart6_apps_clk_src = {
+	.cmd_rcgr = 0x2400c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart6_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2600c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x26020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x2700c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2800c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x28020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x2900c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2a00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2a020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x2b00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2c00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2c020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart4_apps_clk_src = {
+	.cmd_rcgr = 0x2d00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2e00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup5_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2e020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart5_apps_clk_src = {
+	.cmd_rcgr = 0x2f00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart5_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x3000c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup6_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x30020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart6_apps_clk_src = {
+	.cmd_rcgr = 0x3100c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart6_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pdm2_clk_src[] = {
+	F(60000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_pdm2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pdm2_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_tsif_ref_clk_src[] = {
+	F(105495, P_XO, 1, 1, 182),
+	{ }
+};
+
+static struct clk_rcg2 tsif_ref_clk_src = {
+	.cmd_rcgr = 0x36010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_aud_ref_clk_map,
+	.freq_tbl = ftbl_tsif_ref_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "tsif_ref_clk_src",
+		.parent_names = gcc_xo_gpll0_aud_ref_clk,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_sleep_clk_src = {
+	.cmd_rcgr = 0x43014,
+	.hid_width = 5,
+	.parent_map = gcc_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_sleep_clk_src",
+		.parent_names = gcc_sleep_clk,
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 hmss_rbcpr_clk_src = {
+	.cmd_rcgr = 0x48040,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_usb30_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "hmss_rbcpr_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 hmss_gpll0_clk_src = {
+	.cmd_rcgr = 0x48058,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "hmss_gpll0_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gp1_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_gpll0_early_div_map,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp1_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk_gpll0_early_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_gpll0_early_div_map,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp2_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk_gpll0_early_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_gpll0_early_div_map,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp3_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk_gpll0_early_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pcie_aux_clk_src[] = {
+	F(1010526, P_XO, 1, 1, 19),
+	{ }
+};
+
+static struct clk_rcg2 pcie_aux_clk_src = {
+	.cmd_rcgr = 0x6c000,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_sleep_clk_map,
+	.freq_tbl = ftbl_pcie_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie_aux_clk_src",
+		.parent_names = gcc_xo_sleep_clk,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_ufs_axi_clk_src[] = {
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(240000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ufs_axi_clk_src = {
+	.cmd_rcgr = 0x75024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_ufs_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ufs_axi_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 ufs_ice_core_clk_src = {
+	.cmd_rcgr = 0x76014,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ufs_ice_core_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 qspi_ser_clk_src = {
+	.cmd_rcgr = 0x8b00c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "qspi_ser_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div,
+		.num_parents = 6,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb3_axi_clk = {
+	.halt_reg = 0x0f03c,
+	.clkr = {
+		.enable_reg = 0x0f03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_usb3_axi_clk",
+			.parent_names = (const char *[]){ "usb30_master_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_ufs_axi_clk = {
+	.halt_reg = 0x75038,
+	.clkr = {
+		.enable_reg = 0x75038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_ufs_axi_clk",
+			.parent_names = (const char *[]){ "ufs_axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_periph_noc_usb20_ahb_clk = {
+	.halt_reg = 0x6010,
+	.clkr = {
+		.enable_reg = 0x6010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_periph_noc_usb20_ahb_clk",
+			.parent_names = (const char *[]){ "usb20_master_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = {
+	.halt_reg = 0x9008,
+	.clkr = {
+		.enable_reg = 0x9008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_noc_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_bimc_gfx_clk = {
+	.halt_reg = 0x9010,
+	.clkr = {
+		.enable_reg = 0x9010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_bimc_gfx_clk",
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_ROOT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_master_clk = {
+	.halt_reg = 0x0f008,
+	.clkr = {
+		.enable_reg = 0x0f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_master_clk",
+			.parent_names = (const char *[]){ "usb30_master_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_sleep_clk = {
+	.halt_reg = 0x0f00c,
+	.clkr = {
+		.enable_reg = 0x0f00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_sleep_clk",
+			.parent_names = (const char *[]){ "gcc_sleep_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_mock_utmi_clk = {
+	.halt_reg = 0x0f010,
+	.clkr = {
+		.enable_reg = 0x0f010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_mock_utmi_clk",
+			.parent_names = (const char *[]){ "usb30_mock_utmi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_phy_aux_clk = {
+	.halt_reg = 0x50000,
+	.clkr = {
+		.enable_reg = 0x50000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_phy_aux_clk",
+			.parent_names = (const char *[]){ "usb3_phy_aux_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_phy_pipe_clk = {
+	.halt_reg = 0x50004,
+	.clkr = {
+		.enable_reg = 0x50004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_phy_pipe_clk",
+			.parent_names = (const char *[]){ "usb3_phy_pipe_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_master_clk = {
+	.halt_reg = 0x12004,
+	.clkr = {
+		.enable_reg = 0x12004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb20_master_clk",
+			.parent_names = (const char *[]){ "usb20_master_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_sleep_clk = {
+	.halt_reg = 0x12008,
+	.clkr = {
+		.enable_reg = 0x12008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb20_sleep_clk",
+			.parent_names = (const char *[]){ "gcc_sleep_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_mock_utmi_clk = {
+	.halt_reg = 0x1200c,
+	.clkr = {
+		.enable_reg = 0x1200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb20_mock_utmi_clk",
+			.parent_names = (const char *[]){ "usb20_mock_utmi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = {
+	.halt_reg = 0x6a004,
+	.clkr = {
+		.enable_reg = 0x6a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_phy_cfg_ahb2phy_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x13004,
+	.clkr = {
+		.enable_reg = 0x13004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_names = (const char *[]){ "sdcc1_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x13008,
+	.clkr = {
+		.enable_reg = 0x13008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x13038,
+	.clkr = {
+		.enable_reg = 0x13038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_names = (const char *[]){ "sdcc1_ice_core_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_names = (const char *[]){ "sdcc2_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14008,
+	.clkr = {
+		.enable_reg = 0x14008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc3_apps_clk = {
+	.halt_reg = 0x15004,
+	.clkr = {
+		.enable_reg = 0x15004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc3_apps_clk",
+			.parent_names = (const char *[]){ "sdcc3_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc3_ahb_clk = {
+	.halt_reg = 0x15008,
+	.clkr = {
+		.enable_reg = 0x15008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc3_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x16004,
+	.clkr = {
+		.enable_reg = 0x16004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_names = (const char *[]){ "sdcc4_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x16008,
+	.clkr = {
+		.enable_reg = 0x16008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x17004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_sleep_clk = {
+	.halt_reg = 0x17008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_sleep_clk",
+			.parent_names = (const char *[]){ "gcc_sleep_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x19004,
+	.clkr = {
+		.enable_reg = 0x19004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup1_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x19008,
+	.clkr = {
+		.enable_reg = 0x19008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup1_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x1a004,
+	.clkr = {
+		.enable_reg = 0x1a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart1_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x1b004,
+	.clkr = {
+		.enable_reg = 0x1b004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup2_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x1b008,
+	.clkr = {
+		.enable_reg = 0x1b008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup2_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x1c004,
+	.clkr = {
+		.enable_reg = 0x1c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart2_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x1d004,
+	.clkr = {
+		.enable_reg = 0x1d004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup3_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x1d008,
+	.clkr = {
+		.enable_reg = 0x1d008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup3_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart3_apps_clk = {
+	.halt_reg = 0x1e004,
+	.clkr = {
+		.enable_reg = 0x1e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart3_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart3_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x1f004,
+	.clkr = {
+		.enable_reg = 0x1f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup4_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x1f008,
+	.clkr = {
+		.enable_reg = 0x1f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup4_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart4_apps_clk = {
+	.halt_reg = 0x20004,
+	.clkr = {
+		.enable_reg = 0x20004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart4_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart4_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
+	.halt_reg = 0x21004,
+	.clkr = {
+		.enable_reg = 0x21004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup5_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
+	.halt_reg = 0x21008,
+	.clkr = {
+		.enable_reg = 0x21008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup5_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart5_apps_clk = {
+	.halt_reg = 0x22004,
+	.clkr = {
+		.enable_reg = 0x22004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart5_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart5_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
+	.halt_reg = 0x23004,
+	.clkr = {
+		.enable_reg = 0x23004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup6_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
+	.halt_reg = 0x23008,
+	.clkr = {
+		.enable_reg = 0x23008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_qup6_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart6_apps_clk = {
+	.halt_reg = 0x24004,
+	.clkr = {
+		.enable_reg = 0x24004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart6_apps_clk",
+			.parent_names = (const char *[]){ "blsp1_uart6_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_ahb_clk = {
+	.halt_reg = 0x25004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_sleep_clk = {
+	.halt_reg = 0x25008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(14),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_sleep_clk",
+			.parent_names = (const char *[]){ "gcc_sleep_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = {
+	.halt_reg = 0x26004,
+	.clkr = {
+		.enable_reg = 0x26004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup1_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = {
+	.halt_reg = 0x26008,
+	.clkr = {
+		.enable_reg = 0x26008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup1_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart1_apps_clk = {
+	.halt_reg = 0x27004,
+	.clkr = {
+		.enable_reg = 0x27004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart1_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart1_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = {
+	.halt_reg = 0x28004,
+	.clkr = {
+		.enable_reg = 0x28004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup2_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = {
+	.halt_reg = 0x28008,
+	.clkr = {
+		.enable_reg = 0x28008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup2_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart2_apps_clk = {
+	.halt_reg = 0x29004,
+	.clkr = {
+		.enable_reg = 0x29004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart2_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart2_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = {
+	.halt_reg = 0x2a004,
+	.clkr = {
+		.enable_reg = 0x2a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup3_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = {
+	.halt_reg = 0x2a008,
+	.clkr = {
+		.enable_reg = 0x2a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup3_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart3_apps_clk = {
+	.halt_reg = 0x2b004,
+	.clkr = {
+		.enable_reg = 0x2b004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart3_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart3_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = {
+	.halt_reg = 0x2c004,
+	.clkr = {
+		.enable_reg = 0x2c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup4_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = {
+	.halt_reg = 0x2c008,
+	.clkr = {
+		.enable_reg = 0x2c008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup4_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart4_apps_clk = {
+	.halt_reg = 0x2d004,
+	.clkr = {
+		.enable_reg = 0x2d004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart4_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart4_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_spi_apps_clk = {
+	.halt_reg = 0x2e004,
+	.clkr = {
+		.enable_reg = 0x2e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup5_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup5_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_i2c_apps_clk = {
+	.halt_reg = 0x2e008,
+	.clkr = {
+		.enable_reg = 0x2e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup5_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart5_apps_clk = {
+	.halt_reg = 0x2f004,
+	.clkr = {
+		.enable_reg = 0x2f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart5_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart5_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_spi_apps_clk = {
+	.halt_reg = 0x30004,
+	.clkr = {
+		.enable_reg = 0x30004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup6_spi_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup6_spi_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_i2c_apps_clk = {
+	.halt_reg = 0x30008,
+	.clkr = {
+		.enable_reg = 0x30008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_qup6_i2c_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart6_apps_clk = {
+	.halt_reg = 0x31004,
+	.clkr = {
+		.enable_reg = 0x31004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart6_apps_clk",
+			.parent_names = (const char *[]){ "blsp2_uart6_apps_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm2_clk",
+			.parent_names = (const char *[]){ "pdm2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x34004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ahb_clk = {
+	.halt_reg = 0x36004,
+	.clkr = {
+		.enable_reg = 0x36004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ref_clk = {
+	.halt_reg = 0x36008,
+	.clkr = {
+		.enable_reg = 0x36008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ref_clk",
+			.parent_names = (const char *[]){ "tsif_ref_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_inactivity_timers_clk = {
+	.halt_reg = 0x3600c,
+	.clkr = {
+		.enable_reg = 0x3600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_inactivity_timers_clk",
+			.parent_names = (const char *[]){ "gcc_sleep_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_boot_rom_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_gfx_clk = {
+	.halt_reg = 0x46018,
+	.clkr = {
+		.enable_reg = 0x46018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_gfx_clk",
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_ROOT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_rbcpr_clk = {
+	.halt_reg = 0x4800c,
+	.clkr = {
+		.enable_reg = 0x4800c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_rbcpr_clk",
+			.parent_names = (const char *[]){ "hmss_rbcpr_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp1_clk",
+			.parent_names = (const char *[]){ "gp1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_names = (const char *[]){ "gp2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_names = (const char *[]){ "gp3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b008,
+	.clkr = {
+		.enable_reg = 0x6b008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b00c,
+	.clkr = {
+		.enable_reg = 0x6b00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b010,
+	.clkr = {
+		.enable_reg = 0x6b010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b014,
+	.clkr = {
+		.enable_reg = 0x6b014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_names = (const char *[]){ "pcie_aux_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x6b018,
+	.clkr = {
+		.enable_reg = 0x6b018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_pipe_clk",
+			.parent_names = (const char *[]){ "pcie_0_pipe_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_axi_clk = {
+	.halt_reg = 0x6d008,
+	.clkr = {
+		.enable_reg = 0x6d008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_slv_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_mstr_axi_clk = {
+	.halt_reg = 0x6d00c,
+	.clkr = {
+		.enable_reg = 0x6d00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_mstr_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_cfg_ahb_clk = {
+	.halt_reg = 0x6d010,
+	.clkr = {
+		.enable_reg = 0x6d010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_aux_clk = {
+	.halt_reg = 0x6d014,
+	.clkr = {
+		.enable_reg = 0x6d014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_aux_clk",
+			.parent_names = (const char *[]){ "pcie_aux_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_clk = {
+	.halt_reg = 0x6d018,
+	.clkr = {
+		.enable_reg = 0x6d018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_pipe_clk",
+			.parent_names = (const char *[]){ "pcie_1_pipe_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_2_slv_axi_clk = {
+	.halt_reg = 0x6e008,
+	.clkr = {
+		.enable_reg = 0x6e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_2_slv_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_2_mstr_axi_clk = {
+	.halt_reg = 0x6e00c,
+	.clkr = {
+		.enable_reg = 0x6e00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_2_mstr_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_2_cfg_ahb_clk = {
+	.halt_reg = 0x6e010,
+	.clkr = {
+		.enable_reg = 0x6e010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_2_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_2_aux_clk = {
+	.halt_reg = 0x6e014,
+	.clkr = {
+		.enable_reg = 0x6e014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_2_aux_clk",
+			.parent_names = (const char *[]){ "pcie_aux_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_2_pipe_clk = {
+	.halt_reg = 0x6e108,
+	.clkr = {
+		.enable_reg = 0x6e108,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_2_pipe_clk",
+			.parent_names = (const char *[]){ "pcie_2_pipe_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_phy_cfg_ahb_clk = {
+	.halt_reg = 0x6f004,
+	.clkr = {
+		.enable_reg = 0x6f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_phy_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_phy_aux_clk = {
+	.halt_reg = 0x6f008,
+	.clkr = {
+		.enable_reg = 0x6f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_phy_aux_clk",
+			.parent_names = (const char *[]){ "pcie_aux_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_axi_clk = {
+	.halt_reg = 0x75008,
+	.clkr = {
+		.enable_reg = 0x75008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_axi_clk",
+			.parent_names = (const char *[]){ "ufs_axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_ahb_clk = {
+	.halt_reg = 0x7500c,
+	.clkr = {
+		.enable_reg = 0x7500c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor ufs_tx_cfg_clk_src = {
+	.mult = 1,
+	.div = 16,
+	.hw.init = &(struct clk_init_data){
+		.name = "ufs_tx_cfg_clk_src",
+		.parent_names = (const char *[]){ "ufs_axi_clk_src" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_branch gcc_ufs_tx_cfg_clk = {
+	.halt_reg = 0x75010,
+	.clkr = {
+		.enable_reg = 0x75010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_tx_cfg_clk",
+			.parent_names = (const char *[]){ "ufs_tx_cfg_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor ufs_rx_cfg_clk_src = {
+	.mult = 1,
+	.div = 16,
+	.hw.init = &(struct clk_init_data){
+		.name = "ufs_rx_cfg_clk_src",
+		.parent_names = (const char *[]){ "ufs_axi_clk_src" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_cfg_clk = {
+	.halt_reg = 0x75014,
+	.clkr = {
+		.enable_reg = 0x75014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_rx_cfg_clk",
+			.parent_names = (const char *[]){ "ufs_rx_cfg_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_tx_symbol_0_clk = {
+	.halt_reg = 0x75018,
+	.clkr = {
+		.enable_reg = 0x75018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_tx_symbol_0_clk",
+			.parent_names = (const char *[]){ "ufs_tx_symbol_0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_symbol_0_clk = {
+	.halt_reg = 0x7501c,
+	.clkr = {
+		.enable_reg = 0x7501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_rx_symbol_0_clk",
+			.parent_names = (const char *[]){ "ufs_rx_symbol_0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_symbol_1_clk = {
+	.halt_reg = 0x75020,
+	.clkr = {
+		.enable_reg = 0x75020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_rx_symbol_1_clk",
+			.parent_names = (const char *[]){ "ufs_rx_symbol_1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor ufs_ice_core_postdiv_clk_src = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "ufs_ice_core_postdiv_clk_src",
+		.parent_names = (const char *[]){ "ufs_ice_core_clk_src" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_branch gcc_ufs_unipro_core_clk = {
+	.halt_reg = 0x7600c,
+	.clkr = {
+		.enable_reg = 0x7600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_unipro_core_clk",
+			.parent_names = (const char *[]){ "ufs_ice_core_postdiv_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_ice_core_clk = {
+	.halt_reg = 0x76010,
+	.clkr = {
+		.enable_reg = 0x76010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_ice_core_clk",
+			.parent_names = (const char *[]){ "ufs_ice_core_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_sys_clk_core_clk = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x76030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_sys_clk_core_clk",
+			.ops = &clk_branch2_ops,
+			.flags = CLK_IS_ROOT,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_tx_symbol_clk_core_clk = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x76034,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_tx_symbol_clk_core_clk",
+			.ops = &clk_branch2_ops,
+			.flags = CLK_IS_ROOT,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre0_snoc_axi_clk = {
+	.halt_reg = 0x81008,
+	.clkr = {
+		.enable_reg = 0x81008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre0_snoc_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre0_cnoc_ahb_clk = {
+	.halt_reg = 0x8100c,
+	.clkr = {
+		.enable_reg = 0x8100c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre0_cnoc_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_smmu_aggre0_axi_clk = {
+	.halt_reg = 0x81014,
+	.clkr = {
+		.enable_reg = 0x81014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_smmu_aggre0_axi_clk",
+			.parent_names = (const char *[]){ "system_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_smmu_aggre0_ahb_clk = {
+	.halt_reg = 0x81018,
+	.clkr = {
+		.enable_reg = 0x81018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_smmu_aggre0_ahb_clk",
+			.parent_names = (const char *[]){ "config_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre1_pnoc_ahb_clk = {
+	.halt_reg = 0x82014,
+	.clkr = {
+		.enable_reg = 0x82014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre1_pnoc_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre2_ufs_axi_clk = {
+	.halt_reg = 0x83014,
+	.clkr = {
+		.enable_reg = 0x83014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre2_ufs_axi_clk",
+			.parent_names = (const char *[]){ "ufs_axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre2_usb3_axi_clk = {
+	.halt_reg = 0x83018,
+	.clkr = {
+		.enable_reg = 0x83018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre2_usb3_axi_clk",
+			.parent_names = (const char *[]){ "usb30_master_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qspi_ahb_clk = {
+	.halt_reg = 0x8b004,
+	.clkr = {
+		.enable_reg = 0x8b004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qspi_ahb_clk",
+			.parent_names = (const char *[]){ "periph_noc_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qspi_ser_clk = {
+	.halt_reg = 0x8b008,
+	.clkr = {
+		.enable_reg = 0x8b008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qspi_ser_clk",
+			.parent_names = (const char *[]){ "qspi_ser_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_clkref_clk = {
+	.halt_reg = 0x8800C,
+	.clkr = {
+		.enable_reg = 0x8800C,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hdmi_clkref_clk = {
+	.halt_reg = 0x88000,
+	.clkr = {
+		.enable_reg = 0x88000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hdmi_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_clkref_clk = {
+	.halt_reg = 0x88008,
+	.clkr = {
+		.enable_reg = 0x88008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_clkref_clk = {
+	.halt_reg = 0x88010,
+	.clkr = {
+		.enable_reg = 0x88010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_rx2_usb2_clkref_clk = {
+	.halt_reg = 0x88014,
+	.clkr = {
+		.enable_reg = 0x88014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_rx2_usb2_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_rx1_usb2_clkref_clk = {
+	.halt_reg = 0x88018,
+	.clkr = {
+		.enable_reg = 0x88018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_rx1_usb2_clkref_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_hw *gcc_msm8996_hws[] = {
+	&xo.hw,
+	&gpll0_early_div.hw,
+	&ufs_tx_cfg_clk_src.hw,
+	&ufs_rx_cfg_clk_src.hw,
+	&ufs_ice_core_postdiv_clk_src.hw,
+};
+
+static struct clk_regmap *gcc_msm8996_clocks[] = {
+	[GPLL0_EARLY] = &gpll0_early.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL4_EARLY] = &gpll4_early.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[SYSTEM_NOC_CLK_SRC] = &system_noc_clk_src.clkr,
+	[CONFIG_NOC_CLK_SRC] = &config_noc_clk_src.clkr,
+	[PERIPH_NOC_CLK_SRC] = &periph_noc_clk_src.clkr,
+	[USB30_MASTER_CLK_SRC] = &usb30_master_clk_src.clkr,
+	[USB30_MOCK_UTMI_CLK_SRC] = &usb30_mock_utmi_clk_src.clkr,
+	[USB3_PHY_AUX_CLK_SRC] = &usb3_phy_aux_clk_src.clkr,
+	[USB20_MASTER_CLK_SRC] = &usb20_master_clk_src.clkr,
+	[USB20_MOCK_UTMI_CLK_SRC] = &usb20_mock_utmi_clk_src.clkr,
+	[SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr,
+	[SDCC1_ICE_CORE_CLK_SRC] = &sdcc1_ice_core_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[SDCC3_APPS_CLK_SRC] = &sdcc3_apps_clk_src.clkr,
+	[SDCC4_APPS_CLK_SRC] = &sdcc4_apps_clk_src.clkr,
+	[BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr,
+	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
+	[BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr,
+	[BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr,
+	[BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr,
+	[BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr,
+	[BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr,
+	[BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr,
+	[BLSP1_UART3_APPS_CLK_SRC] = &blsp1_uart3_apps_clk_src.clkr,
+	[BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr,
+	[BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr,
+	[BLSP1_UART4_APPS_CLK_SRC] = &blsp1_uart4_apps_clk_src.clkr,
+	[BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr,
+	[BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr,
+	[BLSP1_UART5_APPS_CLK_SRC] = &blsp1_uart5_apps_clk_src.clkr,
+	[BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr,
+	[BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr,
+	[BLSP1_UART6_APPS_CLK_SRC] = &blsp1_uart6_apps_clk_src.clkr,
+	[BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr,
+	[BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr,
+	[BLSP2_UART1_APPS_CLK_SRC] = &blsp2_uart1_apps_clk_src.clkr,
+	[BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr,
+	[BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr,
+	[BLSP2_UART2_APPS_CLK_SRC] = &blsp2_uart2_apps_clk_src.clkr,
+	[BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr,
+	[BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr,
+	[BLSP2_UART3_APPS_CLK_SRC] = &blsp2_uart3_apps_clk_src.clkr,
+	[BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr,
+	[BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr,
+	[BLSP2_UART4_APPS_CLK_SRC] = &blsp2_uart4_apps_clk_src.clkr,
+	[BLSP2_QUP5_SPI_APPS_CLK_SRC] = &blsp2_qup5_spi_apps_clk_src.clkr,
+	[BLSP2_QUP5_I2C_APPS_CLK_SRC] = &blsp2_qup5_i2c_apps_clk_src.clkr,
+	[BLSP2_UART5_APPS_CLK_SRC] = &blsp2_uart5_apps_clk_src.clkr,
+	[BLSP2_QUP6_SPI_APPS_CLK_SRC] = &blsp2_qup6_spi_apps_clk_src.clkr,
+	[BLSP2_QUP6_I2C_APPS_CLK_SRC] = &blsp2_qup6_i2c_apps_clk_src.clkr,
+	[BLSP2_UART6_APPS_CLK_SRC] = &blsp2_uart6_apps_clk_src.clkr,
+	[PDM2_CLK_SRC] = &pdm2_clk_src.clkr,
+	[TSIF_REF_CLK_SRC] = &tsif_ref_clk_src.clkr,
+	[GCC_SLEEP_CLK_SRC] = &gcc_sleep_clk_src.clkr,
+	[HMSS_RBCPR_CLK_SRC] = &hmss_rbcpr_clk_src.clkr,
+	[HMSS_GPLL0_CLK_SRC] = &hmss_gpll0_clk_src.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
+	[PCIE_AUX_CLK_SRC] = &pcie_aux_clk_src.clkr,
+	[UFS_AXI_CLK_SRC] = &ufs_axi_clk_src.clkr,
+	[UFS_ICE_CORE_CLK_SRC] = &ufs_ice_core_clk_src.clkr,
+	[QSPI_SER_CLK_SRC] = &qspi_ser_clk_src.clkr,
+	[GCC_SYS_NOC_USB3_AXI_CLK] = &gcc_sys_noc_usb3_axi_clk.clkr,
+	[GCC_SYS_NOC_UFS_AXI_CLK] = &gcc_sys_noc_ufs_axi_clk.clkr,
+	[GCC_PERIPH_NOC_USB20_AHB_CLK] = &gcc_periph_noc_usb20_ahb_clk.clkr,
+	[GCC_MMSS_NOC_CFG_AHB_CLK] = &gcc_mmss_noc_cfg_ahb_clk.clkr,
+	[GCC_MMSS_BIMC_GFX_CLK] = &gcc_mmss_bimc_gfx_clk.clkr,
+	[GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr,
+	[GCC_USB30_SLEEP_CLK] = &gcc_usb30_sleep_clk.clkr,
+	[GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr,
+	[GCC_USB3_PHY_AUX_CLK] = &gcc_usb3_phy_aux_clk.clkr,
+	[GCC_USB3_PHY_PIPE_CLK] = &gcc_usb3_phy_pipe_clk.clkr,
+	[GCC_USB20_MASTER_CLK] = &gcc_usb20_master_clk.clkr,
+	[GCC_USB20_SLEEP_CLK] = &gcc_usb20_sleep_clk.clkr,
+	[GCC_USB20_MOCK_UTMI_CLK] = &gcc_usb20_mock_utmi_clk.clkr,
+	[GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC3_APPS_CLK] = &gcc_sdcc3_apps_clk.clkr,
+	[GCC_SDCC3_AHB_CLK] = &gcc_sdcc3_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
+	[GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr,
+	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART4_APPS_CLK] = &gcc_blsp1_uart4_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART5_APPS_CLK] = &gcc_blsp1_uart5_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP1_UART6_APPS_CLK] = &gcc_blsp1_uart6_apps_clk.clkr,
+	[GCC_BLSP2_AHB_CLK] = &gcc_blsp2_ahb_clk.clkr,
+	[GCC_BLSP2_SLEEP_CLK] = &gcc_blsp2_sleep_clk.clkr,
+	[GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART1_APPS_CLK] = &gcc_blsp2_uart1_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART2_APPS_CLK] = &gcc_blsp2_uart2_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART3_APPS_CLK] = &gcc_blsp2_uart3_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART4_APPS_CLK] = &gcc_blsp2_uart4_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_SPI_APPS_CLK] = &gcc_blsp2_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_I2C_APPS_CLK] = &gcc_blsp2_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART5_APPS_CLK] = &gcc_blsp2_uart5_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_SPI_APPS_CLK] = &gcc_blsp2_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_I2C_APPS_CLK] = &gcc_blsp2_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP2_UART6_APPS_CLK] = &gcc_blsp2_uart6_apps_clk.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
+	[GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr,
+	[GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr,
+	[GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr,
+	[GCC_HMSS_RBCPR_CLK] = &gcc_hmss_rbcpr_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr,
+	[GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr,
+	[GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr,
+	[GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr,
+	[GCC_PCIE_2_SLV_AXI_CLK] = &gcc_pcie_2_slv_axi_clk.clkr,
+	[GCC_PCIE_2_MSTR_AXI_CLK] = &gcc_pcie_2_mstr_axi_clk.clkr,
+	[GCC_PCIE_2_CFG_AHB_CLK] = &gcc_pcie_2_cfg_ahb_clk.clkr,
+	[GCC_PCIE_2_AUX_CLK] = &gcc_pcie_2_aux_clk.clkr,
+	[GCC_PCIE_2_PIPE_CLK] = &gcc_pcie_2_pipe_clk.clkr,
+	[GCC_PCIE_PHY_CFG_AHB_CLK] = &gcc_pcie_phy_cfg_ahb_clk.clkr,
+	[GCC_PCIE_PHY_AUX_CLK] = &gcc_pcie_phy_aux_clk.clkr,
+	[GCC_UFS_AXI_CLK] = &gcc_ufs_axi_clk.clkr,
+	[GCC_UFS_AHB_CLK] = &gcc_ufs_ahb_clk.clkr,
+	[GCC_UFS_TX_CFG_CLK] = &gcc_ufs_tx_cfg_clk.clkr,
+	[GCC_UFS_RX_CFG_CLK] = &gcc_ufs_rx_cfg_clk.clkr,
+	[GCC_UFS_TX_SYMBOL_0_CLK] = &gcc_ufs_tx_symbol_0_clk.clkr,
+	[GCC_UFS_RX_SYMBOL_0_CLK] = &gcc_ufs_rx_symbol_0_clk.clkr,
+	[GCC_UFS_RX_SYMBOL_1_CLK] = &gcc_ufs_rx_symbol_1_clk.clkr,
+	[GCC_UFS_UNIPRO_CORE_CLK] = &gcc_ufs_unipro_core_clk.clkr,
+	[GCC_UFS_ICE_CORE_CLK] = &gcc_ufs_ice_core_clk.clkr,
+	[GCC_UFS_SYS_CLK_CORE_CLK] = &gcc_ufs_sys_clk_core_clk.clkr,
+	[GCC_UFS_TX_SYMBOL_CLK_CORE_CLK] = &gcc_ufs_tx_symbol_clk_core_clk.clkr,
+	[GCC_AGGRE0_SNOC_AXI_CLK] = &gcc_aggre0_snoc_axi_clk.clkr,
+	[GCC_AGGRE0_CNOC_AHB_CLK] = &gcc_aggre0_cnoc_ahb_clk.clkr,
+	[GCC_SMMU_AGGRE0_AXI_CLK] = &gcc_smmu_aggre0_axi_clk.clkr,
+	[GCC_SMMU_AGGRE0_AHB_CLK] = &gcc_smmu_aggre0_ahb_clk.clkr,
+	[GCC_AGGRE1_PNOC_AHB_CLK] = &gcc_aggre1_pnoc_ahb_clk.clkr,
+	[GCC_AGGRE2_UFS_AXI_CLK] = &gcc_aggre2_ufs_axi_clk.clkr,
+	[GCC_AGGRE2_USB3_AXI_CLK] = &gcc_aggre2_usb3_axi_clk.clkr,
+	[GCC_QSPI_AHB_CLK] = &gcc_qspi_ahb_clk.clkr,
+	[GCC_QSPI_SER_CLK] = &gcc_qspi_ser_clk.clkr,
+	[GCC_USB3_CLKREF_CLK] = &gcc_usb3_clkref_clk.clkr,
+	[GCC_HDMI_CLKREF_CLK] = &gcc_hdmi_clkref_clk.clkr,
+	[GCC_UFS_CLKREF_CLK] = &gcc_ufs_clkref_clk.clkr,
+	[GCC_PCIE_CLKREF_CLK] = &gcc_pcie_clkref_clk.clkr,
+	[GCC_RX2_USB2_CLKREF_CLK] = &gcc_rx2_usb2_clkref_clk.clkr,
+	[GCC_RX1_USB2_CLKREF_CLK] = &gcc_rx1_usb2_clkref_clk.clkr,
+};
+
+static const struct qcom_reset_map gcc_msm8996_resets[] = {
+	[GCC_SYSTEM_NOC_BCR] = { 0x4000 },
+	[GCC_CONFIG_NOC_BCR] = { 0x5000 },
+	[GCC_PERIPH_NOC_BCR] = { 0x6000 },
+	[GCC_IMEM_BCR] = { 0x8000 },
+	[GCC_MMSS_BCR] = { 0x9000 },
+	[GCC_PIMEM_BCR] = { 0x0a000 },
+	[GCC_QDSS_BCR] = { 0x0c000 },
+	[GCC_USB_30_BCR] = { 0x0f000 },
+	[GCC_USB_20_BCR] = { 0x12000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12038 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x1203c },
+	[GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 },
+	[GCC_SDCC1_BCR] = { 0x13000 },
+	[GCC_SDCC2_BCR] = { 0x14000 },
+	[GCC_SDCC3_BCR] = { 0x15000 },
+	[GCC_SDCC4_BCR] = { 0x16000 },
+	[GCC_BLSP1_BCR] = { 0x17000 },
+	[GCC_BLSP1_QUP1_BCR] = { 0x19000 },
+	[GCC_BLSP1_UART1_BCR] = { 0x1a000 },
+	[GCC_BLSP1_QUP2_BCR] = { 0x1b000 },
+	[GCC_BLSP1_UART2_BCR] = { 0x1c000 },
+	[GCC_BLSP1_QUP3_BCR] = { 0x1d000 },
+	[GCC_BLSP1_UART3_BCR] = { 0x1e000 },
+	[GCC_BLSP1_QUP4_BCR] = { 0x1f000 },
+	[GCC_BLSP1_UART4_BCR] = { 0x20000 },
+	[GCC_BLSP1_QUP5_BCR] = { 0x21000 },
+	[GCC_BLSP1_UART5_BCR] = { 0x22000 },
+	[GCC_BLSP1_QUP6_BCR] = { 0x23000 },
+	[GCC_BLSP1_UART6_BCR] = { 0x24000 },
+	[GCC_BLSP2_BCR] = { 0x25000 },
+	[GCC_BLSP2_QUP1_BCR] = { 0x26000 },
+	[GCC_BLSP2_UART1_BCR] = { 0x27000 },
+	[GCC_BLSP2_QUP2_BCR] = { 0x28000 },
+	[GCC_BLSP2_UART2_BCR] = { 0x29000 },
+	[GCC_BLSP2_QUP3_BCR] = { 0x2a000 },
+	[GCC_BLSP2_UART3_BCR] = { 0x2b000 },
+	[GCC_BLSP2_QUP4_BCR] = { 0x2c000 },
+	[GCC_BLSP2_UART4_BCR] = { 0x2d000 },
+	[GCC_BLSP2_QUP5_BCR] = { 0x2e000 },
+	[GCC_BLSP2_UART5_BCR] = { 0x2f000 },
+	[GCC_BLSP2_QUP6_BCR] = { 0x30000 },
+	[GCC_BLSP2_UART6_BCR] = { 0x31000 },
+	[GCC_PDM_BCR] = { 0x33000 },
+	[GCC_PRNG_BCR] = { 0x34000 },
+	[GCC_TSIF_BCR] = { 0x36000 },
+	[GCC_TCSR_BCR] = { 0x37000 },
+	[GCC_BOOT_ROM_BCR] = { 0x38000 },
+	[GCC_MSG_RAM_BCR] = { 0x39000 },
+	[GCC_TLMM_BCR] = { 0x3a000 },
+	[GCC_MPM_BCR] = { 0x3b000 },
+	[GCC_SEC_CTRL_BCR] = { 0x3d000 },
+	[GCC_SPMI_BCR] = { 0x3f000 },
+	[GCC_SPDM_BCR] = { 0x40000 },
+	[GCC_CE1_BCR] = { 0x41000 },
+	[GCC_BIMC_BCR] = { 0x44000 },
+	[GCC_SNOC_BUS_TIMEOUT0_BCR] = { 0x49000 },
+	[GCC_SNOC_BUS_TIMEOUT2_BCR] = { 0x49008 },
+	[GCC_SNOC_BUS_TIMEOUT1_BCR] = { 0x49010 },
+	[GCC_SNOC_BUS_TIMEOUT3_BCR] = { 0x49018 },
+	[GCC_SNOC_BUS_TIMEOUT_EXTREF_BCR] = { 0x49020 },
+	[GCC_PNOC_BUS_TIMEOUT0_BCR] = { 0x4a000 },
+	[GCC_PNOC_BUS_TIMEOUT1_BCR] = { 0x4a008 },
+	[GCC_PNOC_BUS_TIMEOUT2_BCR] = { 0x4a010 },
+	[GCC_PNOC_BUS_TIMEOUT3_BCR] = { 0x4a018 },
+	[GCC_PNOC_BUS_TIMEOUT4_BCR] = { 0x4a020 },
+	[GCC_CNOC_BUS_TIMEOUT0_BCR] = { 0x4b000 },
+	[GCC_CNOC_BUS_TIMEOUT1_BCR] = { 0x4b008 },
+	[GCC_CNOC_BUS_TIMEOUT2_BCR] = { 0x4b010 },
+	[GCC_CNOC_BUS_TIMEOUT3_BCR] = { 0x4b018 },
+	[GCC_CNOC_BUS_TIMEOUT4_BCR] = { 0x4b020 },
+	[GCC_CNOC_BUS_TIMEOUT5_BCR] = { 0x4b028 },
+	[GCC_CNOC_BUS_TIMEOUT6_BCR] = { 0x4b030 },
+	[GCC_CNOC_BUS_TIMEOUT7_BCR] = { 0x4b038 },
+	[GCC_CNOC_BUS_TIMEOUT8_BCR] = { 0x80000 },
+	[GCC_CNOC_BUS_TIMEOUT9_BCR] = { 0x80008 },
+	[GCC_CNOC_BUS_TIMEOUT_EXTREF_BCR] = { 0x80010 },
+	[GCC_APB2JTAG_BCR] = { 0x4c000 },
+	[GCC_RBCPR_CX_BCR] = { 0x4e000 },
+	[GCC_RBCPR_MX_BCR] = { 0x4f000 },
+	[GCC_PCIE_0_BCR] = { 0x6b000 },
+	[GCC_PCIE_0_PHY_BCR] = { 0x6c01c },
+	[GCC_PCIE_1_BCR] = { 0x6d000 },
+	[GCC_PCIE_1_PHY_BCR] = { 0x6d038 },
+	[GCC_PCIE_2_BCR] = { 0x6e000 },
+	[GCC_PCIE_2_PHY_BCR] = { 0x6e038 },
+	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_DCD_BCR] = { 0x70000 },
+	[GCC_OBT_ODT_BCR] = { 0x73000 },
+	[GCC_UFS_BCR] = { 0x75000 },
+	[GCC_SSC_BCR] = { 0x63000 },
+	[GCC_VS_BCR] = { 0x7a000 },
+	[GCC_AGGRE0_NOC_BCR] = { 0x81000 },
+	[GCC_AGGRE1_NOC_BCR] = { 0x82000 },
+	[GCC_AGGRE2_NOC_BCR] = { 0x83000 },
+	[GCC_DCC_BCR] = { 0x84000 },
+	[GCC_IPA_BCR] = { 0x89000 },
+	[GCC_QSPI_BCR] = { 0x8b000 },
+	[GCC_SKL_BCR] = { 0x8c000 },
+	[GCC_MSMPU_BCR] = { 0x8d000 },
+	[GCC_MSS_Q6_BCR] = { 0x8e000 },
+	[GCC_QREFS_VBG_CAL_BCR] = { 0x88020 },
+};
+
+static const struct regmap_config gcc_msm8996_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x8f010,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_msm8996_desc = {
+	.config = &gcc_msm8996_regmap_config,
+	.clks = gcc_msm8996_clocks,
+	.num_clks = ARRAY_SIZE(gcc_msm8996_clocks),
+	.resets = gcc_msm8996_resets,
+	.num_resets = ARRAY_SIZE(gcc_msm8996_resets),
+};
+
+static const struct of_device_id gcc_msm8996_match_table[] = {
+	{ .compatible = "qcom,gcc-msm8996" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_msm8996_match_table);
+
+static int gcc_msm8996_probe(struct platform_device *pdev)
+{
+	struct clk *clk;
+	struct device *dev = &pdev->dev;
+	int i;
+	struct regmap *regmap;
+
+	regmap = qcom_cc_map(pdev, &gcc_msm8996_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/*
+	 * Set the HMSS_AHB_CLK_SLEEP_ENA bit to allow the hmss_ahb_clk to be
+	 * turned off by hardware during certain apps low power modes.
+	 */
+	regmap_update_bits(regmap, 0x52008, BIT(21), BIT(21));
+
+	for (i = 0; i < ARRAY_SIZE(gcc_msm8996_hws); i++) {
+		clk = devm_clk_register(dev, gcc_msm8996_hws[i]);
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+	}
+
+	return qcom_cc_really_probe(pdev, &gcc_msm8996_desc, regmap);
+}
+
+static struct platform_driver gcc_msm8996_driver = {
+	.probe		= gcc_msm8996_probe,
+	.driver		= {
+		.name	= "gcc-msm8996",
+		.of_match_table = gcc_msm8996_match_table,
+	},
+};
+
+static int __init gcc_msm8996_init(void)
+{
+	return platform_driver_register(&gcc_msm8996_driver);
+}
+core_initcall(gcc_msm8996_init);
+
+static void __exit gcc_msm8996_exit(void)
+{
+	platform_driver_unregister(&gcc_msm8996_driver);
+}
+module_exit(gcc_msm8996_exit);
+
+MODULE_DESCRIPTION("QCOM GCC MSM8996 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gcc-msm8996");
diff --git a/drivers/clk/qcom/lcc-ipq806x.c b/drivers/clk/qcom/lcc-ipq806x.c
index db3998e5e2d8..62e79fadd5f7 100644
--- a/drivers/clk/qcom/lcc-ipq806x.c
+++ b/drivers/clk/qcom/lcc-ipq806x.c
@@ -419,6 +419,7 @@ static const struct regmap_config lcc_ipq806x_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0xfc,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc lcc_ipq806x_desc = {
diff --git a/drivers/clk/qcom/lcc-msm8960.c b/drivers/clk/qcom/lcc-msm8960.c
index 4fcf9d1d233c..bf95bb0ea1b8 100644
--- a/drivers/clk/qcom/lcc-msm8960.c
+++ b/drivers/clk/qcom/lcc-msm8960.c
@@ -524,6 +524,7 @@ static const struct regmap_config lcc_msm8960_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0xfc,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc lcc_msm8960_desc = {
diff --git a/drivers/clk/qcom/mmcc-apq8084.c b/drivers/clk/qcom/mmcc-apq8084.c
index 30777f9f1a43..1e703fda8a0f 100644
--- a/drivers/clk/qcom/mmcc-apq8084.c
+++ b/drivers/clk/qcom/mmcc-apq8084.c
@@ -3368,6 +3368,7 @@ static const struct regmap_config mmcc_apq8084_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x5104,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc mmcc_apq8084_desc = {
diff --git a/drivers/clk/qcom/mmcc-msm8960.c b/drivers/clk/qcom/mmcc-msm8960.c
index 00e36192a1de..d73a048d3b9d 100644
--- a/drivers/clk/qcom/mmcc-msm8960.c
+++ b/drivers/clk/qcom/mmcc-msm8960.c
@@ -3029,6 +3029,7 @@ static const struct regmap_config mmcc_msm8960_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x334,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct regmap_config mmcc_apq8064_regmap_config = {
@@ -3037,6 +3038,7 @@ static const struct regmap_config mmcc_apq8064_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x350,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc mmcc_msm8960_desc = {
diff --git a/drivers/clk/qcom/mmcc-msm8974.c b/drivers/clk/qcom/mmcc-msm8974.c
index 9d790bcadf25..bbe28ed93669 100644
--- a/drivers/clk/qcom/mmcc-msm8974.c
+++ b/drivers/clk/qcom/mmcc-msm8974.c
@@ -2594,6 +2594,7 @@ static const struct regmap_config mmcc_msm8974_regmap_config = {
 	.val_bits	= 32,
 	.max_register	= 0x5104,
 	.fast_io	= true,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
 static const struct qcom_cc_desc mmcc_msm8974_desc = {
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
new file mode 100644
index 000000000000..064f3eaa39d0
--- /dev/null
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -0,0 +1,3217 @@
+/*x
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+#include <linux/clk.h>
+
+#include <dt-bindings/clock/qcom,mmcc-msm8996.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-alpha-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+
+#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
+
+enum {
+	P_XO,
+	P_MMPLL0,
+	P_GPLL0,
+	P_GPLL0_DIV,
+	P_MMPLL1,
+	P_MMPLL9,
+	P_MMPLL2,
+	P_MMPLL8,
+	P_MMPLL3,
+	P_DSI0PLL,
+	P_DSI1PLL,
+	P_MMPLL5,
+	P_HDMIPLL,
+	P_DSI0PLL_BYTE,
+	P_DSI1PLL_BYTE,
+	P_MMPLL4,
+};
+
+static const struct parent_map mmss_xo_hdmi_map[] = {
+	{ P_XO, 0 },
+	{ P_HDMIPLL, 1 }
+};
+
+static const char * const mmss_xo_hdmi[] = {
+	"xo",
+	"hdmipll"
+};
+
+static const struct parent_map mmss_xo_dsi0pll_dsi1pll_map[] = {
+	{ P_XO, 0 },
+	{ P_DSI0PLL, 1 },
+	{ P_DSI1PLL, 2 }
+};
+
+static const char * const mmss_xo_dsi0pll_dsi1pll[] = {
+	"xo",
+	"dsi0pll",
+	"dsi1pll"
+};
+
+static const struct parent_map mmss_xo_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_gpll0_gpll0_div[] = {
+	"xo",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_dsibyte_map[] = {
+	{ P_XO, 0 },
+	{ P_DSI0PLL_BYTE, 1 },
+	{ P_DSI1PLL_BYTE, 2 }
+};
+
+static const char * const mmss_xo_dsibyte[] = {
+	"xo",
+	"dsi0pllbyte",
+	"dsi1pllbyte"
+};
+
+static const struct parent_map mmss_xo_mmpll0_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL1, 2 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll1",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL3, 3 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll3",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL5, 2 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll5",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL4, 3 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll4",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL9, 2 },
+	{ P_MMPLL2, 3 },
+	{ P_MMPLL8, 4 },
+	{ P_GPLL0, 5 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0[] = {
+	"xo",
+	"mmpll0",
+	"mmpll9",
+	"mmpll2",
+	"mmpll8",
+	"gpll0"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL9, 2 },
+	{ P_MMPLL2, 3 },
+	{ P_MMPLL8, 4 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll9",
+	"mmpll2",
+	"mmpll8",
+	"gpll0",
+	"gpll0_div"
+};
+
+static const struct parent_map mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map[] = {
+	{ P_XO, 0 },
+	{ P_MMPLL0, 1 },
+	{ P_MMPLL1, 2 },
+	{ P_MMPLL4, 3 },
+	{ P_MMPLL3, 4 },
+	{ P_GPLL0, 5 },
+	{ P_GPLL0_DIV, 6 }
+};
+
+static const char * const mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div[] = {
+	"xo",
+	"mmpll0",
+	"mmpll1",
+	"mmpll4",
+	"mmpll3",
+	"gpll0",
+	"gpll0_div"
+};
+
+static struct clk_fixed_factor gpll0_div = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll0_div",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct pll_vco mmpll_p_vco[] = {
+	{ 250000000, 500000000, 3 },
+	{ 500000000, 1000000000, 2 },
+	{ 1000000000, 1500000000, 1 },
+	{ 1500000000, 2000000000, 0 },
+};
+
+static struct pll_vco mmpll_gfx_vco[] = {
+	{ 400000000, 1000000000, 2 },
+	{ 1000000000, 1500000000, 1 },
+	{ 1500000000, 2000000000, 0 },
+};
+
+static struct pll_vco mmpll_t_vco[] = {
+	{ 500000000, 1500000000, 0 },
+};
+
+static struct clk_alpha_pll mmpll0_early = {
+	.offset = 0x0,
+	.vco_table = mmpll_p_vco,
+	.num_vco = ARRAY_SIZE(mmpll_p_vco),
+	.clkr = {
+		.enable_reg = 0x100,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmpll0_early",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll0 = {
+	.offset = 0x0,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll0",
+		.parent_names = (const char *[]){ "mmpll0_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll1_early = {
+	.offset = 0x30,
+	.vco_table = mmpll_p_vco,
+	.num_vco = ARRAY_SIZE(mmpll_p_vco),
+	.clkr = {
+		.enable_reg = 0x100,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmpll1_early",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll1 = {
+	.offset = 0x30,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll1",
+		.parent_names = (const char *[]){ "mmpll1_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll2_early = {
+	.offset = 0x4100,
+	.vco_table = mmpll_gfx_vco,
+	.num_vco = ARRAY_SIZE(mmpll_gfx_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll2_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll2 = {
+	.offset = 0x4100,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll2",
+		.parent_names = (const char *[]){ "mmpll2_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll3_early = {
+	.offset = 0x60,
+	.vco_table = mmpll_p_vco,
+	.num_vco = ARRAY_SIZE(mmpll_p_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll3_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll3 = {
+	.offset = 0x60,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll3",
+		.parent_names = (const char *[]){ "mmpll3_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll4_early = {
+	.offset = 0x90,
+	.vco_table = mmpll_t_vco,
+	.num_vco = ARRAY_SIZE(mmpll_t_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll4_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll4 = {
+	.offset = 0x90,
+	.width = 2,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll4",
+		.parent_names = (const char *[]){ "mmpll4_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll5_early = {
+	.offset = 0xc0,
+	.vco_table = mmpll_p_vco,
+	.num_vco = ARRAY_SIZE(mmpll_p_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll5_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll5 = {
+	.offset = 0xc0,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll5",
+		.parent_names = (const char *[]){ "mmpll5_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll8_early = {
+	.offset = 0x4130,
+	.vco_table = mmpll_gfx_vco,
+	.num_vco = ARRAY_SIZE(mmpll_gfx_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll8_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll8 = {
+	.offset = 0x4130,
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll8",
+		.parent_names = (const char *[]){ "mmpll8_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll mmpll9_early = {
+	.offset = 0x4200,
+	.vco_table = mmpll_t_vco,
+	.num_vco = ARRAY_SIZE(mmpll_t_vco),
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll9_early",
+		.parent_names = (const char *[]){ "xo" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv mmpll9 = {
+	.offset = 0x4200,
+	.width = 2,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mmpll9",
+		.parent_names = (const char *[]){ "mmpll9_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_ahb_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(40000000, P_GPLL0_DIV, 7.5, 0, 0),
+	F(80000000, P_MMPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ahb_clk_src = {
+	.cmd_rcgr = 0x5000,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_ahb_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ahb_clk_src",
+		.parent_names = mmss_xo_mmpll0_gpll0_gpll0_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_axi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(75000000, P_GPLL0_DIV, 4, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(171430000, P_GPLL0, 3.5, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(320000000, P_MMPLL0, 2.5, 0, 0),
+	F(400000000, P_MMPLL0, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 axi_clk_src = {
+	.cmd_rcgr = 0x5040,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "axi_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 maxi_clk_src = {
+	.cmd_rcgr = 0x5090,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "maxi_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gfx3d_clk_src = {
+	.cmd_rcgr = 0x4000,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gfx3d_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0,
+		.num_parents = 6,
+		.ops = &clk_gfx3d_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_rbbmtimer_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 rbbmtimer_clk_src = {
+	.cmd_rcgr = 0x4090,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_rbbmtimer_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "rbbmtimer_clk_src",
+		.parent_names = mmss_xo_mmpll0_gpll0_gpll0_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 isense_clk_src = {
+	.cmd_rcgr = 0x4010,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_gpll0_div_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "isense_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_rbcpr_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 rbcpr_clk_src = {
+	.cmd_rcgr = 0x4060,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "rbcpr_clk_src",
+		.parent_names = mmss_xo_mmpll0_gpll0_gpll0_div,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_core_clk_src[] = {
+	F(75000000, P_GPLL0_DIV, 4, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(346666667, P_MMPLL3, 3, 0, 0),
+	F(520000000, P_MMPLL3, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_core_clk_src = {
+	.cmd_rcgr = 0x1000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_video_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "video_core_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 video_subcore0_clk_src = {
+	.cmd_rcgr = 0x1060,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_video_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "video_subcore0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 video_subcore1_clk_src = {
+	.cmd_rcgr = 0x1080,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_video_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "video_subcore1_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll3_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 pclk0_clk_src = {
+	.cmd_rcgr = 0x2000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsi0pll_dsi1pll_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pclk0_clk_src",
+		.parent_names = mmss_xo_dsi0pll_dsi1pll,
+		.num_parents = 3,
+		.ops = &clk_pixel_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_rcg2 pclk1_clk_src = {
+	.cmd_rcgr = 0x2020,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsi0pll_dsi1pll_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pclk1_clk_src",
+		.parent_names = mmss_xo_dsi0pll_dsi1pll,
+		.num_parents = 3,
+		.ops = &clk_pixel_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_mdp_clk_src[] = {
+	F(85714286, P_GPLL0, 7, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(171428571, P_GPLL0, 3.5, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(275000000, P_MMPLL5, 3, 0, 0),
+	F(300000000, P_GPLL0, 2, 0, 0),
+	F(330000000, P_MMPLL5, 2.5, 0, 0),
+	F(412500000, P_MMPLL5, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 mdp_clk_src = {
+	.cmd_rcgr = 0x2040,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mdp_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mdp_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl extpclk_freq_tbl[] = {
+	{ .src = P_HDMIPLL },
+	{ }
+};
+
+static struct clk_rcg2 extpclk_clk_src = {
+	.cmd_rcgr = 0x2060,
+	.hid_width = 5,
+	.parent_map = mmss_xo_hdmi_map,
+	.freq_tbl = extpclk_freq_tbl,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "extpclk_clk_src",
+		.parent_names = mmss_xo_hdmi,
+		.num_parents = 2,
+		.ops = &clk_byte_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct freq_tbl ftbl_mdss_vsync_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vsync_clk_src = {
+	.cmd_rcgr = 0x2080,
+	.hid_width = 5,
+	.parent_map = mmss_xo_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mdss_vsync_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vsync_clk_src",
+		.parent_names = mmss_xo_gpll0_gpll0_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_mdss_hdmi_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 hdmi_clk_src = {
+	.cmd_rcgr = 0x2100,
+	.hid_width = 5,
+	.parent_map = mmss_xo_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mdss_hdmi_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "hdmi_clk_src",
+		.parent_names = mmss_xo_gpll0_gpll0_div,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 byte0_clk_src = {
+	.cmd_rcgr = 0x2120,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsibyte_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "byte0_clk_src",
+		.parent_names = mmss_xo_dsibyte,
+		.num_parents = 3,
+		.ops = &clk_byte2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_rcg2 byte1_clk_src = {
+	.cmd_rcgr = 0x2140,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsibyte_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "byte1_clk_src",
+		.parent_names = mmss_xo_dsibyte,
+		.num_parents = 3,
+		.ops = &clk_byte2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct freq_tbl ftbl_mdss_esc0_1_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 esc0_clk_src = {
+	.cmd_rcgr = 0x2160,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsibyte_map,
+	.freq_tbl = ftbl_mdss_esc0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "esc0_clk_src",
+		.parent_names = mmss_xo_dsibyte,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 esc1_clk_src = {
+	.cmd_rcgr = 0x2180,
+	.hid_width = 5,
+	.parent_map = mmss_xo_dsibyte_map,
+	.freq_tbl = ftbl_mdss_esc0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "esc1_clk_src",
+		.parent_names = mmss_xo_dsibyte,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_camss_gp0_clk_src[] = {
+	F(10000, P_XO, 16, 1, 120),
+	F(24000, P_XO, 16, 1, 50),
+	F(6000000, P_GPLL0_DIV, 10, 1, 5),
+	F(12000000, P_GPLL0_DIV, 1, 1, 25),
+	F(13000000, P_GPLL0_DIV, 2, 13, 150),
+	F(24000000, P_GPLL0_DIV, 1, 2, 25),
+	{ }
+};
+
+static struct clk_rcg2 camss_gp0_clk_src = {
+	.cmd_rcgr = 0x3420,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_camss_gp0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 camss_gp1_clk_src = {
+	.cmd_rcgr = 0x3450,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_camss_gp0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp1_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_mclk0_clk_src[] = {
+	F(4800000, P_XO, 4, 0, 0),
+	F(6000000, P_GPLL0_DIV, 10, 1, 5),
+	F(8000000, P_GPLL0_DIV, 1, 2, 75),
+	F(9600000, P_XO, 2, 0, 0),
+	F(16666667, P_GPLL0_DIV, 2, 1, 9),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0_DIV, 1, 2, 25),
+	F(33333333, P_GPLL0_DIV, 1, 1, 9),
+	F(48000000, P_GPLL0, 1, 2, 25),
+	F(66666667, P_GPLL0, 1, 1, 9),
+	{ }
+};
+
+static struct clk_rcg2 mclk0_clk_src = {
+	.cmd_rcgr = 0x3360,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mclk0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk1_clk_src = {
+	.cmd_rcgr = 0x3390,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mclk0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk1_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk2_clk_src = {
+	.cmd_rcgr = 0x33c0,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mclk0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk2_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk3_clk_src = {
+	.cmd_rcgr = 0x33f0,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_mclk0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk3_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cci_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(37500000, P_GPLL0, 16, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cci_clk_src = {
+	.cmd_rcgr = 0x3300,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_cci_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cci_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_csi0phytimer_clk_src[] = {
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(266666667, P_MMPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x3000,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0phytimer_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0phytimer_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x3030,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0phytimer_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1phytimer_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi2phytimer_clk_src = {
+	.cmd_rcgr = 0x3060,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0phytimer_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi2phytimer_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_csiphy0_3p_clk_src[] = {
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(320000000, P_MMPLL4, 3, 0, 0),
+	F(384000000, P_MMPLL4, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 csiphy0_3p_clk_src = {
+	.cmd_rcgr = 0x3240,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csiphy0_3p_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csiphy0_3p_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csiphy1_3p_clk_src = {
+	.cmd_rcgr = 0x3260,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csiphy0_3p_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csiphy1_3p_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csiphy2_3p_clk_src = {
+	.cmd_rcgr = 0x3280,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csiphy0_3p_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csiphy2_3p_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_jpeg0_clk_src[] = {
+	F(75000000, P_GPLL0_DIV, 4, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(228571429, P_MMPLL0, 3.5, 0, 0),
+	F(266666667, P_MMPLL0, 3, 0, 0),
+	F(320000000, P_MMPLL0, 2.5, 0, 0),
+	F(480000000, P_MMPLL4, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 jpeg0_clk_src = {
+	.cmd_rcgr = 0x3500,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_jpeg0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "jpeg0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_jpeg2_clk_src[] = {
+	F(75000000, P_GPLL0_DIV, 4, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(228571429, P_MMPLL0, 3.5, 0, 0),
+	F(266666667, P_MMPLL0, 3, 0, 0),
+	F(320000000, P_MMPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 jpeg2_clk_src = {
+	.cmd_rcgr = 0x3540,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_jpeg2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "jpeg2_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 jpeg_dma_clk_src = {
+	.cmd_rcgr = 0x3560,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_jpeg0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "jpeg_dma_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_vfe0_clk_src[] = {
+	F(75000000, P_GPLL0_DIV, 4, 0, 0),
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(300000000, P_GPLL0, 2, 0, 0),
+	F(320000000, P_MMPLL0, 2.5, 0, 0),
+	F(480000000, P_MMPLL4, 2, 0, 0),
+	F(600000000, P_GPLL0, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vfe0_clk_src = {
+	.cmd_rcgr = 0x3600,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_vfe0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vfe0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 vfe1_clk_src = {
+	.cmd_rcgr = 0x3620,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_vfe0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vfe1_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cpp_clk_src[] = {
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(320000000, P_MMPLL0, 2.5, 0, 0),
+	F(480000000, P_MMPLL4, 2, 0, 0),
+	F(640000000, P_MMPLL4, 1.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cpp_clk_src = {
+	.cmd_rcgr = 0x3640,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_cpp_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cpp_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_csi0_clk_src[] = {
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(266666667, P_MMPLL0, 3, 0, 0),
+	F(480000000, P_MMPLL4, 2, 0, 0),
+	F(600000000, P_GPLL0, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 csi0_clk_src = {
+	.cmd_rcgr = 0x3090,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1_clk_src = {
+	.cmd_rcgr = 0x3100,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi2_clk_src = {
+	.cmd_rcgr = 0x3160,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi2_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi3_clk_src = {
+	.cmd_rcgr = 0x31c0,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_csi0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi3_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll1_mmpll4_mmpll3_gpll0_gpll0_div,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_fd_core_clk_src[] = {
+	F(100000000, P_GPLL0_DIV, 3, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(400000000, P_MMPLL0, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 fd_core_clk_src = {
+	.cmd_rcgr = 0x3b00,
+	.hid_width = 5,
+	.parent_map = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div_map,
+	.freq_tbl = ftbl_fd_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "fd_core_clk_src",
+		.parent_names = mmss_xo_mmpll0_mmpll4_gpll0_gpll0_div,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch mmss_mmagic_ahb_clk = {
+	.halt_reg = 0x5024,
+	.clkr = {
+		.enable_reg = 0x5024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_mmagic_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_mmagic_cfg_ahb_clk = {
+	.halt_reg = 0x5054,
+	.clkr = {
+		.enable_reg = 0x5054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_mmagic_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_misc_ahb_clk = {
+	.halt_reg = 0x5018,
+	.clkr = {
+		.enable_reg = 0x5018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_misc_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_misc_cxo_clk = {
+	.halt_reg = 0x5014,
+	.clkr = {
+		.enable_reg = 0x5014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_misc_cxo_clk",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_mmagic_axi_clk = {
+	.halt_reg = 0x506c,
+	.clkr = {
+		.enable_reg = 0x506c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_mmagic_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_mmagic_maxi_clk = {
+	.halt_reg = 0x5074,
+	.clkr = {
+		.enable_reg = 0x5074,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_mmagic_maxi_clk",
+			.parent_names = (const char *[]){ "maxi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_camss_axi_clk = {
+	.halt_reg = 0x3c44,
+	.clkr = {
+		.enable_reg = 0x3c44,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_camss_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_camss_noc_cfg_ahb_clk = {
+	.halt_reg = 0x3c48,
+	.clkr = {
+		.enable_reg = 0x3c48,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_camss_noc_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "gcc_mmss_noc_cfg_ahb_clk" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_vfe_ahb_clk = {
+	.halt_reg = 0x3c04,
+	.clkr = {
+		.enable_reg = 0x3c04,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_vfe_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_vfe_axi_clk = {
+	.halt_reg = 0x3c08,
+	.clkr = {
+		.enable_reg = 0x3c08,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_vfe_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_cpp_ahb_clk = {
+	.halt_reg = 0x3c14,
+	.clkr = {
+		.enable_reg = 0x3c14,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_cpp_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_cpp_axi_clk = {
+	.halt_reg = 0x3c18,
+	.clkr = {
+		.enable_reg = 0x3c18,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_cpp_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_jpeg_ahb_clk = {
+	.halt_reg = 0x3c24,
+	.clkr = {
+		.enable_reg = 0x3c24,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_jpeg_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_jpeg_axi_clk = {
+	.halt_reg = 0x3c28,
+	.clkr = {
+		.enable_reg = 0x3c28,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_jpeg_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_mdss_axi_clk = {
+	.halt_reg = 0x2474,
+	.clkr = {
+		.enable_reg = 0x2474,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_mdss_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_mdss_noc_cfg_ahb_clk = {
+	.halt_reg = 0x2478,
+	.clkr = {
+		.enable_reg = 0x2478,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_mdss_noc_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "gcc_mmss_noc_cfg_ahb_clk" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_rot_ahb_clk = {
+	.halt_reg = 0x2444,
+	.clkr = {
+		.enable_reg = 0x2444,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_rot_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_rot_axi_clk = {
+	.halt_reg = 0x2448,
+	.clkr = {
+		.enable_reg = 0x2448,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_rot_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_mdp_ahb_clk = {
+	.halt_reg = 0x2454,
+	.clkr = {
+		.enable_reg = 0x2454,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_mdp_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_mdp_axi_clk = {
+	.halt_reg = 0x2458,
+	.clkr = {
+		.enable_reg = 0x2458,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_mdp_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_video_axi_clk = {
+	.halt_reg = 0x1194,
+	.clkr = {
+		.enable_reg = 0x1194,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_video_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_video_noc_cfg_ahb_clk = {
+	.halt_reg = 0x1198,
+	.clkr = {
+		.enable_reg = 0x1198,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_video_noc_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "gcc_mmss_noc_cfg_ahb_clk" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_video_ahb_clk = {
+	.halt_reg = 0x1174,
+	.clkr = {
+		.enable_reg = 0x1174,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_video_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch smmu_video_axi_clk = {
+	.halt_reg = 0x1178,
+	.clkr = {
+		.enable_reg = 0x1178,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "smmu_video_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_bimc_axi_clk = {
+	.halt_reg = 0x5294,
+	.clkr = {
+		.enable_reg = 0x5294,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_bimc_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmagic_bimc_noc_cfg_ahb_clk = {
+	.halt_reg = 0x5298,
+	.clkr = {
+		.enable_reg = 0x5298,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmagic_bimc_noc_cfg_ahb_clk",
+			.parent_names = (const char *[]){ "gcc_mmss_noc_cfg_ahb_clk" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_gx_gfx3d_clk = {
+	.halt_reg = 0x4028,
+	.clkr = {
+		.enable_reg = 0x4028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpu_gx_gfx3d_clk",
+			.parent_names = (const char *[]){ "gfx3d_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_gx_rbbmtimer_clk = {
+	.halt_reg = 0x40b0,
+	.clkr = {
+		.enable_reg = 0x40b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpu_gx_rbbmtimer_clk",
+			.parent_names = (const char *[]){ "rbbmtimer_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_ahb_clk = {
+	.halt_reg = 0x403c,
+	.clkr = {
+		.enable_reg = 0x403c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpu_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_aon_isense_clk = {
+	.halt_reg = 0x4044,
+	.clkr = {
+		.enable_reg = 0x4044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpu_aon_isense_clk",
+			.parent_names = (const char *[]){ "isense_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch vmem_maxi_clk = {
+	.halt_reg = 0x1204,
+	.clkr = {
+		.enable_reg = 0x1204,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "vmem_maxi_clk",
+			.parent_names = (const char *[]){ "maxi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch vmem_ahb_clk = {
+	.halt_reg = 0x1208,
+	.clkr = {
+		.enable_reg = 0x1208,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "vmem_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_rbcpr_clk = {
+	.halt_reg = 0x4084,
+	.clkr = {
+		.enable_reg = 0x4084,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_rbcpr_clk",
+			.parent_names = (const char *[]){ "rbcpr_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mmss_rbcpr_ahb_clk = {
+	.halt_reg = 0x4088,
+	.clkr = {
+		.enable_reg = 0x4088,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mmss_rbcpr_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_core_clk = {
+	.halt_reg = 0x1028,
+	.clkr = {
+		.enable_reg = 0x1028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_core_clk",
+			.parent_names = (const char *[]){ "video_core_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_axi_clk = {
+	.halt_reg = 0x1034,
+	.clkr = {
+		.enable_reg = 0x1034,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_maxi_clk = {
+	.halt_reg = 0x1038,
+	.clkr = {
+		.enable_reg = 0x1038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_maxi_clk",
+			.parent_names = (const char *[]){ "maxi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_ahb_clk = {
+	.halt_reg = 0x1030,
+	.clkr = {
+		.enable_reg = 0x1030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_subcore0_clk = {
+	.halt_reg = 0x1048,
+	.clkr = {
+		.enable_reg = 0x1048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_subcore0_clk",
+			.parent_names = (const char *[]){ "video_subcore0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_subcore1_clk = {
+	.halt_reg = 0x104c,
+	.clkr = {
+		.enable_reg = 0x104c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "video_subcore1_clk",
+			.parent_names = (const char *[]){ "video_subcore1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_ahb_clk = {
+	.halt_reg = 0x2308,
+	.clkr = {
+		.enable_reg = 0x2308,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_hdmi_ahb_clk = {
+	.halt_reg = 0x230c,
+	.clkr = {
+		.enable_reg = 0x230c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_hdmi_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_axi_clk = {
+	.halt_reg = 0x2310,
+	.clkr = {
+		.enable_reg = 0x2310,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_pclk0_clk = {
+	.halt_reg = 0x2314,
+	.clkr = {
+		.enable_reg = 0x2314,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_pclk0_clk",
+			.parent_names = (const char *[]){ "pclk0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_pclk1_clk = {
+	.halt_reg = 0x2318,
+	.clkr = {
+		.enable_reg = 0x2318,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_pclk1_clk",
+			.parent_names = (const char *[]){ "pclk1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_mdp_clk = {
+	.halt_reg = 0x231c,
+	.clkr = {
+		.enable_reg = 0x231c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_mdp_clk",
+			.parent_names = (const char *[]){ "mdp_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_extpclk_clk = {
+	.halt_reg = 0x2324,
+	.clkr = {
+		.enable_reg = 0x2324,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_extpclk_clk",
+			.parent_names = (const char *[]){ "extpclk_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_vsync_clk = {
+	.halt_reg = 0x2328,
+	.clkr = {
+		.enable_reg = 0x2328,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_vsync_clk",
+			.parent_names = (const char *[]){ "vsync_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_hdmi_clk = {
+	.halt_reg = 0x2338,
+	.clkr = {
+		.enable_reg = 0x2338,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_hdmi_clk",
+			.parent_names = (const char *[]){ "hdmi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_byte0_clk = {
+	.halt_reg = 0x233c,
+	.clkr = {
+		.enable_reg = 0x233c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_byte0_clk",
+			.parent_names = (const char *[]){ "byte0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_byte1_clk = {
+	.halt_reg = 0x2340,
+	.clkr = {
+		.enable_reg = 0x2340,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_byte1_clk",
+			.parent_names = (const char *[]){ "byte1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_esc0_clk = {
+	.halt_reg = 0x2344,
+	.clkr = {
+		.enable_reg = 0x2344,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_esc0_clk",
+			.parent_names = (const char *[]){ "esc0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch mdss_esc1_clk = {
+	.halt_reg = 0x2348,
+	.clkr = {
+		.enable_reg = 0x2348,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "mdss_esc1_clk",
+			.parent_names = (const char *[]){ "esc1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_top_ahb_clk = {
+	.halt_reg = 0x3484,
+	.clkr = {
+		.enable_reg = 0x3484,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_top_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_ahb_clk = {
+	.halt_reg = 0x348c,
+	.clkr = {
+		.enable_reg = 0x348c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_micro_ahb_clk = {
+	.halt_reg = 0x3494,
+	.clkr = {
+		.enable_reg = 0x3494,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_micro_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_gp0_clk = {
+	.halt_reg = 0x3444,
+	.clkr = {
+		.enable_reg = 0x3444,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_gp0_clk",
+			.parent_names = (const char *[]){ "camss_gp0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_gp1_clk = {
+	.halt_reg = 0x3474,
+	.clkr = {
+		.enable_reg = 0x3474,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_gp1_clk",
+			.parent_names = (const char *[]){ "camss_gp1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_mclk0_clk = {
+	.halt_reg = 0x3384,
+	.clkr = {
+		.enable_reg = 0x3384,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_mclk0_clk",
+			.parent_names = (const char *[]){ "mclk0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_mclk1_clk = {
+	.halt_reg = 0x33b4,
+	.clkr = {
+		.enable_reg = 0x33b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_mclk1_clk",
+			.parent_names = (const char *[]){ "mclk1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_mclk2_clk = {
+	.halt_reg = 0x33e4,
+	.clkr = {
+		.enable_reg = 0x33e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_mclk2_clk",
+			.parent_names = (const char *[]){ "mclk2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_mclk3_clk = {
+	.halt_reg = 0x3414,
+	.clkr = {
+		.enable_reg = 0x3414,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_mclk3_clk",
+			.parent_names = (const char *[]){ "mclk3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cci_clk = {
+	.halt_reg = 0x3344,
+	.clkr = {
+		.enable_reg = 0x3344,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cci_clk",
+			.parent_names = (const char *[]){ "cci_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cci_ahb_clk = {
+	.halt_reg = 0x3348,
+	.clkr = {
+		.enable_reg = 0x3348,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cci_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0phytimer_clk = {
+	.halt_reg = 0x3024,
+	.clkr = {
+		.enable_reg = 0x3024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0phytimer_clk",
+			.parent_names = (const char *[]){ "csi0phytimer_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1phytimer_clk = {
+	.halt_reg = 0x3054,
+	.clkr = {
+		.enable_reg = 0x3054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1phytimer_clk",
+			.parent_names = (const char *[]){ "csi1phytimer_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2phytimer_clk = {
+	.halt_reg = 0x3084,
+	.clkr = {
+		.enable_reg = 0x3084,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2phytimer_clk",
+			.parent_names = (const char *[]){ "csi2phytimer_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csiphy0_3p_clk = {
+	.halt_reg = 0x3234,
+	.clkr = {
+		.enable_reg = 0x3234,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csiphy0_3p_clk",
+			.parent_names = (const char *[]){ "csiphy0_3p_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csiphy1_3p_clk = {
+	.halt_reg = 0x3254,
+	.clkr = {
+		.enable_reg = 0x3254,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csiphy1_3p_clk",
+			.parent_names = (const char *[]){ "csiphy1_3p_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csiphy2_3p_clk = {
+	.halt_reg = 0x3274,
+	.clkr = {
+		.enable_reg = 0x3274,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csiphy2_3p_clk",
+			.parent_names = (const char *[]){ "csiphy2_3p_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_jpeg0_clk = {
+	.halt_reg = 0x35a8,
+	.clkr = {
+		.enable_reg = 0x35a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_jpeg0_clk",
+			.parent_names = (const char *[]){ "jpeg0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_jpeg2_clk = {
+	.halt_reg = 0x35b0,
+	.clkr = {
+		.enable_reg = 0x35b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_jpeg2_clk",
+			.parent_names = (const char *[]){ "jpeg2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_jpeg_dma_clk = {
+	.halt_reg = 0x35c0,
+	.clkr = {
+		.enable_reg = 0x35c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_jpeg_dma_clk",
+			.parent_names = (const char *[]){ "jpeg_dma_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_jpeg_ahb_clk = {
+	.halt_reg = 0x35b4,
+	.clkr = {
+		.enable_reg = 0x35b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_jpeg_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_jpeg_axi_clk = {
+	.halt_reg = 0x35b8,
+	.clkr = {
+		.enable_reg = 0x35b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_jpeg_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe_ahb_clk = {
+	.halt_reg = 0x36b8,
+	.clkr = {
+		.enable_reg = 0x36b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe_axi_clk = {
+	.halt_reg = 0x36bc,
+	.clkr = {
+		.enable_reg = 0x36bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe0_clk = {
+	.halt_reg = 0x36a8,
+	.clkr = {
+		.enable_reg = 0x36a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe0_clk",
+			.parent_names = (const char *[]){ "vfe0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe0_stream_clk = {
+	.halt_reg = 0x3720,
+	.clkr = {
+		.enable_reg = 0x3720,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe0_stream_clk",
+			.parent_names = (const char *[]){ "vfe0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe0_ahb_clk = {
+	.halt_reg = 0x3668,
+	.clkr = {
+		.enable_reg = 0x3668,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe0_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe1_clk = {
+	.halt_reg = 0x36ac,
+	.clkr = {
+		.enable_reg = 0x36ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe1_clk",
+			.parent_names = (const char *[]){ "vfe1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe1_stream_clk = {
+	.halt_reg = 0x3724,
+	.clkr = {
+		.enable_reg = 0x3724,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe1_stream_clk",
+			.parent_names = (const char *[]){ "vfe1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_vfe1_ahb_clk = {
+	.halt_reg = 0x3678,
+	.clkr = {
+		.enable_reg = 0x3678,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_vfe1_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi_vfe0_clk = {
+	.halt_reg = 0x3704,
+	.clkr = {
+		.enable_reg = 0x3704,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi_vfe0_clk",
+			.parent_names = (const char *[]){ "vfe0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi_vfe1_clk = {
+	.halt_reg = 0x3714,
+	.clkr = {
+		.enable_reg = 0x3714,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi_vfe1_clk",
+			.parent_names = (const char *[]){ "vfe1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cpp_vbif_ahb_clk = {
+	.halt_reg = 0x36c8,
+	.clkr = {
+		.enable_reg = 0x36c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cpp_vbif_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cpp_axi_clk = {
+	.halt_reg = 0x36c4,
+	.clkr = {
+		.enable_reg = 0x36c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cpp_axi_clk",
+			.parent_names = (const char *[]){ "axi_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cpp_clk = {
+	.halt_reg = 0x36b0,
+	.clkr = {
+		.enable_reg = 0x36b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cpp_clk",
+			.parent_names = (const char *[]){ "cpp_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_cpp_ahb_clk = {
+	.halt_reg = 0x36b4,
+	.clkr = {
+		.enable_reg = 0x36b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_cpp_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0_clk = {
+	.halt_reg = 0x30b4,
+	.clkr = {
+		.enable_reg = 0x30b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0_clk",
+			.parent_names = (const char *[]){ "csi0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0_ahb_clk = {
+	.halt_reg = 0x30bc,
+	.clkr = {
+		.enable_reg = 0x30bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0phy_clk = {
+	.halt_reg = 0x30c4,
+	.clkr = {
+		.enable_reg = 0x30c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0phy_clk",
+			.parent_names = (const char *[]){ "csi0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0rdi_clk = {
+	.halt_reg = 0x30d4,
+	.clkr = {
+		.enable_reg = 0x30d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0rdi_clk",
+			.parent_names = (const char *[]){ "csi0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi0pix_clk = {
+	.halt_reg = 0x30e4,
+	.clkr = {
+		.enable_reg = 0x30e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi0pix_clk",
+			.parent_names = (const char *[]){ "csi0_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1_clk = {
+	.halt_reg = 0x3124,
+	.clkr = {
+		.enable_reg = 0x3124,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1_clk",
+			.parent_names = (const char *[]){ "csi1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1_ahb_clk = {
+	.halt_reg = 0x3128,
+	.clkr = {
+		.enable_reg = 0x3128,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1phy_clk = {
+	.halt_reg = 0x3134,
+	.clkr = {
+		.enable_reg = 0x3134,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1phy_clk",
+			.parent_names = (const char *[]){ "csi1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1rdi_clk = {
+	.halt_reg = 0x3144,
+	.clkr = {
+		.enable_reg = 0x3144,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1rdi_clk",
+			.parent_names = (const char *[]){ "csi1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi1pix_clk = {
+	.halt_reg = 0x3154,
+	.clkr = {
+		.enable_reg = 0x3154,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi1pix_clk",
+			.parent_names = (const char *[]){ "csi1_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2_clk = {
+	.halt_reg = 0x3184,
+	.clkr = {
+		.enable_reg = 0x3184,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2_clk",
+			.parent_names = (const char *[]){ "csi2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2_ahb_clk = {
+	.halt_reg = 0x3188,
+	.clkr = {
+		.enable_reg = 0x3188,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2phy_clk = {
+	.halt_reg = 0x3194,
+	.clkr = {
+		.enable_reg = 0x3194,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2phy_clk",
+			.parent_names = (const char *[]){ "csi2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2rdi_clk = {
+	.halt_reg = 0x31a4,
+	.clkr = {
+		.enable_reg = 0x31a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2rdi_clk",
+			.parent_names = (const char *[]){ "csi2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi2pix_clk = {
+	.halt_reg = 0x31b4,
+	.clkr = {
+		.enable_reg = 0x31b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi2pix_clk",
+			.parent_names = (const char *[]){ "csi2_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi3_clk = {
+	.halt_reg = 0x31e4,
+	.clkr = {
+		.enable_reg = 0x31e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi3_clk",
+			.parent_names = (const char *[]){ "csi3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi3_ahb_clk = {
+	.halt_reg = 0x31e8,
+	.clkr = {
+		.enable_reg = 0x31e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi3_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi3phy_clk = {
+	.halt_reg = 0x31f4,
+	.clkr = {
+		.enable_reg = 0x31f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi3phy_clk",
+			.parent_names = (const char *[]){ "csi3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi3rdi_clk = {
+	.halt_reg = 0x3204,
+	.clkr = {
+		.enable_reg = 0x3204,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi3rdi_clk",
+			.parent_names = (const char *[]){ "csi3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_csi3pix_clk = {
+	.halt_reg = 0x3214,
+	.clkr = {
+		.enable_reg = 0x3214,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_csi3pix_clk",
+			.parent_names = (const char *[]){ "csi3_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch camss_ispif_ahb_clk = {
+	.halt_reg = 0x3224,
+	.clkr = {
+		.enable_reg = 0x3224,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "camss_ispif_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch fd_core_clk = {
+	.halt_reg = 0x3b68,
+	.clkr = {
+		.enable_reg = 0x3b68,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "fd_core_clk",
+			.parent_names = (const char *[]){ "fd_core_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch fd_core_uar_clk = {
+	.halt_reg = 0x3b6c,
+	.clkr = {
+		.enable_reg = 0x3b6c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "fd_core_uar_clk",
+			.parent_names = (const char *[]){ "fd_core_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch fd_ahb_clk = {
+	.halt_reg = 0x3ba74,
+	.clkr = {
+		.enable_reg = 0x3ba74,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "fd_ahb_clk",
+			.parent_names = (const char *[]){ "ahb_clk_src" },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_hw *mmcc_msm8996_hws[] = {
+	&gpll0_div.hw,
+};
+
+static struct clk_regmap *mmcc_msm8996_clocks[] = {
+	[MMPLL0_EARLY] = &mmpll0_early.clkr,
+	[MMPLL0_PLL] = &mmpll0.clkr,
+	[MMPLL1_EARLY] = &mmpll1_early.clkr,
+	[MMPLL1_PLL] = &mmpll1.clkr,
+	[MMPLL2_EARLY] = &mmpll2_early.clkr,
+	[MMPLL2_PLL] = &mmpll2.clkr,
+	[MMPLL3_EARLY] = &mmpll3_early.clkr,
+	[MMPLL3_PLL] = &mmpll3.clkr,
+	[MMPLL4_EARLY] = &mmpll4_early.clkr,
+	[MMPLL4_PLL] = &mmpll4.clkr,
+	[MMPLL5_EARLY] = &mmpll5_early.clkr,
+	[MMPLL5_PLL] = &mmpll5.clkr,
+	[MMPLL8_EARLY] = &mmpll8_early.clkr,
+	[MMPLL8_PLL] = &mmpll8.clkr,
+	[MMPLL9_EARLY] = &mmpll9_early.clkr,
+	[MMPLL9_PLL] = &mmpll9.clkr,
+	[AHB_CLK_SRC] = &ahb_clk_src.clkr,
+	[AXI_CLK_SRC] = &axi_clk_src.clkr,
+	[MAXI_CLK_SRC] = &maxi_clk_src.clkr,
+	[GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr,
+	[RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr,
+	[ISENSE_CLK_SRC] = &isense_clk_src.clkr,
+	[RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr,
+	[VIDEO_CORE_CLK_SRC] = &video_core_clk_src.clkr,
+	[VIDEO_SUBCORE0_CLK_SRC] = &video_subcore0_clk_src.clkr,
+	[VIDEO_SUBCORE1_CLK_SRC] = &video_subcore1_clk_src.clkr,
+	[PCLK0_CLK_SRC] = &pclk0_clk_src.clkr,
+	[PCLK1_CLK_SRC] = &pclk1_clk_src.clkr,
+	[MDP_CLK_SRC] = &mdp_clk_src.clkr,
+	[EXTPCLK_CLK_SRC] = &extpclk_clk_src.clkr,
+	[VSYNC_CLK_SRC] = &vsync_clk_src.clkr,
+	[HDMI_CLK_SRC] = &hdmi_clk_src.clkr,
+	[BYTE0_CLK_SRC] = &byte0_clk_src.clkr,
+	[BYTE1_CLK_SRC] = &byte1_clk_src.clkr,
+	[ESC0_CLK_SRC] = &esc0_clk_src.clkr,
+	[ESC1_CLK_SRC] = &esc1_clk_src.clkr,
+	[CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr,
+	[CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr,
+	[MCLK0_CLK_SRC] = &mclk0_clk_src.clkr,
+	[MCLK1_CLK_SRC] = &mclk1_clk_src.clkr,
+	[MCLK2_CLK_SRC] = &mclk2_clk_src.clkr,
+	[MCLK3_CLK_SRC] = &mclk3_clk_src.clkr,
+	[CCI_CLK_SRC] = &cci_clk_src.clkr,
+	[CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr,
+	[CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr,
+	[CSI2PHYTIMER_CLK_SRC] = &csi2phytimer_clk_src.clkr,
+	[CSIPHY0_3P_CLK_SRC] = &csiphy0_3p_clk_src.clkr,
+	[CSIPHY1_3P_CLK_SRC] = &csiphy1_3p_clk_src.clkr,
+	[CSIPHY2_3P_CLK_SRC] = &csiphy2_3p_clk_src.clkr,
+	[JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr,
+	[JPEG2_CLK_SRC] = &jpeg2_clk_src.clkr,
+	[JPEG_DMA_CLK_SRC] = &jpeg_dma_clk_src.clkr,
+	[VFE0_CLK_SRC] = &vfe0_clk_src.clkr,
+	[VFE1_CLK_SRC] = &vfe1_clk_src.clkr,
+	[CPP_CLK_SRC] = &cpp_clk_src.clkr,
+	[CSI0_CLK_SRC] = &csi0_clk_src.clkr,
+	[CSI1_CLK_SRC] = &csi1_clk_src.clkr,
+	[CSI2_CLK_SRC] = &csi2_clk_src.clkr,
+	[CSI3_CLK_SRC] = &csi3_clk_src.clkr,
+	[FD_CORE_CLK_SRC] = &fd_core_clk_src.clkr,
+	[MMSS_MMAGIC_AHB_CLK] = &mmss_mmagic_ahb_clk.clkr,
+	[MMSS_MMAGIC_CFG_AHB_CLK] = &mmss_mmagic_cfg_ahb_clk.clkr,
+	[MMSS_MISC_AHB_CLK] = &mmss_misc_ahb_clk.clkr,
+	[MMSS_MISC_CXO_CLK] = &mmss_misc_cxo_clk.clkr,
+	[MMSS_MMAGIC_AXI_CLK] = &mmss_mmagic_axi_clk.clkr,
+	[MMSS_MMAGIC_MAXI_CLK] = &mmss_mmagic_maxi_clk.clkr,
+	[MMAGIC_CAMSS_AXI_CLK] = &mmagic_camss_axi_clk.clkr,
+	[MMAGIC_CAMSS_NOC_CFG_AHB_CLK] = &mmagic_camss_noc_cfg_ahb_clk.clkr,
+	[SMMU_VFE_AHB_CLK] = &smmu_vfe_ahb_clk.clkr,
+	[SMMU_VFE_AXI_CLK] = &smmu_vfe_axi_clk.clkr,
+	[SMMU_CPP_AHB_CLK] = &smmu_cpp_ahb_clk.clkr,
+	[SMMU_CPP_AXI_CLK] = &smmu_cpp_axi_clk.clkr,
+	[SMMU_JPEG_AHB_CLK] = &smmu_jpeg_ahb_clk.clkr,
+	[SMMU_JPEG_AXI_CLK] = &smmu_jpeg_axi_clk.clkr,
+	[MMAGIC_MDSS_AXI_CLK] = &mmagic_mdss_axi_clk.clkr,
+	[MMAGIC_MDSS_NOC_CFG_AHB_CLK] = &mmagic_mdss_noc_cfg_ahb_clk.clkr,
+	[SMMU_ROT_AHB_CLK] = &smmu_rot_ahb_clk.clkr,
+	[SMMU_ROT_AXI_CLK] = &smmu_rot_axi_clk.clkr,
+	[SMMU_MDP_AHB_CLK] = &smmu_mdp_ahb_clk.clkr,
+	[SMMU_MDP_AXI_CLK] = &smmu_mdp_axi_clk.clkr,
+	[MMAGIC_VIDEO_AXI_CLK] = &mmagic_video_axi_clk.clkr,
+	[MMAGIC_VIDEO_NOC_CFG_AHB_CLK] = &mmagic_video_noc_cfg_ahb_clk.clkr,
+	[SMMU_VIDEO_AHB_CLK] = &smmu_video_ahb_clk.clkr,
+	[SMMU_VIDEO_AXI_CLK] = &smmu_video_axi_clk.clkr,
+	[MMAGIC_BIMC_AXI_CLK] = &mmagic_bimc_axi_clk.clkr,
+	[MMAGIC_BIMC_NOC_CFG_AHB_CLK] = &mmagic_bimc_noc_cfg_ahb_clk.clkr,
+	[GPU_GX_GFX3D_CLK] = &gpu_gx_gfx3d_clk.clkr,
+	[GPU_GX_RBBMTIMER_CLK] = &gpu_gx_rbbmtimer_clk.clkr,
+	[GPU_AHB_CLK] = &gpu_ahb_clk.clkr,
+	[GPU_AON_ISENSE_CLK] = &gpu_aon_isense_clk.clkr,
+	[VMEM_MAXI_CLK] = &vmem_maxi_clk.clkr,
+	[VMEM_AHB_CLK] = &vmem_ahb_clk.clkr,
+	[MMSS_RBCPR_CLK] = &mmss_rbcpr_clk.clkr,
+	[MMSS_RBCPR_AHB_CLK] = &mmss_rbcpr_ahb_clk.clkr,
+	[VIDEO_CORE_CLK] = &video_core_clk.clkr,
+	[VIDEO_AXI_CLK] = &video_axi_clk.clkr,
+	[VIDEO_MAXI_CLK] = &video_maxi_clk.clkr,
+	[VIDEO_AHB_CLK] = &video_ahb_clk.clkr,
+	[VIDEO_SUBCORE0_CLK] = &video_subcore0_clk.clkr,
+	[VIDEO_SUBCORE1_CLK] = &video_subcore1_clk.clkr,
+	[MDSS_AHB_CLK] = &mdss_ahb_clk.clkr,
+	[MDSS_HDMI_AHB_CLK] = &mdss_hdmi_ahb_clk.clkr,
+	[MDSS_AXI_CLK] = &mdss_axi_clk.clkr,
+	[MDSS_PCLK0_CLK] = &mdss_pclk0_clk.clkr,
+	[MDSS_PCLK1_CLK] = &mdss_pclk1_clk.clkr,
+	[MDSS_MDP_CLK] = &mdss_mdp_clk.clkr,
+	[MDSS_EXTPCLK_CLK] = &mdss_extpclk_clk.clkr,
+	[MDSS_VSYNC_CLK] = &mdss_vsync_clk.clkr,
+	[MDSS_HDMI_CLK] = &mdss_hdmi_clk.clkr,
+	[MDSS_BYTE0_CLK] = &mdss_byte0_clk.clkr,
+	[MDSS_BYTE1_CLK] = &mdss_byte1_clk.clkr,
+	[MDSS_ESC0_CLK] = &mdss_esc0_clk.clkr,
+	[MDSS_ESC1_CLK] = &mdss_esc1_clk.clkr,
+	[CAMSS_TOP_AHB_CLK] = &camss_top_ahb_clk.clkr,
+	[CAMSS_AHB_CLK] = &camss_ahb_clk.clkr,
+	[CAMSS_MICRO_AHB_CLK] = &camss_micro_ahb_clk.clkr,
+	[CAMSS_GP0_CLK] = &camss_gp0_clk.clkr,
+	[CAMSS_GP1_CLK] = &camss_gp1_clk.clkr,
+	[CAMSS_MCLK0_CLK] = &camss_mclk0_clk.clkr,
+	[CAMSS_MCLK1_CLK] = &camss_mclk1_clk.clkr,
+	[CAMSS_MCLK2_CLK] = &camss_mclk2_clk.clkr,
+	[CAMSS_MCLK3_CLK] = &camss_mclk3_clk.clkr,
+	[CAMSS_CCI_CLK] = &camss_cci_clk.clkr,
+	[CAMSS_CCI_AHB_CLK] = &camss_cci_ahb_clk.clkr,
+	[CAMSS_CSI0PHYTIMER_CLK] = &camss_csi0phytimer_clk.clkr,
+	[CAMSS_CSI1PHYTIMER_CLK] = &camss_csi1phytimer_clk.clkr,
+	[CAMSS_CSI2PHYTIMER_CLK] = &camss_csi2phytimer_clk.clkr,
+	[CAMSS_CSIPHY0_3P_CLK] = &camss_csiphy0_3p_clk.clkr,
+	[CAMSS_CSIPHY1_3P_CLK] = &camss_csiphy1_3p_clk.clkr,
+	[CAMSS_CSIPHY2_3P_CLK] = &camss_csiphy2_3p_clk.clkr,
+	[CAMSS_JPEG0_CLK] = &camss_jpeg0_clk.clkr,
+	[CAMSS_JPEG2_CLK] = &camss_jpeg2_clk.clkr,
+	[CAMSS_JPEG_DMA_CLK] = &camss_jpeg_dma_clk.clkr,
+	[CAMSS_JPEG_AHB_CLK] = &camss_jpeg_ahb_clk.clkr,
+	[CAMSS_JPEG_AXI_CLK] = &camss_jpeg_axi_clk.clkr,
+	[CAMSS_VFE_AHB_CLK] = &camss_vfe_ahb_clk.clkr,
+	[CAMSS_VFE_AXI_CLK] = &camss_vfe_axi_clk.clkr,
+	[CAMSS_VFE0_CLK] = &camss_vfe0_clk.clkr,
+	[CAMSS_VFE0_STREAM_CLK] = &camss_vfe0_stream_clk.clkr,
+	[CAMSS_VFE0_AHB_CLK] = &camss_vfe0_ahb_clk.clkr,
+	[CAMSS_VFE1_CLK] = &camss_vfe1_clk.clkr,
+	[CAMSS_VFE1_STREAM_CLK] = &camss_vfe1_stream_clk.clkr,
+	[CAMSS_VFE1_AHB_CLK] = &camss_vfe1_ahb_clk.clkr,
+	[CAMSS_CSI_VFE0_CLK] = &camss_csi_vfe0_clk.clkr,
+	[CAMSS_CSI_VFE1_CLK] = &camss_csi_vfe1_clk.clkr,
+	[CAMSS_CPP_VBIF_AHB_CLK] = &camss_cpp_vbif_ahb_clk.clkr,
+	[CAMSS_CPP_AXI_CLK] = &camss_cpp_axi_clk.clkr,
+	[CAMSS_CPP_CLK] = &camss_cpp_clk.clkr,
+	[CAMSS_CPP_AHB_CLK] = &camss_cpp_ahb_clk.clkr,
+	[CAMSS_CSI0_CLK] = &camss_csi0_clk.clkr,
+	[CAMSS_CSI0_AHB_CLK] = &camss_csi0_ahb_clk.clkr,
+	[CAMSS_CSI0PHY_CLK] = &camss_csi0phy_clk.clkr,
+	[CAMSS_CSI0RDI_CLK] = &camss_csi0rdi_clk.clkr,
+	[CAMSS_CSI0PIX_CLK] = &camss_csi0pix_clk.clkr,
+	[CAMSS_CSI1_CLK] = &camss_csi1_clk.clkr,
+	[CAMSS_CSI1_AHB_CLK] = &camss_csi1_ahb_clk.clkr,
+	[CAMSS_CSI1PHY_CLK] = &camss_csi1phy_clk.clkr,
+	[CAMSS_CSI1RDI_CLK] = &camss_csi1rdi_clk.clkr,
+	[CAMSS_CSI1PIX_CLK] = &camss_csi1pix_clk.clkr,
+	[CAMSS_CSI2_CLK] = &camss_csi2_clk.clkr,
+	[CAMSS_CSI2_AHB_CLK] = &camss_csi2_ahb_clk.clkr,
+	[CAMSS_CSI2PHY_CLK] = &camss_csi2phy_clk.clkr,
+	[CAMSS_CSI2RDI_CLK] = &camss_csi2rdi_clk.clkr,
+	[CAMSS_CSI2PIX_CLK] = &camss_csi2pix_clk.clkr,
+	[CAMSS_CSI3_CLK] = &camss_csi3_clk.clkr,
+	[CAMSS_CSI3_AHB_CLK] = &camss_csi3_ahb_clk.clkr,
+	[CAMSS_CSI3PHY_CLK] = &camss_csi3phy_clk.clkr,
+	[CAMSS_CSI3RDI_CLK] = &camss_csi3rdi_clk.clkr,
+	[CAMSS_CSI3PIX_CLK] = &camss_csi3pix_clk.clkr,
+	[CAMSS_ISPIF_AHB_CLK] = &camss_ispif_ahb_clk.clkr,
+	[FD_CORE_CLK] = &fd_core_clk.clkr,
+	[FD_CORE_UAR_CLK] = &fd_core_uar_clk.clkr,
+	[FD_AHB_CLK] = &fd_ahb_clk.clkr,
+};
+
+static const struct qcom_reset_map mmcc_msm8996_resets[] = {
+	[MMAGICAHB_BCR] = { 0x5020 },
+	[MMAGIC_CFG_BCR] = { 0x5050 },
+	[MISC_BCR] = { 0x5010 },
+	[BTO_BCR] = { 0x5030 },
+	[MMAGICAXI_BCR] = { 0x5060 },
+	[MMAGICMAXI_BCR] = { 0x5070 },
+	[DSA_BCR] = { 0x50a0 },
+	[MMAGIC_CAMSS_BCR] = { 0x3c40 },
+	[THROTTLE_CAMSS_BCR] = { 0x3c30 },
+	[SMMU_VFE_BCR] = { 0x3c00 },
+	[SMMU_CPP_BCR] = { 0x3c10 },
+	[SMMU_JPEG_BCR] = { 0x3c20 },
+	[MMAGIC_MDSS_BCR] = { 0x2470 },
+	[THROTTLE_MDSS_BCR] = { 0x2460 },
+	[SMMU_ROT_BCR] = { 0x2440 },
+	[SMMU_MDP_BCR] = { 0x2450 },
+	[MMAGIC_VIDEO_BCR] = { 0x1190 },
+	[THROTTLE_VIDEO_BCR] = { 0x1180 },
+	[SMMU_VIDEO_BCR] = { 0x1170 },
+	[MMAGIC_BIMC_BCR] = { 0x5290 },
+	[GPU_GX_BCR] = { 0x4020 },
+	[GPU_BCR] = { 0x4030 },
+	[GPU_AON_BCR] = { 0x4040 },
+	[VMEM_BCR] = { 0x1200 },
+	[MMSS_RBCPR_BCR] = { 0x4080 },
+	[VIDEO_BCR] = { 0x1020 },
+	[MDSS_BCR] = { 0x2300 },
+	[CAMSS_TOP_BCR] = { 0x3480 },
+	[CAMSS_AHB_BCR] = { 0x3488 },
+	[CAMSS_MICRO_BCR] = { 0x3490 },
+	[CAMSS_CCI_BCR] = { 0x3340 },
+	[CAMSS_PHY0_BCR] = { 0x3020 },
+	[CAMSS_PHY1_BCR] = { 0x3050 },
+	[CAMSS_PHY2_BCR] = { 0x3080 },
+	[CAMSS_CSIPHY0_3P_BCR] = { 0x3230 },
+	[CAMSS_CSIPHY1_3P_BCR] = { 0x3250 },
+	[CAMSS_CSIPHY2_3P_BCR] = { 0x3270 },
+	[CAMSS_JPEG_BCR] = { 0x35a0 },
+	[CAMSS_VFE_BCR] = { 0x36a0 },
+	[CAMSS_VFE0_BCR] = { 0x3660 },
+	[CAMSS_VFE1_BCR] = { 0x3670 },
+	[CAMSS_CSI_VFE0_BCR] = { 0x3700 },
+	[CAMSS_CSI_VFE1_BCR] = { 0x3710 },
+	[CAMSS_CPP_TOP_BCR] = { 0x36c0 },
+	[CAMSS_CPP_BCR] = { 0x36d0 },
+	[CAMSS_CSI0_BCR] = { 0x30b0 },
+	[CAMSS_CSI0RDI_BCR] = { 0x30d0 },
+	[CAMSS_CSI0PIX_BCR] = { 0x30e0 },
+	[CAMSS_CSI1_BCR] = { 0x3120 },
+	[CAMSS_CSI1RDI_BCR] = { 0x3140 },
+	[CAMSS_CSI1PIX_BCR] = { 0x3150 },
+	[CAMSS_CSI2_BCR] = { 0x3180 },
+	[CAMSS_CSI2RDI_BCR] = { 0x31a0 },
+	[CAMSS_CSI2PIX_BCR] = { 0x31b0 },
+	[CAMSS_CSI3_BCR] = { 0x31e0 },
+	[CAMSS_CSI3RDI_BCR] = { 0x3200 },
+	[CAMSS_CSI3PIX_BCR] = { 0x3210 },
+	[CAMSS_ISPIF_BCR] = { 0x3220 },
+	[FD_BCR] = { 0x3b60 },
+	[MMSS_SPDM_RM_BCR] = { 0x300 },
+};
+
+static const struct regmap_config mmcc_msm8996_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0xb008,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc mmcc_msm8996_desc = {
+	.config = &mmcc_msm8996_regmap_config,
+	.clks = mmcc_msm8996_clocks,
+	.num_clks = ARRAY_SIZE(mmcc_msm8996_clocks),
+	.resets = mmcc_msm8996_resets,
+	.num_resets = ARRAY_SIZE(mmcc_msm8996_resets),
+};
+
+static const struct of_device_id mmcc_msm8996_match_table[] = {
+	{ .compatible = "qcom,mmcc-msm8996" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mmcc_msm8996_match_table);
+
+static int mmcc_msm8996_probe(struct platform_device *pdev)
+{
+	struct clk *clk;
+	struct device *dev = &pdev->dev;
+	int i;
+	struct regmap *regmap;
+
+	regmap = qcom_cc_map(pdev, &mmcc_msm8996_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/* Disable the AHB DCD */
+	regmap_update_bits(regmap, 0x50d8, BIT(31), 0);
+	/* Disable the NoC FSM for mmss_mmagic_cfg_ahb_clk */
+	regmap_update_bits(regmap, 0x5054, BIT(15), 0);
+
+	for (i = 0; i < ARRAY_SIZE(mmcc_msm8996_hws); i++) {
+		clk = devm_clk_register(dev, mmcc_msm8996_hws[i]);
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+	}
+
+	return qcom_cc_really_probe(pdev, &mmcc_msm8996_desc, regmap);
+}
+
+static struct platform_driver mmcc_msm8996_driver = {
+	.probe		= mmcc_msm8996_probe,
+	.driver		= {
+		.name	= "mmcc-msm8996",
+		.of_match_table = mmcc_msm8996_match_table,
+	},
+};
+module_platform_driver(mmcc_msm8996_driver);
+
+MODULE_DESCRIPTION("QCOM MMCC MSM8996 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:mmcc-msm8996");
diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile
index b27edd6c8183..80b9a379beb4 100644
--- a/drivers/clk/rockchip/Makefile
+++ b/drivers/clk/rockchip/Makefile
@@ -10,6 +10,8 @@ obj-y	+= clk-inverter.o
 obj-y	+= clk-mmc-phase.o
 obj-$(CONFIG_RESET_CONTROLLER)	+= softrst.o
 
+obj-y	+= clk-rk3036.o
 obj-y	+= clk-rk3188.o
+obj-y	+= clk-rk3228.o
 obj-y	+= clk-rk3288.o
 obj-y	+= clk-rk3368.o
diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c
index 330870a6d8bf..d07374f48caf 100644
--- a/drivers/clk/rockchip/clk-cpu.c
+++ b/drivers/clk/rockchip/clk-cpu.c
@@ -242,8 +242,8 @@ struct clk *rockchip_clk_register_cpuclk(const char *name,
 	struct clk *clk, *cclk;
 	int ret;
 
-	if (num_parents != 2) {
-		pr_err("%s: needs two parent clocks\n", __func__);
+	if (num_parents < 2) {
+		pr_err("%s: needs at least two parent clocks\n", __func__);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c
index 4881eb8a1576..b7e66c9dd9f2 100644
--- a/drivers/clk/rockchip/clk-pll.c
+++ b/drivers/clk/rockchip/clk-pll.c
@@ -2,6 +2,9 @@
  * Copyright (c) 2014 MundoReader S.L.
  * Author: Heiko Stuebner <heiko@sntech.de>
  *
+ * Copyright (c) 2015 Rockchip Electronics Co. Ltd.
+ * Author: Xing Zheng <zhengxing@rock-chips.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -19,6 +22,7 @@
 #include <linux/delay.h>
 #include <linux/clk-provider.h>
 #include <linux/regmap.h>
+#include <linux/clk.h>
 #include "clk.h"
 
 #define PLL_MODE_MASK		0x3
@@ -108,6 +112,252 @@ static int rockchip_pll_wait_lock(struct rockchip_clk_pll *pll)
 }
 
 /**
+ * PLL used in RK3036
+ */
+
+#define RK3036_PLLCON(i)			(i * 0x4)
+#define RK3036_PLLCON0_FBDIV_MASK		0xfff
+#define RK3036_PLLCON0_FBDIV_SHIFT		0
+#define RK3036_PLLCON0_POSTDIV1_MASK		0x7
+#define RK3036_PLLCON0_POSTDIV1_SHIFT		12
+#define RK3036_PLLCON1_REFDIV_MASK		0x3f
+#define RK3036_PLLCON1_REFDIV_SHIFT		0
+#define RK3036_PLLCON1_POSTDIV2_MASK		0x7
+#define RK3036_PLLCON1_POSTDIV2_SHIFT		6
+#define RK3036_PLLCON1_DSMPD_MASK		0x1
+#define RK3036_PLLCON1_DSMPD_SHIFT		12
+#define RK3036_PLLCON2_FRAC_MASK		0xffffff
+#define RK3036_PLLCON2_FRAC_SHIFT		0
+
+#define RK3036_PLLCON1_PWRDOWN			(1 << 13)
+
+static void rockchip_rk3036_pll_get_params(struct rockchip_clk_pll *pll,
+					struct rockchip_pll_rate_table *rate)
+{
+	u32 pllcon;
+
+	pllcon = readl_relaxed(pll->reg_base + RK3036_PLLCON(0));
+	rate->fbdiv = ((pllcon >> RK3036_PLLCON0_FBDIV_SHIFT)
+				& RK3036_PLLCON0_FBDIV_MASK);
+	rate->postdiv1 = ((pllcon >> RK3036_PLLCON0_POSTDIV1_SHIFT)
+				& RK3036_PLLCON0_POSTDIV1_MASK);
+
+	pllcon = readl_relaxed(pll->reg_base + RK3036_PLLCON(1));
+	rate->refdiv = ((pllcon >> RK3036_PLLCON1_REFDIV_SHIFT)
+				& RK3036_PLLCON1_REFDIV_MASK);
+	rate->postdiv2 = ((pllcon >> RK3036_PLLCON1_POSTDIV2_SHIFT)
+				& RK3036_PLLCON1_POSTDIV2_MASK);
+	rate->dsmpd = ((pllcon >> RK3036_PLLCON1_DSMPD_SHIFT)
+				& RK3036_PLLCON1_DSMPD_MASK);
+
+	pllcon = readl_relaxed(pll->reg_base + RK3036_PLLCON(2));
+	rate->frac = ((pllcon >> RK3036_PLLCON2_FRAC_SHIFT)
+				& RK3036_PLLCON2_FRAC_MASK);
+}
+
+static unsigned long rockchip_rk3036_pll_recalc_rate(struct clk_hw *hw,
+						     unsigned long prate)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+	struct rockchip_pll_rate_table cur;
+	u64 rate64 = prate;
+
+	rockchip_rk3036_pll_get_params(pll, &cur);
+
+	rate64 *= cur.fbdiv;
+	do_div(rate64, cur.refdiv);
+
+	if (cur.dsmpd == 0) {
+		/* fractional mode */
+		u64 frac_rate64 = prate * cur.frac;
+
+		do_div(frac_rate64, cur.refdiv);
+		rate64 += frac_rate64 >> 24;
+	}
+
+	do_div(rate64, cur.postdiv1);
+	do_div(rate64, cur.postdiv2);
+
+	return (unsigned long)rate64;
+}
+
+static int rockchip_rk3036_pll_set_params(struct rockchip_clk_pll *pll,
+				const struct rockchip_pll_rate_table *rate)
+{
+	const struct clk_ops *pll_mux_ops = pll->pll_mux_ops;
+	struct clk_mux *pll_mux = &pll->pll_mux;
+	struct rockchip_pll_rate_table cur;
+	u32 pllcon;
+	int rate_change_remuxed = 0;
+	int cur_parent;
+	int ret;
+
+	pr_debug("%s: rate settings for %lu fbdiv: %d, postdiv1: %d, refdiv: %d, postdiv2: %d, dsmpd: %d, frac: %d\n",
+		__func__, rate->rate, rate->fbdiv, rate->postdiv1, rate->refdiv,
+		rate->postdiv2, rate->dsmpd, rate->frac);
+
+	rockchip_rk3036_pll_get_params(pll, &cur);
+	cur.rate = 0;
+
+	cur_parent = pll_mux_ops->get_parent(&pll_mux->hw);
+	if (cur_parent == PLL_MODE_NORM) {
+		pll_mux_ops->set_parent(&pll_mux->hw, PLL_MODE_SLOW);
+		rate_change_remuxed = 1;
+	}
+
+	/* update pll values */
+	writel_relaxed(HIWORD_UPDATE(rate->fbdiv, RK3036_PLLCON0_FBDIV_MASK,
+					  RK3036_PLLCON0_FBDIV_SHIFT) |
+		       HIWORD_UPDATE(rate->postdiv1, RK3036_PLLCON0_POSTDIV1_MASK,
+					     RK3036_PLLCON0_POSTDIV1_SHIFT),
+		       pll->reg_base + RK3036_PLLCON(0));
+
+	writel_relaxed(HIWORD_UPDATE(rate->refdiv, RK3036_PLLCON1_REFDIV_MASK,
+						   RK3036_PLLCON1_REFDIV_SHIFT) |
+		       HIWORD_UPDATE(rate->postdiv2, RK3036_PLLCON1_POSTDIV2_MASK,
+						     RK3036_PLLCON1_POSTDIV2_SHIFT) |
+		       HIWORD_UPDATE(rate->dsmpd, RK3036_PLLCON1_DSMPD_MASK,
+						  RK3036_PLLCON1_DSMPD_SHIFT),
+		       pll->reg_base + RK3036_PLLCON(1));
+
+	/* GPLL CON2 is not HIWORD_MASK */
+	pllcon = readl_relaxed(pll->reg_base + RK3036_PLLCON(2));
+	pllcon &= ~(RK3036_PLLCON2_FRAC_MASK << RK3036_PLLCON2_FRAC_SHIFT);
+	pllcon |= rate->frac << RK3036_PLLCON2_FRAC_SHIFT;
+	writel_relaxed(pllcon, pll->reg_base + RK3036_PLLCON(2));
+
+	/* wait for the pll to lock */
+	ret = rockchip_pll_wait_lock(pll);
+	if (ret) {
+		pr_warn("%s: pll update unsucessful, trying to restore old params\n",
+			__func__);
+		rockchip_rk3036_pll_set_params(pll, &cur);
+	}
+
+	if (rate_change_remuxed)
+		pll_mux_ops->set_parent(&pll_mux->hw, PLL_MODE_NORM);
+
+	return ret;
+}
+
+static int rockchip_rk3036_pll_set_rate(struct clk_hw *hw, unsigned long drate,
+					unsigned long prate)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+	const struct rockchip_pll_rate_table *rate;
+	unsigned long old_rate = rockchip_rk3036_pll_recalc_rate(hw, prate);
+	struct regmap *grf = rockchip_clk_get_grf();
+
+	if (IS_ERR(grf)) {
+		pr_debug("%s: grf regmap not available, aborting rate change\n",
+			 __func__);
+		return PTR_ERR(grf);
+	}
+
+	pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n",
+		 __func__, __clk_get_name(hw->clk), old_rate, drate, prate);
+
+	/* Get required rate settings from table */
+	rate = rockchip_get_pll_settings(pll, drate);
+	if (!rate) {
+		pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__,
+			drate, __clk_get_name(hw->clk));
+		return -EINVAL;
+	}
+
+	return rockchip_rk3036_pll_set_params(pll, rate);
+}
+
+static int rockchip_rk3036_pll_enable(struct clk_hw *hw)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+
+	writel(HIWORD_UPDATE(0, RK3036_PLLCON1_PWRDOWN, 0),
+	       pll->reg_base + RK3036_PLLCON(1));
+
+	return 0;
+}
+
+static void rockchip_rk3036_pll_disable(struct clk_hw *hw)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+
+	writel(HIWORD_UPDATE(RK3036_PLLCON1_PWRDOWN,
+			     RK3036_PLLCON1_PWRDOWN, 0),
+	       pll->reg_base + RK3036_PLLCON(1));
+}
+
+static int rockchip_rk3036_pll_is_enabled(struct clk_hw *hw)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+	u32 pllcon = readl(pll->reg_base + RK3036_PLLCON(1));
+
+	return !(pllcon & RK3036_PLLCON1_PWRDOWN);
+}
+
+static void rockchip_rk3036_pll_init(struct clk_hw *hw)
+{
+	struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw);
+	const struct rockchip_pll_rate_table *rate;
+	struct rockchip_pll_rate_table cur;
+	unsigned long drate;
+
+	if (!(pll->flags & ROCKCHIP_PLL_SYNC_RATE))
+		return;
+
+	drate = clk_hw_get_rate(hw);
+	rate = rockchip_get_pll_settings(pll, drate);
+
+	/* when no rate setting for the current rate, rely on clk_set_rate */
+	if (!rate)
+		return;
+
+	rockchip_rk3036_pll_get_params(pll, &cur);
+
+	pr_debug("%s: pll %s@%lu: Hz\n", __func__, __clk_get_name(hw->clk),
+		 drate);
+	pr_debug("old - fbdiv: %d, postdiv1: %d, refdiv: %d, postdiv2: %d, dsmpd: %d, frac: %d\n",
+		 cur.fbdiv, cur.postdiv1, cur.refdiv, cur.postdiv2,
+		 cur.dsmpd, cur.frac);
+	pr_debug("new - fbdiv: %d, postdiv1: %d, refdiv: %d, postdiv2: %d, dsmpd: %d, frac: %d\n",
+		 rate->fbdiv, rate->postdiv1, rate->refdiv, rate->postdiv2,
+		 rate->dsmpd, rate->frac);
+
+	if (rate->fbdiv != cur.fbdiv || rate->postdiv1 != cur.postdiv1 ||
+		rate->refdiv != cur.refdiv || rate->postdiv2 != cur.postdiv2 ||
+		rate->dsmpd != cur.dsmpd || rate->frac != cur.frac) {
+		struct clk *parent = clk_get_parent(hw->clk);
+
+		if (!parent) {
+			pr_warn("%s: parent of %s not available\n",
+				__func__, __clk_get_name(hw->clk));
+			return;
+		}
+
+		pr_debug("%s: pll %s: rate params do not match rate table, adjusting\n",
+			 __func__, __clk_get_name(hw->clk));
+		rockchip_rk3036_pll_set_params(pll, rate);
+	}
+}
+
+static const struct clk_ops rockchip_rk3036_pll_clk_norate_ops = {
+	.recalc_rate = rockchip_rk3036_pll_recalc_rate,
+	.enable = rockchip_rk3036_pll_enable,
+	.disable = rockchip_rk3036_pll_disable,
+	.is_enabled = rockchip_rk3036_pll_is_enabled,
+};
+
+static const struct clk_ops rockchip_rk3036_pll_clk_ops = {
+	.recalc_rate = rockchip_rk3036_pll_recalc_rate,
+	.round_rate = rockchip_pll_round_rate,
+	.set_rate = rockchip_rk3036_pll_set_rate,
+	.enable = rockchip_rk3036_pll_enable,
+	.disable = rockchip_rk3036_pll_disable,
+	.is_enabled = rockchip_rk3036_pll_is_enabled,
+	.init = rockchip_rk3036_pll_init,
+};
+
+/**
  * PLL used in RK3066, RK3188 and RK3288
  */
 
@@ -376,7 +626,7 @@ struct clk *rockchip_clk_register_pll(enum rockchip_pll_type pll_type,
 	pll_mux->lock = lock;
 	pll_mux->hw.init = &init;
 
-	if (pll_type == pll_rk3066)
+	if (pll_type == pll_rk3036 || pll_type == pll_rk3066)
 		pll_mux->flags |= CLK_MUX_HIWORD_MASK;
 
 	/* the actual muxing is xin24m, pll-output, xin32k */
@@ -421,6 +671,12 @@ struct clk *rockchip_clk_register_pll(enum rockchip_pll_type pll_type,
 	}
 
 	switch (pll_type) {
+	case pll_rk3036:
+		if (!pll->rate_table)
+			init.ops = &rockchip_rk3036_pll_clk_norate_ops;
+		else
+			init.ops = &rockchip_rk3036_pll_clk_ops;
+		break;
 	case pll_rk3066:
 		if (!pll->rate_table)
 			init.ops = &rockchip_rk3066_pll_clk_norate_ops;
diff --git a/drivers/clk/rockchip/clk-rk3036.c b/drivers/clk/rockchip/clk-rk3036.c
new file mode 100644
index 000000000000..ebce98033fbb
--- /dev/null
+++ b/drivers/clk/rockchip/clk-rk3036.c
@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) 2014 MundoReader S.L.
+ * Author: Heiko Stuebner <heiko@sntech.de>
+ *
+ * Copyright (c) 2015 Rockchip Electronics Co. Ltd.
+ * Author: Xing Zheng <zhengxing@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/syscore_ops.h>
+#include <dt-bindings/clock/rk3036-cru.h>
+#include "clk.h"
+
+#define RK3036_GRF_SOC_STATUS0	0x14c
+
+enum rk3036_plls {
+	apll, dpll, gpll,
+};
+
+static struct rockchip_pll_rate_table rk3036_pll_rates[] = {
+	/* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */
+	RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1584000000, 1, 66, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1560000000, 1, 65, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1536000000, 1, 64, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1512000000, 1, 63, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1488000000, 1, 62, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1464000000, 1, 61, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1440000000, 1, 60, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1416000000, 1, 59, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1392000000, 1, 58, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1368000000, 1, 57, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1344000000, 1, 56, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1320000000, 1, 55, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1296000000, 1, 54, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1272000000, 1, 53, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1248000000, 1, 52, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1200000000, 1, 50, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1188000000, 2, 99, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1100000000, 12, 550, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1000000000, 6, 500, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 984000000, 1, 82, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 960000000, 1, 80, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 936000000, 1, 78, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 912000000, 1, 76, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 900000000, 4, 300, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 888000000, 1, 74, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 864000000, 1, 72, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 840000000, 1, 70, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 816000000, 1, 68, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 800000000, 6, 400, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 700000000, 6, 350, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 696000000, 1, 58, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 600000000, 1, 75, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 594000000, 2, 99, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 504000000, 1, 63, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 500000000, 6, 250, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 408000000, 1, 68, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 312000000, 1, 52, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 216000000, 1, 72, 4, 2, 1, 0),
+	RK3036_PLL_RATE(  96000000, 1, 64, 4, 4, 1, 0),
+	{ /* sentinel */ },
+};
+
+#define RK3036_DIV_CPU_MASK		0x1f
+#define RK3036_DIV_CPU_SHIFT		8
+
+#define RK3036_DIV_PERI_MASK		0xf
+#define RK3036_DIV_PERI_SHIFT		0
+#define RK3036_DIV_ACLK_MASK		0x7
+#define RK3036_DIV_ACLK_SHIFT		4
+#define RK3036_DIV_HCLK_MASK		0x3
+#define RK3036_DIV_HCLK_SHIFT		8
+#define RK3036_DIV_PCLK_MASK		0x7
+#define RK3036_DIV_PCLK_SHIFT		12
+
+#define RK3036_CLKSEL1(_core_periph_div)					\
+	{									\
+		.reg = RK2928_CLKSEL_CON(1),					\
+		.val = HIWORD_UPDATE(_core_periph_div, RK3036_DIV_PERI_MASK,	\
+				RK3036_DIV_PERI_SHIFT)				\
+	}
+
+#define RK3036_CPUCLK_RATE(_prate, _core_periph_div)			\
+	{								\
+		.prate = _prate,					\
+		.divs = {						\
+			RK3036_CLKSEL1(_core_periph_div),		\
+		},							\
+	}
+
+static struct rockchip_cpuclk_rate_table rk3036_cpuclk_rates[] __initdata = {
+	RK3036_CPUCLK_RATE(816000000, 4),
+	RK3036_CPUCLK_RATE(600000000, 4),
+	RK3036_CPUCLK_RATE(312000000, 4),
+};
+
+static const struct rockchip_cpuclk_reg_data rk3036_cpuclk_data = {
+	.core_reg = RK2928_CLKSEL_CON(0),
+	.div_core_shift = 0,
+	.div_core_mask = 0x1f,
+	.mux_core_shift = 7,
+};
+
+PNAME(mux_pll_p)		= { "xin24m", "xin24m" };
+
+PNAME(mux_armclk_p)		= { "apll", "gpll_armclk" };
+PNAME(mux_busclk_p)		= { "apll", "dpll_cpu", "gpll_cpu" };
+PNAME(mux_ddrphy_p)		= { "dpll_ddr", "gpll_ddr" };
+PNAME(mux_pll_src_3plls_p)	= { "apll", "dpll", "gpll" };
+PNAME(mux_timer_p)		= { "xin24m", "pclk_peri_src" };
+
+PNAME(mux_pll_src_apll_dpll_gpll_usb480m_p)	= { "apll", "dpll", "gpll" "usb480m" };
+
+PNAME(mux_mmc_src_p)	= { "apll", "dpll", "gpll", "xin24m" };
+PNAME(mux_i2s_pre_p)	= { "i2s_src", "i2s_frac", "ext_i2s", "xin12m" };
+PNAME(mux_i2s_clkout_p)	= { "i2s_pre", "xin12m" };
+PNAME(mux_spdif_p)	= { "spdif_src", "spdif_frac", "xin12m" };
+PNAME(mux_uart0_p)	= { "uart0_src", "uart0_frac", "xin24m" };
+PNAME(mux_uart1_p)	= { "uart1_src", "uart1_frac", "xin24m" };
+PNAME(mux_uart2_p)	= { "uart2_src", "uart2_frac", "xin24m" };
+PNAME(mux_mac_p)	= { "mac_pll_src", "ext_gmac" };
+PNAME(mux_dclk_p)	= { "dclk_lcdc", "dclk_cru" };
+
+static struct rockchip_pll_clock rk3036_pll_clks[] __initdata = {
+	[apll] = PLL(pll_rk3036, PLL_APLL, "apll", mux_pll_p, 0, RK2928_PLL_CON(0),
+		     RK2928_MODE_CON, 0, 5, 0, rk3036_pll_rates),
+	[dpll] = PLL(pll_rk3036, PLL_DPLL, "dpll", mux_pll_p, 0, RK2928_PLL_CON(4),
+		     RK2928_MODE_CON, 4, 4, 0, NULL),
+	[gpll] = PLL(pll_rk3036, PLL_GPLL, "gpll", mux_pll_p, 0, RK2928_PLL_CON(12),
+		     RK2928_MODE_CON, 12, 6, ROCKCHIP_PLL_SYNC_RATE, rk3036_pll_rates),
+};
+
+#define MFLAGS CLK_MUX_HIWORD_MASK
+#define DFLAGS CLK_DIVIDER_HIWORD_MASK
+#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE)
+
+static struct rockchip_clk_branch rk3036_uart0_fracmux __initdata =
+	MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(13), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3036_uart1_fracmux __initdata =
+	MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(14), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3036_uart2_fracmux __initdata =
+	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(15), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3036_i2s_fracmux __initdata =
+	MUX(0, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3036_spdif_fracmux __initdata =
+	MUX(SCLK_SPDIF, "sclk_spdif", mux_spdif_p, 0,
+			RK2928_CLKSEL_CON(5), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = {
+	/*
+	 * Clock-Architecture Diagram 1
+	 */
+
+	GATE(0, "gpll_armclk", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 6, GFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 2
+	 */
+
+	GATE(0, "dpll_ddr", "dpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 2, GFLAGS),
+	GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 8, GFLAGS),
+	COMPOSITE_NOGATE(0, "ddrphy2x", mux_ddrphy_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(26), 8, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO),
+
+	COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(0), 7, GFLAGS),
+	COMPOSITE_NOMUX(0, "aclk_core_pre", "armclk", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(0), 7, GFLAGS),
+
+	GATE(0, "dpll_cpu", "dpll", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS),
+	GATE(0, "gpll_cpu", "gpll", 0, RK2928_CLKGATE_CON(0), 1, GFLAGS),
+	COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_busclk_p, 0,
+			RK2928_CLKSEL_CON(0), 14, 2, MFLAGS, 8, 5, DFLAGS),
+	GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 3, GFLAGS),
+	COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 12, 3, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(0), 5, GFLAGS),
+	COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 8, 2, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(0), 4, GFLAGS),
+
+	COMPOSITE(0, "aclk_peri_src", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(10), 14, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(2), 0, GFLAGS),
+
+	GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0,
+			RK2928_CLKGATE_CON(2), 1, GFLAGS),
+	DIV(0, "pclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO),
+	GATE(PCLK_PERI, "pclk_peri", "pclk_peri_src", 0,
+			RK2928_CLKGATE_CON(2), 3, GFLAGS),
+	DIV(0, "hclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO),
+	GATE(HCLK_PERI, "hclk_peri", "hclk_peri_src", 0,
+			RK2928_CLKGATE_CON(2), 2, GFLAGS),
+
+	COMPOSITE_NODIV(SCLK_TIMER0, "sclk_timer0", mux_timer_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(2), 4, 1, DFLAGS,
+			RK2928_CLKGATE_CON(1), 0, GFLAGS),
+	COMPOSITE_NODIV(SCLK_TIMER1, "sclk_timer1", mux_timer_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(2), 5, 1, DFLAGS,
+			RK2928_CLKGATE_CON(1), 1, GFLAGS),
+	COMPOSITE_NODIV(SCLK_TIMER2, "sclk_timer2", mux_timer_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(2), 6, 1, DFLAGS,
+			RK2928_CLKGATE_CON(2), 4, GFLAGS),
+	COMPOSITE_NODIV(SCLK_TIMER3, "sclk_timer3", mux_timer_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(2), 7, 1, DFLAGS,
+			RK2928_CLKGATE_CON(2), 5, GFLAGS),
+
+	MUX(0, "uart_pll_clk", mux_pll_src_apll_dpll_gpll_usb480m_p, 0,
+			RK2928_CLKSEL_CON(13), 10, 2, MFLAGS),
+	COMPOSITE_NOMUX(0, "uart0_src", "uart_pll_clk", 0,
+			RK2928_CLKSEL_CON(13), 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(1), 8, GFLAGS),
+	COMPOSITE_NOMUX(0, "uart1_src", "uart_pll_clk", 0,
+			RK2928_CLKSEL_CON(13), 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(1), 8, GFLAGS),
+	COMPOSITE_NOMUX(0, "uart2_src", "uart_pll_clk", 0,
+			RK2928_CLKSEL_CON(13), 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(1), 8, GFLAGS),
+	COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(17), 0,
+			RK2928_CLKGATE_CON(1), 9, GFLAGS,
+			&rk3036_uart0_fracmux),
+	COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(18), 0,
+			RK2928_CLKGATE_CON(1), 11, GFLAGS,
+			&rk3036_uart1_fracmux),
+	COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(19), 0,
+			RK2928_CLKGATE_CON(1), 13, GFLAGS,
+			&rk3036_uart2_fracmux),
+
+	COMPOSITE(0, "aclk_vcodec", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(32), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 11, GFLAGS),
+
+	COMPOSITE(0, "aclk_hvec", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(20), 0, 2, MFLAGS, 2, 5, DFLAGS,
+			RK2928_CLKGATE_CON(10), 6, GFLAGS),
+
+	COMPOSITE(0, "aclk_disp1_pre", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(31), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 4, GFLAGS),
+	COMPOSITE(0, "hclk_disp_pre", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(30), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(0), 11, GFLAGS),
+	COMPOSITE(SCLK_LCDC, "dclk_lcdc", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(28), 0, 2, MFLAGS, 8, 8, DFLAGS,
+			RK2928_CLKGATE_CON(3), 2, GFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_sdmmc_src", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(12), 8, 2, DFLAGS,
+			RK2928_CLKGATE_CON(2), 11, GFLAGS),
+	DIV(SCLK_SDMMC, "sclk_sdmmc", "sclk_sdmmc_src", 0,
+			RK2928_CLKSEL_CON(11), 0, 7, DFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_sdio_src", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(12), 10, 2, DFLAGS,
+			RK2928_CLKGATE_CON(2), 13, GFLAGS),
+	DIV(SCLK_SDIO, "sclk_sdio", "sclk_sdio_src", 0,
+			RK2928_CLKSEL_CON(11), 8, 7, DFLAGS),
+
+	COMPOSITE(SCLK_EMMC, "sclk_emmc", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(12), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(2), 14, GFLAGS),
+
+	MMC(SCLK_SDMMC_DRV,    "sdmmc_drv",    "sclk_sdmmc", RK3036_SDMMC_CON0, 1),
+	MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3036_SDMMC_CON1, 0),
+
+	MMC(SCLK_SDIO_DRV,     "sdio_drv",     "sclk_sdio",  RK3036_SDIO_CON0, 1),
+	MMC(SCLK_SDIO_SAMPLE,  "sdio_sample",  "sclk_sdio",  RK3036_SDIO_CON1, 0),
+
+	MMC(SCLK_EMMC_DRV,     "emmc_drv",     "sclk_emmc",  RK3036_EMMC_CON0,  1),
+	MMC(SCLK_EMMC_SAMPLE,  "emmc_sample",  "sclk_emmc",  RK3036_EMMC_CON1,  0),
+
+	COMPOSITE(0, "i2s_src", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(3), 14, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(0), 9, GFLAGS),
+	COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(7), 0,
+			RK2928_CLKGATE_CON(0), 10, GFLAGS,
+			&rk3036_i2s_fracmux),
+	COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_clkout", mux_i2s_clkout_p, 0,
+			RK2928_CLKSEL_CON(3), 12, 1, MFLAGS,
+			RK2928_CLKGATE_CON(0), 13, GFLAGS),
+	GATE(SCLK_I2S, "sclk_i2s", "i2s_pre", CLK_SET_RATE_PARENT,
+			RK2928_CLKGATE_CON(0), 14, GFLAGS),
+
+	COMPOSITE(0, "spdif_src", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(5), 10, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(2), 10, GFLAGS),
+	COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", 0,
+			RK2928_CLKSEL_CON(9), 0,
+			RK2928_CLKGATE_CON(2), 12, GFLAGS,
+			&rk3036_spdif_fracmux),
+
+	GATE(SCLK_OTGPHY0, "sclk_otgphy0", "xin12m", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(1), 5, GFLAGS),
+
+	COMPOSITE(SCLK_GPU, "sclk_gpu", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(34), 8, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 13, GFLAGS),
+
+	COMPOSITE(SCLK_SPI, "sclk_spi", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(25), 8, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(2), 9, GFLAGS),
+
+	COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(16), 8, 2, MFLAGS, 10, 5, DFLAGS,
+			RK2928_CLKGATE_CON(10), 4, GFLAGS),
+
+	COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_apll_dpll_gpll_usb480m_p, 0,
+			RK2928_CLKSEL_CON(16), 0, 2, MFLAGS, 2, 5, DFLAGS,
+			RK2928_CLKGATE_CON(10), 5, GFLAGS),
+
+	COMPOSITE_NOGATE(0, "mac_pll_src", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(21), 0, 2, MFLAGS, 4, 5, DFLAGS),
+	MUX(SCLK_MACREF, "mac_clk_ref", mux_mac_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(21), 3, 1, MFLAGS),
+
+	COMPOSITE_NOMUX(SCLK_MAC, "mac_clk", "mac_clk_ref", 0,
+			RK2928_CLKSEL_CON(21), 9, 5, DFLAGS,
+			RK2928_CLKGATE_CON(2), 6, GFLAGS),
+
+	MUX(SCLK_HDMI, "dclk_hdmi", mux_dclk_p, 0,
+			RK2928_CLKSEL_CON(31), 0, 1, MFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 3
+	 */
+
+	/* aclk_cpu gates */
+	GATE(0, "sclk_intmem", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 12, GFLAGS),
+	GATE(0, "aclk_strc_sys", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 10, GFLAGS),
+
+	/* hclk_cpu gates */
+	GATE(HCLK_ROM, "hclk_rom", "hclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 6, GFLAGS),
+
+	/* pclk_cpu gates */
+	GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS),
+	GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 7, GFLAGS),
+	GATE(PCLK_ACODEC, "pclk_acodec", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 14, GFLAGS),
+	GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS),
+
+	/* aclk_vio gates */
+	GATE(ACLK_VIO, "aclk_vio", "aclk_disp1_pre", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(6), 13, GFLAGS),
+	GATE(ACLK_LCDC, "aclk_lcdc", "aclk_disp1_pre", 0, RK2928_CLKGATE_CON(9), 6, GFLAGS),
+
+	GATE(HCLK_VIO_BUS, "hclk_vio_bus", "hclk_disp_pre", 0, RK2928_CLKGATE_CON(6), 12, GFLAGS),
+	GATE(HCLK_LCDC, "hclk_lcdc", "hclk_disp_pre", 0, RK2928_CLKGATE_CON(9), 5, GFLAGS),
+
+	/* hclk_video gates */
+	GATE(HCLK_VCODEC, "hclk_vcodec", "hclk_disp_pre", 0, RK2928_CLKGATE_CON(3), 12, GFLAGS),
+
+	/* xin24m gates */
+	GATE(SCLK_PVTM_CORE, "sclk_pvtm_core", "xin24m", 0, RK2928_CLKGATE_CON(10), 0, GFLAGS),
+	GATE(SCLK_PVTM_GPU, "sclk_pvtm_gpu", "xin24m", 0, RK2928_CLKGATE_CON(10), 1, GFLAGS),
+
+	/* aclk_peri gates */
+	GATE(0, "aclk_peri_axi_matrix", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 3, GFLAGS),
+	GATE(0, "aclk_cpu_peri", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 2, GFLAGS),
+	GATE(ACLK_DMAC2, "aclk_dmac2", "aclk_peri", 0, RK2928_CLKGATE_CON(5), 1, GFLAGS),
+	GATE(0, "aclk_peri_niu", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 15, GFLAGS),
+
+	/* hclk_peri gates */
+	GATE(0, "hclk_peri_matrix", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 0, GFLAGS),
+	GATE(0, "hclk_usb_peri", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 13, GFLAGS),
+	GATE(0, "hclk_peri_arbi", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 14, GFLAGS),
+	GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 9, GFLAGS),
+	GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 10, GFLAGS),
+	GATE(HCLK_SDIO, "hclk_sdio", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 11, GFLAGS),
+	GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 0, GFLAGS),
+	GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 13, GFLAGS),
+	GATE(HCLK_OTG1, "hclk_otg1", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(7), 3, GFLAGS),
+	GATE(HCLK_I2S, "hclk_i2s", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS),
+	GATE(0, "hclk_sfc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS),
+	GATE(0, "hclk_mac", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 15, GFLAGS),
+
+	/* pclk_peri gates */
+	GATE(0, "pclk_peri_matrix", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 1, GFLAGS),
+	GATE(0, "pclk_efuse", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 2, GFLAGS),
+	GATE(PCLK_TIMER, "pclk_timer", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 7, GFLAGS),
+	GATE(PCLK_PWM, "pclk_pwm", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 10, GFLAGS),
+	GATE(PCLK_SPI, "pclk_spi", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 12, GFLAGS),
+	GATE(PCLK_WDT, "pclk_wdt", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 15, GFLAGS),
+	GATE(PCLK_UART0, "pclk_uart0", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS),
+	GATE(PCLK_UART1, "pclk_uart1", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS),
+	GATE(PCLK_UART2, "pclk_uart2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 2, GFLAGS),
+	GATE(PCLK_I2C0, "pclk_i2c0", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 4, GFLAGS),
+	GATE(PCLK_I2C1, "pclk_i2c1", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 5, GFLAGS),
+	GATE(PCLK_I2C2, "pclk_i2c2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 6, GFLAGS),
+	GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS),
+	GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 10, GFLAGS),
+	GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS),
+};
+
+static const char *const rk3036_critical_clocks[] __initconst = {
+	"aclk_cpu",
+	"aclk_peri",
+	"hclk_peri",
+	"pclk_peri",
+};
+
+static void __init rk3036_clk_init(struct device_node *np)
+{
+	void __iomem *reg_base;
+	struct clk *clk;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: could not map cru region\n", __func__);
+		return;
+	}
+
+	rockchip_clk_init(np, reg_base, CLK_NR_CLKS);
+
+	/* xin12m is created by an cru-internal divider */
+	clk = clk_register_fixed_factor(NULL, "xin12m", "xin24m", 0, 1, 2);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock xin12m: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "usb480m", "xin24m", 0, 20, 1);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock usb480m: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "ddrphy", "ddrphy2x", 0, 1, 2);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock ddrphy: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "hclk_vcodec_pre",
+					"aclk_vcodec", 0, 1, 4);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock hclk_vcodec_pre: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "sclk_macref_out",
+					"hclk_peri_src", 0, 1, 2);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock sclk_macref_out: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	rockchip_clk_register_plls(rk3036_pll_clks,
+				   ARRAY_SIZE(rk3036_pll_clks),
+				   RK3036_GRF_SOC_STATUS0);
+	rockchip_clk_register_branches(rk3036_clk_branches,
+				  ARRAY_SIZE(rk3036_clk_branches));
+	rockchip_clk_protect_critical(rk3036_critical_clocks,
+				      ARRAY_SIZE(rk3036_critical_clocks));
+
+	rockchip_clk_register_armclk(ARMCLK, "armclk",
+			mux_armclk_p, ARRAY_SIZE(mux_armclk_p),
+			&rk3036_cpuclk_data, rk3036_cpuclk_rates,
+			ARRAY_SIZE(rk3036_cpuclk_rates));
+
+	rockchip_register_softrst(np, 9, reg_base + RK2928_SOFTRST_CON(0),
+				  ROCKCHIP_SOFTRST_HIWORD_MASK);
+
+	rockchip_register_restart_notifier(RK2928_GLB_SRST_FST, NULL);
+}
+CLK_OF_DECLARE(rk3036_cru, "rockchip,rk3036-cru", rk3036_clk_init);
diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c
index abb47608713b..7f7444cbf6fc 100644
--- a/drivers/clk/rockchip/clk-rk3188.c
+++ b/drivers/clk/rockchip/clk-rk3188.c
@@ -247,6 +247,30 @@ static struct clk_div_table div_core_peri_t[] = {
 	{ /* sentinel */ },
 };
 
+static struct rockchip_clk_branch common_hsadc_out_fracmux __initdata =
+	MUX(0, "sclk_hsadc_out", mux_sclk_hsadc_p, 0,
+			RK2928_CLKSEL_CON(22), 4, 2, MFLAGS);
+
+static struct rockchip_clk_branch common_spdif_fracmux __initdata =
+	MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(5), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch common_uart0_fracmux __initdata =
+	MUX(SCLK_UART0, "sclk_uart0", mux_sclk_uart0_p, 0,
+			RK2928_CLKSEL_CON(13), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch common_uart1_fracmux __initdata =
+	MUX(SCLK_UART1, "sclk_uart1", mux_sclk_uart1_p, 0,
+			RK2928_CLKSEL_CON(14), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch common_uart2_fracmux __initdata =
+	MUX(SCLK_UART2, "sclk_uart2", mux_sclk_uart2_p, 0,
+			RK2928_CLKSEL_CON(15), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch common_uart3_fracmux __initdata =
+	MUX(SCLK_UART3, "sclk_uart3", mux_sclk_uart3_p, 0,
+			RK2928_CLKSEL_CON(16), 8, 2, MFLAGS);
+
 static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	/*
 	 * Clock-Architecture Diagram 2
@@ -335,11 +359,10 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	COMPOSITE(0, "hsadc_src", mux_pll_src_gpll_cpll_p, 0,
 			RK2928_CLKSEL_CON(22), 0, 1, MFLAGS, 8, 8, DFLAGS,
 			RK2928_CLKGATE_CON(2), 6, GFLAGS),
-	COMPOSITE_FRAC(0, "hsadc_frac", "hsadc_src", 0,
+	COMPOSITE_FRACMUX(0, "hsadc_frac", "hsadc_src", 0,
 			RK2928_CLKSEL_CON(23), 0,
-			RK2928_CLKGATE_CON(2), 7, GFLAGS),
-	MUX(0, "sclk_hsadc_out", mux_sclk_hsadc_p, 0,
-			RK2928_CLKSEL_CON(22), 4, 2, MFLAGS),
+			RK2928_CLKGATE_CON(2), 7, GFLAGS,
+			&common_hsadc_out_fracmux),
 	INVERTER(SCLK_HSADC, "sclk_hsadc", "sclk_hsadc_out",
 			RK2928_CLKSEL_CON(22), 7, IFLAGS),
 
@@ -350,11 +373,10 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "spdif_pre", "i2s_src", 0,
 			RK2928_CLKSEL_CON(5), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(0), 13, GFLAGS),
-	COMPOSITE_FRAC(0, "spdif_frac", "spdif_pre", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_pll", CLK_SET_RATE_PARENT,
 			RK2928_CLKSEL_CON(9), 0,
-			RK2928_CLKGATE_CON(0), 14, GFLAGS),
-	MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, CLK_SET_RATE_PARENT,
-			RK2928_CLKSEL_CON(5), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(0), 14, GFLAGS,
+			&common_spdif_fracmux),
 
 	/*
 	 * Clock-Architecture Diagram 4
@@ -385,35 +407,31 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "uart0_pre", "uart_src", 0,
 			RK2928_CLKSEL_CON(13), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(1), 8, GFLAGS),
-	COMPOSITE_FRAC(0, "uart0_frac", "uart0_pre", 0,
+	COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_pre", 0,
 			RK2928_CLKSEL_CON(17), 0,
-			RK2928_CLKGATE_CON(1), 9, GFLAGS),
-	MUX(SCLK_UART0, "sclk_uart0", mux_sclk_uart0_p, 0,
-			RK2928_CLKSEL_CON(13), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(1), 9, GFLAGS,
+			&common_uart0_fracmux),
 	COMPOSITE_NOMUX(0, "uart1_pre", "uart_src", 0,
 			RK2928_CLKSEL_CON(14), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(1), 10, GFLAGS),
-	COMPOSITE_FRAC(0, "uart1_frac", "uart1_pre", 0,
+	COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_pre", 0,
 			RK2928_CLKSEL_CON(18), 0,
-			RK2928_CLKGATE_CON(1), 11, GFLAGS),
-	MUX(SCLK_UART1, "sclk_uart1", mux_sclk_uart1_p, 0,
-			RK2928_CLKSEL_CON(14), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(1), 11, GFLAGS,
+			&common_uart1_fracmux),
 	COMPOSITE_NOMUX(0, "uart2_pre", "uart_src", 0,
 			RK2928_CLKSEL_CON(15), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(1), 12, GFLAGS),
-	COMPOSITE_FRAC(0, "uart2_frac", "uart2_pre", 0,
+	COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_pre", 0,
 			RK2928_CLKSEL_CON(19), 0,
-			RK2928_CLKGATE_CON(1), 13, GFLAGS),
-	MUX(SCLK_UART2, "sclk_uart2", mux_sclk_uart2_p, 0,
-			RK2928_CLKSEL_CON(15), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(1), 13, GFLAGS,
+			&common_uart2_fracmux),
 	COMPOSITE_NOMUX(0, "uart3_pre", "uart_src", 0,
 			RK2928_CLKSEL_CON(16), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(1), 14, GFLAGS),
-	COMPOSITE_FRAC(0, "uart3_frac", "uart3_pre", 0,
+	COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_pre", 0,
 			RK2928_CLKSEL_CON(20), 0,
-			RK2928_CLKGATE_CON(1), 15, GFLAGS),
-	MUX(SCLK_UART3, "sclk_uart3", mux_sclk_uart3_p, 0,
-			RK2928_CLKSEL_CON(16), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(1), 15, GFLAGS,
+			&common_uart3_fracmux),
 
 	GATE(SCLK_JTAG, "jtag", "ext_jtag", 0, RK2928_CLKGATE_CON(1), 3, GFLAGS),
 
@@ -523,6 +541,18 @@ static struct clk_div_table div_aclk_cpu_t[] = {
 	{ /* sentinel */ },
 };
 
+static struct rockchip_clk_branch rk3066a_i2s0_fracmux __initdata =
+	MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0,
+			RK2928_CLKSEL_CON(2), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3066a_i2s1_fracmux __initdata =
+	MUX(SCLK_I2S1, "sclk_i2s1", mux_sclk_i2s1_p, 0,
+			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3066a_i2s2_fracmux __initdata =
+	MUX(SCLK_I2S2, "sclk_i2s2", mux_sclk_i2s2_p, 0,
+			RK2928_CLKSEL_CON(4), 8, 2, MFLAGS);
+
 static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = {
 	DIVTBL(0, "aclk_cpu_pre", "armclk", 0,
 			RK2928_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, div_aclk_cpu_t),
@@ -584,27 +614,24 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "i2s0_pre", "i2s_src", 0,
 			RK2928_CLKSEL_CON(2), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(0), 7, GFLAGS),
-	COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_pre", 0,
+	COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", 0,
 			RK2928_CLKSEL_CON(6), 0,
-			RK2928_CLKGATE_CON(0), 8, GFLAGS),
-	MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0,
-			RK2928_CLKSEL_CON(2), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(0), 8, GFLAGS,
+			&rk3066a_i2s0_fracmux),
 	COMPOSITE_NOMUX(0, "i2s1_pre", "i2s_src", 0,
 			RK2928_CLKSEL_CON(3), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(0), 9, GFLAGS),
-	COMPOSITE_FRAC(0, "i2s1_frac", "i2s1_pre", 0,
+	COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_pre", 0,
 			RK2928_CLKSEL_CON(7), 0,
-			RK2928_CLKGATE_CON(0), 10, GFLAGS),
-	MUX(SCLK_I2S1, "sclk_i2s1", mux_sclk_i2s1_p, 0,
-			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(0), 10, GFLAGS,
+			&rk3066a_i2s1_fracmux),
 	COMPOSITE_NOMUX(0, "i2s2_pre", "i2s_src", 0,
 			RK2928_CLKSEL_CON(4), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(0), 11, GFLAGS),
-	COMPOSITE_FRAC(0, "i2s2_frac", "i2s2_pre", 0,
+	COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_pre", 0,
 			RK2928_CLKSEL_CON(8), 0,
-			RK2928_CLKGATE_CON(0), 12, GFLAGS),
-	MUX(SCLK_I2S2, "sclk_i2s2", mux_sclk_i2s2_p, 0,
-			RK2928_CLKSEL_CON(4), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(0), 12, GFLAGS,
+			&rk3066a_i2s2_fracmux),
 
 	GATE(HCLK_I2S1, "hclk_i2s1", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS),
 	GATE(HCLK_I2S2, "hclk_i2s2", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS),
@@ -638,6 +665,10 @@ static struct clk_div_table div_rk3188_aclk_core_t[] = {
 PNAME(mux_hsicphy_p)		= { "sclk_otgphy0", "sclk_otgphy1",
 				    "gpll", "cpll" };
 
+static struct rockchip_clk_branch rk3188_i2s0_fracmux __initdata =
+	MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0,
+			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS);
+
 static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX_DIVTBL(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED,
 			RK2928_CLKSEL_CON(1), 3, 3, DFLAGS | CLK_DIVIDER_READ_ONLY,
@@ -691,11 +722,10 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "i2s0_pre", "i2s_src", 0,
 			RK2928_CLKSEL_CON(3), 0, 7, DFLAGS,
 			RK2928_CLKGATE_CON(0), 9, GFLAGS),
-	COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_pre", 0,
+	COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", 0,
 			RK2928_CLKSEL_CON(7), 0,
-			RK2928_CLKGATE_CON(0), 10, GFLAGS),
-	MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0,
-			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS),
+			RK2928_CLKGATE_CON(0), 10, GFLAGS,
+			&rk3188_i2s0_fracmux),
 
 	GATE(0, "hclk_imem0", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS),
 	GATE(0, "hclk_imem1", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 15, GFLAGS),
@@ -750,7 +780,7 @@ static void __init rk3188_common_clk_init(struct device_node *np)
 	rockchip_register_softrst(np, 9, reg_base + RK2928_SOFTRST_CON(0),
 				  ROCKCHIP_SOFTRST_HIWORD_MASK);
 
-	rockchip_register_restart_notifier(RK2928_GLB_SRST_FST);
+	rockchip_register_restart_notifier(RK2928_GLB_SRST_FST, NULL);
 }
 
 static void __init rk3066a_clk_init(struct device_node *np)
diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c
new file mode 100644
index 000000000000..981a50205339
--- /dev/null
+++ b/drivers/clk/rockchip/clk-rk3228.c
@@ -0,0 +1,678 @@
+/*
+ * Copyright (c) 2015 Rockchip Electronics Co. Ltd.
+ * Author: Xing Zheng <zhengxing@rock-chips.com>
+ *         Jeffy Chen <jeffy.chen@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/syscore_ops.h>
+#include <dt-bindings/clock/rk3228-cru.h>
+#include "clk.h"
+
+#define RK3228_GRF_SOC_STATUS0	0x480
+
+enum rk3228_plls {
+	apll, dpll, cpll, gpll,
+};
+
+static struct rockchip_pll_rate_table rk3228_pll_rates[] = {
+	/* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */
+	RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1584000000, 1, 66, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1560000000, 1, 65, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1536000000, 1, 64, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1512000000, 1, 63, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1488000000, 1, 62, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1464000000, 1, 61, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1440000000, 1, 60, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1416000000, 1, 59, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1392000000, 1, 58, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1368000000, 1, 57, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1344000000, 1, 56, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1320000000, 1, 55, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1296000000, 1, 54, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1272000000, 1, 53, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1248000000, 1, 52, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1200000000, 1, 50, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1188000000, 2, 99, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1100000000, 12, 550, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1000000000, 6, 500, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 984000000, 1, 82, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 960000000, 1, 80, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 936000000, 1, 78, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 912000000, 1, 76, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 900000000, 4, 300, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 888000000, 1, 74, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 864000000, 1, 72, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 840000000, 1, 70, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 816000000, 1, 68, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 800000000, 6, 400, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 700000000, 6, 350, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 696000000, 1, 58, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 600000000, 1, 75, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 594000000, 2, 99, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 504000000, 1, 63, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 500000000, 6, 250, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 408000000, 1, 68, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 312000000, 1, 52, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 216000000, 1, 72, 4, 2, 1, 0),
+	RK3036_PLL_RATE(  96000000, 1, 64, 4, 4, 1, 0),
+	{ /* sentinel */ },
+};
+
+#define RK3228_DIV_CPU_MASK		0x1f
+#define RK3228_DIV_CPU_SHIFT		8
+
+#define RK3228_DIV_PERI_MASK		0xf
+#define RK3228_DIV_PERI_SHIFT		0
+#define RK3228_DIV_ACLK_MASK		0x7
+#define RK3228_DIV_ACLK_SHIFT		4
+#define RK3228_DIV_HCLK_MASK		0x3
+#define RK3228_DIV_HCLK_SHIFT		8
+#define RK3228_DIV_PCLK_MASK		0x7
+#define RK3228_DIV_PCLK_SHIFT		12
+
+#define RK3228_CLKSEL1(_core_peri_div)					\
+	{									\
+		.reg = RK2928_CLKSEL_CON(1),					\
+		.val = HIWORD_UPDATE(_core_peri_div, RK3228_DIV_PERI_MASK,	\
+				RK3228_DIV_PERI_SHIFT)				\
+	}
+
+#define RK3228_CPUCLK_RATE(_prate, _core_peri_div)			\
+	{								\
+		.prate = _prate,					\
+		.divs = {						\
+			RK3228_CLKSEL1(_core_peri_div),		\
+		},							\
+	}
+
+static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = {
+	RK3228_CPUCLK_RATE(816000000, 4),
+	RK3228_CPUCLK_RATE(600000000, 4),
+	RK3228_CPUCLK_RATE(312000000, 4),
+};
+
+static const struct rockchip_cpuclk_reg_data rk3228_cpuclk_data = {
+	.core_reg = RK2928_CLKSEL_CON(0),
+	.div_core_shift = 0,
+	.div_core_mask = 0x1f,
+	.mux_core_shift = 6,
+};
+
+PNAME(mux_pll_p)		= { "clk_24m", "xin24m" };
+
+PNAME(mux_ddrphy_p)		= { "dpll_ddr", "gpll_ddr", "apll_ddr" };
+PNAME(mux_armclk_p)		= { "apll_core", "gpll_core", "dpll_core" };
+PNAME(mux_usb480m_phy_p)	= { "usb480m_phy0", "usb480m_phy1" };
+PNAME(mux_usb480m_p)		= { "usb480m_phy", "xin24m" };
+PNAME(mux_hdmiphy_p)		= { "hdmiphy_phy", "xin24m" };
+PNAME(mux_aclk_cpu_src_p)	= { "cpll_aclk_cpu", "gpll_aclk_cpu", "hdmiphy_aclk_cpu" };
+
+PNAME(mux_pll_src_4plls_p)	= { "cpll", "gpll", "hdmiphy" "usb480m" };
+PNAME(mux_pll_src_3plls_p)	= { "cpll", "gpll", "hdmiphy" };
+PNAME(mux_pll_src_2plls_p)	= { "cpll", "gpll" };
+PNAME(mux_sclk_hdmi_cec_p)	= { "cpll", "gpll", "xin24m" };
+PNAME(mux_aclk_peri_src_p)	= { "cpll_peri", "gpll_peri", "hdmiphy_peri" };
+PNAME(mux_mmc_src_p)		= { "cpll", "gpll", "xin24m", "usb480m" };
+PNAME(mux_pll_src_cpll_gpll_usb480m_p)	= { "cpll", "gpll", "usb480m" };
+
+PNAME(mux_sclk_rga_p)		= { "gpll", "cpll", "sclk_rga_src" };
+
+PNAME(mux_sclk_vop_src_p)	= { "gpll_vop", "cpll_vop" };
+PNAME(mux_dclk_vop_p)		= { "hdmiphy", "sclk_vop_pre" };
+
+PNAME(mux_i2s0_p)		= { "i2s0_src", "i2s0_frac", "ext_i2s", "xin12m" };
+PNAME(mux_i2s1_pre_p)		= { "i2s1_src", "i2s1_frac", "ext_i2s", "xin12m" };
+PNAME(mux_i2s_out_p)		= { "i2s1_pre", "xin12m" };
+PNAME(mux_i2s2_p)		= { "i2s2_src", "i2s2_frac", "xin12m" };
+PNAME(mux_sclk_spdif_p)		= { "sclk_spdif_src", "spdif_frac", "xin12m" };
+
+PNAME(mux_aclk_gpu_pre_p)	= { "cpll_gpu", "gpll_gpu", "hdmiphy_gpu", "usb480m_gpu" };
+
+PNAME(mux_uart0_p)		= { "uart0_src", "uart0_frac", "xin24m" };
+PNAME(mux_uart1_p)		= { "uart1_src", "uart1_frac", "xin24m" };
+PNAME(mux_uart2_p)		= { "uart2_src", "uart2_frac", "xin24m" };
+
+PNAME(mux_sclk_macphy_50m_p)	= { "ext_gmac", "phy_50m_out" };
+PNAME(mux_sclk_gmac_pre_p)	= { "sclk_gmac_src", "sclk_macphy_50m" };
+PNAME(mux_sclk_macphy_p)	= { "sclk_gmac_src", "ext_gmac" };
+
+static struct rockchip_pll_clock rk3228_pll_clks[] __initdata = {
+	[apll] = PLL(pll_rk3036, PLL_APLL, "apll", mux_pll_p, 0, RK2928_PLL_CON(0),
+		     RK2928_MODE_CON, 0, 7, 0, rk3228_pll_rates),
+	[dpll] = PLL(pll_rk3036, PLL_DPLL, "dpll", mux_pll_p, 0, RK2928_PLL_CON(3),
+		     RK2928_MODE_CON, 4, 6, 0, NULL),
+	[cpll] = PLL(pll_rk3036, PLL_CPLL, "cpll", mux_pll_p, 0, RK2928_PLL_CON(6),
+		     RK2928_MODE_CON, 8, 8, 0, NULL),
+	[gpll] = PLL(pll_rk3036, PLL_GPLL, "gpll", mux_pll_p, 0, RK2928_PLL_CON(9),
+		     RK2928_MODE_CON, 12, 9, ROCKCHIP_PLL_SYNC_RATE, rk3228_pll_rates),
+};
+
+#define MFLAGS CLK_MUX_HIWORD_MASK
+#define DFLAGS CLK_DIVIDER_HIWORD_MASK
+#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE)
+
+static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = {
+	/*
+	 * Clock-Architecture Diagram 1
+	 */
+
+	DIV(0, "clk_24m", "xin24m", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(4), 8, 5, DFLAGS),
+
+	/* PD_DDR */
+	GATE(0, "apll_ddr", "apll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 2, GFLAGS),
+	GATE(0, "dpll_ddr", "dpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 2, GFLAGS),
+	GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 2, GFLAGS),
+	COMPOSITE(0, "ddrphy4x", mux_ddrphy_p, CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(26), 8, 2, MFLAGS, 0, 3, DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
+			RK2928_CLKGATE_CON(7), 1, GFLAGS),
+	GATE(0, "ddrc", "ddrphy_pre", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(8), 5, GFLAGS),
+	GATE(0, "ddrphy", "ddrphy_pre", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(7), 0, GFLAGS),
+
+	/* PD_CORE */
+	GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 6, GFLAGS),
+	GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 6, GFLAGS),
+	GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 6, GFLAGS),
+	COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(4), 1, GFLAGS),
+	COMPOSITE_NOMUX(0, "armcore", "armclk", CLK_IGNORE_UNUSED,
+			RK2928_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK2928_CLKGATE_CON(4), 0, GFLAGS),
+
+	/* PD_MISC */
+	MUX(0, "hdmiphy", mux_hdmiphy_p, CLK_SET_RATE_PARENT,
+			RK2928_MISC_CON, 13, 1, MFLAGS),
+	MUX(0, "usb480m_phy", mux_usb480m_phy_p, CLK_SET_RATE_PARENT,
+			RK2928_MISC_CON, 14, 1, MFLAGS),
+	MUX(0, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT,
+			RK2928_MISC_CON, 15, 1, MFLAGS),
+
+	/* PD_BUS */
+	GATE(0, "hdmiphy_aclk_cpu", "hdmiphy", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 1, GFLAGS),
+	GATE(0, "gpll_aclk_cpu", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 1, GFLAGS),
+	GATE(0, "cpll_aclk_cpu", "cpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(0), 1, GFLAGS),
+	COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0,
+			RK2928_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS),
+	GATE(ARMCLK, "aclk_cpu", "aclk_cpu_src", 0,
+			RK2928_CLKGATE_CON(6), 0, GFLAGS),
+	COMPOSITE_NOMUX(0, "hclk_cpu", "aclk_cpu_src", 0,
+			RK2928_CLKSEL_CON(1), 8, 2, DFLAGS,
+			RK2928_CLKGATE_CON(6), 1, GFLAGS),
+	COMPOSITE_NOMUX(0, "pclk_bus_src", "aclk_cpu_src", 0,
+			RK2928_CLKSEL_CON(1), 12, 3, DFLAGS,
+			RK2928_CLKGATE_CON(6), 2, GFLAGS),
+	GATE(0, "pclk_cpu", "pclk_bus_src", 0,
+			RK2928_CLKGATE_CON(6), 3, GFLAGS),
+	GATE(0, "pclk_phy_pre", "pclk_bus_src", 0,
+			RK2928_CLKGATE_CON(6), 4, GFLAGS),
+	GATE(0, "pclk_ddr_pre", "pclk_bus_src", 0,
+			RK2928_CLKGATE_CON(6), 13, GFLAGS),
+
+	/* PD_VIDEO */
+	COMPOSITE(0, "aclk_vpu_pre", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(32), 5, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 11, GFLAGS),
+	GATE(0, "hclk_vpu_src", "aclk_vpu_pre", 0,
+			RK2928_CLKGATE_CON(4), 4, GFLAGS),
+
+	COMPOSITE(0, "aclk_rkvdec_pre", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 2, GFLAGS),
+	GATE(0, "hclk_rkvdec_src", "aclk_rkvdec_pre", 0,
+			RK2928_CLKGATE_CON(4), 5, GFLAGS),
+
+	COMPOSITE(0, "sclk_vdec_cabac", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(28), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 3, GFLAGS),
+
+	COMPOSITE(0, "sclk_vdec_core", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(34), 13, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 4, GFLAGS),
+
+	/* PD_VIO */
+	COMPOSITE(0, "aclk_iep_pre", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(31), 5, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 0, GFLAGS),
+	DIV(0, "hclk_vio_pre", "aclk_iep_pre", 0,
+			RK2928_CLKSEL_CON(2), 0, 5, DFLAGS),
+
+	COMPOSITE(0, "aclk_hdcp_pre", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(31), 13, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 4, GFLAGS),
+
+	MUX(0, "sclk_rga_src", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(33), 13, 2, MFLAGS),
+	COMPOSITE_NOMUX(0, "aclk_rga_pre", "sclk_rga_src", 0,
+			RK2928_CLKSEL_CON(33), 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 2, GFLAGS),
+	COMPOSITE(0, "sclk_rga", mux_sclk_rga_p, 0,
+			RK2928_CLKSEL_CON(22), 5, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 6, GFLAGS),
+
+	COMPOSITE(0, "aclk_vop_pre", mux_pll_src_4plls_p, 0,
+			RK2928_CLKSEL_CON(33), 5, 2, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 1, GFLAGS),
+
+	COMPOSITE(0, "sclk_hdcp", mux_pll_src_3plls_p, 0,
+			RK2928_CLKSEL_CON(23), 14, 2, MFLAGS, 8, 6, DFLAGS,
+			RK2928_CLKGATE_CON(3), 5, GFLAGS),
+
+	GATE(0, "sclk_hdmi_hdcp", "xin24m", 0,
+			RK2928_CLKGATE_CON(3), 7, GFLAGS),
+
+	COMPOSITE(0, "sclk_hdmi_cec", mux_sclk_hdmi_cec_p, 0,
+			RK2928_CLKSEL_CON(21), 14, 2, MFLAGS, 0, 14, DFLAGS,
+			RK2928_CLKGATE_CON(3), 8, GFLAGS),
+
+	/* PD_PERI */
+	GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(2), 0, GFLAGS),
+	GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(2), 0, GFLAGS),
+	GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IGNORE_UNUSED,
+			RK2928_CLKGATE_CON(2), 0, GFLAGS),
+	COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, 0,
+			RK2928_CLKSEL_CON(10), 10, 2, MFLAGS, 0, 5, DFLAGS),
+	COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0,
+			RK2928_CLKSEL_CON(10), 12, 3, DFLAGS,
+			RK2928_CLKGATE_CON(5), 2, GFLAGS),
+	COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", 0,
+			RK2928_CLKSEL_CON(10), 8, 2, DFLAGS,
+			RK2928_CLKGATE_CON(5), 1, GFLAGS),
+	GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0,
+			RK2928_CLKGATE_CON(5), 0, GFLAGS),
+
+	GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 5, GFLAGS),
+	GATE(SCLK_TIMER1, "sclk_timer1", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 6, GFLAGS),
+	GATE(SCLK_TIMER2, "sclk_timer2", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 7, GFLAGS),
+	GATE(SCLK_TIMER3, "sclk_timer3", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 8, GFLAGS),
+	GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 9, GFLAGS),
+	GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0,
+			RK2928_CLKGATE_CON(6), 10, GFLAGS),
+
+	COMPOSITE(0, "sclk_crypto", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(24), 5, 1, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(2), 7, GFLAGS),
+
+	COMPOSITE(0, "sclk_tsp", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(22), 15, 1, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(2), 6, GFLAGS),
+
+	GATE(0, "sclk_hsadc", "ext_hsadc", 0,
+			RK3288_CLKGATE_CON(10), 12, GFLAGS),
+
+	COMPOSITE(0, "sclk_wifi", mux_pll_src_cpll_gpll_usb480m_p, 0,
+			RK2928_CLKSEL_CON(23), 5, 2, MFLAGS, 0, 6, DFLAGS,
+			RK2928_CLKGATE_CON(2), 15, GFLAGS),
+
+	COMPOSITE(SCLK_SDMMC, "sclk_sdmmc0", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(11), 8, 2, MFLAGS, 0, 8, DFLAGS,
+			RK2928_CLKGATE_CON(2), 11, GFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_sdio_src", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(11), 10, 2, MFLAGS,
+			RK2928_CLKGATE_CON(2), 13, GFLAGS),
+	DIV(SCLK_SDIO, "sclk_sdio", "sclk_sdio_src", 0,
+			RK2928_CLKSEL_CON(12), 0, 8, DFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_emmc_src", mux_mmc_src_p, 0,
+			RK2928_CLKSEL_CON(11), 12, 2, MFLAGS,
+			RK2928_CLKGATE_CON(2), 14, GFLAGS),
+	DIV(SCLK_EMMC, "sclk_emmc", "sclk_emmc_src", 0,
+			RK2928_CLKSEL_CON(12), 8, 8, DFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 2
+	 */
+
+	GATE(0, "gpll_vop", "gpll", 0,
+			RK2928_CLKGATE_CON(3), 1, GFLAGS),
+	GATE(0, "cpll_vop", "cpll", 0,
+			RK2928_CLKGATE_CON(3), 1, GFLAGS),
+	MUX(0, "sclk_vop_src", mux_sclk_vop_src_p, 0,
+			RK2928_CLKSEL_CON(27), 0, 1, MFLAGS),
+	DIV(0, "dclk_hdmiphy", "sclk_vop_src", 0,
+			RK2928_CLKSEL_CON(29), 0, 3, DFLAGS),
+	DIV(0, "sclk_vop_pre", "sclk_vop_src", 0,
+			RK2928_CLKSEL_CON(27), 8, 8, DFLAGS),
+	MUX(0, "dclk_vop", mux_dclk_vop_p, 0,
+			RK2928_CLKSEL_CON(27), 1, 1, MFLAGS),
+
+	COMPOSITE(0, "i2s0_src", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(9), 15, 1, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(0), 3, GFLAGS),
+	COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(8), 0,
+			RK3288_CLKGATE_CON(0), 4, GFLAGS),
+	COMPOSITE_NODIV(SCLK_I2S0, "sclk_i2s0", mux_i2s0_p, 0,
+			RK2928_CLKSEL_CON(9), 8, 2, MFLAGS,
+			RK2928_CLKGATE_CON(0), 5, GFLAGS),
+
+	COMPOSITE(0, "i2s1_src", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(3), 15, 1, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(0), 10, GFLAGS),
+	COMPOSITE_FRAC(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(7), 0,
+			RK3288_CLKGATE_CON(0), 11, GFLAGS),
+	MUX(0, "i2s1_pre", mux_i2s1_pre_p, 0,
+			RK2928_CLKSEL_CON(3), 8, 2, MFLAGS),
+	GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", 0,
+			RK2928_CLKGATE_CON(0), 14, GFLAGS),
+	COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0,
+			RK2928_CLKSEL_CON(3), 12, 1, MFLAGS,
+			RK2928_CLKGATE_CON(0), 13, GFLAGS),
+
+	COMPOSITE(0, "i2s2_src", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(16), 15, 1, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(0), 7, GFLAGS),
+	COMPOSITE_FRAC(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(30), 0,
+			RK3288_CLKGATE_CON(0), 8, GFLAGS),
+	COMPOSITE_NODIV(SCLK_I2S2, "sclk_i2s2", mux_i2s2_p, 0,
+			RK2928_CLKSEL_CON(16), 8, 2, MFLAGS,
+			RK2928_CLKGATE_CON(0), 9, GFLAGS),
+
+	COMPOSITE(0, "sclk_spdif_src", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(6), 15, 1, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(2), 10, GFLAGS),
+	COMPOSITE_FRAC(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(20), 0,
+			RK3288_CLKGATE_CON(2), 12, GFLAGS),
+	MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, 0,
+			RK2928_CLKSEL_CON(6), 8, 2, MFLAGS),
+
+	GATE(0, "jtag", "ext_jtag", 0,
+			RK2928_CLKGATE_CON(1), 3, GFLAGS),
+
+	GATE(0, "sclk_otgphy0", "xin24m", 0,
+			RK2928_CLKGATE_CON(1), 5, GFLAGS),
+	GATE(0, "sclk_otgphy1", "xin24m", 0,
+			RK2928_CLKGATE_CON(1), 6, GFLAGS),
+
+	COMPOSITE_NOMUX(0, "sclk_tsadc", "xin24m", 0,
+			RK2928_CLKSEL_CON(24), 6, 10, DFLAGS,
+			RK2928_CLKGATE_CON(2), 8, GFLAGS),
+
+	GATE(0, "cpll_gpu", "cpll", 0,
+			RK2928_CLKGATE_CON(3), 13, GFLAGS),
+	GATE(0, "gpll_gpu", "gpll", 0,
+			RK2928_CLKGATE_CON(3), 13, GFLAGS),
+	GATE(0, "hdmiphy_gpu", "hdmiphy", 0,
+			RK2928_CLKGATE_CON(3), 13, GFLAGS),
+	GATE(0, "usb480m_gpu", "usb480m", 0,
+			RK2928_CLKGATE_CON(3), 13, GFLAGS),
+	COMPOSITE_NOGATE(0, "aclk_gpu_pre", mux_aclk_gpu_pre_p, 0,
+			RK2928_CLKSEL_CON(34), 5, 2, MFLAGS, 0, 5, DFLAGS),
+
+	COMPOSITE(SCLK_SPI0, "sclk_spi0", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(25), 8, 1, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(2), 9, GFLAGS),
+
+	/* PD_UART */
+	COMPOSITE(0, "uart0_src", mux_pll_src_cpll_gpll_usb480m_p, 0,
+			RK2928_CLKSEL_CON(13), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(1), 8, GFLAGS),
+	COMPOSITE(0, "uart1_src", mux_pll_src_cpll_gpll_usb480m_p, 0,
+			RK2928_CLKSEL_CON(14), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK2928_CLKGATE_CON(1), 10, GFLAGS),
+	COMPOSITE(0, "uart2_src", mux_pll_src_cpll_gpll_usb480m_p,
+			0, RK2928_CLKSEL_CON(15), 12, 2,
+			MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(1), 12, GFLAGS),
+	COMPOSITE_FRAC(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(17), 0,
+			RK2928_CLKGATE_CON(1), 9, GFLAGS),
+	COMPOSITE_FRAC(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(18), 0,
+			RK2928_CLKGATE_CON(1), 11, GFLAGS),
+	COMPOSITE_FRAC(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(19), 0,
+			RK2928_CLKGATE_CON(1), 13, GFLAGS),
+	MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(13), 8, 2, MFLAGS),
+	MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(14), 8, 2, MFLAGS),
+	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(15), 8, 2, MFLAGS),
+
+	COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(2), 14, 1, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 0, GFLAGS),
+
+	COMPOSITE(0, "sclk_gmac_src", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(5), 7, 1, MFLAGS, 0, 5, DFLAGS,
+			RK2928_CLKGATE_CON(1), 7, GFLAGS),
+	MUX(0, "sclk_macphy_50m", mux_sclk_macphy_50m_p, 0,
+			RK2928_CLKSEL_CON(29), 10, 1, MFLAGS),
+	MUX(0, "sclk_gmac_pre", mux_sclk_gmac_pre_p, 0,
+			RK2928_CLKSEL_CON(5), 5, 1, MFLAGS),
+	GATE(0, "sclk_mac_refout", "sclk_gmac_pre", 0,
+			RK2928_CLKGATE_CON(5), 4, GFLAGS),
+	GATE(0, "sclk_mac_ref", "sclk_gmac_pre", 0,
+			RK2928_CLKGATE_CON(5), 3, GFLAGS),
+	GATE(0, "sclk_mac_rx", "sclk_gmac_pre", 0,
+			RK2928_CLKGATE_CON(5), 5, GFLAGS),
+	GATE(0, "sclk_mac_tx", "sclk_gmac_pre", 0,
+			RK2928_CLKGATE_CON(5), 6, GFLAGS),
+	COMPOSITE(0, "sclk_macphy", mux_sclk_macphy_p, 0,
+			RK2928_CLKSEL_CON(29), 12, 1, MFLAGS, 8, 2, DFLAGS,
+			RK2928_CLKGATE_CON(5), 7, GFLAGS),
+	COMPOSITE(0, "sclk_gmac_out", mux_pll_src_2plls_p, 0,
+			RK2928_CLKSEL_CON(5), 15, 1, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(2), 2, GFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 3
+	 */
+
+	/* PD_VOP */
+	GATE(0, "aclk_rga", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 0, GFLAGS),
+	GATE(0, "aclk_rga_noc", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 11, GFLAGS),
+	GATE(0, "aclk_iep", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 2, GFLAGS),
+	GATE(0, "aclk_iep_noc", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 9, GFLAGS),
+
+	GATE(0, "aclk_vop", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 5, GFLAGS),
+	GATE(0, "aclk_vop_noc", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 12, GFLAGS),
+
+	GATE(0, "aclk_hdcp", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(14), 10, GFLAGS),
+	GATE(0, "aclk_hdcp_noc", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(13), 10, GFLAGS),
+
+	GATE(0, "hclk_rga", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 1, GFLAGS),
+	GATE(0, "hclk_iep", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 3, GFLAGS),
+	GATE(0, "hclk_vop", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 6, GFLAGS),
+	GATE(0, "hclk_vio_ahb_arbi", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 7, GFLAGS),
+	GATE(0, "hclk_vio_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 8, GFLAGS),
+	GATE(0, "hclk_vop_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 13, GFLAGS),
+	GATE(0, "hclk_vio_h2p", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 7, GFLAGS),
+	GATE(0, "hclk_hdcp_mmu", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 12, GFLAGS),
+	GATE(0, "pclk_hdmi_ctrl", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 6, GFLAGS),
+	GATE(0, "pclk_vio_h2p", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 8, GFLAGS),
+	GATE(0, "pclk_hdcp", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 11, GFLAGS),
+
+	/* PD_PERI */
+	GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 0, GFLAGS),
+	GATE(0, "aclk_gmac", "aclk_peri", 0, RK2928_CLKGATE_CON(11), 4, GFLAGS),
+
+	GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 0, GFLAGS),
+	GATE(HCLK_SDIO, "hclk_sdio", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 1, GFLAGS),
+	GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 2, GFLAGS),
+	GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 3, GFLAGS),
+	GATE(0, "hclk_host0", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 6, GFLAGS),
+	GATE(0, "hclk_host0_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 7, GFLAGS),
+	GATE(0, "hclk_host1", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 8, GFLAGS),
+	GATE(0, "hclk_host1_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 9, GFLAGS),
+	GATE(0, "hclk_host2", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 10, GFLAGS),
+	GATE(0, "hclk_otg", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 12, GFLAGS),
+	GATE(0, "hclk_otg_pmu", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 13, GFLAGS),
+	GATE(0, "hclk_host2_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 14, GFLAGS),
+	GATE(0, "hclk_peri_noc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 1, GFLAGS),
+
+	GATE(0, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(11), 5, GFLAGS),
+	GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 2, GFLAGS),
+
+	/* PD_GPU */
+	GATE(0, "aclk_gpu", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(13), 14, GFLAGS),
+	GATE(0, "aclk_gpu_noc", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(13), 15, GFLAGS),
+
+	/* PD_BUS */
+	GATE(0, "sclk_initmem_mbist", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS),
+	GATE(0, "aclk_initmem", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS),
+	GATE(ACLK_DMAC, "aclk_dmac_bus", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 2, GFLAGS),
+	GATE(0, "aclk_bus_noc", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS),
+
+	GATE(0, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 3, GFLAGS),
+	GATE(0, "hclk_i2s0_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS),
+	GATE(0, "hclk_i2s1_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS),
+	GATE(0, "hclk_i2s2_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS),
+	GATE(0, "hclk_spdif_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 10, GFLAGS),
+	GATE(0, "hclk_tsp", "hclk_cpu", 0, RK2928_CLKGATE_CON(10), 11, GFLAGS),
+	GATE(0, "hclk_crypto_mst", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS),
+	GATE(0, "hclk_crypto_slv", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 12, GFLAGS),
+
+	GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 4, GFLAGS),
+	GATE(0, "pclk_ddrmon", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 6, GFLAGS),
+	GATE(0, "pclk_msch_noc", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(10), 2, GFLAGS),
+
+	GATE(0, "pclk_efuse_1024", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS),
+	GATE(0, "pclk_efuse_256", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 14, GFLAGS),
+	GATE(PCLK_I2C0, "pclk_i2c0", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 15, GFLAGS),
+	GATE(PCLK_I2C1, "pclk_i2c1", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 0, GFLAGS),
+	GATE(PCLK_I2C2, "pclk_i2c2", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 1, GFLAGS),
+	GATE(PCLK_I2C3, "pclk_i2c3", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS),
+	GATE(PCLK_TIMER, "pclk_timer0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 4, GFLAGS),
+	GATE(0, "pclk_stimer", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 5, GFLAGS),
+	GATE(PCLK_SPI0, "pclk_spi0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 6, GFLAGS),
+	GATE(PCLK_PWM, "pclk_rk_pwm", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 7, GFLAGS),
+	GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 8, GFLAGS),
+	GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 9, GFLAGS),
+	GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 10, GFLAGS),
+	GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 11, GFLAGS),
+	GATE(PCLK_UART0, "pclk_uart0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 12, GFLAGS),
+	GATE(PCLK_UART1, "pclk_uart1", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 13, GFLAGS),
+	GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 14, GFLAGS),
+	GATE(0, "pclk_tsadc", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 15, GFLAGS),
+	GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 0, GFLAGS),
+	GATE(0, "pclk_cru", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS),
+	GATE(0, "pclk_sgrf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 2, GFLAGS),
+	GATE(0, "pclk_sim", "pclk_cpu", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS),
+
+	GATE(0, "pclk_ddrphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS),
+	GATE(0, "pclk_acodecphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 5, GFLAGS),
+	GATE(0, "pclk_hdmiphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 7, GFLAGS),
+	GATE(0, "pclk_vdacphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS),
+	GATE(0, "pclk_phy_noc", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 9, GFLAGS),
+
+	GATE(0, "aclk_vpu", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 0, GFLAGS),
+	GATE(0, "aclk_vpu_noc", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 4, GFLAGS),
+	GATE(0, "aclk_rkvdec", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 2, GFLAGS),
+	GATE(0, "aclk_rkvdec_noc", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 6, GFLAGS),
+	GATE(0, "hclk_vpu", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 1, GFLAGS),
+	GATE(0, "hclk_vpu_noc", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 5, GFLAGS),
+	GATE(0, "hclk_rkvdec", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 3, GFLAGS),
+	GATE(0, "hclk_rkvdec_noc", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 7, GFLAGS),
+
+	/* PD_MMC */
+	MMC(SCLK_SDMMC_DRV,    "sdmmc_drv",    "sclk_sdmmc", RK3228_SDMMC_CON0, 1),
+	MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3228_SDMMC_CON1, 1),
+
+	MMC(SCLK_SDIO_DRV,     "sdio_drv",     "sclk_sdio",  RK3228_SDIO_CON0,  1),
+	MMC(SCLK_SDIO_SAMPLE,  "sdio_sample",  "sclk_sdio",  RK3228_SDIO_CON1,  1),
+
+	MMC(SCLK_EMMC_DRV,     "emmc_drv",     "sclk_emmc",  RK3228_EMMC_CON0,  1),
+	MMC(SCLK_EMMC_SAMPLE,  "emmc_sample",  "sclk_emmc",  RK3228_EMMC_CON1,  1),
+};
+
+static const char *const rk3228_critical_clocks[] __initconst = {
+	"aclk_cpu",
+	"aclk_peri",
+	"hclk_peri",
+	"pclk_peri",
+};
+
+static void __init rk3228_clk_init(struct device_node *np)
+{
+	void __iomem *reg_base;
+	struct clk *clk;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: could not map cru region\n", __func__);
+		return;
+	}
+
+	rockchip_clk_init(np, reg_base, CLK_NR_CLKS);
+
+	/* xin12m is created by an cru-internal divider */
+	clk = clk_register_fixed_factor(NULL, "xin12m", "xin24m", 0, 1, 2);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock xin12m: %ld\n",
+				__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "ddrphy_pre", "ddrphy4x", 0, 1, 4);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock ddrphy_pre: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "hclk_vpu_pre",
+					"hclk_vpu_src", 0, 1, 4);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock hclk_vpu_pre: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	clk = clk_register_fixed_factor(NULL, "hclk_rkvdec_pre",
+					"hclk_rkvdec_src", 0, 1, 4);
+	if (IS_ERR(clk))
+		pr_warn("%s: could not register clock hclk_rkvdec_pre: %ld\n",
+			__func__, PTR_ERR(clk));
+
+	rockchip_clk_register_plls(rk3228_pll_clks,
+				   ARRAY_SIZE(rk3228_pll_clks),
+				   RK3228_GRF_SOC_STATUS0);
+	rockchip_clk_register_branches(rk3228_clk_branches,
+				  ARRAY_SIZE(rk3228_clk_branches));
+	rockchip_clk_protect_critical(rk3228_critical_clocks,
+				      ARRAY_SIZE(rk3228_critical_clocks));
+
+	rockchip_clk_register_armclk(ARMCLK, "armclk",
+			mux_armclk_p, ARRAY_SIZE(mux_armclk_p),
+			&rk3228_cpuclk_data, rk3228_cpuclk_rates,
+			ARRAY_SIZE(rk3228_cpuclk_rates));
+
+	rockchip_register_softrst(np, 9, reg_base + RK2928_SOFTRST_CON(0),
+				  ROCKCHIP_SOFTRST_HIWORD_MASK);
+
+	rockchip_register_restart_notifier(RK3228_GLB_SRST_FST, NULL);
+}
+CLK_OF_DECLARE(rk3228_cru, "rockchip,rk3228-cru", rk3228_clk_init);
diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c
index 9040878e3e2b..984fc187d12e 100644
--- a/drivers/clk/rockchip/clk-rk3288.c
+++ b/drivers/clk/rockchip/clk-rk3288.c
@@ -225,6 +225,38 @@ static struct clk_div_table div_hclk_cpu_t[] = {
 #define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE)
 #define IFLAGS ROCKCHIP_INVERTER_HIWORD_MASK
 
+static struct rockchip_clk_branch rk3288_i2s_fracmux __initdata =
+	MUX(0, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(4), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_spdif_fracmux __initdata =
+	MUX(0, "spdif_mux", mux_spdif_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(5), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_spdif_8ch_fracmux __initdata =
+	MUX(0, "spdif_8ch_mux", mux_spdif_8ch_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(40), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_uart0_fracmux __initdata =
+	MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(13), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_uart1_fracmux __initdata =
+	MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(14), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_uart2_fracmux __initdata =
+	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(15), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_uart3_fracmux __initdata =
+	MUX(SCLK_UART3, "sclk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(16), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk3288_uart4_fracmux __initdata =
+	MUX(SCLK_UART4, "sclk_uart4", mux_uart4_p, CLK_SET_RATE_PARENT,
+			RK3288_CLKSEL_CON(3), 8, 2, MFLAGS);
+
 static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	/*
 	 * Clock-Architecture Diagram 1
@@ -295,7 +327,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 			RK3288_CLKGATE_CON(0), 4, GFLAGS),
 	GATE(0, "c2c_host", "aclk_cpu_src", 0,
 			RK3288_CLKGATE_CON(13), 8, GFLAGS),
-	COMPOSITE_NOMUX(0, "crypto", "aclk_cpu_pre", 0,
+	COMPOSITE_NOMUX(SCLK_CRYPTO, "crypto", "aclk_cpu_pre", 0,
 			RK3288_CLKSEL_CON(26), 6, 2, DFLAGS,
 			RK3288_CLKGATE_CON(5), 4, GFLAGS),
 	GATE(0, "aclk_bus_2pmu", "aclk_cpu_pre", CLK_IGNORE_UNUSED,
@@ -304,11 +336,10 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	COMPOSITE(0, "i2s_src", mux_pll_src_cpll_gpll_p, 0,
 			RK3288_CLKSEL_CON(4), 15, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(4), 1, GFLAGS),
-	COMPOSITE_FRAC(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(8), 0,
-			RK3288_CLKGATE_CON(4), 2, GFLAGS),
-	MUX(0, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(4), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(4), 2, GFLAGS,
+			&rk3288_i2s_fracmux),
 	COMPOSITE_NODIV(SCLK_I2S0_OUT, "i2s0_clkout", mux_i2s_clkout_p, 0,
 			RK3288_CLKSEL_CON(4), 12, 1, MFLAGS,
 			RK3288_CLKGATE_CON(4), 0, GFLAGS),
@@ -317,23 +348,23 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 
 	MUX(0, "spdif_src", mux_pll_src_cpll_gpll_p, 0,
 			RK3288_CLKSEL_CON(5), 15, 1, MFLAGS),
-	COMPOSITE_NOMUX(0, "spdif_pre", "spdif_src", 0,
+	COMPOSITE_NOMUX(0, "spdif_pre", "spdif_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(5), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(4), 4, GFLAGS),
-	COMPOSITE_FRAC(0, "spdif_frac", "spdif_src", 0,
+	COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(9), 0,
-			RK3288_CLKGATE_CON(4), 5, GFLAGS),
-	COMPOSITE_NODIV(SCLK_SPDIF, "sclk_spdif", mux_spdif_p, 0,
-			RK3288_CLKSEL_CON(5), 8, 2, MFLAGS,
+			RK3288_CLKGATE_CON(4), 5, GFLAGS,
+			&rk3288_spdif_fracmux),
+	GATE(SCLK_SPDIF, "sclk_spdif", "spdif_mux", CLK_SET_RATE_PARENT,
 			RK3288_CLKGATE_CON(4), 6, GFLAGS),
-	COMPOSITE_NOMUX(0, "spdif_8ch_pre", "spdif_src", 0,
+	COMPOSITE_NOMUX(0, "spdif_8ch_pre", "spdif_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(40), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(4), 7, GFLAGS),
-	COMPOSITE_FRAC(0, "spdif_8ch_frac", "spdif_8ch_pre", 0,
+	COMPOSITE_FRACMUX(0, "spdif_8ch_frac", "spdif_8ch_pre", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(41), 0,
-			RK3288_CLKGATE_CON(4), 8, GFLAGS),
-	COMPOSITE_NODIV(SCLK_SPDIF8CH, "sclk_spdif_8ch", mux_spdif_8ch_p, 0,
-			RK3288_CLKSEL_CON(40), 8, 2, MFLAGS,
+			RK3288_CLKGATE_CON(4), 8, GFLAGS,
+			&rk3288_spdif_8ch_fracmux),
+	GATE(SCLK_SPDIF8CH, "sclk_spdif_8ch", "spdif_8ch_mux", CLK_SET_RATE_PARENT,
 			RK3288_CLKGATE_CON(4), 9, GFLAGS),
 
 	GATE(0, "sclk_acc_efuse", "xin24m", 0,
@@ -536,45 +567,40 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	COMPOSITE(0, "uart0_src", mux_pll_src_cpll_gll_usb_npll_p, 0,
 			RK3288_CLKSEL_CON(13), 13, 2, MFLAGS, 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(1), 8, GFLAGS),
-	COMPOSITE_FRAC(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(17), 0,
-			RK3288_CLKGATE_CON(1), 9, GFLAGS),
-	MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(13), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(1), 9, GFLAGS,
+			&rk3288_uart0_fracmux),
 	MUX(0, "uart_src", mux_pll_src_cpll_gpll_p, 0,
 			RK3288_CLKSEL_CON(13), 15, 1, MFLAGS),
 	COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0,
 			RK3288_CLKSEL_CON(14), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(1), 10, GFLAGS),
-	COMPOSITE_FRAC(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(18), 0,
-			RK3288_CLKGATE_CON(1), 11, GFLAGS),
-	MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(14), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(1), 11, GFLAGS,
+			&rk3288_uart1_fracmux),
 	COMPOSITE_NOMUX(0, "uart2_src", "uart_src", 0,
 			RK3288_CLKSEL_CON(15), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(1), 12, GFLAGS),
-	COMPOSITE_FRAC(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(19), 0,
-			RK3288_CLKGATE_CON(1), 13, GFLAGS),
-	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(15), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(1), 13, GFLAGS,
+			&rk3288_uart2_fracmux),
 	COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0,
 			RK3288_CLKSEL_CON(16), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(1), 14, GFLAGS),
-	COMPOSITE_FRAC(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(20), 0,
-			RK3288_CLKGATE_CON(1), 15, GFLAGS),
-	MUX(SCLK_UART3, "sclk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(16), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(1), 15, GFLAGS,
+			&rk3288_uart3_fracmux),
 	COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0,
 			RK3288_CLKSEL_CON(3), 0, 7, DFLAGS,
 			RK3288_CLKGATE_CON(2), 12, GFLAGS),
-	COMPOSITE_FRAC(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT,
 			RK3288_CLKSEL_CON(7), 0,
-			RK3288_CLKGATE_CON(2), 13, GFLAGS),
-	MUX(SCLK_UART4, "sclk_uart4", mux_uart4_p, CLK_SET_RATE_PARENT,
-			RK3288_CLKSEL_CON(3), 8, 2, MFLAGS),
+			RK3288_CLKGATE_CON(2), 13, GFLAGS,
+			&rk3288_uart4_fracmux),
 
 	COMPOSITE(0, "mac_pll_src", mux_pll_src_npll_cpll_gpll_p, 0,
 			RK3288_CLKSEL_CON(21), 0, 2, MFLAGS, 8, 5, DFLAGS,
@@ -644,10 +670,10 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	GATE(PCLK_PUBL0, "pclk_publ0", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 15, GFLAGS),
 	GATE(PCLK_DDRUPCTL1, "pclk_ddrupctl1", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 0, GFLAGS),
 	GATE(PCLK_PUBL1, "pclk_publ1", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 1, GFLAGS),
-	GATE(0, "pclk_efuse_1024", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 2, GFLAGS),
+	GATE(PCLK_EFUSE1024, "pclk_efuse_1024", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 2, GFLAGS),
 	GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 3, GFLAGS),
 	GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 9, GFLAGS),
-	GATE(0, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS),
+	GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS),
 	GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(11), 11, GFLAGS),
 
 	/* ddrctrl [DDR Controller PHY clock] gates */
@@ -709,7 +735,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	GATE(SCLK_LCDC_PWM1, "sclk_lcdc_pwm1", "xin24m", 0, RK3288_CLKGATE_CON(13), 11, GFLAGS),
 	GATE(SCLK_PVTM_CORE, "sclk_pvtm_core", "xin24m", 0, RK3288_CLKGATE_CON(5), 9, GFLAGS),
 	GATE(SCLK_PVTM_GPU, "sclk_pvtm_gpu", "xin24m", 0, RK3288_CLKGATE_CON(5), 10, GFLAGS),
-	GATE(0, "sclk_mipidsi_24m", "xin24m", 0, RK3288_CLKGATE_CON(5), 15, GFLAGS),
+	GATE(SCLK_MIPIDSI_24M, "sclk_mipidsi_24m", "xin24m", 0, RK3288_CLKGATE_CON(5), 15, GFLAGS),
 
 	/* sclk_gpu gates */
 	GATE(ACLK_GPU, "aclk_gpu", "sclk_gpu", 0, RK3288_CLKGATE_CON(18), 0, GFLAGS),
@@ -783,10 +809,10 @@ static const char *const rk3288_critical_clocks[] __initconst = {
 	"pclk_pd_pmu",
 };
 
-#ifdef CONFIG_PM_SLEEP
 static void __iomem *rk3288_cru_base;
 
-/* Some CRU registers will be reset in maskrom when the system
+/*
+ * Some CRU registers will be reset in maskrom when the system
  * wakes up from fastboot.
  * So save them before suspend, restore them after resume.
  */
@@ -840,33 +866,27 @@ static void rk3288_clk_resume(void)
 	}
 }
 
+static void rk3288_clk_shutdown(void)
+{
+	writel_relaxed(0xf3030000, rk3288_cru_base + RK3288_MODE_CON);
+}
+
 static struct syscore_ops rk3288_clk_syscore_ops = {
 	.suspend = rk3288_clk_suspend,
 	.resume = rk3288_clk_resume,
 };
 
-static void rk3288_clk_sleep_init(void __iomem *reg_base)
-{
-	rk3288_cru_base = reg_base;
-	register_syscore_ops(&rk3288_clk_syscore_ops);
-}
-
-#else /* CONFIG_PM_SLEEP */
-static void rk3288_clk_sleep_init(void __iomem *reg_base) {}
-#endif
-
 static void __init rk3288_clk_init(struct device_node *np)
 {
-	void __iomem *reg_base;
 	struct clk *clk;
 
-	reg_base = of_iomap(np, 0);
-	if (!reg_base) {
+	rk3288_cru_base = of_iomap(np, 0);
+	if (!rk3288_cru_base) {
 		pr_err("%s: could not map cru region\n", __func__);
 		return;
 	}
 
-	rockchip_clk_init(np, reg_base, CLK_NR_CLKS);
+	rockchip_clk_init(np, rk3288_cru_base, CLK_NR_CLKS);
 
 	/* xin12m is created by an cru-internal divider */
 	clk = clk_register_fixed_factor(NULL, "xin12m", "xin24m", 0, 1, 2);
@@ -907,10 +927,12 @@ static void __init rk3288_clk_init(struct device_node *np)
 			&rk3288_cpuclk_data, rk3288_cpuclk_rates,
 			ARRAY_SIZE(rk3288_cpuclk_rates));
 
-	rockchip_register_softrst(np, 12, reg_base + RK3288_SOFTRST_CON(0),
+	rockchip_register_softrst(np, 12,
+				  rk3288_cru_base + RK3288_SOFTRST_CON(0),
 				  ROCKCHIP_SOFTRST_HIWORD_MASK);
 
-	rockchip_register_restart_notifier(RK3288_GLB_SRST_FST);
-	rk3288_clk_sleep_init(reg_base);
+	rockchip_register_restart_notifier(RK3288_GLB_SRST_FST,
+					   rk3288_clk_shutdown);
+	register_syscore_ops(&rk3288_clk_syscore_ops);
 }
 CLK_OF_DECLARE(rk3288_cru, "rockchip,rk3288-cru", rk3288_clk_init);
diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c
index 7e6b783e6eee..be0ede522269 100644
--- a/drivers/clk/rockchip/clk-rk3368.c
+++ b/drivers/clk/rockchip/clk-rk3368.c
@@ -184,13 +184,13 @@ static const struct rockchip_cpuclk_reg_data rk3368_cpuclkl_data = {
 
 #define RK3368_CLKSEL0(_offs, _aclkm)					\
 	{								\
-		.reg = RK3288_CLKSEL_CON(0 + _offs),			\
+		.reg = RK3368_CLKSEL_CON(0 + _offs),			\
 		.val = HIWORD_UPDATE(_aclkm, RK3368_DIV_ACLKM_MASK,	\
 				RK3368_DIV_ACLKM_SHIFT),		\
 	}
 #define RK3368_CLKSEL1(_offs, _atclk, _pdbg)				\
 	{								\
-		.reg = RK3288_CLKSEL_CON(1 + _offs),			\
+		.reg = RK3368_CLKSEL_CON(1 + _offs),			\
 		.val = HIWORD_UPDATE(_atclk, RK3368_DIV_ATCLK_MASK,	\
 				RK3368_DIV_ATCLK_SHIFT) |		\
 		       HIWORD_UPDATE(_pdbg, RK3368_DIV_PCLK_DBG_MASK,	\
@@ -819,6 +819,13 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = {
 };
 
 static const char *const rk3368_critical_clocks[] __initconst = {
+	"aclk_bus",
+	"aclk_peri",
+	/*
+	 * pwm1 supplies vdd_logic on a lot of boards, is currently unhandled
+	 * but needs to stay enabled there (including its parents) at all times.
+	 */
+	"pclk_pwm1",
 	"pclk_pd_pmu",
 };
 
@@ -882,6 +889,6 @@ static void __init rk3368_clk_init(struct device_node *np)
 	rockchip_register_softrst(np, 15, reg_base + RK3368_SOFTRST_CON(0),
 				  ROCKCHIP_SOFTRST_HIWORD_MASK);
 
-	rockchip_register_restart_notifier(RK3368_GLB_SRST_FST);
+	rockchip_register_restart_notifier(RK3368_GLB_SRST_FST, NULL);
 }
 CLK_OF_DECLARE(rk3368_cru, "rockchip,rk3368-cru", rk3368_clk_init);
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index be6c7fd8315d..d9a0b5d4d47f 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -102,22 +102,82 @@ static struct clk *rockchip_clk_register_branch(const char *name,
 	return clk;
 }
 
+struct rockchip_clk_frac {
+	struct notifier_block			clk_nb;
+	struct clk_fractional_divider		div;
+	struct clk_gate				gate;
+
+	struct clk_mux				mux;
+	const struct clk_ops			*mux_ops;
+	int					mux_frac_idx;
+
+	bool					rate_change_remuxed;
+	int					rate_change_idx;
+};
+
+#define to_rockchip_clk_frac_nb(nb) \
+			container_of(nb, struct rockchip_clk_frac, clk_nb)
+
+static int rockchip_clk_frac_notifier_cb(struct notifier_block *nb,
+					 unsigned long event, void *data)
+{
+	struct clk_notifier_data *ndata = data;
+	struct rockchip_clk_frac *frac = to_rockchip_clk_frac_nb(nb);
+	struct clk_mux *frac_mux = &frac->mux;
+	int ret = 0;
+
+	pr_debug("%s: event %lu, old_rate %lu, new_rate: %lu\n",
+		 __func__, event, ndata->old_rate, ndata->new_rate);
+	if (event == PRE_RATE_CHANGE) {
+		frac->rate_change_idx = frac->mux_ops->get_parent(&frac_mux->hw);
+		if (frac->rate_change_idx != frac->mux_frac_idx) {
+			frac->mux_ops->set_parent(&frac_mux->hw, frac->mux_frac_idx);
+			frac->rate_change_remuxed = 1;
+		}
+	} else if (event == POST_RATE_CHANGE) {
+		/*
+		 * The POST_RATE_CHANGE notifier runs directly after the
+		 * divider clock is set in clk_change_rate, so we'll have
+		 * remuxed back to the original parent before clk_change_rate
+		 * reaches the mux itself.
+		 */
+		if (frac->rate_change_remuxed) {
+			frac->mux_ops->set_parent(&frac_mux->hw, frac->rate_change_idx);
+			frac->rate_change_remuxed = 0;
+		}
+	}
+
+	return notifier_from_errno(ret);
+}
+
 static struct clk *rockchip_clk_register_frac_branch(const char *name,
 		const char *const *parent_names, u8 num_parents,
 		void __iomem *base, int muxdiv_offset, u8 div_flags,
 		int gate_offset, u8 gate_shift, u8 gate_flags,
-		unsigned long flags, spinlock_t *lock)
+		unsigned long flags, struct rockchip_clk_branch *child,
+		spinlock_t *lock)
 {
+	struct rockchip_clk_frac *frac;
 	struct clk *clk;
 	struct clk_gate *gate = NULL;
 	struct clk_fractional_divider *div = NULL;
 	const struct clk_ops *div_ops = NULL, *gate_ops = NULL;
 
-	if (gate_offset >= 0) {
-		gate = kzalloc(sizeof(*gate), GFP_KERNEL);
-		if (!gate)
-			return ERR_PTR(-ENOMEM);
+	if (muxdiv_offset < 0)
+		return ERR_PTR(-EINVAL);
+
+	if (child && child->branch_type != branch_mux) {
+		pr_err("%s: fractional child clock for %s can only be a mux\n",
+		       __func__, name);
+		return ERR_PTR(-EINVAL);
+	}
 
+	frac = kzalloc(sizeof(*frac), GFP_KERNEL);
+	if (!frac)
+		return ERR_PTR(-ENOMEM);
+
+	if (gate_offset >= 0) {
+		gate = &frac->gate;
 		gate->flags = gate_flags;
 		gate->reg = base + gate_offset;
 		gate->bit_idx = gate_shift;
@@ -125,13 +185,7 @@ static struct clk *rockchip_clk_register_frac_branch(const char *name,
 		gate_ops = &clk_gate_ops;
 	}
 
-	if (muxdiv_offset < 0)
-		return ERR_PTR(-EINVAL);
-
-	div = kzalloc(sizeof(*div), GFP_KERNEL);
-	if (!div)
-		return ERR_PTR(-ENOMEM);
-
+	div = &frac->div;
 	div->flags = div_flags;
 	div->reg = base + muxdiv_offset;
 	div->mshift = 16;
@@ -147,7 +201,61 @@ static struct clk *rockchip_clk_register_frac_branch(const char *name,
 				     NULL, NULL,
 				     &div->hw, div_ops,
 				     gate ? &gate->hw : NULL, gate_ops,
-				     flags);
+				     flags | CLK_SET_RATE_UNGATE);
+	if (IS_ERR(clk)) {
+		kfree(frac);
+		return clk;
+	}
+
+	if (child) {
+		struct clk_mux *frac_mux = &frac->mux;
+		struct clk_init_data init;
+		struct clk *mux_clk;
+		int i, ret;
+
+		frac->mux_frac_idx = -1;
+		for (i = 0; i < child->num_parents; i++) {
+			if (!strcmp(name, child->parent_names[i])) {
+				pr_debug("%s: found fractional parent in mux at pos %d\n",
+					 __func__, i);
+				frac->mux_frac_idx = i;
+				break;
+			}
+		}
+
+		frac->mux_ops = &clk_mux_ops;
+		frac->clk_nb.notifier_call = rockchip_clk_frac_notifier_cb;
+
+		frac_mux->reg = base + child->muxdiv_offset;
+		frac_mux->shift = child->mux_shift;
+		frac_mux->mask = BIT(child->mux_width) - 1;
+		frac_mux->flags = child->mux_flags;
+		frac_mux->lock = lock;
+		frac_mux->hw.init = &init;
+
+		init.name = child->name;
+		init.flags = child->flags | CLK_SET_RATE_PARENT;
+		init.ops = frac->mux_ops;
+		init.parent_names = child->parent_names;
+		init.num_parents = child->num_parents;
+
+		mux_clk = clk_register(NULL, &frac_mux->hw);
+		if (IS_ERR(mux_clk))
+			return clk;
+
+		rockchip_clk_add_lookup(mux_clk, child->id);
+
+		/* notifier on the fraction divider to catch rate changes */
+		if (frac->mux_frac_idx >= 0) {
+			ret = clk_notifier_register(clk, &frac->clk_nb);
+			if (ret)
+				pr_err("%s: failed to register clock notifier for %s\n",
+						__func__, name);
+		} else {
+			pr_warn("%s: could not find %s as parent of %s, rate changes may not work\n",
+				__func__, name, child->name);
+		}
+	}
 
 	return clk;
 }
@@ -251,7 +359,8 @@ void __init rockchip_clk_register_branches(
 				list->parent_names, list->num_parents,
 				reg_base, list->muxdiv_offset, list->div_flags,
 				list->gate_offset, list->gate_shift,
-				list->gate_flags, flags, &clk_lock);
+				list->gate_flags, flags, list->child,
+				&clk_lock);
 			break;
 		case branch_gate:
 			flags |= CLK_SET_RATE_PARENT;
@@ -341,9 +450,13 @@ void __init rockchip_clk_protect_critical(const char *const clocks[],
 }
 
 static unsigned int reg_restart;
+static void (*cb_restart)(void);
 static int rockchip_restart_notify(struct notifier_block *this,
 				   unsigned long mode, void *cmd)
 {
+	if (cb_restart)
+		cb_restart();
+
 	writel(0xfdb9, reg_base + reg_restart);
 	return NOTIFY_DONE;
 }
@@ -353,11 +466,12 @@ static struct notifier_block rockchip_restart_handler = {
 	.priority = 128,
 };
 
-void __init rockchip_register_restart_notifier(unsigned int reg)
+void __init rockchip_register_restart_notifier(unsigned int reg, void (*cb)(void))
 {
 	int ret;
 
 	reg_restart = reg;
+	cb_restart = cb;
 	ret = register_restart_handler(&rockchip_restart_handler);
 	if (ret)
 		pr_err("%s: cannot register restart handler, %d\n",
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index dc8ecb2673b7..ff8bd23a93ec 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -2,6 +2,9 @@
  * Copyright (c) 2014 MundoReader S.L.
  * Author: Heiko Stuebner <heiko@sntech.de>
  *
+ * Copyright (c) 2015 Rockchip Electronics Co. Ltd.
+ * Author: Xing Zheng <zhengxing@rock-chips.com>
+ *
  * based on
  *
  * samsung/clk.h
@@ -30,7 +33,7 @@ struct clk;
 #define HIWORD_UPDATE(val, mask, shift) \
 		((val) << (shift) | (mask) << ((shift) + 16))
 
-/* register positions shared by RK2928, RK3066 and RK3188 */
+/* register positions shared by RK2928, RK3036, RK3066, RK3188 and RK3228 */
 #define RK2928_PLL_CON(x)		((x) * 0x4)
 #define RK2928_MODE_CON		0x40
 #define RK2928_CLKSEL_CON(x)	((x) * 0x4 + 0x44)
@@ -40,6 +43,22 @@ struct clk;
 #define RK2928_SOFTRST_CON(x)	((x) * 0x4 + 0x110)
 #define RK2928_MISC_CON		0x134
 
+#define RK3036_SDMMC_CON0		0x144
+#define RK3036_SDMMC_CON1		0x148
+#define RK3036_SDIO_CON0		0x14c
+#define RK3036_SDIO_CON1		0x150
+#define RK3036_EMMC_CON0		0x154
+#define RK3036_EMMC_CON1		0x158
+
+#define RK3228_GLB_SRST_FST		0x1f0
+#define RK3228_GLB_SRST_SND		0x1f4
+#define RK3228_SDMMC_CON0		0x1c0
+#define RK3228_SDMMC_CON1		0x1c4
+#define RK3228_SDIO_CON0		0x1c8
+#define RK3228_SDIO_CON1		0x1cc
+#define RK3228_EMMC_CON0		0x1d8
+#define RK3228_EMMC_CON1		0x1dc
+
 #define RK3288_PLL_CON(x)		RK2928_PLL_CON(x)
 #define RK3288_MODE_CON			0x50
 #define RK3288_CLKSEL_CON(x)		((x) * 0x4 + 0x60)
@@ -74,9 +93,22 @@ struct clk;
 #define RK3368_EMMC_CON1		0x41c
 
 enum rockchip_pll_type {
+	pll_rk3036,
 	pll_rk3066,
 };
 
+#define RK3036_PLL_RATE(_rate, _refdiv, _fbdiv, _postdiv1,	\
+			_postdiv2, _dsmpd, _frac)		\
+{								\
+	.rate	= _rate##U,					\
+	.fbdiv = _fbdiv,					\
+	.postdiv1 = _postdiv1,					\
+	.refdiv = _refdiv,					\
+	.postdiv2 = _postdiv2,					\
+	.dsmpd = _dsmpd,					\
+	.frac = _frac,						\
+}
+
 #define RK3066_PLL_RATE(_rate, _nr, _nf, _no)	\
 {						\
 	.rate	= _rate##U,			\
@@ -101,6 +133,13 @@ struct rockchip_pll_rate_table {
 	unsigned int nf;
 	unsigned int no;
 	unsigned int nb;
+	/* for RK3036 */
+	unsigned int fbdiv;
+	unsigned int postdiv1;
+	unsigned int refdiv;
+	unsigned int postdiv2;
+	unsigned int dsmpd;
+	unsigned int frac;
 };
 
 /**
@@ -235,6 +274,7 @@ struct rockchip_clk_branch {
 	int				gate_offset;
 	u8				gate_shift;
 	u8				gate_flags;
+	struct rockchip_clk_branch	*child;
 };
 
 #define COMPOSITE(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\
@@ -369,6 +409,24 @@ struct rockchip_clk_branch {
 		.gate_flags	= gf,				\
 	}
 
+#define COMPOSITE_FRACMUX(_id, cname, pname, f, mo, df, go, gs, gf, ch) \
+	{							\
+		.id		= _id,				\
+		.branch_type	= branch_fraction_divider,	\
+		.name		= cname,			\
+		.parent_names	= (const char *[]){ pname },	\
+		.num_parents	= 1,				\
+		.flags		= f,				\
+		.muxdiv_offset	= mo,				\
+		.div_shift	= 16,				\
+		.div_width	= 16,				\
+		.div_flags	= df,				\
+		.gate_offset	= go,				\
+		.gate_shift	= gs,				\
+		.gate_flags	= gf,				\
+		.child		= ch,				\
+	}
+
 #define MUX(_id, cname, pnames, f, o, s, w, mf)			\
 	{							\
 		.id		= _id,				\
@@ -464,7 +522,7 @@ void rockchip_clk_register_armclk(unsigned int lookup_id, const char *name,
 			const struct rockchip_cpuclk_rate_table *rates,
 			int nrates);
 void rockchip_clk_protect_critical(const char *const clocks[], int nclocks);
-void rockchip_register_restart_notifier(unsigned int reg);
+void rockchip_register_restart_notifier(unsigned int reg, void (*cb)(void));
 
 #define ROCKCHIP_SOFTRST_HIWORD_MASK	BIT(0)
 
diff --git a/drivers/clk/samsung/clk-cpu.c b/drivers/clk/samsung/clk-cpu.c
index 2fe37f708dc7..813003d6ce09 100644
--- a/drivers/clk/samsung/clk-cpu.c
+++ b/drivers/clk/samsung/clk-cpu.c
@@ -148,6 +148,7 @@ static int exynos_cpuclk_pre_rate_change(struct clk_notifier_data *ndata,
 	unsigned long alt_prate = clk_get_rate(cpuclk->alt_parent);
 	unsigned long alt_div = 0, alt_div_mask = DIV_MASK;
 	unsigned long div0, div1 = 0, mux_reg;
+	unsigned long flags;
 
 	/* find out the divider values to use for clock data */
 	while ((cfg_data->prate * 1000) != ndata->new_rate) {
@@ -156,7 +157,7 @@ static int exynos_cpuclk_pre_rate_change(struct clk_notifier_data *ndata,
 		cfg_data++;
 	}
 
-	spin_lock(cpuclk->lock);
+	spin_lock_irqsave(cpuclk->lock, flags);
 
 	/*
 	 * For the selected PLL clock frequency, get the pre-defined divider
@@ -212,7 +213,7 @@ static int exynos_cpuclk_pre_rate_change(struct clk_notifier_data *ndata,
 				DIV_MASK_ALL);
 	}
 
-	spin_unlock(cpuclk->lock);
+	spin_unlock_irqrestore(cpuclk->lock, flags);
 	return 0;
 }
 
@@ -223,6 +224,7 @@ static int exynos_cpuclk_post_rate_change(struct clk_notifier_data *ndata,
 	const struct exynos_cpuclk_cfg_data *cfg_data = cpuclk->cfg;
 	unsigned long div = 0, div_mask = DIV_MASK;
 	unsigned long mux_reg;
+	unsigned long flags;
 
 	/* find out the divider values to use for clock data */
 	if (cpuclk->flags & CLK_CPU_NEEDS_DEBUG_ALT_DIV) {
@@ -233,7 +235,7 @@ static int exynos_cpuclk_post_rate_change(struct clk_notifier_data *ndata,
 		}
 	}
 
-	spin_lock(cpuclk->lock);
+	spin_lock_irqsave(cpuclk->lock, flags);
 
 	/* select mout_apll as the alternate parent */
 	mux_reg = readl(base + E4210_SRC_CPU);
@@ -246,7 +248,7 @@ static int exynos_cpuclk_post_rate_change(struct clk_notifier_data *ndata,
 	}
 
 	exynos_set_safe_div(base, div, div_mask);
-	spin_unlock(cpuclk->lock);
+	spin_unlock_irqrestore(cpuclk->lock, flags);
 	return 0;
 }
 
diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c
index 7f370d3e0983..ac03e4fe2871 100644
--- a/drivers/clk/samsung/clk-exynos4.c
+++ b/drivers/clk/samsung/clk-exynos4.c
@@ -1024,6 +1024,7 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = {
 			0, 0),
 	GATE(CLK_AC97, "ac97", "aclk100", GATE_IP_PERIL, 27,
 			0, 0),
+	GATE(CLK_SSS, "sss", "aclk133", GATE_IP_DMC, 4, 0, 0),
 	GATE(CLK_PPMUDMC0, "ppmudmc0", "aclk133", GATE_IP_DMC, 8, 0, 0),
 	GATE(CLK_PPMUDMC1, "ppmudmc1", "aclk133", GATE_IP_DMC, 9, 0, 0),
 	GATE(CLK_PPMUCPU, "ppmucpu", "aclk133", GATE_IP_DMC, 10, 0, 0),
diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c
index 389af3c15ec4..d048dedd8b72 100644
--- a/drivers/clk/samsung/clk-exynos5420.c
+++ b/drivers/clk/samsung/clk-exynos5420.c
@@ -18,6 +18,7 @@
 #include <linux/syscore_ops.h>
 
 #include "clk.h"
+#include "clk-cpu.h"
 
 #define APLL_LOCK		0x0
 #define APLL_CON0		0x100
@@ -616,9 +617,11 @@ static struct samsung_mux_clock exynos5x_mux_clks[] __initdata = {
 	MUX(0, "mout_mspll_kfc", mout_mspll_cpu_p, SRC_TOP7, 8, 2),
 	MUX(0, "mout_mspll_cpu", mout_mspll_cpu_p, SRC_TOP7, 12, 2),
 
-	MUX(0, "mout_apll", mout_apll_p, SRC_CPU, 0, 1),
+	MUX_F(0, "mout_apll", mout_apll_p, SRC_CPU, 0, 1,
+	      CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0),
 	MUX(0, "mout_cpu", mout_cpu_p, SRC_CPU, 16, 1),
-	MUX(0, "mout_kpll", mout_kpll_p, SRC_KFC, 0, 1),
+	MUX_F(0, "mout_kpll", mout_kpll_p, SRC_KFC, 0, 1,
+	      CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0),
 	MUX(0, "mout_kfc", mout_kfc_p, SRC_KFC, 16, 1),
 
 	MUX(0, "mout_aclk200", mout_group1_p, SRC_TOP0, 8, 2),
@@ -677,8 +680,8 @@ static struct samsung_mux_clock exynos5x_mux_clks[] __initdata = {
 			SRC_TOP5, 20, 1),
 	MUX(CLK_MOUT_USER_ACLK300_DISP1, "mout_user_aclk300_disp1",
 			mout_user_aclk300_disp1_p, SRC_TOP5, 24, 1),
-	MUX(0, "mout_user_aclk300_gscl", mout_user_aclk300_gscl_p,
-			SRC_TOP5, 28, 1),
+	MUX(CLK_MOUT_USER_ACLK300_GSCL, "mout_user_aclk300_gscl",
+			mout_user_aclk300_gscl_p, SRC_TOP5, 28, 1),
 
 	MUX(0, "mout_sclk_mpll", mout_mpll_p, SRC_TOP6, 0, 1),
 	MUX(CLK_MOUT_VPLL, "mout_sclk_vpll", mout_vpll_p, SRC_TOP6, 4, 1),
@@ -729,8 +732,8 @@ static struct samsung_mux_clock exynos5x_mux_clks[] __initdata = {
 			SRC_TOP12, 20, 1),
 	MUX(CLK_MOUT_SW_ACLK300, "mout_sw_aclk300_disp1",
 			mout_sw_aclk300_disp1_p, SRC_TOP12, 24, 1),
-	MUX(0, "mout_sw_aclk300_gscl", mout_sw_aclk300_gscl_p,
-			SRC_TOP12, 28, 1),
+	MUX(CLK_MOUT_SW_ACLK300_GSCL, "mout_sw_aclk300_gscl",
+			mout_sw_aclk300_gscl_p, SRC_TOP12, 28, 1),
 
 	/* DISP1 Block */
 	MUX(0, "mout_mipi1", mout_group2_p, SRC_DISP10, 16, 3),
@@ -926,7 +929,7 @@ static struct samsung_gate_clock exynos5x_gate_clks[] __initdata = {
 			GATE_BUS_TOP, 13, 0, 0),
 	GATE(0, "aclk166", "mout_user_aclk166",
 			GATE_BUS_TOP, 14, CLK_IGNORE_UNUSED, 0),
-	GATE(0, "aclk333", "mout_aclk333",
+	GATE(0, "aclk333", "mout_user_aclk333",
 			GATE_BUS_TOP, 15, CLK_IGNORE_UNUSED, 0),
 	GATE(0, "aclk400_isp", "mout_user_aclk400_isp",
 			GATE_BUS_TOP, 16, 0, 0),
@@ -1246,6 +1249,74 @@ static struct samsung_pll_clock exynos5x_plls[nr_plls] __initdata = {
 		KPLL_CON0, NULL),
 };
 
+#define E5420_EGL_DIV0(apll, pclk_dbg, atb, cpud)			\
+		((((apll) << 24) | ((pclk_dbg) << 20) | ((atb) << 16) |	\
+		 ((cpud) << 4)))
+
+static const struct exynos_cpuclk_cfg_data exynos5420_eglclk_d[] __initconst = {
+	{ 1800000, E5420_EGL_DIV0(3, 7, 7, 4), },
+	{ 1700000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1600000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1500000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1400000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1300000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1200000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1100000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1000000, E5420_EGL_DIV0(3, 6, 6, 2), },
+	{  900000, E5420_EGL_DIV0(3, 6, 6, 2), },
+	{  800000, E5420_EGL_DIV0(3, 5, 5, 2), },
+	{  700000, E5420_EGL_DIV0(3, 5, 5, 2), },
+	{  600000, E5420_EGL_DIV0(3, 4, 4, 2), },
+	{  500000, E5420_EGL_DIV0(3, 3, 3, 2), },
+	{  400000, E5420_EGL_DIV0(3, 3, 3, 2), },
+	{  300000, E5420_EGL_DIV0(3, 3, 3, 2), },
+	{  200000, E5420_EGL_DIV0(3, 3, 3, 2), },
+	{  0 },
+};
+
+static const struct exynos_cpuclk_cfg_data exynos5800_eglclk_d[] __initconst = {
+	{ 2000000, E5420_EGL_DIV0(3, 7, 7, 4), },
+	{ 1900000, E5420_EGL_DIV0(3, 7, 7, 4), },
+	{ 1800000, E5420_EGL_DIV0(3, 7, 7, 4), },
+	{ 1700000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1600000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1500000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1400000, E5420_EGL_DIV0(3, 7, 7, 3), },
+	{ 1300000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1200000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1100000, E5420_EGL_DIV0(3, 7, 7, 2), },
+	{ 1000000, E5420_EGL_DIV0(3, 7, 6, 2), },
+	{  900000, E5420_EGL_DIV0(3, 7, 6, 2), },
+	{  800000, E5420_EGL_DIV0(3, 7, 5, 2), },
+	{  700000, E5420_EGL_DIV0(3, 7, 5, 2), },
+	{  600000, E5420_EGL_DIV0(3, 7, 4, 2), },
+	{  500000, E5420_EGL_DIV0(3, 7, 3, 2), },
+	{  400000, E5420_EGL_DIV0(3, 7, 3, 2), },
+	{  300000, E5420_EGL_DIV0(3, 7, 3, 2), },
+	{  200000, E5420_EGL_DIV0(3, 7, 3, 2), },
+	{  0 },
+};
+
+#define E5420_KFC_DIV(kpll, pclk, aclk)					\
+		((((kpll) << 24) | ((pclk) << 20) | ((aclk) << 4)))
+
+static const struct exynos_cpuclk_cfg_data exynos5420_kfcclk_d[] __initconst = {
+	{ 1400000, E5420_KFC_DIV(3, 5, 3), }, /* for Exynos5800 */
+	{ 1300000, E5420_KFC_DIV(3, 5, 2), },
+	{ 1200000, E5420_KFC_DIV(3, 5, 2), },
+	{ 1100000, E5420_KFC_DIV(3, 5, 2), },
+	{ 1000000, E5420_KFC_DIV(3, 5, 2), },
+	{  900000, E5420_KFC_DIV(3, 5, 2), },
+	{  800000, E5420_KFC_DIV(3, 5, 2), },
+	{  700000, E5420_KFC_DIV(3, 4, 2), },
+	{  600000, E5420_KFC_DIV(3, 4, 2), },
+	{  500000, E5420_KFC_DIV(3, 4, 2), },
+	{  400000, E5420_KFC_DIV(3, 3, 2), },
+	{  300000, E5420_KFC_DIV(3, 3, 2), },
+	{  200000, E5420_KFC_DIV(3, 3, 2), },
+	{  0 },
+};
+
 static const struct of_device_id ext_clk_match[] __initconst = {
 	{ .compatible = "samsung,exynos5420-oscclk", .data = (void *)0, },
 	{ },
@@ -1310,6 +1381,19 @@ static void __init exynos5x_clk_init(struct device_node *np,
 				ARRAY_SIZE(exynos5800_gate_clks));
 	}
 
+	if (soc == EXYNOS5420) {
+		exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk",
+			mout_cpu_p[0], mout_cpu_p[1], 0x200,
+			exynos5420_eglclk_d, ARRAY_SIZE(exynos5420_eglclk_d), 0);
+	} else {
+		exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk",
+			mout_cpu_p[0], mout_cpu_p[1], 0x200,
+			exynos5800_eglclk_d, ARRAY_SIZE(exynos5800_eglclk_d), 0);
+	}
+	exynos_register_cpu_clock(ctx, CLK_KFC_CLK, "kfcclk",
+		mout_kfc_p[0], mout_kfc_p[1], 0x28200,
+		exynos5420_kfcclk_d, ARRAY_SIZE(exynos5420_kfcclk_d), 0);
+
 	exynos5420_clk_sleep_init();
 
 	samsung_clk_of_add_provider(np, ctx);
diff --git a/drivers/clk/samsung/clk-s3c2410-dclk.c b/drivers/clk/samsung/clk-s3c2410-dclk.c
index e9eb935d7616..ec6fb14d951c 100644
--- a/drivers/clk/samsung/clk-s3c2410-dclk.c
+++ b/drivers/clk/samsung/clk-s3c2410-dclk.c
@@ -77,12 +77,11 @@ static u8 s3c24xx_clkout_get_parent(struct clk_hw *hw)
 static int s3c24xx_clkout_set_parent(struct clk_hw *hw, u8 index)
 {
 	struct s3c24xx_clkout *clkout = to_s3c24xx_clkout(hw);
-	int ret = 0;
 
 	s3c2410_modify_misccr((clkout->mask << clkout->shift),
 			      (index << clkout->shift));
 
-	return ret;
+	return 0;
 }
 
 static const struct clk_ops s3c24xx_clkout_ops = {
diff --git a/drivers/clk/shmobile/Makefile b/drivers/clk/shmobile/Makefile
index 97c71c885e4f..7e2579b30326 100644
--- a/drivers/clk/shmobile/Makefile
+++ b/drivers/clk/shmobile/Makefile
@@ -1,13 +1,13 @@
 obj-$(CONFIG_ARCH_EMEV2)		+= clk-emev2.o
-obj-$(CONFIG_ARCH_R7S72100)		+= clk-rz.o
-obj-$(CONFIG_ARCH_R8A73A4)		+= clk-r8a73a4.o
-obj-$(CONFIG_ARCH_R8A7740)		+= clk-r8a7740.o
-obj-$(CONFIG_ARCH_R8A7778)		+= clk-r8a7778.o
-obj-$(CONFIG_ARCH_R8A7779)		+= clk-r8a7779.o
-obj-$(CONFIG_ARCH_R8A7790)		+= clk-rcar-gen2.o
-obj-$(CONFIG_ARCH_R8A7791)		+= clk-rcar-gen2.o
-obj-$(CONFIG_ARCH_R8A7793)		+= clk-rcar-gen2.o
-obj-$(CONFIG_ARCH_R8A7794)		+= clk-rcar-gen2.o
-obj-$(CONFIG_ARCH_SH73A0)		+= clk-sh73a0.o
-obj-$(CONFIG_ARCH_SHMOBILE_MULTI)	+= clk-div6.o
-obj-$(CONFIG_ARCH_SHMOBILE_MULTI)	+= clk-mstp.o
+obj-$(CONFIG_ARCH_R7S72100)		+= clk-rz.o clk-mstp.o
+obj-$(CONFIG_ARCH_R8A73A4)		+= clk-r8a73a4.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7740)		+= clk-r8a7740.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7778)		+= clk-r8a7778.o clk-mstp.o
+obj-$(CONFIG_ARCH_R8A7779)		+= clk-r8a7779.o clk-mstp.o
+obj-$(CONFIG_ARCH_R8A7790)		+= clk-rcar-gen2.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7791)		+= clk-rcar-gen2.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7793)		+= clk-rcar-gen2.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7794)		+= clk-rcar-gen2.o clk-mstp.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7795)		+= renesas-cpg-mssr.o \
+					   r8a7795-cpg-mssr.o clk-div6.o
+obj-$(CONFIG_ARCH_SH73A0)		+= clk-sh73a0.o clk-mstp.o clk-div6.o
diff --git a/drivers/clk/shmobile/clk-div6.c b/drivers/clk/shmobile/clk-div6.c
index b4c8d6746f68..999994769450 100644
--- a/drivers/clk/shmobile/clk-div6.c
+++ b/drivers/clk/shmobile/clk-div6.c
@@ -18,6 +18,8 @@
 #include <linux/of_address.h>
 #include <linux/slab.h>
 
+#include "clk-div6.h"
+
 #define CPG_DIV6_CKSTP		BIT(8)
 #define CPG_DIV6_DIV(d)		((d) & 0x3f)
 #define CPG_DIV6_DIV_MASK	0x3f
@@ -172,67 +174,44 @@ static const struct clk_ops cpg_div6_clock_ops = {
 	.set_rate = cpg_div6_clock_set_rate,
 };
 
-static void __init cpg_div6_clock_init(struct device_node *np)
+
+/**
+ * cpg_div6_register - Register a DIV6 clock
+ * @name: Name of the DIV6 clock
+ * @num_parents: Number of parent clocks of the DIV6 clock (1, 4, or 8)
+ * @parent_names: Array containing the names of the parent clocks
+ * @reg: Mapped register used to control the DIV6 clock
+ */
+struct clk * __init cpg_div6_register(const char *name,
+				      unsigned int num_parents,
+				      const char **parent_names,
+				      void __iomem *reg)
 {
-	unsigned int num_parents, valid_parents;
-	const char **parent_names;
+	unsigned int valid_parents;
 	struct clk_init_data init;
 	struct div6_clock *clock;
-	const char *name;
 	struct clk *clk;
 	unsigned int i;
-	int ret;
 
 	clock = kzalloc(sizeof(*clock), GFP_KERNEL);
 	if (!clock)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	num_parents = of_clk_get_parent_count(np);
-	if (num_parents < 1) {
-		pr_err("%s: no parent found for %s DIV6 clock\n",
-		       __func__, np->name);
-		return;
+	clock->parents = kmalloc_array(num_parents, sizeof(*clock->parents),
+				       GFP_KERNEL);
+	if (!clock->parents) {
+		clk = ERR_PTR(-ENOMEM);
+		goto free_clock;
 	}
 
-	clock->parents = kmalloc_array(num_parents, sizeof(*clock->parents),
-		GFP_KERNEL);
-	parent_names = kmalloc_array(num_parents, sizeof(*parent_names),
-				GFP_KERNEL);
-	if (!parent_names)
-		return;
+	clock->reg = reg;
 
-	/* Remap the clock register and read the divisor. Disabling the
-	 * clock overwrites the divisor, so we need to cache its value for the
-	 * enable operation.
+	/*
+	 * Read the divisor. Disabling the clock overwrites the divisor, so we
+	 * need to cache its value for the enable operation.
 	 */
-	clock->reg = of_iomap(np, 0);
-	if (clock->reg == NULL) {
-		pr_err("%s: failed to map %s DIV6 clock register\n",
-		       __func__, np->name);
-		goto error;
-	}
-
 	clock->div = (clk_readl(clock->reg) & CPG_DIV6_DIV_MASK) + 1;
 
-	/* Parse the DT properties. */
-	ret = of_property_read_string(np, "clock-output-names", &name);
-	if (ret < 0) {
-		pr_err("%s: failed to get %s DIV6 clock output name\n",
-		       __func__, np->name);
-		goto error;
-	}
-
-
-	for (i = 0, valid_parents = 0; i < num_parents; i++) {
-		const char *name = of_clk_get_parent_name(np, i);
-
-		if (name) {
-			parent_names[valid_parents] = name;
-			clock->parents[valid_parents] = i;
-			valid_parents++;
-		}
-	}
-
 	switch (num_parents) {
 	case 1:
 		/* fixed parent clock */
@@ -250,8 +229,18 @@ static void __init cpg_div6_clock_init(struct device_node *np)
 		break;
 	default:
 		pr_err("%s: invalid number of parents for DIV6 clock %s\n",
-		       __func__, np->name);
-		goto error;
+		       __func__, name);
+		clk = ERR_PTR(-EINVAL);
+		goto free_parents;
+	}
+
+	/* Filter out invalid parents */
+	for (i = 0, valid_parents = 0; i < num_parents; i++) {
+		if (parent_names[i]) {
+			parent_names[valid_parents] = parent_names[i];
+			clock->parents[valid_parents] = i;
+			valid_parents++;
+		}
 	}
 
 	/* Register the clock. */
@@ -264,6 +253,53 @@ static void __init cpg_div6_clock_init(struct device_node *np)
 	clock->hw.init = &init;
 
 	clk = clk_register(NULL, &clock->hw);
+	if (IS_ERR(clk))
+		goto free_parents;
+
+	return clk;
+
+free_parents:
+	kfree(clock->parents);
+free_clock:
+	kfree(clock);
+	return clk;
+}
+
+static void __init cpg_div6_clock_init(struct device_node *np)
+{
+	unsigned int num_parents;
+	const char **parent_names;
+	const char *clk_name = np->name;
+	void __iomem *reg;
+	struct clk *clk;
+	unsigned int i;
+
+	num_parents = of_clk_get_parent_count(np);
+	if (num_parents < 1) {
+		pr_err("%s: no parent found for %s DIV6 clock\n",
+		       __func__, np->name);
+		return;
+	}
+
+	parent_names = kmalloc_array(num_parents, sizeof(*parent_names),
+				GFP_KERNEL);
+	if (!parent_names)
+		return;
+
+	reg = of_iomap(np, 0);
+	if (reg == NULL) {
+		pr_err("%s: failed to map %s DIV6 clock register\n",
+		       __func__, np->name);
+		goto error;
+	}
+
+	/* Parse the DT properties. */
+	of_property_read_string(np, "clock-output-names", &clk_name);
+
+	for (i = 0; i < num_parents; i++)
+		parent_names[i] = of_clk_get_parent_name(np, i);
+
+	clk = cpg_div6_register(clk_name, num_parents, parent_names, reg);
 	if (IS_ERR(clk)) {
 		pr_err("%s: failed to register %s DIV6 clock (%ld)\n",
 		       __func__, np->name, PTR_ERR(clk));
@@ -276,9 +312,8 @@ static void __init cpg_div6_clock_init(struct device_node *np)
 	return;
 
 error:
-	if (clock->reg)
-		iounmap(clock->reg);
+	if (reg)
+		iounmap(reg);
 	kfree(parent_names);
-	kfree(clock);
 }
 CLK_OF_DECLARE(cpg_div6_clk, "renesas,cpg-div6-clock", cpg_div6_clock_init);
diff --git a/drivers/clk/shmobile/clk-div6.h b/drivers/clk/shmobile/clk-div6.h
new file mode 100644
index 000000000000..9a85a95188da
--- /dev/null
+++ b/drivers/clk/shmobile/clk-div6.h
@@ -0,0 +1,7 @@
+#ifndef __SHMOBILE_CLK_DIV6_H__
+#define __SHMOBILE_CLK_DIV6_H__
+
+struct clk *cpg_div6_register(const char *name, unsigned int num_parents,
+			      const char **parent_names, void __iomem *reg);
+
+#endif
diff --git a/drivers/clk/shmobile/clk-rcar-gen2.c b/drivers/clk/shmobile/clk-rcar-gen2.c
index 745496f7ee9c..841977240305 100644
--- a/drivers/clk/shmobile/clk-rcar-gen2.c
+++ b/drivers/clk/shmobile/clk-rcar-gen2.c
@@ -115,7 +115,7 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 	 *
 	 * Using experimental measurements, it seems that no more than
 	 * ~10 iterations are needed, independently of the CPU rate.
-	 * Since this value might be dependant of external xtal rate, pll1
+	 * Since this value might be dependent on external xtal rate, pll1
 	 * rate or even the other emulation clocks rate, use 1000 as a
 	 * "super" safe value.
 	 */
@@ -262,7 +262,7 @@ static struct clk * __init cpg_adsp_clk_register(struct rcar_gen2_cpg *cpg)
  * 1  1  0	30 / 2		x172/2	x208/2	x106
  * 1  1  1	30 / 2		x172/2	x208/2	x88
  *
- * *1 :	Table 7.6 indicates VCO ouput (PLLx = VCO/2)
+ * *1 :	Table 7.6 indicates VCO output (PLLx = VCO/2)
  */
 #define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 12) | \
 					 (((md) & BIT(13)) >> 12) | \
diff --git a/drivers/clk/shmobile/r8a7795-cpg-mssr.c b/drivers/clk/shmobile/r8a7795-cpg-mssr.c
new file mode 100644
index 000000000000..13e994772dfd
--- /dev/null
+++ b/drivers/clk/shmobile/r8a7795-cpg-mssr.c
@@ -0,0 +1,383 @@
+/*
+ * r8a7795 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2015 Glider bvba
+ *
+ * Based on clk-rcar-gen3.c
+ *
+ * Copyright (C) 2015 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/bug.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+
+#include <dt-bindings/clock/r8a7795-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A7795_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_EXTALR,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL2,
+	CLK_PLL3,
+	CLK_PLL4,
+	CLK_PLL1_DIV2,
+	CLK_PLL1_DIV4,
+	CLK_S0,
+	CLK_S1,
+	CLK_S2,
+	CLK_S3,
+	CLK_SDSRC,
+	CLK_SSPSRC,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+enum r8a7795_clk_types {
+	CLK_TYPE_GEN3_MAIN = CLK_TYPE_CUSTOM,
+	CLK_TYPE_GEN3_PLL0,
+	CLK_TYPE_GEN3_PLL1,
+	CLK_TYPE_GEN3_PLL2,
+	CLK_TYPE_GEN3_PLL3,
+	CLK_TYPE_GEN3_PLL4,
+};
+
+static const struct cpg_core_clk r8a7795_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",  CLK_EXTAL),
+	DEF_INPUT("extalr", CLK_EXTALR),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",       CLK_MAIN, CLK_TYPE_GEN3_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",       CLK_PLL0, CLK_TYPE_GEN3_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",       CLK_PLL1, CLK_TYPE_GEN3_PLL1, CLK_MAIN),
+	DEF_BASE(".pll2",       CLK_PLL2, CLK_TYPE_GEN3_PLL2, CLK_MAIN),
+	DEF_BASE(".pll3",       CLK_PLL3, CLK_TYPE_GEN3_PLL3, CLK_MAIN),
+	DEF_BASE(".pll4",       CLK_PLL4, CLK_TYPE_GEN3_PLL4, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2,     CLK_PLL1,       2, 1),
+	DEF_FIXED(".pll1_div4", CLK_PLL1_DIV4,     CLK_PLL1_DIV2,  2, 1),
+	DEF_FIXED(".s0",        CLK_S0,            CLK_PLL1_DIV2,  2, 1),
+	DEF_FIXED(".s1",        CLK_S1,            CLK_PLL1_DIV2,  3, 1),
+	DEF_FIXED(".s2",        CLK_S2,            CLK_PLL1_DIV2,  4, 1),
+	DEF_FIXED(".s3",        CLK_S3,            CLK_PLL1_DIV2,  6, 1),
+
+	/* Core Clock Outputs */
+	DEF_FIXED("ztr",        R8A7795_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
+	DEF_FIXED("ztrd2",      R8A7795_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1),
+	DEF_FIXED("zt",         R8A7795_CLK_ZT,    CLK_PLL1_DIV2,  4, 1),
+	DEF_FIXED("zx",         R8A7795_CLK_ZX,    CLK_PLL1_DIV2,  2, 1),
+	DEF_FIXED("s0d1",       R8A7795_CLK_S0D1,  CLK_S0,         1, 1),
+	DEF_FIXED("s0d4",       R8A7795_CLK_S0D4,  CLK_S0,         4, 1),
+	DEF_FIXED("s1d1",       R8A7795_CLK_S1D1,  CLK_S1,         1, 1),
+	DEF_FIXED("s1d2",       R8A7795_CLK_S1D2,  CLK_S1,         2, 1),
+	DEF_FIXED("s1d4",       R8A7795_CLK_S1D4,  CLK_S1,         4, 1),
+	DEF_FIXED("s2d1",       R8A7795_CLK_S2D1,  CLK_S2,         1, 1),
+	DEF_FIXED("s2d2",       R8A7795_CLK_S2D2,  CLK_S2,         2, 1),
+	DEF_FIXED("s2d4",       R8A7795_CLK_S2D4,  CLK_S2,         4, 1),
+	DEF_FIXED("s3d1",       R8A7795_CLK_S3D1,  CLK_S3,         1, 1),
+	DEF_FIXED("s3d2",       R8A7795_CLK_S3D2,  CLK_S3,         2, 1),
+	DEF_FIXED("s3d4",       R8A7795_CLK_S3D4,  CLK_S3,         4, 1),
+	DEF_FIXED("cl",         R8A7795_CLK_CL,    CLK_PLL1_DIV2, 48, 1),
+	DEF_FIXED("cp",         R8A7795_CLK_CP,    CLK_EXTAL,      2, 1),
+
+	DEF_DIV6P1("mso",       R8A7795_CLK_MSO,   CLK_PLL1_DIV4, 0x014),
+	DEF_DIV6P1("hdmi",      R8A7795_CLK_HDMI,  CLK_PLL1_DIV2, 0x250),
+};
+
+static const struct mssr_mod_clk r8a7795_mod_clks[] __initconst = {
+	DEF_MOD("scif5",		 202,	R8A7795_CLK_S3D4),
+	DEF_MOD("scif4",		 203,	R8A7795_CLK_S3D4),
+	DEF_MOD("scif3",		 204,	R8A7795_CLK_S3D4),
+	DEF_MOD("scif1",		 206,	R8A7795_CLK_S3D4),
+	DEF_MOD("scif0",		 207,	R8A7795_CLK_S3D4),
+	DEF_MOD("msiof3",		 208,	R8A7795_CLK_MSO),
+	DEF_MOD("msiof2",		 209,	R8A7795_CLK_MSO),
+	DEF_MOD("msiof1",		 210,	R8A7795_CLK_MSO),
+	DEF_MOD("msiof0",		 211,	R8A7795_CLK_MSO),
+	DEF_MOD("sys-dmac2",		 217,	R8A7795_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A7795_CLK_S3D1),
+	DEF_MOD("sys-dmac0",		 219,	R8A7795_CLK_S3D1),
+	DEF_MOD("scif2",		 310,	R8A7795_CLK_S3D4),
+	DEF_MOD("pcie1",		 318,	R8A7795_CLK_S3D1),
+	DEF_MOD("pcie0",		 319,	R8A7795_CLK_S3D1),
+	DEF_MOD("intc-ap",		 408,	R8A7795_CLK_S3D1),
+	DEF_MOD("audmac0",		 502,	R8A7795_CLK_S3D4),
+	DEF_MOD("audmac1",		 501,	R8A7795_CLK_S3D4),
+	DEF_MOD("hscif4",		 516,	R8A7795_CLK_S3D1),
+	DEF_MOD("hscif3",		 517,	R8A7795_CLK_S3D1),
+	DEF_MOD("hscif2",		 518,	R8A7795_CLK_S3D1),
+	DEF_MOD("hscif1",		 519,	R8A7795_CLK_S3D1),
+	DEF_MOD("hscif0",		 520,	R8A7795_CLK_S3D1),
+	DEF_MOD("vspd3",		 620,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspd2",		 621,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspd1",		 622,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspd0",		 623,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspbc",		 624,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspbd",		 626,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspi2",		 629,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspi1",		 630,	R8A7795_CLK_S2D1),
+	DEF_MOD("vspi0",		 631,	R8A7795_CLK_S2D1),
+	DEF_MOD("ehci2",		 701,	R8A7795_CLK_S3D4),
+	DEF_MOD("ehci1",		 702,	R8A7795_CLK_S3D4),
+	DEF_MOD("ehci0",		 703,	R8A7795_CLK_S3D4),
+	DEF_MOD("hsusb",		 704,	R8A7795_CLK_S3D4),
+	DEF_MOD("du3",			 721,	R8A7795_CLK_S2D1),
+	DEF_MOD("du2",			 722,	R8A7795_CLK_S2D1),
+	DEF_MOD("du1",			 723,	R8A7795_CLK_S2D1),
+	DEF_MOD("du0",			 724,	R8A7795_CLK_S2D1),
+	DEF_MOD("hdmi1",		 728,	R8A7795_CLK_HDMI),
+	DEF_MOD("hdmi0",		 729,	R8A7795_CLK_HDMI),
+	DEF_MOD("etheravb",		 812,	R8A7795_CLK_S3D2),
+	DEF_MOD("sata0",		 815,	R8A7795_CLK_S3D2),
+	DEF_MOD("gpio7",		 905,	R8A7795_CLK_CP),
+	DEF_MOD("gpio6",		 906,	R8A7795_CLK_CP),
+	DEF_MOD("gpio5",		 907,	R8A7795_CLK_CP),
+	DEF_MOD("gpio4",		 908,	R8A7795_CLK_CP),
+	DEF_MOD("gpio3",		 909,	R8A7795_CLK_CP),
+	DEF_MOD("gpio2",		 910,	R8A7795_CLK_CP),
+	DEF_MOD("gpio1",		 911,	R8A7795_CLK_CP),
+	DEF_MOD("gpio0",		 912,	R8A7795_CLK_CP),
+	DEF_MOD("i2c6",			 918,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c5",			 919,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c4",			 927,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c3",			 928,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c2",			 929,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c1",			 930,	R8A7795_CLK_S3D2),
+	DEF_MOD("i2c0",			 931,	R8A7795_CLK_S3D2),
+	DEF_MOD("ssi-all",		1005,	R8A7795_CLK_S3D4),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A7795_CLK_S3D4),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src9",		1022,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src8",		1023,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src7",		1024,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src0",		1031,	MOD_CLK_ID(1017)),
+};
+
+static const unsigned int r8a7795_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-AP (GIC) */
+};
+
+
+#define CPG_PLL0CR	0x00d8
+#define CPG_PLL2CR	0x002c
+#define CPG_PLL4CR	0x01f4
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *   MD		EXTAL		PLL0	PLL1	PLL2	PLL3	PLL4
+ * 14 13 19 17	(MHz)
+ *-------------------------------------------------------------------
+ * 0  0  0  0	16.66 x 1	x180	x192	x144	x192	x144
+ * 0  0  0  1	16.66 x 1	x180	x192	x144	x128	x144
+ * 0  0  1  0	Prohibited setting
+ * 0  0  1  1	16.66 x 1	x180	x192	x144	x192	x144
+ * 0  1  0  0	20    x 1	x150	x160	x120	x160	x120
+ * 0  1  0  1	20    x 1	x150	x160	x120	x106	x120
+ * 0  1  1  0	Prohibited setting
+ * 0  1  1  1	20    x 1	x150	x160	x120	x160	x120
+ * 1  0  0  0	25    x 1	x120	x128	x96	x128	x96
+ * 1  0  0  1	25    x 1	x120	x128	x96	x84	x96
+ * 1  0  1  0	Prohibited setting
+ * 1  0  1  1	25    x 1	x120	x128	x96	x128	x96
+ * 1  1  0  0	33.33 / 2	x180	x192	x144	x192	x144
+ * 1  1  0  1	33.33 / 2	x180	x192	x144	x128	x144
+ * 1  1  1  0	Prohibited setting
+ * 1  1  1  1	33.33 / 2	x180	x192	x144	x192	x144
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 11) | \
+					 (((md) & BIT(13)) >> 11) | \
+					 (((md) & BIT(19)) >> 18) | \
+					 (((md) & BIT(17)) >> 17))
+
+struct cpg_pll_config {
+	unsigned int extal_div;
+	unsigned int pll1_mult;
+	unsigned int pll3_mult;
+};
+
+static const struct cpg_pll_config cpg_pll_configs[16] __initconst = {
+	/* EXTAL div	PLL1 mult	PLL3 mult */
+	{ 1,		192,		192,	},
+	{ 1,		192,		128,	},
+	{ 0, /* Prohibited setting */		},
+	{ 1,		192,		192,	},
+	{ 1,		160,		160,	},
+	{ 1,		160,		106,	},
+	{ 0, /* Prohibited setting */		},
+	{ 1,		160,		160,	},
+	{ 1,		128,		128,	},
+	{ 1,		128,		84,	},
+	{ 0, /* Prohibited setting */		},
+	{ 1,		128,		128,	},
+	{ 2,		192,		192,	},
+	{ 2,		192,		128,	},
+	{ 0, /* Prohibited setting */		},
+	{ 2,		192,		192,	},
+};
+
+static const struct cpg_pll_config *cpg_pll_config __initdata;
+
+static
+struct clk * __init r8a7795_cpg_clk_register(struct device *dev,
+					     const struct cpg_core_clk *core,
+					     const struct cpg_mssr_info *info,
+					     struct clk **clks,
+					     void __iomem *base)
+{
+	const struct clk *parent;
+	unsigned int mult = 1;
+	unsigned int div = 1;
+	u32 value;
+
+	parent = clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	switch (core->type) {
+	case CLK_TYPE_GEN3_MAIN:
+		div = cpg_pll_config->extal_div;
+		break;
+
+	case CLK_TYPE_GEN3_PLL0:
+		/*
+		 * PLL0 is a configurable multiplier clock. Register it as a
+		 * fixed factor clock for now as there's no generic multiplier
+		 * clock implementation and we currently have no need to change
+		 * the multiplier value.
+		 */
+		value = readl(base + CPG_PLL0CR);
+		mult = (((value >> 24) & 0x7f) + 1) * 2;
+		break;
+
+	case CLK_TYPE_GEN3_PLL1:
+		mult = cpg_pll_config->pll1_mult;
+		break;
+
+	case CLK_TYPE_GEN3_PLL2:
+		/*
+		 * PLL2 is a configurable multiplier clock. Register it as a
+		 * fixed factor clock for now as there's no generic multiplier
+		 * clock implementation and we currently have no need to change
+		 * the multiplier value.
+		 */
+		value = readl(base + CPG_PLL2CR);
+		mult = (((value >> 24) & 0x7f) + 1) * 2;
+		break;
+
+	case CLK_TYPE_GEN3_PLL3:
+		mult = cpg_pll_config->pll3_mult;
+		break;
+
+	case CLK_TYPE_GEN3_PLL4:
+		/*
+		 * PLL4 is a configurable multiplier clock. Register it as a
+		 * fixed factor clock for now as there's no generic multiplier
+		 * clock implementation and we currently have no need to change
+		 * the multiplier value.
+		 */
+		value = readl(base + CPG_PLL4CR);
+		mult = (((value >> 24) & 0x7f) + 1) * 2;
+		break;
+
+	default:
+		return ERR_PTR(-EINVAL);
+	}
+
+	return clk_register_fixed_factor(NULL, core->name,
+					 __clk_get_name(parent), 0, mult, div);
+}
+
+/*
+ * Reset register definitions.
+ */
+#define MODEMR	0xe6160060
+
+static u32 rcar_gen3_read_mode_pins(void)
+{
+	void __iomem *modemr = ioremap_nocache(MODEMR, 4);
+	u32 mode;
+
+	BUG_ON(!modemr);
+	mode = ioread32(modemr);
+	iounmap(modemr);
+
+	return mode;
+}
+
+static int __init r8a7795_cpg_mssr_init(struct device *dev)
+{
+	u32 cpg_mode = rcar_gen3_read_mode_pins();
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+	if (!cpg_pll_config->extal_div) {
+		dev_err(dev, "Prohibited setting (cpg_mode=0x%x)\n", cpg_mode);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+const struct cpg_mssr_info r8a7795_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a7795_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a7795_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a7795_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a7795_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a7795_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a7795_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a7795_cpg_mssr_init,
+	.cpg_clk_register = r8a7795_cpg_clk_register,
+};
diff --git a/drivers/clk/shmobile/renesas-cpg-mssr.c b/drivers/clk/shmobile/renesas-cpg-mssr.c
new file mode 100644
index 000000000000..9a4d888164bb
--- /dev/null
+++ b/drivers/clk/shmobile/renesas-cpg-mssr.c
@@ -0,0 +1,596 @@
+/*
+ * Renesas Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2015 Glider bvba
+ *
+ * Based on clk-mstp.c, clk-rcar-gen2.c, and clk-rcar-gen3.c
+ *
+ * Copyright (C) 2013 Ideas On Board SPRL
+ * Copyright (C) 2015 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_domain.h>
+#include <linux/slab.h>
+
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "clk-div6.h"
+
+#ifdef DEBUG
+#define WARN_DEBUG(x)	do { } while (0)
+#else
+#define WARN_DEBUG(x)	WARN_ON(x)
+#endif
+
+
+/*
+ * Module Standby and Software Reset register offets.
+ *
+ * If the registers exist, these are valid for SH-Mobile, R-Mobile,
+ * R-Car Gen 2, and R-Car Gen 3.
+ * These are NOT valid for R-Car Gen1 and RZ/A1!
+ */
+
+/*
+ * Module Stop Status Register offsets
+ */
+
+static const u16 mstpsr[] = {
+	0x030, 0x038, 0x040, 0x048, 0x04C, 0x03C, 0x1C0, 0x1C4,
+	0x9A0, 0x9A4, 0x9A8, 0x9AC,
+};
+
+#define	MSTPSR(i)	mstpsr[i]
+
+
+/*
+ * System Module Stop Control Register offsets
+ */
+
+static const u16 smstpcr[] = {
+	0x130, 0x134, 0x138, 0x13C, 0x140, 0x144, 0x148, 0x14C,
+	0x990, 0x994, 0x998, 0x99C,
+};
+
+#define	SMSTPCR(i)	smstpcr[i]
+
+
+/*
+ * Software Reset Register offsets
+ */
+
+static const u16 srcr[] = {
+	0x0A0, 0x0A8, 0x0B0, 0x0B8, 0x0BC, 0x0C4, 0x1C8, 0x1CC,
+	0x920, 0x924, 0x928, 0x92C,
+};
+
+#define	SRCR(i)		srcr[i]
+
+
+/* Realtime Module Stop Control Register offsets */
+#define RMSTPCR(i)	(smstpcr[i] - 0x20)
+
+/* Modem Module Stop Control Register offsets (r8a73a4) */
+#define MMSTPCR(i)	(smstpcr[i] + 0x20)
+
+/* Software Reset Clearing Register offsets */
+#define	SRSTCLR(i)	(0x940 + (i) * 4)
+
+
+/**
+ * Clock Pulse Generator / Module Standby and Software Reset Private Data
+ *
+ * @dev: CPG/MSSR device
+ * @base: CPG/MSSR register block base address
+ * @mstp_lock: protects writes to SMSTPCR
+ * @clks: Array containing all Core and Module Clocks
+ * @num_core_clks: Number of Core Clocks in clks[]
+ * @num_mod_clks: Number of Module Clocks in clks[]
+ * @last_dt_core_clk: ID of the last Core Clock exported to DT
+ */
+struct cpg_mssr_priv {
+	struct device *dev;
+	void __iomem *base;
+	spinlock_t mstp_lock;
+
+	struct clk **clks;
+	unsigned int num_core_clks;
+	unsigned int num_mod_clks;
+	unsigned int last_dt_core_clk;
+};
+
+
+/**
+ * struct mstp_clock - MSTP gating clock
+ * @hw: handle between common and hardware-specific interfaces
+ * @index: MSTP clock number
+ * @priv: CPG/MSSR private data
+ */
+struct mstp_clock {
+	struct clk_hw hw;
+	u32 index;
+	struct cpg_mssr_priv *priv;
+};
+
+#define to_mstp_clock(_hw) container_of(_hw, struct mstp_clock, hw)
+
+static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable)
+{
+	struct mstp_clock *clock = to_mstp_clock(hw);
+	struct cpg_mssr_priv *priv = clock->priv;
+	unsigned int reg = clock->index / 32;
+	unsigned int bit = clock->index % 32;
+	struct device *dev = priv->dev;
+	u32 bitmask = BIT(bit);
+	unsigned long flags;
+	unsigned int i;
+	u32 value;
+
+	dev_dbg(dev, "MSTP %u%02u/%pC %s\n", reg, bit, hw->clk,
+		enable ? "ON" : "OFF");
+	spin_lock_irqsave(&priv->mstp_lock, flags);
+
+	value = clk_readl(priv->base + SMSTPCR(reg));
+	if (enable)
+		value &= ~bitmask;
+	else
+		value |= bitmask;
+	clk_writel(value, priv->base + SMSTPCR(reg));
+
+	spin_unlock_irqrestore(&priv->mstp_lock, flags);
+
+	if (!enable)
+		return 0;
+
+	for (i = 1000; i > 0; --i) {
+		if (!(clk_readl(priv->base + MSTPSR(reg)) &
+		      bitmask))
+			break;
+		cpu_relax();
+	}
+
+	if (!i) {
+		dev_err(dev, "Failed to enable SMSTP %p[%d]\n",
+			priv->base + SMSTPCR(reg), bit);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int cpg_mstp_clock_enable(struct clk_hw *hw)
+{
+	return cpg_mstp_clock_endisable(hw, true);
+}
+
+static void cpg_mstp_clock_disable(struct clk_hw *hw)
+{
+	cpg_mstp_clock_endisable(hw, false);
+}
+
+static int cpg_mstp_clock_is_enabled(struct clk_hw *hw)
+{
+	struct mstp_clock *clock = to_mstp_clock(hw);
+	struct cpg_mssr_priv *priv = clock->priv;
+	u32 value;
+
+	value = clk_readl(priv->base + MSTPSR(clock->index / 32));
+
+	return !(value & BIT(clock->index % 32));
+}
+
+static const struct clk_ops cpg_mstp_clock_ops = {
+	.enable = cpg_mstp_clock_enable,
+	.disable = cpg_mstp_clock_disable,
+	.is_enabled = cpg_mstp_clock_is_enabled,
+};
+
+static
+struct clk *cpg_mssr_clk_src_twocell_get(struct of_phandle_args *clkspec,
+					 void *data)
+{
+	unsigned int clkidx = clkspec->args[1];
+	struct cpg_mssr_priv *priv = data;
+	struct device *dev = priv->dev;
+	unsigned int idx;
+	const char *type;
+	struct clk *clk;
+
+	switch (clkspec->args[0]) {
+	case CPG_CORE:
+		type = "core";
+		if (clkidx > priv->last_dt_core_clk) {
+			dev_err(dev, "Invalid %s clock index %u\n", type,
+			       clkidx);
+			return ERR_PTR(-EINVAL);
+		}
+		clk = priv->clks[clkidx];
+		break;
+
+	case CPG_MOD:
+		type = "module";
+		idx = MOD_CLK_PACK(clkidx);
+		if (clkidx % 100 > 31 || idx >= priv->num_mod_clks) {
+			dev_err(dev, "Invalid %s clock index %u\n", type,
+				clkidx);
+			return ERR_PTR(-EINVAL);
+		}
+		clk = priv->clks[priv->num_core_clks + idx];
+		break;
+
+	default:
+		dev_err(dev, "Invalid CPG clock type %u\n", clkspec->args[0]);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (IS_ERR(clk))
+		dev_err(dev, "Cannot get %s clock %u: %ld", type, clkidx,
+		       PTR_ERR(clk));
+	else
+		dev_dbg(dev, "clock (%u, %u) is %pC at %pCr Hz\n",
+			clkspec->args[0], clkspec->args[1], clk, clk);
+	return clk;
+}
+
+static void __init cpg_mssr_register_core_clk(const struct cpg_core_clk *core,
+					      const struct cpg_mssr_info *info,
+					      struct cpg_mssr_priv *priv)
+{
+	struct clk *clk = NULL, *parent;
+	struct device *dev = priv->dev;
+	unsigned int id = core->id;
+	const char *parent_name;
+
+	WARN_DEBUG(id >= priv->num_core_clks);
+	WARN_DEBUG(PTR_ERR(priv->clks[id]) != -ENOENT);
+
+	switch (core->type) {
+	case CLK_TYPE_IN:
+		clk = of_clk_get_by_name(priv->dev->of_node, core->name);
+		break;
+
+	case CLK_TYPE_FF:
+	case CLK_TYPE_DIV6P1:
+		WARN_DEBUG(core->parent >= priv->num_core_clks);
+		parent = priv->clks[core->parent];
+		if (IS_ERR(parent)) {
+			clk = parent;
+			goto fail;
+		}
+
+		parent_name = __clk_get_name(parent);
+		if (core->type == CLK_TYPE_FF) {
+			clk = clk_register_fixed_factor(NULL, core->name,
+							parent_name, 0,
+							core->mult, core->div);
+		} else {
+			clk = cpg_div6_register(core->name, 1, &parent_name,
+						priv->base + core->offset);
+		}
+		break;
+
+	default:
+		if (info->cpg_clk_register)
+			clk = info->cpg_clk_register(dev, core, info,
+						     priv->clks, priv->base);
+		else
+			dev_err(dev, "%s has unsupported core clock type %u\n",
+				core->name, core->type);
+		break;
+	}
+
+	if (IS_ERR_OR_NULL(clk))
+		goto fail;
+
+	dev_dbg(dev, "Core clock %pC at %pCr Hz\n", clk, clk);
+	priv->clks[id] = clk;
+	return;
+
+fail:
+	dev_err(dev, "Failed to register %s clock %s: %ld\n", "core,",
+		core->name, PTR_ERR(clk));
+}
+
+static void __init cpg_mssr_register_mod_clk(const struct mssr_mod_clk *mod,
+					     const struct cpg_mssr_info *info,
+					     struct cpg_mssr_priv *priv)
+{
+	struct mstp_clock *clock = NULL;
+	struct device *dev = priv->dev;
+	unsigned int id = mod->id;
+	struct clk_init_data init;
+	struct clk *parent, *clk;
+	const char *parent_name;
+	unsigned int i;
+
+	WARN_DEBUG(id < priv->num_core_clks);
+	WARN_DEBUG(id >= priv->num_core_clks + priv->num_mod_clks);
+	WARN_DEBUG(mod->parent >= priv->num_core_clks + priv->num_mod_clks);
+	WARN_DEBUG(PTR_ERR(priv->clks[id]) != -ENOENT);
+
+	parent = priv->clks[mod->parent];
+	if (IS_ERR(parent)) {
+		clk = parent;
+		goto fail;
+	}
+
+	clock = kzalloc(sizeof(*clock), GFP_KERNEL);
+	if (!clock) {
+		clk = ERR_PTR(-ENOMEM);
+		goto fail;
+	}
+
+	init.name = mod->name;
+	init.ops = &cpg_mstp_clock_ops;
+	init.flags = CLK_IS_BASIC | CLK_SET_RATE_PARENT;
+	for (i = 0; i < info->num_crit_mod_clks; i++)
+		if (id == info->crit_mod_clks[i]) {
+#ifdef CLK_ENABLE_HAND_OFF
+			dev_dbg(dev, "MSTP %s setting CLK_ENABLE_HAND_OFF\n",
+				mod->name);
+			init.flags |= CLK_ENABLE_HAND_OFF;
+			break;
+#else
+			dev_dbg(dev, "Ignoring MSTP %s to prevent disabling\n",
+				mod->name);
+			return;
+#endif
+		}
+
+	parent_name = __clk_get_name(parent);
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	clock->index = id - priv->num_core_clks;
+	clock->priv = priv;
+	clock->hw.init = &init;
+
+	clk = clk_register(NULL, &clock->hw);
+	if (IS_ERR(clk))
+		goto fail;
+
+	dev_dbg(dev, "Module clock %pC at %pCr Hz\n", clk, clk);
+	priv->clks[id] = clk;
+	return;
+
+fail:
+	dev_err(dev, "Failed to register %s clock %s: %ld\n", "module,",
+		mod->name, PTR_ERR(clk));
+	kfree(clock);
+}
+
+
+#ifdef CONFIG_PM_GENERIC_DOMAINS_OF
+struct cpg_mssr_clk_domain {
+	struct generic_pm_domain genpd;
+	struct device_node *np;
+	unsigned int num_core_pm_clks;
+	unsigned int core_pm_clks[0];
+};
+
+static bool cpg_mssr_is_pm_clk(const struct of_phandle_args *clkspec,
+			       struct cpg_mssr_clk_domain *pd)
+{
+	unsigned int i;
+
+	if (clkspec->np != pd->np || clkspec->args_count != 2)
+		return false;
+
+	switch (clkspec->args[0]) {
+	case CPG_CORE:
+		for (i = 0; i < pd->num_core_pm_clks; i++)
+			if (clkspec->args[1] == pd->core_pm_clks[i])
+				return true;
+		return false;
+
+	case CPG_MOD:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static int cpg_mssr_attach_dev(struct generic_pm_domain *genpd,
+			       struct device *dev)
+{
+	struct cpg_mssr_clk_domain *pd =
+		container_of(genpd, struct cpg_mssr_clk_domain, genpd);
+	struct device_node *np = dev->of_node;
+	struct of_phandle_args clkspec;
+	struct clk *clk;
+	int i = 0;
+	int error;
+
+	while (!of_parse_phandle_with_args(np, "clocks", "#clock-cells", i,
+					   &clkspec)) {
+		if (cpg_mssr_is_pm_clk(&clkspec, pd))
+			goto found;
+
+		of_node_put(clkspec.np);
+		i++;
+	}
+
+	return 0;
+
+found:
+	clk = of_clk_get_from_provider(&clkspec);
+	of_node_put(clkspec.np);
+
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	error = pm_clk_create(dev);
+	if (error) {
+		dev_err(dev, "pm_clk_create failed %d\n", error);
+		goto fail_put;
+	}
+
+	error = pm_clk_add_clk(dev, clk);
+	if (error) {
+		dev_err(dev, "pm_clk_add_clk %pC failed %d\n", clk, error);
+		goto fail_destroy;
+	}
+
+	return 0;
+
+fail_destroy:
+	pm_clk_destroy(dev);
+fail_put:
+	clk_put(clk);
+	return error;
+}
+
+static void cpg_mssr_detach_dev(struct generic_pm_domain *genpd,
+				struct device *dev)
+{
+	if (!list_empty(&dev->power.subsys_data->clock_list))
+		pm_clk_destroy(dev);
+}
+
+static int __init cpg_mssr_add_clk_domain(struct device *dev,
+					  const unsigned int *core_pm_clks,
+					  unsigned int num_core_pm_clks)
+{
+	struct device_node *np = dev->of_node;
+	struct generic_pm_domain *genpd;
+	struct cpg_mssr_clk_domain *pd;
+	size_t pm_size = num_core_pm_clks * sizeof(core_pm_clks[0]);
+
+	pd = devm_kzalloc(dev, sizeof(*pd) + pm_size, GFP_KERNEL);
+	if (!pd)
+		return -ENOMEM;
+
+	pd->np = np;
+	pd->num_core_pm_clks = num_core_pm_clks;
+	memcpy(pd->core_pm_clks, core_pm_clks, pm_size);
+
+	genpd = &pd->genpd;
+	genpd->name = np->name;
+	genpd->flags = GENPD_FLAG_PM_CLK;
+	pm_genpd_init(genpd, &simple_qos_governor, false);
+	genpd->attach_dev = cpg_mssr_attach_dev;
+	genpd->detach_dev = cpg_mssr_detach_dev;
+
+	of_genpd_add_provider_simple(np, genpd);
+	return 0;
+}
+#else
+static inline int cpg_mssr_add_clk_domain(struct device *dev,
+					  const unsigned int *core_pm_clks,
+					  unsigned int num_core_pm_clks)
+{
+	return 0;
+}
+#endif /* !CONFIG_PM_GENERIC_DOMAINS_OF */
+
+
+static const struct of_device_id cpg_mssr_match[] = {
+#ifdef CONFIG_ARCH_R8A7795
+	{
+		.compatible = "renesas,r8a7795-cpg-mssr",
+		.data = &r8a7795_cpg_mssr_info,
+	},
+#endif
+	{ /* sentinel */ }
+};
+
+static void cpg_mssr_del_clk_provider(void *data)
+{
+	of_clk_del_provider(data);
+}
+
+static int __init cpg_mssr_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	const struct cpg_mssr_info *info;
+	struct cpg_mssr_priv *priv;
+	unsigned int nclks, i;
+	struct resource *res;
+	struct clk **clks;
+	int error;
+
+	info = of_match_node(cpg_mssr_match, np)->data;
+	if (info->init) {
+		error = info->init(dev);
+		if (error)
+			return error;
+	}
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	spin_lock_init(&priv->mstp_lock);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	nclks = info->num_total_core_clks + info->num_hw_mod_clks;
+	clks = devm_kmalloc_array(dev, nclks, sizeof(*clks), GFP_KERNEL);
+	if (!clks)
+		return -ENOMEM;
+
+	priv->clks = clks;
+	priv->num_core_clks = info->num_total_core_clks;
+	priv->num_mod_clks = info->num_hw_mod_clks;
+	priv->last_dt_core_clk = info->last_dt_core_clk;
+
+	for (i = 0; i < nclks; i++)
+		clks[i] = ERR_PTR(-ENOENT);
+
+	for (i = 0; i < info->num_core_clks; i++)
+		cpg_mssr_register_core_clk(&info->core_clks[i], info, priv);
+
+	for (i = 0; i < info->num_mod_clks; i++)
+		cpg_mssr_register_mod_clk(&info->mod_clks[i], info, priv);
+
+	error = of_clk_add_provider(np, cpg_mssr_clk_src_twocell_get, priv);
+	if (error)
+		return error;
+
+	devm_add_action(dev, cpg_mssr_del_clk_provider, np);
+
+	error = cpg_mssr_add_clk_domain(dev, info->core_pm_clks,
+					info->num_core_pm_clks);
+	if (error)
+		return error;
+
+	return 0;
+}
+
+static struct platform_driver cpg_mssr_driver = {
+	.driver		= {
+		.name	= "renesas-cpg-mssr",
+		.of_match_table = cpg_mssr_match,
+	},
+};
+
+static int __init cpg_mssr_init(void)
+{
+	return platform_driver_probe(&cpg_mssr_driver, cpg_mssr_probe);
+}
+
+subsys_initcall(cpg_mssr_init);
+
+MODULE_DESCRIPTION("Renesas CPG/MSSR Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/shmobile/renesas-cpg-mssr.h b/drivers/clk/shmobile/renesas-cpg-mssr.h
new file mode 100644
index 000000000000..e09f03cbf086
--- /dev/null
+++ b/drivers/clk/shmobile/renesas-cpg-mssr.h
@@ -0,0 +1,132 @@
+/*
+ * Renesas Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2015 Glider bvba
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#ifndef __CLK_RENESAS_CPG_MSSR_H__
+#define __CLK_RENESAS_CPG_MSSR_H__
+
+    /*
+     * Definitions of CPG Core Clocks
+     *
+     * These include:
+     *   - Clock outputs exported to DT
+     *   - External input clocks
+     *   - Internal CPG clocks
+     */
+
+struct cpg_core_clk {
+	/* Common */
+	const char *name;
+	unsigned int id;
+	unsigned int type;
+	/* Depending on type */
+	unsigned int parent;	/* Core Clocks only */
+	unsigned int div;
+	unsigned int mult;
+	unsigned int offset;
+};
+
+enum clk_types {
+	/* Generic */
+	CLK_TYPE_IN,		/* External Clock Input */
+	CLK_TYPE_FF,		/* Fixed Factor Clock */
+	CLK_TYPE_DIV6P1,	/* DIV6 Clock with 1 parent clock */
+
+	/* Custom definitions start here */
+	CLK_TYPE_CUSTOM,
+};
+
+#define DEF_TYPE(_name, _id, _type...)	\
+	{ .name = _name, .id = _id, .type = _type }
+#define DEF_BASE(_name, _id, _type, _parent...)	\
+	DEF_TYPE(_name, _id, _type, .parent = _parent)
+
+#define DEF_INPUT(_name, _id) \
+	DEF_TYPE(_name, _id, CLK_TYPE_IN)
+#define DEF_FIXED(_name, _id, _parent, _div, _mult)	\
+	DEF_BASE(_name, _id, CLK_TYPE_FF, _parent, .div = _div, .mult = _mult)
+#define DEF_DIV6P1(_name, _id, _parent, _offset)	\
+	DEF_BASE(_name, _id, CLK_TYPE_DIV6P1, _parent, .offset = _offset)
+
+
+    /*
+     * Definitions of Module Clocks
+     */
+
+struct mssr_mod_clk {
+	const char *name;
+	unsigned int id;
+	unsigned int parent;	/* Add MOD_CLK_BASE for Module Clocks */
+};
+
+/* Convert from sparse base-100 to packed index space */
+#define MOD_CLK_PACK(x)	((x) - ((x) / 100) * (100 - 32))
+
+#define MOD_CLK_ID(x)	(MOD_CLK_BASE + MOD_CLK_PACK(x))
+
+#define DEF_MOD(_name, _mod, _parent...)	\
+	{ .name = _name, .id = MOD_CLK_ID(_mod), .parent = _parent }
+
+
+struct device_node;
+
+    /**
+     * SoC-specific CPG/MSSR Description
+     *
+     * @core_clks: Array of Core Clock definitions
+     * @num_core_clks: Number of entries in core_clks[]
+     * @last_dt_core_clk: ID of the last Core Clock exported to DT
+     * @num_total_core_clks: Total number of Core Clocks (exported + internal)
+     *
+     * @mod_clks: Array of Module Clock definitions
+     * @num_mod_clks: Number of entries in mod_clks[]
+     * @num_hw_mod_clks: Number of Module Clocks supported by the hardware
+     *
+     * @crit_mod_clks: Array with Module Clock IDs of critical clocks that
+     *                 should not be disabled without a knowledgeable driver
+     * @num_crit_mod_clks: Number of entries in crit_mod_clks[]
+     *
+     * @core_pm_clks: Array with IDs of Core Clocks that are suitable for Power
+     *                Management, in addition to Module Clocks
+     * @num_core_pm_clks: Number of entries in core_pm_clks[]
+     *
+     * @init: Optional callback to perform SoC-specific initialization
+     * @cpg_clk_register: Optional callback to handle special Core Clock types
+     */
+
+struct cpg_mssr_info {
+	/* Core Clocks */
+	const struct cpg_core_clk *core_clks;
+	unsigned int num_core_clks;
+	unsigned int last_dt_core_clk;
+	unsigned int num_total_core_clks;
+
+	/* Module Clocks */
+	const struct mssr_mod_clk *mod_clks;
+	unsigned int num_mod_clks;
+	unsigned int num_hw_mod_clks;
+
+	/* Critical Module Clocks that should not be disabled */
+	const unsigned int *crit_mod_clks;
+	unsigned int num_crit_mod_clks;
+
+	/* Core Clocks suitable for PM, in addition to the Module Clocks */
+	const unsigned int *core_pm_clks;
+	unsigned int num_core_pm_clks;
+
+	/* Callbacks */
+	int (*init)(struct device *dev);
+	struct clk *(*cpg_clk_register)(struct device *dev,
+					const struct cpg_core_clk *core,
+					const struct cpg_mssr_info *info,
+					struct clk **clks, void __iomem *base);
+};
+
+extern const struct cpg_mssr_info r8a7795_cpg_mssr_info;
+#endif
diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c
index 576cd0354d48..ccb324d97160 100644
--- a/drivers/clk/st/clkgen-fsyn.c
+++ b/drivers/clk/st/clkgen-fsyn.c
@@ -549,19 +549,20 @@ static int clk_fs660c32_vco_get_params(unsigned long input,
 	return 0;
 }
 
-static long quadfs_pll_fs660c32_round_rate(struct clk_hw *hw, unsigned long rate
-		, unsigned long *prate)
+static long quadfs_pll_fs660c32_round_rate(struct clk_hw *hw,
+					   unsigned long rate,
+					   unsigned long *prate)
 {
 	struct stm_fs params;
 
-	if (!clk_fs660c32_vco_get_params(*prate, rate, &params))
-		clk_fs660c32_vco_get_rate(*prate, &params, &rate);
+	if (clk_fs660c32_vco_get_params(*prate, rate, &params))
+		return rate;
 
-	pr_debug("%s: %s new rate %ld [sdiv=0x%x,md=0x%x,pe=0x%x,nsdiv3=%u]\n",
+	clk_fs660c32_vco_get_rate(*prate, &params, &rate);
+
+	pr_debug("%s: %s new rate %ld [ndiv=%u]\n",
 		 __func__, clk_hw_get_name(hw),
-		 rate, (unsigned int)params.sdiv,
-		 (unsigned int)params.mdiv,
-		 (unsigned int)params.pe, (unsigned int)params.nsdiv);
+		 rate, (unsigned int)params.ndiv);
 
 	return rate;
 }
diff --git a/drivers/clk/sunxi/Makefile b/drivers/clk/sunxi/Makefile
index cb4c299214ce..3fd7901d48e4 100644
--- a/drivers/clk/sunxi/Makefile
+++ b/drivers/clk/sunxi/Makefile
@@ -7,14 +7,19 @@ obj-y += clk-a10-codec.o
 obj-y += clk-a10-hosc.o
 obj-y += clk-a10-mod1.o
 obj-y += clk-a10-pll2.o
+obj-y += clk-a10-ve.o
 obj-y += clk-a20-gmac.o
 obj-y += clk-mod0.o
 obj-y += clk-simple-gates.o
+obj-y += clk-sun8i-bus-gates.o
 obj-y += clk-sun8i-mbus.o
 obj-y += clk-sun9i-core.o
 obj-y += clk-sun9i-mmc.o
 obj-y += clk-usb.o
 
+obj-$(CONFIG_MACH_SUN9I) += clk-sun8i-apb0.o
+obj-$(CONFIG_MACH_SUN9I) += clk-sun9i-cpus.o
+
 obj-$(CONFIG_MFD_SUN6I_PRCM) += \
 	clk-sun6i-ar100.o clk-sun6i-apb0.o clk-sun6i-apb0-gates.o \
 	clk-sun8i-apb0.o
diff --git a/drivers/clk/sunxi/clk-a10-ve.c b/drivers/clk/sunxi/clk-a10-ve.c
new file mode 100644
index 000000000000..044c1717b762
--- /dev/null
+++ b/drivers/clk/sunxi/clk-a10-ve.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2015 Chen-Yu Tsai
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+static DEFINE_SPINLOCK(ve_lock);
+
+#define SUN4I_VE_ENABLE		31
+#define SUN4I_VE_DIVIDER_SHIFT	16
+#define SUN4I_VE_DIVIDER_WIDTH	3
+#define SUN4I_VE_RESET		0
+
+/**
+ * sunxi_ve_reset... - reset bit in ve clk registers handling
+ */
+
+struct ve_reset_data {
+	void __iomem			*reg;
+	spinlock_t			*lock;
+	struct reset_controller_dev	rcdev;
+};
+
+static int sunxi_ve_reset_assert(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	struct ve_reset_data *data = container_of(rcdev,
+						  struct ve_reset_data,
+						  rcdev);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(data->lock, flags);
+
+	reg = readl(data->reg);
+	writel(reg & ~BIT(SUN4I_VE_RESET), data->reg);
+
+	spin_unlock_irqrestore(data->lock, flags);
+
+	return 0;
+}
+
+static int sunxi_ve_reset_deassert(struct reset_controller_dev *rcdev,
+				   unsigned long id)
+{
+	struct ve_reset_data *data = container_of(rcdev,
+						  struct ve_reset_data,
+						  rcdev);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(data->lock, flags);
+
+	reg = readl(data->reg);
+	writel(reg | BIT(SUN4I_VE_RESET), data->reg);
+
+	spin_unlock_irqrestore(data->lock, flags);
+
+	return 0;
+}
+
+static int sunxi_ve_of_xlate(struct reset_controller_dev *rcdev,
+			     const struct of_phandle_args *reset_spec)
+{
+	if (WARN_ON(reset_spec->args_count != 0))
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct reset_control_ops sunxi_ve_reset_ops = {
+	.assert		= sunxi_ve_reset_assert,
+	.deassert	= sunxi_ve_reset_deassert,
+};
+
+static void __init sun4i_ve_clk_setup(struct device_node *node)
+{
+	struct clk *clk;
+	struct clk_divider *div;
+	struct clk_gate *gate;
+	struct ve_reset_data *reset_data;
+	const char *parent;
+	const char *clk_name = node->name;
+	void __iomem *reg;
+	int err;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg))
+		return;
+
+	div = kzalloc(sizeof(*div), GFP_KERNEL);
+	if (!div)
+		goto err_unmap;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate)
+		goto err_free_div;
+
+	of_property_read_string(node, "clock-output-names", &clk_name);
+	parent = of_clk_get_parent_name(node, 0);
+
+	gate->reg = reg;
+	gate->bit_idx = SUN4I_VE_ENABLE;
+	gate->lock = &ve_lock;
+
+	div->reg = reg;
+	div->shift = SUN4I_VE_DIVIDER_SHIFT;
+	div->width = SUN4I_VE_DIVIDER_WIDTH;
+	div->lock = &ve_lock;
+
+	clk = clk_register_composite(NULL, clk_name, &parent, 1,
+				     NULL, NULL,
+				     &div->hw, &clk_divider_ops,
+				     &gate->hw, &clk_gate_ops,
+				     CLK_SET_RATE_PARENT);
+	if (IS_ERR(clk))
+		goto err_free_gate;
+
+	err = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (err)
+		goto err_unregister_clk;
+
+	reset_data = kzalloc(sizeof(*reset_data), GFP_KERNEL);
+	if (!reset_data)
+		goto err_del_provider;
+
+	reset_data->reg = reg;
+	reset_data->lock = &ve_lock;
+	reset_data->rcdev.nr_resets = 1;
+	reset_data->rcdev.ops = &sunxi_ve_reset_ops;
+	reset_data->rcdev.of_node = node;
+	reset_data->rcdev.of_xlate = sunxi_ve_of_xlate;
+	reset_data->rcdev.of_reset_n_cells = 0;
+	err = reset_controller_register(&reset_data->rcdev);
+	if (err)
+		goto err_free_reset;
+
+	return;
+
+err_free_reset:
+	kfree(reset_data);
+err_del_provider:
+	of_clk_del_provider(node);
+err_unregister_clk:
+	clk_unregister(clk);
+err_free_gate:
+	kfree(gate);
+err_free_div:
+	kfree(div);
+err_unmap:
+	iounmap(reg);
+}
+CLK_OF_DECLARE(sun4i_ve, "allwinner,sun4i-a10-ve-clk",
+	       sun4i_ve_clk_setup);
diff --git a/drivers/clk/sunxi/clk-simple-gates.c b/drivers/clk/sunxi/clk-simple-gates.c
index 0214c6548afd..f4da52b5ca0e 100644
--- a/drivers/clk/sunxi/clk-simple-gates.c
+++ b/drivers/clk/sunxi/clk-simple-gates.c
@@ -140,6 +140,8 @@ CLK_OF_DECLARE(sun9i_a80_apb0, "allwinner,sun9i-a80-apb0-gates-clk",
 	       sunxi_simple_gates_init);
 CLK_OF_DECLARE(sun9i_a80_apb1, "allwinner,sun9i-a80-apb1-gates-clk",
 	       sunxi_simple_gates_init);
+CLK_OF_DECLARE(sun9i_a80_apbs, "allwinner,sun9i-a80-apbs-gates-clk",
+	       sunxi_simple_gates_init);
 
 static const int sun4i_a10_ahb_critical_clocks[] __initconst = {
 	14,	/* ahb_sdram */
@@ -158,3 +160,15 @@ CLK_OF_DECLARE(sun5i_a13_ahb, "allwinner,sun5i-a13-ahb-gates-clk",
 	       sun4i_a10_ahb_init);
 CLK_OF_DECLARE(sun7i_a20_ahb, "allwinner,sun7i-a20-ahb-gates-clk",
 	       sun4i_a10_ahb_init);
+
+static const int sun4i_a10_dram_critical_clocks[] __initconst = {
+	15,	/* dram_output */
+};
+
+static void __init sun4i_a10_dram_init(struct device_node *node)
+{
+	sunxi_simple_gates_setup(node, sun4i_a10_dram_critical_clocks,
+				 ARRAY_SIZE(sun4i_a10_dram_critical_clocks));
+}
+CLK_OF_DECLARE(sun4i_a10_dram, "allwinner,sun4i-a10-dram-gates-clk",
+	       sun4i_a10_dram_init);
diff --git a/drivers/clk/sunxi/clk-sun8i-apb0.c b/drivers/clk/sunxi/clk-sun8i-apb0.c
index 7ae5d2c2cde1..7ba61103a6f5 100644
--- a/drivers/clk/sunxi/clk-sun8i-apb0.c
+++ b/drivers/clk/sunxi/clk-sun8i-apb0.c
@@ -17,13 +17,77 @@
 #include <linux/clk-provider.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/platform_device.h>
 
+static struct clk *sun8i_a23_apb0_register(struct device_node *node,
+					   void __iomem *reg)
+{
+	const char *clk_name = node->name;
+	const char *clk_parent;
+	struct clk *clk;
+	int ret;
+
+	clk_parent = of_clk_get_parent_name(node, 0);
+	if (!clk_parent)
+		return ERR_PTR(-EINVAL);
+
+	of_property_read_string(node, "clock-output-names", &clk_name);
+
+	/* The A23 APB0 clock is a standard 2 bit wide divider clock */
+	clk = clk_register_divider(NULL, clk_name, clk_parent, 0, reg,
+				   0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
+	if (IS_ERR(clk))
+		return clk;
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret)
+		goto err_unregister;
+
+	return clk;
+
+err_unregister:
+	clk_unregister_divider(clk);
+
+	return ERR_PTR(ret);
+}
+
+static void sun8i_a23_apb0_setup(struct device_node *node)
+{
+	void __iomem *reg;
+	struct resource res;
+	struct clk *clk;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		/*
+		 * This happens with clk nodes instantiated through mfd,
+		 * as those do not have their resources assigned in the
+		 * device tree. Do not print an error in this case.
+		 */
+		if (PTR_ERR(reg) != -EINVAL)
+			pr_err("Could not get registers for a23-apb0-clk\n");
+
+		return;
+	}
+
+	clk = sun8i_a23_apb0_register(node, reg);
+	if (IS_ERR(clk))
+		goto err_unmap;
+
+	return;
+
+err_unmap:
+	iounmap(reg);
+	of_address_to_resource(node, 0, &res);
+	release_mem_region(res.start, resource_size(&res));
+}
+CLK_OF_DECLARE(sun8i_a23_apb0, "allwinner,sun8i-a23-apb0-clk",
+	       sun8i_a23_apb0_setup);
+
 static int sun8i_a23_apb0_clk_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	const char *clk_name = np->name;
-	const char *clk_parent;
 	struct resource *r;
 	void __iomem *reg;
 	struct clk *clk;
@@ -33,19 +97,11 @@ static int sun8i_a23_apb0_clk_probe(struct platform_device *pdev)
 	if (IS_ERR(reg))
 		return PTR_ERR(reg);
 
-	clk_parent = of_clk_get_parent_name(np, 0);
-	if (!clk_parent)
-		return -EINVAL;
-
-	of_property_read_string(np, "clock-output-names", &clk_name);
-
-	/* The A23 APB0 clock is a standard 2 bit wide divider clock */
-	clk = clk_register_divider(&pdev->dev, clk_name, clk_parent, 0, reg,
-				   0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
+	clk = sun8i_a23_apb0_register(np, reg);
 	if (IS_ERR(clk))
 		return PTR_ERR(clk);
 
-	return of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	return 0;
 }
 
 static const struct of_device_id sun8i_a23_apb0_clk_dt_ids[] = {
diff --git a/drivers/clk/sunxi/clk-sun8i-bus-gates.c b/drivers/clk/sunxi/clk-sun8i-bus-gates.c
new file mode 100644
index 000000000000..e32d18ba252b
--- /dev/null
+++ b/drivers/clk/sunxi/clk-sun8i-bus-gates.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2015 Jens Kuske <jenskuske@gmail.com>
+ *
+ * Based on clk-simple-gates.c, which is:
+ * Copyright 2015 Maxime Ripard
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+static DEFINE_SPINLOCK(gates_lock);
+
+static void __init sun8i_h3_bus_gates_init(struct device_node *node)
+{
+	static const char * const names[] = { "ahb1", "ahb2", "apb1", "apb2" };
+	enum { AHB1, AHB2, APB1, APB2, PARENT_MAX } clk_parent;
+	const char *parents[PARENT_MAX];
+	struct clk_onecell_data *clk_data;
+	const char *clk_name;
+	struct property *prop;
+	struct resource res;
+	void __iomem *clk_reg;
+	void __iomem *reg;
+	const __be32 *p;
+	int number, i;
+	u8 clk_bit;
+	int index;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(names); i++) {
+		int idx = of_property_match_string(node, "clock-names",
+						   names[i]);
+		if (idx < 0)
+			return;
+
+		parents[i] = of_clk_get_parent_name(node, idx);
+	}
+
+	clk_data = kmalloc(sizeof(struct clk_onecell_data), GFP_KERNEL);
+	if (!clk_data)
+		goto err_unmap;
+
+	number = of_property_count_u32_elems(node, "clock-indices");
+	of_property_read_u32_index(node, "clock-indices", number - 1, &number);
+
+	clk_data->clks = kcalloc(number + 1, sizeof(struct clk *), GFP_KERNEL);
+	if (!clk_data->clks)
+		goto err_free_data;
+
+	i = 0;
+	of_property_for_each_u32(node, "clock-indices", prop, p, index) {
+		of_property_read_string_index(node, "clock-output-names",
+					      i, &clk_name);
+
+		if (index == 17 || (index >= 29 && index <= 31))
+			clk_parent = AHB2;
+		else if (index <= 63 || index >= 128)
+			clk_parent = AHB1;
+		else if (index >= 64 && index <= 95)
+			clk_parent = APB1;
+		else if (index >= 96 && index <= 127)
+			clk_parent = APB2;
+
+		clk_reg = reg + 4 * (index / 32);
+		clk_bit = index % 32;
+
+		clk_data->clks[index] = clk_register_gate(NULL, clk_name,
+							  parents[clk_parent],
+							  0, clk_reg, clk_bit,
+							  0, &gates_lock);
+		i++;
+
+		if (IS_ERR(clk_data->clks[index])) {
+			WARN_ON(true);
+			continue;
+		}
+	}
+
+	clk_data->clk_num = number + 1;
+	of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	return;
+
+err_free_data:
+	kfree(clk_data);
+err_unmap:
+	iounmap(reg);
+	of_address_to_resource(node, 0, &res);
+	release_mem_region(res.start, resource_size(&res));
+}
+
+CLK_OF_DECLARE(sun8i_h3_bus_gates, "allwinner,sun8i-h3-bus-gates-clk",
+	       sun8i_h3_bus_gates_init);
diff --git a/drivers/clk/sunxi/clk-sun9i-cpus.c b/drivers/clk/sunxi/clk-sun9i-cpus.c
new file mode 100644
index 000000000000..7626d2194b96
--- /dev/null
+++ b/drivers/clk/sunxi/clk-sun9i-cpus.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright (C) 2015 Chen-Yu Tsai
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ *
+ * Allwinner A80 CPUS clock driver
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+
+static DEFINE_SPINLOCK(sun9i_a80_cpus_lock);
+
+/**
+ * sun9i_a80_cpus_clk_setup() - Setup function for a80 cpus composite clk
+ */
+
+#define SUN9I_CPUS_MAX_PARENTS		4
+#define SUN9I_CPUS_MUX_PARENT_PLL4	3
+#define SUN9I_CPUS_MUX_SHIFT		16
+#define SUN9I_CPUS_MUX_MASK		GENMASK(17, 16)
+#define SUN9I_CPUS_MUX_GET_PARENT(reg)	((reg & SUN9I_CPUS_MUX_MASK) >> \
+						SUN9I_CPUS_MUX_SHIFT)
+
+#define SUN9I_CPUS_DIV_SHIFT		4
+#define SUN9I_CPUS_DIV_MASK		GENMASK(5, 4)
+#define SUN9I_CPUS_DIV_GET(reg)		((reg & SUN9I_CPUS_DIV_MASK) >> \
+						SUN9I_CPUS_DIV_SHIFT)
+#define SUN9I_CPUS_DIV_SET(reg, div)	((reg & ~SUN9I_CPUS_DIV_MASK) | \
+						(div << SUN9I_CPUS_DIV_SHIFT))
+#define SUN9I_CPUS_PLL4_DIV_SHIFT	8
+#define SUN9I_CPUS_PLL4_DIV_MASK	GENMASK(12, 8)
+#define SUN9I_CPUS_PLL4_DIV_GET(reg)	((reg & SUN9I_CPUS_PLL4_DIV_MASK) >> \
+						SUN9I_CPUS_PLL4_DIV_SHIFT)
+#define SUN9I_CPUS_PLL4_DIV_SET(reg, div) ((reg & ~SUN9I_CPUS_PLL4_DIV_MASK) | \
+						(div << SUN9I_CPUS_PLL4_DIV_SHIFT))
+
+struct sun9i_a80_cpus_clk {
+	struct clk_hw hw;
+	void __iomem *reg;
+};
+
+#define to_sun9i_a80_cpus_clk(_hw) container_of(_hw, struct sun9i_a80_cpus_clk, hw)
+
+static unsigned long sun9i_a80_cpus_clk_recalc_rate(struct clk_hw *hw,
+						    unsigned long parent_rate)
+{
+	struct sun9i_a80_cpus_clk *cpus = to_sun9i_a80_cpus_clk(hw);
+	unsigned long rate;
+	u32 reg;
+
+	/* Fetch the register value */
+	reg = readl(cpus->reg);
+
+	/* apply pre-divider first if parent is pll4 */
+	if (SUN9I_CPUS_MUX_GET_PARENT(reg) == SUN9I_CPUS_MUX_PARENT_PLL4)
+		parent_rate /= SUN9I_CPUS_PLL4_DIV_GET(reg) + 1;
+
+	/* clk divider */
+	rate = parent_rate / (SUN9I_CPUS_DIV_GET(reg) + 1);
+
+	return rate;
+}
+
+static long sun9i_a80_cpus_clk_round(unsigned long rate, u8 *divp, u8 *pre_divp,
+				     u8 parent, unsigned long parent_rate)
+{
+	u8 div, pre_div = 1;
+
+	/*
+	 * clock can only divide, so we will never be able to achieve
+	 * frequencies higher than the parent frequency
+	 */
+	if (parent_rate && rate > parent_rate)
+		rate = parent_rate;
+
+	div = DIV_ROUND_UP(parent_rate, rate);
+
+	/* calculate pre-divider if parent is pll4 */
+	if (parent == SUN9I_CPUS_MUX_PARENT_PLL4 && div > 4) {
+		/* pre-divider is 1 ~ 32 */
+		if (div < 32) {
+			pre_div = div;
+			div = 1;
+		} else if (div < 64) {
+			pre_div = DIV_ROUND_UP(div, 2);
+			div = 2;
+		} else if (div < 96) {
+			pre_div = DIV_ROUND_UP(div, 3);
+			div = 3;
+		} else {
+			pre_div = DIV_ROUND_UP(div, 4);
+			div = 4;
+		}
+	}
+
+	/* we were asked to pass back divider values */
+	if (divp) {
+		*divp = div - 1;
+		*pre_divp = pre_div - 1;
+	}
+
+	return parent_rate / pre_div / div;
+}
+
+static int sun9i_a80_cpus_clk_determine_rate(struct clk_hw *clk,
+					     struct clk_rate_request *req)
+{
+	struct clk_hw *parent, *best_parent = NULL;
+	int i, num_parents;
+	unsigned long parent_rate, best = 0, child_rate, best_child_rate = 0;
+	unsigned long rate = req->rate;
+
+	/* find the parent that can help provide the fastest rate <= rate */
+	num_parents = clk_hw_get_num_parents(clk);
+	for (i = 0; i < num_parents; i++) {
+		parent = clk_hw_get_parent_by_index(clk, i);
+		if (!parent)
+			continue;
+		if (clk_hw_get_flags(clk) & CLK_SET_RATE_PARENT)
+			parent_rate = clk_hw_round_rate(parent, rate);
+		else
+			parent_rate = clk_hw_get_rate(parent);
+
+		child_rate = sun9i_a80_cpus_clk_round(rate, NULL, NULL, i,
+						      parent_rate);
+
+		if (child_rate <= rate && child_rate > best_child_rate) {
+			best_parent = parent;
+			best = parent_rate;
+			best_child_rate = child_rate;
+		}
+	}
+
+	if (!best_parent)
+		return -EINVAL;
+
+	req->best_parent_hw = best_parent;
+	req->best_parent_rate = best;
+	req->rate = best_child_rate;
+
+	return 0;
+}
+
+static int sun9i_a80_cpus_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				       unsigned long parent_rate)
+{
+	struct sun9i_a80_cpus_clk *cpus = to_sun9i_a80_cpus_clk(hw);
+	unsigned long flags;
+	u8 div, pre_div, parent;
+	u32 reg;
+
+	spin_lock_irqsave(&sun9i_a80_cpus_lock, flags);
+
+	reg = readl(cpus->reg);
+
+	/* need to know which parent is used to apply pre-divider */
+	parent = SUN9I_CPUS_MUX_GET_PARENT(reg);
+	sun9i_a80_cpus_clk_round(rate, &div, &pre_div, parent, parent_rate);
+
+	reg = SUN9I_CPUS_DIV_SET(reg, div);
+	reg = SUN9I_CPUS_PLL4_DIV_SET(reg, pre_div);
+	writel(reg, cpus->reg);
+
+	spin_unlock_irqrestore(&sun9i_a80_cpus_lock, flags);
+
+	return 0;
+}
+
+static const struct clk_ops sun9i_a80_cpus_clk_ops = {
+	.determine_rate	= sun9i_a80_cpus_clk_determine_rate,
+	.recalc_rate	= sun9i_a80_cpus_clk_recalc_rate,
+	.set_rate	= sun9i_a80_cpus_clk_set_rate,
+};
+
+static void sun9i_a80_cpus_setup(struct device_node *node)
+{
+	const char *clk_name = node->name;
+	const char *parents[SUN9I_CPUS_MAX_PARENTS];
+	struct resource res;
+	struct sun9i_a80_cpus_clk *cpus;
+	struct clk_mux *mux;
+	struct clk *clk;
+	int ret;
+
+	cpus = kzalloc(sizeof(*cpus), GFP_KERNEL);
+	if (!cpus)
+		return;
+
+	cpus->reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(cpus->reg))
+		goto err_free_cpus;
+
+	of_property_read_string(node, "clock-output-names", &clk_name);
+
+	/* we have a mux, we will have >1 parents */
+	ret = of_clk_parent_fill(node, parents, SUN9I_CPUS_MAX_PARENTS);
+
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		goto err_unmap;
+
+	/* set up clock properties */
+	mux->reg = cpus->reg;
+	mux->shift = SUN9I_CPUS_MUX_SHIFT;
+	/* un-shifted mask is what mux_clk expects */
+	mux->mask = SUN9I_CPUS_MUX_MASK >> SUN9I_CPUS_MUX_SHIFT;
+	mux->lock = &sun9i_a80_cpus_lock;
+
+	clk = clk_register_composite(NULL, clk_name, parents, ret,
+				     &mux->hw, &clk_mux_ops,
+				     &cpus->hw, &sun9i_a80_cpus_clk_ops,
+				     NULL, NULL, 0);
+	if (IS_ERR(clk))
+		goto err_free_mux;
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret)
+		goto err_unregister;
+
+	return;
+
+err_unregister:
+	clk_unregister(clk);
+err_free_mux:
+	kfree(mux);
+err_unmap:
+	iounmap(cpus->reg);
+	of_address_to_resource(node, 0, &res);
+	release_mem_region(res.start, resource_size(&res));
+err_free_cpus:
+	kfree(cpus);
+}
+CLK_OF_DECLARE(sun9i_a80_cpus, "allwinner,sun9i-a80-cpus-clk",
+	       sun9i_a80_cpus_setup);
diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c
index 9c79af0c03b2..5ba2188ee99c 100644
--- a/drivers/clk/sunxi/clk-sunxi.c
+++ b/drivers/clk/sunxi/clk-sunxi.c
@@ -778,6 +778,10 @@ static const struct mux_data sun6i_a31_ahb1_mux_data __initconst = {
 	.shift = 12,
 };
 
+static const struct mux_data sun8i_h3_ahb2_mux_data __initconst = {
+	.shift = 0,
+};
+
 static void __init sunxi_mux_clk_setup(struct device_node *node,
 				       struct mux_data *data)
 {
@@ -1130,6 +1134,7 @@ static const struct of_device_id clk_divs_match[] __initconst = {
 static const struct of_device_id clk_mux_match[] __initconst = {
 	{.compatible = "allwinner,sun4i-a10-cpu-clk", .data = &sun4i_cpu_mux_data,},
 	{.compatible = "allwinner,sun6i-a31-ahb1-mux-clk", .data = &sun6i_a31_ahb1_mux_data,},
+	{.compatible = "allwinner,sun8i-h3-ahb2-clk", .data = &sun8i_h3_ahb2_mux_data,},
 	{}
 };
 
@@ -1212,6 +1217,7 @@ CLK_OF_DECLARE(sun6i_a31_clk_init, "allwinner,sun6i-a31", sun6i_init_clocks);
 CLK_OF_DECLARE(sun6i_a31s_clk_init, "allwinner,sun6i-a31s", sun6i_init_clocks);
 CLK_OF_DECLARE(sun8i_a23_clk_init, "allwinner,sun8i-a23", sun6i_init_clocks);
 CLK_OF_DECLARE(sun8i_a33_clk_init, "allwinner,sun8i-a33", sun6i_init_clocks);
+CLK_OF_DECLARE(sun8i_h3_clk_init, "allwinner,sun8i-h3", sun6i_init_clocks);
 
 static void __init sun9i_init_clocks(struct device_node *node)
 {
diff --git a/drivers/clk/sunxi/clk-usb.c b/drivers/clk/sunxi/clk-usb.c
index 1a72cd672839..67b8e38f4ee9 100644
--- a/drivers/clk/sunxi/clk-usb.c
+++ b/drivers/clk/sunxi/clk-usb.c
@@ -243,3 +243,15 @@ static void __init sun9i_a80_usb_phy_setup(struct device_node *node)
 	sunxi_usb_clk_setup(node, &sun9i_a80_usb_phy_data, &a80_usb_phy_lock);
 }
 CLK_OF_DECLARE(sun9i_a80_usb_phy, "allwinner,sun9i-a80-usb-phy-clk", sun9i_a80_usb_phy_setup);
+
+static const struct usb_clk_data sun8i_h3_usb_clk_data __initconst = {
+	.clk_mask =  BIT(19) | BIT(18) | BIT(17) | BIT(16) |
+		     BIT(11) | BIT(10) | BIT(9) | BIT(8),
+	.reset_mask = BIT(3) | BIT(2) | BIT(1) | BIT(0),
+};
+
+static void __init sun8i_h3_usb_setup(struct device_node *node)
+{
+	sunxi_usb_clk_setup(node, &sun8i_h3_usb_clk_data, &sun4i_a10_usb_lock);
+}
+CLK_OF_DECLARE(sun8i_h3_usb, "allwinner,sun8i-h3-usb-clk", sun8i_h3_usb_setup);
diff --git a/drivers/clk/tegra/Makefile b/drivers/clk/tegra/Makefile
index 826c325dc2e8..97984c503bbb 100644
--- a/drivers/clk/tegra/Makefile
+++ b/drivers/clk/tegra/Makefile
@@ -20,3 +20,4 @@ obj-$(CONFIG_ARCH_TEGRA_124_SOC)	+= clk-tegra124.o
 obj-$(CONFIG_ARCH_TEGRA_124_SOC)	+= clk-tegra124-dfll-fcpu.o
 obj-$(CONFIG_ARCH_TEGRA_132_SOC)	+= clk-tegra124.o
 obj-y					+= cvb.o
+obj-$(CONFIG_ARCH_TEGRA_210_SOC)	+= clk-tegra210.o
diff --git a/drivers/clk/tegra/clk-divider.c b/drivers/clk/tegra/clk-divider.c
index 48c83efda4cf..16e0aee14773 100644
--- a/drivers/clk/tegra/clk-divider.c
+++ b/drivers/clk/tegra/clk-divider.c
@@ -32,7 +32,7 @@
 static int get_div(struct tegra_clk_frac_div *divider, unsigned long rate,
 		   unsigned long parent_rate)
 {
-	s64 divider_ux1 = parent_rate;
+	u64 divider_ux1 = parent_rate;
 	u8 flags = divider->flags;
 	int mul;
 
@@ -54,7 +54,7 @@ static int get_div(struct tegra_clk_frac_div *divider, unsigned long rate,
 
 	divider_ux1 -= mul;
 
-	if (divider_ux1 < 0)
+	if ((s64)divider_ux1 < 0)
 		return 0;
 
 	if (divider_ux1 > get_max_div(divider))
diff --git a/drivers/clk/tegra/clk-id.h b/drivers/clk/tegra/clk-id.h
index 60738cc954cb..19ce0738ee76 100644
--- a/drivers/clk/tegra/clk-id.h
+++ b/drivers/clk/tegra/clk-id.h
@@ -13,6 +13,7 @@ enum clk_id {
 	tegra_clk_amx1,
 	tegra_clk_apbdma,
 	tegra_clk_apbif,
+	tegra_clk_ape,
 	tegra_clk_audio0,
 	tegra_clk_audio0_2x,
 	tegra_clk_audio0_mux,
@@ -38,6 +39,7 @@ enum clk_id {
 	tegra_clk_cile,
 	tegra_clk_clk_32k,
 	tegra_clk_clk72Mhz,
+	tegra_clk_clk72Mhz_8,
 	tegra_clk_clk_m,
 	tegra_clk_clk_m_div2,
 	tegra_clk_clk_m_div4,
@@ -51,17 +53,21 @@ enum clk_id {
 	tegra_clk_cml1,
 	tegra_clk_csi,
 	tegra_clk_csite,
+	tegra_clk_csite_8,
 	tegra_clk_csus,
 	tegra_clk_cve,
 	tegra_clk_dam0,
 	tegra_clk_dam1,
 	tegra_clk_dam2,
 	tegra_clk_d_audio,
+	tegra_clk_dbgapb,
 	tegra_clk_dds,
 	tegra_clk_dfll_ref,
 	tegra_clk_dfll_soc,
 	tegra_clk_disp1,
+	tegra_clk_disp1_8,
 	tegra_clk_disp2,
+	tegra_clk_disp2_8,
 	tegra_clk_dp2,
 	tegra_clk_dpaux,
 	tegra_clk_dsialp,
@@ -71,6 +77,7 @@ enum clk_id {
 	tegra_clk_dtv,
 	tegra_clk_emc,
 	tegra_clk_entropy,
+	tegra_clk_entropy_8,
 	tegra_clk_epp,
 	tegra_clk_epp_8,
 	tegra_clk_extern1,
@@ -85,12 +92,16 @@ enum clk_id {
 	tegra_clk_gr3d_8,
 	tegra_clk_hclk,
 	tegra_clk_hda,
+	tegra_clk_hda_8,
 	tegra_clk_hda2codec_2x,
+	tegra_clk_hda2codec_2x_8,
 	tegra_clk_hda2hdmi,
 	tegra_clk_hdmi,
 	tegra_clk_hdmi_audio,
 	tegra_clk_host1x,
 	tegra_clk_host1x_8,
+	tegra_clk_host1x_9,
+	tegra_clk_hsic_trk,
 	tegra_clk_i2c1,
 	tegra_clk_i2c2,
 	tegra_clk_i2c3,
@@ -110,11 +121,14 @@ enum clk_id {
 	tegra_clk_i2s4_sync,
 	tegra_clk_isp,
 	tegra_clk_isp_8,
+	tegra_clk_isp_9,
 	tegra_clk_ispb,
 	tegra_clk_kbc,
 	tegra_clk_kfuse,
 	tegra_clk_la,
+	tegra_clk_maud,
 	tegra_clk_mipi,
+	tegra_clk_mipibif,
 	tegra_clk_mipi_cal,
 	tegra_clk_mpe,
 	tegra_clk_mselect,
@@ -124,15 +138,24 @@ enum clk_id {
 	tegra_clk_ndspeed,
 	tegra_clk_ndspeed_8,
 	tegra_clk_nor,
+	tegra_clk_nvdec,
+	tegra_clk_nvenc,
+	tegra_clk_nvjpg,
 	tegra_clk_owr,
+	tegra_clk_owr_8,
 	tegra_clk_pcie,
 	tegra_clk_pclk,
 	tegra_clk_pll_a,
 	tegra_clk_pll_a_out0,
+	tegra_clk_pll_a1,
 	tegra_clk_pll_c,
 	tegra_clk_pll_c2,
 	tegra_clk_pll_c3,
 	tegra_clk_pll_c4,
+	tegra_clk_pll_c4_out0,
+	tegra_clk_pll_c4_out1,
+	tegra_clk_pll_c4_out2,
+	tegra_clk_pll_c4_out3,
 	tegra_clk_pll_c_out1,
 	tegra_clk_pll_d,
 	tegra_clk_pll_d2,
@@ -140,19 +163,29 @@ enum clk_id {
 	tegra_clk_pll_d_out0,
 	tegra_clk_pll_dp,
 	tegra_clk_pll_e_out0,
+	tegra_clk_pll_g_ref,
 	tegra_clk_pll_m,
 	tegra_clk_pll_m_out1,
+	tegra_clk_pll_mb,
 	tegra_clk_pll_p,
 	tegra_clk_pll_p_out1,
 	tegra_clk_pll_p_out2,
 	tegra_clk_pll_p_out2_int,
 	tegra_clk_pll_p_out3,
 	tegra_clk_pll_p_out4,
+	tegra_clk_pll_p_out4_cpu,
 	tegra_clk_pll_p_out5,
+	tegra_clk_pll_p_out_hsio,
+	tegra_clk_pll_p_out_xusb,
+	tegra_clk_pll_p_out_cpu,
+	tegra_clk_pll_p_out_adsp,
 	tegra_clk_pll_ref,
 	tegra_clk_pll_re_out,
 	tegra_clk_pll_re_vco,
 	tegra_clk_pll_u,
+	tegra_clk_pll_u_out,
+	tegra_clk_pll_u_out1,
+	tegra_clk_pll_u_out2,
 	tegra_clk_pll_u_12m,
 	tegra_clk_pll_u_480m,
 	tegra_clk_pll_u_48m,
@@ -160,53 +193,80 @@ enum clk_id {
 	tegra_clk_pll_x,
 	tegra_clk_pll_x_out0,
 	tegra_clk_pwm,
+	tegra_clk_qspi,
 	tegra_clk_rtc,
 	tegra_clk_sata,
+	tegra_clk_sata_8,
 	tegra_clk_sata_cold,
 	tegra_clk_sata_oob,
+	tegra_clk_sata_oob_8,
 	tegra_clk_sbc1,
 	tegra_clk_sbc1_8,
+	tegra_clk_sbc1_9,
 	tegra_clk_sbc2,
 	tegra_clk_sbc2_8,
+	tegra_clk_sbc2_9,
 	tegra_clk_sbc3,
 	tegra_clk_sbc3_8,
+	tegra_clk_sbc3_9,
 	tegra_clk_sbc4,
 	tegra_clk_sbc4_8,
+	tegra_clk_sbc4_9,
 	tegra_clk_sbc5,
 	tegra_clk_sbc5_8,
 	tegra_clk_sbc6,
 	tegra_clk_sbc6_8,
 	tegra_clk_sclk,
+	tegra_clk_sdmmc_legacy,
 	tegra_clk_sdmmc1,
 	tegra_clk_sdmmc1_8,
+	tegra_clk_sdmmc1_9,
 	tegra_clk_sdmmc2,
 	tegra_clk_sdmmc2_8,
+	tegra_clk_sdmmc2_9,
 	tegra_clk_sdmmc3,
 	tegra_clk_sdmmc3_8,
+	tegra_clk_sdmmc3_9,
 	tegra_clk_sdmmc4,
 	tegra_clk_sdmmc4_8,
+	tegra_clk_sdmmc4_9,
 	tegra_clk_se,
 	tegra_clk_soc_therm,
+	tegra_clk_soc_therm_8,
 	tegra_clk_sor0,
 	tegra_clk_sor0_lvds,
+	tegra_clk_sor1,
+	tegra_clk_sor1_brick,
+	tegra_clk_sor1_src,
 	tegra_clk_spdif,
 	tegra_clk_spdif_2x,
 	tegra_clk_spdif_in,
+	tegra_clk_spdif_in_8,
 	tegra_clk_spdif_in_sync,
 	tegra_clk_spdif_mux,
 	tegra_clk_spdif_out,
 	tegra_clk_timer,
 	tegra_clk_trace,
 	tegra_clk_tsec,
+	tegra_clk_tsec_8,
+	tegra_clk_tsecb,
 	tegra_clk_tsensor,
 	tegra_clk_tvdac,
 	tegra_clk_tvo,
 	tegra_clk_uarta,
+	tegra_clk_uarta_8,
 	tegra_clk_uartb,
+	tegra_clk_uartb_8,
 	tegra_clk_uartc,
+	tegra_clk_uartc_8,
 	tegra_clk_uartd,
+	tegra_clk_uartd_8,
 	tegra_clk_uarte,
+	tegra_clk_uarte_8,
+	tegra_clk_uartape,
 	tegra_clk_usb2,
+	tegra_clk_usb2_hsic_trk,
+	tegra_clk_usb2_trk,
 	tegra_clk_usb3,
 	tegra_clk_usbd,
 	tegra_clk_vcp,
@@ -216,22 +276,35 @@ enum clk_id {
 	tegra_clk_vi,
 	tegra_clk_vi_8,
 	tegra_clk_vi_9,
+	tegra_clk_vi_10,
+	tegra_clk_vi_i2c,
 	tegra_clk_vic03,
+	tegra_clk_vic03_8,
 	tegra_clk_vim2_clk,
 	tegra_clk_vimclk_sync,
 	tegra_clk_vi_sensor,
-	tegra_clk_vi_sensor2,
 	tegra_clk_vi_sensor_8,
+	tegra_clk_vi_sensor_9,
+	tegra_clk_vi_sensor2,
+	tegra_clk_vi_sensor2_8,
 	tegra_clk_xusb_dev,
 	tegra_clk_xusb_dev_src,
+	tegra_clk_xusb_dev_src_8,
 	tegra_clk_xusb_falcon_src,
+	tegra_clk_xusb_falcon_src_8,
 	tegra_clk_xusb_fs_src,
+	tegra_clk_xusb_gate,
 	tegra_clk_xusb_host,
 	tegra_clk_xusb_host_src,
+	tegra_clk_xusb_host_src_8,
 	tegra_clk_xusb_hs_src,
+	tegra_clk_xusb_hs_src_4,
 	tegra_clk_xusb_ss,
 	tegra_clk_xusb_ss_src,
+	tegra_clk_xusb_ss_src_8,
 	tegra_clk_xusb_ss_div2,
+	tegra_clk_xusb_ssp_src,
+	tegra_clk_sclk_mux,
 	tegra_clk_max,
 };
 
diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c
index d6d4ecb88e94..a534bfab30b3 100644
--- a/drivers/clk/tegra/clk-pll.c
+++ b/drivers/clk/tegra/clk-pll.c
@@ -65,6 +65,7 @@
 #define PLLE_BASE_DIVN_WIDTH 8
 #define PLLE_BASE_DIVM_SHIFT 0
 #define PLLE_BASE_DIVM_WIDTH 8
+#define PLLE_BASE_ENABLE BIT(31)
 
 #define PLLE_MISC_SETUP_BASE_SHIFT 16
 #define PLLE_MISC_SETUP_BASE_MASK (0xffff << PLLE_MISC_SETUP_BASE_SHIFT)
@@ -102,6 +103,7 @@
 #define PLLE_AUX_SEQ_ENABLE	BIT(24)
 #define PLLE_AUX_SEQ_START_STATE BIT(25)
 #define PLLE_AUX_PLLRE_SEL	BIT(28)
+#define PLLE_AUX_SS_SEQ_INCLUDE	BIT(31)
 
 #define XUSBIO_PLL_CFG0		0x51c
 #define XUSBIO_PLL_CFG0_PADPLL_RESET_SWCTL	BIT(0)
@@ -187,17 +189,23 @@
 #define pll_readl_base(p) pll_readl(p->params->base_reg, p)
 #define pll_readl_misc(p) pll_readl(p->params->misc_reg, p)
 #define pll_override_readl(offset, p) readl_relaxed(p->pmc + offset)
+#define pll_readl_sdm_din(p) pll_readl(p->params->sdm_din_reg, p)
+#define pll_readl_sdm_ctrl(p) pll_readl(p->params->sdm_ctrl_reg, p)
 
 #define pll_writel(val, offset, p) writel_relaxed(val, p->clk_base + offset)
 #define pll_writel_base(val, p) pll_writel(val, p->params->base_reg, p)
 #define pll_writel_misc(val, p) pll_writel(val, p->params->misc_reg, p)
 #define pll_override_writel(val, offset, p) writel(val, p->pmc + offset)
+#define pll_writel_sdm_din(val, p) pll_writel(val, p->params->sdm_din_reg, p)
+#define pll_writel_sdm_ctrl(val, p) pll_writel(val, p->params->sdm_ctrl_reg, p)
 
 #define mask(w) ((1 << (w)) - 1)
 #define divm_mask(p) mask(p->params->div_nmp->divm_width)
 #define divn_mask(p) mask(p->params->div_nmp->divn_width)
 #define divp_mask(p) (p->params->flags & TEGRA_PLLU ? PLLU_POST_DIVP_MASK :\
 		      mask(p->params->div_nmp->divp_width))
+#define sdm_din_mask(p) p->params->sdm_din_mask
+#define sdm_en_mask(p) p->params->sdm_ctrl_en_mask
 
 #define divm_shift(p) (p)->params->div_nmp->divm_shift
 #define divn_shift(p) (p)->params->div_nmp->divn_shift
@@ -211,6 +219,9 @@
 #define divn_max(p) (divn_mask(p))
 #define divp_max(p) (1 << (divp_mask(p)))
 
+#define sdin_din_to_data(din)	((u16)((din) ? : 0xFFFFU))
+#define sdin_data_to_din(dat)	(((dat) == 0xFFFFU) ? 0 : (s16)dat)
+
 static struct div_nmp default_nmp = {
 	.divn_shift = PLL_BASE_DIVN_SHIFT,
 	.divn_width = PLL_BASE_DIVN_WIDTH,
@@ -269,6 +280,11 @@ static int clk_pll_wait_for_lock(struct tegra_clk_pll *pll)
 	return -1;
 }
 
+int tegra_pll_wait_for_lock(struct tegra_clk_pll *pll)
+{
+	return clk_pll_wait_for_lock(pll);
+}
+
 static int clk_pll_is_enabled(struct clk_hw *hw)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
@@ -290,6 +306,19 @@ static void _clk_pll_enable(struct clk_hw *hw)
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
 	u32 val;
 
+	if (pll->params->iddq_reg) {
+		val = pll_readl(pll->params->iddq_reg, pll);
+		val &= ~BIT(pll->params->iddq_bit_idx);
+		pll_writel(val, pll->params->iddq_reg, pll);
+		udelay(2);
+	}
+
+	if (pll->params->reset_reg) {
+		val = pll_readl(pll->params->reset_reg, pll);
+		val &= ~BIT(pll->params->reset_bit_idx);
+		pll_writel(val, pll->params->reset_reg, pll);
+	}
+
 	clk_pll_enable_lock(pll);
 
 	val = pll_readl_base(pll);
@@ -321,6 +350,19 @@ static void _clk_pll_disable(struct clk_hw *hw)
 		val &= ~PMC_PLLP_WB0_OVERRIDE_PLLM_ENABLE;
 		writel_relaxed(val, pll->pmc + PMC_PLLP_WB0_OVERRIDE);
 	}
+
+	if (pll->params->reset_reg) {
+		val = pll_readl(pll->params->reset_reg, pll);
+		val |= BIT(pll->params->reset_bit_idx);
+		pll_writel(val, pll->params->reset_reg, pll);
+	}
+
+	if (pll->params->iddq_reg) {
+		val = pll_readl(pll->params->iddq_reg, pll);
+		val |= BIT(pll->params->iddq_bit_idx);
+		pll_writel(val, pll->params->iddq_reg, pll);
+		udelay(2);
+	}
 }
 
 static int clk_pll_enable(struct clk_hw *hw)
@@ -359,7 +401,7 @@ static void clk_pll_disable(struct clk_hw *hw)
 static int _p_div_to_hw(struct clk_hw *hw, u8 p_div)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	struct pdiv_map *p_tohw = pll->params->pdiv_tohw;
+	const struct pdiv_map *p_tohw = pll->params->pdiv_tohw;
 
 	if (p_tohw) {
 		while (p_tohw->pdiv) {
@@ -372,10 +414,15 @@ static int _p_div_to_hw(struct clk_hw *hw, u8 p_div)
 	return -EINVAL;
 }
 
+int tegra_pll_p_div_to_hw(struct tegra_clk_pll *pll, u8 p_div)
+{
+	return _p_div_to_hw(&pll->hw, p_div);
+}
+
 static int _hw_to_p_div(struct clk_hw *hw, u8 p_div_hw)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	struct pdiv_map *p_tohw = pll->params->pdiv_tohw;
+	const struct pdiv_map *p_tohw = pll->params->pdiv_tohw;
 
 	if (p_tohw) {
 		while (p_tohw->pdiv) {
@@ -395,6 +442,7 @@ static int _get_table_rate(struct clk_hw *hw,
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
 	struct tegra_clk_pll_freq_table *sel;
+	int p;
 
 	for (sel = pll->params->freq_table; sel->input_rate != 0; sel++)
 		if (sel->input_rate == parent_rate &&
@@ -404,12 +452,21 @@ static int _get_table_rate(struct clk_hw *hw,
 	if (sel->input_rate == 0)
 		return -EINVAL;
 
+	if (pll->params->pdiv_tohw) {
+		p = _p_div_to_hw(hw, sel->p);
+		if (p < 0)
+			return p;
+	} else {
+		p = ilog2(sel->p);
+	}
+
 	cfg->input_rate = sel->input_rate;
 	cfg->output_rate = sel->output_rate;
 	cfg->m = sel->m;
 	cfg->n = sel->n;
-	cfg->p = sel->p;
+	cfg->p = p;
 	cfg->cpcon = sel->cpcon;
+	cfg->sdm_data = sel->sdm_data;
 
 	return 0;
 }
@@ -439,7 +496,7 @@ static int _calc_rate(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 		/*
 		 * PLL_P_OUT1 rate is not listed in PLLA table
 		 */
-		cfreq = parent_rate/(parent_rate/1000000);
+		cfreq = parent_rate / (parent_rate / 1000000);
 		break;
 	default:
 		pr_err("%s Unexpected reference rate %lu\n",
@@ -476,6 +533,42 @@ static int _calc_rate(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 	return 0;
 }
 
+/*
+ * SDM (Sigma Delta Modulator) divisor is 16-bit 2's complement signed number
+ * within (-2^12 ... 2^12-1) range. Represented in PLL data structure as
+ * unsigned 16-bit value, with "0" divisor mapped to 0xFFFF. Data "0" is used
+ * to indicate that SDM is disabled.
+ *
+ * Effective ndiv value when SDM is enabled: ndiv + 1/2 + sdm_din/2^13
+ */
+static void clk_pll_set_sdm_data(struct clk_hw *hw,
+				 struct tegra_clk_pll_freq_table *cfg)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	u32 val;
+	bool enabled;
+
+	if (!pll->params->sdm_din_reg)
+		return;
+
+	if (cfg->sdm_data) {
+		val = pll_readl_sdm_din(pll) & (~sdm_din_mask(pll));
+		val |= sdin_data_to_din(cfg->sdm_data) & sdm_din_mask(pll);
+		pll_writel_sdm_din(val, pll);
+	}
+
+	val = pll_readl_sdm_ctrl(pll);
+	enabled = (val & sdm_en_mask(pll));
+
+	if (cfg->sdm_data == 0 && enabled)
+		val &= ~pll->params->sdm_ctrl_en_mask;
+
+	if (cfg->sdm_data != 0 && !enabled)
+		val |= pll->params->sdm_ctrl_en_mask;
+
+	pll_writel_sdm_ctrl(val, pll);
+}
+
 static void _update_pll_mnp(struct tegra_clk_pll *pll,
 			    struct tegra_clk_pll_freq_table *cfg)
 {
@@ -483,7 +576,7 @@ static void _update_pll_mnp(struct tegra_clk_pll *pll,
 	struct tegra_clk_pll_params *params = pll->params;
 	struct div_nmp *div_nmp = params->div_nmp;
 
-	if ((params->flags & TEGRA_PLLM) &&
+	if ((params->flags & (TEGRA_PLLM | TEGRA_PLLMB)) &&
 		(pll_override_readl(PMC_PLLP_WB0_OVERRIDE, pll) &
 			PMC_PLLP_WB0_OVERRIDE_PLLM_OVERRIDE)) {
 		val = pll_override_readl(params->pmc_divp_reg, pll);
@@ -508,6 +601,8 @@ static void _update_pll_mnp(struct tegra_clk_pll *pll,
 		       (cfg->p << divp_shift(pll));
 
 		pll_writel_base(val, pll);
+
+		clk_pll_set_sdm_data(&pll->hw, cfg);
 	}
 }
 
@@ -518,7 +613,7 @@ static void _get_pll_mnp(struct tegra_clk_pll *pll,
 	struct tegra_clk_pll_params *params = pll->params;
 	struct div_nmp *div_nmp = params->div_nmp;
 
-	if ((params->flags & TEGRA_PLLM) &&
+	if ((params->flags & (TEGRA_PLLM | TEGRA_PLLMB)) &&
 		(pll_override_readl(PMC_PLLP_WB0_OVERRIDE, pll) &
 			PMC_PLLP_WB0_OVERRIDE_PLLM_OVERRIDE)) {
 		val = pll_override_readl(params->pmc_divp_reg, pll);
@@ -533,6 +628,14 @@ static void _get_pll_mnp(struct tegra_clk_pll *pll,
 		cfg->m = (val >> div_nmp->divm_shift) & divm_mask(pll);
 		cfg->n = (val >> div_nmp->divn_shift) & divn_mask(pll);
 		cfg->p = (val >> div_nmp->divp_shift) & divp_mask(pll);
+
+		if (pll->params->sdm_din_reg) {
+			if (sdm_en_mask(pll) & pll_readl_sdm_ctrl(pll)) {
+				val = pll_readl_sdm_din(pll);
+				val &= sdm_din_mask(pll);
+				cfg->sdm_data = sdin_din_to_data(val);
+			}
+		}
 	}
 }
 
@@ -560,16 +663,51 @@ static void _update_pll_cpcon(struct tegra_clk_pll *pll,
 	pll_writel_misc(val, pll);
 }
 
+static void pll_clk_start_ss(struct tegra_clk_pll *pll)
+{
+	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
+		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
+
+		val |= pll->params->ssc_ctrl_en_mask;
+		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
+	}
+}
+
+static void pll_clk_stop_ss(struct tegra_clk_pll *pll)
+{
+	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
+		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
+
+		val &= ~pll->params->ssc_ctrl_en_mask;
+		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
+	}
+}
+
 static int _program_pll(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 			unsigned long rate)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	struct tegra_clk_pll_freq_table old_cfg;
 	int state, ret = 0;
 
 	state = clk_pll_is_enabled(hw);
 
-	if (state)
+	_get_pll_mnp(pll, &old_cfg);
+
+	if (state && pll->params->defaults_set && pll->params->dyn_ramp &&
+			(cfg->m == old_cfg.m) && (cfg->p == old_cfg.p)) {
+		ret = pll->params->dyn_ramp(pll, cfg);
+		if (!ret)
+			return 0;
+	}
+
+	if (state) {
+		pll_clk_stop_ss(pll);
 		_clk_pll_disable(hw);
+	}
+
+	if (!pll->params->defaults_set && pll->params->set_defaults)
+		pll->params->set_defaults(pll);
 
 	_update_pll_mnp(pll, cfg);
 
@@ -579,6 +717,7 @@ static int _program_pll(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 	if (state) {
 		_clk_pll_enable(hw);
 		ret = clk_pll_wait_for_lock(pll);
+		pll_clk_start_ss(pll);
 	}
 
 	return ret;
@@ -603,7 +742,7 @@ static int clk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 	}
 
 	if (_get_table_rate(hw, &cfg, rate, parent_rate) &&
-	    _calc_rate(hw, &cfg, rate, parent_rate)) {
+	    pll->params->calc_rate(hw, &cfg, rate, parent_rate)) {
 		pr_err("%s: Failed to set %s rate %lu\n", __func__,
 		       clk_hw_get_name(hw), rate);
 		WARN_ON(1);
@@ -613,8 +752,11 @@ static int clk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 		spin_lock_irqsave(pll->lock, flags);
 
 	_get_pll_mnp(pll, &old_cfg);
+	if (pll->params->flags & TEGRA_PLL_VCO_OUT)
+		cfg.p = old_cfg.p;
 
-	if (old_cfg.m != cfg.m || old_cfg.n != cfg.n || old_cfg.p != cfg.p)
+	if (old_cfg.m != cfg.m || old_cfg.n != cfg.n || old_cfg.p != cfg.p ||
+		old_cfg.sdm_data != cfg.sdm_data)
 		ret = _program_pll(hw, &cfg, rate);
 
 	if (pll->lock)
@@ -629,15 +771,15 @@ static long clk_pll_round_rate(struct clk_hw *hw, unsigned long rate,
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
 	struct tegra_clk_pll_freq_table cfg;
 
-	if (pll->params->flags & TEGRA_PLL_FIXED)
+	if (pll->params->flags & TEGRA_PLL_FIXED) {
+		/* PLLM/MB are used for memory; we do not change rate */
+		if (pll->params->flags & (TEGRA_PLLM | TEGRA_PLLMB))
+			return clk_hw_get_rate(hw);
 		return pll->params->fixed_rate;
-
-	/* PLLM is used for memory; we do not change rate */
-	if (pll->params->flags & TEGRA_PLLM)
-		return clk_hw_get_rate(hw);
+	}
 
 	if (_get_table_rate(hw, &cfg, rate, *prate) &&
-	    _calc_rate(hw, &cfg, rate, *prate))
+	    pll->params->calc_rate(hw, &cfg, rate, *prate))
 		return -EINVAL;
 
 	return cfg.output_rate;
@@ -658,6 +800,7 @@ static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
 		return parent_rate;
 
 	if ((pll->params->flags & TEGRA_PLL_FIXED) &&
+	    !(pll->params->flags & (TEGRA_PLLM | TEGRA_PLLMB)) &&
 			!(val & PLL_BASE_OVERRIDE)) {
 		struct tegra_clk_pll_freq_table sel;
 		if (_get_table_rate(hw, &sel, pll->params->fixed_rate,
@@ -671,12 +814,20 @@ static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
 
 	_get_pll_mnp(pll, &cfg);
 
-	pdiv = _hw_to_p_div(hw, cfg.p);
-	if (pdiv < 0) {
-		WARN_ON(1);
+	if (pll->params->flags & TEGRA_PLL_VCO_OUT) {
 		pdiv = 1;
+	} else {
+		pdiv = _hw_to_p_div(hw, cfg.p);
+		if (pdiv < 0) {
+			WARN(1, "Clock %s has invalid pdiv value : 0x%x\n",
+			     clk_hw_get_name(hw), cfg.p);
+			pdiv = 1;
+		}
 	}
 
+	if (pll->params->set_gain)
+		pll->params->set_gain(&cfg);
+
 	cfg.m *= pdiv;
 
 	rate *= cfg.n;
@@ -816,19 +967,65 @@ const struct clk_ops tegra_clk_plle_ops = {
 	.enable = clk_plle_enable,
 };
 
-#if defined(CONFIG_ARCH_TEGRA_114_SOC) || \
-	defined(CONFIG_ARCH_TEGRA_124_SOC) || \
-	defined(CONFIG_ARCH_TEGRA_132_SOC)
-
 static int _pll_fixed_mdiv(struct tegra_clk_pll_params *pll_params,
 			   unsigned long parent_rate)
 {
+	u16 mdiv = parent_rate / pll_params->cf_min;
+
+	if (pll_params->flags & TEGRA_MDIV_NEW)
+		return (!pll_params->mdiv_default ? mdiv :
+			min(mdiv, pll_params->mdiv_default));
+
+	if (pll_params->mdiv_default)
+		return pll_params->mdiv_default;
+
 	if (parent_rate > pll_params->cf_max)
 		return 2;
 	else
 		return 1;
 }
 
+static int _calc_dynamic_ramp_rate(struct clk_hw *hw,
+				struct tegra_clk_pll_freq_table *cfg,
+				unsigned long rate, unsigned long parent_rate)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	unsigned int p;
+	int p_div;
+
+	if (!rate)
+		return -EINVAL;
+
+	p = DIV_ROUND_UP(pll->params->vco_min, rate);
+	cfg->m = _pll_fixed_mdiv(pll->params, parent_rate);
+	cfg->output_rate = rate * p;
+	cfg->n = cfg->output_rate * cfg->m / parent_rate;
+	cfg->input_rate = parent_rate;
+
+	p_div = _p_div_to_hw(hw, p);
+	if (p_div < 0)
+		return p_div;
+
+	cfg->p = p_div;
+
+	if (cfg->n > divn_max(pll) || cfg->output_rate > pll->params->vco_max)
+		return -EINVAL;
+
+	return 0;
+}
+
+#if defined(CONFIG_ARCH_TEGRA_114_SOC) || \
+	defined(CONFIG_ARCH_TEGRA_124_SOC) || \
+	defined(CONFIG_ARCH_TEGRA_132_SOC) || \
+	defined(CONFIG_ARCH_TEGRA_210_SOC)
+
+u16 tegra_pll_get_fixed_mdiv(struct clk_hw *hw, unsigned long input_rate)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+
+	return (u16)_pll_fixed_mdiv(pll->params, input_rate);
+}
+
 static unsigned long _clip_vco_min(unsigned long vco_min,
 				   unsigned long parent_rate)
 {
@@ -871,86 +1068,12 @@ static int _setup_dynamic_ramp(struct tegra_clk_pll_params *pll_params,
 	return 0;
 }
 
-static int clk_pll_iddq_enable(struct clk_hw *hw)
-{
-	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	unsigned long flags = 0;
-
-	u32 val;
-	int ret;
-
-	if (pll->lock)
-		spin_lock_irqsave(pll->lock, flags);
-
-	val = pll_readl(pll->params->iddq_reg, pll);
-	val &= ~BIT(pll->params->iddq_bit_idx);
-	pll_writel(val, pll->params->iddq_reg, pll);
-	udelay(2);
-
-	_clk_pll_enable(hw);
-
-	ret = clk_pll_wait_for_lock(pll);
-
-	if (pll->lock)
-		spin_unlock_irqrestore(pll->lock, flags);
-
-	return 0;
-}
-
-static void clk_pll_iddq_disable(struct clk_hw *hw)
-{
-	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	unsigned long flags = 0;
-	u32 val;
-
-	if (pll->lock)
-		spin_lock_irqsave(pll->lock, flags);
-
-	_clk_pll_disable(hw);
-
-	val = pll_readl(pll->params->iddq_reg, pll);
-	val |= BIT(pll->params->iddq_bit_idx);
-	pll_writel(val, pll->params->iddq_reg, pll);
-	udelay(2);
-
-	if (pll->lock)
-		spin_unlock_irqrestore(pll->lock, flags);
-}
-
-static int _calc_dynamic_ramp_rate(struct clk_hw *hw,
-				struct tegra_clk_pll_freq_table *cfg,
-				unsigned long rate, unsigned long parent_rate)
-{
-	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	unsigned int p;
-	int p_div;
-
-	if (!rate)
-		return -EINVAL;
-
-	p = DIV_ROUND_UP(pll->params->vco_min, rate);
-	cfg->m = _pll_fixed_mdiv(pll->params, parent_rate);
-	cfg->output_rate = rate * p;
-	cfg->n = cfg->output_rate * cfg->m / parent_rate;
-
-	p_div = _p_div_to_hw(hw, p);
-	if (p_div < 0)
-		return p_div;
-	else
-		cfg->p = p_div;
-
-	if (cfg->n > divn_max(pll) || cfg->output_rate > pll->params->vco_max)
-		return -EINVAL;
-
-	return 0;
-}
-
 static int _pll_ramp_calc_pll(struct clk_hw *hw,
 			      struct tegra_clk_pll_freq_table *cfg,
 			      unsigned long rate, unsigned long parent_rate)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	int err = 0, p_div;
+	int err = 0;
 
 	err = _get_table_rate(hw, cfg, rate, parent_rate);
 	if (err < 0)
@@ -961,11 +1084,6 @@ static int _pll_ramp_calc_pll(struct clk_hw *hw,
 			err = -EINVAL;
 			goto out;
 		}
-		p_div = _p_div_to_hw(hw, cfg->p);
-		if (p_div < 0)
-			return p_div;
-		else
-			cfg->p = p_div;
 	}
 
 	if (cfg->p >  pll->params->max_p)
@@ -991,6 +1109,8 @@ static int clk_pllxc_set_rate(struct clk_hw *hw, unsigned long rate,
 		spin_lock_irqsave(pll->lock, flags);
 
 	_get_pll_mnp(pll, &old_cfg);
+	if (pll->params->flags & TEGRA_PLL_VCO_OUT)
+		cfg.p = old_cfg.p;
 
 	if (old_cfg.m != cfg.m || old_cfg.n != cfg.n || old_cfg.p != cfg.p)
 		ret = _program_pll(hw, &cfg, rate);
@@ -1004,6 +1124,7 @@ static int clk_pllxc_set_rate(struct clk_hw *hw, unsigned long rate,
 static long clk_pll_ramp_round_rate(struct clk_hw *hw, unsigned long rate,
 				unsigned long *prate)
 {
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
 	struct tegra_clk_pll_freq_table cfg;
 	int ret, p_div;
 	u64 output_rate = *prate;
@@ -1016,46 +1137,15 @@ static long clk_pll_ramp_round_rate(struct clk_hw *hw, unsigned long rate,
 	if (p_div < 0)
 		return p_div;
 
+	if (pll->params->set_gain)
+		pll->params->set_gain(&cfg);
+
 	output_rate *= cfg.n;
 	do_div(output_rate, cfg.m * p_div);
 
 	return output_rate;
 }
 
-static int clk_pllm_set_rate(struct clk_hw *hw, unsigned long rate,
-				unsigned long parent_rate)
-{
-	struct tegra_clk_pll_freq_table cfg;
-	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	unsigned long flags = 0;
-	int state, ret = 0;
-
-	if (pll->lock)
-		spin_lock_irqsave(pll->lock, flags);
-
-	state = clk_pll_is_enabled(hw);
-	if (state) {
-		if (rate != clk_get_rate(hw->clk)) {
-			pr_err("%s: Cannot change active PLLM\n", __func__);
-			ret = -EINVAL;
-			goto out;
-		}
-		goto out;
-	}
-
-	ret = _pll_ramp_calc_pll(hw, &cfg, rate, parent_rate);
-	if (ret < 0)
-		goto out;
-
-	_update_pll_mnp(pll, &cfg);
-
-out:
-	if (pll->lock)
-		spin_unlock_irqrestore(pll->lock, flags);
-
-	return ret;
-}
-
 static void _pllcx_strobe(struct tegra_clk_pll *pll)
 {
 	u32 val;
@@ -1445,6 +1535,17 @@ static struct clk *_tegra_clk_register_pll(struct tegra_clk_pll *pll,
 	init.parent_names = (parent_name ? &parent_name : NULL);
 	init.num_parents = (parent_name ? 1 : 0);
 
+	/* Default to _calc_rate if unspecified */
+	if (!pll->params->calc_rate) {
+		if (pll->params->flags & TEGRA_PLLM)
+			pll->params->calc_rate = _calc_dynamic_ramp_rate;
+		else
+			pll->params->calc_rate = _calc_rate;
+	}
+
+	if (pll->params->set_defaults)
+		pll->params->set_defaults(pll);
+
 	/* Data in .init is copied by clk_register(), so stack variable OK */
 	pll->hw.init = &init;
 
@@ -1460,7 +1561,7 @@ struct clk *tegra_clk_register_pll(const char *name, const char *parent_name,
 	struct clk *clk;
 
 	pll_params->flags |= TEGRA_PLL_BYPASS;
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE;
+
 	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
@@ -1490,8 +1591,7 @@ struct clk *tegra_clk_register_plle(const char *name, const char *parent_name,
 	struct tegra_clk_pll *pll;
 	struct clk *clk;
 
-	pll_params->flags |= TEGRA_PLL_LOCK_MISC | TEGRA_PLL_BYPASS;
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE;
+	pll_params->flags |= TEGRA_PLL_BYPASS;
 
 	if (!pll_params->div_nmp)
 		pll_params->div_nmp = &pll_e_nmp;
@@ -1510,25 +1610,17 @@ struct clk *tegra_clk_register_plle(const char *name, const char *parent_name,
 
 #if defined(CONFIG_ARCH_TEGRA_114_SOC) || \
 	defined(CONFIG_ARCH_TEGRA_124_SOC) || \
-	defined(CONFIG_ARCH_TEGRA_132_SOC)
+	defined(CONFIG_ARCH_TEGRA_132_SOC) || \
+	defined(CONFIG_ARCH_TEGRA_210_SOC)
 static const struct clk_ops tegra_clk_pllxc_ops = {
 	.is_enabled = clk_pll_is_enabled,
-	.enable = clk_pll_iddq_enable,
-	.disable = clk_pll_iddq_disable,
+	.enable = clk_pll_enable,
+	.disable = clk_pll_disable,
 	.recalc_rate = clk_pll_recalc_rate,
 	.round_rate = clk_pll_ramp_round_rate,
 	.set_rate = clk_pllxc_set_rate,
 };
 
-static const struct clk_ops tegra_clk_pllm_ops = {
-	.is_enabled = clk_pll_is_enabled,
-	.enable = clk_pll_iddq_enable,
-	.disable = clk_pll_iddq_disable,
-	.recalc_rate = clk_pll_recalc_rate,
-	.round_rate = clk_pll_ramp_round_rate,
-	.set_rate = clk_pllm_set_rate,
-};
-
 static const struct clk_ops tegra_clk_pllc_ops = {
 	.is_enabled = clk_pll_is_enabled,
 	.enable = clk_pllc_enable,
@@ -1540,8 +1632,8 @@ static const struct clk_ops tegra_clk_pllc_ops = {
 
 static const struct clk_ops tegra_clk_pllre_ops = {
 	.is_enabled = clk_pll_is_enabled,
-	.enable = clk_pll_iddq_enable,
-	.disable = clk_pll_iddq_disable,
+	.enable = clk_pll_enable,
+	.disable = clk_pll_disable,
 	.recalc_rate = clk_pllre_recalc_rate,
 	.round_rate = clk_pllre_round_rate,
 	.set_rate = clk_pllre_set_rate,
@@ -1564,7 +1656,6 @@ struct clk *tegra_clk_register_pllxc(const char *name, const char *parent_name,
 	struct tegra_clk_pll *pll;
 	struct clk *clk, *parent;
 	unsigned long parent_rate;
-	int err;
 	u32 val, val_iddq;
 
 	parent = __clk_lookup(parent_name);
@@ -1581,21 +1672,33 @@ struct clk *tegra_clk_register_pllxc(const char *name, const char *parent_name,
 
 	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
 
-	err = _setup_dynamic_ramp(pll_params, clk_base, parent_rate);
-	if (err)
-		return ERR_PTR(err);
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
 
-	val = readl_relaxed(clk_base + pll_params->base_reg);
-	val_iddq = readl_relaxed(clk_base + pll_params->iddq_reg);
+	/*
+	 * If the pll has a set_defaults callback, it will take care of
+	 * configuring dynamic ramping and setting IDDQ in that path.
+	 */
+	if (!pll_params->set_defaults) {
+		int err;
 
-	if (val & PLL_BASE_ENABLE)
-		WARN_ON(val_iddq & BIT(pll_params->iddq_bit_idx));
-	else {
-		val_iddq |= BIT(pll_params->iddq_bit_idx);
-		writel_relaxed(val_iddq, clk_base + pll_params->iddq_reg);
+		err = _setup_dynamic_ramp(pll_params, clk_base, parent_rate);
+		if (err)
+			return ERR_PTR(err);
+
+		val = readl_relaxed(clk_base + pll_params->base_reg);
+		val_iddq = readl_relaxed(clk_base + pll_params->iddq_reg);
+
+		if (val & PLL_BASE_ENABLE)
+			WARN_ON(val_iddq & BIT(pll_params->iddq_bit_idx));
+		else {
+			val_iddq |= BIT(pll_params->iddq_bit_idx);
+			writel_relaxed(val_iddq,
+				       clk_base + pll_params->iddq_reg);
+		}
 	}
 
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE;
 	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
@@ -1618,10 +1721,12 @@ struct clk *tegra_clk_register_pllre(const char *name, const char *parent_name,
 	struct tegra_clk_pll *pll;
 	struct clk *clk;
 
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_LOCK_MISC;
-
 	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
 
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
 	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
@@ -1630,7 +1735,8 @@ struct clk *tegra_clk_register_pllre(const char *name, const char *parent_name,
 
 	val = pll_readl_base(pll);
 	if (val & PLL_BASE_ENABLE)
-		WARN_ON(val & pll_params->iddq_bit_idx);
+		WARN_ON(readl_relaxed(clk_base + pll_params->iddq_reg) &
+				BIT(pll_params->iddq_bit_idx));
 	else {
 		int m;
 
@@ -1678,15 +1784,18 @@ struct clk *tegra_clk_register_pllm(const char *name, const char *parent_name,
 
 	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
 
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
 	pll_params->flags |= TEGRA_PLL_BYPASS;
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE;
 	pll_params->flags |= TEGRA_PLLM;
 	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
 
 	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
-				      &tegra_clk_pllm_ops);
+				      &tegra_clk_pll_ops);
 	if (IS_ERR(clk))
 		kfree(pll);
 
@@ -1700,7 +1809,7 @@ struct clk *tegra_clk_register_pllc(const char *name, const char *parent_name,
 			  spinlock_t *lock)
 {
 	struct clk *parent, *clk;
-	struct pdiv_map *p_tohw = pll_params->pdiv_tohw;
+	const struct pdiv_map *p_tohw = pll_params->pdiv_tohw;
 	struct tegra_clk_pll *pll;
 	struct tegra_clk_pll_freq_table cfg;
 	unsigned long parent_rate;
@@ -1777,7 +1886,6 @@ struct clk *tegra_clk_register_plle_tegra114(const char *name,
 	struct clk *clk;
 	u32 val, val_aux;
 
-	pll_params->flags |= TEGRA_PLL_HAS_LOCK_ENABLE;
 	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
@@ -1810,8 +1918,8 @@ struct clk *tegra_clk_register_plle_tegra114(const char *name,
 #if defined(CONFIG_ARCH_TEGRA_124_SOC) || defined(CONFIG_ARCH_TEGRA_132_SOC)
 static const struct clk_ops tegra_clk_pllss_ops = {
 	.is_enabled = clk_pll_is_enabled,
-	.enable = clk_pll_iddq_enable,
-	.disable = clk_pll_iddq_disable,
+	.enable = clk_pll_enable,
+	.disable = clk_pll_disable,
 	.recalc_rate = clk_pll_recalc_rate,
 	.round_rate = clk_pll_ramp_round_rate,
 	.set_rate = clk_pllxc_set_rate,
@@ -1826,7 +1934,7 @@ struct clk *tegra_clk_register_pllss(const char *name, const char *parent_name,
 	struct clk *clk, *parent;
 	struct tegra_clk_pll_freq_table cfg;
 	unsigned long parent_rate;
-	u32 val;
+	u32 val, val_iddq;
 	int i;
 
 	if (!pll_params->div_nmp)
@@ -1839,7 +1947,6 @@ struct clk *tegra_clk_register_pllss(const char *name, const char *parent_name,
 		return ERR_PTR(-EINVAL);
 	}
 
-	pll_params->flags = TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_USE_LOCK;
 	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
@@ -1874,6 +1981,337 @@ struct clk *tegra_clk_register_pllss(const char *name, const char *parent_name,
 	pll_writel(PLLSS_CTRL1_DEFAULT, pll_params->ext_misc_reg[2], pll);
 
 	val = pll_readl_base(pll);
+	val_iddq = readl_relaxed(clk_base + pll_params->iddq_reg);
+	if (val & PLL_BASE_ENABLE) {
+		if (val_iddq & BIT(pll_params->iddq_bit_idx)) {
+			WARN(1, "%s is on but IDDQ set\n", name);
+			kfree(pll);
+			return ERR_PTR(-EINVAL);
+		}
+	} else {
+		val_iddq |= BIT(pll_params->iddq_bit_idx);
+		writel_relaxed(val_iddq, clk_base + pll_params->iddq_reg);
+	}
+
+	val &= ~PLLSS_LOCK_OVERRIDE;
+	pll_writel_base(val, pll);
+
+	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
+					&tegra_clk_pllss_ops);
+
+	if (IS_ERR(clk))
+		kfree(pll);
+
+	return clk;
+}
+#endif
+
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+static int clk_plle_tegra210_enable(struct clk_hw *hw)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	struct tegra_clk_pll_freq_table sel;
+	u32 val;
+	int ret;
+	unsigned long flags = 0;
+	unsigned long input_rate = clk_get_rate(clk_get_parent(hw->clk));
+
+	if (_get_table_rate(hw, &sel, pll->params->fixed_rate, input_rate))
+		return -EINVAL;
+
+	if (pll->lock)
+		spin_lock_irqsave(pll->lock, flags);
+
+	val = pll_readl_base(pll);
+	val &= ~BIT(30); /* Disable lock override */
+	pll_writel_base(val, pll);
+
+	val = pll_readl(pll->params->aux_reg, pll);
+	val |= PLLE_AUX_ENABLE_SWCTL;
+	val &= ~PLLE_AUX_SEQ_ENABLE;
+	pll_writel(val, pll->params->aux_reg, pll);
+	udelay(1);
+
+	val = pll_readl_misc(pll);
+	val |= PLLE_MISC_LOCK_ENABLE;
+	val |= PLLE_MISC_IDDQ_SW_CTRL;
+	val &= ~PLLE_MISC_IDDQ_SW_VALUE;
+	val |= PLLE_MISC_PLLE_PTS;
+	val |= PLLE_MISC_VREG_BG_CTRL_MASK | PLLE_MISC_VREG_CTRL_MASK;
+	pll_writel_misc(val, pll);
+	udelay(5);
+
+	val = pll_readl(PLLE_SS_CTRL, pll);
+	val |= PLLE_SS_DISABLE;
+	pll_writel(val, PLLE_SS_CTRL, pll);
+
+	val = pll_readl_base(pll);
+	val &= ~(divp_mask_shifted(pll) | divn_mask_shifted(pll) |
+		 divm_mask_shifted(pll));
+	val &= ~(PLLE_BASE_DIVCML_MASK << PLLE_BASE_DIVCML_SHIFT);
+	val |= sel.m << divm_shift(pll);
+	val |= sel.n << divn_shift(pll);
+	val |= sel.cpcon << PLLE_BASE_DIVCML_SHIFT;
+	pll_writel_base(val, pll);
+	udelay(1);
+
+	val = pll_readl_base(pll);
+	val |= PLLE_BASE_ENABLE;
+	pll_writel_base(val, pll);
+
+	ret = clk_pll_wait_for_lock(pll);
+
+	if (ret < 0)
+		goto out;
+
+	val = pll_readl(PLLE_SS_CTRL, pll);
+	val &= ~(PLLE_SS_CNTL_CENTER | PLLE_SS_CNTL_INVERT);
+	val &= ~PLLE_SS_COEFFICIENTS_MASK;
+	val |= PLLE_SS_COEFFICIENTS_VAL;
+	pll_writel(val, PLLE_SS_CTRL, pll);
+	val &= ~(PLLE_SS_CNTL_SSC_BYP | PLLE_SS_CNTL_BYPASS_SS);
+	pll_writel(val, PLLE_SS_CTRL, pll);
+	udelay(1);
+	val &= ~PLLE_SS_CNTL_INTERP_RESET;
+	pll_writel(val, PLLE_SS_CTRL, pll);
+	udelay(1);
+
+	val = pll_readl_misc(pll);
+	val &= ~PLLE_MISC_IDDQ_SW_CTRL;
+	pll_writel_misc(val, pll);
+
+	val = pll_readl(pll->params->aux_reg, pll);
+	val |= (PLLE_AUX_USE_LOCKDET | PLLE_AUX_SS_SEQ_INCLUDE);
+	val &= ~(PLLE_AUX_ENABLE_SWCTL | PLLE_AUX_SS_SWCTL);
+	pll_writel(val, pll->params->aux_reg, pll);
+	udelay(1);
+	val |= PLLE_AUX_SEQ_ENABLE;
+	pll_writel(val, pll->params->aux_reg, pll);
+
+out:
+	if (pll->lock)
+		spin_unlock_irqrestore(pll->lock, flags);
+
+	return ret;
+}
+
+static void clk_plle_tegra210_disable(struct clk_hw *hw)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	unsigned long flags = 0;
+	u32 val;
+
+	if (pll->lock)
+		spin_lock_irqsave(pll->lock, flags);
+
+	val = pll_readl_base(pll);
+	val &= ~PLLE_BASE_ENABLE;
+	pll_writel_base(val, pll);
+
+	val = pll_readl_misc(pll);
+	val |= PLLE_MISC_IDDQ_SW_CTRL | PLLE_MISC_IDDQ_SW_VALUE;
+	pll_writel_misc(val, pll);
+	udelay(1);
+
+	if (pll->lock)
+		spin_unlock_irqrestore(pll->lock, flags);
+}
+
+static int clk_plle_tegra210_is_enabled(struct clk_hw *hw)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	u32 val;
+
+	val = pll_readl_base(pll);
+
+	return val & PLLE_BASE_ENABLE ? 1 : 0;
+}
+
+static const struct clk_ops tegra_clk_plle_tegra210_ops = {
+	.is_enabled =  clk_plle_tegra210_is_enabled,
+	.enable = clk_plle_tegra210_enable,
+	.disable = clk_plle_tegra210_disable,
+	.recalc_rate = clk_pll_recalc_rate,
+};
+
+struct clk *tegra_clk_register_plle_tegra210(const char *name,
+				const char *parent_name,
+				void __iomem *clk_base, unsigned long flags,
+				struct tegra_clk_pll_params *pll_params,
+				spinlock_t *lock)
+{
+	struct tegra_clk_pll *pll;
+	struct clk *clk;
+	u32 val, val_aux;
+
+	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
+
+	/* ensure parent is set to pll_re_vco */
+
+	val = pll_readl_base(pll);
+	val_aux = pll_readl(pll_params->aux_reg, pll);
+
+	if (val & PLLE_BASE_ENABLE) {
+		if ((val_aux & PLLE_AUX_PLLRE_SEL) ||
+			(val_aux & PLLE_AUX_PLLP_SEL))
+			WARN(1, "pll_e enabled with unsupported parent %s\n",
+			  (val_aux & PLLE_AUX_PLLP_SEL) ? "pllp_out0" :
+					"pll_re_vco");
+	} else {
+		val_aux &= ~(PLLE_AUX_PLLRE_SEL | PLLE_AUX_PLLP_SEL);
+		pll_writel(val_aux, pll_params->aux_reg, pll);
+	}
+
+	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
+				      &tegra_clk_plle_tegra210_ops);
+	if (IS_ERR(clk))
+		kfree(pll);
+
+	return clk;
+}
+
+struct clk *tegra_clk_register_pllc_tegra210(const char *name,
+			const char *parent_name, void __iomem *clk_base,
+			void __iomem *pmc, unsigned long flags,
+			struct tegra_clk_pll_params *pll_params,
+			spinlock_t *lock)
+{
+	struct clk *parent, *clk;
+	const struct pdiv_map *p_tohw = pll_params->pdiv_tohw;
+	struct tegra_clk_pll *pll;
+	unsigned long parent_rate;
+
+	if (!p_tohw)
+		return ERR_PTR(-EINVAL);
+
+	parent = __clk_lookup(parent_name);
+	if (!parent) {
+		WARN(1, "parent clk %s of %s must be registered first\n",
+			name, parent_name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	parent_rate = clk_get_rate(parent);
+
+	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
+
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
+	pll_params->flags |= TEGRA_PLL_BYPASS;
+	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
+
+	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
+				      &tegra_clk_pll_ops);
+	if (IS_ERR(clk))
+		kfree(pll);
+
+	return clk;
+}
+
+struct clk *tegra_clk_register_pllxc_tegra210(const char *name,
+			const char *parent_name, void __iomem *clk_base,
+			void __iomem *pmc, unsigned long flags,
+			struct tegra_clk_pll_params *pll_params,
+			spinlock_t *lock)
+{
+	struct tegra_clk_pll *pll;
+	struct clk *clk, *parent;
+	unsigned long parent_rate;
+
+	parent = __clk_lookup(parent_name);
+	if (!parent) {
+		WARN(1, "parent clk %s of %s must be registered first\n",
+			name, parent_name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!pll_params->pdiv_tohw)
+		return ERR_PTR(-EINVAL);
+
+	parent_rate = clk_get_rate(parent);
+
+	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
+
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
+	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
+
+	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
+				      &tegra_clk_pll_ops);
+	if (IS_ERR(clk))
+		kfree(pll);
+
+	return clk;
+}
+
+struct clk *tegra_clk_register_pllss_tegra210(const char *name,
+				const char *parent_name, void __iomem *clk_base,
+				unsigned long flags,
+				struct tegra_clk_pll_params *pll_params,
+				spinlock_t *lock)
+{
+	struct tegra_clk_pll *pll;
+	struct clk *clk, *parent;
+	struct tegra_clk_pll_freq_table cfg;
+	unsigned long parent_rate;
+	u32 val;
+	int i;
+
+	if (!pll_params->div_nmp)
+		return ERR_PTR(-EINVAL);
+
+	parent = __clk_lookup(parent_name);
+	if (!parent) {
+		WARN(1, "parent clk %s of %s must be registered first\n",
+			name, parent_name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
+
+	val = pll_readl_base(pll);
+	val &= ~PLLSS_REF_SRC_SEL_MASK;
+	pll_writel_base(val, pll);
+
+	parent_rate = clk_get_rate(parent);
+
+	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
+
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
+	/* initialize PLL to minimum rate */
+
+	cfg.m = _pll_fixed_mdiv(pll_params, parent_rate);
+	cfg.n = cfg.m * pll_params->vco_min / parent_rate;
+
+	for (i = 0; pll_params->pdiv_tohw[i].pdiv; i++)
+		;
+	if (!i) {
+		kfree(pll);
+		return ERR_PTR(-EINVAL);
+	}
+
+	cfg.p = pll_params->pdiv_tohw[i-1].hw_val;
+
+	_update_pll_mnp(pll, &cfg);
+
+	pll_writel_misc(PLLSS_MISC_DEFAULT, pll);
+
+	val = pll_readl_base(pll);
 	if (val & PLL_BASE_ENABLE) {
 		if (val & BIT(pll_params->iddq_bit_idx)) {
 			WARN(1, "%s is on but IDDQ set\n", name);
@@ -1887,8 +2325,50 @@ struct clk *tegra_clk_register_pllss(const char *name, const char *parent_name,
 	pll_writel_base(val, pll);
 
 	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
-					&tegra_clk_pllss_ops);
+					&tegra_clk_pll_ops);
+
+	if (IS_ERR(clk))
+		kfree(pll);
+
+	return clk;
+}
+
+struct clk *tegra_clk_register_pllmb(const char *name, const char *parent_name,
+			  void __iomem *clk_base, void __iomem *pmc,
+			  unsigned long flags,
+			  struct tegra_clk_pll_params *pll_params,
+			  spinlock_t *lock)
+{
+	struct tegra_clk_pll *pll;
+	struct clk *clk, *parent;
+	unsigned long parent_rate;
 
+	if (!pll_params->pdiv_tohw)
+		return ERR_PTR(-EINVAL);
+
+	parent = __clk_lookup(parent_name);
+	if (!parent) {
+		WARN(1, "parent clk %s of %s must be registered first\n",
+			parent_name, name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	parent_rate = clk_get_rate(parent);
+
+	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
+
+	if (pll_params->adjust_vco)
+		pll_params->vco_min = pll_params->adjust_vco(pll_params,
+							     parent_rate);
+
+	pll_params->flags |= TEGRA_PLL_BYPASS;
+	pll_params->flags |= TEGRA_PLLMB;
+	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
+
+	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
+				      &tegra_clk_pll_ops);
 	if (IS_ERR(clk))
 		kfree(pll);
 
diff --git a/drivers/clk/tegra/clk-tegra-periph.c b/drivers/clk/tegra/clk-tegra-periph.c
index cb6ab830941d..6ad381a888a6 100644
--- a/drivers/clk/tegra/clk-tegra-periph.c
+++ b/drivers/clk/tegra/clk-tegra-periph.c
@@ -124,6 +124,20 @@
 #define CLK_SOURCE_HDMI_AUDIO 0x668
 #define CLK_SOURCE_VIC03 0x678
 #define CLK_SOURCE_CLK72MHZ 0x66c
+#define CLK_SOURCE_DBGAPB 0x718
+#define CLK_SOURCE_NVENC 0x6a0
+#define CLK_SOURCE_NVDEC 0x698
+#define CLK_SOURCE_NVJPG 0x69c
+#define CLK_SOURCE_APE 0x6c0
+#define CLK_SOURCE_SOR1 0x410
+#define CLK_SOURCE_SDMMC_LEGACY 0x694
+#define CLK_SOURCE_QSPI 0x6c4
+#define CLK_SOURCE_VI_I2C 0x6c8
+#define CLK_SOURCE_MIPIBIF 0x660
+#define CLK_SOURCE_UARTAPE 0x710
+#define CLK_SOURCE_TSECB 0x6d8
+#define CLK_SOURCE_MAUD 0x6d4
+#define CLK_SOURCE_USB2_HSIC_TRK 0x6cc
 
 #define MASK(x) (BIT(x) - 1)
 
@@ -182,6 +196,13 @@
 			TEGRA_DIVIDER_ROUND_UP, _clk_num, 0, _clk_id,\
 			_parents##_idx, 0, NULL)
 
+#define UART8(_name, _parents, _offset,\
+			     _clk_num, _clk_id)			\
+	TEGRA_INIT_DATA_TABLE(_name, NULL, NULL, _parents, _offset,\
+			29, MASK(3), 0, 0, 16, 1, TEGRA_DIVIDER_UART| \
+			TEGRA_DIVIDER_ROUND_UP, _clk_num, 0, _clk_id,\
+			_parents##_idx, 0, NULL)
+
 #define I2C(_name, _parents, _offset,\
 			     _clk_num, _clk_id)			\
 	TEGRA_INIT_DATA_TABLE(_name, NULL, NULL, _parents, _offset,\
@@ -221,8 +242,21 @@
 		.flags = _flags						\
 	}
 
+#define DIV8(_name, _parent_name, _offset, _clk_id, _flags)		\
+	{								\
+		.name = _name,						\
+		.clk_id = _clk_id,					\
+		.p.parent_name = _parent_name,				\
+		.periph = TEGRA_CLK_PERIPH(0, 0, 0, 0, 8, 1,		\
+				TEGRA_DIVIDER_ROUND_UP, 0, 0,		\
+				NULL, NULL),				\
+		.offset = _offset,					\
+		.flags = _flags,					\
+	}
+
 #define PLLP_BASE 0xa0
 #define PLLP_MISC 0xac
+#define PLLP_MISC1 0x680
 #define PLLP_OUTA 0xa4
 #define PLLP_OUTB 0xa8
 #define PLLP_OUTC 0x67c
@@ -234,6 +268,7 @@ static DEFINE_SPINLOCK(PLLP_OUTA_lock);
 static DEFINE_SPINLOCK(PLLP_OUTB_lock);
 static DEFINE_SPINLOCK(PLLP_OUTC_lock);
 static DEFINE_SPINLOCK(sor0_lock);
+static DEFINE_SPINLOCK(sor1_lock);
 
 #define MUX_I2S_SPDIF(_id)						\
 static const char *mux_pllaout0_##_id##_2x_pllp_clkm[] = { "pll_a_out0", \
@@ -285,6 +320,68 @@ static u32 mux_pllp_clkm_idx[] = {
 	[0] = 0, [1] = 3,
 };
 
+static const char *mux_pllp_clkm_2[] = {
+	"pll_p", "clk_m"
+};
+static u32 mux_pllp_clkm_2_idx[] = {
+	[0] = 2, [1] = 6,
+};
+
+static const char *mux_pllc2_c_c3_pllp_plla1_clkm[] = {
+	"pll_c2", "pll_c", "pll_c3", "pll_p", "pll_a1", "clk_m"
+};
+static u32 mux_pllc2_c_c3_pllp_plla1_clkm_idx[] = {
+	[0] = 1, [1] = 2, [2] = 3, [3] = 4, [4] = 6, [5] = 7,
+};
+
+static const char *
+mux_pllc4_out1_pllc_pllc4_out2_pllp_clkm_plla_pllc4_out0[] = {
+	"pll_c4_out1", "pll_c", "pll_c4_out2", "pll_p", "clk_m",
+	"pll_a_out0", "pll_c4_out0"
+};
+static u32 mux_pllc4_out1_pllc_pllc4_out2_pllp_clkm_plla_pllc4_out0_idx[] = {
+	[0] = 0, [1] = 2, [2] = 3, [3] = 4, [4] = 5, [5] = 6, [6] = 7,
+};
+
+static const char *mux_pllc_pllp_plla[] = {
+	"pll_c", "pll_p", "pll_a_out0"
+};
+static u32 mux_pllc_pllp_plla_idx[] = {
+	[0] = 1, [1] = 2, [2] = 3,
+};
+
+static const char *mux_clkm_pllc_pllp_plla[] = {
+	"clk_m", "pll_c", "pll_p", "pll_a_out0"
+};
+#define mux_clkm_pllc_pllp_plla_idx NULL
+
+static const char *mux_pllc_pllp_plla1_pllc2_c3_clkm[] = {
+	"pll_c", "pll_p", "pll_a1", "pll_c2", "pll_c3", "clk_m"
+};
+static u32 mux_pllc_pllp_plla1_pllc2_c3_clkm_idx[] = {
+	[0] = 1, [1] = 2, [2] = 3, [3] = 4, [4] = 5, [5] = 6,
+};
+
+static const char *mux_pllc2_c_c3_pllp_clkm_plla1_pllc4[] = {
+	"pll_c2", "pll_c", "pll_c3", "pll_p", "clk_m", "pll_a1", "pll_c4_out0",
+};
+static u32 mux_pllc2_c_c3_pllp_clkm_plla1_pllc4_idx[] = {
+	[0] = 1, [1] = 2, [2] = 3, [3] = 4, [4] = 5, [5] = 6, [6] = 7,
+};
+
+static const char *mux_pllc_pllp_plla1_pllc2_c3_clkm_pllc4[] = {
+	"pll_c", "pll_p", "pll_a1", "pll_c2", "pll_c3", "clk_m", "pll_c4_out0",
+};
+#define mux_pllc_pllp_plla1_pllc2_c3_clkm_pllc4_idx \
+	mux_pllc2_c_c3_pllp_clkm_plla1_pllc4_idx
+
+static const char *
+mux_plla_pllc4_out0_pllc_pllc4_out1_pllp_pllc4_out2_clkm[] = {
+	"pll_a_out0", "pll_c4_out0", "pll_c", "pll_c4_out1", "pll_p",
+	"pll_c4_out2", "clk_m"
+};
+#define mux_plla_pllc4_out0_pllc_pllc4_out1_pllp_pllc4_out2_clkm_idx NULL
+
 static const char *mux_pllm_pllc2_c_c3_pllp_plla[] = {
 	"pll_m", "pll_c2", "pll_c", "pll_c3", "pll_p", "pll_a_out0"
 };
@@ -302,12 +399,93 @@ static const char *mux_pllm_pllc_pllp_plla[] = {
 #define mux_pllm_pllc_pllp_plla_idx mux_pllp_pllc_pllm_clkm_idx
 
 static const char *mux_pllp_pllc_clkm[] = {
-	"pll_p", "pll_c", "pll_m"
+	"pll_p", "pll_c", "clk_m"
 };
 static u32 mux_pllp_pllc_clkm_idx[] = {
 	[0] = 0, [1] = 1, [2] = 3,
 };
 
+static const char *mux_pllp_pllc_clkm_1[] = {
+	"pll_p", "pll_c", "clk_m"
+};
+static u32 mux_pllp_pllc_clkm_1_idx[] = {
+	[0] = 0, [1] = 2, [2] = 5,
+};
+
+static const char *mux_pllp_pllc_plla_clkm[] = {
+	"pll_p", "pll_c", "pll_a_out0", "clk_m"
+};
+static u32 mux_pllp_pllc_plla_clkm_idx[] = {
+	[0] = 0, [1] = 2, [2] = 4, [3] = 6,
+};
+
+static const char *mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2[] = {
+	"pll_p", "pll_c", "pll_c4_out0", "pll_c4_out1", "clk_m", "pll_c4_out2"
+};
+static u32 mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2_idx[] = {
+	[0] = 0, [1] = 2, [2] = 3, [3] = 5, [4] = 6, [5] = 7,
+};
+
+static const char *
+mux_pllp_pllc_pllc_out1_pllc4_out2_pllc4_out1_clkm_pllc4_out0[] = {
+	"pll_p", "pll_c_out1", "pll_c", "pll_c4_out2", "pll_c4_out1",
+	"clk_m", "pll_c4_out0"
+};
+static u32
+mux_pllp_pllc_pllc_out1_pllc4_out2_pllc4_out1_clkm_pllc4_out0_idx[] = {
+	[0] = 0, [1] = 1, [2] = 2, [3] = 4, [4] = 5, [5] = 6, [6] = 7,
+};
+
+static const char *mux_pllp_pllc4_out2_pllc4_out1_clkm_pllc4_out0[] = {
+	"pll_p", "pll_c4_out2", "pll_c4_out1", "clk_m", "pll_c4_out0"
+};
+static u32 mux_pllp_pllc4_out2_pllc4_out1_clkm_pllc4_out0_idx[] = {
+	[0] = 0, [1] = 3, [2] = 4, [3] = 6, [4] = 7,
+};
+
+static const char *mux_pllp_clkm_pllc4_out2_out1_out0_lj[] = {
+	"pll_p",
+	"pll_c4_out2", "pll_c4_out0",	/* LJ input */
+	"pll_c4_out2", "pll_c4_out1",
+	"pll_c4_out1",			/* LJ input */
+	"clk_m", "pll_c4_out0"
+};
+#define mux_pllp_clkm_pllc4_out2_out1_out0_lj_idx NULL
+
+static const char *mux_pllp_pllc2_c_c3_clkm[] = {
+	"pll_p", "pll_c2", "pll_c", "pll_c3", "clk_m"
+};
+static u32 mux_pllp_pllc2_c_c3_clkm_idx[] = {
+	[0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 6,
+};
+
+static const char *mux_pllp_clkm_clk32_plle[] = {
+	"pll_p", "clk_m", "clk_32k", "pll_e"
+};
+static u32 mux_pllp_clkm_clk32_plle_idx[] = {
+	[0] = 0, [1] = 2, [2] = 4, [3] = 6,
+};
+
+static const char *mux_pllp_pllp_out3_clkm_clk32k_plla[] = {
+	"pll_p", "pll_p_out3", "clk_m", "clk_32k", "pll_a_out0"
+};
+#define mux_pllp_pllp_out3_clkm_clk32k_plla_idx NULL
+
+static const char *mux_pllp_out3_clkm_pllp_pllc4[] = {
+	"pll_p_out3", "clk_m", "pll_p", "pll_c4_out0", "pll_c4_out1",
+	"pll_c4_out2"
+};
+static u32 mux_pllp_out3_clkm_pllp_pllc4_idx[] = {
+	[0] = 0, [1] = 3, [2] = 4, [3] = 5, [4] = 6, [5] = 7,
+};
+
+static const char *mux_clkm_pllp_pllre[] = {
+	"clk_m", "pll_p_out_xusb", "pll_re_out"
+};
+static u32 mux_clkm_pllp_pllre_idx[] = {
+	[0] = 0, [1] = 1, [2] = 5,
+};
+
 static const char *mux_pllp_pllc_clkm_clk32[] = {
 	"pll_p", "pll_c", "clk_m", "clk_32k"
 };
@@ -332,6 +510,11 @@ static u32 mux_clkm_48M_pllp_480M_idx[] = {
 	[0] = 0, [1] = 2, [2] = 4, [3] = 6,
 };
 
+static const char *mux_clkm_pllre_clk32_480M[] = {
+	"clk_m", "pll_re_out", "clk_32k", "pll_u_480M"
+};
+#define mux_clkm_pllre_clk32_480M_idx NULL
+
 static const char *mux_clkm_pllre_clk32_480M_pllc_ref[] = {
 	"clk_m", "pll_re_out", "clk_32k", "pll_u_480M", "pll_c", "pll_ref"
 };
@@ -339,10 +522,27 @@ static u32 mux_clkm_pllre_clk32_480M_pllc_ref_idx[] = {
 	[0] = 0, [1] = 1, [2] = 3, [3] = 3, [4] = 4, [5] = 7,
 };
 
-static const char *mux_ss_60M[] = {
+static const char *mux_pllp_out3_pllp_pllc_clkm[] = {
+	"pll_p_out3", "pll_p", "pll_c", "clk_m"
+};
+static u32 mux_pllp_out3_pllp_pllc_clkm_idx[] = {
+	[0] = 0, [1] = 1, [2] = 2, [3] = 6,
+};
+
+static const char *mux_ss_div2_60M[] = {
 	"xusb_ss_div2", "pll_u_60M"
 };
-#define mux_ss_60M_idx NULL
+#define mux_ss_div2_60M_idx NULL
+
+static const char *mux_ss_div2_60M_ss[] = {
+	"xusb_ss_div2", "pll_u_60M", "xusb_ss_src"
+};
+#define mux_ss_div2_60M_ss_idx NULL
+
+static const char *mux_ss_clkm[] = {
+	"xusb_ss_src", "clk_m"
+};
+#define mux_ss_clkm_idx NULL
 
 static const char *mux_d_audio_clk[] = {
 	"pll_a_out0", "pll_p", "clk_m", "spdif_in_sync", "i2s0_sync",
@@ -386,6 +586,32 @@ static u32 mux_pllm_pllc2_c_c3_pllp_plla_pllc4_idx[] = {
 	[0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 4, [5] = 6, [6] = 7,
 };
 
+/* SOR1 mux'es */
+static const char *mux_pllp_plld_plld2_clkm[] = {
+	"pll_p", "pll_d_out0", "pll_d2_out0", "clk_m"
+};
+static u32 mux_pllp_plld_plld2_clkm_idx[] = {
+	[0] = 0, [1] = 2, [2] = 5, [3] = 6
+};
+
+static const char *mux_plldp_sor1_src[] = {
+	"pll_dp", "clk_sor1_src"
+};
+#define mux_plldp_sor1_src_idx NULL
+
+static const char *mux_clkm_sor1_brick_sor1_src[] = {
+	"clk_m", "sor1_brick", "sor1_src", "sor1_brick"
+};
+#define mux_clkm_sor1_brick_sor1_src_idx NULL
+
+static const char *mux_pllp_pllre_clkm[] = {
+	"pll_p", "pll_re_out1", "clk_m"
+};
+
+static u32 mux_pllp_pllre_clkm_idx[] = {
+	[0] = 0, [1] = 2, [2] = 3,
+};
+
 static const char *mux_clkm_plldp_sor0lvds[] = {
 	"clk_m", "pll_dp", "sor0_lvds",
 };
@@ -401,6 +627,7 @@ static struct tegra_periph_init_data periph_clks[] = {
 	I2C("i2c3", mux_pllp_clkm, CLK_SOURCE_I2C3, 67, tegra_clk_i2c3),
 	I2C("i2c4", mux_pllp_clkm, CLK_SOURCE_I2C4, 103, tegra_clk_i2c4),
 	I2C("i2c5", mux_pllp_clkm, CLK_SOURCE_I2C5, 47, tegra_clk_i2c5),
+	I2C("i2c6", mux_pllp_clkm, CLK_SOURCE_I2C6, 166, tegra_clk_i2c6),
 	INT("vde", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_VDE, 61, 0, tegra_clk_vde),
 	INT("vi", mux_pllm_pllc_pllp_plla, CLK_SOURCE_VI, 20, 0, tegra_clk_vi),
 	INT("epp", mux_pllm_pllc_pllp_plla, CLK_SOURCE_EPP, 19, 0, tegra_clk_epp),
@@ -411,14 +638,19 @@ static struct tegra_periph_init_data periph_clks[] = {
 	INT8("vde", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_VDE, 61, 0, tegra_clk_vde_8),
 	INT8("vi", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI, 20, 0, tegra_clk_vi_8),
 	INT8("vi", mux_pllm_pllc2_c_c3_pllp_plla_pllc4, CLK_SOURCE_VI, 20, 0, tegra_clk_vi_9),
+	INT8("vi", mux_pllc2_c_c3_pllp_clkm_plla1_pllc4, CLK_SOURCE_VI, 20, 0, tegra_clk_vi_10),
 	INT8("epp", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_EPP, 19, 0, tegra_clk_epp_8),
 	INT8("msenc", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_MSENC, 91, TEGRA_PERIPH_WAR_1005168, tegra_clk_msenc),
 	INT8("tsec", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_TSEC, 83, 0, tegra_clk_tsec),
+	INT("tsec", mux_pllp_pllc_clkm, CLK_SOURCE_TSEC, 83, 0, tegra_clk_tsec_8),
 	INT8("host1x", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_HOST1X, 28, 0, tegra_clk_host1x_8),
+	INT8("host1x", mux_pllc4_out1_pllc_pllc4_out2_pllp_clkm_plla_pllc4_out0, CLK_SOURCE_HOST1X, 28, 0, tegra_clk_host1x_9),
 	INT8("se", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SE, 127, TEGRA_PERIPH_ON_APB, tegra_clk_se),
+	INT8("se", mux_pllp_pllc2_c_c3_clkm, CLK_SOURCE_SE, 127, TEGRA_PERIPH_ON_APB, tegra_clk_se),
 	INT8("2d", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_2D, 21, 0, tegra_clk_gr2d_8),
 	INT8("3d", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_3D, 24, 0, tegra_clk_gr3d_8),
 	INT8("vic03", mux_pllm_pllc_pllp_plla_pllc2_c3_clkm, CLK_SOURCE_VIC03, 178, 0, tegra_clk_vic03),
+	INT8("vic03", mux_pllc_pllp_plla1_pllc2_c3_clkm, CLK_SOURCE_VIC03, 178, 0, tegra_clk_vic03_8),
 	INT_FLAGS("mselect", mux_pllp_clkm, CLK_SOURCE_MSELECT, 99, 0, tegra_clk_mselect, CLK_IGNORE_UNUSED),
 	MUX("i2s0", mux_pllaout0_audio0_2x_pllp_clkm, CLK_SOURCE_I2S0, 30, TEGRA_PERIPH_ON_APB, tegra_clk_i2s0),
 	MUX("i2s1", mux_pllaout0_audio1_2x_pllp_clkm, CLK_SOURCE_I2S1, 11, TEGRA_PERIPH_ON_APB, tegra_clk_i2s1),
@@ -427,22 +659,31 @@ static struct tegra_periph_init_data periph_clks[] = {
 	MUX("i2s4", mux_pllaout0_audio4_2x_pllp_clkm, CLK_SOURCE_I2S4, 102, TEGRA_PERIPH_ON_APB, tegra_clk_i2s4),
 	MUX("spdif_out", mux_pllaout0_audio_2x_pllp_clkm, CLK_SOURCE_SPDIF_OUT, 10, TEGRA_PERIPH_ON_APB, tegra_clk_spdif_out),
 	MUX("spdif_in", mux_pllp_pllc_pllm, CLK_SOURCE_SPDIF_IN, 10, TEGRA_PERIPH_ON_APB, tegra_clk_spdif_in),
+	MUX8("spdif_in", mux_pllp_pllc_clkm_1, CLK_SOURCE_SPDIF_IN, 10, TEGRA_PERIPH_ON_APB, tegra_clk_spdif_in_8),
 	MUX("pwm", mux_pllp_pllc_clk32_clkm, CLK_SOURCE_PWM, 17, TEGRA_PERIPH_ON_APB, tegra_clk_pwm),
 	MUX("adx", mux_plla_pllc_pllp_clkm, CLK_SOURCE_ADX, 154, TEGRA_PERIPH_ON_APB, tegra_clk_adx),
 	MUX("amx", mux_plla_pllc_pllp_clkm, CLK_SOURCE_AMX, 153, TEGRA_PERIPH_ON_APB, tegra_clk_amx),
 	MUX("hda", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_HDA, 125, TEGRA_PERIPH_ON_APB, tegra_clk_hda),
+	MUX("hda", mux_pllp_pllc_clkm, CLK_SOURCE_HDA, 125, TEGRA_PERIPH_ON_APB, tegra_clk_hda_8),
 	MUX("hda2codec_2x", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_HDA2CODEC_2X, 111, TEGRA_PERIPH_ON_APB, tegra_clk_hda2codec_2x),
+	MUX8("hda2codec_2x", mux_pllp_pllc_plla_clkm, CLK_SOURCE_HDA2CODEC_2X, 111, TEGRA_PERIPH_ON_APB, tegra_clk_hda2codec_2x_8),
 	MUX("vfir", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_VFIR, 7, TEGRA_PERIPH_ON_APB, tegra_clk_vfir),
 	MUX("sdmmc1", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SDMMC1, 14, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc1),
 	MUX("sdmmc2", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SDMMC2, 9, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc2),
 	MUX("sdmmc3", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SDMMC3, 69, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc3),
 	MUX("sdmmc4", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SDMMC4, 15, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc4),
+	MUX8("sdmmc1", mux_pllp_pllc4_out2_pllc4_out1_clkm_pllc4_out0, CLK_SOURCE_SDMMC1, 14, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc1_9),
+	MUX8("sdmmc2", mux_pllp_clkm_pllc4_out2_out1_out0_lj, CLK_SOURCE_SDMMC2, 9, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc2_9),
+	MUX8("sdmmc3", mux_pllp_pllc4_out2_pllc4_out1_clkm_pllc4_out0, CLK_SOURCE_SDMMC3, 69, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc3_9),
+	MUX8("sdmmc4", mux_pllp_clkm_pllc4_out2_out1_out0_lj, CLK_SOURCE_SDMMC4, 15, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc4_9),
 	MUX("la", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_LA, 76, TEGRA_PERIPH_ON_APB, tegra_clk_la),
 	MUX("trace", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_TRACE, 77, TEGRA_PERIPH_ON_APB, tegra_clk_trace),
 	MUX("owr", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_OWR, 71, TEGRA_PERIPH_ON_APB, tegra_clk_owr),
+	MUX("owr", mux_pllp_pllc_clkm, CLK_SOURCE_OWR, 71, TEGRA_PERIPH_ON_APB, tegra_clk_owr_8),
 	MUX("nor", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_NOR, 42, 0, tegra_clk_nor),
 	MUX("mipi", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_MIPI, 50, TEGRA_PERIPH_ON_APB, tegra_clk_mipi),
 	MUX("vi_sensor", mux_pllm_pllc_pllp_plla, CLK_SOURCE_VI_SENSOR, 20, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor),
+	MUX("vi_sensor", mux_pllc_pllp_plla, CLK_SOURCE_VI_SENSOR, 20, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor_9),
 	MUX("cilab", mux_pllp_pllc_clkm, CLK_SOURCE_CILAB, 144, 0, tegra_clk_cilab),
 	MUX("cilcd", mux_pllp_pllc_clkm, CLK_SOURCE_CILCD, 145, 0, tegra_clk_cilcd),
 	MUX("cile", mux_pllp_pllc_clkm, CLK_SOURCE_CILE, 146, 0, tegra_clk_cile),
@@ -465,10 +706,13 @@ static struct tegra_periph_init_data periph_clks[] = {
 	MUX("ndflash", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_NDFLASH, 13, TEGRA_PERIPH_ON_APB, tegra_clk_ndflash),
 	MUX("ndspeed", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_NDSPEED, 80, TEGRA_PERIPH_ON_APB, tegra_clk_ndspeed),
 	MUX("sata_oob", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SATA_OOB, 123, TEGRA_PERIPH_ON_APB, tegra_clk_sata_oob),
+	MUX("sata_oob", mux_pllp_pllc_clkm, CLK_SOURCE_SATA_OOB, 123, TEGRA_PERIPH_ON_APB, tegra_clk_sata_oob_8),
 	MUX("sata", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SATA, 124, TEGRA_PERIPH_ON_APB, tegra_clk_sata),
+	MUX("sata", mux_pllp_pllc_clkm, CLK_SOURCE_SATA, 124, TEGRA_PERIPH_ON_APB, tegra_clk_sata_8),
 	MUX("adx1", mux_plla_pllc_pllp_clkm, CLK_SOURCE_ADX1, 180, TEGRA_PERIPH_ON_APB, tegra_clk_adx1),
 	MUX("amx1", mux_plla_pllc_pllp_clkm, CLK_SOURCE_AMX1, 185, TEGRA_PERIPH_ON_APB, tegra_clk_amx1),
 	MUX("vi_sensor2", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR2, 165, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor2),
+	MUX("vi_sensor2", mux_pllc_pllp_plla, CLK_SOURCE_VI_SENSOR2, 165, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor2_8),
 	MUX8("sdmmc1", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC1, 14, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc1_8),
 	MUX8("sdmmc2", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC2, 9, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc2_8),
 	MUX8("sdmmc3", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC3, 69, TEGRA_PERIPH_ON_APB, tegra_clk_sdmmc3_8),
@@ -479,6 +723,10 @@ static struct tegra_periph_init_data periph_clks[] = {
 	MUX8("sbc4", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SBC4, 68, TEGRA_PERIPH_ON_APB, tegra_clk_sbc4_8),
 	MUX8("sbc5", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SBC5, 104, TEGRA_PERIPH_ON_APB, tegra_clk_sbc5_8),
 	MUX8("sbc6", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SBC6, 105, TEGRA_PERIPH_ON_APB, tegra_clk_sbc6_8),
+	MUX("sbc1", mux_pllp_pllc_clkm, CLK_SOURCE_SBC1, 41, TEGRA_PERIPH_ON_APB, tegra_clk_sbc1_9),
+	MUX("sbc2", mux_pllp_pllc_clkm, CLK_SOURCE_SBC2, 44, TEGRA_PERIPH_ON_APB, tegra_clk_sbc2_9),
+	MUX("sbc3", mux_pllp_pllc_clkm, CLK_SOURCE_SBC3, 46, TEGRA_PERIPH_ON_APB, tegra_clk_sbc3_9),
+	MUX("sbc4", mux_pllp_pllc_clkm, CLK_SOURCE_SBC4, 68, TEGRA_PERIPH_ON_APB, tegra_clk_sbc4_9),
 	MUX8("ndflash", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_NDFLASH, 13, TEGRA_PERIPH_ON_APB, tegra_clk_ndflash_8),
 	MUX8("ndspeed", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_NDSPEED, 80, TEGRA_PERIPH_ON_APB, tegra_clk_ndspeed_8),
 	MUX8("hdmi", mux_pllp_pllm_plld_plla_pllc_plld2_clkm, CLK_SOURCE_HDMI, 51, 0, tegra_clk_hdmi),
@@ -486,27 +734,59 @@ static struct tegra_periph_init_data periph_clks[] = {
 	MUX8("extern2", mux_plla_clk32_pllp_clkm_plle, CLK_SOURCE_EXTERN2, 121, 0, tegra_clk_extern2),
 	MUX8("extern3", mux_plla_clk32_pllp_clkm_plle, CLK_SOURCE_EXTERN3, 122, 0, tegra_clk_extern3),
 	MUX8("soc_therm", mux_pllm_pllc_pllp_plla, CLK_SOURCE_SOC_THERM, 78, TEGRA_PERIPH_ON_APB, tegra_clk_soc_therm),
+	MUX8("soc_therm", mux_clkm_pllc_pllp_plla, CLK_SOURCE_SOC_THERM, 78, TEGRA_PERIPH_ON_APB, tegra_clk_soc_therm_8),
 	MUX8("vi_sensor", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR, 164, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor_8),
 	MUX8("isp", mux_pllm_pllc_pllp_plla_clkm_pllc4, CLK_SOURCE_ISP, 23, TEGRA_PERIPH_ON_APB, tegra_clk_isp_8),
+	MUX8("isp", mux_pllc_pllp_plla1_pllc2_c3_clkm_pllc4, CLK_SOURCE_ISP, 23, TEGRA_PERIPH_ON_APB, tegra_clk_isp_9),
 	MUX8("entropy", mux_pllp_clkm1, CLK_SOURCE_ENTROPY, 149,  0, tegra_clk_entropy),
+	MUX8("entropy", mux_pllp_clkm_clk32_plle, CLK_SOURCE_ENTROPY, 149,  0, tegra_clk_entropy_8),
 	MUX8("hdmi_audio", mux_pllp3_pllc_clkm, CLK_SOURCE_HDMI_AUDIO, 176, TEGRA_PERIPH_NO_RESET, tegra_clk_hdmi_audio),
 	MUX8("clk72mhz", mux_pllp3_pllc_clkm, CLK_SOURCE_CLK72MHZ, 177, TEGRA_PERIPH_NO_RESET, tegra_clk_clk72Mhz),
+	MUX8("clk72mhz", mux_pllp_out3_pllp_pllc_clkm, CLK_SOURCE_CLK72MHZ, 177, TEGRA_PERIPH_NO_RESET, tegra_clk_clk72Mhz_8),
 	MUX8_NOGATE_LOCK("sor0_lvds", mux_pllp_pllm_plld_plla_pllc_plld2_clkm, CLK_SOURCE_SOR0, tegra_clk_sor0_lvds, &sor0_lock),
 	MUX_FLAGS("csite", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_CSITE, 73, TEGRA_PERIPH_ON_APB, tegra_clk_csite, CLK_IGNORE_UNUSED),
+	MUX_FLAGS("csite", mux_pllp_pllre_clkm, CLK_SOURCE_CSITE, 73, TEGRA_PERIPH_ON_APB, tegra_clk_csite_8, CLK_IGNORE_UNUSED),
 	NODIV("disp1", mux_pllp_pllm_plld_plla_pllc_plld2_clkm, CLK_SOURCE_DISP1, 29, 7, 27, 0, tegra_clk_disp1, NULL),
+	NODIV("disp1", mux_pllp_plld_plld2_clkm, CLK_SOURCE_DISP1, 29, 7, 27, 0, tegra_clk_disp1_8, NULL),
 	NODIV("disp2", mux_pllp_pllm_plld_plla_pllc_plld2_clkm, CLK_SOURCE_DISP2, 29, 7, 26, 0, tegra_clk_disp2, NULL),
+	NODIV("disp2", mux_pllp_plld_plld2_clkm, CLK_SOURCE_DISP2, 29, 7, 26, 0, tegra_clk_disp2_8, NULL),
 	NODIV("sor0", mux_clkm_plldp_sor0lvds, CLK_SOURCE_SOR0, 14, 3, 182, 0, tegra_clk_sor0, &sor0_lock),
 	UART("uarta", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_UARTA, 6, tegra_clk_uarta),
 	UART("uartb", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_UARTB, 7, tegra_clk_uartb),
 	UART("uartc", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_UARTC, 55, tegra_clk_uartc),
 	UART("uartd", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_UARTD, 65, tegra_clk_uartd),
 	UART("uarte", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_UARTE, 66, tegra_clk_uarte),
+	UART8("uarta", mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2, CLK_SOURCE_UARTA, 6, tegra_clk_uarta_8),
+	UART8("uartb", mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2, CLK_SOURCE_UARTB, 7, tegra_clk_uartb_8),
+	UART8("uartc", mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2, CLK_SOURCE_UARTC, 55, tegra_clk_uartc_8),
+	UART8("uartd", mux_pllp_pllc_pllc4_out0_pllc4_out1_clkm_pllc4_out2, CLK_SOURCE_UARTD, 65, tegra_clk_uartd_8),
 	XUSB("xusb_host_src", mux_clkm_pllp_pllc_pllre, CLK_SOURCE_XUSB_HOST_SRC, 143, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_host_src),
+	XUSB("xusb_host_src", mux_clkm_pllp_pllre, CLK_SOURCE_XUSB_HOST_SRC, 143, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_host_src_8),
 	XUSB("xusb_falcon_src", mux_clkm_pllp_pllc_pllre, CLK_SOURCE_XUSB_FALCON_SRC, 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_falcon_src),
+	XUSB("xusb_falcon_src", mux_clkm_pllp_pllre, CLK_SOURCE_XUSB_FALCON_SRC, 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_falcon_src_8),
 	XUSB("xusb_fs_src", mux_clkm_48M_pllp_480M, CLK_SOURCE_XUSB_FS_SRC, 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_fs_src),
 	XUSB("xusb_ss_src", mux_clkm_pllre_clk32_480M_pllc_ref, CLK_SOURCE_XUSB_SS_SRC, 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_ss_src),
-	NODIV("xusb_hs_src", mux_ss_60M, CLK_SOURCE_XUSB_SS_SRC, 25, MASK(1), 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_hs_src, NULL),
+	XUSB("xusb_ss_src", mux_clkm_pllre_clk32_480M, CLK_SOURCE_XUSB_SS_SRC, 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_ss_src_8),
+	NODIV("xusb_hs_src", mux_ss_div2_60M, CLK_SOURCE_XUSB_SS_SRC, 25, MASK(1), 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_hs_src, NULL),
+	NODIV("xusb_hs_src", mux_ss_div2_60M_ss, CLK_SOURCE_XUSB_SS_SRC, 25, MASK(2), 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_hs_src_4, NULL),
+	NODIV("xusb_ssp_src", mux_ss_clkm, CLK_SOURCE_XUSB_SS_SRC, 24, MASK(1), 143, TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_ssp_src, NULL),
 	XUSB("xusb_dev_src", mux_clkm_pllp_pllc_pllre, CLK_SOURCE_XUSB_DEV_SRC, 95, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_dev_src),
+	XUSB("xusb_dev_src", mux_clkm_pllp_pllre, CLK_SOURCE_XUSB_DEV_SRC, 95, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_xusb_dev_src_8),
+	MUX8("dbgapb", mux_pllp_clkm_2, CLK_SOURCE_DBGAPB, 185, TEGRA_PERIPH_NO_RESET, tegra_clk_dbgapb),
+	MUX8("msenc", mux_pllc2_c_c3_pllp_plla1_clkm, CLK_SOURCE_NVENC, 219, 0, tegra_clk_nvenc),
+	MUX8("nvdec", mux_pllc2_c_c3_pllp_plla1_clkm, CLK_SOURCE_NVDEC, 194, 0, tegra_clk_nvdec),
+	MUX8("nvjpg", mux_pllc2_c_c3_pllp_plla1_clkm, CLK_SOURCE_NVJPG, 195, 0, tegra_clk_nvjpg),
+	MUX8("ape", mux_plla_pllc4_out0_pllc_pllc4_out1_pllp_pllc4_out2_clkm, CLK_SOURCE_APE, 198, TEGRA_PERIPH_ON_APB, tegra_clk_ape),
+	MUX8_NOGATE_LOCK("sor1_src", mux_pllp_plld_plld2_clkm, CLK_SOURCE_SOR1, tegra_clk_sor1_src, &sor1_lock),
+	NODIV("sor1_brick", mux_plldp_sor1_src, CLK_SOURCE_SOR1, 14, MASK(1), 183, 0, tegra_clk_sor1_brick, &sor1_lock),
+	NODIV("sor1", mux_clkm_sor1_brick_sor1_src, CLK_SOURCE_SOR1, 15, MASK(1), 183, 0, tegra_clk_sor1, &sor1_lock),
+	MUX8("sdmmc_legacy", mux_pllp_out3_clkm_pllp_pllc4, CLK_SOURCE_SDMMC_LEGACY, 193, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_sdmmc_legacy),
+	MUX8("qspi", mux_pllp_pllc_pllc_out1_pllc4_out2_pllc4_out1_clkm_pllc4_out0, CLK_SOURCE_QSPI, 211, TEGRA_PERIPH_ON_APB, tegra_clk_qspi),
+	MUX("vii2c", mux_pllp_pllc_clkm, CLK_SOURCE_VI_I2C, 208, TEGRA_PERIPH_ON_APB, tegra_clk_vi_i2c),
+	MUX("mipibif", mux_pllp_clkm, CLK_SOURCE_MIPIBIF, 173, TEGRA_PERIPH_ON_APB, tegra_clk_mipibif),
+	MUX("uartape", mux_pllp_pllc_clkm, CLK_SOURCE_UARTAPE, 212, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_uartape),
+	MUX8("tsecb", mux_pllp_pllc2_c_c3_clkm, CLK_SOURCE_TSECB, 206, 0, tegra_clk_tsecb),
+	MUX8("maud", mux_pllp_pllp_out3_clkm_clk32k_plla, CLK_SOURCE_MAUD, 202, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_maud),
 };
 
 static struct tegra_periph_init_data gate_clks[] = {
@@ -543,6 +823,16 @@ static struct tegra_periph_init_data gate_clks[] = {
 	GATE("pcie", "clk_m", 70, 0, tegra_clk_pcie, 0),
 	GATE("dpaux", "clk_m", 181, 0, tegra_clk_dpaux, 0),
 	GATE("gpu", "pll_ref", 184, 0, tegra_clk_gpu, 0),
+	GATE("pllg_ref", "pll_ref", 189, 0, tegra_clk_pll_g_ref, 0),
+	GATE("hsic_trk", "usb2_hsic_trk", 209, TEGRA_PERIPH_NO_RESET, tegra_clk_hsic_trk, 0),
+	GATE("usb2_trk", "usb2_hsic_trk", 210, TEGRA_PERIPH_NO_RESET, tegra_clk_usb2_trk, 0),
+	GATE("xusb_gate", "osc", 143, 0, tegra_clk_xusb_gate, 0),
+	GATE("pll_p_out_cpu", "pll_p", 223, 0, tegra_clk_pll_p_out_cpu, 0),
+	GATE("pll_p_out_adsp", "pll_p", 187, 0, tegra_clk_pll_p_out_adsp, 0),
+};
+
+static struct tegra_periph_init_data div_clks[] = {
+	DIV8("usb2_hsic_trk", "osc", CLK_SOURCE_USB2_HSIC_TRK, tegra_clk_usb2_hsic_trk, 0),
 };
 
 struct pll_out_data {
@@ -633,6 +923,33 @@ static void __init gate_clk_init(void __iomem *clk_base,
 	}
 }
 
+static void __init div_clk_init(void __iomem *clk_base,
+				struct tegra_clk *tegra_clks)
+{
+	int i;
+	struct clk *clk;
+	struct clk **dt_clk;
+
+	for (i = 0; i < ARRAY_SIZE(div_clks); i++) {
+		struct tegra_periph_init_data *data;
+
+		data = div_clks + i;
+
+		dt_clk = tegra_lookup_dt_id(data->clk_id, tegra_clks);
+		if (!dt_clk)
+			continue;
+
+		clk = tegra_clk_register_divider(data->name,
+				data->p.parent_name, clk_base + data->offset,
+				data->flags, data->periph.divider.flags,
+				data->periph.divider.shift,
+				data->periph.divider.width,
+				data->periph.divider.frac_width,
+				data->periph.divider.lock);
+		*dt_clk = clk;
+	}
+}
+
 static void __init init_pllp(void __iomem *clk_base, void __iomem *pmc_base,
 				struct tegra_clk *tegra_clks,
 				struct tegra_clk_pll_params *pll_params)
@@ -669,6 +986,51 @@ static void __init init_pllp(void __iomem *clk_base, void __iomem *pmc_base,
 				data->lock);
 		*dt_clk = clk;
 	}
+
+	dt_clk = tegra_lookup_dt_id(tegra_clk_pll_p_out_cpu,
+			tegra_clks);
+	if (dt_clk) {
+		/*
+		 * Tegra210 has control on enabling/disabling PLLP branches to
+		 * CPU, register a gate clock "pll_p_out_cpu" for this gating
+		 * function and parent "pll_p_out4" to it, so when we are
+		 * re-parenting CPU off from "pll_p_out4" the PLLP branching to
+		 * CPU can be disabled automatically.
+		 */
+		clk = tegra_clk_register_divider("pll_p_out4_div",
+				"pll_p_out_cpu", clk_base + PLLP_OUTB, 0, 0, 24,
+				8, 1, &PLLP_OUTB_lock);
+
+		dt_clk = tegra_lookup_dt_id(tegra_clk_pll_p_out4_cpu, tegra_clks);
+		if (dt_clk) {
+			clk = tegra_clk_register_pll_out("pll_p_out4",
+					"pll_p_out4_div", clk_base + PLLP_OUTB,
+					17, 16, CLK_IGNORE_UNUSED |
+					CLK_SET_RATE_PARENT, 0,
+					&PLLP_OUTB_lock);
+			*dt_clk = clk;
+		}
+	}
+
+	dt_clk = tegra_lookup_dt_id(tegra_clk_pll_p_out_hsio, tegra_clks);
+	if (dt_clk) {
+		/* PLLP_OUT_HSIO */
+		clk = clk_register_gate(NULL, "pll_p_out_hsio", "pll_p",
+				CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+				clk_base + PLLP_MISC1, 29, 0, NULL);
+		*dt_clk = clk;
+	}
+
+	dt_clk = tegra_lookup_dt_id(tegra_clk_pll_p_out_xusb, tegra_clks);
+	if (dt_clk) {
+		/* PLLP_OUT_XUSB */
+		clk = clk_register_gate(NULL, "pll_p_out_xusb",
+				"pll_p_out_hsio", CLK_SET_RATE_PARENT |
+				CLK_IGNORE_UNUSED, clk_base + PLLP_MISC1, 28, 0,
+				NULL);
+		clk_register_clkdev(clk, "pll_p_out_xusb", NULL);
+		*dt_clk = clk;
+	}
 }
 
 void __init tegra_periph_clk_init(void __iomem *clk_base,
@@ -678,4 +1040,5 @@ void __init tegra_periph_clk_init(void __iomem *clk_base,
 	init_pllp(clk_base, pmc_base, tegra_clks, pll_params);
 	periph_clk_init(clk_base, tegra_clks);
 	gate_clk_init(clk_base, tegra_clks);
+	div_clk_init(clk_base, tegra_clks);
 }
diff --git a/drivers/clk/tegra/clk-tegra-super-gen4.c b/drivers/clk/tegra/clk-tegra-super-gen4.c
index 5b1d723932c5..4559a20e3af6 100644
--- a/drivers/clk/tegra/clk-tegra-super-gen4.c
+++ b/drivers/clk/tegra/clk-tegra-super-gen4.c
@@ -34,9 +34,25 @@
 #define CCLKLP_BURST_POLICY 0x370
 #define SCLK_BURST_POLICY 0x028
 #define SYSTEM_CLK_RATE 0x030
+#define SCLK_DIVIDER 0x2c
 
 static DEFINE_SPINLOCK(sysrate_lock);
 
+enum tegra_super_gen {
+	gen4 = 4,
+	gen5,
+};
+
+struct tegra_super_gen_info {
+	enum tegra_super_gen gen;
+	const char **sclk_parents;
+	const char **cclk_g_parents;
+	const char **cclk_lp_parents;
+	int num_sclk_parents;
+	int num_cclk_g_parents;
+	int num_cclk_lp_parents;
+};
+
 static const char *sclk_parents[] = { "clk_m", "pll_c_out1", "pll_p_out4",
 			       "pll_p", "pll_p_out2", "unused",
 			       "clk_32k", "pll_m_out1" };
@@ -51,21 +67,81 @@ static const char *cclk_lp_parents[] = { "clk_m", "pll_c", "clk_32k", "pll_m",
 					 "pll_p", "pll_p_out4", "unused",
 					 "unused", "pll_x", "pll_x_out0" };
 
+const struct tegra_super_gen_info tegra_super_gen_info_gen4 = {
+	.gen = gen4,
+	.sclk_parents = sclk_parents,
+	.cclk_g_parents = cclk_g_parents,
+	.cclk_lp_parents = cclk_lp_parents,
+	.num_sclk_parents = ARRAY_SIZE(sclk_parents),
+	.num_cclk_g_parents = ARRAY_SIZE(cclk_g_parents),
+	.num_cclk_lp_parents = ARRAY_SIZE(cclk_lp_parents),
+};
+
+static const char *sclk_parents_gen5[] = { "clk_m", "pll_c_out1", "pll_c4_out3",
+			       "pll_p", "pll_p_out2", "pll_c4_out1",
+			       "clk_32k", "pll_c4_out2" };
+
+static const char *cclk_g_parents_gen5[] = { "clk_m", "unused", "clk_32k", "unused",
+					"pll_p", "pll_p_out4", "unused",
+					"unused", "pll_x", "unused", "unused",
+					"unused", "unused", "unused", "unused",
+					"dfllCPU_out" };
+
+static const char *cclk_lp_parents_gen5[] = { "clk_m", "unused", "clk_32k", "unused",
+					"pll_p", "pll_p_out4", "unused",
+					"unused", "pll_x", "unused", "unused",
+					"unused", "unused", "unused", "unused",
+					"dfllCPU_out" };
+
+const struct tegra_super_gen_info tegra_super_gen_info_gen5 = {
+	.gen = gen5,
+	.sclk_parents = sclk_parents_gen5,
+	.cclk_g_parents = cclk_g_parents_gen5,
+	.cclk_lp_parents = cclk_lp_parents_gen5,
+	.num_sclk_parents = ARRAY_SIZE(sclk_parents_gen5),
+	.num_cclk_g_parents = ARRAY_SIZE(cclk_g_parents_gen5),
+	.num_cclk_lp_parents = ARRAY_SIZE(cclk_lp_parents_gen5),
+};
+
 static void __init tegra_sclk_init(void __iomem *clk_base,
-				struct tegra_clk *tegra_clks)
+				struct tegra_clk *tegra_clks,
+				const struct tegra_super_gen_info *gen_info)
 {
 	struct clk *clk;
 	struct clk **dt_clk;
 
-	/* SCLK */
-	dt_clk = tegra_lookup_dt_id(tegra_clk_sclk, tegra_clks);
+	/* SCLK_MUX */
+	dt_clk = tegra_lookup_dt_id(tegra_clk_sclk_mux, tegra_clks);
 	if (dt_clk) {
-		clk = tegra_clk_register_super_mux("sclk", sclk_parents,
-						ARRAY_SIZE(sclk_parents),
+		clk = tegra_clk_register_super_mux("sclk_mux",
+						gen_info->sclk_parents,
+						gen_info->num_sclk_parents,
 						CLK_SET_RATE_PARENT,
 						clk_base + SCLK_BURST_POLICY,
 						0, 4, 0, 0, NULL);
 		*dt_clk = clk;
+
+
+		/* SCLK */
+		dt_clk = tegra_lookup_dt_id(tegra_clk_sclk, tegra_clks);
+		if (dt_clk) {
+			clk = clk_register_divider(NULL, "sclk", "sclk_mux", 0,
+						clk_base + SCLK_DIVIDER, 0, 8,
+						0, &sysrate_lock);
+			*dt_clk = clk;
+		}
+	} else {
+		/* SCLK */
+		dt_clk = tegra_lookup_dt_id(tegra_clk_sclk, tegra_clks);
+		if (dt_clk) {
+			clk = tegra_clk_register_super_mux("sclk",
+						gen_info->sclk_parents,
+						gen_info->num_sclk_parents,
+						CLK_SET_RATE_PARENT,
+						clk_base + SCLK_BURST_POLICY,
+						0, 4, 0, 0, NULL);
+			*dt_clk = clk;
+		}
 	}
 
 	/* HCLK */
@@ -95,10 +171,11 @@ static void __init tegra_sclk_init(void __iomem *clk_base,
 	*dt_clk = clk;
 }
 
-void __init tegra_super_clk_gen4_init(void __iomem *clk_base,
+void __init tegra_super_clk_init(void __iomem *clk_base,
 				void __iomem *pmc_base,
 				struct tegra_clk *tegra_clks,
-				struct tegra_clk_pll_params *params)
+				struct tegra_clk_pll_params *params,
+				const struct tegra_super_gen_info *gen_info)
 {
 	struct clk *clk;
 	struct clk **dt_clk;
@@ -106,28 +183,50 @@ void __init tegra_super_clk_gen4_init(void __iomem *clk_base,
 	/* CCLKG */
 	dt_clk = tegra_lookup_dt_id(tegra_clk_cclk_g, tegra_clks);
 	if (dt_clk) {
-		clk = tegra_clk_register_super_mux("cclk_g", cclk_g_parents,
-					ARRAY_SIZE(cclk_g_parents),
+		if (gen_info->gen == gen5) {
+			clk = tegra_clk_register_super_mux("cclk_g",
+					gen_info->cclk_g_parents,
+					gen_info->num_cclk_g_parents,
+					CLK_SET_RATE_PARENT,
+					clk_base + CCLKG_BURST_POLICY,
+					0, 4, 8, 0, NULL);
+		} else {
+			clk = tegra_clk_register_super_mux("cclk_g",
+					gen_info->cclk_g_parents,
+					gen_info->num_cclk_g_parents,
 					CLK_SET_RATE_PARENT,
 					clk_base + CCLKG_BURST_POLICY,
 					0, 4, 0, 0, NULL);
+		}
 		*dt_clk = clk;
 	}
 
 	/* CCLKLP */
 	dt_clk = tegra_lookup_dt_id(tegra_clk_cclk_lp, tegra_clks);
 	if (dt_clk) {
-		clk = tegra_clk_register_super_mux("cclk_lp", cclk_lp_parents,
-					ARRAY_SIZE(cclk_lp_parents),
+		if (gen_info->gen == gen5) {
+			clk = tegra_clk_register_super_mux("cclk_lp",
+					gen_info->cclk_lp_parents,
+					gen_info->num_cclk_lp_parents,
+					CLK_SET_RATE_PARENT,
+					clk_base + CCLKLP_BURST_POLICY,
+					0, 4, 8, 0, NULL);
+		} else {
+			clk = tegra_clk_register_super_mux("cclk_lp",
+					gen_info->cclk_lp_parents,
+					gen_info->num_cclk_lp_parents,
 					CLK_SET_RATE_PARENT,
 					clk_base + CCLKLP_BURST_POLICY,
 					TEGRA_DIVIDER_2, 4, 8, 9, NULL);
+		}
 		*dt_clk = clk;
 	}
 
-	tegra_sclk_init(clk_base, tegra_clks);
+	tegra_sclk_init(clk_base, tegra_clks, gen_info);
 
-#if defined(CONFIG_ARCH_TEGRA_114_SOC) || defined(CONFIG_ARCH_TEGRA_124_SOC)
+#if defined(CONFIG_ARCH_TEGRA_114_SOC) || \
+    defined(CONFIG_ARCH_TEGRA_124_SOC) || \
+    defined(CONFIG_ARCH_TEGRA_210_SOC)
 	/* PLLX */
 	dt_clk = tegra_lookup_dt_id(tegra_clk_pll_x, tegra_clks);
 	if (!dt_clk)
@@ -148,3 +247,20 @@ void __init tegra_super_clk_gen4_init(void __iomem *clk_base,
 #endif
 }
 
+void __init tegra_super_clk_gen4_init(void __iomem *clk_base,
+				void __iomem *pmc_base,
+				struct tegra_clk *tegra_clks,
+				struct tegra_clk_pll_params *params)
+{
+	tegra_super_clk_init(clk_base, pmc_base, tegra_clks, params,
+			     &tegra_super_gen_info_gen4);
+}
+
+void __init tegra_super_clk_gen5_init(void __iomem *clk_base,
+				void __iomem *pmc_base,
+				struct tegra_clk *tegra_clks,
+				struct tegra_clk_pll_params *params)
+{
+	tegra_super_clk_init(clk_base, pmc_base, tegra_clks, params,
+			     &tegra_super_gen_info_gen5);
+}
diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c
index b7d03e9add97..4a24aa4bbdea 100644
--- a/drivers/clk/tegra/clk-tegra114.c
+++ b/drivers/clk/tegra/clk-tegra114.c
@@ -182,40 +182,40 @@ static struct div_nmp pllxc_nmp = {
 	.divp_width = 4,
 };
 
-static struct pdiv_map pllxc_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 3, .hw_val = 2 },
-	{ .pdiv = 4, .hw_val = 3 },
-	{ .pdiv = 5, .hw_val = 4 },
-	{ .pdiv = 6, .hw_val = 5 },
-	{ .pdiv = 8, .hw_val = 6 },
-	{ .pdiv = 10, .hw_val = 7 },
-	{ .pdiv = 12, .hw_val = 8 },
-	{ .pdiv = 16, .hw_val = 9 },
+static const struct pdiv_map pllxc_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
 	{ .pdiv = 12, .hw_val = 10 },
 	{ .pdiv = 16, .hw_val = 11 },
 	{ .pdiv = 20, .hw_val = 12 },
 	{ .pdiv = 24, .hw_val = 13 },
 	{ .pdiv = 32, .hw_val = 14 },
-	{ .pdiv = 0, .hw_val = 0 },
+	{ .pdiv =  0, .hw_val =  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_c_freq_table[] = {
-	{ 12000000, 624000000, 104, 0, 2},
-	{ 12000000, 600000000, 100, 0, 2},
-	{ 13000000, 600000000,  92, 0, 2},	/* actual: 598.0 MHz */
-	{ 16800000, 600000000,  71, 0, 2},	/* actual: 596.4 MHz */
-	{ 19200000, 600000000,  62, 0, 2},	/* actual: 595.2 MHz */
-	{ 26000000, 600000000,  92, 1, 2},	/* actual: 598.0 MHz */
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 624000000, 104, 1, 2, 0 },
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_c_params = {
 	.input_min = 12000000,
 	.input_max = 800000000,
 	.cf_min = 12000000,
-	.cf_max = 19200000,	/* s/w policy, h/w capability 50 MHz */
+	.cf_max = 19200000, /* s/w policy, h/w capability 50 MHz */
 	.vco_min = 600000000,
 	.vco_max = 1400000000,
 	.base_reg = PLLC_BASE,
@@ -232,7 +232,7 @@ static struct tegra_clk_pll_params pll_c_params = {
 	.pdiv_tohw = pllxc_p,
 	.div_nmp = &pllxc_nmp,
 	.freq_table = pll_c_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct div_nmp pllcx_nmp = {
@@ -244,22 +244,22 @@ static struct div_nmp pllcx_nmp = {
 	.divp_width = 3,
 };
 
-static struct pdiv_map pllc_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 4, .hw_val = 3 },
-	{ .pdiv = 8, .hw_val = 5 },
+static const struct pdiv_map pllc_p[] = {
+	{ .pdiv =  1, .hw_val = 0 },
+	{ .pdiv =  2, .hw_val = 1 },
+	{ .pdiv =  4, .hw_val = 3 },
+	{ .pdiv =  8, .hw_val = 5 },
 	{ .pdiv = 16, .hw_val = 7 },
-	{ .pdiv = 0, .hw_val = 0 },
+	{ .pdiv =  0, .hw_val = 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_cx_freq_table[] = {
-	{12000000, 600000000, 100, 0, 2},
-	{13000000, 600000000, 92, 0, 2},	/* actual: 598.0 MHz */
-	{16800000, 600000000, 71, 0, 2},	/* actual: 596.4 MHz */
-	{19200000, 600000000, 62, 0, 2},	/* actual: 595.2 MHz */
-	{26000000, 600000000, 92, 1, 2},	/* actual: 598.0 MHz */
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_c2_params = {
@@ -318,26 +318,26 @@ static struct div_nmp pllm_nmp = {
 	.override_divp_shift = 27,
 };
 
-static struct pdiv_map pllm_p[] = {
+static const struct pdiv_map pllm_p[] = {
 	{ .pdiv = 1, .hw_val = 0 },
 	{ .pdiv = 2, .hw_val = 1 },
 	{ .pdiv = 0, .hw_val = 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_m_freq_table[] = {
-	{12000000, 800000000, 66, 0, 1},	/* actual: 792.0 MHz */
-	{13000000, 800000000, 61, 0, 1},	/* actual: 793.0 MHz */
-	{16800000, 800000000, 47, 0, 1},	/* actual: 789.6 MHz */
-	{19200000, 800000000, 41, 0, 1},	/* actual: 787.2 MHz */
-	{26000000, 800000000, 61, 1, 1},	/* actual: 793.0 MHz */
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 800000000, 66, 1, 1, 0 }, /* actual: 792.0 MHz */
+	{ 13000000, 800000000, 61, 1, 1, 0 }, /* actual: 793.0 MHz */
+	{ 16800000, 800000000, 47, 1, 1, 0 }, /* actual: 789.6 MHz */
+	{ 19200000, 800000000, 41, 1, 1, 0 }, /* actual: 787.2 MHz */
+	{ 26000000, 800000000, 61, 2, 1, 0 }, /* actual: 793.0 MHz */
+	{        0,         0,  0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_m_params = {
 	.input_min = 12000000,
 	.input_max = 500000000,
 	.cf_min = 12000000,
-	.cf_max = 19200000,	/* s/w policy, h/w capability 50 MHz */
+	.cf_max = 19200000, /* s/w policy, h/w capability 50 MHz */
 	.vco_min = 400000000,
 	.vco_max = 1066000000,
 	.base_reg = PLLM_BASE,
@@ -351,7 +351,8 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.pmc_divnm_reg = PMC_PLLM_WB0_OVERRIDE,
 	.pmc_divp_reg = PMC_PLLM_WB0_OVERRIDE_2,
 	.freq_table = pll_m_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE |
+		 TEGRA_PLL_FIXED,
 };
 
 static struct div_nmp pllp_nmp = {
@@ -364,12 +365,12 @@ static struct div_nmp pllp_nmp = {
 };
 
 static struct tegra_clk_pll_freq_table pll_p_freq_table[] = {
-	{12000000, 216000000, 432, 12, 1, 8},
-	{13000000, 216000000, 432, 13, 1, 8},
-	{16800000, 216000000, 360, 14, 1, 8},
-	{19200000, 216000000, 360, 16, 1, 8},
-	{26000000, 216000000, 432, 26, 1, 8},
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 216000000, 432, 12, 2, 8 },
+	{ 13000000, 216000000, 432, 13, 2, 8 },
+	{ 16800000, 216000000, 360, 14, 2, 8 },
+	{ 19200000, 216000000, 360, 16, 2, 8 },
+	{ 26000000, 216000000, 432, 26, 2, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_p_params = {
@@ -386,19 +387,19 @@ static struct tegra_clk_pll_params pll_p_params = {
 	.lock_delay = 300,
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_p_freq_table,
-	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate = 408000000,
 };
 
 static struct tegra_clk_pll_freq_table pll_a_freq_table[] = {
-	{9600000, 282240000, 147, 5, 0, 4},
-	{9600000, 368640000, 192, 5, 0, 4},
-	{9600000, 240000000, 200, 8, 0, 8},
-
-	{28800000, 282240000, 245, 25, 0, 8},
-	{28800000, 368640000, 320, 25, 0, 8},
-	{28800000, 240000000, 200, 24, 0, 8},
-	{0, 0, 0, 0, 0, 0},
+	{  9600000, 282240000, 147,  5, 1, 4 },
+	{  9600000, 368640000, 192,  5, 1, 4 },
+	{  9600000, 240000000, 200,  8, 1, 8 },
+	{ 28800000, 282240000, 245, 25, 1, 8 },
+	{ 28800000, 368640000, 320, 25, 1, 8 },
+	{ 28800000, 240000000, 200, 24, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 
@@ -416,28 +417,26 @@ static struct tegra_clk_pll_params pll_a_params = {
 	.lock_delay = 300,
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_a_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_d_freq_table[] = {
-	{12000000, 216000000, 864, 12, 2, 12},
-	{13000000, 216000000, 864, 13, 2, 12},
-	{16800000, 216000000, 720, 14, 2, 12},
-	{19200000, 216000000, 720, 16, 2, 12},
-	{26000000, 216000000, 864, 26, 2, 12},
-
-	{12000000, 594000000, 594, 12, 0, 12},
-	{13000000, 594000000, 594, 13, 0, 12},
-	{16800000, 594000000, 495, 14, 0, 12},
-	{19200000, 594000000, 495, 16, 0, 12},
-	{26000000, 594000000, 594, 26, 0, 12},
-
-	{12000000, 1000000000, 1000, 12, 0, 12},
-	{13000000, 1000000000, 1000, 13, 0, 12},
-	{19200000, 1000000000, 625, 12, 0, 12},
-	{26000000, 1000000000, 1000, 26, 0, 12},
-
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000,  216000000,  864, 12, 4, 12 },
+	{ 13000000,  216000000,  864, 13, 4, 12 },
+	{ 16800000,  216000000,  720, 14, 4, 12 },
+	{ 19200000,  216000000,  720, 16, 4, 12 },
+	{ 26000000,  216000000,  864, 26, 4, 12 },
+	{ 12000000,  594000000,  594, 12, 1, 12 },
+	{ 13000000,  594000000,  594, 13, 1, 12 },
+	{ 16800000,  594000000,  495, 14, 1, 12 },
+	{ 19200000,  594000000,  495, 16, 1, 12 },
+	{ 26000000,  594000000,  594, 26, 1, 12 },
+	{ 12000000, 1000000000, 1000, 12, 1, 12 },
+	{ 13000000, 1000000000, 1000, 13, 1, 12 },
+	{ 19200000, 1000000000,  625, 12, 1, 12 },
+	{ 26000000, 1000000000, 1000, 26, 1, 12 },
+	{        0,          0,    0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_params pll_d_params = {
@@ -455,7 +454,7 @@ static struct tegra_clk_pll_params pll_d_params = {
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_d2_params = {
@@ -473,10 +472,10 @@ static struct tegra_clk_pll_params pll_d2_params = {
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
-static struct pdiv_map pllu_p[] = {
+static const struct pdiv_map pllu_p[] = {
 	{ .pdiv = 1, .hw_val = 1 },
 	{ .pdiv = 2, .hw_val = 0 },
 	{ .pdiv = 0, .hw_val = 0 },
@@ -492,12 +491,12 @@ static struct div_nmp pllu_nmp = {
 };
 
 static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
-	{12000000, 480000000, 960, 12, 0, 12},
-	{13000000, 480000000, 960, 13, 0, 12},
-	{16800000, 480000000, 400, 7, 0, 5},
-	{19200000, 480000000, 200, 4, 0, 3},
-	{26000000, 480000000, 960, 26, 0, 12},
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 480000000, 960, 12, 2, 12 },
+	{ 13000000, 480000000, 960, 13, 2, 12 },
+	{ 16800000, 480000000, 400,  7, 2,  5 },
+	{ 19200000, 480000000, 200,  4, 2,  3 },
+	{ 26000000, 480000000, 960, 26, 2, 12 },
+	{        0,         0,   0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_params pll_u_params = {
@@ -516,25 +515,24 @@ static struct tegra_clk_pll_params pll_u_params = {
 	.div_nmp = &pllu_nmp,
 	.freq_table = pll_u_freq_table,
 	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_x_freq_table[] = {
 	/* 1 GHz */
-	{12000000, 1000000000, 83, 0, 1},	/* actual: 996.0 MHz */
-	{13000000, 1000000000, 76, 0, 1},	/* actual: 988.0 MHz */
-	{16800000, 1000000000, 59, 0, 1},	/* actual: 991.2 MHz */
-	{19200000, 1000000000, 52, 0, 1},	/* actual: 998.4 MHz */
-	{26000000, 1000000000, 76, 1, 1},	/* actual: 988.0 MHz */
-
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 1000000000, 83, 1, 1, 0 }, /* actual: 996.0 MHz */
+	{ 13000000, 1000000000, 76, 1, 1, 0 }, /* actual: 988.0 MHz */
+	{ 16800000, 1000000000, 59, 1, 1, 0 }, /* actual: 991.2 MHz */
+	{ 19200000, 1000000000, 52, 1, 1, 0 }, /* actual: 998.4 MHz */
+	{ 26000000, 1000000000, 76, 2, 1, 0 }, /* actual: 988.0 MHz */
+	{        0,          0,  0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_x_params = {
 	.input_min = 12000000,
 	.input_max = 800000000,
 	.cf_min = 12000000,
-	.cf_max = 19200000,	/* s/w policy, h/w capability 50 MHz */
+	.cf_max = 19200000, /* s/w policy, h/w capability 50 MHz */
 	.vco_min = 700000000,
 	.vco_max = 2400000000U,
 	.base_reg = PLLX_BASE,
@@ -551,15 +549,34 @@ static struct tegra_clk_pll_params pll_x_params = {
 	.pdiv_tohw = pllxc_p,
 	.div_nmp = &pllxc_nmp,
 	.freq_table = pll_x_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
 	/* PLLE special case: use cpcon field to store cml divider value */
-	{336000000, 100000000, 100, 21, 16, 11},
-	{312000000, 100000000, 200, 26, 24, 13},
-	{12000000, 100000000, 200,  1,  24, 13},
-	{0, 0, 0, 0, 0, 0},
+	{ 336000000, 100000000, 100, 21, 16, 11 },
+	{ 312000000, 100000000, 200, 26, 24, 13 },
+	{  12000000, 100000000, 200,  1, 24, 13 },
+	{         0,         0,   0,  0,  0,  0 },
+};
+
+static const struct pdiv_map plle_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
+	{ .pdiv = 12, .hw_val = 10 },
+	{ .pdiv = 16, .hw_val = 11 },
+	{ .pdiv = 20, .hw_val = 12 },
+	{ .pdiv = 24, .hw_val = 13 },
+	{ .pdiv = 32, .hw_val = 14 },
+	{ .pdiv =  0, .hw_val =  0 }
 };
 
 static struct div_nmp plle_nmp = {
@@ -584,9 +601,10 @@ static struct tegra_clk_pll_params pll_e_params = {
 	.lock_mask = PLLE_MISC_LOCK,
 	.lock_enable_bit_idx = PLLE_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
+	.pdiv_tohw = plle_p,
 	.div_nmp = &plle_nmp,
 	.freq_table = pll_e_freq_table,
-	.flags = TEGRA_PLL_FIXED,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate = 100000000,
 };
 
@@ -614,18 +632,19 @@ static struct tegra_clk_pll_params pll_re_vco_params = {
 	.iddq_reg = PLLRE_MISC,
 	.iddq_bit_idx = PLLRE_IDDQ_BIT,
 	.div_nmp = &pllre_nmp,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE |
+		 TEGRA_PLL_LOCK_MISC,
 };
 
 /* possible OSC frequencies in Hz */
 static unsigned long tegra114_input_freq[] = {
-	[0] = 13000000,
-	[1] = 16800000,
-	[4] = 19200000,
-	[5] = 38400000,
-	[8] = 12000000,
-	[9] = 48000000,
-	[12] = 260000000,
+	[ 0] = 13000000,
+	[ 1] = 16800000,
+	[ 4] = 19200000,
+	[ 5] = 38400000,
+	[ 8] = 12000000,
+	[ 9] = 48000000,
+	[12] = 26000000,
 };
 
 #define MASK(x) (BIT(x) - 1)
@@ -644,21 +663,27 @@ struct utmi_clk_param {
 };
 
 static const struct utmi_clk_param utmi_parameters[] = {
-	{.osc_frequency = 13000000, .enable_delay_count = 0x02,
-	 .stable_count = 0x33, .active_delay_count = 0x05,
-	 .xtal_freq_count = 0x7F},
-	{.osc_frequency = 19200000, .enable_delay_count = 0x03,
-	 .stable_count = 0x4B, .active_delay_count = 0x06,
-	 .xtal_freq_count = 0xBB},
-	{.osc_frequency = 12000000, .enable_delay_count = 0x02,
-	 .stable_count = 0x2F, .active_delay_count = 0x04,
-	 .xtal_freq_count = 0x76},
-	{.osc_frequency = 26000000, .enable_delay_count = 0x04,
-	 .stable_count = 0x66, .active_delay_count = 0x09,
-	 .xtal_freq_count = 0xFE},
-	{.osc_frequency = 16800000, .enable_delay_count = 0x03,
-	 .stable_count = 0x41, .active_delay_count = 0x0A,
-	 .xtal_freq_count = 0xA4},
+	{
+		.osc_frequency = 13000000, .enable_delay_count = 0x02,
+		.stable_count = 0x33, .active_delay_count = 0x05,
+		.xtal_freq_count = 0x7f
+	}, {
+		.osc_frequency = 19200000, .enable_delay_count = 0x03,
+		.stable_count = 0x4b, .active_delay_count = 0x06,
+		.xtal_freq_count = 0xbb
+	}, {
+		.osc_frequency = 12000000, .enable_delay_count = 0x02,
+		.stable_count = 0x2f, .active_delay_count = 0x04,
+		.xtal_freq_count = 0x76
+	}, {
+		.osc_frequency = 26000000, .enable_delay_count = 0x04,
+		.stable_count = 0x66, .active_delay_count = 0x09,
+		.xtal_freq_count = 0xfe
+	}, {
+		.osc_frequency = 16800000, .enable_delay_count = 0x03,
+		.stable_count = 0x41, .active_delay_count = 0x0a,
+		.xtal_freq_count = 0xa4
+	},
 };
 
 /* peripheral mux definitions */
@@ -965,8 +990,8 @@ static void __init tegra114_fixed_clk_init(void __iomem *clk_base)
 
 static __init void tegra114_utmi_param_configure(void __iomem *clk_base)
 {
+	unsigned int i;
 	u32 reg;
-	int i;
 
 	for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) {
 		if (osc_freq == utmi_parameters[i].osc_frequency)
@@ -1173,7 +1198,7 @@ static __init void tegra114_periph_clk_init(void __iomem *clk_base,
 {
 	struct clk *clk;
 	struct tegra_periph_init_data *data;
-	int i;
+	unsigned int i;
 
 	/* xusb_ss_div2 */
 	clk = clk_register_fixed_factor(NULL, "xusb_ss_div2", "xusb_ss_src", 0,
@@ -1278,7 +1303,7 @@ static struct tegra_cpu_car_ops tegra114_cpu_car_ops = {
 
 static const struct of_device_id pmc_match[] __initconst = {
 	{ .compatible = "nvidia,tegra114-pmc" },
-	{},
+	{ },
 };
 
 /*
@@ -1286,37 +1311,37 @@ static const struct of_device_id pmc_match[] __initconst = {
  * breaks
  */
 static struct tegra_clk_init_table init_table[] __initdata = {
-	{TEGRA114_CLK_UARTA, TEGRA114_CLK_PLL_P, 408000000, 0},
-	{TEGRA114_CLK_UARTB, TEGRA114_CLK_PLL_P, 408000000, 0},
-	{TEGRA114_CLK_UARTC, TEGRA114_CLK_PLL_P, 408000000, 0},
-	{TEGRA114_CLK_UARTD, TEGRA114_CLK_PLL_P, 408000000, 0},
-	{TEGRA114_CLK_PLL_A, TEGRA114_CLK_CLK_MAX, 564480000, 1},
-	{TEGRA114_CLK_PLL_A_OUT0, TEGRA114_CLK_CLK_MAX, 11289600, 1},
-	{TEGRA114_CLK_EXTERN1, TEGRA114_CLK_PLL_A_OUT0, 0, 1},
-	{TEGRA114_CLK_CLK_OUT_1_MUX, TEGRA114_CLK_EXTERN1, 0, 1},
-	{TEGRA114_CLK_CLK_OUT_1, TEGRA114_CLK_CLK_MAX, 0, 1},
-	{TEGRA114_CLK_I2S0, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA114_CLK_I2S1, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA114_CLK_I2S2, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA114_CLK_I2S3, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA114_CLK_I2S4, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA114_CLK_HOST1X, TEGRA114_CLK_PLL_P, 136000000, 0},
-	{TEGRA114_CLK_DFLL_SOC, TEGRA114_CLK_PLL_P, 51000000, 1},
-	{TEGRA114_CLK_DFLL_REF, TEGRA114_CLK_PLL_P, 51000000, 1},
-	{TEGRA114_CLK_DISP1, TEGRA114_CLK_PLL_P, 0, 0},
-	{TEGRA114_CLK_DISP2, TEGRA114_CLK_PLL_P, 0, 0},
-	{TEGRA114_CLK_GR2D, TEGRA114_CLK_PLL_C2, 300000000, 0},
-	{TEGRA114_CLK_GR3D, TEGRA114_CLK_PLL_C2, 300000000, 0},
-	{TEGRA114_CLK_DSIALP, TEGRA114_CLK_PLL_P, 68000000, 0},
-	{TEGRA114_CLK_DSIBLP, TEGRA114_CLK_PLL_P, 68000000, 0},
-	{TEGRA114_CLK_PLL_RE_VCO, TEGRA114_CLK_CLK_MAX, 612000000, 0},
-	{TEGRA114_CLK_XUSB_SS_SRC, TEGRA114_CLK_PLL_RE_OUT, 122400000, 0},
-	{TEGRA114_CLK_XUSB_FS_SRC, TEGRA114_CLK_PLL_U_48M, 48000000, 0},
-	{TEGRA114_CLK_XUSB_HS_SRC, TEGRA114_CLK_XUSB_SS_DIV2, 61200000, 0},
-	{TEGRA114_CLK_XUSB_FALCON_SRC, TEGRA114_CLK_PLL_P, 204000000, 0},
-	{TEGRA114_CLK_XUSB_HOST_SRC, TEGRA114_CLK_PLL_P, 102000000, 0},
-	/* This MUST be the last entry. */
-	{TEGRA114_CLK_CLK_MAX, TEGRA114_CLK_CLK_MAX, 0, 0},
+	{ TEGRA114_CLK_UARTA, TEGRA114_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA114_CLK_UARTB, TEGRA114_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA114_CLK_UARTC, TEGRA114_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA114_CLK_UARTD, TEGRA114_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA114_CLK_PLL_A, TEGRA114_CLK_CLK_MAX, 564480000, 1 },
+	{ TEGRA114_CLK_PLL_A_OUT0, TEGRA114_CLK_CLK_MAX, 11289600, 1 },
+	{ TEGRA114_CLK_EXTERN1, TEGRA114_CLK_PLL_A_OUT0, 0, 1 },
+	{ TEGRA114_CLK_CLK_OUT_1_MUX, TEGRA114_CLK_EXTERN1, 0, 1 },
+	{ TEGRA114_CLK_CLK_OUT_1, TEGRA114_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA114_CLK_I2S0, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA114_CLK_I2S1, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA114_CLK_I2S2, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA114_CLK_I2S3, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA114_CLK_I2S4, TEGRA114_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA114_CLK_HOST1X, TEGRA114_CLK_PLL_P, 136000000, 0 },
+	{ TEGRA114_CLK_DFLL_SOC, TEGRA114_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA114_CLK_DFLL_REF, TEGRA114_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA114_CLK_DISP1, TEGRA114_CLK_PLL_P, 0, 0 },
+	{ TEGRA114_CLK_DISP2, TEGRA114_CLK_PLL_P, 0, 0 },
+	{ TEGRA114_CLK_GR2D, TEGRA114_CLK_PLL_C2, 300000000, 0 },
+	{ TEGRA114_CLK_GR3D, TEGRA114_CLK_PLL_C2, 300000000, 0 },
+	{ TEGRA114_CLK_DSIALP, TEGRA114_CLK_PLL_P, 68000000, 0 },
+	{ TEGRA114_CLK_DSIBLP, TEGRA114_CLK_PLL_P, 68000000, 0 },
+	{ TEGRA114_CLK_PLL_RE_VCO, TEGRA114_CLK_CLK_MAX, 612000000, 0 },
+	{ TEGRA114_CLK_XUSB_SS_SRC, TEGRA114_CLK_PLL_RE_OUT, 122400000, 0 },
+	{ TEGRA114_CLK_XUSB_FS_SRC, TEGRA114_CLK_PLL_U_48M, 48000000, 0 },
+	{ TEGRA114_CLK_XUSB_HS_SRC, TEGRA114_CLK_XUSB_SS_DIV2, 61200000, 0 },
+	{ TEGRA114_CLK_XUSB_FALCON_SRC, TEGRA114_CLK_PLL_P, 204000000, 0 },
+	{ TEGRA114_CLK_XUSB_HOST_SRC, TEGRA114_CLK_PLL_P, 102000000, 0 },
+	/* must be the last entry */
+	{ TEGRA114_CLK_CLK_MAX, TEGRA114_CLK_CLK_MAX, 0, 0 },
 };
 
 static void __init tegra114_clock_apply_init_table(void)
diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c
index 87975f7adddc..1627258292d2 100644
--- a/drivers/clk/tegra/clk-tegra124.c
+++ b/drivers/clk/tegra/clk-tegra124.c
@@ -150,13 +150,13 @@ static DEFINE_SPINLOCK(emc_lock);
 
 /* possible OSC frequencies in Hz */
 static unsigned long tegra124_input_freq[] = {
-	[0] = 13000000,
-	[1] = 16800000,
-	[4] = 19200000,
-	[5] = 38400000,
-	[8] = 12000000,
-	[9] = 48000000,
-	[12] = 260000000,
+	[ 0] = 13000000,
+	[ 1] = 16800000,
+	[ 4] = 19200000,
+	[ 5] = 38400000,
+	[ 8] = 12000000,
+	[ 9] = 48000000,
+	[12] = 26000000,
 };
 
 static struct div_nmp pllxc_nmp = {
@@ -168,33 +168,33 @@ static struct div_nmp pllxc_nmp = {
 	.divp_width = 4,
 };
 
-static struct pdiv_map pllxc_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 3, .hw_val = 2 },
-	{ .pdiv = 4, .hw_val = 3 },
-	{ .pdiv = 5, .hw_val = 4 },
-	{ .pdiv = 6, .hw_val = 5 },
-	{ .pdiv = 8, .hw_val = 6 },
-	{ .pdiv = 10, .hw_val = 7 },
-	{ .pdiv = 12, .hw_val = 8 },
-	{ .pdiv = 16, .hw_val = 9 },
+static const struct pdiv_map pllxc_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
 	{ .pdiv = 12, .hw_val = 10 },
 	{ .pdiv = 16, .hw_val = 11 },
 	{ .pdiv = 20, .hw_val = 12 },
 	{ .pdiv = 24, .hw_val = 13 },
 	{ .pdiv = 32, .hw_val = 14 },
-	{ .pdiv = 0, .hw_val = 0 },
+	{ .pdiv =  0, .hw_val =  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_x_freq_table[] = {
 	/* 1 GHz */
-	{12000000, 1000000000, 83, 0, 1},	/* actual: 996.0 MHz */
-	{13000000, 1000000000, 76, 0, 1},	/* actual: 988.0 MHz */
-	{16800000, 1000000000, 59, 0, 1},	/* actual: 991.2 MHz */
-	{19200000, 1000000000, 52, 0, 1},	/* actual: 998.4 MHz */
-	{26000000, 1000000000, 76, 1, 1},	/* actual: 988.0 MHz */
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 1000000000, 83, 1, 1, 0 }, /* actual: 996.0 MHz */
+	{ 13000000, 1000000000, 76, 1, 1, 0 }, /* actual: 988.0 MHz */
+	{ 16800000, 1000000000, 59, 1, 1, 0 }, /* actual: 991.2 MHz */
+	{ 19200000, 1000000000, 52, 1, 1, 0 }, /* actual: 998.4 MHz */
+	{ 26000000, 1000000000, 76, 2, 1, 0 }, /* actual: 988.0 MHz */
+	{        0,          0,  0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_x_params = {
@@ -218,24 +218,24 @@ static struct tegra_clk_pll_params pll_x_params = {
 	.pdiv_tohw = pllxc_p,
 	.div_nmp = &pllxc_nmp,
 	.freq_table = pll_x_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_c_freq_table[] = {
-	{ 12000000, 624000000, 104, 1, 2},
-	{ 12000000, 600000000, 100, 1, 2},
-	{ 13000000, 600000000,  92, 1, 2},	/* actual: 598.0 MHz */
-	{ 16800000, 600000000,  71, 1, 2},	/* actual: 596.4 MHz */
-	{ 19200000, 600000000,  62, 1, 2},	/* actual: 595.2 MHz */
-	{ 26000000, 600000000,  92, 2, 2},	/* actual: 598.0 MHz */
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 624000000, 104, 1, 2, 0 },
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_c_params = {
 	.input_min = 12000000,
 	.input_max = 800000000,
 	.cf_min = 12000000,
-	.cf_max = 19200000,	/* s/w policy, h/w capability 50 MHz */
+	.cf_max = 19200000, /* s/w policy, h/w capability 50 MHz */
 	.vco_min = 600000000,
 	.vco_max = 1400000000,
 	.base_reg = PLLC_BASE,
@@ -252,7 +252,7 @@ static struct tegra_clk_pll_params pll_c_params = {
 	.pdiv_tohw = pllxc_p,
 	.div_nmp = &pllxc_nmp,
 	.freq_table = pll_c_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct div_nmp pllcx_nmp = {
@@ -264,25 +264,25 @@ static struct div_nmp pllcx_nmp = {
 	.divp_width = 3,
 };
 
-static struct pdiv_map pllc_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 3, .hw_val = 2 },
-	{ .pdiv = 4, .hw_val = 3 },
-	{ .pdiv = 6, .hw_val = 4 },
-	{ .pdiv = 8, .hw_val = 5 },
+static const struct pdiv_map pllc_p[] = {
+	{ .pdiv =  1, .hw_val = 0 },
+	{ .pdiv =  2, .hw_val = 1 },
+	{ .pdiv =  3, .hw_val = 2 },
+	{ .pdiv =  4, .hw_val = 3 },
+	{ .pdiv =  6, .hw_val = 4 },
+	{ .pdiv =  8, .hw_val = 5 },
 	{ .pdiv = 12, .hw_val = 6 },
 	{ .pdiv = 16, .hw_val = 7 },
-	{ .pdiv = 0, .hw_val = 0 },
+	{ .pdiv =  0, .hw_val = 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_cx_freq_table[] = {
-	{12000000, 600000000, 100, 1, 2},
-	{13000000, 600000000, 92, 1, 2},	/* actual: 598.0 MHz */
-	{16800000, 600000000, 71, 1, 2},	/* actual: 596.4 MHz */
-	{19200000, 600000000, 62, 1, 2},	/* actual: 595.2 MHz */
-	{26000000, 600000000, 92, 2, 2},	/* actual: 598.0 MHz */
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_c2_params = {
@@ -338,32 +338,32 @@ static struct div_nmp pllss_nmp = {
 	.divp_width = 4,
 };
 
-static struct pdiv_map pll12g_ssd_esd_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 3, .hw_val = 2 },
-	{ .pdiv = 4, .hw_val = 3 },
-	{ .pdiv = 5, .hw_val = 4 },
-	{ .pdiv = 6, .hw_val = 5 },
-	{ .pdiv = 8, .hw_val = 6 },
-	{ .pdiv = 10, .hw_val = 7 },
-	{ .pdiv = 12, .hw_val = 8 },
-	{ .pdiv = 16, .hw_val = 9 },
+static const struct pdiv_map pll12g_ssd_esd_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
 	{ .pdiv = 12, .hw_val = 10 },
 	{ .pdiv = 16, .hw_val = 11 },
 	{ .pdiv = 20, .hw_val = 12 },
 	{ .pdiv = 24, .hw_val = 13 },
 	{ .pdiv = 32, .hw_val = 14 },
-	{ .pdiv = 0, .hw_val = 0 },
+	{ .pdiv =  0, .hw_val =  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_c4_freq_table[] = {
-	{ 12000000, 600000000, 100, 1, 1},
-	{ 13000000, 600000000,  92, 1, 1},      /* actual: 598.0 MHz */
-	{ 16800000, 600000000,  71, 1, 1},      /* actual: 596.4 MHz */
-	{ 19200000, 600000000,  62, 1, 1},      /* actual: 595.2 MHz */
-	{ 26000000, 600000000,  92, 2, 1},      /* actual: 598.0 MHz */
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_c4_params = {
@@ -386,21 +386,35 @@ static struct tegra_clk_pll_params pll_c4_params = {
 	.ext_misc_reg[1] = 0x5b0,
 	.ext_misc_reg[2] = 0x5b4,
 	.freq_table = pll_c4_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
-static struct pdiv_map pllm_p[] = {
-	{ .pdiv = 1, .hw_val = 0 },
-	{ .pdiv = 2, .hw_val = 1 },
-	{ .pdiv = 0, .hw_val = 0 },
+static const struct pdiv_map pllm_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
+	{ .pdiv = 12, .hw_val = 10 },
+	{ .pdiv = 16, .hw_val = 11 },
+	{ .pdiv = 20, .hw_val = 12 },
+	{ .pdiv = 24, .hw_val = 13 },
+	{ .pdiv = 32, .hw_val = 14 },
+	{ .pdiv =  0, .hw_val =  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_m_freq_table[] = {
-	{12000000, 800000000, 66, 1, 1},	/* actual: 792.0 MHz */
-	{13000000, 800000000, 61, 1, 1},	/* actual: 793.0 MHz */
-	{16800000, 800000000, 47, 1, 1},	/* actual: 789.6 MHz */
-	{19200000, 800000000, 41, 1, 1},	/* actual: 787.2 MHz */
-	{26000000, 800000000, 61, 2, 1},	/* actual: 793.0 MHz */
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 800000000, 66, 1, 1, 0 }, /* actual: 792.0 MHz */
+	{ 13000000, 800000000, 61, 1, 1, 0 }, /* actual: 793.0 MHz */
+	{ 16800000, 800000000, 47, 1, 1, 0 }, /* actual: 789.6 MHz */
+	{ 19200000, 800000000, 41, 1, 1, 0 }, /* actual: 787.2 MHz */
+	{ 26000000, 800000000, 61, 2, 1, 0 }, /* actual: 793.0 MHz */
+	{        0,         0,  0, 0, 0, 0},
 };
 
 static struct div_nmp pllm_nmp = {
@@ -427,22 +441,41 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.lock_mask = PLL_BASE_LOCK,
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
-	.max_p = 2,
+	.max_p = 5,
 	.pdiv_tohw = pllm_p,
 	.div_nmp = &pllm_nmp,
 	.pmc_divnm_reg = PMC_PLLM_WB0_OVERRIDE,
 	.pmc_divp_reg = PMC_PLLM_WB0_OVERRIDE_2,
 	.freq_table = pll_m_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
 	/* PLLE special case: use cpcon field to store cml divider value */
-	{336000000, 100000000, 100, 21, 16, 11},
-	{312000000, 100000000, 200, 26, 24, 13},
-	{13000000,  100000000, 200, 1,  26, 13},
-	{12000000,  100000000, 200, 1,  24, 13},
-	{0, 0, 0, 0, 0, 0},
+	{ 336000000, 100000000, 100, 21, 16, 11 },
+	{ 312000000, 100000000, 200, 26, 24, 13 },
+	{  13000000, 100000000, 200,  1, 26, 13 },
+	{  12000000, 100000000, 200,  1, 24, 13 },
+	{         0,         0,   0,  0,  0,  0 },
+};
+
+static const struct pdiv_map plle_p[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv = 10, .hw_val =  7 },
+	{ .pdiv = 12, .hw_val =  8 },
+	{ .pdiv = 16, .hw_val =  9 },
+	{ .pdiv = 12, .hw_val = 10 },
+	{ .pdiv = 16, .hw_val = 11 },
+	{ .pdiv = 20, .hw_val = 12 },
+	{ .pdiv = 24, .hw_val = 13 },
+	{ .pdiv = 32, .hw_val = 14 },
+	{ .pdiv =  1, .hw_val =  0 },
 };
 
 static struct div_nmp plle_nmp = {
@@ -467,9 +500,10 @@ static struct tegra_clk_pll_params pll_e_params = {
 	.lock_mask = PLLE_MISC_LOCK,
 	.lock_enable_bit_idx = PLLE_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
+	.pdiv_tohw = plle_p,
 	.div_nmp = &plle_nmp,
 	.freq_table = pll_e_freq_table,
-	.flags = TEGRA_PLL_FIXED,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate = 100000000,
 };
 
@@ -507,7 +541,8 @@ static struct tegra_clk_pll_params pll_re_vco_params = {
 	.iddq_reg = PLLRE_MISC,
 	.iddq_bit_idx = PLLRE_IDDQ_BIT,
 	.div_nmp = &pllre_nmp,
-	.flags = TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE |
+		 TEGRA_PLL_LOCK_MISC,
 };
 
 static struct div_nmp pllp_nmp = {
@@ -520,12 +555,12 @@ static struct div_nmp pllp_nmp = {
 };
 
 static struct tegra_clk_pll_freq_table pll_p_freq_table[] = {
-	{12000000, 408000000, 408, 12, 0, 8},
-	{13000000, 408000000, 408, 13, 0, 8},
-	{16800000, 408000000, 340, 14, 0, 8},
-	{19200000, 408000000, 340, 16, 0, 8},
-	{26000000, 408000000, 408, 26, 0, 8},
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 408000000, 408, 12, 1, 8 },
+	{ 13000000, 408000000, 408, 13, 1, 8 },
+	{ 16800000, 408000000, 340, 14, 1, 8 },
+	{ 19200000, 408000000, 340, 16, 1, 8 },
+	{ 26000000, 408000000, 408, 26, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_p_params = {
@@ -543,18 +578,18 @@ static struct tegra_clk_pll_params pll_p_params = {
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_p_freq_table,
 	.fixed_rate = 408000000,
-	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_a_freq_table[] = {
-	{9600000, 282240000, 147, 5, 0, 4},
-	{9600000, 368640000, 192, 5, 0, 4},
-	{9600000, 240000000, 200, 8, 0, 8},
-
-	{28800000, 282240000, 245, 25, 0, 8},
-	{28800000, 368640000, 320, 25, 0, 8},
-	{28800000, 240000000, 200, 24, 0, 8},
-	{0, 0, 0, 0, 0, 0},
+	{  9600000, 282240000, 147,  5, 1, 4 },
+	{  9600000, 368640000, 192,  5, 1, 4 },
+	{  9600000, 240000000, 200,  8, 1, 8 },
+	{ 28800000, 282240000, 245, 25, 1, 8 },
+	{ 28800000, 368640000, 320, 25, 1, 8 },
+	{ 28800000, 240000000, 200, 24, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_a_params = {
@@ -571,7 +606,8 @@ static struct tegra_clk_pll_params pll_a_params = {
 	.lock_delay = 300,
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_a_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct div_nmp plld_nmp = {
@@ -584,24 +620,21 @@ static struct div_nmp plld_nmp = {
 };
 
 static struct tegra_clk_pll_freq_table pll_d_freq_table[] = {
-	{12000000, 216000000, 864, 12, 4, 12},
-	{13000000, 216000000, 864, 13, 4, 12},
-	{16800000, 216000000, 720, 14, 4, 12},
-	{19200000, 216000000, 720, 16, 4, 12},
-	{26000000, 216000000, 864, 26, 4, 12},
-
-	{12000000, 594000000, 594, 12, 1, 12},
-	{13000000, 594000000, 594, 13, 1, 12},
-	{16800000, 594000000, 495, 14, 1, 12},
-	{19200000, 594000000, 495, 16, 1, 12},
-	{26000000, 594000000, 594, 26, 1, 12},
-
-	{12000000, 1000000000, 1000, 12, 1, 12},
-	{13000000, 1000000000, 1000, 13, 1, 12},
-	{19200000, 1000000000, 625, 12, 1, 12},
-	{26000000, 1000000000, 1000, 26, 1, 12},
-
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000,  216000000,  864, 12, 4, 12 },
+	{ 13000000,  216000000,  864, 13, 4, 12 },
+	{ 16800000,  216000000,  720, 14, 4, 12 },
+	{ 19200000,  216000000,  720, 16, 4, 12 },
+	{ 26000000,  216000000,  864, 26, 4, 12 },
+	{ 12000000,  594000000,  594, 12, 1, 12 },
+	{ 13000000,  594000000,  594, 13, 1, 12 },
+	{ 16800000,  594000000,  495, 14, 1, 12 },
+	{ 19200000,  594000000,  495, 16, 1, 12 },
+	{ 26000000,  594000000,  594, 26, 1, 12 },
+	{ 12000000, 1000000000, 1000, 12, 1, 12 },
+	{ 13000000, 1000000000, 1000, 13, 1, 12 },
+	{ 19200000, 1000000000,  625, 12, 1, 12 },
+	{ 26000000, 1000000000, 1000, 26, 1, 12 },
+	{        0,          0,    0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_params pll_d_params = {
@@ -619,16 +652,16 @@ static struct tegra_clk_pll_params pll_d_params = {
 	.div_nmp = &plld_nmp,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table tegra124_pll_d2_freq_table[] = {
-	{ 12000000, 594000000,  99, 1, 2},
-	{ 13000000, 594000000,  91, 1, 2},      /* actual: 591.5 MHz */
-	{ 16800000, 594000000,  71, 1, 2},      /* actual: 596.4 MHz */
-	{ 19200000, 594000000,  62, 1, 2},      /* actual: 595.2 MHz */
-	{ 26000000, 594000000,  91, 2, 2},      /* actual: 591.5 MHz */
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 594000000, 99, 1, 2, 0 },
+	{ 13000000, 594000000, 91, 1, 2, 0 }, /* actual: 591.5 MHz */
+	{ 16800000, 594000000, 71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 594000000, 62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 594000000, 91, 2, 2, 0 }, /* actual: 591.5 MHz */
+	{        0,         0,  0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params tegra124_pll_d2_params = {
@@ -652,15 +685,16 @@ static struct tegra_clk_pll_params tegra124_pll_d2_params = {
 	.ext_misc_reg[2] = 0x578,
 	.max_p = 15,
 	.freq_table = tegra124_pll_d2_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_freq_table pll_dp_freq_table[] = {
-	{ 12000000, 600000000, 100, 1, 1},
-	{ 13000000, 600000000,  92, 1, 1},      /* actual: 598.0 MHz */
-	{ 16800000, 600000000,  71, 1, 1},      /* actual: 596.4 MHz */
-	{ 19200000, 600000000,  62, 1, 1},      /* actual: 595.2 MHz */
-	{ 26000000, 600000000,  92, 2, 1},      /* actual: 598.0 MHz */
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 600000000, 100, 1, 2, 0 },
+	{ 13000000, 600000000,  92, 1, 2, 0 }, /* actual: 598.0 MHz */
+	{ 16800000, 600000000,  71, 1, 2, 0 }, /* actual: 596.4 MHz */
+	{ 19200000, 600000000,  62, 1, 2, 0 }, /* actual: 595.2 MHz */
+	{ 26000000, 600000000,  92, 2, 2, 0 }, /* actual: 598.0 MHz */
+	{        0,         0,   0, 0, 0, 0 },
 };
 
 static struct tegra_clk_pll_params pll_dp_params = {
@@ -684,9 +718,10 @@ static struct tegra_clk_pll_params pll_dp_params = {
 	.ext_misc_reg[2] = 0x5a0,
 	.max_p = 5,
 	.freq_table = pll_dp_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
-static struct pdiv_map pllu_p[] = {
+static const struct pdiv_map pllu_p[] = {
 	{ .pdiv = 1, .hw_val = 1 },
 	{ .pdiv = 2, .hw_val = 0 },
 	{ .pdiv = 0, .hw_val = 0 },
@@ -702,12 +737,12 @@ static struct div_nmp pllu_nmp = {
 };
 
 static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
-	{12000000, 480000000, 960, 12, 2, 12},
-	{13000000, 480000000, 960, 13, 2, 12},
-	{16800000, 480000000, 400, 7, 2, 5},
-	{19200000, 480000000, 200, 4, 2, 3},
-	{26000000, 480000000, 960, 26, 2, 12},
-	{0, 0, 0, 0, 0, 0},
+	{ 12000000, 480000000, 960, 12, 2, 12 },
+	{ 13000000, 480000000, 960, 13, 2, 12 },
+	{ 16800000, 480000000, 400,  7, 2,  5 },
+	{ 19200000, 480000000, 200,  4, 2,  3 },
+	{ 26000000, 480000000, 960, 26, 2, 12 },
+	{        0,         0,   0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_params pll_u_params = {
@@ -726,7 +761,7 @@ static struct tegra_clk_pll_params pll_u_params = {
 	.div_nmp = &pllu_nmp,
 	.freq_table = pll_u_freq_table,
 	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 struct utmi_clk_param {
@@ -743,21 +778,27 @@ struct utmi_clk_param {
 };
 
 static const struct utmi_clk_param utmi_parameters[] = {
-	{.osc_frequency = 13000000, .enable_delay_count = 0x02,
-	 .stable_count = 0x33, .active_delay_count = 0x05,
-	 .xtal_freq_count = 0x7F},
-	{.osc_frequency = 19200000, .enable_delay_count = 0x03,
-	 .stable_count = 0x4B, .active_delay_count = 0x06,
-	 .xtal_freq_count = 0xBB},
-	{.osc_frequency = 12000000, .enable_delay_count = 0x02,
-	 .stable_count = 0x2F, .active_delay_count = 0x04,
-	 .xtal_freq_count = 0x76},
-	{.osc_frequency = 26000000, .enable_delay_count = 0x04,
-	 .stable_count = 0x66, .active_delay_count = 0x09,
-	 .xtal_freq_count = 0xFE},
-	{.osc_frequency = 16800000, .enable_delay_count = 0x03,
-	 .stable_count = 0x41, .active_delay_count = 0x0A,
-	 .xtal_freq_count = 0xA4},
+	{
+		.osc_frequency = 13000000, .enable_delay_count = 0x02,
+		.stable_count = 0x33, .active_delay_count = 0x05,
+		.xtal_freq_count = 0x7f
+	}, {
+		.osc_frequency = 19200000, .enable_delay_count = 0x03,
+		.stable_count = 0x4b, .active_delay_count = 0x06,
+		.xtal_freq_count = 0xbb
+	}, {
+		.osc_frequency = 12000000, .enable_delay_count = 0x02,
+		.stable_count = 0x2f, .active_delay_count = 0x04,
+		.xtal_freq_count = 0x76
+	}, {
+		.osc_frequency = 26000000, .enable_delay_count = 0x04,
+		.stable_count = 0x66, .active_delay_count = 0x09,
+		.xtal_freq_count = 0xfe
+	}, {
+		.osc_frequency = 16800000, .enable_delay_count = 0x03,
+		.stable_count = 0x41, .active_delay_count = 0x0a,
+		.xtal_freq_count = 0xa4
+	},
 };
 
 static struct tegra_clk tegra124_clks[tegra_clk_max] __initdata = {
@@ -1024,8 +1065,8 @@ static struct clk **clks;
 
 static void tegra124_utmi_param_configure(void __iomem *clk_base)
 {
+	unsigned int i;
 	u32 reg;
-	int i;
 
 	for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) {
 		if (osc_freq == utmi_parameters[i].osc_frequency)
@@ -1356,65 +1397,65 @@ static struct tegra_cpu_car_ops tegra124_cpu_car_ops = {
 
 static const struct of_device_id pmc_match[] __initconst = {
 	{ .compatible = "nvidia,tegra124-pmc" },
-	{},
+	{ },
 };
 
 static struct tegra_clk_init_table common_init_table[] __initdata = {
-	{TEGRA124_CLK_UARTA, TEGRA124_CLK_PLL_P, 408000000, 0},
-	{TEGRA124_CLK_UARTB, TEGRA124_CLK_PLL_P, 408000000, 0},
-	{TEGRA124_CLK_UARTC, TEGRA124_CLK_PLL_P, 408000000, 0},
-	{TEGRA124_CLK_UARTD, TEGRA124_CLK_PLL_P, 408000000, 0},
-	{TEGRA124_CLK_PLL_A, TEGRA124_CLK_CLK_MAX, 564480000, 1},
-	{TEGRA124_CLK_PLL_A_OUT0, TEGRA124_CLK_CLK_MAX, 11289600, 1},
-	{TEGRA124_CLK_EXTERN1, TEGRA124_CLK_PLL_A_OUT0, 0, 1},
-	{TEGRA124_CLK_CLK_OUT_1_MUX, TEGRA124_CLK_EXTERN1, 0, 1},
-	{TEGRA124_CLK_CLK_OUT_1, TEGRA124_CLK_CLK_MAX, 0, 1},
-	{TEGRA124_CLK_I2S0, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA124_CLK_I2S1, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA124_CLK_I2S2, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA124_CLK_I2S3, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA124_CLK_I2S4, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA124_CLK_VDE, TEGRA124_CLK_PLL_P, 0, 0},
-	{TEGRA124_CLK_HOST1X, TEGRA124_CLK_PLL_P, 136000000, 1},
-	{TEGRA124_CLK_DSIALP, TEGRA124_CLK_PLL_P, 68000000, 0},
-	{TEGRA124_CLK_DSIBLP, TEGRA124_CLK_PLL_P, 68000000, 0},
-	{TEGRA124_CLK_SCLK, TEGRA124_CLK_PLL_P_OUT2, 102000000, 1},
-	{TEGRA124_CLK_DFLL_SOC, TEGRA124_CLK_PLL_P, 51000000, 1},
-	{TEGRA124_CLK_DFLL_REF, TEGRA124_CLK_PLL_P, 51000000, 1},
-	{TEGRA124_CLK_PLL_C, TEGRA124_CLK_CLK_MAX, 768000000, 0},
-	{TEGRA124_CLK_PLL_C_OUT1, TEGRA124_CLK_CLK_MAX, 100000000, 0},
-	{TEGRA124_CLK_SBC4, TEGRA124_CLK_PLL_P, 12000000, 1},
-	{TEGRA124_CLK_TSEC, TEGRA124_CLK_PLL_C3, 0, 0},
-	{TEGRA124_CLK_MSENC, TEGRA124_CLK_PLL_C3, 0, 0},
-	{TEGRA124_CLK_PLL_RE_VCO, TEGRA124_CLK_CLK_MAX, 672000000, 0},
-	{TEGRA124_CLK_XUSB_SS_SRC, TEGRA124_CLK_PLL_U_480M, 120000000, 0},
-	{TEGRA124_CLK_XUSB_FS_SRC, TEGRA124_CLK_PLL_U_48M, 48000000, 0},
-	{TEGRA124_CLK_XUSB_HS_SRC, TEGRA124_CLK_PLL_U_60M, 60000000, 0},
-	{TEGRA124_CLK_XUSB_FALCON_SRC, TEGRA124_CLK_PLL_RE_OUT, 224000000, 0},
-	{TEGRA124_CLK_XUSB_HOST_SRC, TEGRA124_CLK_PLL_RE_OUT, 112000000, 0},
-	{TEGRA124_CLK_SATA, TEGRA124_CLK_PLL_P, 104000000, 0},
-	{TEGRA124_CLK_SATA_OOB, TEGRA124_CLK_PLL_P, 204000000, 0},
-	{TEGRA124_CLK_MSELECT, TEGRA124_CLK_CLK_MAX, 0, 1},
-	{TEGRA124_CLK_CSITE, TEGRA124_CLK_CLK_MAX, 0, 1},
-	{TEGRA124_CLK_TSENSOR, TEGRA124_CLK_CLK_M, 400000, 0},
-	/* This MUST be the last entry. */
-	{TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0},
+	{ TEGRA124_CLK_UARTA, TEGRA124_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA124_CLK_UARTB, TEGRA124_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA124_CLK_UARTC, TEGRA124_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA124_CLK_UARTD, TEGRA124_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA124_CLK_PLL_A, TEGRA124_CLK_CLK_MAX, 564480000, 1 },
+	{ TEGRA124_CLK_PLL_A_OUT0, TEGRA124_CLK_CLK_MAX, 11289600, 1 },
+	{ TEGRA124_CLK_EXTERN1, TEGRA124_CLK_PLL_A_OUT0, 0, 1 },
+	{ TEGRA124_CLK_CLK_OUT_1_MUX, TEGRA124_CLK_EXTERN1, 0, 1 },
+	{ TEGRA124_CLK_CLK_OUT_1, TEGRA124_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA124_CLK_I2S0, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA124_CLK_I2S1, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA124_CLK_I2S2, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA124_CLK_I2S3, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA124_CLK_I2S4, TEGRA124_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA124_CLK_VDE, TEGRA124_CLK_PLL_P, 0, 0 },
+	{ TEGRA124_CLK_HOST1X, TEGRA124_CLK_PLL_P, 136000000, 1 },
+	{ TEGRA124_CLK_DSIALP, TEGRA124_CLK_PLL_P, 68000000, 0 },
+	{ TEGRA124_CLK_DSIBLP, TEGRA124_CLK_PLL_P, 68000000, 0 },
+	{ TEGRA124_CLK_SCLK, TEGRA124_CLK_PLL_P_OUT2, 102000000, 1 },
+	{ TEGRA124_CLK_DFLL_SOC, TEGRA124_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA124_CLK_DFLL_REF, TEGRA124_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA124_CLK_PLL_C, TEGRA124_CLK_CLK_MAX, 768000000, 0 },
+	{ TEGRA124_CLK_PLL_C_OUT1, TEGRA124_CLK_CLK_MAX, 100000000, 0 },
+	{ TEGRA124_CLK_SBC4, TEGRA124_CLK_PLL_P, 12000000, 1 },
+	{ TEGRA124_CLK_TSEC, TEGRA124_CLK_PLL_C3, 0, 0 },
+	{ TEGRA124_CLK_MSENC, TEGRA124_CLK_PLL_C3, 0, 0 },
+	{ TEGRA124_CLK_PLL_RE_VCO, TEGRA124_CLK_CLK_MAX, 672000000, 0 },
+	{ TEGRA124_CLK_XUSB_SS_SRC, TEGRA124_CLK_PLL_U_480M, 120000000, 0 },
+	{ TEGRA124_CLK_XUSB_FS_SRC, TEGRA124_CLK_PLL_U_48M, 48000000, 0 },
+	{ TEGRA124_CLK_XUSB_HS_SRC, TEGRA124_CLK_PLL_U_60M, 60000000, 0 },
+	{ TEGRA124_CLK_XUSB_FALCON_SRC, TEGRA124_CLK_PLL_RE_OUT, 224000000, 0 },
+	{ TEGRA124_CLK_XUSB_HOST_SRC, TEGRA124_CLK_PLL_RE_OUT, 112000000, 0 },
+	{ TEGRA124_CLK_SATA, TEGRA124_CLK_PLL_P, 104000000, 0 },
+	{ TEGRA124_CLK_SATA_OOB, TEGRA124_CLK_PLL_P, 204000000, 0 },
+	{ TEGRA124_CLK_MSELECT, TEGRA124_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA124_CLK_CSITE, TEGRA124_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA124_CLK_TSENSOR, TEGRA124_CLK_CLK_M, 400000, 0 },
+	/* must be the last entry */
+	{ TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0 },
 };
 
 static struct tegra_clk_init_table tegra124_init_table[] __initdata = {
-	{TEGRA124_CLK_SOC_THERM, TEGRA124_CLK_PLL_P, 51000000, 0},
-	{TEGRA124_CLK_CCLK_G, TEGRA124_CLK_CLK_MAX, 0, 1},
-	{TEGRA124_CLK_HDA, TEGRA124_CLK_PLL_P, 102000000, 0},
-	{TEGRA124_CLK_HDA2CODEC_2X, TEGRA124_CLK_PLL_P, 48000000, 0},
-	/* This MUST be the last entry. */
-	{TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0},
+	{ TEGRA124_CLK_SOC_THERM, TEGRA124_CLK_PLL_P, 51000000, 0 },
+	{ TEGRA124_CLK_CCLK_G, TEGRA124_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA124_CLK_HDA, TEGRA124_CLK_PLL_P, 102000000, 0 },
+	{ TEGRA124_CLK_HDA2CODEC_2X, TEGRA124_CLK_PLL_P, 48000000, 0 },
+	/* must be the last entry */
+	{ TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0 },
 };
 
 /* Tegra132 requires the SOC_THERM clock to remain active */
 static struct tegra_clk_init_table tegra132_init_table[] __initdata = {
-	{TEGRA124_CLK_SOC_THERM, TEGRA124_CLK_PLL_P, 51000000, 1},
-	/* This MUST be the last entry. */
-	{TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0},
+	{ TEGRA124_CLK_SOC_THERM, TEGRA124_CLK_PLL_P, 51000000, 1 },
+	/* must be the last entry */
+	{ TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0 },
 };
 
 static struct tegra_audio_clk_info tegra124_audio_plls[] = {
diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c
index bf004f0e4f65..7a48e986c4c9 100644
--- a/drivers/clk/tegra/clk-tegra20.c
+++ b/drivers/clk/tegra/clk-tegra20.c
@@ -166,126 +166,120 @@ static DEFINE_SPINLOCK(emc_lock);
 static struct clk **clks;
 
 static struct tegra_clk_pll_freq_table pll_c_freq_table[] = {
-	{ 12000000, 600000000, 600, 12, 0, 8 },
-	{ 13000000, 600000000, 600, 13, 0, 8 },
-	{ 19200000, 600000000, 500, 16, 0, 6 },
-	{ 26000000, 600000000, 600, 26, 0, 8 },
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 600000000, 600, 12, 1, 8 },
+	{ 13000000, 600000000, 600, 13, 1, 8 },
+	{ 19200000, 600000000, 500, 16, 1, 6 },
+	{ 26000000, 600000000, 600, 26, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_m_freq_table[] = {
-	{ 12000000, 666000000, 666, 12, 0, 8},
-	{ 13000000, 666000000, 666, 13, 0, 8},
-	{ 19200000, 666000000, 555, 16, 0, 8},
-	{ 26000000, 666000000, 666, 26, 0, 8},
-	{ 12000000, 600000000, 600, 12, 0, 8},
-	{ 13000000, 600000000, 600, 13, 0, 8},
-	{ 19200000, 600000000, 375, 12, 0, 6},
-	{ 26000000, 600000000, 600, 26, 0, 8},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 666000000, 666, 12, 1, 8 },
+	{ 13000000, 666000000, 666, 13, 1, 8 },
+	{ 19200000, 666000000, 555, 16, 1, 8 },
+	{ 26000000, 666000000, 666, 26, 1, 8 },
+	{ 12000000, 600000000, 600, 12, 1, 8 },
+	{ 13000000, 600000000, 600, 13, 1, 8 },
+	{ 19200000, 600000000, 375, 12, 1, 6 },
+	{ 26000000, 600000000, 600, 26, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_p_freq_table[] = {
-	{ 12000000, 216000000, 432, 12, 1, 8},
-	{ 13000000, 216000000, 432, 13, 1, 8},
-	{ 19200000, 216000000, 90,   4, 1, 1},
-	{ 26000000, 216000000, 432, 26, 1, 8},
-	{ 12000000, 432000000, 432, 12, 0, 8},
-	{ 13000000, 432000000, 432, 13, 0, 8},
-	{ 19200000, 432000000, 90,   4, 0, 1},
-	{ 26000000, 432000000, 432, 26, 0, 8},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 216000000, 432, 12, 2, 8 },
+	{ 13000000, 216000000, 432, 13, 2, 8 },
+	{ 19200000, 216000000,  90,  4, 2, 1 },
+	{ 26000000, 216000000, 432, 26, 2, 8 },
+	{ 12000000, 432000000, 432, 12, 1, 8 },
+	{ 13000000, 432000000, 432, 13, 1, 8 },
+	{ 19200000, 432000000,  90,  4, 1, 1 },
+	{ 26000000, 432000000, 432, 26, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_a_freq_table[] = {
-	{ 28800000, 56448000, 49, 25, 0, 1},
-	{ 28800000, 73728000, 64, 25, 0, 1},
-	{ 28800000, 24000000,  5,  6, 0, 1},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 28800000, 56448000, 49, 25, 1, 1 },
+	{ 28800000, 73728000, 64, 25, 1, 1 },
+	{ 28800000, 24000000,  5,  6, 1, 1 },
+	{        0,        0,  0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_d_freq_table[] = {
-	{ 12000000, 216000000, 216, 12, 0, 4},
-	{ 13000000, 216000000, 216, 13, 0, 4},
-	{ 19200000, 216000000, 135, 12, 0, 3},
-	{ 26000000, 216000000, 216, 26, 0, 4},
-
-	{ 12000000, 594000000, 594, 12, 0, 8},
-	{ 13000000, 594000000, 594, 13, 0, 8},
-	{ 19200000, 594000000, 495, 16, 0, 8},
-	{ 26000000, 594000000, 594, 26, 0, 8},
-
-	{ 12000000, 1000000000, 1000, 12, 0, 12},
-	{ 13000000, 1000000000, 1000, 13, 0, 12},
-	{ 19200000, 1000000000, 625,  12, 0, 8},
-	{ 26000000, 1000000000, 1000, 26, 0, 12},
-
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000,  216000000,  216, 12, 1,  4 },
+	{ 13000000,  216000000,  216, 13, 1,  4 },
+	{ 19200000,  216000000,  135, 12, 1,  3 },
+	{ 26000000,  216000000,  216, 26, 1,  4 },
+	{ 12000000,  594000000,  594, 12, 1,  8 },
+	{ 13000000,  594000000,  594, 13, 1,  8 },
+	{ 19200000,  594000000,  495, 16, 1,  8 },
+	{ 26000000,  594000000,  594, 26, 1,  8 },
+	{ 12000000, 1000000000, 1000, 12, 1, 12 },
+	{ 13000000, 1000000000, 1000, 13, 1, 12 },
+	{ 19200000, 1000000000,  625, 12, 1,  8 },
+	{ 26000000, 1000000000, 1000, 26, 1, 12 },
+	{        0,          0,    0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
-	{ 12000000, 480000000, 960, 12, 0, 0},
-	{ 13000000, 480000000, 960, 13, 0, 0},
-	{ 19200000, 480000000, 200, 4,  0, 0},
-	{ 26000000, 480000000, 960, 26, 0, 0},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 480000000, 960, 12, 1, 0 },
+	{ 13000000, 480000000, 960, 13, 1, 0 },
+	{ 19200000, 480000000, 200,  4, 1, 0 },
+	{ 26000000, 480000000, 960, 26, 1, 0 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_x_freq_table[] = {
 	/* 1 GHz */
-	{ 12000000, 1000000000, 1000, 12, 0, 12},
-	{ 13000000, 1000000000, 1000, 13, 0, 12},
-	{ 19200000, 1000000000, 625,  12, 0, 8},
-	{ 26000000, 1000000000, 1000, 26, 0, 12},
-
+	{ 12000000, 1000000000, 1000, 12, 1, 12 },
+	{ 13000000, 1000000000, 1000, 13, 1, 12 },
+	{ 19200000, 1000000000,  625, 12, 1,  8 },
+	{ 26000000, 1000000000, 1000, 26, 1, 12 },
 	/* 912 MHz */
-	{ 12000000, 912000000,  912,  12, 0, 12},
-	{ 13000000, 912000000,  912,  13, 0, 12},
-	{ 19200000, 912000000,  760,  16, 0, 8},
-	{ 26000000, 912000000,  912,  26, 0, 12},
-
+	{ 12000000,  912000000,  912, 12, 1, 12 },
+	{ 13000000,  912000000,  912, 13, 1, 12 },
+	{ 19200000,  912000000,  760, 16, 1,  8 },
+	{ 26000000,  912000000,  912, 26, 1, 12 },
 	/* 816 MHz */
-	{ 12000000, 816000000,  816,  12, 0, 12},
-	{ 13000000, 816000000,  816,  13, 0, 12},
-	{ 19200000, 816000000,  680,  16, 0, 8},
-	{ 26000000, 816000000,  816,  26, 0, 12},
-
+	{ 12000000,  816000000,  816, 12, 1, 12 },
+	{ 13000000,  816000000,  816, 13, 1, 12 },
+	{ 19200000,  816000000,  680, 16, 1,  8 },
+	{ 26000000,  816000000,  816, 26, 1, 12 },
 	/* 760 MHz */
-	{ 12000000, 760000000,  760,  12, 0, 12},
-	{ 13000000, 760000000,  760,  13, 0, 12},
-	{ 19200000, 760000000,  950,  24, 0, 8},
-	{ 26000000, 760000000,  760,  26, 0, 12},
-
+	{ 12000000,  760000000,  760, 12, 1, 12 },
+	{ 13000000,  760000000,  760, 13, 1, 12 },
+	{ 19200000,  760000000,  950, 24, 1,  8 },
+	{ 26000000,  760000000,  760, 26, 1, 12 },
 	/* 750 MHz */
-	{ 12000000, 750000000,  750,  12, 0, 12},
-	{ 13000000, 750000000,  750,  13, 0, 12},
-	{ 19200000, 750000000,  625,  16, 0, 8},
-	{ 26000000, 750000000,  750,  26, 0, 12},
-
+	{ 12000000,  750000000,  750, 12, 1, 12 },
+	{ 13000000,  750000000,  750, 13, 1, 12 },
+	{ 19200000,  750000000,  625, 16, 1,  8 },
+	{ 26000000,  750000000,  750, 26, 1, 12 },
 	/* 608 MHz */
-	{ 12000000, 608000000,  608,  12, 0, 12},
-	{ 13000000, 608000000,  608,  13, 0, 12},
-	{ 19200000, 608000000,  380,  12, 0, 8},
-	{ 26000000, 608000000,  608,  26, 0, 12},
-
+	{ 12000000,  608000000,  608, 12, 1, 12 },
+	{ 13000000,  608000000,  608, 13, 1, 12 },
+	{ 19200000,  608000000,  380, 12, 1,  8 },
+	{ 26000000,  608000000,  608, 26, 1, 12 },
 	/* 456 MHz */
-	{ 12000000, 456000000,  456,  12, 0, 12},
-	{ 13000000, 456000000,  456,  13, 0, 12},
-	{ 19200000, 456000000,  380,  16, 0, 8},
-	{ 26000000, 456000000,  456,  26, 0, 12},
-
+	{ 12000000,  456000000,  456, 12, 1, 12 },
+	{ 13000000,  456000000,  456, 13, 1, 12 },
+	{ 19200000,  456000000,  380, 16, 1,  8 },
+	{ 26000000,  456000000,  456, 26, 1, 12 },
 	/* 312 MHz */
-	{ 12000000, 312000000,  312,  12, 0, 12},
-	{ 13000000, 312000000,  312,  13, 0, 12},
-	{ 19200000, 312000000,  260,  16, 0, 8},
-	{ 26000000, 312000000,  312,  26, 0, 12},
+	{ 12000000,  312000000,  312, 12, 1, 12 },
+	{ 13000000,  312000000,  312, 13, 1, 12 },
+	{ 19200000,  312000000,  260, 16, 1,  8 },
+	{ 26000000,  312000000,  312, 26, 1, 12 },
+	{        0,          0,    0,  0, 0,  0 },
+};
 
-	{ 0, 0, 0, 0, 0, 0 },
+static const struct pdiv_map plle_p[] = {
+	{ .pdiv = 1, .hw_val = 1 },
+	{ .pdiv = 0, .hw_val = 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
-	{ 12000000, 100000000,  200,  24, 0, 0 },
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 100000000, 200, 24, 1, 0 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 /* PLL parameters */
@@ -302,7 +296,7 @@ static struct tegra_clk_pll_params pll_c_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_c_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_m_params = {
@@ -318,7 +312,7 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_m_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_p_params = {
@@ -334,7 +328,8 @@ static struct tegra_clk_pll_params pll_p_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_p_freq_table,
-	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_CPCON |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate =  216000000,
 };
 
@@ -351,7 +346,7 @@ static struct tegra_clk_pll_params pll_a_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_a_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_d_params = {
@@ -367,10 +362,10 @@ static struct tegra_clk_pll_params pll_d_params = {
 	.lock_enable_bit_idx = PLLDU_MISC_LOCK_ENABLE,
 	.lock_delay = 1000,
 	.freq_table = pll_d_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
-static struct pdiv_map pllu_p[] = {
+static const struct pdiv_map pllu_p[] = {
 	{ .pdiv = 1, .hw_val = 1 },
 	{ .pdiv = 2, .hw_val = 0 },
 	{ .pdiv = 0, .hw_val = 0 },
@@ -390,7 +385,7 @@ static struct tegra_clk_pll_params pll_u_params = {
 	.lock_delay = 1000,
 	.pdiv_tohw = pllu_p,
 	.freq_table = pll_u_freq_table,
-	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_x_params = {
@@ -406,7 +401,7 @@ static struct tegra_clk_pll_params pll_x_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_x_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_e_params = {
@@ -421,8 +416,10 @@ static struct tegra_clk_pll_params pll_e_params = {
 	.lock_mask = PLLE_MISC_LOCK,
 	.lock_enable_bit_idx = PLLE_MISC_LOCK_ENABLE,
 	.lock_delay = 0,
+	.pdiv_tohw = plle_p,
 	.freq_table = pll_e_freq_table,
-	.flags = TEGRA_PLL_FIXED,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_LOCK_MISC |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate = 100000000,
 };
 
@@ -733,9 +730,9 @@ static void tegra20_super_clk_init(void)
 	clks[TEGRA20_CLK_TWD] = clk;
 }
 
-static const char *audio_parents[] = {"spdif_in", "i2s1", "i2s2", "unused",
-				      "pll_a_out0", "unused", "unused",
-				      "unused"};
+static const char *audio_parents[] = { "spdif_in", "i2s1", "i2s2", "unused",
+				       "pll_a_out0", "unused", "unused",
+				       "unused" };
 
 static void __init tegra20_audio_clk_init(void)
 {
@@ -759,19 +756,18 @@ static void __init tegra20_audio_clk_init(void)
 				    CLK_SET_RATE_PARENT, 89,
 				    periph_clk_enb_refcnt);
 	clks[TEGRA20_CLK_AUDIO_2X] = clk;
-
 }
 
-static const char *i2s1_parents[] = {"pll_a_out0", "audio_2x", "pll_p",
-				     "clk_m"};
-static const char *i2s2_parents[] = {"pll_a_out0", "audio_2x", "pll_p",
-				     "clk_m"};
-static const char *pwm_parents[] = {"pll_p", "pll_c", "audio", "clk_m",
-				    "clk_32k"};
-static const char *mux_pllpcm_clkm[] = {"pll_p", "pll_c", "pll_m", "clk_m"};
-static const char *mux_pllpdc_clkm[] = {"pll_p", "pll_d_out0", "pll_c",
-					"clk_m"};
-static const char *mux_pllmcp_clkm[] = {"pll_m", "pll_c", "pll_p", "clk_m"};
+static const char *i2s1_parents[] = { "pll_a_out0", "audio_2x", "pll_p",
+				      "clk_m" };
+static const char *i2s2_parents[] = { "pll_a_out0", "audio_2x", "pll_p",
+				      "clk_m" };
+static const char *pwm_parents[] = { "pll_p", "pll_c", "audio", "clk_m",
+				     "clk_32k" };
+static const char *mux_pllpcm_clkm[] = { "pll_p", "pll_c", "pll_m", "clk_m" };
+static const char *mux_pllpdc_clkm[] = { "pll_p", "pll_d_out0", "pll_c",
+					 "clk_m" };
+static const char *mux_pllmcp_clkm[] = { "pll_m", "pll_c", "pll_p", "clk_m" };
 
 static struct tegra_periph_init_data tegra_periph_clk_list[] = {
 	TEGRA_INIT_DATA_MUX("i2s1", i2s1_parents,     CLK_SOURCE_I2S1,   11, TEGRA_PERIPH_ON_APB, TEGRA20_CLK_I2S1),
@@ -802,7 +798,7 @@ static void __init tegra20_periph_clk_init(void)
 {
 	struct tegra_periph_init_data *data;
 	struct clk *clk;
-	int i;
+	unsigned int i;
 
 	/* ac97 */
 	clk = tegra_clk_register_periph_gate("ac97", "pll_a_out0",
@@ -1025,44 +1021,45 @@ static struct tegra_cpu_car_ops tegra20_cpu_car_ops = {
 };
 
 static struct tegra_clk_init_table init_table[] __initdata = {
-	{TEGRA20_CLK_PLL_P, TEGRA20_CLK_CLK_MAX, 216000000, 1},
-	{TEGRA20_CLK_PLL_P_OUT1, TEGRA20_CLK_CLK_MAX, 28800000, 1},
-	{TEGRA20_CLK_PLL_P_OUT2, TEGRA20_CLK_CLK_MAX, 48000000, 1},
-	{TEGRA20_CLK_PLL_P_OUT3, TEGRA20_CLK_CLK_MAX, 72000000, 1},
-	{TEGRA20_CLK_PLL_P_OUT4, TEGRA20_CLK_CLK_MAX, 24000000, 1},
-	{TEGRA20_CLK_PLL_C, TEGRA20_CLK_CLK_MAX, 600000000, 1},
-	{TEGRA20_CLK_PLL_C_OUT1, TEGRA20_CLK_CLK_MAX, 120000000, 1},
-	{TEGRA20_CLK_SCLK, TEGRA20_CLK_PLL_C_OUT1, 0, 1},
-	{TEGRA20_CLK_HCLK, TEGRA20_CLK_CLK_MAX, 0, 1},
-	{TEGRA20_CLK_PCLK, TEGRA20_CLK_CLK_MAX, 60000000, 1},
-	{TEGRA20_CLK_CSITE, TEGRA20_CLK_CLK_MAX, 0, 1},
-	{TEGRA20_CLK_EMC, TEGRA20_CLK_CLK_MAX, 0, 1},
-	{TEGRA20_CLK_CCLK, TEGRA20_CLK_CLK_MAX, 0, 1},
-	{TEGRA20_CLK_UARTA, TEGRA20_CLK_PLL_P, 0, 0},
-	{TEGRA20_CLK_UARTB, TEGRA20_CLK_PLL_P, 0, 0},
-	{TEGRA20_CLK_UARTC, TEGRA20_CLK_PLL_P, 0, 0},
-	{TEGRA20_CLK_UARTD, TEGRA20_CLK_PLL_P, 0, 0},
-	{TEGRA20_CLK_UARTE, TEGRA20_CLK_PLL_P, 0, 0},
-	{TEGRA20_CLK_PLL_A, TEGRA20_CLK_CLK_MAX, 56448000, 1},
-	{TEGRA20_CLK_PLL_A_OUT0, TEGRA20_CLK_CLK_MAX, 11289600, 1},
-	{TEGRA20_CLK_CDEV1, TEGRA20_CLK_CLK_MAX, 0, 1},
-	{TEGRA20_CLK_BLINK, TEGRA20_CLK_CLK_MAX, 32768, 1},
-	{TEGRA20_CLK_I2S1, TEGRA20_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA20_CLK_I2S2, TEGRA20_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA20_CLK_SDMMC1, TEGRA20_CLK_PLL_P, 48000000, 0},
-	{TEGRA20_CLK_SDMMC3, TEGRA20_CLK_PLL_P, 48000000, 0},
-	{TEGRA20_CLK_SDMMC4, TEGRA20_CLK_PLL_P, 48000000, 0},
-	{TEGRA20_CLK_SPI, TEGRA20_CLK_PLL_P, 20000000, 0},
-	{TEGRA20_CLK_SBC1, TEGRA20_CLK_PLL_P, 100000000, 0},
-	{TEGRA20_CLK_SBC2, TEGRA20_CLK_PLL_P, 100000000, 0},
-	{TEGRA20_CLK_SBC3, TEGRA20_CLK_PLL_P, 100000000, 0},
-	{TEGRA20_CLK_SBC4, TEGRA20_CLK_PLL_P, 100000000, 0},
-	{TEGRA20_CLK_HOST1X, TEGRA20_CLK_PLL_C, 150000000, 0},
-	{TEGRA20_CLK_DISP1, TEGRA20_CLK_PLL_P, 600000000, 0},
-	{TEGRA20_CLK_DISP2, TEGRA20_CLK_PLL_P, 600000000, 0},
-	{TEGRA20_CLK_GR2D, TEGRA20_CLK_PLL_C, 300000000, 0},
-	{TEGRA20_CLK_GR3D, TEGRA20_CLK_PLL_C, 300000000, 0},
-	{TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0}, /* This MUST be the last entry */
+	{ TEGRA20_CLK_PLL_P, TEGRA20_CLK_CLK_MAX, 216000000, 1 },
+	{ TEGRA20_CLK_PLL_P_OUT1, TEGRA20_CLK_CLK_MAX, 28800000, 1 },
+	{ TEGRA20_CLK_PLL_P_OUT2, TEGRA20_CLK_CLK_MAX, 48000000, 1 },
+	{ TEGRA20_CLK_PLL_P_OUT3, TEGRA20_CLK_CLK_MAX, 72000000, 1 },
+	{ TEGRA20_CLK_PLL_P_OUT4, TEGRA20_CLK_CLK_MAX, 24000000, 1 },
+	{ TEGRA20_CLK_PLL_C, TEGRA20_CLK_CLK_MAX, 600000000, 1 },
+	{ TEGRA20_CLK_PLL_C_OUT1, TEGRA20_CLK_CLK_MAX, 120000000, 1 },
+	{ TEGRA20_CLK_SCLK, TEGRA20_CLK_PLL_C_OUT1, 0, 1 },
+	{ TEGRA20_CLK_HCLK, TEGRA20_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA20_CLK_PCLK, TEGRA20_CLK_CLK_MAX, 60000000, 1 },
+	{ TEGRA20_CLK_CSITE, TEGRA20_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA20_CLK_EMC, TEGRA20_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA20_CLK_CCLK, TEGRA20_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA20_CLK_UARTA, TEGRA20_CLK_PLL_P, 0, 0 },
+	{ TEGRA20_CLK_UARTB, TEGRA20_CLK_PLL_P, 0, 0 },
+	{ TEGRA20_CLK_UARTC, TEGRA20_CLK_PLL_P, 0, 0 },
+	{ TEGRA20_CLK_UARTD, TEGRA20_CLK_PLL_P, 0, 0 },
+	{ TEGRA20_CLK_UARTE, TEGRA20_CLK_PLL_P, 0, 0 },
+	{ TEGRA20_CLK_PLL_A, TEGRA20_CLK_CLK_MAX, 56448000, 1 },
+	{ TEGRA20_CLK_PLL_A_OUT0, TEGRA20_CLK_CLK_MAX, 11289600, 1 },
+	{ TEGRA20_CLK_CDEV1, TEGRA20_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA20_CLK_BLINK, TEGRA20_CLK_CLK_MAX, 32768, 1 },
+	{ TEGRA20_CLK_I2S1, TEGRA20_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA20_CLK_I2S2, TEGRA20_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA20_CLK_SDMMC1, TEGRA20_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA20_CLK_SDMMC3, TEGRA20_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA20_CLK_SDMMC4, TEGRA20_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA20_CLK_SPI, TEGRA20_CLK_PLL_P, 20000000, 0 },
+	{ TEGRA20_CLK_SBC1, TEGRA20_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA20_CLK_SBC2, TEGRA20_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA20_CLK_SBC3, TEGRA20_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA20_CLK_SBC4, TEGRA20_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA20_CLK_HOST1X, TEGRA20_CLK_PLL_C, 150000000, 0 },
+	{ TEGRA20_CLK_DISP1, TEGRA20_CLK_PLL_P, 600000000, 0 },
+	{ TEGRA20_CLK_DISP2, TEGRA20_CLK_PLL_P, 600000000, 0 },
+	{ TEGRA20_CLK_GR2D, TEGRA20_CLK_PLL_C, 300000000, 0 },
+	{ TEGRA20_CLK_GR3D, TEGRA20_CLK_PLL_C, 300000000, 0 },
+	/* must be the last entry */
+	{ TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0 },
 };
 
 static void __init tegra20_clock_apply_init_table(void)
@@ -1076,16 +1073,17 @@ static void __init tegra20_clock_apply_init_table(void)
  * table under two names.
  */
 static struct tegra_clk_duplicate tegra_clk_duplicates[] = {
-	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,   "utmip-pad",    NULL),
-	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,   "tegra-ehci.0", NULL),
-	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,   "tegra-otg",    NULL),
-	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_CCLK,   NULL,           "cpu"),
-	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_CLK_MAX, NULL, NULL), /* Must be the last entry */
+	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,    "utmip-pad",     NULL),
+	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,    "tegra-ehci.0",  NULL),
+	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_USBD,    "tegra-otg",     NULL),
+	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_CCLK,    NULL,           "cpu"),
+	/* must be the last entry */
+	TEGRA_CLK_DUPLICATE(TEGRA20_CLK_CLK_MAX, NULL,            NULL),
 };
 
 static const struct of_device_id pmc_match[] __initconst = {
 	{ .compatible = "nvidia,tegra20-pmc" },
-	{},
+	{ },
 };
 
 static void __init tegra20_clock_init(struct device_node *np)
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
new file mode 100644
index 000000000000..58514c44ea83
--- /dev/null
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -0,0 +1,2852 @@
+/*
+ * Copyright (c) 2012-2014 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/clkdev.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/clk/tegra.h>
+#include <dt-bindings/clock/tegra210-car.h>
+
+#include "clk.h"
+#include "clk-id.h"
+
+/*
+ * TEGRA210_CAR_BANK_COUNT: the number of peripheral clock register
+ * banks present in the Tegra210 CAR IP block.  The banks are
+ * identified by single letters, e.g.: L, H, U, V, W, X, Y.  See
+ * periph_regs[] in drivers/clk/tegra/clk.c
+ */
+#define TEGRA210_CAR_BANK_COUNT			7
+
+#define CLK_SOURCE_CSITE 0x1d4
+#define CLK_SOURCE_EMC 0x19c
+
+#define PLLC_BASE 0x80
+#define PLLC_OUT 0x84
+#define PLLC_MISC0 0x88
+#define PLLC_MISC1 0x8c
+#define PLLC_MISC2 0x5d0
+#define PLLC_MISC3 0x5d4
+
+#define PLLC2_BASE 0x4e8
+#define PLLC2_MISC0 0x4ec
+#define PLLC2_MISC1 0x4f0
+#define PLLC2_MISC2 0x4f4
+#define PLLC2_MISC3 0x4f8
+
+#define PLLC3_BASE 0x4fc
+#define PLLC3_MISC0 0x500
+#define PLLC3_MISC1 0x504
+#define PLLC3_MISC2 0x508
+#define PLLC3_MISC3 0x50c
+
+#define PLLM_BASE 0x90
+#define PLLM_MISC0 0x9c
+#define PLLM_MISC1 0x98
+#define PLLP_BASE 0xa0
+#define PLLP_MISC0 0xac
+#define PLLP_MISC1 0x680
+#define PLLA_BASE 0xb0
+#define PLLA_MISC0 0xbc
+#define PLLA_MISC1 0xb8
+#define PLLA_MISC2 0x5d8
+#define PLLD_BASE 0xd0
+#define PLLD_MISC0 0xdc
+#define PLLD_MISC1 0xd8
+#define PLLU_BASE 0xc0
+#define PLLU_OUTA 0xc4
+#define PLLU_MISC0 0xcc
+#define PLLU_MISC1 0xc8
+#define PLLX_BASE 0xe0
+#define PLLX_MISC0 0xe4
+#define PLLX_MISC1 0x510
+#define PLLX_MISC2 0x514
+#define PLLX_MISC3 0x518
+#define PLLX_MISC4 0x5f0
+#define PLLX_MISC5 0x5f4
+#define PLLE_BASE 0xe8
+#define PLLE_MISC0 0xec
+#define PLLD2_BASE 0x4b8
+#define PLLD2_MISC0 0x4bc
+#define PLLD2_MISC1 0x570
+#define PLLD2_MISC2 0x574
+#define PLLD2_MISC3 0x578
+#define PLLE_AUX 0x48c
+#define PLLRE_BASE 0x4c4
+#define PLLRE_MISC0 0x4c8
+#define PLLDP_BASE 0x590
+#define PLLDP_MISC 0x594
+
+#define PLLC4_BASE 0x5a4
+#define PLLC4_MISC0 0x5a8
+#define PLLC4_OUT 0x5e4
+#define PLLMB_BASE 0x5e8
+#define PLLMB_MISC0 0x5ec
+#define PLLA1_BASE 0x6a4
+#define PLLA1_MISC0 0x6a8
+#define PLLA1_MISC1 0x6ac
+#define PLLA1_MISC2 0x6b0
+#define PLLA1_MISC3 0x6b4
+
+#define PLLU_IDDQ_BIT 31
+#define PLLCX_IDDQ_BIT 27
+#define PLLRE_IDDQ_BIT 24
+#define PLLA_IDDQ_BIT 25
+#define PLLD_IDDQ_BIT 20
+#define PLLSS_IDDQ_BIT 18
+#define PLLM_IDDQ_BIT 5
+#define PLLMB_IDDQ_BIT 17
+#define PLLXP_IDDQ_BIT 3
+
+#define PLLCX_RESET_BIT 30
+
+#define PLL_BASE_LOCK BIT(27)
+#define PLLCX_BASE_LOCK BIT(26)
+#define PLLE_MISC_LOCK BIT(11)
+#define PLLRE_MISC_LOCK BIT(27)
+
+#define PLL_MISC_LOCK_ENABLE 18
+#define PLLC_MISC_LOCK_ENABLE 24
+#define PLLDU_MISC_LOCK_ENABLE 22
+#define PLLU_MISC_LOCK_ENABLE 29
+#define PLLE_MISC_LOCK_ENABLE 9
+#define PLLRE_MISC_LOCK_ENABLE 30
+#define PLLSS_MISC_LOCK_ENABLE 30
+#define PLLP_MISC_LOCK_ENABLE 18
+#define PLLM_MISC_LOCK_ENABLE 4
+#define PLLMB_MISC_LOCK_ENABLE 16
+#define PLLA_MISC_LOCK_ENABLE 28
+#define PLLU_MISC_LOCK_ENABLE 29
+#define PLLD_MISC_LOCK_ENABLE 18
+
+#define PLLA_SDM_DIN_MASK 0xffff
+#define PLLA_SDM_EN_MASK BIT(26)
+
+#define PLLD_SDM_EN_MASK BIT(16)
+
+#define PLLD2_SDM_EN_MASK BIT(31)
+#define PLLD2_SSC_EN_MASK BIT(30)
+
+#define PLLDP_SS_CFG	0x598
+#define PLLDP_SDM_EN_MASK BIT(31)
+#define PLLDP_SSC_EN_MASK BIT(30)
+#define PLLDP_SS_CTRL1	0x59c
+#define PLLDP_SS_CTRL2	0x5a0
+
+#define PMC_PLLM_WB0_OVERRIDE 0x1dc
+#define PMC_PLLM_WB0_OVERRIDE_2 0x2b0
+
+#define UTMIP_PLL_CFG2 0x488
+#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xfff) << 6)
+#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP BIT(1)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP BIT(3)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERUP BIT(5)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN BIT(24)
+#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP BIT(25)
+
+#define UTMIP_PLL_CFG1 0x484
+#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 27)
+#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0)
+#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP BIT(17)
+#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16)
+#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP BIT(15)
+#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14)
+#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12)
+
+#define UTMIPLL_HW_PWRDN_CFG0			0x52c
+#define UTMIPLL_HW_PWRDN_CFG0_UTMIPLL_LOCK	BIT(31)
+#define UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE	BIT(25)
+#define UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE	BIT(24)
+#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE	BIT(7)
+#define UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET	BIT(6)
+#define UTMIPLL_HW_PWRDN_CFG0_SEQ_RESET_INPUT_VALUE	BIT(5)
+#define UTMIPLL_HW_PWRDN_CFG0_SEQ_IN_SWCTL	BIT(4)
+#define UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL	BIT(2)
+#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE	BIT(1)
+#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL	BIT(0)
+
+#define PLLU_HW_PWRDN_CFG0			0x530
+#define PLLU_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE	BIT(28)
+#define PLLU_HW_PWRDN_CFG0_SEQ_ENABLE		BIT(24)
+#define PLLU_HW_PWRDN_CFG0_USE_SWITCH_DETECT	BIT(7)
+#define PLLU_HW_PWRDN_CFG0_USE_LOCKDET		BIT(6)
+#define PLLU_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL	BIT(2)
+#define PLLU_HW_PWRDN_CFG0_CLK_SWITCH_SWCTL	BIT(0)
+
+#define XUSB_PLL_CFG0				0x534
+#define XUSB_PLL_CFG0_UTMIPLL_LOCK_DLY		0x3ff
+#define XUSB_PLL_CFG0_PLLU_LOCK_DLY_MASK	(0x3ff << 14)
+
+#define SPARE_REG0 0x55c
+#define CLK_M_DIVISOR_SHIFT 2
+#define CLK_M_DIVISOR_MASK 0x3
+
+/*
+ * SDM fractional divisor is 16-bit 2's complement signed number within
+ * (-2^12 ... 2^12-1) range. Represented in PLL data structure as unsigned
+ * 16-bit value, with "0" divisor mapped to 0xFFFF. Data "0" is used to
+ * indicate that SDM is disabled.
+ *
+ * Effective ndiv value when SDM is enabled: ndiv + 1/2 + sdm_din/2^13
+ */
+#define PLL_SDM_COEFF BIT(13)
+#define sdin_din_to_data(din)	((u16)((din) ? : 0xFFFFU))
+#define sdin_data_to_din(dat)	(((dat) == 0xFFFFU) ? 0 : (s16)dat)
+
+/* Tegra CPU clock and reset control regs */
+#define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS	0x470
+
+#ifdef CONFIG_PM_SLEEP
+static struct cpu_clk_suspend_context {
+	u32 clk_csite_src;
+} tegra210_cpu_clk_sctx;
+#endif
+
+static void __iomem *clk_base;
+static void __iomem *pmc_base;
+
+static unsigned long osc_freq;
+static unsigned long pll_ref_freq;
+
+static DEFINE_SPINLOCK(pll_d_lock);
+static DEFINE_SPINLOCK(pll_e_lock);
+static DEFINE_SPINLOCK(pll_re_lock);
+static DEFINE_SPINLOCK(pll_u_lock);
+static DEFINE_SPINLOCK(emc_lock);
+
+/* possible OSC frequencies in Hz */
+static unsigned long tegra210_input_freq[] = {
+	[5] = 38400000,
+	[8] = 12000000,
+};
+
+static const char *mux_pllmcp_clkm[] = {
+	"pll_m", "pll_c", "pll_p", "clk_m", "pll_m_ud", "pll_c2", "pll_c3",
+};
+#define mux_pllmcp_clkm_idx NULL
+
+#define PLL_ENABLE			(1 << 30)
+
+#define PLLCX_MISC1_IDDQ		(1 << 27)
+#define PLLCX_MISC0_RESET		(1 << 30)
+
+#define PLLCX_MISC0_DEFAULT_VALUE	0x40080000
+#define PLLCX_MISC0_WRITE_MASK		0x400ffffb
+#define PLLCX_MISC1_DEFAULT_VALUE	0x08000000
+#define PLLCX_MISC1_WRITE_MASK		0x08003cff
+#define PLLCX_MISC2_DEFAULT_VALUE	0x1f720f05
+#define PLLCX_MISC2_WRITE_MASK		0xffffff17
+#define PLLCX_MISC3_DEFAULT_VALUE	0x000000c4
+#define PLLCX_MISC3_WRITE_MASK		0x00ffffff
+
+/* PLLA */
+#define PLLA_BASE_IDDQ			(1 << 25)
+#define PLLA_BASE_LOCK			(1 << 27)
+
+#define PLLA_MISC0_LOCK_ENABLE		(1 << 28)
+#define PLLA_MISC0_LOCK_OVERRIDE	(1 << 27)
+
+#define PLLA_MISC2_EN_SDM		(1 << 26)
+#define PLLA_MISC2_EN_DYNRAMP		(1 << 25)
+
+#define PLLA_MISC0_DEFAULT_VALUE	0x12000020
+#define PLLA_MISC0_WRITE_MASK		0x7fffffff
+#define PLLA_MISC2_DEFAULT_VALUE	0x0
+#define PLLA_MISC2_WRITE_MASK		0x06ffffff
+
+/* PLLD */
+#define PLLD_MISC0_EN_SDM		(1 << 16)
+#define PLLD_MISC0_LOCK_OVERRIDE	(1 << 17)
+#define PLLD_MISC0_LOCK_ENABLE		(1 << 18)
+#define PLLD_MISC0_IDDQ			(1 << 20)
+#define PLLD_MISC0_DSI_CLKENABLE	(1 << 21)
+
+#define PLLD_MISC0_DEFAULT_VALUE	0x00140000
+#define PLLD_MISC0_WRITE_MASK		0x3ff7ffff
+#define PLLD_MISC1_DEFAULT_VALUE	0x20
+#define PLLD_MISC1_WRITE_MASK		0x00ffffff
+
+/* PLLD2 and PLLDP  and PLLC4 */
+#define PLLDSS_BASE_LOCK		(1 << 27)
+#define PLLDSS_BASE_LOCK_OVERRIDE	(1 << 24)
+#define PLLDSS_BASE_IDDQ		(1 << 18)
+#define PLLDSS_BASE_REF_SEL_SHIFT	25
+#define PLLDSS_BASE_REF_SEL_MASK	(0x3 << PLLDSS_BASE_REF_SEL_SHIFT)
+
+#define PLLDSS_MISC0_LOCK_ENABLE	(1 << 30)
+
+#define PLLDSS_MISC1_CFG_EN_SDM		(1 << 31)
+#define PLLDSS_MISC1_CFG_EN_SSC		(1 << 30)
+
+#define PLLD2_MISC0_DEFAULT_VALUE	0x40000020
+#define PLLD2_MISC1_CFG_DEFAULT_VALUE	0x10000000
+#define PLLD2_MISC2_CTRL1_DEFAULT_VALUE	0x0
+#define PLLD2_MISC3_CTRL2_DEFAULT_VALUE	0x0
+
+#define PLLDP_MISC0_DEFAULT_VALUE	0x40000020
+#define PLLDP_MISC1_CFG_DEFAULT_VALUE	0xc0000000
+#define PLLDP_MISC2_CTRL1_DEFAULT_VALUE	0xf400f0da
+#define PLLDP_MISC3_CTRL2_DEFAULT_VALUE	0x2004f400
+
+#define PLLDSS_MISC0_WRITE_MASK		0x47ffffff
+#define PLLDSS_MISC1_CFG_WRITE_MASK	0xf8000000
+#define PLLDSS_MISC2_CTRL1_WRITE_MASK	0xffffffff
+#define PLLDSS_MISC3_CTRL2_WRITE_MASK	0xffffffff
+
+#define PLLC4_MISC0_DEFAULT_VALUE	0x40000000
+
+/* PLLRE */
+#define PLLRE_MISC0_LOCK_ENABLE		(1 << 30)
+#define PLLRE_MISC0_LOCK_OVERRIDE	(1 << 29)
+#define PLLRE_MISC0_LOCK		(1 << 27)
+#define PLLRE_MISC0_IDDQ		(1 << 24)
+
+#define PLLRE_BASE_DEFAULT_VALUE	0x0
+#define PLLRE_MISC0_DEFAULT_VALUE	0x41000000
+
+#define PLLRE_BASE_DEFAULT_MASK		0x1c000000
+#define PLLRE_MISC0_WRITE_MASK		0x67ffffff
+
+/* PLLX */
+#define PLLX_USE_DYN_RAMP		1
+#define PLLX_BASE_LOCK			(1 << 27)
+
+#define PLLX_MISC0_FO_G_DISABLE		(0x1 << 28)
+#define PLLX_MISC0_LOCK_ENABLE		(0x1 << 18)
+
+#define PLLX_MISC2_DYNRAMP_STEPB_SHIFT	24
+#define PLLX_MISC2_DYNRAMP_STEPB_MASK	(0xFF << PLLX_MISC2_DYNRAMP_STEPB_SHIFT)
+#define PLLX_MISC2_DYNRAMP_STEPA_SHIFT	16
+#define PLLX_MISC2_DYNRAMP_STEPA_MASK	(0xFF << PLLX_MISC2_DYNRAMP_STEPA_SHIFT)
+#define PLLX_MISC2_NDIV_NEW_SHIFT	8
+#define PLLX_MISC2_NDIV_NEW_MASK	(0xFF << PLLX_MISC2_NDIV_NEW_SHIFT)
+#define PLLX_MISC2_LOCK_OVERRIDE	(0x1 << 4)
+#define PLLX_MISC2_DYNRAMP_DONE		(0x1 << 2)
+#define PLLX_MISC2_EN_DYNRAMP		(0x1 << 0)
+
+#define PLLX_MISC3_IDDQ			(0x1 << 3)
+
+#define PLLX_MISC0_DEFAULT_VALUE	PLLX_MISC0_LOCK_ENABLE
+#define PLLX_MISC0_WRITE_MASK		0x10c40000
+#define PLLX_MISC1_DEFAULT_VALUE	0x20
+#define PLLX_MISC1_WRITE_MASK		0x00ffffff
+#define PLLX_MISC2_DEFAULT_VALUE	0x0
+#define PLLX_MISC2_WRITE_MASK		0xffffff11
+#define PLLX_MISC3_DEFAULT_VALUE	PLLX_MISC3_IDDQ
+#define PLLX_MISC3_WRITE_MASK		0x01ff0f0f
+#define PLLX_MISC4_DEFAULT_VALUE	0x0
+#define PLLX_MISC4_WRITE_MASK		0x8000ffff
+#define PLLX_MISC5_DEFAULT_VALUE	0x0
+#define PLLX_MISC5_WRITE_MASK		0x0000ffff
+
+#define PLLX_HW_CTRL_CFG		0x548
+#define PLLX_HW_CTRL_CFG_SWCTRL		(0x1 << 0)
+
+/* PLLMB */
+#define PLLMB_BASE_LOCK			(1 << 27)
+
+#define PLLMB_MISC0_LOCK_OVERRIDE	(1 << 18)
+#define PLLMB_MISC0_IDDQ		(1 << 17)
+#define PLLMB_MISC0_LOCK_ENABLE		(1 << 16)
+
+#define PLLMB_MISC0_DEFAULT_VALUE	0x00030000
+#define PLLMB_MISC0_WRITE_MASK		0x0007ffff
+
+/* PLLP */
+#define PLLP_BASE_OVERRIDE		(1 << 28)
+#define PLLP_BASE_LOCK			(1 << 27)
+
+#define PLLP_MISC0_LOCK_ENABLE		(1 << 18)
+#define PLLP_MISC0_LOCK_OVERRIDE	(1 << 17)
+#define PLLP_MISC0_IDDQ			(1 << 3)
+
+#define PLLP_MISC1_HSIO_EN_SHIFT	29
+#define PLLP_MISC1_HSIO_EN		(1 << PLLP_MISC1_HSIO_EN_SHIFT)
+#define PLLP_MISC1_XUSB_EN_SHIFT	28
+#define PLLP_MISC1_XUSB_EN		(1 << PLLP_MISC1_XUSB_EN_SHIFT)
+
+#define PLLP_MISC0_DEFAULT_VALUE	0x00040008
+#define PLLP_MISC1_DEFAULT_VALUE	0x0
+
+#define PLLP_MISC0_WRITE_MASK		0xdc6000f
+#define PLLP_MISC1_WRITE_MASK		0x70ffffff
+
+/* PLLU */
+#define PLLU_BASE_LOCK			(1 << 27)
+#define PLLU_BASE_OVERRIDE		(1 << 24)
+#define PLLU_BASE_CLKENABLE_USB		(1 << 21)
+#define PLLU_BASE_CLKENABLE_HSIC	(1 << 22)
+#define PLLU_BASE_CLKENABLE_ICUSB	(1 << 23)
+#define PLLU_BASE_CLKENABLE_48M		(1 << 25)
+#define PLLU_BASE_CLKENABLE_ALL		(PLLU_BASE_CLKENABLE_USB |\
+					 PLLU_BASE_CLKENABLE_HSIC |\
+					 PLLU_BASE_CLKENABLE_ICUSB |\
+					 PLLU_BASE_CLKENABLE_48M)
+
+#define PLLU_MISC0_IDDQ			(1 << 31)
+#define PLLU_MISC0_LOCK_ENABLE		(1 << 29)
+#define PLLU_MISC1_LOCK_OVERRIDE	(1 << 0)
+
+#define PLLU_MISC0_DEFAULT_VALUE	0xa0000000
+#define PLLU_MISC1_DEFAULT_VALUE	0x0
+
+#define PLLU_MISC0_WRITE_MASK		0xbfffffff
+#define PLLU_MISC1_WRITE_MASK		0x00000007
+
+static inline void _pll_misc_chk_default(void __iomem *base,
+					struct tegra_clk_pll_params *params,
+					u8 misc_num, u32 default_val, u32 mask)
+{
+	u32 boot_val = readl_relaxed(base + params->ext_misc_reg[misc_num]);
+
+	boot_val &= mask;
+	default_val &= mask;
+	if (boot_val != default_val) {
+		pr_warn("boot misc%d 0x%x: expected 0x%x\n",
+			misc_num, boot_val, default_val);
+		pr_warn(" (comparison mask = 0x%x)\n", mask);
+		params->defaults_set = false;
+	}
+}
+
+/*
+ * PLLCX: PLLC, PLLC2, PLLC3, PLLA1
+ * Hybrid PLLs with dynamic ramp. Dynamic ramp is allowed for any transition
+ * that changes NDIV only, while PLL is already locked.
+ */
+static void pllcx_check_defaults(struct tegra_clk_pll_params *params)
+{
+	u32 default_val;
+
+	default_val = PLLCX_MISC0_DEFAULT_VALUE & (~PLLCX_MISC0_RESET);
+	_pll_misc_chk_default(clk_base, params, 0, default_val,
+			PLLCX_MISC0_WRITE_MASK);
+
+	default_val = PLLCX_MISC1_DEFAULT_VALUE & (~PLLCX_MISC1_IDDQ);
+	_pll_misc_chk_default(clk_base, params, 1, default_val,
+			PLLCX_MISC1_WRITE_MASK);
+
+	default_val = PLLCX_MISC2_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, params, 2, default_val,
+			PLLCX_MISC2_WRITE_MASK);
+
+	default_val = PLLCX_MISC3_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, params, 3, default_val,
+			PLLCX_MISC3_WRITE_MASK);
+}
+
+void tegra210_pllcx_set_defaults(const char *name, struct tegra_clk_pll *pllcx)
+{
+	pllcx->params->defaults_set = true;
+
+	if (readl_relaxed(clk_base + pllcx->params->base_reg) &
+			PLL_ENABLE) {
+		/* PLL is ON: only check if defaults already set */
+		pllcx_check_defaults(pllcx->params);
+		pr_warn("%s already enabled. Postponing set full defaults\n",
+			name);
+		return;
+	}
+
+	/* Defaults assert PLL reset, and set IDDQ */
+	writel_relaxed(PLLCX_MISC0_DEFAULT_VALUE,
+			clk_base + pllcx->params->ext_misc_reg[0]);
+	writel_relaxed(PLLCX_MISC1_DEFAULT_VALUE,
+			clk_base + pllcx->params->ext_misc_reg[1]);
+	writel_relaxed(PLLCX_MISC2_DEFAULT_VALUE,
+			clk_base + pllcx->params->ext_misc_reg[2]);
+	writel_relaxed(PLLCX_MISC3_DEFAULT_VALUE,
+			clk_base + pllcx->params->ext_misc_reg[3]);
+	udelay(1);
+}
+
+void _pllc_set_defaults(struct tegra_clk_pll *pllcx)
+{
+	tegra210_pllcx_set_defaults("PLL_C", pllcx);
+}
+
+void _pllc2_set_defaults(struct tegra_clk_pll *pllcx)
+{
+	tegra210_pllcx_set_defaults("PLL_C2", pllcx);
+}
+
+void _pllc3_set_defaults(struct tegra_clk_pll *pllcx)
+{
+	tegra210_pllcx_set_defaults("PLL_C3", pllcx);
+}
+
+void _plla1_set_defaults(struct tegra_clk_pll *pllcx)
+{
+	tegra210_pllcx_set_defaults("PLL_A1", pllcx);
+}
+
+/*
+ * PLLA
+ * PLL with dynamic ramp and fractional SDM. Dynamic ramp is not used.
+ * Fractional SDM is allowed to provide exact audio rates.
+ */
+void tegra210_plla_set_defaults(struct tegra_clk_pll *plla)
+{
+	u32 mask;
+	u32 val = readl_relaxed(clk_base + plla->params->base_reg);
+
+	plla->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		if (val & PLLA_BASE_IDDQ) {
+			pr_warn("PLL_A boot enabled with IDDQ set\n");
+			plla->params->defaults_set = false;
+		}
+
+		pr_warn("PLL_A already enabled. Postponing set full defaults\n");
+
+		val = PLLA_MISC0_DEFAULT_VALUE;	/* ignore lock enable */
+		mask = PLLA_MISC0_LOCK_ENABLE | PLLA_MISC0_LOCK_OVERRIDE;
+		_pll_misc_chk_default(clk_base, plla->params, 0, val,
+				~mask & PLLA_MISC0_WRITE_MASK);
+
+		val = PLLA_MISC2_DEFAULT_VALUE; /* ignore all but control bit */
+		_pll_misc_chk_default(clk_base, plla->params, 2, val,
+				PLLA_MISC2_EN_DYNRAMP);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + plla->params->ext_misc_reg[0]);
+		val &= ~mask;
+		val |= PLLA_MISC0_DEFAULT_VALUE & mask;
+		writel_relaxed(val, clk_base + plla->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect, disable dynamic ramp and SDM */
+	val |= PLLA_BASE_IDDQ;
+	writel_relaxed(val, clk_base + plla->params->base_reg);
+	writel_relaxed(PLLA_MISC0_DEFAULT_VALUE,
+			clk_base + plla->params->ext_misc_reg[0]);
+	writel_relaxed(PLLA_MISC2_DEFAULT_VALUE,
+			clk_base + plla->params->ext_misc_reg[2]);
+	udelay(1);
+}
+
+/*
+ * PLLD
+ * PLL with fractional SDM.
+ */
+void tegra210_plld_set_defaults(struct tegra_clk_pll *plld)
+{
+	u32 val;
+	u32 mask = 0xffff;
+
+	plld->params->defaults_set = true;
+
+	if (readl_relaxed(clk_base + plld->params->base_reg) &
+			PLL_ENABLE) {
+		pr_warn("PLL_D already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		val = PLLD_MISC1_DEFAULT_VALUE;
+		_pll_misc_chk_default(clk_base, plld->params, 1,
+				val, PLLD_MISC1_WRITE_MASK);
+
+		/* ignore lock, DSI and SDM controls, make sure IDDQ not set */
+		val = PLLD_MISC0_DEFAULT_VALUE & (~PLLD_MISC0_IDDQ);
+		mask |= PLLD_MISC0_DSI_CLKENABLE | PLLD_MISC0_LOCK_ENABLE |
+			PLLD_MISC0_LOCK_OVERRIDE | PLLD_MISC0_EN_SDM;
+		_pll_misc_chk_default(clk_base, plld->params, 0, val,
+				~mask & PLLD_MISC0_WRITE_MASK);
+
+		/* Enable lock detect */
+		mask = PLLD_MISC0_LOCK_ENABLE | PLLD_MISC0_LOCK_OVERRIDE;
+		val = readl_relaxed(clk_base + plld->params->ext_misc_reg[0]);
+		val &= ~mask;
+		val |= PLLD_MISC0_DEFAULT_VALUE & mask;
+		writel_relaxed(val, clk_base + plld->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	val = readl_relaxed(clk_base + plld->params->ext_misc_reg[0]);
+	val &= PLLD_MISC0_DSI_CLKENABLE;
+	val |= PLLD_MISC0_DEFAULT_VALUE;
+	/* set IDDQ, enable lock detect, disable SDM */
+	writel_relaxed(val, clk_base + plld->params->ext_misc_reg[0]);
+	writel_relaxed(PLLD_MISC1_DEFAULT_VALUE, clk_base +
+			plld->params->ext_misc_reg[1]);
+	udelay(1);
+}
+
+/*
+ * PLLD2, PLLDP
+ * PLL with fractional SDM and Spread Spectrum (SDM is a must if SSC is used).
+ */
+static void plldss_defaults(const char *pll_name, struct tegra_clk_pll *plldss,
+		u32 misc0_val, u32 misc1_val, u32 misc2_val, u32 misc3_val)
+{
+	u32 default_val;
+	u32 val = readl_relaxed(clk_base + plldss->params->base_reg);
+
+	plldss->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		pr_warn("%s already enabled. Postponing set full defaults\n",
+			 pll_name);
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		if (val & PLLDSS_BASE_IDDQ) {
+			pr_warn("plldss boot enabled with IDDQ set\n");
+			plldss->params->defaults_set = false;
+		}
+
+		/* ignore lock enable */
+		default_val = misc0_val;
+		_pll_misc_chk_default(clk_base, plldss->params, 0, default_val,
+				     PLLDSS_MISC0_WRITE_MASK &
+				     (~PLLDSS_MISC0_LOCK_ENABLE));
+
+		/*
+		 * If SSC is used, check all settings, otherwise just confirm
+		 * that SSC is not used on boot as well. Do nothing when using
+		 * this function for PLLC4 that has only MISC0.
+		 */
+		if (plldss->params->ssc_ctrl_en_mask) {
+			default_val = misc1_val;
+			_pll_misc_chk_default(clk_base, plldss->params, 1,
+				default_val, PLLDSS_MISC1_CFG_WRITE_MASK);
+			default_val = misc2_val;
+			_pll_misc_chk_default(clk_base, plldss->params, 2,
+				default_val, PLLDSS_MISC2_CTRL1_WRITE_MASK);
+			default_val = misc3_val;
+			_pll_misc_chk_default(clk_base, plldss->params, 3,
+				default_val, PLLDSS_MISC3_CTRL2_WRITE_MASK);
+		} else if (plldss->params->ext_misc_reg[1]) {
+			default_val = misc1_val;
+			_pll_misc_chk_default(clk_base, plldss->params, 1,
+				default_val, PLLDSS_MISC1_CFG_WRITE_MASK &
+				(~PLLDSS_MISC1_CFG_EN_SDM));
+		}
+
+		/* Enable lock detect */
+		if (val & PLLDSS_BASE_LOCK_OVERRIDE) {
+			val &= ~PLLDSS_BASE_LOCK_OVERRIDE;
+			writel_relaxed(val, clk_base +
+					plldss->params->base_reg);
+		}
+
+		val = readl_relaxed(clk_base + plldss->params->ext_misc_reg[0]);
+		val &= ~PLLDSS_MISC0_LOCK_ENABLE;
+		val |= misc0_val & PLLDSS_MISC0_LOCK_ENABLE;
+		writel_relaxed(val, clk_base + plldss->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect, configure SDM/SSC  */
+	val |= PLLDSS_BASE_IDDQ;
+	val &= ~PLLDSS_BASE_LOCK_OVERRIDE;
+	writel_relaxed(val, clk_base + plldss->params->base_reg);
+
+	/* When using this function for PLLC4 exit here */
+	if (!plldss->params->ext_misc_reg[1]) {
+		writel_relaxed(misc0_val, clk_base +
+				plldss->params->ext_misc_reg[0]);
+		udelay(1);
+		return;
+	}
+
+	writel_relaxed(misc0_val, clk_base +
+			plldss->params->ext_misc_reg[0]);
+	/* if SSC used set by 1st enable */
+	writel_relaxed(misc1_val & (~PLLDSS_MISC1_CFG_EN_SSC),
+			clk_base + plldss->params->ext_misc_reg[1]);
+	writel_relaxed(misc2_val, clk_base + plldss->params->ext_misc_reg[2]);
+	writel_relaxed(misc3_val, clk_base + plldss->params->ext_misc_reg[3]);
+	udelay(1);
+}
+
+void tegra210_plld2_set_defaults(struct tegra_clk_pll *plld2)
+{
+	plldss_defaults("PLL_D2", plld2, PLLD2_MISC0_DEFAULT_VALUE,
+			PLLD2_MISC1_CFG_DEFAULT_VALUE,
+			PLLD2_MISC2_CTRL1_DEFAULT_VALUE,
+			PLLD2_MISC3_CTRL2_DEFAULT_VALUE);
+}
+
+void tegra210_plldp_set_defaults(struct tegra_clk_pll *plldp)
+{
+	plldss_defaults("PLL_DP", plldp, PLLDP_MISC0_DEFAULT_VALUE,
+			PLLDP_MISC1_CFG_DEFAULT_VALUE,
+			PLLDP_MISC2_CTRL1_DEFAULT_VALUE,
+			PLLDP_MISC3_CTRL2_DEFAULT_VALUE);
+}
+
+/*
+ * PLLC4
+ * Base and misc0 layout is the same as PLLD2/PLLDP, but no SDM/SSC support.
+ * VCO is exposed to the clock tree via fixed 1/3 and 1/5 dividers.
+ */
+void tegra210_pllc4_set_defaults(struct tegra_clk_pll *pllc4)
+{
+	plldss_defaults("PLL_C4", pllc4, PLLC4_MISC0_DEFAULT_VALUE, 0, 0, 0);
+}
+
+/*
+ * PLLRE
+ * VCO is exposed to the clock tree directly along with post-divider output
+ */
+void tegra210_pllre_set_defaults(struct tegra_clk_pll *pllre)
+{
+	u32 mask;
+	u32 val = readl_relaxed(clk_base + pllre->params->base_reg);
+
+	pllre->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		pr_warn("PLL_RE already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		val &= PLLRE_BASE_DEFAULT_MASK;
+		if (val != PLLRE_BASE_DEFAULT_VALUE) {
+			pr_warn("pllre boot base 0x%x : expected 0x%x\n",
+				val, PLLRE_BASE_DEFAULT_VALUE);
+			pr_warn("(comparison mask = 0x%x)\n",
+				PLLRE_BASE_DEFAULT_MASK);
+			pllre->params->defaults_set = false;
+		}
+
+		/* Ignore lock enable */
+		val = PLLRE_MISC0_DEFAULT_VALUE & (~PLLRE_MISC0_IDDQ);
+		mask = PLLRE_MISC0_LOCK_ENABLE | PLLRE_MISC0_LOCK_OVERRIDE;
+		_pll_misc_chk_default(clk_base, pllre->params, 0, val,
+				~mask & PLLRE_MISC0_WRITE_MASK);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + pllre->params->ext_misc_reg[0]);
+		val &= ~mask;
+		val |= PLLRE_MISC0_DEFAULT_VALUE & mask;
+		writel_relaxed(val, clk_base + pllre->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect */
+	val &= ~PLLRE_BASE_DEFAULT_MASK;
+	val |= PLLRE_BASE_DEFAULT_VALUE & PLLRE_BASE_DEFAULT_MASK;
+	writel_relaxed(val, clk_base + pllre->params->base_reg);
+	writel_relaxed(PLLRE_MISC0_DEFAULT_VALUE,
+			clk_base + pllre->params->ext_misc_reg[0]);
+	udelay(1);
+}
+
+static void pllx_get_dyn_steps(struct clk_hw *hw, u32 *step_a, u32 *step_b)
+{
+	unsigned long input_rate;
+
+	if (!IS_ERR_OR_NULL(hw->clk)) {
+		input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+		/* cf rate */
+		input_rate /= tegra_pll_get_fixed_mdiv(hw, input_rate);
+	} else {
+		input_rate = 38400000;
+	}
+
+	switch (input_rate) {
+	case 12000000:
+	case 12800000:
+	case 13000000:
+		*step_a = 0x2B;
+		*step_b = 0x0B;
+		return;
+	case 19200000:
+		*step_a = 0x12;
+		*step_b = 0x08;
+		return;
+	case 38400000:
+		*step_a = 0x04;
+		*step_b = 0x05;
+		return;
+	default:
+		pr_err("%s: Unexpected reference rate %lu\n",
+			__func__, input_rate);
+		BUG();
+	}
+}
+
+static void pllx_check_defaults(struct tegra_clk_pll *pll)
+{
+	u32 default_val;
+
+	default_val = PLLX_MISC0_DEFAULT_VALUE;
+	/* ignore lock enable */
+	_pll_misc_chk_default(clk_base, pll->params, 0, default_val,
+			PLLX_MISC0_WRITE_MASK & (~PLLX_MISC0_LOCK_ENABLE));
+
+	default_val = PLLX_MISC1_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, pll->params, 1, default_val,
+			PLLX_MISC1_WRITE_MASK);
+
+	/* ignore all but control bit */
+	default_val = PLLX_MISC2_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, pll->params, 2,
+			default_val, PLLX_MISC2_EN_DYNRAMP);
+
+	default_val = PLLX_MISC3_DEFAULT_VALUE & (~PLLX_MISC3_IDDQ);
+	_pll_misc_chk_default(clk_base, pll->params, 3, default_val,
+			PLLX_MISC3_WRITE_MASK);
+
+	default_val = PLLX_MISC4_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, pll->params, 4, default_val,
+			PLLX_MISC4_WRITE_MASK);
+
+	default_val = PLLX_MISC5_DEFAULT_VALUE;
+	_pll_misc_chk_default(clk_base, pll->params, 5, default_val,
+			PLLX_MISC5_WRITE_MASK);
+}
+
+void tegra210_pllx_set_defaults(struct tegra_clk_pll *pllx)
+{
+	u32 val;
+	u32 step_a, step_b;
+
+	pllx->params->defaults_set = true;
+
+	/* Get ready dyn ramp state machine settings */
+	pllx_get_dyn_steps(&pllx->hw, &step_a, &step_b);
+	val = PLLX_MISC2_DEFAULT_VALUE & (~PLLX_MISC2_DYNRAMP_STEPA_MASK) &
+		(~PLLX_MISC2_DYNRAMP_STEPB_MASK);
+	val |= step_a << PLLX_MISC2_DYNRAMP_STEPA_SHIFT;
+	val |= step_b << PLLX_MISC2_DYNRAMP_STEPB_SHIFT;
+
+	if (readl_relaxed(clk_base + pllx->params->base_reg) & PLL_ENABLE) {
+		pr_warn("PLL_X already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		pllx_check_defaults(pllx);
+
+		/* Configure dyn ramp, disable lock override */
+		writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[2]);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + pllx->params->ext_misc_reg[0]);
+		val &= ~PLLX_MISC0_LOCK_ENABLE;
+		val |= PLLX_MISC0_DEFAULT_VALUE & PLLX_MISC0_LOCK_ENABLE;
+		writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* Enable lock detect and CPU output */
+	writel_relaxed(PLLX_MISC0_DEFAULT_VALUE, clk_base +
+			pllx->params->ext_misc_reg[0]);
+
+	/* Setup */
+	writel_relaxed(PLLX_MISC1_DEFAULT_VALUE, clk_base +
+			pllx->params->ext_misc_reg[1]);
+
+	/* Configure dyn ramp state machine, disable lock override */
+	writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[2]);
+
+	/* Set IDDQ */
+	writel_relaxed(PLLX_MISC3_DEFAULT_VALUE, clk_base +
+			pllx->params->ext_misc_reg[3]);
+
+	/* Disable SDM */
+	writel_relaxed(PLLX_MISC4_DEFAULT_VALUE, clk_base +
+			pllx->params->ext_misc_reg[4]);
+	writel_relaxed(PLLX_MISC5_DEFAULT_VALUE, clk_base +
+			pllx->params->ext_misc_reg[5]);
+	udelay(1);
+}
+
+/* PLLMB */
+void tegra210_pllmb_set_defaults(struct tegra_clk_pll *pllmb)
+{
+	u32 mask, val = readl_relaxed(clk_base + pllmb->params->base_reg);
+
+	pllmb->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		pr_warn("PLL_MB already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		val = PLLMB_MISC0_DEFAULT_VALUE & (~PLLMB_MISC0_IDDQ);
+		mask = PLLMB_MISC0_LOCK_ENABLE | PLLMB_MISC0_LOCK_OVERRIDE;
+		_pll_misc_chk_default(clk_base, pllmb->params, 0, val,
+				~mask & PLLMB_MISC0_WRITE_MASK);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + pllmb->params->ext_misc_reg[0]);
+		val &= ~mask;
+		val |= PLLMB_MISC0_DEFAULT_VALUE & mask;
+		writel_relaxed(val, clk_base + pllmb->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect */
+	writel_relaxed(PLLMB_MISC0_DEFAULT_VALUE,
+			clk_base + pllmb->params->ext_misc_reg[0]);
+	udelay(1);
+}
+
+/*
+ * PLLP
+ * VCO is exposed to the clock tree directly along with post-divider output.
+ * Both VCO and post-divider output rates are fixed at 408MHz and 204MHz,
+ * respectively.
+ */
+static void pllp_check_defaults(struct tegra_clk_pll *pll, bool enabled)
+{
+	u32 val, mask;
+
+	/* Ignore lock enable (will be set), make sure not in IDDQ if enabled */
+	val = PLLP_MISC0_DEFAULT_VALUE & (~PLLP_MISC0_IDDQ);
+	mask = PLLP_MISC0_LOCK_ENABLE | PLLP_MISC0_LOCK_OVERRIDE;
+	if (!enabled)
+		mask |= PLLP_MISC0_IDDQ;
+	_pll_misc_chk_default(clk_base, pll->params, 0, val,
+			~mask & PLLP_MISC0_WRITE_MASK);
+
+	/* Ignore branch controls */
+	val = PLLP_MISC1_DEFAULT_VALUE;
+	mask = PLLP_MISC1_HSIO_EN | PLLP_MISC1_XUSB_EN;
+	_pll_misc_chk_default(clk_base, pll->params, 1, val,
+			~mask & PLLP_MISC1_WRITE_MASK);
+}
+
+void tegra210_pllp_set_defaults(struct tegra_clk_pll *pllp)
+{
+	u32 mask;
+	u32 val = readl_relaxed(clk_base + pllp->params->base_reg);
+
+	pllp->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		pr_warn("PLL_P already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		pllp_check_defaults(pllp, true);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + pllp->params->ext_misc_reg[0]);
+		mask = PLLP_MISC0_LOCK_ENABLE | PLLP_MISC0_LOCK_OVERRIDE;
+		val &= ~mask;
+		val |= PLLP_MISC0_DEFAULT_VALUE & mask;
+		writel_relaxed(val, clk_base + pllp->params->ext_misc_reg[0]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect */
+	writel_relaxed(PLLP_MISC0_DEFAULT_VALUE,
+			clk_base + pllp->params->ext_misc_reg[0]);
+
+	/* Preserve branch control */
+	val = readl_relaxed(clk_base + pllp->params->ext_misc_reg[1]);
+	mask = PLLP_MISC1_HSIO_EN | PLLP_MISC1_XUSB_EN;
+	val &= mask;
+	val |= ~mask & PLLP_MISC1_DEFAULT_VALUE;
+	writel_relaxed(val, clk_base + pllp->params->ext_misc_reg[1]);
+	udelay(1);
+}
+
+/*
+ * PLLU
+ * VCO is exposed to the clock tree directly along with post-divider output.
+ * Both VCO and post-divider output rates are fixed at 480MHz and 240MHz,
+ * respectively.
+ */
+static void pllu_check_defaults(struct tegra_clk_pll *pll, bool hw_control)
+{
+	u32 val, mask;
+
+	/* Ignore lock enable (will be set) and IDDQ if under h/w control */
+	val = PLLU_MISC0_DEFAULT_VALUE & (~PLLU_MISC0_IDDQ);
+	mask = PLLU_MISC0_LOCK_ENABLE | (hw_control ? PLLU_MISC0_IDDQ : 0);
+	_pll_misc_chk_default(clk_base, pll->params, 0, val,
+			~mask & PLLU_MISC0_WRITE_MASK);
+
+	val = PLLU_MISC1_DEFAULT_VALUE;
+	mask = PLLU_MISC1_LOCK_OVERRIDE;
+	_pll_misc_chk_default(clk_base, pll->params, 1, val,
+			~mask & PLLU_MISC1_WRITE_MASK);
+}
+
+void tegra210_pllu_set_defaults(struct tegra_clk_pll *pllu)
+{
+	u32 val = readl_relaxed(clk_base + pllu->params->base_reg);
+
+	pllu->params->defaults_set = true;
+
+	if (val & PLL_ENABLE) {
+		pr_warn("PLL_U already enabled. Postponing set full defaults\n");
+
+		/*
+		 * PLL is ON: check if defaults already set, then set those
+		 * that can be updated in flight.
+		 */
+		pllu_check_defaults(pllu, false);
+
+		/* Enable lock detect */
+		val = readl_relaxed(clk_base + pllu->params->ext_misc_reg[0]);
+		val &= ~PLLU_MISC0_LOCK_ENABLE;
+		val |= PLLU_MISC0_DEFAULT_VALUE & PLLU_MISC0_LOCK_ENABLE;
+		writel_relaxed(val, clk_base + pllu->params->ext_misc_reg[0]);
+
+		val = readl_relaxed(clk_base + pllu->params->ext_misc_reg[1]);
+		val &= ~PLLU_MISC1_LOCK_OVERRIDE;
+		val |= PLLU_MISC1_DEFAULT_VALUE & PLLU_MISC1_LOCK_OVERRIDE;
+		writel_relaxed(val, clk_base + pllu->params->ext_misc_reg[1]);
+		udelay(1);
+
+		return;
+	}
+
+	/* set IDDQ, enable lock detect */
+	writel_relaxed(PLLU_MISC0_DEFAULT_VALUE,
+			clk_base + pllu->params->ext_misc_reg[0]);
+	writel_relaxed(PLLU_MISC1_DEFAULT_VALUE,
+			clk_base + pllu->params->ext_misc_reg[1]);
+	udelay(1);
+}
+
+#define mask(w) ((1 << (w)) - 1)
+#define divm_mask(p) mask(p->params->div_nmp->divm_width)
+#define divn_mask(p) mask(p->params->div_nmp->divn_width)
+#define divp_mask(p) (p->params->flags & TEGRA_PLLU ? PLLU_POST_DIVP_MASK :\
+		      mask(p->params->div_nmp->divp_width))
+
+#define divm_shift(p) ((p)->params->div_nmp->divm_shift)
+#define divn_shift(p) ((p)->params->div_nmp->divn_shift)
+#define divp_shift(p) ((p)->params->div_nmp->divp_shift)
+
+#define divm_mask_shifted(p) (divm_mask(p) << divm_shift(p))
+#define divn_mask_shifted(p) (divn_mask(p) << divn_shift(p))
+#define divp_mask_shifted(p) (divp_mask(p) << divp_shift(p))
+
+#define PLL_LOCKDET_DELAY 2	/* Lock detection safety delays */
+static int tegra210_wait_for_mask(struct tegra_clk_pll *pll,
+				  u32 reg, u32 mask)
+{
+	int i;
+	u32 val = 0;
+
+	for (i = 0; i < pll->params->lock_delay / PLL_LOCKDET_DELAY + 1; i++) {
+		udelay(PLL_LOCKDET_DELAY);
+		val = readl_relaxed(clk_base + reg);
+		if ((val & mask) == mask) {
+			udelay(PLL_LOCKDET_DELAY);
+			return 0;
+		}
+	}
+	return -ETIMEDOUT;
+}
+
+static int tegra210_pllx_dyn_ramp(struct tegra_clk_pll *pllx,
+		struct tegra_clk_pll_freq_table *cfg)
+{
+	u32 val, base, ndiv_new_mask;
+
+	ndiv_new_mask = (divn_mask(pllx) >> pllx->params->div_nmp->divn_shift)
+			 << PLLX_MISC2_NDIV_NEW_SHIFT;
+
+	val = readl_relaxed(clk_base + pllx->params->ext_misc_reg[2]);
+	val &= (~ndiv_new_mask);
+	val |= cfg->n << PLLX_MISC2_NDIV_NEW_SHIFT;
+	writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[2]);
+	udelay(1);
+
+	val = readl_relaxed(clk_base + pllx->params->ext_misc_reg[2]);
+	val |= PLLX_MISC2_EN_DYNRAMP;
+	writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[2]);
+	udelay(1);
+
+	tegra210_wait_for_mask(pllx, pllx->params->ext_misc_reg[2],
+			       PLLX_MISC2_DYNRAMP_DONE);
+
+	base = readl_relaxed(clk_base + pllx->params->base_reg) &
+		(~divn_mask_shifted(pllx));
+	base |= cfg->n << pllx->params->div_nmp->divn_shift;
+	writel_relaxed(base, clk_base + pllx->params->base_reg);
+	udelay(1);
+
+	val &= ~PLLX_MISC2_EN_DYNRAMP;
+	writel_relaxed(val, clk_base + pllx->params->ext_misc_reg[2]);
+	udelay(1);
+
+	pr_debug("%s: dynamic ramp to m = %u n = %u p = %u, Fout = %lu kHz\n",
+		 __clk_get_name(pllx->hw.clk), cfg->m, cfg->n, cfg->p,
+		 cfg->input_rate / cfg->m * cfg->n /
+		 pllx->params->pdiv_tohw[cfg->p].pdiv / 1000);
+
+	return 0;
+}
+
+/*
+ * Common configuration for PLLs with fixed input divider policy:
+ * - always set fixed M-value based on the reference rate
+ * - always set P-value value 1:1 for output rates above VCO minimum, and
+ *   choose minimum necessary P-value for output rates below VCO maximum
+ * - calculate N-value based on selected M and P
+ * - calculate SDM_DIN fractional part
+ */
+static int tegra210_pll_fixed_mdiv_cfg(struct clk_hw *hw,
+			       struct tegra_clk_pll_freq_table *cfg,
+			       unsigned long rate, unsigned long input_rate)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	struct tegra_clk_pll_params *params = pll->params;
+	int p;
+	unsigned long cf, p_rate;
+	u32 pdiv;
+
+	if (!rate)
+		return -EINVAL;
+
+	if (!(params->flags & TEGRA_PLL_VCO_OUT)) {
+		p = DIV_ROUND_UP(params->vco_min, rate);
+		p = params->round_p_to_pdiv(p, &pdiv);
+	} else {
+		p = rate >= params->vco_min ? 1 : -EINVAL;
+	}
+
+	if (IS_ERR_VALUE(p))
+		return -EINVAL;
+
+	cfg->m = tegra_pll_get_fixed_mdiv(hw, input_rate);
+	cfg->p = p;
+
+	/* Store P as HW value, as that is what is expected */
+	cfg->p = tegra_pll_p_div_to_hw(pll, cfg->p);
+
+	p_rate = rate * p;
+	if (p_rate > params->vco_max)
+		p_rate = params->vco_max;
+	cf = input_rate / cfg->m;
+	cfg->n = p_rate / cf;
+
+	cfg->sdm_data = 0;
+	if (params->sdm_ctrl_reg) {
+		unsigned long rem = p_rate - cf * cfg->n;
+		/* If ssc is enabled SDM enabled as well, even for integer n */
+		if (rem || params->ssc_ctrl_reg) {
+			u64 s = rem * PLL_SDM_COEFF;
+
+			do_div(s, cf);
+			s -= PLL_SDM_COEFF / 2;
+			cfg->sdm_data = sdin_din_to_data(s);
+		}
+	}
+
+	cfg->input_rate = input_rate;
+	cfg->output_rate = rate;
+
+	return 0;
+}
+
+/*
+ * clk_pll_set_gain - set gain to m, n to calculate correct VCO rate
+ *
+ * @cfg: struct tegra_clk_pll_freq_table * cfg
+ *
+ * For Normal mode:
+ *     Fvco = Fref * NDIV / MDIV
+ *
+ * For fractional mode:
+ *     Fvco = Fref * (NDIV + 0.5 + SDM_DIN / PLL_SDM_COEFF) / MDIV
+ */
+static void tegra210_clk_pll_set_gain(struct tegra_clk_pll_freq_table *cfg)
+{
+	cfg->n = cfg->n * PLL_SDM_COEFF + PLL_SDM_COEFF/2 +
+			sdin_data_to_din(cfg->sdm_data);
+	cfg->m *= PLL_SDM_COEFF;
+}
+
+unsigned long tegra210_clk_adjust_vco_min(struct tegra_clk_pll_params *params,
+					  unsigned long parent_rate)
+{
+	unsigned long vco_min = params->vco_min;
+
+	params->vco_min += DIV_ROUND_UP(parent_rate, PLL_SDM_COEFF);
+	vco_min = min(vco_min, params->vco_min);
+
+	return vco_min;
+}
+
+static struct div_nmp pllx_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+};
+/*
+ * PLL post divider maps - two types: quasi-linear and exponential
+ * post divider.
+ */
+#define PLL_QLIN_PDIV_MAX	16
+static const struct pdiv_map pll_qlin_pdiv_to_hw[] = {
+	{ .pdiv =  1, .hw_val =  0 },
+	{ .pdiv =  2, .hw_val =  1 },
+	{ .pdiv =  3, .hw_val =  2 },
+	{ .pdiv =  4, .hw_val =  3 },
+	{ .pdiv =  5, .hw_val =  4 },
+	{ .pdiv =  6, .hw_val =  5 },
+	{ .pdiv =  8, .hw_val =  6 },
+	{ .pdiv =  9, .hw_val =  7 },
+	{ .pdiv = 10, .hw_val =  8 },
+	{ .pdiv = 12, .hw_val =  9 },
+	{ .pdiv = 15, .hw_val = 10 },
+	{ .pdiv = 16, .hw_val = 11 },
+	{ .pdiv = 18, .hw_val = 12 },
+	{ .pdiv = 20, .hw_val = 13 },
+	{ .pdiv = 24, .hw_val = 14 },
+	{ .pdiv = 30, .hw_val = 15 },
+	{ .pdiv = 32, .hw_val = 16 },
+};
+
+static u32 pll_qlin_p_to_pdiv(u32 p, u32 *pdiv)
+{
+	int i;
+
+	if (p) {
+		for (i = 0; i <= PLL_QLIN_PDIV_MAX; i++) {
+			if (p <= pll_qlin_pdiv_to_hw[i].pdiv) {
+				if (pdiv)
+					*pdiv = i;
+				return pll_qlin_pdiv_to_hw[i].pdiv;
+			}
+		}
+	}
+
+	return -EINVAL;
+}
+
+#define PLL_EXPO_PDIV_MAX	7
+static const struct pdiv_map pll_expo_pdiv_to_hw[] = {
+	{ .pdiv =   1, .hw_val = 0 },
+	{ .pdiv =   2, .hw_val = 1 },
+	{ .pdiv =   4, .hw_val = 2 },
+	{ .pdiv =   8, .hw_val = 3 },
+	{ .pdiv =  16, .hw_val = 4 },
+	{ .pdiv =  32, .hw_val = 5 },
+	{ .pdiv =  64, .hw_val = 6 },
+	{ .pdiv = 128, .hw_val = 7 },
+};
+
+static u32 pll_expo_p_to_pdiv(u32 p, u32 *pdiv)
+{
+	if (p) {
+		u32 i = fls(p);
+
+		if (i == ffs(p))
+			i--;
+
+		if (i <= PLL_EXPO_PDIV_MAX) {
+			if (pdiv)
+				*pdiv = i;
+			return 1 << i;
+		}
+	}
+	return -EINVAL;
+}
+
+static struct tegra_clk_pll_freq_table pll_x_freq_table[] = {
+	/* 1 GHz */
+	{ 12000000, 1000000000, 166, 1, 1, 0 }, /* actual: 996.0 MHz */
+	{ 13000000, 1000000000, 153, 1, 1, 0 }, /* actual: 994.0 MHz */
+	{ 38400000, 1000000000, 156, 3, 1, 0 }, /* actual: 998.4 MHz */
+	{        0,          0,   0, 0, 0, 0 },
+};
+
+static struct tegra_clk_pll_params pll_x_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 38400000,
+	.vco_min = 1350000000,
+	.vco_max = 3000000000UL,
+	.base_reg = PLLX_BASE,
+	.misc_reg = PLLX_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.ext_misc_reg[0] = PLLX_MISC0,
+	.ext_misc_reg[1] = PLLX_MISC1,
+	.ext_misc_reg[2] = PLLX_MISC2,
+	.ext_misc_reg[3] = PLLX_MISC3,
+	.ext_misc_reg[4] = PLLX_MISC4,
+	.ext_misc_reg[5] = PLLX_MISC5,
+	.iddq_reg = PLLX_MISC3,
+	.iddq_bit_idx = PLLXP_IDDQ_BIT,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.mdiv_default = 2,
+	.dyn_ramp_reg = PLLX_MISC2,
+	.stepa_shift = 16,
+	.stepb_shift = 24,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllx_nmp,
+	.freq_table = pll_x_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.dyn_ramp = tegra210_pllx_dyn_ramp,
+	.set_defaults = tegra210_pllx_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct div_nmp pllc_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 10,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_freq_table pll_cx_freq_table[] = {
+	{ 12000000, 510000000, 85, 1, 1, 0 },
+	{ 13000000, 510000000, 78, 1, 1, 0 }, /* actual: 507.0 MHz */
+	{ 38400000, 510000000, 79, 3, 1, 0 }, /* actual: 505.6 MHz */
+	{        0,         0,  0, 0, 0, 0 },
+};
+
+static struct tegra_clk_pll_params pll_c_params = {
+	.input_min = 12000000,
+	.input_max = 700000000,
+	.cf_min = 12000000,
+	.cf_max = 50000000,
+	.vco_min = 600000000,
+	.vco_max = 1200000000,
+	.base_reg = PLLC_BASE,
+	.misc_reg = PLLC_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_delay = 300,
+	.iddq_reg = PLLC_MISC1,
+	.iddq_bit_idx = PLLCX_IDDQ_BIT,
+	.reset_reg = PLLC_MISC0,
+	.reset_bit_idx = PLLCX_RESET_BIT,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLC_MISC0,
+	.ext_misc_reg[1] = PLLC_MISC1,
+	.ext_misc_reg[2] = PLLC_MISC2,
+	.ext_misc_reg[3] = PLLC_MISC3,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.mdiv_default = 3,
+	.div_nmp = &pllc_nmp,
+	.freq_table = pll_cx_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = _pllc_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct div_nmp pllcx_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 10,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_params pll_c2_params = {
+	.input_min = 12000000,
+	.input_max = 700000000,
+	.cf_min = 12000000,
+	.cf_max = 50000000,
+	.vco_min = 600000000,
+	.vco_max = 1200000000,
+	.base_reg = PLLC2_BASE,
+	.misc_reg = PLLC2_MISC0,
+	.iddq_reg = PLLC2_MISC1,
+	.iddq_bit_idx = PLLCX_IDDQ_BIT,
+	.reset_reg = PLLC2_MISC0,
+	.reset_bit_idx = PLLCX_RESET_BIT,
+	.lock_mask = PLLCX_BASE_LOCK,
+	.lock_delay = 300,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.mdiv_default = 3,
+	.div_nmp = &pllcx_nmp,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLC2_MISC0,
+	.ext_misc_reg[1] = PLLC2_MISC1,
+	.ext_misc_reg[2] = PLLC2_MISC2,
+	.ext_misc_reg[3] = PLLC2_MISC3,
+	.freq_table = pll_cx_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = _pllc2_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct tegra_clk_pll_params pll_c3_params = {
+	.input_min = 12000000,
+	.input_max = 700000000,
+	.cf_min = 12000000,
+	.cf_max = 50000000,
+	.vco_min = 600000000,
+	.vco_max = 1200000000,
+	.base_reg = PLLC3_BASE,
+	.misc_reg = PLLC3_MISC0,
+	.lock_mask = PLLCX_BASE_LOCK,
+	.lock_delay = 300,
+	.iddq_reg = PLLC3_MISC1,
+	.iddq_bit_idx = PLLCX_IDDQ_BIT,
+	.reset_reg = PLLC3_MISC0,
+	.reset_bit_idx = PLLCX_RESET_BIT,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.mdiv_default = 3,
+	.div_nmp = &pllcx_nmp,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLC3_MISC0,
+	.ext_misc_reg[1] = PLLC3_MISC1,
+	.ext_misc_reg[2] = PLLC3_MISC2,
+	.ext_misc_reg[3] = PLLC3_MISC3,
+	.freq_table = pll_cx_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = _pllc3_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct div_nmp pllss_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 19,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_freq_table pll_c4_vco_freq_table[] = {
+	{ 12000000, 600000000, 50, 1, 0, 0 },
+	{ 13000000, 600000000, 46, 1, 0, 0 }, /* actual: 598.0 MHz */
+	{ 38400000, 600000000, 62, 4, 0, 0 }, /* actual: 595.2 MHz */
+	{        0,         0,  0, 0, 0, 0 },
+};
+
+static const struct clk_div_table pll_vco_post_div_table[] = {
+	{ .val =  0, .div =  1 },
+	{ .val =  1, .div =  2 },
+	{ .val =  2, .div =  3 },
+	{ .val =  3, .div =  4 },
+	{ .val =  4, .div =  5 },
+	{ .val =  5, .div =  6 },
+	{ .val =  6, .div =  8 },
+	{ .val =  7, .div = 10 },
+	{ .val =  8, .div = 12 },
+	{ .val =  9, .div = 16 },
+	{ .val = 10, .div = 12 },
+	{ .val = 11, .div = 16 },
+	{ .val = 12, .div = 20 },
+	{ .val = 13, .div = 24 },
+	{ .val = 14, .div = 32 },
+	{ .val =  0, .div =  0 },
+};
+
+static struct tegra_clk_pll_params pll_c4_vco_params = {
+	.input_min = 9600000,
+	.input_max = 800000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 500000000,
+	.vco_max = 1080000000,
+	.base_reg = PLLC4_BASE,
+	.misc_reg = PLLC4_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLSS_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLC4_MISC0,
+	.iddq_reg = PLLC4_BASE,
+	.iddq_bit_idx = PLLSS_IDDQ_BIT,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.mdiv_default = 3,
+	.div_nmp = &pllss_nmp,
+	.freq_table = pll_c4_vco_freq_table,
+	.set_defaults = tegra210_pllc4_set_defaults,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE |
+		 TEGRA_PLL_VCO_OUT,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct tegra_clk_pll_freq_table pll_m_freq_table[] = {
+	{ 12000000,  800000000,  66, 1, 0, 0 }, /* actual: 792.0 MHz */
+	{ 13000000,  800000000,  61, 1, 0, 0 }, /* actual: 793.0 MHz */
+	{ 38400000,  297600000,  93, 4, 2, 0 },
+	{ 38400000,  400000000, 125, 4, 2, 0 },
+	{ 38400000,  532800000, 111, 4, 1, 0 },
+	{ 38400000,  665600000, 104, 3, 1, 0 },
+	{ 38400000,  800000000, 125, 3, 1, 0 },
+	{ 38400000,  931200000,  97, 4, 0, 0 },
+	{ 38400000, 1065600000, 111, 4, 0, 0 },
+	{ 38400000, 1200000000, 125, 4, 0, 0 },
+	{ 38400000, 1331200000, 104, 3, 0, 0 },
+	{ 38400000, 1459200000,  76, 2, 0, 0 },
+	{ 38400000, 1600000000, 125, 3, 0, 0 },
+	{        0,          0,   0, 0, 0, 0 },
+};
+
+static struct div_nmp pllm_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.override_divm_shift = 0,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.override_divn_shift = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+	.override_divp_shift = 27,
+};
+
+static struct tegra_clk_pll_params pll_m_params = {
+	.input_min = 9600000,
+	.input_max = 500000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 800000000,
+	.vco_max = 1866000000,
+	.base_reg = PLLM_BASE,
+	.misc_reg = PLLM_MISC1,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLM_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.iddq_reg = PLLM_MISC0,
+	.iddq_bit_idx = PLLM_IDDQ_BIT,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLM_MISC0,
+	.ext_misc_reg[0] = PLLM_MISC1,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllm_nmp,
+	.pmc_divnm_reg = PMC_PLLM_WB0_OVERRIDE,
+	.pmc_divp_reg = PMC_PLLM_WB0_OVERRIDE_2,
+	.freq_table = pll_m_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct tegra_clk_pll_params pll_mb_params = {
+	.input_min = 9600000,
+	.input_max = 500000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 800000000,
+	.vco_max = 1866000000,
+	.base_reg = PLLMB_BASE,
+	.misc_reg = PLLMB_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLMB_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.iddq_reg = PLLMB_MISC0,
+	.iddq_bit_idx = PLLMB_IDDQ_BIT,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLMB_MISC0,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllm_nmp,
+	.freq_table = pll_m_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = tegra210_pllmb_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+
+static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
+	/* PLLE special case: use cpcon field to store cml divider value */
+	{ 672000000, 100000000, 125, 42, 0, 13 },
+	{ 624000000, 100000000, 125, 39, 0, 13 },
+	{ 336000000, 100000000, 125, 21, 0, 13 },
+	{ 312000000, 100000000, 200, 26, 0, 14 },
+	{  38400000, 100000000, 125,  2, 0, 14 },
+	{  12000000, 100000000, 200,  1, 0, 14 },
+	{         0,         0,   0,  0, 0,  0 },
+};
+
+static struct div_nmp plle_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 24,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_params pll_e_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 38400000,
+	.vco_min = 1600000000,
+	.vco_max = 2500000000U,
+	.base_reg = PLLE_BASE,
+	.misc_reg = PLLE_MISC0,
+	.aux_reg = PLLE_AUX,
+	.lock_mask = PLLE_MISC_LOCK,
+	.lock_enable_bit_idx = PLLE_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.div_nmp = &plle_nmp,
+	.freq_table = pll_e_freq_table,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_LOCK_MISC | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
+	.fixed_rate = 100000000,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct tegra_clk_pll_freq_table pll_re_vco_freq_table[] = {
+	{ 12000000, 672000000, 56, 1, 0, 0 },
+	{ 13000000, 672000000, 51, 1, 0, 0 }, /* actual: 663.0 MHz */
+	{ 38400000, 672000000, 70, 4, 0, 0 },
+	{        0,         0,  0, 0, 0, 0 },
+};
+
+static struct div_nmp pllre_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 16,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_params pll_re_vco_params = {
+	.input_min = 9600000,
+	.input_max = 800000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 350000000,
+	.vco_max = 700000000,
+	.base_reg = PLLRE_BASE,
+	.misc_reg = PLLRE_MISC0,
+	.lock_mask = PLLRE_MISC_LOCK,
+	.lock_enable_bit_idx = PLLRE_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.ext_misc_reg[0] = PLLRE_MISC0,
+	.iddq_reg = PLLRE_MISC0,
+	.iddq_bit_idx = PLLRE_IDDQ_BIT,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllre_nmp,
+	.freq_table = pll_re_vco_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_LOCK_MISC |
+		 TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_VCO_OUT,
+	.set_defaults = tegra210_pllre_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct div_nmp pllp_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 10,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_freq_table pll_p_freq_table[] = {
+	{ 12000000, 408000000, 34, 1, 0, 0 },
+	{ 38400000, 408000000, 85, 8, 0, 0 }, /* cf = 4.8MHz, allowed exception */
+	{        0,         0,  0, 0, 0, 0 },
+};
+
+static struct tegra_clk_pll_params pll_p_params = {
+	.input_min = 9600000,
+	.input_max = 800000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 350000000,
+	.vco_max = 700000000,
+	.base_reg = PLLP_BASE,
+	.misc_reg = PLLP_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLP_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.iddq_reg = PLLP_MISC0,
+	.iddq_bit_idx = PLLXP_IDDQ_BIT,
+	.ext_misc_reg[0] = PLLP_MISC0,
+	.ext_misc_reg[1] = PLLP_MISC1,
+	.div_nmp = &pllp_nmp,
+	.freq_table = pll_p_freq_table,
+	.fixed_rate = 408000000,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_VCO_OUT,
+	.set_defaults = tegra210_pllp_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct tegra_clk_pll_params pll_a1_params = {
+	.input_min = 12000000,
+	.input_max = 700000000,
+	.cf_min = 12000000,
+	.cf_max = 50000000,
+	.vco_min = 600000000,
+	.vco_max = 1200000000,
+	.base_reg = PLLA1_BASE,
+	.misc_reg = PLLA1_MISC0,
+	.lock_mask = PLLCX_BASE_LOCK,
+	.lock_delay = 300,
+	.iddq_reg = PLLA1_MISC0,
+	.iddq_bit_idx = PLLCX_IDDQ_BIT,
+	.reset_reg = PLLA1_MISC0,
+	.reset_bit_idx = PLLCX_RESET_BIT,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllc_nmp,
+	.ext_misc_reg[0] = PLLA1_MISC0,
+	.ext_misc_reg[1] = PLLA1_MISC1,
+	.ext_misc_reg[2] = PLLA1_MISC2,
+	.ext_misc_reg[3] = PLLA1_MISC3,
+	.freq_table = pll_cx_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = _plla1_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+static struct div_nmp plla_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_freq_table pll_a_freq_table[] = {
+	{ 12000000, 282240000, 47, 1, 1, 1, 0xf148 }, /* actual: 282240234 */
+	{ 12000000, 368640000, 61, 1, 1, 1, 0xfe15 }, /* actual: 368640381 */
+	{ 12000000, 240000000, 60, 1, 2, 1,      0 },
+	{ 13000000, 282240000, 43, 1, 1, 1, 0xfd7d }, /* actual: 282239807 */
+	{ 13000000, 368640000, 56, 1, 1, 1, 0x06d8 }, /* actual: 368640137 */
+	{ 13000000, 240000000, 55, 1, 2, 1,      0 }, /* actual: 238.3 MHz */
+	{ 38400000, 282240000, 44, 3, 1, 1, 0xf333 }, /* actual: 282239844 */
+	{ 38400000, 368640000, 57, 3, 1, 1, 0x0333 }, /* actual: 368639844 */
+	{ 38400000, 240000000, 75, 3, 3, 1,      0 },
+	{        0,         0,  0, 0, 0, 0,      0 },
+};
+
+static struct tegra_clk_pll_params pll_a_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 19200000,
+	.vco_min = 500000000,
+	.vco_max = 1000000000,
+	.base_reg = PLLA_BASE,
+	.misc_reg = PLLA_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLA_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.iddq_reg = PLLA_BASE,
+	.iddq_bit_idx = PLLA_IDDQ_BIT,
+	.div_nmp = &plla_nmp,
+	.sdm_din_reg = PLLA_MISC1,
+	.sdm_din_mask = PLLA_SDM_DIN_MASK,
+	.sdm_ctrl_reg = PLLA_MISC2,
+	.sdm_ctrl_en_mask = PLLA_SDM_EN_MASK,
+	.ext_misc_reg[0] = PLLA_MISC0,
+	.ext_misc_reg[1] = PLLA_MISC1,
+	.ext_misc_reg[2] = PLLA_MISC2,
+	.freq_table = pll_a_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_MDIV_NEW |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
+	.set_defaults = tegra210_plla_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+	.set_gain = tegra210_clk_pll_set_gain,
+	.adjust_vco = tegra210_clk_adjust_vco_min,
+};
+
+static struct div_nmp plld_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 11,
+	.divn_width = 8,
+	.divp_shift = 20,
+	.divp_width = 3,
+};
+
+static struct tegra_clk_pll_freq_table pll_d_freq_table[] = {
+	{ 12000000, 594000000, 99, 1, 1, 0,      0 },
+	{ 13000000, 594000000, 91, 1, 1, 0, 0xfc4f }, /* actual: 594000183 */
+	{ 38400000, 594000000, 30, 1, 1, 0, 0x0e00 },
+	{        0,         0,  0, 0, 0, 0,      0 },
+};
+
+static struct tegra_clk_pll_params pll_d_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 38400000,
+	.vco_min = 750000000,
+	.vco_max = 1500000000,
+	.base_reg = PLLD_BASE,
+	.misc_reg = PLLD_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLD_MISC_LOCK_ENABLE,
+	.lock_delay = 1000,
+	.iddq_reg = PLLD_MISC0,
+	.iddq_bit_idx = PLLD_IDDQ_BIT,
+	.round_p_to_pdiv = pll_expo_p_to_pdiv,
+	.pdiv_tohw = pll_expo_pdiv_to_hw,
+	.div_nmp = &plld_nmp,
+	.sdm_din_reg = PLLD_MISC0,
+	.sdm_din_mask = PLLA_SDM_DIN_MASK,
+	.sdm_ctrl_reg = PLLD_MISC0,
+	.sdm_ctrl_en_mask = PLLD_SDM_EN_MASK,
+	.ext_misc_reg[0] = PLLD_MISC0,
+	.ext_misc_reg[1] = PLLD_MISC1,
+	.freq_table = pll_d_freq_table,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.mdiv_default = 1,
+	.set_defaults = tegra210_plld_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+	.set_gain = tegra210_clk_pll_set_gain,
+	.adjust_vco = tegra210_clk_adjust_vco_min,
+};
+
+static struct tegra_clk_pll_freq_table tegra210_pll_d2_freq_table[] = {
+	{ 12000000, 594000000, 99, 1, 1, 0, 0xf000 },
+	{ 13000000, 594000000, 91, 1, 1, 0, 0xfc4f }, /* actual: 594000183 */
+	{ 38400000, 594000000, 30, 1, 1, 0, 0x0e00 },
+	{        0,         0,  0, 0, 0, 0,      0 },
+};
+
+/* s/w policy, always tegra_pll_ref */
+static struct tegra_clk_pll_params pll_d2_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 38400000,
+	.vco_min = 750000000,
+	.vco_max = 1500000000,
+	.base_reg = PLLD2_BASE,
+	.misc_reg = PLLD2_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLSS_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.iddq_reg = PLLD2_BASE,
+	.iddq_bit_idx = PLLSS_IDDQ_BIT,
+	.sdm_din_reg = PLLD2_MISC3,
+	.sdm_din_mask = PLLA_SDM_DIN_MASK,
+	.sdm_ctrl_reg = PLLD2_MISC1,
+	.sdm_ctrl_en_mask = PLLD2_SDM_EN_MASK,
+	.ssc_ctrl_reg = PLLD2_MISC1,
+	.ssc_ctrl_en_mask = PLLD2_SSC_EN_MASK,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllss_nmp,
+	.ext_misc_reg[0] = PLLD2_MISC0,
+	.ext_misc_reg[1] = PLLD2_MISC1,
+	.ext_misc_reg[2] = PLLD2_MISC2,
+	.ext_misc_reg[3] = PLLD2_MISC3,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.mdiv_default = 1,
+	.freq_table = tegra210_pll_d2_freq_table,
+	.set_defaults = tegra210_plld2_set_defaults,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+	.set_gain = tegra210_clk_pll_set_gain,
+	.adjust_vco = tegra210_clk_adjust_vco_min,
+};
+
+static struct tegra_clk_pll_freq_table pll_dp_freq_table[] = {
+	{ 12000000, 270000000, 90, 1, 3, 0, 0xf000 },
+	{ 13000000, 270000000, 83, 1, 3, 0, 0xf000 }, /* actual: 269.8 MHz */
+	{ 38400000, 270000000, 28, 1, 3, 0, 0xf400 },
+	{        0,         0,  0, 0, 0, 0,      0 },
+};
+
+static struct tegra_clk_pll_params pll_dp_params = {
+	.input_min = 12000000,
+	.input_max = 800000000,
+	.cf_min = 12000000,
+	.cf_max = 38400000,
+	.vco_min = 750000000,
+	.vco_max = 1500000000,
+	.base_reg = PLLDP_BASE,
+	.misc_reg = PLLDP_MISC,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLSS_MISC_LOCK_ENABLE,
+	.lock_delay = 300,
+	.iddq_reg = PLLDP_BASE,
+	.iddq_bit_idx = PLLSS_IDDQ_BIT,
+	.sdm_din_reg = PLLDP_SS_CTRL2,
+	.sdm_din_mask = PLLA_SDM_DIN_MASK,
+	.sdm_ctrl_reg = PLLDP_SS_CFG,
+	.sdm_ctrl_en_mask = PLLDP_SDM_EN_MASK,
+	.ssc_ctrl_reg = PLLDP_SS_CFG,
+	.ssc_ctrl_en_mask = PLLDP_SSC_EN_MASK,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllss_nmp,
+	.ext_misc_reg[0] = PLLDP_MISC,
+	.ext_misc_reg[1] = PLLDP_SS_CFG,
+	.ext_misc_reg[2] = PLLDP_SS_CTRL1,
+	.ext_misc_reg[3] = PLLDP_SS_CTRL2,
+	.max_p = PLL_QLIN_PDIV_MAX,
+	.mdiv_default = 1,
+	.freq_table = pll_dp_freq_table,
+	.set_defaults = tegra210_plldp_set_defaults,
+	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+	.set_gain = tegra210_clk_pll_set_gain,
+	.adjust_vco = tegra210_clk_adjust_vco_min,
+};
+
+static struct div_nmp pllu_nmp = {
+	.divm_shift = 0,
+	.divm_width = 8,
+	.divn_shift = 8,
+	.divn_width = 8,
+	.divp_shift = 16,
+	.divp_width = 5,
+};
+
+static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
+	{ 12000000, 480000000, 40, 1, 0, 0 },
+	{ 13000000, 480000000, 36, 1, 0, 0 }, /* actual: 468.0 MHz */
+	{ 38400000, 480000000, 25, 2, 0, 0 },
+	{        0,         0,  0, 0, 0, 0 },
+};
+
+static struct tegra_clk_pll_params pll_u_vco_params = {
+	.input_min = 9600000,
+	.input_max = 800000000,
+	.cf_min = 9600000,
+	.cf_max = 19200000,
+	.vco_min = 350000000,
+	.vco_max = 700000000,
+	.base_reg = PLLU_BASE,
+	.misc_reg = PLLU_MISC0,
+	.lock_mask = PLL_BASE_LOCK,
+	.lock_enable_bit_idx = PLLU_MISC_LOCK_ENABLE,
+	.lock_delay = 1000,
+	.iddq_reg = PLLU_MISC0,
+	.iddq_bit_idx = PLLU_IDDQ_BIT,
+	.ext_misc_reg[0] = PLLU_MISC0,
+	.ext_misc_reg[1] = PLLU_MISC1,
+	.round_p_to_pdiv = pll_qlin_p_to_pdiv,
+	.pdiv_tohw = pll_qlin_pdiv_to_hw,
+	.div_nmp = &pllu_nmp,
+	.freq_table = pll_u_freq_table,
+	.flags = TEGRA_PLLU | TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE |
+		 TEGRA_PLL_VCO_OUT,
+	.set_defaults = tegra210_pllu_set_defaults,
+	.calc_rate = tegra210_pll_fixed_mdiv_cfg,
+};
+
+struct utmi_clk_param {
+	/* Oscillator Frequency in KHz */
+	u32 osc_frequency;
+	/* UTMIP PLL Enable Delay Count  */
+	u8 enable_delay_count;
+	/* UTMIP PLL Stable count */
+	u16 stable_count;
+	/*  UTMIP PLL Active delay count */
+	u8 active_delay_count;
+	/* UTMIP PLL Xtal frequency count */
+	u16 xtal_freq_count;
+};
+
+static const struct utmi_clk_param utmi_parameters[] = {
+	{
+		.osc_frequency = 38400000, .enable_delay_count = 0x0,
+		.stable_count = 0x0, .active_delay_count = 0x6,
+		.xtal_freq_count = 0x80
+	}, {
+		.osc_frequency = 13000000, .enable_delay_count = 0x02,
+		.stable_count = 0x33, .active_delay_count = 0x05,
+		.xtal_freq_count = 0x7f
+	}, {
+		.osc_frequency = 19200000, .enable_delay_count = 0x03,
+		.stable_count = 0x4b, .active_delay_count = 0x06,
+		.xtal_freq_count = 0xbb
+	}, {
+		.osc_frequency = 12000000, .enable_delay_count = 0x02,
+		.stable_count = 0x2f, .active_delay_count = 0x08,
+		.xtal_freq_count = 0x76
+	}, {
+		.osc_frequency = 26000000, .enable_delay_count = 0x04,
+		.stable_count = 0x66, .active_delay_count = 0x09,
+		.xtal_freq_count = 0xfe
+	}, {
+		.osc_frequency = 16800000, .enable_delay_count = 0x03,
+		.stable_count = 0x41, .active_delay_count = 0x0a,
+		.xtal_freq_count = 0xa4
+	},
+};
+
+static struct tegra_clk tegra210_clks[tegra_clk_max] __initdata = {
+	[tegra_clk_ispb] = { .dt_id = TEGRA210_CLK_ISPB, .present = true },
+	[tegra_clk_rtc] = { .dt_id = TEGRA210_CLK_RTC, .present = true },
+	[tegra_clk_timer] = { .dt_id = TEGRA210_CLK_TIMER, .present = true },
+	[tegra_clk_uarta_8] = { .dt_id = TEGRA210_CLK_UARTA, .present = true },
+	[tegra_clk_sdmmc2_9] = { .dt_id = TEGRA210_CLK_SDMMC2, .present = true },
+	[tegra_clk_i2s1] = { .dt_id = TEGRA210_CLK_I2S1, .present = true },
+	[tegra_clk_i2c1] = { .dt_id = TEGRA210_CLK_I2C1, .present = true },
+	[tegra_clk_sdmmc1_9] = { .dt_id = TEGRA210_CLK_SDMMC1, .present = true },
+	[tegra_clk_sdmmc4_9] = { .dt_id = TEGRA210_CLK_SDMMC4, .present = true },
+	[tegra_clk_pwm] = { .dt_id = TEGRA210_CLK_PWM, .present = true },
+	[tegra_clk_i2s2] = { .dt_id = TEGRA210_CLK_I2S2, .present = true },
+	[tegra_clk_usbd] = { .dt_id = TEGRA210_CLK_USBD, .present = true },
+	[tegra_clk_isp_9] = { .dt_id = TEGRA210_CLK_ISP, .present = true },
+	[tegra_clk_disp2_8] = { .dt_id = TEGRA210_CLK_DISP2, .present = true },
+	[tegra_clk_disp1_8] = { .dt_id = TEGRA210_CLK_DISP1, .present = true },
+	[tegra_clk_host1x_9] = { .dt_id = TEGRA210_CLK_HOST1X, .present = true },
+	[tegra_clk_i2s0] = { .dt_id = TEGRA210_CLK_I2S0, .present = true },
+	[tegra_clk_apbdma] = { .dt_id = TEGRA210_CLK_APBDMA, .present = true },
+	[tegra_clk_kfuse] = { .dt_id = TEGRA210_CLK_KFUSE, .present = true },
+	[tegra_clk_sbc1_9] = { .dt_id = TEGRA210_CLK_SBC1, .present = true },
+	[tegra_clk_sbc2_9] = { .dt_id = TEGRA210_CLK_SBC2, .present = true },
+	[tegra_clk_sbc3_9] = { .dt_id = TEGRA210_CLK_SBC3, .present = true },
+	[tegra_clk_i2c5] = { .dt_id = TEGRA210_CLK_I2C5, .present = true },
+	[tegra_clk_csi] = { .dt_id = TEGRA210_CLK_CSI, .present = true },
+	[tegra_clk_i2c2] = { .dt_id = TEGRA210_CLK_I2C2, .present = true },
+	[tegra_clk_uartc_8] = { .dt_id = TEGRA210_CLK_UARTC, .present = true },
+	[tegra_clk_mipi_cal] = { .dt_id = TEGRA210_CLK_MIPI_CAL, .present = true },
+	[tegra_clk_emc] = { .dt_id = TEGRA210_CLK_EMC, .present = true },
+	[tegra_clk_usb2] = { .dt_id = TEGRA210_CLK_USB2, .present = true },
+	[tegra_clk_bsev] = { .dt_id = TEGRA210_CLK_BSEV, .present = true },
+	[tegra_clk_uartd_8] = { .dt_id = TEGRA210_CLK_UARTD, .present = true },
+	[tegra_clk_i2c3] = { .dt_id = TEGRA210_CLK_I2C3, .present = true },
+	[tegra_clk_sbc4_9] = { .dt_id = TEGRA210_CLK_SBC4, .present = true },
+	[tegra_clk_sdmmc3_9] = { .dt_id = TEGRA210_CLK_SDMMC3, .present = true },
+	[tegra_clk_pcie] = { .dt_id = TEGRA210_CLK_PCIE, .present = true },
+	[tegra_clk_owr_8] = { .dt_id = TEGRA210_CLK_OWR, .present = true },
+	[tegra_clk_afi] = { .dt_id = TEGRA210_CLK_AFI, .present = true },
+	[tegra_clk_csite_8] = { .dt_id = TEGRA210_CLK_CSITE, .present = true },
+	[tegra_clk_soc_therm_8] = { .dt_id = TEGRA210_CLK_SOC_THERM, .present = true },
+	[tegra_clk_dtv] = { .dt_id = TEGRA210_CLK_DTV, .present = true },
+	[tegra_clk_i2cslow] = { .dt_id = TEGRA210_CLK_I2CSLOW, .present = true },
+	[tegra_clk_tsec_8] = { .dt_id = TEGRA210_CLK_TSEC, .present = true },
+	[tegra_clk_xusb_host] = { .dt_id = TEGRA210_CLK_XUSB_HOST, .present = true },
+	[tegra_clk_csus] = { .dt_id = TEGRA210_CLK_CSUS, .present = true },
+	[tegra_clk_mselect] = { .dt_id = TEGRA210_CLK_MSELECT, .present = true },
+	[tegra_clk_tsensor] = { .dt_id = TEGRA210_CLK_TSENSOR, .present = true },
+	[tegra_clk_i2s3] = { .dt_id = TEGRA210_CLK_I2S3, .present = true },
+	[tegra_clk_i2s4] = { .dt_id = TEGRA210_CLK_I2S4, .present = true },
+	[tegra_clk_i2c4] = { .dt_id = TEGRA210_CLK_I2C4, .present = true },
+	[tegra_clk_d_audio] = { .dt_id = TEGRA210_CLK_D_AUDIO, .present = true },
+	[tegra_clk_hda2codec_2x_8] = { .dt_id = TEGRA210_CLK_HDA2CODEC_2X, .present = true },
+	[tegra_clk_spdif_2x] = { .dt_id = TEGRA210_CLK_SPDIF_2X, .present = true },
+	[tegra_clk_actmon] = { .dt_id = TEGRA210_CLK_ACTMON, .present = true },
+	[tegra_clk_extern1] = { .dt_id = TEGRA210_CLK_EXTERN1, .present = true },
+	[tegra_clk_extern2] = { .dt_id = TEGRA210_CLK_EXTERN2, .present = true },
+	[tegra_clk_extern3] = { .dt_id = TEGRA210_CLK_EXTERN3, .present = true },
+	[tegra_clk_sata_oob_8] = { .dt_id = TEGRA210_CLK_SATA_OOB, .present = true },
+	[tegra_clk_sata_8] = { .dt_id = TEGRA210_CLK_SATA, .present = true },
+	[tegra_clk_hda_8] = { .dt_id = TEGRA210_CLK_HDA, .present = true },
+	[tegra_clk_hda2hdmi] = { .dt_id = TEGRA210_CLK_HDA2HDMI, .present = true },
+	[tegra_clk_cilab] = { .dt_id = TEGRA210_CLK_CILAB, .present = true },
+	[tegra_clk_cilcd] = { .dt_id = TEGRA210_CLK_CILCD, .present = true },
+	[tegra_clk_cile] = { .dt_id = TEGRA210_CLK_CILE, .present = true },
+	[tegra_clk_dsialp] = { .dt_id = TEGRA210_CLK_DSIALP, .present = true },
+	[tegra_clk_dsiblp] = { .dt_id = TEGRA210_CLK_DSIBLP, .present = true },
+	[tegra_clk_entropy_8] = { .dt_id = TEGRA210_CLK_ENTROPY, .present = true },
+	[tegra_clk_xusb_ss] = { .dt_id = TEGRA210_CLK_XUSB_SS, .present = true },
+	[tegra_clk_i2c6] = { .dt_id = TEGRA210_CLK_I2C6, .present = true },
+	[tegra_clk_vim2_clk] = { .dt_id = TEGRA210_CLK_VIM2_CLK, .present = true },
+	[tegra_clk_clk72Mhz_8] = { .dt_id = TEGRA210_CLK_CLK72MHZ, .present = true },
+	[tegra_clk_vic03_8] = { .dt_id = TEGRA210_CLK_VIC03, .present = true },
+	[tegra_clk_dpaux] = { .dt_id = TEGRA210_CLK_DPAUX, .present = true },
+	[tegra_clk_sor0] = { .dt_id = TEGRA210_CLK_SOR0, .present = true },
+	[tegra_clk_sor0_lvds] = { .dt_id = TEGRA210_CLK_SOR0_LVDS, .present = true },
+	[tegra_clk_gpu] = { .dt_id = TEGRA210_CLK_GPU, .present = true },
+	[tegra_clk_pll_g_ref] = { .dt_id = TEGRA210_CLK_PLL_G_REF, .present = true, },
+	[tegra_clk_uartb_8] = { .dt_id = TEGRA210_CLK_UARTB, .present = true },
+	[tegra_clk_vfir] = { .dt_id = TEGRA210_CLK_VFIR, .present = true },
+	[tegra_clk_spdif_in_8] = { .dt_id = TEGRA210_CLK_SPDIF_IN, .present = true },
+	[tegra_clk_spdif_out] = { .dt_id = TEGRA210_CLK_SPDIF_OUT, .present = true },
+	[tegra_clk_vi_10] = { .dt_id = TEGRA210_CLK_VI, .present = true },
+	[tegra_clk_vi_sensor_8] = { .dt_id = TEGRA210_CLK_VI_SENSOR, .present = true },
+	[tegra_clk_fuse] = { .dt_id = TEGRA210_CLK_FUSE, .present = true },
+	[tegra_clk_fuse_burn] = { .dt_id = TEGRA210_CLK_FUSE_BURN, .present = true },
+	[tegra_clk_clk_32k] = { .dt_id = TEGRA210_CLK_CLK_32K, .present = true },
+	[tegra_clk_clk_m] = { .dt_id = TEGRA210_CLK_CLK_M, .present = true },
+	[tegra_clk_clk_m_div2] = { .dt_id = TEGRA210_CLK_CLK_M_DIV2, .present = true },
+	[tegra_clk_clk_m_div4] = { .dt_id = TEGRA210_CLK_CLK_M_DIV4, .present = true },
+	[tegra_clk_pll_ref] = { .dt_id = TEGRA210_CLK_PLL_REF, .present = true },
+	[tegra_clk_pll_c] = { .dt_id = TEGRA210_CLK_PLL_C, .present = true },
+	[tegra_clk_pll_c_out1] = { .dt_id = TEGRA210_CLK_PLL_C_OUT1, .present = true },
+	[tegra_clk_pll_c2] = { .dt_id = TEGRA210_CLK_PLL_C2, .present = true },
+	[tegra_clk_pll_c3] = { .dt_id = TEGRA210_CLK_PLL_C3, .present = true },
+	[tegra_clk_pll_m] = { .dt_id = TEGRA210_CLK_PLL_M, .present = true },
+	[tegra_clk_pll_m_out1] = { .dt_id = TEGRA210_CLK_PLL_M_OUT1, .present = true },
+	[tegra_clk_pll_p] = { .dt_id = TEGRA210_CLK_PLL_P, .present = true },
+	[tegra_clk_pll_p_out1] = { .dt_id = TEGRA210_CLK_PLL_P_OUT1, .present = true },
+	[tegra_clk_pll_p_out3] = { .dt_id = TEGRA210_CLK_PLL_P_OUT3, .present = true },
+	[tegra_clk_pll_p_out4_cpu] = { .dt_id = TEGRA210_CLK_PLL_P_OUT4, .present = true },
+	[tegra_clk_pll_p_out_hsio] = { .dt_id = TEGRA210_CLK_PLL_P_OUT_HSIO, .present = true },
+	[tegra_clk_pll_p_out_xusb] = { .dt_id = TEGRA210_CLK_PLL_P_OUT_XUSB, .present = true },
+	[tegra_clk_pll_p_out_cpu] = { .dt_id = TEGRA210_CLK_PLL_P_OUT_CPU, .present = true },
+	[tegra_clk_pll_p_out_adsp] = { .dt_id = TEGRA210_CLK_PLL_P_OUT_ADSP, .present = true },
+	[tegra_clk_pll_a] = { .dt_id = TEGRA210_CLK_PLL_A, .present = true },
+	[tegra_clk_pll_a_out0] = { .dt_id = TEGRA210_CLK_PLL_A_OUT0, .present = true },
+	[tegra_clk_pll_d] = { .dt_id = TEGRA210_CLK_PLL_D, .present = true },
+	[tegra_clk_pll_d_out0] = { .dt_id = TEGRA210_CLK_PLL_D_OUT0, .present = true },
+	[tegra_clk_pll_d2] = { .dt_id = TEGRA210_CLK_PLL_D2, .present = true },
+	[tegra_clk_pll_d2_out0] = { .dt_id = TEGRA210_CLK_PLL_D2_OUT0, .present = true },
+	[tegra_clk_pll_u] = { .dt_id = TEGRA210_CLK_PLL_U, .present = true },
+	[tegra_clk_pll_u_out] = { .dt_id = TEGRA210_CLK_PLL_U_OUT, .present = true },
+	[tegra_clk_pll_u_out1] = { .dt_id = TEGRA210_CLK_PLL_U_OUT1, .present = true },
+	[tegra_clk_pll_u_out2] = { .dt_id = TEGRA210_CLK_PLL_U_OUT2, .present = true },
+	[tegra_clk_pll_u_480m] = { .dt_id = TEGRA210_CLK_PLL_U_480M, .present = true },
+	[tegra_clk_pll_u_60m] = { .dt_id = TEGRA210_CLK_PLL_U_60M, .present = true },
+	[tegra_clk_pll_u_48m] = { .dt_id = TEGRA210_CLK_PLL_U_48M, .present = true },
+	[tegra_clk_pll_x] = { .dt_id = TEGRA210_CLK_PLL_X, .present = true },
+	[tegra_clk_pll_x_out0] = { .dt_id = TEGRA210_CLK_PLL_X_OUT0, .present = true },
+	[tegra_clk_pll_re_vco] = { .dt_id = TEGRA210_CLK_PLL_RE_VCO, .present = true },
+	[tegra_clk_pll_re_out] = { .dt_id = TEGRA210_CLK_PLL_RE_OUT, .present = true },
+	[tegra_clk_spdif_in_sync] = { .dt_id = TEGRA210_CLK_SPDIF_IN_SYNC, .present = true },
+	[tegra_clk_i2s0_sync] = { .dt_id = TEGRA210_CLK_I2S0_SYNC, .present = true },
+	[tegra_clk_i2s1_sync] = { .dt_id = TEGRA210_CLK_I2S1_SYNC, .present = true },
+	[tegra_clk_i2s2_sync] = { .dt_id = TEGRA210_CLK_I2S2_SYNC, .present = true },
+	[tegra_clk_i2s3_sync] = { .dt_id = TEGRA210_CLK_I2S3_SYNC, .present = true },
+	[tegra_clk_i2s4_sync] = { .dt_id = TEGRA210_CLK_I2S4_SYNC, .present = true },
+	[tegra_clk_vimclk_sync] = { .dt_id = TEGRA210_CLK_VIMCLK_SYNC, .present = true },
+	[tegra_clk_audio0] = { .dt_id = TEGRA210_CLK_AUDIO0, .present = true },
+	[tegra_clk_audio1] = { .dt_id = TEGRA210_CLK_AUDIO1, .present = true },
+	[tegra_clk_audio2] = { .dt_id = TEGRA210_CLK_AUDIO2, .present = true },
+	[tegra_clk_audio3] = { .dt_id = TEGRA210_CLK_AUDIO3, .present = true },
+	[tegra_clk_audio4] = { .dt_id = TEGRA210_CLK_AUDIO4, .present = true },
+	[tegra_clk_spdif] = { .dt_id = TEGRA210_CLK_SPDIF, .present = true },
+	[tegra_clk_clk_out_1] = { .dt_id = TEGRA210_CLK_CLK_OUT_1, .present = true },
+	[tegra_clk_clk_out_2] = { .dt_id = TEGRA210_CLK_CLK_OUT_2, .present = true },
+	[tegra_clk_clk_out_3] = { .dt_id = TEGRA210_CLK_CLK_OUT_3, .present = true },
+	[tegra_clk_blink] = { .dt_id = TEGRA210_CLK_BLINK, .present = true },
+	[tegra_clk_xusb_gate] = { .dt_id = TEGRA210_CLK_XUSB_GATE, .present = true },
+	[tegra_clk_xusb_host_src_8] = { .dt_id = TEGRA210_CLK_XUSB_HOST_SRC, .present = true },
+	[tegra_clk_xusb_falcon_src_8] = { .dt_id = TEGRA210_CLK_XUSB_FALCON_SRC, .present = true },
+	[tegra_clk_xusb_fs_src] = { .dt_id = TEGRA210_CLK_XUSB_FS_SRC, .present = true },
+	[tegra_clk_xusb_ss_src_8] = { .dt_id = TEGRA210_CLK_XUSB_SS_SRC, .present = true },
+	[tegra_clk_xusb_ss_div2] = { .dt_id = TEGRA210_CLK_XUSB_SS_DIV2, .present = true },
+	[tegra_clk_xusb_dev_src_8] = { .dt_id = TEGRA210_CLK_XUSB_DEV_SRC, .present = true },
+	[tegra_clk_xusb_dev] = { .dt_id = TEGRA210_CLK_XUSB_DEV, .present = true },
+	[tegra_clk_xusb_hs_src_4] = { .dt_id = TEGRA210_CLK_XUSB_HS_SRC, .present = true },
+	[tegra_clk_xusb_ssp_src] = { .dt_id = TEGRA210_CLK_XUSB_SSP_SRC, .present = true },
+	[tegra_clk_usb2_hsic_trk] = { .dt_id = TEGRA210_CLK_USB2_HSIC_TRK, .present = true },
+	[tegra_clk_hsic_trk] = { .dt_id = TEGRA210_CLK_HSIC_TRK, .present = true },
+	[tegra_clk_usb2_trk] = { .dt_id = TEGRA210_CLK_USB2_TRK, .present = true },
+	[tegra_clk_sclk] = { .dt_id = TEGRA210_CLK_SCLK, .present = true },
+	[tegra_clk_sclk_mux] = { .dt_id = TEGRA210_CLK_SCLK_MUX, .present = true },
+	[tegra_clk_hclk] = { .dt_id = TEGRA210_CLK_HCLK, .present = true },
+	[tegra_clk_pclk] = { .dt_id = TEGRA210_CLK_PCLK, .present = true },
+	[tegra_clk_cclk_g] = { .dt_id = TEGRA210_CLK_CCLK_G, .present = true },
+	[tegra_clk_cclk_lp] = { .dt_id = TEGRA210_CLK_CCLK_LP, .present = true },
+	[tegra_clk_dfll_ref] = { .dt_id = TEGRA210_CLK_DFLL_REF, .present = true },
+	[tegra_clk_dfll_soc] = { .dt_id = TEGRA210_CLK_DFLL_SOC, .present = true },
+	[tegra_clk_vi_sensor2_8] = { .dt_id = TEGRA210_CLK_VI_SENSOR2, .present = true },
+	[tegra_clk_pll_p_out5] = { .dt_id = TEGRA210_CLK_PLL_P_OUT5, .present = true },
+	[tegra_clk_pll_c4] = { .dt_id = TEGRA210_CLK_PLL_C4, .present = true },
+	[tegra_clk_pll_dp] = { .dt_id = TEGRA210_CLK_PLL_DP, .present = true },
+	[tegra_clk_audio0_mux] = { .dt_id = TEGRA210_CLK_AUDIO0_MUX, .present = true },
+	[tegra_clk_audio1_mux] = { .dt_id = TEGRA210_CLK_AUDIO1_MUX, .present = true },
+	[tegra_clk_audio2_mux] = { .dt_id = TEGRA210_CLK_AUDIO2_MUX, .present = true },
+	[tegra_clk_audio3_mux] = { .dt_id = TEGRA210_CLK_AUDIO3_MUX, .present = true },
+	[tegra_clk_audio4_mux] = { .dt_id = TEGRA210_CLK_AUDIO4_MUX, .present = true },
+	[tegra_clk_spdif_mux] = { .dt_id = TEGRA210_CLK_SPDIF_MUX, .present = true },
+	[tegra_clk_clk_out_1_mux] = { .dt_id = TEGRA210_CLK_CLK_OUT_1_MUX, .present = true },
+	[tegra_clk_clk_out_2_mux] = { .dt_id = TEGRA210_CLK_CLK_OUT_2_MUX, .present = true },
+	[tegra_clk_clk_out_3_mux] = { .dt_id = TEGRA210_CLK_CLK_OUT_3_MUX, .present = true },
+	[tegra_clk_maud] = { .dt_id = TEGRA210_CLK_MAUD, .present = true },
+	[tegra_clk_mipibif] = { .dt_id = TEGRA210_CLK_MIPIBIF, .present = true },
+	[tegra_clk_qspi] = { .dt_id = TEGRA210_CLK_QSPI, .present = true },
+	[tegra_clk_sdmmc_legacy] = { .dt_id = TEGRA210_CLK_SDMMC_LEGACY, .present = true },
+	[tegra_clk_tsecb] = { .dt_id = TEGRA210_CLK_TSECB, .present = true },
+	[tegra_clk_uartape] = { .dt_id = TEGRA210_CLK_UARTAPE, .present = true },
+	[tegra_clk_vi_i2c] = { .dt_id = TEGRA210_CLK_VI_I2C, .present = true },
+	[tegra_clk_ape] = { .dt_id = TEGRA210_CLK_APE, .present = true },
+	[tegra_clk_dbgapb] = { .dt_id = TEGRA210_CLK_DBGAPB, .present = true },
+	[tegra_clk_nvdec] = { .dt_id = TEGRA210_CLK_NVDEC, .present = true },
+	[tegra_clk_nvenc] = { .dt_id = TEGRA210_CLK_NVENC, .present = true },
+	[tegra_clk_nvjpg] = { .dt_id = TEGRA210_CLK_NVJPG, .present = true },
+	[tegra_clk_pll_c4_out0] = { .dt_id = TEGRA210_CLK_PLL_C4_OUT0, .present = true },
+	[tegra_clk_pll_c4_out1] = { .dt_id = TEGRA210_CLK_PLL_C4_OUT1, .present = true },
+	[tegra_clk_pll_c4_out2] = { .dt_id = TEGRA210_CLK_PLL_C4_OUT2, .present = true },
+	[tegra_clk_pll_c4_out3] = { .dt_id = TEGRA210_CLK_PLL_C4_OUT3, .present = true },
+};
+
+static struct tegra_devclk devclks[] __initdata = {
+	{ .con_id = "clk_m", .dt_id = TEGRA210_CLK_CLK_M },
+	{ .con_id = "pll_ref", .dt_id = TEGRA210_CLK_PLL_REF },
+	{ .con_id = "clk_32k", .dt_id = TEGRA210_CLK_CLK_32K },
+	{ .con_id = "clk_m_div2", .dt_id = TEGRA210_CLK_CLK_M_DIV2 },
+	{ .con_id = "clk_m_div4", .dt_id = TEGRA210_CLK_CLK_M_DIV4 },
+	{ .con_id = "pll_c", .dt_id = TEGRA210_CLK_PLL_C },
+	{ .con_id = "pll_c_out1", .dt_id = TEGRA210_CLK_PLL_C_OUT1 },
+	{ .con_id = "pll_c2", .dt_id = TEGRA210_CLK_PLL_C2 },
+	{ .con_id = "pll_c3", .dt_id = TEGRA210_CLK_PLL_C3 },
+	{ .con_id = "pll_p", .dt_id = TEGRA210_CLK_PLL_P },
+	{ .con_id = "pll_p_out1", .dt_id = TEGRA210_CLK_PLL_P_OUT1 },
+	{ .con_id = "pll_p_out2", .dt_id = TEGRA210_CLK_PLL_P_OUT2 },
+	{ .con_id = "pll_p_out3", .dt_id = TEGRA210_CLK_PLL_P_OUT3 },
+	{ .con_id = "pll_p_out4", .dt_id = TEGRA210_CLK_PLL_P_OUT4 },
+	{ .con_id = "pll_m", .dt_id = TEGRA210_CLK_PLL_M },
+	{ .con_id = "pll_m_out1", .dt_id = TEGRA210_CLK_PLL_M_OUT1 },
+	{ .con_id = "pll_x", .dt_id = TEGRA210_CLK_PLL_X },
+	{ .con_id = "pll_x_out0", .dt_id = TEGRA210_CLK_PLL_X_OUT0 },
+	{ .con_id = "pll_u", .dt_id = TEGRA210_CLK_PLL_U },
+	{ .con_id = "pll_u_out", .dt_id = TEGRA210_CLK_PLL_U_OUT },
+	{ .con_id = "pll_u_out1", .dt_id = TEGRA210_CLK_PLL_U_OUT1 },
+	{ .con_id = "pll_u_out2", .dt_id = TEGRA210_CLK_PLL_U_OUT2 },
+	{ .con_id = "pll_u_480M", .dt_id = TEGRA210_CLK_PLL_U_480M },
+	{ .con_id = "pll_u_60M", .dt_id = TEGRA210_CLK_PLL_U_60M },
+	{ .con_id = "pll_u_48M", .dt_id = TEGRA210_CLK_PLL_U_48M },
+	{ .con_id = "pll_d", .dt_id = TEGRA210_CLK_PLL_D },
+	{ .con_id = "pll_d_out0", .dt_id = TEGRA210_CLK_PLL_D_OUT0 },
+	{ .con_id = "pll_d2", .dt_id = TEGRA210_CLK_PLL_D2 },
+	{ .con_id = "pll_d2_out0", .dt_id = TEGRA210_CLK_PLL_D2_OUT0 },
+	{ .con_id = "pll_a", .dt_id = TEGRA210_CLK_PLL_A },
+	{ .con_id = "pll_a_out0", .dt_id = TEGRA210_CLK_PLL_A_OUT0 },
+	{ .con_id = "pll_re_vco", .dt_id = TEGRA210_CLK_PLL_RE_VCO },
+	{ .con_id = "pll_re_out", .dt_id = TEGRA210_CLK_PLL_RE_OUT },
+	{ .con_id = "spdif_in_sync", .dt_id = TEGRA210_CLK_SPDIF_IN_SYNC },
+	{ .con_id = "i2s0_sync", .dt_id = TEGRA210_CLK_I2S0_SYNC },
+	{ .con_id = "i2s1_sync", .dt_id = TEGRA210_CLK_I2S1_SYNC },
+	{ .con_id = "i2s2_sync", .dt_id = TEGRA210_CLK_I2S2_SYNC },
+	{ .con_id = "i2s3_sync", .dt_id = TEGRA210_CLK_I2S3_SYNC },
+	{ .con_id = "i2s4_sync", .dt_id = TEGRA210_CLK_I2S4_SYNC },
+	{ .con_id = "vimclk_sync", .dt_id = TEGRA210_CLK_VIMCLK_SYNC },
+	{ .con_id = "audio0", .dt_id = TEGRA210_CLK_AUDIO0 },
+	{ .con_id = "audio1", .dt_id = TEGRA210_CLK_AUDIO1 },
+	{ .con_id = "audio2", .dt_id = TEGRA210_CLK_AUDIO2 },
+	{ .con_id = "audio3", .dt_id = TEGRA210_CLK_AUDIO3 },
+	{ .con_id = "audio4", .dt_id = TEGRA210_CLK_AUDIO4 },
+	{ .con_id = "spdif", .dt_id = TEGRA210_CLK_SPDIF },
+	{ .con_id = "spdif_2x", .dt_id = TEGRA210_CLK_SPDIF_2X },
+	{ .con_id = "extern1", .dev_id = "clk_out_1", .dt_id = TEGRA210_CLK_EXTERN1 },
+	{ .con_id = "extern2", .dev_id = "clk_out_2", .dt_id = TEGRA210_CLK_EXTERN2 },
+	{ .con_id = "extern3", .dev_id = "clk_out_3", .dt_id = TEGRA210_CLK_EXTERN3 },
+	{ .con_id = "blink", .dt_id = TEGRA210_CLK_BLINK },
+	{ .con_id = "cclk_g", .dt_id = TEGRA210_CLK_CCLK_G },
+	{ .con_id = "cclk_lp", .dt_id = TEGRA210_CLK_CCLK_LP },
+	{ .con_id = "sclk", .dt_id = TEGRA210_CLK_SCLK },
+	{ .con_id = "hclk", .dt_id = TEGRA210_CLK_HCLK },
+	{ .con_id = "pclk", .dt_id = TEGRA210_CLK_PCLK },
+	{ .con_id = "fuse", .dt_id = TEGRA210_CLK_FUSE },
+	{ .dev_id = "rtc-tegra", .dt_id = TEGRA210_CLK_RTC },
+	{ .dev_id = "timer", .dt_id = TEGRA210_CLK_TIMER },
+	{ .con_id = "pll_c4_out0", .dt_id = TEGRA210_CLK_PLL_C4_OUT0 },
+	{ .con_id = "pll_c4_out1", .dt_id = TEGRA210_CLK_PLL_C4_OUT1 },
+	{ .con_id = "pll_c4_out2", .dt_id = TEGRA210_CLK_PLL_C4_OUT2 },
+	{ .con_id = "pll_c4_out3", .dt_id = TEGRA210_CLK_PLL_C4_OUT3 },
+	{ .con_id = "dpaux", .dt_id = TEGRA210_CLK_DPAUX },
+	{ .con_id = "sor0", .dt_id = TEGRA210_CLK_SOR0 },
+};
+
+static struct tegra_audio_clk_info tegra210_audio_plls[] = {
+	{ "pll_a", &pll_a_params, tegra_clk_pll_a, "pll_ref" },
+	{ "pll_a1", &pll_a1_params, tegra_clk_pll_a1, "pll_ref" },
+};
+
+static struct clk **clks;
+
+static void tegra210_utmi_param_configure(void __iomem *clk_base)
+{
+	u32 reg;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) {
+		if (osc_freq == utmi_parameters[i].osc_frequency)
+			break;
+	}
+
+	if (i >= ARRAY_SIZE(utmi_parameters)) {
+		pr_err("%s: Unexpected oscillator freq %lu\n", __func__,
+		       osc_freq);
+		return;
+	}
+
+	reg = readl_relaxed(clk_base + PLLU_HW_PWRDN_CFG0);
+	reg |= PLLU_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE |
+	       PLLU_HW_PWRDN_CFG0_USE_SWITCH_DETECT |
+	       PLLU_HW_PWRDN_CFG0_USE_LOCKDET;
+	reg &= ~(PLLU_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL |
+		  PLLU_HW_PWRDN_CFG0_CLK_SWITCH_SWCTL);
+	writel_relaxed(reg, clk_base + PLLU_HW_PWRDN_CFG0);
+
+	reg = readl_relaxed(clk_base + PLLU_HW_PWRDN_CFG0);
+	reg |= PLLU_HW_PWRDN_CFG0_SEQ_ENABLE;
+	writel_relaxed(reg, clk_base + PLLU_HW_PWRDN_CFG0);
+	udelay(1);
+
+	reg = readl_relaxed(clk_base + PLLU_BASE);
+	reg &= ~PLLU_BASE_CLKENABLE_USB;
+	writel_relaxed(reg, clk_base + PLLU_BASE);
+
+	reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0);
+	reg &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE;
+	writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0);
+
+	udelay(10);
+
+	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2);
+
+	/* Program UTMIP PLL stable and active counts */
+	/* [FIXME] arclk_rst.h says WRONG! This should be 1ms -> 0x50 Check! */
+	reg &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0);
+	reg |= UTMIP_PLL_CFG2_STABLE_COUNT(utmi_parameters[i].stable_count);
+
+	reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0);
+
+	reg |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(utmi_parameters[i].
+					    active_delay_count);
+	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2);
+
+	/* Program UTMIP PLL delay and oscillator frequency counts */
+	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1);
+	reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0);
+
+	reg |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(utmi_parameters[i].
+					    enable_delay_count);
+
+	reg &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0);
+	reg |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(utmi_parameters[i].
+					   xtal_freq_count);
+
+	reg |= UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN;
+	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1);
+
+	/* Remove power downs from UTMIP PLL control bits */
+	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1);
+	reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN;
+	reg |= UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP;
+	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1);
+	udelay(1);
+
+	/* Enable samplers for SNPS, XUSB_HOST, XUSB_DEV */
+	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2);
+	reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP;
+	reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP;
+	reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP;
+	reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN;
+	reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN;
+	reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN;
+	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2);
+
+	/* Setup HW control of UTMIPLL */
+	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1);
+	reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN;
+	reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP;
+	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1);
+
+	reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0);
+	reg |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET;
+	reg &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL;
+	writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0);
+
+	udelay(1);
+
+	reg = readl_relaxed(clk_base + XUSB_PLL_CFG0);
+	reg &= ~XUSB_PLL_CFG0_UTMIPLL_LOCK_DLY;
+	writel_relaxed(reg, clk_base + XUSB_PLL_CFG0);
+
+	udelay(1);
+
+	/* Enable HW control UTMIPLL */
+	reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0);
+	reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE;
+	writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0);
+}
+
+static __init void tegra210_periph_clk_init(void __iomem *clk_base,
+					    void __iomem *pmc_base)
+{
+	struct clk *clk;
+
+	/* xusb_ss_div2 */
+	clk = clk_register_fixed_factor(NULL, "xusb_ss_div2", "xusb_ss_src", 0,
+					1, 2);
+	clks[TEGRA210_CLK_XUSB_SS_DIV2] = clk;
+
+	/* pll_d_dsi_out */
+	clk = clk_register_gate(NULL, "pll_d_dsi_out", "pll_d_out0", 0,
+				clk_base + PLLD_MISC0, 21, 0, &pll_d_lock);
+	clks[TEGRA210_CLK_PLL_D_DSI_OUT] = clk;
+
+	/* dsia */
+	clk = tegra_clk_register_periph_gate("dsia", "pll_d_dsi_out", 0,
+					     clk_base, 0, 48,
+					     periph_clk_enb_refcnt);
+	clks[TEGRA210_CLK_DSIA] = clk;
+
+	/* dsib */
+	clk = tegra_clk_register_periph_gate("dsib", "pll_d_dsi_out", 0,
+					     clk_base, 0, 82,
+					     periph_clk_enb_refcnt);
+	clks[TEGRA210_CLK_DSIB] = clk;
+
+	/* emc mux */
+	clk = clk_register_mux(NULL, "emc_mux", mux_pllmcp_clkm,
+			       ARRAY_SIZE(mux_pllmcp_clkm), 0,
+			       clk_base + CLK_SOURCE_EMC,
+			       29, 3, 0, &emc_lock);
+
+	clk = tegra_clk_register_mc("mc", "emc_mux", clk_base + CLK_SOURCE_EMC,
+				    &emc_lock);
+	clks[TEGRA210_CLK_MC] = clk;
+
+	/* cml0 */
+	clk = clk_register_gate(NULL, "cml0", "pll_e", 0, clk_base + PLLE_AUX,
+				0, 0, &pll_e_lock);
+	clk_register_clkdev(clk, "cml0", NULL);
+	clks[TEGRA210_CLK_CML0] = clk;
+
+	/* cml1 */
+	clk = clk_register_gate(NULL, "cml1", "pll_e", 0, clk_base + PLLE_AUX,
+				1, 0, &pll_e_lock);
+	clk_register_clkdev(clk, "cml1", NULL);
+	clks[TEGRA210_CLK_CML1] = clk;
+
+	tegra_periph_clk_init(clk_base, pmc_base, tegra210_clks, &pll_p_params);
+}
+
+static void __init tegra210_pll_init(void __iomem *clk_base,
+				     void __iomem *pmc)
+{
+	u32 val;
+	struct clk *clk;
+
+	/* PLLC */
+	clk = tegra_clk_register_pllxc_tegra210("pll_c", "pll_ref", clk_base,
+			pmc, 0, &pll_c_params, NULL);
+	if (!WARN_ON(IS_ERR(clk)))
+		clk_register_clkdev(clk, "pll_c", NULL);
+	clks[TEGRA210_CLK_PLL_C] = clk;
+
+	/* PLLC_OUT1 */
+	clk = tegra_clk_register_divider("pll_c_out1_div", "pll_c",
+			clk_base + PLLC_OUT, 0, TEGRA_DIVIDER_ROUND_UP,
+			8, 8, 1, NULL);
+	clk = tegra_clk_register_pll_out("pll_c_out1", "pll_c_out1_div",
+				clk_base + PLLC_OUT, 1, 0,
+				CLK_SET_RATE_PARENT, 0, NULL);
+	clk_register_clkdev(clk, "pll_c_out1", NULL);
+	clks[TEGRA210_CLK_PLL_C_OUT1] = clk;
+
+	/* PLLC_UD */
+	clk = clk_register_fixed_factor(NULL, "pll_c_ud", "pll_c",
+					CLK_SET_RATE_PARENT, 1, 1);
+	clk_register_clkdev(clk, "pll_c_ud", NULL);
+	clks[TEGRA210_CLK_PLL_C_UD] = clk;
+
+	/* PLLC2 */
+	clk = tegra_clk_register_pllc_tegra210("pll_c2", "pll_ref", clk_base,
+			     pmc, 0, &pll_c2_params, NULL);
+	clk_register_clkdev(clk, "pll_c2", NULL);
+	clks[TEGRA210_CLK_PLL_C2] = clk;
+
+	/* PLLC3 */
+	clk = tegra_clk_register_pllc_tegra210("pll_c3", "pll_ref", clk_base,
+			     pmc, 0, &pll_c3_params, NULL);
+	clk_register_clkdev(clk, "pll_c3", NULL);
+	clks[TEGRA210_CLK_PLL_C3] = clk;
+
+	/* PLLM */
+	clk = tegra_clk_register_pllm("pll_m", "osc", clk_base, pmc,
+			     CLK_SET_RATE_GATE, &pll_m_params, NULL);
+	clk_register_clkdev(clk, "pll_m", NULL);
+	clks[TEGRA210_CLK_PLL_M] = clk;
+
+	/* PLLMB */
+	clk = tegra_clk_register_pllmb("pll_mb", "osc", clk_base, pmc,
+			     CLK_SET_RATE_GATE, &pll_mb_params, NULL);
+	clk_register_clkdev(clk, "pll_mb", NULL);
+	clks[TEGRA210_CLK_PLL_MB] = clk;
+
+	clk_register_clkdev(clk, "pll_m_out1", NULL);
+	clks[TEGRA210_CLK_PLL_M_OUT1] = clk;
+
+	/* PLLM_UD */
+	clk = clk_register_fixed_factor(NULL, "pll_m_ud", "pll_m",
+					CLK_SET_RATE_PARENT, 1, 1);
+	clk_register_clkdev(clk, "pll_m_ud", NULL);
+	clks[TEGRA210_CLK_PLL_M_UD] = clk;
+
+	/* PLLU_VCO */
+	val = readl(clk_base + pll_u_vco_params.base_reg);
+	val &= ~BIT(24); /* disable PLLU_OVERRIDE */
+	writel(val, clk_base + pll_u_vco_params.base_reg);
+
+	clk = tegra_clk_register_pllre("pll_u_vco", "pll_ref", clk_base, pmc,
+			    0, &pll_u_vco_params, &pll_u_lock, pll_ref_freq);
+	clk_register_clkdev(clk, "pll_u_vco", NULL);
+	clks[TEGRA210_CLK_PLL_U] = clk;
+
+	/* PLLU_OUT */
+	clk = clk_register_divider_table(NULL, "pll_u_out", "pll_u_vco", 0,
+					 clk_base + PLLU_BASE, 16, 4, 0,
+					 pll_vco_post_div_table, NULL);
+	clk_register_clkdev(clk, "pll_u_out", NULL);
+	clks[TEGRA210_CLK_PLL_U_OUT] = clk;
+
+	/* PLLU_OUT1 */
+	clk = tegra_clk_register_divider("pll_u_out1_div", "pll_u_out",
+				clk_base + PLLU_OUTA, 0,
+				TEGRA_DIVIDER_ROUND_UP,
+				8, 8, 1, &pll_u_lock);
+	clk = tegra_clk_register_pll_out("pll_u_out1", "pll_u_out1_div",
+				clk_base + PLLU_OUTA, 1, 0,
+				CLK_SET_RATE_PARENT, 0, &pll_u_lock);
+	clk_register_clkdev(clk, "pll_u_out1", NULL);
+	clks[TEGRA210_CLK_PLL_U_OUT1] = clk;
+
+	/* PLLU_OUT2 */
+	clk = tegra_clk_register_divider("pll_u_out2_div", "pll_u_out",
+				clk_base + PLLU_OUTA, 0,
+				TEGRA_DIVIDER_ROUND_UP,
+				24, 8, 1, &pll_u_lock);
+	clk = tegra_clk_register_pll_out("pll_u_out2", "pll_u_out2_div",
+				clk_base + PLLU_OUTA, 17, 16,
+				CLK_SET_RATE_PARENT, 0, &pll_u_lock);
+	clk_register_clkdev(clk, "pll_u_out2", NULL);
+	clks[TEGRA210_CLK_PLL_U_OUT2] = clk;
+
+	tegra210_utmi_param_configure(clk_base);
+
+	/* PLLU_480M */
+	clk = clk_register_gate(NULL, "pll_u_480M", "pll_u_vco",
+				CLK_SET_RATE_PARENT, clk_base + PLLU_BASE,
+				22, 0, &pll_u_lock);
+	clk_register_clkdev(clk, "pll_u_480M", NULL);
+	clks[TEGRA210_CLK_PLL_U_480M] = clk;
+
+	/* PLLU_60M */
+	clk = clk_register_gate(NULL, "pll_u_60M", "pll_u_out2",
+				CLK_SET_RATE_PARENT, clk_base + PLLU_BASE,
+				23, 0, NULL);
+	clk_register_clkdev(clk, "pll_u_60M", NULL);
+	clks[TEGRA210_CLK_PLL_U_60M] = clk;
+
+	/* PLLU_48M */
+	clk = clk_register_gate(NULL, "pll_u_48M", "pll_u_out1",
+				CLK_SET_RATE_PARENT, clk_base + PLLU_BASE,
+				25, 0, NULL);
+	clk_register_clkdev(clk, "pll_u_48M", NULL);
+	clks[TEGRA210_CLK_PLL_U_48M] = clk;
+
+	/* PLLD */
+	clk = tegra_clk_register_pll("pll_d", "pll_ref", clk_base, pmc, 0,
+			    &pll_d_params, &pll_d_lock);
+	clk_register_clkdev(clk, "pll_d", NULL);
+	clks[TEGRA210_CLK_PLL_D] = clk;
+
+	/* PLLD_OUT0 */
+	clk = clk_register_fixed_factor(NULL, "pll_d_out0", "pll_d",
+					CLK_SET_RATE_PARENT, 1, 2);
+	clk_register_clkdev(clk, "pll_d_out0", NULL);
+	clks[TEGRA210_CLK_PLL_D_OUT0] = clk;
+
+	/* PLLRE */
+	clk = tegra_clk_register_pllre("pll_re_vco", "pll_ref", clk_base, pmc,
+			     0, &pll_re_vco_params, &pll_re_lock, pll_ref_freq);
+	clk_register_clkdev(clk, "pll_re_vco", NULL);
+	clks[TEGRA210_CLK_PLL_RE_VCO] = clk;
+
+	clk = clk_register_divider_table(NULL, "pll_re_out", "pll_re_vco", 0,
+					 clk_base + PLLRE_BASE, 16, 5, 0,
+					 pll_vco_post_div_table, &pll_re_lock);
+	clk_register_clkdev(clk, "pll_re_out", NULL);
+	clks[TEGRA210_CLK_PLL_RE_OUT] = clk;
+
+	/* PLLE */
+	clk = tegra_clk_register_plle_tegra210("pll_e", "pll_ref",
+				      clk_base, 0, &pll_e_params, NULL);
+	clk_register_clkdev(clk, "pll_e", NULL);
+	clks[TEGRA210_CLK_PLL_E] = clk;
+
+	/* PLLC4 */
+	clk = tegra_clk_register_pllre("pll_c4_vco", "pll_ref", clk_base, pmc,
+			     0, &pll_c4_vco_params, NULL, pll_ref_freq);
+	clk_register_clkdev(clk, "pll_c4_vco", NULL);
+	clks[TEGRA210_CLK_PLL_C4] = clk;
+
+	/* PLLC4_OUT0 */
+	clk = clk_register_divider_table(NULL, "pll_c4_out0", "pll_c4_vco", 0,
+					 clk_base + PLLC4_BASE, 19, 4, 0,
+					 pll_vco_post_div_table, NULL);
+	clk_register_clkdev(clk, "pll_c4_out0", NULL);
+	clks[TEGRA210_CLK_PLL_C4_OUT0] = clk;
+
+	/* PLLC4_OUT1 */
+	clk = clk_register_fixed_factor(NULL, "pll_c4_out1", "pll_c4_vco",
+					CLK_SET_RATE_PARENT, 1, 3);
+	clk_register_clkdev(clk, "pll_c4_out1", NULL);
+	clks[TEGRA210_CLK_PLL_C4_OUT1] = clk;
+
+	/* PLLC4_OUT2 */
+	clk = clk_register_fixed_factor(NULL, "pll_c4_out2", "pll_c4_vco",
+					CLK_SET_RATE_PARENT, 1, 5);
+	clk_register_clkdev(clk, "pll_c4_out2", NULL);
+	clks[TEGRA210_CLK_PLL_C4_OUT2] = clk;
+
+	/* PLLC4_OUT3 */
+	clk = tegra_clk_register_divider("pll_c4_out3_div", "pll_c4_out0",
+			clk_base + PLLC4_OUT, 0, TEGRA_DIVIDER_ROUND_UP,
+			8, 8, 1, NULL);
+	clk = tegra_clk_register_pll_out("pll_c4_out3", "pll_c4_out3_div",
+				clk_base + PLLC4_OUT, 1, 0,
+				CLK_SET_RATE_PARENT, 0, NULL);
+	clk_register_clkdev(clk, "pll_c4_out3", NULL);
+	clks[TEGRA210_CLK_PLL_C4_OUT3] = clk;
+
+	/* PLLDP */
+	clk = tegra_clk_register_pllss_tegra210("pll_dp", "pll_ref", clk_base,
+					0, &pll_dp_params, NULL);
+	clk_register_clkdev(clk, "pll_dp", NULL);
+	clks[TEGRA210_CLK_PLL_DP] = clk;
+
+	/* PLLD2 */
+	clk = tegra_clk_register_pllss_tegra210("pll_d2", "pll_ref", clk_base,
+					0, &pll_d2_params, NULL);
+	clk_register_clkdev(clk, "pll_d2", NULL);
+	clks[TEGRA210_CLK_PLL_D2] = clk;
+
+	/* PLLD2_OUT0 */
+	clk = clk_register_fixed_factor(NULL, "pll_d2_out0", "pll_d2",
+					CLK_SET_RATE_PARENT, 1, 1);
+	clk_register_clkdev(clk, "pll_d2_out0", NULL);
+	clks[TEGRA210_CLK_PLL_D2_OUT0] = clk;
+
+	/* PLLP_OUT2 */
+	clk = clk_register_fixed_factor(NULL, "pll_p_out2", "pll_p",
+					CLK_SET_RATE_PARENT, 1, 2);
+	clk_register_clkdev(clk, "pll_p_out2", NULL);
+	clks[TEGRA210_CLK_PLL_P_OUT2] = clk;
+
+}
+
+/* Tegra210 CPU clock and reset control functions */
+static void tegra210_wait_cpu_in_reset(u32 cpu)
+{
+	unsigned int reg;
+
+	do {
+		reg = readl(clk_base + CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
+		cpu_relax();
+	} while (!(reg & (1 << cpu)));  /* check CPU been reset or not */
+}
+
+static void tegra210_disable_cpu_clock(u32 cpu)
+{
+	/* flow controller would take care in the power sequence. */
+}
+
+#ifdef CONFIG_PM_SLEEP
+static void tegra210_cpu_clock_suspend(void)
+{
+	/* switch coresite to clk_m, save off original source */
+	tegra210_cpu_clk_sctx.clk_csite_src =
+				readl(clk_base + CLK_SOURCE_CSITE);
+	writel(3 << 30, clk_base + CLK_SOURCE_CSITE);
+}
+
+static void tegra210_cpu_clock_resume(void)
+{
+	writel(tegra210_cpu_clk_sctx.clk_csite_src,
+				clk_base + CLK_SOURCE_CSITE);
+}
+#endif
+
+static struct tegra_cpu_car_ops tegra210_cpu_car_ops = {
+	.wait_for_reset	= tegra210_wait_cpu_in_reset,
+	.disable_clock	= tegra210_disable_cpu_clock,
+#ifdef CONFIG_PM_SLEEP
+	.suspend	= tegra210_cpu_clock_suspend,
+	.resume		= tegra210_cpu_clock_resume,
+#endif
+};
+
+static const struct of_device_id pmc_match[] __initconst = {
+	{ .compatible = "nvidia,tegra210-pmc" },
+	{ },
+};
+
+static struct tegra_clk_init_table init_table[] __initdata = {
+	{ TEGRA210_CLK_UARTA, TEGRA210_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA210_CLK_UARTB, TEGRA210_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA210_CLK_UARTC, TEGRA210_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA210_CLK_UARTD, TEGRA210_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA210_CLK_PLL_A, TEGRA210_CLK_CLK_MAX, 564480000, 1 },
+	{ TEGRA210_CLK_PLL_A_OUT0, TEGRA210_CLK_CLK_MAX, 11289600, 1 },
+	{ TEGRA210_CLK_EXTERN1, TEGRA210_CLK_PLL_A_OUT0, 0, 1 },
+	{ TEGRA210_CLK_CLK_OUT_1_MUX, TEGRA210_CLK_EXTERN1, 0, 1 },
+	{ TEGRA210_CLK_CLK_OUT_1, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA210_CLK_I2S0, TEGRA210_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA210_CLK_I2S1, TEGRA210_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA210_CLK_I2S2, TEGRA210_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA210_CLK_I2S3, TEGRA210_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA210_CLK_I2S4, TEGRA210_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA210_CLK_HOST1X, TEGRA210_CLK_PLL_P, 136000000, 1 },
+	{ TEGRA210_CLK_SCLK_MUX, TEGRA210_CLK_PLL_P, 0, 1 },
+	{ TEGRA210_CLK_SCLK, TEGRA210_CLK_CLK_MAX, 102000000, 1 },
+	{ TEGRA210_CLK_DFLL_SOC, TEGRA210_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA210_CLK_DFLL_REF, TEGRA210_CLK_PLL_P, 51000000, 1 },
+	{ TEGRA210_CLK_SBC4, TEGRA210_CLK_PLL_P, 12000000, 1 },
+	{ TEGRA210_CLK_PLL_RE_VCO, TEGRA210_CLK_CLK_MAX, 672000000, 1 },
+	{ TEGRA210_CLK_PLL_U_OUT1, TEGRA210_CLK_CLK_MAX, 48000000, 1 },
+	{ TEGRA210_CLK_PLL_U_OUT2, TEGRA210_CLK_CLK_MAX, 60000000, 1 },
+	{ TEGRA210_CLK_XUSB_GATE, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA210_CLK_XUSB_SS_SRC, TEGRA210_CLK_PLL_U_480M, 120000000, 0 },
+	{ TEGRA210_CLK_XUSB_FS_SRC, TEGRA210_CLK_PLL_U_48M, 48000000, 0 },
+	{ TEGRA210_CLK_XUSB_HS_SRC, TEGRA210_CLK_XUSB_SS_SRC, 120000000, 0 },
+	{ TEGRA210_CLK_XUSB_SSP_SRC, TEGRA210_CLK_XUSB_SS_SRC, 120000000, 0 },
+	{ TEGRA210_CLK_XUSB_FALCON_SRC, TEGRA210_CLK_PLL_P_OUT_XUSB, 204000000, 0 },
+	{ TEGRA210_CLK_XUSB_HOST_SRC, TEGRA210_CLK_PLL_P_OUT_XUSB, 102000000, 0 },
+	{ TEGRA210_CLK_XUSB_DEV_SRC, TEGRA210_CLK_PLL_P_OUT_XUSB, 102000000, 0 },
+	{ TEGRA210_CLK_SATA, TEGRA210_CLK_PLL_P, 104000000, 0 },
+	{ TEGRA210_CLK_SATA_OOB, TEGRA210_CLK_PLL_P, 204000000, 0 },
+	{ TEGRA210_CLK_EMC, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA210_CLK_MSELECT, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA210_CLK_CSITE, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA210_CLK_TSENSOR, TEGRA210_CLK_CLK_M, 400000, 0 },
+	{ TEGRA210_CLK_I2C1, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_I2C2, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_I2C3, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_I2C4, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_I2C5, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_I2C6, TEGRA210_CLK_PLL_P, 0, 0 },
+	{ TEGRA210_CLK_PLL_DP, TEGRA210_CLK_CLK_MAX, 270000000, 0 },
+	{ TEGRA210_CLK_SOC_THERM, TEGRA210_CLK_PLL_P, 51000000, 0 },
+	{ TEGRA210_CLK_CCLK_G, TEGRA210_CLK_CLK_MAX, 0, 1 },
+	/* This MUST be the last entry. */
+	{ TEGRA210_CLK_CLK_MAX, TEGRA210_CLK_CLK_MAX, 0, 0 },
+};
+
+/**
+ * tegra210_clock_apply_init_table - initialize clocks on Tegra210 SoCs
+ *
+ * Program an initial clock rate and enable or disable clocks needed
+ * by the rest of the kernel, for Tegra210 SoCs.  It is intended to be
+ * called by assigning a pointer to it to tegra_clk_apply_init_table -
+ * this will be called as an arch_initcall.  No return value.
+ */
+static void __init tegra210_clock_apply_init_table(void)
+{
+	tegra_init_from_table(init_table, clks, TEGRA210_CLK_CLK_MAX);
+}
+
+/**
+ * tegra210_clock_init - Tegra210-specific clock initialization
+ * @np: struct device_node * of the DT node for the SoC CAR IP block
+ *
+ * Register most SoC clocks for the Tegra210 system-on-chip.  Intended
+ * to be called by the OF init code when a DT node with the
+ * "nvidia,tegra210-car" string is encountered, and declared with
+ * CLK_OF_DECLARE.  No return value.
+ */
+static void __init tegra210_clock_init(struct device_node *np)
+{
+	struct device_node *node;
+	u32 value, clk_m_div;
+
+	clk_base = of_iomap(np, 0);
+	if (!clk_base) {
+		pr_err("ioremap tegra210 CAR failed\n");
+		return;
+	}
+
+	node = of_find_matching_node(NULL, pmc_match);
+	if (!node) {
+		pr_err("Failed to find pmc node\n");
+		WARN_ON(1);
+		return;
+	}
+
+	pmc_base = of_iomap(node, 0);
+	if (!pmc_base) {
+		pr_err("Can't map pmc registers\n");
+		WARN_ON(1);
+		return;
+	}
+
+	clks = tegra_clk_init(clk_base, TEGRA210_CLK_CLK_MAX,
+			      TEGRA210_CAR_BANK_COUNT);
+	if (!clks)
+		return;
+
+	value = clk_readl(clk_base + SPARE_REG0) >> CLK_M_DIVISOR_SHIFT;
+	clk_m_div = (value & CLK_M_DIVISOR_MASK) + 1;
+
+	if (tegra_osc_clk_init(clk_base, tegra210_clks, tegra210_input_freq,
+			       ARRAY_SIZE(tegra210_input_freq), clk_m_div,
+			       &osc_freq, &pll_ref_freq) < 0)
+		return;
+
+	tegra_fixed_clk_init(tegra210_clks);
+	tegra210_pll_init(clk_base, pmc_base);
+	tegra210_periph_clk_init(clk_base, pmc_base);
+	tegra_audio_clk_init(clk_base, pmc_base, tegra210_clks,
+			     tegra210_audio_plls,
+			     ARRAY_SIZE(tegra210_audio_plls));
+	tegra_pmc_clk_init(pmc_base, tegra210_clks);
+
+	/* For Tegra210, PLLD is the only source for DSIA & DSIB */
+	value = clk_readl(clk_base + PLLD_BASE);
+	value &= ~BIT(25);
+	clk_writel(value, clk_base + PLLD_BASE);
+
+	tegra_clk_apply_init_table = tegra210_clock_apply_init_table;
+
+	tegra_super_clk_gen5_init(clk_base, pmc_base, tegra210_clks,
+				  &pll_x_params);
+	tegra_add_of_provider(np);
+	tegra_register_devclks(devclks, ARRAY_SIZE(devclks));
+
+	tegra_cpu_car_ops = &tegra210_cpu_car_ops;
+}
+CLK_OF_DECLARE(tegra210, "nvidia,tegra210-car", tegra210_clock_init);
diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c
index b90db615c29e..0478565cf292 100644
--- a/drivers/clk/tegra/clk-tegra30.c
+++ b/drivers/clk/tegra/clk-tegra30.c
@@ -224,188 +224,192 @@ struct utmi_clk_param {
 };
 
 static const struct utmi_clk_param utmi_parameters[] = {
-/*	OSC_FREQUENCY, ENABLE_DLY, STABLE_CNT, ACTIVE_DLY, XTAL_FREQ_CNT */
-	{13000000,     0x02,       0x33,       0x05,       0x7F},
-	{19200000,     0x03,       0x4B,       0x06,       0xBB},
-	{12000000,     0x02,       0x2F,       0x04,       0x76},
-	{26000000,     0x04,       0x66,       0x09,       0xFE},
-	{16800000,     0x03,       0x41,       0x0A,       0xA4},
+	{
+		.osc_frequency = 13000000, .enable_delay_count = 0x02,
+		.stable_count = 0x33, .active_delay_count = 0x05,
+		.xtal_freq_count = 0x7f
+	}, {
+		.osc_frequency = 19200000, .enable_delay_count = 0x03,
+		.stable_count = 0x4b, .active_delay_count = 0x06,
+		.xtal_freq_count = 0xbb
+	}, {
+		.osc_frequency = 12000000, .enable_delay_count = 0x02,
+		.stable_count = 0x2f, .active_delay_count = 0x04,
+		.xtal_freq_count = 0x76
+	}, {
+		.osc_frequency = 26000000, .enable_delay_count = 0x04,
+		.stable_count = 0x66, .active_delay_count = 0x09,
+		.xtal_freq_count = 0xfe
+	}, {
+		.osc_frequency = 16800000, .enable_delay_count = 0x03,
+		.stable_count = 0x41, .active_delay_count = 0x0a,
+		.xtal_freq_count = 0xa4
+	},
 };
 
 static struct tegra_clk_pll_freq_table pll_c_freq_table[] = {
-	{ 12000000, 1040000000, 520,  6, 0, 8},
-	{ 13000000, 1040000000, 480,  6, 0, 8},
-	{ 16800000, 1040000000, 495,  8, 0, 8},	/* actual: 1039.5 MHz */
-	{ 19200000, 1040000000, 325,  6, 0, 6},
-	{ 26000000, 1040000000, 520, 13, 0, 8},
-
-	{ 12000000, 832000000, 416,  6, 0, 8},
-	{ 13000000, 832000000, 832, 13, 0, 8},
-	{ 16800000, 832000000, 396,  8, 0, 8},	/* actual: 831.6 MHz */
-	{ 19200000, 832000000, 260,  6, 0, 8},
-	{ 26000000, 832000000, 416, 13, 0, 8},
-
-	{ 12000000, 624000000, 624, 12, 0, 8},
-	{ 13000000, 624000000, 624, 13, 0, 8},
-	{ 16800000, 600000000, 520, 14, 0, 8},
-	{ 19200000, 624000000, 520, 16, 0, 8},
-	{ 26000000, 624000000, 624, 26, 0, 8},
-
-	{ 12000000, 600000000, 600, 12, 0, 8},
-	{ 13000000, 600000000, 600, 13, 0, 8},
-	{ 16800000, 600000000, 500, 14, 0, 8},
-	{ 19200000, 600000000, 375, 12, 0, 6},
-	{ 26000000, 600000000, 600, 26, 0, 8},
-
-	{ 12000000, 520000000, 520, 12, 0, 8},
-	{ 13000000, 520000000, 520, 13, 0, 8},
-	{ 16800000, 520000000, 495, 16, 0, 8},	/* actual: 519.75 MHz */
-	{ 19200000, 520000000, 325, 12, 0, 6},
-	{ 26000000, 520000000, 520, 26, 0, 8},
-
-	{ 12000000, 416000000, 416, 12, 0, 8},
-	{ 13000000, 416000000, 416, 13, 0, 8},
-	{ 16800000, 416000000, 396, 16, 0, 8},	/* actual: 415.8 MHz */
-	{ 19200000, 416000000, 260, 12, 0, 6},
-	{ 26000000, 416000000, 416, 26, 0, 8},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 1040000000, 520,  6, 1, 8 },
+	{ 13000000, 1040000000, 480,  6, 1, 8 },
+	{ 16800000, 1040000000, 495,  8, 1, 8 }, /* actual: 1039.5 MHz */
+	{ 19200000, 1040000000, 325,  6, 1, 6 },
+	{ 26000000, 1040000000, 520, 13, 1, 8 },
+	{ 12000000,  832000000, 416,  6, 1, 8 },
+	{ 13000000,  832000000, 832, 13, 1, 8 },
+	{ 16800000,  832000000, 396,  8, 1, 8 }, /* actual: 831.6 MHz */
+	{ 19200000,  832000000, 260,  6, 1, 8 },
+	{ 26000000,  832000000, 416, 13, 1, 8 },
+	{ 12000000,  624000000, 624, 12, 1, 8 },
+	{ 13000000,  624000000, 624, 13, 1, 8 },
+	{ 16800000,  600000000, 520, 14, 1, 8 },
+	{ 19200000,  624000000, 520, 16, 1, 8 },
+	{ 26000000,  624000000, 624, 26, 1, 8 },
+	{ 12000000,  600000000, 600, 12, 1, 8 },
+	{ 13000000,  600000000, 600, 13, 1, 8 },
+	{ 16800000,  600000000, 500, 14, 1, 8 },
+	{ 19200000,  600000000, 375, 12, 1, 6 },
+	{ 26000000,  600000000, 600, 26, 1, 8 },
+	{ 12000000,  520000000, 520, 12, 1, 8 },
+	{ 13000000,  520000000, 520, 13, 1, 8 },
+	{ 16800000,  520000000, 495, 16, 1, 8 }, /* actual: 519.75 MHz */
+	{ 19200000,  520000000, 325, 12, 1, 6 },
+	{ 26000000,  520000000, 520, 26, 1, 8 },
+	{ 12000000,  416000000, 416, 12, 1, 8 },
+	{ 13000000,  416000000, 416, 13, 1, 8 },
+	{ 16800000,  416000000, 396, 16, 1, 8 }, /* actual: 415.8 MHz */
+	{ 19200000,  416000000, 260, 12, 1, 6 },
+	{ 26000000,  416000000, 416, 26, 1, 8 },
+	{        0,          0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_m_freq_table[] = {
-	{ 12000000, 666000000, 666, 12, 0, 8},
-	{ 13000000, 666000000, 666, 13, 0, 8},
-	{ 16800000, 666000000, 555, 14, 0, 8},
-	{ 19200000, 666000000, 555, 16, 0, 8},
-	{ 26000000, 666000000, 666, 26, 0, 8},
-	{ 12000000, 600000000, 600, 12, 0, 8},
-	{ 13000000, 600000000, 600, 13, 0, 8},
-	{ 16800000, 600000000, 500, 14, 0, 8},
-	{ 19200000, 600000000, 375, 12, 0, 6},
-	{ 26000000, 600000000, 600, 26, 0, 8},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 666000000, 666, 12, 1, 8 },
+	{ 13000000, 666000000, 666, 13, 1, 8 },
+	{ 16800000, 666000000, 555, 14, 1, 8 },
+	{ 19200000, 666000000, 555, 16, 1, 8 },
+	{ 26000000, 666000000, 666, 26, 1, 8 },
+	{ 12000000, 600000000, 600, 12, 1, 8 },
+	{ 13000000, 600000000, 600, 13, 1, 8 },
+	{ 16800000, 600000000, 500, 14, 1, 8 },
+	{ 19200000, 600000000, 375, 12, 1, 6 },
+	{ 26000000, 600000000, 600, 26, 1, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_p_freq_table[] = {
-	{ 12000000, 216000000, 432, 12, 1, 8},
-	{ 13000000, 216000000, 432, 13, 1, 8},
-	{ 16800000, 216000000, 360, 14, 1, 8},
-	{ 19200000, 216000000, 360, 16, 1, 8},
-	{ 26000000, 216000000, 432, 26, 1, 8},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 216000000, 432, 12, 2, 8 },
+	{ 13000000, 216000000, 432, 13, 2, 8 },
+	{ 16800000, 216000000, 360, 14, 2, 8 },
+	{ 19200000, 216000000, 360, 16, 2, 8 },
+	{ 26000000, 216000000, 432, 26, 2, 8 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_a_freq_table[] = {
-	{ 9600000, 564480000, 294, 5, 0, 4},
-	{ 9600000, 552960000, 288, 5, 0, 4},
-	{ 9600000, 24000000,  5,   2, 0, 1},
-
-	{ 28800000, 56448000, 49, 25, 0, 1},
-	{ 28800000, 73728000, 64, 25, 0, 1},
-	{ 28800000, 24000000,  5,  6, 0, 1},
-	{ 0, 0, 0, 0, 0, 0 },
+	{  9600000, 564480000, 294,  5, 1, 4 },
+	{  9600000, 552960000, 288,  5, 1, 4 },
+	{  9600000,  24000000,   5,  2, 1, 1 },
+	{ 28800000,  56448000,  49, 25, 1, 1 },
+	{ 28800000,  73728000,  64, 25, 1, 1 },
+	{ 28800000,  24000000,   5,  6, 1, 1 },
+	{        0,         0,   0,  0, 0, 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_d_freq_table[] = {
-	{ 12000000, 216000000, 216, 12, 0, 4},
-	{ 13000000, 216000000, 216, 13, 0, 4},
-	{ 16800000, 216000000, 180, 14, 0, 4},
-	{ 19200000, 216000000, 180, 16, 0, 4},
-	{ 26000000, 216000000, 216, 26, 0, 4},
-
-	{ 12000000, 594000000, 594, 12, 0, 8},
-	{ 13000000, 594000000, 594, 13, 0, 8},
-	{ 16800000, 594000000, 495, 14, 0, 8},
-	{ 19200000, 594000000, 495, 16, 0, 8},
-	{ 26000000, 594000000, 594, 26, 0, 8},
-
-	{ 12000000, 1000000000, 1000, 12, 0, 12},
-	{ 13000000, 1000000000, 1000, 13, 0, 12},
-	{ 19200000, 1000000000, 625,  12, 0, 8},
-	{ 26000000, 1000000000, 1000, 26, 0, 12},
-
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000,  216000000,  216, 12, 1,  4 },
+	{ 13000000,  216000000,  216, 13, 1,  4 },
+	{ 16800000,  216000000,  180, 14, 1,  4 },
+	{ 19200000,  216000000,  180, 16, 1,  4 },
+	{ 26000000,  216000000,  216, 26, 1,  4 },
+	{ 12000000,  594000000,  594, 12, 1,  8 },
+	{ 13000000,  594000000,  594, 13, 1,  8 },
+	{ 16800000,  594000000,  495, 14, 1,  8 },
+	{ 19200000,  594000000,  495, 16, 1,  8 },
+	{ 26000000,  594000000,  594, 26, 1,  8 },
+	{ 12000000, 1000000000, 1000, 12, 1, 12 },
+	{ 13000000, 1000000000, 1000, 13, 1, 12 },
+	{ 19200000, 1000000000,  625, 12, 1,  8 },
+	{ 26000000, 1000000000, 1000, 26, 1, 12 },
+	{        0,          0,    0,  0, 0,  0 },
 };
 
-static struct pdiv_map pllu_p[] = {
+static const struct pdiv_map pllu_p[] = {
 	{ .pdiv = 1, .hw_val = 1 },
 	{ .pdiv = 2, .hw_val = 0 },
 	{ .pdiv = 0, .hw_val = 0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
-	{ 12000000, 480000000, 960, 12, 0, 12},
-	{ 13000000, 480000000, 960, 13, 0, 12},
-	{ 16800000, 480000000, 400, 7,  0, 5},
-	{ 19200000, 480000000, 200, 4,  0, 3},
-	{ 26000000, 480000000, 960, 26, 0, 12},
-	{ 0, 0, 0, 0, 0, 0 },
+	{ 12000000, 480000000, 960, 12, 1, 12 },
+	{ 13000000, 480000000, 960, 13, 1, 12 },
+	{ 16800000, 480000000, 400,  7, 1,  5 },
+	{ 19200000, 480000000, 200,  4, 1,  3 },
+	{ 26000000, 480000000, 960, 26, 1, 12 },
+	{        0,         0,   0,  0, 0,  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_x_freq_table[] = {
 	/* 1.7 GHz */
-	{ 12000000, 1700000000, 850,  6,  0, 8},
-	{ 13000000, 1700000000, 915,  7,  0, 8},	/* actual: 1699.2 MHz */
-	{ 16800000, 1700000000, 708,  7,  0, 8},	/* actual: 1699.2 MHz */
-	{ 19200000, 1700000000, 885,  10, 0, 8},	/* actual: 1699.2 MHz */
-	{ 26000000, 1700000000, 850,  13, 0, 8},
-
+	{ 12000000, 1700000000, 850,   6, 1, 8 },
+	{ 13000000, 1700000000, 915,   7, 1, 8 }, /* actual: 1699.2 MHz */
+	{ 16800000, 1700000000, 708,   7, 1, 8 }, /* actual: 1699.2 MHz */
+	{ 19200000, 1700000000, 885,  10, 1, 8 }, /* actual: 1699.2 MHz */
+	{ 26000000, 1700000000, 850,  13, 1, 8 },
 	/* 1.6 GHz */
-	{ 12000000, 1600000000, 800,  6,  0, 8},
-	{ 13000000, 1600000000, 738,  6,  0, 8},	/* actual: 1599.0 MHz */
-	{ 16800000, 1600000000, 857,  9,  0, 8},	/* actual: 1599.7 MHz */
-	{ 19200000, 1600000000, 500,  6,  0, 8},
-	{ 26000000, 1600000000, 800,  13, 0, 8},
-
+	{ 12000000, 1600000000, 800,   6, 1, 8 },
+	{ 13000000, 1600000000, 738,   6, 1, 8 }, /* actual: 1599.0 MHz */
+	{ 16800000, 1600000000, 857,   9, 1, 8 }, /* actual: 1599.7 MHz */
+	{ 19200000, 1600000000, 500,   6, 1, 8 },
+	{ 26000000, 1600000000, 800,  13, 1, 8 },
 	/* 1.5 GHz */
-	{ 12000000, 1500000000, 750,  6,  0, 8},
-	{ 13000000, 1500000000, 923,  8,  0, 8},	/* actual: 1499.8 MHz */
-	{ 16800000, 1500000000, 625,  7,  0, 8},
-	{ 19200000, 1500000000, 625,  8,  0, 8},
-	{ 26000000, 1500000000, 750,  13, 0, 8},
-
+	{ 12000000, 1500000000, 750,   6, 1, 8 },
+	{ 13000000, 1500000000, 923,   8, 1, 8 }, /* actual: 1499.8 MHz */
+	{ 16800000, 1500000000, 625,   7, 1, 8 },
+	{ 19200000, 1500000000, 625,   8, 1, 8 },
+	{ 26000000, 1500000000, 750,  13, 1, 8 },
 	/* 1.4 GHz */
-	{ 12000000, 1400000000, 700,  6,  0, 8},
-	{ 13000000, 1400000000, 969,  9,  0, 8},	/* actual: 1399.7 MHz */
-	{ 16800000, 1400000000, 1000, 12, 0, 8},
-	{ 19200000, 1400000000, 875,  12, 0, 8},
-	{ 26000000, 1400000000, 700,  13, 0, 8},
-
+	{ 12000000, 1400000000,  700,  6, 1, 8 },
+	{ 13000000, 1400000000,  969,  9, 1, 8 }, /* actual: 1399.7 MHz */
+	{ 16800000, 1400000000, 1000, 12, 1, 8 },
+	{ 19200000, 1400000000,  875, 12, 1, 8 },
+	{ 26000000, 1400000000,  700, 13, 1, 8 },
 	/* 1.3 GHz */
-	{ 12000000, 1300000000, 975,  9,  0, 8},
-	{ 13000000, 1300000000, 1000, 10, 0, 8},
-	{ 16800000, 1300000000, 928,  12, 0, 8},	/* actual: 1299.2 MHz */
-	{ 19200000, 1300000000, 812,  12, 0, 8},	/* actual: 1299.2 MHz */
-	{ 26000000, 1300000000, 650,  13, 0, 8},
-
+	{ 12000000, 1300000000,  975,  9, 1, 8 },
+	{ 13000000, 1300000000, 1000, 10, 1, 8 },
+	{ 16800000, 1300000000,  928, 12, 1, 8 }, /* actual: 1299.2 MHz */
+	{ 19200000, 1300000000,  812, 12, 1, 8 }, /* actual: 1299.2 MHz */
+	{ 26000000, 1300000000,  650, 13, 1, 8 },
 	/* 1.2 GHz */
-	{ 12000000, 1200000000, 1000, 10, 0, 8},
-	{ 13000000, 1200000000, 923,  10, 0, 8},	/* actual: 1199.9 MHz */
-	{ 16800000, 1200000000, 1000, 14, 0, 8},
-	{ 19200000, 1200000000, 1000, 16, 0, 8},
-	{ 26000000, 1200000000, 600,  13, 0, 8},
-
+	{ 12000000, 1200000000, 1000, 10, 1, 8 },
+	{ 13000000, 1200000000,  923, 10, 1, 8 }, /* actual: 1199.9 MHz */
+	{ 16800000, 1200000000, 1000, 14, 1, 8 },
+	{ 19200000, 1200000000, 1000, 16, 1, 8 },
+	{ 26000000, 1200000000,  600, 13, 1, 8 },
 	/* 1.1 GHz */
-	{ 12000000, 1100000000, 825,  9,  0, 8},
-	{ 13000000, 1100000000, 846,  10, 0, 8},	/* actual: 1099.8 MHz */
-	{ 16800000, 1100000000, 982,  15, 0, 8},	/* actual: 1099.8 MHz */
-	{ 19200000, 1100000000, 859,  15, 0, 8},	/* actual: 1099.5 MHz */
-	{ 26000000, 1100000000, 550,  13, 0, 8},
-
+	{ 12000000, 1100000000, 825,   9, 1, 8 },
+	{ 13000000, 1100000000, 846,  10, 1, 8 }, /* actual: 1099.8 MHz */
+	{ 16800000, 1100000000, 982,  15, 1, 8 }, /* actual: 1099.8 MHz */
+	{ 19200000, 1100000000, 859,  15, 1, 8 }, /* actual: 1099.5 MHz */
+	{ 26000000, 1100000000, 550,  13, 1, 8 },
 	/* 1 GHz */
-	{ 12000000, 1000000000, 1000, 12, 0, 8},
-	{ 13000000, 1000000000, 1000, 13, 0, 8},
-	{ 16800000, 1000000000, 833,  14, 0, 8},	/* actual: 999.6 MHz */
-	{ 19200000, 1000000000, 625,  12, 0, 8},
-	{ 26000000, 1000000000, 1000, 26, 0, 8},
+	{ 12000000, 1000000000, 1000, 12, 1, 8 },
+	{ 13000000, 1000000000, 1000, 13, 1, 8 },
+	{ 16800000, 1000000000,  833, 14, 1, 8 }, /* actual: 999.6 MHz */
+	{ 19200000, 1000000000,  625, 12, 1, 8 },
+	{ 26000000, 1000000000, 1000, 26, 1, 8 },
+	{        0,          0,    0,  0, 0, 0 },
+};
 
-	{ 0, 0, 0, 0, 0, 0 },
+static const struct pdiv_map plle_p[] = {
+	{ .pdiv = 18, .hw_val = 18 },
+	{ .pdiv = 24, .hw_val = 24 },
+	{ .pdiv =  0, .hw_val =  0 },
 };
 
 static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
 	/* PLLE special case: use cpcon field to store cml divider value */
-	{ 12000000,  100000000, 150, 1,  18, 11},
-	{ 216000000, 100000000, 200, 18, 24, 13},
-	{ 0, 0, 0, 0, 0, 0 },
+	{  12000000, 100000000, 150,  1, 18, 11 },
+	{ 216000000, 100000000, 200, 18, 24, 13 },
+	{         0,         0,   0,  0,  0,  0 },
 };
 
 /* PLL parameters */
@@ -422,7 +426,8 @@ static struct tegra_clk_pll_params pll_c_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_c_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct div_nmp pllm_nmp = {
@@ -454,7 +459,8 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.pmc_divp_reg = PMC_PLLM_WB0_OVERRIDE,
 	.freq_table = pll_m_freq_table,
 	.flags = TEGRA_PLLM | TEGRA_PLL_HAS_CPCON |
-		 TEGRA_PLL_SET_DCCON | TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_SET_DCCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_FIXED,
 };
 
 static struct tegra_clk_pll_params pll_p_params = {
@@ -470,7 +476,8 @@ static struct tegra_clk_pll_params pll_p_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_p_freq_table,
-	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_FIXED | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 	.fixed_rate = 408000000,
 };
 
@@ -487,7 +494,8 @@ static struct tegra_clk_pll_params pll_a_params = {
 	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.freq_table = pll_a_freq_table,
-	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK,
+	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_USE_LOCK |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_d_params = {
@@ -504,8 +512,7 @@ static struct tegra_clk_pll_params pll_d_params = {
 	.lock_delay = 1000,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
-
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_d2_params = {
@@ -522,7 +529,7 @@ static struct tegra_clk_pll_params pll_d2_params = {
 	.lock_delay = 1000,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_u_params = {
@@ -539,7 +546,8 @@ static struct tegra_clk_pll_params pll_u_params = {
 	.lock_delay = 1000,
 	.pdiv_tohw = pllu_p,
 	.freq_table = pll_u_freq_table,
-	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON,
+	.flags = TEGRA_PLLU | TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_x_params = {
@@ -556,7 +564,7 @@ static struct tegra_clk_pll_params pll_x_params = {
 	.lock_delay = 300,
 	.freq_table = pll_x_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_DCCON |
-		 TEGRA_PLL_USE_LOCK,
+		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_e_params = {
@@ -571,19 +579,21 @@ static struct tegra_clk_pll_params pll_e_params = {
 	.lock_mask = PLLE_MISC_LOCK,
 	.lock_enable_bit_idx = PLLE_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
+	.pdiv_tohw = plle_p,
 	.freq_table = pll_e_freq_table,
-	.flags = TEGRA_PLLE_CONFIGURE | TEGRA_PLL_FIXED,
+	.flags = TEGRA_PLLE_CONFIGURE | TEGRA_PLL_FIXED |
+		 TEGRA_PLL_HAS_LOCK_ENABLE | TEGRA_PLL_LOCK_MISC,
 	.fixed_rate = 100000000,
 };
 
 static unsigned long tegra30_input_freq[] = {
-	[0] = 13000000,
-	[1] = 16800000,
-	[4] = 19200000,
-	[5] = 38400000,
-	[8] = 12000000,
-	[9] = 48000000,
-	[12] = 260000000,
+	[ 0] = 13000000,
+	[ 1] = 16800000,
+	[ 4] = 19200000,
+	[ 5] = 38400000,
+	[ 8] = 12000000,
+	[ 9] = 48000000,
+	[12] = 26000000,
 };
 
 static struct tegra_devclk devclks[] __initdata = {
@@ -861,13 +871,12 @@ static struct tegra_clk tegra30_clks[tegra_clk_max] __initdata = {
 	[tegra_clk_pll_p_out4] = { .dt_id = TEGRA30_CLK_PLL_P_OUT4, .present = true },
 	[tegra_clk_pll_a] = { .dt_id = TEGRA30_CLK_PLL_A, .present = true },
 	[tegra_clk_pll_a_out0] = { .dt_id = TEGRA30_CLK_PLL_A_OUT0, .present = true },
-
 };
 
 static void tegra30_utmi_param_configure(void)
 {
+	unsigned int i;
 	u32 reg;
-	int i;
 
 	for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) {
 		if (input_freq == utmi_parameters[i].osc_frequency)
@@ -917,7 +926,7 @@ static void tegra30_utmi_param_configure(void)
 	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1);
 }
 
-static const char *pll_e_parents[] = {"pll_ref", "pll_p"};
+static const char *pll_e_parents[] = { "pll_ref", "pll_p" };
 
 static void __init tegra30_pll_init(void)
 {
@@ -925,7 +934,7 @@ static void __init tegra30_pll_init(void)
 
 	/* PLLC */
 	clk = tegra_clk_register_pll("pll_c", "pll_ref", clk_base, pmc_base, 0,
-				&pll_c_params, NULL);
+				     &pll_c_params, NULL);
 	clks[TEGRA30_CLK_PLL_C] = clk;
 
 	/* PLLC_OUT1 */
@@ -1135,7 +1144,7 @@ static void __init tegra30_periph_clk_init(void)
 {
 	struct tegra_periph_init_data *data;
 	struct clk *clk;
-	int i;
+	unsigned int i;
 
 	/* dsia */
 	clk = tegra_clk_register_periph_gate("dsia", "pll_d_out0", 0, clk_base,
@@ -1224,7 +1233,6 @@ static void tegra30_cpu_out_of_reset(u32 cpu)
 	wmb();
 }
 
-
 static void tegra30_enable_cpu_clock(u32 cpu)
 {
 	unsigned int reg;
@@ -1237,7 +1245,6 @@ static void tegra30_enable_cpu_clock(u32 cpu)
 
 static void tegra30_disable_cpu_clock(u32 cpu)
 {
-
 	unsigned int reg;
 
 	reg = readl(clk_base + TEGRA_CLK_RST_CONTROLLER_CLK_CPU_CMPLX);
@@ -1268,7 +1275,7 @@ static void tegra30_cpu_clock_suspend(void)
 	/* switch coresite to clk_m, save off original source */
 	tegra30_cpu_clk_sctx.clk_csite_src =
 				readl(clk_base + CLK_RESET_SOURCE_CSITE);
-	writel(3<<30, clk_base + CLK_RESET_SOURCE_CSITE);
+	writel(3 << 30, clk_base + CLK_RESET_SOURCE_CSITE);
 
 	tegra30_cpu_clk_sctx.cpu_burst =
 				readl(clk_base + CLK_RESET_CCLK_BURST);
@@ -1335,44 +1342,45 @@ static struct tegra_cpu_car_ops tegra30_cpu_car_ops = {
 };
 
 static struct tegra_clk_init_table init_table[] __initdata = {
-	{TEGRA30_CLK_UARTA, TEGRA30_CLK_PLL_P, 408000000, 0},
-	{TEGRA30_CLK_UARTB, TEGRA30_CLK_PLL_P, 408000000, 0},
-	{TEGRA30_CLK_UARTC, TEGRA30_CLK_PLL_P, 408000000, 0},
-	{TEGRA30_CLK_UARTD, TEGRA30_CLK_PLL_P, 408000000, 0},
-	{TEGRA30_CLK_UARTE, TEGRA30_CLK_PLL_P, 408000000, 0},
-	{TEGRA30_CLK_PLL_A, TEGRA30_CLK_CLK_MAX, 564480000, 1},
-	{TEGRA30_CLK_PLL_A_OUT0, TEGRA30_CLK_CLK_MAX, 11289600, 1},
-	{TEGRA30_CLK_EXTERN1, TEGRA30_CLK_PLL_A_OUT0, 0, 1},
-	{TEGRA30_CLK_CLK_OUT_1_MUX, TEGRA30_CLK_EXTERN1, 0, 0},
-	{TEGRA30_CLK_CLK_OUT_1, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_BLINK, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_I2S0, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA30_CLK_I2S1, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA30_CLK_I2S2, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA30_CLK_I2S3, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA30_CLK_I2S4, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0},
-	{TEGRA30_CLK_SDMMC1, TEGRA30_CLK_PLL_P, 48000000, 0},
-	{TEGRA30_CLK_SDMMC2, TEGRA30_CLK_PLL_P, 48000000, 0},
-	{TEGRA30_CLK_SDMMC3, TEGRA30_CLK_PLL_P, 48000000, 0},
-	{TEGRA30_CLK_PLL_M, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_PCLK, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_CSITE, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_EMC, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_MSELECT, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_SBC1, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_SBC2, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_SBC3, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_SBC4, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_SBC5, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_SBC6, TEGRA30_CLK_PLL_P, 100000000, 0},
-	{TEGRA30_CLK_HOST1X, TEGRA30_CLK_PLL_C, 150000000, 0},
-	{TEGRA30_CLK_DISP1, TEGRA30_CLK_PLL_P, 600000000, 0},
-	{TEGRA30_CLK_DISP2, TEGRA30_CLK_PLL_P, 600000000, 0},
-	{TEGRA30_CLK_TWD, TEGRA30_CLK_CLK_MAX, 0, 1},
-	{TEGRA30_CLK_GR2D, TEGRA30_CLK_PLL_C, 300000000, 0},
-	{TEGRA30_CLK_GR3D, TEGRA30_CLK_PLL_C, 300000000, 0},
-	{TEGRA30_CLK_GR3D2, TEGRA30_CLK_PLL_C, 300000000, 0},
-	{TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0}, /* This MUST be the last entry. */
+	{ TEGRA30_CLK_UARTA, TEGRA30_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA30_CLK_UARTB, TEGRA30_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA30_CLK_UARTC, TEGRA30_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA30_CLK_UARTD, TEGRA30_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA30_CLK_UARTE, TEGRA30_CLK_PLL_P, 408000000, 0 },
+	{ TEGRA30_CLK_PLL_A, TEGRA30_CLK_CLK_MAX, 564480000, 1 },
+	{ TEGRA30_CLK_PLL_A_OUT0, TEGRA30_CLK_CLK_MAX, 11289600, 1 },
+	{ TEGRA30_CLK_EXTERN1, TEGRA30_CLK_PLL_A_OUT0, 0, 1 },
+	{ TEGRA30_CLK_CLK_OUT_1_MUX, TEGRA30_CLK_EXTERN1, 0, 0 },
+	{ TEGRA30_CLK_CLK_OUT_1, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_BLINK, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_I2S0, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA30_CLK_I2S1, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA30_CLK_I2S2, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA30_CLK_I2S3, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA30_CLK_I2S4, TEGRA30_CLK_PLL_A_OUT0, 11289600, 0 },
+	{ TEGRA30_CLK_SDMMC1, TEGRA30_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA30_CLK_SDMMC2, TEGRA30_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA30_CLK_SDMMC3, TEGRA30_CLK_PLL_P, 48000000, 0 },
+	{ TEGRA30_CLK_PLL_M, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_PCLK, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_CSITE, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_EMC, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_MSELECT, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_SBC1, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_SBC2, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_SBC3, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_SBC4, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_SBC5, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_SBC6, TEGRA30_CLK_PLL_P, 100000000, 0 },
+	{ TEGRA30_CLK_HOST1X, TEGRA30_CLK_PLL_C, 150000000, 0 },
+	{ TEGRA30_CLK_DISP1, TEGRA30_CLK_PLL_P, 600000000, 0 },
+	{ TEGRA30_CLK_DISP2, TEGRA30_CLK_PLL_P, 600000000, 0 },
+	{ TEGRA30_CLK_TWD, TEGRA30_CLK_CLK_MAX, 0, 1 },
+	{ TEGRA30_CLK_GR2D, TEGRA30_CLK_PLL_C, 300000000, 0 },
+	{ TEGRA30_CLK_GR3D, TEGRA30_CLK_PLL_C, 300000000, 0 },
+	{ TEGRA30_CLK_GR3D2, TEGRA30_CLK_PLL_C, 300000000, 0 },
+	/* must be the last entry */
+	{ TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0 },
 };
 
 static void __init tegra30_clock_apply_init_table(void)
@@ -1397,12 +1405,13 @@ static struct tegra_clk_duplicate tegra_clk_duplicates[] = {
 	TEGRA_CLK_DUPLICATE(TEGRA30_CLK_CML1, "tegra_sata_cml", NULL),
 	TEGRA_CLK_DUPLICATE(TEGRA30_CLK_CML0, "tegra_pcie", "cml"),
 	TEGRA_CLK_DUPLICATE(TEGRA30_CLK_VCP, "nvavp", "vcp"),
-	TEGRA_CLK_DUPLICATE(TEGRA30_CLK_CLK_MAX, NULL, NULL), /* MUST be the last entry */
+	/* must be the last entry */
+	TEGRA_CLK_DUPLICATE(TEGRA30_CLK_CLK_MAX, NULL, NULL),
 };
 
 static const struct of_device_id pmc_match[] __initconst = {
 	{ .compatible = "nvidia,tegra30-pmc" },
-	{},
+	{ },
 };
 
 static struct tegra_audio_clk_info tegra30_audio_plls[] = {
@@ -1441,7 +1450,6 @@ static void __init tegra30_clock_init(struct device_node *np)
 			       NULL) < 0)
 		return;
 
-
 	tegra_fixed_clk_init(tegra30_clks);
 	tegra30_pll_init();
 	tegra30_super_clk_init();
diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h
index 5d2678914160..4dbcfaec576a 100644
--- a/drivers/clk/tegra/clk.h
+++ b/drivers/clk/tegra/clk.h
@@ -110,14 +110,16 @@ struct clk *tegra_clk_register_mc(const char *name, const char *parent_name,
  * @m:			input divider
  * @p:			post divider
  * @cpcon:		charge pump current
+ * @sdm_data:		fraction divider setting (0 = disabled)
  */
 struct tegra_clk_pll_freq_table {
 	unsigned long	input_rate;
 	unsigned long	output_rate;
-	u16		n;
+	u32		n;
 	u16		m;
 	u8		p;
 	u8		cpcon;
+	u16		sdm_data;
 };
 
 /**
@@ -156,6 +158,10 @@ struct div_nmp {
 	u8		override_divp_shift;
 };
 
+#define MAX_PLL_MISC_REG_COUNT	6
+
+struct tegra_clk_pll;
+
 /**
  * struct tegra_clk_pll_params - PLL parameters
  *
@@ -172,6 +178,14 @@ struct div_nmp {
  * @lock_enable_bit_idx:	Bit index to enable PLL lock
  * @iddq_reg:			PLL IDDQ register offset
  * @iddq_bit_idx:		Bit index to enable PLL IDDQ
+ * @reset_reg:			Register offset of where RESET bit is
+ * @reset_bit_idx:		Shift of reset bit in reset_reg
+ * @sdm_din_reg:		Register offset where SDM settings are
+ * @sdm_din_mask:		Mask of SDM divider bits
+ * @sdm_ctrl_reg:		Register offset where SDM enable is
+ * @sdm_ctrl_en_mask:		Mask of SDM enable bit
+ * @ssc_ctrl_reg:		Register offset where SSC settings are
+ * @ssc_ctrl_en_mask:		Mask of SSC enable bit
  * @aux_reg:			AUX register offset
  * @dyn_ramp_reg:		Dynamic ramp control register offset
  * @ext_misc_reg:		Miscellaneous control register offsets
@@ -182,10 +196,27 @@ struct div_nmp {
  * @stepb_shift:		Dynamic ramp step B field shift
  * @lock_delay:			Delay in us if PLL lock is not used
  * @max_p:			maximum value for the p divider
+ * @defaults_set:		Boolean signaling all reg defaults for PLL set.
  * @pdiv_tohw:			mapping of p divider to register values
  * @div_nmp:			offsets and widths on n, m and p fields
  * @freq_table:			array of frequencies supported by PLL
  * @fixed_rate:			PLL rate if it is fixed
+ * @mdiv_default:		Default value for fixed mdiv for this PLL
+ * @round_p_to_pdiv:		Callback used to round p to the closed pdiv
+ * @set_gain:			Callback to adjust N div for SDM enabled
+ *				PLL's based on fractional divider value.
+ * @calc_rate:			Callback used to change how out of table
+ *				rates (dividers and multipler) are calculated.
+ * @adjust_vco:			Callback to adjust the programming range of the
+ *				divider range (if SDM is present)
+ * @set_defaults:		Callback which will try to initialize PLL
+ *				registers to sane default values. This is first
+ *				tried during PLL registration, but if the PLL
+ *				is already enabled, it will be done the first
+ *				time the rate is changed while the PLL is
+ *				disabled.
+ * @dyn_ramp:			Callback which can be used to define a custom
+ *				dynamic ramp function for a given PLL.
  *
  * Flags:
  * TEGRA_PLL_USE_LOCK - This flag indicated to use lock bits for
@@ -207,6 +238,11 @@ struct div_nmp {
  *     base register.
  * TEGRA_PLL_BYPASS - PLL has bypass bit
  * TEGRA_PLL_HAS_LOCK_ENABLE - PLL has bit to enable lock monitoring
+ * TEGRA_MDIV_NEW - Switch to new method for calculating fixed mdiv
+ *     it may be more accurate (especially if SDM present)
+ * TEGRA_PLLMB - PLLMB has should be treated similar to PLLM. This
+ *     flag indicated that it is PLLMB.
+ * TEGRA_PLL_VCO_OUT - Used to indicate that the PLL has a VCO output
  */
 struct tegra_clk_pll_params {
 	unsigned long	input_min;
@@ -223,9 +259,17 @@ struct tegra_clk_pll_params {
 	u32		lock_enable_bit_idx;
 	u32		iddq_reg;
 	u32		iddq_bit_idx;
+	u32		reset_reg;
+	u32		reset_bit_idx;
+	u32		sdm_din_reg;
+	u32		sdm_din_mask;
+	u32		sdm_ctrl_reg;
+	u32		sdm_ctrl_en_mask;
+	u32		ssc_ctrl_reg;
+	u32		ssc_ctrl_en_mask;
 	u32		aux_reg;
 	u32		dyn_ramp_reg;
-	u32		ext_misc_reg[3];
+	u32		ext_misc_reg[MAX_PLL_MISC_REG_COUNT];
 	u32		pmc_divnm_reg;
 	u32		pmc_divp_reg;
 	u32		flags;
@@ -233,10 +277,22 @@ struct tegra_clk_pll_params {
 	int		stepb_shift;
 	int		lock_delay;
 	int		max_p;
-	struct pdiv_map *pdiv_tohw;
+	bool		defaults_set;
+	const struct pdiv_map *pdiv_tohw;
 	struct div_nmp	*div_nmp;
 	struct tegra_clk_pll_freq_table	*freq_table;
 	unsigned long	fixed_rate;
+	u16		mdiv_default;
+	u32	(*round_p_to_pdiv)(u32 p, u32 *pdiv);
+	void	(*set_gain)(struct tegra_clk_pll_freq_table *cfg);
+	int	(*calc_rate)(struct clk_hw *hw,
+			struct tegra_clk_pll_freq_table *cfg,
+			unsigned long rate, unsigned long parent_rate);
+	unsigned long	(*adjust_vco)(struct tegra_clk_pll_params *pll_params,
+				unsigned long parent_rate);
+	void	(*set_defaults)(struct tegra_clk_pll *pll);
+	int	(*dyn_ramp)(struct tegra_clk_pll *pll,
+			struct tegra_clk_pll_freq_table *cfg);
 };
 
 #define TEGRA_PLL_USE_LOCK BIT(0)
@@ -250,6 +306,9 @@ struct tegra_clk_pll_params {
 #define TEGRA_PLL_LOCK_MISC BIT(8)
 #define TEGRA_PLL_BYPASS BIT(9)
 #define TEGRA_PLL_HAS_LOCK_ENABLE BIT(10)
+#define TEGRA_MDIV_NEW BIT(11)
+#define TEGRA_PLLMB BIT(12)
+#define TEGRA_PLL_VCO_OUT BIT(13)
 
 /**
  * struct tegra_clk_pll - Tegra PLL clock
@@ -303,6 +362,12 @@ struct clk *tegra_clk_register_pllxc(const char *name, const char *parent_name,
 			    struct tegra_clk_pll_params *pll_params,
 			    spinlock_t *lock);
 
+struct clk *tegra_clk_register_pllxc_tegra210(const char *name,
+			const char *parent_name, void __iomem *clk_base,
+			void __iomem *pmc, unsigned long flags,
+			struct tegra_clk_pll_params *pll_params,
+			spinlock_t *lock);
+
 struct clk *tegra_clk_register_pllm(const char *name, const char *parent_name,
 			   void __iomem *clk_base, void __iomem *pmc,
 			   unsigned long flags,
@@ -327,11 +392,35 @@ struct clk *tegra_clk_register_plle_tegra114(const char *name,
 				struct tegra_clk_pll_params *pll_params,
 				spinlock_t *lock);
 
+struct clk *tegra_clk_register_plle_tegra210(const char *name,
+				const char *parent_name,
+				void __iomem *clk_base, unsigned long flags,
+				struct tegra_clk_pll_params *pll_params,
+				spinlock_t *lock);
+
+struct clk *tegra_clk_register_pllc_tegra210(const char *name,
+				const char *parent_name, void __iomem *clk_base,
+				void __iomem *pmc, unsigned long flags,
+				struct tegra_clk_pll_params *pll_params,
+				spinlock_t *lock);
+
+struct clk *tegra_clk_register_pllss_tegra210(const char *name,
+				const char *parent_name, void __iomem *clk_base,
+				unsigned long flags,
+				struct tegra_clk_pll_params *pll_params,
+				spinlock_t *lock);
+
 struct clk *tegra_clk_register_pllss(const char *name, const char *parent_name,
 			   void __iomem *clk_base, unsigned long flags,
 			   struct tegra_clk_pll_params *pll_params,
 			   spinlock_t *lock);
 
+struct clk *tegra_clk_register_pllmb(const char *name, const char *parent_name,
+			   void __iomem *clk_base, void __iomem *pmc,
+			   unsigned long flags,
+			   struct tegra_clk_pll_params *pll_params,
+			   spinlock_t *lock);
+
 /**
  * struct tegra_clk_pll_out - PLL divider down clock
  *
@@ -653,6 +742,9 @@ int tegra_osc_clk_init(void __iomem *clk_base, struct tegra_clk *clks,
 void tegra_super_clk_gen4_init(void __iomem *clk_base,
 			void __iomem *pmc_base, struct tegra_clk *tegra_clks,
 			struct tegra_clk_pll_params *pll_params);
+void tegra_super_clk_gen5_init(void __iomem *clk_base,
+			void __iomem *pmc_base, struct tegra_clk *tegra_clks,
+			struct tegra_clk_pll_params *pll_params);
 
 #ifdef CONFIG_TEGRA_CLK_EMC
 struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np,
@@ -674,5 +766,8 @@ void tegra114_clock_deassert_dfll_dvco_reset(void);
 
 typedef void (*tegra_clk_apply_init_table_func)(void);
 extern tegra_clk_apply_init_table_func tegra_clk_apply_init_table;
+int tegra_pll_wait_for_lock(struct tegra_clk_pll *pll);
+u16 tegra_pll_get_fixed_mdiv(struct clk_hw *hw, unsigned long input_rate);
+int tegra_pll_p_div_to_hw(struct tegra_clk_pll *pll, u8 p_div);
 
 #endif /* TEGRA_CLK_H */
diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c
index f3eab6e79027..b336a8c11e2a 100644
--- a/drivers/clk/ti/apll.c
+++ b/drivers/clk/ti/apll.c
@@ -323,7 +323,7 @@ static void omap2_apll_deny_idle(struct clk_hw_omap *clk)
 	omap2_apll_set_autoidle(clk, OMAP2_APLL_AUTOIDLE_DISABLE);
 }
 
-static struct clk_hw_omap_ops omap2_apll_hwops = {
+static const struct clk_hw_omap_ops omap2_apll_hwops = {
 	.allow_idle	= &omap2_apll_allow_idle,
 	.deny_idle	= &omap2_apll_deny_idle,
 };
diff --git a/drivers/clk/ti/clk-814x.c b/drivers/clk/ti/clk-814x.c
index e172920798ea..9e85fcc74cc9 100644
--- a/drivers/clk/ti/clk-814x.c
+++ b/drivers/clk/ti/clk-814x.c
@@ -14,10 +14,14 @@ static struct ti_dt_clk dm814_clks[] = {
 	DT_CLK(NULL, "devosc_ck", "devosc_ck"),
 	DT_CLK(NULL, "mpu_ck", "mpu_ck"),
 	DT_CLK(NULL, "sysclk4_ck", "sysclk4_ck"),
+	DT_CLK(NULL, "sysclk5_ck", "sysclk5_ck"),
 	DT_CLK(NULL, "sysclk6_ck", "sysclk6_ck"),
+	DT_CLK(NULL, "sysclk8_ck", "sysclk8_ck"),
 	DT_CLK(NULL, "sysclk10_ck", "sysclk10_ck"),
 	DT_CLK(NULL, "sysclk18_ck", "sysclk18_ck"),
 	DT_CLK(NULL, "timer_sys_ck", "devosc_ck"),
+	DT_CLK(NULL, "timer1_fck", "timer1_fck"),
+	DT_CLK(NULL, "timer2_fck", "timer2_fck"),
 	DT_CLK(NULL, "cpsw_125mhz_gclk", "cpsw_125mhz_gclk"),
 	DT_CLK(NULL, "cpsw_cpts_rft_clk", "cpsw_cpts_rft_clk"),
 	{ .node_name = NULL },
diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c
index f4dec00fb684..1c300388782b 100644
--- a/drivers/clk/ti/dpll3xxx.c
+++ b/drivers/clk/ti/dpll3xxx.c
@@ -305,8 +305,9 @@ static void _lookup_sddiv(struct clk_hw_omap *clk, u8 *sd_div, u16 m, u8 n)
 static int omap3_noncore_dpll_program(struct clk_hw_omap *clk, u16 freqsel)
 {
 	struct dpll_data *dd = clk->dpll_data;
-	u8 dco, sd_div;
+	u8 dco, sd_div, ai = 0;
 	u32 v;
+	bool errata_i810;
 
 	/* 3430 ES2 TRM: 4.7.6.9 DPLL Programming Sequence */
 	_omap3_noncore_dpll_bypass(clk);
@@ -350,6 +351,25 @@ static int omap3_noncore_dpll_program(struct clk_hw_omap *clk, u16 freqsel)
 		v |= sd_div << __ffs(dd->sddiv_mask);
 	}
 
+	/*
+	 * Errata i810 - DPLL controller can get stuck while transitioning
+	 * to a power saving state. Software must ensure the DPLL can not
+	 * transition to a low power state while changing M/N values.
+	 * Easiest way to accomplish this is to prevent DPLL autoidle
+	 * before doing the M/N re-program.
+	 */
+	errata_i810 = ti_clk_get_features()->flags & TI_CLK_ERRATA_I810;
+
+	if (errata_i810) {
+		ai = omap3_dpll_autoidle_read(clk);
+		if (ai) {
+			omap3_dpll_deny_idle(clk);
+
+			/* OCP barrier */
+			omap3_dpll_autoidle_read(clk);
+		}
+	}
+
 	ti_clk_ll_ops->clk_writel(v, dd->mult_div1_reg);
 
 	/* Set 4X multiplier and low-power mode */
@@ -379,6 +399,9 @@ static int omap3_noncore_dpll_program(struct clk_hw_omap *clk, u16 freqsel)
 
 	_omap3_noncore_dpll_lock(clk);
 
+	if (errata_i810 && ai)
+		omap3_dpll_allow_idle(clk);
+
 	return 0;
 }
 
diff --git a/drivers/clk/versatile/Kconfig b/drivers/clk/versatile/Kconfig
index fc50b6264bed..a6da2aa09f83 100644
--- a/drivers/clk/versatile/Kconfig
+++ b/drivers/clk/versatile/Kconfig
@@ -1,6 +1,9 @@
 config COMMON_CLK_VERSATILE
 	bool "Clock driver for ARM Reference designs"
-	depends on ARCH_INTEGRATOR || ARCH_REALVIEW || ARCH_VEXPRESS || ARM64 || COMPILE_TEST
+	depends on ARCH_INTEGRATOR || ARCH_REALVIEW || \
+		ARCH_VERSATILE || ARCH_VEXPRESS || ARM64 || \
+		COMPILE_TEST
+	select REGMAP_MMIO
 	---help---
           Supports clocking on ARM Reference designs:
 	  - Integrator/AP and Integrator/CP
diff --git a/drivers/clk/versatile/clk-icst.c b/drivers/clk/versatile/clk-icst.c
index 08c5ee976879..e62f8cb2c9b5 100644
--- a/drivers/clk/versatile/clk-icst.c
+++ b/drivers/clk/versatile/clk-icst.c
@@ -3,7 +3,7 @@
  * We wrap the custom interface from <asm/hardware/icst.h> into the generic
  * clock framework.
  *
- * Copyright (C) 2012 Linus Walleij
+ * Copyright (C) 2012-2015 Linus Walleij
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -19,9 +19,14 @@
 #include <linux/err.h>
 #include <linux/clk-provider.h>
 #include <linux/io.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
 
 #include "clk-icst.h"
 
+/* Magic unlocking token used on all Versatile boards */
+#define VERSATILE_LOCK_VAL	0xA05F
+
 /**
  * struct clk_icst - ICST VCO clock wrapper
  * @hw: corresponding clock hardware entry
@@ -32,8 +37,9 @@
  */
 struct clk_icst {
 	struct clk_hw hw;
-	void __iomem *vcoreg;
-	void __iomem *lockreg;
+	struct regmap *map;
+	u32 vcoreg_off;
+	u32 lockreg_off;
 	struct icst_params *params;
 	unsigned long rate;
 };
@@ -41,53 +47,67 @@ struct clk_icst {
 #define to_icst(_hw) container_of(_hw, struct clk_icst, hw)
 
 /**
- * vco_get() - get ICST VCO settings from a certain register
- * @vcoreg: register containing the VCO settings
+ * vco_get() - get ICST VCO settings from a certain ICST
+ * @icst: the ICST clock to get
+ * @vco: the VCO struct to return the value in
  */
-static struct icst_vco vco_get(void __iomem *vcoreg)
+static int vco_get(struct clk_icst *icst, struct icst_vco *vco)
 {
 	u32 val;
-	struct icst_vco vco;
+	int ret;
 
-	val = readl(vcoreg);
-	vco.v = val & 0x1ff;
-	vco.r = (val >> 9) & 0x7f;
-	vco.s = (val >> 16) & 03;
-	return vco;
+	ret = regmap_read(icst->map, icst->vcoreg_off, &val);
+	if (ret)
+		return ret;
+	vco->v = val & 0x1ff;
+	vco->r = (val >> 9) & 0x7f;
+	vco->s = (val >> 16) & 03;
+	return 0;
 }
 
 /**
  * vco_set() - commit changes to an ICST VCO
- * @locreg: register to poke to unlock the VCO for writing
- * @vcoreg: register containing the VCO settings
- * @vco: ICST VCO parameters to commit
+ * @icst: the ICST clock to set
+ * @vco: the VCO struct to set the changes from
  */
-static void vco_set(void __iomem *lockreg,
-			void __iomem *vcoreg,
-			struct icst_vco vco)
+static int vco_set(struct clk_icst *icst, struct icst_vco vco)
 {
 	u32 val;
+	int ret;
 
-	val = readl(vcoreg) & ~0x7ffff;
+	ret = regmap_read(icst->map, icst->vcoreg_off, &val);
+	if (ret)
+		return ret;
 	val |= vco.v | (vco.r << 9) | (vco.s << 16);
 
 	/* This magic unlocks the VCO so it can be controlled */
-	writel(0xa05f, lockreg);
-	writel(val, vcoreg);
+	ret = regmap_write(icst->map, icst->lockreg_off, VERSATILE_LOCK_VAL);
+	if (ret)
+		return ret;
+	ret = regmap_write(icst->map, icst->vcoreg_off, val);
+	if (ret)
+		return ret;
 	/* This locks the VCO again */
-	writel(0, lockreg);
+	ret = regmap_write(icst->map, icst->lockreg_off, 0);
+	if (ret)
+		return ret;
+	return 0;
 }
 
-
 static unsigned long icst_recalc_rate(struct clk_hw *hw,
 				      unsigned long parent_rate)
 {
 	struct clk_icst *icst = to_icst(hw);
 	struct icst_vco vco;
+	int ret;
 
 	if (parent_rate)
 		icst->params->ref = parent_rate;
-	vco = vco_get(icst->vcoreg);
+	ret = vco_get(icst, &vco);
+	if (ret) {
+		pr_err("ICST: could not get VCO setting\n");
+		return 0;
+	}
 	icst->rate = icst_hz(icst->params, vco);
 	return icst->rate;
 }
@@ -112,8 +132,7 @@ static int icst_set_rate(struct clk_hw *hw, unsigned long rate,
 		icst->params->ref = parent_rate;
 	vco = icst_hz_to_vco(icst->params, rate);
 	icst->rate = icst_hz(icst->params, vco);
-	vco_set(icst->lockreg, icst->vcoreg, vco);
-	return 0;
+	return vco_set(icst, vco);
 }
 
 static const struct clk_ops icst_ops = {
@@ -122,11 +141,11 @@ static const struct clk_ops icst_ops = {
 	.set_rate = icst_set_rate,
 };
 
-struct clk *icst_clk_register(struct device *dev,
-			const struct clk_icst_desc *desc,
-			const char *name,
-			const char *parent_name,
-			void __iomem *base)
+static struct clk *icst_clk_setup(struct device *dev,
+				  const struct clk_icst_desc *desc,
+				  const char *name,
+				  const char *parent_name,
+				  struct regmap *map)
 {
 	struct clk *clk;
 	struct clk_icst *icst;
@@ -151,10 +170,11 @@ struct clk *icst_clk_register(struct device *dev,
 	init.flags = CLK_IS_ROOT;
 	init.parent_names = (parent_name ? &parent_name : NULL);
 	init.num_parents = (parent_name ? 1 : 0);
+	icst->map = map;
 	icst->hw.init = &init;
 	icst->params = pclone;
-	icst->vcoreg = base + desc->vco_offset;
-	icst->lockreg = base + desc->lock_offset;
+	icst->vcoreg_off = desc->vco_offset;
+	icst->lockreg_off = desc->lock_offset;
 
 	clk = clk_register(dev, &icst->hw);
 	if (IS_ERR(clk)) {
@@ -164,4 +184,112 @@ struct clk *icst_clk_register(struct device *dev,
 
 	return clk;
 }
+
+struct clk *icst_clk_register(struct device *dev,
+			const struct clk_icst_desc *desc,
+			const char *name,
+			const char *parent_name,
+			void __iomem *base)
+{
+	struct regmap_config icst_regmap_conf = {
+		.reg_bits = 32,
+		.val_bits = 32,
+		.reg_stride = 4,
+	};
+	struct regmap *map;
+
+	map = regmap_init_mmio(dev, base, &icst_regmap_conf);
+	if (IS_ERR(map)) {
+		pr_err("could not initialize ICST regmap\n");
+		return ERR_CAST(map);
+	}
+	return icst_clk_setup(dev, desc, name, parent_name, map);
+}
 EXPORT_SYMBOL_GPL(icst_clk_register);
+
+#ifdef CONFIG_OF
+/*
+ * In a device tree, an memory-mapped ICST clock appear as a child
+ * of a syscon node. Assume this and probe it only as a child of a
+ * syscon.
+ */
+
+static const struct icst_params icst525_params = {
+	.vco_max	= ICST525_VCO_MAX_5V,
+	.vco_min	= ICST525_VCO_MIN,
+	.vd_min		= 8,
+	.vd_max		= 263,
+	.rd_min		= 3,
+	.rd_max		= 65,
+	.s2div		= icst525_s2div,
+	.idx2s		= icst525_idx2s,
+};
+
+static const struct icst_params icst307_params = {
+	.vco_max	= ICST307_VCO_MAX,
+	.vco_min	= ICST307_VCO_MIN,
+	.vd_min		= 4 + 8,
+	.vd_max		= 511 + 8,
+	.rd_min		= 1 + 2,
+	.rd_max		= 127 + 2,
+	.s2div		= icst307_s2div,
+	.idx2s		= icst307_idx2s,
+};
+
+static void __init of_syscon_icst_setup(struct device_node *np)
+{
+	struct device_node *parent;
+	struct regmap *map;
+	struct clk_icst_desc icst_desc;
+	const char *name = np->name;
+	const char *parent_name;
+	struct clk *regclk;
+
+	/* We do not release this reference, we are using it perpetually */
+	parent = of_get_parent(np);
+	if (!parent) {
+		pr_err("no parent node for syscon ICST clock\n");
+		return;
+	}
+	map = syscon_node_to_regmap(parent);
+	if (IS_ERR(map)) {
+		pr_err("no regmap for syscon ICST clock parent\n");
+		return;
+	}
+
+	if (of_property_read_u32(np, "vco-offset", &icst_desc.vco_offset)) {
+		pr_err("no VCO register offset for ICST clock\n");
+		return;
+	}
+	if (of_property_read_u32(np, "lock-offset", &icst_desc.lock_offset)) {
+		pr_err("no lock register offset for ICST clock\n");
+		return;
+	}
+
+	if (of_device_is_compatible(np, "arm,syscon-icst525"))
+		icst_desc.params = &icst525_params;
+	else if (of_device_is_compatible(np, "arm,syscon-icst307"))
+		icst_desc.params = &icst307_params;
+	else {
+		pr_err("unknown ICST clock %s\n", name);
+		return;
+	}
+
+	/* Parent clock name is not the same as node parent */
+	parent_name = of_clk_get_parent_name(np, 0);
+
+	regclk = icst_clk_setup(NULL, &icst_desc, name, parent_name, map);
+	if (IS_ERR(regclk)) {
+		pr_err("error setting up syscon ICST clock %s\n", name);
+		return;
+	}
+	of_clk_add_provider(np, of_clk_src_simple_get, regclk);
+	pr_debug("registered syscon ICST clock %s\n", name);
+}
+
+CLK_OF_DECLARE(arm_syscon_icst525_clk,
+	       "arm,syscon-icst525", of_syscon_icst_setup);
+CLK_OF_DECLARE(arm_syscon_icst307_clk,
+	       "arm,syscon-icst307", of_syscon_icst_setup);
+
+#endif
diff --git a/drivers/clk/versatile/clk-realview.c b/drivers/clk/versatile/clk-realview.c
index 86f70997d59d..bd4dd2463e23 100644
--- a/drivers/clk/versatile/clk-realview.c
+++ b/drivers/clk/versatile/clk-realview.c
@@ -11,11 +11,15 @@
 #include <linux/io.h>
 #include <linux/clk-provider.h>
 
-#include <mach/hardware.h>
-#include <mach/platform.h>
-
 #include "clk-icst.h"
 
+#define REALVIEW_SYS_OSC0_OFFSET             0x0C
+#define REALVIEW_SYS_OSC1_OFFSET             0x10
+#define REALVIEW_SYS_OSC2_OFFSET             0x14
+#define REALVIEW_SYS_OSC3_OFFSET             0x18
+#define REALVIEW_SYS_OSC4_OFFSET             0x1C	/* OSC1 for RealView/AB */
+#define REALVIEW_SYS_LOCK_OFFSET             0x20
+
 /*
  * Implementation of the ARM RealView clock trees.
  */
diff --git a/drivers/clk/versatile/clk-sp810.c b/drivers/clk/versatile/clk-sp810.c
index a1cdef6b0f90..e78755e0ef78 100644
--- a/drivers/clk/versatile/clk-sp810.c
+++ b/drivers/clk/versatile/clk-sp810.c
@@ -95,13 +95,12 @@ static void __init clk_sp810_of_setup(struct device_node *node)
 	int i;
 	bool deprecated;
 
-	if (!sp810) {
-		pr_err("Failed to allocate memory for SP810!\n");
+	if (!sp810)
 		return;
-	}
 
 	if (of_clk_parent_fill(node, parent_names, num) != num) {
 		pr_warn("Failed to obtain parent clocks for SP810!\n");
+		kfree(sp810);
 		return;
 	}
 
diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c
index b375106844d8..dfad6eb99662 100644
--- a/drivers/clocksource/clksrc-dbx500-prcmu.c
+++ b/drivers/clocksource/clksrc-dbx500-prcmu.c
@@ -12,8 +12,9 @@
  * power domain.  We use the Timer 4 for our always-on clock
  * source on DB8500.
  */
+#include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/clockchips.h>
-#include <linux/clksrc-dbx500-prcmu.h>
 #include <linux/sched_clock.h>
 
 #define RATE_32K		32768
@@ -63,9 +64,9 @@ static u64 notrace dbx500_prcmu_sched_clock_read(void)
 
 #endif
 
-void __init clksrc_dbx500_prcmu_init(void __iomem *base)
+static void __init clksrc_dbx500_prcmu_init(struct device_node *node)
 {
-	clksrc_dbx500_timer_base = base;
+	clksrc_dbx500_timer_base = of_iomap(node, 0);
 
 	/*
 	 * The A9 sub system expects the timer to be configured as
@@ -85,3 +86,5 @@ void __init clksrc_dbx500_prcmu_init(void __iomem *base)
 #endif
 	clocksource_register_hz(&clocksource_dbx500_prcmu, RATE_32K);
 }
+CLOCKSOURCE_OF_DECLARE(dbx500_prcmu, "stericsson,db8500-prcmu-timer-4",
+		       clksrc_dbx500_prcmu_init);
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 9bc37c437874..0ca74d070058 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -142,15 +142,16 @@ static int allocate_resources(int cpu, struct device **cdev,
 
 try_again:
 	cpu_reg = regulator_get_optional(cpu_dev, reg);
-	if (IS_ERR(cpu_reg)) {
+	ret = PTR_ERR_OR_ZERO(cpu_reg);
+	if (ret) {
 		/*
 		 * If cpu's regulator supply node is present, but regulator is
 		 * not yet registered, we should try defering probe.
 		 */
-		if (PTR_ERR(cpu_reg) == -EPROBE_DEFER) {
+		if (ret == -EPROBE_DEFER) {
 			dev_dbg(cpu_dev, "cpu%d regulator not ready, retry\n",
 				cpu);
-			return -EPROBE_DEFER;
+			return ret;
 		}
 
 		/* Try with "cpu-supply" */
@@ -159,18 +160,16 @@ try_again:
 			goto try_again;
 		}
 
-		dev_dbg(cpu_dev, "no regulator for cpu%d: %ld\n",
-			cpu, PTR_ERR(cpu_reg));
+		dev_dbg(cpu_dev, "no regulator for cpu%d: %d\n", cpu, ret);
 	}
 
 	cpu_clk = clk_get(cpu_dev, NULL);
-	if (IS_ERR(cpu_clk)) {
+	ret = PTR_ERR_OR_ZERO(cpu_clk);
+	if (ret) {
 		/* put regulator */
 		if (!IS_ERR(cpu_reg))
 			regulator_put(cpu_reg);
 
-		ret = PTR_ERR(cpu_clk);
-
 		/*
 		 * If cpu's clk node is present, but clock is not yet
 		 * registered, we should try defering probe.
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index c35e7da1ed7a..e979ec78b695 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -48,11 +48,11 @@ static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
 					  bool active)
 {
 	do {
-		policy = list_next_entry(policy, policy_list);
-
 		/* No more policies in the list */
-		if (&policy->policy_list == &cpufreq_policy_list)
+		if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
 			return NULL;
+
+		policy = list_next_entry(policy, policy_list);
 	} while (!suitable_policy(policy, active));
 
 	return policy;
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index bab3a514ec12..e0d111024d48 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -387,16 +387,18 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy,
 	if (!have_governor_per_policy())
 		cdata->gdbs_data = dbs_data;
 
+	policy->governor_data = dbs_data;
+
 	ret = sysfs_create_group(get_governor_parent_kobj(policy),
 				 get_sysfs_attr(dbs_data));
 	if (ret)
 		goto reset_gdbs_data;
 
-	policy->governor_data = dbs_data;
-
 	return 0;
 
 reset_gdbs_data:
+	policy->governor_data = NULL;
+
 	if (!have_governor_per_policy())
 		cdata->gdbs_data = NULL;
 	cdata->exit(dbs_data, !policy->governor->initialized);
@@ -417,16 +419,19 @@ static int cpufreq_governor_exit(struct cpufreq_policy *policy,
 	if (!cdbs->shared || cdbs->shared->policy)
 		return -EBUSY;
 
-	policy->governor_data = NULL;
 	if (!--dbs_data->usage_count) {
 		sysfs_remove_group(get_governor_parent_kobj(policy),
 				   get_sysfs_attr(dbs_data));
 
+		policy->governor_data = NULL;
+
 		if (!have_governor_per_policy())
 			cdata->gdbs_data = NULL;
 
 		cdata->exit(dbs_data, policy->governor->initialized == 1);
 		kfree(dbs_data);
+	} else {
+		policy->governor_data = NULL;
 	}
 
 	free_common_dbs_info(policy, cdata);
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index cb501386eb6e..547890fd9572 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -586,7 +586,7 @@ static int __init powernv_cpufreq_init(void)
 	int rc = 0;
 
 	/* Don't probe on pseries (guest) platforms */
-	if (!firmware_has_feature(FW_FEATURE_OPALv3))
+	if (!firmware_has_feature(FW_FEATURE_OPAL))
 		return -ENODEV;
 
 	/* Discover pstates from device tree and init */
diff --git a/drivers/cpufreq/pxa2xx-cpufreq.c b/drivers/cpufreq/pxa2xx-cpufreq.c
index 1d99c97defa9..096377232747 100644
--- a/drivers/cpufreq/pxa2xx-cpufreq.c
+++ b/drivers/cpufreq/pxa2xx-cpufreq.c
@@ -202,7 +202,7 @@ static void __init pxa_cpufreq_init_voltages(void)
 	}
 }
 #else
-static int pxa_cpufreq_change_voltage(struct pxa_freqs *pxa_freq)
+static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq)
 {
 	return 0;
 }
diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index 8c7930b5a65f..7e48eb5bf0a7 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -19,11 +19,9 @@ config CPU_IDLE_MULTIPLE_DRIVERS
 
 config CPU_IDLE_GOV_LADDER
 	bool "Ladder governor (for periodic timer tick)"
-	default y
 
 config CPU_IDLE_GOV_MENU
 	bool "Menu governor (for tickless system)"
-	default y
 
 config DT_IDLE_STATES
 	bool
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index 344058f8501a..d5657d50ac40 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -119,7 +119,6 @@ struct cpuidle_coupled {
 
 #define CPUIDLE_COUPLED_NOT_IDLE	(-1)
 
-static DEFINE_MUTEX(cpuidle_coupled_lock);
 static DEFINE_PER_CPU(struct call_single_data, cpuidle_coupled_poke_cb);
 
 /*
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index 845bafcfa792..e12dc30d8864 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -264,7 +264,7 @@ static int powernv_idle_probe(void)
 	if (cpuidle_disable != IDLE_NO_OVERRIDE)
 		return -ENODEV;
 
-	if (firmware_has_feature(FW_FEATURE_OPALv3)) {
+	if (firmware_has_feature(FW_FEATURE_OPAL)) {
 		cpuidle_state_table = powernv_states;
 		/* Device tree can indicate more idle states */
 		max_idle_state = powernv_add_idle_states();
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 17a6dc0e2111..f996efc56605 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -79,9 +79,9 @@ static int find_deepest_state(struct cpuidle_driver *drv,
 			      bool freeze)
 {
 	unsigned int latency_req = 0;
-	int i, ret = -ENXIO;
+	int i, ret = 0;
 
-	for (i = 0; i < drv->state_count; i++) {
+	for (i = 1; i < drv->state_count; i++) {
 		struct cpuidle_state *s = &drv->states[i];
 		struct cpuidle_state_usage *su = &dev->states_usage[i];
 
@@ -153,7 +153,7 @@ int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 	 * be frozen safely.
 	 */
 	index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
-	if (index >= 0)
+	if (index > 0)
 		enter_freeze_proper(drv, dev, index);
 
 	return index;
@@ -243,7 +243,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
  * @drv: the cpuidle driver
  * @dev: the cpuidle device
  *
- * Returns the index of the idle state.
+ * Returns the index of the idle state.  The return value must not be negative.
  */
 int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 {
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index 401c0106ed34..63bd5a403e22 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -17,6 +17,7 @@
 #include <linux/pm_qos.h>
 #include <linux/module.h>
 #include <linux/jiffies.h>
+#include <linux/tick.h>
 
 #include <asm/io.h>
 #include <asm/uaccess.h>
@@ -184,6 +185,14 @@ static struct cpuidle_governor ladder_governor = {
  */
 static int __init init_ladder(void)
 {
+	/*
+	 * When NO_HZ is disabled, or when booting with nohz=off, the ladder
+	 * governor is better so give it a higher rating than the menu
+	 * governor.
+	 */
+	if (!tick_nohz_enabled)
+		ladder_governor.rating = 25;
+
 	return cpuidle_register_governor(&ladder_governor);
 }
 
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 7b0971d97cc3..0742b3296673 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -294,8 +294,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 		data->needs_update = 0;
 	}
 
-	data->last_state_idx = CPUIDLE_DRIVER_STATE_START - 1;
-
 	/* Special case when user has set very strict latency requirement */
 	if (unlikely(latency_req == 0))
 		return 0;
@@ -326,20 +324,25 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 	if (latency_req > interactivity_req)
 		latency_req = interactivity_req;
 
-	/*
-	 * We want to default to C1 (hlt), not to busy polling
-	 * unless the timer is happening really really soon.
-	 */
-	if (interactivity_req > 20 &&
-	    !drv->states[CPUIDLE_DRIVER_STATE_START].disabled &&
-		dev->states_usage[CPUIDLE_DRIVER_STATE_START].disable == 0)
+	if (CPUIDLE_DRIVER_STATE_START > 0) {
+		data->last_state_idx = CPUIDLE_DRIVER_STATE_START - 1;
+		/*
+		 * We want to default to C1 (hlt), not to busy polling
+		 * unless the timer is happening really really soon.
+		 */
+		if (interactivity_req > 20 &&
+		    !drv->states[CPUIDLE_DRIVER_STATE_START].disabled &&
+			dev->states_usage[CPUIDLE_DRIVER_STATE_START].disable == 0)
+			data->last_state_idx = CPUIDLE_DRIVER_STATE_START;
+	} else {
 		data->last_state_idx = CPUIDLE_DRIVER_STATE_START;
+	}
 
 	/*
 	 * Find the idle state with the lowest power while satisfying
 	 * our constraints.
 	 */
-	for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
+	for (i = data->last_state_idx + 1; i < drv->state_count; i++) {
 		struct cpuidle_state *s = &drv->states[i];
 		struct cpuidle_state_usage *su = &dev->states_usage[i];
 
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 3dd69df9c970..07d494276aad 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -381,6 +381,7 @@ config CRYPTO_DEV_BFIN_CRC
 
 config CRYPTO_DEV_ATMEL_AES
 	tristate "Support for Atmel AES hw accelerator"
+	depends on HAS_DMA
 	depends on AT_XDMAC || AT_HDMAC || COMPILE_TEST
 	select CRYPTO_AES
 	select CRYPTO_AEAD
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 5621612ee921..6dd3317ca365 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -280,6 +280,7 @@ static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
 	case AES_GCMHR(2):
 	case AES_GCMHR(3):
 		snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
+		break;
 
 	default:
 		snprintf(tmp, sz, "0x%02x", offset);
diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c
index 0ac0ba867611..1e480f140663 100644
--- a/drivers/crypto/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/qat/qat_common/qat_hal.c
@@ -389,7 +389,7 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 {
 	unsigned int base_cnt, cur_cnt;
 	unsigned char ae;
-	unsigned int times = MAX_RETRY_TIMES;
+	int times = MAX_RETRY_TIMES;
 
 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 		qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
@@ -402,7 +402,7 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 			cur_cnt &= 0xffff;
 		} while (times-- && (cur_cnt == base_cnt));
 
-		if (!times) {
+		if (times < 0) {
 			pr_err("QAT: AE%d is inactive!!\n", ae);
 			return -EFAULT;
 		}
@@ -453,7 +453,11 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 	void __iomem *csr_addr =
 			(void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
 			ESRAM_AUTO_INIT_CSR_OFFSET);
-	unsigned int csr_val, times = 30;
+	unsigned int csr_val;
+	int times = 30;
+
+	if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID)
+		return 0;
 
 	csr_val = ADF_CSR_RD(csr_addr, 0);
 	if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
@@ -467,7 +471,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 		qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 		csr_val = ADF_CSR_RD(csr_addr, 0);
 	} while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
-	if ((!times)) {
+	if ((times < 0)) {
 		pr_err("QAT: Fail to init eSram!\n");
 		return -EFAULT;
 	}
@@ -658,7 +662,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 			ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 		} while (ret && times--);
 
-		if (!times) {
+		if (times < 0) {
 			pr_err("QAT: clear GPR of AE %d failed", ae);
 			return -EINVAL;
 		}
@@ -693,14 +697,12 @@ int qat_hal_init(struct adf_accel_dev *accel_dev)
 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 	struct adf_bar *misc_bar =
 			&pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
-	struct adf_bar *sram_bar =
-			&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
+	struct adf_bar *sram_bar;
 
 	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 	if (!handle)
 		return -ENOMEM;
 
-	handle->hal_sram_addr_v = sram_bar->virt_addr;
 	handle->hal_cap_g_ctl_csr_addr_v =
 		(void __iomem *)((uintptr_t)misc_bar->virt_addr +
 				 ICP_QAT_CAP_OFFSET);
@@ -714,6 +716,11 @@ int qat_hal_init(struct adf_accel_dev *accel_dev)
 		(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
 				 LOCAL_TO_XFER_REG_OFFSET);
 	handle->pci_dev = pci_info->pci_dev;
+	if (handle->pci_dev->device != ADF_C3XXX_PCI_DEVICE_ID) {
+		sram_bar =
+			&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
+		handle->hal_sram_addr_v = sram_bar->virt_addr;
+	}
 	handle->fw_auth = (handle->pci_dev->device ==
 			   ADF_DH895XCC_PCI_DEVICE_ID) ? false : true;
 	handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c
index 263af709e536..022c7ab7351a 100644
--- a/drivers/crypto/vmx/aes.c
+++ b/drivers/crypto/vmx/aes.c
@@ -83,10 +83,10 @@ static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,
 
 	preempt_disable();
 	pagefault_disable();
-	enable_kernel_altivec();
 	enable_kernel_vsx();
 	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
 	ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
+	disable_kernel_vsx();
 	pagefault_enable();
 	preempt_enable();
 
@@ -103,9 +103,9 @@ static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	} else {
 		preempt_disable();
 		pagefault_disable();
-		enable_kernel_altivec();
 		enable_kernel_vsx();
 		aes_p8_encrypt(src, dst, &ctx->enc_key);
+		disable_kernel_vsx();
 		pagefault_enable();
 		preempt_enable();
 	}
@@ -120,9 +120,9 @@ static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	} else {
 		preempt_disable();
 		pagefault_disable();
-		enable_kernel_altivec();
 		enable_kernel_vsx();
 		aes_p8_decrypt(src, dst, &ctx->dec_key);
+		disable_kernel_vsx();
 		pagefault_enable();
 		preempt_enable();
 	}
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index 78a978613ca8..495577b6d31b 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -84,10 +84,10 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
 
 	preempt_disable();
 	pagefault_disable();
-	enable_kernel_altivec();
 	enable_kernel_vsx();
 	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
 	ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
+	disable_kernel_vsx();
 	pagefault_enable();
 	preempt_enable();
 
@@ -115,7 +115,6 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
 	} else {
 		preempt_disable();
 		pagefault_disable();
-		enable_kernel_altivec();
 		enable_kernel_vsx();
 
 		blkcipher_walk_init(&walk, dst, src, nbytes);
@@ -129,6 +128,7 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
 			ret = blkcipher_walk_done(desc, &walk, nbytes);
 		}
 
+		disable_kernel_vsx();
 		pagefault_enable();
 		preempt_enable();
 	}
@@ -156,7 +156,6 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
 	} else {
 		preempt_disable();
 		pagefault_disable();
-		enable_kernel_altivec();
 		enable_kernel_vsx();
 
 		blkcipher_walk_init(&walk, dst, src, nbytes);
@@ -170,6 +169,7 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
 			ret = blkcipher_walk_done(desc, &walk, nbytes);
 		}
 
+		disable_kernel_vsx();
 		pagefault_enable();
 		preempt_enable();
 	}
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 1febc4f1d9af..0a3c1b04cf3c 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -81,9 +81,9 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
 	struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
 
 	pagefault_disable();
-	enable_kernel_altivec();
 	enable_kernel_vsx();
 	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
+	disable_kernel_vsx();
 	pagefault_enable();
 
 	ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
@@ -100,9 +100,9 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
 	unsigned int nbytes = walk->nbytes;
 
 	pagefault_disable();
-	enable_kernel_altivec();
 	enable_kernel_vsx();
 	aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key);
+	disable_kernel_vsx();
 	pagefault_enable();
 
 	crypto_xor(keystream, src, nbytes);
@@ -133,7 +133,6 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
 		ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 		while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 			pagefault_disable();
-			enable_kernel_altivec();
 			enable_kernel_vsx();
 			aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
 						    walk.dst.virt.addr,
@@ -142,6 +141,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
 						    AES_BLOCK_SIZE,
 						    &ctx->enc_key,
 						    walk.iv);
+			disable_kernel_vsx();
 			pagefault_enable();
 
 			/* We need to update IV mostly for last bytes/round */
diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c
index 2183a2e77641..6c999cb01b80 100644
--- a/drivers/crypto/vmx/ghash.c
+++ b/drivers/crypto/vmx/ghash.c
@@ -118,10 +118,9 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
 
 	preempt_disable();
 	pagefault_disable();
-	enable_kernel_altivec();
 	enable_kernel_vsx();
-	enable_kernel_fp();
 	gcm_init_p8(ctx->htable, (const u64 *) key);
+	disable_kernel_vsx();
 	pagefault_enable();
 	preempt_enable();
 	return crypto_shash_setkey(ctx->fallback, key, keylen);
@@ -149,11 +148,10 @@ static int p8_ghash_update(struct shash_desc *desc,
 			       GHASH_DIGEST_SIZE - dctx->bytes);
 			preempt_disable();
 			pagefault_disable();
-			enable_kernel_altivec();
 			enable_kernel_vsx();
-			enable_kernel_fp();
 			gcm_ghash_p8(dctx->shash, ctx->htable,
 				     dctx->buffer, GHASH_DIGEST_SIZE);
+			disable_kernel_vsx();
 			pagefault_enable();
 			preempt_enable();
 			src += GHASH_DIGEST_SIZE - dctx->bytes;
@@ -164,10 +162,9 @@ static int p8_ghash_update(struct shash_desc *desc,
 		if (len) {
 			preempt_disable();
 			pagefault_disable();
-			enable_kernel_altivec();
 			enable_kernel_vsx();
-			enable_kernel_fp();
 			gcm_ghash_p8(dctx->shash, ctx->htable, src, len);
+			disable_kernel_vsx();
 			pagefault_enable();
 			preempt_enable();
 			src += len;
@@ -195,11 +192,10 @@ static int p8_ghash_final(struct shash_desc *desc, u8 *out)
 				dctx->buffer[i] = 0;
 			preempt_disable();
 			pagefault_disable();
-			enable_kernel_altivec();
 			enable_kernel_vsx();
-			enable_kernel_fp();
 			gcm_ghash_p8(dctx->shash, ctx->htable,
 				     dctx->buffer, GHASH_DIGEST_SIZE);
+			disable_kernel_vsx();
 			pagefault_enable();
 			preempt_enable();
 			dctx->bytes = 0;
diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c
index f304a0289eda..38bf144ca147 100644
--- a/drivers/devfreq/devfreq-event.c
+++ b/drivers/devfreq/devfreq-event.c
@@ -226,17 +226,12 @@ struct devfreq_event_dev *devfreq_event_get_edev_by_phandle(struct device *dev,
 	struct device_node *node;
 	struct devfreq_event_dev *edev;
 
-	if (!dev->of_node) {
-		dev_err(dev, "device does not have a device node entry\n");
+	if (!dev->of_node)
 		return ERR_PTR(-EINVAL);
-	}
 
 	node = of_parse_phandle(dev->of_node, "devfreq-events", index);
-	if (!node) {
-		dev_err(dev, "failed to get phandle in %s node\n",
-			dev->of_node->full_name);
+	if (!node)
 		return ERR_PTR(-ENODEV);
-	}
 
 	mutex_lock(&devfreq_event_list_lock);
 	list_for_each_entry(edev, &devfreq_event_list, node) {
@@ -248,8 +243,6 @@ out:
 	mutex_unlock(&devfreq_event_list_lock);
 
 	if (!edev) {
-		dev_err(dev, "unable to get devfreq-event device : %s\n",
-			node->name);
 		of_node_put(node);
 		return ERR_PTR(-ENODEV);
 	}
@@ -277,7 +270,7 @@ int devfreq_event_get_edev_count(struct device *dev)
 
 	count = of_property_count_elems_of_size(dev->of_node, "devfreq-events",
 						sizeof(u32));
-	if (count < 0 ) {
+	if (count < 0) {
 		dev_err(dev,
 			"failed to get the count of devfreq-event in %s node\n",
 			dev->of_node->full_name);
@@ -402,7 +395,8 @@ struct devfreq_event_dev *devm_devfreq_event_add_edev(struct device *dev,
 {
 	struct devfreq_event_dev **ptr, *edev;
 
-	ptr = devres_alloc(devm_devfreq_event_release, sizeof(*ptr), GFP_KERNEL);
+	ptr = devres_alloc(devm_devfreq_event_release, sizeof(*ptr),
+				GFP_KERNEL);
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index ca848cc6a8fd..984c5e9e7bdd 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -85,6 +85,46 @@ static int devfreq_get_freq_level(struct devfreq *devfreq, unsigned long freq)
 }
 
 /**
+ * devfreq_set_freq_table() - Initialize freq_table for the frequency
+ * @devfreq:	the devfreq instance
+ */
+static void devfreq_set_freq_table(struct devfreq *devfreq)
+{
+	struct devfreq_dev_profile *profile = devfreq->profile;
+	struct dev_pm_opp *opp;
+	unsigned long freq;
+	int i, count;
+
+	/* Initialize the freq_table from OPP table */
+	count = dev_pm_opp_get_opp_count(devfreq->dev.parent);
+	if (count <= 0)
+		return;
+
+	profile->max_state = count;
+	profile->freq_table = devm_kcalloc(devfreq->dev.parent,
+					profile->max_state,
+					sizeof(*profile->freq_table),
+					GFP_KERNEL);
+	if (!profile->freq_table) {
+		profile->max_state = 0;
+		return;
+	}
+
+	rcu_read_lock();
+	for (i = 0, freq = 0; i < profile->max_state; i++, freq++) {
+		opp = dev_pm_opp_find_freq_ceil(devfreq->dev.parent, &freq);
+		if (IS_ERR(opp)) {
+			devm_kfree(devfreq->dev.parent, profile->freq_table);
+			profile->max_state = 0;
+			rcu_read_unlock();
+			return;
+		}
+		profile->freq_table[i] = freq;
+	}
+	rcu_read_unlock();
+}
+
+/**
  * devfreq_update_status() - Update statistics of devfreq behavior
  * @devfreq:	the devfreq instance
  * @freq:	the update target frequency
@@ -478,6 +518,12 @@ struct devfreq *devfreq_add_device(struct device *dev,
 	devfreq->data = data;
 	devfreq->nb.notifier_call = devfreq_notifier_call;
 
+	if (!devfreq->profile->max_state && !devfreq->profile->freq_table) {
+		mutex_unlock(&devfreq->lock);
+		devfreq_set_freq_table(devfreq);
+		mutex_lock(&devfreq->lock);
+	}
+
 	devfreq->trans_table =	devm_kzalloc(dev, sizeof(unsigned int) *
 						devfreq->profile->max_state *
 						devfreq->profile->max_state,
@@ -921,12 +967,6 @@ unlock:
 	return ret;
 }
 
-static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr,
-			     char *buf)
-{
-	return sprintf(buf, "%lu\n", to_devfreq(dev)->min_freq);
-}
-
 static ssize_t max_freq_store(struct device *dev, struct device_attribute *attr,
 			      const char *buf, size_t count)
 {
@@ -953,13 +993,17 @@ unlock:
 	mutex_unlock(&df->lock);
 	return ret;
 }
-static DEVICE_ATTR_RW(min_freq);
 
-static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr,
-			     char *buf)
-{
-	return sprintf(buf, "%lu\n", to_devfreq(dev)->max_freq);
+#define show_one(name)						\
+static ssize_t name##_show					\
+(struct device *dev, struct device_attribute *attr, char *buf)	\
+{								\
+	return sprintf(buf, "%lu\n", to_devfreq(dev)->name);	\
 }
+show_one(min_freq);
+show_one(max_freq);
+
+static DEVICE_ATTR_RW(min_freq);
 static DEVICE_ATTR_RW(max_freq);
 
 static ssize_t available_frequencies_show(struct device *d,
@@ -1005,11 +1049,13 @@ static ssize_t trans_stat_show(struct device *dev,
 	if (!devfreq->stop_polling &&
 			devfreq_update_status(devfreq, devfreq->previous_freq))
 		return 0;
+	if (max_state == 0)
+		return sprintf(buf, "Not Supported.\n");
 
-	len = sprintf(buf, "   From  :   To\n");
-	len += sprintf(buf + len, "         :");
+	len = sprintf(buf, "     From  :   To\n");
+	len += sprintf(buf + len, "           :");
 	for (i = 0; i < max_state; i++)
-		len += sprintf(buf + len, "%8u",
+		len += sprintf(buf + len, "%10lu",
 				devfreq->profile->freq_table[i]);
 
 	len += sprintf(buf + len, "   time(ms)\n");
@@ -1021,10 +1067,10 @@ static ssize_t trans_stat_show(struct device *dev,
 		} else {
 			len += sprintf(buf + len, " ");
 		}
-		len += sprintf(buf + len, "%8u:",
+		len += sprintf(buf + len, "%10lu:",
 				devfreq->profile->freq_table[i]);
 		for (j = 0; j < max_state; j++)
-			len += sprintf(buf + len, "%8u",
+			len += sprintf(buf + len, "%10u",
 				devfreq->trans_table[(i * max_state) + j]);
 		len += sprintf(buf + len, "%10u\n",
 			jiffies_to_msecs(devfreq->time_in_state[i]));
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 3a8ce67910c2..79b1390f2016 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -444,7 +444,7 @@ config STM32_DMA
 	  here.
 
 config S3C24XX_DMAC
-	tristate "Samsung S3C24XX DMA support"
+	bool "Samsung S3C24XX DMA support"
 	depends on ARCH_S3C24XX
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 39f59666f93f..64f5d1bdbb48 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -1700,6 +1700,7 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan)
 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)
 		at_xdmac_remove_xfer(atchan, desc);
 
+	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
 	spin_unlock_irqrestore(&atchan->lock, flags);
 
@@ -1832,6 +1833,8 @@ static int atmel_xdmac_resume(struct device *dev)
 		atchan = to_at_xdmac_chan(chan);
 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);
 		if (at_xdmac_chan_is_cyclic(atchan)) {
+			if (at_xdmac_chan_is_paused(atchan))
+				at_xdmac_device_resume(chan);
 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);
 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);
 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index 8b20930ade98..e893318560db 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -156,6 +156,7 @@ static void dwc_initialize(struct dw_dma_chan *dwc)
 
 	/* Enable interrupts */
 	channel_set_bit(dw, MASK.XFER, dwc->mask);
+	channel_set_bit(dw, MASK.BLOCK, dwc->mask);
 	channel_set_bit(dw, MASK.ERROR, dwc->mask);
 
 	dwc->initialized = true;
@@ -536,16 +537,17 @@ EXPORT_SYMBOL(dw_dma_get_dst_addr);
 
 /* Called with dwc->lock held and all DMAC interrupts disabled */
 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
-		u32 status_err, u32 status_xfer)
+		u32 status_block, u32 status_err, u32 status_xfer)
 {
 	unsigned long flags;
 
-	if (dwc->mask) {
+	if (status_block & dwc->mask) {
 		void (*callback)(void *param);
 		void *callback_param;
 
 		dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
 				channel_readl(dwc, LLP));
+		dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 
 		callback = dwc->cdesc->period_callback;
 		callback_param = dwc->cdesc->period_callback_param;
@@ -577,6 +579,7 @@ static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
 		channel_writel(dwc, CTL_LO, 0);
 		channel_writel(dwc, CTL_HI, 0);
 
+		dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 		dma_writel(dw, CLEAR.ERROR, dwc->mask);
 		dma_writel(dw, CLEAR.XFER, dwc->mask);
 
@@ -593,10 +596,12 @@ static void dw_dma_tasklet(unsigned long data)
 {
 	struct dw_dma *dw = (struct dw_dma *)data;
 	struct dw_dma_chan *dwc;
+	u32 status_block;
 	u32 status_xfer;
 	u32 status_err;
 	int i;
 
+	status_block = dma_readl(dw, RAW.BLOCK);
 	status_xfer = dma_readl(dw, RAW.XFER);
 	status_err = dma_readl(dw, RAW.ERROR);
 
@@ -605,7 +610,8 @@ static void dw_dma_tasklet(unsigned long data)
 	for (i = 0; i < dw->dma.chancnt; i++) {
 		dwc = &dw->chan[i];
 		if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
-			dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
+			dwc_handle_cyclic(dw, dwc, status_block, status_err,
+					status_xfer);
 		else if (status_err & (1 << i))
 			dwc_handle_error(dw, dwc);
 		else if (status_xfer & (1 << i))
@@ -616,6 +622,7 @@ static void dw_dma_tasklet(unsigned long data)
 	 * Re-enable interrupts.
 	 */
 	channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
+	channel_set_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 }
 
@@ -640,6 +647,7 @@ static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 	 * softirq handler.
 	 */
 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
+	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 
 	status = dma_readl(dw, STATUS_INT);
@@ -650,6 +658,7 @@ static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 
 		/* Try to recover */
 		channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
+		channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
@@ -1116,6 +1125,7 @@ static void dw_dma_off(struct dw_dma *dw)
 	dma_writel(dw, CFG, 0);
 
 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
+	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
@@ -1221,6 +1231,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
 
 	/* Disable interrupts */
 	channel_clear_bit(dw, MASK.XFER, dwc->mask);
+	channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
 	channel_clear_bit(dw, MASK.ERROR, dwc->mask);
 
 	spin_unlock_irqrestore(&dwc->lock, flags);
@@ -1250,7 +1261,6 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
 int dw_dma_cyclic_start(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
-	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);
 	unsigned long		flags;
 
 	if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
@@ -1259,27 +1269,7 @@ int dw_dma_cyclic_start(struct dma_chan *chan)
 	}
 
 	spin_lock_irqsave(&dwc->lock, flags);
-
-	/* Assert channel is idle */
-	if (dma_readl(dw, CH_EN) & dwc->mask) {
-		dev_err(chan2dev(&dwc->chan),
-			"%s: BUG: Attempted to start non-idle channel\n",
-			__func__);
-		dwc_dump_chan_regs(dwc);
-		spin_unlock_irqrestore(&dwc->lock, flags);
-		return -EBUSY;
-	}
-
-	dma_writel(dw, CLEAR.ERROR, dwc->mask);
-	dma_writel(dw, CLEAR.XFER, dwc->mask);
-
-	/* Setup DMAC channel registers */
-	channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
-	channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
-	channel_writel(dwc, CTL_HI, 0);
-
-	channel_set_bit(dw, CH_EN, dwc->mask);
-
+	dwc_dostart(dwc, dwc->cdesc->desc[0]);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	return 0;
@@ -1484,6 +1474,7 @@ void dw_dma_cyclic_free(struct dma_chan *chan)
 
 	dwc_chan_disable(dw, dwc);
 
+	dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 	dma_writel(dw, CLEAR.ERROR, dwc->mask);
 	dma_writel(dw, CLEAR.XFER, dwc->mask);
 
@@ -1572,9 +1563,6 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
 	/* Force dma off, just in case */
 	dw_dma_off(dw);
 
-	/* Disable BLOCK interrupts as well */
-	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
-
 	/* Create a pool of consistent memory blocks for hardware descriptors */
 	dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev,
 					 sizeof(struct dw_desc), 4, 0);
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index 50584015e046..d92d65549406 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -484,7 +484,7 @@ static void edma_read_slot(struct edma_cc *ecc, unsigned slot,
  */
 static int edma_alloc_slot(struct edma_cc *ecc, int slot)
 {
-	if (slot > 0) {
+	if (slot >= 0) {
 		slot = EDMA_CHAN_SLOT(slot);
 		/* Requesting entry paRAM slot for a HW triggered channel. */
 		if (ecc->chmap_exist && slot < ecc->num_channels)
diff --git a/drivers/firmware/broadcom/bcm47xx_nvram.c b/drivers/firmware/broadcom/bcm47xx_nvram.c
index e41594510b97..0c2f0a61b0ea 100644
--- a/drivers/firmware/broadcom/bcm47xx_nvram.c
+++ b/drivers/firmware/broadcom/bcm47xx_nvram.c
@@ -56,9 +56,7 @@ static u32 find_nvram_size(void __iomem *end)
 static int nvram_find_and_copy(void __iomem *iobase, u32 lim)
 {
 	struct nvram_header __iomem *header;
-	int i;
 	u32 off;
-	u32 *src, *dst;
 	u32 size;
 
 	if (nvram_len) {
@@ -95,10 +93,7 @@ static int nvram_find_and_copy(void __iomem *iobase, u32 lim)
 	return -ENXIO;
 
 found:
-	src = (u32 *)header;
-	dst = (u32 *)nvram_buf;
-	for (i = 0; i < sizeof(struct nvram_header); i += 4)
-		*dst++ = __raw_readl(src++);
+	__ioread32_copy(nvram_buf, header, sizeof(*header) / 4);
 	header = (struct nvram_header *)nvram_buf;
 	nvram_len = header->len;
 	if (nvram_len > size) {
@@ -111,8 +106,8 @@ found:
 		nvram_len = NVRAM_SPACE - 1;
 	}
 	/* proceed reading data after header */
-	for (; i < nvram_len; i += 4)
-		*dst++ = readl(src++);
+	__ioread32_copy(nvram_buf + sizeof(*header), header + 1,
+			DIV_ROUND_UP(nvram_len, 4));
 	nvram_buf[NVRAM_SPACE - 1] = '\0';
 
 	return 0;
diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c
index 0e08e665f715..88bebe1968b7 100644
--- a/drivers/firmware/dmi_scan.c
+++ b/drivers/firmware/dmi_scan.c
@@ -321,39 +321,58 @@ static void __init dmi_save_ipmi_device(const struct dmi_header *dm)
 	list_add_tail(&dev->list, &dmi_devices);
 }
 
-static void __init dmi_save_dev_onboard(int instance, int segment, int bus,
-					int devfn, const char *name)
+static void __init dmi_save_dev_pciaddr(int instance, int segment, int bus,
+					int devfn, const char *name, int type)
 {
-	struct dmi_dev_onboard *onboard_dev;
+	struct dmi_dev_onboard *dev;
 
-	onboard_dev = dmi_alloc(sizeof(*onboard_dev) + strlen(name) + 1);
-	if (!onboard_dev)
+	/* Ignore invalid values */
+	if (type == DMI_DEV_TYPE_DEV_SLOT &&
+	    segment == 0xFFFF && bus == 0xFF && devfn == 0xFF)
 		return;
 
-	onboard_dev->instance = instance;
-	onboard_dev->segment = segment;
-	onboard_dev->bus = bus;
-	onboard_dev->devfn = devfn;
+	dev = dmi_alloc(sizeof(*dev) + strlen(name) + 1);
+	if (!dev)
+		return;
 
-	strcpy((char *)&onboard_dev[1], name);
-	onboard_dev->dev.type = DMI_DEV_TYPE_DEV_ONBOARD;
-	onboard_dev->dev.name = (char *)&onboard_dev[1];
-	onboard_dev->dev.device_data = onboard_dev;
+	dev->instance = instance;
+	dev->segment = segment;
+	dev->bus = bus;
+	dev->devfn = devfn;
 
-	list_add(&onboard_dev->dev.list, &dmi_devices);
+	strcpy((char *)&dev[1], name);
+	dev->dev.type = type;
+	dev->dev.name = (char *)&dev[1];
+	dev->dev.device_data = dev;
+
+	list_add(&dev->dev.list, &dmi_devices);
 }
 
 static void __init dmi_save_extended_devices(const struct dmi_header *dm)
 {
-	const u8 *d = (u8 *) dm + 5;
+	const char *name;
+	const u8 *d = (u8 *)dm;
 
 	/* Skip disabled device */
-	if ((*d & 0x80) == 0)
+	if ((d[0x5] & 0x80) == 0)
 		return;
 
-	dmi_save_dev_onboard(*(d+1), *(u16 *)(d+2), *(d+4), *(d+5),
-			     dmi_string_nosave(dm, *(d-1)));
-	dmi_save_one_device(*d & 0x7f, dmi_string_nosave(dm, *(d - 1)));
+	name = dmi_string_nosave(dm, d[0x4]);
+	dmi_save_dev_pciaddr(d[0x6], *(u16 *)(d + 0x7), d[0x9], d[0xA], name,
+			     DMI_DEV_TYPE_DEV_ONBOARD);
+	dmi_save_one_device(d[0x5] & 0x7f, name);
+}
+
+static void __init dmi_save_system_slot(const struct dmi_header *dm)
+{
+	const u8 *d = (u8 *)dm;
+
+	/* Need SMBIOS 2.6+ structure */
+	if (dm->length < 0x11)
+		return;
+	dmi_save_dev_pciaddr(*(u16 *)(d + 0x9), *(u16 *)(d + 0xD), d[0xF],
+			     d[0x10], dmi_string_nosave(dm, d[0x4]),
+			     DMI_DEV_TYPE_DEV_SLOT);
 }
 
 static void __init count_mem_devices(const struct dmi_header *dm, void *v)
@@ -426,6 +445,9 @@ static void __init dmi_decode(const struct dmi_header *dm, void *dummy)
 		dmi_save_ident(dm, DMI_CHASSIS_SERIAL, 7);
 		dmi_save_ident(dm, DMI_CHASSIS_ASSET_TAG, 8);
 		break;
+	case 9:		/* System Slots */
+		dmi_save_system_slot(dm);
+		break;
 	case 10:	/* Onboard Devices Information */
 		dmi_save_devices(dm);
 		break;
@@ -869,7 +891,7 @@ EXPORT_SYMBOL(dmi_name_in_vendors);
  *	@from: previous device found in search, or %NULL for new search.
  *
  *	Iterates through the list of known onboard devices. If a device is
- *	found with a matching @vendor and @device, a pointer to its device
+ *	found with a matching @type and @name, a pointer to its device
  *	structure is returned.  Otherwise, %NULL is returned.
  *	A new search is initiated by passing %NULL as the @from argument.
  *	If @from is not %NULL, searches continue from next device.
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 9c12e18031d5..aaf9c0bab42e 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -22,6 +22,7 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
+UBSAN_SANITIZE			:= n
 
 lib-y				:= efi-stub-helper.o
 
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index b62e2f5dcab3..cf7b7d46302a 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -253,7 +253,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 			sys_table->boottime->free_pool(memory_map);
 			new_fdt_size += EFI_PAGE_SIZE;
 		} else {
-			pr_efi_err(sys_table, "Unable to constuct new device tree.\n");
+			pr_efi_err(sys_table, "Unable to construct new device tree.\n");
 			goto fail_free_mmap;
 		}
 	}
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index b18bea08ff25..c88dd24a4b1f 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -134,8 +134,8 @@ config GPIO_BCM_KONA
 
 config GPIO_BRCMSTB
 	tristate "BRCMSTB GPIO support"
-	default y if ARCH_BRCMSTB
-	depends on OF_GPIO && (ARCH_BRCMSTB || COMPILE_TEST)
+	default y if (ARCH_BRCMSTB || BMIPS_GENERIC)
+	depends on OF_GPIO && (ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST)
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
 	help
@@ -344,13 +344,6 @@ config GPIO_RCAR
 	help
 	  Say yes here to support GPIO on Renesas R-Car SoCs.
 
-config GPIO_SAMSUNG
-	bool
-	depends on PLAT_SAMSUNG
-	help
-	  Legacy GPIO support. Use only for platforms without support for
-	  pinctrl.
-
 config GPIO_SPEAR_SPICS
 	bool "ST SPEAr13xx SPI Chip Select as GPIO support"
 	depends on PLAT_SPEAR
@@ -496,8 +489,21 @@ menu "Port-mapped I/O GPIO drivers"
 
 config GPIO_104_IDIO_16
 	tristate "ACCES 104-IDIO-16 GPIO support"
+	select GPIOLIB_IRQCHIP
 	help
-	  Enables GPIO support for the ACCES 104-IDIO-16 family.
+	  Enables GPIO support for the ACCES 104-IDIO-16 family. The base port
+	  address for the device may be set via the idio_16_base module
+	  parameter. The interrupt line number for the device may be set via the
+	  idio_16_irq module parameter.
+
+config GPIO_104_IDI_48
+	tristate "ACCES 104-IDI-48 GPIO support"
+	select GPIOLIB_IRQCHIP
+	help
+	  Enables GPIO support for the ACCES 104-IDI-48 family. The base port
+	  address for the device may be configured via the idi_48_base module
+	  parameter. The interrupt line number for the device may be configured
+	  via the idi_48_irq module parameter.
 
 config GPIO_F7188X
 	tristate "F71869, F71869A, F71882FG and F71889F GPIO support"
@@ -907,7 +913,6 @@ menu "PCI GPIO expanders"
 
 config GPIO_AMD8111
 	tristate "AMD 8111 GPIO driver"
-	depends on PCI
 	help
 	  The AMD 8111 south bridge contains 32 GPIO pins which can be used.
 
@@ -919,7 +924,7 @@ config GPIO_AMD8111
 
 config GPIO_BT8XX
 	tristate "BT8XX GPIO abuser"
-	depends on PCI && VIDEO_BT848=n
+	depends on VIDEO_BT848=n
 	help
 	  The BT8xx frame grabber chip has 24 GPIO pins that can be abused
 	  as a cheap PCI GPIO card.
@@ -935,14 +940,13 @@ config GPIO_BT8XX
 
 config GPIO_INTEL_MID
 	bool "Intel Mid GPIO support"
-	depends on PCI && X86
+	depends on X86
 	select GPIOLIB_IRQCHIP
 	help
 	  Say Y here to support Intel Mid GPIO.
 
 config GPIO_ML_IOH
 	tristate "OKI SEMICONDUCTOR ML7213 IOH GPIO support"
-	depends on PCI
 	select GENERIC_IRQ_CHIP
 	help
 	  ML7213 is companion chip for Intel Atom E6xx series.
@@ -952,7 +956,7 @@ config GPIO_ML_IOH
 
 config GPIO_PCH
 	tristate "Intel EG20T PCH/LAPIS Semiconductor IOH(ML7223/ML7831) GPIO"
-	depends on PCI && (X86_32 || COMPILE_TEST)
+	depends on X86_32 || MIPS || COMPILE_TEST
 	select GENERIC_IRQ_CHIP
 	help
 	  This driver is for PCH(Platform controller Hub) GPIO of Intel Topcliff
@@ -968,7 +972,6 @@ config GPIO_PCH
 
 config GPIO_RDC321X
 	tristate "RDC R-321x GPIO support"
-	depends on PCI
 	select MFD_CORE
 	select MFD_RDC321X
 	help
@@ -977,7 +980,7 @@ config GPIO_RDC321X
 
 config GPIO_SODAVILLE
 	bool "Intel Sodaville GPIO support"
-	depends on X86 && PCI && OF
+	depends on X86 && OF
 	select GPIO_GENERIC
 	select GENERIC_IRQ_CHIP
 	help
@@ -1028,7 +1031,7 @@ menu "USB GPIO expanders"
 
 config GPIO_VIPERBOARD
 	tristate "Viperboard GPIO a & b support"
-	depends on MFD_VIPERBOARD && USB
+	depends on MFD_VIPERBOARD
 	help
 	  Say yes here to access the GPIO signals of Nano River
 	  Technologies Viperboard. There are two GPIO chips on the
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 986dbd838cea..ece7d7cbdc80 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_GPIO_ACPI)		+= gpiolib-acpi.o
 obj-$(CONFIG_GPIO_GENERIC)	+= gpio-generic.o
 
 obj-$(CONFIG_GPIO_104_IDIO_16)	+= gpio-104-idio-16.o
+obj-$(CONFIG_GPIO_104_IDI_48)	+= gpio-104-idi-48.o
 obj-$(CONFIG_GPIO_74X164)	+= gpio-74x164.o
 obj-$(CONFIG_GPIO_74XX_MMIO)	+= gpio-74xx-mmio.o
 obj-$(CONFIG_GPIO_ADNP)		+= gpio-adnp.o
@@ -79,7 +80,6 @@ obj-$(CONFIG_GPIO_PXA)		+= gpio-pxa.o
 obj-$(CONFIG_GPIO_RC5T583)	+= gpio-rc5t583.o
 obj-$(CONFIG_GPIO_RDC321X)	+= gpio-rdc321x.o
 obj-$(CONFIG_GPIO_RCAR)		+= gpio-rcar.o
-obj-$(CONFIG_GPIO_SAMSUNG)	+= gpio-samsung.o
 obj-$(CONFIG_ARCH_SA1100)	+= gpio-sa1100.o
 obj-$(CONFIG_GPIO_SCH)		+= gpio-sch.o
 obj-$(CONFIG_GPIO_SCH311X)	+= gpio-sch311x.o
diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c
new file mode 100644
index 000000000000..52eed328ce99
--- /dev/null
+++ b/drivers/gpio/gpio-104-idi-48.c
@@ -0,0 +1,343 @@
+/*
+ * GPIO driver for the ACCES 104-IDI-48 family
+ * Copyright (C) 2015 William Breathitt Gray
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/gpio/driver.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/interrupt.h>
+#include <linux/irqdesc.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+static unsigned idi_48_base;
+module_param(idi_48_base, uint, 0);
+MODULE_PARM_DESC(idi_48_base, "ACCES 104-IDI-48 base address");
+static unsigned idi_48_irq;
+module_param(idi_48_irq, uint, 0);
+MODULE_PARM_DESC(idi_48_irq, "ACCES 104-IDI-48 interrupt line number");
+
+/**
+ * struct idi_48_gpio - GPIO device private data structure
+ * @chip:	instance of the gpio_chip
+ * @lock:	synchronization lock to prevent I/O race conditions
+ * @ack_lock:	synchronization lock to prevent IRQ handler race conditions
+ * @irq_mask:	input bits affected by interrupts
+ * @base:	base port address of the GPIO device
+ * @extent:	extent of port address region of the GPIO device
+ * @irq:	Interrupt line number
+ * @cos_enb:	Change-Of-State IRQ enable boundaries mask
+ */
+struct idi_48_gpio {
+	struct gpio_chip chip;
+	spinlock_t lock;
+	spinlock_t ack_lock;
+	unsigned char irq_mask[6];
+	unsigned base;
+	unsigned extent;
+	unsigned irq;
+	unsigned char cos_enb;
+};
+
+static int idi_48_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+{
+	return 1;
+}
+
+static int idi_48_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
+{
+	return 0;
+}
+
+static int idi_48_gpio_get(struct gpio_chip *chip, unsigned offset)
+{
+	struct idi_48_gpio *const idi48gpio = gpiochip_get_data(chip);
+	unsigned i;
+	const unsigned register_offset[6] = { 0, 1, 2, 4, 5, 6 };
+	unsigned base_offset;
+	unsigned mask;
+
+	for (i = 0; i < 48; i += 8)
+		if (offset < i + 8) {
+			base_offset = register_offset[i / 8];
+			mask = BIT(offset - i);
+
+			return !!(inb(idi48gpio->base + base_offset) & mask);
+		}
+
+	/* The following line should never execute since offset < 48 */
+	return 0;
+}
+
+static void idi_48_irq_ack(struct irq_data *data)
+{
+}
+
+static void idi_48_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct idi_48_gpio *const idi48gpio = gpiochip_get_data(chip);
+	const unsigned offset = irqd_to_hwirq(data);
+	unsigned i;
+	unsigned mask;
+	unsigned boundary;
+	unsigned long flags;
+
+	for (i = 0; i < 48; i += 8)
+		if (offset < i + 8) {
+			mask = BIT(offset - i);
+			boundary = i / 8;
+
+			idi48gpio->irq_mask[boundary] &= ~mask;
+
+			if (!idi48gpio->irq_mask[boundary]) {
+				idi48gpio->cos_enb &= ~BIT(boundary);
+
+				spin_lock_irqsave(&idi48gpio->lock, flags);
+
+				outb(idi48gpio->cos_enb, idi48gpio->base + 7);
+
+				spin_unlock_irqrestore(&idi48gpio->lock, flags);
+			}
+
+			return;
+		}
+}
+
+static void idi_48_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct idi_48_gpio *const idi48gpio = gpiochip_get_data(chip);
+	const unsigned offset = irqd_to_hwirq(data);
+	unsigned i;
+	unsigned mask;
+	unsigned boundary;
+	unsigned prev_irq_mask;
+	unsigned long flags;
+
+	for (i = 0; i < 48; i += 8)
+		if (offset < i + 8) {
+			mask = BIT(offset - i);
+			boundary = i / 8;
+			prev_irq_mask = idi48gpio->irq_mask[boundary];
+
+			idi48gpio->irq_mask[boundary] |= mask;
+
+			if (!prev_irq_mask) {
+				idi48gpio->cos_enb |= BIT(boundary);
+
+				spin_lock_irqsave(&idi48gpio->lock, flags);
+
+				outb(idi48gpio->cos_enb, idi48gpio->base + 7);
+
+				spin_unlock_irqrestore(&idi48gpio->lock, flags);
+			}
+
+			return;
+		}
+}
+
+static int idi_48_irq_set_type(struct irq_data *data, unsigned flow_type)
+{
+	/* The only valid irq types are none and both-edges */
+	if (flow_type != IRQ_TYPE_NONE &&
+		(flow_type & IRQ_TYPE_EDGE_BOTH) != IRQ_TYPE_EDGE_BOTH)
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct irq_chip idi_48_irqchip = {
+	.name = "104-idi-48",
+	.irq_ack = idi_48_irq_ack,
+	.irq_mask = idi_48_irq_mask,
+	.irq_unmask = idi_48_irq_unmask,
+	.irq_set_type = idi_48_irq_set_type
+};
+
+static irqreturn_t idi_48_irq_handler(int irq, void *dev_id)
+{
+	struct idi_48_gpio *const idi48gpio = dev_id;
+	unsigned long cos_status;
+	unsigned long boundary;
+	unsigned long irq_mask;
+	unsigned long bit_num;
+	unsigned long gpio;
+	struct gpio_chip *const chip = &idi48gpio->chip;
+
+	spin_lock(&idi48gpio->ack_lock);
+
+	spin_lock(&idi48gpio->lock);
+
+	cos_status = inb(idi48gpio->base + 7);
+
+	spin_unlock(&idi48gpio->lock);
+
+	/* IRQ Status (bit 6) is active low (0 = IRQ generated by device) */
+	if (cos_status & BIT(6)) {
+		spin_unlock(&idi48gpio->ack_lock);
+		return IRQ_NONE;
+	}
+
+	/* Bit 0-5 indicate which Change-Of-State boundary triggered the IRQ */
+	cos_status &= 0x3F;
+
+	for_each_set_bit(boundary, &cos_status, 6) {
+		irq_mask = idi48gpio->irq_mask[boundary];
+
+		for_each_set_bit(bit_num, &irq_mask, 8) {
+			gpio = bit_num + boundary * 8;
+
+			generic_handle_irq(irq_find_mapping(chip->irqdomain,
+				gpio));
+		}
+	}
+
+	spin_unlock(&idi48gpio->ack_lock);
+
+	return IRQ_HANDLED;
+}
+
+static int __init idi_48_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct idi_48_gpio *idi48gpio;
+	const unsigned base = idi_48_base;
+	const unsigned extent = 8;
+	const char *const name = dev_name(dev);
+	int err;
+	const unsigned irq = idi_48_irq;
+
+	idi48gpio = devm_kzalloc(dev, sizeof(*idi48gpio), GFP_KERNEL);
+	if (!idi48gpio)
+		return -ENOMEM;
+
+	if (!request_region(base, extent, name)) {
+		dev_err(dev, "Unable to lock %s port addresses (0x%X-0x%X)\n",
+			name, base, base + extent);
+		err = -EBUSY;
+		goto err_lock_io_port;
+	}
+
+	idi48gpio->chip.label = name;
+	idi48gpio->chip.parent = dev;
+	idi48gpio->chip.owner = THIS_MODULE;
+	idi48gpio->chip.base = -1;
+	idi48gpio->chip.ngpio = 48;
+	idi48gpio->chip.get_direction = idi_48_gpio_get_direction;
+	idi48gpio->chip.direction_input = idi_48_gpio_direction_input;
+	idi48gpio->chip.get = idi_48_gpio_get;
+	idi48gpio->base = base;
+	idi48gpio->extent = extent;
+	idi48gpio->irq = irq;
+
+	spin_lock_init(&idi48gpio->lock);
+
+	dev_set_drvdata(dev, idi48gpio);
+
+	err = gpiochip_add_data(&idi48gpio->chip, idi48gpio);
+	if (err) {
+		dev_err(dev, "GPIO registering failed (%d)\n", err);
+		goto err_gpio_register;
+	}
+
+	/* Disable IRQ by default */
+	outb(0, base + 7);
+	inb(base + 7);
+
+	err = gpiochip_irqchip_add(&idi48gpio->chip, &idi_48_irqchip, 0,
+		handle_edge_irq, IRQ_TYPE_NONE);
+	if (err) {
+		dev_err(dev, "Could not add irqchip (%d)\n", err);
+		goto err_gpiochip_irqchip_add;
+	}
+
+	err = request_irq(irq, idi_48_irq_handler, 0, name, idi48gpio);
+	if (err) {
+		dev_err(dev, "IRQ handler registering failed (%d)\n", err);
+		goto err_request_irq;
+	}
+
+	return 0;
+
+err_request_irq:
+err_gpiochip_irqchip_add:
+	gpiochip_remove(&idi48gpio->chip);
+err_gpio_register:
+	release_region(base, extent);
+err_lock_io_port:
+	return err;
+}
+
+static int idi_48_remove(struct platform_device *pdev)
+{
+	struct idi_48_gpio *const idi48gpio = platform_get_drvdata(pdev);
+
+	free_irq(idi48gpio->irq, idi48gpio);
+	gpiochip_remove(&idi48gpio->chip);
+	release_region(idi48gpio->base, idi48gpio->extent);
+
+	return 0;
+}
+
+static struct platform_device *idi_48_device;
+
+static struct platform_driver idi_48_driver = {
+	.driver = {
+		.name = "104-idi-48"
+	},
+	.remove = idi_48_remove
+};
+
+static void __exit idi_48_exit(void)
+{
+	platform_device_unregister(idi_48_device);
+	platform_driver_unregister(&idi_48_driver);
+}
+
+static int __init idi_48_init(void)
+{
+	int err;
+
+	idi_48_device = platform_device_alloc(idi_48_driver.driver.name, -1);
+	if (!idi_48_device)
+		return -ENOMEM;
+
+	err = platform_device_add(idi_48_device);
+	if (err)
+		goto err_platform_device;
+
+	err = platform_driver_probe(&idi_48_driver, idi_48_probe);
+	if (err)
+		goto err_platform_driver;
+
+	return 0;
+
+err_platform_driver:
+	platform_device_del(idi_48_device);
+err_platform_device:
+	platform_device_put(idi_48_device);
+	return err;
+}
+
+module_init(idi_48_init);
+module_exit(idi_48_exit);
+
+MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
+MODULE_DESCRIPTION("ACCES 104-IDI-48 GPIO driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-104-idio-16.c b/drivers/gpio/gpio-104-idio-16.c
index 5400d7d4d8fd..4d69b50b2d84 100644
--- a/drivers/gpio/gpio-104-idio-16.c
+++ b/drivers/gpio/gpio-104-idio-16.c
@@ -11,11 +11,14 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  */
+#include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/gpio/driver.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
+#include <linux/interrupt.h>
+#include <linux/irqdesc.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
@@ -25,20 +28,27 @@
 static unsigned idio_16_base;
 module_param(idio_16_base, uint, 0);
 MODULE_PARM_DESC(idio_16_base, "ACCES 104-IDIO-16 base address");
+static unsigned idio_16_irq;
+module_param(idio_16_irq, uint, 0);
+MODULE_PARM_DESC(idio_16_irq, "ACCES 104-IDIO-16 interrupt line number");
 
 /**
  * struct idio_16_gpio - GPIO device private data structure
  * @chip:	instance of the gpio_chip
- * @lock:	synchronization lock to prevent gpio_set race conditions
+ * @lock:	synchronization lock to prevent I/O race conditions
+ * @irq_mask:	I/O bits affected by interrupts
  * @base:	base port address of the GPIO device
  * @extent:	extent of port address region of the GPIO device
+ * @irq:	Interrupt line number
  * @out_state:	output bits state
  */
 struct idio_16_gpio {
 	struct gpio_chip chip;
 	spinlock_t lock;
+	unsigned long irq_mask;
 	unsigned base;
 	unsigned extent;
+	unsigned irq;
 	unsigned out_state;
 };
 
@@ -62,29 +72,24 @@ static int idio_16_gpio_direction_output(struct gpio_chip *chip,
 	return 0;
 }
 
-static struct idio_16_gpio *to_idio16gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct idio_16_gpio, chip);
-}
-
 static int idio_16_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct idio_16_gpio *const idio16gpio = to_idio16gpio(chip);
-	const unsigned BIT_MASK = 1U << (offset-16);
+	struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip);
+	const unsigned mask = BIT(offset-16);
 
 	if (offset < 16)
 		return -EINVAL;
 
 	if (offset < 24)
-		return !!(inb(idio16gpio->base + 1) & BIT_MASK);
+		return !!(inb(idio16gpio->base + 1) & mask);
 
-	return !!(inb(idio16gpio->base + 5) & (BIT_MASK>>8));
+	return !!(inb(idio16gpio->base + 5) & (mask>>8));
 }
 
 static void idio_16_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct idio_16_gpio *const idio16gpio = to_idio16gpio(chip);
-	const unsigned BIT_MASK = 1U << offset;
+	struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip);
+	const unsigned mask = BIT(offset);
 	unsigned long flags;
 
 	if (offset > 15)
@@ -93,9 +98,9 @@ static void idio_16_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 	spin_lock_irqsave(&idio16gpio->lock, flags);
 
 	if (value)
-		idio16gpio->out_state |= BIT_MASK;
+		idio16gpio->out_state |= mask;
 	else
-		idio16gpio->out_state &= ~BIT_MASK;
+		idio16gpio->out_state &= ~mask;
 
 	if (offset > 7)
 		outb(idio16gpio->out_state >> 8, idio16gpio->base + 4);
@@ -105,29 +110,106 @@ static void idio_16_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 	spin_unlock_irqrestore(&idio16gpio->lock, flags);
 }
 
+static void idio_16_irq_ack(struct irq_data *data)
+{
+}
+
+static void idio_16_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip);
+	const unsigned long mask = BIT(irqd_to_hwirq(data));
+	unsigned long flags;
+
+	idio16gpio->irq_mask &= ~mask;
+
+	if (!idio16gpio->irq_mask) {
+		spin_lock_irqsave(&idio16gpio->lock, flags);
+
+		outb(0, idio16gpio->base + 2);
+
+		spin_unlock_irqrestore(&idio16gpio->lock, flags);
+	}
+}
+
+static void idio_16_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip);
+	const unsigned long mask = BIT(irqd_to_hwirq(data));
+	const unsigned long prev_irq_mask = idio16gpio->irq_mask;
+	unsigned long flags;
+
+	idio16gpio->irq_mask |= mask;
+
+	if (!prev_irq_mask) {
+		spin_lock_irqsave(&idio16gpio->lock, flags);
+
+		inb(idio16gpio->base + 2);
+
+		spin_unlock_irqrestore(&idio16gpio->lock, flags);
+	}
+}
+
+static int idio_16_irq_set_type(struct irq_data *data, unsigned flow_type)
+{
+	/* The only valid irq types are none and both-edges */
+	if (flow_type != IRQ_TYPE_NONE &&
+		(flow_type & IRQ_TYPE_EDGE_BOTH) != IRQ_TYPE_EDGE_BOTH)
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct irq_chip idio_16_irqchip = {
+	.name = "104-idio-16",
+	.irq_ack = idio_16_irq_ack,
+	.irq_mask = idio_16_irq_mask,
+	.irq_unmask = idio_16_irq_unmask,
+	.irq_set_type = idio_16_irq_set_type
+};
+
+static irqreturn_t idio_16_irq_handler(int irq, void *dev_id)
+{
+	struct idio_16_gpio *const idio16gpio = dev_id;
+	struct gpio_chip *const chip = &idio16gpio->chip;
+	int gpio;
+
+	for_each_set_bit(gpio, &idio16gpio->irq_mask, chip->ngpio)
+		generic_handle_irq(irq_find_mapping(chip->irqdomain, gpio));
+
+	spin_lock(&idio16gpio->lock);
+
+	outb(0, idio16gpio->base + 1);
+
+	spin_unlock(&idio16gpio->lock);
+
+	return IRQ_HANDLED;
+}
+
 static int __init idio_16_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct idio_16_gpio *idio16gpio;
+	const unsigned base = idio_16_base;
+	const unsigned extent = 8;
+	const char *const name = dev_name(dev);
 	int err;
-
-	const unsigned BASE = idio_16_base;
-	const unsigned EXTENT = 8;
-	const char *const NAME = dev_name(dev);
+	const unsigned irq = idio_16_irq;
 
 	idio16gpio = devm_kzalloc(dev, sizeof(*idio16gpio), GFP_KERNEL);
 	if (!idio16gpio)
 		return -ENOMEM;
 
-	if (!request_region(BASE, EXTENT, NAME)) {
+	if (!request_region(base, extent, name)) {
 		dev_err(dev, "Unable to lock %s port addresses (0x%X-0x%X)\n",
-			NAME, BASE, BASE + EXTENT);
+			name, base, base + extent);
 		err = -EBUSY;
 		goto err_lock_io_port;
 	}
 
-	idio16gpio->chip.label = NAME;
-	idio16gpio->chip.dev = dev;
+	idio16gpio->chip.label = name;
+	idio16gpio->chip.parent = dev;
 	idio16gpio->chip.owner = THIS_MODULE;
 	idio16gpio->chip.base = -1;
 	idio16gpio->chip.ngpio = 32;
@@ -136,24 +218,45 @@ static int __init idio_16_probe(struct platform_device *pdev)
 	idio16gpio->chip.direction_output = idio_16_gpio_direction_output;
 	idio16gpio->chip.get = idio_16_gpio_get;
 	idio16gpio->chip.set = idio_16_gpio_set;
-	idio16gpio->base = BASE;
-	idio16gpio->extent = EXTENT;
+	idio16gpio->base = base;
+	idio16gpio->extent = extent;
+	idio16gpio->irq = irq;
 	idio16gpio->out_state = 0xFFFF;
 
 	spin_lock_init(&idio16gpio->lock);
 
 	dev_set_drvdata(dev, idio16gpio);
 
-	err = gpiochip_add(&idio16gpio->chip);
+	err = gpiochip_add_data(&idio16gpio->chip, idio16gpio);
 	if (err) {
 		dev_err(dev, "GPIO registering failed (%d)\n", err);
 		goto err_gpio_register;
 	}
 
+	/* Disable IRQ by default */
+	outb(0, base + 2);
+	outb(0, base + 1);
+
+	err = gpiochip_irqchip_add(&idio16gpio->chip, &idio_16_irqchip, 0,
+		handle_edge_irq, IRQ_TYPE_NONE);
+	if (err) {
+		dev_err(dev, "Could not add irqchip (%d)\n", err);
+		goto err_gpiochip_irqchip_add;
+	}
+
+	err = request_irq(irq, idio_16_irq_handler, 0, name, idio16gpio);
+	if (err) {
+		dev_err(dev, "IRQ handler registering failed (%d)\n", err);
+		goto err_request_irq;
+	}
+
 	return 0;
 
+err_request_irq:
+err_gpiochip_irqchip_add:
+	gpiochip_remove(&idio16gpio->chip);
 err_gpio_register:
-	release_region(BASE, EXTENT);
+	release_region(base, extent);
 err_lock_io_port:
 	return err;
 }
@@ -162,6 +265,7 @@ static int idio_16_remove(struct platform_device *pdev)
 {
 	struct idio_16_gpio *const idio16gpio = platform_get_drvdata(pdev);
 
+	free_irq(idio16gpio->irq, idio16gpio);
 	gpiochip_remove(&idio16gpio->chip);
 	release_region(idio16gpio->base, idio16gpio->extent);
 
diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c
index 60172f835d15..c81224ff2dca 100644
--- a/drivers/gpio/gpio-74x164.c
+++ b/drivers/gpio/gpio-74x164.c
@@ -20,56 +20,34 @@
 #define GEN_74X164_NUMBER_GPIOS	8
 
 struct gen_74x164_chip {
-	u8			*buffer;
 	struct gpio_chip	gpio_chip;
 	struct mutex		lock;
 	u32			registers;
-};
-
-static struct gen_74x164_chip *gpio_to_74x164_chip(struct gpio_chip *gc)
-{
-	return container_of(gc, struct gen_74x164_chip, gpio_chip);
-}
-
-static int __gen_74x164_write_config(struct gen_74x164_chip *chip)
-{
-	struct spi_device *spi = to_spi_device(chip->gpio_chip.dev);
-	struct spi_message message;
-	struct spi_transfer *msg_buf;
-	int i, ret = 0;
-
-	msg_buf = kzalloc(chip->registers * sizeof(struct spi_transfer),
-			GFP_KERNEL);
-	if (!msg_buf)
-		return -ENOMEM;
-
-	spi_message_init(&message);
-
 	/*
 	 * Since the registers are chained, every byte sent will make
 	 * the previous byte shift to the next register in the
-	 * chain. Thus, the first byte send will end up in the last
+	 * chain. Thus, the first byte sent will end up in the last
 	 * register at the end of the transfer. So, to have a logical
-	 * numbering, send the bytes in reverse order so that the last
-	 * byte of the buffer will end up in the last register.
+	 * numbering, store the bytes in reverse order.
 	 */
-	for (i = chip->registers - 1; i >= 0; i--) {
-		msg_buf[i].tx_buf = chip->buffer + i;
-		msg_buf[i].len = sizeof(u8);
-		spi_message_add_tail(msg_buf + i, &message);
-	}
-
-	ret = spi_sync(spi, &message);
+	u8			buffer[0];
+};
 
-	kfree(msg_buf);
+static int __gen_74x164_write_config(struct gen_74x164_chip *chip)
+{
+	struct spi_transfer xfer = {
+		.tx_buf = chip->buffer,
+		.len = chip->registers,
+	};
 
-	return ret;
+	return spi_sync_transfer(to_spi_device(chip->gpio_chip.parent),
+				 &xfer, 1);
 }
 
 static int gen_74x164_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct gen_74x164_chip *chip = gpio_to_74x164_chip(gc);
-	u8 bank = offset / 8;
+	struct gen_74x164_chip *chip = gpiochip_get_data(gc);
+	u8 bank = chip->registers - 1 - offset / 8;
 	u8 pin = offset % 8;
 	int ret;
 
@@ -83,8 +61,8 @@ static int gen_74x164_get_value(struct gpio_chip *gc, unsigned offset)
 static void gen_74x164_set_value(struct gpio_chip *gc,
 		unsigned offset, int val)
 {
-	struct gen_74x164_chip *chip = gpio_to_74x164_chip(gc);
-	u8 bank = offset / 8;
+	struct gen_74x164_chip *chip = gpiochip_get_data(gc);
+	u8 bank = chip->registers - 1 - offset / 8;
 	u8 pin = offset % 8;
 
 	mutex_lock(&chip->lock);
@@ -107,6 +85,7 @@ static int gen_74x164_direction_output(struct gpio_chip *gc,
 static int gen_74x164_probe(struct spi_device *spi)
 {
 	struct gen_74x164_chip *chip;
+	u32 nregs;
 	int ret;
 
 	/*
@@ -118,7 +97,14 @@ static int gen_74x164_probe(struct spi_device *spi)
 	if (ret < 0)
 		return ret;
 
-	chip = devm_kzalloc(&spi->dev, sizeof(*chip), GFP_KERNEL);
+	if (of_property_read_u32(spi->dev.of_node, "registers-number",
+				 &nregs)) {
+		dev_err(&spi->dev,
+			"Missing registers-number property in the DT.\n");
+		return -EINVAL;
+	}
+
+	chip = devm_kzalloc(&spi->dev, sizeof(*chip) + nregs, GFP_KERNEL);
 	if (!chip)
 		return -ENOMEM;
 
@@ -130,20 +116,11 @@ static int gen_74x164_probe(struct spi_device *spi)
 	chip->gpio_chip.set = gen_74x164_set_value;
 	chip->gpio_chip.base = -1;
 
-	if (of_property_read_u32(spi->dev.of_node, "registers-number",
-				 &chip->registers)) {
-		dev_err(&spi->dev,
-			"Missing registers-number property in the DT.\n");
-		return -EINVAL;
-	}
-
+	chip->registers = nregs;
 	chip->gpio_chip.ngpio = GEN_74X164_NUMBER_GPIOS * chip->registers;
-	chip->buffer = devm_kzalloc(&spi->dev, chip->registers, GFP_KERNEL);
-	if (!chip->buffer)
-		return -ENOMEM;
 
 	chip->gpio_chip.can_sleep = true;
-	chip->gpio_chip.dev = &spi->dev;
+	chip->gpio_chip.parent = &spi->dev;
 	chip->gpio_chip.owner = THIS_MODULE;
 
 	mutex_init(&chip->lock);
@@ -154,7 +131,7 @@ static int gen_74x164_probe(struct spi_device *spi)
 		goto exit_destroy;
 	}
 
-	ret = gpiochip_add(&chip->gpio_chip);
+	ret = gpiochip_add_data(&chip->gpio_chip, chip);
 	if (!ret)
 		return 0;
 
diff --git a/drivers/gpio/gpio-74xx-mmio.c b/drivers/gpio/gpio-74xx-mmio.c
index 6b186829087c..372b0e01adc6 100644
--- a/drivers/gpio/gpio-74xx-mmio.c
+++ b/drivers/gpio/gpio-74xx-mmio.c
@@ -10,10 +10,9 @@
  */
 
 #include <linux/err.h>
-#include <linux/gpio.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 
 #define MMIO_74XX_DIR_IN	(0 << 8)
@@ -21,7 +20,7 @@
 #define MMIO_74XX_BIT_CNT(x)	((x) & 0xff)
 
 struct mmio_74xx_gpio_priv {
-	struct bgpio_chip	bgc;
+	struct gpio_chip	gc;
 	unsigned		flags;
 };
 
@@ -78,30 +77,23 @@ static const struct of_device_id mmio_74xx_gpio_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, mmio_74xx_gpio_ids);
 
-static inline struct mmio_74xx_gpio_priv *to_74xx_gpio(struct gpio_chip *gc)
-{
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return container_of(bgc, struct mmio_74xx_gpio_priv, bgc);
-}
-
 static int mmio_74xx_get_direction(struct gpio_chip *gc, unsigned offset)
 {
-	struct mmio_74xx_gpio_priv *priv = to_74xx_gpio(gc);
+	struct mmio_74xx_gpio_priv *priv = gpiochip_get_data(gc);
 
-	return (priv->flags & MMIO_74XX_DIR_OUT) ? GPIOF_DIR_OUT : GPIOF_DIR_IN;
+	return !(priv->flags & MMIO_74XX_DIR_OUT);
 }
 
 static int mmio_74xx_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct mmio_74xx_gpio_priv *priv = to_74xx_gpio(gc);
+	struct mmio_74xx_gpio_priv *priv = gpiochip_get_data(gc);
 
 	return (priv->flags & MMIO_74XX_DIR_OUT) ? -ENOTSUPP : 0;
 }
 
 static int mmio_74xx_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct mmio_74xx_gpio_priv *priv = to_74xx_gpio(gc);
+	struct mmio_74xx_gpio_priv *priv = gpiochip_get_data(gc);
 
 	if (priv->flags & MMIO_74XX_DIR_OUT) {
 		gc->set(gc, gpio, val);
@@ -134,28 +126,29 @@ static int mmio_74xx_gpio_probe(struct platform_device *pdev)
 
 	priv->flags = (uintptr_t) of_id->data;
 
-	err = bgpio_init(&priv->bgc, &pdev->dev,
+	err = bgpio_init(&priv->gc, &pdev->dev,
 			 DIV_ROUND_UP(MMIO_74XX_BIT_CNT(priv->flags), 8),
 			 dat, NULL, NULL, NULL, NULL, 0);
 	if (err)
 		return err;
 
-	priv->bgc.gc.direction_input = mmio_74xx_dir_in;
-	priv->bgc.gc.direction_output = mmio_74xx_dir_out;
-	priv->bgc.gc.get_direction = mmio_74xx_get_direction;
-	priv->bgc.gc.ngpio = MMIO_74XX_BIT_CNT(priv->flags);
-	priv->bgc.gc.owner = THIS_MODULE;
+	priv->gc.direction_input = mmio_74xx_dir_in;
+	priv->gc.direction_output = mmio_74xx_dir_out;
+	priv->gc.get_direction = mmio_74xx_get_direction;
+	priv->gc.ngpio = MMIO_74XX_BIT_CNT(priv->flags);
+	priv->gc.owner = THIS_MODULE;
 
 	platform_set_drvdata(pdev, priv);
 
-	return gpiochip_add(&priv->bgc.gc);
+	return gpiochip_add_data(&priv->gc, priv);
 }
 
 static int mmio_74xx_gpio_remove(struct platform_device *pdev)
 {
 	struct mmio_74xx_gpio_priv *priv = platform_get_drvdata(pdev);
 
-	return bgpio_remove(&priv->bgc);
+	gpiochip_remove(&priv->gc);
+	return 0;
 }
 
 static struct platform_driver mmio_74xx_gpio_driver = {
diff --git a/drivers/gpio/gpio-adnp.c b/drivers/gpio/gpio-adnp.c
index d3d0a90fe542..fb5b47b69f14 100644
--- a/drivers/gpio/gpio-adnp.c
+++ b/drivers/gpio/gpio-adnp.c
@@ -36,18 +36,13 @@ struct adnp {
 	u8 *irq_low;
 };
 
-static inline struct adnp *to_adnp(struct gpio_chip *chip)
-{
-	return container_of(chip, struct adnp, gpio);
-}
-
 static int adnp_read(struct adnp *adnp, unsigned offset, uint8_t *value)
 {
 	int err;
 
 	err = i2c_smbus_read_byte_data(adnp->client, offset);
 	if (err < 0) {
-		dev_err(adnp->gpio.dev, "%s failed: %d\n",
+		dev_err(adnp->gpio.parent, "%s failed: %d\n",
 			"i2c_smbus_read_byte_data()", err);
 		return err;
 	}
@@ -62,7 +57,7 @@ static int adnp_write(struct adnp *adnp, unsigned offset, uint8_t value)
 
 	err = i2c_smbus_write_byte_data(adnp->client, offset, value);
 	if (err < 0) {
-		dev_err(adnp->gpio.dev, "%s failed: %d\n",
+		dev_err(adnp->gpio.parent, "%s failed: %d\n",
 			"i2c_smbus_write_byte_data()", err);
 		return err;
 	}
@@ -72,7 +67,7 @@ static int adnp_write(struct adnp *adnp, unsigned offset, uint8_t value)
 
 static int adnp_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct adnp *adnp = to_adnp(chip);
+	struct adnp *adnp = gpiochip_get_data(chip);
 	unsigned int reg = offset >> adnp->reg_shift;
 	unsigned int pos = offset & 7;
 	u8 value;
@@ -106,7 +101,7 @@ static void __adnp_gpio_set(struct adnp *adnp, unsigned offset, int value)
 
 static void adnp_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct adnp *adnp = to_adnp(chip);
+	struct adnp *adnp = gpiochip_get_data(chip);
 
 	mutex_lock(&adnp->i2c_lock);
 	__adnp_gpio_set(adnp, offset, value);
@@ -115,7 +110,7 @@ static void adnp_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int adnp_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct adnp *adnp = to_adnp(chip);
+	struct adnp *adnp = gpiochip_get_data(chip);
 	unsigned int reg = offset >> adnp->reg_shift;
 	unsigned int pos = offset & 7;
 	u8 value;
@@ -150,7 +145,7 @@ out:
 static int adnp_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 				      int value)
 {
-	struct adnp *adnp = to_adnp(chip);
+	struct adnp *adnp = gpiochip_get_data(chip);
 	unsigned int reg = offset >> adnp->reg_shift;
 	unsigned int pos = offset & 7;
 	int err;
@@ -187,7 +182,7 @@ out:
 
 static void adnp_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct adnp *adnp = to_adnp(chip);
+	struct adnp *adnp = gpiochip_get_data(chip);
 	unsigned int num_regs = 1 << adnp->reg_shift, i, j;
 	int err;
 
@@ -266,11 +261,11 @@ static int adnp_gpio_setup(struct adnp *adnp, unsigned int num_gpios)
 	chip->base = -1;
 	chip->ngpio = num_gpios;
 	chip->label = adnp->client->name;
-	chip->dev = &adnp->client->dev;
-	chip->of_node = chip->dev->of_node;
+	chip->parent = &adnp->client->dev;
+	chip->of_node = chip->parent->of_node;
 	chip->owner = THIS_MODULE;
 
-	err = gpiochip_add(chip);
+	err = gpiochip_add_data(chip, adnp);
 	if (err)
 		return err;
 
@@ -340,7 +335,7 @@ static irqreturn_t adnp_irq(int irq, void *data)
 static void adnp_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct adnp *adnp = to_adnp(gc);
+	struct adnp *adnp = gpiochip_get_data(gc);
 	unsigned int reg = d->hwirq >> adnp->reg_shift;
 	unsigned int pos = d->hwirq & 7;
 
@@ -350,7 +345,7 @@ static void adnp_irq_mask(struct irq_data *d)
 static void adnp_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct adnp *adnp = to_adnp(gc);
+	struct adnp *adnp = gpiochip_get_data(gc);
 	unsigned int reg = d->hwirq >> adnp->reg_shift;
 	unsigned int pos = d->hwirq & 7;
 
@@ -360,7 +355,7 @@ static void adnp_irq_unmask(struct irq_data *d)
 static int adnp_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct adnp *adnp = to_adnp(gc);
+	struct adnp *adnp = gpiochip_get_data(gc);
 	unsigned int reg = d->hwirq >> adnp->reg_shift;
 	unsigned int pos = d->hwirq & 7;
 
@@ -390,7 +385,7 @@ static int adnp_irq_set_type(struct irq_data *d, unsigned int type)
 static void adnp_irq_bus_lock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct adnp *adnp = to_adnp(gc);
+	struct adnp *adnp = gpiochip_get_data(gc);
 
 	mutex_lock(&adnp->irq_lock);
 }
@@ -398,7 +393,7 @@ static void adnp_irq_bus_lock(struct irq_data *d)
 static void adnp_irq_bus_unlock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct adnp *adnp = to_adnp(gc);
+	struct adnp *adnp = gpiochip_get_data(gc);
 	unsigned int num_regs = 1 << adnp->reg_shift, i;
 
 	mutex_lock(&adnp->i2c_lock);
@@ -435,7 +430,8 @@ static int adnp_irq_setup(struct adnp *adnp)
 	 * is chosen to match the register layout of the hardware in that
 	 * each segment contains the corresponding bits for all interrupts.
 	 */
-	adnp->irq_enable = devm_kzalloc(chip->dev, num_regs * 6, GFP_KERNEL);
+	adnp->irq_enable = devm_kzalloc(chip->parent, num_regs * 6,
+					GFP_KERNEL);
 	if (!adnp->irq_enable)
 		return -ENOMEM;
 
@@ -462,12 +458,12 @@ static int adnp_irq_setup(struct adnp *adnp)
 		adnp->irq_enable[i] = 0x00;
 	}
 
-	err = devm_request_threaded_irq(chip->dev, adnp->client->irq,
+	err = devm_request_threaded_irq(chip->parent, adnp->client->irq,
 					NULL, adnp_irq,
 					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
-					dev_name(chip->dev), adnp);
+					dev_name(chip->parent), adnp);
 	if (err != 0) {
-		dev_err(chip->dev, "can't request IRQ#%d: %d\n",
+		dev_err(chip->parent, "can't request IRQ#%d: %d\n",
 			adnp->client->irq, err);
 		return err;
 	}
@@ -478,7 +474,7 @@ static int adnp_irq_setup(struct adnp *adnp)
 				   handle_simple_irq,
 				   IRQ_TYPE_NONE);
 	if (err) {
-		dev_err(chip->dev,
+		dev_err(chip->parent,
 			"could not connect irqchip to gpiochip\n");
 		return err;
 	}
@@ -547,7 +543,6 @@ MODULE_DEVICE_TABLE(of, adnp_of_match);
 static struct i2c_driver adnp_i2c_driver = {
 	.driver = {
 		.name = "gpio-adnp",
-		.owner = THIS_MODULE,
 		.of_match_table = adnp_of_match,
 	},
 	.probe = adnp_i2c_probe,
diff --git a/drivers/gpio/gpio-adp5520.c b/drivers/gpio/gpio-adp5520.c
index caff711ca5a9..4fa7ff1fec9a 100644
--- a/drivers/gpio/gpio-adp5520.c
+++ b/drivers/gpio/gpio-adp5520.c
@@ -27,7 +27,7 @@ static int adp5520_gpio_get_value(struct gpio_chip *chip, unsigned off)
 	struct adp5520_gpio *dev;
 	uint8_t reg_val;
 
-	dev = container_of(chip, struct adp5520_gpio, gpio_chip);
+	dev = gpiochip_get_data(chip);
 
 	/*
 	 * There are dedicated registers for GPIO IN/OUT.
@@ -46,7 +46,7 @@ static void adp5520_gpio_set_value(struct gpio_chip *chip,
 		unsigned off, int val)
 {
 	struct adp5520_gpio *dev;
-	dev = container_of(chip, struct adp5520_gpio, gpio_chip);
+	dev = gpiochip_get_data(chip);
 
 	if (val)
 		adp5520_set_bits(dev->master, ADP5520_GPIO_OUT, dev->lut[off]);
@@ -57,7 +57,7 @@ static void adp5520_gpio_set_value(struct gpio_chip *chip,
 static int adp5520_gpio_direction_input(struct gpio_chip *chip, unsigned off)
 {
 	struct adp5520_gpio *dev;
-	dev = container_of(chip, struct adp5520_gpio, gpio_chip);
+	dev = gpiochip_get_data(chip);
 
 	clear_bit(off, &dev->output);
 
@@ -70,7 +70,7 @@ static int adp5520_gpio_direction_output(struct gpio_chip *chip,
 {
 	struct adp5520_gpio *dev;
 	int ret = 0;
-	dev = container_of(chip, struct adp5520_gpio, gpio_chip);
+	dev = gpiochip_get_data(chip);
 
 	set_bit(off, &dev->output);
 
@@ -153,7 +153,7 @@ static int adp5520_gpio_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	ret = gpiochip_add(&dev->gpio_chip);
+	ret = gpiochip_add_data(&dev->gpio_chip, dev);
 	if (ret)
 		goto err;
 
diff --git a/drivers/gpio/gpio-adp5588.c b/drivers/gpio/gpio-adp5588.c
index 984186ee58a0..19a0eba1e942 100644
--- a/drivers/gpio/gpio-adp5588.c
+++ b/drivers/gpio/gpio-adp5588.c
@@ -65,8 +65,7 @@ static int adp5588_gpio_write(struct i2c_client *client, u8 reg, u8 val)
 
 static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off)
 {
-	struct adp5588_gpio *dev =
-	    container_of(chip, struct adp5588_gpio, gpio_chip);
+	struct adp5588_gpio *dev = gpiochip_get_data(chip);
 	unsigned bank = ADP5588_BANK(off);
 	unsigned bit = ADP5588_BIT(off);
 	int val;
@@ -87,8 +86,7 @@ static void adp5588_gpio_set_value(struct gpio_chip *chip,
 				   unsigned off, int val)
 {
 	unsigned bank, bit;
-	struct adp5588_gpio *dev =
-	    container_of(chip, struct adp5588_gpio, gpio_chip);
+	struct adp5588_gpio *dev = gpiochip_get_data(chip);
 
 	bank = ADP5588_BANK(off);
 	bit = ADP5588_BIT(off);
@@ -108,8 +106,7 @@ static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off)
 {
 	int ret;
 	unsigned bank;
-	struct adp5588_gpio *dev =
-	    container_of(chip, struct adp5588_gpio, gpio_chip);
+	struct adp5588_gpio *dev = gpiochip_get_data(chip);
 
 	bank = ADP5588_BANK(off);
 
@@ -126,8 +123,7 @@ static int adp5588_gpio_direction_output(struct gpio_chip *chip,
 {
 	int ret;
 	unsigned bank, bit;
-	struct adp5588_gpio *dev =
-	    container_of(chip, struct adp5588_gpio, gpio_chip);
+	struct adp5588_gpio *dev = gpiochip_get_data(chip);
 
 	bank = ADP5588_BANK(off);
 	bit = ADP5588_BIT(off);
@@ -152,8 +148,8 @@ static int adp5588_gpio_direction_output(struct gpio_chip *chip,
 #ifdef CONFIG_GPIO_ADP5588_IRQ
 static int adp5588_gpio_to_irq(struct gpio_chip *chip, unsigned off)
 {
-	struct adp5588_gpio *dev =
-		container_of(chip, struct adp5588_gpio, gpio_chip);
+	struct adp5588_gpio *dev = gpiochip_get_data(chip);
+
 	return dev->irq_base + off;
 }
 
@@ -418,7 +414,7 @@ static int adp5588_gpio_probe(struct i2c_client *client,
 		}
 	}
 
-	ret = gpiochip_add(&dev->gpio_chip);
+	ret = gpiochip_add_data(&dev->gpio_chip, dev);
 	if (ret)
 		goto err_irq;
 
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 3e6661bab54a..2aeaebd1c6e7 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -42,11 +42,6 @@ struct altera_gpio_chip {
 	int mapped_irq;
 };
 
-static struct altera_gpio_chip *to_altera(struct gpio_chip *gc)
-{
-	return container_of(gc, struct altera_gpio_chip, mmchip.gc);
-}
-
 static void altera_gpio_irq_unmask(struct irq_data *d)
 {
 	struct altera_gpio_chip *altera_gc;
@@ -54,7 +49,7 @@ static void altera_gpio_irq_unmask(struct irq_data *d)
 	unsigned long flags;
 	u32 intmask;
 
-	altera_gc = to_altera(irq_data_get_irq_chip_data(d));
+	altera_gc = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	mm_gc = &altera_gc->mmchip;
 
 	spin_lock_irqsave(&altera_gc->gpio_lock, flags);
@@ -72,7 +67,7 @@ static void altera_gpio_irq_mask(struct irq_data *d)
 	unsigned long flags;
 	u32 intmask;
 
-	altera_gc = to_altera(irq_data_get_irq_chip_data(d));
+	altera_gc = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	mm_gc = &altera_gc->mmchip;
 
 	spin_lock_irqsave(&altera_gc->gpio_lock, flags);
@@ -92,7 +87,7 @@ static int altera_gpio_irq_set_type(struct irq_data *d,
 {
 	struct altera_gpio_chip *altera_gc;
 
-	altera_gc = to_altera(irq_data_get_irq_chip_data(d));
+	altera_gc = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 
 	if (type == IRQ_TYPE_NONE)
 		return 0;
@@ -145,7 +140,7 @@ static void altera_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 	unsigned int data_reg;
 
 	mm_gc = to_of_mm_gpio_chip(gc);
-	chip = container_of(mm_gc, struct altera_gpio_chip, mmchip);
+	chip = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&chip->gpio_lock, flags);
 	data_reg = readl(mm_gc->regs + ALTERA_GPIO_DATA);
@@ -165,7 +160,7 @@ static int altera_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 	unsigned int gpio_ddr;
 
 	mm_gc = to_of_mm_gpio_chip(gc);
-	chip = container_of(mm_gc, struct altera_gpio_chip, mmchip);
+	chip = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&chip->gpio_lock, flags);
 	/* Set pin as input, assumes software controlled IP */
@@ -186,7 +181,7 @@ static int altera_gpio_direction_output(struct gpio_chip *gc,
 	unsigned int data_reg, gpio_ddr;
 
 	mm_gc = to_of_mm_gpio_chip(gc);
-	chip = container_of(mm_gc, struct altera_gpio_chip, mmchip);
+	chip = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&chip->gpio_lock, flags);
 	/* Sets the GPIO value */
@@ -215,7 +210,7 @@ static void altera_gpio_irq_edge_handler(struct irq_desc *desc)
 	unsigned long status;
 	int i;
 
-	altera_gc = to_altera(irq_desc_get_handler_data(desc));
+	altera_gc = gpiochip_get_data(irq_desc_get_handler_data(desc));
 	chip = irq_desc_get_chip(desc);
 	mm_gc = &altera_gc->mmchip;
 	irqdomain = altera_gc->mmchip.gc.irqdomain;
@@ -244,7 +239,7 @@ static void altera_gpio_irq_leveL_high_handler(struct irq_desc *desc)
 	unsigned long status;
 	int i;
 
-	altera_gc = to_altera(irq_desc_get_handler_data(desc));
+	altera_gc = gpiochip_get_data(irq_desc_get_handler_data(desc));
 	chip = irq_desc_get_chip(desc);
 	mm_gc = &altera_gc->mmchip;
 	irqdomain = altera_gc->mmchip.gc.irqdomain;
@@ -290,9 +285,9 @@ static int altera_gpio_probe(struct platform_device *pdev)
 	altera_gc->mmchip.gc.get		= altera_gpio_get;
 	altera_gc->mmchip.gc.set		= altera_gpio_set;
 	altera_gc->mmchip.gc.owner		= THIS_MODULE;
-	altera_gc->mmchip.gc.dev		= &pdev->dev;
+	altera_gc->mmchip.gc.parent		= &pdev->dev;
 
-	ret = of_mm_gpiochip_add(node, &altera_gc->mmchip);
+	ret = of_mm_gpiochip_add_data(node, &altera_gc->mmchip, altera_gc);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed adding memory mapped gpiochip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-amd8111.c b/drivers/gpio/gpio-amd8111.c
index d00d81928fe8..c7040fffc5b4 100644
--- a/drivers/gpio/gpio-amd8111.c
+++ b/drivers/gpio/gpio-amd8111.c
@@ -75,11 +75,9 @@ struct amd_gpio {
 	u8			orig[32];
 };
 
-#define to_agp(chip)	container_of(chip, struct amd_gpio, chip)
-
 static int amd_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 
 	agp->orig[offset] = ioread8(agp->pm + AMD_REG_GPIO(offset)) &
 		(AMD_GPIO_DEBOUNCE | AMD_GPIO_MODE_MASK | AMD_GPIO_X_MASK);
@@ -91,7 +89,7 @@ static int amd_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void amd_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 
 	dev_dbg(&agp->pdev->dev, "Freed gpio %d, data %x\n", offset, agp->orig[offset]);
 
@@ -100,7 +98,7 @@ static void amd_gpio_free(struct gpio_chip *chip, unsigned offset)
 
 static void amd_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 	u8 temp;
 	unsigned long flags;
 
@@ -115,7 +113,7 @@ static void amd_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int amd_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 	u8 temp;
 
 	temp = ioread8(agp->pm + AMD_REG_GPIO(offset));
@@ -127,7 +125,7 @@ static int amd_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static int amd_gpio_dirout(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 	u8 temp;
 	unsigned long flags;
 
@@ -144,7 +142,7 @@ static int amd_gpio_dirout(struct gpio_chip *chip, unsigned offset, int value)
 
 static int amd_gpio_dirin(struct gpio_chip *chip, unsigned offset)
 {
-	struct amd_gpio *agp = to_agp(chip);
+	struct amd_gpio *agp = gpiochip_get_data(chip);
 	u8 temp;
 	unsigned long flags;
 
@@ -220,12 +218,12 @@ found:
 		goto out;
 	}
 	gp.pdev = pdev;
-	gp.chip.dev = &pdev->dev;
+	gp.chip.parent = &pdev->dev;
 
 	spin_lock_init(&gp.lock);
 
 	printk(KERN_INFO "AMD-8111 GPIO detected\n");
-	err = gpiochip_add(&gp.chip);
+	err = gpiochip_add_data(&gp.chip, &gp);
 	if (err) {
 		printk(KERN_ERR "GPIO registering failed (%d)\n",
 		       err);
diff --git a/drivers/gpio/gpio-amdpt.c b/drivers/gpio/gpio-amdpt.c
index cbbb966d4fc0..c2484046e8e9 100644
--- a/drivers/gpio/gpio-amdpt.c
+++ b/drivers/gpio/gpio-amdpt.c
@@ -31,22 +31,20 @@ struct pt_gpio_chip {
 	spinlock_t               lock;
 };
 
-#define to_pt_gpio(c)	container_of(c, struct pt_gpio_chip, gc)
-
 static int pt_gpio_request(struct gpio_chip *gc, unsigned offset)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 using_pins;
 
-	dev_dbg(gc->dev, "pt_gpio_request offset=%x\n", offset);
+	dev_dbg(gc->parent, "pt_gpio_request offset=%x\n", offset);
 
 	spin_lock_irqsave(&pt_gpio->lock, flags);
 
 	using_pins = readl(pt_gpio->reg_base + PT_SYNC_REG);
 	if (using_pins & BIT(offset)) {
-		dev_warn(gc->dev, "PT GPIO pin %x reconfigured\n",
-			offset);
+		dev_warn(gc->parent, "PT GPIO pin %x reconfigured\n",
+			 offset);
 		spin_unlock_irqrestore(&pt_gpio->lock, flags);
 		return -EINVAL;
 	}
@@ -60,7 +58,7 @@ static int pt_gpio_request(struct gpio_chip *gc, unsigned offset)
 
 static void pt_gpio_free(struct gpio_chip *gc, unsigned offset)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 using_pins;
 
@@ -72,16 +70,16 @@ static void pt_gpio_free(struct gpio_chip *gc, unsigned offset)
 
 	spin_unlock_irqrestore(&pt_gpio->lock, flags);
 
-	dev_dbg(gc->dev, "pt_gpio_free offset=%x\n", offset);
+	dev_dbg(gc->parent, "pt_gpio_free offset=%x\n", offset);
 }
 
 static void pt_gpio_set_value(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 data;
 
-	dev_dbg(gc->dev, "pt_gpio_set_value offset=%x, value=%x\n",
+	dev_dbg(gc->parent, "pt_gpio_set_value offset=%x, value=%x\n",
 		offset, value);
 
 	spin_lock_irqsave(&pt_gpio->lock, flags);
@@ -97,7 +95,7 @@ static void pt_gpio_set_value(struct gpio_chip *gc, unsigned offset, int value)
 
 static int pt_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 data;
 
@@ -116,7 +114,7 @@ static int pt_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 	data >>= offset;
 	data &= 1;
 
-	dev_dbg(gc->dev, "pt_gpio_get_value offset=%x, value=%x\n",
+	dev_dbg(gc->parent, "pt_gpio_get_value offset=%x, value=%x\n",
 		offset, data);
 
 	return data;
@@ -124,11 +122,11 @@ static int pt_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 
 static int pt_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 data;
 
-	dev_dbg(gc->dev, "pt_gpio_dirction_input offset=%x\n", offset);
+	dev_dbg(gc->parent, "pt_gpio_dirction_input offset=%x\n", offset);
 
 	spin_lock_irqsave(&pt_gpio->lock, flags);
 
@@ -144,11 +142,11 @@ static int pt_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 static int pt_gpio_direction_output(struct gpio_chip *gc,
 					unsigned offset, int value)
 {
-	struct pt_gpio_chip *pt_gpio = to_pt_gpio(gc);
+	struct pt_gpio_chip *pt_gpio = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 data;
 
-	dev_dbg(gc->dev, "pt_gpio_direction_output offset=%x, value=%x\n",
+	dev_dbg(gc->parent, "pt_gpio_direction_output offset=%x, value=%x\n",
 		offset, value);
 
 	spin_lock_irqsave(&pt_gpio->lock, flags);
@@ -202,7 +200,7 @@ static int pt_gpio_probe(struct platform_device *pdev)
 
 	pt_gpio->gc.label            = pdev->name;
 	pt_gpio->gc.owner            = THIS_MODULE;
-	pt_gpio->gc.dev              = dev;
+	pt_gpio->gc.parent              = dev;
 	pt_gpio->gc.request          = pt_gpio_request;
 	pt_gpio->gc.free             = pt_gpio_free;
 	pt_gpio->gc.direction_input  = pt_gpio_direction_input;
@@ -214,7 +212,7 @@ static int pt_gpio_probe(struct platform_device *pdev)
 #if defined(CONFIG_OF_GPIO)
 	pt_gpio->gc.of_node          = pdev->dev.of_node;
 #endif
-	ret = gpiochip_add(&pt_gpio->gc);
+	ret = gpiochip_add_data(&pt_gpio->gc, pt_gpio);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register GPIO lib\n");
 		return ret;
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index 624ea5421995..e910c1f41d93 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -28,14 +28,9 @@ struct arizona_gpio {
 	struct gpio_chip gpio_chip;
 };
 
-static inline struct arizona_gpio *to_arizona_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct arizona_gpio, gpio_chip);
-}
-
 static int arizona_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct arizona_gpio *arizona_gpio = to_arizona_gpio(chip);
+	struct arizona_gpio *arizona_gpio = gpiochip_get_data(chip);
 	struct arizona *arizona = arizona_gpio->arizona;
 
 	return regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset,
@@ -44,7 +39,7 @@ static int arizona_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int arizona_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct arizona_gpio *arizona_gpio = to_arizona_gpio(chip);
+	struct arizona_gpio *arizona_gpio = gpiochip_get_data(chip);
 	struct arizona *arizona = arizona_gpio->arizona;
 	unsigned int val;
 	int ret;
@@ -62,7 +57,7 @@ static int arizona_gpio_get(struct gpio_chip *chip, unsigned offset)
 static int arizona_gpio_direction_out(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
-	struct arizona_gpio *arizona_gpio = to_arizona_gpio(chip);
+	struct arizona_gpio *arizona_gpio = gpiochip_get_data(chip);
 	struct arizona *arizona = arizona_gpio->arizona;
 
 	if (value)
@@ -74,7 +69,7 @@ static int arizona_gpio_direction_out(struct gpio_chip *chip,
 
 static void arizona_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct arizona_gpio *arizona_gpio = to_arizona_gpio(chip);
+	struct arizona_gpio *arizona_gpio = gpiochip_get_data(chip);
 	struct arizona *arizona = arizona_gpio->arizona;
 
 	if (value)
@@ -108,7 +103,7 @@ static int arizona_gpio_probe(struct platform_device *pdev)
 
 	arizona_gpio->arizona = arizona;
 	arizona_gpio->gpio_chip = template_chip;
-	arizona_gpio->gpio_chip.dev = &pdev->dev;
+	arizona_gpio->gpio_chip.parent = &pdev->dev;
 #ifdef CONFIG_OF_GPIO
 	arizona_gpio->gpio_chip.of_node = arizona->dev->of_node;
 #endif
@@ -137,7 +132,7 @@ static int arizona_gpio_probe(struct platform_device *pdev)
 	else
 		arizona_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&arizona_gpio->gpio_chip);
+	ret = gpiochip_add_data(&arizona_gpio->gpio_chip, arizona_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n",
 			ret);
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index 5eaea8b812cf..d13dd133a907 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -24,12 +24,10 @@ struct ath79_gpio_ctrl {
 	spinlock_t lock;
 };
 
-#define to_ath79_gpio_ctrl(c) container_of(c, struct ath79_gpio_ctrl, chip)
-
 static void ath79_gpio_set_value(struct gpio_chip *chip,
 				unsigned gpio, int value)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 
 	if (value)
 		__raw_writel(BIT(gpio), ctrl->base + AR71XX_GPIO_REG_SET);
@@ -39,7 +37,7 @@ static void ath79_gpio_set_value(struct gpio_chip *chip,
 
 static int ath79_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 
 	return (__raw_readl(ctrl->base + AR71XX_GPIO_REG_IN) >> gpio) & 1;
 }
@@ -47,7 +45,7 @@ static int ath79_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 static int ath79_gpio_direction_input(struct gpio_chip *chip,
 				       unsigned offset)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctrl->lock, flags);
@@ -64,7 +62,7 @@ static int ath79_gpio_direction_input(struct gpio_chip *chip,
 static int ath79_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset, int value)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctrl->lock, flags);
@@ -85,7 +83,7 @@ static int ath79_gpio_direction_output(struct gpio_chip *chip,
 
 static int ar934x_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctrl->lock, flags);
@@ -102,7 +100,7 @@ static int ar934x_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int ar934x_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 					int value)
 {
-	struct ath79_gpio_ctrl *ctrl = to_ath79_gpio_ctrl(chip);
+	struct ath79_gpio_ctrl *ctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctrl->lock, flags);
@@ -138,7 +136,7 @@ static const struct of_device_id ath79_gpio_of_match[] = {
 
 static int ath79_gpio_probe(struct platform_device *pdev)
 {
-	struct ath79_gpio_platform_data *pdata = pdev->dev.platform_data;
+	struct ath79_gpio_platform_data *pdata = dev_get_platdata(&pdev->dev);
 	struct device_node *np = pdev->dev.of_node;
 	struct ath79_gpio_ctrl *ctrl;
 	struct resource *res;
@@ -177,14 +175,14 @@ static int ath79_gpio_probe(struct platform_device *pdev)
 
 	spin_lock_init(&ctrl->lock);
 	memcpy(&ctrl->chip, &ath79_gpio_chip, sizeof(ctrl->chip));
-	ctrl->chip.dev = &pdev->dev;
+	ctrl->chip.parent = &pdev->dev;
 	ctrl->chip.ngpio = ath79_gpio_count;
 	if (oe_inverted) {
 		ctrl->chip.direction_input = ar934x_gpio_direction_input;
 		ctrl->chip.direction_output = ar934x_gpio_direction_output;
 	}
 
-	err = gpiochip_add(&ctrl->chip);
+	err = gpiochip_add_data(&ctrl->chip, ctrl);
 	if (err) {
 		dev_err(&pdev->dev,
 			"cannot add AR71xx GPIO chip, error=%d", err);
diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c
index 33a1f9779b86..b6c5abe85daf 100644
--- a/drivers/gpio/gpio-bcm-kona.c
+++ b/drivers/gpio/gpio-bcm-kona.c
@@ -78,11 +78,6 @@ struct bcm_kona_gpio_bank {
 	struct bcm_kona_gpio *kona_gpio;
 };
 
-static inline struct bcm_kona_gpio *to_kona_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct bcm_kona_gpio, gpio_chip);
-}
-
 static inline void bcm_kona_gpio_write_lock_regs(void __iomem *reg_base,
 						int bank_id, u32 lockcode)
 {
@@ -124,7 +119,7 @@ static void bcm_kona_gpio_unlock_gpio(struct bcm_kona_gpio *kona_gpio,
 
 static int bcm_kona_gpio_get_dir(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcm_kona_gpio *kona_gpio = to_kona_gpio(chip);
+	struct bcm_kona_gpio *kona_gpio = gpiochip_get_data(chip);
 	void __iomem *reg_base = kona_gpio->reg_base;
 	u32 val;
 
@@ -141,7 +136,7 @@ static void bcm_kona_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 	u32 val, reg_offset;
 	unsigned long flags;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	reg_base = kona_gpio->reg_base;
 	spin_lock_irqsave(&kona_gpio->lock, flags);
 
@@ -168,7 +163,7 @@ static int bcm_kona_gpio_get(struct gpio_chip *chip, unsigned gpio)
 	u32 val, reg_offset;
 	unsigned long flags;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	reg_base = kona_gpio->reg_base;
 	spin_lock_irqsave(&kona_gpio->lock, flags);
 
@@ -188,7 +183,7 @@ static int bcm_kona_gpio_get(struct gpio_chip *chip, unsigned gpio)
 
 static int bcm_kona_gpio_request(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcm_kona_gpio *kona_gpio = to_kona_gpio(chip);
+	struct bcm_kona_gpio *kona_gpio = gpiochip_get_data(chip);
 
 	bcm_kona_gpio_unlock_gpio(kona_gpio, gpio);
 	return 0;
@@ -196,7 +191,7 @@ static int bcm_kona_gpio_request(struct gpio_chip *chip, unsigned gpio)
 
 static void bcm_kona_gpio_free(struct gpio_chip *chip, unsigned gpio)
 {
-	struct bcm_kona_gpio *kona_gpio = to_kona_gpio(chip);
+	struct bcm_kona_gpio *kona_gpio = gpiochip_get_data(chip);
 
 	bcm_kona_gpio_lock_gpio(kona_gpio, gpio);
 }
@@ -208,7 +203,7 @@ static int bcm_kona_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 	u32 val;
 	unsigned long flags;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	reg_base = kona_gpio->reg_base;
 	spin_lock_irqsave(&kona_gpio->lock, flags);
 
@@ -232,7 +227,7 @@ static int bcm_kona_gpio_direction_output(struct gpio_chip *chip,
 	u32 val, reg_offset;
 	unsigned long flags;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	reg_base = kona_gpio->reg_base;
 	spin_lock_irqsave(&kona_gpio->lock, flags);
 
@@ -255,7 +250,7 @@ static int bcm_kona_gpio_to_irq(struct gpio_chip *chip, unsigned gpio)
 {
 	struct bcm_kona_gpio *kona_gpio;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	if (gpio >= kona_gpio->gpio_chip.ngpio)
 		return -ENXIO;
 	return irq_create_mapping(kona_gpio->irq_domain, gpio);
@@ -269,11 +264,11 @@ static int bcm_kona_gpio_set_debounce(struct gpio_chip *chip, unsigned gpio,
 	u32 val, res;
 	unsigned long flags;
 
-	kona_gpio = to_kona_gpio(chip);
+	kona_gpio = gpiochip_get_data(chip);
 	reg_base = kona_gpio->reg_base;
 	/* debounce must be 1-128ms (or 0) */
 	if ((debounce > 0 && debounce < 1000) || debounce > 128000) {
-		dev_err(chip->dev, "Debounce value %u not in range\n",
+		dev_err(chip->parent, "Debounce value %u not in range\n",
 			debounce);
 		return -EINVAL;
 	}
@@ -416,7 +411,7 @@ static int bcm_kona_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	case IRQ_TYPE_LEVEL_LOW:
 		/* BCM GPIO doesn't support level triggering */
 	default:
-		dev_err(kona_gpio->gpio_chip.dev,
+		dev_err(kona_gpio->gpio_chip.parent,
 			"Invalid BCM GPIO irq type 0x%x\n", type);
 		return -EINVAL;
 	}
@@ -477,7 +472,7 @@ static int bcm_kona_gpio_irq_reqres(struct irq_data *d)
 	struct bcm_kona_gpio *kona_gpio = irq_data_get_irq_chip_data(d);
 
 	if (gpiochip_lock_as_irq(&kona_gpio->gpio_chip, d->hwirq)) {
-		dev_err(kona_gpio->gpio_chip.dev,
+		dev_err(kona_gpio->gpio_chip.parent,
 			"unable to lock HW IRQ %lu for IRQ\n",
 			d->hwirq);
 		return -EINVAL;
@@ -635,7 +630,7 @@ static int bcm_kona_gpio_probe(struct platform_device *pdev)
 
 	bcm_kona_gpio_reset(kona_gpio);
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, kona_gpio);
 	if (ret < 0) {
 		dev_err(dev, "Couldn't add GPIO chip -- %d\n", ret);
 		goto err_irq_domain;
diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c
index 4c64627c6bb5..d7644251e869 100644
--- a/drivers/gpio/gpio-brcmstb.c
+++ b/drivers/gpio/gpio-brcmstb.c
@@ -16,7 +16,6 @@
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/module.h>
-#include <linux/basic_mmio_gpio.h>
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/interrupt.h>
@@ -35,7 +34,7 @@
 struct brcmstb_gpio_bank {
 	struct list_head node;
 	int id;
-	struct bgpio_chip bgc;
+	struct gpio_chip gc;
 	struct brcmstb_gpio_priv *parent_priv;
 	u32 width;
 	struct irq_chip irq_chip;
@@ -57,37 +56,30 @@ struct brcmstb_gpio_priv {
 /* assumes MAX_GPIO_PER_BANK is a multiple of 2 */
 #define GPIO_BIT(gpio)          ((gpio) & (MAX_GPIO_PER_BANK - 1))
 
-static inline struct brcmstb_gpio_bank *
-brcmstb_gpio_gc_to_bank(struct gpio_chip *gc)
-{
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	return container_of(bgc, struct brcmstb_gpio_bank, bgc);
-}
-
 static inline struct brcmstb_gpio_priv *
 brcmstb_gpio_gc_to_priv(struct gpio_chip *gc)
 {
-	struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	return bank->parent_priv;
 }
 
 static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
 		unsigned int offset, bool enable)
 {
-	struct bgpio_chip *bgc = &bank->bgc;
+	struct gpio_chip *gc = &bank->gc;
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
-	u32 mask = bgc->pin2mask(bgc, offset);
+	u32 mask = gc->pin2mask(gc, offset);
 	u32 imask;
 	unsigned long flags;
 
-	spin_lock_irqsave(&bgc->lock, flags);
-	imask = bgc->read_reg(priv->reg_base + GIO_MASK(bank->id));
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
+	imask = gc->read_reg(priv->reg_base + GIO_MASK(bank->id));
 	if (enable)
 		imask |= mask;
 	else
 		imask &= ~mask;
-	bgc->write_reg(priv->reg_base + GIO_MASK(bank->id), imask);
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	gc->write_reg(priv->reg_base + GIO_MASK(bank->id), imask);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 /* -------------------- IRQ chip functions -------------------- */
@@ -95,7 +87,7 @@ static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
 static void brcmstb_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 
 	brcmstb_gpio_set_imask(bank, d->hwirq, false);
 }
@@ -103,7 +95,7 @@ static void brcmstb_gpio_irq_mask(struct irq_data *d)
 static void brcmstb_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 
 	brcmstb_gpio_set_imask(bank, d->hwirq, true);
 }
@@ -111,7 +103,7 @@ static void brcmstb_gpio_irq_unmask(struct irq_data *d)
 static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
 	u32 mask = BIT(d->hwirq);
 	u32 edge_insensitive, iedge_insensitive;
@@ -149,23 +141,23 @@ static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 		return -EINVAL;
 	}
 
-	spin_lock_irqsave(&bank->bgc.lock, flags);
+	spin_lock_irqsave(&bank->gc.bgpio_lock, flags);
 
-	iedge_config = bank->bgc.read_reg(priv->reg_base +
+	iedge_config = bank->gc.read_reg(priv->reg_base +
 			GIO_EC(bank->id)) & ~mask;
-	iedge_insensitive = bank->bgc.read_reg(priv->reg_base +
+	iedge_insensitive = bank->gc.read_reg(priv->reg_base +
 			GIO_EI(bank->id)) & ~mask;
-	ilevel = bank->bgc.read_reg(priv->reg_base +
+	ilevel = bank->gc.read_reg(priv->reg_base +
 			GIO_LEVEL(bank->id)) & ~mask;
 
-	bank->bgc.write_reg(priv->reg_base + GIO_EC(bank->id),
+	bank->gc.write_reg(priv->reg_base + GIO_EC(bank->id),
 			iedge_config | edge_config);
-	bank->bgc.write_reg(priv->reg_base + GIO_EI(bank->id),
+	bank->gc.write_reg(priv->reg_base + GIO_EI(bank->id),
 			iedge_insensitive | edge_insensitive);
-	bank->bgc.write_reg(priv->reg_base + GIO_LEVEL(bank->id),
+	bank->gc.write_reg(priv->reg_base + GIO_LEVEL(bank->id),
 			ilevel | level);
 
-	spin_unlock_irqrestore(&bank->bgc.lock, flags);
+	spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags);
 	return 0;
 }
 
@@ -210,29 +202,29 @@ static irqreturn_t brcmstb_gpio_wake_irq_handler(int irq, void *data)
 static void brcmstb_gpio_irq_bank_handler(struct brcmstb_gpio_bank *bank)
 {
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
-	struct irq_domain *irq_domain = bank->bgc.gc.irqdomain;
+	struct irq_domain *irq_domain = bank->gc.irqdomain;
 	void __iomem *reg_base = priv->reg_base;
 	unsigned long status;
 	unsigned long flags;
 
-	spin_lock_irqsave(&bank->bgc.lock, flags);
-	while ((status = bank->bgc.read_reg(reg_base + GIO_STAT(bank->id)) &
-			 bank->bgc.read_reg(reg_base + GIO_MASK(bank->id)))) {
+	spin_lock_irqsave(&bank->gc.bgpio_lock, flags);
+	while ((status = bank->gc.read_reg(reg_base + GIO_STAT(bank->id)) &
+			 bank->gc.read_reg(reg_base + GIO_MASK(bank->id)))) {
 		int bit;
 
 		for_each_set_bit(bit, &status, 32) {
-			u32 stat = bank->bgc.read_reg(reg_base +
+			u32 stat = bank->gc.read_reg(reg_base +
 						      GIO_STAT(bank->id));
 			if (bit >= bank->width)
 				dev_warn(&priv->pdev->dev,
 					 "IRQ for invalid GPIO (bank=%d, offset=%d)\n",
 					 bank->id, bit);
-			bank->bgc.write_reg(reg_base + GIO_STAT(bank->id),
+			bank->gc.write_reg(reg_base + GIO_STAT(bank->id),
 					    stat | BIT(bit));
 			generic_handle_irq(irq_find_mapping(irq_domain, bit));
 		}
 	}
-	spin_unlock_irqrestore(&bank->bgc.lock, flags);
+	spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags);
 }
 
 /* Each UPG GIO block has one IRQ for all banks */
@@ -303,9 +295,7 @@ static int brcmstb_gpio_remove(struct platform_device *pdev)
 	 */
 	list_for_each(pos, &priv->bank_list) {
 		bank = list_entry(pos, struct brcmstb_gpio_bank, node);
-		ret = bgpio_remove(&bank->bgc);
-		if (ret)
-			dev_err(&pdev->dev, "gpiochip_remove fail in cleanup\n");
+		gpiochip_remove(&bank->gc);
 	}
 	if (priv->reboot_notifier.notifier_call) {
 		ret = unregister_reboot_notifier(&priv->reboot_notifier);
@@ -320,7 +310,7 @@ static int brcmstb_gpio_of_xlate(struct gpio_chip *gc,
 		const struct of_phandle_args *gpiospec, u32 *flags)
 {
 	struct brcmstb_gpio_priv *priv = brcmstb_gpio_gc_to_priv(gc);
-	struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	int offset;
 
 	if (gc->of_gpio_n_cells != 2) {
@@ -398,9 +388,9 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 	if (priv->can_wake)
 		bank->irq_chip.irq_set_wake = brcmstb_gpio_irq_set_wake;
 
-	gpiochip_irqchip_add(&bank->bgc.gc, &bank->irq_chip, 0,
+	gpiochip_irqchip_add(&bank->gc, &bank->irq_chip, 0,
 			handle_simple_irq, IRQ_TYPE_NONE);
-	gpiochip_set_chained_irqchip(&bank->bgc.gc, &bank->irq_chip,
+	gpiochip_set_chained_irqchip(&bank->gc, &bank->irq_chip,
 			priv->parent_irq, brcmstb_gpio_irq_handler);
 
 	return 0;
@@ -419,6 +409,7 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
 	int num_banks = 0;
 	int err;
 	static int gpio_base;
+	unsigned long flags = 0;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -448,10 +439,21 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
 	if (brcmstb_gpio_sanity_check_banks(dev, np, res))
 		return -EINVAL;
 
+	/*
+	 * MIPS endianness is configured by boot strap, which also reverses all
+	 * bus endianness (i.e., big-endian CPU + big endian bus ==> native
+	 * endian I/O).
+	 *
+	 * Other architectures (e.g., ARM) either do not support big endian, or
+	 * else leave I/O in little endian mode.
+	 */
+#if defined(CONFIG_MIPS) && defined(__BIG_ENDIAN)
+	flags = BGPIOF_BIG_ENDIAN_BYTE_ORDER;
+#endif
+
 	of_property_for_each_u32(np, "brcm,gpio-bank-widths", prop, p,
 			bank_width) {
 		struct brcmstb_gpio_bank *bank;
-		struct bgpio_chip *bgc;
 		struct gpio_chip *gc;
 
 		bank = devm_kzalloc(dev, sizeof(*bank), GFP_KERNEL);
@@ -473,17 +475,16 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
 		 * Regs are 4 bytes wide, have data reg, no set/clear regs,
 		 * and direction bits have 0 = output and 1 = input
 		 */
-		bgc = &bank->bgc;
-		err = bgpio_init(bgc, dev, 4,
+		gc = &bank->gc;
+		err = bgpio_init(gc, dev, 4,
 				reg_base + GIO_DATA(bank->id),
 				NULL, NULL, NULL,
-				reg_base + GIO_IODIR(bank->id), 0);
+				reg_base + GIO_IODIR(bank->id), flags);
 		if (err) {
 			dev_err(dev, "bgpio_init() failed\n");
 			goto fail;
 		}
 
-		gc = &bgc->gc;
 		gc->of_node = np;
 		gc->owner = THIS_MODULE;
 		gc->label = np->full_name;
@@ -497,9 +498,9 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
 		 * Mask all interrupts by default, since wakeup interrupts may
 		 * be retained from S5 cold boot
 		 */
-		bank->bgc.write_reg(reg_base + GIO_MASK(bank->id), 0);
+		gc->write_reg(reg_base + GIO_MASK(bank->id), 0);
 
-		err = gpiochip_add(gc);
+		err = gpiochip_add_data(gc, bank);
 		if (err) {
 			dev_err(dev, "Could not add gpiochip for bank %d\n",
 					bank->id);
diff --git a/drivers/gpio/gpio-bt8xx.c b/drivers/gpio/gpio-bt8xx.c
index 7e4c43c18960..acefb25e8eca 100644
--- a/drivers/gpio/gpio-bt8xx.c
+++ b/drivers/gpio/gpio-bt8xx.c
@@ -80,7 +80,7 @@ MODULE_PARM_DESC(gpiobase, "The GPIO number base. -1 means dynamic, which is the
 
 static int bt8xxgpio_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 {
-	struct bt8xxgpio *bg = container_of(gpio, struct bt8xxgpio, gpio);
+	struct bt8xxgpio *bg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u32 outen, data;
 
@@ -101,7 +101,7 @@ static int bt8xxgpio_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 
 static int bt8xxgpio_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
-	struct bt8xxgpio *bg = container_of(gpio, struct bt8xxgpio, gpio);
+	struct bt8xxgpio *bg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u32 val;
 
@@ -115,7 +115,7 @@ static int bt8xxgpio_gpio_get(struct gpio_chip *gpio, unsigned nr)
 static int bt8xxgpio_gpio_direction_output(struct gpio_chip *gpio,
 					unsigned nr, int val)
 {
-	struct bt8xxgpio *bg = container_of(gpio, struct bt8xxgpio, gpio);
+	struct bt8xxgpio *bg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u32 outen, data;
 
@@ -140,7 +140,7 @@ static int bt8xxgpio_gpio_direction_output(struct gpio_chip *gpio,
 static void bt8xxgpio_gpio_set(struct gpio_chip *gpio,
 			    unsigned nr, int val)
 {
-	struct bt8xxgpio *bg = container_of(gpio, struct bt8xxgpio, gpio);
+	struct bt8xxgpio *bg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u32 data;
 
@@ -217,7 +217,7 @@ static int bt8xxgpio_probe(struct pci_dev *dev,
 	bgwrite(0, BT848_GPIO_OUT_EN);
 
 	bt8xxgpio_gpio_setup(bg);
-	err = gpiochip_add(&bg->gpio);
+	err = gpiochip_add_data(&bg->gpio, bg);
 	if (err) {
 		printk(KERN_ERR "bt8xxgpio: Failed to register GPIOs\n");
 		goto err_disable;
diff --git a/drivers/gpio/gpio-clps711x.c b/drivers/gpio/gpio-clps711x.c
index b6908f1ff1ab..c84f9551f108 100644
--- a/drivers/gpio/gpio-clps711x.c
+++ b/drivers/gpio/gpio-clps711x.c
@@ -10,24 +10,23 @@
  */
 
 #include <linux/err.h>
-#include <linux/gpio.h>
 #include <linux/module.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 
 static int clps711x_gpio_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	void __iomem *dat, *dir;
-	struct bgpio_chip *bgc;
+	struct gpio_chip *gc;
 	struct resource *res;
 	int err, id = np ? of_alias_get_id(np, "gpio") : pdev->id;
 
 	if ((id < 0) || (id > 4))
 		return -ENODEV;
 
-	bgc = devm_kzalloc(&pdev->dev, sizeof(*bgc), GFP_KERNEL);
-	if (!bgc)
+	gc = devm_kzalloc(&pdev->dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -43,11 +42,11 @@ static int clps711x_gpio_probe(struct platform_device *pdev)
 	switch (id) {
 	case 3:
 		/* PORTD is inverted logic for direction register */
-		err = bgpio_init(bgc, &pdev->dev, 1, dat, NULL, NULL,
+		err = bgpio_init(gc, &pdev->dev, 1, dat, NULL, NULL,
 				 NULL, dir, 0);
 		break;
 	default:
-		err = bgpio_init(bgc, &pdev->dev, 1, dat, NULL, NULL,
+		err = bgpio_init(gc, &pdev->dev, 1, dat, NULL, NULL,
 				 dir, NULL, 0);
 		break;
 	}
@@ -58,24 +57,25 @@ static int clps711x_gpio_probe(struct platform_device *pdev)
 	switch (id) {
 	case 4:
 		/* PORTE is 3 lines only */
-		bgc->gc.ngpio = 3;
+		gc->ngpio = 3;
 		break;
 	default:
 		break;
 	}
 
-	bgc->gc.base = id * 8;
-	bgc->gc.owner = THIS_MODULE;
-	platform_set_drvdata(pdev, bgc);
+	gc->base = id * 8;
+	gc->owner = THIS_MODULE;
+	platform_set_drvdata(pdev, gc);
 
-	return gpiochip_add(&bgc->gc);
+	return gpiochip_add_data(gc, NULL);
 }
 
 static int clps711x_gpio_remove(struct platform_device *pdev)
 {
-	struct bgpio_chip *bgc = platform_get_drvdata(pdev);
+	struct gpio_chip *gc = platform_get_drvdata(pdev);
 
-	return bgpio_remove(bgc);
+	gpiochip_remove(gc);
+	return 0;
 }
 
 static const struct of_device_id __maybe_unused clps711x_gpio_ids[] = {
diff --git a/drivers/gpio/gpio-crystalcove.c b/drivers/gpio/gpio-crystalcove.c
index fddd204dc9b6..7865ef0d3352 100644
--- a/drivers/gpio/gpio-crystalcove.c
+++ b/drivers/gpio/gpio-crystalcove.c
@@ -86,11 +86,6 @@ struct crystalcove_gpio {
 	bool set_irq_mask;
 };
 
-static inline struct crystalcove_gpio *to_cg(struct gpio_chip *gc)
-{
-	return container_of(gc, struct crystalcove_gpio, chip);
-}
-
 static inline int to_reg(int gpio, enum ctrl_register reg_type)
 {
 	int reg;
@@ -134,7 +129,7 @@ static void crystalcove_update_irq_ctrl(struct crystalcove_gpio *cg, int gpio)
 
 static int crystalcove_gpio_dir_in(struct gpio_chip *chip, unsigned gpio)
 {
-	struct crystalcove_gpio *cg = to_cg(chip);
+	struct crystalcove_gpio *cg = gpiochip_get_data(chip);
 
 	if (gpio > CRYSTALCOVE_VGPIO_NUM)
 		return 0;
@@ -146,7 +141,7 @@ static int crystalcove_gpio_dir_in(struct gpio_chip *chip, unsigned gpio)
 static int crystalcove_gpio_dir_out(struct gpio_chip *chip, unsigned gpio,
 				    int value)
 {
-	struct crystalcove_gpio *cg = to_cg(chip);
+	struct crystalcove_gpio *cg = gpiochip_get_data(chip);
 
 	if (gpio > CRYSTALCOVE_VGPIO_NUM)
 		return 0;
@@ -157,7 +152,7 @@ static int crystalcove_gpio_dir_out(struct gpio_chip *chip, unsigned gpio,
 
 static int crystalcove_gpio_get(struct gpio_chip *chip, unsigned gpio)
 {
-	struct crystalcove_gpio *cg = to_cg(chip);
+	struct crystalcove_gpio *cg = gpiochip_get_data(chip);
 	int ret;
 	unsigned int val;
 
@@ -174,7 +169,7 @@ static int crystalcove_gpio_get(struct gpio_chip *chip, unsigned gpio)
 static void crystalcove_gpio_set(struct gpio_chip *chip,
 				 unsigned gpio, int value)
 {
-	struct crystalcove_gpio *cg = to_cg(chip);
+	struct crystalcove_gpio *cg = gpiochip_get_data(chip);
 
 	if (gpio > CRYSTALCOVE_VGPIO_NUM)
 		return;
@@ -187,7 +182,8 @@ static void crystalcove_gpio_set(struct gpio_chip *chip,
 
 static int crystalcove_irq_type(struct irq_data *data, unsigned type)
 {
-	struct crystalcove_gpio *cg = to_cg(irq_data_get_irq_chip_data(data));
+	struct crystalcove_gpio *cg =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 
 	switch (type) {
 	case IRQ_TYPE_NONE:
@@ -213,14 +209,16 @@ static int crystalcove_irq_type(struct irq_data *data, unsigned type)
 
 static void crystalcove_bus_lock(struct irq_data *data)
 {
-	struct crystalcove_gpio *cg = to_cg(irq_data_get_irq_chip_data(data));
+	struct crystalcove_gpio *cg =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 
 	mutex_lock(&cg->buslock);
 }
 
 static void crystalcove_bus_sync_unlock(struct irq_data *data)
 {
-	struct crystalcove_gpio *cg = to_cg(irq_data_get_irq_chip_data(data));
+	struct crystalcove_gpio *cg =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 	int gpio = data->hwirq;
 
 	if (cg->update & UPDATE_IRQ_TYPE)
@@ -234,7 +232,8 @@ static void crystalcove_bus_sync_unlock(struct irq_data *data)
 
 static void crystalcove_irq_unmask(struct irq_data *data)
 {
-	struct crystalcove_gpio *cg = to_cg(irq_data_get_irq_chip_data(data));
+	struct crystalcove_gpio *cg =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 
 	cg->set_irq_mask = false;
 	cg->update |= UPDATE_IRQ_MASK;
@@ -242,7 +241,8 @@ static void crystalcove_irq_unmask(struct irq_data *data)
 
 static void crystalcove_irq_mask(struct irq_data *data)
 {
-	struct crystalcove_gpio *cg = to_cg(irq_data_get_irq_chip_data(data));
+	struct crystalcove_gpio *cg =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 
 	cg->set_irq_mask = true;
 	cg->update |= UPDATE_IRQ_MASK;
@@ -288,7 +288,7 @@ static irqreturn_t crystalcove_gpio_irq_handler(int irq, void *data)
 static void crystalcove_gpio_dbg_show(struct seq_file *s,
 				      struct gpio_chip *chip)
 {
-	struct crystalcove_gpio *cg = to_cg(chip);
+	struct crystalcove_gpio *cg = gpiochip_get_data(chip);
 	int gpio, offset;
 	unsigned int ctlo, ctli, mirqs0, mirqsx, irq;
 
@@ -341,11 +341,11 @@ static int crystalcove_gpio_probe(struct platform_device *pdev)
 	cg->chip.base = -1;
 	cg->chip.ngpio = CRYSTALCOVE_VGPIO_NUM;
 	cg->chip.can_sleep = true;
-	cg->chip.dev = dev;
+	cg->chip.parent = dev;
 	cg->chip.dbg_show = crystalcove_gpio_dbg_show;
 	cg->regmap = pmic->regmap;
 
-	retval = gpiochip_add(&cg->chip);
+	retval = gpiochip_add_data(&cg->chip, cg);
 	if (retval) {
 		dev_warn(&pdev->dev, "add gpio chip error: %d\n", retval);
 		return retval;
diff --git a/drivers/gpio/gpio-cs5535.c b/drivers/gpio/gpio-cs5535.c
index 7b0b198a563d..eccb712e09fb 100644
--- a/drivers/gpio/gpio-cs5535.c
+++ b/drivers/gpio/gpio-cs5535.c
@@ -42,6 +42,10 @@ static ulong mask = GPIO_DEFAULT_MASK;
 module_param_named(mask, mask, ulong, 0444);
 MODULE_PARM_DESC(mask, "GPIO channel mask.");
 
+/*
+ * FIXME: convert this singleton driver to use the state container
+ * design pattern, see Documentation/driver-model/design-patterns.txt
+ */
 static struct cs5535_gpio_chip {
 	struct gpio_chip chip;
 	resource_size_t base;
@@ -201,8 +205,7 @@ EXPORT_SYMBOL_GPL(cs5535_gpio_setup_event);
 
 static int chip_gpio_request(struct gpio_chip *c, unsigned offset)
 {
-	struct cs5535_gpio_chip *chip =
-		container_of(c, struct cs5535_gpio_chip, chip);
+	struct cs5535_gpio_chip *chip = gpiochip_get_data(c);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -242,8 +245,7 @@ static void chip_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 
 static int chip_direction_input(struct gpio_chip *c, unsigned offset)
 {
-	struct cs5535_gpio_chip *chip =
-		container_of(c, struct cs5535_gpio_chip, chip);
+	struct cs5535_gpio_chip *chip = gpiochip_get_data(c);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -256,8 +258,7 @@ static int chip_direction_input(struct gpio_chip *c, unsigned offset)
 
 static int chip_direction_output(struct gpio_chip *c, unsigned offset, int val)
 {
-	struct cs5535_gpio_chip *chip =
-		container_of(c, struct cs5535_gpio_chip, chip);
+	struct cs5535_gpio_chip *chip = gpiochip_get_data(c);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -347,7 +348,7 @@ static int cs5535_gpio_probe(struct platform_device *pdev)
 				mask_orig, mask);
 
 	/* finally, register with the generic GPIO API */
-	err = gpiochip_add(&cs5535_gpio_chip.chip);
+	err = gpiochip_add_data(&cs5535_gpio_chip.chip, &cs5535_gpio_chip);
 	if (err)
 		goto done;
 
diff --git a/drivers/gpio/gpio-da9052.c b/drivers/gpio/gpio-da9052.c
index 2e9578ec0ca1..f9b3247ad14b 100644
--- a/drivers/gpio/gpio-da9052.c
+++ b/drivers/gpio/gpio-da9052.c
@@ -51,11 +51,6 @@ struct da9052_gpio {
 	struct gpio_chip gp;
 };
 
-static inline struct da9052_gpio *to_da9052_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct da9052_gpio, gp);
-}
-
 static unsigned char da9052_gpio_port_odd(unsigned offset)
 {
 	return offset % 2;
@@ -63,7 +58,7 @@ static unsigned char da9052_gpio_port_odd(unsigned offset)
 
 static int da9052_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct da9052_gpio *gpio = to_da9052_gpio(gc);
+	struct da9052_gpio *gpio = gpiochip_get_data(gc);
 	int da9052_port_direction = 0;
 	int ret;
 
@@ -89,15 +84,12 @@ static int da9052_gpio_get(struct gpio_chip *gc, unsigned offset)
 					      DA9052_STATUS_D_REG);
 		if (ret < 0)
 			return ret;
-		if (ret & (1 << DA9052_GPIO_SHIFT_COUNT(offset)))
-			return 1;
-		else
-			return 0;
+		return !!(ret & (1 << DA9052_GPIO_SHIFT_COUNT(offset)));
 	case DA9052_OUTPUT_PUSHPULL:
 		if (da9052_gpio_port_odd(offset))
-			return ret & DA9052_GPIO_ODD_PORT_MODE;
+			return !!(ret & DA9052_GPIO_ODD_PORT_MODE);
 		else
-			return ret & DA9052_GPIO_EVEN_PORT_MODE;
+			return !!(ret & DA9052_GPIO_EVEN_PORT_MODE);
 	default:
 		return -EINVAL;
 	}
@@ -105,7 +97,7 @@ static int da9052_gpio_get(struct gpio_chip *gc, unsigned offset)
 
 static void da9052_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct da9052_gpio *gpio = to_da9052_gpio(gc);
+	struct da9052_gpio *gpio = gpiochip_get_data(gc);
 	int ret;
 
 	if (da9052_gpio_port_odd(offset)) {
@@ -131,7 +123,7 @@ static void da9052_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 
 static int da9052_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct da9052_gpio *gpio = to_da9052_gpio(gc);
+	struct da9052_gpio *gpio = gpiochip_get_data(gc);
 	unsigned char register_value;
 	int ret;
 
@@ -157,7 +149,7 @@ static int da9052_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 static int da9052_gpio_direction_output(struct gpio_chip *gc,
 					unsigned offset, int value)
 {
-	struct da9052_gpio *gpio = to_da9052_gpio(gc);
+	struct da9052_gpio *gpio = gpiochip_get_data(gc);
 	unsigned char register_value;
 	int ret;
 
@@ -182,7 +174,7 @@ static int da9052_gpio_direction_output(struct gpio_chip *gc,
 
 static int da9052_gpio_to_irq(struct gpio_chip *gc, u32 offset)
 {
-	struct da9052_gpio *gpio = to_da9052_gpio(gc);
+	struct da9052_gpio *gpio = gpiochip_get_data(gc);
 	struct da9052 *da9052 = gpio->da9052;
 
 	int irq;
@@ -222,7 +214,7 @@ static int da9052_gpio_probe(struct platform_device *pdev)
 	if (pdata && pdata->gpio_base)
 		gpio->gp.base = pdata->gpio_base;
 
-	ret = gpiochip_add(&gpio->gp);
+	ret = gpiochip_add_data(&gpio->gp, gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-da9055.c b/drivers/gpio/gpio-da9055.c
index 7227e6ed3cb9..18210fb2cb13 100644
--- a/drivers/gpio/gpio-da9055.c
+++ b/drivers/gpio/gpio-da9055.c
@@ -35,14 +35,9 @@ struct da9055_gpio {
 	struct gpio_chip gp;
 };
 
-static inline struct da9055_gpio *to_da9055_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct da9055_gpio, gp);
-}
-
 static int da9055_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct da9055_gpio *gpio = to_da9055_gpio(gc);
+	struct da9055_gpio *gpio = gpiochip_get_data(gc);
 	int gpio_direction = 0;
 	int ret;
 
@@ -71,7 +66,7 @@ static int da9055_gpio_get(struct gpio_chip *gc, unsigned offset)
 
 static void da9055_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct da9055_gpio *gpio = to_da9055_gpio(gc);
+	struct da9055_gpio *gpio = gpiochip_get_data(gc);
 
 	da9055_reg_update(gpio->da9055,
 			DA9055_REG_GPIO_MODE0_2,
@@ -81,7 +76,7 @@ static void da9055_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 
 static int da9055_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct da9055_gpio *gpio = to_da9055_gpio(gc);
+	struct da9055_gpio *gpio = gpiochip_get_data(gc);
 	unsigned char reg_byte;
 
 	reg_byte = (DA9055_ACT_LOW | DA9055_GPI)
@@ -97,7 +92,7 @@ static int da9055_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 static int da9055_gpio_direction_output(struct gpio_chip *gc,
 					unsigned offset, int value)
 {
-	struct da9055_gpio *gpio = to_da9055_gpio(gc);
+	struct da9055_gpio *gpio = gpiochip_get_data(gc);
 	unsigned char reg_byte;
 	int ret;
 
@@ -119,7 +114,7 @@ static int da9055_gpio_direction_output(struct gpio_chip *gc,
 
 static int da9055_gpio_to_irq(struct gpio_chip *gc, u32 offset)
 {
-	struct da9055_gpio *gpio = to_da9055_gpio(gc);
+	struct da9055_gpio *gpio = gpiochip_get_data(gc);
 	struct da9055 *da9055 = gpio->da9055;
 
 	return regmap_irq_get_virq(da9055->irq_data,
@@ -156,7 +151,7 @@ static int da9055_gpio_probe(struct platform_device *pdev)
 	if (pdata && pdata->gpio_base)
 		gpio->gp.base = pdata->gpio_base;
 
-	ret = gpiochip_add(&gpio->gp);
+	ret = gpiochip_add_data(&gpio->gp, gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		goto err_mem;
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index 5e715388803d..ec58f4288649 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -41,9 +41,6 @@ typedef struct irq_chip *(*gpio_get_irq_chip_cb_t)(unsigned int irq);
 
 #define BINTEN	0x8 /* GPIO Interrupt Per-Bank Enable Register */
 
-#define chip2controller(chip)	\
-	container_of(chip, struct davinci_gpio_controller, chip)
-
 static void __iomem *gpio_base;
 
 static struct davinci_gpio_regs __iomem *gpio2regs(unsigned gpio)
@@ -82,7 +79,7 @@ static int davinci_gpio_irq_setup(struct platform_device *pdev);
 static inline int __davinci_direction(struct gpio_chip *chip,
 			unsigned offset, bool out, int value)
 {
-	struct davinci_gpio_controller *d = chip2controller(chip);
+	struct davinci_gpio_controller *d = gpiochip_get_data(chip);
 	struct davinci_gpio_regs __iomem *g = d->regs;
 	unsigned long flags;
 	u32 temp;
@@ -122,10 +119,10 @@ davinci_direction_out(struct gpio_chip *chip, unsigned offset, int value)
  */
 static int davinci_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct davinci_gpio_controller *d = chip2controller(chip);
+	struct davinci_gpio_controller *d = gpiochip_get_data(chip);
 	struct davinci_gpio_regs __iomem *g = d->regs;
 
-	return (1 << offset) & readl_relaxed(&g->in_data);
+	return !!((1 << offset) & readl_relaxed(&g->in_data));
 }
 
 /*
@@ -134,7 +131,7 @@ static int davinci_gpio_get(struct gpio_chip *chip, unsigned offset)
 static void
 davinci_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct davinci_gpio_controller *d = chip2controller(chip);
+	struct davinci_gpio_controller *d = gpiochip_get_data(chip);
 	struct davinci_gpio_regs __iomem *g = d->regs;
 
 	writel_relaxed((1 << offset), value ? &g->set_data : &g->clr_data);
@@ -149,7 +146,7 @@ davinci_gpio_get_pdata(struct platform_device *pdev)
 	u32 val;
 
 	if (!IS_ENABLED(CONFIG_OF) || !pdev->dev.of_node)
-		return pdev->dev.platform_data;
+		return dev_get_platdata(&pdev->dev);
 
 	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
@@ -179,8 +176,8 @@ static int davinci_gpio_of_xlate(struct gpio_chip *gc,
 			     const struct of_phandle_args *gpiospec,
 			     u32 *flags)
 {
-	struct davinci_gpio_controller *chips = dev_get_drvdata(gc->dev);
-	struct davinci_gpio_platform_data *pdata = dev_get_platdata(gc->dev);
+	struct davinci_gpio_controller *chips = dev_get_drvdata(gc->parent);
+	struct davinci_gpio_platform_data *pdata = dev_get_platdata(gc->parent);
 
 	if (gpiospec->args[0] > pdata->ngpio)
 		return -EINVAL;
@@ -254,7 +251,7 @@ static int davinci_gpio_probe(struct platform_device *pdev)
 #ifdef CONFIG_OF_GPIO
 		chips[i].chip.of_gpio_n_cells = 2;
 		chips[i].chip.of_xlate = davinci_gpio_of_xlate;
-		chips[i].chip.dev = dev;
+		chips[i].chip.parent = dev;
 		chips[i].chip.of_node = dev->of_node;
 #endif
 		spin_lock_init(&chips[i].lock);
@@ -265,7 +262,7 @@ static int davinci_gpio_probe(struct platform_device *pdev)
 		chips[i].clr_data = &regs->clr_data;
 		chips[i].in_data = &regs->in_data;
 
-		gpiochip_add(&chips[i].chip);
+		gpiochip_add_data(&chips[i].chip, &chips[i]);
 	}
 
 	platform_set_drvdata(pdev, chips);
@@ -368,7 +365,7 @@ static void gpio_irq_handler(struct irq_desc *desc)
 
 static int gpio_to_irq_banked(struct gpio_chip *chip, unsigned offset)
 {
-	struct davinci_gpio_controller *d = chip2controller(chip);
+	struct davinci_gpio_controller *d = gpiochip_get_data(chip);
 
 	if (d->irq_domain)
 		return irq_create_mapping(d->irq_domain, d->chip.base + offset);
@@ -378,7 +375,7 @@ static int gpio_to_irq_banked(struct gpio_chip *chip, unsigned offset)
 
 static int gpio_to_irq_unbanked(struct gpio_chip *chip, unsigned offset)
 {
-	struct davinci_gpio_controller *d = chip2controller(chip);
+	struct davinci_gpio_controller *d = gpiochip_get_data(chip);
 
 	/*
 	 * NOTE:  we assume for now that only irqs in the first gpio_chip
diff --git a/drivers/gpio/gpio-dln2.c b/drivers/gpio/gpio-dln2.c
index 6685712c15cf..e11a7d126e74 100644
--- a/drivers/gpio/gpio-dln2.c
+++ b/drivers/gpio/gpio-dln2.c
@@ -153,7 +153,7 @@ static int dln2_gpio_pin_set_out_val(struct dln2_gpio *dln2,
 
 static int dln2_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 	struct dln2_gpio_pin req = {
 		.pin = cpu_to_le16(offset),
 	};
@@ -194,14 +194,14 @@ out_disable:
 
 static void dln2_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 
 	dln2_gpio_pin_cmd(dln2, DLN2_GPIO_PIN_DISABLE, offset);
 }
 
 static int dln2_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 
 	if (test_bit(offset, dln2->output_enabled))
 		return GPIOF_DIR_OUT;
@@ -211,7 +211,7 @@ static int dln2_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 
 static int dln2_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 	int dir;
 
 	dir = dln2_gpio_get_direction(chip, offset);
@@ -226,7 +226,7 @@ static int dln2_gpio_get(struct gpio_chip *chip, unsigned int offset)
 
 static void dln2_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 
 	dln2_gpio_pin_set_out_val(dln2, offset, value);
 }
@@ -234,7 +234,7 @@ static void dln2_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int dln2_gpio_set_direction(struct gpio_chip *chip, unsigned offset,
 				   unsigned dir)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 	struct dln2_gpio_pin_val req = {
 		.pin = cpu_to_le16(offset),
 		.value = dir,
@@ -262,7 +262,7 @@ static int dln2_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int dln2_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 				      int value)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 	int ret;
 
 	ret = dln2_gpio_pin_set_out_val(dln2, offset, value);
@@ -275,7 +275,7 @@ static int dln2_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 static int dln2_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
 				  unsigned debounce)
 {
-	struct dln2_gpio *dln2 = container_of(chip, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(chip);
 	__le32 duration = cpu_to_le32(debounce);
 
 	return dln2_transfer_tx(dln2->pdev, DLN2_GPIO_SET_DEBOUNCE,
@@ -302,7 +302,7 @@ static int dln2_gpio_set_event_cfg(struct dln2_gpio *dln2, unsigned pin,
 static void dln2_irq_unmask(struct irq_data *irqd)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
-	struct dln2_gpio *dln2 = container_of(gc, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(gc);
 	int pin = irqd_to_hwirq(irqd);
 
 	set_bit(pin, dln2->unmasked_irqs);
@@ -311,7 +311,7 @@ static void dln2_irq_unmask(struct irq_data *irqd)
 static void dln2_irq_mask(struct irq_data *irqd)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
-	struct dln2_gpio *dln2 = container_of(gc, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(gc);
 	int pin = irqd_to_hwirq(irqd);
 
 	clear_bit(pin, dln2->unmasked_irqs);
@@ -320,7 +320,7 @@ static void dln2_irq_mask(struct irq_data *irqd)
 static int dln2_irq_set_type(struct irq_data *irqd, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
-	struct dln2_gpio *dln2 = container_of(gc, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(gc);
 	int pin = irqd_to_hwirq(irqd);
 
 	switch (type) {
@@ -349,7 +349,7 @@ static int dln2_irq_set_type(struct irq_data *irqd, unsigned type)
 static void dln2_irq_bus_lock(struct irq_data *irqd)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
-	struct dln2_gpio *dln2 = container_of(gc, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(gc);
 
 	mutex_lock(&dln2->irq_lock);
 }
@@ -357,7 +357,7 @@ static void dln2_irq_bus_lock(struct irq_data *irqd)
 static void dln2_irq_bus_unlock(struct irq_data *irqd)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
-	struct dln2_gpio *dln2 = container_of(gc, struct dln2_gpio, gpio);
+	struct dln2_gpio *dln2 = gpiochip_get_data(gc);
 	int pin = irqd_to_hwirq(irqd);
 	int enabled, unmasked;
 	unsigned type;
@@ -377,7 +377,7 @@ static void dln2_irq_bus_unlock(struct irq_data *irqd)
 
 		ret = dln2_gpio_set_event_cfg(dln2, pin, type, 0);
 		if (ret)
-			dev_err(dln2->gpio.dev, "failed to set event\n");
+			dev_err(dln2->gpio.parent, "failed to set event\n");
 	}
 
 	mutex_unlock(&dln2->irq_lock);
@@ -406,19 +406,19 @@ static void dln2_gpio_event(struct platform_device *pdev, u16 echo,
 	struct dln2_gpio *dln2 = platform_get_drvdata(pdev);
 
 	if (len < sizeof(*event)) {
-		dev_err(dln2->gpio.dev, "short event message\n");
+		dev_err(dln2->gpio.parent, "short event message\n");
 		return;
 	}
 
 	pin = le16_to_cpu(event->pin);
 	if (pin >= dln2->gpio.ngpio) {
-		dev_err(dln2->gpio.dev, "out of bounds pin %d\n", pin);
+		dev_err(dln2->gpio.parent, "out of bounds pin %d\n", pin);
 		return;
 	}
 
 	irq = irq_find_mapping(dln2->gpio.irqdomain, pin);
 	if (!irq) {
-		dev_err(dln2->gpio.dev, "pin %d not mapped to IRQ\n", pin);
+		dev_err(dln2->gpio.parent, "pin %d not mapped to IRQ\n", pin);
 		return;
 	}
 
@@ -462,7 +462,7 @@ static int dln2_gpio_probe(struct platform_device *pdev)
 	dln2->pdev = pdev;
 
 	dln2->gpio.label = "dln2";
-	dln2->gpio.dev = dev;
+	dln2->gpio.parent = dev;
 	dln2->gpio.owner = THIS_MODULE;
 	dln2->gpio.base = -1;
 	dln2->gpio.ngpio = pins;
@@ -479,7 +479,7 @@ static int dln2_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, dln2);
 
-	ret = gpiochip_add(&dln2->gpio);
+	ret = gpiochip_add_data(&dln2->gpio, dln2);
 	if (ret < 0) {
 		dev_err(dev, "failed to add gpio chip: %d\n", ret);
 		goto out;
diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c
index fcd5b0acfc72..597de1ef497b 100644
--- a/drivers/gpio/gpio-dwapb.c
+++ b/drivers/gpio/gpio-dwapb.c
@@ -7,7 +7,9 @@
  *
  * All enquiries to support@picochip.com
  */
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
+/* FIXME: for gpio_get_value(), replace this with direct register read */
+#include <linux/gpio.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -66,7 +68,7 @@ struct dwapb_context {
 #endif
 
 struct dwapb_gpio_port {
-	struct bgpio_chip	bgc;
+	struct gpio_chip	gc;
 	bool			is_registered;
 	struct dwapb_gpio	*gpio;
 #ifdef CONFIG_PM_SLEEP
@@ -83,33 +85,26 @@ struct dwapb_gpio {
 	struct irq_domain	*domain;
 };
 
-static inline struct dwapb_gpio_port *
-to_dwapb_gpio_port(struct bgpio_chip *bgc)
-{
-	return container_of(bgc, struct dwapb_gpio_port, bgc);
-}
-
 static inline u32 dwapb_read(struct dwapb_gpio *gpio, unsigned int offset)
 {
-	struct bgpio_chip *bgc	= &gpio->ports[0].bgc;
+	struct gpio_chip *gc	= &gpio->ports[0].gc;
 	void __iomem *reg_base	= gpio->regs;
 
-	return bgc->read_reg(reg_base + offset);
+	return gc->read_reg(reg_base + offset);
 }
 
 static inline void dwapb_write(struct dwapb_gpio *gpio, unsigned int offset,
 			       u32 val)
 {
-	struct bgpio_chip *bgc	= &gpio->ports[0].bgc;
+	struct gpio_chip *gc	= &gpio->ports[0].gc;
 	void __iomem *reg_base	= gpio->regs;
 
-	bgc->write_reg(reg_base + offset, val);
+	gc->write_reg(reg_base + offset, val);
 }
 
 static int dwapb_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	struct dwapb_gpio_port *port = to_dwapb_gpio_port(bgc);
+	struct dwapb_gpio_port *port = gpiochip_get_data(gc);
 	struct dwapb_gpio *gpio = port->gpio;
 
 	return irq_find_mapping(gpio->domain, offset);
@@ -119,7 +114,7 @@ static void dwapb_toggle_trigger(struct dwapb_gpio *gpio, unsigned int offs)
 {
 	u32 v = dwapb_read(gpio, GPIO_INT_POLARITY);
 
-	if (gpio_get_value(gpio->ports[0].bgc.gc.base + offs))
+	if (gpio_get_value(gpio->ports[0].gc.base + offs))
 		v &= ~BIT(offs);
 	else
 		v |= BIT(offs);
@@ -162,39 +157,39 @@ static void dwapb_irq_enable(struct irq_data *d)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
-	struct bgpio_chip *bgc = &gpio->ports[0].bgc;
+	struct gpio_chip *gc = &gpio->ports[0].gc;
 	unsigned long flags;
 	u32 val;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	val = dwapb_read(gpio, GPIO_INTEN);
 	val |= BIT(d->hwirq);
 	dwapb_write(gpio, GPIO_INTEN, val);
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 static void dwapb_irq_disable(struct irq_data *d)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
-	struct bgpio_chip *bgc = &gpio->ports[0].bgc;
+	struct gpio_chip *gc = &gpio->ports[0].gc;
 	unsigned long flags;
 	u32 val;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	val = dwapb_read(gpio, GPIO_INTEN);
 	val &= ~BIT(d->hwirq);
 	dwapb_write(gpio, GPIO_INTEN, val);
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 static int dwapb_irq_reqres(struct irq_data *d)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
-	struct bgpio_chip *bgc = &gpio->ports[0].bgc;
+	struct gpio_chip *gc = &gpio->ports[0].gc;
 
-	if (gpiochip_lock_as_irq(&bgc->gc, irqd_to_hwirq(d))) {
+	if (gpiochip_lock_as_irq(gc, irqd_to_hwirq(d))) {
 		dev_err(gpio->dev, "unable to lock HW IRQ %lu for IRQ\n",
 			irqd_to_hwirq(d));
 		return -EINVAL;
@@ -206,16 +201,16 @@ static void dwapb_irq_relres(struct irq_data *d)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
-	struct bgpio_chip *bgc = &gpio->ports[0].bgc;
+	struct gpio_chip *gc = &gpio->ports[0].gc;
 
-	gpiochip_unlock_as_irq(&bgc->gc, irqd_to_hwirq(d));
+	gpiochip_unlock_as_irq(gc, irqd_to_hwirq(d));
 }
 
 static int dwapb_irq_set_type(struct irq_data *d, u32 type)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
-	struct bgpio_chip *bgc = &gpio->ports[0].bgc;
+	struct gpio_chip *gc = &gpio->ports[0].gc;
 	int bit = d->hwirq;
 	unsigned long level, polarity, flags;
 
@@ -223,7 +218,7 @@ static int dwapb_irq_set_type(struct irq_data *d, u32 type)
 		     IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
 		return -EINVAL;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	level = dwapb_read(gpio, GPIO_INTTYPE_LEVEL);
 	polarity = dwapb_read(gpio, GPIO_INT_POLARITY);
 
@@ -254,7 +249,7 @@ static int dwapb_irq_set_type(struct irq_data *d, u32 type)
 
 	dwapb_write(gpio, GPIO_INTTYPE_LEVEL, level);
 	dwapb_write(gpio, GPIO_INT_POLARITY, polarity);
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
@@ -262,13 +257,12 @@ static int dwapb_irq_set_type(struct irq_data *d, u32 type)
 static int dwapb_gpio_set_debounce(struct gpio_chip *gc,
 				   unsigned offset, unsigned debounce)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	struct dwapb_gpio_port *port = to_dwapb_gpio_port(bgc);
+	struct dwapb_gpio_port *port = gpiochip_get_data(gc);
 	struct dwapb_gpio *gpio = port->gpio;
 	unsigned long flags, val_deb;
-	unsigned long mask = bgc->pin2mask(bgc, offset);
+	unsigned long mask = gc->pin2mask(gc, offset);
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
 	val_deb = dwapb_read(gpio, GPIO_PORTA_DEBOUNCE);
 	if (debounce)
@@ -276,7 +270,7 @@ static int dwapb_gpio_set_debounce(struct gpio_chip *gc,
 	else
 		dwapb_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb & ~mask);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
@@ -295,7 +289,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 				 struct dwapb_gpio_port *port,
 				 struct dwapb_port_property *pp)
 {
-	struct gpio_chip *gc = &port->bgc.gc;
+	struct gpio_chip *gc = &port->gc;
 	struct device_node *node = pp->node;
 	struct irq_chip_generic	*irq_gc = NULL;
 	unsigned int hwirq, ngpio = gc->ngpio;
@@ -369,13 +363,13 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 	for (hwirq = 0 ; hwirq < ngpio ; hwirq++)
 		irq_create_mapping(gpio->domain, hwirq);
 
-	port->bgc.gc.to_irq = dwapb_gpio_to_irq;
+	port->gc.to_irq = dwapb_gpio_to_irq;
 }
 
 static void dwapb_irq_teardown(struct dwapb_gpio *gpio)
 {
 	struct dwapb_gpio_port *port = &gpio->ports[0];
-	struct gpio_chip *gc = &port->bgc.gc;
+	struct gpio_chip *gc = &port->gc;
 	unsigned int ngpio = gc->ngpio;
 	irq_hw_number_t hwirq;
 
@@ -412,7 +406,7 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 	dirout = gpio->regs + GPIO_SWPORTA_DDR +
 		(pp->idx * GPIO_SWPORT_DDR_SIZE);
 
-	err = bgpio_init(&port->bgc, gpio->dev, 4, dat, set, NULL, dirout,
+	err = bgpio_init(&port->gc, gpio->dev, 4, dat, set, NULL, dirout,
 			 NULL, false);
 	if (err) {
 		dev_err(gpio->dev, "failed to init gpio chip for %s\n",
@@ -421,19 +415,19 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 	}
 
 #ifdef CONFIG_OF_GPIO
-	port->bgc.gc.of_node = pp->node;
+	port->gc.of_node = pp->node;
 #endif
-	port->bgc.gc.ngpio = pp->ngpio;
-	port->bgc.gc.base = pp->gpio_base;
+	port->gc.ngpio = pp->ngpio;
+	port->gc.base = pp->gpio_base;
 
 	/* Only port A support debounce */
 	if (pp->idx == 0)
-		port->bgc.gc.set_debounce = dwapb_gpio_set_debounce;
+		port->gc.set_debounce = dwapb_gpio_set_debounce;
 
 	if (pp->irq)
 		dwapb_configure_irqs(gpio, port, pp);
 
-	err = gpiochip_add(&port->bgc.gc);
+	err = gpiochip_add_data(&port->gc, port);
 	if (err)
 		dev_err(gpio->dev, "failed to register gpiochip for %s\n",
 			pp->name);
@@ -449,7 +443,7 @@ static void dwapb_gpio_unregister(struct dwapb_gpio *gpio)
 
 	for (m = 0; m < gpio->nr_ports; ++m)
 		if (gpio->ports[m].is_registered)
-			gpiochip_remove(&gpio->ports[m].bgc.gc);
+			gpiochip_remove(&gpio->ports[m].gc);
 }
 
 static struct dwapb_platform_data *
@@ -591,11 +585,11 @@ static int dwapb_gpio_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct dwapb_gpio *gpio = platform_get_drvdata(pdev);
-	struct bgpio_chip *bgc	= &gpio->ports[0].bgc;
+	struct gpio_chip *gc	= &gpio->ports[0].gc;
 	unsigned long flags;
 	int i;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	for (i = 0; i < gpio->nr_ports; i++) {
 		unsigned int offset;
 		unsigned int idx = gpio->ports[i].idx;
@@ -624,7 +618,7 @@ static int dwapb_gpio_suspend(struct device *dev)
 			dwapb_write(gpio, GPIO_INTMASK, 0xffffffff);
 		}
 	}
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
@@ -633,11 +627,11 @@ static int dwapb_gpio_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct dwapb_gpio *gpio = platform_get_drvdata(pdev);
-	struct bgpio_chip *bgc	= &gpio->ports[0].bgc;
+	struct gpio_chip *gc	= &gpio->ports[0].gc;
 	unsigned long flags;
 	int i;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	for (i = 0; i < gpio->nr_ports; i++) {
 		unsigned int offset;
 		unsigned int idx = gpio->ports[i].idx;
@@ -666,7 +660,7 @@ static int dwapb_gpio_resume(struct device *dev)
 			dwapb_write(gpio, GPIO_PORTA_EOI, 0xffffffff);
 		}
 	}
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-em.c b/drivers/gpio/gpio-em.c
index 6bca1e125e12..8d32ccc980d9 100644
--- a/drivers/gpio/gpio-em.c
+++ b/drivers/gpio/gpio-em.c
@@ -103,7 +103,7 @@ static int em_gio_irq_reqres(struct irq_data *d)
 	struct em_gio_priv *p = irq_data_get_irq_chip_data(d);
 
 	if (gpiochip_lock_as_irq(&p->gpio_chip, irqd_to_hwirq(d))) {
-		dev_err(p->gpio_chip.dev,
+		dev_err(p->gpio_chip.parent,
 			"unable to lock HW IRQ %lu for IRQ\n",
 			irqd_to_hwirq(d));
 		return -EINVAL;
@@ -192,7 +192,7 @@ static irqreturn_t em_gio_irq_handler(int irq, void *dev_id)
 
 static inline struct em_gio_priv *gpio_to_priv(struct gpio_chip *chip)
 {
-	return container_of(chip, struct em_gio_priv, gpio_chip);
+	return gpiochip_get_data(chip);
 }
 
 static int em_gio_direction_input(struct gpio_chip *chip, unsigned offset)
@@ -203,7 +203,7 @@ static int em_gio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int em_gio_get(struct gpio_chip *chip, unsigned offset)
 {
-	return (int)(em_gio_read(gpio_to_priv(chip), GIO_I) & BIT(offset));
+	return !!(em_gio_read(gpio_to_priv(chip), GIO_I) & BIT(offset));
 }
 
 static void __em_gio_set(struct gpio_chip *chip, unsigned int reg,
@@ -332,7 +332,7 @@ static int em_gio_probe(struct platform_device *pdev)
 	gpio_chip->request = em_gio_request;
 	gpio_chip->free = em_gio_free;
 	gpio_chip->label = name;
-	gpio_chip->dev = &pdev->dev;
+	gpio_chip->parent = &pdev->dev;
 	gpio_chip->owner = THIS_MODULE;
 	gpio_chip->base = -1;
 	gpio_chip->ngpio = ngpios;
@@ -368,7 +368,7 @@ static int em_gio_probe(struct platform_device *pdev)
 		goto err1;
 	}
 
-	ret = gpiochip_add(gpio_chip);
+	ret = gpiochip_add_data(gpio_chip, p);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to add GPIO controller\n");
 		goto err1;
diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c
index 3e3947b35c83..ad279078fed7 100644
--- a/drivers/gpio/gpio-ep93xx.c
+++ b/drivers/gpio/gpio-ep93xx.c
@@ -16,10 +16,11 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
-#include <linux/gpio.h>
 #include <linux/irq.h>
 #include <linux/slab.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
+/* FIXME: this is here for gpio_to_irq() - get rid of this! */
+#include <linux/gpio.h>
 
 #include <mach/hardware.h>
 #include <mach/gpio-ep93xx.h>
@@ -28,7 +29,7 @@
 
 struct ep93xx_gpio {
 	void __iomem		*mmio_base;
-	struct bgpio_chip	bgc[8];
+	struct gpio_chip	gc[8];
 };
 
 /*************************************************************************
@@ -319,26 +320,26 @@ static int ep93xx_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 	return 64 + gpio;
 }
 
-static int ep93xx_gpio_add_bank(struct bgpio_chip *bgc, struct device *dev,
+static int ep93xx_gpio_add_bank(struct gpio_chip *gc, struct device *dev,
 	void __iomem *mmio_base, struct ep93xx_gpio_bank *bank)
 {
 	void __iomem *data = mmio_base + bank->data;
 	void __iomem *dir =  mmio_base + bank->dir;
 	int err;
 
-	err = bgpio_init(bgc, dev, 1, data, NULL, NULL, dir, NULL, 0);
+	err = bgpio_init(gc, dev, 1, data, NULL, NULL, dir, NULL, 0);
 	if (err)
 		return err;
 
-	bgc->gc.label = bank->label;
-	bgc->gc.base = bank->base;
+	gc->label = bank->label;
+	gc->base = bank->base;
 
 	if (bank->has_debounce) {
-		bgc->gc.set_debounce = ep93xx_gpio_set_debounce;
-		bgc->gc.to_irq = ep93xx_gpio_to_irq;
+		gc->set_debounce = ep93xx_gpio_set_debounce;
+		gc->to_irq = ep93xx_gpio_to_irq;
 	}
 
-	return gpiochip_add(&bgc->gc);
+	return gpiochip_add_data(gc, NULL);
 }
 
 static int ep93xx_gpio_probe(struct platform_device *pdev)
@@ -358,10 +359,10 @@ static int ep93xx_gpio_probe(struct platform_device *pdev)
 		return PTR_ERR(ep93xx_gpio->mmio_base);
 
 	for (i = 0; i < ARRAY_SIZE(ep93xx_gpio_banks); i++) {
-		struct bgpio_chip *bgc = &ep93xx_gpio->bgc[i];
+		struct gpio_chip *gc = &ep93xx_gpio->gc[i];
 		struct ep93xx_gpio_bank *bank = &ep93xx_gpio_banks[i];
 
-		if (ep93xx_gpio_add_bank(bgc, &pdev->dev,
+		if (ep93xx_gpio_add_bank(gc, &pdev->dev,
 					 ep93xx_gpio->mmio_base, bank))
 			dev_warn(&pdev->dev, "Unable to add gpio bank %s\n",
 				bank->label);
diff --git a/drivers/gpio/gpio-etraxfs.c b/drivers/gpio/gpio-etraxfs.c
index 5c15dd12172d..00b022c9acb3 100644
--- a/drivers/gpio/gpio-etraxfs.c
+++ b/drivers/gpio/gpio-etraxfs.c
@@ -1,12 +1,10 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
-#include <linux/gpio.h>
 #include <linux/gpio/driver.h>
 #include <linux/of_gpio.h>
 #include <linux/io.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
-#include <linux/basic_mmio_gpio.h>
 
 #define ETRAX_FS_rw_pa_dout	0
 #define ETRAX_FS_r_pa_din	4
@@ -67,7 +65,7 @@ struct etraxfs_gpio_block {
 };
 
 struct etraxfs_gpio_chip {
-	struct bgpio_chip bgc;
+	struct gpio_chip gc;
 	struct etraxfs_gpio_block *block;
 };
 
@@ -176,11 +174,6 @@ static const struct etraxfs_gpio_info etraxfs_gpio_artpec3 = {
 	.rw_intr_pins	= ARTPEC3_rw_intr_pins,
 };
 
-static struct etraxfs_gpio_chip *to_etraxfs(struct gpio_chip *gc)
-{
-	return container_of(gc, struct etraxfs_gpio_chip, bgc.gc);
-}
-
 static unsigned int etraxfs_gpio_chip_to_port(struct gpio_chip *gc)
 {
 	return gc->label[0] - 'A';
@@ -220,13 +213,13 @@ static unsigned int etraxfs_gpio_to_group_irq(unsigned int gpio)
 static unsigned int etraxfs_gpio_to_group_pin(struct etraxfs_gpio_chip *chip,
 					      unsigned int gpio)
 {
-	return 4 * etraxfs_gpio_chip_to_port(&chip->bgc.gc) + gpio / 8;
+	return 4 * etraxfs_gpio_chip_to_port(&chip->gc) + gpio / 8;
 }
 
 static void etraxfs_gpio_irq_ack(struct irq_data *d)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 
@@ -236,7 +229,7 @@ static void etraxfs_gpio_irq_ack(struct irq_data *d)
 static void etraxfs_gpio_irq_mask(struct irq_data *d)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 
@@ -249,7 +242,7 @@ static void etraxfs_gpio_irq_mask(struct irq_data *d)
 static void etraxfs_gpio_irq_unmask(struct irq_data *d)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 
@@ -262,7 +255,7 @@ static void etraxfs_gpio_irq_unmask(struct irq_data *d)
 static int etraxfs_gpio_irq_set_type(struct irq_data *d, u32 type)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 	u32 cfg;
@@ -299,7 +292,7 @@ static int etraxfs_gpio_irq_set_type(struct irq_data *d, u32 type)
 static int etraxfs_gpio_irq_request_resources(struct irq_data *d)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 	int ret = -EBUSY;
@@ -308,7 +301,7 @@ static int etraxfs_gpio_irq_request_resources(struct irq_data *d)
 	if (block->group[grpirq])
 		goto out;
 
-	ret = gpiochip_lock_as_irq(&chip->bgc.gc, d->hwirq);
+	ret = gpiochip_lock_as_irq(&chip->gc, d->hwirq);
 	if (ret)
 		goto out;
 
@@ -330,13 +323,13 @@ out:
 static void etraxfs_gpio_irq_release_resources(struct irq_data *d)
 {
 	struct etraxfs_gpio_chip *chip =
-		to_etraxfs(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	struct etraxfs_gpio_block *block = chip->block;
 	unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq);
 
 	spin_lock(&block->lock);
 	block->group[grpirq] = 0;
-	gpiochip_unlock_as_irq(&chip->bgc.gc, d->hwirq);
+	gpiochip_unlock_as_irq(&chip->gc, d->hwirq);
 	spin_unlock(&block->lock);
 }
 
@@ -419,7 +412,7 @@ static int etraxfs_gpio_probe(struct platform_device *pdev)
 
 	for (i = 0; i < info->num_ports; i++) {
 		struct etraxfs_gpio_chip *chip = &chips[i];
-		struct bgpio_chip *bgc = &chip->bgc;
+		struct gpio_chip *gc = &chip->gc;
 		const struct etraxfs_gpio_port *port = &info->ports[i];
 		unsigned long flags = BGPIOF_READ_OUTPUT_REG_SET;
 		void __iomem *dat = regs + port->din;
@@ -433,7 +426,7 @@ static int etraxfs_gpio_probe(struct platform_device *pdev)
 			flags = BGPIOF_NO_OUTPUT;
 		}
 
-		ret = bgpio_init(bgc, dev, 4,
+		ret = bgpio_init(gc, dev, 4,
 				 dat, set, NULL, dirout, NULL,
 				 flags);
 		if (ret) {
@@ -442,28 +435,28 @@ static int etraxfs_gpio_probe(struct platform_device *pdev)
 			continue;
 		}
 
-		bgc->gc.ngpio = port->ngpio;
-		bgc->gc.label = port->label;
+		gc->ngpio = port->ngpio;
+		gc->label = port->label;
 
-		bgc->gc.of_node = dev->of_node;
-		bgc->gc.of_gpio_n_cells = 3;
-		bgc->gc.of_xlate = etraxfs_gpio_of_xlate;
+		gc->of_node = dev->of_node;
+		gc->of_gpio_n_cells = 3;
+		gc->of_xlate = etraxfs_gpio_of_xlate;
 
-		ret = gpiochip_add(&bgc->gc);
+		ret = gpiochip_add_data(gc, chip);
 		if (ret) {
 			dev_err(dev, "Unable to register port %s\n",
-				bgc->gc.label);
+				gc->label);
 			continue;
 		}
 
 		if (i > 0 && !allportsirq)
 			continue;
 
-		ret = gpiochip_irqchip_add(&bgc->gc, &etraxfs_gpio_irq_chip, 0,
+		ret = gpiochip_irqchip_add(gc, &etraxfs_gpio_irq_chip, 0,
 					   handle_level_irq, IRQ_TYPE_NONE);
 		if (ret) {
 			dev_err(dev, "Unable to add irqchip to port %s\n",
-				bgc->gc.label);
+				gc->label);
 		}
 	}
 
diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c
index 5e3c4fa67d82..d62fd6bbaf82 100644
--- a/drivers/gpio/gpio-f7188x.c
+++ b/drivers/gpio/gpio-f7188x.c
@@ -193,8 +193,7 @@ static struct f7188x_gpio_bank f71889_gpio_bank[] = {
 static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
 	int err;
-	struct f7188x_gpio_bank *bank =
-		container_of(chip, struct f7188x_gpio_bank, chip);
+	struct f7188x_gpio_bank *bank = gpiochip_get_data(chip);
 	struct f7188x_sio *sio = bank->data->sio;
 	u8 dir;
 
@@ -215,8 +214,7 @@ static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 static int f7188x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	int err;
-	struct f7188x_gpio_bank *bank =
-		container_of(chip, struct f7188x_gpio_bank, chip);
+	struct f7188x_gpio_bank *bank = gpiochip_get_data(chip);
 	struct f7188x_sio *sio = bank->data->sio;
 	u8 dir, data;
 
@@ -241,8 +239,7 @@ static int f7188x_gpio_direction_out(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
 	int err;
-	struct f7188x_gpio_bank *bank =
-		container_of(chip, struct f7188x_gpio_bank, chip);
+	struct f7188x_gpio_bank *bank = gpiochip_get_data(chip);
 	struct f7188x_sio *sio = bank->data->sio;
 	u8 dir, data_out;
 
@@ -270,8 +267,7 @@ static int f7188x_gpio_direction_out(struct gpio_chip *chip,
 static void f7188x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
 	int err;
-	struct f7188x_gpio_bank *bank =
-		container_of(chip, struct f7188x_gpio_bank, chip);
+	struct f7188x_gpio_bank *bank = gpiochip_get_data(chip);
 	struct f7188x_sio *sio = bank->data->sio;
 	u8 data_out;
 
@@ -298,7 +294,7 @@ static int f7188x_gpio_probe(struct platform_device *pdev)
 {
 	int err;
 	int i;
-	struct f7188x_sio *sio = pdev->dev.platform_data;
+	struct f7188x_sio *sio = dev_get_platdata(&pdev->dev);
 	struct f7188x_gpio_data *data;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
@@ -333,10 +329,10 @@ static int f7188x_gpio_probe(struct platform_device *pdev)
 	for (i = 0; i < data->nr_bank; i++) {
 		struct f7188x_gpio_bank *bank = &data->bank[i];
 
-		bank->chip.dev = &pdev->dev;
+		bank->chip.parent = &pdev->dev;
 		bank->data = data;
 
-		err = gpiochip_add(&bank->chip);
+		err = gpiochip_add_data(&bank->chip, bank);
 		if (err) {
 			dev_err(&pdev->dev,
 				"Failed to register gpiochip %d: %d\n",
diff --git a/drivers/gpio/gpio-ge.c b/drivers/gpio/gpio-ge.c
index f9ac3f351753..cbbec838a9d1 100644
--- a/drivers/gpio/gpio-ge.c
+++ b/drivers/gpio/gpio-ge.c
@@ -24,7 +24,7 @@
 #include <linux/of_gpio.h>
 #include <linux/of_address.h>
 #include <linux/module.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 
 #define GEF_GPIO_DIRECT		0x00
 #define GEF_GPIO_IN		0x04
@@ -55,19 +55,19 @@ static int __init gef_gpio_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *of_id =
 		of_match_device(gef_gpio_ids, &pdev->dev);
-	struct bgpio_chip *bgc;
+	struct gpio_chip *gc;
 	void __iomem *regs;
 	int ret;
 
-	bgc = devm_kzalloc(&pdev->dev, sizeof(*bgc), GFP_KERNEL);
-	if (!bgc)
+	gc = devm_kzalloc(&pdev->dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
 		return -ENOMEM;
 
 	regs = of_iomap(pdev->dev.of_node, 0);
 	if (!regs)
 		return -ENOMEM;
 
-	ret = bgpio_init(bgc, &pdev->dev, 4, regs + GEF_GPIO_IN,
+	ret = bgpio_init(gc, &pdev->dev, 4, regs + GEF_GPIO_IN,
 			 regs + GEF_GPIO_OUT, NULL, NULL,
 			 regs + GEF_GPIO_DIRECT, BGPIOF_BIG_ENDIAN_BYTE_ORDER);
 	if (ret) {
@@ -76,20 +76,20 @@ static int __init gef_gpio_probe(struct platform_device *pdev)
 	}
 
 	/* Setup pointers to chip functions */
-	bgc->gc.label = devm_kstrdup(&pdev->dev, pdev->dev.of_node->full_name,
+	gc->label = devm_kstrdup(&pdev->dev, pdev->dev.of_node->full_name,
 				     GFP_KERNEL);
-	if (!bgc->gc.label) {
+	if (!gc->label) {
 		ret = -ENOMEM;
 		goto err0;
 	}
 
-	bgc->gc.base = -1;
-	bgc->gc.ngpio = (u16)(uintptr_t)of_id->data;
-	bgc->gc.of_gpio_n_cells = 2;
-	bgc->gc.of_node = pdev->dev.of_node;
+	gc->base = -1;
+	gc->ngpio = (u16)(uintptr_t)of_id->data;
+	gc->of_gpio_n_cells = 2;
+	gc->of_node = pdev->dev.of_node;
 
 	/* This function adds a memory mapped GPIO chip */
-	ret = gpiochip_add(&bgc->gc);
+	ret = gpiochip_add_data(gc, NULL);
 	if (ret)
 		goto err0;
 
diff --git a/drivers/gpio/gpio-generic.c b/drivers/gpio/gpio-generic.c
index 88ae70ddb127..2a4f2333a50b 100644
--- a/drivers/gpio/gpio-generic.c
+++ b/drivers/gpio/gpio-generic.c
@@ -56,11 +56,11 @@ o        `                     ~~~~\___/~~~~    ` controller in FPGA is ,.`
 #include <linux/log2.h>
 #include <linux/ioport.h>
 #include <linux/io.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/slab.h>
+#include <linux/bitops.h>
 #include <linux/platform_device.h>
 #include <linux/mod_devicetable.h>
-#include <linux/basic_mmio_gpio.h>
 
 static void bgpio_write8(void __iomem *reg, unsigned long data)
 {
@@ -124,33 +124,30 @@ static unsigned long bgpio_read32be(void __iomem *reg)
 	return ioread32be(reg);
 }
 
-static unsigned long bgpio_pin2mask(struct bgpio_chip *bgc, unsigned int pin)
+static unsigned long bgpio_pin2mask(struct gpio_chip *gc, unsigned int pin)
 {
-	return 1 << pin;
+	return BIT(pin);
 }
 
-static unsigned long bgpio_pin2mask_be(struct bgpio_chip *bgc,
+static unsigned long bgpio_pin2mask_be(struct gpio_chip *gc,
 				       unsigned int pin)
 {
-	return 1 << (bgc->bits - 1 - pin);
+	return BIT(gc->bgpio_bits - 1 - pin);
 }
 
 static int bgpio_get_set(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	unsigned long pinmask = bgc->pin2mask(bgc, gpio);
+	unsigned long pinmask = gc->pin2mask(gc, gpio);
 
-	if (bgc->dir & pinmask)
-		return !!(bgc->read_reg(bgc->reg_set) & pinmask);
+	if (gc->bgpio_dir & pinmask)
+		return !!(gc->read_reg(gc->reg_set) & pinmask);
 	else
-		return !!(bgc->read_reg(bgc->reg_dat) & pinmask);
+		return !!(gc->read_reg(gc->reg_dat) & pinmask);
 }
 
 static int bgpio_get(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return !!(bgc->read_reg(bgc->reg_dat) & bgc->pin2mask(bgc, gpio));
+	return !!(gc->read_reg(gc->reg_dat) & gc->pin2mask(gc, gpio));
 }
 
 static void bgpio_set_none(struct gpio_chip *gc, unsigned int gpio, int val)
@@ -159,53 +156,50 @@ static void bgpio_set_none(struct gpio_chip *gc, unsigned int gpio, int val)
 
 static void bgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	unsigned long mask = bgc->pin2mask(bgc, gpio);
+	unsigned long mask = gc->pin2mask(gc, gpio);
 	unsigned long flags;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
 	if (val)
-		bgc->data |= mask;
+		gc->bgpio_data |= mask;
 	else
-		bgc->data &= ~mask;
+		gc->bgpio_data &= ~mask;
 
-	bgc->write_reg(bgc->reg_dat, bgc->data);
+	gc->write_reg(gc->reg_dat, gc->bgpio_data);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 static void bgpio_set_with_clear(struct gpio_chip *gc, unsigned int gpio,
 				 int val)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	unsigned long mask = bgc->pin2mask(bgc, gpio);
+	unsigned long mask = gc->pin2mask(gc, gpio);
 
 	if (val)
-		bgc->write_reg(bgc->reg_set, mask);
+		gc->write_reg(gc->reg_set, mask);
 	else
-		bgc->write_reg(bgc->reg_clr, mask);
+		gc->write_reg(gc->reg_clr, mask);
 }
 
 static void bgpio_set_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	unsigned long mask = bgc->pin2mask(bgc, gpio);
+	unsigned long mask = gc->pin2mask(gc, gpio);
 	unsigned long flags;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
 	if (val)
-		bgc->data |= mask;
+		gc->bgpio_data |= mask;
 	else
-		bgc->data &= ~mask;
+		gc->bgpio_data &= ~mask;
 
-	bgc->write_reg(bgc->reg_set, bgc->data);
+	gc->write_reg(gc->reg_set, gc->bgpio_data);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
-static void bgpio_multiple_get_masks(struct bgpio_chip *bgc,
+static void bgpio_multiple_get_masks(struct gpio_chip *gc,
 				     unsigned long *mask, unsigned long *bits,
 				     unsigned long *set_mask,
 				     unsigned long *clear_mask)
@@ -215,19 +209,19 @@ static void bgpio_multiple_get_masks(struct bgpio_chip *bgc,
 	*set_mask = 0;
 	*clear_mask = 0;
 
-	for (i = 0; i < bgc->bits; i++) {
+	for (i = 0; i < gc->bgpio_bits; i++) {
 		if (*mask == 0)
 			break;
 		if (__test_and_clear_bit(i, mask)) {
 			if (test_bit(i, bits))
-				*set_mask |= bgc->pin2mask(bgc, i);
+				*set_mask |= gc->pin2mask(gc, i);
 			else
-				*clear_mask |= bgc->pin2mask(bgc, i);
+				*clear_mask |= gc->pin2mask(gc, i);
 		}
 	}
 }
 
-static void bgpio_set_multiple_single_reg(struct bgpio_chip *bgc,
+static void bgpio_set_multiple_single_reg(struct gpio_chip *gc,
 					  unsigned long *mask,
 					  unsigned long *bits,
 					  void __iomem *reg)
@@ -235,47 +229,42 @@ static void bgpio_set_multiple_single_reg(struct bgpio_chip *bgc,
 	unsigned long flags;
 	unsigned long set_mask, clear_mask;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
-	bgpio_multiple_get_masks(bgc, mask, bits, &set_mask, &clear_mask);
+	bgpio_multiple_get_masks(gc, mask, bits, &set_mask, &clear_mask);
 
-	bgc->data |= set_mask;
-	bgc->data &= ~clear_mask;
+	gc->bgpio_data |= set_mask;
+	gc->bgpio_data &= ~clear_mask;
 
-	bgc->write_reg(reg, bgc->data);
+	gc->write_reg(reg, gc->bgpio_data);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 static void bgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
 			       unsigned long *bits)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	bgpio_set_multiple_single_reg(bgc, mask, bits, bgc->reg_dat);
+	bgpio_set_multiple_single_reg(gc, mask, bits, gc->reg_dat);
 }
 
 static void bgpio_set_multiple_set(struct gpio_chip *gc, unsigned long *mask,
 				   unsigned long *bits)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	bgpio_set_multiple_single_reg(bgc, mask, bits, bgc->reg_set);
+	bgpio_set_multiple_single_reg(gc, mask, bits, gc->reg_set);
 }
 
 static void bgpio_set_multiple_with_clear(struct gpio_chip *gc,
 					  unsigned long *mask,
 					  unsigned long *bits)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
 	unsigned long set_mask, clear_mask;
 
-	bgpio_multiple_get_masks(bgc, mask, bits, &set_mask, &clear_mask);
+	bgpio_multiple_get_masks(gc, mask, bits, &set_mask, &clear_mask);
 
 	if (set_mask)
-		bgc->write_reg(bgc->reg_set, set_mask);
+		gc->write_reg(gc->reg_set, set_mask);
 	if (clear_mask)
-		bgc->write_reg(bgc->reg_clr, clear_mask);
+		gc->write_reg(gc->reg_clr, clear_mask);
 }
 
 static int bgpio_simple_dir_in(struct gpio_chip *gc, unsigned int gpio)
@@ -299,111 +288,103 @@ static int bgpio_simple_dir_out(struct gpio_chip *gc, unsigned int gpio,
 
 static int bgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
 	unsigned long flags;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
-	bgc->dir &= ~bgc->pin2mask(bgc, gpio);
-	bgc->write_reg(bgc->reg_dir, bgc->dir);
+	gc->bgpio_dir &= ~gc->pin2mask(gc, gpio);
+	gc->write_reg(gc->reg_dir, gc->bgpio_dir);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
 
 static int bgpio_get_dir(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return (bgc->read_reg(bgc->reg_dir) & bgc->pin2mask(bgc, gpio)) ?
-	       GPIOF_DIR_OUT : GPIOF_DIR_IN;
+	/* Return 0 if output, 1 of input */
+	return !(gc->read_reg(gc->reg_dir) & gc->pin2mask(gc, gpio));
 }
 
 static int bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
 	unsigned long flags;
 
 	gc->set(gc, gpio, val);
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
-	bgc->dir |= bgc->pin2mask(bgc, gpio);
-	bgc->write_reg(bgc->reg_dir, bgc->dir);
+	gc->bgpio_dir |= gc->pin2mask(gc, gpio);
+	gc->write_reg(gc->reg_dir, gc->bgpio_dir);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
 
 static int bgpio_dir_in_inv(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
 	unsigned long flags;
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
-	bgc->dir |= bgc->pin2mask(bgc, gpio);
-	bgc->write_reg(bgc->reg_dir, bgc->dir);
+	gc->bgpio_dir |= gc->pin2mask(gc, gpio);
+	gc->write_reg(gc->reg_dir, gc->bgpio_dir);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
 
 static int bgpio_dir_out_inv(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
 	unsigned long flags;
 
 	gc->set(gc, gpio, val);
 
-	spin_lock_irqsave(&bgc->lock, flags);
+	spin_lock_irqsave(&gc->bgpio_lock, flags);
 
-	bgc->dir &= ~bgc->pin2mask(bgc, gpio);
-	bgc->write_reg(bgc->reg_dir, bgc->dir);
+	gc->bgpio_dir &= ~gc->pin2mask(gc, gpio);
+	gc->write_reg(gc->reg_dir, gc->bgpio_dir);
 
-	spin_unlock_irqrestore(&bgc->lock, flags);
+	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
 }
 
 static int bgpio_get_dir_inv(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return (bgc->read_reg(bgc->reg_dir) & bgc->pin2mask(bgc, gpio)) ?
-	       GPIOF_DIR_IN : GPIOF_DIR_OUT;
+	/* Return 0 if output, 1 if input */
+	return !!(gc->read_reg(gc->reg_dir) & gc->pin2mask(gc, gpio));
 }
 
 static int bgpio_setup_accessors(struct device *dev,
-				 struct bgpio_chip *bgc,
+				 struct gpio_chip *gc,
 				 bool bit_be,
 				 bool byte_be)
 {
 
-	switch (bgc->bits) {
+	switch (gc->bgpio_bits) {
 	case 8:
-		bgc->read_reg	= bgpio_read8;
-		bgc->write_reg	= bgpio_write8;
+		gc->read_reg	= bgpio_read8;
+		gc->write_reg	= bgpio_write8;
 		break;
 	case 16:
 		if (byte_be) {
-			bgc->read_reg	= bgpio_read16be;
-			bgc->write_reg	= bgpio_write16be;
+			gc->read_reg	= bgpio_read16be;
+			gc->write_reg	= bgpio_write16be;
 		} else {
-			bgc->read_reg	= bgpio_read16;
-			bgc->write_reg	= bgpio_write16;
+			gc->read_reg	= bgpio_read16;
+			gc->write_reg	= bgpio_write16;
 		}
 		break;
 	case 32:
 		if (byte_be) {
-			bgc->read_reg	= bgpio_read32be;
-			bgc->write_reg	= bgpio_write32be;
+			gc->read_reg	= bgpio_read32be;
+			gc->write_reg	= bgpio_write32be;
 		} else {
-			bgc->read_reg	= bgpio_read32;
-			bgc->write_reg	= bgpio_write32;
+			gc->read_reg	= bgpio_read32;
+			gc->write_reg	= bgpio_write32;
 		}
 		break;
 #if BITS_PER_LONG >= 64
@@ -413,17 +394,17 @@ static int bgpio_setup_accessors(struct device *dev,
 				"64 bit big endian byte order unsupported\n");
 			return -EINVAL;
 		} else {
-			bgc->read_reg	= bgpio_read64;
-			bgc->write_reg	= bgpio_write64;
+			gc->read_reg	= bgpio_read64;
+			gc->write_reg	= bgpio_write64;
 		}
 		break;
 #endif /* BITS_PER_LONG >= 64 */
 	default:
-		dev_err(dev, "unsupported data width %u bits\n", bgc->bits);
+		dev_err(dev, "unsupported data width %u bits\n", gc->bgpio_bits);
 		return -EINVAL;
 	}
 
-	bgc->pin2mask = bit_be ? bgpio_pin2mask_be : bgpio_pin2mask;
+	gc->pin2mask = bit_be ? bgpio_pin2mask_be : bgpio_pin2mask;
 
 	return 0;
 }
@@ -450,44 +431,44 @@ static int bgpio_setup_accessors(struct device *dev,
  *	- an input direction register (named "dirin") where a 1 bit indicates
  *	the GPIO is an input.
  */
-static int bgpio_setup_io(struct bgpio_chip *bgc,
+static int bgpio_setup_io(struct gpio_chip *gc,
 			  void __iomem *dat,
 			  void __iomem *set,
 			  void __iomem *clr,
 			  unsigned long flags)
 {
 
-	bgc->reg_dat = dat;
-	if (!bgc->reg_dat)
+	gc->reg_dat = dat;
+	if (!gc->reg_dat)
 		return -EINVAL;
 
 	if (set && clr) {
-		bgc->reg_set = set;
-		bgc->reg_clr = clr;
-		bgc->gc.set = bgpio_set_with_clear;
-		bgc->gc.set_multiple = bgpio_set_multiple_with_clear;
+		gc->reg_set = set;
+		gc->reg_clr = clr;
+		gc->set = bgpio_set_with_clear;
+		gc->set_multiple = bgpio_set_multiple_with_clear;
 	} else if (set && !clr) {
-		bgc->reg_set = set;
-		bgc->gc.set = bgpio_set_set;
-		bgc->gc.set_multiple = bgpio_set_multiple_set;
+		gc->reg_set = set;
+		gc->set = bgpio_set_set;
+		gc->set_multiple = bgpio_set_multiple_set;
 	} else if (flags & BGPIOF_NO_OUTPUT) {
-		bgc->gc.set = bgpio_set_none;
-		bgc->gc.set_multiple = NULL;
+		gc->set = bgpio_set_none;
+		gc->set_multiple = NULL;
 	} else {
-		bgc->gc.set = bgpio_set;
-		bgc->gc.set_multiple = bgpio_set_multiple;
+		gc->set = bgpio_set;
+		gc->set_multiple = bgpio_set_multiple;
 	}
 
 	if (!(flags & BGPIOF_UNREADABLE_REG_SET) &&
 	    (flags & BGPIOF_READ_OUTPUT_REG_SET))
-		bgc->gc.get = bgpio_get_set;
+		gc->get = bgpio_get_set;
 	else
-		bgc->gc.get = bgpio_get;
+		gc->get = bgpio_get;
 
 	return 0;
 }
 
-static int bgpio_setup_direction(struct bgpio_chip *bgc,
+static int bgpio_setup_direction(struct gpio_chip *gc,
 				 void __iomem *dirout,
 				 void __iomem *dirin,
 				 unsigned long flags)
@@ -495,21 +476,21 @@ static int bgpio_setup_direction(struct bgpio_chip *bgc,
 	if (dirout && dirin) {
 		return -EINVAL;
 	} else if (dirout) {
-		bgc->reg_dir = dirout;
-		bgc->gc.direction_output = bgpio_dir_out;
-		bgc->gc.direction_input = bgpio_dir_in;
-		bgc->gc.get_direction = bgpio_get_dir;
+		gc->reg_dir = dirout;
+		gc->direction_output = bgpio_dir_out;
+		gc->direction_input = bgpio_dir_in;
+		gc->get_direction = bgpio_get_dir;
 	} else if (dirin) {
-		bgc->reg_dir = dirin;
-		bgc->gc.direction_output = bgpio_dir_out_inv;
-		bgc->gc.direction_input = bgpio_dir_in_inv;
-		bgc->gc.get_direction = bgpio_get_dir_inv;
+		gc->reg_dir = dirin;
+		gc->direction_output = bgpio_dir_out_inv;
+		gc->direction_input = bgpio_dir_in_inv;
+		gc->get_direction = bgpio_get_dir_inv;
 	} else {
 		if (flags & BGPIOF_NO_OUTPUT)
-			bgc->gc.direction_output = bgpio_dir_out_err;
+			gc->direction_output = bgpio_dir_out_err;
 		else
-			bgc->gc.direction_output = bgpio_simple_dir_out;
-		bgc->gc.direction_input = bgpio_simple_dir_in;
+			gc->direction_output = bgpio_simple_dir_out;
+		gc->direction_input = bgpio_simple_dir_in;
 	}
 
 	return 0;
@@ -523,14 +504,7 @@ static int bgpio_request(struct gpio_chip *chip, unsigned gpio_pin)
 	return -EINVAL;
 }
 
-int bgpio_remove(struct bgpio_chip *bgc)
-{
-	gpiochip_remove(&bgc->gc);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(bgpio_remove);
-
-int bgpio_init(struct bgpio_chip *bgc, struct device *dev,
+int bgpio_init(struct gpio_chip *gc, struct device *dev,
 	       unsigned long sz, void __iomem *dat, void __iomem *set,
 	       void __iomem *clr, void __iomem *dirout, void __iomem *dirin,
 	       unsigned long flags)
@@ -540,36 +514,36 @@ int bgpio_init(struct bgpio_chip *bgc, struct device *dev,
 	if (!is_power_of_2(sz))
 		return -EINVAL;
 
-	bgc->bits = sz * 8;
-	if (bgc->bits > BITS_PER_LONG)
+	gc->bgpio_bits = sz * 8;
+	if (gc->bgpio_bits > BITS_PER_LONG)
 		return -EINVAL;
 
-	spin_lock_init(&bgc->lock);
-	bgc->gc.dev = dev;
-	bgc->gc.label = dev_name(dev);
-	bgc->gc.base = -1;
-	bgc->gc.ngpio = bgc->bits;
-	bgc->gc.request = bgpio_request;
+	spin_lock_init(&gc->bgpio_lock);
+	gc->parent = dev;
+	gc->label = dev_name(dev);
+	gc->base = -1;
+	gc->ngpio = gc->bgpio_bits;
+	gc->request = bgpio_request;
 
-	ret = bgpio_setup_io(bgc, dat, set, clr, flags);
+	ret = bgpio_setup_io(gc, dat, set, clr, flags);
 	if (ret)
 		return ret;
 
-	ret = bgpio_setup_accessors(dev, bgc, flags & BGPIOF_BIG_ENDIAN,
+	ret = bgpio_setup_accessors(dev, gc, flags & BGPIOF_BIG_ENDIAN,
 				    flags & BGPIOF_BIG_ENDIAN_BYTE_ORDER);
 	if (ret)
 		return ret;
 
-	ret = bgpio_setup_direction(bgc, dirout, dirin, flags);
+	ret = bgpio_setup_direction(gc, dirout, dirin, flags);
 	if (ret)
 		return ret;
 
-	bgc->data = bgc->read_reg(bgc->reg_dat);
-	if (bgc->gc.set == bgpio_set_set &&
+	gc->bgpio_data = gc->read_reg(gc->reg_dat);
+	if (gc->set == bgpio_set_set &&
 			!(flags & BGPIOF_UNREADABLE_REG_SET))
-		bgc->data = bgc->read_reg(bgc->reg_set);
-	if (bgc->reg_dir && !(flags & BGPIOF_UNREADABLE_REG_DIR))
-		bgc->dir = bgc->read_reg(bgc->reg_dir);
+		gc->bgpio_data = gc->read_reg(gc->reg_set);
+	if (gc->reg_dir && !(flags & BGPIOF_UNREADABLE_REG_DIR))
+		gc->bgpio_dir = gc->read_reg(gc->reg_dir);
 
 	return ret;
 }
@@ -607,7 +581,7 @@ static int bgpio_pdev_probe(struct platform_device *pdev)
 	unsigned long sz;
 	unsigned long flags = pdev->id_entry->driver_data;
 	int err;
-	struct bgpio_chip *bgc;
+	struct gpio_chip *gc;
 	struct bgpio_pdata *pdata = dev_get_platdata(dev);
 
 	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dat");
@@ -636,32 +610,33 @@ static int bgpio_pdev_probe(struct platform_device *pdev)
 	if (IS_ERR(dirin))
 		return PTR_ERR(dirin);
 
-	bgc = devm_kzalloc(&pdev->dev, sizeof(*bgc), GFP_KERNEL);
-	if (!bgc)
+	gc = devm_kzalloc(&pdev->dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
 		return -ENOMEM;
 
-	err = bgpio_init(bgc, dev, sz, dat, set, clr, dirout, dirin, flags);
+	err = bgpio_init(gc, dev, sz, dat, set, clr, dirout, dirin, flags);
 	if (err)
 		return err;
 
 	if (pdata) {
 		if (pdata->label)
-			bgc->gc.label = pdata->label;
-		bgc->gc.base = pdata->base;
+			gc->label = pdata->label;
+		gc->base = pdata->base;
 		if (pdata->ngpio > 0)
-			bgc->gc.ngpio = pdata->ngpio;
+			gc->ngpio = pdata->ngpio;
 	}
 
-	platform_set_drvdata(pdev, bgc);
+	platform_set_drvdata(pdev, gc);
 
-	return gpiochip_add(&bgc->gc);
+	return gpiochip_add_data(gc, NULL);
 }
 
 static int bgpio_pdev_remove(struct platform_device *pdev)
 {
-	struct bgpio_chip *bgc = platform_get_drvdata(pdev);
+	struct gpio_chip *gc = platform_get_drvdata(pdev);
 
-	return bgpio_remove(bgc);
+	gpiochip_remove(gc);
+	return 0;
 }
 
 static const struct platform_device_id bgpio_id_table[] = {
diff --git a/drivers/gpio/gpio-grgpio.c b/drivers/gpio/gpio-grgpio.c
index 801423fe8143..7847dd34f86f 100644
--- a/drivers/gpio/gpio-grgpio.c
+++ b/drivers/gpio/gpio-grgpio.c
@@ -31,7 +31,7 @@
 #include <linux/gpio.h>
 #include <linux/slab.h>
 #include <linux/err.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
@@ -63,7 +63,7 @@ struct grgpio_lirq {
 };
 
 struct grgpio_priv {
-	struct bgpio_chip bgc;
+	struct gpio_chip gc;
 	void __iomem *regs;
 	struct device *dev;
 
@@ -92,29 +92,22 @@ struct grgpio_priv {
 	struct grgpio_lirq lirqs[GRGPIO_MAX_NGPIO];
 };
 
-static inline struct grgpio_priv *grgpio_gc_to_priv(struct gpio_chip *gc)
-{
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return container_of(bgc, struct grgpio_priv, bgc);
-}
-
 static void grgpio_set_imask(struct grgpio_priv *priv, unsigned int offset,
 			     int val)
 {
-	struct bgpio_chip *bgc = &priv->bgc;
-	unsigned long mask = bgc->pin2mask(bgc, offset);
+	struct gpio_chip *gc = &priv->gc;
+	unsigned long mask = gc->pin2mask(gc, offset);
 
 	if (val)
 		priv->imask |= mask;
 	else
 		priv->imask &= ~mask;
-	bgc->write_reg(priv->regs + GRGPIO_IMASK, priv->imask);
+	gc->write_reg(priv->regs + GRGPIO_IMASK, priv->imask);
 }
 
 static int grgpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct grgpio_priv *priv = grgpio_gc_to_priv(gc);
+	struct grgpio_priv *priv = gpiochip_get_data(gc);
 
 	if (offset >= gc->ngpio)
 		return -ENXIO;
@@ -158,15 +151,15 @@ static int grgpio_irq_set_type(struct irq_data *d, unsigned int type)
 		return -EINVAL;
 	}
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
-	ipol = priv->bgc.read_reg(priv->regs + GRGPIO_IPOL) & ~mask;
-	iedge = priv->bgc.read_reg(priv->regs + GRGPIO_IEDGE) & ~mask;
+	ipol = priv->gc.read_reg(priv->regs + GRGPIO_IPOL) & ~mask;
+	iedge = priv->gc.read_reg(priv->regs + GRGPIO_IEDGE) & ~mask;
 
-	priv->bgc.write_reg(priv->regs + GRGPIO_IPOL, ipol | pol);
-	priv->bgc.write_reg(priv->regs + GRGPIO_IEDGE, iedge | edge);
+	priv->gc.write_reg(priv->regs + GRGPIO_IPOL, ipol | pol);
+	priv->gc.write_reg(priv->regs + GRGPIO_IEDGE, iedge | edge);
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 
 	return 0;
 }
@@ -177,11 +170,11 @@ static void grgpio_irq_mask(struct irq_data *d)
 	int offset = d->hwirq;
 	unsigned long flags;
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	grgpio_set_imask(priv, offset, 0);
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 }
 
 static void grgpio_irq_unmask(struct irq_data *d)
@@ -190,11 +183,11 @@ static void grgpio_irq_unmask(struct irq_data *d)
 	int offset = d->hwirq;
 	unsigned long flags;
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	grgpio_set_imask(priv, offset, 1);
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 }
 
 static struct irq_chip grgpio_irq_chip = {
@@ -207,12 +200,12 @@ static struct irq_chip grgpio_irq_chip = {
 static irqreturn_t grgpio_irq_handler(int irq, void *dev)
 {
 	struct grgpio_priv *priv = dev;
-	int ngpio = priv->bgc.gc.ngpio;
+	int ngpio = priv->gc.ngpio;
 	unsigned long flags;
 	int i;
 	int match = 0;
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	/*
 	 * For each gpio line, call its interrupt handler if it its underlying
@@ -228,7 +221,7 @@ static irqreturn_t grgpio_irq_handler(int irq, void *dev)
 		}
 	}
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 
 	if (!match)
 		dev_warn(priv->dev, "No gpio line matched irq %d\n", irq);
@@ -260,7 +253,7 @@ static int grgpio_irq_map(struct irq_domain *d, unsigned int irq,
 	dev_dbg(priv->dev, "Mapping irq %d for gpio line %d\n",
 		irq, offset);
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	/* Request underlying irq if not already requested */
 	lirq->irq = irq;
@@ -273,14 +266,14 @@ static int grgpio_irq_map(struct irq_domain *d, unsigned int irq,
 				"Could not request underlying irq %d\n",
 				uirq->uirq);
 
-			spin_unlock_irqrestore(&priv->bgc.lock, flags);
+			spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 
 			return ret;
 		}
 	}
 	uirq->refcnt++;
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 
 	/* Setup irq  */
 	irq_set_chip_data(irq, priv);
@@ -298,13 +291,13 @@ static void grgpio_irq_unmap(struct irq_domain *d, unsigned int irq)
 	struct grgpio_lirq *lirq;
 	struct grgpio_uirq *uirq;
 	unsigned long flags;
-	int ngpio = priv->bgc.gc.ngpio;
+	int ngpio = priv->gc.ngpio;
 	int i;
 
 	irq_set_chip_and_handler(irq, NULL, NULL);
 	irq_set_chip_data(irq, NULL);
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	/* Free underlying irq if last user unmapped */
 	index = -1;
@@ -326,7 +319,7 @@ static void grgpio_irq_unmap(struct irq_domain *d, unsigned int irq)
 			free_irq(uirq->uirq, priv);
 	}
 
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 }
 
 static const struct irq_domain_ops grgpio_irq_domain_ops = {
@@ -341,7 +334,6 @@ static int grgpio_probe(struct platform_device *ofdev)
 	struct device_node *np = ofdev->dev.of_node;
 	void  __iomem *regs;
 	struct gpio_chip *gc;
-	struct bgpio_chip *bgc;
 	struct grgpio_priv *priv;
 	struct resource *res;
 	int err;
@@ -359,8 +351,8 @@ static int grgpio_probe(struct platform_device *ofdev)
 	if (IS_ERR(regs))
 		return PTR_ERR(regs);
 
-	bgc = &priv->bgc;
-	err = bgpio_init(bgc, &ofdev->dev, 4, regs + GRGPIO_DATA,
+	gc = &priv->gc;
+	err = bgpio_init(gc, &ofdev->dev, 4, regs + GRGPIO_DATA,
 			 regs + GRGPIO_OUTPUT, NULL, regs + GRGPIO_DIR, NULL,
 			 BGPIOF_BIG_ENDIAN_BYTE_ORDER);
 	if (err) {
@@ -369,10 +361,9 @@ static int grgpio_probe(struct platform_device *ofdev)
 	}
 
 	priv->regs = regs;
-	priv->imask = bgc->read_reg(regs + GRGPIO_IMASK);
+	priv->imask = gc->read_reg(regs + GRGPIO_IMASK);
 	priv->dev = &ofdev->dev;
 
-	gc = &bgc->gc;
 	gc->of_node = np;
 	gc->owner = THIS_MODULE;
 	gc->to_irq = grgpio_to_irq;
@@ -435,7 +426,7 @@ static int grgpio_probe(struct platform_device *ofdev)
 
 	platform_set_drvdata(ofdev, priv);
 
-	err = gpiochip_add(gc);
+	err = gpiochip_add_data(gc, priv);
 	if (err) {
 		dev_err(&ofdev->dev, "Could not add gpiochip\n");
 		if (priv->domain)
@@ -456,7 +447,7 @@ static int grgpio_remove(struct platform_device *ofdev)
 	int i;
 	int ret = 0;
 
-	spin_lock_irqsave(&priv->bgc.lock, flags);
+	spin_lock_irqsave(&priv->gc.bgpio_lock, flags);
 
 	if (priv->domain) {
 		for (i = 0; i < GRGPIO_MAX_NGPIO; i++) {
@@ -467,13 +458,13 @@ static int grgpio_remove(struct platform_device *ofdev)
 		}
 	}
 
-	gpiochip_remove(&priv->bgc.gc);
+	gpiochip_remove(&priv->gc);
 
 	if (priv->domain)
 		irq_domain_remove(priv->domain);
 
 out:
-	spin_unlock_irqrestore(&priv->bgc.lock, flags);
+	spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags);
 
 	return ret;
 }
diff --git a/drivers/gpio/gpio-ich.c b/drivers/gpio/gpio-ich.c
index 4ba7ed502131..a4893386abbf 100644
--- a/drivers/gpio/gpio-ich.c
+++ b/drivers/gpio/gpio-ich.c
@@ -282,7 +282,7 @@ static void ichx_gpiolib_setup(struct gpio_chip *chip)
 {
 	chip->owner = THIS_MODULE;
 	chip->label = DRV_NAME;
-	chip->dev = &ichx_priv.dev->dev;
+	chip->parent = &ichx_priv.dev->dev;
 
 	/* Allow chip-specific overrides of request()/get() */
 	chip->request = ichx_priv.desc->request ?
@@ -499,7 +499,7 @@ static int ichx_gpio_probe(struct platform_device *pdev)
 
 init:
 	ichx_gpiolib_setup(&ichx_priv.chip);
-	err = gpiochip_add(&ichx_priv.chip);
+	err = gpiochip_add_data(&ichx_priv.chip, NULL);
 	if (err) {
 		pr_err("Failed to register GPIOs\n");
 		goto add_err;
diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c
index 70097472b02c..cdaba13cb8e8 100644
--- a/drivers/gpio/gpio-intel-mid.c
+++ b/drivers/gpio/gpio-intel-mid.c
@@ -78,15 +78,10 @@ struct intel_mid_gpio {
 	struct pci_dev			*pdev;
 };
 
-static inline struct intel_mid_gpio *to_intel_gpio_priv(struct gpio_chip *gc)
-{
-	return container_of(gc, struct intel_mid_gpio, chip);
-}
-
 static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned offset,
 			      enum GPIO_REG reg_type)
 {
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(chip);
+	struct intel_mid_gpio *priv = gpiochip_get_data(chip);
 	unsigned nreg = chip->ngpio / 32;
 	u8 reg = offset / 32;
 
@@ -96,7 +91,7 @@ static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned offset,
 static void __iomem *gpio_reg_2bit(struct gpio_chip *chip, unsigned offset,
 				   enum GPIO_REG reg_type)
 {
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(chip);
+	struct intel_mid_gpio *priv = gpiochip_get_data(chip);
 	unsigned nreg = chip->ngpio / 32;
 	u8 reg = offset / 16;
 
@@ -120,7 +115,7 @@ static int intel_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	void __iomem *gplr = gpio_reg(chip, offset, GPLR);
 
-	return readl(gplr) & BIT(offset % 32);
+	return !!(readl(gplr) & BIT(offset % 32));
 }
 
 static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
@@ -138,7 +133,7 @@ static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int intel_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(chip);
+	struct intel_mid_gpio *priv = gpiochip_get_data(chip);
 	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
 	u32 value;
 	unsigned long flags;
@@ -161,7 +156,7 @@ static int intel_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int intel_gpio_direction_output(struct gpio_chip *chip,
 			unsigned offset, int value)
 {
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(chip);
+	struct intel_mid_gpio *priv = gpiochip_get_data(chip);
 	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
 	unsigned long flags;
 
@@ -185,7 +180,7 @@ static int intel_gpio_direction_output(struct gpio_chip *chip,
 static int intel_mid_irq_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(gc);
+	struct intel_mid_gpio *priv = gpiochip_get_data(gc);
 	u32 gpio = irqd_to_hwirq(d);
 	unsigned long flags;
 	u32 value;
@@ -304,7 +299,7 @@ MODULE_DEVICE_TABLE(pci, intel_gpio_ids);
 static void intel_mid_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct intel_mid_gpio *priv = to_intel_gpio_priv(gc);
+	struct intel_mid_gpio *priv = gpiochip_get_data(gc);
 	struct irq_data *data = irq_desc_get_irq_data(desc);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	u32 base, gpio, mask;
@@ -392,7 +387,7 @@ static int intel_gpio_probe(struct pci_dev *pdev,
 
 	priv->reg_base = pcim_iomap_table(pdev)[0];
 	priv->chip.label = dev_name(&pdev->dev);
-	priv->chip.dev = &pdev->dev;
+	priv->chip.parent = &pdev->dev;
 	priv->chip.request = intel_gpio_request;
 	priv->chip.direction_input = intel_gpio_direction_input;
 	priv->chip.direction_output = intel_gpio_direction_output;
@@ -406,7 +401,7 @@ static int intel_gpio_probe(struct pci_dev *pdev,
 	spin_lock_init(&priv->lock);
 
 	pci_set_drvdata(pdev, priv);
-	retval = gpiochip_add(&priv->chip);
+	retval = gpiochip_add_data(&priv->chip, priv);
 	if (retval) {
 		dev_err(&pdev->dev, "gpiochip_add error %d\n", retval);
 		return retval;
diff --git a/drivers/gpio/gpio-iop.c b/drivers/gpio/gpio-iop.c
index 2ed0237a8baf..fb65e5850e0c 100644
--- a/drivers/gpio/gpio-iop.c
+++ b/drivers/gpio/gpio-iop.c
@@ -114,7 +114,7 @@ static int iop3xx_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	return gpiochip_add(&iop3xx_chip);
+	return gpiochip_add_data(&iop3xx_chip, NULL);
 }
 
 static struct platform_driver iop3xx_gpio_driver = {
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index 21f6f7c0eb34..b219c82414bf 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -77,11 +77,6 @@ static struct it87_gpio it87_gpio_chip = {
 	.lock = __SPIN_LOCK_UNLOCKED(it87_gpio_chip.lock),
 };
 
-static inline struct it87_gpio *to_it87_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct it87_gpio, chip);
-}
-
 /* Superio chip access functions; copied from wdt_it87 */
 
 static inline int superio_enter(void)
@@ -165,7 +160,7 @@ static int it87_gpio_request(struct gpio_chip *chip, unsigned gpio_num)
 {
 	u8 mask, group;
 	int rc = 0;
-	struct it87_gpio *it87_gpio = to_it87_gpio(chip);
+	struct it87_gpio *it87_gpio = gpiochip_get_data(chip);
 
 	mask = 1 << (gpio_num % 8);
 	group = (gpio_num / 8);
@@ -198,7 +193,7 @@ static int it87_gpio_get(struct gpio_chip *chip, unsigned gpio_num)
 {
 	u16 reg;
 	u8 mask;
-	struct it87_gpio *it87_gpio = to_it87_gpio(chip);
+	struct it87_gpio *it87_gpio = gpiochip_get_data(chip);
 
 	mask = 1 << (gpio_num % 8);
 	reg = (gpio_num / 8) + it87_gpio->io_base;
@@ -210,7 +205,7 @@ static int it87_gpio_direction_in(struct gpio_chip *chip, unsigned gpio_num)
 {
 	u8 mask, group;
 	int rc = 0;
-	struct it87_gpio *it87_gpio = to_it87_gpio(chip);
+	struct it87_gpio *it87_gpio = gpiochip_get_data(chip);
 
 	mask = 1 << (gpio_num % 8);
 	group = (gpio_num / 8);
@@ -236,7 +231,7 @@ static void it87_gpio_set(struct gpio_chip *chip,
 {
 	u8 mask, curr_vals;
 	u16 reg;
-	struct it87_gpio *it87_gpio = to_it87_gpio(chip);
+	struct it87_gpio *it87_gpio = gpiochip_get_data(chip);
 
 	mask = 1 << (gpio_num % 8);
 	reg = (gpio_num / 8) + it87_gpio->io_base;
@@ -253,7 +248,7 @@ static int it87_gpio_direction_out(struct gpio_chip *chip,
 {
 	u8 mask, group;
 	int rc = 0;
-	struct it87_gpio *it87_gpio = to_it87_gpio(chip);
+	struct it87_gpio *it87_gpio = gpiochip_get_data(chip);
 
 	mask = 1 << (gpio_num % 8);
 	group = (gpio_num / 8);
@@ -380,7 +375,7 @@ static int __init it87_gpio_init(void)
 
 	it87_gpio->chip.names = (const char *const*)labels_table;
 
-	rc = gpiochip_add(&it87_gpio->chip);
+	rc = gpiochip_add_data(&it87_gpio->chip, it87_gpio);
 	if (rc)
 		goto labels_free;
 
diff --git a/drivers/gpio/gpio-janz-ttl.c b/drivers/gpio/gpio-janz-ttl.c
index 3a1664335f5e..482aa0353868 100644
--- a/drivers/gpio/gpio-janz-ttl.c
+++ b/drivers/gpio/gpio-janz-ttl.c
@@ -59,7 +59,7 @@ struct ttl_module {
 
 static int ttl_get_value(struct gpio_chip *gpio, unsigned offset)
 {
-	struct ttl_module *mod = dev_get_drvdata(gpio->dev);
+	struct ttl_module *mod = dev_get_drvdata(gpio->parent);
 	u8 *shadow;
 	int ret;
 
@@ -76,12 +76,12 @@ static int ttl_get_value(struct gpio_chip *gpio, unsigned offset)
 	spin_lock(&mod->lock);
 	ret = *shadow & (1 << offset);
 	spin_unlock(&mod->lock);
-	return ret;
+	return !!ret;
 }
 
 static void ttl_set_value(struct gpio_chip *gpio, unsigned offset, int value)
 {
-	struct ttl_module *mod = dev_get_drvdata(gpio->dev);
+	struct ttl_module *mod = dev_get_drvdata(gpio->parent);
 	void __iomem *port;
 	u8 *shadow;
 
@@ -172,7 +172,7 @@ static int ttl_probe(struct platform_device *pdev)
 
 	/* Initialize the GPIO data structures */
 	gpio = &mod->gpio;
-	gpio->dev = &pdev->dev;
+	gpio->parent = &pdev->dev;
 	gpio->label = pdev->name;
 	gpio->get = ttl_get_value;
 	gpio->set = ttl_set_value;
@@ -182,7 +182,7 @@ static int ttl_probe(struct platform_device *pdev)
 	gpio->base = -1;
 	gpio->ngpio = 20;
 
-	ret = gpiochip_add(gpio);
+	ret = gpiochip_add_data(gpio, NULL);
 	if (ret) {
 		dev_err(dev, "unable to add GPIO chip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-kempld.c b/drivers/gpio/gpio-kempld.c
index 83f281dda1e0..01117747b965 100644
--- a/drivers/gpio/gpio-kempld.c
+++ b/drivers/gpio/gpio-kempld.c
@@ -65,17 +65,15 @@ static int kempld_gpio_get_bit(struct kempld_device_data *pld, u8 reg, u8 bit)
 
 static int kempld_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct kempld_gpio_data *gpio
-		= container_of(chip, struct kempld_gpio_data, chip);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
-	return kempld_gpio_get_bit(pld, KEMPLD_GPIO_LVL_NUM(offset), offset);
+	return !!kempld_gpio_get_bit(pld, KEMPLD_GPIO_LVL_NUM(offset), offset);
 }
 
 static void kempld_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct kempld_gpio_data *gpio
-		= container_of(chip, struct kempld_gpio_data, chip);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
@@ -85,8 +83,7 @@ static void kempld_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int kempld_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct kempld_gpio_data *gpio
-		= container_of(chip, struct kempld_gpio_data, chip);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
@@ -99,8 +96,7 @@ static int kempld_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int kempld_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 					int value)
 {
-	struct kempld_gpio_data *gpio
-		= container_of(chip, struct kempld_gpio_data, chip);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
@@ -113,8 +109,7 @@ static int kempld_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static int kempld_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct kempld_gpio_data *gpio
-		= container_of(chip, struct kempld_gpio_data, chip);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
 	return !kempld_gpio_get_bit(pld, KEMPLD_GPIO_DIR_NUM(offset), offset);
@@ -166,7 +161,7 @@ static int kempld_gpio_probe(struct platform_device *pdev)
 	chip = &gpio->chip;
 	chip->label = "gpio-kempld";
 	chip->owner = THIS_MODULE;
-	chip->dev = dev;
+	chip->parent = dev;
 	chip->can_sleep = true;
 	if (pdata && pdata->gpio_base)
 		chip->base = pdata->gpio_base;
@@ -183,7 +178,7 @@ static int kempld_gpio_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, gpio);
 	if (ret) {
 		dev_err(dev, "Could not register GPIO chip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-ks8695.c b/drivers/gpio/gpio-ks8695.c
index cc09b237e88c..9f86ed9c753b 100644
--- a/drivers/gpio/gpio-ks8695.c
+++ b/drivers/gpio/gpio-ks8695.c
@@ -234,7 +234,7 @@ static struct gpio_chip ks8695_gpio_chip = {
 /* Register the GPIOs */
 void ks8695_register_gpios(void)
 {
-	if (gpiochip_add(&ks8695_gpio_chip))
+	if (gpiochip_add_data(&ks8695_gpio_chip, NULL))
 		printk(KERN_ERR "Unable to register core GPIOs\n");
 }
 
diff --git a/drivers/gpio/gpio-loongson.c b/drivers/gpio/gpio-loongson.c
index ccc65a1aea88..92c4fe7b2677 100644
--- a/drivers/gpio/gpio-loongson.c
+++ b/drivers/gpio/gpio-loongson.c
@@ -110,6 +110,6 @@ static struct gpio_chip loongson_chip = {
 
 static int __init loongson_gpio_setup(void)
 {
-	return gpiochip_add(&loongson_chip);
+	return gpiochip_add_data(&loongson_chip, NULL);
 }
 postcore_initcall(loongson_gpio_setup);
diff --git a/drivers/gpio/gpio-lp3943.c b/drivers/gpio/gpio-lp3943.c
index cfc5b12b43ad..1c8e2ae26938 100644
--- a/drivers/gpio/gpio-lp3943.c
+++ b/drivers/gpio/gpio-lp3943.c
@@ -45,14 +45,9 @@ struct lp3943_gpio {
 	u16 input_mask;		/* 1 = GPIO is input direction, 0 = output */
 };
 
-static inline struct lp3943_gpio *to_lp3943_gpio(struct gpio_chip *_chip)
-{
-	return container_of(_chip, struct lp3943_gpio, chip);
-}
-
 static int lp3943_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 	struct lp3943 *lp3943 = lp3943_gpio->lp3943;
 
 	/* Return an error if the pin is already assigned */
@@ -64,7 +59,7 @@ static int lp3943_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void lp3943_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 	struct lp3943 *lp3943 = lp3943_gpio->lp3943;
 
 	clear_bit(offset, &lp3943->pin_used);
@@ -82,7 +77,7 @@ static int lp3943_gpio_set_mode(struct lp3943_gpio *lp3943_gpio, u8 offset,
 
 static int lp3943_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 
 	lp3943_gpio->input_mask |= BIT(offset);
 
@@ -138,7 +133,7 @@ static int lp3943_get_gpio_out_status(struct lp3943_gpio *lp3943_gpio,
 
 static int lp3943_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 
 	/*
 	 * Limitation:
@@ -157,7 +152,7 @@ static int lp3943_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void lp3943_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 	u8 data;
 
 	if (value)
@@ -171,7 +166,7 @@ static void lp3943_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int lp3943_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 					int value)
 {
-	struct lp3943_gpio *lp3943_gpio = to_lp3943_gpio(chip);
+	struct lp3943_gpio *lp3943_gpio = gpiochip_get_data(chip);
 
 	lp3943_gpio_set(chip, offset, value);
 	lp3943_gpio->input_mask &= ~BIT(offset);
@@ -205,11 +200,11 @@ static int lp3943_gpio_probe(struct platform_device *pdev)
 
 	lp3943_gpio->lp3943 = lp3943;
 	lp3943_gpio->chip = lp3943_gpio_chip;
-	lp3943_gpio->chip.dev = &pdev->dev;
+	lp3943_gpio->chip.parent = &pdev->dev;
 
 	platform_set_drvdata(pdev, lp3943_gpio);
 
-	return gpiochip_add(&lp3943_gpio->chip);
+	return gpiochip_add_data(&lp3943_gpio->chip, lp3943_gpio);
 }
 
 static int lp3943_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-lpc18xx.c b/drivers/gpio/gpio-lpc18xx.c
index e39dcb0af8ae..98832c9f614a 100644
--- a/drivers/gpio/gpio-lpc18xx.c
+++ b/drivers/gpio/gpio-lpc18xx.c
@@ -31,27 +31,22 @@ struct lpc18xx_gpio_chip {
 	spinlock_t lock;
 };
 
-static inline struct lpc18xx_gpio_chip *to_lpc18xx_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct lpc18xx_gpio_chip, gpio);
-}
-
 static void lpc18xx_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct lpc18xx_gpio_chip *gc = to_lpc18xx_gpio(chip);
+	struct lpc18xx_gpio_chip *gc = gpiochip_get_data(chip);
 	writeb(value ? 1 : 0, gc->base + offset);
 }
 
 static int lpc18xx_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct lpc18xx_gpio_chip *gc = to_lpc18xx_gpio(chip);
+	struct lpc18xx_gpio_chip *gc = gpiochip_get_data(chip);
 	return !!readb(gc->base + offset);
 }
 
 static int lpc18xx_gpio_direction(struct gpio_chip *chip, unsigned offset,
 				  bool out)
 {
-	struct lpc18xx_gpio_chip *gc = to_lpc18xx_gpio(chip);
+	struct lpc18xx_gpio_chip *gc = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 port, pin, dir;
 
@@ -127,9 +122,9 @@ static int lpc18xx_gpio_probe(struct platform_device *pdev)
 
 	spin_lock_init(&gc->lock);
 
-	gc->gpio.dev = &pdev->dev;
+	gc->gpio.parent = &pdev->dev;
 
-	ret = gpiochip_add(&gc->gpio);
+	ret = gpiochip_add_data(&gc->gpio, gc);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to add gpio chip\n");
 		clk_disable_unprepare(gc->clk);
diff --git a/drivers/gpio/gpio-lpc32xx.c b/drivers/gpio/gpio-lpc32xx.c
index 47e2dde63734..4cecf4ce96c1 100644
--- a/drivers/gpio/gpio-lpc32xx.c
+++ b/drivers/gpio/gpio-lpc32xx.c
@@ -165,12 +165,6 @@ struct lpc32xx_gpio_chip {
 	struct gpio_regs	*gpio_grp;
 };
 
-static inline struct lpc32xx_gpio_chip *to_lpc32xx_gpio(
-	struct gpio_chip *gpc)
-{
-	return container_of(gpc, struct lpc32xx_gpio_chip, chip);
-}
-
 static void __set_gpio_dir_p012(struct lpc32xx_gpio_chip *group,
 	unsigned pin, int input)
 {
@@ -261,7 +255,7 @@ static int __get_gpo_state_p3(struct lpc32xx_gpio_chip *group,
 static int lpc32xx_gpio_dir_input_p012(struct gpio_chip *chip,
 	unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_dir_p012(group, pin, 1);
 
@@ -271,7 +265,7 @@ static int lpc32xx_gpio_dir_input_p012(struct gpio_chip *chip,
 static int lpc32xx_gpio_dir_input_p3(struct gpio_chip *chip,
 	unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_dir_p3(group, pin, 1);
 
@@ -286,29 +280,29 @@ static int lpc32xx_gpio_dir_in_always(struct gpio_chip *chip,
 
 static int lpc32xx_gpio_get_value_p012(struct gpio_chip *chip, unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
-	return __get_gpio_state_p012(group, pin);
+	return !!__get_gpio_state_p012(group, pin);
 }
 
 static int lpc32xx_gpio_get_value_p3(struct gpio_chip *chip, unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
-	return __get_gpio_state_p3(group, pin);
+	return !!__get_gpio_state_p3(group, pin);
 }
 
 static int lpc32xx_gpi_get_value(struct gpio_chip *chip, unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
-	return __get_gpi_state_p3(group, pin);
+	return !!__get_gpi_state_p3(group, pin);
 }
 
 static int lpc32xx_gpio_dir_output_p012(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_level_p012(group, pin, value);
 	__set_gpio_dir_p012(group, pin, 0);
@@ -319,7 +313,7 @@ static int lpc32xx_gpio_dir_output_p012(struct gpio_chip *chip, unsigned pin,
 static int lpc32xx_gpio_dir_output_p3(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_level_p3(group, pin, value);
 	__set_gpio_dir_p3(group, pin, 0);
@@ -330,7 +324,7 @@ static int lpc32xx_gpio_dir_output_p3(struct gpio_chip *chip, unsigned pin,
 static int lpc32xx_gpio_dir_out_always(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpo_level_p3(group, pin, value);
 	return 0;
@@ -339,7 +333,7 @@ static int lpc32xx_gpio_dir_out_always(struct gpio_chip *chip, unsigned pin,
 static void lpc32xx_gpio_set_value_p012(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_level_p012(group, pin, value);
 }
@@ -347,7 +341,7 @@ static void lpc32xx_gpio_set_value_p012(struct gpio_chip *chip, unsigned pin,
 static void lpc32xx_gpio_set_value_p3(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpio_level_p3(group, pin, value);
 }
@@ -355,16 +349,16 @@ static void lpc32xx_gpio_set_value_p3(struct gpio_chip *chip, unsigned pin,
 static void lpc32xx_gpo_set_value(struct gpio_chip *chip, unsigned pin,
 	int value)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
 	__set_gpo_level_p3(group, pin, value);
 }
 
 static int lpc32xx_gpo_get_value(struct gpio_chip *chip, unsigned pin)
 {
-	struct lpc32xx_gpio_chip *group = to_lpc32xx_gpio(chip);
+	struct lpc32xx_gpio_chip *group = gpiochip_get_data(chip);
 
-	return __get_gpo_state_p3(group, pin);
+	return !!__get_gpo_state_p3(group, pin);
 }
 
 static int lpc32xx_gpio_request(struct gpio_chip *chip, unsigned pin)
@@ -553,7 +547,8 @@ static int lpc32xx_gpio_probe(struct platform_device *pdev)
 			lpc32xx_gpiochip[i].chip.of_gpio_n_cells = 3;
 			lpc32xx_gpiochip[i].chip.of_node = pdev->dev.of_node;
 		}
-		gpiochip_add(&lpc32xx_gpiochip[i].chip);
+		gpiochip_add_data(&lpc32xx_gpiochip[i].chip,
+				  &lpc32xx_gpiochip[i]);
 	}
 
 	return 0;
diff --git a/drivers/gpio/gpio-lynxpoint.c b/drivers/gpio/gpio-lynxpoint.c
index 127c37b380ae..13107772be4f 100644
--- a/drivers/gpio/gpio-lynxpoint.c
+++ b/drivers/gpio/gpio-lynxpoint.c
@@ -97,7 +97,7 @@ struct lp_gpio {
 static unsigned long lp_gpio_reg(struct gpio_chip *chip, unsigned offset,
 				 int reg)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	int reg_offset;
 
 	if (reg == LP_CONFIG1 || reg == LP_CONFIG2)
@@ -112,7 +112,7 @@ static unsigned long lp_gpio_reg(struct gpio_chip *chip, unsigned offset,
 
 static int lp_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	unsigned long reg = lp_gpio_reg(chip, offset, LP_CONFIG1);
 	unsigned long conf2 = lp_gpio_reg(chip, offset, LP_CONFIG2);
 	unsigned long acpi_use = lp_gpio_reg(chip, offset, LP_ACPI_OWNED);
@@ -137,7 +137,7 @@ static int lp_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void lp_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	unsigned long conf2 = lp_gpio_reg(chip, offset, LP_CONFIG2);
 
 	/* disable input sensing */
@@ -149,7 +149,7 @@ static void lp_gpio_free(struct gpio_chip *chip, unsigned offset)
 static int lp_irq_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct lp_gpio *lg = container_of(gc, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(gc);
 	u32 hwirq = irqd_to_hwirq(d);
 	unsigned long flags;
 	u32 value;
@@ -191,7 +191,7 @@ static int lp_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void lp_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	unsigned long reg = lp_gpio_reg(chip, offset, LP_CONFIG1);
 	unsigned long flags;
 
@@ -207,7 +207,7 @@ static void lp_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int lp_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	unsigned long reg = lp_gpio_reg(chip, offset, LP_CONFIG1);
 	unsigned long flags;
 
@@ -221,7 +221,7 @@ static int lp_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int lp_gpio_direction_output(struct gpio_chip *chip,
 				      unsigned offset, int value)
 {
-	struct lp_gpio *lg = container_of(chip, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(chip);
 	unsigned long reg = lp_gpio_reg(chip, offset, LP_CONFIG1);
 	unsigned long flags;
 
@@ -238,7 +238,7 @@ static void lp_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct irq_data *data = irq_desc_get_irq_data(desc);
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct lp_gpio *lg = container_of(gc, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(gc);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	u32 base, pin, mask;
 	unsigned long reg, ena, pending;
@@ -273,7 +273,7 @@ static void lp_irq_mask(struct irq_data *d)
 static void lp_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct lp_gpio *lg = container_of(gc, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(gc);
 	u32 hwirq = irqd_to_hwirq(d);
 	unsigned long reg = lp_gpio_reg(&lg->chip, hwirq, LP_INT_ENABLE);
 	unsigned long flags;
@@ -286,7 +286,7 @@ static void lp_irq_enable(struct irq_data *d)
 static void lp_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct lp_gpio *lg = container_of(gc, struct lp_gpio, chip);
+	struct lp_gpio *lg = gpiochip_get_data(gc);
 	u32 hwirq = irqd_to_hwirq(d);
 	unsigned long reg = lp_gpio_reg(&lg->chip, hwirq, LP_INT_ENABLE);
 	unsigned long flags;
@@ -368,9 +368,9 @@ static int lp_gpio_probe(struct platform_device *pdev)
 	gc->base = -1;
 	gc->ngpio = LP_NUM_GPIO;
 	gc->can_sleep = false;
-	gc->dev = dev;
+	gc->parent = dev;
 
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, lg);
 	if (ret) {
 		dev_err(dev, "failed adding lp-gpio chip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-max7300.c b/drivers/gpio/gpio-max7300.c
index 0cc2c279ab5c..1ae9ba851c9a 100644
--- a/drivers/gpio/gpio-max7300.c
+++ b/drivers/gpio/gpio-max7300.c
@@ -65,7 +65,6 @@ MODULE_DEVICE_TABLE(i2c, max7300_id);
 static struct i2c_driver max7300_driver = {
 	.driver = {
 		.name = "max7300",
-		.owner = THIS_MODULE,
 	},
 	.probe = max7300_probe,
 	.remove = max7300_remove,
diff --git a/drivers/gpio/gpio-max730x.c b/drivers/gpio/gpio-max730x.c
index 0f57d2d248ec..08807368f007 100644
--- a/drivers/gpio/gpio-max730x.c
+++ b/drivers/gpio/gpio-max730x.c
@@ -50,7 +50,7 @@
 
 static int max7301_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct max7301 *ts = container_of(chip, struct max7301, chip);
+	struct max7301 *ts = gpiochip_get_data(chip);
 	u8 *config;
 	u8 offset_bits, pin_config;
 	int ret;
@@ -92,7 +92,7 @@ static int __max7301_set(struct max7301 *ts, unsigned offset, int value)
 static int max7301_direction_output(struct gpio_chip *chip, unsigned offset,
 				    int value)
 {
-	struct max7301 *ts = container_of(chip, struct max7301, chip);
+	struct max7301 *ts = gpiochip_get_data(chip);
 	u8 *config;
 	u8 offset_bits;
 	int ret;
@@ -120,7 +120,7 @@ static int max7301_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static int max7301_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct max7301 *ts = container_of(chip, struct max7301, chip);
+	struct max7301 *ts = gpiochip_get_data(chip);
 	int config, level = -EINVAL;
 
 	/* First 4 pins are unused in the controller */
@@ -148,7 +148,7 @@ static int max7301_get(struct gpio_chip *chip, unsigned offset)
 
 static void max7301_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct max7301 *ts = container_of(chip, struct max7301, chip);
+	struct max7301 *ts = gpiochip_get_data(chip);
 
 	/* First 4 pins are unused in the controller */
 	offset += 4;
@@ -189,7 +189,7 @@ int __max730x_probe(struct max7301 *ts)
 
 	ts->chip.ngpio = PIN_NUMBER;
 	ts->chip.can_sleep = true;
-	ts->chip.dev = dev;
+	ts->chip.parent = dev;
 	ts->chip.owner = THIS_MODULE;
 
 	/*
@@ -213,7 +213,7 @@ int __max730x_probe(struct max7301 *ts)
 		}
 	}
 
-	ret = gpiochip_add(&ts->chip);
+	ret = gpiochip_add_data(&ts->chip, ts);
 	if (ret)
 		goto exit_destroy;
 
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index 8c5252c6c327..a9aaf9d822b4 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -158,11 +158,6 @@ struct max732x_chip {
 #endif
 };
 
-static inline struct max732x_chip *to_max732x(struct gpio_chip *gc)
-{
-	return container_of(gc, struct max732x_chip, gpio_chip);
-}
-
 static int max732x_writeb(struct max732x_chip *chip, int group_a, uint8_t val)
 {
 	struct i2c_client *client;
@@ -201,21 +196,21 @@ static inline int is_group_a(struct max732x_chip *chip, unsigned off)
 
 static int max732x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 {
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	uint8_t reg_val;
 	int ret;
 
 	ret = max732x_readb(chip, is_group_a(chip, off), &reg_val);
 	if (ret < 0)
-		return 0;
+		return ret;
 
-	return reg_val & (1u << (off & 0x7));
+	return !!(reg_val & (1u << (off & 0x7)));
 }
 
 static void max732x_gpio_set_mask(struct gpio_chip *gc, unsigned off, int mask,
 				  int val)
 {
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	uint8_t reg_out;
 	int ret;
 
@@ -259,7 +254,7 @@ static void max732x_gpio_set_multiple(struct gpio_chip *gc,
 
 static int max732x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
 {
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	unsigned int mask = 1u << off;
 
 	if ((mask & chip->dir_input) == 0) {
@@ -281,7 +276,7 @@ static int max732x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
 static int max732x_gpio_direction_output(struct gpio_chip *gc,
 		unsigned off, int val)
 {
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	unsigned int mask = 1u << off;
 
 	if ((mask & chip->dir_output) == 0) {
@@ -356,7 +351,7 @@ static void max732x_irq_update_mask(struct max732x_chip *chip)
 static void max732x_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 
 	chip->irq_mask_cur &= ~(1 << d->hwirq);
 }
@@ -364,7 +359,7 @@ static void max732x_irq_mask(struct irq_data *d)
 static void max732x_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 
 	chip->irq_mask_cur |= 1 << d->hwirq;
 }
@@ -372,7 +367,7 @@ static void max732x_irq_unmask(struct irq_data *d)
 static void max732x_irq_bus_lock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 
 	mutex_lock(&chip->irq_lock);
 	chip->irq_mask_cur = chip->irq_mask;
@@ -381,7 +376,7 @@ static void max732x_irq_bus_lock(struct irq_data *d)
 static void max732x_irq_bus_sync_unlock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	uint16_t new_irqs;
 	uint16_t level;
 
@@ -400,7 +395,7 @@ static void max732x_irq_bus_sync_unlock(struct irq_data *d)
 static int max732x_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct max732x_chip *chip = to_max732x(gc);
+	struct max732x_chip *chip = gpiochip_get_data(gc);
 	uint16_t off = d->hwirq;
 	uint16_t mask = 1 << off;
 
@@ -603,7 +598,7 @@ static int max732x_setup_gpio(struct max732x_chip *chip,
 	gc->base = gpio_start;
 	gc->ngpio = port;
 	gc->label = chip->client->name;
-	gc->dev = &chip->client->dev;
+	gc->parent = &chip->client->dev;
 	gc->owner = THIS_MODULE;
 
 	return port;
@@ -649,7 +644,7 @@ static int max732x_probe(struct i2c_client *client,
 	chip->client = client;
 
 	nr_port = max732x_setup_gpio(chip, id, pdata->gpio_base);
-	chip->gpio_chip.dev = &client->dev;
+	chip->gpio_chip.parent = &client->dev;
 
 	addr_a = (client->addr & 0x0f) | 0x60;
 	addr_b = (client->addr & 0x0f) | 0x50;
@@ -694,7 +689,7 @@ static int max732x_probe(struct i2c_client *client,
 			goto out_failed;
 	}
 
-	ret = gpiochip_add(&chip->gpio_chip);
+	ret = gpiochip_add_data(&chip->gpio_chip, chip);
 	if (ret)
 		goto out_failed;
 
@@ -749,7 +744,6 @@ static int max732x_remove(struct i2c_client *client)
 static struct i2c_driver max732x_driver = {
 	.driver = {
 		.name		= "max732x",
-		.owner		= THIS_MODULE,
 		.of_match_table	= of_match_ptr(max732x_of_table),
 	},
 	.probe		= max732x_probe,
diff --git a/drivers/gpio/gpio-mb86s7x.c b/drivers/gpio/gpio-mb86s7x.c
index ee93c0ab0a59..7fffc1d6c055 100644
--- a/drivers/gpio/gpio-mb86s7x.c
+++ b/drivers/gpio/gpio-mb86s7x.c
@@ -44,14 +44,9 @@ struct mb86s70_gpio_chip {
 	spinlock_t lock;
 };
 
-static inline struct mb86s70_gpio_chip *chip_to_mb86s70(struct gpio_chip *gc)
-{
-	return container_of(gc, struct mb86s70_gpio_chip, gc);
-}
-
 static int mb86s70_gpio_request(struct gpio_chip *gc, unsigned gpio)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 val;
 
@@ -73,7 +68,7 @@ static int mb86s70_gpio_request(struct gpio_chip *gc, unsigned gpio)
 
 static void mb86s70_gpio_free(struct gpio_chip *gc, unsigned gpio)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 	unsigned long flags;
 	u32 val;
 
@@ -88,7 +83,7 @@ static void mb86s70_gpio_free(struct gpio_chip *gc, unsigned gpio)
 
 static int mb86s70_gpio_direction_input(struct gpio_chip *gc, unsigned gpio)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char val;
 
@@ -106,7 +101,7 @@ static int mb86s70_gpio_direction_input(struct gpio_chip *gc, unsigned gpio)
 static int mb86s70_gpio_direction_output(struct gpio_chip *gc,
 					 unsigned gpio, int value)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char val;
 
@@ -130,14 +125,14 @@ static int mb86s70_gpio_direction_output(struct gpio_chip *gc,
 
 static int mb86s70_gpio_get(struct gpio_chip *gc, unsigned gpio)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 
 	return !!(readl(gchip->base + PDR(gpio)) & OFFSET(gpio));
 }
 
 static void mb86s70_gpio_set(struct gpio_chip *gc, unsigned gpio, int value)
 {
-	struct mb86s70_gpio_chip *gchip = chip_to_mb86s70(gc);
+	struct mb86s70_gpio_chip *gchip = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char val;
 
@@ -187,12 +182,12 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
 	gchip->gc.label = dev_name(&pdev->dev);
 	gchip->gc.ngpio = 32;
 	gchip->gc.owner = THIS_MODULE;
-	gchip->gc.dev = &pdev->dev;
+	gchip->gc.parent = &pdev->dev;
 	gchip->gc.base = -1;
 
 	platform_set_drvdata(pdev, gchip);
 
-	ret = gpiochip_add(&gchip->gc);
+	ret = gpiochip_add_data(&gchip->gc, gchip);
 	if (ret) {
 		dev_err(&pdev->dev, "couldn't register gpio driver\n");
 		clk_disable_unprepare(gchip->clk);
diff --git a/drivers/gpio/gpio-mc33880.c b/drivers/gpio/gpio-mc33880.c
index 2853731db5bc..0f0df7956264 100644
--- a/drivers/gpio/gpio-mc33880.c
+++ b/drivers/gpio/gpio-mc33880.c
@@ -71,7 +71,7 @@ static int __mc33880_set(struct mc33880 *mc, unsigned offset, int value)
 
 static void mc33880_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct mc33880 *mc = container_of(chip, struct mc33880, chip);
+	struct mc33880 *mc = gpiochip_get_data(chip);
 
 	mutex_lock(&mc->lock);
 
@@ -116,7 +116,7 @@ static int mc33880_probe(struct spi_device *spi)
 	mc->chip.base = pdata->base;
 	mc->chip.ngpio = PIN_NUMBER;
 	mc->chip.can_sleep = true;
-	mc->chip.dev = &spi->dev;
+	mc->chip.parent = &spi->dev;
 	mc->chip.owner = THIS_MODULE;
 
 	mc->port_config = 0x00;
@@ -135,7 +135,7 @@ static int mc33880_probe(struct spi_device *spi)
 		goto exit_destroy;
 	}
 
-	ret = gpiochip_add(&mc->chip);
+	ret = gpiochip_add_data(&mc->chip, mc);
 	if (ret)
 		goto exit_destroy;
 
diff --git a/drivers/gpio/gpio-mc9s08dz60.c b/drivers/gpio/gpio-mc9s08dz60.c
index d62b4f8182bf..ba22fb92a6e7 100644
--- a/drivers/gpio/gpio-mc9s08dz60.c
+++ b/drivers/gpio/gpio-mc9s08dz60.c
@@ -29,12 +29,6 @@ struct mc9s08dz60 {
 	struct gpio_chip chip;
 };
 
-static inline struct mc9s08dz60 *to_mc9s08dz60(struct gpio_chip *gc)
-{
-	return container_of(gc, struct mc9s08dz60, chip);
-}
-
-
 static void mc9s_gpio_to_reg_and_bit(int offset, u8 *reg, u8 *bit)
 {
 	*reg = 0x20 + offset / GPIO_NUM_PER_GROUP;
@@ -45,7 +39,7 @@ static int mc9s08dz60_get_value(struct gpio_chip *gc, unsigned offset)
 {
 	u8 reg, bit;
 	s32 value;
-	struct mc9s08dz60 *mc9s = to_mc9s08dz60(gc);
+	struct mc9s08dz60 *mc9s = gpiochip_get_data(gc);
 
 	mc9s_gpio_to_reg_and_bit(offset, &reg, &bit);
 	value = i2c_smbus_read_byte_data(mc9s->client, reg);
@@ -75,7 +69,7 @@ static int mc9s08dz60_set(struct mc9s08dz60 *mc9s, unsigned offset, int val)
 
 static void mc9s08dz60_set_value(struct gpio_chip *gc, unsigned offset, int val)
 {
-	struct mc9s08dz60 *mc9s = to_mc9s08dz60(gc);
+	struct mc9s08dz60 *mc9s = gpiochip_get_data(gc);
 
 	mc9s08dz60_set(mc9s, offset, val);
 }
@@ -83,7 +77,7 @@ static void mc9s08dz60_set_value(struct gpio_chip *gc, unsigned offset, int val)
 static int mc9s08dz60_direction_output(struct gpio_chip *gc,
 				       unsigned offset, int val)
 {
-	struct mc9s08dz60 *mc9s = to_mc9s08dz60(gc);
+	struct mc9s08dz60 *mc9s = gpiochip_get_data(gc);
 
 	return mc9s08dz60_set(mc9s, offset, val);
 }
@@ -99,7 +93,7 @@ static int mc9s08dz60_probe(struct i2c_client *client,
 
 	mc9s->chip.label = client->name;
 	mc9s->chip.base = -1;
-	mc9s->chip.dev = &client->dev;
+	mc9s->chip.parent = &client->dev;
 	mc9s->chip.owner = THIS_MODULE;
 	mc9s->chip.ngpio = GPIO_NUM;
 	mc9s->chip.can_sleep = true;
@@ -109,7 +103,7 @@ static int mc9s08dz60_probe(struct i2c_client *client,
 	mc9s->client = client;
 	i2c_set_clientdata(client, mc9s);
 
-	return gpiochip_add(&mc9s->chip);
+	return gpiochip_add_data(&mc9s->chip, mc9s);
 }
 
 static int mc9s08dz60_remove(struct i2c_client *client)
@@ -131,7 +125,6 @@ MODULE_DEVICE_TABLE(i2c, mc9s08dz60_id);
 
 static struct i2c_driver mc9s08dz60_i2c_driver = {
 	.driver = {
-		.owner = THIS_MODULE,
 		.name = "mc9s08dz60",
 	},
 	.probe = mc9s08dz60_probe,
diff --git a/drivers/gpio/gpio-mcp23s08.c b/drivers/gpio/gpio-mcp23s08.c
index 4a41694919da..c767879e4dd9 100644
--- a/drivers/gpio/gpio-mcp23s08.c
+++ b/drivers/gpio/gpio-mcp23s08.c
@@ -269,7 +269,7 @@ static const struct mcp23s08_ops mcp23s17_ops = {
 
 static int mcp23s08_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct mcp23s08	*mcp = container_of(chip, struct mcp23s08, chip);
+	struct mcp23s08	*mcp = gpiochip_get_data(chip);
 	int status;
 
 	mutex_lock(&mcp->lock);
@@ -281,7 +281,7 @@ static int mcp23s08_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int mcp23s08_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct mcp23s08	*mcp = container_of(chip, struct mcp23s08, chip);
+	struct mcp23s08	*mcp = gpiochip_get_data(chip);
 	int status;
 
 	mutex_lock(&mcp->lock);
@@ -312,7 +312,7 @@ static int __mcp23s08_set(struct mcp23s08 *mcp, unsigned mask, int value)
 
 static void mcp23s08_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct mcp23s08	*mcp = container_of(chip, struct mcp23s08, chip);
+	struct mcp23s08	*mcp = gpiochip_get_data(chip);
 	unsigned mask = 1 << offset;
 
 	mutex_lock(&mcp->lock);
@@ -323,7 +323,7 @@ static void mcp23s08_set(struct gpio_chip *chip, unsigned offset, int value)
 static int
 mcp23s08_direction_output(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct mcp23s08	*mcp = container_of(chip, struct mcp23s08, chip);
+	struct mcp23s08	*mcp = gpiochip_get_data(chip);
 	unsigned mask = 1 << offset;
 	int status;
 
@@ -377,7 +377,7 @@ static irqreturn_t mcp23s08_irq(int irq, void *data)
 
 static int mcp23s08_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct mcp23s08 *mcp = container_of(chip, struct mcp23s08, chip);
+	struct mcp23s08 *mcp = gpiochip_get_data(chip);
 
 	return irq_find_mapping(mcp->irq_domain, offset);
 }
@@ -446,7 +446,7 @@ static int mcp23s08_irq_reqres(struct irq_data *data)
 	struct mcp23s08 *mcp = irq_data_get_irq_chip_data(data);
 
 	if (gpiochip_lock_as_irq(&mcp->chip, data->hwirq)) {
-		dev_err(mcp->chip.dev,
+		dev_err(mcp->chip.parent,
 			"unable to lock HW IRQ %lu for IRQ usage\n",
 			data->hwirq);
 		return -EINVAL;
@@ -481,7 +481,8 @@ static int mcp23s08_irq_setup(struct mcp23s08 *mcp)
 
 	mutex_init(&mcp->irq_lock);
 
-	mcp->irq_domain = irq_domain_add_linear(chip->dev->of_node, chip->ngpio,
+	mcp->irq_domain = irq_domain_add_linear(chip->parent->of_node,
+						chip->ngpio,
 						&irq_domain_simple_ops, mcp);
 	if (!mcp->irq_domain)
 		return -ENODEV;
@@ -491,10 +492,11 @@ static int mcp23s08_irq_setup(struct mcp23s08 *mcp)
 	else
 		irqflags |= IRQF_TRIGGER_LOW;
 
-	err = devm_request_threaded_irq(chip->dev, mcp->irq, NULL, mcp23s08_irq,
-					irqflags, dev_name(chip->dev), mcp);
+	err = devm_request_threaded_irq(chip->parent, mcp->irq, NULL,
+					mcp23s08_irq,
+					irqflags, dev_name(chip->parent), mcp);
 	if (err != 0) {
-		dev_err(chip->dev, "unable to request IRQ#%d: %d\n",
+		dev_err(chip->parent, "unable to request IRQ#%d: %d\n",
 			mcp->irq, err);
 		return err;
 	}
@@ -542,7 +544,7 @@ static void mcp23s08_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 	int		t;
 	unsigned	mask;
 
-	mcp = container_of(chip, struct mcp23s08, chip);
+	mcp = gpiochip_get_data(chip);
 
 	/* NOTE: we only handle one bank for now ... */
 	bank = '0' + ((mcp->addr >> 1) & 0x7);
@@ -638,7 +640,7 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 
 	mcp->chip.base = pdata->base;
 	mcp->chip.can_sleep = true;
-	mcp->chip.dev = dev;
+	mcp->chip.parent = dev;
 	mcp->chip.owner = THIS_MODULE;
 
 	/* verify MCP_IOCON.SEQOP = 0, so sequential reads work,
@@ -652,7 +654,7 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 	mcp->irq_controller = pdata->irq_controller;
 	if (mcp->irq && mcp->irq_controller) {
 		mcp->irq_active_high =
-			of_property_read_bool(mcp->chip.dev->of_node,
+			of_property_read_bool(mcp->chip.parent->of_node,
 					      "microchip,irq-active-high");
 
 		if (type == MCP_TYPE_017)
@@ -702,7 +704,7 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 			goto fail;
 	}
 
-	status = gpiochip_add(&mcp->chip);
+	status = gpiochip_add_data(&mcp->chip, mcp);
 	if (status < 0)
 		goto fail;
 
diff --git a/drivers/gpio/gpio-ml-ioh.c b/drivers/gpio/gpio-ml-ioh.c
index 5536108aa9db..796a5a4bc4f5 100644
--- a/drivers/gpio/gpio-ml-ioh.c
+++ b/drivers/gpio/gpio-ml-ioh.c
@@ -106,7 +106,7 @@ static const int num_ports[] = {6, 12, 16, 16, 15, 16, 16, 12};
 static void ioh_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 {
 	u32 reg_val;
-	struct ioh_gpio *chip =	container_of(gpio, struct ioh_gpio, gpio);
+	struct ioh_gpio *chip =	gpiochip_get_data(gpio);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->spinlock, flags);
@@ -122,15 +122,15 @@ static void ioh_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 
 static int ioh_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
-	struct ioh_gpio *chip =	container_of(gpio, struct ioh_gpio, gpio);
+	struct ioh_gpio *chip =	gpiochip_get_data(gpio);
 
-	return ioread32(&chip->reg->regs[chip->ch].pi) & (1 << nr);
+	return !!(ioread32(&chip->reg->regs[chip->ch].pi) & (1 << nr));
 }
 
 static int ioh_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 				     int val)
 {
-	struct ioh_gpio *chip =	container_of(gpio, struct ioh_gpio, gpio);
+	struct ioh_gpio *chip =	gpiochip_get_data(gpio);
 	u32 pm;
 	u32 reg_val;
 	unsigned long flags;
@@ -155,7 +155,7 @@ static int ioh_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 
 static int ioh_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 {
-	struct ioh_gpio *chip =	container_of(gpio, struct ioh_gpio, gpio);
+	struct ioh_gpio *chip =	gpiochip_get_data(gpio);
 	u32 pm;
 	unsigned long flags;
 
@@ -225,7 +225,7 @@ static void ioh_gpio_restore_reg_conf(struct ioh_gpio *chip)
 
 static int ioh_gpio_to_irq(struct gpio_chip *gpio, unsigned offset)
 {
-	struct ioh_gpio *chip = container_of(gpio, struct ioh_gpio, gpio);
+	struct ioh_gpio *chip = gpiochip_get_data(gpio);
 	return chip->irq_base + offset;
 }
 
@@ -450,7 +450,7 @@ static int ioh_gpio_probe(struct pci_dev *pdev,
 		chip->ch = i;
 		spin_lock_init(&chip->spinlock);
 		ioh_gpio_setup(chip, num_ports[i]);
-		ret = gpiochip_add(&chip->gpio);
+		ret = gpiochip_add_data(&chip->gpio, chip);
 		if (ret) {
 			dev_err(&pdev->dev, "IOH gpio: Failed to register GPIO\n");
 			goto err_gpiochip_add;
diff --git a/drivers/gpio/gpio-mm-lantiq.c b/drivers/gpio/gpio-mm-lantiq.c
index f67ef2283d64..54e5d8257d34 100644
--- a/drivers/gpio/gpio-mm-lantiq.c
+++ b/drivers/gpio/gpio-mm-lantiq.c
@@ -61,9 +61,7 @@ static void ltq_mm_apply(struct ltq_mm *chip)
  */
 static void ltq_mm_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct ltq_mm *chip =
-		container_of(mm_gc, struct ltq_mm, mmchip);
+	struct ltq_mm *chip = gpiochip_get_data(gc);
 
 	if (value)
 		chip->shadow |= (1 << offset);
@@ -122,7 +120,7 @@ static int ltq_mm_probe(struct platform_device *pdev)
 	if (!of_property_read_u32(pdev->dev.of_node, "lantiq,shadow", &shadow))
 		chip->shadow = shadow;
 
-	return of_mm_gpiochip_add(pdev->dev.of_node, &chip->mmchip);
+	return of_mm_gpiochip_add_data(pdev->dev.of_node, &chip->mmchip, chip);
 }
 
 static int ltq_mm_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-moxart.c b/drivers/gpio/gpio-moxart.c
index d3355a6dc9b1..ca604538ebf7 100644
--- a/drivers/gpio/gpio-moxart.c
+++ b/drivers/gpio/gpio-moxart.c
@@ -14,7 +14,6 @@
 #include <linux/init.h>
 #include <linux/irq.h>
 #include <linux/io.h>
-#include <linux/gpio.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
@@ -23,7 +22,7 @@
 #include <linux/delay.h>
 #include <linux/timer.h>
 #include <linux/bitops.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 
 #define GPIO_DATA_OUT		0x00
 #define GPIO_DATA_IN		0x04
@@ -33,12 +32,12 @@ static int moxart_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct resource *res;
-	struct bgpio_chip *bgc;
+	struct gpio_chip *gc;
 	void __iomem *base;
 	int ret;
 
-	bgc = devm_kzalloc(dev, sizeof(*bgc), GFP_KERNEL);
-	if (!bgc)
+	gc = devm_kzalloc(dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -46,7 +45,7 @@ static int moxart_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	ret = bgpio_init(bgc, dev, 4, base + GPIO_DATA_IN,
+	ret = bgpio_init(gc, dev, 4, base + GPIO_DATA_IN,
 			 base + GPIO_DATA_OUT, NULL,
 			 base + GPIO_PIN_DIRECTION, NULL,
 			 BGPIOF_READ_OUTPUT_REG_SET);
@@ -55,16 +54,16 @@ static int moxart_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	bgc->gc.label = "moxart-gpio";
-	bgc->gc.request = gpiochip_generic_request;
-	bgc->gc.free = gpiochip_generic_free;
-	bgc->data = bgc->read_reg(bgc->reg_set);
-	bgc->gc.base = 0;
-	bgc->gc.ngpio = 32;
-	bgc->gc.dev = dev;
-	bgc->gc.owner = THIS_MODULE;
+	gc->label = "moxart-gpio";
+	gc->request = gpiochip_generic_request;
+	gc->free = gpiochip_generic_free;
+	gc->bgpio_data = gc->read_reg(gc->reg_set);
+	gc->base = 0;
+	gc->ngpio = 32;
+	gc->parent = dev;
+	gc->owner = THIS_MODULE;
 
-	ret = gpiochip_add(&bgc->gc);
+	ret = gpiochip_add_data(gc, NULL);
 	if (ret) {
 		dev_err(dev, "%s: gpiochip_add failed\n",
 			dev->of_node->full_name);
diff --git a/drivers/gpio/gpio-mpc5200.c b/drivers/gpio/gpio-mpc5200.c
index 4c542153e923..0e5a6709f27d 100644
--- a/drivers/gpio/gpio-mpc5200.c
+++ b/drivers/gpio/gpio-mpc5200.c
@@ -71,8 +71,7 @@ static inline void
 __mpc52xx_wkup_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	struct mpc52xx_gpio_wkup __iomem *regs = mm_gc->regs;
 
 	if (val)
@@ -100,8 +99,7 @@ mpc52xx_wkup_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 static int mpc52xx_wkup_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	struct mpc52xx_gpio_wkup __iomem *regs = mm_gc->regs;
 	unsigned long flags;
 
@@ -125,8 +123,7 @@ mpc52xx_wkup_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
 	struct mpc52xx_gpio_wkup __iomem *regs = mm_gc->regs;
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&gpio_lock, flags);
@@ -169,7 +166,7 @@ static int mpc52xx_wkup_gpiochip_probe(struct platform_device *ofdev)
 	gc->get              = mpc52xx_wkup_gpio_get;
 	gc->set              = mpc52xx_wkup_gpio_set;
 
-	ret = of_mm_gpiochip_add(ofdev->dev.of_node, &chip->mmchip);
+	ret = of_mm_gpiochip_add_data(ofdev->dev.of_node, &chip->mmchip, chip);
 	if (ret)
 		return ret;
 
@@ -236,8 +233,7 @@ static inline void
 __mpc52xx_simple_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	struct mpc52xx_gpio __iomem *regs = mm_gc->regs;
 
 	if (val)
@@ -264,8 +260,7 @@ mpc52xx_simple_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 static int mpc52xx_simple_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	struct mpc52xx_gpio __iomem *regs = mm_gc->regs;
 	unsigned long flags;
 
@@ -288,8 +283,7 @@ static int
 mpc52xx_simple_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct mpc52xx_gpiochip *chip = container_of(mm_gc,
-			struct mpc52xx_gpiochip, mmchip);
+	struct mpc52xx_gpiochip *chip = gpiochip_get_data(gc);
 	struct mpc52xx_gpio __iomem *regs = mm_gc->regs;
 	unsigned long flags;
 
@@ -334,7 +328,7 @@ static int mpc52xx_simple_gpiochip_probe(struct platform_device *ofdev)
 	gc->get              = mpc52xx_simple_gpio_get;
 	gc->set              = mpc52xx_simple_gpio_set;
 
-	ret = of_mm_gpiochip_add(ofdev->dev.of_node, &chip->mmchip);
+	ret = of_mm_gpiochip_add_data(ofdev->dev.of_node, &chip->mmchip, chip);
 	if (ret)
 		return ret;
 
@@ -360,15 +354,14 @@ static struct platform_driver mpc52xx_simple_gpiochip_driver = {
 	.remove = mpc52xx_gpiochip_remove,
 };
 
+static struct platform_driver * const drivers[] = {
+	&mpc52xx_wkup_gpiochip_driver,
+	&mpc52xx_simple_gpiochip_driver,
+};
+
 static int __init mpc52xx_gpio_init(void)
 {
-	if (platform_driver_register(&mpc52xx_wkup_gpiochip_driver))
-		printk(KERN_ERR "Unable to register wakeup GPIO driver\n");
-
-	if (platform_driver_register(&mpc52xx_simple_gpiochip_driver))
-		printk(KERN_ERR "Unable to register simple GPIO driver\n");
-
-	return 0;
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
 /* Make sure we get initialised before anyone else tries to use us */
@@ -376,9 +369,7 @@ subsys_initcall(mpc52xx_gpio_init);
 
 static void __exit mpc52xx_gpio_exit(void)
 {
-	platform_driver_unregister(&mpc52xx_wkup_gpiochip_driver);
-
-	platform_driver_unregister(&mpc52xx_simple_gpiochip_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 module_exit(mpc52xx_gpio_exit);
 
diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c
index 48ef368347ab..9d40787e66c0 100644
--- a/drivers/gpio/gpio-mpc8xxx.c
+++ b/drivers/gpio/gpio-mpc8xxx.c
@@ -49,15 +49,10 @@ static inline u32 mpc8xxx_gpio2mask(unsigned int gpio)
 	return 1u << (MPC8XXX_GPIO_PINS - 1 - gpio);
 }
 
-static inline struct mpc8xxx_gpio_chip *
-to_mpc8xxx_gpio_chip(struct of_mm_gpio_chip *mm)
-{
-	return container_of(mm, struct mpc8xxx_gpio_chip, mm_gc);
-}
-
 static void mpc8xxx_gpio_save_regs(struct of_mm_gpio_chip *mm)
 {
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc =
+		container_of(mm, struct mpc8xxx_gpio_chip, mm_gc);
 
 	mpc8xxx_gc->data = in_be32(mm->regs + GPIO_DAT);
 }
@@ -71,7 +66,7 @@ static int mpc8572_gpio_get(struct gpio_chip *gc, unsigned int gpio)
 {
 	u32 val;
 	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 	u32 out_mask, out_shadow;
 
 	out_mask = in_be32(mm->regs + GPIO_DIR);
@@ -79,7 +74,7 @@ static int mpc8572_gpio_get(struct gpio_chip *gc, unsigned int gpio)
 	val = in_be32(mm->regs + GPIO_DAT) & ~out_mask;
 	out_shadow = mpc8xxx_gc->data & out_mask;
 
-	return (val | out_shadow) & mpc8xxx_gpio2mask(gpio);
+	return !!((val | out_shadow) & mpc8xxx_gpio2mask(gpio));
 }
 
 static int mpc8xxx_gpio_get(struct gpio_chip *gc, unsigned int gpio)
@@ -92,7 +87,7 @@ static int mpc8xxx_gpio_get(struct gpio_chip *gc, unsigned int gpio)
 static void mpc8xxx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags);
@@ -111,7 +106,7 @@ static void mpc8xxx_gpio_set_multiple(struct gpio_chip *gc,
 				      unsigned long *mask, unsigned long *bits)
 {
 	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 	int i;
 
@@ -136,7 +131,7 @@ static void mpc8xxx_gpio_set_multiple(struct gpio_chip *gc,
 static int mpc8xxx_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags);
@@ -151,7 +146,7 @@ static int mpc8xxx_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 static int mpc8xxx_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	mpc8xxx_gpio_set(gc, gpio, val);
@@ -185,8 +180,7 @@ static int mpc5125_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val
 
 static int mpc8xxx_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc);
-	struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm);
+	struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
 
 	if (mpc8xxx_gc->irq && offset < MPC8XXX_GPIO_PINS)
 		return irq_create_mapping(mpc8xxx_gc->irq, offset);
@@ -417,7 +411,7 @@ static int mpc8xxx_probe(struct platform_device *pdev)
 	gc->set_multiple = mpc8xxx_gpio_set_multiple;
 	gc->to_irq = mpc8xxx_gpio_to_irq;
 
-	ret = of_mm_gpiochip_add(np, mm_gc);
+	ret = of_mm_gpiochip_add_data(np, mm_gc, mpc8xxx_gc);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c
index 22523aae8abe..d75649787e6c 100644
--- a/drivers/gpio/gpio-msic.c
+++ b/drivers/gpio/gpio-msic.c
@@ -143,7 +143,7 @@ static int msic_gpio_get(struct gpio_chip *chip, unsigned offset)
 	if (ret < 0)
 		return ret;
 
-	return r & MSIC_GPIO_DIN_MASK;
+	return !!(r & MSIC_GPIO_DIN_MASK);
 }
 
 static void msic_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
@@ -179,7 +179,7 @@ static int msic_irq_type(struct irq_data *data, unsigned type)
 
 static int msic_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct msic_gpio *mg = container_of(chip, struct msic_gpio, chip);
+	struct msic_gpio *mg = gpiochip_get_data(chip);
 	return mg->irq_base + offset;
 }
 
@@ -293,11 +293,11 @@ static int platform_msic_gpio_probe(struct platform_device *pdev)
 	mg->chip.base = pdata->gpio_base;
 	mg->chip.ngpio = MSIC_NUM_GPIO;
 	mg->chip.can_sleep = true;
-	mg->chip.dev = dev;
+	mg->chip.parent = dev;
 
 	mutex_init(&mg->buslock);
 
-	retval = gpiochip_add(&mg->chip);
+	retval = gpiochip_add_data(&mg->chip, mg);
 	if (retval) {
 		dev_err(dev, "Adding MSIC gpio chip failed\n");
 		goto err;
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index d428b97876c5..a5eacc1dff09 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -187,8 +187,7 @@ static void __iomem *mvebu_gpioreg_level_mask(struct mvebu_gpio_chip *mvchip)
 
 static void mvebu_gpio_set(struct gpio_chip *chip, unsigned pin, int value)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 u;
 
@@ -204,8 +203,7 @@ static void mvebu_gpio_set(struct gpio_chip *chip, unsigned pin, int value)
 
 static int mvebu_gpio_get(struct gpio_chip *chip, unsigned pin)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	u32 u;
 
 	if (readl_relaxed(mvebu_gpioreg_io_conf(mvchip)) & (1 << pin)) {
@@ -220,8 +218,7 @@ static int mvebu_gpio_get(struct gpio_chip *chip, unsigned pin)
 
 static void mvebu_gpio_blink(struct gpio_chip *chip, unsigned pin, int value)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 u;
 
@@ -237,8 +234,7 @@ static void mvebu_gpio_blink(struct gpio_chip *chip, unsigned pin, int value)
 
 static int mvebu_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	unsigned long flags;
 	int ret;
 	u32 u;
@@ -261,8 +257,7 @@ static int mvebu_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 static int mvebu_gpio_direction_output(struct gpio_chip *chip, unsigned pin,
 				       int value)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	unsigned long flags;
 	int ret;
 	u32 u;
@@ -287,8 +282,7 @@ static int mvebu_gpio_direction_output(struct gpio_chip *chip, unsigned pin,
 
 static int mvebu_gpio_to_irq(struct gpio_chip *chip, unsigned pin)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	return irq_create_mapping(mvchip->domain, pin);
 }
 
@@ -494,8 +488,7 @@ static void mvebu_gpio_irq_handler(struct irq_desc *desc)
 
 static void mvebu_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct mvebu_gpio_chip *mvchip =
-		container_of(chip, struct mvebu_gpio_chip, chip);
+	struct mvebu_gpio_chip *mvchip = gpiochip_get_data(chip);
 	u32 out, io_conf, blink, in_pol, data_in, cause, edg_msk, lvl_msk;
 	int i;
 
@@ -698,7 +691,7 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 
 	mvchip->soc_variant = soc_variant;
 	mvchip->chip.label = dev_name(&pdev->dev);
-	mvchip->chip.dev = &pdev->dev;
+	mvchip->chip.parent = &pdev->dev;
 	mvchip->chip.request = gpiochip_generic_request;
 	mvchip->chip.free = gpiochip_generic_free;
 	mvchip->chip.direction_input = mvebu_gpio_direction_input;
@@ -763,7 +756,7 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 		BUG();
 	}
 
-	gpiochip_add(&mvchip->chip);
+	gpiochip_add_data(&mvchip->chip, mvchip);
 
 	/* Some gpio controllers do not provide irq support */
 	if (!of_irq_count(np))
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index 6ea8df6c7397..7fd21cb53c81 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -26,10 +26,11 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
-#include <linux/gpio.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
+/* FIXME: for gpio_get_value() replace this with direct register read */
+#include <linux/gpio.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/module.h>
@@ -64,7 +65,7 @@ struct mxc_gpio_port {
 	int irq;
 	int irq_high;
 	struct irq_domain *domain;
-	struct bgpio_chip bgc;
+	struct gpio_chip gc;
 	u32 both_edges;
 };
 
@@ -172,7 +173,7 @@ static int gpio_set_irq_type(struct irq_data *d, u32 type)
 	struct mxc_gpio_port *port = gc->private;
 	u32 bit, val;
 	u32 gpio_idx = d->hwirq;
-	u32 gpio = port->bgc.gc.base + gpio_idx;
+	u32 gpio = port->gc.base + gpio_idx;
 	int edge;
 	void __iomem *reg = port->base;
 
@@ -398,9 +399,7 @@ static void mxc_gpio_get_hw(struct platform_device *pdev)
 
 static int mxc_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	struct mxc_gpio_port *port =
-		container_of(bgc, struct mxc_gpio_port, bgc);
+	struct mxc_gpio_port *port = gpiochip_get_data(gc);
 
 	return irq_find_mapping(port->domain, offset);
 }
@@ -451,7 +450,7 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 							 port);
 	}
 
-	err = bgpio_init(&port->bgc, &pdev->dev, 4,
+	err = bgpio_init(&port->gc, &pdev->dev, 4,
 			 port->base + GPIO_PSR,
 			 port->base + GPIO_DR, NULL,
 			 port->base + GPIO_GDIR, NULL,
@@ -459,13 +458,13 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 	if (err)
 		goto out_bgio;
 
-	port->bgc.gc.to_irq = mxc_gpio_to_irq;
-	port->bgc.gc.base = (pdev->id < 0) ? of_alias_get_id(np, "gpio") * 32 :
+	port->gc.to_irq = mxc_gpio_to_irq;
+	port->gc.base = (pdev->id < 0) ? of_alias_get_id(np, "gpio") * 32 :
 					     pdev->id * 32;
 
-	err = gpiochip_add(&port->bgc.gc);
+	err = gpiochip_add_data(&port->gc, port);
 	if (err)
-		goto out_bgpio_remove;
+		goto out_bgio;
 
 	irq_base = irq_alloc_descs(-1, 0, 32, numa_node_id());
 	if (irq_base < 0) {
@@ -494,9 +493,7 @@ out_irqdomain_remove:
 out_irqdesc_free:
 	irq_free_descs(irq_base, 32);
 out_gpiochip_remove:
-	gpiochip_remove(&port->bgc.gc);
-out_bgpio_remove:
-	bgpio_remove(&port->bgc);
+	gpiochip_remove(&port->gc);
 out_bgio:
 	dev_info(&pdev->dev, "%s failed with errno %d\n", __func__, err);
 	return err;
diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c
index a4288f428819..b9daa0bf32a4 100644
--- a/drivers/gpio/gpio-mxs.c
+++ b/drivers/gpio/gpio-mxs.c
@@ -26,13 +26,14 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
-#include <linux/gpio.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
+/* FIXME: for gpio_get_value(), replace this by direct register read */
+#include <linux/gpio.h>
 #include <linux/module.h>
 
 #define MXS_SET		0x4
@@ -64,7 +65,7 @@ struct mxs_gpio_port {
 	int id;
 	int irq;
 	struct irq_domain *domain;
-	struct bgpio_chip bgc;
+	struct gpio_chip gc;
 	enum mxs_gpio_id devid;
 	u32 both_edges;
 };
@@ -93,7 +94,7 @@ static int mxs_gpio_set_irq_type(struct irq_data *d, unsigned int type)
 	port->both_edges &= ~pin_mask;
 	switch (type) {
 	case IRQ_TYPE_EDGE_BOTH:
-		val = gpio_get_value(port->bgc.gc.base + d->hwirq);
+		val = gpio_get_value(port->gc.base + d->hwirq);
 		if (val)
 			edge = GPIO_INT_FALL_EDGE;
 		else
@@ -225,18 +226,14 @@ static int __init mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
 
 static int mxs_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	struct mxs_gpio_port *port =
-		container_of(bgc, struct mxs_gpio_port, bgc);
+	struct mxs_gpio_port *port = gpiochip_get_data(gc);
 
 	return irq_find_mapping(port->domain, offset);
 }
 
 static int mxs_gpio_get_direction(struct gpio_chip *gc, unsigned offset)
 {
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-	struct mxs_gpio_port *port =
-		container_of(bgc, struct mxs_gpio_port, bgc);
+	struct mxs_gpio_port *port = gpiochip_get_data(gc);
 	u32 mask = 1 << offset;
 	u32 dir;
 
@@ -330,26 +327,24 @@ static int mxs_gpio_probe(struct platform_device *pdev)
 	irq_set_chained_handler_and_data(port->irq, mxs_gpio_irq_handler,
 					 port);
 
-	err = bgpio_init(&port->bgc, &pdev->dev, 4,
+	err = bgpio_init(&port->gc, &pdev->dev, 4,
 			 port->base + PINCTRL_DIN(port),
 			 port->base + PINCTRL_DOUT(port) + MXS_SET,
 			 port->base + PINCTRL_DOUT(port) + MXS_CLR,
 			 port->base + PINCTRL_DOE(port), NULL, 0);
 	if (err)
-		goto out_irqdesc_free;
+		goto out_irqdomain_remove;
 
-	port->bgc.gc.to_irq = mxs_gpio_to_irq;
-	port->bgc.gc.get_direction = mxs_gpio_get_direction;
-	port->bgc.gc.base = port->id * 32;
+	port->gc.to_irq = mxs_gpio_to_irq;
+	port->gc.get_direction = mxs_gpio_get_direction;
+	port->gc.base = port->id * 32;
 
-	err = gpiochip_add(&port->bgc.gc);
+	err = gpiochip_add_data(&port->gc, port);
 	if (err)
-		goto out_bgpio_remove;
+		goto out_irqdomain_remove;
 
 	return 0;
 
-out_bgpio_remove:
-	bgpio_remove(&port->bgc);
 out_irqdomain_remove:
 	irq_domain_remove(port->domain);
 out_irqdesc_free:
diff --git a/drivers/gpio/gpio-octeon.c b/drivers/gpio/gpio-octeon.c
index 62ae251d4490..7665ebcd0c1d 100644
--- a/drivers/gpio/gpio-octeon.c
+++ b/drivers/gpio/gpio-octeon.c
@@ -41,7 +41,7 @@ struct octeon_gpio {
 
 static int octeon_gpio_dir_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct octeon_gpio *gpio = container_of(chip, struct octeon_gpio, chip);
+	struct octeon_gpio *gpio = gpiochip_get_data(chip);
 
 	cvmx_write_csr(gpio->register_base + bit_cfg_reg(offset), 0);
 	return 0;
@@ -49,7 +49,7 @@ static int octeon_gpio_dir_in(struct gpio_chip *chip, unsigned offset)
 
 static void octeon_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct octeon_gpio *gpio = container_of(chip, struct octeon_gpio, chip);
+	struct octeon_gpio *gpio = gpiochip_get_data(chip);
 	u64 mask = 1ull << offset;
 	u64 reg = gpio->register_base + (value ? TX_SET : TX_CLEAR);
 	cvmx_write_csr(reg, mask);
@@ -58,7 +58,7 @@ static void octeon_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int octeon_gpio_dir_out(struct gpio_chip *chip, unsigned offset,
 			       int value)
 {
-	struct octeon_gpio *gpio = container_of(chip, struct octeon_gpio, chip);
+	struct octeon_gpio *gpio = gpiochip_get_data(chip);
 	union cvmx_gpio_bit_cfgx cfgx;
 
 	octeon_gpio_set(chip, offset, value);
@@ -72,7 +72,7 @@ static int octeon_gpio_dir_out(struct gpio_chip *chip, unsigned offset,
 
 static int octeon_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct octeon_gpio *gpio = container_of(chip, struct octeon_gpio, chip);
+	struct octeon_gpio *gpio = gpiochip_get_data(chip);
 	u64 read_bits = cvmx_read_csr(gpio->register_base + RX_DAT);
 
 	return ((1ull << offset) & read_bits) != 0;
@@ -108,7 +108,7 @@ static int octeon_gpio_probe(struct platform_device *pdev)
 
 	pdev->dev.platform_data = chip;
 	chip->label = "octeon-gpio";
-	chip->dev = &pdev->dev;
+	chip->parent = &pdev->dev;
 	chip->owner = THIS_MODULE;
 	chip->base = 0;
 	chip->can_sleep = false;
@@ -117,7 +117,7 @@ static int octeon_gpio_probe(struct platform_device *pdev)
 	chip->get = octeon_gpio_get;
 	chip->direction_output = octeon_gpio_dir_out;
 	chip->set = octeon_gpio_set;
-	err = gpiochip_add(chip);
+	err = gpiochip_add_data(chip, gpio);
 	if (err)
 		goto out;
 
@@ -128,7 +128,7 @@ out:
 
 static int octeon_gpio_remove(struct platform_device *pdev)
 {
-	struct gpio_chip *chip = pdev->dev.platform_data;
+	struct gpio_chip *chip = dev_get_platdata(&pdev->dev);
 	gpiochip_remove(chip);
 	return 0;
 }
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index f7fbb46d5d79..189f672bebc1 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -93,7 +93,7 @@ static void omap_gpio_unmask_irq(struct irq_data *d);
 static inline struct gpio_bank *omap_irq_data_get_bank(struct irq_data *d)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
-	return container_of(chip, struct gpio_bank, chip);
+	return gpiochip_get_data(chip);
 }
 
 static void omap_set_gpio_direction(struct gpio_bank *bank, int gpio,
@@ -661,7 +661,7 @@ static int omap_gpio_wake_enable(struct irq_data *d, unsigned int enable)
 
 static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_bank *bank = container_of(chip, struct gpio_bank, chip);
+	struct gpio_bank *bank = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	/*
@@ -681,7 +681,7 @@ static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_bank *bank = container_of(chip, struct gpio_bank, chip);
+	struct gpio_bank *bank = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&bank->lock, flags);
@@ -954,7 +954,7 @@ static int omap_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 	void __iomem *reg;
 	int dir;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 	reg = bank->base + bank->regs->direction;
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	dir = !!(readl_relaxed(reg) & BIT(offset));
@@ -967,7 +967,7 @@ static int omap_gpio_input(struct gpio_chip *chip, unsigned offset)
 	struct gpio_bank *bank;
 	unsigned long flags;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	omap_set_gpio_direction(bank, offset, 1);
 	raw_spin_unlock_irqrestore(&bank->lock, flags);
@@ -978,7 +978,7 @@ static int omap_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	struct gpio_bank *bank;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 
 	if (omap_gpio_is_input(bank, offset))
 		return omap_get_gpio_datain(bank, offset);
@@ -991,7 +991,7 @@ static int omap_gpio_output(struct gpio_chip *chip, unsigned offset, int value)
 	struct gpio_bank *bank;
 	unsigned long flags;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	bank->set_dataout(bank, offset, value);
 	omap_set_gpio_direction(bank, offset, 0);
@@ -1005,7 +1005,7 @@ static int omap_gpio_debounce(struct gpio_chip *chip, unsigned offset,
 	struct gpio_bank *bank;
 	unsigned long flags;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	omap2_set_gpio_debounce(bank, offset, debounce);
@@ -1019,7 +1019,7 @@ static void omap_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 	struct gpio_bank *bank;
 	unsigned long flags;
 
-	bank = container_of(chip, struct gpio_bank, chip);
+	bank = gpiochip_get_data(chip);
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	bank->set_dataout(bank, offset, value);
 	raw_spin_unlock_irqrestore(&bank->lock, flags);
@@ -1090,7 +1090,7 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 	if (bank->is_mpuio) {
 		bank->chip.label = "mpuio";
 		if (bank->regs->wkup_en)
-			bank->chip.dev = &omap_mpuio_device.dev;
+			bank->chip.parent = &omap_mpuio_device.dev;
 		bank->chip.base = OMAP_MPUIO(0);
 	} else {
 		bank->chip.label = "gpio";
@@ -1098,7 +1098,7 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 	}
 	bank->chip.ngpio = bank->width;
 
-	ret = gpiochip_add(&bank->chip);
+	ret = gpiochip_add_data(&bank->chip, bank);
 	if (ret) {
 		dev_err(bank->dev, "Could not register gpio chip %d\n", ret);
 		return ret;
@@ -1197,7 +1197,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
 	}
 
 	bank->dev = dev;
-	bank->chip.dev = dev;
+	bank->chip.parent = dev;
 	bank->chip.owner = THIS_MODULE;
 	bank->dbck_flag = pdata->dbck_flag;
 	bank->stride = pdata->bank_stride;
diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c
index 52b447c071cb..fdfb3b1e0def 100644
--- a/drivers/gpio/gpio-palmas.c
+++ b/drivers/gpio/gpio-palmas.c
@@ -35,14 +35,9 @@ struct palmas_device_data {
 	int ngpio;
 };
 
-static inline struct palmas_gpio *to_palmas_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct palmas_gpio, gpio_chip);
-}
-
 static int palmas_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct palmas_gpio *pg = to_palmas_gpio(gc);
+	struct palmas_gpio *pg = gpiochip_get_data(gc);
 	struct palmas *palmas = pg->palmas;
 	unsigned int val;
 	int ret;
@@ -54,7 +49,7 @@ static int palmas_gpio_get(struct gpio_chip *gc, unsigned offset)
 
 	ret = palmas_read(palmas, PALMAS_GPIO_BASE, reg, &val);
 	if (ret < 0) {
-		dev_err(gc->dev, "Reg 0x%02x read failed, %d\n", reg, ret);
+		dev_err(gc->parent, "Reg 0x%02x read failed, %d\n", reg, ret);
 		return ret;
 	}
 
@@ -65,7 +60,7 @@ static int palmas_gpio_get(struct gpio_chip *gc, unsigned offset)
 
 	ret = palmas_read(palmas, PALMAS_GPIO_BASE, reg, &val);
 	if (ret < 0) {
-		dev_err(gc->dev, "Reg 0x%02x read failed, %d\n", reg, ret);
+		dev_err(gc->parent, "Reg 0x%02x read failed, %d\n", reg, ret);
 		return ret;
 	}
 	return !!(val & BIT(offset));
@@ -74,7 +69,7 @@ static int palmas_gpio_get(struct gpio_chip *gc, unsigned offset)
 static void palmas_gpio_set(struct gpio_chip *gc, unsigned offset,
 			int value)
 {
-	struct palmas_gpio *pg = to_palmas_gpio(gc);
+	struct palmas_gpio *pg = gpiochip_get_data(gc);
 	struct palmas *palmas = pg->palmas;
 	int ret;
 	unsigned int reg;
@@ -90,13 +85,13 @@ static void palmas_gpio_set(struct gpio_chip *gc, unsigned offset,
 
 	ret = palmas_write(palmas, PALMAS_GPIO_BASE, reg, BIT(offset));
 	if (ret < 0)
-		dev_err(gc->dev, "Reg 0x%02x write failed, %d\n", reg, ret);
+		dev_err(gc->parent, "Reg 0x%02x write failed, %d\n", reg, ret);
 }
 
 static int palmas_gpio_output(struct gpio_chip *gc, unsigned offset,
 				int value)
 {
-	struct palmas_gpio *pg = to_palmas_gpio(gc);
+	struct palmas_gpio *pg = gpiochip_get_data(gc);
 	struct palmas *palmas = pg->palmas;
 	int ret;
 	unsigned int reg;
@@ -111,13 +106,14 @@ static int palmas_gpio_output(struct gpio_chip *gc, unsigned offset,
 	ret = palmas_update_bits(palmas, PALMAS_GPIO_BASE, reg,
 				BIT(offset), BIT(offset));
 	if (ret < 0)
-		dev_err(gc->dev, "Reg 0x%02x update failed, %d\n", reg, ret);
+		dev_err(gc->parent, "Reg 0x%02x update failed, %d\n", reg,
+			ret);
 	return ret;
 }
 
 static int palmas_gpio_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct palmas_gpio *pg = to_palmas_gpio(gc);
+	struct palmas_gpio *pg = gpiochip_get_data(gc);
 	struct palmas *palmas = pg->palmas;
 	int ret;
 	unsigned int reg;
@@ -128,13 +124,14 @@ static int palmas_gpio_input(struct gpio_chip *gc, unsigned offset)
 
 	ret = palmas_update_bits(palmas, PALMAS_GPIO_BASE, reg, BIT(offset), 0);
 	if (ret < 0)
-		dev_err(gc->dev, "Reg 0x%02x update failed, %d\n", reg, ret);
+		dev_err(gc->parent, "Reg 0x%02x update failed, %d\n", reg,
+			ret);
 	return ret;
 }
 
 static int palmas_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct palmas_gpio *pg = to_palmas_gpio(gc);
+	struct palmas_gpio *pg = gpiochip_get_data(gc);
 	struct palmas *palmas = pg->palmas;
 
 	return palmas_irq_get_virq(palmas, PALMAS_GPIO_0_IRQ + offset);
@@ -188,7 +185,7 @@ static int palmas_gpio_probe(struct platform_device *pdev)
 	palmas_gpio->gpio_chip.to_irq = palmas_gpio_to_irq;
 	palmas_gpio->gpio_chip.set	= palmas_gpio_set;
 	palmas_gpio->gpio_chip.get	= palmas_gpio_get;
-	palmas_gpio->gpio_chip.dev = &pdev->dev;
+	palmas_gpio->gpio_chip.parent = &pdev->dev;
 #ifdef CONFIG_OF_GPIO
 	palmas_gpio->gpio_chip.of_node = pdev->dev.of_node;
 #endif
@@ -198,7 +195,7 @@ static int palmas_gpio_probe(struct platform_device *pdev)
 	else
 		palmas_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&palmas_gpio->gpio_chip);
+	ret = gpiochip_add_data(&palmas_gpio->gpio_chip, palmas_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 2d4892cc70fb..23196c5fc17c 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -18,9 +18,7 @@
 #include <linux/i2c.h>
 #include <linux/platform_data/pca953x.h>
 #include <linux/slab.h>
-#ifdef CONFIG_OF_GPIO
 #include <linux/of_platform.h>
-#endif
 #include <linux/acpi.h>
 
 #define PCA953X_INPUT		0
@@ -109,11 +107,6 @@ struct pca953x_chip {
 	unsigned long driver_data;
 };
 
-static inline struct pca953x_chip *to_pca(struct gpio_chip *gc)
-{
-	return container_of(gc, struct pca953x_chip, gpio_chip);
-}
-
 static int pca953x_read_single(struct pca953x_chip *chip, int reg, u32 *val,
 				int off)
 {
@@ -216,7 +209,7 @@ static int pca953x_read_regs(struct pca953x_chip *chip, int reg, u8 *val)
 
 static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
 {
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
 	int ret, offset = 0;
 
@@ -245,7 +238,7 @@ exit:
 static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 		unsigned off, int val)
 {
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
 	int ret, offset = 0;
 
@@ -295,7 +288,7 @@ exit:
 
 static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 {
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u32 reg_val;
 	int ret, offset = 0;
 
@@ -323,7 +316,7 @@ static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 
 static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val)
 {
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
 	int ret, offset = 0;
 
@@ -352,6 +345,43 @@ exit:
 	mutex_unlock(&chip->i2c_lock);
 }
 
+
+static void pca953x_gpio_set_multiple(struct gpio_chip *gc,
+		unsigned long *mask, unsigned long *bits)
+{
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
+	u8 reg_val[MAX_BANK];
+	int ret, offset = 0;
+	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+	int bank;
+
+	switch (chip->chip_type) {
+	case PCA953X_TYPE:
+		offset = PCA953X_OUTPUT;
+		break;
+	case PCA957X_TYPE:
+		offset = PCA957X_OUT;
+		break;
+	}
+
+	memcpy(reg_val, chip->reg_output, NBANK(chip));
+	mutex_lock(&chip->i2c_lock);
+	for(bank=0; bank<NBANK(chip); bank++) {
+		unsigned bankmask = mask[bank/4] >> ((bank % 4) * 8);
+		if(bankmask) {
+			unsigned bankval  = bits[bank/4] >> ((bank % 4) * 8);
+			reg_val[bank] = (reg_val[bank] & ~bankmask) | bankval;
+		}
+	}
+	ret = i2c_smbus_write_i2c_block_data(chip->client, offset << bank_shift, NBANK(chip), reg_val);
+	if (ret)
+		goto exit;
+
+	memcpy(chip->reg_output, reg_val, NBANK(chip));
+exit:
+	mutex_unlock(&chip->i2c_lock);
+}
+
 static void pca953x_setup_gpio(struct pca953x_chip *chip, int gpios)
 {
 	struct gpio_chip *gc;
@@ -362,12 +392,13 @@ static void pca953x_setup_gpio(struct pca953x_chip *chip, int gpios)
 	gc->direction_output = pca953x_gpio_direction_output;
 	gc->get = pca953x_gpio_get_value;
 	gc->set = pca953x_gpio_set_value;
+	gc->set_multiple = pca953x_gpio_set_multiple;
 	gc->can_sleep = true;
 
 	gc->base = chip->gpio_start;
 	gc->ngpio = gpios;
 	gc->label = chip->client->name;
-	gc->dev = &chip->client->dev;
+	gc->parent = &chip->client->dev;
 	gc->owner = THIS_MODULE;
 	gc->names = chip->names;
 }
@@ -376,7 +407,7 @@ static void pca953x_setup_gpio(struct pca953x_chip *chip, int gpios)
 static void pca953x_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 
 	chip->irq_mask[d->hwirq / BANK_SZ] &= ~(1 << (d->hwirq % BANK_SZ));
 }
@@ -384,7 +415,7 @@ static void pca953x_irq_mask(struct irq_data *d)
 static void pca953x_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 
 	chip->irq_mask[d->hwirq / BANK_SZ] |= 1 << (d->hwirq % BANK_SZ);
 }
@@ -392,7 +423,7 @@ static void pca953x_irq_unmask(struct irq_data *d)
 static void pca953x_irq_bus_lock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 
 	mutex_lock(&chip->irq_lock);
 }
@@ -400,7 +431,7 @@ static void pca953x_irq_bus_lock(struct irq_data *d)
 static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 new_irqs;
 	int level, i;
 
@@ -423,7 +454,7 @@ static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
 static int pca953x_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pca953x_chip *chip = to_pca(gc);
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	int bank_nb = d->hwirq / BANK_SZ;
 	u8 mask = 1 << (d->hwirq % BANK_SZ);
 
@@ -660,6 +691,8 @@ out:
 	return ret;
 }
 
+static const struct of_device_id pca953x_dt_ids[];
+
 static int pca953x_probe(struct i2c_client *client,
 				   const struct i2c_device_id *id)
 {
@@ -691,12 +724,18 @@ static int pca953x_probe(struct i2c_client *client,
 		chip->driver_data = id->driver_data;
 	} else {
 		const struct acpi_device_id *id;
+		const struct of_device_id *match;
 
-		id = acpi_match_device(pca953x_acpi_ids, &client->dev);
-		if (!id)
-			return -ENODEV;
+		match = of_match_device(pca953x_dt_ids, &client->dev);
+		if (match) {
+			chip->driver_data = (int)(uintptr_t)match->data;
+		} else {
+			id = acpi_match_device(pca953x_acpi_ids, &client->dev);
+			if (!id)
+				return -ENODEV;
 
-		chip->driver_data = id->driver_data;
+			chip->driver_data = id->driver_data;
+		}
 	}
 
 	chip->chip_type = PCA_CHIP_TYPE(chip->driver_data);
@@ -715,7 +754,7 @@ static int pca953x_probe(struct i2c_client *client,
 	if (ret)
 		return ret;
 
-	ret = gpiochip_add(&chip->gpio_chip);
+	ret = gpiochip_add_data(&chip->gpio_chip, chip);
 	if (ret)
 		return ret;
 
@@ -755,33 +794,39 @@ static int pca953x_remove(struct i2c_client *client)
 	return 0;
 }
 
+/* convenience to stop overlong match-table lines */
+#define OF_953X(__nrgpio, __int) (void *)(__nrgpio | PCA953X_TYPE | __int)
+#define OF_957X(__nrgpio, __int) (void *)(__nrgpio | PCA957X_TYPE | __int)
+
 static const struct of_device_id pca953x_dt_ids[] = {
-	{ .compatible = "nxp,pca9505", },
-	{ .compatible = "nxp,pca9534", },
-	{ .compatible = "nxp,pca9535", },
-	{ .compatible = "nxp,pca9536", },
-	{ .compatible = "nxp,pca9537", },
-	{ .compatible = "nxp,pca9538", },
-	{ .compatible = "nxp,pca9539", },
-	{ .compatible = "nxp,pca9554", },
-	{ .compatible = "nxp,pca9555", },
-	{ .compatible = "nxp,pca9556", },
-	{ .compatible = "nxp,pca9557", },
-	{ .compatible = "nxp,pca9574", },
-	{ .compatible = "nxp,pca9575", },
-	{ .compatible = "nxp,pca9698", },
-
-	{ .compatible = "maxim,max7310", },
-	{ .compatible = "maxim,max7312", },
-	{ .compatible = "maxim,max7313", },
-	{ .compatible = "maxim,max7315", },
-
-	{ .compatible = "ti,pca6107", },
-	{ .compatible = "ti,tca6408", },
-	{ .compatible = "ti,tca6416", },
-	{ .compatible = "ti,tca6424", },
-
-	{ .compatible = "exar,xra1202", },
+	{ .compatible = "nxp,pca9505", .data = OF_953X(40, PCA_INT), },
+	{ .compatible = "nxp,pca9534", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "nxp,pca9535", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "nxp,pca9536", .data = OF_953X( 4, 0), },
+	{ .compatible = "nxp,pca9537", .data = OF_953X( 4, PCA_INT), },
+	{ .compatible = "nxp,pca9538", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "nxp,pca9539", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "nxp,pca9554", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "nxp,pca9555", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "nxp,pca9556", .data = OF_953X( 8, 0), },
+	{ .compatible = "nxp,pca9557", .data = OF_953X( 8, 0), },
+	{ .compatible = "nxp,pca9574", .data = OF_957X( 8, PCA_INT), },
+	{ .compatible = "nxp,pca9575", .data = OF_957X(16, PCA_INT), },
+	{ .compatible = "nxp,pca9698", .data = OF_953X(40, 0), },
+
+	{ .compatible = "maxim,max7310", .data = OF_953X( 8, 0), },
+	{ .compatible = "maxim,max7312", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "maxim,max7313", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "maxim,max7315", .data = OF_953X( 8, PCA_INT), },
+
+	{ .compatible = "ti,pca6107", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "ti,tca6408", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "ti,tca6416", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "ti,tca6424", .data = OF_953X(24, PCA_INT), },
+
+	{ .compatible = "onsemi,pca9654", .data = OF_953X( 8, PCA_INT), },
+
+	{ .compatible = "exar,xra1202", .data = OF_953X( 8, 0), },
 	{ }
 };
 
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index 1d4d9bc8b69d..709cd3fc2a70 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -137,7 +137,7 @@ static int i2c_read_le16(struct i2c_client *client)
 
 static int pcf857x_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct pcf857x	*gpio = container_of(chip, struct pcf857x, chip);
+	struct pcf857x	*gpio = gpiochip_get_data(chip);
 	int		status;
 
 	mutex_lock(&gpio->lock);
@@ -150,16 +150,16 @@ static int pcf857x_input(struct gpio_chip *chip, unsigned offset)
 
 static int pcf857x_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct pcf857x	*gpio = container_of(chip, struct pcf857x, chip);
+	struct pcf857x	*gpio = gpiochip_get_data(chip);
 	int		value;
 
 	value = gpio->read(gpio->client);
-	return (value < 0) ? 0 : (value & (1 << offset));
+	return (value < 0) ? value : !!(value & (1 << offset));
 }
 
 static int pcf857x_output(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct pcf857x	*gpio = container_of(chip, struct pcf857x, chip);
+	struct pcf857x	*gpio = gpiochip_get_data(chip);
 	unsigned	bit = 1 << offset;
 	int		status;
 
@@ -293,7 +293,7 @@ static int pcf857x_probe(struct i2c_client *client,
 
 	gpio->chip.base			= pdata ? pdata->gpio_base : -1;
 	gpio->chip.can_sleep		= true;
-	gpio->chip.dev			= &client->dev;
+	gpio->chip.parent		= &client->dev;
 	gpio->chip.owner		= THIS_MODULE;
 	gpio->chip.get			= pcf857x_get;
 	gpio->chip.set			= pcf857x_set;
@@ -372,7 +372,7 @@ static int pcf857x_probe(struct i2c_client *client,
 	gpio->out = ~n_latch;
 	gpio->status = gpio->out;
 
-	status = gpiochip_add(&gpio->chip);
+	status = gpiochip_add_data(&gpio->chip, gpio);
 	if (status < 0)
 		goto fail;
 
@@ -447,7 +447,6 @@ static int pcf857x_remove(struct i2c_client *client)
 static struct i2c_driver pcf857x_driver = {
 	.driver = {
 		.name	= "pcf857x",
-		.owner	= THIS_MODULE,
 		.of_match_table = of_match_ptr(pcf857x_of_table),
 	},
 	.probe	= pcf857x_probe,
diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c
index 34ed176df15a..7c7135da5d4a 100644
--- a/drivers/gpio/gpio-pch.c
+++ b/drivers/gpio/gpio-pch.c
@@ -109,7 +109,7 @@ struct pch_gpio {
 static void pch_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 {
 	u32 reg_val;
-	struct pch_gpio *chip =	container_of(gpio, struct pch_gpio, gpio);
+	struct pch_gpio *chip =	gpiochip_get_data(gpio);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->spinlock, flags);
@@ -125,15 +125,15 @@ static void pch_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 
 static int pch_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
-	struct pch_gpio *chip =	container_of(gpio, struct pch_gpio, gpio);
+	struct pch_gpio *chip =	gpiochip_get_data(gpio);
 
-	return ioread32(&chip->reg->pi) & (1 << nr);
+	return (ioread32(&chip->reg->pi) >> nr) & 1;
 }
 
 static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 				     int val)
 {
-	struct pch_gpio *chip =	container_of(gpio, struct pch_gpio, gpio);
+	struct pch_gpio *chip =	gpiochip_get_data(gpio);
 	u32 pm;
 	u32 reg_val;
 	unsigned long flags;
@@ -158,7 +158,7 @@ static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 
 static int pch_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 {
-	struct pch_gpio *chip =	container_of(gpio, struct pch_gpio, gpio);
+	struct pch_gpio *chip =	gpiochip_get_data(gpio);
 	u32 pm;
 	unsigned long flags;
 
@@ -211,7 +211,7 @@ static void pch_gpio_restore_reg_conf(struct pch_gpio *chip)
 
 static int pch_gpio_to_irq(struct gpio_chip *gpio, unsigned offset)
 {
-	struct pch_gpio *chip = container_of(gpio, struct pch_gpio, gpio);
+	struct pch_gpio *chip = gpiochip_get_data(gpio);
 	return chip->irq_base + offset;
 }
 
@@ -220,7 +220,7 @@ static void pch_gpio_setup(struct pch_gpio *chip)
 	struct gpio_chip *gpio = &chip->gpio;
 
 	gpio->label = dev_name(chip->dev);
-	gpio->dev = chip->dev;
+	gpio->parent = chip->dev;
 	gpio->owner = THIS_MODULE;
 	gpio->direction_input = pch_gpio_direction_input;
 	gpio->get = pch_gpio_get;
@@ -394,7 +394,10 @@ static int pch_gpio_probe(struct pci_dev *pdev,
 	pci_set_drvdata(pdev, chip);
 	spin_lock_init(&chip->spinlock);
 	pch_gpio_setup(chip);
-	ret = gpiochip_add(&chip->gpio);
+#ifdef CONFIG_OF_GPIO
+	chip->gpio.of_node = pdev->dev.of_node;
+#endif
+	ret = gpiochip_add_data(&chip->gpio, chip);
 	if (ret) {
 		dev_err(&pdev->dev, "PCH gpio: Failed to register GPIO\n");
 		goto err_gpiochip_add;
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index 4d4b37676702..5cb38212bbc0 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
+#include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/bitops.h>
@@ -60,7 +61,7 @@ struct pl061_gpio {
 
 static int pl061_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char gpiodir;
 
@@ -79,7 +80,7 @@ static int pl061_direction_input(struct gpio_chip *gc, unsigned offset)
 static int pl061_direction_output(struct gpio_chip *gc, unsigned offset,
 		int value)
 {
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char gpiodir;
 
@@ -104,14 +105,14 @@ static int pl061_direction_output(struct gpio_chip *gc, unsigned offset,
 
 static int pl061_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 
 	return !!readb(chip->base + (BIT(offset + 2)));
 }
 
 static void pl061_set_value(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 
 	writeb(!!value << offset, chip->base + (BIT(offset + 2)));
 }
@@ -119,7 +120,7 @@ static void pl061_set_value(struct gpio_chip *gc, unsigned offset, int value)
 static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	int offset = irqd_to_hwirq(d);
 	unsigned long flags;
 	u8 gpiois, gpioibe, gpioiev;
@@ -131,7 +132,7 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 	if ((trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) &&
 	    (trigger & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)))
 	{
-		dev_err(gc->dev,
+		dev_err(gc->parent,
 			"trying to configure line %d for both level and edge "
 			"detection, choose one!\n",
 			offset);
@@ -158,7 +159,7 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 		else
 			gpioiev &= ~bit;
 		irq_set_handler_locked(d, handle_level_irq);
-		dev_dbg(gc->dev, "line %d: IRQ on %s level\n",
+		dev_dbg(gc->parent, "line %d: IRQ on %s level\n",
 			offset,
 			polarity ? "HIGH" : "LOW");
 	} else if ((trigger & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) {
@@ -167,7 +168,7 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 		/* Select both edges, setting this makes GPIOEV be ignored */
 		gpioibe |= bit;
 		irq_set_handler_locked(d, handle_edge_irq);
-		dev_dbg(gc->dev, "line %d: IRQ on both edges\n", offset);
+		dev_dbg(gc->parent, "line %d: IRQ on both edges\n", offset);
 	} else if ((trigger & IRQ_TYPE_EDGE_RISING) ||
 		   (trigger & IRQ_TYPE_EDGE_FALLING)) {
 		bool rising = trigger & IRQ_TYPE_EDGE_RISING;
@@ -182,7 +183,7 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 		else
 			gpioiev &= ~bit;
 		irq_set_handler_locked(d, handle_edge_irq);
-		dev_dbg(gc->dev, "line %d: IRQ on %s edge\n",
+		dev_dbg(gc->parent, "line %d: IRQ on %s edge\n",
 			offset,
 			rising ? "RISING" : "FALLING");
 	} else {
@@ -191,7 +192,7 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 		gpioibe &= ~bit;
 		gpioiev &= ~bit;
 		irq_set_handler_locked(d, handle_bad_irq);
-		dev_warn(gc->dev, "no trigger selected for line %d\n",
+		dev_warn(gc->parent, "no trigger selected for line %d\n",
 			 offset);
 	}
 
@@ -209,7 +210,7 @@ static void pl061_irq_handler(struct irq_desc *desc)
 	unsigned long pending;
 	int offset;
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 
 	chained_irq_enter(irqchip, desc);
@@ -227,7 +228,7 @@ static void pl061_irq_handler(struct irq_desc *desc)
 static void pl061_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 	u8 gpioie;
 
@@ -240,7 +241,7 @@ static void pl061_irq_mask(struct irq_data *d)
 static void pl061_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 	u8 gpioie;
 
@@ -261,7 +262,7 @@ static void pl061_irq_unmask(struct irq_data *d)
 static void pl061_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = container_of(gc, struct pl061_gpio, gc);
+	struct pl061_gpio *chip = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 
 	spin_lock(&chip->lock);
@@ -269,12 +270,20 @@ static void pl061_irq_ack(struct irq_data *d)
 	spin_unlock(&chip->lock);
 }
 
+static int pl061_irq_set_wake(struct irq_data *d, unsigned int state)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	return irq_set_irq_wake(gc->irq_parent, state);
+}
+
 static struct irq_chip pl061_irqchip = {
 	.name		= "pl061",
 	.irq_ack	= pl061_irq_ack,
 	.irq_mask	= pl061_irq_mask,
 	.irq_unmask	= pl061_irq_unmask,
 	.irq_set_type	= pl061_irq_type,
+	.irq_set_wake	= pl061_irq_set_wake,
 };
 
 static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
@@ -316,10 +325,10 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
 	chip->gc.set = pl061_set_value;
 	chip->gc.ngpio = PL061_GPIO_NR;
 	chip->gc.label = dev_name(dev);
-	chip->gc.dev = dev;
+	chip->gc.parent = dev;
 	chip->gc.owner = THIS_MODULE;
 
-	ret = gpiochip_add(&chip->gc);
+	ret = gpiochip_add_data(&chip->gc, chip);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c
index df2ce550f309..b2b7b78664b8 100644
--- a/drivers/gpio/gpio-pxa.c
+++ b/drivers/gpio/gpio-pxa.c
@@ -24,6 +24,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/syscore_ops.h>
 #include <linux/slab.h>
@@ -64,20 +65,11 @@
 int pxa_last_gpio;
 static int irq_base;
 
-#ifdef CONFIG_OF
-static struct irq_domain *domain;
-static struct device_node *pxa_gpio_of_node;
-#endif
-
-struct pxa_gpio_chip {
-	struct gpio_chip chip;
+struct pxa_gpio_bank {
 	void __iomem	*regbase;
-	char label[10];
-
 	unsigned long	irq_mask;
 	unsigned long	irq_edge_rise;
 	unsigned long	irq_edge_fall;
-	int (*set_wake)(unsigned int gpio, unsigned int on);
 
 #ifdef CONFIG_PM
 	unsigned long	saved_gplr;
@@ -87,6 +79,17 @@ struct pxa_gpio_chip {
 #endif
 };
 
+struct pxa_gpio_chip {
+	struct device *dev;
+	struct gpio_chip chip;
+	struct pxa_gpio_bank *banks;
+	struct irq_domain *irqdomain;
+
+	int irq0;
+	int irq1;
+	int (*set_wake)(unsigned int gpio, unsigned int on);
+};
+
 enum pxa_gpio_type {
 	PXA25X_GPIO = 0,
 	PXA26X_GPIO,
@@ -104,9 +107,8 @@ struct pxa_gpio_id {
 };
 
 static DEFINE_SPINLOCK(gpio_lock);
-static struct pxa_gpio_chip *pxa_gpio_chips;
+static struct pxa_gpio_chip *pxa_gpio_chip;
 static enum pxa_gpio_type gpio_type;
-static void __iomem *gpio_reg_base;
 
 static struct pxa_gpio_id pxa25x_id = {
 	.type		= PXA25X_GPIO,
@@ -148,17 +150,28 @@ static struct pxa_gpio_id pxa1928_id = {
 	.gpio_nums	= 224,
 };
 
-#define for_each_gpio_chip(i, c)			\
-	for (i = 0, c = &pxa_gpio_chips[0]; i <= pxa_last_gpio; i += 32, c++)
+#define for_each_gpio_bank(i, b, pc)					\
+	for (i = 0, b = pc->banks; i <= pxa_last_gpio; i += 32, b++)
 
-static inline void __iomem *gpio_chip_base(struct gpio_chip *c)
+static inline struct pxa_gpio_chip *chip_to_pxachip(struct gpio_chip *c)
 {
-	return container_of(c, struct pxa_gpio_chip, chip)->regbase;
+	struct pxa_gpio_chip *pxa_chip = gpiochip_get_data(c);
+
+	return pxa_chip;
 }
 
-static inline struct pxa_gpio_chip *gpio_to_pxachip(unsigned gpio)
+static inline void __iomem *gpio_bank_base(struct gpio_chip *c, int gpio)
 {
-	return &pxa_gpio_chips[gpio_to_bank(gpio)];
+	struct pxa_gpio_chip *p = gpiochip_get_data(c);
+	struct pxa_gpio_bank *bank = p->banks + (gpio / 32);
+
+	return bank->regbase;
+}
+
+static inline struct pxa_gpio_bank *gpio_to_pxabank(struct gpio_chip *c,
+						    unsigned gpio)
+{
+	return chip_to_pxachip(c)->banks + gpio / 32;
 }
 
 static inline int gpio_is_pxa_type(int type)
@@ -187,15 +200,13 @@ static inline int __gpio_is_inverted(int gpio)
  * is attributed as "occupied" here (I know this terminology isn't
  * accurate, you are welcome to propose a better one :-)
  */
-static inline int __gpio_is_occupied(unsigned gpio)
+static inline int __gpio_is_occupied(struct pxa_gpio_chip *pchip, unsigned gpio)
 {
-	struct pxa_gpio_chip *pxachip;
 	void __iomem *base;
 	unsigned long gafr = 0, gpdr = 0;
 	int ret, af = 0, dir = 0;
 
-	pxachip = gpio_to_pxachip(gpio);
-	base = gpio_chip_base(&pxachip->chip);
+	base = gpio_bank_base(&pchip->chip, gpio);
 	gpdr = readl_relaxed(base + GPDR_OFFSET);
 
 	switch (gpio_type) {
@@ -218,21 +229,35 @@ static inline int __gpio_is_occupied(unsigned gpio)
 	return ret;
 }
 
-static int pxa_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
+int pxa_irq_to_gpio(int irq)
 {
-	return chip->base + offset + irq_base;
+	struct pxa_gpio_chip *pchip = pxa_gpio_chip;
+	int irq_gpio0;
+
+	irq_gpio0 = irq_find_mapping(pchip->irqdomain, 0);
+	if (irq_gpio0 > 0)
+		return irq - irq_gpio0;
+
+	return irq_gpio0;
 }
 
-int pxa_irq_to_gpio(int irq)
+static int pxa_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	return irq - irq_base;
+	struct pxa_gpio_chip *pchip = chip_to_pxachip(chip);
+
+	return irq_find_mapping(pchip->irqdomain, offset);
 }
 
 static int pxa_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	void __iomem *base = gpio_chip_base(chip);
-	uint32_t value, mask = 1 << offset;
+	void __iomem *base = gpio_bank_base(chip, offset);
+	uint32_t value, mask = GPIO_bit(offset);
 	unsigned long flags;
+	int ret;
+
+	ret = pinctrl_gpio_direction_input(chip->base + offset);
+	if (!ret)
+		return 0;
 
 	spin_lock_irqsave(&gpio_lock, flags);
 
@@ -250,12 +275,17 @@ static int pxa_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int pxa_gpio_direction_output(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
-	void __iomem *base = gpio_chip_base(chip);
-	uint32_t tmp, mask = 1 << offset;
+	void __iomem *base = gpio_bank_base(chip, offset);
+	uint32_t tmp, mask = GPIO_bit(offset);
 	unsigned long flags;
+	int ret;
 
 	writel_relaxed(mask, base + (value ? GPSR_OFFSET : GPCR_OFFSET));
 
+	ret = pinctrl_gpio_direction_output(chip->base + offset);
+	if (!ret)
+		return 0;
+
 	spin_lock_irqsave(&gpio_lock, flags);
 
 	tmp = readl_relaxed(base + GPDR_OFFSET);
@@ -271,14 +301,18 @@ static int pxa_gpio_direction_output(struct gpio_chip *chip,
 
 static int pxa_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	u32 gplr = readl_relaxed(gpio_chip_base(chip) + GPLR_OFFSET);
-	return !!(gplr & (1 << offset));
+	void __iomem *base = gpio_bank_base(chip, offset);
+	u32 gplr = readl_relaxed(base + GPLR_OFFSET);
+
+	return !!(gplr & GPIO_bit(offset));
 }
 
 static void pxa_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	writel_relaxed(1 << offset, gpio_chip_base(chip) +
-				(value ? GPSR_OFFSET : GPCR_OFFSET));
+	void __iomem *base = gpio_bank_base(chip, offset);
+
+	writel_relaxed(GPIO_bit(offset),
+		       base + (value ? GPSR_OFFSET : GPCR_OFFSET));
 }
 
 #ifdef CONFIG_OF_GPIO
@@ -289,61 +323,61 @@ static int pxa_gpio_of_xlate(struct gpio_chip *gc,
 	if (gpiospec->args[0] > pxa_last_gpio)
 		return -EINVAL;
 
-	if (gc != &pxa_gpio_chips[gpiospec->args[0] / 32].chip)
-		return -EINVAL;
-
 	if (flags)
 		*flags = gpiospec->args[1];
 
-	return gpiospec->args[0] % 32;
+	return gpiospec->args[0];
 }
 #endif
 
-static int pxa_init_gpio_chip(int gpio_end,
-					int (*set_wake)(unsigned int, unsigned int))
+static int pxa_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
-	int i, gpio, nbanks = gpio_to_bank(gpio_end) + 1;
-	struct pxa_gpio_chip *chips;
-
-	chips = kzalloc(nbanks * sizeof(struct pxa_gpio_chip), GFP_KERNEL);
-	if (chips == NULL) {
-		pr_err("%s: failed to allocate GPIO chips\n", __func__);
-		return -ENOMEM;
-	}
+	return pinctrl_request_gpio(chip->base + offset);
+}
 
-	for (i = 0, gpio = 0; i < nbanks; i++, gpio += 32) {
-		struct gpio_chip *c = &chips[i].chip;
+static void pxa_gpio_free(struct gpio_chip *chip, unsigned int offset)
+{
+	pinctrl_free_gpio(chip->base + offset);
+}
 
-		sprintf(chips[i].label, "gpio-%d", i);
-		chips[i].regbase = gpio_reg_base + BANK_OFF(i);
-		chips[i].set_wake = set_wake;
+static int pxa_init_gpio_chip(struct pxa_gpio_chip *pchip, int ngpio,
+			      struct device_node *np, void __iomem *regbase)
+{
+	int i, gpio, nbanks = DIV_ROUND_UP(ngpio, 32);
+	struct pxa_gpio_bank *bank;
 
-		c->base  = gpio;
-		c->label = chips[i].label;
+	pchip->banks = devm_kcalloc(pchip->dev, nbanks, sizeof(*pchip->banks),
+				    GFP_KERNEL);
+	if (!pchip->banks)
+		return -ENOMEM;
 
-		c->direction_input  = pxa_gpio_direction_input;
-		c->direction_output = pxa_gpio_direction_output;
-		c->get = pxa_gpio_get;
-		c->set = pxa_gpio_set;
-		c->to_irq = pxa_gpio_to_irq;
+	pchip->chip.label = "gpio-pxa";
+	pchip->chip.direction_input  = pxa_gpio_direction_input;
+	pchip->chip.direction_output = pxa_gpio_direction_output;
+	pchip->chip.get = pxa_gpio_get;
+	pchip->chip.set = pxa_gpio_set;
+	pchip->chip.to_irq = pxa_gpio_to_irq;
+	pchip->chip.ngpio = ngpio;
+	pchip->chip.request = pxa_gpio_request;
+	pchip->chip.free = pxa_gpio_free;
 #ifdef CONFIG_OF_GPIO
-		c->of_node = pxa_gpio_of_node;
-		c->of_xlate = pxa_gpio_of_xlate;
-		c->of_gpio_n_cells = 2;
+	pchip->chip.of_node = np;
+	pchip->chip.of_xlate = pxa_gpio_of_xlate;
+	pchip->chip.of_gpio_n_cells = 2;
 #endif
 
-		/* number of GPIOs on last bank may be less than 32 */
-		c->ngpio = (gpio + 31 > gpio_end) ? (gpio_end - gpio + 1) : 32;
-		gpiochip_add(c);
+	for (i = 0, gpio = 0; i < nbanks; i++, gpio += 32) {
+		bank = pchip->banks + i;
+		bank->regbase = regbase + BANK_OFF(i);
 	}
-	pxa_gpio_chips = chips;
-	return 0;
+
+	return gpiochip_add_data(&pchip->chip, pchip);
 }
 
 /* Update only those GRERx and GFERx edge detection register bits if those
  * bits are set in c->irq_mask
  */
-static inline void update_edge_detect(struct pxa_gpio_chip *c)
+static inline void update_edge_detect(struct pxa_gpio_bank *c)
 {
 	uint32_t grer, gfer;
 
@@ -357,12 +391,11 @@ static inline void update_edge_detect(struct pxa_gpio_chip *c)
 
 static int pxa_gpio_irq_type(struct irq_data *d, unsigned int type)
 {
-	struct pxa_gpio_chip *c;
-	int gpio = pxa_irq_to_gpio(d->irq);
+	struct pxa_gpio_chip *pchip = irq_data_get_irq_chip_data(d);
+	unsigned int gpio = irqd_to_hwirq(d);
+	struct pxa_gpio_bank *c = gpio_to_pxabank(&pchip->chip, gpio);
 	unsigned long gpdr, mask = GPIO_bit(gpio);
 
-	c = gpio_to_pxachip(gpio);
-
 	if (type == IRQ_TYPE_PROBE) {
 		/* Don't mess with enabled GPIOs using preconfigured edges or
 		 * GPIOs set to alternate function or to output during probe
@@ -370,7 +403,7 @@ static int pxa_gpio_irq_type(struct irq_data *d, unsigned int type)
 		if ((c->irq_edge_rise | c->irq_edge_fall) & GPIO_bit(gpio))
 			return 0;
 
-		if (__gpio_is_occupied(gpio))
+		if (__gpio_is_occupied(pchip, gpio))
 			return 0;
 
 		type = IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING;
@@ -401,20 +434,16 @@ static int pxa_gpio_irq_type(struct irq_data *d, unsigned int type)
 	return 0;
 }
 
-static void pxa_gpio_demux_handler(struct irq_desc *desc)
+static irqreturn_t pxa_gpio_demux_handler(int in_irq, void *d)
 {
-	struct pxa_gpio_chip *c;
-	int loop, gpio, gpio_base, n;
+	int loop, gpio, n, handled = 0;
 	unsigned long gedr;
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-
-	chained_irq_enter(chip, desc);
+	struct pxa_gpio_chip *pchip = d;
+	struct pxa_gpio_bank *c;
 
 	do {
 		loop = 0;
-		for_each_gpio_chip(gpio, c) {
-			gpio_base = c->chip.base;
-
+		for_each_gpio_bank(gpio, c, pchip) {
 			gedr = readl_relaxed(c->regbase + GEDR_OFFSET);
 			gedr = gedr & c->irq_mask;
 			writel_relaxed(gedr, c->regbase + GEDR_OFFSET);
@@ -422,51 +451,71 @@ static void pxa_gpio_demux_handler(struct irq_desc *desc)
 			for_each_set_bit(n, &gedr, BITS_PER_LONG) {
 				loop = 1;
 
-				generic_handle_irq(gpio_to_irq(gpio_base + n));
+				generic_handle_irq(gpio_to_irq(gpio + n));
 			}
 		}
+		handled += loop;
 	} while (loop);
 
-	chained_irq_exit(chip, desc);
+	return handled ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static irqreturn_t pxa_gpio_direct_handler(int in_irq, void *d)
+{
+	struct pxa_gpio_chip *pchip = d;
+
+	if (in_irq == pchip->irq0) {
+		generic_handle_irq(gpio_to_irq(0));
+	} else if (in_irq == pchip->irq1) {
+		generic_handle_irq(gpio_to_irq(1));
+	} else {
+		pr_err("%s() unknown irq %d\n", __func__, in_irq);
+		return IRQ_NONE;
+	}
+	return IRQ_HANDLED;
 }
 
 static void pxa_ack_muxed_gpio(struct irq_data *d)
 {
-	int gpio = pxa_irq_to_gpio(d->irq);
-	struct pxa_gpio_chip *c = gpio_to_pxachip(gpio);
+	struct pxa_gpio_chip *pchip = irq_data_get_irq_chip_data(d);
+	unsigned int gpio = irqd_to_hwirq(d);
+	void __iomem *base = gpio_bank_base(&pchip->chip, gpio);
 
-	writel_relaxed(GPIO_bit(gpio), c->regbase + GEDR_OFFSET);
+	writel_relaxed(GPIO_bit(gpio), base + GEDR_OFFSET);
 }
 
 static void pxa_mask_muxed_gpio(struct irq_data *d)
 {
-	int gpio = pxa_irq_to_gpio(d->irq);
-	struct pxa_gpio_chip *c = gpio_to_pxachip(gpio);
+	struct pxa_gpio_chip *pchip = irq_data_get_irq_chip_data(d);
+	unsigned int gpio = irqd_to_hwirq(d);
+	struct pxa_gpio_bank *b = gpio_to_pxabank(&pchip->chip, gpio);
+	void __iomem *base = gpio_bank_base(&pchip->chip, gpio);
 	uint32_t grer, gfer;
 
-	c->irq_mask &= ~GPIO_bit(gpio);
+	b->irq_mask &= ~GPIO_bit(gpio);
 
-	grer = readl_relaxed(c->regbase + GRER_OFFSET) & ~GPIO_bit(gpio);
-	gfer = readl_relaxed(c->regbase + GFER_OFFSET) & ~GPIO_bit(gpio);
-	writel_relaxed(grer, c->regbase + GRER_OFFSET);
-	writel_relaxed(gfer, c->regbase + GFER_OFFSET);
+	grer = readl_relaxed(base + GRER_OFFSET) & ~GPIO_bit(gpio);
+	gfer = readl_relaxed(base + GFER_OFFSET) & ~GPIO_bit(gpio);
+	writel_relaxed(grer, base + GRER_OFFSET);
+	writel_relaxed(gfer, base + GFER_OFFSET);
 }
 
 static int pxa_gpio_set_wake(struct irq_data *d, unsigned int on)
 {
-	int gpio = pxa_irq_to_gpio(d->irq);
-	struct pxa_gpio_chip *c = gpio_to_pxachip(gpio);
+	struct pxa_gpio_chip *pchip = irq_data_get_irq_chip_data(d);
+	unsigned int gpio = irqd_to_hwirq(d);
 
-	if (c->set_wake)
-		return c->set_wake(gpio, on);
+	if (pchip->set_wake)
+		return pchip->set_wake(gpio, on);
 	else
 		return 0;
 }
 
 static void pxa_unmask_muxed_gpio(struct irq_data *d)
 {
-	int gpio = pxa_irq_to_gpio(d->irq);
-	struct pxa_gpio_chip *c = gpio_to_pxachip(gpio);
+	struct pxa_gpio_chip *pchip = irq_data_get_irq_chip_data(d);
+	unsigned int gpio = irqd_to_hwirq(d);
+	struct pxa_gpio_bank *c = gpio_to_pxabank(&pchip->chip, gpio);
 
 	c->irq_mask |= GPIO_bit(gpio);
 	update_edge_detect(c);
@@ -506,24 +555,12 @@ static int pxa_gpio_nums(struct platform_device *pdev)
 	return count;
 }
 
-#ifdef CONFIG_OF
-static const struct of_device_id pxa_gpio_dt_ids[] = {
-	{ .compatible = "intel,pxa25x-gpio",	.data = &pxa25x_id, },
-	{ .compatible = "intel,pxa26x-gpio",	.data = &pxa26x_id, },
-	{ .compatible = "intel,pxa27x-gpio",	.data = &pxa27x_id, },
-	{ .compatible = "intel,pxa3xx-gpio",	.data = &pxa3xx_id, },
-	{ .compatible = "marvell,pxa93x-gpio",	.data = &pxa93x_id, },
-	{ .compatible = "marvell,mmp-gpio",	.data = &mmp_id, },
-	{ .compatible = "marvell,mmp2-gpio",	.data = &mmp2_id, },
-	{ .compatible = "marvell,pxa1928-gpio",	.data = &pxa1928_id, },
-	{}
-};
-
 static int pxa_irq_domain_map(struct irq_domain *d, unsigned int irq,
 			      irq_hw_number_t hw)
 {
 	irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip,
 				 handle_edge_irq);
+	irq_set_chip_data(irq, d->host_data);
 	irq_set_noprobe(irq);
 	return 0;
 }
@@ -533,10 +570,23 @@ const struct irq_domain_ops pxa_irq_domain_ops = {
 	.xlate	= irq_domain_xlate_twocell,
 };
 
-static int pxa_gpio_probe_dt(struct platform_device *pdev)
+#ifdef CONFIG_OF
+static const struct of_device_id pxa_gpio_dt_ids[] = {
+	{ .compatible = "intel,pxa25x-gpio",	.data = &pxa25x_id, },
+	{ .compatible = "intel,pxa26x-gpio",	.data = &pxa26x_id, },
+	{ .compatible = "intel,pxa27x-gpio",	.data = &pxa27x_id, },
+	{ .compatible = "intel,pxa3xx-gpio",	.data = &pxa3xx_id, },
+	{ .compatible = "marvell,pxa93x-gpio",	.data = &pxa93x_id, },
+	{ .compatible = "marvell,mmp-gpio",	.data = &mmp_id, },
+	{ .compatible = "marvell,mmp2-gpio",	.data = &mmp2_id, },
+	{ .compatible = "marvell,pxa1928-gpio",	.data = &pxa1928_id, },
+	{}
+};
+
+static int pxa_gpio_probe_dt(struct platform_device *pdev,
+			     struct pxa_gpio_chip *pchip)
 {
-	int ret = 0, nr_gpios;
-	struct device_node *np = pdev->dev.of_node;
+	int nr_gpios;
 	const struct of_device_id *of_id =
 				of_match_device(pxa_gpio_dt_ids, &pdev->dev);
 	const struct pxa_gpio_id *gpio_id;
@@ -554,57 +604,64 @@ static int pxa_gpio_probe_dt(struct platform_device *pdev)
 	irq_base = irq_alloc_descs(-1, 0, nr_gpios, 0);
 	if (irq_base < 0) {
 		dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n");
-		ret = irq_base;
-		goto err;
+		return irq_base;
 	}
-	domain = irq_domain_add_legacy(np, nr_gpios, irq_base, 0,
-				       &pxa_irq_domain_ops, NULL);
-	pxa_gpio_of_node = np;
-	return 0;
-err:
-	iounmap(gpio_reg_base);
-	return ret;
+	return irq_base;
 }
 #else
-#define pxa_gpio_probe_dt(pdev)		(-1)
+#define pxa_gpio_probe_dt(pdev, pchip)		(-1)
 #endif
 
 static int pxa_gpio_probe(struct platform_device *pdev)
 {
-	struct pxa_gpio_chip *c;
+	struct pxa_gpio_chip *pchip;
+	struct pxa_gpio_bank *c;
 	struct resource *res;
 	struct clk *clk;
 	struct pxa_gpio_platform_data *info;
-	int gpio, irq, ret, use_of = 0;
+	void __iomem *gpio_reg_base;
+	int gpio, ret;
 	int irq0 = 0, irq1 = 0, irq_mux, gpio_offset = 0;
 
+	pchip = devm_kzalloc(&pdev->dev, sizeof(*pchip), GFP_KERNEL);
+	if (!pchip)
+		return -ENOMEM;
+	pchip->dev = &pdev->dev;
+
 	info = dev_get_platdata(&pdev->dev);
 	if (info) {
 		irq_base = info->irq_base;
 		if (irq_base <= 0)
 			return -EINVAL;
 		pxa_last_gpio = pxa_gpio_nums(pdev);
+		pchip->set_wake = info->gpio_set_wake;
 	} else {
-		irq_base = 0;
-		use_of = 1;
-		ret = pxa_gpio_probe_dt(pdev);
-		if (ret < 0)
+		irq_base = pxa_gpio_probe_dt(pdev, pchip);
+		if (irq_base < 0)
 			return -EINVAL;
 	}
 
 	if (!pxa_last_gpio)
 		return -EINVAL;
 
+	pchip->irqdomain = irq_domain_add_legacy(pdev->dev.of_node,
+						 pxa_last_gpio + 1, irq_base,
+						 0, &pxa_irq_domain_ops, pchip);
+	if (!pchip->irqdomain)
+		return -ENOMEM;
+
 	irq0 = platform_get_irq_byname(pdev, "gpio0");
 	irq1 = platform_get_irq_byname(pdev, "gpio1");
 	irq_mux = platform_get_irq_byname(pdev, "gpio_mux");
 	if ((irq0 > 0 && irq1 <= 0) || (irq0 <= 0 && irq1 > 0)
 		|| (irq_mux <= 0))
 		return -EINVAL;
+
+	pchip->irq0 = irq0;
+	pchip->irq1 = irq1;
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -EINVAL;
-	gpio_reg_base = ioremap(res->start, resource_size(res));
+	gpio_reg_base = devm_ioremap(&pdev->dev, res->start,
+				     resource_size(res));
 	if (!gpio_reg_base)
 		return -EINVAL;
 
@@ -615,21 +672,24 @@ static int pxa_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "Error %ld to get gpio clock\n",
 			PTR_ERR(clk));
-		iounmap(gpio_reg_base);
 		return PTR_ERR(clk);
 	}
 	ret = clk_prepare_enable(clk);
 	if (ret) {
 		clk_put(clk);
-		iounmap(gpio_reg_base);
 		return ret;
 	}
 
 	/* Initialize GPIO chips */
-	pxa_init_gpio_chip(pxa_last_gpio, info ? info->gpio_set_wake : NULL);
+	ret = pxa_init_gpio_chip(pchip, pxa_last_gpio + 1, pdev->dev.of_node,
+				 gpio_reg_base);
+	if (ret) {
+		clk_put(clk);
+		return ret;
+	}
 
 	/* clear all GPIO edge detects */
-	for_each_gpio_chip(gpio, c) {
+	for_each_gpio_bank(gpio, c, pchip) {
 		writel_relaxed(0, c->regbase + GFER_OFFSET);
 		writel_relaxed(0, c->regbase + GRER_OFFSET);
 		writel_relaxed(~0, c->regbase + GEDR_OFFSET);
@@ -638,34 +698,31 @@ static int pxa_gpio_probe(struct platform_device *pdev)
 			writel_relaxed(~0, c->regbase + ED_MASK_OFFSET);
 	}
 
-	if (!use_of) {
-		if (irq0 > 0) {
-			irq = gpio_to_irq(0);
-			irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip,
-						 handle_edge_irq);
-			irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
-		}
-		if (irq1 > 0) {
-			irq = gpio_to_irq(1);
-			irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip,
-						 handle_edge_irq);
-			irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
-		}
-
-		for (irq  = gpio_to_irq(gpio_offset);
-			irq <= gpio_to_irq(pxa_last_gpio); irq++) {
-			irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip,
-						 handle_edge_irq);
-			irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
-		}
+	if (irq0 > 0) {
+		ret = devm_request_irq(&pdev->dev,
+				       irq0, pxa_gpio_direct_handler, 0,
+				       "gpio-0", pchip);
+		if (ret)
+			dev_err(&pdev->dev, "request of gpio0 irq failed: %d\n",
+				ret);
 	}
+	if (irq1 > 0) {
+		ret = devm_request_irq(&pdev->dev,
+				       irq1, pxa_gpio_direct_handler, 0,
+				       "gpio-1", pchip);
+		if (ret)
+			dev_err(&pdev->dev, "request of gpio1 irq failed: %d\n",
+				ret);
+	}
+	ret = devm_request_irq(&pdev->dev,
+			       irq_mux, pxa_gpio_demux_handler, 0,
+				       "gpio-mux", pchip);
+	if (ret)
+		dev_err(&pdev->dev, "request of gpio-mux irq failed: %d\n",
+				ret);
 
-	if (irq0 > 0)
-		irq_set_chained_handler(irq0, pxa_gpio_demux_handler);
-	if (irq1 > 0)
-		irq_set_chained_handler(irq1, pxa_gpio_demux_handler);
+	pxa_gpio_chip = pchip;
 
-	irq_set_chained_handler(irq_mux, pxa_gpio_demux_handler);
 	return 0;
 }
 
@@ -690,19 +747,32 @@ static struct platform_driver pxa_gpio_driver = {
 	.id_table	= gpio_id_table,
 };
 
-static int __init pxa_gpio_init(void)
+static int __init pxa_gpio_legacy_init(void)
 {
+	if (of_have_populated_dt())
+		return 0;
+
 	return platform_driver_register(&pxa_gpio_driver);
 }
-postcore_initcall(pxa_gpio_init);
+postcore_initcall(pxa_gpio_legacy_init);
+
+static int __init pxa_gpio_dt_init(void)
+{
+	if (of_have_populated_dt())
+		return platform_driver_register(&pxa_gpio_driver);
+
+	return 0;
+}
+device_initcall(pxa_gpio_dt_init);
 
 #ifdef CONFIG_PM
 static int pxa_gpio_suspend(void)
 {
-	struct pxa_gpio_chip *c;
+	struct pxa_gpio_chip *pchip = pxa_gpio_chip;
+	struct pxa_gpio_bank *c;
 	int gpio;
 
-	for_each_gpio_chip(gpio, c) {
+	for_each_gpio_bank(gpio, c, pchip) {
 		c->saved_gplr = readl_relaxed(c->regbase + GPLR_OFFSET);
 		c->saved_gpdr = readl_relaxed(c->regbase + GPDR_OFFSET);
 		c->saved_grer = readl_relaxed(c->regbase + GRER_OFFSET);
@@ -716,10 +786,11 @@ static int pxa_gpio_suspend(void)
 
 static void pxa_gpio_resume(void)
 {
-	struct pxa_gpio_chip *c;
+	struct pxa_gpio_chip *pchip = pxa_gpio_chip;
+	struct pxa_gpio_bank *c;
 	int gpio;
 
-	for_each_gpio_chip(gpio, c) {
+	for_each_gpio_bank(gpio, c, pchip) {
 		/* restore level with set/clear */
 		writel_relaxed(c->saved_gplr, c->regbase + GPSR_OFFSET);
 		writel_relaxed(~c->saved_gplr, c->regbase + GPCR_OFFSET);
diff --git a/drivers/gpio/gpio-rc5t583.c b/drivers/gpio/gpio-rc5t583.c
index 6eabf239676b..1e2d210b3369 100644
--- a/drivers/gpio/gpio-rc5t583.c
+++ b/drivers/gpio/gpio-rc5t583.c
@@ -34,14 +34,9 @@ struct rc5t583_gpio {
 	struct rc5t583 *rc5t583;
 };
 
-static inline struct rc5t583_gpio *to_rc5t583_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct rc5t583_gpio, gpio_chip);
-}
-
 static int rc5t583_gpio_get(struct gpio_chip *gc, unsigned int offset)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 	struct device *parent = rc5t583_gpio->rc5t583->dev;
 	uint8_t val = 0;
 	int ret;
@@ -55,7 +50,7 @@ static int rc5t583_gpio_get(struct gpio_chip *gc, unsigned int offset)
 
 static void rc5t583_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 	struct device *parent = rc5t583_gpio->rc5t583->dev;
 	if (val)
 		rc5t583_set_bits(parent, RC5T583_GPIO_IOOUT, BIT(offset));
@@ -65,7 +60,7 @@ static void rc5t583_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 
 static int rc5t583_gpio_dir_input(struct gpio_chip *gc, unsigned int offset)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 	struct device *parent = rc5t583_gpio->rc5t583->dev;
 	int ret;
 
@@ -80,7 +75,7 @@ static int rc5t583_gpio_dir_input(struct gpio_chip *gc, unsigned int offset)
 static int rc5t583_gpio_dir_output(struct gpio_chip *gc, unsigned offset,
 			int value)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 	struct device *parent = rc5t583_gpio->rc5t583->dev;
 	int ret;
 
@@ -95,7 +90,7 @@ static int rc5t583_gpio_dir_output(struct gpio_chip *gc, unsigned offset,
 
 static int rc5t583_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 
 	if (offset < RC5T583_MAX_GPIO)
 		return rc5t583_gpio->rc5t583->irq_base +
@@ -105,7 +100,7 @@ static int rc5t583_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 
 static void rc5t583_gpio_free(struct gpio_chip *gc, unsigned offset)
 {
-	struct rc5t583_gpio *rc5t583_gpio = to_rc5t583_gpio(gc);
+	struct rc5t583_gpio *rc5t583_gpio = gpiochip_get_data(gc);
 	struct device *parent = rc5t583_gpio->rc5t583->dev;
 
 	rc5t583_set_bits(parent, RC5T583_GPIO_PGSEL, BIT(offset));
@@ -132,7 +127,7 @@ static int rc5t583_gpio_probe(struct platform_device *pdev)
 	rc5t583_gpio->gpio_chip.to_irq = rc5t583_gpio_to_irq,
 	rc5t583_gpio->gpio_chip.ngpio = RC5T583_MAX_GPIO,
 	rc5t583_gpio->gpio_chip.can_sleep = true,
-	rc5t583_gpio->gpio_chip.dev = &pdev->dev;
+	rc5t583_gpio->gpio_chip.parent = &pdev->dev;
 	rc5t583_gpio->gpio_chip.base = -1;
 	rc5t583_gpio->rc5t583 = rc5t583;
 
@@ -141,7 +136,7 @@ static int rc5t583_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, rc5t583_gpio);
 
-	return gpiochip_add(&rc5t583_gpio->gpio_chip);
+	return gpiochip_add_data(&rc5t583_gpio->gpio_chip, rc5t583_gpio);
 }
 
 static int rc5t583_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 2a8122444614..cf41440aff91 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -25,7 +25,6 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/pinctrl/consumer.h>
-#include <linux/platform_data/gpio-rcar.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/spinlock.h>
@@ -34,12 +33,13 @@
 struct gpio_rcar_priv {
 	void __iomem *base;
 	spinlock_t lock;
-	struct gpio_rcar_config config;
 	struct platform_device *pdev;
 	struct gpio_chip gpio_chip;
 	struct irq_chip irq_chip;
-	unsigned int irq_parent;
 	struct clk *clk;
+	unsigned int irq_parent;
+	bool has_both_edge_trigger;
+	bool needs_clk;
 };
 
 #define IOINTSEL 0x00	/* General IO/Interrupt Switching Register */
@@ -84,8 +84,7 @@ static void gpio_rcar_modify_bit(struct gpio_rcar_priv *p, int offs,
 static void gpio_rcar_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct gpio_rcar_priv *p = container_of(gc, struct gpio_rcar_priv,
-						gpio_chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(gc);
 
 	gpio_rcar_write(p, INTMSK, ~BIT(irqd_to_hwirq(d)));
 }
@@ -93,8 +92,7 @@ static void gpio_rcar_irq_disable(struct irq_data *d)
 static void gpio_rcar_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct gpio_rcar_priv *p = container_of(gc, struct gpio_rcar_priv,
-						gpio_chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(gc);
 
 	gpio_rcar_write(p, MSKCLR, BIT(irqd_to_hwirq(d)));
 }
@@ -121,7 +119,7 @@ static void gpio_rcar_config_interrupt_input_mode(struct gpio_rcar_priv *p,
 	gpio_rcar_modify_bit(p, EDGLEVEL, hwirq, !level_trigger);
 
 	/* Select one edge or both edges in BOTHEDGE */
-	if (p->config.has_both_edge_trigger)
+	if (p->has_both_edge_trigger)
 		gpio_rcar_modify_bit(p, BOTHEDGE, hwirq, both);
 
 	/* Select "Interrupt Input Mode" in IOINTSEL */
@@ -137,8 +135,7 @@ static void gpio_rcar_config_interrupt_input_mode(struct gpio_rcar_priv *p,
 static int gpio_rcar_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct gpio_rcar_priv *p = container_of(gc, struct gpio_rcar_priv,
-						gpio_chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(gc);
 	unsigned int hwirq = irqd_to_hwirq(d);
 
 	dev_dbg(&p->pdev->dev, "sense irq = %d, type = %d\n", hwirq, type);
@@ -161,7 +158,7 @@ static int gpio_rcar_irq_set_type(struct irq_data *d, unsigned int type)
 						      false);
 		break;
 	case IRQ_TYPE_EDGE_BOTH:
-		if (!p->config.has_both_edge_trigger)
+		if (!p->has_both_edge_trigger)
 			return -EINVAL;
 		gpio_rcar_config_interrupt_input_mode(p, hwirq, true, false,
 						      true);
@@ -175,8 +172,7 @@ static int gpio_rcar_irq_set_type(struct irq_data *d, unsigned int type)
 static int gpio_rcar_irq_set_wake(struct irq_data *d, unsigned int on)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct gpio_rcar_priv *p = container_of(gc, struct gpio_rcar_priv,
-						gpio_chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(gc);
 	int error;
 
 	if (p->irq_parent) {
@@ -218,16 +214,11 @@ static irqreturn_t gpio_rcar_irq_handler(int irq, void *dev_id)
 	return irqs_handled ? IRQ_HANDLED : IRQ_NONE;
 }
 
-static inline struct gpio_rcar_priv *gpio_to_priv(struct gpio_chip *chip)
-{
-	return container_of(chip, struct gpio_rcar_priv, gpio_chip);
-}
-
 static void gpio_rcar_config_general_input_output_mode(struct gpio_chip *chip,
 						       unsigned int gpio,
 						       bool output)
 {
-	struct gpio_rcar_priv *p = gpio_to_priv(chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	/* follow steps in the GPIO documentation for
@@ -251,7 +242,7 @@ static void gpio_rcar_config_general_input_output_mode(struct gpio_chip *chip,
 
 static int gpio_rcar_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_rcar_priv *p = gpio_to_priv(chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(chip);
 	int error;
 
 	error = pm_runtime_get_sync(&p->pdev->dev);
@@ -267,7 +258,7 @@ static int gpio_rcar_request(struct gpio_chip *chip, unsigned offset)
 
 static void gpio_rcar_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_rcar_priv *p = gpio_to_priv(chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(chip);
 
 	pinctrl_free_gpio(chip->base + offset);
 
@@ -291,15 +282,15 @@ static int gpio_rcar_get(struct gpio_chip *chip, unsigned offset)
 
 	/* testing on r8a7790 shows that INDT does not show correct pin state
 	 * when configured as output, so use OUTDT in case of output pins */
-	if (gpio_rcar_read(gpio_to_priv(chip), INOUTSEL) & bit)
-		return !!(gpio_rcar_read(gpio_to_priv(chip), OUTDT) & bit);
+	if (gpio_rcar_read(gpiochip_get_data(chip), INOUTSEL) & bit)
+		return !!(gpio_rcar_read(gpiochip_get_data(chip), OUTDT) & bit);
 	else
-		return !!(gpio_rcar_read(gpio_to_priv(chip), INDT) & bit);
+		return !!(gpio_rcar_read(gpiochip_get_data(chip), INDT) & bit);
 }
 
 static void gpio_rcar_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct gpio_rcar_priv *p = gpio_to_priv(chip);
+	struct gpio_rcar_priv *p = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&p->lock, flags);
@@ -318,14 +309,17 @@ static int gpio_rcar_direction_output(struct gpio_chip *chip, unsigned offset,
 
 struct gpio_rcar_info {
 	bool has_both_edge_trigger;
+	bool needs_clk;
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen1 = {
 	.has_both_edge_trigger = false,
+	.needs_clk = false,
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen2 = {
 	.has_both_edge_trigger = true,
+	.needs_clk = true,
 };
 
 static const struct of_device_id gpio_rcar_of_table[] = {
@@ -355,39 +349,30 @@ static const struct of_device_id gpio_rcar_of_table[] = {
 
 MODULE_DEVICE_TABLE(of, gpio_rcar_of_table);
 
-static int gpio_rcar_parse_pdata(struct gpio_rcar_priv *p)
+static int gpio_rcar_parse_dt(struct gpio_rcar_priv *p, unsigned int *npins)
 {
-	struct gpio_rcar_config *pdata = dev_get_platdata(&p->pdev->dev);
 	struct device_node *np = p->pdev->dev.of_node;
+	const struct of_device_id *match;
+	const struct gpio_rcar_info *info;
 	struct of_phandle_args args;
 	int ret;
 
-	if (pdata) {
-		p->config = *pdata;
-	} else if (IS_ENABLED(CONFIG_OF) && np) {
-		const struct of_device_id *match;
-		const struct gpio_rcar_info *info;
-
-		match = of_match_node(gpio_rcar_of_table, np);
-		if (!match)
-			return -EINVAL;
+	match = of_match_node(gpio_rcar_of_table, np);
+	if (!match)
+		return -EINVAL;
 
-		info = match->data;
+	info = match->data;
 
-		ret = of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0,
-						       &args);
-		p->config.number_of_pins = ret == 0 ? args.args[2]
-					 : RCAR_MAX_GPIO_PER_BANK;
-		p->config.gpio_base = -1;
-		p->config.has_both_edge_trigger = info->has_both_edge_trigger;
-	}
+	ret = of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &args);
+	*npins = ret == 0 ? args.args[2] : RCAR_MAX_GPIO_PER_BANK;
+	p->has_both_edge_trigger = info->has_both_edge_trigger;
+	p->needs_clk = info->needs_clk;
 
-	if (p->config.number_of_pins == 0 ||
-	    p->config.number_of_pins > RCAR_MAX_GPIO_PER_BANK) {
+	if (*npins == 0 || *npins > RCAR_MAX_GPIO_PER_BANK) {
 		dev_warn(&p->pdev->dev,
-			 "Invalid number of gpio lines %u, using %u\n",
-			 p->config.number_of_pins, RCAR_MAX_GPIO_PER_BANK);
-		p->config.number_of_pins = RCAR_MAX_GPIO_PER_BANK;
+			 "Invalid number of gpio lines %u, using %u\n", *npins,
+			 RCAR_MAX_GPIO_PER_BANK);
+		*npins = RCAR_MAX_GPIO_PER_BANK;
 	}
 
 	return 0;
@@ -401,6 +386,7 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 	struct irq_chip *irq_chip;
 	struct device *dev = &pdev->dev;
 	const char *name = dev_name(dev);
+	unsigned int npins;
 	int ret;
 
 	p = devm_kzalloc(dev, sizeof(*p), GFP_KERNEL);
@@ -410,8 +396,8 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 	p->pdev = pdev;
 	spin_lock_init(&p->lock);
 
-	/* Get device configuration from DT node or platform data. */
-	ret = gpio_rcar_parse_pdata(p);
+	/* Get device configuration from DT node */
+	ret = gpio_rcar_parse_dt(p, &npins);
 	if (ret < 0)
 		return ret;
 
@@ -419,7 +405,11 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 
 	p->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(p->clk)) {
-		dev_warn(dev, "unable to get clock\n");
+		if (p->needs_clk) {
+			dev_err(dev, "unable to get clock\n");
+			ret = PTR_ERR(p->clk);
+			goto err0;
+		}
 		p->clk = NULL;
 	}
 
@@ -449,10 +439,10 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 	gpio_chip->direction_output = gpio_rcar_direction_output;
 	gpio_chip->set = gpio_rcar_set;
 	gpio_chip->label = name;
-	gpio_chip->dev = dev;
+	gpio_chip->parent = dev;
 	gpio_chip->owner = THIS_MODULE;
-	gpio_chip->base = p->config.gpio_base;
-	gpio_chip->ngpio = p->config.number_of_pins;
+	gpio_chip->base = -1;
+	gpio_chip->ngpio = npins;
 
 	irq_chip = &p->irq_chip;
 	irq_chip->name = name;
@@ -462,14 +452,14 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 	irq_chip->irq_set_wake = gpio_rcar_irq_set_wake;
 	irq_chip->flags	= IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
 
-	ret = gpiochip_add(gpio_chip);
+	ret = gpiochip_add_data(gpio_chip, p);
 	if (ret) {
 		dev_err(dev, "failed to add GPIO controller\n");
 		goto err0;
 	}
 
-	ret = gpiochip_irqchip_add(gpio_chip, irq_chip, p->config.irq_base,
-				   handle_level_irq, IRQ_TYPE_NONE);
+	ret = gpiochip_irqchip_add(gpio_chip, irq_chip, 0, handle_level_irq,
+				   IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(dev, "cannot add irqchip\n");
 		goto err1;
@@ -483,22 +473,7 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 		goto err1;
 	}
 
-	dev_info(dev, "driving %d GPIOs\n", p->config.number_of_pins);
-
-	/* warn in case of mismatch if irq base is specified */
-	if (p->config.irq_base) {
-		ret = irq_find_mapping(gpio_chip->irqdomain, 0);
-		if (p->config.irq_base != ret)
-			dev_warn(dev, "irq base mismatch (%u/%u)\n",
-				 p->config.irq_base, ret);
-	}
-
-	if (p->config.pctl_name) {
-		ret = gpiochip_add_pin_range(gpio_chip, p->config.pctl_name, 0,
-					     gpio_chip->base, gpio_chip->ngpio);
-		if (ret < 0)
-			dev_warn(dev, "failed to add pin range\n");
-	}
+	dev_info(dev, "driving %d GPIOs\n", npins);
 
 	return 0;
 
diff --git a/drivers/gpio/gpio-rdc321x.c b/drivers/gpio/gpio-rdc321x.c
index d729bc8a554d..96ddee3f464a 100644
--- a/drivers/gpio/gpio-rdc321x.c
+++ b/drivers/gpio/gpio-rdc321x.c
@@ -47,7 +47,7 @@ static int rdc_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 	u32 value = 0;
 	int reg;
 
-	gpch = container_of(chip, struct rdc321x_gpio, chip);
+	gpch = gpiochip_get_data(chip);
 	reg = gpio < 32 ? gpch->reg1_data_base : gpch->reg2_data_base;
 
 	spin_lock(&gpch->lock);
@@ -65,7 +65,7 @@ static void rdc_gpio_set_value_impl(struct gpio_chip *chip,
 	struct rdc321x_gpio *gpch;
 	int reg = (gpio < 32) ? 0 : 1;
 
-	gpch = container_of(chip, struct rdc321x_gpio, chip);
+	gpch = gpiochip_get_data(chip);
 
 	if (value)
 		gpch->data_reg[reg] |= 1 << (gpio & 0x1f);
@@ -83,7 +83,7 @@ static void rdc_gpio_set_value(struct gpio_chip *chip,
 {
 	struct rdc321x_gpio *gpch;
 
-	gpch = container_of(chip, struct rdc321x_gpio, chip);
+	gpch = gpiochip_get_data(chip);
 	spin_lock(&gpch->lock);
 	rdc_gpio_set_value_impl(chip, gpio, value);
 	spin_unlock(&gpch->lock);
@@ -96,7 +96,7 @@ static int rdc_gpio_config(struct gpio_chip *chip,
 	int err;
 	u32 reg;
 
-	gpch = container_of(chip, struct rdc321x_gpio, chip);
+	gpch = gpiochip_get_data(chip);
 
 	spin_lock(&gpch->lock);
 	err = pci_read_config_dword(gpch->sb_pdev, gpio < 32 ?
@@ -194,7 +194,7 @@ static int rdc321x_gpio_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "registering %d GPIOs\n",
 					rdc321x_gpio_dev->chip.ngpio);
-	return gpiochip_add(&rdc321x_gpio_dev->chip);
+	return gpiochip_add_data(&rdc321x_gpio_dev->chip, rdc321x_gpio_dev);
 }
 
 static int rdc321x_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c
index 990fa9023e22..0c99e8fb9af3 100644
--- a/drivers/gpio/gpio-sa1100.c
+++ b/drivers/gpio/gpio-sa1100.c
@@ -17,7 +17,7 @@
 
 static int sa1100_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	return GPLR & GPIO_GPIO(offset);
+	return !!(GPLR & GPIO_GPIO(offset));
 }
 
 static void sa1100_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
@@ -238,7 +238,7 @@ void __init sa1100_init_gpio(void)
 	GRER = 0;
 	GEDR = -1;
 
-	gpiochip_add(&sa1100_gpio_chip);
+	gpiochip_add_data(&sa1100_gpio_chip, NULL);
 
 	sa1100_gpio_irqdomain = irq_domain_add_simple(NULL,
 			28, IRQ_GPIO0,
diff --git a/drivers/gpio/gpio-samsung.c b/drivers/gpio/gpio-samsung.c
deleted file mode 100644
index 7c288ba4dc87..000000000000
--- a/drivers/gpio/gpio-samsung.c
+++ /dev/null
@@ -1,1328 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *      Ben Dooks <ben@simtec.co.uk>
- *      http://armlinux.simtec.co.uk/
- *
- * SAMSUNG - GPIOlib support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/kernel.h>
-#include <linux/irq.h>
-#include <linux/io.h>
-#include <linux/gpio.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/device.h>
-#include <linux/ioport.h>
-#include <linux/of.h>
-#include <linux/slab.h>
-#include <linux/of_address.h>
-
-#include <asm/irq.h>
-
-#include <mach/map.h>
-#include <mach/regs-gpio.h>
-#include <mach/gpio-samsung.h>
-
-#include <plat/cpu.h>
-#include <plat/gpio-core.h>
-#include <plat/gpio-cfg.h>
-#include <plat/gpio-cfg-helpers.h>
-#include <plat/pm.h>
-
-int samsung_gpio_setpull_updown(struct samsung_gpio_chip *chip,
-				unsigned int off, samsung_gpio_pull_t pull)
-{
-	void __iomem *reg = chip->base + 0x08;
-	int shift = off * 2;
-	u32 pup;
-
-	pup = __raw_readl(reg);
-	pup &= ~(3 << shift);
-	pup |= pull << shift;
-	__raw_writel(pup, reg);
-
-	return 0;
-}
-
-samsung_gpio_pull_t samsung_gpio_getpull_updown(struct samsung_gpio_chip *chip,
-						unsigned int off)
-{
-	void __iomem *reg = chip->base + 0x08;
-	int shift = off * 2;
-	u32 pup = __raw_readl(reg);
-
-	pup >>= shift;
-	pup &= 0x3;
-
-	return (__force samsung_gpio_pull_t)pup;
-}
-
-int s3c2443_gpio_setpull(struct samsung_gpio_chip *chip,
-			 unsigned int off, samsung_gpio_pull_t pull)
-{
-	switch (pull) {
-	case S3C_GPIO_PULL_NONE:
-		pull = 0x01;
-		break;
-	case S3C_GPIO_PULL_UP:
-		pull = 0x00;
-		break;
-	case S3C_GPIO_PULL_DOWN:
-		pull = 0x02;
-		break;
-	}
-	return samsung_gpio_setpull_updown(chip, off, pull);
-}
-
-samsung_gpio_pull_t s3c2443_gpio_getpull(struct samsung_gpio_chip *chip,
-					 unsigned int off)
-{
-	samsung_gpio_pull_t pull;
-
-	pull = samsung_gpio_getpull_updown(chip, off);
-
-	switch (pull) {
-	case 0x00:
-		pull = S3C_GPIO_PULL_UP;
-		break;
-	case 0x01:
-	case 0x03:
-		pull = S3C_GPIO_PULL_NONE;
-		break;
-	case 0x02:
-		pull = S3C_GPIO_PULL_DOWN;
-		break;
-	}
-
-	return pull;
-}
-
-static int s3c24xx_gpio_setpull_1(struct samsung_gpio_chip *chip,
-				  unsigned int off, samsung_gpio_pull_t pull,
-				  samsung_gpio_pull_t updown)
-{
-	void __iomem *reg = chip->base + 0x08;
-	u32 pup = __raw_readl(reg);
-
-	if (pull == updown)
-		pup &= ~(1 << off);
-	else if (pull == S3C_GPIO_PULL_NONE)
-		pup |= (1 << off);
-	else
-		return -EINVAL;
-
-	__raw_writel(pup, reg);
-	return 0;
-}
-
-static samsung_gpio_pull_t s3c24xx_gpio_getpull_1(struct samsung_gpio_chip *chip,
-						  unsigned int off,
-						  samsung_gpio_pull_t updown)
-{
-	void __iomem *reg = chip->base + 0x08;
-	u32 pup = __raw_readl(reg);
-
-	pup &= (1 << off);
-	return pup ? S3C_GPIO_PULL_NONE : updown;
-}
-
-samsung_gpio_pull_t s3c24xx_gpio_getpull_1up(struct samsung_gpio_chip *chip,
-					     unsigned int off)
-{
-	return s3c24xx_gpio_getpull_1(chip, off, S3C_GPIO_PULL_UP);
-}
-
-int s3c24xx_gpio_setpull_1up(struct samsung_gpio_chip *chip,
-			     unsigned int off, samsung_gpio_pull_t pull)
-{
-	return s3c24xx_gpio_setpull_1(chip, off, pull, S3C_GPIO_PULL_UP);
-}
-
-samsung_gpio_pull_t s3c24xx_gpio_getpull_1down(struct samsung_gpio_chip *chip,
-					       unsigned int off)
-{
-	return s3c24xx_gpio_getpull_1(chip, off, S3C_GPIO_PULL_DOWN);
-}
-
-int s3c24xx_gpio_setpull_1down(struct samsung_gpio_chip *chip,
-			       unsigned int off, samsung_gpio_pull_t pull)
-{
-	return s3c24xx_gpio_setpull_1(chip, off, pull, S3C_GPIO_PULL_DOWN);
-}
-
-/*
- * samsung_gpio_setcfg_2bit - Samsung 2bit style GPIO configuration.
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- * @cfg: The configuration value to set.
- *
- * This helper deal with the GPIO cases where the control register
- * has two bits of configuration per gpio, which have the following
- * functions:
- *	00 = input
- *	01 = output
- *	1x = special function
- */
-
-static int samsung_gpio_setcfg_2bit(struct samsung_gpio_chip *chip,
-				    unsigned int off, unsigned int cfg)
-{
-	void __iomem *reg = chip->base;
-	unsigned int shift = off * 2;
-	u32 con;
-
-	if (samsung_gpio_is_cfg_special(cfg)) {
-		cfg &= 0xf;
-		if (cfg > 3)
-			return -EINVAL;
-
-		cfg <<= shift;
-	}
-
-	con = __raw_readl(reg);
-	con &= ~(0x3 << shift);
-	con |= cfg;
-	__raw_writel(con, reg);
-
-	return 0;
-}
-
-/*
- * samsung_gpio_getcfg_2bit - Samsung 2bit style GPIO configuration read.
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- *
- * The reverse of samsung_gpio_setcfg_2bit(). Will return a value which
- * could be directly passed back to samsung_gpio_setcfg_2bit(), from the
- * S3C_GPIO_SPECIAL() macro.
- */
-
-static unsigned int samsung_gpio_getcfg_2bit(struct samsung_gpio_chip *chip,
-					     unsigned int off)
-{
-	u32 con;
-
-	con = __raw_readl(chip->base);
-	con >>= off * 2;
-	con &= 3;
-
-	/* this conversion works for IN and OUT as well as special mode */
-	return S3C_GPIO_SPECIAL(con);
-}
-
-/*
- * samsung_gpio_setcfg_4bit - Samsung 4bit single register GPIO config.
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- * @cfg: The configuration value to set.
- *
- * This helper deal with the GPIO cases where the control register has 4 bits
- * of control per GPIO, generally in the form of:
- *	0000 = Input
- *	0001 = Output
- *	others = Special functions (dependent on bank)
- *
- * Note, since the code to deal with the case where there are two control
- * registers instead of one, we do not have a separate set of functions for
- * each case.
- */
-
-static int samsung_gpio_setcfg_4bit(struct samsung_gpio_chip *chip,
-				    unsigned int off, unsigned int cfg)
-{
-	void __iomem *reg = chip->base;
-	unsigned int shift = (off & 7) * 4;
-	u32 con;
-
-	if (off < 8 && chip->chip.ngpio > 8)
-		reg -= 4;
-
-	if (samsung_gpio_is_cfg_special(cfg)) {
-		cfg &= 0xf;
-		cfg <<= shift;
-	}
-
-	con = __raw_readl(reg);
-	con &= ~(0xf << shift);
-	con |= cfg;
-	__raw_writel(con, reg);
-
-	return 0;
-}
-
-/*
- * samsung_gpio_getcfg_4bit - Samsung 4bit single register GPIO config read.
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- *
- * The reverse of samsung_gpio_setcfg_4bit(), turning a gpio configuration
- * register setting into a value the software can use, such as could be passed
- * to samsung_gpio_setcfg_4bit().
- *
- * @sa samsung_gpio_getcfg_2bit
- */
-
-static unsigned samsung_gpio_getcfg_4bit(struct samsung_gpio_chip *chip,
-					 unsigned int off)
-{
-	void __iomem *reg = chip->base;
-	unsigned int shift = (off & 7) * 4;
-	u32 con;
-
-	if (off < 8 && chip->chip.ngpio > 8)
-		reg -= 4;
-
-	con = __raw_readl(reg);
-	con >>= shift;
-	con &= 0xf;
-
-	/* this conversion works for IN and OUT as well as special mode */
-	return S3C_GPIO_SPECIAL(con);
-}
-
-#ifdef CONFIG_PLAT_S3C24XX
-/*
- * s3c24xx_gpio_setcfg_abank - S3C24XX style GPIO configuration (Bank A)
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- * @cfg: The configuration value to set.
- *
- * This helper deal with the GPIO cases where the control register
- * has one bit of configuration for the gpio, where setting the bit
- * means the pin is in special function mode and unset means output.
- */
-
-static int s3c24xx_gpio_setcfg_abank(struct samsung_gpio_chip *chip,
-				     unsigned int off, unsigned int cfg)
-{
-	void __iomem *reg = chip->base;
-	unsigned int shift = off;
-	u32 con;
-
-	if (samsung_gpio_is_cfg_special(cfg)) {
-		cfg &= 0xf;
-
-		/* Map output to 0, and SFN2 to 1 */
-		cfg -= 1;
-		if (cfg > 1)
-			return -EINVAL;
-
-		cfg <<= shift;
-	}
-
-	con = __raw_readl(reg);
-	con &= ~(0x1 << shift);
-	con |= cfg;
-	__raw_writel(con, reg);
-
-	return 0;
-}
-
-/*
- * s3c24xx_gpio_getcfg_abank - S3C24XX style GPIO configuration read (Bank A)
- * @chip: The gpio chip that is being configured.
- * @off: The offset for the GPIO being configured.
- *
- * The reverse of s3c24xx_gpio_setcfg_abank() turning an GPIO into a usable
- * GPIO configuration value.
- *
- * @sa samsung_gpio_getcfg_2bit
- * @sa samsung_gpio_getcfg_4bit
- */
-
-static unsigned s3c24xx_gpio_getcfg_abank(struct samsung_gpio_chip *chip,
-					  unsigned int off)
-{
-	u32 con;
-
-	con = __raw_readl(chip->base);
-	con >>= off;
-	con &= 1;
-	con++;
-
-	return S3C_GPIO_SFN(con);
-}
-#endif
-
-static void __init samsung_gpiolib_set_cfg(struct samsung_gpio_cfg *chipcfg,
-					   int nr_chips)
-{
-	for (; nr_chips > 0; nr_chips--, chipcfg++) {
-		if (!chipcfg->set_config)
-			chipcfg->set_config = samsung_gpio_setcfg_4bit;
-		if (!chipcfg->get_config)
-			chipcfg->get_config = samsung_gpio_getcfg_4bit;
-		if (!chipcfg->set_pull)
-			chipcfg->set_pull = samsung_gpio_setpull_updown;
-		if (!chipcfg->get_pull)
-			chipcfg->get_pull = samsung_gpio_getpull_updown;
-	}
-}
-
-struct samsung_gpio_cfg s3c24xx_gpiocfg_default = {
-	.set_config	= samsung_gpio_setcfg_2bit,
-	.get_config	= samsung_gpio_getcfg_2bit,
-};
-
-#ifdef CONFIG_PLAT_S3C24XX
-static struct samsung_gpio_cfg s3c24xx_gpiocfg_banka = {
-	.set_config	= s3c24xx_gpio_setcfg_abank,
-	.get_config	= s3c24xx_gpio_getcfg_abank,
-};
-#endif
-
-static struct samsung_gpio_cfg samsung_gpio_cfgs[] = {
-	[0] = {
-		.cfg_eint	= 0x0,
-	},
-	[1] = {
-		.cfg_eint	= 0x3,
-	},
-	[2] = {
-		.cfg_eint	= 0x7,
-	},
-	[3] = {
-		.cfg_eint	= 0xF,
-	},
-	[4] = {
-		.cfg_eint	= 0x0,
-		.set_config	= samsung_gpio_setcfg_2bit,
-		.get_config	= samsung_gpio_getcfg_2bit,
-	},
-	[5] = {
-		.cfg_eint	= 0x2,
-		.set_config	= samsung_gpio_setcfg_2bit,
-		.get_config	= samsung_gpio_getcfg_2bit,
-	},
-	[6] = {
-		.cfg_eint	= 0x3,
-		.set_config	= samsung_gpio_setcfg_2bit,
-		.get_config	= samsung_gpio_getcfg_2bit,
-	},
-	[7] = {
-		.set_config	= samsung_gpio_setcfg_2bit,
-		.get_config	= samsung_gpio_getcfg_2bit,
-	},
-};
-
-/*
- * Default routines for controlling GPIO, based on the original S3C24XX
- * GPIO functions which deal with the case where each gpio bank of the
- * chip is as following:
- *
- * base + 0x00: Control register, 2 bits per gpio
- *	        gpio n: 2 bits starting at (2*n)
- *		00 = input, 01 = output, others mean special-function
- * base + 0x04: Data register, 1 bit per gpio
- *		bit n: data bit n
-*/
-
-static int samsung_gpiolib_2bit_input(struct gpio_chip *chip, unsigned offset)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long flags;
-	unsigned long con;
-
-	samsung_gpio_lock(ourchip, flags);
-
-	con = __raw_readl(base + 0x00);
-	con &= ~(3 << (offset * 2));
-
-	__raw_writel(con, base + 0x00);
-
-	samsung_gpio_unlock(ourchip, flags);
-	return 0;
-}
-
-static int samsung_gpiolib_2bit_output(struct gpio_chip *chip,
-				       unsigned offset, int value)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long flags;
-	unsigned long dat;
-	unsigned long con;
-
-	samsung_gpio_lock(ourchip, flags);
-
-	dat = __raw_readl(base + 0x04);
-	dat &= ~(1 << offset);
-	if (value)
-		dat |= 1 << offset;
-	__raw_writel(dat, base + 0x04);
-
-	con = __raw_readl(base + 0x00);
-	con &= ~(3 << (offset * 2));
-	con |= 1 << (offset * 2);
-
-	__raw_writel(con, base + 0x00);
-	__raw_writel(dat, base + 0x04);
-
-	samsung_gpio_unlock(ourchip, flags);
-	return 0;
-}
-
-/*
- * The samsung_gpiolib_4bit routines are to control the gpio banks where
- * the gpio configuration register (GPxCON) has 4 bits per GPIO, as the
- * following example:
- *
- * base + 0x00: Control register, 4 bits per gpio
- *		gpio n: 4 bits starting at (4*n)
- *		0000 = input, 0001 = output, others mean special-function
- * base + 0x04: Data register, 1 bit per gpio
- *		bit n: data bit n
- *
- * Note, since the data register is one bit per gpio and is at base + 0x4
- * we can use samsung_gpiolib_get and samsung_gpiolib_set to change the
- * state of the output.
- */
-
-static int samsung_gpiolib_4bit_input(struct gpio_chip *chip,
-				      unsigned int offset)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long con;
-
-	con = __raw_readl(base + GPIOCON_OFF);
-	if (ourchip->bitmap_gpio_int & BIT(offset))
-		con |= 0xf << con_4bit_shift(offset);
-	else
-		con &= ~(0xf << con_4bit_shift(offset));
-	__raw_writel(con, base + GPIOCON_OFF);
-
-	pr_debug("%s: %p: CON now %08lx\n", __func__, base, con);
-
-	return 0;
-}
-
-static int samsung_gpiolib_4bit_output(struct gpio_chip *chip,
-				       unsigned int offset, int value)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long con;
-	unsigned long dat;
-
-	con = __raw_readl(base + GPIOCON_OFF);
-	con &= ~(0xf << con_4bit_shift(offset));
-	con |= 0x1 << con_4bit_shift(offset);
-
-	dat = __raw_readl(base + GPIODAT_OFF);
-
-	if (value)
-		dat |= 1 << offset;
-	else
-		dat &= ~(1 << offset);
-
-	__raw_writel(dat, base + GPIODAT_OFF);
-	__raw_writel(con, base + GPIOCON_OFF);
-	__raw_writel(dat, base + GPIODAT_OFF);
-
-	pr_debug("%s: %p: CON %08lx, DAT %08lx\n", __func__, base, con, dat);
-
-	return 0;
-}
-
-/*
- * The next set of routines are for the case where the GPIO configuration
- * registers are 4 bits per GPIO but there is more than one register (the
- * bank has more than 8 GPIOs.
- *
- * This case is the similar to the 4 bit case, but the registers are as
- * follows:
- *
- * base + 0x00: Control register, 4 bits per gpio (lower 8 GPIOs)
- *		gpio n: 4 bits starting at (4*n)
- *		0000 = input, 0001 = output, others mean special-function
- * base + 0x04: Control register, 4 bits per gpio (up to 8 additions GPIOs)
- *		gpio n: 4 bits starting at (4*n)
- *		0000 = input, 0001 = output, others mean special-function
- * base + 0x08: Data register, 1 bit per gpio
- *		bit n: data bit n
- *
- * To allow us to use the samsung_gpiolib_get and samsung_gpiolib_set
- * routines we store the 'base + 0x4' address so that these routines see
- * the data register at ourchip->base + 0x04.
- */
-
-static int samsung_gpiolib_4bit2_input(struct gpio_chip *chip,
-				       unsigned int offset)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	void __iomem *regcon = base;
-	unsigned long con;
-
-	if (offset > 7)
-		offset -= 8;
-	else
-		regcon -= 4;
-
-	con = __raw_readl(regcon);
-	con &= ~(0xf << con_4bit_shift(offset));
-	__raw_writel(con, regcon);
-
-	pr_debug("%s: %p: CON %08lx\n", __func__, base, con);
-
-	return 0;
-}
-
-static int samsung_gpiolib_4bit2_output(struct gpio_chip *chip,
-					unsigned int offset, int value)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	void __iomem *regcon = base;
-	unsigned long con;
-	unsigned long dat;
-	unsigned con_offset = offset;
-
-	if (con_offset > 7)
-		con_offset -= 8;
-	else
-		regcon -= 4;
-
-	con = __raw_readl(regcon);
-	con &= ~(0xf << con_4bit_shift(con_offset));
-	con |= 0x1 << con_4bit_shift(con_offset);
-
-	dat = __raw_readl(base + GPIODAT_OFF);
-
-	if (value)
-		dat |= 1 << offset;
-	else
-		dat &= ~(1 << offset);
-
-	__raw_writel(dat, base + GPIODAT_OFF);
-	__raw_writel(con, regcon);
-	__raw_writel(dat, base + GPIODAT_OFF);
-
-	pr_debug("%s: %p: CON %08lx, DAT %08lx\n", __func__, base, con, dat);
-
-	return 0;
-}
-
-#ifdef CONFIG_PLAT_S3C24XX
-/* The next set of routines are for the case of s3c24xx bank a */
-
-static int s3c24xx_gpiolib_banka_input(struct gpio_chip *chip, unsigned offset)
-{
-	return -EINVAL;
-}
-
-static int s3c24xx_gpiolib_banka_output(struct gpio_chip *chip,
-					unsigned offset, int value)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long flags;
-	unsigned long dat;
-	unsigned long con;
-
-	local_irq_save(flags);
-
-	con = __raw_readl(base + 0x00);
-	dat = __raw_readl(base + 0x04);
-
-	dat &= ~(1 << offset);
-	if (value)
-		dat |= 1 << offset;
-
-	__raw_writel(dat, base + 0x04);
-
-	con &= ~(1 << offset);
-
-	__raw_writel(con, base + 0x00);
-	__raw_writel(dat, base + 0x04);
-
-	local_irq_restore(flags);
-	return 0;
-}
-#endif
-
-static void samsung_gpiolib_set(struct gpio_chip *chip,
-				unsigned offset, int value)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	void __iomem *base = ourchip->base;
-	unsigned long flags;
-	unsigned long dat;
-
-	samsung_gpio_lock(ourchip, flags);
-
-	dat = __raw_readl(base + 0x04);
-	dat &= ~(1 << offset);
-	if (value)
-		dat |= 1 << offset;
-	__raw_writel(dat, base + 0x04);
-
-	samsung_gpio_unlock(ourchip, flags);
-}
-
-static int samsung_gpiolib_get(struct gpio_chip *chip, unsigned offset)
-{
-	struct samsung_gpio_chip *ourchip = to_samsung_gpio(chip);
-	unsigned long val;
-
-	val = __raw_readl(ourchip->base + 0x04);
-	val >>= offset;
-	val &= 1;
-
-	return val;
-}
-
-/*
- * CONFIG_S3C_GPIO_TRACK enables the tracking of the s3c specific gpios
- * for use with the configuration calls, and other parts of the s3c gpiolib
- * support code.
- *
- * Not all s3c support code will need this, as some configurations of cpu
- * may only support one or two different configuration options and have an
- * easy gpio to samsung_gpio_chip mapping function. If this is the case, then
- * the machine support file should provide its own samsung_gpiolib_getchip()
- * and any other necessary functions.
- */
-
-#ifdef CONFIG_S3C_GPIO_TRACK
-struct samsung_gpio_chip *s3c_gpios[S3C_GPIO_END];
-
-static __init void s3c_gpiolib_track(struct samsung_gpio_chip *chip)
-{
-	unsigned int gpn;
-	int i;
-
-	gpn = chip->chip.base;
-	for (i = 0; i < chip->chip.ngpio; i++, gpn++) {
-		BUG_ON(gpn >= ARRAY_SIZE(s3c_gpios));
-		s3c_gpios[gpn] = chip;
-	}
-}
-#endif /* CONFIG_S3C_GPIO_TRACK */
-
-/*
- * samsung_gpiolib_add() - add the Samsung gpio_chip.
- * @chip: The chip to register
- *
- * This is a wrapper to gpiochip_add() that takes our specific gpio chip
- * information and makes the necessary alterations for the platform and
- * notes the information for use with the configuration systems and any
- * other parts of the system.
- */
-
-static void __init samsung_gpiolib_add(struct samsung_gpio_chip *chip)
-{
-	struct gpio_chip *gc = &chip->chip;
-	int ret;
-
-	BUG_ON(!chip->base);
-	BUG_ON(!gc->label);
-	BUG_ON(!gc->ngpio);
-
-	spin_lock_init(&chip->lock);
-
-	if (!gc->direction_input)
-		gc->direction_input = samsung_gpiolib_2bit_input;
-	if (!gc->direction_output)
-		gc->direction_output = samsung_gpiolib_2bit_output;
-	if (!gc->set)
-		gc->set = samsung_gpiolib_set;
-	if (!gc->get)
-		gc->get = samsung_gpiolib_get;
-
-#ifdef CONFIG_PM
-	if (chip->pm != NULL) {
-		if (!chip->pm->save || !chip->pm->resume)
-			pr_err("gpio: %s has missing PM functions\n",
-			       gc->label);
-	} else
-		pr_err("gpio: %s has no PM function\n", gc->label);
-#endif
-
-	/* gpiochip_add() prints own failure message on error. */
-	ret = gpiochip_add(gc);
-	if (ret >= 0)
-		s3c_gpiolib_track(chip);
-}
-
-static void __init s3c24xx_gpiolib_add_chips(struct samsung_gpio_chip *chip,
-					     int nr_chips, void __iomem *base)
-{
-	int i;
-	struct gpio_chip *gc = &chip->chip;
-
-	for (i = 0 ; i < nr_chips; i++, chip++) {
-		/* skip banks not present on SoC */
-		if (chip->chip.base >= S3C_GPIO_END)
-			continue;
-
-		if (!chip->config)
-			chip->config = &s3c24xx_gpiocfg_default;
-		if (!chip->pm)
-			chip->pm = __gpio_pm(&samsung_gpio_pm_2bit);
-		if ((base != NULL) && (chip->base == NULL))
-			chip->base = base + ((i) * 0x10);
-
-		if (!gc->direction_input)
-			gc->direction_input = samsung_gpiolib_2bit_input;
-		if (!gc->direction_output)
-			gc->direction_output = samsung_gpiolib_2bit_output;
-
-		samsung_gpiolib_add(chip);
-	}
-}
-
-static void __init samsung_gpiolib_add_2bit_chips(struct samsung_gpio_chip *chip,
-						  int nr_chips, void __iomem *base,
-						  unsigned int offset)
-{
-	int i;
-
-	for (i = 0 ; i < nr_chips; i++, chip++) {
-		chip->chip.direction_input = samsung_gpiolib_2bit_input;
-		chip->chip.direction_output = samsung_gpiolib_2bit_output;
-
-		if (!chip->config)
-			chip->config = &samsung_gpio_cfgs[7];
-		if (!chip->pm)
-			chip->pm = __gpio_pm(&samsung_gpio_pm_2bit);
-		if ((base != NULL) && (chip->base == NULL))
-			chip->base = base + ((i) * offset);
-
-		samsung_gpiolib_add(chip);
-	}
-}
-
-/*
- * samsung_gpiolib_add_4bit_chips - 4bit single register GPIO config.
- * @chip: The gpio chip that is being configured.
- * @nr_chips: The no of chips (gpio ports) for the GPIO being configured.
- *
- * This helper deal with the GPIO cases where the control register has 4 bits
- * of control per GPIO, generally in the form of:
- * 0000 = Input
- * 0001 = Output
- * others = Special functions (dependent on bank)
- *
- * Note, since the code to deal with the case where there are two control
- * registers instead of one, we do not have a separate set of function
- * (samsung_gpiolib_add_4bit2_chips)for each case.
- */
-
-static void __init samsung_gpiolib_add_4bit_chips(struct samsung_gpio_chip *chip,
-						  int nr_chips, void __iomem *base)
-{
-	int i;
-
-	for (i = 0 ; i < nr_chips; i++, chip++) {
-		chip->chip.direction_input = samsung_gpiolib_4bit_input;
-		chip->chip.direction_output = samsung_gpiolib_4bit_output;
-
-		if (!chip->config)
-			chip->config = &samsung_gpio_cfgs[2];
-		if (!chip->pm)
-			chip->pm = __gpio_pm(&samsung_gpio_pm_4bit);
-		if ((base != NULL) && (chip->base == NULL))
-			chip->base = base + ((i) * 0x20);
-
-		chip->bitmap_gpio_int = 0;
-
-		samsung_gpiolib_add(chip);
-	}
-}
-
-static void __init samsung_gpiolib_add_4bit2_chips(struct samsung_gpio_chip *chip,
-						   int nr_chips)
-{
-	for (; nr_chips > 0; nr_chips--, chip++) {
-		chip->chip.direction_input = samsung_gpiolib_4bit2_input;
-		chip->chip.direction_output = samsung_gpiolib_4bit2_output;
-
-		if (!chip->config)
-			chip->config = &samsung_gpio_cfgs[2];
-		if (!chip->pm)
-			chip->pm = __gpio_pm(&samsung_gpio_pm_4bit);
-
-		samsung_gpiolib_add(chip);
-	}
-}
-
-int samsung_gpiolib_to_irq(struct gpio_chip *chip, unsigned int offset)
-{
-	struct samsung_gpio_chip *samsung_chip = container_of(chip, struct samsung_gpio_chip, chip);
-
-	return samsung_chip->irq_base + offset;
-}
-
-#ifdef CONFIG_PLAT_S3C24XX
-static int s3c24xx_gpiolib_fbank_to_irq(struct gpio_chip *chip, unsigned offset)
-{
-	if (offset < 4) {
-		if (soc_is_s3c2412())
-			return IRQ_EINT0_2412 + offset;
-		else
-			return IRQ_EINT0 + offset;
-	}
-
-	if (offset < 8)
-		return IRQ_EINT4 + offset - 4;
-
-	return -EINVAL;
-}
-#endif
-
-#ifdef CONFIG_ARCH_S3C64XX
-static int s3c64xx_gpiolib_mbank_to_irq(struct gpio_chip *chip, unsigned pin)
-{
-	return pin < 5 ? IRQ_EINT(23) + pin : -ENXIO;
-}
-
-static int s3c64xx_gpiolib_lbank_to_irq(struct gpio_chip *chip, unsigned pin)
-{
-	return pin >= 8 ? IRQ_EINT(16) + pin - 8 : -ENXIO;
-}
-#endif
-
-struct samsung_gpio_chip s3c24xx_gpios[] = {
-#ifdef CONFIG_PLAT_S3C24XX
-	{
-		.config	= &s3c24xx_gpiocfg_banka,
-		.chip	= {
-			.base			= S3C2410_GPA(0),
-			.owner			= THIS_MODULE,
-			.label			= "GPIOA",
-			.ngpio			= 27,
-			.direction_input	= s3c24xx_gpiolib_banka_input,
-			.direction_output	= s3c24xx_gpiolib_banka_output,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPB(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOB",
-			.ngpio	= 11,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPC(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOC",
-			.ngpio	= 16,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPD(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOD",
-			.ngpio	= 16,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPE(0),
-			.label	= "GPIOE",
-			.owner	= THIS_MODULE,
-			.ngpio	= 16,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPF(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOF",
-			.ngpio	= 8,
-			.to_irq	= s3c24xx_gpiolib_fbank_to_irq,
-		},
-	}, {
-		.irq_base = IRQ_EINT8,
-		.chip	= {
-			.base	= S3C2410_GPG(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOG",
-			.ngpio	= 16,
-			.to_irq	= samsung_gpiolib_to_irq,
-		},
-	}, {
-		.chip	= {
-			.base	= S3C2410_GPH(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOH",
-			.ngpio	= 15,
-		},
-	},
-		/* GPIOS for the S3C2443 and later devices. */
-	{
-		.base	= S3C2440_GPJCON,
-		.chip	= {
-			.base	= S3C2410_GPJ(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOJ",
-			.ngpio	= 16,
-		},
-	}, {
-		.base	= S3C2443_GPKCON,
-		.chip	= {
-			.base	= S3C2410_GPK(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOK",
-			.ngpio	= 16,
-		},
-	}, {
-		.base	= S3C2443_GPLCON,
-		.chip	= {
-			.base	= S3C2410_GPL(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOL",
-			.ngpio	= 15,
-		},
-	}, {
-		.base	= S3C2443_GPMCON,
-		.chip	= {
-			.base	= S3C2410_GPM(0),
-			.owner	= THIS_MODULE,
-			.label	= "GPIOM",
-			.ngpio	= 2,
-		},
-	},
-#endif
-};
-
-/*
- * GPIO bank summary:
- *
- * Bank	GPIOs	Style	SlpCon	ExtInt Group
- * A	8	4Bit	Yes	1
- * B	7	4Bit	Yes	1
- * C	8	4Bit	Yes	2
- * D	5	4Bit	Yes	3
- * E	5	4Bit	Yes	None
- * F	16	2Bit	Yes	4 [1]
- * G	7	4Bit	Yes	5
- * H	10	4Bit[2]	Yes	6
- * I	16	2Bit	Yes	None
- * J	12	2Bit	Yes	None
- * K	16	4Bit[2]	No	None
- * L	15	4Bit[2] No	None
- * M	6	4Bit	No	IRQ_EINT
- * N	16	2Bit	No	IRQ_EINT
- * O	16	2Bit	Yes	7
- * P	15	2Bit	Yes	8
- * Q	9	2Bit	Yes	9
- *
- * [1] BANKF pins 14,15 do not form part of the external interrupt sources
- * [2] BANK has two control registers, GPxCON0 and GPxCON1
- */
-
-static struct samsung_gpio_chip s3c64xx_gpios_4bit[] = {
-#ifdef CONFIG_ARCH_S3C64XX
-	{
-		.chip	= {
-			.base	= S3C64XX_GPA(0),
-			.ngpio	= S3C64XX_GPIO_A_NR,
-			.label	= "GPA",
-		},
-	}, {
-		.chip	= {
-			.base	= S3C64XX_GPB(0),
-			.ngpio	= S3C64XX_GPIO_B_NR,
-			.label	= "GPB",
-		},
-	}, {
-		.chip	= {
-			.base	= S3C64XX_GPC(0),
-			.ngpio	= S3C64XX_GPIO_C_NR,
-			.label	= "GPC",
-		},
-	}, {
-		.chip	= {
-			.base	= S3C64XX_GPD(0),
-			.ngpio	= S3C64XX_GPIO_D_NR,
-			.label	= "GPD",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[0],
-		.chip	= {
-			.base	= S3C64XX_GPE(0),
-			.ngpio	= S3C64XX_GPIO_E_NR,
-			.label	= "GPE",
-		},
-	}, {
-		.base	= S3C64XX_GPG_BASE,
-		.chip	= {
-			.base	= S3C64XX_GPG(0),
-			.ngpio	= S3C64XX_GPIO_G_NR,
-			.label	= "GPG",
-		},
-	}, {
-		.base	= S3C64XX_GPM_BASE,
-		.config	= &samsung_gpio_cfgs[1],
-		.chip	= {
-			.base	= S3C64XX_GPM(0),
-			.ngpio	= S3C64XX_GPIO_M_NR,
-			.label	= "GPM",
-			.to_irq = s3c64xx_gpiolib_mbank_to_irq,
-		},
-	},
-#endif
-};
-
-static struct samsung_gpio_chip s3c64xx_gpios_4bit2[] = {
-#ifdef CONFIG_ARCH_S3C64XX
-	{
-		.base	= S3C64XX_GPH_BASE + 0x4,
-		.chip	= {
-			.base	= S3C64XX_GPH(0),
-			.ngpio	= S3C64XX_GPIO_H_NR,
-			.label	= "GPH",
-		},
-	}, {
-		.base	= S3C64XX_GPK_BASE + 0x4,
-		.config	= &samsung_gpio_cfgs[0],
-		.chip	= {
-			.base	= S3C64XX_GPK(0),
-			.ngpio	= S3C64XX_GPIO_K_NR,
-			.label	= "GPK",
-		},
-	}, {
-		.base	= S3C64XX_GPL_BASE + 0x4,
-		.config	= &samsung_gpio_cfgs[1],
-		.chip	= {
-			.base	= S3C64XX_GPL(0),
-			.ngpio	= S3C64XX_GPIO_L_NR,
-			.label	= "GPL",
-			.to_irq = s3c64xx_gpiolib_lbank_to_irq,
-		},
-	},
-#endif
-};
-
-static struct samsung_gpio_chip s3c64xx_gpios_2bit[] = {
-#ifdef CONFIG_ARCH_S3C64XX
-	{
-		.base	= S3C64XX_GPF_BASE,
-		.config	= &samsung_gpio_cfgs[6],
-		.chip	= {
-			.base	= S3C64XX_GPF(0),
-			.ngpio	= S3C64XX_GPIO_F_NR,
-			.label	= "GPF",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[7],
-		.chip	= {
-			.base	= S3C64XX_GPI(0),
-			.ngpio	= S3C64XX_GPIO_I_NR,
-			.label	= "GPI",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[7],
-		.chip	= {
-			.base	= S3C64XX_GPJ(0),
-			.ngpio	= S3C64XX_GPIO_J_NR,
-			.label	= "GPJ",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[6],
-		.chip	= {
-			.base	= S3C64XX_GPO(0),
-			.ngpio	= S3C64XX_GPIO_O_NR,
-			.label	= "GPO",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[6],
-		.chip	= {
-			.base	= S3C64XX_GPP(0),
-			.ngpio	= S3C64XX_GPIO_P_NR,
-			.label	= "GPP",
-		},
-	}, {
-		.config	= &samsung_gpio_cfgs[6],
-		.chip	= {
-			.base	= S3C64XX_GPQ(0),
-			.ngpio	= S3C64XX_GPIO_Q_NR,
-			.label	= "GPQ",
-		},
-	}, {
-		.base	= S3C64XX_GPN_BASE,
-		.irq_base = IRQ_EINT(0),
-		.config	= &samsung_gpio_cfgs[5],
-		.chip	= {
-			.base	= S3C64XX_GPN(0),
-			.ngpio	= S3C64XX_GPIO_N_NR,
-			.label	= "GPN",
-			.to_irq = samsung_gpiolib_to_irq,
-		},
-	},
-#endif
-};
-
-/* TODO: cleanup soc_is_* */
-static __init int samsung_gpiolib_init(void)
-{
-	/*
-	 * Currently there are two drivers that can provide GPIO support for
-	 * Samsung SoCs. For device tree enabled platforms, the new
-	 * pinctrl-samsung driver is used, providing both GPIO and pin control
-	 * interfaces. For legacy (non-DT) platforms this driver is used.
-	 */
-	if (of_have_populated_dt())
-		return -ENODEV;
-
-	samsung_gpiolib_set_cfg(samsung_gpio_cfgs, ARRAY_SIZE(samsung_gpio_cfgs));
-
-	if (soc_is_s3c24xx()) {
-		s3c24xx_gpiolib_add_chips(s3c24xx_gpios,
-				ARRAY_SIZE(s3c24xx_gpios), S3C24XX_VA_GPIO);
-	} else if (soc_is_s3c64xx()) {
-		samsung_gpiolib_add_2bit_chips(s3c64xx_gpios_2bit,
-				ARRAY_SIZE(s3c64xx_gpios_2bit),
-				S3C64XX_VA_GPIO + 0xE0, 0x20);
-		samsung_gpiolib_add_4bit_chips(s3c64xx_gpios_4bit,
-				ARRAY_SIZE(s3c64xx_gpios_4bit),
-				S3C64XX_VA_GPIO);
-		samsung_gpiolib_add_4bit2_chips(s3c64xx_gpios_4bit2,
-				ARRAY_SIZE(s3c64xx_gpios_4bit2));
-	} else {
-		WARN(1, "Unknown SoC in gpio-samsung, no GPIOs added\n");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-core_initcall(samsung_gpiolib_init);
-
-int s3c_gpio_cfgpin(unsigned int pin, unsigned int config)
-{
-	struct samsung_gpio_chip *chip = samsung_gpiolib_getchip(pin);
-	unsigned long flags;
-	int offset;
-	int ret;
-
-	if (!chip)
-		return -EINVAL;
-
-	offset = pin - chip->chip.base;
-
-	samsung_gpio_lock(chip, flags);
-	ret = samsung_gpio_do_setcfg(chip, offset, config);
-	samsung_gpio_unlock(chip, flags);
-
-	return ret;
-}
-EXPORT_SYMBOL(s3c_gpio_cfgpin);
-
-int s3c_gpio_cfgpin_range(unsigned int start, unsigned int nr,
-			  unsigned int cfg)
-{
-	int ret;
-
-	for (; nr > 0; nr--, start++) {
-		ret = s3c_gpio_cfgpin(start, cfg);
-		if (ret != 0)
-			return ret;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(s3c_gpio_cfgpin_range);
-
-int s3c_gpio_cfgall_range(unsigned int start, unsigned int nr,
-			  unsigned int cfg, samsung_gpio_pull_t pull)
-{
-	int ret;
-
-	for (; nr > 0; nr--, start++) {
-		s3c_gpio_setpull(start, pull);
-		ret = s3c_gpio_cfgpin(start, cfg);
-		if (ret != 0)
-			return ret;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(s3c_gpio_cfgall_range);
-
-unsigned s3c_gpio_getcfg(unsigned int pin)
-{
-	struct samsung_gpio_chip *chip = samsung_gpiolib_getchip(pin);
-	unsigned long flags;
-	unsigned ret = 0;
-	int offset;
-
-	if (chip) {
-		offset = pin - chip->chip.base;
-
-		samsung_gpio_lock(chip, flags);
-		ret = samsung_gpio_do_getcfg(chip, offset);
-		samsung_gpio_unlock(chip, flags);
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL(s3c_gpio_getcfg);
-
-int s3c_gpio_setpull(unsigned int pin, samsung_gpio_pull_t pull)
-{
-	struct samsung_gpio_chip *chip = samsung_gpiolib_getchip(pin);
-	unsigned long flags;
-	int offset, ret;
-
-	if (!chip)
-		return -EINVAL;
-
-	offset = pin - chip->chip.base;
-
-	samsung_gpio_lock(chip, flags);
-	ret = samsung_gpio_do_setpull(chip, offset, pull);
-	samsung_gpio_unlock(chip, flags);
-
-	return ret;
-}
-EXPORT_SYMBOL(s3c_gpio_setpull);
-
-samsung_gpio_pull_t s3c_gpio_getpull(unsigned int pin)
-{
-	struct samsung_gpio_chip *chip = samsung_gpiolib_getchip(pin);
-	unsigned long flags;
-	int offset;
-	u32 pup = 0;
-
-	if (chip) {
-		offset = pin - chip->chip.base;
-
-		samsung_gpio_lock(chip, flags);
-		pup = samsung_gpio_do_getpull(chip, offset);
-		samsung_gpio_unlock(chip, flags);
-	}
-
-	return (__force samsung_gpio_pull_t)pup;
-}
-EXPORT_SYMBOL(s3c_gpio_getpull);
-
-#ifdef CONFIG_PLAT_S3C24XX
-unsigned int s3c2410_modify_misccr(unsigned int clear, unsigned int change)
-{
-	unsigned long flags;
-	unsigned long misccr;
-
-	local_irq_save(flags);
-	misccr = __raw_readl(S3C24XX_MISCCR);
-	misccr &= ~clear;
-	misccr ^= change;
-	__raw_writel(misccr, S3C24XX_MISCCR);
-	local_irq_restore(flags);
-
-	return misccr;
-}
-EXPORT_SYMBOL(s3c2410_modify_misccr);
-#endif
diff --git a/drivers/gpio/gpio-sch.c b/drivers/gpio/gpio-sch.c
index b72906f5b999..5314ee4b947d 100644
--- a/drivers/gpio/gpio-sch.c
+++ b/drivers/gpio/gpio-sch.c
@@ -41,8 +41,6 @@ struct sch_gpio {
 	unsigned short resume_base;
 };
 
-#define to_sch_gpio(gc)	container_of(gc, struct sch_gpio, chip)
-
 static unsigned sch_gpio_offset(struct sch_gpio *sch, unsigned gpio,
 				unsigned reg)
 {
@@ -65,7 +63,7 @@ static unsigned sch_gpio_bit(struct sch_gpio *sch, unsigned gpio)
 
 static int sch_gpio_reg_get(struct gpio_chip *gc, unsigned gpio, unsigned reg)
 {
-	struct sch_gpio *sch = to_sch_gpio(gc);
+	struct sch_gpio *sch = gpiochip_get_data(gc);
 	unsigned short offset, bit;
 	u8 reg_val;
 
@@ -80,7 +78,7 @@ static int sch_gpio_reg_get(struct gpio_chip *gc, unsigned gpio, unsigned reg)
 static void sch_gpio_reg_set(struct gpio_chip *gc, unsigned gpio, unsigned reg,
 			     int val)
 {
-	struct sch_gpio *sch = to_sch_gpio(gc);
+	struct sch_gpio *sch = gpiochip_get_data(gc);
 	unsigned short offset, bit;
 	u8 reg_val;
 
@@ -97,7 +95,7 @@ static void sch_gpio_reg_set(struct gpio_chip *gc, unsigned gpio, unsigned reg,
 
 static int sch_gpio_direction_in(struct gpio_chip *gc, unsigned gpio_num)
 {
-	struct sch_gpio *sch = to_sch_gpio(gc);
+	struct sch_gpio *sch = gpiochip_get_data(gc);
 
 	spin_lock(&sch->lock);
 	sch_gpio_reg_set(gc, gpio_num, GIO, 1);
@@ -112,7 +110,7 @@ static int sch_gpio_get(struct gpio_chip *gc, unsigned gpio_num)
 
 static void sch_gpio_set(struct gpio_chip *gc, unsigned gpio_num, int val)
 {
-	struct sch_gpio *sch = to_sch_gpio(gc);
+	struct sch_gpio *sch = gpiochip_get_data(gc);
 
 	spin_lock(&sch->lock);
 	sch_gpio_reg_set(gc, gpio_num, GLV, val);
@@ -122,7 +120,7 @@ static void sch_gpio_set(struct gpio_chip *gc, unsigned gpio_num, int val)
 static int sch_gpio_direction_out(struct gpio_chip *gc, unsigned gpio_num,
 				  int val)
 {
-	struct sch_gpio *sch = to_sch_gpio(gc);
+	struct sch_gpio *sch = gpiochip_get_data(gc);
 
 	spin_lock(&sch->lock);
 	sch_gpio_reg_set(gc, gpio_num, GIO, 0);
@@ -171,7 +169,7 @@ static int sch_gpio_probe(struct platform_device *pdev)
 	sch->iobase = res->start;
 	sch->chip = sch_gpio_chip;
 	sch->chip.label = dev_name(&pdev->dev);
-	sch->chip.dev = &pdev->dev;
+	sch->chip.parent = &pdev->dev;
 
 	switch (pdev->id) {
 	case PCI_DEVICE_ID_INTEL_SCH_LPC:
@@ -217,7 +215,7 @@ static int sch_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sch);
 
-	return gpiochip_add(&sch->chip);
+	return gpiochip_add_data(&sch->chip, sch);
 }
 
 static int sch_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-sch311x.c b/drivers/gpio/gpio-sch311x.c
index 0cb11413e814..1cbd77a04e7b 100644
--- a/drivers/gpio/gpio-sch311x.c
+++ b/drivers/gpio/gpio-sch311x.c
@@ -93,13 +93,6 @@ static struct sch311x_gpio_block_def sch311x_gpio_blocks[] = {
 	},
 };
 
-static inline struct sch311x_gpio_block *
-to_sch311x_gpio_block(struct gpio_chip *chip)
-{
-	return container_of(chip, struct sch311x_gpio_block, chip);
-}
-
-
 /*
  *	Super-IO functions
  */
@@ -142,14 +135,14 @@ static inline void sch311x_sio_outb(int sio_config_port, int reg, int val)
 
 static int sch311x_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 
 	if (block->config_regs[offset] == 0) /* GPIO is not available */
 		return -ENODEV;
 
 	if (!request_region(block->runtime_reg + block->config_regs[offset],
 			    1, DRV_NAME)) {
-		dev_err(chip->dev, "Failed to request region 0x%04x.\n",
+		dev_err(chip->parent, "Failed to request region 0x%04x.\n",
 			block->runtime_reg + block->config_regs[offset]);
 		return -EBUSY;
 	}
@@ -158,7 +151,7 @@ static int sch311x_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void sch311x_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 
 	if (block->config_regs[offset] == 0) /* GPIO is not available */
 		return;
@@ -168,7 +161,7 @@ static void sch311x_gpio_free(struct gpio_chip *chip, unsigned offset)
 
 static int sch311x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 	unsigned char data;
 
 	spin_lock(&block->lock);
@@ -192,7 +185,7 @@ static void __sch311x_gpio_set(struct sch311x_gpio_block *block,
 static void sch311x_gpio_set(struct gpio_chip *chip, unsigned offset,
 			     int value)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 
 	spin_lock(&block->lock);
 	 __sch311x_gpio_set(block, offset, value);
@@ -201,7 +194,7 @@ static void sch311x_gpio_set(struct gpio_chip *chip, unsigned offset,
 
 static int sch311x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 
 	spin_lock(&block->lock);
 	outb(SCH311X_GPIO_CONF_IN, block->runtime_reg +
@@ -214,7 +207,7 @@ static int sch311x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 static int sch311x_gpio_direction_out(struct gpio_chip *chip, unsigned offset,
 				      int value)
 {
-	struct sch311x_gpio_block *block = to_sch311x_gpio_block(chip);
+	struct sch311x_gpio_block *block = gpiochip_get_data(chip);
 
 	spin_lock(&block->lock);
 
@@ -229,7 +222,7 @@ static int sch311x_gpio_direction_out(struct gpio_chip *chip, unsigned offset,
 
 static int sch311x_gpio_probe(struct platform_device *pdev)
 {
-	struct sch311x_pdev_data *pdata = pdev->dev.platform_data;
+	struct sch311x_pdev_data *pdata = dev_get_platdata(&pdev->dev);
 	struct sch311x_gpio_priv *priv;
 	struct sch311x_gpio_block *block;
 	int err, i;
@@ -261,13 +254,13 @@ static int sch311x_gpio_probe(struct platform_device *pdev)
 		block->chip.get = sch311x_gpio_get;
 		block->chip.set = sch311x_gpio_set;
 		block->chip.ngpio = 8;
-		block->chip.dev = &pdev->dev;
+		block->chip.parent = &pdev->dev;
 		block->chip.base = sch311x_gpio_blocks[i].base;
 		block->config_regs = sch311x_gpio_blocks[i].config_regs;
 		block->data_reg = sch311x_gpio_blocks[i].data_reg;
 		block->runtime_reg = pdata->runtime_reg;
 
-		err = gpiochip_add(&block->chip);
+		err = gpiochip_add_data(&block->chip, block);
 		if (err < 0) {
 			dev_err(&pdev->dev,
 				"Could not register gpiochip, %d\n", err);
@@ -289,7 +282,7 @@ exit_err:
 
 static int sch311x_gpio_remove(struct platform_device *pdev)
 {
-	struct sch311x_pdev_data *pdata = pdev->dev.platform_data;
+	struct sch311x_pdev_data *pdata = dev_get_platdata(&pdev->dev);
 	struct sch311x_gpio_priv *priv = platform_get_drvdata(pdev);
 	int i;
 
diff --git a/drivers/gpio/gpio-sodaville.c b/drivers/gpio/gpio-sodaville.c
index 34b02b42ab9e..e3cb6772f6ec 100644
--- a/drivers/gpio/gpio-sodaville.c
+++ b/drivers/gpio/gpio-sodaville.c
@@ -10,7 +10,6 @@
  */
 
 #include <linux/errno.h>
-#include <linux/gpio.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/irq.h>
@@ -20,7 +19,7 @@
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/of_irq.h>
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 
 #define DRV_NAME		"sdv_gpio"
 #define SDV_NUM_PUB_GPIOS	12
@@ -43,7 +42,7 @@ struct sdv_gpio_chip_data {
 	void __iomem *gpio_pub_base;
 	struct irq_domain *id;
 	struct irq_chip_generic *gc;
-	struct bgpio_chip bgpio;
+	struct gpio_chip chip;
 };
 
 static int sdv_gpio_pub_set_type(struct irq_data *d, unsigned int type)
@@ -226,14 +225,14 @@ static int sdv_gpio_probe(struct pci_dev *pdev,
 		writel(mux_val, sd->gpio_pub_base + GPMUXCTL);
 	}
 
-	ret = bgpio_init(&sd->bgpio, &pdev->dev, 4,
+	ret = bgpio_init(&sd->chip, &pdev->dev, 4,
 			sd->gpio_pub_base + GPINR, sd->gpio_pub_base + GPOUTR,
 			NULL, sd->gpio_pub_base + GPOER, NULL, 0);
 	if (ret)
 		goto unmap;
-	sd->bgpio.gc.ngpio = SDV_NUM_PUB_GPIOS;
+	sd->chip.ngpio = SDV_NUM_PUB_GPIOS;
 
-	ret = gpiochip_add(&sd->bgpio.gc);
+	ret = gpiochip_add_data(&sd->chip, sd);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "gpiochip_add() failed.\n");
 		goto unmap;
@@ -265,7 +264,7 @@ static void sdv_gpio_remove(struct pci_dev *pdev)
 	free_irq(pdev->irq, sd);
 	irq_free_descs(sd->irq_base, SDV_NUM_PUB_GPIOS);
 
-	gpiochip_remove(&sd->bgpio.gc);
+	gpiochip_remove(&sd->chip);
 	pci_release_region(pdev, GPIO_BAR);
 	iounmap(sd->gpio_pub_base);
 	pci_disable_device(pdev);
diff --git a/drivers/gpio/gpio-spear-spics.c b/drivers/gpio/gpio-spear-spics.c
index 69ffca5b073b..50fb09080a6b 100644
--- a/drivers/gpio/gpio-spear-spics.c
+++ b/drivers/gpio/gpio-spear-spics.c
@@ -62,8 +62,7 @@ static int spics_get_value(struct gpio_chip *chip, unsigned offset)
 
 static void spics_set_value(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct spear_spics *spics = container_of(chip, struct spear_spics,
-			chip);
+	struct spear_spics *spics = gpiochip_get_data(chip);
 	u32 tmp;
 
 	/* select chip select from register */
@@ -94,8 +93,7 @@ static int spics_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static int spics_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct spear_spics *spics = container_of(chip, struct spear_spics,
-			chip);
+	struct spear_spics *spics = gpiochip_get_data(chip);
 	u32 tmp;
 
 	if (!spics->use_count++) {
@@ -110,8 +108,7 @@ static int spics_request(struct gpio_chip *chip, unsigned offset)
 
 static void spics_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct spear_spics *spics = container_of(chip, struct spear_spics,
-			chip);
+	struct spear_spics *spics = gpiochip_get_data(chip);
 	u32 tmp;
 
 	if (!--spics->use_count) {
@@ -164,11 +161,11 @@ static int spics_gpio_probe(struct platform_device *pdev)
 	spics->chip.get = spics_get_value;
 	spics->chip.set = spics_set_value;
 	spics->chip.label = dev_name(&pdev->dev);
-	spics->chip.dev = &pdev->dev;
+	spics->chip.parent = &pdev->dev;
 	spics->chip.owner = THIS_MODULE;
 	spics->last_off = -1;
 
-	ret = gpiochip_add(&spics->chip);
+	ret = gpiochip_add_data(&spics->chip, spics);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to add gpio chip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-sta2x11.c b/drivers/gpio/gpio-sta2x11.c
index 55e47828ddfc..83af1cb36333 100644
--- a/drivers/gpio/gpio-sta2x11.c
+++ b/drivers/gpio/gpio-sta2x11.c
@@ -74,7 +74,7 @@ static inline u32 __bit(int nr)
 
 static void gsta_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 {
-	struct gsta_gpio *chip = container_of(gpio, struct gsta_gpio, gpio);
+	struct gsta_gpio *chip = gpiochip_get_data(gpio);
 	struct gsta_regs __iomem *regs = __regs(chip, nr);
 	u32 bit = __bit(nr);
 
@@ -86,17 +86,17 @@ static void gsta_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 
 static int gsta_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
-	struct gsta_gpio *chip = container_of(gpio, struct gsta_gpio, gpio);
+	struct gsta_gpio *chip = gpiochip_get_data(gpio);
 	struct gsta_regs __iomem *regs = __regs(chip, nr);
 	u32 bit = __bit(nr);
 
-	return readl(&regs->dat) & bit;
+	return !!(readl(&regs->dat) & bit);
 }
 
 static int gsta_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 				      int val)
 {
-	struct gsta_gpio *chip = container_of(gpio, struct gsta_gpio, gpio);
+	struct gsta_gpio *chip = gpiochip_get_data(gpio);
 	struct gsta_regs __iomem *regs = __regs(chip, nr);
 	u32 bit = __bit(nr);
 
@@ -111,7 +111,7 @@ static int gsta_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 
 static int gsta_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 {
-	struct gsta_gpio *chip = container_of(gpio, struct gsta_gpio, gpio);
+	struct gsta_gpio *chip = gpiochip_get_data(gpio);
 	struct gsta_regs __iomem *regs = __regs(chip, nr);
 	u32 bit = __bit(nr);
 
@@ -121,7 +121,7 @@ static int gsta_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 
 static int gsta_gpio_to_irq(struct gpio_chip *gpio, unsigned offset)
 {
-	struct gsta_gpio *chip = container_of(gpio, struct gsta_gpio, gpio);
+	struct gsta_gpio *chip = gpiochip_get_data(gpio);
 	return chip->irq_base + offset;
 }
 
@@ -409,7 +409,7 @@ static int gsta_probe(struct platform_device *dev)
 		goto err_free_descs;
 	}
 
-	err = gpiochip_add(&chip->gpio);
+	err = gpiochip_add_data(&chip->gpio, chip);
 	if (err < 0) {
 		dev_err(&dev->dev, "sta2x11 gpio: Can't register (%i)\n",
 			-err);
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index dabfb99dddef..5197edf1acfd 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -36,14 +36,9 @@ struct stmpe_gpio {
 	u8 oldregs[CACHE_NR_REGS][CACHE_NR_BANKS];
 };
 
-static inline struct stmpe_gpio *to_stmpe_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct stmpe_gpio, chip);
-}
-
 static int stmpe_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(chip);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	u8 reg = stmpe->regs[STMPE_IDX_GPMR_LSB] - (offset / 8);
 	u8 mask = 1 << (offset % 8);
@@ -58,7 +53,7 @@ static int stmpe_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void stmpe_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(chip);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int which = val ? STMPE_IDX_GPSR_LSB : STMPE_IDX_GPCR_LSB;
 	u8 reg = stmpe->regs[which] - (offset / 8);
@@ -77,7 +72,7 @@ static void stmpe_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 static int stmpe_gpio_direction_output(struct gpio_chip *chip,
 					 unsigned offset, int val)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(chip);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
 	u8 mask = 1 << (offset % 8);
@@ -90,7 +85,7 @@ static int stmpe_gpio_direction_output(struct gpio_chip *chip,
 static int stmpe_gpio_direction_input(struct gpio_chip *chip,
 					unsigned offset)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(chip);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
 	u8 mask = 1 << (offset % 8);
@@ -100,7 +95,7 @@ static int stmpe_gpio_direction_input(struct gpio_chip *chip,
 
 static int stmpe_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(chip);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 
 	if (stmpe_gpio->norequest_mask & (1 << offset))
@@ -123,7 +118,7 @@ static struct gpio_chip template_chip = {
 static int stmpe_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -151,7 +146,7 @@ static int stmpe_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 static void stmpe_gpio_irq_lock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 
 	mutex_lock(&stmpe_gpio->irq_lock);
 }
@@ -159,7 +154,7 @@ static void stmpe_gpio_irq_lock(struct irq_data *d)
 static void stmpe_gpio_irq_sync_unlock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
 	static const u8 regmap[] = {
@@ -193,7 +188,7 @@ static void stmpe_gpio_irq_sync_unlock(struct irq_data *d)
 static void stmpe_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -204,7 +199,7 @@ static void stmpe_gpio_irq_mask(struct irq_data *d)
 static void stmpe_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -216,7 +211,7 @@ static void stmpe_dbg_show_one(struct seq_file *s,
 			       struct gpio_chip *gc,
 			       unsigned offset, unsigned gpio)
 {
-	struct stmpe_gpio *stmpe_gpio = to_stmpe_gpio(gc);
+	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	const char *label = gpiochip_is_requested(gc, offset);
 	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
@@ -356,7 +351,7 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 	stmpe_gpio->stmpe = stmpe;
 	stmpe_gpio->chip = template_chip;
 	stmpe_gpio->chip.ngpio = stmpe->num_gpios;
-	stmpe_gpio->chip.dev = &pdev->dev;
+	stmpe_gpio->chip.parent = &pdev->dev;
 	stmpe_gpio->chip.of_node = np;
 	stmpe_gpio->chip.base = -1;
 
@@ -375,7 +370,7 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_free;
 
-	ret = gpiochip_add(&stmpe_gpio->chip);
+	ret = gpiochip_add_data(&stmpe_gpio->chip, stmpe_gpio);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to add gpiochip: %d\n", ret);
 		goto out_disable;
diff --git a/drivers/gpio/gpio-stp-xway.c b/drivers/gpio/gpio-stp-xway.c
index 81bdbe7ba2a4..d11dd48570b2 100644
--- a/drivers/gpio/gpio-stp-xway.c
+++ b/drivers/gpio/gpio-stp-xway.c
@@ -100,8 +100,7 @@ struct xway_stp {
  */
 static void xway_stp_set(struct gpio_chip *gc, unsigned gpio, int val)
 {
-	struct xway_stp *chip =
-		container_of(gc, struct xway_stp, gc);
+	struct xway_stp *chip = gpiochip_get_data(gc);
 
 	if (val)
 		chip->shadow |= BIT(gpio);
@@ -135,11 +134,10 @@ static int xway_stp_dir_out(struct gpio_chip *gc, unsigned gpio, int val)
  */
 static int xway_stp_request(struct gpio_chip *gc, unsigned gpio)
 {
-	struct xway_stp *chip =
-		container_of(gc, struct xway_stp, gc);
+	struct xway_stp *chip = gpiochip_get_data(gc);
 
 	if ((gpio < 8) && (chip->reserved & BIT(gpio))) {
-		dev_err(gc->dev, "GPIO %d is driven by hardware\n", gpio);
+		dev_err(gc->parent, "GPIO %d is driven by hardware\n", gpio);
 		return -ENODEV;
 	}
 
@@ -214,7 +212,7 @@ static int xway_stp_probe(struct platform_device *pdev)
 	if (IS_ERR(chip->virt))
 		return PTR_ERR(chip->virt);
 
-	chip->gc.dev = &pdev->dev;
+	chip->gc.parent = &pdev->dev;
 	chip->gc.label = "stp-xway";
 	chip->gc.direction_output = xway_stp_dir_out;
 	chip->gc.set = xway_stp_set;
@@ -260,7 +258,7 @@ static int xway_stp_probe(struct platform_device *pdev)
 
 	ret = xway_stp_hw_init(chip);
 	if (!ret)
-		ret = gpiochip_add(&chip->gc);
+		ret = gpiochip_add_data(&chip->gc, chip);
 
 	if (!ret)
 		dev_info(&pdev->dev, "Init done\n");
diff --git a/drivers/gpio/gpio-sx150x.c b/drivers/gpio/gpio-sx150x.c
index 76f920173a2f..e6cff1cabd0c 100644
--- a/drivers/gpio/gpio-sx150x.c
+++ b/drivers/gpio/gpio-sx150x.c
@@ -32,8 +32,19 @@
 #define NO_UPDATE_PENDING	-1
 
 /* The chip models of sx150x */
-#define SX150X_456 0
-#define SX150X_789 1
+#define SX150X_123 0
+#define SX150X_456 1
+#define SX150X_789 2
+
+struct sx150x_123_pri {
+	u8 reg_pld_mode;
+	u8 reg_pld_table0;
+	u8 reg_pld_table1;
+	u8 reg_pld_table2;
+	u8 reg_pld_table3;
+	u8 reg_pld_table4;
+	u8 reg_advance;
+};
 
 struct sx150x_456_pri {
 	u8 reg_pld_mode;
@@ -65,6 +76,7 @@ struct sx150x_device_data {
 	u8 reg_sense;
 	u8 ngpios;
 	union {
+		struct sx150x_123_pri x123;
 		struct sx150x_456_pri x456;
 		struct sx150x_789_pri x789;
 	} pri;
@@ -142,12 +154,33 @@ static const struct sx150x_device_data sx150x_devices[] = {
 		},
 		.ngpios	= 16
 	},
+	[3] = { /* sx1502q */
+		.model = SX150X_123,
+		.reg_pullup	= 0x02,
+		.reg_pulldn	= 0x03,
+		.reg_dir	= 0x01,
+		.reg_data	= 0x00,
+		.reg_irq_mask	= 0x05,
+		.reg_irq_src	= 0x08,
+		.reg_sense	= 0x07,
+		.pri.x123 = {
+			.reg_pld_mode	= 0x10,
+			.reg_pld_table0	= 0x11,
+			.reg_pld_table1	= 0x12,
+			.reg_pld_table2	= 0x13,
+			.reg_pld_table3	= 0x14,
+			.reg_pld_table4	= 0x15,
+			.reg_advance	= 0xad,
+		},
+		.ngpios	= 8,
+	},
 };
 
 static const struct i2c_device_id sx150x_id[] = {
 	{"sx1508q", 0},
 	{"sx1509q", 1},
 	{"sx1506q", 2},
+	{"sx1502q", 3},
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, sx150x_id);
@@ -156,15 +189,11 @@ static const struct of_device_id sx150x_of_match[] = {
 	{ .compatible = "semtech,sx1508q" },
 	{ .compatible = "semtech,sx1509q" },
 	{ .compatible = "semtech,sx1506q" },
+	{ .compatible = "semtech,sx1502q" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sx150x_of_match);
 
-struct sx150x_chip *to_sx150x(struct gpio_chip *gc)
-{
-	return container_of(gc, struct sx150x_chip, gpio_chip);
-}
-
 static s32 sx150x_i2c_write(struct i2c_client *client, u8 reg, u8 val)
 {
 	s32 err = i2c_smbus_write_byte_data(client, reg, val);
@@ -301,7 +330,7 @@ static int sx150x_io_output(struct sx150x_chip *chip, unsigned offset, int val)
 
 static int sx150x_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct sx150x_chip *chip = to_sx150x(gc);
+	struct sx150x_chip *chip = gpiochip_get_data(gc);
 	int status = -EINVAL;
 
 	if (!offset_is_oscio(chip, offset)) {
@@ -310,12 +339,12 @@ static int sx150x_gpio_get(struct gpio_chip *gc, unsigned offset)
 		mutex_unlock(&chip->lock);
 	}
 
-	return status;
+	return (status < 0) ? status : !!status;
 }
 
 static void sx150x_gpio_set(struct gpio_chip *gc, unsigned offset, int val)
 {
-	struct sx150x_chip *chip = to_sx150x(gc);
+	struct sx150x_chip *chip = gpiochip_get_data(gc);
 
 	mutex_lock(&chip->lock);
 	if (offset_is_oscio(chip, offset))
@@ -327,7 +356,7 @@ static void sx150x_gpio_set(struct gpio_chip *gc, unsigned offset, int val)
 
 static int sx150x_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct sx150x_chip *chip = to_sx150x(gc);
+	struct sx150x_chip *chip = gpiochip_get_data(gc);
 	int status = -EINVAL;
 
 	if (!offset_is_oscio(chip, offset)) {
@@ -342,7 +371,7 @@ static int sx150x_gpio_direction_output(struct gpio_chip *gc,
 					unsigned offset,
 					int val)
 {
-	struct sx150x_chip *chip = to_sx150x(gc);
+	struct sx150x_chip *chip = gpiochip_get_data(gc);
 	int status = 0;
 
 	if (!offset_is_oscio(chip, offset)) {
@@ -355,7 +384,7 @@ static int sx150x_gpio_direction_output(struct gpio_chip *gc,
 
 static void sx150x_irq_mask(struct irq_data *d)
 {
-	struct sx150x_chip *chip = to_sx150x(irq_data_get_irq_chip_data(d));
+	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	unsigned n = d->hwirq;
 
 	chip->irq_masked |= (1 << n);
@@ -364,7 +393,7 @@ static void sx150x_irq_mask(struct irq_data *d)
 
 static void sx150x_irq_unmask(struct irq_data *d)
 {
-	struct sx150x_chip *chip = to_sx150x(irq_data_get_irq_chip_data(d));
+	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	unsigned n = d->hwirq;
 
 	chip->irq_masked &= ~(1 << n);
@@ -373,7 +402,7 @@ static void sx150x_irq_unmask(struct irq_data *d)
 
 static int sx150x_irq_set_type(struct irq_data *d, unsigned int flow_type)
 {
-	struct sx150x_chip *chip = to_sx150x(irq_data_get_irq_chip_data(d));
+	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	unsigned n, val = 0;
 
 	if (flow_type & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
@@ -428,14 +457,14 @@ static irqreturn_t sx150x_irq_thread_fn(int irq, void *dev_id)
 
 static void sx150x_irq_bus_lock(struct irq_data *d)
 {
-	struct sx150x_chip *chip = to_sx150x(irq_data_get_irq_chip_data(d));
+	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 
 	mutex_lock(&chip->lock);
 }
 
 static void sx150x_irq_bus_sync_unlock(struct irq_data *d)
 {
-	struct sx150x_chip *chip = to_sx150x(irq_data_get_irq_chip_data(d));
+	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	unsigned n;
 
 	if (chip->irq_update == NO_UPDATE_PENDING)
@@ -473,7 +502,7 @@ static void sx150x_init_chip(struct sx150x_chip *chip,
 
 	chip->client                     = client;
 	chip->dev_cfg                    = &sx150x_devices[driver_data];
-	chip->gpio_chip.dev              = &client->dev;
+	chip->gpio_chip.parent              = &client->dev;
 	chip->gpio_chip.label            = client->name;
 	chip->gpio_chip.direction_input  = sx150x_gpio_direction_input;
 	chip->gpio_chip.direction_output = sx150x_gpio_direction_output;
@@ -545,10 +574,14 @@ static int sx150x_init_hw(struct sx150x_chip *chip,
 		err = sx150x_i2c_write(chip->client,
 				chip->dev_cfg->pri.x789.reg_misc,
 				0x01);
-	else
+	else if (chip->dev_cfg->model == SX150X_456)
 		err = sx150x_i2c_write(chip->client,
 				chip->dev_cfg->pri.x456.reg_advance,
 				0x04);
+	else
+		err = sx150x_i2c_write(chip->client,
+				chip->dev_cfg->pri.x123.reg_advance,
+				0x00);
 	if (err < 0)
 		return err;
 
@@ -574,13 +607,20 @@ static int sx150x_init_hw(struct sx150x_chip *chip,
 				pdata->io_polarity);
 		if (err < 0)
 			return err;
-	} else {
+	} else if (chip->dev_cfg->model == SX150X_456) {
 		/* Set all pins to work in normal mode */
 		err = sx150x_init_io(chip,
 				chip->dev_cfg->pri.x456.reg_pld_mode,
 				0);
 		if (err < 0)
 			return err;
+	} else {
+		/* Set all pins to work in normal mode */
+		err = sx150x_init_io(chip,
+				chip->dev_cfg->pri.x123.reg_pld_mode,
+				0);
+		if (err < 0)
+			return err;
 	}
 
 
@@ -647,7 +687,7 @@ static int sx150x_probe(struct i2c_client *client,
 	if (rc < 0)
 		return rc;
 
-	rc = gpiochip_add(&chip->gpio_chip);
+	rc = gpiochip_add_data(&chip->gpio_chip, chip);
 	if (rc)
 		return rc;
 
@@ -680,7 +720,6 @@ static int sx150x_remove(struct i2c_client *client)
 static struct i2c_driver sx150x_driver = {
 	.driver = {
 		.name = "sx150x",
-		.owner = THIS_MODULE,
 		.of_match_table = of_match_ptr(sx150x_of_match),
 	},
 	.probe    = sx150x_probe,
diff --git a/drivers/gpio/gpio-syscon.c b/drivers/gpio/gpio-syscon.c
index 7b25fdf64802..e5c5b6205886 100644
--- a/drivers/gpio/gpio-syscon.c
+++ b/drivers/gpio/gpio-syscon.c
@@ -59,14 +59,9 @@ struct syscon_gpio_priv {
 	u32				dir_reg_offset;
 };
 
-static inline struct syscon_gpio_priv *to_syscon_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct syscon_gpio_priv, chip);
-}
-
 static int syscon_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct syscon_gpio_priv *priv = to_syscon_gpio(chip);
+	struct syscon_gpio_priv *priv = gpiochip_get_data(chip);
 	unsigned int val, offs;
 	int ret;
 
@@ -82,7 +77,7 @@ static int syscon_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void syscon_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct syscon_gpio_priv *priv = to_syscon_gpio(chip);
+	struct syscon_gpio_priv *priv = gpiochip_get_data(chip);
 	unsigned int offs;
 
 	offs = priv->dreg_offset + priv->data->dat_bit_offset + offset;
@@ -95,7 +90,7 @@ static void syscon_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 
 static int syscon_gpio_dir_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct syscon_gpio_priv *priv = to_syscon_gpio(chip);
+	struct syscon_gpio_priv *priv = gpiochip_get_data(chip);
 
 	if (priv->data->flags & GPIO_SYSCON_FEAT_DIR) {
 		unsigned int offs;
@@ -113,7 +108,7 @@ static int syscon_gpio_dir_in(struct gpio_chip *chip, unsigned offset)
 
 static int syscon_gpio_dir_out(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct syscon_gpio_priv *priv = to_syscon_gpio(chip);
+	struct syscon_gpio_priv *priv = gpiochip_get_data(chip);
 
 	if (priv->data->flags & GPIO_SYSCON_FEAT_DIR) {
 		unsigned int offs;
@@ -144,7 +139,7 @@ static const struct syscon_gpio_data clps711x_mctrl_gpio = {
 
 static void keystone_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct syscon_gpio_priv *priv = to_syscon_gpio(chip);
+	struct syscon_gpio_priv *priv = gpiochip_get_data(chip);
 	unsigned int offs;
 	int ret;
 
@@ -159,7 +154,7 @@ static void keystone_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 			BIT(offs % SYSCON_REG_BITS) | KEYSTONE_LOCK_BIT,
 			BIT(offs % SYSCON_REG_BITS) | KEYSTONE_LOCK_BIT);
 	if (ret < 0)
-		dev_err(chip->dev, "gpio write failed ret(%d)\n", ret);
+		dev_err(chip->parent, "gpio write failed ret(%d)\n", ret);
 }
 
 static const struct syscon_gpio_data keystone_dsp_gpio = {
@@ -228,7 +223,7 @@ static int syscon_gpio_probe(struct platform_device *pdev)
 		priv->dir_reg_offset <<= 3;
 	}
 
-	priv->chip.dev = dev;
+	priv->chip.parent = dev;
 	priv->chip.owner = THIS_MODULE;
 	priv->chip.label = dev_name(dev);
 	priv->chip.base = -1;
@@ -243,7 +238,7 @@ static int syscon_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 
-	return gpiochip_add(&priv->chip);
+	return gpiochip_add_data(&priv->chip, priv);
 }
 
 static int syscon_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c
index 4356e6c20fc5..5eaec20ddbc7 100644
--- a/drivers/gpio/gpio-tb10x.c
+++ b/drivers/gpio/gpio-tb10x.c
@@ -87,14 +87,9 @@ static inline void tb10x_set_bits(struct tb10x_gpio *gpio, unsigned int offs,
 	spin_unlock_irqrestore(&gpio->spinlock, flags);
 }
 
-static inline struct tb10x_gpio *to_tb10x_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct tb10x_gpio, gc);
-}
-
 static int tb10x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct tb10x_gpio *tb10x_gpio = to_tb10x_gpio(chip);
+	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 	int mask = BIT(offset);
 	int val = TB10X_GPIO_DIR_IN << offset;
 
@@ -105,7 +100,7 @@ static int tb10x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int tb10x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct tb10x_gpio *tb10x_gpio = to_tb10x_gpio(chip);
+	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 	int val;
 
 	val = tb10x_reg_read(tb10x_gpio, OFFSET_TO_REG_DATA);
@@ -118,7 +113,7 @@ static int tb10x_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void tb10x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct tb10x_gpio *tb10x_gpio = to_tb10x_gpio(chip);
+	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 	int mask = BIT(offset);
 	int val = value << offset;
 
@@ -128,7 +123,7 @@ static void tb10x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int tb10x_gpio_direction_out(struct gpio_chip *chip,
 					unsigned offset, int value)
 {
-	struct tb10x_gpio *tb10x_gpio = to_tb10x_gpio(chip);
+	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 	int mask = BIT(offset);
 	int val = TB10X_GPIO_DIR_OUT << offset;
 
@@ -140,7 +135,7 @@ static int tb10x_gpio_direction_out(struct gpio_chip *chip,
 
 static int tb10x_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct tb10x_gpio *tb10x_gpio = to_tb10x_gpio(chip);
+	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 
 	return irq_create_mapping(tb10x_gpio->domain, offset);
 }
@@ -197,7 +192,7 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
 		return PTR_ERR(tb10x_gpio->base);
 
 	tb10x_gpio->gc.label		= of_node_full_name(dn);
-	tb10x_gpio->gc.dev		= &pdev->dev;
+	tb10x_gpio->gc.parent		= &pdev->dev;
 	tb10x_gpio->gc.owner		= THIS_MODULE;
 	tb10x_gpio->gc.direction_input	= tb10x_gpio_direction_in;
 	tb10x_gpio->gc.get		= tb10x_gpio_get;
@@ -210,7 +205,7 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
 	tb10x_gpio->gc.can_sleep	= false;
 
 
-	ret = gpiochip_add(&tb10x_gpio->gc);
+	ret = gpiochip_add_data(&tb10x_gpio->gc, tb10x_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not add gpiochip.\n");
 		goto fail_gpiochip_registration;
diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c
index d1d585ddb9ab..05a27ec55add 100644
--- a/drivers/gpio/gpio-tc3589x.c
+++ b/drivers/gpio/gpio-tc3589x.c
@@ -34,14 +34,9 @@ struct tc3589x_gpio {
 	u8 oldregs[CACHE_NR_REGS][CACHE_NR_BANKS];
 };
 
-static inline struct tc3589x_gpio *to_tc3589x_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct tc3589x_gpio, chip);
-}
-
 static int tc3589x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(chip);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODATA0 + (offset / 8) * 2;
 	u8 mask = 1 << (offset % 8);
@@ -51,12 +46,12 @@ static int tc3589x_gpio_get(struct gpio_chip *chip, unsigned offset)
 	if (ret < 0)
 		return ret;
 
-	return ret & mask;
+	return !!(ret & mask);
 }
 
 static void tc3589x_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(chip);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODATA0 + (offset / 8) * 2;
 	unsigned pos = offset % 8;
@@ -68,7 +63,7 @@ static void tc3589x_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 static int tc3589x_gpio_direction_output(struct gpio_chip *chip,
 					 unsigned offset, int val)
 {
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(chip);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODIR0 + offset / 8;
 	unsigned pos = offset % 8;
@@ -81,7 +76,7 @@ static int tc3589x_gpio_direction_output(struct gpio_chip *chip,
 static int tc3589x_gpio_direction_input(struct gpio_chip *chip,
 					unsigned offset)
 {
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(chip);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODIR0 + offset / 8;
 	unsigned pos = offset % 8;
@@ -102,7 +97,7 @@ static struct gpio_chip template_chip = {
 static int tc3589x_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -130,7 +125,7 @@ static int tc3589x_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 static void tc3589x_gpio_irq_lock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(gc);
 
 	mutex_lock(&tc3589x_gpio->irq_lock);
 }
@@ -138,7 +133,7 @@ static void tc3589x_gpio_irq_lock(struct irq_data *d)
 static void tc3589x_gpio_irq_sync_unlock(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(gc);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	static const u8 regmap[] = {
 		[REG_IBE]	= TC3589x_GPIOIBE0,
@@ -167,7 +162,7 @@ static void tc3589x_gpio_irq_sync_unlock(struct irq_data *d)
 static void tc3589x_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -178,7 +173,7 @@ static void tc3589x_gpio_irq_mask(struct irq_data *d)
 static void tc3589x_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc);
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = 1 << (offset % 8);
@@ -258,7 +253,7 @@ static int tc3589x_gpio_probe(struct platform_device *pdev)
 
 	tc3589x_gpio->chip = template_chip;
 	tc3589x_gpio->chip.ngpio = tc3589x->num_gpio;
-	tc3589x_gpio->chip.dev = &pdev->dev;
+	tc3589x_gpio->chip.parent = &pdev->dev;
 	tc3589x_gpio->chip.base = -1;
 	tc3589x_gpio->chip.of_node = np;
 
@@ -277,7 +272,7 @@ static int tc3589x_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	ret = gpiochip_add(&tc3589x_gpio->chip);
+	ret = gpiochip_add_data(&tc3589x_gpio->chip, tc3589x_gpio);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to add gpiochip: %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c
index 896bf29776b0..9a1a7e2ef388 100644
--- a/drivers/gpio/gpio-tegra.c
+++ b/drivers/gpio/gpio-tegra.c
@@ -545,7 +545,7 @@ static int tegra_gpio_probe(struct platform_device *pdev)
 
 	tegra_gpio_chip.of_node = pdev->dev.of_node;
 
-	ret = gpiochip_add(&tegra_gpio_chip);
+	ret = gpiochip_add_data(&tegra_gpio_chip, NULL);
 	if (ret < 0) {
 		irq_domain_remove(irq_domain);
 		return ret;
diff --git a/drivers/gpio/gpio-timberdale.c b/drivers/gpio/gpio-timberdale.c
index 30653e6319e9..a6de10c5275b 100644
--- a/drivers/gpio/gpio-timberdale.c
+++ b/drivers/gpio/gpio-timberdale.c
@@ -53,7 +53,7 @@ struct timbgpio {
 static int timbgpio_update_bit(struct gpio_chip *gpio, unsigned index,
 	unsigned offset, bool enabled)
 {
-	struct timbgpio *tgpio = container_of(gpio, struct timbgpio, gpio);
+	struct timbgpio *tgpio = gpiochip_get_data(gpio);
 	u32 reg;
 
 	spin_lock(&tgpio->lock);
@@ -77,7 +77,7 @@ static int timbgpio_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 
 static int timbgpio_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
-	struct timbgpio *tgpio = container_of(gpio, struct timbgpio, gpio);
+	struct timbgpio *tgpio = gpiochip_get_data(gpio);
 	u32 value;
 
 	value = ioread32(tgpio->membase + TGPIOVAL);
@@ -98,7 +98,7 @@ static void timbgpio_gpio_set(struct gpio_chip *gpio,
 
 static int timbgpio_to_irq(struct gpio_chip *gpio, unsigned offset)
 {
-	struct timbgpio *tgpio = container_of(gpio, struct timbgpio, gpio);
+	struct timbgpio *tgpio = gpiochip_get_data(gpio);
 
 	if (tgpio->irq_base <= 0)
 		return -EINVAL;
@@ -268,7 +268,7 @@ static int timbgpio_probe(struct platform_device *pdev)
 
 	gc->label = dev_name(&pdev->dev);
 	gc->owner = THIS_MODULE;
-	gc->dev = &pdev->dev;
+	gc->parent = &pdev->dev;
 	gc->direction_input = timbgpio_gpio_direction_input;
 	gc->get = timbgpio_gpio_get;
 	gc->direction_output = timbgpio_gpio_direction_output;
@@ -279,7 +279,7 @@ static int timbgpio_probe(struct platform_device *pdev)
 	gc->ngpio = pdata->nr_pins;
 	gc->can_sleep = false;
 
-	err = gpiochip_add(gc);
+	err = gpiochip_add_data(gc, tgpio);
 	if (err)
 		return err;
 
diff --git a/drivers/gpio/gpio-tps6586x.c b/drivers/gpio/gpio-tps6586x.c
index 9c9238e838a9..87de5486a29e 100644
--- a/drivers/gpio/gpio-tps6586x.c
+++ b/drivers/gpio/gpio-tps6586x.c
@@ -38,14 +38,9 @@ struct tps6586x_gpio {
 	struct device *parent;
 };
 
-static inline struct tps6586x_gpio *to_tps6586x_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct tps6586x_gpio, gpio_chip);
-}
-
 static int tps6586x_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps6586x_gpio *tps6586x_gpio = to_tps6586x_gpio(gc);
+	struct tps6586x_gpio *tps6586x_gpio = gpiochip_get_data(gc);
 	uint8_t val;
 	int ret;
 
@@ -59,7 +54,7 @@ static int tps6586x_gpio_get(struct gpio_chip *gc, unsigned offset)
 static void tps6586x_gpio_set(struct gpio_chip *gc, unsigned offset,
 			      int value)
 {
-	struct tps6586x_gpio *tps6586x_gpio = to_tps6586x_gpio(gc);
+	struct tps6586x_gpio *tps6586x_gpio = gpiochip_get_data(gc);
 
 	tps6586x_update(tps6586x_gpio->parent, TPS6586X_GPIOSET2,
 			value << offset, 1 << offset);
@@ -68,7 +63,7 @@ static void tps6586x_gpio_set(struct gpio_chip *gc, unsigned offset,
 static int tps6586x_gpio_output(struct gpio_chip *gc, unsigned offset,
 				int value)
 {
-	struct tps6586x_gpio *tps6586x_gpio = to_tps6586x_gpio(gc);
+	struct tps6586x_gpio *tps6586x_gpio = gpiochip_get_data(gc);
 	uint8_t val, mask;
 
 	tps6586x_gpio_set(gc, offset, value);
@@ -82,7 +77,7 @@ static int tps6586x_gpio_output(struct gpio_chip *gc, unsigned offset,
 
 static int tps6586x_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps6586x_gpio *tps6586x_gpio = to_tps6586x_gpio(gc);
+	struct tps6586x_gpio *tps6586x_gpio = gpiochip_get_data(gc);
 
 	return tps6586x_irq_get_virq(tps6586x_gpio->parent,
 				TPS6586X_INT_PLDO_0 + offset);
@@ -104,7 +99,7 @@ static int tps6586x_gpio_probe(struct platform_device *pdev)
 
 	tps6586x_gpio->gpio_chip.owner = THIS_MODULE;
 	tps6586x_gpio->gpio_chip.label = pdev->name;
-	tps6586x_gpio->gpio_chip.dev = &pdev->dev;
+	tps6586x_gpio->gpio_chip.parent = &pdev->dev;
 	tps6586x_gpio->gpio_chip.ngpio = 4;
 	tps6586x_gpio->gpio_chip.can_sleep = true;
 
@@ -122,7 +117,7 @@ static int tps6586x_gpio_probe(struct platform_device *pdev)
 	else
 		tps6586x_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&tps6586x_gpio->gpio_chip);
+	ret = gpiochip_add_data(&tps6586x_gpio->gpio_chip, tps6586x_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-tps65910.c b/drivers/gpio/gpio-tps65910.c
index 88f1f5ff4e96..e81eee7627a3 100644
--- a/drivers/gpio/gpio-tps65910.c
+++ b/drivers/gpio/gpio-tps65910.c
@@ -27,14 +27,9 @@ struct tps65910_gpio {
 	struct tps65910 *tps65910;
 };
 
-static inline struct tps65910_gpio *to_tps65910_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct tps65910_gpio, gpio_chip);
-}
-
 static int tps65910_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps65910_gpio *tps65910_gpio = to_tps65910_gpio(gc);
+	struct tps65910_gpio *tps65910_gpio = gpiochip_get_data(gc);
 	struct tps65910 *tps65910 = tps65910_gpio->tps65910;
 	unsigned int val;
 
@@ -49,7 +44,7 @@ static int tps65910_gpio_get(struct gpio_chip *gc, unsigned offset)
 static void tps65910_gpio_set(struct gpio_chip *gc, unsigned offset,
 			      int value)
 {
-	struct tps65910_gpio *tps65910_gpio = to_tps65910_gpio(gc);
+	struct tps65910_gpio *tps65910_gpio = gpiochip_get_data(gc);
 	struct tps65910 *tps65910 = tps65910_gpio->tps65910;
 
 	if (value)
@@ -63,7 +58,7 @@ static void tps65910_gpio_set(struct gpio_chip *gc, unsigned offset,
 static int tps65910_gpio_output(struct gpio_chip *gc, unsigned offset,
 				int value)
 {
-	struct tps65910_gpio *tps65910_gpio = to_tps65910_gpio(gc);
+	struct tps65910_gpio *tps65910_gpio = gpiochip_get_data(gc);
 	struct tps65910 *tps65910 = tps65910_gpio->tps65910;
 
 	/* Set the initial value */
@@ -75,7 +70,7 @@ static int tps65910_gpio_output(struct gpio_chip *gc, unsigned offset,
 
 static int tps65910_gpio_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps65910_gpio *tps65910_gpio = to_tps65910_gpio(gc);
+	struct tps65910_gpio *tps65910_gpio = gpiochip_get_data(gc);
 	struct tps65910 *tps65910 = tps65910_gpio->tps65910;
 
 	return tps65910_reg_clear_bits(tps65910, TPS65910_GPIO0 + offset,
@@ -146,7 +141,7 @@ static int tps65910_gpio_probe(struct platform_device *pdev)
 	tps65910_gpio->gpio_chip.direction_output = tps65910_gpio_output;
 	tps65910_gpio->gpio_chip.set	= tps65910_gpio_set;
 	tps65910_gpio->gpio_chip.get	= tps65910_gpio_get;
-	tps65910_gpio->gpio_chip.dev = &pdev->dev;
+	tps65910_gpio->gpio_chip.parent = &pdev->dev;
 #ifdef CONFIG_OF_GPIO
 	tps65910_gpio->gpio_chip.of_node = tps65910->dev->of_node;
 #endif
@@ -175,7 +170,7 @@ static int tps65910_gpio_probe(struct platform_device *pdev)
 	}
 
 skip_init:
-	ret = gpiochip_add(&tps65910_gpio->gpio_chip);
+	ret = gpiochip_add_data(&tps65910_gpio->gpio_chip, tps65910_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-tps65912.c b/drivers/gpio/gpio-tps65912.c
index 9cdbc0c9cb2d..4f2029c7da3a 100644
--- a/drivers/gpio/gpio-tps65912.c
+++ b/drivers/gpio/gpio-tps65912.c
@@ -26,11 +26,9 @@ struct tps65912_gpio_data {
 	struct gpio_chip gpio_chip;
 };
 
-#define to_tgd(gc) container_of(gc, struct tps65912_gpio_data, gpio_chip)
-
 static int tps65912_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps65912_gpio_data *tps65912_gpio = to_tgd(gc);
+	struct tps65912_gpio_data *tps65912_gpio = gpiochip_get_data(gc);
 	struct tps65912 *tps65912 = tps65912_gpio->tps65912;
 	int val;
 
@@ -45,7 +43,7 @@ static int tps65912_gpio_get(struct gpio_chip *gc, unsigned offset)
 static void tps65912_gpio_set(struct gpio_chip *gc, unsigned offset,
 			      int value)
 {
-	struct tps65912_gpio_data *tps65912_gpio = to_tgd(gc);
+	struct tps65912_gpio_data *tps65912_gpio = gpiochip_get_data(gc);
 	struct tps65912 *tps65912 = tps65912_gpio->tps65912;
 
 	if (value)
@@ -59,7 +57,7 @@ static void tps65912_gpio_set(struct gpio_chip *gc, unsigned offset,
 static int tps65912_gpio_output(struct gpio_chip *gc, unsigned offset,
 				int value)
 {
-	struct tps65912_gpio_data *tps65912_gpio = to_tgd(gc);
+	struct tps65912_gpio_data *tps65912_gpio = gpiochip_get_data(gc);
 	struct tps65912 *tps65912 = tps65912_gpio->tps65912;
 
 	/* Set the initial value */
@@ -71,7 +69,7 @@ static int tps65912_gpio_output(struct gpio_chip *gc, unsigned offset,
 
 static int tps65912_gpio_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct tps65912_gpio_data *tps65912_gpio = to_tgd(gc);
+	struct tps65912_gpio_data *tps65912_gpio = gpiochip_get_data(gc);
 	struct tps65912 *tps65912 = tps65912_gpio->tps65912;
 
 	return tps65912_clear_bits(tps65912, TPS65912_GPIO1 + offset,
@@ -104,11 +102,11 @@ static int tps65912_gpio_probe(struct platform_device *pdev)
 
 	tps65912_gpio->tps65912 = tps65912;
 	tps65912_gpio->gpio_chip = template_chip;
-	tps65912_gpio->gpio_chip.dev = &pdev->dev;
+	tps65912_gpio->gpio_chip.parent = &pdev->dev;
 	if (pdata && pdata->gpio_base)
 		tps65912_gpio->gpio_chip.base = pdata->gpio_base;
 
-	ret = gpiochip_add(&tps65912_gpio->gpio_chip);
+	ret = gpiochip_add_data(&tps65912_gpio->gpio_chip, tps65912_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-ts5500.c b/drivers/gpio/gpio-ts5500.c
index b29a102d136b..5f945083f9d8 100644
--- a/drivers/gpio/gpio-ts5500.c
+++ b/drivers/gpio/gpio-ts5500.c
@@ -185,11 +185,6 @@ static const struct ts5500_dio ts5500_lcd[] = {
 	TS5500_DIO_IN_IRQ(0x73, 7, 1),
 };
 
-static inline struct ts5500_priv *ts5500_gc_to_priv(struct gpio_chip *chip)
-{
-	return container_of(chip, struct ts5500_priv, gpio_chip);
-}
-
 static inline void ts5500_set_mask(u8 mask, u8 addr)
 {
 	u8 val = inb(addr);
@@ -206,7 +201,7 @@ static inline void ts5500_clear_mask(u8 mask, u8 addr)
 
 static int ts5500_gpio_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct ts5500_priv *priv = ts5500_gc_to_priv(chip);
+	struct ts5500_priv *priv = gpiochip_get_data(chip);
 	const struct ts5500_dio line = priv->pinout[offset];
 	unsigned long flags;
 
@@ -225,7 +220,7 @@ static int ts5500_gpio_input(struct gpio_chip *chip, unsigned offset)
 
 static int ts5500_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct ts5500_priv *priv = ts5500_gc_to_priv(chip);
+	struct ts5500_priv *priv = gpiochip_get_data(chip);
 	const struct ts5500_dio line = priv->pinout[offset];
 
 	return !!(inb(line.value_addr) & line.value_mask);
@@ -233,7 +228,7 @@ static int ts5500_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static int ts5500_gpio_output(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct ts5500_priv *priv = ts5500_gc_to_priv(chip);
+	struct ts5500_priv *priv = gpiochip_get_data(chip);
 	const struct ts5500_dio line = priv->pinout[offset];
 	unsigned long flags;
 
@@ -255,7 +250,7 @@ static int ts5500_gpio_output(struct gpio_chip *chip, unsigned offset, int val)
 
 static void ts5500_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct ts5500_priv *priv = ts5500_gc_to_priv(chip);
+	struct ts5500_priv *priv = gpiochip_get_data(chip);
 	const struct ts5500_dio line = priv->pinout[offset];
 	unsigned long flags;
 
@@ -269,7 +264,7 @@ static void ts5500_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 
 static int ts5500_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct ts5500_priv *priv = ts5500_gc_to_priv(chip);
+	struct ts5500_priv *priv = gpiochip_get_data(chip);
 	const struct ts5500_dio *block = priv->pinout;
 	const struct ts5500_dio line = block[offset];
 
@@ -315,7 +310,8 @@ static void ts5500_disable_irq(struct ts5500_priv *priv)
 	else if (priv->hwirq == 1)
 		ts5500_clear_mask(BIT(6), 0x7d); /* LCD_RS on IRQ1 */
 	else
-		dev_err(priv->gpio_chip.dev, "invalid hwirq %d\n", priv->hwirq);
+		dev_err(priv->gpio_chip.parent, "invalid hwirq %d\n",
+			priv->hwirq);
 	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
@@ -346,7 +342,7 @@ static int ts5500_dio_probe(struct platform_device *pdev)
 
 	priv->gpio_chip.owner = THIS_MODULE;
 	priv->gpio_chip.label = name;
-	priv->gpio_chip.dev = dev;
+	priv->gpio_chip.parent = dev;
 	priv->gpio_chip.direction_input = ts5500_gpio_input;
 	priv->gpio_chip.direction_output = ts5500_gpio_output;
 	priv->gpio_chip.get = ts5500_gpio_get;
@@ -413,7 +409,7 @@ static int ts5500_dio_probe(struct platform_device *pdev)
 		break;
 	}
 
-	ret = gpiochip_add(&priv->gpio_chip);
+	ret = gpiochip_add_data(&priv->gpio_chip, priv);
 	if (ret) {
 		dev_err(dev, "failed to register the gpio chip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-twl4030.c b/drivers/gpio/gpio-twl4030.c
index 9e1dbb9877c1..4b807b0e0c8e 100644
--- a/drivers/gpio/gpio-twl4030.c
+++ b/drivers/gpio/gpio-twl4030.c
@@ -76,11 +76,6 @@ struct gpio_twl4030_priv {
 
 /*----------------------------------------------------------------------*/
 
-static inline struct gpio_twl4030_priv *to_gpio_twl4030(struct gpio_chip *chip)
-{
-	return container_of(chip, struct gpio_twl4030_priv, gpio_chip);
-}
-
 /*
  * To configure TWL4030 GPIO module registers
  */
@@ -205,7 +200,7 @@ static int twl4030_get_gpio_datain(int gpio)
 
 static int twl_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 	int status = 0;
 
 	mutex_lock(&priv->mutex);
@@ -256,7 +251,7 @@ static int twl_request(struct gpio_chip *chip, unsigned offset)
 		/* optionally have the first two GPIOs switch vMMC1
 		 * and vMMC2 power supplies based on card presence.
 		 */
-		pdata = dev_get_platdata(chip->dev);
+		pdata = dev_get_platdata(chip->parent);
 		if (pdata)
 			value |= pdata->mmc_cd & 0x03;
 
@@ -273,7 +268,7 @@ done:
 
 static void twl_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 
 	mutex_lock(&priv->mutex);
 	if (offset >= TWL4030_GPIO_MAX) {
@@ -293,7 +288,7 @@ out:
 
 static int twl_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 	int ret;
 
 	mutex_lock(&priv->mutex);
@@ -312,7 +307,7 @@ static int twl_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int twl_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 	int ret;
 	int status = 0;
 
@@ -327,7 +322,7 @@ static int twl_get(struct gpio_chip *chip, unsigned offset)
 	else
 		status = twl4030_get_gpio_datain(offset);
 
-	ret = (status <= 0) ? 0 : 1;
+	ret = (status < 0) ? status : !!status;
 out:
 	mutex_unlock(&priv->mutex);
 	return ret;
@@ -335,7 +330,7 @@ out:
 
 static void twl_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 
 	mutex_lock(&priv->mutex);
 	if (offset < TWL4030_GPIO_MAX)
@@ -353,7 +348,7 @@ static void twl_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int twl_direction_out(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 	int ret = 0;
 
 	mutex_lock(&priv->mutex);
@@ -379,7 +374,7 @@ static int twl_direction_out(struct gpio_chip *chip, unsigned offset, int value)
 
 static int twl_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_twl4030_priv *priv = to_gpio_twl4030(chip);
+	struct gpio_twl4030_priv *priv = gpiochip_get_data(chip);
 
 	return (priv->irq_base && (offset < TWL4030_GPIO_MAX))
 		? (priv->irq_base + offset)
@@ -509,7 +504,7 @@ no_irqs:
 	priv->gpio_chip = template_chip;
 	priv->gpio_chip.base = -1;
 	priv->gpio_chip.ngpio = TWL4030_GPIO_MAX;
-	priv->gpio_chip.dev = &pdev->dev;
+	priv->gpio_chip.parent = &pdev->dev;
 
 	mutex_init(&priv->mutex);
 
@@ -544,7 +539,7 @@ no_irqs:
 	if (pdata->use_leds)
 		priv->gpio_chip.ngpio += 2;
 
-	ret = gpiochip_add(&priv->gpio_chip);
+	ret = gpiochip_add_data(&priv->gpio_chip, priv);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "could not register gpiochip, %d\n", ret);
 		priv->gpio_chip.ngpio = 0;
diff --git a/drivers/gpio/gpio-twl6040.c b/drivers/gpio/gpio-twl6040.c
index c946e7eef3ee..8e9e9853f3bd 100644
--- a/drivers/gpio/gpio-twl6040.c
+++ b/drivers/gpio/gpio-twl6040.c
@@ -36,7 +36,7 @@ static struct gpio_chip twl6040gpo_chip;
 
 static int twl6040gpo_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct twl6040 *twl6040 = dev_get_drvdata(chip->dev->parent);
+	struct twl6040 *twl6040 = dev_get_drvdata(chip->parent->parent);
 	int ret = 0;
 
 	ret = twl6040_reg_read(twl6040, TWL6040_REG_GPOCTL);
@@ -55,7 +55,7 @@ static int twl6040gpo_direction_out(struct gpio_chip *chip, unsigned offset,
 
 static void twl6040gpo_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct twl6040 *twl6040 = dev_get_drvdata(chip->dev->parent);
+	struct twl6040 *twl6040 = dev_get_drvdata(chip->parent->parent);
 	int ret;
 	u8 gpoctl;
 
@@ -95,12 +95,12 @@ static int gpo_twl6040_probe(struct platform_device *pdev)
 	else
 		twl6040gpo_chip.ngpio = 1; /* twl6041 have 1 GPO */
 
-	twl6040gpo_chip.dev = &pdev->dev;
+	twl6040gpo_chip.parent = &pdev->dev;
 #ifdef CONFIG_OF_GPIO
 	twl6040gpo_chip.of_node = twl6040_core_dev->of_node;
 #endif
 
-	ret = gpiochip_add(&twl6040gpo_chip);
+	ret = gpiochip_add_data(&twl6040gpo_chip, NULL);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "could not register gpiochip, %d\n", ret);
 		twl6040gpo_chip.ngpio = 0;
diff --git a/drivers/gpio/gpio-tz1090-pdc.c b/drivers/gpio/gpio-tz1090-pdc.c
index 3623d009d808..5b7781741ee9 100644
--- a/drivers/gpio/gpio-tz1090-pdc.c
+++ b/drivers/gpio/gpio-tz1090-pdc.c
@@ -49,7 +49,6 @@ struct tz1090_pdc_gpio {
 	void __iomem *reg;
 	int irq[GPIO_PDC_NIRQ];
 };
-#define to_pdc(c)	container_of(c, struct tz1090_pdc_gpio, chip)
 
 /* Register accesses into the PDC MMIO area */
 
@@ -70,7 +69,7 @@ static inline unsigned int pdc_read(struct tz1090_pdc_gpio *priv,
 static int tz1090_pdc_gpio_direction_input(struct gpio_chip *chip,
 					   unsigned int offset)
 {
-	struct tz1090_pdc_gpio *priv = to_pdc(chip);
+	struct tz1090_pdc_gpio *priv = gpiochip_get_data(chip);
 	u32 value;
 	int lstat;
 
@@ -87,7 +86,7 @@ static int tz1090_pdc_gpio_direction_output(struct gpio_chip *chip,
 					    unsigned int offset,
 					    int output_value)
 {
-	struct tz1090_pdc_gpio *priv = to_pdc(chip);
+	struct tz1090_pdc_gpio *priv = gpiochip_get_data(chip);
 	u32 value;
 	int lstat;
 
@@ -112,14 +111,14 @@ static int tz1090_pdc_gpio_direction_output(struct gpio_chip *chip,
 
 static int tz1090_pdc_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_pdc_gpio *priv = to_pdc(chip);
-	return pdc_read(priv, REG_SOC_GPIO_STATUS) & BIT(offset);
+	struct tz1090_pdc_gpio *priv = gpiochip_get_data(chip);
+	return !!(pdc_read(priv, REG_SOC_GPIO_STATUS) & BIT(offset));
 }
 
 static void tz1090_pdc_gpio_set(struct gpio_chip *chip, unsigned int offset,
 				int output_value)
 {
-	struct tz1090_pdc_gpio *priv = to_pdc(chip);
+	struct tz1090_pdc_gpio *priv = gpiochip_get_data(chip);
 	u32 value;
 	int lstat;
 
@@ -139,7 +138,7 @@ static void tz1090_pdc_gpio_set(struct gpio_chip *chip, unsigned int offset,
 
 static int tz1090_pdc_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_pdc_gpio *priv = to_pdc(chip);
+	struct tz1090_pdc_gpio *priv = gpiochip_get_data(chip);
 	unsigned int syswake = offset - GPIO_PDC_IRQ_FIRST;
 	int irq;
 
@@ -188,7 +187,7 @@ static int tz1090_pdc_gpio_probe(struct platform_device *pdev)
 
 	/* Set up GPIO chip */
 	priv->chip.label		= "tz1090-pdc-gpio";
-	priv->chip.dev			= &pdev->dev;
+	priv->chip.parent		= &pdev->dev;
 	priv->chip.direction_input	= tz1090_pdc_gpio_direction_input;
 	priv->chip.direction_output	= tz1090_pdc_gpio_direction_output;
 	priv->chip.get			= tz1090_pdc_gpio_get;
@@ -207,7 +206,7 @@ static int tz1090_pdc_gpio_probe(struct platform_device *pdev)
 		priv->irq[i] = irq_of_parse_and_map(np, i);
 
 	/* Add the GPIO bank */
-	gpiochip_add(&priv->chip);
+	gpiochip_add_data(&priv->chip, priv);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-tz1090.c b/drivers/gpio/gpio-tz1090.c
index 87bb1b1eee8d..ca958e0f6909 100644
--- a/drivers/gpio/gpio-tz1090.c
+++ b/drivers/gpio/gpio-tz1090.c
@@ -62,7 +62,6 @@ struct tz1090_gpio_bank {
 	int irq;
 	char label[16];
 };
-#define to_bank(c)	container_of(c, struct tz1090_gpio_bank, chip)
 
 /**
  * struct tz1090_gpio - Overall GPIO device private data
@@ -187,7 +186,7 @@ static inline int tz1090_gpio_read_bit(struct tz1090_gpio_bank *bank,
 static int tz1090_gpio_direction_input(struct gpio_chip *chip,
 				       unsigned int offset)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 	tz1090_gpio_set_bit(bank, REG_GPIO_DIR, offset);
 
 	return 0;
@@ -196,7 +195,7 @@ static int tz1090_gpio_direction_input(struct gpio_chip *chip,
 static int tz1090_gpio_direction_output(struct gpio_chip *chip,
 					unsigned int offset, int output_value)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 	int lstat;
 
 	__global_lock2(lstat);
@@ -212,9 +211,9 @@ static int tz1090_gpio_direction_output(struct gpio_chip *chip,
  */
 static int tz1090_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 
-	return tz1090_gpio_read_bit(bank, REG_GPIO_DIN, offset);
+	return !!tz1090_gpio_read_bit(bank, REG_GPIO_DIN, offset);
 }
 
 /*
@@ -223,14 +222,14 @@ static int tz1090_gpio_get(struct gpio_chip *chip, unsigned int offset)
 static void tz1090_gpio_set(struct gpio_chip *chip, unsigned int offset,
 			    int output_value)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 
 	tz1090_gpio_mod_bit(bank, REG_GPIO_DOUT, offset, output_value);
 }
 
 static int tz1090_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 	int ret;
 
 	ret = pinctrl_request_gpio(chip->base + offset);
@@ -245,7 +244,7 @@ static int tz1090_gpio_request(struct gpio_chip *chip, unsigned int offset)
 
 static void tz1090_gpio_free(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 
 	pinctrl_free_gpio(chip->base + offset);
 
@@ -254,7 +253,7 @@ static void tz1090_gpio_free(struct gpio_chip *chip, unsigned int offset)
 
 static int tz1090_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
 {
-	struct tz1090_gpio_bank *bank = to_bank(chip);
+	struct tz1090_gpio_bank *bank = gpiochip_get_data(chip);
 
 	if (!bank->domain)
 		return -EINVAL;
@@ -425,7 +424,7 @@ static int tz1090_gpio_bank_probe(struct tz1090_gpio_bank_info *info)
 	snprintf(bank->label, sizeof(bank->label), "tz1090-gpio-%u",
 		 info->index);
 	bank->chip.label		= bank->label;
-	bank->chip.dev			= dev;
+	bank->chip.parent		= dev;
 	bank->chip.direction_input	= tz1090_gpio_direction_input;
 	bank->chip.direction_output	= tz1090_gpio_direction_output;
 	bank->chip.get			= tz1090_gpio_get;
@@ -440,7 +439,7 @@ static int tz1090_gpio_bank_probe(struct tz1090_gpio_bank_info *info)
 	bank->chip.ngpio		= 30;
 
 	/* Add the GPIO bank */
-	gpiochip_add(&bank->chip);
+	gpiochip_add_data(&bank->chip, bank);
 
 	/* Get the GPIO bank IRQ if provided */
 	bank->irq = irq_of_parse_and_map(np, 0);
diff --git a/drivers/gpio/gpio-ucb1400.c b/drivers/gpio/gpio-ucb1400.c
index d502825159b9..2c5cd46bfa6e 100644
--- a/drivers/gpio/gpio-ucb1400.c
+++ b/drivers/gpio/gpio-ucb1400.c
@@ -15,7 +15,7 @@
 static int ucb1400_gpio_dir_in(struct gpio_chip *gc, unsigned off)
 {
 	struct ucb1400_gpio *gpio;
-	gpio = container_of(gc, struct ucb1400_gpio, gc);
+	gpio = gpiochip_get_data(gc);
 	ucb1400_gpio_set_direction(gpio->ac97, off, 0);
 	return 0;
 }
@@ -23,7 +23,7 @@ static int ucb1400_gpio_dir_in(struct gpio_chip *gc, unsigned off)
 static int ucb1400_gpio_dir_out(struct gpio_chip *gc, unsigned off, int val)
 {
 	struct ucb1400_gpio *gpio;
-	gpio = container_of(gc, struct ucb1400_gpio, gc);
+	gpio = gpiochip_get_data(gc);
 	ucb1400_gpio_set_direction(gpio->ac97, off, 1);
 	ucb1400_gpio_set_value(gpio->ac97, off, val);
 	return 0;
@@ -32,14 +32,15 @@ static int ucb1400_gpio_dir_out(struct gpio_chip *gc, unsigned off, int val)
 static int ucb1400_gpio_get(struct gpio_chip *gc, unsigned off)
 {
 	struct ucb1400_gpio *gpio;
-	gpio = container_of(gc, struct ucb1400_gpio, gc);
-	return ucb1400_gpio_get_value(gpio->ac97, off);
+
+	gpio = gpiochip_get_data(gc);
+	return !!ucb1400_gpio_get_value(gpio->ac97, off);
 }
 
 static void ucb1400_gpio_set(struct gpio_chip *gc, unsigned off, int val)
 {
 	struct ucb1400_gpio *gpio;
-	gpio = container_of(gc, struct ucb1400_gpio, gc);
+	gpio = gpiochip_get_data(gc);
 	ucb1400_gpio_set_value(gpio->ac97, off, val);
 }
 
@@ -66,7 +67,7 @@ static int ucb1400_gpio_probe(struct platform_device *dev)
 	ucb->gc.set = ucb1400_gpio_set;
 	ucb->gc.can_sleep = true;
 
-	err = gpiochip_add(&ucb->gc);
+	err = gpiochip_add_data(&ucb->gc, ucb);
 	if (err)
 		goto err;
 
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 87b950cec6ec..6284bdbe1e0c 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -62,11 +62,6 @@ struct vf610_gpio_port {
 
 static struct irq_chip vf610_gpio_irq_chip;
 
-static struct vf610_gpio_port *to_vf610_gp(struct gpio_chip *gc)
-{
-	return container_of(gc, struct vf610_gpio_port, gc);
-}
-
 static const struct of_device_id vf610_gpio_dt_ids[] = {
 	{ .compatible = "fsl,vf610-gpio" },
 	{ /* sentinel */ }
@@ -84,14 +79,14 @@ static inline u32 vf610_gpio_readl(void __iomem *reg)
 
 static int vf610_gpio_get(struct gpio_chip *gc, unsigned int gpio)
 {
-	struct vf610_gpio_port *port = to_vf610_gp(gc);
+	struct vf610_gpio_port *port = gpiochip_get_data(gc);
 
 	return !!(vf610_gpio_readl(port->gpio_base + GPIO_PDIR) & BIT(gpio));
 }
 
 static void vf610_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
-	struct vf610_gpio_port *port = to_vf610_gp(gc);
+	struct vf610_gpio_port *port = gpiochip_get_data(gc);
 	unsigned long mask = BIT(gpio);
 
 	if (val)
@@ -116,7 +111,7 @@ static int vf610_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 static void vf610_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_desc_get_handler_data(desc));
+		gpiochip_get_data(irq_desc_get_handler_data(desc));
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	int pin;
 	unsigned long irq_isfr;
@@ -137,7 +132,7 @@ static void vf610_gpio_irq_handler(struct irq_desc *desc)
 static void vf610_gpio_irq_ack(struct irq_data *d)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	int gpio = d->hwirq;
 
 	vf610_gpio_writel(BIT(gpio), port->base + PORT_ISFR);
@@ -146,7 +141,7 @@ static void vf610_gpio_irq_ack(struct irq_data *d)
 static int vf610_gpio_irq_set_type(struct irq_data *d, u32 type)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	u8 irqc;
 
 	switch (type) {
@@ -182,7 +177,7 @@ static int vf610_gpio_irq_set_type(struct irq_data *d, u32 type)
 static void vf610_gpio_irq_mask(struct irq_data *d)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	void __iomem *pcr_base = port->base + PORT_PCR(d->hwirq);
 
 	vf610_gpio_writel(0, pcr_base);
@@ -191,7 +186,7 @@ static void vf610_gpio_irq_mask(struct irq_data *d)
 static void vf610_gpio_irq_unmask(struct irq_data *d)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	void __iomem *pcr_base = port->base + PORT_PCR(d->hwirq);
 
 	vf610_gpio_writel(port->irqc[d->hwirq] << PORT_PCR_IRQC_OFFSET,
@@ -201,7 +196,7 @@ static void vf610_gpio_irq_unmask(struct irq_data *d)
 static int vf610_gpio_irq_set_wake(struct irq_data *d, u32 enable)
 {
 	struct vf610_gpio_port *port =
-		to_vf610_gp(irq_data_get_irq_chip_data(d));
+		gpiochip_get_data(irq_data_get_irq_chip_data(d));
 
 	if (enable)
 		enable_irq_wake(port->irq);
@@ -249,7 +244,7 @@ static int vf610_gpio_probe(struct platform_device *pdev)
 
 	gc = &port->gc;
 	gc->of_node = np;
-	gc->dev = dev;
+	gc->parent = dev;
 	gc->label = "vf610-gpio";
 	gc->ngpio = VF610_GPIO_PER_PORT;
 	gc->base = of_alias_get_id(np, "gpio") * VF610_GPIO_PER_PORT;
@@ -261,7 +256,7 @@ static int vf610_gpio_probe(struct platform_device *pdev)
 	gc->direction_output = vf610_gpio_direction_output;
 	gc->set = vf610_gpio_set;
 
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, port);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpio/gpio-viperboard.c b/drivers/gpio/gpio-viperboard.c
index e2a11f27807f..1170b035cb92 100644
--- a/drivers/gpio/gpio-viperboard.c
+++ b/drivers/gpio/gpio-viperboard.c
@@ -88,14 +88,13 @@ static int vprbrd_gpioa_get(struct gpio_chip *chip,
 		unsigned offset)
 {
 	int ret, answer, error = 0;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpioa);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpioa_msg *gamsg = (struct vprbrd_gpioa_msg *)vb->buf;
 
 	/* if io is set to output, just return the saved value */
 	if (gpio->gpioa_out & (1 << offset))
-		return gpio->gpioa_val & (1 << offset);
+		return !!(gpio->gpioa_val & (1 << offset));
 
 	mutex_lock(&vb->lock);
 
@@ -139,8 +138,7 @@ static void vprbrd_gpioa_set(struct gpio_chip *chip,
 		unsigned offset, int value)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpioa);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpioa_msg *gamsg = (struct vprbrd_gpioa_msg *)vb->buf;
 
@@ -173,7 +171,7 @@ static void vprbrd_gpioa_set(struct gpio_chip *chip,
 		mutex_unlock(&vb->lock);
 
 		if (ret != sizeof(struct vprbrd_gpioa_msg))
-			dev_err(chip->dev, "usb error setting pin value\n");
+			dev_err(chip->parent, "usb error setting pin value\n");
 	}
 }
 
@@ -181,8 +179,7 @@ static int vprbrd_gpioa_direction_input(struct gpio_chip *chip,
 			unsigned offset)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpioa);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpioa_msg *gamsg = (struct vprbrd_gpioa_msg *)vb->buf;
 
@@ -219,8 +216,7 @@ static int vprbrd_gpioa_direction_output(struct gpio_chip *chip,
 			unsigned offset, int value)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpioa);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpioa_msg *gamsg = (struct vprbrd_gpioa_msg *)vb->buf;
 
@@ -287,8 +283,7 @@ static int vprbrd_gpiob_get(struct gpio_chip *chip,
 {
 	int ret;
 	u16 val;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpiob);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpiob_msg *gbmsg = (struct vprbrd_gpiob_msg *)vb->buf;
 
@@ -319,8 +314,7 @@ static void vprbrd_gpiob_set(struct gpio_chip *chip,
 		unsigned offset, int value)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpiob);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 	struct vprbrd_gpiob_msg *gbmsg = (struct vprbrd_gpiob_msg *)vb->buf;
 
@@ -345,7 +339,7 @@ static void vprbrd_gpiob_set(struct gpio_chip *chip,
 		mutex_unlock(&vb->lock);
 
 		if (ret != sizeof(struct vprbrd_gpiob_msg))
-			dev_err(chip->dev, "usb error setting pin value\n");
+			dev_err(chip->parent, "usb error setting pin value\n");
 	}
 }
 
@@ -353,8 +347,7 @@ static int vprbrd_gpiob_direction_input(struct gpio_chip *chip,
 			unsigned offset)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpiob);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 
 	gpio->gpiob_out &= ~(1 << offset);
@@ -366,7 +359,7 @@ static int vprbrd_gpiob_direction_input(struct gpio_chip *chip,
 	mutex_unlock(&vb->lock);
 
 	if (ret)
-		dev_err(chip->dev, "usb error setting pin to input\n");
+		dev_err(chip->parent, "usb error setting pin to input\n");
 
 	return ret;
 }
@@ -375,8 +368,7 @@ static int vprbrd_gpiob_direction_output(struct gpio_chip *chip,
 			unsigned offset, int value)
 {
 	int ret;
-	struct vprbrd_gpio *gpio =
-			container_of(chip, struct vprbrd_gpio, gpiob);
+	struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
 	struct vprbrd *vb = gpio->vb;
 
 	gpio->gpiob_out |= (1 << offset);
@@ -385,7 +377,7 @@ static int vprbrd_gpiob_direction_output(struct gpio_chip *chip,
 
 	ret = vprbrd_gpiob_setdir(vb, offset, 1);
 	if (ret)
-		dev_err(chip->dev, "usb error setting pin to output\n");
+		dev_err(chip->parent, "usb error setting pin to output\n");
 
 	mutex_unlock(&vb->lock);
 
@@ -409,7 +401,7 @@ static int vprbrd_gpio_probe(struct platform_device *pdev)
 	vb_gpio->vb = vb;
 	/* registering gpio a */
 	vb_gpio->gpioa.label = "viperboard gpio a";
-	vb_gpio->gpioa.dev = &pdev->dev;
+	vb_gpio->gpioa.parent = &pdev->dev;
 	vb_gpio->gpioa.owner = THIS_MODULE;
 	vb_gpio->gpioa.base = -1;
 	vb_gpio->gpioa.ngpio = 16;
@@ -418,15 +410,15 @@ static int vprbrd_gpio_probe(struct platform_device *pdev)
 	vb_gpio->gpioa.get = vprbrd_gpioa_get;
 	vb_gpio->gpioa.direction_input = vprbrd_gpioa_direction_input;
 	vb_gpio->gpioa.direction_output = vprbrd_gpioa_direction_output;
-	ret = gpiochip_add(&vb_gpio->gpioa);
+	ret = gpiochip_add_data(&vb_gpio->gpioa, vb_gpio);
 	if (ret < 0) {
-		dev_err(vb_gpio->gpioa.dev, "could not add gpio a");
+		dev_err(vb_gpio->gpioa.parent, "could not add gpio a");
 		goto err_gpioa;
 	}
 
 	/* registering gpio b */
 	vb_gpio->gpiob.label = "viperboard gpio b";
-	vb_gpio->gpiob.dev = &pdev->dev;
+	vb_gpio->gpiob.parent = &pdev->dev;
 	vb_gpio->gpiob.owner = THIS_MODULE;
 	vb_gpio->gpiob.base = -1;
 	vb_gpio->gpiob.ngpio = 16;
@@ -435,9 +427,9 @@ static int vprbrd_gpio_probe(struct platform_device *pdev)
 	vb_gpio->gpiob.get = vprbrd_gpiob_get;
 	vb_gpio->gpiob.direction_input = vprbrd_gpiob_direction_input;
 	vb_gpio->gpiob.direction_output = vprbrd_gpiob_direction_output;
-	ret = gpiochip_add(&vb_gpio->gpiob);
+	ret = gpiochip_add_data(&vb_gpio->gpiob, vb_gpio);
 	if (ret < 0) {
-		dev_err(vb_gpio->gpiob.dev, "could not add gpio b");
+		dev_err(vb_gpio->gpiob.parent, "could not add gpio b");
 		goto err_gpiob;
 	}
 
diff --git a/drivers/gpio/gpio-vr41xx.c b/drivers/gpio/gpio-vr41xx.c
index c1caa459c02d..ac8deb01f6f6 100644
--- a/drivers/gpio/gpio-vr41xx.c
+++ b/drivers/gpio/gpio-vr41xx.c
@@ -139,7 +139,7 @@ static void unmask_giuint_low(struct irq_data *d)
 static unsigned int startup_giuint(struct irq_data *data)
 {
 	if (gpiochip_lock_as_irq(&vr41xx_gpio_chip, data->hwirq))
-		dev_err(vr41xx_gpio_chip.dev,
+		dev_err(vr41xx_gpio_chip.parent,
 			"unable to lock HW IRQ %lu for IRQ\n",
 			data->hwirq);
 	/* Satisfy the .enable semantics by unmasking the line */
@@ -542,9 +542,9 @@ static int giu_probe(struct platform_device *pdev)
 	if (!giu_base)
 		return -ENOMEM;
 
-	vr41xx_gpio_chip.dev = &pdev->dev;
+	vr41xx_gpio_chip.parent = &pdev->dev;
 
-	ret = gpiochip_add(&vr41xx_gpio_chip);
+	ret = gpiochip_add_data(&vr41xx_gpio_chip, NULL);
 	if (!ret) {
 		iounmap(giu_base);
 		return -ENODEV;
diff --git a/drivers/gpio/gpio-vx855.c b/drivers/gpio/gpio-vx855.c
index 57b470d5b39e..764999cc0794 100644
--- a/drivers/gpio/gpio-vx855.c
+++ b/drivers/gpio/gpio-vx855.c
@@ -96,7 +96,7 @@ static inline u_int32_t gpio_o_bit(int i)
 static int vx855gpio_direction_input(struct gpio_chip *gpio,
 				     unsigned int nr)
 {
-	struct vx855_gpio *vg = container_of(gpio, struct vx855_gpio, gpio);
+	struct vx855_gpio *vg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u_int32_t reg_out;
 
@@ -120,7 +120,7 @@ static int vx855gpio_direction_input(struct gpio_chip *gpio,
 
 static int vx855gpio_get(struct gpio_chip *gpio, unsigned int nr)
 {
-	struct vx855_gpio *vg = container_of(gpio, struct vx855_gpio, gpio);
+	struct vx855_gpio *vg = gpiochip_get_data(gpio);
 	u_int32_t reg_in;
 	int ret = 0;
 
@@ -146,7 +146,7 @@ static int vx855gpio_get(struct gpio_chip *gpio, unsigned int nr)
 static void vx855gpio_set(struct gpio_chip *gpio, unsigned int nr,
 			  int val)
 {
-	struct vx855_gpio *vg = container_of(gpio, struct vx855_gpio, gpio);
+	struct vx855_gpio *vg = gpiochip_get_data(gpio);
 	unsigned long flags;
 	u_int32_t reg_out;
 
@@ -259,7 +259,7 @@ static int vx855gpio_probe(struct platform_device *pdev)
 
 	vx855gpio_gpio_setup(vg);
 
-	return gpiochip_add(&vg->gpio);
+	return gpiochip_add_data(&vg->gpio, vg);
 }
 
 static int vx855gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-wm831x.c b/drivers/gpio/gpio-wm831x.c
index 58ce75c188b7..98390070fb64 100644
--- a/drivers/gpio/gpio-wm831x.c
+++ b/drivers/gpio/gpio-wm831x.c
@@ -30,14 +30,9 @@ struct wm831x_gpio {
 	struct gpio_chip gpio_chip;
 };
 
-static inline struct wm831x_gpio *to_wm831x_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct wm831x_gpio, gpio_chip);
-}
-
 static int wm831x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 	int val = WM831X_GPN_DIR;
 
@@ -51,7 +46,7 @@ static int wm831x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int wm831x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 	int ret;
 
@@ -67,7 +62,7 @@ static int wm831x_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void wm831x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 
 	wm831x_set_bits(wm831x, WM831X_GPIO_LEVEL, 1 << offset,
@@ -77,7 +72,7 @@ static void wm831x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int wm831x_gpio_direction_out(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 	int val = 0;
 	int ret;
@@ -99,7 +94,7 @@ static int wm831x_gpio_direction_out(struct gpio_chip *chip,
 
 static int wm831x_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 
 	return irq_create_mapping(wm831x->irq_domain,
@@ -109,7 +104,7 @@ static int wm831x_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 static int wm831x_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
 				    unsigned debounce)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 	int reg = WM831X_GPIO1_CONTROL + offset;
 	int ret, fn;
@@ -140,7 +135,7 @@ static int wm831x_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
 #ifdef CONFIG_DEBUG_FS
 static void wm831x_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct wm831x_gpio *wm831x_gpio = to_wm831x_gpio(chip);
+	struct wm831x_gpio *wm831x_gpio = gpiochip_get_data(chip);
 	struct wm831x *wm831x = wm831x_gpio->wm831x;
 	int i, tristated;
 
@@ -258,13 +253,13 @@ static int wm831x_gpio_probe(struct platform_device *pdev)
 	wm831x_gpio->wm831x = wm831x;
 	wm831x_gpio->gpio_chip = template_chip;
 	wm831x_gpio->gpio_chip.ngpio = wm831x->num_gpio;
-	wm831x_gpio->gpio_chip.dev = &pdev->dev;
+	wm831x_gpio->gpio_chip.parent = &pdev->dev;
 	if (pdata && pdata->gpio_base)
 		wm831x_gpio->gpio_chip.base = pdata->gpio_base;
 	else
 		wm831x_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&wm831x_gpio->gpio_chip);
+	ret = gpiochip_add_data(&wm831x_gpio->gpio_chip, wm831x_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-wm8350.c b/drivers/gpio/gpio-wm8350.c
index 060b89303bb6..0a306b4baa73 100644
--- a/drivers/gpio/gpio-wm8350.c
+++ b/drivers/gpio/gpio-wm8350.c
@@ -28,14 +28,9 @@ struct wm8350_gpio_data {
 	struct gpio_chip gpio_chip;
 };
 
-static inline struct wm8350_gpio_data *to_wm8350_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct wm8350_gpio_data, gpio_chip);
-}
-
 static int wm8350_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8350_gpio_data *wm8350_gpio = to_wm8350_gpio(chip);
+	struct wm8350_gpio_data *wm8350_gpio = gpiochip_get_data(chip);
 	struct wm8350 *wm8350 = wm8350_gpio->wm8350;
 
 	return wm8350_set_bits(wm8350, WM8350_GPIO_CONFIGURATION_I_O,
@@ -44,7 +39,7 @@ static int wm8350_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int wm8350_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8350_gpio_data *wm8350_gpio = to_wm8350_gpio(chip);
+	struct wm8350_gpio_data *wm8350_gpio = gpiochip_get_data(chip);
 	struct wm8350 *wm8350 = wm8350_gpio->wm8350;
 	int ret;
 
@@ -60,7 +55,7 @@ static int wm8350_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void wm8350_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct wm8350_gpio_data *wm8350_gpio = to_wm8350_gpio(chip);
+	struct wm8350_gpio_data *wm8350_gpio = gpiochip_get_data(chip);
 	struct wm8350 *wm8350 = wm8350_gpio->wm8350;
 
 	if (value)
@@ -72,7 +67,7 @@ static void wm8350_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 static int wm8350_gpio_direction_out(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
-	struct wm8350_gpio_data *wm8350_gpio = to_wm8350_gpio(chip);
+	struct wm8350_gpio_data *wm8350_gpio = gpiochip_get_data(chip);
 	struct wm8350 *wm8350 = wm8350_gpio->wm8350;
 	int ret;
 
@@ -89,7 +84,7 @@ static int wm8350_gpio_direction_out(struct gpio_chip *chip,
 
 static int wm8350_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8350_gpio_data *wm8350_gpio = to_wm8350_gpio(chip);
+	struct wm8350_gpio_data *wm8350_gpio = gpiochip_get_data(chip);
 	struct wm8350 *wm8350 = wm8350_gpio->wm8350;
 
 	if (!wm8350->irq_base)
@@ -124,13 +119,13 @@ static int wm8350_gpio_probe(struct platform_device *pdev)
 	wm8350_gpio->wm8350 = wm8350;
 	wm8350_gpio->gpio_chip = template_chip;
 	wm8350_gpio->gpio_chip.ngpio = 13;
-	wm8350_gpio->gpio_chip.dev = &pdev->dev;
+	wm8350_gpio->gpio_chip.parent = &pdev->dev;
 	if (pdata && pdata->gpio_base)
 		wm8350_gpio->gpio_chip.base = pdata->gpio_base;
 	else
 		wm8350_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&wm8350_gpio->gpio_chip);
+	ret = gpiochip_add_data(&wm8350_gpio->gpio_chip, wm8350_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
 		return ret;
diff --git a/drivers/gpio/gpio-wm8994.c b/drivers/gpio/gpio-wm8994.c
index 6f5e42db4b9e..3ae4c1597494 100644
--- a/drivers/gpio/gpio-wm8994.c
+++ b/drivers/gpio/gpio-wm8994.c
@@ -31,14 +31,9 @@ struct wm8994_gpio {
 	struct gpio_chip gpio_chip;
 };
 
-static inline struct wm8994_gpio *to_wm8994_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct wm8994_gpio, gpio_chip);
-}
-
 static int wm8994_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 
 	switch (wm8994->type) {
@@ -61,7 +56,7 @@ static int wm8994_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static int wm8994_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 
 	return wm8994_set_bits(wm8994, WM8994_GPIO_1 + offset,
@@ -70,7 +65,7 @@ static int wm8994_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 
 static int wm8994_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 	int ret;
 
@@ -87,7 +82,7 @@ static int wm8994_gpio_get(struct gpio_chip *chip, unsigned offset)
 static int wm8994_gpio_direction_out(struct gpio_chip *chip,
 				     unsigned offset, int value)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 
 	if (value)
@@ -99,7 +94,7 @@ static int wm8994_gpio_direction_out(struct gpio_chip *chip,
 
 static void wm8994_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 
 	if (value)
@@ -110,7 +105,7 @@ static void wm8994_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int wm8994_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 
 	return regmap_irq_get_virq(wm8994->irq_data, offset);
@@ -174,7 +169,7 @@ static const char *wm8994_gpio_fn(u16 fn)
 
 static void wm8994_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct wm8994_gpio *wm8994_gpio = to_wm8994_gpio(chip);
+	struct wm8994_gpio *wm8994_gpio = gpiochip_get_data(chip);
 	struct wm8994 *wm8994 = wm8994_gpio->wm8994;
 	int i;
 
@@ -260,13 +255,13 @@ static int wm8994_gpio_probe(struct platform_device *pdev)
 	wm8994_gpio->wm8994 = wm8994;
 	wm8994_gpio->gpio_chip = template_chip;
 	wm8994_gpio->gpio_chip.ngpio = WM8994_GPIO_MAX;
-	wm8994_gpio->gpio_chip.dev = &pdev->dev;
+	wm8994_gpio->gpio_chip.parent = &pdev->dev;
 	if (pdata && pdata->gpio_base)
 		wm8994_gpio->gpio_chip.base = pdata->gpio_base;
 	else
 		wm8994_gpio->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&wm8994_gpio->gpio_chip);
+	ret = gpiochip_add_data(&wm8994_gpio->gpio_chip, wm8994_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n",
 			ret);
diff --git a/drivers/gpio/gpio-xgene-sb.c b/drivers/gpio/gpio-xgene-sb.c
index d57068b9083e..282004deb5d4 100644
--- a/drivers/gpio/gpio-xgene-sb.c
+++ b/drivers/gpio/gpio-xgene-sb.c
@@ -23,10 +23,8 @@
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/of_gpio.h>
-#include <linux/gpio.h>
 #include <linux/gpio/driver.h>
 #include <linux/acpi.h>
-#include <linux/basic_mmio_gpio.h>
 
 #include "gpiolib.h"
 
@@ -43,38 +41,31 @@
 
 /**
  * struct xgene_gpio_sb - GPIO-Standby private data structure.
- * @bgc:			memory-mapped GPIO controllers.
+ * @gc:				memory-mapped GPIO controllers.
  * @irq:			Mapping GPIO pins and interrupt number
  * nirq:			Number of GPIO pins that supports interrupt
  */
 struct xgene_gpio_sb {
-	struct bgpio_chip	bgc;
+	struct gpio_chip	gc;
 	u32 *irq;
 	u32 nirq;
 };
 
-static inline struct xgene_gpio_sb *to_xgene_gpio_sb(struct gpio_chip *gc)
-{
-	struct bgpio_chip *bgc = to_bgpio_chip(gc);
-
-	return container_of(bgc, struct xgene_gpio_sb, bgc);
-}
-
-static void xgene_gpio_set_bit(struct bgpio_chip *bgc, void __iomem *reg, u32 gpio, int val)
+static void xgene_gpio_set_bit(struct gpio_chip *gc, void __iomem *reg, u32 gpio, int val)
 {
 	u32 data;
 
-	data = bgc->read_reg(reg);
+	data = gc->read_reg(reg);
 	if (val)
 		data |= GPIO_MASK(gpio);
 	else
 		data &= ~GPIO_MASK(gpio);
-	bgc->write_reg(reg, data);
+	gc->write_reg(reg, data);
 }
 
 static int apm_gpio_sb_to_irq(struct gpio_chip *gc, u32 gpio)
 {
-	struct xgene_gpio_sb *priv = to_xgene_gpio_sb(gc);
+	struct xgene_gpio_sb *priv = gpiochip_get_data(gc);
 
 	if (priv->irq[gpio])
 		return priv->irq[gpio];
@@ -99,15 +90,15 @@ static int xgene_gpio_sb_probe(struct platform_device *pdev)
 	if (IS_ERR(regs))
 		return PTR_ERR(regs);
 
-	ret = bgpio_init(&priv->bgc, &pdev->dev, 4,
+	ret = bgpio_init(&priv->gc, &pdev->dev, 4,
 			regs + MPA_GPIO_IN_ADDR,
 			regs + MPA_GPIO_OUT_ADDR, NULL,
 			regs + MPA_GPIO_OE_ADDR, NULL, 0);
         if (ret)
                 return ret;
 
-	priv->bgc.gc.to_irq = apm_gpio_sb_to_irq;
-	priv->bgc.gc.ngpio = XGENE_MAX_GPIO_DS;
+	priv->gc.to_irq = apm_gpio_sb_to_irq;
+	priv->gc.ngpio = XGENE_MAX_GPIO_DS;
 
 	priv->nirq = XGENE_MAX_GPIO_DS_IRQ;
 
@@ -118,14 +109,14 @@ static int xgene_gpio_sb_probe(struct platform_device *pdev)
 
 	for (i = 0; i < priv->nirq; i++) {
 		priv->irq[default_lines[i]] = platform_get_irq(pdev, i);
-		xgene_gpio_set_bit(&priv->bgc, regs + MPA_GPIO_SEL_LO,
+		xgene_gpio_set_bit(&priv->gc, regs + MPA_GPIO_SEL_LO,
                                    default_lines[i] * 2, 1);
-		xgene_gpio_set_bit(&priv->bgc, regs + MPA_GPIO_INT_LVL, i, 1);
+		xgene_gpio_set_bit(&priv->gc, regs + MPA_GPIO_INT_LVL, i, 1);
 	}
 
 	platform_set_drvdata(pdev, priv);
 
-	ret = gpiochip_add(&priv->bgc.gc);
+	ret = gpiochip_add_data(&priv->gc, priv);
 	if (ret)
 		dev_err(&pdev->dev, "failed to register X-Gene GPIO Standby driver\n");
 	else
@@ -133,7 +124,7 @@ static int xgene_gpio_sb_probe(struct platform_device *pdev)
 
 	if (priv->nirq > 0) {
 		/* Register interrupt handlers for gpio signaled acpi events */
-		acpi_gpiochip_request_interrupts(&priv->bgc.gc);
+		acpi_gpiochip_request_interrupts(&priv->gc);
 	}
 
 	return ret;
@@ -144,10 +135,11 @@ static int xgene_gpio_sb_remove(struct platform_device *pdev)
 	struct xgene_gpio_sb *priv = platform_get_drvdata(pdev);
 
 	if (priv->nirq > 0) {
-		acpi_gpiochip_free_interrupts(&priv->bgc.gc);
+		acpi_gpiochip_free_interrupts(&priv->gc);
 	}
 
-	return bgpio_remove(&priv->bgc);
+	gpiochip_remove(&priv->gc);
+	return 0;
 }
 
 static const struct of_device_id xgene_gpio_sb_of_match[] = {
diff --git a/drivers/gpio/gpio-xgene.c b/drivers/gpio/gpio-xgene.c
index 18a8182d4fec..592e9cdf9c53 100644
--- a/drivers/gpio/gpio-xgene.c
+++ b/drivers/gpio/gpio-xgene.c
@@ -47,14 +47,9 @@ struct xgene_gpio {
 #endif
 };
 
-static inline struct xgene_gpio *to_xgene_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct xgene_gpio, chip);
-}
-
 static int xgene_gpio_get(struct gpio_chip *gc, unsigned int offset)
 {
-	struct xgene_gpio *chip = to_xgene_gpio(gc);
+	struct xgene_gpio *chip = gpiochip_get_data(gc);
 	unsigned long bank_offset;
 	u32 bit_offset;
 
@@ -65,7 +60,7 @@ static int xgene_gpio_get(struct gpio_chip *gc, unsigned int offset)
 
 static void __xgene_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 {
-	struct xgene_gpio *chip = to_xgene_gpio(gc);
+	struct xgene_gpio *chip = gpiochip_get_data(gc);
 	unsigned long bank_offset;
 	u32 setval, bit_offset;
 
@@ -82,7 +77,7 @@ static void __xgene_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 
 static void xgene_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 {
-	struct xgene_gpio *chip = to_xgene_gpio(gc);
+	struct xgene_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -92,7 +87,7 @@ static void xgene_gpio_set(struct gpio_chip *gc, unsigned int offset, int val)
 
 static int xgene_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
 {
-	struct xgene_gpio *chip = to_xgene_gpio(gc);
+	struct xgene_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags, bank_offset;
 	u32 dirval, bit_offset;
 
@@ -113,7 +108,7 @@ static int xgene_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
 static int xgene_gpio_dir_out(struct gpio_chip *gc,
 					unsigned int offset, int val)
 {
-	struct xgene_gpio *chip = to_xgene_gpio(gc);
+	struct xgene_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags, bank_offset;
 	u32 dirval, bit_offset;
 
@@ -188,7 +183,7 @@ static int xgene_gpio_probe(struct platform_device *pdev)
 	gpio->chip.ngpio = XGENE_MAX_GPIOS;
 
 	spin_lock_init(&gpio->lock);
-	gpio->chip.dev = &pdev->dev;
+	gpio->chip.parent = &pdev->dev;
 	gpio->chip.direction_input = xgene_gpio_dir_in;
 	gpio->chip.direction_output = xgene_gpio_dir_out;
 	gpio->chip.get = xgene_gpio_get;
@@ -198,7 +193,7 @@ static int xgene_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, gpio);
 
-	err = gpiochip_add(&gpio->chip);
+	err = gpiochip_add_data(&gpio->chip, gpio);
 	if (err) {
 		dev_err(&pdev->dev,
 			"failed to register gpiochip.\n");
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index d5284dfe01fe..d0fbb7f99523 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -92,8 +92,7 @@ static inline int xgpio_offset(struct xgpio_instance *chip, int gpio)
 static int xgpio_get(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct xgpio_instance *chip =
-	    container_of(mm_gc, struct xgpio_instance, mmchip);
+	struct xgpio_instance *chip = gpiochip_get_data(gc);
 	u32 val;
 
 	val = xgpio_readreg(mm_gc->regs + XGPIO_DATA_OFFSET +
@@ -115,8 +114,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	unsigned long flags;
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct xgpio_instance *chip =
-	    container_of(mm_gc, struct xgpio_instance, mmchip);
+	struct xgpio_instance *chip = gpiochip_get_data(gc);
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
@@ -147,8 +145,7 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
 	unsigned long flags;
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct xgpio_instance *chip =
-	    container_of(mm_gc, struct xgpio_instance, mmchip);
+	struct xgpio_instance *chip = gpiochip_get_data(gc);
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
@@ -180,8 +177,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 {
 	unsigned long flags;
 	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
-	struct xgpio_instance *chip =
-	    container_of(mm_gc, struct xgpio_instance, mmchip);
+	struct xgpio_instance *chip = gpiochip_get_data(gc);
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
@@ -212,7 +208,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 static void xgpio_save_regs(struct of_mm_gpio_chip *mm_gc)
 {
 	struct xgpio_instance *chip =
-	    container_of(mm_gc, struct xgpio_instance, mmchip);
+		container_of(mm_gc, struct xgpio_instance, mmchip);
 
 	xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET,	chip->gpio_state[0]);
 	xgpio_writereg(mm_gc->regs + XGPIO_TRI_OFFSET, chip->gpio_dir[0]);
@@ -305,7 +301,7 @@ static int xgpio_probe(struct platform_device *pdev)
 	}
 
 	chip->mmchip.gc.ngpio = chip->gpio_width[0] + chip->gpio_width[1];
-	chip->mmchip.gc.dev = &pdev->dev;
+	chip->mmchip.gc.parent = &pdev->dev;
 	chip->mmchip.gc.direction_input = xgpio_dir_in;
 	chip->mmchip.gc.direction_output = xgpio_dir_out;
 	chip->mmchip.gc.get = xgpio_get;
@@ -314,7 +310,7 @@ static int xgpio_probe(struct platform_device *pdev)
 	chip->mmchip.save_regs = xgpio_save_regs;
 
 	/* Call the OF gpio helper to setup and register the GPIO device */
-	status = of_mm_gpiochip_add(np, &chip->mmchip);
+	status = of_mm_gpiochip_add_data(np, &chip->mmchip, chip);
 	if (status) {
 		pr_err("%s: error in probe function with status %d\n",
 		       np->full_name, status);
diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c
index bc06a2cd2c1d..aa5813d2deb1 100644
--- a/drivers/gpio/gpio-xlp.c
+++ b/drivers/gpio/gpio-xlp.c
@@ -100,11 +100,6 @@ struct xlp_gpio_priv {
 	spinlock_t lock;
 };
 
-static struct xlp_gpio_priv *gpio_chip_to_xlp_priv(struct gpio_chip *gc)
-{
-	return container_of(gc, struct xlp_gpio_priv, chip);
-}
-
 static int xlp_gpio_get_reg(void __iomem *addr, unsigned gpio)
 {
 	u32 pos, regset;
@@ -133,7 +128,7 @@ static void xlp_gpio_set_reg(void __iomem *addr, unsigned gpio, int state)
 static void xlp_gpio_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&priv->lock, flags);
@@ -145,7 +140,7 @@ static void xlp_gpio_irq_disable(struct irq_data *d)
 static void xlp_gpio_irq_mask_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&priv->lock, flags);
@@ -158,7 +153,7 @@ static void xlp_gpio_irq_mask_ack(struct irq_data *d)
 static void xlp_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&priv->lock, flags);
@@ -170,7 +165,7 @@ static void xlp_gpio_irq_unmask(struct irq_data *d)
 static int xlp_gpio_set_irq_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 	int pol, irq_type;
 
 	switch (type) {
@@ -235,7 +230,7 @@ static void xlp_gpio_generic_handler(struct irq_desc *desc)
 
 static int xlp_gpio_dir_output(struct gpio_chip *gc, unsigned gpio, int state)
 {
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 
 	BUG_ON(gpio >= gc->ngpio);
 	xlp_gpio_set_reg(priv->gpio_out_en, gpio, 0x1);
@@ -245,7 +240,7 @@ static int xlp_gpio_dir_output(struct gpio_chip *gc, unsigned gpio, int state)
 
 static int xlp_gpio_dir_input(struct gpio_chip *gc, unsigned gpio)
 {
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 
 	BUG_ON(gpio >= gc->ngpio);
 	xlp_gpio_set_reg(priv->gpio_out_en, gpio, 0x0);
@@ -255,7 +250,7 @@ static int xlp_gpio_dir_input(struct gpio_chip *gc, unsigned gpio)
 
 static int xlp_gpio_get(struct gpio_chip *gc, unsigned gpio)
 {
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 
 	BUG_ON(gpio >= gc->ngpio);
 	return xlp_gpio_get_reg(priv->gpio_paddrv, gpio);
@@ -263,7 +258,7 @@ static int xlp_gpio_get(struct gpio_chip *gc, unsigned gpio)
 
 static void xlp_gpio_set(struct gpio_chip *gc, unsigned gpio, int state)
 {
-	struct xlp_gpio_priv *priv = gpio_chip_to_xlp_priv(gc);
+	struct xlp_gpio_priv *priv = gpiochip_get_data(gc);
 
 	BUG_ON(gpio >= gc->ngpio);
 	xlp_gpio_set_reg(priv->gpio_paddrv, gpio, state);
@@ -373,7 +368,7 @@ static int xlp_gpio_probe(struct platform_device *pdev)
 	gc->owner = THIS_MODULE;
 	gc->label = dev_name(&pdev->dev);
 	gc->base = 0;
-	gc->dev = &pdev->dev;
+	gc->parent = &pdev->dev;
 	gc->ngpio = ngpio;
 	gc->of_node = pdev->dev.of_node;
 	gc->direction_output = xlp_gpio_dir_output;
@@ -388,7 +383,7 @@ static int xlp_gpio_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	err = gpiochip_add(gc);
+	err = gpiochip_add_data(gc, priv);
 	if (err < 0)
 		goto out_free_desc;
 
diff --git a/drivers/gpio/gpio-xtensa.c b/drivers/gpio/gpio-xtensa.c
index 93ec95df67a3..f16c0427952e 100644
--- a/drivers/gpio/gpio-xtensa.c
+++ b/drivers/gpio/gpio-xtensa.c
@@ -148,10 +148,10 @@ static int xtensa_gpio_probe(struct platform_device *pdev)
 {
 	int ret;
 
-	ret = gpiochip_add(&impwire_chip);
+	ret = gpiochip_add_data(&impwire_chip, NULL);
 	if (ret)
 		return ret;
-	return gpiochip_add(&expstate_chip);
+	return gpiochip_add_data(&expstate_chip, NULL);
 }
 
 static struct platform_driver xtensa_gpio_driver = {
diff --git a/drivers/gpio/gpio-zevio.c b/drivers/gpio/gpio-zevio.c
index 6f02d7c4cc57..cda6d922be98 100644
--- a/drivers/gpio/gpio-zevio.c
+++ b/drivers/gpio/gpio-zevio.c
@@ -52,9 +52,6 @@
 #define ZEVIO_GPIO_INPUT			0x18
 #define ZEVIO_GPIO_INT_STICKY		0x20
 
-#define to_zevio_gpio(chip) container_of(to_of_mm_gpio_chip(chip), \
-				struct zevio_gpio, chip)
-
 /* Bit number of GPIO in its section */
 #define ZEVIO_GPIO_BIT(gpio) (gpio&7)
 
@@ -80,7 +77,7 @@ static inline void zevio_gpio_port_set(struct zevio_gpio *c, unsigned pin,
 /* Functions for struct gpio_chip */
 static int zevio_gpio_get(struct gpio_chip *chip, unsigned pin)
 {
-	struct zevio_gpio *controller = to_zevio_gpio(chip);
+	struct zevio_gpio *controller = gpiochip_get_data(chip);
 	u32 val, dir;
 
 	spin_lock(&controller->lock);
@@ -96,7 +93,7 @@ static int zevio_gpio_get(struct gpio_chip *chip, unsigned pin)
 
 static void zevio_gpio_set(struct gpio_chip *chip, unsigned pin, int value)
 {
-	struct zevio_gpio *controller = to_zevio_gpio(chip);
+	struct zevio_gpio *controller = gpiochip_get_data(chip);
 	u32 val;
 
 	spin_lock(&controller->lock);
@@ -112,7 +109,7 @@ static void zevio_gpio_set(struct gpio_chip *chip, unsigned pin, int value)
 
 static int zevio_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 {
-	struct zevio_gpio *controller = to_zevio_gpio(chip);
+	struct zevio_gpio *controller = gpiochip_get_data(chip);
 	u32 val;
 
 	spin_lock(&controller->lock);
@@ -129,7 +126,7 @@ static int zevio_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 static int zevio_gpio_direction_output(struct gpio_chip *chip,
 				       unsigned pin, int value)
 {
-	struct zevio_gpio *controller = to_zevio_gpio(chip);
+	struct zevio_gpio *controller = gpiochip_get_data(chip);
 	u32 val;
 
 	spin_lock(&controller->lock);
@@ -185,9 +182,11 @@ static int zevio_gpio_probe(struct platform_device *pdev)
 
 	/* Copy our reference */
 	controller->chip.gc = zevio_gpio_chip;
-	controller->chip.gc.dev = &pdev->dev;
+	controller->chip.gc.parent = &pdev->dev;
 
-	status = of_mm_gpiochip_add(pdev->dev.of_node, &(controller->chip));
+	status = of_mm_gpiochip_add_data(pdev->dev.of_node,
+					 &(controller->chip),
+					 controller);
 	if (status) {
 		dev_err(&pdev->dev, "failed to add gpiochip: %d\n", status);
 		return status;
@@ -199,7 +198,7 @@ static int zevio_gpio_probe(struct platform_device *pdev)
 	for (i = 0; i < controller->chip.gc.ngpio; i += 8)
 		zevio_gpio_port_set(controller, i, ZEVIO_GPIO_INT_MASK, 0xFF);
 
-	dev_dbg(controller->chip.gc.dev, "ZEVIO GPIO controller set up!\n");
+	dev_dbg(controller->chip.gc.parent, "ZEVIO GPIO controller set up!\n");
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-zx.c b/drivers/gpio/gpio-zx.c
index 1dcf7a66dd36..47c79fa65670 100644
--- a/drivers/gpio/gpio-zx.c
+++ b/drivers/gpio/gpio-zx.c
@@ -43,14 +43,9 @@ struct zx_gpio {
 	struct gpio_chip	gc;
 };
 
-static inline struct zx_gpio *to_zx(struct gpio_chip *gc)
-{
-	return container_of(gc, struct zx_gpio, gc);
-}
-
 static int zx_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 	u16 gpiodir;
 
@@ -69,7 +64,7 @@ static int zx_direction_input(struct gpio_chip *gc, unsigned offset)
 static int zx_direction_output(struct gpio_chip *gc, unsigned offset,
 		int value)
 {
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 	u16 gpiodir;
 
@@ -92,14 +87,14 @@ static int zx_direction_output(struct gpio_chip *gc, unsigned offset,
 
 static int zx_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 
 	return !!(readw_relaxed(chip->base + ZX_GPIO_DI) & BIT(offset));
 }
 
 static void zx_set_value(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 
 	if (value)
 		writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1);
@@ -110,7 +105,7 @@ static void zx_set_value(struct gpio_chip *gc, unsigned offset, int value)
 static int zx_irq_type(struct irq_data *d, unsigned trigger)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	int offset = irqd_to_hwirq(d);
 	unsigned long flags;
 	u16 gpiois, gpioi_epos, gpioi_eneg, gpioiev;
@@ -162,7 +157,7 @@ static void zx_irq_handler(struct irq_desc *desc)
 	unsigned long pending;
 	int offset;
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 
 	chained_irq_enter(irqchip, desc);
@@ -181,7 +176,7 @@ static void zx_irq_handler(struct irq_desc *desc)
 static void zx_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR);
 	u16 gpioie;
 
@@ -196,7 +191,7 @@ static void zx_irq_mask(struct irq_data *d)
 static void zx_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct zx_gpio *chip = to_zx(gc);
+	struct zx_gpio *chip = gpiochip_get_data(gc);
 	u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR);
 	u16 gpioie;
 
@@ -245,10 +240,10 @@ static int zx_gpio_probe(struct platform_device *pdev)
 	chip->gc.base = ZX_GPIO_NR * id;
 	chip->gc.ngpio = ZX_GPIO_NR;
 	chip->gc.label = dev_name(dev);
-	chip->gc.dev = dev;
+	chip->gc.parent = dev;
 	chip->gc.owner = THIS_MODULE;
 
-	ret = gpiochip_add(&chip->gc);
+	ret = gpiochip_add_data(&chip->gc, chip);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index 8abeacac5885..66d3d247d76d 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -131,11 +131,6 @@ struct zynq_platform_data {
 static struct irq_chip zynq_gpio_level_irqchip;
 static struct irq_chip zynq_gpio_edge_irqchip;
 
-static struct zynq_gpio *to_zynq_gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct zynq_gpio, chip);
-}
-
 /**
  * zynq_gpio_get_bank_pin - Get the bank number and pin number within that bank
  * for a given pin in the GPIO device
@@ -183,7 +178,7 @@ static int zynq_gpio_get_value(struct gpio_chip *chip, unsigned int pin)
 {
 	u32 data;
 	unsigned int bank_num, bank_pin_num;
-	struct zynq_gpio *gpio = to_zynq_gpio(chip);
+	struct zynq_gpio *gpio = gpiochip_get_data(chip);
 
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
@@ -207,7 +202,7 @@ static void zynq_gpio_set_value(struct gpio_chip *chip, unsigned int pin,
 				int state)
 {
 	unsigned int reg_offset, bank_num, bank_pin_num;
-	struct zynq_gpio *gpio = to_zynq_gpio(chip);
+	struct zynq_gpio *gpio = gpiochip_get_data(chip);
 
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
@@ -244,7 +239,7 @@ static int zynq_gpio_dir_in(struct gpio_chip *chip, unsigned int pin)
 {
 	u32 reg;
 	unsigned int bank_num, bank_pin_num;
-	struct zynq_gpio *gpio = to_zynq_gpio(chip);
+	struct zynq_gpio *gpio = gpiochip_get_data(chip);
 
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
@@ -277,7 +272,7 @@ static int zynq_gpio_dir_out(struct gpio_chip *chip, unsigned int pin,
 {
 	u32 reg;
 	unsigned int bank_num, bank_pin_num;
-	struct zynq_gpio *gpio = to_zynq_gpio(chip);
+	struct zynq_gpio *gpio = gpiochip_get_data(chip);
 
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
@@ -308,7 +303,7 @@ static void zynq_gpio_irq_mask(struct irq_data *irq_data)
 {
 	unsigned int device_pin_num, bank_num, bank_pin_num;
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_data_get_irq_chip_data(irq_data));
+		gpiochip_get_data(irq_data_get_irq_chip_data(irq_data));
 
 	device_pin_num = irq_data->hwirq;
 	zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num, gpio);
@@ -329,7 +324,7 @@ static void zynq_gpio_irq_unmask(struct irq_data *irq_data)
 {
 	unsigned int device_pin_num, bank_num, bank_pin_num;
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_data_get_irq_chip_data(irq_data));
+		gpiochip_get_data(irq_data_get_irq_chip_data(irq_data));
 
 	device_pin_num = irq_data->hwirq;
 	zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num, gpio);
@@ -349,7 +344,7 @@ static void zynq_gpio_irq_ack(struct irq_data *irq_data)
 {
 	unsigned int device_pin_num, bank_num, bank_pin_num;
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_data_get_irq_chip_data(irq_data));
+		gpiochip_get_data(irq_data_get_irq_chip_data(irq_data));
 
 	device_pin_num = irq_data->hwirq;
 	zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num, gpio);
@@ -400,7 +395,7 @@ static int zynq_gpio_set_irq_type(struct irq_data *irq_data, unsigned int type)
 	u32 int_type, int_pol, int_any;
 	unsigned int device_pin_num, bank_num, bank_pin_num;
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_data_get_irq_chip_data(irq_data));
+		gpiochip_get_data(irq_data_get_irq_chip_data(irq_data));
 
 	device_pin_num = irq_data->hwirq;
 	zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num, gpio);
@@ -464,7 +459,7 @@ static int zynq_gpio_set_irq_type(struct irq_data *irq_data, unsigned int type)
 static int zynq_gpio_set_wake(struct irq_data *data, unsigned int on)
 {
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_data_get_irq_chip_data(data));
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
 
 	irq_set_irq_wake(gpio->irq, on);
 
@@ -530,7 +525,7 @@ static void zynq_gpio_irqhandler(struct irq_desc *desc)
 	u32 int_sts, int_enb;
 	unsigned int bank_num;
 	struct zynq_gpio *gpio =
-		to_zynq_gpio(irq_desc_get_handler_data(desc));
+		gpiochip_get_data(irq_desc_get_handler_data(desc));
 	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 
 	chained_irq_enter(irqchip, desc);
@@ -592,7 +587,7 @@ static int zynq_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
 	int ret;
 
-	ret = pm_runtime_get_sync(chip->dev);
+	ret = pm_runtime_get_sync(chip->parent);
 
 	/*
 	 * If the device is already active pm_runtime_get() will return 1 on
@@ -603,7 +598,7 @@ static int zynq_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void zynq_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	pm_runtime_put(chip->dev);
+	pm_runtime_put(chip->parent);
 }
 
 static const struct dev_pm_ops zynq_gpio_dev_pm_ops = {
@@ -698,7 +693,7 @@ static int zynq_gpio_probe(struct platform_device *pdev)
 	chip = &gpio->chip;
 	chip->label = gpio->p_data->label;
 	chip->owner = THIS_MODULE;
-	chip->dev = &pdev->dev;
+	chip->parent = &pdev->dev;
 	chip->get = zynq_gpio_get_value;
 	chip->set = zynq_gpio_set_value;
 	chip->request = zynq_gpio_request;
@@ -708,23 +703,23 @@ static int zynq_gpio_probe(struct platform_device *pdev)
 	chip->base = -1;
 	chip->ngpio = gpio->p_data->ngpio;
 
-	/* Enable GPIO clock */
+	/* Retrieve GPIO clock */
 	gpio->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(gpio->clk)) {
 		dev_err(&pdev->dev, "input clock not found.\n");
 		return PTR_ERR(gpio->clk);
 	}
-	ret = clk_prepare_enable(gpio->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "Unable to enable clock.\n");
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0)
 		return ret;
-	}
 
 	/* report a bug if gpio chip registration fails */
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, gpio);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to add gpio chip\n");
-		goto err_disable_clk;
+		goto err_pm_put;
 	}
 
 	/* disable interrupts for all banks */
@@ -742,15 +737,14 @@ static int zynq_gpio_probe(struct platform_device *pdev)
 	gpiochip_set_chained_irqchip(chip, &zynq_gpio_edge_irqchip, gpio->irq,
 				     zynq_gpio_irqhandler);
 
-	pm_runtime_set_active(&pdev->dev);
-	pm_runtime_enable(&pdev->dev);
+	pm_runtime_put(&pdev->dev);
 
 	return 0;
 
 err_rm_gpiochip:
 	gpiochip_remove(chip);
-err_disable_clk:
-	clk_disable_unprepare(gpio->clk);
+err_pm_put:
+	pm_runtime_put(&pdev->dev);
 
 	return ret;
 }
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index bc34bc51a948..540cbc88c7a2 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -51,10 +51,10 @@ struct acpi_gpio_chip {
 
 static int acpi_gpiochip_find(struct gpio_chip *gc, void *data)
 {
-	if (!gc->dev)
+	if (!gc->parent)
 		return false;
 
-	return ACPI_HANDLE(gc->dev) == data;
+	return ACPI_HANDLE(gc->parent) == data;
 }
 
 #ifdef CONFIG_PINCTRL
@@ -184,7 +184,7 @@ static acpi_status acpi_gpiochip_request_interrupt(struct acpi_resource *ares,
 	if (agpio->connection_type != ACPI_RESOURCE_GPIO_TYPE_INT)
 		return AE_OK;
 
-	handle = ACPI_HANDLE(chip->dev);
+	handle = ACPI_HANDLE(chip->parent);
 	pin = agpio->pin_table[0];
 
 	if (pin <= 255) {
@@ -208,7 +208,7 @@ static acpi_status acpi_gpiochip_request_interrupt(struct acpi_resource *ares,
 
 	desc = gpiochip_request_own_desc(chip, pin, "ACPI:Event");
 	if (IS_ERR(desc)) {
-		dev_err(chip->dev, "Failed to request GPIO\n");
+		dev_err(chip->parent, "Failed to request GPIO\n");
 		return AE_ERROR;
 	}
 
@@ -216,13 +216,13 @@ static acpi_status acpi_gpiochip_request_interrupt(struct acpi_resource *ares,
 
 	ret = gpiochip_lock_as_irq(chip, pin);
 	if (ret) {
-		dev_err(chip->dev, "Failed to lock GPIO as interrupt\n");
+		dev_err(chip->parent, "Failed to lock GPIO as interrupt\n");
 		goto fail_free_desc;
 	}
 
 	irq = gpiod_to_irq(desc);
 	if (irq < 0) {
-		dev_err(chip->dev, "Failed to translate GPIO to IRQ\n");
+		dev_err(chip->parent, "Failed to translate GPIO to IRQ\n");
 		goto fail_unlock_irq;
 	}
 
@@ -259,7 +259,8 @@ static acpi_status acpi_gpiochip_request_interrupt(struct acpi_resource *ares,
 	ret = request_threaded_irq(event->irq, NULL, handler, irqflags,
 				   "ACPI:Event", event);
 	if (ret) {
-		dev_err(chip->dev, "Failed to setup interrupt handler for %d\n",
+		dev_err(chip->parent,
+			"Failed to setup interrupt handler for %d\n",
 			event->irq);
 		goto fail_free_event;
 	}
@@ -293,10 +294,10 @@ void acpi_gpiochip_request_interrupts(struct gpio_chip *chip)
 	acpi_handle handle;
 	acpi_status status;
 
-	if (!chip->dev || !chip->to_irq)
+	if (!chip->parent || !chip->to_irq)
 		return;
 
-	handle = ACPI_HANDLE(chip->dev);
+	handle = ACPI_HANDLE(chip->parent);
 	if (!handle)
 		return;
 
@@ -323,10 +324,10 @@ void acpi_gpiochip_free_interrupts(struct gpio_chip *chip)
 	acpi_handle handle;
 	acpi_status status;
 
-	if (!chip->dev || !chip->to_irq)
+	if (!chip->parent || !chip->to_irq)
 		return;
 
-	handle = ACPI_HANDLE(chip->dev);
+	handle = ACPI_HANDLE(chip->parent);
 	if (!handle)
 		return;
 
@@ -769,7 +770,7 @@ out:
 static void acpi_gpiochip_request_regions(struct acpi_gpio_chip *achip)
 {
 	struct gpio_chip *chip = achip->chip;
-	acpi_handle handle = ACPI_HANDLE(chip->dev);
+	acpi_handle handle = ACPI_HANDLE(chip->parent);
 	acpi_status status;
 
 	INIT_LIST_HEAD(&achip->conns);
@@ -778,20 +779,22 @@ static void acpi_gpiochip_request_regions(struct acpi_gpio_chip *achip)
 						    acpi_gpio_adr_space_handler,
 						    NULL, achip);
 	if (ACPI_FAILURE(status))
-		dev_err(chip->dev, "Failed to install GPIO OpRegion handler\n");
+		dev_err(chip->parent,
+		        "Failed to install GPIO OpRegion handler\n");
 }
 
 static void acpi_gpiochip_free_regions(struct acpi_gpio_chip *achip)
 {
 	struct gpio_chip *chip = achip->chip;
-	acpi_handle handle = ACPI_HANDLE(chip->dev);
+	acpi_handle handle = ACPI_HANDLE(chip->parent);
 	struct acpi_gpio_connection *conn, *tmp;
 	acpi_status status;
 
 	status = acpi_remove_address_space_handler(handle, ACPI_ADR_SPACE_GPIO,
 						   acpi_gpio_adr_space_handler);
 	if (ACPI_FAILURE(status)) {
-		dev_err(chip->dev, "Failed to remove GPIO OpRegion handler\n");
+		dev_err(chip->parent,
+			"Failed to remove GPIO OpRegion handler\n");
 		return;
 	}
 
@@ -808,16 +811,16 @@ void acpi_gpiochip_add(struct gpio_chip *chip)
 	acpi_handle handle;
 	acpi_status status;
 
-	if (!chip || !chip->dev)
+	if (!chip || !chip->parent)
 		return;
 
-	handle = ACPI_HANDLE(chip->dev);
+	handle = ACPI_HANDLE(chip->parent);
 	if (!handle)
 		return;
 
 	acpi_gpio = kzalloc(sizeof(*acpi_gpio), GFP_KERNEL);
 	if (!acpi_gpio) {
-		dev_err(chip->dev,
+		dev_err(chip->parent,
 			"Failed to allocate memory for ACPI GPIO chip\n");
 		return;
 	}
@@ -827,7 +830,7 @@ void acpi_gpiochip_add(struct gpio_chip *chip)
 
 	status = acpi_attach_data(handle, acpi_gpio_chip_dh, acpi_gpio);
 	if (ACPI_FAILURE(status)) {
-		dev_err(chip->dev, "Failed to attach ACPI GPIO chip\n");
+		dev_err(chip->parent, "Failed to attach ACPI GPIO chip\n");
 		kfree(acpi_gpio);
 		return;
 	}
@@ -841,16 +844,16 @@ void acpi_gpiochip_remove(struct gpio_chip *chip)
 	acpi_handle handle;
 	acpi_status status;
 
-	if (!chip || !chip->dev)
+	if (!chip || !chip->parent)
 		return;
 
-	handle = ACPI_HANDLE(chip->dev);
+	handle = ACPI_HANDLE(chip->parent);
 	if (!handle)
 		return;
 
 	status = acpi_get_data(handle, acpi_gpio_chip_dh, (void **)&acpi_gpio);
 	if (ACPI_FAILURE(status)) {
-		dev_warn(chip->dev, "Failed to retrieve ACPI GPIO chip\n");
+		dev_warn(chip->parent, "Failed to retrieve ACPI GPIO chip\n");
 		return;
 	}
 
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 5fe34a9df3e6..42a4bb7cf49a 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -262,9 +262,10 @@ int of_gpio_simple_xlate(struct gpio_chip *gc,
 EXPORT_SYMBOL(of_gpio_simple_xlate);
 
 /**
- * of_mm_gpiochip_add - Add memory mapped GPIO chip (bank)
+ * of_mm_gpiochip_add_data - Add memory mapped GPIO chip (bank)
  * @np:		device node of the GPIO chip
  * @mm_gc:	pointer to the of_mm_gpio_chip allocated structure
+ * @data:	driver data to store in the struct gpio_chip
  *
  * To use this function you should allocate and fill mm_gc with:
  *
@@ -280,8 +281,9 @@ EXPORT_SYMBOL(of_gpio_simple_xlate);
  * do all necessary work for you. Then you'll able to use .regs
  * to manage GPIOs from the callbacks.
  */
-int of_mm_gpiochip_add(struct device_node *np,
-		       struct of_mm_gpio_chip *mm_gc)
+int of_mm_gpiochip_add_data(struct device_node *np,
+			    struct of_mm_gpio_chip *mm_gc,
+			    void *data)
 {
 	int ret = -ENOMEM;
 	struct gpio_chip *gc = &mm_gc->gc;
@@ -301,7 +303,7 @@ int of_mm_gpiochip_add(struct device_node *np,
 
 	mm_gc->gc.of_node = np;
 
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, data);
 	if (ret)
 		goto err2;
 
@@ -315,7 +317,7 @@ err0:
 	       np->full_name, ret);
 	return ret;
 }
-EXPORT_SYMBOL(of_mm_gpiochip_add);
+EXPORT_SYMBOL(of_mm_gpiochip_add_data);
 
 /**
  * of_mm_gpiochip_remove - Remove memory mapped GPIO chip (bank)
@@ -423,8 +425,8 @@ int of_gpiochip_add(struct gpio_chip *chip)
 {
 	int status;
 
-	if ((!chip->of_node) && (chip->dev))
-		chip->of_node = chip->dev->of_node;
+	if ((!chip->of_node) && (chip->parent))
+		chip->of_node = chip->parent->of_node;
 
 	if (!chip->of_node)
 		return 0;
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index b57ed8e55ab5..405dfcaadc4c 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -605,7 +605,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change)
 	if (chip->names && chip->names[offset])
 		ioname = chip->names[offset];
 
-	dev = device_create_with_groups(&gpio_class, chip->dev,
+	dev = device_create_with_groups(&gpio_class, chip->parent,
 					MKDEV(0, 0), data, gpio_groups,
 					ioname ? ioname : "gpio%u",
 					desc_to_gpio(desc));
@@ -730,7 +730,8 @@ int gpiochip_sysfs_register(struct gpio_chip *chip)
 		return 0;
 
 	/* use chip->base for the ID; it's already known to be unique */
-	dev = device_create_with_groups(&gpio_class, chip->dev, MKDEV(0, 0),
+	dev = device_create_with_groups(&gpio_class, chip->parent,
+					MKDEV(0, 0),
 					chip, gpiochip_groups,
 					"gpiochip%d", chip->base);
 	if (IS_ERR(dev))
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 3346abd29b52..5c1ba879f889 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -182,39 +182,62 @@ EXPORT_SYMBOL_GPL(gpiod_get_direction);
 
 /*
  * Add a new chip to the global chips list, keeping the list of chips sorted
- * by base order.
+ * by range(means [base, base + ngpio - 1]) order.
  *
  * Return -EBUSY if the new chip overlaps with some other chip's integer
  * space.
  */
 static int gpiochip_add_to_list(struct gpio_chip *chip)
 {
-	struct list_head *pos;
-	struct gpio_chip *_chip;
-	int err = 0;
+	struct gpio_chip *iterator;
+	struct gpio_chip *previous = NULL;
 
-	/* find where to insert our chip */
-	list_for_each(pos, &gpio_chips) {
-		_chip = list_entry(pos, struct gpio_chip, list);
-		/* shall we insert before _chip? */
-		if (_chip->base >= chip->base + chip->ngpio)
-			break;
+	if (list_empty(&gpio_chips)) {
+		list_add_tail(&chip->list, &gpio_chips);
+		return 0;
 	}
 
-	/* are we stepping on the chip right before? */
-	if (pos != &gpio_chips && pos->prev != &gpio_chips) {
-		_chip = list_entry(pos->prev, struct gpio_chip, list);
-		if (_chip->base + _chip->ngpio > chip->base) {
-			dev_err(chip->dev,
-			       "GPIO integer space overlap, cannot add chip\n");
-			err = -EBUSY;
+	list_for_each_entry(iterator, &gpio_chips, list) {
+		if (iterator->base >= chip->base + chip->ngpio) {
+			/*
+			 * Iterator is the first GPIO chip so there is no
+			 * previous one
+			 */
+			if (!previous) {
+				goto found;
+			} else {
+				/*
+				 * We found a valid range(means
+				 * [base, base + ngpio - 1]) between previous
+				 * and iterator chip.
+				 */
+				if (previous->base + previous->ngpio
+						<= chip->base)
+					goto found;
+			}
 		}
+		previous = iterator;
+	}
+
+	/*
+	 * We are beyond the last chip in the list and iterator now
+	 * points to the head.
+	 * Let iterator point to the last chip in the list.
+	 */
+
+	iterator = list_last_entry(&gpio_chips, struct gpio_chip, list);
+	if (iterator->base + iterator->ngpio <= chip->base) {
+		list_add(&chip->list, &iterator->list);
+		return 0;
 	}
 
-	if (!err)
-		list_add_tail(&chip->list, pos);
+	dev_err(chip->parent,
+	       "GPIO integer space overlap, cannot add chip\n");
+	return -EBUSY;
 
-	return err;
+found:
+	list_add_tail(&chip->list, &iterator->list);
+	return 0;
 }
 
 /**
@@ -252,7 +275,7 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name)
  * Takes the names from gc->names and checks if they are all unique. If they
  * are, they are assigned to their gpio descriptors.
  *
- * Returns -EEXIST if one of the names is already used for a different GPIO.
+ * Warning if one of the names is already used for a different GPIO.
  */
 static int gpiochip_set_desc_names(struct gpio_chip *gc)
 {
@@ -267,7 +290,7 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc)
 
 		gpio = gpio_name_to_desc(gc->names[i]);
 		if (gpio)
-			dev_warn(gc->dev, "Detected name collision for "
+			dev_warn(gc->parent, "Detected name collision for "
 				 "GPIO name '%s'\n",
 				 gc->names[i]);
 	}
@@ -280,7 +303,7 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc)
 }
 
 /**
- * gpiochip_add() - register a gpio_chip
+ * gpiochip_add_data() - register a gpio_chip
  * @chip: the chip to register, with chip->base initialized
  * Context: potentially before irqs will work
  *
@@ -288,15 +311,15 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc)
  * because the chip->base is invalid or already associated with a
  * different chip.  Otherwise it returns zero as a success code.
  *
- * When gpiochip_add() is called very early during boot, so that GPIOs
- * can be freely used, the chip->dev device must be registered before
+ * When gpiochip_add_data() is called very early during boot, so that GPIOs
+ * can be freely used, the chip->parent device must be registered before
  * the gpio framework's arch_initcall().  Otherwise sysfs initialization
  * for GPIOs will fail rudely.
  *
  * If chip->base is negative, this requests dynamic assignment of
  * a range of valid GPIOs.
  */
-int gpiochip_add(struct gpio_chip *chip)
+int gpiochip_add_data(struct gpio_chip *chip, void *data)
 {
 	unsigned long	flags;
 	int		status = 0;
@@ -308,6 +331,13 @@ int gpiochip_add(struct gpio_chip *chip)
 	if (!descs)
 		return -ENOMEM;
 
+	chip->data = data;
+
+	if (chip->ngpio == 0) {
+		chip_err(chip, "tried to insert a GPIO chip with zero lines\n");
+		return -EINVAL;
+	}
+
 	spin_lock_irqsave(&gpio_lock, flags);
 
 	if (base < 0) {
@@ -348,8 +378,8 @@ int gpiochip_add(struct gpio_chip *chip)
 	INIT_LIST_HEAD(&chip->pin_ranges);
 #endif
 
-	if (!chip->owner && chip->dev && chip->dev->driver)
-		chip->owner = chip->dev->driver->owner;
+	if (!chip->owner && chip->parent && chip->parent->driver)
+		chip->owner = chip->parent->driver->owner;
 
 	status = gpiochip_set_desc_names(chip);
 	if (status)
@@ -389,7 +419,7 @@ err_free_descs:
 		chip->label ? : "generic");
 	return status;
 }
-EXPORT_SYMBOL_GPL(gpiochip_add);
+EXPORT_SYMBOL_GPL(gpiochip_add_data);
 
 /**
  * gpiochip_remove() - unregister a gpio_chip
@@ -424,7 +454,8 @@ void gpiochip_remove(struct gpio_chip *chip)
 	spin_unlock_irqrestore(&gpio_lock, flags);
 
 	if (requested)
-		dev_crit(chip->dev, "REMOVING GPIOCHIP WITH GPIOS STILL REQUESTED\n");
+		dev_crit(chip->parent,
+			 "REMOVING GPIOCHIP WITH GPIOS STILL REQUESTED\n");
 
 	kfree(chip->desc);
 	chip->desc = NULL;
@@ -659,7 +690,7 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
  * gpiochip, providing an irq domain to translate the local IRQs to
  * global irqs in the gpiolib core, and making sure that the gpiochip
  * is passed as chip data to all related functions. Driver callbacks
- * need to use container_of() to get their local state containers back
+ * need to use gpiochip_get_data() to get their local state containers back
  * from the gpiochip passed as chip data. An irqdomain will be stored
  * in the gpiochip that shall be used by the driver to handle IRQ number
  * translation. The gpiochip will need to be initialized and registered
@@ -683,15 +714,16 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 	if (!gpiochip || !irqchip)
 		return -EINVAL;
 
-	if (!gpiochip->dev) {
+	if (!gpiochip->parent) {
 		pr_err("missing gpiochip .dev parent pointer\n");
 		return -EINVAL;
 	}
-	of_node = gpiochip->dev->of_node;
+	of_node = gpiochip->parent->of_node;
 #ifdef CONFIG_OF_GPIO
 	/*
 	 * If the gpiochip has an assigned OF node this takes precedence
-	 * FIXME: get rid of this and use gpiochip->dev->of_node everywhere
+	 * FIXME: get rid of this and use gpiochip->parent->of_node
+	 * everywhere
 	 */
 	if (gpiochip->of_node)
 		of_node = gpiochip->of_node;
@@ -1279,13 +1311,7 @@ static int _gpiod_get_raw_value(const struct gpio_desc *desc)
 	chip = desc->chip;
 	offset = gpio_chip_hwgpio(desc);
 	value = chip->get ? chip->get(chip, offset) : -EIO;
-	/*
-	 * FIXME: fix all drivers to clamp to [0,1] or return negative,
-	 * then change this to:
-	 * value = value < 0 ? value : !!value;
-	 * so we can properly propagate error codes.
-	 */
-	value = !!value;
+	value = value < 0 ? value : !!value;
 	trace_gpio_value(desc_to_gpio(desc), 1, value);
 	return value;
 }
@@ -2512,7 +2538,7 @@ static int gpiolib_seq_show(struct seq_file *s, void *v)
 
 	seq_printf(s, "%sGPIOs %d-%d", (char *)s->private,
 			chip->base, chip->base + chip->ngpio - 1);
-	dev = chip->dev;
+	dev = chip->parent;
 	if (dev)
 		seq_printf(s, ", %s/%s", dev->bus ? dev->bus->name : "no-bus",
 			dev_name(dev));
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index be3a977d0349..99ed3b00ffe9 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -16,7 +16,7 @@
 #include <linux/device.h>
 
 enum of_gpio_flags;
-
+enum gpiod_flags;
 struct acpi_device;
 
 /**
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index c4bf9a1cf4a6..8ae7ab68cb97 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -82,13 +82,13 @@ config DRM_TTM
 
 config DRM_GEM_CMA_HELPER
 	bool
-	depends on DRM && HAVE_DMA_ATTRS
+	depends on DRM
 	help
 	  Choose this if you need the GEM CMA helper functions
 
 config DRM_KMS_CMA_HELPER
 	bool
-	depends on DRM && HAVE_DMA_ATTRS
+	depends on DRM
 	select DRM_GEM_CMA_HELPER
 	select DRM_KMS_FB_HELPER
 	select FB_SYS_FILLRECT
@@ -160,6 +160,7 @@ config DRM_AMDGPU
 	  If M is selected, the module will be called amdgpu.
 
 source "drivers/gpu/drm/amd/amdgpu/Kconfig"
+source "drivers/gpu/drm/amd/powerplay/Kconfig"
 
 source "drivers/gpu/drm/nouveau/Kconfig"
 
@@ -266,3 +267,5 @@ source "drivers/gpu/drm/amd/amdkfd/Kconfig"
 source "drivers/gpu/drm/imx/Kconfig"
 
 source "drivers/gpu/drm/vc4/Kconfig"
+
+source "drivers/gpu/drm/etnaviv/Kconfig"
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 1e9ff4c3e3db..61766dec6a8d 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -21,8 +21,6 @@ drm-$(CONFIG_DRM_PANEL) += drm_panel.o
 drm-$(CONFIG_OF) += drm_of.o
 drm-$(CONFIG_AGP) += drm_agpsupport.o
 
-drm-y += $(drm-m)
-
 drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
 		drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o
 drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
@@ -62,7 +60,7 @@ obj-$(CONFIG_DRM_ARMADA) += armada/
 obj-$(CONFIG_DRM_ATMEL_HLCDC)	+= atmel-hlcdc/
 obj-$(CONFIG_DRM_RCAR_DU) += rcar-du/
 obj-$(CONFIG_DRM_SHMOBILE) +=shmobile/
-obj-$(CONFIG_DRM_OMAP)	+= omapdrm/
+obj-y			+= omapdrm/
 obj-y			+= tilcdc/
 obj-$(CONFIG_DRM_QXL) += qxl/
 obj-$(CONFIG_DRM_BOCHS) += bochs/
@@ -75,3 +73,4 @@ obj-y			+= i2c/
 obj-y			+= panel/
 obj-y			+= bridge/
 obj-$(CONFIG_DRM_FSL_DCU) += fsl-dcu/
+obj-$(CONFIG_DRM_ETNAVIV) += etnaviv/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 04c270757030..66f729eaf00b 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -2,10 +2,13 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/asic_reg \
-	-Idrivers/gpu/drm/amd/include \
-	-Idrivers/gpu/drm/amd/amdgpu \
-	-Idrivers/gpu/drm/amd/scheduler
+FULL_AMD_PATH=$(src)/..
+
+ccflags-y := -Iinclude/drm -I$(FULL_AMD_PATH)/include/asic_reg \
+	-I$(FULL_AMD_PATH)/include \
+	-I$(FULL_AMD_PATH)/amdgpu \
+	-I$(FULL_AMD_PATH)/scheduler \
+	-I$(FULL_AMD_PATH)/powerplay/inc
 
 amdgpu-y := amdgpu_drv.o
 
@@ -44,6 +47,7 @@ amdgpu-y += \
 # add SMC block
 amdgpu-y += \
 	amdgpu_dpm.o \
+	amdgpu_powerplay.o \
 	cz_smc.o cz_dpm.o \
 	tonga_smc.o tonga_dpm.o \
 	fiji_smc.o fiji_dpm.o \
@@ -94,6 +98,14 @@ amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
 amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
 amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
 
+ifneq ($(CONFIG_DRM_AMD_POWERPLAY),)
+
+include $(FULL_AMD_PATH)/powerplay/Makefile
+
+amdgpu-y += $(AMD_POWERPLAY_FILES)
+
+endif
+
 obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
 
 CFLAGS_amdgpu_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 048cfe073dae..82edf95b7740 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -52,6 +52,7 @@
 #include "amdgpu_irq.h"
 #include "amdgpu_ucode.h"
 #include "amdgpu_gds.h"
+#include "amd_powerplay.h"
 
 #include "gpu_scheduler.h"
 
@@ -85,6 +86,7 @@ extern int amdgpu_enable_scheduler;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
 extern int amdgpu_enable_semaphores;
+extern int amdgpu_powerplay;
 
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 #define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
@@ -918,8 +920,8 @@ struct amdgpu_ring {
 #define AMDGPU_VM_FAULT_STOP_ALWAYS	2
 
 struct amdgpu_vm_pt {
-	struct amdgpu_bo	*bo;
-	uint64_t		addr;
+	struct amdgpu_bo_list_entry	entry;
+	uint64_t			addr;
 };
 
 struct amdgpu_vm_id {
@@ -981,9 +983,12 @@ struct amdgpu_vm_manager {
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev,
-					       struct amdgpu_vm *vm,
-					       struct list_head *head);
+void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
+			 struct list_head *validated,
+			 struct amdgpu_bo_list_entry *entry);
+void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates);
+void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
+				  struct amdgpu_vm *vm);
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		      struct amdgpu_sync *sync);
 void amdgpu_vm_flush(struct amdgpu_ring *ring,
@@ -1024,11 +1029,9 @@ int amdgpu_vm_free_job(struct amdgpu_job *job);
  * context related structures
  */
 
-#define AMDGPU_CTX_MAX_CS_PENDING	16
-
 struct amdgpu_ctx_ring {
 	uint64_t		sequence;
-	struct fence		*fences[AMDGPU_CTX_MAX_CS_PENDING];
+	struct fence		**fences;
 	struct amd_sched_entity	entity;
 };
 
@@ -1037,6 +1040,7 @@ struct amdgpu_ctx {
 	struct amdgpu_device    *adev;
 	unsigned		reset_counter;
 	spinlock_t		ring_lock;
+	struct fence            **fences;
 	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
 };
 
@@ -1047,7 +1051,7 @@ struct amdgpu_ctx_mgr {
 	struct idr		ctx_handles;
 };
 
-int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
+int amdgpu_ctx_init(struct amdgpu_device *adev, enum amd_sched_priority pri,
 		    struct amdgpu_ctx *ctx);
 void amdgpu_ctx_fini(struct amdgpu_ctx *ctx);
 
@@ -1254,7 +1258,7 @@ struct amdgpu_cs_parser {
 	unsigned		nchunks;
 	struct amdgpu_cs_chunk	*chunks;
 	/* relocations */
-	struct amdgpu_bo_list_entry	*vm_bos;
+	struct amdgpu_bo_list_entry	vm_pd;
 	struct list_head	validated;
 	struct fence		*fence;
 
@@ -1301,31 +1305,7 @@ struct amdgpu_wb {
 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
 
-/**
- * struct amdgpu_pm - power management datas
- * It keeps track of various data needed to take powermanagement decision.
- */
 
-enum amdgpu_pm_state_type {
-	/* not used for dpm */
-	POWER_STATE_TYPE_DEFAULT,
-	POWER_STATE_TYPE_POWERSAVE,
-	/* user selectable states */
-	POWER_STATE_TYPE_BATTERY,
-	POWER_STATE_TYPE_BALANCED,
-	POWER_STATE_TYPE_PERFORMANCE,
-	/* internal states */
-	POWER_STATE_TYPE_INTERNAL_UVD,
-	POWER_STATE_TYPE_INTERNAL_UVD_SD,
-	POWER_STATE_TYPE_INTERNAL_UVD_HD,
-	POWER_STATE_TYPE_INTERNAL_UVD_HD2,
-	POWER_STATE_TYPE_INTERNAL_UVD_MVC,
-	POWER_STATE_TYPE_INTERNAL_BOOT,
-	POWER_STATE_TYPE_INTERNAL_THERMAL,
-	POWER_STATE_TYPE_INTERNAL_ACPI,
-	POWER_STATE_TYPE_INTERNAL_ULV,
-	POWER_STATE_TYPE_INTERNAL_3DPERF,
-};
 
 enum amdgpu_int_thermal_type {
 	THERMAL_TYPE_NONE,
@@ -1607,8 +1587,8 @@ struct amdgpu_dpm {
 	/* vce requirements */
 	struct amdgpu_vce_state vce_states[AMDGPU_MAX_VCE_LEVELS];
 	enum amdgpu_vce_level vce_level;
-	enum amdgpu_pm_state_type state;
-	enum amdgpu_pm_state_type user_state;
+	enum amd_pm_state_type state;
+	enum amd_pm_state_type user_state;
 	u32                     platform_caps;
 	u32                     voltage_response_time;
 	u32                     backbias_response_time;
@@ -1661,8 +1641,13 @@ struct amdgpu_pm {
 	const struct firmware	*fw;	/* SMC firmware */
 	uint32_t                fw_version;
 	const struct amdgpu_dpm_funcs *funcs;
+	uint32_t                pcie_gen_mask;
+	uint32_t                pcie_mlw_mask;
+	struct amd_pp_display_configuration pm_display_cfg;/* set by DAL */
 };
 
+void amdgpu_get_pcie_info(struct amdgpu_device *adev);
+
 /*
  * UVD
  */
@@ -1830,6 +1815,8 @@ struct amdgpu_cu_info {
  */
 struct amdgpu_asic_funcs {
 	bool (*read_disabled_bios)(struct amdgpu_device *adev);
+	bool (*read_bios_from_rom)(struct amdgpu_device *adev,
+				   u8 *bios, u32 length_bytes);
 	int (*read_register)(struct amdgpu_device *adev, u32 se_num,
 			     u32 sh_num, u32 reg_offset, u32 *value);
 	void (*set_vga_state)(struct amdgpu_device *adev, bool state);
@@ -2060,6 +2047,10 @@ struct amdgpu_device {
 	/* interrupts */
 	struct amdgpu_irq		irq;
 
+	/* powerplay */
+	struct amd_powerplay		powerplay;
+	bool				pp_enabled;
+
 	/* dpm */
 	struct amdgpu_pm		pm;
 	u32				cg_flags;
@@ -2236,6 +2227,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
 #define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
 #define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
+#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
 #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
 #define amdgpu_asic_get_cu_info(adev, info) (adev)->asic_funcs->get_cu_info((adev), (info))
 #define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))
@@ -2277,24 +2269,78 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))
 #define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))
 #define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
-#define amdgpu_dpm_get_temperature(adev) (adev)->pm.funcs->get_temperature((adev))
 #define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
 #define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
 #define amdgpu_dpm_post_set_power_state(adev) (adev)->pm.funcs->post_set_power_state((adev))
 #define amdgpu_dpm_display_configuration_changed(adev) (adev)->pm.funcs->display_configuration_changed((adev))
-#define amdgpu_dpm_get_sclk(adev, l) (adev)->pm.funcs->get_sclk((adev), (l))
-#define amdgpu_dpm_get_mclk(adev, l) (adev)->pm.funcs->get_mclk((adev), (l))
 #define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
-#define amdgpu_dpm_debugfs_print_current_performance_level(adev, m) (adev)->pm.funcs->debugfs_print_current_performance_level((adev), (m))
-#define amdgpu_dpm_force_performance_level(adev, l) (adev)->pm.funcs->force_performance_level((adev), (l))
 #define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
-#define amdgpu_dpm_powergate_uvd(adev, g) (adev)->pm.funcs->powergate_uvd((adev), (g))
-#define amdgpu_dpm_powergate_vce(adev, g) (adev)->pm.funcs->powergate_vce((adev), (g))
 #define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
-#define amdgpu_dpm_set_fan_control_mode(adev, m) (adev)->pm.funcs->set_fan_control_mode((adev), (m))
-#define amdgpu_dpm_get_fan_control_mode(adev) (adev)->pm.funcs->get_fan_control_mode((adev))
-#define amdgpu_dpm_set_fan_speed_percent(adev, s) (adev)->pm.funcs->set_fan_speed_percent((adev), (s))
-#define amdgpu_dpm_get_fan_speed_percent(adev, s) (adev)->pm.funcs->get_fan_speed_percent((adev), (s))
+
+#define amdgpu_dpm_get_temperature(adev) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
+	      (adev)->pm.funcs->get_temperature((adev)))
+
+#define amdgpu_dpm_set_fan_control_mode(adev, m) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)) : \
+	      (adev)->pm.funcs->set_fan_control_mode((adev), (m)))
+
+#define amdgpu_dpm_get_fan_control_mode(adev) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle) : \
+	      (adev)->pm.funcs->get_fan_control_mode((adev)))
+
+#define amdgpu_dpm_set_fan_speed_percent(adev, s) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
+	      (adev)->pm.funcs->set_fan_speed_percent((adev), (s)))
+
+#define amdgpu_dpm_get_fan_speed_percent(adev, s) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
+	      (adev)->pm.funcs->get_fan_speed_percent((adev), (s)))
+
+#define amdgpu_dpm_get_sclk(adev, l) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)) : \
+		(adev)->pm.funcs->get_sclk((adev), (l)))
+
+#define amdgpu_dpm_get_mclk(adev, l)  \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)) : \
+	      (adev)->pm.funcs->get_mclk((adev), (l)))
+
+
+#define amdgpu_dpm_force_performance_level(adev, l) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)) : \
+	      (adev)->pm.funcs->force_performance_level((adev), (l)))
+
+#define amdgpu_dpm_powergate_uvd(adev, g) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)) : \
+	      (adev)->pm.funcs->powergate_uvd((adev), (g)))
+
+#define amdgpu_dpm_powergate_vce(adev, g) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
+	      (adev)->pm.funcs->powergate_vce((adev), (g)))
+
+#define amdgpu_dpm_debugfs_print_current_performance_level(adev, m) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->print_current_performance_level((adev)->powerplay.pp_handle, (m)) : \
+	      (adev)->pm.funcs->debugfs_print_current_performance_level((adev), (m)))
+
+#define amdgpu_dpm_get_current_power_state(adev) \
+	(adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_get_performance_level(adev) \
+	(adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_dispatch_task(adev, event_id, input, output)		\
+	(adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
 
 #define amdgpu_gds_switch(adev, r, v, d, w, a) (adev)->gds.funcs->patch_gds_switch((r), (v), (d), (w), (a))
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index a142d5ae148d..5cd7b736a9de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -29,66 +29,10 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include "amdgpu.h"
-#include "amdgpu_acpi.h"
+#include "amd_acpi.h"
 #include "atom.h"
 
-#define ACPI_AC_CLASS           "ac_adapter"
-
 extern void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev);
-
-struct atif_verify_interface {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u16 version;		/* version */
-	u32 notification_mask;	/* supported notifications mask */
-	u32 function_bits;	/* supported functions bit vector */
-} __packed;
-
-struct atif_system_params {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u32 valid_mask;		/* valid flags mask */
-	u32 flags;		/* flags */
-	u8 command_code;	/* notify command code */
-} __packed;
-
-struct atif_sbios_requests {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u32 pending;		/* pending sbios requests */
-	u8 panel_exp_mode;	/* panel expansion mode */
-	u8 thermal_gfx;		/* thermal state: target gfx controller */
-	u8 thermal_state;	/* thermal state: state id (0: exit state, non-0: state) */
-	u8 forced_power_gfx;	/* forced power state: target gfx controller */
-	u8 forced_power_state;	/* forced power state: state id */
-	u8 system_power_src;	/* system power source */
-	u8 backlight_level;	/* panel backlight level (0-255) */
-} __packed;
-
-#define ATIF_NOTIFY_MASK	0x3
-#define ATIF_NOTIFY_NONE	0
-#define ATIF_NOTIFY_81		1
-#define ATIF_NOTIFY_N		2
-
-struct atcs_verify_interface {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u16 version;		/* version */
-	u32 function_bits;	/* supported functions bit vector */
-} __packed;
-
-#define ATCS_VALID_FLAGS_MASK	0x3
-
-struct atcs_pref_req_input {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u16 client_id;		/* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */
-	u16 valid_flags_mask;	/* valid flags mask */
-	u16 flags;		/* flags */
-	u8 req_type;		/* request type */
-	u8 perf_req;		/* performance request */
-} __packed;
-
-struct atcs_pref_req_output {
-	u16 size;		/* structure size in bytes (includes size field) */
-	u8 ret_val;		/* return value */
-} __packed;
-
 /* Call the ATIF method
  */
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 5a8fbadbd27b..3c895863fcf5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -11,7 +11,7 @@
 #include <linux/acpi.h>
 #include <linux/pci.h>
 
-#include "amdgpu_acpi.h"
+#include "amd_acpi.h"
 
 struct amdgpu_atpx_functions {
 	bool px_params;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index c44c0c6afd1b..80add22375ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -35,6 +35,13 @@
  * BIOS.
  */
 
+#define AMD_VBIOS_SIGNATURE " 761295520"
+#define AMD_VBIOS_SIGNATURE_OFFSET 0x30
+#define AMD_VBIOS_SIGNATURE_SIZE sizeof(AMD_VBIOS_SIGNATURE)
+#define AMD_VBIOS_SIGNATURE_END (AMD_VBIOS_SIGNATURE_OFFSET + AMD_VBIOS_SIGNATURE_SIZE)
+#define AMD_IS_VALID_VBIOS(p) ((p)[0] == 0x55 && (p)[1] == 0xAA)
+#define AMD_VBIOS_LENGTH(p) ((p)[2] << 9)
+
 /* If you boot an IGP board with a discrete card as the primary,
  * the IGP rom is not accessible via the rom bar as the IGP rom is
  * part of the system bios.  On boot, the system bios puts a
@@ -58,7 +65,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
 		return false;
 	}
 
-	if (size == 0 || bios[0] != 0x55 || bios[1] != 0xaa) {
+	if (size == 0 || !AMD_IS_VALID_VBIOS(bios)) {
 		iounmap(bios);
 		return false;
 	}
@@ -74,7 +81,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
 
 bool amdgpu_read_bios(struct amdgpu_device *adev)
 {
-	uint8_t __iomem *bios, val1, val2;
+	uint8_t __iomem *bios, val[2];
 	size_t size;
 
 	adev->bios = NULL;
@@ -84,10 +91,10 @@ bool amdgpu_read_bios(struct amdgpu_device *adev)
 		return false;
 	}
 
-	val1 = readb(&bios[0]);
-	val2 = readb(&bios[1]);
+	val[0] = readb(&bios[0]);
+	val[1] = readb(&bios[1]);
 
-	if (size == 0 || val1 != 0x55 || val2 != 0xaa) {
+	if (size == 0 || !AMD_IS_VALID_VBIOS(val)) {
 		pci_unmap_rom(adev->pdev, bios);
 		return false;
 	}
@@ -101,6 +108,38 @@ bool amdgpu_read_bios(struct amdgpu_device *adev)
 	return true;
 }
 
+static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev)
+{
+	u8 header[AMD_VBIOS_SIGNATURE_END+1] = {0};
+	int len;
+
+	if (!adev->asic_funcs->read_bios_from_rom)
+		return false;
+
+	/* validate VBIOS signature */
+	if (amdgpu_asic_read_bios_from_rom(adev, &header[0], sizeof(header)) == false)
+		return false;
+	header[AMD_VBIOS_SIGNATURE_END] = 0;
+
+	if ((!AMD_IS_VALID_VBIOS(header)) ||
+	    0 != memcmp((char *)&header[AMD_VBIOS_SIGNATURE_OFFSET],
+			AMD_VBIOS_SIGNATURE,
+			strlen(AMD_VBIOS_SIGNATURE)))
+		return false;
+
+	/* valid vbios, go on */
+	len = AMD_VBIOS_LENGTH(header);
+	len = ALIGN(len, 4);
+	adev->bios = kmalloc(len, GFP_KERNEL);
+	if (!adev->bios) {
+		DRM_ERROR("no memory to allocate for BIOS\n");
+		return false;
+	}
+
+	/* read complete BIOS */
+	return amdgpu_asic_read_bios_from_rom(adev, adev->bios, len);
+}
+
 static bool amdgpu_read_platform_bios(struct amdgpu_device *adev)
 {
 	uint8_t __iomem *bios;
@@ -113,7 +152,7 @@ static bool amdgpu_read_platform_bios(struct amdgpu_device *adev)
 		return false;
 	}
 
-	if (size == 0 || bios[0] != 0x55 || bios[1] != 0xaa) {
+	if (size == 0 || !AMD_IS_VALID_VBIOS(bios)) {
 		return false;
 	}
 	adev->bios = kmemdup(bios, size, GFP_KERNEL);
@@ -230,7 +269,7 @@ static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev)
 			break;
 	}
 
-	if (i == 0 || adev->bios[0] != 0x55 || adev->bios[1] != 0xaa) {
+	if (i == 0 || !AMD_IS_VALID_VBIOS(adev->bios)) {
 		kfree(adev->bios);
 		return false;
 	}
@@ -320,6 +359,9 @@ bool amdgpu_get_bios(struct amdgpu_device *adev)
 	if (r == false)
 		r = amdgpu_read_bios(adev);
 	if (r == false) {
+		r = amdgpu_read_bios_from_rom(adev);
+	}
+	if (r == false) {
 		r = amdgpu_read_disabled_bios(adev);
 	}
 	if (r == false) {
@@ -330,7 +372,7 @@ bool amdgpu_get_bios(struct amdgpu_device *adev)
 		adev->bios = NULL;
 		return false;
 	}
-	if (adev->bios[0] != 0x55 || adev->bios[1] != 0xaa) {
+	if (!AMD_IS_VALID_VBIOS(adev->bios)) {
 		printk("BIOS signature incorrect %x %x\n", adev->bios[0], adev->bios[1]);
 		goto free_bios;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 8e995148f56e..a081dda9fa2f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -24,6 +24,7 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
+#include <linux/acpi.h>
 #include <drm/drmP.h>
 #include <linux/firmware.h>
 #include <drm/amdgpu_drm.h>
@@ -32,7 +33,6 @@
 #include "atom.h"
 #include "amdgpu_ucode.h"
 
-
 struct amdgpu_cgs_device {
 	struct cgs_device base;
 	struct amdgpu_device *adev;
@@ -398,6 +398,41 @@ static void amdgpu_cgs_write_pci_config_dword(void *cgs_device, unsigned addr,
 	WARN(ret, "pci_write_config_dword error");
 }
 
+
+static int amdgpu_cgs_get_pci_resource(void *cgs_device,
+				       enum cgs_resource_type resource_type,
+				       uint64_t size,
+				       uint64_t offset,
+				       uint64_t *resource_base)
+{
+	CGS_FUNC_ADEV;
+
+	if (resource_base == NULL)
+		return -EINVAL;
+
+	switch (resource_type) {
+	case CGS_RESOURCE_TYPE_MMIO:
+		if (adev->rmmio_size == 0)
+			return -ENOENT;
+		if ((offset + size) > adev->rmmio_size)
+			return -EINVAL;
+		*resource_base = adev->rmmio_base;
+		return 0;
+	case CGS_RESOURCE_TYPE_DOORBELL:
+		if (adev->doorbell.size == 0)
+			return -ENOENT;
+		if ((offset + size) > adev->doorbell.size)
+			return -EINVAL;
+		*resource_base = adev->doorbell.base;
+		return 0;
+	case CGS_RESOURCE_TYPE_FB:
+	case CGS_RESOURCE_TYPE_IO:
+	case CGS_RESOURCE_TYPE_ROM:
+	default:
+		return -EINVAL;
+	}
+}
+
 static const void *amdgpu_cgs_atom_get_data_table(void *cgs_device,
 						  unsigned table, uint16_t *size,
 						  uint8_t *frev, uint8_t *crev)
@@ -703,6 +738,9 @@ static int amdgpu_cgs_get_firmware_info(void *cgs_device,
 		case CHIP_TONGA:
 			strcpy(fw_name, "amdgpu/tonga_smc.bin");
 			break;
+		case CHIP_FIJI:
+			strcpy(fw_name, "amdgpu/fiji_smc.bin");
+			break;
 		default:
 			DRM_ERROR("SMC firmware not supported\n");
 			return -EINVAL;
@@ -736,6 +774,288 @@ static int amdgpu_cgs_get_firmware_info(void *cgs_device,
 	return 0;
 }
 
+static int amdgpu_cgs_query_system_info(void *cgs_device,
+				struct cgs_system_info *sys_info)
+{
+	CGS_FUNC_ADEV;
+
+	if (NULL == sys_info)
+		return -ENODEV;
+
+	if (sizeof(struct cgs_system_info) != sys_info->size)
+		return -ENODEV;
+
+	switch (sys_info->info_id) {
+	case CGS_SYSTEM_INFO_ADAPTER_BDF_ID:
+		sys_info->value = adev->pdev->devfn | (adev->pdev->bus->number << 8);
+		break;
+	case CGS_SYSTEM_INFO_PCIE_GEN_INFO:
+		sys_info->value = adev->pm.pcie_gen_mask;
+		break;
+	case CGS_SYSTEM_INFO_PCIE_MLW:
+		sys_info->value = adev->pm.pcie_mlw_mask;
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int amdgpu_cgs_get_active_displays_info(void *cgs_device,
+					  struct cgs_display_info *info)
+{
+	CGS_FUNC_ADEV;
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct drm_device *ddev = adev->ddev;
+	struct drm_crtc *crtc;
+	uint32_t line_time_us, vblank_lines;
+
+	if (info == NULL)
+		return -EINVAL;
+
+	if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
+		list_for_each_entry(crtc,
+				&ddev->mode_config.crtc_list, head) {
+			amdgpu_crtc = to_amdgpu_crtc(crtc);
+			if (crtc->enabled) {
+				info->active_display_mask |= (1 << amdgpu_crtc->crtc_id);
+				info->display_count++;
+			}
+			if (info->mode_info != NULL &&
+				crtc->enabled && amdgpu_crtc->enabled &&
+				amdgpu_crtc->hw_mode.clock) {
+				line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) /
+							amdgpu_crtc->hw_mode.clock;
+				vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end -
+							amdgpu_crtc->hw_mode.crtc_vdisplay +
+							(amdgpu_crtc->v_border * 2);
+				info->mode_info->vblank_time_us = vblank_lines * line_time_us;
+				info->mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
+				info->mode_info->ref_clock = adev->clock.spll.reference_freq;
+				info->mode_info++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/** \brief evaluate acpi namespace object, handle or pathname must be valid
+ *  \param cgs_device
+ *  \param info input/output arguments for the control method
+ *  \return status
+ */
+
+#if defined(CONFIG_ACPI)
+static int amdgpu_cgs_acpi_eval_object(void *cgs_device,
+				    struct cgs_acpi_method_info *info)
+{
+	CGS_FUNC_ADEV;
+	acpi_handle handle;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *params = NULL;
+	union acpi_object *obj = NULL;
+	uint8_t name[5] = {'\0'};
+	struct cgs_acpi_method_argument *argument = NULL;
+	uint32_t i, count;
+	acpi_status status;
+	int result;
+	uint32_t func_no = 0xFFFFFFFF;
+
+	handle = ACPI_HANDLE(&adev->pdev->dev);
+	if (!handle)
+		return -ENODEV;
+
+	memset(&input, 0, sizeof(struct acpi_object_list));
+
+	/* validate input info */
+	if (info->size != sizeof(struct cgs_acpi_method_info))
+		return -EINVAL;
+
+	input.count = info->input_count;
+	if (info->input_count > 0) {
+		if (info->pinput_argument == NULL)
+			return -EINVAL;
+		argument = info->pinput_argument;
+		func_no = argument->value;
+		for (i = 0; i < info->input_count; i++) {
+			if (((argument->type == ACPI_TYPE_STRING) ||
+			     (argument->type == ACPI_TYPE_BUFFER)) &&
+			    (argument->pointer == NULL))
+				return -EINVAL;
+			argument++;
+		}
+	}
+
+	if (info->output_count > 0) {
+		if (info->poutput_argument == NULL)
+			return -EINVAL;
+		argument = info->poutput_argument;
+		for (i = 0; i < info->output_count; i++) {
+			if (((argument->type == ACPI_TYPE_STRING) ||
+				(argument->type == ACPI_TYPE_BUFFER))
+				&& (argument->pointer == NULL))
+				return -EINVAL;
+			argument++;
+		}
+	}
+
+	/* The path name passed to acpi_evaluate_object should be null terminated */
+	if ((info->field & CGS_ACPI_FIELD_METHOD_NAME) != 0) {
+		strncpy(name, (char *)&(info->name), sizeof(uint32_t));
+		name[4] = '\0';
+	}
+
+	/* parse input parameters */
+	if (input.count > 0) {
+		input.pointer = params =
+				kzalloc(sizeof(union acpi_object) * input.count, GFP_KERNEL);
+		if (params == NULL)
+			return -EINVAL;
+
+		argument = info->pinput_argument;
+
+		for (i = 0; i < input.count; i++) {
+			params->type = argument->type;
+			switch (params->type) {
+			case ACPI_TYPE_INTEGER:
+				params->integer.value = argument->value;
+				break;
+			case ACPI_TYPE_STRING:
+				params->string.length = argument->method_length;
+				params->string.pointer = argument->pointer;
+				break;
+			case ACPI_TYPE_BUFFER:
+				params->buffer.length = argument->method_length;
+				params->buffer.pointer = argument->pointer;
+				break;
+			default:
+				break;
+			}
+			params++;
+			argument++;
+		}
+	}
+
+	/* parse output info */
+	count = info->output_count;
+	argument = info->poutput_argument;
+
+	/* evaluate the acpi method */
+	status = acpi_evaluate_object(handle, name, &input, &output);
+
+	if (ACPI_FAILURE(status)) {
+		result = -EIO;
+		goto error;
+	}
+
+	/* return the output info */
+	obj = output.pointer;
+
+	if (count > 1) {
+		if ((obj->type != ACPI_TYPE_PACKAGE) ||
+			(obj->package.count != count)) {
+			result = -EIO;
+			goto error;
+		}
+		params = obj->package.elements;
+	} else
+		params = obj;
+
+	if (params == NULL) {
+		result = -EIO;
+		goto error;
+	}
+
+	for (i = 0; i < count; i++) {
+		if (argument->type != params->type) {
+			result = -EIO;
+			goto error;
+		}
+		switch (params->type) {
+		case ACPI_TYPE_INTEGER:
+			argument->value = params->integer.value;
+			break;
+		case ACPI_TYPE_STRING:
+			if ((params->string.length != argument->data_length) ||
+				(params->string.pointer == NULL)) {
+				result = -EIO;
+				goto error;
+			}
+			strncpy(argument->pointer,
+				params->string.pointer,
+				params->string.length);
+			break;
+		case ACPI_TYPE_BUFFER:
+			if (params->buffer.pointer == NULL) {
+				result = -EIO;
+				goto error;
+			}
+			memcpy(argument->pointer,
+				params->buffer.pointer,
+				argument->data_length);
+			break;
+		default:
+			break;
+		}
+		argument++;
+		params++;
+	}
+
+error:
+	if (obj != NULL)
+		kfree(obj);
+	kfree((void *)input.pointer);
+	return result;
+}
+#else
+static int amdgpu_cgs_acpi_eval_object(void *cgs_device,
+				struct cgs_acpi_method_info *info)
+{
+	return -EIO;
+}
+#endif
+
+int amdgpu_cgs_call_acpi_method(void *cgs_device,
+					uint32_t acpi_method,
+					uint32_t acpi_function,
+					void *pinput, void *poutput,
+					uint32_t output_count,
+					uint32_t input_size,
+					uint32_t output_size)
+{
+	struct cgs_acpi_method_argument acpi_input[2] = { {0}, {0} };
+	struct cgs_acpi_method_argument acpi_output = {0};
+	struct cgs_acpi_method_info info = {0};
+
+	acpi_input[0].type = CGS_ACPI_TYPE_INTEGER;
+	acpi_input[0].method_length = sizeof(uint32_t);
+	acpi_input[0].data_length = sizeof(uint32_t);
+	acpi_input[0].value = acpi_function;
+
+	acpi_input[1].type = CGS_ACPI_TYPE_BUFFER;
+	acpi_input[1].method_length = CGS_ACPI_MAX_BUFFER_SIZE;
+	acpi_input[1].data_length = input_size;
+	acpi_input[1].pointer = pinput;
+
+	acpi_output.type = CGS_ACPI_TYPE_BUFFER;
+	acpi_output.method_length = CGS_ACPI_MAX_BUFFER_SIZE;
+	acpi_output.data_length = output_size;
+	acpi_output.pointer = poutput;
+
+	info.size = sizeof(struct cgs_acpi_method_info);
+	info.field = CGS_ACPI_FIELD_METHOD_NAME | CGS_ACPI_FIELD_INPUT_ARGUMENT_COUNT;
+	info.input_count = 2;
+	info.name = acpi_method;
+	info.pinput_argument = acpi_input;
+	info.output_count = output_count;
+	info.poutput_argument = &acpi_output;
+
+	return amdgpu_cgs_acpi_eval_object(cgs_device, &info);
+}
+
 static const struct cgs_ops amdgpu_cgs_ops = {
 	amdgpu_cgs_gpu_mem_info,
 	amdgpu_cgs_gmap_kmem,
@@ -756,6 +1076,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
 	amdgpu_cgs_write_pci_config_byte,
 	amdgpu_cgs_write_pci_config_word,
 	amdgpu_cgs_write_pci_config_dword,
+	amdgpu_cgs_get_pci_resource,
 	amdgpu_cgs_atom_get_data_table,
 	amdgpu_cgs_atom_get_cmd_table_revs,
 	amdgpu_cgs_atom_exec_cmd_table,
@@ -768,7 +1089,10 @@ static const struct cgs_ops amdgpu_cgs_ops = {
 	amdgpu_cgs_set_camera_voltages,
 	amdgpu_cgs_get_firmware_info,
 	amdgpu_cgs_set_powergating_state,
-	amdgpu_cgs_set_clockgating_state
+	amdgpu_cgs_set_clockgating_state,
+	amdgpu_cgs_get_active_displays_info,
+	amdgpu_cgs_call_acpi_method,
+	amdgpu_cgs_query_system_info,
 };
 
 static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 25a3e2485cc2..b882e8175615 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -406,8 +406,8 @@ static int amdgpu_cs_parser_relocs(struct amdgpu_cs_parser *p)
 		amdgpu_cs_buckets_get_list(&buckets, &p->validated);
 	}
 
-	p->vm_bos = amdgpu_vm_get_bos(p->adev, &fpriv->vm,
-				      &p->validated);
+	INIT_LIST_HEAD(&duplicates);
+	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
 
 	if (p->uf.bo)
 		list_add(&p->uf_entry.tv.head, &p->validated);
@@ -415,20 +415,23 @@ static int amdgpu_cs_parser_relocs(struct amdgpu_cs_parser *p)
 	if (need_mmap_lock)
 		down_read(&current->mm->mmap_sem);
 
-	INIT_LIST_HEAD(&duplicates);
 	r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates);
 	if (unlikely(r != 0))
 		goto error_reserve;
 
-	r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &p->validated);
+	amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
+
+	r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &duplicates);
 	if (r)
 		goto error_validate;
 
-	r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &duplicates);
+	r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &p->validated);
 
 error_validate:
-	if (r)
+	if (r) {
+		amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
 		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
+	}
 
 error_reserve:
 	if (need_mmap_lock)
@@ -472,9 +475,12 @@ static int cmp_size_smaller_first(void *priv, struct list_head *a,
  **/
 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
 {
+	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
 	unsigned i;
 
 	if (!error) {
+		amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
+
 		/* Sort the buffer list from the smallest to largest buffer,
 		 * which affects the order of buffers in the LRU list.
 		 * This assures that the smallest buffers are added first
@@ -501,7 +507,6 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo
 	if (parser->bo_list)
 		amdgpu_bo_list_put(parser->bo_list);
 
-	drm_free_large(parser->vm_bos);
 	for (i = 0; i < parser->nchunks; i++)
 		drm_free_large(parser->chunks[i].kdata);
 	kfree(parser->chunks);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index fec65f01c031..17d1fb12128a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -25,7 +25,7 @@
 #include <drm/drmP.h>
 #include "amdgpu.h"
 
-int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
+int amdgpu_ctx_init(struct amdgpu_device *adev, enum amd_sched_priority pri,
 		    struct amdgpu_ctx *ctx)
 {
 	unsigned i, j;
@@ -35,17 +35,25 @@ int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
 	ctx->adev = adev;
 	kref_init(&ctx->refcount);
 	spin_lock_init(&ctx->ring_lock);
-	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
-		ctx->rings[i].sequence = 1;
+	ctx->fences = kzalloc(sizeof(struct fence *) * amdgpu_sched_jobs *
+			 AMDGPU_MAX_RINGS, GFP_KERNEL);
+	if (!ctx->fences)
+		return -ENOMEM;
 
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		ctx->rings[i].sequence = 1;
+		ctx->rings[i].fences = (void *)ctx->fences + sizeof(struct fence *) *
+			amdgpu_sched_jobs * i;
+	}
 	if (amdgpu_enable_scheduler) {
 		/* create context entity for each ring */
 		for (i = 0; i < adev->num_rings; i++) {
 			struct amd_sched_rq *rq;
-			if (kernel)
-				rq = &adev->rings[i]->sched.kernel_rq;
-			else
-				rq = &adev->rings[i]->sched.sched_rq;
+			if (pri >= AMD_SCHED_MAX_PRIORITY) {
+				kfree(ctx->fences);
+				return -EINVAL;
+			}
+			rq = &adev->rings[i]->sched.sched_rq[pri];
 			r = amd_sched_entity_init(&adev->rings[i]->sched,
 						  &ctx->rings[i].entity,
 						  rq, amdgpu_sched_jobs);
@@ -57,7 +65,7 @@ int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
 			for (j = 0; j < i; j++)
 				amd_sched_entity_fini(&adev->rings[j]->sched,
 						      &ctx->rings[j].entity);
-			kfree(ctx);
+			kfree(ctx->fences);
 			return r;
 		}
 	}
@@ -73,8 +81,9 @@ void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 		return;
 
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
-		for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j)
+		for (j = 0; j < amdgpu_sched_jobs; ++j)
 			fence_put(ctx->rings[i].fences[j]);
+	kfree(ctx->fences);
 
 	if (amdgpu_enable_scheduler) {
 		for (i = 0; i < adev->num_rings; i++)
@@ -103,9 +112,13 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 		return r;
 	}
 	*id = (uint32_t)r;
-	r = amdgpu_ctx_init(adev, false, ctx);
+	r = amdgpu_ctx_init(adev, AMD_SCHED_PRIORITY_NORMAL, ctx);
+	if (r) {
+		idr_remove(&mgr->ctx_handles, *id);
+		*id = 0;
+		kfree(ctx);
+	}
 	mutex_unlock(&mgr->lock);
-
 	return r;
 }
 
@@ -239,7 +252,7 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
 	unsigned idx = 0;
 	struct fence *other = NULL;
 
-	idx = seq % AMDGPU_CTX_MAX_CS_PENDING;
+	idx = seq & (amdgpu_sched_jobs - 1);
 	other = cring->fences[idx];
 	if (other) {
 		signed long r;
@@ -274,12 +287,12 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 	}
 
 
-	if (seq + AMDGPU_CTX_MAX_CS_PENDING < cring->sequence) {
+	if (seq + amdgpu_sched_jobs < cring->sequence) {
 		spin_unlock(&ctx->ring_lock);
 		return NULL;
 	}
 
-	fence = fence_get(cring->fences[seq % AMDGPU_CTX_MAX_CS_PENDING]);
+	fence = fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]);
 	spin_unlock(&ctx->ring_lock);
 
 	return fence;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index d5b421330145..65531463f88e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -38,6 +38,7 @@
 #include "amdgpu_i2c.h"
 #include "atom.h"
 #include "amdgpu_atombios.h"
+#include "amd_pcie.h"
 #ifdef CONFIG_DRM_AMDGPU_CIK
 #include "cik.h"
 #endif
@@ -949,6 +950,15 @@ static bool amdgpu_check_pot_argument(int arg)
  */
 static void amdgpu_check_arguments(struct amdgpu_device *adev)
 {
+	if (amdgpu_sched_jobs < 4) {
+		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
+			 amdgpu_sched_jobs);
+		amdgpu_sched_jobs = 4;
+	} else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
+		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
+			 amdgpu_sched_jobs);
+		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
+	}
 	/* vramlimit must be a power of two */
 	if (!amdgpu_check_pot_argument(amdgpu_vram_limit)) {
 		dev_warn(adev->dev, "vram limit (%d) must be a power of 2\n",
@@ -1214,12 +1224,14 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		} else {
 			if (adev->ip_blocks[i].funcs->early_init) {
 				r = adev->ip_blocks[i].funcs->early_init((void *)adev);
-				if (r == -ENOENT)
+				if (r == -ENOENT) {
 					adev->ip_block_status[i].valid = false;
-				else if (r)
+				} else if (r) {
+					DRM_ERROR("early_init %d failed %d\n", i, r);
 					return r;
-				else
+				} else {
 					adev->ip_block_status[i].valid = true;
+				}
 			} else {
 				adev->ip_block_status[i].valid = true;
 			}
@@ -1237,20 +1249,28 @@ static int amdgpu_init(struct amdgpu_device *adev)
 		if (!adev->ip_block_status[i].valid)
 			continue;
 		r = adev->ip_blocks[i].funcs->sw_init((void *)adev);
-		if (r)
+		if (r) {
+			DRM_ERROR("sw_init %d failed %d\n", i, r);
 			return r;
+		}
 		adev->ip_block_status[i].sw = true;
 		/* need to do gmc hw init early so we can allocate gpu mem */
 		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) {
 			r = amdgpu_vram_scratch_init(adev);
-			if (r)
+			if (r) {
+				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
 				return r;
+			}
 			r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
-			if (r)
+			if (r) {
+				DRM_ERROR("hw_init %d failed %d\n", i, r);
 				return r;
+			}
 			r = amdgpu_wb_init(adev);
-			if (r)
+			if (r) {
+				DRM_ERROR("amdgpu_wb_init failed %d\n", r);
 				return r;
+			}
 			adev->ip_block_status[i].hw = true;
 		}
 	}
@@ -1262,8 +1282,10 @@ static int amdgpu_init(struct amdgpu_device *adev)
 		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC)
 			continue;
 		r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
-		if (r)
+		if (r) {
+			DRM_ERROR("hw_init %d failed %d\n", i, r);
 			return r;
+		}
 		adev->ip_block_status[i].hw = true;
 	}
 
@@ -1280,12 +1302,16 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
 		/* enable clockgating to save power */
 		r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
 								    AMD_CG_STATE_GATE);
-		if (r)
+		if (r) {
+			DRM_ERROR("set_clockgating_state(gate) %d failed %d\n", i, r);
 			return r;
+		}
 		if (adev->ip_blocks[i].funcs->late_init) {
 			r = adev->ip_blocks[i].funcs->late_init((void *)adev);
-			if (r)
+			if (r) {
+				DRM_ERROR("late_init %d failed %d\n", i, r);
 				return r;
+			}
 		}
 	}
 
@@ -1306,10 +1332,15 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 		/* ungate blocks before hw fini so that we can shutdown the blocks safely */
 		r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
 								    AMD_CG_STATE_UNGATE);
-		if (r)
+		if (r) {
+			DRM_ERROR("set_clockgating_state(ungate) %d failed %d\n", i, r);
 			return r;
+		}
 		r = adev->ip_blocks[i].funcs->hw_fini((void *)adev);
 		/* XXX handle errors */
+		if (r) {
+			DRM_DEBUG("hw_fini %d failed %d\n", i, r);
+		}
 		adev->ip_block_status[i].hw = false;
 	}
 
@@ -1318,6 +1349,9 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 			continue;
 		r = adev->ip_blocks[i].funcs->sw_fini((void *)adev);
 		/* XXX handle errors */
+		if (r) {
+			DRM_DEBUG("sw_fini %d failed %d\n", i, r);
+		}
 		adev->ip_block_status[i].sw = false;
 		adev->ip_block_status[i].valid = false;
 	}
@@ -1335,9 +1369,15 @@ static int amdgpu_suspend(struct amdgpu_device *adev)
 		/* ungate blocks so that suspend can properly shut them down */
 		r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
 								    AMD_CG_STATE_UNGATE);
+		if (r) {
+			DRM_ERROR("set_clockgating_state(ungate) %d failed %d\n", i, r);
+		}
 		/* XXX handle errors */
 		r = adev->ip_blocks[i].funcs->suspend(adev);
 		/* XXX handle errors */
+		if (r) {
+			DRM_ERROR("suspend %d failed %d\n", i, r);
+		}
 	}
 
 	return 0;
@@ -1351,8 +1391,10 @@ static int amdgpu_resume(struct amdgpu_device *adev)
 		if (!adev->ip_block_status[i].valid)
 			continue;
 		r = adev->ip_blocks[i].funcs->resume(adev);
-		if (r)
+		if (r) {
+			DRM_ERROR("resume %d failed %d\n", i, r);
 			return r;
+		}
 	}
 
 	return 0;
@@ -1484,8 +1526,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 	r = amdgpu_atombios_init(adev);
-	if (r)
+	if (r) {
+		dev_err(adev->dev, "amdgpu_atombios_init failed\n");
 		return r;
+	}
 
 	/* Post card if necessary */
 	if (!amdgpu_card_posted(adev)) {
@@ -1499,21 +1543,26 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 	/* Initialize clocks */
 	r = amdgpu_atombios_get_clock_info(adev);
-	if (r)
+	if (r) {
+		dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
 		return r;
+	}
 	/* init i2c buses */
 	amdgpu_atombios_i2c_init(adev);
 
 	/* Fence driver */
 	r = amdgpu_fence_driver_init(adev);
-	if (r)
+	if (r) {
+		dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
 		return r;
+	}
 
 	/* init the mode config */
 	drm_mode_config_init(adev->ddev);
 
 	r = amdgpu_init(adev);
 	if (r) {
+		dev_err(adev->dev, "amdgpu_init failed\n");
 		amdgpu_fini(adev);
 		return r;
 	}
@@ -1528,7 +1577,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		return r;
 	}
 
-	r = amdgpu_ctx_init(adev, true, &adev->kernel_ctx);
+	r = amdgpu_ctx_init(adev, AMD_SCHED_PRIORITY_KERNEL, &adev->kernel_ctx);
 	if (r) {
 		dev_err(adev->dev, "failed to create kernel context (%d).\n", r);
 		return r;
@@ -1570,8 +1619,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	 * explicit gating rather than handling it automatically.
 	 */
 	r = amdgpu_late_init(adev);
-	if (r)
+	if (r) {
+		dev_err(adev->dev, "amdgpu_late_init failed\n");
 		return r;
+	}
 
 	return 0;
 }
@@ -1788,6 +1839,7 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
 	}
 
 	drm_kms_helper_poll_enable(dev);
+	drm_helper_hpd_irq_event(dev);
 
 	if (fbcon) {
 		amdgpu_fbdev_set_suspend(adev, 0);
@@ -1881,6 +1933,83 @@ retry:
 	return r;
 }
 
+void amdgpu_get_pcie_info(struct amdgpu_device *adev)
+{
+	u32 mask;
+	int ret;
+
+	if (pci_is_root_bus(adev->pdev->bus))
+		return;
+
+	if (amdgpu_pcie_gen2 == 0)
+		return;
+
+	if (adev->flags & AMD_IS_APU)
+		return;
+
+	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
+	if (!ret) {
+		adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+					  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+					  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
+
+		if (mask & DRM_PCIE_SPEED_25)
+			adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
+		if (mask & DRM_PCIE_SPEED_50)
+			adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
+		if (mask & DRM_PCIE_SPEED_80)
+			adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
+	}
+	ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
+	if (!ret) {
+		switch (mask) {
+		case 32:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 16:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 12:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 8:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 4:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 2:
+			adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+						  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+			break;
+		case 1:
+			adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
+			break;
+		default:
+			break;
+		}
+	}
+}
 
 /*
  * Debugfs
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 5580d3420c3a..acd066d0a805 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -518,7 +518,7 @@ static const struct drm_framebuffer_funcs amdgpu_fb_funcs = {
 int
 amdgpu_framebuffer_init(struct drm_device *dev,
 			struct amdgpu_framebuffer *rfb,
-			struct drm_mode_fb_cmd2 *mode_cmd,
+			const struct drm_mode_fb_cmd2 *mode_cmd,
 			struct drm_gem_object *obj)
 {
 	int ret;
@@ -535,7 +535,7 @@ amdgpu_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 amdgpu_user_framebuffer_create(struct drm_device *dev,
 			       struct drm_file *file_priv,
-			       struct drm_mode_fb_cmd2 *mode_cmd)
+			       const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct amdgpu_framebuffer *amdgpu_fb;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 0508c5cd103a..b5dbbb573491 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -79,9 +79,10 @@ int amdgpu_vm_fault_stop = 0;
 int amdgpu_vm_debug = 0;
 int amdgpu_exp_hw_support = 0;
 int amdgpu_enable_scheduler = 1;
-int amdgpu_sched_jobs = 16;
+int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_enable_semaphores = 0;
+int amdgpu_powerplay = -1;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -155,7 +156,7 @@ module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
 MODULE_PARM_DESC(enable_scheduler, "enable SW GPU scheduler (1 = enable (default), 0 = disable)");
 module_param_named(enable_scheduler, amdgpu_enable_scheduler, int, 0444);
 
-MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 16)");
+MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)");
 module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 
 MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
@@ -164,6 +165,11 @@ module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 MODULE_PARM_DESC(enable_semaphores, "Enable semaphores (1 = enable, 0 = disable (default))");
 module_param_named(enable_semaphores, amdgpu_enable_semaphores, int, 0644);
 
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
+module_param_named(powerplay, amdgpu_powerplay, int, 0444);
+#endif
+
 static struct pci_device_id pciidlist[] = {
 #ifdef CONFIG_DRM_AMDGPU_CIK
 	/* Kaveri */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 093a8c618931..919146780a15 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -45,7 +45,6 @@
 struct amdgpu_fbdev {
 	struct drm_fb_helper helper;
 	struct amdgpu_framebuffer rfb;
-	struct list_head fbdev_list;
 	struct amdgpu_device *adev;
 };
 
@@ -264,7 +263,7 @@ out_unref:
 
 	}
 	if (fb && ret) {
-		drm_gem_object_unreference(gobj);
+		drm_gem_object_unreference_unlocked(gobj);
 		drm_framebuffer_unregister_private(fb);
 		drm_framebuffer_cleanup(fb);
 		kfree(fb);
@@ -334,6 +333,10 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev)
 	if (!adev->mode_info.mode_config_initialized)
 		return 0;
 
+	/* don't init fbdev if there are no connectors */
+	if (list_empty(&adev->ddev->mode_config.connector_list))
+		return 0;
+
 	/* select 8 bpp console on low vram cards */
 	if (adev->mc.real_vram_size <= (32*1024*1024))
 		bpp_sel = 8;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 9c253c535d26..7380f782cd14 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -448,7 +448,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 				    struct amdgpu_bo_va *bo_va, uint32_t operation)
 {
 	struct ttm_validate_buffer tv, *entry;
-	struct amdgpu_bo_list_entry *vm_bos;
+	struct amdgpu_bo_list_entry vm_pd;
 	struct ww_acquire_ctx ticket;
 	struct list_head list, duplicates;
 	unsigned domain;
@@ -461,15 +461,14 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 	tv.shared = true;
 	list_add(&tv.head, &list);
 
-	vm_bos = amdgpu_vm_get_bos(adev, bo_va->vm, &list);
-	if (!vm_bos)
-		return;
+	amdgpu_vm_get_pd_bo(bo_va->vm, &list, &vm_pd);
 
 	/* Provide duplicates to avoid -EALREADY */
 	r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
 	if (r)
-		goto error_free;
+		goto error_print;
 
+	amdgpu_vm_get_pt_bos(bo_va->vm, &duplicates);
 	list_for_each_entry(entry, &list, head) {
 		domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
 		/* if anything is swapped out don't swap it in here,
@@ -499,9 +498,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 error_unreserve:
 	ttm_eu_backoff_reservation(&ticket, &list);
 
-error_free:
-	drm_free_large(vm_bos);
-
+error_print:
 	if (r && r != -ERESTARTSYS)
 		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 7c42ff670080..f594cfaa97e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -25,6 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
+#include <linux/irq.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/amdgpu_drm.h>
@@ -312,6 +313,7 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev, unsigned src_id,
 	}
 
 	adev->irq.sources[src_id] = source;
+
 	return 0;
 }
 
@@ -335,15 +337,19 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev,
 		return;
 	}
 
-	src = adev->irq.sources[src_id];
-	if (!src) {
-		DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id);
-		return;
-	}
+	if (adev->irq.virq[src_id]) {
+		generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id));
+	} else {
+		src = adev->irq.sources[src_id];
+		if (!src) {
+			DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id);
+			return;
+		}
 
-	r = src->funcs->process(adev, src, entry);
-	if (r)
-		DRM_ERROR("error processing interrupt (%d)\n", r);
+		r = src->funcs->process(adev, src, entry);
+		if (r)
+			DRM_ERROR("error processing interrupt (%d)\n", r);
+	}
 }
 
 /**
@@ -461,3 +467,90 @@ bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 
 	return !!atomic_read(&src->enabled_types[type]);
 }
+
+/* gen irq */
+static void amdgpu_irq_mask(struct irq_data *irqd)
+{
+	/* XXX */
+}
+
+static void amdgpu_irq_unmask(struct irq_data *irqd)
+{
+	/* XXX */
+}
+
+static struct irq_chip amdgpu_irq_chip = {
+	.name = "amdgpu-ih",
+	.irq_mask = amdgpu_irq_mask,
+	.irq_unmask = amdgpu_irq_unmask,
+};
+
+static int amdgpu_irqdomain_map(struct irq_domain *d,
+				unsigned int irq, irq_hw_number_t hwirq)
+{
+	if (hwirq >= AMDGPU_MAX_IRQ_SRC_ID)
+		return -EPERM;
+
+	irq_set_chip_and_handler(irq,
+				 &amdgpu_irq_chip, handle_simple_irq);
+	return 0;
+}
+
+static struct irq_domain_ops amdgpu_hw_irqdomain_ops = {
+	.map = amdgpu_irqdomain_map,
+};
+
+/**
+ * amdgpu_irq_add_domain - create a linear irq domain
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Create an irq domain for GPU interrupt sources
+ * that may be driven by another driver (e.g., ACP).
+ */
+int amdgpu_irq_add_domain(struct amdgpu_device *adev)
+{
+	adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,
+						 &amdgpu_hw_irqdomain_ops, adev);
+	if (!adev->irq.domain) {
+		DRM_ERROR("GPU irq add domain failed\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_irq_remove_domain - remove the irq domain
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Remove the irq domain for GPU interrupt sources
+ * that may be driven by another driver (e.g., ACP).
+ */
+void amdgpu_irq_remove_domain(struct amdgpu_device *adev)
+{
+	if (adev->irq.domain) {
+		irq_domain_remove(adev->irq.domain);
+		adev->irq.domain = NULL;
+	}
+}
+
+/**
+ * amdgpu_irq_create_mapping - create a mapping between a domain irq and a
+ *                             Linux irq
+ *
+ * @adev: amdgpu device pointer
+ * @src_id: IH source id
+ *
+ * Create a mapping between a domain irq (GPU IH src id) and a Linux irq
+ * Use this for components that generate a GPU interrupt, but are driven
+ * by a different driver (e.g., ACP).
+ * Returns the Linux irq.
+ */
+unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)
+{
+	adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);
+
+	return adev->irq.virq[src_id];
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index 17b01aef4278..e124b59f39c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -24,6 +24,7 @@
 #ifndef __AMDGPU_IRQ_H__
 #define __AMDGPU_IRQ_H__
 
+#include <linux/irqdomain.h>
 #include "amdgpu_ih.h"
 
 #define AMDGPU_MAX_IRQ_SRC_ID	0x100
@@ -65,6 +66,10 @@ struct amdgpu_irq {
 	/* interrupt ring */
 	struct amdgpu_ih_ring		ih;
 	const struct amdgpu_ih_funcs	*ih_funcs;
+
+	/* gen irq stuff */
+	struct irq_domain		*domain; /* GPU irq controller domain */
+	unsigned			virq[AMDGPU_MAX_IRQ_SRC_ID];
 };
 
 void amdgpu_irq_preinstall(struct drm_device *dev);
@@ -90,4 +95,8 @@ int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 			unsigned type);
 
+int amdgpu_irq_add_domain(struct amdgpu_device *adev);
+void amdgpu_irq_remove_domain(struct amdgpu_device *adev);
+unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 064ebb347074..fdc1be8550da 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -35,6 +35,7 @@
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_fixed.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
@@ -556,7 +557,7 @@ int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 
 int amdgpu_framebuffer_init(struct drm_device *dev,
 			     struct amdgpu_framebuffer *rfb,
-			     struct drm_mode_fb_cmd2 *mode_cmd,
+			     const struct drm_mode_fb_cmd2 *mode_cmd,
 			     struct drm_gem_object *obj);
 
 int amdgpufb_remove(struct drm_device *dev, struct drm_framebuffer *fb);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index c3ce103b6a33..a2a16acee34d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -399,7 +399,8 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		}
 		if (fpfn > bo->placements[i].fpfn)
 			bo->placements[i].fpfn = fpfn;
-		if (lpfn && lpfn < bo->placements[i].lpfn)
+		if (!bo->placements[i].lpfn ||
+		    (lpfn && lpfn < bo->placements[i].lpfn))
 			bo->placements[i].lpfn = lpfn;
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index ea756e77b023..5107fb291bdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -96,6 +96,7 @@ static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo)
  */
 static inline u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
 {
+	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
 	return bo->tbo.offset;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 22a8c7d3a3ab..7d8d84eaea4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -30,10 +30,16 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 
+#include "amd_powerplay.h"
+
 static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev);
 
 void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 {
+	if (adev->pp_enabled)
+		/* TODO */
+		return;
+
 	if (adev->pm.dpm_enabled) {
 		mutex_lock(&adev->pm.mutex);
 		if (power_supply_is_system_supplied() > 0)
@@ -52,7 +58,12 @@ static ssize_t amdgpu_get_dpm_state(struct device *dev,
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
-	enum amdgpu_pm_state_type pm = adev->pm.dpm.user_state;
+	enum amd_pm_state_type pm;
+
+	if (adev->pp_enabled) {
+		pm = amdgpu_dpm_get_current_power_state(adev);
+	} else
+		pm = adev->pm.dpm.user_state;
 
 	return snprintf(buf, PAGE_SIZE, "%s\n",
 			(pm == POWER_STATE_TYPE_BATTERY) ? "battery" :
@@ -66,40 +77,57 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
+	enum amd_pm_state_type  state;
 
-	mutex_lock(&adev->pm.mutex);
 	if (strncmp("battery", buf, strlen("battery")) == 0)
-		adev->pm.dpm.user_state = POWER_STATE_TYPE_BATTERY;
+		state = POWER_STATE_TYPE_BATTERY;
 	else if (strncmp("balanced", buf, strlen("balanced")) == 0)
-		adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
+		state = POWER_STATE_TYPE_BALANCED;
 	else if (strncmp("performance", buf, strlen("performance")) == 0)
-		adev->pm.dpm.user_state = POWER_STATE_TYPE_PERFORMANCE;
+		state = POWER_STATE_TYPE_PERFORMANCE;
 	else {
-		mutex_unlock(&adev->pm.mutex);
 		count = -EINVAL;
 		goto fail;
 	}
-	mutex_unlock(&adev->pm.mutex);
 
-	/* Can't set dpm state when the card is off */
-	if (!(adev->flags & AMD_IS_PX) ||
-	    (ddev->switch_power_state == DRM_SWITCH_POWER_ON))
-		amdgpu_pm_compute_clocks(adev);
+	if (adev->pp_enabled) {
+		amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL);
+	} else {
+		mutex_lock(&adev->pm.mutex);
+		adev->pm.dpm.user_state = state;
+		mutex_unlock(&adev->pm.mutex);
+
+		/* Can't set dpm state when the card is off */
+		if (!(adev->flags & AMD_IS_PX) ||
+		    (ddev->switch_power_state == DRM_SWITCH_POWER_ON))
+			amdgpu_pm_compute_clocks(adev);
+	}
 fail:
 	return count;
 }
 
 static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
-						       struct device_attribute *attr,
-						       char *buf)
+						struct device_attribute *attr,
+								char *buf)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
-	enum amdgpu_dpm_forced_level level = adev->pm.dpm.forced_level;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",
-			(level == AMDGPU_DPM_FORCED_LEVEL_AUTO) ? "auto" :
-			(level == AMDGPU_DPM_FORCED_LEVEL_LOW) ? "low" : "high");
+	if (adev->pp_enabled) {
+		enum amd_dpm_forced_level level;
+
+		level = amdgpu_dpm_get_performance_level(adev);
+		return snprintf(buf, PAGE_SIZE, "%s\n",
+				(level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" :
+				(level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" : "high");
+	} else {
+		enum amdgpu_dpm_forced_level level;
+
+		level = adev->pm.dpm.forced_level;
+		return snprintf(buf, PAGE_SIZE, "%s\n",
+				(level == AMDGPU_DPM_FORCED_LEVEL_AUTO) ? "auto" :
+				(level == AMDGPU_DPM_FORCED_LEVEL_LOW) ? "low" : "high");
+	}
 }
 
 static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
@@ -112,7 +140,6 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 	enum amdgpu_dpm_forced_level level;
 	int ret = 0;
 
-	mutex_lock(&adev->pm.mutex);
 	if (strncmp("low", buf, strlen("low")) == 0) {
 		level = AMDGPU_DPM_FORCED_LEVEL_LOW;
 	} else if (strncmp("high", buf, strlen("high")) == 0) {
@@ -123,7 +150,11 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 		count = -EINVAL;
 		goto fail;
 	}
-	if (adev->pm.funcs->force_performance_level) {
+
+	if (adev->pp_enabled)
+		amdgpu_dpm_force_performance_level(adev, level);
+	else {
+		mutex_lock(&adev->pm.mutex);
 		if (adev->pm.dpm.thermal_active) {
 			count = -EINVAL;
 			goto fail;
@@ -131,6 +162,9 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 		ret = amdgpu_dpm_force_performance_level(adev, level);
 		if (ret)
 			count = -EINVAL;
+		else
+			adev->pm.dpm.forced_level = level;
+		mutex_unlock(&adev->pm.mutex);
 	}
 fail:
 	mutex_unlock(&adev->pm.mutex);
@@ -150,10 +184,10 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	int temp;
 
-	if (adev->pm.funcs->get_temperature)
-		temp = amdgpu_dpm_get_temperature(adev);
-	else
+	if (!adev->pp_enabled && !adev->pm.funcs->get_temperature)
 		temp = 0;
+	else
+		temp = amdgpu_dpm_get_temperature(adev);
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", temp);
 }
@@ -181,8 +215,10 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	u32 pwm_mode = 0;
 
-	if (adev->pm.funcs->get_fan_control_mode)
-		pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	if (!adev->pp_enabled && !adev->pm.funcs->get_fan_control_mode)
+		return -EINVAL;
+
+	pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
 
 	/* never 0 (full-speed), fuse or smc-controlled always */
 	return sprintf(buf, "%i\n", pwm_mode == FDO_PWM_MODE_STATIC ? 1 : 2);
@@ -197,7 +233,7 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
 	int err;
 	int value;
 
-	if(!adev->pm.funcs->set_fan_control_mode)
+	if (!adev->pp_enabled && !adev->pm.funcs->set_fan_control_mode)
 		return -EINVAL;
 
 	err = kstrtoint(buf, 10, &value);
@@ -290,11 +326,11 @@ static struct attribute *hwmon_attributes[] = {
 static umode_t hwmon_attributes_visible(struct kobject *kobj,
 					struct attribute *attr, int index)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	umode_t effective_mode = attr->mode;
 
-	/* Skip attributes if DPM is not enabled */
+	/* Skip limit attributes if DPM is not enabled */
 	if (!adev->pm.dpm_enabled &&
 	    (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr ||
 	     attr == &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr ||
@@ -304,6 +340,9 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	     attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
 		return 0;
 
+	if (adev->pp_enabled)
+		return effective_mode;
+
 	/* Skip fan attributes if fan is not present */
 	if (adev->pm.no_fan &&
 	    (attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
@@ -351,7 +390,7 @@ void amdgpu_dpm_thermal_work_handler(struct work_struct *work)
 		container_of(work, struct amdgpu_device,
 			     pm.dpm.thermal.work);
 	/* switch to the thermal state */
-	enum amdgpu_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL;
+	enum amd_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL;
 
 	if (!adev->pm.dpm_enabled)
 		return;
@@ -379,7 +418,7 @@ void amdgpu_dpm_thermal_work_handler(struct work_struct *work)
 }
 
 static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev,
-						     enum amdgpu_pm_state_type dpm_state)
+						     enum amd_pm_state_type dpm_state)
 {
 	int i;
 	struct amdgpu_ps *ps;
@@ -516,7 +555,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 {
 	int i;
 	struct amdgpu_ps *ps;
-	enum amdgpu_pm_state_type dpm_state;
+	enum amd_pm_state_type dpm_state;
 	int ret;
 
 	/* if dpm init failed */
@@ -635,49 +674,54 @@ done:
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->pm.funcs->powergate_uvd) {
-		mutex_lock(&adev->pm.mutex);
-		/* enable/disable UVD */
+	if (adev->pp_enabled)
 		amdgpu_dpm_powergate_uvd(adev, !enable);
-		mutex_unlock(&adev->pm.mutex);
-	} else {
-		if (enable) {
+	else {
+		if (adev->pm.funcs->powergate_uvd) {
 			mutex_lock(&adev->pm.mutex);
-			adev->pm.dpm.uvd_active = true;
-			adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD;
+			/* enable/disable UVD */
+			amdgpu_dpm_powergate_uvd(adev, !enable);
 			mutex_unlock(&adev->pm.mutex);
 		} else {
-			mutex_lock(&adev->pm.mutex);
-			adev->pm.dpm.uvd_active = false;
-			mutex_unlock(&adev->pm.mutex);
+			if (enable) {
+				mutex_lock(&adev->pm.mutex);
+				adev->pm.dpm.uvd_active = true;
+				adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD;
+				mutex_unlock(&adev->pm.mutex);
+			} else {
+				mutex_lock(&adev->pm.mutex);
+				adev->pm.dpm.uvd_active = false;
+				mutex_unlock(&adev->pm.mutex);
+			}
+			amdgpu_pm_compute_clocks(adev);
 		}
 
-		amdgpu_pm_compute_clocks(adev);
 	}
 }
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->pm.funcs->powergate_vce) {
-		mutex_lock(&adev->pm.mutex);
-		/* enable/disable VCE */
+	if (adev->pp_enabled)
 		amdgpu_dpm_powergate_vce(adev, !enable);
-
-		mutex_unlock(&adev->pm.mutex);
-	} else {
-		if (enable) {
+	else {
+		if (adev->pm.funcs->powergate_vce) {
 			mutex_lock(&adev->pm.mutex);
-			adev->pm.dpm.vce_active = true;
-			/* XXX select vce level based on ring/task */
-			adev->pm.dpm.vce_level = AMDGPU_VCE_LEVEL_AC_ALL;
+			amdgpu_dpm_powergate_vce(adev, !enable);
 			mutex_unlock(&adev->pm.mutex);
 		} else {
-			mutex_lock(&adev->pm.mutex);
-			adev->pm.dpm.vce_active = false;
-			mutex_unlock(&adev->pm.mutex);
+			if (enable) {
+				mutex_lock(&adev->pm.mutex);
+				adev->pm.dpm.vce_active = true;
+				/* XXX select vce level based on ring/task */
+				adev->pm.dpm.vce_level = AMDGPU_VCE_LEVEL_AC_ALL;
+				mutex_unlock(&adev->pm.mutex);
+			} else {
+				mutex_lock(&adev->pm.mutex);
+				adev->pm.dpm.vce_active = false;
+				mutex_unlock(&adev->pm.mutex);
+			}
+			amdgpu_pm_compute_clocks(adev);
 		}
-
-		amdgpu_pm_compute_clocks(adev);
 	}
 }
 
@@ -685,10 +729,13 @@ void amdgpu_pm_print_power_states(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->pm.dpm.num_ps; i++) {
-		printk("== power state %d ==\n", i);
+	if (adev->pp_enabled)
+		/* TO DO */
+		return;
+
+	for (i = 0; i < adev->pm.dpm.num_ps; i++)
 		amdgpu_dpm_print_power_state(adev, &adev->pm.dpm.ps[i]);
-	}
+
 }
 
 int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
@@ -698,8 +745,11 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 	if (adev->pm.sysfs_initialized)
 		return 0;
 
-	if (adev->pm.funcs->get_temperature == NULL)
-		return 0;
+	if (!adev->pp_enabled) {
+		if (adev->pm.funcs->get_temperature == NULL)
+			return 0;
+	}
+
 	adev->pm.int_hwmon_dev = hwmon_device_register_with_groups(adev->dev,
 								   DRIVER_NAME, adev,
 								   hwmon_groups);
@@ -748,32 +798,43 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 	if (!adev->pm.dpm_enabled)
 		return;
 
-	mutex_lock(&adev->pm.mutex);
+	if (adev->pp_enabled) {
+		int i = 0;
 
-	/* update active crtc counts */
-	adev->pm.dpm.new_active_crtcs = 0;
-	adev->pm.dpm.new_active_crtc_count = 0;
-	if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
-		list_for_each_entry(crtc,
-				    &ddev->mode_config.crtc_list, head) {
-			amdgpu_crtc = to_amdgpu_crtc(crtc);
-			if (crtc->enabled) {
-				adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id);
-				adev->pm.dpm.new_active_crtc_count++;
+		amdgpu_display_bandwidth_update(adev);
+		mutex_lock(&adev->ring_lock);
+			for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+				struct amdgpu_ring *ring = adev->rings[i];
+				if (ring && ring->ready)
+					amdgpu_fence_wait_empty(ring);
 			}
-		}
-	}
+		mutex_unlock(&adev->ring_lock);
 
-	/* update battery/ac status */
-	if (power_supply_is_system_supplied() > 0)
-		adev->pm.dpm.ac_power = true;
-	else
-		adev->pm.dpm.ac_power = false;
-
-	amdgpu_dpm_change_power_state_locked(adev);
+		amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE, NULL, NULL);
+	} else {
+		mutex_lock(&adev->pm.mutex);
+		adev->pm.dpm.new_active_crtcs = 0;
+		adev->pm.dpm.new_active_crtc_count = 0;
+		if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
+			list_for_each_entry(crtc,
+					    &ddev->mode_config.crtc_list, head) {
+				amdgpu_crtc = to_amdgpu_crtc(crtc);
+				if (crtc->enabled) {
+					adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id);
+					adev->pm.dpm.new_active_crtc_count++;
+				}
+			}
+		}
+		/* update battery/ac status */
+		if (power_supply_is_system_supplied() > 0)
+			adev->pm.dpm.ac_power = true;
+		else
+			adev->pm.dpm.ac_power = false;
 
-	mutex_unlock(&adev->pm.mutex);
+		amdgpu_dpm_change_power_state_locked(adev);
 
+		mutex_unlock(&adev->pm.mutex);
+	}
 }
 
 /*
@@ -787,7 +848,13 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 	struct drm_device *dev = node->minor->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 
-	if (adev->pm.dpm_enabled) {
+	if (!adev->pm.dpm_enabled) {
+		seq_printf(m, "dpm not enabled\n");
+		return 0;
+	}
+	if (adev->pp_enabled) {
+		amdgpu_dpm_debugfs_print_current_performance_level(adev, m);
+	} else {
 		mutex_lock(&adev->pm.mutex);
 		if (adev->pm.funcs->debugfs_print_current_performance_level)
 			amdgpu_dpm_debugfs_print_current_performance_level(adev, m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
new file mode 100644
index 000000000000..b9d0d55f6b47
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#include "atom.h"
+#include "amdgpu.h"
+#include "amd_shared.h"
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include "amdgpu_pm.h"
+#include <drm/amdgpu_drm.h>
+#include "amdgpu_powerplay.h"
+#include "cik_dpm.h"
+#include "vi_dpm.h"
+
+static int amdgpu_powerplay_init(struct amdgpu_device *adev)
+{
+	int ret = 0;
+	struct amd_powerplay *amd_pp;
+
+	amd_pp = &(adev->powerplay);
+
+	if (adev->pp_enabled) {
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+		struct amd_pp_init *pp_init;
+
+		pp_init = kzalloc(sizeof(struct amd_pp_init), GFP_KERNEL);
+
+		if (pp_init == NULL)
+			return -ENOMEM;
+
+		pp_init->chip_family = adev->family;
+		pp_init->chip_id = adev->asic_type;
+		pp_init->device = amdgpu_cgs_create_device(adev);
+
+		ret = amd_powerplay_init(pp_init, amd_pp);
+		kfree(pp_init);
+#endif
+	} else {
+		amd_pp->pp_handle = (void *)adev;
+
+		switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
+		case CHIP_BONAIRE:
+		case CHIP_HAWAII:
+			amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+			break;
+		case CHIP_KABINI:
+		case CHIP_MULLINS:
+		case CHIP_KAVERI:
+			amd_pp->ip_funcs = &kv_dpm_ip_funcs;
+			break;
+#endif
+		case CHIP_TOPAZ:
+			amd_pp->ip_funcs = &iceland_dpm_ip_funcs;
+			break;
+		case CHIP_TONGA:
+			amd_pp->ip_funcs = &tonga_dpm_ip_funcs;
+			break;
+		case CHIP_FIJI:
+			amd_pp->ip_funcs = &fiji_dpm_ip_funcs;
+			break;
+		case CHIP_CARRIZO:
+		case CHIP_STONEY:
+			amd_pp->ip_funcs = &cz_dpm_ip_funcs;
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+	}
+	return ret;
+}
+
+static int amdgpu_pp_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret = 0;
+
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+	switch (adev->asic_type) {
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+		adev->pp_enabled = (amdgpu_powerplay == 0) ? false : true;
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+		adev->pp_enabled = (amdgpu_powerplay > 0) ? true : false;
+		break;
+	/* These chips don't have powerplay implemenations */
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+	case CHIP_KAVERI:
+	case CHIP_TOPAZ:
+	default:
+		adev->pp_enabled = false;
+		break;
+	}
+#else
+	adev->pp_enabled = false;
+#endif
+
+	ret = amdgpu_powerplay_init(adev);
+	if (ret)
+		return ret;
+
+	if (adev->powerplay.ip_funcs->early_init)
+		ret = adev->powerplay.ip_funcs->early_init(
+					adev->powerplay.pp_handle);
+	return ret;
+}
+
+
+static int amdgpu_pp_late_init(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->late_init)
+		ret = adev->powerplay.ip_funcs->late_init(
+					adev->powerplay.pp_handle);
+
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+	if (adev->pp_enabled)
+		amdgpu_pm_sysfs_init(adev);
+#endif
+	return ret;
+}
+
+static int amdgpu_pp_sw_init(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->sw_init)
+		ret = adev->powerplay.ip_funcs->sw_init(
+					adev->powerplay.pp_handle);
+
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+	if (adev->pp_enabled) {
+		if (amdgpu_dpm == 0)
+			adev->pm.dpm_enabled = false;
+		else
+			adev->pm.dpm_enabled = true;
+	}
+#endif
+
+	return ret;
+}
+
+static int amdgpu_pp_sw_fini(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->sw_fini)
+		ret = adev->powerplay.ip_funcs->sw_fini(
+					adev->powerplay.pp_handle);
+	if (ret)
+		return ret;
+
+#ifdef CONFIG_DRM_AMD_POWERPLAY
+	if (adev->pp_enabled) {
+		amdgpu_pm_sysfs_fini(adev);
+		amd_powerplay_fini(adev->powerplay.pp_handle);
+	}
+#endif
+
+	return ret;
+}
+
+static int amdgpu_pp_hw_init(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->pp_enabled && adev->firmware.smu_load)
+		amdgpu_ucode_init_bo(adev);
+
+	if (adev->powerplay.ip_funcs->hw_init)
+		ret = adev->powerplay.ip_funcs->hw_init(
+					adev->powerplay.pp_handle);
+
+	return ret;
+}
+
+static int amdgpu_pp_hw_fini(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->hw_fini)
+		ret = adev->powerplay.ip_funcs->hw_fini(
+					adev->powerplay.pp_handle);
+
+	if (adev->pp_enabled && adev->firmware.smu_load)
+		amdgpu_ucode_fini_bo(adev);
+
+	return ret;
+}
+
+static int amdgpu_pp_suspend(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->suspend)
+		ret = adev->powerplay.ip_funcs->suspend(
+					 adev->powerplay.pp_handle);
+	return ret;
+}
+
+static int amdgpu_pp_resume(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->resume)
+		ret = adev->powerplay.ip_funcs->resume(
+					adev->powerplay.pp_handle);
+	return ret;
+}
+
+static int amdgpu_pp_set_clockgating_state(void *handle,
+					enum amd_clockgating_state state)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->set_clockgating_state)
+		ret = adev->powerplay.ip_funcs->set_clockgating_state(
+				adev->powerplay.pp_handle, state);
+	return ret;
+}
+
+static int amdgpu_pp_set_powergating_state(void *handle,
+					enum amd_powergating_state state)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->set_powergating_state)
+		ret = adev->powerplay.ip_funcs->set_powergating_state(
+				 adev->powerplay.pp_handle, state);
+	return ret;
+}
+
+
+static bool amdgpu_pp_is_idle(void *handle)
+{
+	bool ret = true;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->is_idle)
+		ret = adev->powerplay.ip_funcs->is_idle(
+					adev->powerplay.pp_handle);
+	return ret;
+}
+
+static int amdgpu_pp_wait_for_idle(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->wait_for_idle)
+		ret = adev->powerplay.ip_funcs->wait_for_idle(
+					adev->powerplay.pp_handle);
+	return ret;
+}
+
+static int amdgpu_pp_soft_reset(void *handle)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->soft_reset)
+		ret = adev->powerplay.ip_funcs->soft_reset(
+					adev->powerplay.pp_handle);
+	return ret;
+}
+
+static void amdgpu_pp_print_status(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->powerplay.ip_funcs->print_status)
+		adev->powerplay.ip_funcs->print_status(
+					adev->powerplay.pp_handle);
+}
+
+const struct amd_ip_funcs amdgpu_pp_ip_funcs = {
+	.early_init = amdgpu_pp_early_init,
+	.late_init = amdgpu_pp_late_init,
+	.sw_init = amdgpu_pp_sw_init,
+	.sw_fini = amdgpu_pp_sw_fini,
+	.hw_init = amdgpu_pp_hw_init,
+	.hw_fini = amdgpu_pp_hw_fini,
+	.suspend = amdgpu_pp_suspend,
+	.resume = amdgpu_pp_resume,
+	.is_idle = amdgpu_pp_is_idle,
+	.wait_for_idle = amdgpu_pp_wait_for_idle,
+	.soft_reset = amdgpu_pp_soft_reset,
+	.print_status = amdgpu_pp_print_status,
+	.set_clockgating_state = amdgpu_pp_set_clockgating_state,
+	.set_powergating_state = amdgpu_pp_set_powergating_state,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h
new file mode 100644
index 000000000000..da5cf47cfd99
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h
@@ -0,0 +1,33 @@
+/*
+ *  Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __AMDGPU_POPWERPLAY_H__
+#define __AMDGPU_POPWERPLAY_H__
+
+#include "amd_shared.h"
+
+extern const struct amd_ip_funcs amdgpu_pp_ip_funcs;
+
+#endif /* __AMDSOC_DM_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 78e9b0f14661..d1f234dd2126 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -487,7 +487,7 @@ static int amdgpu_debugfs_ring_info(struct seq_file *m, void *data)
 	seq_printf(m, "rptr: 0x%08x [%5d]\n",
 		   rptr, rptr);
 
-	rptr_next = ~0;
+	rptr_next = le32_to_cpu(*ring->next_rptr_cpu_addr);
 
 	seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
 		   ring->wptr, ring->wptr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index dd005c336c97..181ce39ef5e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -293,7 +293,8 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
 		fence = to_amdgpu_fence(sync->sync_to[i]);
 
 		/* check if we really need to sync */
-		if (!amdgpu_fence_need_sync(fence, ring))
+		if (!amdgpu_enable_scheduler &&
+		    !amdgpu_fence_need_sync(fence, ring))
 			continue;
 
 		/* prevent GPU deadlocks */
@@ -303,7 +304,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
 		}
 
 		if (amdgpu_enable_scheduler || !amdgpu_enable_semaphores) {
-			r = fence_wait(&fence->base, true);
+			r = fence_wait(sync->sync_to[i], true);
 			if (r)
 				return r;
 			continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index b53d273eb7a1..9599f7559b3d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -75,50 +75,77 @@ static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_vm_get_bos - add the vm BOs to a validation list
+ * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  *
  * @vm: vm providing the BOs
- * @head: head of validation list
+ * @validated: head of validation list
+ * @entry: entry to add
  *
  * Add the page directory to the list of BOs to
- * validate for command submission (cayman+).
+ * validate for command submission.
  */
-struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev,
-					  struct amdgpu_vm *vm,
-					  struct list_head *head)
+void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
+			 struct list_head *validated,
+			 struct amdgpu_bo_list_entry *entry)
 {
-	struct amdgpu_bo_list_entry *list;
-	unsigned i, idx;
+	entry->robj = vm->page_directory;
+	entry->prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
+	entry->allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
+	entry->priority = 0;
+	entry->tv.bo = &vm->page_directory->tbo;
+	entry->tv.shared = true;
+	list_add(&entry->tv.head, validated);
+}
 
-	list = drm_malloc_ab(vm->max_pde_used + 2,
-			     sizeof(struct amdgpu_bo_list_entry));
-	if (!list) {
-		return NULL;
-	}
+/**
+ * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
+ *
+ * @vm: vm providing the BOs
+ * @duplicates: head of duplicates list
+ *
+ * Add the page directory to the BO duplicates list
+ * for command submission.
+ */
+void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates)
+{
+	unsigned i;
 
 	/* add the vm page table to the list */
-	list[0].robj = vm->page_directory;
-	list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
-	list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
-	list[0].priority = 0;
-	list[0].tv.bo = &vm->page_directory->tbo;
-	list[0].tv.shared = true;
-	list_add(&list[0].tv.head, head);
-
-	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {
-		if (!vm->page_tables[i].bo)
+	for (i = 0; i <= vm->max_pde_used; ++i) {
+		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
+
+		if (!entry->robj)
 			continue;
 
-		list[idx].robj = vm->page_tables[i].bo;
-		list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
-		list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
-		list[idx].priority = 0;
-		list[idx].tv.bo = &list[idx].robj->tbo;
-		list[idx].tv.shared = true;
-		list_add(&list[idx++].tv.head, head);
+		list_add(&entry->tv.head, duplicates);
 	}
 
-	return list;
+}
+
+/**
+ * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
+ *
+ * @adev: amdgpu device instance
+ * @vm: vm providing the BOs
+ *
+ * Move the PT BOs to the tail of the LRU.
+ */
+void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
+				  struct amdgpu_vm *vm)
+{
+	struct ttm_bo_global *glob = adev->mman.bdev.glob;
+	unsigned i;
+
+	spin_lock(&glob->lru_lock);
+	for (i = 0; i <= vm->max_pde_used; ++i) {
+		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
+
+		if (!entry->robj)
+			continue;
+
+		ttm_bo_move_to_lru_tail(&entry->robj->tbo);
+	}
+	spin_unlock(&glob->lru_lock);
 }
 
 /**
@@ -461,7 +488,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 
 	/* walk over the address space and update the page directory */
 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
-		struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
+		struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
 		uint64_t pde, pt;
 
 		if (bo == NULL)
@@ -638,7 +665,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 	/* walk over the address space and update the page tables */
 	for (addr = start; addr < end; ) {
 		uint64_t pt_idx = addr >> amdgpu_vm_block_size;
-		struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo;
+		struct amdgpu_bo *pt = vm->page_tables[pt_idx].entry.robj;
 		unsigned nptes;
 		uint64_t pte;
 		int r;
@@ -1010,13 +1037,13 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 		return -EINVAL;
 
 	/* make sure object fit at this offset */
-	eaddr = saddr + size;
+	eaddr = saddr + size - 1;
 	if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
 		return -EINVAL;
 
 	last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
-	if (last_pfn > adev->vm_manager.max_pfn) {
-		dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n",
+	if (last_pfn >= adev->vm_manager.max_pfn) {
+		dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
 			last_pfn, adev->vm_manager.max_pfn);
 		return -EINVAL;
 	}
@@ -1025,7 +1052,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
 
 	spin_lock(&vm->it_lock);
-	it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1);
+	it = interval_tree_iter_first(&vm->va, saddr, eaddr);
 	spin_unlock(&vm->it_lock);
 	if (it) {
 		struct amdgpu_bo_va_mapping *tmp;
@@ -1046,7 +1073,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 
 	INIT_LIST_HEAD(&mapping->list);
 	mapping->it.start = saddr;
-	mapping->it.last = eaddr - 1;
+	mapping->it.last = eaddr;
 	mapping->offset = offset;
 	mapping->flags = flags;
 
@@ -1070,9 +1097,11 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	/* walk over the address space and allocate the page tables */
 	for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
 		struct reservation_object *resv = vm->page_directory->tbo.resv;
+		struct amdgpu_bo_list_entry *entry;
 		struct amdgpu_bo *pt;
 
-		if (vm->page_tables[pt_idx].bo)
+		entry = &vm->page_tables[pt_idx].entry;
+		if (entry->robj)
 			continue;
 
 		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
@@ -1094,8 +1123,13 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 			goto error_free;
 		}
 
+		entry->robj = pt;
+		entry->prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
+		entry->allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
+		entry->priority = 0;
+		entry->tv.bo = &entry->robj->tbo;
+		entry->tv.shared = true;
 		vm->page_tables[pt_idx].addr = 0;
-		vm->page_tables[pt_idx].bo = pt;
 	}
 
 	return 0;
@@ -1248,7 +1282,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
 	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
 		AMDGPU_VM_PTE_COUNT * 8);
-	unsigned pd_size, pd_entries, pts_size;
+	unsigned pd_size, pd_entries;
 	int i, r;
 
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
@@ -1266,8 +1300,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	pd_entries = amdgpu_vm_num_pdes(adev);
 
 	/* allocate page table array */
-	pts_size = pd_entries * sizeof(struct amdgpu_vm_pt);
-	vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
+	vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
 	if (vm->page_tables == NULL) {
 		DRM_ERROR("Cannot allocate memory for page table array\n");
 		return -ENOMEM;
@@ -1326,8 +1359,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	}
 
 	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
-		amdgpu_bo_unref(&vm->page_tables[i].bo);
-	kfree(vm->page_tables);
+		amdgpu_bo_unref(&vm->page_tables[i].entry.robj);
+	drm_free_large(vm->page_tables);
 
 	amdgpu_bo_unref(&vm->page_directory);
 	fence_put(vm->page_directory_fence);
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
index 92b6acadfc52..21aacc1f45c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
@@ -243,7 +243,7 @@ static void amdgpu_atombios_dp_get_adjust_train(const u8 link_status[DP_LINK_STA
 
 /* convert bits per color to bits per pixel */
 /* get bpc from the EDID */
-static int amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc)
+static unsigned amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc)
 {
 	if (bpc == 0)
 		return 24;
@@ -251,64 +251,32 @@ static int amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc)
 		return bpc * 3;
 }
 
-/* get the max pix clock supported by the link rate and lane num */
-static int amdgpu_atombios_dp_get_max_dp_pix_clock(int link_rate,
-					    int lane_num,
-					    int bpp)
-{
-	return (link_rate * lane_num * 8) / bpp;
-}
-
 /***** amdgpu specific DP functions *****/
 
-/* First get the min lane# when low rate is used according to pixel clock
- * (prefer low rate), second check max lane# supported by DP panel,
- * if the max lane# < low rate lane# then use max lane# instead.
- */
-static int amdgpu_atombios_dp_get_dp_lane_number(struct drm_connector *connector,
+static int amdgpu_atombios_dp_get_dp_link_config(struct drm_connector *connector,
 						 const u8 dpcd[DP_DPCD_SIZE],
-						 int pix_clock)
-{
-	int bpp = amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector));
-	int max_link_rate = drm_dp_max_link_rate(dpcd);
-	int max_lane_num = drm_dp_max_lane_count(dpcd);
-	int lane_num;
-	int max_dp_pix_clock;
-
-	for (lane_num = 1; lane_num < max_lane_num; lane_num <<= 1) {
-		max_dp_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(max_link_rate, lane_num, bpp);
-		if (pix_clock <= max_dp_pix_clock)
-			break;
-	}
-
-	return lane_num;
-}
-
-static int amdgpu_atombios_dp_get_dp_link_clock(struct drm_connector *connector,
-						const u8 dpcd[DP_DPCD_SIZE],
-						int pix_clock)
+						 unsigned pix_clock,
+						 unsigned *dp_lanes, unsigned *dp_rate)
 {
-	int bpp = amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector));
-	int lane_num, max_pix_clock;
-
-	if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) ==
-	    ENCODER_OBJECT_ID_NUTMEG)
-		return 270000;
-
-	lane_num = amdgpu_atombios_dp_get_dp_lane_number(connector, dpcd, pix_clock);
-	max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(162000, lane_num, bpp);
-	if (pix_clock <= max_pix_clock)
-		return 162000;
-	max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(270000, lane_num, bpp);
-	if (pix_clock <= max_pix_clock)
-		return 270000;
-	if (amdgpu_connector_is_dp12_capable(connector)) {
-		max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(540000, lane_num, bpp);
-		if (pix_clock <= max_pix_clock)
-			return 540000;
+	unsigned bpp =
+		amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector));
+	static const unsigned link_rates[3] = { 162000, 270000, 540000 };
+	unsigned max_link_rate = drm_dp_max_link_rate(dpcd);
+	unsigned max_lane_num = drm_dp_max_lane_count(dpcd);
+	unsigned lane_num, i, max_pix_clock;
+
+	for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) {
+		for (i = 0; i < ARRAY_SIZE(link_rates) && link_rates[i] <= max_link_rate; i++) {
+			max_pix_clock = (lane_num * link_rates[i] * 8) / bpp;
+			if (max_pix_clock >= pix_clock) {
+				*dp_lanes = lane_num;
+				*dp_rate = link_rates[i];
+				return 0;
+			}
+		}
 	}
 
-	return drm_dp_max_link_rate(dpcd);
+	return -EINVAL;
 }
 
 static u8 amdgpu_atombios_dp_encoder_service(struct amdgpu_device *adev,
@@ -422,6 +390,7 @@ void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector,
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 	struct amdgpu_connector_atom_dig *dig_connector;
+	int ret;
 
 	if (!amdgpu_connector->con_priv)
 		return;
@@ -429,10 +398,14 @@ void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector,
 
 	if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
 	    (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) {
-		dig_connector->dp_clock =
-			amdgpu_atombios_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
-		dig_connector->dp_lane_count =
-			amdgpu_atombios_dp_get_dp_lane_number(connector, dig_connector->dpcd, mode->clock);
+		ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd,
+							    mode->clock,
+							    &dig_connector->dp_lane_count,
+							    &dig_connector->dp_clock);
+		if (ret) {
+			dig_connector->dp_clock = 0;
+			dig_connector->dp_lane_count = 0;
+		}
 	}
 }
 
@@ -441,14 +414,17 @@ int amdgpu_atombios_dp_mode_valid_helper(struct drm_connector *connector,
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 	struct amdgpu_connector_atom_dig *dig_connector;
-	int dp_clock;
+	unsigned dp_lanes, dp_clock;
+	int ret;
 
 	if (!amdgpu_connector->con_priv)
 		return MODE_CLOCK_HIGH;
 	dig_connector = amdgpu_connector->con_priv;
 
-	dp_clock =
-		amdgpu_atombios_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
+	ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd,
+						    mode->clock, &dp_lanes, &dp_clock);
+	if (ret)
+		return MODE_CLOCK_HIGH;
 
 	if ((dp_clock == 540000) &&
 	    (!amdgpu_connector_is_dp12_capable(connector)))
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 57a2e347f04d..8b4731d4e10e 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -1395,7 +1395,6 @@ static void ci_thermal_stop_thermal_controller(struct amdgpu_device *adev)
 		ci_fan_ctrl_set_default_mode(adev);
 }
 
-#if 0
 static int ci_read_smc_soft_register(struct amdgpu_device *adev,
 				     u16 reg_offset, u32 *value)
 {
@@ -1405,7 +1404,6 @@ static int ci_read_smc_soft_register(struct amdgpu_device *adev,
 				      pi->soft_regs_start + reg_offset,
 				      value, pi->sram_end);
 }
-#endif
 
 static int ci_write_smc_soft_register(struct amdgpu_device *adev,
 				      u16 reg_offset, u32 value)
@@ -6084,11 +6082,23 @@ ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
 	struct amdgpu_ps *rps = &pi->current_rps;
 	u32 sclk = ci_get_average_sclk_freq(adev);
 	u32 mclk = ci_get_average_mclk_freq(adev);
+	u32 activity_percent = 50;
+	int ret;
+
+	ret = ci_read_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, AverageGraphicsA),
+					&activity_percent);
+
+	if (ret == 0) {
+		activity_percent += 0x80;
+		activity_percent >>= 8;
+		activity_percent = activity_percent > 100 ? 100 : activity_percent;
+	}
 
 	seq_printf(m, "uvd %sabled\n", pi->uvd_enabled ? "en" : "dis");
 	seq_printf(m, "vce %sabled\n", rps->vce_active ? "en" : "dis");
 	seq_printf(m, "power level avg    sclk: %u mclk: %u\n",
 		   sclk, mclk);
+	seq_printf(m, "GPU load: %u %%\n", activity_percent);
 }
 
 static void ci_dpm_print_power_state(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 484710cfdf82..fd9c9588ef46 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -32,6 +32,7 @@
 #include "amdgpu_vce.h"
 #include "cikd.h"
 #include "atom.h"
+#include "amd_pcie.h"
 
 #include "cik.h"
 #include "gmc_v7_0.h"
@@ -65,6 +66,7 @@
 #include "oss/oss_2_0_sh_mask.h"
 
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_powerplay.h"
 
 /*
  * Indirect registers accessor
@@ -929,6 +931,37 @@ static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 	return r;
 }
 
+static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
+				   u8 *bios, u32 length_bytes)
+{
+	u32 *dw_ptr;
+	unsigned long flags;
+	u32 i, length_dw;
+
+	if (bios == NULL)
+		return false;
+	if (length_bytes == 0)
+		return false;
+	/* APU vbios image is part of sbios image */
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	dw_ptr = (u32 *)bios;
+	length_dw = ALIGN(length_bytes, 4) / 4;
+	/* take the smc lock since we are using the smc index */
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	/* set rom index to 0 */
+	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
+	WREG32(mmSMC_IND_DATA_0, 0);
+	/* set index to data for continous read */
+	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
+	for (i = 0; i < length_dw; i++)
+		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return true;
+}
+
 static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 	{mmGRBM_STATUS, false},
 	{mmGB_ADDR_CONFIG, false},
@@ -1563,8 +1596,8 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
 {
 	struct pci_dev *root = adev->pdev->bus->self;
 	int bridge_pos, gpu_pos;
-	u32 speed_cntl, mask, current_data_rate;
-	int ret, i;
+	u32 speed_cntl, current_data_rate;
+	int i;
 	u16 tmp16;
 
 	if (pci_is_root_bus(adev->pdev->bus))
@@ -1576,23 +1609,20 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
 	if (adev->flags & AMD_IS_APU)
 		return;
 
-	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
-	if (ret != 0)
-		return;
-
-	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
 		return;
 
 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
 		if (current_data_rate == 2) {
 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
 			return;
 		}
 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
-	} else if (mask & DRM_PCIE_SPEED_50) {
+	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
 		if (current_data_rate == 1) {
 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
 			return;
@@ -1608,7 +1638,7 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
 	if (!gpu_pos)
 		return;
 
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
 		/* re-try equalization if gen3 is not already enabled */
 		if (current_data_rate != 2) {
 			u16 bridge_cfg, gpu_cfg;
@@ -1703,9 +1733,9 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
 
 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
 	tmp16 &= ~0xf;
-	if (mask & DRM_PCIE_SPEED_80)
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
 		tmp16 |= 3; /* gen3 */
-	else if (mask & DRM_PCIE_SPEED_50)
+	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
 		tmp16 |= 2; /* gen2 */
 	else
 		tmp16 |= 1; /* gen1 */
@@ -1922,7 +1952,7 @@ static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
 		.major = 7,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &ci_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -1990,7 +2020,7 @@ static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
 		.major = 7,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &ci_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -2058,7 +2088,7 @@ static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
 		.major = 7,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &kv_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -2126,7 +2156,7 @@ static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
 		.major = 7,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &kv_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -2194,7 +2224,7 @@ static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
 		.major = 7,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &kv_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -2267,6 +2297,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
 	.read_disabled_bios = &cik_read_disabled_bios,
+	.read_bios_from_rom = &cik_read_bios_from_rom,
 	.read_register = &cik_read_register,
 	.reset = &cik_asic_reset,
 	.set_vga_state = &cik_vga_set_state,
@@ -2417,6 +2448,8 @@ static int cik_common_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	amdgpu_get_pcie_info(adev);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 8993c50cb89f..30c9b3beeef9 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -274,6 +274,11 @@ static void cik_ih_set_rptr(struct amdgpu_device *adev)
 static int cik_ih_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
+
+	ret = amdgpu_irq_add_domain(adev);
+	if (ret)
+		return ret;
 
 	cik_ih_set_interrupt_funcs(adev);
 
@@ -300,6 +305,7 @@ static int cik_ih_sw_fini(void *handle)
 
 	amdgpu_irq_fini(adev);
 	amdgpu_ih_ring_fini(adev);
+	amdgpu_irq_remove_domain(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index 8035d4d6a4f5..4dd17f2dd905 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -1078,6 +1078,37 @@ static uint32_t cz_get_eclk_level(struct amdgpu_device *adev,
 	return i;
 }
 
+static uint32_t cz_get_uvd_level(struct amdgpu_device *adev,
+				 uint32_t clock, uint16_t msg)
+{
+	int i = 0;
+	struct amdgpu_uvd_clock_voltage_dependency_table *table =
+		&adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
+
+	switch (msg) {
+	case PPSMC_MSG_SetUvdSoftMin:
+	case PPSMC_MSG_SetUvdHardMin:
+		for (i = 0; i < table->count; i++)
+			if (clock <= table->entries[i].vclk)
+				break;
+		if (i == table->count)
+			i = table->count - 1;
+		break;
+	case PPSMC_MSG_SetUvdSoftMax:
+	case PPSMC_MSG_SetUvdHardMax:
+		for (i = table->count - 1; i >= 0; i--)
+			if (clock >= table->entries[i].vclk)
+				break;
+		if (i < 0)
+			i = 0;
+		break;
+	default:
+		break;
+	}
+
+	return i;
+}
+
 static int cz_program_bootup_state(struct amdgpu_device *adev)
 {
 	struct cz_power_info *pi = cz_get_pi(adev);
@@ -1739,6 +1770,200 @@ static int cz_dpm_unforce_dpm_levels(struct amdgpu_device *adev)
 	return 0;
 }
 
+static int cz_dpm_uvd_force_highest(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	int ret = 0;
+
+	if (pi->uvd_dpm.soft_min_clk != pi->uvd_dpm.soft_max_clk) {
+		pi->uvd_dpm.soft_min_clk =
+			pi->uvd_dpm.soft_max_clk;
+		ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetUvdSoftMin,
+				cz_get_uvd_level(adev,
+					pi->uvd_dpm.soft_min_clk,
+					PPSMC_MSG_SetUvdSoftMin));
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+static int cz_dpm_uvd_force_lowest(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	int ret = 0;
+
+	if (pi->uvd_dpm.soft_max_clk != pi->uvd_dpm.soft_min_clk) {
+		pi->uvd_dpm.soft_max_clk = pi->uvd_dpm.soft_min_clk;
+		ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetUvdSoftMax,
+				cz_get_uvd_level(adev,
+					pi->uvd_dpm.soft_max_clk,
+					PPSMC_MSG_SetUvdSoftMax));
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+static uint32_t cz_dpm_get_max_uvd_level(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+
+	if (!pi->max_uvd_level) {
+		cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxUvdLevel);
+		pi->max_uvd_level = cz_get_argument(adev) + 1;
+	}
+
+	if (pi->max_uvd_level > CZ_MAX_HARDWARE_POWERLEVELS) {
+		DRM_ERROR("Invalid max uvd level!\n");
+		return -EINVAL;
+	}
+
+	return pi->max_uvd_level;
+}
+
+static int cz_dpm_unforce_uvd_dpm_levels(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	struct amdgpu_uvd_clock_voltage_dependency_table *dep_table =
+		&adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
+	uint32_t level = 0;
+	int ret = 0;
+
+	pi->uvd_dpm.soft_min_clk = dep_table->entries[0].vclk;
+	level = cz_dpm_get_max_uvd_level(adev) - 1;
+	if (level < dep_table->count)
+		pi->uvd_dpm.soft_max_clk = dep_table->entries[level].vclk;
+	else
+		pi->uvd_dpm.soft_max_clk =
+			dep_table->entries[dep_table->count - 1].vclk;
+
+	/* get min/max sclk soft value
+	 * notify SMU to execute */
+	ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetUvdSoftMin,
+				cz_get_uvd_level(adev,
+					pi->uvd_dpm.soft_min_clk,
+					PPSMC_MSG_SetUvdSoftMin));
+	if (ret)
+		return ret;
+
+	ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetUvdSoftMax,
+				cz_get_uvd_level(adev,
+					pi->uvd_dpm.soft_max_clk,
+					PPSMC_MSG_SetUvdSoftMax));
+	if (ret)
+		return ret;
+
+	DRM_DEBUG("DPM uvd unforce state min=%d, max=%d.\n",
+		  pi->uvd_dpm.soft_min_clk,
+		  pi->uvd_dpm.soft_max_clk);
+
+	return 0;
+}
+
+static int cz_dpm_vce_force_highest(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	int ret = 0;
+
+	if (pi->vce_dpm.soft_min_clk != pi->vce_dpm.soft_max_clk) {
+		pi->vce_dpm.soft_min_clk =
+			pi->vce_dpm.soft_max_clk;
+		ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetEclkSoftMin,
+				cz_get_eclk_level(adev,
+					pi->vce_dpm.soft_min_clk,
+					PPSMC_MSG_SetEclkSoftMin));
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+static int cz_dpm_vce_force_lowest(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	int ret = 0;
+
+	if (pi->vce_dpm.soft_max_clk != pi->vce_dpm.soft_min_clk) {
+		pi->vce_dpm.soft_max_clk = pi->vce_dpm.soft_min_clk;
+		ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetEclkSoftMax,
+				cz_get_uvd_level(adev,
+					pi->vce_dpm.soft_max_clk,
+					PPSMC_MSG_SetEclkSoftMax));
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+static uint32_t cz_dpm_get_max_vce_level(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+
+	if (!pi->max_vce_level) {
+		cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxEclkLevel);
+		pi->max_vce_level = cz_get_argument(adev) + 1;
+	}
+
+	if (pi->max_vce_level > CZ_MAX_HARDWARE_POWERLEVELS) {
+		DRM_ERROR("Invalid max vce level!\n");
+		return -EINVAL;
+	}
+
+	return pi->max_vce_level;
+}
+
+static int cz_dpm_unforce_vce_dpm_levels(struct amdgpu_device *adev)
+{
+	struct cz_power_info *pi = cz_get_pi(adev);
+	struct amdgpu_vce_clock_voltage_dependency_table *dep_table =
+		&adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
+	uint32_t level = 0;
+	int ret = 0;
+
+	pi->vce_dpm.soft_min_clk = dep_table->entries[0].ecclk;
+	level = cz_dpm_get_max_vce_level(adev) - 1;
+	if (level < dep_table->count)
+		pi->vce_dpm.soft_max_clk = dep_table->entries[level].ecclk;
+	else
+		pi->vce_dpm.soft_max_clk =
+			dep_table->entries[dep_table->count - 1].ecclk;
+
+	/* get min/max sclk soft value
+	 * notify SMU to execute */
+	ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetEclkSoftMin,
+				cz_get_eclk_level(adev,
+					pi->vce_dpm.soft_min_clk,
+					PPSMC_MSG_SetEclkSoftMin));
+	if (ret)
+		return ret;
+
+	ret = cz_send_msg_to_smc_with_parameter(adev,
+				PPSMC_MSG_SetEclkSoftMax,
+				cz_get_eclk_level(adev,
+					pi->vce_dpm.soft_max_clk,
+					PPSMC_MSG_SetEclkSoftMax));
+	if (ret)
+		return ret;
+
+	DRM_DEBUG("DPM vce unforce state min=%d, max=%d.\n",
+		  pi->vce_dpm.soft_min_clk,
+		  pi->vce_dpm.soft_max_clk);
+
+	return 0;
+}
+
 static int cz_dpm_force_dpm_level(struct amdgpu_device *adev,
 				  enum amdgpu_dpm_forced_level level)
 {
@@ -1746,25 +1971,70 @@ static int cz_dpm_force_dpm_level(struct amdgpu_device *adev,
 
 	switch (level) {
 	case AMDGPU_DPM_FORCED_LEVEL_HIGH:
+		/* sclk */
 		ret = cz_dpm_unforce_dpm_levels(adev);
 		if (ret)
 			return ret;
 		ret = cz_dpm_force_highest(adev);
 		if (ret)
 			return ret;
+
+		/* uvd */
+		ret = cz_dpm_unforce_uvd_dpm_levels(adev);
+		if (ret)
+			return ret;
+		ret = cz_dpm_uvd_force_highest(adev);
+		if (ret)
+			return ret;
+
+		/* vce */
+		ret = cz_dpm_unforce_vce_dpm_levels(adev);
+		if (ret)
+			return ret;
+		ret = cz_dpm_vce_force_highest(adev);
+		if (ret)
+			return ret;
 		break;
 	case AMDGPU_DPM_FORCED_LEVEL_LOW:
+		/* sclk */
 		ret = cz_dpm_unforce_dpm_levels(adev);
 		if (ret)
 			return ret;
 		ret = cz_dpm_force_lowest(adev);
 		if (ret)
 			return ret;
+
+		/* uvd */
+		ret = cz_dpm_unforce_uvd_dpm_levels(adev);
+		if (ret)
+			return ret;
+		ret = cz_dpm_uvd_force_lowest(adev);
+		if (ret)
+			return ret;
+
+		/* vce */
+		ret = cz_dpm_unforce_vce_dpm_levels(adev);
+		if (ret)
+			return ret;
+		ret = cz_dpm_vce_force_lowest(adev);
+		if (ret)
+			return ret;
 		break;
 	case AMDGPU_DPM_FORCED_LEVEL_AUTO:
+		/* sclk */
 		ret = cz_dpm_unforce_dpm_levels(adev);
 		if (ret)
 			return ret;
+
+		/* uvd */
+		ret = cz_dpm_unforce_uvd_dpm_levels(adev);
+		if (ret)
+			return ret;
+
+		/* vce */
+		ret = cz_dpm_unforce_vce_dpm_levels(adev);
+		if (ret)
+			return ret;
 		break;
 	default:
 		break;
@@ -1905,7 +2175,8 @@ static int cz_update_vce_dpm(struct amdgpu_device *adev)
 		pi->vce_dpm.hard_min_clk = table->entries[table->count-1].ecclk;
 
 	} else { /* non-stable p-state cases. without vce.Arbiter.EcclkHardMin */
-		pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;
+		/* leave it as set by user */
+		/*pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;*/
 	}
 
 	cz_send_msg_to_smc_with_parameter(adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.h b/drivers/gpu/drm/amd/amdgpu/cz_dpm.h
index 99e1afc89629..5df8c1faab51 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.h
@@ -183,6 +183,8 @@ struct cz_power_info {
 	uint32_t voltage_drop_threshold;
 	uint32_t gfx_pg_threshold;
 	uint32_t max_sclk_level;
+	uint32_t max_uvd_level;
+	uint32_t max_vce_level;
 	/* flags */
 	bool didt_enabled;
 	bool video_start;
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index bc751bfbcae2..c79638f8e732 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -253,8 +253,14 @@ static void cz_ih_set_rptr(struct amdgpu_device *adev)
 static int cz_ih_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
+
+	ret = amdgpu_irq_add_domain(adev);
+	if (ret)
+		return ret;
 
 	cz_ih_set_interrupt_funcs(adev);
+
 	return 0;
 }
 
@@ -278,6 +284,7 @@ static int cz_ih_sw_fini(void *handle)
 
 	amdgpu_irq_fini(adev);
 	amdgpu_ih_ring_fini(adev);
+	amdgpu_irq_remove_domain(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 4dcc8fba5792..093599aba64b 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -3729,7 +3729,7 @@ static void dce_v10_0_encoder_add(struct amdgpu_device *adev,
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
 		drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-				 DRM_MODE_ENCODER_DAC);
+				 DRM_MODE_ENCODER_DAC, NULL);
 		drm_encoder_helper_add(encoder, &dce_v10_0_dac_helper_funcs);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
@@ -3740,15 +3740,15 @@ static void dce_v10_0_encoder_add(struct amdgpu_device *adev,
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 			amdgpu_encoder->rmx_type = RMX_FULL;
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_lcd_info(amdgpu_encoder);
 		} else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		} else {
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		}
 		drm_encoder_helper_add(encoder, &dce_v10_0_dig_helper_funcs);
@@ -3766,13 +3766,13 @@ static void dce_v10_0_encoder_add(struct amdgpu_device *adev,
 		amdgpu_encoder->is_ext_encoder = true;
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 		else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 		else
 			drm_encoder_init(dev, encoder, &dce_v10_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &dce_v10_0_ext_helper_funcs);
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 8f1e51128b33..8e67249d4367 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -211,9 +211,9 @@ static bool dce_v11_0_is_counter_moving(struct amdgpu_device *adev, int crtc)
  */
 static void dce_v11_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 {
-	unsigned i = 0;
+	unsigned i = 100;
 
-	if (crtc >= adev->mode_info.num_crtc)
+	if (crtc < 0 || crtc >= adev->mode_info.num_crtc)
 		return;
 
 	if (!(RREG32(mmCRTC_CONTROL + crtc_offsets[crtc]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK))
@@ -223,14 +223,16 @@ static void dce_v11_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 	 * wait for another frame.
 	 */
 	while (dce_v11_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v11_0_is_counter_moving(adev, crtc))
 				break;
 		}
 	}
 
 	while (!dce_v11_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v11_0_is_counter_moving(adev, crtc))
 				break;
 		}
@@ -239,7 +241,7 @@ static void dce_v11_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 
 static u32 dce_v11_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
 {
-	if (crtc >= adev->mode_info.num_crtc)
+	if (crtc < 0 || crtc >= adev->mode_info.num_crtc)
 		return 0;
 	else
 		return RREG32(mmCRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
@@ -3384,7 +3386,7 @@ static void dce_v11_0_crtc_vblank_int_ack(struct amdgpu_device *adev,
 {
 	u32 tmp;
 
-	if (crtc >= adev->mode_info.num_crtc) {
+	if (crtc < 0 || crtc >= adev->mode_info.num_crtc) {
 		DRM_DEBUG("invalid crtc %d\n", crtc);
 		return;
 	}
@@ -3399,7 +3401,7 @@ static void dce_v11_0_crtc_vline_int_ack(struct amdgpu_device *adev,
 {
 	u32 tmp;
 
-	if (crtc >= adev->mode_info.num_crtc) {
+	if (crtc < 0 || crtc >= adev->mode_info.num_crtc) {
 		DRM_DEBUG("invalid crtc %d\n", crtc);
 		return;
 	}
@@ -3722,7 +3724,7 @@ static void dce_v11_0_encoder_add(struct amdgpu_device *adev,
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
 		drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-				 DRM_MODE_ENCODER_DAC);
+				 DRM_MODE_ENCODER_DAC, NULL);
 		drm_encoder_helper_add(encoder, &dce_v11_0_dac_helper_funcs);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
@@ -3733,15 +3735,15 @@ static void dce_v11_0_encoder_add(struct amdgpu_device *adev,
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 			amdgpu_encoder->rmx_type = RMX_FULL;
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_lcd_info(amdgpu_encoder);
 		} else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		} else {
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		}
 		drm_encoder_helper_add(encoder, &dce_v11_0_dig_helper_funcs);
@@ -3759,13 +3761,13 @@ static void dce_v11_0_encoder_add(struct amdgpu_device *adev,
 		amdgpu_encoder->is_ext_encoder = true;
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 		else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 		else
 			drm_encoder_init(dev, encoder, &dce_v11_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &dce_v11_0_ext_helper_funcs);
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 42d954dc436d..d0e128c24813 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -3659,7 +3659,7 @@ static void dce_v8_0_encoder_add(struct amdgpu_device *adev,
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
 		drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-				 DRM_MODE_ENCODER_DAC);
+				 DRM_MODE_ENCODER_DAC, NULL);
 		drm_encoder_helper_add(encoder, &dce_v8_0_dac_helper_funcs);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
@@ -3670,15 +3670,15 @@ static void dce_v8_0_encoder_add(struct amdgpu_device *adev,
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 			amdgpu_encoder->rmx_type = RMX_FULL;
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_lcd_info(amdgpu_encoder);
 		} else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		} else {
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
 		}
 		drm_encoder_helper_add(encoder, &dce_v8_0_dig_helper_funcs);
@@ -3696,13 +3696,13 @@ static void dce_v8_0_encoder_add(struct amdgpu_device *adev,
 		amdgpu_encoder->is_ext_encoder = true;
 		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_LVDS);
+					 DRM_MODE_ENCODER_LVDS, NULL);
 		else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_DAC);
+					 DRM_MODE_ENCODER_DAC, NULL);
 		else
 			drm_encoder_init(dev, encoder, &dce_v8_0_encoder_funcs,
-					 DRM_MODE_ENCODER_TMDS);
+					 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &dce_v8_0_ext_helper_funcs);
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
index 8f9845d9a986..4b0e45a27129 100644
--- a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
@@ -24,7 +24,7 @@
 #include <linux/firmware.h>
 #include "drmP.h"
 #include "amdgpu.h"
-#include "fiji_smumgr.h"
+#include "fiji_smum.h"
 
 MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
 
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h b/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h
deleted file mode 100644
index 3c4824082990..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef FIJI_PP_SMC_H
-#define FIJI_PP_SMC_H
-
-#pragma pack(push, 1)
-
-#define PPSMC_SWSTATE_FLAG_DC                           0x01
-#define PPSMC_SWSTATE_FLAG_UVD                          0x02
-#define PPSMC_SWSTATE_FLAG_VCE                          0x04
-
-#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
-#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
-#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff
-
-#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01
-#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02
-#define PPSMC_SYSTEMFLAG_GDDR5                          0x04
-
-#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08
-
-#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10
-#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20
-
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
-#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
-
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
-
-#define PPSMC_DPM2FLAGS_TDPCLMP                         0x01
-#define PPSMC_DPM2FLAGS_PWRSHFT                         0x02
-#define PPSMC_DPM2FLAGS_OCP                             0x04
-
-#define PPSMC_DISPLAY_WATERMARK_LOW                     0
-#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
-
-#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    		0x01
-#define PPSMC_STATEFLAG_POWERBOOST         		0x02
-#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 		0x04
-#define PPSMC_STATEFLAG_POWERSHIFT         		0x08
-#define PPSMC_STATEFLAG_SLOW_READ_MARGIN   		0x10
-#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 		0x20
-#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   		0x40
-
-#define FDO_MODE_HARDWARE 0
-#define FDO_MODE_PIECE_WISE_LINEAR 1
-
-enum FAN_CONTROL {
-	FAN_CONTROL_FUZZY,
-	FAN_CONTROL_TABLE
-};
-
-//Gemini Modes
-#define PPSMC_GeminiModeNone   0  //Single GPU board
-#define PPSMC_GeminiModeMaster 1  //Master GPU on a Gemini board
-#define PPSMC_GeminiModeSlave  2  //Slave GPU on a Gemini board
-
-#define PPSMC_Result_OK             			((uint16_t)0x01)
-#define PPSMC_Result_NoMore         			((uint16_t)0x02)
-#define PPSMC_Result_NotNow         			((uint16_t)0x03)
-#define PPSMC_Result_Failed         			((uint16_t)0xFF)
-#define PPSMC_Result_UnknownCmd     			((uint16_t)0xFE)
-#define PPSMC_Result_UnknownVT      			((uint16_t)0xFD)
-
-typedef uint16_t PPSMC_Result;
-
-#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
-
-#define PPSMC_MSG_Halt                      		((uint16_t)0x10)
-#define PPSMC_MSG_Resume                    		((uint16_t)0x11)
-#define PPSMC_MSG_EnableDPMLevel            		((uint16_t)0x12)
-#define PPSMC_MSG_ZeroLevelsDisabled        		((uint16_t)0x13)
-#define PPSMC_MSG_OneLevelsDisabled         		((uint16_t)0x14)
-#define PPSMC_MSG_TwoLevelsDisabled         		((uint16_t)0x15)
-#define PPSMC_MSG_EnableThermalInterrupt    		((uint16_t)0x16)
-#define PPSMC_MSG_RunningOnAC               		((uint16_t)0x17)
-#define PPSMC_MSG_LevelUp                   		((uint16_t)0x18)
-#define PPSMC_MSG_LevelDown                 		((uint16_t)0x19)
-#define PPSMC_MSG_ResetDPMCounters          		((uint16_t)0x1a)
-#define PPSMC_MSG_SwitchToSwState           		((uint16_t)0x20)
-#define PPSMC_MSG_SwitchToSwStateLast       		((uint16_t)0x3f)
-#define PPSMC_MSG_SwitchToInitialState      		((uint16_t)0x40)
-#define PPSMC_MSG_NoForcedLevel             		((uint16_t)0x41)
-#define PPSMC_MSG_ForceHigh                 		((uint16_t)0x42)
-#define PPSMC_MSG_ForceMediumOrHigh         		((uint16_t)0x43)
-#define PPSMC_MSG_SwitchToMinimumPower      		((uint16_t)0x51)
-#define PPSMC_MSG_ResumeFromMinimumPower    		((uint16_t)0x52)
-#define PPSMC_MSG_EnableCac                 		((uint16_t)0x53)
-#define PPSMC_MSG_DisableCac                		((uint16_t)0x54)
-#define PPSMC_DPMStateHistoryStart          		((uint16_t)0x55)
-#define PPSMC_DPMStateHistoryStop           		((uint16_t)0x56)
-#define PPSMC_CACHistoryStart               		((uint16_t)0x57)
-#define PPSMC_CACHistoryStop                		((uint16_t)0x58)
-#define PPSMC_TDPClampingActive             		((uint16_t)0x59)
-#define PPSMC_TDPClampingInactive           		((uint16_t)0x5A)
-#define PPSMC_StartFanControl               		((uint16_t)0x5B)
-#define PPSMC_StopFanControl                		((uint16_t)0x5C)
-#define PPSMC_NoDisplay                     		((uint16_t)0x5D)
-#define PPSMC_HasDisplay                    		((uint16_t)0x5E)
-#define PPSMC_MSG_UVDPowerOFF               		((uint16_t)0x60)
-#define PPSMC_MSG_UVDPowerON                		((uint16_t)0x61)
-#define PPSMC_MSG_EnableULV                 		((uint16_t)0x62)
-#define PPSMC_MSG_DisableULV                		((uint16_t)0x63)
-#define PPSMC_MSG_EnterULV                  		((uint16_t)0x64)
-#define PPSMC_MSG_ExitULV                   		((uint16_t)0x65)
-#define PPSMC_PowerShiftActive              		((uint16_t)0x6A)
-#define PPSMC_PowerShiftInactive            		((uint16_t)0x6B)
-#define PPSMC_OCPActive                     		((uint16_t)0x6C)
-#define PPSMC_OCPInactive                   		((uint16_t)0x6D)
-#define PPSMC_CACLongTermAvgEnable          		((uint16_t)0x6E)
-#define PPSMC_CACLongTermAvgDisable         		((uint16_t)0x6F)
-#define PPSMC_MSG_InferredStateSweep_Start  		((uint16_t)0x70)
-#define PPSMC_MSG_InferredStateSweep_Stop   		((uint16_t)0x71)
-#define PPSMC_MSG_SwitchToLowestInfState    		((uint16_t)0x72)
-#define PPSMC_MSG_SwitchToNonInfState       		((uint16_t)0x73)
-#define PPSMC_MSG_AllStateSweep_Start       		((uint16_t)0x74)
-#define PPSMC_MSG_AllStateSweep_Stop        		((uint16_t)0x75)
-#define PPSMC_MSG_SwitchNextLowerInfState   		((uint16_t)0x76)
-#define PPSMC_MSG_SwitchNextHigherInfState  		((uint16_t)0x77)
-#define PPSMC_MSG_MclkRetrainingTest        		((uint16_t)0x78)
-#define PPSMC_MSG_ForceTDPClamping          		((uint16_t)0x79)
-#define PPSMC_MSG_CollectCAC_PowerCorreln   		((uint16_t)0x7A)
-#define PPSMC_MSG_CollectCAC_WeightCalib    		((uint16_t)0x7B)
-#define PPSMC_MSG_CollectCAC_SQonly         		((uint16_t)0x7C)
-#define PPSMC_MSG_CollectCAC_TemperaturePwr 		((uint16_t)0x7D)
-#define PPSMC_MSG_ExtremitiesTest_Start     		((uint16_t)0x7E)
-#define PPSMC_MSG_ExtremitiesTest_Stop      		((uint16_t)0x7F)
-#define PPSMC_FlushDataCache                		((uint16_t)0x80)
-#define PPSMC_FlushInstrCache               		((uint16_t)0x81)
-#define PPSMC_MSG_SetEnabledLevels          		((uint16_t)0x82)
-#define PPSMC_MSG_SetForcedLevels           		((uint16_t)0x83)
-#define PPSMC_MSG_ResetToDefaults           		((uint16_t)0x84)
-#define PPSMC_MSG_SetForcedLevelsAndJump    		((uint16_t)0x85)
-#define PPSMC_MSG_SetCACHistoryMode         		((uint16_t)0x86)
-#define PPSMC_MSG_EnableDTE                 		((uint16_t)0x87)
-#define PPSMC_MSG_DisableDTE                		((uint16_t)0x88)
-#define PPSMC_MSG_SmcSpaceSetAddress        		((uint16_t)0x89)
-#define PPSMC_MSG_SmcSpaceWriteDWordInc     		((uint16_t)0x8A)
-#define PPSMC_MSG_SmcSpaceWriteWordInc      		((uint16_t)0x8B)
-#define PPSMC_MSG_SmcSpaceWriteByteInc      		((uint16_t)0x8C)
-
-#define PPSMC_MSG_BREAK                     		((uint16_t)0xF8)
-
-#define PPSMC_MSG_Test                      		((uint16_t)0x100)
-#define PPSMC_MSG_DRV_DRAM_ADDR_HI            		((uint16_t)0x250)
-#define PPSMC_MSG_DRV_DRAM_ADDR_LO            		((uint16_t)0x251)
-#define PPSMC_MSG_SMU_DRAM_ADDR_HI            		((uint16_t)0x252)
-#define PPSMC_MSG_SMU_DRAM_ADDR_LO            		((uint16_t)0x253)
-#define PPSMC_MSG_LoadUcodes                  		((uint16_t)0x254)
-
-typedef uint16_t PPSMC_Msg;
-
-#define PPSMC_EVENT_STATUS_THERMAL          		0x00000001
-#define PPSMC_EVENT_STATUS_REGULATORHOT     		0x00000002
-#define PPSMC_EVENT_STATUS_DC               		0x00000004
-#define PPSMC_EVENT_STATUS_GPIO17           		0x00000008
-
-#pragma pack(pop)
-
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
index bda1249eb871..e35340afd3db 100644
--- a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
@@ -25,7 +25,7 @@
 #include "drmP.h"
 #include "amdgpu.h"
 #include "fiji_ppsmc.h"
-#include "fiji_smumgr.h"
+#include "fiji_smum.h"
 #include "smu_ucode_xfer_vi.h"
 #include "amdgpu_ucode.h"
 
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h b/drivers/gpu/drm/amd/amdgpu/fiji_smum.h
index 1cef03deeac3..1cef03deeac3 100644
--- a/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_smum.h
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index e1dcab98e249..95c0cdfbd1b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -66,6 +66,27 @@
 #define MACRO_TILE_ASPECT(x)				((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
 #define NUM_BANKS(x)					((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
 
+#define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L
+#define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L
+#define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L
+#define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L
+
+/* BPM SERDES CMD */
+#define SET_BPM_SERDES_CMD    1
+#define CLE_BPM_SERDES_CMD    0
+
+/* BPM Register Address*/
+enum {
+	BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */
+	BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */
+	BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */
+	BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */
+	BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */
+	BPM_REG_FGCG_MAX
+};
+
 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
@@ -964,6 +985,322 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
 	return 0;
 }
 
+static const u32 vgpr_init_compute_shader[] =
+{
+	0x7e000209, 0x7e020208,
+	0x7e040207, 0x7e060206,
+	0x7e080205, 0x7e0a0204,
+	0x7e0c0203, 0x7e0e0202,
+	0x7e100201, 0x7e120200,
+	0x7e140209, 0x7e160208,
+	0x7e180207, 0x7e1a0206,
+	0x7e1c0205, 0x7e1e0204,
+	0x7e200203, 0x7e220202,
+	0x7e240201, 0x7e260200,
+	0x7e280209, 0x7e2a0208,
+	0x7e2c0207, 0x7e2e0206,
+	0x7e300205, 0x7e320204,
+	0x7e340203, 0x7e360202,
+	0x7e380201, 0x7e3a0200,
+	0x7e3c0209, 0x7e3e0208,
+	0x7e400207, 0x7e420206,
+	0x7e440205, 0x7e460204,
+	0x7e480203, 0x7e4a0202,
+	0x7e4c0201, 0x7e4e0200,
+	0x7e500209, 0x7e520208,
+	0x7e540207, 0x7e560206,
+	0x7e580205, 0x7e5a0204,
+	0x7e5c0203, 0x7e5e0202,
+	0x7e600201, 0x7e620200,
+	0x7e640209, 0x7e660208,
+	0x7e680207, 0x7e6a0206,
+	0x7e6c0205, 0x7e6e0204,
+	0x7e700203, 0x7e720202,
+	0x7e740201, 0x7e760200,
+	0x7e780209, 0x7e7a0208,
+	0x7e7c0207, 0x7e7e0206,
+	0xbf8a0000, 0xbf810000,
+};
+
+static const u32 sgpr_init_compute_shader[] =
+{
+	0xbe8a0100, 0xbe8c0102,
+	0xbe8e0104, 0xbe900106,
+	0xbe920108, 0xbe940100,
+	0xbe960102, 0xbe980104,
+	0xbe9a0106, 0xbe9c0108,
+	0xbe9e0100, 0xbea00102,
+	0xbea20104, 0xbea40106,
+	0xbea60108, 0xbea80100,
+	0xbeaa0102, 0xbeac0104,
+	0xbeae0106, 0xbeb00108,
+	0xbeb20100, 0xbeb40102,
+	0xbeb60104, 0xbeb80106,
+	0xbeba0108, 0xbebc0100,
+	0xbebe0102, 0xbec00104,
+	0xbec20106, 0xbec40108,
+	0xbec60100, 0xbec80102,
+	0xbee60004, 0xbee70005,
+	0xbeea0006, 0xbeeb0007,
+	0xbee80008, 0xbee90009,
+	0xbefc0000, 0xbf8a0000,
+	0xbf810000, 0x00000000,
+};
+
+static const u32 vgpr_init_regs[] =
+{
+	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
+	mmCOMPUTE_RESOURCE_LIMITS, 0,
+	mmCOMPUTE_NUM_THREAD_X, 256*4,
+	mmCOMPUTE_NUM_THREAD_Y, 1,
+	mmCOMPUTE_NUM_THREAD_Z, 1,
+	mmCOMPUTE_PGM_RSRC2, 20,
+	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
+	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
+	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
+	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
+	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
+	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
+	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
+	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
+	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
+	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
+};
+
+static const u32 sgpr1_init_regs[] =
+{
+	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
+	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
+	mmCOMPUTE_NUM_THREAD_X, 256*5,
+	mmCOMPUTE_NUM_THREAD_Y, 1,
+	mmCOMPUTE_NUM_THREAD_Z, 1,
+	mmCOMPUTE_PGM_RSRC2, 20,
+	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
+	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
+	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
+	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
+	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
+	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
+	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
+	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
+	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
+	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
+};
+
+static const u32 sgpr2_init_regs[] =
+{
+	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
+	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
+	mmCOMPUTE_NUM_THREAD_X, 256*5,
+	mmCOMPUTE_NUM_THREAD_Y, 1,
+	mmCOMPUTE_NUM_THREAD_Z, 1,
+	mmCOMPUTE_PGM_RSRC2, 20,
+	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
+	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
+	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
+	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
+	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
+	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
+	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
+	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
+	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
+	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
+};
+
+static const u32 sec_ded_counter_registers[] =
+{
+	mmCPC_EDC_ATC_CNT,
+	mmCPC_EDC_SCRATCH_CNT,
+	mmCPC_EDC_UCODE_CNT,
+	mmCPF_EDC_ATC_CNT,
+	mmCPF_EDC_ROQ_CNT,
+	mmCPF_EDC_TAG_CNT,
+	mmCPG_EDC_ATC_CNT,
+	mmCPG_EDC_DMA_CNT,
+	mmCPG_EDC_TAG_CNT,
+	mmDC_EDC_CSINVOC_CNT,
+	mmDC_EDC_RESTORE_CNT,
+	mmDC_EDC_STATE_CNT,
+	mmGDS_EDC_CNT,
+	mmGDS_EDC_GRBM_CNT,
+	mmGDS_EDC_OA_DED,
+	mmSPI_EDC_CNT,
+	mmSQC_ATC_EDC_GATCL1_CNT,
+	mmSQC_EDC_CNT,
+	mmSQ_EDC_DED_CNT,
+	mmSQ_EDC_INFO,
+	mmSQ_EDC_SEC_CNT,
+	mmTCC_EDC_CNT,
+	mmTCP_ATC_EDC_GATCL1_CNT,
+	mmTCP_EDC_CNT,
+	mmTD_EDC_CNT
+};
+
+static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
+	struct amdgpu_ib ib;
+	struct fence *f = NULL;
+	int r, i;
+	u32 tmp;
+	unsigned total_size, vgpr_offset, sgpr_offset;
+	u64 gpu_addr;
+
+	/* only supported on CZ */
+	if (adev->asic_type != CHIP_CARRIZO)
+		return 0;
+
+	/* bail if the compute ring is not ready */
+	if (!ring->ready)
+		return 0;
+
+	tmp = RREG32(mmGB_EDC_MODE);
+	WREG32(mmGB_EDC_MODE, 0);
+
+	total_size =
+		(((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
+	total_size +=
+		(((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
+	total_size +=
+		(((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
+	total_size = ALIGN(total_size, 256);
+	vgpr_offset = total_size;
+	total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
+	sgpr_offset = total_size;
+	total_size += sizeof(sgpr_init_compute_shader);
+
+	/* allocate an indirect buffer to put the commands in */
+	memset(&ib, 0, sizeof(ib));
+	r = amdgpu_ib_get(ring, NULL, total_size, &ib);
+	if (r) {
+		DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+		return r;
+	}
+
+	/* load the compute shaders */
+	for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
+		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
+
+	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
+		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
+
+	/* init the ib length to 0 */
+	ib.length_dw = 0;
+
+	/* VGPR */
+	/* write the register state for the compute dispatch */
+	for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
+		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
+	}
+	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
+	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
+	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+
+	/* write dispatch packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib.ptr[ib.length_dw++] = 8; /* x */
+	ib.ptr[ib.length_dw++] = 1; /* y */
+	ib.ptr[ib.length_dw++] = 1; /* z */
+	ib.ptr[ib.length_dw++] =
+		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
+
+	/* write CS partial flush packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+
+	/* SGPR1 */
+	/* write the register state for the compute dispatch */
+	for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
+		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
+	}
+	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
+	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
+	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+
+	/* write dispatch packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib.ptr[ib.length_dw++] = 8; /* x */
+	ib.ptr[ib.length_dw++] = 1; /* y */
+	ib.ptr[ib.length_dw++] = 1; /* z */
+	ib.ptr[ib.length_dw++] =
+		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
+
+	/* write CS partial flush packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+
+	/* SGPR2 */
+	/* write the register state for the compute dispatch */
+	for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
+		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
+	}
+	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
+	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
+	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+
+	/* write dispatch packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib.ptr[ib.length_dw++] = 8; /* x */
+	ib.ptr[ib.length_dw++] = 1; /* y */
+	ib.ptr[ib.length_dw++] = 1; /* z */
+	ib.ptr[ib.length_dw++] =
+		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
+
+	/* write CS partial flush packet */
+	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+
+	/* shedule the ib on the ring */
+	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+						 AMDGPU_FENCE_OWNER_UNDEFINED,
+						 &f);
+	if (r) {
+		DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
+		goto fail;
+	}
+
+	/* wait for the GPU to finish processing the IB */
+	r = fence_wait(f, false);
+	if (r) {
+		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+		goto fail;
+	}
+
+	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
+	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
+	WREG32(mmGB_EDC_MODE, tmp);
+
+	tmp = RREG32(mmCC_GC_EDC_CONFIG);
+	tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
+	WREG32(mmCC_GC_EDC_CONFIG, tmp);
+
+
+	/* read back registers to clear the counters */
+	for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
+		RREG32(sec_ded_counter_registers[i]);
+
+fail:
+	fence_put(f);
+	amdgpu_ib_free(adev, &ib);
+
+	return r;
+}
+
 static void gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
 {
 	u32 gb_addr_config;
@@ -1323,1418 +1660,923 @@ static int gfx_v8_0_sw_fini(void *handle)
 
 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
 {
-	const u32 num_tile_mode_states = 32;
-	const u32 num_secondary_tile_mode_states = 16;
-	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
+	uint32_t *modearray, *mod2array;
+	const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+	const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+	u32 reg_offset;
 
-	switch (adev->gfx.config.mem_row_size_in_kb) {
-	case 1:
-		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
-		break;
-	case 2:
-	default:
-		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
-		break;
-	case 4:
-		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
-		break;
-	}
+	modearray = adev->gfx.config.tile_mode_array;
+	mod2array = adev->gfx.config.macrotile_mode_array;
+
+	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+		modearray[reg_offset] = 0;
+
+	for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)
+		mod2array[reg_offset] = 0;
 
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
-		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 1:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 2:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 3:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 4:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 5:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 6:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 8:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
-						PIPE_CONFIG(ADDR_SURF_P2));
-				break;
-			case 9:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 10:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 11:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 13:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 14:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 15:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 16:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 18:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 19:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 20:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 21:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 22:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 24:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 25:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 26:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 27:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 28:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 29:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 7:
-			case 12:
-			case 17:
-			case 23:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
-		}
-		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 1:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 2:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 3:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 4:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 5:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 6:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 8:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 9:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 10:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 11:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 12:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 13:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 14:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 7:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
-		}
+		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+				PIPE_CONFIG(ADDR_SURF_P2));
+		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+
+		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
+			    reg_offset != 23)
+				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
+
+		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
+			if (reg_offset != 7)
+				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
+
+		break;
 	case CHIP_FIJI:
-		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 1:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 2:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 3:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 4:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 5:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 6:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 7:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 8:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
-				break;
-			case 9:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 10:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 11:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 12:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 13:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 14:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 15:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 16:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 17:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 18:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 19:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 20:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 21:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 22:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 23:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 24:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 25:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 26:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 27:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 28:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 29:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 30:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			default:
-				gb_tile_moden = 0;
-				break;
-			}
-			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
-		}
-		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 1:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 2:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 3:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 4:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 5:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 6:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 8:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 9:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 10:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 11:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 12:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 13:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 14:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_4_BANK));
-				break;
-			case 7:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			}
-			adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
-		}
+		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
+		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+
+		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_4_BANK));
+
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
+
+		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
+			if (reg_offset != 7)
+				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
+
 		break;
 	case CHIP_TONGA:
-		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 1:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 2:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 3:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 4:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 5:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 6:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 7:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 8:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
-				break;
-			case 9:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 10:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 11:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 12:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 13:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 14:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 15:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 16:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 17:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 18:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 19:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 20:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 21:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 22:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 23:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 24:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 25:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 26:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 27:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 28:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 29:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 30:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P4_16x16) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
-		}
-		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 1:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 2:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 3:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 4:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 5:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 6:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 8:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 9:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 10:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 11:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 12:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 13:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_4_BANK));
-				break;
-			case 14:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
-						NUM_BANKS(ADDR_SURF_4_BANK));
-				break;
-			case 7:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
-		}
+		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
+		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+
+		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_4_BANK));
+		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+				 NUM_BANKS(ADDR_SURF_4_BANK));
+
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
+
+		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
+			if (reg_offset != 7)
+				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
+
 		break;
 	case CHIP_STONEY:
-		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 1:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 2:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 3:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 4:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 5:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 6:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 8:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
-						PIPE_CONFIG(ADDR_SURF_P2));
-				break;
-			case 9:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 10:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 11:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 13:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 14:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 15:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 16:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 18:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 19:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 20:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 21:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 22:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 24:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 25:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 26:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 27:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 28:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 29:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 7:
-			case 12:
-			case 17:
-			case 23:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
-		}
-		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 1:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 2:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 3:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 4:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 5:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 6:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 8:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 9:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 10:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 11:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 12:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 13:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 14:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 7:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
-		}
+		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+				PIPE_CONFIG(ADDR_SURF_P2));
+		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+
+		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
+			    reg_offset != 23)
+				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
+
+		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
+			if (reg_offset != 7)
+				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
+
 		break;
-	case CHIP_CARRIZO:
 	default:
-		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 1:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 2:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 3:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 4:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 5:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 6:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
-				break;
-			case 8:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
-						PIPE_CONFIG(ADDR_SURF_P2));
-				break;
-			case 9:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 10:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 11:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 13:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 14:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 15:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 16:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 18:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 19:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 20:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 21:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 22:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 24:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 25:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 26:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
-				break;
-			case 27:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 28:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
-				break;
-			case 29:
-				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
-						PIPE_CONFIG(ADDR_SURF_P2) |
-						MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
-						SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
-				break;
-			case 7:
-			case 12:
-			case 17:
-			case 23:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
-		}
-		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
-			switch (reg_offset) {
-			case 0:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 1:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 2:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 3:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 4:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 5:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 6:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 8:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 9:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 10:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 11:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 12:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 13:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
-						NUM_BANKS(ADDR_SURF_16_BANK));
-				break;
-			case 14:
-				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
-						BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
-						MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
-						NUM_BANKS(ADDR_SURF_8_BANK));
-				break;
-			case 7:
-				/* unused idx */
-				continue;
-			default:
-				gb_tile_moden = 0;
-				break;
-			};
-			adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
-		}
+		dev_warn(adev->dev,
+			 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
+			 adev->asic_type);
+
+	case CHIP_CARRIZO:
+		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+				PIPE_CONFIG(ADDR_SURF_P2));
+		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				PIPE_CONFIG(ADDR_SURF_P2) |
+				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
+		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+				 PIPE_CONFIG(ADDR_SURF_P2) |
+				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
+
+		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_8_BANK));
+		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+				 NUM_BANKS(ADDR_SURF_16_BANK));
+		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				 NUM_BANKS(ADDR_SURF_8_BANK));
+
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
+			    reg_offset != 23)
+				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
+
+		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
+			if (reg_offset != 7)
+				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
+
+		break;
 	}
 }
 
 static u32 gfx_v8_0_create_bitmask(u32 bit_width)
 {
-	u32 i, mask = 0;
-
-	for (i = 0; i < bit_width; i++) {
-		mask <<= 1;
-		mask |= 1;
-	}
-	return mask;
+	return (u32)((1ULL << bit_width) - 1);
 }
 
 void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num)
@@ -2809,7 +2651,7 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev,
 	mutex_lock(&adev->grbm_idx_mutex);
 	for (i = 0; i < se_num; i++) {
 		gfx_v8_0_select_se_sh(adev, i, 0xffffffff);
-		data = 0;
+		data = RREG32(mmPA_SC_RASTER_CONFIG);
 		for (j = 0; j < sh_per_se; j++) {
 			switch (enabled_rbs & 3) {
 			case 0:
@@ -2997,17 +2839,11 @@ static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
 {
 	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
 
-	if (enable) {
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 1);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 1);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 1);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 1);
-	} else {
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 0);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 0);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 0);
-		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 0);
-	}
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
+
 	WREG32(mmCP_INT_CNTL_RING0, tmp);
 }
 
@@ -3087,16 +2923,18 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
 
 	gfx_v8_0_rlc_reset(adev);
 
-	if (!adev->firmware.smu_load) {
-		/* legacy rlc firmware loading */
-		r = gfx_v8_0_rlc_load_microcode(adev);
-		if (r)
-			return r;
-	} else {
-		r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-						AMDGPU_UCODE_ID_RLC_G);
-		if (r)
-			return -EINVAL;
+	if (!adev->pp_enabled) {
+		if (!adev->firmware.smu_load) {
+			/* legacy rlc firmware loading */
+			r = gfx_v8_0_rlc_load_microcode(adev);
+			if (r)
+				return r;
+		} else {
+			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+							AMDGPU_UCODE_ID_RLC_G);
+			if (r)
+				return -EINVAL;
+		}
 	}
 
 	gfx_v8_0_rlc_start(adev);
@@ -3941,6 +3779,11 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
 		tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
 		WREG32(mmCP_HQD_PERSISTENT_STATE, tmp);
 		mqd->cp_hqd_persistent_state = tmp;
+		if (adev->asic_type == CHIP_STONEY) {
+			tmp = RREG32(mmCP_ME1_PIPE3_INT_CNTL);
+			tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE3_INT_CNTL, GENERIC2_INT_ENABLE, 1);
+			WREG32(mmCP_ME1_PIPE3_INT_CNTL, tmp);
+		}
 
 		/* activate the queue */
 		mqd->cp_hqd_active = 1;
@@ -3982,35 +3825,37 @@ static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
 	if (!(adev->flags & AMD_IS_APU))
 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
 
-	if (!adev->firmware.smu_load) {
-		/* legacy firmware loading */
-		r = gfx_v8_0_cp_gfx_load_microcode(adev);
-		if (r)
-			return r;
-
-		r = gfx_v8_0_cp_compute_load_microcode(adev);
-		if (r)
-			return r;
-	} else {
-		r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-						AMDGPU_UCODE_ID_CP_CE);
-		if (r)
-			return -EINVAL;
-
-		r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-						AMDGPU_UCODE_ID_CP_PFP);
-		if (r)
-			return -EINVAL;
-
-		r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-						AMDGPU_UCODE_ID_CP_ME);
-		if (r)
-			return -EINVAL;
+	if (!adev->pp_enabled) {
+		if (!adev->firmware.smu_load) {
+			/* legacy firmware loading */
+			r = gfx_v8_0_cp_gfx_load_microcode(adev);
+			if (r)
+				return r;
 
-		r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-						AMDGPU_UCODE_ID_CP_MEC1);
-		if (r)
-			return -EINVAL;
+			r = gfx_v8_0_cp_compute_load_microcode(adev);
+			if (r)
+				return r;
+		} else {
+			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+							AMDGPU_UCODE_ID_CP_CE);
+			if (r)
+				return -EINVAL;
+
+			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+							AMDGPU_UCODE_ID_CP_PFP);
+			if (r)
+				return -EINVAL;
+
+			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+							AMDGPU_UCODE_ID_CP_ME);
+			if (r)
+				return -EINVAL;
+
+			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+							AMDGPU_UCODE_ID_CP_MEC1);
+			if (r)
+				return -EINVAL;
+		}
 	}
 
 	r = gfx_v8_0_cp_gfx_resume(adev);
@@ -4341,7 +4186,18 @@ static int gfx_v8_0_soft_reset(void *handle)
 		gfx_v8_0_cp_gfx_enable(adev, false);
 
 		/* Disable MEC parsing/prefetching */
-		/* XXX todo */
+		gfx_v8_0_cp_compute_enable(adev, false);
+
+		if (grbm_soft_reset || srbm_soft_reset) {
+			tmp = RREG32(mmGMCON_DEBUG);
+			tmp = REG_SET_FIELD(tmp,
+					    GMCON_DEBUG, GFX_STALL, 1);
+			tmp = REG_SET_FIELD(tmp,
+					    GMCON_DEBUG, GFX_CLEAR, 1);
+			WREG32(mmGMCON_DEBUG, tmp);
+
+			udelay(50);
+		}
 
 		if (grbm_soft_reset) {
 			tmp = RREG32(mmGRBM_SOFT_RESET);
@@ -4370,6 +4226,16 @@ static int gfx_v8_0_soft_reset(void *handle)
 			WREG32(mmSRBM_SOFT_RESET, tmp);
 			tmp = RREG32(mmSRBM_SOFT_RESET);
 		}
+
+		if (grbm_soft_reset || srbm_soft_reset) {
+			tmp = RREG32(mmGMCON_DEBUG);
+			tmp = REG_SET_FIELD(tmp,
+					    GMCON_DEBUG, GFX_STALL, 0);
+			tmp = REG_SET_FIELD(tmp,
+					    GMCON_DEBUG, GFX_CLEAR, 0);
+			WREG32(mmGMCON_DEBUG, tmp);
+		}
+
 		/* Wait a little for things to settle down */
 		udelay(50);
 		gfx_v8_0_print_status((void *)adev);
@@ -4458,15 +4324,261 @@ static int gfx_v8_0_early_init(void *handle)
 	return 0;
 }
 
+static int gfx_v8_0_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int r;
+
+	/* requires IBs so do in late init after IB pool is initialized */
+	r = gfx_v8_0_do_edc_gpr_workarounds(adev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
 static int gfx_v8_0_set_powergating_state(void *handle,
 					  enum amd_powergating_state state)
 {
 	return 0;
 }
 
+static void fiji_send_serdes_cmd(struct amdgpu_device *adev,
+		uint32_t reg_addr, uint32_t cmd)
+{
+	uint32_t data;
+
+	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+
+	WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
+	WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
+
+	data = RREG32(mmRLC_SERDES_WR_CTRL);
+	data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
+			RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
+			RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
+			RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
+			RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
+			RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
+			RLC_SERDES_WR_CTRL__POWER_UP_MASK |
+			RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
+			RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
+			RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
+			RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
+	data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
+			(cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
+			(reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
+			(0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
+
+	WREG32(mmRLC_SERDES_WR_CTRL, data);
+}
+
+static void fiji_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	/* It is disabled by HW by default */
+	if (enable) {
+		/* 1 - RLC memory Light sleep */
+		temp = data = RREG32(mmRLC_MEM_SLP_CNTL);
+		data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
+		if (temp != data)
+			WREG32(mmRLC_MEM_SLP_CNTL, data);
+
+		/* 2 - CP memory Light sleep */
+		temp = data = RREG32(mmCP_MEM_SLP_CNTL);
+		data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
+		if (temp != data)
+			WREG32(mmCP_MEM_SLP_CNTL, data);
+
+		/* 3 - RLC_CGTT_MGCG_OVERRIDE */
+		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+		data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
+
+		if (temp != data)
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
+
+		/* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* 5 - clear mgcg override */
+		fiji_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
+
+		/* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
+		temp = data = RREG32(mmCGTS_SM_CTRL_REG);
+		data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
+		data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
+		data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
+		data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
+		data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
+		data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
+		data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
+		if (temp != data)
+			WREG32(mmCGTS_SM_CTRL_REG, data);
+		udelay(50);
+
+		/* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+	} else {
+		/* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
+		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+		data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
+		if (temp != data)
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
+
+		/* 2 - disable MGLS in RLC */
+		data = RREG32(mmRLC_MEM_SLP_CNTL);
+		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
+			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
+			WREG32(mmRLC_MEM_SLP_CNTL, data);
+		}
+
+		/* 3 - disable MGLS in CP */
+		data = RREG32(mmCP_MEM_SLP_CNTL);
+		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
+			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
+			WREG32(mmCP_MEM_SLP_CNTL, data);
+		}
+
+		/* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
+		temp = data = RREG32(mmCGTS_SM_CTRL_REG);
+		data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
+				CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
+		if (temp != data)
+			WREG32(mmCGTS_SM_CTRL_REG, data);
+
+		/* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* 6 - set mgcg override */
+		fiji_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
+
+		udelay(50);
+
+		/* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+	}
+}
+
+static void fiji_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, temp1, data, data1;
+
+	temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
+
+	if (enable) {
+		/* 1 enable cntx_empty_int_enable/cntx_busy_int_enable/
+		 * Cmp_busy/GFX_Idle interrupts
+		 */
+		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
+
+		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
+		if (temp1 != data1)
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
+
+		/* 2 wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* 3 - clear cgcg override */
+		fiji_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
+
+		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* 4 - write cmd to set CGLS */
+		fiji_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
+
+		/* 5 - enable cgcg */
+		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
+
+		/* enable cgls*/
+		data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
+
+		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
+
+		if (temp1 != data1)
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
+
+		if (temp != data)
+			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
+	} else {
+		/* disable cntx_empty_int_enable & GFX Idle interrupt */
+		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
+
+		/* TEST CGCG */
+		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+		data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
+		if (temp1 != data1)
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
+
+		/* read gfx register to wake up cgcg */
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+
+		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* write cmd to Set CGCG Overrride */
+		fiji_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
+
+		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		gfx_v8_0_wait_for_rlc_serdes(adev);
+
+		/* write cmd to Clear CGLS */
+		fiji_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
+
+		/* disable cgcg, cgls should be disabled too. */
+		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
+				RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
+		if (temp != data)
+			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
+	}
+}
+static int fiji_update_gfx_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	if (enable) {
+		/* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
+		 * ===  MGCG + MGLS + TS(CG/LS) ===
+		 */
+		fiji_update_medium_grain_clock_gating(adev, enable);
+		fiji_update_coarse_grain_clock_gating(adev, enable);
+	} else {
+		/* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
+		 * ===  CGCG + CGLS ===
+		 */
+		fiji_update_coarse_grain_clock_gating(adev, enable);
+		fiji_update_medium_grain_clock_gating(adev, enable);
+	}
+	return 0;
+}
+
 static int gfx_v8_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		fiji_update_gfx_clock_gating(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		break;
+	default:
+		break;
+	}
 	return 0;
 }
 
@@ -4627,7 +4739,7 @@ static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
 				 EVENT_INDEX(5)));
 	amdgpu_ring_write(ring, addr & 0xfffffffc);
-	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 
+	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
 			  DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
 	amdgpu_ring_write(ring, lower_32_bits(seq));
 	amdgpu_ring_write(ring, upper_32_bits(seq));
@@ -4995,7 +5107,7 @@ static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
 
 const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.early_init = gfx_v8_0_early_init,
-	.late_init = NULL,
+	.late_init = gfx_v8_0_late_init,
 	.sw_init = gfx_v8_0_sw_init,
 	.sw_fini = gfx_v8_0_sw_fini,
 	.hw_init = gfx_v8_0_hw_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index ed8abb58a785..3f956065d069 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -370,6 +370,10 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
+	/* In case the PCI BAR is larger than the actual amount of vram */
+	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
+		adev->mc.visible_vram_size = adev->mc.real_vram_size;
+
 	/* unless the user had overridden it, set the gart
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
@@ -1012,7 +1016,6 @@ static int gmc_v7_0_suspend(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	if (adev->vm_manager.enabled) {
-		amdgpu_vm_manager_fini(adev);
 		gmc_v7_0_vm_fini(adev);
 		adev->vm_manager.enabled = false;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index d39028440814..c0c9a0101eb4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -476,6 +476,10 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
+	/* In case the PCI BAR is larger than the actual amount of vram */
+	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
+		adev->mc.visible_vram_size = adev->mc.real_vram_size;
+
 	/* unless the user had overridden it, set the gart
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
@@ -1033,7 +1037,6 @@ static int gmc_v8_0_suspend(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	if (adev->vm_manager.enabled) {
-		amdgpu_vm_manager_fini(adev);
 		gmc_v8_0_vm_fini(adev);
 		adev->vm_manager.enabled = false;
 	}
@@ -1324,9 +1327,181 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t data;
+
+	if (enable) {
+		data = RREG32(mmMC_HUB_MISC_HUB_CG);
+		data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_HUB_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_SIP_CG);
+		data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_SIP_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_VM_CG);
+		data |= MC_HUB_MISC_VM_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_VM_CG, data);
+
+		data = RREG32(mmMC_XPB_CLK_GAT);
+		data |= MC_XPB_CLK_GAT__ENABLE_MASK;
+		WREG32(mmMC_XPB_CLK_GAT, data);
+
+		data = RREG32(mmATC_MISC_CG);
+		data |= ATC_MISC_CG__ENABLE_MASK;
+		WREG32(mmATC_MISC_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_WR_CG);
+		data |= MC_CITF_MISC_WR_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_WR_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_RD_CG);
+		data |= MC_CITF_MISC_RD_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_RD_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_VM_CG);
+		data |= MC_CITF_MISC_VM_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_VM_CG, data);
+
+		data = RREG32(mmVM_L2_CG);
+		data |= VM_L2_CG__ENABLE_MASK;
+		WREG32(mmVM_L2_CG, data);
+	} else {
+		data = RREG32(mmMC_HUB_MISC_HUB_CG);
+		data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_HUB_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_SIP_CG);
+		data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_SIP_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_VM_CG);
+		data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_VM_CG, data);
+
+		data = RREG32(mmMC_XPB_CLK_GAT);
+		data &= ~MC_XPB_CLK_GAT__ENABLE_MASK;
+		WREG32(mmMC_XPB_CLK_GAT, data);
+
+		data = RREG32(mmATC_MISC_CG);
+		data &= ~ATC_MISC_CG__ENABLE_MASK;
+		WREG32(mmATC_MISC_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_WR_CG);
+		data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_WR_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_RD_CG);
+		data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_RD_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_VM_CG);
+		data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_VM_CG, data);
+
+		data = RREG32(mmVM_L2_CG);
+		data &= ~VM_L2_CG__ENABLE_MASK;
+		WREG32(mmVM_L2_CG, data);
+	}
+}
+
+static void fiji_update_mc_light_sleep(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t data;
+
+	if (enable) {
+		data = RREG32(mmMC_HUB_MISC_HUB_CG);
+		data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_HUB_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_SIP_CG);
+		data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_SIP_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_VM_CG);
+		data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_VM_CG, data);
+
+		data = RREG32(mmMC_XPB_CLK_GAT);
+		data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_XPB_CLK_GAT, data);
+
+		data = RREG32(mmATC_MISC_CG);
+		data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmATC_MISC_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_WR_CG);
+		data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_WR_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_RD_CG);
+		data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_RD_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_VM_CG);
+		data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_VM_CG, data);
+
+		data = RREG32(mmVM_L2_CG);
+		data |= VM_L2_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmVM_L2_CG, data);
+	} else {
+		data = RREG32(mmMC_HUB_MISC_HUB_CG);
+		data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_HUB_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_SIP_CG);
+		data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_SIP_CG, data);
+
+		data = RREG32(mmMC_HUB_MISC_VM_CG);
+		data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_HUB_MISC_VM_CG, data);
+
+		data = RREG32(mmMC_XPB_CLK_GAT);
+		data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_XPB_CLK_GAT, data);
+
+		data = RREG32(mmATC_MISC_CG);
+		data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmATC_MISC_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_WR_CG);
+		data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_WR_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_RD_CG);
+		data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_RD_CG, data);
+
+		data = RREG32(mmMC_CITF_MISC_VM_CG);
+		data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmMC_CITF_MISC_VM_CG, data);
+
+		data = RREG32(mmVM_L2_CG);
+		data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK;
+		WREG32(mmVM_L2_CG, data);
+	}
+}
+
 static int gmc_v8_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		fiji_update_mc_medium_grain_clock_gating(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		fiji_update_mc_light_sleep(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		break;
+	default:
+		break;
+	}
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 779532d350ff..679e7394a495 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -253,8 +253,14 @@ static void iceland_ih_set_rptr(struct amdgpu_device *adev)
 static int iceland_ih_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
+
+	ret = amdgpu_irq_add_domain(adev);
+	if (ret)
+		return ret;
 
 	iceland_ih_set_interrupt_funcs(adev);
+
 	return 0;
 }
 
@@ -278,6 +284,7 @@ static int iceland_ih_sw_fini(void *handle)
 
 	amdgpu_irq_fini(adev);
 	amdgpu_ih_ring_fini(adev);
+	amdgpu_irq_remove_domain(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 7253132f04b8..ad54c46751b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -727,18 +727,20 @@ static int sdma_v3_0_start(struct amdgpu_device *adev)
 {
 	int r, i;
 
-	if (!adev->firmware.smu_load) {
-		r = sdma_v3_0_load_microcode(adev);
-		if (r)
-			return r;
-	} else {
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-									 (i == 0) ?
-									 AMDGPU_UCODE_ID_SDMA0 :
-									 AMDGPU_UCODE_ID_SDMA1);
+	if (!adev->pp_enabled) {
+		if (!adev->firmware.smu_load) {
+			r = sdma_v3_0_load_microcode(adev);
 			if (r)
-				return -EINVAL;
+				return r;
+		} else {
+			for (i = 0; i < adev->sdma.num_instances; i++) {
+				r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+										 (i == 0) ?
+										 AMDGPU_UCODE_ID_SDMA0 :
+										 AMDGPU_UCODE_ID_SDMA1);
+				if (r)
+					return -EINVAL;
+			}
 		}
 	}
 
@@ -1427,9 +1429,114 @@ static int sdma_v3_0_process_illegal_inst_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void fiji_update_sdma_medium_grain_clock_gating(
+		struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	if (enable) {
+		temp = data = RREG32(mmSDMA0_CLK_CTRL);
+		data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+		if (data != temp)
+			WREG32(mmSDMA0_CLK_CTRL, data);
+
+		temp = data = RREG32(mmSDMA1_CLK_CTRL);
+		data &= ~(SDMA1_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+
+		if (data != temp)
+			WREG32(mmSDMA1_CLK_CTRL, data);
+	} else {
+		temp = data = RREG32(mmSDMA0_CLK_CTRL);
+		data |= SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK;
+
+		if (data != temp)
+			WREG32(mmSDMA0_CLK_CTRL, data);
+
+		temp = data = RREG32(mmSDMA1_CLK_CTRL);
+		data |= SDMA1_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK;
+
+		if (data != temp)
+			WREG32(mmSDMA1_CLK_CTRL, data);
+	}
+}
+
+static void fiji_update_sdma_medium_grain_light_sleep(
+		struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	if (enable) {
+		temp = data = RREG32(mmSDMA0_POWER_CNTL);
+		data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+
+		if (temp != data)
+			WREG32(mmSDMA0_POWER_CNTL, data);
+
+		temp = data = RREG32(mmSDMA1_POWER_CNTL);
+		data |= SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+
+		if (temp != data)
+			WREG32(mmSDMA1_POWER_CNTL, data);
+	} else {
+		temp = data = RREG32(mmSDMA0_POWER_CNTL);
+		data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+
+		if (temp != data)
+			WREG32(mmSDMA0_POWER_CNTL, data);
+
+		temp = data = RREG32(mmSDMA1_POWER_CNTL);
+		data &= ~SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+
+		if (temp != data)
+			WREG32(mmSDMA1_POWER_CNTL, data);
+	}
+}
+
 static int sdma_v3_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		fiji_update_sdma_medium_grain_clock_gating(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		fiji_update_sdma_medium_grain_light_sleep(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		break;
+	default:
+		break;
+	}
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c b/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c
index 204903897b4f..0497784b3652 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c
@@ -24,7 +24,7 @@
 #include <linux/firmware.h>
 #include "drmP.h"
 #include "amdgpu.h"
-#include "tonga_smumgr.h"
+#include "tonga_smum.h"
 
 MODULE_FIRMWARE("amdgpu/tonga_smc.bin");
 
@@ -122,25 +122,12 @@ static int tonga_dpm_hw_fini(void *handle)
 
 static int tonga_dpm_suspend(void *handle)
 {
-	return 0;
+	return tonga_dpm_hw_fini(handle);
 }
 
 static int tonga_dpm_resume(void *handle)
 {
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-
-	ret = tonga_smu_start(adev);
-	if (ret) {
-		DRM_ERROR("SMU start failed\n");
-		goto fail;
-	}
-
-fail:
-	mutex_unlock(&adev->pm.mutex);
-	return ret;
+	return tonga_dpm_hw_init(handle);
 }
 
 static int tonga_dpm_set_clockgating_state(void *handle,
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 743c372837aa..b6f7d7bff929 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -273,8 +273,14 @@ static void tonga_ih_set_rptr(struct amdgpu_device *adev)
 static int tonga_ih_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
+
+	ret = amdgpu_irq_add_domain(adev);
+	if (ret)
+		return ret;
 
 	tonga_ih_set_interrupt_funcs(adev);
+
 	return 0;
 }
 
@@ -301,6 +307,7 @@ static int tonga_ih_sw_fini(void *handle)
 
 	amdgpu_irq_fini(adev);
 	amdgpu_ih_ring_fini(adev);
+	amdgpu_irq_add_domain(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ppsmc.h b/drivers/gpu/drm/amd/amdgpu/tonga_ppsmc.h
deleted file mode 100644
index 811781f69482..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ppsmc.h
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef TONGA_PP_SMC_H
-#define TONGA_PP_SMC_H
-
-#pragma pack(push, 1)
-
-#define PPSMC_SWSTATE_FLAG_DC                           0x01
-#define PPSMC_SWSTATE_FLAG_UVD                          0x02
-#define PPSMC_SWSTATE_FLAG_VCE                          0x04
-#define PPSMC_SWSTATE_FLAG_PCIE_X1                      0x08
-
-#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
-#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
-#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff 
-
-#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01 
-#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02 
-#define PPSMC_SYSTEMFLAG_GDDR5                          0x04 
-
-#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08 
-
-#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10 
-#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20 
-#define PPSMC_SYSTEMFLAG_12CHANNEL                      0x40
-
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
-#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
-
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
-#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
-
-#define PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH      0x10
-#define PPSMC_EXTRAFLAGS_DRIVER_TO_GPIO17               0x20
-#define PPSMC_EXTRAFLAGS_PCC_TO_GPIO17                  0x40
-
-#define PPSMC_DPM2FLAGS_TDPCLMP                         0x01 
-#define PPSMC_DPM2FLAGS_PWRSHFT                         0x02 
-#define PPSMC_DPM2FLAGS_OCP                             0x04 
-
-#define PPSMC_DISPLAY_WATERMARK_LOW                     0
-#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
-
-#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    		0x01
-#define PPSMC_STATEFLAG_POWERBOOST         		0x02
-#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 		0x04
-#define PPSMC_STATEFLAG_POWERSHIFT         		0x08
-#define PPSMC_STATEFLAG_SLOW_READ_MARGIN   		0x10
-#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 		0x20
-#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   		0x40
-
-#define FDO_MODE_HARDWARE 0
-#define FDO_MODE_PIECE_WISE_LINEAR 1
-
-enum FAN_CONTROL {
-	FAN_CONTROL_FUZZY,
-	FAN_CONTROL_TABLE
-};
-
-#define PPSMC_Result_OK             			((uint16_t)0x01)
-#define PPSMC_Result_NoMore         			((uint16_t)0x02)
-#define PPSMC_Result_NotNow         			((uint16_t)0x03)
-#define PPSMC_Result_Failed         			((uint16_t)0xFF) 
-#define PPSMC_Result_UnknownCmd     			((uint16_t)0xFE) 
-#define PPSMC_Result_UnknownVT      			((uint16_t)0xFD) 
-
-typedef uint16_t PPSMC_Result;
-
-#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
-
-#define PPSMC_MSG_Halt                      		((uint16_t)0x10)
-#define PPSMC_MSG_Resume                    		((uint16_t)0x11)
-#define PPSMC_MSG_EnableDPMLevel            		((uint16_t)0x12)
-#define PPSMC_MSG_ZeroLevelsDisabled        		((uint16_t)0x13)
-#define PPSMC_MSG_OneLevelsDisabled         		((uint16_t)0x14)
-#define PPSMC_MSG_TwoLevelsDisabled         		((uint16_t)0x15)
-#define PPSMC_MSG_EnableThermalInterrupt    		((uint16_t)0x16)
-#define PPSMC_MSG_RunningOnAC               		((uint16_t)0x17)
-#define PPSMC_MSG_LevelUp                   		((uint16_t)0x18)
-#define PPSMC_MSG_LevelDown                 		((uint16_t)0x19)
-#define PPSMC_MSG_ResetDPMCounters          		((uint16_t)0x1a)
-#define PPSMC_MSG_SwitchToSwState           		((uint16_t)0x20)
-#define PPSMC_MSG_SwitchToSwStateLast       		((uint16_t)0x3f)
-#define PPSMC_MSG_SwitchToInitialState      		((uint16_t)0x40)
-#define PPSMC_MSG_NoForcedLevel             		((uint16_t)0x41)
-#define PPSMC_MSG_ForceHigh                 		((uint16_t)0x42)
-#define PPSMC_MSG_ForceMediumOrHigh         		((uint16_t)0x43)
-#define PPSMC_MSG_SwitchToMinimumPower      		((uint16_t)0x51)
-#define PPSMC_MSG_ResumeFromMinimumPower    		((uint16_t)0x52)
-#define PPSMC_MSG_EnableCac                 		((uint16_t)0x53)
-#define PPSMC_MSG_DisableCac                		((uint16_t)0x54)
-#define PPSMC_DPMStateHistoryStart          		((uint16_t)0x55)
-#define PPSMC_DPMStateHistoryStop           		((uint16_t)0x56)
-#define PPSMC_CACHistoryStart               		((uint16_t)0x57)
-#define PPSMC_CACHistoryStop                		((uint16_t)0x58)
-#define PPSMC_TDPClampingActive             		((uint16_t)0x59)
-#define PPSMC_TDPClampingInactive           		((uint16_t)0x5A)
-#define PPSMC_StartFanControl               		((uint16_t)0x5B)
-#define PPSMC_StopFanControl                		((uint16_t)0x5C)
-#define PPSMC_NoDisplay                     		((uint16_t)0x5D)
-#define PPSMC_HasDisplay                    		((uint16_t)0x5E)
-#define PPSMC_MSG_UVDPowerOFF               		((uint16_t)0x60)
-#define PPSMC_MSG_UVDPowerON                		((uint16_t)0x61)
-#define PPSMC_MSG_EnableULV                 		((uint16_t)0x62)
-#define PPSMC_MSG_DisableULV                		((uint16_t)0x63)
-#define PPSMC_MSG_EnterULV                  		((uint16_t)0x64)
-#define PPSMC_MSG_ExitULV                   		((uint16_t)0x65)
-#define PPSMC_PowerShiftActive              		((uint16_t)0x6A)
-#define PPSMC_PowerShiftInactive            		((uint16_t)0x6B)
-#define PPSMC_OCPActive                     		((uint16_t)0x6C)
-#define PPSMC_OCPInactive                   		((uint16_t)0x6D)
-#define PPSMC_CACLongTermAvgEnable          		((uint16_t)0x6E)
-#define PPSMC_CACLongTermAvgDisable         		((uint16_t)0x6F)
-#define PPSMC_MSG_InferredStateSweep_Start  		((uint16_t)0x70)
-#define PPSMC_MSG_InferredStateSweep_Stop   		((uint16_t)0x71)
-#define PPSMC_MSG_SwitchToLowestInfState    		((uint16_t)0x72)
-#define PPSMC_MSG_SwitchToNonInfState       		((uint16_t)0x73)
-#define PPSMC_MSG_AllStateSweep_Start       		((uint16_t)0x74)
-#define PPSMC_MSG_AllStateSweep_Stop        		((uint16_t)0x75)
-#define PPSMC_MSG_SwitchNextLowerInfState   		((uint16_t)0x76)
-#define PPSMC_MSG_SwitchNextHigherInfState  		((uint16_t)0x77)
-#define PPSMC_MSG_MclkRetrainingTest        		((uint16_t)0x78)
-#define PPSMC_MSG_ForceTDPClamping          		((uint16_t)0x79)
-#define PPSMC_MSG_CollectCAC_PowerCorreln   		((uint16_t)0x7A)
-#define PPSMC_MSG_CollectCAC_WeightCalib    		((uint16_t)0x7B)
-#define PPSMC_MSG_CollectCAC_SQonly         		((uint16_t)0x7C)
-#define PPSMC_MSG_CollectCAC_TemperaturePwr 		((uint16_t)0x7D)
-#define PPSMC_MSG_ExtremitiesTest_Start     		((uint16_t)0x7E)
-#define PPSMC_MSG_ExtremitiesTest_Stop      		((uint16_t)0x7F)
-#define PPSMC_FlushDataCache                		((uint16_t)0x80)
-#define PPSMC_FlushInstrCache               		((uint16_t)0x81)
-#define PPSMC_MSG_SetEnabledLevels          		((uint16_t)0x82)
-#define PPSMC_MSG_SetForcedLevels           		((uint16_t)0x83)
-#define PPSMC_MSG_ResetToDefaults           		((uint16_t)0x84)
-#define PPSMC_MSG_SetForcedLevelsAndJump    		((uint16_t)0x85)
-#define PPSMC_MSG_SetCACHistoryMode         		((uint16_t)0x86)
-#define PPSMC_MSG_EnableDTE                 		((uint16_t)0x87)
-#define PPSMC_MSG_DisableDTE                		((uint16_t)0x88)
-#define PPSMC_MSG_SmcSpaceSetAddress        		((uint16_t)0x89)
-#define PPSMC_MSG_SmcSpaceWriteDWordInc     		((uint16_t)0x8A)
-#define PPSMC_MSG_SmcSpaceWriteWordInc      		((uint16_t)0x8B)
-#define PPSMC_MSG_SmcSpaceWriteByteInc      		((uint16_t)0x8C)
-#define PPSMC_MSG_ChangeNearTDPLimit        		((uint16_t)0x90)
-#define PPSMC_MSG_ChangeSafePowerLimit      		((uint16_t)0x91)
-#define PPSMC_MSG_DPMStateSweepStart        		((uint16_t)0x92)
-#define PPSMC_MSG_DPMStateSweepStop         		((uint16_t)0x93)
-#define PPSMC_MSG_OVRDDisableSCLKDS         		((uint16_t)0x94)
-#define PPSMC_MSG_CancelDisableOVRDSCLKDS   		((uint16_t)0x95)
-#define PPSMC_MSG_ThrottleOVRDSCLKDS        		((uint16_t)0x96)
-#define PPSMC_MSG_CancelThrottleOVRDSCLKDS  		((uint16_t)0x97)
-#define PPSMC_MSG_GPIO17                    		((uint16_t)0x98)
-#define PPSMC_MSG_API_SetSvi2Volt_Vddc      		((uint16_t)0x99)
-#define PPSMC_MSG_API_SetSvi2Volt_Vddci     		((uint16_t)0x9A)
-#define PPSMC_MSG_API_SetSvi2Volt_Mvdd      		((uint16_t)0x9B)
-#define PPSMC_MSG_API_GetSvi2Volt_Vddc      		((uint16_t)0x9C)
-#define PPSMC_MSG_API_GetSvi2Volt_Vddci     		((uint16_t)0x9D)
-#define PPSMC_MSG_API_GetSvi2Volt_Mvdd      		((uint16_t)0x9E)
-
-#define PPSMC_MSG_BREAK                     		((uint16_t)0xF8)
-
-#define PPSMC_MSG_Test                      		((uint16_t)0x100)
-#define PPSMC_MSG_DRV_DRAM_ADDR_HI            		((uint16_t)0x250)
-#define PPSMC_MSG_DRV_DRAM_ADDR_LO            		((uint16_t)0x251)
-#define PPSMC_MSG_SMU_DRAM_ADDR_HI            		((uint16_t)0x252)
-#define PPSMC_MSG_SMU_DRAM_ADDR_LO            		((uint16_t)0x253)
-#define PPSMC_MSG_LoadUcodes                  		((uint16_t)0x254)
-
-typedef uint16_t PPSMC_Msg;
-
-#define PPSMC_EVENT_STATUS_THERMAL          		0x00000001
-#define PPSMC_EVENT_STATUS_REGULATORHOT     		0x00000002
-#define PPSMC_EVENT_STATUS_DC               		0x00000004
-#define PPSMC_EVENT_STATUS_GPIO17           		0x00000008
-
-#pragma pack(pop)
-
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
index 5421309c1862..361c49a82323 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
@@ -25,7 +25,7 @@
 #include "drmP.h"
 #include "amdgpu.h"
 #include "tonga_ppsmc.h"
-#include "tonga_smumgr.h"
+#include "tonga_smum.h"
 #include "smu_ucode_xfer_vi.h"
 #include "amdgpu_ucode.h"
 
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smumgr.h b/drivers/gpu/drm/amd/amdgpu/tonga_smum.h
index c031ff99fe3e..c031ff99fe3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smumgr.h
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_smum.h
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 121915bbc3b6..3d5913926436 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -279,6 +279,234 @@ static void uvd_v6_0_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmUVD_VCPU_CACHE_SIZE2, size);
 }
 
+static void cz_set_uvd_clock_gating_branches(struct amdgpu_device *adev,
+		bool enable)
+{
+	u32 data, data1;
+
+	data = RREG32(mmUVD_CGC_GATE);
+	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
+	if (enable) {
+		data |= UVD_CGC_GATE__SYS_MASK |
+				UVD_CGC_GATE__UDEC_MASK |
+				UVD_CGC_GATE__MPEG2_MASK |
+				UVD_CGC_GATE__RBC_MASK |
+				UVD_CGC_GATE__LMI_MC_MASK |
+				UVD_CGC_GATE__IDCT_MASK |
+				UVD_CGC_GATE__MPRD_MASK |
+				UVD_CGC_GATE__MPC_MASK |
+				UVD_CGC_GATE__LBSI_MASK |
+				UVD_CGC_GATE__LRBBM_MASK |
+				UVD_CGC_GATE__UDEC_RE_MASK |
+				UVD_CGC_GATE__UDEC_CM_MASK |
+				UVD_CGC_GATE__UDEC_IT_MASK |
+				UVD_CGC_GATE__UDEC_DB_MASK |
+				UVD_CGC_GATE__UDEC_MP_MASK |
+				UVD_CGC_GATE__WCB_MASK |
+				UVD_CGC_GATE__VCPU_MASK |
+				UVD_CGC_GATE__SCPU_MASK;
+		data1 |= UVD_SUVD_CGC_GATE__SRE_MASK |
+				UVD_SUVD_CGC_GATE__SIT_MASK |
+				UVD_SUVD_CGC_GATE__SMP_MASK |
+				UVD_SUVD_CGC_GATE__SCM_MASK |
+				UVD_SUVD_CGC_GATE__SDB_MASK |
+				UVD_SUVD_CGC_GATE__SRE_H264_MASK |
+				UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SIT_H264_MASK |
+				UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SCM_H264_MASK |
+				UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SDB_H264_MASK |
+				UVD_SUVD_CGC_GATE__SDB_HEVC_MASK;
+	} else {
+		data &= ~(UVD_CGC_GATE__SYS_MASK |
+				UVD_CGC_GATE__UDEC_MASK |
+				UVD_CGC_GATE__MPEG2_MASK |
+				UVD_CGC_GATE__RBC_MASK |
+				UVD_CGC_GATE__LMI_MC_MASK |
+				UVD_CGC_GATE__LMI_UMC_MASK |
+				UVD_CGC_GATE__IDCT_MASK |
+				UVD_CGC_GATE__MPRD_MASK |
+				UVD_CGC_GATE__MPC_MASK |
+				UVD_CGC_GATE__LBSI_MASK |
+				UVD_CGC_GATE__LRBBM_MASK |
+				UVD_CGC_GATE__UDEC_RE_MASK |
+				UVD_CGC_GATE__UDEC_CM_MASK |
+				UVD_CGC_GATE__UDEC_IT_MASK |
+				UVD_CGC_GATE__UDEC_DB_MASK |
+				UVD_CGC_GATE__UDEC_MP_MASK |
+				UVD_CGC_GATE__WCB_MASK |
+				UVD_CGC_GATE__VCPU_MASK |
+				UVD_CGC_GATE__SCPU_MASK);
+		data1 &= ~(UVD_SUVD_CGC_GATE__SRE_MASK |
+				UVD_SUVD_CGC_GATE__SIT_MASK |
+				UVD_SUVD_CGC_GATE__SMP_MASK |
+				UVD_SUVD_CGC_GATE__SCM_MASK |
+				UVD_SUVD_CGC_GATE__SDB_MASK |
+				UVD_SUVD_CGC_GATE__SRE_H264_MASK |
+				UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SIT_H264_MASK |
+				UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SCM_H264_MASK |
+				UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
+				UVD_SUVD_CGC_GATE__SDB_H264_MASK |
+				UVD_SUVD_CGC_GATE__SDB_HEVC_MASK);
+	}
+	WREG32(mmUVD_CGC_GATE, data);
+	WREG32(mmUVD_SUVD_CGC_GATE, data1);
+}
+
+static void tonga_set_uvd_clock_gating_branches(struct amdgpu_device *adev,
+		bool enable)
+{
+	u32 data, data1;
+
+	data = RREG32(mmUVD_CGC_GATE);
+	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
+	if (enable) {
+		data |= UVD_CGC_GATE__SYS_MASK |
+				UVD_CGC_GATE__UDEC_MASK |
+				UVD_CGC_GATE__MPEG2_MASK |
+				UVD_CGC_GATE__RBC_MASK |
+				UVD_CGC_GATE__LMI_MC_MASK |
+				UVD_CGC_GATE__IDCT_MASK |
+				UVD_CGC_GATE__MPRD_MASK |
+				UVD_CGC_GATE__MPC_MASK |
+				UVD_CGC_GATE__LBSI_MASK |
+				UVD_CGC_GATE__LRBBM_MASK |
+				UVD_CGC_GATE__UDEC_RE_MASK |
+				UVD_CGC_GATE__UDEC_CM_MASK |
+				UVD_CGC_GATE__UDEC_IT_MASK |
+				UVD_CGC_GATE__UDEC_DB_MASK |
+				UVD_CGC_GATE__UDEC_MP_MASK |
+				UVD_CGC_GATE__WCB_MASK |
+				UVD_CGC_GATE__VCPU_MASK |
+				UVD_CGC_GATE__SCPU_MASK;
+		data1 |= UVD_SUVD_CGC_GATE__SRE_MASK |
+				UVD_SUVD_CGC_GATE__SIT_MASK |
+				UVD_SUVD_CGC_GATE__SMP_MASK |
+				UVD_SUVD_CGC_GATE__SCM_MASK |
+				UVD_SUVD_CGC_GATE__SDB_MASK;
+	} else {
+		data &= ~(UVD_CGC_GATE__SYS_MASK |
+				UVD_CGC_GATE__UDEC_MASK |
+				UVD_CGC_GATE__MPEG2_MASK |
+				UVD_CGC_GATE__RBC_MASK |
+				UVD_CGC_GATE__LMI_MC_MASK |
+				UVD_CGC_GATE__LMI_UMC_MASK |
+				UVD_CGC_GATE__IDCT_MASK |
+				UVD_CGC_GATE__MPRD_MASK |
+				UVD_CGC_GATE__MPC_MASK |
+				UVD_CGC_GATE__LBSI_MASK |
+				UVD_CGC_GATE__LRBBM_MASK |
+				UVD_CGC_GATE__UDEC_RE_MASK |
+				UVD_CGC_GATE__UDEC_CM_MASK |
+				UVD_CGC_GATE__UDEC_IT_MASK |
+				UVD_CGC_GATE__UDEC_DB_MASK |
+				UVD_CGC_GATE__UDEC_MP_MASK |
+				UVD_CGC_GATE__WCB_MASK |
+				UVD_CGC_GATE__VCPU_MASK |
+				UVD_CGC_GATE__SCPU_MASK);
+		data1 &= ~(UVD_SUVD_CGC_GATE__SRE_MASK |
+				UVD_SUVD_CGC_GATE__SIT_MASK |
+				UVD_SUVD_CGC_GATE__SMP_MASK |
+				UVD_SUVD_CGC_GATE__SCM_MASK |
+				UVD_SUVD_CGC_GATE__SDB_MASK);
+	}
+	WREG32(mmUVD_CGC_GATE, data);
+	WREG32(mmUVD_SUVD_CGC_GATE, data1);
+}
+
+static void uvd_v6_0_set_uvd_dynamic_clock_mode(struct amdgpu_device *adev,
+		bool swmode)
+{
+	u32 data, data1 = 0, data2;
+
+	/* Always un-gate UVD REGS bit */
+	data = RREG32(mmUVD_CGC_GATE);
+	data &= ~(UVD_CGC_GATE__REGS_MASK);
+	WREG32(mmUVD_CGC_GATE, data);
+
+	data = RREG32(mmUVD_CGC_CTRL);
+	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
+			UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
+	data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
+			1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER) |
+			4 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_OFF_DELAY);
+
+	data2 = RREG32(mmUVD_SUVD_CGC_CTRL);
+	if (swmode) {
+		data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
+				UVD_CGC_CTRL__SYS_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_MODE_MASK |
+				UVD_CGC_CTRL__MPEG2_MODE_MASK |
+				UVD_CGC_CTRL__REGS_MODE_MASK |
+				UVD_CGC_CTRL__RBC_MODE_MASK |
+				UVD_CGC_CTRL__LMI_MC_MODE_MASK |
+				UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
+				UVD_CGC_CTRL__IDCT_MODE_MASK |
+				UVD_CGC_CTRL__MPRD_MODE_MASK |
+				UVD_CGC_CTRL__MPC_MODE_MASK |
+				UVD_CGC_CTRL__LBSI_MODE_MASK |
+				UVD_CGC_CTRL__LRBBM_MODE_MASK |
+				UVD_CGC_CTRL__WCB_MODE_MASK |
+				UVD_CGC_CTRL__VCPU_MODE_MASK |
+				UVD_CGC_CTRL__JPEG_MODE_MASK |
+				UVD_CGC_CTRL__SCPU_MODE_MASK);
+		data1 |= UVD_CGC_CTRL2__DYN_OCLK_RAMP_EN_MASK |
+				UVD_CGC_CTRL2__DYN_RCLK_RAMP_EN_MASK;
+		data1 &= ~UVD_CGC_CTRL2__GATER_DIV_ID_MASK;
+		data1 |= 7 << REG_FIELD_SHIFT(UVD_CGC_CTRL2, GATER_DIV_ID);
+		data2 &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SIT_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
+	} else {
+		data |= UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
+				UVD_CGC_CTRL__SYS_MODE_MASK |
+				UVD_CGC_CTRL__UDEC_MODE_MASK |
+				UVD_CGC_CTRL__MPEG2_MODE_MASK |
+				UVD_CGC_CTRL__REGS_MODE_MASK |
+				UVD_CGC_CTRL__RBC_MODE_MASK |
+				UVD_CGC_CTRL__LMI_MC_MODE_MASK |
+				UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
+				UVD_CGC_CTRL__IDCT_MODE_MASK |
+				UVD_CGC_CTRL__MPRD_MODE_MASK |
+				UVD_CGC_CTRL__MPC_MODE_MASK |
+				UVD_CGC_CTRL__LBSI_MODE_MASK |
+				UVD_CGC_CTRL__LRBBM_MODE_MASK |
+				UVD_CGC_CTRL__WCB_MODE_MASK |
+				UVD_CGC_CTRL__VCPU_MODE_MASK |
+				UVD_CGC_CTRL__SCPU_MODE_MASK;
+		data2 |= UVD_SUVD_CGC_CTRL__SRE_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SIT_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
+				UVD_SUVD_CGC_CTRL__SDB_MODE_MASK;
+	}
+	WREG32(mmUVD_CGC_CTRL, data);
+	WREG32(mmUVD_SUVD_CGC_CTRL, data2);
+
+	data = RREG32_UVD_CTX(ixUVD_CGC_CTRL2);
+	data &= ~(REG_FIELD_MASK(UVD_CGC_CTRL2, DYN_OCLK_RAMP_EN) |
+			REG_FIELD_MASK(UVD_CGC_CTRL2, DYN_RCLK_RAMP_EN) |
+			REG_FIELD_MASK(UVD_CGC_CTRL2, GATER_DIV_ID));
+	data1 &= (REG_FIELD_MASK(UVD_CGC_CTRL2, DYN_OCLK_RAMP_EN) |
+			REG_FIELD_MASK(UVD_CGC_CTRL2, DYN_RCLK_RAMP_EN) |
+			REG_FIELD_MASK(UVD_CGC_CTRL2, GATER_DIV_ID));
+	data |= data1;
+	WREG32_UVD_CTX(ixUVD_CGC_CTRL2, data);
+}
+
 /**
  * uvd_v6_0_start - start UVD block
  *
@@ -303,8 +531,19 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 
 	uvd_v6_0_mc_resume(adev);
 
-	/* disable clock gating */
-	WREG32(mmUVD_CGC_GATE, 0);
+	/* Set dynamic clock gating in S/W control mode */
+	if (adev->cg_flags & AMDGPU_CG_SUPPORT_UVD_MGCG) {
+		if (adev->flags & AMD_IS_APU)
+			cz_set_uvd_clock_gating_branches(adev, false);
+		else
+			tonga_set_uvd_clock_gating_branches(adev, false);
+		uvd_v6_0_set_uvd_dynamic_clock_mode(adev, true);
+	} else {
+		/* disable clock gating */
+		uint32_t data = RREG32(mmUVD_CGC_CTRL);
+		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		WREG32(mmUVD_CGC_CTRL, data);
+	}
 
 	/* disable interupt */
 	WREG32_P(mmUVD_MASTINT_EN, 0, ~(1 << 1));
@@ -758,6 +997,24 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
 static int uvd_v6_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
+
+	if (!(adev->cg_flags & AMDGPU_CG_SUPPORT_UVD_MGCG))
+		return 0;
+
+	if (enable) {
+		if (adev->flags & AMD_IS_APU)
+			cz_set_uvd_clock_gating_branches(adev, enable);
+		else
+			tonga_set_uvd_clock_gating_branches(adev, enable);
+		uvd_v6_0_set_uvd_dynamic_clock_mode(adev, true);
+	} else {
+		uint32_t data = RREG32(mmUVD_CGC_CTRL);
+		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		WREG32(mmUVD_CGC_CTRL, data);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 370c6c9d81c2..e99af81e4aec 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -103,6 +103,108 @@ static void vce_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 		WREG32(mmVCE_RB_WPTR2, ring->wptr);
 }
 
+static void vce_v3_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)
+{
+	u32 tmp, data;
+
+	tmp = data = RREG32(mmVCE_RB_ARB_CTRL);
+	if (override)
+		data |= VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
+	else
+		data &= ~VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
+
+	if (tmp != data)
+		WREG32(mmVCE_RB_ARB_CTRL, data);
+}
+
+static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
+					     bool gated)
+{
+	u32 tmp, data;
+	/* Set Override to disable Clock Gating */
+	vce_v3_0_override_vce_clock_gating(adev, true);
+
+	if (!gated) {
+		/* Force CLOCK ON for VCE_CLOCK_GATING_B,
+		 * {*_FORCE_ON, *_FORCE_OFF} = {1, 0}
+		 * VREG can be FORCE ON or set to Dynamic, but can't be OFF
+		 */
+		tmp = data = RREG32(mmVCE_CLOCK_GATING_B);
+		data |= 0x1ff;
+		data &= ~0xef0000;
+		if (tmp != data)
+			WREG32(mmVCE_CLOCK_GATING_B, data);
+
+		/* Force CLOCK ON for VCE_UENC_CLOCK_GATING,
+		 * {*_FORCE_ON, *_FORCE_OFF} = {1, 0}
+		 */
+		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING);
+		data |= 0x3ff000;
+		data &= ~0xffc00000;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_CLOCK_GATING, data);
+
+		/* set VCE_UENC_CLOCK_GATING_2 */
+		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
+		data |= 0x2;
+		data &= ~0x2;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
+
+		/* Force CLOCK ON for VCE_UENC_REG_CLOCK_GATING */
+		tmp = data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
+		data |= 0x37f;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
+
+		/* Force VCE_UENC_DMA_DCLK_CTRL Clock ON */
+		tmp = data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
+		data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
+				VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
+				VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
+				0x8;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
+	} else {
+		/* Force CLOCK OFF for VCE_CLOCK_GATING_B,
+		 * {*, *_FORCE_OFF} = {*, 1}
+		 * set VREG to Dynamic, as it can't be OFF
+		 */
+		tmp = data = RREG32(mmVCE_CLOCK_GATING_B);
+		data &= ~0x80010;
+		data |= 0xe70008;
+		if (tmp != data)
+			WREG32(mmVCE_CLOCK_GATING_B, data);
+		/* Force CLOCK OFF for VCE_UENC_CLOCK_GATING,
+		 * Force ClOCK OFF takes precedent over Force CLOCK ON setting.
+		 * {*_FORCE_ON, *_FORCE_OFF} = {*, 1}
+		 */
+		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING);
+		data |= 0xffc00000;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_CLOCK_GATING, data);
+		/* Set VCE_UENC_CLOCK_GATING_2 */
+		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
+		data |= 0x10000;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
+		/* Set VCE_UENC_REG_CLOCK_GATING to dynamic */
+		tmp = data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
+		data &= ~0xffc00000;
+		if (tmp != data)
+			WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
+		/* Set VCE_UENC_DMA_DCLK_CTRL CG always in dynamic mode */
+		tmp = data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
+		data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
+				VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
+				VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
+				0x8);
+		if (tmp != data)
+			WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
+	}
+	vce_v3_0_override_vce_clock_gating(adev, false);
+}
+
 /**
  * vce_v3_0_start - start VCE block
  *
@@ -121,7 +223,7 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 		if (adev->vce.harvest_config & (1 << idx))
 			continue;
 
-		if(idx == 0)
+		if (idx == 0)
 			WREG32_P(mmGRBM_GFX_INDEX, 0,
 				~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
 		else
@@ -174,6 +276,10 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 		/* clear BUSY flag */
 		WREG32_P(mmVCE_STATUS, 0, ~1);
 
+		/* Set Clock-Gating off */
+		if (adev->cg_flags & AMDGPU_CG_SUPPORT_VCE_MGCG)
+			vce_v3_0_set_vce_sw_clock_gating(adev, false);
+
 		if (r) {
 			DRM_ERROR("VCE not responding, giving up!!!\n");
 			mutex_unlock(&adev->grbm_idx_mutex);
@@ -208,14 +314,11 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev)
 {
 	u32 tmp;
-	unsigned ret;
 
 	/* Fiji, Stoney are single pipe */
 	if ((adev->asic_type == CHIP_FIJI) ||
-	    (adev->asic_type == CHIP_STONEY)){
-		ret = AMDGPU_VCE_HARVEST_VCE1;
-		return ret;
-	}
+	    (adev->asic_type == CHIP_STONEY))
+		return AMDGPU_VCE_HARVEST_VCE1;
 
 	/* Tonga and CZ are dual or single pipe */
 	if (adev->flags & AMD_IS_APU)
@@ -229,19 +332,14 @@ static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev)
 
 	switch (tmp) {
 	case 1:
-		ret = AMDGPU_VCE_HARVEST_VCE0;
-		break;
+		return AMDGPU_VCE_HARVEST_VCE0;
 	case 2:
-		ret = AMDGPU_VCE_HARVEST_VCE1;
-		break;
+		return AMDGPU_VCE_HARVEST_VCE1;
 	case 3:
-		ret = AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1;
-		break;
+		return AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1;
 	default:
-		ret = 0;
+		return 0;
 	}
-
-	return ret;
 }
 
 static int vce_v3_0_early_init(void *handle)
@@ -316,28 +414,22 @@ static int vce_v3_0_sw_fini(void *handle)
 
 static int vce_v3_0_hw_init(void *handle)
 {
-	struct amdgpu_ring *ring;
-	int r;
+	int r, i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	r = vce_v3_0_start(adev);
 	if (r)
 		return r;
 
-	ring = &adev->vce.ring[0];
-	ring->ready = true;
-	r = amdgpu_ring_test_ring(ring);
-	if (r) {
-		ring->ready = false;
-		return r;
-	}
+	adev->vce.ring[0].ready = false;
+	adev->vce.ring[1].ready = false;
 
-	ring = &adev->vce.ring[1];
-	ring->ready = true;
-	r = amdgpu_ring_test_ring(ring);
-	if (r) {
-		ring->ready = false;
-		return r;
+	for (i = 0; i < 2; i++) {
+		r = amdgpu_ring_test_ring(&adev->vce.ring[i]);
+		if (r)
+			return r;
+		else
+			adev->vce.ring[i].ready = true;
 	}
 
 	DRM_INFO("VCE initialized successfully.\n");
@@ -437,17 +529,9 @@ static bool vce_v3_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 mask = 0;
-	int idx;
-
-	for (idx = 0; idx < 2; ++idx) {
-		if (adev->vce.harvest_config & (1 << idx))
-			continue;
 
-		if (idx == 0)
-			mask |= SRBM_STATUS2__VCE0_BUSY_MASK;
-		else
-			mask |= SRBM_STATUS2__VCE1_BUSY_MASK;
-	}
+	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_STATUS2__VCE0_BUSY_MASK;
+	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_STATUS2__VCE1_BUSY_MASK;
 
 	return !(RREG32(mmSRBM_STATUS2) & mask);
 }
@@ -456,23 +540,11 @@ static int vce_v3_0_wait_for_idle(void *handle)
 {
 	unsigned i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	u32 mask = 0;
-	int idx;
-
-	for (idx = 0; idx < 2; ++idx) {
-		if (adev->vce.harvest_config & (1 << idx))
-			continue;
-
-		if (idx == 0)
-			mask |= SRBM_STATUS2__VCE0_BUSY_MASK;
-		else
-			mask |= SRBM_STATUS2__VCE1_BUSY_MASK;
-	}
 
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (!(RREG32(mmSRBM_STATUS2) & mask))
+	for (i = 0; i < adev->usec_timeout; i++)
+		if (vce_v3_0_is_idle(handle))
 			return 0;
-	}
+
 	return -ETIMEDOUT;
 }
 
@@ -480,17 +552,10 @@ static int vce_v3_0_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 mask = 0;
-	int idx;
 
-	for (idx = 0; idx < 2; ++idx) {
-		if (adev->vce.harvest_config & (1 << idx))
-			continue;
+	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK;
+	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK;
 
-		if (idx == 0)
-			mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK;
-		else
-			mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK;
-	}
 	WREG32_P(mmSRBM_SOFT_RESET, mask,
 		 ~(SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK |
 		   SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK));
@@ -592,10 +657,8 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
 
 	switch (entry->src_data) {
 	case 0:
-		amdgpu_fence_process(&adev->vce.ring[0]);
-		break;
 	case 1:
-		amdgpu_fence_process(&adev->vce.ring[1]);
+		amdgpu_fence_process(&adev->vce.ring[entry->src_data]);
 		break;
 	default:
 		DRM_ERROR("Unhandled interrupt: %d %d\n",
@@ -609,6 +672,47 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
 static int vce_v3_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
+	int i;
+
+	if (!(adev->cg_flags & AMDGPU_CG_SUPPORT_VCE_MGCG))
+		return 0;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (i = 0; i < 2; i++) {
+		/* Program VCE Instance 0 or 1 if not harvested */
+		if (adev->vce.harvest_config & (1 << i))
+			continue;
+
+		if (i == 0)
+			WREG32_P(mmGRBM_GFX_INDEX, 0,
+					~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+		else
+			WREG32_P(mmGRBM_GFX_INDEX,
+					GRBM_GFX_INDEX__VCE_INSTANCE_MASK,
+					~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+
+		if (enable) {
+			/* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */
+			uint32_t data = RREG32(mmVCE_CLOCK_GATING_A);
+			data &= ~(0xf | 0xff0);
+			data |= ((0x0 << 0) | (0x04 << 4));
+			WREG32(mmVCE_CLOCK_GATING_A, data);
+
+			/* initialize VCE_UENC_CLOCK_GATING: Clock ON/OFF delay */
+			data = RREG32(mmVCE_UENC_CLOCK_GATING);
+			data &= ~(0xf | 0xff0);
+			data |= ((0x0 << 0) | (0x04 << 4));
+			WREG32(mmVCE_UENC_CLOCK_GATING, data);
+		}
+
+		vce_v3_0_set_vce_sw_clock_gating(adev, enable);
+	}
+
+	WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 2adc1c855e85..652e76644c31 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -31,6 +31,7 @@
 #include "amdgpu_vce.h"
 #include "amdgpu_ucode.h"
 #include "atom.h"
+#include "amd_pcie.h"
 
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
@@ -71,6 +72,7 @@
 #include "uvd_v5_0.h"
 #include "uvd_v6_0.h"
 #include "vce_v3_0.h"
+#include "amdgpu_powerplay.h"
 
 /*
  * Indirect registers accessor
@@ -376,6 +378,38 @@ static bool vi_read_disabled_bios(struct amdgpu_device *adev)
 	WREG32_SMC(ixROM_CNTL, rom_cntl);
 	return r;
 }
+
+static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
+				  u8 *bios, u32 length_bytes)
+{
+	u32 *dw_ptr;
+	unsigned long flags;
+	u32 i, length_dw;
+
+	if (bios == NULL)
+		return false;
+	if (length_bytes == 0)
+		return false;
+	/* APU vbios image is part of sbios image */
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	dw_ptr = (u32 *)bios;
+	length_dw = ALIGN(length_bytes, 4) / 4;
+	/* take the smc lock since we are using the smc index */
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	/* set rom index to 0 */
+	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
+	WREG32(mmSMC_IND_DATA_0, 0);
+	/* set index to data for continous read */
+	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
+	for (i = 0; i < length_dw; i++)
+		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return true;
+}
+
 static struct amdgpu_allowed_register_entry tonga_allowed_read_registers[] = {
 	{mmGB_MACROTILE_MODE7, true},
 };
@@ -1019,9 +1053,6 @@ static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 
 static void vi_pcie_gen3_enable(struct amdgpu_device *adev)
 {
-	u32 mask;
-	int ret;
-
 	if (pci_is_root_bus(adev->pdev->bus))
 		return;
 
@@ -1031,11 +1062,8 @@ static void vi_pcie_gen3_enable(struct amdgpu_device *adev)
 	if (adev->flags & AMD_IS_APU)
 		return;
 
-	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
-	if (ret != 0)
-		return;
-
-	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
 		return;
 
 	/* todo */
@@ -1098,7 +1126,7 @@ static const struct amdgpu_ip_block_version topaz_ip_blocks[] =
 		.major = 7,
 		.minor = 1,
 		.rev = 0,
-		.funcs = &iceland_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_GFX,
@@ -1145,7 +1173,7 @@ static const struct amdgpu_ip_block_version tonga_ip_blocks[] =
 		.major = 7,
 		.minor = 1,
 		.rev = 0,
-		.funcs = &tonga_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -1213,7 +1241,7 @@ static const struct amdgpu_ip_block_version fiji_ip_blocks[] =
 		.major = 7,
 		.minor = 1,
 		.rev = 0,
-		.funcs = &fiji_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs,
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -1281,7 +1309,7 @@ static const struct amdgpu_ip_block_version cz_ip_blocks[] =
 		.major = 8,
 		.minor = 0,
 		.rev = 0,
-		.funcs = &cz_dpm_ip_funcs,
+		.funcs = &amdgpu_pp_ip_funcs
 	},
 	{
 		.type = AMD_IP_BLOCK_TYPE_DCE,
@@ -1354,20 +1382,18 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 
 static uint32_t vi_get_rev_id(struct amdgpu_device *adev)
 {
-	if (adev->asic_type == CHIP_TOPAZ)
-		return (RREG32(mmPCIE_EFUSE4) & PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID_MASK)
-			>> PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID__SHIFT;
-	else if (adev->flags & AMD_IS_APU)
+	if (adev->flags & AMD_IS_APU)
 		return (RREG32_SMC(ATI_REV_ID_FUSE_MACRO__ADDRESS) & ATI_REV_ID_FUSE_MACRO__MASK)
 			>> ATI_REV_ID_FUSE_MACRO__SHIFT;
 	else
-		return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
-			>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
+		return (RREG32(mmPCIE_EFUSE4) & PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID_MASK)
+			>> PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID__SHIFT;
 }
 
 static const struct amdgpu_asic_funcs vi_asic_funcs =
 {
 	.read_disabled_bios = &vi_read_disabled_bios,
+	.read_bios_from_rom = &vi_read_bios_from_rom,
 	.read_register = &vi_read_register,
 	.reset = &vi_asic_reset,
 	.set_vga_state = &vi_vga_set_state,
@@ -1416,7 +1442,8 @@ static int vi_common_early_init(void *handle)
 		break;
 	case CHIP_FIJI:
 		adev->has_uvd = true;
-		adev->cg_flags = 0;
+		adev->cg_flags = AMDGPU_CG_SUPPORT_UVD_MGCG |
+				AMDGPU_CG_SUPPORT_VCE_MGCG;
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x3c;
 		break;
@@ -1442,6 +1469,8 @@ static int vi_common_early_init(void *handle)
 	if (amdgpu_smc_load_fw && smc_enabled)
 		adev->firmware.smu_load = true;
 
+	amdgpu_get_pcie_info(adev);
+
 	return 0;
 }
 
@@ -1515,9 +1544,95 @@ static int vi_common_soft_reset(void *handle)
 	return 0;
 }
 
+static void fiji_update_bif_medium_grain_light_sleep(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	temp = data = RREG32_PCIE(ixPCIE_CNTL2);
+
+	if (enable)
+		data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+				PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+				PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
+	else
+		data &= ~(PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+				PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+				PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
+
+	if (temp != data)
+		WREG32_PCIE(ixPCIE_CNTL2, data);
+}
+
+static void fiji_update_hdp_medium_grain_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	temp = data = RREG32(mmHDP_HOST_PATH_CNTL);
+
+	if (enable)
+		data &= ~HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK;
+	else
+		data |= HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK;
+
+	if (temp != data)
+		WREG32(mmHDP_HOST_PATH_CNTL, data);
+}
+
+static void fiji_update_hdp_light_sleep(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	temp = data = RREG32(mmHDP_MEM_POWER_LS);
+
+	if (enable)
+		data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK;
+	else
+		data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK;
+
+	if (temp != data)
+		WREG32(mmHDP_MEM_POWER_LS, data);
+}
+
+static void fiji_update_rom_medium_grain_clock_gating(struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t temp, data;
+
+	temp = data = RREG32_SMC(ixCGTT_ROM_CLK_CTRL0);
+
+	if (enable)
+		data &= ~(CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK |
+				CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK);
+	else
+		data |= CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK |
+				CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK;
+
+	if (temp != data)
+		WREG32_SMC(ixCGTT_ROM_CLK_CTRL0, data);
+}
+
 static int vi_common_set_clockgating_state(void *handle,
 					    enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		fiji_update_bif_medium_grain_light_sleep(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		fiji_update_hdp_medium_grain_clock_gating(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		fiji_update_hdp_light_sleep(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		fiji_update_rom_medium_grain_clock_gating(adev,
+				state == AMD_CG_STATE_GATE ? true : false);
+		break;
+	default:
+		break;
+	}
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index d321222fd92e..d2b49c026cf6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -558,8 +558,7 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
 		return -EINVAL;
 
 	/* this is the actual buffer to work with */
-
-	args_buff = memdup_user(args_buff,
+	args_buff = memdup_user(cmd_from_user,
 				args->buf_size_in_bytes - sizeof(*args));
 	if (IS_ERR(args_buff))
 		return PTR_ERR(args_buff);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h b/drivers/gpu/drm/amd/include/amd_acpi.h
index 01a29c3d7011..496360eb3fba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h
+++ b/drivers/gpu/drm/amd/include/amd_acpi.h
@@ -21,14 +21,63 @@
  *
  */
 
-#ifndef AMDGPU_ACPI_H
-#define AMDGPU_ACPI_H
+#ifndef AMD_ACPI_H
+#define AMD_ACPI_H
 
-struct amdgpu_device;
-struct acpi_bus_event;
+#define ACPI_AC_CLASS           "ac_adapter"
 
-int amdgpu_atif_handler(struct amdgpu_device *adev,
-		struct acpi_bus_event *event);
+struct atif_verify_interface {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 version;		/* version */
+	u32 notification_mask;	/* supported notifications mask */
+	u32 function_bits;	/* supported functions bit vector */
+} __packed;
+
+struct atif_system_params {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u32 valid_mask;		/* valid flags mask */
+	u32 flags;		/* flags */
+	u8 command_code;	/* notify command code */
+} __packed;
+
+struct atif_sbios_requests {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u32 pending;		/* pending sbios requests */
+	u8 panel_exp_mode;	/* panel expansion mode */
+	u8 thermal_gfx;		/* thermal state: target gfx controller */
+	u8 thermal_state;	/* thermal state: state id (0: exit state, non-0: state) */
+	u8 forced_power_gfx;	/* forced power state: target gfx controller */
+	u8 forced_power_state;	/* forced power state: state id */
+	u8 system_power_src;	/* system power source */
+	u8 backlight_level;	/* panel backlight level (0-255) */
+} __packed;
+
+#define ATIF_NOTIFY_MASK	0x3
+#define ATIF_NOTIFY_NONE	0
+#define ATIF_NOTIFY_81		1
+#define ATIF_NOTIFY_N		2
+
+struct atcs_verify_interface {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 version;		/* version */
+	u32 function_bits;	/* supported functions bit vector */
+} __packed;
+
+#define ATCS_VALID_FLAGS_MASK	0x3
+
+struct atcs_pref_req_input {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 client_id;		/* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */
+	u16 valid_flags_mask;	/* valid flags mask */
+	u16 flags;		/* flags */
+	u8 req_type;		/* request type */
+	u8 perf_req;		/* performance request */
+} __packed;
+
+struct atcs_pref_req_output {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u8 ret_val;		/* return value */
+} __packed;
 
 /* AMD hw uses four ACPI control methods:
  * 1. ATIF
diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h
new file mode 100644
index 000000000000..7c2a916c1e63
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/amd_pcie.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMD_PCIE_H__
+#define __AMD_PCIE_H__
+
+/* Following flags shows PCIe link speed supported in driver which are decided by chipset and ASIC */
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1        0x00010000
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2        0x00020000
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3        0x00040000
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_MASK        0xFFFF0000
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT       16
+
+/* Following flags shows PCIe link speed supported by ASIC H/W.*/
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1   0x00000001
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2   0x00000002
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3   0x00000004
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK   0x0000FFFF
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT  0
+
+/* Following flags shows PCIe lane width switch supported in driver which are decided by chipset and ASIC */
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X1          0x00010000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X2          0x00020000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X4          0x00040000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X8          0x00080000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X12         0x00100000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X16         0x00200000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_X32         0x00400000
+#define CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT       16
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/amd_pcie_helpers.h b/drivers/gpu/drm/amd/include/amd_pcie_helpers.h
new file mode 100644
index 000000000000..5725bf85eacc
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/amd_pcie_helpers.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMD_PCIE_HELPERS_H__
+#define __AMD_PCIE_HELPERS_H__
+
+#include "amd_pcie.h"
+
+static inline bool is_pcie_gen3_supported(uint32_t pcie_link_speed_cap)
+{
+	if (pcie_link_speed_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
+		return true;
+
+	return false;
+}
+
+static inline bool is_pcie_gen2_supported(uint32_t pcie_link_speed_cap)
+{
+	if (pcie_link_speed_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
+		return true;
+
+	return false;
+}
+
+/* Get the new PCIE speed given the ASIC PCIE Cap and the NewState's requested PCIE speed*/
+static inline uint16_t get_pcie_gen_support(uint32_t pcie_link_speed_cap,
+					    uint16_t ns_pcie_gen)
+{
+	uint32_t asic_pcie_link_speed_cap = (pcie_link_speed_cap &
+		CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK);
+	uint32_t sys_pcie_link_speed_cap  = (pcie_link_speed_cap &
+		CAIL_PCIE_LINK_SPEED_SUPPORT_MASK);
+
+	switch (asic_pcie_link_speed_cap) {
+	case CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1:
+		return PP_PCIEGen1;
+
+	case CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2:
+		return PP_PCIEGen2;
+
+	case CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3:
+		return PP_PCIEGen3;
+
+	default:
+		if (is_pcie_gen3_supported(sys_pcie_link_speed_cap) &&
+			(ns_pcie_gen == PP_PCIEGen3)) {
+			return PP_PCIEGen3;
+		} else if (is_pcie_gen2_supported(sys_pcie_link_speed_cap) &&
+			((ns_pcie_gen == PP_PCIEGen3) || (ns_pcie_gen == PP_PCIEGen2))) {
+			return PP_PCIEGen2;
+		}
+	}
+
+	return PP_PCIEGen1;
+}
+
+static inline uint16_t get_pcie_lane_support(uint32_t pcie_lane_width_cap,
+					     uint16_t ns_pcie_lanes)
+{
+	int i, j;
+	uint16_t new_pcie_lanes = ns_pcie_lanes;
+	uint16_t pcie_lanes[7] = {1, 2, 4, 8, 12, 16, 32};
+
+	switch (pcie_lane_width_cap) {
+	case 0:
+		printk(KERN_ERR "No valid PCIE lane width reported");
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
+		new_pcie_lanes = 1;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
+		new_pcie_lanes = 2;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
+		new_pcie_lanes = 4;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
+		new_pcie_lanes = 8;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
+		new_pcie_lanes = 12;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
+		new_pcie_lanes = 16;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
+		new_pcie_lanes = 32;
+		break;
+	default:
+		for (i = 0; i < 7; i++) {
+			if (ns_pcie_lanes == pcie_lanes[i]) {
+				if (pcie_lane_width_cap & (0x10000 << i)) {
+					break;
+				} else {
+					for (j = i - 1; j >= 0; j--) {
+						if (pcie_lane_width_cap & (0x10000 << j)) {
+							new_pcie_lanes = pcie_lanes[j];
+							break;
+						}
+					}
+
+					if (j < 0) {
+						for (j = i + 1; j < 7; j++) {
+							if (pcie_lane_width_cap & (0x10000 << j)) {
+								new_pcie_lanes = pcie_lanes[j];
+								break;
+							}
+						}
+						if (j > 7)
+							printk(KERN_ERR "Cannot find a valid PCIE lane width!");
+					}
+				}
+				break;
+			}
+		}
+		break;
+	}
+
+	return new_pcie_lanes;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index fe28fb353fab..1195d06f55bc 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -85,6 +85,27 @@ enum amd_powergating_state {
 	AMD_PG_STATE_UNGATE,
 };
 
+enum amd_pm_state_type {
+	/* not used for dpm */
+	POWER_STATE_TYPE_DEFAULT,
+	POWER_STATE_TYPE_POWERSAVE,
+	/* user selectable states */
+	POWER_STATE_TYPE_BATTERY,
+	POWER_STATE_TYPE_BALANCED,
+	POWER_STATE_TYPE_PERFORMANCE,
+	/* internal states */
+	POWER_STATE_TYPE_INTERNAL_UVD,
+	POWER_STATE_TYPE_INTERNAL_UVD_SD,
+	POWER_STATE_TYPE_INTERNAL_UVD_HD,
+	POWER_STATE_TYPE_INTERNAL_UVD_HD2,
+	POWER_STATE_TYPE_INTERNAL_UVD_MVC,
+	POWER_STATE_TYPE_INTERNAL_BOOT,
+	POWER_STATE_TYPE_INTERNAL_THERMAL,
+	POWER_STATE_TYPE_INTERNAL_ACPI,
+	POWER_STATE_TYPE_INTERNAL_ULV,
+	POWER_STATE_TYPE_INTERNAL_3DPERF,
+};
+
 struct amd_ip_funcs {
 	/* sets up early driver state (pre sw_init), does not configure hw - Optional */
 	int (*early_init)(void *handle);
diff --git a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
index 92b6ba0047af..293329719bba 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
@@ -596,6 +596,7 @@
 #define mmSWRST_EP_CONTROL_0                                                    0x14ac
 #define mmCPM_CONTROL                                                           0x14b8
 #define mmGSKT_CONTROL                                                          0x14bf
+#define ixSWRST_COMMAND_1                                                       0x1400103
 #define ixLM_CONTROL                                                            0x1400120
 #define ixLM_PCIETXMUX0                                                         0x1400121
 #define ixLM_PCIETXMUX1                                                         0x1400122
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
index daf763ba1a8f..a9b6923192ee 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
@@ -2807,5 +2807,18 @@
 #define ixDIDT_DBR_WEIGHT0_3                                                    0x90
 #define ixDIDT_DBR_WEIGHT4_7                                                    0x91
 #define ixDIDT_DBR_WEIGHT8_11                                                   0x92
+#define mmTD_EDC_CNT                                                            0x252e
+#define mmCPF_EDC_TAG_CNT                                                       0x3188
+#define mmCPF_EDC_ROQ_CNT                                                       0x3189
+#define mmCPF_EDC_ATC_CNT                                                       0x318a
+#define mmCPG_EDC_TAG_CNT                                                       0x318b
+#define mmCPG_EDC_ATC_CNT                                                       0x318c
+#define mmCPG_EDC_DMA_CNT                                                       0x318d
+#define mmCPC_EDC_SCRATCH_CNT                                                   0x318e
+#define mmCPC_EDC_UCODE_CNT                                                     0x318f
+#define mmCPC_EDC_ATC_CNT                                                       0x3190
+#define mmDC_EDC_STATE_CNT                                                      0x3191
+#define mmDC_EDC_CSINVOC_CNT                                                    0x3192
+#define mmDC_EDC_RESTORE_CNT                                                    0x3193
 
 #endif /* GFX_8_0_D_H */
diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h
index 552622675ace..eaf451e26643 100644
--- a/drivers/gpu/drm/amd/include/atombios.h
+++ b/drivers/gpu/drm/amd/include/atombios.h
@@ -550,6 +550,13 @@ typedef struct _COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1
 //MPLL_CNTL_FLAG_BYPASS_AD_PLL has a wrong name, should be BYPASS_DQ_PLL
 #define MPLL_CNTL_FLAG_BYPASS_AD_PLL            0x04
 
+// use for ComputeMemoryClockParamTable
+typedef struct _COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2
+{
+  COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 ulClock;
+  ULONG ulReserved;
+}COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2;
+
 typedef struct _DYNAMICE_MEMORY_SETTINGS_PARAMETER
 {
   ATOM_COMPUTE_CLOCK_FREQ ulClock;
@@ -4988,6 +4995,78 @@ typedef struct  _ATOM_ASIC_PROFILING_INFO_V3_3
   ULONG  ulSDCMargine;
 }ATOM_ASIC_PROFILING_INFO_V3_3;
 
+// for Fiji speed EVV algorithm
+typedef struct  _ATOM_ASIC_PROFILING_INFO_V3_4
+{
+  ATOM_COMMON_TABLE_HEADER         asHeader;
+  ULONG  ulEvvLkgFactor;
+  ULONG  ulBoardCoreTemp;
+  ULONG  ulMaxVddc;
+  ULONG  ulMinVddc;
+  ULONG  ulLoadLineSlop;
+  ULONG  ulLeakageTemp;
+  ULONG  ulLeakageVoltage;
+  EFUSE_LINEAR_FUNC_PARAM sCACm;
+  EFUSE_LINEAR_FUNC_PARAM sCACb;
+  EFUSE_LOGISTIC_FUNC_PARAM sKt_b;
+  EFUSE_LOGISTIC_FUNC_PARAM sKv_m;
+  EFUSE_LOGISTIC_FUNC_PARAM sKv_b;
+  USHORT usLkgEuseIndex;
+  UCHAR  ucLkgEfuseBitLSB;
+  UCHAR  ucLkgEfuseLength;
+  ULONG  ulLkgEncodeLn_MaxDivMin;
+  ULONG  ulLkgEncodeMax;
+  ULONG  ulLkgEncodeMin;
+  ULONG  ulEfuseLogisticAlpha;
+  USHORT usPowerDpm0;
+  USHORT usPowerDpm1;
+  USHORT usPowerDpm2;
+  USHORT usPowerDpm3;
+  USHORT usPowerDpm4;
+  USHORT usPowerDpm5;
+  USHORT usPowerDpm6;
+  USHORT usPowerDpm7;
+  ULONG  ulTdpDerateDPM0;
+  ULONG  ulTdpDerateDPM1;
+  ULONG  ulTdpDerateDPM2;
+  ULONG  ulTdpDerateDPM3;
+  ULONG  ulTdpDerateDPM4;
+  ULONG  ulTdpDerateDPM5;
+  ULONG  ulTdpDerateDPM6;
+  ULONG  ulTdpDerateDPM7;
+  EFUSE_LINEAR_FUNC_PARAM sRoFuse;
+  ULONG  ulEvvDefaultVddc;
+  ULONG  ulEvvNoCalcVddc;
+  USHORT usParamNegFlag;
+  USHORT usSpeed_Model;
+  ULONG  ulSM_A0;
+  ULONG  ulSM_A1;
+  ULONG  ulSM_A2;
+  ULONG  ulSM_A3;
+  ULONG  ulSM_A4;
+  ULONG  ulSM_A5;
+  ULONG  ulSM_A6;
+  ULONG  ulSM_A7;
+  UCHAR  ucSM_A0_sign;
+  UCHAR  ucSM_A1_sign;
+  UCHAR  ucSM_A2_sign;
+  UCHAR  ucSM_A3_sign;
+  UCHAR  ucSM_A4_sign;
+  UCHAR  ucSM_A5_sign;
+  UCHAR  ucSM_A6_sign;
+  UCHAR  ucSM_A7_sign;
+  ULONG ulMargin_RO_a;
+  ULONG ulMargin_RO_b;
+  ULONG ulMargin_RO_c;
+  ULONG ulMargin_fixed;
+  ULONG ulMargin_Fmax_mean;
+  ULONG ulMargin_plat_mean;
+  ULONG ulMargin_Fmax_sigma;
+  ULONG ulMargin_plat_sigma;
+  ULONG ulMargin_DC_sigma;
+  ULONG ulReserved[8];            // Reserved for future ASIC
+}ATOM_ASIC_PROFILING_INFO_V3_4;
+
 typedef struct _ATOM_POWER_SOURCE_OBJECT
 {
    UCHAR  ucPwrSrcId;                                   // Power source
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index 992dcd8a5c6a..713aec954692 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -105,6 +105,34 @@ enum cgs_ucode_id {
 	CGS_UCODE_ID_MAXIMUM,
 };
 
+enum cgs_system_info_id {
+	CGS_SYSTEM_INFO_ADAPTER_BDF_ID = 1,
+	CGS_SYSTEM_INFO_PCIE_GEN_INFO,
+	CGS_SYSTEM_INFO_PCIE_MLW,
+	CGS_SYSTEM_INFO_ID_MAXIMUM,
+};
+
+struct cgs_system_info {
+	uint64_t       size;
+	uint64_t       info_id;
+	union {
+		void           *ptr;
+		uint64_t        value;
+	};
+	uint64_t               padding[13];
+};
+
+/*
+ * enum cgs_resource_type - GPU resource type
+ */
+enum cgs_resource_type {
+	CGS_RESOURCE_TYPE_MMIO = 0,
+	CGS_RESOURCE_TYPE_FB,
+	CGS_RESOURCE_TYPE_IO,
+	CGS_RESOURCE_TYPE_DOORBELL,
+	CGS_RESOURCE_TYPE_ROM,
+};
+
 /**
  * struct cgs_clock_limits - Clock limits
  *
@@ -127,8 +155,53 @@ struct cgs_firmware_info {
 	void			*kptr;
 };
 
+struct cgs_mode_info {
+	uint32_t		refresh_rate;
+	uint32_t		ref_clock;
+	uint32_t		vblank_time_us;
+};
+
+struct cgs_display_info {
+	uint32_t		display_count;
+	uint32_t		active_display_mask;
+	struct cgs_mode_info *mode_info;
+};
+
 typedef unsigned long cgs_handle_t;
 
+#define CGS_ACPI_METHOD_ATCS          0x53435441
+#define CGS_ACPI_METHOD_ATIF          0x46495441
+#define CGS_ACPI_METHOD_ATPX          0x58505441
+#define CGS_ACPI_FIELD_METHOD_NAME                      0x00000001
+#define CGS_ACPI_FIELD_INPUT_ARGUMENT_COUNT             0x00000002
+#define CGS_ACPI_MAX_BUFFER_SIZE     256
+#define CGS_ACPI_TYPE_ANY                      0x00
+#define CGS_ACPI_TYPE_INTEGER               0x01
+#define CGS_ACPI_TYPE_STRING                0x02
+#define CGS_ACPI_TYPE_BUFFER                0x03
+#define CGS_ACPI_TYPE_PACKAGE               0x04
+
+struct cgs_acpi_method_argument {
+	uint32_t type;
+	uint32_t method_length;
+	uint32_t data_length;
+	union{
+		uint32_t value;
+		void *pointer;
+	};
+};
+
+struct cgs_acpi_method_info {
+	uint32_t size;
+	uint32_t field;
+	uint32_t input_count;
+	uint32_t name;
+	struct cgs_acpi_method_argument *pinput_argument;
+	uint32_t output_count;
+	struct cgs_acpi_method_argument *poutput_argument;
+	uint32_t padding[9];
+};
+
 /**
  * cgs_gpu_mem_info() - Return information about memory heaps
  * @cgs_device: opaque device handle
@@ -355,6 +428,23 @@ typedef void (*cgs_write_pci_config_word_t)(void *cgs_device, unsigned addr,
 typedef void (*cgs_write_pci_config_dword_t)(void *cgs_device, unsigned addr,
 					     uint32_t value);
 
+
+/**
+ * cgs_get_pci_resource() - provide access to a device resource (PCI BAR)
+ * @cgs_device:	opaque device handle
+ * @resource_type:	Type of Resource (MMIO, IO, ROM, FB, DOORBELL)
+ * @size:	size of the region
+ * @offset:	offset from the start of the region
+ * @resource_base:	base address (not including offset) returned
+ *
+ * Return: 0 on success, -errno otherwise
+ */
+typedef int (*cgs_get_pci_resource_t)(void *cgs_device,
+				      enum cgs_resource_type resource_type,
+				      uint64_t size,
+				      uint64_t offset,
+				      uint64_t *resource_base);
+
 /**
  * cgs_atom_get_data_table() - Get a pointer to an ATOM BIOS data table
  * @cgs_device:	opaque device handle
@@ -493,6 +583,21 @@ typedef int(*cgs_set_clockgating_state)(void *cgs_device,
 				  enum amd_ip_block_type block_type,
 				  enum amd_clockgating_state state);
 
+typedef int(*cgs_get_active_displays_info)(
+					void *cgs_device,
+					struct cgs_display_info *info);
+
+typedef int (*cgs_call_acpi_method)(void *cgs_device,
+					uint32_t acpi_method,
+					uint32_t acpi_function,
+					void *pinput, void *poutput,
+					uint32_t output_count,
+					uint32_t input_size,
+					uint32_t output_size);
+
+typedef int (*cgs_query_system_info)(void *cgs_device,
+				struct cgs_system_info *sys_info);
+
 struct cgs_ops {
 	/* memory management calls (similar to KFD interface) */
 	cgs_gpu_mem_info_t gpu_mem_info;
@@ -516,6 +621,8 @@ struct cgs_ops {
 	cgs_write_pci_config_byte_t write_pci_config_byte;
 	cgs_write_pci_config_word_t write_pci_config_word;
 	cgs_write_pci_config_dword_t write_pci_config_dword;
+	/* PCI resources */
+	cgs_get_pci_resource_t get_pci_resource;
 	/* ATOM BIOS */
 	cgs_atom_get_data_table_t atom_get_data_table;
 	cgs_atom_get_cmd_table_revs_t atom_get_cmd_table_revs;
@@ -533,7 +640,12 @@ struct cgs_ops {
 	/* cg pg interface*/
 	cgs_set_powergating_state set_powergating_state;
 	cgs_set_clockgating_state set_clockgating_state;
-	/* ACPI (TODO) */
+	/* display manager */
+	cgs_get_active_displays_info get_active_displays_info;
+	/* ACPI */
+	cgs_call_acpi_method call_acpi_method;
+	/* get system info */
+	cgs_query_system_info query_system_info;
 };
 
 struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -620,5 +732,15 @@ struct cgs_device
 	CGS_CALL(set_powergating_state, dev, block_type, state)
 #define cgs_set_clockgating_state(dev, block_type, state)	\
 	CGS_CALL(set_clockgating_state, dev, block_type, state)
+#define cgs_get_active_displays_info(dev, info)	\
+	CGS_CALL(get_active_displays_info, dev, info)
+#define cgs_call_acpi_method(dev, acpi_method, acpi_function, pintput, poutput, output_count, input_size, output_size)	\
+	CGS_CALL(call_acpi_method, dev, acpi_method, acpi_function, pintput, poutput, output_count, input_size, output_size)
+#define cgs_query_system_info(dev, sys_info)	\
+	CGS_CALL(query_system_info, dev, sys_info)
+#define cgs_get_pci_resource(cgs_device, resource_type, size, offset, \
+	resource_base) \
+	CGS_CALL(get_pci_resource, cgs_device, resource_type, size, offset, \
+	resource_base)
 
 #endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/powerplay/Kconfig b/drivers/gpu/drm/amd/powerplay/Kconfig
new file mode 100644
index 000000000000..af380335b425
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/Kconfig
@@ -0,0 +1,6 @@
+config DRM_AMD_POWERPLAY
+	bool  "Enable AMD powerplay component"
+	depends on DRM_AMDGPU
+	default n
+	help
+	  select this option will enable AMD powerplay component.
diff --git a/drivers/gpu/drm/amd/powerplay/Makefile b/drivers/gpu/drm/amd/powerplay/Makefile
new file mode 100644
index 000000000000..e195bf59da86
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/Makefile
@@ -0,0 +1,22 @@
+
+subdir-ccflags-y += -Iinclude/drm  \
+		-Idrivers/gpu/drm/amd/powerplay/inc/  \
+		-Idrivers/gpu/drm/amd/include/asic_reg  \
+		-Idrivers/gpu/drm/amd/include  \
+		-Idrivers/gpu/drm/amd/powerplay/smumgr\
+		-Idrivers/gpu/drm/amd/powerplay/hwmgr \
+		-Idrivers/gpu/drm/amd/powerplay/eventmgr
+
+AMD_PP_PATH = ../powerplay
+
+PP_LIBS = smumgr hwmgr eventmgr
+
+AMD_POWERPLAY = $(addsuffix /Makefile,$(addprefix drivers/gpu/drm/amd/powerplay/,$(PP_LIBS)))
+
+include $(AMD_POWERPLAY)
+
+POWER_MGR = amd_powerplay.o
+
+AMD_PP_POWER = $(addprefix $(AMD_PP_PATH)/,$(POWER_MGR))
+
+AMD_POWERPLAY_FILES += $(AMD_PP_POWER)
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
new file mode 100644
index 000000000000..aa67244a77ae
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -0,0 +1,665 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/gfp.h>
+#include <linux/slab.h>
+#include "amd_shared.h"
+#include "amd_powerplay.h"
+#include "pp_instance.h"
+#include "power_state.h"
+#include "eventmanager.h"
+
+#define PP_CHECK(handle)						\
+	do {								\
+		if ((handle) == NULL || (handle)->pp_valid != PP_VALID)	\
+			return -EINVAL;					\
+	} while (0)
+
+static int pp_early_init(void *handle)
+{
+	return 0;
+}
+
+static int pp_sw_init(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_hwmgr  *hwmgr;
+	int ret = 0;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	hwmgr = pp_handle->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->pptable_func == NULL ||
+	    hwmgr->hwmgr_func == NULL ||
+	    hwmgr->pptable_func->pptable_init == NULL ||
+	    hwmgr->hwmgr_func->backend_init == NULL)
+		return -EINVAL;
+
+	ret = hwmgr->pptable_func->pptable_init(hwmgr);
+
+	if (ret == 0)
+		ret = hwmgr->hwmgr_func->backend_init(hwmgr);
+
+	if (ret)
+		printk("amdgpu: powerplay initialization failed\n");
+	else
+		printk("amdgpu: powerplay initialized\n");
+
+	return ret;
+}
+
+static int pp_sw_fini(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_hwmgr  *hwmgr;
+	int ret = 0;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	hwmgr = pp_handle->hwmgr;
+
+	if (hwmgr != NULL || hwmgr->hwmgr_func != NULL ||
+	    hwmgr->hwmgr_func->backend_fini != NULL)
+		ret = hwmgr->hwmgr_func->backend_fini(hwmgr);
+
+	return ret;
+}
+
+static int pp_hw_init(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_smumgr *smumgr;
+	struct pp_eventmgr *eventmgr;
+	int ret = 0;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	smumgr = pp_handle->smu_mgr;
+
+	if (smumgr == NULL || smumgr->smumgr_funcs == NULL ||
+		smumgr->smumgr_funcs->smu_init == NULL ||
+		smumgr->smumgr_funcs->start_smu == NULL)
+		return -EINVAL;
+
+	ret = smumgr->smumgr_funcs->smu_init(smumgr);
+	if (ret) {
+		printk(KERN_ERR "[ powerplay ] smc initialization failed\n");
+		return ret;
+	}
+
+	ret = smumgr->smumgr_funcs->start_smu(smumgr);
+	if (ret) {
+		printk(KERN_ERR "[ powerplay ] smc start failed\n");
+		smumgr->smumgr_funcs->smu_fini(smumgr);
+		return ret;
+	}
+
+	hw_init_power_state_table(pp_handle->hwmgr);
+	eventmgr = pp_handle->eventmgr;
+
+	if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
+		return -EINVAL;
+
+	ret = eventmgr->pp_eventmgr_init(eventmgr);
+	return 0;
+}
+
+static int pp_hw_fini(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_smumgr *smumgr;
+	struct pp_eventmgr *eventmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	eventmgr = pp_handle->eventmgr;
+
+	if (eventmgr != NULL || eventmgr->pp_eventmgr_fini != NULL)
+		eventmgr->pp_eventmgr_fini(eventmgr);
+
+	smumgr = pp_handle->smu_mgr;
+
+	if (smumgr != NULL || smumgr->smumgr_funcs != NULL ||
+		smumgr->smumgr_funcs->smu_fini != NULL)
+		smumgr->smumgr_funcs->smu_fini(smumgr);
+
+	return 0;
+}
+
+static bool pp_is_idle(void *handle)
+{
+	return 0;
+}
+
+static int pp_wait_for_idle(void *handle)
+{
+	return 0;
+}
+
+static int pp_sw_reset(void *handle)
+{
+	return 0;
+}
+
+static void pp_print_status(void *handle)
+{
+
+}
+
+static int pp_set_clockgating_state(void *handle,
+				    enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int pp_set_powergating_state(void *handle,
+				    enum amd_powergating_state state)
+{
+	return 0;
+}
+
+static int pp_suspend(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_eventmgr *eventmgr;
+	struct pem_event_data event_data = { {0} };
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	eventmgr = pp_handle->eventmgr;
+	pem_handle_event(eventmgr, AMD_PP_EVENT_SUSPEND, &event_data);
+	return 0;
+}
+
+static int pp_resume(void *handle)
+{
+	struct pp_instance *pp_handle;
+	struct pp_eventmgr *eventmgr;
+	struct pem_event_data event_data = { {0} };
+	struct pp_smumgr *smumgr;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+	smumgr = pp_handle->smu_mgr;
+
+	if (smumgr == NULL || smumgr->smumgr_funcs == NULL ||
+		smumgr->smumgr_funcs->start_smu == NULL)
+		return -EINVAL;
+
+	ret = smumgr->smumgr_funcs->start_smu(smumgr);
+	if (ret) {
+		printk(KERN_ERR "[ powerplay ] smc start failed\n");
+		smumgr->smumgr_funcs->smu_fini(smumgr);
+		return ret;
+	}
+
+	eventmgr = pp_handle->eventmgr;
+	pem_handle_event(eventmgr, AMD_PP_EVENT_RESUME, &event_data);
+
+	return 0;
+}
+
+const struct amd_ip_funcs pp_ip_funcs = {
+	.early_init = pp_early_init,
+	.late_init = NULL,
+	.sw_init = pp_sw_init,
+	.sw_fini = pp_sw_fini,
+	.hw_init = pp_hw_init,
+	.hw_fini = pp_hw_fini,
+	.suspend = pp_suspend,
+	.resume = pp_resume,
+	.is_idle = pp_is_idle,
+	.wait_for_idle = pp_wait_for_idle,
+	.soft_reset = pp_sw_reset,
+	.print_status = pp_print_status,
+	.set_clockgating_state = pp_set_clockgating_state,
+	.set_powergating_state = pp_set_powergating_state,
+};
+
+static int pp_dpm_load_fw(void *handle)
+{
+	return 0;
+}
+
+static int pp_dpm_fw_loading_complete(void *handle)
+{
+	return 0;
+}
+
+static int pp_dpm_force_performance_level(void *handle,
+					enum amd_dpm_forced_level level)
+{
+	struct pp_instance *pp_handle;
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	pp_handle = (struct pp_instance *)handle;
+
+	hwmgr = pp_handle->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	    hwmgr->hwmgr_func->force_dpm_level == NULL)
+		return -EINVAL;
+
+	hwmgr->hwmgr_func->force_dpm_level(hwmgr, level);
+
+	return 0;
+}
+
+static enum amd_dpm_forced_level pp_dpm_get_performance_level(
+								void *handle)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	return (((struct pp_instance *)handle)->hwmgr->dpm_level);
+}
+
+static int pp_dpm_get_sclk(void *handle, bool low)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	    hwmgr->hwmgr_func->get_sclk == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_sclk(hwmgr, low);
+}
+
+static int pp_dpm_get_mclk(void *handle, bool low)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	    hwmgr->hwmgr_func->get_mclk == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_mclk(hwmgr, low);
+}
+
+static int pp_dpm_powergate_vce(void *handle, bool gate)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	    hwmgr->hwmgr_func->powergate_vce == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->powergate_vce(hwmgr, gate);
+}
+
+static int pp_dpm_powergate_uvd(void *handle, bool gate)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	    hwmgr->hwmgr_func->powergate_uvd == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->powergate_uvd(hwmgr, gate);
+}
+
+static enum PP_StateUILabel power_state_convert(enum amd_pm_state_type  state)
+{
+	switch (state) {
+	case POWER_STATE_TYPE_BATTERY:
+		return PP_StateUILabel_Battery;
+	case POWER_STATE_TYPE_BALANCED:
+		return PP_StateUILabel_Balanced;
+	case POWER_STATE_TYPE_PERFORMANCE:
+		return PP_StateUILabel_Performance;
+	default:
+		return PP_StateUILabel_None;
+	}
+}
+
+int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input, void *output)
+{
+	int ret = 0;
+	struct pp_instance *pp_handle;
+	struct pem_event_data data = { {0} };
+
+	pp_handle = (struct pp_instance *)handle;
+
+	if (pp_handle == NULL)
+		return -EINVAL;
+
+	switch (event_id) {
+	case AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE:
+		ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
+		break;
+	case AMD_PP_EVENT_ENABLE_USER_STATE:
+	{
+		enum amd_pm_state_type  ps;
+
+		if (input == NULL)
+			return -EINVAL;
+		ps = *(unsigned long *)input;
+
+		data.requested_ui_label = power_state_convert(ps);
+		ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
+	}
+	break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
+{
+	struct pp_hwmgr *hwmgr;
+	struct pp_power_state *state;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->current_ps == NULL)
+		return -EINVAL;
+
+	state = hwmgr->current_ps;
+
+	switch (state->classification.ui_label) {
+	case PP_StateUILabel_Battery:
+		return POWER_STATE_TYPE_BATTERY;
+	case PP_StateUILabel_Balanced:
+		return POWER_STATE_TYPE_BALANCED;
+	case PP_StateUILabel_Performance:
+		return POWER_STATE_TYPE_PERFORMANCE;
+	default:
+		return POWER_STATE_TYPE_DEFAULT;
+	}
+}
+
+static void
+pp_debugfs_print_current_performance_level(void *handle,
+					       struct seq_file *m)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->print_current_perforce_level == NULL)
+		return;
+
+	hwmgr->hwmgr_func->print_current_perforce_level(hwmgr, m);
+}
+
+static int pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->set_fan_control_mode == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->set_fan_control_mode(hwmgr, mode);
+}
+
+static int pp_dpm_get_fan_control_mode(void *handle)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->get_fan_control_mode == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_fan_control_mode(hwmgr);
+}
+
+static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->set_fan_speed_percent == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->set_fan_speed_percent(hwmgr, percent);
+}
+
+static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->get_fan_speed_percent == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_fan_speed_percent(hwmgr, speed);
+}
+
+static int pp_dpm_get_temperature(void *handle)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL ||
+	  hwmgr->hwmgr_func->get_temperature == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_temperature(hwmgr);
+}
+
+const struct amd_powerplay_funcs pp_dpm_funcs = {
+	.get_temperature = pp_dpm_get_temperature,
+	.load_firmware = pp_dpm_load_fw,
+	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
+	.force_performance_level = pp_dpm_force_performance_level,
+	.get_performance_level = pp_dpm_get_performance_level,
+	.get_current_power_state = pp_dpm_get_current_power_state,
+	.get_sclk = pp_dpm_get_sclk,
+	.get_mclk = pp_dpm_get_mclk,
+	.powergate_vce = pp_dpm_powergate_vce,
+	.powergate_uvd = pp_dpm_powergate_uvd,
+	.dispatch_tasks = pp_dpm_dispatch_tasks,
+	.print_current_performance_level = pp_debugfs_print_current_performance_level,
+	.set_fan_control_mode = pp_dpm_set_fan_control_mode,
+	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
+	.set_fan_speed_percent = pp_dpm_set_fan_speed_percent,
+	.get_fan_speed_percent = pp_dpm_get_fan_speed_percent,
+};
+
+static int amd_pp_instance_init(struct amd_pp_init *pp_init,
+				struct amd_powerplay *amd_pp)
+{
+	int ret;
+	struct pp_instance *handle;
+
+	handle = kzalloc(sizeof(struct pp_instance), GFP_KERNEL);
+	if (handle == NULL)
+		return -ENOMEM;
+
+	handle->pp_valid = PP_VALID;
+
+	ret = smum_init(pp_init, handle);
+	if (ret)
+		goto fail_smum;
+
+	ret = hwmgr_init(pp_init, handle);
+	if (ret)
+		goto fail_hwmgr;
+
+	ret = eventmgr_init(handle);
+	if (ret)
+		goto fail_eventmgr;
+
+	amd_pp->pp_handle = handle;
+	return 0;
+
+fail_eventmgr:
+	hwmgr_fini(handle->hwmgr);
+fail_hwmgr:
+	smum_fini(handle->smu_mgr);
+fail_smum:
+	kfree(handle);
+	return ret;
+}
+
+static int amd_pp_instance_fini(void *handle)
+{
+	struct pp_instance *instance = (struct pp_instance *)handle;
+
+	if (instance == NULL)
+		return -EINVAL;
+
+	eventmgr_fini(instance->eventmgr);
+
+	hwmgr_fini(instance->hwmgr);
+
+	smum_fini(instance->smu_mgr);
+
+	kfree(handle);
+	return 0;
+}
+
+int amd_powerplay_init(struct amd_pp_init *pp_init,
+		       struct amd_powerplay *amd_pp)
+{
+	int ret;
+
+	if (pp_init == NULL || amd_pp == NULL)
+		return -EINVAL;
+
+	ret = amd_pp_instance_init(pp_init, amd_pp);
+
+	if (ret)
+		return ret;
+
+	amd_pp->ip_funcs = &pp_ip_funcs;
+	amd_pp->pp_funcs = &pp_dpm_funcs;
+
+	return 0;
+}
+
+int amd_powerplay_fini(void *handle)
+{
+	amd_pp_instance_fini(handle);
+
+	return 0;
+}
+
+/* export this function to DAL */
+
+int amd_powerplay_display_configuration_change(void *handle, const void *input)
+{
+	struct pp_hwmgr  *hwmgr;
+	const struct amd_pp_display_configuration *display_config = input;
+
+	PP_CHECK((struct pp_instance *)handle);
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	phm_store_dal_configuration_data(hwmgr, display_config);
+
+	return 0;
+}
+
+int amd_powerplay_get_display_power_level(void *handle,
+		struct amd_pp_dal_clock_info *output)
+{
+	struct pp_hwmgr  *hwmgr;
+
+	PP_CHECK((struct pp_instance *)handle);
+
+	if (output == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	return phm_get_dal_power_level(hwmgr, output);
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/Makefile b/drivers/gpu/drm/amd/powerplay/eventmgr/Makefile
new file mode 100644
index 000000000000..7509e3850087
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/Makefile
@@ -0,0 +1,11 @@
+#
+# Makefile for the 'event manager' sub-component of powerplay.
+# It provides the event management services for the driver.
+
+EVENT_MGR = eventmgr.o eventinit.o eventmanagement.o  \
+		eventactionchains.o eventsubchains.o eventtasks.o psm.o
+
+AMD_PP_EVENT = $(addprefix $(AMD_PP_PATH)/eventmgr/,$(EVENT_MGR))
+
+AMD_POWERPLAY_FILES += $(AMD_PP_EVENT)
+
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
new file mode 100644
index 000000000000..83be3cf210e0
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
@@ -0,0 +1,289 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "eventmgr.h"
+#include "eventactionchains.h"
+#include "eventsubchains.h"
+
+static const pem_event_action *initialize_event[] = {
+	block_adjust_power_state_tasks,
+	power_budget_tasks,
+	system_config_tasks,
+	setup_asic_tasks,
+	enable_dynamic_state_management_tasks,
+	enable_clock_power_gatings_tasks,
+	get_2d_performance_state_tasks,
+	set_performance_state_tasks,
+	initialize_thermal_controller_tasks,
+	conditionally_force_3d_performance_state_tasks,
+	process_vbios_eventinfo_tasks,
+	broadcast_power_policy_tasks,
+	NULL
+};
+
+const struct action_chain initialize_action_chain = {
+	"Initialize",
+	initialize_event
+};
+
+static const pem_event_action *uninitialize_event[] = {
+	ungate_all_display_phys_tasks,
+	uninitialize_display_phy_access_tasks,
+	disable_gfx_voltage_island_power_gating_tasks,
+	disable_gfx_clock_gating_tasks,
+	set_boot_state_tasks,
+	adjust_power_state_tasks,
+	disable_dynamic_state_management_tasks,
+	disable_clock_power_gatings_tasks,
+	cleanup_asic_tasks,
+	prepare_for_pnp_stop_tasks,
+	NULL
+};
+
+const struct action_chain uninitialize_action_chain = {
+	"Uninitialize",
+	uninitialize_event
+};
+
+static const pem_event_action *power_source_change_event_pp_enabled[] = {
+	set_power_source_tasks,
+	set_power_saving_state_tasks,
+	adjust_power_state_tasks,
+	enable_disable_fps_tasks,
+	set_nbmcu_state_tasks,
+	broadcast_power_policy_tasks,
+	NULL
+};
+
+const struct action_chain power_source_change_action_chain_pp_enabled = {
+	"Power source change - PowerPlay enabled",
+	power_source_change_event_pp_enabled
+};
+
+static const pem_event_action *power_source_change_event_pp_disabled[] = {
+	set_power_source_tasks,
+	set_nbmcu_state_tasks,
+	NULL
+};
+
+const struct action_chain power_source_changes_action_chain_pp_disabled = {
+	"Power source change - PowerPlay disabled",
+	power_source_change_event_pp_disabled
+};
+
+static const pem_event_action *power_source_change_event_hardware_dc[] = {
+	set_power_source_tasks,
+	set_power_saving_state_tasks,
+	adjust_power_state_tasks,
+	enable_disable_fps_tasks,
+	reset_hardware_dc_notification_tasks,
+	set_nbmcu_state_tasks,
+	broadcast_power_policy_tasks,
+	NULL
+};
+
+const struct action_chain power_source_change_action_chain_hardware_dc = {
+	"Power source change - with Hardware DC switching",
+	power_source_change_event_hardware_dc
+};
+
+static const pem_event_action *suspend_event[] = {
+	reset_display_phy_access_tasks,
+	unregister_interrupt_tasks,
+	disable_gfx_voltage_island_power_gating_tasks,
+	disable_gfx_clock_gating_tasks,
+	notify_smu_suspend_tasks,
+	disable_smc_firmware_ctf_tasks,
+	set_boot_state_tasks,
+	adjust_power_state_tasks,
+	disable_fps_tasks,
+	vari_bright_suspend_tasks,
+	reset_fan_speed_to_default_tasks,
+	power_down_asic_tasks,
+	disable_stutter_mode_tasks,
+	set_connected_standby_tasks,
+	block_hw_access_tasks,
+	NULL
+};
+
+const struct action_chain suspend_action_chain = {
+	"Suspend",
+	suspend_event
+};
+
+static const pem_event_action *resume_event[] = {
+	unblock_hw_access_tasks,
+	resume_connected_standby_tasks,
+	notify_smu_resume_tasks,
+	reset_display_configCounter_tasks,
+	update_dal_configuration_tasks,
+	vari_bright_resume_tasks,
+	block_adjust_power_state_tasks,
+	setup_asic_tasks,
+	enable_stutter_mode_tasks, /*must do this in boot state and before SMC is started */
+	enable_dynamic_state_management_tasks,
+	enable_clock_power_gatings_tasks,
+	enable_disable_bapm_tasks,
+	initialize_thermal_controller_tasks,
+	reset_boot_state_tasks,
+	adjust_power_state_tasks,
+	enable_disable_fps_tasks,
+	notify_hw_power_source_tasks,
+	process_vbios_event_info_tasks,
+	enable_gfx_clock_gating_tasks,
+	enable_gfx_voltage_island_power_gating_tasks,
+	reset_clock_gating_tasks,
+	notify_smu_vpu_recovery_end_tasks,
+	disable_vpu_cap_tasks,
+	execute_escape_sequence_tasks,
+	NULL
+};
+
+
+const struct action_chain resume_action_chain = {
+	"resume",
+	resume_event
+};
+
+static const pem_event_action *complete_init_event[] = {
+	adjust_power_state_tasks,
+	enable_gfx_clock_gating_tasks,
+	enable_gfx_voltage_island_power_gating_tasks,
+	notify_power_state_change_tasks,
+	NULL
+};
+
+const struct action_chain complete_init_action_chain = {
+	"complete init",
+	complete_init_event
+};
+
+static const pem_event_action *enable_gfx_clock_gating_event[] = {
+	enable_gfx_clock_gating_tasks,
+	NULL
+};
+
+const struct action_chain enable_gfx_clock_gating_action_chain = {
+	"enable gfx clock gate",
+	enable_gfx_clock_gating_event
+};
+
+static const pem_event_action *disable_gfx_clock_gating_event[] = {
+	disable_gfx_clock_gating_tasks,
+	NULL
+};
+
+const struct action_chain disable_gfx_clock_gating_action_chain = {
+	"disable gfx clock gate",
+	disable_gfx_clock_gating_event
+};
+
+static const pem_event_action *enable_cgpg_event[] = {
+	enable_cgpg_tasks,
+	NULL
+};
+
+const struct action_chain enable_cgpg_action_chain = {
+	"eable cg pg",
+	enable_cgpg_event
+};
+
+static const pem_event_action *disable_cgpg_event[] = {
+	disable_cgpg_tasks,
+	NULL
+};
+
+const struct action_chain disable_cgpg_action_chain = {
+	"disable cg pg",
+	disable_cgpg_event
+};
+
+
+/* Enable user _2d performance and activate */
+
+static const pem_event_action *enable_user_state_event[] = {
+	create_new_user_performance_state_tasks,
+	adjust_power_state_tasks,
+	NULL
+};
+
+const struct action_chain enable_user_state_action_chain = {
+	"Enable user state",
+	enable_user_state_event
+};
+
+static const pem_event_action *enable_user_2d_performance_event[] = {
+	enable_user_2d_performance_tasks,
+	add_user_2d_performance_state_tasks,
+	set_performance_state_tasks,
+	adjust_power_state_tasks,
+	delete_user_2d_performance_state_tasks,
+	NULL
+};
+
+const struct action_chain enable_user_2d_performance_action_chain = {
+	"enable_user_2d_performance_event_activate",
+	enable_user_2d_performance_event
+};
+
+
+static const pem_event_action *disable_user_2d_performance_event[] = {
+	disable_user_2d_performance_tasks,
+	delete_user_2d_performance_state_tasks,
+	NULL
+};
+
+const struct action_chain disable_user_2d_performance_action_chain = {
+	"disable_user_2d_performance_event",
+	disable_user_2d_performance_event
+};
+
+
+static const pem_event_action *display_config_change_event[] = {
+	/* countDisplayConfigurationChangeEventTasks, */
+	unblock_adjust_power_state_tasks,
+	set_cpu_power_state,
+	notify_hw_power_source_tasks,
+	/* updateDALConfigurationTasks,
+	variBrightDisplayConfigurationChangeTasks, */
+	adjust_power_state_tasks,
+	/*enableDisableFPSTasks,
+	setNBMCUStateTasks,
+	notifyPCIEDeviceReadyTasks,*/
+	NULL
+};
+
+const struct action_chain display_config_change_action_chain = {
+	"Display configuration change",
+	display_config_change_event
+};
+
+static const pem_event_action *readjust_power_state_event[] = {
+	adjust_power_state_tasks,
+	NULL
+};
+
+const struct action_chain readjust_power_state_action_chain = {
+	"re-adjust power state",
+	readjust_power_state_event
+};
+
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.h b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.h
new file mode 100644
index 000000000000..f181e53cdcda
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _EVENT_ACTION_CHAINS_H_
+#define _EVENT_ACTION_CHAINS_H_
+#include "eventmgr.h"
+
+extern const struct action_chain initialize_action_chain;
+
+extern const struct action_chain uninitialize_action_chain;
+
+extern const struct action_chain power_source_change_action_chain_pp_enabled;
+
+extern const struct action_chain power_source_changes_action_chain_pp_disabled;
+
+extern const struct action_chain power_source_change_action_chain_hardware_dc;
+
+extern const struct action_chain suspend_action_chain;
+
+extern const struct action_chain resume_action_chain;
+
+extern const struct action_chain complete_init_action_chain;
+
+extern const struct action_chain enable_gfx_clock_gating_action_chain;
+
+extern const struct action_chain disable_gfx_clock_gating_action_chain;
+
+extern const struct action_chain enable_cgpg_action_chain;
+
+extern const struct action_chain disable_cgpg_action_chain;
+
+extern const struct action_chain enable_user_2d_performance_action_chain;
+
+extern const struct action_chain disable_user_2d_performance_action_chain;
+
+extern const struct action_chain enable_user_state_action_chain;
+
+extern const struct action_chain readjust_power_state_action_chain;
+
+extern const struct action_chain display_config_change_action_chain;
+
+#endif /*_EVENT_ACTION_CHAINS_H_*/
+
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c
new file mode 100644
index 000000000000..d5ec8ccbe97d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "eventmgr.h"
+#include "eventinit.h"
+#include "ppinterrupt.h"
+#include "hardwaremanager.h"
+
+void pem_init_feature_info(struct pp_eventmgr *eventmgr)
+{
+
+	/* PowerPlay info */
+	eventmgr->ui_state_info[PP_PowerSource_AC].default_ui_lable =
+					    PP_StateUILabel_Performance;
+
+	eventmgr->ui_state_info[PP_PowerSource_AC].current_ui_label =
+					    PP_StateUILabel_Performance;
+
+	eventmgr->ui_state_info[PP_PowerSource_DC].default_ui_lable =
+						  PP_StateUILabel_Battery;
+
+	eventmgr->ui_state_info[PP_PowerSource_DC].current_ui_label =
+						  PP_StateUILabel_Battery;
+
+	if (phm_cap_enabled(eventmgr->platform_descriptor->platformCaps, PHM_PlatformCaps_PowerPlaySupport)) {
+		eventmgr->features[PP_Feature_PowerPlay].supported = true;
+		eventmgr->features[PP_Feature_PowerPlay].version = PEM_CURRENT_POWERPLAY_FEATURE_VERSION;
+		eventmgr->features[PP_Feature_PowerPlay].enabled_default = true;
+		eventmgr->features[PP_Feature_PowerPlay].enabled = true;
+	} else {
+		eventmgr->features[PP_Feature_PowerPlay].supported = false;
+		eventmgr->features[PP_Feature_PowerPlay].enabled = false;
+		eventmgr->features[PP_Feature_PowerPlay].enabled_default = false;
+	}
+
+	eventmgr->features[PP_Feature_Force3DClock].supported = true;
+	eventmgr->features[PP_Feature_Force3DClock].enabled = false;
+	eventmgr->features[PP_Feature_Force3DClock].enabled_default = false;
+	eventmgr->features[PP_Feature_Force3DClock].version = 1;
+
+	/* over drive*/
+	eventmgr->features[PP_Feature_User2DPerformance].version = 4;
+	eventmgr->features[PP_Feature_User3DPerformance].version = 4;
+	eventmgr->features[PP_Feature_OverdriveTest].version = 4;
+
+	eventmgr->features[PP_Feature_OverDrive].version = 4;
+	eventmgr->features[PP_Feature_OverDrive].enabled = false;
+	eventmgr->features[PP_Feature_OverDrive].enabled_default = false;
+
+	eventmgr->features[PP_Feature_User2DPerformance].supported = false;
+	eventmgr->features[PP_Feature_User2DPerformance].enabled = false;
+	eventmgr->features[PP_Feature_User2DPerformance].enabled_default = false;
+
+	eventmgr->features[PP_Feature_User3DPerformance].supported = false;
+	eventmgr->features[PP_Feature_User3DPerformance].enabled = false;
+	eventmgr->features[PP_Feature_User3DPerformance].enabled_default = false;
+
+	eventmgr->features[PP_Feature_OverdriveTest].supported = false;
+	eventmgr->features[PP_Feature_OverdriveTest].enabled = false;
+	eventmgr->features[PP_Feature_OverdriveTest].enabled_default = false;
+
+	eventmgr->features[PP_Feature_OverDrive].supported = false;
+
+	eventmgr->features[PP_Feature_PowerBudgetWaiver].enabled_default = false;
+	eventmgr->features[PP_Feature_PowerBudgetWaiver].version = 1;
+	eventmgr->features[PP_Feature_PowerBudgetWaiver].supported = false;
+	eventmgr->features[PP_Feature_PowerBudgetWaiver].enabled = false;
+
+	/* Multi UVD States support */
+	eventmgr->features[PP_Feature_MultiUVDState].supported = false;
+	eventmgr->features[PP_Feature_MultiUVDState].enabled = false;
+	eventmgr->features[PP_Feature_MultiUVDState].enabled_default = false;
+
+	/* Dynamic UVD States support */
+	eventmgr->features[PP_Feature_DynamicUVDState].supported = false;
+	eventmgr->features[PP_Feature_DynamicUVDState].enabled = false;
+	eventmgr->features[PP_Feature_DynamicUVDState].enabled_default = false;
+
+	/* VCE DPM support */
+	eventmgr->features[PP_Feature_VCEDPM].supported = false;
+	eventmgr->features[PP_Feature_VCEDPM].enabled = false;
+	eventmgr->features[PP_Feature_VCEDPM].enabled_default = false;
+
+	/* ACP PowerGating support */
+	eventmgr->features[PP_Feature_ACP_POWERGATING].supported = false;
+	eventmgr->features[PP_Feature_ACP_POWERGATING].enabled = false;
+	eventmgr->features[PP_Feature_ACP_POWERGATING].enabled_default = false;
+
+	/* PPM support */
+	eventmgr->features[PP_Feature_PPM].version = 1;
+	eventmgr->features[PP_Feature_PPM].supported = false;
+	eventmgr->features[PP_Feature_PPM].enabled = false;
+
+	/* FFC support (enables fan and temp settings, Gemini needs temp settings) */
+	if (phm_cap_enabled(eventmgr->platform_descriptor->platformCaps, PHM_PlatformCaps_ODFuzzyFanControlSupport) ||
+	    phm_cap_enabled(eventmgr->platform_descriptor->platformCaps, PHM_PlatformCaps_GeminiRegulatorFanControlSupport)) {
+		eventmgr->features[PP_Feature_FFC].version = 1;
+		eventmgr->features[PP_Feature_FFC].supported = true;
+		eventmgr->features[PP_Feature_FFC].enabled = true;
+		eventmgr->features[PP_Feature_FFC].enabled_default = true;
+	} else {
+		eventmgr->features[PP_Feature_FFC].supported = false;
+		eventmgr->features[PP_Feature_FFC].enabled = false;
+		eventmgr->features[PP_Feature_FFC].enabled_default = false;
+	}
+
+	eventmgr->features[PP_Feature_VariBright].supported = false;
+	eventmgr->features[PP_Feature_VariBright].enabled = false;
+	eventmgr->features[PP_Feature_VariBright].enabled_default = false;
+
+	eventmgr->features[PP_Feature_BACO].supported = false;
+	eventmgr->features[PP_Feature_BACO].supported = false;
+	eventmgr->features[PP_Feature_BACO].enabled_default = false;
+
+	/* PowerDown feature support */
+	eventmgr->features[PP_Feature_PowerDown].supported = false;
+	eventmgr->features[PP_Feature_PowerDown].enabled = false;
+	eventmgr->features[PP_Feature_PowerDown].enabled_default = false;
+
+	eventmgr->features[PP_Feature_FPS].version = 1;
+	eventmgr->features[PP_Feature_FPS].supported = false;
+	eventmgr->features[PP_Feature_FPS].enabled_default = false;
+	eventmgr->features[PP_Feature_FPS].enabled = false;
+
+	eventmgr->features[PP_Feature_ViPG].version = 1;
+	eventmgr->features[PP_Feature_ViPG].supported = false;
+	eventmgr->features[PP_Feature_ViPG].enabled_default = false;
+	eventmgr->features[PP_Feature_ViPG].enabled = false;
+}
+
+static int thermal_interrupt_callback(void *private_data,
+				      unsigned src_id, const uint32_t *iv_entry)
+{
+	/* TO DO hanle PEM_Event_ThermalNotification (struct pp_eventmgr *)private_data*/
+	printk("current thermal is out of range \n");
+	return 0;
+}
+
+int pem_register_interrupts(struct pp_eventmgr *eventmgr)
+{
+	int result = 0;
+	struct pp_interrupt_registration_info info;
+
+	info.call_back = thermal_interrupt_callback;
+	info.context = eventmgr;
+
+	result = phm_register_thermal_interrupt(eventmgr->hwmgr, &info);
+
+	/* TODO:
+	 * 2. Register CTF event interrupt
+	 * 3. Register for vbios events interrupt
+	 * 4. Register External Throttle Interrupt
+	 * 5. Register Smc To Host Interrupt
+	 * */
+	return result;
+}
+
+
+int pem_unregister_interrupts(struct pp_eventmgr *eventmgr)
+{
+	return 0;
+}
+
+
+void pem_uninit_featureInfo(struct pp_eventmgr *eventmgr)
+{
+	eventmgr->features[PP_Feature_MultiUVDState].supported = false;
+	eventmgr->features[PP_Feature_VariBright].supported = false;
+	eventmgr->features[PP_Feature_PowerBudgetWaiver].supported = false;
+	eventmgr->features[PP_Feature_OverDrive].supported = false;
+	eventmgr->features[PP_Feature_OverdriveTest].supported = false;
+	eventmgr->features[PP_Feature_User3DPerformance].supported = false;
+	eventmgr->features[PP_Feature_User2DPerformance].supported = false;
+	eventmgr->features[PP_Feature_PowerPlay].supported = false;
+	eventmgr->features[PP_Feature_Force3DClock].supported = false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.h b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.h
new file mode 100644
index 000000000000..9ef96aab3f24
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _EVENTINIT_H_
+#define _EVENTINIT_H_
+
+#define PEM_CURRENT_POWERPLAY_FEATURE_VERSION 4
+
+void pem_init_feature_info(struct pp_eventmgr *eventmgr);
+void pem_uninit_featureInfo(struct pp_eventmgr *eventmgr);
+int pem_register_interrupts(struct pp_eventmgr *eventmgr);
+int pem_unregister_interrupts(struct pp_eventmgr *eventmgr);
+
+#endif /* _EVENTINIT_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.c
new file mode 100644
index 000000000000..1e2ad5603080
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.c
@@ -0,0 +1,215 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "eventmanagement.h"
+#include "eventmgr.h"
+#include "eventactionchains.h"
+
+int pem_init_event_action_chains(struct pp_eventmgr *eventmgr)
+{
+	int i;
+
+	for (i = 0; i < AMD_PP_EVENT_MAX; i++)
+		eventmgr->event_chain[i] = NULL;
+
+	eventmgr->event_chain[AMD_PP_EVENT_SUSPEND] = pem_get_suspend_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_INITIALIZE] = pem_get_initialize_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_UNINITIALIZE] = pem_get_uninitialize_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_POWER_SOURCE_CHANGE] = pem_get_power_source_change_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_HIBERNATE] = pem_get_hibernate_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_RESUME] = pem_get_resume_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_THERMAL_NOTIFICATION] = pem_get_thermal_notification_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_VBIOS_NOTIFICATION] = pem_get_vbios_notification_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENTER_THERMAL_STATE] = pem_get_enter_thermal_state_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_EXIT_THERMAL_STATE] = pem_get_exit_thermal_state_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENABLE_POWER_PLAY] = pem_get_enable_powerplay_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_DISABLE_POWER_PLAY] = pem_get_disable_powerplay_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENABLE_OVER_DRIVE_TEST] = pem_get_enable_overdrive_test_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_DISABLE_OVER_DRIVE_TEST] = pem_get_disable_overdrive_test_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENABLE_GFX_CLOCK_GATING] = pem_get_enable_gfx_clock_gating_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_DISABLE_GFX_CLOCK_GATING] = pem_get_disable_gfx_clock_gating_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENABLE_CGPG] = pem_get_enable_cgpg_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_DISABLE_CGPG] = pem_get_disable_cgpg_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_COMPLETE_INIT] = pem_get_complete_init_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_SCREEN_ON] = pem_get_screen_on_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_SCREEN_OFF] = pem_get_screen_off_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_PRE_SUSPEND] = pem_get_pre_suspend_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_PRE_RESUME] = pem_get_pre_resume_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_ENABLE_USER_STATE] = pem_enable_user_state_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_READJUST_POWER_STATE] = pem_readjust_power_state_action_chain(eventmgr);
+	eventmgr->event_chain[AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE] = pem_display_config_change_action_chain(eventmgr);
+	return 0;
+}
+
+int pem_excute_event_chain(struct pp_eventmgr *eventmgr, const struct action_chain *event_chain, struct pem_event_data *event_data)
+{
+	const pem_event_action **paction_chain;
+	const pem_event_action *psub_chain;
+	int tmp_result = 0;
+	int result = 0;
+
+	if (eventmgr == NULL || event_chain == NULL || event_data == NULL)
+		return -EINVAL;
+
+	for (paction_chain = event_chain->action_chain; NULL != *paction_chain; paction_chain++) {
+		if (0 != result)
+			return result;
+
+		for (psub_chain = *paction_chain; NULL != *psub_chain; psub_chain++) {
+			tmp_result = (*psub_chain)(eventmgr, event_data);
+			if (0 == result)
+				result = tmp_result;
+		}
+	}
+
+	return result;
+}
+
+const struct action_chain *pem_get_suspend_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &suspend_action_chain;
+}
+
+const struct action_chain *pem_get_initialize_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &initialize_action_chain;
+}
+
+const struct action_chain *pem_get_uninitialize_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &uninitialize_action_chain;
+}
+
+const struct action_chain *pem_get_power_source_change_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &power_source_change_action_chain_pp_enabled;  /* other case base on feature info*/
+}
+
+const struct action_chain *pem_get_resume_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &resume_action_chain;
+}
+
+const struct action_chain *pem_get_hibernate_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_thermal_notification_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_vbios_notification_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_enter_thermal_state_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_exit_thermal_state_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_enable_powerplay_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_disable_powerplay_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_enable_overdrive_test_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_disable_overdrive_test_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_enable_gfx_clock_gating_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &enable_gfx_clock_gating_action_chain;
+}
+
+const struct action_chain *pem_get_disable_gfx_clock_gating_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &disable_gfx_clock_gating_action_chain;
+}
+
+const struct action_chain *pem_get_enable_cgpg_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &enable_cgpg_action_chain;
+}
+
+const struct action_chain *pem_get_disable_cgpg_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &disable_cgpg_action_chain;
+}
+
+const struct action_chain *pem_get_complete_init_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &complete_init_action_chain;
+}
+
+const struct action_chain *pem_get_screen_on_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_screen_off_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_pre_suspend_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_get_pre_resume_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return NULL;
+}
+
+const struct action_chain *pem_enable_user_state_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &enable_user_state_action_chain;
+}
+
+const struct action_chain *pem_readjust_power_state_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &readjust_power_state_action_chain;
+}
+
+const struct action_chain *pem_display_config_change_action_chain(struct pp_eventmgr *eventmgr)
+{
+	return &display_config_change_action_chain;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.h b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.h
new file mode 100644
index 000000000000..383d4b295aa9
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmanagement.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _EVENT_MANAGEMENT_H_
+#define _EVENT_MANAGEMENT_H_
+
+#include "eventmgr.h"
+
+int pem_init_event_action_chains(struct pp_eventmgr *eventmgr);
+int pem_excute_event_chain(struct pp_eventmgr *eventmgr, const struct action_chain *event_chain, struct pem_event_data *event_data);
+const struct action_chain *pem_get_suspend_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_initialize_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_uninitialize_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_power_source_change_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_resume_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_hibernate_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_thermal_notification_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_vbios_notification_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_enter_thermal_state_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_exit_thermal_state_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_enable_powerplay_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_disable_powerplay_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_enable_overdrive_test_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_disable_overdrive_test_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_enable_gfx_clock_gating_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_disable_gfx_clock_gating_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_enable_cgpg_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_disable_cgpg_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_complete_init_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_screen_on_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_screen_off_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_pre_suspend_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_get_pre_resume_action_chain(struct pp_eventmgr *eventmgr);
+
+extern const struct action_chain *pem_enable_user_state_action_chain(struct pp_eventmgr *eventmgr);
+extern const struct action_chain *pem_readjust_power_state_action_chain(struct pp_eventmgr *eventmgr);
+const struct action_chain *pem_display_config_change_action_chain(struct pp_eventmgr *eventmgr);
+
+
+#endif /* _EVENT_MANAGEMENT_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c
new file mode 100644
index 000000000000..52a3efc97f05
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "eventmgr.h"
+#include "hwmgr.h"
+#include "eventinit.h"
+#include "eventmanagement.h"
+
+static int pem_init(struct pp_eventmgr *eventmgr)
+{
+	int result = 0;
+	struct pem_event_data event_data;
+
+	/* Initialize PowerPlay feature info */
+	pem_init_feature_info(eventmgr);
+
+	/* Initialize event action chains */
+	pem_init_event_action_chains(eventmgr);
+
+	/* Call initialization event */
+	result = pem_handle_event(eventmgr, AMD_PP_EVENT_INITIALIZE, &event_data);
+
+	if (0 != result)
+		return result;
+
+	/* Register interrupt callback functions */
+	result = pem_register_interrupts(eventmgr);
+	return 0;
+}
+
+static void pem_fini(struct pp_eventmgr *eventmgr)
+{
+	struct pem_event_data event_data;
+
+	pem_uninit_featureInfo(eventmgr);
+	pem_unregister_interrupts(eventmgr);
+
+	pem_handle_event(eventmgr, AMD_PP_EVENT_UNINITIALIZE, &event_data);
+
+	if (eventmgr != NULL)
+		kfree(eventmgr);
+}
+
+int eventmgr_init(struct pp_instance *handle)
+{
+	int result = 0;
+	struct pp_eventmgr *eventmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	eventmgr = kzalloc(sizeof(struct pp_eventmgr), GFP_KERNEL);
+	if (eventmgr == NULL)
+		return -ENOMEM;
+
+	eventmgr->hwmgr = handle->hwmgr;
+	handle->eventmgr = eventmgr;
+
+	eventmgr->platform_descriptor = &(eventmgr->hwmgr->platform_descriptor);
+	eventmgr->pp_eventmgr_init = pem_init;
+	eventmgr->pp_eventmgr_fini = pem_fini;
+
+	return result;
+}
+
+int eventmgr_fini(struct pp_eventmgr *eventmgr)
+{
+	kfree(eventmgr);
+	return 0;
+}
+
+static int pem_handle_event_unlocked(struct pp_eventmgr *eventmgr, enum amd_pp_event event, struct pem_event_data *data)
+{
+	if (eventmgr == NULL || event >= AMD_PP_EVENT_MAX || data == NULL)
+		return -EINVAL;
+
+	return pem_excute_event_chain(eventmgr, eventmgr->event_chain[event], data);
+}
+
+int pem_handle_event(struct pp_eventmgr *eventmgr, enum amd_pp_event event, struct pem_event_data *event_data)
+{
+	int r = 0;
+
+	r = pem_handle_event_unlocked(eventmgr, event, event_data);
+
+	return r;
+}
+
+bool pem_is_hw_access_blocked(struct pp_eventmgr *eventmgr)
+{
+	return (eventmgr->block_adjust_power_state || phm_is_hw_access_blocked(eventmgr->hwmgr));
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.c
new file mode 100644
index 000000000000..9ef2d90e2886
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.c
@@ -0,0 +1,410 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "eventmgr.h"
+#include "eventsubchains.h"
+#include "eventtasks.h"
+#include "hardwaremanager.h"
+
+const pem_event_action reset_display_phy_access_tasks[] = {
+	pem_task_reset_display_phys_access,
+	NULL
+};
+
+const pem_event_action broadcast_power_policy_tasks[] = {
+	/* PEM_Task_BroadcastPowerPolicyChange, */
+	NULL
+};
+
+const pem_event_action unregister_interrupt_tasks[] = {
+	pem_task_unregister_interrupts,
+	NULL
+};
+
+/* Disable GFX Voltage Islands Power Gating */
+const pem_event_action disable_gfx_voltage_island_powergating_tasks[] = {
+	pem_task_disable_voltage_island_power_gating,
+	NULL
+};
+
+const pem_event_action disable_gfx_clockgating_tasks[] = {
+	pem_task_disable_gfx_clock_gating,
+	NULL
+};
+
+const pem_event_action block_adjust_power_state_tasks[] = {
+	pem_task_block_adjust_power_state,
+	NULL
+};
+
+
+const pem_event_action unblock_adjust_power_state_tasks[] = {
+	pem_task_unblock_adjust_power_state,
+	NULL
+};
+
+const pem_event_action set_performance_state_tasks[] = {
+	pem_task_set_performance_state,
+	NULL
+};
+
+const pem_event_action get_2d_performance_state_tasks[] = {
+	pem_task_get_2D_performance_state_id,
+	NULL
+};
+
+const pem_event_action conditionally_force3D_performance_state_tasks[] = {
+	pem_task_conditionally_force_3d_performance_state,
+	NULL
+};
+
+const pem_event_action process_vbios_eventinfo_tasks[] = {
+	/* PEM_Task_ProcessVbiosEventInfo,*/
+	NULL
+};
+
+const pem_event_action enable_dynamic_state_management_tasks[] = {
+	/* PEM_Task_ResetBAPMPolicyChangedFlag,*/
+	pem_task_get_boot_state_id,
+	pem_task_enable_dynamic_state_management,
+	pem_task_register_interrupts,
+	NULL
+};
+
+const pem_event_action enable_clock_power_gatings_tasks[] = {
+	pem_task_enable_clock_power_gatings_tasks,
+	pem_task_powerdown_uvd_tasks,
+	pem_task_powerdown_vce_tasks,
+	NULL
+};
+
+const pem_event_action setup_asic_tasks[] = {
+	pem_task_setup_asic,
+	NULL
+};
+
+const pem_event_action power_budget_tasks[] = {
+	/* TODO
+	 * PEM_Task_PowerBudgetWaiverAvailable,
+	 * PEM_Task_PowerBudgetWarningMessage,
+	 * PEM_Task_PruneStatesBasedOnPowerBudget,
+	*/
+	NULL
+};
+
+const pem_event_action system_config_tasks[] = {
+	/* PEM_Task_PruneStatesBasedOnSystemConfig,*/
+	NULL
+};
+
+
+const pem_event_action conditionally_force_3d_performance_state_tasks[] = {
+	pem_task_conditionally_force_3d_performance_state,
+	NULL
+};
+
+const pem_event_action ungate_all_display_phys_tasks[] = {
+	/* PEM_Task_GetDisplayPhyAccessInfo */
+	NULL
+};
+
+const pem_event_action uninitialize_display_phy_access_tasks[] = {
+	/* PEM_Task_UninitializeDisplayPhysAccess, */
+	NULL
+};
+
+const pem_event_action disable_gfx_voltage_island_power_gating_tasks[] = {
+	/* PEM_Task_DisableVoltageIslandPowerGating, */
+	NULL
+};
+
+const pem_event_action disable_gfx_clock_gating_tasks[] = {
+	pem_task_disable_gfx_clock_gating,
+	NULL
+};
+
+const pem_event_action set_boot_state_tasks[] = {
+	pem_task_get_boot_state_id,
+	pem_task_set_boot_state,
+	NULL
+};
+
+const pem_event_action adjust_power_state_tasks[] = {
+	pem_task_notify_hw_mgr_display_configuration_change,
+	pem_task_adjust_power_state,
+	pem_task_notify_smc_display_config_after_power_state_adjustment,
+	pem_task_update_allowed_performance_levels,
+	/* to do pem_task_Enable_disable_bapm, */
+	NULL
+};
+
+const pem_event_action disable_dynamic_state_management_tasks[] = {
+	pem_task_unregister_interrupts,
+	pem_task_get_boot_state_id,
+	pem_task_disable_dynamic_state_management,
+	NULL
+};
+
+const pem_event_action disable_clock_power_gatings_tasks[] = {
+	pem_task_disable_clock_power_gatings_tasks,
+	NULL
+};
+
+const pem_event_action cleanup_asic_tasks[] = {
+	/* PEM_Task_DisableFPS,*/
+	pem_task_cleanup_asic,
+	NULL
+};
+
+const pem_event_action prepare_for_pnp_stop_tasks[] = {
+	/* PEM_Task_PrepareForPnpStop,*/
+	NULL
+};
+
+const pem_event_action set_power_source_tasks[] = {
+	pem_task_set_power_source,
+	pem_task_notify_hw_of_power_source,
+	NULL
+};
+
+const pem_event_action set_power_saving_state_tasks[] = {
+	pem_task_reset_power_saving_state,
+	pem_task_get_power_saving_state,
+	pem_task_set_power_saving_state,
+	/* PEM_Task_ResetODDCState,
+	 * PEM_Task_GetODDCState,
+	 * PEM_Task_SetODDCState,*/
+	NULL
+};
+
+const pem_event_action enable_disable_fps_tasks[] = {
+	/* PEM_Task_EnableDisableFPS,*/
+	NULL
+};
+
+const pem_event_action set_nbmcu_state_tasks[] = {
+	/* PEM_Task_NBMCUStateChange,*/
+	NULL
+};
+
+const pem_event_action reset_hardware_dc_notification_tasks[] = {
+	/* PEM_Task_ResetHardwareDCNotification,*/
+	NULL
+};
+
+
+const pem_event_action notify_smu_suspend_tasks[] = {
+	/* PEM_Task_NotifySMUSuspend,*/
+	NULL
+};
+
+const pem_event_action disable_smc_firmware_ctf_tasks[] = {
+	/* PEM_Task_DisableSMCFirmwareCTF,*/
+	NULL
+};
+
+const pem_event_action disable_fps_tasks[] = {
+	/* PEM_Task_DisableFPS,*/
+	NULL
+};
+
+const pem_event_action vari_bright_suspend_tasks[] = {
+	/* PEM_Task_VariBright_Suspend,*/
+	NULL
+};
+
+const pem_event_action reset_fan_speed_to_default_tasks[] = {
+	/* PEM_Task_ResetFanSpeedToDefault,*/
+	NULL
+};
+
+const pem_event_action power_down_asic_tasks[] = {
+	/* PEM_Task_DisableFPS,*/
+	pem_task_power_down_asic,
+	NULL
+};
+
+const pem_event_action disable_stutter_mode_tasks[] = {
+	/* PEM_Task_DisableStutterMode,*/
+	NULL
+};
+
+const pem_event_action set_connected_standby_tasks[] = {
+	/* PEM_Task_SetConnectedStandby,*/
+	NULL
+};
+
+const pem_event_action block_hw_access_tasks[] = {
+	pem_task_block_hw_access,
+	NULL
+};
+
+const pem_event_action unblock_hw_access_tasks[] = {
+	pem_task_un_block_hw_access,
+	NULL
+};
+
+const pem_event_action resume_connected_standby_tasks[] = {
+	/* PEM_Task_ResumeConnectedStandby,*/
+	NULL
+};
+
+const pem_event_action notify_smu_resume_tasks[] = {
+	/* PEM_Task_NotifySMUResume,*/
+	NULL
+};
+
+const pem_event_action reset_display_configCounter_tasks[] = {
+	pem_task_reset_display_phys_access,
+	NULL
+};
+
+const pem_event_action update_dal_configuration_tasks[] = {
+	/* PEM_Task_CheckVBlankTime,*/
+	NULL
+};
+
+const pem_event_action vari_bright_resume_tasks[] = {
+	/* PEM_Task_VariBright_Resume,*/
+	NULL
+};
+
+const pem_event_action notify_hw_power_source_tasks[] = {
+	pem_task_notify_hw_of_power_source,
+	NULL
+};
+
+const pem_event_action process_vbios_event_info_tasks[] = {
+	/* PEM_Task_ProcessVbiosEventInfo,*/
+	NULL
+};
+
+const pem_event_action enable_gfx_clock_gating_tasks[] = {
+	pem_task_enable_gfx_clock_gating,
+	NULL
+};
+
+const pem_event_action enable_gfx_voltage_island_power_gating_tasks[] = {
+	pem_task_enable_voltage_island_power_gating,
+	NULL
+};
+
+const pem_event_action reset_clock_gating_tasks[] = {
+	/* PEM_Task_ResetClockGating*/
+	NULL
+};
+
+const pem_event_action notify_smu_vpu_recovery_end_tasks[] = {
+	/* PEM_Task_NotifySmuVPURecoveryEnd,*/
+	NULL
+};
+
+const pem_event_action disable_vpu_cap_tasks[] = {
+	/* PEM_Task_DisableVPUCap,*/
+	NULL
+};
+
+const pem_event_action execute_escape_sequence_tasks[] = {
+	/* PEM_Task_ExecuteEscapesequence,*/
+	NULL
+};
+
+const pem_event_action notify_power_state_change_tasks[] = {
+	pem_task_notify_power_state_change,
+	NULL
+};
+
+const pem_event_action enable_cgpg_tasks[] = {
+	pem_task_enable_cgpg,
+	NULL
+};
+
+const pem_event_action disable_cgpg_tasks[] = {
+	pem_task_disable_cgpg,
+	NULL
+};
+
+const pem_event_action enable_user_2d_performance_tasks[] = {
+	/* PEM_Task_SetUser2DPerformanceFlag,*/
+	/* PEM_Task_UpdateUser2DPerformanceEnableEvents,*/
+	NULL
+};
+
+const pem_event_action add_user_2d_performance_state_tasks[] = {
+	/* PEM_Task_Get2DPerformanceTemplate,*/
+	/* PEM_Task_AllocateNewPowerStateMemory,*/
+	/* PEM_Task_CopyNewPowerStateInfo,*/
+	/* PEM_Task_UpdateNewPowerStateClocks,*/
+	/* PEM_Task_UpdateNewPowerStateUser2DPerformanceFlag,*/
+	/* PEM_Task_AddPowerState,*/
+	/* PEM_Task_ReleaseNewPowerStateMemory,*/
+	NULL
+};
+
+const pem_event_action delete_user_2d_performance_state_tasks[] = {
+	/* PEM_Task_GetCurrentUser2DPerformanceStateID,*/
+	/* PEM_Task_DeletePowerState,*/
+	/* PEM_Task_SetCurrentUser2DPerformanceStateID,*/
+	NULL
+};
+
+const pem_event_action disable_user_2d_performance_tasks[] = {
+	/* PEM_Task_ResetUser2DPerformanceFlag,*/
+	/* PEM_Task_UpdateUser2DPerformanceDisableEvents,*/
+	NULL
+};
+
+const pem_event_action enable_stutter_mode_tasks[] = {
+	pem_task_enable_stutter_mode,
+	NULL
+};
+
+const pem_event_action enable_disable_bapm_tasks[] = {
+	/*PEM_Task_EnableDisableBAPM,*/
+	NULL
+};
+
+const pem_event_action reset_boot_state_tasks[] = {
+	pem_task_reset_boot_state,
+	NULL
+};
+
+const pem_event_action create_new_user_performance_state_tasks[] = {
+	pem_task_create_user_performance_state,
+	NULL
+};
+
+const pem_event_action initialize_thermal_controller_tasks[] = {
+	pem_task_initialize_thermal_controller,
+	NULL
+};
+
+const pem_event_action uninitialize_thermal_controller_tasks[] = {
+	pem_task_uninitialize_thermal_controller,
+	NULL
+};
+
+const pem_event_action set_cpu_power_state[] = {
+	pem_task_set_cpu_power_state,
+	NULL
+};
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.h b/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.h
new file mode 100644
index 000000000000..7714cb927428
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventsubchains.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _EVENT_SUB_CHAINS_H_
+#define _EVENT_SUB_CHAINS_H_
+
+#include "eventmgr.h"
+
+extern const pem_event_action reset_display_phy_access_tasks[];
+extern const pem_event_action broadcast_power_policy_tasks[];
+extern const pem_event_action unregister_interrupt_tasks[];
+extern const pem_event_action disable_GFX_voltage_island_powergating_tasks[];
+extern const pem_event_action disable_GFX_clockgating_tasks[];
+extern const pem_event_action block_adjust_power_state_tasks[];
+extern const pem_event_action unblock_adjust_power_state_tasks[];
+extern const pem_event_action set_performance_state_tasks[];
+extern const pem_event_action get_2D_performance_state_tasks[];
+extern const pem_event_action conditionally_force3D_performance_state_tasks[];
+extern const pem_event_action process_vbios_eventinfo_tasks[];
+extern const pem_event_action enable_dynamic_state_management_tasks[];
+extern const pem_event_action enable_clock_power_gatings_tasks[];
+extern const pem_event_action conditionally_force3D_performance_state_tasks[];
+extern const pem_event_action setup_asic_tasks[];
+extern const pem_event_action power_budget_tasks[];
+extern const pem_event_action system_config_tasks[];
+extern const pem_event_action get_2d_performance_state_tasks[];
+extern const pem_event_action conditionally_force_3d_performance_state_tasks[];
+extern const pem_event_action ungate_all_display_phys_tasks[];
+extern const pem_event_action uninitialize_display_phy_access_tasks[];
+extern const pem_event_action disable_gfx_voltage_island_power_gating_tasks[];
+extern const pem_event_action disable_gfx_clock_gating_tasks[];
+extern const pem_event_action set_boot_state_tasks[];
+extern const pem_event_action adjust_power_state_tasks[];
+extern const pem_event_action disable_dynamic_state_management_tasks[];
+extern const pem_event_action disable_clock_power_gatings_tasks[];
+extern const pem_event_action cleanup_asic_tasks[];
+extern const pem_event_action prepare_for_pnp_stop_tasks[];
+extern const pem_event_action set_power_source_tasks[];
+extern const pem_event_action set_power_saving_state_tasks[];
+extern const pem_event_action enable_disable_fps_tasks[];
+extern const pem_event_action set_nbmcu_state_tasks[];
+extern const pem_event_action reset_hardware_dc_notification_tasks[];
+extern const pem_event_action notify_smu_suspend_tasks[];
+extern const pem_event_action disable_smc_firmware_ctf_tasks[];
+extern const pem_event_action disable_fps_tasks[];
+extern const pem_event_action vari_bright_suspend_tasks[];
+extern const pem_event_action reset_fan_speed_to_default_tasks[];
+extern const pem_event_action power_down_asic_tasks[];
+extern const pem_event_action disable_stutter_mode_tasks[];
+extern const pem_event_action set_connected_standby_tasks[];
+extern const pem_event_action block_hw_access_tasks[];
+extern const pem_event_action unblock_hw_access_tasks[];
+extern const pem_event_action resume_connected_standby_tasks[];
+extern const pem_event_action notify_smu_resume_tasks[];
+extern const pem_event_action reset_display_configCounter_tasks[];
+extern const pem_event_action update_dal_configuration_tasks[];
+extern const pem_event_action vari_bright_resume_tasks[];
+extern const pem_event_action notify_hw_power_source_tasks[];
+extern const pem_event_action process_vbios_event_info_tasks[];
+extern const pem_event_action enable_gfx_clock_gating_tasks[];
+extern const pem_event_action enable_gfx_voltage_island_power_gating_tasks[];
+extern const pem_event_action reset_clock_gating_tasks[];
+extern const pem_event_action notify_smu_vpu_recovery_end_tasks[];
+extern const pem_event_action disable_vpu_cap_tasks[];
+extern const pem_event_action execute_escape_sequence_tasks[];
+extern const pem_event_action notify_power_state_change_tasks[];
+extern const pem_event_action enable_cgpg_tasks[];
+extern const pem_event_action disable_cgpg_tasks[];
+extern const pem_event_action enable_user_2d_performance_tasks[];
+extern const pem_event_action add_user_2d_performance_state_tasks[];
+extern const pem_event_action delete_user_2d_performance_state_tasks[];
+extern const pem_event_action disable_user_2d_performance_tasks[];
+extern const pem_event_action enable_stutter_mode_tasks[];
+extern const pem_event_action enable_disable_bapm_tasks[];
+extern const pem_event_action reset_boot_state_tasks[];
+extern const pem_event_action create_new_user_performance_state_tasks[];
+extern const pem_event_action initialize_thermal_controller_tasks[];
+extern const pem_event_action uninitialize_thermal_controller_tasks[];
+extern const pem_event_action set_cpu_power_state[];
+#endif /* _EVENT_SUB_CHAINS_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
new file mode 100644
index 000000000000..5cd123472db4
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
@@ -0,0 +1,438 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "eventmgr.h"
+#include "eventinit.h"
+#include "eventmanagement.h"
+#include "eventmanager.h"
+#include "hardwaremanager.h"
+#include "eventtasks.h"
+#include "power_state.h"
+#include "hwmgr.h"
+#include "amd_powerplay.h"
+#include "psm.h"
+
+#define TEMP_RANGE_MIN (90 * 1000)
+#define TEMP_RANGE_MAX (120 * 1000)
+
+int pem_task_update_allowed_performance_levels(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+
+	if (pem_is_hw_access_blocked(eventmgr))
+		return 0;
+
+	phm_force_dpm_levels(eventmgr->hwmgr, AMD_DPM_FORCED_LEVEL_AUTO);
+
+	return 0;
+}
+
+/* eventtasks_generic.c */
+int pem_task_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	struct pp_hwmgr *hwmgr;
+
+	if (pem_is_hw_access_blocked(eventmgr))
+		return 0;
+
+	hwmgr = eventmgr->hwmgr;
+	if (event_data->pnew_power_state != NULL)
+		hwmgr->request_ps = event_data->pnew_power_state;
+
+	if (phm_cap_enabled(eventmgr->platform_descriptor->platformCaps, PHM_PlatformCaps_DynamicPatchPowerState))
+		psm_adjust_power_state_dynamic(eventmgr, event_data->skip_state_adjust_rules);
+	else
+		psm_adjust_power_state_static(eventmgr, event_data->skip_state_adjust_rules);
+
+	return 0;
+}
+
+int pem_task_power_down_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_power_down_asic(eventmgr->hwmgr);
+}
+
+int pem_task_set_boot_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	if (pem_is_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID))
+		return psm_set_states(eventmgr, &(event_data->requested_state_id));
+
+	return 0;
+}
+
+int pem_task_reset_boot_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_update_new_power_state_clocks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_system_shutdown(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_register_interrupts(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_unregister_interrupts(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return pem_unregister_interrupts(eventmgr);
+}
+
+int pem_task_get_boot_state_id(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	int result;
+
+	result = psm_get_state_by_classification(eventmgr,
+		PP_StateClassificationFlag_Boot,
+		&(event_data->requested_state_id)
+	);
+
+	if (0 == result)
+		pem_set_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID);
+	else
+		pem_unset_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID);
+
+	return result;
+}
+
+int pem_task_enable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_enable_dynamic_state_management(eventmgr->hwmgr);
+}
+
+int pem_task_disable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_enable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_enable_clock_power_gatings(eventmgr->hwmgr);
+}
+
+int pem_task_powerdown_uvd_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_powerdown_uvd(eventmgr->hwmgr);
+}
+
+int pem_task_powerdown_vce_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	phm_powergate_uvd(eventmgr->hwmgr, true);
+	phm_powergate_vce(eventmgr->hwmgr, true);
+	return 0;
+}
+
+int pem_task_disable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_start_asic_block_usage(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_stop_asic_block_usage(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_setup_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_setup_asic(eventmgr->hwmgr);
+}
+
+int pem_task_cleanup_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_store_dal_configuration(struct pp_eventmgr *eventmgr, const struct amd_display_configuration *display_config)
+{
+	/* TODO */
+	return 0;
+	/*phm_store_dal_configuration_data(eventmgr->hwmgr, display_config) */
+}
+
+int pem_task_notify_hw_mgr_display_configuration_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	if (pem_is_hw_access_blocked(eventmgr))
+		return 0;
+
+	return phm_display_configuration_changed(eventmgr->hwmgr);
+}
+
+int pem_task_notify_hw_mgr_pre_display_configuration_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return 0;
+}
+
+int pem_task_notify_smc_display_config_after_power_state_adjustment(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	if (pem_is_hw_access_blocked(eventmgr))
+		return 0;
+
+	return phm_notify_smc_display_config_after_ps_adjustment(eventmgr->hwmgr);
+}
+
+int pem_task_block_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	eventmgr->block_adjust_power_state = true;
+	/* to do PHM_ResetIPSCounter(pEventMgr->pHwMgr);*/
+	return 0;
+}
+
+int pem_task_unblock_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	eventmgr->block_adjust_power_state = false;
+	return 0;
+}
+
+int pem_task_notify_power_state_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_block_hw_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_un_block_hw_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_reset_display_phys_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_set_cpu_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_set_cpu_power_state(eventmgr->hwmgr);
+}
+
+/*powersaving*/
+
+int pem_task_set_power_source(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_notify_hw_of_power_source(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_get_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_reset_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_set_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_set_screen_state_on(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_set_screen_state_off(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_enable_voltage_island_power_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_disable_voltage_island_power_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_enable_cgpg(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_disable_cgpg(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_enable_clock_power_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+
+int pem_task_enable_gfx_clock_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_disable_gfx_clock_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+
+/* performance */
+int pem_task_set_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	if (pem_is_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID))
+		return psm_set_states(eventmgr, &(event_data->requested_state_id));
+
+	return 0;
+}
+
+int pem_task_conditionally_force_3d_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_enable_stutter_mode(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	/* TODO */
+	return 0;
+}
+
+int pem_task_get_2D_performance_state_id(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	int result;
+
+	if (eventmgr->features[PP_Feature_PowerPlay].supported &&
+		!(eventmgr->features[PP_Feature_PowerPlay].enabled))
+			result = psm_get_state_by_classification(eventmgr,
+					PP_StateClassificationFlag_Boot,
+					&(event_data->requested_state_id));
+	else if (eventmgr->features[PP_Feature_User2DPerformance].enabled)
+			result = psm_get_state_by_classification(eventmgr,
+				   PP_StateClassificationFlag_User2DPerformance,
+					&(event_data->requested_state_id));
+	else
+		result = psm_get_ui_state(eventmgr, PP_StateUILabel_Performance,
+					&(event_data->requested_state_id));
+
+	if (0 == result)
+		pem_set_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID);
+	else
+		pem_unset_event_data_valid(event_data->valid_fields, PEM_EventDataValid_RequestedStateID);
+
+	return result;
+}
+
+int pem_task_create_user_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	struct pp_power_state *state;
+	int table_entries;
+	struct pp_hwmgr *hwmgr = eventmgr->hwmgr;
+	int i;
+
+	table_entries = hwmgr->num_ps;
+	state = hwmgr->ps;
+
+restart_search:
+	for (i = 0; i < table_entries; i++) {
+		if (state->classification.ui_label & event_data->requested_ui_label) {
+			event_data->pnew_power_state = state;
+			return 0;
+		}
+		state = (struct pp_power_state *)((unsigned long)state + hwmgr->ps_size);
+	}
+
+	switch (event_data->requested_ui_label) {
+	case PP_StateUILabel_Battery:
+	case PP_StateUILabel_Balanced:
+		event_data->requested_ui_label = PP_StateUILabel_Performance;
+		goto restart_search;
+	default:
+		break;
+	}
+	return -1;
+}
+
+int pem_task_initialize_thermal_controller(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	struct PP_TemperatureRange range;
+
+	range.max = TEMP_RANGE_MAX;
+	range.min = TEMP_RANGE_MIN;
+
+	if (eventmgr == NULL || eventmgr->platform_descriptor == NULL)
+		return -EINVAL;
+
+	if (phm_cap_enabled(eventmgr->platform_descriptor->platformCaps, PHM_PlatformCaps_ThermalController))
+		return phm_start_thermal_controller(eventmgr->hwmgr, &range);
+
+	return 0;
+}
+
+int pem_task_uninitialize_thermal_controller(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
+{
+	return phm_stop_thermal_controller(eventmgr->hwmgr);
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.h b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.h
new file mode 100644
index 000000000000..6c6297e3b598
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.h
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _EVENT_TASKS_H_
+#define _EVENT_TASKS_H_
+#include "eventmgr.h"
+
+struct amd_display_configuration;
+
+/* eventtasks_generic.c */
+int pem_task_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_power_down_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_get_boot_state_id(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_set_boot_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_reset_boot_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_update_new_power_state_clocks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_system_shutdown(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_register_interrupts(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_unregister_interrupts(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_disable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_powerdown_uvd_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_powerdown_vce_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_disable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_start_asic_block_usage(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_stop_asic_block_usage(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_setup_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_cleanup_asic(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_store_dal_configuration (struct pp_eventmgr *eventmgr, const struct amd_display_configuration *display_config);
+int pem_task_notify_hw_mgr_display_configuration_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_notify_hw_mgr_pre_display_configuration_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_block_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_unblock_adjust_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_notify_power_state_change(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_block_hw_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_un_block_hw_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_reset_display_phys_access(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_set_cpu_power_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_notify_smc_display_config_after_power_state_adjustment(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+/*powersaving*/
+
+int pem_task_set_power_source(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_notify_hw_of_power_source(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_get_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_reset_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_set_power_saving_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_set_screen_state_on(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_set_screen_state_off(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_voltage_island_power_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_disable_voltage_island_power_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_cgpg(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_disable_cgpg(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_gfx_clock_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_disable_gfx_clock_gating(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_enable_stutter_mode(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+
+/* performance */
+int pem_task_set_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_conditionally_force_3d_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_get_2D_performance_state_id(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_create_user_performance_state(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_update_allowed_performance_levels(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+/*thermal */
+int pem_task_initialize_thermal_controller(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+int pem_task_uninitialize_thermal_controller(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data);
+
+#endif /* _EVENT_TASKS_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c
new file mode 100644
index 000000000000..a46225c0fc01
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "psm.h"
+
+int psm_get_ui_state(struct pp_eventmgr *eventmgr, enum PP_StateUILabel ui_label, unsigned long *state_id)
+{
+	struct pp_power_state *state;
+	int table_entries;
+	struct pp_hwmgr *hwmgr = eventmgr->hwmgr;
+	int i;
+
+	table_entries = hwmgr->num_ps;
+	state = hwmgr->ps;
+
+	for (i = 0; i < table_entries; i++) {
+		if (state->classification.ui_label & ui_label) {
+			*state_id = state->id;
+			return 0;
+		}
+		state = (struct pp_power_state *)((unsigned long)state + hwmgr->ps_size);
+	}
+	return -1;
+}
+
+int psm_get_state_by_classification(struct pp_eventmgr *eventmgr, enum PP_StateClassificationFlag flag, unsigned long *state_id)
+{
+	struct pp_power_state *state;
+	int table_entries;
+	struct pp_hwmgr *hwmgr = eventmgr->hwmgr;
+	int i;
+
+	table_entries = hwmgr->num_ps;
+	state = hwmgr->ps;
+
+	for (i = 0; i < table_entries; i++) {
+		if (state->classification.flags & flag) {
+			*state_id = state->id;
+			return 0;
+		}
+		state = (struct pp_power_state *)((unsigned long)state + hwmgr->ps_size);
+	}
+	return -1;
+}
+
+int psm_set_states(struct pp_eventmgr *eventmgr, unsigned long *state_id)
+{
+	struct pp_power_state *state;
+	int table_entries;
+	struct pp_hwmgr *hwmgr = eventmgr->hwmgr;
+	int i;
+
+	table_entries = hwmgr->num_ps;
+	state = hwmgr->ps;
+
+	for (i = 0; i < table_entries; i++) {
+		if (state->id == *state_id) {
+			hwmgr->request_ps = state;
+			return 0;
+		}
+		state = (struct pp_power_state *)((unsigned long)state + hwmgr->ps_size);
+	}
+	return -1;
+}
+
+int psm_adjust_power_state_dynamic(struct pp_eventmgr *eventmgr, bool skip)
+{
+
+	struct pp_power_state *pcurrent;
+	struct pp_power_state *requested;
+	struct pp_hwmgr *hwmgr;
+	bool equal;
+
+	if (skip)
+		return 0;
+
+	hwmgr = eventmgr->hwmgr;
+	pcurrent = hwmgr->current_ps;
+	requested = hwmgr->request_ps;
+
+	if (requested == NULL)
+		return 0;
+
+	if (pcurrent == NULL || (0 != phm_check_states_equal(hwmgr, &pcurrent->hardware, &requested->hardware, &equal)))
+		equal = false;
+
+	if (!equal || phm_check_smc_update_required_for_display_configuration(hwmgr)) {
+		phm_apply_state_adjust_rules(hwmgr, requested, pcurrent);
+		phm_set_power_state(hwmgr, &pcurrent->hardware, &requested->hardware);
+		hwmgr->current_ps = requested;
+	}
+	return 0;
+}
+
+int psm_adjust_power_state_static(struct pp_eventmgr *eventmgr, bool skip)
+{
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/psm.h b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.h
new file mode 100644
index 000000000000..fbdff3e02aa3
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "eventmgr.h"
+#include "eventinit.h"
+#include "eventmanagement.h"
+#include "eventmanager.h"
+#include "power_state.h"
+#include "hardwaremanager.h"
+
+int psm_get_ui_state(struct pp_eventmgr *eventmgr, enum PP_StateUILabel ui_label, unsigned long *state_id);
+
+int psm_get_state_by_classification(struct pp_eventmgr *eventmgr, enum PP_StateClassificationFlag flag, unsigned long *state_id);
+
+int psm_set_states(struct pp_eventmgr *eventmgr, unsigned long *state_id);
+
+int psm_adjust_power_state_dynamic(struct pp_eventmgr *eventmgr, bool skip);
+
+int psm_adjust_power_state_static(struct pp_eventmgr *eventmgr, bool skip);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
new file mode 100644
index 000000000000..b664e34dbcc0
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
@@ -0,0 +1,15 @@
+#
+# Makefile for the 'hw manager' sub-component of powerplay.
+# It provides the hardware management services for the driver.
+
+HARDWARE_MGR = hwmgr.o processpptables.o functiontables.o \
+	       hardwaremanager.o pp_acpi.o cz_hwmgr.o \
+               cz_clockpowergating.o \
+	       tonga_processpptables.o ppatomctrl.o \
+               tonga_hwmgr.o pppcielanes.o  tonga_thermal.o\
+               fiji_powertune.o fiji_hwmgr.o tonga_clockpowergating.o \
+               fiji_clockpowergating.o fiji_thermal.o
+
+AMD_PP_HWMGR = $(addprefix $(AMD_PP_PATH)/hwmgr/,$(HARDWARE_MGR))
+
+AMD_POWERPLAY_FILES += $(AMD_PP_HWMGR)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
new file mode 100644
index 000000000000..ad7700822a1c
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
@@ -0,0 +1,252 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "hwmgr.h"
+#include "cz_clockpowergating.h"
+#include "cz_ppsmc.h"
+
+/* PhyID -> Status Mapping in DDI_PHY_GEN_STATUS
+    0    GFX0L (3:0),                  (27:24),
+    1    GFX0H (7:4),                  (31:28),
+    2    GFX1L (3:0),                  (19:16),
+    3    GFX1H (7:4),                  (23:20),
+    4    DDIL   (3:0),                   (11: 8),
+    5    DDIH  (7:4),                   (15:12),
+    6    DDI2L (3:0),                   ( 3: 0),
+    7    DDI2H (7:4),                   ( 7: 4),
+*/
+#define DDI_PHY_GEN_STATUS_VAL(phyID)   (1 << ((3 - ((phyID & 0x07)/2))*8 + (phyID & 0x01)*4))
+#define IS_PHY_ID_USED_BY_PLL(PhyID)    (((0xF3 & (1 << PhyID)) & 0xFF) ? true : false)
+
+
+int cz_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating)
+{
+	int ret = 0;
+
+	switch (block) {
+	case PHM_AsicBlock_UVD_MVC:
+	case PHM_AsicBlock_UVD:
+	case PHM_AsicBlock_UVD_HD:
+	case PHM_AsicBlock_UVD_SD:
+		if (gating == PHM_ClockGateSetting_StaticOff)
+			ret = cz_dpm_powerdown_uvd(hwmgr);
+		else
+			ret = cz_dpm_powerup_uvd(hwmgr);
+		break;
+	case PHM_AsicBlock_GFX:
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+
+bool cz_phm_is_safe_for_asic_block(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *state, enum PHM_AsicBlock block)
+{
+	return true;
+}
+
+
+int cz_phm_enable_disable_gfx_power_gating(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return 0;
+}
+
+int cz_phm_smu_power_up_down_pcie(struct pp_hwmgr *hwmgr, uint32_t target, bool up, uint32_t args)
+{
+	/* TODO */
+	return 0;
+}
+
+int cz_phm_initialize_display_phy_access(struct pp_hwmgr *hwmgr, bool initialize, bool accesshw)
+{
+	/* TODO */
+	return 0;
+}
+
+int cz_phm_get_display_phy_access_info(struct pp_hwmgr *hwmgr)
+{
+	/* TODO */
+	return 0;
+}
+
+int cz_phm_gate_unused_display_phys(struct pp_hwmgr *hwmgr)
+{
+	/* TODO */
+	return 0;
+}
+
+int cz_phm_ungate_all_display_phys(struct pp_hwmgr *hwmgr)
+{
+	/* TODO */
+	return 0;
+}
+
+static int cz_tf_uvd_power_gating_initialize(struct pp_hwmgr *hwmgr, void *pInput, void *pOutput, void *pStorage, int Result)
+{
+	return 0;
+}
+
+static int cz_tf_vce_power_gating_initialize(struct pp_hwmgr *hwmgr, void *pInput, void *pOutput, void *pStorage, int Result)
+{
+	return 0;
+}
+
+int cz_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	uint32_t dpm_features = 0;
+
+	if (enable &&
+		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				  PHM_PlatformCaps_UVDDPM)) {
+		cz_hwmgr->dpm_flags |= DPMFlags_UVD_Enabled;
+		dpm_features |= UVD_DPM_MASK;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			    PPSMC_MSG_EnableAllSmuFeatures, dpm_features);
+	} else {
+		dpm_features |= UVD_DPM_MASK;
+		cz_hwmgr->dpm_flags &= ~DPMFlags_UVD_Enabled;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			   PPSMC_MSG_DisableAllSmuFeatures, dpm_features);
+	}
+	return 0;
+}
+
+int cz_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	uint32_t dpm_features = 0;
+
+	if (enable && phm_cap_enabled(
+				hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_VCEDPM)) {
+		cz_hwmgr->dpm_flags |= DPMFlags_VCE_Enabled;
+		dpm_features |= VCE_DPM_MASK;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			    PPSMC_MSG_EnableAllSmuFeatures, dpm_features);
+	} else {
+		dpm_features |= VCE_DPM_MASK;
+		cz_hwmgr->dpm_flags &= ~DPMFlags_VCE_Enabled;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			   PPSMC_MSG_DisableAllSmuFeatures, dpm_features);
+	}
+
+	return 0;
+}
+
+
+int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (cz_hwmgr->uvd_power_gated == bgate)
+		return 0;
+
+	cz_hwmgr->uvd_power_gated = bgate;
+
+	if (bgate) {
+		cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_CG_STATE_UNGATE);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_GATE);
+		cz_dpm_update_uvd_dpm(hwmgr, true);
+		cz_dpm_powerdown_uvd(hwmgr);
+	} else {
+		cz_dpm_powerup_uvd(hwmgr);
+		cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_GATE);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_CG_STATE_UNGATE);
+		cz_dpm_update_uvd_dpm(hwmgr, false);
+	}
+
+	return 0;
+}
+
+int cz_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_VCEPowerGating)) {
+		if (cz_hwmgr->vce_power_gated != bgate) {
+			if (bgate) {
+				cgs_set_clockgating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_CG_STATE_UNGATE);
+				cgs_set_powergating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_PG_STATE_GATE);
+				cz_enable_disable_vce_dpm(hwmgr, false);
+			/* TODO: to figure out why vce can't be poweroff*/
+				cz_hwmgr->vce_power_gated = true;
+			} else {
+				cz_dpm_powerup_vce(hwmgr);
+				cz_hwmgr->vce_power_gated = false;
+				cgs_set_clockgating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_PG_STATE_GATE);
+				cgs_set_powergating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_CG_STATE_UNGATE);
+				cz_dpm_update_vce_dpm(hwmgr);
+				cz_enable_disable_vce_dpm(hwmgr, true);
+				return 0;
+			}
+		}
+	} else {
+		cz_dpm_update_vce_dpm(hwmgr);
+		cz_enable_disable_vce_dpm(hwmgr, true);
+		return 0;
+	}
+
+	if (!cz_hwmgr->vce_power_gated)
+		cz_dpm_update_vce_dpm(hwmgr);
+
+	return 0;
+}
+
+
+static struct phm_master_table_item cz_enable_clock_power_gatings_list[] = {
+	/*we don't need an exit table here, because there is only D3 cold on Kv*/
+	{ phm_cf_want_uvd_power_gating, cz_tf_uvd_power_gating_initialize },
+	{ phm_cf_want_vce_power_gating, cz_tf_vce_power_gating_initialize },
+	/* to do { NULL, cz_tf_xdma_power_gating_enable }, */
+	{ NULL, NULL }
+};
+
+struct phm_master_table_header cz_phm_enable_clock_power_gatings_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_enable_clock_power_gatings_list
+};
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.h
new file mode 100644
index 000000000000..bbbc0571320e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _CZ_CLOCK_POWER_GATING_H_
+#define _CZ_CLOCK_POWER_GATING_H_
+
+#include "cz_hwmgr.h"
+#include "pp_asicblocks.h"
+
+extern int cz_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating);
+extern struct phm_master_table_header cz_phm_enable_clock_power_gatings_master;
+extern struct phm_master_table_header cz_phm_disable_clock_power_gatings_master;
+extern int cz_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
+extern int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
+extern int cz_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
+extern int cz_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable);
+#endif /* _CZ_CLOCK_POWER_GATING_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
new file mode 100644
index 000000000000..0874ab42ee95
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -0,0 +1,1737 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "atom-types.h"
+#include "atombios.h"
+#include "processpptables.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "smu/smu_8_0_d.h"
+#include "smu8_fusion.h"
+#include "smu/smu_8_0_sh_mask.h"
+#include "smumgr.h"
+#include "hwmgr.h"
+#include "hardwaremanager.h"
+#include "cz_ppsmc.h"
+#include "cz_hwmgr.h"
+#include "power_state.h"
+#include "cz_clockpowergating.h"
+#include "pp_debug.h"
+
+#define ixSMUSVI_NB_CURRENTVID 0xD8230044
+#define CURRENT_NB_VID_MASK 0xff000000
+#define CURRENT_NB_VID__SHIFT 24
+#define ixSMUSVI_GFX_CURRENTVID  0xD8230048
+#define CURRENT_GFX_VID_MASK 0xff000000
+#define CURRENT_GFX_VID__SHIFT 24
+
+static const unsigned long PhwCz_Magic = (unsigned long) PHM_Cz_Magic;
+
+static struct cz_power_state *cast_PhwCzPowerState(struct pp_hw_power_state *hw_ps)
+{
+	if (PhwCz_Magic != hw_ps->magic)
+		return NULL;
+
+	return (struct cz_power_state *)hw_ps;
+}
+
+static const struct cz_power_state *cast_const_PhwCzPowerState(
+				const struct pp_hw_power_state *hw_ps)
+{
+	if (PhwCz_Magic != hw_ps->magic)
+		return NULL;
+
+	return (struct cz_power_state *)hw_ps;
+}
+
+uint32_t cz_get_eclk_level(struct pp_hwmgr *hwmgr,
+					uint32_t clock, uint32_t msg)
+{
+	int i = 0;
+	struct phm_vce_clock_voltage_dependency_table *ptable =
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+
+	switch (msg) {
+	case PPSMC_MSG_SetEclkSoftMin:
+	case PPSMC_MSG_SetEclkHardMin:
+		for (i = 0; i < (int)ptable->count; i++) {
+			if (clock <= ptable->entries[i].ecclk)
+				break;
+		}
+		break;
+
+	case PPSMC_MSG_SetEclkSoftMax:
+	case PPSMC_MSG_SetEclkHardMax:
+		for (i = ptable->count - 1; i >= 0; i--) {
+			if (clock >= ptable->entries[i].ecclk)
+				break;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return i;
+}
+
+static uint32_t cz_get_sclk_level(struct pp_hwmgr *hwmgr,
+				uint32_t clock, uint32_t msg)
+{
+	int i = 0;
+	struct phm_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.vddc_dependency_on_sclk;
+
+	switch (msg) {
+	case PPSMC_MSG_SetSclkSoftMin:
+	case PPSMC_MSG_SetSclkHardMin:
+		for (i = 0; i < (int)table->count; i++) {
+			if (clock <= table->entries[i].clk)
+				break;
+		}
+		break;
+
+	case PPSMC_MSG_SetSclkSoftMax:
+	case PPSMC_MSG_SetSclkHardMax:
+		for (i = table->count - 1; i >= 0; i--) {
+			if (clock >= table->entries[i].clk)
+				break;
+		}
+		break;
+
+	default:
+		break;
+	}
+	return i;
+}
+
+static uint32_t cz_get_uvd_level(struct pp_hwmgr *hwmgr,
+					uint32_t clock, uint32_t msg)
+{
+	int i = 0;
+	struct phm_uvd_clock_voltage_dependency_table *ptable =
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+
+	switch (msg) {
+	case PPSMC_MSG_SetUvdSoftMin:
+	case PPSMC_MSG_SetUvdHardMin:
+		for (i = 0; i < (int)ptable->count; i++) {
+			if (clock <= ptable->entries[i].vclk)
+				break;
+		}
+		break;
+
+	case PPSMC_MSG_SetUvdSoftMax:
+	case PPSMC_MSG_SetUvdHardMax:
+		for (i = ptable->count - 1; i >= 0; i--) {
+			if (clock >= ptable->entries[i].vclk)
+				break;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return i;
+}
+
+static uint32_t cz_get_max_sclk_level(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (cz_hwmgr->max_sclk_level == 0) {
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetMaxSclkLevel);
+		cz_hwmgr->max_sclk_level = smum_get_argument(hwmgr->smumgr) + 1;
+	}
+
+	return cz_hwmgr->max_sclk_level;
+}
+
+static int cz_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	uint32_t i;
+
+	cz_hwmgr->gfx_ramp_step = 256*25/100;
+
+	cz_hwmgr->gfx_ramp_delay = 1; /* by default, we delay 1us */
+
+	for (i = 0; i < CZ_MAX_HARDWARE_POWERLEVELS; i++)
+		cz_hwmgr->activity_target[i] = CZ_AT_DFLT;
+
+	cz_hwmgr->mgcg_cgtt_local0 = 0x00000000;
+	cz_hwmgr->mgcg_cgtt_local1 = 0x00000000;
+
+	cz_hwmgr->clock_slow_down_freq = 25000;
+
+	cz_hwmgr->skip_clock_slow_down = 1;
+
+	cz_hwmgr->enable_nb_ps_policy = 1; /* disable until UNB is ready, Enabled */
+
+	cz_hwmgr->voltage_drop_in_dce_power_gating = 0; /* disable until fully verified */
+
+	cz_hwmgr->voting_rights_clients = 0x00C00033;
+
+	cz_hwmgr->static_screen_threshold = 8;
+
+	cz_hwmgr->ddi_power_gating_disabled = 0;
+
+	cz_hwmgr->bapm_enabled = 1;
+
+	cz_hwmgr->voltage_drop_threshold = 0;
+
+	cz_hwmgr->gfx_power_gating_threshold = 500;
+
+	cz_hwmgr->vce_slow_sclk_threshold = 20000;
+
+	cz_hwmgr->dce_slow_sclk_threshold = 30000;
+
+	cz_hwmgr->disable_driver_thermal_policy = 1;
+
+	cz_hwmgr->disable_nb_ps3_in_battery = 0;
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+							PHM_PlatformCaps_ABM);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_NonABMSupportInPPLib);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					   PHM_PlatformCaps_SclkDeepSleep);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_DynamicM3Arbiter);
+
+	cz_hwmgr->override_dynamic_mgpg = 1;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				  PHM_PlatformCaps_DynamicPatchPowerState);
+
+	cz_hwmgr->thermal_auto_throttling_treshold = 0;
+
+	cz_hwmgr->tdr_clock = 0;
+
+	cz_hwmgr->disable_gfx_power_gating_in_uvd = 0;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_DynamicUVDState);
+
+	cz_hwmgr->cc6_settings.cpu_cc6_disable = false;
+	cz_hwmgr->cc6_settings.cpu_pstate_disable = false;
+	cz_hwmgr->cc6_settings.nb_pstate_switch_disable = false;
+	cz_hwmgr->cc6_settings.cpu_pstate_separation_time = 0;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				   PHM_PlatformCaps_DisableVoltageIsland);
+
+	return 0;
+}
+
+static uint32_t cz_convert_8Bit_index_to_voltage(
+			struct pp_hwmgr *hwmgr, uint16_t voltage)
+{
+	return 6200 - (voltage * 25);
+}
+
+static int cz_construct_max_power_limits_table(struct pp_hwmgr *hwmgr,
+			struct phm_clock_and_voltage_limits *table)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)hwmgr->backend;
+	struct cz_sys_info *sys_info = &cz_hwmgr->sys_info;
+	struct phm_clock_voltage_dependency_table *dep_table =
+				hwmgr->dyn_state.vddc_dependency_on_sclk;
+
+	if (dep_table->count > 0) {
+		table->sclk = dep_table->entries[dep_table->count-1].clk;
+		table->vddc = cz_convert_8Bit_index_to_voltage(hwmgr,
+		   (uint16_t)dep_table->entries[dep_table->count-1].v);
+	}
+	table->mclk = sys_info->nbp_memory_clock[0];
+	return 0;
+}
+
+static int cz_init_dynamic_state_adjustment_rule_settings(
+			struct pp_hwmgr *hwmgr,
+			ATOM_CLK_VOLT_CAPABILITY *disp_voltage_table)
+{
+	uint32_t table_size =
+		sizeof(struct phm_clock_voltage_dependency_table) +
+		(7 * sizeof(struct phm_clock_voltage_dependency_record));
+
+	struct phm_clock_voltage_dependency_table *table_clk_vlt =
+					kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table_clk_vlt) {
+		printk(KERN_ERR "[ powerplay ] Can not allocate memory!\n");
+		return -ENOMEM;
+	}
+
+	table_clk_vlt->count = 8;
+	table_clk_vlt->entries[0].clk = PP_DAL_POWERLEVEL_0;
+	table_clk_vlt->entries[0].v = 0;
+	table_clk_vlt->entries[1].clk = PP_DAL_POWERLEVEL_1;
+	table_clk_vlt->entries[1].v = 1;
+	table_clk_vlt->entries[2].clk = PP_DAL_POWERLEVEL_2;
+	table_clk_vlt->entries[2].v = 2;
+	table_clk_vlt->entries[3].clk = PP_DAL_POWERLEVEL_3;
+	table_clk_vlt->entries[3].v = 3;
+	table_clk_vlt->entries[4].clk = PP_DAL_POWERLEVEL_4;
+	table_clk_vlt->entries[4].v = 4;
+	table_clk_vlt->entries[5].clk = PP_DAL_POWERLEVEL_5;
+	table_clk_vlt->entries[5].v = 5;
+	table_clk_vlt->entries[6].clk = PP_DAL_POWERLEVEL_6;
+	table_clk_vlt->entries[6].v = 6;
+	table_clk_vlt->entries[7].clk = PP_DAL_POWERLEVEL_7;
+	table_clk_vlt->entries[7].v = 7;
+	hwmgr->dyn_state.vddc_dep_on_dal_pwrl = table_clk_vlt;
+
+	return 0;
+}
+
+static int cz_get_system_info_data(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)hwmgr->backend;
+	ATOM_INTEGRATED_SYSTEM_INFO_V1_9 *info = NULL;
+	uint32_t i;
+	int result = 0;
+	uint8_t frev, crev;
+	uint16_t size;
+
+	info = (ATOM_INTEGRATED_SYSTEM_INFO_V1_9 *) cgs_atom_get_data_table(
+			hwmgr->device,
+			GetIndexIntoMasterTable(DATA, IntegratedSystemInfo),
+			&size, &frev, &crev);
+
+	if (crev != 9) {
+		printk(KERN_ERR "[ powerplay ] Unsupported IGP table: %d %d\n", frev, crev);
+		return -EINVAL;
+	}
+
+	if (info == NULL) {
+		printk(KERN_ERR "[ powerplay ] Could not retrieve the Integrated System Info Table!\n");
+		return -EINVAL;
+	}
+
+	cz_hwmgr->sys_info.bootup_uma_clock =
+				   le32_to_cpu(info->ulBootUpUMAClock);
+
+	cz_hwmgr->sys_info.bootup_engine_clock =
+				le32_to_cpu(info->ulBootUpEngineClock);
+
+	cz_hwmgr->sys_info.dentist_vco_freq =
+				   le32_to_cpu(info->ulDentistVCOFreq);
+
+	cz_hwmgr->sys_info.system_config =
+				     le32_to_cpu(info->ulSystemConfig);
+
+	cz_hwmgr->sys_info.bootup_nb_voltage_index =
+				  le16_to_cpu(info->usBootUpNBVoltage);
+
+	cz_hwmgr->sys_info.htc_hyst_lmt =
+			(info->ucHtcHystLmt == 0) ? 5 : info->ucHtcHystLmt;
+
+	cz_hwmgr->sys_info.htc_tmp_lmt =
+			(info->ucHtcTmpLmt == 0) ? 203 : info->ucHtcTmpLmt;
+
+	if (cz_hwmgr->sys_info.htc_tmp_lmt <=
+			cz_hwmgr->sys_info.htc_hyst_lmt) {
+		printk(KERN_ERR "[ powerplay ] The htcTmpLmt should be larger than htcHystLmt.\n");
+		return -EINVAL;
+	}
+
+	cz_hwmgr->sys_info.nb_dpm_enable =
+				cz_hwmgr->enable_nb_ps_policy &&
+				(le32_to_cpu(info->ulSystemConfig) >> 3 & 0x1);
+
+	for (i = 0; i < CZ_NUM_NBPSTATES; i++) {
+		if (i < CZ_NUM_NBPMEMORYCLOCK) {
+			cz_hwmgr->sys_info.nbp_memory_clock[i] =
+			  le32_to_cpu(info->ulNbpStateMemclkFreq[i]);
+		}
+		cz_hwmgr->sys_info.nbp_n_clock[i] =
+			    le32_to_cpu(info->ulNbpStateNClkFreq[i]);
+	}
+
+	for (i = 0; i < MAX_DISPLAY_CLOCK_LEVEL; i++) {
+		cz_hwmgr->sys_info.display_clock[i] =
+					le32_to_cpu(info->sDispClkVoltageMapping[i].ulMaximumSupportedCLK);
+	}
+
+	/* Here use 4 levels, make sure not exceed */
+	for (i = 0; i < CZ_NUM_NBPSTATES; i++) {
+		cz_hwmgr->sys_info.nbp_voltage_index[i] =
+			     le16_to_cpu(info->usNBPStateVoltage[i]);
+	}
+
+	if (!cz_hwmgr->sys_info.nb_dpm_enable) {
+		for (i = 1; i < CZ_NUM_NBPSTATES; i++) {
+			if (i < CZ_NUM_NBPMEMORYCLOCK) {
+				cz_hwmgr->sys_info.nbp_memory_clock[i] =
+				    cz_hwmgr->sys_info.nbp_memory_clock[0];
+			}
+			cz_hwmgr->sys_info.nbp_n_clock[i] =
+				    cz_hwmgr->sys_info.nbp_n_clock[0];
+			cz_hwmgr->sys_info.nbp_voltage_index[i] =
+				    cz_hwmgr->sys_info.nbp_voltage_index[0];
+		}
+	}
+
+	if (le32_to_cpu(info->ulGPUCapInfo) &
+		SYS_INFO_GPUCAPS__ENABEL_DFS_BYPASS) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_EnableDFSBypass);
+	}
+
+	cz_hwmgr->sys_info.uma_channel_number = info->ucUMAChannelNumber;
+
+	cz_construct_max_power_limits_table (hwmgr,
+				    &hwmgr->dyn_state.max_clock_voltage_on_ac);
+
+	cz_init_dynamic_state_adjustment_rule_settings(hwmgr,
+				    &info->sDISPCLK_Voltage[0]);
+
+	return result;
+}
+
+static int cz_construct_boot_state(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	cz_hwmgr->boot_power_level.engineClock =
+				cz_hwmgr->sys_info.bootup_engine_clock;
+
+	cz_hwmgr->boot_power_level.vddcIndex =
+			(uint8_t)cz_hwmgr->sys_info.bootup_nb_voltage_index;
+
+	cz_hwmgr->boot_power_level.dsDividerIndex = 0;
+
+	cz_hwmgr->boot_power_level.ssDividerIndex = 0;
+
+	cz_hwmgr->boot_power_level.allowGnbSlow = 1;
+
+	cz_hwmgr->boot_power_level.forceNBPstate = 0;
+
+	cz_hwmgr->boot_power_level.hysteresis_up = 0;
+
+	cz_hwmgr->boot_power_level.numSIMDToPowerDown = 0;
+
+	cz_hwmgr->boot_power_level.display_wm = 0;
+
+	cz_hwmgr->boot_power_level.vce_wm = 0;
+
+	return 0;
+}
+
+static int cz_tf_reset_active_process_mask(struct pp_hwmgr *hwmgr, void *input,
+					void *output, void *storage, int result)
+{
+	return 0;
+}
+
+static int cz_tf_upload_pptable_to_smu(struct pp_hwmgr *hwmgr, void *input,
+				       void *output, void *storage, int result)
+{
+	struct SMU8_Fusion_ClkTable *clock_table;
+	int ret;
+	uint32_t i;
+	void *table = NULL;
+	pp_atomctrl_clock_dividers_kong dividers;
+
+	struct phm_clock_voltage_dependency_table *vddc_table =
+		hwmgr->dyn_state.vddc_dependency_on_sclk;
+	struct phm_clock_voltage_dependency_table *vdd_gfx_table =
+		hwmgr->dyn_state.vdd_gfx_dependency_on_sclk;
+	struct phm_acp_clock_voltage_dependency_table *acp_table =
+		hwmgr->dyn_state.acp_clock_voltage_dependency_table;
+	struct phm_uvd_clock_voltage_dependency_table *uvd_table =
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+	struct phm_vce_clock_voltage_dependency_table *vce_table =
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+
+	if (!hwmgr->need_pp_table_upload)
+		return 0;
+
+	ret = smum_download_powerplay_table(hwmgr->smumgr, &table);
+
+	PP_ASSERT_WITH_CODE((0 == ret && NULL != table),
+			    "Fail to get clock table from SMU!", return -EINVAL;);
+
+	clock_table = (struct SMU8_Fusion_ClkTable *)table;
+
+	/* patch clock table */
+	PP_ASSERT_WITH_CODE((vddc_table->count <= CZ_MAX_HARDWARE_POWERLEVELS),
+			    "Dependency table entry exceeds max limit!", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((vdd_gfx_table->count <= CZ_MAX_HARDWARE_POWERLEVELS),
+			    "Dependency table entry exceeds max limit!", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((acp_table->count <= CZ_MAX_HARDWARE_POWERLEVELS),
+			    "Dependency table entry exceeds max limit!", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((uvd_table->count <= CZ_MAX_HARDWARE_POWERLEVELS),
+			    "Dependency table entry exceeds max limit!", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((vce_table->count <= CZ_MAX_HARDWARE_POWERLEVELS),
+			    "Dependency table entry exceeds max limit!", return -EINVAL;);
+
+	for (i = 0; i < CZ_MAX_HARDWARE_POWERLEVELS; i++) {
+
+		/* vddc_sclk */
+		clock_table->SclkBreakdownTable.ClkLevel[i].GnbVid =
+			(i < vddc_table->count) ? (uint8_t)vddc_table->entries[i].v : 0;
+		clock_table->SclkBreakdownTable.ClkLevel[i].Frequency =
+			(i < vddc_table->count) ? vddc_table->entries[i].clk : 0;
+
+		atomctrl_get_engine_pll_dividers_kong(hwmgr,
+						      clock_table->SclkBreakdownTable.ClkLevel[i].Frequency,
+						      &dividers);
+
+		clock_table->SclkBreakdownTable.ClkLevel[i].DfsDid =
+			(uint8_t)dividers.pll_post_divider;
+
+		/* vddgfx_sclk */
+		clock_table->SclkBreakdownTable.ClkLevel[i].GfxVid =
+			(i < vdd_gfx_table->count) ? (uint8_t)vdd_gfx_table->entries[i].v : 0;
+
+		/* acp breakdown */
+		clock_table->AclkBreakdownTable.ClkLevel[i].GfxVid =
+			(i < acp_table->count) ? (uint8_t)acp_table->entries[i].v : 0;
+		clock_table->AclkBreakdownTable.ClkLevel[i].Frequency =
+			(i < acp_table->count) ? acp_table->entries[i].acpclk : 0;
+
+		atomctrl_get_engine_pll_dividers_kong(hwmgr,
+						      clock_table->AclkBreakdownTable.ClkLevel[i].Frequency,
+						      &dividers);
+
+		clock_table->AclkBreakdownTable.ClkLevel[i].DfsDid =
+			(uint8_t)dividers.pll_post_divider;
+
+
+		/* uvd breakdown */
+		clock_table->VclkBreakdownTable.ClkLevel[i].GfxVid =
+			(i < uvd_table->count) ? (uint8_t)uvd_table->entries[i].v : 0;
+		clock_table->VclkBreakdownTable.ClkLevel[i].Frequency =
+			(i < uvd_table->count) ? uvd_table->entries[i].vclk : 0;
+
+		atomctrl_get_engine_pll_dividers_kong(hwmgr,
+						      clock_table->VclkBreakdownTable.ClkLevel[i].Frequency,
+						      &dividers);
+
+		clock_table->VclkBreakdownTable.ClkLevel[i].DfsDid =
+			(uint8_t)dividers.pll_post_divider;
+
+		clock_table->DclkBreakdownTable.ClkLevel[i].GfxVid =
+			(i < uvd_table->count) ? (uint8_t)uvd_table->entries[i].v : 0;
+		clock_table->DclkBreakdownTable.ClkLevel[i].Frequency =
+			(i < uvd_table->count) ? uvd_table->entries[i].dclk : 0;
+
+		atomctrl_get_engine_pll_dividers_kong(hwmgr,
+						      clock_table->DclkBreakdownTable.ClkLevel[i].Frequency,
+						      &dividers);
+
+		clock_table->DclkBreakdownTable.ClkLevel[i].DfsDid =
+			(uint8_t)dividers.pll_post_divider;
+
+		/* vce breakdown */
+		clock_table->EclkBreakdownTable.ClkLevel[i].GfxVid =
+			(i < vce_table->count) ? (uint8_t)vce_table->entries[i].v : 0;
+		clock_table->EclkBreakdownTable.ClkLevel[i].Frequency =
+			(i < vce_table->count) ? vce_table->entries[i].ecclk : 0;
+
+
+		atomctrl_get_engine_pll_dividers_kong(hwmgr,
+						      clock_table->EclkBreakdownTable.ClkLevel[i].Frequency,
+						      &dividers);
+
+		clock_table->EclkBreakdownTable.ClkLevel[i].DfsDid =
+			(uint8_t)dividers.pll_post_divider;
+
+	}
+	ret = smum_upload_powerplay_table(hwmgr->smumgr);
+
+	return ret;
+}
+
+static int cz_tf_init_sclk_limit(struct pp_hwmgr *hwmgr, void *input,
+				 void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_clock_voltage_dependency_table *table =
+					hwmgr->dyn_state.vddc_dependency_on_sclk;
+	unsigned long clock = 0, level;
+
+	if (NULL == table || table->count <= 0)
+		return -EINVAL;
+
+	cz_hwmgr->sclk_dpm.soft_min_clk = table->entries[0].clk;
+	cz_hwmgr->sclk_dpm.hard_min_clk = table->entries[0].clk;
+
+	level = cz_get_max_sclk_level(hwmgr) - 1;
+
+	if (level < table->count)
+		clock = table->entries[level].clk;
+	else
+		clock = table->entries[table->count - 1].clk;
+
+	cz_hwmgr->sclk_dpm.soft_max_clk = clock;
+	cz_hwmgr->sclk_dpm.hard_max_clk = clock;
+
+	return 0;
+}
+
+static int cz_tf_init_uvd_limit(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_uvd_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+	unsigned long clock = 0, level;
+
+	if (NULL == table || table->count <= 0)
+		return -EINVAL;
+
+	cz_hwmgr->uvd_dpm.soft_min_clk = 0;
+	cz_hwmgr->uvd_dpm.hard_min_clk = 0;
+
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetMaxUvdLevel);
+	level = smum_get_argument(hwmgr->smumgr);
+
+	if (level < table->count)
+		clock = table->entries[level].vclk;
+	else
+		clock = table->entries[table->count - 1].vclk;
+
+	cz_hwmgr->uvd_dpm.soft_max_clk = clock;
+	cz_hwmgr->uvd_dpm.hard_max_clk = clock;
+
+	return 0;
+}
+
+static int cz_tf_init_vce_limit(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_vce_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+	unsigned long clock = 0, level;
+
+	if (NULL == table || table->count <= 0)
+		return -EINVAL;
+
+	cz_hwmgr->vce_dpm.soft_min_clk = 0;
+	cz_hwmgr->vce_dpm.hard_min_clk = 0;
+
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetMaxEclkLevel);
+	level = smum_get_argument(hwmgr->smumgr);
+
+	if (level < table->count)
+		clock = table->entries[level].ecclk;
+	else
+		clock = table->entries[table->count - 1].ecclk;
+
+	cz_hwmgr->vce_dpm.soft_max_clk = clock;
+	cz_hwmgr->vce_dpm.hard_max_clk = clock;
+
+	return 0;
+}
+
+static int cz_tf_init_acp_limit(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_acp_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.acp_clock_voltage_dependency_table;
+	unsigned long clock = 0, level;
+
+	if (NULL == table || table->count <= 0)
+		return -EINVAL;
+
+	cz_hwmgr->acp_dpm.soft_min_clk = 0;
+	cz_hwmgr->acp_dpm.hard_min_clk = 0;
+
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetMaxAclkLevel);
+	level = smum_get_argument(hwmgr->smumgr);
+
+	if (level < table->count)
+		clock = table->entries[level].acpclk;
+	else
+		clock = table->entries[table->count - 1].acpclk;
+
+	cz_hwmgr->acp_dpm.soft_max_clk = clock;
+	cz_hwmgr->acp_dpm.hard_max_clk = clock;
+	return 0;
+}
+
+static int cz_tf_init_power_gate_state(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	cz_hwmgr->uvd_power_gated = false;
+	cz_hwmgr->vce_power_gated = false;
+	cz_hwmgr->samu_power_gated = false;
+	cz_hwmgr->acp_power_gated = false;
+	cz_hwmgr->pgacpinit = true;
+
+	return 0;
+}
+
+static int cz_tf_init_sclk_threshold(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	cz_hwmgr->low_sclk_interrupt_threshold = 0;
+
+	return 0;
+}
+static int cz_tf_update_sclk_limit(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_clock_voltage_dependency_table *table =
+					hwmgr->dyn_state.vddc_dependency_on_sclk;
+
+	unsigned long clock = 0;
+	unsigned long level;
+	unsigned long stable_pstate_sclk;
+	struct PP_Clocks clocks;
+	unsigned long percentage;
+
+	cz_hwmgr->sclk_dpm.soft_min_clk = table->entries[0].clk;
+	level = cz_get_max_sclk_level(hwmgr) - 1;
+
+	if (level < table->count)
+		cz_hwmgr->sclk_dpm.soft_max_clk  = table->entries[level].clk;
+	else
+		cz_hwmgr->sclk_dpm.soft_max_clk  = table->entries[table->count - 1].clk;
+
+	/*PECI_GetMinClockSettings(pHwMgr->pPECI, &clocks);*/
+	clock = clocks.engineClock;
+
+	if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) {
+		cz_hwmgr->sclk_dpm.hard_min_clk = clock;
+
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SetSclkHardMin,
+						 cz_get_sclk_level(hwmgr,
+					cz_hwmgr->sclk_dpm.hard_min_clk,
+					     PPSMC_MSG_SetSclkHardMin));
+	}
+
+	clock = cz_hwmgr->sclk_dpm.soft_min_clk;
+
+	/* update minimum clocks for Stable P-State feature */
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				     PHM_PlatformCaps_StablePState)) {
+		percentage = 75;
+		/*Sclk - calculate sclk value based on percentage and find FLOOR sclk from VddcDependencyOnSCLK table  */
+		stable_pstate_sclk = (hwmgr->dyn_state.max_clock_voltage_on_ac.mclk *
+					percentage) / 100;
+
+		if (clock < stable_pstate_sclk)
+			clock = stable_pstate_sclk;
+	} else {
+		if (clock < hwmgr->gfx_arbiter.sclk)
+			clock = hwmgr->gfx_arbiter.sclk;
+	}
+
+	if (cz_hwmgr->sclk_dpm.soft_min_clk != clock) {
+		cz_hwmgr->sclk_dpm.soft_min_clk = clock;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SetSclkSoftMin,
+						cz_get_sclk_level(hwmgr,
+					cz_hwmgr->sclk_dpm.soft_min_clk,
+					     PPSMC_MSG_SetSclkSoftMin));
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_StablePState) &&
+			 cz_hwmgr->sclk_dpm.soft_max_clk != clock) {
+		cz_hwmgr->sclk_dpm.soft_max_clk = clock;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SetSclkSoftMax,
+						cz_get_sclk_level(hwmgr,
+					cz_hwmgr->sclk_dpm.soft_max_clk,
+					PPSMC_MSG_SetSclkSoftMax));
+	}
+
+	return 0;
+}
+
+static int cz_tf_set_deep_sleep_sclk_threshold(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_SclkDeepSleep)) {
+		uint32_t clks = hwmgr->display_config.min_core_set_clock_in_sr;
+		if (clks == 0)
+			clks = CZ_MIN_DEEP_SLEEP_SCLK;
+
+		PP_DBG_LOG("Setting Deep Sleep Clock: %d\n", clks);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetMinDeepSleepSclk,
+				clks);
+	}
+
+	return 0;
+}
+
+static int cz_tf_set_watermark_threshold(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr =
+				  (struct cz_hwmgr *)(hwmgr->backend);
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SetWatermarkFrequency,
+				      cz_hwmgr->sclk_dpm.soft_max_clk);
+
+	return 0;
+}
+
+static int cz_tf_set_enabled_levels(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	return 0;
+}
+
+
+static int cz_tf_enable_nb_dpm(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	int ret = 0;
+
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	unsigned long dpm_features = 0;
+
+	if (!cz_hwmgr->is_nb_dpm_enabled) {
+		PP_DBG_LOG("enabling ALL SMU features.\n");
+		dpm_features |= NB_DPM_MASK;
+		ret = smum_send_msg_to_smc_with_parameter(
+							     hwmgr->smumgr,
+					 PPSMC_MSG_EnableAllSmuFeatures,
+							     dpm_features);
+		if (ret == 0)
+			cz_hwmgr->is_nb_dpm_enabled = true;
+	}
+
+	return ret;
+}
+
+static int cz_nbdpm_pstate_enable_disable(struct pp_hwmgr *hwmgr, bool enable, bool lock)
+{
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (hw_data->is_nb_dpm_enabled) {
+		if (enable) {
+			PP_DBG_LOG("enable Low Memory PState.\n");
+
+			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_EnableLowMemoryPstate,
+						(lock ? 1 : 0));
+		} else {
+			PP_DBG_LOG("disable Low Memory PState.\n");
+
+			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_DisableLowMemoryPstate,
+						(lock ? 1 : 0));
+		}
+	}
+
+	return 0;
+}
+
+static int cz_tf_update_low_mem_pstate(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	bool disable_switch;
+	bool enable_low_mem_state;
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+	const struct phm_set_power_state_input *states = (struct phm_set_power_state_input *)input;
+	const struct cz_power_state *pnew_state = cast_const_PhwCzPowerState(states->pnew_state);
+
+	if (hw_data->sys_info.nb_dpm_enable) {
+		disable_switch = hw_data->cc6_settings.nb_pstate_switch_disable ? true : false;
+		enable_low_mem_state = hw_data->cc6_settings.nb_pstate_switch_disable ? false : true;
+
+		if (pnew_state->action == FORCE_HIGH)
+			cz_nbdpm_pstate_enable_disable(hwmgr, false, disable_switch);
+		else if(pnew_state->action == CANCEL_FORCE_HIGH)
+			cz_nbdpm_pstate_enable_disable(hwmgr, false, disable_switch);
+		else 
+			cz_nbdpm_pstate_enable_disable(hwmgr, enable_low_mem_state, disable_switch);
+	}
+	return 0;
+}
+
+static struct phm_master_table_item cz_set_power_state_list[] = {
+	{NULL, cz_tf_update_sclk_limit},
+	{NULL, cz_tf_set_deep_sleep_sclk_threshold},
+	{NULL, cz_tf_set_watermark_threshold},
+	{NULL, cz_tf_set_enabled_levels},
+	{NULL, cz_tf_enable_nb_dpm},
+	{NULL, cz_tf_update_low_mem_pstate},
+	{NULL, NULL}
+};
+
+static struct phm_master_table_header cz_set_power_state_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_set_power_state_list
+};
+
+static struct phm_master_table_item cz_setup_asic_list[] = {
+	{NULL, cz_tf_reset_active_process_mask},
+	{NULL, cz_tf_upload_pptable_to_smu},
+	{NULL, cz_tf_init_sclk_limit},
+	{NULL, cz_tf_init_uvd_limit},
+	{NULL, cz_tf_init_vce_limit},
+	{NULL, cz_tf_init_acp_limit},
+	{NULL, cz_tf_init_power_gate_state},
+	{NULL, cz_tf_init_sclk_threshold},
+	{NULL, NULL}
+};
+
+static struct phm_master_table_header cz_setup_asic_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_setup_asic_list
+};
+
+static int cz_tf_power_up_display_clock_sys_pll(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+	hw_data->disp_clk_bypass_pending = false;
+	hw_data->disp_clk_bypass = false;
+
+	return 0;
+}
+
+static int cz_tf_clear_nb_dpm_flag(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+	hw_data->is_nb_dpm_enabled = false;
+
+	return 0;
+}
+
+static int cz_tf_reset_cc6_data(struct pp_hwmgr *hwmgr,
+					void *input, void *output,
+					void *storage, int result)
+{
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+
+	hw_data->cc6_settings.cc6_setting_changed = false;
+	hw_data->cc6_settings.cpu_pstate_separation_time = 0;
+	hw_data->cc6_settings.cpu_cc6_disable = false;
+	hw_data->cc6_settings.cpu_pstate_disable = false;
+
+	return 0;
+}
+
+static struct phm_master_table_item cz_power_down_asic_list[] = {
+	{NULL, cz_tf_power_up_display_clock_sys_pll},
+	{NULL, cz_tf_clear_nb_dpm_flag},
+	{NULL, cz_tf_reset_cc6_data},
+	{NULL, NULL}
+};
+
+static struct phm_master_table_header cz_power_down_asic_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_power_down_asic_list
+};
+
+static int cz_tf_program_voting_clients(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	PHMCZ_WRITE_SMC_REGISTER(hwmgr->device, CG_FREQ_TRAN_VOTING_0,
+				PPCZ_VOTINGRIGHTSCLIENTS_DFLT0);
+	return 0;
+}
+
+static int cz_tf_start_dpm(struct pp_hwmgr *hwmgr, void *input, void *output,
+			   void *storage, int result)
+{
+	int res = 0xff;
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	unsigned long dpm_features = 0;
+
+	cz_hwmgr->dpm_flags |= DPMFlags_SCLK_Enabled;
+	dpm_features |= SCLK_DPM_MASK;
+
+	res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_EnableAllSmuFeatures,
+				dpm_features);
+
+	return res;
+}
+
+static int cz_tf_program_bootup_state(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	cz_hwmgr->sclk_dpm.soft_min_clk = cz_hwmgr->sys_info.bootup_engine_clock;
+	cz_hwmgr->sclk_dpm.soft_max_clk = cz_hwmgr->sys_info.bootup_engine_clock;
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMin,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_min_clk,
+				PPSMC_MSG_SetSclkSoftMin));
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMax,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_max_clk,
+				PPSMC_MSG_SetSclkSoftMax));
+
+	return 0;
+}
+
+int cz_tf_reset_acp_boot_level(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	cz_hwmgr->acp_boot_level = 0xff;
+	return 0;
+}
+
+static bool cz_dpm_check_smu_features(struct pp_hwmgr *hwmgr,
+				unsigned long check_feature)
+{
+	int result;
+	unsigned long features;
+
+	result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_GetFeatureStatus, 0);
+	if (result == 0) {
+		features = smum_get_argument(hwmgr->smumgr);
+		if (features & check_feature)
+			return true;
+	}
+
+	return result;
+}
+
+static int cz_tf_check_for_dpm_disabled(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	if (cz_dpm_check_smu_features(hwmgr, SMU_EnabledFeatureScoreboard_SclkDpmOn))
+		return PP_Result_TableImmediateExit;
+	return 0;
+}
+
+static int cz_tf_enable_didt(struct pp_hwmgr *hwmgr, void *input,
+				void *output, void *storage, int result)
+{
+	/* TO DO */
+	return 0;
+}
+
+static int cz_tf_check_for_dpm_enabled(struct pp_hwmgr *hwmgr,
+						void *input, void *output,
+						void *storage, int result)
+{
+	if (!cz_dpm_check_smu_features(hwmgr,
+			     SMU_EnabledFeatureScoreboard_SclkDpmOn))
+		return PP_Result_TableImmediateExit;
+	return 0;
+}
+
+static struct phm_master_table_item cz_disable_dpm_list[] = {
+	{ NULL, cz_tf_check_for_dpm_enabled},
+	{NULL, NULL},
+};
+
+
+static struct phm_master_table_header cz_disable_dpm_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_disable_dpm_list
+};
+
+static struct phm_master_table_item cz_enable_dpm_list[] = {
+	{ NULL, cz_tf_check_for_dpm_disabled },
+	{ NULL, cz_tf_program_voting_clients },
+	{ NULL, cz_tf_start_dpm},
+	{ NULL, cz_tf_program_bootup_state},
+	{ NULL, cz_tf_enable_didt },
+	{ NULL, cz_tf_reset_acp_boot_level },
+	{NULL, NULL},
+};
+
+static struct phm_master_table_header cz_enable_dpm_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	cz_enable_dpm_list
+};
+
+static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				struct pp_power_state  *prequest_ps,
+			const struct pp_power_state *pcurrent_ps)
+{
+	struct cz_power_state *cz_ps =
+				cast_PhwCzPowerState(&prequest_ps->hardware);
+
+	const struct cz_power_state *cz_current_ps =
+				cast_const_PhwCzPowerState(&pcurrent_ps->hardware);
+
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct PP_Clocks clocks;
+	bool force_high;
+	unsigned long  num_of_active_displays = 4;
+
+	cz_ps->evclk = hwmgr->vce_arbiter.evclk;
+	cz_ps->ecclk = hwmgr->vce_arbiter.ecclk;
+
+	cz_ps->need_dfs_bypass = true;
+
+	cz_hwmgr->video_start = (hwmgr->uvd_arbiter.vclk != 0 || hwmgr->uvd_arbiter.dclk != 0 ||
+				hwmgr->vce_arbiter.evclk != 0 || hwmgr->vce_arbiter.ecclk != 0);
+
+	cz_hwmgr->battery_state = (PP_StateUILabel_Battery == prequest_ps->classification.ui_label);
+
+	/* to do PECI_GetMinClockSettings(pHwMgr->pPECI, &clocks); */
+	/* PECI_GetNumberOfActiveDisplays(pHwMgr->pPECI, &numOfActiveDisplays); */
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+		clocks.memoryClock = hwmgr->dyn_state.max_clock_voltage_on_ac.mclk;
+	else
+		clocks.memoryClock = 0;
+
+	if (clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
+		clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
+
+	force_high = (clocks.memoryClock > cz_hwmgr->sys_info.nbp_memory_clock[CZ_NUM_NBPMEMORYCLOCK - 1])
+			|| (num_of_active_displays >= 3);
+
+	cz_ps->action = cz_current_ps->action;
+
+	if ((force_high == false) && (cz_ps->action == FORCE_HIGH))
+		cz_ps->action = CANCEL_FORCE_HIGH;
+	else if ((force_high == true) && (cz_ps->action != FORCE_HIGH))
+		cz_ps->action = FORCE_HIGH;
+	else
+		cz_ps->action = DO_NOTHING;
+
+	return 0;
+}
+
+static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+
+	result = cz_initialize_dpm_defaults(hwmgr);
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] cz_initialize_dpm_defaults failed\n");
+		return result;
+	}
+
+	result = cz_get_system_info_data(hwmgr);
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] cz_get_system_info_data failed\n");
+		return result;
+	}
+
+	cz_construct_boot_state(hwmgr);
+
+	result = phm_construct_table(hwmgr, &cz_setup_asic_master,
+				&(hwmgr->setup_asic));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to construct setup ASIC\n");
+		return result;
+	}
+
+	result = phm_construct_table(hwmgr, &cz_power_down_asic_master,
+				&(hwmgr->power_down_asic));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to construct power down ASIC\n");
+		return result;
+	}
+
+	result = phm_construct_table(hwmgr, &cz_disable_dpm_master,
+				&(hwmgr->disable_dynamic_state_management));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to disable_dynamic_state\n");
+		return result;
+	}
+	result = phm_construct_table(hwmgr, &cz_enable_dpm_master,
+				&(hwmgr->enable_dynamic_state_management));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to enable_dynamic_state\n");
+		return result;
+	}
+	result = phm_construct_table(hwmgr, &cz_set_power_state_master,
+				&(hwmgr->set_power_state));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to construct set_power_state\n");
+		return result;
+	}
+
+	result = phm_construct_table(hwmgr, &cz_phm_enable_clock_power_gatings_master, &(hwmgr->enable_clock_power_gatings));
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] Fail to construct enable_clock_power_gatings\n");
+		return result;
+	}
+	return result;
+}
+
+static int cz_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr != NULL || hwmgr->backend != NULL) {
+		kfree(hwmgr->backend);
+		kfree(hwmgr);
+	}
+	return 0;
+}
+
+int cz_phm_force_dpm_highest(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (cz_hwmgr->sclk_dpm.soft_min_clk !=
+				cz_hwmgr->sclk_dpm.soft_max_clk)
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SetSclkSoftMin,
+						cz_get_sclk_level(hwmgr,
+						cz_hwmgr->sclk_dpm.soft_max_clk,
+						PPSMC_MSG_SetSclkSoftMin));
+	return 0;
+}
+
+int cz_phm_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.vddc_dependency_on_sclk;
+	unsigned long clock = 0, level;
+
+	if (NULL == table || table->count <= 0)
+		return -EINVAL;
+
+	cz_hwmgr->sclk_dpm.soft_min_clk = table->entries[0].clk;
+	cz_hwmgr->sclk_dpm.hard_min_clk = table->entries[0].clk;
+
+	level = cz_get_max_sclk_level(hwmgr) - 1;
+
+	if (level < table->count)
+		clock = table->entries[level].clk;
+	else
+		clock = table->entries[table->count - 1].clk;
+
+	cz_hwmgr->sclk_dpm.soft_max_clk = clock;
+	cz_hwmgr->sclk_dpm.hard_max_clk = clock;
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMin,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_min_clk,
+				PPSMC_MSG_SetSclkSoftMin));
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMax,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_max_clk,
+				PPSMC_MSG_SetSclkSoftMax));
+
+	return 0;
+}
+
+int cz_phm_force_dpm_lowest(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (cz_hwmgr->sclk_dpm.soft_min_clk !=
+				cz_hwmgr->sclk_dpm.soft_max_clk) {
+		cz_hwmgr->sclk_dpm.soft_max_clk =
+			cz_hwmgr->sclk_dpm.soft_min_clk;
+
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMax,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_max_clk,
+				PPSMC_MSG_SetSclkSoftMax));
+	}
+
+	return 0;
+}
+
+static int cz_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
+				enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+		ret = cz_phm_force_dpm_highest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		ret = cz_phm_force_dpm_lowest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		ret = cz_phm_unforce_dpm_levels(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	default:
+		break;
+	}
+
+	hwmgr->dpm_level = level;
+
+	return ret;
+}
+
+int cz_dpm_powerdown_uvd(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_UVDPowerGating))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						     PPSMC_MSG_UVDPowerOFF);
+	return 0;
+}
+
+int cz_dpm_powerup_uvd(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_UVDPowerGating)) {
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				  PHM_PlatformCaps_UVDDynamicPowerGating)) {
+			return smum_send_msg_to_smc_with_parameter(
+								hwmgr->smumgr,
+						   PPSMC_MSG_UVDPowerON, 1);
+		} else {
+			return smum_send_msg_to_smc_with_parameter(
+								hwmgr->smumgr,
+						   PPSMC_MSG_UVDPowerON, 0);
+		}
+	}
+
+	return 0;
+}
+
+int cz_dpm_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_uvd_clock_voltage_dependency_table *ptable =
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+
+	if (!bgate) {
+		/* Stable Pstate is enabled and we need to set the UVD DPM to highest level */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_StablePState)) {
+			cz_hwmgr->uvd_dpm.hard_min_clk =
+				   ptable->entries[ptable->count - 1].vclk;
+
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						     PPSMC_MSG_SetUvdHardMin,
+						      cz_get_uvd_level(hwmgr,
+					     cz_hwmgr->uvd_dpm.hard_min_clk,
+						   PPSMC_MSG_SetUvdHardMin));
+
+			cz_enable_disable_uvd_dpm(hwmgr, true);
+		} else
+			cz_enable_disable_uvd_dpm(hwmgr, true);
+	} else
+		cz_enable_disable_uvd_dpm(hwmgr, false);
+
+	return 0;
+}
+
+int  cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct phm_vce_clock_voltage_dependency_table *ptable =
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+
+	/* Stable Pstate is enabled and we need to set the VCE DPM to highest level */
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_StablePState)) {
+		cz_hwmgr->vce_dpm.hard_min_clk =
+				  ptable->entries[ptable->count - 1].ecclk;
+
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SetEclkHardMin,
+					cz_get_eclk_level(hwmgr,
+					     cz_hwmgr->vce_dpm.hard_min_clk,
+						PPSMC_MSG_SetEclkHardMin));
+	} else {
+		/*EPR# 419220 -HW limitation to to */
+		cz_hwmgr->vce_dpm.hard_min_clk = hwmgr->vce_arbiter.ecclk;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					    PPSMC_MSG_SetEclkHardMin,
+					    cz_get_eclk_level(hwmgr,
+				     cz_hwmgr->vce_dpm.hard_min_clk,
+					  PPSMC_MSG_SetEclkHardMin));
+
+	}
+	return 0;
+}
+
+int cz_dpm_powerdown_vce(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_VCEPowerGating))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						     PPSMC_MSG_VCEPowerOFF);
+	return 0;
+}
+
+int cz_dpm_powerup_vce(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					 PHM_PlatformCaps_VCEPowerGating))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						     PPSMC_MSG_VCEPowerON);
+	return 0;
+}
+
+static int cz_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	return cz_hwmgr->sys_info.bootup_uma_clock;
+}
+
+static int cz_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct cz_power_state  *cz_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	cz_ps = cast_PhwCzPowerState(&ps->hardware);
+
+	if (low)
+		return cz_ps->levels[0].engineClock;
+	else
+		return cz_ps->levels[cz_ps->level-1].engineClock;
+}
+
+static int cz_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
+					struct pp_hw_power_state *hw_ps)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+	struct cz_power_state *cz_ps = cast_PhwCzPowerState(hw_ps);
+
+	cz_ps->level = 1;
+	cz_ps->nbps_flags = 0;
+	cz_ps->bapm_flags = 0;
+	cz_ps->levels[0] = cz_hwmgr->boot_power_level;
+
+	return 0;
+}
+
+static int cz_dpm_get_pp_table_entry_callback(
+						     struct pp_hwmgr *hwmgr,
+					   struct pp_hw_power_state *hw_ps,
+							  unsigned int index,
+						     const void *clock_info)
+{
+	struct cz_power_state *cz_ps = cast_PhwCzPowerState(hw_ps);
+
+	const ATOM_PPLIB_CZ_CLOCK_INFO *cz_clock_info = clock_info;
+
+	struct phm_clock_voltage_dependency_table *table =
+				    hwmgr->dyn_state.vddc_dependency_on_sclk;
+	uint8_t clock_info_index = cz_clock_info->index;
+
+	if (clock_info_index > (uint8_t)(hwmgr->platform_descriptor.hardwareActivityPerformanceLevels - 1))
+		clock_info_index = (uint8_t)(hwmgr->platform_descriptor.hardwareActivityPerformanceLevels - 1);
+
+	cz_ps->levels[index].engineClock = table->entries[clock_info_index].clk;
+	cz_ps->levels[index].vddcIndex = (uint8_t)table->entries[clock_info_index].v;
+
+	cz_ps->level = index + 1;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
+		cz_ps->levels[index].dsDividerIndex = 5;
+		cz_ps->levels[index].ssDividerIndex = 5;
+	}
+
+	return 0;
+}
+
+static int cz_dpm_get_num_of_pp_table_entries(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	unsigned long ret = 0;
+
+	result = pp_tables_get_num_of_entries(hwmgr, &ret);
+
+	return result ? 0 : ret;
+}
+
+static int cz_dpm_get_pp_table_entry(struct pp_hwmgr *hwmgr,
+		    unsigned long entry, struct pp_power_state *ps)
+{
+	int result;
+	struct cz_power_state *cz_ps;
+
+	ps->hardware.magic = PhwCz_Magic;
+
+	cz_ps = cast_PhwCzPowerState(&(ps->hardware));
+
+	result = pp_tables_get_entry(hwmgr, entry, ps,
+			cz_dpm_get_pp_table_entry_callback);
+
+	cz_ps->uvd_clocks.vclk = ps->uvd_clocks.VCLK;
+	cz_ps->uvd_clocks.dclk = ps->uvd_clocks.DCLK;
+
+	return result;
+}
+
+int cz_get_power_state_size(struct pp_hwmgr *hwmgr)
+{
+	return sizeof(struct cz_power_state);
+}
+
+static void
+cz_print_current_perforce_level(struct pp_hwmgr *hwmgr, struct seq_file *m)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	struct phm_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.vddc_dependency_on_sclk;
+
+	struct phm_vce_clock_voltage_dependency_table *vce_table =
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+
+	struct phm_uvd_clock_voltage_dependency_table *uvd_table =
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+
+	uint32_t sclk_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX),
+					TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX);
+	uint32_t uvd_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
+					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_UVD_INDEX);
+	uint32_t vce_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
+					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_VCE_INDEX);
+
+	uint32_t sclk, vclk, dclk, ecclk, tmp, activity_percent;
+	uint16_t vddnb, vddgfx;
+	int result;
+
+	if (sclk_index >= NUM_SCLK_LEVELS) {
+		seq_printf(m, "\n invalid sclk dpm profile %d\n", sclk_index);
+	} else {
+		sclk = table->entries[sclk_index].clk;
+		seq_printf(m, "\n index: %u sclk: %u MHz\n", sclk_index, sclk/100);
+	}
+
+	tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_NB_CURRENTVID) &
+		CURRENT_NB_VID_MASK) >> CURRENT_NB_VID__SHIFT;
+	vddnb = cz_convert_8Bit_index_to_voltage(hwmgr, tmp);
+	tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_GFX_CURRENTVID) &
+		CURRENT_GFX_VID_MASK) >> CURRENT_GFX_VID__SHIFT;
+	vddgfx = cz_convert_8Bit_index_to_voltage(hwmgr, (u16)tmp);
+	seq_printf(m, "\n vddnb: %u vddgfx: %u\n", vddnb, vddgfx);
+
+	seq_printf(m, "\n uvd    %sabled\n", cz_hwmgr->uvd_power_gated ? "dis" : "en");
+	if (!cz_hwmgr->uvd_power_gated) {
+		if (uvd_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
+			seq_printf(m, "\n invalid uvd dpm level %d\n", uvd_index);
+		} else {
+			vclk = uvd_table->entries[uvd_index].vclk;
+			dclk = uvd_table->entries[uvd_index].dclk;
+			seq_printf(m, "\n index: %u uvd vclk: %u MHz dclk: %u MHz\n", uvd_index, vclk/100, dclk/100);
+		}
+	}
+
+	seq_printf(m, "\n vce    %sabled\n", cz_hwmgr->vce_power_gated ? "dis" : "en");
+	if (!cz_hwmgr->vce_power_gated) {
+		if (vce_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
+			seq_printf(m, "\n invalid vce dpm level %d\n", vce_index);
+		} else {
+			ecclk = vce_table->entries[vce_index].ecclk;
+			seq_printf(m, "\n index: %u vce ecclk: %u MHz\n", vce_index, ecclk/100);
+		}
+	}
+
+	result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetAverageGraphicsActivity);
+	if (0 == result) {
+		activity_percent = cgs_read_register(hwmgr->device, mmSMU_MP1_SRBM2P_ARG_0);
+		activity_percent = activity_percent > 100 ? 100 : activity_percent;
+	} else {
+		activity_percent = 50;
+	}
+
+	seq_printf(m, "\n [GPU load]: %u %%\n\n", activity_percent);
+}
+
+static void cz_hw_print_display_cfg(
+	const struct cc6_settings *cc6_settings)
+{
+	PP_DBG_LOG("New Display Configuration:\n");
+
+	PP_DBG_LOG("   cpu_cc6_disable: %d\n",
+			cc6_settings->cpu_cc6_disable);
+	PP_DBG_LOG("   cpu_pstate_disable: %d\n",
+			cc6_settings->cpu_pstate_disable);
+	PP_DBG_LOG("   nb_pstate_switch_disable: %d\n",
+			cc6_settings->nb_pstate_switch_disable);
+	PP_DBG_LOG("   cpu_pstate_separation_time: %d\n\n",
+			cc6_settings->cpu_pstate_separation_time);
+}
+
+ static int cz_set_cpu_power_state(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+	uint32_t data = 0;
+
+	if (hw_data->cc6_settings.cc6_setting_changed == true) {
+
+		hw_data->cc6_settings.cc6_setting_changed = false;
+
+		cz_hw_print_display_cfg(&hw_data->cc6_settings);
+
+		data |= (hw_data->cc6_settings.cpu_pstate_separation_time
+			& PWRMGT_SEPARATION_TIME_MASK)
+			<< PWRMGT_SEPARATION_TIME_SHIFT;
+
+		data|= (hw_data->cc6_settings.cpu_cc6_disable ? 0x1 : 0x0)
+			<< PWRMGT_DISABLE_CPU_CSTATES_SHIFT;
+
+		data|= (hw_data->cc6_settings.cpu_pstate_disable ? 0x1 : 0x0)
+			<< PWRMGT_DISABLE_CPU_PSTATES_SHIFT;
+
+		PP_DBG_LOG("SetDisplaySizePowerParams data: 0x%X\n",
+			data);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SetDisplaySizePowerParams,
+						data);
+	}
+
+	return 0;
+}
+
+
+ static int cz_store_cc6_data(struct pp_hwmgr *hwmgr, uint32_t separation_time,
+			bool cc6_disable, bool pstate_disable, bool pstate_switch_disable)
+ {
+	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
+
+	if (separation_time !=
+		hw_data->cc6_settings.cpu_pstate_separation_time
+		|| cc6_disable !=
+		hw_data->cc6_settings.cpu_cc6_disable
+		|| pstate_disable !=
+		hw_data->cc6_settings.cpu_pstate_disable
+		|| pstate_switch_disable !=
+		hw_data->cc6_settings.nb_pstate_switch_disable) {
+
+		hw_data->cc6_settings.cc6_setting_changed = true;
+
+		hw_data->cc6_settings.cpu_pstate_separation_time =
+			separation_time;
+		hw_data->cc6_settings.cpu_cc6_disable =
+			cc6_disable;
+		hw_data->cc6_settings.cpu_pstate_disable =
+			pstate_disable;
+		hw_data->cc6_settings.nb_pstate_switch_disable =
+			pstate_switch_disable;
+
+	}
+
+	return 0;
+}
+
+ static int cz_get_dal_power_level(struct pp_hwmgr *hwmgr,
+		struct amd_pp_dal_clock_info*info)
+{
+	uint32_t i;
+	const struct phm_clock_voltage_dependency_table * table =
+			hwmgr->dyn_state.vddc_dep_on_dal_pwrl;
+	const struct phm_clock_and_voltage_limits* limits =
+			&hwmgr->dyn_state.max_clock_voltage_on_ac;
+
+	info->engine_max_clock = limits->sclk;
+	info->memory_max_clock = limits->mclk;
+
+	for (i = table->count - 1; i > 0; i--) {
+
+		if (limits->vddc >= table->entries[i].v) {
+			info->level = table->entries[i].clk;
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
+
+static const struct pp_hwmgr_func cz_hwmgr_funcs = {
+	.backend_init = cz_hwmgr_backend_init,
+	.backend_fini = cz_hwmgr_backend_fini,
+	.asic_setup = NULL,
+	.apply_state_adjust_rules = cz_apply_state_adjust_rules,
+	.force_dpm_level = cz_dpm_force_dpm_level,
+	.get_power_state_size = cz_get_power_state_size,
+	.powerdown_uvd = cz_dpm_powerdown_uvd,
+	.powergate_uvd = cz_dpm_powergate_uvd,
+	.powergate_vce = cz_dpm_powergate_vce,
+	.get_mclk = cz_dpm_get_mclk,
+	.get_sclk = cz_dpm_get_sclk,
+	.patch_boot_state = cz_dpm_patch_boot_state,
+	.get_pp_table_entry = cz_dpm_get_pp_table_entry,
+	.get_num_of_pp_table_entries = cz_dpm_get_num_of_pp_table_entries,
+	.print_current_perforce_level = cz_print_current_perforce_level,
+	.set_cpu_power_state = cz_set_cpu_power_state,
+	.store_cc6_data = cz_store_cc6_data,
+	.get_dal_power_level= cz_get_dal_power_level,
+};
+
+int cz_hwmgr_init(struct pp_hwmgr *hwmgr)
+{
+	struct cz_hwmgr *cz_hwmgr;
+	int ret = 0;
+
+	cz_hwmgr = kzalloc(sizeof(struct cz_hwmgr), GFP_KERNEL);
+	if (cz_hwmgr == NULL)
+		return -ENOMEM;
+
+	hwmgr->backend = cz_hwmgr;
+	hwmgr->hwmgr_func = &cz_hwmgr_funcs;
+	hwmgr->pptable_func = &pptable_funcs;
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h
new file mode 100644
index 000000000000..c477f1cf3f23
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h
@@ -0,0 +1,326 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _CZ_HWMGR_H_
+#define _CZ_HWMGR_H_
+
+#include "cgs_common.h"
+#include "ppatomctrl.h"
+
+#define CZ_NUM_NBPSTATES               4
+#define CZ_NUM_NBPMEMORYCLOCK          2
+#define MAX_DISPLAY_CLOCK_LEVEL        8
+#define CZ_AT_DFLT                     30
+#define CZ_MAX_HARDWARE_POWERLEVELS    8
+#define PPCZ_VOTINGRIGHTSCLIENTS_DFLT0   0x3FFFC102
+#define CZ_MIN_DEEP_SLEEP_SCLK         800
+
+/* Carrizo device IDs */
+#define DEVICE_ID_CZ_9870             0x9870
+#define DEVICE_ID_CZ_9874             0x9874
+#define DEVICE_ID_CZ_9875             0x9875
+#define DEVICE_ID_CZ_9876             0x9876
+#define DEVICE_ID_CZ_9877             0x9877
+
+#define PHMCZ_WRITE_SMC_REGISTER(device, reg, value)                            \
+		cgs_write_ind_register(device, CGS_IND_REG__SMC, ix##reg, value)
+
+struct cz_dpm_entry {
+	uint32_t soft_min_clk;
+	uint32_t hard_min_clk;
+	uint32_t soft_max_clk;
+	uint32_t hard_max_clk;
+};
+
+struct cz_sys_info {
+	uint32_t bootup_uma_clock;
+	uint32_t bootup_engine_clock;
+	uint32_t dentist_vco_freq;
+	uint32_t nb_dpm_enable;
+	uint32_t nbp_memory_clock[CZ_NUM_NBPMEMORYCLOCK];
+	uint32_t nbp_n_clock[CZ_NUM_NBPSTATES];
+	uint16_t nbp_voltage_index[CZ_NUM_NBPSTATES];
+	uint32_t display_clock[MAX_DISPLAY_CLOCK_LEVEL];
+	uint16_t bootup_nb_voltage_index;
+	uint8_t htc_tmp_lmt;
+	uint8_t htc_hyst_lmt;
+	uint32_t system_config;
+	uint32_t uma_channel_number;
+};
+
+#define MAX_DISPLAYPHY_IDS			0x8
+#define DISPLAYPHY_LANEMASK			0xF
+#define UNKNOWN_TRANSMITTER_PHY_ID		(-1)
+
+#define DISPLAYPHY_PHYID_SHIFT			24
+#define DISPLAYPHY_LANESELECT_SHIFT		16
+
+#define DISPLAYPHY_RX_SELECT			0x1
+#define DISPLAYPHY_TX_SELECT			0x2
+#define DISPLAYPHY_CORE_SELECT			0x4
+
+#define DDI_POWERGATING_ARG(phyID, lanemask, rx, tx, core) \
+		(((uint32_t)(phyID))<<DISPLAYPHY_PHYID_SHIFT | \
+		((uint32_t)(lanemask))<<DISPLAYPHY_LANESELECT_SHIFT | \
+		((rx) ? DISPLAYPHY_RX_SELECT : 0) | \
+		((tx) ? DISPLAYPHY_TX_SELECT : 0) | \
+		((core) ? DISPLAYPHY_CORE_SELECT : 0))
+
+struct cz_display_phy_info_entry {
+	uint8_t phy_present;
+	uint8_t active_lane_mapping;
+	uint8_t display_config_type;
+	uint8_t active_number_of_lanes;
+};
+
+#define CZ_MAX_DISPLAYPHY_IDS			10
+
+struct cz_display_phy_info {
+	bool display_phy_access_initialized;
+	struct cz_display_phy_info_entry entries[CZ_MAX_DISPLAYPHY_IDS];
+};
+
+struct cz_power_level {
+	uint32_t engineClock;
+	uint8_t vddcIndex;
+	uint8_t dsDividerIndex;
+	uint8_t ssDividerIndex;
+	uint8_t allowGnbSlow;
+	uint8_t forceNBPstate;
+	uint8_t display_wm;
+	uint8_t vce_wm;
+	uint8_t numSIMDToPowerDown;
+	uint8_t hysteresis_up;
+	uint8_t rsv[3];
+};
+
+struct cz_uvd_clocks {
+	uint32_t vclk;
+	uint32_t dclk;
+	uint32_t vclk_low_divider;
+	uint32_t vclk_high_divider;
+	uint32_t dclk_low_divider;
+	uint32_t dclk_high_divider;
+};
+
+enum cz_pstate_previous_action {
+	DO_NOTHING = 1,
+	FORCE_HIGH,
+	CANCEL_FORCE_HIGH
+};
+
+struct pp_disable_nb_ps_flags {
+	union {
+		struct {
+			uint32_t entry : 1;
+			uint32_t display : 1;
+			uint32_t driver: 1;
+			uint32_t vce : 1;
+			uint32_t uvd : 1;
+			uint32_t acp : 1;
+			uint32_t reserved: 26;
+		} bits;
+		uint32_t u32All;
+	};
+};
+
+struct cz_power_state {
+	unsigned int magic;
+	uint32_t level;
+	struct cz_uvd_clocks uvd_clocks;
+	uint32_t evclk;
+	uint32_t ecclk;
+	uint32_t samclk;
+	uint32_t acpclk;
+	bool need_dfs_bypass;
+	uint32_t nbps_flags;
+	uint32_t bapm_flags;
+	uint8_t dpm_0_pg_nb_ps_low;
+	uint8_t dpm_0_pg_nb_ps_high;
+	uint8_t dpm_x_nb_ps_low;
+	uint8_t dpm_x_nb_ps_high;
+	enum cz_pstate_previous_action action;
+	struct cz_power_level levels[CZ_MAX_HARDWARE_POWERLEVELS];
+	struct pp_disable_nb_ps_flags disable_nb_ps_flag;
+};
+
+#define DPMFlags_SCLK_Enabled			0x00000001
+#define DPMFlags_UVD_Enabled			0x00000002
+#define DPMFlags_VCE_Enabled			0x00000004
+#define DPMFlags_ACP_Enabled			0x00000008
+#define DPMFlags_ForceHighestValid		0x40000000
+#define DPMFlags_Debug				0x80000000
+
+#define SMU_EnabledFeatureScoreboard_AcpDpmOn   0x00000001 /* bit 0 */
+#define SMU_EnabledFeatureScoreboard_SclkDpmOn    0x00200000
+#define SMU_EnabledFeatureScoreboard_UvdDpmOn   0x00800000 /* bit 23 */
+#define SMU_EnabledFeatureScoreboard_VceDpmOn   0x01000000 /* bit 24 */
+
+struct cc6_settings {
+	bool cc6_setting_changed;
+	bool nb_pstate_switch_disable;/* controls NB PState switch */
+	bool cpu_cc6_disable; /* controls CPU CState switch ( on or off) */
+	bool cpu_pstate_disable;
+	uint32_t cpu_pstate_separation_time;
+};
+
+struct cz_hwmgr {
+	uint32_t activity_target[CZ_MAX_HARDWARE_POWERLEVELS];
+	uint32_t dpm_interval;
+
+	uint32_t voltage_drop_threshold;
+
+	uint32_t voting_rights_clients;
+
+	uint32_t disable_driver_thermal_policy;
+
+	uint32_t static_screen_threshold;
+
+	uint32_t gfx_power_gating_threshold;
+
+	uint32_t activity_hysteresis;
+	uint32_t bootup_sclk_divider;
+	uint32_t gfx_ramp_step;
+	uint32_t gfx_ramp_delay; /* in micro-seconds */
+
+	uint32_t thermal_auto_throttling_treshold;
+
+	struct cz_sys_info sys_info;
+
+	struct cz_power_level boot_power_level;
+	struct cz_power_state *cz_current_ps;
+	struct cz_power_state *cz_requested_ps;
+
+	uint32_t mgcg_cgtt_local0;
+	uint32_t mgcg_cgtt_local1;
+
+	uint32_t tdr_clock; /* in 10khz unit */
+
+	uint32_t ddi_power_gating_disabled;
+	uint32_t disable_gfx_power_gating_in_uvd;
+	uint32_t disable_nb_ps3_in_battery;
+
+	uint32_t lock_nb_ps_in_uvd_play_back;
+
+	struct cz_display_phy_info display_phy_info;
+	uint32_t vce_slow_sclk_threshold; /* default 200mhz */
+	uint32_t dce_slow_sclk_threshold; /* default 300mhz */
+	uint32_t min_sclk_did;  /* minimum sclk divider */
+
+	bool disp_clk_bypass;
+	bool disp_clk_bypass_pending;
+	uint32_t bapm_enabled;
+	uint32_t clock_slow_down_freq;
+	uint32_t skip_clock_slow_down;
+	uint32_t enable_nb_ps_policy;
+	uint32_t voltage_drop_in_dce_power_gating;
+	uint32_t uvd_dpm_interval;
+	uint32_t override_dynamic_mgpg;
+	uint32_t lclk_deep_enabled;
+
+	uint32_t uvd_performance;
+
+	bool video_start;
+	bool battery_state;
+	uint32_t lowest_valid;
+	uint32_t highest_valid;
+	uint32_t high_voltage_threshold;
+	uint32_t is_nb_dpm_enabled;
+	struct cc6_settings cc6_settings;
+	uint32_t is_voltage_island_enabled;
+
+	bool pgacpinit;
+
+	uint8_t disp_config;
+
+	/* PowerTune */
+	uint32_t power_containment_features;
+	bool cac_enabled;
+	bool disable_uvd_power_tune_feature;
+	bool enable_ba_pm_feature;
+	bool enable_tdc_limit_feature;
+
+	uint32_t sram_end;
+	uint32_t dpm_table_start;
+	uint32_t soft_regs_start;
+
+	uint8_t uvd_level_count;
+	uint8_t vce_level_count;
+
+	uint8_t acp_level_count;
+	uint8_t samu_level_count;
+	uint32_t fps_high_threshold;
+	uint32_t fps_low_threshold;
+
+	uint32_t dpm_flags;
+	struct cz_dpm_entry sclk_dpm;
+	struct cz_dpm_entry uvd_dpm;
+	struct cz_dpm_entry vce_dpm;
+	struct cz_dpm_entry acp_dpm;
+
+	uint8_t uvd_boot_level;
+	uint8_t vce_boot_level;
+	uint8_t acp_boot_level;
+	uint8_t samu_boot_level;
+	uint8_t uvd_interval;
+	uint8_t vce_interval;
+	uint8_t acp_interval;
+	uint8_t samu_interval;
+
+	uint8_t graphics_interval;
+	uint8_t graphics_therm_throttle_enable;
+	uint8_t graphics_voltage_change_enable;
+
+	uint8_t graphics_clk_slow_enable;
+	uint8_t graphics_clk_slow_divider;
+
+	uint32_t display_cac;
+	uint32_t low_sclk_interrupt_threshold;
+
+	uint32_t dram_log_addr_h;
+	uint32_t dram_log_addr_l;
+	uint32_t dram_log_phy_addr_h;
+	uint32_t dram_log_phy_addr_l;
+	uint32_t dram_log_buff_size;
+
+	bool uvd_power_gated;
+	bool vce_power_gated;
+	bool samu_power_gated;
+	bool acp_power_gated;
+	bool acp_power_up_no_dsp;
+	uint32_t active_process_mask;
+
+	uint32_t max_sclk_level;
+	uint32_t num_of_clk_entries;
+};
+
+struct pp_hwmgr;
+
+int cz_hwmgr_init(struct pp_hwmgr *hwmgr);
+int cz_dpm_powerdown_uvd(struct pp_hwmgr *hwmgr);
+int cz_dpm_powerup_uvd(struct pp_hwmgr *hwmgr);
+int cz_dpm_powerdown_vce(struct pp_hwmgr *hwmgr);
+int cz_dpm_powerup_vce(struct pp_hwmgr *hwmgr);
+int cz_dpm_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+int  cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr);
+#endif /* _CZ_HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
new file mode 100644
index 000000000000..e68edf06ed73
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "hwmgr.h"
+#include "fiji_clockpowergating.h"
+#include "fiji_ppsmc.h"
+#include "fiji_hwmgr.h"
+
+int fiji_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = false;
+	data->vce_power_gated = false;
+	data->samu_power_gated = false;
+	data->acp_power_gated = false;
+
+	return 0;
+}
+
+int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (data->uvd_power_gated == bgate)
+		return 0;
+
+	data->uvd_power_gated = bgate;
+
+	if (bgate)
+		fiji_update_uvd_dpm(hwmgr, true);
+	else
+		fiji_update_uvd_dpm(hwmgr, false);
+
+	return 0;
+}
+
+int fiji_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_set_power_state_input states;
+	const struct pp_power_state  *pcurrent;
+	struct pp_power_state  *requested;
+
+	if (data->vce_power_gated == bgate)
+		return 0;
+
+	data->vce_power_gated = bgate;
+
+	pcurrent = hwmgr->current_ps;
+	requested = hwmgr->request_ps;
+
+	states.pcurrent_state = &(pcurrent->hardware);
+	states.pnew_state = &(requested->hardware);
+
+	fiji_update_vce_dpm(hwmgr, &states);
+	fiji_enable_disable_vce_dpm(hwmgr, !bgate);
+
+	return 0;
+}
+
+int fiji_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (data->samu_power_gated == bgate)
+		return 0;
+
+	data->samu_power_gated = bgate;
+
+	if (bgate)
+		fiji_update_samu_dpm(hwmgr, true);
+	else
+		fiji_update_samu_dpm(hwmgr, false);
+
+	return 0;
+}
+
+int fiji_phm_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (data->acp_power_gated == bgate)
+		return 0;
+
+	data->acp_power_gated = bgate;
+
+	if (bgate)
+		fiji_update_acp_dpm(hwmgr, true);
+	else
+		fiji_update_acp_dpm(hwmgr, false);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.h
new file mode 100644
index 000000000000..33af5f511ab8
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _FIJI_CLOCK_POWER_GATING_H_
+#define _FIJI_CLOCK_POWER_GATING_H_
+
+#include "fiji_hwmgr.h"
+#include "pp_asicblocks.h"
+
+extern int fiji_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
+extern int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
+extern int fiji_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate);
+extern int fiji_phm_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate);
+extern int fiji_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
+#endif /* _TONGA_CLOCK_POWER_GATING_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h
new file mode 100644
index 000000000000..32d43e8fecb2
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef FIJI_DYN_DEFAULTS_H
+#define FIJI_DYN_DEFAULTS_H
+
+/** \file
+* Volcanic Islands Dynamic default parameters.
+*/
+
+enum FIJIdpm_TrendDetection
+{
+    FIJIAdpm_TrendDetection_AUTO,
+    FIJIAdpm_TrendDetection_UP,
+    FIJIAdpm_TrendDetection_DOWN
+};
+typedef enum FIJIdpm_TrendDetection FIJIdpm_TrendDetection;
+
+/* We need to fill in the default values!!!!!!!!!!!!!!!!!!!!!!! */
+
+/* Bit vector representing same fields as hardware register. */
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT0    0x3FFFC102  /* CP_Gfx_busy ????
+                                                         * HDP_busy
+                                                         * IH_busy
+                                                         * UVD_busy
+                                                         * VCE_busy
+                                                         * ACP_busy
+                                                         * SAMU_busy
+                                                         * SDMA enabled */
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT1    0x000400  /* FE_Gfx_busy  - Intended for primary usage.   Rest are for flexibility. ????
+                                                       * SH_Gfx_busy
+                                                       * RB_Gfx_busy
+                                                       * VCE_busy */
+
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT2    0xC00080  /* SH_Gfx_busy - Intended for primary usage.   Rest are for flexibility.
+                                                       * FE_Gfx_busy
+                                                       * RB_Gfx_busy
+                                                       * ACP_busy */
+
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT3    0xC00200  /* RB_Gfx_busy - Intended for primary usage.   Rest are for flexibility.
+                                                       * FE_Gfx_busy
+                                                       * SH_Gfx_busy
+                                                       * UVD_busy */
+
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT4    0xC01680  /* UVD_busy
+                                                       * VCE_busy
+                                                       * ACP_busy
+                                                       * SAMU_busy */
+
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT5    0xC00033  /* GFX, HDP */
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT6    0xC00033  /* GFX, HDP */
+#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT7    0x3FFFC000  /* GFX, HDP */
+
+
+/* thermal protection counter (units). */
+#define PPFIJI_THERMALPROTECTCOUNTER_DFLT            0x200 /* ~19us */
+
+/* static screen threshold unit */
+#define PPFIJI_STATICSCREENTHRESHOLDUNIT_DFLT    0
+
+/* static screen threshold */
+#define PPFIJI_STATICSCREENTHRESHOLD_DFLT        0x00C8
+
+/* gfx idle clock stop threshold */
+#define PPFIJI_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200 /* ~19us with static screen threshold unit of 0 */
+
+/* Fixed reference divider to use when building baby stepping tables. */
+#define PPFIJI_REFERENCEDIVIDER_DFLT                  4
+
+/* ULV voltage change delay time
+ * Used to be delay_vreg in N.I. split for S.I.
+ * Using N.I. delay_vreg value as default
+ * ReferenceClock = 2700
+ * VoltageResponseTime = 1000
+ * VDDCDelayTime = (VoltageResponseTime * ReferenceClock) / 1600 = 1687
+ */
+#define PPFIJI_ULVVOLTAGECHANGEDELAY_DFLT             1687
+
+#define PPFIJI_CGULVPARAMETER_DFLT			0x00040035
+#define PPFIJI_CGULVCONTROL_DFLT			0x00007450
+#define PPFIJI_TARGETACTIVITY_DFLT			30 /* 30%*/
+#define PPFIJI_MCLK_TARGETACTIVITY_DFLT		10 /* 10% */
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
new file mode 100644
index 000000000000..28031a7eddba
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
@@ -0,0 +1,5127 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/fb.h>
+#include "linux/delay.h"
+
+#include "hwmgr.h"
+#include "fiji_smumgr.h"
+#include "atombios.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "atombios.h"
+#include "cgs_common.h"
+#include "fiji_dyn_defaults.h"
+#include "fiji_powertune.h"
+#include "smu73.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "pppcielanes.h"
+#include "fiji_hwmgr.h"
+#include "tonga_processpptables.h"
+#include "tonga_pptable.h"
+#include "pp_debug.h"
+#include "pp_acpi.h"
+#include "amd_pcie_helpers.h"
+#include "cgs_linux.h"
+#include "ppinterrupt.h"
+
+#include "fiji_clockpowergating.h"
+#include "fiji_thermal.h"
+
+#define VOLTAGE_SCALE	4
+#define SMC_RAM_END		0x40000
+#define VDDC_VDDCI_DELTA	300
+
+#define MC_SEQ_MISC0_GDDR5_SHIFT 28
+#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
+#define MC_SEQ_MISC0_GDDR5_VALUE 5
+
+#define MC_CG_ARB_FREQ_F0           0x0a /* boot-up default */
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+/* From smc_reg.h */
+#define SMC_CG_IND_START            0xc0030000
+#define SMC_CG_IND_END              0xc0040000  /* First byte after SMC_CG_IND */
+
+#define VOLTAGE_SCALE               4
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+
+#define VDDC_VDDCI_DELTA            300
+
+#define ixSWRST_COMMAND_1           0x1400103
+#define MC_SEQ_CNTL__CAC_EN_MASK    0x40000000
+
+/** Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
+enum DPM_EVENT_SRC {
+    DPM_EVENT_SRC_ANALOG = 0,               /* Internal analog trip point */
+    DPM_EVENT_SRC_EXTERNAL = 1,             /* External (GPIO 17) signal */
+    DPM_EVENT_SRC_DIGITAL = 2,              /* Internal digital trip point (DIG_THERM_DPM) */
+    DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,   /* Internal analog or external */
+    DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4   /* Internal digital or external */
+};
+
+
+/* [2.5%,~2.5%] Clock stretched is multiple of 2.5% vs
+ * not and [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ]
+ */
+uint16_t fiji_clock_stretcher_lookup_table[2][4] = { {600, 1050, 3, 0},
+                                                {600, 1050, 6, 1} };
+
+/* [FF, SS] type, [] 4 voltage ranges, and
+ * [Floor Freq, Boundary Freq, VID min , VID max]
+ */
+uint32_t fiji_clock_stretcher_ddt_table[2][4][4] =
+{ { {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
+  { {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
+
+/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%]
+ * (coming from PWR_CKS_CNTL.stretch_amount reg spec)
+ */
+uint8_t fiji_clock_stretch_amount_conversion[2][6] = { {0, 1, 3, 2, 4, 5},
+                                                  {0, 2, 4, 5, 6, 5} };
+
+const unsigned long PhwFiji_Magic = (unsigned long)(PHM_VIslands_Magic);
+
+struct fiji_power_state *cast_phw_fiji_power_state(
+				  struct pp_hw_power_state *hw_ps)
+{
+	PP_ASSERT_WITH_CODE((PhwFiji_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL;);
+
+	return (struct fiji_power_state *)hw_ps;
+}
+
+const struct fiji_power_state *cast_const_phw_fiji_power_state(
+				 const struct pp_hw_power_state *hw_ps)
+{
+	PP_ASSERT_WITH_CODE((PhwFiji_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL;);
+
+	return (const struct fiji_power_state *)hw_ps;
+}
+
+static bool fiji_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
+
+static void fiji_init_dpm_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_ulv_parm *ulv = &data->ulv;
+
+	ulv->cg_ulv_parameter = PPFIJI_CGULVPARAMETER_DFLT;
+	data->voting_rights_clients0 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT0;
+	data->voting_rights_clients1 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT1;
+	data->voting_rights_clients2 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT2;
+	data->voting_rights_clients3 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT3;
+	data->voting_rights_clients4 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT4;
+	data->voting_rights_clients5 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT5;
+	data->voting_rights_clients6 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT6;
+	data->voting_rights_clients7 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT7;
+
+	data->static_screen_threshold_unit =
+			PPFIJI_STATICSCREENTHRESHOLDUNIT_DFLT;
+	data->static_screen_threshold =
+			PPFIJI_STATICSCREENTHRESHOLD_DFLT;
+
+	/* Unset ABM cap as it moved to DAL.
+	 * Add PHM_PlatformCaps_NonABMSupportInPPLib
+	 * for re-direct ABM related request to DAL
+	 */
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ABM);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_NonABMSupportInPPLib);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DynamicACTiming);
+
+	fiji_initialize_power_tune_defaults(hwmgr);
+
+	data->mclk_stutter_mode_threshold = 60000;
+	data->pcie_gen_performance.max = PP_PCIEGen1;
+	data->pcie_gen_performance.min = PP_PCIEGen3;
+	data->pcie_gen_power_saving.max = PP_PCIEGen1;
+	data->pcie_gen_power_saving.min = PP_PCIEGen3;
+	data->pcie_lane_performance.max = 0;
+	data->pcie_lane_performance.min = 16;
+	data->pcie_lane_power_saving.max = 0;
+	data->pcie_lane_power_saving.min = 16;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DynamicUVDState);
+}
+
+static int fiji_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_voltage_lookup_table *lookup_table,
+	uint16_t virtual_voltage_id, int32_t *sclk)
+{
+	uint8_t entryId;
+	uint8_t voltageId;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -EINVAL);
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
+	for (entryId = 0; entryId < table_info->vdd_dep_on_sclk->count; entryId++) {
+		voltageId = table_info->vdd_dep_on_sclk->entries[entryId].vddInd;
+		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
+			break;
+	}
+
+	PP_ASSERT_WITH_CODE(entryId < table_info->vdd_dep_on_sclk->count,
+			"Can't find requested voltage id in vdd_dep_on_sclk table!",
+			return -EINVAL;
+			);
+
+	*sclk = table_info->vdd_dep_on_sclk->entries[entryId].clk;
+
+	return 0;
+}
+
+/**
+* Get Leakage VDDC based on leakage ID.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int fiji_get_evv_voltages(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint16_t    vv_id;
+	uint16_t    vddc = 0;
+	uint16_t    evv_default = 1150;
+	uint16_t    i, j;
+	uint32_t  sclk = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+	int result;
+
+	for (i = 0; i < FIJI_MAX_LEAKAGE_COUNT; i++) {
+		vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
+		if (!fiji_get_sclk_for_voltage_evv(hwmgr,
+				table_info->vddc_lookup_table, vv_id, &sclk)) {
+			if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_ClockStretcher)) {
+				for (j = 1; j < sclk_table->count; j++) {
+					if (sclk_table->entries[j].clk == sclk &&
+							sclk_table->entries[j].cks_enable == 0) {
+						sclk += 5000;
+						break;
+					}
+				}
+			}
+
+			if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_EnableDriverEVV))
+				result = atomctrl_calculate_voltage_evv_on_sclk(hwmgr,
+						VOLTAGE_TYPE_VDDC, sclk, vv_id, &vddc, i, true);
+			else
+				result = -EINVAL;
+
+			if (result)
+				result = atomctrl_get_voltage_evv_on_sclk(hwmgr,
+						VOLTAGE_TYPE_VDDC, sclk,vv_id, &vddc);
+
+			/* need to make sure vddc is less than 2v or else, it could burn the ASIC. */
+			PP_ASSERT_WITH_CODE((vddc < 2000),
+					"Invalid VDDC value, greater than 2v!", result = -EINVAL;);
+
+			if (result)
+				/* 1.15V is the default safe value for Fiji */
+				vddc = evv_default;
+
+			/* the voltage should not be zero nor equal to leakage ID */
+			if (vddc != 0 && vddc != vv_id) {
+				data->vddc_leakage.actual_voltage
+				[data->vddc_leakage.count] = vddc;
+				data->vddc_leakage.leakage_id
+				[data->vddc_leakage.count] = vv_id;
+				data->vddc_leakage.count++;
+			}
+		}
+	}
+	return 0;
+}
+
+/**
+ * Change virtual leakage voltage to actual value.
+ *
+ * @param     hwmgr  the address of the powerplay hardware manager.
+ * @param     pointer to changing voltage
+ * @param     pointer to leakage table
+ */
+static void fiji_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
+		uint16_t *voltage, struct fiji_leakage_voltage *leakage_table)
+{
+	uint32_t index;
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
+	for (index = 0; index < leakage_table->count; index++) {
+		/* if this voltage matches a leakage voltage ID */
+		/* patch with actual leakage voltage */
+		if (leakage_table->leakage_id[index] == *voltage) {
+			*voltage = leakage_table->actual_voltage[index];
+			break;
+		}
+	}
+
+	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
+		printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
+}
+
+/**
+* Patch voltage lookup table by EVV leakages.
+*
+* @param     hwmgr  the address of the powerplay hardware manager.
+* @param     pointer to voltage lookup table
+* @param     pointer to leakage table
+* @return     always 0
+*/
+static int fiji_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *lookup_table,
+		struct fiji_leakage_voltage *leakage_table)
+{
+	uint32_t i;
+
+	for (i = 0; i < lookup_table->count; i++)
+		fiji_patch_with_vdd_leakage(hwmgr,
+				&lookup_table->entries[i].us_vdd, leakage_table);
+
+	return 0;
+}
+
+static int fiji_patch_clock_voltage_limits_with_vddc_leakage(
+		struct pp_hwmgr *hwmgr, struct fiji_leakage_voltage *leakage_table,
+		uint16_t *vddc)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	fiji_patch_with_vdd_leakage(hwmgr, (uint16_t *)vddc, leakage_table);
+	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
+			table_info->max_clock_voltage_on_dc.vddc;
+	return 0;
+}
+
+static int fiji_patch_voltage_dependency_tables_with_lookup_table(
+		struct pp_hwmgr *hwmgr)
+{
+	uint8_t entryId;
+	uint8_t voltageId;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table =
+			table_info->vdd_dep_on_mclk;
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	for (entryId = 0; entryId < sclk_table->count; ++entryId) {
+		voltageId = sclk_table->entries[entryId].vddInd;
+		sclk_table->entries[entryId].vddc =
+				table_info->vddc_lookup_table->entries[voltageId].us_vdd;
+	}
+
+	for (entryId = 0; entryId < mclk_table->count; ++entryId) {
+		voltageId = mclk_table->entries[entryId].vddInd;
+		mclk_table->entries[entryId].vddc =
+			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
+	}
+
+	for (entryId = 0; entryId < mm_table->count; ++entryId) {
+		voltageId = mm_table->entries[entryId].vddcInd;
+		mm_table->entries[entryId].vddc =
+			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
+	}
+
+	return 0;
+
+}
+
+static int fiji_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	/* Need to determine if we need calculated voltage. */
+	return 0;
+}
+
+static int fiji_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
+{
+	/* Need to determine if we need calculated voltage from mm table. */
+	return 0;
+}
+
+static int fiji_sort_lookup_table(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_voltage_lookup_table *lookup_table)
+{
+	uint32_t table_size, i, j;
+	struct phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
+	table_size = lookup_table->count;
+
+	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
+		"Lookup table is empty", return -EINVAL);
+
+	/* Sorting voltages */
+	for (i = 0; i < table_size - 1; i++) {
+		for (j = i + 1; j > 0; j--) {
+			if (lookup_table->entries[j].us_vdd <
+					lookup_table->entries[j - 1].us_vdd) {
+				tmp_voltage_lookup_record = lookup_table->entries[j - 1];
+				lookup_table->entries[j - 1] = lookup_table->entries[j];
+				lookup_table->entries[j] = tmp_voltage_lookup_record;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_complete_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	int tmp_result;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	tmp_result = fiji_patch_lookup_table_with_leakage(hwmgr,
+			table_info->vddc_lookup_table, &(data->vddc_leakage));
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = fiji_patch_clock_voltage_limits_with_vddc_leakage(hwmgr,
+			&(data->vddc_leakage), &table_info->max_clock_voltage_on_dc.vddc);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = fiji_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = fiji_calc_voltage_dependency_tables(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = fiji_calc_mm_voltage_dependency_table(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = fiji_sort_lookup_table(hwmgr, table_info->vddc_lookup_table);
+	if(tmp_result)
+		result = tmp_result;
+
+	return result;
+}
+
+static int fiji_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
+			table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
+			table_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
+		"VDD dependency on SCLK table is missing. 	\
+		This table is mandatory", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
+		"VDD dependency on SCLK table has to have is missing. 	\
+		This table is mandatory", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
+		"VDD dependency on MCLK table is missing. 	\
+		This table is mandatory", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
+		"VDD dependency on MCLK table has to have is missing.	 \
+		This table is mandatory", return -EINVAL);
+
+	data->min_vddc_in_pptable = (uint16_t)allowed_sclk_vdd_table->entries[0].vddc;
+	data->max_vddc_in_pptable =	(uint16_t)allowed_sclk_vdd_table->
+			entries[allowed_sclk_vdd_table->count - 1].vddc;
+
+	table_info->max_clock_voltage_on_ac.sclk =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
+	table_info->max_clock_voltage_on_ac.mclk =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
+	table_info->max_clock_voltage_on_ac.vddc =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
+	table_info->max_clock_voltage_on_ac.vddci =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
+
+	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk =
+		table_info->max_clock_voltage_on_ac.sclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk =
+		table_info->max_clock_voltage_on_ac.mclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc =
+		table_info->max_clock_voltage_on_ac.vddc;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci =
+		table_info->max_clock_voltage_on_ac.vddci;
+
+	return 0;
+}
+
+static uint16_t fiji_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
+{
+	uint32_t speedCntl = 0;
+
+	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
+	speedCntl = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__PCIE,
+			ixPCIE_LC_SPEED_CNTL);
+	return((uint16_t)PHM_GET_FIELD(speedCntl,
+			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
+}
+
+static int fiji_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
+{
+	uint32_t link_width;
+
+	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
+	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
+			PCIE_LC_LINK_WIDTH_CNTL, LC_LINK_WIDTH_RD);
+
+	PP_ASSERT_WITH_CODE((7 >= link_width),
+			"Invalid PCIe lane width!", return 0);
+
+	return decode_pcie_lane_width(link_width);
+}
+
+/** Patch the Boot State to match VBIOS boot clocks and voltage.
+*
+* @param hwmgr Pointer to the hardware manager.
+* @param pPowerState The address of the PowerState instance being created.
+*
+*/
+static int fiji_patch_boot_state(struct pp_hwmgr *hwmgr,
+		struct pp_hw_power_state *hw_ps)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_power_state *ps = (struct fiji_power_state *)hw_ps;
+	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
+	uint16_t size;
+	uint8_t frev, crev;
+	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+
+	/* First retrieve the Boot clocks and VDDC from the firmware info table.
+	 * We assume here that fw_info is unchanged if this call fails.
+	 */
+	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
+			hwmgr->device, index,
+			&size, &frev, &crev);
+	if (!fw_info)
+		/* During a test, there is no firmware info table. */
+		return 0;
+
+	/* Patch the state. */
+	data->vbios_boot_state.sclk_bootup_value =
+			le32_to_cpu(fw_info->ulDefaultEngineClock);
+	data->vbios_boot_state.mclk_bootup_value =
+			le32_to_cpu(fw_info->ulDefaultMemoryClock);
+	data->vbios_boot_state.mvdd_bootup_value =
+			le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
+	data->vbios_boot_state.vddc_bootup_value =
+			le16_to_cpu(fw_info->usBootUpVDDCVoltage);
+	data->vbios_boot_state.vddci_bootup_value =
+			le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
+	data->vbios_boot_state.pcie_gen_bootup_value =
+			fiji_get_current_pcie_speed(hwmgr);
+	data->vbios_boot_state.pcie_lane_bootup_value =
+			(uint16_t)fiji_get_current_pcie_lane_number(hwmgr);
+
+	/* set boot power state */
+	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
+	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
+	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
+	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
+
+	return 0;
+}
+
+static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t i;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	bool stay_in_boot;
+	int result;
+
+	data->dll_default_on = false;
+	data->sram_end = SMC_RAM_END;
+
+	for (i = 0; i < SMU73_MAX_LEVELS_GRAPHICS; i++)
+		data->activity_target[i] = FIJI_AT_DFLT;
+
+	data->vddc_vddci_delta = VDDC_VDDCI_DELTA;
+
+	data->mclk_activity_target = PPFIJI_MCLK_TARGETACTIVITY_DFLT;
+	data->mclk_dpm0_activity_target = 0xa;
+
+	data->sclk_dpm_key_disabled = 0;
+	data->mclk_dpm_key_disabled = 0;
+	data->pcie_dpm_key_disabled = 0;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_UnTabledHardwareInterface);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TablelessHardwareInterface);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep);
+
+	data->gpio_debug = 0;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DynamicPatchPowerState);
+
+	/* need to set voltage control types before EVV patching */
+	data->voltage_control = FIJI_VOLTAGE_CONTROL_NONE;
+	data->vddci_control = FIJI_VOLTAGE_CONTROL_NONE;
+	data->mvdd_control = FIJI_VOLTAGE_CONTROL_NONE;
+
+	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+			VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
+		data->voltage_control = FIJI_VOLTAGE_CONTROL_BY_SVID2;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EnableMVDDControl))
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
+			data->mvdd_control = FIJI_VOLTAGE_CONTROL_BY_GPIO;
+
+	if (data->mvdd_control == FIJI_VOLTAGE_CONTROL_NONE)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EnableMVDDControl);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDCI)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
+			data->vddci_control = FIJI_VOLTAGE_CONTROL_BY_GPIO;
+		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
+			data->vddci_control = FIJI_VOLTAGE_CONTROL_BY_SVID2;
+	}
+
+	if (data->vddci_control == FIJI_VOLTAGE_CONTROL_NONE)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ControlVDDCI);
+
+	if (table_info && table_info->cac_dtp_table->usClockStretchAmount)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+
+	fiji_init_dpm_defaults(hwmgr);
+
+	/* Get leakage voltage based on leakage ID. */
+	fiji_get_evv_voltages(hwmgr);
+
+	/* Patch our voltage dependency table with actual leakage voltage
+	 * We need to perform leakage translation before it's used by other functions
+	 */
+	fiji_complete_dependency_tables(hwmgr);
+
+	/* Parse pptable data read from VBIOS */
+	fiji_set_private_data_based_on_pptable(hwmgr);
+
+	/* ULV Support */
+	data->ulv.ulv_supported = true; /* ULV feature is enabled by default */
+
+	/* Initalize Dynamic State Adjustment Rule Settings */
+	result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
+
+	if (!result) {
+		data->uvd_enabled = false;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_EnableSMU7ThermalManagement);
+		data->vddc_phase_shed_control = false;
+	}
+
+	stay_in_boot = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StayInBootState);
+
+	if (0 == result) {
+		struct cgs_system_info sys_info = {0};
+
+		data->is_tlu_enabled = 0;
+		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
+				FIJI_MAX_HARDWARE_POWERLEVELS;
+		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
+		hwmgr->platform_descriptor.minimumClocksReductionPercentage  = 50;
+
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_FanSpeedInTableIsRPM);
+
+		if (table_info->cac_dtp_table->usDefaultTargetOperatingTemp &&
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ucFanControlMode) {
+			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM =
+					hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
+			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM =
+					hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM;
+			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMinLimit =
+					table_info->cac_dtp_table->usOperatingTempMinLimit;
+			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMaxLimit =
+					table_info->cac_dtp_table->usOperatingTempMaxLimit;
+			hwmgr->dyn_state.cac_dtp_table->usDefaultTargetOperatingTemp =
+					table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
+			hwmgr->dyn_state.cac_dtp_table->usOperatingTempStep =
+					table_info->cac_dtp_table->usOperatingTempStep;
+			hwmgr->dyn_state.cac_dtp_table->usTargetOperatingTemp =
+					table_info->cac_dtp_table->usTargetOperatingTemp;
+
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_ODFuzzyFanControlSupport);
+		}
+
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_gen_cap = 0x30007;
+		else
+			data->pcie_gen_cap = (uint32_t)sys_info.value;
+		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
+			data->pcie_spc_cap = 20;
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_lane_cap = 0x2f0000;
+		else
+			data->pcie_lane_cap = (uint32_t)sys_info.value;
+	} else {
+		/* Ignore return value in here, we are cleaning up a mess. */
+		tonga_hwmgr_backend_fini(hwmgr);
+	}
+
+	return 0;
+}
+
+/**
+ * Read clock related registers.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int fiji_read_clock_registers(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	data->clock_registers.vCG_SPLL_FUNC_CNTL =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_FUNC_CNTL);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_2 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_FUNC_CNTL_2);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_3 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_FUNC_CNTL_3);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_4 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_FUNC_CNTL_4);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_SPREAD_SPECTRUM);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_SPLL_SPREAD_SPECTRUM_2);
+
+	return 0;
+}
+
+/**
+ * Find out if memory is GDDR5.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int fiji_get_memory_type(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t temp;
+
+	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
+
+	data->is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
+			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
+			 MC_SEQ_MISC0_GDDR5_SHIFT));
+
+	return 0;
+}
+
+/**
+ * Enables Dynamic Power Management by SMC
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int fiji_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			GENERAL_PWRMGT, STATIC_PM_EN, 1);
+
+	return 0;
+}
+
+/**
+ * Initialize PowerGating States for different engines
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int fiji_init_power_gate_state(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = false;
+	data->vce_power_gated = false;
+	data->samu_power_gated = false;
+	data->acp_power_gated = false;
+	data->pg_acp_init = true;
+
+	return 0;
+}
+
+static int fiji_init_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	data->low_sclk_interrupt_threshold = 0;
+
+	return 0;
+}
+
+static int fiji_setup_asic_task(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = fiji_read_clock_registers(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to read clock registers!", result = tmp_result);
+
+	tmp_result = fiji_get_memory_type(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to get memory type!", result = tmp_result);
+
+	tmp_result = fiji_enable_acpi_power_management(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable ACPI power management!", result = tmp_result);
+
+	tmp_result = fiji_init_power_gate_state(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to init power gate state!", result = tmp_result);
+
+	tmp_result = tonga_get_mc_microcode_version(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to get MC microcode version!", result = tmp_result);
+
+	tmp_result = fiji_init_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to init sclk threshold!", result = tmp_result);
+
+	return result;
+}
+
+/**
+* Checks if we want to support voltage control
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+*/
+static bool fiji_voltage_control(const struct pp_hwmgr *hwmgr)
+{
+	const struct fiji_hwmgr *data =
+			(const struct fiji_hwmgr *)(hwmgr->backend);
+
+	return (FIJI_VOLTAGE_CONTROL_NONE != data->voltage_control);
+}
+
+/**
+* Enable voltage control
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int fiji_enable_voltage_control(struct pp_hwmgr *hwmgr)
+{
+	/* enable voltage control */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
+
+	return 0;
+}
+
+/**
+* Remove repeated voltage values and create table with unique values.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    vol_table  the pointer to changing voltage table
+* @return    0 in success
+*/
+
+static int fiji_trim_voltage_table(struct pp_hwmgr *hwmgr,
+		struct pp_atomctrl_voltage_table *vol_table)
+{
+	uint32_t i, j;
+	uint16_t vvalue;
+	bool found = false;
+	struct pp_atomctrl_voltage_table *table;
+
+	PP_ASSERT_WITH_CODE((NULL != vol_table),
+			"Voltage Table empty.", return -EINVAL);
+	table = kzalloc(sizeof(struct pp_atomctrl_voltage_table),
+			GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	table->mask_low = vol_table->mask_low;
+	table->phase_delay = vol_table->phase_delay;
+
+	for (i = 0; i < vol_table->count; i++) {
+		vvalue = vol_table->entries[i].value;
+		found = false;
+
+		for (j = 0; j < table->count; j++) {
+			if (vvalue == table->entries[j].value) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			table->entries[table->count].value = vvalue;
+			table->entries[table->count].smio_low =
+					vol_table->entries[i].smio_low;
+			table->count++;
+		}
+	}
+
+	memcpy(vol_table, table, sizeof(struct pp_atomctrl_voltage_table));
+	kfree(table);
+
+	return 0;
+}
+
+static int fiji_get_svi2_mvdd_voltage_table(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
+{
+	uint32_t i;
+	int result;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_voltage_table *vol_table = &(data->mvdd_voltage_table);
+
+	PP_ASSERT_WITH_CODE((0 != dep_table->count),
+			"Voltage Dependency Table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+	vol_table->count = dep_table->count;
+
+	for (i = 0; i < dep_table->count; i++) {
+		vol_table->entries[i].value = dep_table->entries[i].mvdd;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	result = fiji_trim_voltage_table(hwmgr, vol_table);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to trim MVDD table.", return result);
+
+	return 0;
+}
+
+static int fiji_get_svi2_vddci_voltage_table(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
+{
+	uint32_t i;
+	int result;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_voltage_table *vol_table = &(data->vddci_voltage_table);
+
+	PP_ASSERT_WITH_CODE((0 != dep_table->count),
+			"Voltage Dependency Table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+	vol_table->count = dep_table->count;
+
+	for (i = 0; i < dep_table->count; i++) {
+		vol_table->entries[i].value = dep_table->entries[i].vddci;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	result = fiji_trim_voltage_table(hwmgr, vol_table);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to trim VDDCI table.", return result);
+
+	return 0;
+}
+
+static int fiji_get_svi2_vdd_voltage_table(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *lookup_table)
+{
+	int i = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_voltage_table *vol_table = &(data->vddc_voltage_table);
+
+	PP_ASSERT_WITH_CODE((0 != lookup_table->count),
+			"Voltage Lookup Table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+
+	vol_table->count = lookup_table->count;
+
+	for (i = 0; i < vol_table->count; i++) {
+		vol_table->entries[i].value = lookup_table->entries[i].us_vdd;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	return 0;
+}
+
+/* ---- Voltage Tables ----
+ * If the voltage table would be bigger than
+ * what will fit into the state table on
+ * the SMC keep only the higher entries.
+ */
+static void fiji_trim_voltage_table_to_fit_state_table(struct pp_hwmgr *hwmgr,
+		uint32_t max_vol_steps, struct pp_atomctrl_voltage_table *vol_table)
+{
+	unsigned int i, diff;
+
+	if (vol_table->count <= max_vol_steps)
+		return;
+
+	diff = vol_table->count - max_vol_steps;
+
+	for (i = 0; i < max_vol_steps; i++)
+		vol_table->entries[i] = vol_table->entries[i + diff];
+
+	vol_table->count = max_vol_steps;
+
+	return;
+}
+
+/**
+* Create Voltage Tables.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int fiji_construct_voltage_tables(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	int result;
+
+	if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+				VOLTAGE_TYPE_MVDDC,	VOLTAGE_OBJ_GPIO_LUT,
+				&(data->mvdd_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve MVDD table.",
+				return result);
+	} else if (FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		result = fiji_get_svi2_mvdd_voltage_table(hwmgr,
+				table_info->vdd_dep_on_mclk);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve SVI2 MVDD table from dependancy table.",
+				return result;);
+	}
+
+	if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT,
+				&(data->vddci_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve VDDCI table.",
+				return result);
+	} else if (FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		result = fiji_get_svi2_vddci_voltage_table(hwmgr,
+				table_info->vdd_dep_on_mclk);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve SVI2 VDDCI table from dependancy table.",
+				return result);
+	}
+
+	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		result = fiji_get_svi2_vdd_voltage_table(hwmgr,
+				table_info->vddc_lookup_table);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve SVI2 VDDC table from lookup table.",
+				return result);
+	}
+
+	PP_ASSERT_WITH_CODE(
+			(data->vddc_voltage_table.count <= (SMU73_MAX_LEVELS_VDDC)),
+			"Too many voltage values for VDDC. Trimming to fit state table.",
+			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
+					SMU73_MAX_LEVELS_VDDC, &(data->vddc_voltage_table)));
+
+	PP_ASSERT_WITH_CODE(
+			(data->vddci_voltage_table.count <= (SMU73_MAX_LEVELS_VDDCI)),
+			"Too many voltage values for VDDCI. Trimming to fit state table.",
+			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
+					SMU73_MAX_LEVELS_VDDCI, &(data->vddci_voltage_table)));
+
+	PP_ASSERT_WITH_CODE(
+			(data->mvdd_voltage_table.count <= (SMU73_MAX_LEVELS_MVDD)),
+			"Too many voltage values for MVDD. Trimming to fit state table.",
+			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
+					SMU73_MAX_LEVELS_MVDD, &(data->mvdd_voltage_table)));
+
+	return 0;
+}
+
+static int fiji_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	/* Program additional LP registers
+	 * that are no longer programmed by VBIOS
+	 */
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
+
+	return 0;
+}
+
+/**
+* Programs static screed detection parameters
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int fiji_program_static_screen_threshold_parameters(
+		struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/* Set static screen threshold unit */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
+			data->static_screen_threshold_unit);
+	/* Set static screen threshold */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
+			data->static_screen_threshold);
+
+	return 0;
+}
+
+/**
+* Setup display gap for glitch free memory clock switching.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+static int fiji_enable_display_gap(struct pp_hwmgr *hwmgr)
+{
+	uint32_t displayGap =
+			cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixCG_DISPLAY_GAP_CNTL);
+
+	displayGap = PHM_SET_FIELD(displayGap, CG_DISPLAY_GAP_CNTL,
+			DISP_GAP, DISPLAY_GAP_IGNORE);
+
+	displayGap = PHM_SET_FIELD(displayGap, CG_DISPLAY_GAP_CNTL,
+			DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_DISPLAY_GAP_CNTL, displayGap);
+
+	return 0;
+}
+
+/**
+* Programs activity state transition voting clients
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+static int fiji_program_voting_clients(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/* Clear reset for voting clients before enabling DPM */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
+
+	return 0;
+}
+
+/**
+* Get the location of various tables inside the FW image.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+static int fiji_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, DpmTable),
+			&tmp, data->sram_end);
+
+	if (0 == result)
+		data->dpm_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, SoftRegisters),
+			&tmp, data->sram_end);
+
+	if (!result) {
+		data->soft_regs_start = tmp;
+		smu_data->soft_regs_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, mcRegisterTable),
+			&tmp, data->sram_end);
+
+	if (!result)
+		data->mc_reg_table_start = tmp;
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, FanTable),
+			&tmp, data->sram_end);
+
+	if (!result)
+		data->fan_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, mcArbDramTimingTable),
+			&tmp, data->sram_end);
+
+	if (!result)
+		data->arb_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, Version),
+			&tmp, data->sram_end);
+
+	if (!result)
+		hwmgr->microcode_version_info.SMC = tmp;
+
+	error |= (0 != result);
+
+	return error ? -1 : 0;
+}
+
+/* Copy one arb setting to another and then switch the active set.
+ * arb_src and arb_dest is one of the MC_CG_ARB_FREQ_Fx constants.
+ */
+static int fiji_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
+		uint32_t arb_src, uint32_t arb_dest)
+{
+	uint32_t mc_arb_dram_timing;
+	uint32_t mc_arb_dram_timing2;
+	uint32_t burst_time;
+	uint32_t mc_cg_config;
+
+	switch (arb_src) {
+	case MC_CG_ARB_FREQ_F0:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (arb_dest) {
+	case MC_CG_ARB_FREQ_F0:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
+	mc_cg_config |= 0x0000000F;
+	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
+	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arb_dest);
+
+	return 0;
+}
+
+/**
+* Initial switch from ARB F0->F1
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+* This function is to be called from the SetPowerState table.
+*/
+static int fiji_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
+{
+	return fiji_copy_and_switch_arb_sets(hwmgr,
+			MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+static int fiji_reset_single_dpm_table(struct pp_hwmgr *hwmgr,
+		struct fiji_single_dpm_table *dpm_table, uint32_t count)
+{
+	int i;
+	PP_ASSERT_WITH_CODE(count <= MAX_REGULAR_DPM_NUMBER,
+			"Fatal error, can not set up single DPM table entries "
+			"to exceed max number!",);
+
+	dpm_table->count = count;
+	for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++)
+		dpm_table->dpm_levels[i].enabled = false;
+
+	return 0;
+}
+
+static void fiji_setup_pcie_table_entry(
+	struct fiji_single_dpm_table *dpm_table,
+	uint32_t index, uint32_t pcie_gen,
+	uint32_t pcie_lanes)
+{
+	dpm_table->dpm_levels[index].value = pcie_gen;
+	dpm_table->dpm_levels[index].param1 = pcie_lanes;
+	dpm_table->dpm_levels[index].enabled = 1;
+}
+
+static int fiji_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	uint32_t i, max_entry;
+
+	PP_ASSERT_WITH_CODE((data->use_pcie_performance_levels ||
+			data->use_pcie_power_saving_levels), "No pcie performance levels!",
+			return -EINVAL);
+
+	if (data->use_pcie_performance_levels &&
+			!data->use_pcie_power_saving_levels) {
+		data->pcie_gen_power_saving = data->pcie_gen_performance;
+		data->pcie_lane_power_saving = data->pcie_lane_performance;
+	} else if (!data->use_pcie_performance_levels &&
+			data->use_pcie_power_saving_levels) {
+		data->pcie_gen_performance = data->pcie_gen_power_saving;
+		data->pcie_lane_performance = data->pcie_lane_power_saving;
+	}
+
+	fiji_reset_single_dpm_table(hwmgr,
+			&data->dpm_table.pcie_speed_table, SMU73_MAX_LEVELS_LINK);
+
+	if (pcie_table != NULL) {
+		/* max_entry is used to make sure we reserve one PCIE level
+		 * for boot level (fix for A+A PSPP issue).
+		 * If PCIE table from PPTable have ULV entry + 8 entries,
+		 * then ignore the last entry.*/
+		max_entry = (SMU73_MAX_LEVELS_LINK < pcie_table->count) ?
+				SMU73_MAX_LEVELS_LINK : pcie_table->count;
+		for (i = 1; i < max_entry; i++) {
+			fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i - 1,
+					get_pcie_gen_support(data->pcie_gen_cap,
+							pcie_table->entries[i].gen_speed),
+					get_pcie_lane_support(data->pcie_lane_cap,
+							pcie_table->entries[i].lane_width));
+		}
+		data->dpm_table.pcie_speed_table.count = max_entry - 1;
+	} else {
+		/* Hardcode Pcie Table */
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Min_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Min_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+
+		data->dpm_table.pcie_speed_table.count = 6;
+	}
+	/* Populate last level for boot PCIE level, but do not increment count. */
+	fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
+			data->dpm_table.pcie_speed_table.count,
+			get_pcie_gen_support(data->pcie_gen_cap,
+					PP_Min_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap,
+					PP_Max_PCIELane));
+
+	return 0;
+}
+
+/*
+ * This function is to initalize all DPM state tables
+ * for SMU7 based on the dependency table.
+ * Dynamic state patching function will then trim these
+ * state tables to the allowed range based
+ * on the power policy or external client requests,
+ * such as UVD request, etc.
+ */
+static int fiji_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i;
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table =
+			table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
+			table_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(dep_sclk_table != NULL,
+			"SCLK dependency table is missing. This table is mandatory",
+			return -EINVAL);
+	PP_ASSERT_WITH_CODE(dep_sclk_table->count >= 1,
+			"SCLK dependency table has to have is missing. "
+			"This table is mandatory",
+			return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(dep_mclk_table != NULL,
+			"MCLK dependency table is missing. This table is mandatory",
+			return -EINVAL);
+	PP_ASSERT_WITH_CODE(dep_mclk_table->count >= 1,
+			"MCLK dependency table has to have is missing. "
+			"This table is mandatory",
+			return -EINVAL);
+
+	/* clear the state table to reset everything to default */
+	fiji_reset_single_dpm_table(hwmgr,
+			&data->dpm_table.sclk_table, SMU73_MAX_LEVELS_GRAPHICS);
+	fiji_reset_single_dpm_table(hwmgr,
+			&data->dpm_table.mclk_table, SMU73_MAX_LEVELS_MEMORY);
+
+	/* Initialize Sclk DPM table based on allow Sclk values */
+	data->dpm_table.sclk_table.count = 0;
+	for (i = 0; i < dep_sclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.sclk_table.dpm_levels
+				[data->dpm_table.sclk_table.count - 1].value !=
+						dep_sclk_table->entries[i].clk) {
+			data->dpm_table.sclk_table.dpm_levels
+			[data->dpm_table.sclk_table.count].value =
+					dep_sclk_table->entries[i].clk;
+			data->dpm_table.sclk_table.dpm_levels
+			[data->dpm_table.sclk_table.count].enabled =
+					(i == 0) ? true : false;
+			data->dpm_table.sclk_table.count++;
+		}
+	}
+
+	/* Initialize Mclk DPM table based on allow Mclk values */
+	data->dpm_table.mclk_table.count = 0;
+	for (i=0; i<dep_mclk_table->count; i++) {
+		if ( i==0 || data->dpm_table.mclk_table.dpm_levels
+				[data->dpm_table.mclk_table.count - 1].value !=
+						dep_mclk_table->entries[i].clk) {
+			data->dpm_table.mclk_table.dpm_levels
+			[data->dpm_table.mclk_table.count].value =
+					dep_mclk_table->entries[i].clk;
+			data->dpm_table.mclk_table.dpm_levels
+			[data->dpm_table.mclk_table.count].enabled =
+					(i == 0) ? true : false;
+			data->dpm_table.mclk_table.count++;
+		}
+	}
+
+	/* setup PCIE gen speed levels */
+	fiji_setup_default_pcie_table(hwmgr);
+
+	/* save a copy of the default DPM table */
+	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
+			sizeof(struct fiji_dpm_table));
+
+	return 0;
+}
+
+/**
+ * @brief PhwFiji_GetVoltageOrder
+ *  Returns index of requested voltage record in lookup(table)
+ * @param lookup_table - lookup list to search in
+ * @param voltage - voltage to look for
+ * @return 0 on success
+ */
+uint8_t fiji_get_voltage_index(
+		struct phm_ppt_v1_voltage_lookup_table *lookup_table, uint16_t voltage)
+{
+	uint8_t count = (uint8_t) (lookup_table->count);
+	uint8_t i;
+
+	PP_ASSERT_WITH_CODE((NULL != lookup_table),
+			"Lookup Table empty.", return 0);
+	PP_ASSERT_WITH_CODE((0 != count),
+			"Lookup Table empty.", return 0);
+
+	for (i = 0; i < lookup_table->count; i++) {
+		/* find first voltage equal or bigger than requested */
+		if (lookup_table->entries[i].us_vdd >= voltage)
+			return i;
+	}
+	/* voltage is bigger than max voltage in the table */
+	return i - 1;
+}
+
+/**
+* Preparation of vddc and vddgfx CAC tables for SMC.
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    table  the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index;
+	int result = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
+			table_info->vddc_lookup_table;
+	/* tables is already swapped, so in order to use the value from it,
+	 * we need to swap it back.
+	 * We are populating vddc CAC data to BapmVddc table
+	 * in split and merged mode
+	 */
+	for( count = 0; count<lookup_table->count; count++) {
+		index = fiji_get_voltage_index(lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] = (uint8_t) ((6200 -
+				(lookup_table->entries[index].us_cac_low *
+						VOLTAGE_SCALE)) / 25);
+		table->BapmVddcVidHiSidd[count] = (uint8_t) ((6200 -
+				(lookup_table->entries[index].us_cac_high *
+						VOLTAGE_SCALE)) / 25);
+	}
+
+	return result;
+}
+
+/**
+* Preparation of voltage tables for SMC.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always  0
+*/
+
+int fiji_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result;
+
+	result = fiji_populate_cac_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate CAC voltage tables to SMC",
+			return -EINVAL);
+
+	return 0;
+}
+
+static int fiji_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_Ulv *state)
+{
+	int result = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
+	state->VddcOffsetVid = (uint8_t)( table_info->us_ulv_voltage_offset *
+			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1 );
+
+	state->VddcPhase = (data->vddc_phase_shed_control) ? 0 : 1;
+
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+		CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+	}
+	return result;
+}
+
+static int fiji_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	return fiji_populate_ulv_level(hwmgr, &table->Ulv);
+}
+
+static int32_t fiji_get_dpm_level_enable_mask_value(
+		struct fiji_single_dpm_table* dpm_table)
+{
+	int32_t i;
+	int32_t mask = 0;
+
+	for (i = dpm_table->count; i > 0; i--) {
+		mask = mask << 1;
+		if (dpm_table->dpm_levels[i - 1].enabled)
+			mask |= 0x1;
+		else
+			mask &= 0xFFFFFFFE;
+	}
+	return mask;
+}
+
+static int fiji_populate_smc_link_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_dpm_table *dpm_table = &data->dpm_table;
+	int i;
+
+	/* Index (dpm_table->pcie_speed_table.count)
+	 * is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
+				dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity = 1;
+		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
+	}
+
+	data->smc_state_table.LinkLevelCount =
+			(uint8_t)dpm_table->pcie_speed_table.count;
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+/**
+* Calculates the SCLK dividers using the provided engine clock
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    clock  the engine clock to use to populate the structure
+* @param    sclk   the SMC SCLK structure to be populated
+*/
+static int fiji_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU73_Discrete_GraphicsLevel *sclk)
+{
+	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t ref_clock;
+	uint32_t ref_divider;
+	uint32_t fbdiv;
+	int result;
+
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, clock,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error retrieving Engine Clock dividers from VBIOS.",
+			return result);
+
+	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref. */
+	ref_clock = atomctrl_get_reference_clock(hwmgr);
+	ref_divider = 1 + dividers.uc_pll_ref_div;
+
+	/* low 14 bits is fraction and high 12 bits is divider */
+	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
+
+	/* SPLL_FUNC_CNTL setup */
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_REF_DIV, dividers.uc_pll_ref_div);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_PDIV_A,  dividers.uc_pll_post_div);
+
+	/* SPLL_FUNC_CNTL_3 setup*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
+			SPLL_FB_DIV, fbdiv);
+
+	/* set to use fractional accumulation*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
+			SPLL_DITHEN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
+		struct pp_atomctrl_internal_ss_info ssInfo;
+
+		uint32_t vco_freq = clock * dividers.uc_pll_post_div;
+		if (!atomctrl_get_engine_clock_spread_spectrum(hwmgr,
+				vco_freq, &ssInfo)) {
+			/*
+			 * ss_info.speed_spectrum_percentage -- in unit of 0.01%
+			 * ss_info.speed_spectrum_rate -- in unit of khz
+			 *
+			 * clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2
+			 */
+			uint32_t clk_s = ref_clock * 5 /
+					(ref_divider * ssInfo.speed_spectrum_rate);
+			/* clkv = 2 * D * fbdiv / NS */
+			uint32_t clk_v = 4 * ssInfo.speed_spectrum_percentage *
+					fbdiv / (clk_s * 10000);
+
+			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
+					CG_SPLL_SPREAD_SPECTRUM, CLKS, clk_s);
+			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
+					CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
+			cg_spll_spread_spectrum_2 = PHM_SET_FIELD(cg_spll_spread_spectrum_2,
+					CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clk_v);
+		}
+	}
+
+	sclk->SclkFrequency        = clock;
+	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
+	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
+	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
+	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
+	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
+
+	return 0;
+}
+
+static uint16_t fiji_find_closest_vddci(struct pp_hwmgr *hwmgr, uint16_t vddci)
+{
+	uint32_t  i;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_voltage_table *vddci_table =
+			&(data->vddci_voltage_table);
+
+	for (i = 0; i < vddci_table->count; i++) {
+		if (vddci_table->entries[i].value >= vddci)
+			return vddci_table->entries[i].value;
+	}
+
+	PP_ASSERT_WITH_CODE(false,
+			"VDDCI is larger than max VDDCI in VDDCI Voltage Table!",
+			return vddci_table->entries[i].value);
+}
+
+static int fiji_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_clock_voltage_dependency_table* dep_table,
+		uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
+{
+	uint32_t i;
+	uint16_t vddci;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	*voltage = *mvdd = 0;
+
+	/* clock - voltage dependency table is empty table */
+	if (dep_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < dep_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (dep_table->entries[i].clk >= clock) {
+			*voltage |= (dep_table->entries[i].vddc *
+					VOLTAGE_SCALE) << VDDC_SHIFT;
+			if (FIJI_VOLTAGE_CONTROL_NONE == data->vddci_control)
+				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else if (dep_table->entries[i].vddci)
+				*voltage |= (dep_table->entries[i].vddci *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else {
+				vddci = fiji_find_closest_vddci(hwmgr,
+						(dep_table->entries[i].vddc -
+								(uint16_t)data->vddc_vddci_delta));
+				*voltage |= (vddci * VOLTAGE_SCALE) <<	VDDCI_SHIFT;
+			}
+
+			if (FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
+					VOLTAGE_SCALE;
+			else if (dep_table->entries[i].mvdd)
+				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
+					VOLTAGE_SCALE;
+
+			*voltage |= 1 << PHASES_SHIFT;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+	if (FIJI_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+				VOLTAGE_SCALE) << VDDCI_SHIFT;
+	else if (dep_table->entries[i-1].vddci) {
+		vddci = fiji_find_closest_vddci(hwmgr,
+				(dep_table->entries[i].vddc -
+						(uint16_t)data->vddc_vddci_delta));
+		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+	}
+
+	if (FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
+	else if (dep_table->entries[i].mvdd)
+		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
+
+	return 0;
+}
+/**
+* Populates single SMC SCLK structure using the provided engine clock
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    clock the engine clock to use to populate the structure
+* @param    sclk        the SMC SCLK structure to be populated
+*/
+
+static int fiji_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, uint16_t sclk_al_threshold,
+		struct SMU73_Discrete_GraphicsLevel *level)
+{
+	int result;
+	/* PP_Clocks minClocks; */
+	uint32_t threshold, mvdd;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	result = fiji_calculate_sclk_params(hwmgr, clock, level);
+
+	/* populate graphics levels */
+	result = fiji_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk, clock,
+			&level->MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"can not find VDDC voltage value for "
+			"VDDC engine clock dependency table",
+			return result);
+
+	level->SclkFrequency = clock;
+	level->ActivityLevel = sclk_al_threshold;
+	level->CcPwrDynRm = 0;
+	level->CcPwrDynRm1 = 0;
+	level->EnabledForActivity = 0;
+	level->EnabledForThrottle = 1;
+	level->UpHyst = 10;
+	level->DownHyst = 0;
+	level->VoltageDownHyst = 0;
+	level->PowerThrottle = 0;
+
+	threshold = clock * data->fast_watermark_threshold / 100;
+
+	/*
+	* TODO: get minimum clocks from dal configaration
+	* PECI_GetMinClockSettings(hwmgr->pPECI, &minClocks);
+	*/
+	/* data->DisplayTiming.minClockInSR = minClocks.engineClockInSR; */
+
+	/* get level->DeepSleepDivId
+	if (phm_cap_enabled(hwmgr->platformDescriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
+	{
+	level->DeepSleepDivId = PhwFiji_GetSleepDividerIdFromClock(hwmgr, clock, minClocks.engineClockInSR);
+	} */
+
+	/* Default to slow, highest DPM level will be
+	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
+	 */
+	level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
+
+	return 0;
+}
+/**
+* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+static int fiji_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_dpm_table *dpm_table = &data->dpm_table;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	uint8_t pcie_entry_cnt = (uint8_t) data->dpm_table.pcie_speed_table.count;
+	int result = 0;
+	uint32_t array = data->dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, GraphicsLevel);
+	uint32_t array_size = sizeof(struct SMU73_Discrete_GraphicsLevel) *
+			SMU73_MAX_LEVELS_GRAPHICS;
+	struct SMU73_Discrete_GraphicsLevel *levels =
+			data->smc_state_table.GraphicsLevel;
+	uint32_t i, max_entry;
+	uint8_t hightest_pcie_level_enabled = 0,
+			lowest_pcie_level_enabled = 0,
+			mid_pcie_level_enabled = 0,
+			count = 0;
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		result = fiji_populate_single_graphic_level(hwmgr,
+				dpm_table->sclk_table.dpm_levels[i].value,
+				(uint16_t)data->activity_target[i],
+				&levels[i]);
+		if (result)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			levels[i].DeepSleepDivId = 0;
+	}
+
+	/* Only enable level 0 for now.*/
+	levels[0].EnabledForActivity = 1;
+
+	/* set highest level watermark to high */
+	levels[dpm_table->sclk_table.count - 1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	data->smc_state_table.GraphicsDpmLevelCount =
+			(uint8_t)dpm_table->sclk_table.count;
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
+				"There must be 1 or more PCIE levels defined in PPTable.",
+				return -EINVAL);
+		max_entry = pcie_entry_cnt - 1;
+		for (i = 0; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel =
+					(uint8_t) ((i < max_entry)? i : max_entry);
+	} else {
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (hightest_pcie_level_enabled + 1))) != 0 ))
+			hightest_pcie_level_enabled++;
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << lowest_pcie_level_enabled)) == 0 ))
+			lowest_pcie_level_enabled++;
+
+		while ((count < hightest_pcie_level_enabled) &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0 ))
+			count++;
+
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1+ count) <
+				hightest_pcie_level_enabled?
+						(lowest_pcie_level_enabled + 1 + count) :
+						hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to hightest_pcie_level_enabled */
+		for(i = 2; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled */
+		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled */
+		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change */
+	result = fiji_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, data->sram_end);
+
+	return result;
+}
+
+/**
+ * MCLK Frequency Ratio
+ * SEQ_CG_RESP  Bit[31:24] - 0x0
+ * Bit[27:24] \96 DDR3 Frequency ratio
+ * 0x0 <= 100MHz,       450 < 0x8 <= 500MHz
+ * 100 < 0x1 <= 150MHz,       500 < 0x9 <= 550MHz
+ * 150 < 0x2 <= 200MHz,       550 < 0xA <= 600MHz
+ * 200 < 0x3 <= 250MHz,       600 < 0xB <= 650MHz
+ * 250 < 0x4 <= 300MHz,       650 < 0xC <= 700MHz
+ * 300 < 0x5 <= 350MHz,       700 < 0xD <= 750MHz
+ * 350 < 0x6 <= 400MHz,       750 < 0xE <= 800MHz
+ * 400 < 0x7 <= 450MHz,       800 < 0xF
+ */
+static uint8_t fiji_get_mclk_frequency_ratio(uint32_t mem_clock)
+{
+	if (mem_clock <= 10000) return 0x0;
+	if (mem_clock <= 15000) return 0x1;
+	if (mem_clock <= 20000) return 0x2;
+	if (mem_clock <= 25000) return 0x3;
+	if (mem_clock <= 30000) return 0x4;
+	if (mem_clock <= 35000) return 0x5;
+	if (mem_clock <= 40000) return 0x6;
+	if (mem_clock <= 45000) return 0x7;
+	if (mem_clock <= 50000) return 0x8;
+	if (mem_clock <= 55000) return 0x9;
+	if (mem_clock <= 60000) return 0xa;
+	if (mem_clock <= 65000) return 0xb;
+	if (mem_clock <= 70000) return 0xc;
+	if (mem_clock <= 75000) return 0xd;
+	if (mem_clock <= 80000) return 0xe;
+	/* mem_clock > 800MHz */
+	return 0xf;
+}
+
+/**
+* Populates the SMC MCLK structure using the provided memory clock
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    clock   the memory clock to use to populate the structure
+* @param    sclk    the SMC SCLK structure to be populated
+*/
+static int fiji_calculate_mclk_params(struct pp_hwmgr *hwmgr,
+    uint32_t clock, struct SMU73_Discrete_MemoryLevel *mclk)
+{
+	struct pp_atomctrl_memory_clock_param mem_param;
+	int result;
+
+	result = atomctrl_get_memory_pll_dividers_vi(hwmgr, clock, &mem_param);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to get Memory PLL Dividers.",);
+
+	/* Save the result data to outpupt memory level structure */
+	mclk->MclkFrequency   = clock;
+	mclk->MclkDivider     = (uint8_t)mem_param.mpll_post_divider;
+	mclk->FreqRange       = fiji_get_mclk_frequency_ratio(clock);
+
+	return result;
+}
+
+static int fiji_populate_single_memory_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU73_Discrete_MemoryLevel *mem_level)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+
+	if (table_info->vdd_dep_on_mclk) {
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk, clock,
+				&mem_level->MinVoltage, &mem_level->MinMvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find MinVddc voltage value from memory "
+				"VDDC voltage dependency table", return result);
+	}
+
+	mem_level->EnabledForThrottle = 1;
+	mem_level->EnabledForActivity = 0;
+	mem_level->UpHyst = 0;
+	mem_level->DownHyst = 100;
+	mem_level->VoltageDownHyst = 0;
+	mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	mem_level->StutterEnable = false;
+
+	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	/* enable stutter mode if all the follow condition applied
+	 * PECI_GetNumberOfActiveDisplays(hwmgr->pPECI,
+	 * &(data->DisplayTiming.numExistingDisplays));
+	 */
+	data->display_timing.num_existing_displays = 1;
+
+	if ((data->mclk_stutter_mode_threshold) &&
+		(clock <= data->mclk_stutter_mode_threshold) &&
+		(!data->is_uvd_enabled) &&
+		(PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
+				STUTTER_ENABLE) & 0x1))
+		mem_level->StutterEnable = true;
+
+	result = fiji_calculate_mclk_params(hwmgr, clock, mem_level);
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
+	}
+	return result;
+}
+
+/**
+* Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+static int fiji_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_dpm_table *dpm_table = &data->dpm_table;
+	int result;
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t array = data->dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, MemoryLevel);
+	uint32_t array_size = sizeof(SMU73_Discrete_MemoryLevel) *
+			SMU73_MAX_LEVELS_MEMORY;
+	struct SMU73_Discrete_MemoryLevel *levels =
+			data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+				"can not populate memory level as memory clock is zero",
+				return -EINVAL);
+		result = fiji_populate_single_memory_level(hwmgr,
+				dpm_table->mclk_table.dpm_levels[i].value,
+				&levels[i]);
+		if (result)
+			return result;
+	}
+
+	/* Only enable level 0 for now. */
+	levels[0].EnabledForActivity = 1;
+
+	/* in order to prevent MC activity from stutter mode to push DPM up.
+	 * the UVD change complements this by putting the MCLK in
+	 * a higher state by default such that we are not effected by
+	 * up threshold or and MCLK DPM latency.
+	 */
+	levels[0].ActivityLevel = (uint16_t)data->mclk_dpm0_activity_target;
+	CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel);
+
+	data->smc_state_table.MemoryDpmLevelCount =
+			(uint8_t)dpm_table->mclk_table.count;
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+	/* set highest level watermark to high */
+	levels[dpm_table->mclk_table.count - 1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change */
+	result = fiji_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, data->sram_end);
+
+	return result;
+}
+
+/**
+* Populates the SMC MVDD structure using the provided memory clock.
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
+* @param    voltage     the SMC VOLTAGE structure to be populated
+*/
+int fiji_populate_mvdd_value(struct pp_hwmgr *hwmgr,
+		uint32_t mclk, SMIO_Pattern *smio_pat)
+{
+	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (FIJI_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
+				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
+				"MVDD Voltage is outside the supported range.",
+				return -EINVAL);
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int fiji_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = 0;
+	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	SMIO_Pattern vol_level;
+	uint32_t mvdd;
+	uint16_t us_mvdd;
+	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
+
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	if (!data->sclk_dpm_key_disabled) {
+		/* Get MinVoltage and Frequency from DPM0,
+		 * already converted to SMC_UL */
+		table->ACPILevel.SclkFrequency =
+				data->dpm_table.sclk_table.dpm_levels[0].value;
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_sclk,
+				table->ACPILevel.SclkFrequency,
+				&table->ACPILevel.MinVoltage, &mvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Cannot find ACPI VDDC voltage value "
+				"in Clock Dependency Table",);
+	} else {
+		table->ACPILevel.SclkFrequency =
+				data->vbios_boot_state.sclk_bootup_value;
+		table->ACPILevel.MinVoltage =
+				data->vbios_boot_state.vddc_bootup_value * VOLTAGE_SCALE;
+	}
+
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
+			table->ACPILevel.SclkFrequency,  &dividers);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error retrieving Engine Clock dividers from VBIOS.",
+			return result);
+
+	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
+	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+	table->ACPILevel.DeepSleepDivId = 0;
+
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_PWRON, 0);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_RESET, 1);
+	spll_func_cntl_2 = PHM_SET_FIELD(spll_func_cntl_2, CG_SPLL_FUNC_CNTL_2,
+			SCLK_MUX_SEL, 4);
+
+	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
+	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
+	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	if (!data->mclk_dpm_key_disabled) {
+		/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
+		table->MemoryACPILevel.MclkFrequency =
+				data->dpm_table.mclk_table.dpm_levels[0].value;
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk,
+				table->MemoryACPILevel.MclkFrequency,
+				&table->MemoryACPILevel.MinVoltage, &mvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Cannot find ACPI VDDCI voltage value "
+				"in Clock Dependency Table",);
+	} else {
+		table->MemoryACPILevel.MclkFrequency =
+				data->vbios_boot_state.mclk_bootup_value;
+		table->MemoryACPILevel.MinVoltage =
+				data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE;
+	}
+
+	us_mvdd = 0;
+	if ((FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
+			(data->mclk_dpm_key_disabled))
+		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
+	else {
+		if (!fiji_populate_mvdd_value(hwmgr,
+				data->dpm_table.mclk_table.dpm_levels[0].value,
+				&vol_level))
+			us_mvdd = vol_level.Voltage;
+	}
+
+	table->MemoryACPILevel.MinMvdd =
+			PP_HOST_TO_SMC_UL(us_mvdd * VOLTAGE_SCALE);
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	table->MemoryACPILevel.ActivityLevel =
+			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	table->MemoryACPILevel.StutterEnable = false;
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
+
+	return result;
+}
+
+static int fiji_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	table->VceLevelCount = (uint8_t)(mm_table->count);
+	table->VceBootLevel = 0;
+
+	for(count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage |=
+				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->VceLevel[count].MinVoltage |=
+				((mm_table->entries[count].vddc - data->vddc_vddci_delta) *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/*retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for VCE engine clock",
+				return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	table->AcpLevelCount = (uint8_t)(mm_table->count);
+	table->AcpBootLevel = 0;
+
+	for (count = 0; count < table->AcpLevelCount; count++) {
+		table->AcpLevel[count].Frequency = mm_table->entries[count].aclk;
+		table->AcpLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->AcpLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->AcpLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->AcpLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for engine clock", return result);
+
+		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t)(mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->SamuLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
+		int32_t eng_clock, int32_t mem_clock,
+		struct SMU73_Discrete_MCArbDramTimingTableEntry *arb_regs)
+{
+	uint32_t dram_timing;
+	uint32_t dram_timing2;
+	uint32_t burstTime;
+	ULONG state, trrds, trrdl;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+			eng_clock, mem_clock);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burstTime = cgs_read_register(hwmgr->device, mmMC_ARB_BURST_TIME);
+
+	state = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, STATE0);
+	trrds = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDS0);
+	trrdl = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDL0);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
+	arb_regs->McArbBurstTime   = (uint8_t)burstTime;
+	arb_regs->TRRDS            = (uint8_t)trrds;
+	arb_regs->TRRDL            = (uint8_t)trrdl;
+
+	return 0;
+}
+
+static int fiji_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct SMU73_Discrete_MCArbDramTimingTable arb_regs;
+	uint32_t i, j;
+	int result = 0;
+
+	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
+			result = fiji_populate_memory_timing_parameters(hwmgr,
+					data->dpm_table.sclk_table.dpm_levels[i].value,
+					data->dpm_table.mclk_table.dpm_levels[j].value,
+					&arb_regs.entries[i][j]);
+			if (result)
+				break;
+		}
+	}
+
+	if (!result)
+		result = fiji_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				data->arb_table_start,
+				(uint8_t *)&arb_regs,
+				sizeof(SMU73_Discrete_MCArbDramTimingTable),
+				data->sram_end);
+	return result;
+}
+
+static int fiji_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	table->UvdLevelCount = (uint8_t)(mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->UvdLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].VclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Vclk clock", return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Dclk clock", return result);
+
+		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
+
+	}
+	return result;
+}
+
+static int fiji_find_boot_level(struct fiji_single_dpm_table *table,
+		uint32_t value, uint32_t *boot_level)
+{
+	int result = -EINVAL;
+	uint32_t i;
+
+	for (i = 0; i < table->count; i++) {
+		if (value == table->dpm_levels[i].value) {
+			*boot_level = i;
+			result = 0;
+		}
+	}
+	return result;
+}
+
+static int fiji_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table */
+	result = fiji_find_boot_level(&(data->dpm_table.sclk_table),
+			data->vbios_boot_state.sclk_bootup_value,
+			(uint32_t *)&(table->GraphicsBootLevel));
+
+	result = fiji_find_boot_level(&(data->dpm_table.mclk_table),
+			data->vbios_boot_state.mclk_bootup_value,
+			(uint32_t *)&(table->MemoryBootLevel));
+
+	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
+			VOLTAGE_SCALE;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return 0;
+}
+
+static int fiji_populate_smc_initailial_state(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint8_t count, level;
+
+	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
+	for (level = 0; level < count; level++) {
+		if(table_info->vdd_dep_on_sclk->entries[level].clk >=
+				data->vbios_boot_state.sclk_bootup_value) {
+			data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
+	for (level = 0; level < count; level++) {
+		if(table_info->vdd_dep_on_mclk->entries[level].clk >=
+				data->vbios_boot_state.mclk_bootup_value) {
+			data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t ro, efuse, efuse2, clock_freq, volt_without_cks,
+			volt_with_cks, value;
+	uint16_t clock_freq_u16;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint8_t type, i, j, cks_setting, stretch_amount, stretch_amount2,
+			volt_offset = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+
+	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
+
+	/* Read SMU_Eefuse to read and calculate RO and determine
+	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
+	 */
+	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (146 * 4));
+	efuse2 = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (148 * 4));
+	efuse &= 0xFF000000;
+	efuse = efuse >> 24;
+	efuse2 &= 0xF;
+
+	if (efuse2 == 1)
+		ro = (2300 - 1350) * efuse / 255 + 1350;
+	else
+		ro = (2500 - 1000) * efuse / 255 + 1000;
+
+	if (ro >= 1660)
+		type = 0;
+	else
+		type = 1;
+
+	/* Populate Stretch amount */
+	data->smc_state_table.ClockStretcherAmount = stretch_amount;
+
+	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
+	for (i = 0; i < sclk_table->count; i++) {
+		data->smc_state_table.Sclk_CKS_masterEn0_7 |=
+				sclk_table->entries[i].cks_enable << i;
+		volt_without_cks = (uint32_t)((14041 *
+			(sclk_table->entries[i].clk/100) / 10000 + 3571 + 75 - ro) * 1000 /
+			(4026 - (13924 * (sclk_table->entries[i].clk/100) / 10000)));
+		volt_with_cks = (uint32_t)((13946 *
+			(sclk_table->entries[i].clk/100) / 10000 + 3320 + 45 - ro) * 1000 /
+			(3664 - (11454 * (sclk_table->entries[i].clk/100) / 10000)));
+		if (volt_without_cks >= volt_with_cks)
+			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
+					sclk_table->entries[i].cks_voffset) * 100 / 625) + 1);
+		data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
+	}
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			STRETCH_ENABLE, 0x0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			staticEnable, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x0);
+
+	/* Populate CKS Lookup Table */
+	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
+		stretch_amount2 = 0;
+	else if (stretch_amount == 3 || stretch_amount == 4)
+		stretch_amount2 = 1;
+	else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+		PP_ASSERT_WITH_CODE(false,
+				"Stretch Amount in PPTable not supported\n",
+				return -EINVAL);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL);
+	value &= 0xFFC2FF87;
+	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][0];
+	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][1];
+	clock_freq_u16 = (uint16_t)(PP_SMC_TO_HOST_UL(data->smc_state_table.
+			GraphicsLevel[data->smc_state_table.GraphicsDpmLevelCount - 1].
+			SclkFrequency) / 100);
+	if (fiji_clock_stretcher_lookup_table[stretch_amount2][0] <
+			clock_freq_u16 &&
+	    fiji_clock_stretcher_lookup_table[stretch_amount2][1] >
+			clock_freq_u16) {
+		/* Program PWR_CKS_CNTL. CKS_USE_FOR_LOW_FREQ */
+		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 16;
+		/* Program PWR_CKS_CNTL. CKS_LDO_REFSEL */
+		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][2]) << 18;
+		/* Program PWR_CKS_CNTL. CKS_STRETCH_AMOUNT */
+		value |= (fiji_clock_stretch_amount_conversion
+				[fiji_clock_stretcher_lookup_table[stretch_amount2][3]]
+				 [stretch_amount]) << 3;
+	}
+	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].minFreq);
+	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].maxFreq);
+	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][2] & 0x7F;
+	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting |=
+			(fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 7;
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL, value);
+
+	/* Populate DDT Lookup Table */
+	for (i = 0; i < 4; i++) {
+		/* Assign the minimum and maximum VID stored
+		 * in the last row of Clock Stretcher Voltage Table.
+		 */
+		data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].minVID =
+				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][2];
+		data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].maxVID =
+				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][3];
+		/* Loop through each SCLK and check the frequency
+		 * to see if it lies within the frequency for clock stretcher.
+		 */
+		for (j = 0; j < data->smc_state_table.GraphicsDpmLevelCount; j++) {
+			cks_setting = 0;
+			clock_freq = PP_SMC_TO_HOST_UL(
+					data->smc_state_table.GraphicsLevel[j].SclkFrequency);
+			/* Check the allowed frequency against the sclk level[j].
+			 *  Sclk's endianness has already been converted,
+			 *  and it's in 10Khz unit,
+			 *  as opposed to Data table, which is in Mhz unit.
+			 */
+			if (clock_freq >=
+					(fiji_clock_stretcher_ddt_table[type][i][0]) * 100) {
+				cks_setting |= 0x2;
+				if (clock_freq <
+						(fiji_clock_stretcher_ddt_table[type][i][1]) * 100)
+					cks_setting |= 0x1;
+			}
+			data->smc_state_table.ClockStretcherDataTable.
+			ClockStretcherDataTableEntry[i].setting |= cks_setting << (j * 2);
+		}
+		CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.
+				ClockStretcherDataTable.
+				ClockStretcherDataTableEntry[i].setting);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
+	value &= 0xFFFFFFFE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
+
+	return 0;
+}
+
+/**
+* Populates the SMC VRConfig field in DPM table.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int fiji_populate_vr_config(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint16_t config;
+
+	config = VR_MERGED_WITH_VDDC;
+	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
+
+	/* Set Vddc Voltage Controller */
+	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= config;
+	} else {
+		PP_ASSERT_WITH_CODE(false,
+				"VDDC should be on SVI2 control in merged mode!",);
+	}
+	/* Set Vddci Voltage Controller */
+	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	}
+	/* Set Mvdd Voltage Controller */
+	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		config = VR_SVI2_PLANE_2;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	} else if(FIJI_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		config = VR_SMIO_PATTERN_2;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+/**
+* Initializes the SMC table and uploads it
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput  the pointer to input data (PowerState)
+* @return   always 0
+*/
+static int fiji_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct SMU73_Discrete_DpmTable *table = &(data->smc_state_table);
+	const struct fiji_ulv_parm *ulv = &(data->ulv);
+	uint8_t i;
+	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
+
+	result = fiji_setup_default_dpm_tables(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to setup default DPM tables!", return result);
+
+	if(FIJI_VOLTAGE_CONTROL_NONE != data->voltage_control)
+		fiji_populate_smc_voltage_tables(hwmgr, table);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	if (ulv->ulv_supported && table_info->us_ulv_voltage_offset) {
+		result = fiji_populate_ulv_state(hwmgr, table);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to initialize ULV state!", return result);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_ULV_PARAMETER, ulv->cg_ulv_parameter);
+	}
+
+	result = fiji_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Link Level!", return result);
+
+	result = fiji_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Graphics Level!", return result);
+
+	result = fiji_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Memory Level!", return result);
+
+	result = fiji_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ACPI Level!", return result);
+
+	result = fiji_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize VCE Level!", return result);
+
+	result = fiji_populate_smc_acp_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ACP Level!", return result);
+
+	result = fiji_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize SAMU Level!", return result);
+
+	/* Since only the initial state is completely set up at this point
+	 * (the other states are just copies of the boot state) we only
+	 * need to populate the  ARB settings for the initial state.
+	 */
+	result = fiji_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to Write ARB settings for the initial state.", return result);
+
+	result = fiji_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize UVD Level!", return result);
+
+	result = fiji_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot Level!", return result);
+
+	result = fiji_populate_smc_initailial_state(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot State!", return result);
+
+	result = fiji_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate BAPM Parameters!", return result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = fiji_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to populate Clock Stretcher Data Table!",
+				return result);
+	}
+
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+			table_info->cac_dtp_table->usTargetOperatingTemp *
+			FIJI_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow  =
+			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+			FIJI_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable = 1;
+	table->MemoryInterval = 1;
+	table->VoltageResponseTime = 0;
+	table->PhaseResponseTime = 0;
+	table->MemoryThermThrottleEnable = 1;
+	table->PCIeBootLinkLevel = 0;      /* 0:Gen1 1:Gen2 2:Gen3*/
+	table->PCIeGenInterval = 1;
+
+	result = fiji_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate VRConfig setting!", return result);
+
+	table->ThermGpio = 17;
+	table->SclkStepSize = 0x4000;
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
+		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	} else {
+		table->VRHotGpio = FIJI_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
+			&gpio_pin)) {
+		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	} else {
+		table->AcDcGpio = FIJI_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	/* Thermal Output GPIO */
+	if (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID,
+			&gpio_pin)) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
+
+		/* For porlarity read GPIOPAD_A with assigned Gpio pin
+		 * since VBIOS will program this register to set 'inactive state',
+		 * driver can then determine 'active state' from this and
+		 * program SMU with correct polarity
+		 */
+		table->ThermOutPolarity = (0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
+				(1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO */
+		if(phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot) &&
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_CombinePCCWithThermalSignal))
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+	} else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ThermalOutGPIO);
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	for (i = 0; i < SMU73_MAX_ENTRIES_SMIO; i++)
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = fiji_copy_bytes_to_smc(hwmgr->smumgr,
+			data->dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, SystemFlags),
+			(uint8_t *)&(table->SystemFlags),
+			sizeof(SMU73_Discrete_DpmTable) - 3 * sizeof(SMU73_PIDController),
+			data->sram_end);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to upload dpm data to SMC memory!", return result);
+
+	return 0;
+}
+
+/**
+* Initialize the ARB DRAM timing table's index field.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int fiji_init_arb_table_index(struct pp_hwmgr *hwmgr)
+{
+	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t tmp;
+	int result;
+
+	/* This is a read-modify-write on the first byte of the ARB table.
+	 * The first byte in the SMU73_Discrete_MCArbDramTimingTable structure
+	 * is the field 'current'.
+	 * This solution is ugly, but we never write the whole table only
+	 * individual fields in it.
+	 * In reality this field should not be in that structure
+	 * but in a soft register.
+	 */
+	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
+			data->arb_table_start, &tmp, data->sram_end);
+
+	if (result)
+		return result;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
+
+	return fiji_write_smc_sram_dword(hwmgr->smumgr,
+			data->arb_table_start,  tmp, data->sram_end);
+}
+
+static int fiji_enable_vrhot_gpio_interrupt(struct pp_hwmgr *hwmgr)
+{
+	if(phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_EnableVRHotGPIOInterrupt);
+
+	return 0;
+}
+
+static int fiji_enable_sclk_control(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+			SCLK_PWRMGT_OFF, 0);
+	return 0;
+}
+
+static int fiji_enable_ulv(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_ulv_parm *ulv = &(data->ulv);
+
+	if (ulv->ulv_supported)
+		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_EnableULV);
+
+	return 0;
+}
+
+static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep)) {
+		if (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MASTER_DeepSleep_ON))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to enable Master Deep Sleep switch failed!",
+					return -1);
+	} else {
+		if (smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to disable Master Deep Sleep switch failed!",
+					return -1);
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t   val, val0, val2;
+	uint32_t   i, cpl_cntl, cpl_threshold, mc_threshold;
+
+	/* enable SCLK dpm */
+	if(!data->sclk_dpm_key_disabled)
+		PP_ASSERT_WITH_CODE(
+		(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DPM_Enable)),
+		"Failed to enable SCLK DPM during DPM Start Function!",
+		return -1);
+
+	/* enable MCLK dpm */
+	if(0 == data->mclk_dpm_key_disabled) {
+		cpl_threshold = 0;
+		mc_threshold = 0;
+
+		/* Read per MCD tile (0 - 7) */
+		for (i = 0; i < 8; i++) {
+			PHM_WRITE_FIELD(hwmgr->device, MC_CONFIG_MCD, MC_RD_ENABLE, i);
+			val = cgs_read_register(hwmgr->device, mmMC_SEQ_RESERVE_0_S) & 0xf0000000;
+			if (0xf0000000 != val) {
+				/* count number of MCQ that has channel(s) enabled */
+				cpl_threshold++;
+				/* only harvest 3 or full 4 supported */
+				mc_threshold = val ? 3 : 4;
+			}
+		}
+		PP_ASSERT_WITH_CODE(0 != cpl_threshold,
+				"Number of MCQ is zero!", return -EINVAL;);
+
+		mc_threshold = ((mc_threshold & LCAC_MC0_CNTL__MC0_THRESHOLD_MASK) <<
+				LCAC_MC0_CNTL__MC0_THRESHOLD__SHIFT) |
+						LCAC_MC0_CNTL__MC0_ENABLE_MASK;
+		cpl_cntl = ((cpl_threshold & LCAC_CPL_CNTL__CPL_THRESHOLD_MASK) <<
+				LCAC_CPL_CNTL__CPL_THRESHOLD__SHIFT) |
+						LCAC_CPL_CNTL__CPL_ENABLE_MASK;
+		cpl_cntl = (cpl_cntl | (8 << LCAC_CPL_CNTL__CPL_BLOCK_ID__SHIFT));
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_MC0_CNTL, mc_threshold);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_MC1_CNTL, mc_threshold);
+		if (8 == cpl_threshold) {
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC2_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC3_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC4_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC5_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC6_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC7_CNTL, mc_threshold);
+		}
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_CPL_CNTL, cpl_cntl);
+
+		udelay(5);
+
+		mc_threshold = mc_threshold |
+				(1 << LCAC_MC0_CNTL__MC0_SIGNAL_ID__SHIFT);
+		cpl_cntl = cpl_cntl | (1 << LCAC_CPL_CNTL__CPL_SIGNAL_ID__SHIFT);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_MC0_CNTL, mc_threshold);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_MC1_CNTL, mc_threshold);
+		if (8 == cpl_threshold) {
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC2_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC3_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC4_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC5_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC6_CNTL, mc_threshold);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixLCAC_MC7_CNTL, mc_threshold);
+		}
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixLCAC_CPL_CNTL, cpl_cntl);
+
+		/* Program CAC_EN per MCD (0-7) Tile */
+		val0 = val = cgs_read_register(hwmgr->device, mmMC_CONFIG_MCD);
+		val &= ~(MC_CONFIG_MCD__MCD0_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD1_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD2_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD3_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD4_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD5_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD6_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MCD7_WR_ENABLE_MASK |
+				MC_CONFIG_MCD__MC_RD_ENABLE_MASK);
+
+		for (i = 0; i < 8; i++) {
+			/* Enable MCD i Tile read & write */
+			val2  = (val | (i << MC_CONFIG_MCD__MC_RD_ENABLE__SHIFT) |
+					(1 << i));
+			cgs_write_register(hwmgr->device, mmMC_CONFIG_MCD, val2);
+			/* Enbale CAC_ON MCD i Tile */
+			val2 = cgs_read_register(hwmgr->device, mmMC_SEQ_CNTL);
+			val2 |= MC_SEQ_CNTL__CAC_EN_MASK;
+			cgs_write_register(hwmgr->device, mmMC_SEQ_CNTL, val2);
+		}
+		/* Set MC_CONFIG_MCD back to its default setting val0 */
+		cgs_write_register(hwmgr->device, mmMC_CONFIG_MCD, val0);
+
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_MCLKDPM_Enable)),
+				"Failed to enable MCLK DPM during DPM Start Function!",
+				return -1);
+	}
+	return 0;
+}
+
+static int fiji_start_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/*enable general power management */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+			GLOBAL_PWRMGT_EN, 1);
+	/* enable sclk deep sleep */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+			DYNAMIC_PM_EN, 1);
+	/* prepare for PCIE DPM */
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start + offsetof(SMU73_SoftRegisters,
+					VoltageChangeTimeout), 0x1000);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
+			SWRST_COMMAND_1, RESETLC, 0x0);
+
+	PP_ASSERT_WITH_CODE(
+			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					PPSMC_MSG_Voltage_Cntl_Enable)),
+			"Failed to enable voltage DPM during DPM Start Function!",
+			return -1);
+
+	if (fiji_enable_sclk_mclk_dpm(hwmgr)) {
+		printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
+		return -1;
+	}
+
+	/* enable PCIE dpm */
+	if(!data->pcie_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_Enable)),
+				"Failed to enable pcie DPM during DPM Start Function!",
+				return -1);
+	}
+
+	return 0;
+}
+
+static void fiji_set_dpm_event_sources(struct pp_hwmgr *hwmgr,
+		uint32_t sources)
+{
+	bool protection;
+	enum DPM_EVENT_SRC src;
+
+	switch (sources) {
+	default:
+		printk(KERN_ERR "Unknown throttling event sources.");
+		/* fall through */
+	case 0:
+		protection = false;
+		/* src is unused */
+		break;
+	case (1 << PHM_AutoThrottleSource_Thermal):
+		protection = true;
+		src = DPM_EVENT_SRC_DIGITAL;
+		break;
+	case (1 << PHM_AutoThrottleSource_External):
+		protection = true;
+		src = DPM_EVENT_SRC_EXTERNAL;
+		break;
+	case (1 << PHM_AutoThrottleSource_External) |
+			(1 << PHM_AutoThrottleSource_Thermal):
+		protection = true;
+		src = DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL;
+		break;
+	}
+	/* Order matters - don't enable thermal protection for the wrong source. */
+	if (protection) {
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL,
+				DPM_EVENT_SRC, src);
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+				THERMAL_PROTECTION_DIS,
+				phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_ThermalController));
+	} else
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+				THERMAL_PROTECTION_DIS, 1);
+}
+
+static int fiji_enable_auto_throttle_source(struct pp_hwmgr *hwmgr,
+		PHM_AutoThrottleSource source)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (!(data->active_auto_throttle_sources & (1 << source))) {
+		data->active_auto_throttle_sources |= 1 << source;
+		fiji_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
+	}
+	return 0;
+}
+
+static int fiji_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
+{
+	return fiji_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
+}
+
+static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = (!fiji_is_dpm_running(hwmgr))? 0 : -1;
+	PP_ASSERT_WITH_CODE(result == 0,
+			"DPM is already running right now, no need to enable DPM!",
+			return 0);
+
+	if (fiji_voltage_control(hwmgr)) {
+		tmp_result = fiji_enable_voltage_control(hwmgr);
+		PP_ASSERT_WITH_CODE(tmp_result == 0,
+				"Failed to enable voltage control!",
+				result = tmp_result);
+	}
+
+	if (fiji_voltage_control(hwmgr)) {
+		tmp_result = fiji_construct_voltage_tables(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to contruct voltage tables!",
+				result = tmp_result);
+	}
+
+	tmp_result = fiji_initialize_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize MC reg table!", result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EngineSpreadSpectrumSupport))
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalController))
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 0);
+
+	tmp_result = fiji_program_static_screen_threshold_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to program static screen threshold parameters!",
+			result = tmp_result);
+
+	tmp_result = fiji_enable_display_gap(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable display gap!", result = tmp_result);
+
+	tmp_result = fiji_program_voting_clients(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to program voting clients!", result = tmp_result);
+
+	tmp_result = fiji_process_firmware_header(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to process firmware header!", result = tmp_result);
+
+	tmp_result = fiji_initial_switch_from_arbf0_to_f1(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize switch from ArbF0 to F1!",
+			result = tmp_result);
+
+	tmp_result = fiji_init_smc_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize SMC table!", result = tmp_result);
+
+	tmp_result = fiji_init_arb_table_index(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize ARB table index!", result = tmp_result);
+
+	tmp_result = fiji_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to populate PM fuses!", result = tmp_result);
+
+	tmp_result = fiji_enable_vrhot_gpio_interrupt(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable VR hot GPIO interrupt!", result = tmp_result);
+
+	tmp_result = tonga_notify_smc_display_change(hwmgr, false);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to notify no display!", result = tmp_result);
+
+	tmp_result = fiji_enable_sclk_control(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable SCLK control!", result = tmp_result);
+
+	tmp_result = fiji_enable_ulv(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable ULV!", result = tmp_result);
+
+	tmp_result = fiji_enable_deep_sleep_master_switch(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable deep sleep master switch!", result = tmp_result);
+
+	tmp_result = fiji_start_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to start DPM!", result = tmp_result);
+
+	tmp_result = fiji_enable_smc_cac(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable SMC CAC!", result = tmp_result);
+
+	tmp_result = fiji_enable_power_containment(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable power containment!", result = tmp_result);
+
+	tmp_result = fiji_power_control_set_level(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to power control set level!", result = tmp_result);
+
+	tmp_result = fiji_enable_thermal_auto_throttle(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable thermal auto throttle!", result = tmp_result);
+
+	return result;
+}
+
+static int fiji_force_dpm_highest(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t level, tmp;
+
+	if (!data->sclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SCLKDPM_SetEnabledMask,
+						(1 << level));
+		}
+	}
+
+	if (!data->mclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_MCLKDPM_SetEnabledMask,
+						(1 << level));
+		}
+	}
+
+	if (!data->pcie_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_ForceLevel,
+						(1 << level));
+		}
+	}
+	return 0;
+}
+
+static void fiji_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_clock_voltage_dependency_table *table =
+				table_info->vddc_dep_on_dal_pwrl;
+	struct phm_ppt_v1_clock_voltage_dependency_table *vddc_table;
+	enum PP_DAL_POWERLEVEL dal_power_level = hwmgr->dal_power_level;
+	uint32_t req_vddc = 0, req_volt, i;
+
+	if (!table && !(dal_power_level >= PP_DAL_POWERLEVEL_ULTRALOW &&
+			dal_power_level <= PP_DAL_POWERLEVEL_PERFORMANCE))
+		return;
+
+	for (i= 0; i < table->count; i++) {
+		if (dal_power_level == table->entries[i].clk) {
+			req_vddc = table->entries[i].v;
+			break;
+		}
+	}
+
+	vddc_table = table_info->vdd_dep_on_sclk;
+	for (i= 0; i < vddc_table->count; i++) {
+		if (req_vddc <= vddc_table->entries[i].vddc) {
+			req_volt = (((uint32_t)vddc_table->entries[i].vddc) * VOLTAGE_SCALE)
+					<< VDDC_SHIFT;
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_VddC_Request, req_volt);
+			return;
+		}
+	}
+	printk(KERN_ERR "DAL requested level can not"
+			" found a available voltage in VDDC DPM Table \n");
+}
+
+static int fiji_upload_dpmlevel_enable_mask(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	fiji_apply_dal_min_voltage_request(hwmgr);
+
+	if (!data->sclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+	}
+	return 0;
+}
+
+static int fiji_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (!fiji_is_dpm_running(hwmgr))
+		return -EINVAL;
+
+	if (!data->pcie_dpm_key_disabled) {
+		smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_PCIeDPM_UnForceLevel);
+	}
+
+	return fiji_upload_dpmlevel_enable_mask(hwmgr);
+}
+
+static uint32_t fiji_get_lowest_enabled_level(
+		struct pp_hwmgr *hwmgr, uint32_t mask)
+{
+	uint32_t level = 0;
+
+	while(0 == (mask & (1 << level)))
+		level++;
+
+	return level;
+}
+
+static int fiji_force_dpm_lowest(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data =
+			(struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t level;
+
+	if (!data->sclk_dpm_key_disabled)
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			level = fiji_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_SCLKDPM_SetEnabledMask,
+							    (1 << level));
+
+	}
+
+	if (!data->mclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			level = fiji_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_MCLKDPM_SetEnabledMask,
+							    (1 << level));
+		}
+	}
+
+	if (!data->pcie_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
+			level = fiji_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_PCIeDPM_ForceLevel,
+							    (1 << level));
+		}
+	}
+
+	return 0;
+
+}
+static int fiji_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
+				enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+		ret = fiji_force_dpm_highest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		ret = fiji_force_dpm_lowest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		ret = fiji_unforce_dpm_levels(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	default:
+		break;
+	}
+
+	hwmgr->dpm_level = level;
+
+	return ret;
+}
+
+static int fiji_get_power_state_size(struct pp_hwmgr *hwmgr)
+{
+	return sizeof(struct fiji_power_state);
+}
+
+static int fiji_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
+		void *state, struct pp_power_state *power_state,
+		void *pp_table, uint32_t classification_flag)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_power_state  *fiji_power_state =
+			(struct fiji_power_state *)(&(power_state->hardware));
+	struct fiji_performance_level *performance_level;
+	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
+	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
+			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
+	ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table =
+			(ATOM_Tonga_SCLK_Dependency_Table *)
+			(((unsigned long)powerplay_table) +
+				le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
+	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
+			(ATOM_Tonga_MCLK_Dependency_Table *)
+			(((unsigned long)powerplay_table) +
+				le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
+
+	/* The following fields are not initialized here: id orderedList allStatesList */
+	power_state->classification.ui_label =
+			(le16_to_cpu(state_entry->usClassification) &
+			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
+			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
+	power_state->classification.flags = classification_flag;
+	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
+
+	power_state->classification.temporary_state = false;
+	power_state->classification.to_be_deleted = false;
+
+	power_state->validation.disallowOnDC =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
+					ATOM_Tonga_DISALLOW_ON_DC));
+
+	power_state->pcie.lanes = 0;
+
+	power_state->display.disableFrameModulation = false;
+	power_state->display.limitRefreshrate = false;
+	power_state->display.enableVariBright =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
+					ATOM_Tonga_ENABLE_VARIBRIGHT));
+
+	power_state->validation.supportedPowerLevels = 0;
+	power_state->uvd_clocks.VCLK = 0;
+	power_state->uvd_clocks.DCLK = 0;
+	power_state->temperatures.min = 0;
+	power_state->temperatures.max = 0;
+
+	performance_level = &(fiji_power_state->performance_levels
+			[fiji_power_state->performance_level_count++]);
+
+	PP_ASSERT_WITH_CODE(
+			(fiji_power_state->performance_level_count < SMU73_MAX_LEVELS_GRAPHICS),
+			"Performance levels exceeds SMC limit!",
+			return -1);
+
+	PP_ASSERT_WITH_CODE(
+			(fiji_power_state->performance_level_count <=
+					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
+			"Performance levels exceeds Driver limit!",
+			return -1);
+
+	/* Performance levels are arranged from low to high. */
+	performance_level->memory_clock = mclk_dep_table->entries
+			[state_entry->ucMemoryClockIndexLow].ulMclk;
+	performance_level->engine_clock = sclk_dep_table->entries
+			[state_entry->ucEngineClockIndexLow].ulSclk;
+	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
+			state_entry->ucPCIEGenLow);
+	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
+			state_entry->ucPCIELaneHigh);
+
+	performance_level = &(fiji_power_state->performance_levels
+			[fiji_power_state->performance_level_count++]);
+	performance_level->memory_clock = mclk_dep_table->entries
+			[state_entry->ucMemoryClockIndexHigh].ulMclk;
+	performance_level->engine_clock = sclk_dep_table->entries
+			[state_entry->ucEngineClockIndexHigh].ulSclk;
+	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
+			state_entry->ucPCIEGenHigh);
+	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
+			state_entry->ucPCIELaneHigh);
+
+	return 0;
+}
+
+static int fiji_get_pp_table_entry(struct pp_hwmgr *hwmgr,
+		unsigned long entry_index, struct pp_power_state *state)
+{
+	int result;
+	struct fiji_power_state *ps;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
+			table_info->vdd_dep_on_mclk;
+
+	state->hardware.magic = PHM_VIslands_Magic;
+
+	ps = (struct fiji_power_state *)(&state->hardware);
+
+	result = tonga_get_powerplay_table_entry(hwmgr, entry_index, state,
+			fiji_get_pp_table_entry_callback_func);
+
+	/* This is the earliest time we have all the dependency table and the VBIOS boot state
+	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
+	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
+	 */
+	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
+		if (dep_mclk_table->entries[0].clk !=
+				data->vbios_boot_state.mclk_bootup_value)
+			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot MCLK level");
+		if (dep_mclk_table->entries[0].vddci !=
+				data->vbios_boot_state.vddci_bootup_value)
+			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot VDDCI level");
+	}
+
+	/* set DC compatible flag if this state supports DC */
+	if (!state->validation.disallowOnDC)
+		ps->dc_compatible = true;
+
+	if (state->classification.flags & PP_StateClassificationFlag_ACPI)
+		data->acpi_pcie_gen = ps->performance_levels[0].pcie_gen;
+
+	ps->uvd_clks.vclk = state->uvd_clocks.VCLK;
+	ps->uvd_clks.dclk = state->uvd_clocks.DCLK;
+
+	if (!result) {
+		uint32_t i;
+
+		switch (state->classification.ui_label) {
+		case PP_StateUILabel_Performance:
+			data->use_pcie_performance_levels = true;
+
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_performance.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_performance.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_performance.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.max =
+							ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_performance.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		case PP_StateUILabel_Battery:
+			data->use_pcie_power_saving_levels = true;
+
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_power_saving.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_power_saving.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_power_saving.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.max =
+							ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_power_saving.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+static int fiji_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				struct pp_power_state  *request_ps,
+			const struct pp_power_state *current_ps)
+{
+	struct fiji_power_state *fiji_ps =
+				cast_phw_fiji_power_state(&request_ps->hardware);
+	uint32_t sclk;
+	uint32_t mclk;
+	struct PP_Clocks minimum_clocks = {0};
+	bool disable_mclk_switching;
+	bool disable_mclk_switching_for_frame_lock;
+	struct cgs_display_info info = {0};
+	const struct phm_clock_and_voltage_limits *max_limits;
+	uint32_t i;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int32_t count;
+	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
+
+	data->battery_state = (PP_StateUILabel_Battery ==
+			request_ps->classification.ui_label);
+
+	PP_ASSERT_WITH_CODE(fiji_ps->performance_level_count == 2,
+				 "VI should always have 2 performance levels",);
+
+	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
+			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
+			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
+
+	/* Cap clock DPM tables at DC MAX if it is in DC. */
+	if (PP_PowerSource_DC == hwmgr->power_source) {
+		for (i = 0; i < fiji_ps->performance_level_count; i++) {
+			if (fiji_ps->performance_levels[i].memory_clock > max_limits->mclk)
+				fiji_ps->performance_levels[i].memory_clock = max_limits->mclk;
+			if (fiji_ps->performance_levels[i].engine_clock > max_limits->sclk)
+				fiji_ps->performance_levels[i].engine_clock = max_limits->sclk;
+		}
+	}
+
+	fiji_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
+	fiji_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
+
+	fiji_ps->acp_clk = hwmgr->acp_arbiter.acpclk;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
+
+	/* TO DO GetMinClockSettings(hwmgr->pPECI, &minimum_clocks); */
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState)) {
+		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
+		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
+
+		for (count = table_info->vdd_dep_on_sclk->count - 1;
+				count >= 0; count--) {
+			if (stable_pstate_sclk >=
+					table_info->vdd_dep_on_sclk->entries[count].clk) {
+				stable_pstate_sclk =
+						table_info->vdd_dep_on_sclk->entries[count].clk;
+				break;
+			}
+		}
+
+		if (count < 0)
+			stable_pstate_sclk = table_info->vdd_dep_on_sclk->entries[0].clk;
+
+		stable_pstate_mclk = max_limits->mclk;
+
+		minimum_clocks.engineClock = stable_pstate_sclk;
+		minimum_clocks.memoryClock = stable_pstate_mclk;
+	}
+
+	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
+		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
+
+	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
+		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
+
+	fiji_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
+
+	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
+				hwmgr->platform_descriptor.overdriveLimit.engineClock),
+				"Overdrive sclk exceeds limit",
+				hwmgr->gfx_arbiter.sclk_over_drive =
+						hwmgr->platform_descriptor.overdriveLimit.engineClock);
+
+		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
+			fiji_ps->performance_levels[1].engine_clock =
+					hwmgr->gfx_arbiter.sclk_over_drive;
+	}
+
+	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
+				"Overdrive mclk exceeds limit",
+				hwmgr->gfx_arbiter.mclk_over_drive =
+						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
+
+		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
+			fiji_ps->performance_levels[1].memory_clock =
+					hwmgr->gfx_arbiter.mclk_over_drive;
+	}
+
+	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
+				    hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
+
+	disable_mclk_switching = (1 < info.display_count) ||
+				    disable_mclk_switching_for_frame_lock;
+
+	sclk = fiji_ps->performance_levels[0].engine_clock;
+	mclk = fiji_ps->performance_levels[0].memory_clock;
+
+	if (disable_mclk_switching)
+		mclk = fiji_ps->performance_levels
+		[fiji_ps->performance_level_count - 1].memory_clock;
+
+	if (sclk < minimum_clocks.engineClock)
+		sclk = (minimum_clocks.engineClock > max_limits->sclk) ?
+				max_limits->sclk : minimum_clocks.engineClock;
+
+	if (mclk < minimum_clocks.memoryClock)
+		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ?
+				max_limits->mclk : minimum_clocks.memoryClock;
+
+	fiji_ps->performance_levels[0].engine_clock = sclk;
+	fiji_ps->performance_levels[0].memory_clock = mclk;
+
+	fiji_ps->performance_levels[1].engine_clock =
+		(fiji_ps->performance_levels[1].engine_clock >=
+				fiji_ps->performance_levels[0].engine_clock) ?
+						fiji_ps->performance_levels[1].engine_clock :
+						fiji_ps->performance_levels[0].engine_clock;
+
+	if (disable_mclk_switching) {
+		if (mclk < fiji_ps->performance_levels[1].memory_clock)
+			mclk = fiji_ps->performance_levels[1].memory_clock;
+
+		fiji_ps->performance_levels[0].memory_clock = mclk;
+		fiji_ps->performance_levels[1].memory_clock = mclk;
+	} else {
+		if (fiji_ps->performance_levels[1].memory_clock <
+				fiji_ps->performance_levels[0].memory_clock)
+			fiji_ps->performance_levels[1].memory_clock =
+					fiji_ps->performance_levels[0].memory_clock;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState)) {
+		for (i = 0; i < fiji_ps->performance_level_count; i++) {
+			fiji_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
+			fiji_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
+			fiji_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
+			fiji_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	const struct fiji_power_state *fiji_ps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+	uint32_t sclk = fiji_ps->performance_levels
+			[fiji_ps->performance_level_count - 1].engine_clock;
+	struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+	uint32_t mclk = fiji_ps->performance_levels
+			[fiji_ps->performance_level_count - 1].memory_clock;
+	struct PP_Clocks min_clocks = {0};
+	uint32_t i;
+	struct cgs_display_info info = {0};
+
+	data->need_update_smu7_dpm_table = 0;
+
+	for (i = 0; i < sclk_table->count; i++) {
+		if (sclk == sclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= sclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+	else {
+	/* TODO: Check SCLK in DAL's minimum clocks
+	 * in case DeepSleep divider update is required.
+	 */
+		if(data->display_timing.min_clock_in_sr != min_clocks.engineClockInSR)
+			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
+	}
+
+	for (i = 0; i < mclk_table->count; i++) {
+		if (mclk == mclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= mclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (data->display_timing.num_existing_displays != info.display_count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
+
+	return 0;
+}
+
+static uint16_t fiji_get_maximum_link_speed(struct pp_hwmgr *hwmgr,
+		const struct fiji_power_state *fiji_ps)
+{
+	uint32_t i;
+	uint32_t sclk, max_sclk = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_dpm_table *dpm_table = &data->dpm_table;
+
+	for (i = 0; i < fiji_ps->performance_level_count; i++) {
+		sclk = fiji_ps->performance_levels[i].engine_clock;
+		if (max_sclk < sclk)
+			max_sclk = sclk;
+	}
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		if (dpm_table->sclk_table.dpm_levels[i].value == max_sclk)
+			return (uint16_t) ((i >= dpm_table->pcie_speed_table.count) ?
+					dpm_table->pcie_speed_table.dpm_levels
+					[dpm_table->pcie_speed_table.count - 1].value :
+					dpm_table->pcie_speed_table.dpm_levels[i].value);
+	}
+
+	return 0;
+}
+
+static int fiji_request_link_speed_change_before_state_change(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	const struct fiji_power_state *fiji_nps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+	const struct fiji_power_state *fiji_cps =
+			cast_const_phw_fiji_power_state(states->pcurrent_state);
+
+	uint16_t target_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_nps);
+	uint16_t current_link_speed;
+
+	if (data->force_pcie_gen == PP_PCIEGenInvalid)
+		current_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_cps);
+	else
+		current_link_speed = data->force_pcie_gen;
+
+	data->force_pcie_gen = PP_PCIEGenInvalid;
+	data->pspp_notify_required = false;
+	if (target_link_speed > current_link_speed) {
+		switch(target_link_speed) {
+		case PP_PCIEGen3:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
+				break;
+			data->force_pcie_gen = PP_PCIEGen2;
+			if (current_link_speed == PP_PCIEGen2)
+				break;
+		case PP_PCIEGen2:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
+				break;
+		default:
+			data->force_pcie_gen = fiji_get_current_pcie_speed(hwmgr);
+			break;
+		}
+	} else {
+		if (target_link_speed < current_link_speed)
+			data->pspp_notify_required = true;
+	}
+
+	return 0;
+}
+
+static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+		PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
+				"Trying to freeze SCLK DPM when DPM is disabled",);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_FreezeLevel),
+				"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
+				return -1);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		 DPMTABLE_OD_UPDATE_MCLK)) {
+		PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
+				"Trying to freeze MCLK DPM when DPM is disabled",);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_MCLKDPM_FreezeLevel),
+				"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
+				return -1);
+	}
+
+	return 0;
+}
+
+static int fiji_populate_and_upload_sclk_mclk_dpm_levels(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result = 0;
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	const struct fiji_power_state *fiji_ps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t sclk = fiji_ps->performance_levels
+			[fiji_ps->performance_level_count - 1].engine_clock;
+	uint32_t mclk = fiji_ps->performance_levels
+			[fiji_ps->performance_level_count - 1].memory_clock;
+	struct fiji_dpm_table *dpm_table = &data->dpm_table;
+
+	struct fiji_dpm_table *golden_dpm_table = &data->golden_dpm_table;
+	uint32_t dpm_count, clock_percent;
+	uint32_t i;
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
+		dpm_table->sclk_table.dpm_levels
+		[dpm_table->sclk_table.count - 1].value = sclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_OD6PlusinACSupport) ||
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_OD6PlusinDCSupport)) {
+		/* Need to do calculation based on the golden DPM table
+		 * as the Heatmap GPU Clock axis is also based on the default values
+		 */
+			PP_ASSERT_WITH_CODE(
+				(golden_dpm_table->sclk_table.dpm_levels
+						[golden_dpm_table->sclk_table.count - 1].value != 0),
+				"Divide by 0!",
+				return -1);
+			dpm_count = dpm_table->sclk_table.count < 2 ?
+					0 : dpm_table->sclk_table.count - 2;
+			for (i = dpm_count; i > 1; i--) {
+				if (sclk > golden_dpm_table->sclk_table.dpm_levels
+						[golden_dpm_table->sclk_table.count-1].value) {
+					clock_percent =
+						((sclk - golden_dpm_table->sclk_table.dpm_levels
+							[golden_dpm_table->sclk_table.count-1].value) * 100) /
+						golden_dpm_table->sclk_table.dpm_levels
+							[golden_dpm_table->sclk_table.count-1].value;
+
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value +
+							(golden_dpm_table->sclk_table.dpm_levels[i].value *
+								clock_percent)/100;
+
+				} else if (golden_dpm_table->sclk_table.dpm_levels
+						[dpm_table->sclk_table.count-1].value > sclk) {
+					clock_percent =
+						((golden_dpm_table->sclk_table.dpm_levels
+						[golden_dpm_table->sclk_table.count - 1].value - sclk) *
+								100) /
+						golden_dpm_table->sclk_table.dpm_levels
+							[golden_dpm_table->sclk_table.count-1].value;
+
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value -
+							(golden_dpm_table->sclk_table.dpm_levels[i].value *
+									clock_percent) / 100;
+				} else
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
+		dpm_table->mclk_table.dpm_levels
+			[dpm_table->mclk_table.count - 1].value = mclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_OD6PlusinACSupport) ||
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_OD6PlusinDCSupport)) {
+
+			PP_ASSERT_WITH_CODE(
+					(golden_dpm_table->mclk_table.dpm_levels
+						[golden_dpm_table->mclk_table.count-1].value != 0),
+					"Divide by 0!",
+					return -1);
+			dpm_count = dpm_table->mclk_table.count < 2 ?
+					0 : dpm_table->mclk_table.count - 2;
+			for (i = dpm_count; i > 1; i--) {
+				if (mclk > golden_dpm_table->mclk_table.dpm_levels
+						[golden_dpm_table->mclk_table.count-1].value) {
+					clock_percent = ((mclk -
+							golden_dpm_table->mclk_table.dpm_levels
+							[golden_dpm_table->mclk_table.count-1].value) * 100) /
+							golden_dpm_table->mclk_table.dpm_levels
+							[golden_dpm_table->mclk_table.count-1].value;
+
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value +
+							(golden_dpm_table->mclk_table.dpm_levels[i].value *
+									clock_percent) / 100;
+
+				} else if (golden_dpm_table->mclk_table.dpm_levels
+						[dpm_table->mclk_table.count-1].value > mclk) {
+					clock_percent = ((golden_dpm_table->mclk_table.dpm_levels
+							[golden_dpm_table->mclk_table.count-1].value - mclk) * 100) /
+									golden_dpm_table->mclk_table.dpm_levels
+									[golden_dpm_table->mclk_table.count-1].value;
+
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value -
+							(golden_dpm_table->mclk_table.dpm_levels[i].value *
+									clock_percent) / 100;
+				} else
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
+		result = fiji_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
+
+	if (data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
+		/*populate MCLK dpm table to SMU7 */
+		result = fiji_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
+
+	return result;
+}
+
+static int fiji_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
+			  struct fiji_single_dpm_table * dpm_table,
+			     uint32_t low_limit, uint32_t high_limit)
+{
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->count; i++) {
+		if ((dpm_table->dpm_levels[i].value < low_limit) ||
+		    (dpm_table->dpm_levels[i].value > high_limit))
+			dpm_table->dpm_levels[i].enabled = false;
+		else
+			dpm_table->dpm_levels[i].enabled = true;
+	}
+	return 0;
+}
+
+static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr,
+		const struct fiji_power_state *fiji_ps)
+{
+	int result = 0;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t high_limit_count;
+
+	PP_ASSERT_WITH_CODE((fiji_ps->performance_level_count >= 1),
+			"power state did not have any performance level",
+			return -1);
+
+	high_limit_count = (1 == fiji_ps->performance_level_count) ? 0 : 1;
+
+	fiji_trim_single_dpm_states(hwmgr,
+			&(data->dpm_table.sclk_table),
+			fiji_ps->performance_levels[0].engine_clock,
+			fiji_ps->performance_levels[high_limit_count].engine_clock);
+
+	fiji_trim_single_dpm_states(hwmgr,
+			&(data->dpm_table.mclk_table),
+			fiji_ps->performance_levels[0].memory_clock,
+			fiji_ps->performance_levels[high_limit_count].memory_clock);
+
+	return result;
+}
+
+static int fiji_generate_dpm_level_enable_mask(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result;
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	const struct fiji_power_state *fiji_ps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+
+	result = fiji_trim_dpm_states(hwmgr, fiji_ps);
+	if (result)
+		return result;
+
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
+	data->last_mclk_dpm_enable_mask =
+			data->dpm_level_enable_mask.mclk_dpm_enable_mask;
+
+	if (data->uvd_enabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask & 1)
+			data->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
+	}
+
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
+
+	return 0;
+}
+
+int fiji_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Enable :
+				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Disable);
+}
+
+int fiji_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
+			PPSMC_MSG_VCEDPM_Enable :
+			PPSMC_MSG_VCEDPM_Disable);
+}
+
+int fiji_enable_disable_samu_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
+			PPSMC_MSG_SAMUDPM_Enable :
+			PPSMC_MSG_SAMUDPM_Disable);
+}
+
+int fiji_enable_disable_acp_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
+			PPSMC_MSG_ACPDPM_Enable :
+			PPSMC_MSG_ACPDPM_Disable);
+}
+
+int fiji_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (!bgate) {
+		data->smc_state_table.UvdBootLevel = 0;
+		if (table_info->mm_dep_table->count > 0)
+			data->smc_state_table.UvdBootLevel =
+					(uint8_t) (table_info->mm_dep_table->count - 1);
+		mm_boot_level_offset = data->dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable, UvdBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0x00FFFFFF;
+		mm_boot_level_value |= data->smc_state_table.UvdBootLevel << 24;
+		cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_UVDDPM) ||
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_StablePState))
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_UVDDPM_SetEnabledMask,
+					(uint32_t)(1 << data->smc_state_table.UvdBootLevel));
+	}
+
+	return fiji_enable_disable_uvd_dpm(hwmgr, !bgate);
+}
+
+int fiji_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	const struct fiji_power_state *fiji_nps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+	const struct fiji_power_state *fiji_cps =
+			cast_const_phw_fiji_power_state(states->pcurrent_state);
+
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (fiji_nps->vce_clks.evclk >0 &&
+	(fiji_cps == NULL || fiji_cps->vce_clks.evclk == 0)) {
+		data->smc_state_table.VceBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+
+		mm_boot_level_offset = data->dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable, VceBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0xFF00FFFF;
+		mm_boot_level_value |= data->smc_state_table.VceBootLevel << 16;
+		cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_StablePState)) {
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_VCEDPM_SetEnabledMask,
+					(uint32_t)1 << data->smc_state_table.VceBootLevel);
+
+			fiji_enable_disable_vce_dpm(hwmgr, true);
+		} else if (fiji_nps->vce_clks.evclk == 0 &&
+				fiji_cps != NULL &&
+				fiji_cps->vce_clks.evclk > 0)
+			fiji_enable_disable_vce_dpm(hwmgr, false);
+	}
+
+	return 0;
+}
+
+int fiji_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (!bgate) {
+		data->smc_state_table.SamuBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+		mm_boot_level_offset = data->dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable, SamuBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0xFFFFFF00;
+		mm_boot_level_value |= data->smc_state_table.SamuBootLevel << 0;
+		cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_StablePState))
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SAMUDPM_SetEnabledMask,
+					(uint32_t)(1 << data->smc_state_table.SamuBootLevel));
+	}
+
+	return fiji_enable_disable_samu_dpm(hwmgr, !bgate);
+}
+
+int fiji_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (!bgate) {
+		data->smc_state_table.AcpBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+		mm_boot_level_offset = data->dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable, AcpBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0xFFFF00FF;
+		mm_boot_level_value |= data->smc_state_table.AcpBootLevel << 8;
+		cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_StablePState))
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_ACPDPM_SetEnabledMask,
+						(uint32_t)(1 << data->smc_state_table.AcpBootLevel));
+	}
+
+	return fiji_enable_disable_acp_dpm(hwmgr, !bgate);
+}
+
+static int fiji_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold !=
+				data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold =
+				hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = fiji_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				data->dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				data->sram_end);
+	}
+
+	return result;
+}
+
+static int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return fiji_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+
+		PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
+				"Trying to Unfreeze SCLK DPM when DPM is disabled",);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
+			return -1);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
+
+		PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
+				"Trying to Unfreeze MCLK DPM when DPM is disabled",);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
+		    return -1);
+	}
+
+	data->need_update_smu7_dpm_table = 0;
+
+	return 0;
+}
+
+/* Look up the voltaged based on DAL's requested level.
+ * and then send the requested VDDC voltage to SMC
+ */
+static void fiji_apply_dal_minimum_voltage_request(struct pp_hwmgr *hwmgr)
+{
+	return;
+}
+
+int fiji_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/* Apply minimum voltage based on DAL's request level */
+	fiji_apply_dal_minimum_voltage_request(hwmgr);
+
+	if (0 == data->sclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this,
+		 *  we should skip this message.
+		 */
+		if (!fiji_is_dpm_running(hwmgr))
+			printk(KERN_ERR "[ powerplay ] "
+					"Trying to set Enable Mask when DPM is disabled \n");
+
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == result),
+				"Set Sclk Dpm enable Mask failed", return -1);
+		}
+	}
+
+	if (0 == data->mclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this,
+		 *  we should skip this message.
+		 */
+		if (!fiji_is_dpm_running(hwmgr))
+			printk(KERN_ERR "[ powerplay ]"
+					" Trying to set Enable Mask when DPM is disabled \n");
+
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_MCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == result),
+				"Set Mclk Dpm enable Mask failed", return -1);
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_notify_link_speed_change_after_state_change(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	const struct fiji_power_state *fiji_ps =
+			cast_const_phw_fiji_power_state(states->pnew_state);
+	uint16_t target_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_ps);
+	uint8_t  request;
+
+	if (data->pspp_notify_required) {
+		if (target_link_speed == PP_PCIEGen3)
+			request = PCIE_PERF_REQ_GEN3;
+		else if (target_link_speed == PP_PCIEGen2)
+			request = PCIE_PERF_REQ_GEN2;
+		else
+			request = PCIE_PERF_REQ_GEN1;
+
+		if(request == PCIE_PERF_REQ_GEN1 &&
+				fiji_get_current_pcie_speed(hwmgr) > 0)
+			return 0;
+
+		if (acpi_pcie_perf_request(hwmgr->device, request, false)) {
+			if (PP_PCIEGen2 == target_link_speed)
+				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
+			else
+				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_set_power_state_tasks(struct pp_hwmgr *hwmgr,
+		const void *input)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = fiji_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to find DPM states clocks in DPM table!",
+			result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result =
+			fiji_request_link_speed_change_before_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to request link speed change before state change!",
+				result = tmp_result);
+	}
+
+	tmp_result = fiji_freeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to freeze SCLK MCLK DPM!", result = tmp_result);
+
+	tmp_result = fiji_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to populate and upload SCLK MCLK DPM levels!",
+			result = tmp_result);
+
+	tmp_result = fiji_generate_dpm_level_enable_mask(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to generate DPM level enabled mask!",
+			result = tmp_result);
+
+	tmp_result = fiji_update_vce_dpm(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to update VCE DPM!",
+			result = tmp_result);
+
+	tmp_result = fiji_update_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to update SCLK threshold!",
+			result = tmp_result);
+
+	tmp_result = fiji_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to program memory timing parameters!",
+			result = tmp_result);
+
+	tmp_result = fiji_unfreeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to unfreeze SCLK MCLK DPM!",
+			result = tmp_result);
+
+	tmp_result = fiji_upload_dpm_level_enable_mask(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to upload DPM level enabled mask!",
+			result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result =
+			fiji_notify_link_speed_change_after_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to notify link speed change after state change!",
+				result = tmp_result);
+	}
+
+	return result;
+}
+
+static int fiji_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct fiji_power_state  *fiji_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
+
+	if (low)
+		return fiji_ps->performance_levels[0].engine_clock;
+	else
+		return fiji_ps->performance_levels
+				[fiji_ps->performance_level_count-1].engine_clock;
+}
+
+static int fiji_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct fiji_power_state  *fiji_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
+
+	if (low)
+		return fiji_ps->performance_levels[0].memory_clock;
+	else
+		return fiji_ps->performance_levels
+				[fiji_ps->performance_level_count-1].memory_clock;
+}
+
+static void fiji_print_current_perforce_level(
+		struct pp_hwmgr *hwmgr, struct seq_file *m)
+{
+	uint32_t sclk, mclk, activity_percent = 0;
+	uint32_t offset;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
+
+	sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
+
+	mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+	seq_printf(m, "\n [  mclk  ]: %u MHz\n\n [  sclk  ]: %u MHz\n",
+			mclk / 100, sclk / 100);
+
+	offset = data->soft_regs_start + offsetof(SMU73_SoftRegisters, AverageGraphicsActivity);
+	activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
+	activity_percent += 0x80;
+	activity_percent >>= 8;
+
+	seq_printf(m, "\n [GPU load]: %u%%\n\n", activity_percent > 100 ? 100 : activity_percent);
+}
+
+static int fiji_program_display_gap(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t num_active_displays = 0;
+	uint32_t display_gap = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
+	uint32_t display_gap2;
+	uint32_t pre_vbi_time_in_us;
+	uint32_t frame_time_in_us;
+	uint32_t ref_clock;
+	uint32_t refresh_rate = 0;
+	struct cgs_display_info info = {0};
+	struct cgs_mode_info mode_info;
+
+	info.mode_info = &mode_info;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	num_active_displays = info.display_count;
+
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
+			DISP_GAP, (num_active_displays > 0)?
+			DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_DISPLAY_GAP_CNTL, display_gap);
+
+	ref_clock = mode_info.ref_clock;
+	refresh_rate = mode_info.refresh_rate;
+
+	if (refresh_rate == 0)
+		refresh_rate = 60;
+
+	frame_time_in_us = 1000000 / refresh_rate;
+
+	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
+	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_DISPLAY_GAP_CNTL2, display_gap2);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start +
+			offsetof(SMU73_SoftRegisters, PreVBlankGap), 0x64);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start +
+			offsetof(SMU73_SoftRegisters, VBlankTimeout),
+			(frame_time_in_us - pre_vbi_time_in_us));
+
+	if (num_active_displays == 1)
+		tonga_notify_smc_display_change(hwmgr, true);
+
+	return 0;
+}
+
+int fiji_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
+{
+	return fiji_program_display_gap(hwmgr);
+}
+
+static int fiji_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr,
+		uint16_t us_max_fan_pwm)
+{
+	hwmgr->thermal_controller.
+	advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm);
+}
+
+static int fiji_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr,
+		uint16_t us_max_fan_rpm)
+{
+	hwmgr->thermal_controller.
+	advanceFanControlParameters.usMaxFanRPM = us_max_fan_rpm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetFanRpmMax, us_max_fan_rpm);
+}
+
+int fiji_dpm_set_interrupt_state(void *private_data,
+					 unsigned src_id, unsigned type,
+					 int enabled)
+{
+	uint32_t cg_thermal_int;
+	struct pp_hwmgr *hwmgr = ((struct pp_eventmgr *)private_data)->hwmgr;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	switch (type) {
+	case AMD_THERMAL_IRQ_LOW_TO_HIGH:
+		if (enabled) {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		} else {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		}
+		break;
+
+	case AMD_THERMAL_IRQ_HIGH_TO_LOW:
+		if (enabled) {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		} else {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+int fiji_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
+					const void *thermal_interrupt_info)
+{
+	int result;
+	const struct pp_interrupt_registration_info *info =
+			(const struct pp_interrupt_registration_info *)
+			thermal_interrupt_info;
+
+	if (info == NULL)
+		return -EINVAL;
+
+	result = cgs_add_irq_source(hwmgr->device, 230, AMD_THERMAL_IRQ_LAST,
+				fiji_dpm_set_interrupt_state,
+				info->call_back, info->context);
+
+	if (result)
+		return -EINVAL;
+
+	result = cgs_add_irq_source(hwmgr->device, 231, AMD_THERMAL_IRQ_LAST,
+				fiji_dpm_set_interrupt_state,
+				info->call_back, info->context);
+
+	if (result)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int fiji_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+{
+	if (mode) {
+		/* stop auto-manage */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl))
+			fiji_fan_ctrl_stop_smc_fan_control(hwmgr);
+		fiji_fan_ctrl_set_static_mode(hwmgr, mode);
+	} else
+		/* restart auto-manage */
+		fiji_fan_ctrl_reset_fan_speed_to_default(hwmgr);
+
+	return 0;
+}
+
+static int fiji_get_fan_control_mode(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr->fan_ctrl_is_in_default_mode)
+		return hwmgr->fan_ctrl_default_mode;
+	else
+		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_FDO_CTRL2, FDO_PWM_MODE);
+}
+
+static const struct pp_hwmgr_func fiji_hwmgr_funcs = {
+	.backend_init = &fiji_hwmgr_backend_init,
+	.backend_fini = &tonga_hwmgr_backend_fini,
+	.asic_setup = &fiji_setup_asic_task,
+	.dynamic_state_management_enable = &fiji_enable_dpm_tasks,
+	.force_dpm_level = &fiji_dpm_force_dpm_level,
+	.get_num_of_pp_table_entries = &tonga_get_number_of_powerplay_table_entries,
+	.get_power_state_size = &fiji_get_power_state_size,
+	.get_pp_table_entry = &fiji_get_pp_table_entry,
+	.patch_boot_state = &fiji_patch_boot_state,
+	.apply_state_adjust_rules = &fiji_apply_state_adjust_rules,
+	.power_state_set = &fiji_set_power_state_tasks,
+	.get_sclk = &fiji_dpm_get_sclk,
+	.get_mclk = &fiji_dpm_get_mclk,
+	.print_current_perforce_level = &fiji_print_current_perforce_level,
+	.powergate_uvd = &fiji_phm_powergate_uvd,
+	.powergate_vce = &fiji_phm_powergate_vce,
+	.disable_clock_power_gating = &fiji_phm_disable_clock_power_gating,
+	.notify_smc_display_config_after_ps_adjustment =
+			&tonga_notify_smc_display_config_after_ps_adjustment,
+	.display_config_changed = &fiji_display_configuration_changed_task,
+	.set_max_fan_pwm_output = fiji_set_max_fan_pwm_output,
+	.set_max_fan_rpm_output = fiji_set_max_fan_rpm_output,
+	.get_temperature = fiji_thermal_get_temperature,
+	.stop_thermal_controller = fiji_thermal_stop_thermal_controller,
+	.get_fan_speed_info = fiji_fan_ctrl_get_fan_speed_info,
+	.get_fan_speed_percent = fiji_fan_ctrl_get_fan_speed_percent,
+	.set_fan_speed_percent = fiji_fan_ctrl_set_fan_speed_percent,
+	.reset_fan_speed_to_default = fiji_fan_ctrl_reset_fan_speed_to_default,
+	.get_fan_speed_rpm = fiji_fan_ctrl_get_fan_speed_rpm,
+	.set_fan_speed_rpm = fiji_fan_ctrl_set_fan_speed_rpm,
+	.uninitialize_thermal_controller = fiji_thermal_ctrl_uninitialize_thermal_controller,
+	.register_internal_thermal_interrupt = fiji_register_internal_thermal_interrupt,
+	.set_fan_control_mode = fiji_set_fan_control_mode,
+	.get_fan_control_mode = fiji_get_fan_control_mode,
+};
+
+int fiji_hwmgr_init(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr  *data;
+	int ret = 0;
+
+	data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL);
+	if (data == NULL)
+		return -ENOMEM;
+
+	hwmgr->backend = data;
+	hwmgr->hwmgr_func = &fiji_hwmgr_funcs;
+	hwmgr->pptable_func = &tonga_pptable_funcs;
+	pp_fiji_thermal_initialize(hwmgr);
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
new file mode 100644
index 000000000000..22e273b1c1c5
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
@@ -0,0 +1,361 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _FIJI_HWMGR_H_
+#define _FIJI_HWMGR_H_
+
+#include "hwmgr.h"
+#include "smu73.h"
+#include "smu73_discrete.h"
+#include "ppatomctrl.h"
+#include "fiji_ppsmc.h"
+
+#define FIJI_MAX_HARDWARE_POWERLEVELS	2
+#define FIJI_AT_DFLT	30
+
+#define FIJI_VOLTAGE_CONTROL_NONE                   0x0
+#define FIJI_VOLTAGE_CONTROL_BY_GPIO                0x1
+#define FIJI_VOLTAGE_CONTROL_BY_SVID2               0x2
+#define FIJI_VOLTAGE_CONTROL_MERGED                 0x3
+
+#define DPMTABLE_OD_UPDATE_SCLK     0x00000001
+#define DPMTABLE_OD_UPDATE_MCLK     0x00000002
+#define DPMTABLE_UPDATE_SCLK        0x00000004
+#define DPMTABLE_UPDATE_MCLK        0x00000008
+
+struct fiji_performance_level {
+	uint32_t  memory_clock;
+	uint32_t  engine_clock;
+	uint16_t  pcie_gen;
+	uint16_t  pcie_lane;
+};
+
+struct fiji_uvd_clocks {
+	uint32_t  vclk;
+	uint32_t  dclk;
+};
+
+struct fiji_vce_clocks {
+	uint32_t  evclk;
+	uint32_t  ecclk;
+};
+
+struct fiji_power_state {
+    uint32_t                  magic;
+    struct fiji_uvd_clocks    uvd_clks;
+    struct fiji_vce_clocks    vce_clks;
+    uint32_t                  sam_clk;
+    uint32_t                  acp_clk;
+    uint16_t                  performance_level_count;
+    bool                      dc_compatible;
+    uint32_t                  sclk_threshold;
+    struct fiji_performance_level  performance_levels[FIJI_MAX_HARDWARE_POWERLEVELS];
+};
+
+struct fiji_dpm_level {
+	bool	enabled;
+    uint32_t	value;
+    uint32_t	param1;
+};
+
+#define FIJI_MAX_DEEPSLEEP_DIVIDER_ID 5
+#define MAX_REGULAR_DPM_NUMBER 8
+#define FIJI_MINIMUM_ENGINE_CLOCK 2500
+
+struct fiji_single_dpm_table {
+	uint32_t		count;
+	struct fiji_dpm_level	dpm_levels[MAX_REGULAR_DPM_NUMBER];
+};
+
+struct fiji_dpm_table {
+	struct fiji_single_dpm_table  sclk_table;
+	struct fiji_single_dpm_table  mclk_table;
+	struct fiji_single_dpm_table  pcie_speed_table;
+	struct fiji_single_dpm_table  vddc_table;
+	struct fiji_single_dpm_table  vddci_table;
+	struct fiji_single_dpm_table  mvdd_table;
+};
+
+struct fiji_clock_registers {
+	uint32_t  vCG_SPLL_FUNC_CNTL;
+	uint32_t  vCG_SPLL_FUNC_CNTL_2;
+	uint32_t  vCG_SPLL_FUNC_CNTL_3;
+	uint32_t  vCG_SPLL_FUNC_CNTL_4;
+	uint32_t  vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t  vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t  vDLL_CNTL;
+	uint32_t  vMCLK_PWRMGT_CNTL;
+	uint32_t  vMPLL_AD_FUNC_CNTL;
+	uint32_t  vMPLL_DQ_FUNC_CNTL;
+	uint32_t  vMPLL_FUNC_CNTL;
+	uint32_t  vMPLL_FUNC_CNTL_1;
+	uint32_t  vMPLL_FUNC_CNTL_2;
+	uint32_t  vMPLL_SS1;
+	uint32_t  vMPLL_SS2;
+};
+
+struct fiji_voltage_smio_registers {
+	uint32_t vS0_VID_LOWER_SMIO_CNTL;
+};
+
+#define FIJI_MAX_LEAKAGE_COUNT  8
+struct fiji_leakage_voltage {
+	uint16_t  count;
+	uint16_t  leakage_id[FIJI_MAX_LEAKAGE_COUNT];
+	uint16_t  actual_voltage[FIJI_MAX_LEAKAGE_COUNT];
+};
+
+struct fiji_vbios_boot_state {
+	uint16_t    mvdd_bootup_value;
+	uint16_t    vddc_bootup_value;
+	uint16_t    vddci_bootup_value;
+	uint32_t    sclk_bootup_value;
+	uint32_t    mclk_bootup_value;
+	uint16_t    pcie_gen_bootup_value;
+	uint16_t    pcie_lane_bootup_value;
+};
+
+struct fiji_bacos {
+	uint32_t                       best_match;
+	uint32_t                       baco_flags;
+	struct fiji_performance_level  performance_level;
+};
+
+/* Ultra Low Voltage parameter structure */
+struct fiji_ulv_parm {
+	bool                           ulv_supported;
+	uint32_t                       cg_ulv_parameter;
+	uint32_t                       ulv_volt_change_delay;
+	struct fiji_performance_level  ulv_power_level;
+};
+
+struct fiji_display_timing {
+	uint32_t  min_clock_in_sr;
+	uint32_t  num_existing_displays;
+};
+
+struct fiji_dpmlevel_enable_mask {
+	uint32_t  uvd_dpm_enable_mask;
+	uint32_t  vce_dpm_enable_mask;
+	uint32_t  acp_dpm_enable_mask;
+	uint32_t  samu_dpm_enable_mask;
+	uint32_t  sclk_dpm_enable_mask;
+	uint32_t  mclk_dpm_enable_mask;
+	uint32_t  pcie_dpm_enable_mask;
+};
+
+struct fiji_pcie_perf_range {
+	uint16_t  max;
+	uint16_t  min;
+};
+
+struct fiji_hwmgr {
+	struct fiji_dpm_table			dpm_table;
+	struct fiji_dpm_table			golden_dpm_table;
+
+	uint32_t						voting_rights_clients0;
+	uint32_t						voting_rights_clients1;
+	uint32_t						voting_rights_clients2;
+	uint32_t						voting_rights_clients3;
+	uint32_t						voting_rights_clients4;
+	uint32_t						voting_rights_clients5;
+	uint32_t						voting_rights_clients6;
+	uint32_t						voting_rights_clients7;
+	uint32_t						static_screen_threshold_unit;
+	uint32_t						static_screen_threshold;
+	uint32_t						voltage_control;
+	uint32_t						vddc_vddci_delta;
+
+	uint32_t						active_auto_throttle_sources;
+
+	struct fiji_clock_registers            clock_registers;
+	struct fiji_voltage_smio_registers      voltage_smio_registers;
+
+	bool                           is_memory_gddr5;
+	uint16_t                       acpi_vddc;
+	bool                           pspp_notify_required;
+	uint16_t                       force_pcie_gen;
+	uint16_t                       acpi_pcie_gen;
+	uint32_t                       pcie_gen_cap;
+	uint32_t                       pcie_lane_cap;
+	uint32_t                       pcie_spc_cap;
+	struct fiji_leakage_voltage          vddc_leakage;
+	struct fiji_leakage_voltage          Vddci_leakage;
+
+	uint32_t                             mvdd_control;
+	uint32_t                             vddc_mask_low;
+	uint32_t                             mvdd_mask_low;
+	uint16_t                            max_vddc_in_pptable;
+	uint16_t                            min_vddc_in_pptable;
+	uint16_t                            max_vddci_in_pptable;
+	uint16_t                            min_vddci_in_pptable;
+	uint32_t                             mclk_strobe_mode_threshold;
+	uint32_t                             mclk_stutter_mode_threshold;
+	uint32_t                             mclk_edc_enable_threshold;
+	uint32_t                             mclk_edcwr_enable_threshold;
+	bool                                is_uvd_enabled;
+	struct fiji_vbios_boot_state        vbios_boot_state;
+
+	bool                           battery_state;
+	bool                           is_tlu_enabled;
+
+	/* ---- SMC SRAM Address of firmware header tables ---- */
+	uint32_t                             sram_end;
+	uint32_t                             dpm_table_start;
+	uint32_t                             soft_regs_start;
+	uint32_t                             mc_reg_table_start;
+	uint32_t                             fan_table_start;
+	uint32_t                             arb_table_start;
+	struct SMU73_Discrete_DpmTable       smc_state_table;
+	struct SMU73_Discrete_Ulv            ulv_setting;
+
+	/* ---- Stuff originally coming from Evergreen ---- */
+	uint32_t                             vddci_control;
+	struct pp_atomctrl_voltage_table     vddc_voltage_table;
+	struct pp_atomctrl_voltage_table     vddci_voltage_table;
+	struct pp_atomctrl_voltage_table     mvdd_voltage_table;
+
+	uint32_t                             mgcg_cgtt_local2;
+	uint32_t                             mgcg_cgtt_local3;
+	uint32_t                             gpio_debug;
+	uint32_t                             mc_micro_code_feature;
+	uint32_t                             highest_mclk;
+	uint16_t                             acpi_vddci;
+	uint8_t                              mvdd_high_index;
+	uint8_t                              mvdd_low_index;
+	bool                                 dll_default_on;
+	bool                                 performance_request_registered;
+
+	/* ---- Low Power Features ---- */
+	struct fiji_bacos                    bacos;
+	struct fiji_ulv_parm                 ulv;
+
+	/* ---- CAC Stuff ---- */
+	uint32_t                       cac_table_start;
+	bool                           cac_configuration_required;
+	bool                           driver_calculate_cac_leakage;
+	bool                           cac_enabled;
+
+	/* ---- DPM2 Parameters ---- */
+	uint32_t                       power_containment_features;
+	bool                           enable_dte_feature;
+	bool                           enable_tdc_limit_feature;
+	bool                           enable_pkg_pwr_tracking_feature;
+	bool                           disable_uvd_power_tune_feature;
+	struct fiji_pt_defaults       *power_tune_defaults;
+	struct SMU73_Discrete_PmFuses  power_tune_table;
+	uint32_t                       dte_tj_offset;
+	uint32_t                       fast_watermark_threshold;
+
+	/* ---- Phase Shedding ---- */
+	bool                           vddc_phase_shed_control;
+
+	/* ---- DI/DT ---- */
+	struct fiji_display_timing        display_timing;
+
+	/* ---- Thermal Temperature Setting ---- */
+	struct fiji_dpmlevel_enable_mask     dpm_level_enable_mask;
+	uint32_t                             need_update_smu7_dpm_table;
+	uint32_t                             sclk_dpm_key_disabled;
+	uint32_t                             mclk_dpm_key_disabled;
+	uint32_t                             pcie_dpm_key_disabled;
+	uint32_t                             min_engine_clocks;
+	struct fiji_pcie_perf_range          pcie_gen_performance;
+	struct fiji_pcie_perf_range          pcie_lane_performance;
+	struct fiji_pcie_perf_range          pcie_gen_power_saving;
+	struct fiji_pcie_perf_range          pcie_lane_power_saving;
+	bool                                 use_pcie_performance_levels;
+	bool                                 use_pcie_power_saving_levels;
+	uint32_t                             activity_target[SMU73_MAX_LEVELS_GRAPHICS];
+	uint32_t                             mclk_activity_target;
+	uint32_t                             mclk_dpm0_activity_target;
+	uint32_t                             low_sclk_interrupt_threshold;
+	uint32_t                             last_mclk_dpm_enable_mask;
+	bool                                 uvd_enabled;
+
+	/* ---- Power Gating States ---- */
+	bool                           uvd_power_gated;
+	bool                           vce_power_gated;
+	bool                           samu_power_gated;
+	bool                           acp_power_gated;
+	bool                           pg_acp_init;
+	bool                           frtc_enabled;
+	bool                           frtc_status_changed;
+};
+
+/* To convert to Q8.8 format for firmware */
+#define FIJI_Q88_FORMAT_CONVERSION_UNIT             256
+
+enum Fiji_I2CLineID {
+    Fiji_I2CLineID_DDC1 = 0x90,
+    Fiji_I2CLineID_DDC2 = 0x91,
+    Fiji_I2CLineID_DDC3 = 0x92,
+    Fiji_I2CLineID_DDC4 = 0x93,
+    Fiji_I2CLineID_DDC5 = 0x94,
+    Fiji_I2CLineID_DDC6 = 0x95,
+    Fiji_I2CLineID_SCLSDA = 0x96,
+    Fiji_I2CLineID_DDCVGA = 0x97
+};
+
+#define Fiji_I2C_DDC1DATA          0
+#define Fiji_I2C_DDC1CLK           1
+#define Fiji_I2C_DDC2DATA          2
+#define Fiji_I2C_DDC2CLK           3
+#define Fiji_I2C_DDC3DATA          4
+#define Fiji_I2C_DDC3CLK           5
+#define Fiji_I2C_SDA               40
+#define Fiji_I2C_SCL               41
+#define Fiji_I2C_DDC4DATA          65
+#define Fiji_I2C_DDC4CLK           66
+#define Fiji_I2C_DDC5DATA          0x48
+#define Fiji_I2C_DDC5CLK           0x49
+#define Fiji_I2C_DDC6DATA          0x4a
+#define Fiji_I2C_DDC6CLK           0x4b
+#define Fiji_I2C_DDCVGADATA        0x4c
+#define Fiji_I2C_DDCVGACLK         0x4d
+
+#define FIJI_UNUSED_GPIO_PIN       0x7F
+
+extern int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr);
+extern int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr);
+extern int tonga_get_mc_microcode_version (struct pp_hwmgr *hwmgr);
+extern int tonga_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr);
+extern int tonga_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display);
+int fiji_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input);
+int fiji_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+int fiji_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+int fiji_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+int fiji_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
+
+#define PP_HOST_TO_SMC_UL(X) cpu_to_be32(X)
+#define PP_SMC_TO_HOST_UL(X) be32_to_cpu(X)
+
+#define PP_HOST_TO_SMC_US(X) cpu_to_be16(X)
+#define PP_SMC_TO_HOST_US(X) be16_to_cpu(X)
+
+#define CONVERT_FROM_HOST_TO_SMC_UL(X) ((X) = PP_HOST_TO_SMC_UL(X))
+#define CONVERT_FROM_SMC_TO_HOST_UL(X) ((X) = PP_SMC_TO_HOST_UL(X))
+
+#define CONVERT_FROM_HOST_TO_SMC_US(X) ((X) = PP_HOST_TO_SMC_US(X))
+
+#endif /* _FIJI_HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
new file mode 100644
index 000000000000..6efcb2bac45f
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
@@ -0,0 +1,553 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "hwmgr.h"
+#include "smumgr.h"
+#include "fiji_hwmgr.h"
+#include "fiji_powertune.h"
+#include "fiji_smumgr.h"
+#include "smu73_discrete.h"
+#include "pp_debug.h"
+
+#define VOLTAGE_SCALE  4
+#define POWERTUNE_DEFAULT_SET_MAX    1
+
+struct fiji_pt_defaults fiji_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
+		/*sviLoadLIneEn,  SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc */
+		{1,               0xF,             0xFD,
+		/* TDC_MAWt, TdcWaterfallCtl, DTEAmbientTempBase */
+		0x19,        5,               45}
+};
+
+void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *fiji_hwmgr = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct  phm_ppt_v1_information *table_info =
+			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t tmp = 0;
+
+	if(table_info &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID)
+		fiji_hwmgr->power_tune_defaults =
+				&fiji_power_tune_data_set_array
+				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
+	else
+		fiji_hwmgr->power_tune_defaults = &fiji_power_tune_data_set_array[0];
+
+	/* Assume disabled */
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SQRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DBRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TDRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TCPRamping);
+
+	fiji_hwmgr->dte_tj_offset = tmp;
+
+	if (!tmp) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_PowerContainment);
+
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_CAC);
+
+		fiji_hwmgr->fast_watermark_threshold = 100;
+
+		tmp = 1;
+		fiji_hwmgr->enable_dte_feature = tmp ? false : true;
+		fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false;
+		fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false;
+	}
+}
+
+/* PPGen has the gain setting generated in x * 100 unit
+ * This function is to convert the unit to x * 4096(0x1000) unit.
+ *  This is the unit expected by SMC firmware
+ */
+static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
+{
+	uint32_t tmp;
+	tmp = raw_setting * 4096 / 100;
+	return (uint16_t)tmp;
+}
+
+static void get_scl_sda_value(uint8_t line, uint8_t *scl, uint8_t* sda)
+{
+	switch (line) {
+	case Fiji_I2CLineID_DDC1 :
+		*scl = Fiji_I2C_DDC1CLK;
+		*sda = Fiji_I2C_DDC1DATA;
+		break;
+	case Fiji_I2CLineID_DDC2 :
+		*scl = Fiji_I2C_DDC2CLK;
+		*sda = Fiji_I2C_DDC2DATA;
+		break;
+	case Fiji_I2CLineID_DDC3 :
+		*scl = Fiji_I2C_DDC3CLK;
+		*sda = Fiji_I2C_DDC3DATA;
+		break;
+	case Fiji_I2CLineID_DDC4 :
+		*scl = Fiji_I2C_DDC4CLK;
+		*sda = Fiji_I2C_DDC4DATA;
+		break;
+	case Fiji_I2CLineID_DDC5 :
+		*scl = Fiji_I2C_DDC5CLK;
+		*sda = Fiji_I2C_DDC5DATA;
+		break;
+	case Fiji_I2CLineID_DDC6 :
+		*scl = Fiji_I2C_DDC6CLK;
+		*sda = Fiji_I2C_DDC6DATA;
+		break;
+	case Fiji_I2CLineID_SCLSDA :
+		*scl = Fiji_I2C_SCL;
+		*sda = Fiji_I2C_SDA;
+		break;
+	case Fiji_I2CLineID_DDCVGA :
+		*scl = Fiji_I2C_DDCVGACLK;
+		*sda = Fiji_I2C_DDCVGADATA;
+		break;
+	default:
+		*scl = 0;
+		*sda = 0;
+		break;
+	}
+}
+
+int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct fiji_pt_defaults *defaults = data->power_tune_defaults;
+	SMU73_Discrete_DpmTable  *dpm_table = &(data->smc_state_table);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
+	struct pp_advance_fan_control_parameters *fan_table=
+			&hwmgr->thermal_controller.advanceFanControlParameters;
+	uint8_t uc_scl, uc_sda;
+
+	/* TDP number of fraction bits are changed from 8 to 7 for Fiji
+	 * as requested by SMC team
+	 */
+	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usTDP * 128));
+	dpm_table->TargetTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usTDP * 128));
+
+	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
+			"Target Operating Temp is out of Range!",);
+
+	dpm_table->GpuTjMax = (uint8_t)(cac_dtp_table->usTargetOperatingTemp);
+	dpm_table->GpuTjHyst = 8;
+
+	dpm_table->DTEAmbientTempBase = defaults->DTEAmbientTempBase;
+
+	/* The following are for new Fiji Multi-input fan/thermal control */
+	dpm_table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTargetOperatingTemp * 256);
+	dpm_table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitHotspot * 256);
+	dpm_table->TemperatureLimitLiquid1 = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitLiquid1 * 256);
+	dpm_table->TemperatureLimitLiquid2 = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitLiquid2 * 256);
+	dpm_table->TemperatureLimitVrVddc = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitVrVddc * 256);
+	dpm_table->TemperatureLimitVrMvdd = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitVrMvdd * 256);
+	dpm_table->TemperatureLimitPlx = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitPlx * 256);
+
+	dpm_table->FanGainEdge = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainEdge));
+	dpm_table->FanGainHotspot = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHotspot));
+	dpm_table->FanGainLiquid = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainLiquid));
+	dpm_table->FanGainVrVddc = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainVrVddc));
+	dpm_table->FanGainVrMvdd = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainVrMvdd));
+	dpm_table->FanGainPlx = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainPlx));
+	dpm_table->FanGainHbm = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHbm));
+
+	dpm_table->Liquid1_I2C_address = cac_dtp_table->ucLiquid1_I2C_address;
+	dpm_table->Liquid2_I2C_address = cac_dtp_table->ucLiquid2_I2C_address;
+	dpm_table->Vr_I2C_address = cac_dtp_table->ucVr_I2C_address;
+	dpm_table->Plx_I2C_address = cac_dtp_table->ucPlx_I2C_address;
+
+	get_scl_sda_value(cac_dtp_table->ucLiquid_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Liquid_I2C_LineSCL = uc_scl;
+	dpm_table->Liquid_I2C_LineSDA = uc_sda;
+
+	get_scl_sda_value(cac_dtp_table->ucVr_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Vr_I2C_LineSCL = uc_scl;
+	dpm_table->Vr_I2C_LineSDA = uc_sda;
+
+	get_scl_sda_value(cac_dtp_table->ucPlx_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Plx_I2C_LineSCL = uc_scl;
+	dpm_table->Plx_I2C_LineSDA = uc_sda;
+
+	return 0;
+}
+
+static int fiji_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+    struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+    struct fiji_pt_defaults *defaults = data->power_tune_defaults;
+
+    data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
+    data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
+    data->power_tune_table.SviLoadLineTrimVddC = 3;
+    data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+    return 0;
+}
+
+static int fiji_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct  fiji_pt_defaults *defaults = data->power_tune_defaults;
+
+	/* TDC number of fraction bits are changed from 8 to 7
+	 * for Fiji as requested by SMC team
+	 */
+	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
+	data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
+	data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
+
+	return 0;
+}
+
+static int fiji_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct  fiji_pt_defaults *defaults = data->power_tune_defaults;
+	uint32_t temp;
+
+	if (fiji_read_smc_sram_dword(hwmgr->smumgr,
+			fuse_table_offset +
+			offsetof(SMU73_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, data->sram_end))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
+				return -EINVAL);
+	else {
+		data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
+		data->power_tune_table.LPMLTemperatureMin =
+				(uint8_t)((temp >> 16) & 0xff);
+		data->power_tune_table.LPMLTemperatureMax =
+				(uint8_t)((temp >> 8) & 0xff);
+		data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
+	}
+	return 0;
+}
+
+static int fiji_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		data->power_tune_table.LPMLTemperatureScaler[i] = 0;
+
+	return 0;
+}
+
+static int fiji_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if( (hwmgr->thermal_controller.advanceFanControlParameters.
+			usFanOutputSensitivity & (1 << 15)) ||
+			0 == hwmgr->thermal_controller.advanceFanControlParameters.
+			usFanOutputSensitivity )
+		hwmgr->thermal_controller.advanceFanControlParameters.
+		usFanOutputSensitivity = hwmgr->thermal_controller.
+			advanceFanControlParameters.usDefaultFanOutputSensitivity;
+
+	data->power_tune_table.FuzzyFan_PwmSetDelta =
+			PP_HOST_TO_SMC_US(hwmgr->thermal_controller.
+					advanceFanControlParameters.usFanOutputSensitivity);
+	return 0;
+}
+
+static int fiji_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int fiji_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
+{
+ /*   int  i, min, max;
+    struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+    uint8_t * pHiVID = data->power_tune_table.BapmVddCVidHiSidd;
+    uint8_t * pLoVID = data->power_tune_table.BapmVddCVidLoSidd;
+
+    min = max = pHiVID[0];
+    for (i = 0; i < 8; i++) {
+        if (0 != pHiVID[i]) {
+            if (min > pHiVID[i])
+                min = pHiVID[i];
+            if (max < pHiVID[i])
+                max = pHiVID[i];
+        }
+
+        if (0 != pLoVID[i]) {
+            if (min > pLoVID[i])
+                min = pLoVID[i];
+            if (max < pLoVID[i])
+                max = pLoVID[i];
+        }
+    }
+
+    PP_ASSERT_WITH_CODE((0 != min) && (0 != max), "BapmVddcVidSidd table does not exist!", return int_Failed);
+    data->power_tune_table.GnbLPMLMaxVid = (uint8_t)max;
+    data->power_tune_table.GnbLPMLMinVid = (uint8_t)min;
+*/
+    return 0;
+}
+
+static int fiji_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint16_t HiSidd = data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t LoSidd = data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+
+	HiSidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	LoSidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(HiSidd);
+	data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(LoSidd);
+
+	return 0;
+}
+
+int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	uint32_t pm_fuse_table_offset;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (fiji_read_smc_sram_dword(hwmgr->smumgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU73_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, data->sram_end))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to get pm_fuse_table_offset Failed!",
+					return -EINVAL);
+
+		/* DW6 */
+		if (fiji_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate SviLoadLine Failed!",
+					return -EINVAL);
+		/* DW7 */
+		if (fiji_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed!", return -EINVAL);
+		/* DW8 */
+		if (fiji_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TdcWaterfallCtl, "
+					"LPMLTemperature Min and Max Failed!",
+					return -EINVAL);
+
+		/* DW9-DW12 */
+		if (0 != fiji_populate_temperature_scaler(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate LPMLTemperatureScaler Failed!",
+					return -EINVAL);
+
+		/* DW13-DW14 */
+		if(fiji_populate_fuzzy_fan(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate Fuzzy Fan Control parameters Failed!",
+					return -EINVAL);
+
+		/* DW15-DW18 */
+		if (fiji_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Failed!",
+					return -EINVAL);
+
+		/* DW19 */
+		if (fiji_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Min and Max Vid Failed!",
+					return -EINVAL);
+
+		/* DW20 */
+		if (fiji_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
+					"Sidd Failed!", return -EINVAL);
+
+		if (fiji_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
+				(uint8_t *)&data->power_tune_table,
+				sizeof(struct SMU73_Discrete_PmFuses), data->sram_end))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed!",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	int result = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC)) {
+		int smc_result;
+		smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+				(uint16_t)(PPSMC_MSG_EnableCac));
+		PP_ASSERT_WITH_CODE((0 == smc_result),
+				"Failed to enable CAC in SMC.", result = -1);
+
+        data->cac_enabled = (0 == smc_result) ? true : false;
+	}
+	return result;
+}
+
+int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+	if(data->power_containment_features &
+			POWERCONTAINMENT_FEATURE_PkgPwrLimit)
+		return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_PkgPwrSetLimit, n);
+	return 0;
+}
+
+static int fiji_set_overdriver_target_tdp(struct pp_hwmgr *pHwMgr, uint32_t target_tdp)
+{
+	return smum_send_msg_to_smc_with_parameter(pHwMgr->smumgr,
+			PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
+}
+
+int fiji_enable_power_containment(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int smc_result;
+	int result = 0;
+
+	data->power_containment_features = 0;
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (data->enable_dte_feature) {
+			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+					(uint16_t)(PPSMC_MSG_EnableDTE));
+			PP_ASSERT_WITH_CODE((0 == smc_result),
+					"Failed to enable DTE in SMC.", result = -1;);
+			if (0 == smc_result)
+				data->power_containment_features |= POWERCONTAINMENT_FEATURE_DTE;
+		}
+
+		if (data->enable_tdc_limit_feature) {
+			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+					(uint16_t)(PPSMC_MSG_TDCLimitEnable));
+			PP_ASSERT_WITH_CODE((0 == smc_result),
+					"Failed to enable TDCLimit in SMC.", result = -1;);
+			if (0 == smc_result)
+				data->power_containment_features |=
+						POWERCONTAINMENT_FEATURE_TDCLimit;
+		}
+
+		if (data->enable_pkg_pwr_tracking_feature) {
+			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+					(uint16_t)(PPSMC_MSG_PkgPwrLimitEnable));
+			PP_ASSERT_WITH_CODE((0 == smc_result),
+					"Failed to enable PkgPwrTracking in SMC.", result = -1;);
+			if (0 == smc_result) {
+				struct phm_cac_tdp_table *cac_table =
+						table_info->cac_dtp_table;
+				uint32_t default_limit =
+					(uint32_t)(cac_table->usMaximumPowerDeliveryLimit * 256);
+
+				data->power_containment_features |=
+						POWERCONTAINMENT_FEATURE_PkgPwrLimit;
+
+				if (fiji_set_power_limit(hwmgr, default_limit))
+					printk(KERN_ERR "Failed to set Default Power Limit in SMC!");
+			}
+		}
+	}
+	return result;
+}
+
+int fiji_power_control_set_level(struct pp_hwmgr *hwmgr)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+	int adjust_percent, target_tdp;
+	int result = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		/* adjustment percentage has already been validated */
+		adjust_percent = hwmgr->platform_descriptor.TDPAdjustmentPolarity ?
+				hwmgr->platform_descriptor.TDPAdjustment :
+				(-1 * hwmgr->platform_descriptor.TDPAdjustment);
+		/* SMC requested that target_tdp to be 7 bit fraction in DPM table
+		 * but message to be 8 bit fraction for messages
+		 */
+		target_tdp = ((100 + adjust_percent) * (int)(cac_table->usTDP * 256)) / 100;
+		result = fiji_set_overdriver_target_tdp(hwmgr, (uint32_t)target_tdp);
+	}
+
+	return result;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
new file mode 100644
index 000000000000..55e58200f33a
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef FIJI_POWERTUNE_H
+#define FIJI_POWERTUNE_H
+
+enum fiji_pt_config_reg_type {
+	FIJI_CONFIGREG_MMR = 0,
+	FIJI_CONFIGREG_SMC_IND,
+	FIJI_CONFIGREG_DIDT_IND,
+	FIJI_CONFIGREG_CACHE,
+	FIJI_CONFIGREG_MAX
+};
+
+/* PowerContainment Features */
+#define POWERCONTAINMENT_FEATURE_DTE             0x00000001
+#define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
+#define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
+
+struct fiji_pt_config_reg {
+	uint32_t                           offset;
+	uint32_t                           mask;
+	uint32_t                           shift;
+	uint32_t                           value;
+	enum fiji_pt_config_reg_type       type;
+};
+
+struct fiji_pt_defaults
+{
+    uint8_t   SviLoadLineEn;
+    uint8_t   SviLoadLineVddC;
+    uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
+    uint8_t   TDC_MAWt;
+    uint8_t   TdcWaterfallCtl;
+    uint8_t   DTEAmbientTempBase;
+};
+
+void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr);
+int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr);
+int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr);
+int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr);
+int fiji_enable_power_containment(struct pp_hwmgr *hwmgr);
+int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
+int fiji_power_control_set_level(struct pp_hwmgr *hwmgr);
+
+#endif  /* FIJI_POWERTUNE_H */
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.c
new file mode 100644
index 000000000000..e76a7de9aa32
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.c
@@ -0,0 +1,687 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <asm/div64.h>
+#include "fiji_thermal.h"
+#include "fiji_hwmgr.h"
+#include "fiji_smumgr.h"
+#include "fiji_ppsmc.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+
+int fiji_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
+		struct phm_fan_speed_info *fan_speed_info)
+{
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	fan_speed_info->supports_percent_read = true;
+	fan_speed_info->supports_percent_write = true;
+	fan_speed_info->min_percent = 0;
+	fan_speed_info->max_percent = 100;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_FanSpeedInTableIsRPM) &&
+		hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution) {
+		fan_speed_info->supports_rpm_read = true;
+		fan_speed_info->supports_rpm_write = true;
+		fan_speed_info->min_rpm = hwmgr->thermal_controller.fanInfo.ulMinRPM;
+		fan_speed_info->max_rpm = hwmgr->thermal_controller.fanInfo.ulMaxRPM;
+	} else {
+		fan_speed_info->min_rpm = 0;
+		fan_speed_info->max_rpm = 0;
+	}
+
+	return 0;
+}
+
+int fiji_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
+		uint32_t *speed)
+{
+	uint32_t duty100;
+	uint32_t duty;
+	uint64_t tmp64;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL1, FMAX_DUTY100);
+	duty = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_STATUS, FDO_PWM_DUTY);
+
+	if (duty100 == 0)
+		return -EINVAL;
+
+
+	tmp64 = (uint64_t)duty * 100;
+	do_div(tmp64, duty100);
+	*speed = (uint32_t)tmp64;
+
+	if (*speed > 100)
+		*speed = 100;
+
+	return 0;
+}
+
+int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
+{
+	uint32_t tach_period;
+	uint32_t crystal_clock_freq;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan ||
+			(hwmgr->thermal_controller.fanInfo.
+				ucTachometerPulsesPerRevolution == 0))
+		return 0;
+
+	tach_period = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_TACH_STATUS, TACH_PERIOD);
+
+	if (tach_period == 0)
+		return -EINVAL;
+
+	crystal_clock_freq = tonga_get_xclk(hwmgr);
+
+	*speed = 60 * crystal_clock_freq * 10000/ tach_period;
+
+	return 0;
+}
+
+/**
+* Set Fan Speed Control to static mode, so that the user can decide what speed to use.
+* @param    hwmgr  the address of the powerplay hardware manager.
+*           mode    the fan control mode, 0 default, 1 by percent, 5, by RPM
+* @exception Should always succeed.
+*/
+int fiji_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+{
+
+	if (hwmgr->fan_ctrl_is_in_default_mode) {
+		hwmgr->fan_ctrl_default_mode =
+				PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,	CGS_IND_REG__SMC,
+						CG_FDO_CTRL2, FDO_PWM_MODE);
+		hwmgr->tmin =
+				PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+						CG_FDO_CTRL2, TMIN);
+		hwmgr->fan_ctrl_is_in_default_mode = false;
+	}
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL2, TMIN, 0);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL2, FDO_PWM_MODE, mode);
+
+	return 0;
+}
+
+/**
+* Reset Fan Speed Control to default mode.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @exception Should always succeed.
+*/
+int fiji_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
+{
+	if (!hwmgr->fan_ctrl_is_in_default_mode) {
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_FDO_CTRL2, FDO_PWM_MODE, hwmgr->fan_ctrl_default_mode);
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_FDO_CTRL2, TMIN, hwmgr->tmin);
+		hwmgr->fan_ctrl_is_in_default_mode = true;
+	}
+
+	return 0;
+}
+
+int fiji_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ODFuzzyFanControlSupport)) {
+		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_FUZZY);
+		result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl);
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_FanSpeedInTableIsRPM))
+			hwmgr->hwmgr_func->set_max_fan_rpm_output(hwmgr,
+					hwmgr->thermal_controller.
+					advanceFanControlParameters.usMaxFanRPM);
+		else
+			hwmgr->hwmgr_func->set_max_fan_pwm_output(hwmgr,
+					hwmgr->thermal_controller.
+					advanceFanControlParameters.usMaxFanPWM);
+
+	} else {
+		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_TABLE);
+		result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl);
+	}
+
+	if (!result && hwmgr->thermal_controller.
+			advanceFanControlParameters.ucTargetTemperature)
+		result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanTemperatureTarget,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ucTargetTemperature);
+
+	return result;
+}
+
+
+int fiji_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StopFanControl);
+}
+
+/**
+* Set Fan Speed in percent.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    speed is the percentage value (0% - 100%) to be set.
+* @exception Fails is the 100% setting appears to be 0.
+*/
+int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
+		uint32_t speed)
+{
+	uint32_t duty100;
+	uint32_t duty;
+	uint64_t tmp64;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	if (speed > 100)
+		speed = 100;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl))
+		fiji_fan_ctrl_stop_smc_fan_control(hwmgr);
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (duty100 == 0)
+		return -EINVAL;
+
+	tmp64 = (uint64_t)speed * 100;
+	do_div(tmp64, duty100);
+	duty = (uint32_t)tmp64;
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL0, FDO_STATIC_DUTY, duty);
+
+	return fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+}
+
+/**
+* Reset Fan Speed to default.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @exception Always succeeds.
+*/
+int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl)) {
+		result = fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+		if (!result)
+			result = fiji_fan_ctrl_start_smc_fan_control(hwmgr);
+	} else
+		result = fiji_fan_ctrl_set_default_mode(hwmgr);
+
+	return result;
+}
+
+/**
+* Set Fan Speed in RPM.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    speed is the percentage value (min - max) to be set.
+* @exception Fails is the speed not lie between min and max.
+*/
+int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
+{
+	uint32_t tach_period;
+	uint32_t crystal_clock_freq;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan ||
+			(hwmgr->thermal_controller.fanInfo.
+			ucTachometerPulsesPerRevolution == 0) ||
+			(speed < hwmgr->thermal_controller.fanInfo.ulMinRPM) ||
+			(speed > hwmgr->thermal_controller.fanInfo.ulMaxRPM))
+		return 0;
+
+	crystal_clock_freq = tonga_get_xclk(hwmgr);
+
+	tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_TACH_STATUS, TACH_PERIOD, tach_period);
+
+	return fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+}
+
+/**
+* Reads the remote temperature from the SIslands thermal controller.
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr)
+{
+	int temp;
+
+	temp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_MULT_THERMAL_STATUS, CTF_TEMP);
+
+	/* Bit 9 means the reading is lower than the lowest usable value. */
+	if (temp & 0x200)
+		temp = FIJI_THERMAL_MAXIMUM_TEMP_READING;
+	else
+		temp = temp & 0x1ff;
+
+	temp *= PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	return temp;
+}
+
+/**
+* Set the requested temperature range for high and low alert signals
+*
+* @param    hwmgr The address of the hardware manager.
+* @param    range Temperature range to be programmed for high and low alert signals
+* @exception PP_Result_BadInput if the input data is not valid.
+*/
+static int fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
+		uint32_t low_temp, uint32_t high_temp)
+{
+	uint32_t low = FIJI_THERMAL_MINIMUM_ALERT_TEMP *
+			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	uint32_t high = FIJI_THERMAL_MAXIMUM_ALERT_TEMP *
+			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	if (low < low_temp)
+		low = low_temp;
+	if (high > high_temp)
+		high = high_temp;
+
+	if (low > high)
+		return -EINVAL;
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, DIG_THERM_INTH,
+			(high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, DIG_THERM_INTL,
+			(low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_CTRL, DIG_THERM_DPM,
+			(high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+
+	return 0;
+}
+
+/**
+* Programs thermal controller one-time setting registers
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+static int fiji_thermal_initialize(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution)
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_TACH_CTRL, EDGE_PER_REV,
+				hwmgr->thermal_controller.fanInfo.
+				ucTachometerPulsesPerRevolution - 1);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL2, TACH_PWM_RESP_RATE, 0x28);
+
+	return 0;
+}
+
+/**
+* Enable thermal alerts on the RV770 thermal controller.
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+static int fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr)
+{
+	uint32_t alert;
+
+	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, THERM_INT_MASK);
+	alert &= ~(FIJI_THERMAL_HIGH_ALERT_MASK | FIJI_THERMAL_LOW_ALERT_MASK);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, THERM_INT_MASK, alert);
+
+	/* send message to SMU to enable internal thermal interrupts */
+	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Enable);
+}
+
+/**
+* Disable thermal alerts on the RV770 thermal controller.
+* @param    hwmgr The address of the hardware manager.
+*/
+static int fiji_thermal_disable_alert(struct pp_hwmgr *hwmgr)
+{
+	uint32_t alert;
+
+	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, THERM_INT_MASK);
+	alert |= (FIJI_THERMAL_HIGH_ALERT_MASK | FIJI_THERMAL_LOW_ALERT_MASK);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_THERMAL_INT, THERM_INT_MASK, alert);
+
+	/* send message to SMU to disable internal thermal interrupts */
+	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Disable);
+}
+
+/**
+* Uninitialize the thermal controller.
+* Currently just disables alerts.
+* @param    hwmgr The address of the hardware manager.
+*/
+int fiji_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
+{
+	int result = fiji_thermal_disable_alert(hwmgr);
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		fiji_fan_ctrl_set_default_mode(hwmgr);
+
+	return result;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+	SMU73_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (data->fan_table_start == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (duty100 == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.
+			usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->
+			thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = tonga_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->
+			thermal_controller.advanceFanControlParameters.ulCycleDelay *
+			reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(
+			hwmgr->device, CGS_IND_REG__SMC,
+			CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	res = fiji_copy_bytes_to_smc(hwmgr->smumgr, data->fan_table_start,
+			(uint8_t *)&fan_table, (uint32_t)sizeof(fan_table),
+			data->sram_end);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ucMinimumPWMLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanMinPwm,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ucMinimumPWMLimit);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ulMinFanSCLKAcousticLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanSclkTarget,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ulMinFanSCLKAcousticLimit);
+
+	if (res)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+
+	return 0;
+}
+
+/**
+* Start the fan control on the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_fiji_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+/* If the fantable setup has failed we could have disabled
+ * PHM_PlatformCaps_MicrocodeFanControl even after
+ * this function was included in the table.
+ * Make sure that we still think controlling the fan is OK.
+*/
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl)) {
+		fiji_fan_ctrl_start_smc_fan_control(hwmgr);
+		fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+	}
+
+	return 0;
+}
+
+/**
+* Set temperature range for high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	struct PP_TemperatureRange *range = (struct PP_TemperatureRange *)input;
+
+	if (range == NULL)
+		return -EINVAL;
+
+	return fiji_thermal_set_temperature_range(hwmgr, range->min, range->max);
+}
+
+/**
+* Programs one-time setting registers
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from initialize thermal controller routine
+*/
+int tf_fiji_thermal_initialize(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+    return fiji_thermal_initialize(hwmgr);
+}
+
+/**
+* Enable high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from enable alert routine
+*/
+int tf_fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	return fiji_thermal_enable_alert(hwmgr);
+}
+
+/**
+* Disable high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from disable alert routine
+*/
+static int tf_fiji_thermal_disable_alert(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	return fiji_thermal_disable_alert(hwmgr);
+}
+
+static struct phm_master_table_item
+fiji_thermal_start_thermal_controller_master_list[] = {
+	{NULL, tf_fiji_thermal_initialize},
+	{NULL, tf_fiji_thermal_set_temperature_range},
+	{NULL, tf_fiji_thermal_enable_alert},
+/* We should restrict performance levels to low before we halt the SMC.
+ * On the other hand we are still in boot state when we do this
+ * so it would be pointless.
+ * If this assumption changes we have to revisit this table.
+ */
+	{NULL, tf_fiji_thermal_setup_fan_table},
+	{NULL, tf_fiji_thermal_start_smc_fan_control},
+	{NULL, NULL}
+};
+
+static struct phm_master_table_header
+fiji_thermal_start_thermal_controller_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	fiji_thermal_start_thermal_controller_master_list
+};
+
+static struct phm_master_table_item
+fiji_thermal_set_temperature_range_master_list[] = {
+	{NULL, tf_fiji_thermal_disable_alert},
+	{NULL, tf_fiji_thermal_set_temperature_range},
+	{NULL, tf_fiji_thermal_enable_alert},
+	{NULL, NULL}
+};
+
+struct phm_master_table_header
+fiji_thermal_set_temperature_range_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	fiji_thermal_set_temperature_range_master_list
+};
+
+int fiji_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
+{
+	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
+		fiji_fan_ctrl_set_default_mode(hwmgr);
+	return 0;
+}
+
+/**
+* Initializes the thermal controller related functions in the Hardware Manager structure.
+* @param    hwmgr The address of the hardware manager.
+* @exception Any error code from the low-level communication.
+*/
+int pp_fiji_thermal_initialize(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	result = phm_construct_table(hwmgr,
+			&fiji_thermal_set_temperature_range_master,
+			&(hwmgr->set_temperature_range));
+
+	if (!result) {
+		result = phm_construct_table(hwmgr,
+				&fiji_thermal_start_thermal_controller_master,
+				&(hwmgr->start_thermal_controller));
+		if (result)
+			phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
+	}
+
+	if (!result)
+		hwmgr->fan_ctrl_is_in_default_mode = true;
+	return result;
+}
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h
new file mode 100644
index 000000000000..8621493b8574
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef FIJI_THERMAL_H
+#define FIJI_THERMAL_H
+
+#include "hwmgr.h"
+
+#define FIJI_THERMAL_HIGH_ALERT_MASK         0x1
+#define FIJI_THERMAL_LOW_ALERT_MASK          0x2
+
+#define FIJI_THERMAL_MINIMUM_TEMP_READING    -256
+#define FIJI_THERMAL_MAXIMUM_TEMP_READING    255
+
+#define FIJI_THERMAL_MINIMUM_ALERT_TEMP      0
+#define FIJI_THERMAL_MAXIMUM_ALERT_TEMP      255
+
+#define FDO_PWM_MODE_STATIC  1
+#define FDO_PWM_MODE_STATIC_RPM 5
+
+
+extern int tf_fiji_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+extern int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+extern int tf_fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+
+extern int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr);
+extern int fiji_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int fiji_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
+extern int fiji_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int fiji_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
+extern int fiji_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
+extern int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
+extern int pp_fiji_thermal_initialize(struct pp_hwmgr *hwmgr);
+extern int fiji_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int fiji_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
+extern uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
new file mode 100644
index 000000000000..9deadabbc81c
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "hwmgr.h"
+
+static int phm_run_table(struct pp_hwmgr *hwmgr,
+			 struct phm_runtime_table_header *rt_table,
+			 void *input,
+			 void *output,
+			 void *temp_storage)
+{
+	int result = 0;
+	phm_table_function *function;
+
+	for (function = rt_table->function_list; NULL != *function; function++) {
+		int tmp = (*function)(hwmgr, input, output, temp_storage, result);
+
+		if (tmp == PP_Result_TableImmediateExit)
+			break;
+		if (tmp) {
+			if (0 == result)
+				result = tmp;
+			if (rt_table->exit_error)
+				break;
+		}
+	}
+
+	return result;
+}
+
+int phm_dispatch_table(struct pp_hwmgr *hwmgr,
+		       struct phm_runtime_table_header *rt_table,
+		       void *input, void *output)
+{
+	int result = 0;
+	void *temp_storage = NULL;
+
+	if (hwmgr == NULL || rt_table == NULL || rt_table->function_list == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
+		return 0; /*temp return ture because some function not implement on some asic */
+	}
+
+	if (0 != rt_table->storage_size) {
+		temp_storage = kzalloc(rt_table->storage_size, GFP_KERNEL);
+		if (temp_storage == NULL) {
+			printk(KERN_ERR "[ powerplay ] Could not allocate table temporary storage\n");
+			return -ENOMEM;
+		}
+	}
+
+	result = phm_run_table(hwmgr, rt_table, input, output, temp_storage);
+
+	if (NULL != temp_storage)
+		kfree(temp_storage);
+
+	return result;
+}
+
+int phm_construct_table(struct pp_hwmgr *hwmgr,
+			struct phm_master_table_header *master_table,
+			struct phm_runtime_table_header *rt_table)
+{
+	uint32_t function_count = 0;
+	const struct phm_master_table_item *table_item;
+	uint32_t size;
+	phm_table_function *run_time_list;
+	phm_table_function *rtf;
+
+	if (hwmgr == NULL || master_table == NULL || rt_table == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
+		return -EINVAL;
+	}
+
+	for (table_item = master_table->master_list;
+		NULL != table_item->tableFunction; table_item++) {
+		if ((NULL == table_item->isFunctionNeededInRuntimeTable) ||
+		    (table_item->isFunctionNeededInRuntimeTable(hwmgr)))
+			function_count++;
+	}
+
+	size = (function_count + 1) * sizeof(phm_table_function);
+	run_time_list = kzalloc(size, GFP_KERNEL);
+
+	if (NULL == run_time_list)
+		return -ENOMEM;
+
+	rtf = run_time_list;
+	for (table_item = master_table->master_list;
+		NULL != table_item->tableFunction; table_item++) {
+		if ((rtf - run_time_list) > function_count) {
+			printk(KERN_ERR "[ powerplay ] Check function results have changed\n");
+			kfree(run_time_list);
+			return -EINVAL;
+		}
+
+		if ((NULL == table_item->isFunctionNeededInRuntimeTable) ||
+		     (table_item->isFunctionNeededInRuntimeTable(hwmgr))) {
+			*(rtf++) = table_item->tableFunction;
+		}
+	}
+
+	if ((rtf - run_time_list) > function_count) {
+		printk(KERN_ERR "[ powerplay ] Check function results have changed\n");
+		kfree(run_time_list);
+		return -EINVAL;
+	}
+
+	*rtf = NULL;
+	rt_table->function_list = run_time_list;
+	rt_table->exit_error = (0 != (master_table->flags & PHM_MasterTableFlag_ExitOnError));
+	rt_table->storage_size = master_table->storage_size;
+	return 0;
+}
+
+int phm_destroy_table(struct pp_hwmgr *hwmgr,
+		      struct phm_runtime_table_header *rt_table)
+{
+	if (hwmgr == NULL || rt_table == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Parameter\n");
+		return -EINVAL;
+	}
+
+	if (NULL == rt_table->function_list)
+		return 0;
+
+	kfree(rt_table->function_list);
+
+	rt_table->function_list = NULL;
+	rt_table->storage_size = 0;
+	rt_table->exit_error = false;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
new file mode 100644
index 000000000000..0f2d5e4bc241
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -0,0 +1,334 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/errno.h>
+#include "hwmgr.h"
+#include "hardwaremanager.h"
+#include "power_state.h"
+#include "pp_acpi.h"
+#include "amd_acpi.h"
+#include "amd_powerplay.h"
+
+#define PHM_FUNC_CHECK(hw) \
+	do {							\
+		if ((hw) == NULL || (hw)->hwmgr_func == NULL)	\
+			return -EINVAL;				\
+	} while (0)
+
+void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr)
+{
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableVoltageTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableEngineTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMemoryTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGClockGating);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGCGTSSM);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLSClockGating);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_Force3DClockSupport);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLightSleep);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMCLS);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisablePowerGating);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableDPM);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableSMUUVDHandshake);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ThermalAutoThrottling);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_NoOD5Support);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UserMaxClockForMultiDisplays);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VpuRecoveryInProgress);
+
+	if (acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST) &&
+		acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION))
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
+}
+
+bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr)
+{
+	return hwmgr->block_hw_access;
+}
+
+int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block)
+{
+	hwmgr->block_hw_access = block;
+	return 0;
+}
+
+int phm_setup_asic(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->asic_setup)
+			return hwmgr->hwmgr_func->asic_setup(hwmgr);
+	} else {
+		return phm_dispatch_table(hwmgr, &(hwmgr->setup_asic),
+					  NULL, NULL);
+	}
+
+	return 0;
+}
+
+int phm_power_down_asic(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->power_off_asic)
+			return hwmgr->hwmgr_func->power_off_asic(hwmgr);
+	} else {
+		return phm_dispatch_table(hwmgr, &(hwmgr->power_down_asic),
+					  NULL, NULL);
+	}
+
+	return 0;
+}
+
+int phm_set_power_state(struct pp_hwmgr *hwmgr,
+		    const struct pp_hw_power_state *pcurrent_state,
+		    const struct pp_hw_power_state *pnew_power_state)
+{
+	struct phm_set_power_state_input states;
+
+	PHM_FUNC_CHECK(hwmgr);
+
+	states.pcurrent_state = pcurrent_state;
+	states.pnew_state = pnew_power_state;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->power_state_set)
+			return hwmgr->hwmgr_func->power_state_set(hwmgr, &states);
+	} else {
+		return phm_dispatch_table(hwmgr, &(hwmgr->set_power_state), &states, NULL);
+	}
+
+	return 0;
+}
+
+int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->dynamic_state_management_enable)
+			return hwmgr->hwmgr_func->dynamic_state_management_enable(hwmgr);
+	} else {
+		return phm_dispatch_table(hwmgr,
+				&(hwmgr->enable_dynamic_state_management),
+				NULL, NULL);
+	}
+	return 0;
+}
+
+int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->force_dpm_level != NULL)
+		return hwmgr->hwmgr_func->force_dpm_level(hwmgr, level);
+
+	return 0;
+}
+
+int phm_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				   struct pp_power_state *adjusted_ps,
+			     const struct pp_power_state *current_ps)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->apply_state_adjust_rules != NULL)
+		return hwmgr->hwmgr_func->apply_state_adjust_rules(
+									hwmgr,
+								 adjusted_ps,
+								 current_ps);
+	return 0;
+}
+
+int phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->powerdown_uvd != NULL)
+		return hwmgr->hwmgr_func->powerdown_uvd(hwmgr);
+	return 0;
+}
+
+int phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool gate)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->powergate_uvd != NULL)
+		return hwmgr->hwmgr_func->powergate_uvd(hwmgr, gate);
+	return 0;
+}
+
+int phm_powergate_vce(struct pp_hwmgr *hwmgr, bool gate)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->powergate_vce != NULL)
+		return hwmgr->hwmgr_func->powergate_vce(hwmgr, gate);
+	return 0;
+}
+
+int phm_enable_clock_power_gatings(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->enable_clock_power_gating)
+			return hwmgr->hwmgr_func->enable_clock_power_gating(hwmgr);
+	} else {
+		return phm_dispatch_table(hwmgr, &(hwmgr->enable_clock_power_gatings), NULL, NULL);
+	}
+	return 0;
+}
+
+int phm_display_configuration_changed(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				 PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->display_config_changed)
+			hwmgr->hwmgr_func->display_config_changed(hwmgr);
+	} else
+		return phm_dispatch_table(hwmgr, &hwmgr->display_configuration_changed, NULL, NULL);
+	return 0;
+}
+
+int phm_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				 PHM_PlatformCaps_TablelessHardwareInterface))
+		if (NULL != hwmgr->hwmgr_func->notify_smc_display_config_after_ps_adjustment)
+			hwmgr->hwmgr_func->notify_smc_display_config_after_ps_adjustment(hwmgr);
+
+	return 0;
+}
+
+int phm_stop_thermal_controller(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->stop_thermal_controller == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->stop_thermal_controller(hwmgr);
+}
+
+int phm_register_thermal_interrupt(struct pp_hwmgr *hwmgr, const void *info)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->register_internal_thermal_interrupt == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->register_internal_thermal_interrupt(hwmgr, info);
+}
+
+/**
+* Initializes the thermal controller subsystem.
+*
+* @param    pHwMgr  the address of the powerplay hardware manager.
+* @param    pTemperatureRange the address of the structure holding the temperature range.
+* @exception PP_Result_Failed if any of the paramters is NULL, otherwise the return value from the dispatcher.
+*/
+int phm_start_thermal_controller(struct pp_hwmgr *hwmgr, struct PP_TemperatureRange *temperature_range)
+{
+	return phm_dispatch_table(hwmgr, &(hwmgr->start_thermal_controller), temperature_range, NULL);
+}
+
+
+bool phm_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->check_smc_update_required_for_display_configuration == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->check_smc_update_required_for_display_configuration(hwmgr);
+}
+
+
+int phm_check_states_equal(struct pp_hwmgr *hwmgr,
+				 const struct pp_hw_power_state *pstate1,
+				 const struct pp_hw_power_state *pstate2,
+				 bool *equal)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->check_states_equal == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->check_states_equal(hwmgr, pstate1, pstate2, equal);
+}
+
+int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr,
+		    const struct amd_pp_display_configuration *display_config)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->store_cc6_data == NULL)
+		return -EINVAL;
+
+	hwmgr->display_config = *display_config;
+	/* to do pass other display configuration in furture */
+
+	if (hwmgr->hwmgr_func->store_cc6_data)
+		hwmgr->hwmgr_func->store_cc6_data(hwmgr,
+				display_config->cpu_pstate_separation_time,
+				display_config->cpu_cc6_disable,
+				display_config->cpu_pstate_disable,
+				display_config->nb_pstate_switch_disable);
+
+	return 0;
+}
+
+int phm_get_dal_power_level(struct pp_hwmgr *hwmgr,
+		struct amd_pp_dal_clock_info *info)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (info == NULL || hwmgr->hwmgr_func->get_dal_power_level == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_dal_power_level(hwmgr, info);
+}
+
+int phm_set_cpu_power_state(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->set_cpu_power_state != NULL)
+		return hwmgr->hwmgr_func->set_cpu_power_state(hwmgr);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
new file mode 100644
index 000000000000..5fb98aa2e719
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -0,0 +1,563 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "linux/delay.h"
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "cgs_common.h"
+#include "power_state.h"
+#include "hwmgr.h"
+#include "pppcielanes.h"
+#include "pp_debug.h"
+#include "ppatomctrl.h"
+
+extern int cz_hwmgr_init(struct pp_hwmgr *hwmgr);
+extern int tonga_hwmgr_init(struct pp_hwmgr *hwmgr);
+extern int fiji_hwmgr_init(struct pp_hwmgr *hwmgr);
+
+int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
+{
+	struct pp_hwmgr *hwmgr;
+
+	if ((handle == NULL) || (pp_init == NULL))
+		return -EINVAL;
+
+	hwmgr = kzalloc(sizeof(struct pp_hwmgr), GFP_KERNEL);
+	if (hwmgr == NULL)
+		return -ENOMEM;
+
+	handle->hwmgr = hwmgr;
+	hwmgr->smumgr = handle->smu_mgr;
+	hwmgr->device = pp_init->device;
+	hwmgr->chip_family = pp_init->chip_family;
+	hwmgr->chip_id = pp_init->chip_id;
+	hwmgr->hw_revision = pp_init->rev_id;
+	hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
+	hwmgr->power_source = PP_PowerSource_AC;
+
+	switch (hwmgr->chip_family) {
+	case AMD_FAMILY_CZ:
+		cz_hwmgr_init(hwmgr);
+		break;
+	case AMD_FAMILY_VI:
+		switch (hwmgr->chip_id) {
+		case CHIP_TONGA:
+			tonga_hwmgr_init(hwmgr);
+			break;
+		case CHIP_FIJI:
+			fiji_hwmgr_init(hwmgr);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	phm_init_dynamic_caps(hwmgr);
+
+	return 0;
+}
+
+int hwmgr_fini(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr == NULL || hwmgr->ps == NULL)
+		return -EINVAL;
+
+	kfree(hwmgr->ps);
+	kfree(hwmgr);
+	return 0;
+}
+
+int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	unsigned int i;
+	unsigned int table_entries;
+	struct pp_power_state *state;
+	int size;
+
+	if (hwmgr->hwmgr_func->get_num_of_pp_table_entries == NULL)
+		return -EINVAL;
+
+	if (hwmgr->hwmgr_func->get_power_state_size == NULL)
+		return -EINVAL;
+
+	hwmgr->num_ps = table_entries = hwmgr->hwmgr_func->get_num_of_pp_table_entries(hwmgr);
+
+	hwmgr->ps_size = size = hwmgr->hwmgr_func->get_power_state_size(hwmgr) +
+					  sizeof(struct pp_power_state);
+
+	hwmgr->ps = kzalloc(size * table_entries, GFP_KERNEL);
+
+	if (hwmgr->ps == NULL)
+		return -ENOMEM;
+
+	state = hwmgr->ps;
+
+	for (i = 0; i < table_entries; i++) {
+		result = hwmgr->hwmgr_func->get_pp_table_entry(hwmgr, i, state);
+
+		if (state->classification.flags & PP_StateClassificationFlag_Boot) {
+			hwmgr->boot_ps = state;
+			hwmgr->current_ps = hwmgr->request_ps = state;
+		}
+
+		state->id = i + 1; /* assigned unique num for every power state id */
+
+		if (state->classification.flags & PP_StateClassificationFlag_Uvd)
+			hwmgr->uvd_ps = state;
+		state = (struct pp_power_state *)((unsigned long)state + size);
+	}
+
+	return 0;
+}
+
+
+/**
+ * Returns once the part of the register indicated by the mask has
+ * reached the given value.
+ */
+int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
+			 uint32_t value, uint32_t mask)
+{
+	uint32_t i;
+	uint32_t cur_value;
+
+	if (hwmgr == NULL || hwmgr->device == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < hwmgr->usec_timeout; i++) {
+		cur_value = cgs_read_register(hwmgr->device, index);
+		if ((cur_value & mask) == (value & mask))
+			break;
+		udelay(1);
+	}
+
+	/* timeout means wrong logic*/
+	if (i == hwmgr->usec_timeout)
+		return -1;
+	return 0;
+}
+
+int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr,
+				uint32_t index, uint32_t value, uint32_t mask)
+{
+	uint32_t i;
+	uint32_t cur_value;
+
+	if (hwmgr == NULL || hwmgr->device == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < hwmgr->usec_timeout; i++) {
+		cur_value = cgs_read_register(hwmgr->device, index);
+		if ((cur_value & mask) != (value & mask))
+			break;
+		udelay(1);
+	}
+
+	/* timeout means wrong logic*/
+	if (i == hwmgr->usec_timeout)
+		return -1;
+	return 0;
+}
+
+
+/**
+ * Returns once the part of the register indicated by the mask has
+ * reached the given value.The indirect space is described by giving
+ * the memory-mapped index of the indirect index register.
+ */
+void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
+				uint32_t indirect_port,
+				uint32_t index,
+				uint32_t value,
+				uint32_t mask)
+{
+	if (hwmgr == NULL || hwmgr->device == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+		return;
+	}
+
+	cgs_write_register(hwmgr->device, indirect_port, index);
+	phm_wait_on_register(hwmgr, indirect_port + 1, mask, value);
+}
+
+void phm_wait_for_indirect_register_unequal(struct pp_hwmgr *hwmgr,
+					uint32_t indirect_port,
+					uint32_t index,
+					uint32_t value,
+					uint32_t mask)
+{
+	if (hwmgr == NULL || hwmgr->device == NULL) {
+		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+		return;
+	}
+
+	cgs_write_register(hwmgr->device, indirect_port, index);
+	phm_wait_for_register_unequal(hwmgr, indirect_port + 1,
+				      value, mask);
+}
+
+bool phm_cf_want_uvd_power_gating(struct pp_hwmgr *hwmgr)
+{
+	return phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDPowerGating);
+}
+
+bool phm_cf_want_vce_power_gating(struct pp_hwmgr *hwmgr)
+{
+	return phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VCEPowerGating);
+}
+
+
+int phm_trim_voltage_table(struct pp_atomctrl_voltage_table *vol_table)
+{
+	uint32_t i, j;
+	uint16_t vvalue;
+	bool found = false;
+	struct pp_atomctrl_voltage_table *table;
+
+	PP_ASSERT_WITH_CODE((NULL != vol_table),
+			"Voltage Table empty.", return -EINVAL);
+
+	table = kzalloc(sizeof(struct pp_atomctrl_voltage_table),
+			GFP_KERNEL);
+
+	if (NULL == table)
+		return -EINVAL;
+
+	table->mask_low = vol_table->mask_low;
+	table->phase_delay = vol_table->phase_delay;
+
+	for (i = 0; i < vol_table->count; i++) {
+		vvalue = vol_table->entries[i].value;
+		found = false;
+
+		for (j = 0; j < table->count; j++) {
+			if (vvalue == table->entries[j].value) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			table->entries[table->count].value = vvalue;
+			table->entries[table->count].smio_low =
+					vol_table->entries[i].smio_low;
+			table->count++;
+		}
+	}
+
+	memcpy(vol_table, table, sizeof(struct pp_atomctrl_voltage_table));
+	kfree(table);
+
+	return 0;
+}
+
+int phm_get_svi2_mvdd_voltage_table(struct pp_atomctrl_voltage_table *vol_table,
+		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
+{
+	uint32_t i;
+	int result;
+
+	PP_ASSERT_WITH_CODE((0 != dep_table->count),
+			"Voltage Dependency Table empty.", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE((NULL != vol_table),
+			"vol_table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+	vol_table->count = dep_table->count;
+
+	for (i = 0; i < dep_table->count; i++) {
+		vol_table->entries[i].value = dep_table->entries[i].mvdd;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	result = phm_trim_voltage_table(vol_table);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to trim MVDD table.", return result);
+
+	return 0;
+}
+
+int phm_get_svi2_vddci_voltage_table(struct pp_atomctrl_voltage_table *vol_table,
+		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
+{
+	uint32_t i;
+	int result;
+
+	PP_ASSERT_WITH_CODE((0 != dep_table->count),
+			"Voltage Dependency Table empty.", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE((NULL != vol_table),
+			"vol_table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+	vol_table->count = dep_table->count;
+
+	for (i = 0; i < dep_table->count; i++) {
+		vol_table->entries[i].value = dep_table->entries[i].vddci;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	result = phm_trim_voltage_table(vol_table);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to trim VDDCI table.", return result);
+
+	return 0;
+}
+
+int phm_get_svi2_vdd_voltage_table(struct pp_atomctrl_voltage_table *vol_table,
+		phm_ppt_v1_voltage_lookup_table *lookup_table)
+{
+	int i = 0;
+
+	PP_ASSERT_WITH_CODE((0 != lookup_table->count),
+			"Voltage Lookup Table empty.", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE((NULL != vol_table),
+			"vol_table empty.", return -EINVAL);
+
+	vol_table->mask_low = 0;
+	vol_table->phase_delay = 0;
+
+	vol_table->count = lookup_table->count;
+
+	for (i = 0; i < vol_table->count; i++) {
+		vol_table->entries[i].value = lookup_table->entries[i].us_vdd;
+		vol_table->entries[i].smio_low = 0;
+	}
+
+	return 0;
+}
+
+void phm_trim_voltage_table_to_fit_state_table(uint32_t max_vol_steps,
+				struct pp_atomctrl_voltage_table *vol_table)
+{
+	unsigned int i, diff;
+
+	if (vol_table->count <= max_vol_steps)
+		return;
+
+	diff = vol_table->count - max_vol_steps;
+
+	for (i = 0; i < max_vol_steps; i++)
+		vol_table->entries[i] = vol_table->entries[i + diff];
+
+	vol_table->count = max_vol_steps;
+
+	return;
+}
+
+int phm_reset_single_dpm_table(void *table,
+				uint32_t count, int max)
+{
+	int i;
+
+	struct vi_dpm_table *dpm_table = (struct vi_dpm_table *)table;
+
+	PP_ASSERT_WITH_CODE(count <= max,
+			"Fatal error, can not set up single DPM table entries to exceed max number!",
+			   );
+
+	dpm_table->count = count;
+	for (i = 0; i < max; i++)
+		dpm_table->dpm_level[i].enabled = false;
+
+	return 0;
+}
+
+void phm_setup_pcie_table_entry(
+	void *table,
+	uint32_t index, uint32_t pcie_gen,
+	uint32_t pcie_lanes)
+{
+	struct vi_dpm_table *dpm_table = (struct vi_dpm_table *)table;
+	dpm_table->dpm_level[index].value = pcie_gen;
+	dpm_table->dpm_level[index].param1 = pcie_lanes;
+	dpm_table->dpm_level[index].enabled = 1;
+}
+
+int32_t phm_get_dpm_level_enable_mask_value(void *table)
+{
+	int32_t i;
+	int32_t mask = 0;
+	struct vi_dpm_table *dpm_table = (struct vi_dpm_table *)table;
+
+	for (i = dpm_table->count; i > 0; i--) {
+		mask = mask << 1;
+		if (dpm_table->dpm_level[i - 1].enabled)
+			mask |= 0x1;
+		else
+			mask &= 0xFFFFFFFE;
+	}
+
+	return mask;
+}
+
+uint8_t phm_get_voltage_index(
+		struct phm_ppt_v1_voltage_lookup_table *lookup_table, uint16_t voltage)
+{
+	uint8_t count = (uint8_t) (lookup_table->count);
+	uint8_t i;
+
+	PP_ASSERT_WITH_CODE((NULL != lookup_table),
+			"Lookup Table empty.", return 0);
+	PP_ASSERT_WITH_CODE((0 != count),
+			"Lookup Table empty.", return 0);
+
+	for (i = 0; i < lookup_table->count; i++) {
+		/* find first voltage equal or bigger than requested */
+		if (lookup_table->entries[i].us_vdd >= voltage)
+			return i;
+	}
+	/* voltage is bigger than max voltage in the table */
+	return i - 1;
+}
+
+uint16_t phm_find_closest_vddci(struct pp_atomctrl_voltage_table *vddci_table, uint16_t vddci)
+{
+	uint32_t  i;
+
+	for (i = 0; i < vddci_table->count; i++) {
+		if (vddci_table->entries[i].value >= vddci)
+			return vddci_table->entries[i].value;
+	}
+
+	PP_ASSERT_WITH_CODE(false,
+			"VDDCI is larger than max VDDCI in VDDCI Voltage Table!",
+			return vddci_table->entries[i].value);
+}
+
+int phm_find_boot_level(void *table,
+		uint32_t value, uint32_t *boot_level)
+{
+	int result = -EINVAL;
+	uint32_t i;
+	struct vi_dpm_table *dpm_table = (struct vi_dpm_table *)table;
+
+	for (i = 0; i < dpm_table->count; i++) {
+		if (value == dpm_table->dpm_level[i].value) {
+			*boot_level = i;
+			result = 0;
+		}
+	}
+
+	return result;
+}
+
+int phm_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_voltage_lookup_table *lookup_table,
+	uint16_t virtual_voltage_id, int32_t *sclk)
+{
+	uint8_t entryId;
+	uint8_t voltageId;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -EINVAL);
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
+	for (entryId = 0; entryId < table_info->vdd_dep_on_sclk->count; entryId++) {
+		voltageId = table_info->vdd_dep_on_sclk->entries[entryId].vddInd;
+		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
+			break;
+	}
+
+	PP_ASSERT_WITH_CODE(entryId < table_info->vdd_dep_on_sclk->count,
+			"Can't find requested voltage id in vdd_dep_on_sclk table!",
+			return -EINVAL;
+			);
+
+	*sclk = table_info->vdd_dep_on_sclk->entries[entryId].clk;
+
+	return 0;
+}
+
+/**
+ * Initialize Dynamic State Adjustment Rule Settings
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ */
+int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr)
+{
+	uint32_t table_size;
+	struct phm_clock_voltage_dependency_table *table_clk_vlt;
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	/* initialize vddc_dep_on_dal_pwrl table */
+	table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record);
+	table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table_clk_vlt) {
+		printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
+		return -ENOMEM;
+	} else {
+		table_clk_vlt->count = 4;
+		table_clk_vlt->entries[0].clk = PP_DAL_POWERLEVEL_ULTRALOW;
+		table_clk_vlt->entries[0].v = 0;
+		table_clk_vlt->entries[1].clk = PP_DAL_POWERLEVEL_LOW;
+		table_clk_vlt->entries[1].v = 720;
+		table_clk_vlt->entries[2].clk = PP_DAL_POWERLEVEL_NOMINAL;
+		table_clk_vlt->entries[2].v = 810;
+		table_clk_vlt->entries[3].clk = PP_DAL_POWERLEVEL_PERFORMANCE;
+		table_clk_vlt->entries[3].v = 900;
+		pptable_info->vddc_dep_on_dal_pwrl = table_clk_vlt;
+		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = table_clk_vlt;
+	}
+
+	return 0;
+}
+
+int phm_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->dyn_state.vddc_dep_on_dal_pwrl) {
+		kfree(hwmgr->dyn_state.vddc_dep_on_dal_pwrl);
+		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL;
+	}
+
+	if (NULL != hwmgr->backend) {
+		kfree(hwmgr->backend);
+		hwmgr->backend = NULL;
+	}
+
+	return 0;
+}
+
+uint32_t phm_get_lowest_enabled_level(struct pp_hwmgr *hwmgr, uint32_t mask)
+{
+	uint32_t level = 0;
+
+	while (0 == (mask & (1 << level)))
+		level++;
+
+	return level;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr_ppt.h b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr_ppt.h
new file mode 100644
index 000000000000..c9e6c2d80ea6
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr_ppt.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef PP_HWMGR_PPT_H
+#define PP_HWMGR_PPT_H
+
+#include "hardwaremanager.h"
+#include "smumgr.h"
+#include "atom-types.h"
+
+struct phm_ppt_v1_clock_voltage_dependency_record {
+	uint32_t clk;
+	uint8_t vddInd;
+	uint16_t vdd_offset;
+	uint16_t vddc;
+	uint16_t vddgfx;
+	uint16_t vddci;
+	uint16_t mvdd;
+	uint8_t phases;
+	uint8_t cks_enable;
+	uint8_t cks_voffset;
+};
+
+typedef struct phm_ppt_v1_clock_voltage_dependency_record phm_ppt_v1_clock_voltage_dependency_record;
+
+struct phm_ppt_v1_clock_voltage_dependency_table {
+	uint32_t count;                                            /* Number of entries. */
+	phm_ppt_v1_clock_voltage_dependency_record entries[1];     /* Dynamically allocate count entries. */
+};
+
+typedef struct phm_ppt_v1_clock_voltage_dependency_table phm_ppt_v1_clock_voltage_dependency_table;
+
+
+/* Multimedia Clock Voltage Dependency records and table */
+struct phm_ppt_v1_mm_clock_voltage_dependency_record {
+	uint32_t  dclk;                                              /* UVD D-clock */
+	uint32_t  vclk;                                              /* UVD V-clock */
+	uint32_t  eclk;                                              /* VCE clock */
+	uint32_t  aclk;                                              /* ACP clock */
+	uint32_t  samclock;                                          /* SAMU clock */
+	uint8_t	vddcInd;
+	uint16_t vddgfx_offset;
+	uint16_t vddc;
+	uint16_t vddgfx;
+	uint8_t phases;
+};
+typedef struct phm_ppt_v1_mm_clock_voltage_dependency_record phm_ppt_v1_mm_clock_voltage_dependency_record;
+
+struct phm_ppt_v1_mm_clock_voltage_dependency_table {
+	uint32_t count;													/* Number of entries. */
+	phm_ppt_v1_mm_clock_voltage_dependency_record entries[1];		/* Dynamically allocate count entries. */
+};
+typedef struct phm_ppt_v1_mm_clock_voltage_dependency_table phm_ppt_v1_mm_clock_voltage_dependency_table;
+
+struct phm_ppt_v1_voltage_lookup_record {
+	uint16_t us_calculated;
+	uint16_t us_vdd;												/* Base voltage */
+	uint16_t us_cac_low;
+	uint16_t us_cac_mid;
+	uint16_t us_cac_high;
+};
+typedef struct phm_ppt_v1_voltage_lookup_record phm_ppt_v1_voltage_lookup_record;
+
+struct phm_ppt_v1_voltage_lookup_table {
+	uint32_t count;
+	phm_ppt_v1_voltage_lookup_record entries[1];    /* Dynamically allocate count entries. */
+};
+typedef struct phm_ppt_v1_voltage_lookup_table phm_ppt_v1_voltage_lookup_table;
+
+/* PCIE records and Table */
+
+struct phm_ppt_v1_pcie_record {
+	uint8_t gen_speed;
+	uint8_t lane_width;
+};
+typedef struct phm_ppt_v1_pcie_record phm_ppt_v1_pcie_record;
+
+struct phm_ppt_v1_pcie_table {
+	uint32_t count;                                            /* Number of entries. */
+	phm_ppt_v1_pcie_record entries[1];                         /* Dynamically allocate count entries. */
+};
+typedef struct phm_ppt_v1_pcie_table phm_ppt_v1_pcie_table;
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
new file mode 100644
index 000000000000..7b2d5000292d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
@@ -0,0 +1,76 @@
+#include <linux/errno.h>
+#include "linux/delay.h"
+#include "hwmgr.h"
+#include "amd_acpi.h"
+
+bool acpi_atcs_functions_supported(void *device, uint32_t index)
+{
+	int32_t result;
+	struct atcs_verify_interface output_buf = {0};
+
+	int32_t temp_buffer = 1;
+
+	result = cgs_call_acpi_method(device, CGS_ACPI_METHOD_ATCS,
+						ATCS_FUNCTION_VERIFY_INTERFACE,
+						&temp_buffer,
+						&output_buf,
+						1,
+						sizeof(temp_buffer),
+						sizeof(output_buf));
+
+	return result == 0 ? (output_buf.function_bits & (1 << (index - 1))) != 0 : false;
+}
+
+int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise)
+{
+	struct atcs_pref_req_input atcs_input;
+	struct atcs_pref_req_output atcs_output;
+	u32 retry = 3;
+	int result;
+	struct cgs_system_info info = {0};
+
+	if (!acpi_atcs_functions_supported(device, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST))
+		return -EINVAL;
+
+	info.size = sizeof(struct cgs_system_info);
+	info.info_id = CGS_SYSTEM_INFO_ADAPTER_BDF_ID;
+	result = cgs_query_system_info(device, &info);
+	if (result != 0)
+		return -EINVAL;
+	atcs_input.client_id = (uint16_t)info.value;
+	atcs_input.size = sizeof(struct atcs_pref_req_input);
+	atcs_input.valid_flags_mask = ATCS_VALID_FLAGS_MASK;
+	atcs_input.flags = ATCS_WAIT_FOR_COMPLETION;
+	if (advertise)
+		atcs_input.flags |= ATCS_ADVERTISE_CAPS;
+	atcs_input.req_type = ATCS_PCIE_LINK_SPEED;
+	atcs_input.perf_req = perf_req;
+
+	atcs_output.size = sizeof(struct atcs_pref_req_input);
+
+	while (retry--) {
+		result = cgs_call_acpi_method(device,
+						CGS_ACPI_METHOD_ATCS,
+						ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST,
+						&atcs_input,
+						&atcs_output,
+						0,
+						sizeof(atcs_input),
+						sizeof(atcs_output));
+		if (result != 0)
+			return -EIO;
+
+		switch (atcs_output.ret_val) {
+		case ATCS_REQUEST_REFUSED:
+		default:
+			return -EINVAL;
+		case ATCS_REQUEST_COMPLETE:
+			return 0;
+		case ATCS_REQUEST_IN_PROGRESS:
+			udelay(10);
+			break;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
new file mode 100644
index 000000000000..2a83a4af2904
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -0,0 +1,1207 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/fb.h>
+
+#include "ppatomctrl.h"
+#include "atombios.h"
+#include "cgs_common.h"
+#include "pp_debug.h"
+#include "ppevvmath.h"
+
+#define MEM_ID_MASK           0xff000000
+#define MEM_ID_SHIFT          24
+#define CLOCK_RANGE_MASK      0x00ffffff
+#define CLOCK_RANGE_SHIFT     0
+#define LOW_NIBBLE_MASK       0xf
+#define DATA_EQU_PREV         0
+#define DATA_FROM_TABLE       4
+
+union voltage_object_info {
+	struct _ATOM_VOLTAGE_OBJECT_INFO v1;
+	struct _ATOM_VOLTAGE_OBJECT_INFO_V2 v2;
+	struct _ATOM_VOLTAGE_OBJECT_INFO_V3_1 v3;
+};
+
+static int atomctrl_retrieve_ac_timing(
+		uint8_t index,
+		ATOM_INIT_REG_BLOCK *reg_block,
+		pp_atomctrl_mc_reg_table *table)
+{
+	uint32_t i, j;
+	uint8_t tmem_id;
+	ATOM_MEMORY_SETTING_DATA_BLOCK *reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *)
+		((uint8_t *)reg_block + (2 * sizeof(uint16_t)) + le16_to_cpu(reg_block->usRegIndexTblSize));
+
+	uint8_t num_ranges = 0;
+
+	while (*(uint32_t *)reg_data != END_OF_REG_DATA_BLOCK &&
+			num_ranges < VBIOS_MAX_AC_TIMING_ENTRIES) {
+		tmem_id = (uint8_t)((*(uint32_t *)reg_data & MEM_ID_MASK) >> MEM_ID_SHIFT);
+
+		if (index == tmem_id) {
+			table->mc_reg_table_entry[num_ranges].mclk_max =
+				(uint32_t)((*(uint32_t *)reg_data & CLOCK_RANGE_MASK) >>
+						CLOCK_RANGE_SHIFT);
+
+			for (i = 0, j = 1; i < table->last; i++) {
+				if ((table->mc_reg_address[i].uc_pre_reg_data &
+							LOW_NIBBLE_MASK) == DATA_FROM_TABLE) {
+					table->mc_reg_table_entry[num_ranges].mc_data[i] =
+						(uint32_t)*((uint32_t *)reg_data + j);
+					j++;
+				} else if ((table->mc_reg_address[i].uc_pre_reg_data &
+							LOW_NIBBLE_MASK) == DATA_EQU_PREV) {
+					table->mc_reg_table_entry[num_ranges].mc_data[i] =
+						table->mc_reg_table_entry[num_ranges].mc_data[i-1];
+				}
+			}
+			num_ranges++;
+		}
+
+		reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *)
+			((uint8_t *)reg_data + le16_to_cpu(reg_block->usRegDataBlkSize)) ;
+	}
+
+	PP_ASSERT_WITH_CODE((*(uint32_t *)reg_data == END_OF_REG_DATA_BLOCK),
+			"Invalid VramInfo table.", return -1);
+	table->num_entries = num_ranges;
+
+	return 0;
+}
+
+/**
+ * Get memory clock AC timing registers index from VBIOS table
+ * VBIOS set end of memory clock AC timing registers by ucPreRegDataLength bit6 = 1
+ * @param    reg_block the address ATOM_INIT_REG_BLOCK
+ * @param    table the address of MCRegTable
+ * @return   0
+ */
+static int atomctrl_set_mc_reg_address_table(
+		ATOM_INIT_REG_BLOCK *reg_block,
+		pp_atomctrl_mc_reg_table *table)
+{
+	uint8_t i = 0;
+	uint8_t num_entries = (uint8_t)((le16_to_cpu(reg_block->usRegIndexTblSize))
+			/ sizeof(ATOM_INIT_REG_INDEX_FORMAT));
+	ATOM_INIT_REG_INDEX_FORMAT *format = &reg_block->asRegIndexBuf[0];
+
+	num_entries--;        /* subtract 1 data end mark entry */
+
+	PP_ASSERT_WITH_CODE((num_entries <= VBIOS_MC_REGISTER_ARRAY_SIZE),
+			"Invalid VramInfo table.", return -1);
+
+	/* ucPreRegDataLength bit6 = 1 is the end of memory clock AC timing registers */
+	while ((!(format->ucPreRegDataLength & ACCESS_PLACEHOLDER)) &&
+			(i < num_entries)) {
+		table->mc_reg_address[i].s1 =
+			(uint16_t)(le16_to_cpu(format->usRegIndex));
+		table->mc_reg_address[i].uc_pre_reg_data =
+			format->ucPreRegDataLength;
+
+		i++;
+		format = (ATOM_INIT_REG_INDEX_FORMAT *)
+			((uint8_t *)format + sizeof(ATOM_INIT_REG_INDEX_FORMAT));
+	}
+
+	table->last = i;
+	return 0;
+}
+
+
+int atomctrl_initialize_mc_reg_table(
+		struct pp_hwmgr *hwmgr,
+		uint8_t module_index,
+		pp_atomctrl_mc_reg_table *table)
+{
+	ATOM_VRAM_INFO_HEADER_V2_1 *vram_info;
+	ATOM_INIT_REG_BLOCK *reg_block;
+	int result = 0;
+	u8 frev, crev;
+	u16 size;
+
+	vram_info = (ATOM_VRAM_INFO_HEADER_V2_1 *)
+		cgs_atom_get_data_table(hwmgr->device,
+				GetIndexIntoMasterTable(DATA, VRAM_Info), &size, &frev, &crev);
+
+	if (module_index >= vram_info->ucNumOfVRAMModule) {
+		printk(KERN_ERR "[ powerplay ] Invalid VramInfo table.");
+		result = -1;
+	} else if (vram_info->sHeader.ucTableFormatRevision < 2) {
+		printk(KERN_ERR "[ powerplay ] Invalid VramInfo table.");
+		result = -1;
+	}
+
+	if (0 == result) {
+		reg_block = (ATOM_INIT_REG_BLOCK *)
+			((uint8_t *)vram_info + le16_to_cpu(vram_info->usMemClkPatchTblOffset));
+		result = atomctrl_set_mc_reg_address_table(reg_block, table);
+	}
+
+	if (0 == result) {
+		result = atomctrl_retrieve_ac_timing(module_index,
+					reg_block, table);
+	}
+
+	return result;
+}
+
+/**
+ * Set DRAM timings based on engine clock and memory clock.
+ */
+int atomctrl_set_engine_dram_timings_rv770(
+		struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock,
+		uint32_t memory_clock)
+{
+	SET_ENGINE_CLOCK_PS_ALLOCATION engine_clock_parameters;
+
+	/* They are both in 10KHz Units. */
+	engine_clock_parameters.ulTargetEngineClock =
+		(uint32_t) engine_clock & SET_CLOCK_FREQ_MASK;
+	engine_clock_parameters.ulTargetEngineClock |=
+		(COMPUTE_ENGINE_PLL_PARAM << 24);
+
+	/* in 10 khz units.*/
+	engine_clock_parameters.sReserved.ulClock =
+		(uint32_t) memory_clock & SET_CLOCK_FREQ_MASK;
+	return cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings),
+			&engine_clock_parameters);
+}
+
+/**
+ * Private Function to get the PowerPlay Table Address.
+ * WARNING: The tabled returned by this function is in
+ * dynamically allocated memory.
+ * The caller has to release if by calling kfree.
+ */
+static ATOM_VOLTAGE_OBJECT_INFO *get_voltage_info_table(void *device)
+{
+	int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+	u8 frev, crev;
+	u16 size;
+	union voltage_object_info *voltage_info;
+
+	voltage_info = (union voltage_object_info *)
+		cgs_atom_get_data_table(device, index,
+			&size, &frev, &crev);
+
+	if (voltage_info != NULL)
+		return (ATOM_VOLTAGE_OBJECT_INFO *) &(voltage_info->v3);
+	else
+		return NULL;
+}
+
+static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3(
+		const ATOM_VOLTAGE_OBJECT_INFO_V3_1 * voltage_object_info_table,
+		uint8_t voltage_type, uint8_t voltage_mode)
+{
+	unsigned int size = le16_to_cpu(voltage_object_info_table->sHeader.usStructureSize);
+	unsigned int offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]);
+	uint8_t *start = (uint8_t *)voltage_object_info_table;
+
+	while (offset < size) {
+		const ATOM_VOLTAGE_OBJECT_V3 *voltage_object =
+			(const ATOM_VOLTAGE_OBJECT_V3 *)(start + offset);
+
+		if (voltage_type == voltage_object->asGpioVoltageObj.sHeader.ucVoltageType &&
+			voltage_mode == voltage_object->asGpioVoltageObj.sHeader.ucVoltageMode)
+			return voltage_object;
+
+		offset += le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize);
+	}
+
+	return NULL;
+}
+
+/** atomctrl_get_memory_pll_dividers_si().
+ *
+ * @param hwmgr                 input parameter: pointer to HwMgr
+ * @param clock_value             input parameter: memory clock
+ * @param dividers                 output parameter: memory PLL dividers
+ * @param strobe_mode            input parameter: 1 for strobe mode,  0 for performance mode
+ */
+int atomctrl_get_memory_pll_dividers_si(
+		struct pp_hwmgr *hwmgr,
+		uint32_t clock_value,
+		pp_atomctrl_memory_clock_param *mpll_param,
+		bool strobe_mode)
+{
+	COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 mpll_parameters;
+	int result;
+
+	mpll_parameters.ulClock = (uint32_t) clock_value;
+	mpll_parameters.ucInputFlag = (uint8_t)((strobe_mode) ? 1 : 0);
+
+	result = cgs_atom_exec_cmd_table
+		(hwmgr->device,
+		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
+		 &mpll_parameters);
+
+	if (0 == result) {
+		mpll_param->mpll_fb_divider.clk_frac =
+			mpll_parameters.ulFbDiv.usFbDivFrac;
+		mpll_param->mpll_fb_divider.cl_kf =
+			mpll_parameters.ulFbDiv.usFbDiv;
+		mpll_param->mpll_post_divider =
+			(uint32_t)mpll_parameters.ucPostDiv;
+		mpll_param->vco_mode =
+			(uint32_t)(mpll_parameters.ucPllCntlFlag &
+					MPLL_CNTL_FLAG_VCO_MODE_MASK);
+		mpll_param->yclk_sel =
+			(uint32_t)((mpll_parameters.ucPllCntlFlag &
+						MPLL_CNTL_FLAG_BYPASS_DQ_PLL) ? 1 : 0);
+		mpll_param->qdr =
+			(uint32_t)((mpll_parameters.ucPllCntlFlag &
+						MPLL_CNTL_FLAG_QDR_ENABLE) ? 1 : 0);
+		mpll_param->half_rate =
+			(uint32_t)((mpll_parameters.ucPllCntlFlag &
+						MPLL_CNTL_FLAG_AD_HALF_RATE) ? 1 : 0);
+		mpll_param->dll_speed =
+			(uint32_t)(mpll_parameters.ucDllSpeed);
+		mpll_param->bw_ctrl =
+			(uint32_t)(mpll_parameters.ucBWCntl);
+	}
+
+	return result;
+}
+
+/** atomctrl_get_memory_pll_dividers_vi().
+ *
+ * @param hwmgr                 input parameter: pointer to HwMgr
+ * @param clock_value             input parameter: memory clock
+ * @param dividers               output parameter: memory PLL dividers
+ */
+int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
+		uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param)
+{
+	COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2 mpll_parameters;
+	int result;
+
+	mpll_parameters.ulClock.ulClock = (uint32_t)clock_value;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
+			&mpll_parameters);
+
+	if (!result)
+		mpll_param->mpll_post_divider =
+				(uint32_t)mpll_parameters.ulClock.ucPostDiv;
+
+	return result;
+}
+
+int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
+					  uint32_t clock_value,
+					  pp_atomctrl_clock_dividers_kong *dividers)
+{
+	COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 pll_parameters;
+	int result;
+
+	pll_parameters.ulClock = clock_value;
+
+	result = cgs_atom_exec_cmd_table
+		(hwmgr->device,
+		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
+		 &pll_parameters);
+
+	if (0 == result) {
+		dividers->pll_post_divider = pll_parameters.ucPostDiv;
+		dividers->real_clock = pll_parameters.ulClock;
+	}
+
+	return result;
+}
+
+int atomctrl_get_engine_pll_dividers_vi(
+		struct pp_hwmgr *hwmgr,
+		uint32_t clock_value,
+		pp_atomctrl_clock_dividers_vi *dividers)
+{
+	COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
+	int result;
+
+	pll_patameters.ulClock.ulClock = clock_value;
+	pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK;
+
+	result = cgs_atom_exec_cmd_table
+		(hwmgr->device,
+		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
+		 &pll_patameters);
+
+	if (0 == result) {
+		dividers->pll_post_divider =
+			pll_patameters.ulClock.ucPostDiv;
+		dividers->real_clock =
+			pll_patameters.ulClock.ulClock;
+
+		dividers->ul_fb_div.ul_fb_div_frac =
+			pll_patameters.ulFbDiv.usFbDivFrac;
+		dividers->ul_fb_div.ul_fb_div =
+			pll_patameters.ulFbDiv.usFbDiv;
+
+		dividers->uc_pll_ref_div =
+			pll_patameters.ucPllRefDiv;
+		dividers->uc_pll_post_div =
+			pll_patameters.ucPllPostDiv;
+		dividers->uc_pll_cntl_flag =
+			pll_patameters.ucPllCntlFlag;
+	}
+
+	return result;
+}
+
+int atomctrl_get_dfs_pll_dividers_vi(
+		struct pp_hwmgr *hwmgr,
+		uint32_t clock_value,
+		pp_atomctrl_clock_dividers_vi *dividers)
+{
+	COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
+	int result;
+
+	pll_patameters.ulClock.ulClock = clock_value;
+	pll_patameters.ulClock.ucPostDiv =
+		COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK;
+
+	result = cgs_atom_exec_cmd_table
+		(hwmgr->device,
+		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
+		 &pll_patameters);
+
+	if (0 == result) {
+		dividers->pll_post_divider =
+			pll_patameters.ulClock.ucPostDiv;
+		dividers->real_clock =
+			pll_patameters.ulClock.ulClock;
+
+		dividers->ul_fb_div.ul_fb_div_frac =
+			pll_patameters.ulFbDiv.usFbDivFrac;
+		dividers->ul_fb_div.ul_fb_div =
+			pll_patameters.ulFbDiv.usFbDiv;
+
+		dividers->uc_pll_ref_div =
+			pll_patameters.ucPllRefDiv;
+		dividers->uc_pll_post_div =
+			pll_patameters.ucPllPostDiv;
+		dividers->uc_pll_cntl_flag =
+			pll_patameters.ucPllCntlFlag;
+	}
+
+	return result;
+}
+
+/**
+ * Get the reference clock in 10KHz
+ */
+uint32_t atomctrl_get_reference_clock(struct pp_hwmgr *hwmgr)
+{
+	ATOM_FIRMWARE_INFO *fw_info;
+	u8 frev, crev;
+	u16 size;
+	uint32_t clock;
+
+	fw_info = (ATOM_FIRMWARE_INFO *)
+		cgs_atom_get_data_table(hwmgr->device,
+			GetIndexIntoMasterTable(DATA, FirmwareInfo),
+			&size, &frev, &crev);
+
+	if (fw_info == NULL)
+		clock = 2700;
+	else
+		clock = (uint32_t)(le16_to_cpu(fw_info->usReferenceClock));
+
+	return clock;
+}
+
+/**
+ * Returns true if the given voltage type is controlled by GPIO pins.
+ * voltage_type is one of SET_VOLTAGE_TYPE_ASIC_VDDC,
+ * SET_VOLTAGE_TYPE_ASIC_MVDDC, SET_VOLTAGE_TYPE_ASIC_MVDDQ.
+ * voltage_mode is one of ATOM_SET_VOLTAGE, ATOM_SET_VOLTAGE_PHASE
+ */
+bool atomctrl_is_voltage_controled_by_gpio_v3(
+		struct pp_hwmgr *hwmgr,
+		uint8_t voltage_type,
+		uint8_t voltage_mode)
+{
+	ATOM_VOLTAGE_OBJECT_INFO_V3_1 *voltage_info =
+		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->device);
+	bool ret;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_info),
+			"Could not find Voltage Table in BIOS.", return false;);
+
+	ret = (NULL != atomctrl_lookup_voltage_type_v3
+			(voltage_info, voltage_type, voltage_mode)) ? true : false;
+
+	return ret;
+}
+
+int atomctrl_get_voltage_table_v3(
+		struct pp_hwmgr *hwmgr,
+		uint8_t voltage_type,
+		uint8_t voltage_mode,
+		pp_atomctrl_voltage_table *voltage_table)
+{
+	ATOM_VOLTAGE_OBJECT_INFO_V3_1 *voltage_info =
+		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->device);
+	const ATOM_VOLTAGE_OBJECT_V3 *voltage_object;
+	unsigned int i;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_info),
+			"Could not find Voltage Table in BIOS.", return -1;);
+
+	voltage_object = atomctrl_lookup_voltage_type_v3
+		(voltage_info, voltage_type, voltage_mode);
+
+	if (voltage_object == NULL)
+		return -1;
+
+	PP_ASSERT_WITH_CODE(
+			(voltage_object->asGpioVoltageObj.ucGpioEntryNum <=
+			PP_ATOMCTRL_MAX_VOLTAGE_ENTRIES),
+			"Too many voltage entries!",
+			return -1;
+			);
+
+	for (i = 0; i < voltage_object->asGpioVoltageObj.ucGpioEntryNum; i++) {
+		voltage_table->entries[i].value =
+			voltage_object->asGpioVoltageObj.asVolGpioLut[i].usVoltageValue;
+		voltage_table->entries[i].smio_low =
+			voltage_object->asGpioVoltageObj.asVolGpioLut[i].ulVoltageId;
+	}
+
+	voltage_table->mask_low    =
+		voltage_object->asGpioVoltageObj.ulGpioMaskVal;
+	voltage_table->count      =
+		voltage_object->asGpioVoltageObj.ucGpioEntryNum;
+	voltage_table->phase_delay =
+		voltage_object->asGpioVoltageObj.ucPhaseDelay;
+
+	return 0;
+}
+
+static bool atomctrl_lookup_gpio_pin(
+		ATOM_GPIO_PIN_LUT * gpio_lookup_table,
+		const uint32_t pinId,
+		pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment)
+{
+	unsigned int size = le16_to_cpu(gpio_lookup_table->sHeader.usStructureSize);
+	unsigned int offset = offsetof(ATOM_GPIO_PIN_LUT, asGPIO_Pin[0]);
+	uint8_t *start = (uint8_t *)gpio_lookup_table;
+
+	while (offset < size) {
+		const ATOM_GPIO_PIN_ASSIGNMENT *pin_assignment =
+			(const ATOM_GPIO_PIN_ASSIGNMENT *)(start + offset);
+
+		if (pinId == pin_assignment->ucGPIO_ID) {
+			gpio_pin_assignment->uc_gpio_pin_bit_shift =
+				pin_assignment->ucGpioPinBitShift;
+			gpio_pin_assignment->us_gpio_pin_aindex =
+				le16_to_cpu(pin_assignment->usGpioPin_AIndex);
+			return false;
+		}
+
+		offset += offsetof(ATOM_GPIO_PIN_ASSIGNMENT, ucGPIO_ID) + 1;
+	}
+
+	return true;
+}
+
+/**
+ * Private Function to get the PowerPlay Table Address.
+ * WARNING: The tabled returned by this function is in
+ * dynamically allocated memory.
+ * The caller has to release if by calling kfree.
+ */
+static ATOM_GPIO_PIN_LUT *get_gpio_lookup_table(void *device)
+{
+	u8 frev, crev;
+	u16 size;
+	void *table_address;
+
+	table_address = (ATOM_GPIO_PIN_LUT *)
+		cgs_atom_get_data_table(device,
+				GetIndexIntoMasterTable(DATA, GPIO_Pin_LUT),
+				&size, &frev, &crev);
+
+	PP_ASSERT_WITH_CODE((NULL != table_address),
+			"Error retrieving BIOS Table Address!", return NULL;);
+
+	return (ATOM_GPIO_PIN_LUT *)table_address;
+}
+
+/**
+ * Returns 1 if the given pin id find in lookup table.
+ */
+bool atomctrl_get_pp_assign_pin(
+		struct pp_hwmgr *hwmgr,
+		const uint32_t pinId,
+		pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment)
+{
+	bool bRet = 0;
+	ATOM_GPIO_PIN_LUT *gpio_lookup_table =
+		get_gpio_lookup_table(hwmgr->device);
+
+	PP_ASSERT_WITH_CODE((NULL != gpio_lookup_table),
+			"Could not find GPIO lookup Table in BIOS.", return -1);
+
+	bRet = atomctrl_lookup_gpio_pin(gpio_lookup_table, pinId,
+		gpio_pin_assignment);
+
+	return bRet;
+}
+
+int atomctrl_calculate_voltage_evv_on_sclk(
+		struct pp_hwmgr *hwmgr,
+		uint8_t voltage_type,
+		uint32_t sclk,
+		uint16_t virtual_voltage_Id,
+		uint16_t *voltage,
+		uint16_t dpm_level,
+		bool debug)
+{
+	ATOM_ASIC_PROFILING_INFO_V3_4 *getASICProfilingInfo;
+
+	EFUSE_LINEAR_FUNC_PARAM sRO_fuse;
+	EFUSE_LINEAR_FUNC_PARAM sCACm_fuse;
+	EFUSE_LINEAR_FUNC_PARAM sCACb_fuse;
+	EFUSE_LOGISTIC_FUNC_PARAM sKt_Beta_fuse;
+	EFUSE_LOGISTIC_FUNC_PARAM sKv_m_fuse;
+	EFUSE_LOGISTIC_FUNC_PARAM sKv_b_fuse;
+	EFUSE_INPUT_PARAMETER sInput_FuseValues;
+	READ_EFUSE_VALUE_PARAMETER sOutput_FuseValues;
+
+	uint32_t ul_RO_fused, ul_CACb_fused, ul_CACm_fused, ul_Kt_Beta_fused, ul_Kv_m_fused, ul_Kv_b_fused;
+	fInt fSM_A0, fSM_A1, fSM_A2, fSM_A3, fSM_A4, fSM_A5, fSM_A6, fSM_A7;
+	fInt fMargin_RO_a, fMargin_RO_b, fMargin_RO_c, fMargin_fixed, fMargin_FMAX_mean, fMargin_Plat_mean, fMargin_FMAX_sigma, fMargin_Plat_sigma, fMargin_DC_sigma;
+	fInt fLkg_FT, repeat;
+	fInt fMicro_FMAX, fMicro_CR, fSigma_FMAX, fSigma_CR, fSigma_DC, fDC_SCLK, fSquared_Sigma_DC, fSquared_Sigma_CR, fSquared_Sigma_FMAX;
+	fInt fRLL_LoadLine, fPowerDPMx, fDerateTDP, fVDDC_base, fA_Term, fC_Term, fB_Term, fRO_DC_margin;
+	fInt fRO_fused, fCACm_fused, fCACb_fused, fKv_m_fused, fKv_b_fused, fKt_Beta_fused, fFT_Lkg_V0NORM;
+	fInt fSclk_margin, fSclk, fEVV_V;
+	fInt fV_min, fV_max, fT_prod, fLKG_Factor, fT_FT, fV_FT, fV_x, fTDP_Power, fTDP_Power_right, fTDP_Power_left, fTDP_Current, fV_NL;
+	uint32_t ul_FT_Lkg_V0NORM;
+	fInt fLn_MaxDivMin, fMin, fAverage, fRange;
+	fInt fRoots[2];
+	fInt fStepSize = GetScaledFraction(625, 100000);
+
+	int result;
+
+	getASICProfilingInfo = (ATOM_ASIC_PROFILING_INFO_V3_4 *)
+			cgs_atom_get_data_table(hwmgr->device,
+					GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo),
+					NULL, NULL, NULL);
+
+	if (!getASICProfilingInfo)
+		return -1;
+
+	if(getASICProfilingInfo->asHeader.ucTableFormatRevision < 3 ||
+			(getASICProfilingInfo->asHeader.ucTableFormatRevision == 3 &&
+			getASICProfilingInfo->asHeader.ucTableContentRevision < 4))
+		return -1;
+
+	/*-----------------------------------------------------------
+	 *GETTING MULTI-STEP PARAMETERS RELATED TO CURRENT DPM LEVEL
+	 *-----------------------------------------------------------
+	 */
+	fRLL_LoadLine = Divide(getASICProfilingInfo->ulLoadLineSlop, 1000);
+
+	switch (dpm_level) {
+	case 1:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm1);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM1, 1000);
+		break;
+	case 2:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm2);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM2, 1000);
+		break;
+	case 3:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm3);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM3, 1000);
+		break;
+	case 4:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm4);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM4, 1000);
+		break;
+	case 5:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm5);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM5, 1000);
+		break;
+	case 6:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm6);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM6, 1000);
+		break;
+	case 7:
+		fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm7);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM7, 1000);
+		break;
+	default:
+		printk(KERN_ERR "DPM Level not supported\n");
+		fPowerDPMx = Convert_ULONG_ToFraction(1);
+		fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM0, 1000);
+	}
+
+	/*-------------------------
+	 * DECODING FUSE VALUES
+	 * ------------------------
+	 */
+	/*Decode RO_Fused*/
+	sRO_fuse = getASICProfilingInfo->sRoFuse;
+
+	sInput_FuseValues.usEfuseIndex = sRO_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sRO_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sRO_fuse.ucEfuseLength;
+
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	/* Finally, the actual fuse value */
+	ul_RO_fused = sOutput_FuseValues.ulEfuseValue;
+	fMin = GetScaledFraction(sRO_fuse.ulEfuseMin, 1);
+	fRange = GetScaledFraction(sRO_fuse.ulEfuseEncodeRange, 1);
+	fRO_fused = fDecodeLinearFuse(ul_RO_fused, fMin, fRange, sRO_fuse.ucEfuseLength);
+
+	sCACm_fuse = getASICProfilingInfo->sCACm;
+
+	sInput_FuseValues.usEfuseIndex = sCACm_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sCACm_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sCACm_fuse.ucEfuseLength;
+
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	ul_CACm_fused = sOutput_FuseValues.ulEfuseValue;
+	fMin = GetScaledFraction(sCACm_fuse.ulEfuseMin, 1000);
+	fRange = GetScaledFraction(sCACm_fuse.ulEfuseEncodeRange, 1000);
+
+	fCACm_fused = fDecodeLinearFuse(ul_CACm_fused, fMin, fRange, sCACm_fuse.ucEfuseLength);
+
+	sCACb_fuse = getASICProfilingInfo->sCACb;
+
+	sInput_FuseValues.usEfuseIndex = sCACb_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sCACb_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sCACb_fuse.ucEfuseLength;
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	ul_CACb_fused = sOutput_FuseValues.ulEfuseValue;
+	fMin = GetScaledFraction(sCACb_fuse.ulEfuseMin, 1000);
+	fRange = GetScaledFraction(sCACb_fuse.ulEfuseEncodeRange, 1000);
+
+	fCACb_fused = fDecodeLinearFuse(ul_CACb_fused, fMin, fRange, sCACb_fuse.ucEfuseLength);
+
+	sKt_Beta_fuse = getASICProfilingInfo->sKt_b;
+
+	sInput_FuseValues.usEfuseIndex = sKt_Beta_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sKt_Beta_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sKt_Beta_fuse.ucEfuseLength;
+
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	ul_Kt_Beta_fused = sOutput_FuseValues.ulEfuseValue;
+	fAverage = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeAverage, 1000);
+	fRange = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeRange, 1000);
+
+	fKt_Beta_fused = fDecodeLogisticFuse(ul_Kt_Beta_fused,
+			fAverage, fRange, sKt_Beta_fuse.ucEfuseLength);
+
+	sKv_m_fuse = getASICProfilingInfo->sKv_m;
+
+	sInput_FuseValues.usEfuseIndex = sKv_m_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sKv_m_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sKv_m_fuse.ucEfuseLength;
+
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+	if (result)
+		return result;
+
+	ul_Kv_m_fused = sOutput_FuseValues.ulEfuseValue;
+	fAverage = GetScaledFraction(sKv_m_fuse.ulEfuseEncodeAverage, 1000);
+	fRange = GetScaledFraction((sKv_m_fuse.ulEfuseEncodeRange & 0x7fffffff), 1000);
+	fRange = fMultiply(fRange, ConvertToFraction(-1));
+
+	fKv_m_fused = fDecodeLogisticFuse(ul_Kv_m_fused,
+			fAverage, fRange, sKv_m_fuse.ucEfuseLength);
+
+	sKv_b_fuse = getASICProfilingInfo->sKv_b;
+
+	sInput_FuseValues.usEfuseIndex = sKv_b_fuse.usEfuseIndex;
+	sInput_FuseValues.ucBitShift = sKv_b_fuse.ucEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = sKv_b_fuse.ucEfuseLength;
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	ul_Kv_b_fused = sOutput_FuseValues.ulEfuseValue;
+	fAverage = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeAverage, 1000);
+	fRange = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeRange, 1000);
+
+	fKv_b_fused = fDecodeLogisticFuse(ul_Kv_b_fused,
+			fAverage, fRange, sKv_b_fuse.ucEfuseLength);
+
+	/* Decoding the Leakage - No special struct container */
+	/*
+	 * usLkgEuseIndex=56
+	 * ucLkgEfuseBitLSB=6
+	 * ucLkgEfuseLength=10
+	 * ulLkgEncodeLn_MaxDivMin=69077
+	 * ulLkgEncodeMax=1000000
+	 * ulLkgEncodeMin=1000
+	 * ulEfuseLogisticAlpha=13
+	 */
+
+	sInput_FuseValues.usEfuseIndex = getASICProfilingInfo->usLkgEuseIndex;
+	sInput_FuseValues.ucBitShift = getASICProfilingInfo->ucLkgEfuseBitLSB;
+	sInput_FuseValues.ucBitLength = getASICProfilingInfo->ucLkgEfuseLength;
+
+	sOutput_FuseValues.sEfuse = sInput_FuseValues;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&sOutput_FuseValues);
+
+	if (result)
+		return result;
+
+	ul_FT_Lkg_V0NORM = sOutput_FuseValues.ulEfuseValue;
+	fLn_MaxDivMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeLn_MaxDivMin, 10000);
+	fMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeMin, 10000);
+
+	fFT_Lkg_V0NORM = fDecodeLeakageID(ul_FT_Lkg_V0NORM,
+			fLn_MaxDivMin, fMin, getASICProfilingInfo->ucLkgEfuseLength);
+	fLkg_FT = fFT_Lkg_V0NORM;
+
+	/*-------------------------------------------
+	 * PART 2 - Grabbing all required values
+	 *-------------------------------------------
+	 */
+	fSM_A0 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A0, 1000000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A0_sign)));
+	fSM_A1 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A1, 1000000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A1_sign)));
+	fSM_A2 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A2, 100000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A2_sign)));
+	fSM_A3 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A3, 1000000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A3_sign)));
+	fSM_A4 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A4, 1000000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A4_sign)));
+	fSM_A5 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A5, 1000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A5_sign)));
+	fSM_A6 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A6, 1000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A6_sign)));
+	fSM_A7 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A7, 1000),
+			ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A7_sign)));
+
+	fMargin_RO_a = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_a);
+	fMargin_RO_b = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_b);
+	fMargin_RO_c = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_c);
+
+	fMargin_fixed = ConvertToFraction(getASICProfilingInfo->ulMargin_fixed);
+
+	fMargin_FMAX_mean = GetScaledFraction(
+			getASICProfilingInfo->ulMargin_Fmax_mean, 10000);
+	fMargin_Plat_mean = GetScaledFraction(
+			getASICProfilingInfo->ulMargin_plat_mean, 10000);
+	fMargin_FMAX_sigma = GetScaledFraction(
+			getASICProfilingInfo->ulMargin_Fmax_sigma, 10000);
+	fMargin_Plat_sigma = GetScaledFraction(
+			getASICProfilingInfo->ulMargin_plat_sigma, 10000);
+
+	fMargin_DC_sigma = GetScaledFraction(
+			getASICProfilingInfo->ulMargin_DC_sigma, 100);
+	fMargin_DC_sigma = fDivide(fMargin_DC_sigma, ConvertToFraction(1000));
+
+	fCACm_fused = fDivide(fCACm_fused, ConvertToFraction(100));
+	fCACb_fused = fDivide(fCACb_fused, ConvertToFraction(100));
+	fKt_Beta_fused = fDivide(fKt_Beta_fused, ConvertToFraction(100));
+	fKv_m_fused =  fNegate(fDivide(fKv_m_fused, ConvertToFraction(100)));
+	fKv_b_fused = fDivide(fKv_b_fused, ConvertToFraction(10));
+
+	fSclk = GetScaledFraction(sclk, 100);
+
+	fV_max = fDivide(GetScaledFraction(
+			getASICProfilingInfo->ulMaxVddc, 1000), ConvertToFraction(4));
+	fT_prod = GetScaledFraction(getASICProfilingInfo->ulBoardCoreTemp, 10);
+	fLKG_Factor = GetScaledFraction(getASICProfilingInfo->ulEvvLkgFactor, 100);
+	fT_FT = GetScaledFraction(getASICProfilingInfo->ulLeakageTemp, 10);
+	fV_FT = fDivide(GetScaledFraction(
+			getASICProfilingInfo->ulLeakageVoltage, 1000), ConvertToFraction(4));
+	fV_min = fDivide(GetScaledFraction(
+			getASICProfilingInfo->ulMinVddc, 1000), ConvertToFraction(4));
+
+	/*-----------------------
+	 * PART 3
+	 *-----------------------
+	 */
+
+	fA_Term = fAdd(fMargin_RO_a, fAdd(fMultiply(fSM_A4,fSclk), fSM_A5));
+	fB_Term = fAdd(fAdd(fMultiply(fSM_A2, fSclk), fSM_A6), fMargin_RO_b);
+	fC_Term = fAdd(fMargin_RO_c,
+			fAdd(fMultiply(fSM_A0,fLkg_FT),
+			fAdd(fMultiply(fSM_A1, fMultiply(fLkg_FT,fSclk)),
+			fAdd(fMultiply(fSM_A3, fSclk),
+			fSubtract(fSM_A7,fRO_fused)))));
+
+	fVDDC_base = fSubtract(fRO_fused,
+			fSubtract(fMargin_RO_c,
+					fSubtract(fSM_A3, fMultiply(fSM_A1, fSclk))));
+	fVDDC_base = fDivide(fVDDC_base, fAdd(fMultiply(fSM_A0,fSclk), fSM_A2));
+
+	repeat = fSubtract(fVDDC_base,
+			fDivide(fMargin_DC_sigma, ConvertToFraction(1000)));
+
+	fRO_DC_margin = fAdd(fMultiply(fMargin_RO_a,
+			fGetSquare(repeat)),
+			fAdd(fMultiply(fMargin_RO_b, repeat),
+			fMargin_RO_c));
+
+	fDC_SCLK = fSubtract(fRO_fused,
+			fSubtract(fRO_DC_margin,
+			fSubtract(fSM_A3,
+			fMultiply(fSM_A2, repeat))));
+	fDC_SCLK = fDivide(fDC_SCLK, fAdd(fMultiply(fSM_A0,repeat), fSM_A1));
+
+	fSigma_DC = fSubtract(fSclk, fDC_SCLK);
+
+	fMicro_FMAX = fMultiply(fSclk, fMargin_FMAX_mean);
+	fMicro_CR = fMultiply(fSclk, fMargin_Plat_mean);
+	fSigma_FMAX = fMultiply(fSclk, fMargin_FMAX_sigma);
+	fSigma_CR = fMultiply(fSclk, fMargin_Plat_sigma);
+
+	fSquared_Sigma_DC = fGetSquare(fSigma_DC);
+	fSquared_Sigma_CR = fGetSquare(fSigma_CR);
+	fSquared_Sigma_FMAX = fGetSquare(fSigma_FMAX);
+
+	fSclk_margin = fAdd(fMicro_FMAX,
+			fAdd(fMicro_CR,
+			fAdd(fMargin_fixed,
+			fSqrt(fAdd(fSquared_Sigma_FMAX,
+			fAdd(fSquared_Sigma_DC, fSquared_Sigma_CR))))));
+	/*
+	 fA_Term = fSM_A4 * (fSclk + fSclk_margin) + fSM_A5;
+	 fB_Term = fSM_A2 * (fSclk + fSclk_margin) + fSM_A6;
+	 fC_Term = fRO_DC_margin + fSM_A0 * fLkg_FT + fSM_A1 * fLkg_FT * (fSclk + fSclk_margin) + fSM_A3 * (fSclk + fSclk_margin) + fSM_A7 - fRO_fused;
+	 */
+
+	fA_Term = fAdd(fMultiply(fSM_A4, fAdd(fSclk, fSclk_margin)), fSM_A5);
+	fB_Term = fAdd(fMultiply(fSM_A2, fAdd(fSclk, fSclk_margin)), fSM_A6);
+	fC_Term = fAdd(fRO_DC_margin,
+			fAdd(fMultiply(fSM_A0, fLkg_FT),
+			fAdd(fMultiply(fMultiply(fSM_A1, fLkg_FT),
+			fAdd(fSclk, fSclk_margin)),
+			fAdd(fMultiply(fSM_A3,
+			fAdd(fSclk, fSclk_margin)),
+			fSubtract(fSM_A7, fRO_fused)))));
+
+	SolveQuadracticEqn(fA_Term, fB_Term, fC_Term, fRoots);
+
+	if (GreaterThan(fRoots[0], fRoots[1]))
+		fEVV_V = fRoots[1];
+	else
+		fEVV_V = fRoots[0];
+
+	if (GreaterThan(fV_min, fEVV_V))
+		fEVV_V = fV_min;
+	else if (GreaterThan(fEVV_V, fV_max))
+		fEVV_V = fSubtract(fV_max, fStepSize);
+
+	fEVV_V = fRoundUpByStepSize(fEVV_V, fStepSize, 0);
+
+	/*-----------------
+	 * PART 4
+	 *-----------------
+	 */
+
+	fV_x = fV_min;
+
+	while (GreaterThan(fAdd(fV_max, fStepSize), fV_x)) {
+		fTDP_Power_left = fMultiply(fMultiply(fMultiply(fAdd(
+				fMultiply(fCACm_fused, fV_x), fCACb_fused), fSclk),
+				fGetSquare(fV_x)), fDerateTDP);
+
+		fTDP_Power_right = fMultiply(fFT_Lkg_V0NORM, fMultiply(fLKG_Factor,
+				fMultiply(fExponential(fMultiply(fAdd(fMultiply(fKv_m_fused,
+				fT_prod), fKv_b_fused), fV_x)), fV_x)));
+		fTDP_Power_right = fMultiply(fTDP_Power_right, fExponential(fMultiply(
+				fKt_Beta_fused, fT_prod)));
+		fTDP_Power_right = fDivide(fTDP_Power_right, fExponential(fMultiply(
+				fAdd(fMultiply(fKv_m_fused, fT_prod), fKv_b_fused), fV_FT)));
+		fTDP_Power_right = fDivide(fTDP_Power_right, fExponential(fMultiply(
+				fKt_Beta_fused, fT_FT)));
+
+		fTDP_Power = fAdd(fTDP_Power_left, fTDP_Power_right);
+
+		fTDP_Current = fDivide(fTDP_Power, fV_x);
+
+		fV_NL = fAdd(fV_x, fDivide(fMultiply(fTDP_Current, fRLL_LoadLine),
+				ConvertToFraction(10)));
+
+		fV_NL = fRoundUpByStepSize(fV_NL, fStepSize, 0);
+
+		if (GreaterThan(fV_max, fV_NL) &&
+			(GreaterThan(fV_NL,fEVV_V) ||
+			Equal(fV_NL, fEVV_V))) {
+			fV_NL = fMultiply(fV_NL, ConvertToFraction(1000));
+
+			*voltage = (uint16_t)fV_NL.partial.real;
+			break;
+		} else
+			fV_x = fAdd(fV_x, fStepSize);
+	}
+
+	return result;
+}
+
+/** atomctrl_get_voltage_evv_on_sclk gets voltage via call to ATOM COMMAND table.
+ * @param hwmgr               	input: pointer to hwManager
+ * @param voltage_type            input: type of EVV voltage VDDC or VDDGFX
+ * @param sclk                        input: in 10Khz unit. DPM state SCLK frequency
+ *                                   		which is define in PPTable SCLK/VDDC dependence
+ *				table associated with this virtual_voltage_Id
+ * @param virtual_voltage_Id      input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08
+ * @param voltage		       output: real voltage level in unit of mv
+ */
+int atomctrl_get_voltage_evv_on_sclk(
+		struct pp_hwmgr *hwmgr,
+		uint8_t voltage_type,
+		uint32_t sclk, uint16_t virtual_voltage_Id,
+		uint16_t *voltage)
+{
+	int result;
+	GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space;
+
+	get_voltage_info_param_space.ucVoltageType   =
+		voltage_type;
+	get_voltage_info_param_space.ucVoltageMode   =
+		ATOM_GET_VOLTAGE_EVV_VOLTAGE;
+	get_voltage_info_param_space.usVoltageLevel  =
+		virtual_voltage_Id;
+	get_voltage_info_param_space.ulSCLKFreq      =
+		sclk;
+
+	result = cgs_atom_exec_cmd_table(hwmgr->device,
+			GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
+			&get_voltage_info_param_space);
+
+	if (0 != result)
+		return result;
+
+	*voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
+			(&get_voltage_info_param_space))->usVoltageLevel;
+
+	return result;
+}
+
+/**
+ * Get the mpll reference clock in 10KHz
+ */
+uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr)
+{
+	ATOM_COMMON_TABLE_HEADER *fw_info;
+	uint32_t clock;
+	u8 frev, crev;
+	u16 size;
+
+	fw_info = (ATOM_COMMON_TABLE_HEADER *)
+		cgs_atom_get_data_table(hwmgr->device,
+				GetIndexIntoMasterTable(DATA, FirmwareInfo),
+				&size, &frev, &crev);
+
+	if (fw_info == NULL)
+		clock = 2700;
+	else {
+		if ((fw_info->ucTableFormatRevision == 2) &&
+			(le16_to_cpu(fw_info->usStructureSize) >= sizeof(ATOM_FIRMWARE_INFO_V2_1))) {
+			ATOM_FIRMWARE_INFO_V2_1 *fwInfo_2_1 =
+				(ATOM_FIRMWARE_INFO_V2_1 *)fw_info;
+			clock = (uint32_t)(le16_to_cpu(fwInfo_2_1->usMemoryReferenceClock));
+		} else {
+			ATOM_FIRMWARE_INFO *fwInfo_0_0 =
+				(ATOM_FIRMWARE_INFO *)fw_info;
+			clock = (uint32_t)(le16_to_cpu(fwInfo_0_0->usReferenceClock));
+		}
+	}
+
+	return clock;
+}
+
+/**
+ * Get the asic internal spread spectrum table
+ */
+static ATOM_ASIC_INTERNAL_SS_INFO *asic_internal_ss_get_ss_table(void *device)
+{
+	ATOM_ASIC_INTERNAL_SS_INFO *table = NULL;
+	u8 frev, crev;
+	u16 size;
+
+	table = (ATOM_ASIC_INTERNAL_SS_INFO *)
+		cgs_atom_get_data_table(device,
+			GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info),
+			&size, &frev, &crev);
+
+	return table;
+}
+
+/**
+ * Get the asic internal spread spectrum assignment
+ */
+static int asic_internal_ss_get_ss_asignment(struct pp_hwmgr *hwmgr,
+		const uint8_t clockSource,
+		const uint32_t clockSpeed,
+		pp_atomctrl_internal_ss_info *ssEntry)
+{
+	ATOM_ASIC_INTERNAL_SS_INFO *table;
+	ATOM_ASIC_SS_ASSIGNMENT *ssInfo;
+	int entry_found = 0;
+
+	memset(ssEntry, 0x00, sizeof(pp_atomctrl_internal_ss_info));
+
+	table = asic_internal_ss_get_ss_table(hwmgr->device);
+
+	if (NULL == table)
+		return -1;
+
+	ssInfo = &table->asSpreadSpectrum[0];
+
+	while (((uint8_t *)ssInfo - (uint8_t *)table) <
+		le16_to_cpu(table->sHeader.usStructureSize)) {
+		if ((clockSource == ssInfo->ucClockIndication) &&
+			((uint32_t)clockSpeed <= le32_to_cpu(ssInfo->ulTargetClockRange))) {
+			entry_found = 1;
+			break;
+		}
+
+		ssInfo = (ATOM_ASIC_SS_ASSIGNMENT *)((uint8_t *)ssInfo +
+				sizeof(ATOM_ASIC_SS_ASSIGNMENT));
+	}
+
+	if (entry_found) {
+		ssEntry->speed_spectrum_percentage =
+			ssInfo->usSpreadSpectrumPercentage;
+		ssEntry->speed_spectrum_rate = ssInfo->usSpreadRateInKhz;
+
+		if (((GET_DATA_TABLE_MAJOR_REVISION(table) == 2) &&
+			(GET_DATA_TABLE_MINOR_REVISION(table) >= 2)) ||
+			(GET_DATA_TABLE_MAJOR_REVISION(table) == 3)) {
+			ssEntry->speed_spectrum_rate /= 100;
+		}
+
+		switch (ssInfo->ucSpreadSpectrumMode) {
+		case 0:
+			ssEntry->speed_spectrum_mode =
+				pp_atomctrl_spread_spectrum_mode_down;
+			break;
+		case 1:
+			ssEntry->speed_spectrum_mode =
+				pp_atomctrl_spread_spectrum_mode_center;
+			break;
+		default:
+			ssEntry->speed_spectrum_mode =
+				pp_atomctrl_spread_spectrum_mode_down;
+			break;
+		}
+	}
+
+	return entry_found ? 0 : 1;
+}
+
+/**
+ * Get the memory clock spread spectrum info
+ */
+int atomctrl_get_memory_clock_spread_spectrum(
+		struct pp_hwmgr *hwmgr,
+		const uint32_t memory_clock,
+		pp_atomctrl_internal_ss_info *ssInfo)
+{
+	return asic_internal_ss_get_ss_asignment(hwmgr,
+			ASIC_INTERNAL_MEMORY_SS, memory_clock, ssInfo);
+}
+/**
+ * Get the engine clock spread spectrum info
+ */
+int atomctrl_get_engine_clock_spread_spectrum(
+		struct pp_hwmgr *hwmgr,
+		const uint32_t engine_clock,
+		pp_atomctrl_internal_ss_info *ssInfo)
+{
+	return asic_internal_ss_get_ss_asignment(hwmgr,
+			ASIC_INTERNAL_ENGINE_SS, engine_clock, ssInfo);
+}
+
+int atomctrl_read_efuse(void *device, uint16_t start_index,
+		uint16_t end_index, uint32_t mask, uint32_t *efuse)
+{
+	int result;
+	READ_EFUSE_VALUE_PARAMETER efuse_param;
+
+	efuse_param.sEfuse.usEfuseIndex = (start_index / 32) * 4;
+	efuse_param.sEfuse.ucBitShift = (uint8_t)
+			(start_index - ((start_index / 32) * 32));
+	efuse_param.sEfuse.ucBitLength  = (uint8_t)
+			((end_index - start_index) + 1);
+
+	result = cgs_atom_exec_cmd_table(device,
+			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
+			&efuse_param);
+	if (!result)
+		*efuse = efuse_param.ulEfuseValue & mask;
+
+	return result;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
new file mode 100644
index 000000000000..627420b80a5f
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef PP_ATOMVOLTAGECTRL_H
+#define PP_ATOMVOLTAGECTRL_H
+
+#include "hwmgr.h"
+
+#define MEM_TYPE_GDDR5  0x50
+#define MEM_TYPE_GDDR4  0x40
+#define MEM_TYPE_GDDR3  0x30
+#define MEM_TYPE_DDR2   0x20
+#define MEM_TYPE_GDDR1  0x10
+#define MEM_TYPE_DDR3   0xb0
+#define MEM_TYPE_MASK   0xF0
+
+
+/* As returned from PowerConnectorDetectionTable. */
+#define PP_ATOM_POWER_BUDGET_DISABLE_OVERDRIVE  0x80
+#define PP_ATOM_POWER_BUDGET_SHOW_WARNING       0x40
+#define PP_ATOM_POWER_BUDGET_SHOW_WAIVER        0x20
+#define PP_ATOM_POWER_POWER_BUDGET_BEHAVIOUR    0x0F
+
+/* New functions for Evergreen and beyond. */
+#define PP_ATOMCTRL_MAX_VOLTAGE_ENTRIES 32
+
+struct pp_atomctrl_clock_dividers {
+	uint32_t pll_post_divider;
+	uint32_t pll_feedback_divider;
+	uint32_t pll_ref_divider;
+	bool  enable_post_divider;
+};
+
+typedef struct pp_atomctrl_clock_dividers pp_atomctrl_clock_dividers;
+
+union pp_atomctrl_tcipll_fb_divider {
+	struct {
+		uint32_t ul_fb_div_frac : 14;
+		uint32_t ul_fb_div : 12;
+		uint32_t un_used : 6;
+	};
+	uint32_t ul_fb_divider;
+};
+
+typedef union pp_atomctrl_tcipll_fb_divider pp_atomctrl_tcipll_fb_divider;
+
+struct pp_atomctrl_clock_dividers_rv730 {
+	uint32_t pll_post_divider;
+	pp_atomctrl_tcipll_fb_divider mpll_feedback_divider;
+	uint32_t pll_ref_divider;
+	bool  enable_post_divider;
+	bool  enable_dithen;
+	uint32_t vco_mode;
+};
+typedef struct pp_atomctrl_clock_dividers_rv730 pp_atomctrl_clock_dividers_rv730;
+
+
+struct pp_atomctrl_clock_dividers_kong {
+	uint32_t    pll_post_divider;
+	uint32_t    real_clock;
+};
+typedef struct pp_atomctrl_clock_dividers_kong pp_atomctrl_clock_dividers_kong;
+
+struct pp_atomctrl_clock_dividers_ci {
+	uint32_t    pll_post_divider;               /* post divider value */
+	uint32_t    real_clock;
+	pp_atomctrl_tcipll_fb_divider   ul_fb_div;         /* Output Parameter: PLL FB divider */
+	uint8_t   uc_pll_ref_div;                      /* Output Parameter: PLL ref divider */
+	uint8_t   uc_pll_post_div;                      /* Output Parameter: PLL post divider */
+	uint8_t   uc_pll_cntl_flag;                    /*Output Flags: control flag */
+};
+typedef struct pp_atomctrl_clock_dividers_ci pp_atomctrl_clock_dividers_ci;
+
+struct pp_atomctrl_clock_dividers_vi {
+	uint32_t    pll_post_divider;               /* post divider value */
+	uint32_t    real_clock;
+	pp_atomctrl_tcipll_fb_divider   ul_fb_div;         /*Output Parameter: PLL FB divider */
+	uint8_t   uc_pll_ref_div;                      /*Output Parameter: PLL ref divider */
+	uint8_t   uc_pll_post_div;                     /*Output Parameter: PLL post divider */
+	uint8_t   uc_pll_cntl_flag;                    /*Output Flags: control flag */
+};
+typedef struct pp_atomctrl_clock_dividers_vi pp_atomctrl_clock_dividers_vi;
+
+union pp_atomctrl_s_mpll_fb_divider {
+	struct {
+		uint32_t cl_kf : 12;
+		uint32_t clk_frac : 12;
+		uint32_t un_used : 8;
+	};
+	uint32_t ul_fb_divider;
+};
+typedef union pp_atomctrl_s_mpll_fb_divider pp_atomctrl_s_mpll_fb_divider;
+
+enum pp_atomctrl_spread_spectrum_mode {
+	pp_atomctrl_spread_spectrum_mode_down = 0,
+	pp_atomctrl_spread_spectrum_mode_center
+};
+typedef enum pp_atomctrl_spread_spectrum_mode pp_atomctrl_spread_spectrum_mode;
+
+struct pp_atomctrl_memory_clock_param {
+	pp_atomctrl_s_mpll_fb_divider mpll_fb_divider;
+	uint32_t mpll_post_divider;
+	uint32_t bw_ctrl;
+	uint32_t dll_speed;
+	uint32_t vco_mode;
+	uint32_t yclk_sel;
+	uint32_t qdr;
+	uint32_t half_rate;
+};
+typedef struct pp_atomctrl_memory_clock_param pp_atomctrl_memory_clock_param;
+
+struct pp_atomctrl_internal_ss_info {
+	uint32_t speed_spectrum_percentage;                      /* in 1/100 percentage */
+	uint32_t speed_spectrum_rate;                            /* in KHz */
+	pp_atomctrl_spread_spectrum_mode speed_spectrum_mode;
+};
+typedef struct pp_atomctrl_internal_ss_info pp_atomctrl_internal_ss_info;
+
+#ifndef NUMBER_OF_M3ARB_PARAMS
+#define NUMBER_OF_M3ARB_PARAMS 3
+#endif
+
+#ifndef NUMBER_OF_M3ARB_PARAM_SETS
+#define NUMBER_OF_M3ARB_PARAM_SETS 10
+#endif
+
+struct pp_atomctrl_kong_system_info {
+	uint32_t			ul_bootup_uma_clock;          /* in 10kHz unit */
+	uint16_t			us_max_nb_voltage;            /* high NB voltage, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse; */
+	uint16_t			us_min_nb_voltage;            /* low NB voltage, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse; */
+	uint16_t			us_bootup_nb_voltage;         /* boot up NB voltage */
+	uint8_t			uc_htc_tmp_lmt;               /* bit [22:16] of D24F3x64 Hardware Thermal Control (HTC) Register, may not be needed, TBD */
+	uint8_t			uc_tj_offset;                /* bit [28:22] of D24F3xE4 Thermtrip Status Register,may not be needed, TBD */
+	/* 0: default 1: uvd 2: fs-3d */
+	uint32_t          ul_csr_m3_srb_cntl[NUMBER_OF_M3ARB_PARAM_SETS][NUMBER_OF_M3ARB_PARAMS];/* arrays with values for CSR M3 arbiter for default */
+};
+typedef struct pp_atomctrl_kong_system_info pp_atomctrl_kong_system_info;
+
+struct pp_atomctrl_memory_info {
+	uint8_t memory_vendor;
+	uint8_t memory_type;
+};
+typedef struct pp_atomctrl_memory_info pp_atomctrl_memory_info;
+
+#define MAX_AC_TIMING_ENTRIES 16
+
+struct pp_atomctrl_memory_clock_range_table {
+	uint8_t   num_entries;
+	uint8_t   rsv[3];
+
+	uint32_t mclk[MAX_AC_TIMING_ENTRIES];
+};
+typedef struct pp_atomctrl_memory_clock_range_table pp_atomctrl_memory_clock_range_table;
+
+struct pp_atomctrl_voltage_table_entry {
+	uint16_t value;
+	uint32_t smio_low;
+};
+
+typedef struct pp_atomctrl_voltage_table_entry pp_atomctrl_voltage_table_entry;
+
+struct pp_atomctrl_voltage_table {
+	uint32_t count;
+	uint32_t mask_low;
+	uint32_t phase_delay;   /* Used for ATOM_GPIO_VOLTAGE_OBJECT_V3 and later */
+	pp_atomctrl_voltage_table_entry entries[PP_ATOMCTRL_MAX_VOLTAGE_ENTRIES];
+};
+
+typedef struct pp_atomctrl_voltage_table pp_atomctrl_voltage_table;
+
+#define VBIOS_MC_REGISTER_ARRAY_SIZE           32
+#define VBIOS_MAX_AC_TIMING_ENTRIES            20
+
+struct pp_atomctrl_mc_reg_entry {
+	uint32_t           mclk_max;
+	uint32_t mc_data[VBIOS_MC_REGISTER_ARRAY_SIZE];
+};
+typedef struct pp_atomctrl_mc_reg_entry pp_atomctrl_mc_reg_entry;
+
+struct pp_atomctrl_mc_register_address {
+	uint16_t s1;
+	uint8_t  uc_pre_reg_data;
+};
+
+typedef struct pp_atomctrl_mc_register_address pp_atomctrl_mc_register_address;
+
+struct pp_atomctrl_mc_reg_table {
+	uint8_t                         last;                    /* number of registers */
+	uint8_t                         num_entries;             /* number of AC timing entries */
+	pp_atomctrl_mc_reg_entry        mc_reg_table_entry[VBIOS_MAX_AC_TIMING_ENTRIES];
+	pp_atomctrl_mc_register_address mc_reg_address[VBIOS_MC_REGISTER_ARRAY_SIZE];
+};
+typedef struct pp_atomctrl_mc_reg_table pp_atomctrl_mc_reg_table;
+
+struct pp_atomctrl_gpio_pin_assignment {
+	uint16_t                   us_gpio_pin_aindex;
+	uint8_t                    uc_gpio_pin_bit_shift;
+};
+typedef struct pp_atomctrl_gpio_pin_assignment pp_atomctrl_gpio_pin_assignment;
+
+extern bool atomctrl_get_pp_assign_pin(struct pp_hwmgr *hwmgr, const uint32_t pinId, pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment);
+extern int atomctrl_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage);
+extern uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr);
+extern int atomctrl_get_memory_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t memory_clock, pp_atomctrl_internal_ss_info *ssInfo);
+extern int atomctrl_get_engine_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t engine_clock, pp_atomctrl_internal_ss_info *ssInfo);
+extern int atomctrl_initialize_mc_reg_table(struct pp_hwmgr *hwmgr, uint8_t module_index, pp_atomctrl_mc_reg_table *table);
+extern int atomctrl_set_engine_dram_timings_rv770(struct pp_hwmgr *hwmgr, uint32_t engine_clock, uint32_t memory_clock);
+extern uint32_t atomctrl_get_reference_clock(struct pp_hwmgr *hwmgr);
+extern int atomctrl_get_memory_pll_dividers_si(struct pp_hwmgr *hwmgr, uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param, bool strobe_mode);
+extern int atomctrl_get_engine_pll_dividers_vi(struct pp_hwmgr *hwmgr, uint32_t clock_value, pp_atomctrl_clock_dividers_vi *dividers);
+extern int atomctrl_get_dfs_pll_dividers_vi(struct pp_hwmgr *hwmgr, uint32_t clock_value, pp_atomctrl_clock_dividers_vi *dividers);
+extern bool atomctrl_is_voltage_controled_by_gpio_v3(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint8_t voltage_mode);
+extern int atomctrl_get_voltage_table_v3(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint8_t voltage_mode, pp_atomctrl_voltage_table *voltage_table);
+extern int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
+		uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param);
+extern int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
+						 uint32_t clock_value,
+						 pp_atomctrl_clock_dividers_kong *dividers);
+extern int atomctrl_read_efuse(void *device, uint16_t start_index,
+		uint16_t end_index, uint32_t mask, uint32_t *efuse);
+extern int atomctrl_calculate_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
+		uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage, uint16_t dpm_level, bool debug);
+
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h
new file mode 100644
index 000000000000..b7429a527828
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h
@@ -0,0 +1,612 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <asm/div64.h>
+
+#define SHIFT_AMOUNT 16 /* We multiply all original integers with 2^SHIFT_AMOUNT to get the fInt representation */
+
+#define PRECISION 5 /* Change this value to change the number of decimal places in the final output - 5 is a good default */
+
+#define SHIFTED_2 (2 << SHIFT_AMOUNT)
+#define MAX (1 << (SHIFT_AMOUNT - 1)) - 1 /* 32767 - Might change in the future */
+
+/* -------------------------------------------------------------------------------
+ * NEW TYPE - fINT
+ * -------------------------------------------------------------------------------
+ * A variable of type fInt can be accessed in 3 ways using the dot (.) operator
+ * fInt A;
+ * A.full => The full number as it is. Generally not easy to read
+ * A.partial.real => Only the integer portion
+ * A.partial.decimal => Only the fractional portion
+ */
+typedef union _fInt {
+    int full;
+    struct _partial {
+        unsigned int decimal: SHIFT_AMOUNT; /*Needs to always be unsigned*/
+        int real: 32 - SHIFT_AMOUNT;
+    } partial;
+} fInt;
+
+/* -------------------------------------------------------------------------------
+ * Function Declarations
+ *  -------------------------------------------------------------------------------
+ */
+fInt ConvertToFraction(int);                       /* Use this to convert an INT to a FINT */
+fInt Convert_ULONG_ToFraction(uint32_t);              /* Use this to convert an uint32_t to a FINT */
+fInt GetScaledFraction(int, int);                  /* Use this to convert an INT to a FINT after scaling it by a factor */
+int ConvertBackToInteger(fInt);                    /* Convert a FINT back to an INT that is scaled by 1000 (i.e. last 3 digits are the decimal digits) */
+
+fInt fNegate(fInt);                                /* Returns -1 * input fInt value */
+fInt fAdd (fInt, fInt);                            /* Returns the sum of two fInt numbers */
+fInt fSubtract (fInt A, fInt B);                   /* Returns A-B - Sometimes easier than Adding negative numbers */
+fInt fMultiply (fInt, fInt);                       /* Returns the product of two fInt numbers */
+fInt fDivide (fInt A, fInt B);                     /* Returns A/B */
+fInt fGetSquare(fInt);                             /* Returns the square of a fInt number */
+fInt fSqrt(fInt);                                  /* Returns the Square Root of a fInt number */
+
+int uAbs(int);                                     /* Returns the Absolute value of the Int */
+fInt fAbs(fInt);                                   /* Returns the Absolute value of the fInt */
+int uPow(int base, int exponent);                  /* Returns base^exponent an INT */
+
+void SolveQuadracticEqn(fInt, fInt, fInt, fInt[]); /* Returns the 2 roots via the array */
+bool Equal(fInt, fInt);                         /* Returns true if two fInts are equal to each other */
+bool GreaterThan(fInt A, fInt B);               /* Returns true if A > B */
+
+fInt fExponential(fInt exponent);                  /* Can be used to calculate e^exponent */
+fInt fNaturalLog(fInt value);                      /* Can be used to calculate ln(value) */
+
+/* Fuse decoding functions
+ * -------------------------------------------------------------------------------------
+ */
+fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength);
+fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength);
+fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength);
+
+/* Internal Support Functions - Use these ONLY for testing or adding to internal functions
+ * -------------------------------------------------------------------------------------
+ * Some of the following functions take two INTs as their input - This is unsafe for a variety of reasons.
+ */
+fInt Add (int, int);                               /* Add two INTs and return Sum as FINT */
+fInt Multiply (int, int);                          /* Multiply two INTs and return Product as FINT */
+fInt Divide (int, int);                            /* You get the idea... */
+fInt fNegate(fInt);
+
+int uGetScaledDecimal (fInt);                      /* Internal function */
+int GetReal (fInt A);                              /* Internal function */
+
+/* Future Additions and Incomplete Functions
+ * -------------------------------------------------------------------------------------
+ */
+int GetRoundedValue(fInt);                         /* Incomplete function - Useful only when Precision is lacking */
+                                                   /* Let us say we have 2.126 but can only handle 2 decimal points. We could */
+                                                   /* either chop of 6 and keep 2.12 or use this function to get 2.13, which is more accurate */
+
+/* -------------------------------------------------------------------------------------
+ * TROUBLESHOOTING INFORMATION
+ * -------------------------------------------------------------------------------------
+ * 1) ConvertToFraction - InputOutOfRangeException: Only accepts numbers smaller than MAX (default: 32767)
+ * 2) fAdd - OutputOutOfRangeException: Output bigger than MAX (default: 32767)
+ * 3) fMultiply - OutputOutOfRangeException:
+ * 4) fGetSquare - OutputOutOfRangeException:
+ * 5) fDivide - DivideByZeroException
+ * 6) fSqrt - NegativeSquareRootException: Input cannot be a negative number
+ */
+
+/* -------------------------------------------------------------------------------------
+ * START OF CODE
+ * -------------------------------------------------------------------------------------
+ */
+fInt fExponential(fInt exponent)        /*Can be used to calculate e^exponent*/
+{
+	uint32_t i;
+	bool bNegated = false;
+
+	fInt fPositiveOne = ConvertToFraction(1);
+	fInt fZERO = ConvertToFraction(0);
+
+	fInt lower_bound = Divide(78, 10000);
+	fInt solution = fPositiveOne; /*Starting off with baseline of 1 */
+	fInt error_term;
+
+	uint32_t k_array[11] = {55452, 27726, 13863, 6931, 4055, 2231, 1178, 606, 308, 155, 78};
+	uint32_t expk_array[11] = {2560000, 160000, 40000, 20000, 15000, 12500, 11250, 10625, 10313, 10156, 10078};
+
+	if (GreaterThan(fZERO, exponent)) {
+		exponent = fNegate(exponent);
+		bNegated = true;
+	}
+
+	while (GreaterThan(exponent, lower_bound)) {
+		for (i = 0; i < 11; i++) {
+			if (GreaterThan(exponent, GetScaledFraction(k_array[i], 10000))) {
+				exponent = fSubtract(exponent, GetScaledFraction(k_array[i], 10000));
+				solution = fMultiply(solution, GetScaledFraction(expk_array[i], 10000));
+			}
+		}
+	}
+
+	error_term = fAdd(fPositiveOne, exponent);
+
+	solution = fMultiply(solution, error_term);
+
+	if (bNegated)
+		solution = fDivide(fPositiveOne, solution);
+
+	return solution;
+}
+
+fInt fNaturalLog(fInt value)
+{
+	uint32_t i;
+	fInt upper_bound = Divide(8, 1000);
+	fInt fNegativeOne = ConvertToFraction(-1);
+	fInt solution = ConvertToFraction(0); /*Starting off with baseline of 0 */
+	fInt error_term;
+
+	uint32_t k_array[10] = {160000, 40000, 20000, 15000, 12500, 11250, 10625, 10313, 10156, 10078};
+	uint32_t logk_array[10] = {27726, 13863, 6931, 4055, 2231, 1178, 606, 308, 155, 78};
+
+	while (GreaterThan(fAdd(value, fNegativeOne), upper_bound)) {
+		for (i = 0; i < 10; i++) {
+			if (GreaterThan(value, GetScaledFraction(k_array[i], 10000))) {
+				value = fDivide(value, GetScaledFraction(k_array[i], 10000));
+				solution = fAdd(solution, GetScaledFraction(logk_array[i], 10000));
+			}
+		}
+	}
+
+	error_term = fAdd(fNegativeOne, value);
+
+	return (fAdd(solution, error_term));
+}
+
+fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength)
+{
+	fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value);
+	fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
+
+	fInt f_decoded_value;
+
+	f_decoded_value = fDivide(f_fuse_value, f_bit_max_value);
+	f_decoded_value = fMultiply(f_decoded_value, f_range);
+	f_decoded_value = fAdd(f_decoded_value, f_min);
+
+	return f_decoded_value;
+}
+
+
+fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength)
+{
+	fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value);
+	fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
+
+	fInt f_CONSTANT_NEG13 = ConvertToFraction(-13);
+	fInt f_CONSTANT1 = ConvertToFraction(1);
+
+	fInt f_decoded_value;
+
+	f_decoded_value = fSubtract(fDivide(f_bit_max_value, f_fuse_value), f_CONSTANT1);
+	f_decoded_value = fNaturalLog(f_decoded_value);
+	f_decoded_value = fMultiply(f_decoded_value, fDivide(f_range, f_CONSTANT_NEG13));
+	f_decoded_value = fAdd(f_decoded_value, f_average);
+
+	return f_decoded_value;
+}
+
+fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength)
+{
+	fInt fLeakage;
+	fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
+
+	fLeakage = fMultiply(ln_max_div_min, Convert_ULONG_ToFraction(leakageID_fuse));
+	fLeakage = fDivide(fLeakage, f_bit_max_value);
+	fLeakage = fExponential(fLeakage);
+	fLeakage = fMultiply(fLeakage, f_min);
+
+	return fLeakage;
+}
+
+fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to make fInt a private declaration? */
+{
+	fInt temp;
+
+	if (X <= MAX)
+		temp.full = (X << SHIFT_AMOUNT);
+	else
+		temp.full = 0;
+
+	return temp;
+}
+
+fInt fNegate(fInt X)
+{
+	fInt CONSTANT_NEGONE = ConvertToFraction(-1);
+	return (fMultiply(X, CONSTANT_NEGONE));
+}
+
+fInt Convert_ULONG_ToFraction(uint32_t X)
+{
+	fInt temp;
+
+	if (X <= MAX)
+		temp.full = (X << SHIFT_AMOUNT);
+	else
+		temp.full = 0;
+
+	return temp;
+}
+
+fInt GetScaledFraction(int X, int factor)
+{
+	int times_shifted, factor_shifted;
+	bool bNEGATED;
+	fInt fValue;
+
+	times_shifted = 0;
+	factor_shifted = 0;
+	bNEGATED = false;
+
+	if (X < 0) {
+		X = -1*X;
+		bNEGATED = true;
+	}
+
+	if (factor < 0) {
+		factor = -1*factor;
+		bNEGATED = !bNEGATED; /*If bNEGATED = true due to X < 0, this will cover the case of negative cancelling negative */
+	}
+
+	if ((X > MAX) || factor > MAX) {
+		if ((X/factor) <= MAX) {
+			while (X > MAX) {
+				X = X >> 1;
+				times_shifted++;
+			}
+
+			while (factor > MAX) {
+				factor = factor >> 1;
+				factor_shifted++;
+			}
+		} else {
+			fValue.full = 0;
+			return fValue;
+		}
+	}
+
+	if (factor == 1)
+	return (ConvertToFraction(X));
+
+	fValue = fDivide(ConvertToFraction(X * uPow(-1, bNEGATED)), ConvertToFraction(factor));
+
+	fValue.full = fValue.full << times_shifted;
+	fValue.full = fValue.full >> factor_shifted;
+
+	return fValue;
+}
+
+/* Addition using two fInts */
+fInt fAdd (fInt X, fInt Y)
+{
+	fInt Sum;
+
+	Sum.full = X.full + Y.full;
+
+	return Sum;
+}
+
+/* Addition using two fInts */
+fInt fSubtract (fInt X, fInt Y)
+{
+	fInt Difference;
+
+	Difference.full = X.full - Y.full;
+
+	return Difference;
+}
+
+bool Equal(fInt A, fInt B)
+{
+	if (A.full == B.full)
+		return true;
+	else
+		return false;
+}
+
+bool GreaterThan(fInt A, fInt B)
+{
+	if (A.full > B.full)
+		return true;
+	else
+		return false;
+}
+
+fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */
+{
+	fInt Product;
+	int64_t tempProduct;
+	bool X_LessThanOne, Y_LessThanOne;
+
+	X_LessThanOne = (X.partial.real == 0 && X.partial.decimal != 0 && X.full >= 0);
+	Y_LessThanOne = (Y.partial.real == 0 && Y.partial.decimal != 0 && Y.full >= 0);
+
+	/*The following is for a very specific common case: Non-zero number with ONLY fractional portion*/
+	/* TEMPORARILY DISABLED - CAN BE USED TO IMPROVE PRECISION
+
+	if (X_LessThanOne && Y_LessThanOne) {
+		Product.full = X.full * Y.full;
+		return Product
+	}*/
+
+	tempProduct = ((int64_t)X.full) * ((int64_t)Y.full); /*Q(16,16)*Q(16,16) = Q(32, 32) - Might become a negative number! */
+	tempProduct = tempProduct >> 16; /*Remove lagging 16 bits - Will lose some precision from decimal; */
+	Product.full = (int)tempProduct; /*The int64_t will lose the leading 16 bits that were part of the integer portion */
+
+	return Product;
+}
+
+fInt fDivide (fInt X, fInt Y)
+{
+	fInt fZERO, fQuotient;
+	int64_t longlongX, longlongY;
+
+	fZERO = ConvertToFraction(0);
+
+	if (Equal(Y, fZERO))
+	return fZERO;
+
+	longlongX = (int64_t)X.full;
+	longlongY = (int64_t)Y.full;
+
+	longlongX = longlongX << 16; /*Q(16,16) -> Q(32,32) */
+
+	div64_s64(longlongX, longlongY); /*Q(32,32) divided by Q(16,16) = Q(16,16) Back to original format */
+
+	fQuotient.full = (int)longlongX;
+	return fQuotient;
+}
+
+int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to check with the Golden settings table*/
+{
+	fInt fullNumber, scaledDecimal, scaledReal;
+
+	scaledReal.full = GetReal(A) * uPow(10, PRECISION-1); /* DOUBLE CHECK THISSSS!!! */
+
+	scaledDecimal.full = uGetScaledDecimal(A);
+
+	fullNumber = fAdd(scaledDecimal,scaledReal);
+
+	return fullNumber.full;
+}
+
+fInt fGetSquare(fInt A)
+{
+	return fMultiply(A,A);
+}
+
+/* x_new = x_old - (x_old^2 - C) / (2 * x_old) */
+fInt fSqrt(fInt num)
+{
+	fInt F_divide_Fprime, Fprime;
+	fInt test;
+	fInt twoShifted;
+	int seed, counter, error;
+	fInt x_new, x_old, C, y;
+
+	fInt fZERO = ConvertToFraction(0);
+
+	/* (0 > num) is the same as (num < 0), i.e., num is negative */
+
+	if (GreaterThan(fZERO, num) || Equal(fZERO, num))
+		return fZERO;
+
+	C = num;
+
+	if (num.partial.real > 3000)
+		seed = 60;
+	else if (num.partial.real > 1000)
+		seed = 30;
+	else if (num.partial.real > 100)
+		seed = 10;
+	else
+		seed = 2;
+
+	counter = 0;
+
+	if (Equal(num, fZERO)) /*Square Root of Zero is zero */
+		return fZERO;
+
+	twoShifted = ConvertToFraction(2);
+	x_new = ConvertToFraction(seed);
+
+	do {
+		counter++;
+
+		x_old.full = x_new.full;
+
+		test = fGetSquare(x_old); /*1.75*1.75 is reverting back to 1 when shifted down */
+		y = fSubtract(test, C); /*y = f(x) = x^2 - C; */
+
+		Fprime = fMultiply(twoShifted, x_old);
+		F_divide_Fprime = fDivide(y, Fprime);
+
+		x_new = fSubtract(x_old, F_divide_Fprime);
+
+		error = ConvertBackToInteger(x_new) - ConvertBackToInteger(x_old);
+
+		if (counter > 20) /*20 is already way too many iterations. If we dont have an answer by then, we never will*/
+			return x_new;
+
+	} while (uAbs(error) > 0);
+
+	return (x_new);
+}
+
+void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[])
+{
+	fInt *pRoots = &Roots[0];
+	fInt temp, root_first, root_second;
+	fInt f_CONSTANT10, f_CONSTANT100;
+
+	f_CONSTANT100 = ConvertToFraction(100);
+	f_CONSTANT10 = ConvertToFraction(10);
+
+	while(GreaterThan(A, f_CONSTANT100) || GreaterThan(B, f_CONSTANT100) || GreaterThan(C, f_CONSTANT100)) {
+		A = fDivide(A, f_CONSTANT10);
+		B = fDivide(B, f_CONSTANT10);
+		C = fDivide(C, f_CONSTANT10);
+	}
+
+	temp = fMultiply(ConvertToFraction(4), A); /* root = 4*A */
+	temp = fMultiply(temp, C); /* root = 4*A*C */
+	temp = fSubtract(fGetSquare(B), temp); /* root = b^2 - 4AC */
+	temp = fSqrt(temp); /*root = Sqrt (b^2 - 4AC); */
+
+	root_first = fSubtract(fNegate(B), temp); /* b - Sqrt(b^2 - 4AC) */
+	root_second = fAdd(fNegate(B), temp); /* b + Sqrt(b^2 - 4AC) */
+
+	root_first = fDivide(root_first, ConvertToFraction(2)); /* [b +- Sqrt(b^2 - 4AC)]/[2] */
+	root_first = fDivide(root_first, A); /*[b +- Sqrt(b^2 - 4AC)]/[2*A] */
+
+	root_second = fDivide(root_second, ConvertToFraction(2)); /* [b +- Sqrt(b^2 - 4AC)]/[2] */
+	root_second = fDivide(root_second, A); /*[b +- Sqrt(b^2 - 4AC)]/[2*A] */
+
+	*(pRoots + 0) = root_first;
+	*(pRoots + 1) = root_second;
+}
+
+/* -----------------------------------------------------------------------------
+ * SUPPORT FUNCTIONS
+ * -----------------------------------------------------------------------------
+ */
+
+/* Addition using two normal ints - Temporary - Use only for testing purposes?. */
+fInt Add (int X, int Y)
+{
+	fInt A, B, Sum;
+
+	A.full = (X << SHIFT_AMOUNT);
+	B.full = (Y << SHIFT_AMOUNT);
+
+	Sum.full = A.full + B.full;
+
+	return Sum;
+}
+
+/* Conversion Functions */
+int GetReal (fInt A)
+{
+	return (A.full >> SHIFT_AMOUNT);
+}
+
+/* Temporarily Disabled */
+int GetRoundedValue(fInt A) /*For now, round the 3rd decimal place */
+{
+	/* ROUNDING TEMPORARLY DISABLED
+	int temp = A.full;
+	int decimal_cutoff, decimal_mask = 0x000001FF;
+	decimal_cutoff = temp & decimal_mask;
+	if (decimal_cutoff > 0x147) {
+		temp += 673;
+	}*/
+
+	return ConvertBackToInteger(A)/10000; /*Temporary - in case this was used somewhere else */
+}
+
+fInt Multiply (int X, int Y)
+{
+	fInt A, B, Product;
+
+	A.full = X << SHIFT_AMOUNT;
+	B.full = Y << SHIFT_AMOUNT;
+
+	Product = fMultiply(A, B);
+
+	return Product;
+}
+
+fInt Divide (int X, int Y)
+{
+	fInt A, B, Quotient;
+
+	A.full = X << SHIFT_AMOUNT;
+	B.full = Y << SHIFT_AMOUNT;
+
+	Quotient = fDivide(A, B);
+
+	return Quotient;
+}
+
+int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole integers - Costly function */
+{
+	int dec[PRECISION];
+	int i, scaledDecimal = 0, tmp = A.partial.decimal;
+
+	for (i = 0; i < PRECISION; i++) {
+		dec[i] = tmp / (1 << SHIFT_AMOUNT);
+		tmp = tmp - ((1 << SHIFT_AMOUNT)*dec[i]);
+		tmp *= 10;
+		scaledDecimal = scaledDecimal + dec[i]*uPow(10, PRECISION - 1 -i);
+	}
+
+	return scaledDecimal;
+}
+
+int uPow(int base, int power)
+{
+	if (power == 0)
+		return 1;
+	else
+		return (base)*uPow(base, power - 1);
+}
+
+fInt fAbs(fInt A)
+{
+	if (A.partial.real < 0)
+		return (fMultiply(A, ConvertToFraction(-1)));
+	else
+		return A;
+}
+
+int uAbs(int X)
+{
+	if (X < 0)
+		return (X * -1);
+	else
+		return X;
+}
+
+fInt fRoundUpByStepSize(fInt A, fInt fStepSize, bool error_term)
+{
+	fInt solution;
+
+	solution = fDivide(A, fStepSize);
+	solution.partial.decimal = 0; /*All fractional digits changes to 0 */
+
+	if (error_term)
+		solution.partial.real += 1; /*Error term of 1 added */
+
+	solution = fMultiply(solution, fStepSize);
+	solution = fAdd(solution, fStepSize);
+
+	return solution;
+}
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.c
new file mode 100644
index 000000000000..186496a34cbe
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include "atom-types.h"
+#include "atombios.h"
+#include "pppcielanes.h"
+
+/** \file
+ * Functions related to PCIe lane changes.
+ */
+
+/* For converting from number of lanes to lane bits.  */
+static const unsigned char pp_r600_encode_lanes[] = {
+	0,          /*  0 Not Supported  */
+	1,          /*  1 Lane  */
+	2,          /*  2 Lanes  */
+	0,          /*  3 Not Supported  */
+	3,          /*  4 Lanes  */
+	0,          /*  5 Not Supported  */
+	0,          /*  6 Not Supported  */
+	0,          /*  7 Not Supported  */
+	4,          /*  8 Lanes  */
+	0,          /*  9 Not Supported  */
+	0,          /* 10 Not Supported  */
+	0,          /* 11 Not Supported  */
+	5,          /* 12 Lanes (Not actually supported)  */
+	0,          /* 13 Not Supported  */
+	0,          /* 14 Not Supported  */
+	0,          /* 15 Not Supported  */
+	6           /* 16 Lanes  */
+};
+
+static const unsigned char pp_r600_decoded_lanes[8] = { 16, 1, 2, 4, 8, 12, 16, };
+
+uint8_t encode_pcie_lane_width(uint32_t num_lanes)
+{
+	return pp_r600_encode_lanes[num_lanes];
+}
+
+uint8_t decode_pcie_lane_width(uint32_t num_lanes)
+{
+	return pp_r600_decoded_lanes[num_lanes];
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.h b/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.h
new file mode 100644
index 000000000000..70b163b35570
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef PP_PCIELANES_H
+#define PP_PCIELANES_H
+
+extern uint8_t encode_pcie_lane_width(uint32_t num_lanes);
+extern uint8_t decode_pcie_lane_width(uint32_t num_lanes);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
new file mode 100644
index 000000000000..2f1a14fe05b1
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -0,0 +1,1688 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+
+#include "processpptables.h"
+#include <atom-types.h>
+#include <atombios.h>
+#include "pp_debug.h"
+#include "pptable.h"
+#include "power_state.h"
+#include "hwmgr.h"
+#include "hardwaremanager.h"
+
+
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4 16
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5 18
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6 20
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7 22
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8 24
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V9 26
+
+#define NUM_BITS_CLOCK_INFO_ARRAY_INDEX 6
+
+static uint16_t get_vce_table_offset(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t vce_table_offset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	   sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
+						(const ATOM_PPLIB_EXTENDEDHEADER *)
+						(((unsigned long)powerplay_table3) +
+						le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(extended_header->usSize) >=
+			   SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2)
+				vce_table_offset = le16_to_cpu(extended_header->usVCETableOffset);
+		}
+	}
+
+	return vce_table_offset;
+}
+
+static uint16_t get_vce_clock_info_array_offset(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_table_offset(hwmgr,
+						powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset + 1;
+
+	return 0;
+}
+
+static uint16_t get_vce_clock_info_array_size(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_clock_info_array_offset(hwmgr,
+							powerplay_table);
+	uint16_t table_size = 0;
+
+	if (table_offset > 0) {
+		const VCEClockInfoArray *p = (const VCEClockInfoArray *)
+			(((unsigned long) powerplay_table) + table_offset);
+		table_size = sizeof(uint8_t) + p->ucNumEntries * sizeof(VCEClockInfo);
+	}
+
+	return table_size;
+}
+
+static uint16_t get_vce_clock_voltage_limit_table_offset(struct pp_hwmgr *hwmgr,
+				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_clock_info_array_offset(hwmgr,
+							powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset + get_vce_clock_info_array_size(hwmgr,
+							powerplay_table);
+
+	return 0;
+}
+
+static uint16_t get_vce_clock_voltage_limit_table_size(struct pp_hwmgr *hwmgr,
+							const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_clock_voltage_limit_table_offset(hwmgr, powerplay_table);
+	uint16_t table_size = 0;
+
+	if (table_offset > 0) {
+		const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *ptable =
+			(const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *)(((unsigned long) powerplay_table) + table_offset);
+
+		table_size = sizeof(uint8_t) + ptable->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record);
+	}
+	return table_size;
+}
+
+static uint16_t get_vce_state_table_offset(struct pp_hwmgr *hwmgr, const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_clock_voltage_limit_table_offset(hwmgr, powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset + get_vce_clock_voltage_limit_table_size(hwmgr, powerplay_table);
+
+	return 0;
+}
+
+static const ATOM_PPLIB_VCE_State_Table *get_vce_state_table(
+						struct pp_hwmgr *hwmgr,
+						const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_vce_state_table_offset(hwmgr, powerplay_table);
+
+	if (table_offset > 0)
+		return (const ATOM_PPLIB_VCE_State_Table *)(((unsigned long) powerplay_table) + table_offset);
+
+	return NULL;
+}
+
+static uint16_t get_uvd_table_offset(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t uvd_table_offset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
+					(const ATOM_PPLIB_EXTENDEDHEADER *)
+					(((unsigned long)powerplay_table3) +
+				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(extended_header->usSize) >=
+			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3)
+				uvd_table_offset = le16_to_cpu(extended_header->usUVDTableOffset);
+		}
+	}
+	return uvd_table_offset;
+}
+
+static uint16_t get_uvd_clock_info_array_offset(struct pp_hwmgr *hwmgr,
+			 const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_uvd_table_offset(hwmgr,
+						    powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset + 1;
+	return 0;
+}
+
+static uint16_t get_uvd_clock_info_array_size(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_uvd_clock_info_array_offset(hwmgr,
+						    powerplay_table);
+	uint16_t table_size = 0;
+
+	if (table_offset > 0) {
+		const UVDClockInfoArray *p = (const UVDClockInfoArray *)
+					(((unsigned long) powerplay_table)
+					+ table_offset);
+		table_size = sizeof(UCHAR) +
+			     p->ucNumEntries * sizeof(UVDClockInfo);
+	}
+
+	return table_size;
+}
+
+static uint16_t get_uvd_clock_voltage_limit_table_offset(
+			struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_uvd_clock_info_array_offset(hwmgr,
+						     powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset +
+			get_uvd_clock_info_array_size(hwmgr, powerplay_table);
+
+	return 0;
+}
+
+static uint16_t get_samu_table_offset(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t samu_table_offset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
+				(const ATOM_PPLIB_EXTENDEDHEADER *)
+				(((unsigned long)powerplay_table3) +
+				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(extended_header->usSize) >=
+			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4)
+				samu_table_offset = le16_to_cpu(extended_header->usSAMUTableOffset);
+		}
+	}
+
+	return samu_table_offset;
+}
+
+static uint16_t get_samu_clock_voltage_limit_table_offset(
+			struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t table_offset = get_samu_table_offset(hwmgr,
+					    powerplay_table);
+
+	if (table_offset > 0)
+		return table_offset + 1;
+
+	return 0;
+}
+
+static uint16_t get_acp_table_offset(struct pp_hwmgr *hwmgr,
+				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t acp_table_offset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
+				(const ATOM_PPLIB_EXTENDEDHEADER *)
+				(((unsigned long)powerplay_table3) +
+				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(pExtendedHeader->usSize) >=
+			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6)
+				acp_table_offset = le16_to_cpu(pExtendedHeader->usACPTableOffset);
+		}
+	}
+
+	return acp_table_offset;
+}
+
+static uint16_t get_acp_clock_voltage_limit_table_offset(
+				struct pp_hwmgr *hwmgr,
+				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t tableOffset = get_acp_table_offset(hwmgr, powerplay_table);
+
+	if (tableOffset > 0)
+		return tableOffset + 1;
+
+	return 0;
+}
+
+static uint16_t get_cacp_tdp_table_offset(
+				struct pp_hwmgr *hwmgr,
+				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t cacTdpTableOffset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+				(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
+					(const ATOM_PPLIB_EXTENDEDHEADER *)
+					(((unsigned long)powerplay_table3) +
+				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(pExtendedHeader->usSize) >=
+			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7)
+				cacTdpTableOffset = le16_to_cpu(pExtendedHeader->usPowerTuneTableOffset);
+		}
+	}
+
+	return cacTdpTableOffset;
+}
+
+static int get_cac_tdp_table(struct pp_hwmgr *hwmgr,
+				struct phm_cac_tdp_table **ptable,
+				const ATOM_PowerTune_Table *table,
+				uint16_t us_maximum_power_delivery_limit)
+{
+	unsigned long table_size;
+	struct phm_cac_tdp_table *tdp_table;
+
+	table_size = sizeof(unsigned long) + sizeof(struct phm_cac_tdp_table);
+
+	tdp_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == tdp_table)
+		return -ENOMEM;
+
+	tdp_table->usTDP = le16_to_cpu(table->usTDP);
+	tdp_table->usConfigurableTDP = le16_to_cpu(table->usConfigurableTDP);
+	tdp_table->usTDC = le16_to_cpu(table->usTDC);
+	tdp_table->usBatteryPowerLimit = le16_to_cpu(table->usBatteryPowerLimit);
+	tdp_table->usSmallPowerLimit = le16_to_cpu(table->usSmallPowerLimit);
+	tdp_table->usLowCACLeakage = le16_to_cpu(table->usLowCACLeakage);
+	tdp_table->usHighCACLeakage = le16_to_cpu(table->usHighCACLeakage);
+	tdp_table->usMaximumPowerDeliveryLimit = us_maximum_power_delivery_limit;
+
+	*ptable = tdp_table;
+
+	return 0;
+}
+
+static uint16_t get_sclk_vdd_gfx_table_offset(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t sclk_vdd_gfx_table_offset = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
+		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
+				(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
+			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
+				(const ATOM_PPLIB_EXTENDEDHEADER *)
+				(((unsigned long)powerplay_table3) +
+				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+			if (le16_to_cpu(pExtendedHeader->usSize) >=
+			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8)
+				sclk_vdd_gfx_table_offset =
+					le16_to_cpu(pExtendedHeader->usSclkVddgfxTableOffset);
+		}
+	}
+
+	return sclk_vdd_gfx_table_offset;
+}
+
+static uint16_t get_sclk_vdd_gfx_clock_voltage_dependency_table_offset(
+			struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	uint16_t tableOffset = get_sclk_vdd_gfx_table_offset(hwmgr, powerplay_table);
+
+	if (tableOffset > 0)
+		return tableOffset;
+
+	return 0;
+}
+
+
+static int get_clock_voltage_dependency_table(struct pp_hwmgr *hwmgr,
+		struct phm_clock_voltage_dependency_table **ptable,
+		const ATOM_PPLIB_Clock_Voltage_Dependency_Table *table)
+{
+
+	unsigned long table_size, i;
+	struct phm_clock_voltage_dependency_table *dep_table;
+
+	table_size = sizeof(unsigned long) +
+		sizeof(struct phm_clock_voltage_dependency_table)
+		* table->ucNumEntries;
+
+	dep_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == dep_table)
+		return -ENOMEM;
+
+	dep_table->count = (unsigned long)table->ucNumEntries;
+
+	for (i = 0; i < dep_table->count; i++) {
+		dep_table->entries[i].clk =
+			((unsigned long)table->entries[i].ucClockHigh << 16) |
+			le16_to_cpu(table->entries[i].usClockLow);
+			dep_table->entries[i].v =
+				(unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+	}
+
+	*ptable = dep_table;
+
+	return 0;
+}
+
+static int get_valid_clk(struct pp_hwmgr *hwmgr,
+			struct phm_clock_array **ptable,
+			const struct phm_clock_voltage_dependency_table *table)
+{
+	unsigned long table_size, i;
+	struct phm_clock_array *clock_table;
+
+	table_size = sizeof(unsigned long) + sizeof(unsigned long) * table->count;
+	clock_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == clock_table)
+		return -ENOMEM;
+
+	clock_table->count = (unsigned long)table->count;
+
+	for (i = 0; i < clock_table->count; i++)
+		clock_table->values[i] = (unsigned long)table->entries[i].clk;
+
+	*ptable = clock_table;
+
+	return 0;
+}
+
+static int get_clock_voltage_limit(struct pp_hwmgr *hwmgr,
+			struct phm_clock_and_voltage_limits *limits,
+			const ATOM_PPLIB_Clock_Voltage_Limit_Table *table)
+{
+	limits->sclk = ((unsigned long)table->entries[0].ucSclkHigh << 16) |
+			le16_to_cpu(table->entries[0].usSclkLow);
+	limits->mclk = ((unsigned long)table->entries[0].ucMclkHigh << 16) |
+			le16_to_cpu(table->entries[0].usMclkLow);
+	limits->vddc = (unsigned long)le16_to_cpu(table->entries[0].usVddc);
+	limits->vddci = (unsigned long)le16_to_cpu(table->entries[0].usVddci);
+
+	return 0;
+}
+
+
+static void set_hw_cap(struct pp_hwmgr *hwmgr, bool enable,
+		       enum phm_platform_caps cap)
+{
+	if (enable)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps, cap);
+	else
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, cap);
+}
+
+static int set_platform_caps(struct pp_hwmgr *hwmgr,
+			unsigned long powerplay_caps)
+{
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_POWERPLAY),
+		PHM_PlatformCaps_PowerPlaySupport
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_SBIOSPOWERSOURCE),
+		PHM_PlatformCaps_BiosPowerSourceControl
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s),
+		PHM_PlatformCaps_EnableASPML0s
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1),
+		PHM_PlatformCaps_EnableASPML1
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS),
+		PHM_PlatformCaps_EnableBackbias
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC),
+		PHM_PlatformCaps_AutomaticDCTransition
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_GEMINIPRIMARY),
+		PHM_PlatformCaps_GeminiPrimary
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC),
+		PHM_PlatformCaps_StepVddc
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_VOLTAGECONTROL),
+		PHM_PlatformCaps_EnableVoltageControl
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_SIDEPORTCONTROL),
+		PHM_PlatformCaps_EnableSideportControl
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1),
+		PHM_PlatformCaps_TurnOffPll_ASPML1
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_HTLINKCONTROL),
+		PHM_PlatformCaps_EnableHTLinkControl
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL),
+		PHM_PlatformCaps_EnableMVDDControl
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL),
+		PHM_PlatformCaps_ControlVDDCI
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT),
+		PHM_PlatformCaps_RegulatorHot
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT),
+		PHM_PlatformCaps_BootStateOnAlert
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT),
+		PHM_PlatformCaps_DontWaitForVBlankOnAlert
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_BACO),
+		PHM_PlatformCaps_BACO
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE),
+		PHM_PlatformCaps_NewCACVoltage
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY),
+		PHM_PlatformCaps_RevertGPIO5Polarity
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_OUTPUT_THERMAL2GPIO17),
+		PHM_PlatformCaps_Thermal2GPIO17
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE),
+		PHM_PlatformCaps_VRHotGPIOConfigurable
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_TEMP_INVERSION),
+		PHM_PlatformCaps_TempInversion
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_EVV),
+		PHM_PlatformCaps_EVV
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_COMBINE_PCC_WITH_THERMAL_SIGNAL),
+		PHM_PlatformCaps_CombinePCCWithThermalSignal
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_LOAD_POST_PRODUCTION_FIRMWARE),
+		PHM_PlatformCaps_LoadPostProductionFirmware
+	);
+
+	set_hw_cap(
+		hwmgr,
+		0 != (powerplay_caps & ATOM_PP_PLATFORM_CAP_DISABLE_USING_ACTUAL_TEMPERATURE_FOR_POWER_CALC),
+		PHM_PlatformCaps_DisableUsingActualTemperatureForPowerCalc
+	);
+
+	return 0;
+}
+
+static PP_StateClassificationFlags make_classification_flags(
+						   struct pp_hwmgr *hwmgr,
+						    USHORT classification,
+						   USHORT classification2)
+{
+	PP_StateClassificationFlags result = 0;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_BOOT)
+		result |= PP_StateClassificationFlag_Boot;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+		result |= PP_StateClassificationFlag_Thermal;
+
+	if (classification &
+			ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE)
+		result |= PP_StateClassificationFlag_LimitedPowerSource;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_REST)
+		result |= PP_StateClassificationFlag_Rest;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_FORCED)
+		result |= PP_StateClassificationFlag_Forced;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
+		result |= PP_StateClassificationFlag_3DPerformance;
+
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE)
+		result |= PP_StateClassificationFlag_ACOverdriveTemplate;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+		result |= PP_StateClassificationFlag_Uvd;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_HDSTATE)
+		result |= PP_StateClassificationFlag_UvdHD;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_SDSTATE)
+		result |= PP_StateClassificationFlag_UvdSD;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_HD2STATE)
+		result |= PP_StateClassificationFlag_HD2;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_ACPI)
+		result |= PP_StateClassificationFlag_ACPI;
+
+	if (classification2 & ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2)
+		result |= PP_StateClassificationFlag_LimitedPowerSource_2;
+
+
+	if (classification2 & ATOM_PPLIB_CLASSIFICATION2_ULV)
+		result |= PP_StateClassificationFlag_ULV;
+
+	if (classification2 & ATOM_PPLIB_CLASSIFICATION2_MVC)
+		result |= PP_StateClassificationFlag_UvdMVC;
+
+	return result;
+}
+
+static int init_non_clock_fields(struct pp_hwmgr *hwmgr,
+						struct pp_power_state *ps,
+							    uint8_t version,
+			 const ATOM_PPLIB_NONCLOCK_INFO *pnon_clock_info) {
+	unsigned long rrr_index;
+	unsigned long tmp;
+
+	ps->classification.ui_label = (le16_to_cpu(pnon_clock_info->usClassification) &
+					ATOM_PPLIB_CLASSIFICATION_UI_MASK) >> ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
+	ps->classification.flags = make_classification_flags(hwmgr,
+				le16_to_cpu(pnon_clock_info->usClassification),
+				le16_to_cpu(pnon_clock_info->usClassification2));
+
+	ps->classification.temporary_state = false;
+	ps->classification.to_be_deleted = false;
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_SINGLE_DISPLAY_ONLY;
+
+	ps->validation.singleDisplayOnly = (0 != tmp);
+
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_DISALLOW_ON_DC;
+
+	ps->validation.disallowOnDC = (0 != tmp);
+
+	ps->pcie.lanes = ((le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+				ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >>
+				ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1;
+
+	ps->pcie.lanes = 0;
+
+	ps->display.disableFrameModulation = false;
+
+	rrr_index = (le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+			ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_MASK) >>
+			ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_SHIFT;
+
+	if (rrr_index != ATOM_PPLIB_LIMITED_REFRESHRATE_UNLIMITED) {
+		static const uint8_t look_up[(ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_MASK >> ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_SHIFT) + 1] = \
+								{ 0, 50, 0 };
+
+		ps->display.refreshrateSource = PP_RefreshrateSource_Explicit;
+		ps->display.explicitRefreshrate = look_up[rrr_index];
+		ps->display.limitRefreshrate = true;
+
+		if (ps->display.explicitRefreshrate == 0)
+			ps->display.limitRefreshrate = false;
+	} else
+		ps->display.limitRefreshrate = false;
+
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_ENABLE_VARIBRIGHT;
+
+	ps->display.enableVariBright = (0 != tmp);
+
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_SWSTATE_MEMORY_DLL_OFF;
+
+	ps->memory.dllOff = (0 != tmp);
+
+	ps->memory.m3arb = (le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+			    ATOM_PPLIB_M3ARB_MASK) >> ATOM_PPLIB_M3ARB_SHIFT;
+
+	ps->temperatures.min = PP_TEMPERATURE_UNITS_PER_CENTIGRADES *
+				     pnon_clock_info->ucMinTemperature;
+
+	ps->temperatures.max = PP_TEMPERATURE_UNITS_PER_CENTIGRADES *
+				     pnon_clock_info->ucMaxTemperature;
+
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_SOFTWARE_DISABLE_LOADBALANCING;
+
+	ps->software.disableLoadBalancing = tmp;
+
+	tmp = le32_to_cpu(pnon_clock_info->ulCapsAndSettings) &
+		ATOM_PPLIB_SOFTWARE_ENABLE_SLEEP_FOR_TIMESTAMPS;
+
+	ps->software.enableSleepForTimestamps = (0 != tmp);
+
+	ps->validation.supportedPowerLevels = pnon_clock_info->ucRequiredPower;
+
+	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < version) {
+		ps->uvd_clocks.VCLK = pnon_clock_info->ulVCLK;
+		ps->uvd_clocks.DCLK = pnon_clock_info->ulDCLK;
+	} else {
+		ps->uvd_clocks.VCLK = 0;
+		ps->uvd_clocks.DCLK = 0;
+	}
+
+	return 0;
+}
+
+static ULONG size_of_entry_v2(ULONG num_dpm_levels)
+{
+	return (sizeof(UCHAR) + sizeof(UCHAR) +
+			(num_dpm_levels * sizeof(UCHAR)));
+}
+
+static const ATOM_PPLIB_STATE_V2 *get_state_entry_v2(
+					const StateArray * pstate_arrays,
+							 ULONG entry_index)
+{
+	ULONG i;
+	const ATOM_PPLIB_STATE_V2 *pstate;
+
+	pstate = pstate_arrays->states;
+	if (entry_index <= pstate_arrays->ucNumEntries) {
+		for (i = 0; i < entry_index; i++)
+			pstate = (ATOM_PPLIB_STATE_V2 *)(
+						  (unsigned long)pstate +
+			     size_of_entry_v2(pstate->ucNumDPMLevels));
+	}
+	return pstate;
+}
+
+
+static const ATOM_PPLIB_POWERPLAYTABLE *get_powerplay_table(
+				     struct pp_hwmgr *hwmgr)
+{
+	const void *table_addr = NULL;
+	uint8_t frev, crev;
+	uint16_t size;
+
+	table_addr = cgs_atom_get_data_table(hwmgr->device,
+			GetIndexIntoMasterTable(DATA, PowerPlayInfo),
+			&size, &frev, &crev);
+
+	hwmgr->soft_pp_table = table_addr;
+
+	return (const ATOM_PPLIB_POWERPLAYTABLE *)table_addr;
+}
+
+
+int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr,
+				     unsigned long *num_of_entries)
+{
+	const StateArray *pstate_arrays;
+	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table = get_powerplay_table(hwmgr);
+
+	if (powerplay_table == NULL)
+		return -1;
+
+	if (powerplay_table->sHeader.ucTableFormatRevision >= 6) {
+		pstate_arrays = (StateArray *)(((unsigned long)powerplay_table) +
+					le16_to_cpu(powerplay_table->usStateArrayOffset));
+
+		*num_of_entries = (unsigned long)(pstate_arrays->ucNumEntries);
+	} else
+		*num_of_entries = (unsigned long)(powerplay_table->ucNumStates);
+
+	return 0;
+}
+
+int pp_tables_get_entry(struct pp_hwmgr *hwmgr,
+				unsigned long entry_index,
+				struct pp_power_state *ps,
+			 pp_tables_hw_clock_info_callback func)
+{
+	int i;
+	const StateArray *pstate_arrays;
+	const ATOM_PPLIB_STATE_V2 *pstate_entry_v2;
+	const ATOM_PPLIB_NONCLOCK_INFO *pnon_clock_info;
+	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table = get_powerplay_table(hwmgr);
+	int result = 0;
+	int res = 0;
+
+	const ClockInfoArray *pclock_arrays;
+
+	const NonClockInfoArray *pnon_clock_arrays;
+
+	const ATOM_PPLIB_STATE *pstate_entry;
+
+	if (powerplay_table == NULL)
+		return -1;
+
+	ps->classification.bios_index = entry_index;
+
+	if (powerplay_table->sHeader.ucTableFormatRevision >= 6) {
+		pstate_arrays = (StateArray *)(((unsigned long)powerplay_table) +
+					le16_to_cpu(powerplay_table->usStateArrayOffset));
+
+		if (entry_index > pstate_arrays->ucNumEntries)
+			return -1;
+
+		pstate_entry_v2 = get_state_entry_v2(pstate_arrays, entry_index);
+		pclock_arrays = (ClockInfoArray *)(((unsigned long)powerplay_table) +
+					le16_to_cpu(powerplay_table->usClockInfoArrayOffset));
+
+		pnon_clock_arrays = (NonClockInfoArray *)(((unsigned long)powerplay_table) +
+						le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset));
+
+		pnon_clock_info = (ATOM_PPLIB_NONCLOCK_INFO *)((unsigned long)(pnon_clock_arrays->nonClockInfo) +
+					(pstate_entry_v2->nonClockInfoIndex * pnon_clock_arrays->ucEntrySize));
+
+		result = init_non_clock_fields(hwmgr, ps, pnon_clock_arrays->ucEntrySize, pnon_clock_info);
+
+		for (i = 0; i < pstate_entry_v2->ucNumDPMLevels; i++) {
+			const void *pclock_info = (const void *)(
+							(unsigned long)(pclock_arrays->clockInfo) +
+							(pstate_entry_v2->clockInfoIndex[i] * pclock_arrays->ucEntrySize));
+			res = func(hwmgr, &ps->hardware, i, pclock_info);
+			if ((0 == result) && (0 != res))
+				result = res;
+		}
+	} else {
+		if (entry_index > powerplay_table->ucNumStates)
+			return -1;
+
+		pstate_entry = (ATOM_PPLIB_STATE *)((unsigned long)powerplay_table + powerplay_table->usStateArrayOffset +
+				entry_index * powerplay_table->ucStateEntrySize);
+
+		pnon_clock_info = (ATOM_PPLIB_NONCLOCK_INFO *)((unsigned long)powerplay_table +
+						le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset) +
+						pstate_entry->ucNonClockStateIndex *
+						powerplay_table->ucNonClockSize);
+
+		result = init_non_clock_fields(hwmgr, ps,
+							powerplay_table->ucNonClockSize,
+							pnon_clock_info);
+
+		for (i = 0; i < powerplay_table->ucStateEntrySize-1; i++) {
+			const void *pclock_info = (const void *)((unsigned long)powerplay_table +
+						le16_to_cpu(powerplay_table->usClockInfoArrayOffset) +
+						pstate_entry->ucClockStateIndices[i] *
+						powerplay_table->ucClockInfoSize);
+
+			int res = func(hwmgr, &ps->hardware, i, pclock_info);
+
+			if ((0 == result) && (0 != res))
+					result = res;
+		}
+	}
+
+	if ((0 == result) &&
+		(0 != (ps->classification.flags & PP_StateClassificationFlag_Boot)))
+		result = hwmgr->hwmgr_func->patch_boot_state(hwmgr, &(ps->hardware));
+
+	return result;
+}
+
+
+
+static int init_powerplay_tables(
+			struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table
+)
+{
+	return 0;
+}
+
+
+static int init_thermal_controller(
+			struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	return 0;
+}
+
+static int init_overdrive_limits_V1_4(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table,
+			const ATOM_FIRMWARE_INFO_V1_4 *fw_info)
+{
+	hwmgr->platform_descriptor.overdriveLimit.engineClock =
+				le32_to_cpu(fw_info->ulASICMaxEngineClock);
+
+	hwmgr->platform_descriptor.overdriveLimit.memoryClock =
+				le32_to_cpu(fw_info->ulASICMaxMemoryClock);
+
+	hwmgr->platform_descriptor.maxOverdriveVDDC =
+		le32_to_cpu(fw_info->ul3DAccelerationEngineClock) & 0x7FF;
+
+	hwmgr->platform_descriptor.minOverdriveVDDC =
+			   le16_to_cpu(fw_info->usBootUpVDDCVoltage);
+
+	hwmgr->platform_descriptor.maxOverdriveVDDC =
+			   le16_to_cpu(fw_info->usBootUpVDDCVoltage);
+
+	hwmgr->platform_descriptor.overdriveVDDCStep = 0;
+	return 0;
+}
+
+static int init_overdrive_limits_V2_1(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table,
+			const ATOM_FIRMWARE_INFO_V2_1 *fw_info)
+{
+	const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3;
+	const ATOM_PPLIB_EXTENDEDHEADER *header;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) <
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3))
+		return 0;
+
+	powerplay_table3 = (const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+
+	if (0 == powerplay_table3->usExtendendedHeaderOffset)
+		return 0;
+
+	header = (ATOM_PPLIB_EXTENDEDHEADER *)(((unsigned long) powerplay_table) +
+			le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
+
+	hwmgr->platform_descriptor.overdriveLimit.engineClock = le32_to_cpu(header->ulMaxEngineClock);
+	hwmgr->platform_descriptor.overdriveLimit.memoryClock = le32_to_cpu(header->ulMaxMemoryClock);
+
+
+	hwmgr->platform_descriptor.minOverdriveVDDC = 0;
+	hwmgr->platform_descriptor.maxOverdriveVDDC = 0;
+	hwmgr->platform_descriptor.overdriveVDDCStep = 0;
+
+	return 0;
+}
+
+static int init_overdrive_limits(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	int result;
+	uint8_t frev, crev;
+	uint16_t size;
+
+	const ATOM_COMMON_TABLE_HEADER *fw_info = NULL;
+
+	hwmgr->platform_descriptor.overdriveLimit.engineClock = 0;
+	hwmgr->platform_descriptor.overdriveLimit.memoryClock = 0;
+	hwmgr->platform_descriptor.minOverdriveVDDC = 0;
+	hwmgr->platform_descriptor.maxOverdriveVDDC = 0;
+
+	/* We assume here that fw_info is unchanged if this call fails.*/
+	fw_info = cgs_atom_get_data_table(hwmgr->device,
+			 GetIndexIntoMasterTable(DATA, FirmwareInfo),
+			 &size, &frev, &crev);
+
+	if ((fw_info->ucTableFormatRevision == 1)
+		&& (fw_info->usStructureSize >= sizeof(ATOM_FIRMWARE_INFO_V1_4)))
+		result = init_overdrive_limits_V1_4(hwmgr,
+				powerplay_table,
+				(const ATOM_FIRMWARE_INFO_V1_4 *)fw_info);
+
+	else if ((fw_info->ucTableFormatRevision == 2)
+		&& (fw_info->usStructureSize >= sizeof(ATOM_FIRMWARE_INFO_V2_1)))
+		result = init_overdrive_limits_V2_1(hwmgr,
+				powerplay_table,
+				(const ATOM_FIRMWARE_INFO_V2_1 *)fw_info);
+
+	if (hwmgr->platform_descriptor.overdriveLimit.engineClock > 0
+		&& hwmgr->platform_descriptor.overdriveLimit.memoryClock > 0
+		&& !phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_OverdriveDisabledByPowerBudget))
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ACOverdriveSupport);
+
+	return result;
+}
+
+static int get_uvd_clock_voltage_limit_table(struct pp_hwmgr *hwmgr,
+		struct phm_uvd_clock_voltage_dependency_table **ptable,
+		const ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *table,
+		const UVDClockInfoArray *array)
+{
+	unsigned long table_size, i;
+	struct phm_uvd_clock_voltage_dependency_table *uvd_table;
+
+	table_size = sizeof(unsigned long) +
+		 sizeof(struct phm_uvd_clock_voltage_dependency_table) *
+		 table->numEntries;
+
+	uvd_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == uvd_table)
+		return -ENOMEM;
+
+	uvd_table->count = table->numEntries;
+
+	for (i = 0; i < table->numEntries; i++) {
+		const UVDClockInfo *entry =
+			&array->entries[table->entries[i].ucUVDClockInfoIndex];
+		uvd_table->entries[i].v = (unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+		uvd_table->entries[i].vclk = ((unsigned long)entry->ucVClkHigh << 16)
+					 | le16_to_cpu(entry->usVClkLow);
+		uvd_table->entries[i].dclk = ((unsigned long)entry->ucDClkHigh << 16)
+					 | le16_to_cpu(entry->usDClkLow);
+	}
+
+	*ptable = uvd_table;
+
+	return 0;
+}
+
+static int get_vce_clock_voltage_limit_table(struct pp_hwmgr *hwmgr,
+		struct phm_vce_clock_voltage_dependency_table **ptable,
+		const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *table,
+		const VCEClockInfoArray    *array)
+{
+	unsigned long table_size, i;
+	struct phm_vce_clock_voltage_dependency_table *vce_table = NULL;
+
+	table_size = sizeof(unsigned long) +
+			sizeof(struct phm_vce_clock_voltage_dependency_table)
+			* table->numEntries;
+
+	vce_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == vce_table)
+		return -ENOMEM;
+
+	vce_table->count = table->numEntries;
+	for (i = 0; i < table->numEntries; i++) {
+		const VCEClockInfo *entry = &array->entries[table->entries[i].ucVCEClockInfoIndex];
+
+		vce_table->entries[i].v = (unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+		vce_table->entries[i].evclk = ((unsigned long)entry->ucEVClkHigh << 16)
+					| le16_to_cpu(entry->usEVClkLow);
+		vce_table->entries[i].ecclk = ((unsigned long)entry->ucECClkHigh << 16)
+					| le16_to_cpu(entry->usECClkLow);
+	}
+
+	*ptable = vce_table;
+
+	return 0;
+}
+
+static int get_samu_clock_voltage_limit_table(struct pp_hwmgr *hwmgr,
+		 struct phm_samu_clock_voltage_dependency_table **ptable,
+		 const ATOM_PPLIB_SAMClk_Voltage_Limit_Table *table)
+{
+	unsigned long table_size, i;
+	struct phm_samu_clock_voltage_dependency_table *samu_table;
+
+	table_size = sizeof(unsigned long) +
+		sizeof(struct phm_samu_clock_voltage_dependency_table) *
+		table->numEntries;
+
+	samu_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == samu_table)
+		return -ENOMEM;
+
+	samu_table->count = table->numEntries;
+
+	for (i = 0; i < table->numEntries; i++) {
+		samu_table->entries[i].v = (unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+		samu_table->entries[i].samclk = ((unsigned long)table->entries[i].ucSAMClockHigh << 16)
+					 | le16_to_cpu(table->entries[i].usSAMClockLow);
+	}
+
+	*ptable = samu_table;
+
+	return 0;
+}
+
+static int get_acp_clock_voltage_limit_table(struct pp_hwmgr *hwmgr,
+		struct phm_acp_clock_voltage_dependency_table **ptable,
+		const ATOM_PPLIB_ACPClk_Voltage_Limit_Table *table)
+{
+	unsigned table_size, i;
+	struct phm_acp_clock_voltage_dependency_table *acp_table;
+
+	table_size = sizeof(unsigned long) +
+		sizeof(struct phm_acp_clock_voltage_dependency_table) *
+		table->numEntries;
+
+	acp_table = kzalloc(table_size, GFP_KERNEL);
+	if (NULL == acp_table)
+		return -ENOMEM;
+
+	acp_table->count = (unsigned long)table->numEntries;
+
+	for (i = 0; i < table->numEntries; i++) {
+		acp_table->entries[i].v = (unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+		acp_table->entries[i].acpclk = ((unsigned long)table->entries[i].ucACPClockHigh << 16)
+					 | le16_to_cpu(table->entries[i].usACPClockLow);
+	}
+
+	*ptable = acp_table;
+
+	return 0;
+}
+
+static int init_clock_voltage_dependency(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	ATOM_PPLIB_Clock_Voltage_Dependency_Table *table;
+	ATOM_PPLIB_Clock_Voltage_Limit_Table *limit_table;
+	int result = 0;
+
+	uint16_t vce_clock_info_array_offset;
+	uint16_t uvd_clock_info_array_offset;
+	uint16_t table_offset;
+
+	hwmgr->dyn_state.vddc_dependency_on_sclk = NULL;
+	hwmgr->dyn_state.vddci_dependency_on_mclk = NULL;
+	hwmgr->dyn_state.vddc_dependency_on_mclk = NULL;
+	hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL;
+	hwmgr->dyn_state.mvdd_dependency_on_mclk = NULL;
+	hwmgr->dyn_state.vce_clock_voltage_dependency_table = NULL;
+	hwmgr->dyn_state.uvd_clock_voltage_dependency_table = NULL;
+	hwmgr->dyn_state.samu_clock_voltage_dependency_table = NULL;
+	hwmgr->dyn_state.acp_clock_voltage_dependency_table = NULL;
+	hwmgr->dyn_state.ppm_parameter_table = NULL;
+	hwmgr->dyn_state.vdd_gfx_dependency_on_sclk = NULL;
+
+	vce_clock_info_array_offset = get_vce_clock_info_array_offset(
+						hwmgr, powerplay_table);
+	table_offset = get_vce_clock_voltage_limit_table_offset(hwmgr,
+						powerplay_table);
+	if (vce_clock_info_array_offset > 0 && table_offset > 0) {
+		const VCEClockInfoArray *array = (const VCEClockInfoArray *)
+				(((unsigned long) powerplay_table) +
+				vce_clock_info_array_offset);
+		const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *table =
+				(const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *)
+				(((unsigned long) powerplay_table) + table_offset);
+		result = get_vce_clock_voltage_limit_table(hwmgr,
+				&hwmgr->dyn_state.vce_clock_voltage_dependency_table,
+				table, array);
+	}
+
+	uvd_clock_info_array_offset = get_uvd_clock_info_array_offset(hwmgr, powerplay_table);
+	table_offset = get_uvd_clock_voltage_limit_table_offset(hwmgr, powerplay_table);
+
+	if (uvd_clock_info_array_offset > 0 && table_offset > 0) {
+		const UVDClockInfoArray *array = (const UVDClockInfoArray *)
+				(((unsigned long) powerplay_table) +
+				uvd_clock_info_array_offset);
+		const ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *ptable =
+				(const ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *)
+				(((unsigned long) powerplay_table) + table_offset);
+		result = get_uvd_clock_voltage_limit_table(hwmgr,
+				&hwmgr->dyn_state.uvd_clock_voltage_dependency_table, ptable, array);
+	}
+
+	table_offset = get_samu_clock_voltage_limit_table_offset(hwmgr,
+							    powerplay_table);
+
+	if (table_offset > 0) {
+		const ATOM_PPLIB_SAMClk_Voltage_Limit_Table *ptable =
+				(const ATOM_PPLIB_SAMClk_Voltage_Limit_Table *)
+				(((unsigned long) powerplay_table) + table_offset);
+		result = get_samu_clock_voltage_limit_table(hwmgr,
+				&hwmgr->dyn_state.samu_clock_voltage_dependency_table, ptable);
+	}
+
+	table_offset = get_acp_clock_voltage_limit_table_offset(hwmgr,
+							     powerplay_table);
+
+	if (table_offset > 0) {
+		const ATOM_PPLIB_ACPClk_Voltage_Limit_Table *ptable =
+				(const ATOM_PPLIB_ACPClk_Voltage_Limit_Table *)
+				(((unsigned long) powerplay_table) + table_offset);
+		result = get_acp_clock_voltage_limit_table(hwmgr,
+				&hwmgr->dyn_state.acp_clock_voltage_dependency_table, ptable);
+	}
+
+	table_offset = get_cacp_tdp_table_offset(hwmgr, powerplay_table);
+	if (table_offset > 0) {
+		UCHAR rev_id = *(UCHAR *)(((unsigned long)powerplay_table) + table_offset);
+
+		if (rev_id > 0) {
+			const ATOM_PPLIB_POWERTUNE_Table_V1 *tune_table =
+				(const ATOM_PPLIB_POWERTUNE_Table_V1 *)
+				(((unsigned long) powerplay_table) + table_offset);
+			result = get_cac_tdp_table(hwmgr, &hwmgr->dyn_state.cac_dtp_table,
+				&tune_table->power_tune_table,
+				le16_to_cpu(tune_table->usMaximumPowerDeliveryLimit));
+			hwmgr->dyn_state.cac_dtp_table->usDefaultTargetOperatingTemp =
+				le16_to_cpu(tune_table->usTjMax);
+		} else {
+			const ATOM_PPLIB_POWERTUNE_Table *tune_table =
+				(const ATOM_PPLIB_POWERTUNE_Table *)
+				(((unsigned long) powerplay_table) + table_offset);
+			result = get_cac_tdp_table(hwmgr,
+				&hwmgr->dyn_state.cac_dtp_table,
+				&tune_table->power_tune_table, 255);
+		}
+	}
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+		sizeof(ATOM_PPLIB_POWERPLAYTABLE4)) {
+		const ATOM_PPLIB_POWERPLAYTABLE4 *powerplay_table4 =
+				(const ATOM_PPLIB_POWERPLAYTABLE4 *)powerplay_table;
+		if (0 != powerplay_table4->usVddcDependencyOnSCLKOffset) {
+			table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+				(((unsigned long) powerplay_table4) +
+				powerplay_table4->usVddcDependencyOnSCLKOffset);
+			result = get_clock_voltage_dependency_table(hwmgr,
+				&hwmgr->dyn_state.vddc_dependency_on_sclk, table);
+		}
+
+		if (result == 0 && (0 != powerplay_table4->usVddciDependencyOnMCLKOffset)) {
+			table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+				(((unsigned long) powerplay_table4) +
+				powerplay_table4->usVddciDependencyOnMCLKOffset);
+			result = get_clock_voltage_dependency_table(hwmgr,
+				&hwmgr->dyn_state.vddci_dependency_on_mclk, table);
+		}
+
+		if (result == 0 && (0 != powerplay_table4->usVddcDependencyOnMCLKOffset)) {
+			table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+				(((unsigned long) powerplay_table4) +
+				powerplay_table4->usVddcDependencyOnMCLKOffset);
+			result = get_clock_voltage_dependency_table(hwmgr,
+				&hwmgr->dyn_state.vddc_dependency_on_mclk, table);
+		}
+
+		if (result == 0 && (0 != powerplay_table4->usMaxClockVoltageOnDCOffset)) {
+			limit_table = (ATOM_PPLIB_Clock_Voltage_Limit_Table *)
+				(((unsigned long) powerplay_table4) +
+				powerplay_table4->usMaxClockVoltageOnDCOffset);
+			result = get_clock_voltage_limit(hwmgr,
+				&hwmgr->dyn_state.max_clock_voltage_on_dc, limit_table);
+		}
+
+		if (result == 0 && (NULL != hwmgr->dyn_state.vddc_dependency_on_mclk) &&
+			(0 != hwmgr->dyn_state.vddc_dependency_on_mclk->count))
+			result = get_valid_clk(hwmgr, &hwmgr->dyn_state.valid_mclk_values,
+					hwmgr->dyn_state.vddc_dependency_on_mclk);
+
+		if(result == 0 && (NULL != hwmgr->dyn_state.vddc_dependency_on_sclk) &&
+			(0 != hwmgr->dyn_state.vddc_dependency_on_sclk->count))
+			result = get_valid_clk(hwmgr,
+				&hwmgr->dyn_state.valid_sclk_values,
+				hwmgr->dyn_state.vddc_dependency_on_sclk);
+
+		if (result == 0 && (0 != powerplay_table4->usMvddDependencyOnMCLKOffset)) {
+			table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+				(((unsigned long) powerplay_table4) +
+				powerplay_table4->usMvddDependencyOnMCLKOffset);
+			result = get_clock_voltage_dependency_table(hwmgr,
+				&hwmgr->dyn_state.mvdd_dependency_on_mclk, table);
+		}
+	}
+
+	table_offset = get_sclk_vdd_gfx_clock_voltage_dependency_table_offset(hwmgr,
+								powerplay_table);
+
+	if (table_offset > 0) {
+		table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+			(((unsigned long) powerplay_table) + table_offset);
+		result = get_clock_voltage_dependency_table(hwmgr,
+			&hwmgr->dyn_state.vdd_gfx_dependency_on_sclk, table);
+	}
+
+	return result;
+}
+
+static int get_cac_leakage_table(struct pp_hwmgr *hwmgr,
+				 struct phm_cac_leakage_table **ptable,
+				const ATOM_PPLIB_CAC_Leakage_Table *table)
+{
+	struct phm_cac_leakage_table  *cac_leakage_table;
+	unsigned long            table_size, i;
+
+	if (hwmgr == NULL || table == NULL || ptable == NULL)
+		return -EINVAL;
+
+	table_size = sizeof(ULONG) +
+		(sizeof(struct phm_cac_leakage_table) * table->ucNumEntries);
+
+	cac_leakage_table = kzalloc(table_size, GFP_KERNEL);
+
+	if (cac_leakage_table == NULL)
+		return -ENOMEM;
+
+	cac_leakage_table->count = (ULONG)table->ucNumEntries;
+
+	for (i = 0; i < cac_leakage_table->count; i++) {
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_EVV)) {
+			cac_leakage_table->entries[i].Vddc1 = le16_to_cpu(table->entries[i].usVddc1);
+			cac_leakage_table->entries[i].Vddc2 = le16_to_cpu(table->entries[i].usVddc2);
+			cac_leakage_table->entries[i].Vddc3 = le16_to_cpu(table->entries[i].usVddc3);
+		} else {
+			cac_leakage_table->entries[i].Vddc    = le16_to_cpu(table->entries[i].usVddc);
+			cac_leakage_table->entries[i].Leakage = le32_to_cpu(table->entries[i].ulLeakageValue);
+		}
+	}
+
+	*ptable = cac_leakage_table;
+
+	return 0;
+}
+
+static int get_platform_power_management_table(struct pp_hwmgr *hwmgr,
+			ATOM_PPLIB_PPM_Table *atom_ppm_table)
+{
+	struct phm_ppm_table *ptr = kzalloc(sizeof(struct phm_ppm_table), GFP_KERNEL);
+
+	if (NULL == ptr)
+		return -ENOMEM;
+
+	ptr->ppm_design            = atom_ppm_table->ucPpmDesign;
+	ptr->cpu_core_number        = le16_to_cpu(atom_ppm_table->usCpuCoreNumber);
+	ptr->platform_tdp          = le32_to_cpu(atom_ppm_table->ulPlatformTDP);
+	ptr->small_ac_platform_tdp   = le32_to_cpu(atom_ppm_table->ulSmallACPlatformTDP);
+	ptr->platform_tdc          = le32_to_cpu(atom_ppm_table->ulPlatformTDC);
+	ptr->small_ac_platform_tdc   = le32_to_cpu(atom_ppm_table->ulSmallACPlatformTDC);
+	ptr->apu_tdp               = le32_to_cpu(atom_ppm_table->ulApuTDP);
+	ptr->dgpu_tdp              = le32_to_cpu(atom_ppm_table->ulDGpuTDP);
+	ptr->dgpu_ulv_power         = le32_to_cpu(atom_ppm_table->ulDGpuUlvPower);
+	ptr->tj_max                = le32_to_cpu(atom_ppm_table->ulTjmax);
+	hwmgr->dyn_state.ppm_parameter_table = ptr;
+
+	return 0;
+}
+
+static int init_dpm2_parameters(struct pp_hwmgr *hwmgr,
+			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	int result = 0;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE5)) {
+		const  ATOM_PPLIB_POWERPLAYTABLE5 *ptable5 =
+				(const ATOM_PPLIB_POWERPLAYTABLE5 *)powerplay_table;
+		const  ATOM_PPLIB_POWERPLAYTABLE4 *ptable4 =
+				(const ATOM_PPLIB_POWERPLAYTABLE4 *)
+				(&ptable5->basicTable4);
+		const  ATOM_PPLIB_POWERPLAYTABLE3 *ptable3 =
+				(const ATOM_PPLIB_POWERPLAYTABLE3 *)
+				(&ptable4->basicTable3);
+		const  ATOM_PPLIB_EXTENDEDHEADER  *extended_header;
+		uint16_t table_offset;
+		ATOM_PPLIB_PPM_Table *atom_ppm_table;
+
+		hwmgr->platform_descriptor.TDPLimit     = le32_to_cpu(ptable5->ulTDPLimit);
+		hwmgr->platform_descriptor.nearTDPLimit = le32_to_cpu(ptable5->ulNearTDPLimit);
+
+		hwmgr->platform_descriptor.TDPODLimit   = le16_to_cpu(ptable5->usTDPODLimit);
+		hwmgr->platform_descriptor.TDPAdjustment = 0;
+
+		hwmgr->platform_descriptor.VidAdjustment = 0;
+		hwmgr->platform_descriptor.VidAdjustmentPolarity = 0;
+		hwmgr->platform_descriptor.VidMinLimit     = 0;
+		hwmgr->platform_descriptor.VidMaxLimit     = 1500000;
+		hwmgr->platform_descriptor.VidStep         = 6250;
+
+		hwmgr->platform_descriptor.nearTDPLimitAdjusted = le32_to_cpu(ptable5->ulNearTDPLimit);
+
+		if (hwmgr->platform_descriptor.TDPODLimit != 0)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_PowerControl);
+
+		hwmgr->platform_descriptor.SQRampingThreshold = le32_to_cpu(ptable5->ulSQRampingThreshold);
+
+		hwmgr->platform_descriptor.CACLeakage = le32_to_cpu(ptable5->ulCACLeakage);
+
+		hwmgr->dyn_state.cac_leakage_table = NULL;
+
+		if (0 != ptable5->usCACLeakageTableOffset) {
+			const ATOM_PPLIB_CAC_Leakage_Table *pCAC_leakage_table =
+				(ATOM_PPLIB_CAC_Leakage_Table *)(((unsigned long)ptable5) +
+				le16_to_cpu(ptable5->usCACLeakageTableOffset));
+			result = get_cac_leakage_table(hwmgr,
+				&hwmgr->dyn_state.cac_leakage_table, pCAC_leakage_table);
+		}
+
+		hwmgr->platform_descriptor.LoadLineSlope = le16_to_cpu(ptable5->usLoadLineSlope);
+
+		hwmgr->dyn_state.ppm_parameter_table = NULL;
+
+		if (0 != ptable3->usExtendendedHeaderOffset) {
+			extended_header = (const ATOM_PPLIB_EXTENDEDHEADER *)
+					(((unsigned long)powerplay_table) +
+					le16_to_cpu(ptable3->usExtendendedHeaderOffset));
+			if ((extended_header->usPPMTableOffset > 0) &&
+				le16_to_cpu(extended_header->usSize) >=
+				    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5) {
+				table_offset = le16_to_cpu(extended_header->usPPMTableOffset);
+				atom_ppm_table = (ATOM_PPLIB_PPM_Table *)
+					(((unsigned long)powerplay_table) + table_offset);
+				if (0 == get_platform_power_management_table(hwmgr, atom_ppm_table))
+					phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_EnablePlatformPowerManagement);
+			}
+		}
+	}
+	return result;
+}
+
+static int init_phase_shedding_table(struct pp_hwmgr *hwmgr,
+		const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
+{
+	if (le16_to_cpu(powerplay_table->usTableSize) >=
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE4)) {
+		const ATOM_PPLIB_POWERPLAYTABLE4 *powerplay_table4 =
+				(const ATOM_PPLIB_POWERPLAYTABLE4 *)powerplay_table;
+
+		if (0 != powerplay_table4->usVddcPhaseShedLimitsTableOffset) {
+			const ATOM_PPLIB_PhaseSheddingLimits_Table *ptable =
+				(ATOM_PPLIB_PhaseSheddingLimits_Table *)
+				(((unsigned long)powerplay_table4) +
+				le16_to_cpu(powerplay_table4->usVddcPhaseShedLimitsTableOffset));
+			struct phm_phase_shedding_limits_table *table;
+			unsigned long size, i;
+
+
+			size = sizeof(unsigned long) +
+				(sizeof(struct phm_phase_shedding_limits_table) *
+				ptable->ucNumEntries);
+
+			table = kzalloc(size, GFP_KERNEL);
+
+			if (table == NULL)
+				return -ENOMEM;
+
+			table->count = (unsigned long)ptable->ucNumEntries;
+
+			for (i = 0; i < table->count; i++) {
+				table->entries[i].Voltage = (unsigned long)le16_to_cpu(ptable->entries[i].usVoltage);
+				table->entries[i].Sclk    = ((unsigned long)ptable->entries[i].ucSclkHigh << 16)
+							| le16_to_cpu(ptable->entries[i].usSclkLow);
+				table->entries[i].Mclk    = ((unsigned long)ptable->entries[i].ucMclkHigh << 16)
+							| le16_to_cpu(ptable->entries[i].usMclkLow);
+			}
+			hwmgr->dyn_state.vddc_phase_shed_limits_table = table;
+		}
+	}
+
+	return 0;
+}
+
+int get_number_of_vce_state_table_entries(
+						  struct pp_hwmgr *hwmgr)
+{
+	const ATOM_PPLIB_POWERPLAYTABLE *table =
+					     get_powerplay_table(hwmgr);
+	const ATOM_PPLIB_VCE_State_Table *vce_table =
+				    get_vce_state_table(hwmgr, table);
+
+	if (vce_table > 0)
+		return vce_table->numEntries;
+
+	return 0;
+}
+
+int get_vce_state_table_entry(struct pp_hwmgr *hwmgr,
+							unsigned long i,
+							struct PP_VCEState *vce_state,
+							void **clock_info,
+							unsigned long *flag)
+{
+	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table = get_powerplay_table(hwmgr);
+
+	const ATOM_PPLIB_VCE_State_Table *vce_state_table = get_vce_state_table(hwmgr, powerplay_table);
+
+	unsigned short vce_clock_info_array_offset = get_vce_clock_info_array_offset(hwmgr, powerplay_table);
+
+	const VCEClockInfoArray *vce_clock_info_array = (const VCEClockInfoArray *)(((unsigned long) powerplay_table) + vce_clock_info_array_offset);
+
+	const ClockInfoArray *clock_arrays = (ClockInfoArray *)(((unsigned long)powerplay_table) + powerplay_table->usClockInfoArrayOffset);
+
+	const ATOM_PPLIB_VCE_State_Record *record = &vce_state_table->entries[i];
+
+	const VCEClockInfo *vce_clock_info = &vce_clock_info_array->entries[record->ucVCEClockInfoIndex];
+
+	unsigned long clockInfoIndex = record->ucClockInfoIndex & 0x3F;
+
+	*flag = (record->ucClockInfoIndex >> NUM_BITS_CLOCK_INFO_ARRAY_INDEX);
+
+	vce_state->evclk = ((uint32_t)vce_clock_info->ucEVClkHigh << 16) | vce_clock_info->usEVClkLow;
+	vce_state->ecclk = ((uint32_t)vce_clock_info->ucECClkHigh << 16) | vce_clock_info->usECClkLow;
+
+	*clock_info = (void *)((unsigned long)(clock_arrays->clockInfo) + (clockInfoIndex * clock_arrays->ucEntrySize));
+
+	return 0;
+}
+
+
+static int pp_tables_initialize(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table;
+
+	hwmgr->need_pp_table_upload = true;
+
+	powerplay_table = get_powerplay_table(hwmgr);
+
+	result = init_powerplay_tables(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_powerplay_tables failed", return result);
+
+	result = set_platform_caps(hwmgr,
+				le32_to_cpu(powerplay_table->ulPlatformCaps));
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "set_platform_caps failed", return result);
+
+	result = init_thermal_controller(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_thermal_controller failed", return result);
+
+	result = init_overdrive_limits(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_overdrive_limits failed", return result);
+
+	result = init_clock_voltage_dependency(hwmgr,
+					       powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_clock_voltage_dependency failed", return result);
+
+	result = init_dpm2_parameters(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_dpm2_parameters failed", return result);
+
+	result = init_phase_shedding_table(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_phase_shedding_table failed", return result);
+
+	return result;
+}
+
+static int pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->soft_pp_table) {
+		kfree(hwmgr->soft_pp_table);
+		hwmgr->soft_pp_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vddc_dependency_on_sclk) {
+		kfree(hwmgr->dyn_state.vddc_dependency_on_sclk);
+		hwmgr->dyn_state.vddc_dependency_on_sclk = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vddci_dependency_on_mclk) {
+		kfree(hwmgr->dyn_state.vddci_dependency_on_mclk);
+		hwmgr->dyn_state.vddci_dependency_on_mclk = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vddc_dependency_on_mclk) {
+		kfree(hwmgr->dyn_state.vddc_dependency_on_mclk);
+		hwmgr->dyn_state.vddc_dependency_on_mclk = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.mvdd_dependency_on_mclk) {
+		kfree(hwmgr->dyn_state.mvdd_dependency_on_mclk);
+		hwmgr->dyn_state.mvdd_dependency_on_mclk = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.valid_mclk_values) {
+		kfree(hwmgr->dyn_state.valid_mclk_values);
+		hwmgr->dyn_state.valid_mclk_values = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.valid_sclk_values) {
+		kfree(hwmgr->dyn_state.valid_sclk_values);
+		hwmgr->dyn_state.valid_sclk_values = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.cac_leakage_table) {
+		kfree(hwmgr->dyn_state.cac_leakage_table);
+		hwmgr->dyn_state.cac_leakage_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vddc_phase_shed_limits_table) {
+		kfree(hwmgr->dyn_state.vddc_phase_shed_limits_table);
+		hwmgr->dyn_state.vddc_phase_shed_limits_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vce_clock_voltage_dependency_table) {
+		kfree(hwmgr->dyn_state.vce_clock_voltage_dependency_table);
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.uvd_clock_voltage_dependency_table) {
+		kfree(hwmgr->dyn_state.uvd_clock_voltage_dependency_table);
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.samu_clock_voltage_dependency_table) {
+		kfree(hwmgr->dyn_state.samu_clock_voltage_dependency_table);
+		hwmgr->dyn_state.samu_clock_voltage_dependency_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.acp_clock_voltage_dependency_table) {
+		kfree(hwmgr->dyn_state.acp_clock_voltage_dependency_table);
+		hwmgr->dyn_state.acp_clock_voltage_dependency_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.cac_dtp_table) {
+		kfree(hwmgr->dyn_state.cac_dtp_table);
+		hwmgr->dyn_state.cac_dtp_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.ppm_parameter_table) {
+		kfree(hwmgr->dyn_state.ppm_parameter_table);
+		hwmgr->dyn_state.ppm_parameter_table = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vdd_gfx_dependency_on_sclk) {
+		kfree(hwmgr->dyn_state.vdd_gfx_dependency_on_sclk);
+		hwmgr->dyn_state.vdd_gfx_dependency_on_sclk = NULL;
+	}
+
+	if (NULL != hwmgr->dyn_state.vq_budgeting_table) {
+		kfree(hwmgr->dyn_state.vq_budgeting_table);
+		hwmgr->dyn_state.vq_budgeting_table = NULL;
+	}
+
+	return 0;
+}
+
+const struct pp_table_func pptable_funcs = {
+	.pptable_init = pp_tables_initialize,
+	.pptable_fini = pp_tables_uninitialize,
+	.pptable_get_number_of_vce_state_table_entries =
+				get_number_of_vce_state_table_entries,
+	.pptable_get_vce_state_table_entry =
+						get_vce_state_table_entry,
+};
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h
new file mode 100644
index 000000000000..30434802417e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ * Interface Functions related to the BIOS PowerPlay Tables.
+ *
+ */
+
+#ifndef PROCESSPPTABLES_H
+#define PROCESSPPTABLES_H
+
+struct pp_hwmgr;
+struct pp_power_state;
+struct pp_hw_power_state;
+
+extern const struct pp_table_func pptable_funcs;
+
+typedef int (*pp_tables_hw_clock_info_callback)(struct pp_hwmgr *hwmgr,
+					struct pp_hw_power_state *hw_ps,
+							unsigned int index,
+						 const void *clock_info);
+
+int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr,
+				     unsigned long *num_of_entries);
+
+int pp_tables_get_entry(struct pp_hwmgr *hwmgr,
+						unsigned long entry_index,
+						struct pp_power_state *ps,
+				pp_tables_hw_clock_info_callback func);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c
new file mode 100644
index 000000000000..e58d038a997b
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c
@@ -0,0 +1,350 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "hwmgr.h"
+#include "tonga_clockpowergating.h"
+#include "tonga_ppsmc.h"
+#include "tonga_hwmgr.h"
+
+int tonga_phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cf_want_uvd_power_gating(hwmgr))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						     PPSMC_MSG_UVDPowerOFF);
+	return 0;
+}
+
+int tonga_phm_powerup_uvd(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cf_want_uvd_power_gating(hwmgr)) {
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				  PHM_PlatformCaps_UVDDynamicPowerGating)) {
+			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+								PPSMC_MSG_UVDPowerON, 1);
+		} else {
+			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+								PPSMC_MSG_UVDPowerON, 0);
+		}
+	}
+
+	return 0;
+}
+
+int tonga_phm_powerdown_vce(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cf_want_vce_power_gating(hwmgr))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						  PPSMC_MSG_VCEPowerOFF);
+	return 0;
+}
+
+int tonga_phm_powerup_vce(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cf_want_vce_power_gating(hwmgr))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+						  PPSMC_MSG_VCEPowerON);
+	return 0;
+}
+
+int tonga_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating)
+{
+	int ret = 0;
+
+	switch (block) {
+	case PHM_AsicBlock_UVD_MVC:
+	case PHM_AsicBlock_UVD:
+	case PHM_AsicBlock_UVD_HD:
+	case PHM_AsicBlock_UVD_SD:
+		if (gating == PHM_ClockGateSetting_StaticOff)
+			ret = tonga_phm_powerdown_uvd(hwmgr);
+		else
+			ret = tonga_phm_powerup_uvd(hwmgr);
+		break;
+	case PHM_AsicBlock_GFX:
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+int tonga_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = false;
+	data->vce_power_gated = false;
+
+	tonga_phm_powerup_uvd(hwmgr);
+	tonga_phm_powerup_vce(hwmgr);
+
+	return 0;
+}
+
+int tonga_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	if (data->uvd_power_gated == bgate)
+		return 0;
+
+	data->uvd_power_gated = bgate;
+
+	if (bgate) {
+		cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_CG_STATE_UNGATE);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_GATE);
+		tonga_update_uvd_dpm(hwmgr, true);
+		tonga_phm_powerdown_uvd(hwmgr);
+	} else {
+		tonga_phm_powerup_uvd(hwmgr);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_UNGATE);
+		cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_GATE);
+
+		tonga_update_uvd_dpm(hwmgr, false);
+	}
+
+	return 0;
+}
+
+int tonga_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct phm_set_power_state_input states;
+	const struct pp_power_state  *pcurrent;
+	struct pp_power_state  *requested;
+
+	pcurrent = hwmgr->current_ps;
+	requested = hwmgr->request_ps;
+
+	states.pcurrent_state = &(pcurrent->hardware);
+	states.pnew_state = &(requested->hardware);
+
+	if (phm_cf_want_vce_power_gating(hwmgr)) {
+		if (data->vce_power_gated != bgate) {
+			if (bgate) {
+				cgs_set_clockgating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_CG_STATE_UNGATE);
+				cgs_set_powergating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_PG_STATE_GATE);
+				tonga_enable_disable_vce_dpm(hwmgr, false);
+				data->vce_power_gated = true;
+			} else {
+				tonga_phm_powerup_vce(hwmgr);
+				data->vce_power_gated = false;
+				cgs_set_powergating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_PG_STATE_UNGATE);
+				cgs_set_clockgating_state(
+							hwmgr->device,
+							AMD_IP_BLOCK_TYPE_VCE,
+							AMD_PG_STATE_GATE);
+
+				tonga_update_vce_dpm(hwmgr, &states);
+				tonga_enable_disable_vce_dpm(hwmgr, true);
+				return 0;
+			}
+		}
+	} else {
+		tonga_update_vce_dpm(hwmgr, &states);
+		tonga_enable_disable_vce_dpm(hwmgr, true);
+		return 0;
+	}
+
+	if (!data->vce_power_gated)
+		tonga_update_vce_dpm(hwmgr, &states);
+
+	return 0;
+}
+
+int tonga_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
+					const uint32_t *msg_id)
+{
+	PPSMC_Msg msg;
+	uint32_t value;
+
+	switch ((*msg_id & PP_GROUP_MASK) >> PP_GROUP_SHIFT) {
+	case PP_GROUP_GFX:
+		switch ((*msg_id & PP_BLOCK_MASK) >> PP_BLOCK_SHIFT) {
+		case PP_BLOCK_GFX_CG:
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+				  ? PPSMC_MSG_EnableClockGatingFeature
+				  : PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_GFX_CGCG_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			if (PP_STATE_SUPPORT_LS & *msg_id) {
+				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
+					? PPSMC_MSG_EnableClockGatingFeature
+					: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_GFX_CGLS_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		case PP_BLOCK_GFX_MG:
+			/* For GFX MGCG, there are three different ones;
+			 * CPF, RLC, and all others.  CPF MGCG will not be used for Tonga.
+			 * For GFX MGLS, Tonga will not support it.
+			 * */
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = (CG_RLC_MGCG_MASK | CG_GFX_OTHERS_MGCG_MASK);
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		default:
+			return -1;
+		}
+		break;
+
+	case PP_GROUP_SYS:
+		switch ((*msg_id & PP_BLOCK_MASK) >> PP_BLOCK_SHIFT) {
+		case PP_BLOCK_SYS_BIF:
+			if (PP_STATE_SUPPORT_LS & *msg_id) {
+				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_BIF_MGLS_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		case PP_BLOCK_SYS_MC:
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_MC_MGCG_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+
+			if (PP_STATE_SUPPORT_LS & *msg_id) {
+				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_MC_MGLS_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+
+			}
+			break;
+
+		case PP_BLOCK_SYS_HDP:
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+					? PPSMC_MSG_EnableClockGatingFeature
+					: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_HDP_MGCG_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+
+			if (PP_STATE_SUPPORT_LS & *msg_id) {
+				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
+					? PPSMC_MSG_EnableClockGatingFeature
+					: PPSMC_MSG_DisableClockGatingFeature;
+
+				value = CG_SYS_HDP_MGLS_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		case PP_BLOCK_SYS_SDMA:
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_SDMA_MGCG_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+
+			if (PP_STATE_SUPPORT_LS & *msg_id) {
+				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+
+				value = CG_SYS_SDMA_MGLS_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		case PP_BLOCK_SYS_ROM:
+			if (PP_STATE_SUPPORT_CG & *msg_id) {
+				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
+				? PPSMC_MSG_EnableClockGatingFeature
+				: PPSMC_MSG_DisableClockGatingFeature;
+				value = CG_SYS_ROM_MASK;
+
+				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
+					return -1;
+			}
+			break;
+
+		default:
+			return -1;
+
+		}
+		break;
+
+	default:
+		return -1;
+
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.h
new file mode 100644
index 000000000000..8bc38cb17b7f
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _TONGA_CLOCK_POWER_GATING_H_
+#define _TONGA_CLOCK_POWER_GATING_H_
+
+#include "tonga_hwmgr.h"
+#include "pp_asicblocks.h"
+
+extern int tonga_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating);
+extern int tonga_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
+extern int tonga_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
+extern int tonga_phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
+extern int tonga_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
+extern int tonga_phm_update_clock_gatings(struct pp_hwmgr *hwmgr, const uint32_t *msg_id);
+#endif /* _TONGA_CLOCK_POWER_GATING_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h
new file mode 100644
index 000000000000..080d69d77f04
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef TONGA_DYN_DEFAULTS_H
+#define TONGA_DYN_DEFAULTS_H
+
+
+/** \file
+ * Volcanic Islands Dynamic default parameters.
+ */
+
+enum TONGAdpm_TrendDetection {
+	TONGAdpm_TrendDetection_AUTO,
+	TONGAdpm_TrendDetection_UP,
+	TONGAdpm_TrendDetection_DOWN
+};
+typedef enum TONGAdpm_TrendDetection TONGAdpm_TrendDetection;
+
+/* Bit vector representing same fields as hardware register. */
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT0              0x3FFFC102  /* CP_Gfx_busy */
+/* HDP_busy */
+/* IH_busy */
+/* DRM_busy */
+/* DRMDMA_busy */
+/* UVD_busy */
+/* VCE_busy */
+/* ACP_busy */
+/* SAMU_busy */
+/* AVP_busy  */
+/* SDMA enabled */
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT1              0x000400  /* FE_Gfx_busy  - Intended for primary usage.   Rest are for flexibility. */
+/* SH_Gfx_busy */
+/* RB_Gfx_busy */
+/* VCE_busy */
+
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT2              0xC00080  /* SH_Gfx_busy - Intended for primary usage.   Rest are for flexibility. */
+/* FE_Gfx_busy */
+/* RB_Gfx_busy */
+/* ACP_busy */
+
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT3              0xC00200  /* RB_Gfx_busy - Intended for primary usage.   Rest are for flexibility. */
+/* FE_Gfx_busy */
+/* SH_Gfx_busy */
+/* UVD_busy */
+
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT4              0xC01680  /* UVD_busy */
+/* VCE_busy */
+/* ACP_busy */
+/* SAMU_busy */
+
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT5              0xC00033  /* GFX, HDP, DRMDMA */
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT6              0xC00033  /* GFX, HDP, DRMDMA */
+#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT7              0x3FFFC000  /* GFX, HDP, DRMDMA */
+
+
+/* thermal protection counter (units).*/
+#define PPTONGA_THERMALPROTECTCOUNTER_DFLT            0x200 /* ~19us */
+
+/* static screen threshold unit */
+#define PPTONGA_STATICSCREENTHRESHOLDUNIT_DFLT        0
+
+/* static screen threshold */
+#define PPTONGA_STATICSCREENTHRESHOLD_DFLT            0x00C8
+
+/* gfx idle clock stop threshold */
+#define PPTONGA_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200 /* ~19us with static screen threshold unit of 0 */
+
+/* Fixed reference divider to use when building baby stepping tables. */
+#define PPTONGA_REFERENCEDIVIDER_DFLT                  4
+
+/*
+ * ULV voltage change delay time
+ * Used to be delay_vreg in N.I. split for S.I.
+ * Using N.I. delay_vreg value as default
+ * ReferenceClock = 2700
+ * VoltageResponseTime = 1000
+ * VDDCDelayTime = (VoltageResponseTime * ReferenceClock) / 1600 = 1687
+ */
+
+#define PPTONGA_ULVVOLTAGECHANGEDELAY_DFLT             1687
+
+#define PPTONGA_CGULVPARAMETER_DFLT                    0x00040035
+#define PPTONGA_CGULVCONTROL_DFLT                      0x00007450
+#define PPTONGA_TARGETACTIVITY_DFLT                     30  /*30%  */
+#define PPTONGA_MCLK_TARGETACTIVITY_DFLT                10  /*10%  */
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
new file mode 100644
index 000000000000..44a925006479
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
@@ -0,0 +1,6075 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/fb.h>
+#include "linux/delay.h"
+#include "pp_acpi.h"
+#include "hwmgr.h"
+#include <atombios.h>
+#include "tonga_hwmgr.h"
+#include "pptable.h"
+#include "processpptables.h"
+#include "tonga_processpptables.h"
+#include "tonga_pptable.h"
+#include "pp_debug.h"
+#include "tonga_ppsmc.h"
+#include "cgs_common.h"
+#include "pppcielanes.h"
+#include "tonga_dyn_defaults.h"
+#include "smumgr.h"
+#include "tonga_smumgr.h"
+#include "tonga_clockpowergating.h"
+#include "tonga_thermal.h"
+
+#include "smu/smu_7_1_2_d.h"
+#include "smu/smu_7_1_2_sh_mask.h"
+
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+
+#include "cgs_linux.h"
+#include "eventmgr.h"
+#include "amd_pcie_helpers.h"
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define MC_CG_SEQ_DRAMCONF_S0       0x05
+#define MC_CG_SEQ_DRAMCONF_S1       0x06
+#define MC_CG_SEQ_YCLK_SUSPEND      0x04
+#define MC_CG_SEQ_YCLK_RESUME       0x0a
+
+#define PCIE_BUS_CLK                10000
+#define TCLK                        (PCIE_BUS_CLK / 10)
+
+#define SMC_RAM_END 0x40000
+#define SMC_CG_IND_START            0xc0030000
+#define SMC_CG_IND_END              0xc0040000  /* First byte after SMC_CG_IND*/
+
+#define VOLTAGE_SCALE               4
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+
+#define VDDC_VDDCI_DELTA            200
+#define VDDC_VDDGFX_DELTA           300
+
+#define MC_SEQ_MISC0_GDDR5_SHIFT 28
+#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
+#define MC_SEQ_MISC0_GDDR5_VALUE 5
+
+typedef uint32_t PECI_RegistryValue;
+
+/* [2.5%,~2.5%] Clock stretched is multiple of 2.5% vs not and [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ] */
+uint16_t PP_ClockStretcherLookupTable[2][4] = {
+	{600, 1050, 3, 0},
+	{600, 1050, 6, 1} };
+
+/* [FF, SS] type, [] 4 voltage ranges, and [Floor Freq, Boundary Freq, VID min , VID max] */
+uint32_t PP_ClockStretcherDDTTable[2][4][4] = {
+	{ {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
+	{ {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
+
+/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%] (coming from PWR_CKS_CNTL.stretch_amount reg spec) */
+uint8_t PP_ClockStretchAmountConversion[2][6] = {
+	{0, 1, 3, 2, 4, 5},
+	{0, 2, 4, 5, 6, 5} };
+
+/* Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
+enum DPM_EVENT_SRC {
+	DPM_EVENT_SRC_ANALOG = 0,               /* Internal analog trip point */
+	DPM_EVENT_SRC_EXTERNAL = 1,             /* External (GPIO 17) signal */
+	DPM_EVENT_SRC_DIGITAL = 2,              /* Internal digital trip point (DIG_THERM_DPM) */
+	DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,   /* Internal analog or external */
+	DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4   /* Internal digital or external */
+};
+typedef enum DPM_EVENT_SRC DPM_EVENT_SRC;
+
+const unsigned long PhwTonga_Magic = (unsigned long)(PHM_VIslands_Magic);
+
+struct tonga_power_state *cast_phw_tonga_power_state(
+				  struct pp_hw_power_state *hw_ps)
+{
+	if (hw_ps == NULL)
+		return NULL;
+
+	PP_ASSERT_WITH_CODE((PhwTonga_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL);
+
+	return (struct tonga_power_state *)hw_ps;
+}
+
+const struct tonga_power_state *cast_const_phw_tonga_power_state(
+				 const struct pp_hw_power_state *hw_ps)
+{
+	if (hw_ps == NULL)
+		return NULL;
+
+	PP_ASSERT_WITH_CODE((PhwTonga_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL);
+
+	return (const struct tonga_power_state *)hw_ps;
+}
+
+int tonga_add_voltage(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_voltage_lookup_table *look_up_table,
+	phm_ppt_v1_voltage_lookup_record *record)
+{
+	uint32_t i;
+	PP_ASSERT_WITH_CODE((NULL != look_up_table),
+		"Lookup Table empty.", return -1;);
+	PP_ASSERT_WITH_CODE((0 != look_up_table->count),
+		"Lookup Table empty.", return -1;);
+	PP_ASSERT_WITH_CODE((SMU72_MAX_LEVELS_VDDGFX >= look_up_table->count),
+		"Lookup Table is full.", return -1;);
+
+	/* This is to avoid entering duplicate calculated records. */
+	for (i = 0; i < look_up_table->count; i++) {
+		if (look_up_table->entries[i].us_vdd == record->us_vdd) {
+			if (look_up_table->entries[i].us_calculated == 1)
+				return 0;
+			else
+				break;
+		}
+	}
+
+	look_up_table->entries[i].us_calculated = 1;
+	look_up_table->entries[i].us_vdd = record->us_vdd;
+	look_up_table->entries[i].us_cac_low = record->us_cac_low;
+	look_up_table->entries[i].us_cac_mid = record->us_cac_mid;
+	look_up_table->entries[i].us_cac_high = record->us_cac_high;
+	/* Only increment the count when we're appending, not replacing duplicate entry. */
+	if (i == look_up_table->count)
+		look_up_table->count++;
+
+	return 0;
+}
+
+int tonga_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
+{
+	PPSMC_Msg msg = has_display? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay;
+
+	return (smum_send_msg_to_smc(hwmgr->smumgr, msg) == 0) ?  0 : -1;
+}
+
+uint8_t tonga_get_voltage_id(pp_atomctrl_voltage_table *voltage_table,
+		uint32_t voltage)
+{
+	uint8_t count = (uint8_t) (voltage_table->count);
+	uint8_t i = 0;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_table),
+		"Voltage Table empty.", return 0;);
+	PP_ASSERT_WITH_CODE((0 != count),
+		"Voltage Table empty.", return 0;);
+
+	for (i = 0; i < count; i++) {
+		/* find first voltage bigger than requested */
+		if (voltage_table->entries[i].value >= voltage)
+			return i;
+	}
+
+	/* voltage is bigger than max voltage in the table */
+	return i - 1;
+}
+
+/**
+ * @brief PhwTonga_GetVoltageOrder
+ *  Returns index of requested voltage record in lookup(table)
+ * @param hwmgr - pointer to hardware manager
+ * @param lookupTable - lookup list to search in
+ * @param voltage - voltage to look for
+ * @return 0 on success
+ */
+uint8_t tonga_get_voltage_index(phm_ppt_v1_voltage_lookup_table *look_up_table,
+		uint16_t voltage)
+{
+	uint8_t count = (uint8_t) (look_up_table->count);
+	uint8_t i;
+
+	PP_ASSERT_WITH_CODE((NULL != look_up_table), "Lookup Table empty.", return 0;);
+	PP_ASSERT_WITH_CODE((0 != count), "Lookup Table empty.", return 0;);
+
+	for (i = 0; i < count; i++) {
+		/* find first voltage equal or bigger than requested */
+		if (look_up_table->entries[i].us_vdd >= voltage)
+			return i;
+	}
+
+	/* voltage is bigger than max voltage in the table */
+	return i-1;
+}
+
+bool tonga_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	/*
+	 * We return the status of Voltage Control instead of checking SCLK/MCLK DPM
+	 * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM,
+	 * whereas voltage control is a fundemental change that will not be disabled
+	 */
+
+	return (0 == PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					FEATURE_STATUS, VOLTAGE_CONTROLLER_ON) ? 1 : 0);
+}
+
+/**
+ * Re-generate the DPM level mask value
+ * @param    hwmgr      the address of the hardware manager
+ */
+static uint32_t tonga_get_dpm_level_enable_mask_value(
+			struct tonga_single_dpm_table * dpm_table)
+{
+	uint32_t i;
+	uint32_t mask_value = 0;
+
+	for (i = dpm_table->count; i > 0; i--) {
+		mask_value = mask_value << 1;
+
+		if (dpm_table->dpm_levels[i-1].enabled)
+			mask_value |= 0x1;
+		else
+			mask_value &= 0xFFFFFFFE;
+	}
+	return mask_value;
+}
+
+/**
+ * Retrieve DPM default values from registry (if available)
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ */
+void tonga_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	phw_tonga_ulv_parm *ulv = &(data->ulv);
+	uint32_t tmp;
+
+	ulv->ch_ulv_parameter = PPTONGA_CGULVPARAMETER_DFLT;
+	data->voting_rights_clients0 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT0;
+	data->voting_rights_clients1 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT1;
+	data->voting_rights_clients2 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT2;
+	data->voting_rights_clients3 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT3;
+	data->voting_rights_clients4 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT4;
+	data->voting_rights_clients5 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT5;
+	data->voting_rights_clients6 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT6;
+	data->voting_rights_clients7 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT7;
+
+	data->static_screen_threshold_unit = PPTONGA_STATICSCREENTHRESHOLDUNIT_DFLT;
+	data->static_screen_threshold = PPTONGA_STATICSCREENTHRESHOLD_DFLT;
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_ABM);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_NonABMSupportInPPLib);
+
+	tmp = 0;
+	if (tmp == 0)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DynamicACTiming);
+
+	tmp = 0;
+	if (0 != tmp)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DisableMemoryTransition);
+
+	data->mclk_strobe_mode_threshold = 40000;
+	data->mclk_stutter_mode_threshold = 30000;
+	data->mclk_edc_enable_threshold = 40000;
+	data->mclk_edc_wr_enable_threshold = 40000;
+
+	tmp = 0;
+	if (tmp != 0)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DisableMCLS);
+
+	data->pcie_gen_performance.max = PP_PCIEGen1;
+	data->pcie_gen_performance.min = PP_PCIEGen3;
+	data->pcie_gen_power_saving.max = PP_PCIEGen1;
+	data->pcie_gen_power_saving.min = PP_PCIEGen3;
+
+	data->pcie_lane_performance.max = 0;
+	data->pcie_lane_performance.min = 16;
+	data->pcie_lane_power_saving.max = 0;
+	data->pcie_lane_power_saving.min = 16;
+
+	tmp = 0;
+
+	if (tmp)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_DynamicUVDState);
+
+}
+
+int tonga_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold != data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold = hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold = data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = tonga_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable,
+				LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				data->sram_end
+				);
+	}
+
+	return result;
+}
+
+/**
+ * Find SCLK value that is associated with specified virtual_voltage_Id.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    virtual_voltage_Id  voltageId to look for.
+ * @param    sclk output value .
+ * @return   always 0 if success and 2 if association not found
+ */
+static int tonga_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_voltage_lookup_table *lookup_table,
+	uint16_t virtual_voltage_id, uint32_t *sclk)
+{
+	uint8_t entryId;
+	uint8_t voltageId;
+	struct phm_ppt_v1_information *pptable_info =
+					(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -1);
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
+	for (entryId = 0; entryId < pptable_info->vdd_dep_on_sclk->count; entryId++) {
+		voltageId = pptable_info->vdd_dep_on_sclk->entries[entryId].vddInd;
+		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
+			break;
+	}
+
+	PP_ASSERT_WITH_CODE(entryId < pptable_info->vdd_dep_on_sclk->count,
+			"Can't find requested voltage id in vdd_dep_on_sclk table!",
+			return -1;
+			);
+
+	*sclk = pptable_info->vdd_dep_on_sclk->entries[entryId].clk;
+
+	return 0;
+}
+
+/**
+ * Get Leakage VDDC based on leakage ID.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   2 if vddgfx returned is greater than 2V or if BIOS
+ */
+int tonga_get_evv_voltage(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
+	uint16_t    virtual_voltage_id;
+	uint16_t    vddc = 0;
+	uint16_t    vddgfx = 0;
+	uint16_t    i, j;
+	uint32_t  sclk = 0;
+
+	/* retrieve voltage for leakage ID (0xff01 + i) */
+	for (i = 0; i < TONGA_MAX_LEAKAGE_COUNT; i++) {
+		virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
+
+		/* in split mode we should have only vddgfx EVV leakages */
+		if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
+			if (0 == tonga_get_sclk_for_voltage_evv(hwmgr,
+						pptable_info->vddgfx_lookup_table, virtual_voltage_id, &sclk)) {
+				if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+							PHM_PlatformCaps_ClockStretcher)) {
+					for (j = 1; j < sclk_table->count; j++) {
+						if (sclk_table->entries[j].clk == sclk &&
+								sclk_table->entries[j].cks_enable == 0) {
+							sclk += 5000;
+							break;
+						}
+					}
+				}
+				PP_ASSERT_WITH_CODE(0 == atomctrl_get_voltage_evv_on_sclk
+						(hwmgr, VOLTAGE_TYPE_VDDGFX, sclk,
+						 virtual_voltage_id, &vddgfx),
+						"Error retrieving EVV voltage value!", continue);
+
+				/* need to make sure vddgfx is less than 2v or else, it could burn the ASIC. */
+				PP_ASSERT_WITH_CODE((vddgfx < 2000 && vddgfx != 0), "Invalid VDDGFX value!", return -1);
+
+				/* the voltage should not be zero nor equal to leakage ID */
+				if (vddgfx != 0 && vddgfx != virtual_voltage_id) {
+					data->vddcgfx_leakage.actual_voltage[data->vddcgfx_leakage.count] = vddgfx;
+					data->vddcgfx_leakage.leakage_id[data->vddcgfx_leakage.count] = virtual_voltage_id;
+					data->vddcgfx_leakage.count++;
+				}
+			}
+		} else {
+			/*  in merged mode we have only vddc EVV leakages */
+			if (0 == tonga_get_sclk_for_voltage_evv(hwmgr,
+						pptable_info->vddc_lookup_table,
+						virtual_voltage_id, &sclk)) {
+				PP_ASSERT_WITH_CODE(0 == atomctrl_get_voltage_evv_on_sclk
+						(hwmgr, VOLTAGE_TYPE_VDDC, sclk,
+						 virtual_voltage_id, &vddc),
+						"Error retrieving EVV voltage value!", continue);
+
+				/* need to make sure vddc is less than 2v or else, it could burn the ASIC. */
+				if (vddc > 2000)
+					printk(KERN_ERR "[ powerplay ] Invalid VDDC value! \n");
+
+				/* the voltage should not be zero nor equal to leakage ID */
+				if (vddc != 0 && vddc != virtual_voltage_id) {
+					data->vddc_leakage.actual_voltage[data->vddc_leakage.count] = vddc;
+					data->vddc_leakage.leakage_id[data->vddc_leakage.count] = virtual_voltage_id;
+					data->vddc_leakage.count++;
+				}
+			}
+		}
+	}
+
+	return 0;
+}
+
+int tonga_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* enable SCLK dpm */
+	if (0 == data->sclk_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						   PPSMC_MSG_DPM_Enable)),
+				"Failed to enable SCLK DPM during DPM Start Function!",
+				return -1);
+	}
+
+	/* enable MCLK dpm */
+	if (0 == data->mclk_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					     PPSMC_MSG_MCLKDPM_Enable)),
+				"Failed to enable MCLK DPM during DPM Start Function!",
+				return -1);
+
+		PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_MC0_CNTL, 0x05);/* CH0,1 read */
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_MC1_CNTL, 0x05);/* CH2,3 read */
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_CPL_CNTL, 0x100005);/*Read */
+
+		udelay(10);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_MC0_CNTL, 0x400005);/* CH0,1 write */
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_MC1_CNTL, 0x400005);/* CH2,3 write */
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixLCAC_CPL_CNTL, 0x500005);/* write */
+
+	}
+
+	return 0;
+}
+
+int tonga_start_dpm(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* enable general power management */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, 1);
+	/* enable sclk deep sleep */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, DYNAMIC_PM_EN, 1);
+
+	/* prepare for PCIE DPM */
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start +
+			offsetof(SMU72_SoftRegisters, VoltageChangeTimeout), 0x1000);
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE, SWRST_COMMAND_1, RESETLC, 0x0);
+
+	PP_ASSERT_WITH_CODE(
+			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					PPSMC_MSG_Voltage_Cntl_Enable)),
+			"Failed to enable voltage DPM during DPM Start Function!",
+			return -1);
+
+	if (0 != tonga_enable_sclk_mclk_dpm(hwmgr)) {
+		PP_ASSERT_WITH_CODE(0, "Failed to enable Sclk DPM and Mclk DPM!", return -1);
+	}
+
+	/* enable PCIE dpm */
+	if (0 == data->pcie_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_Enable)),
+				"Failed to enable pcie DPM during DPM Start Function!",
+				return -1
+				);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_Falcon_QuickTransition)) {
+				   smum_send_msg_to_smc(hwmgr->smumgr,
+				    PPSMC_MSG_EnableACDCGPIOInterrupt);
+	}
+
+	return 0;
+}
+
+int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* disable SCLK dpm */
+	if (0 == data->sclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+		PP_ASSERT_WITH_CODE(
+				(0 == tonga_is_dpm_running(hwmgr)),
+				"Trying to Disable SCLK DPM when DPM is disabled",
+				return -1
+				);
+
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						  PPSMC_MSG_DPM_Disable)),
+				"Failed to disable SCLK DPM during DPM stop Function!",
+				return -1);
+	}
+
+	/* disable MCLK dpm */
+	if (0 == data->mclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
+		PP_ASSERT_WITH_CODE(
+				(0 == tonga_is_dpm_running(hwmgr)),
+				"Trying to Disable MCLK DPM when DPM is disabled",
+				return -1
+				);
+
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					    PPSMC_MSG_MCLKDPM_Disable)),
+				"Failed to Disable MCLK DPM during DPM stop Function!",
+				return -1);
+	}
+
+	return 0;
+}
+
+int tonga_stop_dpm(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, 0);
+	/* disable sclk deep sleep*/
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, DYNAMIC_PM_EN, 0);
+
+	/* disable PCIE dpm */
+	if (0 == data->pcie_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+		PP_ASSERT_WITH_CODE(
+				(0 == tonga_is_dpm_running(hwmgr)),
+				"Trying to Disable PCIE DPM when DPM is disabled",
+				return -1
+				);
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_Disable)),
+				"Failed to disable pcie DPM during DPM stop Function!",
+				return -1);
+	}
+
+	if (0 != tonga_disable_sclk_mclk_dpm(hwmgr))
+		PP_ASSERT_WITH_CODE(0, "Failed to disable Sclk DPM and Mclk DPM!", return -1);
+
+	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+	PP_ASSERT_WITH_CODE(
+			(0 == tonga_is_dpm_running(hwmgr)),
+			"Trying to Disable Voltage CNTL when DPM is disabled",
+			return -1
+			);
+
+	PP_ASSERT_WITH_CODE(
+			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					PPSMC_MSG_Voltage_Cntl_Disable)),
+			"Failed to disable voltage DPM during DPM stop Function!",
+			return -1);
+
+	return 0;
+}
+
+int tonga_enable_sclk_control(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, 0);
+
+	return 0;
+}
+
+/**
+ * Send a message to the SMC and return a parameter
+ *
+ * @param    hwmgr:  the address of the powerplay hardware manager.
+ * @param    msg: the message to send.
+ * @param    parameter: pointer to the received parameter
+ * @return   The response that came from the SMC.
+ */
+PPSMC_Result tonga_send_msg_to_smc_return_parameter(
+		struct pp_hwmgr *hwmgr,
+		PPSMC_Msg msg,
+		uint32_t *parameter)
+{
+	int result;
+
+	result = smum_send_msg_to_smc(hwmgr->smumgr, msg);
+
+	if ((0 == result) && parameter) {
+		*parameter = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+	}
+
+	return result;
+}
+
+/**
+ * force DPM power State
+ *
+ * @param    hwmgr:  the address of the powerplay hardware manager.
+ * @param    n     :  DPM level
+ * @return   The response that came from the SMC.
+ */
+int tonga_dpm_force_state(struct pp_hwmgr *hwmgr, uint32_t n)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t level_mask = 1 << n;
+
+	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
+	PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
+			"Trying to force SCLK when DPM is disabled", return -1;);
+	if (0 == data->sclk_dpm_key_disabled)
+		return (0 == smum_send_msg_to_smc_with_parameter(
+							     hwmgr->smumgr,
+		     (PPSMC_Msg)(PPSMC_MSG_SCLKDPM_SetEnabledMask),
+						    level_mask) ? 0 : 1);
+
+	return 0;
+}
+
+/**
+ * force DPM power State
+ *
+ * @param    hwmgr:  the address of the powerplay hardware manager.
+ * @param    n     :  DPM level
+ * @return   The response that came from the SMC.
+ */
+int tonga_dpm_force_state_mclk(struct pp_hwmgr *hwmgr, uint32_t n)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t level_mask = 1 << n;
+
+	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
+	PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
+			"Trying to Force MCLK when DPM is disabled", return -1;);
+	if (0 == data->mclk_dpm_key_disabled)
+		return (0 == smum_send_msg_to_smc_with_parameter(
+								hwmgr->smumgr,
+								(PPSMC_Msg)(PPSMC_MSG_MCLKDPM_SetEnabledMask),
+								level_mask) ? 0 : 1);
+
+	return 0;
+}
+
+/**
+ * force DPM power State
+ *
+ * @param    hwmgr:  the address of the powerplay hardware manager.
+ * @param    n     :  DPM level
+ * @return   The response that came from the SMC.
+ */
+int tonga_dpm_force_state_pcie(struct pp_hwmgr *hwmgr, uint32_t n)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+	PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
+			"Trying to Force PCIE level when DPM is disabled", return -1;);
+	if (0 == data->pcie_dpm_key_disabled)
+		return (0 == smum_send_msg_to_smc_with_parameter(
+							     hwmgr->smumgr,
+			   (PPSMC_Msg)(PPSMC_MSG_PCIeDPM_ForceLevel),
+								n) ? 0 : 1);
+
+	return 0;
+}
+
+/**
+ * Set the initial state by calling SMC to switch to this state directly
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_set_boot_state(struct pp_hwmgr *hwmgr)
+{
+	/*
+	* SMC only stores one state that SW will ask to switch too,
+	* so we switch the the just uploaded one
+	*/
+	return (0 == tonga_disable_sclk_mclk_dpm(hwmgr)) ? 0 : 1;
+}
+
+/**
+ * Get the location of various tables inside the FW image.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *tonga_smu = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	uint32_t tmp;
+	int result;
+	bool error = 0;
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, DpmTable),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		data->dpm_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, SoftRegisters),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		data->soft_regs_start = tmp;
+		tonga_smu->ulSoftRegsStart = tmp;
+	}
+
+	error |= (0 != result);
+
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, mcRegisterTable),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		data->mc_reg_table_start = tmp;
+	}
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, FanTable),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		data->fan_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, mcArbDramTimingTable),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		data->arb_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, Version),
+				&tmp, data->sram_end);
+
+	if (0 == result) {
+		hwmgr->microcode_version_info.SMC = tmp;
+	}
+
+	error |= (0 != result);
+
+	return error ? 1 : 0;
+}
+
+/**
+ * Read clock related registers.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_read_clock_registers(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	data->clock_registers.vCG_SPLL_FUNC_CNTL         =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_2       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_2);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_3       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_3);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_4       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_4);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM   =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM_2);
+	data->clock_registers.vDLL_CNTL                  =
+		cgs_read_register(hwmgr->device, mmDLL_CNTL);
+	data->clock_registers.vMCLK_PWRMGT_CNTL          =
+		cgs_read_register(hwmgr->device, mmMCLK_PWRMGT_CNTL);
+	data->clock_registers.vMPLL_AD_FUNC_CNTL         =
+		cgs_read_register(hwmgr->device, mmMPLL_AD_FUNC_CNTL);
+	data->clock_registers.vMPLL_DQ_FUNC_CNTL         =
+		cgs_read_register(hwmgr->device, mmMPLL_DQ_FUNC_CNTL);
+	data->clock_registers.vMPLL_FUNC_CNTL            =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL);
+	data->clock_registers.vMPLL_FUNC_CNTL_1          =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_1);
+	data->clock_registers.vMPLL_FUNC_CNTL_2          =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_2);
+	data->clock_registers.vMPLL_SS1                  =
+		cgs_read_register(hwmgr->device, mmMPLL_SS1);
+	data->clock_registers.vMPLL_SS2                  =
+		cgs_read_register(hwmgr->device, mmMPLL_SS2);
+
+	return 0;
+}
+
+/**
+ * Find out if memory is GDDR5.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_get_memory_type(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t temp;
+
+	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
+
+	data->is_memory_GDDR5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
+			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
+			 MC_SEQ_MISC0_GDDR5_SHIFT));
+
+	return 0;
+}
+
+/**
+ * Enables Dynamic Power Management by SMC
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, STATIC_PM_EN, 1);
+
+	return 0;
+}
+
+/**
+ * Initialize PowerGating States for different engines
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_init_power_gate_state(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = 0;
+	data->vce_power_gated = 0;
+	data->samu_power_gated = 0;
+	data->acp_power_gated = 0;
+	data->pg_acp_init = 1;
+
+	return 0;
+}
+
+/**
+ * Checks if DPM is enabled
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_check_for_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	/*
+	 * We return the status of Voltage Control instead of checking SCLK/MCLK DPM
+	 * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM,
+	 * whereas voltage control is a fundemental change that will not be disabled
+	 */
+	return (0 == tonga_is_dpm_running(hwmgr) ? 0 : 1);
+}
+
+/**
+ * Checks if DPM is stopped
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_check_for_dpm_stopped(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	if (0 != tonga_is_dpm_running(hwmgr)) {
+		/* If HW Virtualization is enabled, dpm_table_start will not have a valid value */
+		if (!data->dpm_table_start) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Remove repeated voltage values and create table with unique values.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    voltage_table  the pointer to changing voltage table
+ * @return    1 in success
+ */
+
+static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr,
+			pp_atomctrl_voltage_table *voltage_table)
+{
+	uint32_t table_size, i, j;
+	uint16_t vvalue;
+	bool bVoltageFound = 0;
+	pp_atomctrl_voltage_table *table;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_table), "Voltage Table empty.", return -1;);
+	table_size = sizeof(pp_atomctrl_voltage_table);
+	table = kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	memset(table, 0x00, table_size);
+	table->mask_low = voltage_table->mask_low;
+	table->phase_delay = voltage_table->phase_delay;
+
+	for (i = 0; i < voltage_table->count; i++) {
+		vvalue = voltage_table->entries[i].value;
+		bVoltageFound = 0;
+
+		for (j = 0; j < table->count; j++) {
+			if (vvalue == table->entries[j].value) {
+				bVoltageFound = 1;
+				break;
+			}
+		}
+
+		if (!bVoltageFound) {
+			table->entries[table->count].value = vvalue;
+			table->entries[table->count].smio_low =
+				voltage_table->entries[i].smio_low;
+			table->count++;
+		}
+	}
+
+	memcpy(table, voltage_table, sizeof(pp_atomctrl_voltage_table));
+
+	kfree(table);
+
+	return 0;
+}
+
+static int tonga_get_svi2_vdd_ci_voltage_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_clock_voltage_dependency_table *voltage_dependency_table)
+{
+	uint32_t i;
+	int result;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_voltage_table *vddci_voltage_table = &(data->vddci_voltage_table);
+
+	PP_ASSERT_WITH_CODE((0 != voltage_dependency_table->count),
+			"Voltage Dependency Table empty.", return -1;);
+
+	vddci_voltage_table->mask_low = 0;
+	vddci_voltage_table->phase_delay = 0;
+	vddci_voltage_table->count = voltage_dependency_table->count;
+
+	for (i = 0; i < voltage_dependency_table->count; i++) {
+		vddci_voltage_table->entries[i].value =
+			voltage_dependency_table->entries[i].vddci;
+		vddci_voltage_table->entries[i].smio_low = 0;
+	}
+
+	result = tonga_trim_voltage_table(hwmgr, vddci_voltage_table);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to trim VDDCI table.", return result;);
+
+	return 0;
+}
+
+
+
+static int tonga_get_svi2_vdd_voltage_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *look_up_table,
+		pp_atomctrl_voltage_table *voltage_table)
+{
+	uint8_t i = 0;
+
+	PP_ASSERT_WITH_CODE((0 != look_up_table->count),
+			"Voltage Lookup Table empty.", return -1;);
+
+	voltage_table->mask_low = 0;
+	voltage_table->phase_delay = 0;
+
+	voltage_table->count = look_up_table->count;
+
+	for (i = 0; i < voltage_table->count; i++) {
+		voltage_table->entries[i].value = look_up_table->entries[i].us_vdd;
+		voltage_table->entries[i].smio_low = 0;
+	}
+
+	return 0;
+}
+
+/*
+ * -------------------------------------------------------- Voltage Tables --------------------------------------------------------------------------
+ * If the voltage table would be bigger than what will fit into the state table on the SMC keep only the higher entries.
+ */
+
+static void tonga_trim_voltage_table_to_fit_state_table(
+		struct pp_hwmgr *hwmgr,
+		uint32_t max_voltage_steps,
+		pp_atomctrl_voltage_table *voltage_table)
+{
+	unsigned int i, diff;
+
+	if (voltage_table->count <= max_voltage_steps) {
+		return;
+	}
+
+	diff = voltage_table->count - max_voltage_steps;
+
+	for (i = 0; i < max_voltage_steps; i++) {
+		voltage_table->entries[i] = voltage_table->entries[i + diff];
+	}
+
+	voltage_table->count = max_voltage_steps;
+
+	return;
+}
+
+/**
+ * Create Voltage Tables.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_construct_voltage_tables(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result;
+
+	/* MVDD has only GPIO voltage control */
+	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+					VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT, &(data->mvdd_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve MVDD table.", return result;);
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
+		/* GPIO voltage */
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+					VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT, &(data->vddci_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve VDDCI table.", return result;);
+	} else if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
+		/* SVI2 voltage */
+		result = tonga_get_svi2_vdd_ci_voltage_table(hwmgr,
+					pptable_info->vdd_dep_on_mclk);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve SVI2 VDDCI table from dependancy table.", return result;);
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		/* VDDGFX has only SVI2 voltage control */
+		result = tonga_get_svi2_vdd_voltage_table(hwmgr,
+					pptable_info->vddgfx_lookup_table, &(data->vddgfx_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve SVI2 VDDGFX table from lookup table.", return result;);
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		/* VDDC has only SVI2 voltage control */
+		result = tonga_get_svi2_vdd_voltage_table(hwmgr,
+					pptable_info->vddc_lookup_table, &(data->vddc_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve SVI2 VDDC table from lookup table.", return result;);
+	}
+
+	PP_ASSERT_WITH_CODE(
+			(data->vddc_voltage_table.count <= (SMU72_MAX_LEVELS_VDDC)),
+			"Too many voltage values for VDDC. Trimming to fit state table.",
+			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
+			SMU72_MAX_LEVELS_VDDC, &(data->vddc_voltage_table));
+			);
+
+	PP_ASSERT_WITH_CODE(
+			(data->vddgfx_voltage_table.count <= (SMU72_MAX_LEVELS_VDDGFX)),
+			"Too many voltage values for VDDGFX. Trimming to fit state table.",
+			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
+			SMU72_MAX_LEVELS_VDDGFX, &(data->vddgfx_voltage_table));
+			);
+
+	PP_ASSERT_WITH_CODE(
+			(data->vddci_voltage_table.count <= (SMU72_MAX_LEVELS_VDDCI)),
+			"Too many voltage values for VDDCI. Trimming to fit state table.",
+			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
+			SMU72_MAX_LEVELS_VDDCI, &(data->vddci_voltage_table));
+			);
+
+	PP_ASSERT_WITH_CODE(
+			(data->mvdd_voltage_table.count <= (SMU72_MAX_LEVELS_MVDD)),
+			"Too many voltage values for MVDD. Trimming to fit state table.",
+			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
+			SMU72_MAX_LEVELS_MVDD, &(data->mvdd_voltage_table));
+			);
+
+	return 0;
+}
+
+/**
+ * Vddc table preparation for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_smc_vddc_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	unsigned int count;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		table->VddcLevelCount = data->vddc_voltage_table.count;
+		for (count = 0; count < table->VddcLevelCount; count++) {
+			table->VddcTable[count] =
+				PP_HOST_TO_SMC_US(data->vddc_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VddcLevelCount);
+	}
+	return 0;
+}
+
+/**
+ * VddGfx table preparation for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_smc_vdd_gfx_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	unsigned int count;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		table->VddGfxLevelCount = data->vddgfx_voltage_table.count;
+		for (count = 0; count < data->vddgfx_voltage_table.count; count++) {
+			table->VddGfxTable[count] =
+				PP_HOST_TO_SMC_US(data->vddgfx_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VddGfxLevelCount);
+	}
+	return 0;
+}
+
+/**
+ * Vddci table preparation for SMC.
+ *
+ * @param    *hwmgr The address of the hardware manager.
+ * @param    *table The SMC DPM table structure to be populated.
+ * @return   0
+ */
+static int tonga_populate_smc_vdd_ci_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+
+	table->VddciLevelCount = data->vddci_voltage_table.count;
+	for (count = 0; count < table->VddciLevelCount; count++) {
+		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
+			table->VddciTable[count] =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		} else if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
+			table->SmioTable1.Pattern[count].Voltage =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level. */
+			table->SmioTable1.Pattern[count].Smio =
+				(uint8_t) count;
+			table->Smio[count] |=
+				data->vddci_voltage_table.entries[count].smio_low;
+			table->VddciTable[count] =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+	}
+
+	table->SmioMask1 = data->vddci_voltage_table.mask_low;
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VddciLevelCount);
+
+	return 0;
+}
+
+/**
+ * Mvdd table preparation for SMC.
+ *
+ * @param    *hwmgr The address of the hardware manager.
+ * @param    *table The SMC DPM table structure to be populated.
+ * @return   0
+ */
+static int tonga_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+
+	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		table->MvddLevelCount = data->mvdd_voltage_table.count;
+		for (count = 0; count < table->MvddLevelCount; count++) {
+			table->SmioTable2.Pattern[count].Voltage =
+				PP_HOST_TO_SMC_US(data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
+			table->SmioTable2.Pattern[count].Smio =
+				(uint8_t) count;
+			table->Smio[count] |=
+				data->mvdd_voltage_table.entries[count].smio_low;
+		}
+		table->SmioMask2 = data->vddci_voltage_table.mask_low;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->MvddLevelCount);
+	}
+
+	return 0;
+}
+
+/**
+ * Convert a voltage value in mv unit to VID number required by SMU firmware
+ */
+static uint8_t convert_to_vid(uint16_t vddc)
+{
+	return (uint8_t) ((6200 - (vddc * VOLTAGE_SCALE)) / 25);
+}
+
+
+/**
+ * Preparation of vddc and vddgfx CAC tables for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index;
+	int result = 0;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table = pptable_info->vddgfx_lookup_table;
+	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table = pptable_info->vddc_lookup_table;
+
+	/* pTables is already swapped, so in order to use the value from it, we need to swap it back. */
+	uint32_t vddcLevelCount = PP_SMC_TO_HOST_UL(table->VddcLevelCount);
+	uint32_t vddgfxLevelCount = PP_SMC_TO_HOST_UL(table->VddGfxLevelCount);
+
+	for (count = 0; count < vddcLevelCount; count++) {
+		/* We are populating vddc CAC data to BapmVddc table in split and merged mode */
+		index = tonga_get_voltage_index(vddc_lookup_table,
+			data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
+		table->BapmVddcVidHiSidd[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
+		table->BapmVddcVidHiSidd2[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
+	}
+
+	if ((data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2)) {
+		/* We are populating vddgfx CAC data to BapmVddgfx table in split mode */
+		for (count = 0; count < vddgfxLevelCount; count++) {
+			index = tonga_get_voltage_index(vddgfx_lookup_table,
+				data->vddgfx_voltage_table.entries[count].value);
+			table->BapmVddGfxVidLoSidd[count] =
+				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_low);
+			table->BapmVddGfxVidHiSidd[count] =
+				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_mid);
+			table->BapmVddGfxVidHiSidd2[count] =
+				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_high);
+		}
+	} else {
+		for (count = 0; count < vddcLevelCount; count++) {
+			index = tonga_get_voltage_index(vddc_lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+			table->BapmVddGfxVidLoSidd[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
+			table->BapmVddGfxVidHiSidd[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
+			table->BapmVddGfxVidHiSidd2[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
+		}
+	}
+
+	return result;
+}
+
+
+/**
+ * Preparation of voltage tables for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+
+int tonga_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+	SMU72_Discrete_DpmTable *table)
+{
+	int result;
+
+	result = tonga_populate_smc_vddc_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate VDDC voltage table to SMC", return -1);
+
+	result = tonga_populate_smc_vdd_ci_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate VDDCI voltage table to SMC", return -1);
+
+	result = tonga_populate_smc_vdd_gfx_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate VDDGFX voltage table to SMC", return -1);
+
+	result = tonga_populate_smc_mvdd_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate MVDD voltage table to SMC", return -1);
+
+	result = tonga_populate_cac_tables(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate CAC voltage tables to SMC", return -1);
+
+	return 0;
+}
+
+/**
+ * Populates the SMC VRConfig field in DPM table.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_vr_config(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint16_t config;
+
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		/*  Splitted mode */
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
+
+		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+			config = VR_SVI2_PLANE_2;
+			table->VRConfig |= config;
+		} else {
+			printk(KERN_ERR "[ powerplay ] VDDC and VDDGFX should be both on SVI2 control in splitted mode! \n");
+		}
+	} else {
+		/* Merged mode  */
+		config = VR_MERGED_WITH_VDDC;
+		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
+
+		/* Set Vddc Voltage Controller  */
+		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+			config = VR_SVI2_PLANE_1;
+			table->VRConfig |= config;
+		} else {
+			printk(KERN_ERR "[ powerplay ] VDDC should be on SVI2 control in merged mode! \n");
+		}
+	}
+
+	/* Set Vddci Voltage Controller  */
+	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
+	} else if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
+	}
+
+	/* Set Mvdd Voltage Controller */
+	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		config = VR_SMIO_PATTERN_2;
+		table->VRConfig |= (config<<VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+static int tonga_get_dependecy_volt_by_clk(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_clock_voltage_table,
+	uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
+{
+	uint32_t i = 0;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	/* clock - voltage dependency table is empty table */
+	if (allowed_clock_voltage_table->count == 0)
+		return -1;
+
+	for (i = 0; i < allowed_clock_voltage_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (allowed_clock_voltage_table->entries[i].clk >= clock) {
+			voltage->VddGfx = tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+								allowed_clock_voltage_table->entries[i].vddgfx);
+
+			voltage->Vddc = tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+								allowed_clock_voltage_table->entries[i].vddc);
+
+			if (allowed_clock_voltage_table->entries[i].vddci) {
+				voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
+									allowed_clock_voltage_table->entries[i].vddci);
+			} else {
+				voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
+									allowed_clock_voltage_table->entries[i].vddc - data->vddc_vddci_delta);
+			}
+
+			if (allowed_clock_voltage_table->entries[i].mvdd) {
+				*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i].mvdd;
+			}
+
+			voltage->Phases = 1;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	voltage->VddGfx = tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+		allowed_clock_voltage_table->entries[i-1].vddgfx);
+	voltage->Vddc = tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+		allowed_clock_voltage_table->entries[i-1].vddc);
+
+	if (allowed_clock_voltage_table->entries[i-1].vddci) {
+		voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
+			allowed_clock_voltage_table->entries[i-1].vddci);
+	}
+	if (allowed_clock_voltage_table->entries[i-1].mvdd) {
+		*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i-1].mvdd;
+	}
+
+	return 0;
+}
+
+/**
+ * Call SMC to reset S0/S1 to S1 and Reset SMIO to initial value
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_reset_to_default(struct pp_hwmgr *hwmgr)
+{
+	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults) == 0) ? 0 : 1;
+}
+
+int tonga_populate_memory_timing_parameters(
+		struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock,
+		uint32_t memory_clock,
+		struct SMU72_Discrete_MCArbDramTimingTableEntry *arb_regs
+		)
+{
+	uint32_t dramTiming;
+	uint32_t dramTiming2;
+	uint32_t burstTime;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+				engine_clock, memory_clock);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dramTiming  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dramTiming2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burstTime = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dramTiming);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dramTiming2);
+	arb_regs->McArbBurstTime = (uint8_t)burstTime;
+
+	return 0;
+}
+
+/**
+ * Setup parameters for the MC ARB.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ * This function is to be called from the SetPowerState table.
+ */
+int tonga_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	int result = 0;
+	SMU72_Discrete_MCArbDramTimingTable  arb_regs;
+	uint32_t i, j;
+
+	memset(&arb_regs, 0x00, sizeof(SMU72_Discrete_MCArbDramTimingTable));
+
+	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
+			result = tonga_populate_memory_timing_parameters
+				(hwmgr, data->dpm_table.sclk_table.dpm_levels[i].value,
+				 data->dpm_table.mclk_table.dpm_levels[j].value,
+				 &arb_regs.entries[i][j]);
+
+			if (0 != result) {
+				break;
+			}
+		}
+	}
+
+	if (0 == result) {
+		result = tonga_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				data->arb_table_start,
+				(uint8_t *)&arb_regs,
+				sizeof(SMU72_Discrete_MCArbDramTimingTable),
+				data->sram_end
+				);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_link_level(struct pp_hwmgr *hwmgr, SMU72_Discrete_DpmTable *table)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_dpm_table *dpm_table = &data->dpm_table;
+	uint32_t i;
+
+	/* Index (dpm_table->pcie_speed_table.count) is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+			(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount =
+			(uint8_t)encode_pcie_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity =
+			1;
+		table->LinkLevel[i].SPC =
+			(uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold =
+			PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold =
+			PP_HOST_TO_SMC_UL(30);
+	}
+
+	data->smc_state_table.LinkLevelCount =
+		(uint8_t)dpm_table->pcie_speed_table.count;
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+		tonga_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+static int tonga_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+					SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	table->UvdLevelCount = (uint8_t) (mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage.Vddc =
+			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+						mm_table->entries[count].vddc);
+		table->UvdLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
+			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+						mm_table->entries[count].vddgfx) : 0;
+		table->UvdLevel[count].MinVoltage.Vddci =
+			tonga_get_voltage_id(&data->vddci_voltage_table,
+					     mm_table->entries[count].vddc - data->vddc_vddci_delta);
+		table->UvdLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+							  table->UvdLevel[count].VclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				    "can not find divide id for Vclk clock", return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+							  table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				    "can not find divide id for Dclk clock", return result);
+
+		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+		//CONVERT_FROM_HOST_TO_SMC_UL((uint32_t)table->UvdLevel[count].MinVoltage);
+	}
+
+	return result;
+
+}
+
+static int tonga_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	table->VceLevelCount = (uint8_t) (mm_table->count);
+	table->VceBootLevel = 0;
+
+	for (count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency =
+			mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage.Vddc =
+			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+				mm_table->entries[count].vddc);
+		table->VceLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
+			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->VceLevel[count].MinVoltage.Vddci =
+			tonga_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - data->vddc_vddci_delta);
+		table->VceLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+					table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for VCE engine clock", return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	table->AcpLevelCount = (uint8_t) (mm_table->count);
+	table->AcpBootLevel = 0;
+
+	for (count = 0; count < table->AcpLevelCount; count++) {
+		table->AcpLevel[count].Frequency =
+			pptable_info->mm_dep_table->entries[count].aclk;
+		table->AcpLevel[count].MinVoltage.Vddc =
+			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+			mm_table->entries[count].vddc);
+		table->AcpLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
+			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->AcpLevel[count].MinVoltage.Vddci =
+			tonga_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - data->vddc_vddci_delta);
+		table->AcpLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+			table->AcpLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"can not find divide id for engine clock", return result);
+
+		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+	SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t) (mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].Frequency =
+			pptable_info->mm_dep_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage.Vddc =
+			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
+				mm_table->entries[count].vddc);
+		table->SamuLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
+			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->SamuLevel[count].MinVoltage.Vddci =
+			tonga_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - data->vddc_vddci_delta);
+		table->SamuLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+					table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+/**
+ * Populates the SMC MCLK structure using the provided memory clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    memory_clock the memory clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int tonga_calculate_mclk_params(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU72_Discrete_MemoryLevel *mclk,
+		bool strobe_mode,
+		bool dllStateOn
+		)
+{
+	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	uint32_t  dll_cntl = data->clock_registers.vDLL_CNTL;
+	uint32_t  mclk_pwrmgt_cntl = data->clock_registers.vMCLK_PWRMGT_CNTL;
+	uint32_t  mpll_ad_func_cntl = data->clock_registers.vMPLL_AD_FUNC_CNTL;
+	uint32_t  mpll_dq_func_cntl = data->clock_registers.vMPLL_DQ_FUNC_CNTL;
+	uint32_t  mpll_func_cntl = data->clock_registers.vMPLL_FUNC_CNTL;
+	uint32_t  mpll_func_cntl_1 = data->clock_registers.vMPLL_FUNC_CNTL_1;
+	uint32_t  mpll_func_cntl_2 = data->clock_registers.vMPLL_FUNC_CNTL_2;
+	uint32_t  mpll_ss1 = data->clock_registers.vMPLL_SS1;
+	uint32_t  mpll_ss2 = data->clock_registers.vMPLL_SS2;
+
+	pp_atomctrl_memory_clock_param mpll_param;
+	int result;
+
+	result = atomctrl_get_memory_pll_dividers_si(hwmgr,
+				memory_clock, &mpll_param, strobe_mode);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Error retrieving Memory Clock Parameters from VBIOS.", return result);
+
+	/* MPLL_FUNC_CNTL setup*/
+	mpll_func_cntl = PHM_SET_FIELD(mpll_func_cntl, MPLL_FUNC_CNTL, BWCTRL, mpll_param.bw_ctrl);
+
+	/* MPLL_FUNC_CNTL_1 setup*/
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, CLKF, mpll_param.mpll_fb_divider.cl_kf);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, CLKFRAC, mpll_param.mpll_fb_divider.clk_frac);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, VCO_MODE, mpll_param.vco_mode);
+
+	/* MPLL_AD_FUNC_CNTL setup*/
+	mpll_ad_func_cntl = PHM_SET_FIELD(mpll_ad_func_cntl,
+							MPLL_AD_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
+
+	if (data->is_memory_GDDR5) {
+		/* MPLL_DQ_FUNC_CNTL setup*/
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+								MPLL_DQ_FUNC_CNTL, YCLK_SEL, mpll_param.yclk_sel);
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+								MPLL_DQ_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MemorySpreadSpectrumSupport)) {
+		/*
+		 ************************************
+		 Fref = Reference Frequency
+		 NF = Feedback divider ratio
+		 NR = Reference divider ratio
+		 Fnom = Nominal VCO output frequency = Fref * NF / NR
+		 Fs = Spreading Rate
+		 D = Percentage down-spread / 2
+		 Fint = Reference input frequency to PFD = Fref / NR
+		 NS = Spreading rate divider ratio = int(Fint / (2 * Fs))
+		 CLKS = NS - 1 = ISS_STEP_NUM[11:0]
+		 NV = D * Fs / Fnom * 4 * ((Fnom/Fref * NR) ^ 2)
+		 CLKV = 65536 * NV = ISS_STEP_SIZE[25:0]
+		 *************************************
+		 */
+		pp_atomctrl_internal_ss_info ss_info;
+		uint32_t freq_nom;
+		uint32_t tmp;
+		uint32_t reference_clock = atomctrl_get_mpll_reference_clock(hwmgr);
+
+		/* for GDDR5 for all modes and DDR3 */
+		if (1 == mpll_param.qdr)
+			freq_nom = memory_clock * 4 * (1 << mpll_param.mpll_post_divider);
+		else
+			freq_nom = memory_clock * 2 * (1 << mpll_param.mpll_post_divider);
+
+		/* tmp = (freq_nom / reference_clock * reference_divider) ^ 2  Note: S.I. reference_divider = 1*/
+		tmp = (freq_nom / reference_clock);
+		tmp = tmp * tmp;
+
+		if (0 == atomctrl_get_memory_clock_spread_spectrum(hwmgr, freq_nom, &ss_info)) {
+			/* ss_info.speed_spectrum_percentage -- in unit of 0.01% */
+			/* ss.Info.speed_spectrum_rate -- in unit of khz */
+			/* CLKS = reference_clock / (2 * speed_spectrum_rate * reference_divider) * 10 */
+			/*     = reference_clock * 5 / speed_spectrum_rate */
+			uint32_t clks = reference_clock * 5 / ss_info.speed_spectrum_rate;
+
+			/* CLKV = 65536 * speed_spectrum_percentage / 2 * spreadSpecrumRate / freq_nom * 4 / 100000 * ((freq_nom / reference_clock) ^ 2) */
+			/*     = 131 * speed_spectrum_percentage * speed_spectrum_rate / 100 * ((freq_nom / reference_clock) ^ 2) / freq_nom */
+			uint32_t clkv =
+				(uint32_t)((((131 * ss_info.speed_spectrum_percentage *
+							ss_info.speed_spectrum_rate) / 100) * tmp) / freq_nom);
+
+			mpll_ss1 = PHM_SET_FIELD(mpll_ss1, MPLL_SS1, CLKV, clkv);
+			mpll_ss2 = PHM_SET_FIELD(mpll_ss2, MPLL_SS2, CLKS, clks);
+		}
+	}
+
+	/* MCLK_PWRMGT_CNTL setup */
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, DLL_SPEED, mpll_param.dll_speed);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, dllStateOn);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, dllStateOn);
+
+
+	/* Save the result data to outpupt memory level structure */
+	mclk->MclkFrequency   = memory_clock;
+	mclk->MpllFuncCntl    = mpll_func_cntl;
+	mclk->MpllFuncCntl_1  = mpll_func_cntl_1;
+	mclk->MpllFuncCntl_2  = mpll_func_cntl_2;
+	mclk->MpllAdFuncCntl  = mpll_ad_func_cntl;
+	mclk->MpllDqFuncCntl  = mpll_dq_func_cntl;
+	mclk->MclkPwrmgtCntl  = mclk_pwrmgt_cntl;
+	mclk->DllCntl         = dll_cntl;
+	mclk->MpllSs1         = mpll_ss1;
+	mclk->MpllSs2         = mpll_ss2;
+
+	return 0;
+}
+
+static uint8_t tonga_get_mclk_frequency_ratio(uint32_t memory_clock,
+		bool strobe_mode)
+{
+	uint8_t mc_para_index;
+
+	if (strobe_mode) {
+		if (memory_clock < 12500) {
+			mc_para_index = 0x00;
+		} else if (memory_clock > 47500) {
+			mc_para_index = 0x0f;
+		} else {
+			mc_para_index = (uint8_t)((memory_clock - 10000) / 2500);
+		}
+	} else {
+		if (memory_clock < 65000) {
+			mc_para_index = 0x00;
+		} else if (memory_clock > 135000) {
+			mc_para_index = 0x0f;
+		} else {
+			mc_para_index = (uint8_t)((memory_clock - 60000) / 5000);
+		}
+	}
+
+	return mc_para_index;
+}
+
+static uint8_t tonga_get_ddr3_mclk_frequency_ratio(uint32_t memory_clock)
+{
+	uint8_t mc_para_index;
+
+	if (memory_clock < 10000) {
+		mc_para_index = 0;
+	} else if (memory_clock >= 80000) {
+		mc_para_index = 0x0f;
+	} else {
+		mc_para_index = (uint8_t)((memory_clock - 10000) / 5000 + 1);
+	}
+
+	return mc_para_index;
+}
+
+static int tonga_populate_single_memory_level(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU72_Discrete_MemoryLevel *memory_level
+		)
+{
+	uint32_t minMvdd = 0;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+	bool dllStateOn;
+	struct cgs_display_info info = {0};
+
+
+	if (NULL != pptable_info->vdd_dep_on_mclk) {
+		result = tonga_get_dependecy_volt_by_clk(hwmgr,
+			pptable_info->vdd_dep_on_mclk, memory_clock, &memory_level->MinVoltage, &minMvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"can not find MinVddc voltage value from memory VDDC voltage dependency table", return result);
+	}
+
+	if (data->mvdd_control == TONGA_VOLTAGE_CONTROL_NONE) {
+		memory_level->MinMvdd = data->vbios_boot_state.mvdd_bootup_value;
+	} else {
+		memory_level->MinMvdd = minMvdd;
+	}
+	memory_level->EnabledForThrottle = 1;
+	memory_level->EnabledForActivity = 0;
+	memory_level->UpHyst = 0;
+	memory_level->DownHyst = 100;
+	memory_level->VoltageDownHyst = 0;
+
+	/* Indicates maximum activity level for this performance level.*/
+	memory_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	memory_level->StutterEnable = 0;
+	memory_level->StrobeEnable = 0;
+	memory_level->EdcReadEnable = 0;
+	memory_level->EdcWriteEnable = 0;
+	memory_level->RttEnable = 0;
+
+	/* default set to low watermark. Highest level will be set to high later.*/
+	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	data->display_timing.num_existing_displays = info.display_count;
+
+	if ((data->mclk_stutter_mode_threshold != 0) &&
+			(memory_clock <= data->mclk_stutter_mode_threshold) &&
+			(data->is_uvd_enabled == 0)
+#if defined(LINUX)
+			&& (PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE) & 0x1)
+			&& (data->display_timing.num_existing_displays <= 2)
+			&& (data->display_timing.num_existing_displays != 0)
+#endif
+	)
+		memory_level->StutterEnable = 1;
+
+	/* decide strobe mode*/
+	memory_level->StrobeEnable = (data->mclk_strobe_mode_threshold != 0) &&
+		(memory_clock <= data->mclk_strobe_mode_threshold);
+
+	/* decide EDC mode and memory clock ratio*/
+	if (data->is_memory_GDDR5) {
+		memory_level->StrobeRatio = tonga_get_mclk_frequency_ratio(memory_clock,
+					memory_level->StrobeEnable);
+
+		if ((data->mclk_edc_enable_threshold != 0) &&
+				(memory_clock > data->mclk_edc_enable_threshold)) {
+			memory_level->EdcReadEnable = 1;
+		}
+
+		if ((data->mclk_edc_wr_enable_threshold != 0) &&
+				(memory_clock > data->mclk_edc_wr_enable_threshold)) {
+			memory_level->EdcWriteEnable = 1;
+		}
+
+		if (memory_level->StrobeEnable) {
+			if (tonga_get_mclk_frequency_ratio(memory_clock, 1) >=
+					((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC7) >> 16) & 0xf)) {
+				dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+			} else {
+				dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC6) >> 1) & 0x1) ? 1 : 0;
+			}
+
+		} else {
+			dllStateOn = data->dll_defaule_on;
+		}
+	} else {
+		memory_level->StrobeRatio =
+			tonga_get_ddr3_mclk_frequency_ratio(memory_clock);
+		dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+	}
+
+	result = tonga_calculate_mclk_params(hwmgr,
+		memory_clock, memory_level, memory_level->StrobeEnable, dllStateOn);
+
+	if (0 == result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MinMvdd);
+		/* MCLK frequency in units of 10KHz*/
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkFrequency);
+		/* Indicates maximum activity level for this performance level.*/
+		CONVERT_FROM_HOST_TO_SMC_US(memory_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_2);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllAdFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllDqFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkPwrmgtCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->DllCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs2);
+	}
+
+	return result;
+}
+
+/**
+ * Populates the SMC MVDD structure using the provided memory clock.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
+ * @param    voltage     the SMC VOLTAGE structure to be populated
+ */
+int tonga_populate_mvdd_value(struct pp_hwmgr *hwmgr, uint32_t mclk, SMIO_Pattern *smio_pattern)
+{
+	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (TONGA_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < pptable_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= pptable_info->vdd_dep_on_mclk->entries[i].clk) {
+				/* Always round to higher voltage. */
+				smio_pattern->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+
+		PP_ASSERT_WITH_CODE(i < pptable_info->vdd_dep_on_mclk->count,
+			"MVDD Voltage is outside the supported range.", return -1);
+
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+
+static int tonga_populate_smv_acpi_level(struct pp_hwmgr *hwmgr,
+	SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_clock_dividers_vi dividers;
+	SMIO_Pattern voltage_level;
+	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
+	uint32_t dll_cntl          = data->clock_registers.vDLL_CNTL;
+	uint32_t mclk_pwrmgt_cntl  = data->clock_registers.vMCLK_PWRMGT_CNTL;
+
+	/* The ACPI state should not do DPM on DC (or ever).*/
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	table->ACPILevel.MinVoltage = data->smc_state_table.GraphicsLevel[0].MinVoltage;
+
+	/* assign zero for now*/
+	table->ACPILevel.SclkFrequency = atomctrl_get_reference_clock(hwmgr);
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
+		table->ACPILevel.SclkFrequency,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	/* divider ID for required SCLK*/
+	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
+	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+	table->ACPILevel.DeepSleepDivId = 0;
+
+	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
+							CG_SPLL_FUNC_CNTL,   SPLL_PWRON,     0);
+	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
+							CG_SPLL_FUNC_CNTL,   SPLL_RESET,     1);
+	spll_func_cntl_2    = PHM_SET_FIELD(spll_func_cntl_2,
+							CG_SPLL_FUNC_CNTL_2, SCLK_MUX_SEL,   4);
+
+	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
+	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
+	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+
+	/* For various features to be enabled/disabled while this level is active.*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	/* SCLK frequency in units of 10KHz*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	/* table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;*/
+	table->MemoryACPILevel.MinVoltage = data->smc_state_table.MemoryLevel[0].MinVoltage;
+
+	/*  CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);*/
+
+	if (0 == tonga_populate_mvdd_value(hwmgr, 0, &voltage_level))
+		table->MemoryACPILevel.MinMvdd =
+			PP_HOST_TO_SMC_UL(voltage_level.Voltage * VOLTAGE_SCALE);
+	else
+		table->MemoryACPILevel.MinMvdd = 0;
+
+	/* Force reset on DLL*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_RESET, 0x1);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_RESET, 0x1);
+
+	/* Disable DLL in ACPIState*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, 0);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, 0);
+
+	/* Enable DLL bypass signal*/
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK0_BYPASS, 0);
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK1_BYPASS, 0);
+
+	table->MemoryACPILevel.DllCntl            =
+		PP_HOST_TO_SMC_UL(dll_cntl);
+	table->MemoryACPILevel.MclkPwrmgtCntl     =
+		PP_HOST_TO_SMC_UL(mclk_pwrmgt_cntl);
+	table->MemoryACPILevel.MpllAdFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_AD_FUNC_CNTL);
+	table->MemoryACPILevel.MpllDqFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_DQ_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl       =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl_1     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_1);
+	table->MemoryACPILevel.MpllFuncCntl_2     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_2);
+	table->MemoryACPILevel.MpllSs1            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS1);
+	table->MemoryACPILevel.MpllSs2            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS2);
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	/* Indicates maximum activity level for this performance level.*/
+	table->MemoryACPILevel.ActivityLevel = PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	table->MemoryACPILevel.StutterEnable = 0;
+	table->MemoryACPILevel.StrobeEnable = 0;
+	table->MemoryACPILevel.EdcReadEnable = 0;
+	table->MemoryACPILevel.EdcWriteEnable = 0;
+	table->MemoryACPILevel.RttEnable = 0;
+
+	return result;
+}
+
+static int tonga_find_boot_level(struct tonga_single_dpm_table *table, uint32_t value, uint32_t *boot_level)
+{
+	int result = 0;
+	uint32_t i;
+
+	for (i = 0; i < table->count; i++) {
+		if (value == table->dpm_levels[i].value) {
+			*boot_level = i;
+			result = 0;
+		}
+	}
+	return result;
+}
+
+static int tonga_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	table->GraphicsBootLevel  = 0;        /* 0 == DPM[0] (low), etc. */
+	table->MemoryBootLevel    = 0;        /* 0 == DPM[0] (low), etc. */
+
+	/* find boot level from dpm table*/
+	result = tonga_find_boot_level(&(data->dpm_table.sclk_table),
+	data->vbios_boot_state.sclk_bootup_value,
+	(uint32_t *)&(data->smc_state_table.GraphicsBootLevel));
+
+	if (0 != result) {
+		data->smc_state_table.GraphicsBootLevel = 0;
+		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+			in dependency table. Using Graphics DPM level 0!");
+		result = 0;
+	}
+
+	result = tonga_find_boot_level(&(data->dpm_table.mclk_table),
+		data->vbios_boot_state.mclk_bootup_value,
+		(uint32_t *)&(data->smc_state_table.MemoryBootLevel));
+
+	if (0 != result) {
+		data->smc_state_table.MemoryBootLevel = 0;
+		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+			in dependency table. Using Memory DPM level 0!");
+		result = 0;
+	}
+
+	table->BootVoltage.Vddc =
+		tonga_get_voltage_id(&(data->vddc_voltage_table),
+			data->vbios_boot_state.vddc_bootup_value);
+	table->BootVoltage.VddGfx =
+		tonga_get_voltage_id(&(data->vddgfx_voltage_table),
+			data->vbios_boot_state.vddgfx_bootup_value);
+	table->BootVoltage.Vddci =
+		tonga_get_voltage_id(&(data->vddci_voltage_table),
+			data->vbios_boot_state.vddci_bootup_value);
+	table->BootMVdd = data->vbios_boot_state.mvdd_bootup_value;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return result;
+}
+
+
+/**
+ * Calculates the SCLK dividers using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+int tonga_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock, SMU72_Discrete_GraphicsLevel *sclk)
+{
+	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t    reference_clock;
+	uint32_t reference_divider;
+	uint32_t fbdiv;
+	int result;
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, engine_clock,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref.*/
+	reference_clock = atomctrl_get_reference_clock(hwmgr);
+
+	reference_divider = 1 + dividers.uc_pll_ref_div;
+
+	/* low 14 bits is fraction and high 12 bits is divider*/
+	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
+
+	/* SPLL_FUNC_CNTL setup*/
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_REF_DIV, dividers.uc_pll_ref_div);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_PDIV_A,  dividers.uc_pll_post_div);
+
+	/* SPLL_FUNC_CNTL_3 setup*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_FB_DIV, fbdiv);
+
+	/* set to use fractional accumulation*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_DITHEN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
+		pp_atomctrl_internal_ss_info ss_info;
+
+		uint32_t vcoFreq = engine_clock * dividers.uc_pll_post_div;
+		if (0 == atomctrl_get_engine_clock_spread_spectrum(hwmgr, vcoFreq, &ss_info)) {
+			/*
+			* ss_info.speed_spectrum_percentage -- in unit of 0.01%
+			* ss_info.speed_spectrum_rate -- in unit of khz
+			*/
+			/* clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2 */
+			uint32_t clkS = reference_clock * 5 / (reference_divider * ss_info.speed_spectrum_rate);
+
+			/* clkv = 2 * D * fbdiv / NS */
+			uint32_t clkV = 4 * ss_info.speed_spectrum_percentage * fbdiv / (clkS * 10000);
+
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, CLKS, clkS);
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
+			cg_spll_spread_spectrum_2 =
+				PHM_SET_FIELD(cg_spll_spread_spectrum_2, CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clkV);
+		}
+	}
+
+	sclk->SclkFrequency        = engine_clock;
+	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
+	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
+	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
+	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
+	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
+
+	return 0;
+}
+
+/**
+ * Populates single SMC SCLK structure using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int tonga_populate_single_graphic_level(struct pp_hwmgr *hwmgr, uint32_t engine_clock, uint16_t sclk_activity_level_threshold, SMU72_Discrete_GraphicsLevel *graphic_level)
+{
+	int result;
+	uint32_t threshold;
+	uint32_t mvdd;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	result = tonga_calculate_sclk_params(hwmgr, engine_clock, graphic_level);
+
+
+	/* populate graphics levels*/
+	result = tonga_get_dependecy_volt_by_clk(hwmgr,
+		pptable_info->vdd_dep_on_sclk, engine_clock,
+		&graphic_level->MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+		"can not find VDDC voltage value for VDDC	\
+		engine clock dependency table", return result);
+
+	/* SCLK frequency in units of 10KHz*/
+	graphic_level->SclkFrequency = engine_clock;
+
+	/* Indicates maximum activity level for this performance level. 50% for now*/
+	graphic_level->ActivityLevel = sclk_activity_level_threshold;
+
+	graphic_level->CcPwrDynRm = 0;
+	graphic_level->CcPwrDynRm1 = 0;
+	/* this level can be used if activity is high enough.*/
+	graphic_level->EnabledForActivity = 0;
+	/* this level can be used for throttling.*/
+	graphic_level->EnabledForThrottle = 1;
+	graphic_level->UpHyst = 0;
+	graphic_level->DownHyst = 0;
+	graphic_level->VoltageDownHyst = 0;
+	graphic_level->PowerThrottle = 0;
+
+	threshold = engine_clock * data->fast_watemark_threshold / 100;
+/*
+	*get the DAL clock. do it in funture.
+	PECI_GetMinClockSettings(hwmgr->peci, &minClocks);
+	data->display_timing.min_clock_insr = minClocks.engineClockInSR;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
+	{
+		graphic_level->DeepSleepDivId = PhwTonga_GetSleepDividerIdFromClock(hwmgr, engine_clock, minClocks.engineClockInSR);
+	}
+*/
+
+	/* Default to slow, highest DPM level will be set to PPSMC_DISPLAY_WATERMARK_LOW later.*/
+	graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	if (0 == result) {
+		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVoltage);*/
+		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVddcPhases);*/
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(graphic_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl3);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl4);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum2);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm1);
+	}
+
+	return result;
+}
+
+/**
+ * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+ *
+ * @param    hwmgr      the address of the hardware manager
+ */
+static int tonga_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct tonga_dpm_table *dpm_table = &data->dpm_table;
+	phm_ppt_v1_pcie_table *pcie_table = pptable_info->pcie_table;
+	uint8_t pcie_entry_count = (uint8_t) data->dpm_table.pcie_speed_table.count;
+	int result = 0;
+	uint32_t level_array_adress = data->dpm_table_start +
+		offsetof(SMU72_Discrete_DpmTable, GraphicsLevel);
+	uint32_t level_array_size = sizeof(SMU72_Discrete_GraphicsLevel) *
+		SMU72_MAX_LEVELS_GRAPHICS;   /* 64 -> long; 32 -> int*/
+	SMU72_Discrete_GraphicsLevel *levels = data->smc_state_table.GraphicsLevel;
+	uint32_t i, maxEntry;
+	uint8_t highest_pcie_level_enabled = 0, lowest_pcie_level_enabled = 0, mid_pcie_level_enabled = 0, count = 0;
+	PECI_RegistryValue reg_value;
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		result = tonga_populate_single_graphic_level(hwmgr,
+					dpm_table->sclk_table.dpm_levels[i].value,
+					(uint16_t)data->activity_target[i],
+					&(data->smc_state_table.GraphicsLevel[i]));
+
+		if (0 != result)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			data->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
+
+		if (0 == i) {
+			reg_value = 0;
+			if (reg_value != 0)
+				data->smc_state_table.GraphicsLevel[0].UpHyst = (uint8_t)reg_value;
+		}
+
+		if (1 == i) {
+			reg_value = 0;
+			if (reg_value != 0)
+				data->smc_state_table.GraphicsLevel[1].UpHyst = (uint8_t)reg_value;
+		}
+	}
+
+	/* Only enable level 0 for now. */
+	data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
+
+	/* set highest level watermark to high */
+	if (dpm_table->sclk_table.count > 1)
+		data->smc_state_table.GraphicsLevel[dpm_table->sclk_table.count-1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	data->smc_state_table.GraphicsDpmLevelCount =
+		(uint8_t)dpm_table->sclk_table.count;
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+		tonga_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((pcie_entry_count >= 1),
+			"There must be 1 or more PCIE levels defined in PPTable.", return -1);
+		maxEntry = pcie_entry_count - 1; /* for indexing, we need to decrement by 1.*/
+		for (i = 0; i < dpm_table->sclk_table.count; i++) {
+			data->smc_state_table.GraphicsLevel[i].pcieDpmLevel =
+				(uint8_t) ((i < maxEntry) ? i : maxEntry);
+		}
+	} else {
+		if (0 == data->dpm_level_enable_mask.pcie_dpm_enable_mask)
+			printk(KERN_ERR "[ powerplay ] Pcie Dpm Enablemask is 0!");
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<(highest_pcie_level_enabled+1))) != 0)) {
+			highest_pcie_level_enabled++;
+		}
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<lowest_pcie_level_enabled)) == 0)) {
+			lowest_pcie_level_enabled++;
+		}
+
+		while ((count < highest_pcie_level_enabled) &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<(lowest_pcie_level_enabled+1+count))) == 0)) {
+			count++;
+		}
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled+1+count) < highest_pcie_level_enabled ?
+			(lowest_pcie_level_enabled+1+count) : highest_pcie_level_enabled;
+
+
+		/* set pcieDpmLevel to highest_pcie_level_enabled*/
+		for (i = 2; i < dpm_table->sclk_table.count; i++) {
+			data->smc_state_table.GraphicsLevel[i].pcieDpmLevel = highest_pcie_level_enabled;
+		}
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled*/
+		data->smc_state_table.GraphicsLevel[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled*/
+		data->smc_state_table.GraphicsLevel[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change*/
+	result = tonga_copy_bytes_to_smc(hwmgr->smumgr, level_array_adress, (uint8_t *)levels, (uint32_t)level_array_size, data->sram_end);
+
+	if (0 != result)
+		return result;
+
+	return 0;
+}
+
+/**
+ * Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
+ *
+ * @param    hwmgr      the address of the hardware manager
+ */
+
+static int tonga_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_dpm_table *dpm_table = &data->dpm_table;
+	int result;
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t level_array_adress = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, MemoryLevel);
+	uint32_t level_array_size = sizeof(SMU72_Discrete_MemoryLevel) * SMU72_MAX_LEVELS_MEMORY;
+	SMU72_Discrete_MemoryLevel *levels = data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+			"can not populate memory level as memory clock is zero", return -1);
+		result = tonga_populate_single_memory_level(hwmgr, dpm_table->mclk_table.dpm_levels[i].value,
+			&(data->smc_state_table.MemoryLevel[i]));
+		if (0 != result) {
+			return result;
+		}
+	}
+
+	/* Only enable level 0 for now.*/
+	data->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
+
+	/*
+	* in order to prevent MC activity from stutter mode to push DPM up.
+	* the UVD change complements this by putting the MCLK in a higher state
+	* by default such that we are not effected by up threshold or and MCLK DPM latency.
+	*/
+	data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F;
+	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.MemoryLevel[0].ActivityLevel);
+
+	data->smc_state_table.MemoryDpmLevelCount = (uint8_t)dpm_table->mclk_table.count;
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+	/* set highest level watermark to high*/
+	data->smc_state_table.MemoryLevel[dpm_table->mclk_table.count-1].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change*/
+	result = tonga_copy_bytes_to_smc(hwmgr->smumgr,
+		level_array_adress, (uint8_t *)levels, (uint32_t)level_array_size, data->sram_end);
+
+	if (0 != result) {
+		return result;
+	}
+
+	return 0;
+}
+
+struct TONGA_DLL_SPEED_SETTING {
+	uint16_t            Min;                          /* Minimum Data Rate*/
+	uint16_t            Max;                          /* Maximum Data Rate*/
+	uint32_t 			dll_speed;                     /* The desired DLL_SPEED setting*/
+};
+
+static int tonga_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+/* ---------------------------------------- ULV related functions ----------------------------------------------------*/
+
+
+static int tonga_reset_single_dpm_table(
+	struct pp_hwmgr *hwmgr,
+	struct tonga_single_dpm_table *dpm_table,
+	uint32_t count)
+{
+	uint32_t i;
+	if (!(count <= MAX_REGULAR_DPM_NUMBER))
+		printk(KERN_ERR "[ powerplay ] Fatal error, can not set up single DPM \
+			table entries to exceed max number! \n");
+
+	dpm_table->count = count;
+	for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) {
+		dpm_table->dpm_levels[i].enabled = 0;
+	}
+
+	return 0;
+}
+
+static void tonga_setup_pcie_table_entry(
+	struct tonga_single_dpm_table *dpm_table,
+	uint32_t index, uint32_t pcie_gen,
+	uint32_t pcie_lanes)
+{
+	dpm_table->dpm_levels[index].value = pcie_gen;
+	dpm_table->dpm_levels[index].param1 = pcie_lanes;
+	dpm_table->dpm_levels[index].enabled = 1;
+}
+
+static int tonga_setup_default_pcie_tables(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_pcie_table *pcie_table = pptable_info->pcie_table;
+	uint32_t i, maxEntry;
+
+	if (data->use_pcie_performance_levels && !data->use_pcie_power_saving_levels) {
+		data->pcie_gen_power_saving = data->pcie_gen_performance;
+		data->pcie_lane_power_saving = data->pcie_lane_performance;
+	} else if (!data->use_pcie_performance_levels && data->use_pcie_power_saving_levels) {
+		data->pcie_gen_performance = data->pcie_gen_power_saving;
+		data->pcie_lane_performance = data->pcie_lane_power_saving;
+	}
+
+	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.pcie_speed_table, SMU72_MAX_LEVELS_LINK);
+
+	if (pcie_table != NULL) {
+		/*
+		* maxEntry is used to make sure we reserve one PCIE level for boot level (fix for A+A PSPP issue).
+		* If PCIE table from PPTable have ULV entry + 8 entries, then ignore the last entry.
+		*/
+		maxEntry = (SMU72_MAX_LEVELS_LINK < pcie_table->count) ?
+						SMU72_MAX_LEVELS_LINK : pcie_table->count;
+		for (i = 1; i < maxEntry; i++) {
+			tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i-1,
+				get_pcie_gen_support(data->pcie_gen_cap, pcie_table->entries[i].gen_speed),
+				get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		}
+		data->dpm_table.pcie_speed_table.count = maxEntry - 1;
+	} else {
+		/* Hardcode Pcie Table */
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
+			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+		data->dpm_table.pcie_speed_table.count = 6;
+	}
+	/* Populate last level for boot PCIE level, but do not increment count. */
+	tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
+		data->dpm_table.pcie_speed_table.count,
+		get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
+		get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
+
+	return 0;
+
+}
+
+/*
+ * This function is to initalize all DPM state tables for SMU7 based on the dependency table.
+ * Dynamic state patching function will then trim these state tables to the allowed range based
+ * on the power policy or external client requests, such as UVD request, etc.
+ */
+static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i;
+
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_vdd_sclk_table =
+		pptable_info->vdd_dep_on_sclk;
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_vdd_mclk_table =
+		pptable_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL,
+		"SCLK dependency table is missing. This table is mandatory", return -1);
+	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table->count >= 1,
+		"SCLK dependency table has to have is missing. This table is mandatory", return -1);
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
+		"MCLK dependency table is missing. This table is mandatory", return -1);
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table->count >= 1,
+		"VMCLK dependency table has to have is missing. This table is mandatory", return -1);
+
+	/* clear the state table to reset everything to default */
+	memset(&(data->dpm_table), 0x00, sizeof(data->dpm_table));
+	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.sclk_table, SMU72_MAX_LEVELS_GRAPHICS);
+	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.mclk_table, SMU72_MAX_LEVELS_MEMORY);
+	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.VddcTable, SMU72_MAX_LEVELS_VDDC); */
+	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.vdd_gfx_table, SMU72_MAX_LEVELS_VDDGFX);*/
+	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.vdd_ci_table, SMU72_MAX_LEVELS_VDDCI);*/
+	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.mvdd_table, SMU72_MAX_LEVELS_MVDD);*/
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL,
+		"SCLK dependency table is missing. This table is mandatory", return -1);
+	/* Initialize Sclk DPM table based on allow Sclk values*/
+	data->dpm_table.sclk_table.count = 0;
+
+	for (i = 0; i < allowed_vdd_sclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count-1].value !=
+				allowed_vdd_sclk_table->entries[i].clk) {
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
+				allowed_vdd_sclk_table->entries[i].clk;
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; to do */
+			data->dpm_table.sclk_table.count++;
+		}
+	}
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
+		"MCLK dependency table is missing. This table is mandatory", return -1);
+	/* Initialize Mclk DPM table based on allow Mclk values */
+	data->dpm_table.mclk_table.count = 0;
+	for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count-1].value !=
+			allowed_vdd_mclk_table->entries[i].clk) {
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
+				allowed_vdd_mclk_table->entries[i].clk;
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; */
+			data->dpm_table.mclk_table.count++;
+		}
+	}
+
+	/* Initialize Vddc DPM table based on allow Vddc values.  And populate corresponding std values. */
+	for (i = 0; i < allowed_vdd_sclk_table->count; i++) {
+		data->dpm_table.vddc_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].vddc;
+		/* tonga_hwmgr->dpm_table.VddcTable.dpm_levels[i].param1 = stdVoltageTable->entries[i].Leakage; */
+		/* param1 is for corresponding std voltage */
+		data->dpm_table.vddc_table.dpm_levels[i].enabled = 1;
+	}
+	data->dpm_table.vddc_table.count = allowed_vdd_sclk_table->count;
+
+	if (NULL != allowed_vdd_mclk_table) {
+		/* Initialize Vddci DPM table based on allow Mclk values */
+		for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
+			data->dpm_table.vdd_ci_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].vddci;
+			data->dpm_table.vdd_ci_table.dpm_levels[i].enabled = 1;
+			data->dpm_table.mvdd_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].mvdd;
+			data->dpm_table.mvdd_table.dpm_levels[i].enabled = 1;
+		}
+		data->dpm_table.vdd_ci_table.count = allowed_vdd_mclk_table->count;
+		data->dpm_table.mvdd_table.count = allowed_vdd_mclk_table->count;
+	}
+
+	/* setup PCIE gen speed levels*/
+	tonga_setup_default_pcie_tables(hwmgr);
+
+	/* save a copy of the default DPM table*/
+	memcpy(&(data->golden_dpm_table), &(data->dpm_table), sizeof(struct tonga_dpm_table));
+
+	return 0;
+}
+
+int tonga_populate_smc_initial_state(struct pp_hwmgr *hwmgr,
+		const struct tonga_power_state *bootState)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint8_t count, level;
+
+	count = (uint8_t) (pptable_info->vdd_dep_on_sclk->count);
+	for (level = 0; level < count; level++) {
+		if (pptable_info->vdd_dep_on_sclk->entries[level].clk >=
+			bootState->performance_levels[0].engine_clock) {
+			data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t) (pptable_info->vdd_dep_on_mclk->count);
+	for (level = 0; level < count; level++) {
+		if (pptable_info->vdd_dep_on_mclk->entries[level].clk >=
+			bootState->performance_levels[0].memory_clock) {
+			data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Initializes the SMC table and uploads it
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    pInput  the pointer to input data (PowerState)
+ * @return   always 0
+ */
+int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	SMU72_Discrete_DpmTable  *table = &(data->smc_state_table);
+	const phw_tonga_ulv_parm *ulv = &(data->ulv);
+	uint8_t i;
+	PECI_RegistryValue reg_value;
+	pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
+
+	result = tonga_setup_default_dpm_tables(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to setup default DPM tables!", return result;);
+	memset(&(data->smc_state_table), 0x00, sizeof(data->smc_state_table));
+	if (TONGA_VOLTAGE_CONTROL_NONE != data->voltage_control) {
+		tonga_populate_smc_voltage_tables(hwmgr, table);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition)) {
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc)) {
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+	}
+
+	if (data->is_memory_GDDR5) {
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+	}
+
+	i = PHM_READ_FIELD(hwmgr->device, CC_MC_MAX_CHANNEL, NOOFCHAN);
+
+	if (i == 1 || i == 0) {
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_12CHANNEL;
+	}
+
+	if (ulv->ulv_supported && pptable_info->us_ulv_voltage_offset) {
+		PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ULV state!", return result;);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_ULV_PARAMETER, ulv->ch_ulv_parameter);
+	}
+
+	result = tonga_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Link Level!", return result;);
+
+	result = tonga_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Graphics Level!", return result;);
+
+	result = tonga_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Memory Level!", return result;);
+
+	result = tonga_populate_smv_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize ACPI Level!", return result;);
+
+	result = tonga_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize VCE Level!", return result;);
+
+	result = tonga_populate_smc_acp_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize ACP Level!", return result;);
+
+	result = tonga_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize SAMU Level!", return result;);
+
+	/* Since only the initial state is completely set up at this point (the other states are just copies of the boot state) we only */
+	/* need to populate the  ARB settings for the initial state. */
+	result = tonga_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to Write ARB settings for the initial state.", return result;);
+
+	result = tonga_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize UVD Level!", return result;);
+
+	result = tonga_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Boot Level!", return result;);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = tonga_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate Clock Stretcher Data Table!", return result;);
+	}
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+		pptable_info->cac_dtp_table->usTargetOperatingTemp *
+		TONGA_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow =
+		(pptable_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+		TONGA_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable  = 1;
+	table->MemoryInterval  = 1;
+	table->VoltageResponseTime  = 0;
+	table->PhaseResponseTime  = 0;
+	table->MemoryThermThrottleEnable  = 1;
+
+	/*
+	* Cail reads current link status and reports it as cap (we cannot change this due to some previous issues we had)
+	* SMC drops the link status to lowest level after enabling DPM by PowerPlay. After pnp or toggling CF, driver gets reloaded again
+	* but this time Cail reads current link status which was set to low by SMC and reports it as cap to powerplay
+	* To avoid it, we set PCIeBootLinkLevel to highest dpm level
+	*/
+	PP_ASSERT_WITH_CODE((1 <= data->dpm_table.pcie_speed_table.count),
+			"There must be 1 or more PCIE levels defined in PPTable.",
+			return -1);
+
+	table->PCIeBootLinkLevel = (uint8_t) (data->dpm_table.pcie_speed_table.count);
+
+	table->PCIeGenInterval  = 1;
+
+	result = tonga_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to populate VRConfig setting!", return result);
+
+	table->ThermGpio  = 17;
+	table->SclkStepSize = 0x4000;
+
+	reg_value = 0;
+	if ((0 == reg_value) &&
+		(0 == atomctrl_get_pp_assign_pin(hwmgr,
+			VDDC_VRHOT_GPIO_PINID, &gpio_pin_assignment))) {
+		table->VRHotGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot);
+	} else {
+		table->VRHotGpio = TONGA_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot);
+	}
+
+	/* ACDC Switch GPIO */
+	reg_value = 0;
+	if ((0 == reg_value) &&
+		(0 == atomctrl_get_pp_assign_pin(hwmgr,
+			PP_AC_DC_SWITCH_GPIO_PINID, &gpio_pin_assignment))) {
+		table->AcDcGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+	} else {
+		table->AcDcGpio = TONGA_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_Falcon_QuickTransition);
+
+	reg_value = 0;
+	if (1 == reg_value) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_Falcon_QuickTransition);
+	}
+
+	reg_value = 0;
+	if ((0 == reg_value) &&
+		(0 == atomctrl_get_pp_assign_pin(hwmgr,
+			THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin_assignment))) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+
+		table->ThermOutPolarity =
+			(0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
+			(1 << gpio_pin_assignment.uc_gpio_pin_bit_shift))) ? 1:0;
+
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO*/
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot) &&
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CombinePCCWithThermalSignal)){
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+		}
+	} else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	for (i = 0; i < SMU72_MAX_ENTRIES_SMIO; i++) {
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+	}
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = tonga_copy_bytes_to_smc(hwmgr->smumgr, data->dpm_table_start +
+										offsetof(SMU72_Discrete_DpmTable, SystemFlags),
+										(uint8_t *)&(table->SystemFlags),
+										sizeof(SMU72_Discrete_DpmTable)-3 * sizeof(SMU72_PIDController),
+										data->sram_end);
+
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to upload dpm data to SMC memory!", return result;);
+
+	return result;
+}
+
+/* Look up the voltaged based on DAL's requested level. and then send the requested VDDC voltage to SMC*/
+static void tonga_apply_dal_minimum_voltage_request(struct pp_hwmgr *hwmgr)
+{
+	return;
+}
+
+int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
+{
+	PPSMC_Result result;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* Apply minimum voltage based on DAL's request level */
+	tonga_apply_dal_minimum_voltage_request(hwmgr);
+
+	if (0 == data->sclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+		if (0 != tonga_is_dpm_running(hwmgr))
+			printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
+
+		if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			result = smum_send_msg_to_smc_with_parameter(
+								hwmgr->smumgr,
+				(PPSMC_Msg)PPSMC_MSG_SCLKDPM_SetEnabledMask,
+				data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == result),
+				"Set Sclk Dpm enable Mask failed", return -1);
+		}
+	}
+
+	if (0 == data->mclk_dpm_key_disabled) {
+		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
+		if (0 != tonga_is_dpm_running(hwmgr))
+			printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
+
+		if (0 != data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			result = smum_send_msg_to_smc_with_parameter(
+								hwmgr->smumgr,
+				(PPSMC_Msg)PPSMC_MSG_MCLKDPM_SetEnabledMask,
+				data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == result),
+				"Set Mclk Dpm enable Mask failed", return -1);
+		}
+	}
+
+	return 0;
+}
+
+
+int tonga_force_dpm_highest(struct pp_hwmgr *hwmgr)
+{
+	uint32_t level, tmp;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->pcie_dpm_key_disabled) {
+		/* PCIE */
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask != 0) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++ ;
+
+			if (0 != level) {
+				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_pcie(hwmgr, level)),
+					"force highest pcie dpm state failed!", return -1);
+			}
+		}
+	}
+
+	if (0 == data->sclk_dpm_key_disabled) {
+		/* SCLK */
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask != 0) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++ ;
+
+			if (0 != level) {
+				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state(hwmgr, level)),
+					"force highest sclk dpm state failed!", return -1);
+				if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
+					CGS_IND_REG__SMC, TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX) != level)
+					printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
+						Curr_Sclk_Index does not match the level \n");
+
+			}
+		}
+	}
+
+	if (0 == data->mclk_dpm_key_disabled) {
+		/* MCLK */
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask != 0) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++ ;
+
+			if (0 != level) {
+				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_mclk(hwmgr, level)),
+					"force highest mclk dpm state failed!", return -1);
+				if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					TARGET_AND_CURRENT_PROFILE_INDEX, CURR_MCLK_INDEX) != level)
+					printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
+						Curr_Mclk_Index does not match the level \n");
+			}
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Find the MC microcode version and store it in the HwMgr struct
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
+{
+	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
+
+	hwmgr->microcode_version_info.MC = cgs_read_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_DATA);
+
+	return 0;
+}
+
+/**
+ * Initialize Dynamic State Adjustment Rule Settings
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ */
+int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr)
+{
+	uint32_t table_size;
+	struct phm_clock_voltage_dependency_table *table_clk_vlt;
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	hwmgr->dyn_state.mclk_sclk_ratio = 4;
+	hwmgr->dyn_state.sclk_mclk_delta = 15000;      /* 150 MHz */
+	hwmgr->dyn_state.vddc_vddci_delta = 200;       /* 200mV */
+
+	/* initialize vddc_dep_on_dal_pwrl table */
+	table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record);
+	table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table_clk_vlt) {
+		printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
+		return -ENOMEM;
+	} else {
+		table_clk_vlt->count = 4;
+		table_clk_vlt->entries[0].clk = PP_DAL_POWERLEVEL_ULTRALOW;
+		table_clk_vlt->entries[0].v = 0;
+		table_clk_vlt->entries[1].clk = PP_DAL_POWERLEVEL_LOW;
+		table_clk_vlt->entries[1].v = 720;
+		table_clk_vlt->entries[2].clk = PP_DAL_POWERLEVEL_NOMINAL;
+		table_clk_vlt->entries[2].v = 810;
+		table_clk_vlt->entries[3].clk = PP_DAL_POWERLEVEL_PERFORMANCE;
+		table_clk_vlt->entries[3].v = 900;
+		pptable_info->vddc_dep_on_dal_pwrl = table_clk_vlt;
+		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = table_clk_vlt;
+	}
+
+	return 0;
+}
+
+static int tonga_set_private_var_based_on_pptale(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
+		pptable_info->vdd_dep_on_sclk;
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
+		pptable_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
+		"VDD dependency on SCLK table is missing. 	\
+		This table is mandatory", return -1);
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
+		"VDD dependency on SCLK table has to have is missing. 	\
+		This table is mandatory", return -1);
+
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
+		"VDD dependency on MCLK table is missing. 	\
+		This table is mandatory", return -1);
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
+		"VDD dependency on MCLK table has to have is missing.	 \
+		This table is mandatory", return -1);
+
+	data->min_vddc_in_pp_table = (uint16_t)allowed_sclk_vdd_table->entries[0].vddc;
+	data->max_vddc_in_pp_table = (uint16_t)allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
+
+	pptable_info->max_clock_voltage_on_ac.sclk =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
+	pptable_info->max_clock_voltage_on_ac.mclk =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
+	pptable_info->max_clock_voltage_on_ac.vddc =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
+	pptable_info->max_clock_voltage_on_ac.vddci =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
+
+	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk =
+		pptable_info->max_clock_voltage_on_ac.sclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk =
+		pptable_info->max_clock_voltage_on_ac.mclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc =
+		pptable_info->max_clock_voltage_on_ac.vddc;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci =
+		pptable_info->max_clock_voltage_on_ac.vddci;
+
+	return 0;
+}
+
+int tonga_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	int result = 1;
+
+	PP_ASSERT_WITH_CODE (0 == tonga_is_dpm_running(hwmgr),
+		"Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.",
+							return result);
+
+	if (0 == data->pcie_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(
+							     hwmgr->smumgr,
+					PPSMC_MSG_PCIeDPM_UnForceLevel)),
+					   "unforce pcie level failed!",
+								return -1);
+	}
+
+	result = tonga_upload_dpm_level_enable_mask(hwmgr);
+
+	return result;
+}
+
+static uint32_t tonga_get_lowest_enable_level(
+				struct pp_hwmgr *hwmgr, uint32_t level_mask)
+{
+	uint32_t level = 0;
+
+	while (0 == (level_mask & (1 << level)))
+		level++;
+
+	return level;
+}
+
+static int tonga_force_dpm_lowest(struct pp_hwmgr *hwmgr)
+{
+	uint32_t level;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->pcie_dpm_key_disabled) {
+		/* PCIE */
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask != 0) {
+			level = tonga_get_lowest_enable_level(hwmgr,
+							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_pcie(hwmgr, level)),
+					    "force lowest pcie dpm state failed!", return -1);
+		}
+	}
+
+	if (0 == data->sclk_dpm_key_disabled) {
+		/* SCLK */
+		if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			level = tonga_get_lowest_enable_level(hwmgr,
+							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+
+			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state(hwmgr, level)),
+					    "force sclk dpm state failed!", return -1);
+
+			if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
+							 CGS_IND_REG__SMC, TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX) != level)
+				printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index.	\
+				Curr_Sclk_Index does not match the level \n");
+		}
+	}
+
+	if (0 == data->mclk_dpm_key_disabled) {
+		/* MCLK */
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask != 0) {
+			level = tonga_get_lowest_enable_level(hwmgr,
+							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_mclk(hwmgr, level)),
+					    "force lowest mclk dpm state failed!", return -1);
+			if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+							 TARGET_AND_CURRENT_PROFILE_INDEX, CURR_MCLK_INDEX) != level)
+				printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
+						Curr_Mclk_Index does not match the level \n");
+		}
+	}
+
+	return 0;
+}
+
+static int tonga_patch_voltage_dependency_tables_with_lookup_table(struct pp_hwmgr *hwmgr)
+{
+	uint8_t entryId;
+	uint8_t voltageId;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
+	phm_ppt_v1_clock_voltage_dependency_table *mclk_table = pptable_info->vdd_dep_on_mclk;
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
+			voltageId = sclk_table->entries[entryId].vddInd;
+			sclk_table->entries[entryId].vddgfx =
+				pptable_info->vddgfx_lookup_table->entries[voltageId].us_vdd;
+		}
+	} else {
+		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
+			voltageId = sclk_table->entries[entryId].vddInd;
+			sclk_table->entries[entryId].vddc =
+				pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
+		}
+	}
+
+	for (entryId = 0; entryId < mclk_table->count; ++entryId) {
+		voltageId = mclk_table->entries[entryId].vddInd;
+		mclk_table->entries[entryId].vddc =
+			pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
+	}
+
+	for (entryId = 0; entryId < mm_table->count; ++entryId) {
+		voltageId = mm_table->entries[entryId].vddcInd;
+		mm_table->entries[entryId].vddc =
+			pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
+	}
+
+	return 0;
+
+}
+
+static int tonga_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	uint8_t entryId;
+	phm_ppt_v1_voltage_lookup_record v_record;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
+	phm_ppt_v1_clock_voltage_dependency_table *mclk_table = pptable_info->vdd_dep_on_mclk;
+
+	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
+			if (sclk_table->entries[entryId].vdd_offset & (1 << 15))
+				v_record.us_vdd = sclk_table->entries[entryId].vddgfx +
+					sclk_table->entries[entryId].vdd_offset - 0xFFFF;
+			else
+				v_record.us_vdd = sclk_table->entries[entryId].vddgfx +
+					sclk_table->entries[entryId].vdd_offset;
+
+			sclk_table->entries[entryId].vddc =
+				v_record.us_cac_low = v_record.us_cac_mid =
+				v_record.us_cac_high = v_record.us_vdd;
+
+			tonga_add_voltage(hwmgr, pptable_info->vddc_lookup_table, &v_record);
+		}
+
+		for (entryId = 0; entryId < mclk_table->count; ++entryId) {
+			if (mclk_table->entries[entryId].vdd_offset & (1 << 15))
+				v_record.us_vdd = mclk_table->entries[entryId].vddc +
+					mclk_table->entries[entryId].vdd_offset - 0xFFFF;
+			else
+				v_record.us_vdd = mclk_table->entries[entryId].vddc +
+					mclk_table->entries[entryId].vdd_offset;
+
+			mclk_table->entries[entryId].vddgfx = v_record.us_cac_low =
+				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
+			tonga_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
+		}
+	}
+
+	return 0;
+
+}
+
+static int tonga_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t entryId;
+	phm_ppt_v1_voltage_lookup_record v_record;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entryId = 0; entryId < mm_table->count; entryId++) {
+			if (mm_table->entries[entryId].vddgfx_offset & (1 << 15))
+				v_record.us_vdd = mm_table->entries[entryId].vddc +
+					mm_table->entries[entryId].vddgfx_offset - 0xFFFF;
+			else
+				v_record.us_vdd = mm_table->entries[entryId].vddc +
+					mm_table->entries[entryId].vddgfx_offset;
+
+			/* Add the calculated VDDGFX to the VDDGFX lookup table */
+			mm_table->entries[entryId].vddgfx = v_record.us_cac_low =
+				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
+			tonga_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
+		}
+	}
+	return 0;
+}
+
+
+/**
+ * Change virtual leakage voltage to actual value.
+ *
+ * @param     hwmgr  the address of the powerplay hardware manager.
+ * @param     pointer to changing voltage
+ * @param     pointer to leakage table
+ */
+static void tonga_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
+		uint16_t *voltage, phw_tonga_leakage_voltage *pLeakageTable)
+{
+	uint32_t leakage_index;
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
+	for (leakage_index = 0; leakage_index < pLeakageTable->count; leakage_index++) {
+		/* if this voltage matches a leakage voltage ID */
+		/* patch with actual leakage voltage */
+		if (pLeakageTable->leakage_id[leakage_index] == *voltage) {
+			*voltage = pLeakageTable->actual_voltage[leakage_index];
+			break;
+		}
+	}
+
+	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
+		printk(KERN_ERR "[ powerplay ] Voltage value looks like a Leakage ID but it's not patched \n");
+}
+
+/**
+ * Patch voltage lookup table by EVV leakages.
+ *
+ * @param     hwmgr  the address of the powerplay hardware manager.
+ * @param     pointer to voltage lookup table
+ * @param     pointer to leakage table
+ * @return     always 0
+ */
+static int tonga_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *lookup_table,
+		phw_tonga_leakage_voltage *pLeakageTable)
+{
+	uint32_t i;
+
+	for (i = 0; i < lookup_table->count; i++) {
+		tonga_patch_with_vdd_leakage(hwmgr,
+			&lookup_table->entries[i].us_vdd, pLeakageTable);
+	}
+
+	return 0;
+}
+
+static int tonga_patch_clock_voltage_lomits_with_vddc_leakage(struct pp_hwmgr *hwmgr,
+		phw_tonga_leakage_voltage *pLeakageTable, uint16_t *Vddc)
+{
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	tonga_patch_with_vdd_leakage(hwmgr, (uint16_t *)Vddc, pLeakageTable);
+	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
+		pptable_info->max_clock_voltage_on_dc.vddc;
+
+	return 0;
+}
+
+static int tonga_patch_clock_voltage_limits_with_vddgfx_leakage(
+		struct pp_hwmgr *hwmgr, phw_tonga_leakage_voltage *pLeakageTable,
+		uint16_t *Vddgfx)
+{
+	tonga_patch_with_vdd_leakage(hwmgr, (uint16_t *)Vddgfx, pLeakageTable);
+	return 0;
+}
+
+int tonga_sort_lookup_table(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *lookup_table)
+{
+	uint32_t table_size, i, j;
+	phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
+	table_size = lookup_table->count;
+
+	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
+		"Lookup table is empty", return -1);
+
+	/* Sorting voltages */
+	for (i = 0; i < table_size - 1; i++) {
+		for (j = i + 1; j > 0; j--) {
+			if (lookup_table->entries[j].us_vdd < lookup_table->entries[j-1].us_vdd) {
+				tmp_voltage_lookup_record = lookup_table->entries[j-1];
+				lookup_table->entries[j-1] = lookup_table->entries[j];
+				lookup_table->entries[j] = tmp_voltage_lookup_record;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int tonga_complete_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	int tmp_result;
+	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
+		tmp_result = tonga_patch_lookup_table_with_leakage(hwmgr,
+			pptable_info->vddgfx_lookup_table, &(data->vddcgfx_leakage));
+		if (tmp_result != 0)
+			result = tmp_result;
+
+		tmp_result = tonga_patch_clock_voltage_limits_with_vddgfx_leakage(hwmgr,
+			&(data->vddcgfx_leakage), &pptable_info->max_clock_voltage_on_dc.vddgfx);
+		if (tmp_result != 0)
+			result = tmp_result;
+	} else {
+		tmp_result = tonga_patch_lookup_table_with_leakage(hwmgr,
+			pptable_info->vddc_lookup_table, &(data->vddc_leakage));
+		if (tmp_result != 0)
+			result = tmp_result;
+
+		tmp_result = tonga_patch_clock_voltage_lomits_with_vddc_leakage(hwmgr,
+			&(data->vddc_leakage), &pptable_info->max_clock_voltage_on_dc.vddc);
+		if (tmp_result != 0)
+			result = tmp_result;
+	}
+
+	tmp_result = tonga_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
+	if (tmp_result != 0)
+		result = tmp_result;
+
+	tmp_result = tonga_calc_voltage_dependency_tables(hwmgr);
+	if (tmp_result != 0)
+		result = tmp_result;
+
+	tmp_result = tonga_calc_mm_voltage_dependency_table(hwmgr);
+	if (tmp_result != 0)
+		result = tmp_result;
+
+	tmp_result = tonga_sort_lookup_table(hwmgr, pptable_info->vddgfx_lookup_table);
+	if (tmp_result != 0)
+		result = tmp_result;
+
+	tmp_result = tonga_sort_lookup_table(hwmgr, pptable_info->vddc_lookup_table);
+	if (tmp_result != 0)
+		result = tmp_result;
+
+	return result;
+}
+
+int tonga_init_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	data->low_sclk_interrupt_threshold = 0;
+
+	return 0;
+}
+
+int tonga_setup_asic_task(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = tonga_read_clock_registers(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to read clock registers!", result = tmp_result);
+
+	tmp_result = tonga_get_memory_type(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to get memory type!", result = tmp_result);
+
+	tmp_result = tonga_enable_acpi_power_management(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to enable ACPI power management!", result = tmp_result);
+
+	tmp_result = tonga_init_power_gate_state(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to init power gate state!", result = tmp_result);
+
+	tmp_result = tonga_get_mc_microcode_version(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to get MC microcode version!", result = tmp_result);
+
+	tmp_result = tonga_init_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to init sclk threshold!", result = tmp_result);
+
+	return result;
+}
+
+/**
+ * Enable voltage control
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_enable_voltage_control(struct pp_hwmgr *hwmgr)
+{
+	/* enable voltage control */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
+
+	return 0;
+}
+
+/**
+ * Checks if we want to support voltage control
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ */
+bool cf_tonga_voltage_control(const struct pp_hwmgr *hwmgr)
+{
+	const struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	return(TONGA_VOLTAGE_CONTROL_NONE != data->voltage_control);
+}
+
+/*---------------------------MC----------------------------*/
+
+uint8_t tonga_get_memory_modile_index(struct pp_hwmgr *hwmgr)
+{
+	return (uint8_t) (0xFF & (cgs_read_register(hwmgr->device, mmBIOS_SCRATCH_4) >> 16));
+}
+
+bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg)
+{
+	bool result = 1;
+
+	switch (inReg) {
+	case  mmMC_SEQ_RAS_TIMING:
+		*outReg = mmMC_SEQ_RAS_TIMING_LP;
+		break;
+
+	case  mmMC_SEQ_DLL_STBY:
+		*outReg = mmMC_SEQ_DLL_STBY_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD0:
+		*outReg = mmMC_SEQ_G5PDX_CMD0_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD1:
+		*outReg = mmMC_SEQ_G5PDX_CMD1_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CTRL:
+		*outReg = mmMC_SEQ_G5PDX_CTRL_LP;
+		break;
+
+	case mmMC_SEQ_CAS_TIMING:
+		*outReg = mmMC_SEQ_CAS_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING:
+		*outReg = mmMC_SEQ_MISC_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING2:
+		*outReg = mmMC_SEQ_MISC_TIMING2_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CMD:
+		*outReg = mmMC_SEQ_PMG_DVS_CMD_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CTL:
+		*outReg = mmMC_SEQ_PMG_DVS_CTL_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D0:
+		*outReg = mmMC_SEQ_RD_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D1:
+		*outReg = mmMC_SEQ_RD_CTL_D1_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D0:
+		*outReg = mmMC_SEQ_WR_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D1:
+		*outReg = mmMC_SEQ_WR_CTL_D1_LP;
+		break;
+
+	case mmMC_PMG_CMD_EMRS:
+		*outReg = mmMC_SEQ_PMG_CMD_EMRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS:
+		*outReg = mmMC_SEQ_PMG_CMD_MRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS1:
+		*outReg = mmMC_SEQ_PMG_CMD_MRS1_LP;
+		break;
+
+	case mmMC_SEQ_PMG_TIMING:
+		*outReg = mmMC_SEQ_PMG_TIMING_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS2:
+		*outReg = mmMC_SEQ_PMG_CMD_MRS2_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_2:
+		*outReg = mmMC_SEQ_WR_CTL_2_LP;
+		break;
+
+	default:
+		result = 0;
+		break;
+	}
+
+	return result;
+}
+
+int tonga_set_s0_mc_reg_index(phw_tonga_mc_reg_table *table)
+{
+	uint32_t i;
+	uint16_t address;
+
+	for (i = 0; i < table->last; i++) {
+		table->mc_reg_address[i].s0 =
+			tonga_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address)
+			? address : table->mc_reg_address[i].s1;
+	}
+	return 0;
+}
+
+int tonga_copy_vbios_smc_reg_table(const pp_atomctrl_mc_reg_table *table, phw_tonga_mc_reg_table *ni_table)
+{
+	uint8_t i, j;
+
+	PP_ASSERT_WITH_CODE((table->last <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+		"Invalid VramInfo table.", return -1);
+	PP_ASSERT_WITH_CODE((table->num_entries <= MAX_AC_TIMING_ENTRIES),
+		"Invalid VramInfo table.", return -1);
+
+	for (i = 0; i < table->last; i++) {
+		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+	}
+	ni_table->last = table->last;
+
+	for (i = 0; i < table->num_entries; i++) {
+		ni_table->mc_reg_table_entry[i].mclk_max =
+			table->mc_reg_table_entry[i].mclk_max;
+		for (j = 0; j < table->last; j++) {
+			ni_table->mc_reg_table_entry[i].mc_data[j] =
+				table->mc_reg_table_entry[i].mc_data[j];
+		}
+	}
+
+	ni_table->num_entries = table->num_entries;
+
+	return 0;
+}
+
+/**
+ * VBIOS omits some information to reduce size, we need to recover them here.
+ * 1.   when we see mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to  mmMC_PMG_CMD_EMRS /_LP[15:0].
+ *      Bit[15:0] MRS, need to be update mmMC_PMG_CMD_MRS/_LP[15:0]
+ * 2.   when we see mmMC_SEQ_RESERVE_M, bit[15:0] EMRS2, need to be write to mmMC_PMG_CMD_MRS1/_LP[15:0].
+ * 3.   need to set these data for each clock range
+ *
+ * @param    hwmgr the address of the powerplay hardware manager.
+ * @param    table the address of MCRegTable
+ * @return   always 0
+ */
+int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr, phw_tonga_mc_reg_table *table)
+{
+	uint8_t i, j, k;
+	uint32_t temp_reg;
+	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	for (i = 0, j = table->last; i < table->last; i++) {
+		PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+			"Invalid VramInfo table.", return -1);
+		switch (table->mc_reg_address[i].s1) {
+		/*
+		* mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to  mmMC_PMG_CMD_EMRS /_LP[15:0].
+		* Bit[15:0] MRS, need to be update mmMC_PMG_CMD_MRS/_LP[15:0]
+		*/
+		case mmMC_SEQ_MISC1:
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					((temp_reg & 0xffff0000)) |
+					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -1);
+
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+
+				if (!data->is_memory_GDDR5) {
+					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+				}
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -1);
+
+			if (!data->is_memory_GDDR5) {
+				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
+				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
+				for (k = 0; k < table->num_entries; k++) {
+					table->mc_reg_table_entry[k].mc_data[j] =
+						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
+				}
+				j++;
+				PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -1);
+			}
+
+			break;
+
+		case mmMC_SEQ_RESERVE_M:
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -1);
+			break;
+
+		default:
+			break;
+		}
+
+	}
+
+	table->last = j;
+
+	return 0;
+}
+
+int tonga_set_valid_flag(phw_tonga_mc_reg_table *table)
+{
+	uint8_t i, j;
+	for (i = 0; i < table->last; i++) {
+		for (j = 1; j < table->num_entries; j++) {
+			if (table->mc_reg_table_entry[j-1].mc_data[i] !=
+				table->mc_reg_table_entry[j].mc_data[i]) {
+				table->validflag |= (1<<i);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
+int tonga_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_mc_reg_table *table;
+	phw_tonga_mc_reg_table *ni_table = &data->tonga_mc_reg_table;
+	uint8_t module_index = tonga_get_memory_modile_index(hwmgr);
+
+	table = kzalloc(sizeof(pp_atomctrl_mc_reg_table), GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	/* Program additional LP registers that are no longer programmed by VBIOS */
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_DLL_STBY_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_DLL_STBY));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_EMRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS1_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS2_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_2));
+
+	memset(table, 0x00, sizeof(pp_atomctrl_mc_reg_table));
+
+	result = atomctrl_initialize_mc_reg_table(hwmgr, module_index, table);
+
+	if (0 == result)
+		result = tonga_copy_vbios_smc_reg_table(table, ni_table);
+
+	if (0 == result) {
+		tonga_set_s0_mc_reg_index(ni_table);
+		result = tonga_set_mc_special_registers(hwmgr, ni_table);
+	}
+
+	if (0 == result)
+		tonga_set_valid_flag(ni_table);
+
+	kfree(table);
+	return result;
+}
+
+/*
+* Copy one arb setting to another and then switch the active set.
+* arbFreqSrc and arbFreqDest is one of the MC_CG_ARB_FREQ_Fx constants.
+*/
+int tonga_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
+		uint32_t arbFreqSrc, uint32_t arbFreqDest)
+{
+	uint32_t mc_arb_dram_timing;
+	uint32_t mc_arb_dram_timing2;
+	uint32_t burst_time;
+	uint32_t mc_cg_config;
+
+	switch (arbFreqSrc) {
+	case MC_CG_ARB_FREQ_F0:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+		break;
+
+	case MC_CG_ARB_FREQ_F1:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
+		break;
+
+	default:
+		return -1;
+	}
+
+	switch (arbFreqDest) {
+	case MC_CG_ARB_FREQ_F0:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
+		break;
+
+	case MC_CG_ARB_FREQ_F1:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
+		break;
+
+	default:
+		return -1;
+	}
+
+	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
+	mc_cg_config |= 0x0000000F;
+	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
+	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arbFreqDest);
+
+	return 0;
+}
+
+/**
+ * Initial switch from ARB F0->F1
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ * This function is to be called from the SetPowerState table.
+ */
+int tonga_initial_switch_from_arb_f0_to_f1(struct pp_hwmgr *hwmgr)
+{
+	return tonga_copy_and_switch_arb_sets(hwmgr, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+/**
+ * Initialize the ARB DRAM timing table's index field.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_init_arb_table_index(struct pp_hwmgr *hwmgr)
+{
+	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t tmp;
+	int result;
+
+	/*
+	* This is a read-modify-write on the first byte of the ARB table.
+	* The first byte in the SMU72_Discrete_MCArbDramTimingTable structure is the field 'current'.
+	* This solution is ugly, but we never write the whole table only individual fields in it.
+	* In reality this field should not be in that structure but in a soft register.
+	*/
+	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
+				data->arb_table_start, &tmp, data->sram_end);
+
+	if (0 != result)
+		return result;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
+
+	return tonga_write_smc_sram_dword(hwmgr->smumgr,
+			data->arb_table_start,  tmp, data->sram_end);
+}
+
+int tonga_populate_mc_reg_address(struct pp_hwmgr *hwmgr, SMU72_Discrete_MCRegisters *mc_reg_table)
+{
+	const struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < data->tonga_mc_reg_table.last; j++) {
+		if (data->tonga_mc_reg_table.validflag & 1<<j) {
+			PP_ASSERT_WITH_CODE(i < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE,
+				"Index of mc_reg_table->address[] array out of boundary", return -1);
+			mc_reg_table->address[i].s0 =
+				PP_HOST_TO_SMC_US(data->tonga_mc_reg_table.mc_reg_address[j].s0);
+			mc_reg_table->address[i].s1 =
+				PP_HOST_TO_SMC_US(data->tonga_mc_reg_table.mc_reg_address[j].s1);
+			i++;
+		}
+	}
+
+	mc_reg_table->last = (uint8_t)i;
+
+	return 0;
+}
+
+/*convert register values from driver to SMC format */
+void tonga_convert_mc_registers(
+	const phw_tonga_mc_reg_entry * pEntry,
+	SMU72_Discrete_MCRegisterSet *pData,
+	uint32_t numEntries, uint32_t validflag)
+{
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < numEntries; j++) {
+		if (validflag & 1<<j) {
+			pData->value[i] = PP_HOST_TO_SMC_UL(pEntry->mc_data[j]);
+			i++;
+		}
+	}
+}
+
+/* find the entry in the memory range table, then populate the value to SMC's tonga_mc_reg_table */
+int tonga_convert_mc_reg_table_entry_to_smc(
+		struct pp_hwmgr *hwmgr,
+		const uint32_t memory_clock,
+		SMU72_Discrete_MCRegisterSet *mc_reg_table_data
+		)
+{
+	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t i = 0;
+
+	for (i = 0; i < data->tonga_mc_reg_table.num_entries; i++) {
+		if (memory_clock <=
+			data->tonga_mc_reg_table.mc_reg_table_entry[i].mclk_max) {
+			break;
+		}
+	}
+
+	if ((i == data->tonga_mc_reg_table.num_entries) && (i > 0))
+		--i;
+
+	tonga_convert_mc_registers(&data->tonga_mc_reg_table.mc_reg_table_entry[i],
+		mc_reg_table_data, data->tonga_mc_reg_table.last, data->tonga_mc_reg_table.validflag);
+
+	return 0;
+}
+
+int tonga_convert_mc_reg_table_to_smc(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_MCRegisters *mc_reg_table)
+{
+	int result = 0;
+	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	int res;
+	uint32_t i;
+
+	for (i = 0; i < data->dpm_table.mclk_table.count; i++) {
+		res = tonga_convert_mc_reg_table_entry_to_smc(
+				hwmgr,
+				data->dpm_table.mclk_table.dpm_levels[i].value,
+				&mc_reg_table->data[i]
+				);
+
+		if (0 != res)
+			result = res;
+	}
+
+	return result;
+}
+
+int tonga_populate_initial_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	memset(&data->mc_reg_table, 0x00, sizeof(SMU72_Discrete_MCRegisters));
+	result = tonga_populate_mc_reg_address(hwmgr, &(data->mc_reg_table));
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize MCRegTable for the MC register addresses!", return result;);
+
+	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &data->mc_reg_table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize MCRegTable for driver state!", return result;);
+
+	return tonga_copy_bytes_to_smc(hwmgr->smumgr, data->mc_reg_table_start,
+			(uint8_t *)&data->mc_reg_table, sizeof(SMU72_Discrete_MCRegisters), data->sram_end);
+}
+
+/**
+ * Programs static screed detection parameters
+ *
+ * @param   hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_program_static_screen_threshold_parameters(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* Set static screen threshold unit*/
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
+		CGS_IND_REG__SMC, CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
+		data->static_screen_threshold_unit);
+	/* Set static screen threshold*/
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
+		CGS_IND_REG__SMC, CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
+		data->static_screen_threshold);
+
+	return 0;
+}
+
+/**
+ * Setup display gap for glitch free memory clock switching.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_enable_display_gap(struct pp_hwmgr *hwmgr)
+{
+	uint32_t display_gap = cgs_read_ind_register(hwmgr->device,
+							CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
+
+	display_gap = PHM_SET_FIELD(display_gap,
+					CG_DISPLAY_GAP_CNTL, DISP_GAP, DISPLAY_GAP_IGNORE);
+
+	display_gap = PHM_SET_FIELD(display_gap,
+					CG_DISPLAY_GAP_CNTL, DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_DISPLAY_GAP_CNTL, display_gap);
+
+	return 0;
+}
+
+/**
+ * Programs activity state transition voting clients
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_program_voting_clients(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
+
+	/* Clear reset for voting clients before enabling DPM */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+		SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+		SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+		ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
+
+	return 0;
+}
+
+
+int tonga_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = tonga_check_for_dpm_stopped(hwmgr);
+
+	if (cf_tonga_voltage_control(hwmgr)) {
+		tmp_result = tonga_enable_voltage_control(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable voltage control!", result = tmp_result);
+
+		tmp_result = tonga_construct_voltage_tables(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to contruct voltage tables!", result = tmp_result);
+	}
+
+	tmp_result = tonga_initialize_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to initialize MC reg table!", result = tmp_result);
+
+	tmp_result = tonga_program_static_screen_threshold_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to program static screen threshold parameters!", result = tmp_result);
+
+	tmp_result = tonga_enable_display_gap(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to enable display gap!", result = tmp_result);
+
+	tmp_result = tonga_program_voting_clients(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to program voting clients!", result = tmp_result);
+
+	tmp_result = tonga_process_firmware_header(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to process firmware header!", result = tmp_result);
+
+	tmp_result = tonga_initial_switch_from_arb_f0_to_f1(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to initialize switch from ArbF0 to F1!", result = tmp_result);
+
+	tmp_result = tonga_init_smc_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to initialize SMC table!", result = tmp_result);
+
+	tmp_result = tonga_init_arb_table_index(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to initialize ARB table index!", result = tmp_result);
+
+	tmp_result = tonga_populate_initial_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to populate initialize MC Reg table!", result = tmp_result);
+
+	tmp_result = tonga_notify_smc_display_change(hwmgr, false);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to notify no display!", result = tmp_result);
+
+	/* enable SCLK control */
+	tmp_result = tonga_enable_sclk_control(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to enable SCLK control!", result = tmp_result);
+
+	/* enable DPM */
+	tmp_result = tonga_start_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to start DPM!", result = tmp_result);
+
+	return result;
+}
+
+int tonga_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = tonga_check_for_dpm_running(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"SMC is still running!", return 0);
+
+	tmp_result = tonga_stop_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to stop DPM!", result = tmp_result);
+
+	tmp_result = tonga_reset_to_default(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+		"Failed to reset to default!", result = tmp_result);
+
+	return result;
+}
+
+int tonga_reset_asic_tasks(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	result = tonga_set_boot_state(hwmgr);
+	if (0 != result)
+		printk(KERN_ERR "[ powerplay ] Failed to reset asic via set boot state! \n");
+
+	return result;
+}
+
+int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->dyn_state.vddc_dep_on_dal_pwrl) {
+		kfree(hwmgr->dyn_state.vddc_dep_on_dal_pwrl);
+		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL;
+	}
+
+	if (NULL != hwmgr->backend) {
+		kfree(hwmgr->backend);
+		hwmgr->backend = NULL;
+	}
+
+	return 0;
+}
+
+/**
+ * Initializes the Volcanic Islands Hardware Manager
+ *
+ * @param   hwmgr the address of the powerplay hardware manager.
+ * @return   1 if success; otherwise appropriate error code.
+ */
+int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	SMU72_Discrete_DpmTable  *table = NULL;
+	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phw_tonga_ulv_parm *ulv;
+
+	PP_ASSERT_WITH_CODE((NULL != hwmgr),
+		"Invalid Parameter!", return -1;);
+
+	data->dll_defaule_on = 0;
+	data->sram_end = SMC_RAM_END;
+
+	data->activity_target[0] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[1] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[2] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[3] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[4] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[5] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[6] = PPTONGA_TARGETACTIVITY_DFLT;
+	data->activity_target[7] = PPTONGA_TARGETACTIVITY_DFLT;
+
+	data->vddc_vddci_delta = VDDC_VDDCI_DELTA;
+	data->vddc_vddgfx_delta = VDDC_VDDGFX_DELTA;
+	data->mclk_activity_target = PPTONGA_MCLK_TARGETACTIVITY_DFLT;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_DisableVoltageIsland);
+
+	data->sclk_dpm_key_disabled = 0;
+	data->mclk_dpm_key_disabled = 0;
+	data->pcie_dpm_key_disabled = 0;
+	data->pcc_monitor_enabled = 0;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_UnTabledHardwareInterface);
+
+	data->gpio_debug = 0;
+	data->engine_clock_data = 0;
+	data->memory_clock_data = 0;
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_DynamicPatchPowerState);
+
+	/* need to set voltage control types before EVV patching*/
+	data->voltage_control = TONGA_VOLTAGE_CONTROL_NONE;
+	data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_NONE;
+	data->vdd_gfx_control = TONGA_VOLTAGE_CONTROL_NONE;
+	data->mvdd_control = TONGA_VOLTAGE_CONTROL_NONE;
+
+	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) {
+		data->voltage_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDGFX)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+			VOLTAGE_TYPE_VDDGFX, VOLTAGE_OBJ_SVID2)) {
+			data->vdd_gfx_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
+		}
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_NONE == data->vdd_gfx_control) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDGFX);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EnableMVDDControl)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+					VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT)) {
+			data->mvdd_control = TONGA_VOLTAGE_CONTROL_BY_GPIO;
+		}
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_NONE == data->mvdd_control) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EnableMVDDControl);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDCI)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+					VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
+			data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_BY_GPIO;
+		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+						VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
+			data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
+	}
+
+	if (TONGA_VOLTAGE_CONTROL_NONE == data->vdd_ci_control)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_ControlVDDCI);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface);
+
+	if (pptable_info->cac_dtp_table->usClockStretchAmount != 0)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher);
+
+	/* Initializes DPM default values*/
+	tonga_initialize_dpm_defaults(hwmgr);
+
+	/* Get leakage voltage based on leakage ID.*/
+	PP_ASSERT_WITH_CODE((0 == tonga_get_evv_voltage(hwmgr)),
+		"Get EVV Voltage Failed.  Abort Driver loading!", return -1);
+
+	tonga_complete_dependency_tables(hwmgr);
+
+	/* Parse pptable data read from VBIOS*/
+	tonga_set_private_var_based_on_pptale(hwmgr);
+
+	/* ULV Support*/
+	ulv = &(data->ulv);
+	ulv->ulv_supported = 0;
+
+	/* Initalize Dynamic State Adjustment Rule Settings*/
+	result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
+	if (result)
+		printk(KERN_ERR "[ powerplay ] tonga_initializa_dynamic_state_adjustment_rule_settings failed!\n");
+	data->uvd_enabled = 0;
+
+	table = &(data->smc_state_table);
+
+	/*
+	* if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable,
+	* Peak Current Control feature is enabled and we should program PCC HW register
+	*/
+	if (0 == atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
+		uint32_t temp_reg = cgs_read_ind_register(hwmgr->device,
+										CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL);
+
+		switch (gpio_pin_assignment.uc_gpio_pin_bit_shift) {
+		case 0:
+			temp_reg = PHM_SET_FIELD(temp_reg,
+				CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x1);
+			break;
+		case 1:
+			temp_reg = PHM_SET_FIELD(temp_reg,
+				CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x2);
+			break;
+		case 2:
+			temp_reg = PHM_SET_FIELD(temp_reg,
+				CNB_PWRMGT_CNTL, GNB_SLOW, 0x1);
+			break;
+		case 3:
+			temp_reg = PHM_SET_FIELD(temp_reg,
+				CNB_PWRMGT_CNTL, FORCE_NB_PS1, 0x1);
+			break;
+		case 4:
+			temp_reg = PHM_SET_FIELD(temp_reg,
+				CNB_PWRMGT_CNTL, DPM_ENABLED, 0x1);
+			break;
+		default:
+			printk(KERN_ERR "[ powerplay ] Failed to setup PCC HW register!	\
+				Wrong GPIO assigned for VDDC_PCC_GPIO_PINID! \n");
+			break;
+		}
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCNB_PWRMGT_CNTL, temp_reg);
+	}
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_EnableSMU7ThermalManagement);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_SMU7);
+
+	data->vddc_phase_shed_control = 0;
+
+	if (0 == result) {
+		struct cgs_system_info sys_info = {0};
+
+		data->is_tlu_enabled = 0;
+		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
+			TONGA_MAX_HARDWARE_POWERLEVELS;
+		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
+		hwmgr->platform_descriptor.minimumClocksReductionPercentage  = 50;
+
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_gen_cap = 0x30007;
+		else
+			data->pcie_gen_cap = (uint32_t)sys_info.value;
+		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
+			data->pcie_spc_cap = 20;
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_lane_cap = 0x2f0000;
+		else
+			data->pcie_lane_cap = (uint32_t)sys_info.value;
+	} else {
+		/* Ignore return value in here, we are cleaning up a mess. */
+		tonga_hwmgr_backend_fini(hwmgr);
+	}
+
+	return result;
+}
+
+static int tonga_force_dpm_level(struct pp_hwmgr *hwmgr,
+		enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+		ret = tonga_force_dpm_highest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		ret = tonga_force_dpm_lowest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		ret = tonga_unforce_dpm_levels(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	default:
+		break;
+	}
+
+	hwmgr->dpm_level = level;
+	return ret;
+}
+
+static int tonga_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				struct pp_power_state  *prequest_ps,
+			const struct pp_power_state *pcurrent_ps)
+{
+	struct tonga_power_state *tonga_ps =
+				cast_phw_tonga_power_state(&prequest_ps->hardware);
+
+	uint32_t sclk;
+	uint32_t mclk;
+	struct PP_Clocks minimum_clocks = {0};
+	bool disable_mclk_switching;
+	bool disable_mclk_switching_for_frame_lock;
+	struct cgs_display_info info = {0};
+	const struct phm_clock_and_voltage_limits *max_limits;
+	uint32_t i;
+	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	int32_t count;
+	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
+
+	data->battery_state = (PP_StateUILabel_Battery == prequest_ps->classification.ui_label);
+
+	PP_ASSERT_WITH_CODE(tonga_ps->performance_level_count == 2,
+				 "VI should always have 2 performance levels",
+				 );
+
+	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
+			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
+			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
+
+	if (PP_PowerSource_DC == hwmgr->power_source) {
+		for (i = 0; i < tonga_ps->performance_level_count; i++) {
+			if (tonga_ps->performance_levels[i].memory_clock > max_limits->mclk)
+				tonga_ps->performance_levels[i].memory_clock = max_limits->mclk;
+			if (tonga_ps->performance_levels[i].engine_clock > max_limits->sclk)
+				tonga_ps->performance_levels[i].engine_clock = max_limits->sclk;
+		}
+	}
+
+	tonga_ps->vce_clocks.EVCLK = hwmgr->vce_arbiter.evclk;
+	tonga_ps->vce_clocks.ECCLK = hwmgr->vce_arbiter.ecclk;
+
+	tonga_ps->acp_clk = hwmgr->acp_arbiter.acpclk;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
+
+	/* TO DO GetMinClockSettings(hwmgr->pPECI, &minimum_clocks); */
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) {
+
+		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
+		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
+
+		for (count = pptable_info->vdd_dep_on_sclk->count-1; count >= 0; count--) {
+			if (stable_pstate_sclk >= pptable_info->vdd_dep_on_sclk->entries[count].clk) {
+				stable_pstate_sclk = pptable_info->vdd_dep_on_sclk->entries[count].clk;
+				break;
+			}
+		}
+
+		if (count < 0)
+			stable_pstate_sclk = pptable_info->vdd_dep_on_sclk->entries[0].clk;
+
+		stable_pstate_mclk = max_limits->mclk;
+
+		minimum_clocks.engineClock = stable_pstate_sclk;
+		minimum_clocks.memoryClock = stable_pstate_mclk;
+	}
+
+	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
+		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
+
+	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
+		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
+
+	tonga_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
+
+	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <= hwmgr->platform_descriptor.overdriveLimit.engineClock),
+					"Overdrive sclk exceeds limit",
+					hwmgr->gfx_arbiter.sclk_over_drive = hwmgr->platform_descriptor.overdriveLimit.engineClock);
+
+		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
+			tonga_ps->performance_levels[1].engine_clock = hwmgr->gfx_arbiter.sclk_over_drive;
+	}
+
+	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <= hwmgr->platform_descriptor.overdriveLimit.memoryClock),
+			"Overdrive mclk exceeds limit",
+			hwmgr->gfx_arbiter.mclk_over_drive = hwmgr->platform_descriptor.overdriveLimit.memoryClock);
+
+		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
+			tonga_ps->performance_levels[1].memory_clock = hwmgr->gfx_arbiter.mclk_over_drive;
+	}
+
+	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
+				    hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
+
+	disable_mclk_switching = (1 < info.display_count) ||
+				    disable_mclk_switching_for_frame_lock;
+
+	sclk  = tonga_ps->performance_levels[0].engine_clock;
+	mclk  = tonga_ps->performance_levels[0].memory_clock;
+
+	if (disable_mclk_switching)
+		mclk  = tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].memory_clock;
+
+	if (sclk < minimum_clocks.engineClock)
+		sclk = (minimum_clocks.engineClock > max_limits->sclk) ? max_limits->sclk : minimum_clocks.engineClock;
+
+	if (mclk < minimum_clocks.memoryClock)
+		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ? max_limits->mclk : minimum_clocks.memoryClock;
+
+	tonga_ps->performance_levels[0].engine_clock = sclk;
+	tonga_ps->performance_levels[0].memory_clock = mclk;
+
+	tonga_ps->performance_levels[1].engine_clock =
+		(tonga_ps->performance_levels[1].engine_clock >= tonga_ps->performance_levels[0].engine_clock) ?
+			      tonga_ps->performance_levels[1].engine_clock :
+			      tonga_ps->performance_levels[0].engine_clock;
+
+	if (disable_mclk_switching) {
+		if (mclk < tonga_ps->performance_levels[1].memory_clock)
+			mclk = tonga_ps->performance_levels[1].memory_clock;
+
+		tonga_ps->performance_levels[0].memory_clock = mclk;
+		tonga_ps->performance_levels[1].memory_clock = mclk;
+	} else {
+		if (tonga_ps->performance_levels[1].memory_clock < tonga_ps->performance_levels[0].memory_clock)
+			tonga_ps->performance_levels[1].memory_clock = tonga_ps->performance_levels[0].memory_clock;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) {
+		for (i=0; i < tonga_ps->performance_level_count; i++) {
+			tonga_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
+			tonga_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
+			tonga_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
+			tonga_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
+		}
+	}
+
+	return 0;
+}
+
+int tonga_get_power_state_size(struct pp_hwmgr *hwmgr)
+{
+	return sizeof(struct tonga_power_state);
+}
+
+static int tonga_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct tonga_power_state  *tonga_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
+
+	if (low)
+		return tonga_ps->performance_levels[0].memory_clock;
+	else
+		return tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
+}
+
+static int tonga_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct tonga_power_state  *tonga_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
+
+	if (low)
+		return tonga_ps->performance_levels[0].engine_clock;
+	else
+		return tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
+}
+
+static uint16_t tonga_get_current_pcie_speed(
+						   struct pp_hwmgr *hwmgr)
+{
+	uint32_t speed_cntl = 0;
+
+	speed_cntl = cgs_read_ind_register(hwmgr->device,
+						   CGS_IND_REG__PCIE,
+						   ixPCIE_LC_SPEED_CNTL);
+	return((uint16_t)PHM_GET_FIELD(speed_cntl,
+			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
+}
+
+static int tonga_get_current_pcie_lane_number(
+						   struct pp_hwmgr *hwmgr)
+{
+	uint32_t link_width;
+
+	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device,
+							CGS_IND_REG__PCIE,
+						  PCIE_LC_LINK_WIDTH_CNTL,
+							LC_LINK_WIDTH_RD);
+
+	PP_ASSERT_WITH_CODE((7 >= link_width),
+			"Invalid PCIe lane width!", return 0);
+
+	return decode_pcie_lane_width(link_width);
+}
+
+static int tonga_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
+					struct pp_hw_power_state *hw_ps)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_power_state *ps = (struct tonga_power_state *)hw_ps;
+	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
+	uint16_t size;
+	uint8_t frev, crev;
+	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+
+	/* First retrieve the Boot clocks and VDDC from the firmware info table.
+	 * We assume here that fw_info is unchanged if this call fails.
+	 */
+	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
+			hwmgr->device, index,
+			&size, &frev, &crev);
+	if (!fw_info)
+		/* During a test, there is no firmware info table. */
+		return 0;
+
+	/* Patch the state. */
+	data->vbios_boot_state.sclk_bootup_value  = le32_to_cpu(fw_info->ulDefaultEngineClock);
+	data->vbios_boot_state.mclk_bootup_value  = le32_to_cpu(fw_info->ulDefaultMemoryClock);
+	data->vbios_boot_state.mvdd_bootup_value  = le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
+	data->vbios_boot_state.vddc_bootup_value  = le16_to_cpu(fw_info->usBootUpVDDCVoltage);
+	data->vbios_boot_state.vddci_bootup_value = le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
+	data->vbios_boot_state.pcie_gen_bootup_value = tonga_get_current_pcie_speed(hwmgr);
+	data->vbios_boot_state.pcie_lane_bootup_value =
+			(uint16_t)tonga_get_current_pcie_lane_number(hwmgr);
+
+	/* set boot power state */
+	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
+	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
+	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
+	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
+
+	return 0;
+}
+
+static int tonga_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
+		void *state, struct pp_power_state *power_state,
+		void *pp_table, uint32_t classification_flag)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	struct tonga_power_state  *tonga_ps =
+			(struct tonga_power_state *)(&(power_state->hardware));
+
+	struct tonga_performance_level *performance_level;
+
+	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
+
+	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
+			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
+
+	ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table =
+			(ATOM_Tonga_SCLK_Dependency_Table *)
+			(((unsigned long)powerplay_table) +
+			le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
+
+	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
+			(ATOM_Tonga_MCLK_Dependency_Table *)
+			(((unsigned long)powerplay_table) +
+			le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
+
+	/* The following fields are not initialized here: id orderedList allStatesList */
+	power_state->classification.ui_label =
+			(le16_to_cpu(state_entry->usClassification) &
+			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
+			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
+	power_state->classification.flags = classification_flag;
+	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
+
+	power_state->classification.temporary_state = false;
+	power_state->classification.to_be_deleted = false;
+
+	power_state->validation.disallowOnDC =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) & ATOM_Tonga_DISALLOW_ON_DC));
+
+	power_state->pcie.lanes = 0;
+
+	power_state->display.disableFrameModulation = false;
+	power_state->display.limitRefreshrate = false;
+	power_state->display.enableVariBright =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) & ATOM_Tonga_ENABLE_VARIBRIGHT));
+
+	power_state->validation.supportedPowerLevels = 0;
+	power_state->uvd_clocks.VCLK = 0;
+	power_state->uvd_clocks.DCLK = 0;
+	power_state->temperatures.min = 0;
+	power_state->temperatures.max = 0;
+
+	performance_level = &(tonga_ps->performance_levels
+			[tonga_ps->performance_level_count++]);
+
+	PP_ASSERT_WITH_CODE(
+			(tonga_ps->performance_level_count < SMU72_MAX_LEVELS_GRAPHICS),
+			"Performance levels exceeds SMC limit!",
+			return -1);
+
+	PP_ASSERT_WITH_CODE(
+			(tonga_ps->performance_level_count <=
+					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
+			"Performance levels exceeds Driver limit!",
+			return -1);
+
+	/* Performance levels are arranged from low to high. */
+	performance_level->memory_clock =
+				le32_to_cpu(mclk_dep_table->entries[state_entry->ucMemoryClockIndexLow].ulMclk);
+
+	performance_level->engine_clock =
+				le32_to_cpu(sclk_dep_table->entries[state_entry->ucEngineClockIndexLow].ulSclk);
+
+	performance_level->pcie_gen = get_pcie_gen_support(
+							data->pcie_gen_cap,
+					     state_entry->ucPCIEGenLow);
+
+	performance_level->pcie_lane = get_pcie_lane_support(
+						    data->pcie_lane_cap,
+					   state_entry->ucPCIELaneHigh);
+
+	performance_level =
+			&(tonga_ps->performance_levels[tonga_ps->performance_level_count++]);
+
+	performance_level->memory_clock =
+				le32_to_cpu(mclk_dep_table->entries[state_entry->ucMemoryClockIndexHigh].ulMclk);
+
+	performance_level->engine_clock =
+				le32_to_cpu(sclk_dep_table->entries[state_entry->ucEngineClockIndexHigh].ulSclk);
+
+	performance_level->pcie_gen = get_pcie_gen_support(
+							data->pcie_gen_cap,
+					    state_entry->ucPCIEGenHigh);
+
+	performance_level->pcie_lane = get_pcie_lane_support(
+						    data->pcie_lane_cap,
+					   state_entry->ucPCIELaneHigh);
+
+	return 0;
+}
+
+static int tonga_get_pp_table_entry(struct pp_hwmgr *hwmgr,
+		    unsigned long entry_index, struct pp_power_state *ps)
+{
+	int result;
+	struct tonga_power_state *tonga_ps;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
+					   table_info->vdd_dep_on_mclk;
+
+	ps->hardware.magic = PhwTonga_Magic;
+
+	tonga_ps = cast_phw_tonga_power_state(&(ps->hardware));
+
+	result = tonga_get_powerplay_table_entry(hwmgr, entry_index, ps,
+			tonga_get_pp_table_entry_callback_func);
+
+	/* This is the earliest time we have all the dependency table and the VBIOS boot state
+	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
+	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
+	 */
+	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
+		if (dep_mclk_table->entries[0].clk !=
+				data->vbios_boot_state.mclk_bootup_value)
+			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot MCLK level");
+		if (dep_mclk_table->entries[0].vddci !=
+				data->vbios_boot_state.vddci_bootup_value)
+			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot VDDCI level");
+	}
+
+	/* set DC compatible flag if this state supports DC */
+	if (!ps->validation.disallowOnDC)
+		tonga_ps->dc_compatible = true;
+
+	if (ps->classification.flags & PP_StateClassificationFlag_ACPI)
+		data->acpi_pcie_gen = tonga_ps->performance_levels[0].pcie_gen;
+	else if (ps->classification.flags & PP_StateClassificationFlag_Boot) {
+		if (data->bacos.best_match == 0xffff) {
+			/* For V.I. use boot state as base BACO state */
+			data->bacos.best_match = PP_StateClassificationFlag_Boot;
+			data->bacos.performance_level = tonga_ps->performance_levels[0];
+		}
+	}
+
+	tonga_ps->uvd_clocks.VCLK = ps->uvd_clocks.VCLK;
+	tonga_ps->uvd_clocks.DCLK = ps->uvd_clocks.DCLK;
+
+	if (!result) {
+		uint32_t i;
+
+		switch (ps->classification.ui_label) {
+		case PP_StateUILabel_Performance:
+			data->use_pcie_performance_levels = true;
+
+			for (i = 0; i < tonga_ps->performance_level_count; i++) {
+				if (data->pcie_gen_performance.max <
+						tonga_ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.max =
+							tonga_ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_performance.min >
+						tonga_ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.min =
+							tonga_ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_performance.max <
+						tonga_ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.max =
+							tonga_ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_performance.min >
+						tonga_ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.min =
+							tonga_ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		case PP_StateUILabel_Battery:
+			data->use_pcie_power_saving_levels = true;
+
+			for (i = 0; i < tonga_ps->performance_level_count; i++) {
+				if (data->pcie_gen_power_saving.max <
+						tonga_ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.max =
+							tonga_ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_power_saving.min >
+						tonga_ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.min =
+							tonga_ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_power_saving.max <
+						tonga_ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.max =
+							tonga_ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_power_saving.min >
+						tonga_ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.min =
+							tonga_ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+static void
+tonga_print_current_perforce_level(struct pp_hwmgr *hwmgr, struct seq_file *m)
+{
+	uint32_t sclk, mclk, activity_percent;
+	uint32_t offset;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)(PPSMC_MSG_API_GetSclkFrequency));
+
+	sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+
+	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)(PPSMC_MSG_API_GetMclkFrequency));
+
+	mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+	seq_printf(m, "\n [  mclk  ]: %u MHz\n\n [  sclk  ]: %u MHz\n", mclk/100, sclk/100);
+
+
+	offset = data->soft_regs_start + offsetof(SMU72_SoftRegisters, AverageGraphicsActivity);
+	activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
+	activity_percent += 0x80;
+	activity_percent >>= 8;
+
+	seq_printf(m, "\n [GPU load]: %u%%\n\n", activity_percent > 100 ? 100 : activity_percent);
+
+}
+
+static int tonga_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_single_dpm_table *psclk_table = &(data->dpm_table.sclk_table);
+	uint32_t sclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
+	struct tonga_single_dpm_table *pmclk_table = &(data->dpm_table.mclk_table);
+	uint32_t mclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
+	struct PP_Clocks min_clocks = {0};
+	uint32_t i;
+	struct cgs_display_info info = {0};
+
+	data->need_update_smu7_dpm_table = 0;
+
+	for (i = 0; i < psclk_table->count; i++) {
+		if (sclk == psclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= psclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+	else {
+	/* TODO: Check SCLK in DAL's minimum clocks in case DeepSleep divider update is required.*/
+		if(data->display_timing.min_clock_insr != min_clocks.engineClockInSR)
+			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
+	}
+
+	for (i=0; i < pmclk_table->count; i++) {
+		if (mclk == pmclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= pmclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (data->display_timing.num_existing_displays != info.display_count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
+
+	return 0;
+}
+
+static uint16_t tonga_get_maximum_link_speed(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_ps)
+{
+	uint32_t i;
+	uint32_t sclk, max_sclk = 0;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	struct tonga_dpm_table *pdpm_table = &data->dpm_table;
+
+	for (i = 0; i < hw_ps->performance_level_count; i++) {
+		sclk = hw_ps->performance_levels[i].engine_clock;
+		if (max_sclk < sclk)
+			max_sclk = sclk;
+	}
+
+	for (i = 0; i < pdpm_table->sclk_table.count; i++) {
+		if (pdpm_table->sclk_table.dpm_levels[i].value == max_sclk)
+			return (uint16_t) ((i >= pdpm_table->pcie_speed_table.count) ?
+					pdpm_table->pcie_speed_table.dpm_levels[pdpm_table->pcie_speed_table.count-1].value :
+					pdpm_table->pcie_speed_table.dpm_levels[i].value);
+	}
+
+	return 0;
+}
+
+static int tonga_request_link_speed_change_before_state_change(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	const struct tonga_power_state *tonga_nps = cast_const_phw_tonga_power_state(states->pnew_state);
+	const struct tonga_power_state *tonga_cps = cast_const_phw_tonga_power_state(states->pcurrent_state);
+
+	uint16_t target_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_nps);
+	uint16_t current_link_speed;
+
+	if (data->force_pcie_gen == PP_PCIEGenInvalid)
+		current_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_cps);
+	else
+		current_link_speed = data->force_pcie_gen;
+
+	data->force_pcie_gen = PP_PCIEGenInvalid;
+	data->pspp_notify_required = false;
+	if (target_link_speed > current_link_speed) {
+		switch(target_link_speed) {
+		case PP_PCIEGen3:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
+				break;
+			data->force_pcie_gen = PP_PCIEGen2;
+			if (current_link_speed == PP_PCIEGen2)
+				break;
+		case PP_PCIEGen2:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
+				break;
+		default:
+			data->force_pcie_gen = tonga_get_current_pcie_speed(hwmgr);
+			break;
+		}
+	} else {
+		if (target_link_speed < current_link_speed)
+			data->pspp_notify_required = true;
+	}
+
+	return 0;
+}
+
+static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+		PP_ASSERT_WITH_CODE(
+			true == tonga_is_dpm_running(hwmgr),
+			"Trying to freeze SCLK DPM when DPM is disabled",
+			);
+		PP_ASSERT_WITH_CODE(
+			0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					  PPSMC_MSG_SCLKDPM_FreezeLevel),
+			"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
+			return -1);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		 DPMTABLE_OD_UPDATE_MCLK)) {
+		PP_ASSERT_WITH_CODE(true == tonga_is_dpm_running(hwmgr),
+			"Trying to freeze MCLK DPM when DPM is disabled",
+			);
+		PP_ASSERT_WITH_CODE(
+			0 == smum_send_msg_to_smc(hwmgr->smumgr,
+							PPSMC_MSG_MCLKDPM_FreezeLevel),
+			"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
+			return -1);
+	}
+
+	return 0;
+}
+
+static int tonga_populate_and_upload_sclk_mclk_dpm_levels(struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result = 0;
+
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t sclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
+	uint32_t mclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
+	struct tonga_dpm_table *pdpm_table = &data->dpm_table;
+
+	struct tonga_dpm_table *pgolden_dpm_table = &data->golden_dpm_table;
+	uint32_t dpm_count, clock_percent;
+	uint32_t i;
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
+		pdpm_table->sclk_table.dpm_levels[pdpm_table->sclk_table.count-1].value = sclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
+		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
+		/* Need to do calculation based on the golden DPM table
+		 * as the Heatmap GPU Clock axis is also based on the default values
+		 */
+			PP_ASSERT_WITH_CODE(
+				(pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value != 0),
+				"Divide by 0!",
+				return -1);
+			dpm_count = pdpm_table->sclk_table.count < 2 ? 0 : pdpm_table->sclk_table.count-2;
+			for (i = dpm_count; i > 1; i--) {
+				if (sclk > pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value) {
+					clock_percent = ((sclk - pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value)*100) /
+							pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value;
+
+					pdpm_table->sclk_table.dpm_levels[i].value =
+							pgolden_dpm_table->sclk_table.dpm_levels[i].value +
+							(pgolden_dpm_table->sclk_table.dpm_levels[i].value * clock_percent)/100;
+
+				} else if (pgolden_dpm_table->sclk_table.dpm_levels[pdpm_table->sclk_table.count-1].value > sclk) {
+					clock_percent = ((pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value - sclk)*100) /
+								pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value;
+
+					pdpm_table->sclk_table.dpm_levels[i].value =
+							pgolden_dpm_table->sclk_table.dpm_levels[i].value -
+							(pgolden_dpm_table->sclk_table.dpm_levels[i].value * clock_percent)/100;
+				} else
+					pdpm_table->sclk_table.dpm_levels[i].value =
+							pgolden_dpm_table->sclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
+		pdpm_table->mclk_table.dpm_levels[pdpm_table->mclk_table.count-1].value = mclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
+
+			PP_ASSERT_WITH_CODE(
+					(pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value != 0),
+					"Divide by 0!",
+					return -1);
+			dpm_count = pdpm_table->mclk_table.count < 2? 0 : pdpm_table->mclk_table.count-2;
+			for (i = dpm_count; i > 1; i--) {
+				if (mclk > pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value) {
+						clock_percent = ((mclk - pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value)*100) /
+								    pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value;
+
+						pdpm_table->mclk_table.dpm_levels[i].value =
+										pgolden_dpm_table->mclk_table.dpm_levels[i].value +
+										(pgolden_dpm_table->mclk_table.dpm_levels[i].value * clock_percent)/100;
+
+				} else if (pgolden_dpm_table->mclk_table.dpm_levels[pdpm_table->mclk_table.count-1].value > mclk) {
+						clock_percent = ((pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value - mclk)*100) /
+								    pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value;
+
+						pdpm_table->mclk_table.dpm_levels[i].value =
+									pgolden_dpm_table->mclk_table.dpm_levels[i].value -
+									(pgolden_dpm_table->mclk_table.dpm_levels[i].value * clock_percent)/100;
+				} else
+					pdpm_table->mclk_table.dpm_levels[i].value = pgolden_dpm_table->mclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
+		result = tonga_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
+			return result);
+	}
+
+	if (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
+		/*populate MCLK dpm table to SMU7 */
+		result = tonga_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
+
+	return result;
+}
+
+static	int tonga_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
+			  struct tonga_single_dpm_table * pdpm_table,
+			     uint32_t low_limit, uint32_t high_limit)
+{
+	uint32_t i;
+
+	for (i = 0; i < pdpm_table->count; i++) {
+		if ((pdpm_table->dpm_levels[i].value < low_limit) ||
+		    (pdpm_table->dpm_levels[i].value > high_limit))
+			pdpm_table->dpm_levels[i].enabled = false;
+		else
+			pdpm_table->dpm_levels[i].enabled = true;
+	}
+	return 0;
+}
+
+static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_state)
+{
+	int result = 0;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t high_limit_count;
+
+	PP_ASSERT_WITH_CODE((hw_state->performance_level_count >= 1),
+				"power state did not have any performance level",
+				 return -1);
+
+	high_limit_count = (1 == hw_state->performance_level_count) ? 0: 1;
+
+	tonga_trim_single_dpm_states(hwmgr,
+					&(data->dpm_table.sclk_table),
+					hw_state->performance_levels[0].engine_clock,
+					hw_state->performance_levels[high_limit_count].engine_clock);
+
+	tonga_trim_single_dpm_states(hwmgr,
+						&(data->dpm_table.mclk_table),
+						hw_state->performance_levels[0].memory_clock,
+						hw_state->performance_levels[high_limit_count].memory_clock);
+
+	return result;
+}
+
+static int tonga_generate_dpm_level_enable_mask(struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result;
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
+
+	result = tonga_trim_dpm_states(hwmgr, tonga_ps);
+	if (0 != result)
+		return result;
+
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
+	data->last_mclk_dpm_enable_mask = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
+	if (data->uvd_enabled)
+		data->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
+
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
+
+	return 0;
+}
+
+int tonga_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+				  (PPSMC_Msg)PPSMC_MSG_VCEDPM_Enable :
+				  (PPSMC_Msg)PPSMC_MSG_VCEDPM_Disable);
+}
+
+int tonga_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Enable :
+				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Disable);
+}
+
+int tonga_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *ptable_information = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (!bgate) {
+		data->smc_state_table.UvdBootLevel = (uint8_t) (ptable_information->mm_dep_table->count - 1);
+		mm_boot_level_offset = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, UvdBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0x00FFFFFF;
+		mm_boot_level_value |= data->smc_state_table.UvdBootLevel << 24;
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDDPM) ||
+		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_UVDDPM_SetEnabledMask,
+						(uint32_t)(1 << data->smc_state_table.UvdBootLevel));
+	}
+
+	return tonga_enable_disable_uvd_dpm(hwmgr, !bgate);
+}
+
+int tonga_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	const struct tonga_power_state *tonga_nps = cast_const_phw_tonga_power_state(states->pnew_state);
+	const struct tonga_power_state *tonga_cps = cast_const_phw_tonga_power_state(states->pcurrent_state);
+
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (tonga_nps->vce_clocks.EVCLK > 0 && (tonga_cps == NULL || tonga_cps->vce_clocks.EVCLK == 0)) {
+		data->smc_state_table.VceBootLevel = (uint8_t) (pptable_info->mm_dep_table->count - 1);
+
+		mm_boot_level_offset = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, VceBootLevel);
+		mm_boot_level_offset /= 4;
+		mm_boot_level_offset *= 4;
+		mm_boot_level_value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset);
+		mm_boot_level_value &= 0xFF00FFFF;
+		mm_boot_level_value |= data->smc_state_table.VceBootLevel << 16;
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_VCEDPM_SetEnabledMask,
+				(uint32_t)(1 << data->smc_state_table.VceBootLevel));
+
+		tonga_enable_disable_vce_dpm(hwmgr, true);
+	} else if (tonga_nps->vce_clocks.EVCLK == 0 && tonga_cps != NULL && tonga_cps->vce_clocks.EVCLK > 0)
+		tonga_enable_disable_vce_dpm(hwmgr, false);
+
+	return 0;
+}
+
+static int tonga_update_and_upload_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	uint32_t address;
+	int32_t result;
+
+	if (0 == (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
+		return 0;
+
+
+	memset(&data->mc_reg_table, 0, sizeof(SMU72_Discrete_MCRegisters));
+
+	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &(data->mc_reg_table));
+
+	if(result != 0)
+		return result;
+
+
+	address = data->mc_reg_table_start + (uint32_t)offsetof(SMU72_Discrete_MCRegisters, data[0]);
+
+	return  tonga_copy_bytes_to_smc(hwmgr->smumgr, address,
+				 (uint8_t *)&data->mc_reg_table.data[0],
+				sizeof(SMU72_Discrete_MCRegisterSet) * data->dpm_table.mclk_table.count,
+				data->sram_end);
+}
+
+static int tonga_program_memory_timing_parameters_conditionally(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return tonga_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+
+		PP_ASSERT_WITH_CODE(true == tonga_is_dpm_running(hwmgr),
+			"Trying to Unfreeze SCLK DPM when DPM is disabled",
+			);
+		PP_ASSERT_WITH_CODE(
+			 0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
+			return -1);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
+
+		PP_ASSERT_WITH_CODE(
+				true == tonga_is_dpm_running(hwmgr),
+				"Trying to Unfreeze MCLK DPM when DPM is disabled",
+				);
+		PP_ASSERT_WITH_CODE(
+			 0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
+		    return -1);
+	}
+
+	data->need_update_smu7_dpm_table = 0;
+
+	return 0;
+}
+
+static int tonga_notify_link_speed_change_after_state_change(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
+	uint16_t target_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_ps);
+	uint8_t  request;
+
+	if (data->pspp_notify_required  ||
+	    data->pcie_performance_request) {
+		if (target_link_speed == PP_PCIEGen3)
+			request = PCIE_PERF_REQ_GEN3;
+		else if (target_link_speed == PP_PCIEGen2)
+			request = PCIE_PERF_REQ_GEN2;
+		else
+			request = PCIE_PERF_REQ_GEN1;
+
+		if(request == PCIE_PERF_REQ_GEN1 && tonga_get_current_pcie_speed(hwmgr) > 0) {
+			data->pcie_performance_request = false;
+			return 0;
+		}
+
+		if (0 != acpi_pcie_perf_request(hwmgr->device, request, false)) {
+			if (PP_PCIEGen2 == target_link_speed)
+				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
+			else
+				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
+		}
+	}
+
+	data->pcie_performance_request = false;
+	return 0;
+}
+
+static int tonga_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = tonga_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to find DPM states clocks in DPM table!", result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result = tonga_request_link_speed_change_before_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to request link speed change before state change!", result = tmp_result);
+	}
+
+	tmp_result = tonga_freeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to freeze SCLK MCLK DPM!", result = tmp_result);
+
+	tmp_result = tonga_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to populate and upload SCLK MCLK DPM levels!", result = tmp_result);
+
+	tmp_result = tonga_generate_dpm_level_enable_mask(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to generate DPM level enabled mask!", result = tmp_result);
+
+	tmp_result = tonga_update_vce_dpm(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to update VCE DPM!", result = tmp_result);
+
+	tmp_result = tonga_update_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to update SCLK threshold!", result = tmp_result);
+
+	tmp_result = tonga_update_and_upload_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to upload MC reg table!", result = tmp_result);
+
+	tmp_result = tonga_program_memory_timing_parameters_conditionally(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to program memory timing parameters!", result = tmp_result);
+
+	tmp_result = tonga_unfreeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to unfreeze SCLK MCLK DPM!", result = tmp_result);
+
+	tmp_result = tonga_upload_dpm_level_enable_mask(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to upload DPM level enabled mask!", result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result = tonga_notify_link_speed_change_after_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to notify link speed change after state change!", result = tmp_result);
+	}
+
+	return result;
+}
+
+/**
+*  Set maximum target operating fan output PWM
+*
+* @param    pHwMgr:  the address of the powerplay hardware manager.
+* @param    usMaxFanPwm:  max operating fan PWM in percents
+* @return   The response that came from the SMC.
+*/
+static int tonga_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
+{
+	hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm) ? 0 : -1);
+}
+
+int tonga_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
+{
+	uint32_t num_active_displays = 0;
+	struct cgs_display_info info = {0};
+	info.mode_info = NULL;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	num_active_displays = info.display_count;
+
+	if (num_active_displays > 1)  /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */
+		tonga_notify_smc_display_change(hwmgr, false);
+	else
+		tonga_notify_smc_display_change(hwmgr, true);
+
+	return 0;
+}
+
+/**
+* Programs the display gap
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always OK
+*/
+int tonga_program_display_gap(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	uint32_t num_active_displays = 0;
+	uint32_t display_gap = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
+	uint32_t display_gap2;
+	uint32_t pre_vbi_time_in_us;
+	uint32_t frame_time_in_us;
+	uint32_t ref_clock;
+	uint32_t refresh_rate = 0;
+	struct cgs_display_info info = {0};
+	struct cgs_mode_info mode_info;
+
+	info.mode_info = &mode_info;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	num_active_displays = info.display_count;
+
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (num_active_displays > 0)? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL, display_gap);
+
+	ref_clock = mode_info.ref_clock;
+	refresh_rate = mode_info.refresh_rate;
+
+	if(0 == refresh_rate)
+		refresh_rate = 60;
+
+	frame_time_in_us = 1000000 / refresh_rate;
+
+	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
+	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL2, display_gap2);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU72_SoftRegisters, PreVBlankGap), 0x64);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU72_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
+
+	if (num_active_displays == 1)
+		tonga_notify_smc_display_change(hwmgr, true);
+
+	return 0;
+}
+
+int tonga_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
+{
+
+	tonga_program_display_gap(hwmgr);
+
+	/* to do PhwTonga_CacUpdateDisplayConfiguration(pHwMgr); */
+	return 0;
+}
+
+/**
+*  Set maximum target operating fan output RPM
+*
+* @param    pHwMgr:  the address of the powerplay hardware manager.
+* @param    usMaxFanRpm:  max operating fan RPM value.
+* @return   The response that came from the SMC.
+*/
+static int tonga_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
+{
+	hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM = us_max_fan_pwm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanRpmMax, us_max_fan_pwm) ? 0 : -1);
+}
+
+uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr)
+{
+	uint32_t reference_clock;
+	uint32_t tc;
+	uint32_t divide;
+
+	ATOM_FIRMWARE_INFO *fw_info;
+	uint16_t size;
+	uint8_t frev, crev;
+	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+
+	tc = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL_2, MUX_TCLK_TO_XCLK);
+
+	if (tc)
+		return TCLK;
+
+	fw_info = (ATOM_FIRMWARE_INFO *)cgs_atom_get_data_table(hwmgr->device, index,
+						  &size, &frev, &crev);
+
+	if (!fw_info)
+		return 0;
+
+	reference_clock = le16_to_cpu(fw_info->usMinPixelClockPLL_Output);
+
+	divide = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL, XTALIN_DIVIDE);
+
+	if (0 != divide)
+		return reference_clock / 4;
+
+	return reference_clock;
+}
+
+int tonga_dpm_set_interrupt_state(void *private_data,
+					 unsigned src_id, unsigned type,
+					 int enabled)
+{
+	uint32_t cg_thermal_int;
+	struct pp_hwmgr *hwmgr = ((struct pp_eventmgr *)private_data)->hwmgr;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	switch (type) {
+	case AMD_THERMAL_IRQ_LOW_TO_HIGH:
+		if (enabled) {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		} else {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		}
+		break;
+
+	case AMD_THERMAL_IRQ_HIGH_TO_LOW:
+		if (enabled) {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		} else {
+			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
+			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+int tonga_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
+					const void *thermal_interrupt_info)
+{
+	int result;
+	const struct pp_interrupt_registration_info *info =
+			(const struct pp_interrupt_registration_info *)thermal_interrupt_info;
+
+	if (info == NULL)
+		return -EINVAL;
+
+	result = cgs_add_irq_source(hwmgr->device, 230, AMD_THERMAL_IRQ_LAST,
+				tonga_dpm_set_interrupt_state,
+				info->call_back, info->context);
+
+	if (result)
+		return -EINVAL;
+
+	result = cgs_add_irq_source(hwmgr->device, 231, AMD_THERMAL_IRQ_LAST,
+				tonga_dpm_set_interrupt_state,
+				info->call_back, info->context);
+
+	if (result)
+		return -EINVAL;
+
+	return 0;
+}
+
+bool tonga_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	bool is_update_required = false;
+	struct cgs_display_info info = {0,0,NULL};
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (data->display_timing.num_existing_displays != info.display_count)
+		is_update_required = true;
+/* TO DO NEED TO GET DEEP SLEEP CLOCK FROM DAL
+	if (phm_cap_enabled(hwmgr->hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
+		cgs_get_min_clock_settings(hwmgr->device, &min_clocks);
+		if(min_clocks.engineClockInSR != data->display_timing.minClockInSR)
+			is_update_required = true;
+*/
+	return is_update_required;
+}
+
+static inline bool tonga_are_power_levels_equal(const struct tonga_performance_level *pl1,
+							   const struct tonga_performance_level *pl2)
+{
+	return ((pl1->memory_clock == pl2->memory_clock) &&
+		  (pl1->engine_clock == pl2->engine_clock) &&
+		  (pl1->pcie_gen == pl2->pcie_gen) &&
+		  (pl1->pcie_lane == pl2->pcie_lane));
+}
+
+int tonga_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
+{
+	const struct tonga_power_state *psa = cast_const_phw_tonga_power_state(pstate1);
+	const struct tonga_power_state *psb = cast_const_phw_tonga_power_state(pstate2);
+	int i;
+
+	if (equal == NULL || psa == NULL || psb == NULL)
+		return -EINVAL;
+
+	/* If the two states don't even have the same number of performance levels they cannot be the same state. */
+	if (psa->performance_level_count != psb->performance_level_count) {
+		*equal = false;
+		return 0;
+	}
+
+	for (i = 0; i < psa->performance_level_count; i++) {
+		if (!tonga_are_power_levels_equal(&(psa->performance_levels[i]), &(psb->performance_levels[i]))) {
+			/* If we have found even one performance level pair that is different the states are different. */
+			*equal = false;
+			return 0;
+		}
+	}
+
+	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
+	*equal = ((psa->uvd_clocks.VCLK == psb->uvd_clocks.VCLK) && (psa->uvd_clocks.DCLK == psb->uvd_clocks.DCLK));
+	*equal &= ((psa->vce_clocks.EVCLK == psb->vce_clocks.EVCLK) && (psa->vce_clocks.ECCLK == psb->vce_clocks.ECCLK));
+	*equal &= (psa->sclk_threshold == psb->sclk_threshold);
+	*equal &= (psa->acp_clk == psb->acp_clk);
+
+	return 0;
+}
+
+static int tonga_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+{
+	if (mode) {
+		/* stop auto-manage */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl))
+			tonga_fan_ctrl_stop_smc_fan_control(hwmgr);
+		tonga_fan_ctrl_set_static_mode(hwmgr, mode);
+	} else
+		/* restart auto-manage */
+		tonga_fan_ctrl_reset_fan_speed_to_default(hwmgr);
+
+	return 0;
+}
+
+static int tonga_get_fan_control_mode(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr->fan_ctrl_is_in_default_mode)
+		return hwmgr->fan_ctrl_default_mode;
+	else
+		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_FDO_CTRL2, FDO_PWM_MODE);
+}
+
+static const struct pp_hwmgr_func tonga_hwmgr_funcs = {
+	.backend_init = &tonga_hwmgr_backend_init,
+	.backend_fini = &tonga_hwmgr_backend_fini,
+	.asic_setup = &tonga_setup_asic_task,
+	.dynamic_state_management_enable = &tonga_enable_dpm_tasks,
+	.apply_state_adjust_rules = tonga_apply_state_adjust_rules,
+	.force_dpm_level = &tonga_force_dpm_level,
+	.power_state_set = tonga_set_power_state_tasks,
+	.get_power_state_size = tonga_get_power_state_size,
+	.get_mclk = tonga_dpm_get_mclk,
+	.get_sclk = tonga_dpm_get_sclk,
+	.patch_boot_state = tonga_dpm_patch_boot_state,
+	.get_pp_table_entry = tonga_get_pp_table_entry,
+	.get_num_of_pp_table_entries = tonga_get_number_of_powerplay_table_entries,
+	.print_current_perforce_level = tonga_print_current_perforce_level,
+	.powerdown_uvd = tonga_phm_powerdown_uvd,
+	.powergate_uvd = tonga_phm_powergate_uvd,
+	.powergate_vce = tonga_phm_powergate_vce,
+	.disable_clock_power_gating = tonga_phm_disable_clock_power_gating,
+	.notify_smc_display_config_after_ps_adjustment = tonga_notify_smc_display_config_after_ps_adjustment,
+	.display_config_changed = tonga_display_configuration_changed_task,
+	.set_max_fan_pwm_output = tonga_set_max_fan_pwm_output,
+	.set_max_fan_rpm_output = tonga_set_max_fan_rpm_output,
+	.get_temperature = tonga_thermal_get_temperature,
+	.stop_thermal_controller = tonga_thermal_stop_thermal_controller,
+	.get_fan_speed_info = tonga_fan_ctrl_get_fan_speed_info,
+	.get_fan_speed_percent = tonga_fan_ctrl_get_fan_speed_percent,
+	.set_fan_speed_percent = tonga_fan_ctrl_set_fan_speed_percent,
+	.reset_fan_speed_to_default = tonga_fan_ctrl_reset_fan_speed_to_default,
+	.get_fan_speed_rpm = tonga_fan_ctrl_get_fan_speed_rpm,
+	.set_fan_speed_rpm = tonga_fan_ctrl_set_fan_speed_rpm,
+	.uninitialize_thermal_controller = tonga_thermal_ctrl_uninitialize_thermal_controller,
+	.register_internal_thermal_interrupt = tonga_register_internal_thermal_interrupt,
+	.check_smc_update_required_for_display_configuration = tonga_check_smc_update_required_for_display_configuration,
+	.check_states_equal = tonga_check_states_equal,
+	.set_fan_control_mode = tonga_set_fan_control_mode,
+	.get_fan_control_mode = tonga_get_fan_control_mode,
+};
+
+int tonga_hwmgr_init(struct pp_hwmgr *hwmgr)
+{
+	tonga_hwmgr  *data;
+
+	data = kzalloc (sizeof(tonga_hwmgr), GFP_KERNEL);
+	if (data == NULL)
+		return -ENOMEM;
+	memset(data, 0x00, sizeof(tonga_hwmgr));
+
+	hwmgr->backend = data;
+	hwmgr->hwmgr_func = &tonga_hwmgr_funcs;
+	hwmgr->pptable_func = &tonga_pptable_funcs;
+	pp_tonga_thermal_initialize(hwmgr);
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
new file mode 100644
index 000000000000..49168d262ccc
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
@@ -0,0 +1,408 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef TONGA_HWMGR_H
+#define TONGA_HWMGR_H
+
+#include "hwmgr.h"
+#include "smu72_discrete.h"
+#include "ppatomctrl.h"
+#include "ppinterrupt.h"
+#include "tonga_powertune.h"
+
+#define TONGA_MAX_HARDWARE_POWERLEVELS 2
+#define TONGA_DYNCLK_NUMBER_OF_TREND_COEFFICIENTS 15
+
+struct tonga_performance_level {
+	uint32_t	memory_clock;
+	uint32_t	engine_clock;
+	uint16_t    pcie_gen;
+	uint16_t    pcie_lane;
+};
+
+struct _phw_tonga_bacos {
+	uint32_t                          best_match;
+	uint32_t                          baco_flags;
+	struct tonga_performance_level		  performance_level;
+};
+typedef struct _phw_tonga_bacos phw_tonga_bacos;
+
+struct _phw_tonga_uvd_clocks {
+	uint32_t   VCLK;
+	uint32_t   DCLK;
+};
+
+typedef struct _phw_tonga_uvd_clocks phw_tonga_uvd_clocks;
+
+struct _phw_tonga_vce_clocks {
+	uint32_t   EVCLK;
+	uint32_t   ECCLK;
+};
+
+typedef struct _phw_tonga_vce_clocks phw_tonga_vce_clocks;
+
+struct tonga_power_state {
+	uint32_t                    magic;
+	phw_tonga_uvd_clocks        uvd_clocks;
+	phw_tonga_vce_clocks        vce_clocks;
+	uint32_t                    sam_clk;
+	uint32_t                    acp_clk;
+	uint16_t                    performance_level_count;
+	bool                        dc_compatible;
+	uint32_t                    sclk_threshold;
+	struct tonga_performance_level performance_levels[TONGA_MAX_HARDWARE_POWERLEVELS];
+};
+
+struct _phw_tonga_dpm_level {
+	bool  		enabled;
+	uint32_t    value;
+	uint32_t    param1;
+};
+typedef struct _phw_tonga_dpm_level phw_tonga_dpm_level;
+
+#define TONGA_MAX_DEEPSLEEP_DIVIDER_ID 5
+#define MAX_REGULAR_DPM_NUMBER 8
+#define TONGA_MINIMUM_ENGINE_CLOCK 2500
+
+struct tonga_single_dpm_table {
+	uint32_t count;
+	phw_tonga_dpm_level dpm_levels[MAX_REGULAR_DPM_NUMBER];
+};
+
+struct tonga_dpm_table {
+	struct tonga_single_dpm_table  sclk_table;
+	struct tonga_single_dpm_table  mclk_table;
+	struct tonga_single_dpm_table  pcie_speed_table;
+	struct tonga_single_dpm_table  vddc_table;
+	struct tonga_single_dpm_table  vdd_gfx_table;
+	struct tonga_single_dpm_table  vdd_ci_table;
+	struct tonga_single_dpm_table  mvdd_table;
+};
+typedef struct _phw_tonga_dpm_table phw_tonga_dpm_table;
+
+
+struct _phw_tonga_clock_regisiters {
+	uint32_t  vCG_SPLL_FUNC_CNTL;
+	uint32_t  vCG_SPLL_FUNC_CNTL_2;
+	uint32_t  vCG_SPLL_FUNC_CNTL_3;
+	uint32_t  vCG_SPLL_FUNC_CNTL_4;
+	uint32_t  vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t  vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t  vDLL_CNTL;
+	uint32_t  vMCLK_PWRMGT_CNTL;
+	uint32_t  vMPLL_AD_FUNC_CNTL;
+	uint32_t  vMPLL_DQ_FUNC_CNTL;
+	uint32_t  vMPLL_FUNC_CNTL;
+	uint32_t  vMPLL_FUNC_CNTL_1;
+	uint32_t  vMPLL_FUNC_CNTL_2;
+	uint32_t  vMPLL_SS1;
+	uint32_t  vMPLL_SS2;
+};
+typedef struct _phw_tonga_clock_regisiters phw_tonga_clock_registers;
+
+struct _phw_tonga_voltage_smio_registers {
+	uint32_t vs0_vid_lower_smio_cntl;
+};
+typedef struct _phw_tonga_voltage_smio_registers phw_tonga_voltage_smio_registers;
+
+
+struct _phw_tonga_mc_reg_entry {
+	uint32_t mclk_max;
+	uint32_t mc_data[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+typedef struct _phw_tonga_mc_reg_entry phw_tonga_mc_reg_entry;
+
+struct _phw_tonga_mc_reg_table {
+	uint8_t   last;               /* number of registers*/
+	uint8_t   num_entries;        /* number of entries in mc_reg_table_entry used*/
+	uint16_t  validflag;          /* indicate the corresponding register is valid or not. 1: valid, 0: invalid. bit0->address[0], bit1->address[1], etc.*/
+	phw_tonga_mc_reg_entry    mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMU72_Discrete_MCRegisterAddress mc_reg_address[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+typedef struct _phw_tonga_mc_reg_table phw_tonga_mc_reg_table;
+
+#define DISABLE_MC_LOADMICROCODE   1
+#define DISABLE_MC_CFGPROGRAMMING  2
+
+/*Ultra Low Voltage parameter structure */
+struct _phw_tonga_ulv_parm{
+	bool	ulv_supported;
+	uint32_t   ch_ulv_parameter;
+	uint32_t   ulv_volt_change_delay;
+	struct tonga_performance_level   ulv_power_level;
+};
+typedef struct _phw_tonga_ulv_parm phw_tonga_ulv_parm;
+
+#define TONGA_MAX_LEAKAGE_COUNT  8
+
+struct _phw_tonga_leakage_voltage {
+	uint16_t  count;
+	uint16_t  leakage_id[TONGA_MAX_LEAKAGE_COUNT];
+	uint16_t  actual_voltage[TONGA_MAX_LEAKAGE_COUNT];
+};
+typedef struct _phw_tonga_leakage_voltage phw_tonga_leakage_voltage;
+
+struct _phw_tonga_display_timing {
+	uint32_t min_clock_insr;
+	uint32_t num_existing_displays;
+};
+typedef struct _phw_tonga_display_timing phw_tonga_display_timing;
+
+struct _phw_tonga_dpmlevel_enable_mask {
+	uint32_t uvd_dpm_enable_mask;
+	uint32_t vce_dpm_enable_mask;
+	uint32_t acp_dpm_enable_mask;
+	uint32_t samu_dpm_enable_mask;
+	uint32_t sclk_dpm_enable_mask;
+	uint32_t mclk_dpm_enable_mask;
+	uint32_t pcie_dpm_enable_mask;
+};
+typedef struct _phw_tonga_dpmlevel_enable_mask phw_tonga_dpmlevel_enable_mask;
+
+struct _phw_tonga_pcie_perf_range {
+	uint16_t max;
+	uint16_t min;
+};
+typedef struct _phw_tonga_pcie_perf_range phw_tonga_pcie_perf_range;
+
+struct _phw_tonga_vbios_boot_state {
+	uint16_t					mvdd_bootup_value;
+	uint16_t					vddc_bootup_value;
+	uint16_t					vddci_bootup_value;
+	uint16_t					vddgfx_bootup_value;
+	uint32_t					sclk_bootup_value;
+	uint32_t					mclk_bootup_value;
+	uint16_t					pcie_gen_bootup_value;
+	uint16_t					pcie_lane_bootup_value;
+};
+typedef struct _phw_tonga_vbios_boot_state phw_tonga_vbios_boot_state;
+
+#define DPMTABLE_OD_UPDATE_SCLK     0x00000001
+#define DPMTABLE_OD_UPDATE_MCLK     0x00000002
+#define DPMTABLE_UPDATE_SCLK        0x00000004
+#define DPMTABLE_UPDATE_MCLK        0x00000008
+
+/* We need to review which fields are needed. */
+/* This is mostly a copy of the RV7xx/Evergreen structure which is close, but not identical to the N.Islands one. */
+struct tonga_hwmgr {
+	struct tonga_dpm_table               dpm_table;
+	struct tonga_dpm_table               golden_dpm_table;
+
+	uint32_t                           voting_rights_clients0;
+	uint32_t                           voting_rights_clients1;
+	uint32_t                           voting_rights_clients2;
+	uint32_t                           voting_rights_clients3;
+	uint32_t                           voting_rights_clients4;
+	uint32_t                           voting_rights_clients5;
+	uint32_t                           voting_rights_clients6;
+	uint32_t                           voting_rights_clients7;
+	uint32_t                           static_screen_threshold_unit;
+	uint32_t                           static_screen_threshold;
+	uint32_t                           voltage_control;
+	uint32_t                           vdd_gfx_control;
+
+	uint32_t                           vddc_vddci_delta;
+	uint32_t                           vddc_vddgfx_delta;
+
+	struct pp_interrupt_registration_info    internal_high_thermal_interrupt_info;
+	struct pp_interrupt_registration_info    internal_low_thermal_interrupt_info;
+	struct pp_interrupt_registration_info    smc_to_host_interrupt_info;
+	uint32_t                          active_auto_throttle_sources;
+
+	struct pp_interrupt_registration_info    external_throttle_interrupt;
+	irq_handler_func_t             external_throttle_callback;
+	void                             *external_throttle_context;
+
+	struct pp_interrupt_registration_info    ctf_interrupt_info;
+	irq_handler_func_t             ctf_callback;
+	void                             *ctf_context;
+
+	phw_tonga_clock_registers      	  clock_registers;
+	phw_tonga_voltage_smio_registers  voltage_smio_registers;
+
+	bool                         	is_memory_GDDR5;
+	uint16_t                          acpi_vddc;
+	bool                         	pspp_notify_required;        /* Flag to indicate if PSPP notification to SBIOS is required */
+	uint16_t                          force_pcie_gen;            /* The forced PCI-E speed if not 0xffff */
+	uint16_t                          acpi_pcie_gen;             /* The PCI-E speed at ACPI time */
+	uint32_t                           pcie_gen_cap;             /* The PCI-E speed capabilities bitmap from CAIL */
+	uint32_t                           pcie_lane_cap;            /* The PCI-E lane capabilities bitmap from CAIL */
+	uint32_t                           pcie_spc_cap;             /* Symbol Per Clock Capabilities from registry */
+	phw_tonga_leakage_voltage       	vddc_leakage;            /* The Leakage VDDC supported (based on leakage ID).*/
+	phw_tonga_leakage_voltage       	vddcgfx_leakage;         /* The Leakage VDDC supported (based on leakage ID). */
+	phw_tonga_leakage_voltage       	vddci_leakage;           /* The Leakage VDDCI supported (based on leakage ID). */
+
+	uint32_t                           mvdd_control;
+	uint32_t                           vddc_mask_low;
+	uint32_t                           mvdd_mask_low;
+	uint16_t                          max_vddc_in_pp_table;        /* the maximum VDDC value in the powerplay table*/
+	uint16_t                          min_vddc_in_pp_table;
+	uint16_t                          max_vddci_in_pp_table;       /* the maximum VDDCI value in the powerplay table */
+	uint16_t                          min_vddci_in_pp_table;
+	uint32_t                           mclk_strobe_mode_threshold;
+	uint32_t                           mclk_stutter_mode_threshold;
+	uint32_t                           mclk_edc_enable_threshold;
+	uint32_t                           mclk_edc_wr_enable_threshold;
+	bool                         	is_uvd_enabled;
+	bool                         	is_xdma_enabled;
+	phw_tonga_vbios_boot_state      vbios_boot_state;
+
+	bool                         battery_state;
+	bool                         is_tlu_enabled;
+	bool                         pcie_performance_request;
+
+	/* -------------- SMC SRAM Address of firmware header tables ----------------*/
+	uint32_t                           sram_end;                 /* The first address after the SMC SRAM. */
+	uint32_t                           dpm_table_start;          /* The start of the dpm table in the SMC SRAM. */
+	uint32_t                           soft_regs_start;          /* The start of the soft registers in the SMC SRAM. */
+	uint32_t                           mc_reg_table_start;       /* The start of the mc register table in the SMC SRAM. */
+	uint32_t                           fan_table_start;          /* The start of the fan table in the SMC SRAM. */
+	uint32_t                           arb_table_start;          /* The start of the ARB setting table in the SMC SRAM. */
+	SMU72_Discrete_DpmTable         smc_state_table;             /* The carbon copy of the SMC state table. */
+	SMU72_Discrete_MCRegisters      mc_reg_table;
+	SMU72_Discrete_Ulv              ulv_setting;                 /* The carbon copy of ULV setting. */
+	/* -------------- Stuff originally coming from Evergreen --------------------*/
+	phw_tonga_mc_reg_table			tonga_mc_reg_table;
+	uint32_t                         vdd_ci_control;
+	pp_atomctrl_voltage_table        vddc_voltage_table;
+	pp_atomctrl_voltage_table        vddci_voltage_table;
+	pp_atomctrl_voltage_table        vddgfx_voltage_table;
+	pp_atomctrl_voltage_table        mvdd_voltage_table;
+
+	uint32_t                           mgcg_cgtt_local2;
+	uint32_t                           mgcg_cgtt_local3;
+	uint32_t                           gpio_debug;
+	uint32_t							mc_micro_code_feature;
+	uint32_t							highest_mclk;
+	uint16_t                          acpi_vdd_ci;
+	uint8_t                           mvdd_high_index;
+	uint8_t                           mvdd_low_index;
+	bool                         dll_defaule_on;
+	bool                         performance_request_registered;
+
+	/* ----------------- Low Power Features ---------------------*/
+	phw_tonga_bacos					bacos;
+	phw_tonga_ulv_parm              ulv;
+	/* ----------------- CAC Stuff ---------------------*/
+	uint32_t					cac_table_start;
+	bool                         cac_configuration_required;    /* TRUE if PP_CACConfigurationRequired == 1 */
+	bool                         driver_calculate_cac_leakage;  /* TRUE if PP_DriverCalculateCACLeakage == 1 */
+	bool                         cac_enabled;
+	/* ----------------- DPM2 Parameters ---------------------*/
+	uint32_t					power_containment_features;
+	bool                         enable_bapm_feature;
+	bool                         enable_tdc_limit_feature;
+	bool                         enable_pkg_pwr_tracking_feature;
+	bool                         disable_uvd_power_tune_feature;
+	phw_tonga_pt_defaults           *power_tune_defaults;
+	SMU72_Discrete_PmFuses           power_tune_table;
+	uint32_t                           ul_dte_tj_offset;             /* Fudge factor in DPM table to correct HW DTE errors */
+	uint32_t                           fast_watemark_threshold;      /* use fast watermark if clock is equal or above this. In percentage of the target high sclk. */
+
+	/* ----------------- Phase Shedding ---------------------*/
+	bool                         vddc_phase_shed_control;
+	/* --------------------- DI/DT --------------------------*/
+	phw_tonga_display_timing       display_timing;
+	/* --------- ReadRegistry data for memory and engine clock margins ---- */
+	uint32_t                           engine_clock_data;
+	uint32_t                           memory_clock_data;
+	/* -------- Thermal Temperature Setting --------------*/
+	phw_tonga_dpmlevel_enable_mask     dpm_level_enable_mask;
+	uint32_t                           need_update_smu7_dpm_table;
+	uint32_t                           sclk_dpm_key_disabled;
+	uint32_t                           mclk_dpm_key_disabled;
+	uint32_t                           pcie_dpm_key_disabled;
+	uint32_t                           min_engine_clocks; /* used to store the previous dal min sclock */
+	phw_tonga_pcie_perf_range       pcie_gen_performance;
+	phw_tonga_pcie_perf_range       pcie_lane_performance;
+	phw_tonga_pcie_perf_range       pcie_gen_power_saving;
+	phw_tonga_pcie_perf_range       pcie_lane_power_saving;
+	bool                            use_pcie_performance_levels;
+	bool                            use_pcie_power_saving_levels;
+	uint32_t                           activity_target[SMU72_MAX_LEVELS_GRAPHICS]; /* percentage value from 0-100, default 50 */
+	uint32_t                           mclk_activity_target;
+	uint32_t                           low_sclk_interrupt_threshold;
+	uint32_t                           last_mclk_dpm_enable_mask;
+	bool								uvd_enabled;
+	uint32_t                           pcc_monitor_enabled;
+
+	/* --------- Power Gating States ------------*/
+	bool                           uvd_power_gated;  /* 1: gated, 0:not gated */
+	bool                           vce_power_gated;  /* 1: gated, 0:not gated */
+	bool                           samu_power_gated; /* 1: gated, 0:not gated */
+	bool                           acp_power_gated;  /* 1: gated, 0:not gated */
+	bool                           pg_acp_init;
+
+};
+
+typedef struct tonga_hwmgr tonga_hwmgr;
+
+#define TONGA_DPM2_NEAR_TDP_DEC          10
+#define TONGA_DPM2_ABOVE_SAFE_INC        5
+#define TONGA_DPM2_BELOW_SAFE_INC        20
+
+#define TONGA_DPM2_LTA_WINDOW_SIZE       7  /* Log2 of the LTA window size (l2numWin_TDP). Eg. If LTA windows size is 128, then this value should be Log2(128) = 7. */
+
+#define TONGA_DPM2_LTS_TRUNCATE          0
+
+#define TONGA_DPM2_TDP_SAFE_LIMIT_PERCENT            80  /* Maximum 100 */
+
+#define TONGA_DPM2_MAXPS_PERCENT_H                   90  /* Maximum 0xFF */
+#define TONGA_DPM2_MAXPS_PERCENT_M                   90  /* Maximum 0xFF */
+
+#define TONGA_DPM2_PWREFFICIENCYRATIO_MARGIN         50
+
+#define TONGA_DPM2_SQ_RAMP_MAX_POWER                 0x3FFF
+#define TONGA_DPM2_SQ_RAMP_MIN_POWER                 0x12
+#define TONGA_DPM2_SQ_RAMP_MAX_POWER_DELTA           0x15
+#define TONGA_DPM2_SQ_RAMP_SHORT_TERM_INTERVAL_SIZE  0x1E
+#define TONGA_DPM2_SQ_RAMP_LONG_TERM_INTERVAL_RATIO  0xF
+
+#define TONGA_VOLTAGE_CONTROL_NONE                   0x0
+#define TONGA_VOLTAGE_CONTROL_BY_GPIO                0x1
+#define TONGA_VOLTAGE_CONTROL_BY_SVID2               0x2
+#define TONGA_VOLTAGE_CONTROL_MERGED                 0x3
+
+#define TONGA_Q88_FORMAT_CONVERSION_UNIT             256 /*To convert to Q8.8 format for firmware */
+
+#define TONGA_UNUSED_GPIO_PIN                        0x7F
+
+#define PP_HOST_TO_SMC_UL(X) cpu_to_be32(X)
+#define PP_SMC_TO_HOST_UL(X) be32_to_cpu(X)
+
+#define PP_HOST_TO_SMC_US(X) cpu_to_be16(X)
+#define PP_SMC_TO_HOST_US(X) be16_to_cpu(X)
+
+#define CONVERT_FROM_HOST_TO_SMC_UL(X) ((X) = PP_HOST_TO_SMC_UL(X))
+#define CONVERT_FROM_SMC_TO_HOST_UL(X) ((X) = PP_SMC_TO_HOST_UL(X))
+
+#define CONVERT_FROM_HOST_TO_SMC_US(X) ((X) = PP_HOST_TO_SMC_US(X))
+
+int tonga_hwmgr_init(struct pp_hwmgr *hwmgr);
+int tonga_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input);
+int tonga_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+int tonga_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable);
+int tonga_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
+uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_powertune.h
new file mode 100644
index 000000000000..8e6670b3cb67
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_powertune.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef TONGA_POWERTUNE_H
+#define TONGA_POWERTUNE_H
+
+enum _phw_tonga_ptc_config_reg_type {
+	TONGA_CONFIGREG_MMR = 0,
+	TONGA_CONFIGREG_SMC_IND,
+	TONGA_CONFIGREG_DIDT_IND,
+	TONGA_CONFIGREG_CACHE,
+
+	TONGA_CONFIGREG_MAX
+};
+typedef enum _phw_tonga_ptc_config_reg_type phw_tonga_ptc_config_reg_type;
+
+/* PowerContainment Features */
+#define POWERCONTAINMENT_FEATURE_BAPM            0x00000001
+#define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
+#define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
+
+struct _phw_tonga_pt_config_reg {
+	uint32_t                           Offset;
+	uint32_t                           Mask;
+	uint32_t                           Shift;
+	uint32_t                           Value;
+	phw_tonga_ptc_config_reg_type     Type;
+};
+typedef struct _phw_tonga_pt_config_reg phw_tonga_pt_config_reg;
+
+struct _phw_tonga_pt_defaults {
+	uint8_t   svi_load_line_en;
+	uint8_t   svi_load_line_vddC;
+	uint8_t   tdc_vddc_throttle_release_limit_perc;
+	uint8_t   tdc_mawt;
+	uint8_t   tdc_waterfall_ctl;
+	uint8_t   dte_ambient_temp_base;
+	uint32_t  display_cac;
+	uint32_t  bamp_temp_gradient;
+	uint16_t  bapmti_r[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS];
+	uint16_t  bapmti_rc[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS];
+};
+typedef struct _phw_tonga_pt_defaults phw_tonga_pt_defaults;
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_pptable.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_pptable.h
new file mode 100644
index 000000000000..9a4456e6521b
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_pptable.h
@@ -0,0 +1,406 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef TONGA_PPTABLE_H
+#define TONGA_PPTABLE_H
+
+/** \file
+ * This is a PowerPlay table header file
+ */
+#pragma pack(push, 1)
+
+#include "hwmgr.h"
+
+#define ATOM_TONGA_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK 0x0f
+#define ATOM_TONGA_PP_FANPARAMETERS_NOFAN                                 0x80    /* No fan is connected to this controller. */
+
+#define ATOM_TONGA_PP_THERMALCONTROLLER_NONE      0
+#define ATOM_TONGA_PP_THERMALCONTROLLER_LM96163   17
+#define ATOM_TONGA_PP_THERMALCONTROLLER_TONGA     21
+#define ATOM_TONGA_PP_THERMALCONTROLLER_FIJI      22
+
+/*
+ * Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal.
+ * We probably should reserve the bit 0x80 for this use.
+ * To keep the number of these types low we should also use the same code for all ASICs (i.e. do not distinguish RV6xx and RV7xx Internal here).
+ * The driver can pick the correct internal controller based on the ASIC.
+ */
+
+#define ATOM_TONGA_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL   0x89    /* ADT7473 Fan Control + Internal Thermal Controller */
+#define ATOM_TONGA_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL   0x8D    /* EMC2103 Fan Control + Internal Thermal Controller */
+
+/*/* ATOM_TONGA_POWERPLAYTABLE::ulPlatformCaps */
+#define ATOM_TONGA_PP_PLATFORM_CAP_VDDGFX_CONTROL              0x1            /* This cap indicates whether vddgfx will be a separated power rail. */
+#define ATOM_TONGA_PP_PLATFORM_CAP_POWERPLAY                   0x2            /* This cap indicates whether this is a mobile part and CCC need to show Powerplay page. */
+#define ATOM_TONGA_PP_PLATFORM_CAP_SBIOSPOWERSOURCE            0x4            /* This cap indicates whether power source notificaiton is done by SBIOS directly. */
+#define ATOM_TONGA_PP_PLATFORM_CAP_DISABLE_VOLTAGE_ISLAND      0x8            /* Enable the option to overwrite voltage island feature to be disabled, regardless of VddGfx power rail support. */
+#define ____RETIRE16____                                0x10
+#define ATOM_TONGA_PP_PLATFORM_CAP_HARDWAREDC                 0x20            /* This cap indicates whether power source notificaiton is done by GPIO directly. */
+#define ____RETIRE64____                                0x40
+#define ____RETIRE128____                               0x80
+#define ____RETIRE256____                              0x100
+#define ____RETIRE512____                              0x200
+#define ____RETIRE1024____                             0x400
+#define ____RETIRE2048____                             0x800
+#define ATOM_TONGA_PP_PLATFORM_CAP_MVDD_CONTROL             0x1000            /* This cap indicates dynamic MVDD is required. Uncheck to disable it. */
+#define ____RETIRE2000____                            0x2000
+#define ____RETIRE4000____                            0x4000
+#define ATOM_TONGA_PP_PLATFORM_CAP_VDDCI_CONTROL            0x8000            /* This cap indicates dynamic VDDCI is required. Uncheck to disable it. */
+#define ____RETIRE10000____                          0x10000
+#define ATOM_TONGA_PP_PLATFORM_CAP_BACO                    0x20000            /* Enable to indicate the driver supports BACO state. */
+
+#define ATOM_TONGA_PP_PLATFORM_CAP_OUTPUT_THERMAL2GPIO17         0x100000     /* Enable to indicate the driver supports thermal2GPIO17. */
+#define ATOM_TONGA_PP_PLATFORM_COMBINE_PCC_WITH_THERMAL_SIGNAL  0x1000000     /* Enable to indicate if thermal and PCC are sharing the same GPIO */
+#define ATOM_TONGA_PLATFORM_LOAD_POST_PRODUCTION_FIRMWARE       0x2000000
+
+/* ATOM_PPLIB_NONCLOCK_INFO::usClassification */
+#define ATOM_PPLIB_CLASSIFICATION_UI_MASK               0x0007
+#define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT              0
+#define ATOM_PPLIB_CLASSIFICATION_UI_NONE               0
+#define ATOM_PPLIB_CLASSIFICATION_UI_BATTERY            1
+#define ATOM_PPLIB_CLASSIFICATION_UI_BALANCED           3
+#define ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE        5
+/* 2, 4, 6, 7 are reserved */
+
+#define ATOM_PPLIB_CLASSIFICATION_BOOT                  0x0008
+#define ATOM_PPLIB_CLASSIFICATION_THERMAL               0x0010
+#define ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE    0x0020
+#define ATOM_PPLIB_CLASSIFICATION_REST                  0x0040
+#define ATOM_PPLIB_CLASSIFICATION_FORCED                0x0080
+#define ATOM_PPLIB_CLASSIFICATION_ACPI                  0x1000
+
+/* ATOM_PPLIB_NONCLOCK_INFO::usClassification2 */
+#define ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2 0x0001
+
+#define ATOM_Tonga_DISALLOW_ON_DC                       0x00004000
+#define ATOM_Tonga_ENABLE_VARIBRIGHT                    0x00008000
+
+#define ATOM_Tonga_TABLE_REVISION_TONGA                 7
+
+typedef struct _ATOM_Tonga_POWERPLAYTABLE {
+	ATOM_COMMON_TABLE_HEADER sHeader;
+
+	UCHAR  ucTableRevision;
+	USHORT usTableSize;						/*the size of header structure */
+
+	ULONG	ulGoldenPPID;
+	ULONG	ulGoldenRevision;
+	USHORT	usFormatID;
+
+	USHORT	usVoltageTime;					 /*in microseconds */
+	ULONG	ulPlatformCaps;					  /*See ATOM_Tonga_CAPS_* */
+
+	ULONG	ulMaxODEngineClock; 			   /*For Overdrive.  */
+	ULONG	ulMaxODMemoryClock; 			   /*For Overdrive. */
+
+	USHORT	usPowerControlLimit;
+	USHORT	usUlvVoltageOffset;				  /*in mv units */
+
+	USHORT	usStateArrayOffset;				  /*points to ATOM_Tonga_State_Array */
+	USHORT	usFanTableOffset;				  /*points to ATOM_Tonga_Fan_Table */
+	USHORT	usThermalControllerOffset;		   /*points to ATOM_Tonga_Thermal_Controller */
+	USHORT	usReserv;						   /*CustomThermalPolicy removed for Tonga. Keep this filed as reserved. */
+
+	USHORT	usMclkDependencyTableOffset;	   /*points to ATOM_Tonga_MCLK_Dependency_Table */
+	USHORT	usSclkDependencyTableOffset;	   /*points to ATOM_Tonga_SCLK_Dependency_Table */
+	USHORT	usVddcLookupTableOffset;		   /*points to ATOM_Tonga_Voltage_Lookup_Table */
+	USHORT	usVddgfxLookupTableOffset; 		/*points to ATOM_Tonga_Voltage_Lookup_Table */
+
+	USHORT	usMMDependencyTableOffset;		  /*points to ATOM_Tonga_MM_Dependency_Table */
+
+	USHORT	usVCEStateTableOffset;			   /*points to ATOM_Tonga_VCE_State_Table; */
+
+	USHORT	usPPMTableOffset;				  /*points to ATOM_Tonga_PPM_Table */
+	USHORT	usPowerTuneTableOffset;			  /*points to ATOM_PowerTune_Table */
+
+	USHORT	usHardLimitTableOffset; 		   /*points to ATOM_Tonga_Hard_Limit_Table */
+
+	USHORT	usPCIETableOffset;				  /*points to ATOM_Tonga_PCIE_Table */
+
+	USHORT	usGPIOTableOffset;				  /*points to ATOM_Tonga_GPIO_Table */
+
+	USHORT	usReserved[6];					   /*TODO: modify reserved size to fit structure aligning */
+} ATOM_Tonga_POWERPLAYTABLE;
+
+typedef struct _ATOM_Tonga_State {
+	UCHAR  ucEngineClockIndexHigh;
+	UCHAR  ucEngineClockIndexLow;
+
+	UCHAR  ucMemoryClockIndexHigh;
+	UCHAR  ucMemoryClockIndexLow;
+
+	UCHAR  ucPCIEGenLow;
+	UCHAR  ucPCIEGenHigh;
+
+	UCHAR  ucPCIELaneLow;
+	UCHAR  ucPCIELaneHigh;
+
+	USHORT usClassification;
+	ULONG ulCapsAndSettings;
+	USHORT usClassification2;
+	UCHAR  ucUnused[4];
+} ATOM_Tonga_State;
+
+typedef struct _ATOM_Tonga_State_Array {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries;		/* Number of entries. */
+	ATOM_Tonga_State states[1];	/* Dynamically allocate entries. */
+} ATOM_Tonga_State_Array;
+
+typedef struct _ATOM_Tonga_MCLK_Dependency_Record {
+	UCHAR  ucVddcInd;	/* Vddc voltage */
+	USHORT usVddci;
+	USHORT usVddgfxOffset;	/* Offset relative to Vddc voltage */
+	USHORT usMvdd;
+	ULONG ulMclk;
+	USHORT usReserved;
+} ATOM_Tonga_MCLK_Dependency_Record;
+
+typedef struct _ATOM_Tonga_MCLK_Dependency_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries; 										/* Number of entries. */
+	ATOM_Tonga_MCLK_Dependency_Record entries[1];				/* Dynamically allocate entries. */
+} ATOM_Tonga_MCLK_Dependency_Table;
+
+typedef struct _ATOM_Tonga_SCLK_Dependency_Record {
+	UCHAR  ucVddInd;											/* Base voltage */
+	USHORT usVddcOffset;										/* Offset relative to base voltage */
+	ULONG ulSclk;
+	USHORT usEdcCurrent;
+	UCHAR  ucReliabilityTemperature;
+	UCHAR  ucCKSVOffsetandDisable;							  /* Bits 0~6: Voltage offset for CKS, Bit 7: Disable/enable for the SCLK level. */
+} ATOM_Tonga_SCLK_Dependency_Record;
+
+typedef struct _ATOM_Tonga_SCLK_Dependency_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries; 										/* Number of entries. */
+	ATOM_Tonga_SCLK_Dependency_Record entries[1];				 /* Dynamically allocate entries. */
+} ATOM_Tonga_SCLK_Dependency_Table;
+
+typedef struct _ATOM_Tonga_PCIE_Record {
+	UCHAR ucPCIEGenSpeed;
+	UCHAR usPCIELaneWidth;
+	UCHAR ucReserved[2];
+} ATOM_Tonga_PCIE_Record;
+
+typedef struct _ATOM_Tonga_PCIE_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries; 										/* Number of entries. */
+	ATOM_Tonga_PCIE_Record entries[1];							/* Dynamically allocate entries. */
+} ATOM_Tonga_PCIE_Table;
+
+typedef struct _ATOM_Tonga_MM_Dependency_Record {
+	UCHAR   ucVddcInd;											 /* VDDC voltage */
+	USHORT  usVddgfxOffset;									  /* Offset relative to VDDC voltage */
+	ULONG  ulDClk;												/* UVD D-clock */
+	ULONG  ulVClk;												/* UVD V-clock */
+	ULONG  ulEClk;												/* VCE clock */
+	ULONG  ulAClk;												/* ACP clock */
+	ULONG  ulSAMUClk;											/* SAMU clock */
+} ATOM_Tonga_MM_Dependency_Record;
+
+typedef struct _ATOM_Tonga_MM_Dependency_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries; 										/* Number of entries. */
+	ATOM_Tonga_MM_Dependency_Record entries[1]; 			   /* Dynamically allocate entries. */
+} ATOM_Tonga_MM_Dependency_Table;
+
+typedef struct _ATOM_Tonga_Voltage_Lookup_Record {
+	USHORT usVdd;											   /* Base voltage */
+	USHORT usCACLow;
+	USHORT usCACMid;
+	USHORT usCACHigh;
+} ATOM_Tonga_Voltage_Lookup_Record;
+
+typedef struct _ATOM_Tonga_Voltage_Lookup_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries; 										/* Number of entries. */
+	ATOM_Tonga_Voltage_Lookup_Record entries[1];				/* Dynamically allocate entries. */
+} ATOM_Tonga_Voltage_Lookup_Table;
+
+typedef struct _ATOM_Tonga_Fan_Table {
+	UCHAR   ucRevId;						 /* Change this if the table format changes or version changes so that the other fields are not the same. */
+	UCHAR   ucTHyst;						 /* Temperature hysteresis. Integer. */
+	USHORT  usTMin; 						 /* The temperature, in 0.01 centigrades, below which we just run at a minimal PWM. */
+	USHORT  usTMed; 						 /* The middle temperature where we change slopes. */
+	USHORT  usTHigh;						 /* The high point above TMed for adjusting the second slope. */
+	USHORT  usPWMMin;						 /* The minimum PWM value in percent (0.01% increments). */
+	USHORT  usPWMMed;						 /* The PWM value (in percent) at TMed. */
+	USHORT  usPWMHigh;						 /* The PWM value at THigh. */
+	USHORT  usTMax; 						 /* The max temperature */
+	UCHAR   ucFanControlMode;				  /* Legacy or Fuzzy Fan mode */
+	USHORT  usFanPWMMax;					  /* Maximum allowed fan power in percent */
+	USHORT  usFanOutputSensitivity;		  /* Sensitivity of fan reaction to temepature changes */
+	USHORT  usFanRPMMax;					  /* The default value in RPM */
+	ULONG  ulMinFanSCLKAcousticLimit;	   /* Minimum Fan Controller SCLK Frequency Acoustic Limit. */
+	UCHAR   ucTargetTemperature;			 /* Advanced fan controller target temperature. */
+	UCHAR   ucMinimumPWMLimit; 			  /* The minimum PWM that the advanced fan controller can set.	This should be set to the highest PWM that will run the fan at its lowest RPM. */
+	USHORT  usReserved;
+} ATOM_Tonga_Fan_Table;
+
+typedef struct _ATOM_Fiji_Fan_Table {
+	UCHAR   ucRevId;						 /* Change this if the table format changes or version changes so that the other fields are not the same. */
+	UCHAR   ucTHyst;						 /* Temperature hysteresis. Integer. */
+	USHORT  usTMin; 						 /* The temperature, in 0.01 centigrades, below which we just run at a minimal PWM. */
+	USHORT  usTMed; 						 /* The middle temperature where we change slopes. */
+	USHORT  usTHigh;						 /* The high point above TMed for adjusting the second slope. */
+	USHORT  usPWMMin;						 /* The minimum PWM value in percent (0.01% increments). */
+	USHORT  usPWMMed;						 /* The PWM value (in percent) at TMed. */
+	USHORT  usPWMHigh;						 /* The PWM value at THigh. */
+	USHORT  usTMax; 						 /* The max temperature */
+	UCHAR   ucFanControlMode;				  /* Legacy or Fuzzy Fan mode */
+	USHORT  usFanPWMMax;					  /* Maximum allowed fan power in percent */
+	USHORT  usFanOutputSensitivity;		  /* Sensitivity of fan reaction to temepature changes */
+	USHORT  usFanRPMMax;					  /* The default value in RPM */
+	ULONG  ulMinFanSCLKAcousticLimit;		/* Minimum Fan Controller SCLK Frequency Acoustic Limit. */
+	UCHAR   ucTargetTemperature;			 /* Advanced fan controller target temperature. */
+	UCHAR   ucMinimumPWMLimit; 			  /* The minimum PWM that the advanced fan controller can set.	This should be set to the highest PWM that will run the fan at its lowest RPM. */
+	USHORT  usFanGainEdge;
+	USHORT  usFanGainHotspot;
+	USHORT  usFanGainLiquid;
+	USHORT  usFanGainVrVddc;
+	USHORT  usFanGainVrMvdd;
+	USHORT  usFanGainPlx;
+	USHORT  usFanGainHbm;
+	USHORT  usReserved;
+} ATOM_Fiji_Fan_Table;
+
+typedef struct _ATOM_Tonga_Thermal_Controller {
+	UCHAR ucRevId;
+	UCHAR ucType;		   /* one of ATOM_TONGA_PP_THERMALCONTROLLER_* */
+	UCHAR ucI2cLine;		/* as interpreted by DAL I2C */
+	UCHAR ucI2cAddress;
+	UCHAR ucFanParameters;	/* Fan Control Parameters. */
+	UCHAR ucFanMinRPM; 	 /* Fan Minimum RPM (hundreds) -- for display purposes only. */
+	UCHAR ucFanMaxRPM; 	 /* Fan Maximum RPM (hundreds) -- for display purposes only. */
+	UCHAR ucReserved;
+	UCHAR ucFlags;		   /* to be defined */
+} ATOM_Tonga_Thermal_Controller;
+
+typedef struct _ATOM_Tonga_VCE_State_Record {
+	UCHAR  ucVCEClockIndex;	/*index into usVCEDependencyTableOffset of 'ATOM_Tonga_MM_Dependency_Table' type */
+	UCHAR  ucFlag;		/* 2 bits indicates memory p-states */
+	UCHAR  ucSCLKIndex;		/*index into ATOM_Tonga_SCLK_Dependency_Table */
+	UCHAR  ucMCLKIndex;		/*index into ATOM_Tonga_MCLK_Dependency_Table */
+} ATOM_Tonga_VCE_State_Record;
+
+typedef struct _ATOM_Tonga_VCE_State_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries;
+	ATOM_Tonga_VCE_State_Record entries[1];
+} ATOM_Tonga_VCE_State_Table;
+
+typedef struct _ATOM_Tonga_PowerTune_Table {
+	UCHAR  ucRevId;
+	USHORT usTDP;
+	USHORT usConfigurableTDP;
+	USHORT usTDC;
+	USHORT usBatteryPowerLimit;
+	USHORT usSmallPowerLimit;
+	USHORT usLowCACLeakage;
+	USHORT usHighCACLeakage;
+	USHORT usMaximumPowerDeliveryLimit;
+	USHORT usTjMax;
+	USHORT usPowerTuneDataSetID;
+	USHORT usEDCLimit;
+	USHORT usSoftwareShutdownTemp;
+	USHORT usClockStretchAmount;
+	USHORT usReserve[2];
+} ATOM_Tonga_PowerTune_Table;
+
+typedef struct _ATOM_Fiji_PowerTune_Table {
+	UCHAR  ucRevId;
+	USHORT usTDP;
+	USHORT usConfigurableTDP;
+	USHORT usTDC;
+	USHORT usBatteryPowerLimit;
+	USHORT usSmallPowerLimit;
+	USHORT usLowCACLeakage;
+	USHORT usHighCACLeakage;
+	USHORT usMaximumPowerDeliveryLimit;
+	USHORT usTjMax;  /* For Fiji, this is also usTemperatureLimitEdge; */
+	USHORT usPowerTuneDataSetID;
+	USHORT usEDCLimit;
+	USHORT usSoftwareShutdownTemp;
+	USHORT usClockStretchAmount;
+	USHORT usTemperatureLimitHotspot;  /*The following are added for Fiji */
+	USHORT usTemperatureLimitLiquid1;
+	USHORT usTemperatureLimitLiquid2;
+	USHORT usTemperatureLimitVrVddc;
+	USHORT usTemperatureLimitVrMvdd;
+	USHORT usTemperatureLimitPlx;
+	UCHAR  ucLiquid1_I2C_address;  /*Liquid */
+	UCHAR  ucLiquid2_I2C_address;
+	UCHAR  ucLiquid_I2C_Line;
+	UCHAR  ucVr_I2C_address;	/*VR */
+	UCHAR  ucVr_I2C_Line;
+	UCHAR  ucPlx_I2C_address;  /*PLX */
+	UCHAR  ucPlx_I2C_Line;
+	USHORT usReserved;
+} ATOM_Fiji_PowerTune_Table;
+
+#define ATOM_PPM_A_A    1
+#define ATOM_PPM_A_I    2
+typedef struct _ATOM_Tonga_PPM_Table {
+	UCHAR   ucRevId;
+	UCHAR   ucPpmDesign;		  /*A+I or A+A */
+	USHORT  usCpuCoreNumber;
+	ULONG  ulPlatformTDP;
+	ULONG  ulSmallACPlatformTDP;
+	ULONG  ulPlatformTDC;
+	ULONG  ulSmallACPlatformTDC;
+	ULONG  ulApuTDP;
+	ULONG  ulDGpuTDP;
+	ULONG  ulDGpuUlvPower;
+	ULONG  ulTjmax;
+} ATOM_Tonga_PPM_Table;
+
+typedef struct _ATOM_Tonga_Hard_Limit_Record {
+	ULONG  ulSCLKLimit;
+	ULONG  ulMCLKLimit;
+	USHORT  usVddcLimit;
+	USHORT  usVddciLimit;
+	USHORT  usVddgfxLimit;
+} ATOM_Tonga_Hard_Limit_Record;
+
+typedef struct _ATOM_Tonga_Hard_Limit_Table {
+	UCHAR ucRevId;
+	UCHAR ucNumEntries;
+	ATOM_Tonga_Hard_Limit_Record entries[1];
+} ATOM_Tonga_Hard_Limit_Table;
+
+typedef struct _ATOM_Tonga_GPIO_Table {
+	UCHAR  ucRevId;
+	UCHAR  ucVRHotTriggeredSclkDpmIndex;		/* If VRHot signal is triggered SCLK will be limited to this DPM level */
+	UCHAR  ucReserve[5];
+} ATOM_Tonga_GPIO_Table;
+
+typedef struct _PPTable_Generic_SubTable_Header {
+	UCHAR  ucRevId;
+} PPTable_Generic_SubTable_Header;
+
+
+#pragma pack(pop)
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
new file mode 100644
index 000000000000..34f4bef3691f
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
@@ -0,0 +1,1142 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/fb.h>
+
+#include "tonga_processpptables.h"
+#include "ppatomctrl.h"
+#include "atombios.h"
+#include "pp_debug.h"
+#include "hwmgr.h"
+#include "cgs_common.h"
+#include "tonga_pptable.h"
+
+/**
+ * Private Function used during initialization.
+ * @param hwmgr Pointer to the hardware manager.
+ * @param setIt A flag indication if the capability should be set (TRUE) or reset (FALSE).
+ * @param cap Which capability to set/reset.
+ */
+static void set_hw_cap(struct pp_hwmgr *hwmgr, bool setIt, enum phm_platform_caps cap)
+{
+	if (setIt)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps, cap);
+	else
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, cap);
+}
+
+
+/**
+ * Private Function used during initialization.
+ * @param hwmgr Pointer to the hardware manager.
+ * @param powerplay_caps the bit array (from BIOS) of capability bits.
+ * @exception the current implementation always returns 1.
+ */
+static int set_platform_caps(struct pp_hwmgr *hwmgr, uint32_t powerplay_caps)
+{
+	PP_ASSERT_WITH_CODE((~powerplay_caps & ____RETIRE16____),
+		"ATOM_PP_PLATFORM_CAP_ASPM_L1 is not supported!", continue);
+	PP_ASSERT_WITH_CODE((~powerplay_caps & ____RETIRE64____),
+		"ATOM_PP_PLATFORM_CAP_GEMINIPRIMARY is not supported!", continue);
+	PP_ASSERT_WITH_CODE((~powerplay_caps & ____RETIRE512____),
+		"ATOM_PP_PLATFORM_CAP_SIDEPORTCONTROL is not supported!", continue);
+	PP_ASSERT_WITH_CODE((~powerplay_caps & ____RETIRE1024____),
+		"ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1 is not supported!", continue);
+	PP_ASSERT_WITH_CODE((~powerplay_caps & ____RETIRE2048____),
+		"ATOM_PP_PLATFORM_CAP_HTLINKCONTROL is not supported!", continue);
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_POWERPLAY),
+			PHM_PlatformCaps_PowerPlaySupport
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_SBIOSPOWERSOURCE),
+			PHM_PlatformCaps_BiosPowerSourceControl
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_HARDWAREDC),
+			PHM_PlatformCaps_AutomaticDCTransition
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_MVDD_CONTROL),
+			PHM_PlatformCaps_EnableMVDDControl
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_VDDCI_CONTROL),
+			PHM_PlatformCaps_ControlVDDCI
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_VDDGFX_CONTROL),
+			PHM_PlatformCaps_ControlVDDGFX
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_BACO),
+			PHM_PlatformCaps_BACO
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_CAP_DISABLE_VOLTAGE_ISLAND),
+			PHM_PlatformCaps_DisableVoltageIsland
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PP_PLATFORM_COMBINE_PCC_WITH_THERMAL_SIGNAL),
+			PHM_PlatformCaps_CombinePCCWithThermalSignal
+		  );
+
+	set_hw_cap(
+			hwmgr,
+			0 != (powerplay_caps & ATOM_TONGA_PLATFORM_LOAD_POST_PRODUCTION_FIRMWARE),
+			PHM_PlatformCaps_LoadPostProductionFirmware
+		  );
+
+	return 0;
+}
+
+/**
+ * Private Function to get the PowerPlay Table Address.
+ */
+const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
+{
+	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+
+	u16 size;
+	u8 frev, crev;
+	void *table_address;
+
+	table_address = (ATOM_Tonga_POWERPLAYTABLE *)
+		cgs_atom_get_data_table(hwmgr->device, index, &size, &frev, &crev);
+
+	hwmgr->soft_pp_table = table_address;	/*Cache the result in RAM.*/
+
+	return table_address;
+}
+
+static int get_vddc_lookup_table(
+		struct pp_hwmgr	*hwmgr,
+		phm_ppt_v1_voltage_lookup_table	**lookup_table,
+		const ATOM_Tonga_Voltage_Lookup_Table	*vddc_lookup_pp_tables,
+		uint32_t	max_levels
+		)
+{
+	uint32_t table_size, i;
+	phm_ppt_v1_voltage_lookup_table *table;
+
+	PP_ASSERT_WITH_CODE((0 != vddc_lookup_pp_tables->ucNumEntries),
+		"Invalid CAC Leakage PowerPlay Table!", return 1);
+
+	table_size = sizeof(uint32_t) +
+		sizeof(phm_ppt_v1_voltage_lookup_record) * max_levels;
+
+	table = (phm_ppt_v1_voltage_lookup_table *)
+		kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	memset(table, 0x00, table_size);
+
+	table->count = vddc_lookup_pp_tables->ucNumEntries;
+
+	for (i = 0; i < vddc_lookup_pp_tables->ucNumEntries; i++) {
+		table->entries[i].us_calculated = 0;
+		table->entries[i].us_vdd =
+			vddc_lookup_pp_tables->entries[i].usVdd;
+		table->entries[i].us_cac_low =
+			vddc_lookup_pp_tables->entries[i].usCACLow;
+		table->entries[i].us_cac_mid =
+			vddc_lookup_pp_tables->entries[i].usCACMid;
+		table->entries[i].us_cac_high =
+			vddc_lookup_pp_tables->entries[i].usCACHigh;
+	}
+
+	*lookup_table = table;
+
+	return 0;
+}
+
+/**
+ * Private Function used during initialization.
+ * Initialize Platform Power Management Parameter table
+ * @param hwmgr Pointer to the hardware manager.
+ * @param atom_ppm_table Pointer to PPM table in VBIOS
+ */
+static int get_platform_power_management_table(
+		struct pp_hwmgr *hwmgr,
+		ATOM_Tonga_PPM_Table *atom_ppm_table)
+{
+	struct phm_ppm_table *ptr = kzalloc(sizeof(ATOM_Tonga_PPM_Table), GFP_KERNEL);
+	struct phm_ppt_v1_information *pp_table_information =
+		(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (NULL == ptr)
+		return -ENOMEM;
+
+	ptr->ppm_design
+		= atom_ppm_table->ucPpmDesign;
+	ptr->cpu_core_number
+		= atom_ppm_table->usCpuCoreNumber;
+	ptr->platform_tdp
+		= atom_ppm_table->ulPlatformTDP;
+	ptr->small_ac_platform_tdp
+		= atom_ppm_table->ulSmallACPlatformTDP;
+	ptr->platform_tdc
+		= atom_ppm_table->ulPlatformTDC;
+	ptr->small_ac_platform_tdc
+		= atom_ppm_table->ulSmallACPlatformTDC;
+	ptr->apu_tdp
+		= atom_ppm_table->ulApuTDP;
+	ptr->dgpu_tdp
+		= atom_ppm_table->ulDGpuTDP;
+	ptr->dgpu_ulv_power
+		= atom_ppm_table->ulDGpuUlvPower;
+	ptr->tj_max
+		= atom_ppm_table->ulTjmax;
+
+	pp_table_information->ppm_parameter_table = ptr;
+
+	return 0;
+}
+
+/**
+ * Private Function used during initialization.
+ * Initialize TDP limits for DPM2
+ * @param hwmgr Pointer to the hardware manager.
+ * @param powerplay_table Pointer to the PowerPlay Table.
+ */
+static int init_dpm_2_parameters(
+		struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table
+		)
+{
+	int result = 0;
+	struct phm_ppt_v1_information *pp_table_information = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	ATOM_Tonga_PPM_Table *atom_ppm_table;
+	uint32_t disable_ppm = 0;
+	uint32_t disable_power_control = 0;
+
+	pp_table_information->us_ulv_voltage_offset =
+		le16_to_cpu(powerplay_table->usUlvVoltageOffset);
+
+	pp_table_information->ppm_parameter_table = NULL;
+	pp_table_information->vddc_lookup_table = NULL;
+	pp_table_information->vddgfx_lookup_table = NULL;
+	/* TDP limits */
+	hwmgr->platform_descriptor.TDPODLimit =
+		le16_to_cpu(powerplay_table->usPowerControlLimit);
+	hwmgr->platform_descriptor.TDPAdjustment = 0;
+	hwmgr->platform_descriptor.VidAdjustment = 0;
+	hwmgr->platform_descriptor.VidAdjustmentPolarity = 0;
+	hwmgr->platform_descriptor.VidMinLimit = 0;
+	hwmgr->platform_descriptor.VidMaxLimit = 1500000;
+	hwmgr->platform_descriptor.VidStep = 6250;
+
+	disable_power_control = 0;
+	if (0 == disable_power_control) {
+		/* enable TDP overdrive (PowerControl) feature as well if supported */
+		if (hwmgr->platform_descriptor.TDPODLimit != 0)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerControl);
+	}
+
+	if (0 != powerplay_table->usVddcLookupTableOffset) {
+		const ATOM_Tonga_Voltage_Lookup_Table *pVddcCACTable =
+			(ATOM_Tonga_Voltage_Lookup_Table *)(((unsigned long)powerplay_table) +
+			le16_to_cpu(powerplay_table->usVddcLookupTableOffset));
+
+		result = get_vddc_lookup_table(hwmgr,
+			&pp_table_information->vddc_lookup_table, pVddcCACTable, 16);
+	}
+
+	if (0 != powerplay_table->usVddgfxLookupTableOffset) {
+		const ATOM_Tonga_Voltage_Lookup_Table *pVddgfxCACTable =
+			(ATOM_Tonga_Voltage_Lookup_Table *)(((unsigned long)powerplay_table) +
+			le16_to_cpu(powerplay_table->usVddgfxLookupTableOffset));
+
+		result = get_vddc_lookup_table(hwmgr,
+			&pp_table_information->vddgfx_lookup_table, pVddgfxCACTable, 16);
+	}
+
+	disable_ppm = 0;
+	if (0 == disable_ppm) {
+		atom_ppm_table = (ATOM_Tonga_PPM_Table *)
+			(((unsigned long)powerplay_table) + le16_to_cpu(powerplay_table->usPPMTableOffset));
+
+		if (0 != powerplay_table->usPPMTableOffset) {
+			if (1 == get_platform_power_management_table(hwmgr, atom_ppm_table)) {
+				phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_EnablePlatformPowerManagement);
+			}
+		}
+	}
+
+	return result;
+}
+
+static int get_valid_clk(
+		struct pp_hwmgr *hwmgr,
+		struct phm_clock_array **clk_table,
+		const phm_ppt_v1_clock_voltage_dependency_table  * clk_volt_pp_table
+		)
+{
+	uint32_t table_size, i;
+	struct phm_clock_array *table;
+
+	PP_ASSERT_WITH_CODE((0 != clk_volt_pp_table->count),
+		"Invalid PowerPlay Table!", return -1);
+
+	table_size = sizeof(uint32_t) +
+		sizeof(uint32_t) * clk_volt_pp_table->count;
+
+	table = (struct phm_clock_array *)kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	memset(table, 0x00, table_size);
+
+	table->count = (uint32_t)clk_volt_pp_table->count;
+
+	for (i = 0; i < table->count; i++)
+		table->values[i] = (uint32_t)clk_volt_pp_table->entries[i].clk;
+
+	*clk_table = table;
+
+	return 0;
+}
+
+static int get_hard_limits(
+		struct pp_hwmgr *hwmgr,
+		struct phm_clock_and_voltage_limits *limits,
+		const ATOM_Tonga_Hard_Limit_Table * limitable
+		)
+{
+	PP_ASSERT_WITH_CODE((0 != limitable->ucNumEntries), "Invalid PowerPlay Table!", return -1);
+
+	/* currently we always take entries[0] parameters */
+	limits->sclk = (uint32_t)limitable->entries[0].ulSCLKLimit;
+	limits->mclk = (uint32_t)limitable->entries[0].ulMCLKLimit;
+	limits->vddc = (uint16_t)limitable->entries[0].usVddcLimit;
+	limits->vddci = (uint16_t)limitable->entries[0].usVddciLimit;
+	limits->vddgfx = (uint16_t)limitable->entries[0].usVddgfxLimit;
+
+	return 0;
+}
+
+static int get_mclk_voltage_dependency_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_clock_voltage_dependency_table **pp_tonga_mclk_dep_table,
+		const ATOM_Tonga_MCLK_Dependency_Table * mclk_dep_table
+		)
+{
+	uint32_t table_size, i;
+	phm_ppt_v1_clock_voltage_dependency_table *mclk_table;
+
+	PP_ASSERT_WITH_CODE((0 != mclk_dep_table->ucNumEntries),
+		"Invalid PowerPlay Table!", return -1);
+
+	table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record)
+		* mclk_dep_table->ucNumEntries;
+
+	mclk_table = (phm_ppt_v1_clock_voltage_dependency_table *)
+		kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == mclk_table)
+		return -ENOMEM;
+
+	memset(mclk_table, 0x00, table_size);
+
+	mclk_table->count = (uint32_t)mclk_dep_table->ucNumEntries;
+
+	for (i = 0; i < mclk_dep_table->ucNumEntries; i++) {
+		mclk_table->entries[i].vddInd =
+			mclk_dep_table->entries[i].ucVddcInd;
+		mclk_table->entries[i].vdd_offset =
+			mclk_dep_table->entries[i].usVddgfxOffset;
+		mclk_table->entries[i].vddci =
+			mclk_dep_table->entries[i].usVddci;
+		mclk_table->entries[i].mvdd =
+			mclk_dep_table->entries[i].usMvdd;
+		mclk_table->entries[i].clk =
+			mclk_dep_table->entries[i].ulMclk;
+	}
+
+	*pp_tonga_mclk_dep_table = mclk_table;
+
+	return 0;
+}
+
+static int get_sclk_voltage_dependency_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_clock_voltage_dependency_table **pp_tonga_sclk_dep_table,
+		const ATOM_Tonga_SCLK_Dependency_Table * sclk_dep_table
+		)
+{
+	uint32_t table_size, i;
+	phm_ppt_v1_clock_voltage_dependency_table *sclk_table;
+
+	PP_ASSERT_WITH_CODE((0 != sclk_dep_table->ucNumEntries),
+		"Invalid PowerPlay Table!", return -1);
+
+	table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record)
+		* sclk_dep_table->ucNumEntries;
+
+	sclk_table = (phm_ppt_v1_clock_voltage_dependency_table *)
+		kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == sclk_table)
+		return -ENOMEM;
+
+	memset(sclk_table, 0x00, table_size);
+
+	sclk_table->count = (uint32_t)sclk_dep_table->ucNumEntries;
+
+	for (i = 0; i < sclk_dep_table->ucNumEntries; i++) {
+		sclk_table->entries[i].vddInd =
+			sclk_dep_table->entries[i].ucVddInd;
+		sclk_table->entries[i].vdd_offset =
+			sclk_dep_table->entries[i].usVddcOffset;
+		sclk_table->entries[i].clk =
+			sclk_dep_table->entries[i].ulSclk;
+		sclk_table->entries[i].cks_enable =
+			(((sclk_dep_table->entries[i].ucCKSVOffsetandDisable & 0x80) >> 7) == 0) ? 1 : 0;
+		sclk_table->entries[i].cks_voffset =
+			(sclk_dep_table->entries[i].ucCKSVOffsetandDisable & 0x7F);
+	}
+
+	*pp_tonga_sclk_dep_table = sclk_table;
+
+	return 0;
+}
+
+static int get_pcie_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_pcie_table **pp_tonga_pcie_table,
+		const ATOM_Tonga_PCIE_Table * atom_pcie_table
+		)
+{
+	uint32_t table_size, i, pcie_count;
+	phm_ppt_v1_pcie_table *pcie_table;
+	struct phm_ppt_v1_information *pp_table_information =
+		(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	PP_ASSERT_WITH_CODE((0 != atom_pcie_table->ucNumEntries),
+		"Invalid PowerPlay Table!", return -1);
+
+	table_size = sizeof(uint32_t) +
+		sizeof(phm_ppt_v1_pcie_record) * atom_pcie_table->ucNumEntries;
+
+	pcie_table = (phm_ppt_v1_pcie_table *)kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == pcie_table)
+		return -ENOMEM;
+
+	memset(pcie_table, 0x00, table_size);
+
+	/*
+	* Make sure the number of pcie entries are less than or equal to sclk dpm levels.
+	* Since first PCIE entry is for ULV, #pcie has to be <= SclkLevel + 1.
+	*/
+	pcie_count = (pp_table_information->vdd_dep_on_sclk->count) + 1;
+	if ((uint32_t)atom_pcie_table->ucNumEntries <= pcie_count)
+		pcie_count = (uint32_t)atom_pcie_table->ucNumEntries;
+	else
+		printk(KERN_ERR "[ powerplay ] Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
+		Disregarding the excess entries... \n");
+
+	pcie_table->count = pcie_count;
+
+	for (i = 0; i < pcie_count; i++) {
+		pcie_table->entries[i].gen_speed =
+			atom_pcie_table->entries[i].ucPCIEGenSpeed;
+		pcie_table->entries[i].lane_width =
+			atom_pcie_table->entries[i].usPCIELaneWidth;
+	}
+
+	*pp_tonga_pcie_table = pcie_table;
+
+	return 0;
+}
+
+static int get_cac_tdp_table(
+		struct pp_hwmgr *hwmgr,
+		struct phm_cac_tdp_table **cac_tdp_table,
+		const PPTable_Generic_SubTable_Header * table
+		)
+{
+	uint32_t table_size;
+	struct phm_cac_tdp_table *tdp_table;
+
+	table_size = sizeof(uint32_t) + sizeof(struct phm_cac_tdp_table);
+	tdp_table = kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == tdp_table)
+		return -ENOMEM;
+
+	memset(tdp_table, 0x00, table_size);
+
+	hwmgr->dyn_state.cac_dtp_table = kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == hwmgr->dyn_state.cac_dtp_table)
+		return -ENOMEM;
+
+	memset(hwmgr->dyn_state.cac_dtp_table, 0x00, table_size);
+
+	if (table->ucRevId < 3) {
+		const ATOM_Tonga_PowerTune_Table *tonga_table =
+			(ATOM_Tonga_PowerTune_Table *)table;
+		tdp_table->usTDP = tonga_table->usTDP;
+		tdp_table->usConfigurableTDP =
+			tonga_table->usConfigurableTDP;
+		tdp_table->usTDC = tonga_table->usTDC;
+		tdp_table->usBatteryPowerLimit =
+			tonga_table->usBatteryPowerLimit;
+		tdp_table->usSmallPowerLimit =
+			tonga_table->usSmallPowerLimit;
+		tdp_table->usLowCACLeakage =
+			tonga_table->usLowCACLeakage;
+		tdp_table->usHighCACLeakage =
+			tonga_table->usHighCACLeakage;
+		tdp_table->usMaximumPowerDeliveryLimit =
+			tonga_table->usMaximumPowerDeliveryLimit;
+		tdp_table->usDefaultTargetOperatingTemp =
+			tonga_table->usTjMax;
+		tdp_table->usTargetOperatingTemp =
+			tonga_table->usTjMax; /*Set the initial temp to the same as default */
+		tdp_table->usPowerTuneDataSetID =
+			tonga_table->usPowerTuneDataSetID;
+		tdp_table->usSoftwareShutdownTemp =
+			tonga_table->usSoftwareShutdownTemp;
+		tdp_table->usClockStretchAmount =
+			tonga_table->usClockStretchAmount;
+	} else {   /* Fiji and newer */
+		const ATOM_Fiji_PowerTune_Table *fijitable =
+			(ATOM_Fiji_PowerTune_Table *)table;
+		tdp_table->usTDP = fijitable->usTDP;
+		tdp_table->usConfigurableTDP = fijitable->usConfigurableTDP;
+		tdp_table->usTDC = fijitable->usTDC;
+		tdp_table->usBatteryPowerLimit = fijitable->usBatteryPowerLimit;
+		tdp_table->usSmallPowerLimit = fijitable->usSmallPowerLimit;
+		tdp_table->usLowCACLeakage = fijitable->usLowCACLeakage;
+		tdp_table->usHighCACLeakage = fijitable->usHighCACLeakage;
+		tdp_table->usMaximumPowerDeliveryLimit =
+			fijitable->usMaximumPowerDeliveryLimit;
+		tdp_table->usDefaultTargetOperatingTemp =
+			fijitable->usTjMax;
+		tdp_table->usTargetOperatingTemp =
+			fijitable->usTjMax; /*Set the initial temp to the same as default */
+		tdp_table->usPowerTuneDataSetID =
+			fijitable->usPowerTuneDataSetID;
+		tdp_table->usSoftwareShutdownTemp =
+			fijitable->usSoftwareShutdownTemp;
+		tdp_table->usClockStretchAmount =
+			fijitable->usClockStretchAmount;
+		tdp_table->usTemperatureLimitHotspot =
+			fijitable->usTemperatureLimitHotspot;
+		tdp_table->usTemperatureLimitLiquid1 =
+			fijitable->usTemperatureLimitLiquid1;
+		tdp_table->usTemperatureLimitLiquid2 =
+			fijitable->usTemperatureLimitLiquid2;
+		tdp_table->usTemperatureLimitVrVddc =
+			fijitable->usTemperatureLimitVrVddc;
+		tdp_table->usTemperatureLimitVrMvdd =
+			fijitable->usTemperatureLimitVrMvdd;
+		tdp_table->usTemperatureLimitPlx =
+			fijitable->usTemperatureLimitPlx;
+		tdp_table->ucLiquid1_I2C_address =
+			fijitable->ucLiquid1_I2C_address;
+		tdp_table->ucLiquid2_I2C_address =
+			fijitable->ucLiquid2_I2C_address;
+		tdp_table->ucLiquid_I2C_Line =
+			fijitable->ucLiquid_I2C_Line;
+		tdp_table->ucVr_I2C_address = fijitable->ucVr_I2C_address;
+		tdp_table->ucVr_I2C_Line = fijitable->ucVr_I2C_Line;
+		tdp_table->ucPlx_I2C_address = fijitable->ucPlx_I2C_address;
+		tdp_table->ucPlx_I2C_Line = fijitable->ucPlx_I2C_Line;
+	}
+
+	*cac_tdp_table = tdp_table;
+
+	return 0;
+}
+
+static int get_mm_clock_voltage_table(
+		struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_mm_clock_voltage_dependency_table **tonga_mm_table,
+		const ATOM_Tonga_MM_Dependency_Table * mm_dependency_table
+		)
+{
+	uint32_t table_size, i;
+	const ATOM_Tonga_MM_Dependency_Record *mm_dependency_record;
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table;
+
+	PP_ASSERT_WITH_CODE((0 != mm_dependency_table->ucNumEntries),
+		"Invalid PowerPlay Table!", return -1);
+	table_size = sizeof(uint32_t) +
+		sizeof(phm_ppt_v1_mm_clock_voltage_dependency_record)
+		* mm_dependency_table->ucNumEntries;
+	mm_table = (phm_ppt_v1_mm_clock_voltage_dependency_table *)
+		kzalloc(table_size, GFP_KERNEL);
+
+	if (NULL == mm_table)
+		return -ENOMEM;
+
+	memset(mm_table, 0x00, table_size);
+
+	mm_table->count = mm_dependency_table->ucNumEntries;
+
+	for (i = 0; i < mm_dependency_table->ucNumEntries; i++) {
+		mm_dependency_record = &mm_dependency_table->entries[i];
+		mm_table->entries[i].vddcInd = mm_dependency_record->ucVddcInd;
+		mm_table->entries[i].vddgfx_offset = mm_dependency_record->usVddgfxOffset;
+		mm_table->entries[i].aclk = mm_dependency_record->ulAClk;
+		mm_table->entries[i].samclock = mm_dependency_record->ulSAMUClk;
+		mm_table->entries[i].eclk = mm_dependency_record->ulEClk;
+		mm_table->entries[i].vclk = mm_dependency_record->ulVClk;
+		mm_table->entries[i].dclk = mm_dependency_record->ulDClk;
+	}
+
+	*tonga_mm_table = mm_table;
+
+	return 0;
+}
+
+/**
+ * Private Function used during initialization.
+ * Initialize clock voltage dependency
+ * @param hwmgr Pointer to the hardware manager.
+ * @param powerplay_table Pointer to the PowerPlay Table.
+ */
+static int init_clock_voltage_dependency(
+		struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table
+		)
+{
+	int result = 0;
+	struct phm_ppt_v1_information *pp_table_information =
+		(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	const ATOM_Tonga_MM_Dependency_Table *mm_dependency_table =
+		(const ATOM_Tonga_MM_Dependency_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usMMDependencyTableOffset));
+	const PPTable_Generic_SubTable_Header *pPowerTuneTable =
+		(const PPTable_Generic_SubTable_Header *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usPowerTuneTableOffset));
+	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
+		(const ATOM_Tonga_MCLK_Dependency_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
+	const ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table =
+		(const ATOM_Tonga_SCLK_Dependency_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
+	const ATOM_Tonga_Hard_Limit_Table *pHardLimits =
+		(const ATOM_Tonga_Hard_Limit_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usHardLimitTableOffset));
+	const ATOM_Tonga_PCIE_Table *pcie_table =
+		(const ATOM_Tonga_PCIE_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usPCIETableOffset));
+
+	pp_table_information->vdd_dep_on_sclk = NULL;
+	pp_table_information->vdd_dep_on_mclk = NULL;
+	pp_table_information->mm_dep_table = NULL;
+	pp_table_information->pcie_table = NULL;
+
+	if (powerplay_table->usMMDependencyTableOffset != 0)
+		result = get_mm_clock_voltage_table(hwmgr,
+		&pp_table_information->mm_dep_table, mm_dependency_table);
+
+	if (result == 0 && powerplay_table->usPowerTuneTableOffset != 0)
+		result = get_cac_tdp_table(hwmgr,
+		&pp_table_information->cac_dtp_table, pPowerTuneTable);
+
+	if (result == 0 && powerplay_table->usSclkDependencyTableOffset != 0)
+		result = get_sclk_voltage_dependency_table(hwmgr,
+		&pp_table_information->vdd_dep_on_sclk, sclk_dep_table);
+
+	if (result == 0 && powerplay_table->usMclkDependencyTableOffset != 0)
+		result = get_mclk_voltage_dependency_table(hwmgr,
+		&pp_table_information->vdd_dep_on_mclk, mclk_dep_table);
+
+	if (result == 0 && powerplay_table->usPCIETableOffset != 0)
+		result = get_pcie_table(hwmgr,
+		&pp_table_information->pcie_table, pcie_table);
+
+	if (result == 0 && powerplay_table->usHardLimitTableOffset != 0)
+		result = get_hard_limits(hwmgr,
+		&pp_table_information->max_clock_voltage_on_dc, pHardLimits);
+
+	hwmgr->dyn_state.max_clock_voltage_on_dc.sclk =
+		pp_table_information->max_clock_voltage_on_dc.sclk;
+	hwmgr->dyn_state.max_clock_voltage_on_dc.mclk =
+		pp_table_information->max_clock_voltage_on_dc.mclk;
+	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
+		pp_table_information->max_clock_voltage_on_dc.vddc;
+	hwmgr->dyn_state.max_clock_voltage_on_dc.vddci =
+		pp_table_information->max_clock_voltage_on_dc.vddci;
+
+	if (result == 0 && (NULL != pp_table_information->vdd_dep_on_mclk)
+		&& (0 != pp_table_information->vdd_dep_on_mclk->count))
+		result = get_valid_clk(hwmgr, &pp_table_information->valid_mclk_values,
+		pp_table_information->vdd_dep_on_mclk);
+
+	if (result == 0 && (NULL != pp_table_information->vdd_dep_on_sclk)
+		&& (0 != pp_table_information->vdd_dep_on_sclk->count))
+		result = get_valid_clk(hwmgr, &pp_table_information->valid_sclk_values,
+		pp_table_information->vdd_dep_on_sclk);
+
+	return result;
+}
+
+/** Retrieves the (signed) Overdrive limits from VBIOS.
+ * The max engine clock, memory clock and max temperature come from the firmware info table.
+ *
+ * The information is placed into the platform descriptor.
+ *
+ * @param hwmgr source of the VBIOS table and owner of the platform descriptor to be updated.
+ * @param powerplay_table the address of the PowerPlay table.
+ *
+ * @return 1 as long as the firmware info table was present and of a supported version.
+ */
+static int init_over_drive_limits(
+		struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table)
+{
+	hwmgr->platform_descriptor.overdriveLimit.engineClock =
+		le16_to_cpu(powerplay_table->ulMaxODEngineClock);
+	hwmgr->platform_descriptor.overdriveLimit.memoryClock =
+		le16_to_cpu(powerplay_table->ulMaxODMemoryClock);
+
+	hwmgr->platform_descriptor.minOverdriveVDDC = 0;
+	hwmgr->platform_descriptor.maxOverdriveVDDC = 0;
+	hwmgr->platform_descriptor.overdriveVDDCStep = 0;
+
+	if (hwmgr->platform_descriptor.overdriveLimit.engineClock > 0 \
+		&& hwmgr->platform_descriptor.overdriveLimit.memoryClock > 0) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ACOverdriveSupport);
+	}
+
+	return 0;
+}
+
+/**
+ * Private Function used during initialization.
+ * Inspect the PowerPlay table for obvious signs of corruption.
+ * @param hwmgr Pointer to the hardware manager.
+ * @param powerplay_table Pointer to the PowerPlay Table.
+ * @exception This implementation always returns 1.
+ */
+static int init_thermal_controller(
+		struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table
+		)
+{
+	const PPTable_Generic_SubTable_Header *fan_table;
+	ATOM_Tonga_Thermal_Controller *thermal_controller;
+
+	thermal_controller = (ATOM_Tonga_Thermal_Controller *)
+		(((unsigned long)powerplay_table) +
+		le16_to_cpu(powerplay_table->usThermalControllerOffset));
+	PP_ASSERT_WITH_CODE((0 != powerplay_table->usThermalControllerOffset),
+		"Thermal controller table not set!", return -1);
+
+	hwmgr->thermal_controller.ucType = thermal_controller->ucType;
+	hwmgr->thermal_controller.ucI2cLine = thermal_controller->ucI2cLine;
+	hwmgr->thermal_controller.ucI2cAddress = thermal_controller->ucI2cAddress;
+
+	hwmgr->thermal_controller.fanInfo.bNoFan =
+		(0 != (thermal_controller->ucFanParameters & ATOM_TONGA_PP_FANPARAMETERS_NOFAN));
+
+	hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution =
+		thermal_controller->ucFanParameters &
+		ATOM_TONGA_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK;
+
+	hwmgr->thermal_controller.fanInfo.ulMinRPM
+		= thermal_controller->ucFanMinRPM * 100UL;
+	hwmgr->thermal_controller.fanInfo.ulMaxRPM
+		= thermal_controller->ucFanMaxRPM * 100UL;
+
+	set_hw_cap(
+			hwmgr,
+			ATOM_TONGA_PP_THERMALCONTROLLER_NONE != hwmgr->thermal_controller.ucType,
+			PHM_PlatformCaps_ThermalController
+		  );
+
+	if (0 == powerplay_table->usFanTableOffset)
+		return 0;
+
+	fan_table = (const PPTable_Generic_SubTable_Header *)
+		(((unsigned long)powerplay_table) +
+		le16_to_cpu(powerplay_table->usFanTableOffset));
+
+	PP_ASSERT_WITH_CODE((0 != powerplay_table->usFanTableOffset),
+		"Fan table not set!", return -1);
+	PP_ASSERT_WITH_CODE((0 < fan_table->ucRevId),
+		"Unsupported fan table format!", return -1);
+
+	hwmgr->thermal_controller.advanceFanControlParameters.ulCycleDelay
+		= 100000;
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_MicrocodeFanControl);
+
+	if (fan_table->ucRevId < 8) {
+		const ATOM_Tonga_Fan_Table *tonga_fan_table =
+			(ATOM_Tonga_Fan_Table *)fan_table;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst
+			= tonga_fan_table->ucTHyst;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMin
+			= tonga_fan_table->usTMin;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMed
+			= tonga_fan_table->usTMed;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTHigh
+			= tonga_fan_table->usTHigh;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin
+			= tonga_fan_table->usPWMMin;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed
+			= tonga_fan_table->usPWMMed;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh
+			= tonga_fan_table->usPWMHigh;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMax
+			= 10900;                  /* hard coded */
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMax
+			= tonga_fan_table->usTMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucFanControlMode
+			= tonga_fan_table->ucFanControlMode;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM
+			= tonga_fan_table->usFanPWMMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultFanOutputSensitivity
+			= 4836;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity
+			= tonga_fan_table->usFanOutputSensitivity;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM
+			= tonga_fan_table->usFanRPMMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit
+			= (tonga_fan_table->ulMinFanSCLKAcousticLimit / 100); /* PPTable stores it in 10Khz unit for 2 decimal places.  SMC wants MHz. */
+		hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature
+			= tonga_fan_table->ucTargetTemperature;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit
+			= tonga_fan_table->ucMinimumPWMLimit;
+	} else {
+		const ATOM_Fiji_Fan_Table *fiji_fan_table =
+			(ATOM_Fiji_Fan_Table *)fan_table;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst
+			= fiji_fan_table->ucTHyst;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMin
+			= fiji_fan_table->usTMin;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMed
+			= fiji_fan_table->usTMed;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTHigh
+			= fiji_fan_table->usTHigh;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin
+			= fiji_fan_table->usPWMMin;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed
+			= fiji_fan_table->usPWMMed;
+		hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh
+			= fiji_fan_table->usPWMHigh;
+		hwmgr->thermal_controller.advanceFanControlParameters.usTMax
+			= fiji_fan_table->usTMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucFanControlMode
+			= fiji_fan_table->ucFanControlMode;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM
+			= fiji_fan_table->usFanPWMMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultFanOutputSensitivity
+			= 4836;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity
+			= fiji_fan_table->usFanOutputSensitivity;
+		hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM
+			= fiji_fan_table->usFanRPMMax;
+		hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit
+			= (fiji_fan_table->ulMinFanSCLKAcousticLimit / 100); /* PPTable stores it in 10Khz unit for 2 decimal places.  SMC wants MHz. */
+		hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature
+			= fiji_fan_table->ucTargetTemperature;
+		hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit
+			= fiji_fan_table->ucMinimumPWMLimit;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainEdge
+			= fiji_fan_table->usFanGainEdge;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainHotspot
+			= fiji_fan_table->usFanGainHotspot;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainLiquid
+			= fiji_fan_table->usFanGainLiquid;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainVrVddc
+			= fiji_fan_table->usFanGainVrVddc;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainVrMvdd
+			= fiji_fan_table->usFanGainVrMvdd;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainPlx
+			= fiji_fan_table->usFanGainPlx;
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanGainHbm
+			= fiji_fan_table->usFanGainHbm;
+	}
+
+	return 0;
+}
+
+/**
+ * Private Function used during initialization.
+ * Inspect the PowerPlay table for obvious signs of corruption.
+ * @param hwmgr Pointer to the hardware manager.
+ * @param powerplay_table Pointer to the PowerPlay Table.
+ * @exception 2 if the powerplay table is incorrect.
+ */
+static int check_powerplay_tables(
+		struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table
+		)
+{
+	const ATOM_Tonga_State_Array *state_arrays;
+
+	state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)powerplay_table) +
+		le16_to_cpu(powerplay_table->usStateArrayOffset));
+
+	PP_ASSERT_WITH_CODE((ATOM_Tonga_TABLE_REVISION_TONGA <=
+		powerplay_table->sHeader.ucTableFormatRevision),
+		"Unsupported PPTable format!", return -1);
+	PP_ASSERT_WITH_CODE((0 != powerplay_table->usStateArrayOffset),
+		"State table is not set!", return -1);
+	PP_ASSERT_WITH_CODE((0 < powerplay_table->sHeader.usStructureSize),
+		"Invalid PowerPlay Table!", return -1);
+	PP_ASSERT_WITH_CODE((0 < state_arrays->ucNumEntries),
+		"Invalid PowerPlay Table!", return -1);
+
+	return 0;
+}
+
+int tonga_pp_tables_initialize(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	const ATOM_Tonga_POWERPLAYTABLE *powerplay_table;
+
+	hwmgr->pptable = kzalloc(sizeof(struct phm_ppt_v1_information), GFP_KERNEL);
+
+	PP_ASSERT_WITH_CODE((NULL != hwmgr->pptable),
+			    "Failed to allocate hwmgr->pptable!", return -ENOMEM);
+
+	memset(hwmgr->pptable, 0x00, sizeof(struct phm_ppt_v1_information));
+
+	powerplay_table = get_powerplay_table(hwmgr);
+
+	PP_ASSERT_WITH_CODE((NULL != powerplay_table),
+		"Missing PowerPlay Table!", return -1);
+
+	result = check_powerplay_tables(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "check_powerplay_tables failed", return result);
+
+	result = set_platform_caps(hwmgr,
+				   le32_to_cpu(powerplay_table->ulPlatformCaps));
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "set_platform_caps failed", return result);
+
+	result = init_thermal_controller(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_thermal_controller failed", return result);
+
+	result = init_over_drive_limits(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_over_drive_limits failed", return result);
+
+	result = init_clock_voltage_dependency(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_clock_voltage_dependency failed", return result);
+
+	result = init_dpm_2_parameters(hwmgr, powerplay_table);
+
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "init_dpm_2_parameters failed", return result);
+
+	return result;
+}
+
+int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	struct phm_ppt_v1_information *pp_table_information =
+		(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (NULL != hwmgr->soft_pp_table) {
+		kfree(hwmgr->soft_pp_table);
+		hwmgr->soft_pp_table = NULL;
+	}
+
+	if (NULL != pp_table_information->vdd_dep_on_sclk)
+		pp_table_information->vdd_dep_on_sclk = NULL;
+
+	if (NULL != pp_table_information->vdd_dep_on_mclk)
+		pp_table_information->vdd_dep_on_mclk = NULL;
+
+	if (NULL != pp_table_information->valid_mclk_values)
+		pp_table_information->valid_mclk_values = NULL;
+
+	if (NULL != pp_table_information->valid_sclk_values)
+		pp_table_information->valid_sclk_values = NULL;
+
+	if (NULL != pp_table_information->vddc_lookup_table)
+		pp_table_information->vddc_lookup_table = NULL;
+
+	if (NULL != pp_table_information->vddgfx_lookup_table)
+		pp_table_information->vddgfx_lookup_table = NULL;
+
+	if (NULL != pp_table_information->mm_dep_table)
+		pp_table_information->mm_dep_table = NULL;
+
+	if (NULL != pp_table_information->cac_dtp_table)
+		pp_table_information->cac_dtp_table = NULL;
+
+	if (NULL != hwmgr->dyn_state.cac_dtp_table)
+		hwmgr->dyn_state.cac_dtp_table = NULL;
+
+	if (NULL != pp_table_information->ppm_parameter_table)
+		pp_table_information->ppm_parameter_table = NULL;
+
+	if (NULL != pp_table_information->pcie_table)
+		pp_table_information->pcie_table = NULL;
+
+	if (NULL != hwmgr->pptable) {
+		kfree(hwmgr->pptable);
+		hwmgr->pptable = NULL;
+	}
+
+	return result;
+}
+
+const struct pp_table_func tonga_pptable_funcs = {
+	.pptable_init = tonga_pp_tables_initialize,
+	.pptable_fini = tonga_pp_tables_uninitialize,
+};
+
+int tonga_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr)
+{
+	const ATOM_Tonga_State_Array * state_arrays;
+	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+
+	PP_ASSERT_WITH_CODE((NULL != pp_table),
+			"Missing PowerPlay Table!", return -1);
+	PP_ASSERT_WITH_CODE((pp_table->sHeader.ucTableFormatRevision >=
+			ATOM_Tonga_TABLE_REVISION_TONGA),
+			"Incorrect PowerPlay table revision!", return -1);
+
+	state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)pp_table) +
+			le16_to_cpu(pp_table->usStateArrayOffset));
+
+	return (uint32_t)(state_arrays->ucNumEntries);
+}
+
+/**
+* Private function to convert flags stored in the BIOS to software flags in PowerPlay.
+*/
+static uint32_t make_classification_flags(struct pp_hwmgr *hwmgr,
+		uint16_t classification, uint16_t classification2)
+{
+	uint32_t result = 0;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_BOOT)
+		result |= PP_StateClassificationFlag_Boot;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+		result |= PP_StateClassificationFlag_Thermal;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE)
+		result |= PP_StateClassificationFlag_LimitedPowerSource;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_REST)
+		result |= PP_StateClassificationFlag_Rest;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_FORCED)
+		result |= PP_StateClassificationFlag_Forced;
+
+	if (classification & ATOM_PPLIB_CLASSIFICATION_ACPI)
+		result |= PP_StateClassificationFlag_ACPI;
+
+	if (classification2 & ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2)
+		result |= PP_StateClassificationFlag_LimitedPowerSource_2;
+
+	return result;
+}
+
+/**
+* Create a Power State out of an entry in the PowerPlay table.
+* This function is called by the hardware back-end.
+* @param hwmgr Pointer to the hardware manager.
+* @param entry_index The index of the entry to be extracted from the table.
+* @param power_state The address of the PowerState instance being created.
+* @return -1 if the entry cannot be retrieved.
+*/
+int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr,
+		uint32_t entry_index, struct pp_power_state *power_state,
+		int (*call_back_func)(struct pp_hwmgr *, void *,
+				struct pp_power_state *, void *, uint32_t))
+{
+	int result = 0;
+	const ATOM_Tonga_State_Array * state_arrays;
+	const ATOM_Tonga_State *state_entry;
+	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+
+	PP_ASSERT_WITH_CODE((NULL != pp_table), "Missing PowerPlay Table!", return -1;);
+	power_state->classification.bios_index = entry_index;
+
+	if (pp_table->sHeader.ucTableFormatRevision >=
+			ATOM_Tonga_TABLE_REVISION_TONGA) {
+		state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)pp_table) +
+				le16_to_cpu(pp_table->usStateArrayOffset));
+
+		PP_ASSERT_WITH_CODE((0 < pp_table->usStateArrayOffset),
+				"Invalid PowerPlay Table State Array Offset.", return -1);
+		PP_ASSERT_WITH_CODE((0 < state_arrays->ucNumEntries),
+				"Invalid PowerPlay Table State Array.", return -1);
+		PP_ASSERT_WITH_CODE((entry_index <= state_arrays->ucNumEntries),
+				"Invalid PowerPlay Table State Array Entry.", return -1);
+
+		state_entry = &(state_arrays->states[entry_index]);
+
+		result = call_back_func(hwmgr, (void *)state_entry, power_state,
+				(void *)pp_table,
+				make_classification_flags(hwmgr,
+					le16_to_cpu(state_entry->usClassification),
+					le16_to_cpu(state_entry->usClassification2)));
+	}
+
+	if (!result && (power_state->classification.flags &
+			PP_StateClassificationFlag_Boot))
+		result = hwmgr->hwmgr_func->patch_boot_state(hwmgr, &(power_state->hardware));
+
+	return result;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.h
new file mode 100644
index 000000000000..d24b8887f466
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef TONGA_PROCESSPPTABLES_H
+#define TONGA_PROCESSPPTABLES_H
+
+#include "hwmgr.h"
+
+extern const struct pp_table_func tonga_pptable_funcs;
+extern int tonga_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr);
+extern int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr, uint32_t entry_index,
+		struct pp_power_state *power_state, int (*call_back_func)(struct pp_hwmgr *, void *,
+				struct pp_power_state *, void *, uint32_t));
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c
new file mode 100644
index 000000000000..a188174747c9
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c
@@ -0,0 +1,590 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <asm/div64.h>
+#include "tonga_thermal.h"
+#include "tonga_hwmgr.h"
+#include "tonga_smumgr.h"
+#include "tonga_ppsmc.h"
+#include "smu/smu_7_1_2_d.h"
+#include "smu/smu_7_1_2_sh_mask.h"
+
+/**
+* Get Fan Speed Control Parameters.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pSpeed is the address of the structure where the result is to be placed.
+* @exception Always succeeds except if we cannot zero out the output structure.
+*/
+int tonga_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info)
+{
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	fan_speed_info->supports_percent_read = true;
+	fan_speed_info->supports_percent_write = true;
+	fan_speed_info->min_percent = 0;
+	fan_speed_info->max_percent = 100;
+
+	if (0 != hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution) {
+		fan_speed_info->supports_rpm_read = true;
+		fan_speed_info->supports_rpm_write = true;
+		fan_speed_info->min_rpm = hwmgr->thermal_controller.fanInfo.ulMinRPM;
+		fan_speed_info->max_rpm = hwmgr->thermal_controller.fanInfo.ulMaxRPM;
+	} else {
+		fan_speed_info->min_rpm = 0;
+		fan_speed_info->max_rpm = 0;
+	}
+
+	return 0;
+}
+
+/**
+* Get Fan Speed in percent.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pSpeed is the address of the structure where the result is to be placed.
+* @exception Fails is the 100% setting appears to be 0.
+*/
+int tonga_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed)
+{
+	uint32_t duty100;
+	uint32_t duty;
+	uint64_t tmp64;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
+	duty = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_STATUS, FDO_PWM_DUTY);
+
+	if (0 == duty100)
+		return -EINVAL;
+
+
+	tmp64 = (uint64_t)duty * 100;
+	do_div(tmp64, duty100);
+	*speed = (uint32_t)tmp64;
+
+	if (*speed > 100)
+		*speed = 100;
+
+	return 0;
+}
+
+/**
+* Get Fan Speed in RPM.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    speed is the address of the structure where the result is to be placed.
+* @exception Returns not supported if no fan is found or if pulses per revolution are not set
+*/
+int tonga_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
+{
+	return 0;
+}
+
+/**
+* Set Fan Speed Control to static mode, so that the user can decide what speed to use.
+* @param    hwmgr  the address of the powerplay hardware manager.
+*           mode    the fan control mode, 0 default, 1 by percent, 5, by RPM
+* @exception Should always succeed.
+*/
+int tonga_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+{
+
+	if (hwmgr->fan_ctrl_is_in_default_mode) {
+		hwmgr->fan_ctrl_default_mode = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE);
+		hwmgr->tmin = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN);
+		hwmgr->fan_ctrl_is_in_default_mode = false;
+	}
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN, 0);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE, mode);
+
+	return 0;
+}
+
+/**
+* Reset Fan Speed Control to default mode.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @exception Should always succeed.
+*/
+int tonga_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
+{
+	if (!hwmgr->fan_ctrl_is_in_default_mode) {
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE, hwmgr->fan_ctrl_default_mode);
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN, hwmgr->tmin);
+		hwmgr->fan_ctrl_is_in_default_mode = true;
+	}
+
+	return 0;
+}
+
+int tonga_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ODFuzzyFanControlSupport)) {
+		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_FUZZY);
+		result = (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl) == 0) ?  0 : -EINVAL;
+/*
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_FanSpeedInTableIsRPM))
+			hwmgr->set_max_fan_rpm_output(hwmgr, hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM);
+		else
+			hwmgr->set_max_fan_pwm_output(hwmgr, hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM);
+*/
+	} else {
+		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_TABLE);
+		result = (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl) == 0) ?  0 : -EINVAL;
+	}
+/* TO DO FOR SOME DEVICE ID 0X692b, send this msg return invalid command.
+	if (result == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature != 0)
+		result = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanTemperatureTarget, \
+								hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature) ? 0 : -EINVAL);
+*/
+	return result;
+}
+
+
+int tonga_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
+{
+	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StopFanControl) == 0) ?  0 : -EINVAL;
+}
+
+/**
+* Set Fan Speed in percent.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    speed is the percentage value (0% - 100%) to be set.
+* @exception Fails is the 100% setting appears to be 0.
+*/
+int tonga_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed)
+{
+	uint32_t duty100;
+	uint32_t duty;
+	uint64_t tmp64;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return -EINVAL;
+
+	if (speed > 100)
+		speed = 100;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
+		tonga_fan_ctrl_stop_smc_fan_control(hwmgr);
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (0 == duty100)
+		return -EINVAL;
+
+	tmp64 = (uint64_t)speed * 100;
+	do_div(tmp64, duty100);
+	duty = (uint32_t)tmp64;
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL0, FDO_STATIC_DUTY, duty);
+
+	return tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+}
+
+/**
+* Reset Fan Speed to default.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @exception Always succeeds.
+*/
+int tonga_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		return 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl)) {
+		result = tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+		if (0 == result)
+			result = tonga_fan_ctrl_start_smc_fan_control(hwmgr);
+	} else
+		result = tonga_fan_ctrl_set_default_mode(hwmgr);
+
+	return result;
+}
+
+/**
+* Set Fan Speed in RPM.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    speed is the percentage value (min - max) to be set.
+* @exception Fails is the speed not lie between min and max.
+*/
+int tonga_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
+{
+	return 0;
+}
+
+/**
+* Reads the remote temperature from the SIslands thermal controller.
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+int tonga_thermal_get_temperature(struct pp_hwmgr *hwmgr)
+{
+	int temp;
+
+	temp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_STATUS, CTF_TEMP);
+
+/* Bit 9 means the reading is lower than the lowest usable value. */
+	if (0 != (0x200 & temp))
+		temp = TONGA_THERMAL_MAXIMUM_TEMP_READING;
+	else
+		temp = (temp & 0x1ff);
+
+	temp = temp * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	return temp;
+}
+
+/**
+* Set the requested temperature range for high and low alert signals
+*
+* @param    hwmgr The address of the hardware manager.
+* @param    range Temperature range to be programmed for high and low alert signals
+* @exception PP_Result_BadInput if the input data is not valid.
+*/
+static int tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, uint32_t low_temp, uint32_t high_temp)
+{
+	uint32_t low = TONGA_THERMAL_MINIMUM_ALERT_TEMP * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	uint32_t high = TONGA_THERMAL_MAXIMUM_ALERT_TEMP * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	if (low < low_temp)
+		low = low_temp;
+	if (high > high_temp)
+		high = high_temp;
+
+	if (low > high)
+		return -EINVAL;
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL, DIG_THERM_DPM, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+
+	return 0;
+}
+
+/**
+* Programs thermal controller one-time setting registers
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+static int tonga_thermal_initialize(struct pp_hwmgr *hwmgr)
+{
+	if (0 != hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution)
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+						CG_TACH_CTRL, EDGE_PER_REV,
+						hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution - 1);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TACH_PWM_RESP_RATE, 0x28);
+
+	return 0;
+}
+
+/**
+* Enable thermal alerts on the RV770 thermal controller.
+*
+* @param    hwmgr The address of the hardware manager.
+*/
+static int tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr)
+{
+	uint32_t alert;
+
+	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK);
+	alert &= ~(TONGA_THERMAL_HIGH_ALERT_MASK | TONGA_THERMAL_LOW_ALERT_MASK);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK, alert);
+
+	/* send message to SMU to enable internal thermal interrupts */
+	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Enable) == 0) ? 0 : -1;
+}
+
+/**
+* Disable thermal alerts on the RV770 thermal controller.
+* @param    hwmgr The address of the hardware manager.
+*/
+static int tonga_thermal_disable_alert(struct pp_hwmgr *hwmgr)
+{
+	uint32_t alert;
+
+	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK);
+	alert |= (TONGA_THERMAL_HIGH_ALERT_MASK | TONGA_THERMAL_LOW_ALERT_MASK);
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK, alert);
+
+	/* send message to SMU to disable internal thermal interrupts */
+	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Disable) == 0) ? 0 : -1;
+}
+
+/**
+* Uninitialize the thermal controller.
+* Currently just disables alerts.
+* @param    hwmgr The address of the hardware manager.
+*/
+int tonga_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
+{
+	int result = tonga_thermal_disable_alert(hwmgr);
+
+	if (hwmgr->thermal_controller.fanInfo.bNoFan)
+		tonga_fan_ctrl_set_default_mode(hwmgr);
+
+	return result;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+	SMU72_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
+		return 0;
+
+	if (0 == data->fan_table_start) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (0 == duty100) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed - hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh - hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = tonga_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->thermal_controller.advanceFanControlParameters.ulCycleDelay * reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	fan_table.FanControl_GL_Flag = 1;
+
+	res = tonga_copy_bytes_to_smc(hwmgr->smumgr, data->fan_table_start, (uint8_t *)&fan_table, (uint32_t)sizeof(fan_table), data->sram_end);
+/* TO DO FOR SOME DEVICE ID 0X692b, send this msg return invalid command.
+	if (res == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit != 0)
+		res = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanMinPwm, \
+						hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit) ? 0 : -1);
+
+	if (res == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit != 0)
+		res = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanSclkTarget, \
+					hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit) ? 0 : -1);
+
+	if (0 != res)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
+*/
+	return 0;
+}
+
+/**
+* Start the fan control on the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_tonga_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+/* If the fantable setup has failed we could have disabled PHM_PlatformCaps_MicrocodeFanControl even after this function was included in the table.
+ * Make sure that we still think controlling the fan is OK.
+*/
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl)) {
+		tonga_fan_ctrl_start_smc_fan_control(hwmgr);
+		tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+	}
+
+	return 0;
+}
+
+/**
+* Set temperature range for high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tf_tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+	struct PP_TemperatureRange *range = (struct PP_TemperatureRange *)input;
+
+	if (range == NULL)
+		return -EINVAL;
+
+	return tonga_thermal_set_temperature_range(hwmgr, range->min, range->max);
+}
+
+/**
+* Programs one-time setting registers
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from initialize thermal controller routine
+*/
+int tf_tonga_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+    return tonga_thermal_initialize(hwmgr);
+}
+
+/**
+* Enable high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from enable alert routine
+*/
+int tf_tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+	return tonga_thermal_enable_alert(hwmgr);
+}
+
+/**
+* Disable high and low alerts
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from disable alert routine
+*/
+static int tf_tonga_thermal_disable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
+{
+	return tonga_thermal_disable_alert(hwmgr);
+}
+
+static struct phm_master_table_item tonga_thermal_start_thermal_controller_master_list[] = {
+	{ NULL, tf_tonga_thermal_initialize },
+	{ NULL, tf_tonga_thermal_set_temperature_range },
+	{ NULL, tf_tonga_thermal_enable_alert },
+/* We should restrict performance levels to low before we halt the SMC.
+ * On the other hand we are still in boot state when we do this so it would be pointless.
+ * If this assumption changes we have to revisit this table.
+ */
+	{ NULL, tf_tonga_thermal_setup_fan_table},
+	{ NULL, tf_tonga_thermal_start_smc_fan_control},
+	{ NULL, NULL }
+};
+
+static struct phm_master_table_header tonga_thermal_start_thermal_controller_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	tonga_thermal_start_thermal_controller_master_list
+};
+
+static struct phm_master_table_item tonga_thermal_set_temperature_range_master_list[] = {
+	{ NULL, tf_tonga_thermal_disable_alert},
+	{ NULL, tf_tonga_thermal_set_temperature_range},
+	{ NULL, tf_tonga_thermal_enable_alert},
+	{ NULL, NULL }
+};
+
+struct phm_master_table_header tonga_thermal_set_temperature_range_master = {
+	0,
+	PHM_MasterTableFlag_None,
+	tonga_thermal_set_temperature_range_master_list
+};
+
+int tonga_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
+{
+	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
+		tonga_fan_ctrl_set_default_mode(hwmgr);
+	return 0;
+}
+
+/**
+* Initializes the thermal controller related functions in the Hardware Manager structure.
+* @param    hwmgr The address of the hardware manager.
+* @exception Any error code from the low-level communication.
+*/
+int pp_tonga_thermal_initialize(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	result = phm_construct_table(hwmgr, &tonga_thermal_set_temperature_range_master, &(hwmgr->set_temperature_range));
+
+	if (0 == result) {
+		result = phm_construct_table(hwmgr,
+						&tonga_thermal_start_thermal_controller_master,
+						&(hwmgr->start_thermal_controller));
+		if (0 != result)
+			phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
+	}
+
+	if (0 == result)
+		hwmgr->fan_ctrl_is_in_default_mode = true;
+	return result;
+}
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h
new file mode 100644
index 000000000000..aa335f267e25
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef TONGA_THERMAL_H
+#define TONGA_THERMAL_H
+
+#include "hwmgr.h"
+
+#define TONGA_THERMAL_HIGH_ALERT_MASK         0x1
+#define TONGA_THERMAL_LOW_ALERT_MASK          0x2
+
+#define TONGA_THERMAL_MINIMUM_TEMP_READING    -256
+#define TONGA_THERMAL_MAXIMUM_TEMP_READING    255
+
+#define TONGA_THERMAL_MINIMUM_ALERT_TEMP      0
+#define TONGA_THERMAL_MAXIMUM_ALERT_TEMP      255
+
+#define FDO_PWM_MODE_STATIC  1
+#define FDO_PWM_MODE_STATIC_RPM 5
+
+
+extern int tf_tonga_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+extern int tf_tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+extern int tf_tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
+
+extern int tonga_thermal_get_temperature(struct pp_hwmgr *hwmgr);
+extern int tonga_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int tonga_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
+extern int tonga_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int tonga_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
+extern int tonga_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
+extern int tonga_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int tonga_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
+extern int pp_tonga_thermal_initialize(struct pp_hwmgr *hwmgr);
+extern int tonga_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int tonga_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int tonga_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int tonga_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
new file mode 100644
index 000000000000..e61a3e67852e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _AMD_POWERPLAY_H_
+#define _AMD_POWERPLAY_H_
+
+#include <linux/seq_file.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include "amd_shared.h"
+#include "cgs_common.h"
+
+enum amd_pp_event {
+	AMD_PP_EVENT_INITIALIZE = 0,
+	AMD_PP_EVENT_UNINITIALIZE,
+	AMD_PP_EVENT_POWER_SOURCE_CHANGE,
+	AMD_PP_EVENT_SUSPEND,
+	AMD_PP_EVENT_RESUME,
+	AMD_PP_EVENT_ENTER_REST_STATE,
+	AMD_PP_EVENT_EXIT_REST_STATE,
+	AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE,
+	AMD_PP_EVENT_THERMAL_NOTIFICATION,
+	AMD_PP_EVENT_VBIOS_NOTIFICATION,
+	AMD_PP_EVENT_ENTER_THERMAL_STATE,
+	AMD_PP_EVENT_EXIT_THERMAL_STATE,
+	AMD_PP_EVENT_ENTER_FORCED_STATE,
+	AMD_PP_EVENT_EXIT_FORCED_STATE,
+	AMD_PP_EVENT_ENTER_EXCLUSIVE_MODE,
+	AMD_PP_EVENT_EXIT_EXCLUSIVE_MODE,
+	AMD_PP_EVENT_ENTER_SCREEN_SAVER,
+	AMD_PP_EVENT_EXIT_SCREEN_SAVER,
+	AMD_PP_EVENT_VPU_RECOVERY_BEGIN,
+	AMD_PP_EVENT_VPU_RECOVERY_END,
+	AMD_PP_EVENT_ENABLE_POWER_PLAY,
+	AMD_PP_EVENT_DISABLE_POWER_PLAY,
+	AMD_PP_EVENT_CHANGE_POWER_SOURCE_UI_LABEL,
+	AMD_PP_EVENT_ENABLE_USER2D_PERFORMANCE,
+	AMD_PP_EVENT_DISABLE_USER2D_PERFORMANCE,
+	AMD_PP_EVENT_ENABLE_USER3D_PERFORMANCE,
+	AMD_PP_EVENT_DISABLE_USER3D_PERFORMANCE,
+	AMD_PP_EVENT_ENABLE_OVER_DRIVE_TEST,
+	AMD_PP_EVENT_DISABLE_OVER_DRIVE_TEST,
+	AMD_PP_EVENT_ENABLE_REDUCED_REFRESH_RATE,
+	AMD_PP_EVENT_DISABLE_REDUCED_REFRESH_RATE,
+	AMD_PP_EVENT_ENABLE_GFX_CLOCK_GATING,
+	AMD_PP_EVENT_DISABLE_GFX_CLOCK_GATING,
+	AMD_PP_EVENT_ENABLE_CGPG,
+	AMD_PP_EVENT_DISABLE_CGPG,
+	AMD_PP_EVENT_ENTER_TEXT_MODE,
+	AMD_PP_EVENT_EXIT_TEXT_MODE,
+	AMD_PP_EVENT_VIDEO_START,
+	AMD_PP_EVENT_VIDEO_STOP,
+	AMD_PP_EVENT_ENABLE_USER_STATE,
+	AMD_PP_EVENT_DISABLE_USER_STATE,
+	AMD_PP_EVENT_READJUST_POWER_STATE,
+	AMD_PP_EVENT_START_INACTIVITY,
+	AMD_PP_EVENT_STOP_INACTIVITY,
+	AMD_PP_EVENT_LINKED_ADAPTERS_READY,
+	AMD_PP_EVENT_ADAPTER_SAFE_TO_DISABLE,
+	AMD_PP_EVENT_COMPLETE_INIT,
+	AMD_PP_EVENT_CRITICAL_THERMAL_FAULT,
+	AMD_PP_EVENT_BACKLIGHT_CHANGED,
+	AMD_PP_EVENT_ENABLE_VARI_BRIGHT,
+	AMD_PP_EVENT_DISABLE_VARI_BRIGHT,
+	AMD_PP_EVENT_ENABLE_VARI_BRIGHT_ON_POWER_XPRESS,
+	AMD_PP_EVENT_DISABLE_VARI_BRIGHT_ON_POWER_XPRESS,
+	AMD_PP_EVENT_SET_VARI_BRIGHT_LEVEL,
+	AMD_PP_EVENT_VARI_BRIGHT_MONITOR_MEASUREMENT,
+	AMD_PP_EVENT_SCREEN_ON,
+	AMD_PP_EVENT_SCREEN_OFF,
+	AMD_PP_EVENT_PRE_DISPLAY_CONFIG_CHANGE,
+	AMD_PP_EVENT_ENTER_ULP_STATE,
+	AMD_PP_EVENT_EXIT_ULP_STATE,
+	AMD_PP_EVENT_REGISTER_IP_STATE,
+	AMD_PP_EVENT_UNREGISTER_IP_STATE,
+	AMD_PP_EVENT_ENTER_MGPU_MODE,
+	AMD_PP_EVENT_EXIT_MGPU_MODE,
+	AMD_PP_EVENT_ENTER_MULTI_GPU_MODE,
+	AMD_PP_EVENT_PRE_SUSPEND,
+	AMD_PP_EVENT_PRE_RESUME,
+	AMD_PP_EVENT_ENTER_BACOS,
+	AMD_PP_EVENT_EXIT_BACOS,
+	AMD_PP_EVENT_RESUME_BACO,
+	AMD_PP_EVENT_RESET_BACO,
+	AMD_PP_EVENT_PRE_DISPLAY_PHY_ACCESS,
+	AMD_PP_EVENT_POST_DISPLAY_PHY_CCESS,
+	AMD_PP_EVENT_START_COMPUTE_APPLICATION,
+	AMD_PP_EVENT_STOP_COMPUTE_APPLICATION,
+	AMD_PP_EVENT_REDUCE_POWER_LIMIT,
+	AMD_PP_EVENT_ENTER_FRAME_LOCK,
+	AMD_PP_EVENT_EXIT_FRAME_LOOCK,
+	AMD_PP_EVENT_LONG_IDLE_REQUEST_BACO,
+	AMD_PP_EVENT_LONG_IDLE_ENTER_BACO,
+	AMD_PP_EVENT_LONG_IDLE_EXIT_BACO,
+	AMD_PP_EVENT_HIBERNATE,
+	AMD_PP_EVENT_CONNECTED_STANDBY,
+	AMD_PP_EVENT_ENTER_SELF_REFRESH,
+	AMD_PP_EVENT_EXIT_SELF_REFRESH,
+	AMD_PP_EVENT_START_AVFS_BTC,
+	AMD_PP_EVENT_MAX
+};
+
+enum amd_dpm_forced_level {
+	AMD_DPM_FORCED_LEVEL_AUTO = 0,
+	AMD_DPM_FORCED_LEVEL_LOW = 1,
+	AMD_DPM_FORCED_LEVEL_HIGH = 2,
+};
+
+struct amd_pp_init {
+	struct cgs_device *device;
+	uint32_t chip_family;
+	uint32_t chip_id;
+	uint32_t rev_id;
+};
+enum amd_pp_display_config_type{
+	AMD_PP_DisplayConfigType_None = 0,
+	AMD_PP_DisplayConfigType_DP54 ,
+	AMD_PP_DisplayConfigType_DP432 ,
+	AMD_PP_DisplayConfigType_DP324 ,
+	AMD_PP_DisplayConfigType_DP27,
+	AMD_PP_DisplayConfigType_DP243,
+	AMD_PP_DisplayConfigType_DP216,
+	AMD_PP_DisplayConfigType_DP162,
+	AMD_PP_DisplayConfigType_HDMI6G ,
+	AMD_PP_DisplayConfigType_HDMI297 ,
+	AMD_PP_DisplayConfigType_HDMI162,
+	AMD_PP_DisplayConfigType_LVDS,
+	AMD_PP_DisplayConfigType_DVI,
+	AMD_PP_DisplayConfigType_WIRELESS,
+	AMD_PP_DisplayConfigType_VGA
+};
+
+struct single_display_configuration
+{
+	uint32_t controller_index;
+	uint32_t controller_id;
+	uint32_t signal_type;
+	uint32_t display_state;
+	/* phy id for the primary internal transmitter */
+	uint8_t primary_transmitter_phyi_d;
+	/* bitmap with the active lanes */
+	uint8_t primary_transmitter_active_lanemap;
+	/* phy id for the secondary internal transmitter (for dual-link dvi) */
+	uint8_t secondary_transmitter_phy_id;
+	/* bitmap with the active lanes */
+	uint8_t secondary_transmitter_active_lanemap;
+	/* misc phy settings for SMU. */
+	uint32_t config_flags;
+	uint32_t display_type;
+	uint32_t view_resolution_cx;
+	uint32_t view_resolution_cy;
+	enum amd_pp_display_config_type displayconfigtype;
+	uint32_t vertical_refresh; /* for active display */
+};
+
+#define MAX_NUM_DISPLAY 32
+
+struct amd_pp_display_configuration {
+	bool nb_pstate_switch_disable;/* controls NB PState switch */
+	bool cpu_cc6_disable; /* controls CPU CState switch ( on or off) */
+	bool cpu_pstate_disable;
+	uint32_t cpu_pstate_separation_time;
+
+	uint32_t num_display;  /* total number of display*/
+	uint32_t num_path_including_non_display;
+	uint32_t crossfire_display_index;
+	uint32_t min_mem_set_clock;
+	uint32_t min_core_set_clock;
+	/* unit 10KHz x bit*/
+	uint32_t min_bus_bandwidth;
+	/* minimum required stutter sclk, in 10khz uint32_t ulMinCoreSetClk;*/
+	uint32_t min_core_set_clock_in_sr;
+
+	struct single_display_configuration displays[MAX_NUM_DISPLAY];
+
+	uint32_t vrefresh; /* for active display*/
+
+	uint32_t min_vblank_time; /* for active display*/
+	bool multi_monitor_in_sync;
+	/* Controller Index of primary display - used in MCLK SMC switching hang
+	 * SW Workaround*/
+	uint32_t crtc_index;
+	/* htotal*1000/pixelclk - used in MCLK SMC switching hang SW Workaround*/
+	uint32_t line_time_in_us;
+	bool invalid_vblank_time;
+
+	uint32_t display_clk;
+	/*
+	 * for given display configuration if multimonitormnsync == false then
+	 * Memory clock DPMS with this latency or below is allowed, DPMS with
+	 * higher latency not allowed.
+	 */
+	uint32_t dce_tolerable_mclk_in_active_latency;
+};
+
+struct amd_pp_dal_clock_info {
+	uint32_t	engine_max_clock;
+	uint32_t	memory_max_clock;
+	uint32_t	level;
+};
+
+enum {
+	PP_GROUP_UNKNOWN = 0,
+	PP_GROUP_GFX = 1,
+	PP_GROUP_SYS,
+	PP_GROUP_MAX
+};
+
+#define PP_GROUP_MASK        0xF0000000
+#define PP_GROUP_SHIFT       28
+
+#define PP_BLOCK_MASK        0x0FFFFF00
+#define PP_BLOCK_SHIFT       8
+
+#define PP_BLOCK_GFX_CG         0x01
+#define PP_BLOCK_GFX_MG         0x02
+#define PP_BLOCK_SYS_BIF        0x01
+#define PP_BLOCK_SYS_MC         0x02
+#define PP_BLOCK_SYS_ROM        0x04
+#define PP_BLOCK_SYS_DRM        0x08
+#define PP_BLOCK_SYS_HDP        0x10
+#define PP_BLOCK_SYS_SDMA       0x20
+
+#define PP_STATE_MASK           0x0000000F
+#define PP_STATE_SHIFT          0
+#define PP_STATE_SUPPORT_MASK   0x000000F0
+#define PP_STATE_SUPPORT_SHIFT  0
+
+#define PP_STATE_CG             0x01
+#define PP_STATE_LS             0x02
+#define PP_STATE_DS             0x04
+#define PP_STATE_SD             0x08
+#define PP_STATE_SUPPORT_CG     0x10
+#define PP_STATE_SUPPORT_LS     0x20
+#define PP_STATE_SUPPORT_DS     0x40
+#define PP_STATE_SUPPORT_SD     0x80
+
+#define PP_CG_MSG_ID(group, block, support, state) (group << PP_GROUP_SHIFT |\
+								block << PP_BLOCK_SHIFT |\
+								support << PP_STATE_SUPPORT_SHIFT |\
+								state << PP_STATE_SHIFT)
+
+struct amd_powerplay_funcs {
+	int (*get_temperature)(void *handle);
+	int (*load_firmware)(void *handle);
+	int (*wait_for_fw_loading_complete)(void *handle);
+	int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
+	enum amd_dpm_forced_level (*get_performance_level)(void *handle);
+	enum amd_pm_state_type (*get_current_power_state)(void *handle);
+	int (*get_sclk)(void *handle, bool low);
+	int (*get_mclk)(void *handle, bool low);
+	int (*powergate_vce)(void *handle, bool gate);
+	int (*powergate_uvd)(void *handle, bool gate);
+	int (*dispatch_tasks)(void *handle, enum amd_pp_event event_id,
+				   void *input, void *output);
+	void (*print_current_performance_level)(void *handle,
+						      struct seq_file *m);
+	int (*set_fan_control_mode)(void *handle, uint32_t mode);
+	int (*get_fan_control_mode)(void *handle);
+	int (*set_fan_speed_percent)(void *handle, uint32_t percent);
+	int (*get_fan_speed_percent)(void *handle, uint32_t *speed);
+};
+
+struct amd_powerplay {
+	void *pp_handle;
+	const struct amd_ip_funcs *ip_funcs;
+	const struct amd_powerplay_funcs *pp_funcs;
+};
+
+int amd_powerplay_init(struct amd_pp_init *pp_init,
+		       struct amd_powerplay *amd_pp);
+int amd_powerplay_fini(void *handle);
+
+int amd_powerplay_display_configuration_change(void *handle, const void *input);
+
+int amd_powerplay_get_display_power_level(void *handle,
+		struct amd_pp_dal_clock_info *output);
+
+
+#endif /* _AMD_POWERPLAY_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/cz_ppsmc.h
index 273616ab43db..9b698780aed8 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/cz_ppsmc.h
@@ -164,6 +164,7 @@ enum DPM_ARRAY {
 #define PPSMC_MSG_SetLoggerAddressHigh        ((uint16_t) 0x26C)
 #define PPSMC_MSG_SetLoggerAddressLow         ((uint16_t) 0x26D)
 #define PPSMC_MSG_SetWatermarkFrequency       ((uint16_t) 0x26E)
+#define PPSMC_MSG_SetDisplaySizePowerParams   ((uint16_t) 0x26F)
 
 /* REMOVE LATER*/
 #define PPSMC_MSG_DPM_ForceState              ((uint16_t) 0x104)
diff --git a/drivers/gpu/drm/amd/powerplay/inc/eventmanager.h b/drivers/gpu/drm/amd/powerplay/inc/eventmanager.h
new file mode 100644
index 000000000000..b9d84de8a44d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/eventmanager.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _EVENT_MANAGER_H_
+#define _EVENT_MANAGER_H_
+
+#include "power_state.h"
+#include "pp_power_source.h"
+#include "hardwaremanager.h"
+#include "pp_asicblocks.h"
+
+struct pp_eventmgr;
+enum amd_pp_event;
+
+enum PEM_EventDataValid {
+	PEM_EventDataValid_RequestedStateID = 0,
+	PEM_EventDataValid_RequestedUILabel,
+	PEM_EventDataValid_NewPowerState,
+	PEM_EventDataValid_RequestedPowerSource,
+	PEM_EventDataValid_RequestedClocks,
+	PEM_EventDataValid_CurrentTemperature,
+	PEM_EventDataValid_AsicBlocks,
+	PEM_EventDataValid_ODParameters,
+	PEM_EventDataValid_PXAdapterPrefs,
+	PEM_EventDataValid_PXUserPrefs,
+	PEM_EventDataValid_PXSwitchReason,
+	PEM_EventDataValid_PXSwitchPhase,
+	PEM_EventDataValid_HdVideo,
+	PEM_EventDataValid_BacklightLevel,
+	PEM_EventDatavalid_VariBrightParams,
+	PEM_EventDataValid_VariBrightLevel,
+	PEM_EventDataValid_VariBrightImmediateChange,
+	PEM_EventDataValid_PercentWhite,
+	PEM_EventDataValid_SdVideo,
+	PEM_EventDataValid_HTLinkChangeReason,
+	PEM_EventDataValid_HWBlocks,
+	PEM_EventDataValid_RequestedThermalState,
+	PEM_EventDataValid_MvcVideo,
+	PEM_EventDataValid_Max
+};
+
+typedef enum PEM_EventDataValid PEM_EventDataValid;
+
+/* Number of bits in ULONG variable */
+#define PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD (sizeof(unsigned long)*8)
+
+/* Number of ULONG entries used by event data valid bits */
+#define PEM_MAX_NUM_EVENTDATAVALID_ULONG_ENTRIES                                 \
+		((PEM_EventDataValid_Max + PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD - 1) /  \
+		PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD)
+
+static inline void pem_set_event_data_valid(unsigned long *fields, PEM_EventDataValid valid_field)
+{
+	fields[valid_field / PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD] |=
+		(1UL << (valid_field % PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD));
+}
+
+static inline void pem_unset_event_data_valid(unsigned long *fields, PEM_EventDataValid valid_field)
+{
+	fields[valid_field / PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD] &=
+		~(1UL << (valid_field % PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD));
+}
+
+static inline unsigned long pem_is_event_data_valid(const unsigned long *fields, PEM_EventDataValid valid_field)
+{
+	return fields[valid_field / PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD] &
+		(1UL << (valid_field % PEM_MAX_NUM_EVENTDATAVALID_BITS_PER_FIELD));
+}
+
+struct pem_event_data {
+	unsigned long	valid_fields[100];
+	unsigned long   requested_state_id;
+	enum PP_StateUILabel requested_ui_label;
+	struct pp_power_state  *pnew_power_state;
+	enum pp_power_source  requested_power_source;
+	struct PP_Clocks       requested_clocks;
+	bool         skip_state_adjust_rules;
+	struct phm_asic_blocks  asic_blocks;
+	/* to doPP_ThermalState requestedThermalState;
+	enum ThermalStateRequestSrc requestThermalStateSrc;
+	PP_Temperature  currentTemperature;*/
+
+};
+
+int pem_handle_event(struct pp_eventmgr *eventmgr, enum amd_pp_event event,
+		     struct pem_event_data *event_data);
+
+bool pem_is_hw_access_blocked(struct pp_eventmgr *eventmgr);
+
+#endif /* _EVENT_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h b/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h
new file mode 100644
index 000000000000..10437dcfd365
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _EVENTMGR_H_
+#define _EVENTMGR_H_
+
+#include <linux/mutex.h>
+#include "pp_instance.h"
+#include "hardwaremanager.h"
+#include "eventmanager.h"
+#include "pp_feature.h"
+#include "pp_power_source.h"
+#include "power_state.h"
+
+typedef int (*pem_event_action)(struct pp_eventmgr *eventmgr,
+				struct pem_event_data *event_data);
+
+struct action_chain {
+	const char *description;  /* action chain description for debugging purpose */
+	const pem_event_action **action_chain; /* pointer to chain of event actions */
+};
+
+struct pem_power_source_ui_state_info {
+	enum PP_StateUILabel current_ui_label;
+	enum PP_StateUILabel default_ui_lable;
+	unsigned long    configurable_ui_mapping;
+};
+
+struct pp_clock_range {
+	uint32_t min_sclk_khz;
+	uint32_t max_sclk_khz;
+
+	uint32_t min_mclk_khz;
+	uint32_t max_mclk_khz;
+
+	uint32_t min_vclk_khz;
+	uint32_t max_vclk_khz;
+
+	uint32_t min_dclk_khz;
+	uint32_t max_dclk_khz;
+
+	uint32_t min_aclk_khz;
+	uint32_t max_aclk_khz;
+
+	uint32_t min_eclk_khz;
+	uint32_t max_eclk_khz;
+};
+
+enum pp_state {
+	UNINITIALIZED,
+	INACTIVE,
+	ACTIVE
+};
+
+enum pp_ring_index {
+	PP_RING_TYPE_GFX_INDEX = 0,
+	PP_RING_TYPE_DMA_INDEX,
+	PP_RING_TYPE_DMA1_INDEX,
+	PP_RING_TYPE_UVD_INDEX,
+	PP_RING_TYPE_VCE0_INDEX,
+	PP_RING_TYPE_VCE1_INDEX,
+	PP_RING_TYPE_CP1_INDEX,
+	PP_RING_TYPE_CP2_INDEX,
+	PP_NUM_RINGS,
+};
+
+struct pp_request {
+	uint32_t flags;
+	uint32_t sclk;
+	uint32_t sclk_throttle;
+	uint32_t mclk;
+	uint32_t vclk;
+	uint32_t dclk;
+	uint32_t eclk;
+	uint32_t aclk;
+	uint32_t iclk;
+	uint32_t vp8clk;
+	uint32_t rsv[32];
+};
+
+struct pp_eventmgr {
+	struct pp_hwmgr	*hwmgr;
+	struct pp_smumgr *smumgr;
+
+	struct pp_feature_info features[PP_Feature_Max];
+	const struct action_chain *event_chain[AMD_PP_EVENT_MAX];
+	struct phm_platform_descriptor   *platform_descriptor;
+	struct pp_clock_range clock_range;
+	enum pp_power_source  current_power_source;
+	struct pem_power_source_ui_state_info  ui_state_info[PP_PowerSource_Max];
+	enum pp_state states[PP_NUM_RINGS];
+	struct pp_request hi_req;
+	struct list_head context_list;
+	struct mutex lock;
+	bool  block_adjust_power_state;
+	bool enable_cg;
+	bool enable_gfx_cgpg;
+	int (*pp_eventmgr_init)(struct pp_eventmgr *eventmgr);
+	void (*pp_eventmgr_fini)(struct pp_eventmgr *eventmgr);
+};
+
+int eventmgr_init(struct pp_instance *handle);
+int eventmgr_fini(struct pp_eventmgr *eventmgr);
+
+#endif /* _EVENTMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/fiji_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/fiji_ppsmc.h
new file mode 100644
index 000000000000..7ae494569a60
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/fiji_ppsmc.h
@@ -0,0 +1,412 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+
+#ifndef _FIJI_PP_SMC_H_
+#define _FIJI_PP_SMC_H_
+
+#pragma pack(push, 1)
+
+#define PPSMC_SWSTATE_FLAG_DC                           0x01
+#define PPSMC_SWSTATE_FLAG_UVD                          0x02
+#define PPSMC_SWSTATE_FLAG_VCE                          0x04
+
+#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
+#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
+#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff
+
+#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01
+#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02
+#define PPSMC_SYSTEMFLAG_GDDR5                          0x04
+
+#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08
+
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
+#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
+
+/* Defines for DPM 2.0 */
+#define PPSMC_DPM2FLAGS_TDPCLMP                         0x01
+#define PPSMC_DPM2FLAGS_PWRSHFT                         0x02
+#define PPSMC_DPM2FLAGS_OCP                             0x04
+
+/* Defines for display watermark level */
+#define PPSMC_DISPLAY_WATERMARK_LOW                     0
+#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
+
+/* In the HW performance level's state flags: */
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    0x01
+#define PPSMC_STATEFLAG_POWERBOOST         0x02
+#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 0x04
+#define PPSMC_STATEFLAG_POWERSHIFT         0x08
+#define PPSMC_STATEFLAG_SLOW_READ_MARGIN   0x10
+#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20
+#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   0x40
+
+/* Fan control algorithm: */
+#define FDO_MODE_HARDWARE 0
+#define FDO_MODE_PIECE_WISE_LINEAR 1
+
+enum FAN_CONTROL {
+  FAN_CONTROL_FUZZY,
+  FAN_CONTROL_TABLE
+};
+
+/* Gemini Modes*/
+#define PPSMC_GeminiModeNone   0  /*Single GPU board*/
+#define PPSMC_GeminiModeMaster 1  /*Master GPU on a Gemini board*/
+#define PPSMC_GeminiModeSlave  2  /*Slave GPU on a Gemini board*/
+
+
+/* Return codes for driver to SMC communication. */
+#define PPSMC_Result_OK             ((uint16_t)0x01)
+#define PPSMC_Result_NoMore         ((uint16_t)0x02)
+
+#define PPSMC_Result_NotNow         ((uint16_t)0x03)
+
+#define PPSMC_Result_Failed         ((uint16_t)0xFF)
+#define PPSMC_Result_UnknownCmd     ((uint16_t)0xFE)
+#define PPSMC_Result_UnknownVT      ((uint16_t)0xFD)
+
+#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
+
+
+#define PPSMC_MSG_Halt                      ((uint16_t)0x10)
+#define PPSMC_MSG_Resume                    ((uint16_t)0x11)
+#define PPSMC_MSG_EnableDPMLevel            ((uint16_t)0x12)
+#define PPSMC_MSG_ZeroLevelsDisabled        ((uint16_t)0x13)
+#define PPSMC_MSG_OneLevelsDisabled         ((uint16_t)0x14)
+#define PPSMC_MSG_TwoLevelsDisabled         ((uint16_t)0x15)
+#define PPSMC_MSG_EnableThermalInterrupt    ((uint16_t)0x16)
+#define PPSMC_MSG_RunningOnAC               ((uint16_t)0x17)
+#define PPSMC_MSG_LevelUp                   ((uint16_t)0x18)
+#define PPSMC_MSG_LevelDown                 ((uint16_t)0x19)
+#define PPSMC_MSG_ResetDPMCounters          ((uint16_t)0x1a)
+#define PPSMC_MSG_SwitchToSwState           ((uint16_t)0x20)
+
+#define PPSMC_MSG_SwitchToSwStateLast       ((uint16_t)0x3f)
+#define PPSMC_MSG_SwitchToInitialState      ((uint16_t)0x40)
+#define PPSMC_MSG_NoForcedLevel             ((uint16_t)0x41)
+#define PPSMC_MSG_ForceHigh                 ((uint16_t)0x42)
+#define PPSMC_MSG_ForceMediumOrHigh         ((uint16_t)0x43)
+
+#define PPSMC_MSG_SwitchToMinimumPower      ((uint16_t)0x51)
+#define PPSMC_MSG_ResumeFromMinimumPower    ((uint16_t)0x52)
+#define PPSMC_MSG_EnableCac                 ((uint16_t)0x53)
+#define PPSMC_MSG_DisableCac                ((uint16_t)0x54)
+#define PPSMC_DPMStateHistoryStart          ((uint16_t)0x55)
+#define PPSMC_DPMStateHistoryStop           ((uint16_t)0x56)
+#define PPSMC_CACHistoryStart               ((uint16_t)0x57)
+#define PPSMC_CACHistoryStop                ((uint16_t)0x58)
+#define PPSMC_TDPClampingActive             ((uint16_t)0x59)
+#define PPSMC_TDPClampingInactive           ((uint16_t)0x5A)
+#define PPSMC_StartFanControl               ((uint16_t)0x5B)
+#define PPSMC_StopFanControl                ((uint16_t)0x5C)
+#define PPSMC_NoDisplay                     ((uint16_t)0x5D)
+#define PPSMC_HasDisplay                    ((uint16_t)0x5E)
+#define PPSMC_MSG_UVDPowerOFF               ((uint16_t)0x60)
+#define PPSMC_MSG_UVDPowerON                ((uint16_t)0x61)
+#define PPSMC_MSG_EnableULV                 ((uint16_t)0x62)
+#define PPSMC_MSG_DisableULV                ((uint16_t)0x63)
+#define PPSMC_MSG_EnterULV                  ((uint16_t)0x64)
+#define PPSMC_MSG_ExitULV                   ((uint16_t)0x65)
+#define PPSMC_PowerShiftActive              ((uint16_t)0x6A)
+#define PPSMC_PowerShiftInactive            ((uint16_t)0x6B)
+#define PPSMC_OCPActive                     ((uint16_t)0x6C)
+#define PPSMC_OCPInactive                   ((uint16_t)0x6D)
+#define PPSMC_CACLongTermAvgEnable          ((uint16_t)0x6E)
+#define PPSMC_CACLongTermAvgDisable         ((uint16_t)0x6F)
+#define PPSMC_MSG_InferredStateSweep_Start  ((uint16_t)0x70)
+#define PPSMC_MSG_InferredStateSweep_Stop   ((uint16_t)0x71)
+#define PPSMC_MSG_SwitchToLowestInfState    ((uint16_t)0x72)
+#define PPSMC_MSG_SwitchToNonInfState       ((uint16_t)0x73)
+#define PPSMC_MSG_AllStateSweep_Start       ((uint16_t)0x74)
+#define PPSMC_MSG_AllStateSweep_Stop        ((uint16_t)0x75)
+#define PPSMC_MSG_SwitchNextLowerInfState   ((uint16_t)0x76)
+#define PPSMC_MSG_SwitchNextHigherInfState  ((uint16_t)0x77)
+#define PPSMC_MSG_MclkRetrainingTest        ((uint16_t)0x78)
+#define PPSMC_MSG_ForceTDPClamping          ((uint16_t)0x79)
+#define PPSMC_MSG_CollectCAC_PowerCorreln   ((uint16_t)0x7A)
+#define PPSMC_MSG_CollectCAC_WeightCalib    ((uint16_t)0x7B)
+#define PPSMC_MSG_CollectCAC_SQonly         ((uint16_t)0x7C)
+#define PPSMC_MSG_CollectCAC_TemperaturePwr ((uint16_t)0x7D)
+
+#define PPSMC_MSG_ExtremitiesTest_Start     ((uint16_t)0x7E)
+#define PPSMC_MSG_ExtremitiesTest_Stop      ((uint16_t)0x7F)
+#define PPSMC_FlushDataCache                ((uint16_t)0x80)
+#define PPSMC_FlushInstrCache               ((uint16_t)0x81)
+
+#define PPSMC_MSG_SetEnabledLevels          ((uint16_t)0x82)
+#define PPSMC_MSG_SetForcedLevels           ((uint16_t)0x83)
+
+#define PPSMC_MSG_ResetToDefaults           ((uint16_t)0x84)
+
+#define PPSMC_MSG_SetForcedLevelsAndJump      ((uint16_t)0x85)
+#define PPSMC_MSG_SetCACHistoryMode           ((uint16_t)0x86)
+#define PPSMC_MSG_EnableDTE                   ((uint16_t)0x87)
+#define PPSMC_MSG_DisableDTE                  ((uint16_t)0x88)
+
+#define PPSMC_MSG_SmcSpaceSetAddress          ((uint16_t)0x89)
+
+#define PPSMC_MSG_BREAK                       ((uint16_t)0xF8)
+
+/* Trinity Specific Messages*/
+#define PPSMC_MSG_Test                        ((uint16_t) 0x100)
+#define PPSMC_MSG_DPM_Voltage_Pwrmgt          ((uint16_t) 0x101)
+#define PPSMC_MSG_DPM_Config                  ((uint16_t) 0x102)
+#define PPSMC_MSG_PM_Controller_Start         ((uint16_t) 0x103)
+#define PPSMC_MSG_DPM_ForceState              ((uint16_t) 0x104)
+#define PPSMC_MSG_PG_PowerDownSIMD            ((uint16_t) 0x105)
+#define PPSMC_MSG_PG_PowerUpSIMD              ((uint16_t) 0x106)
+#define PPSMC_MSG_PM_Controller_Stop          ((uint16_t) 0x107)
+#define PPSMC_MSG_PG_SIMD_Config              ((uint16_t) 0x108)
+#define PPSMC_MSG_Voltage_Cntl_Enable         ((uint16_t) 0x109)
+#define PPSMC_MSG_Thermal_Cntl_Enable         ((uint16_t) 0x10a)
+#define PPSMC_MSG_Reset_Service               ((uint16_t) 0x10b)
+#define PPSMC_MSG_VCEPowerOFF                 ((uint16_t) 0x10e)
+#define PPSMC_MSG_VCEPowerON                  ((uint16_t) 0x10f)
+#define PPSMC_MSG_DPM_Disable_VCE_HS          ((uint16_t) 0x110)
+#define PPSMC_MSG_DPM_Enable_VCE_HS           ((uint16_t) 0x111)
+#define PPSMC_MSG_DPM_N_LevelsDisabled        ((uint16_t) 0x112)
+#define PPSMC_MSG_DCEPowerOFF                 ((uint16_t) 0x113)
+#define PPSMC_MSG_DCEPowerON                  ((uint16_t) 0x114)
+#define PPSMC_MSG_PCIE_DDIPowerDown           ((uint16_t) 0x117)
+#define PPSMC_MSG_PCIE_DDIPowerUp             ((uint16_t) 0x118)
+#define PPSMC_MSG_PCIE_CascadePLLPowerDown    ((uint16_t) 0x119)
+#define PPSMC_MSG_PCIE_CascadePLLPowerUp      ((uint16_t) 0x11a)
+#define PPSMC_MSG_SYSPLLPowerOff              ((uint16_t) 0x11b)
+#define PPSMC_MSG_SYSPLLPowerOn               ((uint16_t) 0x11c)
+#define PPSMC_MSG_DCE_RemoveVoltageAdjustment ((uint16_t) 0x11d)
+#define PPSMC_MSG_DCE_AllowVoltageAdjustment  ((uint16_t) 0x11e)
+#define PPSMC_MSG_DISPLAYPHYStatusNotify      ((uint16_t) 0x11f)
+#define PPSMC_MSG_EnableBAPM                  ((uint16_t) 0x120)
+#define PPSMC_MSG_DisableBAPM                 ((uint16_t) 0x121)
+#define PPSMC_MSG_Spmi_Enable                 ((uint16_t) 0x122)
+#define PPSMC_MSG_Spmi_Timer                  ((uint16_t) 0x123)
+#define PPSMC_MSG_LCLK_DPM_Config             ((uint16_t) 0x124)
+#define PPSMC_MSG_VddNB_Request               ((uint16_t) 0x125)
+#define PPSMC_MSG_PCIE_DDIPhyPowerDown        ((uint32_t) 0x126)
+#define PPSMC_MSG_PCIE_DDIPhyPowerUp          ((uint32_t) 0x127)
+#define PPSMC_MSG_MCLKDPM_Config              ((uint16_t) 0x128)
+
+#define PPSMC_MSG_UVDDPM_Config               ((uint16_t) 0x129)
+#define PPSMC_MSG_VCEDPM_Config               ((uint16_t) 0x12A)
+#define PPSMC_MSG_ACPDPM_Config               ((uint16_t) 0x12B)
+#define PPSMC_MSG_SAMUDPM_Config              ((uint16_t) 0x12C)
+#define PPSMC_MSG_UVDDPM_SetEnabledMask       ((uint16_t) 0x12D)
+#define PPSMC_MSG_VCEDPM_SetEnabledMask       ((uint16_t) 0x12E)
+#define PPSMC_MSG_ACPDPM_SetEnabledMask       ((uint16_t) 0x12F)
+#define PPSMC_MSG_SAMUDPM_SetEnabledMask      ((uint16_t) 0x130)
+#define PPSMC_MSG_MCLKDPM_ForceState          ((uint16_t) 0x131)
+#define PPSMC_MSG_MCLKDPM_NoForcedLevel       ((uint16_t) 0x132)
+#define PPSMC_MSG_Thermal_Cntl_Disable        ((uint16_t) 0x133)
+#define PPSMC_MSG_SetTDPLimit                 ((uint16_t) 0x134)
+#define PPSMC_MSG_Voltage_Cntl_Disable        ((uint16_t) 0x135)
+#define PPSMC_MSG_PCIeDPM_Enable              ((uint16_t) 0x136)
+#define PPSMC_MSG_ACPPowerOFF                 ((uint16_t) 0x137)
+#define PPSMC_MSG_ACPPowerON                  ((uint16_t) 0x138)
+#define PPSMC_MSG_SAMPowerOFF                 ((uint16_t) 0x139)
+#define PPSMC_MSG_SAMPowerON                  ((uint16_t) 0x13a)
+#define PPSMC_MSG_SDMAPowerOFF                ((uint16_t) 0x13b)
+#define PPSMC_MSG_SDMAPowerON                 ((uint16_t) 0x13c)
+#define PPSMC_MSG_PCIeDPM_Disable             ((uint16_t) 0x13d)
+#define PPSMC_MSG_IOMMUPowerOFF               ((uint16_t) 0x13e)
+#define PPSMC_MSG_IOMMUPowerON                ((uint16_t) 0x13f)
+#define PPSMC_MSG_NBDPM_Enable                ((uint16_t) 0x140)
+#define PPSMC_MSG_NBDPM_Disable               ((uint16_t) 0x141)
+#define PPSMC_MSG_NBDPM_ForceNominal          ((uint16_t) 0x142)
+#define PPSMC_MSG_NBDPM_ForcePerformance      ((uint16_t) 0x143)
+#define PPSMC_MSG_NBDPM_UnForce               ((uint16_t) 0x144)
+#define PPSMC_MSG_SCLKDPM_SetEnabledMask      ((uint16_t) 0x145)
+#define PPSMC_MSG_MCLKDPM_SetEnabledMask      ((uint16_t) 0x146)
+#define PPSMC_MSG_PCIeDPM_ForceLevel          ((uint16_t) 0x147)
+#define PPSMC_MSG_PCIeDPM_UnForceLevel        ((uint16_t) 0x148)
+#define PPSMC_MSG_EnableACDCGPIOInterrupt     ((uint16_t) 0x149)
+#define PPSMC_MSG_EnableVRHotGPIOInterrupt    ((uint16_t) 0x14a)
+#define PPSMC_MSG_SwitchToAC                  ((uint16_t) 0x14b)
+
+#define PPSMC_MSG_XDMAPowerOFF                ((uint16_t) 0x14c)
+#define PPSMC_MSG_XDMAPowerON                 ((uint16_t) 0x14d)
+
+#define PPSMC_MSG_DPM_Enable                  ((uint16_t) 0x14e)
+#define PPSMC_MSG_DPM_Disable                 ((uint16_t) 0x14f)
+#define PPSMC_MSG_MCLKDPM_Enable              ((uint16_t) 0x150)
+#define PPSMC_MSG_MCLKDPM_Disable             ((uint16_t) 0x151)
+#define PPSMC_MSG_LCLKDPM_Enable              ((uint16_t) 0x152)
+#define PPSMC_MSG_LCLKDPM_Disable             ((uint16_t) 0x153)
+#define PPSMC_MSG_UVDDPM_Enable               ((uint16_t) 0x154)
+#define PPSMC_MSG_UVDDPM_Disable              ((uint16_t) 0x155)
+#define PPSMC_MSG_SAMUDPM_Enable              ((uint16_t) 0x156)
+#define PPSMC_MSG_SAMUDPM_Disable             ((uint16_t) 0x157)
+#define PPSMC_MSG_ACPDPM_Enable               ((uint16_t) 0x158)
+#define PPSMC_MSG_ACPDPM_Disable              ((uint16_t) 0x159)
+#define PPSMC_MSG_VCEDPM_Enable               ((uint16_t) 0x15a)
+#define PPSMC_MSG_VCEDPM_Disable              ((uint16_t) 0x15b)
+#define PPSMC_MSG_LCLKDPM_SetEnabledMask      ((uint16_t) 0x15c)
+#define PPSMC_MSG_DPM_FPS_Mode                ((uint16_t) 0x15d)
+#define PPSMC_MSG_DPM_Activity_Mode           ((uint16_t) 0x15e)
+#define PPSMC_MSG_VddC_Request                ((uint16_t) 0x15f)
+#define PPSMC_MSG_MCLKDPM_GetEnabledMask      ((uint16_t) 0x160)
+#define PPSMC_MSG_LCLKDPM_GetEnabledMask      ((uint16_t) 0x161)
+#define PPSMC_MSG_SCLKDPM_GetEnabledMask      ((uint16_t) 0x162)
+#define PPSMC_MSG_UVDDPM_GetEnabledMask       ((uint16_t) 0x163)
+#define PPSMC_MSG_SAMUDPM_GetEnabledMask      ((uint16_t) 0x164)
+#define PPSMC_MSG_ACPDPM_GetEnabledMask       ((uint16_t) 0x165)
+#define PPSMC_MSG_VCEDPM_GetEnabledMask       ((uint16_t) 0x166)
+#define PPSMC_MSG_PCIeDPM_SetEnabledMask      ((uint16_t) 0x167)
+#define PPSMC_MSG_PCIeDPM_GetEnabledMask      ((uint16_t) 0x168)
+#define PPSMC_MSG_TDCLimitEnable              ((uint16_t) 0x169)
+#define PPSMC_MSG_TDCLimitDisable             ((uint16_t) 0x16a)
+#define PPSMC_MSG_DPM_AutoRotate_Mode         ((uint16_t) 0x16b)
+#define PPSMC_MSG_DISPCLK_FROM_FCH            ((uint16_t) 0x16c)
+#define PPSMC_MSG_DISPCLK_FROM_DFS            ((uint16_t) 0x16d)
+#define PPSMC_MSG_DPREFCLK_FROM_FCH           ((uint16_t) 0x16e)
+#define PPSMC_MSG_DPREFCLK_FROM_DFS           ((uint16_t) 0x16f)
+#define PPSMC_MSG_PmStatusLogStart            ((uint16_t) 0x170)
+#define PPSMC_MSG_PmStatusLogSample           ((uint16_t) 0x171)
+#define PPSMC_MSG_SCLK_AutoDPM_ON             ((uint16_t) 0x172)
+#define PPSMC_MSG_MCLK_AutoDPM_ON             ((uint16_t) 0x173)
+#define PPSMC_MSG_LCLK_AutoDPM_ON             ((uint16_t) 0x174)
+#define PPSMC_MSG_UVD_AutoDPM_ON              ((uint16_t) 0x175)
+#define PPSMC_MSG_SAMU_AutoDPM_ON             ((uint16_t) 0x176)
+#define PPSMC_MSG_ACP_AutoDPM_ON              ((uint16_t) 0x177)
+#define PPSMC_MSG_VCE_AutoDPM_ON              ((uint16_t) 0x178)
+#define PPSMC_MSG_PCIe_AutoDPM_ON             ((uint16_t) 0x179)
+#define PPSMC_MSG_MASTER_AutoDPM_ON           ((uint16_t) 0x17a)
+#define PPSMC_MSG_MASTER_AutoDPM_OFF          ((uint16_t) 0x17b)
+#define PPSMC_MSG_DYNAMICDISPPHYPOWER         ((uint16_t) 0x17c)
+#define PPSMC_MSG_CAC_COLLECTION_ON           ((uint16_t) 0x17d)
+#define PPSMC_MSG_CAC_COLLECTION_OFF          ((uint16_t) 0x17e)
+#define PPSMC_MSG_CAC_CORRELATION_ON          ((uint16_t) 0x17f)
+#define PPSMC_MSG_CAC_CORRELATION_OFF         ((uint16_t) 0x180)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_ON        ((uint16_t) 0x181)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_OFF       ((uint16_t) 0x182)
+#define PPSMC_MSG_ALLOW_LOWSCLK_INTERRUPT     ((uint16_t) 0x184)
+#define PPSMC_MSG_PkgPwrLimitEnable           ((uint16_t) 0x185)
+#define PPSMC_MSG_PkgPwrLimitDisable          ((uint16_t) 0x186)
+#define PPSMC_MSG_PkgPwrSetLimit              ((uint16_t) 0x187)
+#define PPSMC_MSG_OverDriveSetTargetTdp       ((uint16_t) 0x188)
+#define PPSMC_MSG_SCLKDPM_FreezeLevel         ((uint16_t) 0x189)
+#define PPSMC_MSG_SCLKDPM_UnfreezeLevel       ((uint16_t) 0x18A)
+#define PPSMC_MSG_MCLKDPM_FreezeLevel         ((uint16_t) 0x18B)
+#define PPSMC_MSG_MCLKDPM_UnfreezeLevel       ((uint16_t) 0x18C)
+#define PPSMC_MSG_START_DRAM_LOGGING          ((uint16_t) 0x18D)
+#define PPSMC_MSG_STOP_DRAM_LOGGING           ((uint16_t) 0x18E)
+#define PPSMC_MSG_MASTER_DeepSleep_ON         ((uint16_t) 0x18F)
+#define PPSMC_MSG_MASTER_DeepSleep_OFF        ((uint16_t) 0x190)
+#define PPSMC_MSG_Remove_DC_Clamp             ((uint16_t) 0x191)
+#define PPSMC_MSG_DisableACDCGPIOInterrupt    ((uint16_t) 0x192)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddc       ((uint16_t) 0x193)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddci      ((uint16_t) 0x194)
+#define PPSMC_MSG_SetVidOffset_1              ((uint16_t) 0x195)
+#define PPSMC_MSG_SetVidOffset_2              ((uint16_t) 0x207)
+#define PPSMC_MSG_GetVidOffset_1              ((uint16_t) 0x196)
+#define PPSMC_MSG_GetVidOffset_2              ((uint16_t) 0x208)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Enable    ((uint16_t) 0x197)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Disable   ((uint16_t) 0x198)
+#define PPSMC_MSG_SetTjMax                    ((uint16_t) 0x199)
+#define PPSMC_MSG_SetFanPwmMax                ((uint16_t) 0x19A)
+#define PPSMC_MSG_WaitForMclkSwitchFinish     ((uint16_t) 0x19B)
+#define PPSMC_MSG_ENABLE_THERMAL_DPM          ((uint16_t) 0x19C)
+#define PPSMC_MSG_DISABLE_THERMAL_DPM         ((uint16_t) 0x19D)
+
+#define PPSMC_MSG_API_GetSclkFrequency        ((uint16_t) 0x200)
+#define PPSMC_MSG_API_GetMclkFrequency        ((uint16_t) 0x201)
+#define PPSMC_MSG_API_GetSclkBusy             ((uint16_t) 0x202)
+#define PPSMC_MSG_API_GetMclkBusy             ((uint16_t) 0x203)
+#define PPSMC_MSG_API_GetAsicPower            ((uint16_t) 0x204)
+#define PPSMC_MSG_SetFanRpmMax                ((uint16_t) 0x205)
+#define PPSMC_MSG_SetFanSclkTarget            ((uint16_t) 0x206)
+#define PPSMC_MSG_SetFanMinPwm                ((uint16_t) 0x209)
+#define PPSMC_MSG_SetFanTemperatureTarget     ((uint16_t) 0x20A)
+
+#define PPSMC_MSG_BACO_StartMonitor           ((uint16_t) 0x240)
+#define PPSMC_MSG_BACO_Cancel                 ((uint16_t) 0x241)
+#define PPSMC_MSG_EnableVddGfx                ((uint16_t) 0x242)
+#define PPSMC_MSG_DisableVddGfx               ((uint16_t) 0x243)
+#define PPSMC_MSG_UcodeAddressLow             ((uint16_t) 0x244)
+#define PPSMC_MSG_UcodeAddressHigh            ((uint16_t) 0x245)
+#define PPSMC_MSG_UcodeLoadStatus             ((uint16_t) 0x246)
+
+#define PPSMC_MSG_DRV_DRAM_ADDR_HI            ((uint16_t) 0x250)
+#define PPSMC_MSG_DRV_DRAM_ADDR_LO            ((uint16_t) 0x251)
+#define PPSMC_MSG_SMU_DRAM_ADDR_HI            ((uint16_t) 0x252)
+#define PPSMC_MSG_SMU_DRAM_ADDR_LO            ((uint16_t) 0x253)
+#define PPSMC_MSG_LoadUcodes                  ((uint16_t) 0x254)
+#define PPSMC_MSG_PowerStateNotify            ((uint16_t) 0x255)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_HI      ((uint16_t) 0x256)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_LO      ((uint16_t) 0x257)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_HI          ((uint16_t) 0x258)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_LO          ((uint16_t) 0x259)
+#define PPSMC_MSG_LoadVBios                   ((uint16_t) 0x25A)
+#define PPSMC_MSG_GetUcodeVersion             ((uint16_t) 0x25B)
+#define DMCUSMC_MSG_PSREntry                  ((uint16_t) 0x25C)
+#define DMCUSMC_MSG_PSRExit                   ((uint16_t) 0x25D)
+#define PPSMC_MSG_EnableClockGatingFeature    ((uint16_t) 0x260)
+#define PPSMC_MSG_DisableClockGatingFeature   ((uint16_t) 0x261)
+#define PPSMC_MSG_IsDeviceRunning             ((uint16_t) 0x262)
+#define PPSMC_MSG_LoadMetaData                ((uint16_t) 0x263)
+#define PPSMC_MSG_TMON_AutoCaliberate_Enable  ((uint16_t) 0x264)
+#define PPSMC_MSG_TMON_AutoCaliberate_Disable ((uint16_t) 0x265)
+#define PPSMC_MSG_GetTelemetry1Slope          ((uint16_t) 0x266)
+#define PPSMC_MSG_GetTelemetry1Offset         ((uint16_t) 0x267)
+#define PPSMC_MSG_GetTelemetry2Slope          ((uint16_t) 0x268)
+#define PPSMC_MSG_GetTelemetry2Offset         ((uint16_t) 0x269)
+#define PPSMC_MSG_EnableAvfs                  ((uint16_t) 0x26A)
+#define PPSMC_MSG_DisableAvfs                 ((uint16_t) 0x26B)
+#define PPSMC_MSG_PerformBtc                  ((uint16_t) 0x26C)
+#define PPSMC_MSG_GetHbmCode                  ((uint16_t) 0x26D)
+#define PPSMC_MSG_GetVrVddcTemperature        ((uint16_t) 0x26E)
+#define PPSMC_MSG_GetVrMvddTemperature        ((uint16_t) 0x26F)
+#define PPSMC_MSG_GetLiquidTemperature        ((uint16_t) 0x270)
+#define PPSMC_MSG_GetPlxTemperature           ((uint16_t) 0x271)
+#define PPSMC_MSG_RequestI2CControl           ((uint16_t) 0x272)
+#define PPSMC_MSG_ReleaseI2CControl           ((uint16_t) 0x273)
+#define PPSMC_MSG_LedConfig                   ((uint16_t) 0x274)
+#define PPSMC_MSG_SetHbmFanCode               ((uint16_t) 0x275)
+#define PPSMC_MSG_SetHbmThrottleCode          ((uint16_t) 0x276)
+
+#define PPSMC_MSG_GetEnabledPsm               ((uint16_t) 0x400)
+#define PPSMC_MSG_AgmStartPsm                 ((uint16_t) 0x401)
+#define PPSMC_MSG_AgmReadPsm                  ((uint16_t) 0x402)
+#define PPSMC_MSG_AgmResetPsm                 ((uint16_t) 0x403)
+#define PPSMC_MSG_ReadVftCell                 ((uint16_t) 0x404)
+
+/* AVFS Only - Remove Later */
+#define PPSMC_MSG_VftTableIsValid             ((uint16_t) 0x666)
+
+/* If the SMC firmware has an event status soft register this is what the individual bits mean.*/
+#define PPSMC_EVENT_STATUS_THERMAL          0x00000001
+#define PPSMC_EVENT_STATUS_REGULATORHOT     0x00000002
+#define PPSMC_EVENT_STATUS_DC               0x00000004
+
+typedef uint16_t PPSMC_Msg;
+
+#pragma pack(pop)
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/fiji_pwrvirus.h b/drivers/gpu/drm/amd/powerplay/inc/fiji_pwrvirus.h
new file mode 100644
index 000000000000..0262ad35502a
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/fiji_pwrvirus.h
@@ -0,0 +1,10299 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _FIJI_PWRVIRUS_H_
+#define _FIJI_PWRVIRUS_H_
+
+#define mmCP_HYP_MEC1_UCODE_ADDR	0xf81a
+#define mmCP_HYP_MEC1_UCODE_DATA	0xf81b
+#define mmCP_HYP_MEC2_UCODE_ADDR	0xf81c
+#define mmCP_HYP_MEC2_UCODE_DATA	0xf81d
+
+enum PWR_Command
+{
+   PwrCmdNull = 0,
+   PwrCmdWrite,
+   PwrCmdEnd,
+   PwrCmdMax
+};
+typedef enum PWR_Command PWR_Command;
+
+struct PWR_Command_Table
+{
+   PWR_Command        command;
+   ULONG              data;
+   ULONG              reg;
+};
+typedef struct PWR_Command_Table PWR_Command_Table;
+
+#define PWR_VIRUS_TABLE_SIZE  10243
+static PWR_Command_Table PwrVirusTable[PWR_VIRUS_TABLE_SIZE] =
+{
+    { PwrCmdWrite, 0x100100b6, mmPCIE_INDEX                               },
+    { PwrCmdWrite, 0x00000000, mmPCIE_DATA                                },
+    { PwrCmdWrite, 0x100100b6, mmPCIE_INDEX                               },
+    { PwrCmdWrite, 0x0300078c, mmPCIE_DATA                                },
+    { PwrCmdWrite, 0x00000000, mmBIF_CLK_CTRL                             },
+    { PwrCmdWrite, 0x00000001, mmBIF_CLK_CTRL                             },
+    { PwrCmdWrite, 0x00000000, mmBIF_CLK_CTRL                             },
+    { PwrCmdWrite, 0x00000003, mmBIF_FB_EN                                },
+    { PwrCmdWrite, 0x00000000, mmBIF_FB_EN                                },
+    { PwrCmdWrite, 0x00000001, mmBIF_DOORBELL_APER_EN                     },
+    { PwrCmdWrite, 0x00000000, mmBIF_DOORBELL_APER_EN                     },
+    { PwrCmdWrite, 0x014000c0, mmPCIE_INDEX                               },
+    { PwrCmdWrite, 0x00000000, mmPCIE_DATA                                },
+    { PwrCmdWrite, 0x014000c0, mmPCIE_INDEX                               },
+    { PwrCmdWrite, 0x22000000, mmPCIE_DATA                                },
+    { PwrCmdWrite, 0x014000c0, mmPCIE_INDEX                               },
+    { PwrCmdWrite, 0x00000000, mmPCIE_DATA                                },
+    /*
+    { PwrCmdWrite, 0x009f0090, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x00000000, mmMC_CITF_CNTL                             },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x009f0090, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x009f0090, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_FB_OFFSET                          },*/
+    { PwrCmdWrite, 0x00000000, mmRLC_CSIB_ADDR_LO                         },
+    { PwrCmdWrite, 0x00000000, mmRLC_CSIB_ADDR_HI                         },
+    { PwrCmdWrite, 0x00000000, mmRLC_CSIB_LENGTH                          },
+    /*
+    { PwrCmdWrite, 0x00000000, mmMC_VM_MX_L1_TLB_CNTL                     },
+    { PwrCmdWrite, 0x00000001, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR           },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR          },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_FB_LOCATION                        },
+    { PwrCmdWrite, 0x009f0090, mmMC_VM_FB_LOCATION                        },*/
+    { PwrCmdWrite, 0x00000000, mmVM_CONTEXT0_CNTL                         },
+    { PwrCmdWrite, 0x00000000, mmVM_CONTEXT1_CNTL                         },
+    /*
+    { PwrCmdWrite, 0x00000000, mmMC_VM_AGP_BASE                           },
+    { PwrCmdWrite, 0x00000002, mmMC_VM_AGP_BOT                            },
+    { PwrCmdWrite, 0x00000000, mmMC_VM_AGP_TOP                            },*/
+    { PwrCmdWrite, 0x04000000, mmATC_VM_APERTURE0_LOW_ADDR                },
+    { PwrCmdWrite, 0x0400ff20, mmATC_VM_APERTURE0_HIGH_ADDR               },
+    { PwrCmdWrite, 0x00000002, mmATC_VM_APERTURE0_CNTL                    },
+    { PwrCmdWrite, 0x0000ffff, mmATC_VM_APERTURE0_CNTL2                   },
+    { PwrCmdWrite, 0x00000001, mmATC_VM_APERTURE1_LOW_ADDR                },
+    { PwrCmdWrite, 0x00000000, mmATC_VM_APERTURE1_HIGH_ADDR               },
+    { PwrCmdWrite, 0x00000000, mmATC_VM_APERTURE1_CNTL                    },
+    { PwrCmdWrite, 0x00000000, mmATC_VM_APERTURE1_CNTL2                   },
+    //{ PwrCmdWrite, 0x00000000, mmMC_ARB_RAMCFG                            },
+    { PwrCmdWrite, 0x12011003, mmGB_ADDR_CONFIG                           },
+    { PwrCmdWrite, 0x00800010, mmGB_TILE_MODE0                            },
+    { PwrCmdWrite, 0x00800810, mmGB_TILE_MODE1                            },
+    { PwrCmdWrite, 0x00801010, mmGB_TILE_MODE2                            },
+    { PwrCmdWrite, 0x00801810, mmGB_TILE_MODE3                            },
+    { PwrCmdWrite, 0x00802810, mmGB_TILE_MODE4                            },
+    { PwrCmdWrite, 0x00802808, mmGB_TILE_MODE5                            },
+    { PwrCmdWrite, 0x00802814, mmGB_TILE_MODE6                            },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE7                            },
+    { PwrCmdWrite, 0x00000004, mmGB_TILE_MODE8                            },
+    { PwrCmdWrite, 0x02000008, mmGB_TILE_MODE9                            },
+    { PwrCmdWrite, 0x02000010, mmGB_TILE_MODE10                           },
+    { PwrCmdWrite, 0x06000014, mmGB_TILE_MODE11                           },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE12                           },
+    { PwrCmdWrite, 0x02400008, mmGB_TILE_MODE13                           },
+    { PwrCmdWrite, 0x02400010, mmGB_TILE_MODE14                           },
+    { PwrCmdWrite, 0x02400030, mmGB_TILE_MODE15                           },
+    { PwrCmdWrite, 0x06400014, mmGB_TILE_MODE16                           },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE17                           },
+    { PwrCmdWrite, 0x0040000c, mmGB_TILE_MODE18                           },
+    { PwrCmdWrite, 0x0100000c, mmGB_TILE_MODE19                           },
+    { PwrCmdWrite, 0x0100001c, mmGB_TILE_MODE20                           },
+    { PwrCmdWrite, 0x01000034, mmGB_TILE_MODE21                           },
+    { PwrCmdWrite, 0x01000024, mmGB_TILE_MODE22                           },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE23                           },
+    { PwrCmdWrite, 0x0040001c, mmGB_TILE_MODE24                           },
+    { PwrCmdWrite, 0x01000020, mmGB_TILE_MODE25                           },
+    { PwrCmdWrite, 0x01000038, mmGB_TILE_MODE26                           },
+    { PwrCmdWrite, 0x02c00008, mmGB_TILE_MODE27                           },
+    { PwrCmdWrite, 0x02c00010, mmGB_TILE_MODE28                           },
+    { PwrCmdWrite, 0x06c00014, mmGB_TILE_MODE29                           },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE30                           },
+    { PwrCmdWrite, 0x00000000, mmGB_TILE_MODE31                           },
+    { PwrCmdWrite, 0x000000a8, mmGB_MACROTILE_MODE0                       },
+    { PwrCmdWrite, 0x000000a4, mmGB_MACROTILE_MODE1                       },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE2                       },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE3                       },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE4                       },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE5                       },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE6                       },
+    { PwrCmdWrite, 0x00000000, mmGB_MACROTILE_MODE7                       },
+    { PwrCmdWrite, 0x000000ee, mmGB_MACROTILE_MODE8                       },
+    { PwrCmdWrite, 0x000000ea, mmGB_MACROTILE_MODE9                       },
+    { PwrCmdWrite, 0x000000e9, mmGB_MACROTILE_MODE10                      },
+    { PwrCmdWrite, 0x000000e5, mmGB_MACROTILE_MODE11                      },
+    { PwrCmdWrite, 0x000000e4, mmGB_MACROTILE_MODE12                      },
+    { PwrCmdWrite, 0x000000e0, mmGB_MACROTILE_MODE13                      },
+    { PwrCmdWrite, 0x00000090, mmGB_MACROTILE_MODE14                      },
+    { PwrCmdWrite, 0x00000000, mmGB_MACROTILE_MODE15                      },
+    { PwrCmdWrite, 0x00900000, mmHDP_NONSURFACE_BASE                      },
+    { PwrCmdWrite, 0x00008000, mmHDP_NONSURFACE_INFO                      },
+    { PwrCmdWrite, 0x3fffffff, mmHDP_NONSURFACE_SIZE                      },
+    { PwrCmdWrite, 0x00000003, mmBIF_FB_EN                                },
+    //{ PwrCmdWrite, 0x00000000, mmMC_VM_FB_OFFSET                          },
+    { PwrCmdWrite, 0x00000000, mmSRBM_CNTL                                },
+    { PwrCmdWrite, 0x00020000, mmSRBM_CNTL                                },
+    { PwrCmdWrite, 0x80000000, mmATC_VMID0_PASID_MAPPING                  },
+    { PwrCmdWrite, 0x00000000, mmATC_VMID_PASID_MAPPING_UPDATE_STATUS     },
+    { PwrCmdWrite, 0x00000000, mmRLC_CNTL                                 },
+    { PwrCmdWrite, 0x00000000, mmRLC_CNTL                                 },
+    { PwrCmdWrite, 0x00000000, mmRLC_CNTL                                 },
+    { PwrCmdWrite, 0xe0000000, mmGRBM_GFX_INDEX                           },
+    { PwrCmdWrite, 0x00000000, mmCGTS_TCC_DISABLE                         },
+    { PwrCmdWrite, 0x00000000, mmTCP_ADDR_CONFIG                          },
+    { PwrCmdWrite, 0x000000ff, mmTCP_ADDR_CONFIG                          },
+    { PwrCmdWrite, 0x76543210, mmTCP_CHAN_STEER_LO                        },
+    { PwrCmdWrite, 0xfedcba98, mmTCP_CHAN_STEER_HI                        },
+    { PwrCmdWrite, 0x00000000, mmDB_DEBUG2                                },
+    { PwrCmdWrite, 0x00000000, mmDB_DEBUG                                 },
+    { PwrCmdWrite, 0x00002b16, mmCP_QUEUE_THRESHOLDS                      },
+    { PwrCmdWrite, 0x00006030, mmCP_MEQ_THRESHOLDS                        },
+    { PwrCmdWrite, 0x01000104, mmSPI_CONFIG_CNTL_1                        },
+    { PwrCmdWrite, 0x98184020, mmPA_SC_FIFO_SIZE                          },
+    { PwrCmdWrite, 0x00000001, mmVGT_NUM_INSTANCES                        },
+    { PwrCmdWrite, 0x00000000, mmCP_PERFMON_CNTL                          },
+    { PwrCmdWrite, 0x01180000, mmSQ_CONFIG                                },
+    { PwrCmdWrite, 0x00000000, mmVGT_CACHE_INVALIDATION                   },
+    { PwrCmdWrite, 0x00000000, mmSQ_THREAD_TRACE_BASE                     },
+    { PwrCmdWrite, 0x0000df80, mmSQ_THREAD_TRACE_MASK                     },
+    { PwrCmdWrite, 0x02249249, mmSQ_THREAD_TRACE_MODE                     },
+    { PwrCmdWrite, 0x00000000, mmPA_SC_LINE_STIPPLE_STATE                 },
+    { PwrCmdWrite, 0x00000000, mmCB_PERFCOUNTER0_SELECT1                  },
+    { PwrCmdWrite, 0x06000100, mmCGTT_VGT_CLK_CTRL                        },
+    { PwrCmdWrite, 0x00000007, mmPA_CL_ENHANCE                            },
+    { PwrCmdWrite, 0x00000001, mmPA_SC_ENHANCE                            },
+    { PwrCmdWrite, 0x00ffffff, mmPA_SC_FORCE_EOV_MAX_CNTS                 },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000010, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000020, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000030, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000040, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000050, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000060, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000070, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000080, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000090, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000a0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000b0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000c0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000d0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000e0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x000000f0, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmRLC_PG_CNTL                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS2                             },
+    { PwrCmdWrite, 0x15000000, mmCP_ME_CNTL                               },
+    { PwrCmdWrite, 0x50000000, mmCP_MEC_CNTL                              },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x0000000e, mmSH_MEM_APE1_BASE                         },
+    { PwrCmdWrite, 0x0000020d, mmSH_MEM_APE1_LIMIT                        },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000320, mmSH_MEM_CONFIG                            },
+    { PwrCmdWrite, 0x00000000, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_RB_VMID                               },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmRLC_CNTL                                 },
+    { PwrCmdWrite, 0x00000000, mmRLC_CNTL                                 },
+    { PwrCmdWrite, 0x00000000, mmRLC_SRM_CNTL                             },
+    { PwrCmdWrite, 0x00000002, mmRLC_SRM_CNTL                             },
+    { PwrCmdWrite, 0x00000000, mmCP_ME_CNTL                               },
+    { PwrCmdWrite, 0x15000000, mmCP_ME_CNTL                               },
+    { PwrCmdWrite, 0x00000000, mmCP_MEC_CNTL                              },
+    { PwrCmdWrite, 0x50000000, mmCP_MEC_CNTL                              },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x0840800a, mmCP_RB0_CNTL                              },
+    { PwrCmdWrite, 0xf30fff0f, mmTCC_CTRL                                 },
+    { PwrCmdWrite, 0x00000002, mmTCC_EXE_DISABLE                          },
+    { PwrCmdWrite, 0x000000ff, mmTCP_ADDR_CONFIG                          },
+    { PwrCmdWrite, 0x540ff000, mmCP_CPC_IC_BASE_LO                        },
+    { PwrCmdWrite, 0x000000b4, mmCP_CPC_IC_BASE_HI                        },
+    { PwrCmdWrite, 0x00010000, mmCP_HYP_MEC1_UCODE_ADDR                   },
+    { PwrCmdWrite, 0x00041b75, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000710e8, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000910dd, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000a1081, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000b016f, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000c0e3c, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000d10ec, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000e0188, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00101b5d, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00150a6c, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00170c5e, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x001d0c8c, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x001e0cfe, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00221408, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00370d7b, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00390dcb, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x003c142f, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x003f0b27, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00400e63, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00500f62, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00460fa7, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00490fa7, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005811d4, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00680ad6, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00760b00, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00780b0c, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00790af7, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x007d1aba, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x007e1abe, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00591260, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005a12fb, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00861ac7, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x008c1b01, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x008d1b34, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a014b9, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a1152e, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a216fb, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a41890, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a31906, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a50b14, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00621387, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005c0b27, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00160a75, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC1_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00010000, mmCP_HYP_MEC2_UCODE_ADDR                   },
+    { PwrCmdWrite, 0x00041b75, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000710e8, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000910dd, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000a1081, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000b016f, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000c0e3c, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000d10ec, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000e0188, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00101b5d, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00150a6c, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00170c5e, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x001d0c8c, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x001e0cfe, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00221408, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00370d7b, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00390dcb, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x003c142f, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x003f0b27, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00400e63, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00500f62, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00460fa7, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00490fa7, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005811d4, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00680ad6, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00760b00, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00780b0c, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00790af7, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x007d1aba, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x007e1abe, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00591260, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005a12fb, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00861ac7, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x008c1b01, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x008d1b34, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a014b9, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a1152e, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a216fb, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a41890, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a31906, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00a50b14, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00621387, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x005c0b27, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x00160a75, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x000f016a, mmCP_HYP_MEC2_UCODE_DATA                   },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x540fe800, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0x7e000200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e020201, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e040204, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e060205, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a080500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a0a0303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbf810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54106f00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000400b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00004000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00804fac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x540fef00, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0xc0031502, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00001e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x540ff000, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000145, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdcc10000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd010000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd410000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080061, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24ccffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3cd08000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1cd0ffcf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d018001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x050c0019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x84c00000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000067, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000006a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000006d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000008f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000099, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800000a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800000af, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x388c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08880002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98800003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000002d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28080001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d808001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc180000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc080000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc0700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d10ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d10c017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0d000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd0130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14cc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c00036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000005d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14d00011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c01b10, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00e0080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000013b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00e0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000013b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x280c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00052, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28180039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x280c0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00052, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28180039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x280c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00052, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28180039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000069, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28080001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca88004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc00006f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000013b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28180080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d10c017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd0130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000013b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd4c0380, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdcc0388, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55dc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdcc038c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce0c0390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce0c0394, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce4c0398, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce4c039c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce8c03a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56a80020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce8c03a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcecc03a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcecc03ac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf0c03b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf0c03b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4c03b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57740020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4c03bc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8c03c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57b80020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8c03c4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfcc03c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57fc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfcc03cc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05dc002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc12009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d200a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc012009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25e01c00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25e40300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25e800c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25ec003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e25c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de5c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xddc10000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02ee000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31100006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc1c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4df0388, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d7038c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d5dc01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4e30390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d70394, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d62001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4e70398, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d7039c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d66401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4eb03a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d6a801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ef03a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703ac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d6ec01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4f303b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d73001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4f703b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703bc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d77401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4fb03c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703c4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d7b801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ff03c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d703cc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d7fc01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc080000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4d70380, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1c88001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0e0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc01e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3cd00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0085, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc006a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc01e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3cd00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc180000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc080000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1c88001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc180000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc080000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400051, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04180018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aac0027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80002f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04080002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08880001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000367, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9880fff3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04080010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08880001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80c0309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80c0319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9880fffc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00e0100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d4001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x155c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e80180, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202c003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000bfc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900091a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05280196, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d4fe04, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800001b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000032b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000350, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000352, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000035f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000701, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000047c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000019f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d98001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0044, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9400036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40005b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40005d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840006d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11540015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a4003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1998003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af0007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1264001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15dc000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d65400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a38003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd5c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7df1c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800045, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411326a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc415326b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293279, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000056, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00059, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c8000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40005a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29988000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000073, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25140fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001b6d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153279, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd00005f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000075, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26f00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15100010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d190004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af07fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001427, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4412e01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0434001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4412e40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04343000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf413267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd1c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0160, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc810001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b4c0057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b700213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b740199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f4f400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55180020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2198003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x248dfffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc12e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af4007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33740003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26d80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ae8003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26680001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253348, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413348, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253348, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x958000d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000315, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04303000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26680001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b342010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1714000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25540800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x459801b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d77400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x199c01e2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e4002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e5c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e540002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc80c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54d00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000282, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc80c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54d00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000282, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000282, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc80c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1334e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01334f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd413350, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813351, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd881334d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3271, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153274, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50cc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cdcc011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05900008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd00006a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0006b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3272, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d594002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54d00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc12e23, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd012e24, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc12e25, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15540002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b340057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b280213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980198, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e40020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20cc003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113274, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01e0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2d540002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x078c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07d40000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001239, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04f80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x057c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd5c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840007c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400069, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c018a6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4412e22, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800007c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c018a2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd4c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680fffc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800002e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800002e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000069, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9540188f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc013cfff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x38d00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdcc30000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c01882, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000304, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x49980198, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e40020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x459801a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04302000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000329, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc812e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04302000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16ec001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1998003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce00000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a18003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4093249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1888003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000671, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419324c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1598001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14d80011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24dc00ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31e00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31dc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c00036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95801827, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14dc0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800006d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51dc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a0000ad, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04080000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af4003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740004d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca88005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24880001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f4b4009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313274, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d33400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a4004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1eecffdd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800003c3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aa80030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a8004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3272, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19e80042, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e8e800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de9c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3271, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50cc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ce8c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd30011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11e80007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd300001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240059, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1660001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e320009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0328000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e72400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0430000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02ac000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d310002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa87600, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280222, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4280058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x22ec003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8380018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57b00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04343108, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13740008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2374007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32a80003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ec0057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e40213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc0199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cecc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ce4c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800003e7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980104, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x49980104, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800003f2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000448, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813279, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf41326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x254c0700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a641fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0726, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a640200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1237b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2264003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8813260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4280034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001427, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c01755, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde830000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0174c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bb80040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100044, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19180024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x551c003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000043d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c8000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840006c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28200000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000043f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x282000f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x195c00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2555fff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0360001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5e124dc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e624001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80fff9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02ee000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2555fff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980158, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x49980158, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980170, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16200010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x195400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1154000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e80488, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0006c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18f807f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e40077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ec0199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6e400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000048e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000494, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004de, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000685, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000686, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800006ac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ccc001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1264000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d79400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e7a400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52a8001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d69401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202c007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aec0028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004cc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419324e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26e8003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aec003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12f4000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d324d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d75401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d290004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f8f4001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f52800f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50e00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004d1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0dc002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x6665fc00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1c011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a644000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f534002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x6665fc00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e76401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004d7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aec003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3257, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12f4000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d75401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1c011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a644000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202c003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e804e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004e7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800004f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000505, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x277401ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf41325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000671, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640fff4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17e00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84131db, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b301ff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26edf000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8413260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05a80507, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000050c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000528, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000057d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800005c2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800005f3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000671, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bd400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c004d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d150005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00063b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2511fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000624, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1be00fe4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000066, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400068, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000671, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bd400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c004d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d150005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400067, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00063b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2511fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000624, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bd400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ed6c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113271, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193272, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d51401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113274, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253276, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400061, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2730000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7db1800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00062, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000063, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400065, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc820001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b700057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b680213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b740199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46ec0188, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17e00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26e01000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c131fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x191807e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x192007ec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1334a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x69dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de20014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x561c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013344, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13345, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425334d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419334e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d334f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213350, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253351, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b680057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b700213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b740199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46ec01b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800068, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2010007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1910003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd00001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd00001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2010003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x191807e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9540000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2511fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1334a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013344, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013345, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180050, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0052, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280042, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813273, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13275, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000068, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400067, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07d40000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00124f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x057c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b680057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b700213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b740199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc820001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46ec0190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2154003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bd800e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd80005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420004d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1e000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28340001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f598004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1be800e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801327a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800005f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000075, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424004c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a4004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x277401ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf41325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xda000068, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113277, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9540002d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1334a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425334d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419334e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d334f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213350, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253351, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b680057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b700213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b740199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46ec01b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1334a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1be000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0360001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc63124dc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e724001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80fff9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02ee000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fc14001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x194c1c03, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc0003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c002d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000697, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x194c00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c004c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27301fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce00005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cf0c00d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0007e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b301ff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25100007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31100005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900008e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000075e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202c007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a9feff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1374000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1774000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d30b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce813265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00ac006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00e0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28880700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0006de, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14cc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30d4000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41530b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19980028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800006c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15600008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8380023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fa38011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1a0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x282c2002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e280008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd3800025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202400d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca48001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d8003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x194c00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c004c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27301fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce00005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cf0c00d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000712, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x194c1c03, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc0003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c002d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e80714, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000071c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000720, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000747, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000071d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800007c4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000732, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000745, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000744, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000072e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0007e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a64008c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b301fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000075e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0fff1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0007e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000723, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41f02f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000743, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8813247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0ffde, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000072e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0007e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15600008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84131db, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b301ff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8413260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc8000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x195800e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd80005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418004c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dd7fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1a001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46200200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04283247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af80057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af40213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6f400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2000025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc6990000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x329c325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x329c3269, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x329c3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01defff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d8009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000078a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fff2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03e7ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3f0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1f30001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03e4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001219, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d30b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bf0003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000b80, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x203c003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf0130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46200008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2000025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31dc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ec0057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e40213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc0199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cecc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ce4c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000448, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31980002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19580066, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15600008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0120001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11980003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da18001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d24db, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580137b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00ee000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113269, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19080070, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x190c00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2518000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05a80809, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000080e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000080f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000898, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000946, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800009e1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04a80811, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000815, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000834, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000085e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000085e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04341001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3045, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c091, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31300021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84002f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293059, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56a8001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000241, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000084a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02f0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4252087, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5668001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000084a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04341001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431ecaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02e0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31300021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84002f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293059, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56a8001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00021d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd410000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84802e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001a41, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43b02f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec80278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56f00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001608, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8813247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80802e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000085e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x950001fa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02e0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aec0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a40006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de6000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10e40008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e2e000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d10ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2110003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d10ff9e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0245301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801325f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0121fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29108eff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e524009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0127ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e524009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0131fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e524009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013279, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000866, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000866, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0100010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd2400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0180003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd1c002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000866, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04a8089a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000089e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800008fa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000945, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000945, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31300022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04183000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d91801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x459801e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2738000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b342010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x172c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef7400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc79d3300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc7a13301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8393300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0260001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce793301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x964012a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c028009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800008d2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x242c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02620c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0260400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6e400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae8001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2f0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800008d2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdf93300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce393301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04182000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000903, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31240022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ec30011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32f80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x67180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bfc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd981325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000915, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c1325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0fff6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f818001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001606, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d838001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16240014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a2801f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e2a000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2264003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00075e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x66d80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1330000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13f40014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33e80010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680ffec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04a80948, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000094c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000099b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800009e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800009e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04183000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d91801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x459801e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2738000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b342010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x172c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef7400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc79d3300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc7a13301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8393300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0260001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce793301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x964011fe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c028009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000978, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x242c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0260010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0260800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6e400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae8001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2f0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000978, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdf93300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce393301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04182000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dda801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e838011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84802e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001802, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x469c0390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04183000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b342010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x172c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef7400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4280011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04182000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0014df, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31280014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce8802ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800062, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31280034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04a809e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800009ec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a45, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a59, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a59, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d91801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a70250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e72401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b342010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x172c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b30c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef7400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x66740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04383000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b38007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33b40003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x4598001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740002f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4002eb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4002ec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4002ed, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4002ee, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04382000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd84802e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001715, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04382000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffbc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04341001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431ecaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a55, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x233c0032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf0130b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49302ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8413247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5198001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193269, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2598000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80002f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53b8001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7db9801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000a5e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c01106, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e02, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e03, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c010fd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ce4c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc80c0072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x58e801fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc01e2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e4002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e5c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e540002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9540000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x44cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55900020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x44cc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd812e01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd012e02, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd412e03, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2264003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1e64001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14d00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ab1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a0010ac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd880003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010de, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc010ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d403f7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0cc009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41b0367, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d958004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d85800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc1e0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d001fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05280adc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000af1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000adf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ae7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000ace, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd8d2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d803f7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc010ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0cc009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11940014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29544001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29544003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000af4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd44d2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd44dc000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000ace, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd8d2c00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000b0a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd44d2c00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28148004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4593240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0105e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2198003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x199c0034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef3400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14e80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a8000af, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c01043, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a01fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3620005c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2464003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc6290ce7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16ac001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ac003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ee6c00d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00fff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000367, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640102e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x199c0037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a00035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0005d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16f8001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9780000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc035f0ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e764009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19b401f8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13740008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e76400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x199c0034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ae4003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000b7c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aec003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a4003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12ec001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1374000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02e4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1774000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc01e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13fc0018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dbd800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d98ff15, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x592c00fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd80000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12e00016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x592c007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12e00015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1264001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1620000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e32000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12e4001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5924007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a4003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013257, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd413258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00fdb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9780f5ca, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001219, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001b6d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d324e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431324d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07740003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x269c003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e4004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f67000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f674002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53740002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab42010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab8c006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a8000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b740000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf40001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000bec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000b47, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b34060b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec00ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a8004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc415325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18580037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x262001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d54001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14f00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd280200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd680208, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcda80210, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc6930200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc6970208, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc69b0210, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd900003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd940003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9400040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010de, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14fc0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24f800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd88130b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d83c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4093249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1888003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000671, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419324c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1598001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14d80011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24e000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x321c0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580ffee, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c30, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9480000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800f29, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800f23, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c00036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800f1a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9600f502, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0f500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000f05, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1f30001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16e4001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640f4f4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33740002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40f4f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aec003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12ec001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1374000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02e4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1774000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00ac005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00e0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc8000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28884900, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ff3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400ee1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c40a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c40c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c40d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d0007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15580010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x255400ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c411, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c40f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c40e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c410, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e80033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ec0034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c414, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c415, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c413, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c412, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc0032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c030011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c038011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431c417, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435c416, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c419, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc418, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf413261, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013262, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13263, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813264, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc0030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d77000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000cd6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51b80020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f97801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000cd6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ca7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc0031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435c40b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4280032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012c2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f8cc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13f4000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bf0060b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000cf4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0677, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13fc0017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc032800b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb7800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ffbc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d42011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d001e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd4c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800e6c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d59401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x596001fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ce0c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x505c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50600020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8240010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e800c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x122c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000d1f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8240010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x566c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413261, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13262, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b740008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x566c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce413261, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec13262, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012c2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f8cc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13f4000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bf0060b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000d57, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0677, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13fc0017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0328009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb7800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ffbc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04143000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd413267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e51001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d2d0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19640057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19580213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19600199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da6400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04142000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd413267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4153267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d001e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d40030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d80034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05280d83, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c424001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000d8a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000d95, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000db1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000d95, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000dbc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11540010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e010001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00187c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d75400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4610000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580f3d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x526c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e80058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e2ec01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc82c0072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5ae0073a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea2800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580f3c6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc3a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980fff5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02a0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01c405, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd441c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580f3b1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c409, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11540010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4610000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580f3a5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00da7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50500020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8280072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5aac007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12d80017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56a00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e82400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e58c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19d4003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28182002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00104f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc011000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c908009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d614011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1a0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb0800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e280008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20880188, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20240090, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca48001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c018001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd901a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1624001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841325f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00038, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ac0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ac0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13f4000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b301ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2330003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00038, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0001a2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc80003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24b00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1330000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ac0024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b304000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a800e5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da9800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1910003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2220003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e2a000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00038, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d40030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d001e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18fc0034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24e8000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80e71, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000edd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e91, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e91, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ea1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000eaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e7c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e7f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e7f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e87, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000e8f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51dc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9e001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a200008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213262, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253261, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a200008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213264, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253263, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc820001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e82005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da1801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1800072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8180072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x59a001fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea2800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce80001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd180001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15980002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421c401, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425c401, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ee6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2580, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac260c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0828, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2440, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0093, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac31dc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac31e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ede, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac7c06, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db07c00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acc337, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0c330, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acc335, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0c336, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac9002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db09001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac9012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db09011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acec70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0ec6f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ebc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5a10000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5a50000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05280eea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ef1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000efe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f11, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f2e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000efe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f1f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce190000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce190000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0f26f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e80058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7daec01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc82c0072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5af8073a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eba800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0f25c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02a0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15980002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c405, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56240020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0f24e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c409, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40f247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce190000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce190000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0f240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439c040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2580, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac260c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0828, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2440, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac2390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac0093, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac31dc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31ac31e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ef2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac7c06, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db07c00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acc337, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0c330, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acc335, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0c336, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39acec70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db0ec6f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac9002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db09002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39ac9012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3db09012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ef1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b740008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c1325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c034001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c038001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e0007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32240003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f88, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e52401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8280072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ac0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26f0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af000fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1334000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24e02000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f63400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e00074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32240003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000f9d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51e40020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5a401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8280072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ac0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26f0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1af000fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13380016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e00039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fa3800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb7800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e0007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1220001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fa3800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e00074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fa3800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1c084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d001e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31140005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31140006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00104f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05280fb7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28140002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fbe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fbe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fc2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fbe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fd1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ff2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ff2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e80039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52a8003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d59401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d69401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140004b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d958004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d150005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x159c0011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31a00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31a40001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e25800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0fff5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fff4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000fef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d100010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01326f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc011000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33b40003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0340008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000ffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c908009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d614011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1a0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb0800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x282c2002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x208801a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e280008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20240030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca48001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28340000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x507c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d7d401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x557c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28342002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000102f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c018001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1cccfe08, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00b33, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da2400f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da28002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1ac002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d2ac002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3ef40010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40f11d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde410000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdcc10000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd010000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd410000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdd810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xddc10000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde010000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c024001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100086, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5510003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001075, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15800f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15c002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d520002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cde0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e20001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001071, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c00036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00b01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc200000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc1c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc180000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc0c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc240000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc40003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4080029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc80003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a800e5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da9800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a400e5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12500009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x248c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x200c006d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x200c0228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410002b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18881fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d4072c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc00d1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd4c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3094000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x38d80000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x311c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30940007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1620001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010c4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x259c007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a00030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010cb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x199c0fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010cb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000aac, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434002e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2020002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07a810d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000bfc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000104c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x200c007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28240007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xde430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc80003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24b00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1330000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a800e5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da9800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d3249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b304000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x192400fd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d59401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06681110, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ac0024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19180070, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19100078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18f40058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5978073a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001117, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001118, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001122, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000112d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001130, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001133, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000117b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24ec0f00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32ec0600, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000117b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24ec0f00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32ec0600, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000117b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001122, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc81c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55e00020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001122, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02a0200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e8e8009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x22a8003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x22a80074, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2774001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13740014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eb6800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25ecffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55700020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15f40010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13740002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x275c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c018001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15dc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39e00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc1c01e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e40008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc2001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e40008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e62000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da58001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001165, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc2001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1a0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e0d000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95000007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e02401e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06640008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05d80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc2401e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da58001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05e00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da2000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9600ffe6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00116b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce00001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81c078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x22640435, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0528117e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x312c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001185, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001182, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001182, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a0400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1198001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d81c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19a000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de2c00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26200010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc415326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc420007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011a3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d654001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c020001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011b6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253279, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc415326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2730003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3b380006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3f38000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0430000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb10004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e57000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e578002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d67c002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0be40001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d3a4002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x202c002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26200010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e640010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce81325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434002e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07a811cf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00feb8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x954009a7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000bfc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1c07c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c07d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c08c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c07e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18f0012f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18f40612, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc00c1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cf7400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x39600004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0140004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11600001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18fc003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9740001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x166c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011ee, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a6c003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800011e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ac007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab00030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aac0fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001205, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x166c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11600001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ffbc00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a2800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c07c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c08c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03ae000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a0800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c07c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c08c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03ae000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a4000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81c07c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c08c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17fc001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30d00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000052, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640090f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1514001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19180038, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d324e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431324d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab0c006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000127f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d3258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313257, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab0c012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e624004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f67800f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53740002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ab42010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a8000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b740000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f6b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf40001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1514001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0012e1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x964008d7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9800036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300677, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012aa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b34060b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f37000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec00ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a8002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef6c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7edec00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4140032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1858003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0cc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0006c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d407f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2598003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d190004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d5d4001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d52000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d514002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d958001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd5c002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc1325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1ccc001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14f00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd980003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9800040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd9c00040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800010de, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33f80003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800051, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc80003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24b00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1330000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18a800e5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1d980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7da9800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b74003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b304000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431326c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b4c00f8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50700020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04e81324, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18ac0024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50600020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30e40004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d71401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x596401fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b74008d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e76400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a640000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000132c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000133b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001344, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42530b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a68003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2024003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11980014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d19000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd0130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce4130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de6800f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce40001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc428000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8240011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de6800f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffe0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00104f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28182002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340035, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140023, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d614011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4100026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05980008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1a0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb0800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3e280008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cb4800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20240030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ca48001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b4c00f8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28340000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x507c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30e40004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d7d401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x557c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28342002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c018001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf81a2a4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c007eb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50500020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d0d001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8100072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x591c01fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45140210, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x595801fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11980009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1624001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400069, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a307fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x23304076, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc00e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0015, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x4514020c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a2001e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a204001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a64003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1264001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15dc000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dcdc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5dc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001427, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340022, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4412e01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0434001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf430000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4412e40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c030, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41c031, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x248dfffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc12e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc812e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45140248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013257, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0434000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdb000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd140001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9980ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8200011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013259, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0337fff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f220009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d01c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f01c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c01c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c000d61, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50500020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001427, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd0c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd0c00072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8240072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd240001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19682011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5a6c01fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12ec0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eeac00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aec0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4180011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf830000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfa0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4380007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d40038, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400029, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9540073d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18c80066, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30880001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00187c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd910000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x4220000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24e80007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24ec0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5310000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001465, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1000072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc82c0072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2c0001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18f02011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5aec01fc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12ec0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aec0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a8146a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1f0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001478, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001478, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000147a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000147a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000147a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f1b400e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000147a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f334002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000147b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e024001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000144a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fbfc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x251001ef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94800007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00187c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42c0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd910000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40d325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800012c2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13f4000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bf0060b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800014a9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d325a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0677, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb81ff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0328007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb7800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13fc0017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ffbc00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03a0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45dc0390, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04183000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b380057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b340213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c424001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c428001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c42c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c430001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c434001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04182000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd813267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a0800fd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x109c000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce080228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9880000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce480250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce880258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0ec75, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc80230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce480250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce880258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52a80020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x66580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc80260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec80288, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf080290, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec80298, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf0802a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4802a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27580001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc802b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80802b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x178c000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b8003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cf8c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8802c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc802c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8802d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf8802d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bc800ea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25b8ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd2800c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5230309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e3a400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001539, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd08034b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd880353, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00163f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b0353, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd14005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000154f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd080238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd08034b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2598ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d200008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc80230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd900309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8100319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340801, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2198003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd910ce7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4190ce6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d918005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d918004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd810ce6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdd1054f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000156e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x090c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdcd050e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x040c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x110c0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc4001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41230a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41230b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41230c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc41230d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc480329, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc48032a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc4802e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f02e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d8003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09940001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x44100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x69100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000157f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970290, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b0288, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d59401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b02a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49f0298, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x041c0040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dcdc002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d924019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d26400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00163f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001579, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d010021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d914019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd480298, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd8802a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10d40010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12180016, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc51f0309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d95800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d62000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdd00309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce113320, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f02e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b02b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18dc01e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9400e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c0001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00163f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015aa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a302b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12240004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e5e400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ab02a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04100000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce4c0319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d9d8002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea14005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015bc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04240001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e624004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d25000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2620000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fff4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd0d3330, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce0802b8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd8802b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ab02e0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aa807f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f02d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49702d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b02c8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49f02c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96800028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d4e000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9600000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d964002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d694001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cde4002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de94001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd64002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d694001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00163f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800015cd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930238, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d698002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd4802d8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x129c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc50f0319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11a0000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1e000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1198000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd953300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e0e000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a8000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce953301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce100319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b70280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73800a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x536c0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9780eb68, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001608, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001609, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30b40000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b400011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b70258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53780020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb3801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7faf8019, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x67b40001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x57b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bb0260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fab8001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf880260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x66f40001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4353247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7f4009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fff7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x269c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a00018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a00060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x269c0018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a40060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11dc0006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29dc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de5c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b70228, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc80230, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f514005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2510000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001644, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd080240, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f130005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001688, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00120d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001219, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001232, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340801, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f130004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01051e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42d051f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ed2c005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96c0fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01051f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000055, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5170309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x195c07f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x196007f6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04340001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x6b740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001665, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a702a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ab0298, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f634014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e76401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56680020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8113320, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce480298, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce8802a0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc5170319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b702b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x255c000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f5f4001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8113330, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf4802b0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11340001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x195c07e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x196007ee, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8353300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1e4001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8353301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce4802d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8100309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8100319, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc48f0250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd4c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x64d80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580005c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc24001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd2000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc435324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7df5c00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800016f1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc033ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2f3000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a7003e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13b80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a7000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07b80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a700064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800016df, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb30002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4392083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffca, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2030007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800016f2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ff9c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001608, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bc800ea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd80802e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18fc0064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00042, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51980020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dd9801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x45980400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b380057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b340213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f7b400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f73400a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14f4001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bf02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x192807fa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bf0258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a70250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53fc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e7e401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x667c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0aec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eebc00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x43300007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7db30011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd3000025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc03ec005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfca200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x192807fa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01f007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1d0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2110007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x203c003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0017f5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18fc01e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00185b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8413247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40ffd5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bf02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0ea24, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14d4001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d52400e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49f0258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a30250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51dc0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400017, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d534002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dae4005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32e0001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec80270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000174f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b740001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00178a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40fff3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001608, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ab0268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7daa4005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32a0001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001765, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01f007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d1d0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2110007d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8013256, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c0017f2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd013254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4113248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b3034b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f13000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001855, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32a4001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8413247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd080260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce880268, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ffc0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ec28001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32e0001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253255, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431324f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e72400c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9680fff7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aa4003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aa400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32680003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a800046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4293260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1aa400e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800017e2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc027ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2e6400ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a4009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4240009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19e403e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26680003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12a80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea68001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19e400e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea68001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea68001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19e40064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x32640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a40005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06640003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce412082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a640003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800017d0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16a40005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce412082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea64002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4292083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ea68005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a80ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc429325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26a400ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40ffca, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2024007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800017e3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4a70280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4ab0278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52640020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7eae8014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e6a401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56680020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce480278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce880280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x042c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec80270, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800017fe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bf02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800017fe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43b02eb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42302ec, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fa3801a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x47b8020c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x15e00008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1220000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2a206032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x513c001e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e3e001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4bf02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000180f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b3c0077, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1330000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd200000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4200007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd3800002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc30001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc1e0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04380032, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf80000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001427, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc413248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3269, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33fc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdfc30000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bfc0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdfc30000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd441326a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x173c0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300303, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3f0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ff3c004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001842, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdfc30000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4413249, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c43c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x23fc003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1326d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdf830000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd441326e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1fb8ffc6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xddc30000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf813265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001852, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c00142b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13252, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013253, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001878, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49f02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c00018, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c420001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13252, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013253, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c3000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c0012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001878, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41f02ed, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42302ee, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc13252, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013253, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e2a0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28340001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x313c0bcc, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x393c051f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d3c050e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x393c0560, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d3c054f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x393c1538, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d3c1537, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b740800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bc800ea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e8007c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c42c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a8189a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000189e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800018c5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800018f2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c414001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0007e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x50580020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d59401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc8140072, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09240002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c418001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4340004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc42130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a24002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2020002c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc418000d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1198001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14cc0004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7cd8c00a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc130b7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce0130b5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd1400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x5978073a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bb80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf800024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd800026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9600e8a8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9640e8a5, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800018a9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc55b0309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d5c0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2598ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dad800c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0ffd2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fff9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x442c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x65180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7df9c00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c13260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd901325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940fff1, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x66d80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x56ec0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26240007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940fff7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000189e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc023007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19e4003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7de1c009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dee000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96000007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c13260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd901325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc421325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x261c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000189e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940fff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000189e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28cc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43d3265, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bc800ea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18e00064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06281911, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14f4001d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24cc0003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x86800000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001915, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x800019af, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001a2b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8000016a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc48032b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc480333, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc48033b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc480343, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98800011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46640400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04203000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b3c0057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b200213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e3e000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e32000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04180000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f438001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00068, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213254, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a1c003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00065, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc01f007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1e0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97800062, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0bb80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x43bc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fcbc001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc7df032b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1fc00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0101, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c0102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001994, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc033ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2f3000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f003e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13b80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07b80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f00064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001982, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb30002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4392083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffcb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2030007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001995, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc1325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98800009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x41bc0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x53fc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e7fc011, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd3c00025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0012, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9bc0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x653c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dbd8001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ff8f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2bfc0008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x043c2000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcfc13267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c410001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc55b0309, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x3d5c0010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2598ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x05540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d91800c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580fff8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09780001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x65180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9580005d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200101, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400058, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dc24001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41d3248, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25dc000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7df9c00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95c00053, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e41c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a70003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a7000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33240003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a400046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1a7000e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001a21, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc033ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2f3000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f270009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x266400ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f003e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27240003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12640004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e724001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e724001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e724001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06640002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f00064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16700005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001a0f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x16700005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e730002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4252083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e724005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x26640001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a40ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x267000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffca, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2030007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001a22, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ff9f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001a31, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8080280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213246, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4253245, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52200020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e26401a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x46640400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04203000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4213267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b180057, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b200213, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1b300199, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e1a000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e32000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce000024, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4970258, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4930250, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x51540020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4af0280, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4b30278, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x52ec0020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140020, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04280000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x65180001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800060, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x8c001628, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4193247, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x25980001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200101, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x30f00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95800056, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb0003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b800046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4393260, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bb000e4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001aa2, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc033ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2f3000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f3b0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf01325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b800ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4300009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9700fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f003e6, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27380003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13b80004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f000e8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07b80002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x19f00064, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33300002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0b300003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001a90, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x17b00005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf012082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01203f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x13300005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb30002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4392083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7fb38005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b80ffdf, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c00034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc00013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc431325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27300010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc439325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27b000ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b00ffca, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2030007b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf00325b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001aa3, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce01325d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04300001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7f2b0014, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ef2c01a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd2400025, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x4664001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000026, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400027, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x06a80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55100001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ff9c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc49b02e9, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99800008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc430000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2b300008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf000013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04302000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcf013267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc4313267, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x244c00ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc4c0200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc44f0200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc410000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d158010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x059cc000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccdd0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0037, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000049, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c003a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500e69a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d0003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d40021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd840004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c003c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x14cc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c00028, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000033, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc438000b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0009, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x27fc0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43dc07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1bfc0078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7ffbc00c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x97c0fffd, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x99000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0120840, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x282c0040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001ae8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0121841, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x282c001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c07c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c08c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c079, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c07e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcec0001b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a200001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9a00ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x166c001f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04200004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9ac0fffb, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc434000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9b40ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc425c07f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8000034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940e66b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800004a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0036, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9900fffe, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18cc0021, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc00047, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc000046, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0039, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c003d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c40c001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24d003ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d47fea, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x18d87ff4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd00004c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd40004e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd80004d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c405, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc02a0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2aa80001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c406, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc414000e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x29540008, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x295c0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8c1325e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcdc0001a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11980002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x4110000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0160800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7d15000a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0164010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c080, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c081, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c082, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc01c083, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c084, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x98c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400048, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c003b, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x94c0ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000c16, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c40a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd901c40d, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c410, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c40e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd801c40f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc40c0040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x09540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9940ffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04140096, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8400013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1c400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc411c401, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9500fffa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424003e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04d00001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x11100002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd01c40c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0180034, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd81c411, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd841c414, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0a540001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcd41c412, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x2468000f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc419c416, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x41980003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc41c003f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7dda0001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x12200002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x10cc0002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xccc1c40c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd901c411, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce41c412, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd8800013, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xce292e40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e01, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e02, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e03, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc412e00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000aa7, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc43c0007, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc120000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x31144000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x95400005, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xdc030000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd800002a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xcc3c000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b70, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x33f80003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd4400078, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x9780e601, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x188cfff0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x04e40002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001190, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400006, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x90000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc424005e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x96400003, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7c408001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x88000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80001b74, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000168, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110501, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120206, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130703, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92100400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92110105, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92120602, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x92130307, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbf810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x54106500, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0x7e000200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e020204, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc00a0505, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbf8c007f, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb8900904, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb8911a04, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb8920304, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb8930b44, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x921c0d0c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x921c1c13, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x921d0c12, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x811c1d1c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x811c111c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x921cff1c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000400, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x921dff10, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000100, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x81181d1c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e040218, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0701000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050102, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xe0501000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80050302, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbf810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x54106900, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0x7e080200, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x7e100204, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbefc00ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00010000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x24200087, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x262200ff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000001f0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x20222282, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x28182111, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000040c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd81a0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000080c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xd86c0000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x1100000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xbf810000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x80000004, mmCP_DFY_CNTL                              },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_ADDR_HI                           },
+    { PwrCmdWrite, 0x54116f00, mmCP_DFY_ADDR_LO                           },
+    { PwrCmdWrite, 0xc0310800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4540fe8, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000041, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000000c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07808000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54116f00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00005301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4540fef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee20, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0310800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb454105e, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000c0, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07808000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54117300, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00005301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4540fef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee20, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0310800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4541065, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000500, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000001c, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07808000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54117700, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00005301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4540fef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee20, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xc0310800, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000040, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4541069, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000444, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x0000008a, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x07808000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xffffffff, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000002, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xaaaaaaaa, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x55555555, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee40, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000010, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000001, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000004, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x54117b00, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00005301, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0xb4540fef, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x540fee20, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x000000b4, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x08000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_DFY_DATA_0                            },
+    { PwrCmdWrite, 0x00000000, mmCP_MEC_CNTL                              },
+    { PwrCmdWrite, 0x00000000, mmCP_MEC_CNTL                              },
+    { PwrCmdWrite, 0x00000004, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x54116f00, mmCP_MQD_BASE_ADDR                         },
+    { PwrCmdWrite, 0x000000b4, mmCP_MQD_BASE_ADDR_HI                      },
+    { PwrCmdWrite, 0xb4540fef, mmCP_HQD_PQ_BASE                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_BASE_HI                        },
+    { PwrCmdWrite, 0x540fee20, mmCP_HQD_PQ_WPTR_POLL_ADDR                 },
+    { PwrCmdWrite, 0x000000b4, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI              },
+    { PwrCmdWrite, 0x00005301, mmCP_HQD_PERSISTENT_STATE                  },
+    { PwrCmdWrite, 0x00010000, mmCP_HQD_VMID                              },
+    { PwrCmdWrite, 0xc8318509, mmCP_HQD_PQ_CONTROL                        },
+    { PwrCmdWrite, 0x00000005, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x54117300, mmCP_MQD_BASE_ADDR                         },
+    { PwrCmdWrite, 0x000000b4, mmCP_MQD_BASE_ADDR_HI                      },
+    { PwrCmdWrite, 0xb4540fef, mmCP_HQD_PQ_BASE                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_BASE_HI                        },
+    { PwrCmdWrite, 0x540fee20, mmCP_HQD_PQ_WPTR_POLL_ADDR                 },
+    { PwrCmdWrite, 0x000000b4, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI              },
+    { PwrCmdWrite, 0x00005301, mmCP_HQD_PERSISTENT_STATE                  },
+    { PwrCmdWrite, 0x00010000, mmCP_HQD_VMID                              },
+    { PwrCmdWrite, 0xc8318509, mmCP_HQD_PQ_CONTROL                        },
+    { PwrCmdWrite, 0x00000006, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x54117700, mmCP_MQD_BASE_ADDR                         },
+    { PwrCmdWrite, 0x000000b4, mmCP_MQD_BASE_ADDR_HI                      },
+    { PwrCmdWrite, 0xb4540fef, mmCP_HQD_PQ_BASE                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_BASE_HI                        },
+    { PwrCmdWrite, 0x540fee20, mmCP_HQD_PQ_WPTR_POLL_ADDR                 },
+    { PwrCmdWrite, 0x000000b4, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI              },
+    { PwrCmdWrite, 0x00005301, mmCP_HQD_PERSISTENT_STATE                  },
+    { PwrCmdWrite, 0x00010000, mmCP_HQD_VMID                              },
+    { PwrCmdWrite, 0xc8318509, mmCP_HQD_PQ_CONTROL                        },
+    { PwrCmdWrite, 0x00000007, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x54117b00, mmCP_MQD_BASE_ADDR                         },
+    { PwrCmdWrite, 0x000000b4, mmCP_MQD_BASE_ADDR_HI                      },
+    { PwrCmdWrite, 0xb4540fef, mmCP_HQD_PQ_BASE                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_BASE_HI                        },
+    { PwrCmdWrite, 0x540fee20, mmCP_HQD_PQ_WPTR_POLL_ADDR                 },
+    { PwrCmdWrite, 0x000000b4, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI              },
+    { PwrCmdWrite, 0x00005301, mmCP_HQD_PERSISTENT_STATE                  },
+    { PwrCmdWrite, 0x00010000, mmCP_HQD_VMID                              },
+    { PwrCmdWrite, 0xc8318509, mmCP_HQD_PQ_CONTROL                        },
+    { PwrCmdWrite, 0x00000004, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000104, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000204, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000304, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000404, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000504, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000604, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000704, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000005, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000105, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000205, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000305, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000405, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000505, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000605, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000705, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000006, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000106, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000206, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000306, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000406, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000506, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000606, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000706, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000007, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000107, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000207, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000307, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000407, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000507, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000607, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000707, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000008, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000108, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000208, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000308, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000408, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000508, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000608, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000708, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000009, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000109, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000209, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000309, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000409, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000509, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000609, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000709, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_RPTR                           },
+    { PwrCmdWrite, 0x00000000, mmCP_HQD_PQ_WPTR                           },
+    { PwrCmdWrite, 0x00000001, mmCP_HQD_ACTIVE                            },
+    { PwrCmdWrite, 0x00000004, mmSRBM_GFX_CNTL                            },
+    { PwrCmdWrite, 0x01010101, mmCP_PQ_WPTR_POLL_CNTL1                    },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdWrite, 0x00000000, mmGRBM_STATUS                              },
+    { PwrCmdEnd,   0x00000000, 0x00000000                                 },
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
new file mode 100644
index 000000000000..91795efe1336
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -0,0 +1,385 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _HARDWARE_MANAGER_H_
+#define _HARDWARE_MANAGER_H_
+
+
+
+struct pp_hwmgr;
+struct pp_hw_power_state;
+struct pp_power_state;
+enum amd_dpm_forced_level;
+struct PP_TemperatureRange;
+
+struct phm_fan_speed_info {
+	uint32_t min_percent;
+	uint32_t max_percent;
+	uint32_t min_rpm;
+	uint32_t max_rpm;
+	bool supports_percent_read;
+	bool supports_percent_write;
+	bool supports_rpm_read;
+	bool supports_rpm_write;
+};
+
+/* Automatic Power State Throttling */
+enum PHM_AutoThrottleSource
+{
+    PHM_AutoThrottleSource_Thermal,
+    PHM_AutoThrottleSource_External
+};
+
+typedef enum PHM_AutoThrottleSource PHM_AutoThrottleSource;
+
+enum phm_platform_caps {
+	PHM_PlatformCaps_AtomBiosPpV1 = 0,
+	PHM_PlatformCaps_PowerPlaySupport,
+	PHM_PlatformCaps_ACOverdriveSupport,
+	PHM_PlatformCaps_BacklightSupport,
+	PHM_PlatformCaps_ThermalController,
+	PHM_PlatformCaps_BiosPowerSourceControl,
+	PHM_PlatformCaps_DisableVoltageTransition,
+	PHM_PlatformCaps_DisableEngineTransition,
+	PHM_PlatformCaps_DisableMemoryTransition,
+	PHM_PlatformCaps_DynamicPowerManagement,
+	PHM_PlatformCaps_EnableASPML0s,
+	PHM_PlatformCaps_EnableASPML1,
+	PHM_PlatformCaps_OD5inACSupport,
+	PHM_PlatformCaps_OD5inDCSupport,
+	PHM_PlatformCaps_SoftStateOD5,
+	PHM_PlatformCaps_NoOD5Support,
+	PHM_PlatformCaps_ContinuousHardwarePerformanceRange,
+	PHM_PlatformCaps_ActivityReporting,
+	PHM_PlatformCaps_EnableBackbias,
+	PHM_PlatformCaps_OverdriveDisabledByPowerBudget,
+	PHM_PlatformCaps_ShowPowerBudgetWarning,
+	PHM_PlatformCaps_PowerBudgetWaiverAvailable,
+	PHM_PlatformCaps_GFXClockGatingSupport,
+	PHM_PlatformCaps_MMClockGatingSupport,
+	PHM_PlatformCaps_AutomaticDCTransition,
+	PHM_PlatformCaps_GeminiPrimary,
+	PHM_PlatformCaps_MemorySpreadSpectrumSupport,
+	PHM_PlatformCaps_EngineSpreadSpectrumSupport,
+	PHM_PlatformCaps_StepVddc,
+	PHM_PlatformCaps_DynamicPCIEGen2Support,
+	PHM_PlatformCaps_SMC,
+	PHM_PlatformCaps_FaultyInternalThermalReading,          /* Internal thermal controller reports faulty temperature value when DAC2 is active */
+	PHM_PlatformCaps_EnableVoltageControl,                  /* indicates voltage can be controlled */
+	PHM_PlatformCaps_EnableSideportControl,                 /* indicates Sideport can be controlled */
+	PHM_PlatformCaps_VideoPlaybackEEUNotification,          /* indicates EEU notification of video start/stop is required */
+	PHM_PlatformCaps_TurnOffPll_ASPML1,                     /* PCIE Turn Off PLL in ASPM L1 */
+	PHM_PlatformCaps_EnableHTLinkControl,                   /* indicates HT Link can be controlled by ACPI or CLMC overrided/automated mode. */
+	PHM_PlatformCaps_PerformanceStateOnly,                  /* indicates only performance power state to be used on current system. */
+	PHM_PlatformCaps_ExclusiveModeAlwaysHigh,               /* In Exclusive (3D) mode always stay in High state. */
+	PHM_PlatformCaps_DisableMGClockGating,                  /* to disable Medium Grain Clock Gating or not */
+	PHM_PlatformCaps_DisableMGCGTSSM,                       /* TO disable Medium Grain Clock Gating Shader Complex control */
+	PHM_PlatformCaps_UVDAlwaysHigh,                         /* In UVD mode always stay in High state */
+	PHM_PlatformCaps_DisablePowerGating,                    /* to disable power gating */
+	PHM_PlatformCaps_CustomThermalPolicy,                   /* indicates only performance power state to be used on current system. */
+	PHM_PlatformCaps_StayInBootState,                       /* Stay in Boot State, do not do clock/voltage or PCIe Lane and Gen switching (RV7xx and up). */
+	PHM_PlatformCaps_SMCAllowSeparateSWThermalState,        /* SMC use separate SW thermal state, instead of the default SMC thermal policy. */
+	PHM_PlatformCaps_MultiUVDStateSupport,                  /* Powerplay state table supports multi UVD states. */
+	PHM_PlatformCaps_EnableSCLKDeepSleepForUVD,             /* With HW ECOs, we don't need to disable SCLK Deep Sleep for UVD state. */
+	PHM_PlatformCaps_EnableMCUHTLinkControl,                /* Enable HT link control by MCU */
+	PHM_PlatformCaps_ABM,                                   /* ABM support.*/
+	PHM_PlatformCaps_KongThermalPolicy,                     /* A thermal policy specific for Kong */
+	PHM_PlatformCaps_SwitchVDDNB,                           /* if the users want to switch VDDNB */
+	PHM_PlatformCaps_ULPS,                                  /* support ULPS mode either through ACPI state or ULPS state */
+	PHM_PlatformCaps_NativeULPS,                            /* hardware capable of ULPS state (other than through the ACPI state) */
+	PHM_PlatformCaps_EnableMVDDControl,                     /* indicates that memory voltage can be controlled */
+	PHM_PlatformCaps_ControlVDDCI,                          /* Control VDDCI separately from VDDC. */
+	PHM_PlatformCaps_DisableDCODT,                          /* indicates if DC ODT apply or not */
+	PHM_PlatformCaps_DynamicACTiming,                       /* if the SMC dynamically re-programs MC SEQ register values */
+	PHM_PlatformCaps_EnableThermalIntByGPIO,                /* enable throttle control through GPIO */
+	PHM_PlatformCaps_BootStateOnAlert,                      /* Go to boot state on alerts, e.g. on an AC->DC transition. */
+	PHM_PlatformCaps_DontWaitForVBlankOnAlert,              /* Do NOT wait for VBLANK during an alert (e.g. AC->DC transition). */
+	PHM_PlatformCaps_Force3DClockSupport,                   /* indicates if the platform supports force 3D clock. */
+	PHM_PlatformCaps_MicrocodeFanControl,                   /* Fan is controlled by the SMC microcode. */
+	PHM_PlatformCaps_AdjustUVDPriorityForSP,
+	PHM_PlatformCaps_DisableLightSleep,                     /* Light sleep for evergreen family. */
+	PHM_PlatformCaps_DisableMCLS,                           /* MC Light sleep */
+	PHM_PlatformCaps_RegulatorHot,                          /* Enable throttling on 'regulator hot' events. */
+	PHM_PlatformCaps_BACO,                                  /* Support Bus Alive Chip Off mode */
+	PHM_PlatformCaps_DisableDPM,                            /* Disable DPM, supported from Llano */
+	PHM_PlatformCaps_DynamicM3Arbiter,                      /* support dynamically change m3 arbitor parameters */
+	PHM_PlatformCaps_SclkDeepSleep,                         /* support sclk deep sleep */
+	PHM_PlatformCaps_DynamicPatchPowerState,                /* this ASIC supports to patch power state dynamically */
+	PHM_PlatformCaps_ThermalAutoThrottling,                 /* enabling auto thermal throttling, */
+	PHM_PlatformCaps_SumoThermalPolicy,                     /* A thermal policy specific for Sumo */
+	PHM_PlatformCaps_PCIEPerformanceRequest,                /* support to change RC voltage */
+	PHM_PlatformCaps_BLControlledByGPU,                     /* support varibright */
+	PHM_PlatformCaps_PowerContainment,                      /* support DPM2 power containment (AKA TDP clamping) */
+	PHM_PlatformCaps_SQRamping,                             /* support DPM2 SQ power throttle */
+	PHM_PlatformCaps_CAC,                                   /* support Capacitance * Activity power estimation */
+	PHM_PlatformCaps_NIChipsets,                            /* Northern Island and beyond chipsets */
+	PHM_PlatformCaps_TrinityChipsets,                       /* Trinity chipset */
+	PHM_PlatformCaps_EvergreenChipsets,                     /* Evergreen family chipset */
+	PHM_PlatformCaps_PowerControl,                          /* Cayman and beyond chipsets */
+	PHM_PlatformCaps_DisableLSClockGating,                  /* to disable Light Sleep control for HDP memories */
+	PHM_PlatformCaps_BoostState,                            /* this ASIC supports boost state */
+	PHM_PlatformCaps_UserMaxClockForMultiDisplays,          /* indicates if max memory clock is used for all status when multiple displays are connected */
+	PHM_PlatformCaps_RegWriteDelay,                         /* indicates if back to back reg write delay is required */
+	PHM_PlatformCaps_NonABMSupportInPPLib,                  /* ABM is not supported in PPLIB, (moved from PPLIB to DAL) */
+	PHM_PlatformCaps_GFXDynamicMGPowerGating,               /* Enable Dynamic MG PowerGating on Trinity */
+	PHM_PlatformCaps_DisableSMUUVDHandshake,                /* Disable SMU UVD Handshake */
+	PHM_PlatformCaps_DTE,                                   /* Support Digital Temperature Estimation */
+	PHM_PlatformCaps_W5100Specifc_SmuSkipMsgDTE,            /* This is for the feature requested by David B., and Tonny W.*/
+	PHM_PlatformCaps_UVDPowerGating,                        /* enable UVD power gating, supported from Llano */
+	PHM_PlatformCaps_UVDDynamicPowerGating,                 /* enable UVD Dynamic power gating, supported from UVD5 */
+	PHM_PlatformCaps_VCEPowerGating,                        /* Enable VCE power gating, supported for TN and later ASICs */
+	PHM_PlatformCaps_SamuPowerGating,                       /* Enable SAMU power gating, supported for KV and later ASICs */
+	PHM_PlatformCaps_UVDDPM,                                /* UVD clock DPM */
+	PHM_PlatformCaps_VCEDPM,                                /* VCE clock DPM */
+	PHM_PlatformCaps_SamuDPM,                               /* SAMU clock DPM */
+	PHM_PlatformCaps_AcpDPM,                                /* ACP clock DPM */
+	PHM_PlatformCaps_SclkDeepSleepAboveLow,                 /* Enable SCLK Deep Sleep on all DPM states */
+	PHM_PlatformCaps_DynamicUVDState,                       /* Dynamic UVD State */
+	PHM_PlatformCaps_WantSAMClkWithDummyBackEnd,            /* Set SAM Clk With Dummy Back End */
+	PHM_PlatformCaps_WantUVDClkWithDummyBackEnd,            /* Set UVD Clk With Dummy Back End */
+	PHM_PlatformCaps_WantVCEClkWithDummyBackEnd,            /* Set VCE Clk With Dummy Back End */
+	PHM_PlatformCaps_WantACPClkWithDummyBackEnd,            /* Set SAM Clk With Dummy Back End */
+	PHM_PlatformCaps_OD6inACSupport,                        /* indicates that the ASIC/back end supports OD6 */
+	PHM_PlatformCaps_OD6inDCSupport,                        /* indicates that the ASIC/back end supports OD6 in DC */
+	PHM_PlatformCaps_EnablePlatformPowerManagement,         /* indicates that Platform Power Management feature is supported */
+	PHM_PlatformCaps_SurpriseRemoval,                       /* indicates that surprise removal feature is requested */
+	PHM_PlatformCaps_NewCACVoltage,                         /* indicates new CAC voltage table support */
+	PHM_PlatformCaps_DBRamping,                             /* for dI/dT feature */
+	PHM_PlatformCaps_TDRamping,                             /* for dI/dT feature */
+	PHM_PlatformCaps_TCPRamping,                            /* for dI/dT feature */
+	PHM_PlatformCaps_EnableSMU7ThermalManagement,           /* SMC will manage thermal events */
+	PHM_PlatformCaps_FPS,                                   /* FPS support */
+	PHM_PlatformCaps_ACP,                                   /* ACP support */
+	PHM_PlatformCaps_SclkThrottleLowNotification,           /* SCLK Throttle Low Notification */
+	PHM_PlatformCaps_XDMAEnabled,                           /* XDMA engine is enabled */
+	PHM_PlatformCaps_UseDummyBackEnd,                       /* use dummy back end */
+	PHM_PlatformCaps_EnableDFSBypass,                       /* Enable DFS bypass */
+	PHM_PlatformCaps_VddNBDirectRequest,
+	PHM_PlatformCaps_PauseMMSessions,
+	PHM_PlatformCaps_UnTabledHardwareInterface,             /* Tableless/direct call hardware interface for CI and newer ASICs */
+	PHM_PlatformCaps_SMU7,                                  /* indicates that vpuRecoveryBegin without SMU shutdown */
+	PHM_PlatformCaps_RevertGPIO5Polarity,                   /* indicates revert GPIO5 plarity table support */
+	PHM_PlatformCaps_Thermal2GPIO17,                        /* indicates thermal2GPIO17 table support */
+	PHM_PlatformCaps_ThermalOutGPIO,                        /* indicates ThermalOutGPIO support, pin number is assigned by VBIOS */
+	PHM_PlatformCaps_DisableMclkSwitchingForFrameLock,      /* Disable memory clock switch during Framelock */
+	PHM_PlatformCaps_VRHotGPIOConfigurable,                 /* indicates VR_HOT GPIO configurable */
+	PHM_PlatformCaps_TempInversion,                         /* enable Temp Inversion feature */
+	PHM_PlatformCaps_IOIC3,
+	PHM_PlatformCaps_ConnectedStandby,
+	PHM_PlatformCaps_EVV,
+	PHM_PlatformCaps_EnableLongIdleBACOSupport,
+	PHM_PlatformCaps_CombinePCCWithThermalSignal,
+	PHM_PlatformCaps_DisableUsingActualTemperatureForPowerCalc,
+	PHM_PlatformCaps_StablePState,
+	PHM_PlatformCaps_OD6PlusinACSupport,
+	PHM_PlatformCaps_OD6PlusinDCSupport,
+	PHM_PlatformCaps_ODThermalLimitUnlock,
+	PHM_PlatformCaps_ReducePowerLimit,
+	PHM_PlatformCaps_ODFuzzyFanControlSupport,
+	PHM_PlatformCaps_GeminiRegulatorFanControlSupport,
+	PHM_PlatformCaps_ControlVDDGFX,
+	PHM_PlatformCaps_BBBSupported,
+	PHM_PlatformCaps_DisableVoltageIsland,
+	PHM_PlatformCaps_FanSpeedInTableIsRPM,
+	PHM_PlatformCaps_GFXClockGatingManagedInCAIL,
+	PHM_PlatformCaps_IcelandULPSSWWorkAround,
+	PHM_PlatformCaps_FPSEnhancement,
+	PHM_PlatformCaps_LoadPostProductionFirmware,
+	PHM_PlatformCaps_VpuRecoveryInProgress,
+	PHM_PlatformCaps_Falcon_QuickTransition,
+	PHM_PlatformCaps_AVFS,
+	PHM_PlatformCaps_ClockStretcher,
+	PHM_PlatformCaps_TablelessHardwareInterface,
+	PHM_PlatformCaps_EnableDriverEVV,
+	PHM_PlatformCaps_Max
+};
+
+#define PHM_MAX_NUM_CAPS_BITS_PER_FIELD (sizeof(uint32_t)*8)
+
+/* Number of uint32_t entries used by CAPS table */
+#define PHM_MAX_NUM_CAPS_ULONG_ENTRIES \
+	((PHM_PlatformCaps_Max + ((PHM_MAX_NUM_CAPS_BITS_PER_FIELD) - 1)) / (PHM_MAX_NUM_CAPS_BITS_PER_FIELD))
+
+struct pp_hw_descriptor {
+	uint32_t hw_caps[PHM_MAX_NUM_CAPS_ULONG_ENTRIES];
+};
+
+enum PHM_PerformanceLevelDesignation {
+	PHM_PerformanceLevelDesignation_Activity,
+	PHM_PerformanceLevelDesignation_PowerContainment
+};
+
+typedef enum PHM_PerformanceLevelDesignation PHM_PerformanceLevelDesignation;
+
+struct PHM_PerformanceLevel {
+    uint32_t    coreClock;
+    uint32_t    memory_clock;
+    uint32_t  vddc;
+    uint32_t  vddci;
+    uint32_t    nonLocalMemoryFreq;
+    uint32_t nonLocalMemoryWidth;
+};
+
+typedef struct PHM_PerformanceLevel PHM_PerformanceLevel;
+
+/* Function for setting a platform cap */
+static inline void phm_cap_set(uint32_t *caps,
+			enum phm_platform_caps c)
+{
+	caps[c / PHM_MAX_NUM_CAPS_BITS_PER_FIELD] |= (1UL <<
+			     (c & (PHM_MAX_NUM_CAPS_BITS_PER_FIELD - 1)));
+}
+
+static inline void phm_cap_unset(uint32_t *caps,
+			enum phm_platform_caps c)
+{
+	caps[c / PHM_MAX_NUM_CAPS_BITS_PER_FIELD] &= ~(1UL << (c & (PHM_MAX_NUM_CAPS_BITS_PER_FIELD - 1)));
+}
+
+static inline bool phm_cap_enabled(const uint32_t *caps, enum phm_platform_caps c)
+{
+	return (0 != (caps[c / PHM_MAX_NUM_CAPS_BITS_PER_FIELD] &
+		  (1UL << (c & (PHM_MAX_NUM_CAPS_BITS_PER_FIELD - 1)))));
+}
+
+#define PP_PCIEGenInvalid  0xffff
+enum PP_PCIEGen {
+    PP_PCIEGen1 = 0,                /* PCIE 1.0 - Transfer rate of 2.5 GT/s */
+    PP_PCIEGen2,                    /*PCIE 2.0 - Transfer rate of 5.0 GT/s */
+    PP_PCIEGen3                     /*PCIE 3.0 - Transfer rate of 8.0 GT/s */
+};
+
+typedef enum PP_PCIEGen PP_PCIEGen;
+
+#define PP_Min_PCIEGen     PP_PCIEGen1
+#define PP_Max_PCIEGen     PP_PCIEGen3
+#define PP_Min_PCIELane    1
+#define PP_Max_PCIELane    32
+
+enum phm_clock_Type {
+	PHM_DispClock = 1,
+	PHM_SClock,
+	PHM_MemClock
+};
+
+#define MAX_NUM_CLOCKS 16
+
+struct PP_Clocks {
+	uint32_t engineClock;
+	uint32_t memoryClock;
+	uint32_t BusBandwidth;
+	uint32_t engineClockInSR;
+};
+
+struct phm_platform_descriptor {
+	uint32_t platformCaps[PHM_MAX_NUM_CAPS_ULONG_ENTRIES];
+	uint32_t vbiosInterruptId;
+	struct PP_Clocks overdriveLimit;
+	struct PP_Clocks clockStep;
+	uint32_t hardwareActivityPerformanceLevels;
+	uint32_t minimumClocksReductionPercentage;
+	uint32_t minOverdriveVDDC;
+	uint32_t maxOverdriveVDDC;
+	uint32_t overdriveVDDCStep;
+	uint32_t hardwarePerformanceLevels;
+	uint16_t powerBudget;
+	uint32_t TDPLimit;
+	uint32_t nearTDPLimit;
+	uint32_t nearTDPLimitAdjusted;
+	uint32_t SQRampingThreshold;
+	uint32_t CACLeakage;
+	uint16_t TDPODLimit;
+	uint32_t TDPAdjustment;
+	bool TDPAdjustmentPolarity;
+	uint16_t LoadLineSlope;
+	uint32_t  VidMinLimit;
+	uint32_t  VidMaxLimit;
+	uint32_t  VidStep;
+	uint32_t  VidAdjustment;
+	bool VidAdjustmentPolarity;
+};
+
+struct phm_clocks {
+	uint32_t num_of_entries;
+	uint32_t clock[MAX_NUM_CLOCKS];
+};
+
+enum PP_DAL_POWERLEVEL {
+	PP_DAL_POWERLEVEL_INVALID = 0,
+	PP_DAL_POWERLEVEL_ULTRALOW,
+	PP_DAL_POWERLEVEL_LOW,
+	PP_DAL_POWERLEVEL_NOMINAL,
+	PP_DAL_POWERLEVEL_PERFORMANCE,
+
+	PP_DAL_POWERLEVEL_0 = PP_DAL_POWERLEVEL_ULTRALOW,
+	PP_DAL_POWERLEVEL_1 = PP_DAL_POWERLEVEL_LOW,
+	PP_DAL_POWERLEVEL_2 = PP_DAL_POWERLEVEL_NOMINAL,
+	PP_DAL_POWERLEVEL_3 = PP_DAL_POWERLEVEL_PERFORMANCE,
+	PP_DAL_POWERLEVEL_4 = PP_DAL_POWERLEVEL_3+1,
+	PP_DAL_POWERLEVEL_5 = PP_DAL_POWERLEVEL_4+1,
+	PP_DAL_POWERLEVEL_6 = PP_DAL_POWERLEVEL_5+1,
+	PP_DAL_POWERLEVEL_7 = PP_DAL_POWERLEVEL_6+1,
+};
+
+
+extern int phm_enable_clock_power_gatings(struct pp_hwmgr *hwmgr);
+extern int phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool gate);
+extern int phm_powergate_vce(struct pp_hwmgr *hwmgr, bool gate);
+extern int phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
+extern int phm_setup_asic(struct pp_hwmgr *hwmgr);
+extern int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr);
+extern void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr);
+extern bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr);
+extern int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block);
+extern int phm_set_power_state(struct pp_hwmgr *hwmgr,
+		    const struct pp_hw_power_state *pcurrent_state,
+		 const struct pp_hw_power_state *pnew_power_state);
+
+extern int phm_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				   struct pp_power_state *adjusted_ps,
+			     const struct pp_power_state *current_ps);
+
+extern int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level);
+extern int phm_display_configuration_changed(struct pp_hwmgr *hwmgr);
+extern int phm_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr);
+extern int phm_register_thermal_interrupt(struct pp_hwmgr *hwmgr, const void *info);
+extern int phm_start_thermal_controller(struct pp_hwmgr *hwmgr, struct PP_TemperatureRange *temperature_range);
+extern int phm_stop_thermal_controller(struct pp_hwmgr *hwmgr);
+extern bool phm_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr);
+
+extern int phm_check_states_equal(struct pp_hwmgr *hwmgr,
+				 const struct pp_hw_power_state *pstate1,
+				 const struct pp_hw_power_state *pstate2,
+				 bool *equal);
+
+extern int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr,
+		const struct amd_pp_display_configuration *display_config);
+
+extern int phm_get_dal_power_level(struct pp_hwmgr *hwmgr,
+		struct amd_pp_dal_clock_info*info);
+
+extern int phm_set_cpu_power_state(struct pp_hwmgr *hwmgr);
+
+extern int phm_power_down_asic(struct pp_hwmgr *hwmgr);
+
+#endif /* _HARDWARE_MANAGER_H_ */
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
new file mode 100644
index 000000000000..aeaa3dbba525
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -0,0 +1,801 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _HWMGR_H_
+#define _HWMGR_H_
+
+#include <linux/seq_file.h>
+#include "amd_powerplay.h"
+#include "pp_instance.h"
+#include "hardwaremanager.h"
+#include "pp_power_source.h"
+#include "hwmgr_ppt.h"
+#include "ppatomctrl.h"
+#include "hwmgr_ppt.h"
+
+struct pp_instance;
+struct pp_hwmgr;
+struct pp_hw_power_state;
+struct pp_power_state;
+struct PP_VCEState;
+struct phm_fan_speed_info;
+struct pp_atomctrl_voltage_table;
+
+
+enum DISPLAY_GAP {
+	DISPLAY_GAP_VBLANK_OR_WM = 0,   /* Wait for vblank or MCHG watermark. */
+	DISPLAY_GAP_VBLANK       = 1,   /* Wait for vblank. */
+	DISPLAY_GAP_WATERMARK    = 2,   /* Wait for MCHG watermark. (Note that HW may deassert WM in VBI depending on DC_STUTTER_CNTL.) */
+	DISPLAY_GAP_IGNORE       = 3    /* Do not wait. */
+};
+typedef enum DISPLAY_GAP DISPLAY_GAP;
+
+
+struct vi_dpm_level {
+	bool enabled;
+	uint32_t value;
+	uint32_t param1;
+};
+
+struct vi_dpm_table {
+	uint32_t count;
+	struct vi_dpm_level dpm_level[1];
+};
+
+enum PP_Result {
+	PP_Result_TableImmediateExit = 0x13,
+};
+
+#define PCIE_PERF_REQ_REMOVE_REGISTRY   0
+#define PCIE_PERF_REQ_FORCE_LOWPOWER    1
+#define PCIE_PERF_REQ_GEN1         2
+#define PCIE_PERF_REQ_GEN2         3
+#define PCIE_PERF_REQ_GEN3         4
+
+enum PHM_BackEnd_Magic {
+	PHM_Dummy_Magic       = 0xAA5555AA,
+	PHM_RV770_Magic       = 0xDCBAABCD,
+	PHM_Kong_Magic        = 0x239478DF,
+	PHM_NIslands_Magic    = 0x736C494E,
+	PHM_Sumo_Magic        = 0x8339FA11,
+	PHM_SIslands_Magic    = 0x369431AC,
+	PHM_Trinity_Magic     = 0x96751873,
+	PHM_CIslands_Magic    = 0x38AC78B0,
+	PHM_Kv_Magic          = 0xDCBBABC0,
+	PHM_VIslands_Magic    = 0x20130307,
+	PHM_Cz_Magic          = 0x67DCBA25
+};
+
+
+#define PHM_PCIE_POWERGATING_TARGET_GFX            0
+#define PHM_PCIE_POWERGATING_TARGET_DDI            1
+#define PHM_PCIE_POWERGATING_TARGET_PLLCASCADE     2
+#define PHM_PCIE_POWERGATING_TARGET_PHY            3
+
+typedef int (*phm_table_function)(struct pp_hwmgr *hwmgr, void *input,
+				  void *output, void *storage, int result);
+
+typedef bool (*phm_check_function)(struct pp_hwmgr *hwmgr);
+
+struct phm_set_power_state_input {
+	const struct pp_hw_power_state *pcurrent_state;
+	const struct pp_hw_power_state *pnew_state;
+};
+
+struct phm_acp_arbiter {
+	uint32_t acpclk;
+};
+
+struct phm_uvd_arbiter {
+	uint32_t vclk;
+	uint32_t dclk;
+	uint32_t vclk_ceiling;
+	uint32_t dclk_ceiling;
+};
+
+struct phm_vce_arbiter {
+	uint32_t   evclk;
+	uint32_t   ecclk;
+};
+
+struct phm_gfx_arbiter {
+	uint32_t sclk;
+	uint32_t mclk;
+	uint32_t sclk_over_drive;
+	uint32_t mclk_over_drive;
+	uint32_t sclk_threshold;
+	uint32_t num_cus;
+};
+
+/* Entries in the master tables */
+struct phm_master_table_item {
+	phm_check_function isFunctionNeededInRuntimeTable;
+	phm_table_function tableFunction;
+};
+
+enum phm_master_table_flag {
+	PHM_MasterTableFlag_None         = 0,
+	PHM_MasterTableFlag_ExitOnError  = 1,
+};
+
+/* The header of the master tables */
+struct phm_master_table_header {
+	uint32_t storage_size;
+	uint32_t flags;
+	struct phm_master_table_item *master_list;
+};
+
+struct phm_runtime_table_header {
+	uint32_t storage_size;
+	bool exit_error;
+	phm_table_function *function_list;
+};
+
+struct phm_clock_array {
+	uint32_t count;
+	uint32_t values[1];
+};
+
+struct phm_clock_voltage_dependency_record {
+	uint32_t clk;
+	uint32_t v;
+};
+
+struct phm_vceclock_voltage_dependency_record {
+	uint32_t ecclk;
+	uint32_t evclk;
+	uint32_t v;
+};
+
+struct phm_uvdclock_voltage_dependency_record {
+	uint32_t vclk;
+	uint32_t dclk;
+	uint32_t v;
+};
+
+struct phm_samuclock_voltage_dependency_record {
+	uint32_t samclk;
+	uint32_t v;
+};
+
+struct phm_acpclock_voltage_dependency_record {
+	uint32_t acpclk;
+	uint32_t v;
+};
+
+struct phm_clock_voltage_dependency_table {
+	uint32_t count;										/* Number of entries. */
+	struct phm_clock_voltage_dependency_record entries[1];		/* Dynamically allocate count entries. */
+};
+
+struct phm_phase_shedding_limits_record {
+	uint32_t  Voltage;
+	uint32_t    Sclk;
+	uint32_t    Mclk;
+};
+
+
+extern int phm_dispatch_table(struct pp_hwmgr *hwmgr,
+			      struct phm_runtime_table_header *rt_table,
+			      void *input, void *output);
+
+extern int phm_construct_table(struct pp_hwmgr *hwmgr,
+			       struct phm_master_table_header *master_table,
+			       struct phm_runtime_table_header *rt_table);
+
+extern int phm_destroy_table(struct pp_hwmgr *hwmgr,
+			     struct phm_runtime_table_header *rt_table);
+
+
+struct phm_uvd_clock_voltage_dependency_record {
+	uint32_t vclk;
+	uint32_t dclk;
+	uint32_t v;
+};
+
+struct phm_uvd_clock_voltage_dependency_table {
+	uint8_t count;
+	struct phm_uvd_clock_voltage_dependency_record entries[1];
+};
+
+struct phm_acp_clock_voltage_dependency_record {
+	uint32_t acpclk;
+	uint32_t v;
+};
+
+struct phm_acp_clock_voltage_dependency_table {
+	uint32_t count;
+	struct phm_acp_clock_voltage_dependency_record entries[1];
+};
+
+struct phm_vce_clock_voltage_dependency_record {
+	uint32_t ecclk;
+	uint32_t evclk;
+	uint32_t v;
+};
+
+struct phm_phase_shedding_limits_table {
+	uint32_t                           count;
+	struct phm_phase_shedding_limits_record  entries[1];
+};
+
+struct phm_vceclock_voltage_dependency_table {
+	uint8_t count;                                    /* Number of entries. */
+	struct phm_vceclock_voltage_dependency_record entries[1]; /* Dynamically allocate count entries. */
+};
+
+struct phm_uvdclock_voltage_dependency_table {
+	uint8_t count;                                    /* Number of entries. */
+	struct phm_uvdclock_voltage_dependency_record entries[1]; /* Dynamically allocate count entries. */
+};
+
+struct phm_samuclock_voltage_dependency_table {
+	uint8_t count;                                    /* Number of entries. */
+	struct phm_samuclock_voltage_dependency_record entries[1]; /* Dynamically allocate count entries. */
+};
+
+struct phm_acpclock_voltage_dependency_table {
+	uint32_t count;                                    /* Number of entries. */
+	struct phm_acpclock_voltage_dependency_record entries[1]; /* Dynamically allocate count entries. */
+};
+
+struct phm_vce_clock_voltage_dependency_table {
+	uint8_t count;
+	struct phm_vce_clock_voltage_dependency_record entries[1];
+};
+
+struct pp_hwmgr_func {
+	int (*backend_init)(struct pp_hwmgr *hw_mgr);
+	int (*backend_fini)(struct pp_hwmgr *hw_mgr);
+	int (*asic_setup)(struct pp_hwmgr *hw_mgr);
+	int (*get_power_state_size)(struct pp_hwmgr *hw_mgr);
+
+	int (*apply_state_adjust_rules)(struct pp_hwmgr *hwmgr,
+				struct pp_power_state  *prequest_ps,
+			const struct pp_power_state *pcurrent_ps);
+
+	int (*force_dpm_level)(struct pp_hwmgr *hw_mgr,
+					enum amd_dpm_forced_level level);
+
+	int (*dynamic_state_management_enable)(
+						struct pp_hwmgr *hw_mgr);
+
+	int (*patch_boot_state)(struct pp_hwmgr *hwmgr,
+				     struct pp_hw_power_state *hw_ps);
+
+	int (*get_pp_table_entry)(struct pp_hwmgr *hwmgr,
+			    unsigned long, struct pp_power_state *);
+	int (*get_num_of_pp_table_entries)(struct pp_hwmgr *hwmgr);
+	int (*powerdown_uvd)(struct pp_hwmgr *hwmgr);
+	int (*powergate_vce)(struct pp_hwmgr *hwmgr, bool bgate);
+	int (*powergate_uvd)(struct pp_hwmgr *hwmgr, bool bgate);
+	int (*get_mclk)(struct pp_hwmgr *hwmgr, bool low);
+	int (*get_sclk)(struct pp_hwmgr *hwmgr, bool low);
+	int (*power_state_set)(struct pp_hwmgr *hwmgr,
+						const void *state);
+	void (*print_current_perforce_level)(struct pp_hwmgr *hwmgr,
+							struct seq_file *m);
+	int (*enable_clock_power_gating)(struct pp_hwmgr *hwmgr);
+	int (*notify_smc_display_config_after_ps_adjustment)(struct pp_hwmgr *hwmgr);
+	int (*display_config_changed)(struct pp_hwmgr *hwmgr);
+	int (*disable_clock_power_gating)(struct pp_hwmgr *hwmgr);
+	int (*update_clock_gatings)(struct pp_hwmgr *hwmgr,
+						const uint32_t *msg_id);
+	int (*set_max_fan_rpm_output)(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm);
+	int (*set_max_fan_pwm_output)(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm);
+	int (*get_temperature)(struct pp_hwmgr *hwmgr);
+	int (*stop_thermal_controller)(struct pp_hwmgr *hwmgr);
+	int (*get_fan_speed_info)(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
+	int (*set_fan_control_mode)(struct pp_hwmgr *hwmgr, uint32_t mode);
+	int (*get_fan_control_mode)(struct pp_hwmgr *hwmgr);
+	int (*set_fan_speed_percent)(struct pp_hwmgr *hwmgr, uint32_t percent);
+	int (*get_fan_speed_percent)(struct pp_hwmgr *hwmgr, uint32_t *speed);
+	int (*set_fan_speed_rpm)(struct pp_hwmgr *hwmgr, uint32_t percent);
+	int (*get_fan_speed_rpm)(struct pp_hwmgr *hwmgr, uint32_t *speed);
+	int (*reset_fan_speed_to_default)(struct pp_hwmgr *hwmgr);
+	int (*uninitialize_thermal_controller)(struct pp_hwmgr *hwmgr);
+	int (*register_internal_thermal_interrupt)(struct pp_hwmgr *hwmgr,
+					const void *thermal_interrupt_info);
+	bool (*check_smc_update_required_for_display_configuration)(struct pp_hwmgr *hwmgr);
+	int (*check_states_equal)(struct pp_hwmgr *hwmgr,
+					const struct pp_hw_power_state *pstate1,
+					const struct pp_hw_power_state *pstate2,
+					bool *equal);
+	int (*set_cpu_power_state)(struct pp_hwmgr *hwmgr);
+	int (*store_cc6_data)(struct pp_hwmgr *hwmgr, uint32_t separation_time,
+				bool cc6_disable, bool pstate_disable,
+				bool pstate_switch_disable);
+	int (*get_dal_power_level)(struct pp_hwmgr *hwmgr,
+				   struct amd_pp_dal_clock_info *info);
+	int (*power_off_asic)(struct pp_hwmgr *hwmgr);
+};
+
+struct pp_table_func {
+	int (*pptable_init)(struct pp_hwmgr *hw_mgr);
+	int (*pptable_fini)(struct pp_hwmgr *hw_mgr);
+	int (*pptable_get_number_of_vce_state_table_entries)(struct pp_hwmgr *hw_mgr);
+	int (*pptable_get_vce_state_table_entry)(
+						struct pp_hwmgr *hwmgr,
+						unsigned long i,
+						struct PP_VCEState *vce_state,
+						void **clock_info,
+						unsigned long *flag);
+};
+
+union phm_cac_leakage_record {
+	struct {
+		uint16_t Vddc;          /* in CI, we use it for StdVoltageHiSidd */
+		uint32_t Leakage;       /* in CI, we use it for StdVoltageLoSidd */
+	};
+	struct {
+		uint16_t Vddc1;
+		uint16_t Vddc2;
+		uint16_t Vddc3;
+	};
+};
+
+struct phm_cac_leakage_table {
+	uint32_t count;
+	union phm_cac_leakage_record entries[1];
+};
+
+struct phm_samu_clock_voltage_dependency_record {
+	uint32_t samclk;
+	uint32_t v;
+};
+
+
+struct phm_samu_clock_voltage_dependency_table {
+	uint8_t count;
+	struct phm_samu_clock_voltage_dependency_record entries[1];
+};
+
+struct phm_cac_tdp_table {
+	uint16_t usTDP;
+	uint16_t usConfigurableTDP;
+	uint16_t usTDC;
+	uint16_t usBatteryPowerLimit;
+	uint16_t usSmallPowerLimit;
+	uint16_t usLowCACLeakage;
+	uint16_t usHighCACLeakage;
+	uint16_t usMaximumPowerDeliveryLimit;
+	uint16_t usOperatingTempMinLimit;
+	uint16_t usOperatingTempMaxLimit;
+	uint16_t usOperatingTempStep;
+	uint16_t usOperatingTempHyst;
+	uint16_t usDefaultTargetOperatingTemp;
+	uint16_t usTargetOperatingTemp;
+	uint16_t usPowerTuneDataSetID;
+	uint16_t usSoftwareShutdownTemp;
+	uint16_t usClockStretchAmount;
+	uint16_t usTemperatureLimitHotspot;
+	uint16_t usTemperatureLimitLiquid1;
+	uint16_t usTemperatureLimitLiquid2;
+	uint16_t usTemperatureLimitVrVddc;
+	uint16_t usTemperatureLimitVrMvdd;
+	uint16_t usTemperatureLimitPlx;
+	uint8_t  ucLiquid1_I2C_address;
+	uint8_t  ucLiquid2_I2C_address;
+	uint8_t  ucLiquid_I2C_Line;
+	uint8_t  ucVr_I2C_address;
+	uint8_t  ucVr_I2C_Line;
+	uint8_t  ucPlx_I2C_address;
+	uint8_t  ucPlx_I2C_Line;
+};
+
+struct phm_ppm_table {
+	uint8_t   ppm_design;
+	uint16_t  cpu_core_number;
+	uint32_t  platform_tdp;
+	uint32_t  small_ac_platform_tdp;
+	uint32_t  platform_tdc;
+	uint32_t  small_ac_platform_tdc;
+	uint32_t  apu_tdp;
+	uint32_t  dgpu_tdp;
+	uint32_t  dgpu_ulv_power;
+	uint32_t  tj_max;
+};
+
+struct phm_vq_budgeting_record {
+	uint32_t ulCUs;
+	uint32_t ulSustainableSOCPowerLimitLow;
+	uint32_t ulSustainableSOCPowerLimitHigh;
+	uint32_t ulMinSclkLow;
+	uint32_t ulMinSclkHigh;
+	uint8_t  ucDispConfig;
+	uint32_t ulDClk;
+	uint32_t ulEClk;
+	uint32_t ulSustainableSclk;
+	uint32_t ulSustainableCUs;
+};
+
+struct phm_vq_budgeting_table {
+	uint8_t numEntries;
+	struct phm_vq_budgeting_record entries[1];
+};
+
+struct phm_clock_and_voltage_limits {
+	uint32_t sclk;
+	uint32_t mclk;
+	uint16_t vddc;
+	uint16_t vddci;
+	uint16_t vddgfx;
+};
+
+/* Structure to hold PPTable information */
+
+struct phm_ppt_v1_information {
+	struct phm_ppt_v1_clock_voltage_dependency_table *vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *vdd_dep_on_mclk;
+	struct phm_clock_array *valid_sclk_values;
+	struct phm_clock_array *valid_mclk_values;
+	struct phm_clock_and_voltage_limits max_clock_voltage_on_dc;
+	struct phm_clock_and_voltage_limits max_clock_voltage_on_ac;
+	struct phm_clock_voltage_dependency_table *vddc_dep_on_dal_pwrl;
+	struct phm_ppm_table *ppm_parameter_table;
+	struct phm_cac_tdp_table *cac_dtp_table;
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_dep_table;
+	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table;
+	struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table;
+	struct phm_ppt_v1_pcie_table *pcie_table;
+	uint16_t us_ulv_voltage_offset;
+};
+
+struct phm_dynamic_state_info {
+	struct phm_clock_voltage_dependency_table *vddc_dependency_on_sclk;
+	struct phm_clock_voltage_dependency_table *vddci_dependency_on_mclk;
+	struct phm_clock_voltage_dependency_table *vddc_dependency_on_mclk;
+	struct phm_clock_voltage_dependency_table *mvdd_dependency_on_mclk;
+	struct phm_clock_voltage_dependency_table *vddc_dep_on_dal_pwrl;
+	struct phm_clock_array                    *valid_sclk_values;
+	struct phm_clock_array                    *valid_mclk_values;
+	struct phm_clock_and_voltage_limits       max_clock_voltage_on_dc;
+	struct phm_clock_and_voltage_limits       max_clock_voltage_on_ac;
+	uint32_t                                  mclk_sclk_ratio;
+	uint32_t                                  sclk_mclk_delta;
+	uint32_t                                  vddc_vddci_delta;
+	uint32_t                                  min_vddc_for_pcie_gen2;
+	struct phm_cac_leakage_table              *cac_leakage_table;
+	struct phm_phase_shedding_limits_table	  *vddc_phase_shed_limits_table;
+
+	struct phm_vce_clock_voltage_dependency_table
+					    *vce_clock_voltage_dependency_table;
+	struct phm_uvd_clock_voltage_dependency_table
+					    *uvd_clock_voltage_dependency_table;
+	struct phm_acp_clock_voltage_dependency_table
+					    *acp_clock_voltage_dependency_table;
+	struct phm_samu_clock_voltage_dependency_table
+					   *samu_clock_voltage_dependency_table;
+
+	struct phm_ppm_table                          *ppm_parameter_table;
+	struct phm_cac_tdp_table                      *cac_dtp_table;
+	struct phm_clock_voltage_dependency_table	  *vdd_gfx_dependency_on_sclk;
+	struct phm_vq_budgeting_table		  		  *vq_budgeting_table;
+};
+
+struct pp_fan_info {
+	bool bNoFan;
+	uint8_t   ucTachometerPulsesPerRevolution;
+	uint32_t   ulMinRPM;
+	uint32_t   ulMaxRPM;
+};
+
+struct pp_advance_fan_control_parameters {
+	uint16_t  usTMin;                          /* The temperature, in 0.01 centigrades, below which we just run at a minimal PWM. */
+	uint16_t  usTMed;                          /* The middle temperature where we change slopes. */
+	uint16_t  usTHigh;                         /* The high temperature for setting the second slope. */
+	uint16_t  usPWMMin;                        /* The minimum PWM value in percent (0.01% increments). */
+	uint16_t  usPWMMed;                        /* The PWM value (in percent) at TMed. */
+	uint16_t  usPWMHigh;                       /* The PWM value at THigh. */
+	uint8_t   ucTHyst;                         /* Temperature hysteresis. Integer. */
+	uint32_t   ulCycleDelay;                   /* The time between two invocations of the fan control routine in microseconds. */
+	uint16_t  usTMax;                          /* The max temperature */
+	uint8_t   ucFanControlMode;
+	uint16_t  usFanPWMMinLimit;
+	uint16_t  usFanPWMMaxLimit;
+	uint16_t  usFanPWMStep;
+	uint16_t  usDefaultMaxFanPWM;
+	uint16_t  usFanOutputSensitivity;
+	uint16_t  usDefaultFanOutputSensitivity;
+	uint16_t  usMaxFanPWM;                     /* The max Fan PWM value for Fuzzy Fan Control feature */
+	uint16_t  usFanRPMMinLimit;                /* Minimum limit range in percentage, need to calculate based on minRPM/MaxRpm */
+	uint16_t  usFanRPMMaxLimit;                /* Maximum limit range in percentage, usually set to 100% by default */
+	uint16_t  usFanRPMStep;                    /* Step increments/decerements, in percent */
+	uint16_t  usDefaultMaxFanRPM;              /* The max Fan RPM value for Fuzzy Fan Control feature, default from PPTable */
+	uint16_t  usMaxFanRPM;                     /* The max Fan RPM value for Fuzzy Fan Control feature, user defined */
+	uint16_t  usFanCurrentLow;                 /* Low current */
+	uint16_t  usFanCurrentHigh;                /* High current */
+	uint16_t  usFanRPMLow;                     /* Low RPM */
+	uint16_t  usFanRPMHigh;                    /* High RPM */
+	uint32_t   ulMinFanSCLKAcousticLimit;      /* Minimum Fan Controller SCLK Frequency Acoustic Limit. */
+	uint8_t   ucTargetTemperature;             /* Advanced fan controller target temperature. */
+	uint8_t   ucMinimumPWMLimit;               /* The minimum PWM that the advanced fan controller can set.  This should be set to the highest PWM that will run the fan at its lowest RPM. */
+	uint16_t  usFanGainEdge;                   /* The following is added for Fiji */
+	uint16_t  usFanGainHotspot;
+	uint16_t  usFanGainLiquid;
+	uint16_t  usFanGainVrVddc;
+	uint16_t  usFanGainVrMvdd;
+	uint16_t  usFanGainPlx;
+	uint16_t  usFanGainHbm;
+};
+
+struct pp_thermal_controller_info {
+	uint8_t ucType;
+	uint8_t ucI2cLine;
+	uint8_t ucI2cAddress;
+	struct pp_fan_info fanInfo;
+	struct pp_advance_fan_control_parameters advanceFanControlParameters;
+};
+
+struct phm_microcode_version_info {
+	uint32_t SMC;
+	uint32_t DMCU;
+	uint32_t MC;
+	uint32_t NB;
+};
+
+/**
+ * The main hardware manager structure.
+ */
+struct pp_hwmgr {
+	uint32_t chip_family;
+	uint32_t chip_id;
+	uint32_t hw_revision;
+	uint32_t sub_sys_id;
+	uint32_t sub_vendor_id;
+
+	void *device;
+	struct pp_smumgr *smumgr;
+	const void *soft_pp_table;
+	bool need_pp_table_upload;
+	enum amd_dpm_forced_level dpm_level;
+	bool block_hw_access;
+	struct phm_gfx_arbiter gfx_arbiter;
+	struct phm_acp_arbiter acp_arbiter;
+	struct phm_uvd_arbiter uvd_arbiter;
+	struct phm_vce_arbiter vce_arbiter;
+	uint32_t usec_timeout;
+	void *pptable;
+	struct phm_platform_descriptor platform_descriptor;
+	void *backend;
+	enum PP_DAL_POWERLEVEL dal_power_level;
+	struct phm_dynamic_state_info dyn_state;
+	struct phm_runtime_table_header setup_asic;
+	struct phm_runtime_table_header power_down_asic;
+	struct phm_runtime_table_header disable_dynamic_state_management;
+	struct phm_runtime_table_header enable_dynamic_state_management;
+	struct phm_runtime_table_header set_power_state;
+	struct phm_runtime_table_header enable_clock_power_gatings;
+	struct phm_runtime_table_header display_configuration_changed;
+	struct phm_runtime_table_header start_thermal_controller;
+	struct phm_runtime_table_header set_temperature_range;
+	const struct pp_hwmgr_func *hwmgr_func;
+	const struct pp_table_func *pptable_func;
+	struct pp_power_state    *ps;
+	enum pp_power_source  power_source;
+	uint32_t num_ps;
+	struct pp_thermal_controller_info thermal_controller;
+	bool fan_ctrl_is_in_default_mode;
+	uint32_t fan_ctrl_default_mode;
+	uint32_t tmin;
+	struct phm_microcode_version_info microcode_version_info;
+	uint32_t ps_size;
+	struct pp_power_state    *current_ps;
+	struct pp_power_state    *request_ps;
+	struct pp_power_state    *boot_ps;
+	struct pp_power_state    *uvd_ps;
+	struct amd_pp_display_configuration display_config;
+};
+
+
+extern int hwmgr_init(struct amd_pp_init *pp_init,
+		      struct pp_instance *handle);
+
+extern int hwmgr_fini(struct pp_hwmgr *hwmgr);
+
+extern int hw_init_power_state_table(struct pp_hwmgr *hwmgr);
+
+extern int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
+				uint32_t value, uint32_t mask);
+
+extern int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr,
+				uint32_t index, uint32_t value, uint32_t mask);
+
+extern uint32_t phm_read_indirect_register(struct pp_hwmgr *hwmgr,
+		uint32_t indirect_port, uint32_t index);
+
+extern void phm_write_indirect_register(struct pp_hwmgr *hwmgr,
+		uint32_t indirect_port,
+		uint32_t index,
+		uint32_t value);
+
+extern void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
+				uint32_t indirect_port,
+				uint32_t index,
+				uint32_t value,
+				uint32_t mask);
+
+extern void phm_wait_for_indirect_register_unequal(
+				struct pp_hwmgr *hwmgr,
+				uint32_t indirect_port,
+				uint32_t index,
+				uint32_t value,
+				uint32_t mask);
+
+extern bool phm_cf_want_uvd_power_gating(struct pp_hwmgr *hwmgr);
+extern bool phm_cf_want_vce_power_gating(struct pp_hwmgr *hwmgr);
+extern bool phm_cf_want_microcode_fan_ctrl(struct pp_hwmgr *hwmgr);
+
+extern int phm_trim_voltage_table(struct pp_atomctrl_voltage_table *vol_table);
+extern int phm_get_svi2_mvdd_voltage_table(struct pp_atomctrl_voltage_table *vol_table, phm_ppt_v1_clock_voltage_dependency_table *dep_table);
+extern int phm_get_svi2_vddci_voltage_table(struct pp_atomctrl_voltage_table *vol_table, phm_ppt_v1_clock_voltage_dependency_table *dep_table);
+extern int phm_get_svi2_vdd_voltage_table(struct pp_atomctrl_voltage_table *vol_table, phm_ppt_v1_voltage_lookup_table *lookup_table);
+extern void phm_trim_voltage_table_to_fit_state_table(uint32_t max_vol_steps, struct pp_atomctrl_voltage_table *vol_table);
+extern int phm_reset_single_dpm_table(void *table, uint32_t count, int max);
+extern void phm_setup_pcie_table_entry(void *table, uint32_t index, uint32_t pcie_gen, uint32_t pcie_lanes);
+extern int32_t phm_get_dpm_level_enable_mask_value(void *table);
+extern uint8_t phm_get_voltage_index(struct phm_ppt_v1_voltage_lookup_table *lookup_table, uint16_t voltage);
+extern uint16_t phm_find_closest_vddci(struct pp_atomctrl_voltage_table *vddci_table, uint16_t vddci);
+extern int phm_find_boot_level(void *table, uint32_t value, uint32_t *boot_level);
+extern int phm_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr, phm_ppt_v1_voltage_lookup_table *lookup_table,
+								uint16_t virtual_voltage_id, int32_t *sclk);
+extern int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr);
+extern int phm_hwmgr_backend_fini(struct pp_hwmgr *hwmgr);
+extern uint32_t phm_get_lowest_enabled_level(struct pp_hwmgr *hwmgr, uint32_t mask);
+
+
+#define PHM_ENTIRE_REGISTER_MASK 0xFFFFFFFFU
+
+#define PHM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
+#define PHM_FIELD_MASK(reg, field) reg##__##field##_MASK
+
+#define PHM_SET_FIELD(origval, reg, field, fieldval)	\
+	(((origval) & ~PHM_FIELD_MASK(reg, field)) |	\
+	 (PHM_FIELD_MASK(reg, field) & ((fieldval) << PHM_FIELD_SHIFT(reg, field))))
+
+#define PHM_GET_FIELD(value, reg, field)	\
+	(((value) & PHM_FIELD_MASK(reg, field)) >>	\
+	 PHM_FIELD_SHIFT(reg, field))
+
+
+#define PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, index, value, mask)	\
+	phm_wait_on_register(hwmgr, index, value, mask)
+
+#define PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, index, value, mask)	\
+	phm_wait_for_register_unequal(hwmgr, index, value, mask)
+
+#define PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
+	phm_wait_on_indirect_register(hwmgr, mm##port##_INDEX, index, value, mask)
+
+#define PHM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
+	phm_wait_for_indirect_register_unequal(hwmgr, mm##port##_INDEX, index, value, mask)
+
+#define PHM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
+	phm_wait_on_indirect_register(hwmgr, mm##port##_INDEX_0, index, value, mask)
+
+#define PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
+	phm_wait_for_indirect_register_unequal(hwmgr, mm##port##_INDEX_0, index, value, mask)
+
+/* Operations on named registers. */
+
+#define PHM_WAIT_REGISTER(hwmgr, reg, value, mask)	\
+	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg, value, mask)
+
+#define PHM_WAIT_REGISTER_UNEQUAL(hwmgr, reg, value, mask)	\
+	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg, value, mask)
+
+#define PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, value, mask)	\
+	PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
+
+#define PHM_WAIT_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, value, mask)	\
+	PHM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
+
+#define PHM_WAIT_VFPF_INDIRECT_REGISTER(hwmgr, port, reg, value, mask)	\
+	PHM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
+
+#define PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, value, mask)	\
+	PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
+
+/* Operations on named fields. */
+
+#define PHM_READ_FIELD(device, reg, field)	\
+	PHM_GET_FIELD(cgs_read_register(device, mm##reg), reg, field)
+
+#define PHM_READ_INDIRECT_FIELD(device, port, reg, field)	\
+	PHM_GET_FIELD(cgs_read_ind_register(device, port, ix##reg),	\
+			reg, field)
+
+#define PHM_READ_VFPF_INDIRECT_FIELD(device, port, reg, field)	\
+	PHM_GET_FIELD(cgs_read_ind_register(device, port, ix##reg),	\
+			reg, field)
+
+#define PHM_WRITE_FIELD(device, reg, field, fieldval)	\
+	cgs_write_register(device, mm##reg, PHM_SET_FIELD(	\
+				cgs_read_register(device, mm##reg), reg, field, fieldval))
+
+#define PHM_WRITE_INDIRECT_FIELD(device, port, reg, field, fieldval)	\
+	cgs_write_ind_register(device, port, ix##reg,	\
+			PHM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg),	\
+				reg, field, fieldval))
+
+#define PHM_WRITE_VFPF_INDIRECT_FIELD(device, port, reg, field, fieldval)	\
+	cgs_write_ind_register(device, port, ix##reg,	\
+			PHM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg),	\
+				reg, field, fieldval))
+
+#define PHM_WAIT_FIELD(hwmgr, reg, field, fieldval)	\
+	PHM_WAIT_REGISTER(hwmgr, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_INDIRECT_FIELD(hwmgr, port, reg, field, fieldval)	\
+	PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, port, reg, field, fieldval)	\
+	PHM_WAIT_VFPF_INDIRECT_REGISTER(hwmgr, port, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_FIELD_UNEQUAL(hwmgr, reg, field, fieldval)	\
+	PHM_WAIT_REGISTER_UNEQUAL(hwmgr, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_INDIRECT_FIELD_UNEQUAL(hwmgr, port, reg, field, fieldval)	\
+	PHM_WAIT_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(hwmgr, port, reg, field, fieldval)	\
+	PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, (fieldval)	\
+			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+
+/* Operations on arrays of registers & fields. */
+
+#define PHM_READ_ARRAY_REGISTER(device, reg, offset)	\
+	cgs_read_register(device, mm##reg + (offset))
+
+#define PHM_WRITE_ARRAY_REGISTER(device, reg, offset, value)	\
+	cgs_write_register(device, mm##reg + (offset), value)
+
+#define PHM_WAIT_ARRAY_REGISTER(hwmgr, reg, offset, value, mask)	\
+	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg + (offset), value, mask)
+
+#define PHM_WAIT_ARRAY_REGISTER_UNEQUAL(hwmgr, reg, offset, value, mask)	\
+	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg + (offset), value, mask)
+
+#define PHM_READ_ARRAY_FIELD(hwmgr, reg, offset, field) \
+	PHM_GET_FIELD(PHM_READ_ARRAY_REGISTER(hwmgr->device, reg, offset), reg, field)
+
+#define PHM_WRITE_ARRAY_FIELD(hwmgr, reg, offset, field, fieldvalue)	\
+	PHM_WRITE_ARRAY_REGISTER(hwmgr->device, reg, offset,	\
+			PHM_SET_FIELD(PHM_READ_ARRAY_REGISTER(hwmgr->device, reg, offset),	\
+				reg, field, fieldvalue))
+
+#define PHM_WAIT_ARRAY_FIELD(hwmgr, reg, offset, field, fieldvalue)	\
+	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg + (offset),	\
+			(fieldvalue) << PHM_FIELD_SHIFT(reg, field),	\
+			PHM_FIELD_MASK(reg, field))
+
+#define PHM_WAIT_ARRAY_FIELD_UNEQUAL(hwmgr, reg, offset, field, fieldvalue)	\
+	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg + (offset),	\
+			(fieldvalue) << PHM_FIELD_SHIFT(reg, field),	\
+			PHM_FIELD_MASK(reg, field))
+
+#endif /* _HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/power_state.h b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
new file mode 100644
index 000000000000..a3f0ce4d5835
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_POWERSTATE_H
+#define PP_POWERSTATE_H
+
+struct pp_hw_power_state {
+	unsigned int magic;
+};
+
+struct pp_power_state;
+
+
+#define PP_INVALID_POWER_STATE_ID (0)
+
+
+/*
+ * An item of a list containing Power States.
+ */
+
+struct PP_StateLinkedList {
+	struct pp_power_state *next;
+	struct pp_power_state *prev;
+};
+
+
+enum PP_StateUILabel {
+	PP_StateUILabel_None,
+	PP_StateUILabel_Battery,
+	PP_StateUILabel_MiddleLow,
+	PP_StateUILabel_Balanced,
+	PP_StateUILabel_MiddleHigh,
+	PP_StateUILabel_Performance,
+	PP_StateUILabel_BACO
+};
+
+enum PP_StateClassificationFlag {
+	PP_StateClassificationFlag_Boot                = 0x0001,
+	PP_StateClassificationFlag_Thermal             = 0x0002,
+	PP_StateClassificationFlag_LimitedPowerSource  = 0x0004,
+	PP_StateClassificationFlag_Rest                = 0x0008,
+	PP_StateClassificationFlag_Forced              = 0x0010,
+	PP_StateClassificationFlag_User3DPerformance   = 0x0020,
+	PP_StateClassificationFlag_User2DPerformance   = 0x0040,
+	PP_StateClassificationFlag_3DPerformance       = 0x0080,
+	PP_StateClassificationFlag_ACOverdriveTemplate   = 0x0100,
+	PP_StateClassificationFlag_Uvd                 = 0x0200,
+	PP_StateClassificationFlag_3DPerformanceLow    = 0x0400,
+	PP_StateClassificationFlag_ACPI                = 0x0800,
+	PP_StateClassificationFlag_HD2                 = 0x1000,
+	PP_StateClassificationFlag_UvdHD               = 0x2000,
+	PP_StateClassificationFlag_UvdSD               = 0x4000,
+	PP_StateClassificationFlag_UserDCPerformance    = 0x8000,
+	PP_StateClassificationFlag_DCOverdriveTemplate   = 0x10000,
+	PP_StateClassificationFlag_BACO                  = 0x20000,
+	PP_StateClassificationFlag_LimitedPowerSource_2  = 0x40000,
+	PP_StateClassificationFlag_ULV                   = 0x80000,
+	PP_StateClassificationFlag_UvdMVC               = 0x100000,
+};
+
+typedef unsigned int PP_StateClassificationFlags;
+
+struct PP_StateClassificationBlock {
+	enum PP_StateUILabel         ui_label;
+	enum PP_StateClassificationFlag  flags;
+	int                          bios_index;
+	bool                      temporary_state;
+	bool                      to_be_deleted;
+};
+
+struct PP_StatePcieBlock {
+	unsigned int lanes;
+};
+
+enum PP_RefreshrateSource {
+	PP_RefreshrateSource_EDID,
+	PP_RefreshrateSource_Explicit
+};
+
+struct PP_StateDisplayBlock {
+	bool              disableFrameModulation;
+	bool              limitRefreshrate;
+	enum PP_RefreshrateSource refreshrateSource;
+	int                  explicitRefreshrate;
+	int                  edidRefreshrateIndex;
+	bool              enableVariBright;
+};
+
+struct PP_StateMemroyBlock {
+	bool              dllOff;
+	uint8_t                 m3arb;
+	uint8_t                 unused[3];
+};
+
+struct PP_StateSoftwareAlgorithmBlock {
+	bool disableLoadBalancing;
+	bool enableSleepForTimestamps;
+};
+
+#define PP_TEMPERATURE_UNITS_PER_CENTIGRADES 1000
+
+/**
+ * Type to hold a temperature range.
+ */
+struct PP_TemperatureRange {
+	uint32_t min;
+	uint32_t max;
+};
+
+struct PP_StateValidationBlock {
+	bool singleDisplayOnly;
+	bool disallowOnDC;
+	uint8_t supportedPowerLevels;
+};
+
+struct PP_UVD_CLOCKS {
+	uint32_t VCLK;
+	uint32_t DCLK;
+};
+
+/**
+* Structure to hold a PowerPlay Power State.
+*/
+struct pp_power_state {
+	uint32_t                            id;
+	struct PP_StateLinkedList                  orderedList;
+	struct PP_StateLinkedList                  allStatesList;
+
+	struct PP_StateClassificationBlock         classification;
+	struct PP_StateValidationBlock             validation;
+	struct PP_StatePcieBlock                   pcie;
+	struct PP_StateDisplayBlock                display;
+	struct PP_StateMemroyBlock                 memory;
+	struct PP_TemperatureRange                 temperatures;
+	struct PP_StateSoftwareAlgorithmBlock      software;
+	struct PP_UVD_CLOCKS                       uvd_clocks;
+	struct pp_hw_power_state  hardware;
+};
+
+
+/*Structure to hold a VCE state entry*/
+struct PP_VCEState {
+	uint32_t evclk;
+	uint32_t ecclk;
+	uint32_t sclk;
+	uint32_t mclk;
+};
+
+enum PP_MMProfilingState {
+	PP_MMProfilingState_NA = 0,
+	PP_MMProfilingState_Started,
+	PP_MMProfilingState_Stopped
+};
+
+struct PP_Clock_Engine_Request {
+	unsigned long clientType;
+	unsigned long ctxid;
+	uint64_t  context_handle;
+	unsigned long sclk;
+	unsigned long sclkHardMin;
+	unsigned long mclk;
+	unsigned long iclk;
+	unsigned long evclk;
+	unsigned long ecclk;
+	unsigned long ecclkHardMin;
+	unsigned long vclk;
+	unsigned long dclk;
+	unsigned long samclk;
+	unsigned long acpclk;
+	unsigned long sclkOverdrive;
+	unsigned long mclkOverdrive;
+	unsigned long sclk_threshold;
+	unsigned long flag;
+	unsigned long vclk_ceiling;
+	unsigned long dclk_ceiling;
+	unsigned long num_cus;
+	unsigned long pmflag;
+	enum PP_MMProfilingState MMProfilingState;
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h
new file mode 100644
index 000000000000..3bd5e69b9045
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+extern bool acpi_atcs_functions_supported(void *device,
+							uint32_t index);
+extern int acpi_pcie_perf_request(void *device,
+						uint8_t perf_req,
+						bool advertise);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_asicblocks.h b/drivers/gpu/drm/amd/powerplay/inc/pp_asicblocks.h
new file mode 100644
index 000000000000..0c1593e53654
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_asicblocks.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_ASICBLOCKS_H
+#define PP_ASICBLOCKS_H
+
+
+enum PHM_AsicBlock {
+	PHM_AsicBlock_GFX,
+	PHM_AsicBlock_UVD_MVC,
+	PHM_AsicBlock_UVD,
+	PHM_AsicBlock_UVD_HD,
+	PHM_AsicBlock_UVD_SD,
+	PHM_AsicBlock_Count
+};
+
+enum PHM_ClockGateSetting {
+	PHM_ClockGateSetting_StaticOn,
+	PHM_ClockGateSetting_StaticOff,
+	PHM_ClockGateSetting_Dynamic
+};
+
+struct phm_asic_blocks {
+	bool gfx : 1;
+	bool uvd : 1;
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
new file mode 100644
index 000000000000..d7d83b7c7f95
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
@@ -0,0 +1,47 @@
+
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_DEBUG_H
+#define PP_DEBUG_H
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+
+#define PP_ASSERT_WITH_CODE(cond, msg, code)	\
+	do {					\
+		if (!(cond)) {			\
+			printk("%s\n", msg);	\
+			code;			\
+		}				\
+	} while (0)
+
+
+#define PP_DBG_LOG(fmt, ...) \
+	do { \
+		if(0)printk(KERN_INFO "[ pp_dbg ] " fmt, ##__VA_ARGS__); \
+	} while (0)
+
+
+#endif /* PP_DEBUG_H */
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_feature.h b/drivers/gpu/drm/amd/powerplay/inc/pp_feature.h
new file mode 100644
index 000000000000..0faf6a25c18b
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_feature.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _PP_FEATURE_H_
+#define _PP_FEATURE_H_
+
+/**
+ * PowerPlay feature ids.
+ */
+enum pp_feature {
+	PP_Feature_PowerPlay = 0,
+	PP_Feature_User2DPerformance,
+	PP_Feature_User3DPerformance,
+	PP_Feature_VariBright,
+	PP_Feature_VariBrightOnPowerXpress,
+	PP_Feature_ReducedRefreshRate,
+	PP_Feature_GFXClockGating,
+	PP_Feature_OverdriveTest,
+	PP_Feature_OverDrive,
+	PP_Feature_PowerBudgetWaiver,
+	PP_Feature_PowerControl,
+	PP_Feature_PowerControl_2,
+	PP_Feature_MultiUVDState,
+	PP_Feature_Force3DClock,
+	PP_Feature_BACO,
+	PP_Feature_PowerDown,
+	PP_Feature_DynamicUVDState,
+	PP_Feature_VCEDPM,
+	PP_Feature_PPM,
+	PP_Feature_ACP_POWERGATING,
+	PP_Feature_FFC,
+	PP_Feature_FPS,
+	PP_Feature_ViPG,
+	PP_Feature_Max
+};
+
+/**
+ * Struct for PowerPlay feature info.
+ */
+struct pp_feature_info {
+	bool supported;               /* feature supported by PowerPlay */
+	bool enabled;                 /* feature enabled in PowerPlay */
+	bool enabled_default;        /* default enable status of the feature */
+	uint32_t version;             /* feature version */
+};
+
+#endif /* _PP_FEATURE_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h b/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h
new file mode 100644
index 000000000000..4d8ed1f33de4
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _PP_INSTANCE_H_
+#define _PP_INSTANCE_H_
+
+#include "smumgr.h"
+#include "hwmgr.h"
+#include "eventmgr.h"
+
+#define PP_VALID  0x1F1F1F1F
+
+struct pp_instance {
+	uint32_t pp_valid;
+	struct pp_smumgr *smu_mgr;
+	struct pp_hwmgr *hwmgr;
+	struct pp_eventmgr *eventmgr;
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_power_source.h b/drivers/gpu/drm/amd/powerplay/inc/pp_power_source.h
new file mode 100644
index 000000000000..b43315cc5d58
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_power_source.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef PP_POWERSOURCE_H
+#define PP_POWERSOURCE_H
+
+enum pp_power_source {
+	PP_PowerSource_AC = 0,
+	PP_PowerSource_DC,
+	PP_PowerSource_LimitedPower,
+	PP_PowerSource_LimitedPower_2,
+	PP_PowerSource_Max
+};
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/ppinterrupt.h b/drivers/gpu/drm/amd/powerplay/inc/ppinterrupt.h
new file mode 100644
index 000000000000..c067e0925b6b
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/ppinterrupt.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _PP_INTERRUPT_H_
+#define _PP_INTERRUPT_H_
+
+enum amd_thermal_irq {
+	AMD_THERMAL_IRQ_LOW_TO_HIGH = 0,
+	AMD_THERMAL_IRQ_HIGH_TO_LOW,
+
+	AMD_THERMAL_IRQ_LAST
+};
+
+/* The type of the interrupt callback functions in PowerPlay */
+typedef int (*irq_handler_func_t)(void *private_data,
+				unsigned src_id, const uint32_t *iv_entry);
+
+/* Event Manager action chain list information */
+struct pp_interrupt_registration_info {
+	irq_handler_func_t call_back; /* Pointer to callback function */
+	void *context;                   /* Pointer to callback function context */
+	uint32_t src_id;               /* Registered interrupt id */
+	const uint32_t *iv_entry;
+};
+
+#endif /* _PP_INTERRUPT_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/smu7.h b/drivers/gpu/drm/amd/powerplay/inc/smu7.h
index 75a380a15292..75a380a15292 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu7.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7.h
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu72.h b/drivers/gpu/drm/amd/powerplay/inc/smu72.h
new file mode 100644
index 000000000000..b73d6b59ac32
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu72.h
@@ -0,0 +1,664 @@
+#ifndef SMU72_H
+#define SMU72_H
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(push, 1)
+#endif
+
+#define SMU__NUM_SCLK_DPM_STATE  8
+#define SMU__NUM_MCLK_DPM_LEVELS 4
+#define SMU__NUM_LCLK_DPM_LEVELS 8
+#define SMU__NUM_PCIE_DPM_LEVELS 8
+
+enum SID_OPTION {
+	SID_OPTION_HI,
+	SID_OPTION_LO,
+	SID_OPTION_COUNT
+};
+
+enum Poly3rdOrderCoeff {
+	LEAKAGE_TEMPERATURE_SCALAR,
+	LEAKAGE_VOLTAGE_SCALAR,
+	DYNAMIC_VOLTAGE_SCALAR,
+	POLY_3RD_ORDER_COUNT
+};
+
+struct SMU7_Poly3rdOrder_Data {
+	int32_t a;
+	int32_t b;
+	int32_t c;
+	int32_t d;
+	uint8_t a_shift;
+	uint8_t b_shift;
+	uint8_t c_shift;
+	uint8_t x_shift;
+};
+
+typedef struct SMU7_Poly3rdOrder_Data SMU7_Poly3rdOrder_Data;
+
+struct Power_Calculator_Data {
+	uint16_t NoLoadVoltage;
+	uint16_t LoadVoltage;
+	uint16_t Resistance;
+	uint16_t Temperature;
+	uint16_t BaseLeakage;
+	uint16_t LkgTempScalar;
+	uint16_t LkgVoltScalar;
+	uint16_t LkgAreaScalar;
+	uint16_t LkgPower;
+	uint16_t DynVoltScalar;
+	uint32_t Cac;
+	uint32_t DynPower;
+	uint32_t TotalCurrent;
+	uint32_t TotalPower;
+};
+
+typedef struct Power_Calculator_Data PowerCalculatorData_t;
+
+struct Gc_Cac_Weight_Data {
+	uint8_t index;
+	uint32_t value;
+};
+
+typedef struct Gc_Cac_Weight_Data GcCacWeight_Data;
+
+
+typedef struct {
+	uint32_t high;
+	uint32_t low;
+} data_64_t;
+
+typedef struct {
+	data_64_t high;
+	data_64_t low;
+} data_128_t;
+
+#define SMU7_CONTEXT_ID_SMC        1
+#define SMU7_CONTEXT_ID_VBIOS      2
+
+#define SMU72_MAX_LEVELS_VDDC            16
+#define SMU72_MAX_LEVELS_VDDGFX          16
+#define SMU72_MAX_LEVELS_VDDCI           8
+#define SMU72_MAX_LEVELS_MVDD            4
+
+#define SMU_MAX_SMIO_LEVELS              4
+
+#define SMU72_MAX_LEVELS_GRAPHICS        SMU__NUM_SCLK_DPM_STATE   /* SCLK + SQ DPM + ULV */
+#define SMU72_MAX_LEVELS_MEMORY          SMU__NUM_MCLK_DPM_LEVELS   /* MCLK Levels DPM */
+#define SMU72_MAX_LEVELS_GIO             SMU__NUM_LCLK_DPM_LEVELS  /* LCLK Levels */
+#define SMU72_MAX_LEVELS_LINK            SMU__NUM_PCIE_DPM_LEVELS  /* PCIe speed and number of lanes. */
+#define SMU72_MAX_LEVELS_UVD             8   /* VCLK/DCLK levels for UVD. */
+#define SMU72_MAX_LEVELS_VCE             8   /* ECLK levels for VCE. */
+#define SMU72_MAX_LEVELS_ACP             8   /* ACLK levels for ACP. */
+#define SMU72_MAX_LEVELS_SAMU            8   /* SAMCLK levels for SAMU. */
+#define SMU72_MAX_ENTRIES_SMIO           32  /* Number of entries in SMIO table. */
+
+#define DPM_NO_LIMIT 0
+#define DPM_NO_UP 1
+#define DPM_GO_DOWN 2
+#define DPM_GO_UP 3
+
+#define SMU7_FIRST_DPM_GRAPHICS_LEVEL    0
+#define SMU7_FIRST_DPM_MEMORY_LEVEL      0
+
+#define GPIO_CLAMP_MODE_VRHOT      1
+#define GPIO_CLAMP_MODE_THERM      2
+#define GPIO_CLAMP_MODE_DC         4
+
+#define SCRATCH_B_TARG_PCIE_INDEX_SHIFT 0
+#define SCRATCH_B_TARG_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_TARG_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_PCIE_INDEX_SHIFT 3
+#define SCRATCH_B_CURR_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_CURR_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_UVD_INDEX_SHIFT  6
+#define SCRATCH_B_TARG_UVD_INDEX_MASK   (0x7<<SCRATCH_B_TARG_UVD_INDEX_SHIFT)
+#define SCRATCH_B_CURR_UVD_INDEX_SHIFT  9
+#define SCRATCH_B_CURR_UVD_INDEX_MASK   (0x7<<SCRATCH_B_CURR_UVD_INDEX_SHIFT)
+#define SCRATCH_B_TARG_VCE_INDEX_SHIFT  12
+#define SCRATCH_B_TARG_VCE_INDEX_MASK   (0x7<<SCRATCH_B_TARG_VCE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_VCE_INDEX_SHIFT  15
+#define SCRATCH_B_CURR_VCE_INDEX_MASK   (0x7<<SCRATCH_B_CURR_VCE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_ACP_INDEX_SHIFT  18
+#define SCRATCH_B_TARG_ACP_INDEX_MASK   (0x7<<SCRATCH_B_TARG_ACP_INDEX_SHIFT)
+#define SCRATCH_B_CURR_ACP_INDEX_SHIFT  21
+#define SCRATCH_B_CURR_ACP_INDEX_MASK   (0x7<<SCRATCH_B_CURR_ACP_INDEX_SHIFT)
+#define SCRATCH_B_TARG_SAMU_INDEX_SHIFT 24
+#define SCRATCH_B_TARG_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_TARG_SAMU_INDEX_SHIFT)
+#define SCRATCH_B_CURR_SAMU_INDEX_SHIFT 27
+#define SCRATCH_B_CURR_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_CURR_SAMU_INDEX_SHIFT)
+
+/* Virtualization Defines */
+#define CG_XDMA_MASK  0x1
+#define CG_XDMA_SHIFT 0
+#define CG_UVD_MASK   0x2
+#define CG_UVD_SHIFT  1
+#define CG_VCE_MASK   0x4
+#define CG_VCE_SHIFT  2
+#define CG_SAMU_MASK  0x8
+#define CG_SAMU_SHIFT 3
+#define CG_GFX_MASK   0x10
+#define CG_GFX_SHIFT  4
+#define CG_SDMA_MASK  0x20
+#define CG_SDMA_SHIFT 5
+#define CG_HDP_MASK   0x40
+#define CG_HDP_SHIFT  6
+#define CG_MC_MASK    0x80
+#define CG_MC_SHIFT   7
+#define CG_DRM_MASK   0x100
+#define CG_DRM_SHIFT  8
+#define CG_ROM_MASK   0x200
+#define CG_ROM_SHIFT  9
+#define CG_BIF_MASK   0x400
+#define CG_BIF_SHIFT  10
+
+#define SMU72_DTE_ITERATIONS 5
+#define SMU72_DTE_SOURCES 3
+#define SMU72_DTE_SINKS 1
+#define SMU72_NUM_CPU_TES 0
+#define SMU72_NUM_GPU_TES 1
+#define SMU72_NUM_NON_TES 2
+#define SMU72_DTE_FAN_SCALAR_MIN 0x100
+#define SMU72_DTE_FAN_SCALAR_MAX 0x166
+#define SMU72_DTE_FAN_TEMP_MAX 93
+#define SMU72_DTE_FAN_TEMP_MIN 83
+
+#if defined SMU__FUSION_ONLY
+#define SMU7_DTE_ITERATIONS 5
+#define SMU7_DTE_SOURCES 5
+#define SMU7_DTE_SINKS 3
+#define SMU7_NUM_CPU_TES 2
+#define SMU7_NUM_GPU_TES 1
+#define SMU7_NUM_NON_TES 2
+#endif
+
+struct SMU7_HystController_Data {
+	uint8_t waterfall_up;
+	uint8_t waterfall_down;
+	uint8_t waterfall_limit;
+	uint8_t spare;
+	uint16_t release_cnt;
+	uint16_t release_limit;
+};
+
+typedef struct SMU7_HystController_Data SMU7_HystController_Data;
+
+struct SMU72_PIDController {
+	uint32_t Ki;
+	int32_t LFWindupUpperLim;
+	int32_t LFWindupLowerLim;
+	uint32_t StatePrecision;
+	uint32_t LfPrecision;
+	uint32_t LfOffset;
+	uint32_t MaxState;
+	uint32_t MaxLfFraction;
+	uint32_t StateShift;
+};
+
+typedef struct SMU72_PIDController SMU72_PIDController;
+
+struct SMU7_LocalDpmScoreboard {
+	uint32_t PercentageBusy;
+
+	int32_t  PIDError;
+	int32_t  PIDIntegral;
+	int32_t  PIDOutput;
+
+	uint32_t SigmaDeltaAccum;
+	uint32_t SigmaDeltaOutput;
+	uint32_t SigmaDeltaLevel;
+
+	uint32_t UtilizationSetpoint;
+
+	uint8_t  TdpClampMode;
+	uint8_t  TdcClampMode;
+	uint8_t  ThermClampMode;
+	uint8_t  VoltageBusy;
+
+	int8_t   CurrLevel;
+	int8_t   TargLevel;
+	uint8_t  LevelChangeInProgress;
+	uint8_t  UpHyst;
+
+	uint8_t  DownHyst;
+	uint8_t  VoltageDownHyst;
+	uint8_t  DpmEnable;
+	uint8_t  DpmRunning;
+
+	uint8_t  DpmForce;
+	uint8_t  DpmForceLevel;
+	uint8_t  DisplayWatermark;
+	uint8_t  McArbIndex;
+
+	uint32_t MinimumPerfSclk;
+
+	uint8_t  AcpiReq;
+	uint8_t  AcpiAck;
+	uint8_t  GfxClkSlow;
+	uint8_t  GpioClampMode; /* bit0 = VRHOT: bit1 = THERM: bit2 = DC */
+
+	uint8_t  FpsFilterWeight;
+	uint8_t  EnabledLevelsChange;
+	uint8_t  DteClampMode;
+	uint8_t  FpsClampMode;
+
+	uint16_t LevelResidencyCounters[SMU72_MAX_LEVELS_GRAPHICS];
+	uint16_t LevelSwitchCounters[SMU72_MAX_LEVELS_GRAPHICS];
+
+	void     (*TargetStateCalculator)(uint8_t);
+	void     (*SavedTargetStateCalculator)(uint8_t);
+
+	uint16_t AutoDpmInterval;
+	uint16_t AutoDpmRange;
+
+	uint8_t  FpsEnabled;
+	uint8_t  MaxPerfLevel;
+	uint8_t  AllowLowClkInterruptToHost;
+	uint8_t  FpsRunning;
+
+	uint32_t MaxAllowedFrequency;
+
+	uint32_t FilteredSclkFrequency;
+	uint32_t LastSclkFrequency;
+	uint32_t FilteredSclkFrequencyCnt;
+};
+
+typedef struct SMU7_LocalDpmScoreboard SMU7_LocalDpmScoreboard;
+
+#define SMU7_MAX_VOLTAGE_CLIENTS 12
+
+typedef uint8_t (*VoltageChangeHandler_t)(uint16_t, uint8_t);
+
+struct SMU_VoltageLevel {
+	uint8_t Vddc;
+	uint8_t Vddci;
+	uint8_t VddGfx;
+	uint8_t Phases;
+};
+
+typedef struct SMU_VoltageLevel SMU_VoltageLevel;
+
+struct SMU7_VoltageScoreboard {
+	SMU_VoltageLevel CurrentVoltage;
+	SMU_VoltageLevel TargetVoltage;
+	uint16_t MaxVid;
+	uint8_t  HighestVidOffset;
+	uint8_t  CurrentVidOffset;
+
+	uint8_t  ControllerBusy;
+	uint8_t  CurrentVid;
+	uint8_t  CurrentVddciVid;
+	uint8_t  VddGfxShutdown; /* 0 = normal mode, 1 = shut down */
+
+	SMU_VoltageLevel RequestedVoltage[SMU7_MAX_VOLTAGE_CLIENTS];
+	uint8_t  EnabledRequest[SMU7_MAX_VOLTAGE_CLIENTS];
+
+	uint8_t  TargetIndex;
+	uint8_t  Delay;
+	uint8_t  ControllerEnable;
+	uint8_t  ControllerRunning;
+	uint16_t CurrentStdVoltageHiSidd;
+	uint16_t CurrentStdVoltageLoSidd;
+	uint8_t  OverrideVoltage;
+	uint8_t  VddcUseUlvOffset;
+	uint8_t  VddGfxUseUlvOffset;
+	uint8_t  padding;
+
+	VoltageChangeHandler_t ChangeVddc;
+	VoltageChangeHandler_t ChangeVddGfx;
+	VoltageChangeHandler_t ChangeVddci;
+	VoltageChangeHandler_t ChangePhase;
+	VoltageChangeHandler_t ChangeMvdd;
+
+	VoltageChangeHandler_t functionLinks[6];
+
+	uint8_t *VddcFollower1;
+	uint8_t *VddcFollower2;
+	int16_t  Driver_OD_RequestedVidOffset1;
+	int16_t  Driver_OD_RequestedVidOffset2;
+
+};
+
+typedef struct SMU7_VoltageScoreboard SMU7_VoltageScoreboard;
+
+#define SMU7_MAX_PCIE_LINK_SPEEDS 3 /* 0:Gen1 1:Gen2 2:Gen3 */
+
+struct SMU7_PCIeLinkSpeedScoreboard {
+	uint8_t     DpmEnable;
+	uint8_t     DpmRunning;
+	uint8_t     DpmForce;
+	uint8_t     DpmForceLevel;
+
+	uint8_t     CurrentLinkSpeed;
+	uint8_t     EnabledLevelsChange;
+	uint16_t    AutoDpmInterval;
+
+	uint16_t    AutoDpmRange;
+	uint16_t    AutoDpmCount;
+
+	uint8_t     DpmMode;
+	uint8_t     AcpiReq;
+	uint8_t     AcpiAck;
+	uint8_t     CurrentLinkLevel;
+
+};
+
+typedef struct SMU7_PCIeLinkSpeedScoreboard SMU7_PCIeLinkSpeedScoreboard;
+
+/* -------------------------------------------------------- CAC table ------------------------------------------------------ */
+#define SMU7_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMU7_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16
+#define SMU7_SCALE_I  7
+#define SMU7_SCALE_R 12
+
+struct SMU7_PowerScoreboard {
+	PowerCalculatorData_t VddGfxPowerData[SID_OPTION_COUNT];
+	PowerCalculatorData_t VddcPowerData[SID_OPTION_COUNT];
+
+	uint32_t TotalGpuPower;
+	uint32_t TdcCurrent;
+
+	uint16_t   VddciTotalPower;
+	uint16_t   sparesasfsdfd;
+	uint16_t   Vddr1Power;
+	uint16_t   RocPower;
+
+	uint16_t   CalcMeasPowerBlend;
+	uint8_t    SidOptionPower;
+	uint8_t    SidOptionCurrent;
+
+	uint32_t   WinTime;
+
+	uint16_t Telemetry_1_slope;
+	uint16_t Telemetry_2_slope;
+	int32_t Telemetry_1_offset;
+	int32_t Telemetry_2_offset;
+
+	uint32_t VddcCurrentTelemetry;
+	uint32_t VddGfxCurrentTelemetry;
+	uint32_t VddcPowerTelemetry;
+	uint32_t VddGfxPowerTelemetry;
+	uint32_t VddciPowerTelemetry;
+
+	uint32_t VddcPower;
+	uint32_t VddGfxPower;
+	uint32_t VddciPower;
+
+	uint32_t TelemetryCurrent[2];
+	uint32_t TelemetryVoltage[2];
+	uint32_t TelemetryPower[2];
+};
+
+typedef struct SMU7_PowerScoreboard SMU7_PowerScoreboard;
+
+struct SMU7_ThermalScoreboard {
+	int16_t  GpuLimit;
+	int16_t  GpuHyst;
+	uint16_t CurrGnbTemp;
+	uint16_t FilteredGnbTemp;
+
+	uint8_t  ControllerEnable;
+	uint8_t  ControllerRunning;
+	uint8_t  AutoTmonCalInterval;
+	uint8_t  AutoTmonCalEnable;
+
+	uint8_t  ThermalDpmEnabled;
+	uint8_t  SclkEnabledMask;
+	uint8_t  spare[2];
+	int32_t  temperature_gradient;
+
+	SMU7_HystController_Data HystControllerData;
+	int32_t  WeightedSensorTemperature;
+	uint16_t TemperatureLimit[SMU72_MAX_LEVELS_GRAPHICS];
+	uint32_t Alpha;
+};
+
+typedef struct SMU7_ThermalScoreboard SMU7_ThermalScoreboard;
+
+/* For FeatureEnables: */
+#define SMU7_SCLK_DPM_CONFIG_MASK                        0x01
+#define SMU7_VOLTAGE_CONTROLLER_CONFIG_MASK              0x02
+#define SMU7_THERMAL_CONTROLLER_CONFIG_MASK              0x04
+#define SMU7_MCLK_DPM_CONFIG_MASK                        0x08
+#define SMU7_UVD_DPM_CONFIG_MASK                         0x10
+#define SMU7_VCE_DPM_CONFIG_MASK                         0x20
+#define SMU7_ACP_DPM_CONFIG_MASK                         0x40
+#define SMU7_SAMU_DPM_CONFIG_MASK                        0x80
+#define SMU7_PCIEGEN_DPM_CONFIG_MASK                    0x100
+
+#define SMU7_ACP_MCLK_HANDSHAKE_DISABLE                  0x00000001
+#define SMU7_ACP_SCLK_HANDSHAKE_DISABLE                  0x00000002
+#define SMU7_UVD_MCLK_HANDSHAKE_DISABLE                  0x00000100
+#define SMU7_UVD_SCLK_HANDSHAKE_DISABLE                  0x00000200
+#define SMU7_VCE_MCLK_HANDSHAKE_DISABLE                  0x00010000
+#define SMU7_VCE_SCLK_HANDSHAKE_DISABLE                  0x00020000
+
+/* All 'soft registers' should be uint32_t. */
+struct SMU72_SoftRegisters {
+	uint32_t        RefClockFrequency;
+	uint32_t        PmTimerPeriod;
+	uint32_t        FeatureEnables;
+
+	uint32_t        PreVBlankGap;
+	uint32_t        VBlankTimeout;
+	uint32_t        TrainTimeGap;
+
+	uint32_t        MvddSwitchTime;
+	uint32_t        LongestAcpiTrainTime;
+	uint32_t        AcpiDelay;
+	uint32_t        G5TrainTime;
+	uint32_t        DelayMpllPwron;
+	uint32_t        VoltageChangeTimeout;
+
+	uint32_t        HandshakeDisables;
+
+	uint8_t         DisplayPhy1Config;
+	uint8_t         DisplayPhy2Config;
+	uint8_t         DisplayPhy3Config;
+	uint8_t         DisplayPhy4Config;
+
+	uint8_t         DisplayPhy5Config;
+	uint8_t         DisplayPhy6Config;
+	uint8_t         DisplayPhy7Config;
+	uint8_t         DisplayPhy8Config;
+
+	uint32_t        AverageGraphicsActivity;
+	uint32_t        AverageMemoryActivity;
+	uint32_t        AverageGioActivity;
+
+	uint8_t         SClkDpmEnabledLevels;
+	uint8_t         MClkDpmEnabledLevels;
+	uint8_t         LClkDpmEnabledLevels;
+	uint8_t         PCIeDpmEnabledLevels;
+
+	uint8_t         UVDDpmEnabledLevels;
+	uint8_t         SAMUDpmEnabledLevels;
+	uint8_t         ACPDpmEnabledLevels;
+	uint8_t         VCEDpmEnabledLevels;
+
+	uint32_t        DRAM_LOG_ADDR_H;
+	uint32_t        DRAM_LOG_ADDR_L;
+	uint32_t        DRAM_LOG_PHY_ADDR_H;
+	uint32_t        DRAM_LOG_PHY_ADDR_L;
+	uint32_t        DRAM_LOG_BUFF_SIZE;
+	uint32_t        UlvEnterCount;
+	uint32_t        UlvTime;
+	uint32_t        UcodeLoadStatus;
+	uint32_t        Reserved[2];
+
+};
+
+typedef struct SMU72_SoftRegisters SMU72_SoftRegisters;
+
+struct SMU72_Firmware_Header {
+	uint32_t Digest[5];
+	uint32_t Version;
+	uint32_t HeaderSize;
+	uint32_t Flags;
+	uint32_t EntryPoint;
+	uint32_t CodeSize;
+	uint32_t ImageSize;
+
+	uint32_t Rtos;
+	uint32_t SoftRegisters;
+	uint32_t DpmTable;
+	uint32_t FanTable;
+	uint32_t CacConfigTable;
+	uint32_t CacStatusTable;
+	uint32_t mcRegisterTable;
+	uint32_t mcArbDramTimingTable;
+	uint32_t PmFuseTable;
+	uint32_t Globals;
+	uint32_t ClockStretcherTable;
+	uint32_t Reserved[41];
+	uint32_t Signature;
+};
+
+typedef struct SMU72_Firmware_Header SMU72_Firmware_Header;
+
+#define SMU72_FIRMWARE_HEADER_LOCATION 0x20000
+
+enum  DisplayConfig {
+	PowerDown = 1,
+	DP54x4,
+	DP54x2,
+	DP54x1,
+	DP27x4,
+	DP27x2,
+	DP27x1,
+	HDMI297,
+	HDMI162,
+	LVDS,
+	DP324x4,
+	DP324x2,
+	DP324x1
+};
+
+#define MC_BLOCK_COUNT 1
+#define CPL_BLOCK_COUNT 5
+#define SE_BLOCK_COUNT 15
+#define GC_BLOCK_COUNT 24
+
+struct SMU7_Local_Cac {
+	uint8_t BlockId;
+	uint8_t SignalId;
+	uint8_t Threshold;
+	uint8_t Padding;
+};
+
+typedef struct SMU7_Local_Cac SMU7_Local_Cac;
+
+struct SMU7_Local_Cac_Table {
+	SMU7_Local_Cac CplLocalCac[CPL_BLOCK_COUNT];
+	SMU7_Local_Cac McLocalCac[MC_BLOCK_COUNT];
+	SMU7_Local_Cac SeLocalCac[SE_BLOCK_COUNT];
+	SMU7_Local_Cac GcLocalCac[GC_BLOCK_COUNT];
+};
+
+typedef struct SMU7_Local_Cac_Table SMU7_Local_Cac_Table;
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(pop)
+#endif
+
+/* Description of Clock Gating bitmask for Tonga: */
+/* System Clock Gating */
+#define CG_SYS_BITMASK_FIRST_BIT      0  /* First bit of Sys CG bitmask */
+#define CG_SYS_BITMASK_LAST_BIT       9  /* Last bit of Sys CG bitmask */
+#define CG_SYS_BIF_MGLS_SHIFT         0
+#define CG_SYS_ROM_SHIFT              1
+#define CG_SYS_MC_MGCG_SHIFT          2
+#define CG_SYS_MC_MGLS_SHIFT          3
+#define CG_SYS_SDMA_MGCG_SHIFT        4
+#define CG_SYS_SDMA_MGLS_SHIFT        5
+#define CG_SYS_DRM_MGCG_SHIFT         6
+#define CG_SYS_HDP_MGCG_SHIFT         7
+#define CG_SYS_HDP_MGLS_SHIFT         8
+#define CG_SYS_DRM_MGLS_SHIFT         9
+
+#define CG_SYS_BIF_MGLS_MASK          0x1
+#define CG_SYS_ROM_MASK               0x2
+#define CG_SYS_MC_MGCG_MASK           0x4
+#define CG_SYS_MC_MGLS_MASK           0x8
+#define CG_SYS_SDMA_MGCG_MASK         0x10
+#define CG_SYS_SDMA_MGLS_MASK         0x20
+#define CG_SYS_DRM_MGCG_MASK          0x40
+#define CG_SYS_HDP_MGCG_MASK          0x80
+#define CG_SYS_HDP_MGLS_MASK          0x100
+#define CG_SYS_DRM_MGLS_MASK          0x200
+
+/* Graphics Clock Gating */
+#define CG_GFX_BITMASK_FIRST_BIT      16 /* First bit of Gfx CG bitmask */
+#define CG_GFX_BITMASK_LAST_BIT       20 /* Last bit of Gfx CG bitmask */
+#define CG_GFX_CGCG_SHIFT             16
+#define CG_GFX_CGLS_SHIFT             17
+#define CG_CPF_MGCG_SHIFT             18
+#define CG_RLC_MGCG_SHIFT             19
+#define CG_GFX_OTHERS_MGCG_SHIFT      20
+
+#define CG_GFX_CGCG_MASK              0x00010000
+#define CG_GFX_CGLS_MASK              0x00020000
+#define CG_CPF_MGCG_MASK              0x00040000
+#define CG_RLC_MGCG_MASK              0x00080000
+#define CG_GFX_OTHERS_MGCG_MASK       0x00100000
+
+/* Voltage Regulator Configuration */
+/* VR Config info is contained in dpmTable.VRConfig */
+
+#define VRCONF_VDDC_MASK         0x000000FF
+#define VRCONF_VDDC_SHIFT        0
+#define VRCONF_VDDGFX_MASK       0x0000FF00
+#define VRCONF_VDDGFX_SHIFT      8
+#define VRCONF_VDDCI_MASK        0x00FF0000
+#define VRCONF_VDDCI_SHIFT       16
+#define VRCONF_MVDD_MASK         0xFF000000
+#define VRCONF_MVDD_SHIFT        24
+
+#define VR_MERGED_WITH_VDDC      0
+#define VR_SVI2_PLANE_1          1
+#define VR_SVI2_PLANE_2          2
+#define VR_SMIO_PATTERN_1        3
+#define VR_SMIO_PATTERN_2        4
+#define VR_STATIC_VOLTAGE        5
+
+/* Clock Stretcher Configuration */
+
+#define CLOCK_STRETCHER_MAX_ENTRIES 0x4
+#define CKS_LOOKUPTable_MAX_ENTRIES 0x4
+
+/* The 'settings' field is subdivided in the following way: */
+#define CLOCK_STRETCHER_SETTING_DDT_MASK             0x01
+#define CLOCK_STRETCHER_SETTING_DDT_SHIFT            0x0
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_MASK  0x1E
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_SHIFT 0x1
+#define CLOCK_STRETCHER_SETTING_ENABLE_MASK          0x80
+#define CLOCK_STRETCHER_SETTING_ENABLE_SHIFT         0x7
+
+struct SMU_ClockStretcherDataTableEntry {
+	uint8_t minVID;
+	uint8_t maxVID;
+
+	uint16_t setting;
+};
+typedef struct SMU_ClockStretcherDataTableEntry SMU_ClockStretcherDataTableEntry;
+
+struct SMU_ClockStretcherDataTable {
+	SMU_ClockStretcherDataTableEntry ClockStretcherDataTableEntry[CLOCK_STRETCHER_MAX_ENTRIES];
+};
+typedef struct SMU_ClockStretcherDataTable SMU_ClockStretcherDataTable;
+
+struct SMU_CKS_LOOKUPTableEntry {
+	uint16_t minFreq;
+	uint16_t maxFreq;
+
+	uint8_t setting;
+	uint8_t padding[3];
+};
+typedef struct SMU_CKS_LOOKUPTableEntry SMU_CKS_LOOKUPTableEntry;
+
+struct SMU_CKS_LOOKUPTable {
+	SMU_CKS_LOOKUPTableEntry CKS_LOOKUPTableEntry[CKS_LOOKUPTable_MAX_ENTRIES];
+};
+typedef struct SMU_CKS_LOOKUPTable SMU_CKS_LOOKUPTable;
+
+#endif
+
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu72_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu72_discrete.h
new file mode 100644
index 000000000000..98f76e925e65
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu72_discrete.h
@@ -0,0 +1,760 @@
+#ifndef SMU72_DISCRETE_H
+#define SMU72_DISCRETE_H
+
+#include "smu72.h"
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(push, 1)
+#endif
+
+struct SMIO_Pattern {
+	uint16_t Voltage;
+	uint8_t  Smio;
+	uint8_t  padding;
+};
+
+typedef struct SMIO_Pattern SMIO_Pattern;
+
+struct SMIO_Table {
+	SMIO_Pattern Pattern[SMU_MAX_SMIO_LEVELS];
+};
+
+typedef struct SMIO_Table SMIO_Table;
+
+struct SMU72_Discrete_GraphicsLevel {
+	SMU_VoltageLevel MinVoltage;
+
+	uint32_t    SclkFrequency;
+
+	uint8_t     pcieDpmLevel;
+	uint8_t     DeepSleepDivId;
+	uint16_t    ActivityLevel;
+
+	uint32_t    CgSpllFuncCntl3;
+	uint32_t    CgSpllFuncCntl4;
+	uint32_t    SpllSpreadSpectrum;
+	uint32_t    SpllSpreadSpectrum2;
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+	uint8_t     SclkDid;
+	uint8_t     DisplayWatermark;
+	uint8_t     EnabledForActivity;
+	uint8_t     EnabledForThrottle;
+	uint8_t     UpHyst;
+	uint8_t     DownHyst;
+	uint8_t     VoltageDownHyst;
+	uint8_t     PowerThrottle;
+};
+
+typedef struct SMU72_Discrete_GraphicsLevel SMU72_Discrete_GraphicsLevel;
+
+struct SMU72_Discrete_ACPILevel {
+	uint32_t    Flags;
+	SMU_VoltageLevel MinVoltage;
+	uint32_t    SclkFrequency;
+	uint8_t     SclkDid;
+	uint8_t     DisplayWatermark;
+	uint8_t     DeepSleepDivId;
+	uint8_t     padding;
+	uint32_t    CgSpllFuncCntl;
+	uint32_t    CgSpllFuncCntl2;
+	uint32_t    CgSpllFuncCntl3;
+	uint32_t    CgSpllFuncCntl4;
+	uint32_t    SpllSpreadSpectrum;
+	uint32_t    SpllSpreadSpectrum2;
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+};
+
+typedef struct SMU72_Discrete_ACPILevel SMU72_Discrete_ACPILevel;
+
+struct SMU72_Discrete_Ulv {
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+	uint16_t    VddcOffset;
+	uint8_t     VddcOffsetVid;
+	uint8_t     VddcPhase;
+	uint32_t    Reserved;
+};
+
+typedef struct SMU72_Discrete_Ulv SMU72_Discrete_Ulv;
+
+struct SMU72_Discrete_MemoryLevel {
+	SMU_VoltageLevel MinVoltage;
+	uint32_t    MinMvdd;
+
+	uint32_t    MclkFrequency;
+
+	uint8_t     EdcReadEnable;
+	uint8_t     EdcWriteEnable;
+	uint8_t     RttEnable;
+	uint8_t     StutterEnable;
+
+	uint8_t     StrobeEnable;
+	uint8_t     StrobeRatio;
+	uint8_t     EnabledForThrottle;
+	uint8_t     EnabledForActivity;
+
+	uint8_t     UpHyst;
+	uint8_t     DownHyst;
+	uint8_t     VoltageDownHyst;
+	uint8_t     padding;
+
+	uint16_t    ActivityLevel;
+	uint8_t     DisplayWatermark;
+	uint8_t     padding1;
+
+	uint32_t    MpllFuncCntl;
+	uint32_t    MpllFuncCntl_1;
+	uint32_t    MpllFuncCntl_2;
+	uint32_t    MpllAdFuncCntl;
+	uint32_t    MpllDqFuncCntl;
+	uint32_t    MclkPwrmgtCntl;
+	uint32_t    DllCntl;
+	uint32_t    MpllSs1;
+	uint32_t    MpllSs2;
+};
+
+typedef struct SMU72_Discrete_MemoryLevel SMU72_Discrete_MemoryLevel;
+
+struct SMU72_Discrete_LinkLevel {
+	uint8_t     PcieGenSpeed;           /*< 0:PciE-gen1 1:PciE-gen2 2:PciE-gen3 */
+	uint8_t     PcieLaneCount;          /*< 1=x1, 2=x2, 3=x4, 4=x8, 5=x12, 6=x16 */
+	uint8_t     EnabledForActivity;
+	uint8_t     SPC;
+	uint32_t    DownThreshold;
+	uint32_t    UpThreshold;
+	uint32_t    Reserved;
+};
+
+typedef struct SMU72_Discrete_LinkLevel SMU72_Discrete_LinkLevel;
+
+/* MC ARB DRAM Timing registers. */
+struct SMU72_Discrete_MCArbDramTimingTableEntry {
+	uint32_t McArbDramTiming;
+	uint32_t McArbDramTiming2;
+	uint8_t  McArbBurstTime;
+	uint8_t  padding[3];
+};
+
+typedef struct SMU72_Discrete_MCArbDramTimingTableEntry SMU72_Discrete_MCArbDramTimingTableEntry;
+
+struct SMU72_Discrete_MCArbDramTimingTable {
+	SMU72_Discrete_MCArbDramTimingTableEntry entries[SMU__NUM_SCLK_DPM_STATE][SMU__NUM_MCLK_DPM_LEVELS];
+};
+
+typedef struct SMU72_Discrete_MCArbDramTimingTable SMU72_Discrete_MCArbDramTimingTable;
+
+/* UVD VCLK/DCLK state (level) definition. */
+struct SMU72_Discrete_UvdLevel {
+	uint32_t VclkFrequency;
+	uint32_t DclkFrequency;
+	SMU_VoltageLevel MinVoltage;
+	uint8_t  VclkDivider;
+	uint8_t  DclkDivider;
+	uint8_t  padding[2];
+};
+
+typedef struct SMU72_Discrete_UvdLevel SMU72_Discrete_UvdLevel;
+
+/* Clocks for other external blocks (VCE, ACP, SAMU). */
+struct SMU72_Discrete_ExtClkLevel {
+	uint32_t Frequency;
+	SMU_VoltageLevel MinVoltage;
+	uint8_t  Divider;
+	uint8_t  padding[3];
+};
+
+typedef struct SMU72_Discrete_ExtClkLevel SMU72_Discrete_ExtClkLevel;
+
+struct SMU72_Discrete_StateInfo {
+	uint32_t SclkFrequency;
+	uint32_t MclkFrequency;
+	uint32_t VclkFrequency;
+	uint32_t DclkFrequency;
+	uint32_t SamclkFrequency;
+	uint32_t AclkFrequency;
+	uint32_t EclkFrequency;
+	uint16_t MvddVoltage;
+	uint16_t padding16;
+	uint8_t  DisplayWatermark;
+	uint8_t  McArbIndex;
+	uint8_t  McRegIndex;
+	uint8_t  SeqIndex;
+	uint8_t  SclkDid;
+	int8_t   SclkIndex;
+	int8_t   MclkIndex;
+	uint8_t  PCIeGen;
+
+};
+
+typedef struct SMU72_Discrete_StateInfo SMU72_Discrete_StateInfo;
+
+struct SMU72_Discrete_DpmTable {
+	/* Multi-DPM controller settings */
+	SMU72_PIDController                  GraphicsPIDController;
+	SMU72_PIDController                  MemoryPIDController;
+	SMU72_PIDController                  LinkPIDController;
+
+	uint32_t                            SystemFlags;
+
+	/* SMIO masks for voltage and phase controls */
+	uint32_t                            VRConfig;
+	uint32_t                            SmioMask1;
+	uint32_t                            SmioMask2;
+	SMIO_Table                          SmioTable1;
+	SMIO_Table                          SmioTable2;
+
+	uint32_t                            VddcLevelCount;
+	uint32_t                            VddciLevelCount;
+	uint32_t                            VddGfxLevelCount;
+	uint32_t                            MvddLevelCount;
+
+	uint16_t                            VddcTable[SMU72_MAX_LEVELS_VDDC];
+	uint16_t                            VddGfxTable[SMU72_MAX_LEVELS_VDDGFX];
+	uint16_t                            VddciTable[SMU72_MAX_LEVELS_VDDCI];
+
+	uint8_t                             BapmVddGfxVidHiSidd[SMU72_MAX_LEVELS_VDDGFX];
+	uint8_t                             BapmVddGfxVidLoSidd[SMU72_MAX_LEVELS_VDDGFX];
+	uint8_t                             BapmVddGfxVidHiSidd2[SMU72_MAX_LEVELS_VDDGFX];
+
+	uint8_t                             BapmVddcVidHiSidd[SMU72_MAX_LEVELS_VDDC];
+	uint8_t                             BapmVddcVidLoSidd[SMU72_MAX_LEVELS_VDDC];
+	uint8_t                             BapmVddcVidHiSidd2[SMU72_MAX_LEVELS_VDDC];
+
+	uint8_t                             GraphicsDpmLevelCount;
+	uint8_t                             MemoryDpmLevelCount;
+	uint8_t                             LinkLevelCount;
+	uint8_t                             MasterDeepSleepControl;
+
+	uint8_t                             UvdLevelCount;
+	uint8_t                             VceLevelCount;
+	uint8_t                             AcpLevelCount;
+	uint8_t                             SamuLevelCount;
+
+	uint8_t                             ThermOutGpio;
+	uint8_t                             ThermOutPolarity;
+	uint8_t                             ThermOutMode;
+	uint8_t                             DPMFreezeAndForced;
+	uint32_t                            Reserved[4];
+
+	/* State table entries for each DPM state */
+	SMU72_Discrete_GraphicsLevel        GraphicsLevel[SMU72_MAX_LEVELS_GRAPHICS];
+	SMU72_Discrete_MemoryLevel          MemoryACPILevel;
+	SMU72_Discrete_MemoryLevel          MemoryLevel[SMU72_MAX_LEVELS_MEMORY];
+	SMU72_Discrete_LinkLevel            LinkLevel[SMU72_MAX_LEVELS_LINK];
+	SMU72_Discrete_ACPILevel            ACPILevel;
+	SMU72_Discrete_UvdLevel             UvdLevel[SMU72_MAX_LEVELS_UVD];
+	SMU72_Discrete_ExtClkLevel          VceLevel[SMU72_MAX_LEVELS_VCE];
+	SMU72_Discrete_ExtClkLevel          AcpLevel[SMU72_MAX_LEVELS_ACP];
+	SMU72_Discrete_ExtClkLevel          SamuLevel[SMU72_MAX_LEVELS_SAMU];
+	SMU72_Discrete_Ulv                  Ulv;
+
+	uint32_t                            SclkStepSize;
+	uint32_t                            Smio[SMU72_MAX_ENTRIES_SMIO];
+
+	uint8_t                             UvdBootLevel;
+	uint8_t                             VceBootLevel;
+	uint8_t                             AcpBootLevel;
+	uint8_t                             SamuBootLevel;
+
+	uint8_t                             GraphicsBootLevel;
+	uint8_t                             GraphicsVoltageChangeEnable;
+	uint8_t                             GraphicsThermThrottleEnable;
+	uint8_t                             GraphicsInterval;
+
+	uint8_t                             VoltageInterval;
+	uint8_t                             ThermalInterval;
+	uint16_t                            TemperatureLimitHigh;
+
+	uint16_t                            TemperatureLimitLow;
+	uint8_t                             MemoryBootLevel;
+	uint8_t                             MemoryVoltageChangeEnable;
+
+	uint16_t                            BootMVdd;
+	uint8_t                             MemoryInterval;
+	uint8_t                             MemoryThermThrottleEnable;
+
+	uint16_t                            VoltageResponseTime;
+	uint16_t                            PhaseResponseTime;
+
+	uint8_t                             PCIeBootLinkLevel;
+	uint8_t                             PCIeGenInterval;
+	uint8_t                             DTEInterval;
+	uint8_t                             DTEMode;
+
+	uint8_t                             SVI2Enable;
+	uint8_t                             VRHotGpio;
+	uint8_t                             AcDcGpio;
+	uint8_t                             ThermGpio;
+
+	uint16_t                            PPM_PkgPwrLimit;
+	uint16_t                            PPM_TemperatureLimit;
+
+	uint16_t                            DefaultTdp;
+	uint16_t                            TargetTdp;
+
+	uint16_t                            FpsHighThreshold;
+	uint16_t                            FpsLowThreshold;
+
+	uint16_t                            BAPMTI_R[SMU72_DTE_ITERATIONS][SMU72_DTE_SOURCES][SMU72_DTE_SINKS];
+	uint16_t                            BAPMTI_RC[SMU72_DTE_ITERATIONS][SMU72_DTE_SOURCES][SMU72_DTE_SINKS];
+
+	uint8_t                             DTEAmbientTempBase;
+	uint8_t                             DTETjOffset;
+	uint8_t                             GpuTjMax;
+	uint8_t                             GpuTjHyst;
+
+	SMU_VoltageLevel                    BootVoltage;
+
+	uint32_t                            BAPM_TEMP_GRADIENT;
+
+	uint32_t                            LowSclkInterruptThreshold;
+	uint32_t                            VddGfxReChkWait;
+
+	uint8_t                             ClockStretcherAmount;
+
+	uint8_t                             Sclk_CKS_masterEn0_7;
+	uint8_t                             Sclk_CKS_masterEn8_15;
+	uint8_t                             padding[1];
+
+	uint8_t                             Sclk_voltageOffset[8];
+
+	SMU_ClockStretcherDataTable         ClockStretcherDataTable;
+	SMU_CKS_LOOKUPTable                 CKS_LOOKUPTable;
+};
+
+typedef struct SMU72_Discrete_DpmTable SMU72_Discrete_DpmTable;
+
+/* --------------------------------------------------- AC Timing Parameters ------------------------------------------------ */
+#define SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE 16
+#define SMU72_DISCRETE_MC_REGISTER_ARRAY_SET_COUNT SMU72_MAX_LEVELS_MEMORY /* DPM */
+
+struct SMU72_Discrete_MCRegisterAddress {
+	uint16_t s0;
+	uint16_t s1;
+};
+
+typedef struct SMU72_Discrete_MCRegisterAddress SMU72_Discrete_MCRegisterAddress;
+
+struct SMU72_Discrete_MCRegisterSet {
+	uint32_t value[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMU72_Discrete_MCRegisterSet SMU72_Discrete_MCRegisterSet;
+
+struct SMU72_Discrete_MCRegisters {
+	uint8_t                             last;
+	uint8_t                             reserved[3];
+	SMU72_Discrete_MCRegisterAddress     address[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+	SMU72_Discrete_MCRegisterSet         data[SMU72_DISCRETE_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMU72_Discrete_MCRegisters SMU72_Discrete_MCRegisters;
+
+
+/* --------------------------------------------------- Fan Table ----------------------------------------------------------- */
+
+struct SMU72_Discrete_FanTable {
+	uint16_t FdoMode;
+	int16_t  TempMin;
+	int16_t  TempMed;
+	int16_t  TempMax;
+	int16_t  Slope1;
+	int16_t  Slope2;
+	int16_t  FdoMin;
+	int16_t  HystUp;
+	int16_t  HystDown;
+	int16_t  HystSlope;
+	int16_t  TempRespLim;
+	int16_t  TempCurr;
+	int16_t  SlopeCurr;
+	int16_t  PwmCurr;
+	uint32_t RefreshPeriod;
+	int16_t  FdoMax;
+	uint8_t  TempSrc;
+	int8_t   FanControl_GL_Flag;
+};
+
+typedef struct SMU72_Discrete_FanTable SMU72_Discrete_FanTable;
+
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG             4
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG_BIT         (0x1 << SMU7_DISCRETE_GPIO_SCLK_DEBUG)
+
+struct SMU7_MclkDpmScoreboard {
+
+	uint32_t PercentageBusy;
+
+	int32_t  PIDError;
+	int32_t  PIDIntegral;
+	int32_t  PIDOutput;
+
+	uint32_t SigmaDeltaAccum;
+	uint32_t SigmaDeltaOutput;
+	uint32_t SigmaDeltaLevel;
+
+	uint32_t UtilizationSetpoint;
+
+	uint8_t  TdpClampMode;
+	uint8_t  TdcClampMode;
+	uint8_t  ThermClampMode;
+	uint8_t  VoltageBusy;
+
+	int8_t   CurrLevel;
+	int8_t   TargLevel;
+	uint8_t  LevelChangeInProgress;
+	uint8_t  UpHyst;
+
+	uint8_t  DownHyst;
+	uint8_t  VoltageDownHyst;
+	uint8_t  DpmEnable;
+	uint8_t  DpmRunning;
+
+	uint8_t  DpmForce;
+	uint8_t  DpmForceLevel;
+	uint8_t  DisplayWatermark;
+	uint8_t  McArbIndex;
+
+	uint32_t MinimumPerfMclk;
+
+	uint8_t  AcpiReq;
+	uint8_t  AcpiAck;
+	uint8_t  MclkSwitchInProgress;
+	uint8_t  MclkSwitchCritical;
+
+	uint8_t  IgnoreVBlank;
+	uint8_t  TargetMclkIndex;
+	uint8_t  TargetMvddIndex;
+	uint8_t  MclkSwitchResult;
+
+	uint16_t VbiFailureCount;
+	uint8_t  VbiWaitCounter;
+	uint8_t  EnabledLevelsChange;
+
+	uint16_t LevelResidencyCountersN[SMU72_MAX_LEVELS_MEMORY];
+	uint16_t LevelSwitchCounters[SMU72_MAX_LEVELS_MEMORY];
+
+	void     (*TargetStateCalculator)(uint8_t);
+	void     (*SavedTargetStateCalculator)(uint8_t);
+
+	uint16_t AutoDpmInterval;
+	uint16_t AutoDpmRange;
+
+	uint16_t VbiTimeoutCount;
+	uint16_t MclkSwitchingTime;
+
+	uint8_t  fastSwitch;
+	uint8_t  Save_PIC_VDDGFX_EXIT;
+	uint8_t  Save_PIC_VDDGFX_ENTER;
+	uint8_t  padding;
+
+};
+
+typedef struct SMU7_MclkDpmScoreboard SMU7_MclkDpmScoreboard;
+
+struct SMU7_UlvScoreboard {
+	uint8_t     EnterUlv;
+	uint8_t     ExitUlv;
+	uint8_t     UlvActive;
+	uint8_t     WaitingForUlv;
+	uint8_t     UlvEnable;
+	uint8_t     UlvRunning;
+	uint8_t     UlvMasterEnable;
+	uint8_t     padding;
+	uint32_t    UlvAbortedCount;
+	uint32_t    UlvTimeStamp;
+};
+
+typedef struct SMU7_UlvScoreboard SMU7_UlvScoreboard;
+
+struct VddgfxSavedRegisters {
+	uint32_t GPU_DBG[3];
+	uint32_t MEC_BaseAddress_Hi;
+	uint32_t MEC_BaseAddress_Lo;
+	uint32_t THM_TMON0_CTRL2__RDIR_PRESENT;
+	uint32_t THM_TMON1_CTRL2__RDIR_PRESENT;
+	uint32_t CP_INT_CNTL;
+};
+
+typedef struct VddgfxSavedRegisters VddgfxSavedRegisters;
+
+struct SMU7_VddGfxScoreboard {
+	uint8_t     VddGfxEnable;
+	uint8_t     VddGfxActive;
+	uint8_t     VPUResetOccured;
+	uint8_t     padding;
+
+	uint32_t    VddGfxEnteredCount;
+	uint32_t    VddGfxAbortedCount;
+
+	uint32_t    VddGfxVid;
+
+	VddgfxSavedRegisters SavedRegisters;
+};
+
+typedef struct SMU7_VddGfxScoreboard SMU7_VddGfxScoreboard;
+
+struct SMU7_TdcLimitScoreboard {
+	uint8_t  Enable;
+	uint8_t  Running;
+	uint16_t Alpha;
+	uint32_t FilteredIddc;
+	uint32_t IddcLimit;
+	uint32_t IddcHyst;
+	SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_TdcLimitScoreboard SMU7_TdcLimitScoreboard;
+
+struct SMU7_PkgPwrLimitScoreboard {
+	uint8_t  Enable;
+	uint8_t  Running;
+	uint16_t Alpha;
+	uint32_t FilteredPkgPwr;
+	uint32_t Limit;
+	uint32_t Hyst;
+	uint32_t LimitFromDriver;
+	SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_PkgPwrLimitScoreboard SMU7_PkgPwrLimitScoreboard;
+
+struct SMU7_BapmScoreboard {
+	uint32_t source_powers[SMU72_DTE_SOURCES];
+	uint32_t source_powers_last[SMU72_DTE_SOURCES];
+	int32_t entity_temperatures[SMU72_NUM_GPU_TES];
+	int32_t initial_entity_temperatures[SMU72_NUM_GPU_TES];
+	int32_t Limit;
+	int32_t Hyst;
+	int32_t therm_influence_coeff_table[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS * 2];
+	int32_t therm_node_table[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS];
+	uint16_t ConfigTDPPowerScalar;
+	uint16_t FanSpeedPowerScalar;
+	uint16_t OverDrivePowerScalar;
+	uint16_t OverDriveLimitScalar;
+	uint16_t FinalPowerScalar;
+	uint8_t VariantID;
+	uint8_t spare997;
+
+	SMU7_HystController_Data HystControllerData;
+
+	int32_t temperature_gradient_slope;
+	int32_t temperature_gradient;
+	uint32_t measured_temperature;
+};
+
+
+typedef struct SMU7_BapmScoreboard SMU7_BapmScoreboard;
+
+struct SMU7_AcpiScoreboard {
+	uint32_t SavedInterruptMask[2];
+	uint8_t LastACPIRequest;
+	uint8_t CgBifResp;
+	uint8_t RequestType;
+	uint8_t Padding;
+	SMU72_Discrete_ACPILevel D0Level;
+};
+
+typedef struct SMU7_AcpiScoreboard SMU7_AcpiScoreboard;
+
+struct SMU72_Discrete_PmFuses {
+	/* dw1  */
+	uint8_t SviLoadLineEn;
+	uint8_t SviLoadLineVddC;
+	uint8_t SviLoadLineTrimVddC;
+	uint8_t SviLoadLineOffsetVddC;
+
+	/* dw2 */
+	uint16_t TDC_VDDC_PkgLimit;
+	uint8_t TDC_VDDC_ThrottleReleaseLimitPerc;
+	uint8_t TDC_MAWt;
+
+	/* dw3 */
+	uint8_t TdcWaterfallCtl;
+	uint8_t LPMLTemperatureMin;
+	uint8_t LPMLTemperatureMax;
+	uint8_t Reserved;
+
+	/* dw4-dw7  */
+	uint8_t LPMLTemperatureScaler[16];
+
+	/* dw8-dw9  */
+	int16_t FuzzyFan_ErrorSetDelta;
+	int16_t FuzzyFan_ErrorRateSetDelta;
+	int16_t FuzzyFan_PwmSetDelta;
+	uint16_t Reserved6;
+
+	/* dw10-dw14  */
+	uint8_t GnbLPML[16];
+
+	/* dw15 */
+	uint8_t GnbLPMLMaxVid;
+	uint8_t GnbLPMLMinVid;
+	uint8_t Reserved1[2];
+
+	/* dw16 */
+	uint16_t BapmVddCBaseLeakageHiSidd;
+	uint16_t BapmVddCBaseLeakageLoSidd;
+};
+
+typedef struct SMU72_Discrete_PmFuses SMU72_Discrete_PmFuses;
+
+struct SMU7_Discrete_Log_Header_Table {
+	uint32_t    version;
+	uint32_t    asic_id;
+	uint16_t    flags;
+	uint16_t    entry_size;
+	uint32_t    total_size;
+	uint32_t    num_of_entries;
+	uint8_t     type;
+	uint8_t     mode;
+	uint8_t     filler_0[2];
+	uint32_t    filler_1[2];
+};
+
+typedef struct SMU7_Discrete_Log_Header_Table SMU7_Discrete_Log_Header_Table;
+
+struct SMU7_Discrete_Log_Cntl {
+	uint8_t             Enabled;
+	uint8_t             Type;
+	uint8_t             padding[2];
+	uint32_t            BufferSize;
+	uint32_t            SamplesLogged;
+	uint32_t            SampleSize;
+	uint32_t            AddrL;
+	uint32_t            AddrH;
+};
+
+typedef struct SMU7_Discrete_Log_Cntl SMU7_Discrete_Log_Cntl;
+
+#define CAC_ACC_NW_NUM_OF_SIGNALS 87
+
+struct SMU7_Discrete_Cac_Collection_Table {
+	uint32_t temperature;
+	uint32_t cac_acc_nw[CAC_ACC_NW_NUM_OF_SIGNALS];
+};
+
+typedef struct SMU7_Discrete_Cac_Collection_Table SMU7_Discrete_Cac_Collection_Table;
+
+struct SMU7_Discrete_Cac_Verification_Table {
+	uint32_t VddcTotalPower;
+	uint32_t VddcLeakagePower;
+	uint32_t VddcConstantPower;
+	uint32_t VddcGfxDynamicPower;
+	uint32_t VddcUvdDynamicPower;
+	uint32_t VddcVceDynamicPower;
+	uint32_t VddcAcpDynamicPower;
+	uint32_t VddcPcieDynamicPower;
+	uint32_t VddcDceDynamicPower;
+	uint32_t VddcCurrent;
+	uint32_t VddcVoltage;
+	uint32_t VddciTotalPower;
+	uint32_t VddciLeakagePower;
+	uint32_t VddciConstantPower;
+	uint32_t VddciDynamicPower;
+	uint32_t Vddr1TotalPower;
+	uint32_t Vddr1LeakagePower;
+	uint32_t Vddr1ConstantPower;
+	uint32_t Vddr1DynamicPower;
+	uint32_t spare[4];
+	uint32_t temperature;
+};
+
+typedef struct SMU7_Discrete_Cac_Verification_Table SMU7_Discrete_Cac_Verification_Table;
+
+struct SMU7_Discrete_Pm_Status_Table {
+	/* Thermal entities */
+	int32_t T_meas_max;
+	int32_t T_meas_acc;
+	int32_t T_calc_max;
+	int32_t T_calc_acc;
+	uint32_t P_scalar_acc;
+	uint32_t P_calc_max;
+	uint32_t P_calc_acc;
+
+	/*Voltage domains */
+	uint32_t I_calc_max;
+	uint32_t I_calc_acc;
+	uint32_t I_calc_acc_vddci;
+	uint32_t V_calc_noload_acc;
+	uint32_t V_calc_load_acc;
+	uint32_t V_calc_noload_acc_vddci;
+	uint32_t P_meas_acc;
+	uint32_t V_meas_noload_acc;
+	uint32_t V_meas_load_acc;
+	uint32_t I_meas_acc;
+	uint32_t P_meas_acc_vddci;
+	uint32_t V_meas_noload_acc_vddci;
+	uint32_t V_meas_load_acc_vddci;
+	uint32_t I_meas_acc_vddci;
+
+	/*Frequency */
+	uint16_t Sclk_dpm_residency[8];
+	uint16_t Uvd_dpm_residency[8];
+	uint16_t Vce_dpm_residency[8];
+	uint16_t Mclk_dpm_residency[4];
+
+	/*Chip */
+	uint32_t P_vddci_acc;
+	uint32_t P_vddr1_acc;
+	uint32_t P_nte1_acc;
+	uint32_t PkgPwr_max;
+	uint32_t PkgPwr_acc;
+	uint32_t MclkSwitchingTime_max;
+	uint32_t MclkSwitchingTime_acc;
+	uint32_t FanPwm_acc;
+	uint32_t FanRpm_acc;
+
+	uint32_t AccCnt;
+};
+
+typedef struct SMU7_Discrete_Pm_Status_Table SMU7_Discrete_Pm_Status_Table;
+
+/*FIXME THESE NEED TO BE UPDATED */
+#define SMU7_SCLK_CAC 0x561
+#define SMU7_MCLK_CAC 0xF9
+#define SMU7_VCLK_CAC 0x2DE
+#define SMU7_DCLK_CAC 0x2DE
+#define SMU7_ECLK_CAC 0x25E
+#define SMU7_ACLK_CAC 0x25E
+#define SMU7_SAMCLK_CAC 0x25E
+#define SMU7_DISPCLK_CAC 0x100
+#define SMU7_CAC_CONSTANT 0x2EE3430
+#define SMU7_CAC_CONSTANT_SHIFT 18
+
+#define SMU7_VDDCI_MCLK_CONST        1765
+#define SMU7_VDDCI_MCLK_CONST_SHIFT  16
+#define SMU7_VDDCI_VDDCI_CONST       50958
+#define SMU7_VDDCI_VDDCI_CONST_SHIFT 14
+#define SMU7_VDDCI_CONST             11781
+
+#define SMU7_12C_VDDCI_MCLK_CONST        1623
+#define SMU7_12C_VDDCI_MCLK_CONST_SHIFT  15
+#define SMU7_12C_VDDCI_VDDCI_CONST       40088
+#define SMU7_12C_VDDCI_VDDCI_CONST_SHIFT 13
+#define SMU7_12C_VDDCI_CONST             20856
+
+#define SMU7_VDDCI_STROBE_PWR        1331
+
+#define SMU7_VDDR1_CONST            693
+#define SMU7_VDDR1_CAC_WEIGHT       20
+#define SMU7_VDDR1_CAC_WEIGHT_SHIFT 19
+#define SMU7_VDDR1_STROBE_PWR       512
+
+#define SMU7_AREA_COEFF_UVD 0xA78
+#define SMU7_AREA_COEFF_VCE 0x190A
+#define SMU7_AREA_COEFF_ACP 0x22D1
+#define SMU7_AREA_COEFF_SAMU 0x534
+
+/*ThermOutMode values */
+#define SMU7_THERM_OUT_MODE_DISABLE       0x0
+#define SMU7_THERM_OUT_MODE_THERM_ONLY    0x1
+#define SMU7_THERM_OUT_MODE_THERM_VRHOT   0x2
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(pop)
+#endif
+
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu73.h b/drivers/gpu/drm/amd/powerplay/inc/smu73.h
new file mode 100644
index 000000000000..c6b12a4c00db
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu73.h
@@ -0,0 +1,720 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _SMU73_H_
+#define _SMU73_H_
+
+#pragma pack(push, 1)
+enum SID_OPTION {
+  SID_OPTION_HI,
+  SID_OPTION_LO,
+  SID_OPTION_COUNT
+};
+
+enum Poly3rdOrderCoeff {
+    LEAKAGE_TEMPERATURE_SCALAR,
+    LEAKAGE_VOLTAGE_SCALAR,
+    DYNAMIC_VOLTAGE_SCALAR,
+    POLY_3RD_ORDER_COUNT
+};
+
+struct SMU7_Poly3rdOrder_Data
+{
+    int32_t a;
+    int32_t b;
+    int32_t c;
+    int32_t d;
+    uint8_t a_shift;
+    uint8_t b_shift;
+    uint8_t c_shift;
+    uint8_t x_shift;
+};
+
+typedef struct SMU7_Poly3rdOrder_Data SMU7_Poly3rdOrder_Data;
+
+struct Power_Calculator_Data
+{
+  uint16_t NoLoadVoltage;
+  uint16_t LoadVoltage;
+  uint16_t Resistance;
+  uint16_t Temperature;
+  uint16_t BaseLeakage;
+  uint16_t LkgTempScalar;
+  uint16_t LkgVoltScalar;
+  uint16_t LkgAreaScalar;
+  uint16_t LkgPower;
+  uint16_t DynVoltScalar;
+  uint32_t Cac;
+  uint32_t DynPower;
+  uint32_t TotalCurrent;
+  uint32_t TotalPower;
+};
+
+typedef struct Power_Calculator_Data PowerCalculatorData_t;
+
+struct Gc_Cac_Weight_Data
+{
+  uint8_t index;
+  uint32_t value;
+};
+
+typedef struct Gc_Cac_Weight_Data GcCacWeight_Data;
+
+
+typedef struct {
+  uint32_t high;
+  uint32_t low;
+} data_64_t;
+
+typedef struct {
+  data_64_t high;
+  data_64_t low;
+} data_128_t;
+
+#define SMU__NUM_SCLK_DPM_STATE  8
+#define SMU__NUM_MCLK_DPM_LEVELS 4
+#define SMU__NUM_LCLK_DPM_LEVELS 8
+#define SMU__NUM_PCIE_DPM_LEVELS 8
+
+#define SMU7_CONTEXT_ID_SMC        1
+#define SMU7_CONTEXT_ID_VBIOS      2
+
+#define SMU73_MAX_LEVELS_VDDC            16
+#define SMU73_MAX_LEVELS_VDDGFX          16
+#define SMU73_MAX_LEVELS_VDDCI           8
+#define SMU73_MAX_LEVELS_MVDD            4
+
+#define SMU_MAX_SMIO_LEVELS              4
+
+#define SMU73_MAX_LEVELS_GRAPHICS        SMU__NUM_SCLK_DPM_STATE   // SCLK + SQ DPM + ULV
+#define SMU73_MAX_LEVELS_MEMORY          SMU__NUM_MCLK_DPM_LEVELS   // MCLK Levels DPM
+#define SMU73_MAX_LEVELS_GIO             SMU__NUM_LCLK_DPM_LEVELS  // LCLK Levels
+#define SMU73_MAX_LEVELS_LINK            SMU__NUM_PCIE_DPM_LEVELS  // PCIe speed and number of lanes.
+#define SMU73_MAX_LEVELS_UVD             8   // VCLK/DCLK levels for UVD.
+#define SMU73_MAX_LEVELS_VCE             8   // ECLK levels for VCE.
+#define SMU73_MAX_LEVELS_ACP             8   // ACLK levels for ACP.
+#define SMU73_MAX_LEVELS_SAMU            8   // SAMCLK levels for SAMU.
+#define SMU73_MAX_ENTRIES_SMIO           32  // Number of entries in SMIO table.
+
+#define DPM_NO_LIMIT 0
+#define DPM_NO_UP 1
+#define DPM_GO_DOWN 2
+#define DPM_GO_UP 3
+
+#define SMU7_FIRST_DPM_GRAPHICS_LEVEL    0
+#define SMU7_FIRST_DPM_MEMORY_LEVEL      0
+
+#define GPIO_CLAMP_MODE_VRHOT      1
+#define GPIO_CLAMP_MODE_THERM      2
+#define GPIO_CLAMP_MODE_DC         4
+
+#define SCRATCH_B_TARG_PCIE_INDEX_SHIFT 0
+#define SCRATCH_B_TARG_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_TARG_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_PCIE_INDEX_SHIFT 3
+#define SCRATCH_B_CURR_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_CURR_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_UVD_INDEX_SHIFT  6
+#define SCRATCH_B_TARG_UVD_INDEX_MASK   (0x7<<SCRATCH_B_TARG_UVD_INDEX_SHIFT)
+#define SCRATCH_B_CURR_UVD_INDEX_SHIFT  9
+#define SCRATCH_B_CURR_UVD_INDEX_MASK   (0x7<<SCRATCH_B_CURR_UVD_INDEX_SHIFT)
+#define SCRATCH_B_TARG_VCE_INDEX_SHIFT  12
+#define SCRATCH_B_TARG_VCE_INDEX_MASK   (0x7<<SCRATCH_B_TARG_VCE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_VCE_INDEX_SHIFT  15
+#define SCRATCH_B_CURR_VCE_INDEX_MASK   (0x7<<SCRATCH_B_CURR_VCE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_ACP_INDEX_SHIFT  18
+#define SCRATCH_B_TARG_ACP_INDEX_MASK   (0x7<<SCRATCH_B_TARG_ACP_INDEX_SHIFT)
+#define SCRATCH_B_CURR_ACP_INDEX_SHIFT  21
+#define SCRATCH_B_CURR_ACP_INDEX_MASK   (0x7<<SCRATCH_B_CURR_ACP_INDEX_SHIFT)
+#define SCRATCH_B_TARG_SAMU_INDEX_SHIFT 24
+#define SCRATCH_B_TARG_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_TARG_SAMU_INDEX_SHIFT)
+#define SCRATCH_B_CURR_SAMU_INDEX_SHIFT 27
+#define SCRATCH_B_CURR_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_CURR_SAMU_INDEX_SHIFT)
+
+// Virtualization Defines
+#define CG_XDMA_MASK  0x1
+#define CG_XDMA_SHIFT 0
+#define CG_UVD_MASK   0x2
+#define CG_UVD_SHIFT  1
+#define CG_VCE_MASK   0x4
+#define CG_VCE_SHIFT  2
+#define CG_SAMU_MASK  0x8
+#define CG_SAMU_SHIFT 3
+#define CG_GFX_MASK   0x10
+#define CG_GFX_SHIFT  4
+#define CG_SDMA_MASK  0x20
+#define CG_SDMA_SHIFT 5
+#define CG_HDP_MASK   0x40
+#define CG_HDP_SHIFT  6
+#define CG_MC_MASK    0x80
+#define CG_MC_SHIFT   7
+#define CG_DRM_MASK   0x100
+#define CG_DRM_SHIFT  8
+#define CG_ROM_MASK   0x200
+#define CG_ROM_SHIFT  9
+#define CG_BIF_MASK   0x400
+#define CG_BIF_SHIFT  10
+
+#define SMU73_DTE_ITERATIONS 5
+#define SMU73_DTE_SOURCES 3
+#define SMU73_DTE_SINKS 1
+#define SMU73_NUM_CPU_TES 0
+#define SMU73_NUM_GPU_TES 1
+#define SMU73_NUM_NON_TES 2
+#define SMU73_DTE_FAN_SCALAR_MIN 0x100
+#define SMU73_DTE_FAN_SCALAR_MAX 0x166
+#define SMU73_DTE_FAN_TEMP_MAX 93
+#define SMU73_DTE_FAN_TEMP_MIN 83
+
+#define SMU73_THERMAL_INPUT_LOOP_COUNT 6
+#define SMU73_THERMAL_CLAMP_MODE_COUNT 8
+
+
+struct SMU7_HystController_Data
+{
+    uint16_t waterfall_up;
+    uint16_t waterfall_down;
+    uint16_t waterfall_limit;
+    uint16_t release_cnt;
+    uint16_t release_limit;
+    uint16_t spare;
+};
+
+typedef struct SMU7_HystController_Data SMU7_HystController_Data;
+
+struct SMU73_PIDController
+{
+    uint32_t Ki;
+    int32_t LFWindupUpperLim;
+    int32_t LFWindupLowerLim;
+    uint32_t StatePrecision;
+
+    uint32_t LfPrecision;
+    uint32_t LfOffset;
+    uint32_t MaxState;
+    uint32_t MaxLfFraction;
+    uint32_t StateShift;
+};
+
+typedef struct SMU73_PIDController SMU73_PIDController;
+
+struct SMU7_LocalDpmScoreboard
+{
+    uint32_t PercentageBusy;
+
+    int32_t  PIDError;
+    int32_t  PIDIntegral;
+    int32_t  PIDOutput;
+
+    uint32_t SigmaDeltaAccum;
+    uint32_t SigmaDeltaOutput;
+    uint32_t SigmaDeltaLevel;
+
+    uint32_t UtilizationSetpoint;
+
+    uint8_t  TdpClampMode;
+    uint8_t  TdcClampMode;
+    uint8_t  ThermClampMode;
+    uint8_t  VoltageBusy;
+
+    int8_t   CurrLevel;
+    int8_t   TargLevel;
+    uint8_t  LevelChangeInProgress;
+    uint8_t  UpHyst;
+
+    uint8_t  DownHyst;
+    uint8_t  VoltageDownHyst;
+    uint8_t  DpmEnable;
+    uint8_t  DpmRunning;
+
+    uint8_t  DpmForce;
+    uint8_t  DpmForceLevel;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+
+    uint32_t MinimumPerfSclk;
+
+    uint8_t  AcpiReq;
+    uint8_t  AcpiAck;
+    uint8_t  GfxClkSlow;
+    uint8_t  GpioClampMode;
+
+    uint8_t  spare2;
+    uint8_t  EnabledLevelsChange;
+    uint8_t  DteClampMode;
+    uint8_t  FpsClampMode;
+
+    uint16_t LevelResidencyCounters [SMU73_MAX_LEVELS_GRAPHICS];
+    uint16_t LevelSwitchCounters [SMU73_MAX_LEVELS_GRAPHICS];
+
+    void     (*TargetStateCalculator)(uint8_t);
+    void     (*SavedTargetStateCalculator)(uint8_t);
+
+    uint16_t AutoDpmInterval;
+    uint16_t AutoDpmRange;
+
+    uint8_t  FpsEnabled;
+    uint8_t  MaxPerfLevel;
+    uint8_t  AllowLowClkInterruptToHost;
+    uint8_t  FpsRunning;
+
+    uint32_t MaxAllowedFrequency;
+
+    uint32_t FilteredSclkFrequency;
+    uint32_t LastSclkFrequency;
+    uint32_t FilteredSclkFrequencyCnt;
+
+    uint8_t  LedEnable;
+    uint8_t  LedPin0;
+    uint8_t  LedPin1;
+    uint8_t  LedPin2;
+    uint32_t LedAndMask;
+
+    uint16_t FpsAlpha;
+    uint16_t DeltaTime;
+    uint32_t CurrentFps;
+    uint32_t FilteredFps;
+    uint32_t FrameCount;
+    uint32_t FrameCountLast;
+    uint16_t FpsTargetScalar;
+    uint16_t FpsWaterfallLimitScalar;
+    uint16_t FpsAlphaScalar;
+    uint16_t spare8;
+    SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_LocalDpmScoreboard SMU7_LocalDpmScoreboard;
+
+#define SMU7_MAX_VOLTAGE_CLIENTS 12
+
+typedef uint8_t (*VoltageChangeHandler_t)(uint16_t, uint8_t);
+
+#define VDDC_MASK    0x00007FFF
+#define VDDC_SHIFT   0
+#define VDDCI_MASK   0x3FFF8000
+#define VDDCI_SHIFT  15
+#define PHASES_MASK  0xC0000000
+#define PHASES_SHIFT 30
+
+typedef uint32_t SMU_VoltageLevel;
+
+struct SMU7_VoltageScoreboard
+{
+    SMU_VoltageLevel TargetVoltage;
+    uint16_t MaxVid;
+    uint8_t  HighestVidOffset;
+    uint8_t  CurrentVidOffset;
+
+    uint16_t CurrentVddc;
+    uint16_t CurrentVddci;
+
+
+    uint8_t  ControllerBusy;
+    uint8_t  CurrentVid;
+    uint8_t  CurrentVddciVid;
+    uint8_t  padding;
+
+    SMU_VoltageLevel RequestedVoltage[SMU7_MAX_VOLTAGE_CLIENTS];
+    SMU_VoltageLevel TargetVoltageState;
+    uint8_t  EnabledRequest[SMU7_MAX_VOLTAGE_CLIENTS];
+
+    uint8_t  padding2;
+    uint8_t  padding3;
+    uint8_t  ControllerEnable;
+    uint8_t  ControllerRunning;
+    uint16_t CurrentStdVoltageHiSidd;
+    uint16_t CurrentStdVoltageLoSidd;
+    uint8_t  OverrideVoltage;
+    uint8_t  padding4;
+    uint8_t  padding5;
+    uint8_t  CurrentPhases;
+
+    VoltageChangeHandler_t ChangeVddc;
+
+    VoltageChangeHandler_t ChangeVddci;
+    VoltageChangeHandler_t ChangePhase;
+    VoltageChangeHandler_t ChangeMvdd;
+
+    VoltageChangeHandler_t functionLinks[6];
+
+    uint16_t * VddcFollower1;
+
+    int16_t  Driver_OD_RequestedVidOffset1;
+    int16_t  Driver_OD_RequestedVidOffset2;
+
+};
+
+typedef struct SMU7_VoltageScoreboard SMU7_VoltageScoreboard;
+
+// -------------------------------------------------------------------------------------------------------------------------
+#define SMU7_MAX_PCIE_LINK_SPEEDS 3 /* 0:Gen1 1:Gen2 2:Gen3 */
+
+struct SMU7_PCIeLinkSpeedScoreboard
+{
+    uint8_t     DpmEnable;
+    uint8_t     DpmRunning;
+    uint8_t     DpmForce;
+    uint8_t     DpmForceLevel;
+
+    uint8_t     CurrentLinkSpeed;
+    uint8_t     EnabledLevelsChange;
+    uint16_t    AutoDpmInterval;
+
+    uint16_t    AutoDpmRange;
+    uint16_t    AutoDpmCount;
+
+    uint8_t     DpmMode;
+    uint8_t     AcpiReq;
+    uint8_t     AcpiAck;
+    uint8_t     CurrentLinkLevel;
+
+};
+
+typedef struct SMU7_PCIeLinkSpeedScoreboard SMU7_PCIeLinkSpeedScoreboard;
+
+// -------------------------------------------------------- CAC table ------------------------------------------------------
+#define SMU7_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMU7_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16
+
+#define SMU7_SCALE_I  7
+#define SMU7_SCALE_R 12
+
+struct SMU7_PowerScoreboard
+{
+    uint32_t GpuPower;
+
+    uint32_t VddcPower;
+    uint32_t VddcVoltage;
+    uint32_t VddcCurrent;
+
+    uint32_t MvddPower;
+    uint32_t MvddVoltage;
+    uint32_t MvddCurrent;
+
+    uint32_t RocPower;
+
+    uint16_t Telemetry_1_slope;
+    uint16_t Telemetry_2_slope;
+    int32_t  Telemetry_1_offset;
+    int32_t  Telemetry_2_offset;
+};
+typedef struct SMU7_PowerScoreboard SMU7_PowerScoreboard;
+
+// For FeatureEnables:
+#define SMU7_SCLK_DPM_CONFIG_MASK                        0x01
+#define SMU7_VOLTAGE_CONTROLLER_CONFIG_MASK              0x02
+#define SMU7_THERMAL_CONTROLLER_CONFIG_MASK              0x04
+#define SMU7_MCLK_DPM_CONFIG_MASK                        0x08
+#define SMU7_UVD_DPM_CONFIG_MASK                         0x10
+#define SMU7_VCE_DPM_CONFIG_MASK                         0x20
+#define SMU7_ACP_DPM_CONFIG_MASK                         0x40
+#define SMU7_SAMU_DPM_CONFIG_MASK                        0x80
+#define SMU7_PCIEGEN_DPM_CONFIG_MASK                    0x100
+
+#define SMU7_ACP_MCLK_HANDSHAKE_DISABLE                  0x00000001
+#define SMU7_ACP_SCLK_HANDSHAKE_DISABLE                  0x00000002
+#define SMU7_UVD_MCLK_HANDSHAKE_DISABLE                  0x00000100
+#define SMU7_UVD_SCLK_HANDSHAKE_DISABLE                  0x00000200
+#define SMU7_VCE_MCLK_HANDSHAKE_DISABLE                  0x00010000
+#define SMU7_VCE_SCLK_HANDSHAKE_DISABLE                  0x00020000
+
+// All 'soft registers' should be uint32_t.
+struct SMU73_SoftRegisters
+{
+    uint32_t        RefClockFrequency;
+    uint32_t        PmTimerPeriod;
+    uint32_t        FeatureEnables;
+
+    uint32_t        PreVBlankGap;
+    uint32_t        VBlankTimeout;
+    uint32_t        TrainTimeGap;
+
+    uint32_t        MvddSwitchTime;
+    uint32_t        LongestAcpiTrainTime;
+    uint32_t        AcpiDelay;
+    uint32_t        G5TrainTime;
+    uint32_t        DelayMpllPwron;
+    uint32_t        VoltageChangeTimeout;
+
+    uint32_t        HandshakeDisables;
+
+    uint8_t         DisplayPhy1Config;
+    uint8_t         DisplayPhy2Config;
+    uint8_t         DisplayPhy3Config;
+    uint8_t         DisplayPhy4Config;
+
+    uint8_t         DisplayPhy5Config;
+    uint8_t         DisplayPhy6Config;
+    uint8_t         DisplayPhy7Config;
+    uint8_t         DisplayPhy8Config;
+
+    uint32_t        AverageGraphicsActivity;
+    uint32_t        AverageMemoryActivity;
+    uint32_t        AverageGioActivity;
+
+    uint8_t         SClkDpmEnabledLevels;
+    uint8_t         MClkDpmEnabledLevels;
+    uint8_t         LClkDpmEnabledLevels;
+    uint8_t         PCIeDpmEnabledLevels;
+
+    uint8_t         UVDDpmEnabledLevels;
+    uint8_t         SAMUDpmEnabledLevels;
+    uint8_t         ACPDpmEnabledLevels;
+    uint8_t         VCEDpmEnabledLevels;
+
+    uint32_t        DRAM_LOG_ADDR_H;
+    uint32_t        DRAM_LOG_ADDR_L;
+    uint32_t        DRAM_LOG_PHY_ADDR_H;
+    uint32_t        DRAM_LOG_PHY_ADDR_L;
+    uint32_t        DRAM_LOG_BUFF_SIZE;
+    uint32_t        UlvEnterCount;
+    uint32_t        UlvTime;
+    uint32_t        UcodeLoadStatus;
+    uint32_t        Reserved[2];
+
+};
+
+typedef struct SMU73_SoftRegisters SMU73_SoftRegisters;
+
+struct SMU73_Firmware_Header
+{
+    uint32_t Digest[5];
+    uint32_t Version;
+    uint32_t HeaderSize;
+    uint32_t Flags;
+    uint32_t EntryPoint;
+    uint32_t CodeSize;
+    uint32_t ImageSize;
+
+    uint32_t Rtos;
+    uint32_t SoftRegisters;
+    uint32_t DpmTable;
+    uint32_t FanTable;
+    uint32_t CacConfigTable;
+    uint32_t CacStatusTable;
+
+
+    uint32_t mcRegisterTable;
+
+
+    uint32_t mcArbDramTimingTable;
+
+
+
+
+    uint32_t PmFuseTable;
+    uint32_t Globals;
+    uint32_t ClockStretcherTable;
+    uint32_t Reserved[41];
+    uint32_t Signature;
+};
+
+typedef struct SMU73_Firmware_Header SMU73_Firmware_Header;
+
+#define SMU7_FIRMWARE_HEADER_LOCATION 0x20000
+
+enum  DisplayConfig {
+    PowerDown = 1,
+    DP54x4,
+    DP54x2,
+    DP54x1,
+    DP27x4,
+    DP27x2,
+    DP27x1,
+    HDMI297,
+    HDMI162,
+    LVDS,
+    DP324x4,
+    DP324x2,
+    DP324x1
+};
+
+
+#define MC_BLOCK_COUNT 1
+#define CPL_BLOCK_COUNT 5
+#define SE_BLOCK_COUNT 15
+#define GC_BLOCK_COUNT 24
+
+struct SMU7_Local_Cac {
+  uint8_t BlockId;
+  uint8_t SignalId;
+  uint8_t Threshold;
+  uint8_t Padding;
+};
+
+typedef struct SMU7_Local_Cac SMU7_Local_Cac;
+
+struct SMU7_Local_Cac_Table {
+
+  SMU7_Local_Cac CplLocalCac[CPL_BLOCK_COUNT];
+  SMU7_Local_Cac McLocalCac[MC_BLOCK_COUNT];
+  SMU7_Local_Cac SeLocalCac[SE_BLOCK_COUNT];
+  SMU7_Local_Cac GcLocalCac[GC_BLOCK_COUNT];
+};
+
+typedef struct SMU7_Local_Cac_Table SMU7_Local_Cac_Table;
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(pop)
+#endif
+
+// Description of Clock Gating bitmask for Tonga:
+// System Clock Gating
+#define CG_SYS_BITMASK_FIRST_BIT      0  // First bit of Sys CG bitmask
+#define CG_SYS_BITMASK_LAST_BIT       9  // Last bit of Sys CG bitmask
+#define CG_SYS_BIF_MGLS_SHIFT         0
+#define CG_SYS_ROM_SHIFT              1
+#define CG_SYS_MC_MGCG_SHIFT          2
+#define CG_SYS_MC_MGLS_SHIFT          3
+#define CG_SYS_SDMA_MGCG_SHIFT        4
+#define CG_SYS_SDMA_MGLS_SHIFT        5
+#define CG_SYS_DRM_MGCG_SHIFT         6
+#define CG_SYS_HDP_MGCG_SHIFT         7
+#define CG_SYS_HDP_MGLS_SHIFT         8
+#define CG_SYS_DRM_MGLS_SHIFT         9
+
+#define CG_SYS_BIF_MGLS_MASK          0x1
+#define CG_SYS_ROM_MASK               0x2
+#define CG_SYS_MC_MGCG_MASK           0x4
+#define CG_SYS_MC_MGLS_MASK           0x8
+#define CG_SYS_SDMA_MGCG_MASK         0x10
+#define CG_SYS_SDMA_MGLS_MASK         0x20
+#define CG_SYS_DRM_MGCG_MASK          0x40
+#define CG_SYS_HDP_MGCG_MASK          0x80
+#define CG_SYS_HDP_MGLS_MASK          0x100
+#define CG_SYS_DRM_MGLS_MASK          0x200
+
+// Graphics Clock Gating
+#define CG_GFX_BITMASK_FIRST_BIT      16 // First bit of Gfx CG bitmask
+#define CG_GFX_BITMASK_LAST_BIT       20 // Last bit of Gfx CG bitmask
+#define CG_GFX_CGCG_SHIFT             16
+#define CG_GFX_CGLS_SHIFT             17
+#define CG_CPF_MGCG_SHIFT             18
+#define CG_RLC_MGCG_SHIFT             19
+#define CG_GFX_OTHERS_MGCG_SHIFT      20
+
+#define CG_GFX_CGCG_MASK              0x00010000
+#define CG_GFX_CGLS_MASK              0x00020000
+#define CG_CPF_MGCG_MASK              0x00040000
+#define CG_RLC_MGCG_MASK              0x00080000
+#define CG_GFX_OTHERS_MGCG_MASK       0x00100000
+
+
+
+// Voltage Regulator Configuration
+// VR Config info is contained in dpmTable.VRConfig
+
+#define VRCONF_VDDC_MASK         0x000000FF
+#define VRCONF_VDDC_SHIFT        0
+#define VRCONF_VDDGFX_MASK       0x0000FF00
+#define VRCONF_VDDGFX_SHIFT      8
+#define VRCONF_VDDCI_MASK        0x00FF0000
+#define VRCONF_VDDCI_SHIFT       16
+#define VRCONF_MVDD_MASK         0xFF000000
+#define VRCONF_MVDD_SHIFT        24
+
+#define VR_MERGED_WITH_VDDC      0
+#define VR_SVI2_PLANE_1          1
+#define VR_SVI2_PLANE_2          2
+#define VR_SMIO_PATTERN_1        3
+#define VR_SMIO_PATTERN_2        4
+#define VR_STATIC_VOLTAGE        5
+
+// Clock Stretcher Configuration
+
+#define CLOCK_STRETCHER_MAX_ENTRIES 0x4
+#define CKS_LOOKUPTable_MAX_ENTRIES 0x4
+
+// The 'settings' field is subdivided in the following way:
+#define CLOCK_STRETCHER_SETTING_DDT_MASK             0x01
+#define CLOCK_STRETCHER_SETTING_DDT_SHIFT            0x0
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_MASK  0x1E
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_SHIFT 0x1
+#define CLOCK_STRETCHER_SETTING_ENABLE_MASK          0x80
+#define CLOCK_STRETCHER_SETTING_ENABLE_SHIFT         0x7
+
+struct SMU_ClockStretcherDataTableEntry {
+  uint8_t minVID;
+  uint8_t maxVID;
+
+
+  uint16_t setting;
+};
+typedef struct SMU_ClockStretcherDataTableEntry SMU_ClockStretcherDataTableEntry;
+
+struct SMU_ClockStretcherDataTable {
+  SMU_ClockStretcherDataTableEntry ClockStretcherDataTableEntry[CLOCK_STRETCHER_MAX_ENTRIES];
+};
+typedef struct SMU_ClockStretcherDataTable SMU_ClockStretcherDataTable;
+
+struct SMU_CKS_LOOKUPTableEntry {
+  uint16_t minFreq;
+  uint16_t maxFreq;
+
+  uint8_t setting;
+  uint8_t padding[3];
+};
+typedef struct SMU_CKS_LOOKUPTableEntry SMU_CKS_LOOKUPTableEntry;
+
+struct SMU_CKS_LOOKUPTable {
+  SMU_CKS_LOOKUPTableEntry CKS_LOOKUPTableEntry[CKS_LOOKUPTable_MAX_ENTRIES];
+};
+typedef struct SMU_CKS_LOOKUPTable SMU_CKS_LOOKUPTable;
+
+struct AgmAvfsData_t {
+  uint16_t avgPsmCount[28];
+  uint16_t minPsmCount[28];
+};
+typedef struct AgmAvfsData_t AgmAvfsData_t;
+
+// AVFS DEFINES
+
+enum VFT_COLUMNS {
+  SCLK0,
+  SCLK1,
+  SCLK2,
+  SCLK3,
+  SCLK4,
+  SCLK5,
+  SCLK6,
+  SCLK7,
+
+  NUM_VFT_COLUMNS
+};
+
+#define TEMP_RANGE_MAXSTEPS 12
+struct VFT_CELL_t {
+  uint16_t Voltage;
+};
+
+typedef struct VFT_CELL_t VFT_CELL_t;
+
+struct VFT_TABLE_t {
+  VFT_CELL_t    Cell[TEMP_RANGE_MAXSTEPS][NUM_VFT_COLUMNS];
+  uint16_t      AvfsGbv [NUM_VFT_COLUMNS];
+  uint16_t      BtcGbv  [NUM_VFT_COLUMNS];
+  uint16_t      Temperature [TEMP_RANGE_MAXSTEPS];
+
+  uint8_t       NumTemperatureSteps;
+  uint8_t       padding[3];
+};
+typedef struct VFT_TABLE_t VFT_TABLE_t;
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu73_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu73_discrete.h
new file mode 100644
index 000000000000..5916be08a7fe
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu73_discrete.h
@@ -0,0 +1,799 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _SMU73_DISCRETE_H_
+#define _SMU73_DISCRETE_H_
+
+#include "smu73.h"
+
+#pragma pack(push, 1)
+
+struct SMIO_Pattern
+{
+  uint16_t Voltage;
+  uint8_t  Smio;
+  uint8_t  padding;
+};
+
+typedef struct SMIO_Pattern SMIO_Pattern;
+
+struct SMIO_Table
+{
+  SMIO_Pattern Pattern[SMU_MAX_SMIO_LEVELS];
+};
+
+typedef struct SMIO_Table SMIO_Table;
+
+struct SMU73_Discrete_GraphicsLevel {
+	uint32_t    MinVoltage;
+
+	uint32_t    SclkFrequency;
+
+	uint8_t     pcieDpmLevel;
+	uint8_t     DeepSleepDivId;
+	uint16_t    ActivityLevel;
+	uint32_t    CgSpllFuncCntl3;
+	uint32_t    CgSpllFuncCntl4;
+	uint32_t    SpllSpreadSpectrum;
+	uint32_t    SpllSpreadSpectrum2;
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+	uint8_t     SclkDid;
+	uint8_t     DisplayWatermark;
+	uint8_t     EnabledForActivity;
+	uint8_t     EnabledForThrottle;
+	uint8_t     UpHyst;
+	uint8_t     DownHyst;
+	uint8_t     VoltageDownHyst;
+	uint8_t     PowerThrottle;
+};
+
+typedef struct SMU73_Discrete_GraphicsLevel SMU73_Discrete_GraphicsLevel;
+
+struct SMU73_Discrete_ACPILevel {
+    uint32_t    Flags;
+    uint32_t MinVoltage;
+    uint32_t    SclkFrequency;
+    uint8_t     SclkDid;
+    uint8_t     DisplayWatermark;
+    uint8_t     DeepSleepDivId;
+    uint8_t     padding;
+    uint32_t    CgSpllFuncCntl;
+    uint32_t    CgSpllFuncCntl2;
+    uint32_t    CgSpllFuncCntl3;
+    uint32_t    CgSpllFuncCntl4;
+    uint32_t    SpllSpreadSpectrum;
+    uint32_t    SpllSpreadSpectrum2;
+    uint32_t    CcPwrDynRm;
+    uint32_t    CcPwrDynRm1;
+};
+
+typedef struct SMU73_Discrete_ACPILevel SMU73_Discrete_ACPILevel;
+
+struct SMU73_Discrete_Ulv {
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+	uint16_t    VddcOffset;
+	uint8_t     VddcOffsetVid;
+	uint8_t     VddcPhase;
+	uint32_t    Reserved;
+};
+
+typedef struct SMU73_Discrete_Ulv SMU73_Discrete_Ulv;
+
+struct SMU73_Discrete_MemoryLevel
+{
+    uint32_t MinVoltage;
+    uint32_t    MinMvdd;
+
+    uint32_t    MclkFrequency;
+
+    uint8_t     StutterEnable;
+    uint8_t     FreqRange;
+    uint8_t     EnabledForThrottle;
+    uint8_t     EnabledForActivity;
+
+    uint8_t     UpHyst;
+    uint8_t     DownHyst;
+    uint8_t     VoltageDownHyst;
+    uint8_t     padding;
+
+    uint16_t    ActivityLevel;
+    uint8_t     DisplayWatermark;
+    uint8_t     MclkDivider;
+};
+
+typedef struct SMU73_Discrete_MemoryLevel SMU73_Discrete_MemoryLevel;
+
+struct SMU73_Discrete_LinkLevel
+{
+    uint8_t     PcieGenSpeed;           ///< 0:PciE-gen1 1:PciE-gen2 2:PciE-gen3
+    uint8_t     PcieLaneCount;          ///< 1=x1, 2=x2, 3=x4, 4=x8, 5=x12, 6=x16 
+    uint8_t     EnabledForActivity;
+    uint8_t     SPC;
+    uint32_t    DownThreshold;
+    uint32_t    UpThreshold;
+    uint32_t    Reserved;
+};
+
+typedef struct SMU73_Discrete_LinkLevel SMU73_Discrete_LinkLevel;
+
+
+// MC ARB DRAM Timing registers.
+struct SMU73_Discrete_MCArbDramTimingTableEntry
+{
+    uint32_t McArbDramTiming;
+    uint32_t McArbDramTiming2;
+    uint8_t  McArbBurstTime;
+    uint8_t  TRRDS;
+    uint8_t  TRRDL;
+    uint8_t  padding;
+};
+
+typedef struct SMU73_Discrete_MCArbDramTimingTableEntry SMU73_Discrete_MCArbDramTimingTableEntry;
+
+struct SMU73_Discrete_MCArbDramTimingTable
+{
+    SMU73_Discrete_MCArbDramTimingTableEntry entries[SMU__NUM_SCLK_DPM_STATE][SMU__NUM_MCLK_DPM_LEVELS];
+};
+
+typedef struct SMU73_Discrete_MCArbDramTimingTable SMU73_Discrete_MCArbDramTimingTable;
+
+// UVD VCLK/DCLK state (level) definition.
+struct SMU73_Discrete_UvdLevel
+{
+    uint32_t VclkFrequency;
+    uint32_t DclkFrequency;
+    uint32_t MinVoltage;
+    uint8_t  VclkDivider;
+    uint8_t  DclkDivider;
+    uint8_t  padding[2];
+};
+
+typedef struct SMU73_Discrete_UvdLevel SMU73_Discrete_UvdLevel;
+
+// Clocks for other external blocks (VCE, ACP, SAMU).
+struct SMU73_Discrete_ExtClkLevel
+{
+    uint32_t Frequency;
+    uint32_t MinVoltage;
+    uint8_t  Divider;
+    uint8_t  padding[3];
+};
+
+typedef struct SMU73_Discrete_ExtClkLevel SMU73_Discrete_ExtClkLevel;
+
+struct SMU73_Discrete_StateInfo
+{
+    uint32_t SclkFrequency;
+    uint32_t MclkFrequency;
+    uint32_t VclkFrequency;
+    uint32_t DclkFrequency;
+    uint32_t SamclkFrequency;
+    uint32_t AclkFrequency;
+    uint32_t EclkFrequency;
+    uint16_t MvddVoltage;
+    uint16_t padding16;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+    uint8_t  McRegIndex;
+    uint8_t  SeqIndex;
+    uint8_t  SclkDid;
+    int8_t   SclkIndex;
+    int8_t   MclkIndex;
+    uint8_t  PCIeGen;
+
+};
+
+typedef struct SMU73_Discrete_StateInfo SMU73_Discrete_StateInfo;
+
+struct SMU73_Discrete_DpmTable
+{
+    // Multi-DPM controller settings
+    SMU73_PIDController                  GraphicsPIDController;
+    SMU73_PIDController                  MemoryPIDController;
+    SMU73_PIDController                  LinkPIDController;
+
+    uint32_t                            SystemFlags;
+
+    // SMIO masks for voltage and phase controls
+    uint32_t                            VRConfig;
+    uint32_t                            SmioMask1;
+    uint32_t                            SmioMask2;
+    SMIO_Table                          SmioTable1;
+    SMIO_Table                          SmioTable2;
+
+    uint32_t                            MvddLevelCount;
+
+
+    uint8_t                             BapmVddcVidHiSidd        [SMU73_MAX_LEVELS_VDDC];
+    uint8_t                             BapmVddcVidLoSidd        [SMU73_MAX_LEVELS_VDDC];
+    uint8_t                             BapmVddcVidHiSidd2       [SMU73_MAX_LEVELS_VDDC];
+
+    uint8_t                             GraphicsDpmLevelCount;
+    uint8_t                             MemoryDpmLevelCount;
+    uint8_t                             LinkLevelCount;
+    uint8_t                             MasterDeepSleepControl;
+
+    uint8_t                             UvdLevelCount;
+    uint8_t                             VceLevelCount;
+    uint8_t                             AcpLevelCount;
+    uint8_t                             SamuLevelCount;
+
+    uint8_t                             ThermOutGpio;
+    uint8_t                             ThermOutPolarity;
+    uint8_t                             ThermOutMode;
+    uint8_t                             BootPhases;
+    uint32_t                            Reserved[4];
+
+    // State table entries for each DPM state
+    SMU73_Discrete_GraphicsLevel        GraphicsLevel           [SMU73_MAX_LEVELS_GRAPHICS];
+    SMU73_Discrete_MemoryLevel          MemoryACPILevel;
+    SMU73_Discrete_MemoryLevel          MemoryLevel             [SMU73_MAX_LEVELS_MEMORY];
+    SMU73_Discrete_LinkLevel            LinkLevel               [SMU73_MAX_LEVELS_LINK];
+    SMU73_Discrete_ACPILevel            ACPILevel;
+    SMU73_Discrete_UvdLevel             UvdLevel                [SMU73_MAX_LEVELS_UVD];
+    SMU73_Discrete_ExtClkLevel          VceLevel                [SMU73_MAX_LEVELS_VCE];
+    SMU73_Discrete_ExtClkLevel          AcpLevel                [SMU73_MAX_LEVELS_ACP];
+    SMU73_Discrete_ExtClkLevel          SamuLevel               [SMU73_MAX_LEVELS_SAMU];
+    SMU73_Discrete_Ulv                  Ulv;
+
+    uint32_t                            SclkStepSize;
+    uint32_t                            Smio                    [SMU73_MAX_ENTRIES_SMIO];
+
+    uint8_t                             UvdBootLevel;
+    uint8_t                             VceBootLevel;
+    uint8_t                             AcpBootLevel;
+    uint8_t                             SamuBootLevel;
+
+    uint8_t                             GraphicsBootLevel;
+    uint8_t                             GraphicsVoltageChangeEnable;
+    uint8_t                             GraphicsThermThrottleEnable;
+    uint8_t                             GraphicsInterval;
+
+    uint8_t                             VoltageInterval;
+    uint8_t                             ThermalInterval;
+    uint16_t                            TemperatureLimitHigh;
+
+    uint16_t                            TemperatureLimitLow;
+    uint8_t                             MemoryBootLevel;
+    uint8_t                             MemoryVoltageChangeEnable;
+
+    uint16_t                            BootMVdd;
+    uint8_t                             MemoryInterval;
+    uint8_t                             MemoryThermThrottleEnable;
+
+    uint16_t                            VoltageResponseTime;
+    uint16_t                            PhaseResponseTime;
+
+    uint8_t                             PCIeBootLinkLevel;
+    uint8_t                             PCIeGenInterval;
+    uint8_t                             DTEInterval;
+    uint8_t                             DTEMode;
+
+    uint8_t                             SVI2Enable;
+    uint8_t                             VRHotGpio;
+    uint8_t                             AcDcGpio;
+    uint8_t                             ThermGpio;
+
+    uint16_t                            PPM_PkgPwrLimit;
+    uint16_t                            PPM_TemperatureLimit;
+
+    uint16_t                            DefaultTdp;
+    uint16_t                            TargetTdp;
+
+    uint16_t                            FpsHighThreshold;
+    uint16_t                            FpsLowThreshold;
+
+    uint16_t                            TemperatureLimitEdge;
+    uint16_t                            TemperatureLimitHotspot;
+    uint16_t                            TemperatureLimitLiquid1;
+    uint16_t                            TemperatureLimitLiquid2;
+    uint16_t                            TemperatureLimitVrVddc;
+    uint16_t                            TemperatureLimitVrMvdd;
+    uint16_t                            TemperatureLimitPlx;
+
+    uint16_t                            FanGainEdge;
+    uint16_t                            FanGainHotspot;
+    uint16_t                            FanGainLiquid;
+    uint16_t                            FanGainVrVddc;
+    uint16_t                            FanGainVrMvdd;
+    uint16_t                            FanGainPlx;
+    uint16_t                            FanGainHbm;
+
+    uint8_t                             Liquid1_I2C_address;
+    uint8_t                             Liquid2_I2C_address;
+    uint8_t                             Vr_I2C_address;
+    uint8_t                             Plx_I2C_address;
+
+    uint8_t                             GeminiMode;
+    uint8_t                             spare17[3];
+    uint32_t                            GeminiApertureHigh;
+    uint32_t                            GeminiApertureLow;
+
+    uint8_t                             Liquid_I2C_LineSCL;
+    uint8_t                             Liquid_I2C_LineSDA;
+    uint8_t                             Vr_I2C_LineSCL;
+    uint8_t                             Vr_I2C_LineSDA;
+    uint8_t                             Plx_I2C_LineSCL;
+    uint8_t                             Plx_I2C_LineSDA;
+
+    uint8_t                             spare1253[2];
+    uint32_t                            spare123[2];
+
+    uint8_t                             DTEAmbientTempBase;
+    uint8_t                             DTETjOffset;
+    uint8_t                             GpuTjMax;
+    uint8_t                             GpuTjHyst;
+
+    uint16_t                            BootVddc;
+    uint16_t                            BootVddci;
+
+    uint32_t                            BAPM_TEMP_GRADIENT;
+
+    uint32_t                            LowSclkInterruptThreshold;
+    uint32_t                            VddGfxReChkWait;
+
+    uint8_t                             ClockStretcherAmount;
+    uint8_t                             Sclk_CKS_masterEn0_7;
+    uint8_t                             Sclk_CKS_masterEn8_15;
+    uint8_t                             DPMFreezeAndForced;
+
+    uint8_t                             Sclk_voltageOffset[8];
+
+    SMU_ClockStretcherDataTable         ClockStretcherDataTable;
+    SMU_CKS_LOOKUPTable                 CKS_LOOKUPTable;
+};
+
+typedef struct SMU73_Discrete_DpmTable SMU73_Discrete_DpmTable;
+
+
+// --------------------------------------------------- Fan Table -----------------------------------------------------------
+struct SMU73_Discrete_FanTable
+{
+    uint16_t FdoMode;
+    int16_t  TempMin;
+    int16_t  TempMed;
+    int16_t  TempMax;
+    int16_t  Slope1;
+    int16_t  Slope2;
+    int16_t  FdoMin;
+    int16_t  HystUp;
+    int16_t  HystDown;
+    int16_t  HystSlope;
+    int16_t  TempRespLim;
+    int16_t  TempCurr;
+    int16_t  SlopeCurr;
+    int16_t  PwmCurr;
+    uint32_t RefreshPeriod;
+    int16_t  FdoMax;
+    uint8_t  TempSrc;
+    int8_t   Padding;
+};
+
+typedef struct SMU73_Discrete_FanTable SMU73_Discrete_FanTable;
+
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG             4
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG_BIT         (0x1 << SMU7_DISCRETE_GPIO_SCLK_DEBUG)
+
+
+
+struct SMU7_MclkDpmScoreboard
+{
+
+    uint32_t PercentageBusy;
+
+    int32_t  PIDError;
+    int32_t  PIDIntegral;
+    int32_t  PIDOutput;
+
+    uint32_t SigmaDeltaAccum;
+    uint32_t SigmaDeltaOutput;
+    uint32_t SigmaDeltaLevel;
+
+    uint32_t UtilizationSetpoint;
+
+    uint8_t  TdpClampMode;
+    uint8_t  TdcClampMode;
+    uint8_t  ThermClampMode;
+    uint8_t  VoltageBusy;
+
+    int8_t   CurrLevel;
+    int8_t   TargLevel;
+    uint8_t  LevelChangeInProgress;
+    uint8_t  UpHyst;
+
+    uint8_t  DownHyst;
+    uint8_t  VoltageDownHyst;
+    uint8_t  DpmEnable;
+    uint8_t  DpmRunning;
+
+    uint8_t  DpmForce;
+    uint8_t  DpmForceLevel;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+
+    uint32_t MinimumPerfMclk;
+
+    uint8_t  AcpiReq;
+    uint8_t  AcpiAck;
+    uint8_t  MclkSwitchInProgress;
+    uint8_t  MclkSwitchCritical;
+
+    uint8_t  IgnoreVBlank;
+    uint8_t  TargetMclkIndex;
+    uint8_t  TargetMvddIndex;
+    uint8_t  MclkSwitchResult;
+
+    uint16_t VbiFailureCount;
+    uint8_t  VbiWaitCounter;
+    uint8_t  EnabledLevelsChange;
+
+    uint16_t LevelResidencyCounters [SMU73_MAX_LEVELS_MEMORY];
+    uint16_t LevelSwitchCounters [SMU73_MAX_LEVELS_MEMORY];
+
+    void     (*TargetStateCalculator)(uint8_t);
+    void     (*SavedTargetStateCalculator)(uint8_t);
+
+    uint16_t AutoDpmInterval;
+    uint16_t AutoDpmRange;
+
+    uint16_t VbiTimeoutCount;
+    uint16_t MclkSwitchingTime;
+
+    uint8_t  fastSwitch;
+    uint8_t  Save_PIC_VDDGFX_EXIT;
+    uint8_t  Save_PIC_VDDGFX_ENTER;
+    uint8_t  padding;
+
+};
+
+typedef struct SMU7_MclkDpmScoreboard SMU7_MclkDpmScoreboard;
+
+struct SMU7_UlvScoreboard
+{
+    uint8_t     EnterUlv;
+    uint8_t     ExitUlv;
+    uint8_t     UlvActive;
+    uint8_t     WaitingForUlv;
+    uint8_t     UlvEnable;
+    uint8_t     UlvRunning;
+    uint8_t     UlvMasterEnable;
+    uint8_t     padding;
+    uint32_t    UlvAbortedCount;
+    uint32_t    UlvTimeStamp;
+};
+
+typedef struct SMU7_UlvScoreboard SMU7_UlvScoreboard;
+
+struct VddgfxSavedRegisters
+{
+  uint32_t GPU_DBG[3];
+  uint32_t MEC_BaseAddress_Hi;
+  uint32_t MEC_BaseAddress_Lo;
+  uint32_t THM_TMON0_CTRL2__RDIR_PRESENT;
+  uint32_t THM_TMON1_CTRL2__RDIR_PRESENT;
+  uint32_t CP_INT_CNTL;
+};
+
+typedef struct VddgfxSavedRegisters VddgfxSavedRegisters;
+
+struct SMU7_VddGfxScoreboard
+{
+    uint8_t     VddGfxEnable;
+    uint8_t     VddGfxActive;
+    uint8_t     VPUResetOccured;
+    uint8_t     padding;
+
+    uint32_t    VddGfxEnteredCount;
+    uint32_t    VddGfxAbortedCount;
+
+    uint32_t    VddGfxVid;
+
+    VddgfxSavedRegisters SavedRegisters;
+};
+
+typedef struct SMU7_VddGfxScoreboard SMU7_VddGfxScoreboard;
+
+struct SMU7_TdcLimitScoreboard {
+  uint8_t  Enable;
+  uint8_t  Running;
+  uint16_t Alpha;
+  uint32_t FilteredIddc;
+  uint32_t IddcLimit;
+  uint32_t IddcHyst;
+  SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_TdcLimitScoreboard SMU7_TdcLimitScoreboard;
+
+struct SMU7_PkgPwrLimitScoreboard {
+  uint8_t  Enable;
+  uint8_t  Running;
+  uint16_t Alpha;
+  uint32_t FilteredPkgPwr;
+  uint32_t Limit;
+  uint32_t Hyst;
+  uint32_t LimitFromDriver;
+  SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_PkgPwrLimitScoreboard SMU7_PkgPwrLimitScoreboard;
+
+struct SMU7_BapmScoreboard {
+  uint32_t source_powers[SMU73_DTE_SOURCES];
+  uint32_t source_powers_last[SMU73_DTE_SOURCES];
+  int32_t entity_temperatures[SMU73_NUM_GPU_TES];
+  int32_t initial_entity_temperatures[SMU73_NUM_GPU_TES];
+  int32_t Limit;
+  int32_t Hyst;
+  int32_t therm_influence_coeff_table[SMU73_DTE_ITERATIONS * SMU73_DTE_SOURCES * SMU73_DTE_SINKS * 2];
+  int32_t therm_node_table[SMU73_DTE_ITERATIONS * SMU73_DTE_SOURCES * SMU73_DTE_SINKS];
+  uint16_t ConfigTDPPowerScalar;
+  uint16_t FanSpeedPowerScalar;
+  uint16_t OverDrivePowerScalar;
+  uint16_t OverDriveLimitScalar;
+  uint16_t FinalPowerScalar;
+  uint8_t VariantID;
+  uint8_t spare997;
+
+  SMU7_HystController_Data HystControllerData;
+
+  int32_t temperature_gradient_slope;
+  int32_t temperature_gradient;
+  uint32_t measured_temperature;
+};
+
+
+typedef struct SMU7_BapmScoreboard SMU7_BapmScoreboard;
+
+struct SMU7_AcpiScoreboard {
+  uint32_t SavedInterruptMask[2];
+  uint8_t LastACPIRequest;
+  uint8_t CgBifResp;
+  uint8_t RequestType;
+  uint8_t Padding;
+  SMU73_Discrete_ACPILevel D0Level;
+};
+
+typedef struct SMU7_AcpiScoreboard SMU7_AcpiScoreboard;
+
+struct SMU_QuadraticCoeffs {
+  int32_t m1;
+  uint32_t b;
+
+  int16_t m2;
+  uint8_t m1_shift;
+  uint8_t m2_shift;
+};
+
+typedef struct SMU_QuadraticCoeffs SMU_QuadraticCoeffs;
+
+struct SMU73_Discrete_PmFuses {
+  /* dw0-dw1 */
+  uint8_t BapmVddCVidHiSidd[8];
+
+  /* dw2-dw3 */
+  uint8_t BapmVddCVidLoSidd[8];
+
+  /* dw4-dw5 */
+  uint8_t VddCVid[8];
+
+  /* dw1*/
+  uint8_t SviLoadLineEn;
+  uint8_t SviLoadLineVddC;
+  uint8_t SviLoadLineTrimVddC;
+  uint8_t SviLoadLineOffsetVddC;
+
+  /* dw2 */
+  uint16_t TDC_VDDC_PkgLimit;
+  uint8_t TDC_VDDC_ThrottleReleaseLimitPerc;
+  uint8_t TDC_MAWt;
+
+  /* dw3 */
+  uint8_t TdcWaterfallCtl;
+  uint8_t LPMLTemperatureMin;
+  uint8_t LPMLTemperatureMax;
+  uint8_t Reserved;
+
+  /* dw4-dw7 */
+  uint8_t LPMLTemperatureScaler[16];
+
+  /* dw8-dw9 */
+  int16_t FuzzyFan_ErrorSetDelta;
+  int16_t FuzzyFan_ErrorRateSetDelta;
+  int16_t FuzzyFan_PwmSetDelta;
+  uint16_t Reserved6;
+
+  /* dw10-dw14 */
+  uint8_t GnbLPML[16];
+
+  /* dw15 */
+  uint8_t GnbLPMLMaxVid;
+  uint8_t GnbLPMLMinVid;
+  uint8_t Reserved1[2];
+
+  /* dw16 */
+  uint16_t BapmVddCBaseLeakageHiSidd;
+  uint16_t BapmVddCBaseLeakageLoSidd;
+
+  /* AVFS */
+  uint16_t  VFT_Temp[3];
+  uint16_t  padding;
+
+  SMU_QuadraticCoeffs VFT_ATE[3];
+
+  SMU_QuadraticCoeffs AVFS_GB;
+  SMU_QuadraticCoeffs ATE_ACBTC_GB;
+
+  SMU_QuadraticCoeffs P2V;
+
+  uint32_t PsmCharzFreq;
+
+  uint16_t InversionVoltage;
+  uint16_t PsmCharzTemp;
+
+  uint32_t EnabledAvfsModules;
+};
+
+typedef struct SMU73_Discrete_PmFuses SMU73_Discrete_PmFuses;
+
+struct SMU7_Discrete_Log_Header_Table {
+  uint32_t    version;
+  uint32_t    asic_id;
+  uint16_t    flags;
+  uint16_t    entry_size;
+  uint32_t    total_size;
+  uint32_t    num_of_entries;
+  uint8_t     type;
+  uint8_t     mode;
+  uint8_t     filler_0[2];
+  uint32_t    filler_1[2];
+};
+
+typedef struct SMU7_Discrete_Log_Header_Table SMU7_Discrete_Log_Header_Table;
+
+struct SMU7_Discrete_Log_Cntl {
+    uint8_t             Enabled;
+    uint8_t             Type;
+    uint8_t             padding[2];
+    uint32_t            BufferSize;
+    uint32_t            SamplesLogged;
+    uint32_t            SampleSize;
+    uint32_t            AddrL;
+    uint32_t            AddrH;
+};
+
+typedef struct SMU7_Discrete_Log_Cntl SMU7_Discrete_Log_Cntl;
+
+#define CAC_ACC_NW_NUM_OF_SIGNALS 87
+
+struct SMU7_Discrete_Cac_Collection_Table {
+  uint32_t temperature;
+  uint32_t cac_acc_nw[CAC_ACC_NW_NUM_OF_SIGNALS];
+};
+
+typedef struct SMU7_Discrete_Cac_Collection_Table SMU7_Discrete_Cac_Collection_Table;
+
+struct SMU7_Discrete_Cac_Verification_Table {
+  uint32_t VddcTotalPower;
+  uint32_t VddcLeakagePower;
+  uint32_t VddcConstantPower;
+  uint32_t VddcGfxDynamicPower;
+  uint32_t VddcUvdDynamicPower;
+  uint32_t VddcVceDynamicPower;
+  uint32_t VddcAcpDynamicPower;
+  uint32_t VddcPcieDynamicPower;
+  uint32_t VddcDceDynamicPower;
+  uint32_t VddcCurrent;
+  uint32_t VddcVoltage;
+  uint32_t VddciTotalPower;
+  uint32_t VddciLeakagePower;
+  uint32_t VddciConstantPower;
+  uint32_t VddciDynamicPower;
+  uint32_t Vddr1TotalPower;
+  uint32_t Vddr1LeakagePower;
+  uint32_t Vddr1ConstantPower;
+  uint32_t Vddr1DynamicPower;
+  uint32_t spare[4];
+  uint32_t temperature;
+};
+
+typedef struct SMU7_Discrete_Cac_Verification_Table SMU7_Discrete_Cac_Verification_Table;
+
+struct SMU7_Discrete_Pm_Status_Table {
+  //Thermal entities
+  int32_t  T_meas_max[SMU73_THERMAL_INPUT_LOOP_COUNT];
+  int32_t  T_meas_acc[SMU73_THERMAL_INPUT_LOOP_COUNT];
+  int32_t  T_meas_acc_cnt[SMU73_THERMAL_INPUT_LOOP_COUNT];
+  uint32_t T_hbm_acc;
+
+  //Voltage domains
+  uint32_t I_calc_max;
+  uint32_t I_calc_acc;
+  uint32_t P_meas_acc;
+  uint32_t V_meas_load_acc;
+  uint32_t I_meas_acc;
+  uint32_t P_meas_acc_vddci;
+  uint32_t V_meas_load_acc_vddci;
+  uint32_t I_meas_acc_vddci;
+
+  //Frequency
+  uint16_t Sclk_dpm_residency[8];
+  uint16_t Uvd_dpm_residency[8];
+  uint16_t Vce_dpm_residency[8];
+
+  //Chip
+  uint32_t P_roc_acc;
+  uint32_t PkgPwr_max;
+  uint32_t PkgPwr_acc;
+  uint32_t MclkSwitchingTime_max;
+  uint32_t MclkSwitchingTime_acc;
+  uint32_t FanPwm_acc;
+  uint32_t FanRpm_acc;
+  uint32_t Gfx_busy_acc;
+  uint32_t Mc_busy_acc;
+  uint32_t Fps_acc;
+
+  uint32_t AccCnt;
+};
+
+typedef struct SMU7_Discrete_Pm_Status_Table SMU7_Discrete_Pm_Status_Table;
+
+//FIXME THESE NEED TO BE UPDATED
+#define SMU7_SCLK_CAC 0x561
+#define SMU7_MCLK_CAC 0xF9
+#define SMU7_VCLK_CAC 0x2DE
+#define SMU7_DCLK_CAC 0x2DE
+#define SMU7_ECLK_CAC 0x25E
+#define SMU7_ACLK_CAC 0x25E
+#define SMU7_SAMCLK_CAC 0x25E
+#define SMU7_DISPCLK_CAC 0x100
+#define SMU7_CAC_CONSTANT 0x2EE3430
+#define SMU7_CAC_CONSTANT_SHIFT 18
+
+#define SMU7_VDDCI_MCLK_CONST        1765
+#define SMU7_VDDCI_MCLK_CONST_SHIFT  16
+#define SMU7_VDDCI_VDDCI_CONST       50958
+#define SMU7_VDDCI_VDDCI_CONST_SHIFT 14
+#define SMU7_VDDCI_CONST             11781
+#define SMU7_VDDCI_STROBE_PWR        1331
+
+#define SMU7_VDDR1_CONST            693
+#define SMU7_VDDR1_CAC_WEIGHT       20
+#define SMU7_VDDR1_CAC_WEIGHT_SHIFT 19
+#define SMU7_VDDR1_STROBE_PWR       512
+
+#define SMU7_AREA_COEFF_UVD 0xA78
+#define SMU7_AREA_COEFF_VCE 0x190A
+#define SMU7_AREA_COEFF_ACP 0x22D1
+#define SMU7_AREA_COEFF_SAMU 0x534
+
+//ThermOutMode values
+#define SMU7_THERM_OUT_MODE_DISABLE       0x0
+#define SMU7_THERM_OUT_MODE_THERM_ONLY    0x1
+#define SMU7_THERM_OUT_MODE_THERM_VRHOT   0x2
+
+#pragma pack(pop)
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/smu7_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu7_discrete.h
index 0b0b404ff091..0b0b404ff091 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu7_discrete.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7_discrete.h
diff --git a/drivers/gpu/drm/amd/amdgpu/smu7_fusion.h b/drivers/gpu/drm/amd/powerplay/inc/smu7_fusion.h
index 78ada9ffd508..78ada9ffd508 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu7_fusion.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7_fusion.h
diff --git a/drivers/gpu/drm/amd/amdgpu/smu8.h b/drivers/gpu/drm/amd/powerplay/inc/smu8.h
index d758d07b6a31..d758d07b6a31 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu8.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu8.h
diff --git a/drivers/gpu/drm/amd/amdgpu/smu8_fusion.h b/drivers/gpu/drm/amd/powerplay/inc/smu8_fusion.h
index 5c9cc3c0bbfa..0c37c94e9414 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu8_fusion.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu8_fusion.h
@@ -48,6 +48,14 @@ struct SMU8_Port80MonitorTable {
 	uint8_t  EnableDramShadow;
 };
 
+/*  Display specific power management parameters */
+#define PWRMGT_SEPARATION_TIME_SHIFT            0
+#define PWRMGT_SEPARATION_TIME_MASK             0xFFFF
+#define PWRMGT_DISABLE_CPU_CSTATES_SHIFT        16
+#define PWRMGT_DISABLE_CPU_CSTATES_MASK         0x1
+#define PWRMGT_DISABLE_CPU_PSTATES_SHIFT        24
+#define PWRMGT_DISABLE_CPU_PSTATES_MASK         0x1
+
 /* Clock Table Definitions */
 #define NUM_SCLK_LEVELS     8
 #define NUM_LCLK_LEVELS     8
diff --git a/drivers/gpu/drm/amd/amdgpu/smu_ucode_xfer_cz.h b/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_cz.h
index f8ba071f39c8..f8ba071f39c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/smu_ucode_xfer_cz.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_cz.h
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_vi.h b/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_vi.h
new file mode 100644
index 000000000000..c24a81eebc7c
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_vi.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU_UCODE_XFER_VI_H
+#define SMU_UCODE_XFER_VI_H
+
+#define SMU_DRAMData_TOC_VERSION  1
+#define MAX_IH_REGISTER_COUNT     65535
+#define SMU_DIGEST_SIZE_BYTES     20
+#define SMU_FB_SIZE_BYTES         1048576
+#define SMU_MAX_ENTRIES           12
+
+#define UCODE_ID_SMU              0
+#define UCODE_ID_SDMA0            1
+#define UCODE_ID_SDMA1            2
+#define UCODE_ID_CP_CE            3
+#define UCODE_ID_CP_PFP           4
+#define UCODE_ID_CP_ME            5
+#define UCODE_ID_CP_MEC           6
+#define UCODE_ID_CP_MEC_JT1       7
+#define UCODE_ID_CP_MEC_JT2       8
+#define UCODE_ID_GMCON_RENG       9
+#define UCODE_ID_RLC_G            10
+#define UCODE_ID_IH_REG_RESTORE   11
+#define UCODE_ID_VBIOS            12
+#define UCODE_ID_MISC_METADATA    13
+#define UCODE_ID_RLC_SCRATCH      32
+#define UCODE_ID_RLC_SRM_ARAM     33
+#define UCODE_ID_RLC_SRM_DRAM     34
+#define UCODE_ID_MEC_STORAGE      35
+#define UCODE_ID_VBIOS_PARAMETERS 36
+#define UCODE_META_DATA           0xFF
+
+#define UCODE_ID_SMU_MASK             0x00000001
+#define UCODE_ID_SDMA0_MASK           0x00000002
+#define UCODE_ID_SDMA1_MASK           0x00000004
+#define UCODE_ID_CP_CE_MASK           0x00000008
+#define UCODE_ID_CP_PFP_MASK          0x00000010
+#define UCODE_ID_CP_ME_MASK           0x00000020
+#define UCODE_ID_CP_MEC_MASK          0x00000040
+#define UCODE_ID_CP_MEC_JT1_MASK      0x00000080
+#define UCODE_ID_CP_MEC_JT2_MASK      0x00000100
+#define UCODE_ID_GMCON_RENG_MASK      0x00000200
+#define UCODE_ID_RLC_G_MASK           0x00000400
+#define UCODE_ID_IH_REG_RESTORE_MASK  0x00000800
+#define UCODE_ID_VBIOS_MASK           0x00001000
+
+#define UCODE_FLAG_UNHALT_MASK   0x1
+
+struct SMU_Entry {
+#ifndef __BIG_ENDIAN
+	uint16_t id;
+	uint16_t version;
+	uint32_t image_addr_high;
+	uint32_t image_addr_low;
+	uint32_t meta_data_addr_high;
+	uint32_t meta_data_addr_low;
+	uint32_t data_size_byte;
+	uint16_t flags;
+	uint16_t num_register_entries;
+#else
+	uint16_t version;
+	uint16_t id;
+	uint32_t image_addr_high;
+	uint32_t image_addr_low;
+	uint32_t meta_data_addr_high;
+	uint32_t meta_data_addr_low;
+	uint32_t data_size_byte;
+	uint16_t num_register_entries;
+	uint16_t flags;
+#endif
+};
+
+struct SMU_DRAMData_TOC {
+	uint32_t structure_version;
+	uint32_t num_entries;
+	struct SMU_Entry entry[SMU_MAX_ENTRIES];
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
new file mode 100644
index 000000000000..504f035d1843
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _SMUMGR_H_
+#define _SMUMGR_H_
+#include <linux/types.h>
+#include "pp_instance.h"
+#include "amd_powerplay.h"
+
+struct pp_smumgr;
+struct pp_instance;
+
+#define smu_lower_32_bits(n) ((uint32_t)(n))
+#define smu_upper_32_bits(n) ((uint32_t)(((n)>>16)>>16))
+
+struct pp_smumgr_func {
+	int (*smu_init)(struct pp_smumgr *smumgr);
+	int (*smu_fini)(struct pp_smumgr *smumgr);
+	int (*start_smu)(struct pp_smumgr *smumgr);
+	int (*check_fw_load_finish)(struct pp_smumgr *smumgr,
+				    uint32_t firmware);
+	int (*request_smu_load_fw)(struct pp_smumgr *smumgr);
+	int (*request_smu_load_specific_fw)(struct pp_smumgr *smumgr,
+					    uint32_t firmware);
+	int (*get_argument)(struct pp_smumgr *smumgr);
+	int (*send_msg_to_smc)(struct pp_smumgr *smumgr, uint16_t msg);
+	int (*send_msg_to_smc_with_parameter)(struct pp_smumgr *smumgr,
+					  uint16_t msg, uint32_t parameter);
+	int (*download_pptable_settings)(struct pp_smumgr *smumgr,
+					 void **table);
+	int (*upload_pptable_settings)(struct pp_smumgr *smumgr);
+};
+
+struct pp_smumgr {
+	uint32_t chip_family;
+	uint32_t chip_id;
+	uint32_t hw_revision;
+	void *device;
+	void *backend;
+	uint32_t usec_timeout;
+	bool reload_fw;
+	const struct pp_smumgr_func *smumgr_funcs;
+};
+
+
+extern int smum_init(struct amd_pp_init *pp_init,
+		     struct pp_instance *handle);
+
+extern int smum_fini(struct pp_smumgr *smumgr);
+
+extern int smum_get_argument(struct pp_smumgr *smumgr);
+
+extern int smum_download_powerplay_table(struct pp_smumgr *smumgr, void **table);
+
+extern int smum_upload_powerplay_table(struct pp_smumgr *smumgr);
+
+extern int smum_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg);
+
+extern int smum_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
+					uint16_t msg, uint32_t parameter);
+
+extern int smum_wait_on_register(struct pp_smumgr *smumgr,
+				uint32_t index, uint32_t value, uint32_t mask);
+
+extern int smum_wait_for_register_unequal(struct pp_smumgr *smumgr,
+				uint32_t index, uint32_t value, uint32_t mask);
+
+extern int smum_wait_on_indirect_register(struct pp_smumgr *smumgr,
+				uint32_t indirect_port, uint32_t index,
+				uint32_t value, uint32_t mask);
+
+
+extern void smum_wait_for_indirect_register_unequal(
+				struct pp_smumgr *smumgr,
+				uint32_t indirect_port, uint32_t index,
+				uint32_t value, uint32_t mask);
+
+extern int smu_allocate_memory(void *device, uint32_t size,
+			 enum cgs_gpu_mem_type type,
+			 uint32_t byte_align, uint64_t *mc_addr,
+			 void **kptr, void *handle);
+
+extern int smu_free_memory(void *device, void *handle);
+
+#define SMUM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
+
+#define SMUM_FIELD_MASK(reg, field) reg##__##field##_MASK
+
+#define SMUM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(smumgr,			\
+					port, index, value, mask)	\
+	smum_wait_on_indirect_register(smumgr,				\
+				mm##port##_INDEX, index, value, mask)
+
+
+#define SMUM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr,         \
+							index, value, mask) \
+		smum_wait_for_register_unequal(smumgr,            \
+					index, value, mask)
+
+#define SMUM_WAIT_REGISTER_UNEQUAL(smumgr, reg, value, mask)		\
+	SMUM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr,			\
+				mm##reg, value, mask)
+
+#define SMUM_WAIT_FIELD_UNEQUAL(smumgr, reg, field, fieldval)		\
+	SMUM_WAIT_REGISTER_UNEQUAL(smumgr, reg,				\
+		(fieldval) << SMUM_FIELD_SHIFT(reg, field),		\
+		SMUM_FIELD_MASK(reg, field))
+
+#define SMUM_GET_FIELD(value, reg, field)				\
+		(((value) & SMUM_FIELD_MASK(reg, field))		\
+		>> SMUM_FIELD_SHIFT(reg, field))
+
+#define SMUM_READ_FIELD(device, reg, field)                           \
+		SMUM_GET_FIELD(cgs_read_register(device, mm##reg), reg, field)
+
+#define SMUM_SET_FIELD(value, reg, field, field_val)                  \
+		(((value) & ~SMUM_FIELD_MASK(reg, field)) |                    \
+		(SMUM_FIELD_MASK(reg, field) & ((field_val) <<                 \
+			SMUM_FIELD_SHIFT(reg, field))))
+
+#define SMUM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(smumgr,		\
+				port, index, value, mask)		\
+	smum_wait_on_indirect_register(smumgr,				\
+		mm##port##_INDEX_0, index, value, mask)
+
+#define SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr,	\
+				port, index, value, mask)		\
+	smum_wait_for_indirect_register_unequal(smumgr,			\
+		mm##port##_INDEX_0, index, value, mask)
+
+
+#define SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, port, reg, value, mask) \
+	SMUM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(smumgr, port, ix##reg, value, mask)
+
+#define SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, value, mask)     \
+		SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, port, ix##reg, value, mask)
+
+
+/*Operations on named fields.*/
+
+#define SMUM_READ_VFPF_INDIRECT_FIELD(device, port, reg, field) \
+		SMUM_GET_FIELD(cgs_read_ind_register(device, port, ix##reg), \
+			reg, field)
+
+#define SMUM_WRITE_FIELD(device, reg, field, fieldval)            \
+		cgs_write_register(device, mm##reg, \
+		SMUM_SET_FIELD(cgs_read_register(device, mm##reg), reg, field, fieldval))
+
+#define SMUM_WRITE_VFPF_INDIRECT_FIELD(device, port, reg, field, fieldval)    \
+		cgs_write_ind_register(device, port, ix##reg, \
+			SMUM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg), \
+			reg, field, fieldval))
+
+#define SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, port, reg, field, fieldval) \
+	SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, port, reg,		\
+		(fieldval) << SMUM_FIELD_SHIFT(reg, field),		\
+		SMUM_FIELD_MASK(reg, field))
+
+#define SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, port, reg, field, fieldval) \
+	SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg,	\
+		(fieldval) << SMUM_FIELD_SHIFT(reg, field),		\
+		SMUM_FIELD_MASK(reg, field))
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/tonga_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/tonga_ppsmc.h
new file mode 100644
index 000000000000..63631296d751
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/tonga_ppsmc.h
@@ -0,0 +1,420 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef TONGA_PP_SMC_H
+#define TONGA_PP_SMC_H
+
+#pragma pack(push, 1)
+
+#define PPSMC_SWSTATE_FLAG_DC				0x01
+#define PPSMC_SWSTATE_FLAG_UVD				0x02
+#define PPSMC_SWSTATE_FLAG_VCE				0x04
+#define PPSMC_SWSTATE_FLAG_PCIE_X1			0x08
+
+#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
+#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
+#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff
+
+#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01
+#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02
+#define PPSMC_SYSTEMFLAG_GDDR5                          0x04
+
+#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08
+
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20
+#define PPSMC_SYSTEMFLAG_12CHANNEL                      0x40
+
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
+#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
+
+#define PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH      0x10
+#define PPSMC_EXTRAFLAGS_DRIVER_TO_GPIO17               0x20
+#define PPSMC_EXTRAFLAGS_PCC_TO_GPIO17                  0x40
+
+/* Defines for DPM 2.0 */
+#define PPSMC_DPM2FLAGS_TDPCLMP                         0x01
+#define PPSMC_DPM2FLAGS_PWRSHFT                         0x02
+#define PPSMC_DPM2FLAGS_OCP                             0x04
+
+/* Defines for display watermark level */
+
+#define PPSMC_DISPLAY_WATERMARK_LOW                     0
+#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
+
+/* In the HW performance level's state flags:*/
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    0x01
+#define PPSMC_STATEFLAG_POWERBOOST         0x02
+#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 0x04
+#define PPSMC_STATEFLAG_POWERSHIFT         0x08
+#define PPSMC_STATEFLAG_SLOW_READ_MARGIN   0x10
+#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20
+#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   0x40
+
+/* Fan control algorithm:*/
+#define FDO_MODE_HARDWARE 0
+#define FDO_MODE_PIECE_WISE_LINEAR 1
+
+enum FAN_CONTROL {
+	FAN_CONTROL_FUZZY,
+	FAN_CONTROL_TABLE
+};
+
+/* Return codes for driver to SMC communication.*/
+
+#define PPSMC_Result_OK             ((uint16_t)0x01)
+#define PPSMC_Result_NoMore         ((uint16_t)0x02)
+#define PPSMC_Result_NotNow         ((uint16_t)0x03)
+
+#define PPSMC_Result_Failed         ((uint16_t)0xFF)
+#define PPSMC_Result_UnknownCmd     ((uint16_t)0xFE)
+#define PPSMC_Result_UnknownVT      ((uint16_t)0xFD)
+
+typedef uint16_t PPSMC_Result;
+
+#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
+
+
+#define PPSMC_MSG_Halt                      ((uint16_t)0x10)
+#define PPSMC_MSG_Resume                    ((uint16_t)0x11)
+#define PPSMC_MSG_EnableDPMLevel            ((uint16_t)0x12)
+#define PPSMC_MSG_ZeroLevelsDisabled        ((uint16_t)0x13)
+#define PPSMC_MSG_OneLevelsDisabled         ((uint16_t)0x14)
+#define PPSMC_MSG_TwoLevelsDisabled         ((uint16_t)0x15)
+#define PPSMC_MSG_EnableThermalInterrupt    ((uint16_t)0x16)
+#define PPSMC_MSG_RunningOnAC               ((uint16_t)0x17)
+#define PPSMC_MSG_LevelUp                   ((uint16_t)0x18)
+#define PPSMC_MSG_LevelDown                 ((uint16_t)0x19)
+#define PPSMC_MSG_ResetDPMCounters          ((uint16_t)0x1a)
+#define PPSMC_MSG_SwitchToSwState           ((uint16_t)0x20)
+
+#define PPSMC_MSG_SwitchToSwStateLast       ((uint16_t)0x3f)
+#define PPSMC_MSG_SwitchToInitialState      ((uint16_t)0x40)
+#define PPSMC_MSG_NoForcedLevel             ((uint16_t)0x41)
+#define PPSMC_MSG_ForceHigh                 ((uint16_t)0x42)
+#define PPSMC_MSG_ForceMediumOrHigh         ((uint16_t)0x43)
+
+#define PPSMC_MSG_SwitchToMinimumPower      ((uint16_t)0x51)
+#define PPSMC_MSG_ResumeFromMinimumPower    ((uint16_t)0x52)
+#define PPSMC_MSG_EnableCac                 ((uint16_t)0x53)
+#define PPSMC_MSG_DisableCac                ((uint16_t)0x54)
+#define PPSMC_DPMStateHistoryStart          ((uint16_t)0x55)
+#define PPSMC_DPMStateHistoryStop           ((uint16_t)0x56)
+#define PPSMC_CACHistoryStart               ((uint16_t)0x57)
+#define PPSMC_CACHistoryStop                ((uint16_t)0x58)
+#define PPSMC_TDPClampingActive             ((uint16_t)0x59)
+#define PPSMC_TDPClampingInactive           ((uint16_t)0x5A)
+#define PPSMC_StartFanControl               ((uint16_t)0x5B)
+#define PPSMC_StopFanControl                ((uint16_t)0x5C)
+#define PPSMC_NoDisplay                     ((uint16_t)0x5D)
+#define PPSMC_HasDisplay                    ((uint16_t)0x5E)
+#define PPSMC_MSG_UVDPowerOFF               ((uint16_t)0x60)
+#define PPSMC_MSG_UVDPowerON                ((uint16_t)0x61)
+#define PPSMC_MSG_EnableULV                 ((uint16_t)0x62)
+#define PPSMC_MSG_DisableULV                ((uint16_t)0x63)
+#define PPSMC_MSG_EnterULV                  ((uint16_t)0x64)
+#define PPSMC_MSG_ExitULV                   ((uint16_t)0x65)
+#define PPSMC_PowerShiftActive              ((uint16_t)0x6A)
+#define PPSMC_PowerShiftInactive            ((uint16_t)0x6B)
+#define PPSMC_OCPActive                     ((uint16_t)0x6C)
+#define PPSMC_OCPInactive                   ((uint16_t)0x6D)
+#define PPSMC_CACLongTermAvgEnable          ((uint16_t)0x6E)
+#define PPSMC_CACLongTermAvgDisable         ((uint16_t)0x6F)
+#define PPSMC_MSG_InferredStateSweep_Start  ((uint16_t)0x70)
+#define PPSMC_MSG_InferredStateSweep_Stop   ((uint16_t)0x71)
+#define PPSMC_MSG_SwitchToLowestInfState    ((uint16_t)0x72)
+#define PPSMC_MSG_SwitchToNonInfState       ((uint16_t)0x73)
+#define PPSMC_MSG_AllStateSweep_Start       ((uint16_t)0x74)
+#define PPSMC_MSG_AllStateSweep_Stop        ((uint16_t)0x75)
+#define PPSMC_MSG_SwitchNextLowerInfState   ((uint16_t)0x76)
+#define PPSMC_MSG_SwitchNextHigherInfState  ((uint16_t)0x77)
+#define PPSMC_MSG_MclkRetrainingTest        ((uint16_t)0x78)
+#define PPSMC_MSG_ForceTDPClamping          ((uint16_t)0x79)
+#define PPSMC_MSG_CollectCAC_PowerCorreln   ((uint16_t)0x7A)
+#define PPSMC_MSG_CollectCAC_WeightCalib    ((uint16_t)0x7B)
+#define PPSMC_MSG_CollectCAC_SQonly         ((uint16_t)0x7C)
+#define PPSMC_MSG_CollectCAC_TemperaturePwr ((uint16_t)0x7D)
+
+#define PPSMC_MSG_ExtremitiesTest_Start     ((uint16_t)0x7E)
+#define PPSMC_MSG_ExtremitiesTest_Stop      ((uint16_t)0x7F)
+#define PPSMC_FlushDataCache                ((uint16_t)0x80)
+#define PPSMC_FlushInstrCache               ((uint16_t)0x81)
+
+#define PPSMC_MSG_SetEnabledLevels          ((uint16_t)0x82)
+#define PPSMC_MSG_SetForcedLevels           ((uint16_t)0x83)
+
+#define PPSMC_MSG_ResetToDefaults           ((uint16_t)0x84)
+
+#define PPSMC_MSG_SetForcedLevelsAndJump    ((uint16_t)0x85)
+#define PPSMC_MSG_SetCACHistoryMode         ((uint16_t)0x86)
+#define PPSMC_MSG_EnableDTE                 ((uint16_t)0x87)
+#define PPSMC_MSG_DisableDTE                ((uint16_t)0x88)
+
+#define PPSMC_MSG_SmcSpaceSetAddress        ((uint16_t)0x89)
+#define PPSMC_MSG_ChangeNearTDPLimit        ((uint16_t)0x90)
+#define PPSMC_MSG_ChangeSafePowerLimit      ((uint16_t)0x91)
+
+#define PPSMC_MSG_DPMStateSweepStart        ((uint16_t)0x92)
+#define PPSMC_MSG_DPMStateSweepStop         ((uint16_t)0x93)
+
+#define PPSMC_MSG_OVRDDisableSCLKDS         ((uint16_t)0x94)
+#define PPSMC_MSG_CancelDisableOVRDSCLKDS   ((uint16_t)0x95)
+#define PPSMC_MSG_ThrottleOVRDSCLKDS        ((uint16_t)0x96)
+#define PPSMC_MSG_CancelThrottleOVRDSCLKDS  ((uint16_t)0x97)
+#define PPSMC_MSG_GPIO17					((uint16_t)0x98)
+
+#define PPSMC_MSG_API_SetSvi2Volt_Vddc      ((uint16_t)0x99)
+#define PPSMC_MSG_API_SetSvi2Volt_Vddci     ((uint16_t)0x9A)
+#define PPSMC_MSG_API_SetSvi2Volt_Mvdd      ((uint16_t)0x9B)
+#define PPSMC_MSG_API_GetSvi2Volt_Vddc      ((uint16_t)0x9C)
+#define PPSMC_MSG_API_GetSvi2Volt_Vddci     ((uint16_t)0x9D)
+#define PPSMC_MSG_API_GetSvi2Volt_Mvdd      ((uint16_t)0x9E)
+
+#define PPSMC_MSG_BREAK                     ((uint16_t)0xF8)
+
+/* Trinity Specific Messages*/
+#define PPSMC_MSG_Test                      ((uint16_t) 0x100)
+#define PPSMC_MSG_DPM_Voltage_Pwrmgt        ((uint16_t) 0x101)
+#define PPSMC_MSG_DPM_Config                ((uint16_t) 0x102)
+#define PPSMC_MSG_PM_Controller_Start       ((uint16_t) 0x103)
+#define PPSMC_MSG_DPM_ForceState            ((uint16_t) 0x104)
+#define PPSMC_MSG_PG_PowerDownSIMD          ((uint16_t) 0x105)
+#define PPSMC_MSG_PG_PowerUpSIMD            ((uint16_t) 0x106)
+#define PPSMC_MSG_PM_Controller_Stop        ((uint16_t) 0x107)
+#define PPSMC_MSG_PG_SIMD_Config            ((uint16_t) 0x108)
+#define PPSMC_MSG_Voltage_Cntl_Enable       ((uint16_t) 0x109)
+#define PPSMC_MSG_Thermal_Cntl_Enable       ((uint16_t) 0x10a)
+#define PPSMC_MSG_Reset_Service             ((uint16_t) 0x10b)
+#define PPSMC_MSG_VCEPowerOFF               ((uint16_t) 0x10e)
+#define PPSMC_MSG_VCEPowerON                ((uint16_t) 0x10f)
+#define PPSMC_MSG_DPM_Disable_VCE_HS        ((uint16_t) 0x110)
+#define PPSMC_MSG_DPM_Enable_VCE_HS         ((uint16_t) 0x111)
+#define PPSMC_MSG_DPM_N_LevelsDisabled      ((uint16_t) 0x112)
+#define PPSMC_MSG_DCEPowerOFF               ((uint16_t) 0x113)
+#define PPSMC_MSG_DCEPowerON                ((uint16_t) 0x114)
+#define PPSMC_MSG_PCIE_DDIPowerDown         ((uint16_t) 0x117)
+#define PPSMC_MSG_PCIE_DDIPowerUp           ((uint16_t) 0x118)
+#define PPSMC_MSG_PCIE_CascadePLLPowerDown  ((uint16_t) 0x119)
+#define PPSMC_MSG_PCIE_CascadePLLPowerUp    ((uint16_t) 0x11a)
+#define PPSMC_MSG_SYSPLLPowerOff            ((uint16_t) 0x11b)
+#define PPSMC_MSG_SYSPLLPowerOn             ((uint16_t) 0x11c)
+#define PPSMC_MSG_DCE_RemoveVoltageAdjustment   ((uint16_t) 0x11d)
+#define PPSMC_MSG_DCE_AllowVoltageAdjustment    ((uint16_t) 0x11e)
+#define PPSMC_MSG_DISPLAYPHYStatusNotify    ((uint16_t) 0x11f)
+#define PPSMC_MSG_EnableBAPM                ((uint16_t) 0x120)
+#define PPSMC_MSG_DisableBAPM               ((uint16_t) 0x121)
+#define PPSMC_MSG_PCIE_PHYPowerDown         ((uint16_t) 0x122)
+#define PPSMC_MSG_PCIE_PHYPowerUp           ((uint16_t) 0x123)
+#define PPSMC_MSG_UVD_DPM_Config            ((uint16_t) 0x124)
+#define PPSMC_MSG_Spmi_Enable               ((uint16_t) 0x122)
+#define PPSMC_MSG_Spmi_Timer                ((uint16_t) 0x123)
+#define PPSMC_MSG_LCLK_DPM_Config           ((uint16_t) 0x124)
+#define PPSMC_MSG_NBDPM_Config             ((uint16_t) 0x125)
+#define PPSMC_MSG_PCIE_DDIPhyPowerDown           ((uint16_t) 0x126)
+#define PPSMC_MSG_PCIE_DDIPhyPowerUp             ((uint16_t) 0x127)
+#define PPSMC_MSG_MCLKDPM_Config                ((uint16_t) 0x128)
+
+#define PPSMC_MSG_UVDDPM_Config               ((uint16_t) 0x129)
+#define PPSMC_MSG_VCEDPM_Config               ((uint16_t) 0x12A)
+#define PPSMC_MSG_ACPDPM_Config               ((uint16_t) 0x12B)
+#define PPSMC_MSG_SAMUDPM_Config              ((uint16_t) 0x12C)
+#define PPSMC_MSG_UVDDPM_SetEnabledMask       ((uint16_t) 0x12D)
+#define PPSMC_MSG_VCEDPM_SetEnabledMask       ((uint16_t) 0x12E)
+#define PPSMC_MSG_ACPDPM_SetEnabledMask       ((uint16_t) 0x12F)
+#define PPSMC_MSG_SAMUDPM_SetEnabledMask      ((uint16_t) 0x130)
+#define PPSMC_MSG_MCLKDPM_ForceState          ((uint16_t) 0x131)
+#define PPSMC_MSG_MCLKDPM_NoForcedLevel       ((uint16_t) 0x132)
+#define PPSMC_MSG_Thermal_Cntl_Disable        ((uint16_t) 0x133)
+#define PPSMC_MSG_SetTDPLimit                 ((uint16_t) 0x134)
+#define PPSMC_MSG_Voltage_Cntl_Disable        ((uint16_t) 0x135)
+#define PPSMC_MSG_PCIeDPM_Enable              ((uint16_t) 0x136)
+#define PPSMC_MSG_ACPPowerOFF                 ((uint16_t) 0x137)
+#define PPSMC_MSG_ACPPowerON                  ((uint16_t) 0x138)
+#define PPSMC_MSG_SAMPowerOFF                 ((uint16_t) 0x139)
+#define PPSMC_MSG_SAMPowerON                  ((uint16_t) 0x13a)
+#define PPSMC_MSG_SDMAPowerOFF                ((uint16_t) 0x13b)
+#define PPSMC_MSG_SDMAPowerON                 ((uint16_t) 0x13c)
+#define PPSMC_MSG_PCIeDPM_Disable             ((uint16_t) 0x13d)
+#define PPSMC_MSG_IOMMUPowerOFF               ((uint16_t) 0x13e)
+#define PPSMC_MSG_IOMMUPowerON                ((uint16_t) 0x13f)
+#define PPSMC_MSG_NBDPM_Enable                ((uint16_t) 0x140)
+#define PPSMC_MSG_NBDPM_Disable               ((uint16_t) 0x141)
+#define PPSMC_MSG_NBDPM_ForceNominal          ((uint16_t) 0x142)
+#define PPSMC_MSG_NBDPM_ForcePerformance      ((uint16_t) 0x143)
+#define PPSMC_MSG_NBDPM_UnForce               ((uint16_t) 0x144)
+#define PPSMC_MSG_SCLKDPM_SetEnabledMask      ((uint16_t) 0x145)
+#define PPSMC_MSG_MCLKDPM_SetEnabledMask      ((uint16_t) 0x146)
+#define PPSMC_MSG_PCIeDPM_ForceLevel          ((uint16_t) 0x147)
+#define PPSMC_MSG_PCIeDPM_UnForceLevel        ((uint16_t) 0x148)
+#define PPSMC_MSG_EnableACDCGPIOInterrupt     ((uint16_t) 0x149)
+#define PPSMC_MSG_EnableVRHotGPIOInterrupt    ((uint16_t) 0x14a)
+#define PPSMC_MSG_SwitchToAC                  ((uint16_t) 0x14b)
+
+#define PPSMC_MSG_XDMAPowerOFF                ((uint16_t) 0x14c)
+#define PPSMC_MSG_XDMAPowerON                 ((uint16_t) 0x14d)
+
+#define PPSMC_MSG_DPM_Enable                  ((uint16_t)0x14e)
+#define PPSMC_MSG_DPM_Disable                 ((uint16_t)0x14f)
+#define PPSMC_MSG_MCLKDPM_Enable              ((uint16_t)0x150)
+#define PPSMC_MSG_MCLKDPM_Disable             ((uint16_t)0x151)
+#define PPSMC_MSG_LCLKDPM_Enable              ((uint16_t)0x152)
+#define PPSMC_MSG_LCLKDPM_Disable             ((uint16_t)0x153)
+#define PPSMC_MSG_UVDDPM_Enable               ((uint16_t)0x154)
+#define PPSMC_MSG_UVDDPM_Disable              ((uint16_t)0x155)
+#define PPSMC_MSG_SAMUDPM_Enable              ((uint16_t)0x156)
+#define PPSMC_MSG_SAMUDPM_Disable             ((uint16_t)0x157)
+#define PPSMC_MSG_ACPDPM_Enable               ((uint16_t)0x158)
+#define PPSMC_MSG_ACPDPM_Disable              ((uint16_t)0x159)
+#define PPSMC_MSG_VCEDPM_Enable               ((uint16_t)0x15a)
+#define PPSMC_MSG_VCEDPM_Disable              ((uint16_t)0x15b)
+#define PPSMC_MSG_LCLKDPM_SetEnabledMask      ((uint16_t)0x15c)
+
+#define PPSMC_MSG_DPM_FPS_Mode                ((uint16_t) 0x15d)
+#define PPSMC_MSG_DPM_Activity_Mode           ((uint16_t) 0x15e)
+#define PPSMC_MSG_VddC_Request                ((uint16_t) 0x15f)
+#define PPSMC_MSG_MCLKDPM_GetEnabledMask      ((uint16_t) 0x160)
+#define PPSMC_MSG_LCLKDPM_GetEnabledMask      ((uint16_t) 0x161)
+#define PPSMC_MSG_SCLKDPM_GetEnabledMask      ((uint16_t) 0x162)
+#define PPSMC_MSG_UVDDPM_GetEnabledMask       ((uint16_t) 0x163)
+#define PPSMC_MSG_SAMUDPM_GetEnabledMask      ((uint16_t) 0x164)
+#define PPSMC_MSG_ACPDPM_GetEnabledMask       ((uint16_t) 0x165)
+#define PPSMC_MSG_VCEDPM_GetEnabledMask       ((uint16_t) 0x166)
+#define PPSMC_MSG_PCIeDPM_SetEnabledMask      ((uint16_t) 0x167)
+#define PPSMC_MSG_PCIeDPM_GetEnabledMask      ((uint16_t) 0x168)
+#define PPSMC_MSG_TDCLimitEnable              ((uint16_t) 0x169)
+#define PPSMC_MSG_TDCLimitDisable             ((uint16_t) 0x16a)
+#define PPSMC_MSG_DPM_AutoRotate_Mode         ((uint16_t) 0x16b)
+#define PPSMC_MSG_DISPCLK_FROM_FCH            ((uint16_t)0x16c)
+#define PPSMC_MSG_DISPCLK_FROM_DFS            ((uint16_t)0x16d)
+#define PPSMC_MSG_DPREFCLK_FROM_FCH           ((uint16_t)0x16e)
+#define PPSMC_MSG_DPREFCLK_FROM_DFS           ((uint16_t)0x16f)
+#define PPSMC_MSG_PmStatusLogStart            ((uint16_t)0x170)
+#define PPSMC_MSG_PmStatusLogSample           ((uint16_t)0x171)
+#define PPSMC_MSG_SCLK_AutoDPM_ON             ((uint16_t) 0x172)
+#define PPSMC_MSG_MCLK_AutoDPM_ON             ((uint16_t) 0x173)
+#define PPSMC_MSG_LCLK_AutoDPM_ON             ((uint16_t) 0x174)
+#define PPSMC_MSG_UVD_AutoDPM_ON              ((uint16_t) 0x175)
+#define PPSMC_MSG_SAMU_AutoDPM_ON             ((uint16_t) 0x176)
+#define PPSMC_MSG_ACP_AutoDPM_ON              ((uint16_t) 0x177)
+#define PPSMC_MSG_VCE_AutoDPM_ON              ((uint16_t) 0x178)
+#define PPSMC_MSG_PCIe_AutoDPM_ON             ((uint16_t) 0x179)
+#define PPSMC_MSG_MASTER_AutoDPM_ON           ((uint16_t) 0x17a)
+#define PPSMC_MSG_MASTER_AutoDPM_OFF          ((uint16_t) 0x17b)
+#define PPSMC_MSG_DYNAMICDISPPHYPOWER         ((uint16_t) 0x17c)
+#define PPSMC_MSG_CAC_COLLECTION_ON           ((uint16_t) 0x17d)
+#define PPSMC_MSG_CAC_COLLECTION_OFF          ((uint16_t) 0x17e)
+#define PPSMC_MSG_CAC_CORRELATION_ON          ((uint16_t) 0x17f)
+#define PPSMC_MSG_CAC_CORRELATION_OFF         ((uint16_t) 0x180)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_ON        ((uint16_t) 0x181)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_OFF       ((uint16_t) 0x182)
+#define PPSMC_MSG_UVD_HANDSHAKE_OFF           ((uint16_t) 0x183)
+#define PPSMC_MSG_ALLOW_LOWSCLK_INTERRUPT     ((uint16_t) 0x184)
+#define PPSMC_MSG_PkgPwrLimitEnable           ((uint16_t) 0x185)
+#define PPSMC_MSG_PkgPwrLimitDisable          ((uint16_t) 0x186)
+#define PPSMC_MSG_PkgPwrSetLimit              ((uint16_t) 0x187)
+#define PPSMC_MSG_OverDriveSetTargetTdp       ((uint16_t) 0x188)
+#define PPSMC_MSG_SCLKDPM_FreezeLevel         ((uint16_t) 0x189)
+#define PPSMC_MSG_SCLKDPM_UnfreezeLevel       ((uint16_t) 0x18A)
+#define PPSMC_MSG_MCLKDPM_FreezeLevel         ((uint16_t) 0x18B)
+#define PPSMC_MSG_MCLKDPM_UnfreezeLevel       ((uint16_t) 0x18C)
+#define PPSMC_MSG_START_DRAM_LOGGING          ((uint16_t) 0x18D)
+#define PPSMC_MSG_STOP_DRAM_LOGGING           ((uint16_t) 0x18E)
+#define PPSMC_MSG_MASTER_DeepSleep_ON         ((uint16_t) 0x18F)
+#define PPSMC_MSG_MASTER_DeepSleep_OFF        ((uint16_t) 0x190)
+#define PPSMC_MSG_Remove_DC_Clamp             ((uint16_t) 0x191)
+#define PPSMC_MSG_DisableACDCGPIOInterrupt    ((uint16_t) 0x192)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddc       ((uint16_t) 0x193)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddci      ((uint16_t) 0x194)
+#define PPSMC_MSG_SetVidOffset_1              ((uint16_t) 0x195)
+#define PPSMC_MSG_SetVidOffset_2              ((uint16_t) 0x207)
+#define PPSMC_MSG_GetVidOffset_1              ((uint16_t) 0x196)
+#define PPSMC_MSG_GetVidOffset_2              ((uint16_t) 0x208)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Enable    ((uint16_t) 0x197)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Disable	  ((uint16_t) 0x198)
+#define PPSMC_MSG_SetTjMax                    ((uint16_t) 0x199)
+#define PPSMC_MSG_SetFanPwmMax                ((uint16_t) 0x19A)
+
+#define PPSMC_MSG_WaitForMclkSwitchFinish	  ((uint16_t) 0x19B)
+#define PPSMC_MSG_ENABLE_THERMAL_DPM          ((uint16_t) 0x19C)
+#define PPSMC_MSG_DISABLE_THERMAL_DPM         ((uint16_t) 0x19D)
+#define PPSMC_MSG_Enable_PCC                  ((uint16_t) 0x19E)
+#define PPSMC_MSG_Disable_PCC                 ((uint16_t) 0x19F)
+
+#define PPSMC_MSG_API_GetSclkFrequency        ((uint16_t) 0x200)
+#define PPSMC_MSG_API_GetMclkFrequency        ((uint16_t) 0x201)
+#define PPSMC_MSG_API_GetSclkBusy             ((uint16_t) 0x202)
+#define PPSMC_MSG_API_GetMclkBusy             ((uint16_t) 0x203)
+#define PPSMC_MSG_API_GetAsicPower            ((uint16_t) 0x204)
+#define PPSMC_MSG_SetFanRpmMax                ((uint16_t) 0x205)
+#define PPSMC_MSG_SetFanSclkTarget            ((uint16_t) 0x206)
+#define PPSMC_MSG_SetFanMinPwm                ((uint16_t) 0x209)
+#define PPSMC_MSG_SetFanTemperatureTarget     ((uint16_t) 0x20A)
+
+#define PPSMC_MSG_BACO_StartMonitor           ((uint16_t) 0x240)
+#define PPSMC_MSG_BACO_Cancel                 ((uint16_t) 0x241)
+#define PPSMC_MSG_EnableVddGfx                ((uint16_t) 0x242)
+#define PPSMC_MSG_DisableVddGfx               ((uint16_t) 0x243)
+#define PPSMC_MSG_UcodeAddressLow             ((uint16_t) 0x244)
+#define PPSMC_MSG_UcodeAddressHigh            ((uint16_t) 0x245)
+#define PPSMC_MSG_UcodeLoadStatus             ((uint16_t) 0x246)
+
+#define PPSMC_MSG_DRV_DRAM_ADDR_HI			  ((uint16_t) 0x250)
+#define PPSMC_MSG_DRV_DRAM_ADDR_LO            ((uint16_t) 0x251)
+#define PPSMC_MSG_SMU_DRAM_ADDR_HI            ((uint16_t) 0x252)
+#define PPSMC_MSG_SMU_DRAM_ADDR_LO            ((uint16_t) 0x253)
+#define PPSMC_MSG_LoadUcodes                  ((uint16_t) 0x254)
+#define PPSMC_MSG_PowerStateNotify            ((uint16_t) 0x255)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_HI      ((uint16_t) 0x256)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_LO      ((uint16_t) 0x257)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_HI          ((uint16_t) 0x258)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_LO          ((uint16_t) 0x259)
+#define PPSMC_MSG_LoadVBios                   ((uint16_t) 0x25A)
+#define PPSMC_MSG_GetUcodeVersion             ((uint16_t) 0x25B)
+#define DMCUSMC_MSG_PSREntry                  ((uint16_t) 0x25C)
+#define DMCUSMC_MSG_PSRExit                   ((uint16_t) 0x25D)
+#define PPSMC_MSG_EnableClockGatingFeature    ((uint16_t) 0x260)
+#define PPSMC_MSG_DisableClockGatingFeature   ((uint16_t) 0x261)
+#define PPSMC_MSG_IsDeviceRunning             ((uint16_t) 0x262)
+#define PPSMC_MSG_LoadMetaData                ((uint16_t) 0x263)
+#define PPSMC_MSG_TMON_AutoCaliberate_Enable  ((uint16_t) 0x264)
+#define PPSMC_MSG_TMON_AutoCaliberate_Disable ((uint16_t) 0x265)
+#define PPSMC_MSG_GetTelemetry1Slope          ((uint16_t) 0x266)
+#define PPSMC_MSG_GetTelemetry1Offset         ((uint16_t) 0x267)
+#define PPSMC_MSG_GetTelemetry2Slope          ((uint16_t) 0x268)
+#define PPSMC_MSG_GetTelemetry2Offset         ((uint16_t) 0x269)
+
+typedef uint16_t PPSMC_Msg;
+
+/* If the SMC firmware has an event status soft register this is what the individual bits mean.*/
+#define PPSMC_EVENT_STATUS_THERMAL          0x00000001
+#define PPSMC_EVENT_STATUS_REGULATORHOT     0x00000002
+#define PPSMC_EVENT_STATUS_DC               0x00000004
+#define PPSMC_EVENT_STATUS_GPIO17           0x00000008
+
+
+#pragma pack(pop)
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
new file mode 100644
index 000000000000..6c4ef135cf01
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
@@ -0,0 +1,9 @@
+#
+# Makefile for the 'smu manager' sub-component of powerplay.
+# It provides the smu management services for the driver.
+
+SMU_MGR = smumgr.o cz_smumgr.o tonga_smumgr.o fiji_smumgr.o
+
+AMD_PP_SMUMGR = $(addprefix $(AMD_PP_PATH)/smumgr/,$(SMU_MGR))
+
+AMD_POWERPLAY_FILES += $(AMD_PP_SMUMGR)
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
new file mode 100644
index 000000000000..ec222c665602
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
@@ -0,0 +1,881 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/gfp.h>
+#include "linux/delay.h"
+#include "cgs_common.h"
+#include "smu/smu_8_0_d.h"
+#include "smu/smu_8_0_sh_mask.h"
+#include "smu8.h"
+#include "smu8_fusion.h"
+#include "cz_smumgr.h"
+#include "cz_ppsmc.h"
+#include "smu_ucode_xfer_cz.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "smumgr.h"
+
+#define SIZE_ALIGN_32(x)    (((x) + 31) / 32 * 32)
+
+static enum cz_scratch_entry firmware_list[] = {
+	CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0,
+	CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
+	CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G,
+};
+
+static int cz_smum_get_argument(struct pp_smumgr *smumgr)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	return cgs_read_register(smumgr->device,
+					mmSMU_MP1_SRBM2P_ARG_0);
+}
+
+static int cz_send_msg_to_smc_async(struct pp_smumgr *smumgr,
+								uint16_t msg)
+{
+	int result = 0;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	result = SMUM_WAIT_FIELD_UNEQUAL(smumgr,
+					SMU_MP1_SRBM2P_RESP_0, CONTENT, 0);
+	if (result != 0) {
+		printk(KERN_ERR "[ powerplay ] cz_send_msg_to_smc_async failed\n");
+		return result;
+	}
+
+	cgs_write_register(smumgr->device, mmSMU_MP1_SRBM2P_RESP_0, 0);
+	cgs_write_register(smumgr->device, mmSMU_MP1_SRBM2P_MSG_0, msg);
+
+	return 0;
+}
+
+/* Send a message to the SMC, and wait for its response.*/
+static int cz_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	int result = 0;
+
+	result = cz_send_msg_to_smc_async(smumgr, msg);
+	if (result != 0)
+		return result;
+
+	result = SMUM_WAIT_FIELD_UNEQUAL(smumgr,
+					SMU_MP1_SRBM2P_RESP_0, CONTENT, 0);
+
+	if (result != 0)
+		return result;
+
+	return 0;
+}
+
+static int cz_set_smc_sram_address(struct pp_smumgr *smumgr,
+				     uint32_t smc_address, uint32_t limit)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	if (0 != (3 & smc_address)) {
+		printk(KERN_ERR "[ powerplay ] SMC address must be 4 byte aligned\n");
+		return -1;
+	}
+
+	if (limit <= (smc_address + 3)) {
+		printk(KERN_ERR "[ powerplay ] SMC address beyond the SMC RAM area\n");
+		return -1;
+	}
+
+	cgs_write_register(smumgr->device, mmMP0PUB_IND_INDEX_0,
+				SMN_MP1_SRAM_START_ADDR + smc_address);
+
+	return 0;
+}
+
+static int cz_write_smc_sram_dword(struct pp_smumgr *smumgr,
+		uint32_t smc_address, uint32_t value, uint32_t limit)
+{
+	int result;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	result = cz_set_smc_sram_address(smumgr, smc_address, limit);
+	cgs_write_register(smumgr->device, mmMP0PUB_IND_DATA_0, value);
+
+	return 0;
+}
+
+static int cz_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
+					  uint16_t msg, uint32_t parameter)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	cgs_write_register(smumgr->device, mmSMU_MP1_SRBM2P_ARG_0, parameter);
+
+	return cz_send_msg_to_smc(smumgr, msg);
+}
+
+static int cz_request_smu_load_fw(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)(smumgr->backend);
+	int result = 0;
+	uint32_t smc_address;
+
+	if (!smumgr->reload_fw) {
+		printk(KERN_INFO "[ powerplay ] skip reloading...\n");
+		return 0;
+	}
+
+	smc_address = SMU8_FIRMWARE_HEADER_LOCATION +
+		offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
+
+	cz_write_smc_sram_dword(smumgr, smc_address, 0, smc_address+4);
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+					PPSMC_MSG_DriverDramAddrHi,
+					cz_smu->toc_buffer.mc_addr_high);
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+					PPSMC_MSG_DriverDramAddrLo,
+					cz_smu->toc_buffer.mc_addr_low);
+
+	cz_send_msg_to_smc(smumgr, PPSMC_MSG_InitJobs);
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+					PPSMC_MSG_ExecuteJob,
+					cz_smu->toc_entry_aram);
+	cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
+				cz_smu->toc_entry_power_profiling_index);
+
+	result = cz_send_msg_to_smc_with_parameter(smumgr,
+					PPSMC_MSG_ExecuteJob,
+					cz_smu->toc_entry_initialize_index);
+
+	return result;
+}
+
+static int cz_check_fw_load_finish(struct pp_smumgr *smumgr,
+				   uint32_t firmware)
+{
+	int i;
+	uint32_t index = SMN_MP1_SRAM_START_ADDR +
+			 SMU8_FIRMWARE_HEADER_LOCATION +
+			 offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	return cgs_read_register(smumgr->device,
+					mmSMU_MP1_SRBM2P_ARG_0);
+
+	cgs_write_register(smumgr->device, mmMP0PUB_IND_INDEX, index);
+
+	for (i = 0; i < smumgr->usec_timeout; i++) {
+		if (firmware ==
+			(cgs_read_register(smumgr->device, mmMP0PUB_IND_DATA) & firmware))
+			break;
+		udelay(1);
+	}
+
+	if (i >= smumgr->usec_timeout) {
+		printk(KERN_ERR "[ powerplay ] SMU check loaded firmware failed.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cz_load_mec_firmware(struct pp_smumgr *smumgr)
+{
+	uint32_t reg_data;
+	uint32_t tmp;
+	int ret = 0;
+	struct cgs_firmware_info info = {0};
+	struct cz_smumgr *cz_smu;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	cz_smu = (struct cz_smumgr *)smumgr->backend;
+	ret = cgs_get_firmware_info(smumgr->device,
+						CGS_UCODE_ID_CP_MEC, &info);
+
+	if (ret)
+		return -EINVAL;
+
+	/* Disable MEC parsing/prefetching */
+	tmp = cgs_read_register(smumgr->device,
+					mmCP_MEC_CNTL);
+	tmp = SMUM_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME1_HALT, 1);
+	tmp = SMUM_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME2_HALT, 1);
+	cgs_write_register(smumgr->device, mmCP_MEC_CNTL, tmp);
+
+	tmp = cgs_read_register(smumgr->device,
+					mmCP_CPC_IC_BASE_CNTL);
+
+	tmp = SMUM_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
+	tmp = SMUM_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ATC, 0);
+	tmp = SMUM_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
+	tmp = SMUM_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, MTYPE, 1);
+	cgs_write_register(smumgr->device, mmCP_CPC_IC_BASE_CNTL, tmp);
+
+	reg_data = smu_lower_32_bits(info.mc_addr) &
+			SMUM_FIELD_MASK(CP_CPC_IC_BASE_LO, IC_BASE_LO);
+	cgs_write_register(smumgr->device, mmCP_CPC_IC_BASE_LO, reg_data);
+
+	reg_data = smu_upper_32_bits(info.mc_addr) &
+			SMUM_FIELD_MASK(CP_CPC_IC_BASE_HI, IC_BASE_HI);
+	cgs_write_register(smumgr->device, mmCP_CPC_IC_BASE_HI, reg_data);
+
+	return 0;
+}
+
+static int cz_start_smu(struct pp_smumgr *smumgr)
+{
+	int ret = 0;
+	uint32_t fw_to_check = UCODE_ID_RLC_G_MASK |
+				UCODE_ID_SDMA0_MASK |
+				UCODE_ID_SDMA1_MASK |
+				UCODE_ID_CP_CE_MASK |
+				UCODE_ID_CP_ME_MASK |
+				UCODE_ID_CP_PFP_MASK |
+				UCODE_ID_CP_MEC_JT1_MASK |
+				UCODE_ID_CP_MEC_JT2_MASK;
+
+	if (smumgr->chip_id == CHIP_STONEY)
+		fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
+
+	cz_request_smu_load_fw(smumgr);
+	cz_check_fw_load_finish(smumgr, fw_to_check);
+
+	ret = cz_load_mec_firmware(smumgr);
+	if (ret)
+		printk(KERN_ERR "[ powerplay ] Mec Firmware load failed\n");
+
+	return ret;
+}
+
+static uint8_t cz_translate_firmware_enum_to_arg(struct pp_smumgr *smumgr,
+			enum cz_scratch_entry firmware_enum)
+{
+	uint8_t ret = 0;
+
+	switch (firmware_enum) {
+	case CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0:
+		ret = UCODE_ID_SDMA0;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1:
+		if (smumgr->chip_id == CHIP_STONEY)
+			ret = UCODE_ID_SDMA0;
+		else
+			ret = UCODE_ID_SDMA1;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE:
+		ret = UCODE_ID_CP_CE;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP:
+		ret = UCODE_ID_CP_PFP;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME:
+		ret = UCODE_ID_CP_ME;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1:
+		ret = UCODE_ID_CP_MEC_JT1;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2:
+		if (smumgr->chip_id == CHIP_STONEY)
+			ret = UCODE_ID_CP_MEC_JT1;
+		else
+			ret = UCODE_ID_CP_MEC_JT2;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_GMCON_RENG:
+		ret = UCODE_ID_GMCON_RENG;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G:
+		ret = UCODE_ID_RLC_G;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH:
+		ret = UCODE_ID_RLC_SCRATCH;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM:
+		ret = UCODE_ID_RLC_SRM_ARAM;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM:
+		ret = UCODE_ID_RLC_SRM_DRAM;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_ERAM:
+		ret = UCODE_ID_DMCU_ERAM;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_IRAM:
+		ret = UCODE_ID_DMCU_IRAM;
+		break;
+	case CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING:
+		ret = TASK_ARG_INIT_MM_PWR_LOG;
+		break;
+	case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_HALT:
+	case CZ_SCRATCH_ENTRY_DATA_ID_SYS_CLOCKGATING:
+	case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_RING_REGS:
+	case CZ_SCRATCH_ENTRY_DATA_ID_NONGFX_REINIT:
+	case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_START:
+	case CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS:
+		ret = TASK_ARG_REG_MMIO;
+		break;
+	case CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE:
+		ret = TASK_ARG_INIT_CLK_TABLE;
+		break;
+	}
+
+	return ret;
+}
+
+static enum cgs_ucode_id cz_convert_fw_type_to_cgs(uint32_t fw_type)
+{
+	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
+
+	switch (fw_type) {
+	case UCODE_ID_SDMA0:
+		result = CGS_UCODE_ID_SDMA0;
+		break;
+	case UCODE_ID_SDMA1:
+		result = CGS_UCODE_ID_SDMA1;
+		break;
+	case UCODE_ID_CP_CE:
+		result = CGS_UCODE_ID_CP_CE;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = CGS_UCODE_ID_CP_PFP;
+		break;
+	case UCODE_ID_CP_ME:
+		result = CGS_UCODE_ID_CP_ME;
+		break;
+	case UCODE_ID_CP_MEC_JT1:
+		result = CGS_UCODE_ID_CP_MEC_JT1;
+		break;
+	case UCODE_ID_CP_MEC_JT2:
+		result = CGS_UCODE_ID_CP_MEC_JT2;
+		break;
+	case UCODE_ID_RLC_G:
+		result = CGS_UCODE_ID_RLC_G;
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+static int cz_smu_populate_single_scratch_task(
+			struct pp_smumgr *smumgr,
+			enum cz_scratch_entry fw_enum,
+			uint8_t type, bool is_last)
+{
+	uint8_t i;
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	struct TOC *toc = (struct TOC *)cz_smu->toc_buffer.kaddr;
+	struct SMU_Task *task = &toc->tasks[cz_smu->toc_entry_used_count++];
+
+	task->type = type;
+	task->arg = cz_translate_firmware_enum_to_arg(smumgr, fw_enum);
+	task->next = is_last ? END_OF_TASK_LIST : cz_smu->toc_entry_used_count;
+
+	for (i = 0; i < cz_smu->scratch_buffer_length; i++)
+		if (cz_smu->scratch_buffer[i].firmware_ID == fw_enum)
+			break;
+
+	if (i >= cz_smu->scratch_buffer_length) {
+		printk(KERN_ERR "[ powerplay ] Invalid Firmware Type\n");
+		return -EINVAL;
+	}
+
+	task->addr.low = cz_smu->scratch_buffer[i].mc_addr_low;
+	task->addr.high = cz_smu->scratch_buffer[i].mc_addr_high;
+	task->size_bytes = cz_smu->scratch_buffer[i].data_size;
+
+	if (CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS == fw_enum) {
+		struct cz_ih_meta_data *pIHReg_restore =
+		     (struct cz_ih_meta_data *)cz_smu->scratch_buffer[i].kaddr;
+		pIHReg_restore->command =
+			METADATA_CMD_MODE0 | METADATA_PERFORM_ON_LOAD;
+	}
+
+	return 0;
+}
+
+static int cz_smu_populate_single_ucode_load_task(
+					struct pp_smumgr *smumgr,
+					enum cz_scratch_entry fw_enum,
+					bool is_last)
+{
+	uint8_t i;
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	struct TOC *toc = (struct TOC *)cz_smu->toc_buffer.kaddr;
+	struct SMU_Task *task = &toc->tasks[cz_smu->toc_entry_used_count++];
+
+	task->type = TASK_TYPE_UCODE_LOAD;
+	task->arg = cz_translate_firmware_enum_to_arg(smumgr, fw_enum);
+	task->next = is_last ? END_OF_TASK_LIST : cz_smu->toc_entry_used_count;
+
+	for (i = 0; i < cz_smu->driver_buffer_length; i++)
+		if (cz_smu->driver_buffer[i].firmware_ID == fw_enum)
+			break;
+
+	if (i >= cz_smu->driver_buffer_length) {
+		printk(KERN_ERR "[ powerplay ] Invalid Firmware Type\n");
+		return -EINVAL;
+	}
+
+	task->addr.low = cz_smu->driver_buffer[i].mc_addr_low;
+	task->addr.high = cz_smu->driver_buffer[i].mc_addr_high;
+	task->size_bytes = cz_smu->driver_buffer[i].data_size;
+
+	return 0;
+}
+
+static int cz_smu_construct_toc_for_rlc_aram_save(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+
+	cz_smu->toc_entry_aram = cz_smu->toc_entry_used_count;
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
+				TASK_TYPE_UCODE_SAVE, true);
+
+	return 0;
+}
+
+static int cz_smu_initialize_toc_empty_job_list(struct pp_smumgr *smumgr)
+{
+	int i;
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	struct TOC *toc = (struct TOC *)cz_smu->toc_buffer.kaddr;
+
+	for (i = 0; i < NUM_JOBLIST_ENTRIES; i++)
+		toc->JobList[i] = (uint8_t)IGNORE_JOB;
+
+	return 0;
+}
+
+static int cz_smu_construct_toc_for_vddgfx_enter(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	struct TOC *toc = (struct TOC *)cz_smu->toc_buffer.kaddr;
+
+	toc->JobList[JOB_GFX_SAVE] = (uint8_t)cz_smu->toc_entry_used_count;
+	cz_smu_populate_single_scratch_task(smumgr,
+				    CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
+				    TASK_TYPE_UCODE_SAVE, false);
+
+	cz_smu_populate_single_scratch_task(smumgr,
+				    CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
+				    TASK_TYPE_UCODE_SAVE, true);
+
+	return 0;
+}
+
+
+static int cz_smu_construct_toc_for_vddgfx_exit(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	struct TOC *toc = (struct TOC *)cz_smu->toc_buffer.kaddr;
+
+	toc->JobList[JOB_GFX_RESTORE] = (uint8_t)cz_smu->toc_entry_used_count;
+
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+
+	if (smumgr->chip_id == CHIP_STONEY)
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+	else
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
+
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, false);
+
+	/* populate scratch */
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
+				TASK_TYPE_UCODE_LOAD, false);
+
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
+				TASK_TYPE_UCODE_LOAD, false);
+
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
+				TASK_TYPE_UCODE_LOAD, true);
+
+	return 0;
+}
+
+static int cz_smu_construct_toc_for_power_profiling(
+						 struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+
+	cz_smu->toc_entry_power_profiling_index = cz_smu->toc_entry_used_count;
+
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
+				TASK_TYPE_INITIALIZE, true);
+	return 0;
+}
+
+static int cz_smu_construct_toc_for_bootup(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+
+	cz_smu->toc_entry_initialize_index = cz_smu->toc_entry_used_count;
+
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
+	if (smumgr->chip_id == CHIP_STONEY)
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
+	else
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+	if (smumgr->chip_id == CHIP_STONEY)
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
+	else
+		cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
+	cz_smu_populate_single_ucode_load_task(smumgr,
+				CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, true);
+
+	return 0;
+}
+
+static int cz_smu_construct_toc_for_clock_table(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+
+	cz_smu->toc_entry_clock_table = cz_smu->toc_entry_used_count;
+
+	cz_smu_populate_single_scratch_task(smumgr,
+				CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE,
+				TASK_TYPE_INITIALIZE, true);
+
+	return 0;
+}
+
+static int cz_smu_construct_toc(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+
+	cz_smu->toc_entry_used_count = 0;
+
+	cz_smu_initialize_toc_empty_job_list(smumgr);
+
+	cz_smu_construct_toc_for_rlc_aram_save(smumgr);
+
+	cz_smu_construct_toc_for_vddgfx_enter(smumgr);
+
+	cz_smu_construct_toc_for_vddgfx_exit(smumgr);
+
+	cz_smu_construct_toc_for_power_profiling(smumgr);
+
+	cz_smu_construct_toc_for_bootup(smumgr);
+
+	cz_smu_construct_toc_for_clock_table(smumgr);
+
+	return 0;
+}
+
+static int cz_smu_populate_firmware_entries(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	uint32_t firmware_type;
+	uint32_t i;
+	int ret;
+	enum cgs_ucode_id ucode_id;
+	struct cgs_firmware_info info = {0};
+
+	cz_smu->driver_buffer_length = 0;
+
+	for (i = 0; i < sizeof(firmware_list)/sizeof(*firmware_list); i++) {
+
+		firmware_type = cz_translate_firmware_enum_to_arg(smumgr,
+					firmware_list[i]);
+
+		ucode_id = cz_convert_fw_type_to_cgs(firmware_type);
+
+		ret = cgs_get_firmware_info(smumgr->device,
+							ucode_id, &info);
+
+		if (ret == 0) {
+			cz_smu->driver_buffer[i].mc_addr_high =
+					smu_upper_32_bits(info.mc_addr);
+
+			cz_smu->driver_buffer[i].mc_addr_low =
+					smu_lower_32_bits(info.mc_addr);
+
+			cz_smu->driver_buffer[i].data_size = info.image_size;
+
+			cz_smu->driver_buffer[i].firmware_ID = firmware_list[i];
+			cz_smu->driver_buffer_length++;
+		}
+	}
+
+	return 0;
+}
+
+static int cz_smu_populate_single_scratch_entry(
+				struct pp_smumgr *smumgr,
+				enum cz_scratch_entry scratch_type,
+				uint32_t ulsize_byte,
+				struct cz_buffer_entry *entry)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	long long mc_addr =
+			((long long)(cz_smu->smu_buffer.mc_addr_high) << 32)
+			| cz_smu->smu_buffer.mc_addr_low;
+
+	uint32_t ulsize_aligned = SIZE_ALIGN_32(ulsize_byte);
+
+	mc_addr += cz_smu->smu_buffer_used_bytes;
+
+	entry->data_size = ulsize_byte;
+	entry->kaddr = (char *) cz_smu->smu_buffer.kaddr +
+				cz_smu->smu_buffer_used_bytes;
+	entry->mc_addr_low = smu_lower_32_bits(mc_addr);
+	entry->mc_addr_high = smu_upper_32_bits(mc_addr);
+	entry->firmware_ID = scratch_type;
+
+	cz_smu->smu_buffer_used_bytes += ulsize_aligned;
+
+	return 0;
+}
+
+static int cz_download_pptable_settings(struct pp_smumgr *smumgr, void **table)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	unsigned long i;
+
+	for (i = 0; i < cz_smu->scratch_buffer_length; i++) {
+		if (cz_smu->scratch_buffer[i].firmware_ID
+			== CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE)
+			break;
+	}
+
+	*table = (struct SMU8_Fusion_ClkTable *)cz_smu->scratch_buffer[i].kaddr;
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_SetClkTableAddrHi,
+				cz_smu->scratch_buffer[i].mc_addr_high);
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_SetClkTableAddrLo,
+				cz_smu->scratch_buffer[i].mc_addr_low);
+
+	cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
+				cz_smu->toc_entry_clock_table);
+
+	cz_send_msg_to_smc(smumgr, PPSMC_MSG_ClkTableXferToDram);
+
+	return 0;
+}
+
+static int cz_upload_pptable_settings(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	unsigned long i;
+
+	for (i = 0; i < cz_smu->scratch_buffer_length; i++) {
+		if (cz_smu->scratch_buffer[i].firmware_ID
+				== CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE)
+			break;
+	}
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_SetClkTableAddrHi,
+				cz_smu->scratch_buffer[i].mc_addr_high);
+
+	cz_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_SetClkTableAddrLo,
+				cz_smu->scratch_buffer[i].mc_addr_low);
+
+	cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
+				cz_smu->toc_entry_clock_table);
+
+	cz_send_msg_to_smc(smumgr, PPSMC_MSG_ClkTableXferToSmu);
+
+	return 0;
+}
+
+static int cz_smu_init(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
+	uint64_t mc_addr = 0;
+	int ret = 0;
+
+	cz_smu->toc_buffer.data_size = 4096;
+	cz_smu->smu_buffer.data_size =
+		ALIGN(UCODE_ID_RLC_SCRATCH_SIZE_BYTE, 32) +
+		ALIGN(UCODE_ID_RLC_SRM_ARAM_SIZE_BYTE, 32) +
+		ALIGN(UCODE_ID_RLC_SRM_DRAM_SIZE_BYTE, 32) +
+		ALIGN(sizeof(struct SMU8_MultimediaPowerLogData), 32) +
+		ALIGN(sizeof(struct SMU8_Fusion_ClkTable), 32);
+
+	ret = smu_allocate_memory(smumgr->device,
+				cz_smu->toc_buffer.data_size,
+				CGS_GPU_MEM_TYPE__GART_CACHEABLE,
+				PAGE_SIZE,
+				&mc_addr,
+				&cz_smu->toc_buffer.kaddr,
+				&cz_smu->toc_buffer.handle);
+	if (ret != 0)
+		return -1;
+
+	cz_smu->toc_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	cz_smu->toc_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	ret = smu_allocate_memory(smumgr->device,
+				cz_smu->smu_buffer.data_size,
+				CGS_GPU_MEM_TYPE__GART_CACHEABLE,
+				PAGE_SIZE,
+				&mc_addr,
+				&cz_smu->smu_buffer.kaddr,
+				&cz_smu->smu_buffer.handle);
+	if (ret != 0)
+		return -1;
+
+	cz_smu->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	cz_smu->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	cz_smu_populate_firmware_entries(smumgr);
+	if (0 != cz_smu_populate_single_scratch_entry(smumgr,
+		CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
+		UCODE_ID_RLC_SCRATCH_SIZE_BYTE,
+		&cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
+		printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+		return -1;
+	}
+
+	if (0 != cz_smu_populate_single_scratch_entry(smumgr,
+		CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
+		UCODE_ID_RLC_SRM_ARAM_SIZE_BYTE,
+		&cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
+		printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+		return -1;
+	}
+	if (0 != cz_smu_populate_single_scratch_entry(smumgr,
+		CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
+		UCODE_ID_RLC_SRM_DRAM_SIZE_BYTE,
+		&cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
+		printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+		return -1;
+	}
+
+	if (0 != cz_smu_populate_single_scratch_entry(smumgr,
+		CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
+		sizeof(struct SMU8_MultimediaPowerLogData),
+		&cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
+		printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+		return -1;
+	}
+
+	if (0 != cz_smu_populate_single_scratch_entry(smumgr,
+		CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE,
+		sizeof(struct SMU8_Fusion_ClkTable),
+		&cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
+		printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+		return -1;
+	}
+	cz_smu_construct_toc(smumgr);
+
+	return 0;
+}
+
+static int cz_smu_fini(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	cz_smu = (struct cz_smumgr *)smumgr->backend;
+	if (cz_smu) {
+		cgs_free_gpu_mem(smumgr->device,
+				cz_smu->toc_buffer.handle);
+		cgs_free_gpu_mem(smumgr->device,
+				cz_smu->smu_buffer.handle);
+		kfree(cz_smu);
+		kfree(smumgr);
+	}
+
+	return 0;
+}
+
+static const struct pp_smumgr_func cz_smu_funcs = {
+	.smu_init = cz_smu_init,
+	.smu_fini = cz_smu_fini,
+	.start_smu = cz_start_smu,
+	.check_fw_load_finish = cz_check_fw_load_finish,
+	.request_smu_load_fw = NULL,
+	.request_smu_load_specific_fw = NULL,
+	.get_argument = cz_smum_get_argument,
+	.send_msg_to_smc = cz_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = cz_send_msg_to_smc_with_parameter,
+	.download_pptable_settings = cz_download_pptable_settings,
+	.upload_pptable_settings = cz_upload_pptable_settings,
+};
+
+int cz_smum_init(struct pp_smumgr *smumgr)
+{
+	struct cz_smumgr *cz_smu;
+
+	cz_smu = kzalloc(sizeof(struct cz_smumgr), GFP_KERNEL);
+	if (cz_smu == NULL)
+		return -ENOMEM;
+
+	smumgr->backend = cz_smu;
+	smumgr->smumgr_funcs = &cz_smu_funcs;
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h
new file mode 100644
index 000000000000..883818039248
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _CZ_SMUMGR_H_
+#define _CZ_SMUMGR_H_
+
+
+#define MAX_NUM_FIRMWARE                        8
+#define MAX_NUM_SCRATCH                         11
+#define CZ_SCRATCH_SIZE_NONGFX_CLOCKGATING      1024
+#define CZ_SCRATCH_SIZE_NONGFX_GOLDENSETTING    2048
+#define CZ_SCRATCH_SIZE_SDMA_METADATA           1024
+#define CZ_SCRATCH_SIZE_IH                      ((2*256+1)*4)
+
+enum cz_scratch_entry {
+	CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0 = 0,
+	CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
+	CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
+	CZ_SCRATCH_ENTRY_UCODE_ID_GMCON_RENG,
+	CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G,
+	CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
+	CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
+	CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
+	CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_ERAM,
+	CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_IRAM,
+	CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
+	CZ_SCRATCH_ENTRY_DATA_ID_SDMA_HALT,
+	CZ_SCRATCH_ENTRY_DATA_ID_SYS_CLOCKGATING,
+	CZ_SCRATCH_ENTRY_DATA_ID_SDMA_RING_REGS,
+	CZ_SCRATCH_ENTRY_DATA_ID_NONGFX_REINIT,
+	CZ_SCRATCH_ENTRY_DATA_ID_SDMA_START,
+	CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS,
+	CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE
+};
+
+struct cz_buffer_entry {
+	uint32_t data_size;
+	uint32_t mc_addr_low;
+	uint32_t mc_addr_high;
+	void *kaddr;
+	enum cz_scratch_entry firmware_ID;
+	unsigned long handle; /* as bo handle used when release bo */
+};
+
+struct cz_register_index_data_pair {
+	uint32_t offset;
+	uint32_t value;
+};
+
+struct cz_ih_meta_data {
+	uint32_t command;
+	struct cz_register_index_data_pair register_index_value_pair[1];
+};
+
+struct cz_smumgr {
+	uint8_t driver_buffer_length;
+	uint8_t scratch_buffer_length;
+	uint16_t toc_entry_used_count;
+	uint16_t toc_entry_initialize_index;
+	uint16_t toc_entry_power_profiling_index;
+	uint16_t toc_entry_aram;
+	uint16_t toc_entry_ih_register_restore_task_index;
+	uint16_t toc_entry_clock_table;
+	uint16_t ih_register_restore_task_size;
+	uint16_t smu_buffer_used_bytes;
+
+	struct cz_buffer_entry toc_buffer;
+	struct cz_buffer_entry smu_buffer;
+	struct cz_buffer_entry firmware_buffer;
+	struct cz_buffer_entry driver_buffer[MAX_NUM_FIRMWARE];
+	struct cz_buffer_entry meta_data_buffer[MAX_NUM_FIRMWARE];
+	struct cz_buffer_entry scratch_buffer[MAX_NUM_SCRATCH];
+};
+
+struct pp_smumgr;
+
+extern int cz_smum_init(struct pp_smumgr *smumgr);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
new file mode 100644
index 000000000000..cdbb9f89bf36
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -0,0 +1,1042 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "smumgr.h"
+#include "smu73.h"
+#include "smu_ucode_xfer_vi.h"
+#include "fiji_smumgr.h"
+#include "fiji_ppsmc.h"
+#include "smu73_discrete.h"
+#include "ppatomctrl.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "gca/gfx_8_0_d.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "pp_debug.h"
+#include "fiji_pwrvirus.h"
+
+#define AVFS_EN_MSB                                        1568
+#define AVFS_EN_LSB                                        1568
+
+#define FIJI_SMC_SIZE 0x20000
+
+struct SMU73_Discrete_GraphicsLevel avfs_graphics_level[8] = {
+		/*  Min        Sclk       pcie     DeepSleep Activity  CgSpll      CgSpll    spllSpread  SpllSpread   CcPwr  CcPwr  Sclk   Display     Enabled     Enabled                       Voltage    Power */
+		/* Voltage,  Frequency,  DpmLevel,  DivId,    Level,  FuncCntl3,  FuncCntl4,  Spectrum,   Spectrum2,  DynRm, DynRm1  Did, Watermark, ForActivity, ForThrottle, UpHyst, DownHyst, DownHyst, Throttle */
+		{ 0x3c0fd047, 0x30750000,   0x00,     0x03,   0x1e00, 0x00200410, 0x87020000, 0x21680000, 0x0c000000,   0,      0,   0x16,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0xa00fd047, 0x409c0000,   0x01,     0x04,   0x1e00, 0x00800510, 0x87020000, 0x21680000, 0x11000000,   0,      0,   0x16,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0x0410d047, 0x50c30000,   0x01,     0x00,   0x1e00, 0x00600410, 0x87020000, 0x21680000, 0x0d000000,   0,      0,   0x0e,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0x6810d047, 0x60ea0000,   0x01,     0x00,   0x1e00, 0x00800410, 0x87020000, 0x21680000, 0x0e000000,   0,      0,   0x0c,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0xcc10d047, 0xe8fd0000,   0x01,     0x00,   0x1e00, 0x00e00410, 0x87020000, 0x21680000, 0x0f000000,   0,      0,   0x0c,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0x3011d047, 0x70110100,   0x01,     0x00,   0x1e00, 0x00400510, 0x87020000, 0x21680000, 0x10000000,   0,      0,   0x0c,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0x9411d047, 0xf8240100,   0x01,     0x00,   0x1e00, 0x00a00510, 0x87020000, 0x21680000, 0x11000000,   0,      0,   0x0c,   0x00,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 },
+		{ 0xf811d047, 0x80380100,   0x01,     0x00,   0x1e00, 0x00000610, 0x87020000, 0x21680000, 0x12000000,   0,      0,   0x0c,   0x01,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 }
+};
+
+static enum cgs_ucode_id fiji_convert_fw_type_to_cgs(uint32_t fw_type)
+{
+	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
+
+	switch (fw_type) {
+	case UCODE_ID_SMU:
+		result = CGS_UCODE_ID_SMU;
+		break;
+	case UCODE_ID_SDMA0:
+		result = CGS_UCODE_ID_SDMA0;
+		break;
+	case UCODE_ID_SDMA1:
+		result = CGS_UCODE_ID_SDMA1;
+		break;
+	case UCODE_ID_CP_CE:
+		result = CGS_UCODE_ID_CP_CE;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = CGS_UCODE_ID_CP_PFP;
+		break;
+	case UCODE_ID_CP_ME:
+		result = CGS_UCODE_ID_CP_ME;
+		break;
+	case UCODE_ID_CP_MEC:
+		result = CGS_UCODE_ID_CP_MEC;
+		break;
+	case UCODE_ID_CP_MEC_JT1:
+		result = CGS_UCODE_ID_CP_MEC_JT1;
+		break;
+	case UCODE_ID_CP_MEC_JT2:
+		result = CGS_UCODE_ID_CP_MEC_JT2;
+		break;
+	case UCODE_ID_RLC_G:
+		result = CGS_UCODE_ID_RLC_G;
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+/**
+* Set the address for reading/writing the SMC SRAM space.
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    smc_addr the address in the SMC RAM to access.
+*/
+static int fiji_set_smc_sram_address(struct pp_smumgr *smumgr,
+		uint32_t smc_addr, uint32_t limit)
+{
+	PP_ASSERT_WITH_CODE((0 == (3 & smc_addr)),
+			"SMC address must be 4 byte aligned.", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((limit > (smc_addr + 3)),
+			"SMC address is beyond the SMC RAM area.", return -EINVAL;);
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, smc_addr);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+
+	return 0;
+}
+
+/**
+* Copy bytes from an array into the SMC RAM space.
+*
+* @param    smumgr  the address of the powerplay SMU manager.
+* @param    smcStartAddress the start address in the SMC RAM to copy bytes to.
+* @param    src the byte array to copy the bytes from.
+* @param    byteCount the number of bytes to copy.
+*/
+int fiji_copy_bytes_to_smc(struct pp_smumgr *smumgr,
+		uint32_t smcStartAddress, const uint8_t *src,
+		uint32_t byteCount, uint32_t limit)
+{
+	int result;
+	uint32_t data, originalData;
+	uint32_t addr, extraShift;
+
+	PP_ASSERT_WITH_CODE((0 == (3 & smcStartAddress)),
+			"SMC address must be 4 byte aligned.", return -EINVAL;);
+	PP_ASSERT_WITH_CODE((limit > (smcStartAddress + byteCount)),
+			"SMC address is beyond the SMC RAM area.", return -EINVAL;);
+
+	addr = smcStartAddress;
+
+	while (byteCount >= 4) {
+		/* Bytes are written into the SMC addres space with the MSB first. */
+		data = src[0] * 0x1000000 + src[1] * 0x10000 + src[2] * 0x100 + src[3];
+
+		result = fiji_set_smc_sram_address(smumgr, addr, limit);
+		if (result)
+			return result;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
+
+		src += 4;
+		byteCount -= 4;
+		addr += 4;
+	}
+
+	if (byteCount) {
+		/* Now write the odd bytes left.
+		 * Do a read modify write cycle.
+		 */
+		data = 0;
+
+		result = fiji_set_smc_sram_address(smumgr, addr, limit);
+		if (result)
+			return result;
+
+		originalData = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
+		extraShift = 8 * (4 - byteCount);
+
+		while (byteCount > 0) {
+			/* Bytes are written into the SMC addres
+			 * space with the MSB first.
+			 */
+			data = (0x100 * data) + *src++;
+			byteCount--;
+		}
+		data <<= extraShift;
+		data |= (originalData & ~((~0UL) << extraShift));
+
+		result = fiji_set_smc_sram_address(smumgr, addr, limit);
+		if (!result)
+			return result;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
+	}
+	return 0;
+}
+
+int fiji_program_jump_on_start(struct pp_smumgr *smumgr)
+{
+	static unsigned char data[] = { 0xE0, 0x00, 0x80, 0x40 };
+
+	fiji_copy_bytes_to_smc(smumgr, 0x0, data, 4, sizeof(data) + 1);
+
+	return 0;
+}
+
+/**
+* Return if the SMC is currently running.
+*
+* @param    smumgr  the address of the powerplay hardware manager.
+*/
+bool fiji_is_smc_ram_running(struct pp_smumgr *smumgr)
+{
+	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device,
+			CGS_IND_REG__SMC,
+			SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
+			&& (0x20100 <= cgs_read_ind_register(smumgr->device,
+					CGS_IND_REG__SMC, ixSMC_PC_C)));
+}
+
+/**
+* Send a message to the SMC, and wait for its response.
+*
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    msg the message to send.
+* @return   The response that came from the SMC.
+*/
+int fiji_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	if (!fiji_is_smc_ram_running(smumgr))
+		return -1;
+
+	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
+		printk(KERN_ERR "Failed to send Previous Message.");
+		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	}
+
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	return 0;
+}
+
+/**
+ * Send a message to the SMC with parameter
+ * @param    smumgr:  the address of the powerplay hardware manager.
+ * @param    msg: the message to send.
+ * @param    parameter: the parameter to send
+ * @return   The response that came from the SMC.
+ */
+int fiji_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
+		uint16_t msg, uint32_t parameter)
+{
+	if (!fiji_is_smc_ram_running(smumgr))
+		return -1;
+
+	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
+		printk(KERN_ERR "Failed to send Previous Message.");
+		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	}
+
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	return 0;
+}
+
+
+/**
+* Send a message to the SMC with parameter, do not wait for response
+*
+* @param    smumgr:  the address of the powerplay hardware manager.
+* @param    msg: the message to send.
+* @param    parameter: the parameter to send
+* @return   The response that came from the SMC.
+*/
+int fiji_send_msg_to_smc_with_parameter_without_waiting(
+		struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
+{
+	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
+		printk(KERN_ERR "Failed to send Previous Message.");
+		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	}
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+
+	return 0;
+}
+
+/**
+* Uploads the SMU firmware from .hex file
+*
+* @param    smumgr  the address of the powerplay SMU manager.
+* @return   0 or -1.
+*/
+
+static int fiji_upload_smu_firmware_image(struct pp_smumgr *smumgr)
+{
+	const uint8_t *src;
+	uint32_t byte_count;
+	uint32_t *data;
+	struct cgs_firmware_info info = {0};
+
+	cgs_get_firmware_info(smumgr->device,
+			fiji_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
+
+	if (info.image_size & 3) {
+		printk(KERN_ERR "SMC ucode is not 4 bytes aligned\n");
+		return -EINVAL;
+	}
+
+	if (info.image_size > FIJI_SMC_SIZE) {
+		printk(KERN_ERR "SMC address is beyond the SMC RAM area\n");
+		return -EINVAL;
+	}
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, 0x20000);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
+
+	byte_count = info.image_size;
+	src = (const uint8_t *)info.kptr;
+
+	data = (uint32_t *)src;
+	for (; byte_count >= 4; data++, byte_count -= 4)
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data[0]);
+
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+	return 0;
+}
+
+/**
+* Read a 32bit value from the SMC SRAM space.
+* ALL PARAMETERS ARE IN HOST BYTE ORDER.
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    smc_addr the address in the SMC RAM to access.
+* @param    value and output parameter for the data read from the SMC SRAM.
+*/
+int fiji_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
+		uint32_t *value, uint32_t limit)
+{
+	int	result = fiji_set_smc_sram_address(smumgr, smc_addr, limit);
+
+	if (result)
+		return result;
+
+	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
+	return 0;
+}
+
+/**
+* Write a 32bit value to the SMC SRAM space.
+* ALL PARAMETERS ARE IN HOST BYTE ORDER.
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    smc_addr the address in the SMC RAM to access.
+* @param    value to write to the SMC SRAM.
+*/
+int fiji_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
+		uint32_t value, uint32_t limit)
+{
+	int result;
+
+	result = fiji_set_smc_sram_address(smumgr, smc_addr, limit);
+
+	if (result)
+		return result;
+
+	cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, value);
+	return 0;
+}
+
+static uint32_t fiji_get_mask_for_firmware_type(uint32_t fw_type)
+{
+	uint32_t result = 0;
+
+	switch (fw_type) {
+	case UCODE_ID_SDMA0:
+		result = UCODE_ID_SDMA0_MASK;
+		break;
+	case UCODE_ID_SDMA1:
+		result = UCODE_ID_SDMA1_MASK;
+		break;
+	case UCODE_ID_CP_CE:
+		result = UCODE_ID_CP_CE_MASK;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = UCODE_ID_CP_PFP_MASK;
+		break;
+	case UCODE_ID_CP_ME:
+		result = UCODE_ID_CP_ME_MASK;
+		break;
+	case UCODE_ID_CP_MEC_JT1:
+		result = UCODE_ID_CP_MEC_MASK | UCODE_ID_CP_MEC_JT1_MASK;
+		break;
+	case UCODE_ID_CP_MEC_JT2:
+		result = UCODE_ID_CP_MEC_MASK | UCODE_ID_CP_MEC_JT2_MASK;
+		break;
+	case UCODE_ID_RLC_G:
+		result = UCODE_ID_RLC_G_MASK;
+		break;
+	default:
+		printk(KERN_ERR "UCode type is out of range!");
+		result = 0;
+	}
+
+	return result;
+}
+
+/* Populate one firmware image to the data structure */
+static int fiji_populate_single_firmware_entry(struct pp_smumgr *smumgr,
+		uint32_t fw_type, struct SMU_Entry *entry)
+{
+	int result;
+	struct cgs_firmware_info info = {0};
+
+	result = cgs_get_firmware_info(
+			smumgr->device,
+			fiji_convert_fw_type_to_cgs(fw_type),
+			&info);
+
+	if (!result) {
+		entry->version = 0;
+		entry->id = (uint16_t)fw_type;
+		entry->image_addr_high = smu_upper_32_bits(info.mc_addr);
+		entry->image_addr_low = smu_lower_32_bits(info.mc_addr);
+		entry->meta_data_addr_high = 0;
+		entry->meta_data_addr_low = 0;
+		entry->data_size_byte = info.image_size;
+		entry->num_register_entries = 0;
+
+		if (fw_type == UCODE_ID_RLC_G)
+			entry->flags = 1;
+		else
+			entry->flags = 0;
+	}
+
+	return result;
+}
+
+static int fiji_request_smu_load_fw(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+	uint32_t fw_to_load;
+	struct SMU_DRAMData_TOC *toc;
+
+	if (priv->soft_regs_start)
+		cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+				priv->soft_regs_start +
+				offsetof(SMU73_SoftRegisters, UcodeLoadStatus),
+				0x0);
+
+	toc = (struct SMU_DRAMData_TOC *)priv->header;
+	toc->num_entries = 0;
+	toc->structure_version = 1;
+
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_RLC_G, &toc->entry[toc->num_entries++]),
+			"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_CE, &toc->entry[toc->num_entries++]),
+			"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
+			"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
+			"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
+			"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
+					"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
+					"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
+					"Failed to Get Firmware Entry.\n" , return -1 );
+	PP_ASSERT_WITH_CODE(
+			0 == fiji_populate_single_firmware_entry(smumgr,
+					UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
+					"Failed to Get Firmware Entry.\n" , return -1 );
+
+	fiji_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_HI,
+			priv->header_buffer.mc_addr_high);
+	fiji_send_msg_to_smc_with_parameter(smumgr,PPSMC_MSG_DRV_DRAM_ADDR_LO,
+			priv->header_buffer.mc_addr_low);
+
+	fw_to_load = UCODE_ID_RLC_G_MASK
+			+ UCODE_ID_SDMA0_MASK
+			+ UCODE_ID_SDMA1_MASK
+			+ UCODE_ID_CP_CE_MASK
+			+ UCODE_ID_CP_ME_MASK
+			+ UCODE_ID_CP_PFP_MASK
+			+ UCODE_ID_CP_MEC_MASK
+			+ UCODE_ID_CP_MEC_JT1_MASK
+			+ UCODE_ID_CP_MEC_JT2_MASK;
+
+	if (fiji_send_msg_to_smc_with_parameter(smumgr,
+			PPSMC_MSG_LoadUcodes, fw_to_load))
+		printk(KERN_ERR "Fail to Request SMU Load uCode");
+
+	return 0;
+}
+
+
+/* Check if the FW has been loaded, SMU will not return
+ * if loading has not finished.
+ */
+static int fiji_check_fw_load_finish(struct pp_smumgr *smumgr,
+		uint32_t fw_type)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+	uint32_t mask = fiji_get_mask_for_firmware_type(fw_type);
+
+	/* Check SOFT_REGISTERS_TABLE_28.UcodeLoadStatus */
+	if (smum_wait_on_indirect_register(smumgr, mmSMC_IND_INDEX,
+			priv->soft_regs_start +
+			offsetof(SMU73_SoftRegisters, UcodeLoadStatus),
+			mask, mask)) {
+		printk(KERN_ERR "check firmware loading failed\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+
+static int fiji_reload_firmware(struct pp_smumgr *smumgr)
+{
+	return smumgr->smumgr_funcs->start_smu(smumgr);
+}
+
+static bool fiji_is_hw_virtualization_enabled(struct pp_smumgr *smumgr)
+{
+	uint32_t value;
+
+	value = cgs_read_register(smumgr->device, mmBIF_IOV_FUNC_IDENTIFIER);
+	if (value & BIF_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK) {
+		/* driver reads on SR-IOV enabled PF: 0x80000000
+		 * driver reads on SR-IOV enabled VF: 0x80000001
+		 * driver reads on SR-IOV disabled:   0x00000000
+		 */
+		return true;
+	}
+	return false;
+}
+
+static int fiji_request_smu_specific_fw_load(struct pp_smumgr *smumgr, uint32_t fw_type)
+{
+	if (fiji_is_hw_virtualization_enabled(smumgr)) {
+		uint32_t masks = fiji_get_mask_for_firmware_type(fw_type);
+		if (fiji_send_msg_to_smc_with_parameter_without_waiting(smumgr,
+				PPSMC_MSG_LoadUcodes, masks))
+			printk(KERN_ERR "Fail to Request SMU Load uCode");
+	}
+	/* For non-virtualization cases,
+	 * SMU loads all FWs at once in fiji_request_smu_load_fw.
+	 */
+	return 0;
+}
+
+static int fiji_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+
+	/* Wait for smc boot up */
+	/* SMUM_WAIT_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+		RCU_UC_EVENTS, boot_seq_done, 0); */
+
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	result = fiji_upload_smu_firmware_image(smumgr);
+	if (result)
+		return result;
+
+	/* Clear status */
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+			ixSMU_STATUS, 0);
+
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	/* De-assert reset */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Wait for ROM firmware to initialize interrupt hendler */
+	/*SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, SMC_IND,
+			SMC_INTR_CNTL_MASK_0, 0x10040, 0xFFFFFFFF); */
+
+	/* Set SMU Auto Start */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMU_INPUT_DATA, AUTO_START, 1);
+
+	/* Clear firmware interrupt enable flag */
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+			ixFIRMWARE_FLAGS, 0);
+
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND, RCU_UC_EVENTS,
+			INTERRUPTS_ENABLED, 1);
+
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, 0x20000);
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, PPSMC_MSG_Test);
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	/* Wait for done bit to be set */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+			SMU_STATUS, SMU_DONE, 0);
+
+	/* Check pass/failed indicator */
+	if (1 != SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMU_STATUS, SMU_PASS)) {
+		PP_ASSERT_WITH_CODE(false,
+				"SMU Firmware start failed!", return -1);
+	}
+
+	/* Wait for firmware to initialize */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND,
+			FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return result;
+}
+
+static int fiji_start_smu_in_non_protection_mode(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+
+	/* wait for smc boot up */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+			RCU_UC_EVENTS, boot_seq_done, 0);
+
+	/* Clear firmware interrupt enable flag */
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+			ixFIRMWARE_FLAGS, 0);
+
+	/* Assert reset */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	result = fiji_upload_smu_firmware_image(smumgr);
+	if (result)
+		return result;
+
+	/* Set smc instruct start point at 0x0 */
+	fiji_program_jump_on_start(smumgr);
+
+	/* Enable clock */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	/* De-assert reset */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Wait for firmware to initialize */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND,
+			FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return result;
+}
+
+int fiji_setup_pwr_virus(struct pp_smumgr *smumgr)
+{
+	int i, result = -1;
+	uint32_t reg, data;
+	PWR_Command_Table *virus = PwrVirusTable;
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	priv->avfs.AvfsBtcStatus = AVFS_LOAD_VIRUS;
+	for (i = 0; (i < PWR_VIRUS_TABLE_SIZE); i++) {
+		switch (virus->command) {
+		case PwrCmdWrite:
+			reg  = virus->reg;
+			data = virus->data;
+			cgs_write_register(smumgr->device, reg, data);
+			break;
+		case PwrCmdEnd:
+			priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_LOADED;
+			result = 0;
+			break;
+		default:
+			printk(KERN_ERR "Table Exit with Invalid Command!");
+			priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_FAIL;
+			result = -1;
+			break;
+		}
+		virus++;
+	}
+	return result;
+}
+
+static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	priv->avfs.AvfsBtcStatus = AVFS_BTC_STARTED;
+	if (priv->avfs.AvfsBtcParam) {
+		if (!fiji_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_PerformBtc, priv->avfs.AvfsBtcParam)) {
+			if (!fiji_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs)) {
+				priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_UNSAVED;
+				result = 0;
+			} else {
+				printk(KERN_ERR "[AVFS][fiji_start_avfs_btc] Attempt"
+						" to Enable AVFS Failed!");
+				fiji_send_msg_to_smc(smumgr, PPSMC_MSG_DisableAvfs);
+				result = -1;
+			}
+		} else {
+			printk(KERN_ERR "[AVFS][fiji_start_avfs_btc] "
+					"PerformBTC SMU msg failed");
+			result = -1;
+		}
+	}
+	/* Soft-Reset to reset the engine before loading uCode */
+	 /* halt */
+	cgs_write_register(smumgr->device, mmCP_MEC_CNTL, 0x50000000);
+	/* reset everything */
+	cgs_write_register(smumgr->device, mmGRBM_SOFT_RESET, 0xffffffff);
+	/* clear reset */
+	cgs_write_register(smumgr->device, mmGRBM_SOFT_RESET, 0);
+
+	return result;
+}
+
+int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+	uint32_t table_start;
+	uint32_t charz_freq_addr, inversion_voltage_addr, charz_freq;
+	uint16_t inversion_voltage;
+
+	charz_freq = 0x30750000; /* In 10KHz units 0x00007530 Actual value */
+	inversion_voltage = 0x1A04; /* mV Q14.2 0x41A Actual value */
+
+	PP_ASSERT_WITH_CODE(0 == fiji_read_smc_sram_dword(smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU73_Firmware_Header,
+					PmFuseTable), &table_start, 0x40000),
+			"[AVFS][Fiji_SetupGfxLvlStruct] SMU could not communicate "
+			"starting address of PmFuse structure",
+			return -1;);
+
+	charz_freq_addr = table_start +
+			offsetof(struct SMU73_Discrete_PmFuses, PsmCharzFreq);
+	inversion_voltage_addr = table_start +
+			offsetof(struct SMU73_Discrete_PmFuses, InversionVoltage);
+
+	result = fiji_copy_bytes_to_smc(smumgr, charz_freq_addr,
+			(uint8_t *)(&charz_freq), sizeof(charz_freq), 0x40000);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"[AVFS][fiji_setup_pm_fuse_for_avfs] charz_freq could not "
+			"be populated.", return -1;);
+
+	result = fiji_copy_bytes_to_smc(smumgr, inversion_voltage_addr,
+			(uint8_t *)(&inversion_voltage), sizeof(inversion_voltage), 0x40000);
+	PP_ASSERT_WITH_CODE(0 == result, "[AVFS][fiji_setup_pm_fuse_for_avfs] "
+			"charz_freq could not be populated.", return -1;);
+
+	return result;
+}
+
+int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
+{
+	int32_t vr_config;
+	uint32_t table_start;
+	uint32_t level_addr, vr_config_addr;
+	uint32_t level_size = sizeof(avfs_graphics_level);
+
+	PP_ASSERT_WITH_CODE(0 == fiji_read_smc_sram_dword(smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, DpmTable),
+			&table_start, 0x40000),
+			"[AVFS][Fiji_SetupGfxLvlStruct] SMU could not "
+			"communicate starting address of DPM table",
+			return -1;);
+
+	/* Default value for vr_config =
+	 * VR_MERGED_WITH_VDDC + VR_STATIC_VOLTAGE(VDDCI) */
+	vr_config = 0x01000500;   /* Real value:0x50001 */
+
+	vr_config_addr = table_start +
+			offsetof(SMU73_Discrete_DpmTable, VRConfig);
+
+	PP_ASSERT_WITH_CODE(0 == fiji_copy_bytes_to_smc(smumgr, vr_config_addr,
+			(uint8_t *)&vr_config, sizeof(int32_t), 0x40000),
+			"[AVFS][Fiji_SetupGfxLvlStruct] Problems copying "
+			"vr_config value over to SMC",
+			return -1;);
+
+	level_addr = table_start + offsetof(SMU73_Discrete_DpmTable, GraphicsLevel);
+
+	PP_ASSERT_WITH_CODE(0 == fiji_copy_bytes_to_smc(smumgr, level_addr,
+			(uint8_t *)(&avfs_graphics_level), level_size, 0x40000),
+			"[AVFS][Fiji_SetupGfxLvlStruct] Copying of DPM table failed!",
+			return -1;);
+
+	return 0;
+}
+
+/* Work in Progress */
+int fiji_restore_vft_table(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	if (AVFS_BTC_COMPLETED_SAVED == priv->avfs.AvfsBtcStatus) {
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
+		return 0;
+	} else
+		return -EINVAL;
+}
+
+/* Work in Progress */
+int fiji_save_vft_table(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	if (AVFS_BTC_COMPLETED_SAVED == priv->avfs.AvfsBtcStatus) {
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
+		return 0;
+	} else
+		return -EINVAL;
+}
+
+int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	switch (priv->avfs.AvfsBtcStatus) {
+	case AVFS_BTC_COMPLETED_SAVED: /*S3 State - Pre SMU Start */
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_RESTOREVFT_FAILED;
+		PP_ASSERT_WITH_CODE(0 == fiji_restore_vft_table(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Could not Copy Graphics "
+				"Level table over to SMU",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
+		break;
+	case AVFS_BTC_COMPLETED_RESTORED: /*S3 State - Post SMU Start*/
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
+		PP_ASSERT_WITH_CODE(0 == fiji_send_msg_to_smc(smumgr,
+				PPSMC_MSG_VftTableIsValid),
+				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
+				"correctly to VftTableIsValid Msg",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
+		PP_ASSERT_WITH_CODE(0 == fiji_send_msg_to_smc(smumgr,
+				PPSMC_MSG_EnableAvfs),
+				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
+				"correctly to EnableAvfs Message Msg",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_SAVED;
+		break;
+	case AVFS_BTC_BOOT: /*Cold Boot State - Post SMU Start*/
+		if (!smu_started)
+			break;
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_FAILED;
+		PP_ASSERT_WITH_CODE(0 == fiji_setup_pm_fuse_for_avfs(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Failure at "
+				"fiji_setup_pm_fuse_for_avfs",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_DPMTABLESETUP_FAILED;
+		PP_ASSERT_WITH_CODE(0 == fiji_setup_graphics_level_structure(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Could not Copy Graphics Level"
+				" table over to SMU",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_FAIL;
+		PP_ASSERT_WITH_CODE(0 == fiji_setup_pwr_virus(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Could not setup "
+				"Pwr Virus for AVFS ",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_FAILED;
+		PP_ASSERT_WITH_CODE(0 == fiji_start_avfs_btc(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Failure at "
+				"fiji_start_avfs_btc. AVFS Disabled",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_SAVEVFT_FAILED;
+		PP_ASSERT_WITH_CODE(0 == fiji_save_vft_table(smumgr),
+				"[AVFS][fiji_avfs_event_mgr] Could not save VFT Table",
+				return -1;);
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_SAVED;
+		break;
+	case AVFS_BTC_DISABLED: /* Do nothing */
+		break;
+	case AVFS_BTC_NOTSUPPORTED: /* Do nothing */
+		break;
+	default:
+		printk(KERN_ERR "[AVFS] Something is broken. See log!");
+		break;
+	}
+	return 0;
+}
+
+static int fiji_start_smu(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+
+	/* Only start SMC if SMC RAM is not running */
+	if (!fiji_is_smc_ram_running(smumgr)) {
+		fiji_avfs_event_mgr(smumgr, false);
+
+		/* Check if SMU is running in protected mode */
+		if (0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device,
+				CGS_IND_REG__SMC,
+				SMU_FIRMWARE, SMU_MODE)) {
+			result = fiji_start_smu_in_non_protection_mode(smumgr);
+			if (result)
+				return result;
+		} else {
+			result = fiji_start_smu_in_protection_mode(smumgr);
+			if (result)
+				return result;
+		}
+		fiji_avfs_event_mgr(smumgr, true);
+	}
+
+	/* To initialize all clock gating before RLC loaded and running.*/
+	cgs_set_clockgating_state(smumgr->device,
+			AMD_IP_BLOCK_TYPE_GFX, AMD_CG_STATE_GATE);
+	cgs_set_clockgating_state(smumgr->device,
+			AMD_IP_BLOCK_TYPE_GMC, AMD_CG_STATE_GATE);
+	cgs_set_clockgating_state(smumgr->device,
+			AMD_IP_BLOCK_TYPE_SDMA, AMD_CG_STATE_GATE);
+	cgs_set_clockgating_state(smumgr->device,
+			AMD_IP_BLOCK_TYPE_COMMON, AMD_CG_STATE_GATE);
+
+	/* Setup SoftRegsStart here for register lookup in case
+	 * DummyBackEnd is used and ProcessFirmwareHeader is not executed
+	 */
+	fiji_read_smc_sram_dword(smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, SoftRegisters),
+			&(priv->soft_regs_start), 0x40000);
+
+	result = fiji_request_smu_load_fw(smumgr);
+
+	return result;
+}
+
+static bool fiji_is_hw_avfs_present(struct pp_smumgr *smumgr)
+{
+
+	uint32_t efuse = 0;
+	uint32_t mask = (1 << ((AVFS_EN_MSB - AVFS_EN_LSB) + 1)) - 1;
+
+	if (!atomctrl_read_efuse(smumgr->device, AVFS_EN_LSB, AVFS_EN_MSB,
+			mask, &efuse)) {
+		if (efuse)
+			return true;
+	}
+	return false;
+}
+
+/**
+* Write a 32bit value to the SMC SRAM space.
+* ALL PARAMETERS ARE IN HOST BYTE ORDER.
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    smc_addr the address in the SMC RAM to access.
+* @param    value to write to the SMC SRAM.
+*/
+static int fiji_smu_init(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+	uint64_t mc_addr;
+
+	priv->header_buffer.data_size =
+			((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
+	smu_allocate_memory(smumgr->device,
+			priv->header_buffer.data_size,
+			CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
+			PAGE_SIZE,
+			&mc_addr,
+			&priv->header_buffer.kaddr,
+			&priv->header_buffer.handle);
+
+	priv->header = priv->header_buffer.kaddr;
+	priv->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	priv->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	PP_ASSERT_WITH_CODE((NULL != priv->header),
+			"Out of memory.",
+			kfree(smumgr->backend);
+			cgs_free_gpu_mem(smumgr->device,
+			(cgs_handle_t)priv->header_buffer.handle);
+			return -1);
+
+	priv->avfs.AvfsBtcStatus = AVFS_BTC_BOOT;
+	if (fiji_is_hw_avfs_present(smumgr))
+		/* AVFS Parameter
+		 * 0 - BTC DC disabled, BTC AC disabled
+		 * 1 - BTC DC enabled,  BTC AC disabled
+		 * 2 - BTC DC disabled, BTC AC enabled
+		 * 3 - BTC DC enabled,  BTC AC enabled
+		 * Default is 0 - BTC DC disabled, BTC AC disabled
+		 */
+		priv->avfs.AvfsBtcParam = 0;
+	else
+		priv->avfs.AvfsBtcStatus = AVFS_BTC_NOTSUPPORTED;
+
+	priv->acpi_optimization = 1;
+
+	return 0;
+}
+
+static int fiji_smu_fini(struct pp_smumgr *smumgr)
+{
+	if (smumgr->backend) {
+		kfree(smumgr->backend);
+		smumgr->backend = NULL;
+	}
+	return 0;
+}
+
+static const struct pp_smumgr_func fiji_smu_funcs = {
+	.smu_init = &fiji_smu_init,
+	.smu_fini = &fiji_smu_fini,
+	.start_smu = &fiji_start_smu,
+	.check_fw_load_finish = &fiji_check_fw_load_finish,
+	.request_smu_load_fw = &fiji_reload_firmware,
+	.request_smu_load_specific_fw = &fiji_request_smu_specific_fw_load,
+	.send_msg_to_smc = &fiji_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &fiji_send_msg_to_smc_with_parameter,
+	.download_pptable_settings = NULL,
+	.upload_pptable_settings = NULL,
+};
+
+int fiji_smum_init(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *fiji_smu = NULL;
+
+	fiji_smu = kzalloc(sizeof(struct fiji_smumgr), GFP_KERNEL);
+
+	if (fiji_smu == NULL)
+		return -ENOMEM;
+
+	smumgr->backend = fiji_smu;
+	smumgr->smumgr_funcs = &fiji_smu_funcs;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
new file mode 100644
index 000000000000..8cd22d9c9140
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _FIJI_SMUMANAGER_H_
+#define _FIJI_SMUMANAGER_H_
+
+enum AVFS_BTC_STATUS {
+	AVFS_BTC_BOOT = 0,
+	AVFS_BTC_BOOT_STARTEDSMU,
+	AVFS_LOAD_VIRUS,
+	AVFS_BTC_VIRUS_LOADED,
+	AVFS_BTC_VIRUS_FAIL,
+	AVFS_BTC_STARTED,
+	AVFS_BTC_FAILED,
+	AVFS_BTC_RESTOREVFT_FAILED,
+	AVFS_BTC_SAVEVFT_FAILED,
+	AVFS_BTC_DPMTABLESETUP_FAILED,
+	AVFS_BTC_COMPLETED_UNSAVED,
+	AVFS_BTC_COMPLETED_SAVED,
+	AVFS_BTC_COMPLETED_RESTORED,
+	AVFS_BTC_DISABLED,
+	AVFS_BTC_NOTSUPPORTED,
+	AVFS_BTC_SMUMSG_ERROR
+};
+
+struct fiji_smu_avfs {
+	enum AVFS_BTC_STATUS AvfsBtcStatus;
+	uint32_t           AvfsBtcParam;
+};
+
+struct fiji_buffer_entry {
+	uint32_t data_size;
+	uint32_t mc_addr_low;
+	uint32_t mc_addr_high;
+	void *kaddr;
+	unsigned long  handle;
+};
+
+struct fiji_smumgr {
+	uint8_t        *header;
+	uint8_t        *mec_image;
+	uint32_t        soft_regs_start;
+	struct fiji_smu_avfs avfs;
+	uint32_t        acpi_optimization;
+
+	struct fiji_buffer_entry header_buffer;
+};
+
+int fiji_smum_init(struct pp_smumgr *smumgr);
+int fiji_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
+		uint32_t *value, uint32_t limit);
+int fiji_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
+		uint32_t value, uint32_t limit);
+int fiji_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smcStartAddress,
+		const uint8_t *src,	uint32_t byteCount, uint32_t limit);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
new file mode 100644
index 000000000000..063ae71c9830
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "pp_instance.h"
+#include "smumgr.h"
+#include "cgs_common.h"
+#include "linux/delay.h"
+#include "cz_smumgr.h"
+#include "tonga_smumgr.h"
+#include "fiji_smumgr.h"
+
+int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
+{
+	struct pp_smumgr *smumgr;
+
+	if ((handle == NULL) || (pp_init == NULL))
+		return -EINVAL;
+
+	smumgr = kzalloc(sizeof(struct pp_smumgr), GFP_KERNEL);
+	if (smumgr == NULL)
+		return -ENOMEM;
+
+	smumgr->device = pp_init->device;
+	smumgr->chip_family = pp_init->chip_family;
+	smumgr->chip_id = pp_init->chip_id;
+	smumgr->hw_revision = pp_init->rev_id;
+	smumgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
+	smumgr->reload_fw = 1;
+	handle->smu_mgr = smumgr;
+
+	switch (smumgr->chip_family) {
+	case AMD_FAMILY_CZ:
+		cz_smum_init(smumgr);
+		break;
+	case AMD_FAMILY_VI:
+		switch (smumgr->chip_id) {
+		case CHIP_TONGA:
+			tonga_smum_init(smumgr);
+			break;
+		case CHIP_FIJI:
+			fiji_smum_init(smumgr);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	default:
+		kfree(smumgr);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int smum_fini(struct pp_smumgr *smumgr)
+{
+	kfree(smumgr);
+	return 0;
+}
+
+int smum_get_argument(struct pp_smumgr *smumgr)
+{
+	if (NULL != smumgr->smumgr_funcs->get_argument)
+		return smumgr->smumgr_funcs->get_argument(smumgr);
+
+	return 0;
+}
+
+int smum_download_powerplay_table(struct pp_smumgr *smumgr,
+								void **table)
+{
+	if (NULL != smumgr->smumgr_funcs->download_pptable_settings)
+		return smumgr->smumgr_funcs->download_pptable_settings(smumgr,
+									table);
+
+	return 0;
+}
+
+int smum_upload_powerplay_table(struct pp_smumgr *smumgr)
+{
+	if (NULL != smumgr->smumgr_funcs->upload_pptable_settings)
+		return smumgr->smumgr_funcs->upload_pptable_settings(smumgr);
+
+	return 0;
+}
+
+int smum_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	if (smumgr == NULL || smumgr->smumgr_funcs->send_msg_to_smc == NULL)
+		return -EINVAL;
+
+	return smumgr->smumgr_funcs->send_msg_to_smc(smumgr, msg);
+}
+
+int smum_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
+					uint16_t msg, uint32_t parameter)
+{
+	if (smumgr == NULL ||
+		smumgr->smumgr_funcs->send_msg_to_smc_with_parameter == NULL)
+		return -EINVAL;
+	return smumgr->smumgr_funcs->send_msg_to_smc_with_parameter(
+						smumgr, msg, parameter);
+}
+
+/*
+ * Returns once the part of the register indicated by the mask has
+ * reached the given value.
+ */
+int smum_wait_on_register(struct pp_smumgr *smumgr,
+				uint32_t index,
+				uint32_t value, uint32_t mask)
+{
+	uint32_t i;
+	uint32_t cur_value;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	for (i = 0; i < smumgr->usec_timeout; i++) {
+		cur_value = cgs_read_register(smumgr->device, index);
+		if ((cur_value & mask) == (value & mask))
+			break;
+		udelay(1);
+	}
+
+	/* timeout means wrong logic*/
+	if (i == smumgr->usec_timeout)
+		return -1;
+
+	return 0;
+}
+
+int smum_wait_for_register_unequal(struct pp_smumgr *smumgr,
+					uint32_t index,
+					uint32_t value, uint32_t mask)
+{
+	uint32_t i;
+	uint32_t cur_value;
+
+	if (smumgr == NULL)
+		return -EINVAL;
+
+	for (i = 0; i < smumgr->usec_timeout; i++) {
+		cur_value = cgs_read_register(smumgr->device,
+									index);
+		if ((cur_value & mask) != (value & mask))
+			break;
+		udelay(1);
+	}
+
+	/* timeout means wrong logic */
+	if (i == smumgr->usec_timeout)
+		return -1;
+
+	return 0;
+}
+
+
+/*
+ * Returns once the part of the register indicated by the mask
+ * has reached the given value.The indirect space is described by
+ * giving the memory-mapped index of the indirect index register.
+ */
+int smum_wait_on_indirect_register(struct pp_smumgr *smumgr,
+					uint32_t indirect_port,
+					uint32_t index,
+					uint32_t value,
+					uint32_t mask)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	cgs_write_register(smumgr->device, indirect_port, index);
+	return smum_wait_on_register(smumgr, indirect_port + 1,
+						mask, value);
+}
+
+void smum_wait_for_indirect_register_unequal(
+						struct pp_smumgr *smumgr,
+						uint32_t indirect_port,
+						uint32_t index,
+						uint32_t value,
+						uint32_t mask)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return;
+	cgs_write_register(smumgr->device, indirect_port, index);
+	smum_wait_for_register_unequal(smumgr, indirect_port + 1,
+						value, mask);
+}
+
+int smu_allocate_memory(void *device, uint32_t size,
+			 enum cgs_gpu_mem_type type,
+			 uint32_t byte_align, uint64_t *mc_addr,
+			 void **kptr, void *handle)
+{
+	int ret = 0;
+	cgs_handle_t cgs_handle;
+
+	if (device == NULL || handle == NULL ||
+	    mc_addr == NULL || kptr == NULL)
+		return -EINVAL;
+
+	ret = cgs_alloc_gpu_mem(device, type, size, byte_align,
+				0, 0, (cgs_handle_t *)handle);
+	if (ret)
+		return -ENOMEM;
+
+	cgs_handle = *(cgs_handle_t *)handle;
+
+	ret = cgs_gmap_gpu_mem(device, cgs_handle, mc_addr);
+	if (ret)
+		goto error_gmap;
+
+	ret = cgs_kmap_gpu_mem(device, cgs_handle, kptr);
+	if (ret)
+		goto error_kmap;
+
+	return 0;
+
+error_kmap:
+	cgs_gunmap_gpu_mem(device, cgs_handle);
+
+error_gmap:
+	cgs_free_gpu_mem(device, cgs_handle);
+	return ret;
+}
+
+int smu_free_memory(void *device, void *handle)
+{
+	cgs_handle_t cgs_handle = (cgs_handle_t)handle;
+
+	if (device == NULL || handle == NULL)
+		return -EINVAL;
+
+	cgs_kunmap_gpu_mem(device, cgs_handle);
+	cgs_gunmap_gpu_mem(device, cgs_handle);
+	cgs_free_gpu_mem(device, cgs_handle);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
new file mode 100644
index 000000000000..ebdb43a8daef
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -0,0 +1,819 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/gfp.h>
+
+#include "smumgr.h"
+#include "tonga_smumgr.h"
+#include "pp_debug.h"
+#include "smu_ucode_xfer_vi.h"
+#include "tonga_ppsmc.h"
+#include "smu/smu_7_1_2_d.h"
+#include "smu/smu_7_1_2_sh_mask.h"
+#include "cgs_common.h"
+
+#define TONGA_SMC_SIZE			0x20000
+#define BUFFER_SIZE			80000
+#define MAX_STRING_SIZE			15
+#define BUFFER_SIZETWO              131072 /*128 *1024*/
+
+/**
+* Set the address for reading/writing the SMC SRAM space.
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    smcAddress the address in the SMC RAM to access.
+*/
+static int tonga_set_smc_sram_address(struct pp_smumgr *smumgr,
+				uint32_t smcAddress, uint32_t limit)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+	PP_ASSERT_WITH_CODE((0 == (3 & smcAddress)),
+		"SMC address must be 4 byte aligned.",
+		return -1;);
+
+	PP_ASSERT_WITH_CODE((limit > (smcAddress + 3)),
+		"SMC address is beyond the SMC RAM area.",
+		return -1;);
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, smcAddress);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0);
+
+	return 0;
+}
+
+/**
+* Copy bytes from an array into the SMC RAM space.
+*
+* @param    smumgr  the address of the powerplay SMU manager.
+* @param    smcStartAddress the start address in the SMC RAM to copy bytes to.
+* @param    src the byte array to copy the bytes from.
+* @param    byteCount the number of bytes to copy.
+*/
+int tonga_copy_bytes_to_smc(struct pp_smumgr *smumgr,
+		uint32_t smcStartAddress, const uint8_t *src,
+		uint32_t byteCount, uint32_t limit)
+{
+	uint32_t addr;
+	uint32_t data, orig_data;
+	int result = 0;
+	uint32_t extra_shift;
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+	PP_ASSERT_WITH_CODE((0 == (3 & smcStartAddress)),
+		"SMC address must be 4 byte aligned.",
+		return 0;);
+
+	PP_ASSERT_WITH_CODE((limit > (smcStartAddress + byteCount)),
+		"SMC address is beyond the SMC RAM area.",
+		return 0;);
+
+	addr = smcStartAddress;
+
+	while (byteCount >= 4) {
+		/*
+		 * Bytes are written into the
+		 * SMC address space with the MSB first
+		 */
+		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
+
+		result = tonga_set_smc_sram_address(smumgr, addr, limit);
+
+		if (result)
+			goto out;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
+
+		src += 4;
+		byteCount -= 4;
+		addr += 4;
+	}
+
+	if (0 != byteCount) {
+		/* Now write odd bytes left, do a read modify write cycle */
+		data = 0;
+
+		result = tonga_set_smc_sram_address(smumgr, addr, limit);
+		if (result)
+			goto out;
+
+		orig_data = cgs_read_register(smumgr->device,
+							mmSMC_IND_DATA_0);
+		extra_shift = 8 * (4 - byteCount);
+
+		while (byteCount > 0) {
+			data = (data << 8) + *src++;
+			byteCount--;
+		}
+
+		data <<= extra_shift;
+		data |= (orig_data & ~((~0UL) << extra_shift));
+
+		result = tonga_set_smc_sram_address(smumgr, addr, limit);
+		if (result)
+			goto out;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
+	}
+
+out:
+	return result;
+}
+
+
+int tonga_program_jump_on_start(struct pp_smumgr *smumgr)
+{
+	static unsigned char pData[] = { 0xE0, 0x00, 0x80, 0x40 };
+
+	tonga_copy_bytes_to_smc(smumgr, 0x0, pData, 4, sizeof(pData)+1);
+
+	return 0;
+}
+
+/**
+* Return if the SMC is currently running.
+*
+* @param    smumgr  the address of the powerplay hardware manager.
+*/
+static int tonga_is_smc_ram_running(struct pp_smumgr *smumgr)
+{
+	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
+			&& (0x20100 <= cgs_read_ind_register(smumgr->device,
+					CGS_IND_REG__SMC, ixSMC_PC_C)));
+}
+
+static int tonga_send_msg_to_smc_offset(struct pp_smumgr *smumgr)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, 0x20000);
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, PPSMC_MSG_Test);
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	return 0;
+}
+
+/**
+* Send a message to the SMC, and wait for its response.
+*
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    msg the message to send.
+* @return   The response that came from the SMC.
+*/
+static int tonga_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	if (!tonga_is_smc_ram_running(smumgr))
+		return -1;
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	PP_ASSERT_WITH_CODE(
+		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
+		"Failed to send Previous Message.",
+		);
+
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	PP_ASSERT_WITH_CODE(
+		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
+		"Failed to send Message.",
+		);
+
+	return 0;
+}
+
+/*
+* Send a message to the SMC, and do not wait for its response.
+*
+* @param    smumgr  the address of the powerplay hardware manager.
+* @param    msg the message to send.
+* @return   The response that came from the SMC.
+*/
+static int tonga_send_msg_to_smc_without_waiting
+				(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	PP_ASSERT_WITH_CODE(
+		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
+		"Failed to send Previous Message.",
+		);
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+
+	return 0;
+}
+
+/*
+* Send a message to the SMC with parameter
+*
+* @param    smumgr:  the address of the powerplay hardware manager.
+* @param    msg: the message to send.
+* @param    parameter: the parameter to send
+* @return   The response that came from the SMC.
+*/
+static int tonga_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
+				uint16_t msg, uint32_t parameter)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	if (!tonga_is_smc_ram_running(smumgr))
+		return PPSMC_Result_Failed;
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+
+	return tonga_send_msg_to_smc(smumgr, msg);
+}
+
+/*
+* Send a message to the SMC with parameter, do not wait for response
+*
+* @param    smumgr:  the address of the powerplay hardware manager.
+* @param    msg: the message to send.
+* @param    parameter: the parameter to send
+* @return   The response that came from the SMC.
+*/
+static int tonga_send_msg_to_smc_with_parameter_without_waiting(
+			struct pp_smumgr *smumgr,
+			uint16_t msg, uint32_t parameter)
+{
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+
+	return tonga_send_msg_to_smc_without_waiting(smumgr, msg);
+}
+
+/*
+ * Read a 32bit value from the SMC SRAM space.
+ * ALL PARAMETERS ARE IN HOST BYTE ORDER.
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    smcAddress the address in the SMC RAM to access.
+ * @param    value and output parameter for the data read from the SMC SRAM.
+ */
+int tonga_read_smc_sram_dword(struct pp_smumgr *smumgr,
+					uint32_t smcAddress, uint32_t *value,
+					uint32_t limit)
+{
+	int result;
+
+	result = tonga_set_smc_sram_address(smumgr, smcAddress, limit);
+
+	if (0 != result)
+		return result;
+
+	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
+
+	return 0;
+}
+
+/*
+ * Write a 32bit value to the SMC SRAM space.
+ * ALL PARAMETERS ARE IN HOST BYTE ORDER.
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    smcAddress the address in the SMC RAM to access.
+ * @param    value to write to the SMC SRAM.
+ */
+int tonga_write_smc_sram_dword(struct pp_smumgr *smumgr,
+					uint32_t smcAddress, uint32_t value,
+					uint32_t limit)
+{
+	int result;
+
+	result = tonga_set_smc_sram_address(smumgr, smcAddress, limit);
+
+	if (0 != result)
+		return result;
+
+	cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, value);
+
+	return 0;
+}
+
+static int tonga_smu_fini(struct pp_smumgr *smumgr)
+{
+	if (smumgr->backend != NULL) {
+		kfree(smumgr->backend);
+		smumgr->backend = NULL;
+	}
+	return 0;
+}
+
+static enum cgs_ucode_id tonga_convert_fw_type_to_cgs(uint32_t fw_type)
+{
+	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
+
+	switch (fw_type) {
+	case UCODE_ID_SMU:
+		result = CGS_UCODE_ID_SMU;
+		break;
+	case UCODE_ID_SDMA0:
+		result = CGS_UCODE_ID_SDMA0;
+		break;
+	case UCODE_ID_SDMA1:
+		result = CGS_UCODE_ID_SDMA1;
+		break;
+	case UCODE_ID_CP_CE:
+		result = CGS_UCODE_ID_CP_CE;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = CGS_UCODE_ID_CP_PFP;
+		break;
+	case UCODE_ID_CP_ME:
+		result = CGS_UCODE_ID_CP_ME;
+		break;
+	case UCODE_ID_CP_MEC:
+		result = CGS_UCODE_ID_CP_MEC;
+		break;
+	case UCODE_ID_CP_MEC_JT1:
+		result = CGS_UCODE_ID_CP_MEC_JT1;
+		break;
+	case UCODE_ID_CP_MEC_JT2:
+		result = CGS_UCODE_ID_CP_MEC_JT2;
+		break;
+	case UCODE_ID_RLC_G:
+		result = CGS_UCODE_ID_RLC_G;
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+/**
+ * Convert the PPIRI firmware type to SMU type mask.
+ * For MEC, we need to check all MEC related type
+*/
+static uint16_t tonga_get_mask_for_firmware_type(uint16_t firmwareType)
+{
+	uint16_t result = 0;
+
+	switch (firmwareType) {
+	case UCODE_ID_SDMA0:
+		result = UCODE_ID_SDMA0_MASK;
+		break;
+	case UCODE_ID_SDMA1:
+		result = UCODE_ID_SDMA1_MASK;
+		break;
+	case UCODE_ID_CP_CE:
+		result = UCODE_ID_CP_CE_MASK;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = UCODE_ID_CP_PFP_MASK;
+		break;
+	case UCODE_ID_CP_ME:
+		result = UCODE_ID_CP_ME_MASK;
+		break;
+	case UCODE_ID_CP_MEC:
+	case UCODE_ID_CP_MEC_JT1:
+	case UCODE_ID_CP_MEC_JT2:
+		result = UCODE_ID_CP_MEC_MASK;
+		break;
+	case UCODE_ID_RLC_G:
+		result = UCODE_ID_RLC_G_MASK;
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+/**
+ * Check if the FW has been loaded,
+ * SMU will not return if loading has not finished.
+*/
+static int tonga_check_fw_load_finish(struct pp_smumgr *smumgr, uint32_t fwType)
+{
+	uint16_t fwMask = tonga_get_mask_for_firmware_type(fwType);
+
+	if (0 != SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, SMC_IND,
+				SOFT_REGISTERS_TABLE_28, fwMask, fwMask)) {
+		printk(KERN_ERR "[ powerplay ] check firmware loading failed\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Populate one firmware image to the data structure */
+static int tonga_populate_single_firmware_entry(struct pp_smumgr *smumgr,
+				uint16_t firmware_type,
+				struct SMU_Entry *pentry)
+{
+	int result;
+	struct cgs_firmware_info info = {0};
+
+	result = cgs_get_firmware_info(
+				smumgr->device,
+				tonga_convert_fw_type_to_cgs(firmware_type),
+				&info);
+
+	if (result == 0) {
+		pentry->version = 0;
+		pentry->id = (uint16_t)firmware_type;
+		pentry->image_addr_high = smu_upper_32_bits(info.mc_addr);
+		pentry->image_addr_low = smu_lower_32_bits(info.mc_addr);
+		pentry->meta_data_addr_high = 0;
+		pentry->meta_data_addr_low = 0;
+		pentry->data_size_byte = info.image_size;
+		pentry->num_register_entries = 0;
+
+		if (firmware_type == UCODE_ID_RLC_G)
+			pentry->flags = 1;
+		else
+			pentry->flags = 0;
+	} else {
+		return result;
+	}
+
+	return result;
+}
+
+static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr)
+{
+	struct tonga_smumgr *tonga_smu =
+		(struct tonga_smumgr *)(smumgr->backend);
+	uint16_t fw_to_load;
+	int result = 0;
+	struct SMU_DRAMData_TOC *toc;
+	/**
+	 * First time this gets called during SmuMgr init,
+	 * we haven't processed SMU header file yet,
+	 * so Soft Register Start offset is unknown.
+	 * However, for this case, UcodeLoadStatus is already 0,
+	 * so we can skip this if the Soft Registers Start offset is 0.
+	 */
+	cgs_write_ind_register(smumgr->device,
+		CGS_IND_REG__SMC, ixSOFT_REGISTERS_TABLE_28, 0);
+
+	tonga_send_msg_to_smc_with_parameter(smumgr,
+		PPSMC_MSG_SMU_DRAM_ADDR_HI,
+		tonga_smu->smu_buffer.mc_addr_high);
+	tonga_send_msg_to_smc_with_parameter(smumgr,
+		PPSMC_MSG_SMU_DRAM_ADDR_LO,
+		tonga_smu->smu_buffer.mc_addr_low);
+
+	toc = (struct SMU_DRAMData_TOC *)tonga_smu->pHeader;
+	toc->num_entries = 0;
+	toc->structure_version = 1;
+
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry(smumgr,
+		UCODE_ID_RLC_G,
+		&toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n",
+		return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry(smumgr,
+		UCODE_ID_CP_CE,
+		&toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n",
+		return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_populate_single_firmware_entry
+		(smumgr, UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
+		"Failed to Get Firmware Entry.\n", return -1);
+
+	tonga_send_msg_to_smc_with_parameter(smumgr,
+		PPSMC_MSG_DRV_DRAM_ADDR_HI,
+		tonga_smu->header_buffer.mc_addr_high);
+	tonga_send_msg_to_smc_with_parameter(smumgr,
+		PPSMC_MSG_DRV_DRAM_ADDR_LO,
+		tonga_smu->header_buffer.mc_addr_low);
+
+	fw_to_load = UCODE_ID_RLC_G_MASK
+			+ UCODE_ID_SDMA0_MASK
+			+ UCODE_ID_SDMA1_MASK
+			+ UCODE_ID_CP_CE_MASK
+			+ UCODE_ID_CP_ME_MASK
+			+ UCODE_ID_CP_PFP_MASK
+			+ UCODE_ID_CP_MEC_MASK;
+
+	PP_ASSERT_WITH_CODE(
+		0 == tonga_send_msg_to_smc_with_parameter_without_waiting(
+		smumgr, PPSMC_MSG_LoadUcodes, fw_to_load),
+		"Fail to Request SMU Load uCode", return 0);
+
+	return result;
+}
+
+static int tonga_request_smu_load_specific_fw(struct pp_smumgr *smumgr,
+				uint32_t firmwareType)
+{
+	return 0;
+}
+
+/**
+ * Upload the SMC firmware to the SMC microcontroller.
+ *
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    pFirmware the data structure containing the various sections of the firmware.
+ */
+static int tonga_smu_upload_firmware_image(struct pp_smumgr *smumgr)
+{
+	const uint8_t *src;
+	uint32_t byte_count;
+	uint32_t *data;
+	struct cgs_firmware_info info = {0};
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	cgs_get_firmware_info(smumgr->device,
+		tonga_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
+
+	if (info.image_size & 3) {
+		printk(KERN_ERR "[ powerplay ] SMC ucode is not 4 bytes aligned\n");
+		return -EINVAL;
+	}
+
+	if (info.image_size > TONGA_SMC_SIZE) {
+		printk(KERN_ERR "[ powerplay ] SMC address is beyond the SMC RAM area\n");
+		return -EINVAL;
+	}
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, 0x20000);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
+
+	byte_count = info.image_size;
+	src = (const uint8_t *)info.kptr;
+
+	data = (uint32_t *)src;
+	for (; byte_count >= 4; data++, byte_count -= 4)
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data[0]);
+
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+
+	return 0;
+}
+
+static int tonga_start_in_protection_mode(struct pp_smumgr *smumgr)
+{
+	int result;
+
+	/* Assert reset */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	result = tonga_smu_upload_firmware_image(smumgr);
+	if (result)
+		return result;
+
+	/* Clear status */
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+		ixSMU_STATUS, 0);
+
+	/* Enable clock */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	/* De-assert reset */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Set SMU Auto Start */
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMU_INPUT_DATA, AUTO_START, 1);
+
+	/* Clear firmware interrupt enable flag */
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+		ixFIRMWARE_FLAGS, 0);
+
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND,
+		RCU_UC_EVENTS, INTERRUPTS_ENABLED, 1);
+
+	/**
+	 * Call Test SMU message with 0x20000 offset to trigger SMU start
+	 */
+	tonga_send_msg_to_smc_offset(smumgr);
+
+	/* Wait for done bit to be set */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+		SMU_STATUS, SMU_DONE, 0);
+
+	/* Check pass/failed indicator */
+	if (1 != SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device,
+				CGS_IND_REG__SMC, SMU_STATUS, SMU_PASS)) {
+		printk(KERN_ERR "[ powerplay ] SMU Firmware start failed\n");
+		return -EINVAL;
+	}
+
+	/* Wait for firmware to initialize */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND,
+		FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return 0;
+}
+
+
+static int tonga_start_in_non_protection_mode(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+
+	/* wait for smc boot up */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+		RCU_UC_EVENTS, boot_seq_done, 0);
+
+	/*Clear firmware interrupt enable flag*/
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+		ixFIRMWARE_FLAGS, 0);
+
+
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	result = tonga_smu_upload_firmware_image(smumgr);
+
+	if (result != 0)
+		return result;
+
+	/* Set smc instruct start point at 0x0 */
+	tonga_program_jump_on_start(smumgr);
+
+
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	/*De-assert reset*/
+	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+		SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Wait for firmware to initialize */
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND,
+		FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return result;
+}
+
+static int tonga_start_smu(struct pp_smumgr *smumgr)
+{
+	int result;
+
+	/* Only start SMC if SMC RAM is not running */
+	if (!tonga_is_smc_ram_running(smumgr)) {
+		/*Check if SMU is running in protected mode*/
+		if (0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+					SMU_FIRMWARE, SMU_MODE)) {
+			result = tonga_start_in_non_protection_mode(smumgr);
+			if (result)
+				return result;
+		} else {
+			result = tonga_start_in_protection_mode(smumgr);
+			if (result)
+				return result;
+		}
+	}
+
+	result = tonga_request_smu_reload_fw(smumgr);
+
+	return result;
+}
+
+/**
+ * Write a 32bit value to the SMC SRAM space.
+ * ALL PARAMETERS ARE IN HOST BYTE ORDER.
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    smcAddress the address in the SMC RAM to access.
+ * @param    value to write to the SMC SRAM.
+ */
+static int tonga_smu_init(struct pp_smumgr *smumgr)
+{
+	struct tonga_smumgr *tonga_smu;
+	uint8_t *internal_buf;
+	uint64_t mc_addr = 0;
+	/* Allocate memory for backend private data */
+	tonga_smu = (struct tonga_smumgr *)(smumgr->backend);
+	tonga_smu->header_buffer.data_size =
+		((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
+	tonga_smu->smu_buffer.data_size = 200*4096;
+
+	smu_allocate_memory(smumgr->device,
+		tonga_smu->header_buffer.data_size,
+		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
+		PAGE_SIZE,
+		&mc_addr,
+		&tonga_smu->header_buffer.kaddr,
+		&tonga_smu->header_buffer.handle);
+
+	tonga_smu->pHeader = tonga_smu->header_buffer.kaddr;
+	tonga_smu->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	tonga_smu->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	PP_ASSERT_WITH_CODE((NULL != tonga_smu->pHeader),
+		"Out of memory.",
+		kfree(smumgr->backend);
+		cgs_free_gpu_mem(smumgr->device,
+		(cgs_handle_t)tonga_smu->header_buffer.handle);
+		return -1);
+
+	smu_allocate_memory(smumgr->device,
+		tonga_smu->smu_buffer.data_size,
+		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
+		PAGE_SIZE,
+		&mc_addr,
+		&tonga_smu->smu_buffer.kaddr,
+		&tonga_smu->smu_buffer.handle);
+
+	internal_buf = tonga_smu->smu_buffer.kaddr;
+	tonga_smu->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	tonga_smu->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	PP_ASSERT_WITH_CODE((NULL != internal_buf),
+		"Out of memory.",
+		kfree(smumgr->backend);
+		cgs_free_gpu_mem(smumgr->device,
+		(cgs_handle_t)tonga_smu->smu_buffer.handle);
+		return -1;);
+
+	return 0;
+}
+
+static const struct pp_smumgr_func tonga_smu_funcs = {
+	.smu_init = &tonga_smu_init,
+	.smu_fini = &tonga_smu_fini,
+	.start_smu = &tonga_start_smu,
+	.check_fw_load_finish = &tonga_check_fw_load_finish,
+	.request_smu_load_fw = &tonga_request_smu_reload_fw,
+	.request_smu_load_specific_fw = &tonga_request_smu_load_specific_fw,
+	.send_msg_to_smc = &tonga_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &tonga_send_msg_to_smc_with_parameter,
+	.download_pptable_settings = NULL,
+	.upload_pptable_settings = NULL,
+};
+
+int tonga_smum_init(struct pp_smumgr *smumgr)
+{
+	struct tonga_smumgr *tonga_smu = NULL;
+
+	tonga_smu = kzalloc(sizeof(struct tonga_smumgr), GFP_KERNEL);
+
+	if (tonga_smu == NULL)
+		return -ENOMEM;
+
+	smumgr->backend = tonga_smu;
+	smumgr->smumgr_funcs = &tonga_smu_funcs;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h
new file mode 100644
index 000000000000..33c788d7f05c
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _TONGA_SMUMGR_H_
+#define _TONGA_SMUMGR_H_
+
+struct tonga_buffer_entry {
+	uint32_t data_size;
+	uint32_t mc_addr_low;
+	uint32_t mc_addr_high;
+	void *kaddr;
+	unsigned long  handle;
+};
+
+struct tonga_smumgr {
+	uint8_t *pHeader;
+	uint8_t *pMecImage;
+	uint32_t ulSoftRegsStart;
+
+	struct tonga_buffer_entry header_buffer;
+	struct tonga_buffer_entry smu_buffer;
+};
+
+extern int tonga_smum_init(struct pp_smumgr *smumgr);
+extern int tonga_copy_bytes_to_smc(struct pp_smumgr *smumgr,
+		uint32_t smcStartAddress, const uint8_t *src,
+		uint32_t byteCount, uint32_t limit);
+extern int tonga_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
+		uint32_t *value, uint32_t limit);
+extern int tonga_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
+		uint32_t value, uint32_t limit);
+
+#endif
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index 3a4820e863ec..8b2becd1aa07 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -47,6 +47,8 @@ static void amd_sched_rq_init(struct amd_sched_rq *rq)
 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
 				    struct amd_sched_entity *entity)
 {
+	if (!list_empty(&entity->list))
+		return;
 	spin_lock(&rq->lock);
 	list_add_tail(&entity->list, &rq->entities);
 	spin_unlock(&rq->lock);
@@ -55,6 +57,8 @@ static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
 				       struct amd_sched_entity *entity)
 {
+	if (list_empty(&entity->list))
+		return;
 	spin_lock(&rq->lock);
 	list_del_init(&entity->list);
 	if (rq->current_entity == entity)
@@ -138,9 +142,6 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 	atomic_set(&entity->fence_seq, 0);
 	entity->fence_context = fence_context_alloc(1);
 
-	/* Add the entity to the run queue */
-	amd_sched_rq_add_entity(rq, entity);
-
 	return 0;
 }
 
@@ -302,9 +303,11 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 	spin_unlock(&entity->queue_lock);
 
 	/* first job wakes up scheduler */
-	if (first)
+	if (first) {
+		/* Add the entity to the run queue */
+		amd_sched_rq_add_entity(entity->rq, entity);
 		amd_sched_wakeup(sched);
-
+	}
 	return added;
 }
 
@@ -349,14 +352,17 @@ static struct amd_sched_entity *
 amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 {
 	struct amd_sched_entity *entity;
+	int i;
 
 	if (!amd_sched_ready(sched))
 		return NULL;
 
 	/* Kernel run queue has higher priority than normal run queue*/
-	entity = amd_sched_rq_select_entity(&sched->kernel_rq);
-	if (entity == NULL)
-		entity = amd_sched_rq_select_entity(&sched->sched_rq);
+	for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) {
+		entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
+		if (entity)
+			break;
+	}
 
 	return entity;
 }
@@ -478,12 +484,13 @@ int amd_sched_init(struct amd_gpu_scheduler *sched,
 		   struct amd_sched_backend_ops *ops,
 		   unsigned hw_submission, long timeout, const char *name)
 {
+	int i;
 	sched->ops = ops;
 	sched->hw_submission_limit = hw_submission;
 	sched->name = name;
 	sched->timeout = timeout;
-	amd_sched_rq_init(&sched->sched_rq);
-	amd_sched_rq_init(&sched->kernel_rq);
+	for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++)
+		amd_sched_rq_init(&sched->sched_rq[i]);
 
 	init_waitqueue_head(&sched->wake_up_worker);
 	init_waitqueue_head(&sched->job_scheduled);
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
index a0f0ae53aacd..9403145d7bee 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
@@ -104,6 +104,12 @@ struct amd_sched_backend_ops {
 	struct fence *(*run_job)(struct amd_sched_job *sched_job);
 };
 
+enum amd_sched_priority {
+	AMD_SCHED_PRIORITY_KERNEL = 0,
+	AMD_SCHED_PRIORITY_NORMAL,
+	AMD_SCHED_MAX_PRIORITY
+};
+
 /**
  * One scheduler is implemented for each hardware ring
 */
@@ -112,8 +118,7 @@ struct amd_gpu_scheduler {
 	uint32_t			hw_submission_limit;
 	long				timeout;
 	const char			*name;
-	struct amd_sched_rq		sched_rq;
-	struct amd_sched_rq		kernel_rq;
+	struct amd_sched_rq		sched_rq[AMD_SCHED_MAX_PRIORITY];
 	wait_queue_head_t		wake_up_worker;
 	wait_queue_head_t		job_scheduled;
 	atomic_t			hw_rq_count;
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index cebcab560626..0293eb74d777 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -928,11 +928,10 @@ static int armada_drm_crtc_cursor_set(struct drm_crtc *crtc,
 		}
 	}
 
-	mutex_lock(&dev->struct_mutex);
 	if (dcrtc->cursor_obj) {
 		dcrtc->cursor_obj->update = NULL;
 		dcrtc->cursor_obj->update_data = NULL;
-		drm_gem_object_unreference(&dcrtc->cursor_obj->obj);
+		drm_gem_object_unreference_unlocked(&dcrtc->cursor_obj->obj);
 	}
 	dcrtc->cursor_obj = obj;
 	dcrtc->cursor_w = w;
@@ -942,14 +941,12 @@ static int armada_drm_crtc_cursor_set(struct drm_crtc *crtc,
 		obj->update_data = dcrtc;
 		obj->update = cursor_update;
 	}
-	mutex_unlock(&dev->struct_mutex);
 
 	return ret;
 }
 
 static int armada_drm_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 {
-	struct drm_device *dev = crtc->dev;
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
 	int ret;
 
@@ -957,11 +954,9 @@ static int armada_drm_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 	if (!dcrtc->variant->has_spu_adv_reg)
 		return -EFAULT;
 
-	mutex_lock(&dev->struct_mutex);
 	dcrtc->cursor_x = x;
 	dcrtc->cursor_y = y;
 	ret = armada_drm_crtc_cursor_update(dcrtc, false);
-	mutex_unlock(&dev->struct_mutex);
 
 	return ret;
 }
@@ -972,7 +967,7 @@ static void armada_drm_crtc_destroy(struct drm_crtc *crtc)
 	struct armada_private *priv = crtc->dev->dev_private;
 
 	if (dcrtc->cursor_obj)
-		drm_gem_object_unreference(&dcrtc->cursor_obj->obj);
+		drm_gem_object_unreference_unlocked(&dcrtc->cursor_obj->obj);
 
 	priv->dcrtc[dcrtc->num] = NULL;
 	drm_crtc_cleanup(&dcrtc->crtc);
@@ -1074,7 +1069,7 @@ armada_drm_crtc_set_property(struct drm_crtc *crtc,
 	return 0;
 }
 
-static struct drm_crtc_funcs armada_crtc_funcs = {
+static const struct drm_crtc_funcs armada_crtc_funcs = {
 	.cursor_set	= armada_drm_crtc_cursor_set,
 	.cursor_move	= armada_drm_crtc_cursor_move,
 	.destroy	= armada_drm_crtc_destroy,
@@ -1216,14 +1211,14 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 				       &armada_primary_plane_funcs,
 				       armada_primary_formats,
 				       ARRAY_SIZE(armada_primary_formats),
-				       DRM_PLANE_TYPE_PRIMARY);
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
 		return ret;
 	}
 
 	ret = drm_crtc_init_with_planes(drm, &dcrtc->crtc, &primary->base, NULL,
-					&armada_crtc_funcs);
+					&armada_crtc_funcs, NULL);
 	if (ret)
 		goto err_crtc_init;
 
diff --git a/drivers/gpu/drm/armada/armada_debugfs.c b/drivers/gpu/drm/armada/armada_debugfs.c
index 471e45627f1e..d4f7ab0a30d4 100644
--- a/drivers/gpu/drm/armada/armada_debugfs.c
+++ b/drivers/gpu/drm/armada/armada_debugfs.c
@@ -21,9 +21,9 @@ static int armada_debugfs_gem_linear_show(struct seq_file *m, void *data)
 	struct armada_private *priv = dev->dev_private;
 	int ret;
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&priv->linear_lock);
 	ret = drm_mm_dump_table(m, &priv->linear);
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&priv->linear_lock);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/armada/armada_drm.h b/drivers/gpu/drm/armada/armada_drm.h
index 4df6f2af2b21..3b2bb6128d40 100644
--- a/drivers/gpu/drm/armada/armada_drm.h
+++ b/drivers/gpu/drm/armada/armada_drm.h
@@ -57,7 +57,8 @@ struct armada_private {
 	DECLARE_KFIFO(fb_unref, struct drm_framebuffer *, 8);
 	struct drm_fb_helper	*fbdev;
 	struct armada_crtc	*dcrtc[2];
-	struct drm_mm		linear;
+	struct drm_mm		linear; /* protected by linear_lock */
+	struct mutex		linear_lock;
 	struct drm_property	*csc_yuv_prop;
 	struct drm_property	*csc_rgb_prop;
 	struct drm_property	*colorkey_prop;
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index 77ab93d60125..3bd7e1cde99e 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -102,6 +102,7 @@ static int armada_drm_load(struct drm_device *dev, unsigned long flags)
 	dev->mode_config.preferred_depth = 24;
 	dev->mode_config.funcs = &armada_drm_mode_config_funcs;
 	drm_mm_init(&priv->linear, mem->start, resource_size(mem));
+	mutex_init(&priv->linear_lock);
 
 	ret = component_bind_all(dev->dev, dev);
 	if (ret)
diff --git a/drivers/gpu/drm/armada/armada_fb.c b/drivers/gpu/drm/armada/armada_fb.c
index 1c90969def3e..5fa4bf20b232 100644
--- a/drivers/gpu/drm/armada/armada_fb.c
+++ b/drivers/gpu/drm/armada/armada_fb.c
@@ -35,7 +35,7 @@ static const struct drm_framebuffer_funcs armada_fb_funcs = {
 };
 
 struct armada_framebuffer *armada_framebuffer_create(struct drm_device *dev,
-	struct drm_mode_fb_cmd2 *mode, struct armada_gem_object *obj)
+	const struct drm_mode_fb_cmd2 *mode, struct armada_gem_object *obj)
 {
 	struct armada_framebuffer *dfb;
 	uint8_t format, config;
@@ -101,7 +101,7 @@ struct armada_framebuffer *armada_framebuffer_create(struct drm_device *dev,
 }
 
 static struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
-	struct drm_file *dfile, struct drm_mode_fb_cmd2 *mode)
+	struct drm_file *dfile, const struct drm_mode_fb_cmd2 *mode)
 {
 	struct armada_gem_object *obj;
 	struct armada_framebuffer *dfb;
diff --git a/drivers/gpu/drm/armada/armada_fb.h b/drivers/gpu/drm/armada/armada_fb.h
index ce3f12ebfc53..48073c4f54d8 100644
--- a/drivers/gpu/drm/armada/armada_fb.h
+++ b/drivers/gpu/drm/armada/armada_fb.h
@@ -19,6 +19,6 @@ struct armada_framebuffer {
 #define drm_fb_obj(fb) drm_fb_to_armada_fb(fb)->obj
 
 struct armada_framebuffer *armada_framebuffer_create(struct drm_device *,
-	struct drm_mode_fb_cmd2 *, struct armada_gem_object *);
+	const struct drm_mode_fb_cmd2 *, struct armada_gem_object *);
 
 #endif
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index 60a688ef81c7..6e731db31aa4 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -46,22 +46,26 @@ static size_t roundup_gem_size(size_t size)
 	return roundup(size, PAGE_SIZE);
 }
 
-/* dev->struct_mutex is held here */
 void armada_gem_free_object(struct drm_gem_object *obj)
 {
 	struct armada_gem_object *dobj = drm_to_armada_gem(obj);
+	struct armada_private *priv = obj->dev->dev_private;
 
 	DRM_DEBUG_DRIVER("release obj %p\n", dobj);
 
 	drm_gem_free_mmap_offset(&dobj->obj);
 
+	might_lock(&priv->linear_lock);
+
 	if (dobj->page) {
 		/* page backed memory */
 		unsigned int order = get_order(dobj->obj.size);
 		__free_pages(dobj->page, order);
 	} else if (dobj->linear) {
 		/* linear backed memory */
+		mutex_lock(&priv->linear_lock);
 		drm_mm_remove_node(dobj->linear);
+		mutex_unlock(&priv->linear_lock);
 		kfree(dobj->linear);
 		if (dobj->addr)
 			iounmap(dobj->addr);
@@ -144,10 +148,10 @@ armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj)
 		if (!node)
 			return -ENOSPC;
 
-		mutex_lock(&dev->struct_mutex);
+		mutex_lock(&priv->linear_lock);
 		ret = drm_mm_insert_node(&priv->linear, node, size, align,
 					 DRM_MM_SEARCH_DEFAULT);
-		mutex_unlock(&dev->struct_mutex);
+		mutex_unlock(&priv->linear_lock);
 		if (ret) {
 			kfree(node);
 			return ret;
@@ -158,9 +162,9 @@ armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj)
 		/* Ensure that the memory we're returning is cleared. */
 		ptr = ioremap_wc(obj->linear->start, size);
 		if (!ptr) {
-			mutex_lock(&dev->struct_mutex);
+			mutex_lock(&priv->linear_lock);
 			drm_mm_remove_node(obj->linear);
-			mutex_unlock(&dev->struct_mutex);
+			mutex_unlock(&priv->linear_lock);
 			kfree(obj->linear);
 			obj->linear = NULL;
 			return -ENOMEM;
@@ -274,18 +278,16 @@ int armada_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 	struct armada_gem_object *obj;
 	int ret = 0;
 
-	mutex_lock(&dev->struct_mutex);
 	obj = armada_gem_object_lookup(dev, file, handle);
 	if (!obj) {
 		DRM_ERROR("failed to lookup gem object\n");
-		ret = -EINVAL;
-		goto err_unlock;
+		return -EINVAL;
 	}
 
 	/* Don't allow imported objects to be mapped */
 	if (obj->obj.import_attach) {
 		ret = -EINVAL;
-		goto err_unlock;
+		goto err_unref;
 	}
 
 	ret = drm_gem_create_mmap_offset(&obj->obj);
@@ -294,9 +296,8 @@ int armada_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 		DRM_DEBUG_DRIVER("handle %#x offset %llx\n", handle, *offset);
 	}
 
-	drm_gem_object_unreference(&obj->obj);
- err_unlock:
-	mutex_unlock(&dev->struct_mutex);
+ err_unref:
+	drm_gem_object_unreference_unlocked(&obj->obj);
 
 	return ret;
 }
@@ -352,13 +353,13 @@ int armada_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 
 	if (!dobj->obj.filp) {
-		drm_gem_object_unreference(&dobj->obj);
+		drm_gem_object_unreference_unlocked(&dobj->obj);
 		return -EINVAL;
 	}
 
 	addr = vm_mmap(dobj->obj.filp, 0, args->size, PROT_READ | PROT_WRITE,
 		       MAP_SHARED, args->offset);
-	drm_gem_object_unreference(&dobj->obj);
+	drm_gem_object_unreference_unlocked(&dobj->obj);
 	if (IS_ERR_VALUE(addr))
 		return addr;
 
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 5c22b380f8f3..148e8a42b2c6 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -460,7 +460,7 @@ int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
 				       &armada_ovl_plane_funcs,
 				       armada_ovl_formats,
 				       ARRAY_SIZE(armada_ovl_formats),
-				       DRM_PLANE_TYPE_OVERLAY);
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret) {
 		kfree(dplane);
 		return ret;
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 05f6522c0457..eb5715994ac2 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -256,7 +256,6 @@ struct ast_framebuffer {
 struct ast_fbdev {
 	struct drm_fb_helper helper;
 	struct ast_framebuffer afb;
-	struct list_head fbdev_list;
 	void *sysram;
 	int size;
 	struct ttm_bo_kmap_obj mapping;
@@ -309,7 +308,7 @@ extern void ast_mode_fini(struct drm_device *dev);
 
 int ast_framebuffer_init(struct drm_device *dev,
 			 struct ast_framebuffer *ast_fb,
-			 struct drm_mode_fb_cmd2 *mode_cmd,
+			 const struct drm_mode_fb_cmd2 *mode_cmd,
 			 struct drm_gem_object *obj);
 
 int ast_fbdev_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index a37e7ea4a00c..5320f8c57884 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -163,7 +163,7 @@ static struct fb_ops astfb_ops = {
 };
 
 static int astfb_create_object(struct ast_fbdev *afbdev,
-			       struct drm_mode_fb_cmd2 *mode_cmd,
+			       const struct drm_mode_fb_cmd2 *mode_cmd,
 			       struct drm_gem_object **gobj_p)
 {
 	struct drm_device *dev = afbdev->helper.dev;
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index 541a610667ad..9759009d1da3 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -309,7 +309,7 @@ static const struct drm_framebuffer_funcs ast_fb_funcs = {
 
 int ast_framebuffer_init(struct drm_device *dev,
 			 struct ast_framebuffer *ast_fb,
-			 struct drm_mode_fb_cmd2 *mode_cmd,
+			 const struct drm_mode_fb_cmd2 *mode_cmd,
 			 struct drm_gem_object *obj)
 {
 	int ret;
@@ -327,7 +327,7 @@ int ast_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 ast_user_framebuffer_create(struct drm_device *dev,
 	       struct drm_file *filp,
-	       struct drm_mode_fb_cmd2 *mode_cmd)
+	       const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct ast_framebuffer *ast_fb;
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 69d19f3304a5..0123458cbd83 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -751,7 +751,7 @@ static int ast_encoder_init(struct drm_device *dev)
 		return -ENOMEM;
 
 	drm_encoder_init(dev, &ast_encoder->base, &ast_enc_funcs,
-			 DRM_MODE_ENCODER_DAC);
+			 DRM_MODE_ENCODER_DAC, NULL);
 	drm_encoder_helper_add(&ast_encoder->base, &ast_enc_helper_funcs);
 
 	ast_encoder->base.possible_crtcs = 1;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index 9f6e234e7029..468a14f266a7 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -344,7 +344,7 @@ int atmel_hlcdc_crtc_create(struct drm_device *dev)
 	ret = drm_crtc_init_with_planes(dev, &crtc->base,
 				&planes->primary->base,
 				planes->cursor ? &planes->cursor->base : NULL,
-				&atmel_hlcdc_crtc_funcs);
+				&atmel_hlcdc_crtc_funcs, NULL);
 	if (ret < 0)
 		goto fail;
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 244df0a440b7..a45b32ba029e 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -333,6 +333,10 @@ static const struct of_device_id atmel_hlcdc_of_match[] = {
 		.data = &atmel_hlcdc_dc_at91sam9x5,
 	},
 	{
+		.compatible = "atmel,sama5d2-hlcdc",
+		.data = &atmel_hlcdc_dc_sama5d4,
+	},
+	{
 		.compatible = "atmel,sama5d3-hlcdc",
 		.data = &atmel_hlcdc_dc_sama5d3,
 	},
@@ -342,6 +346,7 @@ static const struct of_device_id atmel_hlcdc_of_match[] = {
 	},
 	{ /* sentinel */ },
 };
+MODULE_DEVICE_TABLE(of, atmel_hlcdc_of_match);
 
 int atmel_hlcdc_dc_mode_valid(struct atmel_hlcdc_dc *dc,
 			      struct drm_display_mode *mode)
@@ -402,7 +407,7 @@ static irqreturn_t atmel_hlcdc_dc_irq_handler(int irq, void *data)
 }
 
 static struct drm_framebuffer *atmel_hlcdc_fb_create(struct drm_device *dev,
-		struct drm_file *file_priv, struct drm_mode_fb_cmd2 *mode_cmd)
+		struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	return drm_fb_cma_create(dev, file_priv, mode_cmd);
 }
@@ -733,10 +738,6 @@ static int atmel_hlcdc_dc_drm_probe(struct platform_device *pdev)
 	if (!ddev)
 		return -ENOMEM;
 
-	ret = drm_dev_set_unique(ddev, dev_name(ddev->dev));
-	if (ret)
-		goto err_unref;
-
 	ret = atmel_hlcdc_dc_load(ddev);
 	if (ret)
 		goto err_unref;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
index 067e4c144bd6..0f7ec016e7a9 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
@@ -146,7 +146,7 @@ atmel_hlcdc_rgb_encoder_mode_set(struct drm_encoder *encoder,
 			   cfg);
 }
 
-static struct drm_encoder_helper_funcs atmel_hlcdc_panel_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs atmel_hlcdc_panel_encoder_helper_funcs = {
 	.mode_fixup = atmel_hlcdc_panel_encoder_mode_fixup,
 	.mode_set = atmel_hlcdc_rgb_encoder_mode_set,
 	.disable = atmel_hlcdc_panel_encoder_disable,
@@ -192,7 +192,7 @@ atmel_hlcdc_rgb_best_encoder(struct drm_connector *connector)
 	return &rgb->encoder;
 }
 
-static struct drm_connector_helper_funcs atmel_hlcdc_panel_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs atmel_hlcdc_panel_connector_helper_funcs = {
 	.get_modes = atmel_hlcdc_panel_get_modes,
 	.mode_valid = atmel_hlcdc_rgb_mode_valid,
 	.best_encoder = atmel_hlcdc_rgb_best_encoder,
@@ -256,7 +256,7 @@ static int atmel_hlcdc_create_panel_output(struct drm_device *dev,
 			       &atmel_hlcdc_panel_encoder_helper_funcs);
 	ret = drm_encoder_init(dev, &panel->base.encoder,
 			       &atmel_hlcdc_panel_encoder_funcs,
-			       DRM_MODE_ENCODER_LVDS);
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index d0299aed517e..1ffe9c329c46 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -941,7 +941,7 @@ atmel_hlcdc_plane_create(struct drm_device *dev,
 	ret = drm_universal_plane_init(dev, &plane->base, 0,
 				       &layer_plane_funcs,
 				       desc->formats->formats,
-				       desc->formats->nformats, type);
+				       desc->formats->nformats, type, NULL);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/bochs/bochs.h b/drivers/gpu/drm/bochs/bochs.h
index 71f2687fc3cc..19b5adaebe24 100644
--- a/drivers/gpu/drm/bochs/bochs.h
+++ b/drivers/gpu/drm/bochs/bochs.h
@@ -149,7 +149,7 @@ int bochs_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev,
 
 int bochs_framebuffer_init(struct drm_device *dev,
 			   struct bochs_framebuffer *gfb,
-			   struct drm_mode_fb_cmd2 *mode_cmd,
+			   const struct drm_mode_fb_cmd2 *mode_cmd,
 			   struct drm_gem_object *obj);
 int bochs_bo_pin(struct bochs_bo *bo, u32 pl_flag, u64 *gpu_addr);
 int bochs_bo_unpin(struct bochs_bo *bo);
diff --git a/drivers/gpu/drm/bochs/bochs_fbdev.c b/drivers/gpu/drm/bochs/bochs_fbdev.c
index 09a0637aab3e..7520bf81fc25 100644
--- a/drivers/gpu/drm/bochs/bochs_fbdev.c
+++ b/drivers/gpu/drm/bochs/bochs_fbdev.c
@@ -34,7 +34,7 @@ static struct fb_ops bochsfb_ops = {
 };
 
 static int bochsfb_create_object(struct bochs_device *bochs,
-				 struct drm_mode_fb_cmd2 *mode_cmd,
+				 const struct drm_mode_fb_cmd2 *mode_cmd,
 				 struct drm_gem_object **gobj_p)
 {
 	struct drm_device *dev = bochs->dev;
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 26bcd03a8cb6..2849f1b95eec 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -119,7 +119,7 @@ static int bochs_crtc_page_flip(struct drm_crtc *crtc,
 	bochs_crtc_mode_set_base(crtc, 0, 0, old_fb);
 	if (event) {
 		spin_lock_irqsave(&bochs->dev->event_lock, irqflags);
-		drm_send_vblank_event(bochs->dev, -1, event);
+		drm_crtc_send_vblank_event(crtc, event);
 		spin_unlock_irqrestore(&bochs->dev->event_lock, irqflags);
 	}
 	return 0;
@@ -196,7 +196,7 @@ static void bochs_encoder_init(struct drm_device *dev)
 
 	encoder->possible_crtcs = 0x1;
 	drm_encoder_init(dev, encoder, &bochs_encoder_encoder_funcs,
-			 DRM_MODE_ENCODER_DAC);
+			 DRM_MODE_ENCODER_DAC, NULL);
 	drm_encoder_helper_add(encoder, &bochs_encoder_helper_funcs);
 }
 
@@ -245,13 +245,13 @@ static enum drm_connector_status bochs_connector_detect(struct drm_connector
 	return connector_status_connected;
 }
 
-struct drm_connector_helper_funcs bochs_connector_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs bochs_connector_connector_helper_funcs = {
 	.get_modes = bochs_connector_get_modes,
 	.mode_valid = bochs_connector_mode_valid,
 	.best_encoder = bochs_connector_best_encoder,
 };
 
-struct drm_connector_funcs bochs_connector_connector_funcs = {
+static const struct drm_connector_funcs bochs_connector_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = bochs_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -283,7 +283,7 @@ int bochs_kms_init(struct bochs_device *bochs)
 	bochs->dev->mode_config.preferred_depth = 24;
 	bochs->dev->mode_config.prefer_shadow = 0;
 
-	bochs->dev->mode_config.funcs = (void *)&bochs_mode_funcs;
+	bochs->dev->mode_config.funcs = &bochs_mode_funcs;
 
 	bochs_crtc_init(bochs->dev);
 	bochs_encoder_init(bochs->dev);
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index f69e6bf9bb0e..d812ad014da5 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -484,7 +484,7 @@ static const struct drm_framebuffer_funcs bochs_fb_funcs = {
 
 int bochs_framebuffer_init(struct drm_device *dev,
 			   struct bochs_framebuffer *gfb,
-			   struct drm_mode_fb_cmd2 *mode_cmd,
+			   const struct drm_mode_fb_cmd2 *mode_cmd,
 			   struct drm_gem_object *obj)
 {
 	int ret;
@@ -502,7 +502,7 @@ int bochs_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 bochs_user_framebuffer_create(struct drm_device *dev,
 			      struct drm_file *filp,
-			      struct drm_mode_fb_cmd2 *mode_cmd)
+			      const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct bochs_framebuffer *bochs_fb;
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 6dddd392aa42..27e2022de89d 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -22,7 +22,6 @@ config DRM_DW_HDMI_AHB_AUDIO
 	  Designware HDMI block.  This is used in conjunction with
 	  the i.MX6 HDMI driver.
 
-
 config DRM_NXP_PTN3460
 	tristate "NXP PTN3460 DP/LVDS bridge"
 	depends on OF
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index d4e28beec30e..f13c33d67c03 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -1,6 +1,6 @@
 ccflags-y := -Iinclude/drm
 
-obj-$(CONFIG_DRM_DW_HDMI) += dw_hdmi.o
-obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw_hdmi-ahb-audio.o
+obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o
+obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
diff --git a/drivers/gpu/drm/bridge/dw_hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c
index 59f630f1c61a..122bb015f4a9 100644
--- a/drivers/gpu/drm/bridge/dw_hdmi-ahb-audio.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c
@@ -21,7 +21,7 @@
 #include <sound/pcm_drm_eld.h>
 #include <sound/pcm_iec958.h>
 
-#include "dw_hdmi-audio.h"
+#include "dw-hdmi-audio.h"
 
 #define DRIVER_NAME "dw-hdmi-ahb-audio"
 
diff --git a/drivers/gpu/drm/bridge/dw_hdmi-audio.h b/drivers/gpu/drm/bridge/dw-hdmi-audio.h
index 91f631beecc7..91f631beecc7 100644
--- a/drivers/gpu/drm/bridge/dw_hdmi-audio.h
+++ b/drivers/gpu/drm/bridge/dw-hdmi-audio.h
diff --git a/drivers/gpu/drm/bridge/dw_hdmi.c b/drivers/gpu/drm/bridge/dw-hdmi.c
index 56de9f1c95fc..b0aac4733020 100644
--- a/drivers/gpu/drm/bridge/dw_hdmi.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi.c
@@ -22,13 +22,14 @@
 
 #include <drm/drm_of.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder_slave.h>
 #include <drm/bridge/dw_hdmi.h>
 
-#include "dw_hdmi.h"
-#include "dw_hdmi-audio.h"
+#include "dw-hdmi.h"
+#include "dw-hdmi-audio.h"
 
 #define HDMI_EDID_LEN		512
 
@@ -1514,7 +1515,7 @@ static void dw_hdmi_connector_force(struct drm_connector *connector)
 	mutex_unlock(&hdmi->mutex);
 }
 
-static struct drm_connector_funcs dw_hdmi_connector_funcs = {
+static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = dw_hdmi_connector_detect,
@@ -1522,13 +1523,24 @@ static struct drm_connector_funcs dw_hdmi_connector_funcs = {
 	.force = dw_hdmi_connector_force,
 };
 
-static struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
+static const struct drm_connector_funcs dw_hdmi_atomic_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = dw_hdmi_connector_detect,
+	.destroy = dw_hdmi_connector_destroy,
+	.force = dw_hdmi_connector_force,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
 	.get_modes = dw_hdmi_connector_get_modes,
 	.mode_valid = dw_hdmi_connector_mode_valid,
 	.best_encoder = dw_hdmi_connector_best_encoder,
 };
 
-static struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
+static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
 	.enable = dw_hdmi_bridge_enable,
 	.disable = dw_hdmi_bridge_disable,
 	.pre_enable = dw_hdmi_bridge_nop,
@@ -1645,10 +1657,15 @@ static int dw_hdmi_register(struct drm_device *drm, struct dw_hdmi *hdmi)
 
 	drm_connector_helper_add(&hdmi->connector,
 				 &dw_hdmi_connector_helper_funcs);
-	drm_connector_init(drm, &hdmi->connector, &dw_hdmi_connector_funcs,
-			   DRM_MODE_CONNECTOR_HDMIA);
 
-	hdmi->connector.encoder = encoder;
+	if (drm_core_check_feature(drm, DRIVER_ATOMIC))
+		drm_connector_init(drm, &hdmi->connector,
+				   &dw_hdmi_atomic_connector_funcs,
+				   DRM_MODE_CONNECTOR_HDMIA);
+	else
+		drm_connector_init(drm, &hdmi->connector,
+				   &dw_hdmi_connector_funcs,
+				   DRM_MODE_CONNECTOR_HDMIA);
 
 	drm_mode_connector_attach_encoder(&hdmi->connector, encoder);
 
diff --git a/drivers/gpu/drm/bridge/dw_hdmi.h b/drivers/gpu/drm/bridge/dw-hdmi.h
index fc9a560429d6..fc9a560429d6 100644
--- a/drivers/gpu/drm/bridge/dw_hdmi.h
+++ b/drivers/gpu/drm/bridge/dw-hdmi.h
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 0ffa3a6a206a..7ecd59f70b8e 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -242,7 +242,7 @@ static struct drm_encoder *ptn3460_best_encoder(struct drm_connector *connector)
 	return ptn_bridge->bridge.encoder;
 }
 
-static struct drm_connector_helper_funcs ptn3460_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs ptn3460_connector_helper_funcs = {
 	.get_modes = ptn3460_get_modes,
 	.best_encoder = ptn3460_best_encoder,
 };
@@ -258,7 +258,7 @@ static void ptn3460_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs ptn3460_connector_funcs = {
+static const struct drm_connector_funcs ptn3460_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = ptn3460_detect,
@@ -299,7 +299,7 @@ static int ptn3460_bridge_attach(struct drm_bridge *bridge)
 	return ret;
 }
 
-static struct drm_bridge_funcs ptn3460_bridge_funcs = {
+static const struct drm_bridge_funcs ptn3460_bridge_funcs = {
 	.pre_enable = ptn3460_pre_enable,
 	.enable = ptn3460_enable,
 	.disable = ptn3460_disable,
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.h b/drivers/gpu/drm/cirrus/cirrus_drv.h
index 705061537a27..b774d637a00f 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.h
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.h
@@ -153,7 +153,6 @@ struct cirrus_device {
 struct cirrus_fbdev {
 	struct drm_fb_helper helper;
 	struct cirrus_framebuffer gfb;
-	struct list_head fbdev_list;
 	void *sysram;
 	int size;
 	int x1, y1, x2, y2; /* dirty rect */
@@ -207,7 +206,7 @@ int cirrus_dumb_create(struct drm_file *file,
 
 int cirrus_framebuffer_init(struct drm_device *dev,
 			   struct cirrus_framebuffer *gfb,
-			    struct drm_mode_fb_cmd2 *mode_cmd,
+			    const struct drm_mode_fb_cmd2 *mode_cmd,
 			    struct drm_gem_object *obj);
 
 bool cirrus_check_framebuffer(struct cirrus_device *cdev, int width, int height,
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index 589103bcc06c..3b5be7272357 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -135,7 +135,7 @@ static struct fb_ops cirrusfb_ops = {
 };
 
 static int cirrusfb_create_object(struct cirrus_fbdev *afbdev,
-			       struct drm_mode_fb_cmd2 *mode_cmd,
+			       const struct drm_mode_fb_cmd2 *mode_cmd,
 			       struct drm_gem_object **gobj_p)
 {
 	struct drm_device *dev = afbdev->helper.dev;
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index 055fd86ba717..0907715e90fd 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -29,7 +29,7 @@ static const struct drm_framebuffer_funcs cirrus_fb_funcs = {
 
 int cirrus_framebuffer_init(struct drm_device *dev,
 			    struct cirrus_framebuffer *gfb,
-			    struct drm_mode_fb_cmd2 *mode_cmd,
+			    const struct drm_mode_fb_cmd2 *mode_cmd,
 			    struct drm_gem_object *obj)
 {
 	int ret;
@@ -47,7 +47,7 @@ int cirrus_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 cirrus_user_framebuffer_create(struct drm_device *dev,
 			       struct drm_file *filp,
-			       struct drm_mode_fb_cmd2 *mode_cmd)
+			       const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct cirrus_device *cdev = dev->dev_private;
 	struct drm_gem_object *obj;
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index 61385f2298bf..4a02854a6963 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -489,7 +489,7 @@ static struct drm_encoder *cirrus_encoder_init(struct drm_device *dev)
 	encoder->possible_crtcs = 0x1;
 
 	drm_encoder_init(dev, encoder, &cirrus_encoder_encoder_funcs,
-			 DRM_MODE_ENCODER_DAC);
+			 DRM_MODE_ENCODER_DAC, NULL);
 	drm_encoder_helper_add(encoder, &cirrus_encoder_helper_funcs);
 
 	return encoder;
@@ -533,12 +533,12 @@ static void cirrus_connector_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-struct drm_connector_helper_funcs cirrus_vga_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs cirrus_vga_connector_helper_funcs = {
 	.get_modes = cirrus_vga_get_modes,
 	.best_encoder = cirrus_connector_best_encoder,
 };
 
-struct drm_connector_funcs cirrus_vga_connector_funcs = {
+static const struct drm_connector_funcs cirrus_vga_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = cirrus_vga_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index aeee083c7f95..3f74193885f1 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -288,8 +288,8 @@ drm_atomic_get_crtc_state(struct drm_atomic_state *state,
 	state->crtcs[index] = crtc;
 	crtc_state->state = state;
 
-	DRM_DEBUG_ATOMIC("Added [CRTC:%d] %p state to %p\n",
-			 crtc->base.id, crtc_state, state);
+	DRM_DEBUG_ATOMIC("Added [CRTC:%d:%s] %p state to %p\n",
+			 crtc->base.id, crtc->name, crtc_state, state);
 
 	return crtc_state;
 }
@@ -316,8 +316,7 @@ int drm_atomic_set_mode_for_crtc(struct drm_crtc_state *state,
 	if (mode && memcmp(&state->mode, mode, sizeof(*mode)) == 0)
 		return 0;
 
-	if (state->mode_blob)
-		drm_property_unreference_blob(state->mode_blob);
+	drm_property_unreference_blob(state->mode_blob);
 	state->mode_blob = NULL;
 
 	if (mode) {
@@ -363,8 +362,7 @@ int drm_atomic_set_mode_prop_for_crtc(struct drm_crtc_state *state,
 	if (blob == state->mode_blob)
 		return 0;
 
-	if (state->mode_blob)
-		drm_property_unreference_blob(state->mode_blob);
+	drm_property_unreference_blob(state->mode_blob);
 	state->mode_blob = NULL;
 
 	if (blob) {
@@ -419,8 +417,7 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 		struct drm_property_blob *mode =
 			drm_property_lookup_blob(dev, val);
 		ret = drm_atomic_set_mode_prop_for_crtc(state, mode);
-		if (mode)
-			drm_property_unreference_blob(mode);
+		drm_property_unreference_blob(mode);
 		return ret;
 	}
 	else if (crtc->funcs->atomic_set_property)
@@ -432,11 +429,20 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 }
 EXPORT_SYMBOL(drm_atomic_crtc_set_property);
 
-/*
+/**
+ * drm_atomic_crtc_get_property - get property value from CRTC state
+ * @crtc: the drm CRTC to set a property on
+ * @state: the state object to get the property value from
+ * @property: the property to set
+ * @val: return location for the property value
+ *
  * This function handles generic/core properties and calls out to
  * driver's ->atomic_get_property() for driver properties.  To ensure
  * consistent behavior you must call this function rather than the
  * driver hook directly.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
  */
 static int
 drm_atomic_crtc_get_property(struct drm_crtc *crtc,
@@ -480,8 +486,8 @@ static int drm_atomic_crtc_check(struct drm_crtc *crtc,
 	 */
 
 	if (state->active && !state->enable) {
-		DRM_DEBUG_ATOMIC("[CRTC:%d] active without enabled\n",
-				 crtc->base.id);
+		DRM_DEBUG_ATOMIC("[CRTC:%d:%s] active without enabled\n",
+				 crtc->base.id, crtc->name);
 		return -EINVAL;
 	}
 
@@ -490,14 +496,30 @@ static int drm_atomic_crtc_check(struct drm_crtc *crtc,
 	 * be able to trigger. */
 	if (drm_core_check_feature(crtc->dev, DRIVER_ATOMIC) &&
 	    WARN_ON(state->enable && !state->mode_blob)) {
-		DRM_DEBUG_ATOMIC("[CRTC:%d] enabled without mode blob\n",
-				 crtc->base.id);
+		DRM_DEBUG_ATOMIC("[CRTC:%d:%s] enabled without mode blob\n",
+				 crtc->base.id, crtc->name);
 		return -EINVAL;
 	}
 
 	if (drm_core_check_feature(crtc->dev, DRIVER_ATOMIC) &&
 	    WARN_ON(!state->enable && state->mode_blob)) {
-		DRM_DEBUG_ATOMIC("[CRTC:%d] disabled with mode blob\n",
+		DRM_DEBUG_ATOMIC("[CRTC:%d:%s] disabled with mode blob\n",
+				 crtc->base.id, crtc->name);
+		return -EINVAL;
+	}
+
+	/*
+	 * Reject event generation for when a CRTC is off and stays off.
+	 * It wouldn't be hard to implement this, but userspace has a track
+	 * record of happily burning through 100% cpu (or worse, crash) when the
+	 * display pipe is suspended. To avoid all that fun just reject updates
+	 * that ask for events since likely that indicates a bug in the
+	 * compositor's drawing loop. This is consistent with the vblank IOCTL
+	 * and legacy page_flip IOCTL which also reject service on a disabled
+	 * pipe.
+	 */
+	if (state->event && !state->active && !crtc->state->active) {
+		DRM_DEBUG_ATOMIC("[CRTC:%d] requesting event but off\n",
 				 crtc->base.id);
 		return -EINVAL;
 	}
@@ -543,8 +565,8 @@ drm_atomic_get_plane_state(struct drm_atomic_state *state,
 	state->planes[index] = plane;
 	plane_state->state = state;
 
-	DRM_DEBUG_ATOMIC("Added [PLANE:%d] %p state to %p\n",
-			 plane->base.id, plane_state, state);
+	DRM_DEBUG_ATOMIC("Added [PLANE:%d:%s] %p state to %p\n",
+			 plane->base.id, plane->name, plane_state, state);
 
 	if (plane_state->crtc) {
 		struct drm_crtc_state *crtc_state;
@@ -619,11 +641,20 @@ int drm_atomic_plane_set_property(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_atomic_plane_set_property);
 
-/*
+/**
+ * drm_atomic_plane_get_property - get property value from plane state
+ * @plane: the drm plane to set a property on
+ * @state: the state object to get the property value from
+ * @property: the property to set
+ * @val: return location for the property value
+ *
  * This function handles generic/core properties and calls out to
  * driver's ->atomic_get_property() for driver properties.  To ensure
  * consistent behavior you must call this function rather than the
  * driver hook directly.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
  */
 static int
 drm_atomic_plane_get_property(struct drm_plane *plane,
@@ -755,8 +786,8 @@ static int drm_atomic_plane_check(struct drm_plane *plane,
 	}
 
 	if (plane_switching_crtc(state->state, plane, state)) {
-		DRM_DEBUG_ATOMIC("[PLANE:%d] switching CRTC directly\n",
-				 plane->base.id);
+		DRM_DEBUG_ATOMIC("[PLANE:%d:%s] switching CRTC directly\n",
+				 plane->base.id, plane->name);
 		return -EINVAL;
 	}
 
@@ -875,11 +906,20 @@ int drm_atomic_connector_set_property(struct drm_connector *connector,
 }
 EXPORT_SYMBOL(drm_atomic_connector_set_property);
 
-/*
+/**
+ * drm_atomic_connector_get_property - get property value from connector state
+ * @connector: the drm connector to set a property on
+ * @state: the state object to get the property value from
+ * @property: the property to set
+ * @val: return location for the property value
+ *
  * This function handles generic/core properties and calls out to
  * driver's ->atomic_get_property() for driver properties.  To ensure
  * consistent behavior you must call this function rather than the
  * driver hook directly.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
  */
 static int
 drm_atomic_connector_get_property(struct drm_connector *connector,
@@ -980,8 +1020,8 @@ drm_atomic_set_crtc_for_plane(struct drm_plane_state *plane_state,
 	}
 
 	if (crtc)
-		DRM_DEBUG_ATOMIC("Link plane state %p to [CRTC:%d]\n",
-				 plane_state, crtc->base.id);
+		DRM_DEBUG_ATOMIC("Link plane state %p to [CRTC:%d:%s]\n",
+				 plane_state, crtc->base.id, crtc->name);
 	else
 		DRM_DEBUG_ATOMIC("Link plane state %p to [NOCRTC]\n",
 				 plane_state);
@@ -1039,17 +1079,28 @@ drm_atomic_set_crtc_for_connector(struct drm_connector_state *conn_state,
 {
 	struct drm_crtc_state *crtc_state;
 
+	if (conn_state->crtc && conn_state->crtc != crtc) {
+		crtc_state = drm_atomic_get_existing_crtc_state(conn_state->state,
+								conn_state->crtc);
+
+		crtc_state->connector_mask &=
+			~(1 << drm_connector_index(conn_state->connector));
+	}
+
 	if (crtc) {
 		crtc_state = drm_atomic_get_crtc_state(conn_state->state, crtc);
 		if (IS_ERR(crtc_state))
 			return PTR_ERR(crtc_state);
+
+		crtc_state->connector_mask |=
+			1 << drm_connector_index(conn_state->connector);
 	}
 
 	conn_state->crtc = crtc;
 
 	if (crtc)
-		DRM_DEBUG_ATOMIC("Link connector state %p to [CRTC:%d]\n",
-				 conn_state, crtc->base.id);
+		DRM_DEBUG_ATOMIC("Link connector state %p to [CRTC:%d:%s]\n",
+				 conn_state, crtc->base.id, crtc->name);
 	else
 		DRM_DEBUG_ATOMIC("Link connector state %p to [NOCRTC]\n",
 				 conn_state);
@@ -1088,8 +1139,8 @@ drm_atomic_add_affected_connectors(struct drm_atomic_state *state,
 	if (ret)
 		return ret;
 
-	DRM_DEBUG_ATOMIC("Adding all current connectors for [CRTC:%d] to %p\n",
-			 crtc->base.id, state);
+	DRM_DEBUG_ATOMIC("Adding all current connectors for [CRTC:%d:%s] to %p\n",
+			 crtc->base.id, crtc->name, state);
 
 	/*
 	 * Changed connectors are already in @state, so only need to look at the
@@ -1148,35 +1199,6 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state,
 EXPORT_SYMBOL(drm_atomic_add_affected_planes);
 
 /**
- * drm_atomic_connectors_for_crtc - count number of connected outputs
- * @state: atomic state
- * @crtc: DRM crtc
- *
- * This function counts all connectors which will be connected to @crtc
- * according to @state. Useful to recompute the enable state for @crtc.
- */
-int
-drm_atomic_connectors_for_crtc(struct drm_atomic_state *state,
-			       struct drm_crtc *crtc)
-{
-	struct drm_connector *connector;
-	struct drm_connector_state *conn_state;
-
-	int i, num_connected_connectors = 0;
-
-	for_each_connector_in_state(state, connector, conn_state, i) {
-		if (conn_state->crtc == crtc)
-			num_connected_connectors++;
-	}
-
-	DRM_DEBUG_ATOMIC("State %p has %i connectors for [CRTC:%d]\n",
-			 state, num_connected_connectors, crtc->base.id);
-
-	return num_connected_connectors;
-}
-EXPORT_SYMBOL(drm_atomic_connectors_for_crtc);
-
-/**
  * drm_atomic_legacy_backoff - locking backoff for legacy ioctls
  * @state: atomic state
  *
@@ -1191,12 +1213,7 @@ void drm_atomic_legacy_backoff(struct drm_atomic_state *state)
 retry:
 	drm_modeset_backoff(state->acquire_ctx);
 
-	ret = drm_modeset_lock(&state->dev->mode_config.connection_mutex,
-			       state->acquire_ctx);
-	if (ret)
-		goto retry;
-	ret = drm_modeset_lock_all_crtcs(state->dev,
-					 state->acquire_ctx);
+	ret = drm_modeset_lock_all_ctx(state->dev, state->acquire_ctx);
 	if (ret)
 		goto retry;
 }
@@ -1228,8 +1245,8 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
 	for_each_plane_in_state(state, plane, plane_state, i) {
 		ret = drm_atomic_plane_check(plane, plane_state);
 		if (ret) {
-			DRM_DEBUG_ATOMIC("[PLANE:%d] atomic core check failed\n",
-					 plane->base.id);
+			DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic core check failed\n",
+					 plane->base.id, plane->name);
 			return ret;
 		}
 	}
@@ -1237,8 +1254,8 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
 	for_each_crtc_in_state(state, crtc, crtc_state, i) {
 		ret = drm_atomic_crtc_check(crtc, crtc_state);
 		if (ret) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] atomic core check failed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic core check failed\n",
+					 crtc->base.id, crtc->name);
 			return ret;
 		}
 	}
@@ -1249,8 +1266,8 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
 	if (!state->allow_modeset) {
 		for_each_crtc_in_state(state, crtc, crtc_state, i) {
 			if (drm_atomic_crtc_needs_modeset(crtc_state)) {
-				DRM_DEBUG_ATOMIC("[CRTC:%d] requires full modeset\n",
-						 crtc->base.id);
+				DRM_DEBUG_ATOMIC("[CRTC:%d:%s] requires full modeset\n",
+						 crtc->base.id, crtc->name);
 				return -EINVAL;
 			}
 		}
@@ -1433,7 +1450,7 @@ static int atomic_set_prop(struct drm_atomic_state *state,
 }
 
 /**
- * drm_atomic_update_old_fb -- Unset old_fb pointers and set plane->fb pointers.
+ * drm_atomic_clean_old_fb -- Unset old_fb pointers and set plane->fb pointers.
  *
  * @dev: drm device to check.
  * @plane_mask: plane mask for planes that were updated.
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index e5aec45bf985..7c523060a076 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -52,6 +52,12 @@
  * drm_atomic_helper_disable_plane(), drm_atomic_helper_disable_plane() and the
  * various functions to implement set_property callbacks. New drivers must not
  * implement these functions themselves but must use the provided helpers.
+ *
+ * The atomic helper uses the same function table structures as all other
+ * modesetting helpers. See the documentation for struct &drm_crtc_helper_funcs,
+ * struct &drm_encoder_helper_funcs and struct &drm_connector_helper_funcs. It
+ * also shares the struct &drm_plane_helper_funcs function table with the plane
+ * helpers.
  */
 static void
 drm_atomic_helper_plane_changed(struct drm_atomic_state *state,
@@ -80,6 +86,26 @@ drm_atomic_helper_plane_changed(struct drm_atomic_state *state,
 	}
 }
 
+static bool
+check_pending_encoder_assignment(struct drm_atomic_state *state,
+				 struct drm_encoder *new_encoder)
+{
+	struct drm_connector *connector;
+	struct drm_connector_state *conn_state;
+	int i;
+
+	for_each_connector_in_state(state, connector, conn_state, i) {
+		if (conn_state->best_encoder != new_encoder)
+			continue;
+
+		/* encoder already assigned and we're trying to re-steal it! */
+		if (connector->state->best_encoder != conn_state->best_encoder)
+			return false;
+	}
+
+	return true;
+}
+
 static struct drm_crtc *
 get_current_crtc_for_encoder(struct drm_device *dev,
 			     struct drm_encoder *encoder)
@@ -116,9 +142,9 @@ steal_encoder(struct drm_atomic_state *state,
 	 */
 	WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
 
-	DRM_DEBUG_ATOMIC("[ENCODER:%d:%s] in use on [CRTC:%d], stealing it\n",
+	DRM_DEBUG_ATOMIC("[ENCODER:%d:%s] in use on [CRTC:%d:%s], stealing it\n",
 			 encoder->base.id, encoder->name,
-			 encoder_crtc->base.id);
+			 encoder_crtc->base.id, encoder_crtc->name);
 
 	crtc_state = drm_atomic_get_crtc_state(state, encoder_crtc);
 	if (IS_ERR(crtc_state))
@@ -219,16 +245,24 @@ update_connector_routing(struct drm_atomic_state *state, int conn_idx)
 	}
 
 	if (new_encoder == connector_state->best_encoder) {
-		DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] keeps [ENCODER:%d:%s], now on [CRTC:%d]\n",
+		DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] keeps [ENCODER:%d:%s], now on [CRTC:%d:%s]\n",
 				 connector->base.id,
 				 connector->name,
 				 new_encoder->base.id,
 				 new_encoder->name,
-				 connector_state->crtc->base.id);
+				 connector_state->crtc->base.id,
+				 connector_state->crtc->name);
 
 		return 0;
 	}
 
+	if (!check_pending_encoder_assignment(state, new_encoder)) {
+		DRM_DEBUG_ATOMIC("Encoder for [CONNECTOR:%d:%s] already assigned\n",
+				 connector->base.id,
+				 connector->name);
+		return -EINVAL;
+	}
+
 	encoder_crtc = get_current_crtc_for_encoder(state->dev,
 						    new_encoder);
 
@@ -251,12 +285,13 @@ update_connector_routing(struct drm_atomic_state *state, int conn_idx)
 	crtc_state = state->crtc_states[idx];
 	crtc_state->connectors_changed = true;
 
-	DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d]\n",
+	DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d:%s]\n",
 			 connector->base.id,
 			 connector->name,
 			 new_encoder->base.id,
 			 new_encoder->name,
-			 connector_state->crtc->base.id);
+			 connector_state->crtc->base.id,
+			 connector_state->crtc->name);
 
 	return 0;
 }
@@ -340,8 +375,8 @@ mode_fixup(struct drm_atomic_state *state)
 		ret = funcs->mode_fixup(crtc, &crtc_state->mode,
 					&crtc_state->adjusted_mode);
 		if (!ret) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] fixup failed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] fixup failed\n",
+					 crtc->base.id, crtc->name);
 			return -EINVAL;
 		}
 	}
@@ -388,14 +423,14 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 
 	for_each_crtc_in_state(state, crtc, crtc_state, i) {
 		if (!drm_mode_equal(&crtc->state->mode, &crtc_state->mode)) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] mode changed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] mode changed\n",
+					 crtc->base.id, crtc->name);
 			crtc_state->mode_changed = true;
 		}
 
 		if (crtc->state->enable != crtc_state->enable) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] enable changed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] enable changed\n",
+					 crtc->base.id, crtc->name);
 
 			/*
 			 * For clarity this assignment is done here, but
@@ -428,7 +463,8 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 	 * crtc only changed its mode but has the same set of connectors.
 	 */
 	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		int num_connectors;
+		bool has_connectors =
+			!!crtc_state->connector_mask;
 
 		/*
 		 * We must set ->active_changed after walking connectors for
@@ -436,18 +472,18 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 		 * a full modeset because update_connector_routing force that.
 		 */
 		if (crtc->state->active != crtc_state->active) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] active changed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] active changed\n",
+					 crtc->base.id, crtc->name);
 			crtc_state->active_changed = true;
 		}
 
 		if (!drm_atomic_crtc_needs_modeset(crtc_state))
 			continue;
 
-		DRM_DEBUG_ATOMIC("[CRTC:%d] needs all connectors, enable: %c, active: %c\n",
-				 crtc->base.id,
+		DRM_DEBUG_ATOMIC("[CRTC:%d:%s] needs all connectors, enable: %c, active: %c\n",
+				 crtc->base.id, crtc->name,
 				 crtc_state->enable ? 'y' : 'n',
-			      crtc_state->active ? 'y' : 'n');
+				 crtc_state->active ? 'y' : 'n');
 
 		ret = drm_atomic_add_affected_connectors(state, crtc);
 		if (ret != 0)
@@ -457,12 +493,9 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 		if (ret != 0)
 			return ret;
 
-		num_connectors = drm_atomic_connectors_for_crtc(state,
-								crtc);
-
-		if (crtc_state->enable != !!num_connectors) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] enabled/connectors mismatch\n",
-					 crtc->base.id);
+		if (crtc_state->enable != has_connectors) {
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] enabled/connectors mismatch\n",
+					 crtc->base.id, crtc->name);
 
 			return -EINVAL;
 		}
@@ -509,8 +542,8 @@ drm_atomic_helper_check_planes(struct drm_device *dev,
 
 		ret = funcs->atomic_check(plane, plane_state);
 		if (ret) {
-			DRM_DEBUG_ATOMIC("[PLANE:%d] atomic driver check failed\n",
-					 plane->base.id);
+			DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic driver check failed\n",
+					 plane->base.id, plane->name);
 			return ret;
 		}
 	}
@@ -525,8 +558,8 @@ drm_atomic_helper_check_planes(struct drm_device *dev,
 
 		ret = funcs->atomic_check(crtc, state->crtc_states[i]);
 		if (ret) {
-			DRM_DEBUG_ATOMIC("[CRTC:%d] atomic driver check failed\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
+					 crtc->base.id, crtc->name);
 			return ret;
 		}
 	}
@@ -639,8 +672,8 @@ disable_outputs(struct drm_device *dev, struct drm_atomic_state *old_state)
 
 		funcs = crtc->helper_private;
 
-		DRM_DEBUG_ATOMIC("disabling [CRTC:%d]\n",
-				 crtc->base.id);
+		DRM_DEBUG_ATOMIC("disabling [CRTC:%d:%s]\n",
+				 crtc->base.id, crtc->name);
 
 
 		/* Right function depends upon target state. */
@@ -751,8 +784,8 @@ crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *old_state)
 		funcs = crtc->helper_private;
 
 		if (crtc->state->enable && funcs->mode_set_nofb) {
-			DRM_DEBUG_ATOMIC("modeset on [CRTC:%d]\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("modeset on [CRTC:%d:%s]\n",
+					 crtc->base.id, crtc->name);
 
 			funcs->mode_set_nofb(crtc);
 		}
@@ -851,8 +884,8 @@ void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
 		funcs = crtc->helper_private;
 
 		if (crtc->state->enable) {
-			DRM_DEBUG_ATOMIC("enabling [CRTC:%d]\n",
-					 crtc->base.id);
+			DRM_DEBUG_ATOMIC("enabling [CRTC:%d:%s]\n",
+					 crtc->base.id, crtc->name);
 
 			if (funcs->enable)
 				funcs->enable(crtc);
@@ -913,9 +946,23 @@ static void wait_for_fences(struct drm_device *dev,
 	}
 }
 
-static bool framebuffer_changed(struct drm_device *dev,
-				struct drm_atomic_state *old_state,
-				struct drm_crtc *crtc)
+/**
+ * drm_atomic_helper_framebuffer_changed - check if framebuffer has changed
+ * @dev: DRM device
+ * @old_state: atomic state object with old state structures
+ * @crtc: DRM crtc
+ *
+ * Checks whether the framebuffer used for this CRTC changes as a result of
+ * the atomic update.  This is useful for drivers which cannot use
+ * drm_atomic_helper_wait_for_vblanks() and need to reimplement its
+ * functionality.
+ *
+ * Returns:
+ * true if the framebuffer changed.
+ */
+bool drm_atomic_helper_framebuffer_changed(struct drm_device *dev,
+					   struct drm_atomic_state *old_state,
+					   struct drm_crtc *crtc)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state;
@@ -932,6 +979,7 @@ static bool framebuffer_changed(struct drm_device *dev,
 
 	return false;
 }
+EXPORT_SYMBOL(drm_atomic_helper_framebuffer_changed);
 
 /**
  * drm_atomic_helper_wait_for_vblanks - wait for vblank on crtcs
@@ -966,7 +1014,8 @@ drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
 		if (old_state->legacy_cursor_update)
 			continue;
 
-		if (!framebuffer_changed(dev, old_state, crtc))
+		if (!drm_atomic_helper_framebuffer_changed(dev,
+				old_state, crtc))
 			continue;
 
 		ret = drm_crtc_vblank_get(crtc);
@@ -1342,6 +1391,49 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state)
 EXPORT_SYMBOL(drm_atomic_helper_commit_planes_on_crtc);
 
 /**
+ * drm_atomic_helper_disable_planes_on_crtc - helper to disable CRTC's planes
+ * @crtc: CRTC
+ * @atomic: if set, synchronize with CRTC's atomic_begin/flush hooks
+ *
+ * Disables all planes associated with the given CRTC. This can be
+ * used for instance in the CRTC helper disable callback to disable
+ * all planes before shutting down the display pipeline.
+ *
+ * If the atomic-parameter is set the function calls the CRTC's
+ * atomic_begin hook before and atomic_flush hook after disabling the
+ * planes.
+ *
+ * It is a bug to call this function without having implemented the
+ * ->atomic_disable() plane hook.
+ */
+void drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc *crtc,
+					      bool atomic)
+{
+	const struct drm_crtc_helper_funcs *crtc_funcs =
+		crtc->helper_private;
+	struct drm_plane *plane;
+
+	if (atomic && crtc_funcs && crtc_funcs->atomic_begin)
+		crtc_funcs->atomic_begin(crtc, NULL);
+
+	drm_for_each_plane(plane, crtc->dev) {
+		const struct drm_plane_helper_funcs *plane_funcs =
+			plane->helper_private;
+
+		if (plane->state->crtc != crtc || !plane_funcs)
+			continue;
+
+		WARN_ON(!plane_funcs->atomic_disable);
+		if (plane_funcs->atomic_disable)
+			plane_funcs->atomic_disable(plane, NULL);
+	}
+
+	if (atomic && crtc_funcs && crtc_funcs->atomic_flush)
+		crtc_funcs->atomic_flush(crtc, NULL);
+}
+EXPORT_SYMBOL(drm_atomic_helper_disable_planes_on_crtc);
+
+/**
  * drm_atomic_helper_cleanup_planes - cleanup plane resources after commit
  * @dev: DRM device
  * @old_state: atomic state object with old state structures
@@ -1485,12 +1577,12 @@ retry:
 	drm_atomic_set_fb_for_plane(plane_state, fb);
 	plane_state->crtc_x = crtc_x;
 	plane_state->crtc_y = crtc_y;
-	plane_state->crtc_h = crtc_h;
 	plane_state->crtc_w = crtc_w;
+	plane_state->crtc_h = crtc_h;
 	plane_state->src_x = src_x;
 	plane_state->src_y = src_y;
-	plane_state->src_h = src_h;
 	plane_state->src_w = src_w;
+	plane_state->src_h = src_h;
 
 	if (plane == crtc->cursor)
 		state->legacy_cursor_update = true;
@@ -1609,12 +1701,12 @@ int __drm_atomic_helper_disable_plane(struct drm_plane *plane,
 	drm_atomic_set_fb_for_plane(plane_state, NULL);
 	plane_state->crtc_x = 0;
 	plane_state->crtc_y = 0;
-	plane_state->crtc_h = 0;
 	plane_state->crtc_w = 0;
+	plane_state->crtc_h = 0;
 	plane_state->src_x = 0;
 	plane_state->src_y = 0;
-	plane_state->src_h = 0;
 	plane_state->src_w = 0;
+	plane_state->src_h = 0;
 
 	return 0;
 }
@@ -1676,7 +1768,7 @@ static int update_output_state(struct drm_atomic_state *state,
 		if (crtc == set->crtc)
 			continue;
 
-		if (!drm_atomic_connectors_for_crtc(state, crtc)) {
+		if (!crtc_state->connector_mask) {
 			ret = drm_atomic_set_mode_prop_for_crtc(crtc_state,
 								NULL);
 			if (ret < 0)
@@ -1797,16 +1889,16 @@ int __drm_atomic_helper_set_config(struct drm_mode_set *set,
 	drm_atomic_set_fb_for_plane(primary_state, set->fb);
 	primary_state->crtc_x = 0;
 	primary_state->crtc_y = 0;
-	primary_state->crtc_h = vdisplay;
 	primary_state->crtc_w = hdisplay;
+	primary_state->crtc_h = vdisplay;
 	primary_state->src_x = set->x << 16;
 	primary_state->src_y = set->y << 16;
 	if (primary_state->rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))) {
-		primary_state->src_h = hdisplay << 16;
 		primary_state->src_w = vdisplay << 16;
+		primary_state->src_h = hdisplay << 16;
 	} else {
-		primary_state->src_h = vdisplay << 16;
 		primary_state->src_w = hdisplay << 16;
+		primary_state->src_h = vdisplay << 16;
 	}
 
 commit:
@@ -1818,6 +1910,161 @@ commit:
 }
 
 /**
+ * drm_atomic_helper_disable_all - disable all currently active outputs
+ * @dev: DRM device
+ * @ctx: lock acquisition context
+ *
+ * Loops through all connectors, finding those that aren't turned off and then
+ * turns them off by setting their DPMS mode to OFF and deactivating the CRTC
+ * that they are connected to.
+ *
+ * This is used for example in suspend/resume to disable all currently active
+ * functions when suspending.
+ *
+ * Note that if callers haven't already acquired all modeset locks this might
+ * return -EDEADLK, which must be handled by calling drm_modeset_backoff().
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ *
+ * See also:
+ * drm_atomic_helper_suspend(), drm_atomic_helper_resume()
+ */
+int drm_atomic_helper_disable_all(struct drm_device *dev,
+				  struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_atomic_state *state;
+	struct drm_connector *conn;
+	int err;
+
+	state = drm_atomic_state_alloc(dev);
+	if (!state)
+		return -ENOMEM;
+
+	state->acquire_ctx = ctx;
+
+	drm_for_each_connector(conn, dev) {
+		struct drm_crtc *crtc = conn->state->crtc;
+		struct drm_crtc_state *crtc_state;
+
+		if (!crtc || conn->dpms != DRM_MODE_DPMS_ON)
+			continue;
+
+		crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state)) {
+			err = PTR_ERR(crtc_state);
+			goto free;
+		}
+
+		crtc_state->active = false;
+	}
+
+	err = drm_atomic_commit(state);
+
+free:
+	if (err < 0)
+		drm_atomic_state_free(state);
+
+	return err;
+}
+EXPORT_SYMBOL(drm_atomic_helper_disable_all);
+
+/**
+ * drm_atomic_helper_suspend - subsystem-level suspend helper
+ * @dev: DRM device
+ *
+ * Duplicates the current atomic state, disables all active outputs and then
+ * returns a pointer to the original atomic state to the caller. Drivers can
+ * pass this pointer to the drm_atomic_helper_resume() helper upon resume to
+ * restore the output configuration that was active at the time the system
+ * entered suspend.
+ *
+ * Note that it is potentially unsafe to use this. The atomic state object
+ * returned by this function is assumed to be persistent. Drivers must ensure
+ * that this holds true. Before calling this function, drivers must make sure
+ * to suspend fbdev emulation so that nothing can be using the device.
+ *
+ * Returns:
+ * A pointer to a copy of the state before suspend on success or an ERR_PTR()-
+ * encoded error code on failure. Drivers should store the returned atomic
+ * state object and pass it to the drm_atomic_helper_resume() helper upon
+ * resume.
+ *
+ * See also:
+ * drm_atomic_helper_duplicate_state(), drm_atomic_helper_disable_all(),
+ * drm_atomic_helper_resume()
+ */
+struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev)
+{
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_state *state;
+	int err;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+retry:
+	err = drm_modeset_lock_all_ctx(dev, &ctx);
+	if (err < 0) {
+		state = ERR_PTR(err);
+		goto unlock;
+	}
+
+	state = drm_atomic_helper_duplicate_state(dev, &ctx);
+	if (IS_ERR(state))
+		goto unlock;
+
+	err = drm_atomic_helper_disable_all(dev, &ctx);
+	if (err < 0) {
+		drm_atomic_state_free(state);
+		state = ERR_PTR(err);
+		goto unlock;
+	}
+
+unlock:
+	if (PTR_ERR(state) == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+	return state;
+}
+EXPORT_SYMBOL(drm_atomic_helper_suspend);
+
+/**
+ * drm_atomic_helper_resume - subsystem-level resume helper
+ * @dev: DRM device
+ * @state: atomic state to resume to
+ *
+ * Calls drm_mode_config_reset() to synchronize hardware and software states,
+ * grabs all modeset locks and commits the atomic state object. This can be
+ * used in conjunction with the drm_atomic_helper_suspend() helper to
+ * implement suspend/resume for drivers that support atomic mode-setting.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ *
+ * See also:
+ * drm_atomic_helper_suspend()
+ */
+int drm_atomic_helper_resume(struct drm_device *dev,
+			     struct drm_atomic_state *state)
+{
+	struct drm_mode_config *config = &dev->mode_config;
+	int err;
+
+	drm_mode_config_reset(dev);
+	drm_modeset_lock_all(dev);
+	state->acquire_ctx = config->acquire_ctx;
+	err = drm_atomic_commit(state);
+	drm_modeset_unlock_all(dev);
+
+	return err;
+}
+EXPORT_SYMBOL(drm_atomic_helper_resume);
+
+/**
  * drm_atomic_helper_crtc_set_property - helper for crtc properties
  * @crtc: DRM crtc
  * @property: DRM property
@@ -2051,6 +2298,15 @@ retry:
 		goto fail;
 	drm_atomic_set_fb_for_plane(plane_state, fb);
 
+	/* Make sure we don't accidentally do a full modeset. */
+	state->allow_modeset = false;
+	if (!crtc_state->active) {
+		DRM_DEBUG_ATOMIC("[CRTC:%d] disabled, rejecting legacy flip\n",
+				 crtc->base.id);
+		ret = -EINVAL;
+		goto fail;
+	}
+
 	ret = drm_atomic_async_commit(state);
 	if (ret != 0)
 		goto fail;
@@ -2173,6 +2429,12 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_dpms);
  * The simpler solution is to just reset the software state to everything off,
  * which is easiest to do by calling drm_mode_config_reset(). To facilitate this
  * the atomic helpers provide default reset implementations for all hooks.
+ *
+ * On the upside the precise state tracking of atomic simplifies system suspend
+ * and resume a lot. For drivers using drm_mode_config_reset() a complete recipe
+ * is implemented in drm_atomic_helper_suspend() and drm_atomic_helper_resume().
+ * For other drivers the building blocks are split out, see the documentation
+ * for these functions.
  */
 
 /**
@@ -2184,7 +2446,7 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_dpms);
  */
 void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc)
 {
-	if (crtc->state && crtc->state->mode_blob)
+	if (crtc->state)
 		drm_property_unreference_blob(crtc->state->mode_blob);
 	kfree(crtc->state);
 	crtc->state = kzalloc(sizeof(*crtc->state), GFP_KERNEL);
@@ -2252,8 +2514,7 @@ EXPORT_SYMBOL(drm_atomic_helper_crtc_duplicate_state);
 void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc *crtc,
 					    struct drm_crtc_state *state)
 {
-	if (state->mode_blob)
-		drm_property_unreference_blob(state->mode_blob);
+	drm_property_unreference_blob(state->mode_blob);
 }
 EXPORT_SYMBOL(__drm_atomic_helper_crtc_destroy_state);
 
@@ -2368,6 +2629,28 @@ void drm_atomic_helper_plane_destroy_state(struct drm_plane *plane,
 EXPORT_SYMBOL(drm_atomic_helper_plane_destroy_state);
 
 /**
+ * __drm_atomic_helper_connector_reset - reset state on connector
+ * @connector: drm connector
+ * @conn_state: connector state to assign
+ *
+ * Initializes the newly allocated @conn_state and assigns it to
+ * #connector ->state, usually required when initializing the drivers
+ * or when called from the ->reset hook.
+ *
+ * This is useful for drivers that subclass the connector state.
+ */
+void
+__drm_atomic_helper_connector_reset(struct drm_connector *connector,
+				    struct drm_connector_state *conn_state)
+{
+	if (conn_state)
+		conn_state->connector = connector;
+
+	connector->state = conn_state;
+}
+EXPORT_SYMBOL(__drm_atomic_helper_connector_reset);
+
+/**
  * drm_atomic_helper_connector_reset - default ->reset hook for connectors
  * @connector: drm connector
  *
@@ -2377,11 +2660,11 @@ EXPORT_SYMBOL(drm_atomic_helper_plane_destroy_state);
  */
 void drm_atomic_helper_connector_reset(struct drm_connector *connector)
 {
-	kfree(connector->state);
-	connector->state = kzalloc(sizeof(*connector->state), GFP_KERNEL);
+	struct drm_connector_state *conn_state =
+		kzalloc(sizeof(*conn_state), GFP_KERNEL);
 
-	if (connector->state)
-		connector->state->connector = connector;
+	kfree(connector->state);
+	__drm_atomic_helper_connector_reset(connector, conn_state);
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_reset);
 
@@ -2430,7 +2713,9 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_duplicate_state);
  * @ctx: lock acquisition context
  *
  * Makes a copy of the current atomic state by looping over all objects and
- * duplicating their respective states.
+ * duplicating their respective states. This is used for example by suspend/
+ * resume support code to save the state prior to suspend such that it can
+ * be restored upon resume.
  *
  * Note that this treats atomic state as persistent between save and restore.
  * Drivers must make sure that this is possible and won't result in confusion
@@ -2442,6 +2727,9 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_duplicate_state);
  * Returns:
  * A pointer to the copy of the atomic state object on success or an
  * ERR_PTR()-encoded error code on failure.
+ *
+ * See also:
+ * drm_atomic_helper_suspend(), drm_atomic_helper_resume()
  */
 struct drm_atomic_state *
 drm_atomic_helper_duplicate_state(struct drm_device *dev,
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 6b8f7211e543..bd93453afa61 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -31,14 +31,14 @@
 /**
  * DOC: overview
  *
- * drm_bridge represents a device that hangs on to an encoder. These are handy
- * when a regular drm_encoder entity isn't enough to represent the entire
+ * struct &drm_bridge represents a device that hangs on to an encoder. These are
+ * handy when a regular &drm_encoder entity isn't enough to represent the entire
  * encoder chain.
  *
- * A bridge is always associated to a single drm_encoder at a time, but can be
+ * A bridge is always attached to a single &drm_encoder at a time, but can be
  * either connected to it directly, or through an intermediate bridge:
  *
- * encoder ---> bridge B ---> bridge A
+ *     encoder ---> bridge B ---> bridge A
  *
  * Here, the output of the encoder feeds to bridge B, and that furthers feeds to
  * bridge A.
@@ -46,11 +46,16 @@
  * The driver using the bridge is responsible to make the associations between
  * the encoder and bridges. Once these links are made, the bridges will
  * participate along with encoder functions to perform mode_set/enable/disable
- * through the ops provided in drm_bridge_funcs.
+ * through the ops provided in &drm_bridge_funcs.
  *
  * drm_bridge, like drm_panel, aren't drm_mode_object entities like planes,
- * crtcs, encoders or connectors. They just provide additional hooks to get the
- * desired output at the end of the encoder chain.
+ * CRTCs, encoders or connectors and hence are not visible to userspace. They
+ * just provide additional hooks to get the desired output at the end of the
+ * encoder chain.
+ *
+ * Bridges can also be chained up using the next pointer in struct &drm_bridge.
+ *
+ * Both legacy CRTC helpers and the new atomic modeset helpers support bridges.
  */
 
 static DEFINE_MUTEX(bridge_lock);
@@ -122,34 +127,12 @@ EXPORT_SYMBOL(drm_bridge_attach);
 /**
  * DOC: bridge callbacks
  *
- * The drm_bridge_funcs ops are populated by the bridge driver. The drm
- * internals(atomic and crtc helpers) use the helpers defined in drm_bridge.c
- * These helpers call a specific drm_bridge_funcs op for all the bridges
+ * The &drm_bridge_funcs ops are populated by the bridge driver. The DRM
+ * internals (atomic and CRTC helpers) use the helpers defined in drm_bridge.c
+ * These helpers call a specific &drm_bridge_funcs op for all the bridges
  * during encoder configuration.
  *
- * When creating a bridge driver, one can implement drm_bridge_funcs op with
- * the help of these rough rules:
- *
- * pre_enable: this contains things needed to be done for the bridge before
- * its clock and timings are enabled by its source. For a bridge, its source
- * is generally the encoder or bridge just before it in the encoder chain.
- *
- * enable: this contains things needed to be done for the bridge once its
- * source is enabled. In other words, enable is called once the source is
- * ready with clock and timing needed by the bridge.
- *
- * disable: this contains things needed to be done for the bridge assuming
- * that its source is still enabled, i.e. clock and timings are still on.
- *
- * post_disable: this contains things needed to be done for the bridge once
- * its source is disabled, i.e. once clocks and timings are off.
- *
- * mode_fixup: this should fixup the given mode for the bridge. It is called
- * after the encoder's mode fixup. mode_fixup can also reject a mode completely
- * if it's unsuitable for the hardware.
- *
- * mode_set: this sets up the mode for the bridge. It assumes that its source
- * (an encoder or a bridge) has set the mode too.
+ * For detailed specification of the bridge callbacks see &drm_bridge_funcs.
  */
 
 /**
@@ -159,7 +142,7 @@ EXPORT_SYMBOL(drm_bridge_attach);
  * @mode: desired mode to be set for the bridge
  * @adjusted_mode: updated mode that works for this bridge
  *
- * Calls 'mode_fixup' drm_bridge_funcs op for all the bridges in the
+ * Calls ->mode_fixup() &drm_bridge_funcs op for all the bridges in the
  * encoder chain, starting from the first bridge to the last.
  *
  * Note: the bridge passed should be the one closest to the encoder
@@ -186,11 +169,11 @@ bool drm_bridge_mode_fixup(struct drm_bridge *bridge,
 EXPORT_SYMBOL(drm_bridge_mode_fixup);
 
 /**
- * drm_bridge_disable - calls 'disable' drm_bridge_funcs op for all
+ * drm_bridge_disable - calls ->disable() &drm_bridge_funcs op for all
  *			bridges in the encoder chain.
  * @bridge: bridge control structure
  *
- * Calls 'disable' drm_bridge_funcs op for all the bridges in the encoder
+ * Calls ->disable() &drm_bridge_funcs op for all the bridges in the encoder
  * chain, starting from the last bridge to the first. These are called before
  * calling the encoder's prepare op.
  *
@@ -208,11 +191,11 @@ void drm_bridge_disable(struct drm_bridge *bridge)
 EXPORT_SYMBOL(drm_bridge_disable);
 
 /**
- * drm_bridge_post_disable - calls 'post_disable' drm_bridge_funcs op for
+ * drm_bridge_post_disable - calls ->post_disable() &drm_bridge_funcs op for
  *			     all bridges in the encoder chain.
  * @bridge: bridge control structure
  *
- * Calls 'post_disable' drm_bridge_funcs op for all the bridges in the
+ * Calls ->post_disable() &drm_bridge_funcs op for all the bridges in the
  * encoder chain, starting from the first bridge to the last. These are called
  * after completing the encoder's prepare op.
  *
@@ -236,7 +219,7 @@ EXPORT_SYMBOL(drm_bridge_post_disable);
  * @mode: desired mode to be set for the bridge
  * @adjusted_mode: updated mode that works for this bridge
  *
- * Calls 'mode_set' drm_bridge_funcs op for all the bridges in the
+ * Calls ->mode_set() &drm_bridge_funcs op for all the bridges in the
  * encoder chain, starting from the first bridge to the last.
  *
  * Note: the bridge passed should be the one closest to the encoder
@@ -256,11 +239,11 @@ void drm_bridge_mode_set(struct drm_bridge *bridge,
 EXPORT_SYMBOL(drm_bridge_mode_set);
 
 /**
- * drm_bridge_pre_enable - calls 'pre_enable' drm_bridge_funcs op for all
+ * drm_bridge_pre_enable - calls ->pre_enable() &drm_bridge_funcs op for all
  *			   bridges in the encoder chain.
  * @bridge: bridge control structure
  *
- * Calls 'pre_enable' drm_bridge_funcs op for all the bridges in the encoder
+ * Calls ->pre_enable() &drm_bridge_funcs op for all the bridges in the encoder
  * chain, starting from the last bridge to the first. These are called
  * before calling the encoder's commit op.
  *
@@ -278,11 +261,11 @@ void drm_bridge_pre_enable(struct drm_bridge *bridge)
 EXPORT_SYMBOL(drm_bridge_pre_enable);
 
 /**
- * drm_bridge_enable - calls 'enable' drm_bridge_funcs op for all bridges
+ * drm_bridge_enable - calls ->enable() &drm_bridge_funcs op for all bridges
  *		       in the encoder chain.
  * @bridge: bridge control structure
  *
- * Calls 'enable' drm_bridge_funcs op for all the bridges in the encoder
+ * Calls ->enable() &drm_bridge_funcs op for all the bridges in the encoder
  * chain, starting from the first bridge to the last. These are called
  * after completing the encoder's commit op.
  *
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 24c5434abd1c..d40bab29747e 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -45,7 +45,7 @@
 
 static struct drm_framebuffer *
 internal_framebuffer_create(struct drm_device *dev,
-			    struct drm_mode_fb_cmd2 *r,
+			    const struct drm_mode_fb_cmd2 *r,
 			    struct drm_file *file_priv);
 
 /* Avoid boilerplate.  I'm tired of typing. */
@@ -649,6 +649,18 @@ EXPORT_SYMBOL(drm_framebuffer_remove);
 
 DEFINE_WW_CLASS(crtc_ww_class);
 
+static unsigned int drm_num_crtcs(struct drm_device *dev)
+{
+	unsigned int num = 0;
+	struct drm_crtc *tmp;
+
+	drm_for_each_crtc(tmp, dev) {
+		num++;
+	}
+
+	return num;
+}
+
 /**
  * drm_crtc_init_with_planes - Initialise a new CRTC object with
  *    specified primary and cursor planes.
@@ -657,6 +669,7 @@ DEFINE_WW_CLASS(crtc_ww_class);
  * @primary: Primary plane for CRTC
  * @cursor: Cursor plane for CRTC
  * @funcs: callbacks for the new CRTC
+ * @name: printf style format string for the CRTC name, or NULL for default name
  *
  * Inits a new object created as base part of a driver crtc object.
  *
@@ -666,7 +679,8 @@ DEFINE_WW_CLASS(crtc_ww_class);
 int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
 			      struct drm_plane *primary,
 			      struct drm_plane *cursor,
-			      const struct drm_crtc_funcs *funcs)
+			      const struct drm_crtc_funcs *funcs,
+			      const char *name, ...)
 {
 	struct drm_mode_config *config = &dev->mode_config;
 	int ret;
@@ -682,6 +696,21 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
 	if (ret)
 		return ret;
 
+	if (name) {
+		va_list ap;
+
+		va_start(ap, name);
+		crtc->name = kvasprintf(GFP_KERNEL, name, ap);
+		va_end(ap);
+	} else {
+		crtc->name = kasprintf(GFP_KERNEL, "crtc-%d",
+				       drm_num_crtcs(dev));
+	}
+	if (!crtc->name) {
+		drm_mode_object_put(dev, &crtc->base);
+		return -ENOMEM;
+	}
+
 	crtc->base.properties = &crtc->properties;
 
 	list_add_tail(&crtc->head, &config->crtc_list);
@@ -728,6 +757,8 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 	if (crtc->state && crtc->funcs->atomic_destroy_state)
 		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
 
+	kfree(crtc->name);
+
 	memset(crtc, 0, sizeof(*crtc));
 }
 EXPORT_SYMBOL(drm_crtc_cleanup);
@@ -1075,6 +1106,7 @@ EXPORT_SYMBOL(drm_connector_unplug_all);
  * @encoder: the encoder to init
  * @funcs: callbacks for this encoder
  * @encoder_type: user visible type of the encoder
+ * @name: printf style format string for the encoder name, or NULL for default name
  *
  * Initialises a preallocated encoder. Encoder should be
  * subclassed as part of driver encoder objects.
@@ -1085,7 +1117,7 @@ EXPORT_SYMBOL(drm_connector_unplug_all);
 int drm_encoder_init(struct drm_device *dev,
 		      struct drm_encoder *encoder,
 		      const struct drm_encoder_funcs *funcs,
-		      int encoder_type)
+		      int encoder_type, const char *name, ...)
 {
 	int ret;
 
@@ -1098,9 +1130,17 @@ int drm_encoder_init(struct drm_device *dev,
 	encoder->dev = dev;
 	encoder->encoder_type = encoder_type;
 	encoder->funcs = funcs;
-	encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
-				  drm_encoder_enum_list[encoder_type].name,
-				  encoder->base.id);
+	if (name) {
+		va_list ap;
+
+		va_start(ap, name);
+		encoder->name = kvasprintf(GFP_KERNEL, name, ap);
+		va_end(ap);
+	} else {
+		encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
+					  drm_encoder_enum_list[encoder_type].name,
+					  encoder->base.id);
+	}
 	if (!encoder->name) {
 		ret = -ENOMEM;
 		goto out_put;
@@ -1141,6 +1181,18 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 }
 EXPORT_SYMBOL(drm_encoder_cleanup);
 
+static unsigned int drm_num_planes(struct drm_device *dev)
+{
+	unsigned int num = 0;
+	struct drm_plane *tmp;
+
+	drm_for_each_plane(tmp, dev) {
+		num++;
+	}
+
+	return num;
+}
+
 /**
  * drm_universal_plane_init - Initialize a new universal plane object
  * @dev: DRM device
@@ -1150,6 +1202,7 @@ EXPORT_SYMBOL(drm_encoder_cleanup);
  * @formats: array of supported formats (%DRM_FORMAT_*)
  * @format_count: number of elements in @formats
  * @type: type of plane (overlay, primary, cursor)
+ * @name: printf style format string for the plane name, or NULL for default name
  *
  * Initializes a plane object of type @type.
  *
@@ -1160,7 +1213,8 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 			     unsigned long possible_crtcs,
 			     const struct drm_plane_funcs *funcs,
 			     const uint32_t *formats, unsigned int format_count,
-			     enum drm_plane_type type)
+			     enum drm_plane_type type,
+			     const char *name, ...)
 {
 	struct drm_mode_config *config = &dev->mode_config;
 	int ret;
@@ -1182,6 +1236,22 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 		return -ENOMEM;
 	}
 
+	if (name) {
+		va_list ap;
+
+		va_start(ap, name);
+		plane->name = kvasprintf(GFP_KERNEL, name, ap);
+		va_end(ap);
+	} else {
+		plane->name = kasprintf(GFP_KERNEL, "plane-%d",
+					drm_num_planes(dev));
+	}
+	if (!plane->name) {
+		kfree(plane->format_types);
+		drm_mode_object_put(dev, &plane->base);
+		return -ENOMEM;
+	}
+
 	memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
 	plane->format_count = format_count;
 	plane->possible_crtcs = possible_crtcs;
@@ -1240,7 +1310,7 @@ int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
 
 	type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
-					formats, format_count, type);
+					formats, format_count, type, NULL);
 }
 EXPORT_SYMBOL(drm_plane_init);
 
@@ -1272,6 +1342,8 @@ void drm_plane_cleanup(struct drm_plane *plane)
 	if (plane->state && plane->funcs->atomic_destroy_state)
 		plane->funcs->atomic_destroy_state(plane, plane->state);
 
+	kfree(plane->name);
+
 	memset(plane, 0, sizeof(*plane));
 }
 EXPORT_SYMBOL(drm_plane_cleanup);
@@ -1801,7 +1873,8 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
 		copied = 0;
 		crtc_id = (uint32_t __user *)(unsigned long)card_res->crtc_id_ptr;
 		drm_for_each_crtc(crtc, dev) {
-			DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
+			DRM_DEBUG_KMS("[CRTC:%d:%s]\n",
+				      crtc->base.id, crtc->name);
 			if (put_user(crtc->base.id, crtc_id + copied)) {
 				ret = -EFAULT;
 				goto out;
@@ -2646,7 +2719,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 		ret = -ENOENT;
 		goto out;
 	}
-	DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
 	if (crtc_req->mode_valid) {
 		/* If we have a mode we need a framebuffer. */
@@ -3235,7 +3308,7 @@ static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
 
 static struct drm_framebuffer *
 internal_framebuffer_create(struct drm_device *dev,
-			    struct drm_mode_fb_cmd2 *r,
+			    const struct drm_mode_fb_cmd2 *r,
 			    struct drm_file *file_priv)
 {
 	struct drm_mode_config *config = &dev->mode_config;
@@ -4785,9 +4858,7 @@ static int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
 
 	/* Do DPMS ourselves */
 	if (property == connector->dev->mode_config.dpms_property) {
-		ret = 0;
-		if (connector->funcs->dpms)
-			ret = (*connector->funcs->dpms)(connector, (int)value);
+		ret = (*connector->funcs->dpms)(connector, (int)value);
 	} else if (connector->funcs->set_property)
 		ret = connector->funcs->set_property(connector, property, value);
 
@@ -4983,6 +5054,20 @@ int drm_mode_connector_attach_encoder(struct drm_connector *connector,
 {
 	int i;
 
+	/*
+	 * In the past, drivers have attempted to model the static association
+	 * of connector to encoder in simple connector/encoder devices using a
+	 * direct assignment of connector->encoder = encoder. This connection
+	 * is a logical one and the responsibility of the core, so drivers are
+	 * expected not to mess with this.
+	 *
+	 * Note that the error return should've been enough here, but a large
+	 * majority of drivers ignores the return value, so add in a big WARN
+	 * to get people's attention.
+	 */
+	if (WARN_ON(connector->encoder))
+		return -EINVAL;
+
 	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
 		if (connector->encoder_ids[i] == 0) {
 			connector->encoder_ids[i] = encoder->base.id;
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index ef534758a02c..a02a7f9a6a9d 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -51,6 +51,11 @@
  * the same callbacks which drivers can use to e.g. restore the modeset
  * configuration on resume with drm_helper_resume_force_mode().
  *
+ * Note that this helper library doesn't track the current power state of CRTCs
+ * and encoders. It can call callbacks like ->dpms() even though the hardware is
+ * already in the desired state. This deficiency has been fixed in the atomic
+ * helpers.
+ *
  * The driver callbacks are mostly compatible with the atomic modeset helpers,
  * except for the handling of the primary plane: Atomic helpers require that the
  * primary plane is implemented as a real standalone plane and not directly tied
@@ -62,6 +67,11 @@
  * converting to the plane helpers). New drivers must not use these functions
  * but need to implement the atomic interface instead, potentially using the
  * atomic helpers for that.
+ *
+ * These legacy modeset helpers use the same function table structures as
+ * all other modesetting helpers. See the documentation for struct
+ * &drm_crtc_helper_funcs, struct &drm_encoder_helper_funcs and struct
+ * &drm_connector_helper_funcs.
  */
 MODULE_AUTHOR("David Airlie, Jesse Barnes");
 MODULE_DESCRIPTION("DRM KMS helper");
@@ -206,8 +216,8 @@ static void __drm_helper_disable_unused_functions(struct drm_device *dev)
  * @dev: DRM device
  *
  * This function walks through the entire mode setting configuration of @dev. It
- * will remove any crtc links of unused encoders and encoder links of
- * disconnected connectors. Then it will disable all unused encoders and crtcs
+ * will remove any CRTC links of unused encoders and encoder links of
+ * disconnected connectors. Then it will disable all unused encoders and CRTCs
  * either by calling their disable callback if available or by calling their
  * dpms callback with DRM_MODE_DPMS_OFF.
  */
@@ -329,7 +339,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 		DRM_DEBUG_KMS("CRTC fixup failed\n");
 		goto done;
 	}
-	DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
 	crtc->hwmode = *adjusted_mode;
 
@@ -445,11 +455,36 @@ drm_crtc_helper_disable(struct drm_crtc *crtc)
  * drm_crtc_helper_set_config - set a new config from userspace
  * @set: mode set configuration
  *
- * Setup a new configuration, provided by the upper layers (either an ioctl call
- * from userspace or internally e.g. from the fbdev support code) in @set, and
- * enable it. This is the main helper functions for drivers that implement
- * kernel mode setting with the crtc helper functions and the assorted
- * ->prepare(), ->modeset() and ->commit() helper callbacks.
+ * The drm_crtc_helper_set_config() helper function implements the set_config
+ * callback of struct &drm_crtc_funcs for drivers using the legacy CRTC helpers.
+ *
+ * It first tries to locate the best encoder for each connector by calling the
+ * connector ->best_encoder() (struct &drm_connector_helper_funcs) helper
+ * operation.
+ *
+ * After locating the appropriate encoders, the helper function will call the
+ * mode_fixup encoder and CRTC helper operations to adjust the requested mode,
+ * or reject it completely in which case an error will be returned to the
+ * application. If the new configuration after mode adjustment is identical to
+ * the current configuration the helper function will return without performing
+ * any other operation.
+ *
+ * If the adjusted mode is identical to the current mode but changes to the
+ * frame buffer need to be applied, the drm_crtc_helper_set_config() function
+ * will call the CRTC ->mode_set_base() (struct &drm_crtc_helper_funcs) helper
+ * operation.
+ *
+ * If the adjusted mode differs from the current mode, or if the
+ * ->mode_set_base() helper operation is not provided, the helper function
+ * performs a full mode set sequence by calling the ->prepare(), ->mode_set()
+ * and ->commit() CRTC and encoder helper operations, in that order.
+ * Alternatively it can also use the dpms and disable helper operations. For
+ * details see struct &drm_crtc_helper_funcs and struct
+ * &drm_encoder_helper_funcs.
+ *
+ * This function is deprecated.  New drivers must implement atomic modeset
+ * support, for which this function is unsuitable. Instead drivers should use
+ * drm_atomic_helper_set_config().
  *
  * Returns:
  * Returns 0 on success, negative errno numbers on failure.
@@ -484,11 +519,13 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 		set->fb = NULL;
 
 	if (set->fb) {
-		DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n",
-				set->crtc->base.id, set->fb->base.id,
-				(int)set->num_connectors, set->x, set->y);
+		DRM_DEBUG_KMS("[CRTC:%d:%s] [FB:%d] #connectors=%d (x y) (%i %i)\n",
+			      set->crtc->base.id, set->crtc->name,
+			      set->fb->base.id,
+			      (int)set->num_connectors, set->x, set->y);
 	} else {
-		DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id);
+		DRM_DEBUG_KMS("[CRTC:%d:%s] [NOFB]\n",
+			      set->crtc->base.id, set->crtc->name);
 		drm_crtc_helper_disable(set->crtc);
 		return 0;
 	}
@@ -628,12 +665,12 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 			connector->encoder->crtc = new_crtc;
 		}
 		if (new_crtc) {
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d]\n",
-				connector->base.id, connector->name,
-				new_crtc->base.id);
+			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d:%s]\n",
+				      connector->base.id, connector->name,
+				      new_crtc->base.id, new_crtc->name);
 		} else {
 			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n",
-				connector->base.id, connector->name);
+				      connector->base.id, connector->name);
 		}
 	}
 
@@ -650,8 +687,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 			if (!drm_crtc_helper_set_mode(set->crtc, set->mode,
 						      set->x, set->y,
 						      save_set.fb)) {
-				DRM_ERROR("failed to set mode on [CRTC:%d]\n",
-					  set->crtc->base.id);
+				DRM_ERROR("failed to set mode on [CRTC:%d:%s]\n",
+					  set->crtc->base.id, set->crtc->name);
 				set->crtc->primary->fb = save_set.fb;
 				ret = -EINVAL;
 				goto fail;
@@ -758,10 +795,18 @@ static int drm_helper_choose_crtc_dpms(struct drm_crtc *crtc)
  * @connector: affected connector
  * @mode: DPMS mode
  *
- * This is the main helper function provided by the crtc helper framework for
+ * The drm_helper_connector_dpms() helper function implements the ->dpms()
+ * callback of struct &drm_connector_funcs for drivers using the legacy CRTC helpers.
+ *
+ * This is the main helper function provided by the CRTC helper framework for
  * implementing the DPMS connector attribute. It computes the new desired DPMS
- * state for all encoders and crtcs in the output mesh and calls the ->dpms()
- * callback provided by the driver appropriately.
+ * state for all encoders and CRTCs in the output mesh and calls the ->dpms()
+ * callbacks provided by the driver in struct &drm_crtc_helper_funcs and struct
+ * &drm_encoder_helper_funcs appropriately.
+ *
+ * This function is deprecated.  New drivers must implement atomic modeset
+ * support, for which this function is unsuitable. Instead drivers should use
+ * drm_atomic_helper_connector_dpms().
  *
  * Returns:
  * Always returns 0.
@@ -818,7 +863,7 @@ EXPORT_SYMBOL(drm_helper_connector_dpms);
  * metadata fields.
  */
 void drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
-				    struct drm_mode_fb_cmd2 *mode_cmd)
+				    const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	int i;
 
@@ -855,6 +900,12 @@ EXPORT_SYMBOL(drm_helper_mode_fill_fb_struct);
  * due to slight differences in allocating shared resources when the
  * configuration is restored in a different order than when userspace set it up)
  * need to use their own restore logic.
+ *
+ * This function is deprecated. New drivers should implement atomic mode-
+ * setting and use the atomic suspend/resume helpers.
+ *
+ * See also:
+ * drm_atomic_helper_suspend(), drm_atomic_helper_resume()
  */
 void drm_helper_resume_force_mode(struct drm_device *dev)
 {
@@ -913,9 +964,9 @@ EXPORT_SYMBOL(drm_helper_resume_force_mode);
  * @old_fb: previous framebuffer
  *
  * This function implements a callback useable as the ->mode_set callback
- * required by the crtc helpers. Besides the atomic plane helper functions for
+ * required by the CRTC helpers. Besides the atomic plane helper functions for
  * the primary plane the driver must also provide the ->mode_set_nofb callback
- * to set up the crtc.
+ * to set up the CRTC.
  *
  * This is a transitional helper useful for converting drivers to the atomic
  * interfaces.
@@ -979,7 +1030,7 @@ EXPORT_SYMBOL(drm_helper_crtc_mode_set);
  * @old_fb: previous framebuffer
  *
  * This function implements a callback useable as the ->mode_set_base used
- * required by the crtc helpers. The driver must provide the atomic plane helper
+ * required by the CRTC helpers. The driver must provide the atomic plane helper
  * functions for the primary plane.
  *
  * This is a transitional helper useful for converting drivers to the atomic
diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index 809959d56d78..6ed90a2437e5 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -666,7 +666,9 @@ static int build_enum_path_resources(struct drm_dp_sideband_msg_tx *msg, int por
 }
 
 static int build_allocate_payload(struct drm_dp_sideband_msg_tx *msg, int port_num,
-				  u8 vcpi, uint16_t pbn)
+				  u8 vcpi, uint16_t pbn,
+				  u8 number_sdp_streams,
+				  u8 *sdp_stream_sink)
 {
 	struct drm_dp_sideband_msg_req_body req;
 	memset(&req, 0, sizeof(req));
@@ -674,6 +676,9 @@ static int build_allocate_payload(struct drm_dp_sideband_msg_tx *msg, int port_n
 	req.u.allocate_payload.port_number = port_num;
 	req.u.allocate_payload.vcpi = vcpi;
 	req.u.allocate_payload.pbn = pbn;
+	req.u.allocate_payload.number_sdp_streams = number_sdp_streams;
+	memcpy(req.u.allocate_payload.sdp_stream_sink, sdp_stream_sink,
+		   number_sdp_streams);
 	drm_dp_encode_sideband_req(&req, msg);
 	msg->path_msg = true;
 	return 0;
@@ -973,17 +978,17 @@ static struct drm_dp_mst_port *drm_dp_get_port(struct drm_dp_mst_branch *mstb, u
 static u8 drm_dp_calculate_rad(struct drm_dp_mst_port *port,
 				 u8 *rad)
 {
-	int lct = port->parent->lct;
+	int parent_lct = port->parent->lct;
 	int shift = 4;
-	int idx = lct / 2;
-	if (lct > 1) {
-		memcpy(rad, port->parent->rad, idx);
-		shift = (lct % 2) ? 4 : 0;
+	int idx = (parent_lct - 1) / 2;
+	if (parent_lct > 1) {
+		memcpy(rad, port->parent->rad, idx + 1);
+		shift = (parent_lct % 2) ? 4 : 0;
 	} else
 		rad[0] = 0;
 
 	rad[idx] |= port->port_num << shift;
-	return lct + 1;
+	return parent_lct + 1;
 }
 
 /*
@@ -1039,7 +1044,7 @@ static void build_mst_prop_path(const struct drm_dp_mst_branch *mstb,
 	snprintf(proppath, proppath_size, "mst:%d", mstb->mgr->conn_base_id);
 	for (i = 0; i < (mstb->lct - 1); i++) {
 		int shift = (i % 2) ? 0 : 4;
-		int port_num = mstb->rad[i / 2] >> shift;
+		int port_num = (mstb->rad[i / 2] >> shift) & 0xf;
 		snprintf(temp, sizeof(temp), "-%d", port_num);
 		strlcat(proppath, temp, proppath_size);
 	}
@@ -1190,7 +1195,7 @@ static struct drm_dp_mst_branch *drm_dp_get_mst_branch_device(struct drm_dp_mst_
 
 	for (i = 0; i < lct - 1; i++) {
 		int shift = (i % 2) ? 0 : 4;
-		int port_num = rad[i / 2] >> shift;
+		int port_num = (rad[i / 2] >> shift) & 0xf;
 
 		list_for_each_entry(port, &mstb->ports, next) {
 			if (port->port_num == port_num) {
@@ -1210,6 +1215,50 @@ out:
 	return mstb;
 }
 
+static struct drm_dp_mst_branch *get_mst_branch_device_by_guid_helper(
+	struct drm_dp_mst_branch *mstb,
+	uint8_t *guid)
+{
+	struct drm_dp_mst_branch *found_mstb;
+	struct drm_dp_mst_port *port;
+
+	list_for_each_entry(port, &mstb->ports, next) {
+		if (!port->mstb)
+			continue;
+
+		if (port->guid_valid && memcmp(port->guid, guid, 16) == 0)
+			return port->mstb;
+
+		found_mstb = get_mst_branch_device_by_guid_helper(port->mstb, guid);
+
+		if (found_mstb)
+			return found_mstb;
+	}
+
+	return NULL;
+}
+
+static struct drm_dp_mst_branch *drm_dp_get_mst_branch_device_by_guid(
+	struct drm_dp_mst_topology_mgr *mgr,
+	uint8_t *guid)
+{
+	struct drm_dp_mst_branch *mstb;
+
+	/* find the port by iterating down */
+	mutex_lock(&mgr->lock);
+
+	if (mgr->guid_valid && memcmp(mgr->guid, guid, 16) == 0)
+		mstb = mgr->mst_primary;
+	else
+		mstb = get_mst_branch_device_by_guid_helper(mgr->mst_primary, guid);
+
+	if (mstb)
+		kref_get(&mstb->kref);
+
+	mutex_unlock(&mgr->lock);
+	return mstb;
+}
+
 static void drm_dp_check_and_send_link_address(struct drm_dp_mst_topology_mgr *mgr,
 					       struct drm_dp_mst_branch *mstb)
 {
@@ -1320,6 +1369,7 @@ static int set_hdr_from_dst_qlock(struct drm_dp_sideband_msg_hdr *hdr,
 				  struct drm_dp_sideband_msg_tx *txmsg)
 {
 	struct drm_dp_mst_branch *mstb = txmsg->dst;
+	u8 req_type;
 
 	/* both msg slots are full */
 	if (txmsg->seqno == -1) {
@@ -1336,7 +1386,13 @@ static int set_hdr_from_dst_qlock(struct drm_dp_sideband_msg_hdr *hdr,
 			txmsg->seqno = 1;
 		mstb->tx_slots[txmsg->seqno] = txmsg;
 	}
-	hdr->broadcast = 0;
+
+	req_type = txmsg->msg[0] & 0x7f;
+	if (req_type == DP_CONNECTION_STATUS_NOTIFY ||
+		req_type == DP_RESOURCE_STATUS_NOTIFY)
+		hdr->broadcast = 1;
+	else
+		hdr->broadcast = 0;
 	hdr->path_msg = txmsg->path_msg;
 	hdr->lct = mstb->lct;
 	hdr->lcr = mstb->lct - 1;
@@ -1438,26 +1494,18 @@ static void process_single_down_tx_qlock(struct drm_dp_mst_topology_mgr *mgr)
 }
 
 /* called holding qlock */
-static void process_single_up_tx_qlock(struct drm_dp_mst_topology_mgr *mgr)
+static void process_single_up_tx_qlock(struct drm_dp_mst_topology_mgr *mgr,
+				       struct drm_dp_sideband_msg_tx *txmsg)
 {
-	struct drm_dp_sideband_msg_tx *txmsg;
 	int ret;
 
 	/* construct a chunk from the first msg in the tx_msg queue */
-	if (list_empty(&mgr->tx_msg_upq)) {
-		mgr->tx_up_in_progress = false;
-		return;
-	}
-
-	txmsg = list_first_entry(&mgr->tx_msg_upq, struct drm_dp_sideband_msg_tx, next);
 	ret = process_single_tx_qlock(mgr, txmsg, true);
-	if (ret == 1) {
-		/* up txmsgs aren't put in slots - so free after we send it */
-		list_del(&txmsg->next);
-		kfree(txmsg);
-	} else if (ret)
+
+	if (ret != 1)
 		DRM_DEBUG_KMS("failed to send msg in q %d\n", ret);
-	mgr->tx_up_in_progress = true;
+
+	txmsg->dst->tx_slots[txmsg->seqno] = NULL;
 }
 
 static void drm_dp_queue_down_tx(struct drm_dp_mst_topology_mgr *mgr,
@@ -1562,6 +1610,8 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr,
 	struct drm_dp_sideband_msg_tx *txmsg;
 	struct drm_dp_mst_branch *mstb;
 	int len, ret;
+	u8 sinks[DRM_DP_MAX_SDP_STREAMS];
+	int i;
 
 	mstb = drm_dp_get_validated_mstb_ref(mgr, port->parent);
 	if (!mstb)
@@ -1573,10 +1623,13 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr,
 		goto fail_put;
 	}
 
+	for (i = 0; i < port->num_sdp_streams; i++)
+		sinks[i] = i;
+
 	txmsg->dst = mstb;
 	len = build_allocate_payload(txmsg, port->port_num,
 				     id,
-				     pbn);
+				     pbn, port->num_sdp_streams, sinks);
 
 	drm_dp_queue_down_tx(mgr, txmsg);
 
@@ -1673,6 +1726,7 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr)
 		if (mgr->proposed_vcpis[i]) {
 			port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi);
 			req_payload.num_slots = mgr->proposed_vcpis[i]->num_slots;
+			req_payload.vcpi = mgr->proposed_vcpis[i]->vcpi;
 		} else {
 			port = NULL;
 			req_payload.num_slots = 0;
@@ -1688,6 +1742,7 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr)
 			if (req_payload.num_slots) {
 				drm_dp_create_payload_step1(mgr, mgr->proposed_vcpis[i]->vcpi, &req_payload);
 				mgr->payloads[i].num_slots = req_payload.num_slots;
+				mgr->payloads[i].vcpi = req_payload.vcpi;
 			} else if (mgr->payloads[i].num_slots) {
 				mgr->payloads[i].num_slots = 0;
 				drm_dp_destroy_payload_step1(mgr, port, port->vcpi.vcpi, &mgr->payloads[i]);
@@ -1823,7 +1878,7 @@ static int drm_dp_encode_up_ack_reply(struct drm_dp_sideband_msg_tx *msg, u8 req
 {
 	struct drm_dp_sideband_msg_reply_body reply;
 
-	reply.reply_type = 1;
+	reply.reply_type = 0;
 	reply.req_type = req_type;
 	drm_dp_encode_sideband_reply(&reply, msg);
 	return 0;
@@ -1844,11 +1899,12 @@ static int drm_dp_send_up_ack_reply(struct drm_dp_mst_topology_mgr *mgr,
 	drm_dp_encode_up_ack_reply(txmsg, req_type);
 
 	mutex_lock(&mgr->qlock);
-	list_add_tail(&txmsg->next, &mgr->tx_msg_upq);
-	if (!mgr->tx_up_in_progress) {
-		process_single_up_tx_qlock(mgr);
-	}
+
+	process_single_up_tx_qlock(mgr, txmsg);
+
 	mutex_unlock(&mgr->qlock);
+
+	kfree(txmsg);
 	return 0;
 }
 
@@ -2145,28 +2201,50 @@ static int drm_dp_mst_handle_up_req(struct drm_dp_mst_topology_mgr *mgr)
 
 	if (mgr->up_req_recv.have_eomt) {
 		struct drm_dp_sideband_msg_req_body msg;
-		struct drm_dp_mst_branch *mstb;
+		struct drm_dp_mst_branch *mstb = NULL;
 		bool seqno;
-		mstb = drm_dp_get_mst_branch_device(mgr,
-						    mgr->up_req_recv.initial_hdr.lct,
-						    mgr->up_req_recv.initial_hdr.rad);
-		if (!mstb) {
-			DRM_DEBUG_KMS("Got MST reply from unknown device %d\n", mgr->up_req_recv.initial_hdr.lct);
-			memset(&mgr->up_req_recv, 0, sizeof(struct drm_dp_sideband_msg_rx));
-			return 0;
+
+		if (!mgr->up_req_recv.initial_hdr.broadcast) {
+			mstb = drm_dp_get_mst_branch_device(mgr,
+							    mgr->up_req_recv.initial_hdr.lct,
+							    mgr->up_req_recv.initial_hdr.rad);
+			if (!mstb) {
+				DRM_DEBUG_KMS("Got MST reply from unknown device %d\n", mgr->up_req_recv.initial_hdr.lct);
+				memset(&mgr->up_req_recv, 0, sizeof(struct drm_dp_sideband_msg_rx));
+				return 0;
+			}
 		}
 
 		seqno = mgr->up_req_recv.initial_hdr.seqno;
 		drm_dp_sideband_parse_req(&mgr->up_req_recv, &msg);
 
 		if (msg.req_type == DP_CONNECTION_STATUS_NOTIFY) {
-			drm_dp_send_up_ack_reply(mgr, mstb, msg.req_type, seqno, false);
+			drm_dp_send_up_ack_reply(mgr, mgr->mst_primary, msg.req_type, seqno, false);
+
+			if (!mstb)
+				mstb = drm_dp_get_mst_branch_device_by_guid(mgr, msg.u.conn_stat.guid);
+
+			if (!mstb) {
+				DRM_DEBUG_KMS("Got MST reply from unknown device %d\n", mgr->up_req_recv.initial_hdr.lct);
+				memset(&mgr->up_req_recv, 0, sizeof(struct drm_dp_sideband_msg_rx));
+				return 0;
+			}
+
 			drm_dp_update_port(mstb, &msg.u.conn_stat);
 			DRM_DEBUG_KMS("Got CSN: pn: %d ldps:%d ddps: %d mcs: %d ip: %d pdt: %d\n", msg.u.conn_stat.port_number, msg.u.conn_stat.legacy_device_plug_status, msg.u.conn_stat.displayport_device_plug_status, msg.u.conn_stat.message_capability_status, msg.u.conn_stat.input_port, msg.u.conn_stat.peer_device_type);
 			(*mgr->cbs->hotplug)(mgr);
 
 		} else if (msg.req_type == DP_RESOURCE_STATUS_NOTIFY) {
-			drm_dp_send_up_ack_reply(mgr, mstb, msg.req_type, seqno, false);
+			drm_dp_send_up_ack_reply(mgr, mgr->mst_primary, msg.req_type, seqno, false);
+			if (!mstb)
+				mstb = drm_dp_get_mst_branch_device_by_guid(mgr, msg.u.resource_stat.guid);
+
+			if (!mstb) {
+				DRM_DEBUG_KMS("Got MST reply from unknown device %d\n", mgr->up_req_recv.initial_hdr.lct);
+				memset(&mgr->up_req_recv, 0, sizeof(struct drm_dp_sideband_msg_rx));
+				return 0;
+			}
+
 			DRM_DEBUG_KMS("Got RSN: pn: %d avail_pbn %d\n", msg.u.resource_stat.port_number, msg.u.resource_stat.available_pbn);
 		}
 
@@ -2259,6 +2337,27 @@ out:
 EXPORT_SYMBOL(drm_dp_mst_detect_port);
 
 /**
+ * drm_dp_mst_port_has_audio() - Check whether port has audio capability or not
+ * @mgr: manager for this port
+ * @port: unverified pointer to a port.
+ *
+ * This returns whether the port supports audio or not.
+ */
+bool drm_dp_mst_port_has_audio(struct drm_dp_mst_topology_mgr *mgr,
+					struct drm_dp_mst_port *port)
+{
+	bool ret = false;
+
+	port = drm_dp_get_validated_port_ref(mgr, port);
+	if (!port)
+		return ret;
+	ret = port->has_audio;
+	drm_dp_put_port(port);
+	return ret;
+}
+EXPORT_SYMBOL(drm_dp_mst_port_has_audio);
+
+/**
  * drm_dp_mst_get_edid() - get EDID for an MST port
  * @connector: toplevel connector to get EDID for
  * @mgr: manager for this port
@@ -2283,6 +2382,7 @@ struct edid *drm_dp_mst_get_edid(struct drm_connector *connector, struct drm_dp_
 		edid = drm_get_edid(connector, &port->aux.ddc);
 		drm_mode_connector_set_tile_property(connector);
 	}
+	port->has_audio = drm_detect_monitor_audio(edid);
 	drm_dp_put_port(port);
 	return edid;
 }
@@ -2566,7 +2666,7 @@ static void drm_dp_mst_dump_mstb(struct seq_file *m,
 
 	seq_printf(m, "%smst: %p, %d\n", prefix, mstb, mstb->num_ports);
 	list_for_each_entry(port, &mstb->ports, next) {
-		seq_printf(m, "%sport: %d: ddps: %d ldps: %d, %p, conn: %p\n", prefix, port->port_num, port->ddps, port->ldps, port, port->connector);
+		seq_printf(m, "%sport: %d: ddps: %d ldps: %d, sdp: %d/%d, %p, conn: %p\n", prefix, port->port_num, port->ddps, port->ldps, port->num_sdp_streams, port->num_sdp_stream_sinks, port, port->connector);
 		if (port->mstb)
 			drm_dp_mst_dump_mstb(m, port->mstb);
 	}
@@ -2736,7 +2836,6 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 	mutex_init(&mgr->qlock);
 	mutex_init(&mgr->payload_lock);
 	mutex_init(&mgr->destroy_connector_lock);
-	INIT_LIST_HEAD(&mgr->tx_msg_upq);
 	INIT_LIST_HEAD(&mgr->tx_msg_downq);
 	INIT_LIST_HEAD(&mgr->destroy_connector_list);
 	INIT_WORK(&mgr->work, drm_dp_mst_link_probe_work);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 7dd6728dd092..167c8d3d4a31 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -44,10 +44,6 @@ MODULE_AUTHOR(CORE_AUTHOR);
 MODULE_DESCRIPTION(CORE_DESC);
 MODULE_LICENSE("GPL and additional rights");
 MODULE_PARM_DESC(debug, "Enable debug output");
-MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
-MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
-MODULE_PARM_DESC(timestamp_monotonic, "Use monotonic timestamps");
-
 module_param_named(debug, drm_debug, int, 0600);
 
 static DEFINE_SPINLOCK(drm_minor_lock);
@@ -633,8 +629,17 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
 		}
 	}
 
+	if (parent) {
+		ret = drm_dev_set_unique(dev, dev_name(parent));
+		if (ret)
+			goto err_setunique;
+	}
+
 	return dev;
 
+err_setunique:
+	if (drm_core_check_feature(dev, DRIVER_GEM))
+		drm_gem_destroy(dev);
 err_ctxbitmap:
 	drm_legacy_ctxbitmap_cleanup(dev);
 	drm_ht_remove(&dev->map_hash);
@@ -797,23 +802,18 @@ EXPORT_SYMBOL(drm_dev_unregister);
 /**
  * drm_dev_set_unique - Set the unique name of a DRM device
  * @dev: device of which to set the unique name
- * @fmt: format string for unique name
+ * @name: unique name
  *
- * Sets the unique name of a DRM device using the specified format string and
- * a variable list of arguments. Drivers can use this at driver probe time if
- * the unique name of the devices they drive is static.
+ * Sets the unique name of a DRM device using the specified string. Drivers
+ * can use this at driver probe time if the unique name of the devices they
+ * drive is static.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int drm_dev_set_unique(struct drm_device *dev, const char *fmt, ...)
+int drm_dev_set_unique(struct drm_device *dev, const char *name)
 {
-	va_list ap;
-
 	kfree(dev->unique);
-
-	va_start(ap, fmt);
-	dev->unique = kvasprintf(GFP_KERNEL, fmt, ap);
-	va_end(ap);
+	dev->unique = kstrdup(name, GFP_KERNEL);
 
 	return dev->unique ? 0 : -ENOMEM;
 }
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index d5d2c03fd136..04cb4877fabd 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -637,8 +637,12 @@ static const struct minimode extra_modes[] = {
 /*
  * Probably taken from CEA-861 spec.
  * This table is converted from xorg's hw/xfree86/modes/xf86EdidModes.c.
+ *
+ * Index using the VIC.
  */
 static const struct drm_display_mode edid_cea_modes[] = {
+	/* 0 - dummy, VICs start at 1 */
+	{ },
 	/* 1 - 640x480@60Hz */
 	{ DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
 		   752, 800, 0, 480, 490, 492, 525, 0,
@@ -987,9 +991,11 @@ static const struct drm_display_mode edid_cea_modes[] = {
 };
 
 /*
- * HDMI 1.4 4k modes.
+ * HDMI 1.4 4k modes. Index using the VIC.
  */
 static const struct drm_display_mode edid_4k_modes[] = {
+	/* 0 - dummy, VICs start at 1 */
+	{ },
 	/* 1 - 3840x2160@30Hz */
 	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000,
 		   3840, 4016, 4104, 4400, 0,
@@ -2545,6 +2551,33 @@ cea_mode_alternate_clock(const struct drm_display_mode *cea_mode)
 	return clock;
 }
 
+static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_match,
+					     unsigned int clock_tolerance)
+{
+	u8 vic;
+
+	if (!to_match->clock)
+		return 0;
+
+	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
+		const struct drm_display_mode *cea_mode = &edid_cea_modes[vic];
+		unsigned int clock1, clock2;
+
+		/* Check both 60Hz and 59.94Hz */
+		clock1 = cea_mode->clock;
+		clock2 = cea_mode_alternate_clock(cea_mode);
+
+		if (abs(to_match->clock - clock1) > clock_tolerance &&
+		    abs(to_match->clock - clock2) > clock_tolerance)
+			continue;
+
+		if (drm_mode_equal_no_clocks(to_match, cea_mode))
+			return vic;
+	}
+
+	return 0;
+}
+
 /**
  * drm_match_cea_mode - look for a CEA mode matching given mode
  * @to_match: display mode
@@ -2554,13 +2587,13 @@ cea_mode_alternate_clock(const struct drm_display_mode *cea_mode)
  */
 u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 {
-	u8 mode;
+	u8 vic;
 
 	if (!to_match->clock)
 		return 0;
 
-	for (mode = 0; mode < ARRAY_SIZE(edid_cea_modes); mode++) {
-		const struct drm_display_mode *cea_mode = &edid_cea_modes[mode];
+	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
+		const struct drm_display_mode *cea_mode = &edid_cea_modes[vic];
 		unsigned int clock1, clock2;
 
 		/* Check both 60Hz and 59.94Hz */
@@ -2570,12 +2603,17 @@ u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 		if ((KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock1) ||
 		     KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock2)) &&
 		    drm_mode_equal_no_clocks_no_stereo(to_match, cea_mode))
-			return mode + 1;
+			return vic;
 	}
 	return 0;
 }
 EXPORT_SYMBOL(drm_match_cea_mode);
 
+static bool drm_valid_cea_vic(u8 vic)
+{
+	return vic > 0 && vic < ARRAY_SIZE(edid_cea_modes);
+}
+
 /**
  * drm_get_cea_aspect_ratio - get the picture aspect ratio corresponding to
  * the input VIC from the CEA mode list
@@ -2585,10 +2623,7 @@ EXPORT_SYMBOL(drm_match_cea_mode);
  */
 enum hdmi_picture_aspect drm_get_cea_aspect_ratio(const u8 video_code)
 {
-	/* return picture aspect ratio for video_code - 1 to access the
-	 * right array element
-	*/
-	return edid_cea_modes[video_code-1].picture_aspect_ratio;
+	return edid_cea_modes[video_code].picture_aspect_ratio;
 }
 EXPORT_SYMBOL(drm_get_cea_aspect_ratio);
 
@@ -2609,6 +2644,33 @@ hdmi_mode_alternate_clock(const struct drm_display_mode *hdmi_mode)
 	return cea_mode_alternate_clock(hdmi_mode);
 }
 
+static u8 drm_match_hdmi_mode_clock_tolerance(const struct drm_display_mode *to_match,
+					      unsigned int clock_tolerance)
+{
+	u8 vic;
+
+	if (!to_match->clock)
+		return 0;
+
+	for (vic = 1; vic < ARRAY_SIZE(edid_4k_modes); vic++) {
+		const struct drm_display_mode *hdmi_mode = &edid_4k_modes[vic];
+		unsigned int clock1, clock2;
+
+		/* Make sure to also match alternate clocks */
+		clock1 = hdmi_mode->clock;
+		clock2 = hdmi_mode_alternate_clock(hdmi_mode);
+
+		if (abs(to_match->clock - clock1) > clock_tolerance &&
+		    abs(to_match->clock - clock2) > clock_tolerance)
+			continue;
+
+		if (drm_mode_equal_no_clocks(to_match, hdmi_mode))
+			return vic;
+	}
+
+	return 0;
+}
+
 /*
  * drm_match_hdmi_mode - look for a HDMI mode matching given mode
  * @to_match: display mode
@@ -2619,13 +2681,13 @@ hdmi_mode_alternate_clock(const struct drm_display_mode *hdmi_mode)
  */
 static u8 drm_match_hdmi_mode(const struct drm_display_mode *to_match)
 {
-	u8 mode;
+	u8 vic;
 
 	if (!to_match->clock)
 		return 0;
 
-	for (mode = 0; mode < ARRAY_SIZE(edid_4k_modes); mode++) {
-		const struct drm_display_mode *hdmi_mode = &edid_4k_modes[mode];
+	for (vic = 1; vic < ARRAY_SIZE(edid_4k_modes); vic++) {
+		const struct drm_display_mode *hdmi_mode = &edid_4k_modes[vic];
 		unsigned int clock1, clock2;
 
 		/* Make sure to also match alternate clocks */
@@ -2635,11 +2697,16 @@ static u8 drm_match_hdmi_mode(const struct drm_display_mode *to_match)
 		if ((KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock1) ||
 		     KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock2)) &&
 		    drm_mode_equal_no_clocks_no_stereo(to_match, hdmi_mode))
-			return mode + 1;
+			return vic;
 	}
 	return 0;
 }
 
+static bool drm_valid_hdmi_vic(u8 vic)
+{
+	return vic > 0 && vic < ARRAY_SIZE(edid_4k_modes);
+}
+
 static int
 add_alternate_cea_modes(struct drm_connector *connector, struct edid *edid)
 {
@@ -2659,16 +2726,16 @@ add_alternate_cea_modes(struct drm_connector *connector, struct edid *edid)
 	list_for_each_entry(mode, &connector->probed_modes, head) {
 		const struct drm_display_mode *cea_mode = NULL;
 		struct drm_display_mode *newmode;
-		u8 mode_idx = drm_match_cea_mode(mode) - 1;
+		u8 vic = drm_match_cea_mode(mode);
 		unsigned int clock1, clock2;
 
-		if (mode_idx < ARRAY_SIZE(edid_cea_modes)) {
-			cea_mode = &edid_cea_modes[mode_idx];
+		if (drm_valid_cea_vic(vic)) {
+			cea_mode = &edid_cea_modes[vic];
 			clock2 = cea_mode_alternate_clock(cea_mode);
 		} else {
-			mode_idx = drm_match_hdmi_mode(mode) - 1;
-			if (mode_idx < ARRAY_SIZE(edid_4k_modes)) {
-				cea_mode = &edid_4k_modes[mode_idx];
+			vic = drm_match_hdmi_mode(mode);
+			if (drm_valid_hdmi_vic(vic)) {
+				cea_mode = &edid_4k_modes[vic];
 				clock2 = hdmi_mode_alternate_clock(cea_mode);
 			}
 		}
@@ -2719,17 +2786,17 @@ drm_display_mode_from_vic_index(struct drm_connector *connector,
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *newmode;
-	u8 cea_mode;
+	u8 vic;
 
 	if (video_db == NULL || video_index >= video_len)
 		return NULL;
 
 	/* CEA modes are numbered 1..127 */
-	cea_mode = (video_db[video_index] & 127) - 1;
-	if (cea_mode >= ARRAY_SIZE(edid_cea_modes))
+	vic = (video_db[video_index] & 127);
+	if (!drm_valid_cea_vic(vic))
 		return NULL;
 
-	newmode = drm_mode_duplicate(dev, &edid_cea_modes[cea_mode]);
+	newmode = drm_mode_duplicate(dev, &edid_cea_modes[vic]);
 	if (!newmode)
 		return NULL;
 
@@ -2824,8 +2891,7 @@ static int add_hdmi_mode(struct drm_connector *connector, u8 vic)
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *newmode;
 
-	vic--; /* VICs start at 1 */
-	if (vic >= ARRAY_SIZE(edid_4k_modes)) {
+	if (!drm_valid_hdmi_vic(vic)) {
 		DRM_ERROR("Unknown HDMI VIC: %d\n", vic);
 		return 0;
 	}
@@ -3116,20 +3182,24 @@ static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode)
 {
 	const struct drm_display_mode *cea_mode;
 	int clock1, clock2, clock;
-	u8 mode_idx;
+	u8 vic;
 	const char *type;
 
-	mode_idx = drm_match_cea_mode(mode) - 1;
-	if (mode_idx < ARRAY_SIZE(edid_cea_modes)) {
+	/*
+	 * allow 5kHz clock difference either way to account for
+	 * the 10kHz clock resolution limit of detailed timings.
+	 */
+	vic = drm_match_cea_mode_clock_tolerance(mode, 5);
+	if (drm_valid_cea_vic(vic)) {
 		type = "CEA";
-		cea_mode = &edid_cea_modes[mode_idx];
+		cea_mode = &edid_cea_modes[vic];
 		clock1 = cea_mode->clock;
 		clock2 = cea_mode_alternate_clock(cea_mode);
 	} else {
-		mode_idx = drm_match_hdmi_mode(mode) - 1;
-		if (mode_idx < ARRAY_SIZE(edid_4k_modes)) {
+		vic = drm_match_hdmi_mode_clock_tolerance(mode, 5);
+		if (drm_valid_hdmi_vic(vic)) {
 			type = "HDMI";
-			cea_mode = &edid_4k_modes[mode_idx];
+			cea_mode = &edid_4k_modes[vic];
 			clock1 = cea_mode->clock;
 			clock2 = hdmi_mode_alternate_clock(cea_mode);
 		} else {
@@ -3147,7 +3217,7 @@ static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode)
 		return;
 
 	DRM_DEBUG("detailed mode matches %s VIC %d, adjusting clock %d -> %d\n",
-		  type, mode_idx + 1, mode->clock, clock);
+		  type, vic, mode->clock, clock);
 	mode->clock = clock;
 }
 
diff --git a/drivers/gpu/drm/drm_encoder_slave.c b/drivers/gpu/drm/drm_encoder_slave.c
index d18b88b755c3..e8629076de32 100644
--- a/drivers/gpu/drm/drm_encoder_slave.c
+++ b/drivers/gpu/drm/drm_encoder_slave.c
@@ -124,7 +124,7 @@ EXPORT_SYMBOL(drm_i2c_encoder_destroy);
  * Wrapper fxns which can be plugged in to drm_encoder_helper_funcs:
  */
 
-static inline struct drm_encoder_slave_funcs *
+static inline const struct drm_encoder_slave_funcs *
 get_slave_funcs(struct drm_encoder *enc)
 {
 	return to_encoder_slave(enc)->slave_funcs;
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
index c19a62561183..c895b6fddbd8 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -74,7 +74,7 @@ static struct drm_framebuffer_funcs drm_fb_cma_funcs = {
 };
 
 static struct drm_fb_cma *drm_fb_cma_alloc(struct drm_device *dev,
-	struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_cma_object **obj,
+	const const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_cma_object **obj,
 	unsigned int num_planes)
 {
 	struct drm_fb_cma *fb_cma;
@@ -107,7 +107,7 @@ static struct drm_fb_cma *drm_fb_cma_alloc(struct drm_device *dev,
  * checked before calling this function.
  */
 struct drm_framebuffer *drm_fb_cma_create(struct drm_device *dev,
-	struct drm_file *file_priv, struct drm_mode_fb_cmd2 *mode_cmd)
+	struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_fb_cma *fb_cma;
 	struct drm_gem_cma_object *objs[4];
@@ -266,7 +266,7 @@ static int drm_fbdev_cma_create(struct drm_fb_helper *helper,
 	fbi = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(fbi)) {
 		ret = PTR_ERR(fbi);
-		goto err_drm_gem_cma_free_object;
+		goto err_gem_free_object;
 	}
 
 	fbdev_cma->fb = drm_fb_cma_alloc(dev, &mode_cmd, &obj, 1);
@@ -299,8 +299,8 @@ static int drm_fbdev_cma_create(struct drm_fb_helper *helper,
 
 err_fb_info_destroy:
 	drm_fb_helper_release_fbi(helper);
-err_drm_gem_cma_free_object:
-	drm_gem_cma_free_object(&obj->base);
+err_gem_free_object:
+	dev->driver->gem_free_object(&obj->base);
 	return ret;
 }
 
@@ -348,9 +348,6 @@ struct drm_fbdev_cma *drm_fbdev_cma_init(struct drm_device *dev,
 
 	}
 
-	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(dev);
-
 	ret = drm_fb_helper_initial_config(helper, preferred_bpp);
 	if (ret < 0) {
 		dev_err(dev->dev, "Failed to set initial hw configuration.\n");
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 69cbab5e5c81..1e103c4c6ee0 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -1251,7 +1251,7 @@ retry:
 			goto fail;
 
 		plane = mode_set->crtc->primary;
-		plane_mask |= drm_plane_index(plane);
+		plane_mask |= (1 << drm_plane_index(plane));
 		plane->old_fb = plane->fb;
 	}
 
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 6b5625e66119..1ea8790e5090 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -226,6 +226,8 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor)
 	init_waitqueue_head(&priv->event_wait);
 	priv->event_space = 4096; /* set aside 4k for event buffer */
 
+	mutex_init(&priv->event_read_lock);
+
 	if (drm_core_check_feature(dev, DRIVER_GEM))
 		drm_gem_open(dev, priv);
 
@@ -511,14 +513,28 @@ ssize_t drm_read(struct file *filp, char __user *buffer,
 {
 	struct drm_file *file_priv = filp->private_data;
 	struct drm_device *dev = file_priv->minor->dev;
-	ssize_t ret = 0;
+	ssize_t ret;
 
 	if (!access_ok(VERIFY_WRITE, buffer, count))
 		return -EFAULT;
 
-	spin_lock_irq(&dev->event_lock);
+	ret = mutex_lock_interruptible(&file_priv->event_read_lock);
+	if (ret)
+		return ret;
+
 	for (;;) {
-		if (list_empty(&file_priv->event_list)) {
+		struct drm_pending_event *e = NULL;
+
+		spin_lock_irq(&dev->event_lock);
+		if (!list_empty(&file_priv->event_list)) {
+			e = list_first_entry(&file_priv->event_list,
+					struct drm_pending_event, link);
+			file_priv->event_space += e->event->length;
+			list_del(&e->link);
+		}
+		spin_unlock_irq(&dev->event_lock);
+
+		if (e == NULL) {
 			if (ret)
 				break;
 
@@ -527,36 +543,36 @@ ssize_t drm_read(struct file *filp, char __user *buffer,
 				break;
 			}
 
-			spin_unlock_irq(&dev->event_lock);
+			mutex_unlock(&file_priv->event_read_lock);
 			ret = wait_event_interruptible(file_priv->event_wait,
 						       !list_empty(&file_priv->event_list));
-			spin_lock_irq(&dev->event_lock);
-			if (ret < 0)
-				break;
-
-			ret = 0;
+			if (ret >= 0)
+				ret = mutex_lock_interruptible(&file_priv->event_read_lock);
+			if (ret)
+				return ret;
 		} else {
-			struct drm_pending_event *e;
-
-			e = list_first_entry(&file_priv->event_list,
-					     struct drm_pending_event, link);
-			if (e->event->length + ret > count)
+			unsigned length = e->event->length;
+
+			if (length > count - ret) {
+put_back_event:
+				spin_lock_irq(&dev->event_lock);
+				file_priv->event_space -= length;
+				list_add(&e->link, &file_priv->event_list);
+				spin_unlock_irq(&dev->event_lock);
 				break;
+			}
 
-			if (__copy_to_user_inatomic(buffer + ret,
-						    e->event, e->event->length)) {
+			if (copy_to_user(buffer + ret, e->event, length)) {
 				if (ret == 0)
 					ret = -EFAULT;
-				break;
+				goto put_back_event;
 			}
 
-			file_priv->event_space += e->event->length;
-			ret += e->event->length;
-			list_del(&e->link);
+			ret += length;
 			e->destroy(e);
 		}
 	}
-	spin_unlock_irq(&dev->event_lock);
+	mutex_unlock(&file_priv->event_read_lock);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index c7de454e8e88..2e8c77e71e1f 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -220,6 +220,9 @@ static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
 static void
 drm_gem_object_handle_unreference_unlocked(struct drm_gem_object *obj)
 {
+	struct drm_device *dev = obj->dev;
+	bool final = false;
+
 	if (WARN_ON(obj->handle_count == 0))
 		return;
 
@@ -229,14 +232,39 @@ drm_gem_object_handle_unreference_unlocked(struct drm_gem_object *obj)
 	* checked for a name
 	*/
 
-	mutex_lock(&obj->dev->object_name_lock);
+	mutex_lock(&dev->object_name_lock);
 	if (--obj->handle_count == 0) {
 		drm_gem_object_handle_free(obj);
 		drm_gem_object_exported_dma_buf_free(obj);
+		final = true;
 	}
-	mutex_unlock(&obj->dev->object_name_lock);
+	mutex_unlock(&dev->object_name_lock);
 
-	drm_gem_object_unreference_unlocked(obj);
+	if (final)
+		drm_gem_object_unreference_unlocked(obj);
+}
+
+/*
+ * Called at device or object close to release the file's
+ * handle references on objects.
+ */
+static int
+drm_gem_object_release_handle(int id, void *ptr, void *data)
+{
+	struct drm_file *file_priv = data;
+	struct drm_gem_object *obj = ptr;
+	struct drm_device *dev = obj->dev;
+
+	if (drm_core_check_feature(dev, DRIVER_PRIME))
+		drm_gem_remove_prime_handles(obj, file_priv);
+	drm_vma_node_revoke(&obj->vma_node, file_priv->filp);
+
+	if (dev->driver->gem_close_object)
+		dev->driver->gem_close_object(obj, file_priv);
+
+	drm_gem_object_handle_unreference_unlocked(obj);
+
+	return 0;
 }
 
 /**
@@ -244,8 +272,9 @@ drm_gem_object_handle_unreference_unlocked(struct drm_gem_object *obj)
  * @filp: drm file-private structure to use for the handle look up
  * @handle: userspace handle to delete
  *
- * Removes the GEM handle from the @filp lookup table and if this is the last
- * handle also cleans up linked resources like GEM names.
+ * Removes the GEM handle from the @filp lookup table which has been added with
+ * drm_gem_handle_create(). If this is the last handle also cleans up linked
+ * resources like GEM names.
  */
 int
 drm_gem_handle_delete(struct drm_file *filp, u32 handle)
@@ -276,14 +305,7 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 	idr_remove(&filp->object_idr, handle);
 	spin_unlock(&filp->table_lock);
 
-	if (drm_core_check_feature(dev, DRIVER_PRIME))
-		drm_gem_remove_prime_handles(obj, filp);
-	drm_vma_node_revoke(&obj->vma_node, filp->filp);
-
-	if (dev->driver->gem_close_object)
-		dev->driver->gem_close_object(obj, filp);
-	drm_gem_object_handle_unreference_unlocked(obj);
-
+	drm_gem_object_release_handle(handle, obj, filp);
 	return 0;
 }
 EXPORT_SYMBOL(drm_gem_handle_delete);
@@ -314,6 +336,10 @@ EXPORT_SYMBOL(drm_gem_dumb_destroy);
  * This expects the dev->object_name_lock to be held already and will drop it
  * before returning. Used to avoid races in establishing new handles when
  * importing an object from either an flink name or a dma-buf.
+ *
+ * Handles must be release again through drm_gem_handle_delete(). This is done
+ * when userspace closes @file_priv for all attached handles, or through the
+ * GEM_CLOSE ioctl for individual handles.
  */
 int
 drm_gem_handle_create_tail(struct drm_file *file_priv,
@@ -321,9 +347,12 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
 			   u32 *handlep)
 {
 	struct drm_device *dev = obj->dev;
+	u32 handle;
 	int ret;
 
 	WARN_ON(!mutex_is_locked(&dev->object_name_lock));
+	if (obj->handle_count++ == 0)
+		drm_gem_object_reference(obj);
 
 	/*
 	 * Get the user-visible handle using idr.  Preload and perform
@@ -333,32 +362,38 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
 	spin_lock(&file_priv->table_lock);
 
 	ret = idr_alloc(&file_priv->object_idr, obj, 1, 0, GFP_NOWAIT);
-	drm_gem_object_reference(obj);
-	obj->handle_count++;
+
 	spin_unlock(&file_priv->table_lock);
 	idr_preload_end();
+
 	mutex_unlock(&dev->object_name_lock);
-	if (ret < 0) {
-		drm_gem_object_handle_unreference_unlocked(obj);
-		return ret;
-	}
-	*handlep = ret;
+	if (ret < 0)
+		goto err_unref;
+
+	handle = ret;
 
 	ret = drm_vma_node_allow(&obj->vma_node, file_priv->filp);
-	if (ret) {
-		drm_gem_handle_delete(file_priv, *handlep);
-		return ret;
-	}
+	if (ret)
+		goto err_remove;
 
 	if (dev->driver->gem_open_object) {
 		ret = dev->driver->gem_open_object(obj, file_priv);
-		if (ret) {
-			drm_gem_handle_delete(file_priv, *handlep);
-			return ret;
-		}
+		if (ret)
+			goto err_revoke;
 	}
 
+	*handlep = handle;
 	return 0;
+
+err_revoke:
+	drm_vma_node_revoke(&obj->vma_node, file_priv->filp);
+err_remove:
+	spin_lock(&file_priv->table_lock);
+	idr_remove(&file_priv->object_idr, handle);
+	spin_unlock(&file_priv->table_lock);
+err_unref:
+	drm_gem_object_handle_unreference_unlocked(obj);
+	return ret;
 }
 
 /**
@@ -541,7 +576,17 @@ void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
 }
 EXPORT_SYMBOL(drm_gem_put_pages);
 
-/** Returns a reference to the object named by the handle. */
+/**
+ * drm_gem_object_lookup - look up a GEM object from it's handle
+ * @dev: DRM device
+ * @filp: DRM file private date
+ * @handle: userspace handle
+ *
+ * Returns:
+ *
+ * A reference to the object named by the handle if such exists on @filp, NULL
+ * otherwise.
+ */
 struct drm_gem_object *
 drm_gem_object_lookup(struct drm_device *dev, struct drm_file *filp,
 		      u32 handle)
@@ -615,7 +660,6 @@ drm_gem_flink_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 
 	mutex_lock(&dev->object_name_lock);
-	idr_preload(GFP_KERNEL);
 	/* prevent races with concurrent gem_close. */
 	if (obj->handle_count == 0) {
 		ret = -ENOENT;
@@ -623,7 +667,7 @@ drm_gem_flink_ioctl(struct drm_device *dev, void *data,
 	}
 
 	if (!obj->name) {
-		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_NOWAIT);
+		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
 		if (ret < 0)
 			goto err;
 
@@ -634,7 +678,6 @@ drm_gem_flink_ioctl(struct drm_device *dev, void *data,
 	ret = 0;
 
 err:
-	idr_preload_end();
 	mutex_unlock(&dev->object_name_lock);
 	drm_gem_object_unreference_unlocked(obj);
 	return ret;
@@ -699,29 +742,6 @@ drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
 	spin_lock_init(&file_private->table_lock);
 }
 
-/*
- * Called at device close to release the file's
- * handle references on objects.
- */
-static int
-drm_gem_object_release_handle(int id, void *ptr, void *data)
-{
-	struct drm_file *file_priv = data;
-	struct drm_gem_object *obj = ptr;
-	struct drm_device *dev = obj->dev;
-
-	if (drm_core_check_feature(dev, DRIVER_PRIME))
-		drm_gem_remove_prime_handles(obj, file_priv);
-	drm_vma_node_revoke(&obj->vma_node, file_priv->filp);
-
-	if (dev->driver->gem_close_object)
-		dev->driver->gem_close_object(obj, file_priv);
-
-	drm_gem_object_handle_unreference_unlocked(obj);
-
-	return 0;
-}
-
 /**
  * drm_gem_release - release file-private GEM resources
  * @dev: drm_device which is being closed by userspace
@@ -774,6 +794,13 @@ drm_gem_object_free(struct kref *kref)
 }
 EXPORT_SYMBOL(drm_gem_object_free);
 
+/**
+ * drm_gem_vm_open - vma->ops->open implementation for GEM
+ * @vma: VM area structure
+ *
+ * This function implements the #vm_operations_struct open() callback for GEM
+ * drivers. This must be used together with drm_gem_vm_close().
+ */
 void drm_gem_vm_open(struct vm_area_struct *vma)
 {
 	struct drm_gem_object *obj = vma->vm_private_data;
@@ -782,6 +809,13 @@ void drm_gem_vm_open(struct vm_area_struct *vma)
 }
 EXPORT_SYMBOL(drm_gem_vm_open);
 
+/**
+ * drm_gem_vm_close - vma->ops->close implementation for GEM
+ * @vma: VM area structure
+ *
+ * This function implements the #vm_operations_struct close() callback for GEM
+ * drivers. This must be used together with drm_gem_vm_open().
+ */
 void drm_gem_vm_close(struct vm_area_struct *vma)
 {
 	struct drm_gem_object *obj = vma->vm_private_data;
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index e109b49cd25d..e5df53b6e229 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -59,11 +59,13 @@ __drm_gem_cma_create(struct drm_device *drm, size_t size)
 	struct drm_gem_object *gem_obj;
 	int ret;
 
-	cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
-	if (!cma_obj)
+	if (drm->driver->gem_create_object)
+		gem_obj = drm->driver->gem_create_object(drm, size);
+	else
+		gem_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
+	if (!gem_obj)
 		return ERR_PTR(-ENOMEM);
-
-	gem_obj = &cma_obj->base;
+	cma_obj = container_of(gem_obj, struct drm_gem_cma_object, base);
 
 	ret = drm_gem_object_init(drm, gem_obj, size);
 	if (ret)
@@ -119,7 +121,7 @@ struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
 	return cma_obj;
 
 error:
-	drm_gem_cma_free_object(&cma_obj->base);
+	drm->driver->gem_free_object(&cma_obj->base);
 	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_create);
@@ -169,7 +171,7 @@ drm_gem_cma_create_with_handle(struct drm_file *file_priv,
 	return cma_obj;
 
 err_handle_create:
-	drm_gem_cma_free_object(gem_obj);
+	drm->driver->gem_free_object(gem_obj);
 
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c
index c3b80fd65d62..7b30b307674b 100644
--- a/drivers/gpu/drm/drm_hashtab.c
+++ b/drivers/gpu/drm/drm_hashtab.c
@@ -198,10 +198,7 @@ EXPORT_SYMBOL(drm_ht_remove_item);
 void drm_ht_remove(struct drm_open_hash *ht)
 {
 	if (ht->table) {
-		if ((PAGE_SIZE / sizeof(*ht->table)) >> ht->order)
-			kfree(ht->table);
-		else
-			vfree(ht->table);
+		kvfree(ht->table);
 		ht->table = NULL;
 	}
 }
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 607f493ae801..d12a4efa651b 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -73,6 +73,9 @@ static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
 module_param_named(timestamp_monotonic, drm_timestamp_monotonic, int, 0600);
+MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
+MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
+MODULE_PARM_DESC(timestamp_monotonic, "Use monotonic timestamps");
 
 static void store_vblank(struct drm_device *dev, unsigned int pipe,
 			 u32 vblank_count_inc,
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index 2d5ca8eec13a..6e6a9c58d404 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -365,6 +365,44 @@ int mipi_dsi_create_packet(struct mipi_dsi_packet *packet,
 }
 EXPORT_SYMBOL(mipi_dsi_create_packet);
 
+/**
+ * mipi_dsi_shutdown_peripheral() - sends a Shutdown Peripheral command
+ * @dsi: DSI peripheral device
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int mipi_dsi_shutdown_peripheral(struct mipi_dsi_device *dsi)
+{
+	struct mipi_dsi_msg msg = {
+		.channel = dsi->channel,
+		.type = MIPI_DSI_SHUTDOWN_PERIPHERAL,
+		.tx_buf = (u8 [2]) { 0, 0 },
+		.tx_len = 2,
+	};
+
+	return mipi_dsi_device_transfer(dsi, &msg);
+}
+EXPORT_SYMBOL(mipi_dsi_shutdown_peripheral);
+
+/**
+ * mipi_dsi_turn_on_peripheral() - sends a Turn On Peripheral command
+ * @dsi: DSI peripheral device
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int mipi_dsi_turn_on_peripheral(struct mipi_dsi_device *dsi)
+{
+	struct mipi_dsi_msg msg = {
+		.channel = dsi->channel,
+		.type = MIPI_DSI_TURN_ON_PERIPHERAL,
+		.tx_buf = (u8 [2]) { 0, 0 },
+		.tx_len = 2,
+	};
+
+	return mipi_dsi_device_transfer(dsi, &msg);
+}
+EXPORT_SYMBOL(mipi_dsi_turn_on_peripheral);
+
 /*
  * mipi_dsi_set_maximum_return_packet_size() - specify the maximum size of the
  *    the payload in a long packet transmitted from the peripheral back to the
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index cd74a0953f42..20775c05235a 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -553,10 +553,10 @@ EXPORT_SYMBOL(drm_gtf_mode_complex);
  * drivers/video/fbmon.c
  *
  * Standard GTF parameters:
- * M = 600
- * C = 40
- * K = 128
- * J = 20
+ *     M = 600
+ *     C = 40
+ *     K = 128
+ *     J = 20
  *
  * Returns:
  * The modeline based on the GTF algorithm stored in a drm_display_mode object.
@@ -708,7 +708,8 @@ void drm_mode_set_name(struct drm_display_mode *mode)
 }
 EXPORT_SYMBOL(drm_mode_set_name);
 
-/** drm_mode_hsync - get the hsync of a mode
+/**
+ * drm_mode_hsync - get the hsync of a mode
  * @mode: mode
  *
  * Returns:
@@ -917,13 +918,30 @@ bool drm_mode_equal(const struct drm_display_mode *mode1, const struct drm_displ
 	} else if (mode1->clock != mode2->clock)
 		return false;
 
+	return drm_mode_equal_no_clocks(mode1, mode2);
+}
+EXPORT_SYMBOL(drm_mode_equal);
+
+/**
+ * drm_mode_equal_no_clocks - test modes for equality
+ * @mode1: first mode
+ * @mode2: second mode
+ *
+ * Check to see if @mode1 and @mode2 are equivalent, but
+ * don't check the pixel clocks.
+ *
+ * Returns:
+ * True if the modes are equal, false otherwise.
+ */
+bool drm_mode_equal_no_clocks(const struct drm_display_mode *mode1, const struct drm_display_mode *mode2)
+{
 	if ((mode1->flags & DRM_MODE_FLAG_3D_MASK) !=
 	    (mode2->flags & DRM_MODE_FLAG_3D_MASK))
 		return false;
 
 	return drm_mode_equal_no_clocks_no_stereo(mode1, mode2);
 }
-EXPORT_SYMBOL(drm_mode_equal);
+EXPORT_SYMBOL(drm_mode_equal_no_clocks);
 
 /**
  * drm_mode_equal_no_clocks_no_stereo - test modes for equality
@@ -1056,7 +1074,7 @@ static const char * const drm_mode_status_names[] = {
 	MODE_STATUS(ONE_SIZE),
 	MODE_STATUS(NO_REDUCED),
 	MODE_STATUS(NO_STEREO),
-	MODE_STATUS(UNVERIFIED),
+	MODE_STATUS(STALE),
 	MODE_STATUS(BAD),
 	MODE_STATUS(ERROR),
 };
@@ -1154,7 +1172,6 @@ EXPORT_SYMBOL(drm_mode_sort);
 /**
  * drm_mode_connector_list_update - update the mode list for the connector
  * @connector: the connector to update
- * @merge_type_bits: whether to merge or overwrite type bits
  *
  * This moves the modes from the @connector probed_modes list
  * to the actual mode list. It compares the probed mode against the current
@@ -1163,33 +1180,48 @@ EXPORT_SYMBOL(drm_mode_sort);
  * This is just a helper functions doesn't validate any modes itself and also
  * doesn't prune any invalid modes. Callers need to do that themselves.
  */
-void drm_mode_connector_list_update(struct drm_connector *connector,
-				    bool merge_type_bits)
+void drm_mode_connector_list_update(struct drm_connector *connector)
 {
-	struct drm_display_mode *mode;
 	struct drm_display_mode *pmode, *pt;
-	int found_it;
 
 	WARN_ON(!mutex_is_locked(&connector->dev->mode_config.mutex));
 
-	list_for_each_entry_safe(pmode, pt, &connector->probed_modes,
-				 head) {
-		found_it = 0;
+	list_for_each_entry_safe(pmode, pt, &connector->probed_modes, head) {
+		struct drm_display_mode *mode;
+		bool found_it = false;
+
 		/* go through current modes checking for the new probed mode */
 		list_for_each_entry(mode, &connector->modes, head) {
-			if (drm_mode_equal(pmode, mode)) {
-				found_it = 1;
-				/* if equal delete the probed mode */
-				mode->status = pmode->status;
-				/* Merge type bits together */
-				if (merge_type_bits)
-					mode->type |= pmode->type;
-				else
-					mode->type = pmode->type;
-				list_del(&pmode->head);
-				drm_mode_destroy(connector->dev, pmode);
-				break;
+			if (!drm_mode_equal(pmode, mode))
+				continue;
+
+			found_it = true;
+
+			/*
+			 * If the old matching mode is stale (ie. left over
+			 * from a previous probe) just replace it outright.
+			 * Otherwise just merge the type bits between all
+			 * equal probed modes.
+			 *
+			 * If two probed modes are considered equal, pick the
+			 * actual timings from the one that's marked as
+			 * preferred (in case the match isn't 100%). If
+			 * multiple or zero preferred modes are present, favor
+			 * the mode added to the probed_modes list first.
+			 */
+			if (mode->status == MODE_STALE) {
+				drm_mode_copy(mode, pmode);
+			} else if ((mode->type & DRM_MODE_TYPE_PREFERRED) == 0 &&
+				   (pmode->type & DRM_MODE_TYPE_PREFERRED) != 0) {
+				pmode->type |= mode->type;
+				drm_mode_copy(mode, pmode);
+			} else {
+				mode->type |= pmode->type;
 			}
+
+			list_del(&pmode->head);
+			drm_mode_destroy(connector->dev, pmode);
+			break;
 		}
 
 		if (!found_it) {
@@ -1212,7 +1244,7 @@ EXPORT_SYMBOL(drm_mode_connector_list_update);
  * This uses the same parameters as the fb modedb.c, except for an extra
  * force-enable, force-enable-digital and force-disable bit at the end:
  *
- *	<xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
+ * <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
  *
  * The intermediate drm_cmdline_mode structure is required to store additional
  * options from the command line modline like the force-enable/disable flag.
@@ -1491,4 +1523,4 @@ int drm_mode_convert_umode(struct drm_display_mode *out,
 
 out:
 	return ret;
-}
\ No newline at end of file
+}
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index 6675b1428410..e3a4adf03e7b 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -40,28 +40,33 @@
  * The basic usage pattern is to:
  *
  *     drm_modeset_acquire_init(&ctx)
- *   retry:
+ *     retry:
  *     foreach (lock in random_ordered_set_of_locks) {
- *       ret = drm_modeset_lock(lock, &ctx)
- *       if (ret == -EDEADLK) {
- *          drm_modeset_backoff(&ctx);
- *          goto retry;
- *       }
+ *         ret = drm_modeset_lock(lock, &ctx)
+ *         if (ret == -EDEADLK) {
+ *             drm_modeset_backoff(&ctx);
+ *             goto retry;
+ *         }
  *     }
- *
  *     ... do stuff ...
- *
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
  */
 
 /**
  * drm_modeset_lock_all - take all modeset locks
- * @dev: drm device
+ * @dev: DRM device
  *
  * This function takes all modeset locks, suitable where a more fine-grained
- * scheme isn't (yet) implemented. Locks must be dropped with
- * drm_modeset_unlock_all.
+ * scheme isn't (yet) implemented. Locks must be dropped by calling the
+ * drm_modeset_unlock_all() function.
+ *
+ * This function is deprecated. It allocates a lock acquisition context and
+ * stores it in the DRM device's ->mode_config. This facilitate conversion of
+ * existing code because it removes the need to manually deal with the
+ * acquisition context, but it is also brittle because the context is global
+ * and care must be taken not to nest calls. New code should use the
+ * drm_modeset_lock_all_ctx() function and pass in the context explicitly.
  */
 void drm_modeset_lock_all(struct drm_device *dev)
 {
@@ -78,39 +83,43 @@ void drm_modeset_lock_all(struct drm_device *dev)
 	drm_modeset_acquire_init(ctx, 0);
 
 retry:
-	ret = drm_modeset_lock(&config->connection_mutex, ctx);
-	if (ret)
-		goto fail;
-	ret = drm_modeset_lock_all_crtcs(dev, ctx);
-	if (ret)
-		goto fail;
+	ret = drm_modeset_lock_all_ctx(dev, ctx);
+	if (ret < 0) {
+		if (ret == -EDEADLK) {
+			drm_modeset_backoff(ctx);
+			goto retry;
+		}
+
+		drm_modeset_acquire_fini(ctx);
+		kfree(ctx);
+		return;
+	}
 
 	WARN_ON(config->acquire_ctx);
 
-	/* now we hold the locks, so now that it is safe, stash the
-	 * ctx for drm_modeset_unlock_all():
+	/*
+	 * We hold the locks now, so it is safe to stash the acquisition
+	 * context for drm_modeset_unlock_all().
 	 */
 	config->acquire_ctx = ctx;
 
 	drm_warn_on_modeset_not_all_locked(dev);
-
-	return;
-
-fail:
-	if (ret == -EDEADLK) {
-		drm_modeset_backoff(ctx);
-		goto retry;
-	}
-
-	kfree(ctx);
 }
 EXPORT_SYMBOL(drm_modeset_lock_all);
 
 /**
  * drm_modeset_unlock_all - drop all modeset locks
- * @dev: device
+ * @dev: DRM device
  *
- * This function drop all modeset locks taken by drm_modeset_lock_all.
+ * This function drops all modeset locks taken by a previous call to the
+ * drm_modeset_lock_all() function.
+ *
+ * This function is deprecated. It uses the lock acquisition context stored
+ * in the DRM device's ->mode_config. This facilitates conversion of existing
+ * code because it removes the need to manually deal with the acquisition
+ * context, but it is also brittle because the context is global and care must
+ * be taken not to nest calls. New code should pass the acquisition context
+ * directly to the drm_modeset_drop_locks() function.
  */
 void drm_modeset_unlock_all(struct drm_device *dev)
 {
@@ -431,14 +440,34 @@ void drm_modeset_unlock(struct drm_modeset_lock *lock)
 }
 EXPORT_SYMBOL(drm_modeset_unlock);
 
-/* In some legacy codepaths it's convenient to just grab all the crtc and plane
- * related locks. */
-int drm_modeset_lock_all_crtcs(struct drm_device *dev,
-		struct drm_modeset_acquire_ctx *ctx)
+/**
+ * drm_modeset_lock_all_ctx - take all modeset locks
+ * @dev: DRM device
+ * @ctx: lock acquisition context
+ *
+ * This function takes all modeset locks, suitable where a more fine-grained
+ * scheme isn't (yet) implemented.
+ *
+ * Unlike drm_modeset_lock_all(), it doesn't take the dev->mode_config.mutex
+ * since that lock isn't required for modeset state changes. Callers which
+ * need to grab that lock too need to do so outside of the acquire context
+ * @ctx.
+ *
+ * Locks acquired with this function should be released by calling the
+ * drm_modeset_drop_locks() function on @ctx.
+ *
+ * Returns: 0 on success or a negative error-code on failure.
+ */
+int drm_modeset_lock_all_ctx(struct drm_device *dev,
+			     struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_crtc *crtc;
 	struct drm_plane *plane;
-	int ret = 0;
+	int ret;
+
+	ret = drm_modeset_lock(&dev->mode_config.connection_mutex, ctx);
+	if (ret)
+		return ret;
 
 	drm_for_each_crtc(crtc, dev) {
 		ret = drm_modeset_lock(&crtc->mutex, ctx);
@@ -454,4 +483,4 @@ int drm_modeset_lock_all_crtcs(struct drm_device *dev,
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_modeset_lock_all_crtcs);
+EXPORT_SYMBOL(drm_modeset_lock_all_ctx);
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index fcd2a86acd2c..a1fff1179a97 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -410,6 +410,26 @@ int drm_pcie_get_speed_cap_mask(struct drm_device *dev, u32 *mask)
 }
 EXPORT_SYMBOL(drm_pcie_get_speed_cap_mask);
 
+int drm_pcie_get_max_link_width(struct drm_device *dev, u32 *mlw)
+{
+	struct pci_dev *root;
+	u32 lnkcap;
+
+	*mlw = 0;
+	if (!dev->pdev)
+		return -EINVAL;
+
+	root = dev->pdev->bus->self;
+
+	pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
+
+	*mlw = (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4;
+
+	DRM_INFO("probing mlw for device %x:%x = %x\n", root->vendor, root->device, lnkcap);
+	return 0;
+}
+EXPORT_SYMBOL(drm_pcie_get_max_link_width);
+
 #else
 
 int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index d384ebcf0aaf..369d2898ff9e 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -57,6 +57,10 @@
  * by the atomic helpers.
  *
  * Again drivers are strongly urged to switch to the new interfaces.
+ *
+ * The plane helpers share the function table structures with other helpers,
+ * specifically also the atomic helpers. See struct &drm_plane_helper_funcs for
+ * the details.
  */
 
 /*
@@ -164,6 +168,8 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 	vscale = drm_rect_calc_vscale(src, dest, min_scale, max_scale);
 	if (hscale < 0 || vscale < 0) {
 		DRM_DEBUG_KMS("Invalid scaling of plane\n");
+		drm_rect_debug_print("src: ", src, true);
+		drm_rect_debug_print("dst: ", dest, false);
 		return -ERANGE;
 	}
 
@@ -180,6 +186,8 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 
 	if (!can_position && !drm_rect_equals(dest, clip)) {
 		DRM_DEBUG_KMS("Plane must cover entire CRTC\n");
+		drm_rect_debug_print("dst: ", dest, false);
+		drm_rect_debug_print("clip: ", clip, false);
 		return -EINVAL;
 	}
 
@@ -367,7 +375,7 @@ static struct drm_plane *create_primary_plane(struct drm_device *dev)
 				       &drm_primary_helper_funcs,
 				       safe_modeset_formats,
 				       ARRAY_SIZE(safe_modeset_formats),
-				       DRM_PLANE_TYPE_PRIMARY);
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
 		primary = NULL;
@@ -394,7 +402,8 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 	struct drm_plane *primary;
 
 	primary = create_primary_plane(dev);
-	return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs);
+	return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs,
+					 NULL);
 }
 EXPORT_SYMBOL(drm_crtc_init);
 
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 9f935f55d74c..27aa7183b20b 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -313,19 +313,15 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
  *
  * Export callbacks:
  *
- *  - @gem_prime_pin (optional): prepare a GEM object for exporting
- *
- *  - @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
- *
- *  - @gem_prime_vmap: vmap a buffer exported by your driver
- *
- *  - @gem_prime_vunmap: vunmap a buffer exported by your driver
- *
- *  - @gem_prime_mmap (optional): mmap a buffer exported by your driver
+ *  * @gem_prime_pin (optional): prepare a GEM object for exporting
+ *  * @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
+ *  * @gem_prime_vmap: vmap a buffer exported by your driver
+ *  * @gem_prime_vunmap: vunmap a buffer exported by your driver
+ *  * @gem_prime_mmap (optional): mmap a buffer exported by your driver
  *
  * Import callback:
  *
- *  - @gem_prime_import_sg_table (import): produce a GEM object from another
+ *  * @gem_prime_import_sg_table (import): produce a GEM object from another
  *    driver's scatter/gather table
  */
 
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index f8b5fcfa91a2..e714b5a7955f 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -53,6 +53,9 @@
  * This helper library can be used independently of the modeset helper library.
  * Drivers can also overwrite different parts e.g. use their own hotplug
  * handling code to avoid probing unrelated outputs.
+ *
+ * The probe helpers share the function table structures with other display
+ * helper libraries. See struct &drm_connector_helper_funcs for the details.
  */
 
 static bool drm_kms_helper_poll = true;
@@ -126,9 +129,64 @@ void drm_kms_helper_poll_enable_locked(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_enable_locked);
 
-
-static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connector *connector,
-							      uint32_t maxX, uint32_t maxY, bool merge_type_bits)
+/**
+ * drm_helper_probe_single_connector_modes - get complete set of display modes
+ * @connector: connector to probe
+ * @maxX: max width for modes
+ * @maxY: max height for modes
+ *
+ * Based on the helper callbacks implemented by @connector in struct
+ * &drm_connector_helper_funcs try to detect all valid modes.  Modes will first
+ * be added to the connector's probed_modes list, then culled (based on validity
+ * and the @maxX, @maxY parameters) and put into the normal modes list.
+ *
+ * Intended to be used as a generic implementation of the ->fill_modes()
+ * @connector vfunc for drivers that use the CRTC helpers for output mode
+ * filtering and detection.
+ *
+ * The basic procedure is as follows
+ *
+ * 1. All modes currently on the connector's modes list are marked as stale
+ *
+ * 2. New modes are added to the connector's probed_modes list with
+ *    drm_mode_probed_add(). New modes start their life with status as OK.
+ *    Modes are added from a single source using the following priority order.
+ *
+ *    - debugfs 'override_edid' (used for testing only)
+ *    - firmware EDID (drm_load_edid_firmware())
+ *    - connector helper ->get_modes() vfunc
+ *    - if the connector status is connector_status_connected, standard
+ *      VESA DMT modes up to 1024x768 are automatically added
+ *      (drm_add_modes_noedid())
+ *
+ *    Finally modes specified via the kernel command line (video=...) are
+ *    added in addition to what the earlier probes produced
+ *    (drm_helper_probe_add_cmdline_mode()). These modes are generated
+ *    using the VESA GTF/CVT formulas.
+ *
+ * 3. Modes are moved from the probed_modes list to the modes list. Potential
+ *    duplicates are merged together (see drm_mode_connector_list_update()).
+ *    After this step the probed_modes list will be empty again.
+ *
+ * 4. Any non-stale mode on the modes list then undergoes validation
+ *
+ *    - drm_mode_validate_basic() performs basic sanity checks
+ *    - drm_mode_validate_size() filters out modes larger than @maxX and @maxY
+ *      (if specified)
+ *    - drm_mode_validate_flag() checks the modes againt basic connector
+ *      capabilites (interlace_allowed,doublescan_allowed,stereo_allowed)
+ *    - the optional connector ->mode_valid() helper can perform driver and/or
+ *      hardware specific checks
+ *
+ * 5. Any mode whose status is not OK is pruned from the connector's modes list,
+ *    accompanied by a debug message indicating the reason for the mode's
+ *    rejection (see drm_mode_prune_invalid()).
+ *
+ * Returns:
+ * The number of modes found on @connector.
+ */
+int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
+					    uint32_t maxX, uint32_t maxY)
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *mode;
@@ -143,9 +201,11 @@ static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connect
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id,
 			connector->name);
-	/* set all modes to the unverified state */
+	/* set all old modes to the stale state */
 	list_for_each_entry(mode, &connector->modes, head)
-		mode->status = MODE_UNVERIFIED;
+		mode->status = MODE_STALE;
+
+	old_status = connector->status;
 
 	if (connector->force) {
 		if (connector->force == DRM_FORCE_ON ||
@@ -156,33 +216,32 @@ static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connect
 		if (connector->funcs->force)
 			connector->funcs->force(connector);
 	} else {
-		old_status = connector->status;
-
 		connector->status = connector->funcs->detect(connector, true);
+	}
+
+	/*
+	 * Normally either the driver's hpd code or the poll loop should
+	 * pick up any changes and fire the hotplug event. But if
+	 * userspace sneaks in a probe, we might miss a change. Hence
+	 * check here, and if anything changed start the hotplug code.
+	 */
+	if (old_status != connector->status) {
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
+			      connector->base.id,
+			      connector->name,
+			      drm_get_connector_status_name(old_status),
+			      drm_get_connector_status_name(connector->status));
 
 		/*
-		 * Normally either the driver's hpd code or the poll loop should
-		 * pick up any changes and fire the hotplug event. But if
-		 * userspace sneaks in a probe, we might miss a change. Hence
-		 * check here, and if anything changed start the hotplug code.
+		 * The hotplug event code might call into the fb
+		 * helpers, and so expects that we do not hold any
+		 * locks. Fire up the poll struct instead, it will
+		 * disable itself again.
 		 */
-		if (old_status != connector->status) {
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
-				      connector->base.id,
-				      connector->name,
-				      old_status, connector->status);
-
-			/*
-			 * The hotplug event code might call into the fb
-			 * helpers, and so expects that we do not hold any
-			 * locks. Fire up the poll struct instead, it will
-			 * disable itself again.
-			 */
-			dev->mode_config.delayed_event = true;
-			if (dev->mode_config.poll_enabled)
-				schedule_delayed_work(&dev->mode_config.output_poll_work,
-						      0);
-		}
+		dev->mode_config.delayed_event = true;
+		if (dev->mode_config.poll_enabled)
+			schedule_delayed_work(&dev->mode_config.output_poll_work,
+					      0);
 	}
 
 	/* Re-enable polling in case the global poll config changed. */
@@ -199,17 +258,16 @@ static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connect
 		goto prune;
 	}
 
+	if (connector->override_edid) {
+		struct edid *edid = (struct edid *) connector->edid_blob_ptr->data;
+
+		count = drm_add_edid_modes(connector, edid);
+		drm_edid_to_eld(connector, edid);
+	} else {
 #ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE
-	count = drm_load_edid_firmware(connector);
-	if (count == 0)
+		count = drm_load_edid_firmware(connector);
+		if (count == 0)
 #endif
-	{
-		if (connector->override_edid) {
-			struct edid *edid = (struct edid *) connector->edid_blob_ptr->data;
-
-			count = drm_add_edid_modes(connector, edid);
-			drm_edid_to_eld(connector, edid);
-		} else
 			count = (*connector_funcs->get_modes)(connector);
 	}
 
@@ -219,7 +277,7 @@ static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connect
 	if (count == 0)
 		goto prune;
 
-	drm_mode_connector_list_update(connector, merge_type_bits);
+	drm_mode_connector_list_update(connector);
 
 	if (connector->interlace_allowed)
 		mode_flags |= DRM_MODE_FLAG_INTERLACE;
@@ -263,49 +321,9 @@ prune:
 
 	return count;
 }
-
-/**
- * drm_helper_probe_single_connector_modes - get complete set of display modes
- * @connector: connector to probe
- * @maxX: max width for modes
- * @maxY: max height for modes
- *
- * Based on the helper callbacks implemented by @connector try to detect all
- * valid modes.  Modes will first be added to the connector's probed_modes list,
- * then culled (based on validity and the @maxX, @maxY parameters) and put into
- * the normal modes list.
- *
- * Intended to be use as a generic implementation of the ->fill_modes()
- * @connector vfunc for drivers that use the crtc helpers for output mode
- * filtering and detection.
- *
- * Returns:
- * The number of modes found on @connector.
- */
-int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
-					    uint32_t maxX, uint32_t maxY)
-{
-	return drm_helper_probe_single_connector_modes_merge_bits(connector, maxX, maxY, true);
-}
 EXPORT_SYMBOL(drm_helper_probe_single_connector_modes);
 
 /**
- * drm_helper_probe_single_connector_modes_nomerge - get complete set of display modes
- * @connector: connector to probe
- * @maxX: max width for modes
- * @maxY: max height for modes
- *
- * This operates like drm_hehlper_probe_single_connector_modes except it
- * replaces the mode bits instead of merging them for preferred modes.
- */
-int drm_helper_probe_single_connector_modes_nomerge(struct drm_connector *connector,
-					    uint32_t maxX, uint32_t maxY)
-{
-	return drm_helper_probe_single_connector_modes_merge_bits(connector, maxX, maxY, false);
-}
-EXPORT_SYMBOL(drm_helper_probe_single_connector_modes_nomerge);
-
-/**
  * drm_kms_helper_hotplug_event - fire off KMS hotplug events
  * @dev: drm_device whose connector state changed
  *
diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
index 531ac4cc9756..a8e2c8603945 100644
--- a/drivers/gpu/drm/drm_rect.c
+++ b/drivers/gpu/drm/drm_rect.c
@@ -275,22 +275,23 @@ EXPORT_SYMBOL(drm_rect_calc_vscale_relaxed);
 
 /**
  * drm_rect_debug_print - print the rectangle information
+ * @prefix: prefix string
  * @r: rectangle to print
  * @fixed_point: rectangle is in 16.16 fixed point format
  */
-void drm_rect_debug_print(const struct drm_rect *r, bool fixed_point)
+void drm_rect_debug_print(const char *prefix, const struct drm_rect *r, bool fixed_point)
 {
 	int w = drm_rect_width(r);
 	int h = drm_rect_height(r);
 
 	if (fixed_point)
-		DRM_DEBUG_KMS("%d.%06ux%d.%06u%+d.%06u%+d.%06u\n",
+		DRM_DEBUG_KMS("%s%d.%06ux%d.%06u%+d.%06u%+d.%06u\n", prefix,
 			      w >> 16, ((w & 0xffff) * 15625) >> 10,
 			      h >> 16, ((h & 0xffff) * 15625) >> 10,
 			      r->x1 >> 16, ((r->x1 & 0xffff) * 15625) >> 10,
 			      r->y1 >> 16, ((r->y1 & 0xffff) * 15625) >> 10);
 	else
-		DRM_DEBUG_KMS("%dx%d%+d%+d\n", w, h, r->x1, r->y1);
+		DRM_DEBUG_KMS("%s%dx%d%+d%+d\n", prefix, w, h, r->x1, r->y1);
 }
 EXPORT_SYMBOL(drm_rect_debug_print);
 
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 615b7e667320..d503f8e8c2d1 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -167,47 +167,35 @@ static ssize_t status_store(struct device *device,
 {
 	struct drm_connector *connector = to_drm_connector(device);
 	struct drm_device *dev = connector->dev;
-	enum drm_connector_status old_status;
+	enum drm_connector_force old_force;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->mode_config.mutex);
 	if (ret)
 		return ret;
 
-	old_status = connector->status;
+	old_force = connector->force;
 
-	if (sysfs_streq(buf, "detect")) {
+	if (sysfs_streq(buf, "detect"))
 		connector->force = 0;
-		connector->status = connector->funcs->detect(connector, true);
-	} else if (sysfs_streq(buf, "on")) {
+	else if (sysfs_streq(buf, "on"))
 		connector->force = DRM_FORCE_ON;
-	} else if (sysfs_streq(buf, "on-digital")) {
+	else if (sysfs_streq(buf, "on-digital"))
 		connector->force = DRM_FORCE_ON_DIGITAL;
-	} else if (sysfs_streq(buf, "off")) {
+	else if (sysfs_streq(buf, "off"))
 		connector->force = DRM_FORCE_OFF;
-	} else
+	else
 		ret = -EINVAL;
 
-	if (ret == 0 && connector->force) {
-		if (connector->force == DRM_FORCE_ON ||
-		    connector->force == DRM_FORCE_ON_DIGITAL)
-			connector->status = connector_status_connected;
-		else
-			connector->status = connector_status_disconnected;
-		if (connector->funcs->force)
-			connector->funcs->force(connector);
-	}
-
-	if (old_status != connector->status) {
-		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
+	if (old_force != connector->force || !connector->force) {
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force updated from %d to %d or reprobing\n",
 			      connector->base.id,
 			      connector->name,
-			      old_status, connector->status);
+			      old_force, connector->force);
 
-		dev->mode_config.delayed_event = true;
-		if (dev->mode_config.poll_enabled)
-			schedule_delayed_work(&dev->mode_config.output_poll_work,
-					      0);
+		connector->funcs->fill_modes(connector,
+					     dev->mode_config.max_width,
+					     dev->mode_config.max_height);
 	}
 
 	mutex_unlock(&dev->mode_config.mutex);
@@ -252,27 +240,33 @@ static ssize_t edid_show(struct file *filp, struct kobject *kobj,
 			 struct bin_attribute *attr, char *buf, loff_t off,
 			 size_t count)
 {
-	struct device *connector_dev = container_of(kobj, struct device, kobj);
+	struct device *connector_dev = kobj_to_dev(kobj);
 	struct drm_connector *connector = to_drm_connector(connector_dev);
 	unsigned char *edid;
 	size_t size;
+	ssize_t ret = 0;
 
+	mutex_lock(&connector->dev->mode_config.mutex);
 	if (!connector->edid_blob_ptr)
-		return 0;
+		goto unlock;
 
 	edid = connector->edid_blob_ptr->data;
 	size = connector->edid_blob_ptr->length;
 	if (!edid)
-		return 0;
+		goto unlock;
 
 	if (off >= size)
-		return 0;
+		goto unlock;
 
 	if (off + count > size)
 		count = size - off;
 	memcpy(buf, edid + off, count);
 
-	return count;
+	ret = count;
+unlock:
+	mutex_unlock(&connector->dev->mode_config.mutex);
+
+	return ret;
 }
 
 static ssize_t modes_show(struct device *device,
@@ -283,10 +277,12 @@ static ssize_t modes_show(struct device *device,
 	struct drm_display_mode *mode;
 	int written = 0;
 
+	mutex_lock(&connector->dev->mode_config.mutex);
 	list_for_each_entry(mode, &connector->modes, head) {
 		written += snprintf(buf + written, PAGE_SIZE - written, "%s\n",
 				    mode->name);
 	}
+	mutex_unlock(&connector->dev->mode_config.mutex);
 
 	return written;
 }
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
new file mode 100644
index 000000000000..2cde7a5442fb
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/Kconfig
@@ -0,0 +1,20 @@
+
+config DRM_ETNAVIV
+	tristate "ETNAVIV (DRM support for Vivante GPU IP cores)"
+	depends on DRM
+	depends on ARCH_MXC || ARCH_DOVE
+	select SHMEM
+	select TMPFS
+	select IOMMU_API
+	select IOMMU_SUPPORT
+	select WANT_DEV_COREDUMP
+	help
+	  DRM driver for Vivante GPUs.
+
+config DRM_ETNAVIV_REGISTER_LOGGING
+	bool "enable ETNAVIV register logging"
+	depends on DRM_ETNAVIV
+	help
+	  Compile in support for logging register reads/writes in a format
+	  that can be parsed by envytools demsm tool.  If enabled, register
+	  logging can be switched on via etnaviv.reglog=y module param.
diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile
new file mode 100644
index 000000000000..1086e9876f91
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/Makefile
@@ -0,0 +1,14 @@
+etnaviv-y := \
+	etnaviv_buffer.o \
+	etnaviv_cmd_parser.o \
+	etnaviv_drv.o \
+	etnaviv_dump.o \
+	etnaviv_gem_prime.o \
+	etnaviv_gem_submit.o \
+	etnaviv_gem.o \
+	etnaviv_gpu.o \
+	etnaviv_iommu_v2.o \
+	etnaviv_iommu.o \
+	etnaviv_mmu.o
+
+obj-$(CONFIG_DRM_ETNAVIV)	+= etnaviv.o
diff --git a/drivers/gpu/drm/etnaviv/cmdstream.xml.h b/drivers/gpu/drm/etnaviv/cmdstream.xml.h
new file mode 100644
index 000000000000..8c44ba9a694e
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/cmdstream.xml.h
@@ -0,0 +1,218 @@
+#ifndef CMDSTREAM_XML
+#define CMDSTREAM_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- cmdstream.xml (  12589 bytes, from 2014-02-17 14:57:56)
+- common.xml    (  18437 bytes, from 2015-03-25 11:27:41)
+
+Copyright (C) 2014
+*/
+
+
+#define FE_OPCODE_LOAD_STATE					0x00000001
+#define FE_OPCODE_END						0x00000002
+#define FE_OPCODE_NOP						0x00000003
+#define FE_OPCODE_DRAW_2D					0x00000004
+#define FE_OPCODE_DRAW_PRIMITIVES				0x00000005
+#define FE_OPCODE_DRAW_INDEXED_PRIMITIVES			0x00000006
+#define FE_OPCODE_WAIT						0x00000007
+#define FE_OPCODE_LINK						0x00000008
+#define FE_OPCODE_STALL						0x00000009
+#define FE_OPCODE_CALL						0x0000000a
+#define FE_OPCODE_RETURN					0x0000000b
+#define FE_OPCODE_CHIP_SELECT					0x0000000d
+#define PRIMITIVE_TYPE_POINTS					0x00000001
+#define PRIMITIVE_TYPE_LINES					0x00000002
+#define PRIMITIVE_TYPE_LINE_STRIP				0x00000003
+#define PRIMITIVE_TYPE_TRIANGLES				0x00000004
+#define PRIMITIVE_TYPE_TRIANGLE_STRIP				0x00000005
+#define PRIMITIVE_TYPE_TRIANGLE_FAN				0x00000006
+#define PRIMITIVE_TYPE_LINE_LOOP				0x00000007
+#define PRIMITIVE_TYPE_QUADS					0x00000008
+#define VIV_FE_LOAD_STATE					0x00000000
+
+#define VIV_FE_LOAD_STATE_HEADER				0x00000000
+#define VIV_FE_LOAD_STATE_HEADER_OP__MASK			0xf8000000
+#define VIV_FE_LOAD_STATE_HEADER_OP__SHIFT			27
+#define VIV_FE_LOAD_STATE_HEADER_OP_LOAD_STATE			0x08000000
+#define VIV_FE_LOAD_STATE_HEADER_FIXP				0x04000000
+#define VIV_FE_LOAD_STATE_HEADER_COUNT__MASK			0x03ff0000
+#define VIV_FE_LOAD_STATE_HEADER_COUNT__SHIFT			16
+#define VIV_FE_LOAD_STATE_HEADER_COUNT(x)			(((x) << VIV_FE_LOAD_STATE_HEADER_COUNT__SHIFT) & VIV_FE_LOAD_STATE_HEADER_COUNT__MASK)
+#define VIV_FE_LOAD_STATE_HEADER_OFFSET__MASK			0x0000ffff
+#define VIV_FE_LOAD_STATE_HEADER_OFFSET__SHIFT			0
+#define VIV_FE_LOAD_STATE_HEADER_OFFSET(x)			(((x) << VIV_FE_LOAD_STATE_HEADER_OFFSET__SHIFT) & VIV_FE_LOAD_STATE_HEADER_OFFSET__MASK)
+#define VIV_FE_LOAD_STATE_HEADER_OFFSET__SHR			2
+
+#define VIV_FE_END						0x00000000
+
+#define VIV_FE_END_HEADER					0x00000000
+#define VIV_FE_END_HEADER_EVENT_ID__MASK			0x0000001f
+#define VIV_FE_END_HEADER_EVENT_ID__SHIFT			0
+#define VIV_FE_END_HEADER_EVENT_ID(x)				(((x) << VIV_FE_END_HEADER_EVENT_ID__SHIFT) & VIV_FE_END_HEADER_EVENT_ID__MASK)
+#define VIV_FE_END_HEADER_EVENT_ENABLE				0x00000100
+#define VIV_FE_END_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_END_HEADER_OP__SHIFT				27
+#define VIV_FE_END_HEADER_OP_END				0x10000000
+
+#define VIV_FE_NOP						0x00000000
+
+#define VIV_FE_NOP_HEADER					0x00000000
+#define VIV_FE_NOP_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_NOP_HEADER_OP__SHIFT				27
+#define VIV_FE_NOP_HEADER_OP_NOP				0x18000000
+
+#define VIV_FE_DRAW_2D						0x00000000
+
+#define VIV_FE_DRAW_2D_HEADER					0x00000000
+#define VIV_FE_DRAW_2D_HEADER_COUNT__MASK			0x0000ff00
+#define VIV_FE_DRAW_2D_HEADER_COUNT__SHIFT			8
+#define VIV_FE_DRAW_2D_HEADER_COUNT(x)				(((x) << VIV_FE_DRAW_2D_HEADER_COUNT__SHIFT) & VIV_FE_DRAW_2D_HEADER_COUNT__MASK)
+#define VIV_FE_DRAW_2D_HEADER_DATA_COUNT__MASK			0x07ff0000
+#define VIV_FE_DRAW_2D_HEADER_DATA_COUNT__SHIFT			16
+#define VIV_FE_DRAW_2D_HEADER_DATA_COUNT(x)			(((x) << VIV_FE_DRAW_2D_HEADER_DATA_COUNT__SHIFT) & VIV_FE_DRAW_2D_HEADER_DATA_COUNT__MASK)
+#define VIV_FE_DRAW_2D_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_DRAW_2D_HEADER_OP__SHIFT				27
+#define VIV_FE_DRAW_2D_HEADER_OP_DRAW_2D			0x20000000
+
+#define VIV_FE_DRAW_2D_TOP_LEFT					0x00000008
+#define VIV_FE_DRAW_2D_TOP_LEFT_X__MASK				0x0000ffff
+#define VIV_FE_DRAW_2D_TOP_LEFT_X__SHIFT			0
+#define VIV_FE_DRAW_2D_TOP_LEFT_X(x)				(((x) << VIV_FE_DRAW_2D_TOP_LEFT_X__SHIFT) & VIV_FE_DRAW_2D_TOP_LEFT_X__MASK)
+#define VIV_FE_DRAW_2D_TOP_LEFT_Y__MASK				0xffff0000
+#define VIV_FE_DRAW_2D_TOP_LEFT_Y__SHIFT			16
+#define VIV_FE_DRAW_2D_TOP_LEFT_Y(x)				(((x) << VIV_FE_DRAW_2D_TOP_LEFT_Y__SHIFT) & VIV_FE_DRAW_2D_TOP_LEFT_Y__MASK)
+
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT				0x0000000c
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_X__MASK			0x0000ffff
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_X__SHIFT			0
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_X(x)			(((x) << VIV_FE_DRAW_2D_BOTTOM_RIGHT_X__SHIFT) & VIV_FE_DRAW_2D_BOTTOM_RIGHT_X__MASK)
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_Y__MASK			0xffff0000
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_Y__SHIFT			16
+#define VIV_FE_DRAW_2D_BOTTOM_RIGHT_Y(x)			(((x) << VIV_FE_DRAW_2D_BOTTOM_RIGHT_Y__SHIFT) & VIV_FE_DRAW_2D_BOTTOM_RIGHT_Y__MASK)
+
+#define VIV_FE_DRAW_PRIMITIVES					0x00000000
+
+#define VIV_FE_DRAW_PRIMITIVES_HEADER				0x00000000
+#define VIV_FE_DRAW_PRIMITIVES_HEADER_OP__MASK			0xf8000000
+#define VIV_FE_DRAW_PRIMITIVES_HEADER_OP__SHIFT			27
+#define VIV_FE_DRAW_PRIMITIVES_HEADER_OP_DRAW_PRIMITIVES	0x28000000
+
+#define VIV_FE_DRAW_PRIMITIVES_COMMAND				0x00000004
+#define VIV_FE_DRAW_PRIMITIVES_COMMAND_TYPE__MASK		0x000000ff
+#define VIV_FE_DRAW_PRIMITIVES_COMMAND_TYPE__SHIFT		0
+#define VIV_FE_DRAW_PRIMITIVES_COMMAND_TYPE(x)			(((x) << VIV_FE_DRAW_PRIMITIVES_COMMAND_TYPE__SHIFT) & VIV_FE_DRAW_PRIMITIVES_COMMAND_TYPE__MASK)
+
+#define VIV_FE_DRAW_PRIMITIVES_START				0x00000008
+
+#define VIV_FE_DRAW_PRIMITIVES_COUNT				0x0000000c
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES				0x00000000
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_HEADER			0x00000000
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_HEADER_OP__MASK		0xf8000000
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_HEADER_OP__SHIFT		27
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_HEADER_OP_DRAW_INDEXED_PRIMITIVES	0x30000000
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND			0x00000004
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND_TYPE__MASK	0x000000ff
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND_TYPE__SHIFT	0
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND_TYPE(x)		(((x) << VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND_TYPE__SHIFT) & VIV_FE_DRAW_INDEXED_PRIMITIVES_COMMAND_TYPE__MASK)
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_START			0x00000008
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_COUNT			0x0000000c
+
+#define VIV_FE_DRAW_INDEXED_PRIMITIVES_OFFSET			0x00000010
+
+#define VIV_FE_WAIT						0x00000000
+
+#define VIV_FE_WAIT_HEADER					0x00000000
+#define VIV_FE_WAIT_HEADER_DELAY__MASK				0x0000ffff
+#define VIV_FE_WAIT_HEADER_DELAY__SHIFT				0
+#define VIV_FE_WAIT_HEADER_DELAY(x)				(((x) << VIV_FE_WAIT_HEADER_DELAY__SHIFT) & VIV_FE_WAIT_HEADER_DELAY__MASK)
+#define VIV_FE_WAIT_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_WAIT_HEADER_OP__SHIFT				27
+#define VIV_FE_WAIT_HEADER_OP_WAIT				0x38000000
+
+#define VIV_FE_LINK						0x00000000
+
+#define VIV_FE_LINK_HEADER					0x00000000
+#define VIV_FE_LINK_HEADER_PREFETCH__MASK			0x0000ffff
+#define VIV_FE_LINK_HEADER_PREFETCH__SHIFT			0
+#define VIV_FE_LINK_HEADER_PREFETCH(x)				(((x) << VIV_FE_LINK_HEADER_PREFETCH__SHIFT) & VIV_FE_LINK_HEADER_PREFETCH__MASK)
+#define VIV_FE_LINK_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_LINK_HEADER_OP__SHIFT				27
+#define VIV_FE_LINK_HEADER_OP_LINK				0x40000000
+
+#define VIV_FE_LINK_ADDRESS					0x00000004
+
+#define VIV_FE_STALL						0x00000000
+
+#define VIV_FE_STALL_HEADER					0x00000000
+#define VIV_FE_STALL_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_STALL_HEADER_OP__SHIFT				27
+#define VIV_FE_STALL_HEADER_OP_STALL				0x48000000
+
+#define VIV_FE_STALL_TOKEN					0x00000004
+#define VIV_FE_STALL_TOKEN_FROM__MASK				0x0000001f
+#define VIV_FE_STALL_TOKEN_FROM__SHIFT				0
+#define VIV_FE_STALL_TOKEN_FROM(x)				(((x) << VIV_FE_STALL_TOKEN_FROM__SHIFT) & VIV_FE_STALL_TOKEN_FROM__MASK)
+#define VIV_FE_STALL_TOKEN_TO__MASK				0x00001f00
+#define VIV_FE_STALL_TOKEN_TO__SHIFT				8
+#define VIV_FE_STALL_TOKEN_TO(x)				(((x) << VIV_FE_STALL_TOKEN_TO__SHIFT) & VIV_FE_STALL_TOKEN_TO__MASK)
+
+#define VIV_FE_CALL						0x00000000
+
+#define VIV_FE_CALL_HEADER					0x00000000
+#define VIV_FE_CALL_HEADER_PREFETCH__MASK			0x0000ffff
+#define VIV_FE_CALL_HEADER_PREFETCH__SHIFT			0
+#define VIV_FE_CALL_HEADER_PREFETCH(x)				(((x) << VIV_FE_CALL_HEADER_PREFETCH__SHIFT) & VIV_FE_CALL_HEADER_PREFETCH__MASK)
+#define VIV_FE_CALL_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_CALL_HEADER_OP__SHIFT				27
+#define VIV_FE_CALL_HEADER_OP_CALL				0x50000000
+
+#define VIV_FE_CALL_ADDRESS					0x00000004
+
+#define VIV_FE_CALL_RETURN_PREFETCH				0x00000008
+
+#define VIV_FE_CALL_RETURN_ADDRESS				0x0000000c
+
+#define VIV_FE_RETURN						0x00000000
+
+#define VIV_FE_RETURN_HEADER					0x00000000
+#define VIV_FE_RETURN_HEADER_OP__MASK				0xf8000000
+#define VIV_FE_RETURN_HEADER_OP__SHIFT				27
+#define VIV_FE_RETURN_HEADER_OP_RETURN				0x58000000
+
+#define VIV_FE_CHIP_SELECT					0x00000000
+
+#define VIV_FE_CHIP_SELECT_HEADER				0x00000000
+#define VIV_FE_CHIP_SELECT_HEADER_OP__MASK			0xf8000000
+#define VIV_FE_CHIP_SELECT_HEADER_OP__SHIFT			27
+#define VIV_FE_CHIP_SELECT_HEADER_OP_CHIP_SELECT		0x68000000
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP15			0x00008000
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP14			0x00004000
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP13			0x00002000
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP12			0x00001000
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP11			0x00000800
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP10			0x00000400
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP9			0x00000200
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP8			0x00000100
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP7			0x00000080
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP6			0x00000040
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP5			0x00000020
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP4			0x00000010
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP3			0x00000008
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP2			0x00000004
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP1			0x00000002
+#define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP0			0x00000001
+
+
+#endif /* CMDSTREAM_XML */
diff --git a/drivers/gpu/drm/etnaviv/common.xml.h b/drivers/gpu/drm/etnaviv/common.xml.h
new file mode 100644
index 000000000000..e881482b5971
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/common.xml.h
@@ -0,0 +1,292 @@
+#ifndef COMMON_XML
+#define COMMON_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- state_hi.xml (  24309 bytes, from 2015-12-12 09:02:53)
+- common.xml   (  18379 bytes, from 2015-12-12 09:02:53)
+
+Copyright (C) 2015
+*/
+
+
+#define PIPE_ID_PIPE_3D						0x00000000
+#define PIPE_ID_PIPE_2D						0x00000001
+#define SYNC_RECIPIENT_FE					0x00000001
+#define SYNC_RECIPIENT_RA					0x00000005
+#define SYNC_RECIPIENT_PE					0x00000007
+#define SYNC_RECIPIENT_DE					0x0000000b
+#define SYNC_RECIPIENT_VG					0x0000000f
+#define SYNC_RECIPIENT_TESSELATOR				0x00000010
+#define SYNC_RECIPIENT_VG2					0x00000011
+#define SYNC_RECIPIENT_TESSELATOR2				0x00000012
+#define SYNC_RECIPIENT_VG3					0x00000013
+#define SYNC_RECIPIENT_TESSELATOR3				0x00000014
+#define ENDIAN_MODE_NO_SWAP					0x00000000
+#define ENDIAN_MODE_SWAP_16					0x00000001
+#define ENDIAN_MODE_SWAP_32					0x00000002
+#define chipModel_GC200						0x00000200
+#define chipModel_GC300						0x00000300
+#define chipModel_GC320						0x00000320
+#define chipModel_GC328						0x00000328
+#define chipModel_GC350						0x00000350
+#define chipModel_GC355						0x00000355
+#define chipModel_GC400						0x00000400
+#define chipModel_GC410						0x00000410
+#define chipModel_GC420						0x00000420
+#define chipModel_GC428						0x00000428
+#define chipModel_GC450						0x00000450
+#define chipModel_GC500						0x00000500
+#define chipModel_GC520						0x00000520
+#define chipModel_GC530						0x00000530
+#define chipModel_GC600						0x00000600
+#define chipModel_GC700						0x00000700
+#define chipModel_GC800						0x00000800
+#define chipModel_GC860						0x00000860
+#define chipModel_GC880						0x00000880
+#define chipModel_GC1000					0x00001000
+#define chipModel_GC1500					0x00001500
+#define chipModel_GC2000					0x00002000
+#define chipModel_GC2100					0x00002100
+#define chipModel_GC2200					0x00002200
+#define chipModel_GC2500					0x00002500
+#define chipModel_GC3000					0x00003000
+#define chipModel_GC4000					0x00004000
+#define chipModel_GC5000					0x00005000
+#define chipModel_GC5200					0x00005200
+#define chipModel_GC6400					0x00006400
+#define RGBA_BITS_R						0x00000001
+#define RGBA_BITS_G						0x00000002
+#define RGBA_BITS_B						0x00000004
+#define RGBA_BITS_A						0x00000008
+#define chipFeatures_FAST_CLEAR					0x00000001
+#define chipFeatures_SPECIAL_ANTI_ALIASING			0x00000002
+#define chipFeatures_PIPE_3D					0x00000004
+#define chipFeatures_DXT_TEXTURE_COMPRESSION			0x00000008
+#define chipFeatures_DEBUG_MODE					0x00000010
+#define chipFeatures_Z_COMPRESSION				0x00000020
+#define chipFeatures_YUV420_SCALER				0x00000040
+#define chipFeatures_MSAA					0x00000080
+#define chipFeatures_DC						0x00000100
+#define chipFeatures_PIPE_2D					0x00000200
+#define chipFeatures_ETC1_TEXTURE_COMPRESSION			0x00000400
+#define chipFeatures_FAST_SCALER				0x00000800
+#define chipFeatures_HIGH_DYNAMIC_RANGE				0x00001000
+#define chipFeatures_YUV420_TILER				0x00002000
+#define chipFeatures_MODULE_CG					0x00004000
+#define chipFeatures_MIN_AREA					0x00008000
+#define chipFeatures_NO_EARLY_Z					0x00010000
+#define chipFeatures_NO_422_TEXTURE				0x00020000
+#define chipFeatures_BUFFER_INTERLEAVING			0x00040000
+#define chipFeatures_BYTE_WRITE_2D				0x00080000
+#define chipFeatures_NO_SCALER					0x00100000
+#define chipFeatures_YUY2_AVERAGING				0x00200000
+#define chipFeatures_HALF_PE_CACHE				0x00400000
+#define chipFeatures_HALF_TX_CACHE				0x00800000
+#define chipFeatures_YUY2_RENDER_TARGET				0x01000000
+#define chipFeatures_MEM32					0x02000000
+#define chipFeatures_PIPE_VG					0x04000000
+#define chipFeatures_VGTS					0x08000000
+#define chipFeatures_FE20					0x10000000
+#define chipFeatures_BYTE_WRITE_3D				0x20000000
+#define chipFeatures_RS_YUV_TARGET				0x40000000
+#define chipFeatures_32_BIT_INDICES				0x80000000
+#define chipMinorFeatures0_FLIP_Y				0x00000001
+#define chipMinorFeatures0_DUAL_RETURN_BUS			0x00000002
+#define chipMinorFeatures0_ENDIANNESS_CONFIG			0x00000004
+#define chipMinorFeatures0_TEXTURE_8K				0x00000008
+#define chipMinorFeatures0_CORRECT_TEXTURE_CONVERTER		0x00000010
+#define chipMinorFeatures0_SPECIAL_MSAA_LOD			0x00000020
+#define chipMinorFeatures0_FAST_CLEAR_FLUSH			0x00000040
+#define chipMinorFeatures0_2DPE20				0x00000080
+#define chipMinorFeatures0_CORRECT_AUTO_DISABLE			0x00000100
+#define chipMinorFeatures0_RENDERTARGET_8K			0x00000200
+#define chipMinorFeatures0_2BITPERTILE				0x00000400
+#define chipMinorFeatures0_SEPARATE_TILE_STATUS_WHEN_INTERLEAVED	0x00000800
+#define chipMinorFeatures0_SUPER_TILED				0x00001000
+#define chipMinorFeatures0_VG_20				0x00002000
+#define chipMinorFeatures0_TS_EXTENDED_COMMANDS			0x00004000
+#define chipMinorFeatures0_COMPRESSION_FIFO_FIXED		0x00008000
+#define chipMinorFeatures0_HAS_SIGN_FLOOR_CEIL			0x00010000
+#define chipMinorFeatures0_VG_FILTER				0x00020000
+#define chipMinorFeatures0_VG_21				0x00040000
+#define chipMinorFeatures0_SHADER_HAS_W				0x00080000
+#define chipMinorFeatures0_HAS_SQRT_TRIG			0x00100000
+#define chipMinorFeatures0_MORE_MINOR_FEATURES			0x00200000
+#define chipMinorFeatures0_MC20					0x00400000
+#define chipMinorFeatures0_MSAA_SIDEBAND			0x00800000
+#define chipMinorFeatures0_BUG_FIXES0				0x01000000
+#define chipMinorFeatures0_VAA					0x02000000
+#define chipMinorFeatures0_BYPASS_IN_MSAA			0x04000000
+#define chipMinorFeatures0_HZ					0x08000000
+#define chipMinorFeatures0_NEW_TEXTURE				0x10000000
+#define chipMinorFeatures0_2D_A8_TARGET				0x20000000
+#define chipMinorFeatures0_CORRECT_STENCIL			0x40000000
+#define chipMinorFeatures0_ENHANCE_VR				0x80000000
+#define chipMinorFeatures1_RSUV_SWIZZLE				0x00000001
+#define chipMinorFeatures1_V2_COMPRESSION			0x00000002
+#define chipMinorFeatures1_VG_DOUBLE_BUFFER			0x00000004
+#define chipMinorFeatures1_EXTRA_EVENT_STATES			0x00000008
+#define chipMinorFeatures1_NO_STRIPING_NEEDED			0x00000010
+#define chipMinorFeatures1_TEXTURE_STRIDE			0x00000020
+#define chipMinorFeatures1_BUG_FIXES3				0x00000040
+#define chipMinorFeatures1_AUTO_DISABLE				0x00000080
+#define chipMinorFeatures1_AUTO_RESTART_TS			0x00000100
+#define chipMinorFeatures1_DISABLE_PE_GATING			0x00000200
+#define chipMinorFeatures1_L2_WINDOWING				0x00000400
+#define chipMinorFeatures1_HALF_FLOAT				0x00000800
+#define chipMinorFeatures1_PIXEL_DITHER				0x00001000
+#define chipMinorFeatures1_TWO_STENCIL_REFERENCE		0x00002000
+#define chipMinorFeatures1_EXTENDED_PIXEL_FORMAT		0x00004000
+#define chipMinorFeatures1_CORRECT_MIN_MAX_DEPTH		0x00008000
+#define chipMinorFeatures1_2D_DITHER				0x00010000
+#define chipMinorFeatures1_BUG_FIXES5				0x00020000
+#define chipMinorFeatures1_NEW_2D				0x00040000
+#define chipMinorFeatures1_NEW_FP				0x00080000
+#define chipMinorFeatures1_TEXTURE_HALIGN			0x00100000
+#define chipMinorFeatures1_NON_POWER_OF_TWO			0x00200000
+#define chipMinorFeatures1_LINEAR_TEXTURE_SUPPORT		0x00400000
+#define chipMinorFeatures1_HALTI0				0x00800000
+#define chipMinorFeatures1_CORRECT_OVERFLOW_VG			0x01000000
+#define chipMinorFeatures1_NEGATIVE_LOG_FIX			0x02000000
+#define chipMinorFeatures1_RESOLVE_OFFSET			0x04000000
+#define chipMinorFeatures1_OK_TO_GATE_AXI_CLOCK			0x08000000
+#define chipMinorFeatures1_MMU_VERSION				0x10000000
+#define chipMinorFeatures1_WIDE_LINE				0x20000000
+#define chipMinorFeatures1_BUG_FIXES6				0x40000000
+#define chipMinorFeatures1_FC_FLUSH_STALL			0x80000000
+#define chipMinorFeatures2_LINE_LOOP				0x00000001
+#define chipMinorFeatures2_LOGIC_OP				0x00000002
+#define chipMinorFeatures2_UNK2					0x00000004
+#define chipMinorFeatures2_SUPERTILED_TEXTURE			0x00000008
+#define chipMinorFeatures2_UNK4					0x00000010
+#define chipMinorFeatures2_RECT_PRIMITIVE			0x00000020
+#define chipMinorFeatures2_COMPOSITION				0x00000040
+#define chipMinorFeatures2_CORRECT_AUTO_DISABLE_COUNT		0x00000080
+#define chipMinorFeatures2_UNK8					0x00000100
+#define chipMinorFeatures2_UNK9					0x00000200
+#define chipMinorFeatures2_UNK10				0x00000400
+#define chipMinorFeatures2_HALTI1				0x00000800
+#define chipMinorFeatures2_UNK12				0x00001000
+#define chipMinorFeatures2_UNK13				0x00002000
+#define chipMinorFeatures2_UNK14				0x00004000
+#define chipMinorFeatures2_EXTRA_TEXTURE_STATE			0x00008000
+#define chipMinorFeatures2_FULL_DIRECTFB			0x00010000
+#define chipMinorFeatures2_2D_TILING				0x00020000
+#define chipMinorFeatures2_THREAD_WALKER_IN_PS			0x00040000
+#define chipMinorFeatures2_TILE_FILLER				0x00080000
+#define chipMinorFeatures2_UNK20				0x00100000
+#define chipMinorFeatures2_2D_MULTI_SOURCE_BLIT			0x00200000
+#define chipMinorFeatures2_UNK22				0x00400000
+#define chipMinorFeatures2_UNK23				0x00800000
+#define chipMinorFeatures2_UNK24				0x01000000
+#define chipMinorFeatures2_MIXED_STREAMS			0x02000000
+#define chipMinorFeatures2_2D_420_L2CACHE			0x04000000
+#define chipMinorFeatures2_UNK27				0x08000000
+#define chipMinorFeatures2_2D_NO_INDEX8_BRUSH			0x10000000
+#define chipMinorFeatures2_TEXTURE_TILED_READ			0x20000000
+#define chipMinorFeatures2_UNK30				0x40000000
+#define chipMinorFeatures2_UNK31				0x80000000
+#define chipMinorFeatures3_ROTATION_STALL_FIX			0x00000001
+#define chipMinorFeatures3_UNK1					0x00000002
+#define chipMinorFeatures3_2D_MULTI_SOURCE_BLT_EX		0x00000004
+#define chipMinorFeatures3_UNK3					0x00000008
+#define chipMinorFeatures3_UNK4					0x00000010
+#define chipMinorFeatures3_UNK5					0x00000020
+#define chipMinorFeatures3_UNK6					0x00000040
+#define chipMinorFeatures3_UNK7					0x00000080
+#define chipMinorFeatures3_FAST_MSAA				0x00000100
+#define chipMinorFeatures3_UNK9					0x00000200
+#define chipMinorFeatures3_BUG_FIXES10				0x00000400
+#define chipMinorFeatures3_UNK11				0x00000800
+#define chipMinorFeatures3_BUG_FIXES11				0x00001000
+#define chipMinorFeatures3_UNK13				0x00002000
+#define chipMinorFeatures3_UNK14				0x00004000
+#define chipMinorFeatures3_UNK15				0x00008000
+#define chipMinorFeatures3_UNK16				0x00010000
+#define chipMinorFeatures3_UNK17				0x00020000
+#define chipMinorFeatures3_ACE					0x00040000
+#define chipMinorFeatures3_UNK19				0x00080000
+#define chipMinorFeatures3_UNK20				0x00100000
+#define chipMinorFeatures3_UNK21				0x00200000
+#define chipMinorFeatures3_UNK22				0x00400000
+#define chipMinorFeatures3_UNK23				0x00800000
+#define chipMinorFeatures3_UNK24				0x01000000
+#define chipMinorFeatures3_UNK25				0x02000000
+#define chipMinorFeatures3_NEW_HZ				0x04000000
+#define chipMinorFeatures3_UNK27				0x08000000
+#define chipMinorFeatures3_UNK28				0x10000000
+#define chipMinorFeatures3_UNK29				0x20000000
+#define chipMinorFeatures3_UNK30				0x40000000
+#define chipMinorFeatures3_UNK31				0x80000000
+#define chipMinorFeatures4_UNK0					0x00000001
+#define chipMinorFeatures4_UNK1					0x00000002
+#define chipMinorFeatures4_UNK2					0x00000004
+#define chipMinorFeatures4_UNK3					0x00000008
+#define chipMinorFeatures4_UNK4					0x00000010
+#define chipMinorFeatures4_UNK5					0x00000020
+#define chipMinorFeatures4_UNK6					0x00000040
+#define chipMinorFeatures4_UNK7					0x00000080
+#define chipMinorFeatures4_UNK8					0x00000100
+#define chipMinorFeatures4_UNK9					0x00000200
+#define chipMinorFeatures4_UNK10				0x00000400
+#define chipMinorFeatures4_UNK11				0x00000800
+#define chipMinorFeatures4_UNK12				0x00001000
+#define chipMinorFeatures4_UNK13				0x00002000
+#define chipMinorFeatures4_UNK14				0x00004000
+#define chipMinorFeatures4_UNK15				0x00008000
+#define chipMinorFeatures4_HALTI2				0x00010000
+#define chipMinorFeatures4_UNK17				0x00020000
+#define chipMinorFeatures4_SMALL_MSAA				0x00040000
+#define chipMinorFeatures4_UNK19				0x00080000
+#define chipMinorFeatures4_UNK20				0x00100000
+#define chipMinorFeatures4_UNK21				0x00200000
+#define chipMinorFeatures4_UNK22				0x00400000
+#define chipMinorFeatures4_UNK23				0x00800000
+#define chipMinorFeatures4_UNK24				0x01000000
+#define chipMinorFeatures4_UNK25				0x02000000
+#define chipMinorFeatures4_UNK26				0x04000000
+#define chipMinorFeatures4_UNK27				0x08000000
+#define chipMinorFeatures4_UNK28				0x10000000
+#define chipMinorFeatures4_UNK29				0x20000000
+#define chipMinorFeatures4_UNK30				0x40000000
+#define chipMinorFeatures4_UNK31				0x80000000
+#define chipMinorFeatures5_UNK0					0x00000001
+#define chipMinorFeatures5_UNK1					0x00000002
+#define chipMinorFeatures5_UNK2					0x00000004
+#define chipMinorFeatures5_UNK3					0x00000008
+#define chipMinorFeatures5_UNK4					0x00000010
+#define chipMinorFeatures5_UNK5					0x00000020
+#define chipMinorFeatures5_UNK6					0x00000040
+#define chipMinorFeatures5_UNK7					0x00000080
+#define chipMinorFeatures5_UNK8					0x00000100
+#define chipMinorFeatures5_HALTI3				0x00000200
+#define chipMinorFeatures5_UNK10				0x00000400
+#define chipMinorFeatures5_UNK11				0x00000800
+#define chipMinorFeatures5_UNK12				0x00001000
+#define chipMinorFeatures5_UNK13				0x00002000
+#define chipMinorFeatures5_UNK14				0x00004000
+#define chipMinorFeatures5_UNK15				0x00008000
+#define chipMinorFeatures5_UNK16				0x00010000
+#define chipMinorFeatures5_UNK17				0x00020000
+#define chipMinorFeatures5_UNK18				0x00040000
+#define chipMinorFeatures5_UNK19				0x00080000
+#define chipMinorFeatures5_UNK20				0x00100000
+#define chipMinorFeatures5_UNK21				0x00200000
+#define chipMinorFeatures5_UNK22				0x00400000
+#define chipMinorFeatures5_UNK23				0x00800000
+#define chipMinorFeatures5_UNK24				0x01000000
+#define chipMinorFeatures5_UNK25				0x02000000
+#define chipMinorFeatures5_UNK26				0x04000000
+#define chipMinorFeatures5_UNK27				0x08000000
+#define chipMinorFeatures5_UNK28				0x10000000
+#define chipMinorFeatures5_UNK29				0x20000000
+#define chipMinorFeatures5_UNK30				0x40000000
+#define chipMinorFeatures5_UNK31				0x80000000
+
+#endif /* COMMON_XML */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
new file mode 100644
index 000000000000..332c55ebba6d
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
@@ -0,0 +1,268 @@
+/*
+ * Copyright (C) 2014 Etnaviv Project
+ * Author: Christian Gmeiner <christian.gmeiner@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "etnaviv_gpu.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_mmu.h"
+
+#include "common.xml.h"
+#include "state.xml.h"
+#include "cmdstream.xml.h"
+
+/*
+ * Command Buffer helper:
+ */
+
+
+static inline void OUT(struct etnaviv_cmdbuf *buffer, u32 data)
+{
+	u32 *vaddr = (u32 *)buffer->vaddr;
+
+	BUG_ON(buffer->user_size >= buffer->size);
+
+	vaddr[buffer->user_size / 4] = data;
+	buffer->user_size += 4;
+}
+
+static inline void CMD_LOAD_STATE(struct etnaviv_cmdbuf *buffer,
+	u32 reg, u32 value)
+{
+	u32 index = reg >> VIV_FE_LOAD_STATE_HEADER_OFFSET__SHR;
+
+	buffer->user_size = ALIGN(buffer->user_size, 8);
+
+	/* write a register via cmd stream */
+	OUT(buffer, VIV_FE_LOAD_STATE_HEADER_OP_LOAD_STATE |
+		    VIV_FE_LOAD_STATE_HEADER_COUNT(1) |
+		    VIV_FE_LOAD_STATE_HEADER_OFFSET(index));
+	OUT(buffer, value);
+}
+
+static inline void CMD_END(struct etnaviv_cmdbuf *buffer)
+{
+	buffer->user_size = ALIGN(buffer->user_size, 8);
+
+	OUT(buffer, VIV_FE_END_HEADER_OP_END);
+}
+
+static inline void CMD_WAIT(struct etnaviv_cmdbuf *buffer)
+{
+	buffer->user_size = ALIGN(buffer->user_size, 8);
+
+	OUT(buffer, VIV_FE_WAIT_HEADER_OP_WAIT | 200);
+}
+
+static inline void CMD_LINK(struct etnaviv_cmdbuf *buffer,
+	u16 prefetch, u32 address)
+{
+	buffer->user_size = ALIGN(buffer->user_size, 8);
+
+	OUT(buffer, VIV_FE_LINK_HEADER_OP_LINK |
+		    VIV_FE_LINK_HEADER_PREFETCH(prefetch));
+	OUT(buffer, address);
+}
+
+static inline void CMD_STALL(struct etnaviv_cmdbuf *buffer,
+	u32 from, u32 to)
+{
+	buffer->user_size = ALIGN(buffer->user_size, 8);
+
+	OUT(buffer, VIV_FE_STALL_HEADER_OP_STALL);
+	OUT(buffer, VIV_FE_STALL_TOKEN_FROM(from) | VIV_FE_STALL_TOKEN_TO(to));
+}
+
+static void etnaviv_cmd_select_pipe(struct etnaviv_cmdbuf *buffer, u8 pipe)
+{
+	u32 flush;
+	u32 stall;
+
+	/*
+	 * This assumes that if we're switching to 2D, we're switching
+	 * away from 3D, and vice versa.  Hence, if we're switching to
+	 * the 2D core, we need to flush the 3D depth and color caches,
+	 * otherwise we need to flush the 2D pixel engine cache.
+	 */
+	if (pipe == ETNA_PIPE_2D)
+		flush = VIVS_GL_FLUSH_CACHE_DEPTH | VIVS_GL_FLUSH_CACHE_COLOR;
+	else
+		flush = VIVS_GL_FLUSH_CACHE_PE2D;
+
+	stall = VIVS_GL_SEMAPHORE_TOKEN_FROM(SYNC_RECIPIENT_FE) |
+		VIVS_GL_SEMAPHORE_TOKEN_TO(SYNC_RECIPIENT_PE);
+
+	CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE, flush);
+	CMD_LOAD_STATE(buffer, VIVS_GL_SEMAPHORE_TOKEN, stall);
+
+	CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+
+	CMD_LOAD_STATE(buffer, VIVS_GL_PIPE_SELECT,
+		       VIVS_GL_PIPE_SELECT_PIPE(pipe));
+}
+
+static u32 gpu_va(struct etnaviv_gpu *gpu, struct etnaviv_cmdbuf *buf)
+{
+	return buf->paddr - gpu->memory_base;
+}
+
+static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,
+	struct etnaviv_cmdbuf *buf, u32 off, u32 len)
+{
+	u32 size = buf->size;
+	u32 *ptr = buf->vaddr + off;
+
+	dev_info(gpu->dev, "virt %p phys 0x%08x free 0x%08x\n",
+			ptr, gpu_va(gpu, buf) + off, size - len * 4 - off);
+
+	print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
+			ptr, len * 4, 0);
+}
+
+u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+
+	/* initialize buffer */
+	buffer->user_size = 0;
+
+	CMD_WAIT(buffer);
+	CMD_LINK(buffer, 2, gpu_va(gpu, buffer) + buffer->user_size - 4);
+
+	return buffer->user_size / 8;
+}
+
+void etnaviv_buffer_end(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+
+	/* Replace the last WAIT with an END */
+	buffer->user_size -= 16;
+
+	CMD_END(buffer);
+	mb();
+}
+
+void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
+	struct etnaviv_cmdbuf *cmdbuf)
+{
+	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	u32 *lw = buffer->vaddr + buffer->user_size - 16;
+	u32 back, link_target, link_size, reserve_size, extra_size = 0;
+
+	if (drm_debug & DRM_UT_DRIVER)
+		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
+
+	/*
+	 * If we need to flush the MMU prior to submitting this buffer, we
+	 * will need to append a mmu flush load state, followed by a new
+	 * link to this buffer - a total of four additional words.
+	 */
+	if (gpu->mmu->need_flush || gpu->switch_context) {
+		/* link command */
+		extra_size += 2;
+		/* flush command */
+		if (gpu->mmu->need_flush)
+			extra_size += 2;
+		/* pipe switch commands */
+		if (gpu->switch_context)
+			extra_size += 8;
+	}
+
+	reserve_size = (6 + extra_size) * 4;
+
+	/*
+	 * if we are going to completely overflow the buffer, we need to wrap.
+	 */
+	if (buffer->user_size + reserve_size > buffer->size)
+		buffer->user_size = 0;
+
+	/* save offset back into main buffer */
+	back = buffer->user_size + reserve_size - 6 * 4;
+	link_target = gpu_va(gpu, buffer) + buffer->user_size;
+	link_size = 6;
+
+	/* Skip over any extra instructions */
+	link_target += extra_size * sizeof(u32);
+
+	if (drm_debug & DRM_UT_DRIVER)
+		pr_info("stream link to 0x%08x @ 0x%08x %p\n",
+			link_target, gpu_va(gpu, cmdbuf), cmdbuf->vaddr);
+
+	/* jump back from cmd to main buffer */
+	CMD_LINK(cmdbuf, link_size, link_target);
+
+	link_target = gpu_va(gpu, cmdbuf);
+	link_size = cmdbuf->size / 8;
+
+
+
+	if (drm_debug & DRM_UT_DRIVER) {
+		print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
+			       cmdbuf->vaddr, cmdbuf->size, 0);
+
+		pr_info("link op: %p\n", lw);
+		pr_info("link addr: %p\n", lw + 1);
+		pr_info("addr: 0x%08x\n", link_target);
+		pr_info("back: 0x%08x\n", gpu_va(gpu, buffer) + back);
+		pr_info("event: %d\n", event);
+	}
+
+	if (gpu->mmu->need_flush || gpu->switch_context) {
+		u32 new_target = gpu_va(gpu, buffer) + buffer->user_size;
+
+		if (gpu->mmu->need_flush) {
+			/* Add the MMU flush */
+			CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
+				       VIVS_GL_FLUSH_MMU_FLUSH_FEMMU |
+				       VIVS_GL_FLUSH_MMU_FLUSH_UNK1 |
+				       VIVS_GL_FLUSH_MMU_FLUSH_UNK2 |
+				       VIVS_GL_FLUSH_MMU_FLUSH_PEMMU |
+				       VIVS_GL_FLUSH_MMU_FLUSH_UNK4);
+
+			gpu->mmu->need_flush = false;
+		}
+
+		if (gpu->switch_context) {
+			etnaviv_cmd_select_pipe(buffer, cmdbuf->exec_state);
+			gpu->switch_context = false;
+		}
+
+		/* And the link to the first buffer */
+		CMD_LINK(buffer, link_size, link_target);
+
+		/* Update the link target to point to above instructions */
+		link_target = new_target;
+		link_size = extra_size;
+	}
+
+	/* trigger event */
+	CMD_LOAD_STATE(buffer, VIVS_GL_EVENT, VIVS_GL_EVENT_EVENT_ID(event) |
+		       VIVS_GL_EVENT_FROM_PE);
+
+	/* append WAIT/LINK to main buffer */
+	CMD_WAIT(buffer);
+	CMD_LINK(buffer, 2, gpu_va(gpu, buffer) + (buffer->user_size - 4));
+
+	/* Change WAIT into a LINK command; write the address first. */
+	*(lw + 1) = link_target;
+	mb();
+	*(lw) = VIV_FE_LINK_HEADER_OP_LINK |
+		VIV_FE_LINK_HEADER_PREFETCH(link_size);
+	mb();
+
+	if (drm_debug & DRM_UT_DRIVER)
+		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
new file mode 100644
index 000000000000..dcfd565c88d1
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
@@ -0,0 +1,209 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+
+#include "etnaviv_gem.h"
+#include "etnaviv_gpu.h"
+
+#include "cmdstream.xml.h"
+
+#define EXTRACT(val, field) (((val) & field##__MASK) >> field##__SHIFT)
+
+struct etna_validation_state {
+	struct etnaviv_gpu *gpu;
+	const struct drm_etnaviv_gem_submit_reloc *relocs;
+	unsigned int num_relocs;
+	u32 *start;
+};
+
+static const struct {
+	u16 offset;
+	u16 size;
+} etnaviv_sensitive_states[] __initconst = {
+#define ST(start, num) { (start) >> 2, (num) }
+	/* 2D */
+	ST(0x1200, 1),
+	ST(0x1228, 1),
+	ST(0x1238, 1),
+	ST(0x1284, 1),
+	ST(0x128c, 1),
+	ST(0x1304, 1),
+	ST(0x1310, 1),
+	ST(0x1318, 1),
+	ST(0x12800, 4),
+	ST(0x128a0, 4),
+	ST(0x128c0, 4),
+	ST(0x12970, 4),
+	ST(0x12a00, 8),
+	ST(0x12b40, 8),
+	ST(0x12b80, 8),
+	ST(0x12ce0, 8),
+	/* 3D */
+	ST(0x0644, 1),
+	ST(0x064c, 1),
+	ST(0x0680, 8),
+	ST(0x1410, 1),
+	ST(0x1430, 1),
+	ST(0x1458, 1),
+	ST(0x1460, 8),
+	ST(0x1480, 8),
+	ST(0x1500, 8),
+	ST(0x1520, 8),
+	ST(0x1608, 1),
+	ST(0x1610, 1),
+	ST(0x1658, 1),
+	ST(0x165c, 1),
+	ST(0x1664, 1),
+	ST(0x1668, 1),
+	ST(0x16a4, 1),
+	ST(0x16c0, 8),
+	ST(0x16e0, 8),
+	ST(0x1740, 8),
+	ST(0x2400, 14 * 16),
+	ST(0x10800, 32 * 16),
+#undef ST
+};
+
+#define ETNAVIV_STATES_SIZE (VIV_FE_LOAD_STATE_HEADER_OFFSET__MASK + 1u)
+static DECLARE_BITMAP(etnaviv_states, ETNAVIV_STATES_SIZE);
+
+void __init etnaviv_validate_init(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(etnaviv_sensitive_states); i++)
+		bitmap_set(etnaviv_states, etnaviv_sensitive_states[i].offset,
+			   etnaviv_sensitive_states[i].size);
+}
+
+static void etnaviv_warn_if_non_sensitive(struct etna_validation_state *state,
+	unsigned int buf_offset, unsigned int state_addr)
+{
+	if (state->num_relocs && state->relocs->submit_offset < buf_offset) {
+		dev_warn_once(state->gpu->dev,
+			      "%s: relocation for non-sensitive state 0x%x at offset %u\n",
+			      __func__, state_addr,
+			      state->relocs->submit_offset);
+		while (state->num_relocs &&
+		       state->relocs->submit_offset < buf_offset) {
+			state->relocs++;
+			state->num_relocs--;
+		}
+	}
+}
+
+static bool etnaviv_validate_load_state(struct etna_validation_state *state,
+	u32 *ptr, unsigned int state_offset, unsigned int num)
+{
+	unsigned int size = min(ETNAVIV_STATES_SIZE, state_offset + num);
+	unsigned int st_offset = state_offset, buf_offset;
+
+	for_each_set_bit_from(st_offset, etnaviv_states, size) {
+		buf_offset = (ptr - state->start +
+			      st_offset - state_offset) * 4;
+
+		etnaviv_warn_if_non_sensitive(state, buf_offset, st_offset * 4);
+		if (state->num_relocs &&
+		    state->relocs->submit_offset == buf_offset) {
+			state->relocs++;
+			state->num_relocs--;
+			continue;
+		}
+
+		dev_warn_ratelimited(state->gpu->dev,
+				     "%s: load state touches restricted state 0x%x at offset %u\n",
+				     __func__, st_offset * 4, buf_offset);
+		return false;
+	}
+
+	if (state->num_relocs) {
+		buf_offset = (ptr - state->start + num) * 4;
+		etnaviv_warn_if_non_sensitive(state, buf_offset, st_offset * 4 +
+					      state->relocs->submit_offset -
+					      buf_offset);
+	}
+
+	return true;
+}
+
+static uint8_t cmd_length[32] = {
+	[FE_OPCODE_DRAW_PRIMITIVES] = 4,
+	[FE_OPCODE_DRAW_INDEXED_PRIMITIVES] = 6,
+	[FE_OPCODE_NOP] = 2,
+	[FE_OPCODE_STALL] = 2,
+};
+
+bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu, u32 *stream,
+			      unsigned int size,
+			      struct drm_etnaviv_gem_submit_reloc *relocs,
+			      unsigned int reloc_size)
+{
+	struct etna_validation_state state;
+	u32 *buf = stream;
+	u32 *end = buf + size;
+
+	state.gpu = gpu;
+	state.relocs = relocs;
+	state.num_relocs = reloc_size;
+	state.start = stream;
+
+	while (buf < end) {
+		u32 cmd = *buf;
+		unsigned int len, n, off;
+		unsigned int op = cmd >> 27;
+
+		switch (op) {
+		case FE_OPCODE_LOAD_STATE:
+			n = EXTRACT(cmd, VIV_FE_LOAD_STATE_HEADER_COUNT);
+			len = ALIGN(1 + n, 2);
+			if (buf + len > end)
+				break;
+
+			off = EXTRACT(cmd, VIV_FE_LOAD_STATE_HEADER_OFFSET);
+			if (!etnaviv_validate_load_state(&state, buf + 1,
+							 off, n))
+				return false;
+			break;
+
+		case FE_OPCODE_DRAW_2D:
+			n = EXTRACT(cmd, VIV_FE_DRAW_2D_HEADER_COUNT);
+			if (n == 0)
+				n = 256;
+			len = 2 + n * 2;
+			break;
+
+		default:
+			len = cmd_length[op];
+			if (len == 0) {
+				dev_err(gpu->dev, "%s: op %u not permitted at offset %tu\n",
+					__func__, op, buf - state.start);
+				return false;
+			}
+			break;
+		}
+
+		buf += len;
+	}
+
+	if (buf > end) {
+		dev_err(gpu->dev, "%s: commands overflow end of buffer: %tu > %u\n",
+			__func__, buf - state.start, size);
+		return false;
+	}
+
+	return true;
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
new file mode 100644
index 000000000000..e8858985f01e
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -0,0 +1,706 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/component.h>
+#include <linux/of_platform.h>
+
+#include "etnaviv_drv.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_mmu.h"
+#include "etnaviv_gem.h"
+
+#ifdef CONFIG_DRM_ETNAVIV_REGISTER_LOGGING
+static bool reglog;
+MODULE_PARM_DESC(reglog, "Enable register read/write logging");
+module_param(reglog, bool, 0600);
+#else
+#define reglog 0
+#endif
+
+void __iomem *etnaviv_ioremap(struct platform_device *pdev, const char *name,
+		const char *dbgname)
+{
+	struct resource *res;
+	void __iomem *ptr;
+
+	if (name)
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	else
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	ptr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(ptr)) {
+		dev_err(&pdev->dev, "failed to ioremap %s: %ld\n", name,
+			PTR_ERR(ptr));
+		return ptr;
+	}
+
+	if (reglog)
+		dev_printk(KERN_DEBUG, &pdev->dev, "IO:region %s 0x%p %08zx\n",
+			   dbgname, ptr, (size_t)resource_size(res));
+
+	return ptr;
+}
+
+void etnaviv_writel(u32 data, void __iomem *addr)
+{
+	if (reglog)
+		printk(KERN_DEBUG "IO:W %p %08x\n", addr, data);
+
+	writel(data, addr);
+}
+
+u32 etnaviv_readl(const void __iomem *addr)
+{
+	u32 val = readl(addr);
+
+	if (reglog)
+		printk(KERN_DEBUG "IO:R %p %08x\n", addr, val);
+
+	return val;
+}
+
+/*
+ * DRM operations:
+ */
+
+
+static void load_gpu(struct drm_device *dev)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	unsigned int i;
+
+	for (i = 0; i < ETNA_MAX_PIPES; i++) {
+		struct etnaviv_gpu *g = priv->gpu[i];
+
+		if (g) {
+			int ret;
+
+			ret = etnaviv_gpu_init(g);
+			if (ret) {
+				dev_err(g->dev, "hw init failed: %d\n", ret);
+				priv->gpu[i] = NULL;
+			}
+		}
+	}
+}
+
+static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
+{
+	struct etnaviv_file_private *ctx;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	file->driver_priv = ctx;
+
+	return 0;
+}
+
+static void etnaviv_preclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct etnaviv_file_private *ctx = file->driver_priv;
+	unsigned int i;
+
+	for (i = 0; i < ETNA_MAX_PIPES; i++) {
+		struct etnaviv_gpu *gpu = priv->gpu[i];
+
+		if (gpu) {
+			mutex_lock(&gpu->lock);
+			if (gpu->lastctx == ctx)
+				gpu->lastctx = NULL;
+			mutex_unlock(&gpu->lock);
+		}
+	}
+
+	kfree(ctx);
+}
+
+/*
+ * DRM debugfs:
+ */
+
+#ifdef CONFIG_DEBUG_FS
+static int etnaviv_gem_show(struct drm_device *dev, struct seq_file *m)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+
+	etnaviv_gem_describe_objects(priv, m);
+
+	return 0;
+}
+
+static int etnaviv_mm_show(struct drm_device *dev, struct seq_file *m)
+{
+	int ret;
+
+	read_lock(&dev->vma_offset_manager->vm_lock);
+	ret = drm_mm_dump_table(m, &dev->vma_offset_manager->vm_addr_space_mm);
+	read_unlock(&dev->vma_offset_manager->vm_lock);
+
+	return ret;
+}
+
+static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m)
+{
+	seq_printf(m, "Active Objects (%s):\n", dev_name(gpu->dev));
+
+	mutex_lock(&gpu->mmu->lock);
+	drm_mm_dump_table(m, &gpu->mmu->mm);
+	mutex_unlock(&gpu->mmu->lock);
+
+	return 0;
+}
+
+static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, struct seq_file *m)
+{
+	struct etnaviv_cmdbuf *buf = gpu->buffer;
+	u32 size = buf->size;
+	u32 *ptr = buf->vaddr;
+	u32 i;
+
+	seq_printf(m, "virt %p - phys 0x%llx - free 0x%08x\n",
+			buf->vaddr, (u64)buf->paddr, size - buf->user_size);
+
+	for (i = 0; i < size / 4; i++) {
+		if (i && !(i % 4))
+			seq_puts(m, "\n");
+		if (i % 4 == 0)
+			seq_printf(m, "\t0x%p: ", ptr + i);
+		seq_printf(m, "%08x ", *(ptr + i));
+	}
+	seq_puts(m, "\n");
+}
+
+static int etnaviv_ring_show(struct etnaviv_gpu *gpu, struct seq_file *m)
+{
+	seq_printf(m, "Ring Buffer (%s): ", dev_name(gpu->dev));
+
+	mutex_lock(&gpu->lock);
+	etnaviv_buffer_dump(gpu, m);
+	mutex_unlock(&gpu->lock);
+
+	return 0;
+}
+
+static int show_unlocked(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	int (*show)(struct drm_device *dev, struct seq_file *m) =
+			node->info_ent->data;
+
+	return show(dev, m);
+}
+
+static int show_each_gpu(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct etnaviv_gpu *gpu;
+	int (*show)(struct etnaviv_gpu *gpu, struct seq_file *m) =
+			node->info_ent->data;
+	unsigned int i;
+	int ret = 0;
+
+	for (i = 0; i < ETNA_MAX_PIPES; i++) {
+		gpu = priv->gpu[i];
+		if (!gpu)
+			continue;
+
+		ret = show(gpu, m);
+		if (ret < 0)
+			break;
+	}
+
+	return ret;
+}
+
+static struct drm_info_list etnaviv_debugfs_list[] = {
+		{"gpu", show_each_gpu, 0, etnaviv_gpu_debugfs},
+		{"gem", show_unlocked, 0, etnaviv_gem_show},
+		{ "mm", show_unlocked, 0, etnaviv_mm_show },
+		{"mmu", show_each_gpu, 0, etnaviv_mmu_show},
+		{"ring", show_each_gpu, 0, etnaviv_ring_show},
+};
+
+static int etnaviv_debugfs_init(struct drm_minor *minor)
+{
+	struct drm_device *dev = minor->dev;
+	int ret;
+
+	ret = drm_debugfs_create_files(etnaviv_debugfs_list,
+			ARRAY_SIZE(etnaviv_debugfs_list),
+			minor->debugfs_root, minor);
+
+	if (ret) {
+		dev_err(dev->dev, "could not install etnaviv_debugfs_list\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static void etnaviv_debugfs_cleanup(struct drm_minor *minor)
+{
+	drm_debugfs_remove_files(etnaviv_debugfs_list,
+			ARRAY_SIZE(etnaviv_debugfs_list), minor);
+}
+#endif
+
+/*
+ * DRM ioctls:
+ */
+
+static int etnaviv_ioctl_get_param(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct drm_etnaviv_param *args = data;
+	struct etnaviv_gpu *gpu;
+
+	if (args->pipe >= ETNA_MAX_PIPES)
+		return -EINVAL;
+
+	gpu = priv->gpu[args->pipe];
+	if (!gpu)
+		return -ENXIO;
+
+	return etnaviv_gpu_get_param(gpu, args->param, &args->value);
+}
+
+static int etnaviv_ioctl_gem_new(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_etnaviv_gem_new *args = data;
+
+	if (args->flags & ~(ETNA_BO_CACHED | ETNA_BO_WC | ETNA_BO_UNCACHED |
+			    ETNA_BO_FORCE_MMU))
+		return -EINVAL;
+
+	return etnaviv_gem_new_handle(dev, file, args->size,
+			args->flags, &args->handle);
+}
+
+#define TS(t) ((struct timespec){ \
+	.tv_sec = (t).tv_sec, \
+	.tv_nsec = (t).tv_nsec \
+})
+
+static int etnaviv_ioctl_gem_cpu_prep(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_etnaviv_gem_cpu_prep *args = data;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (args->op & ~(ETNA_PREP_READ | ETNA_PREP_WRITE | ETNA_PREP_NOSYNC))
+		return -EINVAL;
+
+	obj = drm_gem_object_lookup(dev, file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	ret = etnaviv_gem_cpu_prep(obj, args->op, &TS(args->timeout));
+
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
+}
+
+static int etnaviv_ioctl_gem_cpu_fini(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_etnaviv_gem_cpu_fini *args = data;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (args->flags)
+		return -EINVAL;
+
+	obj = drm_gem_object_lookup(dev, file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	ret = etnaviv_gem_cpu_fini(obj);
+
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
+}
+
+static int etnaviv_ioctl_gem_info(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_etnaviv_gem_info *args = data;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (args->pad)
+		return -EINVAL;
+
+	obj = drm_gem_object_lookup(dev, file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	ret = etnaviv_gem_mmap_offset(obj, &args->offset);
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
+}
+
+static int etnaviv_ioctl_wait_fence(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_etnaviv_wait_fence *args = data;
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct timespec *timeout = &TS(args->timeout);
+	struct etnaviv_gpu *gpu;
+
+	if (args->flags & ~(ETNA_WAIT_NONBLOCK))
+		return -EINVAL;
+
+	if (args->pipe >= ETNA_MAX_PIPES)
+		return -EINVAL;
+
+	gpu = priv->gpu[args->pipe];
+	if (!gpu)
+		return -ENXIO;
+
+	if (args->flags & ETNA_WAIT_NONBLOCK)
+		timeout = NULL;
+
+	return etnaviv_gpu_wait_fence_interruptible(gpu, args->fence,
+						    timeout);
+}
+
+static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data,
+	struct drm_file *file)
+{
+	struct drm_etnaviv_gem_userptr *args = data;
+	int access;
+
+	if (args->flags & ~(ETNA_USERPTR_READ|ETNA_USERPTR_WRITE) ||
+	    args->flags == 0)
+		return -EINVAL;
+
+	if (offset_in_page(args->user_ptr | args->user_size) ||
+	    (uintptr_t)args->user_ptr != args->user_ptr ||
+	    (u32)args->user_size != args->user_size ||
+	    args->user_ptr & ~PAGE_MASK)
+		return -EINVAL;
+
+	if (args->flags & ETNA_USERPTR_WRITE)
+		access = VERIFY_WRITE;
+	else
+		access = VERIFY_READ;
+
+	if (!access_ok(access, (void __user *)(unsigned long)args->user_ptr,
+		       args->user_size))
+		return -EFAULT;
+
+	return etnaviv_gem_new_userptr(dev, file, args->user_ptr,
+				       args->user_size, args->flags,
+				       &args->handle);
+}
+
+static int etnaviv_ioctl_gem_wait(struct drm_device *dev, void *data,
+	struct drm_file *file)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct drm_etnaviv_gem_wait *args = data;
+	struct timespec *timeout = &TS(args->timeout);
+	struct drm_gem_object *obj;
+	struct etnaviv_gpu *gpu;
+	int ret;
+
+	if (args->flags & ~(ETNA_WAIT_NONBLOCK))
+		return -EINVAL;
+
+	if (args->pipe >= ETNA_MAX_PIPES)
+		return -EINVAL;
+
+	gpu = priv->gpu[args->pipe];
+	if (!gpu)
+		return -ENXIO;
+
+	obj = drm_gem_object_lookup(dev, file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	if (args->flags & ETNA_WAIT_NONBLOCK)
+		timeout = NULL;
+
+	ret = etnaviv_gem_wait_bo(gpu, obj, timeout);
+
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
+}
+
+static const struct drm_ioctl_desc etnaviv_ioctls[] = {
+#define ETNA_IOCTL(n, func, flags) \
+	DRM_IOCTL_DEF_DRV(ETNAVIV_##n, etnaviv_ioctl_##func, flags)
+	ETNA_IOCTL(GET_PARAM,    get_param,    DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_NEW,      gem_new,      DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_INFO,     gem_info,     DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_CPU_PREP, gem_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_CPU_FINI, gem_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_SUBMIT,   gem_submit,   DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(WAIT_FENCE,   wait_fence,   DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_USERPTR,  gem_userptr,  DRM_AUTH|DRM_RENDER_ALLOW),
+	ETNA_IOCTL(GEM_WAIT,     gem_wait,     DRM_AUTH|DRM_RENDER_ALLOW),
+};
+
+static const struct vm_operations_struct vm_ops = {
+	.fault = etnaviv_gem_fault,
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static const struct file_operations fops = {
+	.owner              = THIS_MODULE,
+	.open               = drm_open,
+	.release            = drm_release,
+	.unlocked_ioctl     = drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl       = drm_compat_ioctl,
+#endif
+	.poll               = drm_poll,
+	.read               = drm_read,
+	.llseek             = no_llseek,
+	.mmap               = etnaviv_gem_mmap,
+};
+
+static struct drm_driver etnaviv_drm_driver = {
+	.driver_features    = DRIVER_HAVE_IRQ |
+				DRIVER_GEM |
+				DRIVER_PRIME |
+				DRIVER_RENDER,
+	.open               = etnaviv_open,
+	.preclose           = etnaviv_preclose,
+	.set_busid          = drm_platform_set_busid,
+	.gem_free_object    = etnaviv_gem_free_object,
+	.gem_vm_ops         = &vm_ops,
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_export   = drm_gem_prime_export,
+	.gem_prime_import   = drm_gem_prime_import,
+	.gem_prime_pin      = etnaviv_gem_prime_pin,
+	.gem_prime_unpin    = etnaviv_gem_prime_unpin,
+	.gem_prime_get_sg_table = etnaviv_gem_prime_get_sg_table,
+	.gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table,
+	.gem_prime_vmap     = etnaviv_gem_prime_vmap,
+	.gem_prime_vunmap   = etnaviv_gem_prime_vunmap,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init       = etnaviv_debugfs_init,
+	.debugfs_cleanup    = etnaviv_debugfs_cleanup,
+#endif
+	.ioctls             = etnaviv_ioctls,
+	.num_ioctls         = DRM_ETNAVIV_NUM_IOCTLS,
+	.fops               = &fops,
+	.name               = "etnaviv",
+	.desc               = "etnaviv DRM",
+	.date               = "20151214",
+	.major              = 1,
+	.minor              = 0,
+};
+
+/*
+ * Platform driver:
+ */
+static int etnaviv_bind(struct device *dev)
+{
+	struct etnaviv_drm_private *priv;
+	struct drm_device *drm;
+	int ret;
+
+	drm = drm_dev_alloc(&etnaviv_drm_driver, dev);
+	if (!drm)
+		return -ENOMEM;
+
+	drm->platformdev = to_platform_device(dev);
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		dev_err(dev, "failed to allocate private data\n");
+		ret = -ENOMEM;
+		goto out_unref;
+	}
+	drm->dev_private = priv;
+
+	priv->wq = alloc_ordered_workqueue("etnaviv", 0);
+	if (!priv->wq) {
+		ret = -ENOMEM;
+		goto out_wq;
+	}
+
+	mutex_init(&priv->gem_lock);
+	INIT_LIST_HEAD(&priv->gem_list);
+	priv->num_gpus = 0;
+
+	dev_set_drvdata(dev, drm);
+
+	ret = component_bind_all(dev, drm);
+	if (ret < 0)
+		goto out_bind;
+
+	load_gpu(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto out_register;
+
+	return 0;
+
+out_register:
+	component_unbind_all(dev, drm);
+out_bind:
+	flush_workqueue(priv->wq);
+	destroy_workqueue(priv->wq);
+out_wq:
+	kfree(priv);
+out_unref:
+	drm_dev_unref(drm);
+
+	return ret;
+}
+
+static void etnaviv_unbind(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct etnaviv_drm_private *priv = drm->dev_private;
+
+	drm_dev_unregister(drm);
+
+	flush_workqueue(priv->wq);
+	destroy_workqueue(priv->wq);
+
+	component_unbind_all(dev, drm);
+
+	drm->dev_private = NULL;
+	kfree(priv);
+
+	drm_put_dev(drm);
+}
+
+static const struct component_master_ops etnaviv_master_ops = {
+	.bind = etnaviv_bind,
+	.unbind = etnaviv_unbind,
+};
+
+static int compare_of(struct device *dev, void *data)
+{
+	struct device_node *np = data;
+
+	return dev->of_node == np;
+}
+
+static int compare_str(struct device *dev, void *data)
+{
+	return !strcmp(dev_name(dev), data);
+}
+
+static int etnaviv_pdev_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct component_match *match = NULL;
+
+	dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+
+	if (node) {
+		struct device_node *core_node;
+		int i;
+
+		for (i = 0; ; i++) {
+			core_node = of_parse_phandle(node, "cores", i);
+			if (!core_node)
+				break;
+
+			component_match_add(&pdev->dev, &match, compare_of,
+					    core_node);
+			of_node_put(core_node);
+		}
+	} else if (dev->platform_data) {
+		char **names = dev->platform_data;
+		unsigned i;
+
+		for (i = 0; names[i]; i++)
+			component_match_add(dev, &match, compare_str, names[i]);
+	}
+
+	return component_master_add_with_match(dev, &etnaviv_master_ops, match);
+}
+
+static int etnaviv_pdev_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &etnaviv_master_ops);
+
+	return 0;
+}
+
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "fsl,imx-gpu-subsystem" },
+	{ .compatible = "marvell,dove-gpu-subsystem" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dt_match);
+
+static struct platform_driver etnaviv_platform_driver = {
+	.probe      = etnaviv_pdev_probe,
+	.remove     = etnaviv_pdev_remove,
+	.driver     = {
+		.name   = "etnaviv",
+		.of_match_table = dt_match,
+	},
+};
+
+static int __init etnaviv_init(void)
+{
+	int ret;
+
+	etnaviv_validate_init();
+
+	ret = platform_driver_register(&etnaviv_gpu_driver);
+	if (ret != 0)
+		return ret;
+
+	ret = platform_driver_register(&etnaviv_platform_driver);
+	if (ret != 0)
+		platform_driver_unregister(&etnaviv_gpu_driver);
+
+	return ret;
+}
+module_init(etnaviv_init);
+
+static void __exit etnaviv_exit(void)
+{
+	platform_driver_unregister(&etnaviv_gpu_driver);
+	platform_driver_unregister(&etnaviv_platform_driver);
+}
+module_exit(etnaviv_exit);
+
+MODULE_AUTHOR("Christian Gmeiner <christian.gmeiner@gmail.com>");
+MODULE_AUTHOR("Russell King <rmk+kernel@arm.linux.org.uk>");
+MODULE_AUTHOR("Lucas Stach <l.stach@pengutronix.de>");
+MODULE_DESCRIPTION("etnaviv DRM Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:etnaviv");
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
new file mode 100644
index 000000000000..1cd6046e76b1
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_DRV_H__
+#define __ETNAVIV_DRV_H__
+
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/iommu.h>
+#include <linux/types.h>
+#include <linux/sizes.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem.h>
+#include <drm/etnaviv_drm.h>
+
+struct etnaviv_cmdbuf;
+struct etnaviv_gpu;
+struct etnaviv_mmu;
+struct etnaviv_gem_object;
+struct etnaviv_gem_submit;
+
+struct etnaviv_file_private {
+	/* currently we don't do anything useful with this.. but when
+	 * per-context address spaces are supported we'd keep track of
+	 * the context's page-tables here.
+	 */
+	int dummy;
+};
+
+struct etnaviv_drm_private {
+	int num_gpus;
+	struct etnaviv_gpu *gpu[ETNA_MAX_PIPES];
+
+	/* list of GEM objects: */
+	struct mutex gem_lock;
+	struct list_head gem_list;
+
+	struct workqueue_struct *wq;
+};
+
+static inline void etnaviv_queue_work(struct drm_device *dev,
+	struct work_struct *w)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+
+	queue_work(priv->wq, w);
+}
+
+int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
+		struct drm_file *file);
+
+int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset);
+int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu,
+	struct drm_gem_object *obj, u32 *iova);
+void etnaviv_gem_put_iova(struct etnaviv_gpu *gpu, struct drm_gem_object *obj);
+struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj);
+void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj);
+void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
+	struct dma_buf_attachment *attach, struct sg_table *sg);
+int etnaviv_gem_prime_pin(struct drm_gem_object *obj);
+void etnaviv_gem_prime_unpin(struct drm_gem_object *obj);
+void *etnaviv_gem_vmap(struct drm_gem_object *obj);
+int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
+		struct timespec *timeout);
+int etnaviv_gem_cpu_fini(struct drm_gem_object *obj);
+void etnaviv_gem_free_object(struct drm_gem_object *obj);
+int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
+		u32 size, u32 flags, u32 *handle);
+struct drm_gem_object *etnaviv_gem_new_locked(struct drm_device *dev,
+		u32 size, u32 flags);
+struct drm_gem_object *etnaviv_gem_new(struct drm_device *dev,
+		u32 size, u32 flags);
+int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
+	uintptr_t ptr, u32 size, u32 flags, u32 *handle);
+u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu);
+void etnaviv_buffer_end(struct etnaviv_gpu *gpu);
+void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
+	struct etnaviv_cmdbuf *cmdbuf);
+void etnaviv_validate_init(void);
+bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu,
+	u32 *stream, unsigned int size,
+	struct drm_etnaviv_gem_submit_reloc *relocs, unsigned int reloc_size);
+
+#ifdef CONFIG_DEBUG_FS
+void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
+	struct seq_file *m);
+#endif
+
+void __iomem *etnaviv_ioremap(struct platform_device *pdev, const char *name,
+		const char *dbgname);
+void etnaviv_writel(u32 data, void __iomem *addr);
+u32 etnaviv_readl(const void __iomem *addr);
+
+#define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
+#define VERB(fmt, ...) if (0) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
+
+/*
+ * Return the storage size of a structure with a variable length array.
+ * The array is nelem elements of elem_size, where the base structure
+ * is defined by base.  If the size overflows size_t, return zero.
+ */
+static inline size_t size_vstruct(size_t nelem, size_t elem_size, size_t base)
+{
+	if (elem_size && nelem > (SIZE_MAX - base) / elem_size)
+		return 0;
+	return base + nelem * elem_size;
+}
+
+/* returns true if fence a comes after fence b */
+static inline bool fence_after(u32 a, u32 b)
+{
+	return (s32)(a - b) > 0;
+}
+
+static inline bool fence_after_eq(u32 a, u32 b)
+{
+	return (s32)(a - b) >= 0;
+}
+
+static inline unsigned long etnaviv_timeout_to_jiffies(
+	const struct timespec *timeout)
+{
+	unsigned long timeout_jiffies = timespec_to_jiffies(timeout);
+	unsigned long start_jiffies = jiffies;
+	unsigned long remaining_jiffies;
+
+	if (time_after(start_jiffies, timeout_jiffies))
+		remaining_jiffies = 0;
+	else
+		remaining_jiffies = timeout_jiffies - start_jiffies;
+
+	return remaining_jiffies;
+}
+
+#endif /* __ETNAVIV_DRV_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
new file mode 100644
index 000000000000..4a29eeadbf1e
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/devcoredump.h>
+#include "etnaviv_dump.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
+#include "state.xml.h"
+#include "state_hi.xml.h"
+
+struct core_dump_iterator {
+	void *start;
+	struct etnaviv_dump_object_header *hdr;
+	void *data;
+};
+
+static const unsigned short etnaviv_dump_registers[] = {
+	VIVS_HI_AXI_STATUS,
+	VIVS_HI_CLOCK_CONTROL,
+	VIVS_HI_IDLE_STATE,
+	VIVS_HI_AXI_CONFIG,
+	VIVS_HI_INTR_ENBL,
+	VIVS_HI_CHIP_IDENTITY,
+	VIVS_HI_CHIP_FEATURE,
+	VIVS_HI_CHIP_MODEL,
+	VIVS_HI_CHIP_REV,
+	VIVS_HI_CHIP_DATE,
+	VIVS_HI_CHIP_TIME,
+	VIVS_HI_CHIP_MINOR_FEATURE_0,
+	VIVS_HI_CACHE_CONTROL,
+	VIVS_HI_AXI_CONTROL,
+	VIVS_PM_POWER_CONTROLS,
+	VIVS_PM_MODULE_CONTROLS,
+	VIVS_PM_MODULE_STATUS,
+	VIVS_PM_PULSE_EATER,
+	VIVS_MC_MMU_FE_PAGE_TABLE,
+	VIVS_MC_MMU_TX_PAGE_TABLE,
+	VIVS_MC_MMU_PE_PAGE_TABLE,
+	VIVS_MC_MMU_PEZ_PAGE_TABLE,
+	VIVS_MC_MMU_RA_PAGE_TABLE,
+	VIVS_MC_DEBUG_MEMORY,
+	VIVS_MC_MEMORY_BASE_ADDR_RA,
+	VIVS_MC_MEMORY_BASE_ADDR_FE,
+	VIVS_MC_MEMORY_BASE_ADDR_TX,
+	VIVS_MC_MEMORY_BASE_ADDR_PEZ,
+	VIVS_MC_MEMORY_BASE_ADDR_PE,
+	VIVS_MC_MEMORY_TIMING_CONTROL,
+	VIVS_MC_BUS_CONFIG,
+	VIVS_FE_DMA_STATUS,
+	VIVS_FE_DMA_DEBUG_STATE,
+	VIVS_FE_DMA_ADDRESS,
+	VIVS_FE_DMA_LOW,
+	VIVS_FE_DMA_HIGH,
+	VIVS_FE_AUTO_FLUSH,
+};
+
+static void etnaviv_core_dump_header(struct core_dump_iterator *iter,
+	u32 type, void *data_end)
+{
+	struct etnaviv_dump_object_header *hdr = iter->hdr;
+
+	hdr->magic = cpu_to_le32(ETDUMP_MAGIC);
+	hdr->type = cpu_to_le32(type);
+	hdr->file_offset = cpu_to_le32(iter->data - iter->start);
+	hdr->file_size = cpu_to_le32(data_end - iter->data);
+
+	iter->hdr++;
+	iter->data += hdr->file_size;
+}
+
+static void etnaviv_core_dump_registers(struct core_dump_iterator *iter,
+	struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_dump_registers *reg = iter->data;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(etnaviv_dump_registers); i++, reg++) {
+		reg->reg = etnaviv_dump_registers[i];
+		reg->value = gpu_read(gpu, etnaviv_dump_registers[i]);
+	}
+
+	etnaviv_core_dump_header(iter, ETDUMP_BUF_REG, reg);
+}
+
+static void etnaviv_core_dump_mmu(struct core_dump_iterator *iter,
+	struct etnaviv_gpu *gpu, size_t mmu_size)
+{
+	etnaviv_iommu_dump(gpu->mmu, iter->data);
+
+	etnaviv_core_dump_header(iter, ETDUMP_BUF_MMU, iter->data + mmu_size);
+}
+
+static void etnaviv_core_dump_mem(struct core_dump_iterator *iter, u32 type,
+	void *ptr, size_t size, u64 iova)
+{
+	memcpy(iter->data, ptr, size);
+
+	iter->hdr->iova = cpu_to_le64(iova);
+
+	etnaviv_core_dump_header(iter, type, iter->data + size);
+}
+
+void etnaviv_core_dump(struct etnaviv_gpu *gpu)
+{
+	struct core_dump_iterator iter;
+	struct etnaviv_vram_mapping *vram;
+	struct etnaviv_gem_object *obj;
+	struct etnaviv_cmdbuf *cmd;
+	unsigned int n_obj, n_bomap_pages;
+	size_t file_size, mmu_size;
+	__le64 *bomap, *bomap_start;
+
+	mmu_size = etnaviv_iommu_dump_size(gpu->mmu);
+
+	/* We always dump registers, mmu, ring and end marker */
+	n_obj = 4;
+	n_bomap_pages = 0;
+	file_size = ARRAY_SIZE(etnaviv_dump_registers) *
+			sizeof(struct etnaviv_dump_registers) +
+		    mmu_size + gpu->buffer->size;
+
+	/* Add in the active command buffers */
+	list_for_each_entry(cmd, &gpu->active_cmd_list, node) {
+		file_size += cmd->size;
+		n_obj++;
+	}
+
+	/* Add in the active buffer objects */
+	list_for_each_entry(vram, &gpu->mmu->mappings, mmu_node) {
+		if (!vram->use)
+			continue;
+
+		obj = vram->object;
+		file_size += obj->base.size;
+		n_bomap_pages += obj->base.size >> PAGE_SHIFT;
+		n_obj++;
+	}
+
+	/* If we have any buffer objects, add a bomap object */
+	if (n_bomap_pages) {
+		file_size += n_bomap_pages * sizeof(__le64);
+		n_obj++;
+	}
+
+	/* Add the size of the headers */
+	file_size += sizeof(*iter.hdr) * n_obj;
+
+	/* Allocate the file in vmalloc memory, it's likely to be big */
+	iter.start = vmalloc(file_size);
+	if (!iter.start) {
+		dev_warn(gpu->dev, "failed to allocate devcoredump file\n");
+		return;
+	}
+
+	/* Point the data member after the headers */
+	iter.hdr = iter.start;
+	iter.data = &iter.hdr[n_obj];
+
+	memset(iter.hdr, 0, iter.data - iter.start);
+
+	etnaviv_core_dump_registers(&iter, gpu);
+	etnaviv_core_dump_mmu(&iter, gpu, mmu_size);
+	etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer->vaddr,
+			      gpu->buffer->size, gpu->buffer->paddr);
+
+	list_for_each_entry(cmd, &gpu->active_cmd_list, node)
+		etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD, cmd->vaddr,
+				      cmd->size, cmd->paddr);
+
+	/* Reserve space for the bomap */
+	if (n_bomap_pages) {
+		bomap_start = bomap = iter.data;
+		memset(bomap, 0, sizeof(*bomap) * n_bomap_pages);
+		etnaviv_core_dump_header(&iter, ETDUMP_BUF_BOMAP,
+					 bomap + n_bomap_pages);
+	} else {
+		/* Silence warning */
+		bomap_start = bomap = NULL;
+	}
+
+	list_for_each_entry(vram, &gpu->mmu->mappings, mmu_node) {
+		struct page **pages;
+		void *vaddr;
+
+		if (vram->use == 0)
+			continue;
+
+		obj = vram->object;
+
+		mutex_lock(&obj->lock);
+		pages = etnaviv_gem_get_pages(obj);
+		mutex_unlock(&obj->lock);
+		if (pages) {
+			int j;
+
+			iter.hdr->data[0] = bomap - bomap_start;
+
+			for (j = 0; j < obj->base.size >> PAGE_SHIFT; j++)
+				*bomap++ = cpu_to_le64(page_to_phys(*pages++));
+		}
+
+		iter.hdr->iova = cpu_to_le64(vram->iova);
+
+		vaddr = etnaviv_gem_vmap(&obj->base);
+		if (vaddr)
+			memcpy(iter.data, vaddr, obj->base.size);
+
+		etnaviv_core_dump_header(&iter, ETDUMP_BUF_BO, iter.data +
+					 obj->base.size);
+	}
+
+	etnaviv_core_dump_header(&iter, ETDUMP_BUF_END, iter.data);
+
+	dev_coredumpv(gpu->dev, iter.start, iter.data - iter.start, GFP_KERNEL);
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.h b/drivers/gpu/drm/etnaviv/etnaviv_dump.h
new file mode 100644
index 000000000000..97f2f8db9133
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Etnaviv devcoredump file definitions
+ */
+#ifndef ETNAVIV_DUMP_H
+#define ETNAVIV_DUMP_H
+
+#include <linux/types.h>
+
+enum {
+	ETDUMP_MAGIC = 0x414e5445,
+	ETDUMP_BUF_REG = 0,
+	ETDUMP_BUF_MMU,
+	ETDUMP_BUF_RING,
+	ETDUMP_BUF_CMD,
+	ETDUMP_BUF_BOMAP,
+	ETDUMP_BUF_BO,
+	ETDUMP_BUF_END,
+};
+
+struct etnaviv_dump_object_header {
+	__le32 magic;
+	__le32 type;
+	__le32 file_offset;
+	__le32 file_size;
+	__le64 iova;
+	__le32 data[2];
+};
+
+/* Registers object, an array of these */
+struct etnaviv_dump_registers {
+	__le32 reg;
+	__le32 value;
+};
+
+#ifdef __KERNEL__
+struct etnaviv_gpu;
+void etnaviv_core_dump(struct etnaviv_gpu *gpu);
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
new file mode 100644
index 000000000000..4b519e4309b2
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -0,0 +1,915 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/spinlock.h>
+#include <linux/shmem_fs.h>
+
+#include "etnaviv_drv.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
+
+static void etnaviv_gem_scatter_map(struct etnaviv_gem_object *etnaviv_obj)
+{
+	struct drm_device *dev = etnaviv_obj->base.dev;
+	struct sg_table *sgt = etnaviv_obj->sgt;
+
+	/*
+	 * For non-cached buffers, ensure the new pages are clean
+	 * because display controller, GPU, etc. are not coherent.
+	 */
+	if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK)
+		dma_map_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL);
+}
+
+static void etnaviv_gem_scatterlist_unmap(struct etnaviv_gem_object *etnaviv_obj)
+{
+	struct drm_device *dev = etnaviv_obj->base.dev;
+	struct sg_table *sgt = etnaviv_obj->sgt;
+
+	/*
+	 * For non-cached buffers, ensure the new pages are clean
+	 * because display controller, GPU, etc. are not coherent:
+	 *
+	 * WARNING: The DMA API does not support concurrent CPU
+	 * and device access to the memory area.  With BIDIRECTIONAL,
+	 * we will clean the cache lines which overlap the region,
+	 * and invalidate all cache lines (partially) contained in
+	 * the region.
+	 *
+	 * If you have dirty data in the overlapping cache lines,
+	 * that will corrupt the GPU-written data.  If you have
+	 * written into the remainder of the region, this can
+	 * discard those writes.
+	 */
+	if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK)
+		dma_unmap_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL);
+}
+
+/* called with etnaviv_obj->lock held */
+static int etnaviv_gem_shmem_get_pages(struct etnaviv_gem_object *etnaviv_obj)
+{
+	struct drm_device *dev = etnaviv_obj->base.dev;
+	struct page **p = drm_gem_get_pages(&etnaviv_obj->base);
+
+	if (IS_ERR(p)) {
+		dev_err(dev->dev, "could not get pages: %ld\n", PTR_ERR(p));
+		return PTR_ERR(p);
+	}
+
+	etnaviv_obj->pages = p;
+
+	return 0;
+}
+
+static void put_pages(struct etnaviv_gem_object *etnaviv_obj)
+{
+	if (etnaviv_obj->sgt) {
+		etnaviv_gem_scatterlist_unmap(etnaviv_obj);
+		sg_free_table(etnaviv_obj->sgt);
+		kfree(etnaviv_obj->sgt);
+		etnaviv_obj->sgt = NULL;
+	}
+	if (etnaviv_obj->pages) {
+		drm_gem_put_pages(&etnaviv_obj->base, etnaviv_obj->pages,
+				  true, false);
+
+		etnaviv_obj->pages = NULL;
+	}
+}
+
+struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *etnaviv_obj)
+{
+	int ret;
+
+	lockdep_assert_held(&etnaviv_obj->lock);
+
+	if (!etnaviv_obj->pages) {
+		ret = etnaviv_obj->ops->get_pages(etnaviv_obj);
+		if (ret < 0)
+			return ERR_PTR(ret);
+	}
+
+	if (!etnaviv_obj->sgt) {
+		struct drm_device *dev = etnaviv_obj->base.dev;
+		int npages = etnaviv_obj->base.size >> PAGE_SHIFT;
+		struct sg_table *sgt;
+
+		sgt = drm_prime_pages_to_sg(etnaviv_obj->pages, npages);
+		if (IS_ERR(sgt)) {
+			dev_err(dev->dev, "failed to allocate sgt: %ld\n",
+				PTR_ERR(sgt));
+			return ERR_CAST(sgt);
+		}
+
+		etnaviv_obj->sgt = sgt;
+
+		etnaviv_gem_scatter_map(etnaviv_obj);
+	}
+
+	return etnaviv_obj->pages;
+}
+
+void etnaviv_gem_put_pages(struct etnaviv_gem_object *etnaviv_obj)
+{
+	lockdep_assert_held(&etnaviv_obj->lock);
+	/* when we start tracking the pin count, then do something here */
+}
+
+static int etnaviv_gem_mmap_obj(struct drm_gem_object *obj,
+		struct vm_area_struct *vma)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	pgprot_t vm_page_prot;
+
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_flags |= VM_MIXEDMAP;
+
+	vm_page_prot = vm_get_page_prot(vma->vm_flags);
+
+	if (etnaviv_obj->flags & ETNA_BO_WC) {
+		vma->vm_page_prot = pgprot_writecombine(vm_page_prot);
+	} else if (etnaviv_obj->flags & ETNA_BO_UNCACHED) {
+		vma->vm_page_prot = pgprot_noncached(vm_page_prot);
+	} else {
+		/*
+		 * Shunt off cached objs to shmem file so they have their own
+		 * address_space (so unmap_mapping_range does what we want,
+		 * in particular in the case of mmap'd dmabufs)
+		 */
+		fput(vma->vm_file);
+		get_file(obj->filp);
+		vma->vm_pgoff = 0;
+		vma->vm_file  = obj->filp;
+
+		vma->vm_page_prot = vm_page_prot;
+	}
+
+	return 0;
+}
+
+int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct etnaviv_gem_object *obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret) {
+		DBG("mmap failed: %d", ret);
+		return ret;
+	}
+
+	obj = to_etnaviv_bo(vma->vm_private_data);
+	return etnaviv_gem_mmap_obj(vma->vm_private_data, vma);
+}
+
+int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct page **pages, *page;
+	pgoff_t pgoff;
+	int ret;
+
+	/*
+	 * Make sure we don't parallel update on a fault, nor move or remove
+	 * something from beneath our feet.  Note that vm_insert_page() is
+	 * specifically coded to take care of this, so we don't have to.
+	 */
+	ret = mutex_lock_interruptible(&etnaviv_obj->lock);
+	if (ret)
+		goto out;
+
+	/* make sure we have pages attached now */
+	pages = etnaviv_gem_get_pages(etnaviv_obj);
+	mutex_unlock(&etnaviv_obj->lock);
+
+	if (IS_ERR(pages)) {
+		ret = PTR_ERR(pages);
+		goto out;
+	}
+
+	/* We don't use vmf->pgoff since that has the fake offset: */
+	pgoff = ((unsigned long)vmf->virtual_address -
+			vma->vm_start) >> PAGE_SHIFT;
+
+	page = pages[pgoff];
+
+	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
+	     page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT);
+
+	ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page);
+
+out:
+	switch (ret) {
+	case -EAGAIN:
+	case 0:
+	case -ERESTARTSYS:
+	case -EINTR:
+	case -EBUSY:
+		/*
+		 * EBUSY is ok: this just means that another thread
+		 * already did the job.
+		 */
+		return VM_FAULT_NOPAGE;
+	case -ENOMEM:
+		return VM_FAULT_OOM;
+	default:
+		return VM_FAULT_SIGBUS;
+	}
+}
+
+int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset)
+{
+	int ret;
+
+	/* Make it mmapable */
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret)
+		dev_err(obj->dev->dev, "could not allocate mmap offset\n");
+	else
+		*offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+	return ret;
+}
+
+static struct etnaviv_vram_mapping *
+etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj,
+			     struct etnaviv_iommu *mmu)
+{
+	struct etnaviv_vram_mapping *mapping;
+
+	list_for_each_entry(mapping, &obj->vram_list, obj_node) {
+		if (mapping->mmu == mmu)
+			return mapping;
+	}
+
+	return NULL;
+}
+
+int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu,
+	struct drm_gem_object *obj, u32 *iova)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct etnaviv_vram_mapping *mapping;
+	struct page **pages;
+	int ret = 0;
+
+	mutex_lock(&etnaviv_obj->lock);
+	mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu);
+	if (mapping) {
+		/*
+		 * Holding the object lock prevents the use count changing
+		 * beneath us.  If the use count is zero, the MMU might be
+		 * reaping this object, so take the lock and re-check that
+		 * the MMU owns this mapping to close this race.
+		 */
+		if (mapping->use == 0) {
+			mutex_lock(&gpu->mmu->lock);
+			if (mapping->mmu == gpu->mmu)
+				mapping->use += 1;
+			else
+				mapping = NULL;
+			mutex_unlock(&gpu->mmu->lock);
+			if (mapping)
+				goto out;
+		} else {
+			mapping->use += 1;
+			goto out;
+		}
+	}
+
+	pages = etnaviv_gem_get_pages(etnaviv_obj);
+	if (IS_ERR(pages)) {
+		ret = PTR_ERR(pages);
+		goto out;
+	}
+
+	/*
+	 * See if we have a reaped vram mapping we can re-use before
+	 * allocating a fresh mapping.
+	 */
+	mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, NULL);
+	if (!mapping) {
+		mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
+		if (!mapping) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		INIT_LIST_HEAD(&mapping->scan_node);
+		mapping->object = etnaviv_obj;
+	} else {
+		list_del(&mapping->obj_node);
+	}
+
+	mapping->mmu = gpu->mmu;
+	mapping->use = 1;
+
+	ret = etnaviv_iommu_map_gem(gpu->mmu, etnaviv_obj, gpu->memory_base,
+				    mapping);
+	if (ret < 0)
+		kfree(mapping);
+	else
+		list_add_tail(&mapping->obj_node, &etnaviv_obj->vram_list);
+
+out:
+	mutex_unlock(&etnaviv_obj->lock);
+
+	if (!ret) {
+		/* Take a reference on the object */
+		drm_gem_object_reference(obj);
+		*iova = mapping->iova;
+	}
+
+	return ret;
+}
+
+void etnaviv_gem_put_iova(struct etnaviv_gpu *gpu, struct drm_gem_object *obj)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct etnaviv_vram_mapping *mapping;
+
+	mutex_lock(&etnaviv_obj->lock);
+	mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu);
+
+	WARN_ON(mapping->use == 0);
+	mapping->use -= 1;
+	mutex_unlock(&etnaviv_obj->lock);
+
+	drm_gem_object_unreference_unlocked(obj);
+}
+
+void *etnaviv_gem_vmap(struct drm_gem_object *obj)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+	if (etnaviv_obj->vaddr)
+		return etnaviv_obj->vaddr;
+
+	mutex_lock(&etnaviv_obj->lock);
+	/*
+	 * Need to check again, as we might have raced with another thread
+	 * while waiting for the mutex.
+	 */
+	if (!etnaviv_obj->vaddr)
+		etnaviv_obj->vaddr = etnaviv_obj->ops->vmap(etnaviv_obj);
+	mutex_unlock(&etnaviv_obj->lock);
+
+	return etnaviv_obj->vaddr;
+}
+
+static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
+{
+	struct page **pages;
+
+	lockdep_assert_held(&obj->lock);
+
+	pages = etnaviv_gem_get_pages(obj);
+	if (IS_ERR(pages))
+		return NULL;
+
+	return vmap(pages, obj->base.size >> PAGE_SHIFT,
+			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+}
+
+static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
+{
+	if (op & ETNA_PREP_READ)
+		return DMA_FROM_DEVICE;
+	else if (op & ETNA_PREP_WRITE)
+		return DMA_TO_DEVICE;
+	else
+		return DMA_BIDIRECTIONAL;
+}
+
+int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
+		struct timespec *timeout)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct drm_device *dev = obj->dev;
+	bool write = !!(op & ETNA_PREP_WRITE);
+	int ret;
+
+	if (op & ETNA_PREP_NOSYNC) {
+		if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv,
+							  write))
+			return -EBUSY;
+	} else {
+		unsigned long remain = etnaviv_timeout_to_jiffies(timeout);
+
+		ret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv,
+							  write, true, remain);
+		if (ret <= 0)
+			return ret == 0 ? -ETIMEDOUT : ret;
+	}
+
+	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+		if (!etnaviv_obj->sgt) {
+			void *ret;
+
+			mutex_lock(&etnaviv_obj->lock);
+			ret = etnaviv_gem_get_pages(etnaviv_obj);
+			mutex_unlock(&etnaviv_obj->lock);
+			if (IS_ERR(ret))
+				return PTR_ERR(ret);
+		}
+
+		dma_sync_sg_for_cpu(dev->dev, etnaviv_obj->sgt->sgl,
+				    etnaviv_obj->sgt->nents,
+				    etnaviv_op_to_dma_dir(op));
+		etnaviv_obj->last_cpu_prep_op = op;
+	}
+
+	return 0;
+}
+
+int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+		/* fini without a prep is almost certainly a userspace error */
+		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
+		dma_sync_sg_for_device(dev->dev, etnaviv_obj->sgt->sgl,
+			etnaviv_obj->sgt->nents,
+			etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op));
+		etnaviv_obj->last_cpu_prep_op = 0;
+	}
+
+	return 0;
+}
+
+int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj,
+	struct timespec *timeout)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+	return etnaviv_gpu_wait_obj_inactive(gpu, etnaviv_obj, timeout);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static void etnaviv_gem_describe_fence(struct fence *fence,
+	const char *type, struct seq_file *m)
+{
+	if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		seq_printf(m, "\t%9s: %s %s seq %u\n",
+			   type,
+			   fence->ops->get_driver_name(fence),
+			   fence->ops->get_timeline_name(fence),
+			   fence->seqno);
+}
+
+static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct reservation_object *robj = etnaviv_obj->resv;
+	struct reservation_object_list *fobj;
+	struct fence *fence;
+	unsigned long off = drm_vma_node_start(&obj->vma_node);
+
+	seq_printf(m, "%08x: %c %2d (%2d) %08lx %p %zd\n",
+			etnaviv_obj->flags, is_active(etnaviv_obj) ? 'A' : 'I',
+			obj->name, obj->refcount.refcount.counter,
+			off, etnaviv_obj->vaddr, obj->size);
+
+	rcu_read_lock();
+	fobj = rcu_dereference(robj->fence);
+	if (fobj) {
+		unsigned int i, shared_count = fobj->shared_count;
+
+		for (i = 0; i < shared_count; i++) {
+			fence = rcu_dereference(fobj->shared[i]);
+			etnaviv_gem_describe_fence(fence, "Shared", m);
+		}
+	}
+
+	fence = rcu_dereference(robj->fence_excl);
+	if (fence)
+		etnaviv_gem_describe_fence(fence, "Exclusive", m);
+	rcu_read_unlock();
+}
+
+void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
+	struct seq_file *m)
+{
+	struct etnaviv_gem_object *etnaviv_obj;
+	int count = 0;
+	size_t size = 0;
+
+	mutex_lock(&priv->gem_lock);
+	list_for_each_entry(etnaviv_obj, &priv->gem_list, gem_node) {
+		struct drm_gem_object *obj = &etnaviv_obj->base;
+
+		seq_puts(m, "   ");
+		etnaviv_gem_describe(obj, m);
+		count++;
+		size += obj->size;
+	}
+	mutex_unlock(&priv->gem_lock);
+
+	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
+}
+#endif
+
+static void etnaviv_gem_shmem_release(struct etnaviv_gem_object *etnaviv_obj)
+{
+	if (etnaviv_obj->vaddr)
+		vunmap(etnaviv_obj->vaddr);
+	put_pages(etnaviv_obj);
+}
+
+static const struct etnaviv_gem_ops etnaviv_gem_shmem_ops = {
+	.get_pages = etnaviv_gem_shmem_get_pages,
+	.release = etnaviv_gem_shmem_release,
+	.vmap = etnaviv_gem_vmap_impl,
+};
+
+void etnaviv_gem_free_object(struct drm_gem_object *obj)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct etnaviv_vram_mapping *mapping, *tmp;
+
+	/* object should not be active */
+	WARN_ON(is_active(etnaviv_obj));
+
+	list_del(&etnaviv_obj->gem_node);
+
+	list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list,
+				 obj_node) {
+		struct etnaviv_iommu *mmu = mapping->mmu;
+
+		WARN_ON(mapping->use);
+
+		if (mmu)
+			etnaviv_iommu_unmap_gem(mmu, mapping);
+
+		list_del(&mapping->obj_node);
+		kfree(mapping);
+	}
+
+	drm_gem_free_mmap_offset(obj);
+	etnaviv_obj->ops->release(etnaviv_obj);
+	if (etnaviv_obj->resv == &etnaviv_obj->_resv)
+		reservation_object_fini(&etnaviv_obj->_resv);
+	drm_gem_object_release(obj);
+
+	kfree(etnaviv_obj);
+}
+
+int etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+	mutex_lock(&priv->gem_lock);
+	list_add_tail(&etnaviv_obj->gem_node, &priv->gem_list);
+	mutex_unlock(&priv->gem_lock);
+
+	return 0;
+}
+
+static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags,
+	struct reservation_object *robj, const struct etnaviv_gem_ops *ops,
+	struct drm_gem_object **obj)
+{
+	struct etnaviv_gem_object *etnaviv_obj;
+	unsigned sz = sizeof(*etnaviv_obj);
+	bool valid = true;
+
+	/* validate flags */
+	switch (flags & ETNA_BO_CACHE_MASK) {
+	case ETNA_BO_UNCACHED:
+	case ETNA_BO_CACHED:
+	case ETNA_BO_WC:
+		break;
+	default:
+		valid = false;
+	}
+
+	if (!valid) {
+		dev_err(dev->dev, "invalid cache flag: %x\n",
+			(flags & ETNA_BO_CACHE_MASK));
+		return -EINVAL;
+	}
+
+	etnaviv_obj = kzalloc(sz, GFP_KERNEL);
+	if (!etnaviv_obj)
+		return -ENOMEM;
+
+	etnaviv_obj->flags = flags;
+	etnaviv_obj->ops = ops;
+	if (robj) {
+		etnaviv_obj->resv = robj;
+	} else {
+		etnaviv_obj->resv = &etnaviv_obj->_resv;
+		reservation_object_init(&etnaviv_obj->_resv);
+	}
+
+	mutex_init(&etnaviv_obj->lock);
+	INIT_LIST_HEAD(&etnaviv_obj->vram_list);
+
+	*obj = &etnaviv_obj->base;
+
+	return 0;
+}
+
+static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
+		u32 size, u32 flags)
+{
+	struct drm_gem_object *obj = NULL;
+	int ret;
+
+	size = PAGE_ALIGN(size);
+
+	ret = etnaviv_gem_new_impl(dev, size, flags, NULL,
+				   &etnaviv_gem_shmem_ops, &obj);
+	if (ret)
+		goto fail;
+
+	ret = drm_gem_object_init(dev, obj, size);
+	if (ret == 0) {
+		struct address_space *mapping;
+
+		/*
+		 * Our buffers are kept pinned, so allocating them
+		 * from the MOVABLE zone is a really bad idea, and
+		 * conflicts with CMA.  See coments above new_inode()
+		 * why this is required _and_ expected if you're
+		 * going to pin these pages.
+		 */
+		mapping = file_inode(obj->filp)->i_mapping;
+		mapping_set_gfp_mask(mapping, GFP_HIGHUSER);
+	}
+
+	if (ret)
+		goto fail;
+
+	return obj;
+
+fail:
+	if (obj)
+		drm_gem_object_unreference_unlocked(obj);
+
+	return ERR_PTR(ret);
+}
+
+/* convenience method to construct a GEM buffer object, and userspace handle */
+int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
+		u32 size, u32 flags, u32 *handle)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	obj = __etnaviv_gem_new(dev, size, flags);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	ret = etnaviv_gem_obj_add(dev, obj);
+	if (ret < 0) {
+		drm_gem_object_unreference_unlocked(obj);
+		return ret;
+	}
+
+	ret = drm_gem_handle_create(file, obj, handle);
+
+	/* drop reference from allocate - handle holds it now */
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
+}
+
+struct drm_gem_object *etnaviv_gem_new(struct drm_device *dev,
+		u32 size, u32 flags)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	obj = __etnaviv_gem_new(dev, size, flags);
+	if (IS_ERR(obj))
+		return obj;
+
+	ret = etnaviv_gem_obj_add(dev, obj);
+	if (ret < 0) {
+		drm_gem_object_unreference_unlocked(obj);
+		return ERR_PTR(ret);
+	}
+
+	return obj;
+}
+
+int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags,
+	struct reservation_object *robj, const struct etnaviv_gem_ops *ops,
+	struct etnaviv_gem_object **res)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	ret = etnaviv_gem_new_impl(dev, size, flags, robj, ops, &obj);
+	if (ret)
+		return ret;
+
+	drm_gem_private_object_init(dev, obj, size);
+
+	*res = to_etnaviv_bo(obj);
+
+	return 0;
+}
+
+struct get_pages_work {
+	struct work_struct work;
+	struct mm_struct *mm;
+	struct task_struct *task;
+	struct etnaviv_gem_object *etnaviv_obj;
+};
+
+static struct page **etnaviv_gem_userptr_do_get_pages(
+	struct etnaviv_gem_object *etnaviv_obj, struct mm_struct *mm, struct task_struct *task)
+{
+	int ret = 0, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
+	struct page **pvec;
+	uintptr_t ptr;
+
+	pvec = drm_malloc_ab(npages, sizeof(struct page *));
+	if (!pvec)
+		return ERR_PTR(-ENOMEM);
+
+	pinned = 0;
+	ptr = etnaviv_obj->userptr.ptr;
+
+	down_read(&mm->mmap_sem);
+	while (pinned < npages) {
+		ret = get_user_pages(task, mm, ptr, npages - pinned,
+				     !etnaviv_obj->userptr.ro, 0,
+				     pvec + pinned, NULL);
+		if (ret < 0)
+			break;
+
+		ptr += ret * PAGE_SIZE;
+		pinned += ret;
+	}
+	up_read(&mm->mmap_sem);
+
+	if (ret < 0) {
+		release_pages(pvec, pinned, 0);
+		drm_free_large(pvec);
+		return ERR_PTR(ret);
+	}
+
+	return pvec;
+}
+
+static void __etnaviv_gem_userptr_get_pages(struct work_struct *_work)
+{
+	struct get_pages_work *work = container_of(_work, typeof(*work), work);
+	struct etnaviv_gem_object *etnaviv_obj = work->etnaviv_obj;
+	struct page **pvec;
+
+	pvec = etnaviv_gem_userptr_do_get_pages(etnaviv_obj, work->mm, work->task);
+
+	mutex_lock(&etnaviv_obj->lock);
+	if (IS_ERR(pvec)) {
+		etnaviv_obj->userptr.work = ERR_CAST(pvec);
+	} else {
+		etnaviv_obj->userptr.work = NULL;
+		etnaviv_obj->pages = pvec;
+	}
+
+	mutex_unlock(&etnaviv_obj->lock);
+	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+
+	mmput(work->mm);
+	put_task_struct(work->task);
+	kfree(work);
+}
+
+static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj)
+{
+	struct page **pvec = NULL;
+	struct get_pages_work *work;
+	struct mm_struct *mm;
+	int ret, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
+
+	if (etnaviv_obj->userptr.work) {
+		if (IS_ERR(etnaviv_obj->userptr.work)) {
+			ret = PTR_ERR(etnaviv_obj->userptr.work);
+			etnaviv_obj->userptr.work = NULL;
+		} else {
+			ret = -EAGAIN;
+		}
+		return ret;
+	}
+
+	mm = get_task_mm(etnaviv_obj->userptr.task);
+	pinned = 0;
+	if (mm == current->mm) {
+		pvec = drm_malloc_ab(npages, sizeof(struct page *));
+		if (!pvec) {
+			mmput(mm);
+			return -ENOMEM;
+		}
+
+		pinned = __get_user_pages_fast(etnaviv_obj->userptr.ptr, npages,
+					       !etnaviv_obj->userptr.ro, pvec);
+		if (pinned < 0) {
+			drm_free_large(pvec);
+			mmput(mm);
+			return pinned;
+		}
+
+		if (pinned == npages) {
+			etnaviv_obj->pages = pvec;
+			mmput(mm);
+			return 0;
+		}
+	}
+
+	release_pages(pvec, pinned, 0);
+	drm_free_large(pvec);
+
+	work = kmalloc(sizeof(*work), GFP_KERNEL);
+	if (!work) {
+		mmput(mm);
+		return -ENOMEM;
+	}
+
+	get_task_struct(current);
+	drm_gem_object_reference(&etnaviv_obj->base);
+
+	work->mm = mm;
+	work->task = current;
+	work->etnaviv_obj = etnaviv_obj;
+
+	etnaviv_obj->userptr.work = &work->work;
+	INIT_WORK(&work->work, __etnaviv_gem_userptr_get_pages);
+
+	etnaviv_queue_work(etnaviv_obj->base.dev, &work->work);
+
+	return -EAGAIN;
+}
+
+static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj)
+{
+	if (etnaviv_obj->sgt) {
+		etnaviv_gem_scatterlist_unmap(etnaviv_obj);
+		sg_free_table(etnaviv_obj->sgt);
+		kfree(etnaviv_obj->sgt);
+	}
+	if (etnaviv_obj->pages) {
+		int npages = etnaviv_obj->base.size >> PAGE_SHIFT;
+
+		release_pages(etnaviv_obj->pages, npages, 0);
+		drm_free_large(etnaviv_obj->pages);
+	}
+	put_task_struct(etnaviv_obj->userptr.task);
+}
+
+static const struct etnaviv_gem_ops etnaviv_gem_userptr_ops = {
+	.get_pages = etnaviv_gem_userptr_get_pages,
+	.release = etnaviv_gem_userptr_release,
+	.vmap = etnaviv_gem_vmap_impl,
+};
+
+int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
+	uintptr_t ptr, u32 size, u32 flags, u32 *handle)
+{
+	struct etnaviv_gem_object *etnaviv_obj;
+	int ret;
+
+	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, NULL,
+				      &etnaviv_gem_userptr_ops, &etnaviv_obj);
+	if (ret)
+		return ret;
+
+	etnaviv_obj->userptr.ptr = ptr;
+	etnaviv_obj->userptr.task = current;
+	etnaviv_obj->userptr.ro = !(flags & ETNA_USERPTR_WRITE);
+	get_task_struct(current);
+
+	ret = etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
+	if (ret) {
+		drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+		return ret;
+	}
+
+	ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle);
+
+	/* drop reference from allocate - handle holds it now */
+	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
new file mode 100644
index 000000000000..ab5df8147a5f
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_GEM_H__
+#define __ETNAVIV_GEM_H__
+
+#include <linux/reservation.h>
+#include "etnaviv_drv.h"
+
+struct etnaviv_gem_ops;
+struct etnaviv_gem_object;
+
+struct etnaviv_gem_userptr {
+	uintptr_t ptr;
+	struct task_struct *task;
+	struct work_struct *work;
+	bool ro;
+};
+
+struct etnaviv_vram_mapping {
+	struct list_head obj_node;
+	struct list_head scan_node;
+	struct list_head mmu_node;
+	struct etnaviv_gem_object *object;
+	struct etnaviv_iommu *mmu;
+	struct drm_mm_node vram_node;
+	unsigned int use;
+	u32 iova;
+};
+
+struct etnaviv_gem_object {
+	struct drm_gem_object base;
+	const struct etnaviv_gem_ops *ops;
+	struct mutex lock;
+
+	u32 flags;
+
+	struct list_head gem_node;
+	struct etnaviv_gpu *gpu;     /* non-null if active */
+	atomic_t gpu_active;
+	u32 access;
+
+	struct page **pages;
+	struct sg_table *sgt;
+	void *vaddr;
+
+	/* normally (resv == &_resv) except for imported bo's */
+	struct reservation_object *resv;
+	struct reservation_object _resv;
+
+	struct list_head vram_list;
+
+	/* cache maintenance */
+	u32 last_cpu_prep_op;
+
+	struct etnaviv_gem_userptr userptr;
+};
+
+static inline
+struct etnaviv_gem_object *to_etnaviv_bo(struct drm_gem_object *obj)
+{
+	return container_of(obj, struct etnaviv_gem_object, base);
+}
+
+struct etnaviv_gem_ops {
+	int (*get_pages)(struct etnaviv_gem_object *);
+	void (*release)(struct etnaviv_gem_object *);
+	void *(*vmap)(struct etnaviv_gem_object *);
+};
+
+static inline bool is_active(struct etnaviv_gem_object *etnaviv_obj)
+{
+	return atomic_read(&etnaviv_obj->gpu_active) != 0;
+}
+
+#define MAX_CMDS 4
+
+/* Created per submit-ioctl, to track bo's and cmdstream bufs, etc,
+ * associated with the cmdstream submission for synchronization (and
+ * make it easier to unwind when things go wrong, etc).  This only
+ * lasts for the duration of the submit-ioctl.
+ */
+struct etnaviv_gem_submit {
+	struct drm_device *dev;
+	struct etnaviv_gpu *gpu;
+	struct ww_acquire_ctx ticket;
+	u32 fence;
+	unsigned int nr_bos;
+	struct {
+		u32 flags;
+		struct etnaviv_gem_object *obj;
+		u32 iova;
+	} bos[0];
+};
+
+int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj,
+	struct timespec *timeout);
+int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags,
+	struct reservation_object *robj, const struct etnaviv_gem_ops *ops,
+	struct etnaviv_gem_object **res);
+int etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj);
+struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *obj);
+void etnaviv_gem_put_pages(struct etnaviv_gem_object *obj);
+
+#endif /* __ETNAVIV_GEM_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
new file mode 100644
index 000000000000..4e67395f5fa1
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/dma-buf.h>
+#include "etnaviv_drv.h"
+#include "etnaviv_gem.h"
+
+
+struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+	BUG_ON(!etnaviv_obj->sgt);  /* should have already pinned! */
+
+	return etnaviv_obj->sgt;
+}
+
+void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj)
+{
+	return etnaviv_gem_vmap(obj);
+}
+
+void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+	/* TODO msm_gem_vunmap() */
+}
+
+int etnaviv_gem_prime_pin(struct drm_gem_object *obj)
+{
+	if (!obj->import_attach) {
+		struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+		mutex_lock(&etnaviv_obj->lock);
+		etnaviv_gem_get_pages(etnaviv_obj);
+		mutex_unlock(&etnaviv_obj->lock);
+	}
+	return 0;
+}
+
+void etnaviv_gem_prime_unpin(struct drm_gem_object *obj)
+{
+	if (!obj->import_attach) {
+		struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+
+		mutex_lock(&etnaviv_obj->lock);
+		etnaviv_gem_put_pages(to_etnaviv_bo(obj));
+		mutex_unlock(&etnaviv_obj->lock);
+	}
+}
+
+static void etnaviv_gem_prime_release(struct etnaviv_gem_object *etnaviv_obj)
+{
+	if (etnaviv_obj->vaddr)
+		dma_buf_vunmap(etnaviv_obj->base.import_attach->dmabuf,
+			       etnaviv_obj->vaddr);
+
+	/* Don't drop the pages for imported dmabuf, as they are not
+	 * ours, just free the array we allocated:
+	 */
+	if (etnaviv_obj->pages)
+		drm_free_large(etnaviv_obj->pages);
+
+	drm_prime_gem_destroy(&etnaviv_obj->base, etnaviv_obj->sgt);
+}
+
+static void *etnaviv_gem_prime_vmap_impl(struct etnaviv_gem_object *etnaviv_obj)
+{
+	lockdep_assert_held(&etnaviv_obj->lock);
+
+	return dma_buf_vmap(etnaviv_obj->base.import_attach->dmabuf);
+}
+
+static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
+	/* .get_pages should never be called */
+	.release = etnaviv_gem_prime_release,
+	.vmap = etnaviv_gem_prime_vmap_impl,
+};
+
+struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
+	struct dma_buf_attachment *attach, struct sg_table *sgt)
+{
+	struct etnaviv_gem_object *etnaviv_obj;
+	size_t size = PAGE_ALIGN(attach->dmabuf->size);
+	int ret, npages;
+
+	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
+				      attach->dmabuf->resv,
+				      &etnaviv_gem_prime_ops, &etnaviv_obj);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	npages = size / PAGE_SIZE;
+
+	etnaviv_obj->sgt = sgt;
+	etnaviv_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
+	if (!etnaviv_obj->pages) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, etnaviv_obj->pages,
+					       NULL, npages);
+	if (ret)
+		goto fail;
+
+	ret = etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
+	if (ret)
+		goto fail;
+
+	return &etnaviv_obj->base;
+
+fail:
+	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
new file mode 100644
index 000000000000..1aba01a999df
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -0,0 +1,443 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/reservation.h>
+#include "etnaviv_drv.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_gem.h"
+
+/*
+ * Cmdstream submission:
+ */
+
+#define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE)
+/* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */
+#define BO_LOCKED   0x4000
+#define BO_PINNED   0x2000
+
+static inline void __user *to_user_ptr(u64 address)
+{
+	return (void __user *)(uintptr_t)address;
+}
+
+static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
+		struct etnaviv_gpu *gpu, size_t nr)
+{
+	struct etnaviv_gem_submit *submit;
+	size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit));
+
+	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
+	if (submit) {
+		submit->dev = dev;
+		submit->gpu = gpu;
+
+		/* initially, until copy_from_user() and bo lookup succeeds: */
+		submit->nr_bos = 0;
+
+		ww_acquire_init(&submit->ticket, &reservation_ww_class);
+	}
+
+	return submit;
+}
+
+static int submit_lookup_objects(struct etnaviv_gem_submit *submit,
+	struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos,
+	unsigned nr_bos)
+{
+	struct drm_etnaviv_gem_submit_bo *bo;
+	unsigned i;
+	int ret = 0;
+
+	spin_lock(&file->table_lock);
+
+	for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) {
+		struct drm_gem_object *obj;
+
+		if (bo->flags & BO_INVALID_FLAGS) {
+			DRM_ERROR("invalid flags: %x\n", bo->flags);
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+
+		submit->bos[i].flags = bo->flags;
+
+		/* normally use drm_gem_object_lookup(), but for bulk lookup
+		 * all under single table_lock just hit object_idr directly:
+		 */
+		obj = idr_find(&file->object_idr, bo->handle);
+		if (!obj) {
+			DRM_ERROR("invalid handle %u at index %u\n",
+				  bo->handle, i);
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+
+		/*
+		 * Take a refcount on the object. The file table lock
+		 * prevents the object_idr's refcount on this being dropped.
+		 */
+		drm_gem_object_reference(obj);
+
+		submit->bos[i].obj = to_etnaviv_bo(obj);
+	}
+
+out_unlock:
+	submit->nr_bos = i;
+	spin_unlock(&file->table_lock);
+
+	return ret;
+}
+
+static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
+{
+	if (submit->bos[i].flags & BO_LOCKED) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+
+		ww_mutex_unlock(&etnaviv_obj->resv->lock);
+		submit->bos[i].flags &= ~BO_LOCKED;
+	}
+}
+
+static int submit_lock_objects(struct etnaviv_gem_submit *submit)
+{
+	int contended, slow_locked = -1, i, ret = 0;
+
+retry:
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+
+		if (slow_locked == i)
+			slow_locked = -1;
+
+		contended = i;
+
+		if (!(submit->bos[i].flags & BO_LOCKED)) {
+			ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock,
+					&submit->ticket);
+			if (ret == -EALREADY)
+				DRM_ERROR("BO at index %u already on submit list\n",
+					  i);
+			if (ret)
+				goto fail;
+			submit->bos[i].flags |= BO_LOCKED;
+		}
+	}
+
+	ww_acquire_done(&submit->ticket);
+
+	return 0;
+
+fail:
+	for (; i >= 0; i--)
+		submit_unlock_object(submit, i);
+
+	if (slow_locked > 0)
+		submit_unlock_object(submit, slow_locked);
+
+	if (ret == -EDEADLK) {
+		struct etnaviv_gem_object *etnaviv_obj;
+
+		etnaviv_obj = submit->bos[contended].obj;
+
+		/* we lost out in a seqno race, lock and retry.. */
+		ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock,
+				&submit->ticket);
+		if (!ret) {
+			submit->bos[contended].flags |= BO_LOCKED;
+			slow_locked = contended;
+			goto retry;
+		}
+	}
+
+	return ret;
+}
+
+static int submit_fence_sync(const struct etnaviv_gem_submit *submit)
+{
+	unsigned int context = submit->gpu->fence_context;
+	int i, ret = 0;
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE;
+
+		ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static void submit_unpin_objects(struct etnaviv_gem_submit *submit)
+{
+	int i;
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+
+		if (submit->bos[i].flags & BO_PINNED)
+			etnaviv_gem_put_iova(submit->gpu, &etnaviv_obj->base);
+
+		submit->bos[i].iova = 0;
+		submit->bos[i].flags &= ~BO_PINNED;
+	}
+}
+
+static int submit_pin_objects(struct etnaviv_gem_submit *submit)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		u32 iova;
+
+		ret = etnaviv_gem_get_iova(submit->gpu, &etnaviv_obj->base,
+					   &iova);
+		if (ret)
+			break;
+
+		submit->bos[i].flags |= BO_PINNED;
+		submit->bos[i].iova = iova;
+	}
+
+	return ret;
+}
+
+static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx,
+		struct etnaviv_gem_object **obj, u32 *iova)
+{
+	if (idx >= submit->nr_bos) {
+		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
+				idx, submit->nr_bos);
+		return -EINVAL;
+	}
+
+	if (obj)
+		*obj = submit->bos[idx].obj;
+	if (iova)
+		*iova = submit->bos[idx].iova;
+
+	return 0;
+}
+
+/* process the reloc's and patch up the cmdstream as needed: */
+static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,
+		u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs,
+		u32 nr_relocs)
+{
+	u32 i, last_offset = 0;
+	u32 *ptr = stream;
+	int ret;
+
+	for (i = 0; i < nr_relocs; i++) {
+		const struct drm_etnaviv_gem_submit_reloc *r = relocs + i;
+		struct etnaviv_gem_object *bobj;
+		u32 iova, off;
+
+		if (unlikely(r->flags)) {
+			DRM_ERROR("invalid reloc flags\n");
+			return -EINVAL;
+		}
+
+		if (r->submit_offset % 4) {
+			DRM_ERROR("non-aligned reloc offset: %u\n",
+				  r->submit_offset);
+			return -EINVAL;
+		}
+
+		/* offset in dwords: */
+		off = r->submit_offset / 4;
+
+		if ((off >= size ) ||
+				(off < last_offset)) {
+			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
+			return -EINVAL;
+		}
+
+		ret = submit_bo(submit, r->reloc_idx, &bobj, &iova);
+		if (ret)
+			return ret;
+
+		if (r->reloc_offset >=
+		    bobj->base.size - sizeof(*ptr)) {
+			DRM_ERROR("relocation %u outside object", i);
+			return -EINVAL;
+		}
+
+		ptr[off] = iova + r->reloc_offset;
+
+		last_offset = off;
+	}
+
+	return 0;
+}
+
+static void submit_cleanup(struct etnaviv_gem_submit *submit)
+{
+	unsigned i;
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+
+		submit_unlock_object(submit, i);
+		drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+	}
+
+	ww_acquire_fini(&submit->ticket);
+	kfree(submit);
+}
+
+int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct etnaviv_drm_private *priv = dev->dev_private;
+	struct drm_etnaviv_gem_submit *args = data;
+	struct drm_etnaviv_gem_submit_reloc *relocs;
+	struct drm_etnaviv_gem_submit_bo *bos;
+	struct etnaviv_gem_submit *submit;
+	struct etnaviv_cmdbuf *cmdbuf;
+	struct etnaviv_gpu *gpu;
+	void *stream;
+	int ret;
+
+	if (args->pipe >= ETNA_MAX_PIPES)
+		return -EINVAL;
+
+	gpu = priv->gpu[args->pipe];
+	if (!gpu)
+		return -ENXIO;
+
+	if (args->stream_size % 4) {
+		DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
+			  args->stream_size);
+		return -EINVAL;
+	}
+
+	if (args->exec_state != ETNA_PIPE_3D &&
+	    args->exec_state != ETNA_PIPE_2D &&
+	    args->exec_state != ETNA_PIPE_VG) {
+		DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state);
+		return -EINVAL;
+	}
+
+	/*
+	 * Copy the command submission and bo array to kernel space in
+	 * one go, and do this outside of any locks.
+	 */
+	bos = drm_malloc_ab(args->nr_bos, sizeof(*bos));
+	relocs = drm_malloc_ab(args->nr_relocs, sizeof(*relocs));
+	stream = drm_malloc_ab(1, args->stream_size);
+	cmdbuf = etnaviv_gpu_cmdbuf_new(gpu, ALIGN(args->stream_size, 8) + 8,
+					args->nr_bos);
+	if (!bos || !relocs || !stream || !cmdbuf) {
+		ret = -ENOMEM;
+		goto err_submit_cmds;
+	}
+
+	cmdbuf->exec_state = args->exec_state;
+	cmdbuf->ctx = file->driver_priv;
+
+	ret = copy_from_user(bos, to_user_ptr(args->bos),
+			     args->nr_bos * sizeof(*bos));
+	if (ret) {
+		ret = -EFAULT;
+		goto err_submit_cmds;
+	}
+
+	ret = copy_from_user(relocs, to_user_ptr(args->relocs),
+			     args->nr_relocs * sizeof(*relocs));
+	if (ret) {
+		ret = -EFAULT;
+		goto err_submit_cmds;
+	}
+
+	ret = copy_from_user(stream, to_user_ptr(args->stream),
+			     args->stream_size);
+	if (ret) {
+		ret = -EFAULT;
+		goto err_submit_cmds;
+	}
+
+	submit = submit_create(dev, gpu, args->nr_bos);
+	if (!submit) {
+		ret = -ENOMEM;
+		goto err_submit_cmds;
+	}
+
+	ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
+	if (ret)
+		goto err_submit_objects;
+
+	ret = submit_lock_objects(submit);
+	if (ret)
+		goto err_submit_objects;
+
+	if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4,
+				      relocs, args->nr_relocs)) {
+		ret = -EINVAL;
+		goto err_submit_objects;
+	}
+
+	ret = submit_fence_sync(submit);
+	if (ret)
+		goto err_submit_objects;
+
+	ret = submit_pin_objects(submit);
+	if (ret)
+		goto out;
+
+	ret = submit_reloc(submit, stream, args->stream_size / 4,
+			   relocs, args->nr_relocs);
+	if (ret)
+		goto out;
+
+	memcpy(cmdbuf->vaddr, stream, args->stream_size);
+	cmdbuf->user_size = ALIGN(args->stream_size, 8);
+
+	ret = etnaviv_gpu_submit(gpu, submit, cmdbuf);
+	if (ret == 0)
+		cmdbuf = NULL;
+
+	args->fence = submit->fence;
+
+out:
+	submit_unpin_objects(submit);
+
+	/*
+	 * If we're returning -EAGAIN, it may be due to the userptr code
+	 * wanting to run its workqueue outside of any locks. Flush our
+	 * workqueue to ensure that it is run in a timely manner.
+	 */
+	if (ret == -EAGAIN)
+		flush_workqueue(priv->wq);
+
+err_submit_objects:
+	submit_cleanup(submit);
+
+err_submit_cmds:
+	/* if we still own the cmdbuf */
+	if (cmdbuf)
+		etnaviv_gpu_cmdbuf_free(cmdbuf);
+	if (stream)
+		drm_free_large(stream);
+	if (bos)
+		drm_free_large(bos);
+	if (relocs)
+		drm_free_large(relocs);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
new file mode 100644
index 000000000000..a33162cf4f4c
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -0,0 +1,1712 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/component.h>
+#include <linux/fence.h>
+#include <linux/moduleparam.h>
+#include <linux/of_device.h>
+#include "etnaviv_dump.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_mmu.h"
+#include "etnaviv_iommu.h"
+#include "etnaviv_iommu_v2.h"
+#include "common.xml.h"
+#include "state.xml.h"
+#include "state_hi.xml.h"
+#include "cmdstream.xml.h"
+
+static const struct platform_device_id gpu_ids[] = {
+	{ .name = "etnaviv-gpu,2d" },
+	{ },
+};
+
+static bool etnaviv_dump_core = true;
+module_param_named(dump_core, etnaviv_dump_core, bool, 0600);
+
+/*
+ * Driver functions:
+ */
+
+int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
+{
+	switch (param) {
+	case ETNAVIV_PARAM_GPU_MODEL:
+		*value = gpu->identity.model;
+		break;
+
+	case ETNAVIV_PARAM_GPU_REVISION:
+		*value = gpu->identity.revision;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_0:
+		*value = gpu->identity.features;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_1:
+		*value = gpu->identity.minor_features0;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_2:
+		*value = gpu->identity.minor_features1;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_3:
+		*value = gpu->identity.minor_features2;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_4:
+		*value = gpu->identity.minor_features3;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_5:
+		*value = gpu->identity.minor_features4;
+		break;
+
+	case ETNAVIV_PARAM_GPU_FEATURES_6:
+		*value = gpu->identity.minor_features5;
+		break;
+
+	case ETNAVIV_PARAM_GPU_STREAM_COUNT:
+		*value = gpu->identity.stream_count;
+		break;
+
+	case ETNAVIV_PARAM_GPU_REGISTER_MAX:
+		*value = gpu->identity.register_max;
+		break;
+
+	case ETNAVIV_PARAM_GPU_THREAD_COUNT:
+		*value = gpu->identity.thread_count;
+		break;
+
+	case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE:
+		*value = gpu->identity.vertex_cache_size;
+		break;
+
+	case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT:
+		*value = gpu->identity.shader_core_count;
+		break;
+
+	case ETNAVIV_PARAM_GPU_PIXEL_PIPES:
+		*value = gpu->identity.pixel_pipes;
+		break;
+
+	case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE:
+		*value = gpu->identity.vertex_output_buffer_size;
+		break;
+
+	case ETNAVIV_PARAM_GPU_BUFFER_SIZE:
+		*value = gpu->identity.buffer_size;
+		break;
+
+	case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT:
+		*value = gpu->identity.instruction_count;
+		break;
+
+	case ETNAVIV_PARAM_GPU_NUM_CONSTANTS:
+		*value = gpu->identity.num_constants;
+		break;
+
+	case ETNAVIV_PARAM_GPU_NUM_VARYINGS:
+		*value = gpu->identity.varyings_count;
+		break;
+
+	default:
+		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+#define etnaviv_is_model_rev(gpu, mod, rev) \
+	((gpu)->identity.model == chipModel_##mod && \
+	 (gpu)->identity.revision == rev)
+#define etnaviv_field(val, field) \
+	(((val) & field##__MASK) >> field##__SHIFT)
+
+static void etnaviv_hw_specs(struct etnaviv_gpu *gpu)
+{
+	if (gpu->identity.minor_features0 &
+	    chipMinorFeatures0_MORE_MINOR_FEATURES) {
+		u32 specs[4];
+		unsigned int streams;
+
+		specs[0] = gpu_read(gpu, VIVS_HI_CHIP_SPECS);
+		specs[1] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_2);
+		specs[2] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_3);
+		specs[3] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_4);
+
+		gpu->identity.stream_count = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_STREAM_COUNT);
+		gpu->identity.register_max = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_REGISTER_MAX);
+		gpu->identity.thread_count = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_THREAD_COUNT);
+		gpu->identity.vertex_cache_size = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE);
+		gpu->identity.shader_core_count = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT);
+		gpu->identity.pixel_pipes = etnaviv_field(specs[0],
+					VIVS_HI_CHIP_SPECS_PIXEL_PIPES);
+		gpu->identity.vertex_output_buffer_size =
+			etnaviv_field(specs[0],
+				VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE);
+
+		gpu->identity.buffer_size = etnaviv_field(specs[1],
+					VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE);
+		gpu->identity.instruction_count = etnaviv_field(specs[1],
+					VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT);
+		gpu->identity.num_constants = etnaviv_field(specs[1],
+					VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS);
+
+		gpu->identity.varyings_count = etnaviv_field(specs[2],
+					VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT);
+
+		/* This overrides the value from older register if non-zero */
+		streams = etnaviv_field(specs[3],
+					VIVS_HI_CHIP_SPECS_4_STREAM_COUNT);
+		if (streams)
+			gpu->identity.stream_count = streams;
+	}
+
+	/* Fill in the stream count if not specified */
+	if (gpu->identity.stream_count == 0) {
+		if (gpu->identity.model >= 0x1000)
+			gpu->identity.stream_count = 4;
+		else
+			gpu->identity.stream_count = 1;
+	}
+
+	/* Convert the register max value */
+	if (gpu->identity.register_max)
+		gpu->identity.register_max = 1 << gpu->identity.register_max;
+	else if (gpu->identity.model == chipModel_GC400)
+		gpu->identity.register_max = 32;
+	else
+		gpu->identity.register_max = 64;
+
+	/* Convert thread count */
+	if (gpu->identity.thread_count)
+		gpu->identity.thread_count = 1 << gpu->identity.thread_count;
+	else if (gpu->identity.model == chipModel_GC400)
+		gpu->identity.thread_count = 64;
+	else if (gpu->identity.model == chipModel_GC500 ||
+		 gpu->identity.model == chipModel_GC530)
+		gpu->identity.thread_count = 128;
+	else
+		gpu->identity.thread_count = 256;
+
+	if (gpu->identity.vertex_cache_size == 0)
+		gpu->identity.vertex_cache_size = 8;
+
+	if (gpu->identity.shader_core_count == 0) {
+		if (gpu->identity.model >= 0x1000)
+			gpu->identity.shader_core_count = 2;
+		else
+			gpu->identity.shader_core_count = 1;
+	}
+
+	if (gpu->identity.pixel_pipes == 0)
+		gpu->identity.pixel_pipes = 1;
+
+	/* Convert virtex buffer size */
+	if (gpu->identity.vertex_output_buffer_size) {
+		gpu->identity.vertex_output_buffer_size =
+			1 << gpu->identity.vertex_output_buffer_size;
+	} else if (gpu->identity.model == chipModel_GC400) {
+		if (gpu->identity.revision < 0x4000)
+			gpu->identity.vertex_output_buffer_size = 512;
+		else if (gpu->identity.revision < 0x4200)
+			gpu->identity.vertex_output_buffer_size = 256;
+		else
+			gpu->identity.vertex_output_buffer_size = 128;
+	} else {
+		gpu->identity.vertex_output_buffer_size = 512;
+	}
+
+	switch (gpu->identity.instruction_count) {
+	case 0:
+		if (etnaviv_is_model_rev(gpu, GC2000, 0x5108) ||
+		    gpu->identity.model == chipModel_GC880)
+			gpu->identity.instruction_count = 512;
+		else
+			gpu->identity.instruction_count = 256;
+		break;
+
+	case 1:
+		gpu->identity.instruction_count = 1024;
+		break;
+
+	case 2:
+		gpu->identity.instruction_count = 2048;
+		break;
+
+	default:
+		gpu->identity.instruction_count = 256;
+		break;
+	}
+
+	if (gpu->identity.num_constants == 0)
+		gpu->identity.num_constants = 168;
+
+	if (gpu->identity.varyings_count == 0) {
+		if (gpu->identity.minor_features1 & chipMinorFeatures1_HALTI0)
+			gpu->identity.varyings_count = 12;
+		else
+			gpu->identity.varyings_count = 8;
+	}
+
+	/*
+	 * For some cores, two varyings are consumed for position, so the
+	 * maximum varying count needs to be reduced by one.
+	 */
+	if (etnaviv_is_model_rev(gpu, GC5000, 0x5434) ||
+	    etnaviv_is_model_rev(gpu, GC4000, 0x5222) ||
+	    etnaviv_is_model_rev(gpu, GC4000, 0x5245) ||
+	    etnaviv_is_model_rev(gpu, GC4000, 0x5208) ||
+	    etnaviv_is_model_rev(gpu, GC3000, 0x5435) ||
+	    etnaviv_is_model_rev(gpu, GC2200, 0x5244) ||
+	    etnaviv_is_model_rev(gpu, GC2100, 0x5108) ||
+	    etnaviv_is_model_rev(gpu, GC2000, 0x5108) ||
+	    etnaviv_is_model_rev(gpu, GC1500, 0x5246) ||
+	    etnaviv_is_model_rev(gpu, GC880, 0x5107) ||
+	    etnaviv_is_model_rev(gpu, GC880, 0x5106))
+		gpu->identity.varyings_count -= 1;
+}
+
+static void etnaviv_hw_identify(struct etnaviv_gpu *gpu)
+{
+	u32 chipIdentity;
+
+	chipIdentity = gpu_read(gpu, VIVS_HI_CHIP_IDENTITY);
+
+	/* Special case for older graphic cores. */
+	if (etnaviv_field(chipIdentity, VIVS_HI_CHIP_IDENTITY_FAMILY) == 0x01) {
+		gpu->identity.model    = chipModel_GC500;
+		gpu->identity.revision = etnaviv_field(chipIdentity,
+					 VIVS_HI_CHIP_IDENTITY_REVISION);
+	} else {
+
+		gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL);
+		gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV);
+
+		/*
+		 * !!!! HACK ALERT !!!!
+		 * Because people change device IDs without letting software
+		 * know about it - here is the hack to make it all look the
+		 * same.  Only for GC400 family.
+		 */
+		if ((gpu->identity.model & 0xff00) == 0x0400 &&
+		    gpu->identity.model != chipModel_GC420) {
+			gpu->identity.model = gpu->identity.model & 0x0400;
+		}
+
+		/* Another special case */
+		if (etnaviv_is_model_rev(gpu, GC300, 0x2201)) {
+			u32 chipDate = gpu_read(gpu, VIVS_HI_CHIP_DATE);
+			u32 chipTime = gpu_read(gpu, VIVS_HI_CHIP_TIME);
+
+			if (chipDate == 0x20080814 && chipTime == 0x12051100) {
+				/*
+				 * This IP has an ECO; put the correct
+				 * revision in it.
+				 */
+				gpu->identity.revision = 0x1051;
+			}
+		}
+	}
+
+	dev_info(gpu->dev, "model: GC%x, revision: %x\n",
+		 gpu->identity.model, gpu->identity.revision);
+
+	gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE);
+
+	/* Disable fast clear on GC700. */
+	if (gpu->identity.model == chipModel_GC700)
+		gpu->identity.features &= ~chipFeatures_FAST_CLEAR;
+
+	if ((gpu->identity.model == chipModel_GC500 &&
+	     gpu->identity.revision < 2) ||
+	    (gpu->identity.model == chipModel_GC300 &&
+	     gpu->identity.revision < 0x2000)) {
+
+		/*
+		 * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these
+		 * registers.
+		 */
+		gpu->identity.minor_features0 = 0;
+		gpu->identity.minor_features1 = 0;
+		gpu->identity.minor_features2 = 0;
+		gpu->identity.minor_features3 = 0;
+		gpu->identity.minor_features4 = 0;
+		gpu->identity.minor_features5 = 0;
+	} else
+		gpu->identity.minor_features0 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_0);
+
+	if (gpu->identity.minor_features0 &
+	    chipMinorFeatures0_MORE_MINOR_FEATURES) {
+		gpu->identity.minor_features1 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_1);
+		gpu->identity.minor_features2 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_2);
+		gpu->identity.minor_features3 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_3);
+		gpu->identity.minor_features4 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_4);
+		gpu->identity.minor_features5 =
+				gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_5);
+	}
+
+	/* GC600 idle register reports zero bits where modules aren't present */
+	if (gpu->identity.model == chipModel_GC600) {
+		gpu->idle_mask = VIVS_HI_IDLE_STATE_TX |
+				 VIVS_HI_IDLE_STATE_RA |
+				 VIVS_HI_IDLE_STATE_SE |
+				 VIVS_HI_IDLE_STATE_PA |
+				 VIVS_HI_IDLE_STATE_SH |
+				 VIVS_HI_IDLE_STATE_PE |
+				 VIVS_HI_IDLE_STATE_DE |
+				 VIVS_HI_IDLE_STATE_FE;
+	} else {
+		gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP;
+	}
+
+	etnaviv_hw_specs(gpu);
+}
+
+static void etnaviv_gpu_load_clock(struct etnaviv_gpu *gpu, u32 clock)
+{
+	gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock |
+		  VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD);
+	gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock);
+}
+
+static int etnaviv_hw_reset(struct etnaviv_gpu *gpu)
+{
+	u32 control, idle;
+	unsigned long timeout;
+	bool failed = true;
+
+	/* TODO
+	 *
+	 * - clock gating
+	 * - puls eater
+	 * - what about VG?
+	 */
+
+	/* We hope that the GPU resets in under one second */
+	timeout = jiffies + msecs_to_jiffies(1000);
+
+	while (time_is_after_jiffies(timeout)) {
+		control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS |
+			  VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40);
+
+		/* enable clock */
+		etnaviv_gpu_load_clock(gpu, control);
+
+		/* Wait for stable clock.  Vivante's code waited for 1ms */
+		usleep_range(1000, 10000);
+
+		/* isolate the GPU. */
+		control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU;
+		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
+
+		/* set soft reset. */
+		control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET;
+		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
+
+		/* wait for reset. */
+		msleep(1);
+
+		/* reset soft reset bit. */
+		control &= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET;
+		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
+
+		/* reset GPU isolation. */
+		control &= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU;
+		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
+
+		/* read idle register. */
+		idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
+
+		/* try reseting again if FE it not idle */
+		if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) {
+			dev_dbg(gpu->dev, "FE is not idle\n");
+			continue;
+		}
+
+		/* read reset register. */
+		control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL);
+
+		/* is the GPU idle? */
+		if (((control & VIVS_HI_CLOCK_CONTROL_IDLE_3D) == 0) ||
+		    ((control & VIVS_HI_CLOCK_CONTROL_IDLE_2D) == 0)) {
+			dev_dbg(gpu->dev, "GPU is not idle\n");
+			continue;
+		}
+
+		failed = false;
+		break;
+	}
+
+	if (failed) {
+		idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
+		control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL);
+
+		dev_err(gpu->dev, "GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle\n",
+			idle & VIVS_HI_IDLE_STATE_FE ? "" : "not ",
+			control & VIVS_HI_CLOCK_CONTROL_IDLE_3D ? "" : "not ",
+			control & VIVS_HI_CLOCK_CONTROL_IDLE_2D ? "" : "not ");
+
+		return -EBUSY;
+	}
+
+	/* We rely on the GPU running, so program the clock */
+	control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS |
+		  VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40);
+
+	/* enable clock */
+	etnaviv_gpu_load_clock(gpu, control);
+
+	return 0;
+}
+
+static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
+{
+	u16 prefetch;
+
+	if ((etnaviv_is_model_rev(gpu, GC320, 0x5007) ||
+	     etnaviv_is_model_rev(gpu, GC320, 0x5220)) &&
+	    gpu_read(gpu, VIVS_HI_CHIP_TIME) != 0x2062400) {
+		u32 mc_memory_debug;
+
+		mc_memory_debug = gpu_read(gpu, VIVS_MC_DEBUG_MEMORY) & ~0xff;
+
+		if (gpu->identity.revision == 0x5007)
+			mc_memory_debug |= 0x0c;
+		else
+			mc_memory_debug |= 0x08;
+
+		gpu_write(gpu, VIVS_MC_DEBUG_MEMORY, mc_memory_debug);
+	}
+
+	/*
+	 * Update GPU AXI cache atttribute to "cacheable, no allocate".
+	 * This is necessary to prevent the iMX6 SoC locking up.
+	 */
+	gpu_write(gpu, VIVS_HI_AXI_CONFIG,
+		  VIVS_HI_AXI_CONFIG_AWCACHE(2) |
+		  VIVS_HI_AXI_CONFIG_ARCACHE(2));
+
+	/* GC2000 rev 5108 needs a special bus config */
+	if (etnaviv_is_model_rev(gpu, GC2000, 0x5108)) {
+		u32 bus_config = gpu_read(gpu, VIVS_MC_BUS_CONFIG);
+		bus_config &= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK |
+				VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK);
+		bus_config |= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1) |
+			      VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0);
+		gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config);
+	}
+
+	/* set base addresses */
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base);
+
+	/* setup the MMU page table pointers */
+	etnaviv_iommu_domain_restore(gpu, gpu->mmu->domain);
+
+	/* Start command processor */
+	prefetch = etnaviv_buffer_init(gpu);
+
+	gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U);
+	gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS,
+		  gpu->buffer->paddr - gpu->memory_base);
+	gpu_write(gpu, VIVS_FE_COMMAND_CONTROL,
+		  VIVS_FE_COMMAND_CONTROL_ENABLE |
+		  VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch));
+}
+
+int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
+{
+	int ret, i;
+	struct iommu_domain *iommu;
+	enum etnaviv_iommu_version version;
+	bool mmuv2;
+
+	ret = pm_runtime_get_sync(gpu->dev);
+	if (ret < 0)
+		return ret;
+
+	etnaviv_hw_identify(gpu);
+
+	if (gpu->identity.model == 0) {
+		dev_err(gpu->dev, "Unknown GPU model\n");
+		ret = -ENXIO;
+		goto fail;
+	}
+
+	/* Exclude VG cores with FE2.0 */
+	if (gpu->identity.features & chipFeatures_PIPE_VG &&
+	    gpu->identity.features & chipFeatures_FE20) {
+		dev_info(gpu->dev, "Ignoring GPU with VG and FE2.0\n");
+		ret = -ENXIO;
+		goto fail;
+	}
+
+	ret = etnaviv_hw_reset(gpu);
+	if (ret)
+		goto fail;
+
+	/* Setup IOMMU.. eventually we will (I think) do this once per context
+	 * and have separate page tables per context.  For now, to keep things
+	 * simple and to get something working, just use a single address space:
+	 */
+	mmuv2 = gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION;
+	dev_dbg(gpu->dev, "mmuv2: %d\n", mmuv2);
+
+	if (!mmuv2) {
+		iommu = etnaviv_iommu_domain_alloc(gpu);
+		version = ETNAVIV_IOMMU_V1;
+	} else {
+		iommu = etnaviv_iommu_v2_domain_alloc(gpu);
+		version = ETNAVIV_IOMMU_V2;
+	}
+
+	if (!iommu) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	gpu->mmu = etnaviv_iommu_new(gpu, iommu, version);
+	if (!gpu->mmu) {
+		iommu_domain_free(iommu);
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	/* Create buffer: */
+	gpu->buffer = etnaviv_gpu_cmdbuf_new(gpu, PAGE_SIZE, 0);
+	if (!gpu->buffer) {
+		ret = -ENOMEM;
+		dev_err(gpu->dev, "could not create command buffer\n");
+		goto destroy_iommu;
+	}
+	if (gpu->buffer->paddr - gpu->memory_base > 0x80000000) {
+		ret = -EINVAL;
+		dev_err(gpu->dev,
+			"command buffer outside valid memory window\n");
+		goto free_buffer;
+	}
+
+	/* Setup event management */
+	spin_lock_init(&gpu->event_spinlock);
+	init_completion(&gpu->event_free);
+	for (i = 0; i < ARRAY_SIZE(gpu->event); i++) {
+		gpu->event[i].used = false;
+		complete(&gpu->event_free);
+	}
+
+	/* Now program the hardware */
+	mutex_lock(&gpu->lock);
+	etnaviv_gpu_hw_init(gpu);
+	mutex_unlock(&gpu->lock);
+
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+
+	return 0;
+
+free_buffer:
+	etnaviv_gpu_cmdbuf_free(gpu->buffer);
+	gpu->buffer = NULL;
+destroy_iommu:
+	etnaviv_iommu_destroy(gpu->mmu);
+	gpu->mmu = NULL;
+fail:
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+
+	return ret;
+}
+
+#ifdef CONFIG_DEBUG_FS
+struct dma_debug {
+	u32 address[2];
+	u32 state[2];
+};
+
+static void verify_dma(struct etnaviv_gpu *gpu, struct dma_debug *debug)
+{
+	u32 i;
+
+	debug->address[0] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
+	debug->state[0]   = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE);
+
+	for (i = 0; i < 500; i++) {
+		debug->address[1] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
+		debug->state[1]   = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE);
+
+		if (debug->address[0] != debug->address[1])
+			break;
+
+		if (debug->state[0] != debug->state[1])
+			break;
+	}
+}
+
+int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m)
+{
+	struct dma_debug debug;
+	u32 dma_lo, dma_hi, axi, idle;
+	int ret;
+
+	seq_printf(m, "%s Status:\n", dev_name(gpu->dev));
+
+	ret = pm_runtime_get_sync(gpu->dev);
+	if (ret < 0)
+		return ret;
+
+	dma_lo = gpu_read(gpu, VIVS_FE_DMA_LOW);
+	dma_hi = gpu_read(gpu, VIVS_FE_DMA_HIGH);
+	axi = gpu_read(gpu, VIVS_HI_AXI_STATUS);
+	idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
+
+	verify_dma(gpu, &debug);
+
+	seq_puts(m, "\tfeatures\n");
+	seq_printf(m, "\t minor_features0: 0x%08x\n",
+		   gpu->identity.minor_features0);
+	seq_printf(m, "\t minor_features1: 0x%08x\n",
+		   gpu->identity.minor_features1);
+	seq_printf(m, "\t minor_features2: 0x%08x\n",
+		   gpu->identity.minor_features2);
+	seq_printf(m, "\t minor_features3: 0x%08x\n",
+		   gpu->identity.minor_features3);
+	seq_printf(m, "\t minor_features4: 0x%08x\n",
+		   gpu->identity.minor_features4);
+	seq_printf(m, "\t minor_features5: 0x%08x\n",
+		   gpu->identity.minor_features5);
+
+	seq_puts(m, "\tspecs\n");
+	seq_printf(m, "\t stream_count:  %d\n",
+			gpu->identity.stream_count);
+	seq_printf(m, "\t register_max: %d\n",
+			gpu->identity.register_max);
+	seq_printf(m, "\t thread_count: %d\n",
+			gpu->identity.thread_count);
+	seq_printf(m, "\t vertex_cache_size: %d\n",
+			gpu->identity.vertex_cache_size);
+	seq_printf(m, "\t shader_core_count: %d\n",
+			gpu->identity.shader_core_count);
+	seq_printf(m, "\t pixel_pipes: %d\n",
+			gpu->identity.pixel_pipes);
+	seq_printf(m, "\t vertex_output_buffer_size: %d\n",
+			gpu->identity.vertex_output_buffer_size);
+	seq_printf(m, "\t buffer_size: %d\n",
+			gpu->identity.buffer_size);
+	seq_printf(m, "\t instruction_count: %d\n",
+			gpu->identity.instruction_count);
+	seq_printf(m, "\t num_constants: %d\n",
+			gpu->identity.num_constants);
+	seq_printf(m, "\t varyings_count: %d\n",
+			gpu->identity.varyings_count);
+
+	seq_printf(m, "\taxi: 0x%08x\n", axi);
+	seq_printf(m, "\tidle: 0x%08x\n", idle);
+	idle |= ~gpu->idle_mask & ~VIVS_HI_IDLE_STATE_AXI_LP;
+	if ((idle & VIVS_HI_IDLE_STATE_FE) == 0)
+		seq_puts(m, "\t FE is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_DE) == 0)
+		seq_puts(m, "\t DE is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_PE) == 0)
+		seq_puts(m, "\t PE is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_SH) == 0)
+		seq_puts(m, "\t SH is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_PA) == 0)
+		seq_puts(m, "\t PA is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_SE) == 0)
+		seq_puts(m, "\t SE is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_RA) == 0)
+		seq_puts(m, "\t RA is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_TX) == 0)
+		seq_puts(m, "\t TX is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_VG) == 0)
+		seq_puts(m, "\t VG is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_IM) == 0)
+		seq_puts(m, "\t IM is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_FP) == 0)
+		seq_puts(m, "\t FP is not idle\n");
+	if ((idle & VIVS_HI_IDLE_STATE_TS) == 0)
+		seq_puts(m, "\t TS is not idle\n");
+	if (idle & VIVS_HI_IDLE_STATE_AXI_LP)
+		seq_puts(m, "\t AXI low power mode\n");
+
+	if (gpu->identity.features & chipFeatures_DEBUG_MODE) {
+		u32 read0 = gpu_read(gpu, VIVS_MC_DEBUG_READ0);
+		u32 read1 = gpu_read(gpu, VIVS_MC_DEBUG_READ1);
+		u32 write = gpu_read(gpu, VIVS_MC_DEBUG_WRITE);
+
+		seq_puts(m, "\tMC\n");
+		seq_printf(m, "\t read0: 0x%08x\n", read0);
+		seq_printf(m, "\t read1: 0x%08x\n", read1);
+		seq_printf(m, "\t write: 0x%08x\n", write);
+	}
+
+	seq_puts(m, "\tDMA ");
+
+	if (debug.address[0] == debug.address[1] &&
+	    debug.state[0] == debug.state[1]) {
+		seq_puts(m, "seems to be stuck\n");
+	} else if (debug.address[0] == debug.address[1]) {
+		seq_puts(m, "adress is constant\n");
+	} else {
+		seq_puts(m, "is runing\n");
+	}
+
+	seq_printf(m, "\t address 0: 0x%08x\n", debug.address[0]);
+	seq_printf(m, "\t address 1: 0x%08x\n", debug.address[1]);
+	seq_printf(m, "\t state 0: 0x%08x\n", debug.state[0]);
+	seq_printf(m, "\t state 1: 0x%08x\n", debug.state[1]);
+	seq_printf(m, "\t last fetch 64 bit word: 0x%08x 0x%08x\n",
+		   dma_lo, dma_hi);
+
+	ret = 0;
+
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+
+	return ret;
+}
+#endif
+
+/*
+ * Power Management:
+ */
+static int enable_clk(struct etnaviv_gpu *gpu)
+{
+	if (gpu->clk_core)
+		clk_prepare_enable(gpu->clk_core);
+	if (gpu->clk_shader)
+		clk_prepare_enable(gpu->clk_shader);
+
+	return 0;
+}
+
+static int disable_clk(struct etnaviv_gpu *gpu)
+{
+	if (gpu->clk_core)
+		clk_disable_unprepare(gpu->clk_core);
+	if (gpu->clk_shader)
+		clk_disable_unprepare(gpu->clk_shader);
+
+	return 0;
+}
+
+static int enable_axi(struct etnaviv_gpu *gpu)
+{
+	if (gpu->clk_bus)
+		clk_prepare_enable(gpu->clk_bus);
+
+	return 0;
+}
+
+static int disable_axi(struct etnaviv_gpu *gpu)
+{
+	if (gpu->clk_bus)
+		clk_disable_unprepare(gpu->clk_bus);
+
+	return 0;
+}
+
+/*
+ * Hangcheck detection for locked gpu:
+ */
+static void recover_worker(struct work_struct *work)
+{
+	struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu,
+					       recover_work);
+	unsigned long flags;
+	unsigned int i;
+
+	dev_err(gpu->dev, "hangcheck recover!\n");
+
+	if (pm_runtime_get_sync(gpu->dev) < 0)
+		return;
+
+	mutex_lock(&gpu->lock);
+
+	/* Only catch the first event, or when manually re-armed */
+	if (etnaviv_dump_core) {
+		etnaviv_core_dump(gpu);
+		etnaviv_dump_core = false;
+	}
+
+	etnaviv_hw_reset(gpu);
+
+	/* complete all events, the GPU won't do it after the reset */
+	spin_lock_irqsave(&gpu->event_spinlock, flags);
+	for (i = 0; i < ARRAY_SIZE(gpu->event); i++) {
+		if (!gpu->event[i].used)
+			continue;
+		fence_signal(gpu->event[i].fence);
+		gpu->event[i].fence = NULL;
+		gpu->event[i].used = false;
+		complete(&gpu->event_free);
+		/*
+		 * Decrement the PM count for each stuck event. This is safe
+		 * even in atomic context as we use ASYNC RPM here.
+		 */
+		pm_runtime_put_autosuspend(gpu->dev);
+	}
+	spin_unlock_irqrestore(&gpu->event_spinlock, flags);
+	gpu->completed_fence = gpu->active_fence;
+
+	etnaviv_gpu_hw_init(gpu);
+	gpu->switch_context = true;
+
+	mutex_unlock(&gpu->lock);
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+
+	/* Retire the buffer objects in a work */
+	etnaviv_queue_work(gpu->drm, &gpu->retire_work);
+}
+
+static void hangcheck_timer_reset(struct etnaviv_gpu *gpu)
+{
+	DBG("%s", dev_name(gpu->dev));
+	mod_timer(&gpu->hangcheck_timer,
+		  round_jiffies_up(jiffies + DRM_ETNAVIV_HANGCHECK_JIFFIES));
+}
+
+static void hangcheck_handler(unsigned long data)
+{
+	struct etnaviv_gpu *gpu = (struct etnaviv_gpu *)data;
+	u32 fence = gpu->completed_fence;
+	bool progress = false;
+
+	if (fence != gpu->hangcheck_fence) {
+		gpu->hangcheck_fence = fence;
+		progress = true;
+	}
+
+	if (!progress) {
+		u32 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
+		int change = dma_addr - gpu->hangcheck_dma_addr;
+
+		if (change < 0 || change > 16) {
+			gpu->hangcheck_dma_addr = dma_addr;
+			progress = true;
+		}
+	}
+
+	if (!progress && fence_after(gpu->active_fence, fence)) {
+		dev_err(gpu->dev, "hangcheck detected gpu lockup!\n");
+		dev_err(gpu->dev, "     completed fence: %u\n", fence);
+		dev_err(gpu->dev, "     active fence: %u\n",
+			gpu->active_fence);
+		etnaviv_queue_work(gpu->drm, &gpu->recover_work);
+	}
+
+	/* if still more pending work, reset the hangcheck timer: */
+	if (fence_after(gpu->active_fence, gpu->hangcheck_fence))
+		hangcheck_timer_reset(gpu);
+}
+
+static void hangcheck_disable(struct etnaviv_gpu *gpu)
+{
+	del_timer_sync(&gpu->hangcheck_timer);
+	cancel_work_sync(&gpu->recover_work);
+}
+
+/* fence object management */
+struct etnaviv_fence {
+	struct etnaviv_gpu *gpu;
+	struct fence base;
+};
+
+static inline struct etnaviv_fence *to_etnaviv_fence(struct fence *fence)
+{
+	return container_of(fence, struct etnaviv_fence, base);
+}
+
+static const char *etnaviv_fence_get_driver_name(struct fence *fence)
+{
+	return "etnaviv";
+}
+
+static const char *etnaviv_fence_get_timeline_name(struct fence *fence)
+{
+	struct etnaviv_fence *f = to_etnaviv_fence(fence);
+
+	return dev_name(f->gpu->dev);
+}
+
+static bool etnaviv_fence_enable_signaling(struct fence *fence)
+{
+	return true;
+}
+
+static bool etnaviv_fence_signaled(struct fence *fence)
+{
+	struct etnaviv_fence *f = to_etnaviv_fence(fence);
+
+	return fence_completed(f->gpu, f->base.seqno);
+}
+
+static void etnaviv_fence_release(struct fence *fence)
+{
+	struct etnaviv_fence *f = to_etnaviv_fence(fence);
+
+	kfree_rcu(f, base.rcu);
+}
+
+static const struct fence_ops etnaviv_fence_ops = {
+	.get_driver_name = etnaviv_fence_get_driver_name,
+	.get_timeline_name = etnaviv_fence_get_timeline_name,
+	.enable_signaling = etnaviv_fence_enable_signaling,
+	.signaled = etnaviv_fence_signaled,
+	.wait = fence_default_wait,
+	.release = etnaviv_fence_release,
+};
+
+static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_fence *f;
+
+	f = kzalloc(sizeof(*f), GFP_KERNEL);
+	if (!f)
+		return NULL;
+
+	f->gpu = gpu;
+
+	fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock,
+		   gpu->fence_context, ++gpu->next_fence);
+
+	return &f->base;
+}
+
+int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
+	unsigned int context, bool exclusive)
+{
+	struct reservation_object *robj = etnaviv_obj->resv;
+	struct reservation_object_list *fobj;
+	struct fence *fence;
+	int i, ret;
+
+	if (!exclusive) {
+		ret = reservation_object_reserve_shared(robj);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * If we have any shared fences, then the exclusive fence
+	 * should be ignored as it will already have been signalled.
+	 */
+	fobj = reservation_object_get_list(robj);
+	if (!fobj || fobj->shared_count == 0) {
+		/* Wait on any existing exclusive fence which isn't our own */
+		fence = reservation_object_get_excl(robj);
+		if (fence && fence->context != context) {
+			ret = fence_wait(fence, true);
+			if (ret)
+				return ret;
+		}
+	}
+
+	if (!exclusive || !fobj)
+		return 0;
+
+	for (i = 0; i < fobj->shared_count; i++) {
+		fence = rcu_dereference_protected(fobj->shared[i],
+						reservation_object_held(robj));
+		if (fence->context != context) {
+			ret = fence_wait(fence, true);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * event management:
+ */
+
+static unsigned int event_alloc(struct etnaviv_gpu *gpu)
+{
+	unsigned long ret, flags;
+	unsigned int i, event = ~0U;
+
+	ret = wait_for_completion_timeout(&gpu->event_free,
+					  msecs_to_jiffies(10 * 10000));
+	if (!ret)
+		dev_err(gpu->dev, "wait_for_completion_timeout failed");
+
+	spin_lock_irqsave(&gpu->event_spinlock, flags);
+
+	/* find first free event */
+	for (i = 0; i < ARRAY_SIZE(gpu->event); i++) {
+		if (gpu->event[i].used == false) {
+			gpu->event[i].used = true;
+			event = i;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&gpu->event_spinlock, flags);
+
+	return event;
+}
+
+static void event_free(struct etnaviv_gpu *gpu, unsigned int event)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&gpu->event_spinlock, flags);
+
+	if (gpu->event[event].used == false) {
+		dev_warn(gpu->dev, "event %u is already marked as free",
+			 event);
+		spin_unlock_irqrestore(&gpu->event_spinlock, flags);
+	} else {
+		gpu->event[event].used = false;
+		spin_unlock_irqrestore(&gpu->event_spinlock, flags);
+
+		complete(&gpu->event_free);
+	}
+}
+
+/*
+ * Cmdstream submission/retirement:
+ */
+
+struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size,
+	size_t nr_bos)
+{
+	struct etnaviv_cmdbuf *cmdbuf;
+	size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo[0]),
+				 sizeof(*cmdbuf));
+
+	cmdbuf = kzalloc(sz, GFP_KERNEL);
+	if (!cmdbuf)
+		return NULL;
+
+	cmdbuf->vaddr = dma_alloc_writecombine(gpu->dev, size, &cmdbuf->paddr,
+					       GFP_KERNEL);
+	if (!cmdbuf->vaddr) {
+		kfree(cmdbuf);
+		return NULL;
+	}
+
+	cmdbuf->gpu = gpu;
+	cmdbuf->size = size;
+
+	return cmdbuf;
+}
+
+void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf)
+{
+	dma_free_writecombine(cmdbuf->gpu->dev, cmdbuf->size,
+			      cmdbuf->vaddr, cmdbuf->paddr);
+	kfree(cmdbuf);
+}
+
+static void retire_worker(struct work_struct *work)
+{
+	struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu,
+					       retire_work);
+	u32 fence = gpu->completed_fence;
+	struct etnaviv_cmdbuf *cmdbuf, *tmp;
+	unsigned int i;
+
+	mutex_lock(&gpu->lock);
+	list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) {
+		if (!fence_is_signaled(cmdbuf->fence))
+			break;
+
+		list_del(&cmdbuf->node);
+		fence_put(cmdbuf->fence);
+
+		for (i = 0; i < cmdbuf->nr_bos; i++) {
+			struct etnaviv_gem_object *etnaviv_obj = cmdbuf->bo[i];
+
+			atomic_dec(&etnaviv_obj->gpu_active);
+			/* drop the refcount taken in etnaviv_gpu_submit */
+			etnaviv_gem_put_iova(gpu, &etnaviv_obj->base);
+		}
+
+		etnaviv_gpu_cmdbuf_free(cmdbuf);
+	}
+
+	gpu->retired_fence = fence;
+
+	mutex_unlock(&gpu->lock);
+
+	wake_up_all(&gpu->fence_event);
+}
+
+int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu,
+	u32 fence, struct timespec *timeout)
+{
+	int ret;
+
+	if (fence_after(fence, gpu->next_fence)) {
+		DRM_ERROR("waiting on invalid fence: %u (of %u)\n",
+				fence, gpu->next_fence);
+		return -EINVAL;
+	}
+
+	if (!timeout) {
+		/* No timeout was requested: just test for completion */
+		ret = fence_completed(gpu, fence) ? 0 : -EBUSY;
+	} else {
+		unsigned long remaining = etnaviv_timeout_to_jiffies(timeout);
+
+		ret = wait_event_interruptible_timeout(gpu->fence_event,
+						fence_completed(gpu, fence),
+						remaining);
+		if (ret == 0) {
+			DBG("timeout waiting for fence: %u (retired: %u completed: %u)",
+				fence, gpu->retired_fence,
+				gpu->completed_fence);
+			ret = -ETIMEDOUT;
+		} else if (ret != -ERESTARTSYS) {
+			ret = 0;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * Wait for an object to become inactive.  This, on it's own, is not race
+ * free: the object is moved by the retire worker off the active list, and
+ * then the iova is put.  Moreover, the object could be re-submitted just
+ * after we notice that it's become inactive.
+ *
+ * Although the retirement happens under the gpu lock, we don't want to hold
+ * that lock in this function while waiting.
+ */
+int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu,
+	struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout)
+{
+	unsigned long remaining;
+	long ret;
+
+	if (!timeout)
+		return !is_active(etnaviv_obj) ? 0 : -EBUSY;
+
+	remaining = etnaviv_timeout_to_jiffies(timeout);
+
+	ret = wait_event_interruptible_timeout(gpu->fence_event,
+					       !is_active(etnaviv_obj),
+					       remaining);
+	if (ret > 0) {
+		struct etnaviv_drm_private *priv = gpu->drm->dev_private;
+
+		/* Synchronise with the retire worker */
+		flush_workqueue(priv->wq);
+		return 0;
+	} else if (ret == -ERESTARTSYS) {
+		return -ERESTARTSYS;
+	} else {
+		return -ETIMEDOUT;
+	}
+}
+
+int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu)
+{
+	return pm_runtime_get_sync(gpu->dev);
+}
+
+void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu)
+{
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+}
+
+/* add bo's to gpu's ring, and kick gpu: */
+int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
+	struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf)
+{
+	struct fence *fence;
+	unsigned int event, i;
+	int ret;
+
+	ret = etnaviv_gpu_pm_get_sync(gpu);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&gpu->lock);
+
+	/*
+	 * TODO
+	 *
+	 * - flush
+	 * - data endian
+	 * - prefetch
+	 *
+	 */
+
+	event = event_alloc(gpu);
+	if (unlikely(event == ~0U)) {
+		DRM_ERROR("no free event\n");
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	fence = etnaviv_gpu_fence_alloc(gpu);
+	if (!fence) {
+		event_free(gpu, event);
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
+	gpu->event[event].fence = fence;
+	submit->fence = fence->seqno;
+	gpu->active_fence = submit->fence;
+
+	if (gpu->lastctx != cmdbuf->ctx) {
+		gpu->mmu->need_flush = true;
+		gpu->switch_context = true;
+		gpu->lastctx = cmdbuf->ctx;
+	}
+
+	etnaviv_buffer_queue(gpu, event, cmdbuf);
+
+	cmdbuf->fence = fence;
+	list_add_tail(&cmdbuf->node, &gpu->active_cmd_list);
+
+	/* We're committed to adding this command buffer, hold a PM reference */
+	pm_runtime_get_noresume(gpu->dev);
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		u32 iova;
+
+		/* Each cmdbuf takes a refcount on the iova */
+		etnaviv_gem_get_iova(gpu, &etnaviv_obj->base, &iova);
+		cmdbuf->bo[i] = etnaviv_obj;
+		atomic_inc(&etnaviv_obj->gpu_active);
+
+		if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)
+			reservation_object_add_excl_fence(etnaviv_obj->resv,
+							  fence);
+		else
+			reservation_object_add_shared_fence(etnaviv_obj->resv,
+							    fence);
+	}
+	cmdbuf->nr_bos = submit->nr_bos;
+	hangcheck_timer_reset(gpu);
+	ret = 0;
+
+out_unlock:
+	mutex_unlock(&gpu->lock);
+
+	etnaviv_gpu_pm_put(gpu);
+
+	return ret;
+}
+
+/*
+ * Init/Cleanup:
+ */
+static irqreturn_t irq_handler(int irq, void *data)
+{
+	struct etnaviv_gpu *gpu = data;
+	irqreturn_t ret = IRQ_NONE;
+
+	u32 intr = gpu_read(gpu, VIVS_HI_INTR_ACKNOWLEDGE);
+
+	if (intr != 0) {
+		int event;
+
+		pm_runtime_mark_last_busy(gpu->dev);
+
+		dev_dbg(gpu->dev, "intr 0x%08x\n", intr);
+
+		if (intr & VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR) {
+			dev_err(gpu->dev, "AXI bus error\n");
+			intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR;
+		}
+
+		while ((event = ffs(intr)) != 0) {
+			struct fence *fence;
+
+			event -= 1;
+
+			intr &= ~(1 << event);
+
+			dev_dbg(gpu->dev, "event %u\n", event);
+
+			fence = gpu->event[event].fence;
+			gpu->event[event].fence = NULL;
+			fence_signal(fence);
+
+			/*
+			 * Events can be processed out of order.  Eg,
+			 * - allocate and queue event 0
+			 * - allocate event 1
+			 * - event 0 completes, we process it
+			 * - allocate and queue event 0
+			 * - event 1 and event 0 complete
+			 * we can end up processing event 0 first, then 1.
+			 */
+			if (fence_after(fence->seqno, gpu->completed_fence))
+				gpu->completed_fence = fence->seqno;
+
+			event_free(gpu, event);
+
+			/*
+			 * We need to balance the runtime PM count caused by
+			 * each submission.  Upon submission, we increment
+			 * the runtime PM counter, and allocate one event.
+			 * So here, we put the runtime PM count for each
+			 * completed event.
+			 */
+			pm_runtime_put_autosuspend(gpu->dev);
+		}
+
+		/* Retire the buffer objects in a work */
+		etnaviv_queue_work(gpu->drm, &gpu->retire_work);
+
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu)
+{
+	int ret;
+
+	ret = enable_clk(gpu);
+	if (ret)
+		return ret;
+
+	ret = enable_axi(gpu);
+	if (ret) {
+		disable_clk(gpu);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu)
+{
+	int ret;
+
+	ret = disable_axi(gpu);
+	if (ret)
+		return ret;
+
+	ret = disable_clk(gpu);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu)
+{
+	if (gpu->buffer) {
+		unsigned long timeout;
+
+		/* Replace the last WAIT with END */
+		etnaviv_buffer_end(gpu);
+
+		/*
+		 * We know that only the FE is busy here, this should
+		 * happen quickly (as the WAIT is only 200 cycles).  If
+		 * we fail, just warn and continue.
+		 */
+		timeout = jiffies + msecs_to_jiffies(100);
+		do {
+			u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
+
+			if ((idle & gpu->idle_mask) == gpu->idle_mask)
+				break;
+
+			if (time_is_before_jiffies(timeout)) {
+				dev_warn(gpu->dev,
+					 "timed out waiting for idle: idle=0x%x\n",
+					 idle);
+				break;
+			}
+
+			udelay(5);
+		} while (1);
+	}
+
+	return etnaviv_gpu_clk_disable(gpu);
+}
+
+#ifdef CONFIG_PM
+static int etnaviv_gpu_hw_resume(struct etnaviv_gpu *gpu)
+{
+	u32 clock;
+	int ret;
+
+	ret = mutex_lock_killable(&gpu->lock);
+	if (ret)
+		return ret;
+
+	clock = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS |
+		VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40);
+
+	etnaviv_gpu_load_clock(gpu, clock);
+	etnaviv_gpu_hw_init(gpu);
+
+	gpu->switch_context = true;
+
+	mutex_unlock(&gpu->lock);
+
+	return 0;
+}
+#endif
+
+static int etnaviv_gpu_bind(struct device *dev, struct device *master,
+	void *data)
+{
+	struct drm_device *drm = data;
+	struct etnaviv_drm_private *priv = drm->dev_private;
+	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
+	int ret;
+
+#ifdef CONFIG_PM
+	ret = pm_runtime_get_sync(gpu->dev);
+#else
+	ret = etnaviv_gpu_clk_enable(gpu);
+#endif
+	if (ret < 0)
+		return ret;
+
+	gpu->drm = drm;
+	gpu->fence_context = fence_context_alloc(1);
+	spin_lock_init(&gpu->fence_spinlock);
+
+	INIT_LIST_HEAD(&gpu->active_cmd_list);
+	INIT_WORK(&gpu->retire_work, retire_worker);
+	INIT_WORK(&gpu->recover_work, recover_worker);
+	init_waitqueue_head(&gpu->fence_event);
+
+	setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
+			(unsigned long)gpu);
+
+	priv->gpu[priv->num_gpus++] = gpu;
+
+	pm_runtime_mark_last_busy(gpu->dev);
+	pm_runtime_put_autosuspend(gpu->dev);
+
+	return 0;
+}
+
+static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
+	void *data)
+{
+	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
+
+	DBG("%s", dev_name(gpu->dev));
+
+	hangcheck_disable(gpu);
+
+#ifdef CONFIG_PM
+	pm_runtime_get_sync(gpu->dev);
+	pm_runtime_put_sync_suspend(gpu->dev);
+#else
+	etnaviv_gpu_hw_suspend(gpu);
+#endif
+
+	if (gpu->buffer) {
+		etnaviv_gpu_cmdbuf_free(gpu->buffer);
+		gpu->buffer = NULL;
+	}
+
+	if (gpu->mmu) {
+		etnaviv_iommu_destroy(gpu->mmu);
+		gpu->mmu = NULL;
+	}
+
+	gpu->drm = NULL;
+}
+
+static const struct component_ops gpu_ops = {
+	.bind = etnaviv_gpu_bind,
+	.unbind = etnaviv_gpu_unbind,
+};
+
+static const struct of_device_id etnaviv_gpu_match[] = {
+	{
+		.compatible = "vivante,gc"
+	},
+	{ /* sentinel */ }
+};
+
+static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct etnaviv_gpu *gpu;
+	int err = 0;
+
+	gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL);
+	if (!gpu)
+		return -ENOMEM;
+
+	gpu->dev = &pdev->dev;
+	mutex_init(&gpu->lock);
+
+	/*
+	 * Set the GPU base address to the start of physical memory.  This
+	 * ensures that if we have up to 2GB, the v1 MMU can address the
+	 * highest memory.  This is important as command buffers may be
+	 * allocated outside of this limit.
+	 */
+	gpu->memory_base = PHYS_OFFSET;
+
+	/* Map registers: */
+	gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev));
+	if (IS_ERR(gpu->mmio))
+		return PTR_ERR(gpu->mmio);
+
+	/* Get Interrupt: */
+	gpu->irq = platform_get_irq(pdev, 0);
+	if (gpu->irq < 0) {
+		err = gpu->irq;
+		dev_err(dev, "failed to get irq: %d\n", err);
+		goto fail;
+	}
+
+	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
+			       dev_name(gpu->dev), gpu);
+	if (err) {
+		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
+		goto fail;
+	}
+
+	/* Get Clocks: */
+	gpu->clk_bus = devm_clk_get(&pdev->dev, "bus");
+	DBG("clk_bus: %p", gpu->clk_bus);
+	if (IS_ERR(gpu->clk_bus))
+		gpu->clk_bus = NULL;
+
+	gpu->clk_core = devm_clk_get(&pdev->dev, "core");
+	DBG("clk_core: %p", gpu->clk_core);
+	if (IS_ERR(gpu->clk_core))
+		gpu->clk_core = NULL;
+
+	gpu->clk_shader = devm_clk_get(&pdev->dev, "shader");
+	DBG("clk_shader: %p", gpu->clk_shader);
+	if (IS_ERR(gpu->clk_shader))
+		gpu->clk_shader = NULL;
+
+	/* TODO: figure out max mapped size */
+	dev_set_drvdata(dev, gpu);
+
+	/*
+	 * We treat the device as initially suspended.  The runtime PM
+	 * autosuspend delay is rather arbitary: no measurements have
+	 * yet been performed to determine an appropriate value.
+	 */
+	pm_runtime_use_autosuspend(gpu->dev);
+	pm_runtime_set_autosuspend_delay(gpu->dev, 200);
+	pm_runtime_enable(gpu->dev);
+
+	err = component_add(&pdev->dev, &gpu_ops);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to register component: %d\n", err);
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	return err;
+}
+
+static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &gpu_ops);
+	pm_runtime_disable(&pdev->dev);
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int etnaviv_gpu_rpm_suspend(struct device *dev)
+{
+	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
+	u32 idle, mask;
+
+	/* If we have outstanding fences, we're not idle */
+	if (gpu->completed_fence != gpu->active_fence)
+		return -EBUSY;
+
+	/* Check whether the hardware (except FE) is idle */
+	mask = gpu->idle_mask & ~VIVS_HI_IDLE_STATE_FE;
+	idle = gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask;
+	if (idle != mask)
+		return -EBUSY;
+
+	return etnaviv_gpu_hw_suspend(gpu);
+}
+
+static int etnaviv_gpu_rpm_resume(struct device *dev)
+{
+	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
+	int ret;
+
+	ret = etnaviv_gpu_clk_enable(gpu);
+	if (ret)
+		return ret;
+
+	/* Re-initialise the basic hardware state */
+	if (gpu->drm && gpu->buffer) {
+		ret = etnaviv_gpu_hw_resume(gpu);
+		if (ret) {
+			etnaviv_gpu_clk_disable(gpu);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
+	SET_RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume,
+			   NULL)
+};
+
+struct platform_driver etnaviv_gpu_driver = {
+	.driver = {
+		.name = "etnaviv-gpu",
+		.owner = THIS_MODULE,
+		.pm = &etnaviv_gpu_pm_ops,
+		.of_match_table = etnaviv_gpu_match,
+	},
+	.probe = etnaviv_gpu_platform_probe,
+	.remove = etnaviv_gpu_platform_remove,
+	.id_table = gpu_ids,
+};
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
new file mode 100644
index 000000000000..f233ac4c7c1c
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -0,0 +1,218 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_GPU_H__
+#define __ETNAVIV_GPU_H__
+
+#include <linux/clk.h>
+#include <linux/regulator/consumer.h>
+
+#include "etnaviv_drv.h"
+
+struct etnaviv_gem_submit;
+
+struct etnaviv_chip_identity {
+	/* Chip model. */
+	u32 model;
+
+	/* Revision value.*/
+	u32 revision;
+
+	/* Supported feature fields. */
+	u32 features;
+
+	/* Supported minor feature fields. */
+	u32 minor_features0;
+
+	/* Supported minor feature 1 fields. */
+	u32 minor_features1;
+
+	/* Supported minor feature 2 fields. */
+	u32 minor_features2;
+
+	/* Supported minor feature 3 fields. */
+	u32 minor_features3;
+
+	/* Supported minor feature 4 fields. */
+	u32 minor_features4;
+
+	/* Supported minor feature 5 fields. */
+	u32 minor_features5;
+
+	/* Number of streams supported. */
+	u32 stream_count;
+
+	/* Total number of temporary registers per thread. */
+	u32 register_max;
+
+	/* Maximum number of threads. */
+	u32 thread_count;
+
+	/* Number of shader cores. */
+	u32 shader_core_count;
+
+	/* Size of the vertex cache. */
+	u32 vertex_cache_size;
+
+	/* Number of entries in the vertex output buffer. */
+	u32 vertex_output_buffer_size;
+
+	/* Number of pixel pipes. */
+	u32 pixel_pipes;
+
+	/* Number of instructions. */
+	u32 instruction_count;
+
+	/* Number of constants. */
+	u32 num_constants;
+
+	/* Buffer size */
+	u32 buffer_size;
+
+	/* Number of varyings */
+	u8 varyings_count;
+};
+
+struct etnaviv_event {
+	bool used;
+	struct fence *fence;
+};
+
+struct etnaviv_cmdbuf;
+
+struct etnaviv_gpu {
+	struct drm_device *drm;
+	struct device *dev;
+	struct mutex lock;
+	struct etnaviv_chip_identity identity;
+	struct etnaviv_file_private *lastctx;
+	bool switch_context;
+
+	/* 'ring'-buffer: */
+	struct etnaviv_cmdbuf *buffer;
+
+	/* bus base address of memory  */
+	u32 memory_base;
+
+	/* event management: */
+	struct etnaviv_event event[30];
+	struct completion event_free;
+	spinlock_t event_spinlock;
+
+	/* list of currently in-flight command buffers */
+	struct list_head active_cmd_list;
+
+	u32 idle_mask;
+
+	/* Fencing support */
+	u32 next_fence;
+	u32 active_fence;
+	u32 completed_fence;
+	u32 retired_fence;
+	wait_queue_head_t fence_event;
+	unsigned int fence_context;
+	spinlock_t fence_spinlock;
+
+	/* worker for handling active-list retiring: */
+	struct work_struct retire_work;
+
+	void __iomem *mmio;
+	int irq;
+
+	struct etnaviv_iommu *mmu;
+
+	/* Power Control: */
+	struct clk *clk_bus;
+	struct clk *clk_core;
+	struct clk *clk_shader;
+
+	/* Hang Detction: */
+#define DRM_ETNAVIV_HANGCHECK_PERIOD 500 /* in ms */
+#define DRM_ETNAVIV_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_ETNAVIV_HANGCHECK_PERIOD)
+	struct timer_list hangcheck_timer;
+	u32 hangcheck_fence;
+	u32 hangcheck_dma_addr;
+	struct work_struct recover_work;
+};
+
+struct etnaviv_cmdbuf {
+	/* device this cmdbuf is allocated for */
+	struct etnaviv_gpu *gpu;
+	/* user context key, must be unique between all active users */
+	struct etnaviv_file_private *ctx;
+	/* cmdbuf properties */
+	void *vaddr;
+	dma_addr_t paddr;
+	u32 size;
+	u32 user_size;
+	/* fence after which this buffer is to be disposed */
+	struct fence *fence;
+	/* target exec state */
+	u32 exec_state;
+	/* per GPU in-flight list */
+	struct list_head node;
+	/* BOs attached to this command buffer */
+	unsigned int nr_bos;
+	struct etnaviv_gem_object *bo[0];
+};
+
+static inline void gpu_write(struct etnaviv_gpu *gpu, u32 reg, u32 data)
+{
+	etnaviv_writel(data, gpu->mmio + reg);
+}
+
+static inline u32 gpu_read(struct etnaviv_gpu *gpu, u32 reg)
+{
+	return etnaviv_readl(gpu->mmio + reg);
+}
+
+static inline bool fence_completed(struct etnaviv_gpu *gpu, u32 fence)
+{
+	return fence_after_eq(gpu->completed_fence, fence);
+}
+
+static inline bool fence_retired(struct etnaviv_gpu *gpu, u32 fence)
+{
+	return fence_after_eq(gpu->retired_fence, fence);
+}
+
+int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value);
+
+int etnaviv_gpu_init(struct etnaviv_gpu *gpu);
+
+#ifdef CONFIG_DEBUG_FS
+int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m);
+#endif
+
+int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
+	unsigned int context, bool exclusive);
+
+void etnaviv_gpu_retire(struct etnaviv_gpu *gpu);
+int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu,
+	u32 fence, struct timespec *timeout);
+int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu,
+	struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout);
+int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
+	struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf);
+struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu,
+					      u32 size, size_t nr_bos);
+void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf);
+int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu);
+void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
+
+extern struct platform_driver etnaviv_gpu_driver;
+
+#endif /* __ETNAVIV_GPU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
new file mode 100644
index 000000000000..522cfd447892
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/iommu.h>
+#include <linux/platform_device.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/bitops.h>
+
+#include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
+#include "etnaviv_iommu.h"
+#include "state_hi.xml.h"
+
+#define PT_SIZE		SZ_2M
+#define PT_ENTRIES	(PT_SIZE / sizeof(u32))
+
+#define GPU_MEM_START	0x80000000
+
+struct etnaviv_iommu_domain_pgtable {
+	u32 *pgtable;
+	dma_addr_t paddr;
+};
+
+struct etnaviv_iommu_domain {
+	struct iommu_domain domain;
+	struct device *dev;
+	void *bad_page_cpu;
+	dma_addr_t bad_page_dma;
+	struct etnaviv_iommu_domain_pgtable pgtable;
+	spinlock_t map_lock;
+};
+
+static struct etnaviv_iommu_domain *to_etnaviv_domain(struct iommu_domain *domain)
+{
+	return container_of(domain, struct etnaviv_iommu_domain, domain);
+}
+
+static int pgtable_alloc(struct etnaviv_iommu_domain_pgtable *pgtable,
+			 size_t size)
+{
+	pgtable->pgtable = dma_alloc_coherent(NULL, size, &pgtable->paddr, GFP_KERNEL);
+	if (!pgtable->pgtable)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void pgtable_free(struct etnaviv_iommu_domain_pgtable *pgtable,
+			 size_t size)
+{
+	dma_free_coherent(NULL, size, pgtable->pgtable, pgtable->paddr);
+}
+
+static u32 pgtable_read(struct etnaviv_iommu_domain_pgtable *pgtable,
+			   unsigned long iova)
+{
+	/* calcuate index into page table */
+	unsigned int index = (iova - GPU_MEM_START) / SZ_4K;
+	phys_addr_t paddr;
+
+	paddr = pgtable->pgtable[index];
+
+	return paddr;
+}
+
+static void pgtable_write(struct etnaviv_iommu_domain_pgtable *pgtable,
+			  unsigned long iova, phys_addr_t paddr)
+{
+	/* calcuate index into page table */
+	unsigned int index = (iova - GPU_MEM_START) / SZ_4K;
+
+	pgtable->pgtable[index] = paddr;
+}
+
+static int __etnaviv_iommu_init(struct etnaviv_iommu_domain *etnaviv_domain)
+{
+	u32 *p;
+	int ret, i;
+
+	etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev,
+						  SZ_4K,
+						  &etnaviv_domain->bad_page_dma,
+						  GFP_KERNEL);
+	if (!etnaviv_domain->bad_page_cpu)
+		return -ENOMEM;
+
+	p = etnaviv_domain->bad_page_cpu;
+	for (i = 0; i < SZ_4K / 4; i++)
+		*p++ = 0xdead55aa;
+
+	ret = pgtable_alloc(&etnaviv_domain->pgtable, PT_SIZE);
+	if (ret < 0) {
+		dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+				  etnaviv_domain->bad_page_cpu,
+				  etnaviv_domain->bad_page_dma);
+		return ret;
+	}
+
+	for (i = 0; i < PT_ENTRIES; i++)
+		etnaviv_domain->pgtable.pgtable[i] =
+			etnaviv_domain->bad_page_dma;
+
+	spin_lock_init(&etnaviv_domain->map_lock);
+
+	return 0;
+}
+
+static void etnaviv_domain_free(struct iommu_domain *domain)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+
+	pgtable_free(&etnaviv_domain->pgtable, PT_SIZE);
+
+	dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+			  etnaviv_domain->bad_page_cpu,
+			  etnaviv_domain->bad_page_dma);
+
+	kfree(etnaviv_domain);
+}
+
+static int etnaviv_iommuv1_map(struct iommu_domain *domain, unsigned long iova,
+	   phys_addr_t paddr, size_t size, int prot)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+
+	if (size != SZ_4K)
+		return -EINVAL;
+
+	spin_lock(&etnaviv_domain->map_lock);
+	pgtable_write(&etnaviv_domain->pgtable, iova, paddr);
+	spin_unlock(&etnaviv_domain->map_lock);
+
+	return 0;
+}
+
+static size_t etnaviv_iommuv1_unmap(struct iommu_domain *domain,
+	unsigned long iova, size_t size)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+
+	if (size != SZ_4K)
+		return -EINVAL;
+
+	spin_lock(&etnaviv_domain->map_lock);
+	pgtable_write(&etnaviv_domain->pgtable, iova,
+		      etnaviv_domain->bad_page_dma);
+	spin_unlock(&etnaviv_domain->map_lock);
+
+	return SZ_4K;
+}
+
+static phys_addr_t etnaviv_iommu_iova_to_phys(struct iommu_domain *domain,
+	dma_addr_t iova)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+
+	return pgtable_read(&etnaviv_domain->pgtable, iova);
+}
+
+static size_t etnaviv_iommuv1_dump_size(struct iommu_domain *domain)
+{
+	return PT_SIZE;
+}
+
+static void etnaviv_iommuv1_dump(struct iommu_domain *domain, void *buf)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+
+	memcpy(buf, etnaviv_domain->pgtable.pgtable, PT_SIZE);
+}
+
+static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
+	.ops = {
+		.domain_free = etnaviv_domain_free,
+		.map = etnaviv_iommuv1_map,
+		.unmap = etnaviv_iommuv1_unmap,
+		.iova_to_phys = etnaviv_iommu_iova_to_phys,
+		.pgsize_bitmap = SZ_4K,
+	},
+	.dump_size = etnaviv_iommuv1_dump_size,
+	.dump = etnaviv_iommuv1_dump,
+};
+
+void etnaviv_iommu_domain_restore(struct etnaviv_gpu *gpu,
+	struct iommu_domain *domain)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+	u32 pgtable;
+
+	/* set page table address in MC */
+	pgtable = (u32)etnaviv_domain->pgtable.paddr;
+
+	gpu_write(gpu, VIVS_MC_MMU_FE_PAGE_TABLE, pgtable);
+	gpu_write(gpu, VIVS_MC_MMU_TX_PAGE_TABLE, pgtable);
+	gpu_write(gpu, VIVS_MC_MMU_PE_PAGE_TABLE, pgtable);
+	gpu_write(gpu, VIVS_MC_MMU_PEZ_PAGE_TABLE, pgtable);
+	gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable);
+}
+
+struct iommu_domain *etnaviv_iommu_domain_alloc(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_iommu_domain *etnaviv_domain;
+	int ret;
+
+	etnaviv_domain = kzalloc(sizeof(*etnaviv_domain), GFP_KERNEL);
+	if (!etnaviv_domain)
+		return NULL;
+
+	etnaviv_domain->dev = gpu->dev;
+
+	etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING;
+	etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops;
+	etnaviv_domain->domain.geometry.aperture_start = GPU_MEM_START;
+	etnaviv_domain->domain.geometry.aperture_end = GPU_MEM_START + PT_ENTRIES * SZ_4K - 1;
+
+	ret = __etnaviv_iommu_init(etnaviv_domain);
+	if (ret)
+		goto out_free;
+
+	return &etnaviv_domain->domain;
+
+out_free:
+	kfree(etnaviv_domain);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
new file mode 100644
index 000000000000..cf45503f6b6f
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
+  *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_IOMMU_H__
+#define __ETNAVIV_IOMMU_H__
+
+#include <linux/iommu.h>
+struct etnaviv_gpu;
+
+struct iommu_domain *etnaviv_iommu_domain_alloc(struct etnaviv_gpu *gpu);
+void etnaviv_iommu_domain_restore(struct etnaviv_gpu *gpu,
+	struct iommu_domain *domain);
+struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu);
+
+#endif /* __ETNAVIV_IOMMU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
new file mode 100644
index 000000000000..fbb4aed3dc80
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
+  *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/iommu.h>
+#include <linux/platform_device.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/bitops.h>
+
+#include "etnaviv_gpu.h"
+#include "etnaviv_iommu.h"
+#include "state_hi.xml.h"
+
+
+struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu)
+{
+	/* TODO */
+	return NULL;
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h
new file mode 100644
index 000000000000..603ea41c5389
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
+  *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_IOMMU_V2_H__
+#define __ETNAVIV_IOMMU_V2_H__
+
+#include <linux/iommu.h>
+struct etnaviv_gpu;
+
+struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu);
+
+#endif /* __ETNAVIV_IOMMU_V2_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
new file mode 100644
index 000000000000..6743bc648dc8
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "etnaviv_drv.h"
+#include "etnaviv_gem.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
+
+static int etnaviv_fault_handler(struct iommu_domain *iommu, struct device *dev,
+		unsigned long iova, int flags, void *arg)
+{
+	DBG("*** fault: iova=%08lx, flags=%d", iova, flags);
+	return 0;
+}
+
+int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova,
+		struct sg_table *sgt, unsigned len, int prot)
+{
+	struct iommu_domain *domain = iommu->domain;
+	struct scatterlist *sg;
+	unsigned int da = iova;
+	unsigned int i, j;
+	int ret;
+
+	if (!domain || !sgt)
+		return -EINVAL;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
+		u32 pa = sg_dma_address(sg) - sg->offset;
+		size_t bytes = sg_dma_len(sg) + sg->offset;
+
+		VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes);
+
+		ret = iommu_map(domain, da, pa, bytes, prot);
+		if (ret)
+			goto fail;
+
+		da += bytes;
+	}
+
+	return 0;
+
+fail:
+	da = iova;
+
+	for_each_sg(sgt->sgl, sg, i, j) {
+		size_t bytes = sg_dma_len(sg) + sg->offset;
+
+		iommu_unmap(domain, da, bytes);
+		da += bytes;
+	}
+	return ret;
+}
+
+int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova,
+		struct sg_table *sgt, unsigned len)
+{
+	struct iommu_domain *domain = iommu->domain;
+	struct scatterlist *sg;
+	unsigned int da = iova;
+	int i;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
+		size_t bytes = sg_dma_len(sg) + sg->offset;
+		size_t unmapped;
+
+		unmapped = iommu_unmap(domain, da, bytes);
+		if (unmapped < bytes)
+			return unmapped;
+
+		VERB("unmap[%d]: %08x(%zx)", i, iova, bytes);
+
+		BUG_ON(!PAGE_ALIGNED(bytes));
+
+		da += bytes;
+	}
+
+	return 0;
+}
+
+static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu *mmu,
+	struct etnaviv_vram_mapping *mapping)
+{
+	struct etnaviv_gem_object *etnaviv_obj = mapping->object;
+
+	etnaviv_iommu_unmap(mmu, mapping->vram_node.start,
+			    etnaviv_obj->sgt, etnaviv_obj->base.size);
+	drm_mm_remove_node(&mapping->vram_node);
+}
+
+int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
+	struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
+	struct etnaviv_vram_mapping *mapping)
+{
+	struct etnaviv_vram_mapping *free = NULL;
+	struct sg_table *sgt = etnaviv_obj->sgt;
+	struct drm_mm_node *node;
+	int ret;
+
+	lockdep_assert_held(&etnaviv_obj->lock);
+
+	mutex_lock(&mmu->lock);
+
+	/* v1 MMU can optimize single entry (contiguous) scatterlists */
+	if (sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) {
+		u32 iova;
+
+		iova = sg_dma_address(sgt->sgl) - memory_base;
+		if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
+			mapping->iova = iova;
+			list_add_tail(&mapping->mmu_node, &mmu->mappings);
+			mutex_unlock(&mmu->lock);
+			return 0;
+		}
+	}
+
+	node = &mapping->vram_node;
+	while (1) {
+		struct etnaviv_vram_mapping *m, *n;
+		struct list_head list;
+		bool found;
+
+		ret = drm_mm_insert_node_in_range(&mmu->mm, node,
+			etnaviv_obj->base.size, 0, mmu->last_iova, ~0UL,
+			DRM_MM_SEARCH_DEFAULT);
+
+		if (ret != -ENOSPC)
+			break;
+
+		/*
+		 * If we did not search from the start of the MMU region,
+		 * try again in case there are free slots.
+		 */
+		if (mmu->last_iova) {
+			mmu->last_iova = 0;
+			mmu->need_flush = true;
+			continue;
+		}
+
+		/* Try to retire some entries */
+		drm_mm_init_scan(&mmu->mm, etnaviv_obj->base.size, 0, 0);
+
+		found = 0;
+		INIT_LIST_HEAD(&list);
+		list_for_each_entry(free, &mmu->mappings, mmu_node) {
+			/* If this vram node has not been used, skip this. */
+			if (!free->vram_node.mm)
+				continue;
+
+			/*
+			 * If the iova is pinned, then it's in-use,
+			 * so we must keep its mapping.
+			 */
+			if (free->use)
+				continue;
+
+			list_add(&free->scan_node, &list);
+			if (drm_mm_scan_add_block(&free->vram_node)) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			/* Nothing found, clean up and fail */
+			list_for_each_entry_safe(m, n, &list, scan_node)
+				BUG_ON(drm_mm_scan_remove_block(&m->vram_node));
+			break;
+		}
+
+		/*
+		 * drm_mm does not allow any other operations while
+		 * scanning, so we have to remove all blocks first.
+		 * If drm_mm_scan_remove_block() returns false, we
+		 * can leave the block pinned.
+		 */
+		list_for_each_entry_safe(m, n, &list, scan_node)
+			if (!drm_mm_scan_remove_block(&m->vram_node))
+				list_del_init(&m->scan_node);
+
+		/*
+		 * Unmap the blocks which need to be reaped from the MMU.
+		 * Clear the mmu pointer to prevent the get_iova finding
+		 * this mapping.
+		 */
+		list_for_each_entry_safe(m, n, &list, scan_node) {
+			etnaviv_iommu_remove_mapping(mmu, m);
+			m->mmu = NULL;
+			list_del_init(&m->mmu_node);
+			list_del_init(&m->scan_node);
+		}
+
+		/*
+		 * We removed enough mappings so that the new allocation will
+		 * succeed.  Ensure that the MMU will be flushed before the
+		 * associated commit requesting this mapping, and retry the
+		 * allocation one more time.
+		 */
+		mmu->need_flush = true;
+	}
+
+	if (ret < 0) {
+		mutex_unlock(&mmu->lock);
+		return ret;
+	}
+
+	mmu->last_iova = node->start + etnaviv_obj->base.size;
+	mapping->iova = node->start;
+	ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size,
+				IOMMU_READ | IOMMU_WRITE);
+
+	if (ret < 0) {
+		drm_mm_remove_node(node);
+		mutex_unlock(&mmu->lock);
+		return ret;
+	}
+
+	list_add_tail(&mapping->mmu_node, &mmu->mappings);
+	mutex_unlock(&mmu->lock);
+
+	return ret;
+}
+
+void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,
+	struct etnaviv_vram_mapping *mapping)
+{
+	WARN_ON(mapping->use);
+
+	mutex_lock(&mmu->lock);
+
+	/* If the vram node is on the mm, unmap and remove the node */
+	if (mapping->vram_node.mm == &mmu->mm)
+		etnaviv_iommu_remove_mapping(mmu, mapping);
+
+	list_del(&mapping->mmu_node);
+	mutex_unlock(&mmu->lock);
+}
+
+void etnaviv_iommu_destroy(struct etnaviv_iommu *mmu)
+{
+	drm_mm_takedown(&mmu->mm);
+	iommu_domain_free(mmu->domain);
+	kfree(mmu);
+}
+
+struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu,
+	struct iommu_domain *domain, enum etnaviv_iommu_version version)
+{
+	struct etnaviv_iommu *mmu;
+
+	mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
+	if (!mmu)
+		return ERR_PTR(-ENOMEM);
+
+	mmu->domain = domain;
+	mmu->gpu = gpu;
+	mmu->version = version;
+	mutex_init(&mmu->lock);
+	INIT_LIST_HEAD(&mmu->mappings);
+
+	drm_mm_init(&mmu->mm, domain->geometry.aperture_start,
+		    domain->geometry.aperture_end -
+		      domain->geometry.aperture_start + 1);
+
+	iommu_set_fault_handler(domain, etnaviv_fault_handler, gpu->dev);
+
+	return mmu;
+}
+
+size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu)
+{
+	struct etnaviv_iommu_ops *ops;
+
+	ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
+
+	return ops->dump_size(iommu->domain);
+}
+
+void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf)
+{
+	struct etnaviv_iommu_ops *ops;
+
+	ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
+
+	ops->dump(iommu->domain, buf);
+}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
new file mode 100644
index 000000000000..fff215a47630
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2015 Etnaviv Project
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ETNAVIV_MMU_H__
+#define __ETNAVIV_MMU_H__
+
+#include <linux/iommu.h>
+
+enum etnaviv_iommu_version {
+	ETNAVIV_IOMMU_V1 = 0,
+	ETNAVIV_IOMMU_V2,
+};
+
+struct etnaviv_gpu;
+struct etnaviv_vram_mapping;
+
+struct etnaviv_iommu_ops {
+	struct iommu_ops ops;
+	size_t (*dump_size)(struct iommu_domain *);
+	void (*dump)(struct iommu_domain *, void *);
+};
+
+struct etnaviv_iommu {
+	struct etnaviv_gpu *gpu;
+	struct iommu_domain *domain;
+
+	enum etnaviv_iommu_version version;
+
+	/* memory manager for GPU address area */
+	struct mutex lock;
+	struct list_head mappings;
+	struct drm_mm mm;
+	u32 last_iova;
+	bool need_flush;
+};
+
+struct etnaviv_gem_object;
+
+int etnaviv_iommu_attach(struct etnaviv_iommu *iommu, const char **names,
+	int cnt);
+int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova,
+	struct sg_table *sgt, unsigned len, int prot);
+int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova,
+	struct sg_table *sgt, unsigned len);
+int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
+	struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
+	struct etnaviv_vram_mapping *mapping);
+void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,
+	struct etnaviv_vram_mapping *mapping);
+void etnaviv_iommu_destroy(struct etnaviv_iommu *iommu);
+
+size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu);
+void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf);
+
+struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu,
+	struct iommu_domain *domain, enum etnaviv_iommu_version version);
+
+#endif /* __ETNAVIV_MMU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/state.xml.h b/drivers/gpu/drm/etnaviv/state.xml.h
new file mode 100644
index 000000000000..368218304566
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/state.xml.h
@@ -0,0 +1,351 @@
+#ifndef STATE_XML
+#define STATE_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- state.xml    (  18882 bytes, from 2015-03-25 11:42:32)
+- common.xml   (  18437 bytes, from 2015-03-25 11:27:41)
+- state_hi.xml (  23420 bytes, from 2015-03-25 11:47:21)
+- state_2d.xml (  51549 bytes, from 2015-03-25 11:25:06)
+- state_3d.xml (  54600 bytes, from 2015-03-25 11:25:19)
+- state_vg.xml (   5973 bytes, from 2015-03-25 11:26:01)
+
+Copyright (C) 2015
+*/
+
+
+#define VARYING_COMPONENT_USE_UNUSED				0x00000000
+#define VARYING_COMPONENT_USE_USED				0x00000001
+#define VARYING_COMPONENT_USE_POINTCOORD_X			0x00000002
+#define VARYING_COMPONENT_USE_POINTCOORD_Y			0x00000003
+#define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__MASK		0x000000ff
+#define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__SHIFT		0
+#define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE(x)		(((x) << FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__SHIFT) & FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__MASK)
+#define VIVS_FE							0x00000000
+
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG(i0)		       (0x00000600 + 0x4*(i0))
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG__ESIZE			0x00000004
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG__LEN			0x00000010
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__MASK		0x0000000f
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__SHIFT		0
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_BYTE			0x00000000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_BYTE	0x00000001
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_SHORT		0x00000002
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_SHORT	0x00000003
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_INT			0x00000004
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_INT		0x00000005
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_FLOAT		0x00000008
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_HALF_FLOAT		0x00000009
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_FIXED		0x0000000b
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_INT_10_10_10_2	0x0000000c
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_INT_10_10_10_2	0x0000000d
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__MASK		0x00000030
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__SHIFT		4
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN(x)			(((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__MASK)
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NONCONSECUTIVE		0x00000080
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_STREAM__MASK		0x00000700
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_STREAM__SHIFT		8
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_STREAM(x)			(((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_STREAM__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_STREAM__MASK)
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NUM__MASK			0x00003000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NUM__SHIFT		12
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NUM(x)			(((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_NUM__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_NUM__MASK)
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NORMALIZE__MASK		0x0000c000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NORMALIZE__SHIFT		14
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NORMALIZE_OFF		0x00000000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_NORMALIZE_ON		0x00008000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_START__MASK		0x00ff0000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_START__SHIFT		16
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_START(x)			(((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_START__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_START__MASK)
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_END__MASK			0xff000000
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_END__SHIFT		24
+#define VIVS_FE_VERTEX_ELEMENT_CONFIG_END(x)			(((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_END__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_END__MASK)
+
+#define VIVS_FE_CMD_STREAM_BASE_ADDR				0x00000640
+
+#define VIVS_FE_INDEX_STREAM_BASE_ADDR				0x00000644
+
+#define VIVS_FE_INDEX_STREAM_CONTROL				0x00000648
+#define VIVS_FE_INDEX_STREAM_CONTROL_TYPE__MASK			0x00000003
+#define VIVS_FE_INDEX_STREAM_CONTROL_TYPE__SHIFT		0
+#define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_CHAR		0x00000000
+#define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_SHORT	0x00000001
+#define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_INT		0x00000002
+
+#define VIVS_FE_VERTEX_STREAM_BASE_ADDR				0x0000064c
+
+#define VIVS_FE_VERTEX_STREAM_CONTROL				0x00000650
+
+#define VIVS_FE_COMMAND_ADDRESS					0x00000654
+
+#define VIVS_FE_COMMAND_CONTROL					0x00000658
+#define VIVS_FE_COMMAND_CONTROL_PREFETCH__MASK			0x0000ffff
+#define VIVS_FE_COMMAND_CONTROL_PREFETCH__SHIFT			0
+#define VIVS_FE_COMMAND_CONTROL_PREFETCH(x)			(((x) << VIVS_FE_COMMAND_CONTROL_PREFETCH__SHIFT) & VIVS_FE_COMMAND_CONTROL_PREFETCH__MASK)
+#define VIVS_FE_COMMAND_CONTROL_ENABLE				0x00010000
+
+#define VIVS_FE_DMA_STATUS					0x0000065c
+
+#define VIVS_FE_DMA_DEBUG_STATE					0x00000660
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE__MASK			0x0000001f
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE__SHIFT		0
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_IDLE			0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_DEC			0x00000001
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_ADR0			0x00000002
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_LOAD0			0x00000003
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_ADR1			0x00000004
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_LOAD1			0x00000005
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_3DADR			0x00000006
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_3DCMD			0x00000007
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_3DCNTL		0x00000008
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_3DIDXCNTL		0x00000009
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_INITREQDMA		0x0000000a
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_DRAWIDX		0x0000000b
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_DRAW			0x0000000c
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_2DRECT0		0x0000000d
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_2DRECT1		0x0000000e
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_2DDATA0		0x0000000f
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_2DDATA1		0x00000010
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_WAITFIFO		0x00000011
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_WAIT			0x00000012
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_LINK			0x00000013
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_END			0x00000014
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_STATE_STALL			0x00000015
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE__MASK		0x00000300
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE__SHIFT		8
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE_IDLE		0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE_START		0x00000100
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE_REQ		0x00000200
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_DMA_STATE_END		0x00000300
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_FETCH_STATE__MASK		0x00000c00
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_FETCH_STATE__SHIFT		10
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_FETCH_STATE_IDLE		0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_FETCH_STATE_RAMVALID	0x00000400
+#define VIVS_FE_DMA_DEBUG_STATE_CMD_FETCH_STATE_VALID		0x00000800
+#define VIVS_FE_DMA_DEBUG_STATE_REQ_DMA_STATE__MASK		0x00003000
+#define VIVS_FE_DMA_DEBUG_STATE_REQ_DMA_STATE__SHIFT		12
+#define VIVS_FE_DMA_DEBUG_STATE_REQ_DMA_STATE_IDLE		0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_REQ_DMA_STATE_WAITIDX		0x00001000
+#define VIVS_FE_DMA_DEBUG_STATE_REQ_DMA_STATE_CAL		0x00002000
+#define VIVS_FE_DMA_DEBUG_STATE_CAL_STATE__MASK			0x0000c000
+#define VIVS_FE_DMA_DEBUG_STATE_CAL_STATE__SHIFT		14
+#define VIVS_FE_DMA_DEBUG_STATE_CAL_STATE_IDLE			0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_CAL_STATE_LDADR			0x00004000
+#define VIVS_FE_DMA_DEBUG_STATE_CAL_STATE_IDXCALC		0x00008000
+#define VIVS_FE_DMA_DEBUG_STATE_VE_REQ_STATE__MASK		0x00030000
+#define VIVS_FE_DMA_DEBUG_STATE_VE_REQ_STATE__SHIFT		16
+#define VIVS_FE_DMA_DEBUG_STATE_VE_REQ_STATE_IDLE		0x00000000
+#define VIVS_FE_DMA_DEBUG_STATE_VE_REQ_STATE_CKCACHE		0x00010000
+#define VIVS_FE_DMA_DEBUG_STATE_VE_REQ_STATE_MISS		0x00020000
+
+#define VIVS_FE_DMA_ADDRESS					0x00000664
+
+#define VIVS_FE_DMA_LOW						0x00000668
+
+#define VIVS_FE_DMA_HIGH					0x0000066c
+
+#define VIVS_FE_AUTO_FLUSH					0x00000670
+
+#define VIVS_FE_UNK00678					0x00000678
+
+#define VIVS_FE_UNK0067C					0x0000067c
+
+#define VIVS_FE_VERTEX_STREAMS(i0)			       (0x00000000 + 0x4*(i0))
+#define VIVS_FE_VERTEX_STREAMS__ESIZE				0x00000004
+#define VIVS_FE_VERTEX_STREAMS__LEN				0x00000008
+
+#define VIVS_FE_VERTEX_STREAMS_BASE_ADDR(i0)		       (0x00000680 + 0x4*(i0))
+
+#define VIVS_FE_VERTEX_STREAMS_CONTROL(i0)		       (0x000006a0 + 0x4*(i0))
+
+#define VIVS_FE_UNK00700(i0)				       (0x00000700 + 0x4*(i0))
+#define VIVS_FE_UNK00700__ESIZE					0x00000004
+#define VIVS_FE_UNK00700__LEN					0x00000010
+
+#define VIVS_FE_UNK00740(i0)				       (0x00000740 + 0x4*(i0))
+#define VIVS_FE_UNK00740__ESIZE					0x00000004
+#define VIVS_FE_UNK00740__LEN					0x00000010
+
+#define VIVS_FE_UNK00780(i0)				       (0x00000780 + 0x4*(i0))
+#define VIVS_FE_UNK00780__ESIZE					0x00000004
+#define VIVS_FE_UNK00780__LEN					0x00000010
+
+#define VIVS_GL							0x00000000
+
+#define VIVS_GL_PIPE_SELECT					0x00003800
+#define VIVS_GL_PIPE_SELECT_PIPE__MASK				0x00000001
+#define VIVS_GL_PIPE_SELECT_PIPE__SHIFT				0
+#define VIVS_GL_PIPE_SELECT_PIPE(x)				(((x) << VIVS_GL_PIPE_SELECT_PIPE__SHIFT) & VIVS_GL_PIPE_SELECT_PIPE__MASK)
+
+#define VIVS_GL_EVENT						0x00003804
+#define VIVS_GL_EVENT_EVENT_ID__MASK				0x0000001f
+#define VIVS_GL_EVENT_EVENT_ID__SHIFT				0
+#define VIVS_GL_EVENT_EVENT_ID(x)				(((x) << VIVS_GL_EVENT_EVENT_ID__SHIFT) & VIVS_GL_EVENT_EVENT_ID__MASK)
+#define VIVS_GL_EVENT_FROM_FE					0x00000020
+#define VIVS_GL_EVENT_FROM_PE					0x00000040
+#define VIVS_GL_EVENT_SOURCE__MASK				0x00001f00
+#define VIVS_GL_EVENT_SOURCE__SHIFT				8
+#define VIVS_GL_EVENT_SOURCE(x)					(((x) << VIVS_GL_EVENT_SOURCE__SHIFT) & VIVS_GL_EVENT_SOURCE__MASK)
+
+#define VIVS_GL_SEMAPHORE_TOKEN					0x00003808
+#define VIVS_GL_SEMAPHORE_TOKEN_FROM__MASK			0x0000001f
+#define VIVS_GL_SEMAPHORE_TOKEN_FROM__SHIFT			0
+#define VIVS_GL_SEMAPHORE_TOKEN_FROM(x)				(((x) << VIVS_GL_SEMAPHORE_TOKEN_FROM__SHIFT) & VIVS_GL_SEMAPHORE_TOKEN_FROM__MASK)
+#define VIVS_GL_SEMAPHORE_TOKEN_TO__MASK			0x00001f00
+#define VIVS_GL_SEMAPHORE_TOKEN_TO__SHIFT			8
+#define VIVS_GL_SEMAPHORE_TOKEN_TO(x)				(((x) << VIVS_GL_SEMAPHORE_TOKEN_TO__SHIFT) & VIVS_GL_SEMAPHORE_TOKEN_TO__MASK)
+
+#define VIVS_GL_FLUSH_CACHE					0x0000380c
+#define VIVS_GL_FLUSH_CACHE_DEPTH				0x00000001
+#define VIVS_GL_FLUSH_CACHE_COLOR				0x00000002
+#define VIVS_GL_FLUSH_CACHE_TEXTURE				0x00000004
+#define VIVS_GL_FLUSH_CACHE_PE2D				0x00000008
+#define VIVS_GL_FLUSH_CACHE_TEXTUREVS				0x00000010
+#define VIVS_GL_FLUSH_CACHE_SHADER_L1				0x00000020
+#define VIVS_GL_FLUSH_CACHE_SHADER_L2				0x00000040
+
+#define VIVS_GL_FLUSH_MMU					0x00003810
+#define VIVS_GL_FLUSH_MMU_FLUSH_FEMMU				0x00000001
+#define VIVS_GL_FLUSH_MMU_FLUSH_UNK1				0x00000002
+#define VIVS_GL_FLUSH_MMU_FLUSH_UNK2				0x00000004
+#define VIVS_GL_FLUSH_MMU_FLUSH_PEMMU				0x00000008
+#define VIVS_GL_FLUSH_MMU_FLUSH_UNK4				0x00000010
+
+#define VIVS_GL_VERTEX_ELEMENT_CONFIG				0x00003814
+
+#define VIVS_GL_MULTI_SAMPLE_CONFIG				0x00003818
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES__MASK		0x00000003
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES__SHIFT		0
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES_NONE		0x00000000
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES_2X		0x00000001
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES_4X		0x00000002
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_SAMPLES_MASK		0x00000008
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES__MASK		0x000000f0
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES__SHIFT		4
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES(x)		(((x) << VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES__SHIFT) & VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES__MASK)
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_MSAA_ENABLES_MASK		0x00000100
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12__MASK			0x00007000
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12__SHIFT		12
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12(x)			(((x) << VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12__SHIFT) & VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12__MASK)
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK12_MASK			0x00008000
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16__MASK			0x00030000
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16__SHIFT		16
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16(x)			(((x) << VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16__SHIFT) & VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16__MASK)
+#define VIVS_GL_MULTI_SAMPLE_CONFIG_UNK16_MASK			0x00080000
+
+#define VIVS_GL_VARYING_TOTAL_COMPONENTS			0x0000381c
+#define VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__MASK		0x000000ff
+#define VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__SHIFT		0
+#define VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM(x)			(((x) << VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__SHIFT) & VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__MASK)
+
+#define VIVS_GL_VARYING_NUM_COMPONENTS				0x00003820
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__MASK		0x00000007
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__SHIFT		0
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__MASK		0x00000070
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__SHIFT		4
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__MASK		0x00000700
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__SHIFT		8
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__MASK		0x00007000
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__SHIFT		12
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__MASK		0x00070000
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__SHIFT		16
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__MASK		0x00700000
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__SHIFT		20
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__MASK		0x07000000
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__SHIFT		24
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__MASK)
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__MASK		0x70000000
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__SHIFT		28
+#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7(x)			(((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__MASK)
+
+#define VIVS_GL_VARYING_COMPONENT_USE(i0)		       (0x00003828 + 0x4*(i0))
+#define VIVS_GL_VARYING_COMPONENT_USE__ESIZE			0x00000004
+#define VIVS_GL_VARYING_COMPONENT_USE__LEN			0x00000002
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP0__MASK		0x00000003
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP0__SHIFT		0
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP0(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP0__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP0__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP1__MASK		0x0000000c
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP1__SHIFT		2
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP1(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP1__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP1__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP2__MASK		0x00000030
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP2__SHIFT		4
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP2(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP2__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP2__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP3__MASK		0x000000c0
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP3__SHIFT		6
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP3(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP3__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP3__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP4__MASK		0x00000300
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP4__SHIFT		8
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP4(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP4__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP4__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP5__MASK		0x00000c00
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP5__SHIFT		10
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP5(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP5__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP5__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP6__MASK		0x00003000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP6__SHIFT		12
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP6(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP6__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP6__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP7__MASK		0x0000c000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP7__SHIFT		14
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP7(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP7__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP7__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP8__MASK		0x00030000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP8__SHIFT		16
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP8(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP8__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP8__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP9__MASK		0x000c0000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP9__SHIFT		18
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP9(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP9__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP9__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP10__MASK		0x00300000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP10__SHIFT		20
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP10(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP10__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP10__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP11__MASK		0x00c00000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP11__SHIFT		22
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP11(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP11__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP11__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP12__MASK		0x03000000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP12__SHIFT		24
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP12(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP12__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP12__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP13__MASK		0x0c000000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP13__SHIFT		26
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP13(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP13__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP13__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP14__MASK		0x30000000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP14__SHIFT		28
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP14(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP14__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP14__MASK)
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP15__MASK		0xc0000000
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP15__SHIFT		30
+#define VIVS_GL_VARYING_COMPONENT_USE_COMP15(x)			(((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP15__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP15__MASK)
+
+#define VIVS_GL_UNK03834					0x00003834
+
+#define VIVS_GL_UNK03838					0x00003838
+
+#define VIVS_GL_API_MODE					0x0000384c
+#define VIVS_GL_API_MODE_OPENGL					0x00000000
+#define VIVS_GL_API_MODE_OPENVG					0x00000001
+#define VIVS_GL_API_MODE_OPENCL					0x00000002
+
+#define VIVS_GL_CONTEXT_POINTER					0x00003850
+
+#define VIVS_GL_UNK03A00					0x00003a00
+
+#define VIVS_GL_STALL_TOKEN					0x00003c00
+#define VIVS_GL_STALL_TOKEN_FROM__MASK				0x0000001f
+#define VIVS_GL_STALL_TOKEN_FROM__SHIFT				0
+#define VIVS_GL_STALL_TOKEN_FROM(x)				(((x) << VIVS_GL_STALL_TOKEN_FROM__SHIFT) & VIVS_GL_STALL_TOKEN_FROM__MASK)
+#define VIVS_GL_STALL_TOKEN_TO__MASK				0x00001f00
+#define VIVS_GL_STALL_TOKEN_TO__SHIFT				8
+#define VIVS_GL_STALL_TOKEN_TO(x)				(((x) << VIVS_GL_STALL_TOKEN_TO__SHIFT) & VIVS_GL_STALL_TOKEN_TO__MASK)
+#define VIVS_GL_STALL_TOKEN_FLIP0				0x40000000
+#define VIVS_GL_STALL_TOKEN_FLIP1				0x80000000
+
+#define VIVS_DUMMY						0x00000000
+
+#define VIVS_DUMMY_DUMMY					0x0003fffc
+
+
+#endif /* STATE_XML */
diff --git a/drivers/gpu/drm/etnaviv/state_hi.xml.h b/drivers/gpu/drm/etnaviv/state_hi.xml.h
new file mode 100644
index 000000000000..6a7de5f1454a
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/state_hi.xml.h
@@ -0,0 +1,429 @@
+#ifndef STATE_HI_XML
+#define STATE_HI_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://0x04.net/cgit/index.cgi/rules-ng-ng
+git clone git://0x04.net/rules-ng-ng
+
+The rules-ng-ng source files this header was generated from are:
+- state_hi.xml (  24309 bytes, from 2015-12-12 09:02:53)
+- common.xml   (  18437 bytes, from 2015-12-12 09:02:53)
+
+Copyright (C) 2015
+*/
+
+
+#define MMU_EXCEPTION_SLAVE_NOT_PRESENT				0x00000001
+#define MMU_EXCEPTION_PAGE_NOT_PRESENT				0x00000002
+#define MMU_EXCEPTION_WRITE_VIOLATION				0x00000003
+#define VIVS_HI							0x00000000
+
+#define VIVS_HI_CLOCK_CONTROL					0x00000000
+#define VIVS_HI_CLOCK_CONTROL_CLK3D_DIS				0x00000001
+#define VIVS_HI_CLOCK_CONTROL_CLK2D_DIS				0x00000002
+#define VIVS_HI_CLOCK_CONTROL_FSCALE_VAL__MASK			0x000001fc
+#define VIVS_HI_CLOCK_CONTROL_FSCALE_VAL__SHIFT			2
+#define VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(x)			(((x) << VIVS_HI_CLOCK_CONTROL_FSCALE_VAL__SHIFT) & VIVS_HI_CLOCK_CONTROL_FSCALE_VAL__MASK)
+#define VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD			0x00000200
+#define VIVS_HI_CLOCK_CONTROL_DISABLE_RAM_CLK_GATING		0x00000400
+#define VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS		0x00000800
+#define VIVS_HI_CLOCK_CONTROL_SOFT_RESET			0x00001000
+#define VIVS_HI_CLOCK_CONTROL_IDLE_3D				0x00010000
+#define VIVS_HI_CLOCK_CONTROL_IDLE_2D				0x00020000
+#define VIVS_HI_CLOCK_CONTROL_IDLE_VG				0x00040000
+#define VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU			0x00080000
+#define VIVS_HI_CLOCK_CONTROL_DEBUG_PIXEL_PIPE__MASK		0x00f00000
+#define VIVS_HI_CLOCK_CONTROL_DEBUG_PIXEL_PIPE__SHIFT		20
+#define VIVS_HI_CLOCK_CONTROL_DEBUG_PIXEL_PIPE(x)		(((x) << VIVS_HI_CLOCK_CONTROL_DEBUG_PIXEL_PIPE__SHIFT) & VIVS_HI_CLOCK_CONTROL_DEBUG_PIXEL_PIPE__MASK)
+
+#define VIVS_HI_IDLE_STATE					0x00000004
+#define VIVS_HI_IDLE_STATE_FE					0x00000001
+#define VIVS_HI_IDLE_STATE_DE					0x00000002
+#define VIVS_HI_IDLE_STATE_PE					0x00000004
+#define VIVS_HI_IDLE_STATE_SH					0x00000008
+#define VIVS_HI_IDLE_STATE_PA					0x00000010
+#define VIVS_HI_IDLE_STATE_SE					0x00000020
+#define VIVS_HI_IDLE_STATE_RA					0x00000040
+#define VIVS_HI_IDLE_STATE_TX					0x00000080
+#define VIVS_HI_IDLE_STATE_VG					0x00000100
+#define VIVS_HI_IDLE_STATE_IM					0x00000200
+#define VIVS_HI_IDLE_STATE_FP					0x00000400
+#define VIVS_HI_IDLE_STATE_TS					0x00000800
+#define VIVS_HI_IDLE_STATE_AXI_LP				0x80000000
+
+#define VIVS_HI_AXI_CONFIG					0x00000008
+#define VIVS_HI_AXI_CONFIG_AWID__MASK				0x0000000f
+#define VIVS_HI_AXI_CONFIG_AWID__SHIFT				0
+#define VIVS_HI_AXI_CONFIG_AWID(x)				(((x) << VIVS_HI_AXI_CONFIG_AWID__SHIFT) & VIVS_HI_AXI_CONFIG_AWID__MASK)
+#define VIVS_HI_AXI_CONFIG_ARID__MASK				0x000000f0
+#define VIVS_HI_AXI_CONFIG_ARID__SHIFT				4
+#define VIVS_HI_AXI_CONFIG_ARID(x)				(((x) << VIVS_HI_AXI_CONFIG_ARID__SHIFT) & VIVS_HI_AXI_CONFIG_ARID__MASK)
+#define VIVS_HI_AXI_CONFIG_AWCACHE__MASK			0x00000f00
+#define VIVS_HI_AXI_CONFIG_AWCACHE__SHIFT			8
+#define VIVS_HI_AXI_CONFIG_AWCACHE(x)				(((x) << VIVS_HI_AXI_CONFIG_AWCACHE__SHIFT) & VIVS_HI_AXI_CONFIG_AWCACHE__MASK)
+#define VIVS_HI_AXI_CONFIG_ARCACHE__MASK			0x0000f000
+#define VIVS_HI_AXI_CONFIG_ARCACHE__SHIFT			12
+#define VIVS_HI_AXI_CONFIG_ARCACHE(x)				(((x) << VIVS_HI_AXI_CONFIG_ARCACHE__SHIFT) & VIVS_HI_AXI_CONFIG_ARCACHE__MASK)
+
+#define VIVS_HI_AXI_STATUS					0x0000000c
+#define VIVS_HI_AXI_STATUS_WR_ERR_ID__MASK			0x0000000f
+#define VIVS_HI_AXI_STATUS_WR_ERR_ID__SHIFT			0
+#define VIVS_HI_AXI_STATUS_WR_ERR_ID(x)				(((x) << VIVS_HI_AXI_STATUS_WR_ERR_ID__SHIFT) & VIVS_HI_AXI_STATUS_WR_ERR_ID__MASK)
+#define VIVS_HI_AXI_STATUS_RD_ERR_ID__MASK			0x000000f0
+#define VIVS_HI_AXI_STATUS_RD_ERR_ID__SHIFT			4
+#define VIVS_HI_AXI_STATUS_RD_ERR_ID(x)				(((x) << VIVS_HI_AXI_STATUS_RD_ERR_ID__SHIFT) & VIVS_HI_AXI_STATUS_RD_ERR_ID__MASK)
+#define VIVS_HI_AXI_STATUS_DET_WR_ERR				0x00000100
+#define VIVS_HI_AXI_STATUS_DET_RD_ERR				0x00000200
+
+#define VIVS_HI_INTR_ACKNOWLEDGE				0x00000010
+#define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__MASK			0x7fffffff
+#define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__SHIFT		0
+#define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC(x)			(((x) << VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__SHIFT) & VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__MASK)
+#define VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR			0x80000000
+
+#define VIVS_HI_INTR_ENBL					0x00000014
+#define VIVS_HI_INTR_ENBL_INTR_ENBL_VEC__MASK			0xffffffff
+#define VIVS_HI_INTR_ENBL_INTR_ENBL_VEC__SHIFT			0
+#define VIVS_HI_INTR_ENBL_INTR_ENBL_VEC(x)			(((x) << VIVS_HI_INTR_ENBL_INTR_ENBL_VEC__SHIFT) & VIVS_HI_INTR_ENBL_INTR_ENBL_VEC__MASK)
+
+#define VIVS_HI_CHIP_IDENTITY					0x00000018
+#define VIVS_HI_CHIP_IDENTITY_FAMILY__MASK			0xff000000
+#define VIVS_HI_CHIP_IDENTITY_FAMILY__SHIFT			24
+#define VIVS_HI_CHIP_IDENTITY_FAMILY(x)				(((x) << VIVS_HI_CHIP_IDENTITY_FAMILY__SHIFT) & VIVS_HI_CHIP_IDENTITY_FAMILY__MASK)
+#define VIVS_HI_CHIP_IDENTITY_PRODUCT__MASK			0x00ff0000
+#define VIVS_HI_CHIP_IDENTITY_PRODUCT__SHIFT			16
+#define VIVS_HI_CHIP_IDENTITY_PRODUCT(x)			(((x) << VIVS_HI_CHIP_IDENTITY_PRODUCT__SHIFT) & VIVS_HI_CHIP_IDENTITY_PRODUCT__MASK)
+#define VIVS_HI_CHIP_IDENTITY_REVISION__MASK			0x0000f000
+#define VIVS_HI_CHIP_IDENTITY_REVISION__SHIFT			12
+#define VIVS_HI_CHIP_IDENTITY_REVISION(x)			(((x) << VIVS_HI_CHIP_IDENTITY_REVISION__SHIFT) & VIVS_HI_CHIP_IDENTITY_REVISION__MASK)
+
+#define VIVS_HI_CHIP_FEATURE					0x0000001c
+
+#define VIVS_HI_CHIP_MODEL					0x00000020
+
+#define VIVS_HI_CHIP_REV					0x00000024
+
+#define VIVS_HI_CHIP_DATE					0x00000028
+
+#define VIVS_HI_CHIP_TIME					0x0000002c
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_0				0x00000034
+
+#define VIVS_HI_CACHE_CONTROL					0x00000038
+
+#define VIVS_HI_MEMORY_COUNTER_RESET				0x0000003c
+
+#define VIVS_HI_PROFILE_READ_BYTES8				0x00000040
+
+#define VIVS_HI_PROFILE_WRITE_BYTES8				0x00000044
+
+#define VIVS_HI_CHIP_SPECS					0x00000048
+#define VIVS_HI_CHIP_SPECS_STREAM_COUNT__MASK			0x0000000f
+#define VIVS_HI_CHIP_SPECS_STREAM_COUNT__SHIFT			0
+#define VIVS_HI_CHIP_SPECS_STREAM_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_STREAM_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_STREAM_COUNT__MASK)
+#define VIVS_HI_CHIP_SPECS_REGISTER_MAX__MASK			0x000000f0
+#define VIVS_HI_CHIP_SPECS_REGISTER_MAX__SHIFT			4
+#define VIVS_HI_CHIP_SPECS_REGISTER_MAX(x)			(((x) << VIVS_HI_CHIP_SPECS_REGISTER_MAX__SHIFT) & VIVS_HI_CHIP_SPECS_REGISTER_MAX__MASK)
+#define VIVS_HI_CHIP_SPECS_THREAD_COUNT__MASK			0x00000f00
+#define VIVS_HI_CHIP_SPECS_THREAD_COUNT__SHIFT			8
+#define VIVS_HI_CHIP_SPECS_THREAD_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_THREAD_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_THREAD_COUNT__MASK)
+#define VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__MASK		0x0001f000
+#define VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__SHIFT		12
+#define VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE(x)			(((x) << VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__SHIFT) & VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE__MASK)
+#define VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__MASK		0x01f00000
+#define VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__SHIFT		20
+#define VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT__MASK)
+#define VIVS_HI_CHIP_SPECS_PIXEL_PIPES__MASK			0x0e000000
+#define VIVS_HI_CHIP_SPECS_PIXEL_PIPES__SHIFT			25
+#define VIVS_HI_CHIP_SPECS_PIXEL_PIPES(x)			(((x) << VIVS_HI_CHIP_SPECS_PIXEL_PIPES__SHIFT) & VIVS_HI_CHIP_SPECS_PIXEL_PIPES__MASK)
+#define VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__MASK	0xf0000000
+#define VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__SHIFT	28
+#define VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE(x)		(((x) << VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__SHIFT) & VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE__MASK)
+
+#define VIVS_HI_PROFILE_WRITE_BURSTS				0x0000004c
+
+#define VIVS_HI_PROFILE_WRITE_REQUESTS				0x00000050
+
+#define VIVS_HI_PROFILE_READ_BURSTS				0x00000058
+
+#define VIVS_HI_PROFILE_READ_REQUESTS				0x0000005c
+
+#define VIVS_HI_PROFILE_READ_LASTS				0x00000060
+
+#define VIVS_HI_GP_OUT0						0x00000064
+
+#define VIVS_HI_GP_OUT1						0x00000068
+
+#define VIVS_HI_GP_OUT2						0x0000006c
+
+#define VIVS_HI_AXI_CONTROL					0x00000070
+#define VIVS_HI_AXI_CONTROL_WR_FULL_BURST_MODE			0x00000001
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_1				0x00000074
+
+#define VIVS_HI_PROFILE_TOTAL_CYCLES				0x00000078
+
+#define VIVS_HI_PROFILE_IDLE_CYCLES				0x0000007c
+
+#define VIVS_HI_CHIP_SPECS_2					0x00000080
+#define VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__MASK			0x000000ff
+#define VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__SHIFT			0
+#define VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE(x)			(((x) << VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__SHIFT) & VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE__MASK)
+#define VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__MASK		0x0000ff00
+#define VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__SHIFT		8
+#define VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT(x)		(((x) << VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT__MASK)
+#define VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__MASK		0xffff0000
+#define VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__SHIFT		16
+#define VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS(x)			(((x) << VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__SHIFT) & VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS__MASK)
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_2				0x00000084
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_3				0x00000088
+
+#define VIVS_HI_CHIP_SPECS_3					0x0000008c
+#define VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT__MASK		0x000001f0
+#define VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT__SHIFT		4
+#define VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT__MASK)
+#define VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__MASK		0x00000007
+#define VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__SHIFT		0
+#define VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__MASK)
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_4				0x00000094
+
+#define VIVS_HI_CHIP_SPECS_4					0x0000009c
+#define VIVS_HI_CHIP_SPECS_4_STREAM_COUNT__MASK			0x0001f000
+#define VIVS_HI_CHIP_SPECS_4_STREAM_COUNT__SHIFT		12
+#define VIVS_HI_CHIP_SPECS_4_STREAM_COUNT(x)			(((x) << VIVS_HI_CHIP_SPECS_4_STREAM_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_4_STREAM_COUNT__MASK)
+
+#define VIVS_HI_CHIP_MINOR_FEATURE_5				0x000000a0
+
+#define VIVS_HI_CHIP_PRODUCT_ID					0x000000a8
+
+#define VIVS_PM							0x00000000
+
+#define VIVS_PM_POWER_CONTROLS					0x00000100
+#define VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING	0x00000001
+#define VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING	0x00000002
+#define VIVS_PM_POWER_CONTROLS_DISABLE_STARVE_MODULE_CLOCK_GATING	0x00000004
+#define VIVS_PM_POWER_CONTROLS_TURN_ON_COUNTER__MASK		0x000000f0
+#define VIVS_PM_POWER_CONTROLS_TURN_ON_COUNTER__SHIFT		4
+#define VIVS_PM_POWER_CONTROLS_TURN_ON_COUNTER(x)		(((x) << VIVS_PM_POWER_CONTROLS_TURN_ON_COUNTER__SHIFT) & VIVS_PM_POWER_CONTROLS_TURN_ON_COUNTER__MASK)
+#define VIVS_PM_POWER_CONTROLS_TURN_OFF_COUNTER__MASK		0xffff0000
+#define VIVS_PM_POWER_CONTROLS_TURN_OFF_COUNTER__SHIFT		16
+#define VIVS_PM_POWER_CONTROLS_TURN_OFF_COUNTER(x)		(((x) << VIVS_PM_POWER_CONTROLS_TURN_OFF_COUNTER__SHIFT) & VIVS_PM_POWER_CONTROLS_TURN_OFF_COUNTER__MASK)
+
+#define VIVS_PM_MODULE_CONTROLS					0x00000104
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_FE	0x00000001
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_DE	0x00000002
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE	0x00000004
+
+#define VIVS_PM_MODULE_STATUS					0x00000108
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_FE		0x00000001
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_DE		0x00000002
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_PE		0x00000004
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_SH		0x00000008
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_PA		0x00000010
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_SE		0x00000020
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_RA		0x00000040
+#define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_TX		0x00000080
+
+#define VIVS_PM_PULSE_EATER					0x0000010c
+
+#define VIVS_MMUv2						0x00000000
+
+#define VIVS_MMUv2_SAFE_ADDRESS					0x00000180
+
+#define VIVS_MMUv2_CONFIGURATION				0x00000184
+#define VIVS_MMUv2_CONFIGURATION_MODE__MASK			0x00000001
+#define VIVS_MMUv2_CONFIGURATION_MODE__SHIFT			0
+#define VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K			0x00000000
+#define VIVS_MMUv2_CONFIGURATION_MODE_MODE1_K			0x00000001
+#define VIVS_MMUv2_CONFIGURATION_MODE_MASK			0x00000008
+#define VIVS_MMUv2_CONFIGURATION_FLUSH__MASK			0x00000010
+#define VIVS_MMUv2_CONFIGURATION_FLUSH__SHIFT			4
+#define VIVS_MMUv2_CONFIGURATION_FLUSH_FLUSH			0x00000010
+#define VIVS_MMUv2_CONFIGURATION_FLUSH_MASK			0x00000080
+#define VIVS_MMUv2_CONFIGURATION_ADDRESS_MASK			0x00000100
+#define VIVS_MMUv2_CONFIGURATION_ADDRESS__MASK			0xfffffc00
+#define VIVS_MMUv2_CONFIGURATION_ADDRESS__SHIFT			10
+#define VIVS_MMUv2_CONFIGURATION_ADDRESS(x)			(((x) << VIVS_MMUv2_CONFIGURATION_ADDRESS__SHIFT) & VIVS_MMUv2_CONFIGURATION_ADDRESS__MASK)
+
+#define VIVS_MMUv2_STATUS					0x00000188
+#define VIVS_MMUv2_STATUS_EXCEPTION0__MASK			0x00000003
+#define VIVS_MMUv2_STATUS_EXCEPTION0__SHIFT			0
+#define VIVS_MMUv2_STATUS_EXCEPTION0(x)				(((x) << VIVS_MMUv2_STATUS_EXCEPTION0__SHIFT) & VIVS_MMUv2_STATUS_EXCEPTION0__MASK)
+#define VIVS_MMUv2_STATUS_EXCEPTION1__MASK			0x00000030
+#define VIVS_MMUv2_STATUS_EXCEPTION1__SHIFT			4
+#define VIVS_MMUv2_STATUS_EXCEPTION1(x)				(((x) << VIVS_MMUv2_STATUS_EXCEPTION1__SHIFT) & VIVS_MMUv2_STATUS_EXCEPTION1__MASK)
+#define VIVS_MMUv2_STATUS_EXCEPTION2__MASK			0x00000300
+#define VIVS_MMUv2_STATUS_EXCEPTION2__SHIFT			8
+#define VIVS_MMUv2_STATUS_EXCEPTION2(x)				(((x) << VIVS_MMUv2_STATUS_EXCEPTION2__SHIFT) & VIVS_MMUv2_STATUS_EXCEPTION2__MASK)
+#define VIVS_MMUv2_STATUS_EXCEPTION3__MASK			0x00003000
+#define VIVS_MMUv2_STATUS_EXCEPTION3__SHIFT			12
+#define VIVS_MMUv2_STATUS_EXCEPTION3(x)				(((x) << VIVS_MMUv2_STATUS_EXCEPTION3__SHIFT) & VIVS_MMUv2_STATUS_EXCEPTION3__MASK)
+
+#define VIVS_MMUv2_CONTROL					0x0000018c
+#define VIVS_MMUv2_CONTROL_ENABLE				0x00000001
+
+#define VIVS_MMUv2_EXCEPTION_ADDR(i0)			       (0x00000190 + 0x4*(i0))
+#define VIVS_MMUv2_EXCEPTION_ADDR__ESIZE			0x00000004
+#define VIVS_MMUv2_EXCEPTION_ADDR__LEN				0x00000004
+
+#define VIVS_MC							0x00000000
+
+#define VIVS_MC_MMU_FE_PAGE_TABLE				0x00000400
+
+#define VIVS_MC_MMU_TX_PAGE_TABLE				0x00000404
+
+#define VIVS_MC_MMU_PE_PAGE_TABLE				0x00000408
+
+#define VIVS_MC_MMU_PEZ_PAGE_TABLE				0x0000040c
+
+#define VIVS_MC_MMU_RA_PAGE_TABLE				0x00000410
+
+#define VIVS_MC_DEBUG_MEMORY					0x00000414
+#define VIVS_MC_DEBUG_MEMORY_SPECIAL_PATCH_GC320		0x00000008
+#define VIVS_MC_DEBUG_MEMORY_FAST_CLEAR_BYPASS			0x00100000
+#define VIVS_MC_DEBUG_MEMORY_COMPRESSION_BYPASS			0x00200000
+
+#define VIVS_MC_MEMORY_BASE_ADDR_RA				0x00000418
+
+#define VIVS_MC_MEMORY_BASE_ADDR_FE				0x0000041c
+
+#define VIVS_MC_MEMORY_BASE_ADDR_TX				0x00000420
+
+#define VIVS_MC_MEMORY_BASE_ADDR_PEZ				0x00000424
+
+#define VIVS_MC_MEMORY_BASE_ADDR_PE				0x00000428
+
+#define VIVS_MC_MEMORY_TIMING_CONTROL				0x0000042c
+
+#define VIVS_MC_MEMORY_FLUSH					0x00000430
+
+#define VIVS_MC_PROFILE_CYCLE_COUNTER				0x00000438
+
+#define VIVS_MC_DEBUG_READ0					0x0000043c
+
+#define VIVS_MC_DEBUG_READ1					0x00000440
+
+#define VIVS_MC_DEBUG_WRITE					0x00000444
+
+#define VIVS_MC_PROFILE_RA_READ					0x00000448
+
+#define VIVS_MC_PROFILE_TX_READ					0x0000044c
+
+#define VIVS_MC_PROFILE_FE_READ					0x00000450
+
+#define VIVS_MC_PROFILE_PE_READ					0x00000454
+
+#define VIVS_MC_PROFILE_DE_READ					0x00000458
+
+#define VIVS_MC_PROFILE_SH_READ					0x0000045c
+
+#define VIVS_MC_PROFILE_PA_READ					0x00000460
+
+#define VIVS_MC_PROFILE_SE_READ					0x00000464
+
+#define VIVS_MC_PROFILE_MC_READ					0x00000468
+
+#define VIVS_MC_PROFILE_HI_READ					0x0000046c
+
+#define VIVS_MC_PROFILE_CONFIG0					0x00000470
+#define VIVS_MC_PROFILE_CONFIG0_FE__MASK			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG0_FE__SHIFT			0
+#define VIVS_MC_PROFILE_CONFIG0_FE_RESET			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG0_DE__MASK			0x00000f00
+#define VIVS_MC_PROFILE_CONFIG0_DE__SHIFT			8
+#define VIVS_MC_PROFILE_CONFIG0_DE_RESET			0x00000f00
+#define VIVS_MC_PROFILE_CONFIG0_PE__MASK			0x000f0000
+#define VIVS_MC_PROFILE_CONFIG0_PE__SHIFT			16
+#define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_KILLED_BY_COLOR_PIPE	0x00000000
+#define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_KILLED_BY_DEPTH_PIPE	0x00010000
+#define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_DRAWN_BY_COLOR_PIPE	0x00020000
+#define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_DRAWN_BY_DEPTH_PIPE	0x00030000
+#define VIVS_MC_PROFILE_CONFIG0_PE_PIXELS_RENDERED_2D		0x000b0000
+#define VIVS_MC_PROFILE_CONFIG0_PE_RESET			0x000f0000
+#define VIVS_MC_PROFILE_CONFIG0_SH__MASK			0x0f000000
+#define VIVS_MC_PROFILE_CONFIG0_SH__SHIFT			24
+#define VIVS_MC_PROFILE_CONFIG0_SH_SHADER_CYCLES		0x04000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_PS_INST_COUNTER		0x07000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_RENDERED_PIXEL_COUNTER	0x08000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_VS_INST_COUNTER		0x09000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_RENDERED_VERTICE_COUNTER	0x0a000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_VTX_BRANCH_INST_COUNTER	0x0b000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_VTX_TEXLD_INST_COUNTER	0x0c000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_PXL_BRANCH_INST_COUNTER	0x0d000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_PXL_TEXLD_INST_COUNTER	0x0e000000
+#define VIVS_MC_PROFILE_CONFIG0_SH_RESET			0x0f000000
+
+#define VIVS_MC_PROFILE_CONFIG1					0x00000474
+#define VIVS_MC_PROFILE_CONFIG1_PA__MASK			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG1_PA__SHIFT			0
+#define VIVS_MC_PROFILE_CONFIG1_PA_INPUT_VTX_COUNTER		0x00000003
+#define VIVS_MC_PROFILE_CONFIG1_PA_INPUT_PRIM_COUNTER		0x00000004
+#define VIVS_MC_PROFILE_CONFIG1_PA_OUTPUT_PRIM_COUNTER		0x00000005
+#define VIVS_MC_PROFILE_CONFIG1_PA_DEPTH_CLIPPED_COUNTER	0x00000006
+#define VIVS_MC_PROFILE_CONFIG1_PA_TRIVIAL_REJECTED_COUNTER	0x00000007
+#define VIVS_MC_PROFILE_CONFIG1_PA_CULLED_COUNTER		0x00000008
+#define VIVS_MC_PROFILE_CONFIG1_PA_RESET			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG1_SE__MASK			0x00000f00
+#define VIVS_MC_PROFILE_CONFIG1_SE__SHIFT			8
+#define VIVS_MC_PROFILE_CONFIG1_SE_CULLED_TRIANGLE_COUNT	0x00000000
+#define VIVS_MC_PROFILE_CONFIG1_SE_CULLED_LINES_COUNT		0x00000100
+#define VIVS_MC_PROFILE_CONFIG1_SE_RESET			0x00000f00
+#define VIVS_MC_PROFILE_CONFIG1_RA__MASK			0x000f0000
+#define VIVS_MC_PROFILE_CONFIG1_RA__SHIFT			16
+#define VIVS_MC_PROFILE_CONFIG1_RA_VALID_PIXEL_COUNT		0x00000000
+#define VIVS_MC_PROFILE_CONFIG1_RA_TOTAL_QUAD_COUNT		0x00010000
+#define VIVS_MC_PROFILE_CONFIG1_RA_VALID_QUAD_COUNT_AFTER_EARLY_Z	0x00020000
+#define VIVS_MC_PROFILE_CONFIG1_RA_TOTAL_PRIMITIVE_COUNT	0x00030000
+#define VIVS_MC_PROFILE_CONFIG1_RA_PIPE_CACHE_MISS_COUNTER	0x00090000
+#define VIVS_MC_PROFILE_CONFIG1_RA_PREFETCH_CACHE_MISS_COUNTER	0x000a0000
+#define VIVS_MC_PROFILE_CONFIG1_RA_CULLED_QUAD_COUNT		0x000b0000
+#define VIVS_MC_PROFILE_CONFIG1_RA_RESET			0x000f0000
+#define VIVS_MC_PROFILE_CONFIG1_TX__MASK			0x0f000000
+#define VIVS_MC_PROFILE_CONFIG1_TX__SHIFT			24
+#define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_BILINEAR_REQUESTS	0x00000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_TRILINEAR_REQUESTS	0x01000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_DISCARDED_TEXTURE_REQUESTS	0x02000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_TEXTURE_REQUESTS	0x03000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_UNKNOWN			0x04000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_MEM_READ_COUNT		0x05000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_MEM_READ_IN_8B_COUNT		0x06000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_CACHE_MISS_COUNT		0x07000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_CACHE_HIT_TEXEL_COUNT	0x08000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_CACHE_MISS_TEXEL_COUNT	0x09000000
+#define VIVS_MC_PROFILE_CONFIG1_TX_RESET			0x0f000000
+
+#define VIVS_MC_PROFILE_CONFIG2					0x00000478
+#define VIVS_MC_PROFILE_CONFIG2_MC__MASK			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG2_MC__SHIFT			0
+#define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_READ_REQ_8B_FROM_PIPELINE	0x00000001
+#define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_READ_REQ_8B_FROM_IP	0x00000002
+#define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_WRITE_REQ_8B_FROM_PIPELINE	0x00000003
+#define VIVS_MC_PROFILE_CONFIG2_MC_RESET			0x0000000f
+#define VIVS_MC_PROFILE_CONFIG2_HI__MASK			0x00000f00
+#define VIVS_MC_PROFILE_CONFIG2_HI__SHIFT			8
+#define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_READ_REQUEST_STALLED	0x00000000
+#define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_WRITE_REQUEST_STALLED	0x00000100
+#define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_WRITE_DATA_STALLED	0x00000200
+#define VIVS_MC_PROFILE_CONFIG2_HI_RESET			0x00000f00
+
+#define VIVS_MC_PROFILE_CONFIG3					0x0000047c
+
+#define VIVS_MC_BUS_CONFIG					0x00000480
+#define VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK			0x0000000f
+#define VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__SHIFT			0
+#define VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(x)			(((x) << VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__SHIFT) & VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK)
+#define VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK			0x000000f0
+#define VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__SHIFT			4
+#define VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(x)			(((x) << VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__SHIFT) & VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK)
+
+#define VIVS_MC_START_COMPOSITION				0x00000554
+
+#define VIVS_MC_128B_MERGE					0x00000558
+
+
+#endif /* STATE_HI_XML */
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 96e86cf4455b..83efca941388 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -118,7 +118,7 @@ config DRM_EXYNOS_ROTATOR
 
 config DRM_EXYNOS_GSC
 	bool "GScaler"
-	depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && !ARCH_MULTIPLATFORM
+	depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && !VIDEO_SAMSUNG_EXYNOS_GSC
 	help
 	  Choose this option if you want to use Exynos GSC for DRM.
 
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index fbe1b3174f75..1bf6a21130c7 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -21,11 +21,11 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
+#include "exynos_drm_fb.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_iommu.h"
 
 #define WINDOWS_NR	3
-#define CURSOR_WIN	2
 #define MIN_FB_WIDTH_FOR_16WORD_BURST	128
 
 static const char * const decon_clks_name[] = {
@@ -56,6 +56,7 @@ struct decon_context {
 	struct drm_device		*drm_dev;
 	struct exynos_drm_crtc		*crtc;
 	struct exynos_drm_plane		planes[WINDOWS_NR];
+	struct exynos_drm_plane_config	configs[WINDOWS_NR];
 	void __iomem			*addr;
 	struct clk			*clks[ARRAY_SIZE(decon_clks_name)];
 	int				pipe;
@@ -71,6 +72,12 @@ static const uint32_t decon_formats[] = {
 	DRM_FORMAT_ARGB8888,
 };
 
+static const enum drm_plane_type decon_win_types[WINDOWS_NR] = {
+	DRM_PLANE_TYPE_PRIMARY,
+	DRM_PLANE_TYPE_OVERLAY,
+	DRM_PLANE_TYPE_CURSOR,
+};
+
 static inline void decon_set_bits(struct decon_context *ctx, u32 reg, u32 mask,
 				  u32 val)
 {
@@ -241,15 +248,16 @@ static void decon_shadow_protect_win(struct decon_context *ctx, int win,
 		       protect ? ~0 : 0);
 }
 
-static void decon_atomic_begin(struct exynos_drm_crtc *crtc,
-					struct exynos_drm_plane *plane)
+static void decon_atomic_begin(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
+	int i;
 
 	if (test_bit(BIT_SUSPENDED, &ctx->flags))
 		return;
 
-	decon_shadow_protect_win(ctx, plane->zpos, true);
+	for (i = ctx->first_win; i < WINDOWS_NR; i++)
+		decon_shadow_protect_win(ctx, i, true);
 }
 
 #define BIT_VAL(x, e, s) (((x) & ((1 << ((e) - (s) + 1)) - 1)) << (s))
@@ -259,21 +267,24 @@ static void decon_atomic_begin(struct exynos_drm_crtc *crtc,
 static void decon_update_plane(struct exynos_drm_crtc *crtc,
 			       struct exynos_drm_plane *plane)
 {
+	struct exynos_drm_plane_state *state =
+				to_exynos_plane_state(plane->base.state);
 	struct decon_context *ctx = crtc->ctx;
-	struct drm_plane_state *state = plane->base.state;
-	unsigned int win = plane->zpos;
-	unsigned int bpp = state->fb->bits_per_pixel >> 3;
-	unsigned int pitch = state->fb->pitches[0];
+	struct drm_framebuffer *fb = state->base.fb;
+	unsigned int win = plane->index;
+	unsigned int bpp = fb->bits_per_pixel >> 3;
+	unsigned int pitch = fb->pitches[0];
+	dma_addr_t dma_addr = exynos_drm_fb_dma_addr(fb, 0);
 	u32 val;
 
 	if (test_bit(BIT_SUSPENDED, &ctx->flags))
 		return;
 
-	val = COORDINATE_X(plane->crtc_x) | COORDINATE_Y(plane->crtc_y);
+	val = COORDINATE_X(state->crtc.x) | COORDINATE_Y(state->crtc.y);
 	writel(val, ctx->addr + DECON_VIDOSDxA(win));
 
-	val = COORDINATE_X(plane->crtc_x + plane->crtc_w - 1) |
-		COORDINATE_Y(plane->crtc_y + plane->crtc_h - 1);
+	val = COORDINATE_X(state->crtc.x + state->crtc.w - 1) |
+		COORDINATE_Y(state->crtc.y + state->crtc.h - 1);
 	writel(val, ctx->addr + DECON_VIDOSDxB(win));
 
 	val = VIDOSD_Wx_ALPHA_R_F(0x0) | VIDOSD_Wx_ALPHA_G_F(0x0) |
@@ -284,20 +295,20 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 		VIDOSD_Wx_ALPHA_B_F(0x0);
 	writel(val, ctx->addr + DECON_VIDOSDxD(win));
 
-	writel(plane->dma_addr[0], ctx->addr + DECON_VIDW0xADD0B0(win));
+	writel(dma_addr, ctx->addr + DECON_VIDW0xADD0B0(win));
 
-	val = plane->dma_addr[0] + pitch * plane->crtc_h;
+	val = dma_addr + pitch * state->src.h;
 	writel(val, ctx->addr + DECON_VIDW0xADD1B0(win));
 
 	if (ctx->out_type != IFTYPE_HDMI)
-		val = BIT_VAL(pitch - plane->crtc_w * bpp, 27, 14)
-			| BIT_VAL(plane->crtc_w * bpp, 13, 0);
+		val = BIT_VAL(pitch - state->crtc.w * bpp, 27, 14)
+			| BIT_VAL(state->crtc.w * bpp, 13, 0);
 	else
-		val = BIT_VAL(pitch - plane->crtc_w * bpp, 29, 15)
-			| BIT_VAL(plane->crtc_w * bpp, 14, 0);
+		val = BIT_VAL(pitch - state->crtc.w * bpp, 29, 15)
+			| BIT_VAL(state->crtc.w * bpp, 14, 0);
 	writel(val, ctx->addr + DECON_VIDW0xADD2(win));
 
-	decon_win_set_pixfmt(ctx, win, state->fb);
+	decon_win_set_pixfmt(ctx, win, fb);
 
 	/* window enable */
 	decon_set_bits(ctx, DECON_WINCONx(win), WINCONx_ENWIN_F, ~0);
@@ -310,7 +321,7 @@ static void decon_disable_plane(struct exynos_drm_crtc *crtc,
 				struct exynos_drm_plane *plane)
 {
 	struct decon_context *ctx = crtc->ctx;
-	unsigned int win = plane->zpos;
+	unsigned int win = plane->index;
 
 	if (test_bit(BIT_SUSPENDED, &ctx->flags))
 		return;
@@ -326,15 +337,16 @@ static void decon_disable_plane(struct exynos_drm_crtc *crtc,
 	decon_set_bits(ctx, DECON_UPDATE, STANDALONE_UPDATE_F, ~0);
 }
 
-static void decon_atomic_flush(struct exynos_drm_crtc *crtc,
-				struct exynos_drm_plane *plane)
+static void decon_atomic_flush(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
+	int i;
 
 	if (test_bit(BIT_SUSPENDED, &ctx->flags))
 		return;
 
-	decon_shadow_protect_win(ctx, plane->zpos, false);
+	for (i = ctx->first_win; i < WINDOWS_NR; i++)
+		decon_shadow_protect_win(ctx, i, false);
 
 	if (ctx->out_type == IFTYPE_I80)
 		set_bit(BIT_WIN_UPDATED, &ctx->flags);
@@ -377,20 +389,12 @@ static void decon_swreset(struct decon_context *ctx)
 static void decon_enable(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
-	int ret;
-	int i;
 
 	if (!test_and_clear_bit(BIT_SUSPENDED, &ctx->flags))
 		return;
 
 	pm_runtime_get_sync(ctx->dev);
 
-	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
-		ret = clk_prepare_enable(ctx->clks[i]);
-		if (ret < 0)
-			goto err;
-	}
-
 	set_bit(BIT_CLKS_ENABLED, &ctx->flags);
 
 	/* if vblank was enabled status, enable it again. */
@@ -399,11 +403,6 @@ static void decon_enable(struct exynos_drm_crtc *crtc)
 
 	decon_commit(ctx->crtc);
 
-	return;
-err:
-	while (--i >= 0)
-		clk_disable_unprepare(ctx->clks[i]);
-
 	set_bit(BIT_SUSPENDED, &ctx->flags);
 }
 
@@ -425,9 +424,6 @@ static void decon_disable(struct exynos_drm_crtc *crtc)
 
 	decon_swreset(ctx);
 
-	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++)
-		clk_disable_unprepare(ctx->clks[i]);
-
 	clear_bit(BIT_CLKS_ENABLED, &ctx->flags);
 
 	pm_runtime_put_sync(ctx->dev);
@@ -478,7 +474,6 @@ err:
 static struct exynos_drm_crtc_ops decon_crtc_ops = {
 	.enable			= decon_enable,
 	.disable		= decon_disable,
-	.commit			= decon_commit,
 	.enable_vblank		= decon_enable_vblank,
 	.disable_vblank		= decon_disable_vblank,
 	.atomic_begin		= decon_atomic_begin,
@@ -495,7 +490,6 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_private *priv = drm_dev->dev_private;
 	struct exynos_drm_plane *exynos_plane;
 	enum exynos_drm_output_type out_type;
-	enum drm_plane_type type;
 	unsigned int win;
 	int ret;
 
@@ -505,10 +499,13 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 	for (win = ctx->first_win; win < WINDOWS_NR; win++) {
 		int tmp = (win == ctx->first_win) ? 0 : win;
 
-		type = exynos_plane_get_type(tmp, CURSOR_WIN);
-		ret = exynos_plane_init(drm_dev, &ctx->planes[win],
-				1 << ctx->pipe, type, decon_formats,
-				ARRAY_SIZE(decon_formats), win);
+		ctx->configs[win].pixel_formats = decon_formats;
+		ctx->configs[win].num_pixel_formats = ARRAY_SIZE(decon_formats);
+		ctx->configs[win].zpos = win;
+		ctx->configs[win].type = decon_win_types[tmp];
+
+		ret = exynos_plane_init(drm_dev, &ctx->planes[win], win,
+					1 << ctx->pipe, &ctx->configs[win]);
 		if (ret)
 			return ret;
 	}
@@ -581,6 +578,44 @@ out:
 	return IRQ_HANDLED;
 }
 
+#ifdef CONFIG_PM
+static int exynos5433_decon_suspend(struct device *dev)
+{
+	struct decon_context *ctx = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++)
+		clk_disable_unprepare(ctx->clks[i]);
+
+	return 0;
+}
+
+static int exynos5433_decon_resume(struct device *dev)
+{
+	struct decon_context *ctx = dev_get_drvdata(dev);
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
+		ret = clk_prepare_enable(ctx->clks[i]);
+		if (ret < 0)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	while (--i >= 0)
+		clk_disable_unprepare(ctx->clks[i]);
+
+	return ret;
+}
+#endif
+
+static const struct dev_pm_ops exynos5433_decon_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos5433_decon_suspend, exynos5433_decon_resume,
+			   NULL)
+};
+
 static const struct of_device_id exynos5433_decon_driver_dt_match[] = {
 	{
 		.compatible = "samsung,exynos5433-decon",
@@ -684,6 +719,7 @@ struct platform_driver exynos5433_decon_driver = {
 	.remove		= exynos5433_decon_remove,
 	.driver		= {
 		.name	= "exynos5433-decon",
+		.pm	= &exynos5433_decon_pm_ops,
 		.of_match_table = exynos5433_decon_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index ead2b16e237d..52bda3b42fe0 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -30,6 +30,7 @@
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_drv.h"
+#include "exynos_drm_fb.h"
 #include "exynos_drm_fbdev.h"
 #include "exynos_drm_iommu.h"
 
@@ -40,13 +41,13 @@
 #define MIN_FB_WIDTH_FOR_16WORD_BURST 128
 
 #define WINDOWS_NR	2
-#define CURSOR_WIN	1
 
 struct decon_context {
 	struct device			*dev;
 	struct drm_device		*drm_dev;
 	struct exynos_drm_crtc		*crtc;
 	struct exynos_drm_plane		planes[WINDOWS_NR];
+	struct exynos_drm_plane_config	configs[WINDOWS_NR];
 	struct clk			*pclk;
 	struct clk			*aclk;
 	struct clk			*eclk;
@@ -81,6 +82,11 @@ static const uint32_t decon_formats[] = {
 	DRM_FORMAT_BGRA8888,
 };
 
+static const enum drm_plane_type decon_win_types[WINDOWS_NR] = {
+	DRM_PLANE_TYPE_PRIMARY,
+	DRM_PLANE_TYPE_CURSOR,
+};
+
 static void decon_wait_for_vblank(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
@@ -119,13 +125,8 @@ static void decon_clear_channels(struct exynos_drm_crtc *crtc)
 	}
 
 	/* Wait for vsync, as disable channel takes effect at next vsync */
-	if (ch_enabled) {
-		unsigned int state = ctx->suspended;
-
-		ctx->suspended = 0;
+	if (ch_enabled)
 		decon_wait_for_vblank(ctx->crtc);
-		ctx->suspended = state;
-	}
 }
 
 static int decon_ctx_initialize(struct decon_context *ctx,
@@ -384,30 +385,32 @@ static void decon_shadow_protect_win(struct decon_context *ctx,
 	writel(val, ctx->regs + SHADOWCON);
 }
 
-static void decon_atomic_begin(struct exynos_drm_crtc *crtc,
-					struct exynos_drm_plane *plane)
+static void decon_atomic_begin(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
+	int i;
 
 	if (ctx->suspended)
 		return;
 
-	decon_shadow_protect_win(ctx, plane->zpos, true);
+	for (i = 0; i < WINDOWS_NR; i++)
+		decon_shadow_protect_win(ctx, i, true);
 }
 
 static void decon_update_plane(struct exynos_drm_crtc *crtc,
 			       struct exynos_drm_plane *plane)
 {
+	struct exynos_drm_plane_state *state =
+				to_exynos_plane_state(plane->base.state);
 	struct decon_context *ctx = crtc->ctx;
-	struct drm_display_mode *mode = &crtc->base.state->adjusted_mode;
-	struct drm_plane_state *state = plane->base.state;
+	struct drm_framebuffer *fb = state->base.fb;
 	int padding;
 	unsigned long val, alpha;
 	unsigned int last_x;
 	unsigned int last_y;
-	unsigned int win = plane->zpos;
-	unsigned int bpp = state->fb->bits_per_pixel >> 3;
-	unsigned int pitch = state->fb->pitches[0];
+	unsigned int win = plane->index;
+	unsigned int bpp = fb->bits_per_pixel >> 3;
+	unsigned int pitch = fb->pitches[0];
 
 	if (ctx->suspended)
 		return;
@@ -423,41 +426,32 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	 */
 
 	/* buffer start address */
-	val = (unsigned long)plane->dma_addr[0];
+	val = (unsigned long)exynos_drm_fb_dma_addr(fb, 0);
 	writel(val, ctx->regs + VIDW_BUF_START(win));
 
-	padding = (pitch / bpp) - state->fb->width;
+	padding = (pitch / bpp) - fb->width;
 
 	/* buffer size */
-	writel(state->fb->width + padding, ctx->regs + VIDW_WHOLE_X(win));
-	writel(state->fb->height, ctx->regs + VIDW_WHOLE_Y(win));
+	writel(fb->width + padding, ctx->regs + VIDW_WHOLE_X(win));
+	writel(fb->height, ctx->regs + VIDW_WHOLE_Y(win));
 
 	/* offset from the start of the buffer to read */
-	writel(plane->src_x, ctx->regs + VIDW_OFFSET_X(win));
-	writel(plane->src_y, ctx->regs + VIDW_OFFSET_Y(win));
+	writel(state->src.x, ctx->regs + VIDW_OFFSET_X(win));
+	writel(state->src.y, ctx->regs + VIDW_OFFSET_Y(win));
 
 	DRM_DEBUG_KMS("start addr = 0x%lx\n",
 			(unsigned long)val);
 	DRM_DEBUG_KMS("ovl_width = %d, ovl_height = %d\n",
-			plane->crtc_w, plane->crtc_h);
-
-	/*
-	 * OSD position.
-	 * In case the window layout goes of LCD layout, DECON fails.
-	 */
-	if ((plane->crtc_x + plane->crtc_w) > mode->hdisplay)
-		plane->crtc_x = mode->hdisplay - plane->crtc_w;
-	if ((plane->crtc_y + plane->crtc_h) > mode->vdisplay)
-		plane->crtc_y = mode->vdisplay - plane->crtc_h;
+			state->crtc.w, state->crtc.h);
 
-	val = VIDOSDxA_TOPLEFT_X(plane->crtc_x) |
-		VIDOSDxA_TOPLEFT_Y(plane->crtc_y);
+	val = VIDOSDxA_TOPLEFT_X(state->crtc.x) |
+		VIDOSDxA_TOPLEFT_Y(state->crtc.y);
 	writel(val, ctx->regs + VIDOSD_A(win));
 
-	last_x = plane->crtc_x + plane->crtc_w;
+	last_x = state->crtc.x + state->crtc.w;
 	if (last_x)
 		last_x--;
-	last_y = plane->crtc_y + plane->crtc_h;
+	last_y = state->crtc.y + state->crtc.h;
 	if (last_y)
 		last_y--;
 
@@ -466,7 +460,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	writel(val, ctx->regs + VIDOSD_B(win));
 
 	DRM_DEBUG_KMS("osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
-			plane->crtc_x, plane->crtc_y, last_x, last_y);
+			state->crtc.x, state->crtc.y, last_x, last_y);
 
 	/* OSD alpha */
 	alpha = VIDOSDxC_ALPHA0_R_F(0x0) |
@@ -481,7 +475,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 
 	writel(alpha, ctx->regs + VIDOSD_D(win));
 
-	decon_win_set_pixfmt(ctx, win, state->fb);
+	decon_win_set_pixfmt(ctx, win, fb);
 
 	/* hardware window 0 doesn't support color key. */
 	if (win != 0)
@@ -505,7 +499,7 @@ static void decon_disable_plane(struct exynos_drm_crtc *crtc,
 				struct exynos_drm_plane *plane)
 {
 	struct decon_context *ctx = crtc->ctx;
-	unsigned int win = plane->zpos;
+	unsigned int win = plane->index;
 	u32 val;
 
 	if (ctx->suspended)
@@ -524,15 +518,16 @@ static void decon_disable_plane(struct exynos_drm_crtc *crtc,
 	writel(val, ctx->regs + DECON_UPDATE);
 }
 
-static void decon_atomic_flush(struct exynos_drm_crtc *crtc,
-					struct exynos_drm_plane *plane)
+static void decon_atomic_flush(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
+	int i;
 
 	if (ctx->suspended)
 		return;
 
-	decon_shadow_protect_win(ctx, plane->zpos, false);
+	for (i = 0; i < WINDOWS_NR; i++)
+		decon_shadow_protect_win(ctx, i, false);
 }
 
 static void decon_init(struct decon_context *ctx)
@@ -555,39 +550,12 @@ static void decon_init(struct decon_context *ctx)
 static void decon_enable(struct exynos_drm_crtc *crtc)
 {
 	struct decon_context *ctx = crtc->ctx;
-	int ret;
 
 	if (!ctx->suspended)
 		return;
 
-	ctx->suspended = false;
-
 	pm_runtime_get_sync(ctx->dev);
 
-	ret = clk_prepare_enable(ctx->pclk);
-	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the pclk [%d]\n", ret);
-		return;
-	}
-
-	ret = clk_prepare_enable(ctx->aclk);
-	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the aclk [%d]\n", ret);
-		return;
-	}
-
-	ret = clk_prepare_enable(ctx->eclk);
-	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the eclk [%d]\n", ret);
-		return;
-	}
-
-	ret = clk_prepare_enable(ctx->vclk);
-	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the vclk [%d]\n", ret);
-		return;
-	}
-
 	decon_init(ctx);
 
 	/* if vblank was enabled status, enable it again. */
@@ -595,6 +563,8 @@ static void decon_enable(struct exynos_drm_crtc *crtc)
 		decon_enable_vblank(ctx->crtc);
 
 	decon_commit(ctx->crtc);
+
+	ctx->suspended = false;
 }
 
 static void decon_disable(struct exynos_drm_crtc *crtc)
@@ -613,11 +583,6 @@ static void decon_disable(struct exynos_drm_crtc *crtc)
 	for (i = 0; i < WINDOWS_NR; i++)
 		decon_disable_plane(crtc, &ctx->planes[i]);
 
-	clk_disable_unprepare(ctx->vclk);
-	clk_disable_unprepare(ctx->eclk);
-	clk_disable_unprepare(ctx->aclk);
-	clk_disable_unprepare(ctx->pclk);
-
 	pm_runtime_put_sync(ctx->dev);
 
 	ctx->suspended = true;
@@ -679,8 +644,7 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 	struct decon_context *ctx = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = data;
 	struct exynos_drm_plane *exynos_plane;
-	enum drm_plane_type type;
-	unsigned int zpos;
+	unsigned int i;
 	int ret;
 
 	ret = decon_ctx_initialize(ctx, drm_dev);
@@ -689,11 +653,14 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
-	for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
-		type = exynos_plane_get_type(zpos, CURSOR_WIN);
-		ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
-					1 << ctx->pipe, type, decon_formats,
-					ARRAY_SIZE(decon_formats), zpos);
+	for (i = 0; i < WINDOWS_NR; i++) {
+		ctx->configs[i].pixel_formats = decon_formats;
+		ctx->configs[i].num_pixel_formats = ARRAY_SIZE(decon_formats);
+		ctx->configs[i].zpos = i;
+		ctx->configs[i].type = decon_win_types[i];
+
+		ret = exynos_plane_init(drm_dev, &ctx->planes[i], i,
+					1 << ctx->pipe, &ctx->configs[i]);
 		if (ret)
 			return ret;
 	}
@@ -843,11 +810,63 @@ static int decon_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int exynos7_decon_suspend(struct device *dev)
+{
+	struct decon_context *ctx = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(ctx->vclk);
+	clk_disable_unprepare(ctx->eclk);
+	clk_disable_unprepare(ctx->aclk);
+	clk_disable_unprepare(ctx->pclk);
+
+	return 0;
+}
+
+static int exynos7_decon_resume(struct device *dev)
+{
+	struct decon_context *ctx = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(ctx->pclk);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the pclk [%d]\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(ctx->aclk);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the aclk [%d]\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(ctx->eclk);
+	if  (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the eclk [%d]\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(ctx->vclk);
+	if  (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the vclk [%d]\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos7_decon_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos7_decon_suspend, exynos7_decon_resume,
+			   NULL)
+};
+
 struct platform_driver decon_driver = {
 	.probe		= decon_probe,
 	.remove		= decon_remove,
 	.driver		= {
 		.name	= "exynos-decon",
+		.pm	= &exynos7_decon_pm_ops,
 		.of_match_table = decon_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_dp_core.c b/drivers/gpu/drm/exynos/exynos_dp_core.c
index 124fb9a56f02..b79c316c2ad2 100644
--- a/drivers/gpu/drm/exynos/exynos_dp_core.c
+++ b/drivers/gpu/drm/exynos/exynos_dp_core.c
@@ -953,7 +953,7 @@ static void exynos_dp_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs exynos_dp_connector_funcs = {
+static const struct drm_connector_funcs exynos_dp_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = exynos_dp_detect,
@@ -998,7 +998,7 @@ static struct drm_encoder *exynos_dp_best_encoder(
 	return &dp->encoder;
 }
 
-static struct drm_connector_helper_funcs exynos_dp_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs exynos_dp_connector_helper_funcs = {
 	.get_modes = exynos_dp_get_modes,
 	.best_encoder = exynos_dp_best_encoder,
 };
@@ -1009,9 +1009,9 @@ static int exynos_drm_attach_lcd_bridge(struct exynos_dp_device *dp,
 {
 	int ret;
 
-	encoder->bridge = dp->bridge;
-	dp->bridge->encoder = encoder;
-	ret = drm_bridge_attach(encoder->dev, dp->bridge);
+	encoder->bridge->next = dp->ptn_bridge;
+	dp->ptn_bridge->encoder = encoder;
+	ret = drm_bridge_attach(encoder->dev, dp->ptn_bridge);
 	if (ret) {
 		DRM_ERROR("Failed to attach bridge to drm\n");
 		return ret;
@@ -1020,14 +1020,15 @@ static int exynos_drm_attach_lcd_bridge(struct exynos_dp_device *dp,
 	return 0;
 }
 
-static int exynos_dp_create_connector(struct drm_encoder *encoder)
+static int exynos_dp_bridge_attach(struct drm_bridge *bridge)
 {
-	struct exynos_dp_device *dp = encoder_to_dp(encoder);
+	struct exynos_dp_device *dp = bridge->driver_private;
+	struct drm_encoder *encoder = &dp->encoder;
 	struct drm_connector *connector = &dp->connector;
 	int ret;
 
 	/* Pre-empt DP connector creation if there's a bridge */
-	if (dp->bridge) {
+	if (dp->ptn_bridge) {
 		ret = exynos_drm_attach_lcd_bridge(dp, encoder);
 		if (!ret)
 			return 0;
@@ -1052,27 +1053,16 @@ static int exynos_dp_create_connector(struct drm_encoder *encoder)
 	return ret;
 }
 
-static bool exynos_dp_mode_fixup(struct drm_encoder *encoder,
-				 const struct drm_display_mode *mode,
-				 struct drm_display_mode *adjusted_mode)
-{
-	return true;
-}
-
-static void exynos_dp_mode_set(struct drm_encoder *encoder,
-			       struct drm_display_mode *mode,
-			       struct drm_display_mode *adjusted_mode)
-{
-}
-
-static void exynos_dp_enable(struct drm_encoder *encoder)
+static void exynos_dp_bridge_enable(struct drm_bridge *bridge)
 {
-	struct exynos_dp_device *dp = encoder_to_dp(encoder);
+	struct exynos_dp_device *dp = bridge->driver_private;
 	struct exynos_drm_crtc *crtc = dp_to_crtc(dp);
 
 	if (dp->dpms_mode == DRM_MODE_DPMS_ON)
 		return;
 
+	pm_runtime_get_sync(dp->dev);
+
 	if (dp->panel) {
 		if (drm_panel_prepare(dp->panel)) {
 			DRM_ERROR("failed to setup the panel\n");
@@ -1083,7 +1073,6 @@ static void exynos_dp_enable(struct drm_encoder *encoder)
 	if (crtc->ops->clock_enable)
 		crtc->ops->clock_enable(dp_to_crtc(dp), true);
 
-	clk_prepare_enable(dp->clock);
 	phy_power_on(dp->phy);
 	exynos_dp_init_dp(dp);
 	enable_irq(dp->irq);
@@ -1092,9 +1081,9 @@ static void exynos_dp_enable(struct drm_encoder *encoder)
 	dp->dpms_mode = DRM_MODE_DPMS_ON;
 }
 
-static void exynos_dp_disable(struct drm_encoder *encoder)
+static void exynos_dp_bridge_disable(struct drm_bridge *bridge)
 {
-	struct exynos_dp_device *dp = encoder_to_dp(encoder);
+	struct exynos_dp_device *dp = bridge->driver_private;
 	struct exynos_drm_crtc *crtc = dp_to_crtc(dp);
 
 	if (dp->dpms_mode != DRM_MODE_DPMS_ON)
@@ -1110,7 +1099,6 @@ static void exynos_dp_disable(struct drm_encoder *encoder)
 	disable_irq(dp->irq);
 	flush_work(&dp->hotplug_work);
 	phy_power_off(dp->phy);
-	clk_disable_unprepare(dp->clock);
 
 	if (crtc->ops->clock_enable)
 		crtc->ops->clock_enable(dp_to_crtc(dp), false);
@@ -1120,17 +1108,82 @@ static void exynos_dp_disable(struct drm_encoder *encoder)
 			DRM_ERROR("failed to turnoff the panel\n");
 	}
 
+	pm_runtime_put_sync(dp->dev);
+
 	dp->dpms_mode = DRM_MODE_DPMS_OFF;
 }
 
-static struct drm_encoder_helper_funcs exynos_dp_encoder_helper_funcs = {
+static void exynos_dp_bridge_nop(struct drm_bridge *bridge)
+{
+	/* do nothing */
+}
+
+static const struct drm_bridge_funcs exynos_dp_bridge_funcs = {
+	.enable = exynos_dp_bridge_enable,
+	.disable = exynos_dp_bridge_disable,
+	.pre_enable = exynos_dp_bridge_nop,
+	.post_disable = exynos_dp_bridge_nop,
+	.attach = exynos_dp_bridge_attach,
+};
+
+static int exynos_dp_create_connector(struct drm_encoder *encoder)
+{
+	struct exynos_dp_device *dp = encoder_to_dp(encoder);
+	struct drm_device *drm_dev = dp->drm_dev;
+	struct drm_bridge *bridge;
+	int ret;
+
+	bridge = devm_kzalloc(drm_dev->dev, sizeof(*bridge), GFP_KERNEL);
+	if (!bridge) {
+		DRM_ERROR("failed to allocate for drm bridge\n");
+		return -ENOMEM;
+	}
+
+	dp->bridge = bridge;
+
+	encoder->bridge = bridge;
+	bridge->driver_private = dp;
+	bridge->encoder = encoder;
+	bridge->funcs = &exynos_dp_bridge_funcs;
+
+	ret = drm_bridge_attach(drm_dev, bridge);
+	if (ret) {
+		DRM_ERROR("failed to attach drm bridge\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static bool exynos_dp_mode_fixup(struct drm_encoder *encoder,
+				 const struct drm_display_mode *mode,
+				 struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void exynos_dp_mode_set(struct drm_encoder *encoder,
+			       struct drm_display_mode *mode,
+			       struct drm_display_mode *adjusted_mode)
+{
+}
+
+static void exynos_dp_enable(struct drm_encoder *encoder)
+{
+}
+
+static void exynos_dp_disable(struct drm_encoder *encoder)
+{
+}
+
+static const struct drm_encoder_helper_funcs exynos_dp_encoder_helper_funcs = {
 	.mode_fixup = exynos_dp_mode_fixup,
 	.mode_set = exynos_dp_mode_set,
 	.enable = exynos_dp_enable,
 	.disable = exynos_dp_disable,
 };
 
-static struct drm_encoder_funcs exynos_dp_encoder_funcs = {
+static const struct drm_encoder_funcs exynos_dp_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -1238,7 +1291,7 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 		}
 	}
 
-	if (!dp->panel && !dp->bridge) {
+	if (!dp->panel && !dp->ptn_bridge) {
 		ret = exynos_dp_dt_parse_panel(dp);
 		if (ret)
 			return ret;
@@ -1289,10 +1342,6 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 
 	INIT_WORK(&dp->hotplug_work, exynos_dp_hotplug);
 
-	phy_power_on(dp->phy);
-
-	exynos_dp_init_dp(dp);
-
 	ret = devm_request_irq(&pdev->dev, dp->irq, exynos_dp_irq_handler,
 			irq_flags, "exynos-dp", dp);
 	if (ret) {
@@ -1313,7 +1362,7 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
 	drm_encoder_init(drm_dev, encoder, &exynos_dp_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dp_encoder_helper_funcs);
 
@@ -1343,8 +1392,9 @@ static const struct component_ops exynos_dp_ops = {
 static int exynos_dp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *panel_node, *bridge_node, *endpoint;
+	struct device_node *panel_node = NULL, *bridge_node, *endpoint = NULL;
 	struct exynos_dp_device *dp;
+	int ret;
 
 	dp = devm_kzalloc(&pdev->dev, sizeof(struct exynos_dp_device),
 				GFP_KERNEL);
@@ -1353,36 +1403,96 @@ static int exynos_dp_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, dp);
 
+	/* This is for the backward compatibility. */
 	panel_node = of_parse_phandle(dev->of_node, "panel", 0);
 	if (panel_node) {
 		dp->panel = of_drm_find_panel(panel_node);
 		of_node_put(panel_node);
 		if (!dp->panel)
 			return -EPROBE_DEFER;
+	} else {
+		endpoint = of_graph_get_next_endpoint(dev->of_node, NULL);
+		if (endpoint) {
+			panel_node = of_graph_get_remote_port_parent(endpoint);
+			if (panel_node) {
+				dp->panel = of_drm_find_panel(panel_node);
+				of_node_put(panel_node);
+				if (!dp->panel)
+					return -EPROBE_DEFER;
+			} else {
+				DRM_ERROR("no port node for panel device.\n");
+				return -EINVAL;
+			}
+		}
 	}
 
+	if (endpoint)
+		goto out;
+
 	endpoint = of_graph_get_next_endpoint(dev->of_node, NULL);
 	if (endpoint) {
 		bridge_node = of_graph_get_remote_port_parent(endpoint);
 		if (bridge_node) {
-			dp->bridge = of_drm_find_bridge(bridge_node);
+			dp->ptn_bridge = of_drm_find_bridge(bridge_node);
 			of_node_put(bridge_node);
-			if (!dp->bridge)
+			if (!dp->ptn_bridge)
 				return -EPROBE_DEFER;
 		} else
 			return -EPROBE_DEFER;
 	}
 
-	return component_add(&pdev->dev, &exynos_dp_ops);
+out:
+	pm_runtime_enable(dev);
+
+	ret = component_add(&pdev->dev, &exynos_dp_ops);
+	if (ret)
+		goto err_disable_pm_runtime;
+
+	return ret;
+
+err_disable_pm_runtime:
+	pm_runtime_disable(dev);
+
+	return ret;
 }
 
 static int exynos_dp_remove(struct platform_device *pdev)
 {
+	pm_runtime_disable(&pdev->dev);
 	component_del(&pdev->dev, &exynos_dp_ops);
 
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int exynos_dp_suspend(struct device *dev)
+{
+	struct exynos_dp_device *dp = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(dp->clock);
+
+	return 0;
+}
+
+static int exynos_dp_resume(struct device *dev)
+{
+	struct exynos_dp_device *dp = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(dp->clock);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos_dp_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos_dp_suspend, exynos_dp_resume, NULL)
+};
+
 static const struct of_device_id exynos_dp_match[] = {
 	{ .compatible = "samsung,exynos5-dp" },
 	{},
@@ -1395,6 +1505,7 @@ struct platform_driver dp_driver = {
 	.driver		= {
 		.name	= "exynos-dp",
 		.owner	= THIS_MODULE,
+		.pm	= &exynos_dp_pm_ops,
 		.of_match_table = exynos_dp_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_dp_core.h b/drivers/gpu/drm/exynos/exynos_dp_core.h
index e413b6f7b0e7..66eec4b2d5c6 100644
--- a/drivers/gpu/drm/exynos/exynos_dp_core.h
+++ b/drivers/gpu/drm/exynos/exynos_dp_core.h
@@ -153,6 +153,7 @@ struct exynos_dp_device {
 	struct drm_connector	connector;
 	struct drm_panel	*panel;
 	struct drm_bridge	*bridge;
+	struct drm_bridge	*ptn_bridge;
 	struct clk		*clock;
 	unsigned int		irq;
 	void __iomem		*reg_base;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index e69357172ffb..e36579c1c025 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -68,35 +68,23 @@ static void exynos_crtc_atomic_begin(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_crtc_state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
-	struct drm_plane *plane;
 
 	exynos_crtc->event = crtc->state->event;
 
-	drm_atomic_crtc_for_each_plane(plane, crtc) {
-		struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
-
-		if (exynos_crtc->ops->atomic_begin)
-			exynos_crtc->ops->atomic_begin(exynos_crtc,
-							exynos_plane);
-	}
+	if (exynos_crtc->ops->atomic_begin)
+		exynos_crtc->ops->atomic_begin(exynos_crtc);
 }
 
 static void exynos_crtc_atomic_flush(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_crtc_state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
-	struct drm_plane *plane;
 
-	drm_atomic_crtc_for_each_plane(plane, crtc) {
-		struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
-
-		if (exynos_crtc->ops->atomic_flush)
-			exynos_crtc->ops->atomic_flush(exynos_crtc,
-							exynos_plane);
-	}
+	if (exynos_crtc->ops->atomic_flush)
+		exynos_crtc->ops->atomic_flush(exynos_crtc);
 }
 
-static struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
+static const struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
 	.enable		= exynos_drm_crtc_enable,
 	.disable	= exynos_drm_crtc_disable,
 	.mode_set_nofb	= exynos_drm_crtc_mode_set_nofb,
@@ -116,7 +104,7 @@ static void exynos_drm_crtc_destroy(struct drm_crtc *crtc)
 	kfree(exynos_crtc);
 }
 
-static struct drm_crtc_funcs exynos_crtc_funcs = {
+static const struct drm_crtc_funcs exynos_crtc_funcs = {
 	.set_config	= drm_atomic_helper_set_config,
 	.page_flip	= drm_atomic_helper_page_flip,
 	.destroy	= exynos_drm_crtc_destroy,
@@ -153,7 +141,7 @@ struct exynos_drm_crtc *exynos_drm_crtc_create(struct drm_device *drm_dev,
 	private->crtc[pipe] = crtc;
 
 	ret = drm_crtc_init_with_planes(drm_dev, crtc, plane, NULL,
-					&exynos_crtc_funcs);
+					&exynos_crtc_funcs, NULL);
 	if (ret < 0)
 		goto err_crtc;
 
@@ -215,29 +203,6 @@ void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc,
 	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 }
 
-void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb)
-{
-	struct exynos_drm_crtc *exynos_crtc;
-	struct drm_device *dev = fb->dev;
-	struct drm_crtc *crtc;
-
-	/*
-	 * make sure that overlay data are updated to real hardware
-	 * for all encoders.
-	 */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		exynos_crtc = to_exynos_crtc(crtc);
-
-		/*
-		 * wait for vblank interrupt
-		 * - this makes sure that overlay data are updated to
-		 *	real hardware.
-		 */
-		if (exynos_crtc->ops->wait_for_vblank)
-			exynos_crtc->ops->wait_for_vblank(exynos_crtc);
-	}
-}
-
 int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
 				       enum exynos_drm_output_type out_type)
 {
@@ -261,3 +226,29 @@ void exynos_drm_crtc_te_handler(struct drm_crtc *crtc)
 	if (exynos_crtc->ops->te_handler)
 		exynos_crtc->ops->te_handler(exynos_crtc);
 }
+
+void exynos_drm_crtc_cancel_page_flip(struct drm_crtc *crtc,
+					struct drm_file *file)
+{
+	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+	struct drm_pending_vblank_event *e;
+	unsigned long flags;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+	e = exynos_crtc->event;
+	if (e && e->base.file_priv == file) {
+		exynos_crtc->event = NULL;
+		/*
+		 * event will be destroyed by core part
+		 * so below line should be removed later with core changes
+		 */
+		e->base.destroy(&e->base);
+		/*
+		 * event_space will be increased by core part
+		 * so below line should be removed later with core changes.
+		 */
+		file->event_space += sizeof(e->event);
+		atomic_dec(&exynos_crtc->pending_update);
+	}
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.h b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
index f9f365bd0257..cfdcf3e4eb1b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
@@ -28,7 +28,6 @@ void exynos_drm_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe);
 void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc);
 void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc,
 				   struct exynos_drm_plane *exynos_plane);
-void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb);
 
 /* This function gets pipe value to crtc device matched with out_type. */
 int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
@@ -41,4 +40,8 @@ int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
  */
 void exynos_drm_crtc_te_handler(struct drm_crtc *crtc);
 
+/* This function cancels a page flip request. */
+void exynos_drm_crtc_cancel_page_flip(struct drm_crtc *crtc,
+					struct drm_file *file);
+
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
index c748b8790de3..05350ae0785b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -57,7 +57,7 @@ static void exynos_dpi_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs exynos_dpi_connector_funcs = {
+static const struct drm_connector_funcs exynos_dpi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = exynos_dpi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -100,7 +100,7 @@ exynos_dpi_best_encoder(struct drm_connector *connector)
 	return &ctx->encoder;
 }
 
-static struct drm_connector_helper_funcs exynos_dpi_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs exynos_dpi_connector_helper_funcs = {
 	.get_modes = exynos_dpi_get_modes,
 	.best_encoder = exynos_dpi_best_encoder,
 };
@@ -161,14 +161,14 @@ static void exynos_dpi_disable(struct drm_encoder *encoder)
 	}
 }
 
-static struct drm_encoder_helper_funcs exynos_dpi_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs exynos_dpi_encoder_helper_funcs = {
 	.mode_fixup = exynos_dpi_mode_fixup,
 	.mode_set = exynos_dpi_mode_set,
 	.enable = exynos_dpi_enable,
 	.disable = exynos_dpi_disable,
 };
 
-static struct drm_encoder_funcs exynos_dpi_encoder_funcs = {
+static const struct drm_encoder_funcs exynos_dpi_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -309,7 +309,7 @@ int exynos_dpi_bind(struct drm_device *dev, struct drm_encoder *encoder)
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
 	drm_encoder_init(dev, encoder, &exynos_dpi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dpi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 2c6019d6a205..68f0f36f6e7e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -304,45 +304,6 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int exynos_drm_suspend(struct drm_device *dev, pm_message_t state)
-{
-	struct drm_connector *connector;
-
-	drm_modeset_lock_all(dev);
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		int old_dpms = connector->dpms;
-
-		if (connector->funcs->dpms)
-			connector->funcs->dpms(connector, DRM_MODE_DPMS_OFF);
-
-		/* Set the old mode back to the connector for resume */
-		connector->dpms = old_dpms;
-	}
-	drm_modeset_unlock_all(dev);
-
-	return 0;
-}
-
-static int exynos_drm_resume(struct drm_device *dev)
-{
-	struct drm_connector *connector;
-
-	drm_modeset_lock_all(dev);
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->funcs->dpms) {
-			int dpms = connector->dpms;
-
-			connector->dpms = DRM_MODE_DPMS_OFF;
-			connector->funcs->dpms(connector, dpms);
-		}
-	}
-	drm_modeset_unlock_all(dev);
-
-	return 0;
-}
-#endif
-
 static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_exynos_file_private *file_priv;
@@ -369,7 +330,12 @@ err_file_priv_free:
 static void exynos_drm_preclose(struct drm_device *dev,
 					struct drm_file *file)
 {
+	struct drm_crtc *crtc;
+
 	exynos_drm_subdrv_close(dev, file);
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+		exynos_drm_crtc_cancel_page_flip(crtc, file);
 }
 
 static void exynos_drm_postclose(struct drm_device *dev, struct drm_file *file)
@@ -476,31 +442,54 @@ static struct drm_driver exynos_drm_driver = {
 };
 
 #ifdef CONFIG_PM_SLEEP
-static int exynos_drm_sys_suspend(struct device *dev)
+static int exynos_drm_suspend(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	pm_message_t message;
+	struct drm_connector *connector;
 
 	if (pm_runtime_suspended(dev) || !drm_dev)
 		return 0;
 
-	message.event = PM_EVENT_SUSPEND;
-	return exynos_drm_suspend(drm_dev, message);
+	drm_modeset_lock_all(drm_dev);
+	drm_for_each_connector(connector, drm_dev) {
+		int old_dpms = connector->dpms;
+
+		if (connector->funcs->dpms)
+			connector->funcs->dpms(connector, DRM_MODE_DPMS_OFF);
+
+		/* Set the old mode back to the connector for resume */
+		connector->dpms = old_dpms;
+	}
+	drm_modeset_unlock_all(drm_dev);
+
+	return 0;
 }
 
-static int exynos_drm_sys_resume(struct device *dev)
+static int exynos_drm_resume(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
+	struct drm_connector *connector;
 
 	if (pm_runtime_suspended(dev) || !drm_dev)
 		return 0;
 
-	return exynos_drm_resume(drm_dev);
+	drm_modeset_lock_all(drm_dev);
+	drm_for_each_connector(connector, drm_dev) {
+		if (connector->funcs->dpms) {
+			int dpms = connector->dpms;
+
+			connector->dpms = DRM_MODE_DPMS_OFF;
+			connector->funcs->dpms(connector, dpms);
+		}
+	}
+	drm_modeset_unlock_all(drm_dev);
+
+	return 0;
 }
 #endif
 
 static const struct dev_pm_ops exynos_drm_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(exynos_drm_sys_suspend, exynos_drm_sys_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(exynos_drm_suspend, exynos_drm_resume)
 };
 
 /* forward declaration */
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index f1eda7fa4e3c..17b5ded72ff1 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -38,25 +38,46 @@ enum exynos_drm_output_type {
 	EXYNOS_DISPLAY_TYPE_VIDI,
 };
 
+struct exynos_drm_rect {
+	unsigned int x, y;
+	unsigned int w, h;
+};
+
 /*
- * Exynos drm common overlay structure.
+ * Exynos drm plane state structure.
  *
- * @base: plane object
- * @src_x: offset x on a framebuffer to be displayed.
- *	- the unit is screen coordinates.
- * @src_y: offset y on a framebuffer to be displayed.
- *	- the unit is screen coordinates.
- * @src_w: width of a partial image to be displayed from framebuffer.
- * @src_h: height of a partial image to be displayed from framebuffer.
- * @crtc_x: offset x on hardware screen.
- * @crtc_y: offset y on hardware screen.
- * @crtc_w: window width to be displayed (hardware screen).
- * @crtc_h: window height to be displayed (hardware screen).
+ * @base: plane_state object (contains drm_framebuffer pointer)
+ * @src: rectangle of the source image data to be displayed (clipped to
+ *       visible part).
+ * @crtc: rectangle of the target image position on hardware screen
+ *       (clipped to visible part).
  * @h_ratio: horizontal scaling ratio, 16.16 fixed point
  * @v_ratio: vertical scaling ratio, 16.16 fixed point
- * @dma_addr: array of bus(accessed by dma) address to the memory region
- *	      allocated for a overlay.
- * @zpos: order of overlay layer(z position).
+ *
+ * this structure consists plane state data that will be applied to hardware
+ * specific overlay info.
+ */
+
+struct exynos_drm_plane_state {
+	struct drm_plane_state base;
+	struct exynos_drm_rect crtc;
+	struct exynos_drm_rect src;
+	unsigned int h_ratio;
+	unsigned int v_ratio;
+	unsigned int zpos;
+};
+
+static inline struct exynos_drm_plane_state *
+to_exynos_plane_state(struct drm_plane_state *state)
+{
+	return container_of(state, struct exynos_drm_plane_state, base);
+}
+
+/*
+ * Exynos drm common overlay structure.
+ *
+ * @base: plane object
+ * @index: hardware index of the overlay layer
  *
  * this structure is common to exynos SoC and its contents would be copied
  * to hardware specific overlay info.
@@ -64,21 +85,33 @@ enum exynos_drm_output_type {
 
 struct exynos_drm_plane {
 	struct drm_plane base;
-	unsigned int src_x;
-	unsigned int src_y;
-	unsigned int src_w;
-	unsigned int src_h;
-	unsigned int crtc_x;
-	unsigned int crtc_y;
-	unsigned int crtc_w;
-	unsigned int crtc_h;
-	unsigned int h_ratio;
-	unsigned int v_ratio;
-	dma_addr_t dma_addr[MAX_FB_BUFFER];
-	unsigned int zpos;
+	const struct exynos_drm_plane_config *config;
+	unsigned int index;
 	struct drm_framebuffer *pending_fb;
 };
 
+#define EXYNOS_DRM_PLANE_CAP_DOUBLE	(1 << 0)
+#define EXYNOS_DRM_PLANE_CAP_SCALE	(1 << 1)
+#define EXYNOS_DRM_PLANE_CAP_ZPOS	(1 << 2)
+
+/*
+ * Exynos DRM plane configuration structure.
+ *
+ * @zpos: initial z-position of the plane.
+ * @type: type of the plane (primary, cursor or overlay).
+ * @pixel_formats: supported pixel formats.
+ * @num_pixel_formats: number of elements in 'pixel_formats'.
+ * @capabilities: supported features (see EXYNOS_DRM_PLANE_CAP_*)
+ */
+
+struct exynos_drm_plane_config {
+	unsigned int zpos;
+	enum drm_plane_type type;
+	const uint32_t *pixel_formats;
+	unsigned int num_pixel_formats;
+	unsigned int capabilities;
+};
+
 /*
  * Exynos drm crtc ops
  *
@@ -90,8 +123,8 @@ struct exynos_drm_plane {
  * @wait_for_vblank: wait for vblank interrupt to make sure that
  *	hardware overlay is updated.
  * @atomic_check: validate state
- * @atomic_begin: prepare a window to receive a update
- * @atomic_flush: mark the end of a window update
+ * @atomic_begin: prepare device to receive an update
+ * @atomic_flush: mark the end of device update
  * @update_plane: apply hardware specific overlay data to registers.
  * @disable_plane: disable hardware specific overlay.
  * @te_handler: trigger to transfer video image at the tearing effect
@@ -111,14 +144,12 @@ struct exynos_drm_crtc_ops {
 	void (*wait_for_vblank)(struct exynos_drm_crtc *crtc);
 	int (*atomic_check)(struct exynos_drm_crtc *crtc,
 			    struct drm_crtc_state *state);
-	void (*atomic_begin)(struct exynos_drm_crtc *crtc,
-			      struct exynos_drm_plane *plane);
+	void (*atomic_begin)(struct exynos_drm_crtc *crtc);
 	void (*update_plane)(struct exynos_drm_crtc *crtc,
 			     struct exynos_drm_plane *plane);
 	void (*disable_plane)(struct exynos_drm_crtc *crtc,
 			      struct exynos_drm_plane *plane);
-	void (*atomic_flush)(struct exynos_drm_crtc *crtc,
-			      struct exynos_drm_plane *plane);
+	void (*atomic_flush)(struct exynos_drm_crtc *crtc);
 	void (*te_handler)(struct exynos_drm_crtc *crtc);
 	void (*clock_enable)(struct exynos_drm_crtc *crtc, bool enable);
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index 12b03b364703..d84a498ef099 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -1458,66 +1458,6 @@ static const struct mipi_dsi_host_ops exynos_dsi_ops = {
 	.transfer = exynos_dsi_host_transfer,
 };
 
-static int exynos_dsi_poweron(struct exynos_dsi *dsi)
-{
-	struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	int ret, i;
-
-	ret = regulator_bulk_enable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
-	if (ret < 0) {
-		dev_err(dsi->dev, "cannot enable regulators %d\n", ret);
-		return ret;
-	}
-
-	for (i = 0; i < driver_data->num_clks; i++) {
-		ret = clk_prepare_enable(dsi->clks[i]);
-		if (ret < 0)
-			goto err_clk;
-	}
-
-	ret = phy_power_on(dsi->phy);
-	if (ret < 0) {
-		dev_err(dsi->dev, "cannot enable phy %d\n", ret);
-		goto err_clk;
-	}
-
-	return 0;
-
-err_clk:
-	while (--i > -1)
-		clk_disable_unprepare(dsi->clks[i]);
-	regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
-
-	return ret;
-}
-
-static void exynos_dsi_poweroff(struct exynos_dsi *dsi)
-{
-	struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	int ret, i;
-
-	usleep_range(10000, 20000);
-
-	if (dsi->state & DSIM_STATE_INITIALIZED) {
-		dsi->state &= ~DSIM_STATE_INITIALIZED;
-
-		exynos_dsi_disable_clock(dsi);
-
-		exynos_dsi_disable_irq(dsi);
-	}
-
-	dsi->state &= ~DSIM_STATE_CMD_LPM;
-
-	phy_power_off(dsi->phy);
-
-	for (i = driver_data->num_clks - 1; i > -1; i--)
-		clk_disable_unprepare(dsi->clks[i]);
-
-	ret = regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
-	if (ret < 0)
-		dev_err(dsi->dev, "cannot disable regulators %d\n", ret);
-}
-
 static void exynos_dsi_enable(struct drm_encoder *encoder)
 {
 	struct exynos_dsi *dsi = encoder_to_dsi(encoder);
@@ -1526,16 +1466,14 @@ static void exynos_dsi_enable(struct drm_encoder *encoder)
 	if (dsi->state & DSIM_STATE_ENABLED)
 		return;
 
-	ret = exynos_dsi_poweron(dsi);
-	if (ret < 0)
-		return;
+	pm_runtime_get_sync(dsi->dev);
 
 	dsi->state |= DSIM_STATE_ENABLED;
 
 	ret = drm_panel_prepare(dsi->panel);
 	if (ret < 0) {
 		dsi->state &= ~DSIM_STATE_ENABLED;
-		exynos_dsi_poweroff(dsi);
+		pm_runtime_put_sync(dsi->dev);
 		return;
 	}
 
@@ -1547,7 +1485,7 @@ static void exynos_dsi_enable(struct drm_encoder *encoder)
 		dsi->state &= ~DSIM_STATE_ENABLED;
 		exynos_dsi_set_display_enable(dsi, false);
 		drm_panel_unprepare(dsi->panel);
-		exynos_dsi_poweroff(dsi);
+		pm_runtime_put_sync(dsi->dev);
 		return;
 	}
 
@@ -1569,7 +1507,7 @@ static void exynos_dsi_disable(struct drm_encoder *encoder)
 
 	dsi->state &= ~DSIM_STATE_ENABLED;
 
-	exynos_dsi_poweroff(dsi);
+	pm_runtime_put_sync(dsi->dev);
 }
 
 static enum drm_connector_status
@@ -1603,7 +1541,7 @@ static void exynos_dsi_connector_destroy(struct drm_connector *connector)
 	connector->dev = NULL;
 }
 
-static struct drm_connector_funcs exynos_dsi_connector_funcs = {
+static const struct drm_connector_funcs exynos_dsi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = exynos_dsi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -1631,7 +1569,7 @@ exynos_dsi_best_encoder(struct drm_connector *connector)
 	return &dsi->encoder;
 }
 
-static struct drm_connector_helper_funcs exynos_dsi_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs exynos_dsi_connector_helper_funcs = {
 	.get_modes = exynos_dsi_get_modes,
 	.best_encoder = exynos_dsi_best_encoder,
 };
@@ -1684,14 +1622,14 @@ static void exynos_dsi_mode_set(struct drm_encoder *encoder,
 	vm->hsync_len = m->hsync_end - m->hsync_start;
 }
 
-static struct drm_encoder_helper_funcs exynos_dsi_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs exynos_dsi_encoder_helper_funcs = {
 	.mode_fixup = exynos_dsi_mode_fixup,
 	.mode_set = exynos_dsi_mode_set,
 	.enable = exynos_dsi_enable,
 	.disable = exynos_dsi_disable,
 };
 
-static struct drm_encoder_funcs exynos_dsi_encoder_funcs = {
+static const struct drm_encoder_funcs exynos_dsi_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -1797,13 +1735,13 @@ static int exynos_dsi_parse_dt(struct exynos_dsi *dsi)
 
 	ep = of_graph_get_next_endpoint(node, NULL);
 	if (!ep) {
-		ret = -ENXIO;
+		ret = -EINVAL;
 		goto end;
 	}
 
 	dsi->bridge_node = of_graph_get_remote_port_parent(ep);
 	if (!dsi->bridge_node) {
-		ret = -ENXIO;
+		ret = -EINVAL;
 		goto end;
 	}
 end:
@@ -1831,7 +1769,7 @@ static int exynos_dsi_bind(struct device *dev, struct device *master,
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
 	drm_encoder_init(drm_dev, encoder, &exynos_dsi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dsi_encoder_helper_funcs);
 
@@ -1954,22 +1892,99 @@ static int exynos_dsi_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, &dsi->encoder);
 
+	pm_runtime_enable(dev);
+
 	return component_add(dev, &exynos_dsi_component_ops);
 }
 
 static int exynos_dsi_remove(struct platform_device *pdev)
 {
+	pm_runtime_disable(&pdev->dev);
+
 	component_del(&pdev->dev, &exynos_dsi_component_ops);
 
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int exynos_dsi_suspend(struct device *dev)
+{
+	struct drm_encoder *encoder = dev_get_drvdata(dev);
+	struct exynos_dsi *dsi = encoder_to_dsi(encoder);
+	struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
+	int ret, i;
+
+	usleep_range(10000, 20000);
+
+	if (dsi->state & DSIM_STATE_INITIALIZED) {
+		dsi->state &= ~DSIM_STATE_INITIALIZED;
+
+		exynos_dsi_disable_clock(dsi);
+
+		exynos_dsi_disable_irq(dsi);
+	}
+
+	dsi->state &= ~DSIM_STATE_CMD_LPM;
+
+	phy_power_off(dsi->phy);
+
+	for (i = driver_data->num_clks - 1; i > -1; i--)
+		clk_disable_unprepare(dsi->clks[i]);
+
+	ret = regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+	if (ret < 0)
+		dev_err(dsi->dev, "cannot disable regulators %d\n", ret);
+
+	return 0;
+}
+
+static int exynos_dsi_resume(struct device *dev)
+{
+	struct drm_encoder *encoder = dev_get_drvdata(dev);
+	struct exynos_dsi *dsi = encoder_to_dsi(encoder);
+	struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
+	int ret, i;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+	if (ret < 0) {
+		dev_err(dsi->dev, "cannot enable regulators %d\n", ret);
+		return ret;
+	}
+
+	for (i = 0; i < driver_data->num_clks; i++) {
+		ret = clk_prepare_enable(dsi->clks[i]);
+		if (ret < 0)
+			goto err_clk;
+	}
+
+	ret = phy_power_on(dsi->phy);
+	if (ret < 0) {
+		dev_err(dsi->dev, "cannot enable phy %d\n", ret);
+		goto err_clk;
+	}
+
+	return 0;
+
+err_clk:
+	while (--i > -1)
+		clk_disable_unprepare(dsi->clks[i]);
+	regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+
+	return ret;
+}
+#endif
+
+static const struct dev_pm_ops exynos_dsi_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos_dsi_suspend, exynos_dsi_resume, NULL)
+};
+
 struct platform_driver dsi_driver = {
 	.probe = exynos_dsi_probe,
 	.remove = exynos_dsi_remove,
 	.driver = {
 		   .name = "exynos-dsi",
 		   .owner = THIS_MODULE,
+		   .pm = &exynos_dsi_pm_ops,
 		   .of_match_table = exynos_dsi_of_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index fcea28bdbc42..d614194644c8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -37,6 +37,7 @@
 struct exynos_drm_fb {
 	struct drm_framebuffer	fb;
 	struct exynos_drm_gem	*exynos_gem[MAX_FB_BUFFER];
+	dma_addr_t			dma_addr[MAX_FB_BUFFER];
 };
 
 static int check_fb_gem_memory_type(struct drm_device *drm_dev,
@@ -70,9 +71,6 @@ static void exynos_drm_fb_destroy(struct drm_framebuffer *fb)
 	struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
 	unsigned int i;
 
-	/* make sure that overlay data are updated before relesing fb. */
-	exynos_drm_crtc_complete_scanout(fb);
-
 	drm_framebuffer_cleanup(fb);
 
 	for (i = 0; i < ARRAY_SIZE(exynos_fb->exynos_gem); i++) {
@@ -109,7 +107,7 @@ static int exynos_drm_fb_dirty(struct drm_framebuffer *fb,
 	return 0;
 }
 
-static struct drm_framebuffer_funcs exynos_drm_fb_funcs = {
+static const struct drm_framebuffer_funcs exynos_drm_fb_funcs = {
 	.destroy	= exynos_drm_fb_destroy,
 	.create_handle	= exynos_drm_fb_create_handle,
 	.dirty		= exynos_drm_fb_dirty,
@@ -117,7 +115,7 @@ static struct drm_framebuffer_funcs exynos_drm_fb_funcs = {
 
 struct drm_framebuffer *
 exynos_drm_framebuffer_init(struct drm_device *dev,
-			    struct drm_mode_fb_cmd2 *mode_cmd,
+			    const struct drm_mode_fb_cmd2 *mode_cmd,
 			    struct exynos_drm_gem **exynos_gem,
 			    int count)
 {
@@ -135,6 +133,8 @@ exynos_drm_framebuffer_init(struct drm_device *dev,
 			goto err;
 
 		exynos_fb->exynos_gem[i] = exynos_gem[i];
+		exynos_fb->dma_addr[i] = exynos_gem[i]->dma_addr
+						+ mode_cmd->offsets[i];
 	}
 
 	drm_helper_mode_fill_fb_struct(&exynos_fb->fb, mode_cmd);
@@ -154,7 +154,7 @@ err:
 
 static struct drm_framebuffer *
 exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
-		      struct drm_mode_fb_cmd2 *mode_cmd)
+		      const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct exynos_drm_gem *exynos_gem[MAX_FB_BUFFER];
 	struct drm_gem_object *obj;
@@ -189,21 +189,14 @@ err:
 	return ERR_PTR(ret);
 }
 
-struct exynos_drm_gem *exynos_drm_fb_gem(struct drm_framebuffer *fb, int index)
+dma_addr_t exynos_drm_fb_dma_addr(struct drm_framebuffer *fb, int index)
 {
 	struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
-	struct exynos_drm_gem *exynos_gem;
 
 	if (index >= MAX_FB_BUFFER)
-		return NULL;
-
-	exynos_gem = exynos_fb->exynos_gem[index];
-	if (!exynos_gem)
-		return NULL;
-
-	DRM_DEBUG_KMS("dma_addr: 0x%lx\n", (unsigned long)exynos_gem->dma_addr);
+		return DMA_ERROR_CODE;
 
-	return exynos_gem;
+	return exynos_fb->dma_addr[index];
 }
 
 static void exynos_drm_output_poll_changed(struct drm_device *dev)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.h b/drivers/gpu/drm/exynos/exynos_drm_fb.h
index 726a2d44371f..3a9e75b2cf6b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.h
@@ -12,18 +12,17 @@
  */
 
 #ifndef _EXYNOS_DRM_FB_H_
-#define _EXYNOS_DRM_FB_H
+#define _EXYNOS_DRM_FB_H_
 
 #include "exynos_drm_gem.h"
 
 struct drm_framebuffer *
 exynos_drm_framebuffer_init(struct drm_device *dev,
-			    struct drm_mode_fb_cmd2 *mode_cmd,
+			    const struct drm_mode_fb_cmd2 *mode_cmd,
 			    struct exynos_drm_gem **exynos_gem,
 			    int count);
 
-/* get gem object of a drm framebuffer */
-struct exynos_drm_gem *exynos_drm_fb_gem(struct drm_framebuffer *fb, int index);
+dma_addr_t exynos_drm_fb_dma_addr(struct drm_framebuffer *fb, int index);
 
 void exynos_drm_mode_config_init(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index bd75c1531cac..70194d0e4fe4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -29,6 +29,7 @@
 #include <drm/exynos_drm.h>
 
 #include "exynos_drm_drv.h"
+#include "exynos_drm_fb.h"
 #include "exynos_drm_fbdev.h"
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_plane.h"
@@ -87,7 +88,6 @@
 
 /* FIMD has totally five hardware windows. */
 #define WINDOWS_NR	5
-#define CURSOR_WIN	4
 
 struct fimd_driver_data {
 	unsigned int timing_base;
@@ -150,6 +150,7 @@ struct fimd_context {
 	struct drm_device		*drm_dev;
 	struct exynos_drm_crtc		*crtc;
 	struct exynos_drm_plane		planes[WINDOWS_NR];
+	struct exynos_drm_plane_config	configs[WINDOWS_NR];
 	struct clk			*bus_clk;
 	struct clk			*lcd_clk;
 	void __iomem			*regs;
@@ -187,6 +188,14 @@ static const struct of_device_id fimd_driver_dt_match[] = {
 };
 MODULE_DEVICE_TABLE(of, fimd_driver_dt_match);
 
+static const enum drm_plane_type fimd_win_types[WINDOWS_NR] = {
+	DRM_PLANE_TYPE_PRIMARY,
+	DRM_PLANE_TYPE_OVERLAY,
+	DRM_PLANE_TYPE_OVERLAY,
+	DRM_PLANE_TYPE_OVERLAY,
+	DRM_PLANE_TYPE_CURSOR,
+};
+
 static const uint32_t fimd_formats[] = {
 	DRM_FORMAT_C8,
 	DRM_FORMAT_XRGB1555,
@@ -478,7 +487,7 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 
 
 static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win,
-				struct drm_framebuffer *fb)
+				uint32_t pixel_format, int width)
 {
 	unsigned long val;
 
@@ -489,11 +498,11 @@ static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win,
 	 * So the request format is ARGB8888 then change it to XRGB8888.
 	 */
 	if (ctx->driver_data->has_limited_fmt && !win) {
-		if (fb->pixel_format == DRM_FORMAT_ARGB8888)
-			fb->pixel_format = DRM_FORMAT_XRGB8888;
+		if (pixel_format == DRM_FORMAT_ARGB8888)
+			pixel_format = DRM_FORMAT_XRGB8888;
 	}
 
-	switch (fb->pixel_format) {
+	switch (pixel_format) {
 	case DRM_FORMAT_C8:
 		val |= WINCON0_BPPMODE_8BPP_PALETTE;
 		val |= WINCONx_BURSTLEN_8WORD;
@@ -529,17 +538,15 @@ static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win,
 		break;
 	}
 
-	DRM_DEBUG_KMS("bpp = %d\n", fb->bits_per_pixel);
-
 	/*
-	 * In case of exynos, setting dma-burst to 16Word causes permanent
-	 * tearing for very small buffers, e.g. cursor buffer. Burst Mode
-	 * switching which is based on plane size is not recommended as
-	 * plane size varies alot towards the end of the screen and rapid
-	 * movement causes unstable DMA which results into iommu crash/tear.
+	 * Setting dma-burst to 16Word causes permanent tearing for very small
+	 * buffers, e.g. cursor buffer. Burst Mode switching which based on
+	 * plane size is not recommended as plane size varies alot towards the
+	 * end of the screen and rapid movement causes unstable DMA, but it is
+	 * still better to change dma-burst than displaying garbage.
 	 */
 
-	if (fb->width < MIN_FB_WIDTH_FOR_16WORD_BURST) {
+	if (width < MIN_FB_WIDTH_FOR_16WORD_BURST) {
 		val &= ~WINCONx_BURSTLEN_MASK;
 		val |= WINCONx_BURSTLEN_4WORD;
 	}
@@ -615,64 +622,68 @@ static void fimd_shadow_protect_win(struct fimd_context *ctx,
 	writel(val, ctx->regs + reg);
 }
 
-static void fimd_atomic_begin(struct exynos_drm_crtc *crtc,
-			       struct exynos_drm_plane *plane)
+static void fimd_atomic_begin(struct exynos_drm_crtc *crtc)
 {
 	struct fimd_context *ctx = crtc->ctx;
+	int i;
 
 	if (ctx->suspended)
 		return;
 
-	fimd_shadow_protect_win(ctx, plane->zpos, true);
+	for (i = 0; i < WINDOWS_NR; i++)
+		fimd_shadow_protect_win(ctx, i, true);
 }
 
-static void fimd_atomic_flush(struct exynos_drm_crtc *crtc,
-			       struct exynos_drm_plane *plane)
+static void fimd_atomic_flush(struct exynos_drm_crtc *crtc)
 {
 	struct fimd_context *ctx = crtc->ctx;
+	int i;
 
 	if (ctx->suspended)
 		return;
 
-	fimd_shadow_protect_win(ctx, plane->zpos, false);
+	for (i = 0; i < WINDOWS_NR; i++)
+		fimd_shadow_protect_win(ctx, i, false);
 }
 
 static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 			      struct exynos_drm_plane *plane)
 {
+	struct exynos_drm_plane_state *state =
+				to_exynos_plane_state(plane->base.state);
 	struct fimd_context *ctx = crtc->ctx;
-	struct drm_plane_state *state = plane->base.state;
+	struct drm_framebuffer *fb = state->base.fb;
 	dma_addr_t dma_addr;
 	unsigned long val, size, offset;
 	unsigned int last_x, last_y, buf_offsize, line_size;
-	unsigned int win = plane->zpos;
-	unsigned int bpp = state->fb->bits_per_pixel >> 3;
-	unsigned int pitch = state->fb->pitches[0];
+	unsigned int win = plane->index;
+	unsigned int bpp = fb->bits_per_pixel >> 3;
+	unsigned int pitch = fb->pitches[0];
 
 	if (ctx->suspended)
 		return;
 
-	offset = plane->src_x * bpp;
-	offset += plane->src_y * pitch;
+	offset = state->src.x * bpp;
+	offset += state->src.y * pitch;
 
 	/* buffer start address */
-	dma_addr = plane->dma_addr[0] + offset;
+	dma_addr = exynos_drm_fb_dma_addr(fb, 0) + offset;
 	val = (unsigned long)dma_addr;
 	writel(val, ctx->regs + VIDWx_BUF_START(win, 0));
 
 	/* buffer end address */
-	size = pitch * plane->crtc_h;
+	size = pitch * state->crtc.h;
 	val = (unsigned long)(dma_addr + size);
 	writel(val, ctx->regs + VIDWx_BUF_END(win, 0));
 
 	DRM_DEBUG_KMS("start addr = 0x%lx, end addr = 0x%lx, size = 0x%lx\n",
 			(unsigned long)dma_addr, val, size);
 	DRM_DEBUG_KMS("ovl_width = %d, ovl_height = %d\n",
-			plane->crtc_w, plane->crtc_h);
+			state->crtc.w, state->crtc.h);
 
 	/* buffer size */
-	buf_offsize = pitch - (plane->crtc_w * bpp);
-	line_size = plane->crtc_w * bpp;
+	buf_offsize = pitch - (state->crtc.w * bpp);
+	line_size = state->crtc.w * bpp;
 	val = VIDW_BUF_SIZE_OFFSET(buf_offsize) |
 		VIDW_BUF_SIZE_PAGEWIDTH(line_size) |
 		VIDW_BUF_SIZE_OFFSET_E(buf_offsize) |
@@ -680,16 +691,16 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 	writel(val, ctx->regs + VIDWx_BUF_SIZE(win, 0));
 
 	/* OSD position */
-	val = VIDOSDxA_TOPLEFT_X(plane->crtc_x) |
-		VIDOSDxA_TOPLEFT_Y(plane->crtc_y) |
-		VIDOSDxA_TOPLEFT_X_E(plane->crtc_x) |
-		VIDOSDxA_TOPLEFT_Y_E(plane->crtc_y);
+	val = VIDOSDxA_TOPLEFT_X(state->crtc.x) |
+		VIDOSDxA_TOPLEFT_Y(state->crtc.y) |
+		VIDOSDxA_TOPLEFT_X_E(state->crtc.x) |
+		VIDOSDxA_TOPLEFT_Y_E(state->crtc.y);
 	writel(val, ctx->regs + VIDOSD_A(win));
 
-	last_x = plane->crtc_x + plane->crtc_w;
+	last_x = state->crtc.x + state->crtc.w;
 	if (last_x)
 		last_x--;
-	last_y = plane->crtc_y + plane->crtc_h;
+	last_y = state->crtc.y + state->crtc.h;
 	if (last_y)
 		last_y--;
 
@@ -699,20 +710,20 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 	writel(val, ctx->regs + VIDOSD_B(win));
 
 	DRM_DEBUG_KMS("osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
-			plane->crtc_x, plane->crtc_y, last_x, last_y);
+			state->crtc.x, state->crtc.y, last_x, last_y);
 
 	/* OSD size */
 	if (win != 3 && win != 4) {
 		u32 offset = VIDOSD_D(win);
 		if (win == 0)
 			offset = VIDOSD_C(win);
-		val = plane->crtc_w * plane->crtc_h;
+		val = state->crtc.w * state->crtc.h;
 		writel(val, ctx->regs + offset);
 
 		DRM_DEBUG_KMS("osd size = 0x%x\n", (unsigned int)val);
 	}
 
-	fimd_win_set_pixfmt(ctx, win, state->fb);
+	fimd_win_set_pixfmt(ctx, win, fb->pixel_format, state->src.w);
 
 	/* hardware window 0 doesn't support color key. */
 	if (win != 0)
@@ -731,7 +742,7 @@ static void fimd_disable_plane(struct exynos_drm_crtc *crtc,
 			       struct exynos_drm_plane *plane)
 {
 	struct fimd_context *ctx = crtc->ctx;
-	unsigned int win = plane->zpos;
+	unsigned int win = plane->index;
 
 	if (ctx->suspended)
 		return;
@@ -745,7 +756,6 @@ static void fimd_disable_plane(struct exynos_drm_crtc *crtc,
 static void fimd_enable(struct exynos_drm_crtc *crtc)
 {
 	struct fimd_context *ctx = crtc->ctx;
-	int ret;
 
 	if (!ctx->suspended)
 		return;
@@ -754,18 +764,6 @@ static void fimd_enable(struct exynos_drm_crtc *crtc)
 
 	pm_runtime_get_sync(ctx->dev);
 
-	ret = clk_prepare_enable(ctx->bus_clk);
-	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the bus clk [%d]\n", ret);
-		return;
-	}
-
-	ret = clk_prepare_enable(ctx->lcd_clk);
-	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the lcd clk [%d]\n", ret);
-		return;
-	}
-
 	/* if vblank was enabled status, enable it again. */
 	if (test_and_clear_bit(0, &ctx->irq_flags))
 		fimd_enable_vblank(ctx->crtc);
@@ -795,11 +793,7 @@ static void fimd_disable(struct exynos_drm_crtc *crtc)
 
 	writel(0, ctx->regs + VIDCON0);
 
-	clk_disable_unprepare(ctx->lcd_clk);
-	clk_disable_unprepare(ctx->bus_clk);
-
 	pm_runtime_put_sync(ctx->dev);
-
 	ctx->suspended = true;
 }
 
@@ -941,18 +935,19 @@ static int fimd_bind(struct device *dev, struct device *master, void *data)
 	struct drm_device *drm_dev = data;
 	struct exynos_drm_private *priv = drm_dev->dev_private;
 	struct exynos_drm_plane *exynos_plane;
-	enum drm_plane_type type;
-	unsigned int zpos;
+	unsigned int i;
 	int ret;
 
 	ctx->drm_dev = drm_dev;
 	ctx->pipe = priv->pipe++;
 
-	for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
-		type = exynos_plane_get_type(zpos, CURSOR_WIN);
-		ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
-					1 << ctx->pipe, type, fimd_formats,
-					ARRAY_SIZE(fimd_formats), zpos);
+	for (i = 0; i < WINDOWS_NR; i++) {
+		ctx->configs[i].pixel_formats = fimd_formats;
+		ctx->configs[i].num_pixel_formats = ARRAY_SIZE(fimd_formats);
+		ctx->configs[i].zpos = i;
+		ctx->configs[i].type = fimd_win_types[i];
+		ret = exynos_plane_init(drm_dev, &ctx->planes[i], i,
+					1 << ctx->pipe, &ctx->configs[i]);
 		if (ret)
 			return ret;
 	}
@@ -1121,12 +1116,49 @@ static int fimd_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int exynos_fimd_suspend(struct device *dev)
+{
+	struct fimd_context *ctx = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(ctx->lcd_clk);
+	clk_disable_unprepare(ctx->bus_clk);
+
+	return 0;
+}
+
+static int exynos_fimd_resume(struct device *dev)
+{
+	struct fimd_context *ctx = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(ctx->bus_clk);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the bus clk [%d]\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(ctx->lcd_clk);
+	if  (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the lcd clk [%d]\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos_fimd_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos_fimd_suspend, exynos_fimd_resume, NULL)
+};
+
 struct platform_driver fimd_driver = {
 	.probe		= fimd_probe,
 	.remove		= fimd_remove,
 	.driver		= {
 		.name	= "exynos4-fb",
 		.owner	= THIS_MODULE,
+		.pm	= &exynos_fimd_pm_ops,
 		.of_match_table = fimd_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index 252eb301470c..32358c5e3db4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -14,6 +14,7 @@
 
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
+#include <linux/pfn_t.h>
 #include <drm/exynos_drm.h>
 
 #include "exynos_drm_drv.h"
@@ -490,7 +491,8 @@ int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 
 	pfn = page_to_pfn(exynos_gem->pages[page_offset]);
-	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
+	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
+			__pfn_to_pfn_t(pfn, PFN_DEV));
 
 out:
 	switch (ret) {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h
index 37ab8b282db6..9ca5047959ec 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h
@@ -55,8 +55,6 @@ struct exynos_drm_gem {
 	struct sg_table		*sgt;
 };
 
-struct page **exynos_gem_get_pages(struct drm_gem_object *obj, gfp_t gfpmask);
-
 /* destroy a buffer with gem object */
 void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem);
 
@@ -91,10 +89,6 @@ void exynos_drm_gem_put_dma_addr(struct drm_device *dev,
 					unsigned int gem_handle,
 					struct drm_file *filp);
 
-/* map user space allocated by malloc to pages. */
-int exynos_drm_gem_userptr_ioctl(struct drm_device *dev, void *data,
-				      struct drm_file *file_priv);
-
 /* get buffer information to memory region allocated by gem. */
 int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv);
@@ -123,28 +117,6 @@ int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
 /* set vm_flags and we can change the vm attribute to other one at here. */
 int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 
-static inline int vma_is_io(struct vm_area_struct *vma)
-{
-	return !!(vma->vm_flags & (VM_IO | VM_PFNMAP));
-}
-
-/* get a copy of a virtual memory region. */
-struct vm_area_struct *exynos_gem_get_vma(struct vm_area_struct *vma);
-
-/* release a userspace virtual memory area. */
-void exynos_gem_put_vma(struct vm_area_struct *vma);
-
-/* get pages from user space. */
-int exynos_gem_get_pages_from_userptr(unsigned long start,
-						unsigned int npages,
-						struct page **pages,
-						struct vm_area_struct *vma);
-
-/* drop the reference to pages. */
-void exynos_gem_put_pages_to_userptr(struct page **pages,
-					unsigned int npages,
-					struct vm_area_struct *vma);
-
 /* map sgt with dma region. */
 int exynos_gem_map_sgt_with_dma(struct drm_device *drm_dev,
 				struct sg_table *sgt,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 11b87d2a7913..7aecd23cfa11 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -15,7 +15,8 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/pm_runtime.h>
-#include <plat/map-base.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 
 #include <drm/drmP.h>
 #include <drm/exynos_drm.h>
@@ -126,6 +127,7 @@ struct gsc_capability {
  * @ippdrv: prepare initialization using ippdrv.
  * @regs_res: register resources.
  * @regs: memory mapped io registers.
+ * @sysreg: handle to SYSREG block regmap.
  * @lock: locking of operations.
  * @gsc_clk: gsc gate clock.
  * @sc: scaler infomations.
@@ -138,6 +140,7 @@ struct gsc_context {
 	struct exynos_drm_ippdrv	ippdrv;
 	struct resource	*regs_res;
 	void __iomem	*regs;
+	struct regmap	*sysreg;
 	struct mutex	lock;
 	struct clk	*gsc_clk;
 	struct gsc_scaler	sc;
@@ -437,9 +440,12 @@ static int gsc_sw_reset(struct gsc_context *ctx)
 
 static void gsc_set_gscblk_fimd_wb(struct gsc_context *ctx, bool enable)
 {
-	u32 gscblk_cfg;
+	unsigned int gscblk_cfg;
 
-	gscblk_cfg = readl(SYSREG_GSCBLK_CFG1);
+	if (!ctx->sysreg)
+		return;
+
+	regmap_read(ctx->sysreg, SYSREG_GSCBLK_CFG1, &gscblk_cfg);
 
 	if (enable)
 		gscblk_cfg |= GSC_BLK_DISP1WB_DEST(ctx->id) |
@@ -448,7 +454,7 @@ static void gsc_set_gscblk_fimd_wb(struct gsc_context *ctx, bool enable)
 	else
 		gscblk_cfg |= GSC_BLK_PXLASYNC_LO_MASK_WB(ctx->id);
 
-	writel(gscblk_cfg, SYSREG_GSCBLK_CFG1);
+	regmap_write(ctx->sysreg, SYSREG_GSCBLK_CFG1, gscblk_cfg);
 }
 
 static void gsc_handle_irq(struct gsc_context *ctx, bool enable,
@@ -1215,10 +1221,10 @@ static int gsc_clk_ctrl(struct gsc_context *ctx, bool enable)
 	DRM_DEBUG_KMS("enable[%d]\n", enable);
 
 	if (enable) {
-		clk_enable(ctx->gsc_clk);
+		clk_prepare_enable(ctx->gsc_clk);
 		ctx->suspended = false;
 	} else {
-		clk_disable(ctx->gsc_clk);
+		clk_disable_unprepare(ctx->gsc_clk);
 		ctx->suspended = true;
 	}
 
@@ -1663,6 +1669,15 @@ static int gsc_probe(struct platform_device *pdev)
 	if (!ctx)
 		return -ENOMEM;
 
+	if (dev->of_node) {
+		ctx->sysreg = syscon_regmap_lookup_by_phandle(dev->of_node,
+							"samsung,sysreg");
+		if (IS_ERR(ctx->sysreg)) {
+			dev_warn(dev, "failed to get system register.\n");
+			ctx->sysreg = NULL;
+		}
+	}
+
 	/* clock control */
 	ctx->gsc_clk = devm_clk_get(dev, "gscl");
 	if (IS_ERR(ctx->gsc_clk)) {
@@ -1713,7 +1728,6 @@ static int gsc_probe(struct platform_device *pdev)
 	mutex_init(&ctx->lock);
 	platform_set_drvdata(pdev, ctx);
 
-	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
 
 	ret = exynos_drm_ippdrv_register(ippdrv);
@@ -1797,6 +1811,12 @@ static const struct dev_pm_ops gsc_pm_ops = {
 	SET_RUNTIME_PM_OPS(gsc_runtime_suspend, gsc_runtime_resume, NULL)
 };
 
+static const struct of_device_id exynos_drm_gsc_of_match[] = {
+	{ .compatible = "samsung,exynos5-gsc" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, exynos_drm_gsc_of_match);
+
 struct platform_driver gsc_driver = {
 	.probe		= gsc_probe,
 	.remove		= gsc_remove,
@@ -1804,6 +1824,7 @@ struct platform_driver gsc_driver = {
 		.name	= "exynos-drm-gsc",
 		.owner	= THIS_MODULE,
 		.pm	= &gsc_pm_ops,
+		.of_match_table = of_match_ptr(exynos_drm_gsc_of_match),
 	},
 };
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index 8994eab56ba8..4eaef36aec5a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -389,7 +389,7 @@ already_disabled:
 	mutex_unlock(&mic_mutex);
 }
 
-struct drm_bridge_funcs mic_bridge_funcs = {
+static const struct drm_bridge_funcs mic_bridge_funcs = {
 	.disable = mic_disable,
 	.post_disable = mic_post_disable,
 	.pre_enable = mic_pre_enable,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 179311760bb7..d86227236f55 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -56,93 +56,213 @@ static int exynos_plane_get_size(int start, unsigned length, unsigned last)
 	return size;
 }
 
-static void exynos_plane_mode_set(struct drm_plane *plane,
-				  struct drm_crtc *crtc,
-				  struct drm_framebuffer *fb,
-				  int crtc_x, int crtc_y,
-				  unsigned int crtc_w, unsigned int crtc_h,
-				  uint32_t src_x, uint32_t src_y,
-				  uint32_t src_w, uint32_t src_h)
+static void exynos_plane_mode_set(struct exynos_drm_plane_state *exynos_state)
+
 {
-	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
+	struct drm_plane_state *state = &exynos_state->base;
+	struct drm_crtc *crtc = exynos_state->base.crtc;
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	int crtc_x, crtc_y;
+	unsigned int crtc_w, crtc_h;
+	unsigned int src_x, src_y;
+	unsigned int src_w, src_h;
 	unsigned int actual_w;
 	unsigned int actual_h;
 
+	/*
+	 * The original src/dest coordinates are stored in exynos_state->base,
+	 * but we want to keep another copy internal to our driver that we can
+	 * clip/modify ourselves.
+	 */
+
+	crtc_x = state->crtc_x;
+	crtc_y = state->crtc_y;
+	crtc_w = state->crtc_w;
+	crtc_h = state->crtc_h;
+
+	src_x = state->src_x >> 16;
+	src_y = state->src_y >> 16;
+	src_w = state->src_w >> 16;
+	src_h = state->src_h >> 16;
+
+	/* set ratio */
+	exynos_state->h_ratio = (src_w << 16) / crtc_w;
+	exynos_state->v_ratio = (src_h << 16) / crtc_h;
+
+	/* clip to visible area */
 	actual_w = exynos_plane_get_size(crtc_x, crtc_w, mode->hdisplay);
 	actual_h = exynos_plane_get_size(crtc_y, crtc_h, mode->vdisplay);
 
 	if (crtc_x < 0) {
 		if (actual_w)
-			src_x -= crtc_x;
+			src_x += ((-crtc_x) * exynos_state->h_ratio) >> 16;
 		crtc_x = 0;
 	}
 
 	if (crtc_y < 0) {
 		if (actual_h)
-			src_y -= crtc_y;
+			src_y += ((-crtc_y) * exynos_state->v_ratio) >> 16;
 		crtc_y = 0;
 	}
 
-	/* set ratio */
-	exynos_plane->h_ratio = (src_w << 16) / crtc_w;
-	exynos_plane->v_ratio = (src_h << 16) / crtc_h;
-
 	/* set drm framebuffer data. */
-	exynos_plane->src_x = src_x;
-	exynos_plane->src_y = src_y;
-	exynos_plane->src_w = (actual_w * exynos_plane->h_ratio) >> 16;
-	exynos_plane->src_h = (actual_h * exynos_plane->v_ratio) >> 16;
+	exynos_state->src.x = src_x;
+	exynos_state->src.y = src_y;
+	exynos_state->src.w = (actual_w * exynos_state->h_ratio) >> 16;
+	exynos_state->src.h = (actual_h * exynos_state->v_ratio) >> 16;
 
 	/* set plane range to be displayed. */
-	exynos_plane->crtc_x = crtc_x;
-	exynos_plane->crtc_y = crtc_y;
-	exynos_plane->crtc_w = actual_w;
-	exynos_plane->crtc_h = actual_h;
+	exynos_state->crtc.x = crtc_x;
+	exynos_state->crtc.y = crtc_y;
+	exynos_state->crtc.w = actual_w;
+	exynos_state->crtc.h = actual_h;
 
 	DRM_DEBUG_KMS("plane : offset_x/y(%d,%d), width/height(%d,%d)",
-			exynos_plane->crtc_x, exynos_plane->crtc_y,
-			exynos_plane->crtc_w, exynos_plane->crtc_h);
+			exynos_state->crtc.x, exynos_state->crtc.y,
+			exynos_state->crtc.w, exynos_state->crtc.h);
+}
+
+static void exynos_drm_plane_reset(struct drm_plane *plane)
+{
+	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
+	struct exynos_drm_plane_state *exynos_state;
+
+	if (plane->state) {
+		exynos_state = to_exynos_plane_state(plane->state);
+		if (exynos_state->base.fb)
+			drm_framebuffer_unreference(exynos_state->base.fb);
+		kfree(exynos_state);
+		plane->state = NULL;
+	}
+
+	exynos_state = kzalloc(sizeof(*exynos_state), GFP_KERNEL);
+	if (exynos_state) {
+		exynos_state->zpos = exynos_plane->config->zpos;
+		plane->state = &exynos_state->base;
+		plane->state->plane = plane;
+	}
+}
+
+static struct drm_plane_state *
+exynos_drm_plane_duplicate_state(struct drm_plane *plane)
+{
+	struct exynos_drm_plane_state *exynos_state;
+	struct exynos_drm_plane_state *copy;
+
+	exynos_state = to_exynos_plane_state(plane->state);
+	copy = kzalloc(sizeof(*exynos_state), GFP_KERNEL);
+	if (!copy)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &copy->base);
+	copy->zpos = exynos_state->zpos;
+	return &copy->base;
+}
+
+static void exynos_drm_plane_destroy_state(struct drm_plane *plane,
+					   struct drm_plane_state *old_state)
+{
+	struct exynos_drm_plane_state *old_exynos_state =
+					to_exynos_plane_state(old_state);
+	__drm_atomic_helper_plane_destroy_state(plane, old_state);
+	kfree(old_exynos_state);
+}
+
+static int exynos_drm_plane_atomic_set_property(struct drm_plane *plane,
+						struct drm_plane_state *state,
+						struct drm_property *property,
+						uint64_t val)
+{
+	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
+	struct exynos_drm_plane_state *exynos_state =
+					to_exynos_plane_state(state);
+	struct exynos_drm_private *dev_priv = plane->dev->dev_private;
+	const struct exynos_drm_plane_config *config = exynos_plane->config;
 
-	plane->crtc = crtc;
+	if (property == dev_priv->plane_zpos_property &&
+	    (config->capabilities & EXYNOS_DRM_PLANE_CAP_ZPOS))
+		exynos_state->zpos = val;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int exynos_drm_plane_atomic_get_property(struct drm_plane *plane,
+					  const struct drm_plane_state *state,
+					  struct drm_property *property,
+					  uint64_t *val)
+{
+	const struct exynos_drm_plane_state *exynos_state =
+		container_of(state, const struct exynos_drm_plane_state, base);
+	struct exynos_drm_private *dev_priv = plane->dev->dev_private;
+
+	if (property == dev_priv->plane_zpos_property)
+		*val = exynos_state->zpos;
+	else
+		return -EINVAL;
+
+	return 0;
 }
 
 static struct drm_plane_funcs exynos_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
-	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.set_property	= drm_atomic_helper_plane_set_property,
+	.reset		= exynos_drm_plane_reset,
+	.atomic_duplicate_state = exynos_drm_plane_duplicate_state,
+	.atomic_destroy_state = exynos_drm_plane_destroy_state,
+	.atomic_set_property = exynos_drm_plane_atomic_set_property,
+	.atomic_get_property = exynos_drm_plane_atomic_get_property,
 };
 
+static int
+exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config,
+			    struct exynos_drm_plane_state *state)
+{
+	bool width_ok = false, height_ok = false;
+
+	if (config->capabilities & EXYNOS_DRM_PLANE_CAP_SCALE)
+		return 0;
+
+	if (state->src.w == state->crtc.w)
+		width_ok = true;
+
+	if (state->src.h == state->crtc.h)
+		height_ok = true;
+
+	if ((config->capabilities & EXYNOS_DRM_PLANE_CAP_DOUBLE) &&
+	    state->h_ratio == (1 << 15))
+		width_ok = true;
+
+	if ((config->capabilities & EXYNOS_DRM_PLANE_CAP_DOUBLE) &&
+	    state->v_ratio == (1 << 15))
+		height_ok = true;
+
+	if (width_ok & height_ok)
+		return 0;
+
+	DRM_DEBUG_KMS("scaling mode is not supported");
+	return -ENOTSUPP;
+}
+
 static int exynos_plane_atomic_check(struct drm_plane *plane,
 				     struct drm_plane_state *state)
 {
 	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
-	int nr;
-	int i;
+	struct exynos_drm_plane_state *exynos_state =
+						to_exynos_plane_state(state);
+	int ret = 0;
 
-	if (!state->fb)
+	if (!state->crtc || !state->fb)
 		return 0;
 
-	nr = drm_format_num_planes(state->fb->pixel_format);
-	for (i = 0; i < nr; i++) {
-		struct exynos_drm_gem *exynos_gem =
-					exynos_drm_fb_gem(state->fb, i);
-		if (!exynos_gem) {
-			DRM_DEBUG_KMS("gem object is null\n");
-			return -EFAULT;
-		}
-
-		exynos_plane->dma_addr[i] = exynos_gem->dma_addr +
-					    state->fb->offsets[i];
+	/* translate state into exynos_state */
+	exynos_plane_mode_set(exynos_state);
 
-		DRM_DEBUG_KMS("buffer: %d, dma_addr = 0x%lx\n",
-				i, (unsigned long)exynos_plane->dma_addr[i]);
-	}
-
-	return 0;
+	ret = exynos_drm_plane_check_size(exynos_plane->config, exynos_state);
+	return ret;
 }
 
 static void exynos_plane_atomic_update(struct drm_plane *plane,
@@ -155,12 +275,7 @@ static void exynos_plane_atomic_update(struct drm_plane *plane,
 	if (!state->crtc)
 		return;
 
-	exynos_plane_mode_set(plane, state->crtc, state->fb,
-			      state->crtc_x, state->crtc_y,
-			      state->crtc_w, state->crtc_h,
-			      state->src_x >> 16, state->src_y >> 16,
-			      state->src_w >> 16, state->src_h >> 16);
-
+	plane->crtc = state->crtc;
 	exynos_plane->pending_fb = state->fb;
 
 	if (exynos_crtc->ops->update_plane)
@@ -177,8 +292,7 @@ static void exynos_plane_atomic_disable(struct drm_plane *plane,
 		return;
 
 	if (exynos_crtc->ops->disable_plane)
-		exynos_crtc->ops->disable_plane(exynos_crtc,
-						exynos_plane);
+		exynos_crtc->ops->disable_plane(exynos_crtc, exynos_plane);
 }
 
 static const struct drm_plane_helper_funcs plane_helper_funcs = {
@@ -196,8 +310,8 @@ static void exynos_plane_attach_zpos_property(struct drm_plane *plane,
 
 	prop = dev_priv->plane_zpos_property;
 	if (!prop) {
-		prop = drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE,
-						 "zpos", 0, MAX_PLANE - 1);
+		prop = drm_property_create_range(dev, 0, "zpos",
+						 0, MAX_PLANE - 1);
 		if (!prop)
 			return;
 
@@ -207,28 +321,19 @@ static void exynos_plane_attach_zpos_property(struct drm_plane *plane,
 	drm_object_attach_property(&plane->base, prop, zpos);
 }
 
-enum drm_plane_type exynos_plane_get_type(unsigned int zpos,
-					  unsigned int cursor_win)
-{
-		if (zpos == DEFAULT_WIN)
-			return DRM_PLANE_TYPE_PRIMARY;
-		else if (zpos == cursor_win)
-			return DRM_PLANE_TYPE_CURSOR;
-		else
-			return DRM_PLANE_TYPE_OVERLAY;
-}
-
 int exynos_plane_init(struct drm_device *dev,
 		      struct exynos_drm_plane *exynos_plane,
-		      unsigned long possible_crtcs, enum drm_plane_type type,
-		      const uint32_t *formats, unsigned int fcount,
-		      unsigned int zpos)
+		      unsigned int index, unsigned long possible_crtcs,
+		      const struct exynos_drm_plane_config *config)
 {
 	int err;
 
-	err = drm_universal_plane_init(dev, &exynos_plane->base, possible_crtcs,
-				       &exynos_plane_funcs, formats, fcount,
-				       type);
+	err = drm_universal_plane_init(dev, &exynos_plane->base,
+				       possible_crtcs,
+				       &exynos_plane_funcs,
+				       config->pixel_formats,
+				       config->num_pixel_formats,
+				       config->type, NULL);
 	if (err) {
 		DRM_ERROR("failed to initialize plane\n");
 		return err;
@@ -236,10 +341,10 @@ int exynos_plane_init(struct drm_device *dev,
 
 	drm_plane_helper_add(&exynos_plane->base, &plane_helper_funcs);
 
-	exynos_plane->zpos = zpos;
+	exynos_plane->index = index;
+	exynos_plane->config = config;
 
-	if (type == DRM_PLANE_TYPE_OVERLAY)
-		exynos_plane_attach_zpos_property(&exynos_plane->base, zpos);
+	exynos_plane_attach_zpos_property(&exynos_plane->base, config->zpos);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.h b/drivers/gpu/drm/exynos/exynos_drm_plane.h
index abb641e64c23..9aafad164cdf 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.h
@@ -9,10 +9,7 @@
  *
  */
 
-enum drm_plane_type exynos_plane_get_type(unsigned int zpos,
-					  unsigned int cursor_win);
 int exynos_plane_init(struct drm_device *dev,
-		      struct exynos_drm_plane *exynos_plane,
-		      unsigned long possible_crtcs, enum drm_plane_type type,
-		      const uint32_t *formats, unsigned int fcount,
-		      unsigned int zpos);
+		      struct exynos_drm_plane *exynos_plane, unsigned int index,
+		      unsigned long possible_crtcs,
+		      const struct exynos_drm_plane_config *config);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 2f5c118f4c8e..bea0f7826d30 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -790,10 +790,10 @@ static int rotator_remove(struct platform_device *pdev)
 static int rotator_clk_crtl(struct rot_context *rot, bool enable)
 {
 	if (enable) {
-		clk_enable(rot->clock);
+		clk_prepare_enable(rot->clock);
 		rot->suspended = false;
 	} else {
-		clk_disable(rot->clock);
+		clk_disable_unprepare(rot->clock);
 		rot->suspended = true;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 669362c53f49..62ac4e5fa51d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -24,12 +24,12 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
+#include "exynos_drm_fb.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_vidi.h"
 
 /* vidi has totally three virtual windows. */
 #define WINDOWS_NR		3
-#define CURSOR_WIN		2
 
 #define ctx_from_connector(c)	container_of(c, struct vidi_context, \
 					connector)
@@ -89,6 +89,12 @@ static const uint32_t formats[] = {
 	DRM_FORMAT_NV12,
 };
 
+static const enum drm_plane_type vidi_win_types[WINDOWS_NR] = {
+	DRM_PLANE_TYPE_PRIMARY,
+	DRM_PLANE_TYPE_OVERLAY,
+	DRM_PLANE_TYPE_CURSOR,
+};
+
 static int vidi_enable_vblank(struct exynos_drm_crtc *crtc)
 {
 	struct vidi_context *ctx = crtc->ctx;
@@ -125,12 +131,15 @@ static void vidi_disable_vblank(struct exynos_drm_crtc *crtc)
 static void vidi_update_plane(struct exynos_drm_crtc *crtc,
 			      struct exynos_drm_plane *plane)
 {
+	struct drm_plane_state *state = plane->base.state;
 	struct vidi_context *ctx = crtc->ctx;
+	dma_addr_t addr;
 
 	if (ctx->suspended)
 		return;
 
-	DRM_DEBUG_KMS("dma_addr = %pad\n", plane->dma_addr);
+	addr = exynos_drm_fb_dma_addr(state->fb, 0);
+	DRM_DEBUG_KMS("dma_addr = %pad\n", &addr);
 
 	if (ctx->vblank_on)
 		schedule_work(&ctx->work);
@@ -330,7 +339,7 @@ static void vidi_connector_destroy(struct drm_connector *connector)
 {
 }
 
-static struct drm_connector_funcs vidi_connector_funcs = {
+static const struct drm_connector_funcs vidi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = vidi_detect,
@@ -374,7 +383,7 @@ static struct drm_encoder *vidi_best_encoder(struct drm_connector *connector)
 	return &ctx->encoder;
 }
 
-static struct drm_connector_helper_funcs vidi_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs vidi_connector_helper_funcs = {
 	.get_modes = vidi_get_modes,
 	.best_encoder = vidi_best_encoder,
 };
@@ -422,14 +431,14 @@ static void exynos_vidi_disable(struct drm_encoder *encoder)
 {
 }
 
-static struct drm_encoder_helper_funcs exynos_vidi_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs exynos_vidi_encoder_helper_funcs = {
 	.mode_fixup = exynos_vidi_mode_fixup,
 	.mode_set = exynos_vidi_mode_set,
 	.enable = exynos_vidi_enable,
 	.disable = exynos_vidi_disable,
 };
 
-static struct drm_encoder_funcs exynos_vidi_encoder_funcs = {
+static const struct drm_encoder_funcs exynos_vidi_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -439,17 +448,21 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 	struct drm_device *drm_dev = data;
 	struct drm_encoder *encoder = &ctx->encoder;
 	struct exynos_drm_plane *exynos_plane;
-	enum drm_plane_type type;
-	unsigned int zpos;
+	struct exynos_drm_plane_config plane_config = { 0 };
+	unsigned int i;
 	int pipe, ret;
 
 	vidi_ctx_initialize(ctx, drm_dev);
 
-	for (zpos = 0; zpos < WINDOWS_NR; zpos++) {
-		type = exynos_plane_get_type(zpos, CURSOR_WIN);
-		ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
-					1 << ctx->pipe, type, formats,
-					ARRAY_SIZE(formats), zpos);
+	plane_config.pixel_formats = formats;
+	plane_config.num_pixel_formats = ARRAY_SIZE(formats);
+
+	for (i = 0; i < WINDOWS_NR; i++) {
+		plane_config.zpos = i;
+		plane_config.type = vidi_win_types[i];
+
+		ret = exynos_plane_init(drm_dev, &ctx->planes[i], i,
+					1 << ctx->pipe, &plane_config);
 		if (ret)
 			return ret;
 	}
@@ -473,7 +486,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
 	drm_encoder_init(drm_dev, encoder, &exynos_vidi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_vidi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 57b675563e94..21a29dbce18c 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -113,7 +113,7 @@ struct hdmi_context {
 	void __iomem			*regs_hdmiphy;
 	struct i2c_client		*hdmiphy_port;
 	struct i2c_adapter		*ddc_adpt;
-	struct gpio_desc 		*hpd_gpio;
+	struct gpio_desc		*hpd_gpio;
 	int				irq;
 	struct regmap			*pmureg;
 	struct clk			*hdmi;
@@ -956,7 +956,7 @@ static void hdmi_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs hdmi_connector_funcs = {
+static const struct drm_connector_funcs hdmi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = hdmi_detect,
@@ -1030,7 +1030,7 @@ static struct drm_encoder *hdmi_best_encoder(struct drm_connector *connector)
 	return &hdata->encoder;
 }
 
-static struct drm_connector_helper_funcs hdmi_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs hdmi_connector_helper_funcs = {
 	.get_modes = hdmi_get_modes,
 	.mode_valid = hdmi_mode_valid,
 	.best_encoder = hdmi_best_encoder,
@@ -1588,8 +1588,6 @@ static void hdmi_enable(struct drm_encoder *encoder)
 	if (hdata->powered)
 		return;
 
-	hdata->powered = true;
-
 	pm_runtime_get_sync(hdata->dev);
 
 	if (regulator_bulk_enable(ARRAY_SIZE(supply), hdata->regul_bulk))
@@ -1599,10 +1597,9 @@ static void hdmi_enable(struct drm_encoder *encoder)
 	regmap_update_bits(hdata->pmureg, PMU_HDMI_PHY_CONTROL,
 			PMU_HDMI_PHY_ENABLE_BIT, 1);
 
-	clk_prepare_enable(hdata->hdmi);
-	clk_prepare_enable(hdata->sclk_hdmi);
-
 	hdmi_conf_apply(hdata);
+
+	hdata->powered = true;
 }
 
 static void hdmi_disable(struct drm_encoder *encoder)
@@ -1633,9 +1630,6 @@ static void hdmi_disable(struct drm_encoder *encoder)
 
 	cancel_delayed_work(&hdata->hotplug_work);
 
-	clk_disable_unprepare(hdata->sclk_hdmi);
-	clk_disable_unprepare(hdata->hdmi);
-
 	/* reset pmu hdmiphy control bit to disable hdmiphy */
 	regmap_update_bits(hdata->pmureg, PMU_HDMI_PHY_CONTROL,
 			PMU_HDMI_PHY_ENABLE_BIT, 0);
@@ -1647,14 +1641,14 @@ static void hdmi_disable(struct drm_encoder *encoder)
 	hdata->powered = false;
 }
 
-static struct drm_encoder_helper_funcs exynos_hdmi_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs exynos_hdmi_encoder_helper_funcs = {
 	.mode_fixup	= hdmi_mode_fixup,
 	.mode_set	= hdmi_mode_set,
 	.enable		= hdmi_enable,
 	.disable	= hdmi_disable,
 };
 
-static struct drm_encoder_funcs exynos_hdmi_encoder_funcs = {
+static const struct drm_encoder_funcs exynos_hdmi_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -1793,7 +1787,7 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
 	drm_encoder_init(drm_dev, encoder, &exynos_hdmi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_hdmi_encoder_helper_funcs);
 
@@ -1978,12 +1972,49 @@ static int hdmi_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int exynos_hdmi_suspend(struct device *dev)
+{
+	struct hdmi_context *hdata = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(hdata->sclk_hdmi);
+	clk_disable_unprepare(hdata->hdmi);
+
+	return 0;
+}
+
+static int exynos_hdmi_resume(struct device *dev)
+{
+	struct hdmi_context *hdata = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(hdata->hdmi);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the hdmi clk [%d]\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(hdata->sclk_hdmi);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the sclk_mixer clk [%d]\n",
+			  ret);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos_hdmi_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos_hdmi_suspend, exynos_hdmi_resume, NULL)
+};
+
 struct platform_driver hdmi_driver = {
 	.probe		= hdmi_probe,
 	.remove		= hdmi_remove,
 	.driver		= {
 		.name	= "exynos-hdmi",
 		.owner	= THIS_MODULE,
+		.pm	= &exynos_hdmi_pm_ops,
 		.of_match_table = hdmi_match_types,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index d09f8f9a8939..b5fbc1cbf024 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -37,12 +37,12 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
+#include "exynos_drm_fb.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_iommu.h"
 
 #define MIXER_WIN_NR		3
 #define VP_DEFAULT_WIN		2
-#define CURSOR_WIN		1
 
 /* The pixelformats that are natively supported by the mixer. */
 #define MXR_FORMAT_RGB565	4
@@ -76,7 +76,9 @@ enum mixer_flag_bits {
 
 static const uint32_t mixer_formats[] = {
 	DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_RGB565,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_ARGB8888,
@@ -111,6 +113,31 @@ struct mixer_drv_data {
 	bool					has_sclk;
 };
 
+static const struct exynos_drm_plane_config plane_configs[MIXER_WIN_NR] = {
+	{
+		.zpos = 0,
+		.type = DRM_PLANE_TYPE_PRIMARY,
+		.pixel_formats = mixer_formats,
+		.num_pixel_formats = ARRAY_SIZE(mixer_formats),
+		.capabilities = EXYNOS_DRM_PLANE_CAP_DOUBLE |
+				EXYNOS_DRM_PLANE_CAP_ZPOS,
+	}, {
+		.zpos = 1,
+		.type = DRM_PLANE_TYPE_CURSOR,
+		.pixel_formats = mixer_formats,
+		.num_pixel_formats = ARRAY_SIZE(mixer_formats),
+		.capabilities = EXYNOS_DRM_PLANE_CAP_DOUBLE |
+				EXYNOS_DRM_PLANE_CAP_ZPOS,
+	}, {
+		.zpos = 2,
+		.type = DRM_PLANE_TYPE_OVERLAY,
+		.pixel_formats = vp_formats,
+		.num_pixel_formats = ARRAY_SIZE(vp_formats),
+		.capabilities = EXYNOS_DRM_PLANE_CAP_SCALE |
+				EXYNOS_DRM_PLANE_CAP_ZPOS,
+	},
+};
+
 static const u8 filter_y_horiz_tap8[] = {
 	0,	-1,	-1,	-1,	-1,	-1,	-1,	-1,
 	-1,	-1,	-1,	-1,	-1,	0,	0,	0,
@@ -140,6 +167,18 @@ static const u8 filter_cr_horiz_tap4[] = {
 	70,	59,	48,	37,	27,	19,	11,	5,
 };
 
+static inline bool is_alpha_format(unsigned int pixel_format)
+{
+	switch (pixel_format) {
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_ARGB4444:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static inline u32 vp_reg_read(struct mixer_resources *res, u32 reg_id)
 {
 	return readl(res->vp_regs + reg_id);
@@ -269,6 +308,37 @@ static void vp_default_filter(struct mixer_resources *res)
 		filter_cr_horiz_tap4, sizeof(filter_cr_horiz_tap4));
 }
 
+static void mixer_cfg_gfx_blend(struct mixer_context *ctx, unsigned int win,
+				bool alpha)
+{
+	struct mixer_resources *res = &ctx->mixer_res;
+	u32 val;
+
+	val  = MXR_GRP_CFG_COLOR_KEY_DISABLE; /* no blank key */
+	if (alpha) {
+		/* blending based on pixel alpha */
+		val |= MXR_GRP_CFG_BLEND_PRE_MUL;
+		val |= MXR_GRP_CFG_PIXEL_BLEND_EN;
+	}
+	mixer_reg_writemask(res, MXR_GRAPHIC_CFG(win),
+			    val, MXR_GRP_CFG_MISC_MASK);
+}
+
+static void mixer_cfg_vp_blend(struct mixer_context *ctx)
+{
+	struct mixer_resources *res = &ctx->mixer_res;
+	u32 val;
+
+	/*
+	 * No blending at the moment since the NV12/NV21 pixelformats don't
+	 * have an alpha channel. However the mixer supports a global alpha
+	 * value for a layer. Once this functionality is exposed, we can
+	 * support blending of the video layer through this.
+	 */
+	val = 0;
+	mixer_reg_write(res, MXR_VIDEO_CFG, val);
+}
+
 static void mixer_vsync_set_update(struct mixer_context *ctx, bool enable)
 {
 	struct mixer_resources *res = &ctx->mixer_res;
@@ -350,7 +420,7 @@ static void mixer_cfg_rgb_fmt(struct mixer_context *ctx, unsigned int height)
 }
 
 static void mixer_cfg_layer(struct mixer_context *ctx, unsigned int win,
-				bool enable)
+			    unsigned int priority, bool enable)
 {
 	struct mixer_resources *res = &ctx->mixer_res;
 	u32 val = enable ? ~0 : 0;
@@ -358,20 +428,24 @@ static void mixer_cfg_layer(struct mixer_context *ctx, unsigned int win,
 	switch (win) {
 	case 0:
 		mixer_reg_writemask(res, MXR_CFG, val, MXR_CFG_GRP0_ENABLE);
+		mixer_reg_writemask(res, MXR_LAYER_CFG,
+				    MXR_LAYER_CFG_GRP0_VAL(priority),
+				    MXR_LAYER_CFG_GRP0_MASK);
 		break;
 	case 1:
 		mixer_reg_writemask(res, MXR_CFG, val, MXR_CFG_GRP1_ENABLE);
+		mixer_reg_writemask(res, MXR_LAYER_CFG,
+				    MXR_LAYER_CFG_GRP1_VAL(priority),
+				    MXR_LAYER_CFG_GRP1_MASK);
 		break;
-	case 2:
+	case VP_DEFAULT_WIN:
 		if (ctx->vp_enabled) {
 			vp_reg_writemask(res, VP_ENABLE, val, VP_ENABLE_ON);
 			mixer_reg_writemask(res, MXR_CFG, val,
 				MXR_CFG_VP_ENABLE);
-
-			/* control blending of graphic layer 0 */
-			mixer_reg_writemask(res, MXR_GRAPHIC_CFG(0), val,
-					MXR_GRP_CFG_BLEND_PRE_MUL |
-					MXR_GRP_CFG_PIXEL_BLEND_EN);
+			mixer_reg_writemask(res, MXR_LAYER_CFG,
+					    MXR_LAYER_CFG_VP_VAL(priority),
+					    MXR_LAYER_CFG_VP_MASK);
 		}
 		break;
 	}
@@ -399,10 +473,11 @@ static void mixer_stop(struct mixer_context *ctx)
 static void vp_video_buffer(struct mixer_context *ctx,
 			    struct exynos_drm_plane *plane)
 {
+	struct exynos_drm_plane_state *state =
+				to_exynos_plane_state(plane->base.state);
+	struct drm_display_mode *mode = &state->base.crtc->state->adjusted_mode;
 	struct mixer_resources *res = &ctx->mixer_res;
-	struct drm_plane_state *state = plane->base.state;
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_display_mode *mode = &state->crtc->mode;
+	struct drm_framebuffer *fb = state->base.fb;
 	unsigned long flags;
 	dma_addr_t luma_addr[2], chroma_addr[2];
 	bool tiled_mode = false;
@@ -422,8 +497,8 @@ static void vp_video_buffer(struct mixer_context *ctx,
 		return;
 	}
 
-	luma_addr[0] = plane->dma_addr[0];
-	chroma_addr[0] = plane->dma_addr[1];
+	luma_addr[0] = exynos_drm_fb_dma_addr(fb, 0);
+	chroma_addr[0] = exynos_drm_fb_dma_addr(fb, 1);
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 		ctx->interlace = true;
@@ -441,7 +516,6 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	}
 
 	spin_lock_irqsave(&res->reg_slock, flags);
-	mixer_vsync_set_update(ctx, false);
 
 	/* interlace or progressive scan mode */
 	val = (ctx->interlace ? ~0 : 0);
@@ -459,24 +533,24 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	vp_reg_write(res, VP_IMG_SIZE_C, VP_IMG_HSIZE(fb->pitches[0]) |
 		VP_IMG_VSIZE(fb->height / 2));
 
-	vp_reg_write(res, VP_SRC_WIDTH, plane->src_w);
-	vp_reg_write(res, VP_SRC_HEIGHT, plane->src_h);
+	vp_reg_write(res, VP_SRC_WIDTH, state->src.w);
+	vp_reg_write(res, VP_SRC_HEIGHT, state->src.h);
 	vp_reg_write(res, VP_SRC_H_POSITION,
-			VP_SRC_H_POSITION_VAL(plane->src_x));
-	vp_reg_write(res, VP_SRC_V_POSITION, plane->src_y);
+			VP_SRC_H_POSITION_VAL(state->src.x));
+	vp_reg_write(res, VP_SRC_V_POSITION, state->src.y);
 
-	vp_reg_write(res, VP_DST_WIDTH, plane->crtc_w);
-	vp_reg_write(res, VP_DST_H_POSITION, plane->crtc_x);
+	vp_reg_write(res, VP_DST_WIDTH, state->crtc.w);
+	vp_reg_write(res, VP_DST_H_POSITION, state->crtc.x);
 	if (ctx->interlace) {
-		vp_reg_write(res, VP_DST_HEIGHT, plane->crtc_h / 2);
-		vp_reg_write(res, VP_DST_V_POSITION, plane->crtc_y / 2);
+		vp_reg_write(res, VP_DST_HEIGHT, state->crtc.h / 2);
+		vp_reg_write(res, VP_DST_V_POSITION, state->crtc.y / 2);
 	} else {
-		vp_reg_write(res, VP_DST_HEIGHT, plane->crtc_h);
-		vp_reg_write(res, VP_DST_V_POSITION, plane->crtc_y);
+		vp_reg_write(res, VP_DST_HEIGHT, state->crtc.h);
+		vp_reg_write(res, VP_DST_V_POSITION, state->crtc.y);
 	}
 
-	vp_reg_write(res, VP_H_RATIO, plane->h_ratio);
-	vp_reg_write(res, VP_V_RATIO, plane->v_ratio);
+	vp_reg_write(res, VP_H_RATIO, state->h_ratio);
+	vp_reg_write(res, VP_V_RATIO, state->v_ratio);
 
 	vp_reg_write(res, VP_ENDIAN_MODE, VP_ENDIAN_MODE_LITTLE);
 
@@ -488,10 +562,10 @@ static void vp_video_buffer(struct mixer_context *ctx,
 
 	mixer_cfg_scan(ctx, mode->vdisplay);
 	mixer_cfg_rgb_fmt(ctx, mode->vdisplay);
-	mixer_cfg_layer(ctx, plane->zpos, true);
+	mixer_cfg_layer(ctx, plane->index, state->zpos + 1, true);
+	mixer_cfg_vp_blend(ctx);
 	mixer_run(ctx);
 
-	mixer_vsync_set_update(ctx, true);
 	spin_unlock_irqrestore(&res->reg_slock, flags);
 
 	mixer_regs_dump(ctx);
@@ -505,39 +579,16 @@ static void mixer_layer_update(struct mixer_context *ctx)
 	mixer_reg_writemask(res, MXR_CFG, ~0, MXR_CFG_LAYER_UPDATE);
 }
 
-static int mixer_setup_scale(const struct exynos_drm_plane *plane,
-		unsigned int *x_ratio, unsigned int *y_ratio)
-{
-	if (plane->crtc_w != plane->src_w) {
-		if (plane->crtc_w == 2 * plane->src_w)
-			*x_ratio = 1;
-		else
-			goto fail;
-	}
-
-	if (plane->crtc_h != plane->src_h) {
-		if (plane->crtc_h == 2 * plane->src_h)
-			*y_ratio = 1;
-		else
-			goto fail;
-	}
-
-	return 0;
-
-fail:
-	DRM_DEBUG_KMS("only 2x width/height scaling of plane supported\n");
-	return -ENOTSUPP;
-}
-
 static void mixer_graph_buffer(struct mixer_context *ctx,
 			       struct exynos_drm_plane *plane)
 {
+	struct exynos_drm_plane_state *state =
+				to_exynos_plane_state(plane->base.state);
+	struct drm_display_mode *mode = &state->base.crtc->state->adjusted_mode;
 	struct mixer_resources *res = &ctx->mixer_res;
-	struct drm_plane_state *state = plane->base.state;
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_display_mode *mode = &state->crtc->mode;
+	struct drm_framebuffer *fb = state->base.fb;
 	unsigned long flags;
-	unsigned int win = plane->zpos;
+	unsigned int win = plane->index;
 	unsigned int x_ratio = 0, y_ratio = 0;
 	unsigned int src_x_offset, src_y_offset, dst_x_offset, dst_y_offset;
 	dma_addr_t dma_addr;
@@ -546,10 +597,12 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 
 	switch (fb->pixel_format) {
 	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_ARGB4444:
 		fmt = MXR_FORMAT_ARGB4444;
 		break;
 
 	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_ARGB1555:
 		fmt = MXR_FORMAT_ARGB1555;
 		break;
 
@@ -567,17 +620,17 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 		return;
 	}
 
-	/* check if mixer supports requested scaling setup */
-	if (mixer_setup_scale(plane, &x_ratio, &y_ratio))
-		return;
+	/* ratio is already checked by common plane code */
+	x_ratio = state->h_ratio == (1 << 15);
+	y_ratio = state->v_ratio == (1 << 15);
 
-	dst_x_offset = plane->crtc_x;
-	dst_y_offset = plane->crtc_y;
+	dst_x_offset = state->crtc.x;
+	dst_y_offset = state->crtc.y;
 
 	/* converting dma address base and source offset */
-	dma_addr = plane->dma_addr[0]
-		+ (plane->src_x * fb->bits_per_pixel >> 3)
-		+ (plane->src_y * fb->pitches[0]);
+	dma_addr = exynos_drm_fb_dma_addr(fb, 0)
+		+ (state->src.x * fb->bits_per_pixel >> 3)
+		+ (state->src.y * fb->pitches[0]);
 	src_x_offset = 0;
 	src_y_offset = 0;
 
@@ -587,7 +640,6 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 		ctx->interlace = false;
 
 	spin_lock_irqsave(&res->reg_slock, flags);
-	mixer_vsync_set_update(ctx, false);
 
 	/* setup format */
 	mixer_reg_writemask(res, MXR_GRAPHIC_CFG(win),
@@ -605,8 +657,8 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 		mixer_reg_write(res, MXR_RESOLUTION, val);
 	}
 
-	val  = MXR_GRP_WH_WIDTH(plane->src_w);
-	val |= MXR_GRP_WH_HEIGHT(plane->src_h);
+	val  = MXR_GRP_WH_WIDTH(state->src.w);
+	val |= MXR_GRP_WH_HEIGHT(state->src.h);
 	val |= MXR_GRP_WH_H_SCALE(x_ratio);
 	val |= MXR_GRP_WH_V_SCALE(y_ratio);
 	mixer_reg_write(res, MXR_GRAPHIC_WH(win), val);
@@ -626,7 +678,8 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 
 	mixer_cfg_scan(ctx, mode->vdisplay);
 	mixer_cfg_rgb_fmt(ctx, mode->vdisplay);
-	mixer_cfg_layer(ctx, win, true);
+	mixer_cfg_layer(ctx, win, state->zpos + 1, true);
+	mixer_cfg_gfx_blend(ctx, win, is_alpha_format(fb->pixel_format));
 
 	/* layer update mandatory for mixer 16.0.33.0 */
 	if (ctx->mxr_ver == MXR_VER_16_0_33_0 ||
@@ -635,7 +688,6 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 
 	mixer_run(ctx);
 
-	mixer_vsync_set_update(ctx, true);
 	spin_unlock_irqrestore(&res->reg_slock, flags);
 
 	mixer_regs_dump(ctx);
@@ -660,10 +712,8 @@ static void mixer_win_reset(struct mixer_context *ctx)
 {
 	struct mixer_resources *res = &ctx->mixer_res;
 	unsigned long flags;
-	u32 val; /* value stored to register */
 
 	spin_lock_irqsave(&res->reg_slock, flags);
-	mixer_vsync_set_update(ctx, false);
 
 	mixer_reg_writemask(res, MXR_CFG, MXR_CFG_DST_HDMI, MXR_CFG_DST_MASK);
 
@@ -674,40 +724,14 @@ static void mixer_win_reset(struct mixer_context *ctx)
 	mixer_reg_writemask(res, MXR_STATUS, MXR_STATUS_16_BURST,
 		MXR_STATUS_BURST_MASK);
 
-	/* setting default layer priority: layer1 > layer0 > video
-	 * because typical usage scenario would be
-	 * layer1 - OSD
-	 * layer0 - framebuffer
-	 * video - video overlay
-	 */
-	val = MXR_LAYER_CFG_GRP1_VAL(3);
-	val |= MXR_LAYER_CFG_GRP0_VAL(2);
-	if (ctx->vp_enabled)
-		val |= MXR_LAYER_CFG_VP_VAL(1);
-	mixer_reg_write(res, MXR_LAYER_CFG, val);
+	/* reset default layer priority */
+	mixer_reg_write(res, MXR_LAYER_CFG, 0);
 
 	/* setting background color */
 	mixer_reg_write(res, MXR_BG_COLOR0, 0x008080);
 	mixer_reg_write(res, MXR_BG_COLOR1, 0x008080);
 	mixer_reg_write(res, MXR_BG_COLOR2, 0x008080);
 
-	/* setting graphical layers */
-	val  = MXR_GRP_CFG_COLOR_KEY_DISABLE; /* no blank key */
-	val |= MXR_GRP_CFG_WIN_BLEND_EN;
-	val |= MXR_GRP_CFG_ALPHA_VAL(0xff); /* non-transparent alpha */
-
-	/* Don't blend layer 0 onto the mixer background */
-	mixer_reg_write(res, MXR_GRAPHIC_CFG(0), val);
-
-	/* Blend layer 1 into layer 0 */
-	val |= MXR_GRP_CFG_BLEND_PRE_MUL;
-	val |= MXR_GRP_CFG_PIXEL_BLEND_EN;
-	mixer_reg_write(res, MXR_GRAPHIC_CFG(1), val);
-
-	/* setting video layers */
-	val = MXR_GRP_CFG_ALPHA_VAL(0);
-	mixer_reg_write(res, MXR_VIDEO_CFG, val);
-
 	if (ctx->vp_enabled) {
 		/* configuration of Video Processor Registers */
 		vp_win_reset(ctx);
@@ -720,7 +744,6 @@ static void mixer_win_reset(struct mixer_context *ctx)
 	if (ctx->vp_enabled)
 		mixer_reg_writemask(res, MXR_CFG, 0, MXR_CFG_VP_ENABLE);
 
-	mixer_vsync_set_update(ctx, true);
 	spin_unlock_irqrestore(&res->reg_slock, flags);
 }
 
@@ -951,17 +974,27 @@ static void mixer_disable_vblank(struct exynos_drm_crtc *crtc)
 	mixer_reg_writemask(res, MXR_INT_EN, 0, MXR_INT_EN_VSYNC);
 }
 
+static void mixer_atomic_begin(struct exynos_drm_crtc *crtc)
+{
+	struct mixer_context *mixer_ctx = crtc->ctx;
+
+	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
+		return;
+
+	mixer_vsync_set_update(mixer_ctx, false);
+}
+
 static void mixer_update_plane(struct exynos_drm_crtc *crtc,
 			       struct exynos_drm_plane *plane)
 {
 	struct mixer_context *mixer_ctx = crtc->ctx;
 
-	DRM_DEBUG_KMS("win: %d\n", plane->zpos);
+	DRM_DEBUG_KMS("win: %d\n", plane->index);
 
 	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
 		return;
 
-	if (plane->zpos > 1 && mixer_ctx->vp_enabled)
+	if (plane->index == VP_DEFAULT_WIN)
 		vp_video_buffer(mixer_ctx, plane);
 	else
 		mixer_graph_buffer(mixer_ctx, plane);
@@ -974,18 +1007,24 @@ static void mixer_disable_plane(struct exynos_drm_crtc *crtc,
 	struct mixer_resources *res = &mixer_ctx->mixer_res;
 	unsigned long flags;
 
-	DRM_DEBUG_KMS("win: %d\n", plane->zpos);
+	DRM_DEBUG_KMS("win: %d\n", plane->index);
 
 	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
 		return;
 
 	spin_lock_irqsave(&res->reg_slock, flags);
-	mixer_vsync_set_update(mixer_ctx, false);
+	mixer_cfg_layer(mixer_ctx, plane->index, 0, false);
+	spin_unlock_irqrestore(&res->reg_slock, flags);
+}
 
-	mixer_cfg_layer(mixer_ctx, plane->zpos, false);
+static void mixer_atomic_flush(struct exynos_drm_crtc *crtc)
+{
+	struct mixer_context *mixer_ctx = crtc->ctx;
+
+	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
+		return;
 
 	mixer_vsync_set_update(mixer_ctx, true);
-	spin_unlock_irqrestore(&res->reg_slock, flags);
 }
 
 static void mixer_wait_for_vblank(struct exynos_drm_crtc *crtc)
@@ -1020,42 +1059,13 @@ static void mixer_enable(struct exynos_drm_crtc *crtc)
 {
 	struct mixer_context *ctx = crtc->ctx;
 	struct mixer_resources *res = &ctx->mixer_res;
-	int ret;
 
 	if (test_bit(MXR_BIT_POWERED, &ctx->flags))
 		return;
 
 	pm_runtime_get_sync(ctx->dev);
 
-	ret = clk_prepare_enable(res->mixer);
-	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the mixer clk [%d]\n", ret);
-		return;
-	}
-	ret = clk_prepare_enable(res->hdmi);
-	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the hdmi clk [%d]\n", ret);
-		return;
-	}
-	if (ctx->vp_enabled) {
-		ret = clk_prepare_enable(res->vp);
-		if (ret < 0) {
-			DRM_ERROR("Failed to prepare_enable the vp clk [%d]\n",
-				  ret);
-			return;
-		}
-		if (ctx->has_sclk) {
-			ret = clk_prepare_enable(res->sclk_mixer);
-			if (ret < 0) {
-				DRM_ERROR("Failed to prepare_enable the " \
-					   "sclk_mixer clk [%d]\n",
-					  ret);
-				return;
-			}
-		}
-	}
-
-	set_bit(MXR_BIT_POWERED, &ctx->flags);
+	mixer_vsync_set_update(ctx, false);
 
 	mixer_reg_writemask(res, MXR_STATUS, ~0, MXR_STATUS_SOFT_RESET);
 
@@ -1064,12 +1074,15 @@ static void mixer_enable(struct exynos_drm_crtc *crtc)
 		mixer_reg_writemask(res, MXR_INT_EN, ~0, MXR_INT_EN_VSYNC);
 	}
 	mixer_win_reset(ctx);
+
+	mixer_vsync_set_update(ctx, true);
+
+	set_bit(MXR_BIT_POWERED, &ctx->flags);
 }
 
 static void mixer_disable(struct exynos_drm_crtc *crtc)
 {
 	struct mixer_context *ctx = crtc->ctx;
-	struct mixer_resources *res = &ctx->mixer_res;
 	int i;
 
 	if (!test_bit(MXR_BIT_POWERED, &ctx->flags))
@@ -1081,17 +1094,9 @@ static void mixer_disable(struct exynos_drm_crtc *crtc)
 	for (i = 0; i < MIXER_WIN_NR; i++)
 		mixer_disable_plane(crtc, &ctx->planes[i]);
 
-	clear_bit(MXR_BIT_POWERED, &ctx->flags);
+	pm_runtime_put(ctx->dev);
 
-	clk_disable_unprepare(res->hdmi);
-	clk_disable_unprepare(res->mixer);
-	if (ctx->vp_enabled) {
-		clk_disable_unprepare(res->vp);
-		if (ctx->has_sclk)
-			clk_disable_unprepare(res->sclk_mixer);
-	}
-
-	pm_runtime_put_sync(ctx->dev);
+	clear_bit(MXR_BIT_POWERED, &ctx->flags);
 }
 
 /* Only valid for Mixer version 16.0.33.0 */
@@ -1122,8 +1127,10 @@ static const struct exynos_drm_crtc_ops mixer_crtc_ops = {
 	.enable_vblank		= mixer_enable_vblank,
 	.disable_vblank		= mixer_disable_vblank,
 	.wait_for_vblank	= mixer_wait_for_vblank,
+	.atomic_begin		= mixer_atomic_begin,
 	.update_plane		= mixer_update_plane,
 	.disable_plane		= mixer_disable_plane,
+	.atomic_flush		= mixer_atomic_flush,
 	.atomic_check		= mixer_atomic_check,
 };
 
@@ -1187,30 +1194,19 @@ static int mixer_bind(struct device *dev, struct device *manager, void *data)
 	struct mixer_context *ctx = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = data;
 	struct exynos_drm_plane *exynos_plane;
-	unsigned int zpos;
+	unsigned int i;
 	int ret;
 
 	ret = mixer_initialize(ctx, drm_dev);
 	if (ret)
 		return ret;
 
-	for (zpos = 0; zpos < MIXER_WIN_NR; zpos++) {
-		enum drm_plane_type type;
-		const uint32_t *formats;
-		unsigned int fcount;
-
-		if (zpos < VP_DEFAULT_WIN) {
-			formats = mixer_formats;
-			fcount = ARRAY_SIZE(mixer_formats);
-		} else {
-			formats = vp_formats;
-			fcount = ARRAY_SIZE(vp_formats);
-		}
+	for (i = 0; i < MIXER_WIN_NR; i++) {
+		if (i == VP_DEFAULT_WIN && !ctx->vp_enabled)
+			continue;
 
-		type = exynos_plane_get_type(zpos, CURSOR_WIN);
-		ret = exynos_plane_init(drm_dev, &ctx->planes[zpos],
-					1 << ctx->pipe, type, formats, fcount,
-					zpos);
+		ret = exynos_plane_init(drm_dev, &ctx->planes[i], i,
+					1 << ctx->pipe, &plane_configs[i]);
 		if (ret)
 			return ret;
 	}
@@ -1293,10 +1289,70 @@ static int mixer_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int exynos_mixer_suspend(struct device *dev)
+{
+	struct mixer_context *ctx = dev_get_drvdata(dev);
+	struct mixer_resources *res = &ctx->mixer_res;
+
+	clk_disable_unprepare(res->hdmi);
+	clk_disable_unprepare(res->mixer);
+	if (ctx->vp_enabled) {
+		clk_disable_unprepare(res->vp);
+		if (ctx->has_sclk)
+			clk_disable_unprepare(res->sclk_mixer);
+	}
+
+	return 0;
+}
+
+static int exynos_mixer_resume(struct device *dev)
+{
+	struct mixer_context *ctx = dev_get_drvdata(dev);
+	struct mixer_resources *res = &ctx->mixer_res;
+	int ret;
+
+	ret = clk_prepare_enable(res->mixer);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the mixer clk [%d]\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(res->hdmi);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the hdmi clk [%d]\n", ret);
+		return ret;
+	}
+	if (ctx->vp_enabled) {
+		ret = clk_prepare_enable(res->vp);
+		if (ret < 0) {
+			DRM_ERROR("Failed to prepare_enable the vp clk [%d]\n",
+				  ret);
+			return ret;
+		}
+		if (ctx->has_sclk) {
+			ret = clk_prepare_enable(res->sclk_mixer);
+			if (ret < 0) {
+				DRM_ERROR("Failed to prepare_enable the " \
+					   "sclk_mixer clk [%d]\n",
+					  ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos_mixer_pm_ops = {
+	SET_RUNTIME_PM_OPS(exynos_mixer_suspend, exynos_mixer_resume, NULL)
+};
+
 struct platform_driver mixer_driver = {
 	.driver = {
 		.name = "exynos-mixer",
 		.owner = THIS_MODULE,
+		.pm = &exynos_mixer_pm_ops,
 		.of_match_table = mixer_match_types,
 	},
 	.probe = mixer_probe,
diff --git a/drivers/gpu/drm/exynos/regs-gsc.h b/drivers/gpu/drm/exynos/regs-gsc.h
index 9ad592707aaf..4704a993cbb7 100644
--- a/drivers/gpu/drm/exynos/regs-gsc.h
+++ b/drivers/gpu/drm/exynos/regs-gsc.h
@@ -273,12 +273,12 @@
 #define GSC_CLK_GATE_MODE_SNOOP_CNT(x)	((x) << 0)
 
 /* SYSCON. GSCBLK_CFG */
-#define SYSREG_GSCBLK_CFG1		(S3C_VA_SYS + 0x0224)
+#define SYSREG_GSCBLK_CFG1		0x0224
 #define GSC_BLK_DISP1WB_DEST(x)		(x << 10)
 #define GSC_BLK_SW_RESET_WB_DEST(x)	(1 << (18 + x))
 #define GSC_BLK_PXLASYNC_LO_MASK_WB(x)	(0 << (14 + x))
 #define GSC_BLK_GSCL_WB_IN_SRC_SEL(x)	(1 << (2 * x))
-#define SYSREG_GSCBLK_CFG2		(S3C_VA_SYS + 0x2000)
+#define SYSREG_GSCBLK_CFG2		0x2000
 #define PXLASYNC_LO_MASK_CAMIF_GSCL(x)	(1 << (x))
 
 #endif /* EXYNOS_REGS_GSC_H_ */
diff --git a/drivers/gpu/drm/exynos/regs-mixer.h b/drivers/gpu/drm/exynos/regs-mixer.h
index ac60260c2389..7f22df5bf707 100644
--- a/drivers/gpu/drm/exynos/regs-mixer.h
+++ b/drivers/gpu/drm/exynos/regs-mixer.h
@@ -113,6 +113,7 @@
 #define MXR_GRP_CFG_BLEND_PRE_MUL	(1 << 20)
 #define MXR_GRP_CFG_WIN_BLEND_EN	(1 << 17)
 #define MXR_GRP_CFG_PIXEL_BLEND_EN	(1 << 16)
+#define MXR_GRP_CFG_MISC_MASK		((3 << 16) | (3 << 20))
 #define MXR_GRP_CFG_FORMAT_VAL(x)	MXR_MASK_VAL(x, 11, 8)
 #define MXR_GRP_CFG_FORMAT_MASK		MXR_GRP_CFG_FORMAT_VAL(~0)
 #define MXR_GRP_CFG_ALPHA_VAL(x)	MXR_MASK_VAL(x, 7, 0)
@@ -145,8 +146,11 @@
 
 /* bit for MXR_LAYER_CFG */
 #define MXR_LAYER_CFG_GRP1_VAL(x)	MXR_MASK_VAL(x, 11, 8)
+#define MXR_LAYER_CFG_GRP1_MASK		MXR_LAYER_CFG_GRP1_VAL(~0)
 #define MXR_LAYER_CFG_GRP0_VAL(x)	MXR_MASK_VAL(x, 7, 4)
+#define MXR_LAYER_CFG_GRP0_MASK		MXR_LAYER_CFG_GRP0_VAL(~0)
 #define MXR_LAYER_CFG_VP_VAL(x)		MXR_MASK_VAL(x, 3, 0)
+#define MXR_LAYER_CFG_VP_MASK		MXR_LAYER_CFG_VP_VAL(~0)
 
 #endif /* SAMSUNG_REGS_MIXER_H */
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index 82a3d311e164..d8ab8f0af10c 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -175,7 +175,7 @@ int fsl_dcu_drm_crtc_create(struct fsl_dcu_drm_device *fsl_dev)
 
 	primary = fsl_dcu_drm_primary_create_plane(fsl_dev->drm);
 	ret = drm_crtc_init_with_planes(fsl_dev->drm, crtc, primary, NULL,
-					&fsl_dcu_drm_crtc_funcs);
+					&fsl_dcu_drm_crtc_funcs, NULL);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 1930234ba5f1..fca97d3fc846 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -363,7 +363,6 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	fsl_dev->np = dev->of_node;
 	drm->dev_private = fsl_dev;
 	dev_set_drvdata(dev, fsl_dev);
-	drm_dev_set_unique(drm, dev_name(dev));
 
 	ret = drm_dev_register(drm, 0);
 	if (ret < 0)
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index 51daaea40b4d..4b13cf919575 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -249,7 +249,7 @@ struct drm_plane *fsl_dcu_drm_primary_create_plane(struct drm_device *dev)
 				       &fsl_dcu_drm_plane_funcs,
 				       fsl_dcu_drm_plane_formats,
 				       ARRAY_SIZE(fsl_dcu_drm_plane_formats),
-				       DRM_PLANE_TYPE_PRIMARY);
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
 		primary = NULL;
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index fe8ab5da04fb..8780deba5e8a 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -57,7 +57,7 @@ int fsl_dcu_drm_encoder_create(struct fsl_dcu_drm_device *fsl_dev,
 
 	encoder->possible_crtcs = 1;
 	ret = drm_encoder_init(fsl_dev->drm, encoder, &encoder_funcs,
-			       DRM_MODE_ENCODER_LVDS);
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index 3531f90e53d0..8745971a7680 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -619,6 +619,8 @@ const struct psb_ops cdv_chip_ops = {
 	.init_pm = cdv_init_pm,
 	.save_regs = cdv_save_display_registers,
 	.restore_regs = cdv_restore_display_registers,
+	.save_crtc = gma_crtc_save,
+	.restore_crtc = gma_crtc_restore,
 	.power_down = cdv_power_down,
 	.power_up = cdv_power_up,
 	.update_wm = cdv_update_wm,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c
index 248c33a35ebf..d0717a85c7ec 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_crt.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c
@@ -273,7 +273,7 @@ void cdv_intel_crt_init(struct drm_device *dev,
 
 	encoder = &gma_encoder->base;
 	drm_encoder_init(dev, encoder,
-		&cdv_intel_crt_enc_funcs, DRM_MODE_ENCODER_DAC);
+		&cdv_intel_crt_enc_funcs, DRM_MODE_ENCODER_DAC, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_display.c b/drivers/gpu/drm/gma500/cdv_intel_display.c
index 7d47b3d5cc0d..6126546295e9 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_display.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_display.c
@@ -983,8 +983,6 @@ const struct drm_crtc_helper_funcs cdv_intel_helper_funcs = {
 };
 
 const struct drm_crtc_funcs cdv_intel_crtc_funcs = {
-	.save = gma_crtc_save,
-	.restore = gma_crtc_restore,
 	.cursor_set = gma_crtc_cursor_set,
 	.cursor_move = gma_crtc_cursor_move,
 	.gamma_set = gma_crtc_gamma_set,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index 17cea400ae32..7bb1f1aff932 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -2020,7 +2020,8 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 	encoder = &gma_encoder->base;
 
 	drm_connector_init(dev, connector, &cdv_intel_dp_connector_funcs, type);
-	drm_encoder_init(dev, encoder, &cdv_intel_dp_enc_funcs, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(dev, encoder, &cdv_intel_dp_enc_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index 6b1d3340ba14..ddf2d7700759 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -270,8 +270,6 @@ static const struct drm_connector_helper_funcs
 
 static const struct drm_connector_funcs cdv_hdmi_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = cdv_hdmi_save,
-	.restore = cdv_hdmi_restore,
 	.detect = cdv_hdmi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = cdv_hdmi_set_property,
@@ -306,13 +304,16 @@ void cdv_hdmi_init(struct drm_device *dev,
 
 	connector = &gma_connector->base;
 	connector->polled = DRM_CONNECTOR_POLL_HPD;
+	gma_connector->save = cdv_hdmi_save;
+	gma_connector->restore = cdv_hdmi_restore;
+
 	encoder = &gma_encoder->base;
 	drm_connector_init(dev, connector,
 			   &cdv_hdmi_connector_funcs,
 			   DRM_MODE_CONNECTOR_DVID);
 
 	drm_encoder_init(dev, encoder, &psb_intel_lvds_enc_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 	gma_encoder->type = INTEL_OUTPUT_HDMI;
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index 211069b2b951..813ef23a8054 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -530,8 +530,6 @@ static const struct drm_connector_helper_funcs
 
 static const struct drm_connector_funcs cdv_intel_lvds_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = cdv_intel_lvds_save,
-	.restore = cdv_intel_lvds_restore,
 	.detect = cdv_intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = cdv_intel_lvds_set_property,
@@ -643,6 +641,8 @@ void cdv_intel_lvds_init(struct drm_device *dev,
 	gma_encoder->dev_priv = lvds_priv;
 
 	connector = &gma_connector->base;
+	gma_connector->save = cdv_intel_lvds_save;
+	gma_connector->restore = cdv_intel_lvds_restore;
 	encoder = &gma_encoder->base;
 
 
@@ -652,7 +652,7 @@ void cdv_intel_lvds_init(struct drm_device *dev,
 
 	drm_encoder_init(dev, encoder,
 			 &cdv_intel_lvds_enc_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 2eaf1b31c7bd..cb95765050cc 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -21,6 +21,7 @@
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/string.h>
+#include <linux/pfn_t.h>
 #include <linux/mm.h>
 #include <linux/tty.h>
 #include <linux/slab.h>
@@ -132,7 +133,8 @@ static int psbfb_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	for (i = 0; i < page_num; i++) {
 		pfn = (phys_addr >> PAGE_SHIFT);
 
-		ret = vm_insert_mixed(vma, address, pfn);
+		ret = vm_insert_mixed(vma, address,
+				__pfn_to_pfn_t(pfn, PFN_DEV));
 		if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
 			break;
 		else if (unlikely(ret != 0)) {
@@ -241,7 +243,7 @@ static struct fb_ops psbfb_unaccel_ops = {
  */
 static int psb_framebuffer_init(struct drm_device *dev,
 					struct psb_framebuffer *fb,
-					struct drm_mode_fb_cmd2 *mode_cmd,
+					const struct drm_mode_fb_cmd2 *mode_cmd,
 					struct gtt_range *gt)
 {
 	u32 bpp, depth;
@@ -284,7 +286,7 @@ static int psb_framebuffer_init(struct drm_device *dev,
 
 static struct drm_framebuffer *psb_framebuffer_create
 			(struct drm_device *dev,
-			 struct drm_mode_fb_cmd2 *mode_cmd,
+			 const struct drm_mode_fb_cmd2 *mode_cmd,
 			 struct gtt_range *gt)
 {
 	struct psb_framebuffer *fb;
@@ -406,8 +408,6 @@ static int psbfb_create(struct psb_fbdev *fbdev,
 
 	memset(dev_priv->vram_addr + backing->offset, 0, size);
 
-	mutex_lock(&dev->struct_mutex);
-
 	info = drm_fb_helper_alloc_fbi(&fbdev->psb_fb_helper);
 	if (IS_ERR(info)) {
 		ret = PTR_ERR(info);
@@ -463,17 +463,15 @@ static int psbfb_create(struct psb_fbdev *fbdev,
 	dev_dbg(dev->dev, "allocated %dx%d fb\n",
 					psbfb->base.width, psbfb->base.height);
 
-	mutex_unlock(&dev->struct_mutex);
 	return 0;
 out_unref:
 	if (backing->stolen)
 		psb_gtt_free_range(dev, backing);
 	else
-		drm_gem_object_unreference(&backing->gem);
+		drm_gem_object_unreference_unlocked(&backing->gem);
 
 	drm_fb_helper_release_fbi(&fbdev->psb_fb_helper);
 out_err1:
-	mutex_unlock(&dev->struct_mutex);
 	psb_gtt_free_range(dev, backing);
 	return ret;
 }
@@ -488,7 +486,7 @@ out_err1:
  */
 static struct drm_framebuffer *psb_user_framebuffer_create
 			(struct drm_device *dev, struct drm_file *filp,
-			 struct drm_mode_fb_cmd2 *cmd)
+			 const struct drm_mode_fb_cmd2 *cmd)
 {
 	struct gtt_range *r;
 	struct drm_gem_object *obj;
@@ -569,7 +567,7 @@ static int psb_fbdev_destroy(struct drm_device *dev, struct psb_fbdev *fbdev)
 	drm_framebuffer_cleanup(&psbfb->base);
 
 	if (psbfb->gtt)
-		drm_gem_object_unreference(&psbfb->gtt->gem);
+		drm_gem_object_unreference_unlocked(&psbfb->gtt->gem);
 	return 0;
 }
 
@@ -784,12 +782,8 @@ void psb_modeset_cleanup(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	if (dev_priv->modeset) {
-		mutex_lock(&dev->struct_mutex);
-
 		drm_kms_helper_poll_fini(dev);
 		psb_fbdev_fini(dev);
 		drm_mode_config_cleanup(dev);
-
-		mutex_unlock(&dev->struct_mutex);
 	}
 }
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index c707fa6fca85..506224b3a0ad 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -62,15 +62,10 @@ int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
 	int ret = 0;
 	struct drm_gem_object *obj;
 
-	mutex_lock(&dev->struct_mutex);
-
 	/* GEM does all our handle to object mapping */
 	obj = drm_gem_object_lookup(dev, file, handle);
-	if (obj == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
-	/* What validation is needed here ? */
+	if (obj == NULL)
+		return -ENOENT;
 
 	/* Make it mmapable */
 	ret = drm_gem_create_mmap_offset(obj);
@@ -78,9 +73,7 @@ int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
 		goto out;
 	*offset = drm_vma_node_offset_addr(&obj->vma_node);
 out:
-	drm_gem_object_unreference(obj);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
+	drm_gem_object_unreference_unlocked(obj);
 	return ret;
 }
 
@@ -130,7 +123,7 @@ int psb_gem_create(struct drm_file *file, struct drm_device *dev, u64 size,
 		return ret;
 	}
 	/* We have the initial and handle reference but need only one now */
-	drm_gem_object_unreference(&r->gem);
+	drm_gem_object_unreference_unlocked(&r->gem);
 	*handlep = handle;
 	return 0;
 }
@@ -189,7 +182,7 @@ int psb_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
 	/* Make sure we don't parallel update on a fault, nor move or remove
 	   something from beneath our feet */
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&dev_priv->mmap_mutex);
 
 	/* For now the mmap pins the object and it stays pinned. As things
 	   stand that will do us no harm */
@@ -215,7 +208,7 @@ int psb_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 
 fail:
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&dev_priv->mmap_mutex);
 	switch (ret) {
 	case 0:
 	case -ERESTARTSYS:
diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c
index 001b450b27b3..ff17af4cfc64 100644
--- a/drivers/gpu/drm/gma500/gma_display.c
+++ b/drivers/gpu/drm/gma500/gma_display.c
@@ -349,8 +349,6 @@ int gma_crtc_cursor_set(struct drm_crtc *crtc,
 	/* If we didn't get a handle then turn the cursor off */
 	if (!handle) {
 		temp = CURSOR_MODE_DISABLE;
-		mutex_lock(&dev->struct_mutex);
-
 		if (gma_power_begin(dev, false)) {
 			REG_WRITE(control, temp);
 			REG_WRITE(base, 0);
@@ -362,11 +360,9 @@ int gma_crtc_cursor_set(struct drm_crtc *crtc,
 			gt = container_of(gma_crtc->cursor_obj,
 					  struct gtt_range, gem);
 			psb_gtt_unpin(gt);
-			drm_gem_object_unreference(gma_crtc->cursor_obj);
+			drm_gem_object_unreference_unlocked(gma_crtc->cursor_obj);
 			gma_crtc->cursor_obj = NULL;
 		}
-
-		mutex_unlock(&dev->struct_mutex);
 		return 0;
 	}
 
@@ -376,7 +372,6 @@ int gma_crtc_cursor_set(struct drm_crtc *crtc,
 		return -EINVAL;
 	}
 
-	mutex_lock(&dev->struct_mutex);
 	obj = drm_gem_object_lookup(dev, file_priv, handle);
 	if (!obj) {
 		ret = -ENOENT;
@@ -441,17 +436,15 @@ int gma_crtc_cursor_set(struct drm_crtc *crtc,
 	if (gma_crtc->cursor_obj) {
 		gt = container_of(gma_crtc->cursor_obj, struct gtt_range, gem);
 		psb_gtt_unpin(gt);
-		drm_gem_object_unreference(gma_crtc->cursor_obj);
+		drm_gem_object_unreference_unlocked(gma_crtc->cursor_obj);
 	}
 
 	gma_crtc->cursor_obj = obj;
 unlock:
-	mutex_unlock(&dev->struct_mutex);
 	return ret;
 
 unref_cursor:
-	drm_gem_object_unreference(obj);
-	mutex_unlock(&dev->struct_mutex);
+	drm_gem_object_unreference_unlocked(obj);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c
index ce015db59dc6..8f69225ce2b4 100644
--- a/drivers/gpu/drm/gma500/gtt.c
+++ b/drivers/gpu/drm/gma500/gtt.c
@@ -425,6 +425,7 @@ int psb_gtt_init(struct drm_device *dev, int resume)
 
 	if (!resume) {
 		mutex_init(&dev_priv->gtt_mutex);
+		mutex_init(&dev_priv->mmap_mutex);
 		psb_gtt_alloc(dev);
 	}
 
diff --git a/drivers/gpu/drm/gma500/mdfld_device.c b/drivers/gpu/drm/gma500/mdfld_device.c
index 265ad0de44a6..e2ab858122f9 100644
--- a/drivers/gpu/drm/gma500/mdfld_device.c
+++ b/drivers/gpu/drm/gma500/mdfld_device.c
@@ -546,6 +546,8 @@ const struct psb_ops mdfld_chip_ops = {
 
 	.save_regs = mdfld_save_registers,
 	.restore_regs = mdfld_restore_registers,
+	.save_crtc = gma_crtc_save,
+	.restore_crtc = gma_crtc_restore,
 	.power_down = mdfld_power_down,
 	.power_up = mdfld_power_up,
 };
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c
index d4813e03f5ee..7cd87a0c2385 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c
@@ -821,14 +821,18 @@ void mdfld_dsi_dpi_mode_set(struct drm_encoder *encoder,
 	struct drm_device *dev = dsi_config->dev;
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	int pipe = mdfld_dsi_encoder_get_pipe(dsi_encoder);
-
 	u32 pipeconf_reg = PIPEACONF;
 	u32 dspcntr_reg = DSPACNTR;
+	u32 pipeconf, dspcntr;
 
-	u32 pipeconf = dev_priv->pipeconf[pipe];
-	u32 dspcntr = dev_priv->dspcntr[pipe];
 	u32 mipi = MIPI_PORT_EN | PASS_FROM_SPHY_TO_AFE | SEL_FLOPPED_HSTX;
 
+	if (WARN_ON(pipe < 0))
+		return;
+
+	pipeconf = dev_priv->pipeconf[pipe];
+	dspcntr = dev_priv->dspcntr[pipe];
+
 	if (pipe) {
 		pipeconf_reg = PIPECCONF;
 		dspcntr_reg = DSPCCNTR;
@@ -994,7 +998,7 @@ struct mdfld_dsi_encoder *mdfld_dsi_dpi_init(struct drm_device *dev,
 	drm_encoder_init(dev,
 			encoder,
 			p_funcs->encoder_funcs,
-			DRM_MODE_ENCODER_LVDS);
+			DRM_MODE_ENCODER_LVDS, NULL);
 	drm_encoder_helper_add(encoder,
 				p_funcs->encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
index 89f705c3a5eb..d758f4cc6805 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
@@ -405,8 +405,6 @@ static struct drm_encoder *mdfld_dsi_connector_best_encoder(
 /*DSI connector funcs*/
 static const struct drm_connector_funcs mdfld_dsi_connector_funcs = {
 	.dpms = /*drm_helper_connector_dpms*/mdfld_dsi_connector_dpms,
-	.save = mdfld_dsi_connector_save,
-	.restore = mdfld_dsi_connector_restore,
 	.detect = mdfld_dsi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = mdfld_dsi_connector_set_property,
@@ -563,6 +561,9 @@ void mdfld_dsi_output_init(struct drm_device *dev,
 
 
 	connector = &dsi_connector->base.base;
+	dsi_connector->base.save = mdfld_dsi_connector_save;
+	dsi_connector->base.restore = mdfld_dsi_connector_restore;
+
 	drm_connector_init(dev, connector, &mdfld_dsi_connector_funcs,
 						DRM_MODE_CONNECTOR_LVDS);
 	drm_connector_helper_add(connector, &mdfld_dsi_connector_helper_funcs);
diff --git a/drivers/gpu/drm/gma500/oaktrail_device.c b/drivers/gpu/drm/gma500/oaktrail_device.c
index 368a03ae3010..ba30b43a3412 100644
--- a/drivers/gpu/drm/gma500/oaktrail_device.c
+++ b/drivers/gpu/drm/gma500/oaktrail_device.c
@@ -568,6 +568,8 @@ const struct psb_ops oaktrail_chip_ops = {
 
 	.save_regs = oaktrail_save_display_registers,
 	.restore_regs = oaktrail_restore_display_registers,
+	.save_crtc = gma_crtc_save,
+	.restore_crtc = gma_crtc_restore,
 	.power_down = oaktrail_power_down,
 	.power_up = oaktrail_power_up,
 
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 2310d879cdc2..2d18499d6060 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -654,7 +654,7 @@ void oaktrail_hdmi_init(struct drm_device *dev,
 
 	drm_encoder_init(dev, encoder,
 			 &oaktrail_hdmi_enc_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index 83bbc271bcfb..f7038f12ac76 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -323,7 +323,7 @@ void oaktrail_lvds_init(struct drm_device *dev,
 			   DRM_MODE_CONNECTOR_LVDS);
 
 	drm_encoder_init(dev, encoder, &psb_intel_lvds_enc_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 	gma_encoder->type = INTEL_OUTPUT_LVDS;
diff --git a/drivers/gpu/drm/gma500/power.c b/drivers/gpu/drm/gma500/power.c
index b6b135fcd59c..bea8578846d1 100644
--- a/drivers/gpu/drm/gma500/power.c
+++ b/drivers/gpu/drm/gma500/power.c
@@ -187,7 +187,7 @@ static bool gma_resume_pci(struct pci_dev *pdev)
  */
 int gma_power_suspend(struct device *_dev)
 {
-	struct pci_dev *pdev = container_of(_dev, struct pci_dev, dev);
+	struct pci_dev *pdev = to_pci_dev(_dev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct drm_psb_private *dev_priv = dev->dev_private;
 
@@ -214,7 +214,7 @@ int gma_power_suspend(struct device *_dev)
  */
 int gma_power_resume(struct device *_dev)
 {
-	struct pci_dev *pdev = container_of(_dev, struct pci_dev, dev);
+	struct pci_dev *pdev = to_pci_dev(_dev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
 
 	mutex_lock(&power_mutex);
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index 07df7d4eea72..dc0f8527570c 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -181,7 +181,7 @@ static int psb_save_display_registers(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	struct drm_crtc *crtc;
-	struct drm_connector *connector;
+	struct gma_connector *connector;
 	struct psb_state *regs = &dev_priv->regs.psb;
 
 	/* Display arbitration control + watermarks */
@@ -198,12 +198,12 @@ static int psb_save_display_registers(struct drm_device *dev)
 	drm_modeset_lock_all(dev);
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		if (drm_helper_crtc_in_use(crtc))
-			crtc->funcs->save(crtc);
+			dev_priv->ops->save_crtc(crtc);
 	}
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head)
-		if (connector->funcs->save)
-			connector->funcs->save(connector);
+	list_for_each_entry(connector, &dev->mode_config.connector_list, base.head)
+		if (connector->save)
+			connector->save(&connector->base);
 
 	drm_modeset_unlock_all(dev);
 	return 0;
@@ -219,7 +219,7 @@ static int psb_restore_display_registers(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	struct drm_crtc *crtc;
-	struct drm_connector *connector;
+	struct gma_connector *connector;
 	struct psb_state *regs = &dev_priv->regs.psb;
 
 	/* Display arbitration + watermarks */
@@ -238,11 +238,11 @@ static int psb_restore_display_registers(struct drm_device *dev)
 	drm_modeset_lock_all(dev);
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
 		if (drm_helper_crtc_in_use(crtc))
-			crtc->funcs->restore(crtc);
+			dev_priv->ops->restore_crtc(crtc);
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head)
-		if (connector->funcs->restore)
-			connector->funcs->restore(connector);
+	list_for_each_entry(connector, &dev->mode_config.connector_list, base.head)
+		if (connector->restore)
+			connector->restore(&connector->base);
 
 	drm_modeset_unlock_all(dev);
 	return 0;
@@ -354,6 +354,8 @@ const struct psb_ops psb_chip_ops = {
 	.init_pm = psb_init_pm,
 	.save_regs = psb_save_display_registers,
 	.restore_regs = psb_restore_display_registers,
+	.save_crtc = gma_crtc_save,
+	.restore_crtc = gma_crtc_restore,
 	.power_down = psb_power_down,
 	.power_up = psb_power_up,
 };
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index e21726ecac32..b74372760d7f 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -465,6 +465,8 @@ struct drm_psb_private {
 	struct mutex gtt_mutex;
 	struct resource *gtt_mem;	/* Our PCI resource */
 
+	struct mutex mmap_mutex;
+
 	struct psb_mmu_driver *mmu;
 	struct psb_mmu_pd *pf_pd;
 
@@ -651,6 +653,8 @@ struct psb_ops {
 	void (*init_pm)(struct drm_device *dev);
 	int (*save_regs)(struct drm_device *dev);
 	int (*restore_regs)(struct drm_device *dev);
+	void (*save_crtc)(struct drm_crtc *crtc);
+	void (*restore_crtc)(struct drm_crtc *crtc);
 	int (*power_up)(struct drm_device *dev);
 	int (*power_down)(struct drm_device *dev);
 	void (*update_wm)(struct drm_device *dev, struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 6659da88fe5b..dcdbc37e55e1 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -439,8 +439,6 @@ const struct drm_crtc_helper_funcs psb_intel_helper_funcs = {
 };
 
 const struct drm_crtc_funcs psb_intel_crtc_funcs = {
-	.save = gma_crtc_save,
-	.restore = gma_crtc_restore,
 	.cursor_set = gma_crtc_cursor_set,
 	.cursor_move = gma_crtc_cursor_move,
 	.gamma_set = gma_crtc_gamma_set,
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index 860dd2177ca1..2a3b7c684db2 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -140,6 +140,9 @@ struct gma_encoder {
 struct gma_connector {
 	struct drm_connector base;
 	struct gma_encoder *encoder;
+
+	void (*save)(struct drm_connector *connector);
+	void (*restore)(struct drm_connector *connector);
 };
 
 struct psb_intel_crtc_state {
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index ce0645d0c1e5..b1b93317d054 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -653,8 +653,6 @@ const struct drm_connector_helper_funcs
 
 const struct drm_connector_funcs psb_intel_lvds_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = psb_intel_lvds_save,
-	.restore = psb_intel_lvds_restore,
 	.detect = psb_intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = psb_intel_lvds_set_property,
@@ -715,6 +713,9 @@ void psb_intel_lvds_init(struct drm_device *dev,
 	gma_encoder->dev_priv = lvds_priv;
 
 	connector = &gma_connector->base;
+	gma_connector->save = psb_intel_lvds_save;
+	gma_connector->restore = psb_intel_lvds_restore;
+
 	encoder = &gma_encoder->base;
 	drm_connector_init(dev, connector,
 			   &psb_intel_lvds_connector_funcs,
@@ -722,7 +723,7 @@ void psb_intel_lvds_init(struct drm_device *dev,
 
 	drm_encoder_init(dev, encoder,
 			 &psb_intel_lvds_enc_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 	gma_encoder->type = INTEL_OUTPUT_LVDS;
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index 58529cea575d..e787d376ba67 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -1837,8 +1837,6 @@ static const struct drm_encoder_helper_funcs psb_intel_sdvo_helper_funcs = {
 
 static const struct drm_connector_funcs psb_intel_sdvo_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = psb_intel_sdvo_save,
-	.restore = psb_intel_sdvo_restore,
 	.detect = psb_intel_sdvo_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = psb_intel_sdvo_set_property,
@@ -2021,6 +2019,9 @@ psb_intel_sdvo_connector_init(struct psb_intel_sdvo_connector *connector,
 	connector->base.base.doublescan_allowed = 0;
 	connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB;
 
+	connector->base.save = psb_intel_sdvo_save;
+	connector->base.restore = psb_intel_sdvo_restore;
+
 	gma_connector_attach_encoder(&connector->base, &encoder->base);
 	drm_connector_register(&connector->base.base);
 }
@@ -2525,7 +2526,8 @@ bool psb_intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
 	/* encoder type will be decided later */
 	gma_encoder = &psb_intel_sdvo->base;
 	gma_encoder->type = INTEL_OUTPUT_SDVO;
-	drm_encoder_init(dev, &gma_encoder->base, &psb_intel_sdvo_enc_funcs, 0);
+	drm_encoder_init(dev, &gma_encoder->base, &psb_intel_sdvo_enc_funcs,
+			 0, NULL);
 
 	/* Read the regs to test if we can talk to the device */
 	for (i = 0; i < 0x40; i++) {
diff --git a/drivers/gpu/drm/i2c/adv7511.c b/drivers/gpu/drm/i2c/adv7511.c
index 00416f23b5cb..533d1e3d4a99 100644
--- a/drivers/gpu/drm/i2c/adv7511.c
+++ b/drivers/gpu/drm/i2c/adv7511.c
@@ -752,7 +752,7 @@ static void adv7511_encoder_mode_set(struct drm_encoder *encoder,
 	adv7511->f_tmds = mode->clock;
 }
 
-static struct drm_encoder_slave_funcs adv7511_encoder_funcs = {
+static const struct drm_encoder_slave_funcs adv7511_encoder_funcs = {
 	.dpms = adv7511_encoder_dpms,
 	.mode_valid = adv7511_encoder_mode_valid,
 	.mode_set = adv7511_encoder_mode_set,
diff --git a/drivers/gpu/drm/i2c/ch7006_drv.c b/drivers/gpu/drm/i2c/ch7006_drv.c
index d9a72c96e56c..90db5f4dcce5 100644
--- a/drivers/gpu/drm/i2c/ch7006_drv.c
+++ b/drivers/gpu/drm/i2c/ch7006_drv.c
@@ -371,7 +371,7 @@ static int ch7006_encoder_set_property(struct drm_encoder *encoder,
 	return 0;
 }
 
-static struct drm_encoder_slave_funcs ch7006_encoder_funcs = {
+static const struct drm_encoder_slave_funcs ch7006_encoder_funcs = {
 	.set_config = ch7006_encoder_set_config,
 	.destroy = ch7006_encoder_destroy,
 	.dpms = ch7006_encoder_dpms,
diff --git a/drivers/gpu/drm/i2c/sil164_drv.c b/drivers/gpu/drm/i2c/sil164_drv.c
index 002ce7874332..c400428f6c8c 100644
--- a/drivers/gpu/drm/i2c/sil164_drv.c
+++ b/drivers/gpu/drm/i2c/sil164_drv.c
@@ -341,7 +341,7 @@ sil164_encoder_destroy(struct drm_encoder *encoder)
 	drm_i2c_encoder_destroy(encoder);
 }
 
-static struct drm_encoder_slave_funcs sil164_encoder_funcs = {
+static const struct drm_encoder_slave_funcs sil164_encoder_funcs = {
 	.set_config = sil164_encoder_set_config,
 	.destroy = sil164_encoder_destroy,
 	.dpms = sil164_encoder_dpms,
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 896b6aaf8c4d..34e38749a817 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -22,6 +22,7 @@
 #include <sound/asoundef.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_of.h>
@@ -855,18 +856,6 @@ static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
 	priv->dpms = mode;
 }
 
-static void
-tda998x_encoder_save(struct drm_encoder *encoder)
-{
-	DBG("");
-}
-
-static void
-tda998x_encoder_restore(struct drm_encoder *encoder)
-{
-	DBG("");
-}
-
 static bool
 tda998x_encoder_mode_fixup(struct drm_encoder *encoder,
 			  const struct drm_display_mode *mode,
@@ -878,7 +867,10 @@ tda998x_encoder_mode_fixup(struct drm_encoder *encoder,
 static int tda998x_connector_mode_valid(struct drm_connector *connector,
 					struct drm_display_mode *mode)
 {
-	if (mode->clock > 150000)
+	/* TDA19988 dotclock can go up to 165MHz */
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+
+	if (mode->clock > ((priv->rev == TDA19988) ? 165000 : 150000))
 		return MODE_CLOCK_HIGH;
 	if (mode->htotal >= BIT(13))
 		return MODE_BAD_HVALUE;
@@ -1351,8 +1343,6 @@ static void tda998x_encoder_commit(struct drm_encoder *encoder)
 
 static const struct drm_encoder_helper_funcs tda998x_encoder_helper_funcs = {
 	.dpms = tda998x_encoder_dpms,
-	.save = tda998x_encoder_save,
-	.restore = tda998x_encoder_restore,
 	.mode_fixup = tda998x_encoder_mode_fixup,
 	.prepare = tda998x_encoder_prepare,
 	.commit = tda998x_encoder_commit,
@@ -1393,10 +1383,13 @@ static void tda998x_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs tda998x_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = drm_atomic_helper_connector_dpms,
+	.reset = drm_atomic_helper_connector_reset,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = tda998x_connector_detect,
 	.destroy = tda998x_connector_destroy,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static int tda998x_bind(struct device *dev, struct device *master, void *data)
@@ -1437,7 +1430,7 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 
 	drm_encoder_helper_add(&priv->encoder, &tda998x_encoder_helper_funcs);
 	ret = drm_encoder_init(drm, &priv->encoder, &tda998x_encoder_funcs,
-			       DRM_MODE_ENCODER_TMDS);
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret)
 		goto err_encoder;
 
@@ -1453,7 +1446,6 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		goto err_sysfs;
 
-	priv->connector.encoder = &priv->encoder;
 	drm_mode_connector_attach_encoder(&priv->connector, &priv->encoder);
 
 	return 0;
@@ -1472,6 +1464,7 @@ static void tda998x_unbind(struct device *dev, struct device *master,
 {
 	struct tda998x_priv *priv = dev_get_drvdata(dev);
 
+	drm_connector_unregister(&priv->connector);
 	drm_connector_cleanup(&priv->connector);
 	drm_encoder_cleanup(&priv->encoder);
 	tda998x_destroy(priv);
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 051eab33e4c7..fcd77b27514d 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -10,6 +10,7 @@ config DRM_I915
 	# the shmem_readpage() which depends upon tmpfs
 	select SHMEM
 	select TMPFS
+	select STOP_MACHINE
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	select DRM_MIPI_DSI
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 44d290ae1999..0851de07bd13 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -77,6 +77,7 @@ i915-y += dvo_ch7017.o \
 	  dvo_tfp410.o \
 	  intel_crt.o \
 	  intel_ddi.o \
+	  intel_dp_link_training.o \
 	  intel_dp_mst.o \
 	  intel_dp.o \
 	  intel_dsi.o \
diff --git a/drivers/gpu/drm/i915/dvo.h b/drivers/gpu/drm/i915/dvo.h
index 0e2c1b9648a7..5e6a3013da49 100644
--- a/drivers/gpu/drm/i915/dvo.h
+++ b/drivers/gpu/drm/i915/dvo.h
@@ -32,7 +32,8 @@ struct intel_dvo_device {
 	const char *name;
 	int type;
 	/* DVOA/B/C output register */
-	u32 dvo_reg;
+	i915_reg_t dvo_reg;
+	i915_reg_t dvo_srcdim_reg;
 	/* GPIO register used for i2c bus to control this device */
 	u32 gpio;
 	int slave_addr;
@@ -128,11 +129,11 @@ struct intel_dvo_dev_ops {
 	void (*dump_regs)(struct intel_dvo_device *dvo);
 };
 
-extern struct intel_dvo_dev_ops sil164_ops;
-extern struct intel_dvo_dev_ops ch7xxx_ops;
-extern struct intel_dvo_dev_ops ivch_ops;
-extern struct intel_dvo_dev_ops tfp410_ops;
-extern struct intel_dvo_dev_ops ch7017_ops;
-extern struct intel_dvo_dev_ops ns2501_ops;
+extern const struct intel_dvo_dev_ops sil164_ops;
+extern const struct intel_dvo_dev_ops ch7xxx_ops;
+extern const struct intel_dvo_dev_ops ivch_ops;
+extern const struct intel_dvo_dev_ops tfp410_ops;
+extern const struct intel_dvo_dev_ops ch7017_ops;
+extern const struct intel_dvo_dev_ops ns2501_ops;
 
 #endif /* _INTEL_DVO_H */
diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c
index cbb22027a3ce..b3c7c199200c 100644
--- a/drivers/gpu/drm/i915/dvo_ch7017.c
+++ b/drivers/gpu/drm/i915/dvo_ch7017.c
@@ -402,7 +402,7 @@ static void ch7017_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops ch7017_ops = {
+const struct intel_dvo_dev_ops ch7017_ops = {
 	.init = ch7017_init,
 	.detect = ch7017_detect,
 	.mode_valid = ch7017_mode_valid,
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index 4b4acc1a06fe..44b3159f2fe8 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -356,7 +356,7 @@ static void ch7xxx_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops ch7xxx_ops = {
+const struct intel_dvo_dev_ops ch7xxx_ops = {
 	.init = ch7xxx_init,
 	.detect = ch7xxx_detect,
 	.mode_valid = ch7xxx_mode_valid,
diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c
index ff9f1b077d83..4950b82f5b49 100644
--- a/drivers/gpu/drm/i915/dvo_ivch.c
+++ b/drivers/gpu/drm/i915/dvo_ivch.c
@@ -490,7 +490,7 @@ static void ivch_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops ivch_ops = {
+const struct intel_dvo_dev_ops ivch_ops = {
 	.init = ivch_init,
 	.dpms = ivch_dpms,
 	.get_hw_state = ivch_get_hw_state,
diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c
index 063859fff0f0..2379c33cfe51 100644
--- a/drivers/gpu/drm/i915/dvo_ns2501.c
+++ b/drivers/gpu/drm/i915/dvo_ns2501.c
@@ -698,7 +698,7 @@ static void ns2501_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops ns2501_ops = {
+const struct intel_dvo_dev_ops ns2501_ops = {
 	.init = ns2501_init,
 	.detect = ns2501_detect,
 	.mode_valid = ns2501_mode_valid,
diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c
index 26f13eb634f9..1c1a0674dbab 100644
--- a/drivers/gpu/drm/i915/dvo_sil164.c
+++ b/drivers/gpu/drm/i915/dvo_sil164.c
@@ -267,7 +267,7 @@ static void sil164_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops sil164_ops = {
+const struct intel_dvo_dev_ops sil164_ops = {
 	.init = sil164_init,
 	.detect = sil164_detect,
 	.mode_valid = sil164_mode_valid,
diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c
index 6f1a0a6d4e22..31e181da93db 100644
--- a/drivers/gpu/drm/i915/dvo_tfp410.c
+++ b/drivers/gpu/drm/i915/dvo_tfp410.c
@@ -306,7 +306,7 @@ static void tfp410_destroy(struct intel_dvo_device *dvo)
 	}
 }
 
-struct intel_dvo_dev_ops tfp410_ops = {
+const struct intel_dvo_dev_ops tfp410_ops = {
 	.init = tfp410_init,
 	.detect = tfp410_detect,
 	.mode_valid = tfp410_mode_valid,
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index db58c8d664c2..814d894ed925 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -407,14 +407,14 @@ static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = {
  * LRI.
  */
 struct drm_i915_reg_descriptor {
-	u32 addr;
+	i915_reg_t addr;
 	u32 mask;
 	u32 value;
 };
 
 /* Convenience macro for adding 32-bit registers. */
-#define REG32(address, ...)                             \
-	{ .addr = address, __VA_ARGS__ }
+#define REG32(_reg, ...) \
+	{ .addr = (_reg), __VA_ARGS__ }
 
 /*
  * Convenience macro for adding 64-bit registers.
@@ -423,8 +423,13 @@ struct drm_i915_reg_descriptor {
  * access commands only allow 32-bit accesses. Hence, we have to include
  * entries for both halves of the 64-bit registers.
  */
-#define REG64(addr)                                     \
-	REG32(addr), REG32(addr + sizeof(u32))
+#define REG64(_reg) \
+	{ .addr = _reg }, \
+	{ .addr = _reg ## _UDW }
+
+#define REG64_IDX(_reg, idx) \
+	{ .addr = _reg(idx) }, \
+	{ .addr = _reg ## _UDW(idx) }
 
 static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
 	REG64(GPGPU_THREADS_DISPATCHED),
@@ -451,14 +456,14 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
 	REG32(GEN7_GPGPU_DISPATCHDIMX),
 	REG32(GEN7_GPGPU_DISPATCHDIMY),
 	REG32(GEN7_GPGPU_DISPATCHDIMZ),
-	REG64(GEN7_SO_NUM_PRIMS_WRITTEN(0)),
-	REG64(GEN7_SO_NUM_PRIMS_WRITTEN(1)),
-	REG64(GEN7_SO_NUM_PRIMS_WRITTEN(2)),
-	REG64(GEN7_SO_NUM_PRIMS_WRITTEN(3)),
-	REG64(GEN7_SO_PRIM_STORAGE_NEEDED(0)),
-	REG64(GEN7_SO_PRIM_STORAGE_NEEDED(1)),
-	REG64(GEN7_SO_PRIM_STORAGE_NEEDED(2)),
-	REG64(GEN7_SO_PRIM_STORAGE_NEEDED(3)),
+	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0),
+	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1),
+	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2),
+	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3),
+	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0),
+	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1),
+	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2),
+	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3),
 	REG32(GEN7_SO_WRITE_OFFSET(0)),
 	REG32(GEN7_SO_WRITE_OFFSET(1)),
 	REG32(GEN7_SO_WRITE_OFFSET(2)),
@@ -592,7 +597,7 @@ static bool check_sorted(int ring_id,
 	bool ret = true;
 
 	for (i = 0; i < reg_count; i++) {
-		u32 curr = reg_table[i].addr;
+		u32 curr = i915_mmio_reg_offset(reg_table[i].addr);
 
 		if (curr < previous) {
 			DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n",
@@ -847,7 +852,7 @@ find_reg(const struct drm_i915_reg_descriptor *table,
 		int i;
 
 		for (i = 0; i < count; i++) {
-			if (table[i].addr == addr)
+			if (i915_mmio_reg_offset(table[i].addr) == addr)
 				return &table[i];
 		}
 	}
@@ -1023,7 +1028,7 @@ static bool check_cmd(const struct intel_engine_cs *ring,
 			 * to the register. Hence, limit OACONTROL writes to
 			 * only MI_LOAD_REGISTER_IMM commands.
 			 */
-			if (reg_addr == OACONTROL) {
+			if (reg_addr == i915_mmio_reg_offset(OACONTROL)) {
 				if (desc->cmd.value == MI_LOAD_REGISTER_MEM) {
 					DRM_DEBUG_DRIVER("CMD: Rejected LRM to OACONTROL\n");
 					return false;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 8aab974b0564..0fc38bb7276c 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1142,8 +1142,34 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   MEMSTAT_VID_SHIFT);
 		seq_printf(m, "Current P-state: %d\n",
 			   (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
-	} else if (IS_GEN6(dev) || (IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) ||
-		   IS_BROADWELL(dev) || IS_GEN9(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+		u32 freq_sts;
+
+		mutex_lock(&dev_priv->rps.hw_lock);
+		freq_sts = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+		seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts);
+		seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq);
+
+		seq_printf(m, "actual GPU freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff));
+
+		seq_printf(m, "current GPU freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq));
+
+		seq_printf(m, "max GPU freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.max_freq));
+
+		seq_printf(m, "min GPU freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.min_freq));
+
+		seq_printf(m, "idle GPU freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq));
+
+		seq_printf(m,
+			   "efficient (RPe) frequency: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
+		mutex_unlock(&dev_priv->rps.hw_lock);
+	} else if (INTEL_INFO(dev)->gen >= 6) {
 		u32 rp_state_limits;
 		u32 gt_perf_status;
 		u32 rp_state_cap;
@@ -1252,18 +1278,21 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 
 		max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 0 :
 			    rp_state_cap >> 16) & 0xff;
-		max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (rp_state_cap & 0xff00) >> 8;
-		max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 16 :
 			    rp_state_cap >> 0) & 0xff;
-		max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 		seq_printf(m, "Max overclocked frequency: %dMHz\n",
@@ -1281,33 +1310,6 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		seq_printf(m,
 			   "efficient (RPe) frequency: %d MHz\n",
 			   intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
-	} else if (IS_VALLEYVIEW(dev)) {
-		u32 freq_sts;
-
-		mutex_lock(&dev_priv->rps.hw_lock);
-		freq_sts = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
-		seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts);
-		seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq);
-
-		seq_printf(m, "actual GPU freq: %d MHz\n",
-			   intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff));
-
-		seq_printf(m, "current GPU freq: %d MHz\n",
-			   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq));
-
-		seq_printf(m, "max GPU freq: %d MHz\n",
-			   intel_gpu_freq(dev_priv, dev_priv->rps.max_freq));
-
-		seq_printf(m, "min GPU freq: %d MHz\n",
-			   intel_gpu_freq(dev_priv, dev_priv->rps.min_freq));
-
-		seq_printf(m, "idle GPU freq: %d MHz\n",
-			   intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq));
-
-		seq_printf(m,
-			   "efficient (RPe) frequency: %d MHz\n",
-			   intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
-		mutex_unlock(&dev_priv->rps.hw_lock);
 	} else {
 		seq_puts(m, "no P-state info available\n");
 	}
@@ -1523,7 +1525,7 @@ static int gen6_drpc_info(struct seq_file *m)
 		seq_printf(m, "RC information accurate: %s\n", yesno(count < 51));
 	}
 
-	gt_core_status = readl(dev_priv->regs + GEN6_GT_CORE_STATUS);
+	gt_core_status = I915_READ_FW(GEN6_GT_CORE_STATUS);
 	trace_i915_reg_rw(false, GEN6_GT_CORE_STATUS, gt_core_status, 4, true);
 
 	rpmodectl1 = I915_READ(GEN6_RP_CONTROL);
@@ -1599,7 +1601,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
 	struct drm_info_node *node = m->private;
 	struct drm_device *dev = node->minor->dev;
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		return vlv_drpc_info(m);
 	else if (INTEL_INFO(dev)->gen >= 6)
 		return gen6_drpc_info(m);
@@ -1636,11 +1638,11 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 	intel_runtime_pm_get(dev_priv);
 	mutex_lock(&dev_priv->fbc.lock);
 
-	if (intel_fbc_enabled(dev_priv))
+	if (intel_fbc_is_active(dev_priv))
 		seq_puts(m, "FBC enabled\n");
 	else
 		seq_printf(m, "FBC disabled: %s\n",
-			  intel_no_fbc_reason_str(dev_priv->fbc.no_fbc_reason));
+			   dev_priv->fbc.no_fbc_reason);
 
 	if (INTEL_INFO(dev_priv)->gen >= 7)
 		seq_printf(m, "Compressing: %s\n",
@@ -1740,7 +1742,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 		sr_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 	else if (IS_PINEVIEW(dev))
 		sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 
 	intel_runtime_pm_put(dev_priv);
@@ -1801,7 +1803,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 	if (ret)
 		goto out;
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		/* Convert GT frequency to 50 HZ units */
 		min_gpu_freq =
 			dev_priv->rps.min_freq_softlimit / GEN9_FREQ_SCALER;
@@ -1821,7 +1823,8 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 				       &ia_freq);
 		seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
 			   intel_gpu_freq(dev_priv, (gpu_freq *
-				(IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1))),
+				(IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+				 GEN9_FREQ_SCALER : 1))),
 			   ((ia_freq >> 0) & 0xff) * 100,
 			   ((ia_freq >> 8) & 0xff) * 100);
 	}
@@ -1839,25 +1842,31 @@ static int i915_opregion(struct seq_file *m, void *unused)
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_opregion *opregion = &dev_priv->opregion;
-	void *data = kmalloc(OPREGION_SIZE, GFP_KERNEL);
 	int ret;
 
-	if (data == NULL)
-		return -ENOMEM;
-
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		goto out;
 
-	if (opregion->header) {
-		memcpy(data, opregion->header, OPREGION_SIZE);
-		seq_write(m, data, OPREGION_SIZE);
-	}
+	if (opregion->header)
+		seq_write(m, opregion->header, OPREGION_SIZE);
 
 	mutex_unlock(&dev->struct_mutex);
 
 out:
-	kfree(data);
+	return 0;
+}
+
+static int i915_vbt(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_opregion *opregion = &dev_priv->opregion;
+
+	if (opregion->vbt)
+		seq_write(m, opregion->vbt, opregion->vbt_size);
+
 	return 0;
 }
 
@@ -1865,31 +1874,29 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = m->private;
 	struct drm_device *dev = node->minor->dev;
-	struct intel_fbdev *ifbdev = NULL;
-	struct intel_framebuffer *fb;
+	struct intel_framebuffer *fbdev_fb = NULL;
 	struct drm_framebuffer *drm_fb;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	ifbdev = dev_priv->fbdev;
-	fb = to_intel_framebuffer(ifbdev->helper.fb);
-
-	seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
-		   fb->base.width,
-		   fb->base.height,
-		   fb->base.depth,
-		   fb->base.bits_per_pixel,
-		   fb->base.modifier[0],
-		   atomic_read(&fb->base.refcount.refcount));
-	describe_obj(m, fb->obj);
-	seq_putc(m, '\n');
+       if (to_i915(dev)->fbdev) {
+               fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb);
+
+               seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
+                         fbdev_fb->base.width,
+                         fbdev_fb->base.height,
+                         fbdev_fb->base.depth,
+                         fbdev_fb->base.bits_per_pixel,
+                         fbdev_fb->base.modifier[0],
+                         atomic_read(&fbdev_fb->base.refcount.refcount));
+               describe_obj(m, fbdev_fb->obj);
+               seq_putc(m, '\n');
+       }
 #endif
 
 	mutex_lock(&dev->mode_config.fb_lock);
 	drm_for_each_fb(drm_fb, dev) {
-		fb = to_intel_framebuffer(drm_fb);
-		if (ifbdev && &fb->base == ifbdev->helper.fb)
+		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
+		if (fb == fbdev_fb)
 			continue;
 
 		seq_printf(m, "user size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
@@ -2402,6 +2409,12 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data)
 		guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted);
 	seq_printf(m, "\tversion found: %d.%d\n",
 		guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found);
+	seq_printf(m, "\theader: offset is %d; size = %d\n",
+		guc_fw->header_offset, guc_fw->header_size);
+	seq_printf(m, "\tuCode: offset is %d; size = %d\n",
+		guc_fw->ucode_offset, guc_fw->ucode_size);
+	seq_printf(m, "\tRSA: offset is %d; size = %d\n",
+		guc_fw->rsa_offset, guc_fw->rsa_size);
 
 	tmp = I915_READ(GUC_STATUS);
 
@@ -2461,15 +2474,15 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	if (!HAS_GUC_SCHED(dev_priv->dev))
 		return 0;
 
+	if (mutex_lock_interruptible(&dev->struct_mutex))
+		return 0;
+
 	/* Take a local copy of the GuC data, so we can dump it at leisure */
-	spin_lock(&dev_priv->guc.host2guc_lock);
 	guc = dev_priv->guc;
-	if (guc.execbuf_client) {
-		spin_lock(&guc.execbuf_client->wq_lock);
+	if (guc.execbuf_client)
 		client = *guc.execbuf_client;
-		spin_unlock(&guc.execbuf_client->wq_lock);
-	}
-	spin_unlock(&dev_priv->guc.host2guc_lock);
+
+	mutex_unlock(&dev->struct_mutex);
 
 	seq_printf(m, "GuC total action count: %llu\n", guc.action_count);
 	seq_printf(m, "GuC action failure count: %u\n", guc.action_fail);
@@ -2550,7 +2563,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 		   yesno(work_busy(&dev_priv->psr.work.work)));
 
 	if (HAS_DDI(dev))
-		enabled = I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE;
+		enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE;
 	else {
 		for_each_pipe(dev_priv, pipe) {
 			stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) &
@@ -2570,9 +2583,12 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 		}
 	seq_puts(m, "\n");
 
-	/* CHV PSR has no kind of performance counter */
-	if (HAS_DDI(dev)) {
-		psrperf = I915_READ(EDP_PSR_PERF_CNT(dev)) &
+	/*
+	 * VLV/CHV PSR has no kind of performance counter
+	 * SKL+ Perf counter is reset to 0 everytime DC state is entered
+	 */
+	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+		psrperf = I915_READ(EDP_PSR_PERF_CNT) &
 			EDP_PSR_PERF_CNT_MASK;
 
 		seq_printf(m, "Performance_Counter: %u\n", psrperf);
@@ -2673,77 +2689,6 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static const char *power_domain_str(enum intel_display_power_domain domain)
-{
-	switch (domain) {
-	case POWER_DOMAIN_PIPE_A:
-		return "PIPE_A";
-	case POWER_DOMAIN_PIPE_B:
-		return "PIPE_B";
-	case POWER_DOMAIN_PIPE_C:
-		return "PIPE_C";
-	case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
-		return "PIPE_A_PANEL_FITTER";
-	case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
-		return "PIPE_B_PANEL_FITTER";
-	case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
-		return "PIPE_C_PANEL_FITTER";
-	case POWER_DOMAIN_TRANSCODER_A:
-		return "TRANSCODER_A";
-	case POWER_DOMAIN_TRANSCODER_B:
-		return "TRANSCODER_B";
-	case POWER_DOMAIN_TRANSCODER_C:
-		return "TRANSCODER_C";
-	case POWER_DOMAIN_TRANSCODER_EDP:
-		return "TRANSCODER_EDP";
-	case POWER_DOMAIN_PORT_DDI_A_2_LANES:
-		return "PORT_DDI_A_2_LANES";
-	case POWER_DOMAIN_PORT_DDI_A_4_LANES:
-		return "PORT_DDI_A_4_LANES";
-	case POWER_DOMAIN_PORT_DDI_B_2_LANES:
-		return "PORT_DDI_B_2_LANES";
-	case POWER_DOMAIN_PORT_DDI_B_4_LANES:
-		return "PORT_DDI_B_4_LANES";
-	case POWER_DOMAIN_PORT_DDI_C_2_LANES:
-		return "PORT_DDI_C_2_LANES";
-	case POWER_DOMAIN_PORT_DDI_C_4_LANES:
-		return "PORT_DDI_C_4_LANES";
-	case POWER_DOMAIN_PORT_DDI_D_2_LANES:
-		return "PORT_DDI_D_2_LANES";
-	case POWER_DOMAIN_PORT_DDI_D_4_LANES:
-		return "PORT_DDI_D_4_LANES";
-	case POWER_DOMAIN_PORT_DDI_E_2_LANES:
-		return "PORT_DDI_E_2_LANES";
-	case POWER_DOMAIN_PORT_DSI:
-		return "PORT_DSI";
-	case POWER_DOMAIN_PORT_CRT:
-		return "PORT_CRT";
-	case POWER_DOMAIN_PORT_OTHER:
-		return "PORT_OTHER";
-	case POWER_DOMAIN_VGA:
-		return "VGA";
-	case POWER_DOMAIN_AUDIO:
-		return "AUDIO";
-	case POWER_DOMAIN_PLLS:
-		return "PLLS";
-	case POWER_DOMAIN_AUX_A:
-		return "AUX_A";
-	case POWER_DOMAIN_AUX_B:
-		return "AUX_B";
-	case POWER_DOMAIN_AUX_C:
-		return "AUX_C";
-	case POWER_DOMAIN_AUX_D:
-		return "AUX_D";
-	case POWER_DOMAIN_GMBUS:
-		return "GMBUS";
-	case POWER_DOMAIN_INIT:
-		return "INIT";
-	default:
-		MISSING_CASE(domain);
-		return "?";
-	}
-}
-
 static int i915_power_domain_info(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = m->private;
@@ -2769,7 +2714,7 @@ static int i915_power_domain_info(struct seq_file *m, void *unused)
 				continue;
 
 			seq_printf(m, "  %-23s %d\n",
-				 power_domain_str(power_domain),
+				 intel_display_power_domain_str(power_domain),
 				 power_domains->domain_use_count[power_domain]);
 		}
 	}
@@ -2779,6 +2724,51 @@ static int i915_power_domain_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
+static int i915_dmc_info(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_csr *csr;
+
+	if (!HAS_CSR(dev)) {
+		seq_puts(m, "not supported\n");
+		return 0;
+	}
+
+	csr = &dev_priv->csr;
+
+	intel_runtime_pm_get(dev_priv);
+
+	seq_printf(m, "fw loaded: %s\n", yesno(csr->dmc_payload != NULL));
+	seq_printf(m, "path: %s\n", csr->fw_path);
+
+	if (!csr->dmc_payload)
+		goto out;
+
+	seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version),
+		   CSR_VERSION_MINOR(csr->version));
+
+	if (IS_SKYLAKE(dev) && csr->version >= CSR_VERSION(1, 6)) {
+		seq_printf(m, "DC3 -> DC5 count: %d\n",
+			   I915_READ(SKL_CSR_DC3_DC5_COUNT));
+		seq_printf(m, "DC5 -> DC6 count: %d\n",
+			   I915_READ(SKL_CSR_DC5_DC6_COUNT));
+	} else if (IS_BROXTON(dev) && csr->version >= CSR_VERSION(1, 4)) {
+		seq_printf(m, "DC3 -> DC5 count: %d\n",
+			   I915_READ(BXT_CSR_DC3_DC5_COUNT));
+	}
+
+out:
+	seq_printf(m, "program base: 0x%08x\n", I915_READ(CSR_PROGRAM(0)));
+	seq_printf(m, "ssp base: 0x%08x\n", I915_READ(CSR_SSP_BASE));
+	seq_printf(m, "htp: 0x%08x\n", I915_READ(CSR_HTP_SKL));
+
+	intel_runtime_pm_put(dev_priv);
+
+	return 0;
+}
+
 static void intel_seq_print_mode(struct seq_file *m, int tabs,
 				 struct drm_display_mode *mode)
 {
@@ -2865,6 +2855,20 @@ static void intel_dp_info(struct seq_file *m,
 		intel_panel_info(m, &intel_connector->panel);
 }
 
+static void intel_dp_mst_info(struct seq_file *m,
+			  struct intel_connector *intel_connector)
+{
+	struct intel_encoder *intel_encoder = intel_connector->encoder;
+	struct intel_dp_mst_encoder *intel_mst =
+		enc_to_mst(&intel_encoder->base);
+	struct intel_digital_port *intel_dig_port = intel_mst->primary;
+	struct intel_dp *intel_dp = &intel_dig_port->dp;
+	bool has_audio = drm_dp_mst_port_has_audio(&intel_dp->mst_mgr,
+					intel_connector->port);
+
+	seq_printf(m, "\taudio support: %s\n", yesno(has_audio));
+}
+
 static void intel_hdmi_info(struct seq_file *m,
 			    struct intel_connector *intel_connector)
 {
@@ -2908,6 +2912,8 @@ static void intel_connector_info(struct seq_file *m,
 			intel_hdmi_info(m, intel_connector);
 		else if (intel_encoder->type == INTEL_OUTPUT_LVDS)
 			intel_lvds_info(m, intel_connector);
+		else if (intel_encoder->type == INTEL_OUTPUT_DP_MST)
+			intel_dp_mst_info(m, intel_connector);
 	}
 
 	seq_printf(m, "\tmodes:\n");
@@ -2946,6 +2952,107 @@ static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y)
 	return cursor_active(dev, pipe);
 }
 
+static const char *plane_type(enum drm_plane_type type)
+{
+	switch (type) {
+	case DRM_PLANE_TYPE_OVERLAY:
+		return "OVL";
+	case DRM_PLANE_TYPE_PRIMARY:
+		return "PRI";
+	case DRM_PLANE_TYPE_CURSOR:
+		return "CUR";
+	/*
+	 * Deliberately omitting default: to generate compiler warnings
+	 * when a new drm_plane_type gets added.
+	 */
+	}
+
+	return "unknown";
+}
+
+static const char *plane_rotation(unsigned int rotation)
+{
+	static char buf[48];
+	/*
+	 * According to doc only one DRM_ROTATE_ is allowed but this
+	 * will print them all to visualize if the values are misused
+	 */
+	snprintf(buf, sizeof(buf),
+		 "%s%s%s%s%s%s(0x%08x)",
+		 (rotation & BIT(DRM_ROTATE_0)) ? "0 " : "",
+		 (rotation & BIT(DRM_ROTATE_90)) ? "90 " : "",
+		 (rotation & BIT(DRM_ROTATE_180)) ? "180 " : "",
+		 (rotation & BIT(DRM_ROTATE_270)) ? "270 " : "",
+		 (rotation & BIT(DRM_REFLECT_X)) ? "FLIPX " : "",
+		 (rotation & BIT(DRM_REFLECT_Y)) ? "FLIPY " : "",
+		 rotation);
+
+	return buf;
+}
+
+static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct intel_plane *intel_plane;
+
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		struct drm_plane_state *state;
+		struct drm_plane *plane = &intel_plane->base;
+
+		if (!plane->state) {
+			seq_puts(m, "plane->state is NULL!\n");
+			continue;
+		}
+
+		state = plane->state;
+
+		seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n",
+			   plane->base.id,
+			   plane_type(intel_plane->base.type),
+			   state->crtc_x, state->crtc_y,
+			   state->crtc_w, state->crtc_h,
+			   (state->src_x >> 16),
+			   ((state->src_x & 0xffff) * 15625) >> 10,
+			   (state->src_y >> 16),
+			   ((state->src_y & 0xffff) * 15625) >> 10,
+			   (state->src_w >> 16),
+			   ((state->src_w & 0xffff) * 15625) >> 10,
+			   (state->src_h >> 16),
+			   ((state->src_h & 0xffff) * 15625) >> 10,
+			   state->fb ? drm_get_format_name(state->fb->pixel_format) : "N/A",
+			   plane_rotation(state->rotation));
+	}
+}
+
+static void intel_scaler_info(struct seq_file *m, struct intel_crtc *intel_crtc)
+{
+	struct intel_crtc_state *pipe_config;
+	int num_scalers = intel_crtc->num_scalers;
+	int i;
+
+	pipe_config = to_intel_crtc_state(intel_crtc->base.state);
+
+	/* Not all platformas have a scaler */
+	if (num_scalers) {
+		seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d",
+			   num_scalers,
+			   pipe_config->scaler_state.scaler_users,
+			   pipe_config->scaler_state.scaler_id);
+
+		for (i = 0; i < SKL_NUM_SCALERS; i++) {
+			struct intel_scaler *sc =
+					&pipe_config->scaler_state.scalers[i];
+
+			seq_printf(m, ", scalers[%d]: use=%s, mode=%x",
+				   i, yesno(sc->in_use), sc->mode);
+		}
+		seq_puts(m, "\n");
+	} else {
+		seq_puts(m, "\tNo scalers available on this platform\n");
+	}
+}
+
 static int i915_display_info(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = m->private;
@@ -2965,10 +3072,12 @@ static int i915_display_info(struct seq_file *m, void *unused)
 
 		pipe_config = to_intel_crtc_state(crtc->base.state);
 
-		seq_printf(m, "CRTC %d: pipe: %c, active=%s (size=%dx%d)\n",
+		seq_printf(m, "CRTC %d: pipe: %c, active=%s, (size=%dx%d), dither=%s, bpp=%d\n",
 			   crtc->base.base.id, pipe_name(crtc->pipe),
 			   yesno(pipe_config->base.active),
-			   pipe_config->pipe_src_w, pipe_config->pipe_src_h);
+			   pipe_config->pipe_src_w, pipe_config->pipe_src_h,
+			   yesno(pipe_config->dither), pipe_config->pipe_bpp);
+
 		if (pipe_config->base.active) {
 			intel_crtc_info(m, crtc);
 
@@ -2978,6 +3087,8 @@ static int i915_display_info(struct seq_file *m, void *unused)
 				   x, y, crtc->base.cursor->state->crtc_w,
 				   crtc->base.cursor->state->crtc_h,
 				   crtc->cursor_addr, yesno(active));
+			intel_scaler_info(m, crtc);
+			intel_plane_info(m, crtc);
 		}
 
 		seq_printf(m, "\tunderrun reporting: cpu=%s pch=%s \n",
@@ -3112,7 +3223,8 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 
 	seq_printf(m, "Workarounds applied: %d\n", dev_priv->workarounds.count);
 	for (i = 0; i < dev_priv->workarounds.count; ++i) {
-		u32 addr, mask, value, read;
+		i915_reg_t addr;
+		u32 mask, value, read;
 		bool ok;
 
 		addr = dev_priv->workarounds.reg[i].addr;
@@ -3121,7 +3233,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 		read = I915_READ(addr);
 		ok = (value & mask) == (read & mask);
 		seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X, read: 0x%08x, status: %s\n",
-			   addr, value, mask, read, ok ? "OK" : "FAIL");
+			   i915_mmio_reg_offset(addr), value, mask, read, ok ? "OK" : "FAIL");
 	}
 
 	intel_runtime_pm_put(dev_priv);
@@ -3892,7 +4004,7 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
 		ret = i8xx_pipe_crc_ctl_reg(&source, &val);
 	else if (INTEL_INFO(dev)->gen < 5)
 		ret = i9xx_pipe_crc_ctl_reg(dev, pipe, &source, &val);
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		ret = vlv_pipe_crc_ctl_reg(dev, pipe, &source, &val);
 	else if (IS_GEN5(dev) || IS_GEN6(dev))
 		ret = ilk_pipe_crc_ctl_reg(&source, &val);
@@ -3961,7 +4073,7 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
 
 		if (IS_G4X(dev))
 			g4x_undo_pipe_scramble_reset(dev, pipe);
-		else if (IS_VALLEYVIEW(dev))
+		else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 			vlv_undo_pipe_scramble_reset(dev, pipe);
 		else if (IS_HASWELL(dev) && pipe == PIPE_A)
 			hsw_trans_edp_pipe_A_crc_wa(dev, false);
@@ -4351,7 +4463,8 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 		 * - WM1+ latency values in 0.5us units
 		 * - latencies are in us on gen9/vlv/chv
 		 */
-		if (INTEL_INFO(dev)->gen >= 9 || IS_VALLEYVIEW(dev))
+		if (INTEL_INFO(dev)->gen >= 9 || IS_VALLEYVIEW(dev) ||
+		    IS_CHERRYVIEW(dev))
 			latency *= 10;
 		else if (level > 0)
 			latency *= 5;
@@ -5025,7 +5138,7 @@ static void gen9_sseu_device_status(struct drm_device *dev,
 
 		stat->slice_total++;
 
-		if (IS_SKYLAKE(dev))
+		if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 			ss_cnt = INTEL_INFO(dev)->subslice_per_slice;
 
 		for (ss = 0; ss < ss_max; ss++) {
@@ -5225,6 +5338,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_ips_status", i915_ips_status, 0},
 	{"i915_sr_status", i915_sr_status, 0},
 	{"i915_opregion", i915_opregion, 0},
+	{"i915_vbt", i915_vbt, 0},
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
 	{"i915_context_status", i915_context_status, 0},
 	{"i915_dump_lrc", i915_dump_lrc, 0},
@@ -5238,6 +5352,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_energy_uJ", i915_energy_uJ, 0},
 	{"i915_runtime_pm_status", i915_runtime_pm_status, 0},
 	{"i915_power_domain_info", i915_power_domain_info, 0},
+	{"i915_dmc_info", i915_dmc_info, 0},
 	{"i915_display_info", i915_display_info, 0},
 	{"i915_semaphore_status", i915_semaphore_status, 0},
 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index b4741d121a74..d70d96fe553b 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -28,7 +28,6 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/async.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
@@ -170,6 +169,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_RESOURCE_STREAMER:
 		value = HAS_RESOURCE_STREAMER(dev);
 		break;
+	case I915_PARAM_HAS_EXEC_SOFTPIN:
+		value = 1;
+		break;
 	default:
 		DRM_DEBUG("Unknown parameter %d\n", param->param);
 		return -EINVAL;
@@ -257,7 +259,7 @@ intel_setup_mchbar(struct drm_device *dev)
 	u32 temp;
 	bool enabled;
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		return;
 
 	dev_priv->mchbar_need_disable = false;
@@ -338,7 +340,7 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_
 		i915_resume_switcheroo(dev);
 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
 	} else {
-		pr_err("switched off\n");
+		pr_info("switched off\n");
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 		i915_suspend_switcheroo(dev, pmm);
 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
@@ -368,7 +370,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 
-	ret = intel_parse_bios(dev);
+	ret = intel_bios_init(dev_priv);
 	if (ret)
 		DRM_INFO("failed to find VBIOS tables\n");
 
@@ -396,12 +398,16 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	if (ret)
 		goto cleanup_vga_switcheroo;
 
-	intel_power_domains_init_hw(dev_priv);
+	intel_power_domains_init_hw(dev_priv, false);
+
+	intel_csr_ucode_init(dev_priv);
 
 	ret = intel_irq_install(dev_priv);
 	if (ret)
 		goto cleanup_gem_stolen;
 
+	intel_setup_gmbus(dev);
+
 	/* Important: The output setup functions called by modeset_init need
 	 * working irqs for e.g. gmbus and dp aux transfers. */
 	intel_modeset_init(dev);
@@ -437,7 +443,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	 * scanning against hotplug events. Hence do this first and ignore the
 	 * tiny window where we will loose hotplug notifactions.
 	 */
-	async_schedule(intel_fbdev_initial_config, dev_priv);
+	intel_fbdev_initial_config_async(dev);
 
 	drm_kms_helper_poll_init(dev);
 
@@ -451,6 +457,7 @@ cleanup_gem:
 cleanup_irq:
 	intel_guc_ucode_fini(dev);
 	drm_irq_uninstall(dev);
+	intel_teardown_gmbus(dev);
 cleanup_gem_stolen:
 	i915_gem_cleanup_stolen(dev);
 cleanup_vga_switcheroo:
@@ -663,7 +670,8 @@ static void gen9_sseu_info_init(struct drm_device *dev)
 	 * supports EU power gating on devices with more than one EU
 	 * pair per subslice.
 	*/
-	info->has_slice_pg = (IS_SKYLAKE(dev) && (info->slice_total > 1));
+	info->has_slice_pg = ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
+			       (info->slice_total > 1));
 	info->has_subslice_pg = (IS_BROXTON(dev) && (info->subslice_total > 1));
 	info->has_eu_pg = (info->eu_per_subslice > 2);
 }
@@ -777,7 +785,7 @@ static void intel_device_info_runtime_init(struct drm_device *dev)
 		info->num_sprites[PIPE_A] = 2;
 		info->num_sprites[PIPE_B] = 2;
 		info->num_sprites[PIPE_C] = 1;
-	} else if (IS_VALLEYVIEW(dev))
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 2;
 	else
@@ -789,7 +797,7 @@ static void intel_device_info_runtime_init(struct drm_device *dev)
 		info->num_pipes = 0;
 	} else if (info->num_pipes > 0 &&
 		   (INTEL_INFO(dev)->gen == 7 || INTEL_INFO(dev)->gen == 8) &&
-		   !IS_VALLEYVIEW(dev)) {
+		   HAS_PCH_SPLIT(dev)) {
 		u32 fuse_strap = I915_READ(FUSE_STRAP);
 		u32 sfuse_strap = I915_READ(SFUSE_STRAP);
 
@@ -834,9 +842,6 @@ static void intel_device_info_runtime_init(struct drm_device *dev)
 
 static void intel_init_dpio(struct drm_i915_private *dev_priv)
 {
-	if (!IS_VALLEYVIEW(dev_priv))
-		return;
-
 	/*
 	 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
 	 * CHV x1 PHY (DP/HDMI D)
@@ -845,7 +850,7 @@ static void intel_init_dpio(struct drm_i915_private *dev_priv)
 	if (IS_CHERRYVIEW(dev_priv)) {
 		DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
 		DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
-	} else {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
 	}
 }
@@ -890,11 +895,12 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	spin_lock_init(&dev_priv->mmio_flip_lock);
 	mutex_init(&dev_priv->sb_lock);
 	mutex_init(&dev_priv->modeset_restore_lock);
-	mutex_init(&dev_priv->csr_lock);
 	mutex_init(&dev_priv->av_mutex);
 
 	intel_pm_setup(dev);
 
+	intel_runtime_pm_get(dev_priv);
+
 	intel_display_crc_init(dev);
 
 	i915_dump_device_info(dev_priv);
@@ -937,9 +943,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	intel_uncore_init(dev);
 
-	/* Load CSR Firmware for SKL */
-	intel_csr_ucode_init(dev);
-
 	ret = i915_gem_gtt_init(dev);
 	if (ret)
 		goto out_freecsr;
@@ -1028,7 +1031,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	/* Try to make sure MCHBAR is enabled before poking at it */
 	intel_setup_mchbar(dev);
-	intel_setup_gmbus(dev);
 	intel_opregion_setup(dev);
 
 	i915_gem_load(dev);
@@ -1087,6 +1089,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	i915_audio_component_init(dev_priv);
 
+	intel_runtime_pm_put(dev_priv);
+
 	return 0;
 
 out_power_well:
@@ -1099,7 +1103,6 @@ out_gem_unload:
 	if (dev->pdev->msi_enabled)
 		pci_disable_msi(dev->pdev);
 
-	intel_teardown_gmbus(dev);
 	intel_teardown_mchbar(dev);
 	pm_qos_remove_request(&dev_priv->pm_qos);
 	destroy_workqueue(dev_priv->gpu_error.hangcheck_wq);
@@ -1113,7 +1116,7 @@ out_mtrrfree:
 out_gtt:
 	i915_global_gtt_cleanup(dev);
 out_freecsr:
-	intel_csr_ucode_fini(dev);
+	intel_csr_ucode_fini(dev_priv);
 	intel_uncore_fini(dev);
 	pci_iounmap(dev->pdev, dev_priv->regs);
 put_bridge:
@@ -1122,6 +1125,9 @@ free_priv:
 	kmem_cache_destroy(dev_priv->requests);
 	kmem_cache_destroy(dev_priv->vmas);
 	kmem_cache_destroy(dev_priv->objects);
+
+	intel_runtime_pm_put(dev_priv);
+
 	kfree(dev_priv);
 	return ret;
 }
@@ -1131,6 +1137,8 @@ int i915_driver_unload(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 
+	intel_fbdev_fini(dev);
+
 	i915_audio_component_cleanup(dev_priv);
 
 	ret = i915_gem_suspend(dev);
@@ -1153,8 +1161,6 @@ int i915_driver_unload(struct drm_device *dev)
 
 	acpi_video_unregister();
 
-	intel_fbdev_fini(dev);
-
 	drm_vblank_cleanup(dev);
 
 	intel_modeset_cleanup(dev);
@@ -1196,9 +1202,8 @@ int i915_driver_unload(struct drm_device *dev)
 	intel_fbc_cleanup_cfb(dev_priv);
 	i915_gem_cleanup_stolen(dev);
 
-	intel_csr_ucode_fini(dev);
+	intel_csr_ucode_fini(dev_priv);
 
-	intel_teardown_gmbus(dev);
 	intel_teardown_mchbar(dev);
 
 	destroy_workqueue(dev_priv->hotplug.dp_wq);
@@ -1264,8 +1269,6 @@ void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 
-	if (file_priv && file_priv->bsd_ring)
-		file_priv->bsd_ring = NULL;
 	kfree(file_priv);
 }
 
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 760e0ce4aa26..3ac616d7363b 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -228,161 +228,111 @@ static const struct intel_device_info intel_sandybridge_m_info = {
 	.need_gfx_hws = 1, .has_hotplug = 1, \
 	.has_fbc = 1, \
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
-	.has_llc = 1
+	.has_llc = 1, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	IVB_CURSOR_OFFSETS
 
 static const struct intel_device_info intel_ivybridge_d_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_ivybridge_m_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
 	.is_mobile = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_ivybridge_q_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
 	.num_pipes = 0, /* legal, last one wins */
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
+#define VLV_FEATURES  \
+	.gen = 7, .num_pipes = 2, \
+	.need_gfx_hws = 1, .has_hotplug = 1, \
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+	.display_mmio_offset = VLV_DISPLAY_BASE, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_valleyview_m_info = {
-	GEN7_FEATURES,
-	.is_mobile = 1,
-	.num_pipes = 2,
+	VLV_FEATURES,
 	.is_valleyview = 1,
-	.display_mmio_offset = VLV_DISPLAY_BASE,
-	.has_fbc = 0, /* legal, last one wins */
-	.has_llc = 0, /* legal, last one wins */
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.is_mobile = 1,
 };
 
 static const struct intel_device_info intel_valleyview_d_info = {
-	GEN7_FEATURES,
-	.num_pipes = 2,
+	VLV_FEATURES,
 	.is_valleyview = 1,
-	.display_mmio_offset = VLV_DISPLAY_BASE,
-	.has_fbc = 0, /* legal, last one wins */
-	.has_llc = 0, /* legal, last one wins */
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
+#define HSW_FEATURES  \
+	GEN7_FEATURES, \
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
+	.has_ddi = 1, \
+	.has_fpga_dbg = 1
+
 static const struct intel_device_info intel_haswell_d_info = {
-	GEN7_FEATURES,
+	HSW_FEATURES,
 	.is_haswell = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_haswell_m_info = {
-	GEN7_FEATURES,
+	HSW_FEATURES,
 	.is_haswell = 1,
 	.is_mobile = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_broadwell_d_info = {
-	.gen = 8, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
+	HSW_FEATURES,
+	.gen = 8,
 };
 
 static const struct intel_device_info intel_broadwell_m_info = {
-	.gen = 8, .is_mobile = 1, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
+	HSW_FEATURES,
+	.gen = 8, .is_mobile = 1,
 };
 
 static const struct intel_device_info intel_broadwell_gt3d_info = {
-	.gen = 8, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	HSW_FEATURES,
+	.gen = 8,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_broadwell_gt3m_info = {
-	.gen = 8, .is_mobile = 1, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	HSW_FEATURES,
+	.gen = 8, .is_mobile = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_cherryview_info = {
 	.gen = 8, .num_pipes = 3,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	.is_valleyview = 1,
+	.is_cherryview = 1,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
 	GEN_CHV_PIPEOFFSETS,
 	CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_skylake_info = {
+	HSW_FEATURES,
 	.is_skylake = 1,
-	.gen = 9, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
+	.gen = 9,
 };
 
 static const struct intel_device_info intel_skylake_gt3_info = {
+	HSW_FEATURES,
 	.is_skylake = 1,
-	.gen = 9, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	.gen = 9,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-	.has_llc = 1,
-	.has_ddi = 1,
-	.has_fpga_dbg = 1,
-	.has_fbc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	IVB_CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_broxton_info = {
 	.is_preliminary = 1,
+	.is_broxton = 1,
 	.gen = 9,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
@@ -394,50 +344,67 @@ static const struct intel_device_info intel_broxton_info = {
 	IVB_CURSOR_OFFSETS,
 };
 
+static const struct intel_device_info intel_kabylake_info = {
+	HSW_FEATURES,
+	.is_preliminary = 1,
+	.is_kabylake = 1,
+	.gen = 9,
+};
+
+static const struct intel_device_info intel_kabylake_gt3_info = {
+	HSW_FEATURES,
+	.is_preliminary = 1,
+	.is_kabylake = 1,
+	.gen = 9,
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
 /*
  * Make sure any device matches here are from most specific to most
  * general.  For example, since the Quanta match is based on the subsystem
  * and subvendor IDs, we need it to come before the more general IVB
  * PCI ID matches, otherwise we'll use the wrong info struct above.
  */
-#define INTEL_PCI_IDS \
-	INTEL_I830_IDS(&intel_i830_info),	\
-	INTEL_I845G_IDS(&intel_845g_info),	\
-	INTEL_I85X_IDS(&intel_i85x_info),	\
-	INTEL_I865G_IDS(&intel_i865g_info),	\
-	INTEL_I915G_IDS(&intel_i915g_info),	\
-	INTEL_I915GM_IDS(&intel_i915gm_info),	\
-	INTEL_I945G_IDS(&intel_i945g_info),	\
-	INTEL_I945GM_IDS(&intel_i945gm_info),	\
-	INTEL_I965G_IDS(&intel_i965g_info),	\
-	INTEL_G33_IDS(&intel_g33_info),		\
-	INTEL_I965GM_IDS(&intel_i965gm_info),	\
-	INTEL_GM45_IDS(&intel_gm45_info), 	\
-	INTEL_G45_IDS(&intel_g45_info), 	\
-	INTEL_PINEVIEW_IDS(&intel_pineview_info),	\
-	INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),	\
-	INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),	\
-	INTEL_SNB_D_IDS(&intel_sandybridge_d_info),	\
-	INTEL_SNB_M_IDS(&intel_sandybridge_m_info),	\
-	INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ \
-	INTEL_IVB_M_IDS(&intel_ivybridge_m_info),	\
-	INTEL_IVB_D_IDS(&intel_ivybridge_d_info),	\
-	INTEL_HSW_D_IDS(&intel_haswell_d_info), \
-	INTEL_HSW_M_IDS(&intel_haswell_m_info), \
-	INTEL_VLV_M_IDS(&intel_valleyview_m_info),	\
-	INTEL_VLV_D_IDS(&intel_valleyview_d_info),	\
-	INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),	\
-	INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),	\
-	INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),	\
-	INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \
-	INTEL_CHV_IDS(&intel_cherryview_info),	\
-	INTEL_SKL_GT1_IDS(&intel_skylake_info),	\
-	INTEL_SKL_GT2_IDS(&intel_skylake_info),	\
-	INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),	\
-	INTEL_BXT_IDS(&intel_broxton_info)
-
-static const struct pci_device_id pciidlist[] = {		/* aka */
-	INTEL_PCI_IDS,
+static const struct pci_device_id pciidlist[] = {
+	INTEL_I830_IDS(&intel_i830_info),
+	INTEL_I845G_IDS(&intel_845g_info),
+	INTEL_I85X_IDS(&intel_i85x_info),
+	INTEL_I865G_IDS(&intel_i865g_info),
+	INTEL_I915G_IDS(&intel_i915g_info),
+	INTEL_I915GM_IDS(&intel_i915gm_info),
+	INTEL_I945G_IDS(&intel_i945g_info),
+	INTEL_I945GM_IDS(&intel_i945gm_info),
+	INTEL_I965G_IDS(&intel_i965g_info),
+	INTEL_G33_IDS(&intel_g33_info),
+	INTEL_I965GM_IDS(&intel_i965gm_info),
+	INTEL_GM45_IDS(&intel_gm45_info),
+	INTEL_G45_IDS(&intel_g45_info),
+	INTEL_PINEVIEW_IDS(&intel_pineview_info),
+	INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),
+	INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),
+	INTEL_SNB_D_IDS(&intel_sandybridge_d_info),
+	INTEL_SNB_M_IDS(&intel_sandybridge_m_info),
+	INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */
+	INTEL_IVB_M_IDS(&intel_ivybridge_m_info),
+	INTEL_IVB_D_IDS(&intel_ivybridge_d_info),
+	INTEL_HSW_D_IDS(&intel_haswell_d_info),
+	INTEL_HSW_M_IDS(&intel_haswell_m_info),
+	INTEL_VLV_M_IDS(&intel_valleyview_m_info),
+	INTEL_VLV_D_IDS(&intel_valleyview_d_info),
+	INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),
+	INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),
+	INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),
+	INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info),
+	INTEL_CHV_IDS(&intel_cherryview_info),
+	INTEL_SKL_GT1_IDS(&intel_skylake_info),
+	INTEL_SKL_GT2_IDS(&intel_skylake_info),
+	INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),
+	INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info),
+	INTEL_BXT_IDS(&intel_broxton_info),
+	INTEL_KBL_GT1_IDS(&intel_kabylake_info),
+	INTEL_KBL_GT2_IDS(&intel_kabylake_info),
+	INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info),
+	INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info),
 	{0, 0, 0}
 };
 
@@ -463,7 +430,7 @@ static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
 	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 		ret = PCH_LPT;
 		DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
-	} else if (IS_SKYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		ret = PCH_SPT;
 		DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
 	}
@@ -526,12 +493,15 @@ void intel_detect_pch(struct drm_device *dev)
 			} else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
-				WARN_ON(!IS_SKYLAKE(dev));
+				WARN_ON(!IS_SKYLAKE(dev) &&
+					!IS_KABYLAKE(dev));
 			} else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
-				WARN_ON(!IS_SKYLAKE(dev));
-			} else if (id == INTEL_PCH_P2X_DEVICE_ID_TYPE) {
+				WARN_ON(!IS_SKYLAKE(dev) &&
+					!IS_KABYLAKE(dev));
+			} else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
+				   (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE)) {
 				dev_priv->pch_type = intel_virt_detect_pch(dev);
 			} else
 				continue;
@@ -570,47 +540,31 @@ bool i915_semaphore_is_enabled(struct drm_device *dev)
 	return true;
 }
 
-void i915_firmware_load_error_print(const char *fw_path, int err)
-{
-	DRM_ERROR("failed to load firmware %s (%d)\n", fw_path, err);
-
-	/*
-	 * If the reason is not known assume -ENOENT since that's the most
-	 * usual failure mode.
-	 */
-	if (!err)
-		err = -ENOENT;
-
-	if (!(IS_BUILTIN(CONFIG_DRM_I915) && err == -ENOENT))
-		return;
-
-	DRM_ERROR(
-	  "The driver is built-in, so to load the firmware you need to\n"
-	  "include it either in the kernel (see CONFIG_EXTRA_FIRMWARE) or\n"
-	  "in your initrd/initramfs image.\n");
-}
-
 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = dev_priv->dev;
-	struct drm_encoder *encoder;
+	struct intel_encoder *encoder;
 
 	drm_modeset_lock_all(dev);
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
-
-		if (intel_encoder->suspend)
-			intel_encoder->suspend(intel_encoder);
-	}
+	for_each_intel_encoder(dev, encoder)
+		if (encoder->suspend)
+			encoder->suspend(encoder);
 	drm_modeset_unlock_all(dev);
 }
 
 static int intel_suspend_complete(struct drm_i915_private *dev_priv);
 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
 			      bool rpm_resume);
-static int skl_resume_prepare(struct drm_i915_private *dev_priv);
 static int bxt_resume_prepare(struct drm_i915_private *dev_priv);
 
+static bool suspend_to_idle(struct drm_i915_private *dev_priv)
+{
+#if IS_ENABLED(CONFIG_ACPI_SLEEP)
+	if (acpi_target_system_state() < ACPI_STATE_S3)
+		return true;
+#endif
+	return false;
+}
 
 static int i915_drm_suspend(struct drm_device *dev)
 {
@@ -623,6 +577,8 @@ static int i915_drm_suspend(struct drm_device *dev)
 	dev_priv->modeset_restore = MODESET_SUSPENDED;
 	mutex_unlock(&dev_priv->modeset_restore_lock);
 
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	/* We do a lot of poking in a lot of registers, make sure they work
 	 * properly. */
 	intel_display_set_init_power(dev_priv, true);
@@ -635,7 +591,7 @@ static int i915_drm_suspend(struct drm_device *dev)
 	if (error) {
 		dev_err(&dev->pdev->dev,
 			"GEM idle failed, resume might fail\n");
-		return error;
+		goto out;
 	}
 
 	intel_guc_suspend(dev);
@@ -663,11 +619,7 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	i915_save_state(dev);
 
-	opregion_target_state = PCI_D3cold;
-#if IS_ENABLED(CONFIG_ACPI_SLEEP)
-	if (acpi_target_system_state() < ACPI_STATE_S3)
-		opregion_target_state = PCI_D1;
-#endif
+	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
 	intel_opregion_notify_adapter(dev, opregion_target_state);
 
 	intel_uncore_forcewake_reset(dev, false);
@@ -679,20 +631,42 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	intel_display_set_init_power(dev_priv, false);
 
-	return 0;
+	if (HAS_CSR(dev_priv))
+		flush_work(&dev_priv->csr.work);
+
+out:
+	enable_rpm_wakeref_asserts(dev_priv);
+
+	return error;
 }
 
 static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 {
 	struct drm_i915_private *dev_priv = drm_dev->dev_private;
+	bool fw_csr;
 	int ret;
 
+	disable_rpm_wakeref_asserts(dev_priv);
+
+	fw_csr = suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
+	/*
+	 * In case of firmware assisted context save/restore don't manually
+	 * deinit the power domains. This also means the CSR/DMC firmware will
+	 * stay active, it will power down any HW resources as required and
+	 * also enable deeper system power states that would be blocked if the
+	 * firmware was inactive.
+	 */
+	if (!fw_csr)
+		intel_power_domains_suspend(dev_priv);
+
 	ret = intel_suspend_complete(dev_priv);
 
 	if (ret) {
 		DRM_ERROR("Suspend complete failed: %d\n", ret);
+		if (!fw_csr)
+			intel_power_domains_init_hw(dev_priv, true);
 
-		return ret;
+		goto out;
 	}
 
 	pci_disable_device(drm_dev->pdev);
@@ -711,7 +685,12 @@ static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 	if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
 		pci_set_power_state(drm_dev->pdev, PCI_D3hot);
 
-	return 0;
+	dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
+
+out:
+	enable_rpm_wakeref_asserts(dev_priv);
+
+	return ret;
 }
 
 int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
@@ -742,6 +721,8 @@ static int i915_drm_resume(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	mutex_lock(&dev->struct_mutex);
 	i915_gem_restore_gtt_mappings(dev);
 	mutex_unlock(&dev->struct_mutex);
@@ -806,6 +787,8 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	drm_kms_helper_poll_enable(dev);
 
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return 0;
 }
 
@@ -823,12 +806,16 @@ static int i915_drm_resume_early(struct drm_device *dev)
 	 * FIXME: This should be solved with a special hdmi sink device or
 	 * similar so that power domains can be employed.
 	 */
-	if (pci_enable_device(dev->pdev))
-		return -EIO;
+	if (pci_enable_device(dev->pdev)) {
+		ret = -EIO;
+		goto out;
+	}
 
 	pci_set_master(dev->pdev);
 
-	if (IS_VALLEYVIEW(dev_priv))
+	disable_rpm_wakeref_asserts(dev_priv);
+
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		ret = vlv_resume_prepare(dev_priv, false);
 	if (ret)
 		DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
@@ -838,13 +825,18 @@ static int i915_drm_resume_early(struct drm_device *dev)
 
 	if (IS_BROXTON(dev))
 		ret = bxt_resume_prepare(dev_priv);
-	else if (IS_SKYLAKE(dev_priv))
-		ret = skl_resume_prepare(dev_priv);
 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		hsw_disable_pc8(dev_priv);
 
 	intel_uncore_sanitize(dev);
-	intel_power_domains_init_hw(dev_priv);
+
+	if (!(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
+		intel_power_domains_init_hw(dev_priv, true);
+
+out:
+	dev_priv->suspended_to_idle = false;
+
+	enable_rpm_wakeref_asserts(dev_priv);
 
 	return ret;
 }
@@ -1051,15 +1043,6 @@ static int i915_pm_resume(struct device *dev)
 	return i915_drm_resume(drm_dev);
 }
 
-static int skl_suspend_complete(struct drm_i915_private *dev_priv)
-{
-	/* Enabling DC6 is not a hard requirement to enter runtime D3 */
-
-	skl_uninit_cdclk(dev_priv);
-
-	return 0;
-}
-
 static int hsw_suspend_complete(struct drm_i915_private *dev_priv)
 {
 	hsw_enable_pc8(dev_priv);
@@ -1099,16 +1082,6 @@ static int bxt_resume_prepare(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-static int skl_resume_prepare(struct drm_i915_private *dev_priv)
-{
-	struct drm_device *dev = dev_priv->dev;
-
-	skl_init_cdclk(dev_priv);
-	intel_csr_load_program(dev);
-
-	return 0;
-}
-
 /*
  * Save all Gunit registers that may be lost after a D3 and a subsequent
  * S0i[R123] transition. The list of registers needing a save/restore is
@@ -1497,6 +1470,9 @@ static int intel_runtime_suspend(struct device *device)
 
 		return -EAGAIN;
 	}
+
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	/*
 	 * We are safe here against re-faults, since the fault handler takes
 	 * an RPM reference.
@@ -1504,6 +1480,8 @@ static int intel_runtime_suspend(struct device *device)
 	i915_gem_release_all_mmaps(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 
+	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
+
 	intel_guc_suspend(dev);
 
 	intel_suspend_gt_powersave(dev);
@@ -1514,11 +1492,15 @@ static int intel_runtime_suspend(struct device *device)
 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
 		intel_runtime_pm_enable_interrupts(dev_priv);
 
+		enable_rpm_wakeref_asserts(dev_priv);
+
 		return ret;
 	}
 
-	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
 	intel_uncore_forcewake_reset(dev, false);
+
+	enable_rpm_wakeref_asserts(dev_priv);
+	WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
 	dev_priv->pm.suspended = true;
 
 	/*
@@ -1562,6 +1544,9 @@ static int intel_runtime_resume(struct device *device)
 
 	DRM_DEBUG_KMS("Resuming device\n");
 
+	WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	intel_opregion_notify_adapter(dev, PCI_D0);
 	dev_priv->pm.suspended = false;
 
@@ -1572,11 +1557,9 @@ static int intel_runtime_resume(struct device *device)
 
 	if (IS_BROXTON(dev))
 		ret = bxt_resume_prepare(dev_priv);
-	else if (IS_SKYLAKE(dev))
-		ret = skl_resume_prepare(dev_priv);
 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		hsw_disable_pc8(dev_priv);
-	else if (IS_VALLEYVIEW(dev_priv))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		ret = vlv_resume_prepare(dev_priv, true);
 
 	/*
@@ -1593,11 +1576,13 @@ static int intel_runtime_resume(struct device *device)
 	 * power well, so hpd is reinitialized from there. For
 	 * everyone else do it here.
 	 */
-	if (!IS_VALLEYVIEW(dev_priv))
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 		intel_hpd_init(dev_priv);
 
 	intel_enable_gt_powersave(dev);
 
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	if (ret)
 		DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
 	else
@@ -1616,11 +1601,9 @@ static int intel_suspend_complete(struct drm_i915_private *dev_priv)
 
 	if (IS_BROXTON(dev_priv))
 		ret = bxt_suspend_complete(dev_priv);
-	else if (IS_SKYLAKE(dev_priv))
-		ret = skl_suspend_complete(dev_priv);
 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		ret = hsw_suspend_complete(dev_priv);
-	else if (IS_VALLEYVIEW(dev_priv))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		ret = vlv_suspend_complete(dev_priv);
 	else
 		ret = 0;
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index f4af19a0d569..f0f75d7c0d94 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -33,6 +33,7 @@
 #include <uapi/drm/i915_drm.h>
 #include <uapi/drm/drm_fourcc.h>
 
+#include <drm/drmP.h>
 #include "i915_reg.h"
 #include "intel_bios.h"
 #include "intel_ringbuffer.h"
@@ -57,7 +58,7 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20151010"
+#define DRIVER_DATE		"20151218"
 
 #undef WARN_ON
 /* Many gcc seem to no see through this and fall over :( */
@@ -180,15 +181,11 @@ enum intel_display_power_domain {
 	POWER_DOMAIN_TRANSCODER_B,
 	POWER_DOMAIN_TRANSCODER_C,
 	POWER_DOMAIN_TRANSCODER_EDP,
-	POWER_DOMAIN_PORT_DDI_A_2_LANES,
-	POWER_DOMAIN_PORT_DDI_A_4_LANES,
-	POWER_DOMAIN_PORT_DDI_B_2_LANES,
-	POWER_DOMAIN_PORT_DDI_B_4_LANES,
-	POWER_DOMAIN_PORT_DDI_C_2_LANES,
-	POWER_DOMAIN_PORT_DDI_C_4_LANES,
-	POWER_DOMAIN_PORT_DDI_D_2_LANES,
-	POWER_DOMAIN_PORT_DDI_D_4_LANES,
-	POWER_DOMAIN_PORT_DDI_E_2_LANES,
+	POWER_DOMAIN_PORT_DDI_A_LANES,
+	POWER_DOMAIN_PORT_DDI_B_LANES,
+	POWER_DOMAIN_PORT_DDI_C_LANES,
+	POWER_DOMAIN_PORT_DDI_D_LANES,
+	POWER_DOMAIN_PORT_DDI_E_LANES,
 	POWER_DOMAIN_PORT_DSI,
 	POWER_DOMAIN_PORT_CRT,
 	POWER_DOMAIN_PORT_OTHER,
@@ -200,6 +197,7 @@ enum intel_display_power_domain {
 	POWER_DOMAIN_AUX_C,
 	POWER_DOMAIN_AUX_D,
 	POWER_DOMAIN_GMBUS,
+	POWER_DOMAIN_MODESET,
 	POWER_DOMAIN_INIT,
 
 	POWER_DOMAIN_NUM,
@@ -289,7 +287,7 @@ struct i915_hotplug {
 	list_for_each_entry(intel_plane,				\
 			    &(dev)->mode_config.plane_list,		\
 			    base.head)					\
-		if ((intel_plane)->pipe == (intel_crtc)->pipe)
+		for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe)
 
 #define for_each_intel_crtc(dev, intel_crtc) \
 	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head)
@@ -306,15 +304,15 @@ struct i915_hotplug {
 
 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
 	list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
-		if ((intel_encoder)->base.crtc == (__crtc))
+		for_each_if ((intel_encoder)->base.crtc == (__crtc))
 
 #define for_each_connector_on_encoder(dev, __encoder, intel_connector) \
 	list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \
-		if ((intel_connector)->base.encoder == (__encoder))
+		for_each_if ((intel_connector)->base.encoder == (__encoder))
 
 #define for_each_power_domain(domain, mask)				\
 	for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)	\
-		if ((1 << (domain)) & (mask))
+		for_each_if ((1 << (domain)) & (mask))
 
 struct drm_i915_private;
 struct i915_mm_struct;
@@ -460,7 +458,9 @@ struct intel_opregion {
 	u32 swsci_gbda_sub_functions;
 	u32 swsci_sbcb_sub_functions;
 	struct opregion_asle *asle;
-	void *vbt;
+	void *rvda;
+	const void *vbt;
+	u32 vbt_size;
 	u32 *lid_state;
 	struct work_struct asle_work;
 };
@@ -631,11 +631,9 @@ struct drm_i915_display_funcs {
 			  int target, int refclk,
 			  struct dpll *match_clock,
 			  struct dpll *best_clock);
+	int (*compute_pipe_wm)(struct intel_crtc *crtc,
+			       struct drm_atomic_state *state);
 	void (*update_wm)(struct drm_crtc *crtc);
-	void (*update_sprite_wm)(struct drm_plane *plane,
-				 struct drm_crtc *crtc,
-				 uint32_t sprite_width, uint32_t sprite_height,
-				 int pixel_size, bool enable, bool scaled);
 	int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
 	void (*modeset_commit_cdclk)(struct drm_atomic_state *state);
 	/* Returns the active state of the crtc, and if the crtc is active,
@@ -693,18 +691,18 @@ struct intel_uncore_funcs {
 	void (*force_wake_put)(struct drm_i915_private *dev_priv,
 							enum forcewake_domains domains);
 
-	uint8_t  (*mmio_readb)(struct drm_i915_private *dev_priv, off_t offset, bool trace);
-	uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv, off_t offset, bool trace);
-	uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv, off_t offset, bool trace);
-	uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv, off_t offset, bool trace);
+	uint8_t  (*mmio_readb)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace);
+	uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace);
+	uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace);
+	uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace);
 
-	void (*mmio_writeb)(struct drm_i915_private *dev_priv, off_t offset,
+	void (*mmio_writeb)(struct drm_i915_private *dev_priv, i915_reg_t r,
 				uint8_t val, bool trace);
-	void (*mmio_writew)(struct drm_i915_private *dev_priv, off_t offset,
+	void (*mmio_writew)(struct drm_i915_private *dev_priv, i915_reg_t r,
 				uint16_t val, bool trace);
-	void (*mmio_writel)(struct drm_i915_private *dev_priv, off_t offset,
+	void (*mmio_writel)(struct drm_i915_private *dev_priv, i915_reg_t r,
 				uint32_t val, bool trace);
-	void (*mmio_writeq)(struct drm_i915_private *dev_priv, off_t offset,
+	void (*mmio_writeq)(struct drm_i915_private *dev_priv, i915_reg_t r,
 				uint64_t val, bool trace);
 };
 
@@ -721,11 +719,11 @@ struct intel_uncore {
 		enum forcewake_domain_id id;
 		unsigned wake_count;
 		struct timer_list timer;
-		u32 reg_set;
+		i915_reg_t reg_set;
 		u32 val_set;
 		u32 val_clear;
-		u32 reg_ack;
-		u32 reg_post;
+		i915_reg_t reg_ack;
+		i915_reg_t reg_post;
 		u32 val_reset;
 	} fw_domain[FW_DOMAIN_ID_COUNT];
 };
@@ -735,25 +733,24 @@ struct intel_uncore {
 	for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \
 	     (i__) < FW_DOMAIN_ID_COUNT; \
 	     (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \
-		if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))
+		for_each_if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))
 
 #define for_each_fw_domain(domain__, dev_priv__, i__) \
 	for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__)
 
-enum csr_state {
-	FW_UNINITIALIZED = 0,
-	FW_LOADED,
-	FW_FAILED
-};
+#define CSR_VERSION(major, minor)	((major) << 16 | (minor))
+#define CSR_VERSION_MAJOR(version)	((version) >> 16)
+#define CSR_VERSION_MINOR(version)	((version) & 0xffff)
 
 struct intel_csr {
+	struct work_struct work;
 	const char *fw_path;
 	uint32_t *dmc_payload;
 	uint32_t dmc_fw_size;
+	uint32_t version;
 	uint32_t mmio_count;
-	uint32_t mmioaddr[8];
+	i915_reg_t mmioaddr[8];
 	uint32_t mmiodata[8];
-	enum csr_state state;
 };
 
 #define DEV_INFO_FOR_EACH_FLAG(func, sep) \
@@ -769,8 +766,11 @@ struct intel_csr {
 	func(is_crestline) sep \
 	func(is_ivybridge) sep \
 	func(is_valleyview) sep \
+	func(is_cherryview) sep \
 	func(is_haswell) sep \
 	func(is_skylake) sep \
+	func(is_broxton) sep \
+	func(is_kabylake) sep \
 	func(is_preliminary) sep \
 	func(has_fbc) sep \
 	func(has_pipe_cxsr) sep \
@@ -906,7 +906,6 @@ struct i915_fbc {
 	/* This is always the inner lock when overlapping with struct_mutex and
 	 * it's the outer lock when overlapping with stolen_lock. */
 	struct mutex lock;
-	unsigned long uncompressed_size;
 	unsigned threshold;
 	unsigned int fb_id;
 	unsigned int possible_framebuffer_bits;
@@ -919,38 +918,21 @@ struct i915_fbc {
 
 	bool false_color;
 
-	/* Tracks whether the HW is actually enabled, not whether the feature is
-	 * possible. */
 	bool enabled;
+	bool active;
 
 	struct intel_fbc_work {
-		struct delayed_work work;
-		struct intel_crtc *crtc;
+		bool scheduled;
+		struct work_struct work;
 		struct drm_framebuffer *fb;
-	} *fbc_work;
-
-	enum no_fbc_reason {
-		FBC_OK, /* FBC is enabled */
-		FBC_UNSUPPORTED, /* FBC is not supported by this chipset */
-		FBC_NO_OUTPUT, /* no outputs enabled to compress */
-		FBC_STOLEN_TOO_SMALL, /* not enough space for buffers */
-		FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */
-		FBC_MODE_TOO_LARGE, /* mode too large for compression */
-		FBC_BAD_PLANE, /* fbc not supported on plane */
-		FBC_NOT_TILED, /* buffer not tiled */
-		FBC_MULTIPLE_PIPES, /* more than one pipe active */
-		FBC_MODULE_PARAM,
-		FBC_CHIP_DEFAULT, /* disabled by default on this chip */
-		FBC_ROTATION, /* rotation is not supported */
-		FBC_IN_DBG_MASTER, /* kernel debugger is active */
-		FBC_BAD_STRIDE, /* stride is not supported */
-		FBC_PIXEL_RATE, /* pixel rate is too big */
-		FBC_PIXEL_FORMAT /* pixel format is invalid */
-	} no_fbc_reason;
-
-	bool (*fbc_enabled)(struct drm_i915_private *dev_priv);
-	void (*enable_fbc)(struct intel_crtc *crtc);
-	void (*disable_fbc)(struct drm_i915_private *dev_priv);
+		unsigned long enable_jiffies;
+	} work;
+
+	const char *no_fbc_reason;
+
+	bool (*is_active)(struct drm_i915_private *dev_priv);
+	void (*activate)(struct intel_crtc *crtc);
+	void (*deactivate)(struct drm_i915_private *dev_priv);
 };
 
 /**
@@ -1020,7 +1002,7 @@ struct intel_gmbus {
 	struct i2c_adapter adapter;
 	u32 force_bit;
 	u32 reg0;
-	u32 gpio_reg;
+	i915_reg_t gpio_reg;
 	struct i2c_algo_bit_data bit_algo;
 	struct drm_i915_private *dev_priv;
 };
@@ -1623,6 +1605,8 @@ struct skl_wm_level {
  * For more, read the Documentation/power/runtime_pm.txt.
  */
 struct i915_runtime_pm {
+	atomic_t wakeref_count;
+	atomic_t atomic_seq;
 	bool suspended;
 	bool irqs_enabled;
 };
@@ -1669,7 +1653,7 @@ struct i915_frontbuffer_tracking {
 };
 
 struct i915_wa_reg {
-	u32 addr;
+	i915_reg_t addr;
 	u32 value;
 	/* bitmask representing WA bits */
 	u32 mask;
@@ -1698,6 +1682,13 @@ struct i915_execbuffer_params {
 	struct drm_i915_gem_request     *request;
 };
 
+/* used in computing the new watermarks state */
+struct intel_wm_config {
+	unsigned int num_pipes_active;
+	bool sprites_enabled;
+	bool sprites_scaled;
+};
+
 struct drm_i915_private {
 	struct drm_device *dev;
 	struct kmem_cache *objects;
@@ -1718,9 +1709,6 @@ struct drm_i915_private {
 
 	struct intel_csr csr;
 
-	/* Display CSR-related protection */
-	struct mutex csr_lock;
-
 	struct intel_gmbus gmbus[GMBUS_NUM_PINS];
 
 	/** gmbus_mutex protects against concurrent usage of the single hw gmbus
@@ -1735,6 +1723,8 @@ struct drm_i915_private {
 	/* MMIO base address for MIPI regs */
 	uint32_t mipi_mmio_base;
 
+	uint32_t psr_mmio_base;
+
 	wait_queue_head_t gmbus_wait_queue;
 
 	struct pci_dev *bridge_dev;
@@ -1900,6 +1890,7 @@ struct drm_i915_private {
 	u32 chv_phy_control;
 
 	u32 suspend_count;
+	bool suspended_to_idle;
 	struct i915_suspend_saved_registers regfile;
 	struct vlv_s0ix_state vlv_s0ix_state;
 
@@ -1922,6 +1913,9 @@ struct drm_i915_private {
 		 */
 		uint16_t skl_latency[8];
 
+		/* Committed wm config */
+		struct intel_wm_config config;
+
 		/*
 		 * The skl_wm_values structure is a bit too big for stack
 		 * allocation, so we keep the staging struct where we store
@@ -1956,6 +1950,8 @@ struct drm_i915_private {
 	/* perform PHY state sanity checks? */
 	bool chv_phy_assert[2];
 
+	struct intel_encoder *dig_port_map[I915_MAX_PORTS];
+
 	/*
 	 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
 	 * will be rejected. Instead look for a better place.
@@ -1980,7 +1976,7 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 /* Iterate over initialised rings */
 #define for_each_ring(ring__, dev_priv__, i__) \
 	for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \
-		if (((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__)))
+		for_each_if ((((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__))))
 
 enum hdmi_force_audio {
 	HDMI_AUDIO_OFF_DVI = -2,	/* no aux data for HDMI-DVI converter */
@@ -2445,6 +2441,15 @@ struct drm_i915_cmd_table {
 #define INTEL_DEVID(p)	(INTEL_INFO(p)->device_id)
 #define INTEL_REVID(p)	(__I915__(p)->dev->pdev->revision)
 
+#define REVID_FOREVER		0xff
+/*
+ * Return true if revision is in range [since,until] inclusive.
+ *
+ * Use 0 for open-ended since, and REVID_FOREVER for open-ended until.
+ */
+#define IS_REVID(p, since, until) \
+	(INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
+
 #define IS_I830(dev)		(INTEL_DEVID(dev) == 0x3577)
 #define IS_845G(dev)		(INTEL_DEVID(dev) == 0x2562)
 #define IS_I85X(dev)		(INTEL_INFO(dev)->is_i85x)
@@ -2467,11 +2472,12 @@ struct drm_i915_cmd_table {
 				 INTEL_DEVID(dev) == 0x0152 || \
 				 INTEL_DEVID(dev) == 0x015a)
 #define IS_VALLEYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview)
-#define IS_CHERRYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
+#define IS_CHERRYVIEW(dev)	(INTEL_INFO(dev)->is_cherryview)
 #define IS_HASWELL(dev)	(INTEL_INFO(dev)->is_haswell)
-#define IS_BROADWELL(dev)	(!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
+#define IS_BROADWELL(dev)	(!INTEL_INFO(dev)->is_cherryview && IS_GEN8(dev))
 #define IS_SKYLAKE(dev)	(INTEL_INFO(dev)->is_skylake)
-#define IS_BROXTON(dev)	(!INTEL_INFO(dev)->is_skylake && IS_GEN9(dev))
+#define IS_BROXTON(dev)		(INTEL_INFO(dev)->is_broxton)
+#define IS_KABYLAKE(dev)	(INTEL_INFO(dev)->is_kabylake)
 #define IS_MOBILE(dev)		(INTEL_INFO(dev)->is_mobile)
 #define IS_HSW_EARLY_SDV(dev)	(IS_HASWELL(dev) && \
 				 (INTEL_DEVID(dev) & 0xFF00) == 0x0C00)
@@ -2499,6 +2505,14 @@ struct drm_i915_cmd_table {
 #define IS_SKL_ULX(dev)		(INTEL_DEVID(dev) == 0x190E || \
 				 INTEL_DEVID(dev) == 0x1915 || \
 				 INTEL_DEVID(dev) == 0x191E)
+#define IS_KBL_ULT(dev)		(INTEL_DEVID(dev) == 0x5906 || \
+				 INTEL_DEVID(dev) == 0x5913 || \
+				 INTEL_DEVID(dev) == 0x5916 || \
+				 INTEL_DEVID(dev) == 0x5921 || \
+				 INTEL_DEVID(dev) == 0x5926)
+#define IS_KBL_ULX(dev)		(INTEL_DEVID(dev) == 0x590E || \
+				 INTEL_DEVID(dev) == 0x5915 || \
+				 INTEL_DEVID(dev) == 0x591E)
 #define IS_SKL_GT3(dev)		(IS_SKYLAKE(dev) && \
 				 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
 #define IS_SKL_GT4(dev)		(IS_SKYLAKE(dev) && \
@@ -2506,16 +2520,21 @@ struct drm_i915_cmd_table {
 
 #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
 
-#define SKL_REVID_A0		(0x0)
-#define SKL_REVID_B0		(0x1)
-#define SKL_REVID_C0		(0x2)
-#define SKL_REVID_D0		(0x3)
-#define SKL_REVID_E0		(0x4)
-#define SKL_REVID_F0		(0x5)
+#define SKL_REVID_A0		0x0
+#define SKL_REVID_B0		0x1
+#define SKL_REVID_C0		0x2
+#define SKL_REVID_D0		0x3
+#define SKL_REVID_E0		0x4
+#define SKL_REVID_F0		0x5
+
+#define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until))
+
+#define BXT_REVID_A0		0x0
+#define BXT_REVID_A1		0x1
+#define BXT_REVID_B0		0x3
+#define BXT_REVID_C0		0x9
 
-#define BXT_REVID_A0		(0x0)
-#define BXT_REVID_B0		(0x3)
-#define BXT_REVID_C0		(0x9)
+#define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until))
 
 /*
  * The genX designation typically refers to the render engine, so render
@@ -2587,23 +2606,25 @@ struct drm_i915_cmd_table {
 #define HAS_FPGA_DBG_UNCLAIMED(dev)	(INTEL_INFO(dev)->has_fpga_dbg)
 #define HAS_PSR(dev)		(IS_HASWELL(dev) || IS_BROADWELL(dev) || \
 				 IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev) || \
-				 IS_SKYLAKE(dev))
+				 IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 #define HAS_RUNTIME_PM(dev)	(IS_GEN6(dev) || IS_HASWELL(dev) || \
 				 IS_BROADWELL(dev) || IS_VALLEYVIEW(dev) || \
-				 IS_SKYLAKE(dev))
+				 IS_CHERRYVIEW(dev) || IS_SKYLAKE(dev) || \
+				 IS_KABYLAKE(dev))
 #define HAS_RC6(dev)		(INTEL_INFO(dev)->gen >= 6)
 #define HAS_RC6p(dev)		(INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev))
 
 #define HAS_CSR(dev)	(IS_GEN9(dev))
 
-#define HAS_GUC_UCODE(dev)	(IS_GEN9(dev))
-#define HAS_GUC_SCHED(dev)	(IS_GEN9(dev))
+#define HAS_GUC_UCODE(dev)	(IS_GEN9(dev) && !IS_KABYLAKE(dev))
+#define HAS_GUC_SCHED(dev)	(IS_GEN9(dev) && !IS_KABYLAKE(dev))
 
 #define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \
 				    INTEL_INFO(dev)->gen >= 8)
 
 #define HAS_CORE_RING_FREQ(dev)	(INTEL_INFO(dev)->gen >= 6 && \
-				 !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
+				 !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && \
+				 !IS_BROXTON(dev))
 
 #define INTEL_PCH_DEVICE_ID_MASK		0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE		0x3b00
@@ -2614,17 +2635,20 @@ struct drm_i915_cmd_table {
 #define INTEL_PCH_SPT_DEVICE_ID_TYPE		0xA100
 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE		0x9D00
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE		0x7100
+#define INTEL_PCH_QEMU_DEVICE_ID_TYPE		0x2900 /* qemu q35 has 2918 */
 
 #define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type)
 #define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT)
 #define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT)
 #define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
+#define HAS_PCH_LPT_H(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
 #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
 #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
 #define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP)
 #define HAS_PCH_SPLIT(dev) (INTEL_PCH_TYPE(dev) != PCH_NONE)
 
-#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev))
+#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->gen < 5 || \
+			       IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 
 /* DPF == dynamic parity feature */
 #define HAS_L3_DPF(dev) (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
@@ -2650,6 +2674,7 @@ struct i915_params {
 	int panel_use_ssc;
 	int vbt_sdvo_panel_type;
 	int enable_rc6;
+	int enable_dc;
 	int enable_fbc;
 	int enable_ppgtt;
 	int enable_execlists;
@@ -2698,7 +2723,6 @@ extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
-void i915_firmware_load_error_print(const char *fw_path, int err);
 
 /* intel_hotplug.c */
 void intel_hpd_irq_handler(struct drm_device *dev, u32 pin_mask, u32 long_mask);
@@ -2755,17 +2779,47 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv);
 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 				   uint32_t mask,
 				   uint32_t bits);
-void
-ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask);
-void
-ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask);
+void ilk_update_display_irq(struct drm_i915_private *dev_priv,
+			    uint32_t interrupt_mask,
+			    uint32_t enabled_irq_mask);
+static inline void
+ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits)
+{
+	ilk_update_display_irq(dev_priv, bits, bits);
+}
+static inline void
+ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits)
+{
+	ilk_update_display_irq(dev_priv, bits, 0);
+}
+void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
+			 enum pipe pipe,
+			 uint32_t interrupt_mask,
+			 uint32_t enabled_irq_mask);
+static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv,
+				       enum pipe pipe, uint32_t bits)
+{
+	bdw_update_pipe_irq(dev_priv, pipe, bits, bits);
+}
+static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv,
+					enum pipe pipe, uint32_t bits)
+{
+	bdw_update_pipe_irq(dev_priv, pipe, bits, 0);
+}
 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 				  uint32_t interrupt_mask,
 				  uint32_t enabled_irq_mask);
-#define ibx_enable_display_interrupt(dev_priv, bits) \
-	ibx_display_interrupt_update((dev_priv), (bits), (bits))
-#define ibx_disable_display_interrupt(dev_priv, bits) \
-	ibx_display_interrupt_update((dev_priv), (bits), 0)
+static inline void
+ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits)
+{
+	ibx_display_interrupt_update(dev_priv, bits, bits);
+}
+static inline void
+ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits)
+{
+	ibx_display_interrupt_update(dev_priv, bits, 0);
+}
+
 
 /* i915_gem.c */
 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
@@ -2834,6 +2888,7 @@ void i915_gem_vma_destroy(struct i915_vma *vma);
 #define PIN_UPDATE	(1<<5)
 #define PIN_ZONE_4G	(1<<6)
 #define PIN_HIGH	(1<<7)
+#define PIN_OFFSET_FIXED	(1<<8)
 #define PIN_OFFSET_MASK (~4095)
 int __must_check
 i915_gem_object_pin(struct drm_i915_gem_object *obj,
@@ -2869,6 +2924,9 @@ static inline int __sg_page_count(struct scatterlist *sg)
 	return sg->length >> PAGE_SHIFT;
 }
 
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
+
 static inline struct page *
 i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
 {
@@ -3008,8 +3066,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
 int __must_check
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     u32 alignment,
-				     struct intel_engine_cs *pipelined,
-				     struct drm_i915_gem_request **pipelined_request,
 				     const struct i915_ggtt_view *view);
 void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
 					      const struct i915_ggtt_view *view);
@@ -3184,6 +3240,7 @@ int __must_check i915_gem_evict_something(struct drm_device *dev,
 					  unsigned long start,
 					  unsigned long end,
 					  unsigned flags);
+int __must_check i915_gem_evict_for_vma(struct i915_vma *target);
 int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
 
 /* belongs in i915_gem_gtt.h */
@@ -3312,6 +3369,10 @@ static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
 }
 extern void intel_i2c_reset(struct drm_device *dev);
 
+/* intel_bios.c */
+int intel_bios_init(struct drm_i915_private *dev_priv);
+bool intel_bios_is_valid_vbt(const void *buf, size_t size);
+
 /* intel_opregion.c */
 #ifdef CONFIG_ACPI
 extern int intel_opregion_setup(struct drm_device *dev);
@@ -3364,7 +3425,6 @@ extern void intel_set_rps(struct drm_device *dev, u8 val);
 extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
 				  bool enable);
 extern void intel_detect_pch(struct drm_device *dev);
-extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
 extern int intel_enable_rc6(const struct drm_device *dev);
 
 extern bool i915_semaphore_is_enabled(struct drm_device *dev);
@@ -3447,6 +3507,32 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
 #define POSTING_READ(reg)	(void)I915_READ_NOTRACE(reg)
 #define POSTING_READ16(reg)	(void)I915_READ16_NOTRACE(reg)
 
+#define __raw_read(x, s) \
+static inline uint##x##_t __raw_i915_read##x(struct drm_i915_private *dev_priv, \
+					     i915_reg_t reg) \
+{ \
+	return read##s(dev_priv->regs + i915_mmio_reg_offset(reg)); \
+}
+
+#define __raw_write(x, s) \
+static inline void __raw_i915_write##x(struct drm_i915_private *dev_priv, \
+				       i915_reg_t reg, uint##x##_t val) \
+{ \
+	write##s(val, dev_priv->regs + i915_mmio_reg_offset(reg)); \
+}
+__raw_read(8, b)
+__raw_read(16, w)
+__raw_read(32, l)
+__raw_read(64, q)
+
+__raw_write(8, b)
+__raw_write(16, w)
+__raw_write(32, l)
+__raw_write(64, q)
+
+#undef __raw_read
+#undef __raw_write
+
 /* These are untraced mmio-accessors that are only valid to be used inside
  * criticial sections inside IRQ handlers where forcewake is explicitly
  * controlled.
@@ -3454,8 +3540,8 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
  * Note: Should only be used between intel_uncore_forcewake_irqlock() and
  * intel_uncore_forcewake_irqunlock().
  */
-#define I915_READ_FW(reg__) readl(dev_priv->regs + (reg__))
-#define I915_WRITE_FW(reg__, val__) writel(val__, dev_priv->regs + (reg__))
+#define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__))
+#define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__))
 #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__)
 
 /* "Broadcast RGB" property */
@@ -3463,9 +3549,9 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
 #define INTEL_BROADCAST_RGB_FULL 1
 #define INTEL_BROADCAST_RGB_LIMITED 2
 
-static inline uint32_t i915_vgacntrl_reg(struct drm_device *dev)
+static inline i915_reg_t i915_vgacntrl_reg(struct drm_device *dev)
 {
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		return VLV_VGACNTRL;
 	else if (INTEL_INFO(dev)->gen >= 5)
 		return CPU_VGACNTRL;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index f56af0aaafde..ddc21d4b388d 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2797,6 +2797,8 @@ static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
 					struct intel_engine_cs *ring)
 {
+	struct intel_ringbuffer *buffer;
+
 	while (!list_empty(&ring->active_list)) {
 		struct drm_i915_gem_object *obj;
 
@@ -2812,18 +2814,16 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
 	 * are the ones that keep the context and ringbuffer backing objects
 	 * pinned in place.
 	 */
-	while (!list_empty(&ring->execlist_queue)) {
-		struct drm_i915_gem_request *submit_req;
 
-		submit_req = list_first_entry(&ring->execlist_queue,
-				struct drm_i915_gem_request,
-				execlist_link);
-		list_del(&submit_req->execlist_link);
+	if (i915.enable_execlists) {
+		spin_lock_irq(&ring->execlist_lock);
 
-		if (submit_req->ctx != ring->default_context)
-			intel_lr_context_unpin(submit_req);
+		/* list_splice_tail_init checks for empty lists */
+		list_splice_tail_init(&ring->execlist_queue,
+				      &ring->execlist_retired_req_list);
 
-		i915_gem_request_unreference(submit_req);
+		spin_unlock_irq(&ring->execlist_lock);
+		intel_execlists_retire_requests(ring);
 	}
 
 	/*
@@ -2842,6 +2842,18 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
 
 		i915_gem_request_retire(request);
 	}
+
+	/* Having flushed all requests from all queues, we know that all
+	 * ringbuffers must now be empty. However, since we do not reclaim
+	 * all space when retiring the request (to prevent HEADs colliding
+	 * with rapid ringbuffer wraparound) the amount of available space
+	 * upon reset is less than when we start. Do one more pass over
+	 * all the ringbuffers to reset last_retired_head.
+	 */
+	list_for_each_entry(buffer, &ring->buffers, link) {
+		buffer->last_retired_head = buffer->tail;
+		intel_ring_update_space(buffer);
+	}
 }
 
 void i915_gem_reset(struct drm_device *dev)
@@ -2982,6 +2994,10 @@ i915_gem_idle_work_handler(struct work_struct *work)
 		if (!list_empty(&ring->request_list))
 			return;
 
+	/* we probably should sync with hangcheck here, using cancel_work_sync.
+	 * Also locking seems to be fubar here, ring->request_list is protected
+	 * by dev->struct_mutex. */
+
 	intel_mark_idle(dev);
 
 	if (mutex_trylock(&dev->struct_mutex)) {
@@ -3106,7 +3122,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		if (ret == 0)
 			ret = __i915_wait_request(req[i], reset_counter, true,
 						  args->timeout_ns > 0 ? &args->timeout_ns : NULL,
-						  file->driver_priv);
+						  to_rps_client(file));
 		i915_gem_request_unreference__unlocked(req[i]);
 	}
 	return ret;
@@ -3472,7 +3488,7 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
 	if (flags & PIN_MAPPABLE)
 		end = min_t(u64, end, dev_priv->gtt.mappable_end);
 	if (flags & PIN_ZONE_4G)
-		end = min_t(u64, end, (1ULL << 32));
+		end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
 
 	if (alignment == 0)
 		alignment = flags & PIN_MAPPABLE ? fence_alignment :
@@ -3509,30 +3525,50 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
 	if (IS_ERR(vma))
 		goto err_unpin;
 
-	if (flags & PIN_HIGH) {
-		search_flag = DRM_MM_SEARCH_BELOW;
-		alloc_flag = DRM_MM_CREATE_TOP;
+	if (flags & PIN_OFFSET_FIXED) {
+		uint64_t offset = flags & PIN_OFFSET_MASK;
+
+		if (offset & (alignment - 1) || offset + size > end) {
+			ret = -EINVAL;
+			goto err_free_vma;
+		}
+		vma->node.start = offset;
+		vma->node.size = size;
+		vma->node.color = obj->cache_level;
+		ret = drm_mm_reserve_node(&vm->mm, &vma->node);
+		if (ret) {
+			ret = i915_gem_evict_for_vma(vma);
+			if (ret == 0)
+				ret = drm_mm_reserve_node(&vm->mm, &vma->node);
+		}
+		if (ret)
+			goto err_free_vma;
 	} else {
-		search_flag = DRM_MM_SEARCH_DEFAULT;
-		alloc_flag = DRM_MM_CREATE_DEFAULT;
-	}
+		if (flags & PIN_HIGH) {
+			search_flag = DRM_MM_SEARCH_BELOW;
+			alloc_flag = DRM_MM_CREATE_TOP;
+		} else {
+			search_flag = DRM_MM_SEARCH_DEFAULT;
+			alloc_flag = DRM_MM_CREATE_DEFAULT;
+		}
 
 search_free:
-	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
-						  size, alignment,
-						  obj->cache_level,
-						  start, end,
-						  search_flag,
-						  alloc_flag);
-	if (ret) {
-		ret = i915_gem_evict_something(dev, vm, size, alignment,
-					       obj->cache_level,
-					       start, end,
-					       flags);
-		if (ret == 0)
-			goto search_free;
+		ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
+							  size, alignment,
+							  obj->cache_level,
+							  start, end,
+							  search_flag,
+							  alloc_flag);
+		if (ret) {
+			ret = i915_gem_evict_something(dev, vm, size, alignment,
+						       obj->cache_level,
+						       start, end,
+						       flags);
+			if (ret == 0)
+				goto search_free;
 
-		goto err_free_vma;
+			goto err_free_vma;
+		}
 	}
 	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
 		ret = -EINVAL;
@@ -3886,7 +3922,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 		 * cacheline, whereas normally such cachelines would get
 		 * invalidated.
 		 */
-		if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
+		if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 			return -ENODEV;
 
 		level = I915_CACHE_LLC;
@@ -3929,17 +3965,11 @@ rpm_put:
 int
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     u32 alignment,
-				     struct intel_engine_cs *pipelined,
-				     struct drm_i915_gem_request **pipelined_request,
 				     const struct i915_ggtt_view *view)
 {
 	u32 old_read_domains, old_write_domain;
 	int ret;
 
-	ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
-	if (ret)
-		return ret;
-
 	/* Mark the pin_display early so that we account for the
 	 * display coherency whilst setting up the cache domains.
 	 */
@@ -4129,6 +4159,10 @@ i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
 	    vma->node.start < (flags & PIN_OFFSET_MASK))
 		return true;
 
+	if (flags & PIN_OFFSET_FIXED &&
+	    vma->node.start != (flags & PIN_OFFSET_MASK))
+		return true;
+
 	return false;
 }
 
@@ -4541,10 +4575,8 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
 {
 	struct i915_vma *vma;
 	list_for_each_entry(vma, &obj->vma_list, vma_link) {
-		if (i915_is_ggtt(vma->vm) &&
-		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
-			continue;
-		if (vma->vm == vm)
+		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
+		    vma->vm == vm)
 			return vma;
 	}
 	return NULL;
@@ -4633,7 +4665,6 @@ int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
 	struct intel_engine_cs *ring = req->ring;
 	struct drm_device *dev = ring->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
 	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
 	int i, ret;
 
@@ -4649,10 +4680,10 @@ int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
 	 * here because no other code should access these registers other than
 	 * at initialization time.
 	 */
-	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
+	for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(ring, reg_base + i);
-		intel_ring_emit(ring, remap_info[i/4]);
+		intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
+		intel_ring_emit(ring, remap_info[i]);
 	}
 
 	intel_ring_advance(ring);
@@ -4820,18 +4851,9 @@ i915_gem_init_hw(struct drm_device *dev)
 	if (HAS_GUC_UCODE(dev)) {
 		ret = intel_guc_ucode_load(dev);
 		if (ret) {
-			/*
-			 * If we got an error and GuC submission is enabled, map
-			 * the error to -EIO so the GPU will be declared wedged.
-			 * OTOH, if we didn't intend to use the GuC anyway, just
-			 * discard the error and carry on.
-			 */
-			DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
-				  i915.enable_guc_submission ? "" :
-				  " (ignored)");
-			ret = i915.enable_guc_submission ? -EIO : 0;
-			if (ret)
-				goto out;
+			DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
+			ret = -EIO;
+			goto out;
 		}
 	}
 
@@ -4894,14 +4916,6 @@ int i915_gem_init(struct drm_device *dev)
 
 	mutex_lock(&dev->struct_mutex);
 
-	if (IS_VALLEYVIEW(dev)) {
-		/* VLVA0 (potential hack), BIOS isn't actually waking us */
-		I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
-		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
-			      VLV_GTLC_ALLOWWAKEACK), 10))
-			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
-	}
-
 	if (!i915.enable_execlists) {
 		dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
 		dev_priv->gt.init_rings = i915_gem_init_rings;
@@ -5019,7 +5033,7 @@ i915_gem_load(struct drm_device *dev)
 
 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
 
-	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
+	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
 		dev_priv->num_fence_regs = 32;
 	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
 		dev_priv->num_fence_regs = 16;
@@ -5240,6 +5254,21 @@ bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
 	return false;
 }
 
+/* Like i915_gem_object_get_page(), but mark the returned page dirty */
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
+{
+	struct page *page;
+
+	/* Only default objects have per-page dirty tracking */
+	if (WARN_ON(obj->ops != &i915_gem_object_ops))
+		return NULL;
+
+	page = i915_gem_object_get_page(obj, n);
+	set_page_dirty(page);
+	return page;
+}
+
 /* Allocate a new GEM object and fill it with the supplied data */
 struct drm_i915_gem_object *
 i915_gem_object_create_from_data(struct drm_device *dev,
@@ -5265,6 +5294,7 @@ i915_gem_object_create_from_data(struct drm_device *dev,
 	i915_gem_object_pin_pages(obj);
 	sg = obj->pages;
 	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
+	obj->dirty = 1;		/* Backing store is now out of date */
 	i915_gem_object_unpin_pages(obj);
 
 	if (WARN_ON(bytes != size)) {
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 02ceb7a4b481..c25083c78ba7 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -189,8 +189,15 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
 	 * shouldn't touch the cache level, especially as that
 	 * would make the object snooped which might have a
 	 * negative performance impact.
+	 *
+	 * Snooping is required on non-llc platforms in execlist
+	 * mode, but since all GGTT accesses use PAT entry 0 we
+	 * get snooping anyway regardless of cache_level.
+	 *
+	 * This is only applicable for Ivy Bridge devices since
+	 * later platforms don't have L3 control bits in the PTE.
 	 */
-	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) {
+	if (IS_IVYBRIDGE(dev)) {
 		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
 		/* Failure shouldn't ever happen this early */
 		if (WARN_ON(ret)) {
@@ -340,6 +347,10 @@ void i915_gem_context_reset(struct drm_device *dev)
 			i915_gem_context_unreference(lctx);
 			ring->last_context = NULL;
 		}
+
+		/* Force the GPU state to be reinitialised on enabling */
+		if (ring->default_context)
+			ring->default_context->legacy_hw_ctx.initialized = false;
 	}
 }
 
@@ -554,7 +565,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 				if (signaller == ring)
 					continue;
 
-				intel_ring_emit(ring, RING_PSMI_CTL(signaller->mmio_base));
+				intel_ring_emit_reg(ring, RING_PSMI_CTL(signaller->mmio_base));
 				intel_ring_emit(ring, _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 			}
 		}
@@ -579,7 +590,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 				if (signaller == ring)
 					continue;
 
-				intel_ring_emit(ring, RING_PSMI_CTL(signaller->mmio_base));
+				intel_ring_emit_reg(ring, RING_PSMI_CTL(signaller->mmio_base));
 				intel_ring_emit(ring, _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 			}
 		}
@@ -708,7 +719,7 @@ static int do_switch(struct drm_i915_gem_request *req)
 	if (ret)
 		goto unpin_out;
 
-	if (!to->legacy_hw_ctx.initialized) {
+	if (!to->legacy_hw_ctx.initialized || i915_gem_context_is_default(to)) {
 		hw_flags |= MI_RESTORE_INHIBIT;
 		/* NB: If we inhibit the restore, the context is not allowed to
 		 * die because future work may end up depending on valid address
@@ -923,6 +934,14 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
 	case I915_CONTEXT_PARAM_NO_ZEROMAP:
 		args->value = ctx->flags & CONTEXT_NO_ZEROMAP;
 		break;
+	case I915_CONTEXT_PARAM_GTT_SIZE:
+		if (ctx->ppgtt)
+			args->value = ctx->ppgtt->base.total;
+		else if (to_i915(dev)->mm.aliasing_ppgtt)
+			args->value = to_i915(dev)->mm.aliasing_ppgtt->base.total;
+		else
+			args->value = to_i915(dev)->gtt.base.total;
+		break;
 	default:
 		ret = -EINVAL;
 		break;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index d71a133ceff5..07c6e4d320c9 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -199,6 +199,45 @@ found:
 	return ret;
 }
 
+int
+i915_gem_evict_for_vma(struct i915_vma *target)
+{
+	struct drm_mm_node *node, *next;
+
+	list_for_each_entry_safe(node, next,
+			&target->vm->mm.head_node.node_list,
+			node_list) {
+		struct i915_vma *vma;
+		int ret;
+
+		if (node->start + node->size <= target->node.start)
+			continue;
+		if (node->start >= target->node.start + target->node.size)
+			break;
+
+		vma = container_of(node, typeof(*vma), node);
+
+		if (vma->pin_count) {
+			if (!vma->exec_entry || (vma->pin_count > 1))
+				/* Object is pinned for some other use */
+				return -EBUSY;
+
+			/* We need to evict a buffer in the same batch */
+			if (vma->exec_entry->flags & EXEC_OBJECT_PINNED)
+				/* Overlapping fixed objects in the same batch */
+				return -EINVAL;
+
+			return -ENOSPC;
+		}
+
+		ret = i915_vma_unbind(vma);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 /**
  * i915_gem_evict_vm - Evict all idle vmas from a vm
  * @vm: Address space to cleanse
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 6ed7d63a0688..dccb517361b3 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -249,6 +249,31 @@ static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 		obj->cache_level != I915_CACHE_NONE);
 }
 
+/* Used to convert any address to canonical form.
+ * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
+ * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
+ * addresses to be in a canonical form:
+ * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
+ * canonical form [63:48] == [47]."
+ */
+#define GEN8_HIGH_ADDRESS_BIT 47
+static inline uint64_t gen8_canonical_addr(uint64_t address)
+{
+	return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
+}
+
+static inline uint64_t gen8_noncanonical_addr(uint64_t address)
+{
+	return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
+}
+
+static inline uint64_t
+relocation_target(struct drm_i915_gem_relocation_entry *reloc,
+		  uint64_t target_offset)
+{
+	return gen8_canonical_addr((int)reloc->delta + target_offset);
+}
+
 static int
 relocate_entry_cpu(struct drm_i915_gem_object *obj,
 		   struct drm_i915_gem_relocation_entry *reloc,
@@ -256,7 +281,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj,
 {
 	struct drm_device *dev = obj->base.dev;
 	uint32_t page_offset = offset_in_page(reloc->offset);
-	uint64_t delta = reloc->delta + target_offset;
+	uint64_t delta = relocation_target(reloc, target_offset);
 	char *vaddr;
 	int ret;
 
@@ -264,7 +289,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 				reloc->offset >> PAGE_SHIFT));
 	*(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
 
@@ -273,7 +298,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj,
 
 		if (page_offset == 0) {
 			kunmap_atomic(vaddr);
-			vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 		}
 
@@ -292,7 +317,7 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
 {
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint64_t delta = reloc->delta + target_offset;
+	uint64_t delta = relocation_target(reloc, target_offset);
 	uint64_t offset;
 	void __iomem *reloc_page;
 	int ret;
@@ -347,7 +372,7 @@ relocate_entry_clflush(struct drm_i915_gem_object *obj,
 {
 	struct drm_device *dev = obj->base.dev;
 	uint32_t page_offset = offset_in_page(reloc->offset);
-	uint64_t delta = (int)reloc->delta + target_offset;
+	uint64_t delta = relocation_target(reloc, target_offset);
 	char *vaddr;
 	int ret;
 
@@ -355,7 +380,7 @@ relocate_entry_clflush(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 				reloc->offset >> PAGE_SHIFT));
 	clflush_write32(vaddr + page_offset, lower_32_bits(delta));
 
@@ -364,7 +389,7 @@ relocate_entry_clflush(struct drm_i915_gem_object *obj,
 
 		if (page_offset == 0) {
 			kunmap_atomic(vaddr);
-			vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 		}
 
@@ -395,7 +420,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 	target_i915_obj = target_vma->obj;
 	target_obj = &target_vma->obj->base;
 
-	target_offset = target_vma->node.start;
+	target_offset = gen8_canonical_addr(target_vma->node.start);
 
 	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
 	 * pipe_control writes because the gpu doesn't properly redirect them
@@ -599,6 +624,8 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 			flags |= PIN_GLOBAL | PIN_MAPPABLE;
 		if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
 			flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+		if (entry->flags & EXEC_OBJECT_PINNED)
+			flags |= entry->offset | PIN_OFFSET_FIXED;
 		if ((flags & PIN_MAPPABLE) == 0)
 			flags |= PIN_HIGH;
 	}
@@ -670,6 +697,10 @@ eb_vma_misplaced(struct i915_vma *vma)
 	    vma->node.start & (entry->alignment - 1))
 		return true;
 
+	if (entry->flags & EXEC_OBJECT_PINNED &&
+	    vma->node.start != entry->offset)
+		return true;
+
 	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
 	    vma->node.start < BATCH_OFFSET_BIAS)
 		return true;
@@ -695,6 +726,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
 	struct i915_vma *vma;
 	struct i915_address_space *vm;
 	struct list_head ordered_vmas;
+	struct list_head pinned_vmas;
 	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 	int retry;
 
@@ -703,6 +735,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
 	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
 
 	INIT_LIST_HEAD(&ordered_vmas);
+	INIT_LIST_HEAD(&pinned_vmas);
 	while (!list_empty(vmas)) {
 		struct drm_i915_gem_exec_object2 *entry;
 		bool need_fence, need_mappable;
@@ -721,7 +754,9 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
 			obj->tiling_mode != I915_TILING_NONE;
 		need_mappable = need_fence || need_reloc_mappable(vma);
 
-		if (need_mappable) {
+		if (entry->flags & EXEC_OBJECT_PINNED)
+			list_move_tail(&vma->exec_list, &pinned_vmas);
+		else if (need_mappable) {
 			entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
 			list_move(&vma->exec_list, &ordered_vmas);
 		} else
@@ -731,6 +766,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
 		obj->base.pending_write_domain = 0;
 	}
 	list_splice(&ordered_vmas, vmas);
+	list_splice(&pinned_vmas, vmas);
 
 	/* Attempt to pin all of the buffers into the GTT.
 	 * This is done in 3 phases:
@@ -983,6 +1019,21 @@ validate_exec_list(struct drm_device *dev,
 		if (exec[i].flags & invalid_flags)
 			return -EINVAL;
 
+		/* Offset can be used as input (EXEC_OBJECT_PINNED), reject
+		 * any non-page-aligned or non-canonical addresses.
+		 */
+		if (exec[i].flags & EXEC_OBJECT_PINNED) {
+			if (exec[i].offset !=
+			    gen8_canonical_addr(exec[i].offset & PAGE_MASK))
+				return -EINVAL;
+
+			/* From drm_mm perspective address space is continuous,
+			 * so from this point we're always using non-canonical
+			 * form internally.
+			 */
+			exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
+		}
+
 		if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
 			return -EINVAL;
 
@@ -1114,7 +1165,7 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev,
 
 	for (i = 0; i < 4; i++) {
 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
+		intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
 		intel_ring_emit(ring, 0);
 	}
 
@@ -1241,7 +1292,7 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
 
 		intel_ring_emit(ring, MI_NOOP);
 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(ring, INSTPM);
+		intel_ring_emit_reg(ring, INSTPM);
 		intel_ring_emit(ring, instp_mask << 16 | instp_mode);
 		intel_ring_advance(ring);
 
@@ -1317,7 +1368,8 @@ eb_get_batch(struct eb_vmas *eb)
 	 * Note that actual hangs have only been observed on gen7, but for
 	 * paranoia do it everywhere.
 	 */
-	vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
+		vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 
 	return vma->obj;
 }
@@ -1675,6 +1727,8 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
 		/* Copy the new buffer offsets back to the user's exec list. */
 		for (i = 0; i < args->buffer_count; i++) {
+			exec2_list[i].offset =
+				gen8_canonical_addr(exec2_list[i].offset);
 			ret = __copy_to_user(&user_exec_list[i].offset,
 					     &exec2_list[i].offset,
 					     sizeof(user_exec_list[i].offset));
@@ -1740,6 +1794,8 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		int i;
 
 		for (i = 0; i < args->buffer_count; i++) {
+			exec2_list[i].offset =
+				gen8_canonical_addr(exec2_list[i].offset);
 			ret = __copy_to_user(&user_exec_list[i].offset,
 					     &exec2_list[i].offset,
 					     sizeof(user_exec_list[i].offset));
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index f010391b87f5..598198543dcd 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -59,7 +59,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg,
 				 struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int fence_reg_lo, fence_reg_hi;
+	i915_reg_t fence_reg_lo, fence_reg_hi;
 	int fence_pitch_shift;
 
 	if (INTEL_INFO(dev)->gen >= 6) {
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 86c7500454b4..56f4f2e58d53 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -24,6 +24,7 @@
  */
 
 #include <linux/seq_file.h>
+#include <linux/stop_machine.h>
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
@@ -104,9 +105,11 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 {
 	bool has_aliasing_ppgtt;
 	bool has_full_ppgtt;
+	bool has_full_48bit_ppgtt;
 
 	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
 	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
+	has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9;
 
 	if (intel_vgpu_active(dev))
 		has_full_ppgtt = false; /* emulation is too hard */
@@ -125,6 +128,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 	if (enable_ppgtt == 2 && has_full_ppgtt)
 		return 2;
 
+	if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
+		return 3;
+
 #ifdef CONFIG_INTEL_IOMMU
 	/* Disable ppgtt on SNB if VT-d is on. */
 	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
@@ -134,14 +140,13 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 #endif
 
 	/* Early VLV doesn't have this */
-	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
-	    dev->pdev->revision < 0xb) {
+	if (IS_VALLEYVIEW(dev) && dev->pdev->revision < 0xb) {
 		DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 		return 0;
 	}
 
 	if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
-		return 2;
+		return has_full_48bit_ppgtt ? 3 : 2;
 	else
 		return has_aliasing_ppgtt ? 1 : 0;
 }
@@ -661,10 +666,10 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req,
 		return ret;
 
 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
+	intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(ring, entry));
 	intel_ring_emit(ring, upper_32_bits(addr));
 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
+	intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(ring, entry));
 	intel_ring_emit(ring, lower_32_bits(addr));
 	intel_ring_advance(ring);
 
@@ -764,10 +769,10 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 		gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
 					   scratch_pte);
 	} else {
-		uint64_t templ4, pml4e;
+		uint64_t pml4e;
 		struct i915_page_directory_pointer *pdp;
 
-		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
+		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
 			gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
 						   scratch_pte);
 		}
@@ -833,10 +838,10 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 					      cache_level);
 	} else {
 		struct i915_page_directory_pointer *pdp;
-		uint64_t templ4, pml4e;
+		uint64_t pml4e;
 		uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
 
-		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
+		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
 			gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
 						      start, cache_level);
 		}
@@ -904,14 +909,13 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 	enum vgt_g2v_type msg;
 	struct drm_device *dev = ppgtt->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	unsigned int offset = vgtif_reg(pdp0_lo);
 	int i;
 
 	if (USES_FULL_48BIT_PPGTT(dev)) {
 		u64 daddr = px_dma(&ppgtt->pml4);
 
-		I915_WRITE(offset, lower_32_bits(daddr));
-		I915_WRITE(offset + 4, upper_32_bits(daddr));
+		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
+		I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
 
 		msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
 				VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
@@ -919,10 +923,8 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 		for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
 			u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
 
-			I915_WRITE(offset, lower_32_bits(daddr));
-			I915_WRITE(offset + 4, upper_32_bits(daddr));
-
-			offset += 8;
+			I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
+			I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
 		}
 
 		msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
@@ -1017,10 +1019,9 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
 {
 	struct drm_device *dev = vm->dev;
 	struct i915_page_table *pt;
-	uint64_t temp;
 	uint32_t pde;
 
-	gen8_for_each_pde(pt, pd, start, length, temp, pde) {
+	gen8_for_each_pde(pt, pd, start, length, pde) {
 		/* Don't reallocate page tables */
 		if (test_bit(pde, pd->used_pdes)) {
 			/* Scratch is never allocated this way */
@@ -1079,13 +1080,12 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
 {
 	struct drm_device *dev = vm->dev;
 	struct i915_page_directory *pd;
-	uint64_t temp;
 	uint32_t pdpe;
 	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
 
 	WARN_ON(!bitmap_empty(new_pds, pdpes));
 
-	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		if (test_bit(pdpe, pdp->used_pdpes))
 			continue;
 
@@ -1133,12 +1133,11 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
 {
 	struct drm_device *dev = vm->dev;
 	struct i915_page_directory_pointer *pdp;
-	uint64_t temp;
 	uint32_t pml4e;
 
 	WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
 
-	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 		if (!test_bit(pml4e, pml4->used_pml4es)) {
 			pdp = alloc_pdp(dev);
 			if (IS_ERR(pdp))
@@ -1222,7 +1221,6 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 	struct i915_page_directory *pd;
 	const uint64_t orig_start = start;
 	const uint64_t orig_length = length;
-	uint64_t temp;
 	uint32_t pdpe;
 	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
 	int ret;
@@ -1249,7 +1247,7 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 	}
 
 	/* For every page directory referenced, allocate page tables */
-	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
 						new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
 		if (ret)
@@ -1261,7 +1259,7 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 
 	/* Allocations have completed successfully, so set the bitmaps, and do
 	 * the mappings. */
-	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		gen8_pde_t *const page_directory = kmap_px(pd);
 		struct i915_page_table *pt;
 		uint64_t pd_len = length;
@@ -1271,7 +1269,7 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 		/* Every pd should be allocated, we just did that above. */
 		WARN_ON(!pd);
 
-		gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
+		gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
 			/* Same reasoning as pd */
 			WARN_ON(!pt);
 			WARN_ON(!pd_len);
@@ -1308,6 +1306,8 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 
 err_out:
 	while (pdpe--) {
+		unsigned long temp;
+
 		for_each_set_bit(temp, new_page_tables + pdpe *
 				BITS_TO_LONGS(I915_PDES), I915_PDES)
 			free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
@@ -1330,7 +1330,7 @@ static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 			container_of(vm, struct i915_hw_ppgtt, base);
 	struct i915_page_directory_pointer *pdp;
-	uint64_t temp, pml4e;
+	uint64_t pml4e;
 	int ret = 0;
 
 	/* Do the pml4 allocations first, so we don't need to track the newly
@@ -1349,7 +1349,7 @@ static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
 	     "The allocation has spanned more than 512GB. "
 	     "It is highly likely this is incorrect.");
 
-	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 		WARN_ON(!pdp);
 
 		ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
@@ -1389,10 +1389,9 @@ static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
 			  struct seq_file *m)
 {
 	struct i915_page_directory *pd;
-	uint64_t temp;
 	uint32_t pdpe;
 
-	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
+	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		struct i915_page_table *pt;
 		uint64_t pd_len = length;
 		uint64_t pd_start = start;
@@ -1402,7 +1401,7 @@ static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
 			continue;
 
 		seq_printf(m, "\tPDPE #%d\n", pdpe);
-		gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
+		gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
 			uint32_t  pte;
 			gen8_pte_t *pt_vaddr;
 
@@ -1452,11 +1451,11 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
 		gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
 	} else {
-		uint64_t templ4, pml4e;
+		uint64_t pml4e;
 		struct i915_pml4 *pml4 = &ppgtt->pml4;
 		struct i915_page_directory_pointer *pdp;
 
-		gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) {
+		gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 			if (!test_bit(pml4e, pml4->used_pml4es))
 				continue;
 
@@ -1662,9 +1661,9 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 		return ret;
 
 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
-	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
+	intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring));
 	intel_ring_emit(ring, PP_DIR_DCLV_2G);
-	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
+	intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring));
 	intel_ring_emit(ring, get_pd_offset(ppgtt));
 	intel_ring_emit(ring, MI_NOOP);
 	intel_ring_advance(ring);
@@ -1699,9 +1698,9 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 		return ret;
 
 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
-	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
+	intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring));
 	intel_ring_emit(ring, PP_DIR_DCLV_2G);
-	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
+	intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring));
 	intel_ring_emit(ring, get_pd_offset(ppgtt));
 	intel_ring_emit(ring, MI_NOOP);
 	intel_ring_advance(ring);
@@ -2352,6 +2351,9 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	int i = 0;
 	struct sg_page_iter sg_iter;
 	dma_addr_t addr = 0; /* shut up gcc */
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
 		addr = sg_dma_address(sg_iter.sg) +
@@ -2378,6 +2380,34 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
+
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+}
+
+struct insert_entries {
+	struct i915_address_space *vm;
+	struct sg_table *st;
+	uint64_t start;
+	enum i915_cache_level level;
+	u32 flags;
+};
+
+static int gen8_ggtt_insert_entries__cb(void *_arg)
+{
+	struct insert_entries *arg = _arg;
+	gen8_ggtt_insert_entries(arg->vm, arg->st,
+				 arg->start, arg->level, arg->flags);
+	return 0;
+}
+
+static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
+					  struct sg_table *st,
+					  uint64_t start,
+					  enum i915_cache_level level,
+					  u32 flags)
+{
+	struct insert_entries arg = { vm, st, start, level, flags };
+	stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
 }
 
 /*
@@ -2398,6 +2428,9 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 	int i = 0;
 	struct sg_page_iter sg_iter;
 	dma_addr_t addr = 0;
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
 		addr = sg_page_iter_dma_address(&sg_iter);
@@ -2422,6 +2455,8 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
+
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
@@ -2436,6 +2471,9 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 		(gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
 	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
 	int i;
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	if (WARN(num_entries > max_entries,
 		 "First entry = %d; Num entries = %d (max=%d)\n",
@@ -2448,6 +2486,8 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 	for (i = 0; i < num_entries; i++)
 		gen8_set_pte(&gtt_base[i], scratch_pte);
 	readl(gtt_base);
+
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
@@ -2462,6 +2502,9 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 		(gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
 	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
 	int i;
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	if (WARN(num_entries > max_entries,
 		 "First entry = %d; Num entries = %d (max=%d)\n",
@@ -2474,6 +2517,8 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 	for (i = 0; i < num_entries; i++)
 		iowrite32(scratch_pte, &gtt_base[i]);
 	readl(gtt_base);
+
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
@@ -2481,11 +2526,17 @@ static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 				     uint64_t start,
 				     enum i915_cache_level cache_level, u32 unused)
 {
+	struct drm_i915_private *dev_priv = vm->dev->dev_private;
 	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
 
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+
 }
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,
@@ -2493,9 +2544,16 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm,
 				  uint64_t length,
 				  bool unused)
 {
+	struct drm_i915_private *dev_priv = vm->dev->dev_private;
 	unsigned first_entry = start >> PAGE_SHIFT;
 	unsigned num_entries = length >> PAGE_SHIFT;
+	int rpm_atomic_seq;
+
+	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
+
 	intel_gtt_clear_range(first_entry, num_entries);
+
+	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static int ggtt_bind_vma(struct i915_vma *vma,
@@ -2996,6 +3054,9 @@ static int gen8_gmch_probe(struct drm_device *dev,
 	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
 	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
 
+	if (IS_CHERRYVIEW(dev_priv))
+		dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries__BKL;
+
 	return ret;
 }
 
@@ -3303,7 +3364,7 @@ static struct sg_table *
 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
 			  struct drm_i915_gem_object *obj)
 {
-	struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
+	struct intel_rotation_info *rot_info = &ggtt_view->params.rotation_info;
 	unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
 	unsigned int size_pages_uv;
 	struct sg_page_iter sg_iter;
@@ -3535,7 +3596,7 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj,
 	if (view->type == I915_GGTT_VIEW_NORMAL) {
 		return obj->base.size;
 	} else if (view->type == I915_GGTT_VIEW_ROTATED) {
-		return view->rotation_info.size;
+		return view->params.rotation_info.size;
 	} else if (view->type == I915_GGTT_VIEW_PARTIAL) {
 		return view->params.partial.size << PAGE_SHIFT;
 	} else {
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index a216397ead52..b448ad832dcf 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -156,13 +156,10 @@ struct i915_ggtt_view {
 			u64 offset;
 			unsigned int size;
 		} partial;
+		struct intel_rotation_info rotation_info;
 	} params;
 
 	struct sg_table *pages;
-
-	union {
-		struct intel_rotation_info rotation_info;
-	};
 };
 
 extern const struct i915_ggtt_view i915_ggtt_view_normal;
@@ -458,32 +455,29 @@ static inline uint32_t gen6_pde_index(uint32_t addr)
  * between from start until start + length. On gen8+ it simply iterates
  * over every page directory entry in a page directory.
  */
-#define gen8_for_each_pde(pt, pd, start, length, temp, iter)		\
-	for (iter = gen8_pde_index(start); \
-	     length > 0 && iter < I915_PDES ? \
-			(pt = (pd)->page_table[iter]), 1 : 0; \
-	     iter++,				\
-	     temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT) - start,	\
-	     temp = min(temp, length),					\
-	     start += temp, length -= temp)
-
-#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)	\
-	for (iter = gen8_pdpe_index(start); \
-	     length > 0 && (iter < I915_PDPES_PER_PDP(dev)) ? \
-			(pd = (pdp)->page_directory[iter]), 1 : 0; \
-	     iter++,				\
-	     temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start,	\
-	     temp = min(temp, length),					\
-	     start += temp, length -= temp)
-
-#define gen8_for_each_pml4e(pdp, pml4, start, length, temp, iter)	\
-	for (iter = gen8_pml4e_index(start);	\
-	     length > 0 && iter < GEN8_PML4ES_PER_PML4 ? \
-			(pdp = (pml4)->pdps[iter]), 1 : 0; \
-	     iter++,				\
-	     temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT) - start,	\
-	     temp = min(temp, length),					\
-	     start += temp, length -= temp)
+#define gen8_for_each_pde(pt, pd, start, length, iter)			\
+	for (iter = gen8_pde_index(start);				\
+	     length > 0 && iter < I915_PDES &&				\
+		(pt = (pd)->page_table[iter], true);			\
+	     ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT);		\
+		    temp = min(temp - start, length);			\
+		    start += temp, length -= temp; }), ++iter)
+
+#define gen8_for_each_pdpe(pd, pdp, start, length, iter)		\
+	for (iter = gen8_pdpe_index(start);				\
+	     length > 0 && iter < I915_PDPES_PER_PDP(dev) &&		\
+		(pd = (pdp)->page_directory[iter], true);		\
+	     ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT);	\
+		    temp = min(temp - start, length);			\
+		    start += temp, length -= temp; }), ++iter)
+
+#define gen8_for_each_pml4e(pdp, pml4, start, length, iter)		\
+	for (iter = gen8_pml4e_index(start);				\
+	     length > 0 && iter < GEN8_PML4ES_PER_PML4 &&		\
+		(pdp = (pml4)->pdps[iter], true);			\
+	     ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT);	\
+		    temp = min(temp - start, length);			\
+		    start += temp, length -= temp; }), ++iter)
 
 static inline uint32_t gen8_pte_index(uint64_t address)
 {
@@ -556,7 +550,7 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a,
 
 	if (a->type != b->type)
 		return false;
-	if (a->type == I915_GGTT_VIEW_PARTIAL)
+	if (a->type != I915_GGTT_VIEW_NORMAL)
 		return !memcmp(&a->params, &b->params, sizeof(a->params));
 	return true;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 5026a6267a88..fc7e6d5c6251 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -103,7 +103,7 @@ static int render_state_setup(struct render_state *so)
 	if (ret)
 		return ret;
 
-	page = sg_page(so->obj->pages->sgl);
+	page = i915_gem_object_get_dirty_page(so->obj, 0);
 	d = kmap(page);
 
 	while (i < rodata->batch_items) {
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 87e919a06b27..3476877fc0d6 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -433,7 +433,8 @@ int i915_gem_init_stolen(struct drm_device *dev)
 					 &reserved_size);
 		break;
 	default:
-		if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+		if (IS_BROADWELL(dev_priv) ||
+		    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev))
 			bdw_get_stolen_reserved(dev_priv, &reserved_base,
 						&reserved_size);
 		else
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 8a6717cc265c..7410f6c962e7 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -176,6 +176,8 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		return -EINVAL;
 	}
 
+	intel_runtime_pm_get(dev_priv);
+
 	mutex_lock(&dev->struct_mutex);
 	if (obj->pin_display || obj->framebuffer_references) {
 		ret = -EBUSY;
@@ -269,6 +271,8 @@ err:
 	drm_gem_object_unreference(&obj->base);
 	mutex_unlock(&dev->struct_mutex);
 
+	intel_runtime_pm_put(dev_priv);
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 2f04e4f2ff35..06ca4082735b 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -366,6 +366,17 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	err_printf(m, "Suspend count: %u\n", error->suspend_count);
 	err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device);
 	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
+
+	if (HAS_CSR(dev)) {
+		struct intel_csr *csr = &dev_priv->csr;
+
+		err_printf(m, "DMC loaded: %s\n",
+			   yesno(csr->dmc_payload != NULL));
+		err_printf(m, "DMC fw version: %d.%d\n",
+			   CSR_VERSION_MAJOR(csr->version),
+			   CSR_VERSION_MINOR(csr->version));
+	}
+
 	err_printf(m, "EIR: 0x%08x\n", error->eir);
 	err_printf(m, "IER: 0x%08x\n", error->ier);
 	if (INTEL_INFO(dev)->gen >= 8) {
@@ -862,7 +873,7 @@ static void i915_record_ring_state(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	if (INTEL_INFO(dev)->gen >= 6) {
-		ering->rc_psmi = I915_READ(ring->mmio_base + 0x50);
+		ering->rc_psmi = I915_READ(RING_PSMI_CTL(ring->mmio_base));
 		ering->fault_reg = I915_READ(RING_FAULT_REG(ring));
 		if (INTEL_INFO(dev)->gen >= 8)
 			gen8_record_semaphore_state(dev_priv, error, ring, ering);
@@ -899,7 +910,7 @@ static void i915_record_ring_state(struct drm_device *dev,
 	ering->ctl = I915_READ_CTL(ring);
 
 	if (I915_NEED_GFX_HWS(dev)) {
-		int mmio;
+		i915_reg_t mmio;
 
 		if (IS_GEN7(dev)) {
 			switch (ring->id) {
@@ -1071,6 +1082,25 @@ static void i915_gem_record_rings(struct drm_device *dev,
 		list_for_each_entry(request, &ring->request_list, list) {
 			struct drm_i915_error_request *erq;
 
+			if (count >= error->ring[i].num_requests) {
+				/*
+				 * If the ring request list was changed in
+				 * between the point where the error request
+				 * list was created and dimensioned and this
+				 * point then just exit early to avoid crashes.
+				 *
+				 * We don't need to communicate that the
+				 * request list changed state during error
+				 * state capture and that the error state is
+				 * slightly incorrect as a consequence since we
+				 * are typically only interested in the request
+				 * list state at the point of error state
+				 * capture, not in any changes happening during
+				 * the capture.
+				 */
+				break;
+			}
+
 			erq = &error->ring[i].requests[count++];
 			erq->seqno = request->seqno;
 			erq->jiffies = request->emitted_jiffies;
@@ -1181,7 +1211,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 	if (IS_VALLEYVIEW(dev)) {
 		error->gtier[0] = I915_READ(GTIER);
 		error->ier = I915_READ(VLV_IER);
-		error->forcewake = I915_READ(FORCEWAKE_VLV);
+		error->forcewake = I915_READ_FW(FORCEWAKE_VLV);
 	}
 
 	if (IS_GEN7(dev))
@@ -1193,14 +1223,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 	}
 
 	if (IS_GEN6(dev)) {
-		error->forcewake = I915_READ(FORCEWAKE);
+		error->forcewake = I915_READ_FW(FORCEWAKE);
 		error->gab_ctl = I915_READ(GAB_CTL);
 		error->gfx_mode = I915_READ(GFX_MODE);
 	}
 
 	/* 2: Registers which belong to multiple generations */
 	if (INTEL_INFO(dev)->gen >= 7)
-		error->forcewake = I915_READ(FORCEWAKE_MT);
+		error->forcewake = I915_READ_FW(FORCEWAKE_MT);
 
 	if (INTEL_INFO(dev)->gen >= 6) {
 		error->derrmr = I915_READ(DERRMR);
diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h
index c4cb1c0c4d0d..685c7991e24f 100644
--- a/drivers/gpu/drm/i915/i915_guc_reg.h
+++ b/drivers/gpu/drm/i915/i915_guc_reg.h
@@ -26,7 +26,7 @@
 
 /* Definitions of GuC H/W registers, bits, etc */
 
-#define GUC_STATUS			0xc000
+#define GUC_STATUS			_MMIO(0xc000)
 #define   GS_BOOTROM_SHIFT		1
 #define   GS_BOOTROM_MASK		  (0x7F << GS_BOOTROM_SHIFT)
 #define   GS_BOOTROM_RSA_FAILED		  (0x50 << GS_BOOTROM_SHIFT)
@@ -39,40 +39,41 @@
 #define   GS_MIA_MASK			  (0x07 << GS_MIA_SHIFT)
 #define   GS_MIA_CORE_STATE		  (1 << GS_MIA_SHIFT)
 
-#define SOFT_SCRATCH(n)			(0xc180 + ((n) * 4))
+#define SOFT_SCRATCH(n)			_MMIO(0xc180 + (n) * 4)
 
-#define UOS_RSA_SCRATCH(i)		(0xc200 + (i) * 4)
-#define DMA_ADDR_0_LOW			0xc300
-#define DMA_ADDR_0_HIGH			0xc304
-#define DMA_ADDR_1_LOW			0xc308
-#define DMA_ADDR_1_HIGH			0xc30c
+#define UOS_RSA_SCRATCH(i)		_MMIO(0xc200 + (i) * 4)
+#define   UOS_RSA_SCRATCH_MAX_COUNT	  64
+#define DMA_ADDR_0_LOW			_MMIO(0xc300)
+#define DMA_ADDR_0_HIGH			_MMIO(0xc304)
+#define DMA_ADDR_1_LOW			_MMIO(0xc308)
+#define DMA_ADDR_1_HIGH			_MMIO(0xc30c)
 #define   DMA_ADDRESS_SPACE_WOPCM	  (7 << 16)
 #define   DMA_ADDRESS_SPACE_GTT		  (8 << 16)
-#define DMA_COPY_SIZE			0xc310
-#define DMA_CTRL			0xc314
+#define DMA_COPY_SIZE			_MMIO(0xc310)
+#define DMA_CTRL			_MMIO(0xc314)
 #define   UOS_MOVE			  (1<<4)
 #define   START_DMA			  (1<<0)
-#define DMA_GUC_WOPCM_OFFSET		0xc340
+#define DMA_GUC_WOPCM_OFFSET		_MMIO(0xc340)
 #define   GUC_WOPCM_OFFSET_VALUE	  0x80000	/* 512KB */
-#define GUC_MAX_IDLE_COUNT		0xC3E4
+#define GUC_MAX_IDLE_COUNT		_MMIO(0xC3E4)
 
-#define GUC_WOPCM_SIZE			0xc050
+#define GUC_WOPCM_SIZE			_MMIO(0xc050)
 #define   GUC_WOPCM_SIZE_VALUE  	  (0x80 << 12)	/* 512KB */
 
 /* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */
 #define	GUC_WOPCM_TOP			(GUC_WOPCM_SIZE_VALUE)
 
-#define GEN8_GT_PM_CONFIG		0x138140
-#define GEN9LP_GT_PM_CONFIG		0x138140
-#define GEN9_GT_PM_CONFIG		0x13816c
+#define GEN8_GT_PM_CONFIG		_MMIO(0x138140)
+#define GEN9LP_GT_PM_CONFIG		_MMIO(0x138140)
+#define GEN9_GT_PM_CONFIG		_MMIO(0x13816c)
 #define   GT_DOORBELL_ENABLE		  (1<<0)
 
-#define GEN8_GTCR			0x4274
+#define GEN8_GTCR			_MMIO(0x4274)
 #define   GEN8_GTCR_INVALIDATE		  (1<<0)
 
-#define GUC_ARAT_C6DIS			0xA178
+#define GUC_ARAT_C6DIS			_MMIO(0xA178)
 
-#define GUC_SHIM_CONTROL		0xc064
+#define GUC_SHIM_CONTROL		_MMIO(0xc064)
 #define   GUC_DISABLE_SRAM_INIT_TO_ZEROES	(1<<0)
 #define   GUC_ENABLE_READ_CACHE_LOGIC		(1<<1)
 #define   GUC_ENABLE_MIA_CACHING		(1<<2)
@@ -89,21 +90,21 @@
 				 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA	| \
 				 GUC_ENABLE_MIA_CLOCK_GATING)
 
-#define HOST2GUC_INTERRUPT		0xc4c8
+#define HOST2GUC_INTERRUPT		_MMIO(0xc4c8)
 #define   HOST2GUC_TRIGGER		  (1<<0)
 
 #define DRBMISC1			0x1984
 #define   DOORBELL_ENABLE		  (1<<0)
 
-#define GEN8_DRBREGL(x)			(0x1000 + (x) * 8)
+#define GEN8_DRBREGL(x)			_MMIO(0x1000 + (x) * 8)
 #define   GEN8_DRB_VALID		  (1<<0)
-#define GEN8_DRBREGU(x)			(GEN8_DRBREGL(x) + 4)
+#define GEN8_DRBREGU(x)			_MMIO(0x1000 + (x) * 8 + 4)
 
-#define DE_GUCRMR			0x44054
+#define DE_GUCRMR			_MMIO(0x44054)
 
-#define GUC_BCS_RCS_IER			0xC550
-#define GUC_VCS2_VCS1_IER		0xC554
-#define GUC_WD_VECS_IER			0xC558
-#define GUC_PM_P24C_IER			0xC55C
+#define GUC_BCS_RCS_IER			_MMIO(0xC550)
+#define GUC_VCS2_VCS1_IER		_MMIO(0xC554)
+#define GUC_WD_VECS_IER			_MMIO(0xC558)
+#define GUC_PM_P24C_IER			_MMIO(0xC55C)
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index 036b42bae827..05aa7e61cbe0 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -27,7 +27,7 @@
 #include "intel_guc.h"
 
 /**
- * DOC: GuC Client
+ * DOC: GuC-based command submission
  *
  * i915_guc_client:
  * We use the term client to avoid confusion with contexts. A i915_guc_client is
@@ -86,7 +86,6 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 		return -EINVAL;
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-	spin_lock(&dev_priv->guc.host2guc_lock);
 
 	dev_priv->guc.action_count += 1;
 	dev_priv->guc.action_cmd = data[0];
@@ -119,7 +118,6 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 	}
 	dev_priv->guc.action_status = status;
 
-	spin_unlock(&dev_priv->guc.host2guc_lock);
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
 	return ret;
@@ -161,9 +159,9 @@ static int host2guc_sample_forcewake(struct intel_guc *guc,
 	data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE;
 	/* WaRsDisableCoarsePowerGating:skl,bxt */
 	if (!intel_enable_rc6(dev_priv->dev) ||
-	    (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) ||
-	    (IS_SKL_GT3(dev) && (INTEL_REVID(dev) <= SKL_REVID_E0)) ||
-	    (IS_SKL_GT4(dev) && (INTEL_REVID(dev) <= SKL_REVID_E0)))
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1) ||
+	    (IS_SKL_GT3(dev) && IS_SKL_REVID(dev, 0, SKL_REVID_E0)) ||
+	    (IS_SKL_GT4(dev) && IS_SKL_REVID(dev, 0, SKL_REVID_E0)))
 		data[1] = 0;
 	else
 		/* bit 0 and 1 are for Render and Media domain separately */
@@ -258,7 +256,7 @@ static void guc_disable_doorbell(struct intel_guc *guc,
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct guc_doorbell_info *doorbell;
 	void *base;
-	int drbreg = GEN8_DRBREGL(client->doorbell_id);
+	i915_reg_t drbreg = GEN8_DRBREGL(client->doorbell_id);
 	int value;
 
 	base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0));
@@ -292,16 +290,12 @@ static uint32_t select_doorbell_cacheline(struct intel_guc *guc)
 	const uint32_t cacheline_size = cache_line_size();
 	uint32_t offset;
 
-	spin_lock(&guc->host2guc_lock);
-
 	/* Doorbell uses a single cache line within a page */
 	offset = offset_in_page(guc->db_cacheline);
 
 	/* Moving to next cache line to reduce contention */
 	guc->db_cacheline += cacheline_size;
 
-	spin_unlock(&guc->host2guc_lock);
-
 	DRM_DEBUG_DRIVER("selected doorbell cacheline 0x%x, next 0x%x, linesize %u\n",
 			offset, guc->db_cacheline, cacheline_size);
 
@@ -322,13 +316,11 @@ static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority)
 	const uint16_t end = start + half;
 	uint16_t id;
 
-	spin_lock(&guc->host2guc_lock);
 	id = find_next_zero_bit(guc->doorbell_bitmap, end, start);
 	if (id == end)
 		id = GUC_INVALID_DOORBELL_ID;
 	else
 		bitmap_set(guc->doorbell_bitmap, id, 1);
-	spin_unlock(&guc->host2guc_lock);
 
 	DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n",
 			hi_pri ? "high" : "normal", id);
@@ -338,9 +330,7 @@ static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority)
 
 static void release_doorbell(struct intel_guc *guc, uint16_t id)
 {
-	spin_lock(&guc->host2guc_lock);
 	bitmap_clear(guc->doorbell_bitmap, id, 1);
-	spin_unlock(&guc->host2guc_lock);
 }
 
 /*
@@ -487,16 +477,13 @@ static int guc_get_workqueue_space(struct i915_guc_client *gc, u32 *offset)
 	struct guc_process_desc *desc;
 	void *base;
 	u32 size = sizeof(struct guc_wq_item);
-	int ret = 0, timeout_counter = 200;
+	int ret = -ETIMEDOUT, timeout_counter = 200;
 
 	base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 0));
 	desc = base + gc->proc_desc_offset;
 
 	while (timeout_counter-- > 0) {
-		ret = wait_for_atomic(CIRC_SPACE(gc->wq_tail, desc->head,
-				gc->wq_size) >= size, 1);
-
-		if (!ret) {
+		if (CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size) >= size) {
 			*offset = gc->wq_tail;
 
 			/* advance the tail for next workqueue item */
@@ -505,7 +492,11 @@ static int guc_get_workqueue_space(struct i915_guc_client *gc, u32 *offset)
 
 			/* this will break the loop */
 			timeout_counter = 0;
+			ret = 0;
 		}
+
+		if (timeout_counter)
+			usleep_range(1000, 2000);
 	};
 
 	kunmap_atomic(base);
@@ -577,7 +568,7 @@ static void lr_context_update(struct drm_i915_gem_request *rq)
 	WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
 	WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
 
-	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
 	reg_state = kmap_atomic(page);
 
 	reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj);
@@ -588,8 +579,7 @@ static void lr_context_update(struct drm_i915_gem_request *rq)
 /**
  * i915_guc_submit() - Submit commands through GuC
  * @client:	the guc client where commands will go through
- * @ctx:	LRC where commands come from
- * @ring:	HW engine that will excute the commands
+ * @rq:		request associated with the commands
  *
  * Return:	0 if succeed
  */
@@ -598,15 +588,12 @@ int i915_guc_submit(struct i915_guc_client *client,
 {
 	struct intel_guc *guc = client->guc;
 	enum intel_ring_id ring_id = rq->ring->id;
-	unsigned long flags;
 	int q_ret, b_ret;
 
 	/* Need this because of the deferred pin ctx and ring */
 	/* Shall we move this right after ring is pinned? */
 	lr_context_update(rq);
 
-	spin_lock_irqsave(&client->wq_lock, flags);
-
 	q_ret = guc_add_workqueue_item(client, rq);
 	if (q_ret == 0)
 		b_ret = guc_ring_doorbell(client);
@@ -621,12 +608,8 @@ int i915_guc_submit(struct i915_guc_client *client,
 	} else {
 		client->retcode = 0;
 	}
-	spin_unlock_irqrestore(&client->wq_lock, flags);
-
-	spin_lock(&guc->host2guc_lock);
 	guc->submissions[ring_id] += 1;
 	guc->last_seqno[ring_id] = rq->seqno;
-	spin_unlock(&guc->host2guc_lock);
 
 	return q_ret;
 }
@@ -678,7 +661,7 @@ static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev,
 /**
  * gem_release_guc_obj() - Release gem object allocated for GuC usage
  * @obj:	gem obj to be released
-  */
+ */
 static void gem_release_guc_obj(struct drm_i915_gem_object *obj)
 {
 	if (!obj)
@@ -731,7 +714,8 @@ static void guc_client_free(struct drm_device *dev,
  * 		The kernel client to replace ExecList submission is created with
  * 		NORMAL priority. Priority of a client for scheduler can be HIGH,
  * 		while a preemption context can use CRITICAL.
- * @ctx		the context to own the client (we use the default render context)
+ * @ctx:	the context that owns the client (we use the default render
+ * 		context)
  *
  * Return:	An i915_guc_client object if success.
  */
@@ -768,7 +752,6 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
 	client->client_obj = obj;
 	client->wq_offset = GUC_DB_SIZE;
 	client->wq_size = GUC_WQ_SIZE;
-	spin_lock_init(&client->wq_lock);
 
 	client->doorbell_offset = select_doorbell_cacheline(guc);
 
@@ -871,8 +854,6 @@ int i915_guc_submission_init(struct drm_device *dev)
 	if (!guc->ctx_pool_obj)
 		return -ENOMEM;
 
-	spin_lock_init(&dev_priv->guc.host2guc_lock);
-
 	ida_init(&guc->ctx_ids);
 
 	guc_create_log(guc);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 0d228f909dcb..fa8afa7860ae 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -139,7 +139,8 @@ static const u32 hpd_bxt[HPD_NUM_PINS] = {
 /*
  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
  */
-static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
+static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
+				    i915_reg_t reg)
 {
 	u32 val = I915_READ(reg);
 
@@ -147,7 +148,7 @@ static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
 		return;
 
 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
-	     reg, val);
+	     i915_mmio_reg_offset(reg), val);
 	I915_WRITE(reg, 0xffffffff);
 	POSTING_READ(reg);
 	I915_WRITE(reg, 0xffffffff);
@@ -214,9 +215,9 @@ void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
-static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
-				   uint32_t interrupt_mask,
-				   uint32_t enabled_irq_mask)
+void ilk_update_display_irq(struct drm_i915_private *dev_priv,
+			    uint32_t interrupt_mask,
+			    uint32_t enabled_irq_mask)
 {
 	uint32_t new_val;
 
@@ -238,18 +239,6 @@ static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
 	}
 }
 
-void
-ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
-{
-	ilk_update_display_irq(dev_priv, mask, mask);
-}
-
-void
-ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
-{
-	ilk_update_display_irq(dev_priv, mask, 0);
-}
-
 /**
  * ilk_update_gt_irq - update GTIMR
  * @dev_priv: driver private
@@ -283,27 +272,27 @@ void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 	ilk_update_gt_irq(dev_priv, mask, 0);
 }
 
-static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
+static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 {
 	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 }
 
-static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
+static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
 {
 	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 }
 
-static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
+static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
 {
 	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 }
 
 /**
-  * snb_update_pm_irq - update GEN6_PMIMR
-  * @dev_priv: driver private
-  * @interrupt_mask: mask of interrupt bits to update
-  * @enabled_irq_mask: mask of interrupt bits to enable
-  */
+ * snb_update_pm_irq - update GEN6_PMIMR
+ * @dev_priv: driver private
+ * @interrupt_mask: mask of interrupt bits to update
+ * @enabled_irq_mask: mask of interrupt bits to enable
+ */
 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 			      uint32_t interrupt_mask,
 			      uint32_t enabled_irq_mask)
@@ -350,7 +339,7 @@ void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 void gen6_reset_rps_interrupts(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t reg = gen6_pm_iir(dev_priv);
+	i915_reg_t reg = gen6_pm_iir(dev_priv);
 
 	spin_lock_irq(&dev_priv->irq_lock);
 	I915_WRITE(reg, dev_priv->pm_rps_events);
@@ -417,11 +406,11 @@ void gen6_disable_rps_interrupts(struct drm_device *dev)
 }
 
 /**
-  * bdw_update_port_irq - update DE port interrupt
-  * @dev_priv: driver private
-  * @interrupt_mask: mask of interrupt bits to update
-  * @enabled_irq_mask: mask of interrupt bits to enable
-  */
+ * bdw_update_port_irq - update DE port interrupt
+ * @dev_priv: driver private
+ * @interrupt_mask: mask of interrupt bits to update
+ * @enabled_irq_mask: mask of interrupt bits to enable
+ */
 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 				uint32_t interrupt_mask,
 				uint32_t enabled_irq_mask)
@@ -449,6 +438,38 @@ static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 }
 
 /**
+ * bdw_update_pipe_irq - update DE pipe interrupt
+ * @dev_priv: driver private
+ * @pipe: pipe whose interrupt to update
+ * @interrupt_mask: mask of interrupt bits to update
+ * @enabled_irq_mask: mask of interrupt bits to enable
+ */
+void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
+			 enum pipe pipe,
+			 uint32_t interrupt_mask,
+			 uint32_t enabled_irq_mask)
+{
+	uint32_t new_val;
+
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	WARN_ON(enabled_irq_mask & ~interrupt_mask);
+
+	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
+		return;
+
+	new_val = dev_priv->de_irq_mask[pipe];
+	new_val &= ~interrupt_mask;
+	new_val |= (~enabled_irq_mask & interrupt_mask);
+
+	if (new_val != dev_priv->de_irq_mask[pipe]) {
+		dev_priv->de_irq_mask[pipe] = new_val;
+		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+	}
+}
+
+/**
  * ibx_display_interrupt_update - update SDEIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
@@ -477,7 +498,7 @@ static void
 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 		       u32 enable_mask, u32 status_mask)
 {
-	u32 reg = PIPESTAT(pipe);
+	i915_reg_t reg = PIPESTAT(pipe);
 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 
 	assert_spin_locked(&dev_priv->irq_lock);
@@ -504,7 +525,7 @@ static void
 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 		        u32 enable_mask, u32 status_mask)
 {
-	u32 reg = PIPESTAT(pipe);
+	i915_reg_t reg = PIPESTAT(pipe);
 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 
 	assert_spin_locked(&dev_priv->irq_lock);
@@ -560,7 +581,7 @@ i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 {
 	u32 enable_mask;
 
-	if (IS_VALLEYVIEW(dev_priv->dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 							   status_mask);
 	else
@@ -574,7 +595,7 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 {
 	u32 enable_mask;
 
-	if (IS_VALLEYVIEW(dev_priv->dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 							   status_mask);
 	else
@@ -665,8 +686,7 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	unsigned long high_frame;
-	unsigned long low_frame;
+	i915_reg_t high_frame, low_frame;
 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 	struct intel_crtc *intel_crtc =
 		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
@@ -717,9 +737,7 @@ static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
 }
 
-/* raw reads, only for fast reads of display block, no need for forcewake etc. */
-#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
-
+/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
@@ -733,9 +751,9 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 		vtotal /= 2;
 
 	if (IS_GEN2(dev))
-		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
+		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 	else
-		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
+		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 
 	/*
 	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
@@ -827,7 +845,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 		 * We can split this into vertical and horizontal
 		 * scanout position.
 		 */
-		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
+		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 
 		/* convert to pixel counts */
 		vbl_start *= htotal;
@@ -1085,6 +1103,14 @@ static void gen6_pm_rps_work(struct work_struct *work)
 		spin_unlock_irq(&dev_priv->irq_lock);
 		return;
 	}
+
+	/*
+	 * The RPS work is synced during runtime suspend, we don't require a
+	 * wakeref. TODO: instead of disabling the asserts make sure that we
+	 * always hold an RPM reference while the work is running.
+	 */
+	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
+
 	pm_iir = dev_priv->rps.pm_iir;
 	dev_priv->rps.pm_iir = 0;
 	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
@@ -1097,7 +1123,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 
 	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
-		return;
+		goto out;
 
 	mutex_lock(&dev_priv->rps.hw_lock);
 
@@ -1152,6 +1178,8 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	intel_set_rps(dev_priv->dev, new_delay);
 
 	mutex_unlock(&dev_priv->rps.hw_lock);
+out:
+	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
 }
 
 
@@ -1188,7 +1216,7 @@ static void ivybridge_parity_work(struct work_struct *work)
 	POSTING_READ(GEN7_MISCCPCTL);
 
 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
-		u32 reg;
+		i915_reg_t reg;
 
 		slice--;
 		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
@@ -1196,7 +1224,7 @@ static void ivybridge_parity_work(struct work_struct *work)
 
 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
 
-		reg = GEN7_L3CDERRST1 + (slice * 0x200);
+		reg = GEN7_L3CDERRST1(slice);
 
 		error_status = I915_READ(reg);
 		row = GEN7_PARITY_ERROR_ROW(error_status);
@@ -1290,70 +1318,69 @@ static void snb_gt_irq_handler(struct drm_device *dev,
 		ivybridge_parity_error_irq_handler(dev, gt_iir);
 }
 
+static __always_inline void
+gen8_cs_irq_handler(struct intel_engine_cs *ring, u32 iir, int test_shift)
+{
+	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
+		notify_ring(ring);
+	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
+		intel_lrc_irq_handler(ring);
+}
+
 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
 				       u32 master_ctl)
 {
 	irqreturn_t ret = IRQ_NONE;
 
 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
-		u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
-		if (tmp) {
-			I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
+		u32 iir = I915_READ_FW(GEN8_GT_IIR(0));
+		if (iir) {
+			I915_WRITE_FW(GEN8_GT_IIR(0), iir);
 			ret = IRQ_HANDLED;
 
-			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
-				intel_lrc_irq_handler(&dev_priv->ring[RCS]);
-			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
-				notify_ring(&dev_priv->ring[RCS]);
+			gen8_cs_irq_handler(&dev_priv->ring[RCS],
+					iir, GEN8_RCS_IRQ_SHIFT);
 
-			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
-				intel_lrc_irq_handler(&dev_priv->ring[BCS]);
-			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
-				notify_ring(&dev_priv->ring[BCS]);
+			gen8_cs_irq_handler(&dev_priv->ring[BCS],
+					iir, GEN8_BCS_IRQ_SHIFT);
 		} else
 			DRM_ERROR("The master control interrupt lied (GT0)!\n");
 	}
 
 	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
-		u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
-		if (tmp) {
-			I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
+		u32 iir = I915_READ_FW(GEN8_GT_IIR(1));
+		if (iir) {
+			I915_WRITE_FW(GEN8_GT_IIR(1), iir);
 			ret = IRQ_HANDLED;
 
-			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
-				intel_lrc_irq_handler(&dev_priv->ring[VCS]);
-			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
-				notify_ring(&dev_priv->ring[VCS]);
+			gen8_cs_irq_handler(&dev_priv->ring[VCS],
+					iir, GEN8_VCS1_IRQ_SHIFT);
 
-			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
-				intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
-			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
-				notify_ring(&dev_priv->ring[VCS2]);
+			gen8_cs_irq_handler(&dev_priv->ring[VCS2],
+					iir, GEN8_VCS2_IRQ_SHIFT);
 		} else
 			DRM_ERROR("The master control interrupt lied (GT1)!\n");
 	}
 
 	if (master_ctl & GEN8_GT_VECS_IRQ) {
-		u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
-		if (tmp) {
-			I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
+		u32 iir = I915_READ_FW(GEN8_GT_IIR(3));
+		if (iir) {
+			I915_WRITE_FW(GEN8_GT_IIR(3), iir);
 			ret = IRQ_HANDLED;
 
-			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
-				intel_lrc_irq_handler(&dev_priv->ring[VECS]);
-			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
-				notify_ring(&dev_priv->ring[VECS]);
+			gen8_cs_irq_handler(&dev_priv->ring[VECS],
+					iir, GEN8_VECS_IRQ_SHIFT);
 		} else
 			DRM_ERROR("The master control interrupt lied (GT3)!\n");
 	}
 
 	if (master_ctl & GEN8_GT_PM_IRQ) {
-		u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
-		if (tmp & dev_priv->pm_rps_events) {
+		u32 iir = I915_READ_FW(GEN8_GT_IIR(2));
+		if (iir & dev_priv->pm_rps_events) {
 			I915_WRITE_FW(GEN8_GT_IIR(2),
-				      tmp & dev_priv->pm_rps_events);
+				      iir & dev_priv->pm_rps_events);
 			ret = IRQ_HANDLED;
-			gen6_rps_irq_handler(dev_priv, tmp);
+			gen6_rps_irq_handler(dev_priv, iir);
 		} else
 			DRM_ERROR("The master control interrupt lied (PM)!\n");
 	}
@@ -1625,7 +1652,7 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
 
 	spin_lock(&dev_priv->irq_lock);
 	for_each_pipe(dev_priv, pipe) {
-		int reg;
+		i915_reg_t reg;
 		u32 mask, iir_bit = 0;
 
 		/*
@@ -1706,7 +1733,7 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev)
 	 */
 	POSTING_READ(PORT_HOTPLUG_STAT);
 
-	if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
 
 		if (hotplug_trigger) {
@@ -1741,6 +1768,9 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	while (true) {
 		/* Find, clear, then process each source of interrupt */
 
@@ -1775,6 +1805,8 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 	}
 
 out:
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -1788,6 +1820,9 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	for (;;) {
 		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
 		iir = I915_READ(VLV_IIR);
@@ -1818,6 +1853,8 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 		POSTING_READ(GEN8_MASTER_IRQ);
 	}
 
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -1827,8 +1864,24 @@ static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
 
+	/*
+	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
+	 * unless we touch the hotplug register, even if hotplug_trigger is
+	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
+	 * errors.
+	 */
 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
+	if (!hotplug_trigger) {
+		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
+			PORTD_HOTPLUG_STATUS_MASK |
+			PORTC_HOTPLUG_STATUS_MASK |
+			PORTB_HOTPLUG_STATUS_MASK;
+		dig_hotplug_reg &= ~mask;
+	}
+
 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
+	if (!hotplug_trigger)
+		return;
 
 	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
 			   dig_hotplug_reg, hpd,
@@ -1843,8 +1896,7 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
 	int pipe;
 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
 
-	if (hotplug_trigger)
-		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
+	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
 
 	if (pch_iir & SDE_AUDIO_POWER_MASK) {
 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
@@ -1937,8 +1989,7 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
 	int pipe;
 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
 
-	if (hotplug_trigger)
-		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
+	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
 
 	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
@@ -2134,6 +2185,9 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	/* We get interrupts on unclaimed registers, so check for this before we
 	 * do any I915_{READ,WRITE}. */
 	intel_uncore_check_errors(dev);
@@ -2192,6 +2246,9 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 		POSTING_READ(SDEIER);
 	}
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -2224,6 +2281,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	if (INTEL_INFO(dev_priv)->gen >= 9)
 		aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
 			GEN9_AUX_CHANNEL_D;
@@ -2231,7 +2291,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
 	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
 	if (!master_ctl)
-		return IRQ_NONE;
+		goto out;
 
 	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
 
@@ -2354,14 +2414,21 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 				spt_irq_handler(dev, pch_iir);
 			else
 				cpt_irq_handler(dev, pch_iir);
-		} else
-			DRM_ERROR("The master control interrupt lied (SDE)!\n");
-
+		} else {
+			/*
+			 * Like on previous PCH there seems to be something
+			 * fishy going on with forwarding PCH interrupts.
+			 */
+			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
+		}
 	}
 
 	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
 	POSTING_READ_FW(GEN8_MASTER_IRQ);
 
+out:
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -2644,7 +2711,7 @@ static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
 						     DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	ironlake_enable_display_irq(dev_priv, bit);
+	ilk_enable_display_irq(dev_priv, bit);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
 	return 0;
@@ -2669,10 +2736,9 @@ static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
-	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
-	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+
 	return 0;
 }
 
@@ -2699,7 +2765,7 @@ static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
 						     DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	ironlake_disable_display_irq(dev_priv, bit);
+	ilk_disable_display_irq(dev_priv, bit);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
@@ -2720,9 +2786,7 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
-	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
-	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
@@ -2961,6 +3025,13 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 	if (!i915.enable_hangcheck)
 		return;
 
+	/*
+	 * The hangcheck work is synced during runtime suspend, we don't
+	 * require a wakeref. TODO: instead of disabling the asserts make
+	 * sure that we hold a reference when this work is running.
+	 */
+	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
+
 	for_each_ring(ring, dev_priv, i) {
 		u64 acthd;
 		u32 seqno;
@@ -3052,13 +3123,18 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 		}
 	}
 
-	if (rings_hung)
-		return i915_handle_error(dev, true, "Ring hung");
+	if (rings_hung) {
+		i915_handle_error(dev, true, "Ring hung");
+		goto out;
+	}
 
 	if (busy_count)
 		/* Reset timer case chip hangs without another request
 		 * being added */
 		i915_queue_hangcheck(dev);
+
+out:
+	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
 }
 
 void i915_queue_hangcheck(struct drm_device *dev)
@@ -3451,7 +3527,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 		 * setup is guaranteed to run in single-threaded context. But we
 		 * need it to make the assert_spin_locked happy. */
 		spin_lock_irq(&dev_priv->irq_lock);
-		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
+		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
 		spin_unlock_irq(&dev_priv->irq_lock);
 	}
 
@@ -3850,13 +3926,18 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 	u16 flip_mask =
 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
 		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
+	irqreturn_t ret;
 
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
+	ret = IRQ_NONE;
 	iir = I915_READ16(IIR);
 	if (iir == 0)
-		return IRQ_NONE;
+		goto out;
 
 	while (iir & ~flip_mask) {
 		/* Can't rely on pipestat interrupt bit in iir as it might
@@ -3869,7 +3950,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
 
 		for_each_pipe(dev_priv, pipe) {
-			int reg = PIPESTAT(pipe);
+			i915_reg_t reg = PIPESTAT(pipe);
 			pipe_stats[pipe] = I915_READ(reg);
 
 			/*
@@ -3905,8 +3986,12 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
 		iir = new_iir;
 	}
+	ret = IRQ_HANDLED;
+
+out:
+	enable_rpm_wakeref_asserts(dev_priv);
 
-	return IRQ_HANDLED;
+	return ret;
 }
 
 static void i8xx_irq_uninstall(struct drm_device * dev)
@@ -4035,6 +4120,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	iir = I915_READ(IIR);
 	do {
 		bool irq_received = (iir & ~flip_mask) != 0;
@@ -4050,7 +4138,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
 
 		for_each_pipe(dev_priv, pipe) {
-			int reg = PIPESTAT(pipe);
+			i915_reg_t reg = PIPESTAT(pipe);
 			pipe_stats[pipe] = I915_READ(reg);
 
 			/* Clear the PIPE*STAT regs before the IIR */
@@ -4117,6 +4205,8 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		iir = new_iir;
 	} while (iir & ~flip_mask);
 
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -4256,6 +4346,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
 
+	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
+	disable_rpm_wakeref_asserts(dev_priv);
+
 	iir = I915_READ(IIR);
 
 	for (;;) {
@@ -4272,7 +4365,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
 
 		for_each_pipe(dev_priv, pipe) {
-			int reg = PIPESTAT(pipe);
+			i915_reg_t reg = PIPESTAT(pipe);
 			pipe_stats[pipe] = I915_READ(reg);
 
 			/*
@@ -4341,6 +4434,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		iir = new_iir;
 	}
 
+	enable_rpm_wakeref_asserts(dev_priv);
+
 	return ret;
 }
 
@@ -4384,7 +4479,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
 
 	/* Let's track the enabled rps events */
-	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
+	if (IS_VALLEYVIEW(dev_priv))
 		/* WaGsvRC0ResidencyMethod:vlv */
 		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
 	else
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 4be13a5eb932..835d6099c769 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -32,6 +32,7 @@ struct i915_params i915 __read_mostly = {
 	.panel_use_ssc = -1,
 	.vbt_sdvo_panel_type = -1,
 	.enable_rc6 = -1,
+	.enable_dc = -1,
 	.enable_fbc = -1,
 	.enable_execlists = -1,
 	.enable_hangcheck = true,
@@ -80,6 +81,11 @@ MODULE_PARM_DESC(enable_rc6,
 	"For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. "
 	"default: -1 (use per-chip default)");
 
+module_param_named_unsafe(enable_dc, i915.enable_dc, int, 0400);
+MODULE_PARM_DESC(enable_dc,
+	"Enable power-saving display C-states. "
+	"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)");
+
 module_param_named_unsafe(enable_fbc, i915.enable_fbc, int, 0600);
 MODULE_PARM_DESC(enable_fbc,
 	"Enable frame buffer compression for power savings "
@@ -112,7 +118,7 @@ MODULE_PARM_DESC(enable_hangcheck,
 module_param_named_unsafe(enable_ppgtt, i915.enable_ppgtt, int, 0400);
 MODULE_PARM_DESC(enable_ppgtt,
 	"Override PPGTT usage. "
-	"(-1=auto [default], 0=disabled, 1=aliasing, 2=full)");
+	"(-1=auto [default], 0=disabled, 1=aliasing, 2=full, 3=full with extended address space)");
 
 module_param_named_unsafe(enable_execlists, i915.enable_execlists, int, 0400);
 MODULE_PARM_DESC(enable_execlists,
@@ -126,7 +132,7 @@ module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, i
 MODULE_PARM_DESC(preliminary_hw_support,
 	"Enable preliminary hardware support.");
 
-module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0600);
+module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400);
 MODULE_PARM_DESC(disable_power_well,
 	"Disable display power wells when possible "
 	"(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)");
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index bc7b8faba84d..007ae83a4086 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -25,14 +25,43 @@
 #ifndef _I915_REG_H_
 #define _I915_REG_H_
 
+typedef struct {
+	uint32_t reg;
+} i915_reg_t;
+
+#define _MMIO(r) ((const i915_reg_t){ .reg = (r) })
+
+#define INVALID_MMIO_REG _MMIO(0)
+
+static inline uint32_t i915_mmio_reg_offset(i915_reg_t reg)
+{
+	return reg.reg;
+}
+
+static inline bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b)
+{
+	return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b);
+}
+
+static inline bool i915_mmio_reg_valid(i915_reg_t reg)
+{
+	return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG);
+}
+
 #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a)))
+#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b))
 #define _PLANE(plane, a, b) _PIPE(plane, a, b)
-#define _TRANSCODER(tran, a, b) ((a) + (tran)*((b)-(a)))
+#define _MMIO_PLANE(plane, a, b) _MMIO_PIPE(plane, a, b)
+#define _TRANS(tran, a, b) ((a) + (tran)*((b)-(a)))
+#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b))
 #define _PORT(port, a, b) ((a) + (port)*((b)-(a)))
+#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b))
 #define _PIPE3(pipe, a, b, c) ((pipe) == PIPE_A ? (a) : \
 			       (pipe) == PIPE_B ? (b) : (c))
+#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PIPE3(pipe, a, b, c))
 #define _PORT3(port, a, b, c) ((port) == PORT_A ? (a) : \
 			       (port) == PORT_B ? (b) : (c))
+#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PORT3(pipe, a, b, c))
 
 #define _MASKED_FIELD(mask, value) ({					   \
 	if (__builtin_constant_p(mask))					   \
@@ -105,14 +134,14 @@
 #define  GRDOM_RESET_STATUS (1<<1)
 #define  GRDOM_RESET_ENABLE (1<<0)
 
-#define ILK_GDSR (MCHBAR_MIRROR_BASE + 0x2ca4)
+#define ILK_GDSR _MMIO(MCHBAR_MIRROR_BASE + 0x2ca4)
 #define  ILK_GRDOM_FULL		(0<<1)
 #define  ILK_GRDOM_RENDER	(1<<1)
 #define  ILK_GRDOM_MEDIA	(3<<1)
 #define  ILK_GRDOM_MASK		(3<<1)
 #define  ILK_GRDOM_RESET_ENABLE (1<<0)
 
-#define GEN6_MBCUNIT_SNPCR	0x900c /* for LLC config */
+#define GEN6_MBCUNIT_SNPCR	_MMIO(0x900c) /* for LLC config */
 #define   GEN6_MBC_SNPCR_SHIFT	21
 #define   GEN6_MBC_SNPCR_MASK	(3<<21)
 #define   GEN6_MBC_SNPCR_MAX	(0<<21)
@@ -120,31 +149,31 @@
 #define   GEN6_MBC_SNPCR_LOW	(2<<21)
 #define   GEN6_MBC_SNPCR_MIN	(3<<21) /* only 1/16th of the cache is shared */
 
-#define VLV_G3DCTL		0x9024
-#define VLV_GSCKGCTL		0x9028
+#define VLV_G3DCTL		_MMIO(0x9024)
+#define VLV_GSCKGCTL		_MMIO(0x9028)
 
-#define GEN6_MBCTL		0x0907c
+#define GEN6_MBCTL		_MMIO(0x0907c)
 #define   GEN6_MBCTL_ENABLE_BOOT_FETCH	(1 << 4)
 #define   GEN6_MBCTL_CTX_FETCH_NEEDED	(1 << 3)
 #define   GEN6_MBCTL_BME_UPDATE_ENABLE	(1 << 2)
 #define   GEN6_MBCTL_MAE_UPDATE_ENABLE	(1 << 1)
 #define   GEN6_MBCTL_BOOT_FETCH_MECH	(1 << 0)
 
-#define GEN6_GDRST	0x941c
+#define GEN6_GDRST	_MMIO(0x941c)
 #define  GEN6_GRDOM_FULL		(1 << 0)
 #define  GEN6_GRDOM_RENDER		(1 << 1)
 #define  GEN6_GRDOM_MEDIA		(1 << 2)
 #define  GEN6_GRDOM_BLT			(1 << 3)
 
-#define RING_PP_DIR_BASE(ring)		((ring)->mmio_base+0x228)
-#define RING_PP_DIR_BASE_READ(ring)	((ring)->mmio_base+0x518)
-#define RING_PP_DIR_DCLV(ring)		((ring)->mmio_base+0x220)
+#define RING_PP_DIR_BASE(ring)		_MMIO((ring)->mmio_base+0x228)
+#define RING_PP_DIR_BASE_READ(ring)	_MMIO((ring)->mmio_base+0x518)
+#define RING_PP_DIR_DCLV(ring)		_MMIO((ring)->mmio_base+0x220)
 #define   PP_DIR_DCLV_2G		0xffffffff
 
-#define GEN8_RING_PDP_UDW(ring, n)	((ring)->mmio_base+0x270 + ((n) * 8 + 4))
-#define GEN8_RING_PDP_LDW(ring, n)	((ring)->mmio_base+0x270 + (n) * 8)
+#define GEN8_RING_PDP_UDW(ring, n)	_MMIO((ring)->mmio_base+0x270 + (n) * 8 + 4)
+#define GEN8_RING_PDP_LDW(ring, n)	_MMIO((ring)->mmio_base+0x270 + (n) * 8)
 
-#define GEN8_R_PWR_CLK_STATE		0x20C8
+#define GEN8_R_PWR_CLK_STATE		_MMIO(0x20C8)
 #define   GEN8_RPCS_ENABLE		(1 << 31)
 #define   GEN8_RPCS_S_CNT_ENABLE	(1 << 18)
 #define   GEN8_RPCS_S_CNT_SHIFT		15
@@ -157,7 +186,7 @@
 #define   GEN8_RPCS_EU_MIN_SHIFT	0
 #define   GEN8_RPCS_EU_MIN_MASK		(0xf << GEN8_RPCS_EU_MIN_SHIFT)
 
-#define GAM_ECOCHK			0x4090
+#define GAM_ECOCHK			_MMIO(0x4090)
 #define   BDW_DISABLE_HDC_INVALIDATION	(1<<25)
 #define   ECOCHK_SNB_BIT		(1<<10)
 #define   ECOCHK_DIS_TLB		(1<<8)
@@ -170,15 +199,15 @@
 #define   ECOCHK_PPGTT_WT_HSW		(0x2<<3)
 #define   ECOCHK_PPGTT_WB_HSW		(0x3<<3)
 
-#define GAC_ECO_BITS			0x14090
+#define GAC_ECO_BITS			_MMIO(0x14090)
 #define   ECOBITS_SNB_BIT		(1<<13)
 #define   ECOBITS_PPGTT_CACHE64B	(3<<8)
 #define   ECOBITS_PPGTT_CACHE4B		(0<<8)
 
-#define GAB_CTL				0x24000
+#define GAB_CTL				_MMIO(0x24000)
 #define   GAB_CTL_CONT_AFTER_PAGEFAULT	(1<<8)
 
-#define GEN6_STOLEN_RESERVED		0x1082C0
+#define GEN6_STOLEN_RESERVED		_MMIO(0x1082C0)
 #define GEN6_STOLEN_RESERVED_ADDR_MASK	(0xFFF << 20)
 #define GEN7_STOLEN_RESERVED_ADDR_MASK	(0x3FFF << 18)
 #define GEN6_STOLEN_RESERVED_SIZE_MASK	(3 << 4)
@@ -200,6 +229,7 @@
 #define VGA_ST01_MDA 0x3ba
 #define VGA_ST01_CGA 0x3da
 
+#define _VGA_MSR_WRITE _MMIO(0x3c2)
 #define VGA_MSR_WRITE 0x3c2
 #define VGA_MSR_READ 0x3cc
 #define   VGA_MSR_MEM_EN (1<<1)
@@ -377,10 +407,12 @@
 #define MI_BATCH_BUFFER_START_GEN8	MI_INSTR(0x31, 1)
 #define   MI_BATCH_RESOURCE_STREAMER (1<<10)
 
-#define MI_PREDICATE_SRC0	(0x2400)
-#define MI_PREDICATE_SRC1	(0x2408)
+#define MI_PREDICATE_SRC0	_MMIO(0x2400)
+#define MI_PREDICATE_SRC0_UDW	_MMIO(0x2400 + 4)
+#define MI_PREDICATE_SRC1	_MMIO(0x2408)
+#define MI_PREDICATE_SRC1_UDW	_MMIO(0x2408 + 4)
 
-#define MI_PREDICATE_RESULT_2	(0x2214)
+#define MI_PREDICATE_RESULT_2	_MMIO(0x2214)
 #define  LOWER_SLICE_ENABLED	(1<<0)
 #define  LOWER_SLICE_DISABLED	(0<<0)
 
@@ -509,49 +541,61 @@
 /*
  * Registers used only by the command parser
  */
-#define BCS_SWCTRL 0x22200
-
-#define GPGPU_THREADS_DISPATCHED        0x2290
-#define HS_INVOCATION_COUNT             0x2300
-#define DS_INVOCATION_COUNT             0x2308
-#define IA_VERTICES_COUNT               0x2310
-#define IA_PRIMITIVES_COUNT             0x2318
-#define VS_INVOCATION_COUNT             0x2320
-#define GS_INVOCATION_COUNT             0x2328
-#define GS_PRIMITIVES_COUNT             0x2330
-#define CL_INVOCATION_COUNT             0x2338
-#define CL_PRIMITIVES_COUNT             0x2340
-#define PS_INVOCATION_COUNT             0x2348
-#define PS_DEPTH_COUNT                  0x2350
+#define BCS_SWCTRL _MMIO(0x22200)
+
+#define GPGPU_THREADS_DISPATCHED        _MMIO(0x2290)
+#define GPGPU_THREADS_DISPATCHED_UDW	_MMIO(0x2290 + 4)
+#define HS_INVOCATION_COUNT             _MMIO(0x2300)
+#define HS_INVOCATION_COUNT_UDW		_MMIO(0x2300 + 4)
+#define DS_INVOCATION_COUNT             _MMIO(0x2308)
+#define DS_INVOCATION_COUNT_UDW		_MMIO(0x2308 + 4)
+#define IA_VERTICES_COUNT               _MMIO(0x2310)
+#define IA_VERTICES_COUNT_UDW		_MMIO(0x2310 + 4)
+#define IA_PRIMITIVES_COUNT             _MMIO(0x2318)
+#define IA_PRIMITIVES_COUNT_UDW		_MMIO(0x2318 + 4)
+#define VS_INVOCATION_COUNT             _MMIO(0x2320)
+#define VS_INVOCATION_COUNT_UDW		_MMIO(0x2320 + 4)
+#define GS_INVOCATION_COUNT             _MMIO(0x2328)
+#define GS_INVOCATION_COUNT_UDW		_MMIO(0x2328 + 4)
+#define GS_PRIMITIVES_COUNT             _MMIO(0x2330)
+#define GS_PRIMITIVES_COUNT_UDW		_MMIO(0x2330 + 4)
+#define CL_INVOCATION_COUNT             _MMIO(0x2338)
+#define CL_INVOCATION_COUNT_UDW		_MMIO(0x2338 + 4)
+#define CL_PRIMITIVES_COUNT             _MMIO(0x2340)
+#define CL_PRIMITIVES_COUNT_UDW		_MMIO(0x2340 + 4)
+#define PS_INVOCATION_COUNT             _MMIO(0x2348)
+#define PS_INVOCATION_COUNT_UDW		_MMIO(0x2348 + 4)
+#define PS_DEPTH_COUNT                  _MMIO(0x2350)
+#define PS_DEPTH_COUNT_UDW		_MMIO(0x2350 + 4)
 
 /* There are the 4 64-bit counter registers, one for each stream output */
-#define GEN7_SO_NUM_PRIMS_WRITTEN(n) (0x5200 + (n) * 8)
+#define GEN7_SO_NUM_PRIMS_WRITTEN(n)		_MMIO(0x5200 + (n) * 8)
+#define GEN7_SO_NUM_PRIMS_WRITTEN_UDW(n)	_MMIO(0x5200 + (n) * 8 + 4)
 
-#define GEN7_SO_PRIM_STORAGE_NEEDED(n)  (0x5240 + (n) * 8)
+#define GEN7_SO_PRIM_STORAGE_NEEDED(n)		_MMIO(0x5240 + (n) * 8)
+#define GEN7_SO_PRIM_STORAGE_NEEDED_UDW(n)	_MMIO(0x5240 + (n) * 8 + 4)
 
-#define GEN7_3DPRIM_END_OFFSET          0x2420
-#define GEN7_3DPRIM_START_VERTEX        0x2430
-#define GEN7_3DPRIM_VERTEX_COUNT        0x2434
-#define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
-#define GEN7_3DPRIM_START_INSTANCE      0x243C
-#define GEN7_3DPRIM_BASE_VERTEX         0x2440
+#define GEN7_3DPRIM_END_OFFSET          _MMIO(0x2420)
+#define GEN7_3DPRIM_START_VERTEX        _MMIO(0x2430)
+#define GEN7_3DPRIM_VERTEX_COUNT        _MMIO(0x2434)
+#define GEN7_3DPRIM_INSTANCE_COUNT      _MMIO(0x2438)
+#define GEN7_3DPRIM_START_INSTANCE      _MMIO(0x243C)
+#define GEN7_3DPRIM_BASE_VERTEX         _MMIO(0x2440)
 
-#define GEN7_GPGPU_DISPATCHDIMX         0x2500
-#define GEN7_GPGPU_DISPATCHDIMY         0x2504
-#define GEN7_GPGPU_DISPATCHDIMZ         0x2508
+#define GEN7_GPGPU_DISPATCHDIMX         _MMIO(0x2500)
+#define GEN7_GPGPU_DISPATCHDIMY         _MMIO(0x2504)
+#define GEN7_GPGPU_DISPATCHDIMZ         _MMIO(0x2508)
 
-#define OACONTROL 0x2360
+#define OACONTROL _MMIO(0x2360)
 
 #define _GEN7_PIPEA_DE_LOAD_SL	0x70068
 #define _GEN7_PIPEB_DE_LOAD_SL	0x71068
-#define GEN7_PIPE_DE_LOAD_SL(pipe) _PIPE(pipe, \
-					 _GEN7_PIPEA_DE_LOAD_SL, \
-					 _GEN7_PIPEB_DE_LOAD_SL)
+#define GEN7_PIPE_DE_LOAD_SL(pipe) _MMIO_PIPE(pipe, _GEN7_PIPEA_DE_LOAD_SL, _GEN7_PIPEB_DE_LOAD_SL)
 
 /*
  * Reset registers
  */
-#define DEBUG_RESET_I830		0x6070
+#define DEBUG_RESET_I830		_MMIO(0x6070)
 #define  DEBUG_RESET_FULL		(1<<7)
 #define  DEBUG_RESET_RENDER		(1<<8)
 #define  DEBUG_RESET_DISPLAY		(1<<9)
@@ -559,7 +603,7 @@
 /*
  * IOSF sideband
  */
-#define VLV_IOSF_DOORBELL_REQ			(VLV_DISPLAY_BASE + 0x2100)
+#define VLV_IOSF_DOORBELL_REQ			_MMIO(VLV_DISPLAY_BASE + 0x2100)
 #define   IOSF_DEVFN_SHIFT			24
 #define   IOSF_OPCODE_SHIFT			16
 #define   IOSF_PORT_SHIFT			8
@@ -576,8 +620,8 @@
 #define   IOSF_PORT_CCU				0xA9
 #define   IOSF_PORT_GPS_CORE			0x48
 #define   IOSF_PORT_FLISDSI			0x1B
-#define VLV_IOSF_DATA				(VLV_DISPLAY_BASE + 0x2104)
-#define VLV_IOSF_ADDR				(VLV_DISPLAY_BASE + 0x2108)
+#define VLV_IOSF_DATA				_MMIO(VLV_DISPLAY_BASE + 0x2104)
+#define VLV_IOSF_ADDR				_MMIO(VLV_DISPLAY_BASE + 0x2108)
 
 /* See configdb bunit SB addr map */
 #define BUNIT_REG_BISOC				0x11
@@ -609,6 +653,7 @@
 
 /* See the PUNIT HAS v0.8 for the below bits */
 enum punit_power_well {
+	/* These numbers are fixed and must match the position of the pw bits */
 	PUNIT_POWER_WELL_RENDER			= 0,
 	PUNIT_POWER_WELL_MEDIA			= 1,
 	PUNIT_POWER_WELL_DISP2D			= 3,
@@ -621,10 +666,12 @@ enum punit_power_well {
 	PUNIT_POWER_WELL_DPIO_RX1		= 11,
 	PUNIT_POWER_WELL_DPIO_CMN_D		= 12,
 
-	PUNIT_POWER_WELL_NUM,
+	/* Not actual bit groups. Used as IDs for lookup_power_well() */
+	PUNIT_POWER_WELL_ALWAYS_ON,
 };
 
 enum skl_disp_power_wells {
+	/* These numbers are fixed and must match the position of the pw bits */
 	SKL_DISP_PW_MISC_IO,
 	SKL_DISP_PW_DDI_A_E,
 	SKL_DISP_PW_DDI_B,
@@ -632,6 +679,10 @@ enum skl_disp_power_wells {
 	SKL_DISP_PW_DDI_D,
 	SKL_DISP_PW_1 = 14,
 	SKL_DISP_PW_2,
+
+	/* Not actual bit groups. Used as IDs for lookup_power_well() */
+	SKL_DISP_PW_ALWAYS_ON,
+	SKL_DISP_PW_DC_OFF,
 };
 
 #define SKL_POWER_WELL_STATE(pw) (1 << ((pw) * 2))
@@ -804,35 +855,35 @@ enum skl_disp_power_wells {
  *
  * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
  * digital port D (CHV) or port A (BXT).
- */
-/*
- * Dual channel PHY (VLV/CHV/BXT)
- * ---------------------------------
- * |      CH0      |      CH1      |
- * |  CMN/PLL/REF  |  CMN/PLL/REF  |
- * |---------------|---------------| Display PHY
- * | PCS01 | PCS23 | PCS01 | PCS23 |
- * |-------|-------|-------|-------|
- * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
- * ---------------------------------
- * |     DDI0      |     DDI1      | DP/HDMI ports
- * ---------------------------------
  *
- * Single channel PHY (CHV/BXT)
- * -----------------
- * |      CH0      |
- * |  CMN/PLL/REF  |
- * |---------------| Display PHY
- * | PCS01 | PCS23 |
- * |-------|-------|
- * |TX0|TX1|TX2|TX3|
- * -----------------
- * |     DDI2      | DP/HDMI port
- * -----------------
+ *
+ *     Dual channel PHY (VLV/CHV/BXT)
+ *     ---------------------------------
+ *     |      CH0      |      CH1      |
+ *     |  CMN/PLL/REF  |  CMN/PLL/REF  |
+ *     |---------------|---------------| Display PHY
+ *     | PCS01 | PCS23 | PCS01 | PCS23 |
+ *     |-------|-------|-------|-------|
+ *     |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
+ *     ---------------------------------
+ *     |     DDI0      |     DDI1      | DP/HDMI ports
+ *     ---------------------------------
+ *
+ *     Single channel PHY (CHV/BXT)
+ *     -----------------
+ *     |      CH0      |
+ *     |  CMN/PLL/REF  |
+ *     |---------------| Display PHY
+ *     | PCS01 | PCS23 |
+ *     |-------|-------|
+ *     |TX0|TX1|TX2|TX3|
+ *     -----------------
+ *     |     DDI2      | DP/HDMI port
+ *     -----------------
  */
 #define DPIO_DEVFN			0
 
-#define DPIO_CTL			(VLV_DISPLAY_BASE + 0x2110)
+#define DPIO_CTL			_MMIO(VLV_DISPLAY_BASE + 0x2110)
 #define  DPIO_MODSEL1			(1<<3) /* if ref clk b == 27 */
 #define  DPIO_MODSEL0			(1<<2) /* if ref clk a == 27 */
 #define  DPIO_SFR_BYPASS		(1<<1)
@@ -1185,9 +1236,9 @@ enum skl_disp_power_wells {
 #define   DPIO_UPAR_SHIFT		30
 
 /* BXT PHY registers */
-#define _BXT_PHY(phy, a, b)		_PIPE((phy), (a), (b))
+#define _BXT_PHY(phy, a, b)		_MMIO_PIPE((phy), (a), (b))
 
-#define BXT_P_CR_GT_DISP_PWRON		0x138090
+#define BXT_P_CR_GT_DISP_PWRON		_MMIO(0x138090)
 #define   GT_DISPLAY_POWER_ON(phy)	(1 << (phy))
 
 #define _PHY_CTL_FAMILY_EDP		0x64C80
@@ -1203,7 +1254,7 @@ enum skl_disp_power_wells {
 #define   PORT_PLL_ENABLE		(1 << 31)
 #define   PORT_PLL_LOCK			(1 << 30)
 #define   PORT_PLL_REF_SEL		(1 << 27)
-#define BXT_PORT_PLL_ENABLE(port)	_PORT(port, _PORT_PLL_A, _PORT_PLL_B)
+#define BXT_PORT_PLL_ENABLE(port)	_MMIO_PORT(port, _PORT_PLL_A, _PORT_PLL_B)
 
 #define _PORT_PLL_EBB_0_A		0x162034
 #define _PORT_PLL_EBB_0_B		0x6C034
@@ -1214,7 +1265,7 @@ enum skl_disp_power_wells {
 #define   PORT_PLL_P2_SHIFT		8
 #define   PORT_PLL_P2_MASK		(0x1f << PORT_PLL_P2_SHIFT)
 #define   PORT_PLL_P2(x)		((x)  << PORT_PLL_P2_SHIFT)
-#define BXT_PORT_PLL_EBB_0(port)	_PORT3(port, _PORT_PLL_EBB_0_A, \
+#define BXT_PORT_PLL_EBB_0(port)	_MMIO_PORT3(port, _PORT_PLL_EBB_0_A, \
 						_PORT_PLL_EBB_0_B,	\
 						_PORT_PLL_EBB_0_C)
 
@@ -1223,7 +1274,7 @@ enum skl_disp_power_wells {
 #define _PORT_PLL_EBB_4_C		0x6C344
 #define   PORT_PLL_10BIT_CLK_ENABLE	(1 << 13)
 #define   PORT_PLL_RECALIBRATE		(1 << 14)
-#define BXT_PORT_PLL_EBB_4(port)	_PORT3(port, _PORT_PLL_EBB_4_A, \
+#define BXT_PORT_PLL_EBB_4(port)	_MMIO_PORT3(port, _PORT_PLL_EBB_4_A, \
 						_PORT_PLL_EBB_4_B,	\
 						_PORT_PLL_EBB_4_C)
 
@@ -1259,7 +1310,7 @@ enum skl_disp_power_wells {
 #define _PORT_PLL_BASE(port)		_PORT3(port, _PORT_PLL_0_A,	\
 						_PORT_PLL_0_B,		\
 						_PORT_PLL_0_C)
-#define BXT_PORT_PLL(port, idx)		(_PORT_PLL_BASE(port) + (idx) * 4)
+#define BXT_PORT_PLL(port, idx)		_MMIO(_PORT_PLL_BASE(port) + (idx) * 4)
 
 /* BXT PHY common lane registers */
 #define _PORT_CL1CM_DW0_A		0x162000
@@ -1297,7 +1348,7 @@ enum skl_disp_power_wells {
 							_PORT_CL1CM_DW30_A)
 
 /* Defined for PHY0 only */
-#define BXT_PORT_CL2CM_DW6_BC		0x6C358
+#define BXT_PORT_CL2CM_DW6_BC		_MMIO(0x6C358)
 #define   DW6_OLDO_DYN_PWR_DOWN_EN	(1 << 28)
 
 /* BXT PHY Ref registers */
@@ -1337,10 +1388,10 @@ enum skl_disp_power_wells {
 #define _PORT_PCS_DW10_GRP_A		0x162C28
 #define _PORT_PCS_DW10_GRP_B		0x6CC28
 #define _PORT_PCS_DW10_GRP_C		0x6CE28
-#define BXT_PORT_PCS_DW10_LN01(port)	_PORT3(port, _PORT_PCS_DW10_LN01_A, \
+#define BXT_PORT_PCS_DW10_LN01(port)	_MMIO_PORT3(port, _PORT_PCS_DW10_LN01_A, \
 						     _PORT_PCS_DW10_LN01_B, \
 						     _PORT_PCS_DW10_LN01_C)
-#define BXT_PORT_PCS_DW10_GRP(port)	_PORT3(port, _PORT_PCS_DW10_GRP_A,  \
+#define BXT_PORT_PCS_DW10_GRP(port)	_MMIO_PORT3(port, _PORT_PCS_DW10_GRP_A,  \
 						     _PORT_PCS_DW10_GRP_B,  \
 						     _PORT_PCS_DW10_GRP_C)
 #define   TX2_SWING_CALC_INIT		(1 << 31)
@@ -1357,13 +1408,13 @@ enum skl_disp_power_wells {
 #define _PORT_PCS_DW12_GRP_C		0x6CE30
 #define   LANESTAGGER_STRAP_OVRD	(1 << 6)
 #define   LANE_STAGGER_MASK		0x1F
-#define BXT_PORT_PCS_DW12_LN01(port)	_PORT3(port, _PORT_PCS_DW12_LN01_A, \
+#define BXT_PORT_PCS_DW12_LN01(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_LN01_A, \
 						     _PORT_PCS_DW12_LN01_B, \
 						     _PORT_PCS_DW12_LN01_C)
-#define BXT_PORT_PCS_DW12_LN23(port)	_PORT3(port, _PORT_PCS_DW12_LN23_A, \
+#define BXT_PORT_PCS_DW12_LN23(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_LN23_A, \
 						     _PORT_PCS_DW12_LN23_B, \
 						     _PORT_PCS_DW12_LN23_C)
-#define BXT_PORT_PCS_DW12_GRP(port)	_PORT3(port, _PORT_PCS_DW12_GRP_A, \
+#define BXT_PORT_PCS_DW12_GRP(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_GRP_A, \
 						     _PORT_PCS_DW12_GRP_B, \
 						     _PORT_PCS_DW12_GRP_C)
 
@@ -1377,10 +1428,10 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW2_GRP_A		0x162D08
 #define _PORT_TX_DW2_GRP_B		0x6CD08
 #define _PORT_TX_DW2_GRP_C		0x6CF08
-#define BXT_PORT_TX_DW2_GRP(port)	_PORT3(port, _PORT_TX_DW2_GRP_A,  \
+#define BXT_PORT_TX_DW2_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW2_GRP_A,  \
 						     _PORT_TX_DW2_GRP_B,  \
 						     _PORT_TX_DW2_GRP_C)
-#define BXT_PORT_TX_DW2_LN0(port)	_PORT3(port, _PORT_TX_DW2_LN0_A,  \
+#define BXT_PORT_TX_DW2_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW2_LN0_A,  \
 						     _PORT_TX_DW2_LN0_B,  \
 						     _PORT_TX_DW2_LN0_C)
 #define   MARGIN_000_SHIFT		16
@@ -1394,10 +1445,10 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW3_GRP_A		0x162D0C
 #define _PORT_TX_DW3_GRP_B		0x6CD0C
 #define _PORT_TX_DW3_GRP_C		0x6CF0C
-#define BXT_PORT_TX_DW3_GRP(port)	_PORT3(port, _PORT_TX_DW3_GRP_A,  \
+#define BXT_PORT_TX_DW3_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW3_GRP_A,  \
 						     _PORT_TX_DW3_GRP_B,  \
 						     _PORT_TX_DW3_GRP_C)
-#define BXT_PORT_TX_DW3_LN0(port)	_PORT3(port, _PORT_TX_DW3_LN0_A,  \
+#define BXT_PORT_TX_DW3_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW3_LN0_A,  \
 						     _PORT_TX_DW3_LN0_B,  \
 						     _PORT_TX_DW3_LN0_C)
 #define   SCALE_DCOMP_METHOD		(1 << 26)
@@ -1409,10 +1460,10 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW4_GRP_A		0x162D10
 #define _PORT_TX_DW4_GRP_B		0x6CD10
 #define _PORT_TX_DW4_GRP_C		0x6CF10
-#define BXT_PORT_TX_DW4_LN0(port)	_PORT3(port, _PORT_TX_DW4_LN0_A,  \
+#define BXT_PORT_TX_DW4_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW4_LN0_A,  \
 						     _PORT_TX_DW4_LN0_B,  \
 						     _PORT_TX_DW4_LN0_C)
-#define BXT_PORT_TX_DW4_GRP(port)	_PORT3(port, _PORT_TX_DW4_GRP_A,  \
+#define BXT_PORT_TX_DW4_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW4_GRP_A,  \
 						     _PORT_TX_DW4_GRP_B,  \
 						     _PORT_TX_DW4_GRP_C)
 #define   DEEMPH_SHIFT			24
@@ -1423,17 +1474,17 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW14_LN0_C		0x6C938
 #define   LATENCY_OPTIM_SHIFT		30
 #define   LATENCY_OPTIM			(1 << LATENCY_OPTIM_SHIFT)
-#define BXT_PORT_TX_DW14_LN(port, lane)	(_PORT3((port), _PORT_TX_DW14_LN0_A,   \
+#define BXT_PORT_TX_DW14_LN(port, lane)	_MMIO(_PORT3((port), _PORT_TX_DW14_LN0_A,   \
 							_PORT_TX_DW14_LN0_B,   \
 							_PORT_TX_DW14_LN0_C) + \
 					 _BXT_LANE_OFFSET(lane))
 
 /* UAIMI scratch pad register 1 */
-#define UAIMI_SPR1			0x4F074
+#define UAIMI_SPR1			_MMIO(0x4F074)
 /* SKL VccIO mask */
 #define SKL_VCCIO_MASK			0x1
 /* SKL balance leg register */
-#define DISPIO_CR_TX_BMU_CR0		0x6C00C
+#define DISPIO_CR_TX_BMU_CR0		_MMIO(0x6C00C)
 /* I_boost values */
 #define BALANCE_LEG_SHIFT(port)		(8+3*(port))
 #define BALANCE_LEG_MASK(port)		(7<<(8+3*(port)))
@@ -1450,7 +1501,7 @@ enum skl_disp_power_wells {
  * [0-15] @ 0x100000 gen6,vlv,chv
  * [0-31] @ 0x100000 gen7+
  */
-#define FENCE_REG(i)			(0x2000 + (((i) & 8) << 9) + ((i) & 7) * 4)
+#define FENCE_REG(i)			_MMIO(0x2000 + (((i) & 8) << 9) + ((i) & 7) * 4)
 #define   I830_FENCE_START_MASK		0x07f80000
 #define   I830_FENCE_TILING_Y_SHIFT	12
 #define   I830_FENCE_SIZE_BITS(size)	((ffs((size) >> 19) - 1) << 8)
@@ -1463,21 +1514,21 @@ enum skl_disp_power_wells {
 #define   I915_FENCE_START_MASK		0x0ff00000
 #define   I915_FENCE_SIZE_BITS(size)	((ffs((size) >> 20) - 1) << 8)
 
-#define FENCE_REG_965_LO(i)		(0x03000 + (i) * 8)
-#define FENCE_REG_965_HI(i)		(0x03000 + (i) * 8 + 4)
+#define FENCE_REG_965_LO(i)		_MMIO(0x03000 + (i) * 8)
+#define FENCE_REG_965_HI(i)		_MMIO(0x03000 + (i) * 8 + 4)
 #define   I965_FENCE_PITCH_SHIFT	2
 #define   I965_FENCE_TILING_Y_SHIFT	1
 #define   I965_FENCE_REG_VALID		(1<<0)
 #define   I965_FENCE_MAX_PITCH_VAL	0x0400
 
-#define FENCE_REG_GEN6_LO(i)	(0x100000 + (i) * 8)
-#define FENCE_REG_GEN6_HI(i)	(0x100000 + (i) * 8 + 4)
+#define FENCE_REG_GEN6_LO(i)		_MMIO(0x100000 + (i) * 8)
+#define FENCE_REG_GEN6_HI(i)		_MMIO(0x100000 + (i) * 8 + 4)
 #define   GEN6_FENCE_PITCH_SHIFT	32
 #define   GEN7_FENCE_MAX_PITCH_VAL	0x0800
 
 
 /* control register for cpu gtt access */
-#define TILECTL				0x101000
+#define TILECTL				_MMIO(0x101000)
 #define   TILECTL_SWZCTL			(1 << 0)
 #define   TILECTL_TLBPF			(1 << 1)
 #define   TILECTL_TLB_PREFETCH_DIS	(1 << 2)
@@ -1486,30 +1537,30 @@ enum skl_disp_power_wells {
 /*
  * Instruction and interrupt control regs
  */
-#define PGTBL_CTL	0x02020
+#define PGTBL_CTL	_MMIO(0x02020)
 #define   PGTBL_ADDRESS_LO_MASK	0xfffff000 /* bits [31:12] */
 #define   PGTBL_ADDRESS_HI_MASK	0x000000f0 /* bits [35:32] (gen4) */
-#define PGTBL_ER	0x02024
-#define PRB0_BASE (0x2030-0x30)
-#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */
-#define PRB2_BASE (0x2050-0x30) /* gen3 */
-#define SRB0_BASE (0x2100-0x30) /* gen2 */
-#define SRB1_BASE (0x2110-0x30) /* gen2 */
-#define SRB2_BASE (0x2120-0x30) /* 830 */
-#define SRB3_BASE (0x2130-0x30) /* 830 */
+#define PGTBL_ER	_MMIO(0x02024)
+#define PRB0_BASE	(0x2030-0x30)
+#define PRB1_BASE	(0x2040-0x30) /* 830,gen3 */
+#define PRB2_BASE	(0x2050-0x30) /* gen3 */
+#define SRB0_BASE	(0x2100-0x30) /* gen2 */
+#define SRB1_BASE	(0x2110-0x30) /* gen2 */
+#define SRB2_BASE	(0x2120-0x30) /* 830 */
+#define SRB3_BASE	(0x2130-0x30) /* 830 */
 #define RENDER_RING_BASE	0x02000
 #define BSD_RING_BASE		0x04000
 #define GEN6_BSD_RING_BASE	0x12000
 #define GEN8_BSD2_RING_BASE	0x1c000
 #define VEBOX_RING_BASE		0x1a000
 #define BLT_RING_BASE		0x22000
-#define RING_TAIL(base)		((base)+0x30)
-#define RING_HEAD(base)		((base)+0x34)
-#define RING_START(base)	((base)+0x38)
-#define RING_CTL(base)		((base)+0x3c)
-#define RING_SYNC_0(base)	((base)+0x40)
-#define RING_SYNC_1(base)	((base)+0x44)
-#define RING_SYNC_2(base)	((base)+0x48)
+#define RING_TAIL(base)		_MMIO((base)+0x30)
+#define RING_HEAD(base)		_MMIO((base)+0x34)
+#define RING_START(base)	_MMIO((base)+0x38)
+#define RING_CTL(base)		_MMIO((base)+0x3c)
+#define RING_SYNC_0(base)	_MMIO((base)+0x40)
+#define RING_SYNC_1(base)	_MMIO((base)+0x44)
+#define RING_SYNC_2(base)	_MMIO((base)+0x48)
 #define GEN6_RVSYNC	(RING_SYNC_0(RENDER_RING_BASE))
 #define GEN6_RBSYNC	(RING_SYNC_1(RENDER_RING_BASE))
 #define GEN6_RVESYNC	(RING_SYNC_2(RENDER_RING_BASE))
@@ -1522,51 +1573,52 @@ enum skl_disp_power_wells {
 #define GEN6_VEBSYNC	(RING_SYNC_0(VEBOX_RING_BASE))
 #define GEN6_VERSYNC	(RING_SYNC_1(VEBOX_RING_BASE))
 #define GEN6_VEVSYNC	(RING_SYNC_2(VEBOX_RING_BASE))
-#define GEN6_NOSYNC 0
-#define RING_PSMI_CTL(base)	((base)+0x50)
-#define RING_MAX_IDLE(base)	((base)+0x54)
-#define RING_HWS_PGA(base)	((base)+0x80)
-#define RING_HWS_PGA_GEN6(base)	((base)+0x2080)
-#define RING_RESET_CTL(base)	((base)+0xd0)
+#define GEN6_NOSYNC	INVALID_MMIO_REG
+#define RING_PSMI_CTL(base)	_MMIO((base)+0x50)
+#define RING_MAX_IDLE(base)	_MMIO((base)+0x54)
+#define RING_HWS_PGA(base)	_MMIO((base)+0x80)
+#define RING_HWS_PGA_GEN6(base)	_MMIO((base)+0x2080)
+#define RING_RESET_CTL(base)	_MMIO((base)+0xd0)
 #define   RESET_CTL_REQUEST_RESET  (1 << 0)
 #define   RESET_CTL_READY_TO_RESET (1 << 1)
 
-#define HSW_GTT_CACHE_EN	0x4024
+#define HSW_GTT_CACHE_EN	_MMIO(0x4024)
 #define   GTT_CACHE_EN_ALL	0xF0007FFF
-#define GEN7_WR_WATERMARK	0x4028
-#define GEN7_GFX_PRIO_CTRL	0x402C
-#define ARB_MODE		0x4030
+#define GEN7_WR_WATERMARK	_MMIO(0x4028)
+#define GEN7_GFX_PRIO_CTRL	_MMIO(0x402C)
+#define ARB_MODE		_MMIO(0x4030)
 #define   ARB_MODE_SWIZZLE_SNB	(1<<4)
 #define   ARB_MODE_SWIZZLE_IVB	(1<<5)
-#define GEN7_GFX_PEND_TLB0	0x4034
-#define GEN7_GFX_PEND_TLB1	0x4038
+#define GEN7_GFX_PEND_TLB0	_MMIO(0x4034)
+#define GEN7_GFX_PEND_TLB1	_MMIO(0x4038)
 /* L3, CVS, ZTLB, RCC, CASC LRA min, max values */
-#define GEN7_LRA_LIMITS(i)	(0x403C + (i) * 4)
+#define GEN7_LRA_LIMITS(i)	_MMIO(0x403C + (i) * 4)
 #define GEN7_LRA_LIMITS_REG_NUM	13
-#define GEN7_MEDIA_MAX_REQ_COUNT	0x4070
-#define GEN7_GFX_MAX_REQ_COUNT		0x4074
+#define GEN7_MEDIA_MAX_REQ_COUNT	_MMIO(0x4070)
+#define GEN7_GFX_MAX_REQ_COUNT		_MMIO(0x4074)
 
-#define GAMTARBMODE		0x04a08
+#define GAMTARBMODE		_MMIO(0x04a08)
 #define   ARB_MODE_BWGTLB_DISABLE (1<<9)
 #define   ARB_MODE_SWIZZLE_BDW	(1<<1)
-#define RENDER_HWS_PGA_GEN7	(0x04080)
-#define RING_FAULT_REG(ring)	(0x4094 + 0x100*(ring)->id)
+#define RENDER_HWS_PGA_GEN7	_MMIO(0x04080)
+#define RING_FAULT_REG(ring)	_MMIO(0x4094 + 0x100*(ring)->id)
 #define   RING_FAULT_GTTSEL_MASK (1<<11)
 #define   RING_FAULT_SRCID(x)	(((x) >> 3) & 0xff)
 #define   RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3)
 #define   RING_FAULT_VALID	(1<<0)
-#define DONE_REG		0x40b0
-#define GEN8_PRIVATE_PAT_LO	0x40e0
-#define GEN8_PRIVATE_PAT_HI	(0x40e0 + 4)
-#define BSD_HWS_PGA_GEN7	(0x04180)
-#define BLT_HWS_PGA_GEN7	(0x04280)
-#define VEBOX_HWS_PGA_GEN7	(0x04380)
-#define RING_ACTHD(base)	((base)+0x74)
-#define RING_ACTHD_UDW(base)	((base)+0x5c)
-#define RING_NOPID(base)	((base)+0x94)
-#define RING_IMR(base)		((base)+0xa8)
-#define RING_HWSTAM(base)	((base)+0x98)
-#define RING_TIMESTAMP(base)	((base)+0x358)
+#define DONE_REG		_MMIO(0x40b0)
+#define GEN8_PRIVATE_PAT_LO	_MMIO(0x40e0)
+#define GEN8_PRIVATE_PAT_HI	_MMIO(0x40e0 + 4)
+#define BSD_HWS_PGA_GEN7	_MMIO(0x04180)
+#define BLT_HWS_PGA_GEN7	_MMIO(0x04280)
+#define VEBOX_HWS_PGA_GEN7	_MMIO(0x04380)
+#define RING_ACTHD(base)	_MMIO((base)+0x74)
+#define RING_ACTHD_UDW(base)	_MMIO((base)+0x5c)
+#define RING_NOPID(base)	_MMIO((base)+0x94)
+#define RING_IMR(base)		_MMIO((base)+0xa8)
+#define RING_HWSTAM(base)	_MMIO((base)+0x98)
+#define RING_TIMESTAMP(base)		_MMIO((base)+0x358)
+#define RING_TIMESTAMP_UDW(base)	_MMIO((base)+0x358 + 4)
 #define   TAIL_ADDR		0x001FFFF8
 #define   HEAD_WRAP_COUNT	0xFFE00000
 #define   HEAD_WRAP_ONE		0x00200000
@@ -1583,57 +1635,65 @@ enum skl_disp_power_wells {
 #define   RING_WAIT		(1<<11) /* gen3+, PRBx_CTL */
 #define   RING_WAIT_SEMAPHORE	(1<<10) /* gen6+ */
 
-#define GEN7_TLB_RD_ADDR	0x4700
+#define GEN7_TLB_RD_ADDR	_MMIO(0x4700)
 
 #if 0
-#define PRB0_TAIL	0x02030
-#define PRB0_HEAD	0x02034
-#define PRB0_START	0x02038
-#define PRB0_CTL	0x0203c
-#define PRB1_TAIL	0x02040 /* 915+ only */
-#define PRB1_HEAD	0x02044 /* 915+ only */
-#define PRB1_START	0x02048 /* 915+ only */
-#define PRB1_CTL	0x0204c /* 915+ only */
+#define PRB0_TAIL	_MMIO(0x2030)
+#define PRB0_HEAD	_MMIO(0x2034)
+#define PRB0_START	_MMIO(0x2038)
+#define PRB0_CTL	_MMIO(0x203c)
+#define PRB1_TAIL	_MMIO(0x2040) /* 915+ only */
+#define PRB1_HEAD	_MMIO(0x2044) /* 915+ only */
+#define PRB1_START	_MMIO(0x2048) /* 915+ only */
+#define PRB1_CTL	_MMIO(0x204c) /* 915+ only */
 #endif
-#define IPEIR_I965	0x02064
-#define IPEHR_I965	0x02068
-#define GEN7_SC_INSTDONE	0x07100
-#define GEN7_SAMPLER_INSTDONE	0x0e160
-#define GEN7_ROW_INSTDONE	0x0e164
+#define IPEIR_I965	_MMIO(0x2064)
+#define IPEHR_I965	_MMIO(0x2068)
+#define GEN7_SC_INSTDONE	_MMIO(0x7100)
+#define GEN7_SAMPLER_INSTDONE	_MMIO(0xe160)
+#define GEN7_ROW_INSTDONE	_MMIO(0xe164)
 #define I915_NUM_INSTDONE_REG	4
-#define RING_IPEIR(base)	((base)+0x64)
-#define RING_IPEHR(base)	((base)+0x68)
+#define RING_IPEIR(base)	_MMIO((base)+0x64)
+#define RING_IPEHR(base)	_MMIO((base)+0x68)
 /*
  * On GEN4, only the render ring INSTDONE exists and has a different
  * layout than the GEN7+ version.
  * The GEN2 counterpart of this register is GEN2_INSTDONE.
  */
-#define RING_INSTDONE(base)	((base)+0x6c)
-#define RING_INSTPS(base)	((base)+0x70)
-#define RING_DMA_FADD(base)	((base)+0x78)
-#define RING_DMA_FADD_UDW(base)	((base)+0x60) /* gen8+ */
-#define RING_INSTPM(base)	((base)+0xc0)
-#define RING_MI_MODE(base)	((base)+0x9c)
-#define INSTPS		0x02070 /* 965+ only */
-#define GEN4_INSTDONE1	0x0207c /* 965+ only, aka INSTDONE_2 on SNB */
-#define ACTHD_I965	0x02074
-#define HWS_PGA		0x02080
+#define RING_INSTDONE(base)	_MMIO((base)+0x6c)
+#define RING_INSTPS(base)	_MMIO((base)+0x70)
+#define RING_DMA_FADD(base)	_MMIO((base)+0x78)
+#define RING_DMA_FADD_UDW(base)	_MMIO((base)+0x60) /* gen8+ */
+#define RING_INSTPM(base)	_MMIO((base)+0xc0)
+#define RING_MI_MODE(base)	_MMIO((base)+0x9c)
+#define INSTPS		_MMIO(0x2070) /* 965+ only */
+#define GEN4_INSTDONE1	_MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */
+#define ACTHD_I965	_MMIO(0x2074)
+#define HWS_PGA		_MMIO(0x2080)
 #define HWS_ADDRESS_MASK	0xfffff000
 #define HWS_START_ADDRESS_SHIFT	4
-#define PWRCTXA		0x2088 /* 965GM+ only */
+#define PWRCTXA		_MMIO(0x2088) /* 965GM+ only */
 #define   PWRCTX_EN	(1<<0)
-#define IPEIR		0x02088
-#define IPEHR		0x0208c
-#define GEN2_INSTDONE	0x02090
-#define NOPID		0x02094
-#define HWSTAM		0x02098
-#define DMA_FADD_I8XX	0x020d0
-#define RING_BBSTATE(base)	((base)+0x110)
-#define RING_BBADDR(base)	((base)+0x140)
-#define RING_BBADDR_UDW(base)	((base)+0x168) /* gen8+ */
-
-#define ERROR_GEN6	0x040a0
-#define GEN7_ERR_INT	0x44040
+#define IPEIR		_MMIO(0x2088)
+#define IPEHR		_MMIO(0x208c)
+#define GEN2_INSTDONE	_MMIO(0x2090)
+#define NOPID		_MMIO(0x2094)
+#define HWSTAM		_MMIO(0x2098)
+#define DMA_FADD_I8XX	_MMIO(0x20d0)
+#define RING_BBSTATE(base)	_MMIO((base)+0x110)
+#define   RING_BB_PPGTT		(1 << 5)
+#define RING_SBBADDR(base)	_MMIO((base)+0x114) /* hsw+ */
+#define RING_SBBSTATE(base)	_MMIO((base)+0x118) /* hsw+ */
+#define RING_SBBADDR_UDW(base)	_MMIO((base)+0x11c) /* gen8+ */
+#define RING_BBADDR(base)	_MMIO((base)+0x140)
+#define RING_BBADDR_UDW(base)	_MMIO((base)+0x168) /* gen8+ */
+#define RING_BB_PER_CTX_PTR(base)	_MMIO((base)+0x1c0) /* gen8+ */
+#define RING_INDIRECT_CTX(base)		_MMIO((base)+0x1c4) /* gen8+ */
+#define RING_INDIRECT_CTX_OFFSET(base)	_MMIO((base)+0x1c8) /* gen8+ */
+#define RING_CTX_TIMESTAMP(base)	_MMIO((base)+0x3a8) /* gen8+ */
+
+#define ERROR_GEN6	_MMIO(0x40a0)
+#define GEN7_ERR_INT	_MMIO(0x44040)
 #define   ERR_INT_POISON		(1<<31)
 #define   ERR_INT_MMIO_UNCLAIMED	(1<<13)
 #define   ERR_INT_PIPE_CRC_DONE_C	(1<<8)
@@ -1645,13 +1705,13 @@ enum skl_disp_power_wells {
 #define   ERR_INT_FIFO_UNDERRUN_A	(1<<0)
 #define   ERR_INT_FIFO_UNDERRUN(pipe)	(1<<((pipe)*3))
 
-#define GEN8_FAULT_TLB_DATA0		0x04b10
-#define GEN8_FAULT_TLB_DATA1		0x04b14
+#define GEN8_FAULT_TLB_DATA0		_MMIO(0x4b10)
+#define GEN8_FAULT_TLB_DATA1		_MMIO(0x4b14)
 
-#define FPGA_DBG		0x42300
+#define FPGA_DBG		_MMIO(0x42300)
 #define   FPGA_DBG_RM_NOCLAIM	(1<<31)
 
-#define DERRMR		0x44050
+#define DERRMR		_MMIO(0x44050)
 /* Note that HBLANK events are reserved on bdw+ */
 #define   DERRMR_PIPEA_SCANLINE		(1<<0)
 #define   DERRMR_PIPEA_PRI_FLIP_DONE	(1<<1)
@@ -1675,29 +1735,29 @@ enum skl_disp_power_wells {
  * for various sorts of correct behavior.  The top 16 bits of each are
  * the enables for writing to the corresponding low bit.
  */
-#define _3D_CHICKEN	0x02084
+#define _3D_CHICKEN	_MMIO(0x2084)
 #define  _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB	(1 << 10)
-#define _3D_CHICKEN2	0x0208c
+#define _3D_CHICKEN2	_MMIO(0x208c)
 /* Disables pipelining of read flushes past the SF-WIZ interface.
  * Required on all Ironlake steppings according to the B-Spec, but the
  * particular danger of not doing so is not specified.
  */
 # define _3D_CHICKEN2_WM_READ_PIPELINED			(1 << 14)
-#define _3D_CHICKEN3	0x02090
+#define _3D_CHICKEN3	_MMIO(0x2090)
 #define  _3D_CHICKEN_SF_DISABLE_OBJEND_CULL		(1 << 10)
 #define  _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL		(1 << 5)
 #define  _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x)	((x)<<1) /* gen8+ */
 #define  _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH	(1 << 1) /* gen6 */
 
-#define MI_MODE		0x0209c
+#define MI_MODE		_MMIO(0x209c)
 # define VS_TIMER_DISPATCH				(1 << 6)
 # define MI_FLUSH_ENABLE				(1 << 12)
 # define ASYNC_FLIP_PERF_DISABLE			(1 << 14)
 # define MODE_IDLE					(1 << 9)
 # define STOP_RING					(1 << 8)
 
-#define GEN6_GT_MODE	0x20d0
-#define GEN7_GT_MODE	0x7008
+#define GEN6_GT_MODE	_MMIO(0x20d0)
+#define GEN7_GT_MODE	_MMIO(0x7008)
 #define   GEN6_WIZ_HASHING(hi, lo)			(((hi) << 9) | ((lo) << 7))
 #define   GEN6_WIZ_HASHING_8x8				GEN6_WIZ_HASHING(0, 0)
 #define   GEN6_WIZ_HASHING_8x4				GEN6_WIZ_HASHING(0, 1)
@@ -1707,9 +1767,9 @@ enum skl_disp_power_wells {
 #define   GEN9_IZ_HASHING_MASK(slice)			(0x3 << ((slice) * 2))
 #define   GEN9_IZ_HASHING(slice, val)			((val) << ((slice) * 2))
 
-#define GFX_MODE	0x02520
-#define GFX_MODE_GEN7	0x0229c
-#define RING_MODE_GEN7(ring)	((ring)->mmio_base+0x29c)
+#define GFX_MODE	_MMIO(0x2520)
+#define GFX_MODE_GEN7	_MMIO(0x229c)
+#define RING_MODE_GEN7(ring)	_MMIO((ring)->mmio_base+0x29c)
 #define   GFX_RUN_LIST_ENABLE		(1<<15)
 #define   GFX_INTERRUPT_STEERING	(1<<14)
 #define   GFX_TLB_INVALIDATE_EXPLICIT	(1<<13)
@@ -1727,36 +1787,36 @@ enum skl_disp_power_wells {
 #define VLV_DISPLAY_BASE 0x180000
 #define VLV_MIPI_BASE VLV_DISPLAY_BASE
 
-#define VLV_GU_CTL0	(VLV_DISPLAY_BASE + 0x2030)
-#define VLV_GU_CTL1	(VLV_DISPLAY_BASE + 0x2034)
-#define SCPD0		0x0209c /* 915+ only */
-#define IER		0x020a0
-#define IIR		0x020a4
-#define IMR		0x020a8
-#define ISR		0x020ac
-#define VLV_GUNIT_CLOCK_GATE	(VLV_DISPLAY_BASE + 0x2060)
+#define VLV_GU_CTL0	_MMIO(VLV_DISPLAY_BASE + 0x2030)
+#define VLV_GU_CTL1	_MMIO(VLV_DISPLAY_BASE + 0x2034)
+#define SCPD0		_MMIO(0x209c) /* 915+ only */
+#define IER		_MMIO(0x20a0)
+#define IIR		_MMIO(0x20a4)
+#define IMR		_MMIO(0x20a8)
+#define ISR		_MMIO(0x20ac)
+#define VLV_GUNIT_CLOCK_GATE	_MMIO(VLV_DISPLAY_BASE + 0x2060)
 #define   GINT_DIS		(1<<22)
 #define   GCFG_DIS		(1<<8)
-#define VLV_GUNIT_CLOCK_GATE2	(VLV_DISPLAY_BASE + 0x2064)
-#define VLV_IIR_RW	(VLV_DISPLAY_BASE + 0x2084)
-#define VLV_IER		(VLV_DISPLAY_BASE + 0x20a0)
-#define VLV_IIR		(VLV_DISPLAY_BASE + 0x20a4)
-#define VLV_IMR		(VLV_DISPLAY_BASE + 0x20a8)
-#define VLV_ISR		(VLV_DISPLAY_BASE + 0x20ac)
-#define VLV_PCBR	(VLV_DISPLAY_BASE + 0x2120)
+#define VLV_GUNIT_CLOCK_GATE2	_MMIO(VLV_DISPLAY_BASE + 0x2064)
+#define VLV_IIR_RW	_MMIO(VLV_DISPLAY_BASE + 0x2084)
+#define VLV_IER		_MMIO(VLV_DISPLAY_BASE + 0x20a0)
+#define VLV_IIR		_MMIO(VLV_DISPLAY_BASE + 0x20a4)
+#define VLV_IMR		_MMIO(VLV_DISPLAY_BASE + 0x20a8)
+#define VLV_ISR		_MMIO(VLV_DISPLAY_BASE + 0x20ac)
+#define VLV_PCBR	_MMIO(VLV_DISPLAY_BASE + 0x2120)
 #define VLV_PCBR_ADDR_SHIFT	12
 
 #define   DISPLAY_PLANE_FLIP_PENDING(plane) (1<<(11-(plane))) /* A and B only */
-#define EIR		0x020b0
-#define EMR		0x020b4
-#define ESR		0x020b8
+#define EIR		_MMIO(0x20b0)
+#define EMR		_MMIO(0x20b4)
+#define ESR		_MMIO(0x20b8)
 #define   GM45_ERROR_PAGE_TABLE				(1<<5)
 #define   GM45_ERROR_MEM_PRIV				(1<<4)
 #define   I915_ERROR_PAGE_TABLE				(1<<4)
 #define   GM45_ERROR_CP_PRIV				(1<<3)
 #define   I915_ERROR_MEMORY_REFRESH			(1<<1)
 #define   I915_ERROR_INSTRUCTION			(1<<0)
-#define INSTPM	        0x020c0
+#define INSTPM	        _MMIO(0x20c0)
 #define   INSTPM_SELF_EN (1<<12) /* 915GM only */
 #define   INSTPM_AGPBUSY_INT_EN (1<<11) /* gen3: when disabled, pending interrupts
 					will not assert AGPBUSY# and will only
@@ -1764,14 +1824,14 @@ enum skl_disp_power_wells {
 #define   INSTPM_FORCE_ORDERING				(1<<7) /* GEN6+ */
 #define   INSTPM_TLB_INVALIDATE	(1<<9)
 #define   INSTPM_SYNC_FLUSH	(1<<5)
-#define ACTHD	        0x020c8
-#define MEM_MODE	0x020cc
+#define ACTHD	        _MMIO(0x20c8)
+#define MEM_MODE	_MMIO(0x20cc)
 #define   MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1<<3) /* 830 only */
 #define   MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1<<2) /* 830/845 only */
 #define   MEM_DISPLAY_TRICKLE_FEED_DISABLE (1<<2) /* 85x only */
-#define FW_BLC		0x020d8
-#define FW_BLC2		0x020dc
-#define FW_BLC_SELF	0x020e0 /* 915+ only */
+#define FW_BLC		_MMIO(0x20d8)
+#define FW_BLC2		_MMIO(0x20dc)
+#define FW_BLC_SELF	_MMIO(0x20e0) /* 915+ only */
 #define   FW_BLC_SELF_EN_MASK      (1<<31)
 #define   FW_BLC_SELF_FIFO_MASK    (1<<16) /* 945 only */
 #define   FW_BLC_SELF_EN           (1<<15) /* 945 only */
@@ -1779,7 +1839,7 @@ enum skl_disp_power_wells {
 #define MM_FIFO_WATERMARK   0x0001F000
 #define LM_BURST_LENGTH     0x00000700
 #define LM_FIFO_WATERMARK   0x0000001F
-#define MI_ARB_STATE	0x020e4 /* 915+ only */
+#define MI_ARB_STATE	_MMIO(0x20e4) /* 915+ only */
 
 /* Make render/texture TLB fetches lower priorty than associated data
  *   fetches. This is not turned on by default
@@ -1843,11 +1903,11 @@ enum skl_disp_power_wells {
 #define   MI_ARB_DISPLAY_PRIORITY_A_B		(0 << 0)	/* display A > display B */
 #define   MI_ARB_DISPLAY_PRIORITY_B_A		(1 << 0)	/* display B > display A */
 
-#define MI_STATE	0x020e4 /* gen2 only */
+#define MI_STATE	_MMIO(0x20e4) /* gen2 only */
 #define   MI_AGPBUSY_INT_EN			(1 << 1) /* 85x only */
 #define   MI_AGPBUSY_830_MODE			(1 << 0) /* 85x only */
 
-#define CACHE_MODE_0	0x02120 /* 915+ only */
+#define CACHE_MODE_0	_MMIO(0x2120) /* 915+ only */
 #define   CM0_PIPELINED_RENDER_FLUSH_DISABLE (1<<8)
 #define   CM0_IZ_OPT_DISABLE      (1<<6)
 #define   CM0_ZR_OPT_DISABLE      (1<<5)
@@ -1856,32 +1916,32 @@ enum skl_disp_power_wells {
 #define   CM0_COLOR_EVICT_DISABLE (1<<3)
 #define   CM0_DEPTH_WRITE_DISABLE (1<<1)
 #define   CM0_RC_OP_FLUSH_DISABLE (1<<0)
-#define GFX_FLSH_CNTL	0x02170 /* 915+ only */
-#define GFX_FLSH_CNTL_GEN6	0x101008
+#define GFX_FLSH_CNTL	_MMIO(0x2170) /* 915+ only */
+#define GFX_FLSH_CNTL_GEN6	_MMIO(0x101008)
 #define   GFX_FLSH_CNTL_EN	(1<<0)
-#define ECOSKPD		0x021d0
+#define ECOSKPD		_MMIO(0x21d0)
 #define   ECO_GATING_CX_ONLY	(1<<3)
 #define   ECO_FLIP_DONE		(1<<0)
 
-#define CACHE_MODE_0_GEN7	0x7000 /* IVB+ */
+#define CACHE_MODE_0_GEN7	_MMIO(0x7000) /* IVB+ */
 #define RC_OP_FLUSH_ENABLE (1<<0)
 #define   HIZ_RAW_STALL_OPT_DISABLE (1<<2)
-#define CACHE_MODE_1		0x7004 /* IVB+ */
+#define CACHE_MODE_1		_MMIO(0x7004) /* IVB+ */
 #define   PIXEL_SUBSPAN_COLLECT_OPT_DISABLE	(1<<6)
 #define   GEN8_4x4_STC_OPTIMIZATION_DISABLE	(1<<6)
 #define   GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE	(1<<1)
 
-#define GEN6_BLITTER_ECOSKPD	0x221d0
+#define GEN6_BLITTER_ECOSKPD	_MMIO(0x221d0)
 #define   GEN6_BLITTER_LOCK_SHIFT			16
 #define   GEN6_BLITTER_FBC_NOTIFY			(1<<3)
 
-#define GEN6_RC_SLEEP_PSMI_CONTROL	0x2050
+#define GEN6_RC_SLEEP_PSMI_CONTROL	_MMIO(0x2050)
 #define   GEN6_PSMI_SLEEP_MSG_DISABLE	(1 << 0)
 #define   GEN8_RC_SEMA_IDLE_MSG_DISABLE	(1 << 12)
 #define   GEN8_FF_DOP_CLOCK_GATE_DISABLE	(1<<10)
 
 /* Fuse readout registers for GT */
-#define CHV_FUSE_GT			(VLV_DISPLAY_BASE + 0x2168)
+#define CHV_FUSE_GT			_MMIO(VLV_DISPLAY_BASE + 0x2168)
 #define   CHV_FGT_DISABLE_SS0		(1 << 10)
 #define   CHV_FGT_DISABLE_SS1		(1 << 11)
 #define   CHV_FGT_EU_DIS_SS0_R0_SHIFT	16
@@ -1893,7 +1953,7 @@ enum skl_disp_power_wells {
 #define   CHV_FGT_EU_DIS_SS1_R1_SHIFT	28
 #define   CHV_FGT_EU_DIS_SS1_R1_MASK	(0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT)
 
-#define GEN8_FUSE2			0x9120
+#define GEN8_FUSE2			_MMIO(0x9120)
 #define   GEN8_F2_SS_DIS_SHIFT		21
 #define   GEN8_F2_SS_DIS_MASK		(0x7 << GEN8_F2_SS_DIS_SHIFT)
 #define   GEN8_F2_S_ENA_SHIFT		25
@@ -1902,22 +1962,22 @@ enum skl_disp_power_wells {
 #define   GEN9_F2_SS_DIS_SHIFT		20
 #define   GEN9_F2_SS_DIS_MASK		(0xf << GEN9_F2_SS_DIS_SHIFT)
 
-#define GEN8_EU_DISABLE0		0x9134
+#define GEN8_EU_DISABLE0		_MMIO(0x9134)
 #define   GEN8_EU_DIS0_S0_MASK		0xffffff
 #define   GEN8_EU_DIS0_S1_SHIFT		24
 #define   GEN8_EU_DIS0_S1_MASK		(0xff << GEN8_EU_DIS0_S1_SHIFT)
 
-#define GEN8_EU_DISABLE1		0x9138
+#define GEN8_EU_DISABLE1		_MMIO(0x9138)
 #define   GEN8_EU_DIS1_S1_MASK		0xffff
 #define   GEN8_EU_DIS1_S2_SHIFT		16
 #define   GEN8_EU_DIS1_S2_MASK		(0xffff << GEN8_EU_DIS1_S2_SHIFT)
 
-#define GEN8_EU_DISABLE2		0x913c
+#define GEN8_EU_DISABLE2		_MMIO(0x913c)
 #define   GEN8_EU_DIS2_S2_MASK		0xff
 
-#define GEN9_EU_DISABLE(slice)		(0x9134 + (slice)*0x4)
+#define GEN9_EU_DISABLE(slice)		_MMIO(0x9134 + (slice)*0x4)
 
-#define GEN6_BSD_SLEEP_PSMI_CONTROL	0x12050
+#define GEN6_BSD_SLEEP_PSMI_CONTROL	_MMIO(0x12050)
 #define   GEN6_BSD_SLEEP_MSG_DISABLE	(1 << 0)
 #define   GEN6_BSD_SLEEP_FLUSH_DISABLE	(1 << 2)
 #define   GEN6_BSD_SLEEP_INDICATOR	(1 << 3)
@@ -1995,9 +2055,9 @@ enum skl_disp_power_wells {
 #define I915_ASLE_INTERRUPT				(1<<0)
 #define I915_BSD_USER_INTERRUPT				(1<<25)
 
-#define GEN6_BSD_RNCID			0x12198
+#define GEN6_BSD_RNCID			_MMIO(0x12198)
 
-#define GEN7_FF_THREAD_MODE		0x20a0
+#define GEN7_FF_THREAD_MODE		_MMIO(0x20a0)
 #define   GEN7_FF_SCHED_MASK		0x0077070
 #define   GEN8_FF_DS_REF_CNT_FFME	(1 << 19)
 #define   GEN7_FF_TS_SCHED_HS1		(0x5<<16)
@@ -2018,9 +2078,9 @@ enum skl_disp_power_wells {
  * Framebuffer compression (915+ only)
  */
 
-#define FBC_CFB_BASE		0x03200 /* 4k page aligned */
-#define FBC_LL_BASE		0x03204 /* 4k page aligned */
-#define FBC_CONTROL		0x03208
+#define FBC_CFB_BASE		_MMIO(0x3200) /* 4k page aligned */
+#define FBC_LL_BASE		_MMIO(0x3204) /* 4k page aligned */
+#define FBC_CONTROL		_MMIO(0x3208)
 #define   FBC_CTL_EN		(1<<31)
 #define   FBC_CTL_PERIODIC	(1<<30)
 #define   FBC_CTL_INTERVAL_SHIFT (16)
@@ -2028,14 +2088,14 @@ enum skl_disp_power_wells {
 #define   FBC_CTL_C3_IDLE	(1<<13)
 #define   FBC_CTL_STRIDE_SHIFT	(5)
 #define   FBC_CTL_FENCENO_SHIFT	(0)
-#define FBC_COMMAND		0x0320c
+#define FBC_COMMAND		_MMIO(0x320c)
 #define   FBC_CMD_COMPRESS	(1<<0)
-#define FBC_STATUS		0x03210
+#define FBC_STATUS		_MMIO(0x3210)
 #define   FBC_STAT_COMPRESSING	(1<<31)
 #define   FBC_STAT_COMPRESSED	(1<<30)
 #define   FBC_STAT_MODIFIED	(1<<29)
 #define   FBC_STAT_CURRENT_LINE_SHIFT	(0)
-#define FBC_CONTROL2		0x03214
+#define FBC_CONTROL2		_MMIO(0x3214)
 #define   FBC_CTL_FENCE_DBL	(0<<4)
 #define   FBC_CTL_IDLE_IMM	(0<<2)
 #define   FBC_CTL_IDLE_FULL	(1<<2)
@@ -2043,17 +2103,17 @@ enum skl_disp_power_wells {
 #define   FBC_CTL_IDLE_DEBUG	(3<<2)
 #define   FBC_CTL_CPU_FENCE	(1<<1)
 #define   FBC_CTL_PLANE(plane)	((plane)<<0)
-#define FBC_FENCE_OFF		0x03218 /* BSpec typo has 321Bh */
-#define FBC_TAG(i)		(0x03300 + (i) * 4)
+#define FBC_FENCE_OFF		_MMIO(0x3218) /* BSpec typo has 321Bh */
+#define FBC_TAG(i)		_MMIO(0x3300 + (i) * 4)
 
-#define FBC_STATUS2		0x43214
+#define FBC_STATUS2		_MMIO(0x43214)
 #define  FBC_COMPRESSION_MASK	0x7ff
 
 #define FBC_LL_SIZE		(1536)
 
 /* Framebuffer compression for GM45+ */
-#define DPFC_CB_BASE		0x3200
-#define DPFC_CONTROL		0x3208
+#define DPFC_CB_BASE		_MMIO(0x3200)
+#define DPFC_CONTROL		_MMIO(0x3208)
 #define   DPFC_CTL_EN		(1<<31)
 #define   DPFC_CTL_PLANE(plane)	((plane)<<30)
 #define   IVB_DPFC_CTL_PLANE(plane)	((plane)<<29)
@@ -2064,37 +2124,37 @@ enum skl_disp_power_wells {
 #define   DPFC_CTL_LIMIT_1X	(0<<6)
 #define   DPFC_CTL_LIMIT_2X	(1<<6)
 #define   DPFC_CTL_LIMIT_4X	(2<<6)
-#define DPFC_RECOMP_CTL		0x320c
+#define DPFC_RECOMP_CTL		_MMIO(0x320c)
 #define   DPFC_RECOMP_STALL_EN	(1<<27)
 #define   DPFC_RECOMP_STALL_WM_SHIFT (16)
 #define   DPFC_RECOMP_STALL_WM_MASK (0x07ff0000)
 #define   DPFC_RECOMP_TIMER_COUNT_SHIFT (0)
 #define   DPFC_RECOMP_TIMER_COUNT_MASK (0x0000003f)
-#define DPFC_STATUS		0x3210
+#define DPFC_STATUS		_MMIO(0x3210)
 #define   DPFC_INVAL_SEG_SHIFT  (16)
 #define   DPFC_INVAL_SEG_MASK	(0x07ff0000)
 #define   DPFC_COMP_SEG_SHIFT	(0)
 #define   DPFC_COMP_SEG_MASK	(0x000003ff)
-#define DPFC_STATUS2		0x3214
-#define DPFC_FENCE_YOFF		0x3218
-#define DPFC_CHICKEN		0x3224
+#define DPFC_STATUS2		_MMIO(0x3214)
+#define DPFC_FENCE_YOFF		_MMIO(0x3218)
+#define DPFC_CHICKEN		_MMIO(0x3224)
 #define   DPFC_HT_MODIFY	(1<<31)
 
 /* Framebuffer compression for Ironlake */
-#define ILK_DPFC_CB_BASE	0x43200
-#define ILK_DPFC_CONTROL	0x43208
+#define ILK_DPFC_CB_BASE	_MMIO(0x43200)
+#define ILK_DPFC_CONTROL	_MMIO(0x43208)
 #define   FBC_CTL_FALSE_COLOR	(1<<10)
 /* The bit 28-8 is reserved */
 #define   DPFC_RESERVED		(0x1FFFFF00)
-#define ILK_DPFC_RECOMP_CTL	0x4320c
-#define ILK_DPFC_STATUS		0x43210
-#define ILK_DPFC_FENCE_YOFF	0x43218
-#define ILK_DPFC_CHICKEN	0x43224
-#define ILK_FBC_RT_BASE		0x2128
+#define ILK_DPFC_RECOMP_CTL	_MMIO(0x4320c)
+#define ILK_DPFC_STATUS		_MMIO(0x43210)
+#define ILK_DPFC_FENCE_YOFF	_MMIO(0x43218)
+#define ILK_DPFC_CHICKEN	_MMIO(0x43224)
+#define ILK_FBC_RT_BASE		_MMIO(0x2128)
 #define   ILK_FBC_RT_VALID	(1<<0)
 #define   SNB_FBC_FRONT_BUFFER	(1<<1)
 
-#define ILK_DISPLAY_CHICKEN1	0x42000
+#define ILK_DISPLAY_CHICKEN1	_MMIO(0x42000)
 #define   ILK_FBCQ_DIS		(1<<22)
 #define	  ILK_PABSTRETCH_DIS	(1<<21)
 
@@ -2104,31 +2164,31 @@ enum skl_disp_power_wells {
  *
  * The following two registers are of type GTTMMADR
  */
-#define SNB_DPFC_CTL_SA		0x100100
+#define SNB_DPFC_CTL_SA		_MMIO(0x100100)
 #define   SNB_CPU_FENCE_ENABLE	(1<<29)
-#define DPFC_CPU_FENCE_OFFSET	0x100104
+#define DPFC_CPU_FENCE_OFFSET	_MMIO(0x100104)
 
 /* Framebuffer compression for Ivybridge */
-#define IVB_FBC_RT_BASE			0x7020
+#define IVB_FBC_RT_BASE			_MMIO(0x7020)
 
-#define IPS_CTL		0x43408
+#define IPS_CTL		_MMIO(0x43408)
 #define   IPS_ENABLE	(1 << 31)
 
-#define MSG_FBC_REND_STATE	0x50380
+#define MSG_FBC_REND_STATE	_MMIO(0x50380)
 #define   FBC_REND_NUKE		(1<<2)
 #define   FBC_REND_CACHE_CLEAN	(1<<1)
 
 /*
  * GPIO regs
  */
-#define GPIOA			0x5010
-#define GPIOB			0x5014
-#define GPIOC			0x5018
-#define GPIOD			0x501c
-#define GPIOE			0x5020
-#define GPIOF			0x5024
-#define GPIOG			0x5028
-#define GPIOH			0x502c
+#define GPIOA			_MMIO(0x5010)
+#define GPIOB			_MMIO(0x5014)
+#define GPIOC			_MMIO(0x5018)
+#define GPIOD			_MMIO(0x501c)
+#define GPIOE			_MMIO(0x5020)
+#define GPIOF			_MMIO(0x5024)
+#define GPIOG			_MMIO(0x5028)
+#define GPIOH			_MMIO(0x502c)
 # define GPIO_CLOCK_DIR_MASK		(1 << 0)
 # define GPIO_CLOCK_DIR_IN		(0 << 1)
 # define GPIO_CLOCK_DIR_OUT		(1 << 1)
@@ -2144,7 +2204,7 @@ enum skl_disp_power_wells {
 # define GPIO_DATA_VAL_IN		(1 << 12)
 # define GPIO_DATA_PULLUP_DISABLE	(1 << 13)
 
-#define GMBUS0			(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */
+#define GMBUS0			_MMIO(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */
 #define   GMBUS_RATE_100KHZ	(0<<8)
 #define   GMBUS_RATE_50KHZ	(1<<8)
 #define   GMBUS_RATE_400KHZ	(2<<8) /* reserved on Pineview */
@@ -2163,7 +2223,7 @@ enum skl_disp_power_wells {
 #define   GMBUS_PIN_2_BXT	2
 #define   GMBUS_PIN_3_BXT	3
 #define   GMBUS_NUM_PINS	7 /* including 0 */
-#define GMBUS1			(dev_priv->gpio_mmio_base + 0x5104) /* command/status */
+#define GMBUS1			_MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */
 #define   GMBUS_SW_CLR_INT	(1<<31)
 #define   GMBUS_SW_RDY		(1<<30)
 #define   GMBUS_ENT		(1<<29) /* enable timeout */
@@ -2177,7 +2237,7 @@ enum skl_disp_power_wells {
 #define   GMBUS_SLAVE_ADDR_SHIFT 1
 #define   GMBUS_SLAVE_READ	(1<<0)
 #define   GMBUS_SLAVE_WRITE	(0<<0)
-#define GMBUS2			(dev_priv->gpio_mmio_base + 0x5108) /* status */
+#define GMBUS2			_MMIO(dev_priv->gpio_mmio_base + 0x5108) /* status */
 #define   GMBUS_INUSE		(1<<15)
 #define   GMBUS_HW_WAIT_PHASE	(1<<14)
 #define   GMBUS_STALL_TIMEOUT	(1<<13)
@@ -2185,14 +2245,14 @@ enum skl_disp_power_wells {
 #define   GMBUS_HW_RDY		(1<<11)
 #define   GMBUS_SATOER		(1<<10)
 #define   GMBUS_ACTIVE		(1<<9)
-#define GMBUS3			(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */
-#define GMBUS4			(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */
+#define GMBUS3			_MMIO(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */
+#define GMBUS4			_MMIO(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */
 #define   GMBUS_SLAVE_TIMEOUT_EN (1<<4)
 #define   GMBUS_NAK_EN		(1<<3)
 #define   GMBUS_IDLE_EN		(1<<2)
 #define   GMBUS_HW_WAIT_EN	(1<<1)
 #define   GMBUS_HW_RDY_EN	(1<<0)
-#define GMBUS5			(dev_priv->gpio_mmio_base + 0x5120) /* byte index */
+#define GMBUS5			_MMIO(dev_priv->gpio_mmio_base + 0x5120) /* byte index */
 #define   GMBUS_2BYTE_INDEX_EN	(1<<31)
 
 /*
@@ -2201,11 +2261,11 @@ enum skl_disp_power_wells {
 #define _DPLL_A (dev_priv->info.display_mmio_offset + 0x6014)
 #define _DPLL_B (dev_priv->info.display_mmio_offset + 0x6018)
 #define _CHV_DPLL_C (dev_priv->info.display_mmio_offset + 0x6030)
-#define DPLL(pipe) _PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C)
+#define DPLL(pipe) _MMIO_PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C)
 
-#define VGA0	0x6000
-#define VGA1	0x6004
-#define VGA_PD	0x6010
+#define VGA0	_MMIO(0x6000)
+#define VGA1	_MMIO(0x6004)
+#define VGA_PD	_MMIO(0x6010)
 #define   VGA0_PD_P2_DIV_4	(1 << 7)
 #define   VGA0_PD_P1_DIV_2	(1 << 5)
 #define   VGA0_PD_P1_SHIFT	0
@@ -2241,9 +2301,9 @@ enum skl_disp_power_wells {
 #define   DPLL_FPA01_P1_POST_DIV_MASK_I830	0x001f0000
 
 /* Additional CHV pll/phy registers */
-#define DPIO_PHY_STATUS			(VLV_DISPLAY_BASE + 0x6240)
+#define DPIO_PHY_STATUS			_MMIO(VLV_DISPLAY_BASE + 0x6240)
 #define   DPLL_PORTD_READY_MASK		(0xf)
-#define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100)
+#define DISPLAY_PHY_CONTROL _MMIO(VLV_DISPLAY_BASE + 0x60100)
 #define   PHY_CH_POWER_DOWN_OVRD_EN(phy, ch)	(1 << (2*(phy)+(ch)+27))
 #define   PHY_LDO_DELAY_0NS			0x0
 #define   PHY_LDO_DELAY_200NS			0x1
@@ -2254,7 +2314,7 @@ enum skl_disp_power_wells {
 #define   PHY_CH_DEEP_PSR			0x7
 #define   PHY_CH_POWER_MODE(mode, phy, ch)	((mode) << (6*(phy)+3*(ch)+2))
 #define   PHY_COM_LANE_RESET_DEASSERT(phy)	(1 << (phy))
-#define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104)
+#define DISPLAY_PHY_STATUS _MMIO(VLV_DISPLAY_BASE + 0x60104)
 #define   PHY_POWERGOOD(phy)	(((phy) == DPIO_PHY0) ? (1<<31) : (1<<30))
 #define   PHY_STATUS_CMN_LDO(phy, ch)                   (1 << (6-(6*(phy)+3*(ch))))
 #define   PHY_STATUS_SPLINE_LDO(phy, ch, spline)        (1 << (8-(6*(phy)+3*(ch)+(spline))))
@@ -2300,7 +2360,7 @@ enum skl_disp_power_wells {
 #define _DPLL_A_MD (dev_priv->info.display_mmio_offset + 0x601c)
 #define _DPLL_B_MD (dev_priv->info.display_mmio_offset + 0x6020)
 #define _CHV_DPLL_C_MD (dev_priv->info.display_mmio_offset + 0x603c)
-#define DPLL_MD(pipe) _PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD)
+#define DPLL_MD(pipe) _MMIO_PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD)
 
 /*
  * UDI pixel divider, controlling how many pixels are stuffed into a packet.
@@ -2339,12 +2399,12 @@ enum skl_disp_power_wells {
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK	0x0000003f
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT	0
 
-#define _FPA0	0x06040
-#define _FPA1	0x06044
-#define _FPB0	0x06048
-#define _FPB1	0x0604c
-#define FP0(pipe) _PIPE(pipe, _FPA0, _FPB0)
-#define FP1(pipe) _PIPE(pipe, _FPA1, _FPB1)
+#define _FPA0	0x6040
+#define _FPA1	0x6044
+#define _FPB0	0x6048
+#define _FPB1	0x604c
+#define FP0(pipe) _MMIO_PIPE(pipe, _FPA0, _FPB0)
+#define FP1(pipe) _MMIO_PIPE(pipe, _FPA1, _FPB1)
 #define   FP_N_DIV_MASK		0x003f0000
 #define   FP_N_PINEVIEW_DIV_MASK	0x00ff0000
 #define   FP_N_DIV_SHIFT		16
@@ -2353,7 +2413,7 @@ enum skl_disp_power_wells {
 #define   FP_M2_DIV_MASK	0x0000003f
 #define   FP_M2_PINEVIEW_DIV_MASK	0x000000ff
 #define   FP_M2_DIV_SHIFT		 0
-#define DPLL_TEST	0x606c
+#define DPLL_TEST	_MMIO(0x606c)
 #define   DPLLB_TEST_SDVO_DIV_1		(0 << 22)
 #define   DPLLB_TEST_SDVO_DIV_2		(1 << 22)
 #define   DPLLB_TEST_SDVO_DIV_4		(2 << 22)
@@ -2364,12 +2424,12 @@ enum skl_disp_power_wells {
 #define   DPLLA_TEST_N_BYPASS		(1 << 3)
 #define   DPLLA_TEST_M_BYPASS		(1 << 2)
 #define   DPLLA_INPUT_BUFFER_ENABLE	(1 << 0)
-#define D_STATE		0x6104
+#define D_STATE		_MMIO(0x6104)
 #define  DSTATE_GFX_RESET_I830			(1<<6)
 #define  DSTATE_PLL_D3_OFF			(1<<3)
 #define  DSTATE_GFX_CLOCK_GATING		(1<<1)
 #define  DSTATE_DOT_CLOCK_GATING		(1<<0)
-#define DSPCLK_GATE_D	(dev_priv->info.display_mmio_offset + 0x6200)
+#define DSPCLK_GATE_D	_MMIO(dev_priv->info.display_mmio_offset + 0x6200)
 # define DPUNIT_B_CLOCK_GATE_DISABLE		(1 << 30) /* 965 */
 # define VSUNIT_CLOCK_GATE_DISABLE		(1 << 29) /* 965 */
 # define VRHUNIT_CLOCK_GATE_DISABLE		(1 << 28) /* 965 */
@@ -2408,7 +2468,7 @@ enum skl_disp_power_wells {
 # define ZVUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 830 */
 # define OVLUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 845,865 */
 
-#define RENCLK_GATE_D1		0x6204
+#define RENCLK_GATE_D1		_MMIO(0x6204)
 # define BLITTER_CLOCK_GATE_DISABLE		(1 << 13) /* 945GM only */
 # define MPEG_CLOCK_GATE_DISABLE		(1 << 12) /* 945GM only */
 # define PC_FE_CLOCK_GATE_DISABLE		(1 << 11)
@@ -2472,35 +2532,35 @@ enum skl_disp_power_wells {
 # define I965_FT_CLOCK_GATE_DISABLE		(1 << 1)
 # define I965_DM_CLOCK_GATE_DISABLE		(1 << 0)
 
-#define RENCLK_GATE_D2		0x6208
+#define RENCLK_GATE_D2		_MMIO(0x6208)
 #define VF_UNIT_CLOCK_GATE_DISABLE		(1 << 9)
 #define GS_UNIT_CLOCK_GATE_DISABLE		(1 << 7)
 #define CL_UNIT_CLOCK_GATE_DISABLE		(1 << 6)
 
-#define VDECCLK_GATE_D		0x620C		/* g4x only */
+#define VDECCLK_GATE_D		_MMIO(0x620C)		/* g4x only */
 #define  VCP_UNIT_CLOCK_GATE_DISABLE		(1 << 4)
 
-#define RAMCLK_GATE_D		0x6210		/* CRL only */
-#define DEUC			0x6214          /* CRL only */
+#define RAMCLK_GATE_D		_MMIO(0x6210)		/* CRL only */
+#define DEUC			_MMIO(0x6214)          /* CRL only */
 
-#define FW_BLC_SELF_VLV		(VLV_DISPLAY_BASE + 0x6500)
+#define FW_BLC_SELF_VLV		_MMIO(VLV_DISPLAY_BASE + 0x6500)
 #define  FW_CSPWRDWNEN		(1<<15)
 
-#define MI_ARB_VLV		(VLV_DISPLAY_BASE + 0x6504)
+#define MI_ARB_VLV		_MMIO(VLV_DISPLAY_BASE + 0x6504)
 
-#define CZCLK_CDCLK_FREQ_RATIO	(VLV_DISPLAY_BASE + 0x6508)
+#define CZCLK_CDCLK_FREQ_RATIO	_MMIO(VLV_DISPLAY_BASE + 0x6508)
 #define   CDCLK_FREQ_SHIFT	4
 #define   CDCLK_FREQ_MASK	(0x1f << CDCLK_FREQ_SHIFT)
 #define   CZCLK_FREQ_MASK	0xf
 
-#define GCI_CONTROL		(VLV_DISPLAY_BASE + 0x650C)
+#define GCI_CONTROL		_MMIO(VLV_DISPLAY_BASE + 0x650C)
 #define   PFI_CREDIT_63		(9 << 28)		/* chv only */
 #define   PFI_CREDIT_31		(8 << 28)		/* chv only */
 #define   PFI_CREDIT(x)		(((x) - 8) << 28)	/* 8-15 */
 #define   PFI_CREDIT_RESEND	(1 << 27)
 #define   VGA_FAST_MODE_DISABLE	(1 << 14)
 
-#define GMBUSFREQ_VLV		(VLV_DISPLAY_BASE + 0x6510)
+#define GMBUSFREQ_VLV		_MMIO(VLV_DISPLAY_BASE + 0x6510)
 
 /*
  * Palette regs
@@ -2508,8 +2568,8 @@ enum skl_disp_power_wells {
 #define PALETTE_A_OFFSET 0xa000
 #define PALETTE_B_OFFSET 0xa800
 #define CHV_PALETTE_C_OFFSET 0xc000
-#define PALETTE(pipe, i) (dev_priv->info.palette_offsets[pipe] + \
-			  dev_priv->info.display_mmio_offset + (i) * 4)
+#define PALETTE(pipe, i) _MMIO(dev_priv->info.palette_offsets[pipe] +	\
+			      dev_priv->info.display_mmio_offset + (i) * 4)
 
 /* MCH MMIO space */
 
@@ -2527,37 +2587,37 @@ enum skl_disp_power_wells {
 
 #define MCHBAR_MIRROR_BASE_SNB	0x140000
 
-#define CTG_STOLEN_RESERVED		(MCHBAR_MIRROR_BASE + 0x34)
-#define ELK_STOLEN_RESERVED		(MCHBAR_MIRROR_BASE + 0x48)
+#define CTG_STOLEN_RESERVED		_MMIO(MCHBAR_MIRROR_BASE + 0x34)
+#define ELK_STOLEN_RESERVED		_MMIO(MCHBAR_MIRROR_BASE + 0x48)
 #define G4X_STOLEN_RESERVED_ADDR1_MASK	(0xFFFF << 16)
 #define G4X_STOLEN_RESERVED_ADDR2_MASK	(0xFFF << 4)
 
 /* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
-#define DCLK (MCHBAR_MIRROR_BASE_SNB + 0x5e04)
+#define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
 
 /* 915-945 and GM965 MCH register controlling DRAM channel access */
-#define DCC			0x10200
+#define DCC			_MMIO(MCHBAR_MIRROR_BASE + 0x200)
 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL		(0 << 0)
 #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC	(1 << 0)
 #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
 #define DCC_ADDRESSING_MODE_MASK			(3 << 0)
 #define DCC_CHANNEL_XOR_DISABLE				(1 << 10)
 #define DCC_CHANNEL_XOR_BIT_17				(1 << 9)
-#define DCC2			0x10204
+#define DCC2			_MMIO(MCHBAR_MIRROR_BASE + 0x204)
 #define DCC2_MODIFIED_ENHANCED_DISABLE			(1 << 20)
 
 /* Pineview MCH register contains DDR3 setting */
-#define CSHRDDR3CTL            0x101a8
+#define CSHRDDR3CTL            _MMIO(MCHBAR_MIRROR_BASE + 0x1a8)
 #define CSHRDDR3CTL_DDR3       (1 << 2)
 
 /* 965 MCH register controlling DRAM channel configuration */
-#define C0DRB3			0x10206
-#define C1DRB3			0x10606
+#define C0DRB3			_MMIO(MCHBAR_MIRROR_BASE + 0x206)
+#define C1DRB3			_MMIO(MCHBAR_MIRROR_BASE + 0x606)
 
 /* snb MCH registers for reading the DRAM channel configuration */
-#define MAD_DIMM_C0			(MCHBAR_MIRROR_BASE_SNB + 0x5004)
-#define MAD_DIMM_C1			(MCHBAR_MIRROR_BASE_SNB + 0x5008)
-#define MAD_DIMM_C2			(MCHBAR_MIRROR_BASE_SNB + 0x500C)
+#define MAD_DIMM_C0			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004)
+#define MAD_DIMM_C1			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008)
+#define MAD_DIMM_C2			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
 #define   MAD_DIMM_ECC_MASK		(0x3 << 24)
 #define   MAD_DIMM_ECC_OFF		(0x0 << 24)
 #define   MAD_DIMM_ECC_IO_ON_LOGIC_OFF	(0x1 << 24)
@@ -2577,14 +2637,14 @@ enum skl_disp_power_wells {
 #define   MAD_DIMM_A_SIZE_MASK		(0xff << MAD_DIMM_A_SIZE_SHIFT)
 
 /* snb MCH registers for priority tuning */
-#define MCH_SSKPD			(MCHBAR_MIRROR_BASE_SNB + 0x5d10)
+#define MCH_SSKPD			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10)
 #define   MCH_SSKPD_WM0_MASK		0x3f
 #define   MCH_SSKPD_WM0_VAL		0xc
 
-#define MCH_SECP_NRG_STTS		(MCHBAR_MIRROR_BASE_SNB + 0x592c)
+#define MCH_SECP_NRG_STTS		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x592c)
 
 /* Clocking configuration register */
-#define CLKCFG			0x10c00
+#define CLKCFG			_MMIO(MCHBAR_MIRROR_BASE + 0xc00)
 #define CLKCFG_FSB_400					(5 << 0)	/* hrawclk 100 */
 #define CLKCFG_FSB_533					(1 << 0)	/* hrawclk 133 */
 #define CLKCFG_FSB_667					(3 << 0)	/* hrawclk 166 */
@@ -2600,26 +2660,26 @@ enum skl_disp_power_wells {
 #define CLKCFG_MEM_800					(3 << 4)
 #define CLKCFG_MEM_MASK					(7 << 4)
 
-#define HPLLVCO                 (MCHBAR_MIRROR_BASE + 0xc38)
-#define HPLLVCO_MOBILE          (MCHBAR_MIRROR_BASE + 0xc0f)
+#define HPLLVCO                 _MMIO(MCHBAR_MIRROR_BASE + 0xc38)
+#define HPLLVCO_MOBILE          _MMIO(MCHBAR_MIRROR_BASE + 0xc0f)
 
-#define TSC1			0x11001
+#define TSC1			_MMIO(0x11001)
 #define   TSE			(1<<0)
-#define TR1			0x11006
-#define TSFS			0x11020
+#define TR1			_MMIO(0x11006)
+#define TSFS			_MMIO(0x11020)
 #define   TSFS_SLOPE_MASK	0x0000ff00
 #define   TSFS_SLOPE_SHIFT	8
 #define   TSFS_INTR_MASK	0x000000ff
 
-#define CRSTANDVID		0x11100
-#define PXVFREQ(i)		(0x11110 + (i) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */
+#define CRSTANDVID		_MMIO(0x11100)
+#define PXVFREQ(fstart)		_MMIO(0x11110 + (fstart) * 4)  /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */
 #define   PXVFREQ_PX_MASK	0x7f000000
 #define   PXVFREQ_PX_SHIFT	24
-#define VIDFREQ_BASE		0x11110
-#define VIDFREQ1		0x11110 /* VIDFREQ1-4 (0x1111c) (Cantiga) */
-#define VIDFREQ2		0x11114
-#define VIDFREQ3		0x11118
-#define VIDFREQ4		0x1111c
+#define VIDFREQ_BASE		_MMIO(0x11110)
+#define VIDFREQ1		_MMIO(0x11110) /* VIDFREQ1-4 (0x1111c) (Cantiga) */
+#define VIDFREQ2		_MMIO(0x11114)
+#define VIDFREQ3		_MMIO(0x11118)
+#define VIDFREQ4		_MMIO(0x1111c)
 #define   VIDFREQ_P0_MASK	0x1f000000
 #define   VIDFREQ_P0_SHIFT	24
 #define   VIDFREQ_P0_CSCLK_MASK	0x00f00000
@@ -2631,8 +2691,8 @@ enum skl_disp_power_wells {
 #define   VIDFREQ_P1_CSCLK_MASK	0x000000f0
 #define   VIDFREQ_P1_CSCLK_SHIFT 4
 #define   VIDFREQ_P1_CRCLK_MASK	0x0000000f
-#define INTTOEXT_BASE_ILK	0x11300
-#define INTTOEXT_BASE		0x11120 /* INTTOEXT1-8 (0x1113c) */
+#define INTTOEXT_BASE_ILK	_MMIO(0x11300)
+#define INTTOEXT_BASE		_MMIO(0x11120) /* INTTOEXT1-8 (0x1113c) */
 #define   INTTOEXT_MAP3_SHIFT	24
 #define   INTTOEXT_MAP3_MASK	(0x1f << INTTOEXT_MAP3_SHIFT)
 #define   INTTOEXT_MAP2_SHIFT	16
@@ -2641,7 +2701,7 @@ enum skl_disp_power_wells {
 #define   INTTOEXT_MAP1_MASK	(0x1f << INTTOEXT_MAP1_SHIFT)
 #define   INTTOEXT_MAP0_SHIFT	0
 #define   INTTOEXT_MAP0_MASK	(0x1f << INTTOEXT_MAP0_SHIFT)
-#define MEMSWCTL		0x11170 /* Ironlake only */
+#define MEMSWCTL		_MMIO(0x11170) /* Ironlake only */
 #define   MEMCTL_CMD_MASK	0xe000
 #define   MEMCTL_CMD_SHIFT	13
 #define   MEMCTL_CMD_RCLK_OFF	0
@@ -2656,8 +2716,8 @@ enum skl_disp_power_wells {
 #define   MEMCTL_FREQ_SHIFT	8
 #define   MEMCTL_SFCAVM		(1<<7)
 #define   MEMCTL_TGT_VID_MASK	0x007f
-#define MEMIHYST		0x1117c
-#define MEMINTREN		0x11180 /* 16 bits */
+#define MEMIHYST		_MMIO(0x1117c)
+#define MEMINTREN		_MMIO(0x11180) /* 16 bits */
 #define   MEMINT_RSEXIT_EN	(1<<8)
 #define   MEMINT_CX_SUPR_EN	(1<<7)
 #define   MEMINT_CONT_BUSY_EN	(1<<6)
@@ -2667,7 +2727,7 @@ enum skl_disp_power_wells {
 #define   MEMINT_UP_EVAL_EN	(1<<2)
 #define   MEMINT_DOWN_EVAL_EN	(1<<1)
 #define   MEMINT_SW_CMD_EN	(1<<0)
-#define MEMINTRSTR		0x11182 /* 16 bits */
+#define MEMINTRSTR		_MMIO(0x11182) /* 16 bits */
 #define   MEM_RSEXIT_MASK	0xc000
 #define   MEM_RSEXIT_SHIFT	14
 #define   MEM_CONT_BUSY_MASK	0x3000
@@ -2687,7 +2747,7 @@ enum skl_disp_power_wells {
 #define   MEM_INT_STEER_CMR	1
 #define   MEM_INT_STEER_SMI	2
 #define   MEM_INT_STEER_SCI	3
-#define MEMINTRSTS		0x11184
+#define MEMINTRSTS		_MMIO(0x11184)
 #define   MEMINT_RSEXIT		(1<<7)
 #define   MEMINT_CONT_BUSY	(1<<6)
 #define   MEMINT_AVG_BUSY	(1<<5)
@@ -2696,7 +2756,7 @@ enum skl_disp_power_wells {
 #define   MEMINT_UP_EVAL	(1<<2)
 #define   MEMINT_DOWN_EVAL	(1<<1)
 #define   MEMINT_SW_CMD		(1<<0)
-#define MEMMODECTL		0x11190
+#define MEMMODECTL		_MMIO(0x11190)
 #define   MEMMODE_BOOST_EN	(1<<31)
 #define   MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */
 #define   MEMMODE_BOOST_FREQ_SHIFT 24
@@ -2713,8 +2773,8 @@ enum skl_disp_power_wells {
 #define   MEMMODE_FMAX_MASK	0x000000f0 /* max jitter, 0-15 */
 #define   MEMMODE_FMAX_SHIFT	4
 #define   MEMMODE_FMIN_MASK	0x0000000f /* min jitter, 0-15 */
-#define RCBMAXAVG		0x1119c
-#define MEMSWCTL2		0x1119e /* Cantiga only */
+#define RCBMAXAVG		_MMIO(0x1119c)
+#define MEMSWCTL2		_MMIO(0x1119e) /* Cantiga only */
 #define   SWMEMCMD_RENDER_OFF	(0 << 13)
 #define   SWMEMCMD_RENDER_ON	(1 << 13)
 #define   SWMEMCMD_SWFREQ	(2 << 13)
@@ -2726,11 +2786,11 @@ enum skl_disp_power_wells {
 #define   SWFREQ_MASK		0x0380 /* P0-7 */
 #define   SWFREQ_SHIFT		7
 #define   TARVID_MASK		0x001f
-#define MEMSTAT_CTG		0x111a0
-#define RCBMINAVG		0x111a0
-#define RCUPEI			0x111b0
-#define RCDNEI			0x111b4
-#define RSTDBYCTL		0x111b8
+#define MEMSTAT_CTG		_MMIO(0x111a0)
+#define RCBMINAVG		_MMIO(0x111a0)
+#define RCUPEI			_MMIO(0x111b0)
+#define RCDNEI			_MMIO(0x111b4)
+#define RSTDBYCTL		_MMIO(0x111b8)
 #define   RS1EN			(1<<31)
 #define   RS2EN			(1<<30)
 #define   RS3EN			(1<<29)
@@ -2774,10 +2834,10 @@ enum skl_disp_power_wells {
 #define   RS_CSTATE_C367_RS2	(3<<4)
 #define   REDSAVES		(1<<3) /* no context save if was idle during rs0 */
 #define   REDRESTORES		(1<<2) /* no restore if was idle during rs0 */
-#define VIDCTL			0x111c0
-#define VIDSTS			0x111c8
-#define VIDSTART		0x111cc /* 8 bits */
-#define MEMSTAT_ILK			0x111f8
+#define VIDCTL			_MMIO(0x111c0)
+#define VIDSTS			_MMIO(0x111c8)
+#define VIDSTART		_MMIO(0x111cc) /* 8 bits */
+#define MEMSTAT_ILK		_MMIO(0x111f8)
 #define   MEMSTAT_VID_MASK	0x7f00
 #define   MEMSTAT_VID_SHIFT	8
 #define   MEMSTAT_PSTATE_MASK	0x00f8
@@ -2788,55 +2848,55 @@ enum skl_disp_power_wells {
 #define   MEMSTAT_SRC_CTL_TRB	1
 #define   MEMSTAT_SRC_CTL_THM	2
 #define   MEMSTAT_SRC_CTL_STDBY 3
-#define RCPREVBSYTUPAVG		0x113b8
-#define RCPREVBSYTDNAVG		0x113bc
-#define PMMISC			0x11214
+#define RCPREVBSYTUPAVG		_MMIO(0x113b8)
+#define RCPREVBSYTDNAVG		_MMIO(0x113bc)
+#define PMMISC			_MMIO(0x11214)
 #define   MCPPCE_EN		(1<<0) /* enable PM_MSG from PCH->MPC */
-#define SDEW			0x1124c
-#define CSIEW0			0x11250
-#define CSIEW1			0x11254
-#define CSIEW2			0x11258
-#define PEW(i)			(0x1125c + (i) * 4) /* 5 registers */
-#define DEW(i)			(0x11270 + (i) * 4) /* 3 registers */
-#define MCHAFE			0x112c0
-#define CSIEC			0x112e0
-#define DMIEC			0x112e4
-#define DDREC			0x112e8
-#define PEG0EC			0x112ec
-#define PEG1EC			0x112f0
-#define GFXEC			0x112f4
-#define RPPREVBSYTUPAVG		0x113b8
-#define RPPREVBSYTDNAVG		0x113bc
-#define ECR			0x11600
+#define SDEW			_MMIO(0x1124c)
+#define CSIEW0			_MMIO(0x11250)
+#define CSIEW1			_MMIO(0x11254)
+#define CSIEW2			_MMIO(0x11258)
+#define PEW(i)			_MMIO(0x1125c + (i) * 4) /* 5 registers */
+#define DEW(i)			_MMIO(0x11270 + (i) * 4) /* 3 registers */
+#define MCHAFE			_MMIO(0x112c0)
+#define CSIEC			_MMIO(0x112e0)
+#define DMIEC			_MMIO(0x112e4)
+#define DDREC			_MMIO(0x112e8)
+#define PEG0EC			_MMIO(0x112ec)
+#define PEG1EC			_MMIO(0x112f0)
+#define GFXEC			_MMIO(0x112f4)
+#define RPPREVBSYTUPAVG		_MMIO(0x113b8)
+#define RPPREVBSYTDNAVG		_MMIO(0x113bc)
+#define ECR			_MMIO(0x11600)
 #define   ECR_GPFE		(1<<31)
 #define   ECR_IMONE		(1<<30)
 #define   ECR_CAP_MASK		0x0000001f /* Event range, 0-31 */
-#define OGW0			0x11608
-#define OGW1			0x1160c
-#define EG0			0x11610
-#define EG1			0x11614
-#define EG2			0x11618
-#define EG3			0x1161c
-#define EG4			0x11620
-#define EG5			0x11624
-#define EG6			0x11628
-#define EG7			0x1162c
-#define PXW(i)			(0x11664 + (i) * 4) /* 4 registers */
-#define PXWL(i)			(0x11680 + (i) * 4) /* 8 registers */
-#define LCFUSE02		0x116c0
+#define OGW0			_MMIO(0x11608)
+#define OGW1			_MMIO(0x1160c)
+#define EG0			_MMIO(0x11610)
+#define EG1			_MMIO(0x11614)
+#define EG2			_MMIO(0x11618)
+#define EG3			_MMIO(0x1161c)
+#define EG4			_MMIO(0x11620)
+#define EG5			_MMIO(0x11624)
+#define EG6			_MMIO(0x11628)
+#define EG7			_MMIO(0x1162c)
+#define PXW(i)			_MMIO(0x11664 + (i) * 4) /* 4 registers */
+#define PXWL(i)			_MMIO(0x11680 + (i) * 8) /* 8 registers */
+#define LCFUSE02		_MMIO(0x116c0)
 #define   LCFUSE_HIV_MASK	0x000000ff
-#define CSIPLL0			0x12c10
-#define DDRMPLL1		0X12c20
-#define PEG_BAND_GAP_DATA	0x14d68
+#define CSIPLL0			_MMIO(0x12c10)
+#define DDRMPLL1		_MMIO(0X12c20)
+#define PEG_BAND_GAP_DATA	_MMIO(0x14d68)
 
-#define GEN6_GT_THREAD_STATUS_REG 0x13805c
+#define GEN6_GT_THREAD_STATUS_REG _MMIO(0x13805c)
 #define GEN6_GT_THREAD_STATUS_CORE_MASK 0x7
 
-#define GEN6_GT_PERF_STATUS	(MCHBAR_MIRROR_BASE_SNB + 0x5948)
-#define BXT_GT_PERF_STATUS      (MCHBAR_MIRROR_BASE_SNB + 0x7070)
-#define GEN6_RP_STATE_LIMITS	(MCHBAR_MIRROR_BASE_SNB + 0x5994)
-#define GEN6_RP_STATE_CAP	(MCHBAR_MIRROR_BASE_SNB + 0x5998)
-#define BXT_RP_STATE_CAP        0x138170
+#define GEN6_GT_PERF_STATUS	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948)
+#define BXT_GT_PERF_STATUS      _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070)
+#define GEN6_RP_STATE_LIMITS	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994)
+#define GEN6_RP_STATE_CAP	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998)
+#define BXT_RP_STATE_CAP        _MMIO(0x138170)
 
 #define INTERVAL_1_28_US(us)	(((us) * 100) >> 7)
 #define INTERVAL_1_33_US(us)	(((us) * 3)   >> 2)
@@ -2850,7 +2910,7 @@ enum skl_disp_power_wells {
 /*
  * Logical Context regs
  */
-#define CCID			0x2180
+#define CCID			_MMIO(0x2180)
 #define   CCID_EN		(1<<0)
 /*
  * Notes on SNB/IVB/VLV context size:
@@ -2865,7 +2925,7 @@ enum skl_disp_power_wells {
  * - GT1 size just indicates how much of render context
  *   doesn't need saving on GT1
  */
-#define CXT_SIZE		0x21a0
+#define CXT_SIZE		_MMIO(0x21a0)
 #define GEN6_CXT_POWER_SIZE(cxt_reg)	(((cxt_reg) >> 24) & 0x3f)
 #define GEN6_CXT_RING_SIZE(cxt_reg)	(((cxt_reg) >> 18) & 0x3f)
 #define GEN6_CXT_RENDER_SIZE(cxt_reg)	(((cxt_reg) >> 12) & 0x3f)
@@ -2874,7 +2934,7 @@ enum skl_disp_power_wells {
 #define GEN6_CXT_TOTAL_SIZE(cxt_reg)	(GEN6_CXT_RING_SIZE(cxt_reg) + \
 					GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \
 					GEN6_CXT_PIPELINE_SIZE(cxt_reg))
-#define GEN7_CXT_SIZE		0x21a8
+#define GEN7_CXT_SIZE		_MMIO(0x21a8)
 #define GEN7_CXT_POWER_SIZE(ctx_reg)	(((ctx_reg) >> 25) & 0x7f)
 #define GEN7_CXT_RING_SIZE(ctx_reg)	(((ctx_reg) >> 22) & 0x7)
 #define GEN7_CXT_RENDER_SIZE(ctx_reg)	(((ctx_reg) >> 16) & 0x3f)
@@ -2894,23 +2954,30 @@ enum skl_disp_power_wells {
 /* Same as Haswell, but 72064 bytes now. */
 #define GEN8_CXT_TOTAL_SIZE		(18 * PAGE_SIZE)
 
-#define CHV_CLK_CTL1			0x101100
-#define VLV_CLK_CTL2			0x101104
+#define CHV_CLK_CTL1			_MMIO(0x101100)
+#define VLV_CLK_CTL2			_MMIO(0x101104)
 #define   CLK_CTL2_CZCOUNT_30NS_SHIFT	28
 
 /*
  * Overlay regs
  */
 
-#define OVADD			0x30000
-#define DOVSTA			0x30008
+#define OVADD			_MMIO(0x30000)
+#define DOVSTA			_MMIO(0x30008)
 #define OC_BUF			(0x3<<20)
-#define OGAMC5			0x30010
-#define OGAMC4			0x30014
-#define OGAMC3			0x30018
-#define OGAMC2			0x3001c
-#define OGAMC1			0x30020
-#define OGAMC0			0x30024
+#define OGAMC5			_MMIO(0x30010)
+#define OGAMC4			_MMIO(0x30014)
+#define OGAMC3			_MMIO(0x30018)
+#define OGAMC2			_MMIO(0x3001c)
+#define OGAMC1			_MMIO(0x30020)
+#define OGAMC0			_MMIO(0x30024)
+
+/*
+ * GEN9 clock gating regs
+ */
+#define GEN9_CLKGATE_DIS_0		_MMIO(0x46530)
+#define   PWM2_GATING_DIS		(1 << 14)
+#define   PWM1_GATING_DIS		(1 << 13)
 
 /*
  * Display engine regs
@@ -2970,28 +3037,18 @@ enum skl_disp_power_wells {
 #define _PIPE_CRC_RES_4_B_IVB		0x61070
 #define _PIPE_CRC_RES_5_B_IVB		0x61074
 
-#define PIPE_CRC_CTL(pipe) _TRANSCODER2(pipe, _PIPE_CRC_CTL_A)
-#define PIPE_CRC_RES_1_IVB(pipe)	\
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_1_A_IVB)
-#define PIPE_CRC_RES_2_IVB(pipe)	\
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_2_A_IVB)
-#define PIPE_CRC_RES_3_IVB(pipe)	\
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_3_A_IVB)
-#define PIPE_CRC_RES_4_IVB(pipe)	\
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_4_A_IVB)
-#define PIPE_CRC_RES_5_IVB(pipe)	\
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_5_A_IVB)
-
-#define PIPE_CRC_RES_RED(pipe) \
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_RED_A)
-#define PIPE_CRC_RES_GREEN(pipe) \
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_GREEN_A)
-#define PIPE_CRC_RES_BLUE(pipe) \
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_BLUE_A)
-#define PIPE_CRC_RES_RES1_I915(pipe) \
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_RES1_A_I915)
-#define PIPE_CRC_RES_RES2_G4X(pipe) \
-	_TRANSCODER2(pipe, _PIPE_CRC_RES_RES2_A_G4X)
+#define PIPE_CRC_CTL(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_CTL_A)
+#define PIPE_CRC_RES_1_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_1_A_IVB)
+#define PIPE_CRC_RES_2_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_2_A_IVB)
+#define PIPE_CRC_RES_3_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_3_A_IVB)
+#define PIPE_CRC_RES_4_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_4_A_IVB)
+#define PIPE_CRC_RES_5_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_5_A_IVB)
+
+#define PIPE_CRC_RES_RED(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RED_A)
+#define PIPE_CRC_RES_GREEN(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_GREEN_A)
+#define PIPE_CRC_RES_BLUE(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_RES_BLUE_A)
+#define PIPE_CRC_RES_RES1_I915(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES1_A_I915)
+#define PIPE_CRC_RES_RES2_G4X(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES2_A_G4X)
 
 /* Pipe A timing regs */
 #define _HTOTAL_A	0x60000
@@ -3023,20 +3080,20 @@ enum skl_disp_power_wells {
 #define CHV_TRANSCODER_C_OFFSET 0x63000
 #define TRANSCODER_EDP_OFFSET 0x6f000
 
-#define _TRANSCODER2(pipe, reg) (dev_priv->info.trans_offsets[(pipe)] - \
+#define _MMIO_TRANS2(pipe, reg) _MMIO(dev_priv->info.trans_offsets[(pipe)] - \
 	dev_priv->info.trans_offsets[TRANSCODER_A] + (reg) + \
 	dev_priv->info.display_mmio_offset)
 
-#define HTOTAL(trans) _TRANSCODER2(trans, _HTOTAL_A)
-#define HBLANK(trans) _TRANSCODER2(trans, _HBLANK_A)
-#define HSYNC(trans) _TRANSCODER2(trans, _HSYNC_A)
-#define VTOTAL(trans) _TRANSCODER2(trans, _VTOTAL_A)
-#define VBLANK(trans) _TRANSCODER2(trans, _VBLANK_A)
-#define VSYNC(trans) _TRANSCODER2(trans, _VSYNC_A)
-#define BCLRPAT(trans) _TRANSCODER2(trans, _BCLRPAT_A)
-#define VSYNCSHIFT(trans) _TRANSCODER2(trans, _VSYNCSHIFT_A)
-#define PIPESRC(trans) _TRANSCODER2(trans, _PIPEASRC)
-#define PIPE_MULT(trans) _TRANSCODER2(trans, _PIPE_MULT_A)
+#define HTOTAL(trans)		_MMIO_TRANS2(trans, _HTOTAL_A)
+#define HBLANK(trans)		_MMIO_TRANS2(trans, _HBLANK_A)
+#define HSYNC(trans)		_MMIO_TRANS2(trans, _HSYNC_A)
+#define VTOTAL(trans)		_MMIO_TRANS2(trans, _VTOTAL_A)
+#define VBLANK(trans)		_MMIO_TRANS2(trans, _VBLANK_A)
+#define VSYNC(trans)		_MMIO_TRANS2(trans, _VSYNC_A)
+#define BCLRPAT(trans)		_MMIO_TRANS2(trans, _BCLRPAT_A)
+#define VSYNCSHIFT(trans)	_MMIO_TRANS2(trans, _VSYNCSHIFT_A)
+#define PIPESRC(trans)		_MMIO_TRANS2(trans, _PIPEASRC)
+#define PIPE_MULT(trans)	_MMIO_TRANS2(trans, _PIPE_MULT_A)
 
 /* VLV eDP PSR registers */
 #define _PSRCTLA				(VLV_DISPLAY_BASE + 0x60090)
@@ -3052,14 +3109,14 @@ enum skl_disp_power_wells {
 #define  VLV_EDP_PSR_DBL_FRAME			(1<<10)
 #define  VLV_EDP_PSR_FRAME_COUNT_MASK		(0xff<<16)
 #define  VLV_EDP_PSR_IDLE_FRAME_SHIFT		16
-#define VLV_PSRCTL(pipe) _PIPE(pipe, _PSRCTLA, _PSRCTLB)
+#define VLV_PSRCTL(pipe)	_MMIO_PIPE(pipe, _PSRCTLA, _PSRCTLB)
 
 #define _VSCSDPA			(VLV_DISPLAY_BASE + 0x600a0)
 #define _VSCSDPB			(VLV_DISPLAY_BASE + 0x610a0)
 #define  VLV_EDP_PSR_SDP_FREQ_MASK	(3<<30)
 #define  VLV_EDP_PSR_SDP_FREQ_ONCE	(1<<31)
 #define  VLV_EDP_PSR_SDP_FREQ_EVFRAME	(1<<30)
-#define VLV_VSCSDP(pipe)	_PIPE(pipe, _VSCSDPA, _VSCSDPB)
+#define VLV_VSCSDP(pipe)	_MMIO_PIPE(pipe, _VSCSDPA, _VSCSDPB)
 
 #define _PSRSTATA			(VLV_DISPLAY_BASE + 0x60094)
 #define _PSRSTATB			(VLV_DISPLAY_BASE + 0x61094)
@@ -3072,11 +3129,12 @@ enum skl_disp_power_wells {
 #define  VLV_EDP_PSR_ACTIVE_SF_UPDATE	(4<<0)
 #define  VLV_EDP_PSR_EXIT		(5<<0)
 #define  VLV_EDP_PSR_IN_TRANS		(1<<7)
-#define VLV_PSRSTAT(pipe) _PIPE(pipe, _PSRSTATA, _PSRSTATB)
+#define VLV_PSRSTAT(pipe)	_MMIO_PIPE(pipe, _PSRSTATA, _PSRSTATB)
 
 /* HSW+ eDP PSR registers */
-#define EDP_PSR_BASE(dev)                       (IS_HASWELL(dev) ? 0x64800 : 0x6f800)
-#define EDP_PSR_CTL(dev)			(EDP_PSR_BASE(dev) + 0)
+#define HSW_EDP_PSR_BASE	0x64800
+#define BDW_EDP_PSR_BASE	0x6f800
+#define EDP_PSR_CTL				_MMIO(dev_priv->psr_mmio_base + 0)
 #define   EDP_PSR_ENABLE			(1<<31)
 #define   BDW_PSR_SINGLE_FRAME			(1<<30)
 #define   EDP_PSR_LINK_STANDBY			(1<<27)
@@ -3099,14 +3157,10 @@ enum skl_disp_power_wells {
 #define   EDP_PSR_TP1_TIME_0us			(3<<4)
 #define   EDP_PSR_IDLE_FRAME_SHIFT		0
 
-#define EDP_PSR_AUX_CTL(dev)			(EDP_PSR_BASE(dev) + 0x10)
-#define EDP_PSR_AUX_DATA1(dev)			(EDP_PSR_BASE(dev) + 0x14)
-#define EDP_PSR_AUX_DATA2(dev)			(EDP_PSR_BASE(dev) + 0x18)
-#define EDP_PSR_AUX_DATA3(dev)			(EDP_PSR_BASE(dev) + 0x1c)
-#define EDP_PSR_AUX_DATA4(dev)			(EDP_PSR_BASE(dev) + 0x20)
-#define EDP_PSR_AUX_DATA5(dev)			(EDP_PSR_BASE(dev) + 0x24)
+#define EDP_PSR_AUX_CTL				_MMIO(dev_priv->psr_mmio_base + 0x10)
+#define EDP_PSR_AUX_DATA(i)			_MMIO(dev_priv->psr_mmio_base + 0x14 + (i) * 4) /* 5 registers */
 
-#define EDP_PSR_STATUS_CTL(dev)			(EDP_PSR_BASE(dev) + 0x40)
+#define EDP_PSR_STATUS_CTL			_MMIO(dev_priv->psr_mmio_base + 0x40)
 #define   EDP_PSR_STATUS_STATE_MASK		(7<<29)
 #define   EDP_PSR_STATUS_STATE_IDLE		(0<<29)
 #define   EDP_PSR_STATUS_STATE_SRDONACK		(1<<29)
@@ -3130,15 +3184,15 @@ enum skl_disp_power_wells {
 #define   EDP_PSR_STATUS_SENDING_TP1		(1<<4)
 #define   EDP_PSR_STATUS_IDLE_MASK		0xf
 
-#define EDP_PSR_PERF_CNT(dev)		(EDP_PSR_BASE(dev) + 0x44)
+#define EDP_PSR_PERF_CNT		_MMIO(dev_priv->psr_mmio_base + 0x44)
 #define   EDP_PSR_PERF_CNT_MASK		0xffffff
 
-#define EDP_PSR_DEBUG_CTL(dev)		(EDP_PSR_BASE(dev) + 0x60)
+#define EDP_PSR_DEBUG_CTL		_MMIO(dev_priv->psr_mmio_base + 0x60)
 #define   EDP_PSR_DEBUG_MASK_LPSP	(1<<27)
 #define   EDP_PSR_DEBUG_MASK_MEMUP	(1<<26)
 #define   EDP_PSR_DEBUG_MASK_HPD	(1<<25)
 
-#define EDP_PSR2_CTL			0x6f900
+#define EDP_PSR2_CTL			_MMIO(0x6f900)
 #define   EDP_PSR2_ENABLE		(1<<31)
 #define   EDP_SU_TRACK_ENABLE		(1<<30)
 #define   EDP_MAX_SU_DISABLE_TIME(t)	((t)<<20)
@@ -3153,9 +3207,9 @@ enum skl_disp_power_wells {
 #define   EDP_PSR2_IDLE_MASK		0xf
 
 /* VGA port control */
-#define ADPA			0x61100
-#define PCH_ADPA                0xe1100
-#define VLV_ADPA		(VLV_DISPLAY_BASE + ADPA)
+#define ADPA			_MMIO(0x61100)
+#define PCH_ADPA                _MMIO(0xe1100)
+#define VLV_ADPA		_MMIO(VLV_DISPLAY_BASE + 0x61100)
 
 #define   ADPA_DAC_ENABLE	(1<<31)
 #define   ADPA_DAC_DISABLE	0
@@ -3201,7 +3255,7 @@ enum skl_disp_power_wells {
 
 
 /* Hotplug control (945+ only) */
-#define PORT_HOTPLUG_EN		(dev_priv->info.display_mmio_offset + 0x61110)
+#define PORT_HOTPLUG_EN		_MMIO(dev_priv->info.display_mmio_offset + 0x61110)
 #define   PORTB_HOTPLUG_INT_EN			(1 << 29)
 #define   PORTC_HOTPLUG_INT_EN			(1 << 28)
 #define   PORTD_HOTPLUG_INT_EN			(1 << 27)
@@ -3231,7 +3285,7 @@ enum skl_disp_power_wells {
 #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV	(0 << 2)
 #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV	(1 << 2)
 
-#define PORT_HOTPLUG_STAT	(dev_priv->info.display_mmio_offset + 0x61114)
+#define PORT_HOTPLUG_STAT	_MMIO(dev_priv->info.display_mmio_offset + 0x61114)
 /*
  * HDMI/DP bits are gen4+
  *
@@ -3296,21 +3350,23 @@ enum skl_disp_power_wells {
 
 /* SDVO and HDMI port control.
  * The same register may be used for SDVO or HDMI */
-#define GEN3_SDVOB	0x61140
-#define GEN3_SDVOC	0x61160
+#define _GEN3_SDVOB	0x61140
+#define _GEN3_SDVOC	0x61160
+#define GEN3_SDVOB	_MMIO(_GEN3_SDVOB)
+#define GEN3_SDVOC	_MMIO(_GEN3_SDVOC)
 #define GEN4_HDMIB	GEN3_SDVOB
 #define GEN4_HDMIC	GEN3_SDVOC
-#define VLV_HDMIB	(VLV_DISPLAY_BASE + GEN4_HDMIB)
-#define VLV_HDMIC	(VLV_DISPLAY_BASE + GEN4_HDMIC)
-#define CHV_HDMID	(VLV_DISPLAY_BASE + 0x6116C)
-#define PCH_SDVOB	0xe1140
+#define VLV_HDMIB	_MMIO(VLV_DISPLAY_BASE + 0x61140)
+#define VLV_HDMIC	_MMIO(VLV_DISPLAY_BASE + 0x61160)
+#define CHV_HDMID	_MMIO(VLV_DISPLAY_BASE + 0x6116C)
+#define PCH_SDVOB	_MMIO(0xe1140)
 #define PCH_HDMIB	PCH_SDVOB
-#define PCH_HDMIC	0xe1150
-#define PCH_HDMID	0xe1160
+#define PCH_HDMIC	_MMIO(0xe1150)
+#define PCH_HDMID	_MMIO(0xe1160)
 
-#define PORT_DFT_I9XX				0x61150
+#define PORT_DFT_I9XX				_MMIO(0x61150)
 #define   DC_BALANCE_RESET			(1 << 25)
-#define PORT_DFT2_G4X		(dev_priv->info.display_mmio_offset + 0x61154)
+#define PORT_DFT2_G4X		_MMIO(dev_priv->info.display_mmio_offset + 0x61154)
 #define   DC_BALANCE_RESET_VLV			(1 << 31)
 #define   PIPE_SCRAMBLE_RESET_MASK		((1 << 14) | (0x3 << 0))
 #define   PIPE_C_SCRAMBLE_RESET			(1 << 14) /* chv */
@@ -3370,9 +3426,12 @@ enum skl_disp_power_wells {
 
 
 /* DVO port control */
-#define DVOA			0x61120
-#define DVOB			0x61140
-#define DVOC			0x61160
+#define _DVOA			0x61120
+#define DVOA			_MMIO(_DVOA)
+#define _DVOB			0x61140
+#define DVOB			_MMIO(_DVOB)
+#define _DVOC			0x61160
+#define DVOC			_MMIO(_DVOC)
 #define   DVO_ENABLE			(1 << 31)
 #define   DVO_PIPE_B_SELECT		(1 << 30)
 #define   DVO_PIPE_STALL_UNUSED		(0 << 28)
@@ -3397,14 +3456,14 @@ enum skl_disp_power_wells {
 #define   DVO_OUTPUT_CSTATE_PIXELS	(1 << 1)	/* SDG only */
 #define   DVO_OUTPUT_SOURCE_SIZE_PIXELS	(1 << 0)	/* SDG only */
 #define   DVO_PRESERVE_MASK		(0x7<<24)
-#define DVOA_SRCDIM		0x61124
-#define DVOB_SRCDIM		0x61144
-#define DVOC_SRCDIM		0x61164
+#define DVOA_SRCDIM		_MMIO(0x61124)
+#define DVOB_SRCDIM		_MMIO(0x61144)
+#define DVOC_SRCDIM		_MMIO(0x61164)
 #define   DVO_SRCDIM_HORIZONTAL_SHIFT	12
 #define   DVO_SRCDIM_VERTICAL_SHIFT	0
 
 /* LVDS port control */
-#define LVDS			0x61180
+#define LVDS			_MMIO(0x61180)
 /*
  * Enables the LVDS port.  This bit must be set before DPLLs are enabled, as
  * the DPLL semantics change when the LVDS is assigned to that pipe.
@@ -3454,13 +3513,13 @@ enum skl_disp_power_wells {
 #define   LVDS_B0B3_POWER_UP		(3 << 2)
 
 /* Video Data Island Packet control */
-#define VIDEO_DIP_DATA		0x61178
+#define VIDEO_DIP_DATA		_MMIO(0x61178)
 /* Read the description of VIDEO_DIP_DATA (before Haswell) or VIDEO_DIP_ECC
  * (Haswell and newer) to see which VIDEO_DIP_DATA byte corresponds to each byte
  * of the infoframe structure specified by CEA-861. */
 #define   VIDEO_DIP_DATA_SIZE	32
 #define   VIDEO_DIP_VSC_DATA_SIZE	36
-#define VIDEO_DIP_CTL		0x61170
+#define VIDEO_DIP_CTL		_MMIO(0x61170)
 /* Pre HSW: */
 #define   VIDEO_DIP_ENABLE		(1 << 31)
 #define   VIDEO_DIP_PORT(port)		((port) << 29)
@@ -3487,7 +3546,7 @@ enum skl_disp_power_wells {
 #define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
 
 /* Panel power sequencing */
-#define PP_STATUS	0x61200
+#define PP_STATUS	_MMIO(0x61200)
 #define   PP_ON		(1 << 31)
 /*
  * Indicates that all dependencies of the panel are on:
@@ -3513,14 +3572,14 @@ enum skl_disp_power_wells {
 #define   PP_SEQUENCE_STATE_ON_S1_2	(0xa << 0)
 #define   PP_SEQUENCE_STATE_ON_S1_3	(0xb << 0)
 #define   PP_SEQUENCE_STATE_RESET	(0xf << 0)
-#define PP_CONTROL	0x61204
+#define PP_CONTROL	_MMIO(0x61204)
 #define   POWER_TARGET_ON	(1 << 0)
-#define PP_ON_DELAYS	0x61208
-#define PP_OFF_DELAYS	0x6120c
-#define PP_DIVISOR	0x61210
+#define PP_ON_DELAYS	_MMIO(0x61208)
+#define PP_OFF_DELAYS	_MMIO(0x6120c)
+#define PP_DIVISOR	_MMIO(0x61210)
 
 /* Panel fitting */
-#define PFIT_CONTROL	(dev_priv->info.display_mmio_offset + 0x61230)
+#define PFIT_CONTROL	_MMIO(dev_priv->info.display_mmio_offset + 0x61230)
 #define   PFIT_ENABLE		(1 << 31)
 #define   PFIT_PIPE_MASK	(3 << 29)
 #define   PFIT_PIPE_SHIFT	29
@@ -3538,7 +3597,7 @@ enum skl_disp_power_wells {
 #define   PFIT_SCALING_PROGRAMMED (1 << 26)
 #define   PFIT_SCALING_PILLAR	(2 << 26)
 #define   PFIT_SCALING_LETTER	(3 << 26)
-#define PFIT_PGM_RATIOS	(dev_priv->info.display_mmio_offset + 0x61234)
+#define PFIT_PGM_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61234)
 /* Pre-965 */
 #define		PFIT_VERT_SCALE_SHIFT		20
 #define		PFIT_VERT_SCALE_MASK		0xfff00000
@@ -3550,25 +3609,25 @@ enum skl_disp_power_wells {
 #define		PFIT_HORIZ_SCALE_SHIFT_965	0
 #define		PFIT_HORIZ_SCALE_MASK_965	0x00001fff
 
-#define PFIT_AUTO_RATIOS (dev_priv->info.display_mmio_offset + 0x61238)
+#define PFIT_AUTO_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61238)
 
 #define _VLV_BLC_PWM_CTL2_A (dev_priv->info.display_mmio_offset + 0x61250)
 #define _VLV_BLC_PWM_CTL2_B (dev_priv->info.display_mmio_offset + 0x61350)
-#define VLV_BLC_PWM_CTL2(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
-				     _VLV_BLC_PWM_CTL2_B)
+#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
+					 _VLV_BLC_PWM_CTL2_B)
 
 #define _VLV_BLC_PWM_CTL_A (dev_priv->info.display_mmio_offset + 0x61254)
 #define _VLV_BLC_PWM_CTL_B (dev_priv->info.display_mmio_offset + 0x61354)
-#define VLV_BLC_PWM_CTL(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
-				    _VLV_BLC_PWM_CTL_B)
+#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
+					_VLV_BLC_PWM_CTL_B)
 
 #define _VLV_BLC_HIST_CTL_A (dev_priv->info.display_mmio_offset + 0x61260)
 #define _VLV_BLC_HIST_CTL_B (dev_priv->info.display_mmio_offset + 0x61360)
-#define VLV_BLC_HIST_CTL(pipe) _PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
-				     _VLV_BLC_HIST_CTL_B)
+#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
+					 _VLV_BLC_HIST_CTL_B)
 
 /* Backlight control */
-#define BLC_PWM_CTL2	(dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */
+#define BLC_PWM_CTL2	_MMIO(dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */
 #define   BLM_PWM_ENABLE		(1 << 31)
 #define   BLM_COMBINATION_MODE		(1 << 30) /* gen4 only */
 #define   BLM_PIPE_SELECT		(1 << 29)
@@ -3591,7 +3650,7 @@ enum skl_disp_power_wells {
 #define   BLM_PHASE_IN_COUNT_MASK	(0xff << 8)
 #define   BLM_PHASE_IN_INCR_SHIFT	(0)
 #define   BLM_PHASE_IN_INCR_MASK	(0xff << 0)
-#define BLC_PWM_CTL	(dev_priv->info.display_mmio_offset + 0x61254)
+#define BLC_PWM_CTL	_MMIO(dev_priv->info.display_mmio_offset + 0x61254)
 /*
  * This is the most significant 15 bits of the number of backlight cycles in a
  * complete cycle of the modulated backlight control.
@@ -3613,25 +3672,25 @@ enum skl_disp_power_wells {
 #define   BACKLIGHT_DUTY_CYCLE_MASK_PNV		(0xfffe)
 #define   BLM_POLARITY_PNV			(1 << 0) /* pnv only */
 
-#define BLC_HIST_CTL	(dev_priv->info.display_mmio_offset + 0x61260)
+#define BLC_HIST_CTL	_MMIO(dev_priv->info.display_mmio_offset + 0x61260)
 #define  BLM_HISTOGRAM_ENABLE			(1 << 31)
 
 /* New registers for PCH-split platforms. Safe where new bits show up, the
  * register layout machtes with gen4 BLC_PWM_CTL[12]. */
-#define BLC_PWM_CPU_CTL2	0x48250
-#define BLC_PWM_CPU_CTL		0x48254
+#define BLC_PWM_CPU_CTL2	_MMIO(0x48250)
+#define BLC_PWM_CPU_CTL		_MMIO(0x48254)
 
-#define HSW_BLC_PWM2_CTL	0x48350
+#define HSW_BLC_PWM2_CTL	_MMIO(0x48350)
 
 /* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is
  * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */
-#define BLC_PWM_PCH_CTL1	0xc8250
+#define BLC_PWM_PCH_CTL1	_MMIO(0xc8250)
 #define   BLM_PCH_PWM_ENABLE			(1 << 31)
 #define   BLM_PCH_OVERRIDE_ENABLE		(1 << 30)
 #define   BLM_PCH_POLARITY			(1 << 29)
-#define BLC_PWM_PCH_CTL2	0xc8254
+#define BLC_PWM_PCH_CTL2	_MMIO(0xc8254)
 
-#define UTIL_PIN_CTL		0x48400
+#define UTIL_PIN_CTL		_MMIO(0x48400)
 #define   UTIL_PIN_ENABLE	(1 << 31)
 
 #define   UTIL_PIN_PIPE(x)     ((x) << 29)
@@ -3651,18 +3710,18 @@ enum skl_disp_power_wells {
 #define _BXT_BLC_PWM_FREQ2			0xC8354
 #define _BXT_BLC_PWM_DUTY2			0xC8358
 
-#define BXT_BLC_PWM_CTL(controller)    _PIPE(controller, \
+#define BXT_BLC_PWM_CTL(controller)    _MMIO_PIPE(controller,		\
 					_BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2)
-#define BXT_BLC_PWM_FREQ(controller)   _PIPE(controller, \
+#define BXT_BLC_PWM_FREQ(controller)   _MMIO_PIPE(controller, \
 					_BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2)
-#define BXT_BLC_PWM_DUTY(controller)   _PIPE(controller, \
+#define BXT_BLC_PWM_DUTY(controller)   _MMIO_PIPE(controller, \
 					_BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2)
 
-#define PCH_GTC_CTL		0xe7000
+#define PCH_GTC_CTL		_MMIO(0xe7000)
 #define   PCH_GTC_ENABLE	(1 << 31)
 
 /* TV port control */
-#define TV_CTL			0x68000
+#define TV_CTL			_MMIO(0x68000)
 /* Enables the TV encoder */
 # define TV_ENC_ENABLE			(1 << 31)
 /* Sources the TV encoder input from pipe B instead of A. */
@@ -3729,7 +3788,7 @@ enum skl_disp_power_wells {
 # define TV_TEST_MODE_MONITOR_DETECT	(7 << 0)
 # define TV_TEST_MODE_MASK		(7 << 0)
 
-#define TV_DAC			0x68004
+#define TV_DAC			_MMIO(0x68004)
 # define TV_DAC_SAVE		0x00ffff00
 /*
  * Reports that DAC state change logic has reported change (RO).
@@ -3780,13 +3839,13 @@ enum skl_disp_power_wells {
  * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with
  * -1 (0x3) being the only legal negative value.
  */
-#define TV_CSC_Y		0x68010
+#define TV_CSC_Y		_MMIO(0x68010)
 # define TV_RY_MASK			0x07ff0000
 # define TV_RY_SHIFT			16
 # define TV_GY_MASK			0x00000fff
 # define TV_GY_SHIFT			0
 
-#define TV_CSC_Y2		0x68014
+#define TV_CSC_Y2		_MMIO(0x68014)
 # define TV_BY_MASK			0x07ff0000
 # define TV_BY_SHIFT			16
 /*
@@ -3797,13 +3856,13 @@ enum skl_disp_power_wells {
 # define TV_AY_MASK			0x000003ff
 # define TV_AY_SHIFT			0
 
-#define TV_CSC_U		0x68018
+#define TV_CSC_U		_MMIO(0x68018)
 # define TV_RU_MASK			0x07ff0000
 # define TV_RU_SHIFT			16
 # define TV_GU_MASK			0x000007ff
 # define TV_GU_SHIFT			0
 
-#define TV_CSC_U2		0x6801c
+#define TV_CSC_U2		_MMIO(0x6801c)
 # define TV_BU_MASK			0x07ff0000
 # define TV_BU_SHIFT			16
 /*
@@ -3814,13 +3873,13 @@ enum skl_disp_power_wells {
 # define TV_AU_MASK			0x000003ff
 # define TV_AU_SHIFT			0
 
-#define TV_CSC_V		0x68020
+#define TV_CSC_V		_MMIO(0x68020)
 # define TV_RV_MASK			0x0fff0000
 # define TV_RV_SHIFT			16
 # define TV_GV_MASK			0x000007ff
 # define TV_GV_SHIFT			0
 
-#define TV_CSC_V2		0x68024
+#define TV_CSC_V2		_MMIO(0x68024)
 # define TV_BV_MASK			0x07ff0000
 # define TV_BV_SHIFT			16
 /*
@@ -3831,7 +3890,7 @@ enum skl_disp_power_wells {
 # define TV_AV_MASK			0x000007ff
 # define TV_AV_SHIFT			0
 
-#define TV_CLR_KNOBS		0x68028
+#define TV_CLR_KNOBS		_MMIO(0x68028)
 /* 2s-complement brightness adjustment */
 # define TV_BRIGHTNESS_MASK		0xff000000
 # define TV_BRIGHTNESS_SHIFT		24
@@ -3845,7 +3904,7 @@ enum skl_disp_power_wells {
 # define TV_HUE_MASK			0x000000ff
 # define TV_HUE_SHIFT			0
 
-#define TV_CLR_LEVEL		0x6802c
+#define TV_CLR_LEVEL		_MMIO(0x6802c)
 /* Controls the DAC level for black */
 # define TV_BLACK_LEVEL_MASK		0x01ff0000
 # define TV_BLACK_LEVEL_SHIFT		16
@@ -3853,7 +3912,7 @@ enum skl_disp_power_wells {
 # define TV_BLANK_LEVEL_MASK		0x000001ff
 # define TV_BLANK_LEVEL_SHIFT		0
 
-#define TV_H_CTL_1		0x68030
+#define TV_H_CTL_1		_MMIO(0x68030)
 /* Number of pixels in the hsync. */
 # define TV_HSYNC_END_MASK		0x1fff0000
 # define TV_HSYNC_END_SHIFT		16
@@ -3861,7 +3920,7 @@ enum skl_disp_power_wells {
 # define TV_HTOTAL_MASK			0x00001fff
 # define TV_HTOTAL_SHIFT		0
 
-#define TV_H_CTL_2		0x68034
+#define TV_H_CTL_2		_MMIO(0x68034)
 /* Enables the colorburst (needed for non-component color) */
 # define TV_BURST_ENA			(1 << 31)
 /* Offset of the colorburst from the start of hsync, in pixels minus one. */
@@ -3871,7 +3930,7 @@ enum skl_disp_power_wells {
 # define TV_HBURST_LEN_SHIFT		0
 # define TV_HBURST_LEN_MASK		0x0001fff
 
-#define TV_H_CTL_3		0x68038
+#define TV_H_CTL_3		_MMIO(0x68038)
 /* End of hblank, measured in pixels minus one from start of hsync */
 # define TV_HBLANK_END_SHIFT		16
 # define TV_HBLANK_END_MASK		0x1fff0000
@@ -3879,7 +3938,7 @@ enum skl_disp_power_wells {
 # define TV_HBLANK_START_SHIFT		0
 # define TV_HBLANK_START_MASK		0x0001fff
 
-#define TV_V_CTL_1		0x6803c
+#define TV_V_CTL_1		_MMIO(0x6803c)
 /* XXX */
 # define TV_NBR_END_SHIFT		16
 # define TV_NBR_END_MASK		0x07ff0000
@@ -3890,7 +3949,7 @@ enum skl_disp_power_wells {
 # define TV_VI_END_F2_SHIFT		0
 # define TV_VI_END_F2_MASK		0x0000003f
 
-#define TV_V_CTL_2		0x68040
+#define TV_V_CTL_2		_MMIO(0x68040)
 /* Length of vsync, in half lines */
 # define TV_VSYNC_LEN_MASK		0x07ff0000
 # define TV_VSYNC_LEN_SHIFT		16
@@ -3906,7 +3965,7 @@ enum skl_disp_power_wells {
 # define TV_VSYNC_START_F2_MASK		0x0000007f
 # define TV_VSYNC_START_F2_SHIFT	0
 
-#define TV_V_CTL_3		0x68044
+#define TV_V_CTL_3		_MMIO(0x68044)
 /* Enables generation of the equalization signal */
 # define TV_EQUAL_ENA			(1 << 31)
 /* Length of vsync, in half lines */
@@ -3924,7 +3983,7 @@ enum skl_disp_power_wells {
 # define TV_VEQ_START_F2_MASK		0x000007f
 # define TV_VEQ_START_F2_SHIFT		0
 
-#define TV_V_CTL_4		0x68048
+#define TV_V_CTL_4		_MMIO(0x68048)
 /*
  * Offset to start of vertical colorburst, measured in one less than the
  * number of lines from vertical start.
@@ -3938,7 +3997,7 @@ enum skl_disp_power_wells {
 # define TV_VBURST_END_F1_MASK		0x000000ff
 # define TV_VBURST_END_F1_SHIFT		0
 
-#define TV_V_CTL_5		0x6804c
+#define TV_V_CTL_5		_MMIO(0x6804c)
 /*
  * Offset to start of vertical colorburst, measured in one less than the
  * number of lines from vertical start.
@@ -3952,7 +4011,7 @@ enum skl_disp_power_wells {
 # define TV_VBURST_END_F2_MASK		0x000000ff
 # define TV_VBURST_END_F2_SHIFT		0
 
-#define TV_V_CTL_6		0x68050
+#define TV_V_CTL_6		_MMIO(0x68050)
 /*
  * Offset to start of vertical colorburst, measured in one less than the
  * number of lines from vertical start.
@@ -3966,7 +4025,7 @@ enum skl_disp_power_wells {
 # define TV_VBURST_END_F3_MASK		0x000000ff
 # define TV_VBURST_END_F3_SHIFT		0
 
-#define TV_V_CTL_7		0x68054
+#define TV_V_CTL_7		_MMIO(0x68054)
 /*
  * Offset to start of vertical colorburst, measured in one less than the
  * number of lines from vertical start.
@@ -3980,7 +4039,7 @@ enum skl_disp_power_wells {
 # define TV_VBURST_END_F4_MASK		0x000000ff
 # define TV_VBURST_END_F4_SHIFT		0
 
-#define TV_SC_CTL_1		0x68060
+#define TV_SC_CTL_1		_MMIO(0x68060)
 /* Turns on the first subcarrier phase generation DDA */
 # define TV_SC_DDA1_EN			(1 << 31)
 /* Turns on the first subcarrier phase generation DDA */
@@ -4002,7 +4061,7 @@ enum skl_disp_power_wells {
 # define TV_SCDDA1_INC_MASK		0x00000fff
 # define TV_SCDDA1_INC_SHIFT		0
 
-#define TV_SC_CTL_2		0x68064
+#define TV_SC_CTL_2		_MMIO(0x68064)
 /* Sets the rollover for the second subcarrier phase generation DDA */
 # define TV_SCDDA2_SIZE_MASK		0x7fff0000
 # define TV_SCDDA2_SIZE_SHIFT		16
@@ -4010,7 +4069,7 @@ enum skl_disp_power_wells {
 # define TV_SCDDA2_INC_MASK		0x00007fff
 # define TV_SCDDA2_INC_SHIFT		0
 
-#define TV_SC_CTL_3		0x68068
+#define TV_SC_CTL_3		_MMIO(0x68068)
 /* Sets the rollover for the third subcarrier phase generation DDA */
 # define TV_SCDDA3_SIZE_MASK		0x7fff0000
 # define TV_SCDDA3_SIZE_SHIFT		16
@@ -4018,7 +4077,7 @@ enum skl_disp_power_wells {
 # define TV_SCDDA3_INC_MASK		0x00007fff
 # define TV_SCDDA3_INC_SHIFT		0
 
-#define TV_WIN_POS		0x68070
+#define TV_WIN_POS		_MMIO(0x68070)
 /* X coordinate of the display from the start of horizontal active */
 # define TV_XPOS_MASK			0x1fff0000
 # define TV_XPOS_SHIFT			16
@@ -4026,7 +4085,7 @@ enum skl_disp_power_wells {
 # define TV_YPOS_MASK			0x00000fff
 # define TV_YPOS_SHIFT			0
 
-#define TV_WIN_SIZE		0x68074
+#define TV_WIN_SIZE		_MMIO(0x68074)
 /* Horizontal size of the display window, measured in pixels*/
 # define TV_XSIZE_MASK			0x1fff0000
 # define TV_XSIZE_SHIFT			16
@@ -4038,7 +4097,7 @@ enum skl_disp_power_wells {
 # define TV_YSIZE_MASK			0x00000fff
 # define TV_YSIZE_SHIFT			0
 
-#define TV_FILTER_CTL_1		0x68080
+#define TV_FILTER_CTL_1		_MMIO(0x68080)
 /*
  * Enables automatic scaling calculation.
  *
@@ -4071,7 +4130,7 @@ enum skl_disp_power_wells {
 # define TV_HSCALE_FRAC_MASK		0x00003fff
 # define TV_HSCALE_FRAC_SHIFT		0
 
-#define TV_FILTER_CTL_2		0x68084
+#define TV_FILTER_CTL_2		_MMIO(0x68084)
 /*
  * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
  *
@@ -4087,7 +4146,7 @@ enum skl_disp_power_wells {
 # define TV_VSCALE_FRAC_MASK		0x00007fff
 # define TV_VSCALE_FRAC_SHIFT		0
 
-#define TV_FILTER_CTL_3		0x68088
+#define TV_FILTER_CTL_3		_MMIO(0x68088)
 /*
  * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
  *
@@ -4107,7 +4166,7 @@ enum skl_disp_power_wells {
 # define TV_VSCALE_IP_FRAC_MASK		0x00007fff
 # define TV_VSCALE_IP_FRAC_SHIFT		0
 
-#define TV_CC_CONTROL		0x68090
+#define TV_CC_CONTROL		_MMIO(0x68090)
 # define TV_CC_ENABLE			(1 << 31)
 /*
  * Specifies which field to send the CC data in.
@@ -4123,7 +4182,7 @@ enum skl_disp_power_wells {
 # define TV_CC_LINE_MASK		0x0000003f
 # define TV_CC_LINE_SHIFT		0
 
-#define TV_CC_DATA		0x68094
+#define TV_CC_DATA		_MMIO(0x68094)
 # define TV_CC_RDY			(1 << 31)
 /* Second word of CC data to be transmitted. */
 # define TV_CC_DATA_2_MASK		0x007f0000
@@ -4132,20 +4191,20 @@ enum skl_disp_power_wells {
 # define TV_CC_DATA_1_MASK		0x0000007f
 # define TV_CC_DATA_1_SHIFT		0
 
-#define TV_H_LUMA(i)		(0x68100 + (i) * 4) /* 60 registers */
-#define TV_H_CHROMA(i)		(0x68200 + (i) * 4) /* 60 registers */
-#define TV_V_LUMA(i)		(0x68300 + (i) * 4) /* 43 registers */
-#define TV_V_CHROMA(i)		(0x68400 + (i) * 4) /* 43 registers */
+#define TV_H_LUMA(i)		_MMIO(0x68100 + (i) * 4) /* 60 registers */
+#define TV_H_CHROMA(i)		_MMIO(0x68200 + (i) * 4) /* 60 registers */
+#define TV_V_LUMA(i)		_MMIO(0x68300 + (i) * 4) /* 43 registers */
+#define TV_V_CHROMA(i)		_MMIO(0x68400 + (i) * 4) /* 43 registers */
 
 /* Display Port */
-#define DP_A				0x64000 /* eDP */
-#define DP_B				0x64100
-#define DP_C				0x64200
-#define DP_D				0x64300
+#define DP_A			_MMIO(0x64000) /* eDP */
+#define DP_B			_MMIO(0x64100)
+#define DP_C			_MMIO(0x64200)
+#define DP_D			_MMIO(0x64300)
 
-#define VLV_DP_B			(VLV_DISPLAY_BASE + DP_B)
-#define VLV_DP_C			(VLV_DISPLAY_BASE + DP_C)
-#define CHV_DP_D			(VLV_DISPLAY_BASE + DP_D)
+#define VLV_DP_B		_MMIO(VLV_DISPLAY_BASE + 0x64100)
+#define VLV_DP_C		_MMIO(VLV_DISPLAY_BASE + 0x64200)
+#define CHV_DP_D		_MMIO(VLV_DISPLAY_BASE + 0x64300)
 
 #define   DP_PORT_EN			(1 << 31)
 #define   DP_PIPEB_SELECT		(1 << 30)
@@ -4199,7 +4258,7 @@ enum skl_disp_power_wells {
 
 /* eDP */
 #define   DP_PLL_FREQ_270MHZ		(0 << 16)
-#define   DP_PLL_FREQ_160MHZ		(1 << 16)
+#define   DP_PLL_FREQ_162MHZ		(1 << 16)
 #define   DP_PLL_FREQ_MASK		(3 << 16)
 
 /* locked once port is enabled */
@@ -4232,33 +4291,36 @@ enum skl_disp_power_wells {
  * is 20 bytes in each direction, hence the 5 fixed
  * data registers
  */
-#define DPA_AUX_CH_CTL			0x64010
-#define DPA_AUX_CH_DATA1		0x64014
-#define DPA_AUX_CH_DATA2		0x64018
-#define DPA_AUX_CH_DATA3		0x6401c
-#define DPA_AUX_CH_DATA4		0x64020
-#define DPA_AUX_CH_DATA5		0x64024
-
-#define DPB_AUX_CH_CTL			0x64110
-#define DPB_AUX_CH_DATA1		0x64114
-#define DPB_AUX_CH_DATA2		0x64118
-#define DPB_AUX_CH_DATA3		0x6411c
-#define DPB_AUX_CH_DATA4		0x64120
-#define DPB_AUX_CH_DATA5		0x64124
-
-#define DPC_AUX_CH_CTL			0x64210
-#define DPC_AUX_CH_DATA1		0x64214
-#define DPC_AUX_CH_DATA2		0x64218
-#define DPC_AUX_CH_DATA3		0x6421c
-#define DPC_AUX_CH_DATA4		0x64220
-#define DPC_AUX_CH_DATA5		0x64224
-
-#define DPD_AUX_CH_CTL			0x64310
-#define DPD_AUX_CH_DATA1		0x64314
-#define DPD_AUX_CH_DATA2		0x64318
-#define DPD_AUX_CH_DATA3		0x6431c
-#define DPD_AUX_CH_DATA4		0x64320
-#define DPD_AUX_CH_DATA5		0x64324
+#define _DPA_AUX_CH_CTL		(dev_priv->info.display_mmio_offset + 0x64010)
+#define _DPA_AUX_CH_DATA1	(dev_priv->info.display_mmio_offset + 0x64014)
+#define _DPA_AUX_CH_DATA2	(dev_priv->info.display_mmio_offset + 0x64018)
+#define _DPA_AUX_CH_DATA3	(dev_priv->info.display_mmio_offset + 0x6401c)
+#define _DPA_AUX_CH_DATA4	(dev_priv->info.display_mmio_offset + 0x64020)
+#define _DPA_AUX_CH_DATA5	(dev_priv->info.display_mmio_offset + 0x64024)
+
+#define _DPB_AUX_CH_CTL		(dev_priv->info.display_mmio_offset + 0x64110)
+#define _DPB_AUX_CH_DATA1	(dev_priv->info.display_mmio_offset + 0x64114)
+#define _DPB_AUX_CH_DATA2	(dev_priv->info.display_mmio_offset + 0x64118)
+#define _DPB_AUX_CH_DATA3	(dev_priv->info.display_mmio_offset + 0x6411c)
+#define _DPB_AUX_CH_DATA4	(dev_priv->info.display_mmio_offset + 0x64120)
+#define _DPB_AUX_CH_DATA5	(dev_priv->info.display_mmio_offset + 0x64124)
+
+#define _DPC_AUX_CH_CTL		(dev_priv->info.display_mmio_offset + 0x64210)
+#define _DPC_AUX_CH_DATA1	(dev_priv->info.display_mmio_offset + 0x64214)
+#define _DPC_AUX_CH_DATA2	(dev_priv->info.display_mmio_offset + 0x64218)
+#define _DPC_AUX_CH_DATA3	(dev_priv->info.display_mmio_offset + 0x6421c)
+#define _DPC_AUX_CH_DATA4	(dev_priv->info.display_mmio_offset + 0x64220)
+#define _DPC_AUX_CH_DATA5	(dev_priv->info.display_mmio_offset + 0x64224)
+
+#define _DPD_AUX_CH_CTL		(dev_priv->info.display_mmio_offset + 0x64310)
+#define _DPD_AUX_CH_DATA1	(dev_priv->info.display_mmio_offset + 0x64314)
+#define _DPD_AUX_CH_DATA2	(dev_priv->info.display_mmio_offset + 0x64318)
+#define _DPD_AUX_CH_DATA3	(dev_priv->info.display_mmio_offset + 0x6431c)
+#define _DPD_AUX_CH_DATA4	(dev_priv->info.display_mmio_offset + 0x64320)
+#define _DPD_AUX_CH_DATA5	(dev_priv->info.display_mmio_offset + 0x64324)
+
+#define DP_AUX_CH_CTL(port)	_MMIO_PORT(port, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL)
+#define DP_AUX_CH_DATA(port, i)	_MMIO(_PORT(port, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
 
 #define   DP_AUX_CH_CTL_SEND_BUSY	    (1 << 31)
 #define   DP_AUX_CH_CTL_DONE		    (1 << 30)
@@ -4335,10 +4397,10 @@ enum skl_disp_power_wells {
 #define _PIPEB_LINK_N_G4X	0x71064
 #define   PIPEA_DP_LINK_N_MASK			(0xffffff)
 
-#define PIPE_DATA_M_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X)
-#define PIPE_DATA_N_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X)
-#define PIPE_LINK_M_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X)
-#define PIPE_LINK_N_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X)
+#define PIPE_DATA_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X)
+#define PIPE_DATA_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X)
+#define PIPE_LINK_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X)
+#define PIPE_LINK_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X)
 
 /* Display & cursor control */
 
@@ -4454,15 +4516,15 @@ enum skl_disp_power_wells {
  */
 #define PIPE_EDP_OFFSET	0x7f000
 
-#define _PIPE2(pipe, reg) (dev_priv->info.pipe_offsets[pipe] - \
+#define _MMIO_PIPE2(pipe, reg) _MMIO(dev_priv->info.pipe_offsets[pipe] - \
 	dev_priv->info.pipe_offsets[PIPE_A] + (reg) + \
 	dev_priv->info.display_mmio_offset)
 
-#define PIPECONF(pipe) _PIPE2(pipe, _PIPEACONF)
-#define PIPEDSL(pipe)  _PIPE2(pipe, _PIPEADSL)
-#define PIPEFRAME(pipe) _PIPE2(pipe, _PIPEAFRAMEHIGH)
-#define PIPEFRAMEPIXEL(pipe)  _PIPE2(pipe, _PIPEAFRAMEPIXEL)
-#define PIPESTAT(pipe) _PIPE2(pipe, _PIPEASTAT)
+#define PIPECONF(pipe)		_MMIO_PIPE2(pipe, _PIPEACONF)
+#define PIPEDSL(pipe)		_MMIO_PIPE2(pipe, _PIPEADSL)
+#define PIPEFRAME(pipe)		_MMIO_PIPE2(pipe, _PIPEAFRAMEHIGH)
+#define PIPEFRAMEPIXEL(pipe)	_MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL)
+#define PIPESTAT(pipe)		_MMIO_PIPE2(pipe, _PIPEASTAT)
 
 #define _PIPE_MISC_A			0x70030
 #define _PIPE_MISC_B			0x71030
@@ -4474,9 +4536,9 @@ enum skl_disp_power_wells {
 #define   PIPEMISC_DITHER_ENABLE	(1<<4)
 #define   PIPEMISC_DITHER_TYPE_MASK	(3<<2)
 #define   PIPEMISC_DITHER_TYPE_SP	(0<<2)
-#define PIPEMISC(pipe) _PIPE2(pipe, _PIPE_MISC_A)
+#define PIPEMISC(pipe)			_MMIO_PIPE2(pipe, _PIPE_MISC_A)
 
-#define VLV_DPFLIPSTAT				(VLV_DISPLAY_BASE + 0x70028)
+#define VLV_DPFLIPSTAT				_MMIO(VLV_DISPLAY_BASE + 0x70028)
 #define   PIPEB_LINE_COMPARE_INT_EN		(1<<29)
 #define   PIPEB_HLINE_INT_EN			(1<<28)
 #define   PIPEB_VBLANK_INT_EN			(1<<27)
@@ -4497,7 +4559,7 @@ enum skl_disp_power_wells {
 #define   SPRITEE_FLIPDONE_INT_EN		(1<<9)
 #define   PLANEC_FLIPDONE_INT_EN		(1<<8)
 
-#define DPINVGTT				(VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */
+#define DPINVGTT				_MMIO(VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */
 #define   SPRITEF_INVALID_GTT_INT_EN		(1<<27)
 #define   SPRITEE_INVALID_GTT_INT_EN		(1<<26)
 #define   PLANEC_INVALID_GTT_INT_EN		(1<<25)
@@ -4527,7 +4589,7 @@ enum skl_disp_power_wells {
 #define   DPINVGTT_STATUS_MASK			0xff
 #define   DPINVGTT_STATUS_MASK_CHV		0xfff
 
-#define DSPARB			(dev_priv->info.display_mmio_offset + 0x70030)
+#define DSPARB			_MMIO(dev_priv->info.display_mmio_offset + 0x70030)
 #define   DSPARB_CSTART_MASK	(0x7f << 7)
 #define   DSPARB_CSTART_SHIFT	7
 #define   DSPARB_BSTART_MASK	(0x7f)
@@ -4542,7 +4604,7 @@ enum skl_disp_power_wells {
 #define   DSPARB_SPRITEC_MASK_VLV	(0xff << 16)
 #define   DSPARB_SPRITED_SHIFT_VLV	24
 #define   DSPARB_SPRITED_MASK_VLV	(0xff << 24)
-#define DSPARB2			(VLV_DISPLAY_BASE + 0x70060) /* vlv/chv */
+#define DSPARB2				_MMIO(VLV_DISPLAY_BASE + 0x70060) /* vlv/chv */
 #define   DSPARB_SPRITEA_HI_SHIFT_VLV	0
 #define   DSPARB_SPRITEA_HI_MASK_VLV	(0x1 << 0)
 #define   DSPARB_SPRITEB_HI_SHIFT_VLV	4
@@ -4555,14 +4617,14 @@ enum skl_disp_power_wells {
 #define   DSPARB_SPRITEE_HI_MASK_VLV	(0x1 << 16)
 #define   DSPARB_SPRITEF_HI_SHIFT_VLV	20
 #define   DSPARB_SPRITEF_HI_MASK_VLV	(0x1 << 20)
-#define DSPARB3			(VLV_DISPLAY_BASE + 0x7006c) /* chv */
+#define DSPARB3				_MMIO(VLV_DISPLAY_BASE + 0x7006c) /* chv */
 #define   DSPARB_SPRITEE_SHIFT_VLV	0
 #define   DSPARB_SPRITEE_MASK_VLV	(0xff << 0)
 #define   DSPARB_SPRITEF_SHIFT_VLV	8
 #define   DSPARB_SPRITEF_MASK_VLV	(0xff << 8)
 
 /* pnv/gen4/g4x/vlv/chv */
-#define DSPFW1			(dev_priv->info.display_mmio_offset + 0x70034)
+#define DSPFW1		_MMIO(dev_priv->info.display_mmio_offset + 0x70034)
 #define   DSPFW_SR_SHIFT		23
 #define   DSPFW_SR_MASK			(0x1ff<<23)
 #define   DSPFW_CURSORB_SHIFT		16
@@ -4573,7 +4635,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_PLANEA_SHIFT		0
 #define   DSPFW_PLANEA_MASK		(0x7f<<0)
 #define   DSPFW_PLANEA_MASK_VLV		(0xff<<0) /* vlv/chv */
-#define DSPFW2			(dev_priv->info.display_mmio_offset + 0x70038)
+#define DSPFW2		_MMIO(dev_priv->info.display_mmio_offset + 0x70038)
 #define   DSPFW_FBC_SR_EN		(1<<31)	  /* g4x */
 #define   DSPFW_FBC_SR_SHIFT		28
 #define   DSPFW_FBC_SR_MASK		(0x7<<28) /* g4x */
@@ -4589,7 +4651,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_SPRITEA_SHIFT		0
 #define   DSPFW_SPRITEA_MASK		(0x7f<<0) /* g4x */
 #define   DSPFW_SPRITEA_MASK_VLV	(0xff<<0) /* vlv/chv */
-#define DSPFW3			(dev_priv->info.display_mmio_offset + 0x7003c)
+#define DSPFW3		_MMIO(dev_priv->info.display_mmio_offset + 0x7003c)
 #define   DSPFW_HPLL_SR_EN		(1<<31)
 #define   PINEVIEW_SELF_REFRESH_EN	(1<<30)
 #define   DSPFW_CURSOR_SR_SHIFT		24
@@ -4600,14 +4662,14 @@ enum skl_disp_power_wells {
 #define   DSPFW_HPLL_SR_MASK		(0x1ff<<0)
 
 /* vlv/chv */
-#define DSPFW4			(VLV_DISPLAY_BASE + 0x70070)
+#define DSPFW4		_MMIO(VLV_DISPLAY_BASE + 0x70070)
 #define   DSPFW_SPRITEB_WM1_SHIFT	16
 #define   DSPFW_SPRITEB_WM1_MASK	(0xff<<16)
 #define   DSPFW_CURSORA_WM1_SHIFT	8
 #define   DSPFW_CURSORA_WM1_MASK	(0x3f<<8)
 #define   DSPFW_SPRITEA_WM1_SHIFT	0
 #define   DSPFW_SPRITEA_WM1_MASK	(0xff<<0)
-#define DSPFW5			(VLV_DISPLAY_BASE + 0x70074)
+#define DSPFW5		_MMIO(VLV_DISPLAY_BASE + 0x70074)
 #define   DSPFW_PLANEB_WM1_SHIFT	24
 #define   DSPFW_PLANEB_WM1_MASK		(0xff<<24)
 #define   DSPFW_PLANEA_WM1_SHIFT	16
@@ -4616,11 +4678,11 @@ enum skl_disp_power_wells {
 #define   DSPFW_CURSORB_WM1_MASK	(0x3f<<8)
 #define   DSPFW_CURSOR_SR_WM1_SHIFT	0
 #define   DSPFW_CURSOR_SR_WM1_MASK	(0x3f<<0)
-#define DSPFW6			(VLV_DISPLAY_BASE + 0x70078)
+#define DSPFW6		_MMIO(VLV_DISPLAY_BASE + 0x70078)
 #define   DSPFW_SR_WM1_SHIFT		0
 #define   DSPFW_SR_WM1_MASK		(0x1ff<<0)
-#define DSPFW7			(VLV_DISPLAY_BASE + 0x7007c)
-#define DSPFW7_CHV		(VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */
+#define DSPFW7		_MMIO(VLV_DISPLAY_BASE + 0x7007c)
+#define DSPFW7_CHV	_MMIO(VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */
 #define   DSPFW_SPRITED_WM1_SHIFT	24
 #define   DSPFW_SPRITED_WM1_MASK	(0xff<<24)
 #define   DSPFW_SPRITED_SHIFT		16
@@ -4629,7 +4691,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_SPRITEC_WM1_MASK	(0xff<<8)
 #define   DSPFW_SPRITEC_SHIFT		0
 #define   DSPFW_SPRITEC_MASK_VLV	(0xff<<0)
-#define DSPFW8_CHV		(VLV_DISPLAY_BASE + 0x700b8)
+#define DSPFW8_CHV	_MMIO(VLV_DISPLAY_BASE + 0x700b8)
 #define   DSPFW_SPRITEF_WM1_SHIFT	24
 #define   DSPFW_SPRITEF_WM1_MASK	(0xff<<24)
 #define   DSPFW_SPRITEF_SHIFT		16
@@ -4638,7 +4700,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_SPRITEE_WM1_MASK	(0xff<<8)
 #define   DSPFW_SPRITEE_SHIFT		0
 #define   DSPFW_SPRITEE_MASK_VLV	(0xff<<0)
-#define DSPFW9_CHV		(VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */
+#define DSPFW9_CHV	_MMIO(VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */
 #define   DSPFW_PLANEC_WM1_SHIFT	24
 #define   DSPFW_PLANEC_WM1_MASK		(0xff<<24)
 #define   DSPFW_PLANEC_SHIFT		16
@@ -4649,7 +4711,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_CURSORC_MASK		(0x3f<<0)
 
 /* vlv/chv high order bits */
-#define DSPHOWM			(VLV_DISPLAY_BASE + 0x70064)
+#define DSPHOWM		_MMIO(VLV_DISPLAY_BASE + 0x70064)
 #define   DSPFW_SR_HI_SHIFT		24
 #define   DSPFW_SR_HI_MASK		(3<<24) /* 2 bits for chv, 1 for vlv */
 #define   DSPFW_SPRITEF_HI_SHIFT	23
@@ -4670,7 +4732,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_SPRITEA_HI_MASK		(1<<4)
 #define   DSPFW_PLANEA_HI_SHIFT		0
 #define   DSPFW_PLANEA_HI_MASK		(1<<0)
-#define DSPHOWM1		(VLV_DISPLAY_BASE + 0x70068)
+#define DSPHOWM1	_MMIO(VLV_DISPLAY_BASE + 0x70068)
 #define   DSPFW_SR_WM1_HI_SHIFT		24
 #define   DSPFW_SR_WM1_HI_MASK		(3<<24) /* 2 bits for chv, 1 for vlv */
 #define   DSPFW_SPRITEF_WM1_HI_SHIFT	23
@@ -4693,7 +4755,7 @@ enum skl_disp_power_wells {
 #define   DSPFW_PLANEA_WM1_HI_MASK	(1<<0)
 
 /* drain latency register values*/
-#define VLV_DDL(pipe)			(VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe))
+#define VLV_DDL(pipe)			_MMIO(VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe))
 #define DDL_CURSOR_SHIFT		24
 #define DDL_SPRITE_SHIFT(sprite)	(8+8*(sprite))
 #define DDL_PLANE_SHIFT			0
@@ -4701,7 +4763,7 @@ enum skl_disp_power_wells {
 #define DDL_PRECISION_LOW		(0<<7)
 #define DRAIN_LATENCY_MASK		0x7f
 
-#define CBR1_VLV			(VLV_DISPLAY_BASE + 0x70400)
+#define CBR1_VLV			_MMIO(VLV_DISPLAY_BASE + 0x70400)
 #define  CBR_PND_DEADLINE_DISABLE	(1<<31)
 #define  CBR_PWM_CLOCK_MUX_SELECT	(1<<30)
 
@@ -4739,51 +4801,51 @@ enum skl_disp_power_wells {
 #define I965_CURSOR_DFT_WM	8
 
 /* Watermark register definitions for SKL */
-#define CUR_WM_A_0		0x70140
-#define CUR_WM_B_0		0x71140
-#define PLANE_WM_1_A_0		0x70240
-#define PLANE_WM_1_B_0		0x71240
-#define PLANE_WM_2_A_0		0x70340
-#define PLANE_WM_2_B_0		0x71340
-#define PLANE_WM_TRANS_1_A_0	0x70268
-#define PLANE_WM_TRANS_1_B_0	0x71268
-#define PLANE_WM_TRANS_2_A_0	0x70368
-#define PLANE_WM_TRANS_2_B_0	0x71368
-#define CUR_WM_TRANS_A_0	0x70168
-#define CUR_WM_TRANS_B_0	0x71168
+#define _CUR_WM_A_0		0x70140
+#define _CUR_WM_B_0		0x71140
+#define _PLANE_WM_1_A_0		0x70240
+#define _PLANE_WM_1_B_0		0x71240
+#define _PLANE_WM_2_A_0		0x70340
+#define _PLANE_WM_2_B_0		0x71340
+#define _PLANE_WM_TRANS_1_A_0	0x70268
+#define _PLANE_WM_TRANS_1_B_0	0x71268
+#define _PLANE_WM_TRANS_2_A_0	0x70368
+#define _PLANE_WM_TRANS_2_B_0	0x71368
+#define _CUR_WM_TRANS_A_0	0x70168
+#define _CUR_WM_TRANS_B_0	0x71168
 #define   PLANE_WM_EN		(1 << 31)
 #define   PLANE_WM_LINES_SHIFT	14
 #define   PLANE_WM_LINES_MASK	0x1f
 #define   PLANE_WM_BLOCKS_MASK	0x3ff
 
-#define CUR_WM_0(pipe) _PIPE(pipe, CUR_WM_A_0, CUR_WM_B_0)
-#define CUR_WM(pipe, level) (CUR_WM_0(pipe) + ((4) * (level)))
-#define CUR_WM_TRANS(pipe) _PIPE(pipe, CUR_WM_TRANS_A_0, CUR_WM_TRANS_B_0)
+#define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0)
+#define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level)))
+#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A_0, _CUR_WM_TRANS_B_0)
 
-#define _PLANE_WM_1(pipe) _PIPE(pipe, PLANE_WM_1_A_0, PLANE_WM_1_B_0)
-#define _PLANE_WM_2(pipe) _PIPE(pipe, PLANE_WM_2_A_0, PLANE_WM_2_B_0)
+#define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0)
+#define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0)
 #define _PLANE_WM_BASE(pipe, plane)	\
 			_PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe))
 #define PLANE_WM(pipe, plane, level)	\
-			(_PLANE_WM_BASE(pipe, plane) + ((4) * (level)))
+			_MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level)))
 #define _PLANE_WM_TRANS_1(pipe)	\
-			_PIPE(pipe, PLANE_WM_TRANS_1_A_0, PLANE_WM_TRANS_1_B_0)
+			_PIPE(pipe, _PLANE_WM_TRANS_1_A_0, _PLANE_WM_TRANS_1_B_0)
 #define _PLANE_WM_TRANS_2(pipe)	\
-			_PIPE(pipe, PLANE_WM_TRANS_2_A_0, PLANE_WM_TRANS_2_B_0)
+			_PIPE(pipe, _PLANE_WM_TRANS_2_A_0, _PLANE_WM_TRANS_2_B_0)
 #define PLANE_WM_TRANS(pipe, plane)	\
-		_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe))
+	_MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe)))
 
 /* define the Watermark register on Ironlake */
-#define WM0_PIPEA_ILK		0x45100
+#define WM0_PIPEA_ILK		_MMIO(0x45100)
 #define  WM0_PIPE_PLANE_MASK	(0xffff<<16)
 #define  WM0_PIPE_PLANE_SHIFT	16
 #define  WM0_PIPE_SPRITE_MASK	(0xff<<8)
 #define  WM0_PIPE_SPRITE_SHIFT	8
 #define  WM0_PIPE_CURSOR_MASK	(0xff)
 
-#define WM0_PIPEB_ILK		0x45104
-#define WM0_PIPEC_IVB		0x45200
-#define WM1_LP_ILK		0x45108
+#define WM0_PIPEB_ILK		_MMIO(0x45104)
+#define WM0_PIPEC_IVB		_MMIO(0x45200)
+#define WM1_LP_ILK		_MMIO(0x45108)
 #define  WM1_LP_SR_EN		(1<<31)
 #define  WM1_LP_LATENCY_SHIFT	24
 #define  WM1_LP_LATENCY_MASK	(0x7f<<24)
@@ -4793,13 +4855,13 @@ enum skl_disp_power_wells {
 #define  WM1_LP_SR_MASK		(0x7ff<<8)
 #define  WM1_LP_SR_SHIFT	8
 #define  WM1_LP_CURSOR_MASK	(0xff)
-#define WM2_LP_ILK		0x4510c
+#define WM2_LP_ILK		_MMIO(0x4510c)
 #define  WM2_LP_EN		(1<<31)
-#define WM3_LP_ILK		0x45110
+#define WM3_LP_ILK		_MMIO(0x45110)
 #define  WM3_LP_EN		(1<<31)
-#define WM1S_LP_ILK		0x45120
-#define WM2S_LP_IVB		0x45124
-#define WM3S_LP_IVB		0x45128
+#define WM1S_LP_ILK		_MMIO(0x45120)
+#define WM2S_LP_IVB		_MMIO(0x45124)
+#define WM3S_LP_IVB		_MMIO(0x45128)
 #define  WM1S_LP_EN		(1<<31)
 
 #define HSW_WM_LP_VAL(lat, fbc, pri, cur) \
@@ -4807,7 +4869,7 @@ enum skl_disp_power_wells {
 	 ((fbc) << WM1_LP_FBC_SHIFT) | ((pri) << WM1_LP_SR_SHIFT) | (cur))
 
 /* Memory latency timer register */
-#define MLTR_ILK		0x11222
+#define MLTR_ILK		_MMIO(0x11222)
 #define  MLTR_WM1_SHIFT		0
 #define  MLTR_WM2_SHIFT		8
 /* the unit of memory self-refresh latency time is 0.5us */
@@ -4815,7 +4877,7 @@ enum skl_disp_power_wells {
 
 
 /* the address where we get all kinds of latency value */
-#define SSKPD			0x5d10
+#define SSKPD			_MMIO(0x5d10)
 #define SSKPD_WM_MASK		0x3f
 #define SSKPD_WM0_SHIFT		0
 #define SSKPD_WM1_SHIFT		8
@@ -4848,8 +4910,8 @@ enum skl_disp_power_wells {
 /* GM45+ just has to be different */
 #define _PIPEA_FRMCOUNT_G4X	0x70040
 #define _PIPEA_FLIPCOUNT_G4X	0x70044
-#define PIPE_FRMCOUNT_G4X(pipe) _PIPE2(pipe, _PIPEA_FRMCOUNT_G4X)
-#define PIPE_FLIPCOUNT_G4X(pipe) _PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X)
+#define PIPE_FRMCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FRMCOUNT_G4X)
+#define PIPE_FLIPCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X)
 
 /* Cursor A & B regs */
 #define _CURACNTR		0x70080
@@ -4887,7 +4949,7 @@ enum skl_disp_power_wells {
 #define   CURSOR_POS_SIGN       0x8000
 #define   CURSOR_X_SHIFT        0
 #define   CURSOR_Y_SHIFT        16
-#define CURSIZE			0x700a0
+#define CURSIZE			_MMIO(0x700a0)
 #define _CURBCNTR		0x700c0
 #define _CURBBASE		0x700c4
 #define _CURBPOS		0x700c8
@@ -4896,7 +4958,7 @@ enum skl_disp_power_wells {
 #define _CURBBASE_IVB		0x71084
 #define _CURBPOS_IVB		0x71088
 
-#define _CURSOR2(pipe, reg) (dev_priv->info.cursor_offsets[(pipe)] - \
+#define _CURSOR2(pipe, reg) _MMIO(dev_priv->info.cursor_offsets[(pipe)] - \
 	dev_priv->info.cursor_offsets[PIPE_A] + (reg) + \
 	dev_priv->info.display_mmio_offset)
 
@@ -4957,16 +5019,16 @@ enum skl_disp_power_wells {
 #define _DSPAOFFSET				0x701A4 /* HSW */
 #define _DSPASURFLIVE				0x701AC
 
-#define DSPCNTR(plane) _PIPE2(plane, _DSPACNTR)
-#define DSPADDR(plane) _PIPE2(plane, _DSPAADDR)
-#define DSPSTRIDE(plane) _PIPE2(plane, _DSPASTRIDE)
-#define DSPPOS(plane) _PIPE2(plane, _DSPAPOS)
-#define DSPSIZE(plane) _PIPE2(plane, _DSPASIZE)
-#define DSPSURF(plane) _PIPE2(plane, _DSPASURF)
-#define DSPTILEOFF(plane) _PIPE2(plane, _DSPATILEOFF)
-#define DSPLINOFF(plane) DSPADDR(plane)
-#define DSPOFFSET(plane) _PIPE2(plane, _DSPAOFFSET)
-#define DSPSURFLIVE(plane) _PIPE2(plane, _DSPASURFLIVE)
+#define DSPCNTR(plane)		_MMIO_PIPE2(plane, _DSPACNTR)
+#define DSPADDR(plane)		_MMIO_PIPE2(plane, _DSPAADDR)
+#define DSPSTRIDE(plane)	_MMIO_PIPE2(plane, _DSPASTRIDE)
+#define DSPPOS(plane)		_MMIO_PIPE2(plane, _DSPAPOS)
+#define DSPSIZE(plane)		_MMIO_PIPE2(plane, _DSPASIZE)
+#define DSPSURF(plane)		_MMIO_PIPE2(plane, _DSPASURF)
+#define DSPTILEOFF(plane)	_MMIO_PIPE2(plane, _DSPATILEOFF)
+#define DSPLINOFF(plane)	DSPADDR(plane)
+#define DSPOFFSET(plane)	_MMIO_PIPE2(plane, _DSPAOFFSET)
+#define DSPSURFLIVE(plane)	_MMIO_PIPE2(plane, _DSPASURFLIVE)
 
 /* CHV pipe B blender and primary plane */
 #define _CHV_BLEND_A		0x60a00
@@ -4980,11 +5042,11 @@ enum skl_disp_power_wells {
 #define _PRIMCNSTALPHA_A	0x60a10
 #define   PRIM_CONST_ALPHA_ENABLE	(1<<31)
 
-#define CHV_BLEND(pipe) _TRANSCODER2(pipe, _CHV_BLEND_A)
-#define CHV_CANVAS(pipe) _TRANSCODER2(pipe, _CHV_CANVAS_A)
-#define PRIMPOS(plane) _TRANSCODER2(plane, _PRIMPOS_A)
-#define PRIMSIZE(plane) _TRANSCODER2(plane, _PRIMSIZE_A)
-#define PRIMCNSTALPHA(plane) _TRANSCODER2(plane, _PRIMCNSTALPHA_A)
+#define CHV_BLEND(pipe)		_MMIO_TRANS2(pipe, _CHV_BLEND_A)
+#define CHV_CANVAS(pipe)	_MMIO_TRANS2(pipe, _CHV_CANVAS_A)
+#define PRIMPOS(plane)		_MMIO_TRANS2(plane, _PRIMPOS_A)
+#define PRIMSIZE(plane)		_MMIO_TRANS2(plane, _PRIMSIZE_A)
+#define PRIMCNSTALPHA(plane)	_MMIO_TRANS2(plane, _PRIMCNSTALPHA_A)
 
 /* Display/Sprite base address macros */
 #define DISP_BASEADDR_MASK	(0xfffff000)
@@ -5002,9 +5064,10 @@ enum skl_disp_power_wells {
  * [10:1f] all
  * [30:32] all
  */
-#define SWF0(i)	(dev_priv->info.display_mmio_offset + 0x70410 + (i) * 4)
-#define SWF1(i)	(dev_priv->info.display_mmio_offset + 0x71410 + (i) * 4)
-#define SWF3(i)	(dev_priv->info.display_mmio_offset + 0x72414 + (i) * 4)
+#define SWF0(i)	_MMIO(dev_priv->info.display_mmio_offset + 0x70410 + (i) * 4)
+#define SWF1(i)	_MMIO(dev_priv->info.display_mmio_offset + 0x71410 + (i) * 4)
+#define SWF3(i)	_MMIO(dev_priv->info.display_mmio_offset + 0x72414 + (i) * 4)
+#define SWF_ILK(i)	_MMIO(0x4F000 + (i) * 4)
 
 /* Pipe B */
 #define _PIPEBDSL		(dev_priv->info.display_mmio_offset + 0x71000)
@@ -5086,18 +5149,18 @@ enum skl_disp_power_wells {
 #define _DVSBSCALE		0x73204
 #define _DVSBGAMC		0x73300
 
-#define DVSCNTR(pipe) _PIPE(pipe, _DVSACNTR, _DVSBCNTR)
-#define DVSLINOFF(pipe) _PIPE(pipe, _DVSALINOFF, _DVSBLINOFF)
-#define DVSSTRIDE(pipe) _PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE)
-#define DVSPOS(pipe) _PIPE(pipe, _DVSAPOS, _DVSBPOS)
-#define DVSSURF(pipe) _PIPE(pipe, _DVSASURF, _DVSBSURF)
-#define DVSKEYMAX(pipe) _PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL)
-#define DVSSIZE(pipe) _PIPE(pipe, _DVSASIZE, _DVSBSIZE)
-#define DVSSCALE(pipe) _PIPE(pipe, _DVSASCALE, _DVSBSCALE)
-#define DVSTILEOFF(pipe) _PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF)
-#define DVSKEYVAL(pipe) _PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL)
-#define DVSKEYMSK(pipe) _PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK)
-#define DVSSURFLIVE(pipe) _PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE)
+#define DVSCNTR(pipe) _MMIO_PIPE(pipe, _DVSACNTR, _DVSBCNTR)
+#define DVSLINOFF(pipe) _MMIO_PIPE(pipe, _DVSALINOFF, _DVSBLINOFF)
+#define DVSSTRIDE(pipe) _MMIO_PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE)
+#define DVSPOS(pipe) _MMIO_PIPE(pipe, _DVSAPOS, _DVSBPOS)
+#define DVSSURF(pipe) _MMIO_PIPE(pipe, _DVSASURF, _DVSBSURF)
+#define DVSKEYMAX(pipe) _MMIO_PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL)
+#define DVSSIZE(pipe) _MMIO_PIPE(pipe, _DVSASIZE, _DVSBSIZE)
+#define DVSSCALE(pipe) _MMIO_PIPE(pipe, _DVSASCALE, _DVSBSCALE)
+#define DVSTILEOFF(pipe) _MMIO_PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF)
+#define DVSKEYVAL(pipe) _MMIO_PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL)
+#define DVSKEYMSK(pipe) _MMIO_PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK)
+#define DVSSURFLIVE(pipe) _MMIO_PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE)
 
 #define _SPRA_CTL		0x70280
 #define   SPRITE_ENABLE			(1<<31)
@@ -5160,20 +5223,20 @@ enum skl_disp_power_wells {
 #define _SPRB_SCALE		0x71304
 #define _SPRB_GAMC		0x71400
 
-#define SPRCTL(pipe) _PIPE(pipe, _SPRA_CTL, _SPRB_CTL)
-#define SPRLINOFF(pipe) _PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF)
-#define SPRSTRIDE(pipe) _PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE)
-#define SPRPOS(pipe) _PIPE(pipe, _SPRA_POS, _SPRB_POS)
-#define SPRSIZE(pipe) _PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE)
-#define SPRKEYVAL(pipe) _PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL)
-#define SPRKEYMSK(pipe) _PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK)
-#define SPRSURF(pipe) _PIPE(pipe, _SPRA_SURF, _SPRB_SURF)
-#define SPRKEYMAX(pipe) _PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX)
-#define SPRTILEOFF(pipe) _PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF)
-#define SPROFFSET(pipe) _PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET)
-#define SPRSCALE(pipe) _PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE)
-#define SPRGAMC(pipe) _PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC)
-#define SPRSURFLIVE(pipe) _PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE)
+#define SPRCTL(pipe) _MMIO_PIPE(pipe, _SPRA_CTL, _SPRB_CTL)
+#define SPRLINOFF(pipe) _MMIO_PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF)
+#define SPRSTRIDE(pipe) _MMIO_PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE)
+#define SPRPOS(pipe) _MMIO_PIPE(pipe, _SPRA_POS, _SPRB_POS)
+#define SPRSIZE(pipe) _MMIO_PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE)
+#define SPRKEYVAL(pipe) _MMIO_PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL)
+#define SPRKEYMSK(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK)
+#define SPRSURF(pipe) _MMIO_PIPE(pipe, _SPRA_SURF, _SPRB_SURF)
+#define SPRKEYMAX(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX)
+#define SPRTILEOFF(pipe) _MMIO_PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF)
+#define SPROFFSET(pipe) _MMIO_PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET)
+#define SPRSCALE(pipe) _MMIO_PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE)
+#define SPRGAMC(pipe) _MMIO_PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC)
+#define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE)
 
 #define _SPACNTR		(VLV_DISPLAY_BASE + 0x72180)
 #define   SP_ENABLE			(1<<31)
@@ -5223,18 +5286,18 @@ enum skl_disp_power_wells {
 #define _SPBCONSTALPHA		(VLV_DISPLAY_BASE + 0x722a8)
 #define _SPBGAMC		(VLV_DISPLAY_BASE + 0x722f4)
 
-#define SPCNTR(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPACNTR, _SPBCNTR)
-#define SPLINOFF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPALINOFF, _SPBLINOFF)
-#define SPSTRIDE(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASTRIDE, _SPBSTRIDE)
-#define SPPOS(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAPOS, _SPBPOS)
-#define SPSIZE(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASIZE, _SPBSIZE)
-#define SPKEYMINVAL(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMINVAL, _SPBKEYMINVAL)
-#define SPKEYMSK(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMSK, _SPBKEYMSK)
-#define SPSURF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPASURF, _SPBSURF)
-#define SPKEYMAXVAL(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
-#define SPTILEOFF(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPATILEOFF, _SPBTILEOFF)
-#define SPCONSTALPHA(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPACONSTALPHA, _SPBCONSTALPHA)
-#define SPGAMC(pipe, plane) _PIPE((pipe) * 2 + (plane), _SPAGAMC, _SPBGAMC)
+#define SPCNTR(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPACNTR, _SPBCNTR)
+#define SPLINOFF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPALINOFF, _SPBLINOFF)
+#define SPSTRIDE(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASTRIDE, _SPBSTRIDE)
+#define SPPOS(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAPOS, _SPBPOS)
+#define SPSIZE(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASIZE, _SPBSIZE)
+#define SPKEYMINVAL(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMINVAL, _SPBKEYMINVAL)
+#define SPKEYMSK(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMSK, _SPBKEYMSK)
+#define SPSURF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASURF, _SPBSURF)
+#define SPKEYMAXVAL(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
+#define SPTILEOFF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPATILEOFF, _SPBTILEOFF)
+#define SPCONSTALPHA(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPACONSTALPHA, _SPBCONSTALPHA)
+#define SPGAMC(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAGAMC, _SPBGAMC)
 
 /*
  * CHV pipe B sprite CSC
@@ -5243,29 +5306,29 @@ enum skl_disp_power_wells {
  * |yg| = |c3 c4 c5| x |yg + yg_ioff| + |yg_ooff|
  * |cb|   |c6 c7 c8|   |cb + cr_ioff|   |cb_ooff|
  */
-#define SPCSCYGOFF(sprite)	(VLV_DISPLAY_BASE + 0x6d900 + (sprite) * 0x1000)
-#define SPCSCCBOFF(sprite)	(VLV_DISPLAY_BASE + 0x6d904 + (sprite) * 0x1000)
-#define SPCSCCROFF(sprite)	(VLV_DISPLAY_BASE + 0x6d908 + (sprite) * 0x1000)
+#define SPCSCYGOFF(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d900 + (sprite) * 0x1000)
+#define SPCSCCBOFF(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d904 + (sprite) * 0x1000)
+#define SPCSCCROFF(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d908 + (sprite) * 0x1000)
 #define  SPCSC_OOFF(x)		(((x) & 0x7ff) << 16) /* s11 */
 #define  SPCSC_IOFF(x)		(((x) & 0x7ff) << 0) /* s11 */
 
-#define SPCSCC01(sprite)	(VLV_DISPLAY_BASE + 0x6d90c + (sprite) * 0x1000)
-#define SPCSCC23(sprite)	(VLV_DISPLAY_BASE + 0x6d910 + (sprite) * 0x1000)
-#define SPCSCC45(sprite)	(VLV_DISPLAY_BASE + 0x6d914 + (sprite) * 0x1000)
-#define SPCSCC67(sprite)	(VLV_DISPLAY_BASE + 0x6d918 + (sprite) * 0x1000)
-#define SPCSCC8(sprite)		(VLV_DISPLAY_BASE + 0x6d91c + (sprite) * 0x1000)
+#define SPCSCC01(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d90c + (sprite) * 0x1000)
+#define SPCSCC23(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d910 + (sprite) * 0x1000)
+#define SPCSCC45(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d914 + (sprite) * 0x1000)
+#define SPCSCC67(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d918 + (sprite) * 0x1000)
+#define SPCSCC8(sprite)		_MMIO(VLV_DISPLAY_BASE + 0x6d91c + (sprite) * 0x1000)
 #define  SPCSC_C1(x)		(((x) & 0x7fff) << 16) /* s3.12 */
 #define  SPCSC_C0(x)		(((x) & 0x7fff) << 0) /* s3.12 */
 
-#define SPCSCYGICLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d920 + (sprite) * 0x1000)
-#define SPCSCCBICLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d924 + (sprite) * 0x1000)
-#define SPCSCCRICLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d928 + (sprite) * 0x1000)
+#define SPCSCYGICLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d920 + (sprite) * 0x1000)
+#define SPCSCCBICLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d924 + (sprite) * 0x1000)
+#define SPCSCCRICLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d928 + (sprite) * 0x1000)
 #define  SPCSC_IMAX(x)		(((x) & 0x7ff) << 16) /* s11 */
 #define  SPCSC_IMIN(x)		(((x) & 0x7ff) << 0) /* s11 */
 
-#define SPCSCYGOCLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d92c + (sprite) * 0x1000)
-#define SPCSCCBOCLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d930 + (sprite) * 0x1000)
-#define SPCSCCROCLAMP(sprite)	(VLV_DISPLAY_BASE + 0x6d934 + (sprite) * 0x1000)
+#define SPCSCYGOCLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d92c + (sprite) * 0x1000)
+#define SPCSCCBOCLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d930 + (sprite) * 0x1000)
+#define SPCSCCROCLAMP(sprite)	_MMIO(VLV_DISPLAY_BASE + 0x6d934 + (sprite) * 0x1000)
 #define  SPCSC_OMAX(x)		((x) << 16) /* u10 */
 #define  SPCSC_OMIN(x)		((x) << 0) /* u10 */
 
@@ -5346,7 +5409,7 @@ enum skl_disp_power_wells {
 #define _PLANE_CTL_2(pipe)	_PIPE(pipe, _PLANE_CTL_2_A, _PLANE_CTL_2_B)
 #define _PLANE_CTL_3(pipe)	_PIPE(pipe, _PLANE_CTL_3_A, _PLANE_CTL_3_B)
 #define PLANE_CTL(pipe, plane)	\
-	_PLANE(plane, _PLANE_CTL_1(pipe), _PLANE_CTL_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_CTL_1(pipe), _PLANE_CTL_2(pipe))
 
 #define _PLANE_STRIDE_1_B			0x71188
 #define _PLANE_STRIDE_2_B			0x71288
@@ -5358,7 +5421,7 @@ enum skl_disp_power_wells {
 #define _PLANE_STRIDE_3(pipe)	\
 	_PIPE(pipe, _PLANE_STRIDE_3_A, _PLANE_STRIDE_3_B)
 #define PLANE_STRIDE(pipe, plane)	\
-	_PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe))
 
 #define _PLANE_POS_1_B				0x7118c
 #define _PLANE_POS_2_B				0x7128c
@@ -5367,7 +5430,7 @@ enum skl_disp_power_wells {
 #define _PLANE_POS_2(pipe)	_PIPE(pipe, _PLANE_POS_2_A, _PLANE_POS_2_B)
 #define _PLANE_POS_3(pipe)	_PIPE(pipe, _PLANE_POS_3_A, _PLANE_POS_3_B)
 #define PLANE_POS(pipe, plane)	\
-	_PLANE(plane, _PLANE_POS_1(pipe), _PLANE_POS_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_POS_1(pipe), _PLANE_POS_2(pipe))
 
 #define _PLANE_SIZE_1_B				0x71190
 #define _PLANE_SIZE_2_B				0x71290
@@ -5376,7 +5439,7 @@ enum skl_disp_power_wells {
 #define _PLANE_SIZE_2(pipe)	_PIPE(pipe, _PLANE_SIZE_2_A, _PLANE_SIZE_2_B)
 #define _PLANE_SIZE_3(pipe)	_PIPE(pipe, _PLANE_SIZE_3_A, _PLANE_SIZE_3_B)
 #define PLANE_SIZE(pipe, plane)	\
-	_PLANE(plane, _PLANE_SIZE_1(pipe), _PLANE_SIZE_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_SIZE_1(pipe), _PLANE_SIZE_2(pipe))
 
 #define _PLANE_SURF_1_B				0x7119c
 #define _PLANE_SURF_2_B				0x7129c
@@ -5385,35 +5448,35 @@ enum skl_disp_power_wells {
 #define _PLANE_SURF_2(pipe)	_PIPE(pipe, _PLANE_SURF_2_A, _PLANE_SURF_2_B)
 #define _PLANE_SURF_3(pipe)	_PIPE(pipe, _PLANE_SURF_3_A, _PLANE_SURF_3_B)
 #define PLANE_SURF(pipe, plane)	\
-	_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe))
 
 #define _PLANE_OFFSET_1_B			0x711a4
 #define _PLANE_OFFSET_2_B			0x712a4
 #define _PLANE_OFFSET_1(pipe) _PIPE(pipe, _PLANE_OFFSET_1_A, _PLANE_OFFSET_1_B)
 #define _PLANE_OFFSET_2(pipe) _PIPE(pipe, _PLANE_OFFSET_2_A, _PLANE_OFFSET_2_B)
 #define PLANE_OFFSET(pipe, plane)	\
-	_PLANE(plane, _PLANE_OFFSET_1(pipe), _PLANE_OFFSET_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_OFFSET_1(pipe), _PLANE_OFFSET_2(pipe))
 
 #define _PLANE_KEYVAL_1_B			0x71194
 #define _PLANE_KEYVAL_2_B			0x71294
 #define _PLANE_KEYVAL_1(pipe) _PIPE(pipe, _PLANE_KEYVAL_1_A, _PLANE_KEYVAL_1_B)
 #define _PLANE_KEYVAL_2(pipe) _PIPE(pipe, _PLANE_KEYVAL_2_A, _PLANE_KEYVAL_2_B)
 #define PLANE_KEYVAL(pipe, plane)	\
-	_PLANE(plane, _PLANE_KEYVAL_1(pipe), _PLANE_KEYVAL_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_KEYVAL_1(pipe), _PLANE_KEYVAL_2(pipe))
 
 #define _PLANE_KEYMSK_1_B			0x71198
 #define _PLANE_KEYMSK_2_B			0x71298
 #define _PLANE_KEYMSK_1(pipe) _PIPE(pipe, _PLANE_KEYMSK_1_A, _PLANE_KEYMSK_1_B)
 #define _PLANE_KEYMSK_2(pipe) _PIPE(pipe, _PLANE_KEYMSK_2_A, _PLANE_KEYMSK_2_B)
 #define PLANE_KEYMSK(pipe, plane)	\
-	_PLANE(plane, _PLANE_KEYMSK_1(pipe), _PLANE_KEYMSK_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_KEYMSK_1(pipe), _PLANE_KEYMSK_2(pipe))
 
 #define _PLANE_KEYMAX_1_B			0x711a0
 #define _PLANE_KEYMAX_2_B			0x712a0
 #define _PLANE_KEYMAX_1(pipe) _PIPE(pipe, _PLANE_KEYMAX_1_A, _PLANE_KEYMAX_1_B)
 #define _PLANE_KEYMAX_2(pipe) _PIPE(pipe, _PLANE_KEYMAX_2_A, _PLANE_KEYMAX_2_B)
 #define PLANE_KEYMAX(pipe, plane)	\
-	_PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe))
 
 #define _PLANE_BUF_CFG_1_B			0x7127c
 #define _PLANE_BUF_CFG_2_B			0x7137c
@@ -5422,7 +5485,7 @@ enum skl_disp_power_wells {
 #define _PLANE_BUF_CFG_2(pipe)	\
 	_PIPE(pipe, _PLANE_BUF_CFG_2_A, _PLANE_BUF_CFG_2_B)
 #define PLANE_BUF_CFG(pipe, plane)	\
-	_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe))
 
 #define _PLANE_NV12_BUF_CFG_1_B		0x71278
 #define _PLANE_NV12_BUF_CFG_2_B		0x71378
@@ -5431,26 +5494,26 @@ enum skl_disp_power_wells {
 #define _PLANE_NV12_BUF_CFG_2(pipe)	\
 	_PIPE(pipe, _PLANE_NV12_BUF_CFG_2_A, _PLANE_NV12_BUF_CFG_2_B)
 #define PLANE_NV12_BUF_CFG(pipe, plane)	\
-	_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe))
+	_MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe))
 
 /* SKL new cursor registers */
 #define _CUR_BUF_CFG_A				0x7017c
 #define _CUR_BUF_CFG_B				0x7117c
-#define CUR_BUF_CFG(pipe)	_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B)
+#define CUR_BUF_CFG(pipe)	_MMIO_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B)
 
 /* VBIOS regs */
-#define VGACNTRL		0x71400
+#define VGACNTRL		_MMIO(0x71400)
 # define VGA_DISP_DISABLE			(1 << 31)
 # define VGA_2X_MODE				(1 << 30)
 # define VGA_PIPE_B_SELECT			(1 << 29)
 
-#define VLV_VGACNTRL		(VLV_DISPLAY_BASE + 0x71400)
+#define VLV_VGACNTRL		_MMIO(VLV_DISPLAY_BASE + 0x71400)
 
 /* Ironlake */
 
-#define CPU_VGACNTRL	0x41000
+#define CPU_VGACNTRL	_MMIO(0x41000)
 
-#define DIGITAL_PORT_HOTPLUG_CNTRL	0x44030
+#define DIGITAL_PORT_HOTPLUG_CNTRL	_MMIO(0x44030)
 #define  DIGITAL_PORTA_HOTPLUG_ENABLE		(1 << 4)
 #define  DIGITAL_PORTA_PULSE_DURATION_2ms	(0 << 2) /* pre-HSW */
 #define  DIGITAL_PORTA_PULSE_DURATION_4_5ms	(1 << 2) /* pre-HSW */
@@ -5463,26 +5526,26 @@ enum skl_disp_power_wells {
 #define  DIGITAL_PORTA_HOTPLUG_LONG_DETECT	(2 << 0)
 
 /* refresh rate hardware control */
-#define RR_HW_CTL       0x45300
+#define RR_HW_CTL       _MMIO(0x45300)
 #define  RR_HW_LOW_POWER_FRAMES_MASK    0xff
 #define  RR_HW_HIGH_POWER_FRAMES_MASK   0xff00
 
-#define FDI_PLL_BIOS_0  0x46000
+#define FDI_PLL_BIOS_0  _MMIO(0x46000)
 #define  FDI_PLL_FB_CLOCK_MASK  0xff
-#define FDI_PLL_BIOS_1  0x46004
-#define FDI_PLL_BIOS_2  0x46008
-#define DISPLAY_PORT_PLL_BIOS_0         0x4600c
-#define DISPLAY_PORT_PLL_BIOS_1         0x46010
-#define DISPLAY_PORT_PLL_BIOS_2         0x46014
+#define FDI_PLL_BIOS_1  _MMIO(0x46004)
+#define FDI_PLL_BIOS_2  _MMIO(0x46008)
+#define DISPLAY_PORT_PLL_BIOS_0         _MMIO(0x4600c)
+#define DISPLAY_PORT_PLL_BIOS_1         _MMIO(0x46010)
+#define DISPLAY_PORT_PLL_BIOS_2         _MMIO(0x46014)
 
-#define PCH_3DCGDIS0		0x46020
+#define PCH_3DCGDIS0		_MMIO(0x46020)
 # define MARIUNIT_CLOCK_GATE_DISABLE		(1 << 18)
 # define SVSMUNIT_CLOCK_GATE_DISABLE		(1 << 1)
 
-#define PCH_3DCGDIS1		0x46024
+#define PCH_3DCGDIS1		_MMIO(0x46024)
 # define VFMUNIT_CLOCK_GATE_DISABLE		(1 << 11)
 
-#define FDI_PLL_FREQ_CTL        0x46030
+#define FDI_PLL_FREQ_CTL        _MMIO(0x46030)
 #define  FDI_PLL_FREQ_CHANGE_REQUEST    (1<<24)
 #define  FDI_PLL_FREQ_LOCK_LIMIT_MASK   0xfff00
 #define  FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK  0xff
@@ -5519,14 +5582,14 @@ enum skl_disp_power_wells {
 #define _PIPEB_LINK_M2		0x61048
 #define _PIPEB_LINK_N2		0x6104c
 
-#define PIPE_DATA_M1(tran) _TRANSCODER2(tran, _PIPEA_DATA_M1)
-#define PIPE_DATA_N1(tran) _TRANSCODER2(tran, _PIPEA_DATA_N1)
-#define PIPE_DATA_M2(tran) _TRANSCODER2(tran, _PIPEA_DATA_M2)
-#define PIPE_DATA_N2(tran) _TRANSCODER2(tran, _PIPEA_DATA_N2)
-#define PIPE_LINK_M1(tran) _TRANSCODER2(tran, _PIPEA_LINK_M1)
-#define PIPE_LINK_N1(tran) _TRANSCODER2(tran, _PIPEA_LINK_N1)
-#define PIPE_LINK_M2(tran) _TRANSCODER2(tran, _PIPEA_LINK_M2)
-#define PIPE_LINK_N2(tran) _TRANSCODER2(tran, _PIPEA_LINK_N2)
+#define PIPE_DATA_M1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M1)
+#define PIPE_DATA_N1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N1)
+#define PIPE_DATA_M2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M2)
+#define PIPE_DATA_N2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N2)
+#define PIPE_LINK_M1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M1)
+#define PIPE_LINK_N1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N1)
+#define PIPE_LINK_M2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M2)
+#define PIPE_LINK_N2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N2)
 
 /* CPU panel fitter */
 /* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */
@@ -5549,11 +5612,11 @@ enum skl_disp_power_wells {
 #define _PFA_HSCALE		0x68090
 #define _PFB_HSCALE		0x68890
 
-#define PF_CTL(pipe)		_PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1)
-#define PF_WIN_SZ(pipe)		_PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ)
-#define PF_WIN_POS(pipe)	_PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS)
-#define PF_VSCALE(pipe)		_PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE)
-#define PF_HSCALE(pipe)		_PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE)
+#define PF_CTL(pipe)		_MMIO_PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1)
+#define PF_WIN_SZ(pipe)		_MMIO_PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ)
+#define PF_WIN_POS(pipe)	_MMIO_PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS)
+#define PF_VSCALE(pipe)		_MMIO_PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE)
+#define PF_HSCALE(pipe)		_MMIO_PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE)
 
 #define _PSA_CTL		0x68180
 #define _PSB_CTL		0x68980
@@ -5563,9 +5626,9 @@ enum skl_disp_power_wells {
 #define _PSA_WIN_POS		0x68170
 #define _PSB_WIN_POS		0x68970
 
-#define PS_CTL(pipe)		_PIPE(pipe, _PSA_CTL, _PSB_CTL)
-#define PS_WIN_SZ(pipe)		_PIPE(pipe, _PSA_WIN_SZ, _PSB_WIN_SZ)
-#define PS_WIN_POS(pipe)	_PIPE(pipe, _PSA_WIN_POS, _PSB_WIN_POS)
+#define PS_CTL(pipe)		_MMIO_PIPE(pipe, _PSA_CTL, _PSB_CTL)
+#define PS_WIN_SZ(pipe)		_MMIO_PIPE(pipe, _PSA_WIN_SZ, _PSB_WIN_SZ)
+#define PS_WIN_POS(pipe)	_MMIO_PIPE(pipe, _PSA_WIN_POS, _PSB_WIN_POS)
 
 /*
  * Skylake scalers
@@ -5654,48 +5717,63 @@ enum skl_disp_power_wells {
 #define _PS_ECC_STAT_1C     0x691D0
 
 #define _ID(id, a, b) ((a) + (id)*((b)-(a)))
-#define SKL_PS_CTRL(pipe, id) _PIPE(pipe,        \
+#define SKL_PS_CTRL(pipe, id) _MMIO_PIPE(pipe,        \
 			_ID(id, _PS_1A_CTRL, _PS_2A_CTRL),       \
 			_ID(id, _PS_1B_CTRL, _PS_2B_CTRL))
-#define SKL_PS_PWR_GATE(pipe, id) _PIPE(pipe,    \
+#define SKL_PS_PWR_GATE(pipe, id) _MMIO_PIPE(pipe,    \
 			_ID(id, _PS_PWR_GATE_1A, _PS_PWR_GATE_2A), \
 			_ID(id, _PS_PWR_GATE_1B, _PS_PWR_GATE_2B))
-#define SKL_PS_WIN_POS(pipe, id) _PIPE(pipe,     \
+#define SKL_PS_WIN_POS(pipe, id) _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_WIN_POS_1A, _PS_WIN_POS_2A), \
 			_ID(id, _PS_WIN_POS_1B, _PS_WIN_POS_2B))
-#define SKL_PS_WIN_SZ(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_WIN_SZ(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_WIN_SZ_1A, _PS_WIN_SZ_2A),   \
 			_ID(id, _PS_WIN_SZ_1B, _PS_WIN_SZ_2B))
-#define SKL_PS_VSCALE(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_VSCALE(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_VSCALE_1A, _PS_VSCALE_2A),   \
 			_ID(id, _PS_VSCALE_1B, _PS_VSCALE_2B))
-#define SKL_PS_HSCALE(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_HSCALE(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_HSCALE_1A, _PS_HSCALE_2A),   \
 			_ID(id, _PS_HSCALE_1B, _PS_HSCALE_2B))
-#define SKL_PS_VPHASE(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_VPHASE(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_VPHASE_1A, _PS_VPHASE_2A),   \
 			_ID(id, _PS_VPHASE_1B, _PS_VPHASE_2B))
-#define SKL_PS_HPHASE(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_HPHASE(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_HPHASE_1A, _PS_HPHASE_2A),   \
 			_ID(id, _PS_HPHASE_1B, _PS_HPHASE_2B))
-#define SKL_PS_ECC_STAT(pipe, id)  _PIPE(pipe,     \
+#define SKL_PS_ECC_STAT(pipe, id)  _MMIO_PIPE(pipe,     \
 			_ID(id, _PS_ECC_STAT_1A, _PS_ECC_STAT_2A),   \
-			_ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B)
+			_ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B))
 
 /* legacy palette */
 #define _LGC_PALETTE_A           0x4a000
 #define _LGC_PALETTE_B           0x4a800
-#define LGC_PALETTE(pipe, i) (_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4)
+#define LGC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4)
 
 #define _GAMMA_MODE_A		0x4a480
 #define _GAMMA_MODE_B		0x4ac80
-#define GAMMA_MODE(pipe) _PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B)
+#define GAMMA_MODE(pipe) _MMIO_PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B)
 #define GAMMA_MODE_MODE_MASK	(3 << 0)
 #define GAMMA_MODE_MODE_8BIT	(0 << 0)
 #define GAMMA_MODE_MODE_10BIT	(1 << 0)
 #define GAMMA_MODE_MODE_12BIT	(2 << 0)
 #define GAMMA_MODE_MODE_SPLIT	(3 << 0)
 
+/* DMC/CSR */
+#define CSR_PROGRAM(i)		_MMIO(0x80000 + (i) * 4)
+#define CSR_SSP_BASE_ADDR_GEN9	0x00002FC0
+#define CSR_HTP_ADDR_SKL	0x00500034
+#define CSR_SSP_BASE		_MMIO(0x8F074)
+#define CSR_HTP_SKL		_MMIO(0x8F004)
+#define CSR_LAST_WRITE		_MMIO(0x8F034)
+#define CSR_LAST_WRITE_VALUE	0xc003b400
+/* MMIO address range for CSR program (0x80000 - 0x82FFF) */
+#define CSR_MMIO_START_RANGE	0x80000
+#define CSR_MMIO_END_RANGE	0x8FFFF
+#define SKL_CSR_DC3_DC5_COUNT	_MMIO(0x80030)
+#define SKL_CSR_DC5_DC6_COUNT	_MMIO(0x8002C)
+#define BXT_CSR_DC3_DC5_COUNT	_MMIO(0x80038)
+
 /* interrupts */
 #define DE_MASTER_IRQ_CONTROL   (1 << 31)
 #define DE_SPRITEB_FLIP_DONE    (1 << 29)
@@ -5747,20 +5825,20 @@ enum skl_disp_power_wells {
 #define DE_PIPEA_VBLANK_IVB		(1<<0)
 #define DE_PIPE_VBLANK_IVB(pipe)	(1 << ((pipe) * 5))
 
-#define VLV_MASTER_IER			0x4400c /* Gunit master IER */
+#define VLV_MASTER_IER			_MMIO(0x4400c) /* Gunit master IER */
 #define   MASTER_INTERRUPT_ENABLE	(1<<31)
 
-#define DEISR   0x44000
-#define DEIMR   0x44004
-#define DEIIR   0x44008
-#define DEIER   0x4400c
+#define DEISR   _MMIO(0x44000)
+#define DEIMR   _MMIO(0x44004)
+#define DEIIR   _MMIO(0x44008)
+#define DEIER   _MMIO(0x4400c)
 
-#define GTISR   0x44010
-#define GTIMR   0x44014
-#define GTIIR   0x44018
-#define GTIER   0x4401c
+#define GTISR   _MMIO(0x44010)
+#define GTIMR   _MMIO(0x44014)
+#define GTIIR   _MMIO(0x44018)
+#define GTIER   _MMIO(0x4401c)
 
-#define GEN8_MASTER_IRQ			0x44200
+#define GEN8_MASTER_IRQ			_MMIO(0x44200)
 #define  GEN8_MASTER_IRQ_CONTROL	(1<<31)
 #define  GEN8_PCU_IRQ			(1<<30)
 #define  GEN8_DE_PCH_IRQ		(1<<23)
@@ -5777,10 +5855,10 @@ enum skl_disp_power_wells {
 #define  GEN8_GT_BCS_IRQ		(1<<1)
 #define  GEN8_GT_RCS_IRQ		(1<<0)
 
-#define GEN8_GT_ISR(which) (0x44300 + (0x10 * (which)))
-#define GEN8_GT_IMR(which) (0x44304 + (0x10 * (which)))
-#define GEN8_GT_IIR(which) (0x44308 + (0x10 * (which)))
-#define GEN8_GT_IER(which) (0x4430c + (0x10 * (which)))
+#define GEN8_GT_ISR(which) _MMIO(0x44300 + (0x10 * (which)))
+#define GEN8_GT_IMR(which) _MMIO(0x44304 + (0x10 * (which)))
+#define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which)))
+#define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which)))
 
 #define GEN8_RCS_IRQ_SHIFT 0
 #define GEN8_BCS_IRQ_SHIFT 16
@@ -5789,10 +5867,10 @@ enum skl_disp_power_wells {
 #define GEN8_VECS_IRQ_SHIFT 0
 #define GEN8_WD_IRQ_SHIFT 16
 
-#define GEN8_DE_PIPE_ISR(pipe) (0x44400 + (0x10 * (pipe)))
-#define GEN8_DE_PIPE_IMR(pipe) (0x44404 + (0x10 * (pipe)))
-#define GEN8_DE_PIPE_IIR(pipe) (0x44408 + (0x10 * (pipe)))
-#define GEN8_DE_PIPE_IER(pipe) (0x4440c + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_ISR(pipe) _MMIO(0x44400 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IMR(pipe) _MMIO(0x44404 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IIR(pipe) _MMIO(0x44408 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IER(pipe) _MMIO(0x4440c + (0x10 * (pipe)))
 #define  GEN8_PIPE_FIFO_UNDERRUN	(1 << 31)
 #define  GEN8_PIPE_CDCLK_CRC_ERROR	(1 << 29)
 #define  GEN8_PIPE_CDCLK_CRC_DONE	(1 << 28)
@@ -5825,10 +5903,10 @@ enum skl_disp_power_wells {
 	 GEN9_PIPE_PLANE2_FAULT | \
 	 GEN9_PIPE_PLANE1_FAULT)
 
-#define GEN8_DE_PORT_ISR 0x44440
-#define GEN8_DE_PORT_IMR 0x44444
-#define GEN8_DE_PORT_IIR 0x44448
-#define GEN8_DE_PORT_IER 0x4444c
+#define GEN8_DE_PORT_ISR _MMIO(0x44440)
+#define GEN8_DE_PORT_IMR _MMIO(0x44444)
+#define GEN8_DE_PORT_IIR _MMIO(0x44448)
+#define GEN8_DE_PORT_IER _MMIO(0x4444c)
 #define  GEN9_AUX_CHANNEL_D		(1 << 27)
 #define  GEN9_AUX_CHANNEL_C		(1 << 26)
 #define  GEN9_AUX_CHANNEL_B		(1 << 25)
@@ -5842,23 +5920,23 @@ enum skl_disp_power_wells {
 #define  BXT_DE_PORT_GMBUS		(1 << 1)
 #define  GEN8_AUX_CHANNEL_A		(1 << 0)
 
-#define GEN8_DE_MISC_ISR 0x44460
-#define GEN8_DE_MISC_IMR 0x44464
-#define GEN8_DE_MISC_IIR 0x44468
-#define GEN8_DE_MISC_IER 0x4446c
+#define GEN8_DE_MISC_ISR _MMIO(0x44460)
+#define GEN8_DE_MISC_IMR _MMIO(0x44464)
+#define GEN8_DE_MISC_IIR _MMIO(0x44468)
+#define GEN8_DE_MISC_IER _MMIO(0x4446c)
 #define  GEN8_DE_MISC_GSE		(1 << 27)
 
-#define GEN8_PCU_ISR 0x444e0
-#define GEN8_PCU_IMR 0x444e4
-#define GEN8_PCU_IIR 0x444e8
-#define GEN8_PCU_IER 0x444ec
+#define GEN8_PCU_ISR _MMIO(0x444e0)
+#define GEN8_PCU_IMR _MMIO(0x444e4)
+#define GEN8_PCU_IIR _MMIO(0x444e8)
+#define GEN8_PCU_IER _MMIO(0x444ec)
 
-#define ILK_DISPLAY_CHICKEN2	0x42004
+#define ILK_DISPLAY_CHICKEN2	_MMIO(0x42004)
 /* Required on all Ironlake and Sandybridge according to the B-Spec. */
 #define  ILK_ELPIN_409_SELECT	(1 << 25)
 #define  ILK_DPARB_GATE	(1<<22)
 #define  ILK_VSDPFD_FULL	(1<<21)
-#define FUSE_STRAP			0x42014
+#define FUSE_STRAP			_MMIO(0x42014)
 #define  ILK_INTERNAL_GRAPHICS_DISABLE	(1 << 31)
 #define  ILK_INTERNAL_DISPLAY_DISABLE	(1 << 30)
 #define  ILK_DISPLAY_DEBUG_DISABLE	(1 << 29)
@@ -5867,18 +5945,18 @@ enum skl_disp_power_wells {
 #define  HSW_CDCLK_LIMIT		(1 << 24)
 #define  ILK_DESKTOP			(1 << 23)
 
-#define ILK_DSPCLK_GATE_D			0x42020
+#define ILK_DSPCLK_GATE_D			_MMIO(0x42020)
 #define   ILK_VRHUNIT_CLOCK_GATE_DISABLE	(1 << 28)
 #define   ILK_DPFCUNIT_CLOCK_GATE_DISABLE	(1 << 9)
 #define   ILK_DPFCRUNIT_CLOCK_GATE_DISABLE	(1 << 8)
 #define   ILK_DPFDUNIT_CLOCK_GATE_ENABLE	(1 << 7)
 #define   ILK_DPARBUNIT_CLOCK_GATE_ENABLE	(1 << 5)
 
-#define IVB_CHICKEN3	0x4200c
+#define IVB_CHICKEN3	_MMIO(0x4200c)
 # define CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE	(1 << 5)
 # define CHICKEN3_DGMG_DONE_FIX_DISABLE		(1 << 2)
 
-#define CHICKEN_PAR1_1		0x42080
+#define CHICKEN_PAR1_1		_MMIO(0x42080)
 #define  DPA_MASK_VBLANK_SRD	(1 << 15)
 #define  FORCE_ARB_IDLE_PLANES	(1 << 14)
 
@@ -5886,70 +5964,70 @@ enum skl_disp_power_wells {
 #define _CHICKEN_PIPESL_1_B	0x420b4
 #define  HSW_FBCQ_DIS			(1 << 22)
 #define  BDW_DPRS_MASK_VBLANK_SRD	(1 << 0)
-#define CHICKEN_PIPESL_1(pipe) _PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
+#define CHICKEN_PIPESL_1(pipe) _MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
 
-#define DISP_ARB_CTL	0x45000
+#define DISP_ARB_CTL	_MMIO(0x45000)
 #define  DISP_TILE_SURFACE_SWIZZLING	(1<<13)
 #define  DISP_FBC_WM_DIS		(1<<15)
-#define DISP_ARB_CTL2	0x45004
+#define DISP_ARB_CTL2	_MMIO(0x45004)
 #define  DISP_DATA_PARTITION_5_6	(1<<6)
-#define DBUF_CTL	0x45008
+#define DBUF_CTL	_MMIO(0x45008)
 #define  DBUF_POWER_REQUEST		(1<<31)
 #define  DBUF_POWER_STATE		(1<<30)
-#define GEN7_MSG_CTL	0x45010
+#define GEN7_MSG_CTL	_MMIO(0x45010)
 #define  WAIT_FOR_PCH_RESET_ACK		(1<<1)
 #define  WAIT_FOR_PCH_FLR_ACK		(1<<0)
-#define HSW_NDE_RSTWRN_OPT	0x46408
+#define HSW_NDE_RSTWRN_OPT	_MMIO(0x46408)
 #define  RESET_PCH_HANDSHAKE_ENABLE	(1<<4)
 
-#define SKL_DFSM			0x51000
+#define SKL_DFSM			_MMIO(0x51000)
 #define SKL_DFSM_CDCLK_LIMIT_MASK	(3 << 23)
 #define SKL_DFSM_CDCLK_LIMIT_675	(0 << 23)
 #define SKL_DFSM_CDCLK_LIMIT_540	(1 << 23)
 #define SKL_DFSM_CDCLK_LIMIT_450	(2 << 23)
 #define SKL_DFSM_CDCLK_LIMIT_337_5	(3 << 23)
 
-#define FF_SLICE_CS_CHICKEN2			0x20e4
+#define FF_SLICE_CS_CHICKEN2			_MMIO(0x20e4)
 #define  GEN9_TSG_BARRIER_ACK_DISABLE		(1<<8)
 
 /* GEN7 chicken */
-#define GEN7_COMMON_SLICE_CHICKEN1		0x7010
+#define GEN7_COMMON_SLICE_CHICKEN1		_MMIO(0x7010)
 # define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC	((1<<10) | (1<<26))
 # define GEN9_RHWO_OPTIMIZATION_DISABLE		(1<<14)
-#define COMMON_SLICE_CHICKEN2			0x7014
+#define COMMON_SLICE_CHICKEN2			_MMIO(0x7014)
 # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE	(1<<0)
 
-#define HIZ_CHICKEN					0x7018
+#define HIZ_CHICKEN					_MMIO(0x7018)
 # define CHV_HZ_8X8_MODE_IN_1X				(1<<15)
 # define BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE	(1<<3)
 
-#define GEN9_SLICE_COMMON_ECO_CHICKEN0		0x7308
+#define GEN9_SLICE_COMMON_ECO_CHICKEN0		_MMIO(0x7308)
 #define  DISABLE_PIXEL_MASK_CAMMING		(1<<14)
 
-#define GEN7_L3SQCREG1				0xB010
+#define GEN7_L3SQCREG1				_MMIO(0xB010)
 #define  VLV_B0_WA_L3SQCREG1_VALUE		0x00D30000
 
-#define GEN8_L3SQCREG1				0xB100
+#define GEN8_L3SQCREG1				_MMIO(0xB100)
 #define  BDW_WA_L3SQCREG1_DEFAULT		0x784000
 
-#define GEN7_L3CNTLREG1				0xB01C
+#define GEN7_L3CNTLREG1				_MMIO(0xB01C)
 #define  GEN7_WA_FOR_GEN7_L3_CONTROL			0x3C47FF8C
 #define  GEN7_L3AGDIS				(1<<19)
-#define GEN7_L3CNTLREG2				0xB020
-#define GEN7_L3CNTLREG3				0xB024
+#define GEN7_L3CNTLREG2				_MMIO(0xB020)
+#define GEN7_L3CNTLREG3				_MMIO(0xB024)
 
-#define GEN7_L3_CHICKEN_MODE_REGISTER		0xB030
+#define GEN7_L3_CHICKEN_MODE_REGISTER		_MMIO(0xB030)
 #define  GEN7_WA_L3_CHICKEN_MODE				0x20000000
 
-#define GEN7_L3SQCREG4				0xb034
+#define GEN7_L3SQCREG4				_MMIO(0xb034)
 #define  L3SQ_URB_READ_CAM_MATCH_DISABLE	(1<<27)
 
-#define GEN8_L3SQCREG4				0xb118
+#define GEN8_L3SQCREG4				_MMIO(0xb118)
 #define  GEN8_LQSC_RO_PERF_DIS			(1<<27)
 #define  GEN8_LQSC_FLUSH_COHERENT_LINES		(1<<21)
 
 /* GEN8 chicken */
-#define HDC_CHICKEN0				0x7300
+#define HDC_CHICKEN0				_MMIO(0x7300)
 #define  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE	(1<<15)
 #define  HDC_FENCE_DEST_SLM_DISABLE		(1<<14)
 #define  HDC_DONOT_FETCH_MEM_WHEN_MASKED	(1<<11)
@@ -5958,17 +6036,17 @@ enum skl_disp_power_wells {
 #define  HDC_BARRIER_PERFORMANCE_DISABLE	(1<<10)
 
 /* GEN9 chicken */
-#define SLICE_ECO_CHICKEN0			0x7308
+#define SLICE_ECO_CHICKEN0			_MMIO(0x7308)
 #define   PIXEL_MASK_CAMMING_DISABLE		(1 << 14)
 
 /* WaCatErrorRejectionIssue */
-#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG		0x9030
+#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG		_MMIO(0x9030)
 #define  GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB	(1<<11)
 
-#define HSW_SCRATCH1				0xb038
+#define HSW_SCRATCH1				_MMIO(0xb038)
 #define  HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE	(1<<27)
 
-#define BDW_SCRATCH1					0xb11c
+#define BDW_SCRATCH1					_MMIO(0xb11c)
 #define  GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE	(1<<2)
 
 /* PCH */
@@ -6062,12 +6140,12 @@ enum skl_disp_power_wells {
 				 SDE_FDI_RXB_CPT | \
 				 SDE_FDI_RXA_CPT)
 
-#define SDEISR  0xc4000
-#define SDEIMR  0xc4004
-#define SDEIIR  0xc4008
-#define SDEIER  0xc400c
+#define SDEISR  _MMIO(0xc4000)
+#define SDEIMR  _MMIO(0xc4004)
+#define SDEIIR  _MMIO(0xc4008)
+#define SDEIER  _MMIO(0xc400c)
 
-#define SERR_INT			0xc4040
+#define SERR_INT			_MMIO(0xc4040)
 #define  SERR_INT_POISON		(1<<31)
 #define  SERR_INT_TRANS_C_FIFO_UNDERRUN	(1<<6)
 #define  SERR_INT_TRANS_B_FIFO_UNDERRUN	(1<<3)
@@ -6075,7 +6153,7 @@ enum skl_disp_power_wells {
 #define  SERR_INT_TRANS_FIFO_UNDERRUN(pipe)	(1<<((pipe)*3))
 
 /* digital port hotplug */
-#define PCH_PORT_HOTPLUG		0xc4030	/* SHOTPLUG_CTL */
+#define PCH_PORT_HOTPLUG		_MMIO(0xc4030)	/* SHOTPLUG_CTL */
 #define  PORTA_HOTPLUG_ENABLE		(1 << 28) /* LPT:LP+ & BXT */
 #define  PORTA_HOTPLUG_STATUS_MASK	(3 << 24) /* SPT+ & BXT */
 #define  PORTA_HOTPLUG_NO_DETECT	(0 << 24) /* SPT+ & BXT */
@@ -6112,42 +6190,42 @@ enum skl_disp_power_wells {
 #define  PORTB_HOTPLUG_SHORT_DETECT	(1 << 0)
 #define  PORTB_HOTPLUG_LONG_DETECT	(2 << 0)
 
-#define PCH_PORT_HOTPLUG2		0xc403C	/* SHOTPLUG_CTL2 SPT+ */
+#define PCH_PORT_HOTPLUG2		_MMIO(0xc403C)	/* SHOTPLUG_CTL2 SPT+ */
 #define  PORTE_HOTPLUG_ENABLE		(1 << 4)
 #define  PORTE_HOTPLUG_STATUS_MASK	(3 << 0)
 #define  PORTE_HOTPLUG_NO_DETECT	(0 << 0)
 #define  PORTE_HOTPLUG_SHORT_DETECT	(1 << 0)
 #define  PORTE_HOTPLUG_LONG_DETECT	(2 << 0)
 
-#define PCH_GPIOA               0xc5010
-#define PCH_GPIOB               0xc5014
-#define PCH_GPIOC               0xc5018
-#define PCH_GPIOD               0xc501c
-#define PCH_GPIOE               0xc5020
-#define PCH_GPIOF               0xc5024
+#define PCH_GPIOA               _MMIO(0xc5010)
+#define PCH_GPIOB               _MMIO(0xc5014)
+#define PCH_GPIOC               _MMIO(0xc5018)
+#define PCH_GPIOD               _MMIO(0xc501c)
+#define PCH_GPIOE               _MMIO(0xc5020)
+#define PCH_GPIOF               _MMIO(0xc5024)
 
-#define PCH_GMBUS0		0xc5100
-#define PCH_GMBUS1		0xc5104
-#define PCH_GMBUS2		0xc5108
-#define PCH_GMBUS3		0xc510c
-#define PCH_GMBUS4		0xc5110
-#define PCH_GMBUS5		0xc5120
+#define PCH_GMBUS0		_MMIO(0xc5100)
+#define PCH_GMBUS1		_MMIO(0xc5104)
+#define PCH_GMBUS2		_MMIO(0xc5108)
+#define PCH_GMBUS3		_MMIO(0xc510c)
+#define PCH_GMBUS4		_MMIO(0xc5110)
+#define PCH_GMBUS5		_MMIO(0xc5120)
 
 #define _PCH_DPLL_A              0xc6014
 #define _PCH_DPLL_B              0xc6018
-#define PCH_DPLL(pll) (pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B)
+#define PCH_DPLL(pll) _MMIO(pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B)
 
 #define _PCH_FPA0                0xc6040
 #define  FP_CB_TUNE		(0x3<<22)
 #define _PCH_FPA1                0xc6044
 #define _PCH_FPB0                0xc6048
 #define _PCH_FPB1                0xc604c
-#define PCH_FP0(pll) (pll == 0 ? _PCH_FPA0 : _PCH_FPB0)
-#define PCH_FP1(pll) (pll == 0 ? _PCH_FPA1 : _PCH_FPB1)
+#define PCH_FP0(pll) _MMIO(pll == 0 ? _PCH_FPA0 : _PCH_FPB0)
+#define PCH_FP1(pll) _MMIO(pll == 0 ? _PCH_FPA1 : _PCH_FPB1)
 
-#define PCH_DPLL_TEST           0xc606c
+#define PCH_DPLL_TEST           _MMIO(0xc606c)
 
-#define PCH_DREF_CONTROL        0xC6200
+#define PCH_DREF_CONTROL        _MMIO(0xC6200)
 #define  DREF_CONTROL_MASK      0x7fc3
 #define  DREF_CPU_SOURCE_OUTPUT_DISABLE         (0<<13)
 #define  DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD      (2<<13)
@@ -6170,19 +6248,19 @@ enum skl_disp_power_wells {
 #define  DREF_SSC4_DISABLE                      (0)
 #define  DREF_SSC4_ENABLE                       (1)
 
-#define PCH_RAWCLK_FREQ         0xc6204
+#define PCH_RAWCLK_FREQ         _MMIO(0xc6204)
 #define  FDL_TP1_TIMER_SHIFT    12
 #define  FDL_TP1_TIMER_MASK     (3<<12)
 #define  FDL_TP2_TIMER_SHIFT    10
 #define  FDL_TP2_TIMER_MASK     (3<<10)
 #define  RAWCLK_FREQ_MASK       0x3ff
 
-#define PCH_DPLL_TMR_CFG        0xc6208
+#define PCH_DPLL_TMR_CFG        _MMIO(0xc6208)
 
-#define PCH_SSC4_PARMS          0xc6210
-#define PCH_SSC4_AUX_PARMS      0xc6214
+#define PCH_SSC4_PARMS          _MMIO(0xc6210)
+#define PCH_SSC4_AUX_PARMS      _MMIO(0xc6214)
 
-#define PCH_DPLL_SEL		0xc7000
+#define PCH_DPLL_SEL		_MMIO(0xc7000)
 #define	 TRANS_DPLLB_SEL(pipe)		(1 << ((pipe) * 4))
 #define	 TRANS_DPLLA_SEL(pipe)		0
 #define  TRANS_DPLL_ENABLE(pipe)	(1 << ((pipe) * 4 + 3))
@@ -6230,79 +6308,73 @@ enum skl_disp_power_wells {
 #define _VIDEO_DIP_DATA_B        0xe1208
 #define _VIDEO_DIP_GCP_B         0xe1210
 
-#define TVIDEO_DIP_CTL(pipe) _PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B)
-#define TVIDEO_DIP_DATA(pipe) _PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B)
-#define TVIDEO_DIP_GCP(pipe) _PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B)
+#define TVIDEO_DIP_CTL(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B)
+#define TVIDEO_DIP_DATA(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B)
+#define TVIDEO_DIP_GCP(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B)
 
 /* Per-transcoder DIP controls (VLV) */
-#define VLV_VIDEO_DIP_CTL_A		(VLV_DISPLAY_BASE + 0x60200)
-#define VLV_VIDEO_DIP_DATA_A		(VLV_DISPLAY_BASE + 0x60208)
-#define VLV_VIDEO_DIP_GDCP_PAYLOAD_A	(VLV_DISPLAY_BASE + 0x60210)
+#define _VLV_VIDEO_DIP_CTL_A		(VLV_DISPLAY_BASE + 0x60200)
+#define _VLV_VIDEO_DIP_DATA_A		(VLV_DISPLAY_BASE + 0x60208)
+#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_A	(VLV_DISPLAY_BASE + 0x60210)
 
-#define VLV_VIDEO_DIP_CTL_B		(VLV_DISPLAY_BASE + 0x61170)
-#define VLV_VIDEO_DIP_DATA_B		(VLV_DISPLAY_BASE + 0x61174)
-#define VLV_VIDEO_DIP_GDCP_PAYLOAD_B	(VLV_DISPLAY_BASE + 0x61178)
+#define _VLV_VIDEO_DIP_CTL_B		(VLV_DISPLAY_BASE + 0x61170)
+#define _VLV_VIDEO_DIP_DATA_B		(VLV_DISPLAY_BASE + 0x61174)
+#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_B	(VLV_DISPLAY_BASE + 0x61178)
 
-#define CHV_VIDEO_DIP_CTL_C		(VLV_DISPLAY_BASE + 0x611f0)
-#define CHV_VIDEO_DIP_DATA_C		(VLV_DISPLAY_BASE + 0x611f4)
-#define CHV_VIDEO_DIP_GDCP_PAYLOAD_C	(VLV_DISPLAY_BASE + 0x611f8)
+#define _CHV_VIDEO_DIP_CTL_C		(VLV_DISPLAY_BASE + 0x611f0)
+#define _CHV_VIDEO_DIP_DATA_C		(VLV_DISPLAY_BASE + 0x611f4)
+#define _CHV_VIDEO_DIP_GDCP_PAYLOAD_C	(VLV_DISPLAY_BASE + 0x611f8)
 
 #define VLV_TVIDEO_DIP_CTL(pipe) \
-	_PIPE3((pipe), VLV_VIDEO_DIP_CTL_A, \
-	       VLV_VIDEO_DIP_CTL_B, CHV_VIDEO_DIP_CTL_C)
+	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_CTL_A, \
+	       _VLV_VIDEO_DIP_CTL_B, _CHV_VIDEO_DIP_CTL_C)
 #define VLV_TVIDEO_DIP_DATA(pipe) \
-	_PIPE3((pipe), VLV_VIDEO_DIP_DATA_A, \
-	       VLV_VIDEO_DIP_DATA_B, CHV_VIDEO_DIP_DATA_C)
+	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_DATA_A, \
+	       _VLV_VIDEO_DIP_DATA_B, _CHV_VIDEO_DIP_DATA_C)
 #define VLV_TVIDEO_DIP_GCP(pipe) \
-	_PIPE3((pipe), VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \
-		VLV_VIDEO_DIP_GDCP_PAYLOAD_B, CHV_VIDEO_DIP_GDCP_PAYLOAD_C)
+	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \
+		_VLV_VIDEO_DIP_GDCP_PAYLOAD_B, _CHV_VIDEO_DIP_GDCP_PAYLOAD_C)
 
 /* Haswell DIP controls */
-#define HSW_VIDEO_DIP_CTL_A		0x60200
-#define HSW_VIDEO_DIP_AVI_DATA_A	0x60220
-#define HSW_VIDEO_DIP_VS_DATA_A		0x60260
-#define HSW_VIDEO_DIP_SPD_DATA_A	0x602A0
-#define HSW_VIDEO_DIP_GMP_DATA_A	0x602E0
-#define HSW_VIDEO_DIP_VSC_DATA_A	0x60320
-#define HSW_VIDEO_DIP_AVI_ECC_A		0x60240
-#define HSW_VIDEO_DIP_VS_ECC_A		0x60280
-#define HSW_VIDEO_DIP_SPD_ECC_A		0x602C0
-#define HSW_VIDEO_DIP_GMP_ECC_A		0x60300
-#define HSW_VIDEO_DIP_VSC_ECC_A		0x60344
-#define HSW_VIDEO_DIP_GCP_A		0x60210
-
-#define HSW_VIDEO_DIP_CTL_B		0x61200
-#define HSW_VIDEO_DIP_AVI_DATA_B	0x61220
-#define HSW_VIDEO_DIP_VS_DATA_B		0x61260
-#define HSW_VIDEO_DIP_SPD_DATA_B	0x612A0
-#define HSW_VIDEO_DIP_GMP_DATA_B	0x612E0
-#define HSW_VIDEO_DIP_VSC_DATA_B	0x61320
-#define HSW_VIDEO_DIP_BVI_ECC_B		0x61240
-#define HSW_VIDEO_DIP_VS_ECC_B		0x61280
-#define HSW_VIDEO_DIP_SPD_ECC_B		0x612C0
-#define HSW_VIDEO_DIP_GMP_ECC_B		0x61300
-#define HSW_VIDEO_DIP_VSC_ECC_B		0x61344
-#define HSW_VIDEO_DIP_GCP_B		0x61210
-
-#define HSW_TVIDEO_DIP_CTL(trans) \
-	 _TRANSCODER2(trans, HSW_VIDEO_DIP_CTL_A)
-#define HSW_TVIDEO_DIP_AVI_DATA(trans, i) \
-	(_TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A) + (i) * 4)
-#define HSW_TVIDEO_DIP_VS_DATA(trans, i) \
-	(_TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A) + (i) * 4)
-#define HSW_TVIDEO_DIP_SPD_DATA(trans, i) \
-	(_TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A) + (i) * 4)
-#define HSW_TVIDEO_DIP_GCP(trans) \
-	_TRANSCODER2(trans, HSW_VIDEO_DIP_GCP_A)
-#define HSW_TVIDEO_DIP_VSC_DATA(trans, i) \
-	(_TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A) + (i) * 4)
-
-#define HSW_STEREO_3D_CTL_A	0x70020
-#define   S3D_ENABLE		(1<<31)
-#define HSW_STEREO_3D_CTL_B	0x71020
-
-#define HSW_STEREO_3D_CTL(trans) \
-	_PIPE2(trans, HSW_STEREO_3D_CTL_A)
+
+#define _HSW_VIDEO_DIP_CTL_A		0x60200
+#define _HSW_VIDEO_DIP_AVI_DATA_A	0x60220
+#define _HSW_VIDEO_DIP_VS_DATA_A	0x60260
+#define _HSW_VIDEO_DIP_SPD_DATA_A	0x602A0
+#define _HSW_VIDEO_DIP_GMP_DATA_A	0x602E0
+#define _HSW_VIDEO_DIP_VSC_DATA_A	0x60320
+#define _HSW_VIDEO_DIP_AVI_ECC_A	0x60240
+#define _HSW_VIDEO_DIP_VS_ECC_A		0x60280
+#define _HSW_VIDEO_DIP_SPD_ECC_A	0x602C0
+#define _HSW_VIDEO_DIP_GMP_ECC_A	0x60300
+#define _HSW_VIDEO_DIP_VSC_ECC_A	0x60344
+#define _HSW_VIDEO_DIP_GCP_A		0x60210
+
+#define _HSW_VIDEO_DIP_CTL_B		0x61200
+#define _HSW_VIDEO_DIP_AVI_DATA_B	0x61220
+#define _HSW_VIDEO_DIP_VS_DATA_B	0x61260
+#define _HSW_VIDEO_DIP_SPD_DATA_B	0x612A0
+#define _HSW_VIDEO_DIP_GMP_DATA_B	0x612E0
+#define _HSW_VIDEO_DIP_VSC_DATA_B	0x61320
+#define _HSW_VIDEO_DIP_BVI_ECC_B	0x61240
+#define _HSW_VIDEO_DIP_VS_ECC_B		0x61280
+#define _HSW_VIDEO_DIP_SPD_ECC_B	0x612C0
+#define _HSW_VIDEO_DIP_GMP_ECC_B	0x61300
+#define _HSW_VIDEO_DIP_VSC_ECC_B	0x61344
+#define _HSW_VIDEO_DIP_GCP_B		0x61210
+
+#define HSW_TVIDEO_DIP_CTL(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A)
+#define HSW_TVIDEO_DIP_AVI_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_VS_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_SPD_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A)
+#define HSW_TVIDEO_DIP_VSC_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4)
+
+#define _HSW_STEREO_3D_CTL_A		0x70020
+#define   S3D_ENABLE			(1<<31)
+#define _HSW_STEREO_3D_CTL_B		0x71020
+
+#define HSW_STEREO_3D_CTL(trans)	_MMIO_PIPE2(trans, _HSW_STEREO_3D_CTL_A)
 
 #define _PCH_TRANS_HTOTAL_B          0xe1000
 #define _PCH_TRANS_HBLANK_B          0xe1004
@@ -6310,16 +6382,15 @@ enum skl_disp_power_wells {
 #define _PCH_TRANS_VTOTAL_B          0xe100c
 #define _PCH_TRANS_VBLANK_B          0xe1010
 #define _PCH_TRANS_VSYNC_B           0xe1014
-#define _PCH_TRANS_VSYNCSHIFT_B	 0xe1028
+#define _PCH_TRANS_VSYNCSHIFT_B 0xe1028
 
-#define PCH_TRANS_HTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B)
-#define PCH_TRANS_HBLANK(pipe) _PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B)
-#define PCH_TRANS_HSYNC(pipe) _PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B)
-#define PCH_TRANS_VTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B)
-#define PCH_TRANS_VBLANK(pipe) _PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B)
-#define PCH_TRANS_VSYNC(pipe) _PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B)
-#define PCH_TRANS_VSYNCSHIFT(pipe) _PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, \
-					 _PCH_TRANS_VSYNCSHIFT_B)
+#define PCH_TRANS_HTOTAL(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B)
+#define PCH_TRANS_HBLANK(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B)
+#define PCH_TRANS_HSYNC(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B)
+#define PCH_TRANS_VTOTAL(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B)
+#define PCH_TRANS_VBLANK(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B)
+#define PCH_TRANS_VSYNC(pipe)		_MMIO_PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B)
+#define PCH_TRANS_VSYNCSHIFT(pipe)	_MMIO_PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, _PCH_TRANS_VSYNCSHIFT_B)
 
 #define _PCH_TRANSB_DATA_M1	0xe1030
 #define _PCH_TRANSB_DATA_N1	0xe1034
@@ -6330,19 +6401,19 @@ enum skl_disp_power_wells {
 #define _PCH_TRANSB_LINK_M2	0xe1048
 #define _PCH_TRANSB_LINK_N2	0xe104c
 
-#define PCH_TRANS_DATA_M1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1)
-#define PCH_TRANS_DATA_N1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1)
-#define PCH_TRANS_DATA_M2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2)
-#define PCH_TRANS_DATA_N2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2)
-#define PCH_TRANS_LINK_M1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1)
-#define PCH_TRANS_LINK_N1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1)
-#define PCH_TRANS_LINK_M2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2)
-#define PCH_TRANS_LINK_N2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2)
+#define PCH_TRANS_DATA_M1(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1)
+#define PCH_TRANS_DATA_N1(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1)
+#define PCH_TRANS_DATA_M2(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2)
+#define PCH_TRANS_DATA_N2(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2)
+#define PCH_TRANS_LINK_M1(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1)
+#define PCH_TRANS_LINK_N1(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1)
+#define PCH_TRANS_LINK_M2(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2)
+#define PCH_TRANS_LINK_N2(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2)
 
 #define _PCH_TRANSACONF              0xf0008
 #define _PCH_TRANSBCONF              0xf1008
-#define PCH_TRANSCONF(pipe) _PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF)
-#define LPT_TRANSCONF		_PCH_TRANSACONF /* lpt has only one transcoder */
+#define PCH_TRANSCONF(pipe)	_MMIO_PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF)
+#define LPT_TRANSCONF		PCH_TRANSCONF(PIPE_A) /* lpt has only one transcoder */
 #define  TRANS_DISABLE          (0<<31)
 #define  TRANS_ENABLE           (1<<31)
 #define  TRANS_STATE_MASK       (1<<30)
@@ -6363,47 +6434,47 @@ enum skl_disp_power_wells {
 
 #define _TRANSA_CHICKEN1	 0xf0060
 #define _TRANSB_CHICKEN1	 0xf1060
-#define TRANS_CHICKEN1(pipe) _PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1)
+#define TRANS_CHICKEN1(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1)
 #define  TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE	(1<<10)
 #define  TRANS_CHICKEN1_DP0UNIT_GC_DISABLE	(1<<4)
 #define _TRANSA_CHICKEN2	 0xf0064
 #define _TRANSB_CHICKEN2	 0xf1064
-#define TRANS_CHICKEN2(pipe) _PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2)
+#define TRANS_CHICKEN2(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2)
 #define  TRANS_CHICKEN2_TIMING_OVERRIDE			(1<<31)
 #define  TRANS_CHICKEN2_FDI_POLARITY_REVERSED		(1<<29)
 #define  TRANS_CHICKEN2_FRAME_START_DELAY_MASK		(3<<27)
 #define  TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER	(1<<26)
 #define  TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH	(1<<25)
 
-#define SOUTH_CHICKEN1		0xc2000
+#define SOUTH_CHICKEN1		_MMIO(0xc2000)
 #define  FDIA_PHASE_SYNC_SHIFT_OVR	19
 #define  FDIA_PHASE_SYNC_SHIFT_EN	18
 #define  FDI_PHASE_SYNC_OVR(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
 #define  FDI_PHASE_SYNC_EN(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
 #define  FDI_BC_BIFURCATION_SELECT	(1 << 12)
 #define  SPT_PWM_GRANULARITY		(1<<0)
-#define SOUTH_CHICKEN2		0xc2004
+#define SOUTH_CHICKEN2		_MMIO(0xc2004)
 #define  FDI_MPHY_IOSFSB_RESET_STATUS	(1<<13)
 #define  FDI_MPHY_IOSFSB_RESET_CTL	(1<<12)
 #define  LPT_PWM_GRANULARITY		(1<<5)
 #define  DPLS_EDP_PPS_FIX_DIS		(1<<0)
 
-#define _FDI_RXA_CHICKEN         0xc200c
-#define _FDI_RXB_CHICKEN         0xc2010
+#define _FDI_RXA_CHICKEN        0xc200c
+#define _FDI_RXB_CHICKEN        0xc2010
 #define  FDI_RX_PHASE_SYNC_POINTER_OVR	(1<<1)
 #define  FDI_RX_PHASE_SYNC_POINTER_EN	(1<<0)
-#define FDI_RX_CHICKEN(pipe) _PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
+#define FDI_RX_CHICKEN(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
 
-#define SOUTH_DSPCLK_GATE_D	0xc2020
+#define SOUTH_DSPCLK_GATE_D	_MMIO(0xc2020)
 #define  PCH_DPLUNIT_CLOCK_GATE_DISABLE (1<<30)
 #define  PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1<<29)
 #define  PCH_CPUNIT_CLOCK_GATE_DISABLE (1<<14)
 #define  PCH_LP_PARTITION_LEVEL_DISABLE  (1<<12)
 
 /* CPU: FDI_TX */
-#define _FDI_TXA_CTL             0x60100
-#define _FDI_TXB_CTL             0x61100
-#define FDI_TX_CTL(pipe) _PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL)
+#define _FDI_TXA_CTL            0x60100
+#define _FDI_TXB_CTL            0x61100
+#define FDI_TX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL)
 #define  FDI_TX_DISABLE         (0<<31)
 #define  FDI_TX_ENABLE          (1<<31)
 #define  FDI_LINK_TRAIN_PATTERN_1       (0<<28)
@@ -6453,7 +6524,7 @@ enum skl_disp_power_wells {
 /* FDI_RX, FDI_X is hard-wired to Transcoder_X */
 #define _FDI_RXA_CTL             0xf000c
 #define _FDI_RXB_CTL             0xf100c
-#define FDI_RX_CTL(pipe) _PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL)
+#define FDI_RX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL)
 #define  FDI_RX_ENABLE          (1<<31)
 /* train, dp width same as FDI_TX */
 #define  FDI_FS_ERRC_ENABLE		(1<<27)
@@ -6489,14 +6560,14 @@ enum skl_disp_power_wells {
 #define  FDI_RX_TP1_TO_TP2_48		(2<<20)
 #define  FDI_RX_TP1_TO_TP2_64		(3<<20)
 #define  FDI_RX_FDI_DELAY_90		(0x90<<0)
-#define FDI_RX_MISC(pipe) _PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC)
+#define FDI_RX_MISC(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC)
 
-#define _FDI_RXA_TUSIZE1         0xf0030
-#define _FDI_RXA_TUSIZE2         0xf0038
-#define _FDI_RXB_TUSIZE1         0xf1030
-#define _FDI_RXB_TUSIZE2         0xf1038
-#define FDI_RX_TUSIZE1(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1)
-#define FDI_RX_TUSIZE2(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2)
+#define _FDI_RXA_TUSIZE1        0xf0030
+#define _FDI_RXA_TUSIZE2        0xf0038
+#define _FDI_RXB_TUSIZE1        0xf1030
+#define _FDI_RXB_TUSIZE2        0xf1038
+#define FDI_RX_TUSIZE1(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1)
+#define FDI_RX_TUSIZE2(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2)
 
 /* FDI_RX interrupt register format */
 #define FDI_RX_INTER_LANE_ALIGN         (1<<10)
@@ -6511,44 +6582,41 @@ enum skl_disp_power_wells {
 #define FDI_RX_CROSS_CLOCK_OVERFLOW     (1<<1)
 #define FDI_RX_SYMBOL_QUEUE_OVERFLOW    (1<<0)
 
-#define _FDI_RXA_IIR             0xf0014
-#define _FDI_RXA_IMR             0xf0018
-#define _FDI_RXB_IIR             0xf1014
-#define _FDI_RXB_IMR             0xf1018
-#define FDI_RX_IIR(pipe) _PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR)
-#define FDI_RX_IMR(pipe) _PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR)
+#define _FDI_RXA_IIR            0xf0014
+#define _FDI_RXA_IMR            0xf0018
+#define _FDI_RXB_IIR            0xf1014
+#define _FDI_RXB_IMR            0xf1018
+#define FDI_RX_IIR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR)
+#define FDI_RX_IMR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR)
 
-#define FDI_PLL_CTL_1           0xfe000
-#define FDI_PLL_CTL_2           0xfe004
+#define FDI_PLL_CTL_1           _MMIO(0xfe000)
+#define FDI_PLL_CTL_2           _MMIO(0xfe004)
 
-#define PCH_LVDS	0xe1180
+#define PCH_LVDS	_MMIO(0xe1180)
 #define  LVDS_DETECTED	(1 << 1)
 
 /* vlv has 2 sets of panel control regs. */
-#define PIPEA_PP_STATUS         (VLV_DISPLAY_BASE + 0x61200)
-#define PIPEA_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61204)
-#define PIPEA_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61208)
+#define _PIPEA_PP_STATUS         (VLV_DISPLAY_BASE + 0x61200)
+#define _PIPEA_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61204)
+#define _PIPEA_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61208)
 #define  PANEL_PORT_SELECT_VLV(port)	((port) << 30)
-#define PIPEA_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6120c)
-#define PIPEA_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61210)
-
-#define PIPEB_PP_STATUS         (VLV_DISPLAY_BASE + 0x61300)
-#define PIPEB_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61304)
-#define PIPEB_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61308)
-#define PIPEB_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6130c)
-#define PIPEB_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61310)
-
-#define VLV_PIPE_PP_STATUS(pipe) _PIPE(pipe, PIPEA_PP_STATUS, PIPEB_PP_STATUS)
-#define VLV_PIPE_PP_CONTROL(pipe) _PIPE(pipe, PIPEA_PP_CONTROL, PIPEB_PP_CONTROL)
-#define VLV_PIPE_PP_ON_DELAYS(pipe) \
-		_PIPE(pipe, PIPEA_PP_ON_DELAYS, PIPEB_PP_ON_DELAYS)
-#define VLV_PIPE_PP_OFF_DELAYS(pipe) \
-		_PIPE(pipe, PIPEA_PP_OFF_DELAYS, PIPEB_PP_OFF_DELAYS)
-#define VLV_PIPE_PP_DIVISOR(pipe) \
-		_PIPE(pipe, PIPEA_PP_DIVISOR, PIPEB_PP_DIVISOR)
-
-#define PCH_PP_STATUS		0xc7200
-#define PCH_PP_CONTROL		0xc7204
+#define _PIPEA_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6120c)
+#define _PIPEA_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61210)
+
+#define _PIPEB_PP_STATUS         (VLV_DISPLAY_BASE + 0x61300)
+#define _PIPEB_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61304)
+#define _PIPEB_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61308)
+#define _PIPEB_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6130c)
+#define _PIPEB_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61310)
+
+#define VLV_PIPE_PP_STATUS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_STATUS, _PIPEB_PP_STATUS)
+#define VLV_PIPE_PP_CONTROL(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_CONTROL, _PIPEB_PP_CONTROL)
+#define VLV_PIPE_PP_ON_DELAYS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_ON_DELAYS, _PIPEB_PP_ON_DELAYS)
+#define VLV_PIPE_PP_OFF_DELAYS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_OFF_DELAYS, _PIPEB_PP_OFF_DELAYS)
+#define VLV_PIPE_PP_DIVISOR(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_DIVISOR, _PIPEB_PP_DIVISOR)
+
+#define _PCH_PP_STATUS		0xc7200
+#define _PCH_PP_CONTROL		0xc7204
 #define  PANEL_UNLOCK_REGS	(0xabcd << 16)
 #define  PANEL_UNLOCK_MASK	(0xffff << 16)
 #define  BXT_POWER_CYCLE_DELAY_MASK	(0x1f0)
@@ -6558,7 +6626,7 @@ enum skl_disp_power_wells {
 #define  PANEL_POWER_RESET	(1 << 1)
 #define  PANEL_POWER_OFF	(0 << 0)
 #define  PANEL_POWER_ON		(1 << 0)
-#define PCH_PP_ON_DELAYS	0xc7208
+#define _PCH_PP_ON_DELAYS	0xc7208
 #define  PANEL_PORT_SELECT_MASK	(3 << 30)
 #define  PANEL_PORT_SELECT_LVDS	(0 << 30)
 #define  PANEL_PORT_SELECT_DPA	(1 << 30)
@@ -6569,52 +6637,64 @@ enum skl_disp_power_wells {
 #define  PANEL_LIGHT_ON_DELAY_MASK	(0x1fff)
 #define  PANEL_LIGHT_ON_DELAY_SHIFT	0
 
-#define PCH_PP_OFF_DELAYS	0xc720c
+#define _PCH_PP_OFF_DELAYS		0xc720c
 #define  PANEL_POWER_DOWN_DELAY_MASK	(0x1fff0000)
 #define  PANEL_POWER_DOWN_DELAY_SHIFT	16
 #define  PANEL_LIGHT_OFF_DELAY_MASK	(0x1fff)
 #define  PANEL_LIGHT_OFF_DELAY_SHIFT	0
 
-#define PCH_PP_DIVISOR		0xc7210
+#define _PCH_PP_DIVISOR			0xc7210
 #define  PP_REFERENCE_DIVIDER_MASK	(0xffffff00)
 #define  PP_REFERENCE_DIVIDER_SHIFT	8
 #define  PANEL_POWER_CYCLE_DELAY_MASK	(0x1f)
 #define  PANEL_POWER_CYCLE_DELAY_SHIFT	0
 
+#define PCH_PP_STATUS			_MMIO(_PCH_PP_STATUS)
+#define PCH_PP_CONTROL			_MMIO(_PCH_PP_CONTROL)
+#define PCH_PP_ON_DELAYS		_MMIO(_PCH_PP_ON_DELAYS)
+#define PCH_PP_OFF_DELAYS		_MMIO(_PCH_PP_OFF_DELAYS)
+#define PCH_PP_DIVISOR			_MMIO(_PCH_PP_DIVISOR)
+
 /* BXT PPS changes - 2nd set of PPS registers */
 #define _BXT_PP_STATUS2 	0xc7300
 #define _BXT_PP_CONTROL2 	0xc7304
 #define _BXT_PP_ON_DELAYS2	0xc7308
 #define _BXT_PP_OFF_DELAYS2	0xc730c
 
-#define BXT_PP_STATUS(n)	_PIPE(n, PCH_PP_STATUS, _BXT_PP_STATUS2)
-#define BXT_PP_CONTROL(n)	_PIPE(n, PCH_PP_CONTROL, _BXT_PP_CONTROL2)
-#define BXT_PP_ON_DELAYS(n)	_PIPE(n, PCH_PP_ON_DELAYS, _BXT_PP_ON_DELAYS2)
-#define BXT_PP_OFF_DELAYS(n)	_PIPE(n, PCH_PP_OFF_DELAYS, _BXT_PP_OFF_DELAYS2)
-
-#define PCH_DP_B		0xe4100
-#define PCH_DPB_AUX_CH_CTL	0xe4110
-#define PCH_DPB_AUX_CH_DATA1	0xe4114
-#define PCH_DPB_AUX_CH_DATA2	0xe4118
-#define PCH_DPB_AUX_CH_DATA3	0xe411c
-#define PCH_DPB_AUX_CH_DATA4	0xe4120
-#define PCH_DPB_AUX_CH_DATA5	0xe4124
-
-#define PCH_DP_C		0xe4200
-#define PCH_DPC_AUX_CH_CTL	0xe4210
-#define PCH_DPC_AUX_CH_DATA1	0xe4214
-#define PCH_DPC_AUX_CH_DATA2	0xe4218
-#define PCH_DPC_AUX_CH_DATA3	0xe421c
-#define PCH_DPC_AUX_CH_DATA4	0xe4220
-#define PCH_DPC_AUX_CH_DATA5	0xe4224
-
-#define PCH_DP_D		0xe4300
-#define PCH_DPD_AUX_CH_CTL	0xe4310
-#define PCH_DPD_AUX_CH_DATA1	0xe4314
-#define PCH_DPD_AUX_CH_DATA2	0xe4318
-#define PCH_DPD_AUX_CH_DATA3	0xe431c
-#define PCH_DPD_AUX_CH_DATA4	0xe4320
-#define PCH_DPD_AUX_CH_DATA5	0xe4324
+#define BXT_PP_STATUS(n)	_MMIO_PIPE(n, _PCH_PP_STATUS, _BXT_PP_STATUS2)
+#define BXT_PP_CONTROL(n)	_MMIO_PIPE(n, _PCH_PP_CONTROL, _BXT_PP_CONTROL2)
+#define BXT_PP_ON_DELAYS(n)	_MMIO_PIPE(n, _PCH_PP_ON_DELAYS, _BXT_PP_ON_DELAYS2)
+#define BXT_PP_OFF_DELAYS(n)	_MMIO_PIPE(n, _PCH_PP_OFF_DELAYS, _BXT_PP_OFF_DELAYS2)
+
+#define _PCH_DP_B		0xe4100
+#define PCH_DP_B		_MMIO(_PCH_DP_B)
+#define _PCH_DPB_AUX_CH_CTL	0xe4110
+#define _PCH_DPB_AUX_CH_DATA1	0xe4114
+#define _PCH_DPB_AUX_CH_DATA2	0xe4118
+#define _PCH_DPB_AUX_CH_DATA3	0xe411c
+#define _PCH_DPB_AUX_CH_DATA4	0xe4120
+#define _PCH_DPB_AUX_CH_DATA5	0xe4124
+
+#define _PCH_DP_C		0xe4200
+#define PCH_DP_C		_MMIO(_PCH_DP_C)
+#define _PCH_DPC_AUX_CH_CTL	0xe4210
+#define _PCH_DPC_AUX_CH_DATA1	0xe4214
+#define _PCH_DPC_AUX_CH_DATA2	0xe4218
+#define _PCH_DPC_AUX_CH_DATA3	0xe421c
+#define _PCH_DPC_AUX_CH_DATA4	0xe4220
+#define _PCH_DPC_AUX_CH_DATA5	0xe4224
+
+#define _PCH_DP_D		0xe4300
+#define PCH_DP_D		_MMIO(_PCH_DP_D)
+#define _PCH_DPD_AUX_CH_CTL	0xe4310
+#define _PCH_DPD_AUX_CH_DATA1	0xe4314
+#define _PCH_DPD_AUX_CH_DATA2	0xe4318
+#define _PCH_DPD_AUX_CH_DATA3	0xe431c
+#define _PCH_DPD_AUX_CH_DATA4	0xe4320
+#define _PCH_DPD_AUX_CH_DATA5	0xe4324
+
+#define PCH_DP_AUX_CH_CTL(port)		_MMIO_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_CTL, _PCH_DPC_AUX_CH_CTL)
+#define PCH_DP_AUX_CH_DATA(port, i)	_MMIO(_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_DATA1, _PCH_DPC_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
 
 /* CPT */
 #define  PORT_TRANS_A_SEL_CPT	0
@@ -6627,10 +6707,10 @@ enum skl_disp_power_wells {
 #define  SDVO_PORT_TO_PIPE_CHV(val)	(((val) & (3<<24)) >> 24)
 #define  DP_PORT_TO_PIPE_CHV(val)	(((val) & (3<<16)) >> 16)
 
-#define TRANS_DP_CTL_A		0xe0300
-#define TRANS_DP_CTL_B		0xe1300
-#define TRANS_DP_CTL_C		0xe2300
-#define TRANS_DP_CTL(pipe)	_PIPE(pipe, TRANS_DP_CTL_A, TRANS_DP_CTL_B)
+#define _TRANS_DP_CTL_A		0xe0300
+#define _TRANS_DP_CTL_B		0xe1300
+#define _TRANS_DP_CTL_C		0xe2300
+#define TRANS_DP_CTL(pipe)	_MMIO_PIPE(pipe, _TRANS_DP_CTL_A, _TRANS_DP_CTL_B)
 #define  TRANS_DP_OUTPUT_ENABLE	(1<<31)
 #define  TRANS_DP_PORT_SEL_B	(0<<29)
 #define  TRANS_DP_PORT_SEL_C	(1<<29)
@@ -6683,40 +6763,40 @@ enum skl_disp_power_wells {
 
 #define  EDP_LINK_TRAIN_VOL_EMP_MASK_IVB	(0x3f<<22)
 
-#define  VLV_PMWGICZ				0x1300a4
+#define  VLV_PMWGICZ				_MMIO(0x1300a4)
 
-#define  FORCEWAKE				0xA18C
-#define  FORCEWAKE_VLV				0x1300b0
-#define  FORCEWAKE_ACK_VLV			0x1300b4
-#define  FORCEWAKE_MEDIA_VLV			0x1300b8
-#define  FORCEWAKE_ACK_MEDIA_VLV		0x1300bc
-#define  FORCEWAKE_ACK_HSW			0x130044
-#define  FORCEWAKE_ACK				0x130090
-#define  VLV_GTLC_WAKE_CTRL			0x130090
+#define  FORCEWAKE				_MMIO(0xA18C)
+#define  FORCEWAKE_VLV				_MMIO(0x1300b0)
+#define  FORCEWAKE_ACK_VLV			_MMIO(0x1300b4)
+#define  FORCEWAKE_MEDIA_VLV			_MMIO(0x1300b8)
+#define  FORCEWAKE_ACK_MEDIA_VLV		_MMIO(0x1300bc)
+#define  FORCEWAKE_ACK_HSW			_MMIO(0x130044)
+#define  FORCEWAKE_ACK				_MMIO(0x130090)
+#define  VLV_GTLC_WAKE_CTRL			_MMIO(0x130090)
 #define   VLV_GTLC_RENDER_CTX_EXISTS		(1 << 25)
 #define   VLV_GTLC_MEDIA_CTX_EXISTS		(1 << 24)
 #define   VLV_GTLC_ALLOWWAKEREQ			(1 << 0)
 
-#define  VLV_GTLC_PW_STATUS			0x130094
+#define  VLV_GTLC_PW_STATUS			_MMIO(0x130094)
 #define   VLV_GTLC_ALLOWWAKEACK			(1 << 0)
 #define   VLV_GTLC_ALLOWWAKEERR			(1 << 1)
 #define   VLV_GTLC_PW_MEDIA_STATUS_MASK		(1 << 5)
 #define   VLV_GTLC_PW_RENDER_STATUS_MASK	(1 << 7)
-#define  FORCEWAKE_MT				0xa188 /* multi-threaded */
-#define  FORCEWAKE_MEDIA_GEN9			0xa270
-#define  FORCEWAKE_RENDER_GEN9			0xa278
-#define  FORCEWAKE_BLITTER_GEN9			0xa188
-#define  FORCEWAKE_ACK_MEDIA_GEN9		0x0D88
-#define  FORCEWAKE_ACK_RENDER_GEN9		0x0D84
-#define  FORCEWAKE_ACK_BLITTER_GEN9		0x130044
+#define  FORCEWAKE_MT				_MMIO(0xa188) /* multi-threaded */
+#define  FORCEWAKE_MEDIA_GEN9			_MMIO(0xa270)
+#define  FORCEWAKE_RENDER_GEN9			_MMIO(0xa278)
+#define  FORCEWAKE_BLITTER_GEN9			_MMIO(0xa188)
+#define  FORCEWAKE_ACK_MEDIA_GEN9		_MMIO(0x0D88)
+#define  FORCEWAKE_ACK_RENDER_GEN9		_MMIO(0x0D84)
+#define  FORCEWAKE_ACK_BLITTER_GEN9		_MMIO(0x130044)
 #define   FORCEWAKE_KERNEL			0x1
 #define   FORCEWAKE_USER			0x2
-#define  FORCEWAKE_MT_ACK			0x130040
-#define  ECOBUS					0xa180
+#define  FORCEWAKE_MT_ACK			_MMIO(0x130040)
+#define  ECOBUS					_MMIO(0xa180)
 #define    FORCEWAKE_MT_ENABLE			(1<<5)
-#define  VLV_SPAREG2H				0xA194
+#define  VLV_SPAREG2H				_MMIO(0xA194)
 
-#define  GTFIFODBG				0x120000
+#define  GTFIFODBG				_MMIO(0x120000)
 #define    GT_FIFO_SBDROPERR			(1<<6)
 #define    GT_FIFO_BLOBDROPERR			(1<<5)
 #define    GT_FIFO_SB_READ_ABORTERR		(1<<4)
@@ -6725,23 +6805,23 @@ enum skl_disp_power_wells {
 #define    GT_FIFO_IAWRERR			(1<<1)
 #define    GT_FIFO_IARDERR			(1<<0)
 
-#define  GTFIFOCTL				0x120008
+#define  GTFIFOCTL				_MMIO(0x120008)
 #define    GT_FIFO_FREE_ENTRIES_MASK		0x7f
 #define    GT_FIFO_NUM_RESERVED_ENTRIES		20
 #define    GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL	(1 << 12)
 #define    GT_FIFO_CTL_RC6_POLICY_STALL		(1 << 11)
 
-#define  HSW_IDICR				0x9008
+#define  HSW_IDICR				_MMIO(0x9008)
 #define    IDIHASHMSK(x)			(((x) & 0x3f) << 16)
-#define  HSW_EDRAM_PRESENT			0x120010
+#define  HSW_EDRAM_PRESENT			_MMIO(0x120010)
 #define    EDRAM_ENABLED			0x1
 
-#define GEN6_UCGCTL1				0x9400
+#define GEN6_UCGCTL1				_MMIO(0x9400)
 # define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE		(1 << 16)
 # define GEN6_BLBUNIT_CLOCK_GATE_DISABLE		(1 << 5)
 # define GEN6_CSUNIT_CLOCK_GATE_DISABLE			(1 << 7)
 
-#define GEN6_UCGCTL2				0x9404
+#define GEN6_UCGCTL2				_MMIO(0x9404)
 # define GEN6_VFUNIT_CLOCK_GATE_DISABLE			(1 << 31)
 # define GEN7_VDSUNIT_CLOCK_GATE_DISABLE		(1 << 30)
 # define GEN7_TDLUNIT_CLOCK_GATE_DISABLE		(1 << 22)
@@ -6749,30 +6829,30 @@ enum skl_disp_power_wells {
 # define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE		(1 << 12)
 # define GEN6_RCCUNIT_CLOCK_GATE_DISABLE		(1 << 11)
 
-#define GEN6_UCGCTL3				0x9408
+#define GEN6_UCGCTL3				_MMIO(0x9408)
 
-#define GEN7_UCGCTL4				0x940c
+#define GEN7_UCGCTL4				_MMIO(0x940c)
 #define  GEN7_L3BANK2X_CLOCK_GATE_DISABLE	(1<<25)
 
-#define GEN6_RCGCTL1				0x9410
-#define GEN6_RCGCTL2				0x9414
-#define GEN6_RSTCTL				0x9420
+#define GEN6_RCGCTL1				_MMIO(0x9410)
+#define GEN6_RCGCTL2				_MMIO(0x9414)
+#define GEN6_RSTCTL				_MMIO(0x9420)
 
-#define GEN8_UCGCTL6				0x9430
+#define GEN8_UCGCTL6				_MMIO(0x9430)
 #define   GEN8_GAPSUNIT_CLOCK_GATE_DISABLE	(1<<24)
 #define   GEN8_SDEUNIT_CLOCK_GATE_DISABLE	(1<<14)
 #define   GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ (1<<28)
 
-#define GEN6_GFXPAUSE				0xA000
-#define GEN6_RPNSWREQ				0xA008
+#define GEN6_GFXPAUSE				_MMIO(0xA000)
+#define GEN6_RPNSWREQ				_MMIO(0xA008)
 #define   GEN6_TURBO_DISABLE			(1<<31)
 #define   GEN6_FREQUENCY(x)			((x)<<25)
 #define   HSW_FREQUENCY(x)			((x)<<24)
 #define   GEN9_FREQUENCY(x)			((x)<<23)
 #define   GEN6_OFFSET(x)			((x)<<19)
 #define   GEN6_AGGRESSIVE_TURBO			(0<<15)
-#define GEN6_RC_VIDEO_FREQ			0xA00C
-#define GEN6_RC_CONTROL				0xA090
+#define GEN6_RC_VIDEO_FREQ			_MMIO(0xA00C)
+#define GEN6_RC_CONTROL				_MMIO(0xA090)
 #define   GEN6_RC_CTL_RC6pp_ENABLE		(1<<16)
 #define   GEN6_RC_CTL_RC6p_ENABLE		(1<<17)
 #define   GEN6_RC_CTL_RC6_ENABLE		(1<<18)
@@ -6782,16 +6862,16 @@ enum skl_disp_power_wells {
 #define   GEN7_RC_CTL_TO_MODE			(1<<28)
 #define   GEN6_RC_CTL_EI_MODE(x)		((x)<<27)
 #define   GEN6_RC_CTL_HW_ENABLE			(1<<31)
-#define GEN6_RP_DOWN_TIMEOUT			0xA010
-#define GEN6_RP_INTERRUPT_LIMITS		0xA014
-#define GEN6_RPSTAT1				0xA01C
+#define GEN6_RP_DOWN_TIMEOUT			_MMIO(0xA010)
+#define GEN6_RP_INTERRUPT_LIMITS		_MMIO(0xA014)
+#define GEN6_RPSTAT1				_MMIO(0xA01C)
 #define   GEN6_CAGF_SHIFT			8
 #define   HSW_CAGF_SHIFT			7
 #define   GEN9_CAGF_SHIFT			23
 #define   GEN6_CAGF_MASK			(0x7f << GEN6_CAGF_SHIFT)
 #define   HSW_CAGF_MASK				(0x7f << HSW_CAGF_SHIFT)
 #define   GEN9_CAGF_MASK			(0x1ff << GEN9_CAGF_SHIFT)
-#define GEN6_RP_CONTROL				0xA024
+#define GEN6_RP_CONTROL				_MMIO(0xA024)
 #define   GEN6_RP_MEDIA_TURBO			(1<<11)
 #define   GEN6_RP_MEDIA_MODE_MASK		(3<<9)
 #define   GEN6_RP_MEDIA_HW_TURBO_MODE		(3<<9)
@@ -6805,53 +6885,53 @@ enum skl_disp_power_wells {
 #define   GEN6_RP_UP_BUSY_CONT			(0x4<<3)
 #define   GEN6_RP_DOWN_IDLE_AVG			(0x2<<0)
 #define   GEN6_RP_DOWN_IDLE_CONT		(0x1<<0)
-#define GEN6_RP_UP_THRESHOLD			0xA02C
-#define GEN6_RP_DOWN_THRESHOLD			0xA030
-#define GEN6_RP_CUR_UP_EI			0xA050
+#define GEN6_RP_UP_THRESHOLD			_MMIO(0xA02C)
+#define GEN6_RP_DOWN_THRESHOLD			_MMIO(0xA030)
+#define GEN6_RP_CUR_UP_EI			_MMIO(0xA050)
 #define   GEN6_CURICONT_MASK			0xffffff
-#define GEN6_RP_CUR_UP				0xA054
+#define GEN6_RP_CUR_UP				_MMIO(0xA054)
 #define   GEN6_CURBSYTAVG_MASK			0xffffff
-#define GEN6_RP_PREV_UP				0xA058
-#define GEN6_RP_CUR_DOWN_EI			0xA05C
+#define GEN6_RP_PREV_UP				_MMIO(0xA058)
+#define GEN6_RP_CUR_DOWN_EI			_MMIO(0xA05C)
 #define   GEN6_CURIAVG_MASK			0xffffff
-#define GEN6_RP_CUR_DOWN			0xA060
-#define GEN6_RP_PREV_DOWN			0xA064
-#define GEN6_RP_UP_EI				0xA068
-#define GEN6_RP_DOWN_EI				0xA06C
-#define GEN6_RP_IDLE_HYSTERSIS			0xA070
-#define GEN6_RPDEUHWTC				0xA080
-#define GEN6_RPDEUC				0xA084
-#define GEN6_RPDEUCSW				0xA088
-#define GEN6_RC_STATE				0xA094
-#define GEN6_RC1_WAKE_RATE_LIMIT		0xA098
-#define GEN6_RC6_WAKE_RATE_LIMIT		0xA09C
-#define GEN6_RC6pp_WAKE_RATE_LIMIT		0xA0A0
-#define GEN6_RC_EVALUATION_INTERVAL		0xA0A8
-#define GEN6_RC_IDLE_HYSTERSIS			0xA0AC
-#define GEN6_RC_SLEEP				0xA0B0
-#define GEN6_RCUBMABDTMR			0xA0B0
-#define GEN6_RC1e_THRESHOLD			0xA0B4
-#define GEN6_RC6_THRESHOLD			0xA0B8
-#define GEN6_RC6p_THRESHOLD			0xA0BC
-#define VLV_RCEDATA				0xA0BC
-#define GEN6_RC6pp_THRESHOLD			0xA0C0
-#define GEN6_PMINTRMSK				0xA168
+#define GEN6_RP_CUR_DOWN			_MMIO(0xA060)
+#define GEN6_RP_PREV_DOWN			_MMIO(0xA064)
+#define GEN6_RP_UP_EI				_MMIO(0xA068)
+#define GEN6_RP_DOWN_EI				_MMIO(0xA06C)
+#define GEN6_RP_IDLE_HYSTERSIS			_MMIO(0xA070)
+#define GEN6_RPDEUHWTC				_MMIO(0xA080)
+#define GEN6_RPDEUC				_MMIO(0xA084)
+#define GEN6_RPDEUCSW				_MMIO(0xA088)
+#define GEN6_RC_STATE				_MMIO(0xA094)
+#define GEN6_RC1_WAKE_RATE_LIMIT		_MMIO(0xA098)
+#define GEN6_RC6_WAKE_RATE_LIMIT		_MMIO(0xA09C)
+#define GEN6_RC6pp_WAKE_RATE_LIMIT		_MMIO(0xA0A0)
+#define GEN6_RC_EVALUATION_INTERVAL		_MMIO(0xA0A8)
+#define GEN6_RC_IDLE_HYSTERSIS			_MMIO(0xA0AC)
+#define GEN6_RC_SLEEP				_MMIO(0xA0B0)
+#define GEN6_RCUBMABDTMR			_MMIO(0xA0B0)
+#define GEN6_RC1e_THRESHOLD			_MMIO(0xA0B4)
+#define GEN6_RC6_THRESHOLD			_MMIO(0xA0B8)
+#define GEN6_RC6p_THRESHOLD			_MMIO(0xA0BC)
+#define VLV_RCEDATA				_MMIO(0xA0BC)
+#define GEN6_RC6pp_THRESHOLD			_MMIO(0xA0C0)
+#define GEN6_PMINTRMSK				_MMIO(0xA168)
 #define GEN8_PMINTR_REDIRECT_TO_NON_DISP	(1<<31)
-#define VLV_PWRDWNUPCTL				0xA294
-#define GEN9_MEDIA_PG_IDLE_HYSTERESIS		0xA0C4
-#define GEN9_RENDER_PG_IDLE_HYSTERESIS		0xA0C8
-#define GEN9_PG_ENABLE				0xA210
+#define VLV_PWRDWNUPCTL				_MMIO(0xA294)
+#define GEN9_MEDIA_PG_IDLE_HYSTERESIS		_MMIO(0xA0C4)
+#define GEN9_RENDER_PG_IDLE_HYSTERESIS		_MMIO(0xA0C8)
+#define GEN9_PG_ENABLE				_MMIO(0xA210)
 #define GEN9_RENDER_PG_ENABLE			(1<<0)
 #define GEN9_MEDIA_PG_ENABLE			(1<<1)
 
-#define VLV_CHICKEN_3				(VLV_DISPLAY_BASE + 0x7040C)
+#define VLV_CHICKEN_3				_MMIO(VLV_DISPLAY_BASE + 0x7040C)
 #define  PIXEL_OVERLAP_CNT_MASK			(3 << 30)
 #define  PIXEL_OVERLAP_CNT_SHIFT		30
 
-#define GEN6_PMISR				0x44020
-#define GEN6_PMIMR				0x44024 /* rps_lock */
-#define GEN6_PMIIR				0x44028
-#define GEN6_PMIER				0x4402C
+#define GEN6_PMISR				_MMIO(0x44020)
+#define GEN6_PMIMR				_MMIO(0x44024) /* rps_lock */
+#define GEN6_PMIIR				_MMIO(0x44028)
+#define GEN6_PMIER				_MMIO(0x4402C)
 #define  GEN6_PM_MBOX_EVENT			(1<<25)
 #define  GEN6_PM_THERMAL_EVENT			(1<<24)
 #define  GEN6_PM_RP_DOWN_TIMEOUT		(1<<6)
@@ -6863,30 +6943,30 @@ enum skl_disp_power_wells {
 						 GEN6_PM_RP_DOWN_THRESHOLD | \
 						 GEN6_PM_RP_DOWN_TIMEOUT)
 
-#define GEN7_GT_SCRATCH(i)			(0x4F100 + (i) * 4)
+#define GEN7_GT_SCRATCH(i)			_MMIO(0x4F100 + (i) * 4)
 #define GEN7_GT_SCRATCH_REG_NUM			8
 
-#define VLV_GTLC_SURVIVABILITY_REG              0x130098
+#define VLV_GTLC_SURVIVABILITY_REG              _MMIO(0x130098)
 #define VLV_GFX_CLK_STATUS_BIT			(1<<3)
 #define VLV_GFX_CLK_FORCE_ON_BIT		(1<<2)
 
-#define GEN6_GT_GFX_RC6_LOCKED			0x138104
-#define VLV_COUNTER_CONTROL			0x138104
+#define GEN6_GT_GFX_RC6_LOCKED			_MMIO(0x138104)
+#define VLV_COUNTER_CONTROL			_MMIO(0x138104)
 #define   VLV_COUNT_RANGE_HIGH			(1<<15)
 #define   VLV_MEDIA_RC0_COUNT_EN		(1<<5)
 #define   VLV_RENDER_RC0_COUNT_EN		(1<<4)
 #define   VLV_MEDIA_RC6_COUNT_EN		(1<<1)
 #define   VLV_RENDER_RC6_COUNT_EN		(1<<0)
-#define GEN6_GT_GFX_RC6				0x138108
-#define VLV_GT_RENDER_RC6			0x138108
-#define VLV_GT_MEDIA_RC6			0x13810C
+#define GEN6_GT_GFX_RC6				_MMIO(0x138108)
+#define VLV_GT_RENDER_RC6			_MMIO(0x138108)
+#define VLV_GT_MEDIA_RC6			_MMIO(0x13810C)
 
-#define GEN6_GT_GFX_RC6p			0x13810C
-#define GEN6_GT_GFX_RC6pp			0x138110
-#define VLV_RENDER_C0_COUNT			0x138118
-#define VLV_MEDIA_C0_COUNT			0x13811C
+#define GEN6_GT_GFX_RC6p			_MMIO(0x13810C)
+#define GEN6_GT_GFX_RC6pp			_MMIO(0x138110)
+#define VLV_RENDER_C0_COUNT			_MMIO(0x138118)
+#define VLV_MEDIA_C0_COUNT			_MMIO(0x13811C)
 
-#define GEN6_PCODE_MAILBOX			0x138124
+#define GEN6_PCODE_MAILBOX			_MMIO(0x138124)
 #define   GEN6_PCODE_READY			(1<<31)
 #define	  GEN6_PCODE_WRITE_RC6VIDS		0x4
 #define	  GEN6_PCODE_READ_RC6VIDS		0x5
@@ -6909,12 +6989,12 @@ enum skl_disp_power_wells {
 #define   HSW_PCODE_DE_WRITE_FREQ_REQ		0x17
 #define   DISPLAY_IPS_CONTROL			0x19
 #define	  HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL	0x1A
-#define GEN6_PCODE_DATA				0x138128
+#define GEN6_PCODE_DATA				_MMIO(0x138128)
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT	8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT	16
-#define GEN6_PCODE_DATA1			0x13812C
+#define GEN6_PCODE_DATA1			_MMIO(0x13812C)
 
-#define GEN6_GT_CORE_STATUS		0x138060
+#define GEN6_GT_CORE_STATUS		_MMIO(0x138060)
 #define   GEN6_CORE_CPD_STATE_MASK	(7<<4)
 #define   GEN6_RCn_MASK			7
 #define   GEN6_RC0			0
@@ -6922,26 +7002,26 @@ enum skl_disp_power_wells {
 #define   GEN6_RC6			3
 #define   GEN6_RC7			4
 
-#define GEN8_GT_SLICE_INFO		0x138064
+#define GEN8_GT_SLICE_INFO		_MMIO(0x138064)
 #define   GEN8_LSLICESTAT_MASK		0x7
 
-#define CHV_POWER_SS0_SIG1		0xa720
-#define CHV_POWER_SS1_SIG1		0xa728
+#define CHV_POWER_SS0_SIG1		_MMIO(0xa720)
+#define CHV_POWER_SS1_SIG1		_MMIO(0xa728)
 #define   CHV_SS_PG_ENABLE		(1<<1)
 #define   CHV_EU08_PG_ENABLE		(1<<9)
 #define   CHV_EU19_PG_ENABLE		(1<<17)
 #define   CHV_EU210_PG_ENABLE		(1<<25)
 
-#define CHV_POWER_SS0_SIG2		0xa724
-#define CHV_POWER_SS1_SIG2		0xa72c
+#define CHV_POWER_SS0_SIG2		_MMIO(0xa724)
+#define CHV_POWER_SS1_SIG2		_MMIO(0xa72c)
 #define   CHV_EU311_PG_ENABLE		(1<<1)
 
-#define GEN9_SLICE_PGCTL_ACK(slice)	(0x804c + (slice)*0x4)
+#define GEN9_SLICE_PGCTL_ACK(slice)	_MMIO(0x804c + (slice)*0x4)
 #define   GEN9_PGCTL_SLICE_ACK		(1 << 0)
 #define   GEN9_PGCTL_SS_ACK(subslice)	(1 << (2 + (subslice)*2))
 
-#define GEN9_SS01_EU_PGCTL_ACK(slice)	(0x805c + (slice)*0x8)
-#define GEN9_SS23_EU_PGCTL_ACK(slice)	(0x8060 + (slice)*0x8)
+#define GEN9_SS01_EU_PGCTL_ACK(slice)	_MMIO(0x805c + (slice)*0x8)
+#define GEN9_SS23_EU_PGCTL_ACK(slice)	_MMIO(0x8060 + (slice)*0x8)
 #define   GEN9_PGCTL_SSA_EU08_ACK	(1 << 0)
 #define   GEN9_PGCTL_SSA_EU19_ACK	(1 << 2)
 #define   GEN9_PGCTL_SSA_EU210_ACK	(1 << 4)
@@ -6951,18 +7031,17 @@ enum skl_disp_power_wells {
 #define   GEN9_PGCTL_SSB_EU210_ACK	(1 << 12)
 #define   GEN9_PGCTL_SSB_EU311_ACK	(1 << 14)
 
-#define GEN7_MISCCPCTL			(0x9424)
+#define GEN7_MISCCPCTL				_MMIO(0x9424)
 #define   GEN7_DOP_CLOCK_GATE_ENABLE		(1<<0)
 #define   GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE	(1<<2)
 #define   GEN8_DOP_CLOCK_GATE_GUC_ENABLE	(1<<4)
 #define   GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE     (1<<6)
 
-#define GEN8_GARBCNTL                   0xB004
+#define GEN8_GARBCNTL                   _MMIO(0xB004)
 #define   GEN9_GAPS_TSV_CREDIT_DISABLE  (1<<7)
 
 /* IVYBRIDGE DPF */
-#define GEN7_L3CDERRST1			0xB008 /* L3CD Error Status 1 */
-#define HSW_L3CDERRST11			0xB208 /* L3CD Error Status register 1 slice 1 */
+#define GEN7_L3CDERRST1(slice)		_MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */
 #define   GEN7_L3CDERRST1_ROW_MASK	(0x7ff<<14)
 #define   GEN7_PARITY_ERROR_VALID	(1<<13)
 #define   GEN7_L3CDERRST1_BANK_MASK	(3<<11)
@@ -6975,119 +7054,102 @@ enum skl_disp_power_wells {
 		((reg & GEN7_L3CDERRST1_SUBBANK_MASK) >> 8)
 #define   GEN7_L3CDERRST1_ENABLE	(1<<7)
 
-#define GEN7_L3LOG_BASE			0xB070
-#define HSW_L3LOG_BASE_SLICE1		0xB270
+#define GEN7_L3LOG(slice, i)		_MMIO(0xB070 + (slice) * 0x200 + (i) * 4)
 #define GEN7_L3LOG_SIZE			0x80
 
-#define GEN7_HALF_SLICE_CHICKEN1	0xe100 /* IVB GT1 + VLV */
-#define GEN7_HALF_SLICE_CHICKEN1_GT2	0xf100
+#define GEN7_HALF_SLICE_CHICKEN1	_MMIO(0xe100) /* IVB GT1 + VLV */
+#define GEN7_HALF_SLICE_CHICKEN1_GT2	_MMIO(0xf100)
 #define   GEN7_MAX_PS_THREAD_DEP		(8<<12)
 #define   GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE	(1<<10)
 #define   GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE	(1<<4)
 #define   GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE	(1<<3)
 
-#define GEN9_HALF_SLICE_CHICKEN5	0xe188
+#define GEN9_HALF_SLICE_CHICKEN5	_MMIO(0xe188)
 #define   GEN9_DG_MIRROR_FIX_ENABLE	(1<<5)
 #define   GEN9_CCS_TLB_PREFETCH_ENABLE	(1<<3)
 
-#define GEN8_ROW_CHICKEN		0xe4f0
+#define GEN8_ROW_CHICKEN		_MMIO(0xe4f0)
 #define   PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE	(1<<8)
 #define   STALL_DOP_GATING_DISABLE		(1<<5)
 
-#define GEN7_ROW_CHICKEN2		0xe4f4
-#define GEN7_ROW_CHICKEN2_GT2		0xf4f4
+#define GEN7_ROW_CHICKEN2		_MMIO(0xe4f4)
+#define GEN7_ROW_CHICKEN2_GT2		_MMIO(0xf4f4)
 #define   DOP_CLOCK_GATING_DISABLE	(1<<0)
 
-#define HSW_ROW_CHICKEN3		0xe49c
+#define HSW_ROW_CHICKEN3		_MMIO(0xe49c)
 #define  HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE    (1 << 6)
 
-#define HALF_SLICE_CHICKEN2		0xe180
+#define HALF_SLICE_CHICKEN2		_MMIO(0xe180)
 #define   GEN8_ST_PO_DISABLE		(1<<13)
 
-#define HALF_SLICE_CHICKEN3		0xe184
+#define HALF_SLICE_CHICKEN3		_MMIO(0xe184)
 #define   HSW_SAMPLE_C_PERFORMANCE	(1<<9)
 #define   GEN8_CENTROID_PIXEL_OPT_DIS	(1<<8)
 #define   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC	(1<<5)
 #define   GEN8_SAMPLER_POWER_BYPASS_DIS	(1<<1)
 
-#define GEN9_HALF_SLICE_CHICKEN7	0xe194
+#define GEN9_HALF_SLICE_CHICKEN7	_MMIO(0xe194)
 #define   GEN9_ENABLE_YV12_BUGFIX	(1<<4)
 
 /* Audio */
-#define G4X_AUD_VID_DID			(dev_priv->info.display_mmio_offset + 0x62020)
+#define G4X_AUD_VID_DID			_MMIO(dev_priv->info.display_mmio_offset + 0x62020)
 #define   INTEL_AUDIO_DEVCL		0x808629FB
 #define   INTEL_AUDIO_DEVBLC		0x80862801
 #define   INTEL_AUDIO_DEVCTG		0x80862802
 
-#define G4X_AUD_CNTL_ST			0x620B4
+#define G4X_AUD_CNTL_ST			_MMIO(0x620B4)
 #define   G4X_ELDV_DEVCL_DEVBLC		(1 << 13)
 #define   G4X_ELDV_DEVCTG		(1 << 14)
 #define   G4X_ELD_ADDR_MASK		(0xf << 5)
 #define   G4X_ELD_ACK			(1 << 4)
-#define G4X_HDMIW_HDMIEDID		0x6210C
+#define G4X_HDMIW_HDMIEDID		_MMIO(0x6210C)
 
 #define _IBX_HDMIW_HDMIEDID_A		0xE2050
 #define _IBX_HDMIW_HDMIEDID_B		0xE2150
-#define IBX_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
-					_IBX_HDMIW_HDMIEDID_A, \
-					_IBX_HDMIW_HDMIEDID_B)
+#define IBX_HDMIW_HDMIEDID(pipe)	_MMIO_PIPE(pipe, _IBX_HDMIW_HDMIEDID_A, \
+						  _IBX_HDMIW_HDMIEDID_B)
 #define _IBX_AUD_CNTL_ST_A		0xE20B4
 #define _IBX_AUD_CNTL_ST_B		0xE21B4
-#define IBX_AUD_CNTL_ST(pipe) _PIPE(pipe, \
-					_IBX_AUD_CNTL_ST_A, \
-					_IBX_AUD_CNTL_ST_B)
+#define IBX_AUD_CNTL_ST(pipe)		_MMIO_PIPE(pipe, _IBX_AUD_CNTL_ST_A, \
+						  _IBX_AUD_CNTL_ST_B)
 #define   IBX_ELD_BUFFER_SIZE_MASK	(0x1f << 10)
 #define   IBX_ELD_ADDRESS_MASK		(0x1f << 5)
 #define   IBX_ELD_ACK			(1 << 4)
-#define IBX_AUD_CNTL_ST2		0xE20C0
+#define IBX_AUD_CNTL_ST2		_MMIO(0xE20C0)
 #define   IBX_CP_READY(port)		((1 << 1) << (((port) - 1) * 4))
 #define   IBX_ELD_VALID(port)		((1 << 0) << (((port) - 1) * 4))
 
 #define _CPT_HDMIW_HDMIEDID_A		0xE5050
 #define _CPT_HDMIW_HDMIEDID_B		0xE5150
-#define CPT_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
-					_CPT_HDMIW_HDMIEDID_A, \
-					_CPT_HDMIW_HDMIEDID_B)
+#define CPT_HDMIW_HDMIEDID(pipe)	_MMIO_PIPE(pipe, _CPT_HDMIW_HDMIEDID_A, _CPT_HDMIW_HDMIEDID_B)
 #define _CPT_AUD_CNTL_ST_A		0xE50B4
 #define _CPT_AUD_CNTL_ST_B		0xE51B4
-#define CPT_AUD_CNTL_ST(pipe) _PIPE(pipe, \
-					_CPT_AUD_CNTL_ST_A, \
-					_CPT_AUD_CNTL_ST_B)
-#define CPT_AUD_CNTRL_ST2		0xE50C0
+#define CPT_AUD_CNTL_ST(pipe)		_MMIO_PIPE(pipe, _CPT_AUD_CNTL_ST_A, _CPT_AUD_CNTL_ST_B)
+#define CPT_AUD_CNTRL_ST2		_MMIO(0xE50C0)
 
 #define _VLV_HDMIW_HDMIEDID_A		(VLV_DISPLAY_BASE + 0x62050)
 #define _VLV_HDMIW_HDMIEDID_B		(VLV_DISPLAY_BASE + 0x62150)
-#define VLV_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
-					_VLV_HDMIW_HDMIEDID_A, \
-					_VLV_HDMIW_HDMIEDID_B)
+#define VLV_HDMIW_HDMIEDID(pipe)	_MMIO_PIPE(pipe, _VLV_HDMIW_HDMIEDID_A, _VLV_HDMIW_HDMIEDID_B)
 #define _VLV_AUD_CNTL_ST_A		(VLV_DISPLAY_BASE + 0x620B4)
 #define _VLV_AUD_CNTL_ST_B		(VLV_DISPLAY_BASE + 0x621B4)
-#define VLV_AUD_CNTL_ST(pipe) _PIPE(pipe, \
-					_VLV_AUD_CNTL_ST_A, \
-					_VLV_AUD_CNTL_ST_B)
-#define VLV_AUD_CNTL_ST2		(VLV_DISPLAY_BASE + 0x620C0)
+#define VLV_AUD_CNTL_ST(pipe)		_MMIO_PIPE(pipe, _VLV_AUD_CNTL_ST_A, _VLV_AUD_CNTL_ST_B)
+#define VLV_AUD_CNTL_ST2		_MMIO(VLV_DISPLAY_BASE + 0x620C0)
 
 /* These are the 4 32-bit write offset registers for each stream
  * output buffer.  It determines the offset from the
  * 3DSTATE_SO_BUFFERs that the next streamed vertex output goes to.
  */
-#define GEN7_SO_WRITE_OFFSET(n)		(0x5280 + (n) * 4)
+#define GEN7_SO_WRITE_OFFSET(n)		_MMIO(0x5280 + (n) * 4)
 
 #define _IBX_AUD_CONFIG_A		0xe2000
 #define _IBX_AUD_CONFIG_B		0xe2100
-#define IBX_AUD_CFG(pipe) _PIPE(pipe, \
-					_IBX_AUD_CONFIG_A, \
-					_IBX_AUD_CONFIG_B)
+#define IBX_AUD_CFG(pipe)		_MMIO_PIPE(pipe, _IBX_AUD_CONFIG_A, _IBX_AUD_CONFIG_B)
 #define _CPT_AUD_CONFIG_A		0xe5000
 #define _CPT_AUD_CONFIG_B		0xe5100
-#define CPT_AUD_CFG(pipe) _PIPE(pipe, \
-					_CPT_AUD_CONFIG_A, \
-					_CPT_AUD_CONFIG_B)
+#define CPT_AUD_CFG(pipe)		_MMIO_PIPE(pipe, _CPT_AUD_CONFIG_A, _CPT_AUD_CONFIG_B)
 #define _VLV_AUD_CONFIG_A		(VLV_DISPLAY_BASE + 0x62000)
 #define _VLV_AUD_CONFIG_B		(VLV_DISPLAY_BASE + 0x62100)
-#define VLV_AUD_CFG(pipe) _PIPE(pipe, \
-					_VLV_AUD_CONFIG_A, \
-					_VLV_AUD_CONFIG_B)
+#define VLV_AUD_CFG(pipe)		_MMIO_PIPE(pipe, _VLV_AUD_CONFIG_A, _VLV_AUD_CONFIG_B)
 
 #define   AUD_CONFIG_N_VALUE_INDEX		(1 << 29)
 #define   AUD_CONFIG_N_PROG_ENABLE		(1 << 28)
@@ -7112,72 +7174,62 @@ enum skl_disp_power_wells {
 /* HSW Audio */
 #define _HSW_AUD_CONFIG_A		0x65000
 #define _HSW_AUD_CONFIG_B		0x65100
-#define HSW_AUD_CFG(pipe) _PIPE(pipe, \
-					_HSW_AUD_CONFIG_A, \
-					_HSW_AUD_CONFIG_B)
+#define HSW_AUD_CFG(pipe)		_MMIO_PIPE(pipe, _HSW_AUD_CONFIG_A, _HSW_AUD_CONFIG_B)
 
 #define _HSW_AUD_MISC_CTRL_A		0x65010
 #define _HSW_AUD_MISC_CTRL_B		0x65110
-#define HSW_AUD_MISC_CTRL(pipe) _PIPE(pipe, \
-					_HSW_AUD_MISC_CTRL_A, \
-					_HSW_AUD_MISC_CTRL_B)
+#define HSW_AUD_MISC_CTRL(pipe)		_MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B)
 
 #define _HSW_AUD_DIP_ELD_CTRL_ST_A	0x650b4
 #define _HSW_AUD_DIP_ELD_CTRL_ST_B	0x651b4
-#define HSW_AUD_DIP_ELD_CTRL(pipe) _PIPE(pipe, \
-					_HSW_AUD_DIP_ELD_CTRL_ST_A, \
-					_HSW_AUD_DIP_ELD_CTRL_ST_B)
+#define HSW_AUD_DIP_ELD_CTRL(pipe)	_MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B)
 
 /* Audio Digital Converter */
 #define _HSW_AUD_DIG_CNVT_1		0x65080
 #define _HSW_AUD_DIG_CNVT_2		0x65180
-#define AUD_DIG_CNVT(pipe) _PIPE(pipe, \
-					_HSW_AUD_DIG_CNVT_1, \
-					_HSW_AUD_DIG_CNVT_2)
+#define AUD_DIG_CNVT(pipe)		_MMIO_PIPE(pipe, _HSW_AUD_DIG_CNVT_1, _HSW_AUD_DIG_CNVT_2)
 #define DIP_PORT_SEL_MASK		0x3
 
 #define _HSW_AUD_EDID_DATA_A		0x65050
 #define _HSW_AUD_EDID_DATA_B		0x65150
-#define HSW_AUD_EDID_DATA(pipe) _PIPE(pipe, \
-					_HSW_AUD_EDID_DATA_A, \
-					_HSW_AUD_EDID_DATA_B)
+#define HSW_AUD_EDID_DATA(pipe)		_MMIO_PIPE(pipe, _HSW_AUD_EDID_DATA_A, _HSW_AUD_EDID_DATA_B)
 
-#define HSW_AUD_PIPE_CONV_CFG		0x6507c
-#define HSW_AUD_PIN_ELD_CP_VLD		0x650c0
+#define HSW_AUD_PIPE_CONV_CFG		_MMIO(0x6507c)
+#define HSW_AUD_PIN_ELD_CP_VLD		_MMIO(0x650c0)
 #define   AUDIO_INACTIVE(trans)		((1 << 3) << ((trans) * 4))
 #define   AUDIO_OUTPUT_ENABLE(trans)	((1 << 2) << ((trans) * 4))
 #define   AUDIO_CP_READY(trans)		((1 << 1) << ((trans) * 4))
 #define   AUDIO_ELD_VALID(trans)	((1 << 0) << ((trans) * 4))
 
-#define HSW_AUD_CHICKENBIT			0x65f10
+#define HSW_AUD_CHICKENBIT			_MMIO(0x65f10)
 #define   SKL_AUD_CODEC_WAKE_SIGNAL		(1 << 15)
 
 /* HSW Power Wells */
-#define HSW_PWR_WELL_BIOS			0x45400 /* CTL1 */
-#define HSW_PWR_WELL_DRIVER			0x45404 /* CTL2 */
-#define HSW_PWR_WELL_KVMR			0x45408 /* CTL3 */
-#define HSW_PWR_WELL_DEBUG			0x4540C /* CTL4 */
+#define HSW_PWR_WELL_BIOS			_MMIO(0x45400) /* CTL1 */
+#define HSW_PWR_WELL_DRIVER			_MMIO(0x45404) /* CTL2 */
+#define HSW_PWR_WELL_KVMR			_MMIO(0x45408) /* CTL3 */
+#define HSW_PWR_WELL_DEBUG			_MMIO(0x4540C) /* CTL4 */
 #define   HSW_PWR_WELL_ENABLE_REQUEST		(1<<31)
 #define   HSW_PWR_WELL_STATE_ENABLED		(1<<30)
-#define HSW_PWR_WELL_CTL5			0x45410
+#define HSW_PWR_WELL_CTL5			_MMIO(0x45410)
 #define   HSW_PWR_WELL_ENABLE_SINGLE_STEP	(1<<31)
 #define   HSW_PWR_WELL_PWR_GATE_OVERRIDE	(1<<20)
 #define   HSW_PWR_WELL_FORCE_ON			(1<<19)
-#define HSW_PWR_WELL_CTL6			0x45414
+#define HSW_PWR_WELL_CTL6			_MMIO(0x45414)
 
 /* SKL Fuse Status */
-#define SKL_FUSE_STATUS				0x42000
+#define SKL_FUSE_STATUS				_MMIO(0x42000)
 #define  SKL_FUSE_DOWNLOAD_STATUS              (1<<31)
 #define  SKL_FUSE_PG0_DIST_STATUS              (1<<27)
 #define  SKL_FUSE_PG1_DIST_STATUS              (1<<26)
 #define  SKL_FUSE_PG2_DIST_STATUS              (1<<25)
 
 /* Per-pipe DDI Function Control */
-#define TRANS_DDI_FUNC_CTL_A		0x60400
-#define TRANS_DDI_FUNC_CTL_B		0x61400
-#define TRANS_DDI_FUNC_CTL_C		0x62400
-#define TRANS_DDI_FUNC_CTL_EDP		0x6F400
-#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER2(tran, TRANS_DDI_FUNC_CTL_A)
+#define _TRANS_DDI_FUNC_CTL_A		0x60400
+#define _TRANS_DDI_FUNC_CTL_B		0x61400
+#define _TRANS_DDI_FUNC_CTL_C		0x62400
+#define _TRANS_DDI_FUNC_CTL_EDP		0x6F400
+#define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL_A)
 
 #define  TRANS_DDI_FUNC_ENABLE		(1<<31)
 /* Those bits are ignored by pipe EDP since it can only connect to DDI A */
@@ -7207,9 +7259,9 @@ enum skl_disp_power_wells {
 #define  TRANS_DDI_BFI_ENABLE		(1<<4)
 
 /* DisplayPort Transport Control */
-#define DP_TP_CTL_A			0x64040
-#define DP_TP_CTL_B			0x64140
-#define DP_TP_CTL(port) _PORT(port, DP_TP_CTL_A, DP_TP_CTL_B)
+#define _DP_TP_CTL_A			0x64040
+#define _DP_TP_CTL_B			0x64140
+#define DP_TP_CTL(port) _MMIO_PORT(port, _DP_TP_CTL_A, _DP_TP_CTL_B)
 #define  DP_TP_CTL_ENABLE			(1<<31)
 #define  DP_TP_CTL_MODE_SST			(0<<27)
 #define  DP_TP_CTL_MODE_MST			(1<<27)
@@ -7225,9 +7277,9 @@ enum skl_disp_power_wells {
 #define  DP_TP_CTL_SCRAMBLE_DISABLE		(1<<7)
 
 /* DisplayPort Transport Status */
-#define DP_TP_STATUS_A			0x64044
-#define DP_TP_STATUS_B			0x64144
-#define DP_TP_STATUS(port) _PORT(port, DP_TP_STATUS_A, DP_TP_STATUS_B)
+#define _DP_TP_STATUS_A			0x64044
+#define _DP_TP_STATUS_B			0x64144
+#define DP_TP_STATUS(port) _MMIO_PORT(port, _DP_TP_STATUS_A, _DP_TP_STATUS_B)
 #define  DP_TP_STATUS_IDLE_DONE			(1<<25)
 #define  DP_TP_STATUS_ACT_SENT			(1<<24)
 #define  DP_TP_STATUS_MODE_STATUS_MST		(1<<23)
@@ -7237,9 +7289,9 @@ enum skl_disp_power_wells {
 #define  DP_TP_STATUS_PAYLOAD_MAPPING_VC0	(3 << 0)
 
 /* DDI Buffer Control */
-#define DDI_BUF_CTL_A				0x64000
-#define DDI_BUF_CTL_B				0x64100
-#define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B)
+#define _DDI_BUF_CTL_A				0x64000
+#define _DDI_BUF_CTL_B				0x64100
+#define DDI_BUF_CTL(port) _MMIO_PORT(port, _DDI_BUF_CTL_A, _DDI_BUF_CTL_B)
 #define  DDI_BUF_CTL_ENABLE			(1<<31)
 #define  DDI_BUF_TRANS_SELECT(n)	((n) << 24)
 #define  DDI_BUF_EMP_MASK			(0xf<<24)
@@ -7252,17 +7304,17 @@ enum skl_disp_power_wells {
 #define  DDI_INIT_DISPLAY_DETECTED		(1<<0)
 
 /* DDI Buffer Translations */
-#define DDI_BUF_TRANS_A				0x64E00
-#define DDI_BUF_TRANS_B				0x64E60
-#define DDI_BUF_TRANS_LO(port, i) (_PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B) + (i) * 8)
-#define DDI_BUF_TRANS_HI(port, i) (_PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B) + (i) * 8 + 4)
+#define _DDI_BUF_TRANS_A		0x64E00
+#define _DDI_BUF_TRANS_B		0x64E60
+#define DDI_BUF_TRANS_LO(port, i)	_MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8)
+#define DDI_BUF_TRANS_HI(port, i)	_MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8 + 4)
 
 /* Sideband Interface (SBI) is programmed indirectly, via
  * SBI_ADDR, which contains the register offset; and SBI_DATA,
  * which contains the payload */
-#define SBI_ADDR			0xC6000
-#define SBI_DATA			0xC6004
-#define SBI_CTL_STAT			0xC6008
+#define SBI_ADDR			_MMIO(0xC6000)
+#define SBI_DATA			_MMIO(0xC6004)
+#define SBI_CTL_STAT			_MMIO(0xC6008)
 #define  SBI_CTL_DEST_ICLK		(0x0<<16)
 #define  SBI_CTL_DEST_MPHY		(0x1<<16)
 #define  SBI_CTL_OP_IORD		(0x2<<8)
@@ -7275,6 +7327,7 @@ enum skl_disp_power_wells {
 #define  SBI_READY			(0x0<<0)
 
 /* SBI offsets */
+#define  SBI_SSCDIVINTPHASE			0x0200
 #define  SBI_SSCDIVINTPHASE6			0x0600
 #define   SBI_SSCDIVINTPHASE_DIVSEL_MASK	((0x7f)<<1)
 #define   SBI_SSCDIVINTPHASE_DIVSEL(x)		((x)<<1)
@@ -7282,6 +7335,7 @@ enum skl_disp_power_wells {
 #define   SBI_SSCDIVINTPHASE_INCVAL(x)		((x)<<8)
 #define   SBI_SSCDIVINTPHASE_DIR(x)		((x)<<15)
 #define   SBI_SSCDIVINTPHASE_PROPAGATE		(1<<0)
+#define  SBI_SSCDITHPHASE			0x0204
 #define  SBI_SSCCTL				0x020c
 #define  SBI_SSCCTL6				0x060C
 #define   SBI_SSCCTL_PATHALT			(1<<3)
@@ -7293,12 +7347,12 @@ enum skl_disp_power_wells {
 #define   SBI_GEN0_CFG_BUFFENABLE_DISABLE	(1<<0)
 
 /* LPT PIXCLK_GATE */
-#define PIXCLK_GATE			0xC6020
+#define PIXCLK_GATE			_MMIO(0xC6020)
 #define  PIXCLK_GATE_UNGATE		(1<<0)
 #define  PIXCLK_GATE_GATE		(0<<0)
 
 /* SPLL */
-#define SPLL_CTL			0x46020
+#define SPLL_CTL			_MMIO(0x46020)
 #define  SPLL_PLL_ENABLE		(1<<31)
 #define  SPLL_PLL_SSC			(1<<28)
 #define  SPLL_PLL_NON_SSC		(2<<28)
@@ -7310,9 +7364,9 @@ enum skl_disp_power_wells {
 #define  SPLL_PLL_FREQ_MASK		(3<<26)
 
 /* WRPLL */
-#define WRPLL_CTL1			0x46040
-#define WRPLL_CTL2			0x46060
-#define WRPLL_CTL(pll)			(pll == 0 ? WRPLL_CTL1 : WRPLL_CTL2)
+#define _WRPLL_CTL1			0x46040
+#define _WRPLL_CTL2			0x46060
+#define WRPLL_CTL(pll)			_MMIO_PIPE(pll, _WRPLL_CTL1, _WRPLL_CTL2)
 #define  WRPLL_PLL_ENABLE		(1<<31)
 #define  WRPLL_PLL_SSC			(1<<28)
 #define  WRPLL_PLL_NON_SSC		(2<<28)
@@ -7329,9 +7383,9 @@ enum skl_disp_power_wells {
 #define  WRPLL_DIVIDER_FB_MASK		(0xff<<16)
 
 /* Port clock selection */
-#define PORT_CLK_SEL_A			0x46100
-#define PORT_CLK_SEL_B			0x46104
-#define PORT_CLK_SEL(port) _PORT(port, PORT_CLK_SEL_A, PORT_CLK_SEL_B)
+#define _PORT_CLK_SEL_A			0x46100
+#define _PORT_CLK_SEL_B			0x46104
+#define PORT_CLK_SEL(port) _MMIO_PORT(port, _PORT_CLK_SEL_A, _PORT_CLK_SEL_B)
 #define  PORT_CLK_SEL_LCPLL_2700	(0<<29)
 #define  PORT_CLK_SEL_LCPLL_1350	(1<<29)
 #define  PORT_CLK_SEL_LCPLL_810		(2<<29)
@@ -7343,18 +7397,18 @@ enum skl_disp_power_wells {
 #define  PORT_CLK_SEL_MASK		(7<<29)
 
 /* Transcoder clock selection */
-#define TRANS_CLK_SEL_A			0x46140
-#define TRANS_CLK_SEL_B			0x46144
-#define TRANS_CLK_SEL(tran) _TRANSCODER(tran, TRANS_CLK_SEL_A, TRANS_CLK_SEL_B)
+#define _TRANS_CLK_SEL_A		0x46140
+#define _TRANS_CLK_SEL_B		0x46144
+#define TRANS_CLK_SEL(tran) _MMIO_TRANS(tran, _TRANS_CLK_SEL_A, _TRANS_CLK_SEL_B)
 /* For each transcoder, we need to select the corresponding port clock */
 #define  TRANS_CLK_SEL_DISABLED		(0x0<<29)
 #define  TRANS_CLK_SEL_PORT(x)		(((x)+1)<<29)
 
-#define TRANSA_MSA_MISC			0x60410
-#define TRANSB_MSA_MISC			0x61410
-#define TRANSC_MSA_MISC			0x62410
-#define TRANS_EDP_MSA_MISC		0x6f410
-#define TRANS_MSA_MISC(tran) _TRANSCODER2(tran, TRANSA_MSA_MISC)
+#define _TRANSA_MSA_MISC		0x60410
+#define _TRANSB_MSA_MISC		0x61410
+#define _TRANSC_MSA_MISC		0x62410
+#define _TRANS_EDP_MSA_MISC		0x6f410
+#define TRANS_MSA_MISC(tran) _MMIO_TRANS2(tran, _TRANSA_MSA_MISC)
 
 #define  TRANS_MSA_SYNC_CLK		(1<<0)
 #define  TRANS_MSA_6_BPC		(0<<5)
@@ -7364,7 +7418,7 @@ enum skl_disp_power_wells {
 #define  TRANS_MSA_16_BPC		(4<<5)
 
 /* LCPLL Control */
-#define LCPLL_CTL			0x130040
+#define LCPLL_CTL			_MMIO(0x130040)
 #define  LCPLL_PLL_DISABLE		(1<<31)
 #define  LCPLL_PLL_LOCK			(1<<30)
 #define  LCPLL_CLK_FREQ_MASK		(3<<26)
@@ -7384,7 +7438,7 @@ enum skl_disp_power_wells {
  */
 
 /* CDCLK_CTL */
-#define CDCLK_CTL			0x46000
+#define CDCLK_CTL			_MMIO(0x46000)
 #define  CDCLK_FREQ_SEL_MASK		(3<<26)
 #define  CDCLK_FREQ_450_432		(0<<26)
 #define  CDCLK_FREQ_540			(1<<26)
@@ -7400,12 +7454,12 @@ enum skl_disp_power_wells {
 #define  BXT_CDCLK_SSA_PRECHARGE_ENABLE	(1<<16)
 
 /* LCPLL_CTL */
-#define LCPLL1_CTL		0x46010
-#define LCPLL2_CTL		0x46014
+#define LCPLL1_CTL		_MMIO(0x46010)
+#define LCPLL2_CTL		_MMIO(0x46014)
 #define  LCPLL_PLL_ENABLE	(1<<31)
 
 /* DPLL control1 */
-#define DPLL_CTRL1		0x6C058
+#define DPLL_CTRL1		_MMIO(0x6C058)
 #define  DPLL_CTRL1_HDMI_MODE(id)		(1<<((id)*6+5))
 #define  DPLL_CTRL1_SSC(id)			(1<<((id)*6+4))
 #define  DPLL_CTRL1_LINK_RATE_MASK(id)		(7<<((id)*6+1))
@@ -7420,7 +7474,7 @@ enum skl_disp_power_wells {
 #define  DPLL_CTRL1_LINK_RATE_2160		5
 
 /* DPLL control2 */
-#define DPLL_CTRL2				0x6C05C
+#define DPLL_CTRL2				_MMIO(0x6C05C)
 #define  DPLL_CTRL2_DDI_CLK_OFF(port)		(1<<((port)+15))
 #define  DPLL_CTRL2_DDI_CLK_SEL_MASK(port)	(3<<((port)*3+1))
 #define  DPLL_CTRL2_DDI_CLK_SEL_SHIFT(port)    ((port)*3+1)
@@ -7428,21 +7482,21 @@ enum skl_disp_power_wells {
 #define  DPLL_CTRL2_DDI_SEL_OVERRIDE(port)     (1<<((port)*3))
 
 /* DPLL Status */
-#define DPLL_STATUS	0x6C060
+#define DPLL_STATUS	_MMIO(0x6C060)
 #define  DPLL_LOCK(id) (1<<((id)*8))
 
 /* DPLL cfg */
-#define DPLL1_CFGCR1	0x6C040
-#define DPLL2_CFGCR1	0x6C048
-#define DPLL3_CFGCR1	0x6C050
+#define _DPLL1_CFGCR1	0x6C040
+#define _DPLL2_CFGCR1	0x6C048
+#define _DPLL3_CFGCR1	0x6C050
 #define  DPLL_CFGCR1_FREQ_ENABLE	(1<<31)
 #define  DPLL_CFGCR1_DCO_FRACTION_MASK	(0x7fff<<9)
 #define  DPLL_CFGCR1_DCO_FRACTION(x)	((x)<<9)
 #define  DPLL_CFGCR1_DCO_INTEGER_MASK	(0x1ff)
 
-#define DPLL1_CFGCR2	0x6C044
-#define DPLL2_CFGCR2	0x6C04C
-#define DPLL3_CFGCR2	0x6C054
+#define _DPLL1_CFGCR2	0x6C044
+#define _DPLL2_CFGCR2	0x6C04C
+#define _DPLL3_CFGCR2	0x6C054
 #define  DPLL_CFGCR2_QDIV_RATIO_MASK	(0xff<<8)
 #define  DPLL_CFGCR2_QDIV_RATIO(x)	((x)<<8)
 #define  DPLL_CFGCR2_QDIV_MODE(x)	((x)<<7)
@@ -7460,58 +7514,59 @@ enum skl_disp_power_wells {
 #define  DPLL_CFGCR2_PDIV_7 (4<<2)
 #define  DPLL_CFGCR2_CENTRAL_FREQ_MASK	(3)
 
-#define DPLL_CFGCR1(id) (DPLL1_CFGCR1 + ((id) - SKL_DPLL1) * 8)
-#define DPLL_CFGCR2(id) (DPLL1_CFGCR2 + ((id) - SKL_DPLL1) * 8)
+#define DPLL_CFGCR1(id)	_MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR1, _DPLL2_CFGCR2)
+#define DPLL_CFGCR2(id)	_MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR2, _DPLL2_CFGCR2)
 
 /* BXT display engine PLL */
-#define BXT_DE_PLL_CTL			0x6d000
+#define BXT_DE_PLL_CTL			_MMIO(0x6d000)
 #define   BXT_DE_PLL_RATIO(x)		(x)	/* {60,65,100} * 19.2MHz */
 #define   BXT_DE_PLL_RATIO_MASK		0xff
 
-#define BXT_DE_PLL_ENABLE		0x46070
+#define BXT_DE_PLL_ENABLE		_MMIO(0x46070)
 #define   BXT_DE_PLL_PLL_ENABLE		(1 << 31)
 #define   BXT_DE_PLL_LOCK		(1 << 30)
 
 /* GEN9 DC */
-#define DC_STATE_EN			0x45504
+#define DC_STATE_EN			_MMIO(0x45504)
+#define  DC_STATE_DISABLE		0
 #define  DC_STATE_EN_UPTO_DC5		(1<<0)
 #define  DC_STATE_EN_DC9		(1<<3)
 #define  DC_STATE_EN_UPTO_DC6		(2<<0)
 #define  DC_STATE_EN_UPTO_DC5_DC6_MASK   0x3
 
-#define  DC_STATE_DEBUG                  0x45520
+#define  DC_STATE_DEBUG                  _MMIO(0x45520)
 #define  DC_STATE_DEBUG_MASK_MEMORY_UP	(1<<1)
 
 /* Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register,
  * since on HSW we can't write to it using I915_WRITE. */
-#define D_COMP_HSW			(MCHBAR_MIRROR_BASE_SNB + 0x5F0C)
-#define D_COMP_BDW			0x138144
+#define D_COMP_HSW			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5F0C)
+#define D_COMP_BDW			_MMIO(0x138144)
 #define  D_COMP_RCOMP_IN_PROGRESS	(1<<9)
 #define  D_COMP_COMP_FORCE		(1<<8)
 #define  D_COMP_COMP_DISABLE		(1<<0)
 
 /* Pipe WM_LINETIME - watermark line time */
-#define PIPE_WM_LINETIME_A		0x45270
-#define PIPE_WM_LINETIME_B		0x45274
-#define PIPE_WM_LINETIME(pipe) _PIPE(pipe, PIPE_WM_LINETIME_A, \
-					   PIPE_WM_LINETIME_B)
+#define _PIPE_WM_LINETIME_A		0x45270
+#define _PIPE_WM_LINETIME_B		0x45274
+#define PIPE_WM_LINETIME(pipe) _MMIO_PIPE(pipe, _PIPE_WM_LINETIME_A, _PIPE_WM_LINETIME_B)
 #define   PIPE_WM_LINETIME_MASK			(0x1ff)
 #define   PIPE_WM_LINETIME_TIME(x)		((x))
 #define   PIPE_WM_LINETIME_IPS_LINETIME_MASK	(0x1ff<<16)
 #define   PIPE_WM_LINETIME_IPS_LINETIME(x)	((x)<<16)
 
 /* SFUSE_STRAP */
-#define SFUSE_STRAP			0xc2014
+#define SFUSE_STRAP			_MMIO(0xc2014)
 #define  SFUSE_STRAP_FUSE_LOCK		(1<<13)
 #define  SFUSE_STRAP_DISPLAY_DISABLED	(1<<7)
+#define  SFUSE_STRAP_CRT_DISABLED	(1<<6)
 #define  SFUSE_STRAP_DDIB_DETECTED	(1<<2)
 #define  SFUSE_STRAP_DDIC_DETECTED	(1<<1)
 #define  SFUSE_STRAP_DDID_DETECTED	(1<<0)
 
-#define WM_MISC				0x45260
+#define WM_MISC				_MMIO(0x45260)
 #define  WM_MISC_DATA_PARTITION_5_6	(1 << 0)
 
-#define WM_DBG				0x45280
+#define WM_DBG				_MMIO(0x45280)
 #define  WM_DBG_DISALLOW_MULTIPLE_LP	(1<<0)
 #define  WM_DBG_DISALLOW_MAXFIFO	(1<<1)
 #define  WM_DBG_DISALLOW_SPRITE		(1<<2)
@@ -7548,28 +7603,29 @@ enum skl_disp_power_wells {
 #define _PIPE_B_CSC_POSTOFF_ME	0x49144
 #define _PIPE_B_CSC_POSTOFF_LO	0x49148
 
-#define PIPE_CSC_COEFF_RY_GY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY)
-#define PIPE_CSC_COEFF_BY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY)
-#define PIPE_CSC_COEFF_RU_GU(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU)
-#define PIPE_CSC_COEFF_BU(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BU, _PIPE_B_CSC_COEFF_BU)
-#define PIPE_CSC_COEFF_RV_GV(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RV_GV, _PIPE_B_CSC_COEFF_RV_GV)
-#define PIPE_CSC_COEFF_BV(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BV, _PIPE_B_CSC_COEFF_BV)
-#define PIPE_CSC_MODE(pipe) _PIPE(pipe, _PIPE_A_CSC_MODE, _PIPE_B_CSC_MODE)
-#define PIPE_CSC_PREOFF_HI(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_HI, _PIPE_B_CSC_PREOFF_HI)
-#define PIPE_CSC_PREOFF_ME(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_ME, _PIPE_B_CSC_PREOFF_ME)
-#define PIPE_CSC_PREOFF_LO(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_LO, _PIPE_B_CSC_PREOFF_LO)
-#define PIPE_CSC_POSTOFF_HI(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_HI, _PIPE_B_CSC_POSTOFF_HI)
-#define PIPE_CSC_POSTOFF_ME(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME)
-#define PIPE_CSC_POSTOFF_LO(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO)
+#define PIPE_CSC_COEFF_RY_GY(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY)
+#define PIPE_CSC_COEFF_BY(pipe)		_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY)
+#define PIPE_CSC_COEFF_RU_GU(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU)
+#define PIPE_CSC_COEFF_BU(pipe)		_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BU, _PIPE_B_CSC_COEFF_BU)
+#define PIPE_CSC_COEFF_RV_GV(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RV_GV, _PIPE_B_CSC_COEFF_RV_GV)
+#define PIPE_CSC_COEFF_BV(pipe)		_MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BV, _PIPE_B_CSC_COEFF_BV)
+#define PIPE_CSC_MODE(pipe)		_MMIO_PIPE(pipe, _PIPE_A_CSC_MODE, _PIPE_B_CSC_MODE)
+#define PIPE_CSC_PREOFF_HI(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_HI, _PIPE_B_CSC_PREOFF_HI)
+#define PIPE_CSC_PREOFF_ME(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_ME, _PIPE_B_CSC_PREOFF_ME)
+#define PIPE_CSC_PREOFF_LO(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_LO, _PIPE_B_CSC_PREOFF_LO)
+#define PIPE_CSC_POSTOFF_HI(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_HI, _PIPE_B_CSC_POSTOFF_HI)
+#define PIPE_CSC_POSTOFF_ME(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME)
+#define PIPE_CSC_POSTOFF_LO(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO)
 
 /* MIPI DSI registers */
 
 #define _MIPI_PORT(port, a, c)	_PORT3(port, a, 0, c)	/* ports A and C only */
+#define _MMIO_MIPI(port, a, c)	_MMIO(_MIPI_PORT(port, a, c))
 
 /* BXT MIPI clock controls */
 #define BXT_MAX_VAR_OUTPUT_KHZ			39500
 
-#define BXT_MIPI_CLOCK_CTL			0x46090
+#define BXT_MIPI_CLOCK_CTL			_MMIO(0x46090)
 #define  BXT_MIPI1_DIV_SHIFT			26
 #define  BXT_MIPI2_DIV_SHIFT			10
 #define  BXT_MIPI_DIV_SHIFT(port)		\
@@ -7631,20 +7687,20 @@ enum skl_disp_power_wells {
 /* BXT MIPI mode configure */
 #define  _BXT_MIPIA_TRANS_HACTIVE			0x6B0F8
 #define  _BXT_MIPIC_TRANS_HACTIVE			0x6B8F8
-#define  BXT_MIPI_TRANS_HACTIVE(tc)	_MIPI_PORT(tc, \
+#define  BXT_MIPI_TRANS_HACTIVE(tc)	_MMIO_MIPI(tc, \
 		_BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE)
 
 #define  _BXT_MIPIA_TRANS_VACTIVE			0x6B0FC
 #define  _BXT_MIPIC_TRANS_VACTIVE			0x6B8FC
-#define  BXT_MIPI_TRANS_VACTIVE(tc)	_MIPI_PORT(tc, \
+#define  BXT_MIPI_TRANS_VACTIVE(tc)	_MMIO_MIPI(tc, \
 		_BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE)
 
 #define  _BXT_MIPIA_TRANS_VTOTAL			0x6B100
 #define  _BXT_MIPIC_TRANS_VTOTAL			0x6B900
-#define  BXT_MIPI_TRANS_VTOTAL(tc)	_MIPI_PORT(tc, \
+#define  BXT_MIPI_TRANS_VTOTAL(tc)	_MMIO_MIPI(tc, \
 		_BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL)
 
-#define BXT_DSI_PLL_CTL			0x161000
+#define BXT_DSI_PLL_CTL			_MMIO(0x161000)
 #define  BXT_DSI_PLL_PVD_RATIO_SHIFT	16
 #define  BXT_DSI_PLL_PVD_RATIO_MASK	(3 << BXT_DSI_PLL_PVD_RATIO_SHIFT)
 #define  BXT_DSI_PLL_PVD_RATIO_1	(1 << BXT_DSI_PLL_PVD_RATIO_SHIFT)
@@ -7660,21 +7716,20 @@ enum skl_disp_power_wells {
 #define BXT_DSI_PLL_RATIO_MAX		0x7D
 #define BXT_DSI_PLL_RATIO_MIN		0x22
 #define BXT_DSI_PLL_RATIO_MASK		0xFF
-#define BXT_REF_CLOCK_KHZ		19500
+#define BXT_REF_CLOCK_KHZ		19200
 
-#define BXT_DSI_PLL_ENABLE		0x46080
+#define BXT_DSI_PLL_ENABLE		_MMIO(0x46080)
 #define  BXT_DSI_PLL_DO_ENABLE		(1 << 31)
 #define  BXT_DSI_PLL_LOCKED		(1 << 30)
 
 #define _MIPIA_PORT_CTRL			(VLV_DISPLAY_BASE + 0x61190)
 #define _MIPIC_PORT_CTRL			(VLV_DISPLAY_BASE + 0x61700)
-#define MIPI_PORT_CTRL(port)	_MIPI_PORT(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL)
+#define MIPI_PORT_CTRL(port)	_MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL)
 
  /* BXT port control */
 #define _BXT_MIPIA_PORT_CTRL				0x6B0C0
 #define _BXT_MIPIC_PORT_CTRL				0x6B8C0
-#define BXT_MIPI_PORT_CTRL(tc)	_MIPI_PORT(tc, _BXT_MIPIA_PORT_CTRL, \
-						_BXT_MIPIC_PORT_CTRL)
+#define BXT_MIPI_PORT_CTRL(tc)	_MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL)
 
 #define  DPI_ENABLE					(1 << 31) /* A + C */
 #define  MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT		27
@@ -7718,8 +7773,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_TEARING_CTRL			(VLV_DISPLAY_BASE + 0x61194)
 #define _MIPIC_TEARING_CTRL			(VLV_DISPLAY_BASE + 0x61704)
-#define MIPI_TEARING_CTRL(port)			_MIPI_PORT(port, \
-				_MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL)
+#define MIPI_TEARING_CTRL(port)			_MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL)
 #define  TEARING_EFFECT_DELAY_SHIFT			0
 #define  TEARING_EFFECT_DELAY_MASK			(0xffff << 0)
 
@@ -7730,8 +7784,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_DEVICE_READY		(dev_priv->mipi_mmio_base + 0xb000)
 #define _MIPIC_DEVICE_READY		(dev_priv->mipi_mmio_base + 0xb800)
-#define MIPI_DEVICE_READY(port)		_MIPI_PORT(port, _MIPIA_DEVICE_READY, \
-						_MIPIC_DEVICE_READY)
+#define MIPI_DEVICE_READY(port)		_MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY)
 #define  BUS_POSSESSION					(1 << 3) /* set to give bus to receiver */
 #define  ULPS_STATE_MASK				(3 << 1)
 #define  ULPS_STATE_ENTER				(2 << 1)
@@ -7741,12 +7794,10 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_INTR_STAT		(dev_priv->mipi_mmio_base + 0xb004)
 #define _MIPIC_INTR_STAT		(dev_priv->mipi_mmio_base + 0xb804)
-#define MIPI_INTR_STAT(port)		_MIPI_PORT(port, _MIPIA_INTR_STAT, \
-					_MIPIC_INTR_STAT)
+#define MIPI_INTR_STAT(port)		_MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT)
 #define _MIPIA_INTR_EN			(dev_priv->mipi_mmio_base + 0xb008)
 #define _MIPIC_INTR_EN			(dev_priv->mipi_mmio_base + 0xb808)
-#define MIPI_INTR_EN(port)		_MIPI_PORT(port, _MIPIA_INTR_EN, \
-					_MIPIC_INTR_EN)
+#define MIPI_INTR_EN(port)		_MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN)
 #define  TEARING_EFFECT					(1 << 31)
 #define  SPL_PKT_SENT_INTERRUPT				(1 << 30)
 #define  GEN_READ_DATA_AVAIL				(1 << 29)
@@ -7782,8 +7833,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_DSI_FUNC_PRG		(dev_priv->mipi_mmio_base + 0xb00c)
 #define _MIPIC_DSI_FUNC_PRG		(dev_priv->mipi_mmio_base + 0xb80c)
-#define MIPI_DSI_FUNC_PRG(port)		_MIPI_PORT(port, _MIPIA_DSI_FUNC_PRG, \
-						_MIPIC_DSI_FUNC_PRG)
+#define MIPI_DSI_FUNC_PRG(port)		_MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG)
 #define  CMD_MODE_DATA_WIDTH_MASK			(7 << 13)
 #define  CMD_MODE_NOT_SUPPORTED				(0 << 13)
 #define  CMD_MODE_DATA_WIDTH_16_BIT			(1 << 13)
@@ -7806,32 +7856,27 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_HS_TX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb010)
 #define _MIPIC_HS_TX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb810)
-#define MIPI_HS_TX_TIMEOUT(port)	_MIPI_PORT(port, _MIPIA_HS_TX_TIMEOUT, \
-					_MIPIC_HS_TX_TIMEOUT)
+#define MIPI_HS_TX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT)
 #define  HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK		0xffffff
 
 #define _MIPIA_LP_RX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb014)
 #define _MIPIC_LP_RX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb814)
-#define MIPI_LP_RX_TIMEOUT(port)	_MIPI_PORT(port, _MIPIA_LP_RX_TIMEOUT, \
-					_MIPIC_LP_RX_TIMEOUT)
+#define MIPI_LP_RX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT)
 #define  LOW_POWER_RX_TIMEOUT_COUNTER_MASK		0xffffff
 
 #define _MIPIA_TURN_AROUND_TIMEOUT	(dev_priv->mipi_mmio_base + 0xb018)
 #define _MIPIC_TURN_AROUND_TIMEOUT	(dev_priv->mipi_mmio_base + 0xb818)
-#define MIPI_TURN_AROUND_TIMEOUT(port)	_MIPI_PORT(port, \
-			_MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT)
+#define MIPI_TURN_AROUND_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT)
 #define  TURN_AROUND_TIMEOUT_MASK			0x3f
 
 #define _MIPIA_DEVICE_RESET_TIMER	(dev_priv->mipi_mmio_base + 0xb01c)
 #define _MIPIC_DEVICE_RESET_TIMER	(dev_priv->mipi_mmio_base + 0xb81c)
-#define MIPI_DEVICE_RESET_TIMER(port)	_MIPI_PORT(port, \
-			_MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER)
+#define MIPI_DEVICE_RESET_TIMER(port)	_MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER)
 #define  DEVICE_RESET_TIMER_MASK			0xffff
 
 #define _MIPIA_DPI_RESOLUTION		(dev_priv->mipi_mmio_base + 0xb020)
 #define _MIPIC_DPI_RESOLUTION		(dev_priv->mipi_mmio_base + 0xb820)
-#define MIPI_DPI_RESOLUTION(port)	_MIPI_PORT(port, _MIPIA_DPI_RESOLUTION, \
-					_MIPIC_DPI_RESOLUTION)
+#define MIPI_DPI_RESOLUTION(port)	_MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION)
 #define  VERTICAL_ADDRESS_SHIFT				16
 #define  VERTICAL_ADDRESS_MASK				(0xffff << 16)
 #define  HORIZONTAL_ADDRESS_SHIFT			0
@@ -7839,8 +7884,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_DBI_FIFO_THROTTLE	(dev_priv->mipi_mmio_base + 0xb024)
 #define _MIPIC_DBI_FIFO_THROTTLE	(dev_priv->mipi_mmio_base + 0xb824)
-#define MIPI_DBI_FIFO_THROTTLE(port)	_MIPI_PORT(port, \
-			_MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE)
+#define MIPI_DBI_FIFO_THROTTLE(port)	_MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE)
 #define  DBI_FIFO_EMPTY_HALF				(0 << 0)
 #define  DBI_FIFO_EMPTY_QUARTER				(1 << 0)
 #define  DBI_FIFO_EMPTY_7_LOCATIONS			(2 << 0)
@@ -7848,50 +7892,41 @@ enum skl_disp_power_wells {
 /* regs below are bits 15:0 */
 #define _MIPIA_HSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb028)
 #define _MIPIC_HSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb828)
-#define MIPI_HSYNC_PADDING_COUNT(port)	_MIPI_PORT(port, \
-			_MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT)
+#define MIPI_HSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT)
 
 #define _MIPIA_HBP_COUNT		(dev_priv->mipi_mmio_base + 0xb02c)
 #define _MIPIC_HBP_COUNT		(dev_priv->mipi_mmio_base + 0xb82c)
-#define MIPI_HBP_COUNT(port)		_MIPI_PORT(port, _MIPIA_HBP_COUNT, \
-					_MIPIC_HBP_COUNT)
+#define MIPI_HBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT)
 
 #define _MIPIA_HFP_COUNT		(dev_priv->mipi_mmio_base + 0xb030)
 #define _MIPIC_HFP_COUNT		(dev_priv->mipi_mmio_base + 0xb830)
-#define MIPI_HFP_COUNT(port)		_MIPI_PORT(port, _MIPIA_HFP_COUNT, \
-					_MIPIC_HFP_COUNT)
+#define MIPI_HFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT)
 
 #define _MIPIA_HACTIVE_AREA_COUNT	(dev_priv->mipi_mmio_base + 0xb034)
 #define _MIPIC_HACTIVE_AREA_COUNT	(dev_priv->mipi_mmio_base + 0xb834)
-#define MIPI_HACTIVE_AREA_COUNT(port)	_MIPI_PORT(port, \
-			_MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT)
+#define MIPI_HACTIVE_AREA_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT)
 
 #define _MIPIA_VSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb038)
 #define _MIPIC_VSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb838)
-#define MIPI_VSYNC_PADDING_COUNT(port)	_MIPI_PORT(port, \
-			_MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT)
+#define MIPI_VSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT)
 
 #define _MIPIA_VBP_COUNT		(dev_priv->mipi_mmio_base + 0xb03c)
 #define _MIPIC_VBP_COUNT		(dev_priv->mipi_mmio_base + 0xb83c)
-#define MIPI_VBP_COUNT(port)		_MIPI_PORT(port, _MIPIA_VBP_COUNT, \
-					_MIPIC_VBP_COUNT)
+#define MIPI_VBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT)
 
 #define _MIPIA_VFP_COUNT		(dev_priv->mipi_mmio_base + 0xb040)
 #define _MIPIC_VFP_COUNT		(dev_priv->mipi_mmio_base + 0xb840)
-#define MIPI_VFP_COUNT(port)		_MIPI_PORT(port, _MIPIA_VFP_COUNT, \
-					_MIPIC_VFP_COUNT)
+#define MIPI_VFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT)
 
 #define _MIPIA_HIGH_LOW_SWITCH_COUNT	(dev_priv->mipi_mmio_base + 0xb044)
 #define _MIPIC_HIGH_LOW_SWITCH_COUNT	(dev_priv->mipi_mmio_base + 0xb844)
-#define MIPI_HIGH_LOW_SWITCH_COUNT(port)	_MIPI_PORT(port,	\
-		_MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT)
+#define MIPI_HIGH_LOW_SWITCH_COUNT(port)	_MMIO_MIPI(port,	_MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT)
 
 /* regs above are bits 15:0 */
 
 #define _MIPIA_DPI_CONTROL		(dev_priv->mipi_mmio_base + 0xb048)
 #define _MIPIC_DPI_CONTROL		(dev_priv->mipi_mmio_base + 0xb848)
-#define MIPI_DPI_CONTROL(port)		_MIPI_PORT(port, _MIPIA_DPI_CONTROL, \
-					_MIPIC_DPI_CONTROL)
+#define MIPI_DPI_CONTROL(port)		_MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL)
 #define  DPI_LP_MODE					(1 << 6)
 #define  BACKLIGHT_OFF					(1 << 5)
 #define  BACKLIGHT_ON					(1 << 4)
@@ -7902,29 +7937,26 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_DPI_DATA			(dev_priv->mipi_mmio_base + 0xb04c)
 #define _MIPIC_DPI_DATA			(dev_priv->mipi_mmio_base + 0xb84c)
-#define MIPI_DPI_DATA(port)		_MIPI_PORT(port, _MIPIA_DPI_DATA, \
-					_MIPIC_DPI_DATA)
+#define MIPI_DPI_DATA(port)		_MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA)
 #define  COMMAND_BYTE_SHIFT				0
 #define  COMMAND_BYTE_MASK				(0x3f << 0)
 
 #define _MIPIA_INIT_COUNT		(dev_priv->mipi_mmio_base + 0xb050)
 #define _MIPIC_INIT_COUNT		(dev_priv->mipi_mmio_base + 0xb850)
-#define MIPI_INIT_COUNT(port)		_MIPI_PORT(port, _MIPIA_INIT_COUNT, \
-					_MIPIC_INIT_COUNT)
+#define MIPI_INIT_COUNT(port)		_MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT)
 #define  MASTER_INIT_TIMER_SHIFT			0
 #define  MASTER_INIT_TIMER_MASK				(0xffff << 0)
 
 #define _MIPIA_MAX_RETURN_PKT_SIZE	(dev_priv->mipi_mmio_base + 0xb054)
 #define _MIPIC_MAX_RETURN_PKT_SIZE	(dev_priv->mipi_mmio_base + 0xb854)
-#define MIPI_MAX_RETURN_PKT_SIZE(port)	_MIPI_PORT(port, \
+#define MIPI_MAX_RETURN_PKT_SIZE(port)	_MMIO_MIPI(port, \
 			_MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE)
 #define  MAX_RETURN_PKT_SIZE_SHIFT			0
 #define  MAX_RETURN_PKT_SIZE_MASK			(0x3ff << 0)
 
 #define _MIPIA_VIDEO_MODE_FORMAT	(dev_priv->mipi_mmio_base + 0xb058)
 #define _MIPIC_VIDEO_MODE_FORMAT	(dev_priv->mipi_mmio_base + 0xb858)
-#define MIPI_VIDEO_MODE_FORMAT(port)	_MIPI_PORT(port, \
-			_MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT)
+#define MIPI_VIDEO_MODE_FORMAT(port)	_MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT)
 #define  RANDOM_DPI_DISPLAY_RESOLUTION			(1 << 4)
 #define  DISABLE_VIDEO_BTA				(1 << 3)
 #define  IP_TG_CONFIG					(1 << 2)
@@ -7934,8 +7966,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_EOT_DISABLE		(dev_priv->mipi_mmio_base + 0xb05c)
 #define _MIPIC_EOT_DISABLE		(dev_priv->mipi_mmio_base + 0xb85c)
-#define MIPI_EOT_DISABLE(port)		_MIPI_PORT(port, _MIPIA_EOT_DISABLE, \
-					_MIPIC_EOT_DISABLE)
+#define MIPI_EOT_DISABLE(port)		_MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE)
 #define  LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE		(1 << 7)
 #define  HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE		(1 << 6)
 #define  LOW_CONTENTION_RECOVERY_DISABLE		(1 << 5)
@@ -7947,31 +7978,26 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_LP_BYTECLK		(dev_priv->mipi_mmio_base + 0xb060)
 #define _MIPIC_LP_BYTECLK		(dev_priv->mipi_mmio_base + 0xb860)
-#define MIPI_LP_BYTECLK(port)		_MIPI_PORT(port, _MIPIA_LP_BYTECLK, \
-					_MIPIC_LP_BYTECLK)
+#define MIPI_LP_BYTECLK(port)		_MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK)
 #define  LP_BYTECLK_SHIFT				0
 #define  LP_BYTECLK_MASK				(0xffff << 0)
 
 /* bits 31:0 */
 #define _MIPIA_LP_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb064)
 #define _MIPIC_LP_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb864)
-#define MIPI_LP_GEN_DATA(port)		_MIPI_PORT(port, _MIPIA_LP_GEN_DATA, \
-					_MIPIC_LP_GEN_DATA)
+#define MIPI_LP_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA)
 
 /* bits 31:0 */
 #define _MIPIA_HS_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb068)
 #define _MIPIC_HS_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb868)
-#define MIPI_HS_GEN_DATA(port)		_MIPI_PORT(port, _MIPIA_HS_GEN_DATA, \
-					_MIPIC_HS_GEN_DATA)
+#define MIPI_HS_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA)
 
 #define _MIPIA_LP_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb06c)
 #define _MIPIC_LP_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb86c)
-#define MIPI_LP_GEN_CTRL(port)		_MIPI_PORT(port, _MIPIA_LP_GEN_CTRL, \
-					_MIPIC_LP_GEN_CTRL)
+#define MIPI_LP_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL)
 #define _MIPIA_HS_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb070)
 #define _MIPIC_HS_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb870)
-#define MIPI_HS_GEN_CTRL(port)		_MIPI_PORT(port, _MIPIA_HS_GEN_CTRL, \
-					_MIPIC_HS_GEN_CTRL)
+#define MIPI_HS_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL)
 #define  LONG_PACKET_WORD_COUNT_SHIFT			8
 #define  LONG_PACKET_WORD_COUNT_MASK			(0xffff << 8)
 #define  SHORT_PACKET_PARAM_SHIFT			8
@@ -7984,8 +8010,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_GEN_FIFO_STAT		(dev_priv->mipi_mmio_base + 0xb074)
 #define _MIPIC_GEN_FIFO_STAT		(dev_priv->mipi_mmio_base + 0xb874)
-#define MIPI_GEN_FIFO_STAT(port)	_MIPI_PORT(port, _MIPIA_GEN_FIFO_STAT, \
-					_MIPIC_GEN_FIFO_STAT)
+#define MIPI_GEN_FIFO_STAT(port)	_MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT)
 #define  DPI_FIFO_EMPTY					(1 << 28)
 #define  DBI_FIFO_EMPTY					(1 << 27)
 #define  LP_CTRL_FIFO_EMPTY				(1 << 26)
@@ -8003,16 +8028,14 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_HS_LS_DBI_ENABLE		(dev_priv->mipi_mmio_base + 0xb078)
 #define _MIPIC_HS_LS_DBI_ENABLE		(dev_priv->mipi_mmio_base + 0xb878)
-#define MIPI_HS_LP_DBI_ENABLE(port)	_MIPI_PORT(port, \
-			_MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE)
+#define MIPI_HS_LP_DBI_ENABLE(port)	_MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE)
 #define  DBI_HS_LP_MODE_MASK				(1 << 0)
 #define  DBI_LP_MODE					(1 << 0)
 #define  DBI_HS_MODE					(0 << 0)
 
 #define _MIPIA_DPHY_PARAM		(dev_priv->mipi_mmio_base + 0xb080)
 #define _MIPIC_DPHY_PARAM		(dev_priv->mipi_mmio_base + 0xb880)
-#define MIPI_DPHY_PARAM(port)		_MIPI_PORT(port, _MIPIA_DPHY_PARAM, \
-					_MIPIC_DPHY_PARAM)
+#define MIPI_DPHY_PARAM(port)		_MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM)
 #define  EXIT_ZERO_COUNT_SHIFT				24
 #define  EXIT_ZERO_COUNT_MASK				(0x3f << 24)
 #define  TRAIL_COUNT_SHIFT				16
@@ -8025,15 +8048,11 @@ enum skl_disp_power_wells {
 /* bits 31:0 */
 #define _MIPIA_DBI_BW_CTRL		(dev_priv->mipi_mmio_base + 0xb084)
 #define _MIPIC_DBI_BW_CTRL		(dev_priv->mipi_mmio_base + 0xb884)
-#define MIPI_DBI_BW_CTRL(port)		_MIPI_PORT(port, _MIPIA_DBI_BW_CTRL, \
-					_MIPIC_DBI_BW_CTRL)
-
-#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base \
-							+ 0xb088)
-#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base \
-							+ 0xb888)
-#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port)	_MIPI_PORT(port, \
-	_MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT)
+#define MIPI_DBI_BW_CTRL(port)		_MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL)
+
+#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base + 0xb088)
+#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base + 0xb888)
+#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port)	_MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT)
 #define  LP_HS_SSW_CNT_SHIFT				16
 #define  LP_HS_SSW_CNT_MASK				(0xffff << 16)
 #define  HS_LP_PWR_SW_CNT_SHIFT				0
@@ -8041,19 +8060,16 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_STOP_STATE_STALL		(dev_priv->mipi_mmio_base + 0xb08c)
 #define _MIPIC_STOP_STATE_STALL		(dev_priv->mipi_mmio_base + 0xb88c)
-#define MIPI_STOP_STATE_STALL(port)	_MIPI_PORT(port, \
-			_MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL)
+#define MIPI_STOP_STATE_STALL(port)	_MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL)
 #define  STOP_STATE_STALL_COUNTER_SHIFT			0
 #define  STOP_STATE_STALL_COUNTER_MASK			(0xff << 0)
 
 #define _MIPIA_INTR_STAT_REG_1		(dev_priv->mipi_mmio_base + 0xb090)
 #define _MIPIC_INTR_STAT_REG_1		(dev_priv->mipi_mmio_base + 0xb890)
-#define MIPI_INTR_STAT_REG_1(port)	_MIPI_PORT(port, \
-				_MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1)
+#define MIPI_INTR_STAT_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1)
 #define _MIPIA_INTR_EN_REG_1		(dev_priv->mipi_mmio_base + 0xb094)
 #define _MIPIC_INTR_EN_REG_1		(dev_priv->mipi_mmio_base + 0xb894)
-#define MIPI_INTR_EN_REG_1(port)	_MIPI_PORT(port, _MIPIA_INTR_EN_REG_1, \
-					_MIPIC_INTR_EN_REG_1)
+#define MIPI_INTR_EN_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1)
 #define  RX_CONTENTION_DETECTED				(1 << 0)
 
 /* XXX: only pipe A ?!? */
@@ -8073,8 +8089,7 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_CTRL			(dev_priv->mipi_mmio_base + 0xb104)
 #define _MIPIC_CTRL			(dev_priv->mipi_mmio_base + 0xb904)
-#define MIPI_CTRL(port)			_MIPI_PORT(port, _MIPIA_CTRL, \
-					_MIPIC_CTRL)
+#define MIPI_CTRL(port)			_MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL)
 #define  ESCAPE_CLOCK_DIVIDER_SHIFT			5 /* A only */
 #define  ESCAPE_CLOCK_DIVIDER_MASK			(3 << 5)
 #define  ESCAPE_CLOCK_DIVIDER_1				(0 << 5)
@@ -8087,29 +8102,24 @@ enum skl_disp_power_wells {
 #define  RGB_FLIP_TO_BGR				(1 << 2)
 
 #define  BXT_PIPE_SELECT_MASK				(7 << 7)
-#define  BXT_PIPE_SELECT_C				(2 << 7)
-#define  BXT_PIPE_SELECT_B				(1 << 7)
-#define  BXT_PIPE_SELECT_A				(0 << 7)
+#define  BXT_PIPE_SELECT(pipe)				((pipe) << 7)
 
 #define _MIPIA_DATA_ADDRESS		(dev_priv->mipi_mmio_base + 0xb108)
 #define _MIPIC_DATA_ADDRESS		(dev_priv->mipi_mmio_base + 0xb908)
-#define MIPI_DATA_ADDRESS(port)		_MIPI_PORT(port, _MIPIA_DATA_ADDRESS, \
-					_MIPIC_DATA_ADDRESS)
+#define MIPI_DATA_ADDRESS(port)		_MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS)
 #define  DATA_MEM_ADDRESS_SHIFT				5
 #define  DATA_MEM_ADDRESS_MASK				(0x7ffffff << 5)
 #define  DATA_VALID					(1 << 0)
 
 #define _MIPIA_DATA_LENGTH		(dev_priv->mipi_mmio_base + 0xb10c)
 #define _MIPIC_DATA_LENGTH		(dev_priv->mipi_mmio_base + 0xb90c)
-#define MIPI_DATA_LENGTH(port)		_MIPI_PORT(port, _MIPIA_DATA_LENGTH, \
-					_MIPIC_DATA_LENGTH)
+#define MIPI_DATA_LENGTH(port)		_MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH)
 #define  DATA_LENGTH_SHIFT				0
 #define  DATA_LENGTH_MASK				(0xfffff << 0)
 
 #define _MIPIA_COMMAND_ADDRESS		(dev_priv->mipi_mmio_base + 0xb110)
 #define _MIPIC_COMMAND_ADDRESS		(dev_priv->mipi_mmio_base + 0xb910)
-#define MIPI_COMMAND_ADDRESS(port)	_MIPI_PORT(port, \
-				_MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS)
+#define MIPI_COMMAND_ADDRESS(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS)
 #define  COMMAND_MEM_ADDRESS_SHIFT			5
 #define  COMMAND_MEM_ADDRESS_MASK			(0x7ffffff << 5)
 #define  AUTO_PWG_ENABLE				(1 << 2)
@@ -8118,21 +8128,17 @@ enum skl_disp_power_wells {
 
 #define _MIPIA_COMMAND_LENGTH		(dev_priv->mipi_mmio_base + 0xb114)
 #define _MIPIC_COMMAND_LENGTH		(dev_priv->mipi_mmio_base + 0xb914)
-#define MIPI_COMMAND_LENGTH(port)	_MIPI_PORT(port, _MIPIA_COMMAND_LENGTH, \
-					_MIPIC_COMMAND_LENGTH)
+#define MIPI_COMMAND_LENGTH(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH)
 #define  COMMAND_LENGTH_SHIFT(n)			(8 * (n)) /* n: 0...3 */
 #define  COMMAND_LENGTH_MASK(n)				(0xff << (8 * (n)))
 
 #define _MIPIA_READ_DATA_RETURN0	(dev_priv->mipi_mmio_base + 0xb118)
 #define _MIPIC_READ_DATA_RETURN0	(dev_priv->mipi_mmio_base + 0xb918)
-#define MIPI_READ_DATA_RETURN(port, n) \
-	(_MIPI_PORT(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) \
-					+ 4 * (n)) /* n: 0...7 */
+#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */
 
 #define _MIPIA_READ_DATA_VALID		(dev_priv->mipi_mmio_base + 0xb138)
 #define _MIPIC_READ_DATA_VALID		(dev_priv->mipi_mmio_base + 0xb938)
-#define MIPI_READ_DATA_VALID(port)	_MIPI_PORT(port, \
-				_MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID)
+#define MIPI_READ_DATA_VALID(port)	_MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID)
 #define  READ_DATA_VALID(n)				(1 << (n))
 
 /* For UMS only (deprecated): */
@@ -8140,12 +8146,12 @@ enum skl_disp_power_wells {
 #define _PALETTE_B (dev_priv->info.display_mmio_offset + 0xa800)
 
 /* MOCS (Memory Object Control State) registers */
-#define GEN9_LNCFCMOCS0		0xb020	/* L3 Cache Control base */
+#define GEN9_LNCFCMOCS(i)	_MMIO(0xb020 + (i) * 4)	/* L3 Cache Control */
 
-#define GEN9_GFX_MOCS_0		0xc800	/* Graphics MOCS base register*/
-#define GEN9_MFX0_MOCS_0	0xc900	/* Media 0 MOCS base register*/
-#define GEN9_MFX1_MOCS_0	0xca00	/* Media 1 MOCS base register*/
-#define GEN9_VEBOX_MOCS_0	0xcb00	/* Video MOCS base register*/
-#define GEN9_BLT_MOCS_0		0xcc00	/* Blitter MOCS base register*/
+#define GEN9_GFX_MOCS(i)	_MMIO(0xc800 + (i) * 4)	/* Graphics MOCS registers */
+#define GEN9_MFX0_MOCS(i)	_MMIO(0xc900 + (i) * 4)	/* Media 0 MOCS registers */
+#define GEN9_MFX1_MOCS(i)	_MMIO(0xca00 + (i) * 4)	/* Media 1 MOCS registers */
+#define GEN9_VEBOX_MOCS(i)	_MMIO(0xcb00 + (i) * 4)	/* Video MOCS registers */
+#define GEN9_BLT_MOCS(i)	_MMIO(0xcc00 + (i) * 4)	/* Blitter MOCS registers */
 
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 2d9182189422..a2aa09ce3202 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -49,7 +49,7 @@ static void i915_save_display(struct drm_device *dev)
 		dev_priv->regfile.savePP_ON_DELAYS = I915_READ(PCH_PP_ON_DELAYS);
 		dev_priv->regfile.savePP_OFF_DELAYS = I915_READ(PCH_PP_OFF_DELAYS);
 		dev_priv->regfile.savePP_DIVISOR = I915_READ(PCH_PP_DIVISOR);
-	} else if (!IS_VALLEYVIEW(dev)) {
+	} else if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 		dev_priv->regfile.savePP_CONTROL = I915_READ(PP_CONTROL);
 		dev_priv->regfile.savePP_ON_DELAYS = I915_READ(PP_ON_DELAYS);
 		dev_priv->regfile.savePP_OFF_DELAYS = I915_READ(PP_OFF_DELAYS);
@@ -84,7 +84,7 @@ static void i915_restore_display(struct drm_device *dev)
 		I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
 		I915_WRITE(PCH_PP_DIVISOR, dev_priv->regfile.savePP_DIVISOR);
 		I915_WRITE(PCH_PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
-	} else if (!IS_VALLEYVIEW(dev)) {
+	} else if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 		I915_WRITE(PP_ON_DELAYS, dev_priv->regfile.savePP_ON_DELAYS);
 		I915_WRITE(PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
 		I915_WRITE(PP_DIVISOR, dev_priv->regfile.savePP_DIVISOR);
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 50ce9ce2b269..37e3f0ddf8e0 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -35,7 +35,8 @@
 #define dev_to_drm_minor(d) dev_get_drvdata((d))
 
 #ifdef CONFIG_PM
-static u32 calc_residency(struct drm_device *dev, const u32 reg)
+static u32 calc_residency(struct drm_device *dev,
+			  i915_reg_t reg)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u64 raw_time; /* 32b value may overflow during fixed point math */
@@ -48,7 +49,7 @@ static u32 calc_residency(struct drm_device *dev, const u32 reg)
 	intel_runtime_pm_get(dev_priv);
 
 	/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		units = 1;
 		div = dev_priv->czclk_freq;
 
@@ -283,7 +284,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 	intel_runtime_pm_get(dev_priv);
 
 	mutex_lock(&dev_priv->rps.hw_lock);
-	if (IS_VALLEYVIEW(dev_priv->dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		u32 freq;
 		freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
 		ret = intel_gpu_freq(dev_priv, (freq >> 8) & 0xff);
@@ -597,7 +598,7 @@ void i915_setup_sysfs(struct drm_device *dev)
 		if (ret)
 			DRM_ERROR("RC6p residency sysfs setup failed\n");
 	}
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		ret = sysfs_merge_group(&dev->primary->kdev->kobj,
 					&media_rc6_attr_group);
 		if (ret)
@@ -618,7 +619,7 @@ void i915_setup_sysfs(struct drm_device *dev)
 	}
 
 	ret = 0;
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		ret = sysfs_create_files(&dev->primary->kdev->kobj, vlv_attrs);
 	else if (INTEL_INFO(dev)->gen >= 6)
 		ret = sysfs_create_files(&dev->primary->kdev->kobj, gen6_attrs);
@@ -634,7 +635,7 @@ void i915_setup_sysfs(struct drm_device *dev)
 void i915_teardown_sysfs(struct drm_device *dev)
 {
 	sysfs_remove_bin_file(&dev->primary->kdev->kobj, &error_state_attr);
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		sysfs_remove_files(&dev->primary->kdev->kobj, vlv_attrs);
 	else
 		sysfs_remove_files(&dev->primary->kdev->kobj, gen6_attrs);
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 04fe8491c8b6..52b2d409945d 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -664,7 +664,7 @@ TRACE_EVENT(i915_flip_complete,
 );
 
 TRACE_EVENT_CONDITION(i915_reg_rw,
-	TP_PROTO(bool write, u32 reg, u64 val, int len, bool trace),
+	TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace),
 
 	TP_ARGS(write, reg, val, len, trace),
 
@@ -679,7 +679,7 @@ TRACE_EVENT_CONDITION(i915_reg_rw,
 
 	TP_fast_assign(
 		__entry->val = (u64)val;
-		__entry->reg = reg;
+		__entry->reg = i915_mmio_reg_offset(reg);
 		__entry->write = write;
 		__entry->len = len;
 		),
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index 5eee75bff170..dea7429be4d0 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -69,13 +69,13 @@ void i915_check_vgpu(struct drm_device *dev)
 	if (!IS_HASWELL(dev))
 		return;
 
-	magic = readq(dev_priv->regs + vgtif_reg(magic));
+	magic = __raw_i915_read64(dev_priv, vgtif_reg(magic));
 	if (magic != VGT_MAGIC)
 		return;
 
 	version = INTEL_VGT_IF_VERSION_ENCODE(
-		readw(dev_priv->regs + vgtif_reg(version_major)),
-		readw(dev_priv->regs + vgtif_reg(version_minor)));
+		__raw_i915_read16(dev_priv, vgtif_reg(version_major)),
+		__raw_i915_read16(dev_priv, vgtif_reg(version_minor)));
 	if (version != INTEL_VGT_IF_VERSION) {
 		DRM_INFO("VGT interface version mismatch!\n");
 		return;
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 21c97f44d637..3c83b47b5f69 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -92,14 +92,10 @@ struct vgt_if {
 	uint32_t g2v_notify;
 	uint32_t rsv6[7];
 
-	uint32_t pdp0_lo;
-	uint32_t pdp0_hi;
-	uint32_t pdp1_lo;
-	uint32_t pdp1_hi;
-	uint32_t pdp2_lo;
-	uint32_t pdp2_hi;
-	uint32_t pdp3_lo;
-	uint32_t pdp3_hi;
+	struct {
+		uint32_t lo;
+		uint32_t hi;
+	} pdp[4];
 
 	uint32_t execlist_context_descriptor_lo;
 	uint32_t execlist_context_descriptor_hi;
@@ -108,7 +104,7 @@ struct vgt_if {
 } __packed;
 
 #define vgtif_reg(x) \
-	(VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x)
+	_MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
 
 /* vGPU display status to be used by the host side */
 #define VGT_DRV_DISPLAY_NOT_READY 0
diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c
index f1975f267710..d0b1c9afa35e 100644
--- a/drivers/gpu/drm/i915/intel_atomic.c
+++ b/drivers/gpu/drm/i915/intel_atomic.c
@@ -94,6 +94,9 @@ intel_crtc_duplicate_state(struct drm_crtc *crtc)
 	__drm_atomic_helper_crtc_duplicate_state(crtc, &crtc_state->base);
 
 	crtc_state->update_pipe = false;
+	crtc_state->disable_lp_wm = false;
+	crtc_state->disable_cxsr = false;
+	crtc_state->wm_changed = false;
 
 	return &crtc_state->base;
 }
@@ -205,8 +208,6 @@ int intel_atomic_setup_scalers(struct drm_device *dev,
 				 * but since this plane is unchanged just do the
 				 * minimum required validation.
 				 */
-				if (plane->type == DRM_PLANE_TYPE_PRIMARY)
-					intel_crtc->atomic.wait_for_flips = true;
 				crtc_state->base.planes_changed = true;
 			}
 
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index a11980696595..c6bb0fc1edfb 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -84,6 +84,7 @@ intel_plane_duplicate_state(struct drm_plane *plane)
 	state = &intel_state->base;
 
 	__drm_atomic_helper_plane_duplicate_state(plane, state);
+	intel_state->wait_req = NULL;
 
 	return state;
 }
@@ -100,6 +101,7 @@ void
 intel_plane_destroy_state(struct drm_plane *plane,
 			  struct drm_plane_state *state)
 {
+	WARN_ON(state && to_intel_plane_state(state)->wait_req);
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 4dccd9b003a1..31f6d212fb1b 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -161,9 +161,9 @@ static bool audio_rate_need_prog(struct intel_crtc *crtc,
 }
 
 static bool intel_eld_uptodate(struct drm_connector *connector,
-			       int reg_eldv, uint32_t bits_eldv,
-			       int reg_elda, uint32_t bits_elda,
-			       int reg_edid)
+			       i915_reg_t reg_eldv, uint32_t bits_eldv,
+			       i915_reg_t reg_elda, uint32_t bits_elda,
+			       i915_reg_t reg_edid)
 {
 	struct drm_i915_private *dev_priv = connector->dev->dev_private;
 	uint8_t *eld = connector->eld;
@@ -262,7 +262,8 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 	tmp |= AUD_CONFIG_N_PROG_ENABLE;
 	tmp &= ~AUD_CONFIG_UPPER_N_MASK;
 	tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-	if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+	if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT) ||
+	    intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DP_MST))
 		tmp |= AUD_CONFIG_N_VALUE_INDEX;
 	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
 
@@ -364,8 +365,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 	enum port port = intel_dig_port->port;
 	enum pipe pipe = intel_crtc->pipe;
 	uint32_t tmp, eldv;
-	int aud_config;
-	int aud_cntrl_st2;
+	i915_reg_t aud_config, aud_cntrl_st2;
 
 	DRM_DEBUG_KMS("Disable audio codec on port %c, pipe %c\n",
 		      port_name(port), pipe_name(pipe));
@@ -376,7 +376,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 	if (HAS_PCH_IBX(dev_priv->dev)) {
 		aud_config = IBX_AUD_CFG(pipe);
 		aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
-	} else if (IS_VALLEYVIEW(dev_priv)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		aud_config = VLV_AUD_CFG(pipe);
 		aud_cntrl_st2 = VLV_AUD_CNTL_ST2;
 	} else {
@@ -416,10 +416,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 	uint32_t eldv;
 	uint32_t tmp;
 	int len, i;
-	int hdmiw_hdmiedid;
-	int aud_config;
-	int aud_cntl_st;
-	int aud_cntrl_st2;
+	i915_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2;
 
 	DRM_DEBUG_KMS("Enable audio codec on port %c, pipe %c, %u bytes ELD\n",
 		      port_name(port), pipe_name(pipe), drm_eld_size(eld));
@@ -439,7 +436,8 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 		aud_config = IBX_AUD_CFG(pipe);
 		aud_cntl_st = IBX_AUD_CNTL_ST(pipe);
 		aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
-	} else if (IS_VALLEYVIEW(connector->dev)) {
+	} else if (IS_VALLEYVIEW(connector->dev) ||
+		   IS_CHERRYVIEW(connector->dev)) {
 		hdmiw_hdmiedid = VLV_HDMIW_HDMIEDID(pipe);
 		aud_config = VLV_AUD_CFG(pipe);
 		aud_cntl_st = VLV_AUD_CNTL_ST(pipe);
@@ -478,7 +476,8 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
 	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
 	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-	if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+	if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT) ||
+	    intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DP_MST))
 		tmp |= AUD_CONFIG_N_VALUE_INDEX;
 	else
 		tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
@@ -516,7 +515,8 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 
 	/* ELD Conn_Type */
 	connector->eld[5] &= ~(3 << 2);
-	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
+	    intel_pipe_has_type(crtc, INTEL_OUTPUT_DP_MST))
 		connector->eld[5] |= (1 << 2);
 
 	connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
@@ -525,6 +525,10 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 		dev_priv->display.audio_codec_enable(connector, intel_encoder,
 						     adjusted_mode);
 
+	mutex_lock(&dev_priv->av_mutex);
+	intel_dig_port->audio_connector = connector;
+	mutex_unlock(&dev_priv->av_mutex);
+
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
 		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
 }
@@ -548,6 +552,10 @@ void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
 	if (dev_priv->display.audio_codec_disable)
 		dev_priv->display.audio_codec_disable(intel_encoder);
 
+	mutex_lock(&dev_priv->av_mutex);
+	intel_dig_port->audio_connector = NULL;
+	mutex_unlock(&dev_priv->av_mutex);
+
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
 		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
 }
@@ -563,7 +571,7 @@ void intel_init_audio(struct drm_device *dev)
 	if (IS_G4X(dev)) {
 		dev_priv->display.audio_codec_enable = g4x_audio_codec_enable;
 		dev_priv->display.audio_codec_disable = g4x_audio_codec_disable;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		dev_priv->display.audio_codec_enable = ilk_audio_codec_enable;
 		dev_priv->display.audio_codec_disable = ilk_audio_codec_disable;
 	} else if (IS_HASWELL(dev) || INTEL_INFO(dev)->gen >= 8) {
@@ -591,7 +599,7 @@ static void i915_audio_component_codec_wake_override(struct device *dev,
 	struct drm_i915_private *dev_priv = dev_to_i915(dev);
 	u32 tmp;
 
-	if (!IS_SKYLAKE(dev_priv))
+	if (!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv))
 		return;
 
 	/*
@@ -632,44 +640,40 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 						int port, int rate)
 {
 	struct drm_i915_private *dev_priv = dev_to_i915(dev);
-	struct drm_device *drm_dev = dev_priv->dev;
 	struct intel_encoder *intel_encoder;
-	struct intel_digital_port *intel_dig_port;
 	struct intel_crtc *crtc;
 	struct drm_display_mode *mode;
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	enum pipe pipe = -1;
+	enum pipe pipe = INVALID_PIPE;
 	u32 tmp;
 	int n;
+	int err = 0;
 
-	/* HSW, BDW SKL need this fix */
+	/* HSW, BDW, SKL, KBL need this fix */
 	if (!IS_SKYLAKE(dev_priv) &&
-		!IS_BROADWELL(dev_priv) &&
-		!IS_HASWELL(dev_priv))
+	    !IS_KABYLAKE(dev_priv) &&
+	    !IS_BROADWELL(dev_priv) &&
+	    !IS_HASWELL(dev_priv))
 		return 0;
 
 	mutex_lock(&dev_priv->av_mutex);
 	/* 1. get the pipe */
-	for_each_intel_encoder(drm_dev, intel_encoder) {
-		if (intel_encoder->type != INTEL_OUTPUT_HDMI)
-			continue;
-		intel_dig_port = enc_to_dig_port(&intel_encoder->base);
-		if (port == intel_dig_port->port) {
-			crtc = to_intel_crtc(intel_encoder->base.crtc);
-			if (!crtc) {
-				DRM_DEBUG_KMS("%s: crtc is NULL\n", __func__);
-				continue;
-			}
-			pipe = crtc->pipe;
-			break;
-		}
+	intel_encoder = dev_priv->dig_port_map[port];
+	/* intel_encoder might be NULL for DP MST */
+	if (!intel_encoder || !intel_encoder->base.crtc ||
+	    intel_encoder->type != INTEL_OUTPUT_HDMI) {
+		DRM_DEBUG_KMS("no valid port %c\n", port_name(port));
+		err = -ENODEV;
+		goto unlock;
 	}
-
+	crtc = to_intel_crtc(intel_encoder->base.crtc);
+	pipe = crtc->pipe;
 	if (pipe == INVALID_PIPE) {
 		DRM_DEBUG_KMS("no pipe for the port %c\n", port_name(port));
-		mutex_unlock(&dev_priv->av_mutex);
-		return -ENODEV;
+		err = -ENODEV;
+		goto unlock;
 	}
+
 	DRM_DEBUG_KMS("pipe %c connects port %c\n",
 				  pipe_name(pipe), port_name(port));
 	mode = &crtc->config->base.adjusted_mode;
@@ -682,8 +686,7 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 		tmp = I915_READ(HSW_AUD_CFG(pipe));
 		tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
 		I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-		mutex_unlock(&dev_priv->av_mutex);
-		return 0;
+		goto unlock;
 	}
 
 	n = audio_config_get_n(mode, rate);
@@ -693,8 +696,7 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 		tmp = I915_READ(HSW_AUD_CFG(pipe));
 		tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
 		I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-		mutex_unlock(&dev_priv->av_mutex);
-		return 0;
+		goto unlock;
 	}
 
 	/* 3. set the N/CTS/M */
@@ -702,8 +704,37 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 	tmp = audio_config_setup_n_reg(n, tmp);
 	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
 
+ unlock:
 	mutex_unlock(&dev_priv->av_mutex);
-	return 0;
+	return err;
+}
+
+static int i915_audio_component_get_eld(struct device *dev, int port,
+					bool *enabled,
+					unsigned char *buf, int max_bytes)
+{
+	struct drm_i915_private *dev_priv = dev_to_i915(dev);
+	struct intel_encoder *intel_encoder;
+	struct intel_digital_port *intel_dig_port;
+	const u8 *eld;
+	int ret = -EINVAL;
+
+	mutex_lock(&dev_priv->av_mutex);
+	intel_encoder = dev_priv->dig_port_map[port];
+	/* intel_encoder might be NULL for DP MST */
+	if (intel_encoder) {
+		ret = 0;
+		intel_dig_port = enc_to_dig_port(&intel_encoder->base);
+		*enabled = intel_dig_port->audio_connector != NULL;
+		if (*enabled) {
+			eld = intel_dig_port->audio_connector->eld;
+			ret = drm_eld_size(eld);
+			memcpy(buf, eld, min(max_bytes, ret));
+		}
+	}
+
+	mutex_unlock(&dev_priv->av_mutex);
+	return ret;
 }
 
 static const struct i915_audio_component_ops i915_audio_component_ops = {
@@ -713,6 +744,7 @@ static const struct i915_audio_component_ops i915_audio_component_ops = {
 	.codec_wake_override = i915_audio_component_codec_wake_override,
 	.get_cdclk_freq	= i915_audio_component_get_cdclk_freq,
 	.sync_audio_rate = i915_audio_component_sync_audio_rate,
+	.get_eld	= i915_audio_component_get_eld,
 };
 
 static int i915_audio_component_bind(struct device *i915_dev,
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index ce82f9c7df24..eba3e0f87181 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -24,7 +24,7 @@
  *    Eric Anholt <eric@anholt.net>
  *
  */
-#include <linux/dmi.h>
+
 #include <drm/drm_dp_helper.h>
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
@@ -332,10 +332,10 @@ parse_sdvo_panel_data(struct drm_i915_private *dev_priv,
 	drm_mode_debug_printmodeline(panel_fixed_mode);
 }
 
-static int intel_bios_ssc_frequency(struct drm_device *dev,
+static int intel_bios_ssc_frequency(struct drm_i915_private *dev_priv,
 				    bool alternate)
 {
-	switch (INTEL_INFO(dev)->gen) {
+	switch (INTEL_INFO(dev_priv)->gen) {
 	case 2:
 		return alternate ? 66667 : 48000;
 	case 3:
@@ -350,26 +350,29 @@ static void
 parse_general_features(struct drm_i915_private *dev_priv,
 		       const struct bdb_header *bdb)
 {
-	struct drm_device *dev = dev_priv->dev;
 	const struct bdb_general_features *general;
 
 	general = find_section(bdb, BDB_GENERAL_FEATURES);
-	if (general) {
-		dev_priv->vbt.int_tv_support = general->int_tv_support;
+	if (!general)
+		return;
+
+	dev_priv->vbt.int_tv_support = general->int_tv_support;
+	/* int_crt_support can't be trusted on earlier platforms */
+	if (bdb->version >= 155 &&
+	    (HAS_DDI(dev_priv) || IS_VALLEYVIEW(dev_priv)))
 		dev_priv->vbt.int_crt_support = general->int_crt_support;
-		dev_priv->vbt.lvds_use_ssc = general->enable_ssc;
-		dev_priv->vbt.lvds_ssc_freq =
-			intel_bios_ssc_frequency(dev, general->ssc_freq);
-		dev_priv->vbt.display_clock_mode = general->display_clock_mode;
-		dev_priv->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
-		DRM_DEBUG_KMS("BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n",
-			      dev_priv->vbt.int_tv_support,
-			      dev_priv->vbt.int_crt_support,
-			      dev_priv->vbt.lvds_use_ssc,
-			      dev_priv->vbt.lvds_ssc_freq,
-			      dev_priv->vbt.display_clock_mode,
-			      dev_priv->vbt.fdi_rx_polarity_inverted);
-	}
+	dev_priv->vbt.lvds_use_ssc = general->enable_ssc;
+	dev_priv->vbt.lvds_ssc_freq =
+		intel_bios_ssc_frequency(dev_priv, general->ssc_freq);
+	dev_priv->vbt.display_clock_mode = general->display_clock_mode;
+	dev_priv->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
+	DRM_DEBUG_KMS("BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n",
+		      dev_priv->vbt.int_tv_support,
+		      dev_priv->vbt.int_crt_support,
+		      dev_priv->vbt.lvds_use_ssc,
+		      dev_priv->vbt.lvds_ssc_freq,
+		      dev_priv->vbt.display_clock_mode,
+		      dev_priv->vbt.fdi_rx_polarity_inverted);
 }
 
 static void
@@ -1054,10 +1057,9 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 static void parse_ddi_ports(struct drm_i915_private *dev_priv,
 			    const struct bdb_header *bdb)
 {
-	struct drm_device *dev = dev_priv->dev;
 	enum port port;
 
-	if (!HAS_DDI(dev))
+	if (!HAS_DDI(dev_priv))
 		return;
 
 	if (!dev_priv->vbt.child_dev_num)
@@ -1170,7 +1172,6 @@ parse_device_mapping(struct drm_i915_private *dev_priv,
 static void
 init_vbt_defaults(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = dev_priv->dev;
 	enum port port;
 
 	dev_priv->vbt.crt_ddc_pin = GMBUS_PIN_VGADDC;
@@ -1195,8 +1196,8 @@ init_vbt_defaults(struct drm_i915_private *dev_priv)
 	 * Core/SandyBridge/IvyBridge use alternative (120MHz) reference
 	 * clock for LVDS.
 	 */
-	dev_priv->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(dev,
-			!HAS_PCH_SPLIT(dev));
+	dev_priv->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(dev_priv,
+			!HAS_PCH_SPLIT(dev_priv));
 	DRM_DEBUG_KMS("Set default to SSC at %d kHz\n", dev_priv->vbt.lvds_ssc_freq);
 
 	for (port = PORT_A; port < I915_MAX_PORTS; port++) {
@@ -1211,88 +1212,79 @@ init_vbt_defaults(struct drm_i915_private *dev_priv)
 	}
 }
 
-static int intel_no_opregion_vbt_callback(const struct dmi_system_id *id)
+static const struct bdb_header *get_bdb_header(const struct vbt_header *vbt)
 {
-	DRM_DEBUG_KMS("Falling back to manually reading VBT from "
-		      "VBIOS ROM for %s\n",
-		      id->ident);
-	return 1;
+	const void *_vbt = vbt;
+
+	return _vbt + vbt->bdb_offset;
 }
 
-static const struct dmi_system_id intel_no_opregion_vbt[] = {
-	{
-		.callback = intel_no_opregion_vbt_callback,
-		.ident = "ThinkCentre A57",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "97027RG"),
-		},
-	},
-	{ }
-};
-
-static const struct bdb_header *validate_vbt(const void *base,
-					     size_t size,
-					     const void *_vbt,
-					     const char *source)
+/**
+ * intel_bios_is_valid_vbt - does the given buffer contain a valid VBT
+ * @buf:	pointer to a buffer to validate
+ * @size:	size of the buffer
+ *
+ * Returns true on valid VBT.
+ */
+bool intel_bios_is_valid_vbt(const void *buf, size_t size)
 {
-	size_t offset = _vbt - base;
-	const struct vbt_header *vbt = _vbt;
+	const struct vbt_header *vbt = buf;
 	const struct bdb_header *bdb;
 
-	if (offset + sizeof(struct vbt_header) > size) {
+	if (!vbt)
+		return false;
+
+	if (sizeof(struct vbt_header) > size) {
 		DRM_DEBUG_DRIVER("VBT header incomplete\n");
-		return NULL;
+		return false;
 	}
 
 	if (memcmp(vbt->signature, "$VBT", 4)) {
 		DRM_DEBUG_DRIVER("VBT invalid signature\n");
-		return NULL;
+		return false;
 	}
 
-	offset += vbt->bdb_offset;
-	if (offset + sizeof(struct bdb_header) > size) {
+	if (vbt->bdb_offset + sizeof(struct bdb_header) > size) {
 		DRM_DEBUG_DRIVER("BDB header incomplete\n");
-		return NULL;
+		return false;
 	}
 
-	bdb = base + offset;
-	if (offset + bdb->bdb_size > size) {
+	bdb = get_bdb_header(vbt);
+	if (vbt->bdb_offset + bdb->bdb_size > size) {
 		DRM_DEBUG_DRIVER("BDB incomplete\n");
-		return NULL;
+		return false;
 	}
 
-	DRM_DEBUG_KMS("Using VBT from %s: %20s\n",
-		      source, vbt->signature);
-	return bdb;
+	return vbt;
 }
 
-static const struct bdb_header *find_vbt(void __iomem *bios, size_t size)
+static const struct vbt_header *find_vbt(void __iomem *bios, size_t size)
 {
-	const struct bdb_header *bdb = NULL;
 	size_t i;
 
 	/* Scour memory looking for the VBT signature. */
 	for (i = 0; i + 4 < size; i++) {
-		if (ioread32(bios + i) == *((const u32 *) "$VBT")) {
-			/*
-			 * This is the one place where we explicitly discard the
-			 * address space (__iomem) of the BIOS/VBT. From now on
-			 * everything is based on 'base', and treated as regular
-			 * memory.
-			 */
-			void *_bios = (void __force *) bios;
+		void *vbt;
 
-			bdb = validate_vbt(_bios, size, _bios + i, "PCI ROM");
-			break;
-		}
+		if (ioread32(bios + i) != *((const u32 *) "$VBT"))
+			continue;
+
+		/*
+		 * This is the one place where we explicitly discard the address
+		 * space (__iomem) of the BIOS/VBT.
+		 */
+		vbt = (void __force *) bios + i;
+		if (intel_bios_is_valid_vbt(vbt, size - i))
+			return vbt;
+
+		break;
 	}
 
-	return bdb;
+	return NULL;
 }
 
 /**
- * intel_parse_bios - find VBT and initialize settings from the BIOS
+ * intel_bios_init - find VBT and initialize settings from the BIOS
  * @dev: DRM device
  *
  * Loads the Video BIOS and checks that the VBT exists.  Sets scratch registers
@@ -1301,37 +1293,39 @@ static const struct bdb_header *find_vbt(void __iomem *bios, size_t size)
  * Returns 0 on success, nonzero on failure.
  */
 int
-intel_parse_bios(struct drm_device *dev)
+intel_bios_init(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct pci_dev *pdev = dev->pdev;
-	const struct bdb_header *bdb = NULL;
+	struct pci_dev *pdev = dev_priv->dev->pdev;
+	const struct vbt_header *vbt = dev_priv->opregion.vbt;
+	const struct bdb_header *bdb;
 	u8 __iomem *bios = NULL;
 
-	if (HAS_PCH_NOP(dev))
+	if (HAS_PCH_NOP(dev_priv))
 		return -ENODEV;
 
 	init_vbt_defaults(dev_priv);
 
-	/* XXX Should this validation be moved to intel_opregion.c? */
-	if (!dmi_check_system(intel_no_opregion_vbt) && dev_priv->opregion.vbt)
-		bdb = validate_vbt(dev_priv->opregion.header, OPREGION_SIZE,
-				   dev_priv->opregion.vbt, "OpRegion");
-
-	if (bdb == NULL) {
+	if (!vbt) {
 		size_t size;
 
 		bios = pci_map_rom(pdev, &size);
 		if (!bios)
 			return -1;
 
-		bdb = find_vbt(bios, size);
-		if (!bdb) {
+		vbt = find_vbt(bios, size);
+		if (!vbt) {
 			pci_unmap_rom(pdev, bios);
 			return -1;
 		}
+
+		DRM_DEBUG_KMS("Found valid VBT in PCI ROM\n");
 	}
 
+	bdb = get_bdb_header(vbt);
+
+	DRM_DEBUG_KMS("VBT signature \"%.*s\", BDB version %d\n",
+		      (int)sizeof(vbt->signature), vbt->signature, bdb->version);
+
 	/* Grab useful general definitions */
 	parse_general_features(dev_priv, bdb);
 	parse_general_definitions(dev_priv, bdb);
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index 7ec8c9aefb84..54eac1003a1e 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -28,8 +28,6 @@
 #ifndef _I830_BIOS_H_
 #define _I830_BIOS_H_
 
-#include <drm/drmP.h>
-
 struct vbt_header {
 	u8 signature[20];		/**< Always starts with 'VBT$' */
 	u16 version;			/**< decimal */
@@ -588,8 +586,6 @@ struct bdb_psr {
 	struct psr_table psr_table[16];
 } __packed;
 
-int intel_parse_bios(struct drm_device *dev);
-
 /*
  * Driver<->VBIOS interaction occurs through scratch bits in
  * GR18 & SWF*.
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 6a2c76e367a5..9c89df1af036 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -50,7 +50,7 @@ struct intel_crt {
 	 * encoder's enable/disable callbacks */
 	struct intel_connector *connector;
 	bool force_hotplug_required;
-	u32 adpa_reg;
+	i915_reg_t adpa_reg;
 };
 
 static struct intel_crt *intel_encoder_to_crt(struct intel_encoder *encoder)
@@ -480,12 +480,8 @@ intel_crt_load_detect(struct intel_crt *crt)
 	uint32_t vsample;
 	uint32_t vblank, vblank_start, vblank_end;
 	uint32_t dsl;
-	uint32_t bclrpat_reg;
-	uint32_t vtotal_reg;
-	uint32_t vblank_reg;
-	uint32_t vsync_reg;
-	uint32_t pipeconf_reg;
-	uint32_t pipe_dsl_reg;
+	i915_reg_t bclrpat_reg, vtotal_reg,
+		vblank_reg, vsync_reg, pipeconf_reg, pipe_dsl_reg;
 	uint8_t	st00;
 	enum drm_connector_status status;
 
@@ -518,7 +514,7 @@ intel_crt_load_detect(struct intel_crt *crt)
 		/* Wait for next Vblank to substitue
 		 * border color for Color info */
 		intel_wait_for_vblank(dev, pipe);
-		st00 = I915_READ8(VGA_MSR_WRITE);
+		st00 = I915_READ8(_VGA_MSR_WRITE);
 		status = ((st00 & (1 << 4)) != 0) ?
 			connector_status_connected :
 			connector_status_disconnected;
@@ -563,7 +559,7 @@ intel_crt_load_detect(struct intel_crt *crt)
 		do {
 			count++;
 			/* Read the ST00 VGA status register */
-			st00 = I915_READ8(VGA_MSR_WRITE);
+			st00 = I915_READ8(_VGA_MSR_WRITE);
 			if (st00 & (1 << 4))
 				detect++;
 		} while ((I915_READ(pipe_dsl_reg) == dsl));
@@ -781,11 +777,37 @@ void intel_crt_init(struct drm_device *dev)
 	struct intel_crt *crt;
 	struct intel_connector *intel_connector;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	i915_reg_t adpa_reg;
+	u32 adpa;
 
 	/* Skip machines without VGA that falsely report hotplug events */
 	if (dmi_check_system(intel_no_crt))
 		return;
 
+	if (HAS_PCH_SPLIT(dev))
+		adpa_reg = PCH_ADPA;
+	else if (IS_VALLEYVIEW(dev))
+		adpa_reg = VLV_ADPA;
+	else
+		adpa_reg = ADPA;
+
+	adpa = I915_READ(adpa_reg);
+	if ((adpa & ADPA_DAC_ENABLE) == 0) {
+		/*
+		 * On some machines (some IVB at least) CRT can be
+		 * fused off, but there's no known fuse bit to
+		 * indicate that. On these machine the ADPA register
+		 * works normally, except the DAC enable bit won't
+		 * take. So the only way to tell is attempt to enable
+		 * it and see what happens.
+		 */
+		I915_WRITE(adpa_reg, adpa | ADPA_DAC_ENABLE |
+			   ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE);
+		if ((I915_READ(adpa_reg) & ADPA_DAC_ENABLE) == 0)
+			return;
+		I915_WRITE(adpa_reg, adpa);
+	}
+
 	crt = kzalloc(sizeof(struct intel_crt), GFP_KERNEL);
 	if (!crt)
 		return;
@@ -802,7 +824,7 @@ void intel_crt_init(struct drm_device *dev)
 			   &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA);
 
 	drm_encoder_init(dev, &crt->base.base, &intel_crt_enc_funcs,
-			 DRM_MODE_ENCODER_DAC);
+			 DRM_MODE_ENCODER_DAC, NULL);
 
 	intel_connector_attach_encoder(intel_connector, &crt->base);
 
@@ -819,15 +841,10 @@ void intel_crt_init(struct drm_device *dev)
 		connector->interlace_allowed = 1;
 	connector->doublescan_allowed = 0;
 
-	if (HAS_PCH_SPLIT(dev))
-		crt->adpa_reg = PCH_ADPA;
-	else if (IS_VALLEYVIEW(dev))
-		crt->adpa_reg = VLV_ADPA;
-	else
-		crt->adpa_reg = ADPA;
+	crt->adpa_reg = adpa_reg;
 
 	crt->base.compute_config = intel_crt_compute_config;
-	if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev)) {
+	if (HAS_PCH_SPLIT(dev)) {
 		crt->base.disable = pch_disable_crt;
 		crt->base.post_disable = pch_post_disable_crt;
 	} else {
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index 9e530a739354..9bb63a85997a 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -47,21 +47,10 @@
 MODULE_FIRMWARE(I915_CSR_SKL);
 MODULE_FIRMWARE(I915_CSR_BXT);
 
-/*
-* SKL CSR registers for DC5 and DC6
-*/
-#define CSR_PROGRAM(i)			(0x80000 + (i) * 4)
-#define CSR_SSP_BASE_ADDR_GEN9		0x00002FC0
-#define CSR_HTP_ADDR_SKL		0x00500034
-#define CSR_SSP_BASE			0x8F074
-#define CSR_HTP_SKL			0x8F004
-#define CSR_LAST_WRITE			0x8F034
-#define CSR_LAST_WRITE_VALUE		0xc003b400
-/* MMIO address range for CSR program (0x80000 - 0x82FFF) */
+#define SKL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 23)
+
 #define CSR_MAX_FW_SIZE			0x2FFF
 #define CSR_DEFAULT_FW_OFFSET		0xFFFFFFFF
-#define CSR_MMIO_START_RANGE	0x80000
-#define CSR_MMIO_END_RANGE		0x8FFFF
 
 struct intel_css_header {
 	/* 0x09 for DMC */
@@ -177,167 +166,146 @@ struct stepping_info {
 	char substepping;
 };
 
+/*
+ * Kabylake derivated from Skylake H0, so SKL H0
+ * is the right firmware for KBL A0 (revid 0).
+ */
+static const struct stepping_info kbl_stepping_info[] = {
+	{'H', '0'}, {'I', '0'}
+};
+
 static const struct stepping_info skl_stepping_info[] = {
-		{'A', '0'}, {'B', '0'}, {'C', '0'},
-		{'D', '0'}, {'E', '0'}, {'F', '0'},
-		{'G', '0'}, {'H', '0'}, {'I', '0'}
+	{'A', '0'}, {'B', '0'}, {'C', '0'},
+	{'D', '0'}, {'E', '0'}, {'F', '0'},
+	{'G', '0'}, {'H', '0'}, {'I', '0'}
 };
 
-static struct stepping_info bxt_stepping_info[] = {
+static const struct stepping_info bxt_stepping_info[] = {
 	{'A', '0'}, {'A', '1'}, {'A', '2'},
 	{'B', '0'}, {'B', '1'}, {'B', '2'}
 };
 
-static char intel_get_stepping(struct drm_device *dev)
-{
-	if (IS_SKYLAKE(dev) && (dev->pdev->revision <
-			ARRAY_SIZE(skl_stepping_info)))
-		return skl_stepping_info[dev->pdev->revision].stepping;
-	else if (IS_BROXTON(dev) && (dev->pdev->revision <
-				ARRAY_SIZE(bxt_stepping_info)))
-		return bxt_stepping_info[dev->pdev->revision].stepping;
-	else
-		return -ENODATA;
-}
-
-static char intel_get_substepping(struct drm_device *dev)
-{
-	if (IS_SKYLAKE(dev) && (dev->pdev->revision <
-			ARRAY_SIZE(skl_stepping_info)))
-		return skl_stepping_info[dev->pdev->revision].substepping;
-	else if (IS_BROXTON(dev) && (dev->pdev->revision <
-			ARRAY_SIZE(bxt_stepping_info)))
-		return bxt_stepping_info[dev->pdev->revision].substepping;
-	else
-		return -ENODATA;
-}
-
-/**
- * intel_csr_load_status_get() - to get firmware loading status.
- * @dev_priv: i915 device.
- *
- * This function helps to get the firmware loading status.
- *
- * Return: Firmware loading status.
- */
-enum csr_state intel_csr_load_status_get(struct drm_i915_private *dev_priv)
+static const struct stepping_info *intel_get_stepping_info(struct drm_device *dev)
 {
-	enum csr_state state;
-
-	mutex_lock(&dev_priv->csr_lock);
-	state = dev_priv->csr.state;
-	mutex_unlock(&dev_priv->csr_lock);
+	const struct stepping_info *si;
+	unsigned int size;
+
+	if (IS_KABYLAKE(dev)) {
+		size = ARRAY_SIZE(kbl_stepping_info);
+		si = kbl_stepping_info;
+	} else if (IS_SKYLAKE(dev)) {
+		size = ARRAY_SIZE(skl_stepping_info);
+		si = skl_stepping_info;
+	} else if (IS_BROXTON(dev)) {
+		size = ARRAY_SIZE(bxt_stepping_info);
+		si = bxt_stepping_info;
+	} else {
+		return NULL;
+	}
 
-	return state;
-}
+	if (INTEL_REVID(dev) < size)
+		return si + INTEL_REVID(dev);
 
-/**
- * intel_csr_load_status_set() - help to set firmware loading status.
- * @dev_priv: i915 device.
- * @state: enumeration of firmware loading status.
- *
- * Set the firmware loading status.
- */
-void intel_csr_load_status_set(struct drm_i915_private *dev_priv,
-			enum csr_state state)
-{
-	mutex_lock(&dev_priv->csr_lock);
-	dev_priv->csr.state = state;
-	mutex_unlock(&dev_priv->csr_lock);
+	return NULL;
 }
 
 /**
  * intel_csr_load_program() - write the firmware from memory to register.
- * @dev: drm device.
+ * @dev_priv: i915 drm device.
  *
  * CSR firmware is read from a .bin file and kept in internal memory one time.
  * Everytime display comes back from low power state this function is called to
  * copy the firmware from internal memory to registers.
  */
-void intel_csr_load_program(struct drm_device *dev)
+void intel_csr_load_program(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 *payload = dev_priv->csr.dmc_payload;
 	uint32_t i, fw_size;
 
-	if (!IS_GEN9(dev)) {
+	if (!IS_GEN9(dev_priv)) {
 		DRM_ERROR("No CSR support available for this platform\n");
 		return;
 	}
 
-	/*
-	 * FIXME: Firmware gets lost on S3/S4, but not when entering system
-	 * standby or suspend-to-idle (which is just like forced runtime pm).
-	 * Unfortunately the ACPI subsystem doesn't yet give us a way to
-	 * differentiate this, hence figure it out with this hack.
-	 */
-	if (I915_READ(CSR_PROGRAM(0)))
+	if (!dev_priv->csr.dmc_payload) {
+		DRM_ERROR("Tried to program CSR with empty payload\n");
 		return;
+	}
 
-	mutex_lock(&dev_priv->csr_lock);
 	fw_size = dev_priv->csr.dmc_fw_size;
 	for (i = 0; i < fw_size; i++)
 		I915_WRITE(CSR_PROGRAM(i), payload[i]);
 
 	for (i = 0; i < dev_priv->csr.mmio_count; i++) {
 		I915_WRITE(dev_priv->csr.mmioaddr[i],
-			dev_priv->csr.mmiodata[i]);
+			   dev_priv->csr.mmiodata[i]);
 	}
-
-	dev_priv->csr.state = FW_LOADED;
-	mutex_unlock(&dev_priv->csr_lock);
 }
 
-static void finish_csr_load(const struct firmware *fw, void *context)
+static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv,
+			      const struct firmware *fw)
 {
-	struct drm_i915_private *dev_priv = context;
 	struct drm_device *dev = dev_priv->dev;
 	struct intel_css_header *css_header;
 	struct intel_package_header *package_header;
 	struct intel_dmc_header *dmc_header;
 	struct intel_csr *csr = &dev_priv->csr;
-	char stepping = intel_get_stepping(dev);
-	char substepping = intel_get_substepping(dev);
+	const struct stepping_info *stepping_info = intel_get_stepping_info(dev);
+	char stepping, substepping;
 	uint32_t dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes;
 	uint32_t i;
 	uint32_t *dmc_payload;
-	bool fw_loaded = false;
 
-	if (!fw) {
-		i915_firmware_load_error_print(csr->fw_path, 0);
-		goto out;
-	}
+	if (!fw)
+		return NULL;
 
-	if ((stepping == -ENODATA) || (substepping == -ENODATA)) {
+	if (!stepping_info) {
 		DRM_ERROR("Unknown stepping info, firmware loading failed\n");
-		goto out;
+		return NULL;
 	}
 
+	stepping = stepping_info->stepping;
+	substepping = stepping_info->substepping;
+
 	/* Extract CSS Header information*/
 	css_header = (struct intel_css_header *)fw->data;
 	if (sizeof(struct intel_css_header) !=
-		(css_header->header_len * 4)) {
+	    (css_header->header_len * 4)) {
 		DRM_ERROR("Firmware has wrong CSS header length %u bytes\n",
-			(css_header->header_len * 4));
-		goto out;
+			  (css_header->header_len * 4));
+		return NULL;
+	}
+
+	csr->version = css_header->version;
+
+	if (IS_SKYLAKE(dev) && csr->version < SKL_CSR_VERSION_REQUIRED) {
+		DRM_INFO("Refusing to load old Skylake DMC firmware v%u.%u,"
+			 " please upgrade to v%u.%u or later"
+			 " [https://01.org/linuxgraphics/intel-linux-graphics-firmwares].\n",
+			 CSR_VERSION_MAJOR(csr->version),
+			 CSR_VERSION_MINOR(csr->version),
+			 CSR_VERSION_MAJOR(SKL_CSR_VERSION_REQUIRED),
+			 CSR_VERSION_MINOR(SKL_CSR_VERSION_REQUIRED));
+		return NULL;
 	}
+
 	readcount += sizeof(struct intel_css_header);
 
 	/* Extract Package Header information*/
 	package_header = (struct intel_package_header *)
-					&fw->data[readcount];
+		&fw->data[readcount];
 	if (sizeof(struct intel_package_header) !=
-		(package_header->header_len * 4)) {
+	    (package_header->header_len * 4)) {
 		DRM_ERROR("Firmware has wrong package header length %u bytes\n",
-			(package_header->header_len * 4));
-		goto out;
+			  (package_header->header_len * 4));
+		return NULL;
 	}
 	readcount += sizeof(struct intel_package_header);
 
 	/* Search for dmc_offset to find firware binary. */
 	for (i = 0; i < package_header->num_entries; i++) {
 		if (package_header->fw_info[i].substepping == '*' &&
-			stepping == package_header->fw_info[i].stepping) {
+		    stepping == package_header->fw_info[i].stepping) {
 			dmc_offset = package_header->fw_info[i].offset;
 			break;
 		} else if (stepping == package_header->fw_info[i].stepping &&
@@ -345,12 +313,12 @@ static void finish_csr_load(const struct firmware *fw, void *context)
 			dmc_offset = package_header->fw_info[i].offset;
 			break;
 		} else if (package_header->fw_info[i].stepping == '*' &&
-			package_header->fw_info[i].substepping == '*')
+			   package_header->fw_info[i].substepping == '*')
 			dmc_offset = package_header->fw_info[i].offset;
 	}
 	if (dmc_offset == CSR_DEFAULT_FW_OFFSET) {
 		DRM_ERROR("Firmware not supported for %c stepping\n", stepping);
-		goto out;
+		return NULL;
 	}
 	readcount += dmc_offset;
 
@@ -358,26 +326,26 @@ static void finish_csr_load(const struct firmware *fw, void *context)
 	dmc_header = (struct intel_dmc_header *)&fw->data[readcount];
 	if (sizeof(struct intel_dmc_header) != (dmc_header->header_len)) {
 		DRM_ERROR("Firmware has wrong dmc header length %u bytes\n",
-				(dmc_header->header_len));
-		goto out;
+			  (dmc_header->header_len));
+		return NULL;
 	}
 	readcount += sizeof(struct intel_dmc_header);
 
 	/* Cache the dmc header info. */
 	if (dmc_header->mmio_count > ARRAY_SIZE(csr->mmioaddr)) {
 		DRM_ERROR("Firmware has wrong mmio count %u\n",
-						dmc_header->mmio_count);
-		goto out;
+			  dmc_header->mmio_count);
+		return NULL;
 	}
 	csr->mmio_count = dmc_header->mmio_count;
 	for (i = 0; i < dmc_header->mmio_count; i++) {
 		if (dmc_header->mmioaddr[i] < CSR_MMIO_START_RANGE ||
-			dmc_header->mmioaddr[i] > CSR_MMIO_END_RANGE) {
+		    dmc_header->mmioaddr[i] > CSR_MMIO_END_RANGE) {
 			DRM_ERROR(" Firmware has wrong mmio address 0x%x\n",
-						dmc_header->mmioaddr[i]);
-			goto out;
+				  dmc_header->mmioaddr[i]);
+			return NULL;
 		}
-		csr->mmioaddr[i] = dmc_header->mmioaddr[i];
+		csr->mmioaddr[i] = _MMIO(dmc_header->mmioaddr[i]);
 		csr->mmiodata[i] = dmc_header->mmiodata[i];
 	}
 
@@ -385,56 +353,80 @@ static void finish_csr_load(const struct firmware *fw, void *context)
 	nbytes = dmc_header->fw_size * 4;
 	if (nbytes > CSR_MAX_FW_SIZE) {
 		DRM_ERROR("CSR firmware too big (%u) bytes\n", nbytes);
-		goto out;
+		return NULL;
 	}
 	csr->dmc_fw_size = dmc_header->fw_size;
 
-	csr->dmc_payload = kmalloc(nbytes, GFP_KERNEL);
-	if (!csr->dmc_payload) {
+	dmc_payload = kmalloc(nbytes, GFP_KERNEL);
+	if (!dmc_payload) {
 		DRM_ERROR("Memory allocation failed for dmc payload\n");
-		goto out;
+		return NULL;
 	}
 
-	dmc_payload = csr->dmc_payload;
 	memcpy(dmc_payload, &fw->data[readcount], nbytes);
 
+	return dmc_payload;
+}
+
+static void csr_load_work_fn(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv;
+	struct intel_csr *csr;
+	const struct firmware *fw;
+	int ret;
+
+	dev_priv = container_of(work, typeof(*dev_priv), csr.work);
+	csr = &dev_priv->csr;
+
+	ret = request_firmware(&fw, dev_priv->csr.fw_path,
+			       &dev_priv->dev->pdev->dev);
+	if (!fw)
+		goto out;
+
+	dev_priv->csr.dmc_payload = parse_csr_fw(dev_priv, fw);
+	if (!dev_priv->csr.dmc_payload)
+		goto out;
+
 	/* load csr program during system boot, as needed for DC states */
-	intel_csr_load_program(dev);
-	fw_loaded = true;
+	intel_csr_load_program(dev_priv);
 
-	DRM_DEBUG_KMS("Finished loading %s\n", dev_priv->csr.fw_path);
 out:
-	if (fw_loaded)
-		intel_runtime_pm_put(dev_priv);
-	else
-		intel_csr_load_status_set(dev_priv, FW_FAILED);
+	if (dev_priv->csr.dmc_payload) {
+		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
+
+		DRM_INFO("Finished loading %s (v%u.%u)\n",
+			 dev_priv->csr.fw_path,
+			 CSR_VERSION_MAJOR(csr->version),
+			 CSR_VERSION_MINOR(csr->version));
+	} else {
+		DRM_ERROR("Failed to load DMC firmware, disabling rpm\n");
+	}
 
 	release_firmware(fw);
 }
 
 /**
  * intel_csr_ucode_init() - initialize the firmware loading.
- * @dev: drm device.
+ * @dev_priv: i915 drm device.
  *
  * This function is called at the time of loading the display driver to read
  * firmware from a .bin file and copied into a internal memory.
  */
-void intel_csr_ucode_init(struct drm_device *dev)
+void intel_csr_ucode_init(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_csr *csr = &dev_priv->csr;
-	int ret;
 
-	if (!HAS_CSR(dev))
+	INIT_WORK(&dev_priv->csr.work, csr_load_work_fn);
+
+	if (!HAS_CSR(dev_priv))
 		return;
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev_priv))
 		csr->fw_path = I915_CSR_SKL;
 	else if (IS_BROXTON(dev_priv))
 		csr->fw_path = I915_CSR_BXT;
 	else {
 		DRM_ERROR("Unexpected: no known CSR firmware for platform\n");
-		intel_csr_load_status_set(dev_priv, FW_FAILED);
 		return;
 	}
 
@@ -444,43 +436,24 @@ void intel_csr_ucode_init(struct drm_device *dev)
 	 * Obtain a runtime pm reference, until CSR is loaded,
 	 * to avoid entering runtime-suspend.
 	 */
-	intel_runtime_pm_get(dev_priv);
-
-	/* CSR supported for platform, load firmware */
-	ret = request_firmware_nowait(THIS_MODULE, true, csr->fw_path,
-				&dev_priv->dev->pdev->dev,
-				GFP_KERNEL, dev_priv,
-				finish_csr_load);
-	if (ret) {
-		i915_firmware_load_error_print(csr->fw_path, ret);
-		intel_csr_load_status_set(dev_priv, FW_FAILED);
-	}
+	intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+
+	schedule_work(&dev_priv->csr.work);
 }
 
 /**
  * intel_csr_ucode_fini() - unload the CSR firmware.
- * @dev: drm device.
+ * @dev_priv: i915 drm device.
  *
  * Firmmware unloading includes freeing the internal momory and reset the
  * firmware loading status.
  */
-void intel_csr_ucode_fini(struct drm_device *dev)
+void intel_csr_ucode_fini(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (!HAS_CSR(dev))
+	if (!HAS_CSR(dev_priv))
 		return;
 
-	intel_csr_load_status_set(dev_priv, FW_FAILED);
-	kfree(dev_priv->csr.dmc_payload);
-}
+	flush_work(&dev_priv->csr.work);
 
-void assert_csr_loaded(struct drm_i915_private *dev_priv)
-{
-	WARN_ONCE(intel_csr_load_status_get(dev_priv) != FW_LOADED,
-		  "CSR is not loaded.\n");
-	WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
-		  "CSR program storage start is NULL\n");
-	WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
-	WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
+	kfree(dev_priv->csr.dmc_payload);
 }
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index a6752a61d99f..e6408e5583d7 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -133,12 +133,12 @@ static const struct ddi_buf_trans skl_ddi_translations_dp[] = {
 	{ 0x00002016, 0x000000A0, 0x0 },
 	{ 0x00005012, 0x0000009B, 0x0 },
 	{ 0x00007011, 0x00000088, 0x0 },
-	{ 0x00009010, 0x000000C7, 0x0 },
+	{ 0x80009010, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 	{ 0x00002016, 0x0000009B, 0x0 },
 	{ 0x00005012, 0x00000088, 0x0 },
-	{ 0x00007011, 0x000000C7, 0x0 },
+	{ 0x80007011, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 	{ 0x00002016, 0x000000DF, 0x0 },
-	{ 0x00005012, 0x000000C7, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 };
 
 /* Skylake U */
@@ -146,12 +146,12 @@ static const struct ddi_buf_trans skl_u_ddi_translations_dp[] = {
 	{ 0x0000201B, 0x000000A2, 0x0 },
 	{ 0x00005012, 0x00000088, 0x0 },
 	{ 0x00007011, 0x00000087, 0x0 },
-	{ 0x80009010, 0x000000C7, 0x1 },	/* Uses I_boost level 0x1 */
+	{ 0x80009010, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 	{ 0x0000201B, 0x0000009D, 0x0 },
-	{ 0x00005012, 0x000000C7, 0x0 },
-	{ 0x00007011, 0x000000C7, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
+	{ 0x80007011, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 	{ 0x00002016, 0x00000088, 0x0 },
-	{ 0x00005012, 0x000000C7, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x1 },	/* Uses I_boost level 0x1 */
 };
 
 /* Skylake Y */
@@ -159,12 +159,12 @@ static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = {
 	{ 0x00000018, 0x000000A2, 0x0 },
 	{ 0x00005012, 0x00000088, 0x0 },
 	{ 0x00007011, 0x00000087, 0x0 },
-	{ 0x80009010, 0x000000C7, 0x3 },	/* Uses I_boost level 0x3 */
+	{ 0x80009010, 0x000000C0, 0x3 },	/* Uses I_boost level 0x3 */
 	{ 0x00000018, 0x0000009D, 0x0 },
-	{ 0x00005012, 0x000000C7, 0x0 },
-	{ 0x00007011, 0x000000C7, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },	/* Uses I_boost level 0x3 */
+	{ 0x80007011, 0x000000C0, 0x3 },	/* Uses I_boost level 0x3 */
 	{ 0x00000018, 0x00000088, 0x0 },
-	{ 0x00005012, 0x000000C7, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },	/* Uses I_boost level 0x3 */
 };
 
 /*
@@ -345,7 +345,7 @@ enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
 static bool
 intel_dig_port_supports_hdmi(const struct intel_digital_port *intel_dig_port)
 {
-	return intel_dig_port->hdmi.hdmi_reg;
+	return i915_mmio_reg_valid(intel_dig_port->hdmi.hdmi_reg);
 }
 
 static const struct ddi_buf_trans *skl_get_buf_trans_dp(struct drm_device *dev,
@@ -353,10 +353,10 @@ static const struct ddi_buf_trans *skl_get_buf_trans_dp(struct drm_device *dev,
 {
 	const struct ddi_buf_trans *ddi_translations;
 
-	if (IS_SKL_ULX(dev)) {
+	if (IS_SKL_ULX(dev) || IS_KBL_ULX(dev)) {
 		ddi_translations = skl_y_ddi_translations_dp;
 		*n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp);
-	} else if (IS_SKL_ULT(dev)) {
+	} else if (IS_SKL_ULT(dev) || IS_KBL_ULT(dev)) {
 		ddi_translations = skl_u_ddi_translations_dp;
 		*n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp);
 	} else {
@@ -373,7 +373,7 @@ static const struct ddi_buf_trans *skl_get_buf_trans_edp(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	const struct ddi_buf_trans *ddi_translations;
 
-	if (IS_SKL_ULX(dev)) {
+	if (IS_SKL_ULX(dev) || IS_KBL_ULX(dev)) {
 		if (dev_priv->edp_low_vswing) {
 			ddi_translations = skl_y_ddi_translations_edp;
 			*n_entries = ARRAY_SIZE(skl_y_ddi_translations_edp);
@@ -381,7 +381,7 @@ static const struct ddi_buf_trans *skl_get_buf_trans_edp(struct drm_device *dev,
 			ddi_translations = skl_y_ddi_translations_dp;
 			*n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp);
 		}
-	} else if (IS_SKL_ULT(dev)) {
+	} else if (IS_SKL_ULT(dev) || IS_KBL_ULT(dev)) {
 		if (dev_priv->edp_low_vswing) {
 			ddi_translations = skl_u_ddi_translations_edp;
 			*n_entries = ARRAY_SIZE(skl_u_ddi_translations_edp);
@@ -408,7 +408,7 @@ skl_get_buf_trans_hdmi(struct drm_device *dev,
 {
 	const struct ddi_buf_trans *ddi_translations;
 
-	if (IS_SKL_ULX(dev)) {
+	if (IS_SKL_ULX(dev) || IS_KBL_ULX(dev)) {
 		ddi_translations = skl_y_ddi_translations_hdmi;
 		*n_entries = ARRAY_SIZE(skl_y_ddi_translations_hdmi);
 	} else {
@@ -448,7 +448,7 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port,
 		bxt_ddi_vswing_sequence(dev, hdmi_level, port,
 					INTEL_OUTPUT_HDMI);
 		return;
-	} else if (IS_SKYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		ddi_translations_fdi = NULL;
 		ddi_translations_dp =
 				skl_get_buf_trans_dp(dev, &n_dp_entries);
@@ -576,7 +576,7 @@ void intel_prepare_ddi(struct drm_device *dev)
 static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
 				    enum port port)
 {
-	uint32_t reg = DDI_BUF_CTL(port);
+	i915_reg_t reg = DDI_BUF_CTL(port);
 	int i;
 
 	for (i = 0; i < 16; i++) {
@@ -675,15 +675,16 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 		temp = I915_READ(DP_TP_STATUS(PORT_E));
 		if (temp & DP_TP_STATUS_AUTOTRAIN_DONE) {
 			DRM_DEBUG_KMS("FDI link training done on step %d\n", i);
+			break;
+		}
 
-			/* Enable normal pixel sending for FDI */
-			I915_WRITE(DP_TP_CTL(PORT_E),
-				   DP_TP_CTL_FDI_AUTOTRAIN |
-				   DP_TP_CTL_LINK_TRAIN_NORMAL |
-				   DP_TP_CTL_ENHANCED_FRAME_ENABLE |
-				   DP_TP_CTL_ENABLE);
-
-			return;
+		/*
+		 * Leave things enabled even if we failed to train FDI.
+		 * Results in less fireworks from the state checker.
+		 */
+		if (i == ARRAY_SIZE(hsw_ddi_translations_fdi) * 2 - 1) {
+			DRM_ERROR("FDI link training failed!\n");
+			break;
 		}
 
 		temp = I915_READ(DDI_BUF_CTL(PORT_E));
@@ -712,7 +713,12 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 		POSTING_READ(FDI_RX_MISC(PIPE_A));
 	}
 
-	DRM_ERROR("FDI link training failed!\n");
+	/* Enable normal pixel sending for FDI */
+	I915_WRITE(DP_TP_CTL(PORT_E),
+		   DP_TP_CTL_FDI_AUTOTRAIN |
+		   DP_TP_CTL_LINK_TRAIN_NORMAL |
+		   DP_TP_CTL_ENHANCED_FRAME_ENABLE |
+		   DP_TP_CTL_ENABLE);
 }
 
 void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder)
@@ -931,7 +937,8 @@ static void hsw_wrpll_update_rnp(uint64_t freq2k, unsigned budget,
 	/* Otherwise a < c && b >= d, do nothing */
 }
 
-static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, int reg)
+static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv,
+				   i915_reg_t reg)
 {
 	int refclk = LC_FREQ;
 	int n, p, r;
@@ -967,7 +974,7 @@ static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, int reg)
 static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv,
 			       uint32_t dpll)
 {
-	uint32_t cfgcr1_reg, cfgcr2_reg;
+	i915_reg_t cfgcr1_reg, cfgcr2_reg;
 	uint32_t cfgcr1_val, cfgcr2_val;
 	uint32_t p0, p1, p2, dco_freq;
 
@@ -1112,10 +1119,10 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder,
 		link_clock = 270000;
 		break;
 	case PORT_CLK_SEL_WRPLL1:
-		link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL1);
+		link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(0));
 		break;
 	case PORT_CLK_SEL_WRPLL2:
-		link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL2);
+		link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(1));
 		break;
 	case PORT_CLK_SEL_SPLL:
 		pll = I915_READ(SPLL_CTL) & SPLL_PLL_FREQ_MASK;
@@ -1184,7 +1191,7 @@ void intel_ddi_clock_get(struct intel_encoder *encoder,
 
 	if (INTEL_INFO(dev)->gen <= 8)
 		hsw_ddi_clock_get(encoder, pipe_config);
-	else if (IS_SKYLAKE(dev))
+	else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		skl_ddi_clock_get(encoder, pipe_config);
 	else if (IS_BROXTON(dev))
 		bxt_ddi_clock_get(encoder, pipe_config);
@@ -1780,7 +1787,7 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc,
 	struct intel_encoder *intel_encoder =
 		intel_ddi_get_crtc_new_encoder(crtc_state);
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		return skl_ddi_pll_select(intel_crtc, crtc_state,
 					  intel_encoder);
 	else if (IS_BROXTON(dev))
@@ -1942,7 +1949,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
 				       enum transcoder cpu_transcoder)
 {
-	uint32_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder);
+	i915_reg_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder);
 	uint32_t val = I915_READ(reg);
 
 	val &= ~(TRANS_DDI_FUNC_ENABLE | TRANS_DDI_PORT_MASK | TRANS_DDI_DP_VC_PAYLOAD_ALLOC);
@@ -2097,21 +2104,21 @@ static void skl_ddi_set_iboost(struct drm_device *dev, u32 level,
 			iboost = dp_iboost;
 		} else {
 			ddi_translations = skl_get_buf_trans_dp(dev, &n_entries);
-			iboost = ddi_translations[port].i_boost;
+			iboost = ddi_translations[level].i_boost;
 		}
 	} else if (type == INTEL_OUTPUT_EDP) {
 		if (dp_iboost) {
 			iboost = dp_iboost;
 		} else {
 			ddi_translations = skl_get_buf_trans_edp(dev, &n_entries);
-			iboost = ddi_translations[port].i_boost;
+			iboost = ddi_translations[level].i_boost;
 		}
 	} else if (type == INTEL_OUTPUT_HDMI) {
 		if (hdmi_iboost) {
 			iboost = hdmi_iboost;
 		} else {
 			ddi_translations = skl_get_buf_trans_hdmi(dev, &n_entries);
-			iboost = ddi_translations[port].i_boost;
+			iboost = ddi_translations[level].i_boost;
 		}
 	} else {
 		return;
@@ -2263,7 +2270,7 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 
 	level = translate_signal_level(signal_levels);
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		skl_ddi_set_iboost(dev, level, port, encoder->type);
 	else if (IS_BROXTON(dev))
 		bxt_ddi_vswing_sequence(dev, level, port, encoder->type);
@@ -2271,30 +2278,21 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 	return DDI_BUF_TRANS_SELECT(level);
 }
 
-static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
+void intel_ddi_clk_select(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
-	enum port port = intel_ddi_get_encoder_port(intel_encoder);
-	int type = intel_encoder->type;
-	int hdmi_level;
-
-	if (type == INTEL_OUTPUT_EDP) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-		intel_edp_panel_on(intel_dp);
-	}
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = intel_ddi_get_encoder_port(encoder);
 
-	if (IS_SKYLAKE(dev)) {
-		uint32_t dpll = crtc->config->ddi_pll_sel;
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		uint32_t dpll = pipe_config->ddi_pll_sel;
 		uint32_t val;
 
 		/*
 		 * DPLL0 is used for eDP and is the only "private" DPLL (as
 		 * opposed to shared) on SKL
 		 */
-		if (type == INTEL_OUTPUT_EDP) {
+		if (encoder->type == INTEL_OUTPUT_EDP) {
 			WARN_ON(dpll != SKL_DPLL0);
 
 			val = I915_READ(DPLL_CTRL1);
@@ -2302,7 +2300,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
 			val &= ~(DPLL_CTRL1_HDMI_MODE(dpll) |
 				 DPLL_CTRL1_SSC(dpll) |
 				 DPLL_CTRL1_LINK_RATE_MASK(dpll));
-			val |= crtc->config->dpll_hw_state.ctrl1 << (dpll * 6);
+			val |= pipe_config->dpll_hw_state.ctrl1 << (dpll * 6);
 
 			I915_WRITE(DPLL_CTRL1, val);
 			POSTING_READ(DPLL_CTRL1);
@@ -2318,11 +2316,29 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
 
 		I915_WRITE(DPLL_CTRL2, val);
 
-	} else if (INTEL_INFO(dev)->gen < 9) {
-		WARN_ON(crtc->config->ddi_pll_sel == PORT_CLK_SEL_NONE);
-		I915_WRITE(PORT_CLK_SEL(port), crtc->config->ddi_pll_sel);
+	} else if (INTEL_INFO(dev_priv)->gen < 9) {
+		WARN_ON(pipe_config->ddi_pll_sel == PORT_CLK_SEL_NONE);
+		I915_WRITE(PORT_CLK_SEL(port), pipe_config->ddi_pll_sel);
+	}
+}
+
+static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
+{
+	struct drm_encoder *encoder = &intel_encoder->base;
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+	enum port port = intel_ddi_get_encoder_port(intel_encoder);
+	int type = intel_encoder->type;
+	int hdmi_level;
+
+	if (type == INTEL_OUTPUT_EDP) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+		intel_edp_panel_on(intel_dp);
 	}
 
+	intel_ddi_clk_select(intel_encoder, crtc->config);
+
 	if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
@@ -2381,7 +2397,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
 		intel_edp_panel_off(intel_dp);
 	}
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) |
 					DPLL_CTRL2_DDI_CLK_OFF(port)));
 	else if (INTEL_INFO(dev)->gen < 9)
@@ -2553,7 +2569,7 @@ static const char * const skl_ddi_pll_names[] = {
 };
 
 struct skl_dpll_regs {
-	u32 ctl, cfgcr1, cfgcr2;
+	i915_reg_t ctl, cfgcr1, cfgcr2;
 };
 
 /* this array is indexed by the *shared* pll id */
@@ -2566,13 +2582,13 @@ static const struct skl_dpll_regs skl_dpll_regs[3] = {
 	},
 	{
 		/* DPLL 2 */
-		.ctl = WRPLL_CTL1,
+		.ctl = WRPLL_CTL(0),
 		.cfgcr1 = DPLL_CFGCR1(SKL_DPLL2),
 		.cfgcr2 = DPLL_CFGCR2(SKL_DPLL2),
 	},
 	{
 		/* DPLL 3 */
-		.ctl = WRPLL_CTL2,
+		.ctl = WRPLL_CTL(1),
 		.cfgcr1 = DPLL_CFGCR1(SKL_DPLL3),
 		.cfgcr2 = DPLL_CFGCR2(SKL_DPLL3),
 	},
@@ -2992,22 +3008,22 @@ void intel_ddi_pll_init(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	uint32_t val = I915_READ(LCPLL_CTL);
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		skl_shared_dplls_init(dev_priv);
 	else if (IS_BROXTON(dev))
 		bxt_shared_dplls_init(dev_priv);
 	else
 		hsw_shared_dplls_init(dev_priv);
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		int cdclk_freq;
 
 		cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
 		dev_priv->skl_boot_cdclk = cdclk_freq;
+		if (skl_sanitize_cdclk(dev_priv))
+			DRM_DEBUG_KMS("Sanitized cdclk programmed by pre-os\n");
 		if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE))
 			DRM_ERROR("LCPLL1 is disabled\n");
-		else
-			intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
 	} else if (IS_BROXTON(dev)) {
 		broxton_init_cdclk(dev);
 		broxton_ddi_phy_init(dev);
@@ -3026,11 +3042,11 @@ void intel_ddi_pll_init(struct drm_device *dev)
 	}
 }
 
-void intel_ddi_prepare_link_retrain(struct drm_encoder *encoder)
+void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-	struct intel_dp *intel_dp = &intel_dig_port->dp;
-	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	struct drm_i915_private *dev_priv =
+		to_i915(intel_dig_port->base.base.dev);
 	enum port port = intel_dig_port->port;
 	uint32_t val;
 	bool wait = false;
@@ -3098,6 +3114,19 @@ void intel_ddi_fdi_disable(struct drm_crtc *crtc)
 	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
 }
 
+bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
+				 struct intel_crtc *intel_crtc)
+{
+	u32 temp;
+
+	if (intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO)) {
+		temp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
+		if (temp & AUDIO_OUTPUT_ENABLE(intel_crtc->pipe))
+			return true;
+	}
+	return false;
+}
+
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config)
 {
@@ -3141,7 +3170,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 		pipe_config->has_hdmi_sink = true;
 		intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 
-		if (intel_hdmi->infoframe_enabled(&encoder->base))
+		if (intel_hdmi->infoframe_enabled(&encoder->base, pipe_config))
 			pipe_config->has_infoframe = true;
 		break;
 	case TRANS_DDI_MODE_SELECT_DVI:
@@ -3158,11 +3187,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 		break;
 	}
 
-	if (intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO)) {
-		temp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
-		if (temp & AUDIO_OUTPUT_ENABLE(intel_crtc->pipe))
-			pipe_config->has_audio = true;
-	}
+	pipe_config->has_audio =
+		intel_ddi_is_audio_enabled(dev_priv, intel_crtc);
 
 	if (encoder->type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp_bpp &&
 	    pipe_config->pipe_bpp > dev_priv->vbt.edp_bpp) {
@@ -3274,7 +3300,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	encoder = &intel_encoder->base;
 
 	drm_encoder_init(dev, encoder, &intel_ddi_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	intel_encoder->compute_config = intel_ddi_compute_config;
 	intel_encoder->enable = intel_enable_ddi;
@@ -3285,10 +3311,25 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	intel_encoder->get_config = intel_ddi_get_config;
 
 	intel_dig_port->port = port;
+	dev_priv->dig_port_map[port] = intel_encoder;
 	intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) &
 					  (DDI_BUF_PORT_REVERSAL |
 					   DDI_A_4_LANES);
 
+	/*
+	 * Bspec says that DDI_A_4_LANES is the only supported configuration
+	 * for Broxton.  Yet some BIOS fail to set this bit on port A if eDP
+	 * wasn't lit up at boot.  Force this bit on in our internal
+	 * configuration so that we use the proper lane count for our
+	 * calculations.
+	 */
+	if (IS_BROXTON(dev) && port == PORT_A) {
+		if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) {
+			DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n");
+			intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
+		}
+	}
+
 	intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 	intel_encoder->cloneable = 0;
@@ -3302,8 +3343,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 		 * On BXT A0/A1, sw needs to activate DDIA HPD logic and
 		 * interrupts to check the external panel connection.
 		 */
-		if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0)
-					 && port == PORT_B)
+		if (IS_BXT_REVID(dev, 0, BXT_REVID_A1) && port == PORT_B)
 			dev_priv->hotplug.irq_port[PORT_A] = intel_dig_port;
 		else
 			dev_priv->hotplug.irq_port[port] = intel_dig_port;
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 32cf97346978..2f00828ccc6e 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -44,6 +44,8 @@
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_rect.h>
 #include <linux/dma_remapping.h>
+#include <linux/reservation.h>
+#include <linux/dma-buf.h>
 
 /* Primary plane formats for gen <= 3 */
 static const uint32_t i8xx_primary_formats[] = {
@@ -186,7 +188,7 @@ int intel_hrawclk(struct drm_device *dev)
 	uint32_t clkcfg;
 
 	/* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		return 200;
 
 	clkcfg = I915_READ(CLKCFG);
@@ -214,7 +216,7 @@ int intel_hrawclk(struct drm_device *dev)
 
 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 {
-	if (!IS_VALLEYVIEW(dev_priv))
+	if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 		return;
 
 	dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
@@ -715,11 +717,12 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
 	if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 		INTELPllInvalid("m1 out of range\n");
 
-	if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
+	if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) &&
+	    !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev))
 		if (clock->m1 <= clock->m2)
 			INTELPllInvalid("m1 <= m2\n");
 
-	if (!IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) {
+	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev)) {
 		if (clock->p < limit->p.min || limit->p.max < clock->p)
 			INTELPllInvalid("p out of range\n");
 		if (clock->m < limit->m.min || limit->m.max < clock->m)
@@ -1096,7 +1099,7 @@ enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 reg = PIPEDSL(pipe);
+	i915_reg_t reg = PIPEDSL(pipe);
 	u32 line1, line2;
 	u32 line_mask;
 
@@ -1136,7 +1139,7 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
 	enum pipe pipe = crtc->pipe;
 
 	if (INTEL_INFO(dev)->gen >= 4) {
-		int reg = PIPECONF(cpu_transcoder);
+		i915_reg_t reg = PIPECONF(cpu_transcoder);
 
 		/* Wait for the Pipe State to go off */
 		if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
@@ -1286,7 +1289,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 			   enum pipe pipe)
 {
 	struct drm_device *dev = dev_priv->dev;
-	int pp_reg;
+	i915_reg_t pp_reg;
 	u32 val;
 	enum pipe panel_pipe = PIPE_A;
 	bool locked = true;
@@ -1304,7 +1307,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 		    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
 			panel_pipe = PIPE_B;
 		/* XXX: else fix for eDP */
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/* presumably write lock depends on pipe, not port select */
 		pp_reg = VLV_PIPE_PP_CONTROL(pipe);
 		panel_pipe = pipe;
@@ -1422,7 +1425,7 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
 			     "plane %d assertion failure, should be off on pipe %c but is still active\n",
 			     sprite, pipe_name(pipe));
 		}
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		for_each_sprite(dev_priv, pipe, sprite) {
 			u32 val = I915_READ(SPCNTR(pipe, sprite));
 			I915_STATE_WARN(val & SP_ENABLE,
@@ -1481,8 +1484,7 @@ static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
 		return false;
 
 	if (HAS_PCH_CPT(dev_priv->dev)) {
-		u32	trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
-		u32	trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
+		u32 trans_dp_ctl = I915_READ(TRANS_DP_CTL(pipe));
 		if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
 			return false;
 	} else if (IS_CHERRYVIEW(dev_priv->dev)) {
@@ -1546,12 +1548,13 @@ static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
 }
 
 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
-				   enum pipe pipe, int reg, u32 port_sel)
+				   enum pipe pipe, i915_reg_t reg,
+				   u32 port_sel)
 {
 	u32 val = I915_READ(reg);
 	I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
 	     "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
-	     reg, pipe_name(pipe));
+	     i915_mmio_reg_offset(reg), pipe_name(pipe));
 
 	I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & DP_PORT_EN) == 0
 	     && (val & DP_PIPEB_SELECT),
@@ -1559,12 +1562,12 @@ static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
 }
 
 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
-				     enum pipe pipe, int reg)
+				     enum pipe pipe, i915_reg_t reg)
 {
 	u32 val = I915_READ(reg);
 	I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val),
 	     "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
-	     reg, pipe_name(pipe));
+	     i915_mmio_reg_offset(reg), pipe_name(pipe));
 
 	I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & SDVO_ENABLE) == 0
 	     && (val & SDVO_PIPE_B_SELECT),
@@ -1600,14 +1603,11 @@ static void vlv_enable_pll(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int reg = DPLL(crtc->pipe);
+	i915_reg_t reg = DPLL(crtc->pipe);
 	u32 dpll = pipe_config->dpll_hw_state.dpll;
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
 
-	/* No really, not for ILK+ */
-	BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
-
 	/* PLL is protected by panel, make sure we can write it */
 	if (IS_MOBILE(dev_priv->dev))
 		assert_panel_unlocked(dev_priv, crtc->pipe);
@@ -1645,8 +1645,6 @@ static void chv_enable_pll(struct intel_crtc *crtc,
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
 
-	BUG_ON(!IS_CHERRYVIEW(dev_priv->dev));
-
 	mutex_lock(&dev_priv->sb_lock);
 
 	/* Enable back the 10bit clock to display controller */
@@ -1689,7 +1687,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int reg = DPLL(crtc->pipe);
+	i915_reg_t reg = DPLL(crtc->pipe);
 	u32 dpll = crtc->config->dpll_hw_state.dpll;
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
@@ -1838,7 +1836,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 			 unsigned int expected_mask)
 {
 	u32 port_mask;
-	int dpll_reg;
+	i915_reg_t dpll_reg;
 
 	switch (dport->port) {
 	case PORT_B:
@@ -1963,7 +1961,8 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 	struct drm_device *dev = dev_priv->dev;
 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	uint32_t reg, val, pipeconf_val;
+	i915_reg_t reg;
+	uint32_t val, pipeconf_val;
 
 	/* PCH only available on ILK+ */
 	BUG_ON(!HAS_PCH_SPLIT(dev));
@@ -2052,7 +2051,8 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 					    enum pipe pipe)
 {
 	struct drm_device *dev = dev_priv->dev;
-	uint32_t reg, val;
+	i915_reg_t reg;
+	uint32_t val;
 
 	/* FDI relies on the transcoder */
 	assert_fdi_tx_disabled(dev_priv, pipe);
@@ -2069,7 +2069,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 	if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
 		DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
 
-	if (!HAS_PCH_IBX(dev)) {
+	if (HAS_PCH_CPT(dev)) {
 		/* Workaround: Clear the timing override chicken bit again. */
 		reg = TRANS_CHICKEN2(pipe);
 		val = I915_READ(reg);
@@ -2107,10 +2107,9 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	enum pipe pipe = crtc->pipe;
-	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
-								      pipe);
+	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 	enum pipe pch_transcoder;
-	int reg;
+	i915_reg_t reg;
 	u32 val;
 
 	DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
@@ -2130,7 +2129,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	 * need the check.
 	 */
 	if (HAS_GMCH_DISPLAY(dev_priv->dev))
-		if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI))
+		if (crtc->config->has_dsi_encoder)
 			assert_dsi_pll_enabled(dev_priv);
 		else
 			assert_pll_enabled(dev_priv, pipe);
@@ -2171,7 +2170,7 @@ static void intel_disable_pipe(struct intel_crtc *crtc)
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
-	int reg;
+	i915_reg_t reg;
 	u32 val;
 
 	DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
@@ -2270,20 +2269,20 @@ intel_fb_align_height(struct drm_device *dev, unsigned int height,
 					       fb_format_modifier, 0));
 }
 
-static int
+static void
 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb,
 			const struct drm_plane_state *plane_state)
 {
-	struct intel_rotation_info *info = &view->rotation_info;
+	struct intel_rotation_info *info = &view->params.rotation_info;
 	unsigned int tile_height, tile_pitch;
 
 	*view = i915_ggtt_view_normal;
 
 	if (!plane_state)
-		return 0;
+		return;
 
 	if (!intel_rotation_90_or_270(plane_state->rotation))
-		return 0;
+		return;
 
 	*view = i915_ggtt_view_rotated;
 
@@ -2310,8 +2309,6 @@ intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb,
 		info->size_uv = info->width_pages_uv * info->height_pages_uv *
 				PAGE_SIZE;
 	}
-
-	return 0;
 }
 
 static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
@@ -2319,7 +2316,7 @@ static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
 	if (INTEL_INFO(dev_priv)->gen >= 9)
 		return 256 * 1024;
 	else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
-		 IS_VALLEYVIEW(dev_priv))
+		 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return 128 * 1024;
 	else if (INTEL_INFO(dev_priv)->gen >= 4)
 		return 4 * 1024;
@@ -2330,9 +2327,7 @@ static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
 int
 intel_pin_and_fence_fb_obj(struct drm_plane *plane,
 			   struct drm_framebuffer *fb,
-			   const struct drm_plane_state *plane_state,
-			   struct intel_engine_cs *pipelined,
-			   struct drm_i915_gem_request **pipelined_request)
+			   const struct drm_plane_state *plane_state)
 {
 	struct drm_device *dev = fb->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2367,9 +2362,7 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane,
 		return -EINVAL;
 	}
 
-	ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
-	if (ret)
-		return ret;
+	intel_fill_fb_ggtt_view(&view, fb, plane_state);
 
 	/* Note that the w/a also requires 64 PTE of padding following the
 	 * bo. We currently fill all unused PTE with the shadow page and so
@@ -2388,11 +2381,10 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane,
 	 */
 	intel_runtime_pm_get(dev_priv);
 
-	dev_priv->mm.interruptible = false;
-	ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined,
-						   pipelined_request, &view);
+	ret = i915_gem_object_pin_to_display_plane(obj, alignment,
+						   &view);
 	if (ret)
-		goto err_interruptible;
+		goto err_pm;
 
 	/* Install a fence for tiled scan-out. Pre-i965 always needs a
 	 * fence, whereas 965+ only requires a fence if using
@@ -2418,14 +2410,12 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane,
 		i915_gem_object_pin_fence(obj);
 	}
 
-	dev_priv->mm.interruptible = true;
 	intel_runtime_pm_put(dev_priv);
 	return 0;
 
 err_unpin:
 	i915_gem_object_unpin_from_display_plane(obj, &view);
-err_interruptible:
-	dev_priv->mm.interruptible = true;
+err_pm:
 	intel_runtime_pm_put(dev_priv);
 	return ret;
 }
@@ -2435,12 +2425,10 @@ static void intel_unpin_fb_obj(struct drm_framebuffer *fb,
 {
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct i915_ggtt_view view;
-	int ret;
 
 	WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
 
-	ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
-	WARN_ONCE(ret, "Couldn't get view from plane state!");
+	intel_fill_fb_ggtt_view(&view, fb, plane_state);
 
 	if (view.type == I915_GGTT_VIEW_NORMAL)
 		i915_gem_object_unpin_fence(obj);
@@ -2695,7 +2683,7 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
 	int plane = intel_crtc->plane;
 	unsigned long linear_offset;
 	u32 dspcntr;
-	u32 reg = DSPCNTR(plane);
+	i915_reg_t reg = DSPCNTR(plane);
 	int pixel_size;
 
 	if (!visible || !fb) {
@@ -2825,7 +2813,7 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
 	int plane = intel_crtc->plane;
 	unsigned long linear_offset;
 	u32 dspcntr;
-	u32 reg = DSPCNTR(plane);
+	i915_reg_t reg = DSPCNTR(plane);
 	int pixel_size;
 
 	if (!visible || !fb) {
@@ -2950,30 +2938,32 @@ u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier,
 	}
 }
 
-unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
-				     struct drm_i915_gem_object *obj,
-				     unsigned int plane)
+u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
+			   struct drm_i915_gem_object *obj,
+			   unsigned int plane)
 {
-	const struct i915_ggtt_view *view = &i915_ggtt_view_normal;
+	struct i915_ggtt_view view;
 	struct i915_vma *vma;
-	unsigned char *offset;
+	u64 offset;
 
-	if (intel_rotation_90_or_270(intel_plane->base.state->rotation))
-		view = &i915_ggtt_view_rotated;
+	intel_fill_fb_ggtt_view(&view, intel_plane->base.fb,
+				intel_plane->base.state);
 
-	vma = i915_gem_obj_to_ggtt_view(obj, view);
+	vma = i915_gem_obj_to_ggtt_view(obj, &view);
 	if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
-		view->type))
+		view.type))
 		return -1;
 
-	offset = (unsigned char *)vma->node.start;
+	offset = vma->node.start;
 
 	if (plane == 1) {
-		offset += vma->ggtt_view.rotation_info.uv_start_page *
+		offset += vma->ggtt_view.params.rotation_info.uv_start_page *
 			  PAGE_SIZE;
 	}
 
-	return (unsigned long)offset;
+	WARN_ON(upper_32_bits(offset));
+
+	return lower_32_bits(offset);
 }
 
 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
@@ -3099,7 +3089,7 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc,
 	u32 tile_height, plane_offset, plane_size;
 	unsigned int rotation;
 	int x_offset, y_offset;
-	unsigned long surf_addr;
+	u32 surf_addr;
 	struct intel_crtc_state *crtc_state = intel_crtc->config;
 	struct intel_plane_state *plane_state;
 	int src_x = 0, src_y = 0, src_w = 0, src_h = 0;
@@ -3197,8 +3187,8 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	if (dev_priv->fbc.disable_fbc)
-		dev_priv->fbc.disable_fbc(dev_priv);
+	if (dev_priv->fbc.deactivate)
+		dev_priv->fbc.deactivate(dev_priv);
 
 	dev_priv->display.update_primary_plane(crtc, fb, x, y);
 
@@ -3227,10 +3217,9 @@ static void intel_update_primary_planes(struct drm_device *dev)
 		struct intel_plane_state *plane_state;
 
 		drm_modeset_lock_crtc(crtc, &plane->base);
-
 		plane_state = to_intel_plane_state(plane->base.state);
 
-		if (plane_state->base.fb)
+		if (crtc->state->active && plane_state->base.fb)
 			plane->commit_plane(&plane->base, plane_state);
 
 		drm_modeset_unlock_crtc(crtc);
@@ -3306,32 +3295,6 @@ void intel_finish_reset(struct drm_device *dev)
 	drm_modeset_unlock_all(dev);
 }
 
-static void
-intel_finish_fb(struct drm_framebuffer *old_fb)
-{
-	struct drm_i915_gem_object *obj = intel_fb_obj(old_fb);
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	bool was_interruptible = dev_priv->mm.interruptible;
-	int ret;
-
-	/* Big Hammer, we also need to ensure that any pending
-	 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
-	 * current scanout is retired before unpinning the old
-	 * framebuffer. Note that we rely on userspace rendering
-	 * into the buffer attached to the pipe they are waiting
-	 * on. If not, userspace generates a GPU hang with IPEHR
-	 * point to the MI_WAIT_FOR_EVENT.
-	 *
-	 * This should only fail upon a hung GPU, in which case we
-	 * can safely continue.
-	 */
-	dev_priv->mm.interruptible = false;
-	ret = i915_gem_object_wait_rendering(obj, true);
-	dev_priv->mm.interruptible = was_interruptible;
-
-	WARN_ON(ret);
-}
-
 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
@@ -3401,7 +3364,8 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
+	i915_reg_t reg;
+	u32 temp;
 
 	/* enable normal train */
 	reg = FDI_TX_CTL(pipe);
@@ -3443,7 +3407,8 @@ static void ironlake_fdi_link_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp, tries;
+	i915_reg_t reg;
+	u32 temp, tries;
 
 	/* FDI needs bits from pipe first */
 	assert_pipe_enabled(dev_priv, pipe);
@@ -3543,7 +3508,8 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp, i, retry;
+	i915_reg_t reg;
+	u32 temp, i, retry;
 
 	/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
 	   for train result */
@@ -3675,7 +3641,8 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp, i, j;
+	i915_reg_t reg;
+	u32 temp, i, j;
 
 	/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
 	   for train result */
@@ -3792,8 +3759,8 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
-
+	i915_reg_t reg;
+	u32 temp;
 
 	/* enable PCH FDI RX PLL, wait warmup plus DMI latency */
 	reg = FDI_RX_CTL(pipe);
@@ -3829,7 +3796,8 @@ static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
+	i915_reg_t reg;
+	u32 temp;
 
 	/* Switch from PCDclk to Rawclk */
 	reg = FDI_RX_CTL(pipe);
@@ -3859,7 +3827,8 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
+	i915_reg_t reg;
+	u32 temp;
 
 	/* disable CPU FDI tx and PCH FDI rx */
 	reg = FDI_TX_CTL(pipe);
@@ -3952,15 +3921,23 @@ static void page_flip_completed(struct intel_crtc *intel_crtc)
 				 work->pending_flip_obj);
 }
 
-void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
+static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	long ret;
 
 	WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
-	if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
-				       !intel_crtc_has_pending_flip(crtc),
-				       60*HZ) == 0)) {
+
+	ret = wait_event_interruptible_timeout(
+					dev_priv->pending_flip_queue,
+					!intel_crtc_has_pending_flip(crtc),
+					60*HZ);
+
+	if (ret < 0)
+		return ret;
+
+	if (ret == 0) {
 		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
 		spin_lock_irq(&dev->event_lock);
@@ -3971,11 +3948,22 @@ void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 		spin_unlock_irq(&dev->event_lock);
 	}
 
-	if (crtc->primary->fb) {
-		mutex_lock(&dev->struct_mutex);
-		intel_finish_fb(crtc->primary->fb);
-		mutex_unlock(&dev->struct_mutex);
-	}
+	return 0;
+}
+
+static void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
+{
+	u32 temp;
+
+	I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
+
+	mutex_lock(&dev_priv->sb_lock);
+
+	temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
+	temp |= SBI_SSCCTL_DISABLE;
+	intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
+
+	mutex_unlock(&dev_priv->sb_lock);
 }
 
 /* Program iCLKIP clock to the desired frequency */
@@ -3987,18 +3975,7 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
 	u32 divsel, phaseinc, auxdiv, phasedir = 0;
 	u32 temp;
 
-	mutex_lock(&dev_priv->sb_lock);
-
-	/* It is necessary to ungate the pixclk gate prior to programming
-	 * the divisors, and gate it back when it is done.
-	 */
-	I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
-
-	/* Disable SSCCTL */
-	intel_sbi_write(dev_priv, SBI_SSCCTL6,
-			intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK) |
-				SBI_SSCCTL_DISABLE,
-			SBI_ICLK);
+	lpt_disable_iclkip(dev_priv);
 
 	/* 20MHz is a corner case which is out of range for the 7-bit divisor */
 	if (clock == 20000) {
@@ -4016,7 +3993,7 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
 		u32 iclk_pi_range = 64;
 		u32 desired_divisor, msb_divisor_value, pi_value;
 
-		desired_divisor = (iclk_virtual_root_freq / clock);
+		desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq, clock);
 		msb_divisor_value = desired_divisor / iclk_pi_range;
 		pi_value = desired_divisor % iclk_pi_range;
 
@@ -4038,6 +4015,8 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
 			phasedir,
 			phaseinc);
 
+	mutex_lock(&dev_priv->sb_lock);
+
 	/* Program SSCDIVINTPHASE6 */
 	temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
 	temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
@@ -4059,12 +4038,12 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
 	temp &= ~SBI_SSCCTL_DISABLE;
 	intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
 
+	mutex_unlock(&dev_priv->sb_lock);
+
 	/* Wait for initialization time */
 	udelay(24);
 
 	I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
-
-	mutex_unlock(&dev_priv->sb_lock);
 }
 
 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
@@ -4135,6 +4114,22 @@ static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
 	}
 }
 
+/* Return which DP Port should be selected for Transcoder DP control */
+static enum port
+intel_trans_dp_port_sel(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct intel_encoder *encoder;
+
+	for_each_encoder_on_crtc(dev, crtc, encoder) {
+		if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+		    encoder->type == INTEL_OUTPUT_EDP)
+			return enc_to_dig_port(&encoder->base)->port;
+	}
+
+	return -1;
+}
+
 /*
  * Enable PCH resources required for PCH ports:
  *   - PCH PLLs
@@ -4149,7 +4144,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
+	u32 temp;
 
 	assert_pch_transcoder_disabled(dev_priv, pipe);
 
@@ -4161,6 +4156,12 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	I915_WRITE(FDI_RX_TUSIZE1(pipe),
 		   I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
 
+	/*
+	 * Sometimes spurious CPU pipe underruns happen during FDI
+	 * training, at least with VGA+HDMI cloning. Suppress them.
+	 */
+	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+
 	/* For PCH output, training FDI link */
 	dev_priv->display.fdi_link_train(crtc);
 
@@ -4194,10 +4195,14 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
 	intel_fdi_normal_train(crtc);
 
+	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+
 	/* For PCH DP, enable TRANS_DP_CTL */
 	if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
+		const struct drm_display_mode *adjusted_mode =
+			&intel_crtc->config->base.adjusted_mode;
 		u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
-		reg = TRANS_DP_CTL(pipe);
+		i915_reg_t reg = TRANS_DP_CTL(pipe);
 		temp = I915_READ(reg);
 		temp &= ~(TRANS_DP_PORT_SEL_MASK |
 			  TRANS_DP_SYNC_MASK |
@@ -4205,19 +4210,19 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 		temp |= TRANS_DP_OUTPUT_ENABLE;
 		temp |= bpc << 9; /* same format but at 11:9 */
 
-		if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
+		if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
 			temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
-		if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
+		if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
 			temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
 
 		switch (intel_trans_dp_port_sel(crtc)) {
-		case PCH_DP_B:
+		case PORT_B:
 			temp |= TRANS_DP_PORT_SEL_B;
 			break;
-		case PCH_DP_C:
+		case PORT_C:
 			temp |= TRANS_DP_PORT_SEL_C;
 			break;
-		case PCH_DP_D:
+		case PORT_D:
 			temp |= TRANS_DP_PORT_SEL_D;
 			break;
 		default:
@@ -4357,7 +4362,7 @@ static void intel_shared_dpll_commit(struct drm_atomic_state *state)
 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int dslreg = PIPEDSL(pipe);
+	i915_reg_t dslreg = PIPEDSL(pipe);
 	u32 temp;
 
 	temp = I915_READ(dslreg);
@@ -4650,7 +4655,7 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc)
 		return;
 
 	if (HAS_GMCH_DISPLAY(dev_priv->dev)) {
-		if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI))
+		if (intel_crtc->config->has_dsi_encoder)
 			assert_dsi_pll_enabled(dev_priv);
 		else
 			assert_pll_enabled(dev_priv, pipe);
@@ -4667,7 +4672,7 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc)
 	}
 
 	for (i = 0; i < 256; i++) {
-		u32 palreg;
+		i915_reg_t palreg;
 
 		if (HAS_GMCH_DISPLAY(dev))
 			palreg = PALETTE(pipe, i);
@@ -4721,14 +4726,6 @@ intel_post_enable_primary(struct drm_crtc *crtc)
 	int pipe = intel_crtc->pipe;
 
 	/*
-	 * BDW signals flip done immediately if the plane
-	 * is disabled, even if the plane enable is already
-	 * armed to occur at the next vblank :(
-	 */
-	if (IS_BROADWELL(dev))
-		intel_wait_for_vblank(dev, pipe);
-
-	/*
 	 * FIXME IPS should be fine as long as one plane is
 	 * enabled, but in practice it seems to have problems
 	 * when going from primary only to sprite only and vice
@@ -4746,9 +4743,9 @@ intel_post_enable_primary(struct drm_crtc *crtc)
 	if (IS_GEN2(dev))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
-	/* Underruns don't raise interrupts, so check manually. */
-	if (HAS_GMCH_DISPLAY(dev))
-		i9xx_check_fifo_underruns(dev_priv);
+	/* Underruns don't always raise interrupts, so check manually. */
+	intel_check_cpu_fifo_underruns(dev_priv);
+	intel_check_pch_fifo_underruns(dev_priv);
 }
 
 /**
@@ -4805,31 +4802,26 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
 static void intel_post_plane_update(struct intel_crtc *crtc)
 {
 	struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
+	struct intel_crtc_state *pipe_config =
+		to_intel_crtc_state(crtc->base.state);
 	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_plane *plane;
 
 	if (atomic->wait_vblank)
 		intel_wait_for_vblank(dev, crtc->pipe);
 
 	intel_frontbuffer_flip(dev, atomic->fb_bits);
 
-	if (atomic->disable_cxsr)
-		crtc->wm.cxsr_allowed = true;
+	crtc->wm.cxsr_allowed = true;
 
-	if (crtc->atomic.update_wm_post)
+	if (pipe_config->wm_changed && pipe_config->base.active)
 		intel_update_watermarks(&crtc->base);
 
 	if (atomic->update_fbc)
-		intel_fbc_update(dev_priv);
+		intel_fbc_update(crtc);
 
 	if (atomic->post_enable_primary)
 		intel_post_enable_primary(&crtc->base);
 
-	drm_for_each_plane_mask(plane, dev, atomic->update_sprite_watermarks)
-		intel_update_sprite_watermarks(plane, &crtc->base,
-					       0, 0, 0, false, false);
-
 	memset(atomic, 0, sizeof(*atomic));
 }
 
@@ -4838,23 +4830,11 @@ static void intel_pre_plane_update(struct intel_crtc *crtc)
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
-	struct drm_plane *p;
-
-	/* Track fb's for any planes being disabled */
-	drm_for_each_plane_mask(p, dev, atomic->disabled_planes) {
-		struct intel_plane *plane = to_intel_plane(p);
-
-		mutex_lock(&dev->struct_mutex);
-		i915_gem_track_fb(intel_fb_obj(plane->base.fb), NULL,
-				  plane->frontbuffer_bit);
-		mutex_unlock(&dev->struct_mutex);
-	}
-
-	if (atomic->wait_for_flips)
-		intel_crtc_wait_for_pending_flips(&crtc->base);
+	struct intel_crtc_state *pipe_config =
+		to_intel_crtc_state(crtc->base.state);
 
 	if (atomic->disable_fbc)
-		intel_fbc_disable_crtc(crtc);
+		intel_fbc_deactivate(crtc);
 
 	if (crtc->atomic.disable_ips)
 		hsw_disable_ips(crtc);
@@ -4862,10 +4842,13 @@ static void intel_pre_plane_update(struct intel_crtc *crtc)
 	if (atomic->pre_disable_primary)
 		intel_pre_disable_primary(&crtc->base);
 
-	if (atomic->disable_cxsr) {
+	if (pipe_config->disable_cxsr) {
 		crtc->wm.cxsr_allowed = false;
 		intel_set_memory_cxsr(dev_priv, false);
 	}
+
+	if (!needs_modeset(&pipe_config->base) && pipe_config->wm_changed)
+		intel_update_watermarks(&crtc->base);
 }
 
 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
@@ -4900,6 +4883,9 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 		return;
 
 	if (intel_crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
+
+	if (intel_crtc->config->has_pch_encoder)
 		intel_prepare_shared_dpll(intel_crtc);
 
 	if (intel_crtc->config->has_dp_encoder)
@@ -4917,7 +4903,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 	intel_crtc->active = true;
 
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
-	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
 
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		if (encoder->pre_enable)
@@ -4955,6 +4940,13 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 
 	if (HAS_PCH_CPT(dev))
 		cpt_verify_modeset(dev, intel_crtc->pipe);
+
+	/* Must wait for vblank to avoid spurious PCH FIFO underruns */
+	if (intel_crtc->config->has_pch_encoder)
+		intel_wait_for_vblank(dev, pipe);
+	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
+
+	intel_fbc_enable(intel_crtc);
 }
 
 /* IPS only exists on ULT machines and is tied to pipe A. */
@@ -4972,11 +4964,14 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	int pipe = intel_crtc->pipe, hsw_workaround_pipe;
 	struct intel_crtc_state *pipe_config =
 		to_intel_crtc_state(crtc->state);
-	bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
 
 	if (WARN_ON(intel_crtc->active))
 		return;
 
+	if (intel_crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
+						      false);
+
 	if (intel_crtc_to_shared_dpll(intel_crtc))
 		intel_enable_shared_dpll(intel_crtc);
 
@@ -5001,21 +4996,20 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 
 	intel_crtc->active = true;
 
-	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+	if (intel_crtc->config->has_pch_encoder)
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+	else
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+
 	for_each_encoder_on_crtc(dev, crtc, encoder) {
-		if (encoder->pre_pll_enable)
-			encoder->pre_pll_enable(encoder);
 		if (encoder->pre_enable)
 			encoder->pre_enable(encoder);
 	}
 
-	if (intel_crtc->config->has_pch_encoder) {
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      true);
+	if (intel_crtc->config->has_pch_encoder)
 		dev_priv->display.fdi_link_train(crtc);
-	}
 
-	if (!is_dsi)
+	if (!intel_crtc->config->has_dsi_encoder)
 		intel_ddi_enable_pipe_clock(intel_crtc);
 
 	if (INTEL_INFO(dev)->gen >= 9)
@@ -5030,7 +5024,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	intel_crtc_load_lut(crtc);
 
 	intel_ddi_set_pipe_settings(crtc);
-	if (!is_dsi)
+	if (!intel_crtc->config->has_dsi_encoder)
 		intel_ddi_enable_transcoder_func(crtc);
 
 	intel_update_watermarks(crtc);
@@ -5039,7 +5033,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	if (intel_crtc->config->has_pch_encoder)
 		lpt_pch_enable(crtc);
 
-	if (intel_crtc->config->dp_encoder_is_mst && !is_dsi)
+	if (intel_crtc->config->dp_encoder_is_mst)
 		intel_ddi_set_vc_payload_alloc(crtc, true);
 
 	assert_vblank_disabled(crtc);
@@ -5050,6 +5044,14 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 		intel_opregion_notify_encoder(encoder, true);
 	}
 
+	if (intel_crtc->config->has_pch_encoder) {
+		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev, pipe);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
+						      true);
+	}
+
 	/* If we change the relative order between pipe/planes enabling, we need
 	 * to change the workaround. */
 	hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
@@ -5057,6 +5059,8 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 		intel_wait_for_vblank(dev, hsw_workaround_pipe);
 		intel_wait_for_vblank(dev, hsw_workaround_pipe);
 	}
+
+	intel_fbc_enable(intel_crtc);
 }
 
 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
@@ -5081,7 +5085,9 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp;
+
+	if (intel_crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
 
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		encoder->disable(encoder);
@@ -5089,15 +5095,22 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
 
+	/*
+	 * Sometimes spurious CPU pipe underruns happen when the
+	 * pipe is already disabled, but FDI RX/TX is still enabled.
+	 * Happens at least with VGA+HDMI cloning. Suppress them.
+	 */
 	if (intel_crtc->config->has_pch_encoder)
-		intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
 	intel_disable_pipe(intel_crtc);
 
 	ironlake_pfit_disable(intel_crtc, false);
 
-	if (intel_crtc->config->has_pch_encoder)
+	if (intel_crtc->config->has_pch_encoder) {
 		ironlake_fdi_disable(crtc);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+	}
 
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		if (encoder->post_disable)
@@ -5107,6 +5120,9 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 		ironlake_disable_pch_transcoder(dev_priv, pipe);
 
 		if (HAS_PCH_CPT(dev)) {
+			i915_reg_t reg;
+			u32 temp;
+
 			/* disable TRANS_DP_CTL */
 			reg = TRANS_DP_CTL(pipe);
 			temp = I915_READ(reg);
@@ -5123,6 +5139,10 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
 		ironlake_fdi_pll_disable(intel_crtc);
 	}
+
+	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
+
+	intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void haswell_crtc_disable(struct drm_crtc *crtc)
@@ -5132,7 +5152,10 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_encoder *encoder;
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
-	bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
+
+	if (intel_crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
+						      false);
 
 	for_each_encoder_on_crtc(dev, crtc, encoder) {
 		intel_opregion_notify_encoder(encoder, false);
@@ -5142,15 +5165,12 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
 
-	if (intel_crtc->config->has_pch_encoder)
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      false);
 	intel_disable_pipe(intel_crtc);
 
 	if (intel_crtc->config->dp_encoder_is_mst)
 		intel_ddi_set_vc_payload_alloc(crtc, false);
 
-	if (!is_dsi)
+	if (!intel_crtc->config->has_dsi_encoder)
 		intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
 
 	if (INTEL_INFO(dev)->gen >= 9)
@@ -5158,17 +5178,23 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	else
 		ironlake_pfit_disable(intel_crtc, false);
 
-	if (!is_dsi)
+	if (!intel_crtc->config->has_dsi_encoder)
 		intel_ddi_disable_pipe_clock(intel_crtc);
 
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		if (encoder->post_disable)
+			encoder->post_disable(encoder);
+
 	if (intel_crtc->config->has_pch_encoder) {
 		lpt_disable_pch_transcoder(dev_priv);
+		lpt_disable_iclkip(dev_priv);
 		intel_ddi_fdi_disable(crtc);
+
+		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
+						      true);
 	}
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->post_disable)
-			encoder->post_disable(encoder);
+	intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
@@ -5199,15 +5225,15 @@ static enum intel_display_power_domain port_to_power_domain(enum port port)
 {
 	switch (port) {
 	case PORT_A:
-		return POWER_DOMAIN_PORT_DDI_A_4_LANES;
+		return POWER_DOMAIN_PORT_DDI_A_LANES;
 	case PORT_B:
-		return POWER_DOMAIN_PORT_DDI_B_4_LANES;
+		return POWER_DOMAIN_PORT_DDI_B_LANES;
 	case PORT_C:
-		return POWER_DOMAIN_PORT_DDI_C_4_LANES;
+		return POWER_DOMAIN_PORT_DDI_C_LANES;
 	case PORT_D:
-		return POWER_DOMAIN_PORT_DDI_D_4_LANES;
+		return POWER_DOMAIN_PORT_DDI_D_LANES;
 	case PORT_E:
-		return POWER_DOMAIN_PORT_DDI_E_2_LANES;
+		return POWER_DOMAIN_PORT_DDI_E_LANES;
 	default:
 		MISSING_CASE(port);
 		return POWER_DOMAIN_PORT_OTHER;
@@ -5234,10 +5260,6 @@ static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
 	}
 }
 
-#define for_each_power_domain(domain, mask)				\
-	for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)	\
-		if ((1 << (domain)) & (mask))
-
 enum intel_display_power_domain
 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
 {
@@ -5302,13 +5324,11 @@ static unsigned long get_crtc_power_domains(struct drm_crtc *crtc)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	enum pipe pipe = intel_crtc->pipe;
 	unsigned long mask;
-	enum transcoder transcoder;
+	enum transcoder transcoder = intel_crtc->config->cpu_transcoder;
 
 	if (!crtc->state->active)
 		return 0;
 
-	transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
-
 	mask = BIT(POWER_DOMAIN_PIPE(pipe));
 	mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
 	if (intel_crtc->config->pch_pfit.enabled ||
@@ -5395,7 +5415,7 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
 
 		if (limit == SKL_DFSM_CDCLK_LIMIT_675)
@@ -5452,7 +5472,7 @@ static void intel_update_cdclk(struct drm_device *dev)
 	 * BSpec erroneously claims we should aim for 4MHz, but
 	 * in fact 1MHz is the correct frequency.
 	 */
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/*
 		 * Program the gmbus_freq based on the cdclk frequency.
 		 * BSpec erroneously claims we should aim for 4MHz, but
@@ -5812,32 +5832,16 @@ void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 	if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
 		DRM_ERROR("DBuf power disable timeout\n");
 
-	/*
-	 * DMC assumes ownership of LCPLL and will get confused if we touch it.
-	 */
-	if (dev_priv->csr.dmc_payload) {
-		/* disable DPLL0 */
-		I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) &
-					~LCPLL_PLL_ENABLE);
-		if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
-			DRM_ERROR("Couldn't disable DPLL0\n");
-	}
-
-	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
+	/* disable DPLL0 */
+	I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
+	if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
+		DRM_ERROR("Couldn't disable DPLL0\n");
 }
 
 void skl_init_cdclk(struct drm_i915_private *dev_priv)
 {
-	u32 val;
 	unsigned int required_vco;
 
-	/* enable PCH reset handshake */
-	val = I915_READ(HSW_NDE_RSTWRN_OPT);
-	I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
-
-	/* enable PG1 and Misc I/O */
-	intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
-
 	/* DPLL0 not enabled (happens on early BIOS versions) */
 	if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) {
 		/* enable DPLL0 */
@@ -5858,6 +5862,45 @@ void skl_init_cdclk(struct drm_i915_private *dev_priv)
 		DRM_ERROR("DBuf power enable timeout\n");
 }
 
+int skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
+{
+	uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
+	uint32_t cdctl = I915_READ(CDCLK_CTL);
+	int freq = dev_priv->skl_boot_cdclk;
+
+	/*
+	 * check if the pre-os intialized the display
+	 * There is SWF18 scratchpad register defined which is set by the
+	 * pre-os which can be used by the OS drivers to check the status
+	 */
+	if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
+		goto sanitize;
+
+	/* Is PLL enabled and locked ? */
+	if (!((lcpll1 & LCPLL_PLL_ENABLE) && (lcpll1 & LCPLL_PLL_LOCK)))
+		goto sanitize;
+
+	/* DPLL okay; verify the cdclock
+	 *
+	 * Noticed in some instances that the freq selection is correct but
+	 * decimal part is programmed wrong from BIOS where pre-os does not
+	 * enable display. Verify the same as well.
+	 */
+	if (cdctl == ((cdctl & CDCLK_FREQ_SEL_MASK) | skl_cdclk_decimal(freq)))
+		/* All well; nothing to sanitize */
+		return false;
+sanitize:
+	/*
+	 * As of now initialize with max cdclk till
+	 * we get dynamic cdclk support
+	 * */
+	dev_priv->skl_boot_cdclk = dev_priv->max_cdclk_freq;
+	skl_init_cdclk(dev_priv);
+
+	/* we did have to sanitize */
+	return true;
+}
+
 /* Adjust CDclk dividers to allow high res or save power if possible */
 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
 {
@@ -6139,13 +6182,10 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
-	bool is_dsi;
 
 	if (WARN_ON(intel_crtc->active))
 		return;
 
-	is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
-
 	if (intel_crtc->config->has_dp_encoder)
 		intel_dp_set_m_n(intel_crtc, M1_N1);
 
@@ -6168,7 +6208,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 		if (encoder->pre_pll_enable)
 			encoder->pre_pll_enable(encoder);
 
-	if (!is_dsi) {
+	if (!intel_crtc->config->has_dsi_encoder) {
 		if (IS_CHERRYVIEW(dev)) {
 			chv_prepare_pll(intel_crtc, intel_crtc->config);
 			chv_enable_pll(intel_crtc, intel_crtc->config);
@@ -6247,6 +6287,8 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		encoder->enable(encoder);
+
+	intel_fbc_enable(intel_crtc);
 }
 
 static void i9xx_pfit_disable(struct intel_crtc *crtc)
@@ -6294,7 +6336,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 		if (encoder->post_disable)
 			encoder->post_disable(encoder);
 
-	if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) {
+	if (!intel_crtc->config->has_dsi_encoder) {
 		if (IS_CHERRYVIEW(dev))
 			chv_disable_pll(dev_priv, pipe);
 		else if (IS_VALLEYVIEW(dev))
@@ -6309,6 +6351,8 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 
 	if (!IS_GEN2(dev))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+
+	intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
@@ -6322,7 +6366,8 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
 		return;
 
 	if (to_intel_plane_state(crtc->primary->state)->visible) {
-		intel_crtc_wait_for_pending_flips(crtc);
+		WARN_ON(intel_crtc->unpin_work);
+
 		intel_pre_disable_primary(crtc);
 
 		intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
@@ -6447,13 +6492,11 @@ static void intel_connector_check_state(struct intel_connector *connector)
 
 int intel_connector_init(struct intel_connector *connector)
 {
-	struct drm_connector_state *connector_state;
+	drm_atomic_helper_connector_reset(&connector->base);
 
-	connector_state = kzalloc(sizeof *connector_state, GFP_KERNEL);
-	if (!connector_state)
+	if (!connector->base.state)
 		return -ENOMEM;
 
-	connector->base.state = connector_state;
 	return 0;
 }
 
@@ -6642,6 +6685,15 @@ static void hsw_compute_ips_config(struct intel_crtc *crtc,
 		pipe_config_supports_ips(dev_priv, pipe_config);
 }
 
+static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
+{
+	const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+	/* GDG double wide on either pipe, otherwise pipe A only */
+	return INTEL_INFO(dev_priv)->gen < 4 &&
+		(crtc->pipe == PIPE_A || IS_I915G(dev_priv));
+}
+
 static int intel_crtc_compute_config(struct intel_crtc *crtc,
 				     struct intel_crtc_state *pipe_config)
 {
@@ -6651,23 +6703,24 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 
 	/* FIXME should check pixel clock limits on all platforms */
 	if (INTEL_INFO(dev)->gen < 4) {
-		int clock_limit = dev_priv->max_cdclk_freq;
+		int clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
 
 		/*
-		 * Enable pixel doubling when the dot clock
+		 * Enable double wide mode when the dot clock
 		 * is > 90% of the (display) core speed.
-		 *
-		 * GDG double wide on either pipe,
-		 * otherwise pipe A only.
 		 */
-		if ((crtc->pipe == PIPE_A || IS_I915G(dev)) &&
-		    adjusted_mode->crtc_clock > clock_limit * 9 / 10) {
+		if (intel_crtc_supports_double_wide(crtc) &&
+		    adjusted_mode->crtc_clock > clock_limit) {
 			clock_limit *= 2;
 			pipe_config->double_wide = true;
 		}
 
-		if (adjusted_mode->crtc_clock > clock_limit * 9 / 10)
+		if (adjusted_mode->crtc_clock > clock_limit) {
+			DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
+				      adjusted_mode->crtc_clock, clock_limit,
+				      yesno(pipe_config->double_wide));
 			return -EINVAL;
+		}
 	}
 
 	/*
@@ -7144,7 +7197,7 @@ static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
 
 	WARN_ON(!crtc_state->base.state);
 
-	if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) {
 		refclk = 100000;
 	} else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
 	    intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
@@ -7432,7 +7485,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc,
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int pipe = crtc->pipe;
-	int dpll_reg = DPLL(crtc->pipe);
+	i915_reg_t dpll_reg = DPLL(crtc->pipe);
 	enum dpio_channel port = vlv_pipe_to_channel(pipe);
 	u32 loopfilter, tribuf_calcntr;
 	u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
@@ -7843,7 +7896,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 		pipeconf |= PIPECONF_DOUBLE_WIDE;
 
 	/* only g4x and later have fancy bpc/dither controls */
-	if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/* Bspec claims that we can't use dithering for 30bpp pipes. */
 		if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
 			pipeconf |= PIPECONF_DITHER_EN |
@@ -7883,7 +7936,8 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 	} else
 		pipeconf |= PIPECONF_PROGRESSIVE;
 
-	if (IS_VALLEYVIEW(dev) && intel_crtc->config->limited_color_range)
+	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	     intel_crtc->config->limited_color_range)
 		pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
 	I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
@@ -7898,8 +7952,6 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
 	int refclk, num_connectors = 0;
 	intel_clock_t clock;
 	bool ok;
-	bool is_dsi = false;
-	struct intel_encoder *encoder;
 	const intel_limit_t *limit;
 	struct drm_atomic_state *state = crtc_state->base.state;
 	struct drm_connector *connector;
@@ -7909,26 +7961,14 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
 	memset(&crtc_state->dpll_hw_state, 0,
 	       sizeof(crtc_state->dpll_hw_state));
 
-	for_each_connector_in_state(state, connector, connector_state, i) {
-		if (connector_state->crtc != &crtc->base)
-			continue;
-
-		encoder = to_intel_encoder(connector_state->best_encoder);
-
-		switch (encoder->type) {
-		case INTEL_OUTPUT_DSI:
-			is_dsi = true;
-			break;
-		default:
-			break;
-		}
+	if (crtc_state->has_dsi_encoder)
+		return 0;
 
-		num_connectors++;
+	for_each_connector_in_state(state, connector, connector_state, i) {
+		if (connector_state->crtc == &crtc->base)
+			num_connectors++;
 	}
 
-	if (is_dsi)
-		return 0;
-
 	if (!crtc_state->clock_set) {
 		refclk = i9xx_get_refclk(crtc_state, num_connectors);
 
@@ -8144,7 +8184,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	if (!(tmp & PIPECONF_ENABLE))
 		return false;
 
-	if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		switch (tmp & PIPECONF_BPC_MASK) {
 		case PIPECONF_6BPC:
 			pipe_config->pipe_bpp = 18;
@@ -8160,7 +8200,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 		}
 	}
 
-	if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT))
+	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	    (tmp & PIPECONF_COLOR_RANGE_SELECT))
 		pipe_config->limited_color_range = true;
 
 	if (INTEL_INFO(dev)->gen < 4)
@@ -8188,7 +8229,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 		pipe_config->pixel_multiplier = 1;
 	}
 	pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
-	if (!IS_VALLEYVIEW(dev)) {
+	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 		/*
 		 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
 		 * on 830. Filter it out here so that we don't
@@ -8540,6 +8581,67 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
+#define BEND_IDX(steps) ((50 + (steps)) / 5)
+
+static const uint16_t sscdivintphase[] = {
+	[BEND_IDX( 50)] = 0x3B23,
+	[BEND_IDX( 45)] = 0x3B23,
+	[BEND_IDX( 40)] = 0x3C23,
+	[BEND_IDX( 35)] = 0x3C23,
+	[BEND_IDX( 30)] = 0x3D23,
+	[BEND_IDX( 25)] = 0x3D23,
+	[BEND_IDX( 20)] = 0x3E23,
+	[BEND_IDX( 15)] = 0x3E23,
+	[BEND_IDX( 10)] = 0x3F23,
+	[BEND_IDX(  5)] = 0x3F23,
+	[BEND_IDX(  0)] = 0x0025,
+	[BEND_IDX( -5)] = 0x0025,
+	[BEND_IDX(-10)] = 0x0125,
+	[BEND_IDX(-15)] = 0x0125,
+	[BEND_IDX(-20)] = 0x0225,
+	[BEND_IDX(-25)] = 0x0225,
+	[BEND_IDX(-30)] = 0x0325,
+	[BEND_IDX(-35)] = 0x0325,
+	[BEND_IDX(-40)] = 0x0425,
+	[BEND_IDX(-45)] = 0x0425,
+	[BEND_IDX(-50)] = 0x0525,
+};
+
+/*
+ * Bend CLKOUT_DP
+ * steps -50 to 50 inclusive, in steps of 5
+ * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
+ * change in clock period = -(steps / 10) * 5.787 ps
+ */
+static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
+{
+	uint32_t tmp;
+	int idx = BEND_IDX(steps);
+
+	if (WARN_ON(steps % 5 != 0))
+		return;
+
+	if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
+		return;
+
+	mutex_lock(&dev_priv->sb_lock);
+
+	if (steps % 10 != 0)
+		tmp = 0xAAAAAAAB;
+	else
+		tmp = 0x00000000;
+	intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
+
+	tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
+	tmp &= 0xffff0000;
+	tmp |= sscdivintphase[idx];
+	intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
+
+	mutex_unlock(&dev_priv->sb_lock);
+}
+
+#undef BEND_IDX
+
 static void lpt_init_pch_refclk(struct drm_device *dev)
 {
 	struct intel_encoder *encoder;
@@ -8555,10 +8657,12 @@ static void lpt_init_pch_refclk(struct drm_device *dev)
 		}
 	}
 
-	if (has_vga)
+	if (has_vga) {
+		lpt_bend_clkout_dp(to_i915(dev), 0);
 		lpt_enable_clkout_dp(dev, true, true);
-	else
+	} else {
 		lpt_disable_clkout_dp(dev);
+	}
 }
 
 /*
@@ -8921,7 +9025,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 	memset(&crtc_state->dpll_hw_state, 0,
 	       sizeof(crtc_state->dpll_hw_state));
 
-	is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS);
+	is_lvds = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS);
 
 	WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
 	     "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
@@ -9350,8 +9454,8 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 
 	I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
 	I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
-	I915_STATE_WARN(I915_READ(WRPLL_CTL1) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
-	I915_STATE_WARN(I915_READ(WRPLL_CTL2) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
+	I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
+	I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
 	I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
 	I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
 	     "CPU PWM1 enabled\n");
@@ -9695,14 +9799,10 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
 	else
 		cdclk = 337500;
 
-	/*
-	 * FIXME move the cdclk caclulation to
-	 * compute_config() so we can fail gracegully.
-	 */
 	if (cdclk > dev_priv->max_cdclk_freq) {
-		DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
-			  cdclk, dev_priv->max_cdclk_freq);
-		cdclk = dev_priv->max_cdclk_freq;
+		DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
+			      cdclk, dev_priv->max_cdclk_freq);
+		return -EINVAL;
 	}
 
 	to_intel_atomic_state(state)->cdclk = cdclk;
@@ -9797,6 +9897,7 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
 		break;
 	case PORT_CLK_SEL_SPLL:
 		pipe_config->shared_dpll = DPLL_ID_SPLL;
+		break;
 	}
 }
 
@@ -9813,7 +9914,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
 	port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		skylake_get_ddi_pll(dev_priv, port, pipe_config);
 	else if (IS_BROXTON(dev))
 		bxt_get_ddi_pll(dev_priv, port, pipe_config);
@@ -10154,20 +10255,17 @@ __intel_framebuffer_create(struct drm_device *dev,
 	int ret;
 
 	intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
-	if (!intel_fb) {
-		drm_gem_object_unreference(&obj->base);
+	if (!intel_fb)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
 	if (ret)
 		goto err;
 
 	return &intel_fb->base;
+
 err:
-	drm_gem_object_unreference(&obj->base);
 	kfree(intel_fb);
-
 	return ERR_PTR(ret);
 }
 
@@ -10207,6 +10305,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev,
 				  struct drm_display_mode *mode,
 				  int depth, int bpp)
 {
+	struct drm_framebuffer *fb;
 	struct drm_i915_gem_object *obj;
 	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
 
@@ -10221,7 +10320,11 @@ intel_framebuffer_create_for_mode(struct drm_device *dev,
 								bpp);
 	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
 
-	return intel_framebuffer_create(dev, &mode_cmd, obj);
+	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
+	if (IS_ERR(fb))
+		drm_gem_object_unreference_unlocked(&obj->base);
+
+	return fb;
 }
 
 static struct drm_framebuffer *
@@ -11124,7 +11227,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 	 */
 	if (ring->id == RCS) {
 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(ring, DERRMR);
+		intel_ring_emit_reg(ring, DERRMR);
 		intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
 					DERRMR_PIPEB_PRI_FLIP_DONE |
 					DERRMR_PIPEC_PRI_FLIP_DONE));
@@ -11134,7 +11237,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 		else
 			intel_ring_emit(ring, MI_STORE_REGISTER_MEM |
 					      MI_SRM_LRM_GLOBAL_GTT);
-		intel_ring_emit(ring, DERRMR);
+		intel_ring_emit_reg(ring, DERRMR);
 		intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
 		if (IS_GEN8(dev)) {
 			intel_ring_emit(ring, 0);
@@ -11174,18 +11277,23 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
 		return true;
 	else if (i915.enable_execlists)
 		return true;
+	else if (obj->base.dma_buf &&
+		 !reservation_object_test_signaled_rcu(obj->base.dma_buf->resv,
+						       false))
+		return true;
 	else
 		return ring != i915_gem_request_get_ring(obj->last_write_req);
 }
 
 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
+			     unsigned int rotation,
 			     struct intel_unpin_work *work)
 {
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
 	const enum pipe pipe = intel_crtc->pipe;
-	u32 ctl, stride;
+	u32 ctl, stride, tile_height;
 
 	ctl = I915_READ(PLANE_CTL(pipe, 0));
 	ctl &= ~PLANE_CTL_TILED_MASK;
@@ -11209,9 +11317,16 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 	 * The stride is either expressed as a multiple of 64 bytes chunks for
 	 * linear buffers or in number of tiles for tiled buffers.
 	 */
-	stride = fb->pitches[0] /
-		 intel_fb_stride_alignment(dev, fb->modifier[0],
-					   fb->pixel_format);
+	if (intel_rotation_90_or_270(rotation)) {
+		/* stride = Surface height in tiles */
+		tile_height = intel_tile_height(dev, fb->pixel_format,
+						fb->modifier[0], 0);
+		stride = DIV_ROUND_UP(fb->height, tile_height);
+	} else {
+		stride = fb->pitches[0] /
+				intel_fb_stride_alignment(dev, fb->modifier[0],
+							  fb->pixel_format);
+	}
 
 	/*
 	 * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
@@ -11232,10 +11347,9 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
 	struct intel_framebuffer *intel_fb =
 		to_intel_framebuffer(intel_crtc->base.primary->fb);
 	struct drm_i915_gem_object *obj = intel_fb->obj;
+	i915_reg_t reg = DSPCNTR(intel_crtc->plane);
 	u32 dspcntr;
-	u32 reg;
 
-	reg = DSPCNTR(intel_crtc->plane);
 	dspcntr = I915_READ(reg);
 
 	if (obj->tiling_mode != I915_TILING_NONE)
@@ -11269,7 +11383,7 @@ static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip)
 	intel_pipe_update_start(crtc);
 
 	if (INTEL_INFO(mmio_flip->i915)->gen >= 9)
-		skl_do_mmio_flip(crtc, work);
+		skl_do_mmio_flip(crtc, mmio_flip->rotation, work);
 	else
 		/* use_mmio_flip() retricts MMIO flips to ilk+ */
 		ilk_do_mmio_flip(crtc, work);
@@ -11281,6 +11395,9 @@ static void intel_mmio_flip_work_func(struct work_struct *work)
 {
 	struct intel_mmio_flip *mmio_flip =
 		container_of(work, struct intel_mmio_flip, work);
+	struct intel_framebuffer *intel_fb =
+		to_intel_framebuffer(mmio_flip->crtc->base.primary->fb);
+	struct drm_i915_gem_object *obj = intel_fb->obj;
 
 	if (mmio_flip->req) {
 		WARN_ON(__i915_wait_request(mmio_flip->req,
@@ -11290,16 +11407,19 @@ static void intel_mmio_flip_work_func(struct work_struct *work)
 		i915_gem_request_unreference__unlocked(mmio_flip->req);
 	}
 
+	/* For framebuffer backed by dmabuf, wait for fence */
+	if (obj->base.dma_buf)
+		WARN_ON(reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
+							    false, false,
+							    MAX_SCHEDULE_TIMEOUT) < 0);
+
 	intel_do_mmio_flip(mmio_flip);
 	kfree(mmio_flip);
 }
 
 static int intel_queue_mmio_flip(struct drm_device *dev,
 				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct intel_engine_cs *ring,
-				 uint32_t flags)
+				 struct drm_i915_gem_object *obj)
 {
 	struct intel_mmio_flip *mmio_flip;
 
@@ -11310,6 +11430,7 @@ static int intel_queue_mmio_flip(struct drm_device *dev,
 	mmio_flip->i915 = to_i915(dev);
 	mmio_flip->req = i915_gem_request_reference(obj->last_write_req);
 	mmio_flip->crtc = to_intel_crtc(crtc);
+	mmio_flip->rotation = crtc->primary->state->rotation;
 
 	INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func);
 	schedule_work(&mmio_flip->work);
@@ -11493,7 +11614,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
 		work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		ring = &dev_priv->ring[BCS];
 		if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
 			/* vlv: DISPLAY_FLIP fails to change tiling */
@@ -11515,9 +11636,14 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	 * synchronisation, so all we want here is to pin the framebuffer
 	 * into the display plane and skip any waits.
 	 */
+	if (!mmio_flip) {
+		ret = i915_gem_object_sync(obj, ring, &request);
+		if (ret)
+			goto cleanup_pending;
+	}
+
 	ret = intel_pin_and_fence_fb_obj(crtc->primary, fb,
-					 crtc->primary->state,
-					 mmio_flip ? i915_gem_request_get_ring(obj->last_write_req) : ring, &request);
+					 crtc->primary->state);
 	if (ret)
 		goto cleanup_pending;
 
@@ -11526,8 +11652,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	work->gtt_offset += intel_crtc->dspaddr_offset;
 
 	if (mmio_flip) {
-		ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
-					    page_flip_flags);
+		ret = intel_queue_mmio_flip(dev, crtc, obj);
 		if (ret)
 			goto cleanup_unpin;
 
@@ -11558,7 +11683,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 			  to_intel_plane(primary)->frontbuffer_bit);
 	mutex_unlock(&dev->struct_mutex);
 
-	intel_fbc_disable_crtc(intel_crtc);
+	intel_fbc_deactivate(intel_crtc);
 	intel_frontbuffer_flip_prepare(dev,
 				       to_intel_plane(primary)->frontbuffer_bit);
 
@@ -11641,21 +11766,41 @@ retry:
 static bool intel_wm_need_update(struct drm_plane *plane,
 				 struct drm_plane_state *state)
 {
-	/* Update watermarks on tiling changes. */
-	if (!plane->state->fb || !state->fb ||
-	    plane->state->fb->modifier[0] != state->fb->modifier[0] ||
-	    plane->state->rotation != state->rotation)
+	struct intel_plane_state *new = to_intel_plane_state(state);
+	struct intel_plane_state *cur = to_intel_plane_state(plane->state);
+
+	/* Update watermarks on tiling or size changes. */
+	if (new->visible != cur->visible)
 		return true;
 
-	if (plane->state->crtc_w != state->crtc_w)
+	if (!cur->base.fb || !new->base.fb)
+		return false;
+
+	if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] ||
+	    cur->base.rotation != new->base.rotation ||
+	    drm_rect_width(&new->src) != drm_rect_width(&cur->src) ||
+	    drm_rect_height(&new->src) != drm_rect_height(&cur->src) ||
+	    drm_rect_width(&new->dst) != drm_rect_width(&cur->dst) ||
+	    drm_rect_height(&new->dst) != drm_rect_height(&cur->dst))
 		return true;
 
 	return false;
 }
 
+static bool needs_scaling(struct intel_plane_state *state)
+{
+	int src_w = drm_rect_width(&state->src) >> 16;
+	int src_h = drm_rect_height(&state->src) >> 16;
+	int dst_w = drm_rect_width(&state->dst);
+	int dst_h = drm_rect_height(&state->dst);
+
+	return (src_w != dst_w || src_h != dst_h);
+}
+
 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 				    struct drm_plane_state *plane_state)
 {
+	struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
 	struct drm_crtc *crtc = crtc_state->crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct drm_plane *plane = plane_state->plane;
@@ -11668,7 +11813,6 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 	bool mode_changed = needs_modeset(crtc_state);
 	bool was_crtc_enabled = crtc->state->active;
 	bool is_crtc_enabled = crtc_state->active;
-
 	bool turn_off, turn_on, visible, was_visible;
 	struct drm_framebuffer *fb = plane_state->fb;
 
@@ -11681,14 +11825,6 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 			return ret;
 	}
 
-	/*
-	 * Disabling a plane is always okay; we just need to update
-	 * fb tracking in a special way since cleanup_fb() won't
-	 * get called by the plane helpers.
-	 */
-	if (old_plane_state->base.fb && !fb)
-		intel_crtc->atomic.disabled_planes |= 1 << i;
-
 	was_visible = old_plane_state->visible;
 	visible = to_intel_plane_state(plane_state)->visible;
 
@@ -11711,25 +11847,17 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 			 plane->base.id, was_visible, visible,
 			 turn_off, turn_on, mode_changed);
 
-	if (turn_on) {
-		intel_crtc->atomic.update_wm_pre = true;
-		/* must disable cxsr around plane enable/disable */
-		if (plane->type != DRM_PLANE_TYPE_CURSOR) {
-			intel_crtc->atomic.disable_cxsr = true;
-			/* to potentially re-enable cxsr */
-			intel_crtc->atomic.wait_vblank = true;
-			intel_crtc->atomic.update_wm_post = true;
-		}
-	} else if (turn_off) {
-		intel_crtc->atomic.update_wm_post = true;
+	if (turn_on || turn_off) {
+		pipe_config->wm_changed = true;
+
 		/* must disable cxsr around plane enable/disable */
 		if (plane->type != DRM_PLANE_TYPE_CURSOR) {
 			if (is_crtc_enabled)
 				intel_crtc->atomic.wait_vblank = true;
-			intel_crtc->atomic.disable_cxsr = true;
+			pipe_config->disable_cxsr = true;
 		}
 	} else if (intel_wm_need_update(plane, plane_state)) {
-		intel_crtc->atomic.update_wm_pre = true;
+		pipe_config->wm_changed = true;
 	}
 
 	if (visible || was_visible)
@@ -11738,7 +11866,6 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 
 	switch (plane->type) {
 	case DRM_PLANE_TYPE_PRIMARY:
-		intel_crtc->atomic.wait_for_flips = true;
 		intel_crtc->atomic.pre_disable_primary = turn_off;
 		intel_crtc->atomic.post_enable_primary = turn_on;
 
@@ -11786,11 +11913,23 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 	case DRM_PLANE_TYPE_CURSOR:
 		break;
 	case DRM_PLANE_TYPE_OVERLAY:
-		if (turn_off && !mode_changed) {
+		/*
+		 * WaCxSRDisabledForSpriteScaling:ivb
+		 *
+		 * cstate->update_wm was already set above, so this flag will
+		 * take effect when we commit and program watermarks.
+		 */
+		if (IS_IVYBRIDGE(dev) &&
+		    needs_scaling(to_intel_plane_state(plane_state)) &&
+		    !needs_scaling(old_plane_state)) {
+			to_intel_crtc_state(crtc_state)->disable_lp_wm = true;
+		} else if (turn_off && !mode_changed) {
 			intel_crtc->atomic.wait_vblank = true;
 			intel_crtc->atomic.update_sprite_watermarks |=
 				1 << i;
 		}
+
+		break;
 	}
 	return 0;
 }
@@ -11863,7 +12002,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 	}
 
 	if (mode_changed && !crtc_state->active)
-		intel_crtc->atomic.update_wm_post = true;
+		pipe_config->wm_changed = true;
 
 	if (mode_changed && crtc_state->enable &&
 	    dev_priv->display.crtc_compute_clock &&
@@ -11875,6 +12014,12 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 	}
 
 	ret = 0;
+	if (dev_priv->display.compute_pipe_wm) {
+		ret = dev_priv->display.compute_pipe_wm(intel_crtc, state);
+		if (ret)
+			return ret;
+	}
+
 	if (INTEL_INFO(dev)->gen >= 9) {
 		if (mode_changed)
 			ret = skl_update_scaler_crtc(pipe_config);
@@ -11948,7 +12093,7 @@ compute_baseline_pipe_bpp(struct intel_crtc *crtc,
 	struct drm_connector_state *connector_state;
 	int bpp, i;
 
-	if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)))
+	if ((IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)))
 		bpp = 10*3;
 	else if (INTEL_INFO(dev)->gen >= 5)
 		bpp = 12*3;
@@ -12064,7 +12209,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 			      pipe_config->dpll_hw_state.pll9,
 			      pipe_config->dpll_hw_state.pll10,
 			      pipe_config->dpll_hw_state.pcsdw12);
-	} else if (IS_SKYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
 			      "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
 			      pipe_config->ddi_pll_sel,
@@ -12322,6 +12467,18 @@ intel_modeset_update_crtc_state(struct drm_atomic_state *state)
 			crtc->hwmode = crtc->state->adjusted_mode;
 		else
 			crtc->hwmode.crtc_clock = 0;
+
+		/*
+		 * Update legacy state to satisfy fbc code. This can
+		 * be removed when fbc uses the atomic state.
+		 */
+		if (drm_atomic_get_existing_plane_state(state, crtc->primary)) {
+			struct drm_plane_state *plane_state = crtc->primary->state;
+
+			crtc->primary->fb = plane_state->fb;
+			crtc->x = plane_state->src_x >> 16;
+			crtc->y = plane_state->src_y >> 16;
+		}
 	}
 }
 
@@ -12347,7 +12504,7 @@ static bool intel_fuzzy_clock_check(int clock1, int clock2)
 	list_for_each_entry((intel_crtc), \
 			    &(dev)->mode_config.crtc_list, \
 			    base.head) \
-		if (mask & (1 <<(intel_crtc)->pipe))
+		for_each_if (mask & (1 <<(intel_crtc)->pipe))
 
 static bool
 intel_compare_m_n(unsigned int m, unsigned int n,
@@ -12531,6 +12688,8 @@ intel_pipe_config_compare(struct drm_device *dev,
 	} else
 		PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
 
+	PIPE_CONF_CHECK_I(has_dsi_encoder);
+
 	PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
 	PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
 	PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
@@ -12548,7 +12707,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 	PIPE_CONF_CHECK_I(pixel_multiplier);
 	PIPE_CONF_CHECK_I(has_hdmi_sink);
 	if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
-	    IS_VALLEYVIEW(dev))
+	    IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		PIPE_CONF_CHECK_I(limited_color_range);
 	PIPE_CONF_CHECK_I(has_infoframe);
 
@@ -13085,6 +13244,45 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 	return 0;
 }
 
+/*
+ * Handle calculation of various watermark data at the end of the atomic check
+ * phase.  The code here should be run after the per-crtc and per-plane 'check'
+ * handlers to ensure that all derived state has been updated.
+ */
+static void calc_watermark_data(struct drm_atomic_state *state)
+{
+	struct drm_device *dev = state->dev;
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *cstate;
+	struct drm_plane *plane;
+	struct drm_plane_state *pstate;
+
+	/*
+	 * Calculate watermark configuration details now that derived
+	 * plane/crtc state is all properly updated.
+	 */
+	drm_for_each_crtc(crtc, dev) {
+		cstate = drm_atomic_get_existing_crtc_state(state, crtc) ?:
+			crtc->state;
+
+		if (cstate->active)
+			intel_state->wm_config.num_pipes_active++;
+	}
+	drm_for_each_legacy_plane(plane, dev) {
+		pstate = drm_atomic_get_existing_plane_state(state, plane) ?:
+			plane->state;
+
+		if (!to_intel_plane_state(pstate)->visible)
+			continue;
+
+		intel_state->wm_config.sprites_enabled = true;
+		if (pstate->crtc_w != pstate->src_w >> 16 ||
+		    pstate->crtc_h != pstate->src_h >> 16)
+			intel_state->wm_config.sprites_scaled = true;
+	}
+}
+
 /**
  * intel_atomic_check - validate state object
  * @dev: drm device
@@ -13093,6 +13291,7 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 static int intel_atomic_check(struct drm_device *dev,
 			      struct drm_atomic_state *state)
 {
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
 	int ret, i;
@@ -13160,10 +13359,81 @@ static int intel_atomic_check(struct drm_device *dev,
 		if (ret)
 			return ret;
 	} else
-		to_intel_atomic_state(state)->cdclk =
-			to_i915(state->dev)->cdclk_freq;
+		intel_state->cdclk = to_i915(state->dev)->cdclk_freq;
+
+	ret = drm_atomic_helper_check_planes(state->dev, state);
+	if (ret)
+		return ret;
 
-	return drm_atomic_helper_check_planes(state->dev, state);
+	calc_watermark_data(state);
+
+	return 0;
+}
+
+static int intel_atomic_prepare_commit(struct drm_device *dev,
+				       struct drm_atomic_state *state,
+				       bool async)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_plane_state *plane_state;
+	struct drm_crtc_state *crtc_state;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	int i, ret;
+
+	if (async) {
+		DRM_DEBUG_KMS("i915 does not yet support async commit\n");
+		return -EINVAL;
+	}
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		ret = intel_crtc_wait_for_pending_flips(crtc);
+		if (ret)
+			return ret;
+
+		if (atomic_read(&to_intel_crtc(crtc)->unpin_work_count) >= 2)
+			flush_workqueue(dev_priv->wq);
+	}
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (!ret && !async && !i915_reset_in_progress(&dev_priv->gpu_error)) {
+		u32 reset_counter;
+
+		reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
+		mutex_unlock(&dev->struct_mutex);
+
+		for_each_plane_in_state(state, plane, plane_state, i) {
+			struct intel_plane_state *intel_plane_state =
+				to_intel_plane_state(plane_state);
+
+			if (!intel_plane_state->wait_req)
+				continue;
+
+			ret = __i915_wait_request(intel_plane_state->wait_req,
+						  reset_counter, true,
+						  NULL, NULL);
+
+			/* Swallow -EIO errors to allow updates during hw lockup. */
+			if (ret == -EIO)
+				ret = 0;
+
+			if (ret)
+				break;
+		}
+
+		if (!ret)
+			return 0;
+
+		mutex_lock(&dev->struct_mutex);
+		drm_atomic_helper_cleanup_planes(dev, state);
+	}
+
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
 }
 
 /**
@@ -13187,22 +13457,20 @@ static int intel_atomic_commit(struct drm_device *dev,
 			       bool async)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
 	int ret = 0;
 	int i;
 	bool any_ms = false;
 
-	if (async) {
-		DRM_DEBUG_KMS("i915 does not yet support async commit\n");
-		return -EINVAL;
-	}
-
-	ret = drm_atomic_helper_prepare_planes(dev, state);
-	if (ret)
+	ret = intel_atomic_prepare_commit(dev, state, async);
+	if (ret) {
+		DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
 		return ret;
+	}
 
 	drm_atomic_helper_swap_state(dev, state);
+	dev_priv->wm.config = to_intel_atomic_state(state)->wm_config;
 
 	for_each_crtc_in_state(state, crtc, crtc_state, i) {
 		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -13218,6 +13486,16 @@ static int intel_atomic_commit(struct drm_device *dev,
 			dev_priv->display.crtc_disable(crtc);
 			intel_crtc->active = false;
 			intel_disable_shared_dpll(intel_crtc);
+
+			/*
+			 * Underruns don't always raise
+			 * interrupts, so check manually.
+			 */
+			intel_check_cpu_fifo_underruns(dev_priv);
+			intel_check_pch_fifo_underruns(dev_priv);
+
+			if (!crtc->state->active)
+				intel_update_watermarks(crtc);
 		}
 	}
 
@@ -13240,6 +13518,9 @@ static int intel_atomic_commit(struct drm_device *dev,
 			to_intel_crtc_state(crtc->state)->update_pipe;
 		unsigned long put_domains = 0;
 
+		if (modeset)
+			intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
+
 		if (modeset && crtc->state->active) {
 			update_scanline_offset(to_intel_crtc(crtc));
 			dev_priv->display.crtc_enable(crtc);
@@ -13255,18 +13536,26 @@ static int intel_atomic_commit(struct drm_device *dev,
 		if (!modeset)
 			intel_pre_plane_update(intel_crtc);
 
-		drm_atomic_helper_commit_planes_on_crtc(crtc_state);
+		if (crtc->state->active &&
+		    (crtc->state->planes_changed || update_pipe))
+			drm_atomic_helper_commit_planes_on_crtc(crtc_state);
 
 		if (put_domains)
 			modeset_put_power_domains(dev_priv, put_domains);
 
 		intel_post_plane_update(intel_crtc);
+
+		if (modeset)
+			intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
 	}
 
 	/* FIXME: add subpixel order */
 
 	drm_atomic_helper_wait_for_vblanks(dev, state);
+
+	mutex_lock(&dev->struct_mutex);
 	drm_atomic_helper_cleanup_planes(dev, state);
+	mutex_unlock(&dev->struct_mutex);
 
 	if (any_ms)
 		intel_modeset_check_state(dev, state);
@@ -13435,6 +13724,8 @@ static void intel_shared_dpll_init(struct drm_device *dev)
  * bits.  Some older platforms need special physical address handling for
  * cursor planes.
  *
+ * Must be called with struct_mutex held.
+ *
  * Returns 0 on success, negative error code on failure.
  */
 int
@@ -13445,28 +13736,71 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 	struct drm_framebuffer *fb = new_state->fb;
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
-	struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
+	struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
 	int ret = 0;
 
-	if (!obj)
+	if (!obj && !old_obj)
 		return 0;
 
-	mutex_lock(&dev->struct_mutex);
+	if (old_obj) {
+		struct drm_crtc_state *crtc_state =
+			drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc);
+
+		/* Big Hammer, we also need to ensure that any pending
+		 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
+		 * current scanout is retired before unpinning the old
+		 * framebuffer. Note that we rely on userspace rendering
+		 * into the buffer attached to the pipe they are waiting
+		 * on. If not, userspace generates a GPU hang with IPEHR
+		 * point to the MI_WAIT_FOR_EVENT.
+		 *
+		 * This should only fail upon a hung GPU, in which case we
+		 * can safely continue.
+		 */
+		if (needs_modeset(crtc_state))
+			ret = i915_gem_object_wait_rendering(old_obj, true);
+
+		/* Swallow -EIO errors to allow updates during hw lockup. */
+		if (ret && ret != -EIO)
+			return ret;
+	}
+
+	/* For framebuffer backed by dmabuf, wait for fence */
+	if (obj && obj->base.dma_buf) {
+		long lret;
+
+		lret = reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
+							   false, true,
+							   MAX_SCHEDULE_TIMEOUT);
+		if (lret == -ERESTARTSYS)
+			return lret;
+
+		WARN(lret < 0, "waiting returns %li\n", lret);
+	}
 
-	if (plane->type == DRM_PLANE_TYPE_CURSOR &&
+	if (!obj) {
+		ret = 0;
+	} else if (plane->type == DRM_PLANE_TYPE_CURSOR &&
 	    INTEL_INFO(dev)->cursor_needs_physical) {
 		int align = IS_I830(dev) ? 16 * 1024 : 256;
 		ret = i915_gem_object_attach_phys(obj, align);
 		if (ret)
 			DRM_DEBUG_KMS("failed to attach phys object\n");
 	} else {
-		ret = intel_pin_and_fence_fb_obj(plane, fb, new_state, NULL, NULL);
+		ret = intel_pin_and_fence_fb_obj(plane, fb, new_state);
 	}
 
-	if (ret == 0)
-		i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
+	if (ret == 0) {
+		if (obj) {
+			struct intel_plane_state *plane_state =
+				to_intel_plane_state(new_state);
 
-	mutex_unlock(&dev->struct_mutex);
+			i915_gem_request_assign(&plane_state->wait_req,
+						obj->last_write_req);
+		}
+
+		i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
+	}
 
 	return ret;
 }
@@ -13477,23 +13811,35 @@ intel_prepare_plane_fb(struct drm_plane *plane,
  * @fb: old framebuffer that was on plane
  *
  * Cleans up a framebuffer that has just been removed from a plane.
+ *
+ * Must be called with struct_mutex held.
  */
 void
 intel_cleanup_plane_fb(struct drm_plane *plane,
 		       const struct drm_plane_state *old_state)
 {
 	struct drm_device *dev = plane->dev;
-	struct drm_i915_gem_object *obj = intel_fb_obj(old_state->fb);
+	struct intel_plane *intel_plane = to_intel_plane(plane);
+	struct intel_plane_state *old_intel_state;
+	struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
+	struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
 
-	if (!obj)
+	old_intel_state = to_intel_plane_state(old_state);
+
+	if (!obj && !old_obj)
 		return;
 
-	if (plane->type != DRM_PLANE_TYPE_CURSOR ||
-	    !INTEL_INFO(dev)->cursor_needs_physical) {
-		mutex_lock(&dev->struct_mutex);
+	if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR ||
+	    !INTEL_INFO(dev)->cursor_needs_physical))
 		intel_unpin_fb_obj(old_state->fb, old_state);
-		mutex_unlock(&dev->struct_mutex);
-	}
+
+	/* prepare_fb aborted? */
+	if ((old_obj && (old_obj->frontbuffer_bits & intel_plane->frontbuffer_bit)) ||
+	    (obj && !(obj->frontbuffer_bits & intel_plane->frontbuffer_bit)))
+		i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
+
+	i915_gem_request_assign(&old_intel_state->wait_req, NULL);
+
 }
 
 int
@@ -13512,7 +13858,7 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
 	crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
 	cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
 
-	if (!crtc_clock || !cdclk)
+	if (WARN_ON_ONCE(!crtc_clock || cdclk < crtc_clock))
 		return DRM_PLANE_HELPER_NO_SCALING;
 
 	/*
@@ -13560,18 +13906,8 @@ intel_commit_primary_plane(struct drm_plane *plane,
 	struct drm_framebuffer *fb = state->base.fb;
 	struct drm_device *dev = plane->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc;
-	struct drm_rect *src = &state->src;
 
 	crtc = crtc ? crtc : plane->crtc;
-	intel_crtc = to_intel_crtc(crtc);
-
-	plane->fb = fb;
-	crtc->x = src->x1 >> 16;
-	crtc->y = src->y1 >> 16;
-
-	if (!crtc->state->active)
-		return;
 
 	dev_priv->display.update_primary_plane(crtc, fb,
 					       state->src.x1 >> 16,
@@ -13597,12 +13933,8 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 		to_intel_crtc_state(old_crtc_state);
 	bool modeset = needs_modeset(crtc->state);
 
-	if (intel_crtc->atomic.update_wm_pre)
-		intel_update_watermarks(crtc);
-
 	/* Perform vblank evasion around commit operation */
-	if (crtc->state->active)
-		intel_pipe_update_start(intel_crtc);
+	intel_pipe_update_start(intel_crtc);
 
 	if (modeset)
 		return;
@@ -13618,8 +13950,7 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	if (crtc->state->active)
-		intel_pipe_update_end(intel_crtc);
+	intel_pipe_update_end(intel_crtc);
 }
 
 /**
@@ -13696,7 +14027,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
 	drm_universal_plane_init(dev, &primary->base, 0,
 				 &intel_plane_funcs,
 				 intel_primary_formats, num_formats,
-				 DRM_PLANE_TYPE_PRIMARY);
+				 DRM_PLANE_TYPE_PRIMARY, NULL);
 
 	if (INTEL_INFO(dev)->gen >= 4)
 		intel_create_rotation_property(dev, primary);
@@ -13848,7 +14179,7 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
 				 &intel_plane_funcs,
 				 intel_cursor_formats,
 				 ARRAY_SIZE(intel_cursor_formats),
-				 DRM_PLANE_TYPE_CURSOR);
+				 DRM_PLANE_TYPE_CURSOR, NULL);
 
 	if (INTEL_INFO(dev)->gen >= 4) {
 		if (!dev->mode_config.rotation_property)
@@ -13925,7 +14256,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 		goto fail;
 
 	ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
-					cursor, &intel_crtc_funcs);
+					cursor, &intel_crtc_funcs, NULL);
 	if (ret)
 		goto fail;
 
@@ -14051,7 +14382,14 @@ static bool intel_crt_present(struct drm_device *dev)
 	if (IS_CHERRYVIEW(dev))
 		return false;
 
-	if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support)
+	if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
+		return false;
+
+	/* DDI E can't be used if DDI A requires 4 lanes */
+	if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
+		return false;
+
+	if (!dev_priv->vbt.int_crt_support)
 		return false;
 
 	return true;
@@ -14087,7 +14425,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 		 */
 		found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
 		/* WaIgnoreDDIAStrap: skl */
-		if (found || IS_SKYLAKE(dev))
+		if (found || IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 			intel_ddi_init(dev, PORT_A);
 
 		/* DDI B, C and D detection is indicated by the SFUSE_STRAP
@@ -14103,7 +14441,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 		/*
 		 * On SKL we don't have a way to detect DDI-E so we rely on VBT.
 		 */
-		if (IS_SKYLAKE(dev) &&
+		if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
 		    (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
 		     dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
 		     dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
@@ -14118,7 +14456,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
 			/* PCH SDVOB multiplex with HDMIB */
-			found = intel_sdvo_init(dev, PCH_SDVOB, true);
+			found = intel_sdvo_init(dev, PCH_SDVOB, PORT_B);
 			if (!found)
 				intel_hdmi_init(dev, PCH_HDMIB, PORT_B);
 			if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
@@ -14136,7 +14474,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(PCH_DP_D) & DP_DETECTED)
 			intel_dp_init(dev, PCH_DP_D, PORT_D);
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/*
 		 * The DP_DETECTED bit is the latched state of the DDC
 		 * SDA pin at boot. However since eDP doesn't require DDC
@@ -14174,7 +14512,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
 			DRM_DEBUG_KMS("probing SDVOB\n");
-			found = intel_sdvo_init(dev, GEN3_SDVOB, true);
+			found = intel_sdvo_init(dev, GEN3_SDVOB, PORT_B);
 			if (!found && IS_G4X(dev)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
 				intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
@@ -14188,7 +14526,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
 			DRM_DEBUG_KMS("probing SDVOC\n");
-			found = intel_sdvo_init(dev, GEN3_SDVOC, false);
+			found = intel_sdvo_init(dev, GEN3_SDVOC, PORT_C);
 		}
 
 		if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
@@ -14285,7 +14623,7 @@ u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
 		 *  pixels and 32K bytes."
 		 */
 		 return min(8192*drm_format_plane_cpp(pixel_format, 0), 32768);
-	} else if (gen >= 5 && !IS_VALLEYVIEW(dev)) {
+	} else if (gen >= 5 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 		return 32*1024;
 	} else if (gen >= 4) {
 		if (fb_modifier == I915_FORMAT_MOD_X_TILED)
@@ -14389,7 +14727,8 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		}
 		break;
 	case DRM_FORMAT_ABGR8888:
-		if (!IS_VALLEYVIEW(dev) && INTEL_INFO(dev)->gen < 9) {
+		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
+		    INTEL_INFO(dev)->gen < 9) {
 			DRM_DEBUG("unsupported pixel format: %s\n",
 				  drm_get_format_name(mode_cmd->pixel_format));
 			return -EINVAL;
@@ -14405,7 +14744,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		}
 		break;
 	case DRM_FORMAT_ABGR2101010:
-		if (!IS_VALLEYVIEW(dev)) {
+		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 			DRM_DEBUG("unsupported pixel format: %s\n",
 				  drm_get_format_name(mode_cmd->pixel_format));
 			return -EINVAL;
@@ -14454,8 +14793,9 @@ static int intel_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 intel_user_framebuffer_create(struct drm_device *dev,
 			      struct drm_file *filp,
-			      struct drm_mode_fb_cmd2 *user_mode_cmd)
+			      const struct drm_mode_fb_cmd2 *user_mode_cmd)
 {
+	struct drm_framebuffer *fb;
 	struct drm_i915_gem_object *obj;
 	struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
 
@@ -14464,7 +14804,11 @@ intel_user_framebuffer_create(struct drm_device *dev,
 	if (&obj->base == NULL)
 		return ERR_PTR(-ENOENT);
 
-	return intel_framebuffer_create(dev, &mode_cmd, obj);
+	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
+	if (IS_ERR(fb))
+		drm_gem_object_unreference_unlocked(&obj->base);
+
+	return fb;
 }
 
 #ifndef CONFIG_DRM_FBDEV_EMULATION
@@ -14528,7 +14872,7 @@ static void intel_init_display(struct drm_device *dev)
 		dev_priv->display.crtc_disable = ironlake_crtc_disable;
 		dev_priv->display.update_primary_plane =
 			ironlake_update_primary_plane;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
 		dev_priv->display.get_initial_plane_config =
 			i9xx_get_initial_plane_config;
@@ -14549,7 +14893,7 @@ static void intel_init_display(struct drm_device *dev)
 	}
 
 	/* Returns the core display clock speed */
-	if (IS_SKYLAKE(dev))
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 		dev_priv->display.get_display_clock_speed =
 			skylake_get_display_clock_speed;
 	else if (IS_BROXTON(dev))
@@ -14561,7 +14905,7 @@ static void intel_init_display(struct drm_device *dev)
 	else if (IS_HASWELL(dev))
 		dev_priv->display.get_display_clock_speed =
 			haswell_get_display_clock_speed;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		dev_priv->display.get_display_clock_speed =
 			valleyview_get_display_clock_speed;
 	else if (IS_GEN5(dev))
@@ -14589,9 +14933,6 @@ static void intel_init_display(struct drm_device *dev)
 	else if (IS_I945GM(dev) || IS_845G(dev))
 		dev_priv->display.get_display_clock_speed =
 			i9xx_misc_get_display_clock_speed;
-	else if (IS_PINEVIEW(dev))
-		dev_priv->display.get_display_clock_speed =
-			pnv_get_display_clock_speed;
 	else if (IS_I915GM(dev))
 		dev_priv->display.get_display_clock_speed =
 			i915gm_get_display_clock_speed;
@@ -14622,7 +14963,7 @@ static void intel_init_display(struct drm_device *dev)
 			dev_priv->display.modeset_calc_cdclk =
 				broadwell_modeset_calc_cdclk;
 		}
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		dev_priv->display.modeset_commit_cdclk =
 			valleyview_modeset_commit_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
@@ -14838,7 +15179,7 @@ static void i915_disable_vga(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u8 sr1;
-	u32 vga_reg = i915_vgacntrl_reg(dev);
+	i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
 
 	/* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
 	vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
@@ -14954,9 +15295,6 @@ void intel_modeset_init(struct drm_device *dev)
 	i915_disable_vga(dev);
 	intel_setup_outputs(dev);
 
-	/* Just in case the BIOS is doing something questionable. */
-	intel_fbc_disable(dev_priv);
-
 	drm_modeset_lock_all(dev);
 	intel_modeset_setup_hw_state(dev);
 	drm_modeset_unlock_all(dev);
@@ -15043,10 +15381,9 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 reg;
+	i915_reg_t reg = PIPECONF(crtc->config->cpu_transcoder);
 
 	/* Clear any frame start delays used for debugging left by the BIOS */
-	reg = PIPECONF(crtc->config->cpu_transcoder);
 	I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
 
 	/* restore vblank interrupts to correct state */
@@ -15113,6 +15450,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 		WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, NULL) < 0);
 		crtc->base.state->active = crtc->active;
 		crtc->base.enabled = crtc->active;
+		crtc->base.state->connector_mask = 0;
 
 		/* Because we only establish the connector -> encoder ->
 		 * crtc links if something is active, this means the
@@ -15200,7 +15538,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 void i915_redisable_vga_power_on(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 vga_reg = i915_vgacntrl_reg(dev);
+	i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
 
 	if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
 		DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
@@ -15239,7 +15577,7 @@ static void readout_plane_state(struct intel_crtc *crtc)
 	struct intel_plane_state *plane_state =
 		to_intel_plane_state(primary->state);
 
-	plane_state->visible =
+	plane_state->visible = crtc->active &&
 		primary_get_hw_state(to_intel_plane(primary));
 
 	if (plane_state->visible)
@@ -15315,7 +15653,21 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 	for_each_intel_connector(dev, connector) {
 		if (connector->get_hw_state(connector)) {
 			connector->base.dpms = DRM_MODE_DPMS_ON;
-			connector->base.encoder = &connector->encoder->base;
+
+			encoder = connector->encoder;
+			connector->base.encoder = &encoder->base;
+
+			if (encoder->base.crtc &&
+			    encoder->base.crtc->state->active) {
+				/*
+				 * This has to be done during hardware readout
+				 * because anything calling .crtc_disable may
+				 * rely on the connector_mask being accurate.
+				 */
+				encoder->base.crtc->state->connector_mask |=
+					1 << drm_connector_index(&connector->base);
+			}
+
 		} else {
 			connector->base.dpms = DRM_MODE_DPMS_OFF;
 			connector->base.encoder = NULL;
@@ -15400,7 +15752,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev)
 		pll->on = false;
 	}
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_wm_get_hw_state(dev);
 	else if (IS_GEN9(dev))
 		skl_wm_get_hw_state(dev);
@@ -15496,8 +15848,7 @@ void intel_modeset_gem_init(struct drm_device *dev)
 		mutex_lock(&dev->struct_mutex);
 		ret = intel_pin_and_fence_fb_obj(c->primary,
 						 c->primary->fb,
-						 c->primary->state,
-						 NULL, NULL);
+						 c->primary->state);
 		mutex_unlock(&dev->struct_mutex);
 		if (ret) {
 			DRM_ERROR("failed to pin boot fb on pipe %d\n",
@@ -15524,7 +15875,7 @@ void intel_connector_unregister(struct intel_connector *intel_connector)
 void intel_modeset_cleanup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_connector *connector;
+	struct intel_connector *connector;
 
 	intel_disable_gt_powersave(dev);
 
@@ -15551,12 +15902,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
 	flush_scheduled_work();
 
 	/* destroy the backlight and sysfs files before encoders/connectors */
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		struct intel_connector *intel_connector;
-
-		intel_connector = to_intel_connector(connector);
-		intel_connector->unregister(intel_connector);
-	}
+	for_each_intel_connector(dev, connector)
+		connector->unregister(connector);
 
 	drm_mode_config_cleanup(dev);
 
@@ -15565,6 +15912,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
 	mutex_lock(&dev->struct_mutex);
 	intel_cleanup_gt_powersave(dev);
 	mutex_unlock(&dev->struct_mutex);
+
+	intel_teardown_gmbus(dev);
 }
 
 /*
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 78b8ec84d576..796e3d313cb9 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -389,8 +389,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 	 * We don't have power sequencer currently.
 	 * Pick one that's not used by other ports.
 	 */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-			    base.head) {
+	for_each_intel_encoder(dev, encoder) {
 		struct intel_dp *tmp;
 
 		if (encoder->type != INTEL_OUTPUT_EDP)
@@ -517,7 +516,7 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	struct intel_encoder *encoder;
 
-	if (WARN_ON(!IS_VALLEYVIEW(dev)))
+	if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)))
 		return;
 
 	/*
@@ -530,7 +529,7 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	 * should use them always.
 	 */
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+	for_each_intel_encoder(dev, encoder) {
 		struct intel_dp *intel_dp;
 
 		if (encoder->type != INTEL_OUTPUT_EDP)
@@ -541,7 +540,8 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	}
 }
 
-static u32 _pp_ctrl_reg(struct intel_dp *intel_dp)
+static i915_reg_t
+_pp_ctrl_reg(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
@@ -553,7 +553,8 @@ static u32 _pp_ctrl_reg(struct intel_dp *intel_dp)
 		return VLV_PIPE_PP_CONTROL(vlv_power_sequencer_pipe(intel_dp));
 }
 
-static u32 _pp_stat_reg(struct intel_dp *intel_dp)
+static i915_reg_t
+_pp_stat_reg(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
@@ -580,9 +581,9 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 
 	pps_lock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
-		u32 pp_ctrl_reg, pp_div_reg;
+		i915_reg_t pp_ctrl_reg, pp_div_reg;
 		u32 pp_div;
 
 		pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
@@ -608,7 +609,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	if (IS_VALLEYVIEW(dev) &&
+	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
 	    intel_dp->pps_pipe == INVALID_PIPE)
 		return false;
 
@@ -622,7 +623,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	if (IS_VALLEYVIEW(dev) &&
+	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
 	    intel_dp->pps_pipe == INVALID_PIPE)
 		return false;
 
@@ -652,7 +653,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t ch_ctl = intel_dp->aux_ch_ctl_reg;
+	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
 	uint32_t status;
 	bool done;
 
@@ -679,7 +680,7 @@ static uint32_t i9xx_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	 * The clock divider is based off the hrawclk, and would like to run at
 	 * 2MHz.  So, take the hrawclk value and divide by 2 and use that
 	 */
-	return index ? 0 : intel_hrawclk(dev) / 2;
+	return index ? 0 : DIV_ROUND_CLOSEST(intel_hrawclk(dev), 2);
 }
 
 static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
@@ -692,10 +693,10 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 		return 0;
 
 	if (intel_dig_port->port == PORT_A) {
-		return DIV_ROUND_UP(dev_priv->cdclk_freq, 2000);
+		return DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 2000);
 
 	} else {
-		return DIV_ROUND_UP(intel_pch_rawclk(dev), 2);
+		return DIV_ROUND_CLOSEST(intel_pch_rawclk(dev), 2);
 	}
 }
 
@@ -709,7 +710,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 		if (index)
 			return 0;
 		return DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 2000);
-	} else if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
+	} else if (HAS_PCH_LPT_H(dev_priv)) {
 		/* Workaround for non-ULT HSW */
 		switch (index) {
 		case 0: return 63;
@@ -717,7 +718,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 		default: return 0;
 		}
 	} else  {
-		return index ? 0 : DIV_ROUND_UP(intel_pch_rawclk(dev), 2);
+		return index ? 0 : DIV_ROUND_CLOSEST(intel_pch_rawclk(dev), 2);
 	}
 }
 
@@ -750,7 +751,7 @@ static uint32_t i9xx_get_aux_send_ctl(struct intel_dp *intel_dp,
 	else
 		precharge = 5;
 
-	if (IS_BROADWELL(dev) && intel_dp->aux_ch_ctl_reg == DPA_AUX_CH_CTL)
+	if (IS_BROADWELL(dev) && intel_dig_port->port == PORT_A)
 		timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
 	else
 		timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
@@ -789,8 +790,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t ch_ctl = intel_dp->aux_ch_ctl_reg;
-	uint32_t ch_data = ch_ctl + 4;
+	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
 	uint32_t aux_clock_divider;
 	int i, ret, recv_bytes;
 	uint32_t status;
@@ -854,7 +854,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
 		for (try = 0; try < 5; try++) {
 			/* Load the send data into the aux channel data registers */
 			for (i = 0; i < send_bytes; i += 4)
-				I915_WRITE(ch_data + i,
+				I915_WRITE(intel_dp->aux_ch_data_reg[i >> 2],
 					   intel_dp_pack_aux(send + i,
 							     send_bytes - i));
 
@@ -914,11 +914,32 @@ done:
 	/* Unload any bytes sent back from the other side */
 	recv_bytes = ((status & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >>
 		      DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT);
+
+	/*
+	 * By BSpec: "Message sizes of 0 or >20 are not allowed."
+	 * We have no idea of what happened so we return -EBUSY so
+	 * drm layer takes care for the necessary retries.
+	 */
+	if (recv_bytes == 0 || recv_bytes > 20) {
+		DRM_DEBUG_KMS("Forbidden recv_bytes = %d on aux transaction\n",
+			      recv_bytes);
+		/*
+		 * FIXME: This patch was created on top of a series that
+		 * organize the retries at drm level. There EBUSY should
+		 * also take care for 1ms wait before retrying.
+		 * That aux retries re-org is still needed and after that is
+		 * merged we remove this sleep from here.
+		 */
+		usleep_range(1000, 1500);
+		ret = -EBUSY;
+		goto out;
+	}
+
 	if (recv_bytes > recv_size)
 		recv_bytes = recv_size;
 
 	for (i = 0; i < recv_bytes; i += 4)
-		intel_dp_unpack_aux(I915_READ(ch_data + i),
+		intel_dp_unpack_aux(I915_READ(intel_dp->aux_ch_data_reg[i >> 2]),
 				    recv + i, recv_bytes - i);
 
 	ret = recv_bytes;
@@ -1005,96 +1026,206 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 	return ret;
 }
 
-static void
-intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector)
+static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv,
+				       enum port port)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	enum port port = intel_dig_port->port;
-	struct ddi_vbt_port_info *info = &dev_priv->vbt.ddi_port_info[port];
-	const char *name = NULL;
-	uint32_t porte_aux_ctl_reg = DPA_AUX_CH_CTL;
-	int ret;
+	switch (port) {
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		return DP_AUX_CH_CTL(port);
+	default:
+		MISSING_CASE(port);
+		return DP_AUX_CH_CTL(PORT_B);
+	}
+}
 
-	/* On SKL we don't have Aux for port E so we rely on VBT to set
-	 * a proper alternate aux channel.
-	 */
-	if (IS_SKYLAKE(dev) && port == PORT_E) {
-		switch (info->alternate_aux_channel) {
-		case DP_AUX_B:
-			porte_aux_ctl_reg = DPB_AUX_CH_CTL;
-			break;
-		case DP_AUX_C:
-			porte_aux_ctl_reg = DPC_AUX_CH_CTL;
-			break;
-		case DP_AUX_D:
-			porte_aux_ctl_reg = DPD_AUX_CH_CTL;
-			break;
-		case DP_AUX_A:
-		default:
-			porte_aux_ctl_reg = DPA_AUX_CH_CTL;
-		}
+static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv,
+					enum port port, int index)
+{
+	switch (port) {
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		return DP_AUX_CH_DATA(port, index);
+	default:
+		MISSING_CASE(port);
+		return DP_AUX_CH_DATA(PORT_B, index);
 	}
+}
 
+static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv,
+				       enum port port)
+{
 	switch (port) {
 	case PORT_A:
-		intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL;
-		name = "DPDDC-A";
-		break;
+		return DP_AUX_CH_CTL(port);
 	case PORT_B:
-		intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL;
-		name = "DPDDC-B";
-		break;
 	case PORT_C:
-		intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL;
-		name = "DPDDC-C";
-		break;
 	case PORT_D:
-		intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL;
-		name = "DPDDC-D";
-		break;
-	case PORT_E:
-		intel_dp->aux_ch_ctl_reg = porte_aux_ctl_reg;
-		name = "DPDDC-E";
-		break;
+		return PCH_DP_AUX_CH_CTL(port);
 	default:
-		BUG();
+		MISSING_CASE(port);
+		return DP_AUX_CH_CTL(PORT_A);
 	}
+}
 
-	/*
-	 * The AUX_CTL register is usually DP_CTL + 0x10.
-	 *
-	 * On Haswell and Broadwell though:
-	 *   - Both port A DDI_BUF_CTL and DDI_AUX_CTL are on the CPU
-	 *   - Port B/C/D AUX channels are on the PCH, DDI_BUF_CTL on the CPU
-	 *
-	 * Skylake moves AUX_CTL back next to DDI_BUF_CTL, on the CPU.
-	 */
-	if (!IS_HASWELL(dev) && !IS_BROADWELL(dev) && port != PORT_E)
-		intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10;
+static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv,
+					enum port port, int index)
+{
+	switch (port) {
+	case PORT_A:
+		return DP_AUX_CH_DATA(port, index);
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		return PCH_DP_AUX_CH_DATA(port, index);
+	default:
+		MISSING_CASE(port);
+		return DP_AUX_CH_DATA(PORT_A, index);
+	}
+}
+
+/*
+ * On SKL we don't have Aux for port E so we rely
+ * on VBT to set a proper alternate aux channel.
+ */
+static enum port skl_porte_aux_port(struct drm_i915_private *dev_priv)
+{
+	const struct ddi_vbt_port_info *info =
+		&dev_priv->vbt.ddi_port_info[PORT_E];
+
+	switch (info->alternate_aux_channel) {
+	case DP_AUX_A:
+		return PORT_A;
+	case DP_AUX_B:
+		return PORT_B;
+	case DP_AUX_C:
+		return PORT_C;
+	case DP_AUX_D:
+		return PORT_D;
+	default:
+		MISSING_CASE(info->alternate_aux_channel);
+		return PORT_A;
+	}
+}
+
+static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv,
+				       enum port port)
+{
+	if (port == PORT_E)
+		port = skl_porte_aux_port(dev_priv);
+
+	switch (port) {
+	case PORT_A:
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		return DP_AUX_CH_CTL(port);
+	default:
+		MISSING_CASE(port);
+		return DP_AUX_CH_CTL(PORT_A);
+	}
+}
+
+static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv,
+					enum port port, int index)
+{
+	if (port == PORT_E)
+		port = skl_porte_aux_port(dev_priv);
+
+	switch (port) {
+	case PORT_A:
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		return DP_AUX_CH_DATA(port, index);
+	default:
+		MISSING_CASE(port);
+		return DP_AUX_CH_DATA(PORT_A, index);
+	}
+}
+
+static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv,
+					 enum port port)
+{
+	if (INTEL_INFO(dev_priv)->gen >= 9)
+		return skl_aux_ctl_reg(dev_priv, port);
+	else if (HAS_PCH_SPLIT(dev_priv))
+		return ilk_aux_ctl_reg(dev_priv, port);
+	else
+		return g4x_aux_ctl_reg(dev_priv, port);
+}
+
+static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv,
+					  enum port port, int index)
+{
+	if (INTEL_INFO(dev_priv)->gen >= 9)
+		return skl_aux_data_reg(dev_priv, port, index);
+	else if (HAS_PCH_SPLIT(dev_priv))
+		return ilk_aux_data_reg(dev_priv, port, index);
+	else
+		return g4x_aux_data_reg(dev_priv, port, index);
+}
+
+static void intel_aux_reg_init(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
+	enum port port = dp_to_dig_port(intel_dp)->port;
+	int i;
+
+	intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port);
+	for (i = 0; i < ARRAY_SIZE(intel_dp->aux_ch_data_reg); i++)
+		intel_dp->aux_ch_data_reg[i] = intel_aux_data_reg(dev_priv, port, i);
+}
+
+static void
+intel_dp_aux_fini(struct intel_dp *intel_dp)
+{
+	drm_dp_aux_unregister(&intel_dp->aux);
+	kfree(intel_dp->aux.name);
+}
+
+static int
+intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector)
+{
+	struct drm_device *dev = intel_dp_to_dev(intel_dp);
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	enum port port = intel_dig_port->port;
+	int ret;
+
+	intel_aux_reg_init(intel_dp);
+
+	intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", port_name(port));
+	if (!intel_dp->aux.name)
+		return -ENOMEM;
 
-	intel_dp->aux.name = name;
 	intel_dp->aux.dev = dev->dev;
 	intel_dp->aux.transfer = intel_dp_aux_transfer;
 
-	DRM_DEBUG_KMS("registering %s bus for %s\n", name,
+	DRM_DEBUG_KMS("registering %s bus for %s\n",
+		      intel_dp->aux.name,
 		      connector->base.kdev->kobj.name);
 
 	ret = drm_dp_aux_register(&intel_dp->aux);
 	if (ret < 0) {
 		DRM_ERROR("drm_dp_aux_register() for %s failed (%d)\n",
-			  name, ret);
-		return;
+			  intel_dp->aux.name, ret);
+		kfree(intel_dp->aux.name);
+		return ret;
 	}
 
 	ret = sysfs_create_link(&connector->base.kdev->kobj,
 				&intel_dp->aux.ddc.dev.kobj,
 				intel_dp->aux.ddc.dev.kobj.name);
 	if (ret < 0) {
-		DRM_ERROR("sysfs_create_link() for %s failed (%d)\n", name, ret);
-		drm_dp_aux_unregister(&intel_dp->aux);
+		DRM_ERROR("sysfs_create_link() for %s failed (%d)\n",
+			  intel_dp->aux.name, ret);
+		intel_dp_aux_fini(intel_dp);
+		return ret;
 	}
+
+	return 0;
 }
 
 static void
@@ -1186,10 +1317,13 @@ intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates)
 	return (intel_dp_max_link_bw(intel_dp) >> 3) + 1;
 }
 
-static bool intel_dp_source_supports_hbr2(struct drm_device *dev)
+bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp)
 {
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = dig_port->base.base.dev;
+
 	/* WaDisableHBR2:skl */
-	if (IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0)
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0))
 		return false;
 
 	if ((IS_HASWELL(dev) && !IS_HSW_ULX(dev)) || IS_BROADWELL(dev) ||
@@ -1200,14 +1334,16 @@ static bool intel_dp_source_supports_hbr2(struct drm_device *dev)
 }
 
 static int
-intel_dp_source_rates(struct drm_device *dev, const int **source_rates)
+intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
 {
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = dig_port->base.base.dev;
 	int size;
 
 	if (IS_BROXTON(dev)) {
 		*source_rates = bxt_rates;
 		size = ARRAY_SIZE(bxt_rates);
-	} else if (IS_SKYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		*source_rates = skl_rates;
 		size = ARRAY_SIZE(skl_rates);
 	} else {
@@ -1216,7 +1352,7 @@ intel_dp_source_rates(struct drm_device *dev, const int **source_rates)
 	}
 
 	/* This depends on the fact that 5.4 is last value in the array */
-	if (!intel_dp_source_supports_hbr2(dev))
+	if (!intel_dp_source_supports_hbr2(intel_dp))
 		size--;
 
 	return size;
@@ -1281,12 +1417,11 @@ static int intersect_rates(const int *source_rates, int source_len,
 static int intel_dp_common_rates(struct intel_dp *intel_dp,
 				 int *common_rates)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	const int *source_rates, *sink_rates;
 	int source_len, sink_len;
 
 	sink_len = intel_dp_sink_rates(intel_dp, &sink_rates);
-	source_len = intel_dp_source_rates(dev, &source_rates);
+	source_len = intel_dp_source_rates(intel_dp, &source_rates);
 
 	return intersect_rates(source_rates, source_len,
 			       sink_rates, sink_len,
@@ -1311,7 +1446,6 @@ static void snprintf_int_array(char *str, size_t len,
 
 static void intel_dp_print_rates(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	const int *source_rates, *sink_rates;
 	int source_len, sink_len, common_len;
 	int common_rates[DP_MAX_SUPPORTED_RATES];
@@ -1320,7 +1454,7 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp)
 	if ((drm_debug & DRM_UT_KMS) == 0)
 		return;
 
-	source_len = intel_dp_source_rates(dev, &source_rates);
+	source_len = intel_dp_source_rates(intel_dp, &source_rates);
 	snprintf_int_array(str, sizeof(str), source_rates, source_len);
 	DRM_DEBUG_KMS("source rates: %s\n", str);
 
@@ -1362,8 +1496,8 @@ int intel_dp_rate_select(struct intel_dp *intel_dp, int rate)
 	return rate_to_index(rate, intel_dp->sink_rates);
 }
 
-static void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
-				  uint8_t *link_bw, uint8_t *rate_select)
+void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
+			   uint8_t *link_bw, uint8_t *rate_select)
 {
 	if (intel_dp->num_sink_rates) {
 		*link_bw = 0;
@@ -1423,7 +1557,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 				return ret;
 		}
 
-		if (!HAS_PCH_SPLIT(dev))
+		if (HAS_GMCH_DISPLAY(dev))
 			intel_gmch_panel_fitting(intel_crtc, pipe_config,
 						 intel_connector->panel.fitting_mode);
 		else
@@ -1527,7 +1661,7 @@ found:
 				&pipe_config->dp_m2_n2);
 	}
 
-	if (IS_SKYLAKE(dev) && is_edp(intel_dp))
+	if ((IS_SKYLAKE(dev)  || IS_KABYLAKE(dev)) && is_edp(intel_dp))
 		skl_edp_set_pll_config(pipe_config);
 	else if (IS_BROXTON(dev))
 		/* handled in ddi */;
@@ -1539,37 +1673,6 @@ found:
 	return true;
 }
 
-static void ironlake_set_pll_cpu_edp(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc);
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dpa_ctl;
-
-	DRM_DEBUG_KMS("eDP PLL enable for clock %d\n",
-		      crtc->config->port_clock);
-	dpa_ctl = I915_READ(DP_A);
-	dpa_ctl &= ~DP_PLL_FREQ_MASK;
-
-	if (crtc->config->port_clock == 162000) {
-		/* For a long time we've carried around a ILK-DevA w/a for the
-		 * 160MHz clock. If we're really unlucky, it's still required.
-		 */
-		DRM_DEBUG_KMS("160MHz cpu eDP clock, might need ilk devA w/a\n");
-		dpa_ctl |= DP_PLL_FREQ_160MHZ;
-		intel_dp->DP |= DP_PLL_FREQ_160MHZ;
-	} else {
-		dpa_ctl |= DP_PLL_FREQ_270MHZ;
-		intel_dp->DP |= DP_PLL_FREQ_270MHZ;
-	}
-
-	I915_WRITE(DP_A, dpa_ctl);
-
-	POSTING_READ(DP_A);
-	udelay(500);
-}
-
 void intel_dp_set_link_params(struct intel_dp *intel_dp,
 			      const struct intel_crtc_state *pipe_config)
 {
@@ -1614,9 +1717,6 @@ static void intel_dp_prepare(struct intel_encoder *encoder)
 	intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
 	intel_dp->DP |= DP_PORT_WIDTH(crtc->config->lane_count);
 
-	if (crtc->config->has_audio)
-		intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
-
 	/* Split out the IBX/CPU vs CPT settings */
 
 	if (IS_GEN7(dev) && port == PORT_A) {
@@ -1643,7 +1743,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder)
 		I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
 	} else {
 		if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-		    crtc->config->limited_color_range)
+		    !IS_CHERRYVIEW(dev) && crtc->config->limited_color_range)
 			intel_dp->DP |= DP_COLOR_RANGE_16_235;
 
 		if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1677,7 +1777,7 @@ static void wait_panel_status(struct intel_dp *intel_dp,
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 pp_stat_reg, pp_ctrl_reg;
+	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -1767,7 +1867,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	enum intel_display_power_domain power_domain;
 	u32 pp;
-	u32 pp_stat_reg, pp_ctrl_reg;
+	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 	bool need_to_disable = !intel_dp->want_panel_vdd;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
@@ -1843,7 +1943,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 	struct intel_encoder *intel_encoder = &intel_dig_port->base;
 	enum intel_display_power_domain power_domain;
 	u32 pp;
-	u32 pp_stat_reg, pp_ctrl_reg;
+	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -1930,7 +2030,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 pp;
-	u32 pp_ctrl_reg;
+	i915_reg_t pp_ctrl_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -1992,7 +2092,7 @@ static void edp_panel_off(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	enum intel_display_power_domain power_domain;
 	u32 pp;
-	u32 pp_ctrl_reg;
+	i915_reg_t pp_ctrl_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -2043,7 +2143,7 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 pp;
-	u32 pp_ctrl_reg;
+	i915_reg_t pp_ctrl_reg;
 
 	/*
 	 * If we enable the backlight right away following a panel power
@@ -2084,7 +2184,7 @@ static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 pp;
-	u32 pp_ctrl_reg;
+	i915_reg_t pp_ctrl_reg;
 
 	if (!is_edp(intel_dp))
 		return;
@@ -2143,27 +2243,61 @@ static void intel_edp_backlight_power(struct intel_connector *connector,
 		_intel_edp_backlight_off(intel_dp);
 }
 
+static const char *state_string(bool enabled)
+{
+	return enabled ? "on" : "off";
+}
+
+static void assert_dp_port(struct intel_dp *intel_dp, bool state)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
+	bool cur_state = I915_READ(intel_dp->output_reg) & DP_PORT_EN;
+
+	I915_STATE_WARN(cur_state != state,
+			"DP port %c state assertion failure (expected %s, current %s)\n",
+			port_name(dig_port->port),
+			state_string(state), state_string(cur_state));
+}
+#define assert_dp_port_disabled(d) assert_dp_port((d), false)
+
+static void assert_edp_pll(struct drm_i915_private *dev_priv, bool state)
+{
+	bool cur_state = I915_READ(DP_A) & DP_PLL_ENABLE;
+
+	I915_STATE_WARN(cur_state != state,
+			"eDP PLL state assertion failure (expected %s, current %s)\n",
+			state_string(state), state_string(cur_state));
+}
+#define assert_edp_pll_enabled(d) assert_edp_pll((d), true)
+#define assert_edp_pll_disabled(d) assert_edp_pll((d), false)
+
 static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_crtc *crtc = intel_dig_port->base.base.crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dpa_ctl;
+	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	assert_pipe_disabled(dev_priv,
-			     to_intel_crtc(crtc)->pipe);
+	assert_pipe_disabled(dev_priv, crtc->pipe);
+	assert_dp_port_disabled(intel_dp);
+	assert_edp_pll_disabled(dev_priv);
+
+	DRM_DEBUG_KMS("enabling eDP PLL for clock %d\n",
+		      crtc->config->port_clock);
+
+	intel_dp->DP &= ~DP_PLL_FREQ_MASK;
+
+	if (crtc->config->port_clock == 162000)
+		intel_dp->DP |= DP_PLL_FREQ_162MHZ;
+	else
+		intel_dp->DP |= DP_PLL_FREQ_270MHZ;
+
+	I915_WRITE(DP_A, intel_dp->DP);
+	POSTING_READ(DP_A);
+	udelay(500);
 
-	DRM_DEBUG_KMS("\n");
-	dpa_ctl = I915_READ(DP_A);
-	WARN(dpa_ctl & DP_PLL_ENABLE, "dp pll on, should be off\n");
-	WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n");
-
-	/* We don't adjust intel_dp->DP while tearing down the link, to
-	 * facilitate link retraining (e.g. after hotplug). Hence clear all
-	 * enable bits here to ensure that we don't enable too much. */
-	intel_dp->DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE);
 	intel_dp->DP |= DP_PLL_ENABLE;
+
 	I915_WRITE(DP_A, intel_dp->DP);
 	POSTING_READ(DP_A);
 	udelay(200);
@@ -2172,24 +2306,18 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
 static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_crtc *crtc = intel_dig_port->base.base.crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dpa_ctl;
+	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+	assert_pipe_disabled(dev_priv, crtc->pipe);
+	assert_dp_port_disabled(intel_dp);
+	assert_edp_pll_enabled(dev_priv);
 
-	assert_pipe_disabled(dev_priv,
-			     to_intel_crtc(crtc)->pipe);
+	DRM_DEBUG_KMS("disabling eDP PLL\n");
 
-	dpa_ctl = I915_READ(DP_A);
-	WARN((dpa_ctl & DP_PLL_ENABLE) == 0,
-	     "dp pll off, should be on\n");
-	WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n");
+	intel_dp->DP &= ~DP_PLL_ENABLE;
 
-	/* We can't rely on the value tracked for the DP register in
-	 * intel_dp->DP because link_down must not change that (otherwise link
-	 * re-training will fail. */
-	dpa_ctl &= ~DP_PLL_ENABLE;
-	I915_WRITE(DP_A, dpa_ctl);
+	I915_WRITE(DP_A, intel_dp->DP);
 	POSTING_READ(DP_A);
 	udelay(200);
 }
@@ -2258,7 +2386,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 		}
 
 		DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n",
-			      intel_dp->output_reg);
+			      i915_mmio_reg_offset(intel_dp->output_reg));
 	} else if (IS_CHERRYVIEW(dev)) {
 		*pipe = DP_PORT_TO_PIPE_CHV(tmp);
 	} else {
@@ -2310,7 +2438,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 	pipe_config->base.adjusted_mode.flags |= flags;
 
 	if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-	    tmp & DP_COLOR_RANGE_16_235)
+	    !IS_CHERRYVIEW(dev) && tmp & DP_COLOR_RANGE_16_235)
 		pipe_config->limited_color_range = true;
 
 	pipe_config->has_dp_encoder = true;
@@ -2321,7 +2449,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 	intel_dp_get_m_n(crtc, pipe_config);
 
 	if (port == PORT_A) {
-		if ((I915_READ(DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_160MHZ)
+		if ((I915_READ(DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_162MHZ)
 			pipe_config->port_clock = 162000;
 		else
 			pipe_config->port_clock = 270000;
@@ -2386,6 +2514,8 @@ static void ilk_post_disable_dp(struct intel_encoder *encoder)
 	enum port port = dp_to_dig_port(intel_dp)->port;
 
 	intel_dp_link_down(intel_dp);
+
+	/* Only ilk+ has port A */
 	if (port == PORT_A)
 		ironlake_edp_pll_off(intel_dp);
 }
@@ -2545,6 +2675,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *crtc =
+		to_intel_crtc(dp_to_dig_port(intel_dp)->base.base.crtc);
 
 	/* enable with pattern 1 (as per spec) */
 	_intel_dp_set_link_train(intel_dp, &intel_dp->DP,
@@ -2560,6 +2692,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp)
 	 * fail when the power sequencer is freshly used for this port.
 	 */
 	intel_dp->DP |= DP_PORT_EN;
+	if (crtc->config->has_audio)
+		intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
 
 	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
 	POSTING_READ(intel_dp->output_reg);
@@ -2572,24 +2706,49 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 	uint32_t dp_reg = I915_READ(intel_dp->output_reg);
+	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum pipe pipe = crtc->pipe;
 
 	if (WARN_ON(dp_reg & DP_PORT_EN))
 		return;
 
 	pps_lock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_init_panel_power_sequencer(intel_dp);
 
+	/*
+	 * We get an occasional spurious underrun between the port
+	 * enable and vdd enable, when enabling port A eDP.
+	 *
+	 * FIXME: Not sure if this applies to (PCH) port D eDP as well
+	 */
+	if (port == PORT_A)
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+
 	intel_dp_enable_port(intel_dp);
 
+	if (port == PORT_A && IS_GEN5(dev_priv)) {
+		/*
+		 * Underrun reporting for the other pipe was disabled in
+		 * g4x_pre_enable_dp(). The eDP PLL and port have now been
+		 * enabled, so it's now safe to re-enable underrun reporting.
+		 */
+		intel_wait_for_vblank_if_active(dev_priv->dev, !pipe);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, !pipe, true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, !pipe, true);
+	}
+
 	edp_panel_vdd_on(intel_dp);
 	edp_panel_on(intel_dp);
 	edp_panel_vdd_off(intel_dp, true);
 
+	if (port == PORT_A)
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+
 	pps_unlock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		unsigned int lane_mask = 0x0;
 
 		if (IS_CHERRYVIEW(dev))
@@ -2605,7 +2764,7 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 
 	if (crtc->config->has_audio) {
 		DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
-				 pipe_name(crtc->pipe));
+				 pipe_name(pipe));
 		intel_audio_codec_enable(encoder);
 	}
 }
@@ -2628,16 +2787,29 @@ static void vlv_enable_dp(struct intel_encoder *encoder)
 
 static void g4x_pre_enable_dp(struct intel_encoder *encoder)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
+	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
 
 	intel_dp_prepare(encoder);
 
+	if (port == PORT_A && IS_GEN5(dev_priv)) {
+		/*
+		 * We get FIFO underruns on the other pipe when
+		 * enabling the CPU eDP PLL, and when enabling CPU
+		 * eDP port. We could potentially avoid the PLL
+		 * underrun with a vblank wait just prior to enabling
+		 * the PLL, but that doesn't appear to help the port
+		 * enable case. Just sweep it all under the rug.
+		 */
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, !pipe, false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, !pipe, false);
+	}
+
 	/* Only ilk+ has port A */
-	if (dport->port == PORT_A) {
-		ironlake_set_pll_cpu_edp(intel_dp);
+	if (port == PORT_A)
 		ironlake_edp_pll_on(intel_dp);
-	}
 }
 
 static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
@@ -2645,7 +2817,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = intel_dig_port->base.base.dev->dev_private;
 	enum pipe pipe = intel_dp->pps_pipe;
-	int pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
+	i915_reg_t pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
 
 	edp_panel_vdd_off_sync(intel_dp);
 
@@ -2677,8 +2849,7 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
 	if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
 		return;
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-			    base.head) {
+	for_each_intel_encoder(dev, encoder) {
 		struct intel_dp *intel_dp;
 		enum port port;
 
@@ -3043,7 +3214,7 @@ intel_dp_dpcd_read_wake(struct drm_dp_aux *aux, unsigned int offset,
  * Fetch AUX CH registers 0x202 - 0x207 which contain
  * link status information
  */
-static bool
+bool
 intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE])
 {
 	return intel_dp_dpcd_read_wake(&intel_dp->aux,
@@ -3053,7 +3224,7 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_
 }
 
 /* These are source-specific values. */
-static uint8_t
+uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
@@ -3066,7 +3237,7 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
 		if (dev_priv->edp_low_vswing && port == PORT_A)
 			return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
-	} else if (IS_VALLEYVIEW(dev))
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
 	else if (IS_GEN7(dev) && port == PORT_A)
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
@@ -3076,7 +3247,7 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
 }
 
-static uint8_t
+uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
@@ -3107,7 +3278,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 		default:
 			return DP_TRAIN_PRE_EMPH_LEVEL_0;
 		}
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
 		case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
 			return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3418,38 +3589,6 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp)
 	return 0;
 }
 
-static void
-intel_get_adjust_train(struct intel_dp *intel_dp,
-		       const uint8_t link_status[DP_LINK_STATUS_SIZE])
-{
-	uint8_t v = 0;
-	uint8_t p = 0;
-	int lane;
-	uint8_t voltage_max;
-	uint8_t preemph_max;
-
-	for (lane = 0; lane < intel_dp->lane_count; lane++) {
-		uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane);
-		uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane);
-
-		if (this_v > v)
-			v = this_v;
-		if (this_p > p)
-			p = this_p;
-	}
-
-	voltage_max = intel_dp_voltage_max(intel_dp);
-	if (v >= voltage_max)
-		v = voltage_max | DP_TRAIN_MAX_SWING_REACHED;
-
-	preemph_max = intel_dp_pre_emphasis_max(intel_dp, v);
-	if (p >= preemph_max)
-		p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
-
-	for (lane = 0; lane < 4; lane++)
-		intel_dp->train_set[lane] = v | p;
-}
-
 static uint32_t
 gen4_signal_levels(uint8_t train_set)
 {
@@ -3547,13 +3686,13 @@ gen7_edp_signal_levels(uint8_t train_set)
 	}
 }
 
-/* Properly updates "DP" with the correct signal levels. */
-static void
-intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
+void
+intel_dp_set_signal_levels(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	enum port port = intel_dig_port->port;
 	struct drm_device *dev = intel_dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t signal_levels, mask = 0;
 	uint8_t train_set = intel_dp->train_set[0];
 
@@ -3588,74 +3727,27 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
 		(train_set & DP_TRAIN_PRE_EMPHASIS_MASK) >>
 			DP_TRAIN_PRE_EMPHASIS_SHIFT);
 
-	*DP = (*DP & ~mask) | signal_levels;
-}
-
-static bool
-intel_dp_set_link_train(struct intel_dp *intel_dp,
-			uint32_t *DP,
-			uint8_t dp_train_pat)
-{
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv =
-		to_i915(intel_dig_port->base.base.dev);
-	uint8_t buf[sizeof(intel_dp->train_set) + 1];
-	int ret, len;
-
-	_intel_dp_set_link_train(intel_dp, DP, dp_train_pat);
+	intel_dp->DP = (intel_dp->DP & ~mask) | signal_levels;
 
-	I915_WRITE(intel_dp->output_reg, *DP);
+	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
 	POSTING_READ(intel_dp->output_reg);
-
-	buf[0] = dp_train_pat;
-	if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) ==
-	    DP_TRAINING_PATTERN_DISABLE) {
-		/* don't write DP_TRAINING_LANEx_SET on disable */
-		len = 1;
-	} else {
-		/* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */
-		memcpy(buf + 1, intel_dp->train_set, intel_dp->lane_count);
-		len = intel_dp->lane_count + 1;
-	}
-
-	ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET,
-				buf, len);
-
-	return ret == len;
 }
 
-static bool
-intel_dp_reset_link_train(struct intel_dp *intel_dp, uint32_t *DP,
-			uint8_t dp_train_pat)
-{
-	if (!intel_dp->train_set_valid)
-		memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set));
-	intel_dp_set_signal_levels(intel_dp, DP);
-	return intel_dp_set_link_train(intel_dp, DP, dp_train_pat);
-}
-
-static bool
-intel_dp_update_link_train(struct intel_dp *intel_dp, uint32_t *DP,
-			   const uint8_t link_status[DP_LINK_STATUS_SIZE])
+void
+intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
+				       uint8_t dp_train_pat)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv =
 		to_i915(intel_dig_port->base.base.dev);
-	int ret;
 
-	intel_get_adjust_train(intel_dp, link_status);
-	intel_dp_set_signal_levels(intel_dp, DP);
+	_intel_dp_set_link_train(intel_dp, &intel_dp->DP, dp_train_pat);
 
-	I915_WRITE(intel_dp->output_reg, *DP);
+	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
 	POSTING_READ(intel_dp->output_reg);
-
-	ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET,
-				intel_dp->train_set, intel_dp->lane_count);
-
-	return ret == intel_dp->lane_count;
 }
 
-static void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
+void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = intel_dig_port->base.base.dev;
@@ -3686,232 +3778,6 @@ static void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 		DRM_ERROR("Timed out waiting for DP idle patterns\n");
 }
 
-/* Enable corresponding port and start training pattern 1 */
-static void
-intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
-{
-	struct drm_encoder *encoder = &dp_to_dig_port(intel_dp)->base.base;
-	struct drm_device *dev = encoder->dev;
-	int i;
-	uint8_t voltage;
-	int voltage_tries, loop_tries;
-	uint32_t DP = intel_dp->DP;
-	uint8_t link_config[2];
-	uint8_t link_bw, rate_select;
-
-	if (HAS_DDI(dev))
-		intel_ddi_prepare_link_retrain(encoder);
-
-	intel_dp_compute_rate(intel_dp, intel_dp->link_rate,
-			      &link_bw, &rate_select);
-
-	/* Write the link configuration data */
-	link_config[0] = link_bw;
-	link_config[1] = intel_dp->lane_count;
-	if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
-		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
-	drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
-	if (intel_dp->num_sink_rates)
-		drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET,
-				  &rate_select, 1);
-
-	link_config[0] = 0;
-	link_config[1] = DP_SET_ANSI_8B10B;
-	drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2);
-
-	DP |= DP_PORT_EN;
-
-	/* clock recovery */
-	if (!intel_dp_reset_link_train(intel_dp, &DP,
-				       DP_TRAINING_PATTERN_1 |
-				       DP_LINK_SCRAMBLING_DISABLE)) {
-		DRM_ERROR("failed to enable link training\n");
-		return;
-	}
-
-	voltage = 0xff;
-	voltage_tries = 0;
-	loop_tries = 0;
-	for (;;) {
-		uint8_t link_status[DP_LINK_STATUS_SIZE];
-
-		drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd);
-		if (!intel_dp_get_link_status(intel_dp, link_status)) {
-			DRM_ERROR("failed to get link status\n");
-			break;
-		}
-
-		if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) {
-			DRM_DEBUG_KMS("clock recovery OK\n");
-			break;
-		}
-
-		/*
-		 * if we used previously trained voltage and pre-emphasis values
-		 * and we don't get clock recovery, reset link training values
-		 */
-		if (intel_dp->train_set_valid) {
-			DRM_DEBUG_KMS("clock recovery not ok, reset");
-			/* clear the flag as we are not reusing train set */
-			intel_dp->train_set_valid = false;
-			if (!intel_dp_reset_link_train(intel_dp, &DP,
-						       DP_TRAINING_PATTERN_1 |
-						       DP_LINK_SCRAMBLING_DISABLE)) {
-				DRM_ERROR("failed to enable link training\n");
-				return;
-			}
-			continue;
-		}
-
-		/* Check to see if we've tried the max voltage */
-		for (i = 0; i < intel_dp->lane_count; i++)
-			if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
-				break;
-		if (i == intel_dp->lane_count) {
-			++loop_tries;
-			if (loop_tries == 5) {
-				DRM_ERROR("too many full retries, give up\n");
-				break;
-			}
-			intel_dp_reset_link_train(intel_dp, &DP,
-						  DP_TRAINING_PATTERN_1 |
-						  DP_LINK_SCRAMBLING_DISABLE);
-			voltage_tries = 0;
-			continue;
-		}
-
-		/* Check to see if we've tried the same voltage 5 times */
-		if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
-			++voltage_tries;
-			if (voltage_tries == 5) {
-				DRM_ERROR("too many voltage retries, give up\n");
-				break;
-			}
-		} else
-			voltage_tries = 0;
-		voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
-
-		/* Update training set as requested by target */
-		if (!intel_dp_update_link_train(intel_dp, &DP, link_status)) {
-			DRM_ERROR("failed to update link training\n");
-			break;
-		}
-	}
-
-	intel_dp->DP = DP;
-}
-
-static void
-intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-	bool channel_eq = false;
-	int tries, cr_tries;
-	uint32_t DP = intel_dp->DP;
-	uint32_t training_pattern = DP_TRAINING_PATTERN_2;
-
-	/*
-	 * Training Pattern 3 for HBR2 or 1.2 devices that support it.
-	 *
-	 * Intel platforms that support HBR2 also support TPS3. TPS3 support is
-	 * also mandatory for downstream devices that support HBR2.
-	 *
-	 * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is
-	 * supported but still not enabled.
-	 */
-	if (intel_dp_source_supports_hbr2(dev) &&
-	    drm_dp_tps3_supported(intel_dp->dpcd))
-		training_pattern = DP_TRAINING_PATTERN_3;
-	else if (intel_dp->link_rate == 540000)
-		DRM_ERROR("5.4 Gbps link rate without HBR2/TPS3 support\n");
-
-	/* channel equalization */
-	if (!intel_dp_set_link_train(intel_dp, &DP,
-				     training_pattern |
-				     DP_LINK_SCRAMBLING_DISABLE)) {
-		DRM_ERROR("failed to start channel equalization\n");
-		return;
-	}
-
-	tries = 0;
-	cr_tries = 0;
-	channel_eq = false;
-	for (;;) {
-		uint8_t link_status[DP_LINK_STATUS_SIZE];
-
-		if (cr_tries > 5) {
-			DRM_ERROR("failed to train DP, aborting\n");
-			break;
-		}
-
-		drm_dp_link_train_channel_eq_delay(intel_dp->dpcd);
-		if (!intel_dp_get_link_status(intel_dp, link_status)) {
-			DRM_ERROR("failed to get link status\n");
-			break;
-		}
-
-		/* Make sure clock is still ok */
-		if (!drm_dp_clock_recovery_ok(link_status,
-					      intel_dp->lane_count)) {
-			intel_dp->train_set_valid = false;
-			intel_dp_link_training_clock_recovery(intel_dp);
-			intel_dp_set_link_train(intel_dp, &DP,
-						training_pattern |
-						DP_LINK_SCRAMBLING_DISABLE);
-			cr_tries++;
-			continue;
-		}
-
-		if (drm_dp_channel_eq_ok(link_status,
-					 intel_dp->lane_count)) {
-			channel_eq = true;
-			break;
-		}
-
-		/* Try 5 times, then try clock recovery if that fails */
-		if (tries > 5) {
-			intel_dp->train_set_valid = false;
-			intel_dp_link_training_clock_recovery(intel_dp);
-			intel_dp_set_link_train(intel_dp, &DP,
-						training_pattern |
-						DP_LINK_SCRAMBLING_DISABLE);
-			tries = 0;
-			cr_tries++;
-			continue;
-		}
-
-		/* Update training set as requested by target */
-		if (!intel_dp_update_link_train(intel_dp, &DP, link_status)) {
-			DRM_ERROR("failed to update link training\n");
-			break;
-		}
-		++tries;
-	}
-
-	intel_dp_set_idle_link_train(intel_dp);
-
-	intel_dp->DP = DP;
-
-	if (channel_eq) {
-		intel_dp->train_set_valid = true;
-		DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n");
-	}
-}
-
-void intel_dp_stop_link_train(struct intel_dp *intel_dp)
-{
-	intel_dp_set_link_train(intel_dp, &intel_dp->DP,
-				DP_TRAINING_PATTERN_DISABLE);
-}
-
-void
-intel_dp_start_link_train(struct intel_dp *intel_dp)
-{
-	intel_dp_link_training_clock_recovery(intel_dp);
-	intel_dp_link_training_channel_equalization(intel_dp);
-}
-
 static void
 intel_dp_link_down(struct intel_dp *intel_dp)
 {
@@ -3954,6 +3820,13 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 	 * matching HDMI port to be enabled on transcoder A.
 	 */
 	if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B && port != PORT_A) {
+		/*
+		 * We get CPU/PCH FIFO underruns on the other pipe when
+		 * doing the workaround. Sweep them under the rug.
+		 */
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+
 		/* always enable with pattern 1 (as per spec) */
 		DP &= ~(DP_PIPEB_SELECT | DP_LINK_TRAIN_MASK);
 		DP |= DP_PORT_EN | DP_LINK_TRAIN_PAT_1;
@@ -3963,9 +3836,15 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 		DP &= ~DP_PORT_EN;
 		I915_WRITE(intel_dp->output_reg, DP);
 		POSTING_READ(intel_dp->output_reg);
+
+		intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
 
 	msleep(intel_dp->panel_power_down_delay);
+
+	intel_dp->DP = DP;
 }
 
 static bool
@@ -4013,7 +3892,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 	}
 
 	DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
-		      yesno(intel_dp_source_supports_hbr2(dev)),
+		      yesno(intel_dp_source_supports_hbr2(intel_dp)),
 		      yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
 
 	/* Intermediate frequency support */
@@ -4103,9 +3982,12 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
 static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = dig_port->base.base.dev;
 	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
 	u8 buf;
 	int ret = 0;
+	int count = 0;
+	int attempts = 10;
 
 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) {
 		DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n");
@@ -4120,7 +4002,22 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 		goto out;
 	}
 
-	intel_dp->sink_crc.started = false;
+	do {
+		intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+		if (drm_dp_dpcd_readb(&intel_dp->aux,
+				      DP_TEST_SINK_MISC, &buf) < 0) {
+			ret = -EIO;
+			goto out;
+		}
+		count = buf & DP_TEST_COUNT_MASK;
+	} while (--attempts && count);
+
+	if (attempts == 0) {
+		DRM_ERROR("TIMEOUT: Sink CRC counter is not zeroed\n");
+		ret = -ETIMEDOUT;
+	}
+
  out:
 	hsw_enable_ips(intel_crtc);
 	return ret;
@@ -4129,27 +4026,26 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = dig_port->base.base.dev;
 	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
 	u8 buf;
 	int ret;
 
-	if (intel_dp->sink_crc.started) {
-		ret = intel_dp_sink_crc_stop(intel_dp);
-		if (ret)
-			return ret;
-	}
-
 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0)
 		return -EIO;
 
 	if (!(buf & DP_TEST_CRC_SUPPORTED))
 		return -ENOTTY;
 
-	intel_dp->sink_crc.last_count = buf & DP_TEST_COUNT_MASK;
-
 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0)
 		return -EIO;
 
+	if (buf & DP_TEST_SINK_START) {
+		ret = intel_dp_sink_crc_stop(intel_dp);
+		if (ret)
+			return ret;
+	}
+
 	hsw_disable_ips(intel_crtc);
 
 	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
@@ -4158,7 +4054,7 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 		return -EIO;
 	}
 
-	intel_dp->sink_crc.started = true;
+	intel_wait_for_vblank(dev, intel_crtc->pipe);
 	return 0;
 }
 
@@ -4170,7 +4066,6 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 	u8 buf;
 	int count, ret;
 	int attempts = 6;
-	bool old_equal_new;
 
 	ret = intel_dp_sink_crc_start(intel_dp);
 	if (ret)
@@ -4186,35 +4081,17 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 		}
 		count = buf & DP_TEST_COUNT_MASK;
 
-		/*
-		 * Count might be reset during the loop. In this case
-		 * last known count needs to be reset as well.
-		 */
-		if (count == 0)
-			intel_dp->sink_crc.last_count = 0;
-
-		if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) {
-			ret = -EIO;
-			goto stop;
-		}
-
-		old_equal_new = (count == intel_dp->sink_crc.last_count &&
-				 !memcmp(intel_dp->sink_crc.last_crc, crc,
-					 6 * sizeof(u8)));
-
-	} while (--attempts && (count == 0 || old_equal_new));
-
-	intel_dp->sink_crc.last_count = buf & DP_TEST_COUNT_MASK;
-	memcpy(intel_dp->sink_crc.last_crc, crc, 6 * sizeof(u8));
+	} while (--attempts && count == 0);
 
 	if (attempts == 0) {
-		if (old_equal_new) {
-			DRM_DEBUG_KMS("Unreliable Sink CRC counter: Current returned CRC is identical to the previous one\n");
-		} else {
-			DRM_ERROR("Panel is unable to calculate any CRC after 6 vblanks\n");
-			ret = -ETIMEDOUT;
-			goto stop;
-		}
+		DRM_ERROR("Panel is unable to calculate any CRC after 6 vblanks\n");
+		ret = -ETIMEDOUT;
+		goto stop;
+	}
+
+	if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) {
+		ret = -EIO;
+		goto stop;
 	}
 
 stop:
@@ -4314,13 +4191,6 @@ static void intel_dp_handle_test_request(struct intel_dp *intel_dp)
 	uint8_t rxdata = 0;
 	int status = 0;
 
-	intel_dp->compliance_test_active = 0;
-	intel_dp->compliance_test_type = 0;
-	intel_dp->compliance_test_data = 0;
-
-	intel_dp->aux.i2c_nack_count = 0;
-	intel_dp->aux.i2c_defer_count = 0;
-
 	status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_REQUEST, &rxdata, 1);
 	if (status <= 0) {
 		DRM_DEBUG_KMS("Could not read test request from sink\n");
@@ -4436,6 +4306,14 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 
 	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
 
+	/*
+	 * Clearing compliance test variables to allow capturing
+	 * of values for next automated test request.
+	 */
+	intel_dp->compliance_test_active = 0;
+	intel_dp->compliance_test_type = 0;
+	intel_dp->compliance_test_data = 0;
+
 	if (!intel_encoder->base.crtc)
 		return;
 
@@ -4466,7 +4344,9 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 			DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n");
 	}
 
-	if (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) {
+	/* if link training is requested we should perform it always */
+	if ((intel_dp->compliance_test_type == DP_TEST_LINK_TRAINING) ||
+		(!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count))) {
 		DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n",
 			      intel_encoder->base.name);
 		intel_dp_start_link_train(intel_dp);
@@ -4678,47 +4558,12 @@ bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
 		return cpt_digital_port_connected(dev_priv, port);
 	else if (IS_BROXTON(dev_priv))
 		return bxt_digital_port_connected(dev_priv, port);
-	else if (IS_VALLEYVIEW(dev_priv))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return vlv_digital_port_connected(dev_priv, port);
 	else
 		return g4x_digital_port_connected(dev_priv, port);
 }
 
-static enum drm_connector_status
-ironlake_dp_detect(struct intel_dp *intel_dp)
-{
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-
-	if (!intel_digital_port_connected(dev_priv, intel_dig_port))
-		return connector_status_disconnected;
-
-	return intel_dp_detect_dpcd(intel_dp);
-}
-
-static enum drm_connector_status
-g4x_dp_detect(struct intel_dp *intel_dp)
-{
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-
-	/* Can't disconnect eDP, but you can close the lid... */
-	if (is_edp(intel_dp)) {
-		enum drm_connector_status status;
-
-		status = intel_panel_detect(dev);
-		if (status == connector_status_unknown)
-			status = connector_status_connected;
-		return status;
-	}
-
-	if (!intel_digital_port_connected(dev->dev_private, intel_dig_port))
-		return connector_status_disconnected;
-
-	return intel_dp_detect_dpcd(intel_dp);
-}
-
 static struct edid *
 intel_dp_get_edid(struct intel_dp *intel_dp)
 {
@@ -4791,12 +4636,19 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 	/* Can't disconnect eDP, but you can close the lid... */
 	if (is_edp(intel_dp))
 		status = edp_detect(intel_dp);
-	else if (HAS_PCH_SPLIT(dev))
-		status = ironlake_dp_detect(intel_dp);
+	else if (intel_digital_port_connected(to_i915(dev),
+					      dp_to_dig_port(intel_dp)))
+		status = intel_dp_detect_dpcd(intel_dp);
 	else
-		status = g4x_dp_detect(intel_dp);
-	if (status != connector_status_connected)
+		status = connector_status_disconnected;
+
+	if (status != connector_status_connected) {
+		intel_dp->compliance_test_active = 0;
+		intel_dp->compliance_test_type = 0;
+		intel_dp->compliance_test_data = 0;
+
 		goto out;
+	}
 
 	intel_dp_probe_oui(intel_dp);
 
@@ -4810,6 +4662,14 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 		goto out;
 	}
 
+	/*
+	 * Clearing NACK and defer counts to get their exact values
+	 * while reading EDID which are required by Compliance tests
+	 * 4.2.2.4 and 4.2.2.5
+	 */
+	intel_dp->aux.i2c_nack_count = 0;
+	intel_dp->aux.i2c_defer_count = 0;
+
 	intel_dp_set_edid(intel_dp);
 
 	if (intel_encoder->type != INTEL_OUTPUT_EDP)
@@ -5014,7 +4874,7 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder)
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 
-	drm_dp_aux_unregister(&intel_dp->aux);
+	intel_dp_aux_fini(intel_dp);
 	intel_dp_mst_encoder_cleanup(intel_dig_port);
 	if (is_edp(intel_dp)) {
 		cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
@@ -5092,7 +4952,7 @@ static void intel_dp_encoder_reset(struct drm_encoder *encoder)
 	 * Read out the current power sequencer assignment,
 	 * in case the BIOS did something with it.
 	 */
-	if (IS_VALLEYVIEW(encoder->dev))
+	if (IS_VALLEYVIEW(encoder->dev) || IS_CHERRYVIEW(encoder->dev))
 		vlv_initial_power_sequencer_setup(intel_dp);
 
 	intel_edp_panel_vdd_sanitize(intel_dp);
@@ -5204,25 +5064,6 @@ put_power:
 	return ret;
 }
 
-/* Return which DP Port should be selected for Transcoder DP control */
-int
-intel_trans_dp_port_sel(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct intel_encoder *intel_encoder;
-	struct intel_dp *intel_dp;
-
-	for_each_encoder_on_crtc(dev, crtc, intel_encoder) {
-		intel_dp = enc_to_intel_dp(&intel_encoder->base);
-
-		if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
-		    intel_encoder->type == INTEL_OUTPUT_EDP)
-			return intel_dp->output_reg;
-	}
-
-	return -1;
-}
-
 /* check the VBT to see whether the eDP is on another port */
 bool intel_dp_is_edp(struct drm_device *dev, enum port port)
 {
@@ -5294,7 +5135,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 	struct edp_power_seq cur, vbt, spec,
 		*final = &intel_dp->pps_delays;
 	u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0;
-	int pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg = 0;
+	i915_reg_t pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -5416,7 +5257,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 pp_on, pp_off, pp_div, port_sel = 0;
 	int div = HAS_PCH_SPLIT(dev) ? intel_pch_rawclk(dev) : intel_hrawclk(dev);
-	int pp_on_reg, pp_off_reg, pp_div_reg = 0, pp_ctrl_reg;
+	i915_reg_t pp_on_reg, pp_off_reg, pp_div_reg, pp_ctrl_reg;
 	enum port port = dp_to_dig_port(intel_dp)->port;
 	const struct edp_power_seq *seq = &intel_dp->pps_delays;
 
@@ -5471,7 +5312,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
 	/* Haswell doesn't have any port selection bits for the panel
 	 * power sequencer any more. */
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		port_sel = PANEL_PORT_SELECT_VLV(port);
 	} else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
 		if (port == PORT_A)
@@ -5578,17 +5419,17 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 			DRM_ERROR("Unsupported refreshrate type\n");
 		}
 	} else if (INTEL_INFO(dev)->gen > 6) {
-		u32 reg = PIPECONF(intel_crtc->config->cpu_transcoder);
+		i915_reg_t reg = PIPECONF(intel_crtc->config->cpu_transcoder);
 		u32 val;
 
 		val = I915_READ(reg);
 		if (index > DRRS_HIGH_RR) {
-			if (IS_VALLEYVIEW(dev))
+			if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 				val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV;
 			else
 				val |= PIPECONF_EDP_RR_MODE_SWITCH;
 		} else {
-			if (IS_VALLEYVIEW(dev))
+			if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 				val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV;
 			else
 				val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
@@ -5955,7 +5796,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	}
 	mutex_unlock(&dev->mode_config.mutex);
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		intel_dp->edp_notifier.notifier_call = edp_notify_handler;
 		register_reboot_notifier(&intel_dp->edp_notifier);
 
@@ -5996,14 +5837,14 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	enum port port = intel_dig_port->port;
-	int type;
+	int type, ret;
 
 	intel_dp->pps_pipe = INVALID_PIPE;
 
 	/* intel_dp vfuncs */
 	if (INTEL_INFO(dev)->gen >= 9)
 		intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider;
 	else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
 		intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
@@ -6017,6 +5858,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	else
 		intel_dp->get_aux_send_ctl = i9xx_get_aux_send_ctl;
 
+	if (HAS_DDI(dev))
+		intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain;
+
 	/* Preserve the current hw state. */
 	intel_dp->DP = I915_READ(intel_dp->output_reg);
 	intel_dp->attached_connector = intel_connector;
@@ -6035,8 +5879,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 		intel_encoder->type = INTEL_OUTPUT_EDP;
 
 	/* eDP only on port B and/or C on vlv/chv */
-	if (WARN_ON(IS_VALLEYVIEW(dev) && is_edp(intel_dp) &&
-		    port != PORT_B && port != PORT_C))
+	if (WARN_ON((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+		    is_edp(intel_dp) && port != PORT_B && port != PORT_C))
 		return false;
 
 	DRM_DEBUG_KMS("Adding %s connector on port %c\n",
@@ -6068,7 +5912,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 		break;
 	case PORT_B:
 		intel_encoder->hpd_pin = HPD_PORT_B;
-		if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0))
+		if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 			intel_encoder->hpd_pin = HPD_PORT_A;
 		break;
 	case PORT_C:
@@ -6087,14 +5931,16 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	if (is_edp(intel_dp)) {
 		pps_lock(intel_dp);
 		intel_dp_init_panel_power_timestamps(intel_dp);
-		if (IS_VALLEYVIEW(dev))
+		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 			vlv_initial_power_sequencer_setup(intel_dp);
 		else
 			intel_dp_init_panel_power_sequencer(dev, intel_dp);
 		pps_unlock(intel_dp);
 	}
 
-	intel_dp_aux_init(intel_dp, intel_connector);
+	ret = intel_dp_aux_init(intel_dp, intel_connector);
+	if (ret)
+		goto fail;
 
 	/* init MST on ports that can support it */
 	if (HAS_DP_MST(dev) &&
@@ -6103,20 +5949,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 					  intel_connector->base.base.id);
 
 	if (!intel_edp_init_connector(intel_dp, intel_connector)) {
-		drm_dp_aux_unregister(&intel_dp->aux);
-		if (is_edp(intel_dp)) {
-			cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
-			/*
-			 * vdd might still be enabled do to the delayed vdd off.
-			 * Make sure vdd is actually turned off here.
-			 */
-			pps_lock(intel_dp);
-			edp_panel_vdd_off_sync(intel_dp);
-			pps_unlock(intel_dp);
-		}
-		drm_connector_unregister(connector);
-		drm_connector_cleanup(connector);
-		return false;
+		intel_dp_aux_fini(intel_dp);
+		intel_dp_mst_encoder_cleanup(intel_dig_port);
+		goto fail;
 	}
 
 	intel_dp_add_properties(intel_dp, connector);
@@ -6133,10 +5968,27 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	i915_debugfs_connector_add(connector);
 
 	return true;
+
+fail:
+	if (is_edp(intel_dp)) {
+		cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
+		/*
+		 * vdd might still be enabled do to the delayed vdd off.
+		 * Make sure vdd is actually turned off here.
+		 */
+		pps_lock(intel_dp);
+		edp_panel_vdd_off_sync(intel_dp);
+		pps_unlock(intel_dp);
+	}
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+
+	return false;
 }
 
 void
-intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
+intel_dp_init(struct drm_device *dev,
+	      i915_reg_t output_reg, enum port port)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_digital_port *intel_dig_port;
@@ -6155,8 +6007,9 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
 	intel_encoder = &intel_dig_port->base;
 	encoder = &intel_encoder->base;
 
-	drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+	if (drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs,
+			     DRM_MODE_ENCODER_TMDS, NULL))
+		goto err_encoder_init;
 
 	intel_encoder->compute_config = intel_dp_compute_config;
 	intel_encoder->disable = intel_disable_dp;
@@ -6182,6 +6035,7 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
 	}
 
 	intel_dig_port->port = port;
+	dev_priv->dig_port_map[port] = intel_encoder;
 	intel_dig_port->dp.output_reg = output_reg;
 
 	intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
@@ -6205,6 +6059,7 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
 
 err_init_connector:
 	drm_encoder_cleanup(encoder);
+err_encoder_init:
 	kfree(intel_connector);
 err_connector_alloc:
 	kfree(intel_dig_port);
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
new file mode 100644
index 000000000000..88887938e0bf
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -0,0 +1,323 @@
+/*
+ * Copyright © 2008-2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "intel_drv.h"
+
+static void
+intel_get_adjust_train(struct intel_dp *intel_dp,
+		       const uint8_t link_status[DP_LINK_STATUS_SIZE])
+{
+	uint8_t v = 0;
+	uint8_t p = 0;
+	int lane;
+	uint8_t voltage_max;
+	uint8_t preemph_max;
+
+	for (lane = 0; lane < intel_dp->lane_count; lane++) {
+		uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane);
+		uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane);
+
+		if (this_v > v)
+			v = this_v;
+		if (this_p > p)
+			p = this_p;
+	}
+
+	voltage_max = intel_dp_voltage_max(intel_dp);
+	if (v >= voltage_max)
+		v = voltage_max | DP_TRAIN_MAX_SWING_REACHED;
+
+	preemph_max = intel_dp_pre_emphasis_max(intel_dp, v);
+	if (p >= preemph_max)
+		p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+
+	for (lane = 0; lane < 4; lane++)
+		intel_dp->train_set[lane] = v | p;
+}
+
+static bool
+intel_dp_set_link_train(struct intel_dp *intel_dp,
+			uint8_t dp_train_pat)
+{
+	uint8_t buf[sizeof(intel_dp->train_set) + 1];
+	int ret, len;
+
+	intel_dp_program_link_training_pattern(intel_dp, dp_train_pat);
+
+	buf[0] = dp_train_pat;
+	if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) ==
+	    DP_TRAINING_PATTERN_DISABLE) {
+		/* don't write DP_TRAINING_LANEx_SET on disable */
+		len = 1;
+	} else {
+		/* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */
+		memcpy(buf + 1, intel_dp->train_set, intel_dp->lane_count);
+		len = intel_dp->lane_count + 1;
+	}
+
+	ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET,
+				buf, len);
+
+	return ret == len;
+}
+
+static bool
+intel_dp_reset_link_train(struct intel_dp *intel_dp,
+			uint8_t dp_train_pat)
+{
+	if (!intel_dp->train_set_valid)
+		memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set));
+	intel_dp_set_signal_levels(intel_dp);
+	return intel_dp_set_link_train(intel_dp, dp_train_pat);
+}
+
+static bool
+intel_dp_update_link_train(struct intel_dp *intel_dp)
+{
+	int ret;
+
+	intel_dp_set_signal_levels(intel_dp);
+
+	ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET,
+				intel_dp->train_set, intel_dp->lane_count);
+
+	return ret == intel_dp->lane_count;
+}
+
+/* Enable corresponding port and start training pattern 1 */
+static void
+intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
+{
+	int i;
+	uint8_t voltage;
+	int voltage_tries, loop_tries;
+	uint8_t link_config[2];
+	uint8_t link_bw, rate_select;
+
+	if (intel_dp->prepare_link_retrain)
+		intel_dp->prepare_link_retrain(intel_dp);
+
+	intel_dp_compute_rate(intel_dp, intel_dp->link_rate,
+			      &link_bw, &rate_select);
+
+	/* Write the link configuration data */
+	link_config[0] = link_bw;
+	link_config[1] = intel_dp->lane_count;
+	if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
+		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+	drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
+	if (intel_dp->num_sink_rates)
+		drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET,
+				  &rate_select, 1);
+
+	link_config[0] = 0;
+	link_config[1] = DP_SET_ANSI_8B10B;
+	drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2);
+
+	intel_dp->DP |= DP_PORT_EN;
+
+	/* clock recovery */
+	if (!intel_dp_reset_link_train(intel_dp,
+				       DP_TRAINING_PATTERN_1 |
+				       DP_LINK_SCRAMBLING_DISABLE)) {
+		DRM_ERROR("failed to enable link training\n");
+		return;
+	}
+
+	voltage = 0xff;
+	voltage_tries = 0;
+	loop_tries = 0;
+	for (;;) {
+		uint8_t link_status[DP_LINK_STATUS_SIZE];
+
+		drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd);
+		if (!intel_dp_get_link_status(intel_dp, link_status)) {
+			DRM_ERROR("failed to get link status\n");
+			break;
+		}
+
+		if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) {
+			DRM_DEBUG_KMS("clock recovery OK\n");
+			break;
+		}
+
+		/*
+		 * if we used previously trained voltage and pre-emphasis values
+		 * and we don't get clock recovery, reset link training values
+		 */
+		if (intel_dp->train_set_valid) {
+			DRM_DEBUG_KMS("clock recovery not ok, reset");
+			/* clear the flag as we are not reusing train set */
+			intel_dp->train_set_valid = false;
+			if (!intel_dp_reset_link_train(intel_dp,
+						       DP_TRAINING_PATTERN_1 |
+						       DP_LINK_SCRAMBLING_DISABLE)) {
+				DRM_ERROR("failed to enable link training\n");
+				return;
+			}
+			continue;
+		}
+
+		/* Check to see if we've tried the max voltage */
+		for (i = 0; i < intel_dp->lane_count; i++)
+			if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
+				break;
+		if (i == intel_dp->lane_count) {
+			++loop_tries;
+			if (loop_tries == 5) {
+				DRM_ERROR("too many full retries, give up\n");
+				break;
+			}
+			intel_dp_reset_link_train(intel_dp,
+						  DP_TRAINING_PATTERN_1 |
+						  DP_LINK_SCRAMBLING_DISABLE);
+			voltage_tries = 0;
+			continue;
+		}
+
+		/* Check to see if we've tried the same voltage 5 times */
+		if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
+			++voltage_tries;
+			if (voltage_tries == 5) {
+				DRM_ERROR("too many voltage retries, give up\n");
+				break;
+			}
+		} else
+			voltage_tries = 0;
+		voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
+
+		/* Update training set as requested by target */
+		intel_get_adjust_train(intel_dp, link_status);
+		if (!intel_dp_update_link_train(intel_dp)) {
+			DRM_ERROR("failed to update link training\n");
+			break;
+		}
+	}
+}
+
+static void
+intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
+{
+	bool channel_eq = false;
+	int tries, cr_tries;
+	uint32_t training_pattern = DP_TRAINING_PATTERN_2;
+
+	/*
+	 * Training Pattern 3 for HBR2 or 1.2 devices that support it.
+	 *
+	 * Intel platforms that support HBR2 also support TPS3. TPS3 support is
+	 * also mandatory for downstream devices that support HBR2.
+	 *
+	 * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is
+	 * supported but still not enabled.
+	 */
+	if (intel_dp_source_supports_hbr2(intel_dp) &&
+	    drm_dp_tps3_supported(intel_dp->dpcd))
+		training_pattern = DP_TRAINING_PATTERN_3;
+	else if (intel_dp->link_rate == 540000)
+		DRM_ERROR("5.4 Gbps link rate without HBR2/TPS3 support\n");
+
+	/* channel equalization */
+	if (!intel_dp_set_link_train(intel_dp,
+				     training_pattern |
+				     DP_LINK_SCRAMBLING_DISABLE)) {
+		DRM_ERROR("failed to start channel equalization\n");
+		return;
+	}
+
+	tries = 0;
+	cr_tries = 0;
+	channel_eq = false;
+	for (;;) {
+		uint8_t link_status[DP_LINK_STATUS_SIZE];
+
+		if (cr_tries > 5) {
+			DRM_ERROR("failed to train DP, aborting\n");
+			break;
+		}
+
+		drm_dp_link_train_channel_eq_delay(intel_dp->dpcd);
+		if (!intel_dp_get_link_status(intel_dp, link_status)) {
+			DRM_ERROR("failed to get link status\n");
+			break;
+		}
+
+		/* Make sure clock is still ok */
+		if (!drm_dp_clock_recovery_ok(link_status,
+					      intel_dp->lane_count)) {
+			intel_dp->train_set_valid = false;
+			intel_dp_link_training_clock_recovery(intel_dp);
+			intel_dp_set_link_train(intel_dp,
+						training_pattern |
+						DP_LINK_SCRAMBLING_DISABLE);
+			cr_tries++;
+			continue;
+		}
+
+		if (drm_dp_channel_eq_ok(link_status,
+					 intel_dp->lane_count)) {
+			channel_eq = true;
+			break;
+		}
+
+		/* Try 5 times, then try clock recovery if that fails */
+		if (tries > 5) {
+			intel_dp->train_set_valid = false;
+			intel_dp_link_training_clock_recovery(intel_dp);
+			intel_dp_set_link_train(intel_dp,
+						training_pattern |
+						DP_LINK_SCRAMBLING_DISABLE);
+			tries = 0;
+			cr_tries++;
+			continue;
+		}
+
+		/* Update training set as requested by target */
+		intel_get_adjust_train(intel_dp, link_status);
+		if (!intel_dp_update_link_train(intel_dp)) {
+			DRM_ERROR("failed to update link training\n");
+			break;
+		}
+		++tries;
+	}
+
+	intel_dp_set_idle_link_train(intel_dp);
+
+	if (channel_eq) {
+		intel_dp->train_set_valid = true;
+		DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n");
+	}
+}
+
+void intel_dp_stop_link_train(struct intel_dp *intel_dp)
+{
+	intel_dp_set_link_train(intel_dp,
+				DP_TRAINING_PATTERN_DISABLE);
+}
+
+void
+intel_dp_start_link_train(struct intel_dp *intel_dp)
+{
+	intel_dp_link_training_clock_recovery(intel_dp);
+	intel_dp_link_training_channel_equalization(intel_dp);
+}
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 0639275fc471..fa0dabf578dc 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -78,6 +78,8 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 		return false;
 	}
 
+	if (drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, found->port))
+		pipe_config->has_audio = true;
 	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
 
 	pipe_config->pbn = mst_pbn;
@@ -102,6 +104,11 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder)
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc = encoder->base.crtc;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
 	int ret;
 
 	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
@@ -112,6 +119,10 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder)
 	if (ret) {
 		DRM_ERROR("failed to update payload %d\n", ret);
 	}
+	if (intel_crtc->config->has_audio) {
+		intel_audio_codec_disable(encoder);
+		intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
+	}
 }
 
 static void intel_mst_post_disable_dp(struct intel_encoder *encoder)
@@ -173,20 +184,14 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
 	intel_mst->port = found->port;
 
 	if (intel_dp->active_mst_links == 0) {
-		enum port port = intel_ddi_get_encoder_port(encoder);
+		intel_ddi_clk_select(encoder, intel_crtc->config);
 
 		intel_dp_set_link_params(intel_dp, intel_crtc->config);
 
-		/* FIXME: add support for SKL */
-		if (INTEL_INFO(dev)->gen < 9)
-			I915_WRITE(PORT_CLK_SEL(port),
-				   intel_crtc->config->ddi_pll_sel);
-
 		intel_ddi_init_dp_buf_reg(&intel_dig_port->base);
 
 		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
 
-
 		intel_dp_start_link_train(intel_dp);
 		intel_dp_stop_link_train(intel_dp);
 	}
@@ -214,6 +219,7 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder)
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 	enum port port = intel_dig_port->port;
 	int ret;
 
@@ -226,6 +232,13 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder)
 	ret = drm_dp_check_act_status(&intel_dp->mst_mgr);
 
 	ret = drm_dp_update_payload_part2(&intel_dp->mst_mgr);
+
+	if (crtc->config->has_audio) {
+		DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
+				 pipe_name(crtc->pipe));
+		intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
+		intel_audio_codec_enable(encoder);
+	}
 }
 
 static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder,
@@ -251,6 +264,9 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
 
 	pipe_config->has_dp_encoder = true;
 
+	pipe_config->has_audio =
+		intel_ddi_is_audio_enabled(dev_priv, crtc);
+
 	temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
 	if (temp & TRANS_DDI_PHSYNC)
 		flags |= DRM_MODE_FLAG_PHSYNC;
@@ -414,7 +430,10 @@ static void intel_connector_add_to_fbdev(struct intel_connector *connector)
 {
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper, &connector->base);
+
+	if (dev_priv->fbdev)
+		drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper,
+						&connector->base);
 #endif
 }
 
@@ -422,7 +441,10 @@ static void intel_connector_remove_from_fbdev(struct intel_connector *connector)
 {
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper, &connector->base);
+
+	if (dev_priv->fbdev)
+		drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper,
+						   &connector->base);
 #endif
 }
 
@@ -512,7 +534,7 @@ static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
 	drm_kms_helper_hotplug_event(dev);
 }
 
-static struct drm_dp_mst_topology_cbs mst_cbs = {
+static const struct drm_dp_mst_topology_cbs mst_cbs = {
 	.add_connector = intel_dp_add_mst_connector,
 	.register_connector = intel_dp_register_mst_connector,
 	.destroy_connector = intel_dp_destroy_mst_connector,
@@ -536,7 +558,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
 	intel_mst->primary = intel_dig_port;
 
 	drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs,
-			 DRM_MODE_ENCODER_DPMST);
+			 DRM_MODE_ENCODER_DPMST, NULL);
 
 	intel_encoder->type = INTEL_OUTPUT_DP_MST;
 	intel_encoder->crtc_mask = 0x7;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 0d00f07b7163..ea5415851c6e 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -123,8 +123,6 @@ struct intel_framebuffer {
 struct intel_fbdev {
 	struct drm_fb_helper helper;
 	struct intel_framebuffer *fb;
-	struct list_head fbdev_list;
-	struct drm_display_mode *our_mode;
 	int preferred_bpp;
 };
 
@@ -250,6 +248,7 @@ struct intel_atomic_state {
 	unsigned int cdclk;
 	bool dpll_set;
 	struct intel_shared_dpll_config shared_dpll[I915_NUM_PLLS];
+	struct intel_wm_config wm_config;
 };
 
 struct intel_plane_state {
@@ -280,6 +279,9 @@ struct intel_plane_state {
 	int scaler_id;
 
 	struct drm_intel_sprite_colorkey ckey;
+
+	/* async flip related structures */
+	struct drm_i915_gem_request *wait_req;
 };
 
 struct intel_initial_plane_config {
@@ -334,6 +336,21 @@ struct intel_crtc_scaler_state {
 /* drm_mode->private_flags */
 #define I915_MODE_FLAG_INHERITED 1
 
+struct intel_pipe_wm {
+	struct intel_wm_level wm[5];
+	uint32_t linetime;
+	bool fbc_wm_enabled;
+	bool pipe_enabled;
+	bool sprites_enabled;
+	bool sprites_scaled;
+};
+
+struct skl_pipe_wm {
+	struct skl_wm_level wm[8];
+	struct skl_wm_level trans_wm;
+	uint32_t linetime;
+};
+
 struct intel_crtc_state {
 	struct drm_crtc_state base;
 
@@ -348,7 +365,9 @@ struct intel_crtc_state {
 #define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS	(1<<0) /* unreliable sync mode.flags */
 	unsigned long quirks;
 
-	bool update_pipe;
+	bool update_pipe; /* can a fast modeset be performed? */
+	bool disable_cxsr;
+	bool wm_changed; /* watermarks are updated */
 
 	/* Pipe source size (ie. panel fitter input size)
 	 * All planes will be positioned inside this space,
@@ -376,6 +395,9 @@ struct intel_crtc_state {
 	 * accordingly. */
 	bool has_dp_encoder;
 
+	/* DSI has special cases */
+	bool has_dsi_encoder;
+
 	/* Whether we should send NULL infoframes. Required for audio. */
 	bool has_hdmi_sink;
 
@@ -468,6 +490,20 @@ struct intel_crtc_state {
 
 	/* w/a for waiting 2 vblanks during crtc enable */
 	enum pipe hsw_workaround_pipe;
+
+	/* IVB sprite scaling w/a (WaCxSRDisabledForSpriteScaling:ivb) */
+	bool disable_lp_wm;
+
+	struct {
+		/*
+		 * optimal watermarks, programmed post-vblank when this state
+		 * is committed
+		 */
+		union {
+			struct intel_pipe_wm ilk;
+			struct skl_pipe_wm skl;
+		} optimal;
+	} wm;
 };
 
 struct vlv_wm_state {
@@ -479,26 +515,12 @@ struct vlv_wm_state {
 	bool cxsr;
 };
 
-struct intel_pipe_wm {
-	struct intel_wm_level wm[5];
-	uint32_t linetime;
-	bool fbc_wm_enabled;
-	bool pipe_enabled;
-	bool sprites_enabled;
-	bool sprites_scaled;
-};
-
 struct intel_mmio_flip {
 	struct work_struct work;
 	struct drm_i915_private *i915;
 	struct drm_i915_gem_request *req;
 	struct intel_crtc *crtc;
-};
-
-struct skl_pipe_wm {
-	struct skl_wm_level wm[8];
-	struct skl_wm_level trans_wm;
-	uint32_t linetime;
+	unsigned int rotation;
 };
 
 /*
@@ -509,13 +531,9 @@ struct skl_pipe_wm {
  */
 struct intel_crtc_atomic_commit {
 	/* Sleepable operations to perform before commit */
-	bool wait_for_flips;
 	bool disable_fbc;
 	bool disable_ips;
-	bool disable_cxsr;
 	bool pre_disable_primary;
-	bool update_wm_pre, update_wm_post;
-	unsigned disabled_planes;
 
 	/* Sleepable operations to perform after commit */
 	unsigned fb_bits;
@@ -567,9 +585,10 @@ struct intel_crtc {
 	/* per-pipe watermark state */
 	struct {
 		/* watermarks currently being used  */
-		struct intel_pipe_wm active;
-		/* SKL wm values currently in use */
-		struct skl_pipe_wm skl_active;
+		union {
+			struct intel_pipe_wm ilk;
+			struct skl_pipe_wm skl;
+		} active;
 		/* allow CxSR on this pipe */
 		bool cxsr_allowed;
 	} wm;
@@ -677,7 +696,7 @@ struct cxsr_latency {
 #define intel_fb_obj(x) (x ? to_intel_framebuffer(x)->obj : NULL)
 
 struct intel_hdmi {
-	u32 hdmi_reg;
+	i915_reg_t hdmi_reg;
 	int ddc_bus;
 	bool limited_color_range;
 	bool color_range_auto;
@@ -693,7 +712,8 @@ struct intel_hdmi {
 	void (*set_infoframes)(struct drm_encoder *encoder,
 			       bool enable,
 			       const struct drm_display_mode *adjusted_mode);
-	bool (*infoframe_enabled)(struct drm_encoder *encoder);
+	bool (*infoframe_enabled)(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config);
 };
 
 struct intel_dp_mst_encoder;
@@ -719,15 +739,10 @@ enum link_m_n_set {
 	M2_N2
 };
 
-struct sink_crc {
-	bool started;
-	u8 last_crc[6];
-	int last_count;
-};
-
 struct intel_dp {
-	uint32_t output_reg;
-	uint32_t aux_ch_ctl_reg;
+	i915_reg_t output_reg;
+	i915_reg_t aux_ch_ctl_reg;
+	i915_reg_t aux_ch_data_reg[5];
 	uint32_t DP;
 	int link_rate;
 	uint8_t lane_count;
@@ -741,7 +756,6 @@ struct intel_dp {
 	/* sink rates as reported by DP_SUPPORTED_LINK_RATES */
 	uint8_t num_sink_rates;
 	int sink_rates[DP_MAX_SUPPORTED_RATES];
-	struct sink_crc sink_crc;
 	struct drm_dp_aux aux;
 	uint8_t train_set[4];
 	int panel_power_up_delay;
@@ -783,6 +797,10 @@ struct intel_dp {
 				     bool has_aux_irq,
 				     int send_bytes,
 				     uint32_t aux_clock_divider);
+
+	/* This is called before a link training is starterd */
+	void (*prepare_link_retrain)(struct intel_dp *intel_dp);
+
 	bool train_set_valid;
 
 	/* Displayport compliance testing */
@@ -799,6 +817,8 @@ struct intel_digital_port {
 	struct intel_hdmi hdmi;
 	enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool);
 	bool release_cl2_override;
+	/* for communication with audio component; protected by av_mutex */
+	const struct drm_connector *audio_connector;
 };
 
 struct intel_dp_mst_encoder {
@@ -942,7 +962,8 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 					 enum pipe pipe);
 void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 					 enum transcoder pch_transcoder);
-void i9xx_check_fifo_underruns(struct drm_i915_private *dev_priv);
+void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv);
+void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv);
 
 /* i915_irq.c */
 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
@@ -973,6 +994,8 @@ void intel_crt_init(struct drm_device *dev);
 
 
 /* intel_ddi.c */
+void intel_ddi_clk_select(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *pipe_config);
 void intel_prepare_ddi(struct drm_device *dev);
 void hsw_fdi_link_train(struct drm_crtc *crtc);
 void intel_ddi_init(struct drm_device *dev, enum port port);
@@ -987,9 +1010,11 @@ void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc);
 bool intel_ddi_pll_select(struct intel_crtc *crtc,
 			  struct intel_crtc_state *crtc_state);
 void intel_ddi_set_pipe_settings(struct drm_crtc *crtc);
-void intel_ddi_prepare_link_retrain(struct drm_encoder *encoder);
+void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp);
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
 void intel_ddi_fdi_disable(struct drm_crtc *crtc);
+bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
+				 struct intel_crtc *intel_crtc);
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config);
 struct intel_encoder *
@@ -1055,6 +1080,15 @@ intel_wait_for_vblank(struct drm_device *dev, int pipe)
 {
 	drm_wait_one_vblank(dev, pipe);
 }
+static inline void
+intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe)
+{
+	const struct intel_crtc *crtc =
+		to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
+
+	if (crtc->active)
+		intel_wait_for_vblank(dev, pipe);
+}
 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp);
 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 			 struct intel_digital_port *dport,
@@ -1068,9 +1102,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
 				    struct drm_modeset_acquire_ctx *ctx);
 int intel_pin_and_fence_fb_obj(struct drm_plane *plane,
 			       struct drm_framebuffer *fb,
-			       const struct drm_plane_state *plane_state,
-			       struct intel_engine_cs *pipelined,
-			       struct drm_i915_gem_request **pipelined_request);
+			       const struct drm_plane_state *plane_state);
 struct drm_framebuffer *
 __intel_framebuffer_create(struct drm_device *dev,
 			   struct drm_mode_fb_cmd2 *mode_cmd,
@@ -1151,7 +1183,10 @@ void broxton_ddi_phy_uninit(struct drm_device *dev);
 void bxt_enable_dc9(struct drm_i915_private *dev_priv);
 void bxt_disable_dc9(struct drm_i915_private *dev_priv);
 void skl_init_cdclk(struct drm_i915_private *dev_priv);
+int skl_sanitize_cdclk(struct drm_i915_private *dev_priv);
 void skl_uninit_cdclk(struct drm_i915_private *dev_priv);
+void skl_enable_dc6(struct drm_i915_private *dev_priv);
+void skl_disable_dc6(struct drm_i915_private *dev_priv);
 void intel_dp_get_m_n(struct intel_crtc *crtc,
 		      struct intel_crtc_state *pipe_config);
 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n);
@@ -1172,31 +1207,26 @@ enum intel_display_power_domain
 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder);
 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 				 struct intel_crtc_state *pipe_config);
-void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc);
 void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file);
 
 int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state);
 int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state);
 
-unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
-				     struct drm_i915_gem_object *obj,
-				     unsigned int plane);
+u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
+			   struct drm_i915_gem_object *obj,
+			   unsigned int plane);
 
 u32 skl_plane_ctl_format(uint32_t pixel_format);
 u32 skl_plane_ctl_tiling(uint64_t fb_modifier);
 u32 skl_plane_ctl_rotation(unsigned int rotation);
 
 /* intel_csr.c */
-void intel_csr_ucode_init(struct drm_device *dev);
-enum csr_state intel_csr_load_status_get(struct drm_i915_private *dev_priv);
-void intel_csr_load_status_set(struct drm_i915_private *dev_priv,
-					enum csr_state state);
-void intel_csr_load_program(struct drm_device *dev);
-void intel_csr_ucode_fini(struct drm_device *dev);
-void assert_csr_loaded(struct drm_i915_private *dev_priv);
+void intel_csr_ucode_init(struct drm_i915_private *);
+void intel_csr_load_program(struct drm_i915_private *);
+void intel_csr_ucode_fini(struct drm_i915_private *);
 
 /* intel_dp.c */
-void intel_dp_init(struct drm_device *dev, int output_reg, enum port port);
+void intel_dp_init(struct drm_device *dev, i915_reg_t output_reg, enum port port);
 bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 			     struct intel_connector *intel_connector);
 void intel_dp_set_link_params(struct intel_dp *intel_dp,
@@ -1234,6 +1264,22 @@ bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
 					 struct intel_digital_port *port);
 void hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config);
 
+void
+intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
+				       uint8_t dp_train_pat);
+void
+intel_dp_set_signal_levels(struct intel_dp *intel_dp);
+void intel_dp_set_idle_link_train(struct intel_dp *intel_dp);
+uint8_t
+intel_dp_voltage_max(struct intel_dp *intel_dp);
+uint8_t
+intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing);
+void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
+			   uint8_t *link_bw, uint8_t *rate_select);
+bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp);
+bool
+intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]);
+
 /* intel_dp_mst.c */
 int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
@@ -1248,7 +1294,7 @@ void intel_dvo_init(struct drm_device *dev);
 /* legacy fbdev emulation in intel_fbdev.c */
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 extern int intel_fbdev_init(struct drm_device *dev);
-extern void intel_fbdev_initial_config(void *data, async_cookie_t cookie);
+extern void intel_fbdev_initial_config_async(struct drm_device *dev);
 extern void intel_fbdev_fini(struct drm_device *dev);
 extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
 extern void intel_fbdev_output_poll_changed(struct drm_device *dev);
@@ -1259,7 +1305,7 @@ static inline int intel_fbdev_init(struct drm_device *dev)
 	return 0;
 }
 
-static inline void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
+static inline void intel_fbdev_initial_config_async(struct drm_device *dev)
 {
 }
 
@@ -1277,9 +1323,11 @@ static inline void intel_fbdev_restore_mode(struct drm_device *dev)
 #endif
 
 /* intel_fbc.c */
-bool intel_fbc_enabled(struct drm_i915_private *dev_priv);
-void intel_fbc_update(struct drm_i915_private *dev_priv);
+bool intel_fbc_is_active(struct drm_i915_private *dev_priv);
+void intel_fbc_deactivate(struct intel_crtc *crtc);
+void intel_fbc_update(struct intel_crtc *crtc);
 void intel_fbc_init(struct drm_i915_private *dev_priv);
+void intel_fbc_enable(struct intel_crtc *crtc);
 void intel_fbc_disable(struct drm_i915_private *dev_priv);
 void intel_fbc_disable_crtc(struct intel_crtc *crtc);
 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
@@ -1287,11 +1335,10 @@ void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
 			  enum fb_op_origin origin);
 void intel_fbc_flush(struct drm_i915_private *dev_priv,
 		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
-const char *intel_no_fbc_reason_str(enum no_fbc_reason reason);
 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
 
 /* intel_hdmi.c */
-void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port);
+void intel_hdmi_init(struct drm_device *dev, i915_reg_t hdmi_reg, enum port port);
 void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 			       struct intel_connector *intel_connector);
 struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
@@ -1367,8 +1414,13 @@ void intel_psr_single_frame_update(struct drm_device *dev,
 /* intel_runtime_pm.c */
 int intel_power_domains_init(struct drm_i915_private *);
 void intel_power_domains_fini(struct drm_i915_private *);
-void intel_power_domains_init_hw(struct drm_i915_private *dev_priv);
+void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume);
+void intel_power_domains_suspend(struct drm_i915_private *dev_priv);
+void skl_pw1_misc_io_init(struct drm_i915_private *dev_priv);
+void skl_pw1_misc_io_fini(struct drm_i915_private *dev_priv);
 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv);
+const char *
+intel_display_power_domain_str(enum intel_display_power_domain domain);
 
 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 				    enum intel_display_power_domain domain);
@@ -1378,6 +1430,89 @@ void intel_display_power_get(struct drm_i915_private *dev_priv,
 			     enum intel_display_power_domain domain);
 void intel_display_power_put(struct drm_i915_private *dev_priv,
 			     enum intel_display_power_domain domain);
+
+static inline void
+assert_rpm_device_not_suspended(struct drm_i915_private *dev_priv)
+{
+	WARN_ONCE(dev_priv->pm.suspended,
+		  "Device suspended during HW access\n");
+}
+
+static inline void
+assert_rpm_wakelock_held(struct drm_i915_private *dev_priv)
+{
+	assert_rpm_device_not_suspended(dev_priv);
+	/* FIXME: Needs to be converted back to WARN_ONCE, but currently causes
+	 * too much noise. */
+	if (!atomic_read(&dev_priv->pm.wakeref_count))
+		DRM_DEBUG_DRIVER("RPM wakelock ref not held during HW access");
+}
+
+static inline int
+assert_rpm_atomic_begin(struct drm_i915_private *dev_priv)
+{
+	int seq = atomic_read(&dev_priv->pm.atomic_seq);
+
+	assert_rpm_wakelock_held(dev_priv);
+
+	return seq;
+}
+
+static inline void
+assert_rpm_atomic_end(struct drm_i915_private *dev_priv, int begin_seq)
+{
+	WARN_ONCE(atomic_read(&dev_priv->pm.atomic_seq) != begin_seq,
+		  "HW access outside of RPM atomic section\n");
+}
+
+/**
+ * disable_rpm_wakeref_asserts - disable the RPM assert checks
+ * @dev_priv: i915 device instance
+ *
+ * This function disable asserts that check if we hold an RPM wakelock
+ * reference, while keeping the device-not-suspended checks still enabled.
+ * It's meant to be used only in special circumstances where our rule about
+ * the wakelock refcount wrt. the device power state doesn't hold. According
+ * to this rule at any point where we access the HW or want to keep the HW in
+ * an active state we must hold an RPM wakelock reference acquired via one of
+ * the intel_runtime_pm_get() helpers. Currently there are a few special spots
+ * where this rule doesn't hold: the IRQ and suspend/resume handlers, the
+ * forcewake release timer, and the GPU RPS and hangcheck works. All other
+ * users should avoid using this function.
+ *
+ * Any calls to this function must have a symmetric call to
+ * enable_rpm_wakeref_asserts().
+ */
+static inline void
+disable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv)
+{
+	atomic_inc(&dev_priv->pm.wakeref_count);
+}
+
+/**
+ * enable_rpm_wakeref_asserts - re-enable the RPM assert checks
+ * @dev_priv: i915 device instance
+ *
+ * This function re-enables the RPM assert checks after disabling them with
+ * disable_rpm_wakeref_asserts. It's meant to be used only in special
+ * circumstances otherwise its use should be avoided.
+ *
+ * Any calls to this function must have a symmetric call to
+ * disable_rpm_wakeref_asserts().
+ */
+static inline void
+enable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv)
+{
+	atomic_dec(&dev_priv->pm.wakeref_count);
+}
+
+/* TODO: convert users of these to rely instead on proper RPM refcounting */
+#define DISABLE_RPM_WAKEREF_ASSERTS(dev_priv)	\
+	disable_rpm_wakeref_asserts(dev_priv)
+
+#define ENABLE_RPM_WAKEREF_ASSERTS(dev_priv)	\
+	enable_rpm_wakeref_asserts(dev_priv)
+
 void intel_runtime_pm_get(struct drm_i915_private *dev_priv);
 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv);
 void intel_runtime_pm_put(struct drm_i915_private *dev_priv);
@@ -1395,12 +1530,6 @@ void intel_init_clock_gating(struct drm_device *dev);
 void intel_suspend_hw(struct drm_device *dev);
 int ilk_wm_max_level(const struct drm_device *dev);
 void intel_update_watermarks(struct drm_crtc *crtc);
-void intel_update_sprite_watermarks(struct drm_plane *plane,
-				    struct drm_crtc *crtc,
-				    uint32_t sprite_width,
-				    uint32_t sprite_height,
-				    int pixel_size,
-				    bool enabled, bool scaled);
 void intel_init_pm(struct drm_device *dev);
 void intel_pm_setup(struct drm_device *dev);
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
@@ -1428,7 +1557,8 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
 
 /* intel_sdvo.c */
-bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob);
+bool intel_sdvo_init(struct drm_device *dev,
+		     i915_reg_t reg, enum port port);
 
 
 /* intel_sprite.c */
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 170ae6f4866e..44742fa2f616 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -60,7 +60,8 @@ static void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 		DRM_ERROR("DPI FIFOs are not empty\n");
 }
 
-static void write_data(struct drm_i915_private *dev_priv, u32 reg,
+static void write_data(struct drm_i915_private *dev_priv,
+		       i915_reg_t reg,
 		       const u8 *data, u32 len)
 {
 	u32 i, j;
@@ -75,7 +76,8 @@ static void write_data(struct drm_i915_private *dev_priv, u32 reg,
 	}
 }
 
-static void read_data(struct drm_i915_private *dev_priv, u32 reg,
+static void read_data(struct drm_i915_private *dev_priv,
+		      i915_reg_t reg,
 		      u8 *data, u32 len)
 {
 	u32 i, j;
@@ -98,7 +100,8 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 	struct mipi_dsi_packet packet;
 	ssize_t ret;
 	const u8 *header, *data;
-	u32 data_reg, data_mask, ctrl_reg, ctrl_mask;
+	i915_reg_t data_reg, ctrl_reg;
+	u32 data_mask, ctrl_mask;
 
 	ret = mipi_dsi_create_packet(&packet, msg);
 	if (ret < 0)
@@ -263,16 +266,18 @@ static inline bool is_cmd_mode(struct intel_dsi *intel_dsi)
 }
 
 static bool intel_dsi_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *config)
+				     struct intel_crtc_state *pipe_config)
 {
 	struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi,
 						   base);
 	struct intel_connector *intel_connector = intel_dsi->attached_connector;
 	struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode;
-	struct drm_display_mode *adjusted_mode = &config->base.adjusted_mode;
+	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
 	DRM_DEBUG_KMS("\n");
 
+	pipe_config->has_dsi_encoder = true;
+
 	if (fixed_mode)
 		intel_fixed_panel_mode(fixed_mode, adjusted_mode);
 
@@ -364,7 +369,7 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder)
 {
 	struct drm_device *dev = encoder->base.dev;
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_dsi_device_ready(encoder);
 	else if (IS_BROXTON(dev))
 		bxt_dsi_device_ready(encoder);
@@ -377,10 +382,10 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
-	u32 temp;
-	u32 port_ctrl;
 
 	if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) {
+		u32 temp;
+
 		temp = I915_READ(VLV_CHICKEN_3);
 		temp &= ~PIXEL_OVERLAP_CNT_MASK |
 					intel_dsi->pixel_overlap <<
@@ -389,8 +394,9 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		port_ctrl = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
-						MIPI_PORT_CTRL(port);
+		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		u32 temp;
 
 		temp = I915_READ(port_ctrl);
 
@@ -416,13 +422,13 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
-	u32 temp;
-	u32 port_ctrl;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
+		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		u32 temp;
+
 		/* de-assert ip_tg_enable signal */
-		port_ctrl = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
-						MIPI_PORT_CTRL(port);
 		temp = I915_READ(port_ctrl);
 		I915_WRITE(port_ctrl, temp & ~DPI_ENABLE);
 		POSTING_READ(port_ctrl);
@@ -458,6 +464,8 @@ static void intel_dsi_enable(struct intel_encoder *encoder)
 	intel_panel_enable_backlight(intel_dsi->attached_connector);
 }
 
+static void intel_dsi_prepare(struct intel_encoder *intel_encoder);
+
 static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 {
 	struct drm_device *dev = encoder->base.dev;
@@ -470,13 +478,16 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 
 	DRM_DEBUG_KMS("\n");
 
+	intel_dsi_prepare(encoder);
+	intel_enable_dsi_pll(encoder);
+
 	/* Panel Enable over CRC PMIC */
 	if (intel_dsi->gpio_panel)
 		gpiod_set_value_cansleep(intel_dsi->gpio_panel, 1);
 
 	msleep(intel_dsi->panel_on_delay);
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/*
 		 * Disable DPOunit clock gating, can stall pipe
 		 * and we need DPLL REFA always enabled
@@ -580,11 +591,13 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
-	u32 val;
-	u32 port_ctrl = 0;
 
 	DRM_DEBUG_KMS("\n");
 	for_each_dsi_port(port, intel_dsi->ports) {
+		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
+		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
+		u32 val;
 
 		I915_WRITE(MIPI_DEVICE_READY(port), DEVICE_READY |
 							ULPS_STATE_ENTER);
@@ -598,12 +611,6 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 							ULPS_STATE_ENTER);
 		usleep_range(2000, 2500);
 
-		if (IS_BROXTON(dev))
-			port_ctrl = BXT_MIPI_PORT_CTRL(port);
-		else if (IS_VALLEYVIEW(dev))
-			/* Common bit for both MIPI Port A & MIPI Port C */
-			port_ctrl = MIPI_PORT_CTRL(PORT_A);
-
 		/* Wait till Clock lanes are in LP-00 state for MIPI Port A
 		 * only. MIPI Port C has no similar bit for checking
 		 */
@@ -656,7 +663,6 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	struct drm_device *dev = encoder->base.dev;
 	enum intel_display_power_domain power_domain;
-	u32 dpi_enabled, func, ctrl_reg;
 	enum port port;
 
 	DRM_DEBUG_KMS("\n");
@@ -667,17 +673,18 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 
 	/* XXX: this only works for one DSI output */
 	for_each_dsi_port(port, intel_dsi->ports) {
+		i915_reg_t ctrl_reg = IS_BROXTON(dev) ?
+			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		u32 dpi_enabled, func;
+
 		func = I915_READ(MIPI_DSI_FUNC_PRG(port));
-		ctrl_reg = IS_BROXTON(dev) ? BXT_MIPI_PORT_CTRL(port) :
-						MIPI_PORT_CTRL(port);
 		dpi_enabled = I915_READ(ctrl_reg) & DPI_ENABLE;
 
 		/* Due to some hardware limitations on BYT, MIPI Port C DPI
 		 * Enable bit does not get set. To check whether DSI Port C
 		 * was enabled in BIOS, check the Pipe B enable bit
 		 */
-		if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
-		    (port == PORT_C))
+		if (IS_VALLEYVIEW(dev) && port == PORT_C)
 			dpi_enabled = I915_READ(PIPECONF(PIPE_B)) &
 							PIPECONF_ENABLE;
 
@@ -698,6 +705,8 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 	u32 pclk = 0;
 	DRM_DEBUG_KMS("\n");
 
+	pipe_config->has_dsi_encoder = true;
+
 	/*
 	 * DPLL_MD is not used in case of DSI, reading will get some default value
 	 * set dpll_md = 0
@@ -706,7 +715,8 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 
 	if (IS_BROXTON(encoder->base.dev))
 		pclk = bxt_get_dsi_pclk(encoder, pipe_config->pipe_bpp);
-	else if (IS_VALLEYVIEW(encoder->base.dev))
+	else if (IS_VALLEYVIEW(encoder->base.dev) ||
+		 IS_CHERRYVIEW(encoder->base.dev))
 		pclk = vlv_get_dsi_pclk(encoder, pipe_config->pipe_bpp);
 
 	if (!pclk)
@@ -859,7 +869,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (IS_VALLEYVIEW(dev)) {
+		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 			/*
 			 * escape clock divider, 20MHz, shared for A and C.
 			 * device ready must be off when doing this! txclkesc?
@@ -875,21 +885,12 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
 			I915_WRITE(MIPI_CTRL(port), tmp |
 					READ_REQUEST_PRIORITY_HIGH);
 		} else if (IS_BROXTON(dev)) {
-			/*
-			 * FIXME:
-			 * BXT can connect any PIPE to any MIPI port.
-			 * Select the pipe based on the MIPI port read from
-			 * VBT for now. Pick PIPE A for MIPI port A and C
-			 * for port C.
-			 */
+			enum pipe pipe = intel_crtc->pipe;
+
 			tmp = I915_READ(MIPI_CTRL(port));
 			tmp &= ~BXT_PIPE_SELECT_MASK;
 
-			if (port == PORT_A)
-				tmp |= BXT_PIPE_SELECT_A;
-			else if (port == PORT_C)
-				tmp |= BXT_PIPE_SELECT_C;
-
+			tmp |= BXT_PIPE_SELECT(pipe);
 			I915_WRITE(MIPI_CTRL(port), tmp);
 		}
 
@@ -1025,15 +1026,6 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
 	}
 }
 
-static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder)
-{
-	DRM_DEBUG_KMS("\n");
-
-	intel_dsi_prepare(encoder);
-	intel_enable_dsi_pll(encoder);
-
-}
-
 static enum drm_connector_status
 intel_dsi_detect(struct drm_connector *connector, bool force)
 {
@@ -1128,7 +1120,7 @@ void intel_dsi_init(struct drm_device *dev)
 	if (!dev_priv->vbt.has_mipi)
 		return;
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
 	} else {
 		DRM_ERROR("Unsupported Mipi device to reg base");
@@ -1151,11 +1143,10 @@ void intel_dsi_init(struct drm_device *dev)
 
 	connector = &intel_connector->base;
 
-	drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI);
+	drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI,
+			 NULL);
 
-	/* XXX: very likely not all of these are needed */
 	intel_encoder->compute_config = intel_dsi_compute_config;
-	intel_encoder->pre_pll_enable = intel_dsi_pre_pll_enable;
 	intel_encoder->pre_enable = intel_dsi_pre_enable;
 	intel_encoder->enable = intel_dsi_enable_nop;
 	intel_encoder->disable = intel_dsi_pre_disable;
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index e6cb25239941..02551ff228c2 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -117,7 +117,7 @@ static inline struct intel_dsi_host *to_intel_dsi_host(struct mipi_dsi_host *h)
 
 #define for_each_dsi_port(__port, __ports_mask) \
 	for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++)	\
-		if ((__ports_mask) & (1 << (__port)))
+		for_each_if ((__ports_mask) & (1 << (__port)))
 
 static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder)
 {
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
index cb3cf3986212..fbd2b51810ca 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
@@ -561,7 +561,7 @@ void intel_enable_dsi_pll(struct intel_encoder *encoder)
 {
 	struct drm_device *dev = encoder->base.dev;
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_enable_dsi_pll(encoder);
 	else if (IS_BROXTON(dev))
 		bxt_enable_dsi_pll(encoder);
@@ -571,7 +571,7 @@ void intel_disable_dsi_pll(struct intel_encoder *encoder)
 {
 	struct drm_device *dev = encoder->base.dev;
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_disable_dsi_pll(encoder);
 	else if (IS_BROXTON(dev))
 		bxt_disable_dsi_pll(encoder);
@@ -599,6 +599,6 @@ void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 
 	if (IS_BROXTON(dev))
 		bxt_dsi_reset_clocks(encoder, port);
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_dsi_reset_clocks(encoder, port);
 }
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 8492053e0ff0..286baec979c8 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -44,6 +44,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_TMDS,
 		.name = "sil164",
 		.dvo_reg = DVOC,
+		.dvo_srcdim_reg = DVOC_SRCDIM,
 		.slave_addr = SIL164_ADDR,
 		.dev_ops = &sil164_ops,
 	},
@@ -51,6 +52,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_TMDS,
 		.name = "ch7xxx",
 		.dvo_reg = DVOC,
+		.dvo_srcdim_reg = DVOC_SRCDIM,
 		.slave_addr = CH7xxx_ADDR,
 		.dev_ops = &ch7xxx_ops,
 	},
@@ -58,6 +60,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_TMDS,
 		.name = "ch7xxx",
 		.dvo_reg = DVOC,
+		.dvo_srcdim_reg = DVOC_SRCDIM,
 		.slave_addr = 0x75, /* For some ch7010 */
 		.dev_ops = &ch7xxx_ops,
 	},
@@ -65,6 +68,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_LVDS,
 		.name = "ivch",
 		.dvo_reg = DVOA,
+		.dvo_srcdim_reg = DVOA_SRCDIM,
 		.slave_addr = 0x02, /* Might also be 0x44, 0x84, 0xc4 */
 		.dev_ops = &ivch_ops,
 	},
@@ -72,6 +76,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_TMDS,
 		.name = "tfp410",
 		.dvo_reg = DVOC,
+		.dvo_srcdim_reg = DVOC_SRCDIM,
 		.slave_addr = TFP410_ADDR,
 		.dev_ops = &tfp410_ops,
 	},
@@ -79,6 +84,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.type = INTEL_DVO_CHIP_LVDS,
 		.name = "ch7017",
 		.dvo_reg = DVOC,
+		.dvo_srcdim_reg = DVOC_SRCDIM,
 		.slave_addr = 0x75,
 		.gpio = GMBUS_PIN_DPB,
 		.dev_ops = &ch7017_ops,
@@ -87,6 +93,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 	        .type = INTEL_DVO_CHIP_TMDS,
 		.name = "ns2501",
 		.dvo_reg = DVOB,
+		.dvo_srcdim_reg = DVOB_SRCDIM,
 		.slave_addr = NS2501_ADDR,
 		.dev_ops = &ns2501_ops,
        }
@@ -171,7 +178,7 @@ static void intel_disable_dvo(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
-	u32 dvo_reg = intel_dvo->dev.dvo_reg;
+	i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
 	u32 temp = I915_READ(dvo_reg);
 
 	intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false);
@@ -184,7 +191,7 @@ static void intel_enable_dvo(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	u32 dvo_reg = intel_dvo->dev.dvo_reg;
+	i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
 	u32 temp = I915_READ(dvo_reg);
 
 	intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
@@ -255,20 +262,8 @@ static void intel_dvo_pre_enable(struct intel_encoder *encoder)
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	int pipe = crtc->pipe;
 	u32 dvo_val;
-	u32 dvo_reg = intel_dvo->dev.dvo_reg, dvo_srcdim_reg;
-
-	switch (dvo_reg) {
-	case DVOA:
-	default:
-		dvo_srcdim_reg = DVOA_SRCDIM;
-		break;
-	case DVOB:
-		dvo_srcdim_reg = DVOB_SRCDIM;
-		break;
-	case DVOC:
-		dvo_srcdim_reg = DVOC_SRCDIM;
-		break;
-	}
+	i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
+	i915_reg_t dvo_srcdim_reg = intel_dvo->dev.dvo_srcdim_reg;
 
 	/* Save the data order, since I don't know what it should be set to. */
 	dvo_val = I915_READ(dvo_reg) &
@@ -434,7 +429,7 @@ void intel_dvo_init(struct drm_device *dev)
 
 	intel_encoder = &intel_dvo->base;
 	drm_encoder_init(dev, &intel_encoder->base,
-			 &intel_dvo_enc_funcs, encoder_type);
+			 &intel_dvo_enc_funcs, encoder_type, NULL);
 
 	intel_encoder->disable = intel_disable_dvo;
 	intel_encoder->enable = intel_enable_dvo;
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index cf47352b7b8e..a1988a486b92 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -43,7 +43,17 @@
 
 static inline bool fbc_supported(struct drm_i915_private *dev_priv)
 {
-	return dev_priv->fbc.enable_fbc != NULL;
+	return dev_priv->fbc.activate != NULL;
+}
+
+static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv)
+{
+	return IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8;
+}
+
+static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv)
+{
+	return INTEL_INFO(dev_priv)->gen < 4;
 }
 
 /*
@@ -59,11 +69,51 @@ static unsigned int get_crtc_fence_y_offset(struct intel_crtc *crtc)
 	return crtc->base.y - crtc->adjusted_y;
 }
 
-static void i8xx_fbc_disable(struct drm_i915_private *dev_priv)
+/*
+ * For SKL+, the plane source size used by the hardware is based on the value we
+ * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value
+ * we wrote to PIPESRC.
+ */
+static void intel_fbc_get_plane_source_size(struct intel_crtc *crtc,
+					    int *width, int *height)
+{
+	struct intel_plane_state *plane_state =
+			to_intel_plane_state(crtc->base.primary->state);
+	int w, h;
+
+	if (intel_rotation_90_or_270(plane_state->base.rotation)) {
+		w = drm_rect_height(&plane_state->src) >> 16;
+		h = drm_rect_width(&plane_state->src) >> 16;
+	} else {
+		w = drm_rect_width(&plane_state->src) >> 16;
+		h = drm_rect_height(&plane_state->src) >> 16;
+	}
+
+	if (width)
+		*width = w;
+	if (height)
+		*height = h;
+}
+
+static int intel_fbc_calculate_cfb_size(struct intel_crtc *crtc,
+					struct drm_framebuffer *fb)
+{
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+	int lines;
+
+	intel_fbc_get_plane_source_size(crtc, NULL, &lines);
+	if (INTEL_INFO(dev_priv)->gen >= 7)
+		lines = min(lines, 2048);
+
+	/* Hardware needs the full buffer stride, not just the active area. */
+	return lines * fb->pitches[0];
+}
+
+static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)
 {
 	u32 fbc_ctl;
 
-	dev_priv->fbc.enabled = false;
+	dev_priv->fbc.active = false;
 
 	/* Disable compression */
 	fbc_ctl = I915_READ(FBC_CONTROL);
@@ -78,11 +128,9 @@ static void i8xx_fbc_disable(struct drm_i915_private *dev_priv)
 		DRM_DEBUG_KMS("FBC idle timed out\n");
 		return;
 	}
-
-	DRM_DEBUG_KMS("disabled FBC\n");
 }
 
-static void i8xx_fbc_enable(struct intel_crtc *crtc)
+static void i8xx_fbc_activate(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	struct drm_framebuffer *fb = crtc->base.primary->fb;
@@ -91,10 +139,10 @@ static void i8xx_fbc_enable(struct intel_crtc *crtc)
 	int i;
 	u32 fbc_ctl;
 
-	dev_priv->fbc.enabled = true;
+	dev_priv->fbc.active = true;
 
 	/* Note: fbc.threshold == 1 for i8xx */
-	cfb_pitch = dev_priv->fbc.uncompressed_size / FBC_LL_SIZE;
+	cfb_pitch = intel_fbc_calculate_cfb_size(crtc, fb) / FBC_LL_SIZE;
 	if (fb->pitches[0] < cfb_pitch)
 		cfb_pitch = fb->pitches[0];
 
@@ -127,24 +175,21 @@ static void i8xx_fbc_enable(struct intel_crtc *crtc)
 	fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
 	fbc_ctl |= obj->fence_reg;
 	I915_WRITE(FBC_CONTROL, fbc_ctl);
-
-	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
-		      cfb_pitch, crtc->base.y, plane_name(crtc->plane));
 }
 
-static bool i8xx_fbc_enabled(struct drm_i915_private *dev_priv)
+static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv)
 {
 	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
 }
 
-static void g4x_fbc_enable(struct intel_crtc *crtc)
+static void g4x_fbc_activate(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	struct drm_framebuffer *fb = crtc->base.primary->fb;
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	u32 dpfc_ctl;
 
-	dev_priv->fbc.enabled = true;
+	dev_priv->fbc.active = true;
 
 	dpfc_ctl = DPFC_CTL_PLANE(crtc->plane) | DPFC_SR_EN;
 	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
@@ -157,38 +202,35 @@ static void g4x_fbc_enable(struct intel_crtc *crtc)
 
 	/* enable it... */
 	I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
-
-	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(crtc->plane));
 }
 
-static void g4x_fbc_disable(struct drm_i915_private *dev_priv)
+static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv)
 {
 	u32 dpfc_ctl;
 
-	dev_priv->fbc.enabled = false;
+	dev_priv->fbc.active = false;
 
 	/* Disable compression */
 	dpfc_ctl = I915_READ(DPFC_CONTROL);
 	if (dpfc_ctl & DPFC_CTL_EN) {
 		dpfc_ctl &= ~DPFC_CTL_EN;
 		I915_WRITE(DPFC_CONTROL, dpfc_ctl);
-
-		DRM_DEBUG_KMS("disabled FBC\n");
 	}
 }
 
-static bool g4x_fbc_enabled(struct drm_i915_private *dev_priv)
+static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv)
 {
 	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
 }
 
-static void intel_fbc_nuke(struct drm_i915_private *dev_priv)
+/* This function forces a CFB recompression through the nuke operation. */
+static void intel_fbc_recompress(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(MSG_FBC_REND_STATE, FBC_REND_NUKE);
 	POSTING_READ(MSG_FBC_REND_STATE);
 }
 
-static void ilk_fbc_enable(struct intel_crtc *crtc)
+static void ilk_fbc_activate(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	struct drm_framebuffer *fb = crtc->base.primary->fb;
@@ -197,7 +239,7 @@ static void ilk_fbc_enable(struct intel_crtc *crtc)
 	int threshold = dev_priv->fbc.threshold;
 	unsigned int y_offset;
 
-	dev_priv->fbc.enabled = true;
+	dev_priv->fbc.active = true;
 
 	dpfc_ctl = DPFC_CTL_PLANE(crtc->plane);
 	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
@@ -231,33 +273,29 @@ static void ilk_fbc_enable(struct intel_crtc *crtc)
 		I915_WRITE(DPFC_CPU_FENCE_OFFSET, y_offset);
 	}
 
-	intel_fbc_nuke(dev_priv);
-
-	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(crtc->plane));
+	intel_fbc_recompress(dev_priv);
 }
 
-static void ilk_fbc_disable(struct drm_i915_private *dev_priv)
+static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv)
 {
 	u32 dpfc_ctl;
 
-	dev_priv->fbc.enabled = false;
+	dev_priv->fbc.active = false;
 
 	/* Disable compression */
 	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
 	if (dpfc_ctl & DPFC_CTL_EN) {
 		dpfc_ctl &= ~DPFC_CTL_EN;
 		I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
-
-		DRM_DEBUG_KMS("disabled FBC\n");
 	}
 }
 
-static bool ilk_fbc_enabled(struct drm_i915_private *dev_priv)
+static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv)
 {
 	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
 }
 
-static void gen7_fbc_enable(struct intel_crtc *crtc)
+static void gen7_fbc_activate(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	struct drm_framebuffer *fb = crtc->base.primary->fb;
@@ -265,7 +303,7 @@ static void gen7_fbc_enable(struct intel_crtc *crtc)
 	u32 dpfc_ctl;
 	int threshold = dev_priv->fbc.threshold;
 
-	dev_priv->fbc.enabled = true;
+	dev_priv->fbc.active = true;
 
 	dpfc_ctl = 0;
 	if (IS_IVYBRIDGE(dev_priv))
@@ -310,155 +348,120 @@ static void gen7_fbc_enable(struct intel_crtc *crtc)
 		   SNB_CPU_FENCE_ENABLE | obj->fence_reg);
 	I915_WRITE(DPFC_CPU_FENCE_OFFSET, get_crtc_fence_y_offset(crtc));
 
-	intel_fbc_nuke(dev_priv);
-
-	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(crtc->plane));
+	intel_fbc_recompress(dev_priv);
 }
 
 /**
- * intel_fbc_enabled - Is FBC enabled?
+ * intel_fbc_is_active - Is FBC active?
  * @dev_priv: i915 device instance
  *
  * This function is used to verify the current state of FBC.
  * FIXME: This should be tracked in the plane config eventually
  *        instead of queried at runtime for most callers.
  */
-bool intel_fbc_enabled(struct drm_i915_private *dev_priv)
+bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
 {
-	return dev_priv->fbc.enabled;
+	return dev_priv->fbc.active;
 }
 
-static void intel_fbc_enable(struct intel_crtc *crtc,
-			     const struct drm_framebuffer *fb)
+static void intel_fbc_activate(const struct drm_framebuffer *fb)
 {
-	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+	struct drm_i915_private *dev_priv = fb->dev->dev_private;
+	struct intel_crtc *crtc = dev_priv->fbc.crtc;
 
-	dev_priv->fbc.enable_fbc(crtc);
+	dev_priv->fbc.activate(crtc);
 
-	dev_priv->fbc.crtc = crtc;
 	dev_priv->fbc.fb_id = fb->base.id;
 	dev_priv->fbc.y = crtc->base.y;
 }
 
 static void intel_fbc_work_fn(struct work_struct *__work)
 {
-	struct intel_fbc_work *work =
-		container_of(to_delayed_work(__work),
-			     struct intel_fbc_work, work);
-	struct drm_i915_private *dev_priv = work->crtc->base.dev->dev_private;
-	struct drm_framebuffer *crtc_fb = work->crtc->base.primary->fb;
+	struct drm_i915_private *dev_priv =
+		container_of(__work, struct drm_i915_private, fbc.work.work);
+	struct intel_fbc_work *work = &dev_priv->fbc.work;
+	struct intel_crtc *crtc = dev_priv->fbc.crtc;
+	int delay_ms = 50;
+
+retry:
+	/* Delay the actual enabling to let pageflipping cease and the
+	 * display to settle before starting the compression. Note that
+	 * this delay also serves a second purpose: it allows for a
+	 * vblank to pass after disabling the FBC before we attempt
+	 * to modify the control registers.
+	 *
+	 * A more complicated solution would involve tracking vblanks
+	 * following the termination of the page-flipping sequence
+	 * and indeed performing the enable as a co-routine and not
+	 * waiting synchronously upon the vblank.
+	 *
+	 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
+	 */
+	wait_remaining_ms_from_jiffies(work->enable_jiffies, delay_ms);
 
 	mutex_lock(&dev_priv->fbc.lock);
-	if (work == dev_priv->fbc.fbc_work) {
-		/* Double check that we haven't switched fb without cancelling
-		 * the prior work.
-		 */
-		if (crtc_fb == work->fb)
-			intel_fbc_enable(work->crtc, work->fb);
 
-		dev_priv->fbc.fbc_work = NULL;
+	/* Were we cancelled? */
+	if (!work->scheduled)
+		goto out;
+
+	/* Were we delayed again while this function was sleeping? */
+	if (time_after(work->enable_jiffies + msecs_to_jiffies(delay_ms),
+		       jiffies)) {
+		mutex_unlock(&dev_priv->fbc.lock);
+		goto retry;
 	}
-	mutex_unlock(&dev_priv->fbc.lock);
 
-	kfree(work);
+	if (crtc->base.primary->fb == work->fb)
+		intel_fbc_activate(work->fb);
+
+	work->scheduled = false;
+
+out:
+	mutex_unlock(&dev_priv->fbc.lock);
 }
 
 static void intel_fbc_cancel_work(struct drm_i915_private *dev_priv)
 {
 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
-
-	if (dev_priv->fbc.fbc_work == NULL)
-		return;
-
-	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
-
-	/* Synchronisation is provided by struct_mutex and checking of
-	 * dev_priv->fbc.fbc_work, so we can perform the cancellation
-	 * entirely asynchronously.
-	 */
-	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
-		/* tasklet was killed before being run, clean up */
-		kfree(dev_priv->fbc.fbc_work);
-
-	/* Mark the work as no longer wanted so that if it does
-	 * wake-up (because the work was already running and waiting
-	 * for our mutex), it will discover that is no longer
-	 * necessary to run.
-	 */
-	dev_priv->fbc.fbc_work = NULL;
+	dev_priv->fbc.work.scheduled = false;
 }
 
-static void intel_fbc_schedule_enable(struct intel_crtc *crtc)
+static void intel_fbc_schedule_activation(struct intel_crtc *crtc)
 {
-	struct intel_fbc_work *work;
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+	struct intel_fbc_work *work = &dev_priv->fbc.work;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
 
-	intel_fbc_cancel_work(dev_priv);
-
-	work = kzalloc(sizeof(*work), GFP_KERNEL);
-	if (work == NULL) {
-		DRM_ERROR("Failed to allocate FBC work structure\n");
-		intel_fbc_enable(crtc, crtc->base.primary->fb);
-		return;
-	}
-
-	work->crtc = crtc;
+	/* It is useless to call intel_fbc_cancel_work() in this function since
+	 * we're not releasing fbc.lock, so it won't have an opportunity to grab
+	 * it to discover that it was cancelled. So we just update the expected
+	 * jiffy count. */
 	work->fb = crtc->base.primary->fb;
-	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
-
-	dev_priv->fbc.fbc_work = work;
+	work->scheduled = true;
+	work->enable_jiffies = jiffies;
 
-	/* Delay the actual enabling to let pageflipping cease and the
-	 * display to settle before starting the compression. Note that
-	 * this delay also serves a second purpose: it allows for a
-	 * vblank to pass after disabling the FBC before we attempt
-	 * to modify the control registers.
-	 *
-	 * A more complicated solution would involve tracking vblanks
-	 * following the termination of the page-flipping sequence
-	 * and indeed performing the enable as a co-routine and not
-	 * waiting synchronously upon the vblank.
-	 *
-	 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
-	 */
-	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
+	schedule_work(&work->work);
 }
 
-static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
+static void __intel_fbc_deactivate(struct drm_i915_private *dev_priv)
 {
 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
 
 	intel_fbc_cancel_work(dev_priv);
 
-	dev_priv->fbc.disable_fbc(dev_priv);
-	dev_priv->fbc.crtc = NULL;
-}
-
-/**
- * intel_fbc_disable - disable FBC
- * @dev_priv: i915 device instance
- *
- * This function disables FBC.
- */
-void intel_fbc_disable(struct drm_i915_private *dev_priv)
-{
-	if (!fbc_supported(dev_priv))
-		return;
-
-	mutex_lock(&dev_priv->fbc.lock);
-	__intel_fbc_disable(dev_priv);
-	mutex_unlock(&dev_priv->fbc.lock);
+	if (dev_priv->fbc.active)
+		dev_priv->fbc.deactivate(dev_priv);
 }
 
 /*
- * intel_fbc_disable_crtc - disable FBC if it's associated with crtc
+ * intel_fbc_deactivate - deactivate FBC if it's associated with crtc
  * @crtc: the CRTC
  *
- * This function disables FBC if it's associated with the provided CRTC.
+ * This function deactivates FBC if it's associated with the provided CRTC.
  */
-void intel_fbc_disable_crtc(struct intel_crtc *crtc)
+void intel_fbc_deactivate(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 
@@ -467,85 +470,42 @@ void intel_fbc_disable_crtc(struct intel_crtc *crtc)
 
 	mutex_lock(&dev_priv->fbc.lock);
 	if (dev_priv->fbc.crtc == crtc)
-		__intel_fbc_disable(dev_priv);
+		__intel_fbc_deactivate(dev_priv);
 	mutex_unlock(&dev_priv->fbc.lock);
 }
 
-const char *intel_no_fbc_reason_str(enum no_fbc_reason reason)
-{
-	switch (reason) {
-	case FBC_OK:
-		return "FBC enabled but currently disabled in hardware";
-	case FBC_UNSUPPORTED:
-		return "unsupported by this chipset";
-	case FBC_NO_OUTPUT:
-		return "no output";
-	case FBC_STOLEN_TOO_SMALL:
-		return "not enough stolen memory";
-	case FBC_UNSUPPORTED_MODE:
-		return "mode incompatible with compression";
-	case FBC_MODE_TOO_LARGE:
-		return "mode too large for compression";
-	case FBC_BAD_PLANE:
-		return "FBC unsupported on plane";
-	case FBC_NOT_TILED:
-		return "framebuffer not tiled or fenced";
-	case FBC_MULTIPLE_PIPES:
-		return "more than one pipe active";
-	case FBC_MODULE_PARAM:
-		return "disabled per module param";
-	case FBC_CHIP_DEFAULT:
-		return "disabled per chip default";
-	case FBC_ROTATION:
-		return "rotation unsupported";
-	case FBC_IN_DBG_MASTER:
-		return "Kernel debugger is active";
-	case FBC_BAD_STRIDE:
-		return "framebuffer stride not supported";
-	case FBC_PIXEL_RATE:
-		return "pixel rate is too big";
-	case FBC_PIXEL_FORMAT:
-		return "pixel format is invalid";
-	default:
-		MISSING_CASE(reason);
-		return "unknown reason";
-	}
-}
-
 static void set_no_fbc_reason(struct drm_i915_private *dev_priv,
-			      enum no_fbc_reason reason)
+			      const char *reason)
 {
 	if (dev_priv->fbc.no_fbc_reason == reason)
 		return;
 
 	dev_priv->fbc.no_fbc_reason = reason;
-	DRM_DEBUG_KMS("Disabling FBC: %s\n", intel_no_fbc_reason_str(reason));
+	DRM_DEBUG_KMS("Disabling FBC: %s\n", reason);
 }
 
-static struct drm_crtc *intel_fbc_find_crtc(struct drm_i915_private *dev_priv)
+static bool crtc_can_fbc(struct intel_crtc *crtc)
 {
-	struct drm_crtc *crtc = NULL, *tmp_crtc;
-	enum pipe pipe;
-	bool pipe_a_only = false;
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 
-	if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)
-		pipe_a_only = true;
+	if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A)
+		return false;
 
-	for_each_pipe(dev_priv, pipe) {
-		tmp_crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+	if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A)
+		return false;
 
-		if (intel_crtc_active(tmp_crtc) &&
-		    to_intel_plane_state(tmp_crtc->primary->state)->visible)
-			crtc = tmp_crtc;
+	return true;
+}
 
-		if (pipe_a_only)
-			break;
-	}
+static bool crtc_is_valid(struct intel_crtc *crtc)
+{
+	if (!intel_crtc_active(&crtc->base))
+		return false;
 
-	if (!crtc || crtc->primary->fb == NULL)
-		return NULL;
+	if (!to_intel_plane_state(crtc->base.primary->state)->visible)
+		return false;
 
-	return crtc;
+	return true;
 }
 
 static bool multiple_pipes_ok(struct drm_i915_private *dev_priv)
@@ -581,7 +541,8 @@ static int find_compression_threshold(struct drm_i915_private *dev_priv,
 	 * reserved range size, so it always assumes the maximum (8mb) is used.
 	 * If we enable FBC using a CFB on that memory range we'll get FIFO
 	 * underruns, even if that range is not reserved by the BIOS. */
-	if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+	if (IS_BROADWELL(dev_priv) ||
+	    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		end = dev_priv->gtt.stolen_size - 8 * 1024 * 1024;
 	else
 		end = dev_priv->gtt.stolen_usable_size;
@@ -617,11 +578,17 @@ again:
 	}
 }
 
-static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, int size,
-			       int fb_cpp)
+static int intel_fbc_alloc_cfb(struct intel_crtc *crtc)
 {
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+	struct drm_framebuffer *fb = crtc->base.primary->state->fb;
 	struct drm_mm_node *uninitialized_var(compressed_llb);
-	int ret;
+	int size, fb_cpp, ret;
+
+	WARN_ON(drm_mm_node_allocated(&dev_priv->fbc.compressed_fb));
+
+	size = intel_fbc_calculate_cfb_size(crtc, fb);
+	fb_cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
 	ret = find_compression_threshold(dev_priv, &dev_priv->fbc.compressed_fb,
 					 size, fb_cpp);
@@ -656,8 +623,6 @@ static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, int size,
 			   dev_priv->mm.stolen_base + compressed_llb->start);
 	}
 
-	dev_priv->fbc.uncompressed_size = size;
-
 	DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n",
 		      dev_priv->fbc.compressed_fb.size,
 		      dev_priv->fbc.threshold);
@@ -674,18 +639,15 @@ err_llb:
 
 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
 {
-	if (dev_priv->fbc.uncompressed_size == 0)
-		return;
-
-	i915_gem_stolen_remove_node(dev_priv, &dev_priv->fbc.compressed_fb);
+	if (drm_mm_node_allocated(&dev_priv->fbc.compressed_fb))
+		i915_gem_stolen_remove_node(dev_priv,
+					    &dev_priv->fbc.compressed_fb);
 
 	if (dev_priv->fbc.compressed_llb) {
 		i915_gem_stolen_remove_node(dev_priv,
 					    dev_priv->fbc.compressed_llb);
 		kfree(dev_priv->fbc.compressed_llb);
 	}
-
-	dev_priv->fbc.uncompressed_size = 0;
 }
 
 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
@@ -698,63 +660,6 @@ void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->fbc.lock);
 }
 
-/*
- * For SKL+, the plane source size used by the hardware is based on the value we
- * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value
- * we wrote to PIPESRC.
- */
-static void intel_fbc_get_plane_source_size(struct intel_crtc *crtc,
-					    int *width, int *height)
-{
-	struct intel_plane_state *plane_state =
-			to_intel_plane_state(crtc->base.primary->state);
-	int w, h;
-
-	if (intel_rotation_90_or_270(plane_state->base.rotation)) {
-		w = drm_rect_height(&plane_state->src) >> 16;
-		h = drm_rect_width(&plane_state->src) >> 16;
-	} else {
-		w = drm_rect_width(&plane_state->src) >> 16;
-		h = drm_rect_height(&plane_state->src) >> 16;
-	}
-
-	if (width)
-		*width = w;
-	if (height)
-		*height = h;
-}
-
-static int intel_fbc_calculate_cfb_size(struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
-	struct drm_framebuffer *fb = crtc->base.primary->fb;
-	int lines;
-
-	intel_fbc_get_plane_source_size(crtc, NULL, &lines);
-	if (INTEL_INFO(dev_priv)->gen >= 7)
-		lines = min(lines, 2048);
-
-	return lines * fb->pitches[0];
-}
-
-static int intel_fbc_setup_cfb(struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
-	struct drm_framebuffer *fb = crtc->base.primary->fb;
-	int size, cpp;
-
-	size = intel_fbc_calculate_cfb_size(crtc);
-	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-
-	if (size <= dev_priv->fbc.uncompressed_size)
-		return 0;
-
-	/* Release any current block */
-	__intel_fbc_cleanup_cfb(dev_priv);
-
-	return intel_fbc_alloc_cfb(dev_priv, size, cpp);
-}
-
 static bool stride_is_valid(struct drm_i915_private *dev_priv,
 			    unsigned int stride)
 {
@@ -829,87 +734,46 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
 }
 
 /**
- * __intel_fbc_update - enable/disable FBC as needed, unlocked
- * @dev_priv: i915 device instance
- *
- * Set up the framebuffer compression hardware at mode set time.  We
- * enable it if possible:
- *   - plane A only (on pre-965)
- *   - no pixel mulitply/line duplication
- *   - no alpha buffer discard
- *   - no dual wide
- *   - framebuffer <= max_hdisplay in width, max_vdisplay in height
- *
- * We can't assume that any compression will take place (worst case),
- * so the compressed buffer has to be the same size as the uncompressed
- * one.  It also must reside (along with the line length buffer) in
- * stolen memory.
+ * __intel_fbc_update - activate/deactivate FBC as needed, unlocked
+ * @crtc: the CRTC that triggered the update
  *
- * We need to enable/disable FBC on a global basis.
+ * This function completely reevaluates the status of FBC, then activates,
+ * deactivates or maintains it on the same state.
  */
-static void __intel_fbc_update(struct drm_i915_private *dev_priv)
+static void __intel_fbc_update(struct intel_crtc *crtc)
 {
-	struct drm_crtc *crtc = NULL;
-	struct intel_crtc *intel_crtc;
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 	struct drm_framebuffer *fb;
 	struct drm_i915_gem_object *obj;
 	const struct drm_display_mode *adjusted_mode;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
 
-	/* disable framebuffer compression in vGPU */
-	if (intel_vgpu_active(dev_priv->dev))
-		i915.enable_fbc = 0;
-
-	if (i915.enable_fbc < 0) {
-		set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT);
+	if (!multiple_pipes_ok(dev_priv)) {
+		set_no_fbc_reason(dev_priv, "more than one pipe active");
 		goto out_disable;
 	}
 
-	if (!i915.enable_fbc) {
-		set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM);
-		goto out_disable;
-	}
+	if (!dev_priv->fbc.enabled || dev_priv->fbc.crtc != crtc)
+		return;
 
-	/*
-	 * If FBC is already on, we just have to verify that we can
-	 * keep it that way...
-	 * Need to disable if:
-	 *   - more than one pipe is active
-	 *   - changing FBC params (stride, fence, mode)
-	 *   - new fb is too large to fit in compressed buffer
-	 *   - going to an unsupported config (interlace, pixel multiply, etc.)
-	 */
-	crtc = intel_fbc_find_crtc(dev_priv);
-	if (!crtc) {
-		set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT);
+	if (!crtc_is_valid(crtc)) {
+		set_no_fbc_reason(dev_priv, "no output");
 		goto out_disable;
 	}
 
-	if (!multiple_pipes_ok(dev_priv)) {
-		set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES);
-		goto out_disable;
-	}
-
-	intel_crtc = to_intel_crtc(crtc);
-	fb = crtc->primary->fb;
+	fb = crtc->base.primary->fb;
 	obj = intel_fb_obj(fb);
-	adjusted_mode = &intel_crtc->config->base.adjusted_mode;
+	adjusted_mode = &crtc->config->base.adjusted_mode;
 
 	if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ||
 	    (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
-		set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE);
+		set_no_fbc_reason(dev_priv, "incompatible mode");
 		goto out_disable;
 	}
 
-	if (!intel_fbc_hw_tracking_covers_screen(intel_crtc)) {
-		set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE);
-		goto out_disable;
-	}
-
-	if ((INTEL_INFO(dev_priv)->gen < 4 || HAS_DDI(dev_priv)) &&
-	    intel_crtc->plane != PLANE_A) {
-		set_no_fbc_reason(dev_priv, FBC_BAD_PLANE);
+	if (!intel_fbc_hw_tracking_covers_screen(crtc)) {
+		set_no_fbc_reason(dev_priv, "mode too large for compression");
 		goto out_disable;
 	}
 
@@ -918,41 +782,46 @@ static void __intel_fbc_update(struct drm_i915_private *dev_priv)
 	 */
 	if (obj->tiling_mode != I915_TILING_X ||
 	    obj->fence_reg == I915_FENCE_REG_NONE) {
-		set_no_fbc_reason(dev_priv, FBC_NOT_TILED);
+		set_no_fbc_reason(dev_priv, "framebuffer not tiled or fenced");
 		goto out_disable;
 	}
 	if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) &&
-	    crtc->primary->state->rotation != BIT(DRM_ROTATE_0)) {
-		set_no_fbc_reason(dev_priv, FBC_ROTATION);
+	    crtc->base.primary->state->rotation != BIT(DRM_ROTATE_0)) {
+		set_no_fbc_reason(dev_priv, "rotation unsupported");
 		goto out_disable;
 	}
 
 	if (!stride_is_valid(dev_priv, fb->pitches[0])) {
-		set_no_fbc_reason(dev_priv, FBC_BAD_STRIDE);
+		set_no_fbc_reason(dev_priv, "framebuffer stride not supported");
 		goto out_disable;
 	}
 
 	if (!pixel_format_is_valid(fb)) {
-		set_no_fbc_reason(dev_priv, FBC_PIXEL_FORMAT);
-		goto out_disable;
-	}
-
-	/* If the kernel debugger is active, always disable compression */
-	if (in_dbg_master()) {
-		set_no_fbc_reason(dev_priv, FBC_IN_DBG_MASTER);
+		set_no_fbc_reason(dev_priv, "pixel format is invalid");
 		goto out_disable;
 	}
 
 	/* WaFbcExceedCdClockThreshold:hsw,bdw */
 	if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&
-	    ilk_pipe_pixel_rate(intel_crtc->config) >=
+	    ilk_pipe_pixel_rate(crtc->config) >=
 	    dev_priv->cdclk_freq * 95 / 100) {
-		set_no_fbc_reason(dev_priv, FBC_PIXEL_RATE);
+		set_no_fbc_reason(dev_priv, "pixel rate is too big");
 		goto out_disable;
 	}
 
-	if (intel_fbc_setup_cfb(intel_crtc)) {
-		set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL);
+	/* It is possible for the required CFB size change without a
+	 * crtc->disable + crtc->enable since it is possible to change the
+	 * stride without triggering a full modeset. Since we try to
+	 * over-allocate the CFB, there's a chance we may keep FBC enabled even
+	 * if this happens, but if we exceed the current CFB size we'll have to
+	 * disable FBC. Notice that it would be possible to disable FBC, wait
+	 * for a frame, free the stolen node, then try to reenable FBC in case
+	 * we didn't get any invalidate/deactivate calls, but this would require
+	 * a lot of tracking just for a specific case. If we conclude it's an
+	 * important case, we can implement it later. */
+	if (intel_fbc_calculate_cfb_size(crtc, fb) >
+	    dev_priv->fbc.compressed_fb.size * dev_priv->fbc.threshold) {
+		set_no_fbc_reason(dev_priv, "CFB requirements changed");
 		goto out_disable;
 	}
 
@@ -961,12 +830,13 @@ static void __intel_fbc_update(struct drm_i915_private *dev_priv)
 	 * cannot be unpinned (and have its GTT offset and fence revoked)
 	 * without first being decoupled from the scanout and FBC disabled.
 	 */
-	if (dev_priv->fbc.crtc == intel_crtc &&
+	if (dev_priv->fbc.crtc == crtc &&
 	    dev_priv->fbc.fb_id == fb->base.id &&
-	    dev_priv->fbc.y == crtc->y)
+	    dev_priv->fbc.y == crtc->base.y &&
+	    dev_priv->fbc.active)
 		return;
 
-	if (intel_fbc_enabled(dev_priv)) {
+	if (intel_fbc_is_active(dev_priv)) {
 		/* We update FBC along two paths, after changing fb/crtc
 		 * configuration (modeswitching) and after page-flipping
 		 * finishes. For the latter, we know that not only did
@@ -990,36 +860,37 @@ static void __intel_fbc_update(struct drm_i915_private *dev_priv)
 		 * disabling paths we do need to wait for a vblank at
 		 * some point. And we wait before enabling FBC anyway.
 		 */
-		DRM_DEBUG_KMS("disabling active FBC for update\n");
-		__intel_fbc_disable(dev_priv);
+		DRM_DEBUG_KMS("deactivating FBC for update\n");
+		__intel_fbc_deactivate(dev_priv);
 	}
 
-	intel_fbc_schedule_enable(intel_crtc);
-	dev_priv->fbc.no_fbc_reason = FBC_OK;
+	intel_fbc_schedule_activation(crtc);
+	dev_priv->fbc.no_fbc_reason = "FBC enabled (not necessarily active)";
 	return;
 
 out_disable:
 	/* Multiple disables should be harmless */
-	if (intel_fbc_enabled(dev_priv)) {
-		DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
-		__intel_fbc_disable(dev_priv);
+	if (intel_fbc_is_active(dev_priv)) {
+		DRM_DEBUG_KMS("unsupported config, deactivating FBC\n");
+		__intel_fbc_deactivate(dev_priv);
 	}
-	__intel_fbc_cleanup_cfb(dev_priv);
 }
 
 /*
- * intel_fbc_update - enable/disable FBC as needed
- * @dev_priv: i915 device instance
+ * intel_fbc_update - activate/deactivate FBC as needed
+ * @crtc: the CRTC that triggered the update
  *
- * This function reevaluates the overall state and enables or disables FBC.
+ * This function reevaluates the overall state and activates or deactivates FBC.
  */
-void intel_fbc_update(struct drm_i915_private *dev_priv)
+void intel_fbc_update(struct intel_crtc *crtc)
 {
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
 	if (!fbc_supported(dev_priv))
 		return;
 
 	mutex_lock(&dev_priv->fbc.lock);
-	__intel_fbc_update(dev_priv);
+	__intel_fbc_update(crtc);
 	mutex_unlock(&dev_priv->fbc.lock);
 }
 
@@ -1039,16 +910,13 @@ void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
 
 	if (dev_priv->fbc.enabled)
 		fbc_bits = INTEL_FRONTBUFFER_PRIMARY(dev_priv->fbc.crtc->pipe);
-	else if (dev_priv->fbc.fbc_work)
-		fbc_bits = INTEL_FRONTBUFFER_PRIMARY(
-					dev_priv->fbc.fbc_work->crtc->pipe);
 	else
 		fbc_bits = dev_priv->fbc.possible_framebuffer_bits;
 
 	dev_priv->fbc.busy_bits |= (fbc_bits & frontbuffer_bits);
 
 	if (dev_priv->fbc.busy_bits)
-		__intel_fbc_disable(dev_priv);
+		__intel_fbc_deactivate(dev_priv);
 
 	mutex_unlock(&dev_priv->fbc.lock);
 }
@@ -1066,11 +934,136 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
 
 	dev_priv->fbc.busy_bits &= ~frontbuffer_bits;
 
-	if (!dev_priv->fbc.busy_bits) {
+	if (!dev_priv->fbc.busy_bits && dev_priv->fbc.enabled) {
+		if (origin != ORIGIN_FLIP && dev_priv->fbc.active) {
+			intel_fbc_recompress(dev_priv);
+		} else {
+			__intel_fbc_deactivate(dev_priv);
+			__intel_fbc_update(dev_priv->fbc.crtc);
+		}
+	}
+
+	mutex_unlock(&dev_priv->fbc.lock);
+}
+
+/**
+ * intel_fbc_enable: tries to enable FBC on the CRTC
+ * @crtc: the CRTC
+ *
+ * This function checks if it's possible to enable FBC on the following CRTC,
+ * then enables it. Notice that it doesn't activate FBC.
+ */
+void intel_fbc_enable(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
+	if (!fbc_supported(dev_priv))
+		return;
+
+	mutex_lock(&dev_priv->fbc.lock);
+
+	if (dev_priv->fbc.enabled) {
+		WARN_ON(dev_priv->fbc.crtc == crtc);
+		goto out;
+	}
+
+	WARN_ON(dev_priv->fbc.active);
+	WARN_ON(dev_priv->fbc.crtc != NULL);
+
+	if (intel_vgpu_active(dev_priv->dev)) {
+		set_no_fbc_reason(dev_priv, "VGPU is active");
+		goto out;
+	}
+
+	if (i915.enable_fbc < 0) {
+		set_no_fbc_reason(dev_priv, "disabled per chip default");
+		goto out;
+	}
+
+	if (!i915.enable_fbc) {
+		set_no_fbc_reason(dev_priv, "disabled per module param");
+		goto out;
+	}
+
+	if (!crtc_can_fbc(crtc)) {
+		set_no_fbc_reason(dev_priv, "no enabled pipes can have FBC");
+		goto out;
+	}
+
+	if (intel_fbc_alloc_cfb(crtc)) {
+		set_no_fbc_reason(dev_priv, "not enough stolen memory");
+		goto out;
+	}
+
+	DRM_DEBUG_KMS("Enabling FBC on pipe %c\n", pipe_name(crtc->pipe));
+	dev_priv->fbc.no_fbc_reason = "FBC enabled but not active yet\n";
+
+	dev_priv->fbc.enabled = true;
+	dev_priv->fbc.crtc = crtc;
+out:
+	mutex_unlock(&dev_priv->fbc.lock);
+}
+
+/**
+ * __intel_fbc_disable - disable FBC
+ * @dev_priv: i915 device instance
+ *
+ * This is the low level function that actually disables FBC. Callers should
+ * grab the FBC lock.
+ */
+static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
+{
+	struct intel_crtc *crtc = dev_priv->fbc.crtc;
+
+	WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock));
+	WARN_ON(!dev_priv->fbc.enabled);
+	WARN_ON(dev_priv->fbc.active);
+	assert_pipe_disabled(dev_priv, crtc->pipe);
+
+	DRM_DEBUG_KMS("Disabling FBC on pipe %c\n", pipe_name(crtc->pipe));
+
+	__intel_fbc_cleanup_cfb(dev_priv);
+
+	dev_priv->fbc.enabled = false;
+	dev_priv->fbc.crtc = NULL;
+}
+
+/**
+ * intel_fbc_disable_crtc - disable FBC if it's associated with crtc
+ * @crtc: the CRTC
+ *
+ * This function disables FBC if it's associated with the provided CRTC.
+ */
+void intel_fbc_disable_crtc(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
+	if (!fbc_supported(dev_priv))
+		return;
+
+	mutex_lock(&dev_priv->fbc.lock);
+	if (dev_priv->fbc.crtc == crtc) {
+		WARN_ON(!dev_priv->fbc.enabled);
+		WARN_ON(dev_priv->fbc.active);
 		__intel_fbc_disable(dev_priv);
-		__intel_fbc_update(dev_priv);
 	}
+	mutex_unlock(&dev_priv->fbc.lock);
+}
 
+/**
+ * intel_fbc_disable - globally disable FBC
+ * @dev_priv: i915 device instance
+ *
+ * This function disables FBC regardless of which CRTC is associated with it.
+ */
+void intel_fbc_disable(struct drm_i915_private *dev_priv)
+{
+	if (!fbc_supported(dev_priv))
+		return;
+
+	mutex_lock(&dev_priv->fbc.lock);
+	if (dev_priv->fbc.enabled)
+		__intel_fbc_disable(dev_priv);
 	mutex_unlock(&dev_priv->fbc.lock);
 }
 
@@ -1084,11 +1077,14 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 {
 	enum pipe pipe;
 
+	INIT_WORK(&dev_priv->fbc.work.work, intel_fbc_work_fn);
 	mutex_init(&dev_priv->fbc.lock);
+	dev_priv->fbc.enabled = false;
+	dev_priv->fbc.active = false;
+	dev_priv->fbc.work.scheduled = false;
 
 	if (!HAS_FBC(dev_priv)) {
-		dev_priv->fbc.enabled = false;
-		dev_priv->fbc.no_fbc_reason = FBC_UNSUPPORTED;
+		dev_priv->fbc.no_fbc_reason = "unsupported by this chipset";
 		return;
 	}
 
@@ -1096,30 +1092,34 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 		dev_priv->fbc.possible_framebuffer_bits |=
 				INTEL_FRONTBUFFER_PRIMARY(pipe);
 
-		if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)
+		if (fbc_on_pipe_a_only(dev_priv))
 			break;
 	}
 
 	if (INTEL_INFO(dev_priv)->gen >= 7) {
-		dev_priv->fbc.fbc_enabled = ilk_fbc_enabled;
-		dev_priv->fbc.enable_fbc = gen7_fbc_enable;
-		dev_priv->fbc.disable_fbc = ilk_fbc_disable;
+		dev_priv->fbc.is_active = ilk_fbc_is_active;
+		dev_priv->fbc.activate = gen7_fbc_activate;
+		dev_priv->fbc.deactivate = ilk_fbc_deactivate;
 	} else if (INTEL_INFO(dev_priv)->gen >= 5) {
-		dev_priv->fbc.fbc_enabled = ilk_fbc_enabled;
-		dev_priv->fbc.enable_fbc = ilk_fbc_enable;
-		dev_priv->fbc.disable_fbc = ilk_fbc_disable;
+		dev_priv->fbc.is_active = ilk_fbc_is_active;
+		dev_priv->fbc.activate = ilk_fbc_activate;
+		dev_priv->fbc.deactivate = ilk_fbc_deactivate;
 	} else if (IS_GM45(dev_priv)) {
-		dev_priv->fbc.fbc_enabled = g4x_fbc_enabled;
-		dev_priv->fbc.enable_fbc = g4x_fbc_enable;
-		dev_priv->fbc.disable_fbc = g4x_fbc_disable;
+		dev_priv->fbc.is_active = g4x_fbc_is_active;
+		dev_priv->fbc.activate = g4x_fbc_activate;
+		dev_priv->fbc.deactivate = g4x_fbc_deactivate;
 	} else {
-		dev_priv->fbc.fbc_enabled = i8xx_fbc_enabled;
-		dev_priv->fbc.enable_fbc = i8xx_fbc_enable;
-		dev_priv->fbc.disable_fbc = i8xx_fbc_disable;
+		dev_priv->fbc.is_active = i8xx_fbc_is_active;
+		dev_priv->fbc.activate = i8xx_fbc_activate;
+		dev_priv->fbc.deactivate = i8xx_fbc_deactivate;
 
 		/* This value was pulled out of someone's hat */
 		I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
 	}
 
-	dev_priv->fbc.enabled = dev_priv->fbc.fbc_enabled(dev_priv);
+	/* We still don't have any sort of hardware state readout for FBC, so
+	 * deactivate it in case the BIOS activated it to make sure software
+	 * matches the hardware state. */
+	if (dev_priv->fbc.is_active(dev_priv))
+		dev_priv->fbc.deactivate(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 4fd5fdfef6bd..bea75cafc623 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -119,7 +119,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 {
 	struct intel_fbdev *ifbdev =
 		container_of(helper, struct intel_fbdev, helper);
-	struct drm_framebuffer *fb;
+	struct drm_framebuffer *fb = NULL;
 	struct drm_device *dev = helper->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_mode_fb_cmd2 mode_cmd = {};
@@ -138,6 +138,8 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
 							  sizes->surface_depth);
 
+	mutex_lock(&dev->struct_mutex);
+
 	size = mode_cmd.pitches[0] * mode_cmd.height;
 	size = PAGE_ALIGN(size);
 
@@ -156,26 +158,21 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 
 	fb = __intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb)) {
+		drm_gem_object_unreference(&obj->base);
 		ret = PTR_ERR(fb);
-		goto out_unref;
+		goto out;
 	}
 
-	/* Flush everything out, we'll be doing GTT only from now on */
-	ret = intel_pin_and_fence_fb_obj(NULL, fb, NULL, NULL, NULL);
-	if (ret) {
-		DRM_ERROR("failed to pin obj: %d\n", ret);
-		goto out_fb;
-	}
+	mutex_unlock(&dev->struct_mutex);
 
 	ifbdev->fb = to_intel_framebuffer(fb);
 
 	return 0;
 
-out_fb:
-	drm_framebuffer_remove(fb);
-out_unref:
-	drm_gem_object_unreference(&obj->base);
 out:
+	mutex_unlock(&dev->struct_mutex);
+	if (!IS_ERR_OR_NULL(fb))
+		drm_framebuffer_unreference(fb);
 	return ret;
 }
 
@@ -193,8 +190,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	int size, ret;
 	bool prealloc = false;
 
-	mutex_lock(&dev->struct_mutex);
-
 	if (intel_fb &&
 	    (sizes->fb_width > intel_fb->base.width ||
 	     sizes->fb_height > intel_fb->base.height)) {
@@ -209,7 +204,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 		DRM_DEBUG_KMS("no BIOS fb, allocating a new one\n");
 		ret = intelfb_alloc(helper, sizes);
 		if (ret)
-			goto out_unlock;
+			return ret;
 		intel_fb = ifbdev->fb;
 	} else {
 		DRM_DEBUG_KMS("re-using BIOS fb\n");
@@ -221,8 +216,19 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	obj = intel_fb->obj;
 	size = obj->base.size;
 
+	mutex_lock(&dev->struct_mutex);
+
+	/* Pin the GGTT vma for our access via info->screen_base.
+	 * This also validates that any existing fb inherited from the
+	 * BIOS is suitable for own access.
+	 */
+	ret = intel_pin_and_fence_fb_obj(NULL, &ifbdev->fb->base, NULL);
+	if (ret)
+		goto out_unlock;
+
 	info = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(info)) {
+		DRM_ERROR("Failed to allocate fb_info\n");
 		ret = PTR_ERR(info);
 		goto out_unpin;
 	}
@@ -249,6 +255,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 		ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
 			   size);
 	if (!info->screen_base) {
+		DRM_ERROR("Failed to remap framebuffer into virtual memory\n");
 		ret = -ENOSPC;
 		goto out_destroy_fbi;
 	}
@@ -281,7 +288,6 @@ out_destroy_fbi:
 	drm_fb_helper_release_fbi(helper);
 out_unpin:
 	i915_gem_object_ggtt_unpin(obj);
-	drm_gem_object_unreference(&obj->base);
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
@@ -520,14 +526,20 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
 static void intel_fbdev_destroy(struct drm_device *dev,
 				struct intel_fbdev *ifbdev)
 {
+	/* We rely on the object-free to release the VMA pinning for
+	 * the info->screen_base mmaping. Leaking the VMA is simpler than
+	 * trying to rectify all the possible error paths leading here.
+	 */
 
 	drm_fb_helper_unregister_fbi(&ifbdev->helper);
 	drm_fb_helper_release_fbi(&ifbdev->helper);
 
 	drm_fb_helper_fini(&ifbdev->helper);
 
-	drm_framebuffer_unregister_private(&ifbdev->fb->base);
-	drm_framebuffer_remove(&ifbdev->fb->base);
+	if (ifbdev->fb) {
+		drm_framebuffer_unregister_private(&ifbdev->fb->base);
+		drm_framebuffer_remove(&ifbdev->fb->base);
+	}
 }
 
 /*
@@ -702,13 +714,20 @@ int intel_fbdev_init(struct drm_device *dev)
 	return 0;
 }
 
-void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
+static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 {
 	struct drm_i915_private *dev_priv = data;
 	struct intel_fbdev *ifbdev = dev_priv->fbdev;
 
 	/* Due to peculiar init order wrt to hpd handling this is separate. */
-	drm_fb_helper_initial_config(&ifbdev->helper, ifbdev->preferred_bpp);
+	if (drm_fb_helper_initial_config(&ifbdev->helper,
+					 ifbdev->preferred_bpp))
+		intel_fbdev_fini(dev_priv->dev);
+}
+
+void intel_fbdev_initial_config_async(struct drm_device *dev)
+{
+	async_schedule(intel_fbdev_initial_config, to_i915(dev));
 }
 
 void intel_fbdev_fini(struct drm_device *dev)
@@ -719,7 +738,8 @@ void intel_fbdev_fini(struct drm_device *dev)
 
 	flush_work(&dev_priv->fbdev_suspend_work);
 
-	async_synchronize_full();
+	if (!current_is_async())
+		async_synchronize_full();
 	intel_fbdev_destroy(dev, dev_priv->fbdev);
 	kfree(dev_priv->fbdev);
 	dev_priv->fbdev = NULL;
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index 54daa66c6970..bda526660e20 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -84,38 +84,21 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev)
 	return true;
 }
 
-/**
- * i9xx_check_fifo_underruns - check for fifo underruns
- * @dev_priv: i915 device instance
- *
- * This function checks for fifo underruns on GMCH platforms. This needs to be
- * done manually on modeset to make sure that we catch all underruns since they
- * do not generate an interrupt by themselves on these platforms.
- */
-void i9xx_check_fifo_underruns(struct drm_i915_private *dev_priv)
+static void i9xx_check_fifo_underruns(struct intel_crtc *crtc)
 {
-	struct intel_crtc *crtc;
-
-	spin_lock_irq(&dev_priv->irq_lock);
-
-	for_each_intel_crtc(dev_priv->dev, crtc) {
-		u32 reg = PIPESTAT(crtc->pipe);
-		u32 pipestat;
-
-		if (crtc->cpu_fifo_underrun_disabled)
-			continue;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	i915_reg_t reg = PIPESTAT(crtc->pipe);
+	u32 pipestat = I915_READ(reg) & 0xffff0000;
 
-		pipestat = I915_READ(reg) & 0xffff0000;
-		if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0)
-			continue;
+	assert_spin_locked(&dev_priv->irq_lock);
 
-		I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
-		POSTING_READ(reg);
+	if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0)
+		return;
 
-		DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe));
-	}
+	I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
+	POSTING_READ(reg);
 
-	spin_unlock_irq(&dev_priv->irq_lock);
+	DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe));
 }
 
 static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
@@ -123,7 +106,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
 					     bool enable, bool old)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 reg = PIPESTAT(pipe);
+	i915_reg_t reg = PIPESTAT(pipe);
 	u32 pipestat = I915_READ(reg) & 0xffff0000;
 
 	assert_spin_locked(&dev_priv->irq_lock);
@@ -145,9 +128,26 @@ static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
 					  DE_PIPEB_FIFO_UNDERRUN;
 
 	if (enable)
-		ironlake_enable_display_irq(dev_priv, bit);
+		ilk_enable_display_irq(dev_priv, bit);
 	else
-		ironlake_disable_display_irq(dev_priv, bit);
+		ilk_disable_display_irq(dev_priv, bit);
+}
+
+static void ivybridge_check_fifo_underruns(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+	uint32_t err_int = I915_READ(GEN7_ERR_INT);
+
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	if ((err_int & ERR_INT_FIFO_UNDERRUN(pipe)) == 0)
+		return;
+
+	I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
+	POSTING_READ(GEN7_ERR_INT);
+
+	DRM_ERROR("fifo underrun on pipe %c\n", pipe_name(pipe));
 }
 
 static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
@@ -161,9 +161,9 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 		if (!ivb_can_enable_err_int(dev))
 			return;
 
-		ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
+		ilk_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
 	} else {
-		ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
+		ilk_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
 
 		if (old &&
 		    I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) {
@@ -178,14 +178,10 @@ static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	assert_spin_locked(&dev_priv->irq_lock);
-
 	if (enable)
-		dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN;
+		bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN);
 	else
-		dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN;
-	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
-	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+		bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN);
 }
 
 static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
@@ -202,6 +198,24 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
 		ibx_disable_display_interrupt(dev_priv, bit);
 }
 
+static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum transcoder pch_transcoder = (enum transcoder) crtc->pipe;
+	uint32_t serr_int = I915_READ(SERR_INT);
+
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	if ((serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) == 0)
+		return;
+
+	I915_WRITE(SERR_INT, SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
+	POSTING_READ(SERR_INT);
+
+	DRM_ERROR("pch fifo underrun on pch transcoder %c\n",
+		  transcoder_name(pch_transcoder));
+}
+
 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 					    enum transcoder pch_transcoder,
 					    bool enable, bool old)
@@ -375,3 +389,56 @@ void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 		DRM_ERROR("PCH transcoder %c FIFO underrun\n",
 			  transcoder_name(pch_transcoder));
 }
+
+/**
+ * intel_check_cpu_fifo_underruns - check for CPU fifo underruns immediately
+ * @dev_priv: i915 device instance
+ *
+ * Check for CPU fifo underruns immediately. Useful on IVB/HSW where the shared
+ * error interrupt may have been disabled, and so CPU fifo underruns won't
+ * necessarily raise an interrupt, and on GMCH platforms where underruns never
+ * raise an interrupt.
+ */
+void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv)
+{
+	struct intel_crtc *crtc;
+
+	spin_lock_irq(&dev_priv->irq_lock);
+
+	for_each_intel_crtc(dev_priv->dev, crtc) {
+		if (crtc->cpu_fifo_underrun_disabled)
+			continue;
+
+		if (HAS_GMCH_DISPLAY(dev_priv))
+			i9xx_check_fifo_underruns(crtc);
+		else if (IS_GEN7(dev_priv))
+			ivybridge_check_fifo_underruns(crtc);
+	}
+
+	spin_unlock_irq(&dev_priv->irq_lock);
+}
+
+/**
+ * intel_check_pch_fifo_underruns - check for PCH fifo underruns immediately
+ * @dev_priv: i915 device instance
+ *
+ * Check for PCH fifo underruns immediately. Useful on CPT/PPT where the shared
+ * error interrupt may have been disabled, and so PCH fifo underruns won't
+ * necessarily raise an interrupt.
+ */
+void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv)
+{
+	struct intel_crtc *crtc;
+
+	spin_lock_irq(&dev_priv->irq_lock);
+
+	for_each_intel_crtc(dev_priv->dev, crtc) {
+		if (crtc->pch_fifo_underrun_disabled)
+			continue;
+
+		if (HAS_PCH_CPT(dev_priv))
+			cpt_check_pch_fifo_underruns(crtc);
+	}
+
+	spin_unlock_irq(&dev_priv->irq_lock);
+}
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 081d5f648d26..822952235dcf 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -42,8 +42,6 @@ struct i915_guc_client {
 
 	uint32_t wq_offset;
 	uint32_t wq_size;
-
-	spinlock_t wq_lock;		/* Protects all data below	*/
 	uint32_t wq_tail;
 
 	/* GuC submission statistics & status */
@@ -76,11 +74,17 @@ struct intel_guc_fw {
 	uint16_t			guc_fw_minor_wanted;
 	uint16_t			guc_fw_major_found;
 	uint16_t			guc_fw_minor_found;
+
+	uint32_t header_size;
+	uint32_t header_offset;
+	uint32_t rsa_size;
+	uint32_t rsa_offset;
+	uint32_t ucode_size;
+	uint32_t ucode_offset;
 };
 
 struct intel_guc {
 	struct intel_guc_fw guc_fw;
-
 	uint32_t log_flags;
 	struct drm_i915_gem_object *log_obj;
 
@@ -89,8 +93,6 @@ struct intel_guc {
 
 	struct i915_guc_client *execbuf_client;
 
-	spinlock_t host2guc_lock;	/* Protects all data below	*/
-
 	DECLARE_BITMAP(doorbell_bitmap, GUC_MAX_DOORBELLS);
 	uint32_t db_cacheline;		/* Cyclic counter mod pagesize	*/
 
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 593d2f585978..40b2ea572e16 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -122,6 +122,78 @@
 
 #define GUC_CTL_MAX_DWORDS		(GUC_CTL_RSRVD + 1)
 
+/**
+ * DOC: GuC Firmware Layout
+ *
+ * The GuC firmware layout looks like this:
+ *
+ *     +-------------------------------+
+ *     |        guc_css_header         |
+ *     | contains major/minor version  |
+ *     +-------------------------------+
+ *     |             uCode             |
+ *     +-------------------------------+
+ *     |         RSA signature         |
+ *     +-------------------------------+
+ *     |          modulus key          |
+ *     +-------------------------------+
+ *     |          exponent val         |
+ *     +-------------------------------+
+ *
+ * The firmware may or may not have modulus key and exponent data. The header,
+ * uCode and RSA signature are must-have components that will be used by driver.
+ * Length of each components, which is all in dwords, can be found in header.
+ * In the case that modulus and exponent are not present in fw, a.k.a truncated
+ * image, the length value still appears in header.
+ *
+ * Driver will do some basic fw size validation based on the following rules:
+ *
+ * 1. Header, uCode and RSA are must-have components.
+ * 2. All firmware components, if they present, are in the sequence illustrated
+ * in the layout table above.
+ * 3. Length info of each component can be found in header, in dwords.
+ * 4. Modulus and exponent key are not required by driver. They may not appear
+ * in fw. So driver will load a truncated firmware in this case.
+ */
+
+struct guc_css_header {
+	uint32_t module_type;
+	/* header_size includes all non-uCode bits, including css_header, rsa
+	 * key, modulus key and exponent data. */
+	uint32_t header_size_dw;
+	uint32_t header_version;
+	uint32_t module_id;
+	uint32_t module_vendor;
+	union {
+		struct {
+			uint8_t day;
+			uint8_t month;
+			uint16_t year;
+		};
+		uint32_t date;
+	};
+	uint32_t size_dw; /* uCode plus header_size_dw */
+	uint32_t key_size_dw;
+	uint32_t modulus_size_dw;
+	uint32_t exponent_size_dw;
+	union {
+		struct {
+			uint8_t hour;
+			uint8_t min;
+			uint16_t sec;
+		};
+		uint32_t time;
+	};
+
+	char username[8];
+	char buildnumber[12];
+	uint32_t device_id;
+	uint32_t guc_sw_version;
+	uint32_t prod_preprod_fw;
+	uint32_t reserved[12];
+	uint32_t header_info;
+} __packed;
+
 struct guc_doorbell_info {
 	u32 db_status;
 	u32 cookie;
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 3541f76c65a7..550921f2ef7d 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -31,7 +31,7 @@
 #include "intel_guc.h"
 
 /**
- * DOC: GuC
+ * DOC: GuC-specific firmware loader
  *
  * intel_guc:
  * Top level structure of guc. It handles firmware loading and manages client
@@ -208,16 +208,6 @@ static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
 /*
  * Transfer the firmware image to RAM for execution by the microcontroller.
  *
- * GuC Firmware layout:
- * +-------------------------------+  ----
- * |          CSS header           |  128B
- * | contains major/minor version  |
- * +-------------------------------+  ----
- * |             uCode             |
- * +-------------------------------+  ----
- * |         RSA signature         |  256B
- * +-------------------------------+  ----
- *
  * Architecturally, the DMA engine is bidirectional, and can potentially even
  * transfer between GTT locations. This functionality is left out of the API
  * for now as there is no need for it.
@@ -225,33 +215,29 @@ static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
  * Note that GuC needs the CSS header plus uKernel code to be copied by the
  * DMA engine in one operation, whereas the RSA signature is loaded via MMIO.
  */
-
-#define UOS_CSS_HEADER_OFFSET		0
-#define UOS_VER_MINOR_OFFSET		0x44
-#define UOS_VER_MAJOR_OFFSET		0x46
-#define UOS_CSS_HEADER_SIZE		0x80
-#define UOS_RSA_SIG_SIZE		0x100
-
 static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 	struct drm_i915_gem_object *fw_obj = guc_fw->guc_fw_obj;
 	unsigned long offset;
 	struct sg_table *sg = fw_obj->pages;
-	u32 status, ucode_size, rsa[UOS_RSA_SIG_SIZE / sizeof(u32)];
+	u32 status, rsa[UOS_RSA_SCRATCH_MAX_COUNT];
 	int i, ret = 0;
 
-	/* uCode size, also is where RSA signature starts */
-	offset = ucode_size = guc_fw->guc_fw_size - UOS_RSA_SIG_SIZE;
-	I915_WRITE(DMA_COPY_SIZE, ucode_size);
+	/* where RSA signature starts */
+	offset = guc_fw->rsa_offset;
 
 	/* Copy RSA signature from the fw image to HW for verification */
-	sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, UOS_RSA_SIG_SIZE, offset);
-	for (i = 0; i < UOS_RSA_SIG_SIZE / sizeof(u32); i++)
+	sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa), offset);
+	for (i = 0; i < UOS_RSA_SCRATCH_MAX_COUNT; i++)
 		I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
 
+	/* The header plus uCode will be copied to WOPCM via DMA, excluding any
+	 * other components */
+	I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size);
+
 	/* Set the source address for the new blob */
-	offset = i915_gem_obj_ggtt_offset(fw_obj);
+	offset = i915_gem_obj_ggtt_offset(fw_obj) + guc_fw->header_offset;
 	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
 	I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
 
@@ -322,8 +308,8 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 	I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
 
 	/* WaDisableMinuteIaClockGating:skl,bxt */
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) == BXT_REVID_A0)) {
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
 					      ~GUC_ENABLE_MIA_CLOCK_GATING));
 	}
@@ -378,6 +364,9 @@ int intel_guc_ucode_load(struct drm_device *dev)
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 	int err = 0;
 
+	if (!i915.enable_guc_submission)
+		return 0;
+
 	DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n",
 		intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
 		intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
@@ -457,10 +446,8 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 {
 	struct drm_i915_gem_object *obj;
 	const struct firmware *fw;
-	const u8 *css_header;
-	const size_t minsize = UOS_CSS_HEADER_SIZE + UOS_RSA_SIG_SIZE;
-	const size_t maxsize = GUC_WOPCM_SIZE_VALUE + UOS_RSA_SIG_SIZE
-			- 0x8000; /* 32k reserved (8K stack + 24k context) */
+	struct guc_css_header *css;
+	size_t size;
 	int err;
 
 	DRM_DEBUG_DRIVER("before requesting firmware: GuC fw fetch status %s\n",
@@ -474,12 +461,52 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 
 	DRM_DEBUG_DRIVER("fetch GuC fw from %s succeeded, fw %p\n",
 		guc_fw->guc_fw_path, fw);
-	DRM_DEBUG_DRIVER("firmware file size %zu (minimum %zu, maximum %zu)\n",
-		fw->size, minsize, maxsize);
 
-	/* Check the size of the blob befoe examining buffer contents */
-	if (fw->size < minsize || fw->size > maxsize)
+	/* Check the size of the blob before examining buffer contents */
+	if (fw->size < sizeof(struct guc_css_header)) {
+		DRM_ERROR("Firmware header is missing\n");
 		goto fail;
+	}
+
+	css = (struct guc_css_header *)fw->data;
+
+	/* Firmware bits always start from header */
+	guc_fw->header_offset = 0;
+	guc_fw->header_size = (css->header_size_dw - css->modulus_size_dw -
+		css->key_size_dw - css->exponent_size_dw) * sizeof(u32);
+
+	if (guc_fw->header_size != sizeof(struct guc_css_header)) {
+		DRM_ERROR("CSS header definition mismatch\n");
+		goto fail;
+	}
+
+	/* then, uCode */
+	guc_fw->ucode_offset = guc_fw->header_offset + guc_fw->header_size;
+	guc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32);
+
+	/* now RSA */
+	if (css->key_size_dw != UOS_RSA_SCRATCH_MAX_COUNT) {
+		DRM_ERROR("RSA key size is bad\n");
+		goto fail;
+	}
+	guc_fw->rsa_offset = guc_fw->ucode_offset + guc_fw->ucode_size;
+	guc_fw->rsa_size = css->key_size_dw * sizeof(u32);
+
+	/* At least, it should have header, uCode and RSA. Size of all three. */
+	size = guc_fw->header_size + guc_fw->ucode_size + guc_fw->rsa_size;
+	if (fw->size < size) {
+		DRM_ERROR("Missing firmware components\n");
+		goto fail;
+	}
+
+	/* Header and uCode will be loaded to WOPCM. Size of the two. */
+	size = guc_fw->header_size + guc_fw->ucode_size;
+
+	/* Top 32k of WOPCM is reserved (8K stack + 24k RC6 context). */
+	if (size > GUC_WOPCM_SIZE_VALUE - 0x8000) {
+		DRM_ERROR("Firmware is too large to fit in WOPCM\n");
+		goto fail;
+	}
 
 	/*
 	 * The GuC firmware image has the version number embedded at a well-known
@@ -487,9 +514,8 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 	 * TWO bytes each (i.e. u16), although all pointers and offsets are defined
 	 * in terms of bytes (u8).
 	 */
-	css_header = fw->data + UOS_CSS_HEADER_OFFSET;
-	guc_fw->guc_fw_major_found = *(u16 *)(css_header + UOS_VER_MAJOR_OFFSET);
-	guc_fw->guc_fw_minor_found = *(u16 *)(css_header + UOS_VER_MINOR_OFFSET);
+	guc_fw->guc_fw_major_found = css->guc_sw_version >> 16;
+	guc_fw->guc_fw_minor_found = css->guc_sw_version & 0xFFFF;
 
 	if (guc_fw->guc_fw_major_found != guc_fw->guc_fw_major_wanted ||
 	    guc_fw->guc_fw_minor_found < guc_fw->guc_fw_minor_wanted) {
@@ -566,6 +592,9 @@ void intel_guc_ucode_init(struct drm_device *dev)
 		fw_path = "";	/* unknown device */
 	}
 
+	if (!i915.enable_guc_submission)
+		return;
+
 	guc_fw->guc_dev = dev;
 	guc_fw->guc_fw_path = fw_path;
 	guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE;
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index e6c035b0fc1c..4a77639a489d 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -78,7 +78,7 @@ static u32 g4x_infoframe_index(enum hdmi_infoframe_type type)
 	case HDMI_INFOFRAME_TYPE_VENDOR:
 		return VIDEO_DIP_SELECT_VENDOR;
 	default:
-		DRM_DEBUG_DRIVER("unknown info frame type %d\n", type);
+		MISSING_CASE(type);
 		return 0;
 	}
 }
@@ -93,7 +93,7 @@ static u32 g4x_infoframe_enable(enum hdmi_infoframe_type type)
 	case HDMI_INFOFRAME_TYPE_VENDOR:
 		return VIDEO_DIP_ENABLE_VENDOR;
 	default:
-		DRM_DEBUG_DRIVER("unknown info frame type %d\n", type);
+		MISSING_CASE(type);
 		return 0;
 	}
 }
@@ -108,15 +108,16 @@ static u32 hsw_infoframe_enable(enum hdmi_infoframe_type type)
 	case HDMI_INFOFRAME_TYPE_VENDOR:
 		return VIDEO_DIP_ENABLE_VS_HSW;
 	default:
-		DRM_DEBUG_DRIVER("unknown info frame type %d\n", type);
+		MISSING_CASE(type);
 		return 0;
 	}
 }
 
-static u32 hsw_dip_data_reg(struct drm_i915_private *dev_priv,
-			    enum transcoder cpu_transcoder,
-			    enum hdmi_infoframe_type type,
-			    int i)
+static i915_reg_t
+hsw_dip_data_reg(struct drm_i915_private *dev_priv,
+		 enum transcoder cpu_transcoder,
+		 enum hdmi_infoframe_type type,
+		 int i)
 {
 	switch (type) {
 	case HDMI_INFOFRAME_TYPE_AVI:
@@ -126,8 +127,8 @@ static u32 hsw_dip_data_reg(struct drm_i915_private *dev_priv,
 	case HDMI_INFOFRAME_TYPE_VENDOR:
 		return HSW_TVIDEO_DIP_VS_DATA(cpu_transcoder, i);
 	default:
-		DRM_DEBUG_DRIVER("unknown info frame type %d\n", type);
-		return 0;
+		MISSING_CASE(type);
+		return INVALID_MMIO_REG;
 	}
 }
 
@@ -168,10 +169,10 @@ static void g4x_write_infoframe(struct drm_encoder *encoder,
 	POSTING_READ(VIDEO_DIP_CTL);
 }
 
-static bool g4x_infoframe_enabled(struct drm_encoder *encoder)
+static bool g4x_infoframe_enabled(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
 	u32 val = I915_READ(VIDEO_DIP_CTL);
 
@@ -193,8 +194,9 @@ static void ibx_write_infoframe(struct drm_encoder *encoder,
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-	int i, reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
+	int i;
 
 	WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n");
 
@@ -223,13 +225,13 @@ static void ibx_write_infoframe(struct drm_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool ibx_infoframe_enabled(struct drm_encoder *encoder)
+static bool ibx_infoframe_enabled(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-	int reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe;
+	i915_reg_t reg = TVIDEO_DIP_CTL(pipe);
 	u32 val = I915_READ(reg);
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
@@ -251,8 +253,9 @@ static void cpt_write_infoframe(struct drm_encoder *encoder,
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-	int i, reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
+	int i;
 
 	WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n");
 
@@ -284,13 +287,12 @@ static void cpt_write_infoframe(struct drm_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool cpt_infoframe_enabled(struct drm_encoder *encoder)
+static bool cpt_infoframe_enabled(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-	int reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
-	u32 val = I915_READ(reg);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+	enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe;
+	u32 val = I915_READ(TVIDEO_DIP_CTL(pipe));
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
 		return false;
@@ -308,8 +310,9 @@ static void vlv_write_infoframe(struct drm_encoder *encoder,
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-	int i, reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
+	int i;
 
 	WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n");
 
@@ -338,14 +341,13 @@ static void vlv_write_infoframe(struct drm_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool vlv_infoframe_enabled(struct drm_encoder *encoder)
+static bool vlv_infoframe_enabled(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-	int reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
-	u32 val = I915_READ(reg);
+	enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe;
+	u32 val = I915_READ(VLV_TVIDEO_DIP_CTL(pipe));
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
 		return false;
@@ -367,14 +369,12 @@ static void hsw_write_infoframe(struct drm_encoder *encoder,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
-	u32 ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
-	u32 data_reg;
+	i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
+	i915_reg_t data_reg;
 	int i;
 	u32 val = I915_READ(ctl_reg);
 
 	data_reg = hsw_dip_data_reg(dev_priv, cpu_transcoder, type, 0);
-	if (data_reg == 0)
-		return;
 
 	val &= ~hsw_infoframe_enable(type);
 	I915_WRITE(ctl_reg, val);
@@ -396,13 +396,11 @@ static void hsw_write_infoframe(struct drm_encoder *encoder,
 	POSTING_READ(ctl_reg);
 }
 
-static bool hsw_infoframe_enabled(struct drm_encoder *encoder)
+static bool hsw_infoframe_enabled(struct drm_encoder *encoder,
+				  const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
-	u32 ctl_reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder);
-	u32 val = I915_READ(ctl_reg);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+	u32 val = I915_READ(HSW_TVIDEO_DIP_CTL(pipe_config->cpu_transcoder));
 
 	return val & (VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_AVI_HSW |
 		      VIDEO_DIP_ENABLE_GCP_HSW | VIDEO_DIP_ENABLE_VS_HSW |
@@ -513,7 +511,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
 	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
 	struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
-	u32 reg = VIDEO_DIP_CTL;
+	i915_reg_t reg = VIDEO_DIP_CTL;
 	u32 val = I915_READ(reg);
 	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
 
@@ -633,11 +631,12 @@ static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
 	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
-	u32 reg, val = 0;
+	i915_reg_t reg;
+	u32 val = 0;
 
 	if (HAS_DDI(dev_priv))
 		reg = HSW_TVIDEO_DIP_GCP(crtc->config->cpu_transcoder);
-	else if (IS_VALLEYVIEW(dev_priv))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		reg = VLV_TVIDEO_DIP_GCP(crtc->pipe);
 	else if (HAS_PCH_SPLIT(dev_priv->dev))
 		reg = TVIDEO_DIP_GCP(crtc->pipe);
@@ -666,7 +665,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
 	struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
-	u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
 	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
 
@@ -717,7 +716,7 @@ static void cpt_set_infoframes(struct drm_encoder *encoder,
 	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
 
 	assert_hdmi_port_disabled(intel_hdmi);
@@ -760,7 +759,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder,
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	u32 reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
+	i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
 	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
 
@@ -811,7 +810,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder,
 	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	u32 reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder);
+	i915_reg_t reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder);
 	u32 val = I915_READ(reg);
 
 	assert_hdmi_port_disabled(intel_hdmi);
@@ -925,7 +924,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	if (tmp & HDMI_MODE_SELECT_HDMI)
 		pipe_config->has_hdmi_sink = true;
 
-	if (intel_hdmi->infoframe_enabled(&encoder->base))
+	if (intel_hdmi->infoframe_enabled(&encoder->base, pipe_config))
 		pipe_config->has_infoframe = true;
 
 	if (tmp & SDVO_AUDIO_ENABLE)
@@ -1108,6 +1107,13 @@ static void intel_disable_hdmi(struct intel_encoder *encoder)
 	 * matching DP port to be enabled on transcoder A.
 	 */
 	if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B) {
+		/*
+		 * We get CPU/PCH FIFO underruns on the other pipe when
+		 * doing the workaround. Sweep them under the rug.
+		 */
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+
 		temp &= ~SDVO_PIPE_B_SELECT;
 		temp |= SDVO_ENABLE;
 		/*
@@ -1122,6 +1128,10 @@ static void intel_disable_hdmi(struct intel_encoder *encoder)
 		temp &= ~SDVO_ENABLE;
 		I915_WRITE(intel_hdmi->hdmi_reg, temp);
 		POSTING_READ(intel_hdmi->hdmi_reg);
+
+		intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
 
 	intel_hdmi->set_infoframes(&encoder->base, false, NULL);
@@ -1338,14 +1348,15 @@ intel_hdmi_set_edid(struct drm_connector *connector, bool force)
 	struct edid *edid = NULL;
 	bool connected = false;
 
-	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
+	if (force) {
+		intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
 
-	if (force)
 		edid = drm_get_edid(connector,
 				    intel_gmbus_get_adapter(dev_priv,
 				    intel_hdmi->ddc_bus));
 
-	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
+		intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
+	}
 
 	to_intel_connector(connector)->detect_edid = edid;
 	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
@@ -2040,7 +2051,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 		 * On BXT A0/A1, sw needs to activate DDIA HPD logic and
 		 * interrupts to check the external panel connection.
 		 */
-		if (IS_BROXTON(dev_priv) && (INTEL_REVID(dev) < BXT_REVID_B0))
+		if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 			intel_encoder->hpd_pin = HPD_PORT_A;
 		else
 			intel_encoder->hpd_pin = HPD_PORT_B;
@@ -2088,7 +2099,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 		BUG();
 	}
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		intel_hdmi->write_infoframe = vlv_write_infoframe;
 		intel_hdmi->set_infoframes = vlv_set_infoframes;
 		intel_hdmi->infoframe_enabled = vlv_infoframe_enabled;
@@ -2132,8 +2143,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 	}
 }
 
-void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port)
+void intel_hdmi_init(struct drm_device *dev,
+		     i915_reg_t hdmi_reg, enum port port)
 {
+	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_digital_port *intel_dig_port;
 	struct intel_encoder *intel_encoder;
 	struct intel_connector *intel_connector;
@@ -2151,7 +2164,7 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port)
 	intel_encoder = &intel_dig_port->base;
 
 	drm_encoder_init(dev, &intel_encoder->base, &intel_hdmi_enc_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	intel_encoder->compute_config = intel_hdmi_compute_config;
 	if (HAS_PCH_SPLIT(dev)) {
@@ -2202,8 +2215,9 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port)
 		intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
 	intel_dig_port->port = port;
+	dev_priv->dig_port_map[port] = intel_encoder;
 	intel_dig_port->hdmi.hdmi_reg = hdmi_reg;
-	intel_dig_port->dp.output_reg = 0;
+	intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
 
 	intel_hdmi_init_connector(intel_dig_port, intel_connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index b17785719598..bee673005d48 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -407,7 +407,7 @@ void intel_hpd_irq_handler(struct drm_device *dev,
 			 * hotplug bits itself. So only WARN about unexpected
 			 * interrupts on saner platforms.
 			 */
-			WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
+			WARN_ONCE(!HAS_GMCH_DISPLAY(dev),
 				  "Received HPD interrupt on pin %d although disabled\n", i);
 			continue;
 		}
@@ -468,9 +468,14 @@ void intel_hpd_init(struct drm_i915_private *dev_priv)
 	list_for_each_entry(connector, &mode_config->connector_list, head) {
 		struct intel_connector *intel_connector = to_intel_connector(connector);
 		connector->polled = intel_connector->polled;
-		if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
-			connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+		/* MST has a dynamic intel_connector->encoder and it's reprobing
+		 * is all handled by the MST helpers. */
 		if (intel_connector->mst_port)
+			continue;
+
+		if (!connector->polled && I915_HAS_HOTPLUG(dev) &&
+		    intel_connector->encoder->hpd_pin > HPD_NONE)
 			connector->polled = DRM_CONNECTOR_POLL_HPD;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 8324654037b6..25254b5c1ac5 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -36,7 +36,7 @@
 
 struct gmbus_pin {
 	const char *name;
-	int reg;
+	i915_reg_t reg;
 };
 
 /* Map gmbus pin pairs to names and registers. */
@@ -63,9 +63,9 @@ static const struct gmbus_pin gmbus_pins_skl[] = {
 };
 
 static const struct gmbus_pin gmbus_pins_bxt[] = {
-	[GMBUS_PIN_1_BXT] = { "dpb", PCH_GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpc", PCH_GPIOC },
-	[GMBUS_PIN_3_BXT] = { "misc", PCH_GPIOD },
+	[GMBUS_PIN_1_BXT] = { "dpb", GPIOB },
+	[GMBUS_PIN_2_BXT] = { "dpc", GPIOC },
+	[GMBUS_PIN_3_BXT] = { "misc", GPIOD },
 };
 
 /* pin is expected to be valid */
@@ -74,7 +74,7 @@ static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *dev_priv,
 {
 	if (IS_BROXTON(dev_priv))
 		return &gmbus_pins_bxt[pin];
-	else if (IS_SKYLAKE(dev_priv))
+	else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		return &gmbus_pins_skl[pin];
 	else if (IS_BROADWELL(dev_priv))
 		return &gmbus_pins_bdw[pin];
@@ -89,14 +89,15 @@ bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv,
 
 	if (IS_BROXTON(dev_priv))
 		size = ARRAY_SIZE(gmbus_pins_bxt);
-	else if (IS_SKYLAKE(dev_priv))
+	else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		size = ARRAY_SIZE(gmbus_pins_skl);
 	else if (IS_BROADWELL(dev_priv))
 		size = ARRAY_SIZE(gmbus_pins_bdw);
 	else
 		size = ARRAY_SIZE(gmbus_pins);
 
-	return pin < size && get_gmbus_pin(dev_priv, pin)->reg;
+	return pin < size &&
+		i915_mmio_reg_valid(get_gmbus_pin(dev_priv, pin)->reg);
 }
 
 /* Intel GPIO access functions */
@@ -240,9 +241,8 @@ intel_gpio_setup(struct intel_gmbus *bus, unsigned int pin)
 
 	algo = &bus->bit_algo;
 
-	bus->gpio_reg = dev_priv->gpio_mmio_base +
-		get_gmbus_pin(dev_priv, pin)->reg;
-
+	bus->gpio_reg = _MMIO(dev_priv->gpio_mmio_base +
+			      i915_mmio_reg_offset(get_gmbus_pin(dev_priv, pin)->reg));
 	bus->adapter.algo_data = algo;
 	algo->setsda = set_data;
 	algo->setscl = set_clock;
@@ -472,9 +472,7 @@ gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct i2c_msg *msgs)
 }
 
 static int
-gmbus_xfer(struct i2c_adapter *adapter,
-	   struct i2c_msg *msgs,
-	   int num)
+do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 {
 	struct intel_gmbus *bus = container_of(adapter,
 					       struct intel_gmbus,
@@ -483,14 +481,6 @@ gmbus_xfer(struct i2c_adapter *adapter,
 	int i = 0, inc, try = 0;
 	int ret = 0;
 
-	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
-	mutex_lock(&dev_priv->gmbus_mutex);
-
-	if (bus->force_bit) {
-		ret = i2c_bit_algo.master_xfer(adapter, msgs, num);
-		goto out;
-	}
-
 retry:
 	I915_WRITE(GMBUS0, bus->reg0);
 
@@ -505,17 +495,13 @@ retry:
 			ret = gmbus_xfer_write(dev_priv, &msgs[i]);
 		}
 
+		if (!ret)
+			ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_WAIT_PHASE,
+						   GMBUS_HW_WAIT_EN);
 		if (ret == -ETIMEDOUT)
 			goto timeout;
-		if (ret == -ENXIO)
-			goto clear_err;
-
-		ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_WAIT_PHASE,
-					   GMBUS_HW_WAIT_EN);
-		if (ret == -ENXIO)
+		else if (ret)
 			goto clear_err;
-		if (ret)
-			goto timeout;
 	}
 
 	/* Generate a STOP condition on the bus. Note that gmbus can't generata
@@ -589,13 +575,34 @@ timeout:
 		 bus->adapter.name, bus->reg0 & 0xff);
 	I915_WRITE(GMBUS0, 0);
 
-	/* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */
+	/*
+	 * Hardware may not support GMBUS over these pins? Try GPIO bitbanging
+	 * instead. Use EAGAIN to have i2c core retry.
+	 */
 	bus->force_bit = 1;
-	ret = i2c_bit_algo.master_xfer(adapter, msgs, num);
+	ret = -EAGAIN;
 
 out:
-	mutex_unlock(&dev_priv->gmbus_mutex);
+	return ret;
+}
 
+static int
+gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
+{
+	struct intel_gmbus *bus = container_of(adapter, struct intel_gmbus,
+					       adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+	int ret;
+
+	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
+	mutex_lock(&dev_priv->gmbus_mutex);
+
+	if (bus->force_bit)
+		ret = i2c_bit_algo.master_xfer(adapter, msgs, num);
+	else
+		ret = do_gmbus_xfer(adapter, msgs, num);
+
+	mutex_unlock(&dev_priv->gmbus_mutex);
 	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
 
 	return ret;
@@ -628,12 +635,13 @@ int intel_setup_gmbus(struct drm_device *dev)
 
 	if (HAS_PCH_NOP(dev))
 		return 0;
-	else if (HAS_PCH_SPLIT(dev))
-		dev_priv->gpio_mmio_base = PCH_GPIOA - GPIOA;
-	else if (IS_VALLEYVIEW(dev))
+
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE;
-	else
-		dev_priv->gpio_mmio_base = 0;
+	else if (!HAS_GMCH_DISPLAY(dev_priv))
+		dev_priv->gpio_mmio_base =
+			i915_mmio_reg_offset(PCH_GPIOA) -
+			i915_mmio_reg_offset(GPIOA);
 
 	mutex_init(&dev_priv->gmbus_mutex);
 	init_waitqueue_head(&dev_priv->gmbus_wait_queue);
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 88e12bdf79e2..3aa614731d7e 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -190,16 +190,21 @@
 #define GEN8_CTX_L3LLC_COHERENT (1<<5)
 #define GEN8_CTX_PRIVILEGE (1<<8)
 
-#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) { \
+#define ASSIGN_CTX_REG(reg_state, pos, reg, val) do { \
+	(reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \
+	(reg_state)[(pos)+1] = (val); \
+} while (0)
+
+#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do {		\
 	const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n));	\
 	reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
 	reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
-}
+} while (0)
 
-#define ASSIGN_CTX_PML4(ppgtt, reg_state) { \
+#define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
 	reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
 	reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
-}
+} while (0)
 
 enum {
 	ADVANCED_CONTEXT = 0,
@@ -284,8 +289,8 @@ static bool disable_lite_restore_wa(struct intel_engine_cs *ring)
 {
 	struct drm_device *dev = ring->dev;
 
-	return ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) ||
-		(IS_BROXTON(dev) && INTEL_REVID(dev) == BXT_REVID_A0)) &&
+	return (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+		IS_BXT_REVID(dev, 0, BXT_REVID_A1)) &&
 	       (ring->id == VCS || ring->id == VCS2);
 }
 
@@ -367,7 +372,7 @@ static int execlists_update_context(struct drm_i915_gem_request *rq)
 	WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
 	WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
 
-	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
 	reg_state = kmap_atomic(page);
 
 	reg_state[CTX_RING_TAIL+1] = rq->tail;
@@ -921,7 +926,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
 
 		intel_logical_ring_emit(ringbuf, MI_NOOP);
 		intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
-		intel_logical_ring_emit(ringbuf, INSTPM);
+		intel_logical_ring_emit_reg(ringbuf, INSTPM);
 		intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
 		intel_logical_ring_advance(ringbuf);
 
@@ -1096,7 +1101,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
 
 	intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));
 	for (i = 0; i < w->count; i++) {
-		intel_logical_ring_emit(ringbuf, w->reg[i].addr);
+		intel_logical_ring_emit_reg(ringbuf, w->reg[i].addr);
 		intel_logical_ring_emit(ringbuf, w->reg[i].value);
 	}
 	intel_logical_ring_emit(ringbuf, MI_NOOP);
@@ -1120,6 +1125,8 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
 		batch[__index] = (cmd);					\
 	} while (0)
 
+#define wa_ctx_emit_reg(batch, index, reg) \
+	wa_ctx_emit((batch), (index), i915_mmio_reg_offset(reg))
 
 /*
  * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
@@ -1149,17 +1156,17 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *ring,
 	 * this batch updates GEN8_L3SQCREG4 with default value we need to
 	 * set this bit here to retain the WA during flush.
 	 */
-	if (IS_SKYLAKE(ring->dev) && INTEL_REVID(ring->dev) <= SKL_REVID_E0)
+	if (IS_SKL_REVID(ring->dev, 0, SKL_REVID_E0))
 		l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS;
 
 	wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
 				   MI_SRM_LRM_GLOBAL_GTT));
-	wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
+	wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
 	wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
 	wa_ctx_emit(batch, index, 0);
 
 	wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
-	wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
+	wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
 	wa_ctx_emit(batch, index, l3sqc4_flush);
 
 	wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
@@ -1172,7 +1179,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *ring,
 
 	wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 |
 				   MI_SRM_LRM_GLOBAL_GTT));
-	wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
+	wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
 	wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
 	wa_ctx_emit(batch, index, 0);
 
@@ -1314,8 +1321,8 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *ring,
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
 	/* WaDisableCtxRestoreArbitration:skl,bxt */
-	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_D0)) ||
-	    (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0)))
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 
 	/* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
@@ -1340,18 +1347,18 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *ring,
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
 	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_B0)) ||
-	    (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0))) {
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
-		wa_ctx_emit(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
+		wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
 		wa_ctx_emit(batch, index,
 			    _MASKED_BIT_ENABLE(DISABLE_PIXEL_MASK_CAMMING));
 		wa_ctx_emit(batch, index, MI_NOOP);
 	}
 
 	/* WaDisableCtxRestoreArbitration:skl,bxt */
-	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_D0)) ||
-	    (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0)))
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
 
 	wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
@@ -1418,7 +1425,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *ring)
 		return ret;
 	}
 
-	page = i915_gem_object_get_page(wa_ctx->obj, 0);
+	page = i915_gem_object_get_dirty_page(wa_ctx->obj, 0);
 	batch = kmap_atomic(page);
 	offset = 0;
 
@@ -1472,12 +1479,6 @@ static int gen8_init_common_ring(struct intel_engine_cs *ring)
 	I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 	I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
 
-	if (ring->status_page.obj) {
-		I915_WRITE(RING_HWS_PGA(ring->mmio_base),
-			   (u32)ring->status_page.gfx_addr);
-		POSTING_READ(RING_HWS_PGA(ring->mmio_base));
-	}
-
 	I915_WRITE(RING_MODE_GEN7(ring),
 		   _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
 		   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
@@ -1562,9 +1563,9 @@ static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
 	for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 
-		intel_logical_ring_emit(ringbuf, GEN8_RING_PDP_UDW(ring, i));
+		intel_logical_ring_emit_reg(ringbuf, GEN8_RING_PDP_UDW(ring, i));
 		intel_logical_ring_emit(ringbuf, upper_32_bits(pd_daddr));
-		intel_logical_ring_emit(ringbuf, GEN8_RING_PDP_LDW(ring, i));
+		intel_logical_ring_emit_reg(ringbuf, GEN8_RING_PDP_LDW(ring, i));
 		intel_logical_ring_emit(ringbuf, lower_32_bits(pd_daddr));
 	}
 
@@ -1893,8 +1894,10 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
 
 	dev_priv = ring->dev->dev_private;
 
-	intel_logical_ring_stop(ring);
-	WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
+	if (ring->buffer) {
+		intel_logical_ring_stop(ring);
+		WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
+	}
 
 	if (ring->cleanup)
 		ring->cleanup(ring);
@@ -1908,6 +1911,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
 	}
 
 	lrc_destroy_wa_ctx_obj(ring);
+	ring->dev = NULL;
 }
 
 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
@@ -1923,17 +1927,18 @@ static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *rin
 	i915_gem_batch_pool_init(dev, &ring->batch_pool);
 	init_waitqueue_head(&ring->irq_queue);
 
+	INIT_LIST_HEAD(&ring->buffers);
 	INIT_LIST_HEAD(&ring->execlist_queue);
 	INIT_LIST_HEAD(&ring->execlist_retired_req_list);
 	spin_lock_init(&ring->execlist_lock);
 
 	ret = i915_cmd_parser_init_ring(ring);
 	if (ret)
-		return ret;
+		goto error;
 
 	ret = intel_lr_context_deferred_alloc(ring->default_context, ring);
 	if (ret)
-		return ret;
+		goto error;
 
 	/* As this is the default context, always pin it */
 	ret = intel_lr_context_do_pin(
@@ -1944,9 +1949,13 @@ static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *rin
 		DRM_ERROR(
 			"Failed to pin and map ringbuffer %s: %d\n",
 			ring->name, ret);
-		return ret;
+		goto error;
 	}
 
+	return 0;
+
+error:
+	intel_logical_ring_cleanup(ring);
 	return ret;
 }
 
@@ -1972,7 +1981,7 @@ static int logical_render_ring_init(struct drm_device *dev)
 		ring->init_hw = gen8_init_render_ring;
 	ring->init_context = gen8_init_rcs_context;
 	ring->cleanup = intel_fini_pipe_control;
-	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		ring->get_seqno = bxt_a_get_seqno;
 		ring->set_seqno = bxt_a_set_seqno;
 	} else {
@@ -2024,7 +2033,7 @@ static int logical_bsd_ring_init(struct drm_device *dev)
 		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 
 	ring->init_hw = gen8_init_common_ring;
-	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		ring->get_seqno = bxt_a_get_seqno;
 		ring->set_seqno = bxt_a_set_seqno;
 	} else {
@@ -2079,7 +2088,7 @@ static int logical_blt_ring_init(struct drm_device *dev)
 		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 
 	ring->init_hw = gen8_init_common_ring;
-	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		ring->get_seqno = bxt_a_get_seqno;
 		ring->set_seqno = bxt_a_set_seqno;
 	} else {
@@ -2109,7 +2118,7 @@ static int logical_vebox_ring_init(struct drm_device *dev)
 		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 
 	ring->init_hw = gen8_init_common_ring;
-	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		ring->get_seqno = bxt_a_get_seqno;
 		ring->set_seqno = bxt_a_set_seqno;
 	} else {
@@ -2255,7 +2264,7 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o
 
 	/* The second page of the context object contains some fields which must
 	 * be set up prior to the first execution. */
-	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
 	reg_state = kmap_atomic(page);
 
 	/* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
@@ -2263,46 +2272,31 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o
 	 * only for the first context restore: on a subsequent save, the GPU will
 	 * recreate this batchbuffer with new values (including all the missing
 	 * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
-	if (ring->id == RCS)
-		reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
-	else
-		reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
-	reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
-	reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
-	reg_state[CTX_CONTEXT_CONTROL+1] =
-		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
-				   CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
-				   CTX_CTRL_RS_CTX_ENABLE);
-	reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
-	reg_state[CTX_RING_HEAD+1] = 0;
-	reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
-	reg_state[CTX_RING_TAIL+1] = 0;
-	reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
+	reg_state[CTX_LRI_HEADER_0] =
+		MI_LOAD_REGISTER_IMM(ring->id == RCS ? 14 : 11) | MI_LRI_FORCE_POSTED;
+	ASSIGN_CTX_REG(reg_state, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(ring),
+		       _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
+					  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
+					  CTX_CTRL_RS_CTX_ENABLE));
+	ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(ring->mmio_base), 0);
 	/* Ring buffer start address is not known until the buffer is pinned.
 	 * It is written to the context image in execlists_update_context()
 	 */
-	reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
-	reg_state[CTX_RING_BUFFER_CONTROL+1] =
-			((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
-	reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
-	reg_state[CTX_BB_HEAD_U+1] = 0;
-	reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
-	reg_state[CTX_BB_HEAD_L+1] = 0;
-	reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
-	reg_state[CTX_BB_STATE+1] = (1<<5);
-	reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
-	reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
-	reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
-	reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
-	reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
-	reg_state[CTX_SECOND_BB_STATE+1] = 0;
+	ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_START, RING_START(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL, RING_CTL(ring->mmio_base),
+		       ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID);
+	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U, RING_BBADDR_UDW(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L, RING_BBADDR(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_BB_STATE, RING_BBSTATE(ring->mmio_base),
+		       RING_BB_PPGTT);
+	ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(ring->mmio_base), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_STATE, RING_SBBSTATE(ring->mmio_base), 0);
 	if (ring->id == RCS) {
-		reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
-		reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
-		reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
-		reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
-		reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
-		reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
+		ASSIGN_CTX_REG(reg_state, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(ring->mmio_base), 0);
+		ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(ring->mmio_base), 0);
+		ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX_OFFSET, RING_INDIRECT_CTX_OFFSET(ring->mmio_base), 0);
 		if (ring->wa_ctx.obj) {
 			struct i915_ctx_workarounds *wa_ctx = &ring->wa_ctx;
 			uint32_t ggtt_offset = i915_gem_obj_ggtt_offset(wa_ctx->obj);
@@ -2319,18 +2313,17 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o
 				0x01;
 		}
 	}
-	reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
-	reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
-	reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
-	reg_state[CTX_CTX_TIMESTAMP+1] = 0;
-	reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
-	reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
-	reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
-	reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
-	reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
-	reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
-	reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
-	reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
+	reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;
+	ASSIGN_CTX_REG(reg_state, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(ring->mmio_base), 0);
+	/* PDP values well be assigned later if needed */
+	ASSIGN_CTX_REG(reg_state, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(ring, 3), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(ring, 3), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(ring, 2), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(ring, 2), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(ring, 1), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(ring, 1), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(ring, 0), 0);
+	ASSIGN_CTX_REG(reg_state, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(ring, 0), 0);
 
 	if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
 		/* 64b PPGTT (48bit canonical)
@@ -2352,14 +2345,11 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o
 
 	if (ring->id == RCS) {
 		reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
-		reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE;
-		reg_state[CTX_R_PWR_CLK_STATE+1] = make_rpcs(dev);
+		ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
+			       make_rpcs(dev));
 	}
 
 	kunmap_atomic(reg_state);
-
-	ctx_obj->dirty = 1;
-	set_page_dirty(page);
 	i915_gem_object_unpin_pages(ctx_obj);
 
 	return 0;
@@ -2543,7 +2533,7 @@ void intel_lr_context_reset(struct drm_device *dev,
 			WARN(1, "Failed get_pages for context obj\n");
 			continue;
 		}
-		page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+		page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
 		reg_state = kmap_atomic(page);
 
 		reg_state[CTX_RING_HEAD+1] = 0;
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 4e60d54ba66d..0b821b91723a 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -29,16 +29,16 @@
 #define GEN8_CSB_PTR_MASK 0x07
 
 /* Execlists regs */
-#define RING_ELSP(ring)			((ring)->mmio_base+0x230)
-#define RING_EXECLIST_STATUS_LO(ring)	((ring)->mmio_base+0x234)
-#define RING_EXECLIST_STATUS_HI(ring)	((ring)->mmio_base+0x234 + 4)
-#define RING_CONTEXT_CONTROL(ring)	((ring)->mmio_base+0x244)
+#define RING_ELSP(ring)				_MMIO((ring)->mmio_base + 0x230)
+#define RING_EXECLIST_STATUS_LO(ring)		_MMIO((ring)->mmio_base + 0x234)
+#define RING_EXECLIST_STATUS_HI(ring)		_MMIO((ring)->mmio_base + 0x234 + 4)
+#define RING_CONTEXT_CONTROL(ring)		_MMIO((ring)->mmio_base + 0x244)
 #define	  CTX_CTRL_INHIBIT_SYN_CTX_SWITCH	(1 << 3)
 #define	  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT	(1 << 0)
 #define   CTX_CTRL_RS_CTX_ENABLE                (1 << 1)
-#define RING_CONTEXT_STATUS_BUF_LO(ring, i)	((ring)->mmio_base+0x370 + (i) * 8)
-#define RING_CONTEXT_STATUS_BUF_HI(ring, i)	((ring)->mmio_base+0x370 + (i) * 8 + 4)
-#define RING_CONTEXT_STATUS_PTR(ring)	((ring)->mmio_base+0x3a0)
+#define RING_CONTEXT_STATUS_BUF_LO(ring, i)	_MMIO((ring)->mmio_base + 0x370 + (i) * 8)
+#define RING_CONTEXT_STATUS_BUF_HI(ring, i)	_MMIO((ring)->mmio_base + 0x370 + (i) * 8 + 4)
+#define RING_CONTEXT_STATUS_PTR(ring)		_MMIO((ring)->mmio_base + 0x3a0)
 
 /* Logical Rings */
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request);
@@ -70,6 +70,11 @@ static inline void intel_logical_ring_emit(struct intel_ringbuffer *ringbuf,
 	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
 	ringbuf->tail += 4;
 }
+static inline void intel_logical_ring_emit_reg(struct intel_ringbuffer *ringbuf,
+					       i915_reg_t reg)
+{
+	intel_logical_ring_emit(ringbuf, i915_mmio_reg_offset(reg));
+}
 
 /* Logical Ring Contexts */
 
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 7f39b8ad88ae..0da0240caf81 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -51,7 +51,7 @@ struct intel_lvds_encoder {
 	struct intel_encoder base;
 
 	bool is_dual_link;
-	u32 reg;
+	i915_reg_t reg;
 	u32 a3_power;
 
 	struct intel_lvds_connector *attached_connector;
@@ -210,7 +210,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder)
 	struct intel_connector *intel_connector =
 		&lvds_encoder->attached_connector->base;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 ctl_reg, stat_reg;
+	i915_reg_t ctl_reg, stat_reg;
 
 	if (HAS_PCH_SPLIT(dev)) {
 		ctl_reg = PCH_PP_CONTROL;
@@ -235,7 +235,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder)
 	struct drm_device *dev = encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 ctl_reg, stat_reg;
+	i915_reg_t ctl_reg, stat_reg;
 
 	if (HAS_PCH_SPLIT(dev)) {
 		ctl_reg = PCH_PP_CONTROL;
@@ -939,7 +939,7 @@ void intel_lvds_init(struct drm_device *dev)
 	struct drm_display_mode *downclock_mode = NULL;
 	struct edid *edid;
 	struct drm_crtc *crtc;
-	u32 lvds_reg;
+	i915_reg_t lvds_reg;
 	u32 lvds;
 	int pipe;
 	u8 pin;
@@ -1025,7 +1025,7 @@ void intel_lvds_init(struct drm_device *dev)
 			   DRM_MODE_CONNECTOR_LVDS);
 
 	drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 
 	intel_encoder->enable = intel_enable_lvds;
 	intel_encoder->pre_enable = intel_pre_enable_lvds;
@@ -1164,8 +1164,7 @@ out:
 	DRM_DEBUG_KMS("detected %s-link lvds configuration\n",
 		      lvds_encoder->is_dual_link ? "dual" : "single");
 
-	lvds_encoder->a3_power = I915_READ(lvds_encoder->reg) &
-				 LVDS_A3_POWER_MASK;
+	lvds_encoder->a3_power = lvds & LVDS_A3_POWER_MASK;
 
 	lvds_connector->lid_notifier.notifier_call = intel_lid_notify;
 	if (acpi_lid_notifier_register(&lvds_connector->lid_notifier)) {
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index 6d3c6c0a5c62..fed7bea19cc9 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -143,7 +143,7 @@ static bool get_mocs_settings(struct drm_device *dev,
 {
 	bool result = false;
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		table->size  = ARRAY_SIZE(skylake_mocs_table);
 		table->table = skylake_mocs_table;
 		result = true;
@@ -159,11 +159,30 @@ static bool get_mocs_settings(struct drm_device *dev,
 	return result;
 }
 
+static i915_reg_t mocs_register(enum intel_ring_id ring, int index)
+{
+	switch (ring) {
+	case RCS:
+		return GEN9_GFX_MOCS(index);
+	case VCS:
+		return GEN9_MFX0_MOCS(index);
+	case BCS:
+		return GEN9_BLT_MOCS(index);
+	case VECS:
+		return GEN9_VEBOX_MOCS(index);
+	case VCS2:
+		return GEN9_MFX1_MOCS(index);
+	default:
+		MISSING_CASE(ring);
+		return INVALID_MMIO_REG;
+	}
+}
+
 /**
  * emit_mocs_control_table() - emit the mocs control table
  * @req:	Request to set up the MOCS table for.
  * @table:	The values to program into the control regs.
- * @reg_base:	The base for the engine that needs to be programmed.
+ * @ring:	The engine for whom to emit the registers.
  *
  * This function simply emits a MI_LOAD_REGISTER_IMM command for the
  * given table starting at the given address.
@@ -172,7 +191,7 @@ static bool get_mocs_settings(struct drm_device *dev,
  */
 static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 				   const struct drm_i915_mocs_table *table,
-				   u32 reg_base)
+				   enum intel_ring_id ring)
 {
 	struct intel_ringbuffer *ringbuf = req->ringbuf;
 	unsigned int index;
@@ -191,7 +210,7 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 				MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES));
 
 	for (index = 0; index < table->size; index++) {
-		intel_logical_ring_emit(ringbuf, reg_base + index * 4);
+		intel_logical_ring_emit_reg(ringbuf, mocs_register(ring, index));
 		intel_logical_ring_emit(ringbuf,
 					table->table[index].control_value);
 	}
@@ -205,7 +224,7 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 	 * that value to all the used entries.
 	 */
 	for (; index < GEN9_NUM_MOCS_ENTRIES; index++) {
-		intel_logical_ring_emit(ringbuf, reg_base + index * 4);
+		intel_logical_ring_emit_reg(ringbuf, mocs_register(ring, index));
 		intel_logical_ring_emit(ringbuf, table->table[0].control_value);
 	}
 
@@ -253,7 +272,7 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
 		value = (table->table[count].l3cc_value & 0xffff) |
 			((table->table[count + 1].l3cc_value & 0xffff) << 16);
 
-		intel_logical_ring_emit(ringbuf, GEN9_LNCFCMOCS0 + i * 4);
+		intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i));
 		intel_logical_ring_emit(ringbuf, value);
 	}
 
@@ -270,7 +289,7 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
 	 * they are reserved by the hardware.
 	 */
 	for (; i < GEN9_NUM_MOCS_ENTRIES / 2; i++) {
-		intel_logical_ring_emit(ringbuf, GEN9_LNCFCMOCS0 + i * 4);
+		intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i));
 		intel_logical_ring_emit(ringbuf, value);
 
 		value = filler;
@@ -304,26 +323,16 @@ int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req)
 	int ret;
 
 	if (get_mocs_settings(req->ring->dev, &t)) {
-		/* Program the control registers */
-		ret = emit_mocs_control_table(req, &t, GEN9_GFX_MOCS_0);
-		if (ret)
-			return ret;
-
-		ret = emit_mocs_control_table(req, &t, GEN9_MFX0_MOCS_0);
-		if (ret)
-			return ret;
+		struct drm_i915_private *dev_priv = req->i915;
+		struct intel_engine_cs *ring;
+		enum intel_ring_id ring_id;
 
-		ret = emit_mocs_control_table(req, &t, GEN9_MFX1_MOCS_0);
-		if (ret)
-			return ret;
-
-		ret = emit_mocs_control_table(req, &t, GEN9_VEBOX_MOCS_0);
-		if (ret)
-			return ret;
-
-		ret = emit_mocs_control_table(req, &t, GEN9_BLT_MOCS_0);
-		if (ret)
-			return ret;
+		/* Program the control registers */
+		for_each_ring(ring, dev_priv, ring_id) {
+			ret = emit_mocs_control_table(req, &t, ring_id);
+			if (ret)
+				return ret;
+		}
 
 		/* Now program the l3cc registers */
 		ret = emit_mocs_l3cc_table(req, &t);
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 6dc13c02c28e..c15718b4862a 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -26,6 +26,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/dmi.h>
 #include <acpi/video.h>
 
 #include <drm/drmP.h>
@@ -46,6 +47,7 @@
 #define OPREGION_SWSCI_OFFSET  0x200
 #define OPREGION_ASLE_OFFSET   0x300
 #define OPREGION_VBT_OFFSET    0x400
+#define OPREGION_ASLE_EXT_OFFSET	0x1C00
 
 #define OPREGION_SIGNATURE "IntelGraphicsMem"
 #define MBOX_ACPI      (1<<0)
@@ -120,7 +122,16 @@ struct opregion_asle {
 	u64 fdss;
 	u32 fdsp;
 	u32 stat;
-	u8 rsvd[70];
+	u64 rvda;	/* Physical address of raw vbt data */
+	u32 rvds;	/* Size of raw vbt data */
+	u8 rsvd[58];
+} __packed;
+
+/* OpRegion mailbox #5: ASLE ext */
+struct opregion_asle_ext {
+	u32 phed;	/* Panel Header */
+	u8 bddc[256];	/* Panel EDID */
+	u8 rsvd[764];
 } __packed;
 
 /* Driver readiness indicator */
@@ -411,7 +422,7 @@ int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state)
 static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_connector *intel_connector;
+	struct intel_connector *connector;
 	struct opregion_asle *asle = dev_priv->opregion.asle;
 
 	DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
@@ -435,8 +446,8 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 	 * only one).
 	 */
 	DRM_DEBUG_KMS("updating opregion backlight %d/255\n", bclp);
-	list_for_each_entry(intel_connector, &dev->mode_config.connector_list, base.head)
-		intel_panel_set_backlight_acpi(intel_connector, bclp, 255);
+	for_each_intel_connector(dev, connector)
+		intel_panel_set_backlight_acpi(connector, bclp, 255);
 	asle->cblv = DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID;
 
 	drm_modeset_unlock(&dev->mode_config.connection_mutex);
@@ -682,7 +693,7 @@ static void intel_didl_outputs(struct drm_device *dev)
 	}
 
 	if (!acpi_video_bus) {
-		DRM_ERROR("No ACPI video bus found\n");
+		DRM_DEBUG_KMS("No ACPI video bus found\n");
 		return;
 	}
 
@@ -826,6 +837,10 @@ void intel_opregion_fini(struct drm_device *dev)
 
 	/* just clear all opregion memory pointers now */
 	memunmap(opregion->header);
+	if (opregion->rvda) {
+		memunmap(opregion->rvda);
+		opregion->rvda = NULL;
+	}
 	opregion->header = NULL;
 	opregion->acpi = NULL;
 	opregion->swsci = NULL;
@@ -894,6 +909,25 @@ static void swsci_setup(struct drm_device *dev)
 static inline void swsci_setup(struct drm_device *dev) {}
 #endif  /* CONFIG_ACPI */
 
+static int intel_no_opregion_vbt_callback(const struct dmi_system_id *id)
+{
+	DRM_DEBUG_KMS("Falling back to manually reading VBT from "
+		      "VBIOS ROM for %s\n", id->ident);
+	return 1;
+}
+
+static const struct dmi_system_id intel_no_opregion_vbt[] = {
+	{
+		.callback = intel_no_opregion_vbt_callback,
+		.ident = "ThinkCentre A57",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "97027RG"),
+		},
+	},
+	{ }
+};
+
 int intel_opregion_setup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -907,6 +941,7 @@ int intel_opregion_setup(struct drm_device *dev)
 	BUILD_BUG_ON(sizeof(struct opregion_acpi) != 0x100);
 	BUILD_BUG_ON(sizeof(struct opregion_swsci) != 0x100);
 	BUILD_BUG_ON(sizeof(struct opregion_asle) != 0x100);
+	BUILD_BUG_ON(sizeof(struct opregion_asle_ext) != 0x400);
 
 	pci_read_config_dword(dev->pdev, PCI_ASLS, &asls);
 	DRM_DEBUG_DRIVER("graphic opregion physical addr: 0x%x\n", asls);
@@ -931,8 +966,6 @@ int intel_opregion_setup(struct drm_device *dev)
 		goto err_out;
 	}
 	opregion->header = base;
-	opregion->vbt = base + OPREGION_VBT_OFFSET;
-
 	opregion->lid_state = base + ACPI_CLID;
 
 	mboxes = opregion->header->mboxes;
@@ -946,6 +979,7 @@ int intel_opregion_setup(struct drm_device *dev)
 		opregion->swsci = base + OPREGION_SWSCI_OFFSET;
 		swsci_setup(dev);
 	}
+
 	if (mboxes & MBOX_ASLE) {
 		DRM_DEBUG_DRIVER("ASLE supported\n");
 		opregion->asle = base + OPREGION_ASLE_OFFSET;
@@ -953,6 +987,37 @@ int intel_opregion_setup(struct drm_device *dev)
 		opregion->asle->ardy = ASLE_ARDY_NOT_READY;
 	}
 
+	if (mboxes & MBOX_ASLE_EXT)
+		DRM_DEBUG_DRIVER("ASLE extension supported\n");
+
+	if (!dmi_check_system(intel_no_opregion_vbt)) {
+		const void *vbt = NULL;
+		u32 vbt_size = 0;
+
+		if (opregion->header->opregion_ver >= 2 && opregion->asle &&
+		    opregion->asle->rvda && opregion->asle->rvds) {
+			opregion->rvda = memremap(opregion->asle->rvda,
+						  opregion->asle->rvds,
+						  MEMREMAP_WB);
+			vbt = opregion->rvda;
+			vbt_size = opregion->asle->rvds;
+		}
+
+		if (intel_bios_is_valid_vbt(vbt, vbt_size)) {
+			DRM_DEBUG_KMS("Found valid VBT in ACPI OpRegion (RVDA)\n");
+			opregion->vbt = vbt;
+			opregion->vbt_size = vbt_size;
+		} else {
+			vbt = base + OPREGION_VBT_OFFSET;
+			vbt_size = OPREGION_ASLE_EXT_OFFSET - OPREGION_VBT_OFFSET;
+			if (intel_bios_is_valid_vbt(vbt, vbt_size)) {
+				DRM_DEBUG_KMS("Found valid VBT in ACPI OpRegion (Mailbox #4)\n");
+				opregion->vbt = vbt;
+				opregion->vbt_size = vbt_size;
+			}
+		}
+	}
+
 	return 0;
 
 err_out:
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 444542696a2c..76f1980a7541 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -749,7 +749,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret != 0)
 		return ret;
 
-	ret = i915_gem_object_pin_to_display_plane(new_bo, 0, NULL, NULL,
+	ret = i915_gem_object_pin_to_display_plane(new_bo, 0,
 						   &i915_ggtt_view_normal);
 	if (ret != 0)
 		return ret;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index a24df35e11e7..21ee6477bf98 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -461,8 +461,7 @@ static inline u32 scale_hw_to_user(struct intel_connector *connector,
 static u32 intel_panel_compute_brightness(struct intel_connector *connector,
 					  u32 val)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
 	WARN_ON(panel->backlight.max == 0);
@@ -480,45 +479,40 @@ static u32 intel_panel_compute_brightness(struct intel_connector *connector,
 
 static u32 lpt_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 
 	return I915_READ(BLC_PWM_PCH_CTL2) & BACKLIGHT_DUTY_CYCLE_MASK;
 }
 
 static u32 pch_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 
 	return I915_READ(BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
 }
 
 static u32 i9xx_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 val;
 
 	val = I915_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
-	if (INTEL_INFO(dev)->gen < 4)
+	if (INTEL_INFO(dev_priv)->gen < 4)
 		val >>= 1;
 
 	if (panel->backlight.combination_mode) {
 		u8 lbpc;
 
-		pci_read_config_byte(dev->pdev, PCI_LBPC, &lbpc);
+		pci_read_config_byte(dev_priv->dev->pdev, PCI_LBPC, &lbpc);
 		val *= lbpc;
 	}
 
 	return val;
 }
 
-static u32 _vlv_get_backlight(struct drm_device *dev, enum pipe pipe)
+static u32 _vlv_get_backlight(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
 	if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B))
 		return 0;
 
@@ -527,17 +521,16 @@ static u32 _vlv_get_backlight(struct drm_device *dev, enum pipe pipe)
 
 static u32 vlv_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 
-	return _vlv_get_backlight(dev, pipe);
+	return _vlv_get_backlight(dev_priv, pipe);
 }
 
 static u32 bxt_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	return I915_READ(BXT_BLC_PWM_DUTY(panel->backlight.controller));
 }
@@ -553,8 +546,7 @@ static u32 pwm_get_backlight(struct intel_connector *connector)
 
 static u32 intel_panel_get_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 val = 0;
 
@@ -573,16 +565,14 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector)
 
 static void lpt_set_backlight(struct intel_connector *connector, u32 level)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 val = I915_READ(BLC_PWM_PCH_CTL2) & ~BACKLIGHT_DUTY_CYCLE_MASK;
 	I915_WRITE(BLC_PWM_PCH_CTL2, val | level);
 }
 
 static void pch_set_backlight(struct intel_connector *connector, u32 level)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
 	tmp = I915_READ(BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK;
@@ -591,8 +581,7 @@ static void pch_set_backlight(struct intel_connector *connector, u32 level)
 
 static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 tmp, mask;
 
@@ -603,10 +592,10 @@ static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
 
 		lbpc = level * 0xfe / panel->backlight.max + 1;
 		level /= lbpc;
-		pci_write_config_byte(dev->pdev, PCI_LBPC, lbpc);
+		pci_write_config_byte(dev_priv->dev->pdev, PCI_LBPC, lbpc);
 	}
 
-	if (IS_GEN4(dev)) {
+	if (IS_GEN4(dev_priv)) {
 		mask = BACKLIGHT_DUTY_CYCLE_MASK;
 	} else {
 		level <<= 1;
@@ -619,8 +608,7 @@ static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
 
 static void vlv_set_backlight(struct intel_connector *connector, u32 level)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 tmp;
 
@@ -633,8 +621,7 @@ static void vlv_set_backlight(struct intel_connector *connector, u32 level)
 
 static void bxt_set_backlight(struct intel_connector *connector, u32 level)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
 	I915_WRITE(BXT_BLC_PWM_DUTY(panel->backlight.controller), level);
@@ -663,8 +650,7 @@ intel_panel_actually_set_backlight(struct intel_connector *connector, u32 level)
 static void intel_panel_set_backlight(struct intel_connector *connector,
 				      u32 user_level, u32 user_max)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 hw_level;
 
@@ -690,8 +676,7 @@ static void intel_panel_set_backlight(struct intel_connector *connector,
 void intel_panel_set_backlight_acpi(struct intel_connector *connector,
 				    u32 user_level, u32 user_max)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 hw_level;
@@ -726,8 +711,7 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector,
 
 static void lpt_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
 	intel_panel_actually_set_backlight(connector, 0);
@@ -752,8 +736,7 @@ static void lpt_disable_backlight(struct intel_connector *connector)
 
 static void pch_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
 	intel_panel_actually_set_backlight(connector, 0);
@@ -772,8 +755,7 @@ static void i9xx_disable_backlight(struct intel_connector *connector)
 
 static void i965_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 tmp;
 
 	intel_panel_actually_set_backlight(connector, 0);
@@ -784,8 +766,7 @@ static void i965_disable_backlight(struct intel_connector *connector)
 
 static void vlv_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 tmp;
 
@@ -800,8 +781,7 @@ static void vlv_disable_backlight(struct intel_connector *connector)
 
 static void bxt_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 tmp, val;
 
@@ -830,8 +810,7 @@ static void pwm_disable_backlight(struct intel_connector *connector)
 
 void intel_panel_disable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
 	if (!panel->backlight.present)
@@ -843,7 +822,7 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
 	 * backlight. This will leave the backlight on unnecessarily when
 	 * another client is not activated.
 	 */
-	if (dev->switch_power_state == DRM_SWITCH_POWER_CHANGING) {
+	if (dev_priv->dev->switch_power_state == DRM_SWITCH_POWER_CHANGING) {
 		DRM_DEBUG_DRIVER("Skipping backlight disable on vga switch\n");
 		return;
 	}
@@ -860,8 +839,7 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
 
 static void lpt_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 pch_ctl1, pch_ctl2;
 
@@ -893,8 +871,7 @@ static void lpt_enable_backlight(struct intel_connector *connector)
 
 static void pch_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	enum transcoder cpu_transcoder =
@@ -940,8 +917,7 @@ static void pch_enable_backlight(struct intel_connector *connector)
 
 static void i9xx_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 ctl, freq;
 
@@ -958,7 +934,7 @@ static void i9xx_enable_backlight(struct intel_connector *connector)
 	ctl = freq << 17;
 	if (panel->backlight.combination_mode)
 		ctl |= BLM_LEGACY_MODE;
-	if (IS_PINEVIEW(dev) && panel->backlight.active_low_pwm)
+	if (IS_PINEVIEW(dev_priv) && panel->backlight.active_low_pwm)
 		ctl |= BLM_POLARITY_PNV;
 
 	I915_WRITE(BLC_PWM_CTL, ctl);
@@ -972,14 +948,13 @@ static void i9xx_enable_backlight(struct intel_connector *connector)
 	 * 855gm only, but checking for gen2 is safe, as 855gm is the only gen2
 	 * that has backlight.
 	 */
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		I915_WRITE(BLC_HIST_CTL, BLM_HISTOGRAM_ENABLE);
 }
 
 static void i965_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 ctl, ctl2, freq;
@@ -1012,8 +987,7 @@ static void i965_enable_backlight(struct intel_connector *connector)
 
 static void vlv_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 ctl, ctl2;
@@ -1044,8 +1018,7 @@ static void vlv_enable_backlight(struct intel_connector *connector)
 
 static void bxt_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 	u32 pwm_ctl, val;
@@ -1102,8 +1075,7 @@ static void pwm_enable_backlight(struct intel_connector *connector)
 
 void intel_panel_enable_backlight(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	enum pipe pipe = intel_get_pipe_from_connector(connector);
 
@@ -1264,14 +1236,21 @@ static void intel_backlight_device_unregister(struct intel_connector *connector)
 #endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
 
 /*
+ * BXT: PWM clock frequency = 19.2 MHz.
+ */
+static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
+{
+	return KHz(19200) / pwm_freq_hz;
+}
+
+/*
  * SPT: This value represents the period of the PWM stream in clock periods
  * multiplied by 16 (default increment) or 128 (alternate increment selected in
  * SCHICKEN_1 bit 0). PWM clock is 24 MHz.
  */
 static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 mul, clock;
 
 	if (I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY)
@@ -1291,8 +1270,7 @@ static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
  */
 static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	u32 mul, clock;
 
 	if (I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY)
@@ -1300,7 +1278,7 @@ static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 	else
 		mul = 128;
 
-	if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
+	if (HAS_PCH_LPT_H(dev_priv))
 		clock = MHz(135); /* LPT:H */
 	else
 		clock = MHz(24); /* LPT:LP */
@@ -1335,22 +1313,28 @@ static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 	int clock;
 
 	if (IS_PINEVIEW(dev))
-		clock = intel_hrawclk(dev);
+		clock = MHz(intel_hrawclk(dev));
 	else
-		clock = 1000 * dev_priv->display.get_display_clock_speed(dev);
+		clock = 1000 * dev_priv->cdclk_freq;
 
 	return clock / (pwm_freq_hz * 32);
 }
 
 /*
  * Gen4: This value represents the period of the PWM stream in display core
- * clocks multiplied by 128.
+ * clocks ([DevCTG] HRAW clocks) multiplied by 128.
+ *
  */
 static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
 	struct drm_device *dev = connector->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int clock = 1000 * dev_priv->display.get_display_clock_speed(dev);
+	int clock;
+
+	if (IS_G4X(dev_priv))
+		clock = MHz(intel_hrawclk(dev));
+	else
+		clock = 1000 * dev_priv->cdclk_freq;
 
 	return clock / (pwm_freq_hz * 128);
 }
@@ -1379,20 +1363,23 @@ static u32 vlv_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 
 static u32 get_backlight_max_vbt(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u16 pwm_freq_hz = dev_priv->vbt.backlight.pwm_freq_hz;
 	u32 pwm;
 
-	if (!pwm_freq_hz) {
-		DRM_DEBUG_KMS("backlight frequency not specified in VBT\n");
+	if (!panel->backlight.hz_to_pwm) {
+		DRM_DEBUG_KMS("backlight frequency conversion not supported\n");
 		return 0;
 	}
 
-	if (!panel->backlight.hz_to_pwm) {
-		DRM_DEBUG_KMS("backlight frequency setting from VBT currently not supported on this platform\n");
-		return 0;
+	if (pwm_freq_hz) {
+		DRM_DEBUG_KMS("VBT defined backlight frequency %u Hz\n",
+			      pwm_freq_hz);
+	} else {
+		pwm_freq_hz = 200;
+		DRM_DEBUG_KMS("default backlight frequency %u Hz\n",
+			      pwm_freq_hz);
 	}
 
 	pwm = panel->backlight.hz_to_pwm(connector, pwm_freq_hz);
@@ -1401,8 +1388,6 @@ static u32 get_backlight_max_vbt(struct intel_connector *connector)
 		return 0;
 	}
 
-	DRM_DEBUG_KMS("backlight frequency %u Hz from VBT\n", pwm_freq_hz);
-
 	return pwm;
 }
 
@@ -1411,8 +1396,7 @@ static u32 get_backlight_max_vbt(struct intel_connector *connector)
  */
 static u32 get_backlight_min_vbt(struct intel_connector *connector)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	int min;
 
@@ -1437,8 +1421,7 @@ static u32 get_backlight_min_vbt(struct intel_connector *connector)
 
 static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 pch_ctl1, pch_ctl2, val;
 
@@ -1467,8 +1450,7 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus
 
 static int pch_setup_backlight(struct intel_connector *connector, enum pipe unused)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 cpu_ctl2, pch_ctl1, pch_ctl2, val;
 
@@ -1498,17 +1480,16 @@ static int pch_setup_backlight(struct intel_connector *connector, enum pipe unus
 
 static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unused)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 ctl, val;
 
 	ctl = I915_READ(BLC_PWM_CTL);
 
-	if (IS_GEN2(dev) || IS_I915GM(dev) || IS_I945GM(dev))
+	if (IS_GEN2(dev_priv) || IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
 		panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE;
 
-	if (IS_PINEVIEW(dev))
+	if (IS_PINEVIEW(dev_priv))
 		panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV;
 
 	panel->backlight.max = ctl >> 17;
@@ -1536,8 +1517,7 @@ static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unu
 
 static int i965_setup_backlight(struct intel_connector *connector, enum pipe unused)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 ctl, ctl2, val;
 
@@ -1570,8 +1550,7 @@ static int i965_setup_backlight(struct intel_connector *connector, enum pipe unu
 
 static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 ctl, ctl2, val;
 
@@ -1592,7 +1571,7 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe
 
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
-	val = _vlv_get_backlight(dev, pipe);
+	val = _vlv_get_backlight(dev_priv, pipe);
 	panel->backlight.level = intel_panel_compute_brightness(connector, val);
 
 	panel->backlight.enabled = (ctl2 & BLM_PWM_ENABLE) &&
@@ -1604,8 +1583,7 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe
 static int
 bxt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 {
-	struct drm_device *dev = connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 pwm_ctl, val;
 
@@ -1683,8 +1661,7 @@ static int pwm_setup_backlight(struct intel_connector *connector,
 
 int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe)
 {
-	struct drm_device *dev = connector->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 	struct intel_connector *intel_connector = to_intel_connector(connector);
 	struct intel_panel *panel = &intel_connector->panel;
 	int ret;
@@ -1739,35 +1716,35 @@ void intel_panel_destroy_backlight(struct drm_connector *connector)
 static void
 intel_panel_init_backlight_funcs(struct intel_panel *panel)
 {
-	struct intel_connector *intel_connector =
+	struct intel_connector *connector =
 		container_of(panel, struct intel_connector, panel);
-	struct drm_device *dev = intel_connector->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 
-	if (IS_BROXTON(dev)) {
+	if (IS_BROXTON(dev_priv)) {
 		panel->backlight.setup = bxt_setup_backlight;
 		panel->backlight.enable = bxt_enable_backlight;
 		panel->backlight.disable = bxt_disable_backlight;
 		panel->backlight.set = bxt_set_backlight;
 		panel->backlight.get = bxt_get_backlight;
-	} else if (HAS_PCH_LPT(dev) || HAS_PCH_SPT(dev)) {
+		panel->backlight.hz_to_pwm = bxt_hz_to_pwm;
+	} else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_SPT(dev_priv)) {
 		panel->backlight.setup = lpt_setup_backlight;
 		panel->backlight.enable = lpt_enable_backlight;
 		panel->backlight.disable = lpt_disable_backlight;
 		panel->backlight.set = lpt_set_backlight;
 		panel->backlight.get = lpt_get_backlight;
-		if (HAS_PCH_LPT(dev))
+		if (HAS_PCH_LPT(dev_priv))
 			panel->backlight.hz_to_pwm = lpt_hz_to_pwm;
 		else
 			panel->backlight.hz_to_pwm = spt_hz_to_pwm;
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		panel->backlight.setup = pch_setup_backlight;
 		panel->backlight.enable = pch_enable_backlight;
 		panel->backlight.disable = pch_disable_backlight;
 		panel->backlight.set = pch_set_backlight;
 		panel->backlight.get = pch_get_backlight;
 		panel->backlight.hz_to_pwm = pch_hz_to_pwm;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		if (dev_priv->vbt.has_mipi) {
 			panel->backlight.setup = pwm_setup_backlight;
 			panel->backlight.enable = pwm_enable_backlight;
@@ -1782,7 +1759,7 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel)
 			panel->backlight.get = vlv_get_backlight;
 			panel->backlight.hz_to_pwm = vlv_hz_to_pwm;
 		}
-	} else if (IS_GEN4(dev)) {
+	} else if (IS_GEN4(dev_priv)) {
 		panel->backlight.setup = i965_setup_backlight;
 		panel->backlight.enable = i965_enable_backlight;
 		panel->backlight.disable = i965_disable_backlight;
@@ -1828,7 +1805,7 @@ void intel_backlight_register(struct drm_device *dev)
 {
 	struct intel_connector *connector;
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, base.head)
+	for_each_intel_connector(dev, connector)
 		intel_backlight_device_register(connector);
 }
 
@@ -1836,6 +1813,6 @@ void intel_backlight_unregister(struct drm_device *dev)
 {
 	struct intel_connector *connector;
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, base.head)
+	for_each_intel_connector(dev, connector)
 		intel_backlight_device_unregister(connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index f091ad12d694..eb5fa05cf476 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -66,6 +66,14 @@ static void bxt_init_clock_gating(struct drm_device *dev)
 	 */
 	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 		   GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
+
+	/*
+	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
+	 * to stay fully on.
+	 */
+	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
+		I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
+			   PWM1_GATING_DIS | PWM2_GATING_DIS);
 }
 
 static void i915_pineview_get_mem_freq(struct drm_device *dev)
@@ -283,7 +291,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 	struct drm_device *dev = dev_priv->dev;
 	u32 val;
 
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 		POSTING_READ(FW_BLC_SELF_VLV);
 		dev_priv->wm.vlv.cxsr = enable;
@@ -1708,13 +1716,6 @@ static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
 	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
 }
 
-struct skl_pipe_wm_parameters {
-	bool active;
-	uint32_t pipe_htotal;
-	uint32_t pixel_rate; /* in KHz */
-	struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
-};
-
 struct ilk_wm_maximums {
 	uint16_t pri;
 	uint16_t spr;
@@ -1722,13 +1723,6 @@ struct ilk_wm_maximums {
 	uint16_t fbc;
 };
 
-/* used in computing the new watermarks state */
-struct intel_wm_config {
-	unsigned int num_pipes_active;
-	bool sprites_enabled;
-	bool sprites_scaled;
-};
-
 /*
  * For both WM_PIPE and WM_LP.
  * mem_value must be in 0.1us units.
@@ -1979,9 +1973,11 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
 				 const struct intel_crtc *intel_crtc,
 				 int level,
 				 struct intel_crtc_state *cstate,
+				 struct intel_plane_state *pristate,
+				 struct intel_plane_state *sprstate,
+				 struct intel_plane_state *curstate,
 				 struct intel_wm_level *result)
 {
-	struct intel_plane *intel_plane;
 	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
 	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
 	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
@@ -1993,29 +1989,11 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
 		cur_latency *= 5;
 	}
 
-	for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) {
-		struct intel_plane_state *pstate =
-			to_intel_plane_state(intel_plane->base.state);
-
-		switch (intel_plane->base.type) {
-		case DRM_PLANE_TYPE_PRIMARY:
-			result->pri_val = ilk_compute_pri_wm(cstate, pstate,
-							     pri_latency,
-							     level);
-			result->fbc_val = ilk_compute_fbc_wm(cstate, pstate,
-							     result->pri_val);
-			break;
-		case DRM_PLANE_TYPE_OVERLAY:
-			result->spr_val = ilk_compute_spr_wm(cstate, pstate,
-							     spr_latency);
-			break;
-		case DRM_PLANE_TYPE_CURSOR:
-			result->cur_val = ilk_compute_cur_wm(cstate, pstate,
-							     cur_latency);
-			break;
-		}
-	}
-
+	result->pri_val = ilk_compute_pri_wm(cstate, pristate,
+					     pri_latency, level);
+	result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
+	result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
+	result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
 	result->enable = true;
 }
 
@@ -2274,34 +2252,19 @@ static void skl_setup_wm_latency(struct drm_device *dev)
 	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
 }
 
-static void ilk_compute_wm_config(struct drm_device *dev,
-				  struct intel_wm_config *config)
-{
-	struct intel_crtc *intel_crtc;
-
-	/* Compute the currently _active_ config */
-	for_each_intel_crtc(dev, intel_crtc) {
-		const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
-
-		if (!wm->pipe_enabled)
-			continue;
-
-		config->sprites_enabled |= wm->sprites_enabled;
-		config->sprites_scaled |= wm->sprites_scaled;
-		config->num_pipes_active++;
-	}
-}
-
 /* Compute new watermarks for the pipe */
-static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
-				  struct intel_pipe_wm *pipe_wm)
+static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc,
+			       struct drm_atomic_state *state)
 {
-	struct drm_crtc *crtc = cstate->base.crtc;
-	struct drm_device *dev = crtc->dev;
+	struct intel_pipe_wm *pipe_wm;
+	struct drm_device *dev = intel_crtc->base.dev;
 	const struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc_state *cstate = NULL;
 	struct intel_plane *intel_plane;
+	struct drm_plane_state *ps;
+	struct intel_plane_state *pristate = NULL;
 	struct intel_plane_state *sprstate = NULL;
+	struct intel_plane_state *curstate = NULL;
 	int level, max_level = ilk_wm_max_level(dev);
 	/* LP0 watermark maximums depend on this pipe alone */
 	struct intel_wm_config config = {
@@ -2309,11 +2272,24 @@ static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
 	};
 	struct ilk_wm_maximums max;
 
+	cstate = intel_atomic_get_crtc_state(state, intel_crtc);
+	if (IS_ERR(cstate))
+		return PTR_ERR(cstate);
+
+	pipe_wm = &cstate->wm.optimal.ilk;
+
 	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
-		if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) {
-			sprstate = to_intel_plane_state(intel_plane->base.state);
-			break;
-		}
+		ps = drm_atomic_get_plane_state(state,
+						&intel_plane->base);
+		if (IS_ERR(ps))
+			return PTR_ERR(ps);
+
+		if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY)
+			pristate = to_intel_plane_state(ps);
+		else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY)
+			sprstate = to_intel_plane_state(ps);
+		else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
+			curstate = to_intel_plane_state(ps);
 	}
 
 	config.sprites_enabled = sprstate->visible;
@@ -2322,7 +2298,7 @@ static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
 		drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
 
 	pipe_wm->pipe_enabled = cstate->base.active;
-	pipe_wm->sprites_enabled = sprstate->visible;
+	pipe_wm->sprites_enabled = config.sprites_enabled;
 	pipe_wm->sprites_scaled = config.sprites_scaled;
 
 	/* ILK/SNB: LP2+ watermarks only w/o sprites */
@@ -2333,24 +2309,27 @@ static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
 	if (config.sprites_scaled)
 		max_level = 0;
 
-	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]);
+	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
+			     pristate, sprstate, curstate, &pipe_wm->wm[0]);
 
 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
-		pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
+		pipe_wm->linetime = hsw_compute_linetime_wm(dev,
+							    &intel_crtc->base);
 
 	/* LP0 watermarks always use 1/2 DDB partitioning */
 	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
 
 	/* At least LP0 must be valid */
 	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
-		return false;
+		return -EINVAL;
 
 	ilk_compute_wm_reg_maximums(dev, 1, &max);
 
 	for (level = 1; level <= max_level; level++) {
 		struct intel_wm_level wm = {};
 
-		ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm);
+		ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
+				     pristate, sprstate, curstate, &wm);
 
 		/*
 		 * Disable any watermark level that exceeds the
@@ -2363,7 +2342,7 @@ static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
 		pipe_wm->wm[level] = wm;
 	}
 
-	return true;
+	return 0;
 }
 
 /*
@@ -2378,7 +2357,9 @@ static void ilk_merge_wm_level(struct drm_device *dev,
 	ret_wm->enable = true;
 
 	for_each_intel_crtc(dev, intel_crtc) {
-		const struct intel_pipe_wm *active = &intel_crtc->wm.active;
+		const struct intel_crtc_state *cstate =
+			to_intel_crtc_state(intel_crtc->base.state);
+		const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
 		const struct intel_wm_level *wm = &active->wm[level];
 
 		if (!active->pipe_enabled)
@@ -2449,7 +2430,7 @@ static void ilk_wm_merge(struct drm_device *dev,
 	 * enabled sometime later.
 	 */
 	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
-	    intel_fbc_enabled(dev_priv)) {
+	    intel_fbc_is_active(dev_priv)) {
 		for (level = 2; level <= max_level; level++) {
 			struct intel_wm_level *wm = &merged->wm[level];
 
@@ -2526,14 +2507,15 @@ static void ilk_compute_wm_results(struct drm_device *dev,
 
 	/* LP0 register values */
 	for_each_intel_crtc(dev, intel_crtc) {
+		const struct intel_crtc_state *cstate =
+			to_intel_crtc_state(intel_crtc->base.state);
 		enum pipe pipe = intel_crtc->pipe;
-		const struct intel_wm_level *r =
-			&intel_crtc->wm.active.wm[0];
+		const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0];
 
 		if (WARN_ON(!r->enable))
 			continue;
 
-		results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
+		results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime;
 
 		results->wm_pipe[pipe] =
 			(r->pri_val << WM0_PIPE_PLANE_SHIFT) |
@@ -2755,18 +2737,40 @@ static bool ilk_disable_lp_wm(struct drm_device *dev)
 #define SKL_DDB_SIZE		896	/* in blocks */
 #define BXT_DDB_SIZE		512
 
+/*
+ * Return the index of a plane in the SKL DDB and wm result arrays.  Primary
+ * plane is always in slot 0, cursor is always in slot I915_MAX_PLANES-1, and
+ * other universal planes are in indices 1..n.  Note that this may leave unused
+ * indices between the top "sprite" plane and the cursor.
+ */
+static int
+skl_wm_plane_id(const struct intel_plane *plane)
+{
+	switch (plane->base.type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+		return 0;
+	case DRM_PLANE_TYPE_CURSOR:
+		return PLANE_CURSOR;
+	case DRM_PLANE_TYPE_OVERLAY:
+		return plane->plane + 1;
+	default:
+		MISSING_CASE(plane->base.type);
+		return plane->plane;
+	}
+}
+
 static void
 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
-				   struct drm_crtc *for_crtc,
+				   const struct intel_crtc_state *cstate,
 				   const struct intel_wm_config *config,
-				   const struct skl_pipe_wm_parameters *params,
 				   struct skl_ddb_entry *alloc /* out */)
 {
+	struct drm_crtc *for_crtc = cstate->base.crtc;
 	struct drm_crtc *crtc;
 	unsigned int pipe_size, ddb_size;
 	int nth_active_pipe;
 
-	if (!params->active) {
+	if (!cstate->base.active) {
 		alloc->start = 0;
 		alloc->end = 0;
 		return;
@@ -2837,19 +2841,29 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 }
 
 static unsigned int
-skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
+skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
+			     const struct drm_plane_state *pstate,
+			     int y)
 {
+	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
+	struct drm_framebuffer *fb = pstate->fb;
 
 	/* for planar format */
-	if (p->y_bytes_per_pixel) {
+	if (fb->pixel_format == DRM_FORMAT_NV12) {
 		if (y)  /* y-plane data rate */
-			return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel;
+			return intel_crtc->config->pipe_src_w *
+				intel_crtc->config->pipe_src_h *
+				drm_format_plane_cpp(fb->pixel_format, 0);
 		else    /* uv-plane data rate */
-			return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel;
+			return (intel_crtc->config->pipe_src_w/2) *
+				(intel_crtc->config->pipe_src_h/2) *
+				drm_format_plane_cpp(fb->pixel_format, 1);
 	}
 
 	/* for packed formats */
-	return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
+	return intel_crtc->config->pipe_src_w *
+		intel_crtc->config->pipe_src_h *
+		drm_format_plane_cpp(fb->pixel_format, 0);
 }
 
 /*
@@ -2858,46 +2872,55 @@ skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
  *   3 * 4096 * 8192  * 4 < 2^32
  */
 static unsigned int
-skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
-				 const struct skl_pipe_wm_parameters *params)
+skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate)
 {
+	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
+	struct drm_device *dev = intel_crtc->base.dev;
+	const struct intel_plane *intel_plane;
 	unsigned int total_data_rate = 0;
-	int plane;
 
-	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
-		const struct intel_plane_wm_parameters *p;
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		const struct drm_plane_state *pstate = intel_plane->base.state;
 
-		p = &params->plane[plane];
-		if (!p->enabled)
+		if (pstate->fb == NULL)
 			continue;
 
-		total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */
-		if (p->y_bytes_per_pixel) {
-			total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */
-		}
+		if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
+			continue;
+
+		/* packed/uv */
+		total_data_rate += skl_plane_relative_data_rate(cstate,
+								pstate,
+								0);
+
+		if (pstate->fb->pixel_format == DRM_FORMAT_NV12)
+			/* y-plane */
+			total_data_rate += skl_plane_relative_data_rate(cstate,
+									pstate,
+									1);
 	}
 
 	return total_data_rate;
 }
 
 static void
-skl_allocate_pipe_ddb(struct drm_crtc *crtc,
-		      const struct intel_wm_config *config,
-		      const struct skl_pipe_wm_parameters *params,
+skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		      struct skl_ddb_allocation *ddb /* out */)
 {
+	struct drm_crtc *crtc = cstate->base.crtc;
 	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_wm_config *config = &dev_priv->wm.config;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_plane *intel_plane;
 	enum pipe pipe = intel_crtc->pipe;
 	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
 	uint16_t alloc_size, start, cursor_blocks;
 	uint16_t minimum[I915_MAX_PLANES];
 	uint16_t y_minimum[I915_MAX_PLANES];
 	unsigned int total_data_rate;
-	int plane;
 
-	skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
+	skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc);
 	alloc_size = skl_ddb_entry_size(alloc);
 	if (alloc_size == 0) {
 		memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
@@ -2914,17 +2937,20 @@ skl_allocate_pipe_ddb(struct drm_crtc *crtc,
 	alloc->end -= cursor_blocks;
 
 	/* 1. Allocate the mininum required blocks for each active plane */
-	for_each_plane(dev_priv, pipe, plane) {
-		const struct intel_plane_wm_parameters *p;
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		struct drm_plane *plane = &intel_plane->base;
+		struct drm_framebuffer *fb = plane->state->fb;
+		int id = skl_wm_plane_id(intel_plane);
 
-		p = &params->plane[plane];
-		if (!p->enabled)
+		if (fb == NULL)
+			continue;
+		if (plane->type == DRM_PLANE_TYPE_CURSOR)
 			continue;
 
-		minimum[plane] = 8;
-		alloc_size -= minimum[plane];
-		y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0;
-		alloc_size -= y_minimum[plane];
+		minimum[id] = 8;
+		alloc_size -= minimum[id];
+		y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0;
+		alloc_size -= y_minimum[id];
 	}
 
 	/*
@@ -2933,45 +2959,50 @@ skl_allocate_pipe_ddb(struct drm_crtc *crtc,
 	 *
 	 * FIXME: we may not allocate every single block here.
 	 */
-	total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
+	total_data_rate = skl_get_total_relative_data_rate(cstate);
 
 	start = alloc->start;
-	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
-		const struct intel_plane_wm_parameters *p;
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		struct drm_plane *plane = &intel_plane->base;
+		struct drm_plane_state *pstate = intel_plane->base.state;
 		unsigned int data_rate, y_data_rate;
 		uint16_t plane_blocks, y_plane_blocks = 0;
+		int id = skl_wm_plane_id(intel_plane);
 
-		p = &params->plane[plane];
-		if (!p->enabled)
+		if (pstate->fb == NULL)
+			continue;
+		if (plane->type == DRM_PLANE_TYPE_CURSOR)
 			continue;
 
-		data_rate = skl_plane_relative_data_rate(p, 0);
+		data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
 
 		/*
 		 * allocation for (packed formats) or (uv-plane part of planar format):
 		 * promote the expression to 64 bits to avoid overflowing, the
 		 * result is < available as data_rate / total_data_rate < 1
 		 */
-		plane_blocks = minimum[plane];
+		plane_blocks = minimum[id];
 		plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
 					total_data_rate);
 
-		ddb->plane[pipe][plane].start = start;
-		ddb->plane[pipe][plane].end = start + plane_blocks;
+		ddb->plane[pipe][id].start = start;
+		ddb->plane[pipe][id].end = start + plane_blocks;
 
 		start += plane_blocks;
 
 		/*
 		 * allocation for y_plane part of planar format:
 		 */
-		if (p->y_bytes_per_pixel) {
-			y_data_rate = skl_plane_relative_data_rate(p, 1);
-			y_plane_blocks = y_minimum[plane];
+		if (pstate->fb->pixel_format == DRM_FORMAT_NV12) {
+			y_data_rate = skl_plane_relative_data_rate(cstate,
+								   pstate,
+								   1);
+			y_plane_blocks = y_minimum[id];
 			y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
 						total_data_rate);
 
-			ddb->y_plane[pipe][plane].start = start;
-			ddb->y_plane[pipe][plane].end = start + y_plane_blocks;
+			ddb->y_plane[pipe][id].start = start;
+			ddb->y_plane[pipe][id].end = start + y_plane_blocks;
 
 			start += y_plane_blocks;
 		}
@@ -3041,104 +3072,27 @@ static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
-	enum pipe pipe = intel_crtc->pipe;
 
-	if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
-		   sizeof(new_ddb->plane[pipe])))
-		return true;
-
-	if (memcmp(&new_ddb->plane[pipe][PLANE_CURSOR], &cur_ddb->plane[pipe][PLANE_CURSOR],
-		    sizeof(new_ddb->plane[pipe][PLANE_CURSOR])))
+	/*
+	 * If ddb allocation of pipes changed, it may require recalculation of
+	 * watermarks
+	 */
+	if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe)))
 		return true;
 
 	return false;
 }
 
-static void skl_compute_wm_global_parameters(struct drm_device *dev,
-					     struct intel_wm_config *config)
-{
-	struct drm_crtc *crtc;
-	struct drm_plane *plane;
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-		config->num_pipes_active += to_intel_crtc(crtc)->active;
-
-	/* FIXME: I don't think we need those two global parameters on SKL */
-	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
-		struct intel_plane *intel_plane = to_intel_plane(plane);
-
-		config->sprites_enabled |= intel_plane->wm.enabled;
-		config->sprites_scaled |= intel_plane->wm.scaled;
-	}
-}
-
-static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
-					   struct skl_pipe_wm_parameters *p)
-{
-	struct drm_device *dev = crtc->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	enum pipe pipe = intel_crtc->pipe;
-	struct drm_plane *plane;
-	struct drm_framebuffer *fb;
-	int i = 1; /* Index for sprite planes start */
-
-	p->active = intel_crtc->active;
-	if (p->active) {
-		p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
-		p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
-
-		fb = crtc->primary->state->fb;
-		/* For planar: Bpp is for uv plane, y_Bpp is for y plane */
-		if (fb) {
-			p->plane[0].enabled = true;
-			p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
-				drm_format_plane_cpp(fb->pixel_format, 1) :
-				drm_format_plane_cpp(fb->pixel_format, 0);
-			p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
-				drm_format_plane_cpp(fb->pixel_format, 0) : 0;
-			p->plane[0].tiling = fb->modifier[0];
-		} else {
-			p->plane[0].enabled = false;
-			p->plane[0].bytes_per_pixel = 0;
-			p->plane[0].y_bytes_per_pixel = 0;
-			p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
-		}
-		p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
-		p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
-		p->plane[0].rotation = crtc->primary->state->rotation;
-
-		fb = crtc->cursor->state->fb;
-		p->plane[PLANE_CURSOR].y_bytes_per_pixel = 0;
-		if (fb) {
-			p->plane[PLANE_CURSOR].enabled = true;
-			p->plane[PLANE_CURSOR].bytes_per_pixel = fb->bits_per_pixel / 8;
-			p->plane[PLANE_CURSOR].horiz_pixels = crtc->cursor->state->crtc_w;
-			p->plane[PLANE_CURSOR].vert_pixels = crtc->cursor->state->crtc_h;
-		} else {
-			p->plane[PLANE_CURSOR].enabled = false;
-			p->plane[PLANE_CURSOR].bytes_per_pixel = 0;
-			p->plane[PLANE_CURSOR].horiz_pixels = 64;
-			p->plane[PLANE_CURSOR].vert_pixels = 64;
-		}
-	}
-
-	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
-		struct intel_plane *intel_plane = to_intel_plane(plane);
-
-		if (intel_plane->pipe == pipe &&
-			plane->type == DRM_PLANE_TYPE_OVERLAY)
-			p->plane[i++] = intel_plane->wm;
-	}
-}
-
 static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
-				 struct skl_pipe_wm_parameters *p,
-				 struct intel_plane_wm_parameters *p_params,
+				 struct intel_crtc_state *cstate,
+				 struct intel_plane *intel_plane,
 				 uint16_t ddb_allocation,
 				 int level,
 				 uint16_t *out_blocks, /* out */
 				 uint8_t *out_lines /* out */)
 {
+	struct drm_plane *plane = &intel_plane->base;
+	struct drm_framebuffer *fb = plane->state->fb;
 	uint32_t latency = dev_priv->wm.skl_latency[level];
 	uint32_t method1, method2;
 	uint32_t plane_bytes_per_line, plane_blocks_per_line;
@@ -3146,31 +3100,33 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	uint32_t selected_result;
 	uint8_t bytes_per_pixel;
 
-	if (latency == 0 || !p->active || !p_params->enabled)
+	if (latency == 0 || !cstate->base.active || !fb)
 		return false;
 
-	bytes_per_pixel = p_params->y_bytes_per_pixel ?
-		p_params->y_bytes_per_pixel :
-		p_params->bytes_per_pixel;
-	method1 = skl_wm_method1(p->pixel_rate,
+	bytes_per_pixel = drm_format_plane_cpp(fb->pixel_format, 0);
+	method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate),
 				 bytes_per_pixel,
 				 latency);
-	method2 = skl_wm_method2(p->pixel_rate,
-				 p->pipe_htotal,
-				 p_params->horiz_pixels,
+	method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate),
+				 cstate->base.adjusted_mode.crtc_htotal,
+				 cstate->pipe_src_w,
 				 bytes_per_pixel,
-				 p_params->tiling,
+				 fb->modifier[0],
 				 latency);
 
-	plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel;
+	plane_bytes_per_line = cstate->pipe_src_w * bytes_per_pixel;
 	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
 
-	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
-	    p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
+	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
+	    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
 		uint32_t min_scanlines = 4;
 		uint32_t y_tile_minimum;
-		if (intel_rotation_90_or_270(p_params->rotation)) {
-			switch (p_params->bytes_per_pixel) {
+		if (intel_rotation_90_or_270(plane->state->rotation)) {
+			int bpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
+				drm_format_plane_cpp(fb->pixel_format, 1) :
+				drm_format_plane_cpp(fb->pixel_format, 0);
+
+			switch (bpp) {
 			case 1:
 				min_scanlines = 16;
 				break;
@@ -3194,8 +3150,8 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
 
 	if (level >= 1 && level <= 7) {
-		if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
-		    p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
+		if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
+		    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
 			res_lines += 4;
 		else
 			res_blocks++;
@@ -3212,84 +3168,80 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
 				 struct skl_ddb_allocation *ddb,
-				 struct skl_pipe_wm_parameters *p,
-				 enum pipe pipe,
+				 struct intel_crtc_state *cstate,
 				 int level,
-				 int num_planes,
 				 struct skl_wm_level *result)
 {
+	struct drm_device *dev = dev_priv->dev;
+	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
+	struct intel_plane *intel_plane;
 	uint16_t ddb_blocks;
-	int i;
+	enum pipe pipe = intel_crtc->pipe;
+
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		int i = skl_wm_plane_id(intel_plane);
 
-	for (i = 0; i < num_planes; i++) {
 		ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
 
 		result->plane_en[i] = skl_compute_plane_wm(dev_priv,
-						p, &p->plane[i],
+						cstate,
+						intel_plane,
 						ddb_blocks,
 						level,
 						&result->plane_res_b[i],
 						&result->plane_res_l[i]);
 	}
-
-	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][PLANE_CURSOR]);
-	result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p,
-						 &p->plane[PLANE_CURSOR],
-						 ddb_blocks, level,
-						 &result->plane_res_b[PLANE_CURSOR],
-						 &result->plane_res_l[PLANE_CURSOR]);
 }
 
 static uint32_t
-skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
+skl_compute_linetime_wm(struct intel_crtc_state *cstate)
 {
-	if (!to_intel_crtc(crtc)->active)
+	if (!cstate->base.active)
 		return 0;
 
-	if (WARN_ON(p->pixel_rate == 0))
+	if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0))
 		return 0;
 
-	return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
+	return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
+			    skl_pipe_pixel_rate(cstate));
 }
 
-static void skl_compute_transition_wm(struct drm_crtc *crtc,
-				      struct skl_pipe_wm_parameters *params,
+static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
 				      struct skl_wm_level *trans_wm /* out */)
 {
+	struct drm_crtc *crtc = cstate->base.crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int i;
+	struct intel_plane *intel_plane;
 
-	if (!params->active)
+	if (!cstate->base.active)
 		return;
 
 	/* Until we know more, just disable transition WMs */
-	for (i = 0; i < intel_num_planes(intel_crtc); i++)
+	for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
+		int i = skl_wm_plane_id(intel_plane);
+
 		trans_wm->plane_en[i] = false;
-	trans_wm->plane_en[PLANE_CURSOR] = false;
+	}
 }
 
-static void skl_compute_pipe_wm(struct drm_crtc *crtc,
+static void skl_compute_pipe_wm(struct intel_crtc_state *cstate,
 				struct skl_ddb_allocation *ddb,
-				struct skl_pipe_wm_parameters *params,
 				struct skl_pipe_wm *pipe_wm)
 {
-	struct drm_device *dev = crtc->dev;
+	struct drm_device *dev = cstate->base.crtc->dev;
 	const struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int level, max_level = ilk_wm_max_level(dev);
 
 	for (level = 0; level <= max_level; level++) {
-		skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
-				     level, intel_num_planes(intel_crtc),
-				     &pipe_wm->wm[level]);
+		skl_compute_wm_level(dev_priv, ddb, cstate,
+				     level, &pipe_wm->wm[level]);
 	}
-	pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
+	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
 
-	skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
+	skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
 }
 
 static void skl_compute_wm_results(struct drm_device *dev,
-				   struct skl_pipe_wm_parameters *p,
 				   struct skl_pipe_wm *p_wm,
 				   struct skl_wm_values *r,
 				   struct intel_crtc *intel_crtc)
@@ -3346,7 +3298,8 @@ static void skl_compute_wm_results(struct drm_device *dev,
 	r->wm_linetime[pipe] = p_wm->linetime;
 }
 
-static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
+static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
+				i915_reg_t reg,
 				const struct skl_ddb_entry *entry)
 {
 	if (entry->end)
@@ -3361,7 +3314,7 @@ static void skl_write_wm_values(struct drm_i915_private *dev_priv,
 	struct drm_device *dev = dev_priv->dev;
 	struct intel_crtc *crtc;
 
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+	for_each_intel_crtc(dev, crtc) {
 		int i, level, max_level = ilk_wm_max_level(dev);
 		enum pipe pipe = crtc->pipe;
 
@@ -3533,28 +3486,25 @@ static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
 }
 
 static bool skl_update_pipe_wm(struct drm_crtc *crtc,
-			       struct skl_pipe_wm_parameters *params,
-			       struct intel_wm_config *config,
 			       struct skl_ddb_allocation *ddb, /* out */
 			       struct skl_pipe_wm *pipe_wm /* out */)
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
 
-	skl_compute_wm_pipe_parameters(crtc, params);
-	skl_allocate_pipe_ddb(crtc, config, params, ddb);
-	skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
+	skl_allocate_pipe_ddb(cstate, ddb);
+	skl_compute_pipe_wm(cstate, ddb, pipe_wm);
 
-	if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
+	if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
 		return false;
 
-	intel_crtc->wm.skl_active = *pipe_wm;
+	intel_crtc->wm.active.skl = *pipe_wm;
 
 	return true;
 }
 
 static void skl_update_other_pipe_wm(struct drm_device *dev,
 				     struct drm_crtc *crtc,
-				     struct intel_wm_config *config,
 				     struct skl_wm_values *r)
 {
 	struct intel_crtc *intel_crtc;
@@ -3573,9 +3523,7 @@ static void skl_update_other_pipe_wm(struct drm_device *dev,
 	 * Otherwise, because of this_crtc being freshly enabled/disabled, the
 	 * other active pipes need new DDB allocation and WM values.
 	 */
-	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
-				base.head) {
-		struct skl_pipe_wm_parameters params = {};
+	for_each_intel_crtc(dev, intel_crtc) {
 		struct skl_pipe_wm pipe_wm = {};
 		bool wm_changed;
 
@@ -3586,7 +3534,6 @@ static void skl_update_other_pipe_wm(struct drm_device *dev,
 			continue;
 
 		wm_changed = skl_update_pipe_wm(&intel_crtc->base,
-						&params, config,
 						&r->ddb, &pipe_wm);
 
 		/*
@@ -3596,7 +3543,7 @@ static void skl_update_other_pipe_wm(struct drm_device *dev,
 		 */
 		WARN_ON(!wm_changed);
 
-		skl_compute_wm_results(dev, &params, &pipe_wm, r, intel_crtc);
+		skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
 		r->dirty[intel_crtc->pipe] = true;
 	}
 }
@@ -3626,10 +3573,9 @@ static void skl_update_wm(struct drm_crtc *crtc)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct skl_pipe_wm_parameters params = {};
 	struct skl_wm_values *results = &dev_priv->wm.skl_results;
-	struct skl_pipe_wm pipe_wm = {};
-	struct intel_wm_config config = {};
+	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+	struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl;
 
 
 	/* Clear all dirty flags */
@@ -3637,16 +3583,13 @@ static void skl_update_wm(struct drm_crtc *crtc)
 
 	skl_clear_wm(results, intel_crtc->pipe);
 
-	skl_compute_wm_global_parameters(dev, &config);
-
-	if (!skl_update_pipe_wm(crtc, &params, &config,
-				&results->ddb, &pipe_wm))
+	if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
 		return;
 
-	skl_compute_wm_results(dev, &params, &pipe_wm, results, intel_crtc);
+	skl_compute_wm_results(dev, pipe_wm, results, intel_crtc);
 	results->dirty[intel_crtc->pipe] = true;
 
-	skl_update_other_pipe_wm(dev, crtc, &config, results);
+	skl_update_other_pipe_wm(dev, crtc, results);
 	skl_write_wm_values(dev_priv, results);
 	skl_flush_wm_values(dev_priv, results);
 
@@ -3654,71 +3597,23 @@ static void skl_update_wm(struct drm_crtc *crtc)
 	dev_priv->wm.skl_hw = *results;
 }
 
-static void
-skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
-		     uint32_t sprite_width, uint32_t sprite_height,
-		     int pixel_size, bool enabled, bool scaled)
-{
-	struct intel_plane *intel_plane = to_intel_plane(plane);
-	struct drm_framebuffer *fb = plane->state->fb;
-
-	intel_plane->wm.enabled = enabled;
-	intel_plane->wm.scaled = scaled;
-	intel_plane->wm.horiz_pixels = sprite_width;
-	intel_plane->wm.vert_pixels = sprite_height;
-	intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
-
-	/* For planar: Bpp is for UV plane, y_Bpp is for Y plane */
-	intel_plane->wm.bytes_per_pixel =
-		(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
-		drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size;
-	intel_plane->wm.y_bytes_per_pixel =
-		(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
-		drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0;
-
-	/*
-	 * Framebuffer can be NULL on plane disable, but it does not
-	 * matter for watermarks if we assume no tiling in that case.
-	 */
-	if (fb)
-		intel_plane->wm.tiling = fb->modifier[0];
-	intel_plane->wm.rotation = plane->state->rotation;
-
-	skl_update_wm(crtc);
-}
-
-static void ilk_update_wm(struct drm_crtc *crtc)
+static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_device *dev = dev_priv->dev;
+	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
 	struct ilk_wm_maximums max;
+	struct intel_wm_config *config = &dev_priv->wm.config;
 	struct ilk_wm_values results = {};
 	enum intel_ddb_partitioning partitioning;
-	struct intel_pipe_wm pipe_wm = {};
-	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
-	struct intel_wm_config config = {};
-
-	WARN_ON(cstate->base.active != intel_crtc->active);
 
-	intel_compute_pipe_wm(cstate, &pipe_wm);
-
-	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
-		return;
-
-	intel_crtc->wm.active = pipe_wm;
-
-	ilk_compute_wm_config(dev, &config);
-
-	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
-	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
+	ilk_compute_wm_maximums(dev, 1, config, INTEL_DDB_PART_1_2, &max);
+	ilk_wm_merge(dev, config, &max, &lp_wm_1_2);
 
 	/* 5/6 split only in single pipe config on IVB+ */
 	if (INTEL_INFO(dev)->gen >= 7 &&
-	    config.num_pipes_active == 1 && config.sprites_enabled) {
-		ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
-		ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
+	    config->num_pipes_active == 1 && config->sprites_enabled) {
+		ilk_compute_wm_maximums(dev, 1, config, INTEL_DDB_PART_5_6, &max);
+		ilk_wm_merge(dev, config, &max, &lp_wm_5_6);
 
 		best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
 	} else {
@@ -3733,14 +3628,13 @@ static void ilk_update_wm(struct drm_crtc *crtc)
 	ilk_write_wm_values(dev_priv, &results);
 }
 
-static void
-ilk_update_sprite_wm(struct drm_plane *plane,
-		     struct drm_crtc *crtc,
-		     uint32_t sprite_width, uint32_t sprite_height,
-		     int pixel_size, bool enabled, bool scaled)
+static void ilk_update_wm(struct drm_crtc *crtc)
 {
-	struct drm_device *dev = plane->dev;
-	struct intel_plane *intel_plane = to_intel_plane(plane);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+
+	WARN_ON(cstate->base.active != intel_crtc->active);
 
 	/*
 	 * IVB workaround: must disable low power watermarks for at least
@@ -3749,10 +3643,14 @@ ilk_update_sprite_wm(struct drm_plane *plane,
 	 *
 	 * WaCxSRDisabledForSpriteScaling:ivb
 	 */
-	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
-		intel_wait_for_vblank(dev, intel_plane->pipe);
+	if (cstate->disable_lp_wm) {
+		ilk_disable_lp_wm(crtc->dev);
+		intel_wait_for_vblank(crtc->dev, intel_crtc->pipe);
+	}
+
+	intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk;
 
-	ilk_update_wm(crtc);
+	ilk_program_watermarks(dev_priv);
 }
 
 static void skl_pipe_wm_active_state(uint32_t val,
@@ -3805,7 +3703,8 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
+	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+	struct skl_pipe_wm *active = &cstate->wm.optimal.skl;
 	enum pipe pipe = intel_crtc->pipe;
 	int level, i, max_level;
 	uint32_t temp;
@@ -3849,6 +3748,8 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 
 	temp = hw->plane_trans[pipe][PLANE_CURSOR];
 	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
+
+	intel_crtc->wm.active.skl = *active;
 }
 
 void skl_wm_get_hw_state(struct drm_device *dev)
@@ -3868,9 +3769,10 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct ilk_wm_values *hw = &dev_priv->wm.hw;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_pipe_wm *active = &intel_crtc->wm.active;
+	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+	struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
 	enum pipe pipe = intel_crtc->pipe;
-	static const unsigned int wm0_pipe_reg[] = {
+	static const i915_reg_t wm0_pipe_reg[] = {
 		[PIPE_A] = WM0_PIPEA_ILK,
 		[PIPE_B] = WM0_PIPEB_ILK,
 		[PIPE_C] = WM0_PIPEC_IVB,
@@ -3907,6 +3809,8 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 		for (level = 0; level <= max_level; level++)
 			active->wm[level].enable = true;
 	}
+
+	intel_crtc->wm.active.ilk = *active;
 }
 
 #define _FW_WM(value, plane) \
@@ -4132,21 +4036,6 @@ void intel_update_watermarks(struct drm_crtc *crtc)
 		dev_priv->display.update_wm(crtc);
 }
 
-void intel_update_sprite_watermarks(struct drm_plane *plane,
-				    struct drm_crtc *crtc,
-				    uint32_t sprite_width,
-				    uint32_t sprite_height,
-				    int pixel_size,
-				    bool enabled, bool scaled)
-{
-	struct drm_i915_private *dev_priv = plane->dev->dev_private;
-
-	if (dev_priv->display.update_sprite_wm)
-		dev_priv->display.update_sprite_wm(plane, crtc,
-						   sprite_width, sprite_height,
-						   pixel_size, enabled, scaled);
-}
-
 /**
  * Lock protecting IPS related data structures
  */
@@ -4414,7 +4303,7 @@ static void gen6_set_rps(struct drm_device *dev, u8 val)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
-	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		return;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -4515,7 +4404,7 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
 
 	mutex_lock(&dev_priv->rps.hw_lock);
 	if (dev_priv->rps.enabled) {
-		if (IS_VALLEYVIEW(dev))
+		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 			vlv_set_rps_idle(dev_priv);
 		else
 			gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
@@ -4568,7 +4457,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
 
 void intel_set_rps(struct drm_device *dev, u8 val)
 {
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		valleyview_set_rps(dev, val);
 	else
 		gen6_set_rps(dev, val);
@@ -4612,7 +4501,7 @@ static void valleyview_disable_rps(struct drm_device *dev)
 
 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
 {
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
 			mode = GEN6_RC_CTL_RC6_ENABLE;
 		else
@@ -4689,7 +4578,8 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
 	dev_priv->rps.max_freq		= dev_priv->rps.rp0_freq;
 
 	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev) || IS_SKYLAKE(dev)) {
+	if (IS_HASWELL(dev) || IS_BROADWELL(dev) ||
+	    IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		ret = sandybridge_pcode_read(dev_priv,
 					HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
 					&ddcc_status);
@@ -4701,7 +4591,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
 					dev_priv->rps.max_freq);
 	}
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		/* Store the frequency values in 16.66 MHZ units, which is
 		   the natural hardware unit for SKL */
 		dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
@@ -4738,7 +4628,7 @@ static void gen9_enable_rps(struct drm_device *dev)
 	gen6_init_rps_frequencies(dev);
 
 	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
-	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 		return;
 	}
@@ -4806,8 +4696,8 @@ static void gen9_enable_rc6(struct drm_device *dev)
 	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
 			"on" : "off");
 	/* WaRsUseTimeoutMode */
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0)) {
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
 		I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
 			   GEN7_RC_CTL_TO_MODE |
@@ -5055,7 +4945,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
 	/* convert DDR frequency from units of 266.6MHz to bandwidth */
 	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
 
-	if (IS_SKYLAKE(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 		/* Convert GT frequency to 50 HZ units */
 		min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
 		max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
@@ -5073,7 +4963,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
 		int diff = max_gpu_freq - gpu_freq;
 		unsigned int ia_freq = 0, ring_freq = 0;
 
-		if (IS_SKYLAKE(dev)) {
+		if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
 			/*
 			 * ring_freq = 2 * GT. ring_freq is in 100MHz units
 			 * No floor required for ring frequency on SKL.
@@ -5208,7 +5098,17 @@ static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
 
 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
 {
-	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
+	u32 val;
+
+	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
+	/*
+	 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
+	 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
+	 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
+	 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
+	 * to make sure it matches what Punit accepts.
+	 */
+	return max_t(u32, val, 0xc0);
 }
 
 /* Check that the pctx buffer wasn't move under us. */
@@ -6113,7 +6013,17 @@ static void intel_init_emon(struct drm_device *dev)
 
 void intel_init_gt_powersave(struct drm_device *dev)
 {
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
 	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
+	/*
+	 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
+	 * requirement.
+	 */
+	if (!i915.enable_rc6) {
+		DRM_INFO("RC6 disabled, disabling runtime PM support\n");
+		intel_runtime_pm_get(dev_priv);
+	}
 
 	if (IS_CHERRYVIEW(dev))
 		cherryview_init_gt_powersave(dev);
@@ -6123,10 +6033,15 @@ void intel_init_gt_powersave(struct drm_device *dev)
 
 void intel_cleanup_gt_powersave(struct drm_device *dev)
 {
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
 	if (IS_CHERRYVIEW(dev))
 		return;
 	else if (IS_VALLEYVIEW(dev))
 		valleyview_cleanup_gt_powersave(dev);
+
+	if (!i915.enable_rc6)
+		intel_runtime_pm_put(dev_priv);
 }
 
 static void gen6_suspend_rps(struct drm_device *dev)
@@ -6201,7 +6116,7 @@ static void intel_gen6_powersave_work(struct work_struct *work)
 	} else if (INTEL_INFO(dev)->gen >= 9) {
 		gen9_enable_rc6(dev);
 		gen9_enable_rps(dev);
-		if (IS_SKYLAKE(dev))
+		if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
 			__gen6_update_ring_freq(dev);
 	} else if (IS_BROADWELL(dev)) {
 		gen8_enable_rps(dev);
@@ -7057,7 +6972,6 @@ void intel_init_pm(struct drm_device *dev)
 			dev_priv->display.init_clock_gating =
 				bxt_init_clock_gating;
 		dev_priv->display.update_wm = skl_update_wm;
-		dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
 	} else if (HAS_PCH_SPLIT(dev)) {
 		ilk_setup_wm_latency(dev);
 
@@ -7066,7 +6980,7 @@ void intel_init_pm(struct drm_device *dev)
 		    (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
 		     dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
 			dev_priv->display.update_wm = ilk_update_wm;
-			dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
+			dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
 		} else {
 			DRM_DEBUG_KMS("Failed to read display plane latency. "
 				      "Disable CxSR\n");
@@ -7331,4 +7245,6 @@ void intel_pm_setup(struct drm_device *dev)
 	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
 
 	dev_priv->pm.suspended = false;
+	atomic_set(&dev_priv->pm.wakeref_count, 0);
+	atomic_set(&dev_priv->pm.atomic_seq, 0);
 }
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 213581c215b3..9ccff3011523 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -80,7 +80,7 @@ static void intel_psr_write_vsc(struct intel_dp *intel_dp,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc);
 	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
-	u32 ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
+	i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
 	uint32_t *data = (uint32_t *) vsc_psr;
 	unsigned int i;
 
@@ -151,13 +151,31 @@ static void vlv_psr_enable_sink(struct intel_dp *intel_dp)
 			   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
 }
 
+static i915_reg_t psr_aux_ctl_reg(struct drm_i915_private *dev_priv,
+				       enum port port)
+{
+	if (INTEL_INFO(dev_priv)->gen >= 9)
+		return DP_AUX_CH_CTL(port);
+	else
+		return EDP_PSR_AUX_CTL;
+}
+
+static i915_reg_t psr_aux_data_reg(struct drm_i915_private *dev_priv,
+					enum port port, int index)
+{
+	if (INTEL_INFO(dev_priv)->gen >= 9)
+		return DP_AUX_CH_DATA(port, index);
+	else
+		return EDP_PSR_AUX_DATA(index);
+}
+
 static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	uint32_t aux_clock_divider;
-	uint32_t aux_data_reg, aux_ctl_reg;
+	i915_reg_t aux_ctl_reg;
 	int precharge = 0x3;
 	static const uint8_t aux_msg[] = {
 		[0] = DP_AUX_NATIVE_WRITE << 4,
@@ -166,29 +184,24 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 		[3] = 1 - 1,
 		[4] = DP_SET_POWER_D0,
 	};
+	enum port port = dig_port->port;
 	int i;
 
 	BUILD_BUG_ON(sizeof(aux_msg) > 20);
 
 	aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
-			   DP_PSR_ENABLE & ~DP_PSR_MAIN_LINK_ACTIVE);
-
 	/* Enable AUX frame sync at sink */
 	if (dev_priv->psr.aux_frame_sync)
 		drm_dp_dpcd_writeb(&intel_dp->aux,
 				DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF,
 				DP_AUX_FRAME_SYNC_ENABLE);
 
-	aux_data_reg = (INTEL_INFO(dev)->gen >= 9) ?
-				DPA_AUX_CH_DATA1 : EDP_PSR_AUX_DATA1(dev);
-	aux_ctl_reg = (INTEL_INFO(dev)->gen >= 9) ?
-				DPA_AUX_CH_CTL : EDP_PSR_AUX_CTL(dev);
+	aux_ctl_reg = psr_aux_ctl_reg(dev_priv, port);
 
 	/* Setup AUX registers */
 	for (i = 0; i < sizeof(aux_msg); i += 4)
-		I915_WRITE(aux_data_reg + i,
+		I915_WRITE(psr_aux_data_reg(dev_priv, port, i >> 2),
 			   intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
 
 	if (INTEL_INFO(dev)->gen >= 9) {
@@ -254,30 +267,20 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	uint32_t max_sleep_time = 0x1f;
-	/* Lately it was identified that depending on panel idle frame count
-	 * calculated at HW can be off by 1. So let's use what came
-	 * from VBT + 1.
-	 * There are also other cases where panel demands at least 4
-	 * but VBT is not being set. To cover these 2 cases lets use
-	 * at least 5 when VBT isn't set to be on the safest side.
+	/*
+	 * Let's respect VBT in case VBT asks a higher idle_frame value.
+	 * Let's use 6 as the minimum to cover all known cases including
+	 * the off-by-one issue that HW has in some cases. Also there are
+	 * cases where sink should be able to train
+	 * with the 5 or 6 idle patterns.
 	 */
-	uint32_t idle_frames = dev_priv->vbt.psr.idle_frames ?
-			       dev_priv->vbt.psr.idle_frames + 1 : 5;
+	uint32_t idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 	uint32_t val = 0x0;
-	const uint32_t link_entry_time = EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
-
-	if (intel_dp->psr_dpcd[1] & DP_PSR_NO_TRAIN_ON_EXIT) {
-		/* It doesn't mean we shouldn't send TPS patters, so let's
-		   send the minimal TP1 possible and skip TP2. */
-		val |= EDP_PSR_TP1_TIME_100us;
-		val |= EDP_PSR_TP2_TP3_TIME_0us;
-		val |= EDP_PSR_SKIP_AUX_EXIT;
-		/* Sink should be able to train with the 5 or 6 idle patterns */
-		idle_frames += 4;
-	}
 
-	I915_WRITE(EDP_PSR_CTL(dev), val |
-		   (IS_BROADWELL(dev) ? 0 : link_entry_time) |
+	if (IS_HASWELL(dev))
+		val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
+
+	I915_WRITE(EDP_PSR_CTL, val |
 		   max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT |
 		   idle_frames << EDP_PSR_IDLE_FRAME_SHIFT |
 		   EDP_PSR_ENABLE);
@@ -324,8 +327,8 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
 		return false;
 	}
 
-	if (!IS_VALLEYVIEW(dev) && ((dev_priv->vbt.psr.full_link) ||
-				    (dig_port->port != PORT_A))) {
+	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
+	    ((dev_priv->vbt.psr.full_link) || (dig_port->port != PORT_A))) {
 		DRM_DEBUG_KMS("PSR condition failed: Link Standby requested/needed but not supported on this platform\n");
 		return false;
 	}
@@ -340,7 +343,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	WARN_ON(I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE);
+	WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
 	WARN_ON(dev_priv->psr.active);
 	lockdep_assert_held(&dev_priv->psr.lock);
 
@@ -403,9 +406,14 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 				skl_psr_setup_su_vsc(intel_dp);
 		}
 
-		/* Avoid continuous PSR exit by masking memup and hpd */
-		I915_WRITE(EDP_PSR_DEBUG_CTL(dev), EDP_PSR_DEBUG_MASK_MEMUP |
-			   EDP_PSR_DEBUG_MASK_HPD);
+		/*
+		 * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD.
+		 * Also mask LPSP to avoid dependency on other drivers that
+		 * might block runtime_pm besides preventing other hw tracking
+		 * issues now we can rely on frontbuffer tracking.
+		 */
+		I915_WRITE(EDP_PSR_DEBUG_CTL, EDP_PSR_DEBUG_MASK_MEMUP |
+			   EDP_PSR_DEBUG_MASK_HPD | EDP_PSR_DEBUG_MASK_LPSP);
 
 		/* Enable PSR on the panel */
 		hsw_psr_enable_sink(intel_dp);
@@ -427,6 +435,19 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 		vlv_psr_enable_source(intel_dp);
 	}
 
+	/*
+	 * FIXME: Activation should happen immediately since this function
+	 * is just called after pipe is fully trained and enabled.
+	 * However on every platform we face issues when first activation
+	 * follows a modeset so quickly.
+	 *     - On VLV/CHV we get bank screen on first activation
+	 *     - On HSW/BDW we get a recoverable frozen screen until next
+	 *       exit-activate sequence.
+	 */
+	if (INTEL_INFO(dev)->gen < 9)
+		schedule_delayed_work(&dev_priv->psr.work,
+				      msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5));
+
 	dev_priv->psr.enabled = intel_dp;
 unlock:
 	mutex_unlock(&dev_priv->psr.lock);
@@ -466,17 +487,17 @@ static void hsw_psr_disable(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	if (dev_priv->psr.active) {
-		I915_WRITE(EDP_PSR_CTL(dev),
-			   I915_READ(EDP_PSR_CTL(dev)) & ~EDP_PSR_ENABLE);
+		I915_WRITE(EDP_PSR_CTL,
+			   I915_READ(EDP_PSR_CTL) & ~EDP_PSR_ENABLE);
 
 		/* Wait till PSR is idle */
-		if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL(dev)) &
+		if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL) &
 			       EDP_PSR_STATUS_STATE_MASK) == 0, 2000, 10))
 			DRM_ERROR("Timed out waiting for PSR Idle State\n");
 
 		dev_priv->psr.active = false;
 	} else {
-		WARN_ON(I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE);
+		WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
 	}
 }
 
@@ -498,11 +519,15 @@ void intel_psr_disable(struct intel_dp *intel_dp)
 		return;
 	}
 
+	/* Disable PSR on Source */
 	if (HAS_DDI(dev))
 		hsw_psr_disable(intel_dp);
 	else
 		vlv_psr_disable(intel_dp);
 
+	/* Disable PSR on Sink */
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+
 	dev_priv->psr.enabled = NULL;
 	mutex_unlock(&dev_priv->psr.lock);
 
@@ -523,7 +548,7 @@ static void intel_psr_work(struct work_struct *work)
 	 * and be ready for re-enable.
 	 */
 	if (HAS_DDI(dev_priv->dev)) {
-		if (wait_for((I915_READ(EDP_PSR_STATUS_CTL(dev_priv->dev)) &
+		if (wait_for((I915_READ(EDP_PSR_STATUS_CTL) &
 			      EDP_PSR_STATUS_STATE_MASK) == 0, 50)) {
 			DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
 			return;
@@ -566,11 +591,11 @@ static void intel_psr_exit(struct drm_device *dev)
 		return;
 
 	if (HAS_DDI(dev)) {
-		val = I915_READ(EDP_PSR_CTL(dev));
+		val = I915_READ(EDP_PSR_CTL);
 
 		WARN_ON(!(val & EDP_PSR_ENABLE));
 
-		I915_WRITE(EDP_PSR_CTL(dev), val & ~EDP_PSR_ENABLE);
+		I915_WRITE(EDP_PSR_CTL, val & ~EDP_PSR_ENABLE);
 	} else {
 		val = I915_READ(VLV_PSRCTL(pipe));
 
@@ -620,7 +645,7 @@ void intel_psr_single_frame_update(struct drm_device *dev,
 	 * Single frame update is already supported on BDW+ but it requires
 	 * many W/A and it isn't really needed.
 	 */
-	if (!IS_VALLEYVIEW(dev))
+	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
 		return;
 
 	mutex_lock(&dev_priv->psr.lock);
@@ -700,7 +725,6 @@ void intel_psr_flush(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_crtc *crtc;
 	enum pipe pipe;
-	int delay_ms = HAS_DDI(dev) ? 100 : 500;
 
 	mutex_lock(&dev_priv->psr.lock);
 	if (!dev_priv->psr.enabled) {
@@ -714,29 +738,14 @@ void intel_psr_flush(struct drm_device *dev,
 	frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe);
 	dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits;
 
-	if (HAS_DDI(dev)) {
-		/*
-		 * By definition every flush should mean invalidate + flush,
-		 * however on core platforms let's minimize the
-		 * disable/re-enable so we can avoid the invalidate when flip
-		 * originated the flush.
-		 */
-		if (frontbuffer_bits && origin != ORIGIN_FLIP)
-			intel_psr_exit(dev);
-	} else {
-		/*
-		 * On Valleyview and Cherryview we don't use hardware tracking
-		 * so any plane updates or cursor moves don't result in a PSR
-		 * invalidating. Which means we need to manually fake this in
-		 * software for all flushes.
-		 */
-		if (frontbuffer_bits)
-			intel_psr_exit(dev);
-	}
+	/* By definition flush = invalidate + flush */
+	if (frontbuffer_bits)
+		intel_psr_exit(dev);
 
 	if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)
-		schedule_delayed_work(&dev_priv->psr.work,
-				      msecs_to_jiffies(delay_ms));
+		if (!work_busy(&dev_priv->psr.work.work))
+			schedule_delayed_work(&dev_priv->psr.work,
+					      msecs_to_jiffies(100));
 	mutex_unlock(&dev_priv->psr.lock);
 }
 
@@ -751,6 +760,9 @@ void intel_psr_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
+	dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ?
+		HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE;
+
 	INIT_DELAYED_WORK(&dev_priv->psr.work, intel_psr_work);
 	mutex_init(&dev_priv->psr.lock);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 9461a238f5d5..339701d7a9a5 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -27,29 +27,13 @@
  *
  */
 
+#include <linux/log2.h>
 #include <drm/drmP.h>
 #include "i915_drv.h"
 #include <drm/i915_drm.h>
 #include "i915_trace.h"
 #include "intel_drv.h"
 
-bool
-intel_ring_initialized(struct intel_engine_cs *ring)
-{
-	struct drm_device *dev = ring->dev;
-
-	if (!dev)
-		return false;
-
-	if (i915.enable_execlists) {
-		struct intel_context *dctx = ring->default_context;
-		struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf;
-
-		return ringbuf->obj;
-	} else
-		return ring->buffer && ring->buffer->obj;
-}
-
 int __intel_ring_space(int head, int tail, int size)
 {
 	int space = head - tail;
@@ -481,7 +465,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
 {
 	struct drm_device *dev = ring->dev;
 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
-	u32 mmio = 0;
+	i915_reg_t mmio;
 
 	/* The ring status page addresses are no longer next to the rest of
 	 * the ring registers as of gen7.
@@ -524,7 +508,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
 	 * invalidating the TLB?
 	 */
 	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
-		u32 reg = RING_INSTPM(ring->mmio_base);
+		i915_reg_t reg = RING_INSTPM(ring->mmio_base);
 
 		/* ring should be idle before issuing a sync flush*/
 		WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
@@ -733,7 +717,7 @@ static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 
 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 	for (i = 0; i < w->count; i++) {
-		intel_ring_emit(ring, w->reg[i].addr);
+		intel_ring_emit_reg(ring, w->reg[i].addr);
 		intel_ring_emit(ring, w->reg[i].value);
 	}
 	intel_ring_emit(ring, MI_NOOP);
@@ -766,7 +750,8 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 }
 
 static int wa_add(struct drm_i915_private *dev_priv,
-		  const u32 addr, const u32 mask, const u32 val)
+		  i915_reg_t addr,
+		  const u32 mask, const u32 val)
 {
 	const u32 idx = dev_priv->workarounds.count;
 
@@ -924,17 +909,15 @@ static int gen9_init_workarounds(struct intel_engine_cs *ring)
 	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 			  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
-	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) == SKL_REVID_A0 ||
-	    INTEL_REVID(dev) == SKL_REVID_B0)) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
-		/* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
+	/* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 				  GEN9_DG_MIRROR_FIX_ENABLE);
-	}
 
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
-		/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
+	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
 				  GEN9_RHWO_OPTIMIZATION_DISABLE);
 		/*
@@ -944,12 +927,10 @@ static int gen9_init_workarounds(struct intel_engine_cs *ring)
 		 */
 	}
 
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) >= SKL_REVID_C0) ||
-	    IS_BROXTON(dev)) {
-		/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
+	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
+	if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER) || IS_BROXTON(dev))
 		WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 				  GEN9_ENABLE_YV12_BUGFIX);
-	}
 
 	/* Wa4x4STCOptimizationDisable:skl,bxt */
 	/* WaDisablePartialResolveInVc:skl,bxt */
@@ -961,24 +942,22 @@ static int gen9_init_workarounds(struct intel_engine_cs *ring)
 			  GEN9_CCS_TLB_PREFETCH_ENABLE);
 
 	/* WaDisableMaskBasedCammingInRCC:skl,bxt */
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_C0) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0))
+	if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 				  PIXEL_MASK_CAMMING_DISABLE);
 
 	/* WaForceContextSaveRestoreNonCoherent:skl,bxt */
 	tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT;
-	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_F0) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) >= BXT_REVID_B0))
+	if (IS_SKL_REVID(dev, SKL_REVID_F0, SKL_REVID_F0) ||
+	    IS_BXT_REVID(dev, BXT_REVID_B0, REVID_FOREVER))
 		tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
 	WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
 
 	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
-	if (IS_SKYLAKE(dev) ||
-	    (IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_B0)) {
+	if (IS_SKYLAKE(dev) || IS_BXT_REVID(dev, 0, BXT_REVID_B0))
 		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 				  GEN8_SAMPLER_POWER_BYPASS_DIS);
-	}
 
 	/* WaDisableSTUnitPowerOptimization:skl,bxt */
 	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
@@ -1000,7 +979,7 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *ring)
 		 * Only consider slices where one, and only one, subslice has 7
 		 * EUs
 		 */
-		if (hweight8(dev_priv->info.subslice_7eu[i]) != 1)
+		if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
 			continue;
 
 		/*
@@ -1038,11 +1017,7 @@ static int skl_init_workarounds(struct intel_engine_cs *ring)
 	if (ret)
 		return ret;
 
-	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
-		/* WaDisableHDCInvalidation:skl */
-		I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
-			   BDW_DISABLE_HDC_INVALIDATION);
-
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) {
 		/* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
 		I915_WRITE(FF_SLICE_CS_CHICKEN2,
 			   _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
@@ -1051,23 +1026,23 @@ static int skl_init_workarounds(struct intel_engine_cs *ring)
 	/* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
 	 * involving this register should also be added to WA batch as required.
 	 */
-	if (INTEL_REVID(dev) <= SKL_REVID_E0)
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_E0))
 		/* WaDisableLSQCROPERFforOCL:skl */
 		I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
 			   GEN8_LQSC_RO_PERF_DIS);
 
 	/* WaEnableGapsTsvCreditFix:skl */
-	if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
+	if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) {
 		I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
 					   GEN9_GAPS_TSV_CREDIT_DISABLE));
 	}
 
 	/* WaDisablePowerCompilerClockGating:skl */
-	if (INTEL_REVID(dev) == SKL_REVID_B0)
+	if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0))
 		WA_SET_BIT_MASKED(HIZ_CHICKEN,
 				  BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
 
-	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) {
 		/*
 		 *Use Force Non-Coherent whenever executing a 3D context. This
 		 * is a workaround for a possible hang in the unlikely event
@@ -1076,21 +1051,23 @@ static int skl_init_workarounds(struct intel_engine_cs *ring)
 		/* WaForceEnableNonCoherent:skl */
 		WA_SET_BIT_MASKED(HDC_CHICKEN0,
 				  HDC_FORCE_NON_COHERENT);
+
+		/* WaDisableHDCInvalidation:skl */
+		I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+			   BDW_DISABLE_HDC_INVALIDATION);
 	}
 
-	if (INTEL_REVID(dev) == SKL_REVID_C0 ||
-	    INTEL_REVID(dev) == SKL_REVID_D0)
-		/* WaBarrierPerformanceFixDisable:skl */
+	/* WaBarrierPerformanceFixDisable:skl */
+	if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0))
 		WA_SET_BIT_MASKED(HDC_CHICKEN0,
 				  HDC_FENCE_DEST_SLM_DISABLE |
 				  HDC_BARRIER_PERFORMANCE_DISABLE);
 
 	/* WaDisableSbeCacheDispatchPortSharing:skl */
-	if (INTEL_REVID(dev) <= SKL_REVID_F0) {
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_F0))
 		WA_SET_BIT_MASKED(
 			GEN7_HALF_SLICE_CHICKEN1,
 			GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
-	}
 
 	return skl_tune_iz_hashing(ring);
 }
@@ -1107,11 +1084,11 @@ static int bxt_init_workarounds(struct intel_engine_cs *ring)
 
 	/* WaStoreMultiplePTEenable:bxt */
 	/* This is a requirement according to Hardware specification */
-	if (INTEL_REVID(dev) == BXT_REVID_A0)
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 		I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
 
 	/* WaSetClckGatingDisableMedia:bxt */
-	if (INTEL_REVID(dev) == BXT_REVID_A0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 		I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
 					    ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
 	}
@@ -1121,7 +1098,7 @@ static int bxt_init_workarounds(struct intel_engine_cs *ring)
 			  STALL_DOP_GATING_DISABLE);
 
 	/* WaDisableSbeCacheDispatchPortSharing:bxt */
-	if (INTEL_REVID(dev) <= BXT_REVID_B0) {
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
 		WA_SET_BIT_MASKED(
 			GEN7_HALF_SLICE_CHICKEN1,
 			GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
@@ -1319,11 +1296,13 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
 		return ret;
 
 	for_each_ring(useless, dev_priv, i) {
-		u32 mbox_reg = signaller->semaphore.mbox.signal[i];
-		if (mbox_reg != GEN6_NOSYNC) {
+		i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[i];
+
+		if (i915_mmio_reg_valid(mbox_reg)) {
 			u32 seqno = i915_gem_request_get_seqno(signaller_req);
+
 			intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
-			intel_ring_emit(signaller, mbox_reg);
+			intel_ring_emit_reg(signaller, mbox_reg);
 			intel_ring_emit(signaller, seqno);
 		}
 	}
@@ -2004,11 +1983,35 @@ static int init_phys_status_page(struct intel_engine_cs *ring)
 
 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
 {
-	iounmap(ringbuf->virtual_start);
+	if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
+		vunmap(ringbuf->virtual_start);
+	else
+		iounmap(ringbuf->virtual_start);
 	ringbuf->virtual_start = NULL;
 	i915_gem_object_ggtt_unpin(ringbuf->obj);
 }
 
+static u32 *vmap_obj(struct drm_i915_gem_object *obj)
+{
+	struct sg_page_iter sg_iter;
+	struct page **pages;
+	void *addr;
+	int i;
+
+	pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages));
+	if (pages == NULL)
+		return NULL;
+
+	i = 0;
+	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0)
+		pages[i++] = sg_page_iter_page(&sg_iter);
+
+	addr = vmap(pages, i, 0, PAGE_KERNEL);
+	drm_free_large(pages);
+
+	return addr;
+}
+
 int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
 				     struct intel_ringbuffer *ringbuf)
 {
@@ -2016,21 +2019,39 @@ int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
 	struct drm_i915_gem_object *obj = ringbuf->obj;
 	int ret;
 
-	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
-	if (ret)
-		return ret;
+	if (HAS_LLC(dev_priv) && !obj->stolen) {
+		ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, 0);
+		if (ret)
+			return ret;
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, true);
-	if (ret) {
-		i915_gem_object_ggtt_unpin(obj);
-		return ret;
-	}
+		ret = i915_gem_object_set_to_cpu_domain(obj, true);
+		if (ret) {
+			i915_gem_object_ggtt_unpin(obj);
+			return ret;
+		}
 
-	ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
-			i915_gem_obj_ggtt_offset(obj), ringbuf->size);
-	if (ringbuf->virtual_start == NULL) {
-		i915_gem_object_ggtt_unpin(obj);
-		return -EINVAL;
+		ringbuf->virtual_start = vmap_obj(obj);
+		if (ringbuf->virtual_start == NULL) {
+			i915_gem_object_ggtt_unpin(obj);
+			return -ENOMEM;
+		}
+	} else {
+		ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
+		if (ret)
+			return ret;
+
+		ret = i915_gem_object_set_to_gtt_domain(obj, true);
+		if (ret) {
+			i915_gem_object_ggtt_unpin(obj);
+			return ret;
+		}
+
+		ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
+						    i915_gem_obj_ggtt_offset(obj), ringbuf->size);
+		if (ringbuf->virtual_start == NULL) {
+			i915_gem_object_ggtt_unpin(obj);
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -2070,10 +2091,14 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
 	int ret;
 
 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
-	if (ring == NULL)
+	if (ring == NULL) {
+		DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
+				 engine->name);
 		return ERR_PTR(-ENOMEM);
+	}
 
 	ring->ring = engine;
+	list_add(&ring->link, &engine->buffers);
 
 	ring->size = size;
 	/* Workaround an erratum on the i830 which causes a hang if
@@ -2089,8 +2114,9 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
 
 	ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
 	if (ret) {
-		DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
-			  engine->name, ret);
+		DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
+				 engine->name, ret);
+		list_del(&ring->link);
 		kfree(ring);
 		return ERR_PTR(ret);
 	}
@@ -2102,6 +2128,7 @@ void
 intel_ringbuffer_free(struct intel_ringbuffer *ring)
 {
 	intel_destroy_ringbuffer_obj(ring);
+	list_del(&ring->link);
 	kfree(ring);
 }
 
@@ -2117,14 +2144,17 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
 	INIT_LIST_HEAD(&ring->execlist_queue);
+	INIT_LIST_HEAD(&ring->buffers);
 	i915_gem_batch_pool_init(dev, &ring->batch_pool);
 	memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
 
 	init_waitqueue_head(&ring->irq_queue);
 
 	ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
-	if (IS_ERR(ringbuf))
-		return PTR_ERR(ringbuf);
+	if (IS_ERR(ringbuf)) {
+		ret = PTR_ERR(ringbuf);
+		goto error;
+	}
 	ring->buffer = ringbuf;
 
 	if (I915_NEED_GFX_HWS(dev)) {
@@ -2153,8 +2183,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 	return 0;
 
 error:
-	intel_ringbuffer_free(ringbuf);
-	ring->buffer = NULL;
+	intel_cleanup_ring_buffer(ring);
 	return ret;
 }
 
@@ -2167,12 +2196,14 @@ void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
 
 	dev_priv = to_i915(ring->dev);
 
-	intel_stop_ring_buffer(ring);
-	WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
+	if (ring->buffer) {
+		intel_stop_ring_buffer(ring);
+		WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
 
-	intel_unpin_ringbuffer_obj(ring->buffer);
-	intel_ringbuffer_free(ring->buffer);
-	ring->buffer = NULL;
+		intel_unpin_ringbuffer_obj(ring->buffer);
+		intel_ringbuffer_free(ring->buffer);
+		ring->buffer = NULL;
+	}
 
 	if (ring->cleanup)
 		ring->cleanup(ring);
@@ -2181,6 +2212,7 @@ void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
 
 	i915_cmd_parser_fini_ring(ring);
 	i915_gem_batch_pool_fini(&ring->batch_pool);
+	ring->dev = NULL;
 }
 
 static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 49fa41dc0eb6..49574ffe54bc 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -100,6 +100,7 @@ struct intel_ringbuffer {
 	void __iomem *virtual_start;
 
 	struct intel_engine_cs *ring;
+	struct list_head link;
 
 	u32 head;
 	u32 tail;
@@ -157,6 +158,7 @@ struct  intel_engine_cs {
 	u32		mmio_base;
 	struct		drm_device *dev;
 	struct intel_ringbuffer *buffer;
+	struct list_head buffers;
 
 	/*
 	 * A pool of objects to use as shadow copies of client batch buffers
@@ -247,7 +249,7 @@ struct  intel_engine_cs {
 				/* our mbox written by others */
 				u32		wait[I915_NUM_RINGS];
 				/* mboxes this ring signals to */
-				u32		signal[I915_NUM_RINGS];
+				i915_reg_t	signal[I915_NUM_RINGS];
 			} mbox;
 			u64		signal_ggtt[I915_NUM_RINGS];
 		};
@@ -348,7 +350,11 @@ struct  intel_engine_cs {
 	u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
 
-bool intel_ring_initialized(struct intel_engine_cs *ring);
+static inline bool
+intel_ring_initialized(struct intel_engine_cs *ring)
+{
+	return ring->dev != NULL;
+}
 
 static inline unsigned
 intel_ring_flag(struct intel_engine_cs *ring)
@@ -441,6 +447,11 @@ static inline void intel_ring_emit(struct intel_engine_cs *ring,
 	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
 	ringbuf->tail += 4;
 }
+static inline void intel_ring_emit_reg(struct intel_engine_cs *ring,
+				       i915_reg_t reg)
+{
+	intel_ring_emit(ring, i915_mmio_reg_offset(reg));
+}
 static inline void intel_ring_advance(struct intel_engine_cs *ring)
 {
 	struct intel_ringbuffer *ringbuf = ring->buffer;
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 7e23d65c9b24..ddbdbffe829a 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -49,25 +49,88 @@
  * present for a given platform.
  */
 
-#define GEN9_ENABLE_DC5(dev) 0
-#define SKL_ENABLE_DC6(dev) IS_SKYLAKE(dev)
-
 #define for_each_power_well(i, power_well, domain_mask, power_domains)	\
 	for (i = 0;							\
 	     i < (power_domains)->power_well_count &&			\
 		 ((power_well) = &(power_domains)->power_wells[i]);	\
 	     i++)							\
-		if ((power_well)->domains & (domain_mask))
+		for_each_if ((power_well)->domains & (domain_mask))
 
 #define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
 	for (i = (power_domains)->power_well_count - 1;			 \
 	     i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
 	     i--)							 \
-		if ((power_well)->domains & (domain_mask))
+		for_each_if ((power_well)->domains & (domain_mask))
 
 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
 				    int power_well_id);
 
+const char *
+intel_display_power_domain_str(enum intel_display_power_domain domain)
+{
+	switch (domain) {
+	case POWER_DOMAIN_PIPE_A:
+		return "PIPE_A";
+	case POWER_DOMAIN_PIPE_B:
+		return "PIPE_B";
+	case POWER_DOMAIN_PIPE_C:
+		return "PIPE_C";
+	case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
+		return "PIPE_A_PANEL_FITTER";
+	case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
+		return "PIPE_B_PANEL_FITTER";
+	case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
+		return "PIPE_C_PANEL_FITTER";
+	case POWER_DOMAIN_TRANSCODER_A:
+		return "TRANSCODER_A";
+	case POWER_DOMAIN_TRANSCODER_B:
+		return "TRANSCODER_B";
+	case POWER_DOMAIN_TRANSCODER_C:
+		return "TRANSCODER_C";
+	case POWER_DOMAIN_TRANSCODER_EDP:
+		return "TRANSCODER_EDP";
+	case POWER_DOMAIN_PORT_DDI_A_LANES:
+		return "PORT_DDI_A_LANES";
+	case POWER_DOMAIN_PORT_DDI_B_LANES:
+		return "PORT_DDI_B_LANES";
+	case POWER_DOMAIN_PORT_DDI_C_LANES:
+		return "PORT_DDI_C_LANES";
+	case POWER_DOMAIN_PORT_DDI_D_LANES:
+		return "PORT_DDI_D_LANES";
+	case POWER_DOMAIN_PORT_DDI_E_LANES:
+		return "PORT_DDI_E_LANES";
+	case POWER_DOMAIN_PORT_DSI:
+		return "PORT_DSI";
+	case POWER_DOMAIN_PORT_CRT:
+		return "PORT_CRT";
+	case POWER_DOMAIN_PORT_OTHER:
+		return "PORT_OTHER";
+	case POWER_DOMAIN_VGA:
+		return "VGA";
+	case POWER_DOMAIN_AUDIO:
+		return "AUDIO";
+	case POWER_DOMAIN_PLLS:
+		return "PLLS";
+	case POWER_DOMAIN_AUX_A:
+		return "AUX_A";
+	case POWER_DOMAIN_AUX_B:
+		return "AUX_B";
+	case POWER_DOMAIN_AUX_C:
+		return "AUX_C";
+	case POWER_DOMAIN_AUX_D:
+		return "AUX_D";
+	case POWER_DOMAIN_GMBUS:
+		return "GMBUS";
+	case POWER_DOMAIN_INIT:
+		return "INIT";
+	case POWER_DOMAIN_MODESET:
+		return "MODESET";
+	default:
+		MISSING_CASE(domain);
+		return "?";
+	}
+}
+
 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 				    struct i915_power_well *power_well)
 {
@@ -244,12 +307,6 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 		gen8_irq_power_well_post_enable(dev_priv,
 						1 << PIPE_C | 1 << PIPE_B);
 	}
-
-	if (power_well->data == SKL_DISP_PW_1) {
-		if (!dev_priv->power_domains.initializing)
-			intel_prepare_ddi(dev);
-		gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_A);
-	}
 }
 
 static void hsw_set_power_well(struct drm_i915_private *dev_priv,
@@ -292,58 +349,38 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
 	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
 	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
 	BIT(POWER_DOMAIN_AUX_B) |                       \
 	BIT(POWER_DOMAIN_AUX_C) |			\
 	BIT(POWER_DOMAIN_AUX_D) |			\
 	BIT(POWER_DOMAIN_AUDIO) |			\
 	BIT(POWER_DOMAIN_VGA) |				\
 	BIT(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS (		\
-	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	BIT(POWER_DOMAIN_PLLS) |			\
-	BIT(POWER_DOMAIN_PIPE_A) |			\
-	BIT(POWER_DOMAIN_TRANSCODER_EDP) |		\
-	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |		\
-	BIT(POWER_DOMAIN_AUX_A) |			\
-	BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
 	BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_B_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
 	BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_C_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
 	BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_D_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
 	BIT(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_MISC_IO_POWER_DOMAINS (		\
-	SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |		\
-	BIT(POWER_DOMAIN_PLLS) |			\
+#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT(POWER_DOMAIN_MODESET) |			\
+	BIT(POWER_DOMAIN_AUX_A) |			\
 	BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS (		\
-	(POWER_DOMAIN_MASK & ~(SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |	\
+	(POWER_DOMAIN_MASK & ~(				\
 	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	SKL_DISPLAY_DDI_A_E_POWER_DOMAINS |		\
-	SKL_DISPLAY_DDI_B_POWER_DOMAINS |		\
-	SKL_DISPLAY_DDI_C_POWER_DOMAINS |		\
-	SKL_DISPLAY_DDI_D_POWER_DOMAINS |		\
-	SKL_DISPLAY_MISC_IO_POWER_DOMAINS)) |		\
+	SKL_DISPLAY_DC_OFF_POWER_DOMAINS)) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
@@ -354,10 +391,8 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
 	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
 	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
 	BIT(POWER_DOMAIN_AUX_B) |			\
 	BIT(POWER_DOMAIN_AUX_C) |			\
 	BIT(POWER_DOMAIN_AUDIO) |			\
@@ -369,11 +404,15 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 	BIT(POWER_DOMAIN_PIPE_A) |			\
 	BIT(POWER_DOMAIN_TRANSCODER_EDP) |		\
 	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
 	BIT(POWER_DOMAIN_AUX_A) |			\
 	BIT(POWER_DOMAIN_PLLS) |			\
 	BIT(POWER_DOMAIN_INIT))
+#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT(POWER_DOMAIN_MODESET) |			\
+	BIT(POWER_DOMAIN_AUX_A) |			\
+	BIT(POWER_DOMAIN_INIT))
 #define BXT_DISPLAY_ALWAYS_ON_POWER_DOMAINS (		\
 	(POWER_DOMAIN_MASK & ~(BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS |	\
 	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS)) |	\
@@ -417,46 +456,74 @@ static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 	  */
 }
 
-void bxt_enable_dc9(struct drm_i915_private *dev_priv)
+static void gen9_set_dc_state_debugmask_memory_up(
+			struct drm_i915_private *dev_priv)
 {
 	uint32_t val;
 
-	assert_can_enable_dc9(dev_priv);
+	/* The below bit doesn't need to be cleared ever afterwards */
+	val = I915_READ(DC_STATE_DEBUG);
+	if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) {
+		val |= DC_STATE_DEBUG_MASK_MEMORY_UP;
+		I915_WRITE(DC_STATE_DEBUG, val);
+		POSTING_READ(DC_STATE_DEBUG);
+	}
+}
 
-	DRM_DEBUG_KMS("Enabling DC9\n");
+static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
+{
+	uint32_t val;
+	uint32_t mask;
+
+	mask = DC_STATE_EN_UPTO_DC5;
+	if (IS_BROXTON(dev_priv))
+		mask |= DC_STATE_EN_DC9;
+	else
+		mask |= DC_STATE_EN_UPTO_DC6;
+
+	WARN_ON_ONCE(state & ~mask);
+
+	if (i915.enable_dc == 0)
+		state = DC_STATE_DISABLE;
+	else if (i915.enable_dc == 1 && state > DC_STATE_EN_UPTO_DC5)
+		state = DC_STATE_EN_UPTO_DC5;
+
+	if (state & DC_STATE_EN_UPTO_DC5_DC6_MASK)
+		gen9_set_dc_state_debugmask_memory_up(dev_priv);
 
 	val = I915_READ(DC_STATE_EN);
-	val |= DC_STATE_EN_DC9;
+	DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
+		      val & mask, state);
+	val &= ~mask;
+	val |= state;
 	I915_WRITE(DC_STATE_EN, val);
 	POSTING_READ(DC_STATE_EN);
 }
 
-void bxt_disable_dc9(struct drm_i915_private *dev_priv)
+void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 {
-	uint32_t val;
+	assert_can_enable_dc9(dev_priv);
+
+	DRM_DEBUG_KMS("Enabling DC9\n");
 
+	gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
+}
+
+void bxt_disable_dc9(struct drm_i915_private *dev_priv)
+{
 	assert_can_disable_dc9(dev_priv);
 
 	DRM_DEBUG_KMS("Disabling DC9\n");
 
-	val = I915_READ(DC_STATE_EN);
-	val &= ~DC_STATE_EN_DC9;
-	I915_WRITE(DC_STATE_EN, val);
-	POSTING_READ(DC_STATE_EN);
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 }
 
-static void gen9_set_dc_state_debugmask_memory_up(
-			struct drm_i915_private *dev_priv)
+static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 {
-	uint32_t val;
-
-	/* The below bit doesn't need to be cleared ever afterwards */
-	val = I915_READ(DC_STATE_DEBUG);
-	if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) {
-		val |= DC_STATE_DEBUG_MASK_MEMORY_UP;
-		I915_WRITE(DC_STATE_DEBUG, val);
-		POSTING_READ(DC_STATE_DEBUG);
-	}
+	WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
+		  "CSR program storage start is NULL\n");
+	WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
+	WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 }
 
 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
@@ -471,16 +538,13 @@ static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 
 	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 		  "DC5 already programmed to be enabled.\n");
-	WARN_ONCE(dev_priv->pm.suspended,
-		  "DC5 cannot be enabled, if platform is runtime-suspended.\n");
+	assert_rpm_wakelock_held(dev_priv);
 
 	assert_csr_loaded(dev_priv);
 }
 
 static void assert_can_disable_dc5(struct drm_i915_private *dev_priv)
 {
-	bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
-					SKL_DISP_PW_2);
 	/*
 	 * During initialization, the firmware may not be loaded yet.
 	 * We still want to make sure that the DC enabling flag is cleared.
@@ -488,40 +552,16 @@ static void assert_can_disable_dc5(struct drm_i915_private *dev_priv)
 	if (dev_priv->power_domains.initializing)
 		return;
 
-	WARN_ONCE(!pg2_enabled, "PG2 not enabled to disable DC5.\n");
-	WARN_ONCE(dev_priv->pm.suspended,
-		"Disabling of DC5 while platform is runtime-suspended should never happen.\n");
+	assert_rpm_wakelock_held(dev_priv);
 }
 
 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 {
-	uint32_t val;
-
 	assert_can_enable_dc5(dev_priv);
 
 	DRM_DEBUG_KMS("Enabling DC5\n");
 
-	gen9_set_dc_state_debugmask_memory_up(dev_priv);
-
-	val = I915_READ(DC_STATE_EN);
-	val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
-	val |= DC_STATE_EN_UPTO_DC5;
-	I915_WRITE(DC_STATE_EN, val);
-	POSTING_READ(DC_STATE_EN);
-}
-
-static void gen9_disable_dc5(struct drm_i915_private *dev_priv)
-{
-	uint32_t val;
-
-	assert_can_disable_dc5(dev_priv);
-
-	DRM_DEBUG_KMS("Disabling DC5\n");
-
-	val = I915_READ(DC_STATE_EN);
-	val &= ~DC_STATE_EN_UPTO_DC5;
-	I915_WRITE(DC_STATE_EN, val);
-	POSTING_READ(DC_STATE_EN);
+	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 }
 
 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
@@ -547,40 +587,37 @@ static void assert_can_disable_dc6(struct drm_i915_private *dev_priv)
 	if (dev_priv->power_domains.initializing)
 		return;
 
-	assert_csr_loaded(dev_priv);
 	WARN_ONCE(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 		  "DC6 already programmed to be disabled.\n");
 }
 
-static void skl_enable_dc6(struct drm_i915_private *dev_priv)
+static void gen9_disable_dc5_dc6(struct drm_i915_private *dev_priv)
 {
-	uint32_t val;
+	assert_can_disable_dc5(dev_priv);
+
+	if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1)
+		assert_can_disable_dc6(dev_priv);
 
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+}
+
+void skl_enable_dc6(struct drm_i915_private *dev_priv)
+{
 	assert_can_enable_dc6(dev_priv);
 
 	DRM_DEBUG_KMS("Enabling DC6\n");
 
-	gen9_set_dc_state_debugmask_memory_up(dev_priv);
+	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 
-	val = I915_READ(DC_STATE_EN);
-	val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
-	val |= DC_STATE_EN_UPTO_DC6;
-	I915_WRITE(DC_STATE_EN, val);
-	POSTING_READ(DC_STATE_EN);
 }
 
-static void skl_disable_dc6(struct drm_i915_private *dev_priv)
+void skl_disable_dc6(struct drm_i915_private *dev_priv)
 {
-	uint32_t val;
-
 	assert_can_disable_dc6(dev_priv);
 
 	DRM_DEBUG_KMS("Disabling DC6\n");
 
-	val = I915_READ(DC_STATE_EN);
-	val &= ~DC_STATE_EN_UPTO_DC6;
-	I915_WRITE(DC_STATE_EN, val);
-	POSTING_READ(DC_STATE_EN);
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 }
 
 static void skl_set_power_well(struct drm_i915_private *dev_priv,
@@ -630,20 +667,16 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 				!I915_READ(HSW_PWR_WELL_BIOS),
 				"Invalid for power well status to be enabled, unless done by the BIOS, \
 				when request is to disable!\n");
-			if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
-				power_well->data == SKL_DISP_PW_2) {
-				if (SKL_ENABLE_DC6(dev)) {
-					skl_disable_dc6(dev_priv);
-					/*
-					 * DDI buffer programming unnecessary during driver-load/resume
-					 * as it's already done during modeset initialization then.
-					 * It's also invalid here as encoder list is still uninitialized.
-					 */
-					if (!dev_priv->power_domains.initializing)
-						intel_prepare_ddi(dev);
-				} else {
-					gen9_disable_dc5(dev_priv);
-				}
+			if (power_well->data == SKL_DISP_PW_2) {
+				/*
+				 * DDI buffer programming unnecessary during
+				 * driver-load/resume as it's already done
+				 * during modeset initialization then. It's
+				 * also invalid here as encoder list is still
+				 * uninitialized.
+				 */
+				if (!dev_priv->power_domains.initializing)
+					intel_prepare_ddi(dev);
 			}
 			I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
 		}
@@ -658,34 +691,9 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 		}
 	} else {
 		if (enable_requested) {
-			if (IS_SKYLAKE(dev) &&
-				(power_well->data == SKL_DISP_PW_1) &&
-				(intel_csr_load_status_get(dev_priv) == FW_LOADED))
-				DRM_DEBUG_KMS("Not Disabling PW1, dmc will handle\n");
-			else {
-				I915_WRITE(HSW_PWR_WELL_DRIVER,	tmp & ~req_mask);
-				POSTING_READ(HSW_PWR_WELL_DRIVER);
-				DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
-			}
-
-			if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
-				power_well->data == SKL_DISP_PW_2) {
-				enum csr_state state;
-				/* TODO: wait for a completion event or
-				 * similar here instead of busy
-				 * waiting using wait_for function.
-				 */
-				wait_for((state = intel_csr_load_status_get(dev_priv)) !=
-						FW_UNINITIALIZED, 1000);
-				if (state != FW_LOADED)
-					DRM_DEBUG("CSR firmware not ready (%d)\n",
-							state);
-				else
-					if (SKL_ENABLE_DC6(dev))
-						skl_enable_dc6(dev_priv);
-					else
-						gen9_enable_dc5(dev_priv);
-			}
+			I915_WRITE(HSW_PWR_WELL_DRIVER,	tmp & ~req_mask);
+			POSTING_READ(HSW_PWR_WELL_DRIVER);
+			DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
 		}
 	}
 
@@ -760,6 +768,41 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv,
 	skl_set_power_well(dev_priv, power_well, false);
 }
 
+static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
+					   struct i915_power_well *power_well)
+{
+	return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
+}
+
+static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	gen9_disable_dc5_dc6(dev_priv);
+}
+
+static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
+					   struct i915_power_well *power_well)
+{
+	if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1)
+		skl_enable_dc6(dev_priv);
+	else
+		gen9_enable_dc5(dev_priv);
+}
+
+static void gen9_dc_off_power_well_sync_hw(struct drm_i915_private *dev_priv,
+					   struct i915_power_well *power_well)
+{
+	if (power_well->count > 0) {
+		gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+	} else {
+		if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 &&
+		    i915.enable_dc != 1)
+			gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
+		else
+			gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
+	}
+}
+
 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
@@ -974,10 +1017,12 @@ static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_pr
 						 int power_well_id)
 {
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
-	struct i915_power_well *power_well;
 	int i;
 
-	for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
+	for (i = 0; i < power_domains->power_well_count; i++) {
+		struct i915_power_well *power_well;
+
+		power_well = &power_domains->power_wells[i];
 		if (power_well->data == power_well_id)
 			return power_well;
 	}
@@ -1452,13 +1497,17 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 
 	mutex_lock(&power_domains->lock);
 
-	WARN_ON(!power_domains->domain_use_count[domain]);
+	WARN(!power_domains->domain_use_count[domain],
+	     "Use count on domain %s is already zero\n",
+	     intel_display_power_domain_str(domain));
 	power_domains->domain_use_count[domain]--;
 
 	for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
-		WARN_ON(!power_well->count);
+		WARN(!power_well->count,
+		     "Use count on power well %s is already zero",
+		     power_well->name);
 
-		if (!--power_well->count && i915.disable_power_well)
+		if (!--power_well->count)
 			intel_power_well_disable(dev_priv, power_well);
 	}
 
@@ -1470,14 +1519,10 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 #define HSW_ALWAYS_ON_POWER_DOMAINS (			\
 	BIT(POWER_DOMAIN_PIPE_A) |			\
 	BIT(POWER_DOMAIN_TRANSCODER_EDP) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
 	BIT(POWER_DOMAIN_PORT_CRT) |			\
 	BIT(POWER_DOMAIN_PLLS) |			\
 	BIT(POWER_DOMAIN_AUX_A) |			\
@@ -1501,49 +1546,42 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 #define VLV_DISPLAY_POWER_DOMAINS	POWER_DOMAIN_MASK
 
 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
 	BIT(POWER_DOMAIN_PORT_CRT) |		\
 	BIT(POWER_DOMAIN_AUX_B) |		\
 	BIT(POWER_DOMAIN_AUX_C) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (	\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_B) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (	\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_B) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_C) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_C) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_B) |		\
 	BIT(POWER_DOMAIN_AUX_C) |		\
 	BIT(POWER_DOMAIN_INIT))
 
 #define CHV_DPIO_CMN_D_POWER_DOMAINS (		\
-	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |	\
-	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |	\
+	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |	\
 	BIT(POWER_DOMAIN_AUX_D) |		\
 	BIT(POWER_DOMAIN_INIT))
 
@@ -1591,6 +1629,13 @@ static const struct i915_power_well_ops skl_power_well_ops = {
 	.is_enabled = skl_power_well_enabled,
 };
 
+static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
+	.sync_hw = gen9_dc_off_power_well_sync_hw,
+	.enable = gen9_dc_off_power_well_enable,
+	.disable = gen9_dc_off_power_well_disable,
+	.is_enabled = gen9_dc_off_power_well_enabled,
+};
+
 static struct i915_power_well hsw_power_wells[] = {
 	{
 		.name = "always-on",
@@ -1646,6 +1691,7 @@ static struct i915_power_well vlv_power_wells[] = {
 		.always_on = 1,
 		.domains = VLV_ALWAYS_ON_POWER_DOMAINS,
 		.ops = &i9xx_always_on_power_well_ops,
+		.data = PUNIT_POWER_WELL_ALWAYS_ON,
 	},
 	{
 		.name = "display",
@@ -1747,20 +1793,29 @@ static struct i915_power_well skl_power_wells[] = {
 		.always_on = 1,
 		.domains = SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS,
 		.ops = &i9xx_always_on_power_well_ops,
+		.data = SKL_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
-		.domains = SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS,
+		/* Handled by the DMC firmware */
+		.domains = 0,
 		.ops = &skl_power_well_ops,
 		.data = SKL_DISP_PW_1,
 	},
 	{
 		.name = "MISC IO power well",
-		.domains = SKL_DISPLAY_MISC_IO_POWER_DOMAINS,
+		/* Handled by the DMC firmware */
+		.domains = 0,
 		.ops = &skl_power_well_ops,
 		.data = SKL_DISP_PW_MISC_IO,
 	},
 	{
+		.name = "DC off",
+		.domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
+		.ops = &gen9_dc_off_power_well_ops,
+		.data = SKL_DISP_PW_DC_OFF,
+	},
+	{
 		.name = "power well 2",
 		.domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
@@ -1792,6 +1847,34 @@ static struct i915_power_well skl_power_wells[] = {
 	},
 };
 
+void skl_pw1_misc_io_init(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_well *well;
+
+	if (!IS_SKYLAKE(dev_priv))
+		return;
+
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+	intel_power_well_enable(dev_priv, well);
+
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
+	intel_power_well_enable(dev_priv, well);
+}
+
+void skl_pw1_misc_io_fini(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_well *well;
+
+	if (!IS_SKYLAKE(dev_priv))
+		return;
+
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+	intel_power_well_disable(dev_priv, well);
+
+	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
+	intel_power_well_disable(dev_priv, well);
+}
+
 static struct i915_power_well bxt_power_wells[] = {
 	{
 		.name = "always-on",
@@ -1806,11 +1889,17 @@ static struct i915_power_well bxt_power_wells[] = {
 		.data = SKL_DISP_PW_1,
 	},
 	{
+		.name = "DC off",
+		.domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
+		.ops = &gen9_dc_off_power_well_ops,
+		.data = SKL_DISP_PW_DC_OFF,
+	},
+	{
 		.name = "power well 2",
 		.domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
 		.data = SKL_DISP_PW_2,
-	}
+	},
 };
 
 static int
@@ -1820,7 +1909,7 @@ sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
 	if (disable_power_well >= 0)
 		return !!disable_power_well;
 
-	if (IS_SKYLAKE(dev_priv)) {
+	if (IS_BROXTON(dev_priv)) {
 		DRM_DEBUG_KMS("Disabling display power well support\n");
 		return 0;
 	}
@@ -1859,7 +1948,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 		set_power_wells(power_domains, hsw_power_wells);
 	} else if (IS_BROADWELL(dev_priv->dev)) {
 		set_power_wells(power_domains, bdw_power_wells);
-	} else if (IS_SKYLAKE(dev_priv->dev)) {
+	} else if (IS_SKYLAKE(dev_priv->dev) || IS_KABYLAKE(dev_priv->dev)) {
 		set_power_wells(power_domains, skl_power_wells);
 	} else if (IS_BROXTON(dev_priv->dev)) {
 		set_power_wells(power_domains, bxt_power_wells);
@@ -1874,21 +1963,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
-{
-	struct drm_device *dev = dev_priv->dev;
-	struct device *device = &dev->pdev->dev;
-
-	if (!HAS_RUNTIME_PM(dev))
-		return;
-
-	if (!intel_enable_rc6(dev))
-		return;
-
-	/* Make sure we're not suspended first. */
-	pm_runtime_get_sync(device);
-}
-
 /**
  * intel_power_domains_fini - finalizes the power domain structures
  * @dev_priv: i915 device instance
@@ -1899,15 +1973,32 @@ static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
 {
-	intel_runtime_pm_disable(dev_priv);
+	struct device *device = &dev_priv->dev->pdev->dev;
 
-	/* The i915.ko module is still not prepared to be loaded when
+	/*
+	 * The i915.ko module is still not prepared to be loaded when
 	 * the power well is not enabled, so just enable it in case
-	 * we're going to unload/reload. */
+	 * we're going to unload/reload.
+	 * The following also reacquires the RPM reference the core passed
+	 * to the driver during loading, which is dropped in
+	 * intel_runtime_pm_enable(). We have to hand back the control of the
+	 * device to the core with this reference held.
+	 */
 	intel_display_set_init_power(dev_priv, true);
+
+	/* Remove the refcount we took to keep power well support disabled. */
+	if (!i915.disable_power_well)
+		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
+
+	/*
+	 * Remove the refcount we took in intel_runtime_pm_enable() in case
+	 * the platform doesn't support runtime PM.
+	 */
+	if (!HAS_RUNTIME_PM(dev_priv))
+		pm_runtime_put(device);
 }
 
-static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
+static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
 {
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
 	struct i915_power_well *power_well;
@@ -1922,6 +2013,47 @@ static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
 	mutex_unlock(&power_domains->lock);
 }
 
+static void skl_display_core_init(struct drm_i915_private *dev_priv,
+				  bool resume)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	uint32_t val;
+
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+	/* enable PCH reset handshake */
+	val = I915_READ(HSW_NDE_RSTWRN_OPT);
+	I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
+
+	/* enable PG1 and Misc I/O */
+	mutex_lock(&power_domains->lock);
+	skl_pw1_misc_io_init(dev_priv);
+	mutex_unlock(&power_domains->lock);
+
+	if (!resume)
+		return;
+
+	skl_init_cdclk(dev_priv);
+
+	if (dev_priv->csr.dmc_payload)
+		intel_csr_load_program(dev_priv);
+}
+
+static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+
+	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+	skl_uninit_cdclk(dev_priv);
+
+	/* The spec doesn't call for removing the reset handshake flag */
+	/* disable PG1 and Misc I/O */
+	mutex_lock(&power_domains->lock);
+	skl_pw1_misc_io_fini(dev_priv);
+	mutex_unlock(&power_domains->lock);
+}
+
 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 {
 	struct i915_power_well *cmn_bc =
@@ -2044,14 +2176,16 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
  * This function initializes the hardware power domain state and enables all
  * power domains using intel_display_set_init_power().
  */
-void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
+void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 {
 	struct drm_device *dev = dev_priv->dev;
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
 	power_domains->initializing = true;
 
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+		skl_display_core_init(dev_priv, resume);
+	} else if (IS_CHERRYVIEW(dev)) {
 		mutex_lock(&power_domains->lock);
 		chv_phy_control_init(dev_priv);
 		mutex_unlock(&power_domains->lock);
@@ -2063,11 +2197,34 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
 
 	/* For now, we need the power well to be always enabled. */
 	intel_display_set_init_power(dev_priv, true);
-	intel_power_domains_resume(dev_priv);
+	/* Disable power support if the user asked so. */
+	if (!i915.disable_power_well)
+		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+	intel_power_domains_sync_hw(dev_priv);
 	power_domains->initializing = false;
 }
 
 /**
+ * intel_power_domains_suspend - suspend power domain state
+ * @dev_priv: i915 device instance
+ *
+ * This function prepares the hardware power domain state before entering
+ * system suspend. It must be paired with intel_power_domains_init_hw().
+ */
+void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
+{
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+		skl_display_core_uninit(dev_priv);
+
+	/*
+	 * Even if power well support was disabled we still want to disable
+	 * power wells while we are system suspended.
+	 */
+	if (!i915.disable_power_well)
+		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
+}
+
+/**
  * intel_runtime_pm_get - grab a runtime pm reference
  * @dev_priv: i915 device instance
  *
@@ -2082,11 +2239,10 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	struct device *device = &dev->pdev->dev;
 
-	if (!HAS_RUNTIME_PM(dev))
-		return;
-
 	pm_runtime_get_sync(device);
-	WARN(dev_priv->pm.suspended, "Device still suspended.\n");
+
+	atomic_inc(&dev_priv->pm.wakeref_count);
+	assert_rpm_wakelock_held(dev_priv);
 }
 
 /**
@@ -2111,11 +2267,10 @@ void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	struct device *device = &dev->pdev->dev;
 
-	if (!HAS_RUNTIME_PM(dev))
-		return;
-
-	WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n");
+	assert_rpm_wakelock_held(dev_priv);
 	pm_runtime_get_noresume(device);
+
+	atomic_inc(&dev_priv->pm.wakeref_count);
 }
 
 /**
@@ -2131,8 +2286,9 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	struct device *device = &dev->pdev->dev;
 
-	if (!HAS_RUNTIME_PM(dev))
-		return;
+	assert_rpm_wakelock_held(dev_priv);
+	if (atomic_dec_and_test(&dev_priv->pm.wakeref_count))
+		atomic_inc(&dev_priv->pm.atomic_seq);
 
 	pm_runtime_mark_last_busy(device);
 	pm_runtime_put_autosuspend(device);
@@ -2153,22 +2309,27 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	struct device *device = &dev->pdev->dev;
 
-	if (!HAS_RUNTIME_PM(dev))
-		return;
+	pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
+	pm_runtime_mark_last_busy(device);
 
 	/*
-	 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
-	 * requirement.
+	 * Take a permanent reference to disable the RPM functionality and drop
+	 * it only when unloading the driver. Use the low level get/put helpers,
+	 * so the driver's own RPM reference tracking asserts also work on
+	 * platforms without RPM support.
 	 */
-	if (!intel_enable_rc6(dev)) {
-		DRM_INFO("RC6 disabled, disabling runtime PM support\n");
-		return;
+	if (!HAS_RUNTIME_PM(dev)) {
+		pm_runtime_dont_use_autosuspend(device);
+		pm_runtime_get_sync(device);
+	} else {
+		pm_runtime_use_autosuspend(device);
 	}
 
-	pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
-	pm_runtime_mark_last_busy(device);
-	pm_runtime_use_autosuspend(device);
-
+	/*
+	 * The core calls the driver load handler with an RPM reference held.
+	 * We drop that here and will reacquire it during unloading in
+	 * intel_power_domains_fini().
+	 */
 	pm_runtime_put_autosuspend(device);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index c42b636c2087..2e1da060b0e1 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -74,7 +74,7 @@ struct intel_sdvo {
 	struct i2c_adapter ddc;
 
 	/* Register for the SDVO device: SDVOB or SDVOC */
-	uint32_t sdvo_reg;
+	i915_reg_t sdvo_reg;
 
 	/* Active outputs controlled by this SDVO output */
 	uint16_t controlled_output;
@@ -120,8 +120,7 @@ struct intel_sdvo {
 	 */
 	bool is_tv;
 
-	/* On different gens SDVOB is at different places. */
-	bool is_sdvob;
+	enum port port;
 
 	/* This is for current tv format name */
 	int tv_format_index;
@@ -245,7 +244,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 	u32 bval = val, cval = val;
 	int i;
 
-	if (intel_sdvo->sdvo_reg == PCH_SDVOB) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		I915_WRITE(intel_sdvo->sdvo_reg, val);
 		POSTING_READ(intel_sdvo->sdvo_reg);
 		/*
@@ -259,7 +258,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 		return;
 	}
 
-	if (intel_sdvo->sdvo_reg == GEN3_SDVOB)
+	if (intel_sdvo->port == PORT_B)
 		cval = I915_READ(GEN3_SDVOC);
 	else
 		bval = I915_READ(GEN3_SDVOB);
@@ -422,7 +421,7 @@ static const struct _sdvo_cmd_name {
 	SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_HBUF_DATA),
 };
 
-#define SDVO_NAME(svdo) ((svdo)->is_sdvob ? "SDVOB" : "SDVOC")
+#define SDVO_NAME(svdo) ((svdo)->port == PORT_B ? "SDVOB" : "SDVOC")
 
 static void intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd,
 				   const void *args, int args_len)
@@ -1282,14 +1281,10 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 			sdvox |= SDVO_BORDER_ENABLE;
 	} else {
 		sdvox = I915_READ(intel_sdvo->sdvo_reg);
-		switch (intel_sdvo->sdvo_reg) {
-		case GEN3_SDVOB:
+		if (intel_sdvo->port == PORT_B)
 			sdvox &= SDVOB_PRESERVE_MASK;
-			break;
-		case GEN3_SDVOC:
+		else
 			sdvox &= SDVOC_PRESERVE_MASK;
-			break;
-		}
 		sdvox |= (9 << 19) | SDVO_BORDER_ENABLE;
 	}
 
@@ -1464,12 +1459,23 @@ static void intel_disable_sdvo(struct intel_encoder *encoder)
 	 * matching DP port to be enabled on transcoder A.
 	 */
 	if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B) {
+		/*
+		 * We get CPU/PCH FIFO underruns on the other pipe when
+		 * doing the workaround. Sweep them under the rug.
+		 */
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
+
 		temp &= ~SDVO_PIPE_B_SELECT;
 		temp |= SDVO_ENABLE;
 		intel_sdvo_write_sdvox(intel_sdvo, temp);
 
 		temp &= ~SDVO_ENABLE;
 		intel_sdvo_write_sdvox(intel_sdvo, temp);
+
+		intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
 }
 
@@ -2251,7 +2257,7 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv,
 {
 	struct sdvo_device_mapping *mapping;
 
-	if (sdvo->is_sdvob)
+	if (sdvo->port == PORT_B)
 		mapping = &(dev_priv->sdvo_mappings[0]);
 	else
 		mapping = &(dev_priv->sdvo_mappings[1]);
@@ -2269,7 +2275,7 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
 	struct sdvo_device_mapping *mapping;
 	u8 pin;
 
-	if (sdvo->is_sdvob)
+	if (sdvo->port == PORT_B)
 		mapping = &dev_priv->sdvo_mappings[0];
 	else
 		mapping = &dev_priv->sdvo_mappings[1];
@@ -2307,7 +2313,7 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct sdvo_device_mapping *my_mapping, *other_mapping;
 
-	if (sdvo->is_sdvob) {
+	if (sdvo->port == PORT_B) {
 		my_mapping = &dev_priv->sdvo_mappings[0];
 		other_mapping = &dev_priv->sdvo_mappings[1];
 	} else {
@@ -2332,7 +2338,7 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
 	/* No SDVO device info is found for another DVO port,
 	 * so use mapping assumption we had before BIOS parsing.
 	 */
-	if (sdvo->is_sdvob)
+	if (sdvo->port == PORT_B)
 		return 0x70;
 	else
 		return 0x72;
@@ -2939,18 +2945,31 @@ intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo,
 	return i2c_add_adapter(&sdvo->ddc) == 0;
 }
 
-bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
+static void assert_sdvo_port_valid(const struct drm_i915_private *dev_priv,
+				   enum port port)
+{
+	if (HAS_PCH_SPLIT(dev_priv))
+		WARN_ON(port != PORT_B);
+	else
+		WARN_ON(port != PORT_B && port != PORT_C);
+}
+
+bool intel_sdvo_init(struct drm_device *dev,
+		     i915_reg_t sdvo_reg, enum port port)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_encoder *intel_encoder;
 	struct intel_sdvo *intel_sdvo;
 	int i;
+
+	assert_sdvo_port_valid(dev_priv, port);
+
 	intel_sdvo = kzalloc(sizeof(*intel_sdvo), GFP_KERNEL);
 	if (!intel_sdvo)
 		return false;
 
 	intel_sdvo->sdvo_reg = sdvo_reg;
-	intel_sdvo->is_sdvob = is_sdvob;
+	intel_sdvo->port = port;
 	intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, intel_sdvo) >> 1;
 	intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo);
 	if (!intel_sdvo_init_ddc_proxy(intel_sdvo, dev))
@@ -2959,7 +2978,8 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
 	/* encoder type will be decided later */
 	intel_encoder = &intel_sdvo->base;
 	intel_encoder->type = INTEL_OUTPUT_SDVO;
-	drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0);
+	drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0,
+			 NULL);
 
 	/* Read the regs to test if we can talk to the device */
 	for (i = 0; i < 0x40; i++) {
@@ -3000,8 +3020,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
 	 * hotplug lines.
 	 */
 	if (intel_sdvo->hotplug_active) {
-		intel_encoder->hpd_pin =
-			intel_sdvo->is_sdvob ?  HPD_SDVO_B : HPD_SDVO_C;
+		if (intel_sdvo->port == PORT_B)
+			intel_encoder->hpd_pin = HPD_SDVO_B;
+		else
+			intel_encoder->hpd_pin = HPD_SDVO_C;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 56dc132e8e20..4ff7a1f4183e 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -192,10 +192,9 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc,
 	const int pipe = intel_plane->pipe;
 	const int plane = intel_plane->plane + 1;
 	u32 plane_ctl, stride_div, stride;
-	int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
 	const struct drm_intel_sprite_colorkey *key =
 		&to_intel_plane_state(drm_plane->state)->ckey;
-	unsigned long surf_addr;
+	u32 surf_addr;
 	u32 tile_height, plane_offset, plane_size;
 	unsigned int rotation;
 	int x_offset, y_offset;
@@ -212,10 +211,6 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc,
 	rotation = drm_plane->state->rotation;
 	plane_ctl |= skl_plane_ctl_rotation(rotation);
 
-	intel_update_sprite_watermarks(drm_plane, crtc, src_w, src_h,
-				       pixel_size, true,
-				       src_w != crtc_w || src_h != crtc_h);
-
 	stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
 					       fb->pixel_format);
 
@@ -297,8 +292,6 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 
 	I915_WRITE(PLANE_SURF(pipe, plane), 0);
 	POSTING_READ(PLANE_SURF(pipe, plane));
-
-	intel_update_sprite_watermarks(dplane, crtc, 0, 0, 0, false, false);
 }
 
 static void
@@ -541,10 +534,6 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
 		sprctl |= SPRITE_PIPE_CSC_ENABLE;
 
-	intel_update_sprite_watermarks(plane, crtc, src_w, src_h, pixel_size,
-				       true,
-				       src_w != crtc_w || src_h != crtc_h);
-
 	/* Sizes are 0 based */
 	src_w--;
 	src_h--;
@@ -678,10 +667,6 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	if (IS_GEN6(dev))
 		dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */
 
-	intel_update_sprite_watermarks(plane, crtc, src_w, src_h,
-				       pixel_size, true,
-				       src_w != crtc_w || src_h != crtc_h);
-
 	/* Sizes are 0 based */
 	src_w--;
 	src_h--;
@@ -832,8 +817,8 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
 		if (hscale < 0) {
 			DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n");
-			drm_rect_debug_print(src, true);
-			drm_rect_debug_print(dst, false);
+			drm_rect_debug_print("src: ", src, true);
+			drm_rect_debug_print("dst: ", dst, false);
 
 			return hscale;
 		}
@@ -841,8 +826,8 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
 		if (vscale < 0) {
 			DRM_DEBUG_KMS("Vertical scaling factor out of limits\n");
-			drm_rect_debug_print(src, true);
-			drm_rect_debug_print(dst, false);
+			drm_rect_debug_print("src: ", src, true);
+			drm_rect_debug_print("dst: ", dst, false);
 
 			return vscale;
 		}
@@ -938,9 +923,6 @@ intel_commit_sprite_plane(struct drm_plane *plane,
 
 	crtc = crtc ? crtc : plane->crtc;
 
-	if (!crtc->state->active)
-		return;
-
 	if (state->visible) {
 		intel_plane->update_plane(plane, crtc, fb,
 					  state->dst.x1, state->dst.y1,
@@ -969,7 +951,7 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 	if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
 		return -EINVAL;
 
-	if (IS_VALLEYVIEW(dev) &&
+	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
 	    set->flags & I915_SET_COLORKEY_DESTINATION)
 		return -EINVAL;
 
@@ -1104,7 +1086,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 			intel_plane->max_downscale = 1;
 		}
 
-		if (IS_VALLEYVIEW(dev)) {
+		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 			intel_plane->update_plane = vlv_update_plane;
 			intel_plane->disable_plane = vlv_disable_plane;
 
@@ -1141,7 +1123,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 	ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
 				       &intel_plane_funcs,
 				       plane_formats, num_plane_formats,
-				       DRM_PLANE_TYPE_OVERLAY);
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret) {
 		kfree(intel_plane);
 		goto out;
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 6bea78944cd6..948cbff6c62e 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1645,7 +1645,7 @@ intel_tv_init(struct drm_device *dev)
 			   DRM_MODE_CONNECTOR_SVIDEO);
 
 	drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs,
-			 DRM_MODE_ENCODER_TVDAC);
+			 DRM_MODE_ENCODER_TVDAC, NULL);
 
 	intel_encoder->compute_config = intel_tv_compute_config;
 	intel_encoder->get_config = intel_tv_get_config;
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 43cba129a0c0..277e60ae0e47 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -29,19 +29,7 @@
 
 #define FORCEWAKE_ACK_TIMEOUT_MS 50
 
-#define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
-#define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
-
-#define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
-#define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
-
-#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
-#define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
-
-#define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
-#define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
-
-#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
+#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
 
 static const char * const forcewake_domain_names[] = {
 	"render",
@@ -62,17 +50,10 @@ intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 	return "unknown";
 }
 
-static void
-assert_device_not_suspended(struct drm_i915_private *dev_priv)
-{
-	WARN_ONCE(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
-		  "Device suspended\n");
-}
-
 static inline void
 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 {
-	WARN_ON(d->reg_set == 0);
+	WARN_ON(!i915_mmio_reg_valid(d->reg_set));
 	__raw_i915_write32(d->i915, d->reg_set, d->val_reset);
 }
 
@@ -118,7 +99,7 @@ static inline void
 fw_domain_posting_read(const struct intel_uncore_forcewake_domain *d)
 {
 	/* something from same cacheline, but not from the set register */
-	if (d->reg_post)
+	if (i915_mmio_reg_valid(d->reg_post))
 		__raw_posting_read(d->i915, d->reg_post);
 }
 
@@ -248,7 +229,7 @@ static void intel_uncore_fw_release_timer(unsigned long arg)
 	struct intel_uncore_forcewake_domain *domain = (void *)arg;
 	unsigned long irqflags;
 
-	assert_device_not_suspended(domain->i915);
+	assert_rpm_device_not_suspended(domain->i915);
 
 	spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
 	if (WARN_ON(domain->wake_count == 0))
@@ -423,7 +404,7 @@ void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 	if (!dev_priv->uncore.funcs.force_wake_get)
 		return;
 
-	WARN_ON(dev_priv->pm.suspended);
+	assert_rpm_wakelock_held(dev_priv);
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 	__intel_uncore_forcewake_get(dev_priv, fw_domains);
@@ -525,8 +506,7 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 }
 
 /* We give fast paths for the really cool registers */
-#define NEEDS_FORCE_WAKE(reg) \
-	 ((reg) < 0x40000 && (reg) != FORCEWAKE)
+#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
 
 #define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
 
@@ -589,7 +569,7 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 	REG_RANGE((reg), 0x9400, 0x9800)
 
 #define FORCEWAKE_GEN9_BLITTER_RANGE_OFFSET(reg) \
-	((reg) < 0x40000 &&\
+	((reg) < 0x40000 && \
 	 !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) && \
 	 !FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) && \
 	 !FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) && \
@@ -605,8 +585,8 @@ ilk_dummy_write(struct drm_i915_private *dev_priv)
 }
 
 static void
-hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read,
-			bool before)
+hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv,
+			i915_reg_t reg, bool read, bool before)
 {
 	const char *op = read ? "reading" : "writing to";
 	const char *when = before ? "before" : "after";
@@ -616,7 +596,7 @@ hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read,
 
 	if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 		WARN(1, "Unclaimed register detected %s %s register 0x%x\n",
-		     when, op, reg);
+		     when, op, i915_mmio_reg_offset(reg));
 		__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 		i915.mmio_debug--; /* Only report the first N failures */
 	}
@@ -641,7 +621,7 @@ hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv)
 
 #define GEN2_READ_HEADER(x) \
 	u##x val = 0; \
-	assert_device_not_suspended(dev_priv);
+	assert_rpm_wakelock_held(dev_priv);
 
 #define GEN2_READ_FOOTER \
 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
@@ -649,7 +629,7 @@ hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv)
 
 #define __gen2_read(x) \
 static u##x \
-gen2_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	GEN2_READ_HEADER(x); \
 	val = __raw_i915_read##x(dev_priv, reg); \
 	GEN2_READ_FOOTER; \
@@ -657,7 +637,7 @@ gen2_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 
 #define __gen5_read(x) \
 static u##x \
-gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	GEN2_READ_HEADER(x); \
 	ilk_dummy_write(dev_priv); \
 	val = __raw_i915_read##x(dev_priv, reg); \
@@ -680,9 +660,10 @@ __gen2_read(64)
 #undef GEN2_READ_HEADER
 
 #define GEN6_READ_HEADER(x) \
+	u32 offset = i915_mmio_reg_offset(reg); \
 	unsigned long irqflags; \
 	u##x val = 0; \
-	assert_device_not_suspended(dev_priv); \
+	assert_rpm_wakelock_held(dev_priv); \
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 
 #define GEN6_READ_FOOTER \
@@ -714,20 +695,12 @@ static inline void __force_wake_get(struct drm_i915_private *dev_priv,
 		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
 }
 
-#define __vgpu_read(x) \
-static u##x \
-vgpu_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
-	GEN6_READ_HEADER(x); \
-	val = __raw_i915_read##x(dev_priv, reg); \
-	GEN6_READ_FOOTER; \
-}
-
 #define __gen6_read(x) \
 static u##x \
-gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	GEN6_READ_HEADER(x); \
 	hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
-	if (NEEDS_FORCE_WAKE(reg)) \
+	if (NEEDS_FORCE_WAKE(offset)) \
 		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
 	val = __raw_i915_read##x(dev_priv, reg); \
 	hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
@@ -736,47 +709,56 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 
 #define __vlv_read(x) \
 static u##x \
-vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+vlv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+	enum forcewake_domains fw_engine = 0; \
 	GEN6_READ_HEADER(x); \
-	if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \
-		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
-	else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \
-		__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
+	if (!NEEDS_FORCE_WAKE(offset)) \
+		fw_engine = 0; \
+	else if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_RENDER; \
+	else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_MEDIA; \
+	if (fw_engine) \
+		__force_wake_get(dev_priv, fw_engine); \
 	val = __raw_i915_read##x(dev_priv, reg); \
 	GEN6_READ_FOOTER; \
 }
 
 #define __chv_read(x) \
 static u##x \
-chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+chv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+	enum forcewake_domains fw_engine = 0; \
 	GEN6_READ_HEADER(x); \
-	if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
-		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
-	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
-		__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
-	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
-		__force_wake_get(dev_priv, \
-				 FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
+	if (!NEEDS_FORCE_WAKE(offset)) \
+		fw_engine = 0; \
+	else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_RENDER; \
+	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_MEDIA; \
+	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+	if (fw_engine) \
+		__force_wake_get(dev_priv, fw_engine); \
 	val = __raw_i915_read##x(dev_priv, reg); \
 	GEN6_READ_FOOTER; \
 }
 
 #define SKL_NEEDS_FORCE_WAKE(reg) \
-	 ((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg))
+	((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg))
 
 #define __gen9_read(x) \
 static u##x \
-gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+gen9_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_READ_HEADER(x); \
 	hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
-	if (!SKL_NEEDS_FORCE_WAKE(reg)) \
+	if (!SKL_NEEDS_FORCE_WAKE(offset)) \
 		fw_engine = 0; \
-	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
 	else \
 		fw_engine = FORCEWAKE_BLITTER; \
@@ -787,10 +769,6 @@ gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 	GEN6_READ_FOOTER; \
 }
 
-__vgpu_read(8)
-__vgpu_read(16)
-__vgpu_read(32)
-__vgpu_read(64)
 __gen9_read(8)
 __gen9_read(16)
 __gen9_read(32)
@@ -812,19 +790,46 @@ __gen6_read(64)
 #undef __chv_read
 #undef __vlv_read
 #undef __gen6_read
-#undef __vgpu_read
 #undef GEN6_READ_FOOTER
 #undef GEN6_READ_HEADER
 
+#define VGPU_READ_HEADER(x) \
+	unsigned long irqflags; \
+	u##x val = 0; \
+	assert_rpm_device_not_suspended(dev_priv); \
+	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
+
+#define VGPU_READ_FOOTER \
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
+	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
+	return val
+
+#define __vgpu_read(x) \
+static u##x \
+vgpu_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+	VGPU_READ_HEADER(x); \
+	val = __raw_i915_read##x(dev_priv, reg); \
+	VGPU_READ_FOOTER; \
+}
+
+__vgpu_read(8)
+__vgpu_read(16)
+__vgpu_read(32)
+__vgpu_read(64)
+
+#undef __vgpu_read
+#undef VGPU_READ_FOOTER
+#undef VGPU_READ_HEADER
+
 #define GEN2_WRITE_HEADER \
 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
-	assert_device_not_suspended(dev_priv); \
+	assert_rpm_wakelock_held(dev_priv); \
 
 #define GEN2_WRITE_FOOTER
 
 #define __gen2_write(x) \
 static void \
-gen2_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	GEN2_WRITE_HEADER; \
 	__raw_i915_write##x(dev_priv, reg, val); \
 	GEN2_WRITE_FOOTER; \
@@ -832,7 +837,7 @@ gen2_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
 
 #define __gen5_write(x) \
 static void \
-gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	GEN2_WRITE_HEADER; \
 	ilk_dummy_write(dev_priv); \
 	__raw_i915_write##x(dev_priv, reg, val); \
@@ -855,9 +860,10 @@ __gen2_write(64)
 #undef GEN2_WRITE_HEADER
 
 #define GEN6_WRITE_HEADER \
+	u32 offset = i915_mmio_reg_offset(reg); \
 	unsigned long irqflags; \
 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
-	assert_device_not_suspended(dev_priv); \
+	assert_rpm_wakelock_held(dev_priv); \
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 
 #define GEN6_WRITE_FOOTER \
@@ -865,10 +871,10 @@ __gen2_write(64)
 
 #define __gen6_write(x) \
 static void \
-gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	u32 __fifo_ret = 0; \
 	GEN6_WRITE_HEADER; \
-	if (NEEDS_FORCE_WAKE(reg)) { \
+	if (NEEDS_FORCE_WAKE(offset)) { \
 		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 	} \
 	__raw_i915_write##x(dev_priv, reg, val); \
@@ -880,10 +886,10 @@ gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
 
 #define __hsw_write(x) \
 static void \
-hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+hsw_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	u32 __fifo_ret = 0; \
 	GEN6_WRITE_HEADER; \
-	if (NEEDS_FORCE_WAKE(reg)) { \
+	if (NEEDS_FORCE_WAKE(offset)) { \
 		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 	} \
 	hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
@@ -896,15 +902,7 @@ hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace)
 	GEN6_WRITE_FOOTER; \
 }
 
-#define __vgpu_write(x) \
-static void vgpu_write##x(struct drm_i915_private *dev_priv, \
-			  off_t reg, u##x val, bool trace) { \
-	GEN6_WRITE_HEADER; \
-	__raw_i915_write##x(dev_priv, reg, val); \
-	GEN6_WRITE_FOOTER; \
-}
-
-static const u32 gen8_shadowed_regs[] = {
+static const i915_reg_t gen8_shadowed_regs[] = {
 	FORCEWAKE_MT,
 	GEN6_RPNSWREQ,
 	GEN6_RC_VIDEO_FREQ,
@@ -915,11 +913,12 @@ static const u32 gen8_shadowed_regs[] = {
 	/* TODO: Other registers are not yet used */
 };
 
-static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
+static bool is_gen8_shadowed(struct drm_i915_private *dev_priv,
+			     i915_reg_t reg)
 {
 	int i;
 	for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
-		if (reg == gen8_shadowed_regs[i])
+		if (i915_mmio_reg_equal(reg, gen8_shadowed_regs[i]))
 			return true;
 
 	return false;
@@ -927,10 +926,10 @@ static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 
 #define __gen8_write(x) \
 static void \
-gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	GEN6_WRITE_HEADER; \
 	hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
-	if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \
+	if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(dev_priv, reg)) \
 		__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
 	__raw_i915_write##x(dev_priv, reg, val); \
 	hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
@@ -940,22 +939,25 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
 
 #define __chv_write(x) \
 static void \
-chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
-	bool shadowed = is_gen8_shadowed(dev_priv, reg); \
+chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+	enum forcewake_domains fw_engine = 0; \
 	GEN6_WRITE_HEADER; \
-	if (!shadowed) { \
-		if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
-			__force_wake_get(dev_priv, FORCEWAKE_RENDER); \
-		else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
-			__force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
-		else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
-			__force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
-	} \
+	if (!NEEDS_FORCE_WAKE(offset) || \
+	    is_gen8_shadowed(dev_priv, reg)) \
+		fw_engine = 0; \
+	else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_RENDER; \
+	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_MEDIA; \
+	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
+		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+	if (fw_engine) \
+		__force_wake_get(dev_priv, fw_engine); \
 	__raw_i915_write##x(dev_priv, reg, val); \
 	GEN6_WRITE_FOOTER; \
 }
 
-static const u32 gen9_shadowed_regs[] = {
+static const i915_reg_t gen9_shadowed_regs[] = {
 	RING_TAIL(RENDER_RING_BASE),
 	RING_TAIL(GEN6_BSD_RING_BASE),
 	RING_TAIL(VEBOX_RING_BASE),
@@ -968,11 +970,12 @@ static const u32 gen9_shadowed_regs[] = {
 	/* TODO: Other registers are not yet used */
 };
 
-static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg)
+static bool is_gen9_shadowed(struct drm_i915_private *dev_priv,
+			     i915_reg_t reg)
 {
 	int i;
 	for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++)
-		if (reg == gen9_shadowed_regs[i])
+		if (i915_mmio_reg_equal(reg, gen9_shadowed_regs[i]))
 			return true;
 
 	return false;
@@ -980,19 +983,19 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 
 #define __gen9_write(x) \
 static void \
-gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \
+gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
 		bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_WRITE_HEADER; \
 	hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
-	if (!SKL_NEEDS_FORCE_WAKE(reg) || \
+	if (!SKL_NEEDS_FORCE_WAKE(offset) || \
 	    is_gen9_shadowed(dev_priv, reg)) \
 		fw_engine = 0; \
-	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
 		fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
 	else \
 		fw_engine = FORCEWAKE_BLITTER; \
@@ -1024,20 +1027,41 @@ __gen6_write(8)
 __gen6_write(16)
 __gen6_write(32)
 __gen6_write(64)
-__vgpu_write(8)
-__vgpu_write(16)
-__vgpu_write(32)
-__vgpu_write(64)
 
 #undef __gen9_write
 #undef __chv_write
 #undef __gen8_write
 #undef __hsw_write
 #undef __gen6_write
-#undef __vgpu_write
 #undef GEN6_WRITE_FOOTER
 #undef GEN6_WRITE_HEADER
 
+#define VGPU_WRITE_HEADER \
+	unsigned long irqflags; \
+	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
+	assert_rpm_device_not_suspended(dev_priv); \
+	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
+
+#define VGPU_WRITE_FOOTER \
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
+
+#define __vgpu_write(x) \
+static void vgpu_write##x(struct drm_i915_private *dev_priv, \
+			  i915_reg_t reg, u##x val, bool trace) { \
+	VGPU_WRITE_HEADER; \
+	__raw_i915_write##x(dev_priv, reg, val); \
+	VGPU_WRITE_FOOTER; \
+}
+
+__vgpu_write(8)
+__vgpu_write(16)
+__vgpu_write(32)
+__vgpu_write(64)
+
+#undef __vgpu_write
+#undef VGPU_WRITE_FOOTER
+#undef VGPU_WRITE_HEADER
+
 #define ASSIGN_WRITE_MMIO_VFUNCS(x) \
 do { \
 	dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
@@ -1057,7 +1081,8 @@ do { \
 
 static void fw_domain_init(struct drm_i915_private *dev_priv,
 			   enum forcewake_domain_id domain_id,
-			   u32 reg_set, u32 reg_ack)
+			   i915_reg_t reg_set,
+			   i915_reg_t reg_ack)
 {
 	struct intel_uncore_forcewake_domain *d;
 
@@ -1083,12 +1108,10 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
 		d->val_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
 	}
 
-	if (IS_VALLEYVIEW(dev_priv))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		d->reg_post = FORCEWAKE_ACK_VLV;
 	else if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv) || IS_GEN8(dev_priv))
 		d->reg_post = ECOBUS;
-	else
-		d->reg_post = 0;
 
 	d->i915 = dev_priv;
 	d->id = domain_id;
@@ -1118,7 +1141,7 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
 			       FORCEWAKE_ACK_BLITTER_GEN9);
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
 			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
 		if (!IS_CHERRYVIEW(dev))
 			dev_priv->uncore.funcs.force_wake_put =
@@ -1262,12 +1285,14 @@ void intel_uncore_fini(struct drm_device *dev)
 #define GEN_RANGE(l, h) GENMASK(h, l)
 
 static const struct register_whitelist {
-	uint64_t offset;
+	i915_reg_t offset_ldw, offset_udw;
 	uint32_t size;
 	/* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 	uint32_t gen_bitmask;
 } whitelist[] = {
-	{ RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 9) },
+	{ .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
+	  .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
+	  .size = 8, .gen_bitmask = GEN_RANGE(4, 9) },
 };
 
 int i915_reg_read_ioctl(struct drm_device *dev,
@@ -1277,11 +1302,11 @@ int i915_reg_read_ioctl(struct drm_device *dev,
 	struct drm_i915_reg_read *reg = data;
 	struct register_whitelist const *entry = whitelist;
 	unsigned size;
-	u64 offset;
+	i915_reg_t offset_ldw, offset_udw;
 	int i, ret = 0;
 
 	for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
-		if (entry->offset == (reg->offset & -entry->size) &&
+		if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
 		    (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 			break;
 	}
@@ -1293,27 +1318,28 @@ int i915_reg_read_ioctl(struct drm_device *dev,
 	 * be naturally aligned (and those that are not so aligned merely
 	 * limit the available flags for that register).
 	 */
-	offset = entry->offset;
+	offset_ldw = entry->offset_ldw;
+	offset_udw = entry->offset_udw;
 	size = entry->size;
-	size |= reg->offset ^ offset;
+	size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw);
 
 	intel_runtime_pm_get(dev_priv);
 
 	switch (size) {
 	case 8 | 1:
-		reg->val = I915_READ64_2x32(offset, offset+4);
+		reg->val = I915_READ64_2x32(offset_ldw, offset_udw);
 		break;
 	case 8:
-		reg->val = I915_READ64(offset);
+		reg->val = I915_READ64(offset_ldw);
 		break;
 	case 4:
-		reg->val = I915_READ(offset);
+		reg->val = I915_READ(offset_ldw);
 		break;
 	case 2:
-		reg->val = I915_READ16(offset);
+		reg->val = I915_READ16(offset_ldw);
 		break;
 	case 1:
-		reg->val = I915_READ8(offset);
+		reg->val = I915_READ8(offset_ldw);
 		break;
 	default:
 		ret = -EINVAL;
@@ -1470,7 +1496,7 @@ static int gen6_do_reset(struct drm_device *dev)
 }
 
 static int wait_for_register(struct drm_i915_private *dev_priv,
-			     const u32 reg,
+			     i915_reg_t reg,
 			     const u32 mask,
 			     const u32 value,
 			     const unsigned long timeout_ms)
diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig
index 2b81a417cf29..a1844b50546c 100644
--- a/drivers/gpu/drm/imx/Kconfig
+++ b/drivers/gpu/drm/imx/Kconfig
@@ -5,20 +5,11 @@ config DRM_IMX
 	select VIDEOMODE_HELPERS
 	select DRM_GEM_CMA_HELPER
 	select DRM_KMS_CMA_HELPER
-	depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM) && HAVE_DMA_ATTRS
+	depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM)
 	depends on IMX_IPUV3_CORE
 	help
 	  enable i.MX graphics support
 
-config DRM_IMX_FB_HELPER
-	tristate "provide legacy framebuffer /dev/fb0"
-	select DRM_KMS_CMA_HELPER
-	depends on DRM_IMX
-	help
-	  The DRM framework can provide a legacy /dev/fb0 framebuffer
-	  for your device. This is necessary to get a framebuffer console
-	  and also for applications using the legacy framebuffer API
-
 config DRM_IMX_PARALLEL_DISPLAY
 	tristate "Support for parallel displays"
 	select DRM_PANEL
diff --git a/drivers/gpu/drm/imx/dw_hdmi-imx.c b/drivers/gpu/drm/imx/dw_hdmi-imx.c
index 98605ea2ad9d..063825fecbe2 100644
--- a/drivers/gpu/drm/imx/dw_hdmi-imx.c
+++ b/drivers/gpu/drm/imx/dw_hdmi-imx.c
@@ -137,7 +137,7 @@ static void dw_hdmi_imx_encoder_prepare(struct drm_encoder *encoder)
 	imx_drm_set_bus_format(encoder, MEDIA_BUS_FMT_RGB888_1X24);
 }
 
-static struct drm_encoder_helper_funcs dw_hdmi_imx_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs dw_hdmi_imx_encoder_helper_funcs = {
 	.mode_fixup = dw_hdmi_imx_encoder_mode_fixup,
 	.mode_set   = dw_hdmi_imx_encoder_mode_set,
 	.prepare    = dw_hdmi_imx_encoder_prepare,
@@ -145,7 +145,7 @@ static struct drm_encoder_helper_funcs dw_hdmi_imx_encoder_helper_funcs = {
 	.disable    = dw_hdmi_imx_encoder_disable,
 };
 
-static struct drm_encoder_funcs dw_hdmi_imx_encoder_funcs = {
+static const struct drm_encoder_funcs dw_hdmi_imx_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -251,7 +251,7 @@ static int dw_hdmi_imx_bind(struct device *dev, struct device *master,
 
 	drm_encoder_helper_add(encoder, &dw_hdmi_imx_encoder_helper_funcs);
 	drm_encoder_init(drm, encoder, &dw_hdmi_imx_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	return dw_hdmi_bind(dev, master, data, encoder, iores, irq, plat_data);
 }
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 7b990b4e96d2..2f57d7967417 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -39,46 +39,41 @@ struct imx_drm_component {
 struct imx_drm_device {
 	struct drm_device			*drm;
 	struct imx_drm_crtc			*crtc[MAX_CRTC];
-	int					pipes;
+	unsigned int				pipes;
 	struct drm_fbdev_cma			*fbhelper;
 };
 
 struct imx_drm_crtc {
 	struct drm_crtc				*crtc;
-	int					pipe;
 	struct imx_drm_crtc_helper_funcs	imx_drm_helper_funcs;
 };
 
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
 static int legacyfb_depth = 16;
 module_param(legacyfb_depth, int, 0444);
+#endif
 
-int imx_drm_crtc_id(struct imx_drm_crtc *crtc)
+unsigned int imx_drm_crtc_id(struct imx_drm_crtc *crtc)
 {
-	return crtc->pipe;
+	return drm_crtc_index(crtc->crtc);
 }
 EXPORT_SYMBOL_GPL(imx_drm_crtc_id);
 
 static void imx_drm_driver_lastclose(struct drm_device *drm)
 {
-#if IS_ENABLED(CONFIG_DRM_IMX_FB_HELPER)
 	struct imx_drm_device *imxdrm = drm->dev_private;
 
 	drm_fbdev_cma_restore_mode(imxdrm->fbhelper);
-#endif
 }
 
 static int imx_drm_driver_unload(struct drm_device *drm)
 {
-#if IS_ENABLED(CONFIG_DRM_IMX_FB_HELPER)
 	struct imx_drm_device *imxdrm = drm->dev_private;
-#endif
 
 	drm_kms_helper_poll_fini(drm);
 
-#if IS_ENABLED(CONFIG_DRM_IMX_FB_HELPER)
 	if (imxdrm->fbhelper)
 		drm_fbdev_cma_fini(imxdrm->fbhelper);
-#endif
 
 	component_unbind_all(drm->dev, drm);
 
@@ -128,19 +123,19 @@ EXPORT_SYMBOL_GPL(imx_drm_set_bus_format);
 
 int imx_drm_crtc_vblank_get(struct imx_drm_crtc *imx_drm_crtc)
 {
-	return drm_vblank_get(imx_drm_crtc->crtc->dev, imx_drm_crtc->pipe);
+	return drm_crtc_vblank_get(imx_drm_crtc->crtc);
 }
 EXPORT_SYMBOL_GPL(imx_drm_crtc_vblank_get);
 
 void imx_drm_crtc_vblank_put(struct imx_drm_crtc *imx_drm_crtc)
 {
-	drm_vblank_put(imx_drm_crtc->crtc->dev, imx_drm_crtc->pipe);
+	drm_crtc_vblank_put(imx_drm_crtc->crtc);
 }
 EXPORT_SYMBOL_GPL(imx_drm_crtc_vblank_put);
 
 void imx_drm_handle_vblank(struct imx_drm_crtc *imx_drm_crtc)
 {
-	drm_handle_vblank(imx_drm_crtc->crtc->dev, imx_drm_crtc->pipe);
+	drm_crtc_handle_vblank(imx_drm_crtc->crtc);
 }
 EXPORT_SYMBOL_GPL(imx_drm_handle_vblank);
 
@@ -214,14 +209,12 @@ EXPORT_SYMBOL_GPL(imx_drm_encoder_destroy);
 
 static void imx_drm_output_poll_changed(struct drm_device *drm)
 {
-#if IS_ENABLED(CONFIG_DRM_IMX_FB_HELPER)
 	struct imx_drm_device *imxdrm = drm->dev_private;
 
 	drm_fbdev_cma_hotplug_event(imxdrm->fbhelper);
-#endif
 }
 
-static struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
+static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
 	.fb_create = drm_fb_cma_create,
 	.output_poll_changed = imx_drm_output_poll_changed,
 };
@@ -307,11 +300,12 @@ static int imx_drm_driver_load(struct drm_device *drm, unsigned long flags)
 	 * The fb helper takes copies of key hardware information, so the
 	 * crtcs/connectors/encoders must not change after this point.
 	 */
-#if IS_ENABLED(CONFIG_DRM_IMX_FB_HELPER)
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
 	if (legacyfb_depth != 16 && legacyfb_depth != 32) {
 		dev_warn(drm->dev, "Invalid legacyfb_depth.  Defaulting to 16bpp\n");
 		legacyfb_depth = 16;
 	}
+	drm_helper_disable_unused_functions(drm);
 	imxdrm->fbhelper = drm_fbdev_cma_init(drm, legacyfb_depth,
 				drm->mode_config.num_crtc, MAX_CRTC);
 	if (IS_ERR(imxdrm->fbhelper)) {
@@ -362,12 +356,11 @@ int imx_drm_add_crtc(struct drm_device *drm, struct drm_crtc *crtc,
 		return -ENOMEM;
 
 	imx_drm_crtc->imx_drm_helper_funcs = *imx_drm_helper_funcs;
-	imx_drm_crtc->pipe = imxdrm->pipes++;
 	imx_drm_crtc->crtc = crtc;
 
 	crtc->port = port;
 
-	imxdrm->crtc[imx_drm_crtc->pipe] = imx_drm_crtc;
+	imxdrm->crtc[imxdrm->pipes++] = imx_drm_crtc;
 
 	*new_crtc = imx_drm_crtc;
 
@@ -379,12 +372,12 @@ int imx_drm_add_crtc(struct drm_device *drm, struct drm_crtc *crtc,
 			imx_drm_crtc->imx_drm_helper_funcs.crtc_helper_funcs);
 
 	drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
-			imx_drm_crtc->imx_drm_helper_funcs.crtc_funcs);
+			imx_drm_crtc->imx_drm_helper_funcs.crtc_funcs, NULL);
 
 	return 0;
 
 err_register:
-	imxdrm->crtc[imx_drm_crtc->pipe] = NULL;
+	imxdrm->crtc[--imxdrm->pipes] = NULL;
 	kfree(imx_drm_crtc);
 	return ret;
 }
@@ -396,10 +389,11 @@ EXPORT_SYMBOL_GPL(imx_drm_add_crtc);
 int imx_drm_remove_crtc(struct imx_drm_crtc *imx_drm_crtc)
 {
 	struct imx_drm_device *imxdrm = imx_drm_crtc->crtc->dev->dev_private;
+	unsigned int pipe = drm_crtc_index(imx_drm_crtc->crtc);
 
 	drm_crtc_cleanup(imx_drm_crtc->crtc);
 
-	imxdrm->crtc[imx_drm_crtc->pipe] = NULL;
+	imxdrm->crtc[pipe] = NULL;
 
 	kfree(imx_drm_crtc);
 
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index 83284b4d4be1..71cf6d9c714f 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -13,7 +13,7 @@ struct drm_plane;
 struct imx_drm_crtc;
 struct platform_device;
 
-int imx_drm_crtc_id(struct imx_drm_crtc *crtc);
+unsigned int imx_drm_crtc_id(struct imx_drm_crtc *crtc);
 
 struct imx_drm_crtc_helper_funcs {
 	int (*enable_vblank)(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index abacc8f67469..22ac482231ed 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -358,23 +358,23 @@ static void imx_ldb_encoder_disable(struct drm_encoder *encoder)
 	drm_panel_unprepare(imx_ldb_ch->panel);
 }
 
-static struct drm_connector_funcs imx_ldb_connector_funcs = {
+static const struct drm_connector_funcs imx_ldb_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = imx_ldb_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 };
 
-static struct drm_connector_helper_funcs imx_ldb_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs imx_ldb_connector_helper_funcs = {
 	.get_modes = imx_ldb_connector_get_modes,
 	.best_encoder = imx_ldb_connector_best_encoder,
 };
 
-static struct drm_encoder_funcs imx_ldb_encoder_funcs = {
+static const struct drm_encoder_funcs imx_ldb_encoder_funcs = {
 	.destroy = imx_drm_encoder_destroy,
 };
 
-static struct drm_encoder_helper_funcs imx_ldb_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs imx_ldb_encoder_helper_funcs = {
 	.dpms = imx_ldb_encoder_dpms,
 	.mode_fixup = imx_ldb_encoder_mode_fixup,
 	.prepare = imx_ldb_encoder_prepare,
@@ -422,7 +422,7 @@ static int imx_ldb_register(struct drm_device *drm,
 	drm_encoder_helper_add(&imx_ldb_ch->encoder,
 			&imx_ldb_encoder_helper_funcs);
 	drm_encoder_init(drm, &imx_ldb_ch->encoder, &imx_ldb_encoder_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 
 	drm_connector_helper_add(&imx_ldb_ch->connector,
 			&imx_ldb_connector_helper_funcs);
diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c
index f9597146dc67..292349f0b132 100644
--- a/drivers/gpu/drm/imx/imx-tve.c
+++ b/drivers/gpu/drm/imx/imx-tve.c
@@ -360,24 +360,24 @@ static void imx_tve_encoder_disable(struct drm_encoder *encoder)
 	tve_disable(tve);
 }
 
-static struct drm_connector_funcs imx_tve_connector_funcs = {
+static const struct drm_connector_funcs imx_tve_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = imx_tve_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 };
 
-static struct drm_connector_helper_funcs imx_tve_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs imx_tve_connector_helper_funcs = {
 	.get_modes = imx_tve_connector_get_modes,
 	.best_encoder = imx_tve_connector_best_encoder,
 	.mode_valid = imx_tve_connector_mode_valid,
 };
 
-static struct drm_encoder_funcs imx_tve_encoder_funcs = {
+static const struct drm_encoder_funcs imx_tve_encoder_funcs = {
 	.destroy = imx_drm_encoder_destroy,
 };
 
-static struct drm_encoder_helper_funcs imx_tve_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs imx_tve_encoder_helper_funcs = {
 	.dpms = imx_tve_encoder_dpms,
 	.mode_fixup = imx_tve_encoder_mode_fixup,
 	.prepare = imx_tve_encoder_prepare,
@@ -508,7 +508,7 @@ static int imx_tve_register(struct drm_device *drm, struct imx_tve *tve)
 
 	drm_encoder_helper_add(&tve->encoder, &imx_tve_encoder_helper_funcs);
 	drm_encoder_init(drm, &tve->encoder, &imx_tve_encoder_funcs,
-			 encoder_type);
+			 encoder_type, NULL);
 
 	drm_connector_helper_add(&tve->connector,
 			&imx_tve_connector_helper_funcs);
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 4ab841eebee1..30a57185bdb4 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -270,7 +270,7 @@ static void ipu_crtc_commit(struct drm_crtc *crtc)
 	ipu_fb_enable(ipu_crtc);
 }
 
-static struct drm_crtc_helper_funcs ipu_helper_funcs = {
+static const struct drm_crtc_helper_funcs ipu_helper_funcs = {
 	.dpms = ipu_crtc_dpms,
 	.mode_fixup = ipu_crtc_mode_fixup,
 	.mode_set = ipu_crtc_mode_set,
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index e2ff410bab74..591ba2f1ae03 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -401,7 +401,8 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 
 	ret = drm_universal_plane_init(dev, &ipu_plane->base, possible_crtcs,
 				       &ipu_plane_funcs, ipu_plane_formats,
-				       ARRAY_SIZE(ipu_plane_formats), type);
+				       ARRAY_SIZE(ipu_plane_formats), type,
+				       NULL);
 	if (ret) {
 		DRM_ERROR("failed to initialize plane\n");
 		kfree(ipu_plane);
diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
index 2e9b9f1b5cd2..0ffef172afb4 100644
--- a/drivers/gpu/drm/imx/parallel-display.c
+++ b/drivers/gpu/drm/imx/parallel-display.c
@@ -148,23 +148,23 @@ static void imx_pd_encoder_disable(struct drm_encoder *encoder)
 	drm_panel_unprepare(imxpd->panel);
 }
 
-static struct drm_connector_funcs imx_pd_connector_funcs = {
+static const struct drm_connector_funcs imx_pd_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = imx_pd_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 };
 
-static struct drm_connector_helper_funcs imx_pd_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs imx_pd_connector_helper_funcs = {
 	.get_modes = imx_pd_connector_get_modes,
 	.best_encoder = imx_pd_connector_best_encoder,
 };
 
-static struct drm_encoder_funcs imx_pd_encoder_funcs = {
+static const struct drm_encoder_funcs imx_pd_encoder_funcs = {
 	.destroy = imx_drm_encoder_destroy,
 };
 
-static struct drm_encoder_helper_funcs imx_pd_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs imx_pd_encoder_helper_funcs = {
 	.dpms = imx_pd_encoder_dpms,
 	.mode_fixup = imx_pd_encoder_mode_fixup,
 	.prepare = imx_pd_encoder_prepare,
@@ -192,7 +192,7 @@ static int imx_pd_register(struct drm_device *drm,
 
 	drm_encoder_helper_add(&imxpd->encoder, &imx_pd_encoder_helper_funcs);
 	drm_encoder_init(drm, &imxpd->encoder, &imx_pd_encoder_funcs,
-			 DRM_MODE_ENCODER_NONE);
+			 DRM_MODE_ENCODER_NONE, NULL);
 
 	drm_connector_helper_add(&imxpd->connector,
 			&imx_pd_connector_helper_funcs);
@@ -204,8 +204,6 @@ static int imx_pd_register(struct drm_device *drm,
 
 	drm_mode_connector_attach_encoder(&imxpd->connector, &imxpd->encoder);
 
-	imxpd->connector.encoder = &imxpd->encoder;
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 912151c36d59..205b2801d3b8 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -252,7 +252,7 @@ void mgag200_fbdev_fini(struct mga_device *mdev);
 				/* mgag200_main.c */
 int mgag200_framebuffer_init(struct drm_device *dev,
 			     struct mga_framebuffer *mfb,
-			     struct drm_mode_fb_cmd2 *mode_cmd,
+			     const struct drm_mode_fb_cmd2 *mode_cmd,
 			     struct drm_gem_object *obj);
 
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index b35b5b2db4ec..d9b04b008feb 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -138,7 +138,7 @@ static struct fb_ops mgag200fb_ops = {
 };
 
 static int mgag200fb_create_object(struct mga_fbdev *afbdev,
-				   struct drm_mode_fb_cmd2 *mode_cmd,
+				   const struct drm_mode_fb_cmd2 *mode_cmd,
 				   struct drm_gem_object **gobj_p)
 {
 	struct drm_device *dev = afbdev->helper.dev;
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index b1a0f5656175..9147444d5bf2 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -29,7 +29,7 @@ static const struct drm_framebuffer_funcs mga_fb_funcs = {
 
 int mgag200_framebuffer_init(struct drm_device *dev,
 			     struct mga_framebuffer *gfb,
-			     struct drm_mode_fb_cmd2 *mode_cmd,
+			     const struct drm_mode_fb_cmd2 *mode_cmd,
 			     struct drm_gem_object *obj)
 {
 	int ret;
@@ -47,7 +47,7 @@ int mgag200_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 mgag200_user_framebuffer_create(struct drm_device *dev,
 				struct drm_file *filp,
-				struct drm_mode_fb_cmd2 *mode_cmd)
+				const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct mga_framebuffer *mga_fb;
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index c99d3fe12881..dc13c4857e6f 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -1538,7 +1538,7 @@ static struct drm_encoder *mga_encoder_init(struct drm_device *dev)
 	encoder->possible_crtcs = 0x1;
 
 	drm_encoder_init(dev, encoder, &mga_encoder_encoder_funcs,
-			 DRM_MODE_ENCODER_DAC);
+			 DRM_MODE_ENCODER_DAC, NULL);
 	drm_encoder_helper_add(encoder, &mga_encoder_helper_funcs);
 
 	return encoder;
@@ -1564,7 +1564,7 @@ static uint32_t mga_vga_calculate_mode_bandwidth(struct drm_display_mode *mode,
 							int bits_per_pixel)
 {
 	uint32_t total_area, divisor;
-	int64_t active_area, pixels_per_second, bandwidth;
+	uint64_t active_area, pixels_per_second, bandwidth;
 	uint64_t bytes_per_pixel = (bits_per_pixel + 7) / 8;
 
 	divisor = 1024;
@@ -1684,13 +1684,13 @@ static void mga_connector_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-struct drm_connector_helper_funcs mga_vga_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs mga_vga_connector_helper_funcs = {
 	.get_modes = mga_vga_get_modes,
 	.mode_valid = mga_vga_mode_valid,
 	.best_encoder = mga_connector_best_encoder,
 };
 
-struct drm_connector_funcs mga_vga_connector_funcs = {
+static const struct drm_connector_funcs mga_vga_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = mga_vga_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 84d3ec98e6b9..215495c2780c 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -54,3 +54,11 @@ config DRM_MSM_DSI_20NM_PHY
 	default y
 	help
 	  Choose this option if the 20nm DSI PHY is used on the platform.
+
+config DRM_MSM_DSI_28NM_8960_PHY
+	bool "Enable DSI 28nm 8960 PHY driver in MSM DRM"
+	depends on DRM_MSM_DSI
+	default y
+	help
+	  Choose this option if the 28nm DSI PHY 8960 variant is used on the
+	  platform.
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 1c90290be716..065ad4138799 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -54,6 +54,7 @@ msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
 msm-$(CONFIG_COMMON_CLK) += mdp/mdp4/mdp4_lvds_pll.o
 
 msm-$(CONFIG_DRM_MSM_DSI) += dsi/dsi.o \
+			mdp/mdp4/mdp4_dsi_encoder.o \
 			dsi/dsi_cfg.o \
 			dsi/dsi_host.o \
 			dsi/dsi_manager.o \
@@ -62,10 +63,12 @@ msm-$(CONFIG_DRM_MSM_DSI) += dsi/dsi.o \
 
 msm-$(CONFIG_DRM_MSM_DSI_28NM_PHY) += dsi/phy/dsi_phy_28nm.o
 msm-$(CONFIG_DRM_MSM_DSI_20NM_PHY) += dsi/phy/dsi_phy_20nm.o
+msm-$(CONFIG_DRM_MSM_DSI_28NM_8960_PHY) += dsi/phy/dsi_phy_28nm_8960.o
 
 ifeq ($(CONFIG_DRM_MSM_DSI_PLL),y)
 msm-y += dsi/pll/dsi_pll.o
 msm-$(CONFIG_DRM_MSM_DSI_28NM_PHY) += dsi/pll/dsi_pll_28nm.o
+msm-$(CONFIG_DRM_MSM_DSI_28NM_8960_PHY) += dsi/pll/dsi_pll_28nm_8960.o
 endif
 
 obj-$(CONFIG_DRM_MSM)	+= msm.o
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 1ea2df524fac..950d27d26b30 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -19,10 +19,6 @@
 
 #include "adreno_gpu.h"
 
-#if defined(DOWNSTREAM_CONFIG_MSM_BUS_SCALING) && !defined(CONFIG_OF)
-#  include <mach/kgsl.h>
-#endif
-
 #define ANY_ID 0xff
 
 bool hang_debug = false;
@@ -168,7 +164,6 @@ static void set_gpu_pdev(struct drm_device *dev,
 static int adreno_bind(struct device *dev, struct device *master, void *data)
 {
 	static struct adreno_platform_config config = {};
-#ifdef CONFIG_OF
 	struct device_node *child, *node = dev->of_node;
 	u32 val;
 	int ret;
@@ -205,53 +200,6 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
 		return -ENXIO;
 	}
 
-#else
-	struct kgsl_device_platform_data *pdata = dev->platform_data;
-	uint32_t version = socinfo_get_version();
-	if (cpu_is_apq8064ab()) {
-		config.fast_rate = 450000000;
-		config.slow_rate = 27000000;
-		config.bus_freq  = 4;
-		config.rev = ADRENO_REV(3, 2, 1, 0);
-	} else if (cpu_is_apq8064()) {
-		config.fast_rate = 400000000;
-		config.slow_rate = 27000000;
-		config.bus_freq  = 4;
-
-		if (SOCINFO_VERSION_MAJOR(version) == 2)
-			config.rev = ADRENO_REV(3, 2, 0, 2);
-		else if ((SOCINFO_VERSION_MAJOR(version) == 1) &&
-				(SOCINFO_VERSION_MINOR(version) == 1))
-			config.rev = ADRENO_REV(3, 2, 0, 1);
-		else
-			config.rev = ADRENO_REV(3, 2, 0, 0);
-
-	} else if (cpu_is_msm8960ab()) {
-		config.fast_rate = 400000000;
-		config.slow_rate = 320000000;
-		config.bus_freq  = 4;
-
-		if (SOCINFO_VERSION_MINOR(version) == 0)
-			config.rev = ADRENO_REV(3, 2, 1, 0);
-		else
-			config.rev = ADRENO_REV(3, 2, 1, 1);
-
-	} else if (cpu_is_msm8930()) {
-		config.fast_rate = 400000000;
-		config.slow_rate = 27000000;
-		config.bus_freq  = 3;
-
-		if ((SOCINFO_VERSION_MAJOR(version) == 1) &&
-			(SOCINFO_VERSION_MINOR(version) == 2))
-			config.rev = ADRENO_REV(3, 0, 5, 2);
-		else
-			config.rev = ADRENO_REV(3, 0, 5, 0);
-
-	}
-#  ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-	config.bus_scale_table = pdata->bus_scale_table;
-#  endif
-#endif
 	dev->platform_data = &config;
 	set_gpu_pdev(dev_get_drvdata(master), to_platform_device(dev));
 	return 0;
diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
index 5f5a3732cdf6..749fbb28ec3d 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.h
@@ -31,10 +31,12 @@ enum msm_dsi_phy_type {
 	MSM_DSI_PHY_28NM_HPM,
 	MSM_DSI_PHY_28NM_LP,
 	MSM_DSI_PHY_20NM,
+	MSM_DSI_PHY_28NM_8960,
 	MSM_DSI_PHY_MAX
 };
 
 #define DSI_DEV_REGULATOR_MAX	8
+#define DSI_BUS_CLK_MAX		4
 
 /* Regulators for DSI devices */
 struct dsi_reg_entry {
@@ -89,7 +91,7 @@ int msm_dsi_manager_phy_enable(int id,
 		u32 *clk_pre, u32 *clk_post);
 void msm_dsi_manager_phy_disable(int id);
 int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg);
-bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 iova, u32 len);
+bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len);
 int msm_dsi_manager_register(struct msm_dsi *msm_dsi);
 void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi);
 
@@ -143,7 +145,7 @@ int msm_dsi_host_cmd_tx(struct mipi_dsi_host *host,
 int msm_dsi_host_cmd_rx(struct mipi_dsi_host *host,
 					const struct mipi_dsi_msg *msg);
 void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host,
-					u32 iova, u32 len);
+					u32 dma_base, u32 len);
 int msm_dsi_host_enable(struct mipi_dsi_host *host);
 int msm_dsi_host_disable(struct mipi_dsi_host *host);
 int msm_dsi_host_power_on(struct mipi_dsi_host *host);
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
index 5872d5e5934f..2a827d8093a2 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
@@ -13,9 +13,26 @@
 
 #include "dsi_cfg.h"
 
-/* DSI v2 has not been supported by now */
-static const struct msm_dsi_config dsi_v2_cfg = {
+static const char * const dsi_v2_bus_clk_names[] = {
+	"core_mmss_clk", "iface_clk", "bus_clk",
+};
+
+static const struct msm_dsi_config apq8064_dsi_cfg = {
 	.io_offset = 0,
+	.reg_cfg = {
+		.num = 3,
+		.regs = {
+			{"vdda", 1200000, 1200000, 100000, 100},
+			{"avdd", 3000000, 3000000, 110000, 100},
+			{"vddio", 1800000, 1800000, 100000, 100},
+		},
+	},
+	.bus_clk_names = dsi_v2_bus_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v2_bus_clk_names),
+};
+
+static const char * const dsi_6g_bus_clk_names[] = {
+	"mdp_core_clk", "iface_clk", "bus_clk", "core_mmss_clk",
 };
 
 static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = {
@@ -29,6 +46,12 @@ static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = {
 			{"vddio", 1800000, 1800000, 100000, 100},
 		},
 	},
+	.bus_clk_names = dsi_6g_bus_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
+};
+
+static const char * const dsi_8916_bus_clk_names[] = {
+	"mdp_core_clk", "iface_clk", "bus_clk",
 };
 
 static const struct msm_dsi_config msm8916_dsi_cfg = {
@@ -42,6 +65,8 @@ static const struct msm_dsi_config msm8916_dsi_cfg = {
 			{"vddio", 1800000, 1800000, 100000, 100},
 		},
 	},
+	.bus_clk_names = dsi_8916_bus_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_8916_bus_clk_names),
 };
 
 static const struct msm_dsi_config msm8994_dsi_cfg = {
@@ -57,11 +82,13 @@ static const struct msm_dsi_config msm8994_dsi_cfg = {
 			{"lab_reg", -1, -1, -1, -1},
 			{"ibb_reg", -1, -1, -1, -1},
 		},
-	}
+	},
+	.bus_clk_names = dsi_6g_bus_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
 };
 
 static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = {
-	{MSM_DSI_VER_MAJOR_V2, U32_MAX, &dsi_v2_cfg},
+	{MSM_DSI_VER_MAJOR_V2, MSM_DSI_V2_VER_MINOR_8064, &apq8064_dsi_cfg},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_0,
 						&msm8974_apq8084_dsi_cfg},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_1,
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
index 4cf887240177..a68c836744a3 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
@@ -25,11 +25,15 @@
 #define MSM_DSI_6G_VER_MINOR_V1_3	0x10030000
 #define MSM_DSI_6G_VER_MINOR_V1_3_1	0x10030001
 
+#define MSM_DSI_V2_VER_MINOR_8064	0x0
+
 #define DSI_6G_REG_SHIFT	4
 
 struct msm_dsi_config {
 	u32 io_offset;
 	struct dsi_reg_config reg_cfg;
+	const char * const *bus_clk_names;
+	const int num_bus_clks;
 };
 
 struct msm_dsi_cfg_handler {
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 4c49868efcda..48f9967b4a1b 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -24,26 +24,36 @@
 #include <linux/of_graph.h>
 #include <linux/regulator/consumer.h>
 #include <linux/spinlock.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 #include <video/mipi_display.h>
 
 #include "dsi.h"
 #include "dsi.xml.h"
+#include "sfpb.xml.h"
 #include "dsi_cfg.h"
 
 static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor)
 {
 	u32 ver;
-	u32 ver_6g;
 
 	if (!major || !minor)
 		return -EINVAL;
 
-	/* From DSI6G(v3), addition of a 6G_HW_VERSION register at offset 0
+	/*
+	 * From DSI6G(v3), addition of a 6G_HW_VERSION register at offset 0
 	 * makes all other registers 4-byte shifted down.
+	 *
+	 * In order to identify between DSI6G(v3) and beyond, and DSIv2 and
+	 * older, we read the DSI_VERSION register without any shift(offset
+	 * 0x1f0). In the case of DSIv2, this hast to be a non-zero value. In
+	 * the case of DSI6G, this has to be zero (the offset points to a
+	 * scratch register which we never touch)
 	 */
-	ver_6g = msm_readl(base + REG_DSI_6G_HW_VERSION);
-	if (ver_6g == 0) {
-		ver = msm_readl(base + REG_DSI_VERSION);
+
+	ver = msm_readl(base + REG_DSI_VERSION);
+	if (ver) {
+		/* older dsi host, there is no register shift */
 		ver = FIELD(ver, DSI_VERSION_MAJOR);
 		if (ver <= MSM_DSI_VER_MAJOR_V2) {
 			/* old versions */
@@ -54,12 +64,17 @@ static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor)
 			return -EINVAL;
 		}
 	} else {
+		/*
+		 * newer host, offset 0 has 6G_HW_VERSION, the rest of the
+		 * registers are shifted down, read DSI_VERSION again with
+		 * the shifted offset
+		 */
 		ver = msm_readl(base + DSI_6G_REG_SHIFT + REG_DSI_VERSION);
 		ver = FIELD(ver, DSI_VERSION_MAJOR);
 		if (ver == MSM_DSI_VER_MAJOR_6G) {
 			/* 6G version */
 			*major = ver;
-			*minor = ver_6g;
+			*minor = msm_readl(base + REG_DSI_6G_HW_VERSION);
 			return 0;
 		} else {
 			return -EINVAL;
@@ -91,10 +106,9 @@ struct msm_dsi_host {
 
 	void __iomem *ctrl_base;
 	struct regulator_bulk_data supplies[DSI_DEV_REGULATOR_MAX];
-	struct clk *mdp_core_clk;
-	struct clk *ahb_clk;
-	struct clk *axi_clk;
-	struct clk *mmss_misc_ahb_clk;
+
+	struct clk *bus_clks[DSI_BUS_CLK_MAX];
+
 	struct clk *byte_clk;
 	struct clk *esc_clk;
 	struct clk *pixel_clk;
@@ -102,6 +116,14 @@ struct msm_dsi_host {
 	struct clk *pixel_clk_src;
 
 	u32 byte_clk_rate;
+	u32 esc_clk_rate;
+
+	/* DSI v2 specific clocks */
+	struct clk *src_clk;
+	struct clk *esc_clk_src;
+	struct clk *dsi_clk_src;
+
+	u32 src_clk_rate;
 
 	struct gpio_desc *disp_en_gpio;
 	struct gpio_desc *te_gpio;
@@ -119,9 +141,19 @@ struct msm_dsi_host {
 	struct work_struct err_work;
 	struct workqueue_struct *workqueue;
 
+	/* DSI 6G TX buffer*/
 	struct drm_gem_object *tx_gem_obj;
+
+	/* DSI v2 TX buffer */
+	void *tx_buf;
+	dma_addr_t tx_buf_paddr;
+
+	int tx_size;
+
 	u8 *rx_buf;
 
+	struct regmap *sfpb;
+
 	struct drm_display_mode *mode;
 
 	/* connected device info */
@@ -165,21 +197,31 @@ static const struct msm_dsi_cfg_handler *dsi_get_config(
 						struct msm_dsi_host *msm_host)
 {
 	const struct msm_dsi_cfg_handler *cfg_hnd = NULL;
+	struct device *dev = &msm_host->pdev->dev;
 	struct regulator *gdsc_reg;
+	struct clk *ahb_clk;
 	int ret;
 	u32 major = 0, minor = 0;
 
-	gdsc_reg = regulator_get(&msm_host->pdev->dev, "gdsc");
+	gdsc_reg = regulator_get(dev, "gdsc");
 	if (IS_ERR(gdsc_reg)) {
 		pr_err("%s: cannot get gdsc\n", __func__);
 		goto exit;
 	}
+
+	ahb_clk = clk_get(dev, "iface_clk");
+	if (IS_ERR(ahb_clk)) {
+		pr_err("%s: cannot get interface clock\n", __func__);
+		goto put_gdsc;
+	}
+
 	ret = regulator_enable(gdsc_reg);
 	if (ret) {
 		pr_err("%s: unable to enable gdsc\n", __func__);
-		goto put_gdsc;
+		goto put_clk;
 	}
-	ret = clk_prepare_enable(msm_host->ahb_clk);
+
+	ret = clk_prepare_enable(ahb_clk);
 	if (ret) {
 		pr_err("%s: unable to enable ahb_clk\n", __func__);
 		goto disable_gdsc;
@@ -196,9 +238,11 @@ static const struct msm_dsi_cfg_handler *dsi_get_config(
 	DBG("%s: Version %x:%x\n", __func__, major, minor);
 
 disable_clks:
-	clk_disable_unprepare(msm_host->ahb_clk);
+	clk_disable_unprepare(ahb_clk);
 disable_gdsc:
 	regulator_disable(gdsc_reg);
+put_clk:
+	clk_put(ahb_clk);
 put_gdsc:
 	regulator_put(gdsc_reg);
 exit:
@@ -295,40 +339,23 @@ static int dsi_regulator_init(struct msm_dsi_host *msm_host)
 static int dsi_clk_init(struct msm_dsi_host *msm_host)
 {
 	struct device *dev = &msm_host->pdev->dev;
-	int ret = 0;
-
-	msm_host->mdp_core_clk = devm_clk_get(dev, "mdp_core_clk");
-	if (IS_ERR(msm_host->mdp_core_clk)) {
-		ret = PTR_ERR(msm_host->mdp_core_clk);
-		pr_err("%s: Unable to get mdp core clk. ret=%d\n",
-			__func__, ret);
-		goto exit;
-	}
-
-	msm_host->ahb_clk = devm_clk_get(dev, "iface_clk");
-	if (IS_ERR(msm_host->ahb_clk)) {
-		ret = PTR_ERR(msm_host->ahb_clk);
-		pr_err("%s: Unable to get mdss ahb clk. ret=%d\n",
-			__func__, ret);
-		goto exit;
-	}
-
-	msm_host->axi_clk = devm_clk_get(dev, "bus_clk");
-	if (IS_ERR(msm_host->axi_clk)) {
-		ret = PTR_ERR(msm_host->axi_clk);
-		pr_err("%s: Unable to get axi bus clk. ret=%d\n",
-			__func__, ret);
-		goto exit;
-	}
-
-	msm_host->mmss_misc_ahb_clk = devm_clk_get(dev, "core_mmss_clk");
-	if (IS_ERR(msm_host->mmss_misc_ahb_clk)) {
-		ret = PTR_ERR(msm_host->mmss_misc_ahb_clk);
-		pr_err("%s: Unable to get mmss misc ahb clk. ret=%d\n",
-			__func__, ret);
-		goto exit;
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+	const struct msm_dsi_config *cfg = cfg_hnd->cfg;
+	int i, ret = 0;
+
+	/* get bus clocks */
+	for (i = 0; i < cfg->num_bus_clks; i++) {
+		msm_host->bus_clks[i] = devm_clk_get(dev,
+						cfg->bus_clk_names[i]);
+		if (IS_ERR(msm_host->bus_clks[i])) {
+			ret = PTR_ERR(msm_host->bus_clks[i]);
+			pr_err("%s: Unable to get %s, ret = %d\n",
+				__func__, cfg->bus_clk_names[i], ret);
+			goto exit;
+		}
 	}
 
+	/* get link and source clocks */
 	msm_host->byte_clk = devm_clk_get(dev, "byte_clk");
 	if (IS_ERR(msm_host->byte_clk)) {
 		ret = PTR_ERR(msm_host->byte_clk);
@@ -356,80 +383,85 @@ static int dsi_clk_init(struct msm_dsi_host *msm_host)
 		goto exit;
 	}
 
-	msm_host->byte_clk_src = devm_clk_get(dev, "byte_clk_src");
-	if (IS_ERR(msm_host->byte_clk_src)) {
-		ret = PTR_ERR(msm_host->byte_clk_src);
+	msm_host->byte_clk_src = clk_get_parent(msm_host->byte_clk);
+	if (!msm_host->byte_clk_src) {
+		ret = -ENODEV;
 		pr_err("%s: can't find byte_clk_src. ret=%d\n", __func__, ret);
-		msm_host->byte_clk_src = NULL;
 		goto exit;
 	}
 
-	msm_host->pixel_clk_src = devm_clk_get(dev, "pixel_clk_src");
-	if (IS_ERR(msm_host->pixel_clk_src)) {
-		ret = PTR_ERR(msm_host->pixel_clk_src);
+	msm_host->pixel_clk_src = clk_get_parent(msm_host->pixel_clk);
+	if (!msm_host->pixel_clk_src) {
+		ret = -ENODEV;
 		pr_err("%s: can't find pixel_clk_src. ret=%d\n", __func__, ret);
-		msm_host->pixel_clk_src = NULL;
 		goto exit;
 	}
 
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
+		msm_host->src_clk = devm_clk_get(dev, "src_clk");
+		if (IS_ERR(msm_host->src_clk)) {
+			ret = PTR_ERR(msm_host->src_clk);
+			pr_err("%s: can't find dsi_src_clk. ret=%d\n",
+				__func__, ret);
+			msm_host->src_clk = NULL;
+			goto exit;
+		}
+
+		msm_host->esc_clk_src = clk_get_parent(msm_host->esc_clk);
+		if (!msm_host->esc_clk_src) {
+			ret = -ENODEV;
+			pr_err("%s: can't get esc_clk_src. ret=%d\n",
+				__func__, ret);
+			goto exit;
+		}
+
+		msm_host->dsi_clk_src = clk_get_parent(msm_host->src_clk);
+		if (!msm_host->dsi_clk_src) {
+			ret = -ENODEV;
+			pr_err("%s: can't get dsi_clk_src. ret=%d\n",
+				__func__, ret);
+		}
+	}
 exit:
 	return ret;
 }
 
 static int dsi_bus_clk_enable(struct msm_dsi_host *msm_host)
 {
-	int ret;
+	const struct msm_dsi_config *cfg = msm_host->cfg_hnd->cfg;
+	int i, ret;
 
 	DBG("id=%d", msm_host->id);
 
-	ret = clk_prepare_enable(msm_host->mdp_core_clk);
-	if (ret) {
-		pr_err("%s: failed to enable mdp_core_clock, %d\n",
-							 __func__, ret);
-		goto core_clk_err;
-	}
-
-	ret = clk_prepare_enable(msm_host->ahb_clk);
-	if (ret) {
-		pr_err("%s: failed to enable ahb clock, %d\n", __func__, ret);
-		goto ahb_clk_err;
-	}
-
-	ret = clk_prepare_enable(msm_host->axi_clk);
-	if (ret) {
-		pr_err("%s: failed to enable ahb clock, %d\n", __func__, ret);
-		goto axi_clk_err;
-	}
-
-	ret = clk_prepare_enable(msm_host->mmss_misc_ahb_clk);
-	if (ret) {
-		pr_err("%s: failed to enable mmss misc ahb clk, %d\n",
-			__func__, ret);
-		goto misc_ahb_clk_err;
+	for (i = 0; i < cfg->num_bus_clks; i++) {
+		ret = clk_prepare_enable(msm_host->bus_clks[i]);
+		if (ret) {
+			pr_err("%s: failed to enable bus clock %d ret %d\n",
+				__func__, i, ret);
+			goto err;
+		}
 	}
 
 	return 0;
+err:
+	for (; i > 0; i--)
+		clk_disable_unprepare(msm_host->bus_clks[i]);
 
-misc_ahb_clk_err:
-	clk_disable_unprepare(msm_host->axi_clk);
-axi_clk_err:
-	clk_disable_unprepare(msm_host->ahb_clk);
-ahb_clk_err:
-	clk_disable_unprepare(msm_host->mdp_core_clk);
-core_clk_err:
 	return ret;
 }
 
 static void dsi_bus_clk_disable(struct msm_dsi_host *msm_host)
 {
+	const struct msm_dsi_config *cfg = msm_host->cfg_hnd->cfg;
+	int i;
+
 	DBG("");
-	clk_disable_unprepare(msm_host->mmss_misc_ahb_clk);
-	clk_disable_unprepare(msm_host->axi_clk);
-	clk_disable_unprepare(msm_host->ahb_clk);
-	clk_disable_unprepare(msm_host->mdp_core_clk);
+
+	for (i = cfg->num_bus_clks - 1; i >= 0; i--)
+		clk_disable_unprepare(msm_host->bus_clks[i]);
 }
 
-static int dsi_link_clk_enable(struct msm_dsi_host *msm_host)
+static int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host)
 {
 	int ret;
 
@@ -476,11 +508,98 @@ error:
 	return ret;
 }
 
-static void dsi_link_clk_disable(struct msm_dsi_host *msm_host)
+static int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host)
 {
+	int ret;
+
+	DBG("Set clk rates: pclk=%d, byteclk=%d, esc_clk=%d, dsi_src_clk=%d",
+		msm_host->mode->clock, msm_host->byte_clk_rate,
+		msm_host->esc_clk_rate, msm_host->src_clk_rate);
+
+	ret = clk_set_rate(msm_host->byte_clk, msm_host->byte_clk_rate);
+	if (ret) {
+		pr_err("%s: Failed to set rate byte clk, %d\n", __func__, ret);
+		goto error;
+	}
+
+	ret = clk_set_rate(msm_host->esc_clk, msm_host->esc_clk_rate);
+	if (ret) {
+		pr_err("%s: Failed to set rate esc clk, %d\n", __func__, ret);
+		goto error;
+	}
+
+	ret = clk_set_rate(msm_host->src_clk, msm_host->src_clk_rate);
+	if (ret) {
+		pr_err("%s: Failed to set rate src clk, %d\n", __func__, ret);
+		goto error;
+	}
+
+	ret = clk_set_rate(msm_host->pixel_clk, msm_host->mode->clock * 1000);
+	if (ret) {
+		pr_err("%s: Failed to set rate pixel clk, %d\n", __func__, ret);
+		goto error;
+	}
+
+	ret = clk_prepare_enable(msm_host->byte_clk);
+	if (ret) {
+		pr_err("%s: Failed to enable dsi byte clk\n", __func__);
+		goto error;
+	}
+
+	ret = clk_prepare_enable(msm_host->esc_clk);
+	if (ret) {
+		pr_err("%s: Failed to enable dsi esc clk\n", __func__);
+		goto esc_clk_err;
+	}
+
+	ret = clk_prepare_enable(msm_host->src_clk);
+	if (ret) {
+		pr_err("%s: Failed to enable dsi src clk\n", __func__);
+		goto src_clk_err;
+	}
+
+	ret = clk_prepare_enable(msm_host->pixel_clk);
+	if (ret) {
+		pr_err("%s: Failed to enable dsi pixel clk\n", __func__);
+		goto pixel_clk_err;
+	}
+
+	return 0;
+
+pixel_clk_err:
+	clk_disable_unprepare(msm_host->src_clk);
+src_clk_err:
 	clk_disable_unprepare(msm_host->esc_clk);
-	clk_disable_unprepare(msm_host->pixel_clk);
+esc_clk_err:
 	clk_disable_unprepare(msm_host->byte_clk);
+error:
+	return ret;
+}
+
+static int dsi_link_clk_enable(struct msm_dsi_host *msm_host)
+{
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G)
+		return dsi_link_clk_enable_6g(msm_host);
+	else
+		return dsi_link_clk_enable_v2(msm_host);
+}
+
+static void dsi_link_clk_disable(struct msm_dsi_host *msm_host)
+{
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
+		clk_disable_unprepare(msm_host->esc_clk);
+		clk_disable_unprepare(msm_host->pixel_clk);
+		clk_disable_unprepare(msm_host->byte_clk);
+	} else {
+		clk_disable_unprepare(msm_host->pixel_clk);
+		clk_disable_unprepare(msm_host->src_clk);
+		clk_disable_unprepare(msm_host->esc_clk);
+		clk_disable_unprepare(msm_host->byte_clk);
+	}
 }
 
 static int dsi_clk_ctrl(struct msm_dsi_host *msm_host, bool enable)
@@ -515,6 +634,7 @@ unlock_ret:
 static int dsi_calc_clk_rate(struct msm_dsi_host *msm_host)
 {
 	struct drm_display_mode *mode = msm_host->mode;
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	u8 lanes = msm_host->lanes;
 	u32 bpp = dsi_get_bpp(msm_host->format);
 	u32 pclk_rate;
@@ -534,6 +654,47 @@ static int dsi_calc_clk_rate(struct msm_dsi_host *msm_host)
 
 	DBG("pclk=%d, bclk=%d", pclk_rate, msm_host->byte_clk_rate);
 
+	msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk);
+
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
+		unsigned int esc_mhz, esc_div;
+		unsigned long byte_mhz;
+
+		msm_host->src_clk_rate = (pclk_rate * bpp) / 8;
+
+		/*
+		 * esc clock is byte clock followed by a 4 bit divider,
+		 * we need to find an escape clock frequency within the
+		 * mipi DSI spec range within the maximum divider limit
+		 * We iterate here between an escape clock frequencey
+		 * between 20 Mhz to 5 Mhz and pick up the first one
+		 * that can be supported by our divider
+		 */
+
+		byte_mhz = msm_host->byte_clk_rate / 1000000;
+
+		for (esc_mhz = 20; esc_mhz >= 5; esc_mhz--) {
+			esc_div = DIV_ROUND_UP(byte_mhz, esc_mhz);
+
+			/*
+			 * TODO: Ideally, we shouldn't know what sort of divider
+			 * is available in mmss_cc, we're just assuming that
+			 * it'll always be a 4 bit divider. Need to come up with
+			 * a better way here.
+			 */
+			if (esc_div >= 1 && esc_div <= 16)
+				break;
+		}
+
+		if (esc_mhz < 5)
+			return -EINVAL;
+
+		msm_host->esc_clk_rate = msm_host->byte_clk_rate / esc_div;
+
+		DBG("esc=%d, src=%d", msm_host->esc_clk_rate,
+			msm_host->src_clk_rate);
+	}
+
 	return 0;
 }
 
@@ -835,29 +996,46 @@ static void dsi_wait4video_eng_busy(struct msm_dsi_host *msm_host)
 static int dsi_tx_buf_alloc(struct msm_dsi_host *msm_host, int size)
 {
 	struct drm_device *dev = msm_host->dev;
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
 	u32 iova;
 
-	mutex_lock(&dev->struct_mutex);
-	msm_host->tx_gem_obj = msm_gem_new(dev, size, MSM_BO_UNCACHED);
-	if (IS_ERR(msm_host->tx_gem_obj)) {
-		ret = PTR_ERR(msm_host->tx_gem_obj);
-		pr_err("%s: failed to allocate gem, %d\n", __func__, ret);
-		msm_host->tx_gem_obj = NULL;
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
+		mutex_lock(&dev->struct_mutex);
+		msm_host->tx_gem_obj = msm_gem_new(dev, size, MSM_BO_UNCACHED);
+		if (IS_ERR(msm_host->tx_gem_obj)) {
+			ret = PTR_ERR(msm_host->tx_gem_obj);
+			pr_err("%s: failed to allocate gem, %d\n",
+				__func__, ret);
+			msm_host->tx_gem_obj = NULL;
+			mutex_unlock(&dev->struct_mutex);
+			return ret;
+		}
+
+		ret = msm_gem_get_iova_locked(msm_host->tx_gem_obj, 0, &iova);
 		mutex_unlock(&dev->struct_mutex);
-		return ret;
-	}
+		if (ret) {
+			pr_err("%s: failed to get iova, %d\n", __func__, ret);
+			return ret;
+		}
 
-	ret = msm_gem_get_iova_locked(msm_host->tx_gem_obj, 0, &iova);
-	if (ret) {
-		pr_err("%s: failed to get iova, %d\n", __func__, ret);
-		return ret;
-	}
-	mutex_unlock(&dev->struct_mutex);
+		if (iova & 0x07) {
+			pr_err("%s: buf NOT 8 bytes aligned\n", __func__);
+			return -EINVAL;
+		}
 
-	if (iova & 0x07) {
-		pr_err("%s: buf NOT 8 bytes aligned\n", __func__);
-		return -EINVAL;
+		msm_host->tx_size = msm_host->tx_gem_obj->size;
+	} else {
+		msm_host->tx_buf = dma_alloc_coherent(dev->dev, size,
+					&msm_host->tx_buf_paddr, GFP_KERNEL);
+		if (!msm_host->tx_buf) {
+			ret = -ENOMEM;
+			pr_err("%s: failed to allocate tx buf, %d\n",
+				__func__, ret);
+			return ret;
+		}
+
+		msm_host->tx_size = size;
 	}
 
 	return 0;
@@ -874,14 +1052,19 @@ static void dsi_tx_buf_free(struct msm_dsi_host *msm_host)
 		msm_host->tx_gem_obj = NULL;
 		mutex_unlock(&dev->struct_mutex);
 	}
+
+	if (msm_host->tx_buf)
+		dma_free_coherent(dev->dev, msm_host->tx_size, msm_host->tx_buf,
+			msm_host->tx_buf_paddr);
 }
 
 /*
  * prepare cmd buffer to be txed
  */
-static int dsi_cmd_dma_add(struct drm_gem_object *tx_gem,
-			const struct mipi_dsi_msg *msg)
+static int dsi_cmd_dma_add(struct msm_dsi_host *msm_host,
+			   const struct mipi_dsi_msg *msg)
 {
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	struct mipi_dsi_packet packet;
 	int len;
 	int ret;
@@ -894,17 +1077,20 @@ static int dsi_cmd_dma_add(struct drm_gem_object *tx_gem,
 	}
 	len = (packet.size + 3) & (~0x3);
 
-	if (len > tx_gem->size) {
+	if (len > msm_host->tx_size) {
 		pr_err("%s: packet size is too big\n", __func__);
 		return -EINVAL;
 	}
 
-	data = msm_gem_vaddr(tx_gem);
-
-	if (IS_ERR(data)) {
-		ret = PTR_ERR(data);
-		pr_err("%s: get vaddr failed, %d\n", __func__, ret);
-		return ret;
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
+		data = msm_gem_vaddr(msm_host->tx_gem_obj);
+		if (IS_ERR(data)) {
+			ret = PTR_ERR(data);
+			pr_err("%s: get vaddr failed, %d\n", __func__, ret);
+			return ret;
+		}
+	} else {
+		data = msm_host->tx_buf;
 	}
 
 	/* MSM specific command format in memory */
@@ -970,17 +1156,21 @@ static int dsi_long_read_resp(u8 *buf, const struct mipi_dsi_msg *msg)
 	return msg->rx_len;
 }
 
-
 static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len)
 {
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
-	u32 iova;
+	u32 dma_base;
 	bool triggered;
 
-	ret = msm_gem_get_iova(msm_host->tx_gem_obj, 0, &iova);
-	if (ret) {
-		pr_err("%s: failed to get iova: %d\n", __func__, ret);
-		return ret;
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
+		ret = msm_gem_get_iova(msm_host->tx_gem_obj, 0, &dma_base);
+		if (ret) {
+			pr_err("%s: failed to get iova: %d\n", __func__, ret);
+			return ret;
+		}
+	} else {
+		dma_base = msm_host->tx_buf_paddr;
 	}
 
 	reinit_completion(&msm_host->dma_comp);
@@ -988,7 +1178,7 @@ static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len)
 	dsi_wait4video_eng_busy(msm_host);
 
 	triggered = msm_dsi_manager_cmd_xfer_trigger(
-						msm_host->id, iova, len);
+						msm_host->id, dma_base, len);
 	if (triggered) {
 		ret = wait_for_completion_timeout(&msm_host->dma_comp,
 					msecs_to_jiffies(200));
@@ -1060,7 +1250,7 @@ static int dsi_cmds2buf_tx(struct msm_dsi_host *msm_host,
 	int bllp_len = msm_host->mode->hdisplay *
 			dsi_get_bpp(msm_host->format) / 8;
 
-	len = dsi_cmd_dma_add(msm_host->tx_gem_obj, msg);
+	len = dsi_cmd_dma_add(msm_host, msg);
 	if (!len) {
 		pr_err("%s: failed to add cmd type = 0x%x\n",
 			__func__,  msg->type);
@@ -1383,6 +1573,16 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host)
 
 	msm_host->device_node = device_node;
 
+	if (of_property_read_bool(np, "syscon-sfpb")) {
+		msm_host->sfpb = syscon_regmap_lookup_by_phandle(np,
+					"syscon-sfpb");
+		if (IS_ERR(msm_host->sfpb)) {
+			dev_err(dev, "%s: failed to get sfpb regmap\n",
+				__func__);
+			return PTR_ERR(msm_host->sfpb);
+		}
+	}
+
 	return 0;
 }
 
@@ -1408,12 +1608,6 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 		goto fail;
 	}
 
-	ret = dsi_clk_init(msm_host);
-	if (ret) {
-		pr_err("%s: unable to initialize dsi clks\n", __func__);
-		goto fail;
-	}
-
 	msm_host->ctrl_base = msm_ioremap(pdev, "dsi_ctrl", "DSI CTRL");
 	if (IS_ERR(msm_host->ctrl_base)) {
 		pr_err("%s: unable to map Dsi ctrl base\n", __func__);
@@ -1437,6 +1631,12 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 		goto fail;
 	}
 
+	ret = dsi_clk_init(msm_host);
+	if (ret) {
+		pr_err("%s: unable to initialize dsi clks\n", __func__);
+		goto fail;
+	}
+
 	msm_host->rx_buf = devm_kzalloc(&pdev->dev, SZ_4K, GFP_KERNEL);
 	if (!msm_host->rx_buf) {
 		pr_err("%s: alloc rx temp buf failed\n", __func__);
@@ -1750,11 +1950,12 @@ int msm_dsi_host_cmd_rx(struct mipi_dsi_host *host,
 	return ret;
 }
 
-void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host, u32 iova, u32 len)
+void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host, u32 dma_base,
+				  u32 len)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
 
-	dsi_write(msm_host, REG_DSI_DMA_BASE, iova);
+	dsi_write(msm_host, REG_DSI_DMA_BASE, dma_base);
 	dsi_write(msm_host, REG_DSI_DMA_LEN, len);
 	dsi_write(msm_host, REG_DSI_TRIG_DMA, 1);
 
@@ -1766,6 +1967,7 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host,
 	struct msm_dsi_pll *src_pll)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	struct clk *byte_clk_provider, *pixel_clk_provider;
 	int ret;
 
@@ -1791,6 +1993,22 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host,
 		goto exit;
 	}
 
+	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
+		ret = clk_set_parent(msm_host->dsi_clk_src, pixel_clk_provider);
+		if (ret) {
+			pr_err("%s: can't set parent to dsi_clk_src. ret=%d\n",
+				__func__, ret);
+			goto exit;
+		}
+
+		ret = clk_set_parent(msm_host->esc_clk_src, byte_clk_provider);
+		if (ret) {
+			pr_err("%s: can't set parent to esc_clk_src. ret=%d\n",
+				__func__, ret);
+			goto exit;
+		}
+	}
+
 exit:
 	return ret;
 }
@@ -1828,6 +2046,20 @@ int msm_dsi_host_disable(struct mipi_dsi_host *host)
 	return 0;
 }
 
+static void msm_dsi_sfpb_config(struct msm_dsi_host *msm_host, bool enable)
+{
+	enum sfpb_ahb_arb_master_port_en en;
+
+	if (!msm_host->sfpb)
+		return;
+
+	en = enable ? SFPB_MASTER_PORT_ENABLE : SFPB_MASTER_PORT_DISABLE;
+
+	regmap_update_bits(msm_host->sfpb, REG_SFPB_GPREG,
+			SFPB_GPREG_MASTER_PORT_EN__MASK,
+			SFPB_GPREG_MASTER_PORT_EN(en));
+}
+
 int msm_dsi_host_power_on(struct mipi_dsi_host *host)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
@@ -1840,6 +2072,8 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host)
 		goto unlock_ret;
 	}
 
+	msm_dsi_sfpb_config(msm_host, true);
+
 	ret = dsi_calc_clk_rate(msm_host);
 	if (ret) {
 		pr_err("%s: unable to calc clk rate, %d\n", __func__, ret);
@@ -1862,7 +2096,7 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host)
 	dsi_phy_sw_reset(msm_host);
 	ret = msm_dsi_manager_phy_enable(msm_host->id,
 					msm_host->byte_clk_rate * 8,
-					clk_get_rate(msm_host->esc_clk),
+					msm_host->esc_clk_rate,
 					&clk_pre, &clk_post);
 	dsi_bus_clk_disable(msm_host);
 	if (ret) {
@@ -1927,6 +2161,8 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host)
 
 	dsi_host_regulator_disable(msm_host);
 
+	msm_dsi_sfpb_config(msm_host, false);
+
 	DBG("-");
 
 	msm_host->power_on = false;
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index 0455ff75074a..58ba7ec17f51 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -774,7 +774,7 @@ restore_host0:
 	return ret;
 }
 
-bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 iova, u32 len)
+bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len)
 {
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct msm_dsi *msm_dsi0 = dsi_mgr_get_dsi(DSI_0);
@@ -784,9 +784,9 @@ bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 iova, u32 len)
 		return false;
 
 	if (IS_SYNC_NEEDED() && msm_dsi0)
-		msm_dsi_host_cmd_xfer_commit(msm_dsi0->host, iova, len);
+		msm_dsi_host_cmd_xfer_commit(msm_dsi0->host, dma_base, len);
 
-	msm_dsi_host_cmd_xfer_commit(host, iova, len);
+	msm_dsi_host_cmd_xfer_commit(host, dma_base, len);
 
 	return true;
 }
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
index f1f955f571fa..91a95fb04a4a 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
@@ -277,6 +277,10 @@ static const struct of_device_id dsi_phy_dt_match[] = {
 	{ .compatible = "qcom,dsi-phy-20nm",
 	  .data = &dsi_phy_20nm_cfgs },
 #endif
+#ifdef CONFIG_DRM_MSM_DSI_28NM_8960_PHY
+	{ .compatible = "qcom,dsi-phy-28nm-8960",
+	  .data = &dsi_phy_28nm_8960_cfgs },
+#endif
 	{}
 };
 
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index 0456b253239f..0d54ed00386d 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -43,6 +43,7 @@ struct msm_dsi_phy_cfg {
 extern const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs;
+extern const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs;
 
 struct msm_dsi_dphy_timing {
 	u32 clk_pre;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
new file mode 100644
index 000000000000..197b039ca1f1
--- /dev/null
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dsi_phy.h"
+#include "dsi.xml.h"
+
+static void dsi_28nm_dphy_set_timing(struct msm_dsi_phy *phy,
+		struct msm_dsi_dphy_timing *timing)
+{
+	void __iomem *base = phy->base;
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_0,
+		DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_1,
+		DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_2,
+		DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_3, 0x0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_4,
+		DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_5,
+		DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_6,
+		DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_7,
+		DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_8,
+		DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_9,
+		DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) |
+		DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_10,
+		DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get));
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_11,
+		DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
+}
+
+static void dsi_28nm_phy_regulator_init(struct msm_dsi_phy *phy)
+{
+	void __iomem *base = phy->reg_base;
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_0, 0x3);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_1, 1);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2, 1);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3, 0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4,
+		0x100);
+}
+
+static void dsi_28nm_phy_regulator_ctrl(struct msm_dsi_phy *phy)
+{
+	void __iomem *base = phy->reg_base;
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_0, 0x3);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_1, 0xa);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2, 0x4);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3, 0x0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4, 0x20);
+}
+
+static void dsi_28nm_phy_calibration(struct msm_dsi_phy *phy)
+{
+	void __iomem *base = phy->reg_base;
+	u32 status;
+	int i = 5000;
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CAL_PWR_CFG,
+			0x3);
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_2, 0x0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_1, 0x5a);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_3, 0x10);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_4, 0x1);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_0, 0x1);
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_TRIGGER, 0x1);
+	usleep_range(5000, 6000);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_TRIGGER, 0x0);
+
+	do {
+		status = dsi_phy_read(base +
+				REG_DSI_28nm_8960_PHY_MISC_CAL_STATUS);
+
+		if (!(status & DSI_28nm_8960_PHY_MISC_CAL_STATUS_CAL_BUSY))
+			break;
+
+		udelay(1);
+	} while (--i > 0);
+}
+
+static void dsi_28nm_phy_lane_config(struct msm_dsi_phy *phy)
+{
+	void __iomem *base = phy->base;
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_0(i), 0x80);
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_1(i), 0x45);
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_2(i), 0x00);
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_DATAPATH(i),
+			0x00);
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_0(i),
+			0x01);
+		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_1(i),
+			0x66);
+	}
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_CFG_0, 0x40);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_CFG_1, 0x67);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_CFG_2, 0x0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_TEST_DATAPATH, 0x0);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_TEST_STR0, 0x1);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_TEST_STR1, 0x88);
+}
+
+static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy, int src_pll_id,
+		const unsigned long bit_rate, const unsigned long esc_rate)
+{
+	struct msm_dsi_dphy_timing *timing = &phy->timing;
+	void __iomem *base = phy->base;
+
+	DBG("");
+
+	if (msm_dsi_dphy_timing_calc(timing, bit_rate, esc_rate)) {
+		dev_err(&phy->pdev->dev,
+			"%s: D-PHY timing calculation failed\n", __func__);
+		return -EINVAL;
+	}
+
+	dsi_28nm_phy_regulator_init(phy);
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LDO_CTRL, 0x04);
+
+	/* strength control */
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_STRENGTH_0, 0xff);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_STRENGTH_1, 0x00);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_STRENGTH_2, 0x06);
+
+	/* phy ctrl */
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_CTRL_0, 0x5f);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_CTRL_1, 0x00);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_CTRL_2, 0x00);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_CTRL_3, 0x10);
+
+	dsi_28nm_phy_regulator_ctrl(phy);
+
+	dsi_28nm_phy_calibration(phy);
+
+	dsi_28nm_phy_lane_config(phy);
+
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_BIST_CTRL_4, 0x0f);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_BIST_CTRL_1, 0x03);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_BIST_CTRL_0, 0x03);
+	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_BIST_CTRL_4, 0x0);
+
+	dsi_28nm_dphy_set_timing(phy, timing);
+
+	return 0;
+}
+
+static void dsi_28nm_phy_disable(struct msm_dsi_phy *phy)
+{
+	dsi_phy_write(phy->base + REG_DSI_28nm_8960_PHY_CTRL_0, 0x0);
+
+	/*
+	 * Wait for the registers writes to complete in order to
+	 * ensure that the phy is completely disabled
+	 */
+	wmb();
+}
+
+const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs = {
+	.type = MSM_DSI_PHY_28NM_8960,
+	.src_pll_truthtable = { {true, true}, {false, true} },
+	.reg_cfg = {
+		.num = 1,
+		.regs = {
+			{"vddio", 1800000, 1800000, 100000, 100},
+		},
+	},
+	.ops = {
+		.enable = dsi_28nm_phy_enable,
+		.disable = dsi_28nm_phy_disable,
+	},
+};
diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll.c
index 5104fc9f9a53..5cd438f91afe 100644
--- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll.c
+++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll.c
@@ -151,6 +151,9 @@ struct msm_dsi_pll *msm_dsi_pll_init(struct platform_device *pdev,
 	case MSM_DSI_PHY_28NM_LP:
 		pll = msm_dsi_pll_28nm_init(pdev, type, id);
 		break;
+	case MSM_DSI_PHY_28NM_8960:
+		pll = msm_dsi_pll_28nm_8960_init(pdev, id);
+		break;
 	default:
 		pll = ERR_PTR(-ENXIO);
 		break;
diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll.h b/drivers/gpu/drm/msm/dsi/pll/dsi_pll.h
index 063caa2c5740..80b6038334a6 100644
--- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll.h
+++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll.h
@@ -93,6 +93,16 @@ static inline struct msm_dsi_pll *msm_dsi_pll_28nm_init(
 	return ERR_PTR(-ENODEV);
 }
 #endif
+#ifdef CONFIG_DRM_MSM_DSI_28NM_8960_PHY
+struct msm_dsi_pll *msm_dsi_pll_28nm_8960_init(struct platform_device *pdev,
+					       int id);
+#else
+struct msm_dsi_pll *msm_dsi_pll_28nm_8960_init(struct platform_device *pdev,
+					       int id)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
 
 #endif /* __DSI_PLL_H__ */
 
diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm_8960.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm_8960.c
new file mode 100644
index 000000000000..38c90e1eb002
--- /dev/null
+++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm_8960.c
@@ -0,0 +1,533 @@
+/*
+ * Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+
+#include "dsi_pll.h"
+#include "dsi.xml.h"
+
+/*
+ * DSI PLL 28nm (8960/A family) - clock diagram (eg: DSI1):
+ *
+ *
+ *                        +------+
+ *  dsi1vco_clk ----o-----| DIV1 |---dsi1pllbit (not exposed as clock)
+ *  F * byte_clk    |     +------+
+ *                  | bit clock divider (F / 8)
+ *                  |
+ *                  |     +------+
+ *                  o-----| DIV2 |---dsi0pllbyte---o---> To byte RCG
+ *                  |     +------+                 | (sets parent rate)
+ *                  | byte clock divider (F)       |
+ *                  |                              |
+ *                  |                              o---> To esc RCG
+ *                  |                                (doesn't set parent rate)
+ *                  |
+ *                  |     +------+
+ *                  o-----| DIV3 |----dsi0pll------o---> To dsi RCG
+ *                        +------+                 | (sets parent rate)
+ *                  dsi clock divider (F * magic)  |
+ *                                                 |
+ *                                                 o---> To pixel rcg
+ *                                                  (doesn't set parent rate)
+ */
+
+#define POLL_MAX_READS		8000
+#define POLL_TIMEOUT_US		1
+
+#define NUM_PROVIDED_CLKS	2
+
+#define VCO_REF_CLK_RATE	27000000
+#define VCO_MIN_RATE		600000000
+#define VCO_MAX_RATE		1200000000
+
+#define DSI_BYTE_PLL_CLK	0
+#define DSI_PIXEL_PLL_CLK	1
+
+#define VCO_PREF_DIV_RATIO	27
+
+struct pll_28nm_cached_state {
+	unsigned long vco_rate;
+	u8 postdiv3;
+	u8 postdiv2;
+	u8 postdiv1;
+};
+
+struct clk_bytediv {
+	struct clk_hw hw;
+	void __iomem *reg;
+};
+
+struct dsi_pll_28nm {
+	struct msm_dsi_pll base;
+
+	int id;
+	struct platform_device *pdev;
+	void __iomem *mmio;
+
+	/* custom byte clock divider */
+	struct clk_bytediv *bytediv;
+
+	/* private clocks: */
+	struct clk *clks[NUM_DSI_CLOCKS_MAX];
+	u32 num_clks;
+
+	/* clock-provider: */
+	struct clk *provided_clks[NUM_PROVIDED_CLKS];
+	struct clk_onecell_data clk_data;
+
+	struct pll_28nm_cached_state cached_state;
+};
+
+#define to_pll_28nm(x)	container_of(x, struct dsi_pll_28nm, base)
+
+static bool pll_28nm_poll_for_ready(struct dsi_pll_28nm *pll_28nm,
+				    int nb_tries, int timeout_us)
+{
+	bool pll_locked = false;
+	u32 val;
+
+	while (nb_tries--) {
+		val = pll_read(pll_28nm->mmio + REG_DSI_28nm_8960_PHY_PLL_RDY);
+		pll_locked = !!(val & DSI_28nm_8960_PHY_PLL_RDY_PLL_RDY);
+
+		if (pll_locked)
+			break;
+
+		udelay(timeout_us);
+	}
+	DBG("DSI PLL is %slocked", pll_locked ? "" : "*not* ");
+
+	return pll_locked;
+}
+
+/*
+ * Clock Callbacks
+ */
+static int dsi_pll_28nm_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct msm_dsi_pll *pll = hw_clk_to_pll(hw);
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+	void __iomem *base = pll_28nm->mmio;
+	u32 val, temp, fb_divider;
+
+	DBG("rate=%lu, parent's=%lu", rate, parent_rate);
+
+	temp = rate / 10;
+	val = VCO_REF_CLK_RATE / 10;
+	fb_divider = (temp * VCO_PREF_DIV_RATIO) / val;
+	fb_divider = fb_divider / 2 - 1;
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_1,
+			fb_divider & 0xff);
+
+	val = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2);
+
+	val |= (fb_divider >> 8) & 0x07;
+
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2,
+			val);
+
+	val = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3);
+
+	val |= (VCO_PREF_DIV_RATIO - 1) & 0x3f;
+
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3,
+			val);
+
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_6,
+			0xf);
+
+	val = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8);
+	val |= 0x7 << 4;
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8,
+			val);
+
+	return 0;
+}
+
+static int dsi_pll_28nm_clk_is_enabled(struct clk_hw *hw)
+{
+	struct msm_dsi_pll *pll = hw_clk_to_pll(hw);
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+
+	return pll_28nm_poll_for_ready(pll_28nm, POLL_MAX_READS,
+					POLL_TIMEOUT_US);
+}
+
+static unsigned long dsi_pll_28nm_clk_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct msm_dsi_pll *pll = hw_clk_to_pll(hw);
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+	void __iomem *base = pll_28nm->mmio;
+	unsigned long vco_rate;
+	u32 status, fb_divider, temp, ref_divider;
+
+	VERB("parent_rate=%lu", parent_rate);
+
+	status = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_0);
+
+	if (status & DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE) {
+		fb_divider = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_1);
+		fb_divider &= 0xff;
+		temp = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2) & 0x07;
+		fb_divider = (temp << 8) | fb_divider;
+		fb_divider += 1;
+
+		ref_divider = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3);
+		ref_divider &= 0x3f;
+		ref_divider += 1;
+
+		/* multiply by 2 */
+		vco_rate = (parent_rate / ref_divider) * fb_divider * 2;
+	} else {
+		vco_rate = 0;
+	}
+
+	DBG("returning vco rate = %lu", vco_rate);
+
+	return vco_rate;
+}
+
+static const struct clk_ops clk_ops_dsi_pll_28nm_vco = {
+	.round_rate = msm_dsi_pll_helper_clk_round_rate,
+	.set_rate = dsi_pll_28nm_clk_set_rate,
+	.recalc_rate = dsi_pll_28nm_clk_recalc_rate,
+	.prepare = msm_dsi_pll_helper_clk_prepare,
+	.unprepare = msm_dsi_pll_helper_clk_unprepare,
+	.is_enabled = dsi_pll_28nm_clk_is_enabled,
+};
+
+/*
+ * Custom byte clock divier clk_ops
+ *
+ * This clock is the entry point to configuring the PLL. The user (dsi host)
+ * will set this clock's rate to the desired byte clock rate. The VCO lock
+ * frequency is a multiple of the byte clock rate. The multiplication factor
+ * (shown as F in the diagram above) is a function of the byte clock rate.
+ *
+ * This custom divider clock ensures that its parent (VCO) is set to the
+ * desired rate, and that the byte clock postdivider (POSTDIV2) is configured
+ * accordingly
+ */
+#define to_clk_bytediv(_hw) container_of(_hw, struct clk_bytediv, hw)
+
+static unsigned long clk_bytediv_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct clk_bytediv *bytediv = to_clk_bytediv(hw);
+	unsigned int div;
+
+	div = pll_read(bytediv->reg) & 0xff;
+
+	return parent_rate / (div + 1);
+}
+
+/* find multiplication factor(wrt byte clock) at which the VCO should be set */
+static unsigned int get_vco_mul_factor(unsigned long byte_clk_rate)
+{
+	unsigned long bit_mhz;
+
+	/* convert to bit clock in Mhz */
+	bit_mhz = (byte_clk_rate * 8) / 1000000;
+
+	if (bit_mhz < 125)
+		return 64;
+	else if (bit_mhz < 250)
+		return 32;
+	else if (bit_mhz < 600)
+		return 16;
+	else
+		return 8;
+}
+
+static long clk_bytediv_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *prate)
+{
+	unsigned long best_parent;
+	unsigned int factor;
+
+	factor = get_vco_mul_factor(rate);
+
+	best_parent = rate * factor;
+	*prate = clk_hw_round_rate(clk_hw_get_parent(hw), best_parent);
+
+	return *prate / factor;
+}
+
+static int clk_bytediv_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct clk_bytediv *bytediv = to_clk_bytediv(hw);
+	u32 val;
+	unsigned int factor;
+
+	factor = get_vco_mul_factor(rate);
+
+	val = pll_read(bytediv->reg);
+	val |= (factor - 1) & 0xff;
+	pll_write(bytediv->reg, val);
+
+	return 0;
+}
+
+/* Our special byte clock divider ops */
+static const struct clk_ops clk_bytediv_ops = {
+	.round_rate = clk_bytediv_round_rate,
+	.set_rate = clk_bytediv_set_rate,
+	.recalc_rate = clk_bytediv_recalc_rate,
+};
+
+/*
+ * PLL Callbacks
+ */
+static int dsi_pll_28nm_enable_seq(struct msm_dsi_pll *pll)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+	struct device *dev = &pll_28nm->pdev->dev;
+	void __iomem *base = pll_28nm->mmio;
+	bool locked;
+	unsigned int bit_div, byte_div;
+	int max_reads = 1000, timeout_us = 100;
+	u32 val;
+
+	DBG("id=%d", pll_28nm->id);
+
+	/*
+	 * before enabling the PLL, configure the bit clock divider since we
+	 * don't expose it as a clock to the outside world
+	 * 1: read back the byte clock divider that should already be set
+	 * 2: divide by 8 to get bit clock divider
+	 * 3: write it to POSTDIV1
+	 */
+	val = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9);
+	byte_div = val + 1;
+	bit_div = byte_div / 8;
+
+	val = pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8);
+	val &= ~0xf;
+	val |= (bit_div - 1);
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8, val);
+
+	/* enable the PLL */
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_0,
+			DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE);
+
+	locked = pll_28nm_poll_for_ready(pll_28nm, max_reads, timeout_us);
+
+	if (unlikely(!locked))
+		dev_err(dev, "DSI PLL lock failed\n");
+	else
+		DBG("DSI PLL lock success");
+
+	return locked ? 0 : -EINVAL;
+}
+
+static void dsi_pll_28nm_disable_seq(struct msm_dsi_pll *pll)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+
+	DBG("id=%d", pll_28nm->id);
+	pll_write(pll_28nm->mmio + REG_DSI_28nm_8960_PHY_PLL_CTRL_0, 0x00);
+}
+
+static void dsi_pll_28nm_save_state(struct msm_dsi_pll *pll)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+	struct pll_28nm_cached_state *cached_state = &pll_28nm->cached_state;
+	void __iomem *base = pll_28nm->mmio;
+
+	cached_state->postdiv3 =
+			pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_10);
+	cached_state->postdiv2 =
+			pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9);
+	cached_state->postdiv1 =
+			pll_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8);
+
+	cached_state->vco_rate = clk_hw_get_rate(&pll->clk_hw);
+}
+
+static int dsi_pll_28nm_restore_state(struct msm_dsi_pll *pll)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+	struct pll_28nm_cached_state *cached_state = &pll_28nm->cached_state;
+	void __iomem *base = pll_28nm->mmio;
+	int ret;
+
+	ret = dsi_pll_28nm_clk_set_rate(&pll->clk_hw,
+					cached_state->vco_rate, 0);
+	if (ret) {
+		dev_err(&pll_28nm->pdev->dev,
+			"restore vco rate failed. ret=%d\n", ret);
+		return ret;
+	}
+
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_10,
+			cached_state->postdiv3);
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9,
+			cached_state->postdiv2);
+	pll_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8,
+			cached_state->postdiv1);
+
+	return 0;
+}
+
+static int dsi_pll_28nm_get_provider(struct msm_dsi_pll *pll,
+				struct clk **byte_clk_provider,
+				struct clk **pixel_clk_provider)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+
+	if (byte_clk_provider)
+		*byte_clk_provider = pll_28nm->provided_clks[DSI_BYTE_PLL_CLK];
+	if (pixel_clk_provider)
+		*pixel_clk_provider =
+				pll_28nm->provided_clks[DSI_PIXEL_PLL_CLK];
+
+	return 0;
+}
+
+static void dsi_pll_28nm_destroy(struct msm_dsi_pll *pll)
+{
+	struct dsi_pll_28nm *pll_28nm = to_pll_28nm(pll);
+
+	msm_dsi_pll_helper_unregister_clks(pll_28nm->pdev,
+					pll_28nm->clks, pll_28nm->num_clks);
+}
+
+static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm)
+{
+	char *clk_name, *parent_name, *vco_name;
+	struct clk_init_data vco_init = {
+		.parent_names = (const char *[]){ "pxo" },
+		.num_parents = 1,
+		.ops = &clk_ops_dsi_pll_28nm_vco,
+	};
+	struct device *dev = &pll_28nm->pdev->dev;
+	struct clk **clks = pll_28nm->clks;
+	struct clk **provided_clks = pll_28nm->provided_clks;
+	struct clk_bytediv *bytediv;
+	struct clk_init_data bytediv_init = { };
+	int ret, num = 0;
+
+	DBG("%d", pll_28nm->id);
+
+	bytediv = devm_kzalloc(dev, sizeof(*bytediv), GFP_KERNEL);
+	if (!bytediv)
+		return -ENOMEM;
+
+	vco_name = devm_kzalloc(dev, 32, GFP_KERNEL);
+	if (!vco_name)
+		return -ENOMEM;
+
+	parent_name = devm_kzalloc(dev, 32, GFP_KERNEL);
+	if (!parent_name)
+		return -ENOMEM;
+
+	clk_name = devm_kzalloc(dev, 32, GFP_KERNEL);
+	if (!clk_name)
+		return -ENOMEM;
+
+	pll_28nm->bytediv = bytediv;
+
+	snprintf(vco_name, 32, "dsi%dvco_clk", pll_28nm->id);
+	vco_init.name = vco_name;
+
+	pll_28nm->base.clk_hw.init = &vco_init;
+
+	clks[num++] = clk_register(dev, &pll_28nm->base.clk_hw);
+
+	/* prepare and register bytediv */
+	bytediv->hw.init = &bytediv_init;
+	bytediv->reg = pll_28nm->mmio + REG_DSI_28nm_8960_PHY_PLL_CTRL_9;
+
+	snprintf(parent_name, 32, "dsi%dvco_clk", pll_28nm->id);
+	snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->id);
+
+	bytediv_init.name = clk_name;
+	bytediv_init.ops = &clk_bytediv_ops;
+	bytediv_init.flags = CLK_SET_RATE_PARENT;
+	bytediv_init.parent_names = (const char * const *) &parent_name;
+	bytediv_init.num_parents = 1;
+
+	/* DIV2 */
+	clks[num++] = provided_clks[DSI_BYTE_PLL_CLK] =
+			clk_register(dev, &bytediv->hw);
+
+	snprintf(clk_name, 32, "dsi%dpll", pll_28nm->id);
+	/* DIV3 */
+	clks[num++] = provided_clks[DSI_PIXEL_PLL_CLK] =
+			clk_register_divider(dev, clk_name,
+				parent_name, 0, pll_28nm->mmio +
+				REG_DSI_28nm_8960_PHY_PLL_CTRL_10,
+				0, 8, 0, NULL);
+
+	pll_28nm->num_clks = num;
+
+	pll_28nm->clk_data.clk_num = NUM_PROVIDED_CLKS;
+	pll_28nm->clk_data.clks = provided_clks;
+
+	ret = of_clk_add_provider(dev->of_node,
+			of_clk_src_onecell_get, &pll_28nm->clk_data);
+	if (ret) {
+		dev_err(dev, "failed to register clk provider: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+struct msm_dsi_pll *msm_dsi_pll_28nm_8960_init(struct platform_device *pdev,
+					       int id)
+{
+	struct dsi_pll_28nm *pll_28nm;
+	struct msm_dsi_pll *pll;
+	int ret;
+
+	if (!pdev)
+		return ERR_PTR(-ENODEV);
+
+	pll_28nm = devm_kzalloc(&pdev->dev, sizeof(*pll_28nm), GFP_KERNEL);
+	if (!pll_28nm)
+		return ERR_PTR(-ENOMEM);
+
+	pll_28nm->pdev = pdev;
+	pll_28nm->id = id + 1;
+
+	pll_28nm->mmio = msm_ioremap(pdev, "dsi_pll", "DSI_PLL");
+	if (IS_ERR_OR_NULL(pll_28nm->mmio)) {
+		dev_err(&pdev->dev, "%s: failed to map pll base\n", __func__);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	pll = &pll_28nm->base;
+	pll->min_rate = VCO_MIN_RATE;
+	pll->max_rate = VCO_MAX_RATE;
+	pll->get_provider = dsi_pll_28nm_get_provider;
+	pll->destroy = dsi_pll_28nm_destroy;
+	pll->disable_seq = dsi_pll_28nm_disable_seq;
+	pll->save_state = dsi_pll_28nm_save_state;
+	pll->restore_state = dsi_pll_28nm_restore_state;
+
+	pll->en_seq_cnt = 1;
+	pll->enable_seqs[0] = dsi_pll_28nm_enable_seq;
+
+	ret = pll_28nm_register(pll_28nm);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register PLL: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	return pll;
+}
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index 1f4a95eeb348..9a0989c0b4de 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -17,6 +17,8 @@
  */
 
 #include <linux/of_irq.h>
+#include <linux/of_gpio.h>
+
 #include "hdmi.h"
 
 void hdmi_set_mode(struct hdmi *hdmi, bool power_on)
@@ -322,8 +324,6 @@ fail:
  * The hdmi device:
  */
 
-#include <linux/of_gpio.h>
-
 #define HDMI_CFG(item, entry) \
 	.item ## _names = item ##_names_ ## entry, \
 	.item ## _cnt   = ARRAY_SIZE(item ## _names_ ## entry)
@@ -388,17 +388,6 @@ static struct hdmi_platform_config hdmi_tx_8996_config = {
 		.hpd_freq      = hpd_clk_freq_8x74,
 };
 
-static const struct of_device_id dt_match[] = {
-	{ .compatible = "qcom,hdmi-tx-8996", .data = &hdmi_tx_8996_config },
-	{ .compatible = "qcom,hdmi-tx-8994", .data = &hdmi_tx_8994_config },
-	{ .compatible = "qcom,hdmi-tx-8084", .data = &hdmi_tx_8084_config },
-	{ .compatible = "qcom,hdmi-tx-8974", .data = &hdmi_tx_8974_config },
-	{ .compatible = "qcom,hdmi-tx-8960", .data = &hdmi_tx_8960_config },
-	{ .compatible = "qcom,hdmi-tx-8660", .data = &hdmi_tx_8660_config },
-	{}
-};
-
-#ifdef CONFIG_OF
 static int get_gpio(struct device *dev, struct device_node *of_node, const char *name)
 {
 	int gpio = of_get_named_gpio(of_node, name, 0);
@@ -413,7 +402,6 @@ static int get_gpio(struct device *dev, struct device_node *of_node, const char
 	}
 	return gpio;
 }
-#endif
 
 static int hdmi_bind(struct device *dev, struct device *master, void *data)
 {
@@ -421,16 +409,12 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 	struct msm_drm_private *priv = drm->dev_private;
 	static struct hdmi_platform_config *hdmi_cfg;
 	struct hdmi *hdmi;
-#ifdef CONFIG_OF
 	struct device_node *of_node = dev->of_node;
-	const struct of_device_id *match;
 
-	match = of_match_node(dt_match, of_node);
-	if (match && match->data) {
-		hdmi_cfg = (struct hdmi_platform_config *)match->data;
-		DBG("hdmi phy: %s", match->compatible);
-	} else {
-		dev_err(dev, "unknown phy: %s\n", of_node->name);
+	hdmi_cfg = (struct hdmi_platform_config *)
+			of_device_get_match_data(dev);
+	if (!hdmi_cfg) {
+		dev_err(dev, "unknown hdmi_cfg: %s\n", of_node->name);
 		return -ENXIO;
 	}
 
@@ -443,55 +427,6 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 	hdmi_cfg->mux_sel_gpio  = get_gpio(dev, of_node, "qcom,hdmi-tx-mux-sel");
 	hdmi_cfg->mux_lpm_gpio  = get_gpio(dev, of_node, "qcom,hdmi-tx-mux-lpm");
 
-#else
-	static struct hdmi_platform_config config = {};
-	static const char *hpd_clk_names[] = {
-			"core_clk", "master_iface_clk", "slave_iface_clk",
-	};
-	if (cpu_is_apq8064()) {
-		static const char *hpd_reg_names[] = {"8921_hdmi_mvs"};
-		config.phy_init      = hdmi_phy_8960_init;
-		config.hpd_reg_names = hpd_reg_names;
-		config.hpd_reg_cnt   = ARRAY_SIZE(hpd_reg_names);
-		config.hpd_clk_names = hpd_clk_names;
-		config.hpd_clk_cnt   = ARRAY_SIZE(hpd_clk_names);
-		config.ddc_clk_gpio  = 70;
-		config.ddc_data_gpio = 71;
-		config.hpd_gpio      = 72;
-		config.mux_en_gpio   = -1;
-		config.mux_sel_gpio  = -1;
-	} else if (cpu_is_msm8960() || cpu_is_msm8960ab()) {
-		static const char *hpd_reg_names[] = {"8921_hdmi_mvs"};
-		config.phy_init      = hdmi_phy_8960_init;
-		config.hpd_reg_names = hpd_reg_names;
-		config.hpd_reg_cnt   = ARRAY_SIZE(hpd_reg_names);
-		config.hpd_clk_names = hpd_clk_names;
-		config.hpd_clk_cnt   = ARRAY_SIZE(hpd_clk_names);
-		config.ddc_clk_gpio  = 100;
-		config.ddc_data_gpio = 101;
-		config.hpd_gpio      = 102;
-		config.mux_en_gpio   = -1;
-		config.mux_sel_gpio  = -1;
-	} else if (cpu_is_msm8x60()) {
-		static const char *hpd_reg_names[] = {
-				"8901_hdmi_mvs", "8901_mpp0"
-		};
-		config.phy_init      = hdmi_phy_8x60_init;
-		config.hpd_reg_names = hpd_reg_names;
-		config.hpd_reg_cnt   = ARRAY_SIZE(hpd_reg_names);
-		config.hpd_clk_names = hpd_clk_names;
-		config.hpd_clk_cnt   = ARRAY_SIZE(hpd_clk_names);
-		config.ddc_clk_gpio  = 170;
-		config.ddc_data_gpio = 171;
-		config.hpd_gpio      = 172;
-		config.mux_en_gpio   = -1;
-		config.mux_sel_gpio  = -1;
-	}
-	config.mmio_name     = "hdmi_msm_hdmi_addr";
-	config.qfprom_mmio_name = "hdmi_msm_qfprom_addr";
-
-	hdmi_cfg = &config;
-#endif
 	dev->platform_data = hdmi_cfg;
 
 	hdmi = hdmi_init(to_platform_device(dev));
@@ -529,6 +464,16 @@ static int hdmi_dev_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "qcom,hdmi-tx-8996", .data = &hdmi_tx_8996_config },
+	{ .compatible = "qcom,hdmi-tx-8994", .data = &hdmi_tx_8994_config },
+	{ .compatible = "qcom,hdmi-tx-8084", .data = &hdmi_tx_8084_config },
+	{ .compatible = "qcom,hdmi-tx-8974", .data = &hdmi_tx_8974_config },
+	{ .compatible = "qcom,hdmi-tx-8960", .data = &hdmi_tx_8960_config },
+	{ .compatible = "qcom,hdmi-tx-8660", .data = &hdmi_tx_8660_config },
+	{}
+};
+
 static struct platform_driver hdmi_driver = {
 	.probe = hdmi_dev_probe,
 	.remove = hdmi_dev_remove,
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
index 6ac9aa165768..28df397c3b04 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
@@ -678,7 +678,8 @@ struct drm_crtc *mdp4_crtc_init(struct drm_device *dev,
 	drm_flip_work_init(&mdp4_crtc->unref_cursor_work,
 			"unref cursor", unref_cursor_worker);
 
-	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp4_crtc_funcs);
+	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp4_crtc_funcs,
+				  NULL);
 	drm_crtc_helper_add(crtc, &mdp4_crtc_helper_funcs);
 	plane->crtc = crtc;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dsi_encoder.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dsi_encoder.c
new file mode 100644
index 000000000000..2f57e9453b67
--- /dev/null
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dsi_encoder.c
@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2014, Inforce Computing. All rights reserved.
+ *
+ * Author: Vinay Simha <vinaysimha@inforcecomputing.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mdp4_kms.h"
+
+#include "drm_crtc.h"
+#include "drm_crtc_helper.h"
+
+struct mdp4_dsi_encoder {
+	struct drm_encoder base;
+	struct drm_panel *panel;
+	bool enabled;
+};
+#define to_mdp4_dsi_encoder(x) container_of(x, struct mdp4_dsi_encoder, base)
+
+static struct mdp4_kms *get_kms(struct drm_encoder *encoder)
+{
+	struct msm_drm_private *priv = encoder->dev->dev_private;
+	return to_mdp4_kms(to_mdp_kms(priv->kms));
+}
+
+static void mdp4_dsi_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct mdp4_dsi_encoder *mdp4_dsi_encoder = to_mdp4_dsi_encoder(encoder);
+
+	drm_encoder_cleanup(encoder);
+	kfree(mdp4_dsi_encoder);
+}
+
+static const struct drm_encoder_funcs mdp4_dsi_encoder_funcs = {
+	.destroy = mdp4_dsi_encoder_destroy,
+};
+
+static bool mdp4_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void mdp4_dsi_encoder_mode_set(struct drm_encoder *encoder,
+				      struct drm_display_mode *mode,
+				      struct drm_display_mode *adjusted_mode)
+{
+	struct mdp4_kms *mdp4_kms = get_kms(encoder);
+	uint32_t dsi_hsync_skew, vsync_period, vsync_len, ctrl_pol;
+	uint32_t display_v_start, display_v_end;
+	uint32_t hsync_start_x, hsync_end_x;
+
+	mode = adjusted_mode;
+
+	DBG("set mode: %d:\"%s\" %d %d %d %d %d %d %d %d %d %d 0x%x 0x%x",
+			mode->base.id, mode->name,
+			mode->vrefresh, mode->clock,
+			mode->hdisplay, mode->hsync_start,
+			mode->hsync_end, mode->htotal,
+			mode->vdisplay, mode->vsync_start,
+			mode->vsync_end, mode->vtotal,
+			mode->type, mode->flags);
+
+	ctrl_pol = 0;
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		ctrl_pol |= MDP4_DSI_CTRL_POLARITY_HSYNC_LOW;
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		ctrl_pol |= MDP4_DSI_CTRL_POLARITY_VSYNC_LOW;
+	/* probably need to get DATA_EN polarity from panel.. */
+
+	dsi_hsync_skew = 0;  /* get this from panel? */
+
+	hsync_start_x = (mode->htotal - mode->hsync_start);
+	hsync_end_x = mode->htotal - (mode->hsync_start - mode->hdisplay) - 1;
+
+	vsync_period = mode->vtotal * mode->htotal;
+	vsync_len = (mode->vsync_end - mode->vsync_start) * mode->htotal;
+	display_v_start = (mode->vtotal - mode->vsync_start) * mode->htotal + dsi_hsync_skew;
+	display_v_end = vsync_period - ((mode->vsync_start - mode->vdisplay) * mode->htotal) + dsi_hsync_skew - 1;
+
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_HSYNC_CTRL,
+			MDP4_DSI_HSYNC_CTRL_PULSEW(mode->hsync_end - mode->hsync_start) |
+			MDP4_DSI_HSYNC_CTRL_PERIOD(mode->htotal));
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_VSYNC_PERIOD, vsync_period);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_VSYNC_LEN, vsync_len);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_DISPLAY_HCTRL,
+			MDP4_DSI_DISPLAY_HCTRL_START(hsync_start_x) |
+			MDP4_DSI_DISPLAY_HCTRL_END(hsync_end_x));
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_DISPLAY_VSTART, display_v_start);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_DISPLAY_VEND, display_v_end);
+
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_CTRL_POLARITY, ctrl_pol);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_UNDERFLOW_CLR,
+			MDP4_DSI_UNDERFLOW_CLR_ENABLE_RECOVERY |
+			MDP4_DSI_UNDERFLOW_CLR_COLOR(0xff));
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_ACTIVE_HCTL,
+			MDP4_DSI_ACTIVE_HCTL_START(0) |
+			MDP4_DSI_ACTIVE_HCTL_END(0));
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_HSYNC_SKEW, dsi_hsync_skew);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_BORDER_CLR, 0);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_ACTIVE_VSTART, 0);
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_ACTIVE_VEND, 0);
+}
+
+static void mdp4_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct mdp4_dsi_encoder *mdp4_dsi_encoder = to_mdp4_dsi_encoder(encoder);
+	struct mdp4_kms *mdp4_kms = get_kms(encoder);
+
+	if (!mdp4_dsi_encoder->enabled)
+		return;
+
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_ENABLE, 0);
+
+	/*
+	 * Wait for a vsync so we know the ENABLE=0 latched before
+	 * the (connector) source of the vsync's gets disabled,
+	 * otherwise we end up in a funny state if we re-enable
+	 * before the disable latches, which results that some of
+	 * the settings changes for the new modeset (like new
+	 * scanout buffer) don't latch properly..
+	 */
+	mdp_irq_wait(&mdp4_kms->base, MDP4_IRQ_PRIMARY_VSYNC);
+
+	mdp4_dsi_encoder->enabled = false;
+}
+
+static void mdp4_dsi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct mdp4_dsi_encoder *mdp4_dsi_encoder = to_mdp4_dsi_encoder(encoder);
+	struct mdp4_kms *mdp4_kms = get_kms(encoder);
+
+	if (mdp4_dsi_encoder->enabled)
+		return;
+
+	 mdp4_crtc_set_config(encoder->crtc,
+			MDP4_DMA_CONFIG_PACK_ALIGN_MSB |
+			MDP4_DMA_CONFIG_DEFLKR_EN |
+			MDP4_DMA_CONFIG_DITHER_EN |
+			MDP4_DMA_CONFIG_R_BPC(BPC8) |
+			MDP4_DMA_CONFIG_G_BPC(BPC8) |
+			MDP4_DMA_CONFIG_B_BPC(BPC8) |
+			MDP4_DMA_CONFIG_PACK(0x21));
+
+	mdp4_crtc_set_intf(encoder->crtc, INTF_DSI_VIDEO, 0);
+
+	mdp4_write(mdp4_kms, REG_MDP4_DSI_ENABLE, 1);
+
+	mdp4_dsi_encoder->enabled = true;
+}
+
+static const struct drm_encoder_helper_funcs mdp4_dsi_encoder_helper_funcs = {
+	.mode_fixup = mdp4_dsi_encoder_mode_fixup,
+	.mode_set = mdp4_dsi_encoder_mode_set,
+	.disable = mdp4_dsi_encoder_disable,
+	.enable = mdp4_dsi_encoder_enable,
+};
+
+/* initialize encoder */
+struct drm_encoder *mdp4_dsi_encoder_init(struct drm_device *dev)
+{
+	struct drm_encoder *encoder = NULL;
+	struct mdp4_dsi_encoder *mdp4_dsi_encoder;
+	int ret;
+
+	mdp4_dsi_encoder = kzalloc(sizeof(*mdp4_dsi_encoder), GFP_KERNEL);
+	if (!mdp4_dsi_encoder) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	encoder = &mdp4_dsi_encoder->base;
+
+	drm_encoder_init(dev, encoder, &mdp4_dsi_encoder_funcs,
+			 DRM_MODE_ENCODER_DSI, NULL);
+	drm_encoder_helper_add(encoder, &mdp4_dsi_encoder_helper_funcs);
+
+	return encoder;
+
+fail:
+	if (encoder)
+		mdp4_dsi_encoder_destroy(encoder);
+
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
index 89614c6a6c1b..a21df54cb50f 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
@@ -262,7 +262,7 @@ struct drm_encoder *mdp4_dtv_encoder_init(struct drm_device *dev)
 	encoder = &mdp4_dtv_encoder->base;
 
 	drm_encoder_init(dev, encoder, &mdp4_dtv_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 	drm_encoder_helper_add(encoder, &mdp4_dtv_encoder_helper_funcs);
 
 	mdp4_dtv_encoder->src_clk = devm_clk_get(dev->dev, "src_clk");
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
index 5ed38cf548a1..a521207db8a1 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
@@ -29,7 +29,7 @@ void mdp4_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
 
 static void mdp4_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
 {
-	DRM_ERROR("errors: %08x\n", irqstatus);
+	DRM_ERROR_RATELIMITED("errors: %08x\n", irqstatus);
 }
 
 void mdp4_irq_preinstall(struct msm_kms *kms)
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
index 077f7521a971..5a8e3d6bcbff 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
@@ -169,7 +169,14 @@ static long mdp4_round_pixclk(struct msm_kms *kms, unsigned long rate,
 		struct drm_encoder *encoder)
 {
 	/* if we had >1 encoder, we'd need something more clever: */
-	return mdp4_dtv_round_pixclk(encoder, rate);
+	switch (encoder->encoder_type) {
+	case DRM_MODE_ENCODER_TMDS:
+		return mdp4_dtv_round_pixclk(encoder, rate);
+	case DRM_MODE_ENCODER_LVDS:
+	case DRM_MODE_ENCODER_DSI:
+	default:
+		return rate;
+	}
 }
 
 static void mdp4_preclose(struct msm_kms *kms, struct drm_file *file)
@@ -240,19 +247,18 @@ int mdp4_enable(struct mdp4_kms *mdp4_kms)
 	return 0;
 }
 
-#ifdef CONFIG_OF
-static struct drm_panel *detect_panel(struct drm_device *dev)
+static struct device_node *mdp4_detect_lcdc_panel(struct drm_device *dev)
 {
 	struct device_node *endpoint, *panel_node;
 	struct device_node *np = dev->dev->of_node;
-	struct drm_panel *panel = NULL;
 
 	endpoint = of_graph_get_next_endpoint(np, NULL);
 	if (!endpoint) {
-		dev_err(dev->dev, "no valid endpoint\n");
-		return ERR_PTR(-ENODEV);
+		DBG("no endpoint in MDP4 to fetch LVDS panel\n");
+		return NULL;
 	}
 
+	/* don't proceed if we have an endpoint but no panel_node tied to it */
 	panel_node = of_graph_get_remote_port_parent(endpoint);
 	if (!panel_node) {
 		dev_err(dev->dev, "no valid panel node\n");
@@ -262,132 +268,185 @@ static struct drm_panel *detect_panel(struct drm_device *dev)
 
 	of_node_put(endpoint);
 
-	panel = of_drm_find_panel(panel_node);
-	if (!panel) {
-		of_node_put(panel_node);
-		return ERR_PTR(-EPROBE_DEFER);
-	}
-
-	return panel;
+	return panel_node;
 }
-#else
-static struct drm_panel *detect_panel(struct drm_device *dev)
-{
-	// ??? maybe use a module param to specify which panel is attached?
-}
-#endif
 
-static int modeset_init(struct mdp4_kms *mdp4_kms)
+static int mdp4_modeset_init_intf(struct mdp4_kms *mdp4_kms,
+				  int intf_type)
 {
 	struct drm_device *dev = mdp4_kms->dev;
 	struct msm_drm_private *priv = dev->dev_private;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc;
 	struct drm_encoder *encoder;
 	struct drm_connector *connector;
-	struct drm_panel *panel;
+	struct device_node *panel_node;
+	struct drm_encoder *dsi_encs[MSM_DSI_ENCODER_NUM];
+	int i, dsi_id;
 	int ret;
 
-	/* construct non-private planes: */
-	plane = mdp4_plane_init(dev, VG1, false);
-	if (IS_ERR(plane)) {
-		dev_err(dev->dev, "failed to construct plane for VG1\n");
-		ret = PTR_ERR(plane);
-		goto fail;
-	}
-	priv->planes[priv->num_planes++] = plane;
+	switch (intf_type) {
+	case DRM_MODE_ENCODER_LVDS:
+		/*
+		 * bail out early if:
+		 * - there is no panel node (no need to initialize lcdc
+		 *   encoder and lvds connector), or
+		 * - panel node is a bad pointer
+		 */
+		panel_node = mdp4_detect_lcdc_panel(dev);
+		if (IS_ERR_OR_NULL(panel_node))
+			return PTR_ERR(panel_node);
+
+		encoder = mdp4_lcdc_encoder_init(dev, panel_node);
+		if (IS_ERR(encoder)) {
+			dev_err(dev->dev, "failed to construct LCDC encoder\n");
+			return PTR_ERR(encoder);
+		}
 
-	plane = mdp4_plane_init(dev, VG2, false);
-	if (IS_ERR(plane)) {
-		dev_err(dev->dev, "failed to construct plane for VG2\n");
-		ret = PTR_ERR(plane);
-		goto fail;
-	}
-	priv->planes[priv->num_planes++] = plane;
+		/* LCDC can be hooked to DMA_P (TODO: Add DMA_S later?) */
+		encoder->possible_crtcs = 1 << DMA_P;
 
-	/*
-	 * Setup the LCDC/LVDS path: RGB2 -> DMA_P -> LCDC -> LVDS:
-	 */
+		connector = mdp4_lvds_connector_init(dev, panel_node, encoder);
+		if (IS_ERR(connector)) {
+			dev_err(dev->dev, "failed to initialize LVDS connector\n");
+			return PTR_ERR(connector);
+		}
 
-	panel = detect_panel(dev);
-	if (IS_ERR(panel)) {
-		ret = PTR_ERR(panel);
-		dev_err(dev->dev, "failed to detect LVDS panel: %d\n", ret);
-		goto fail;
-	}
+		priv->encoders[priv->num_encoders++] = encoder;
+		priv->connectors[priv->num_connectors++] = connector;
 
-	plane = mdp4_plane_init(dev, RGB2, true);
-	if (IS_ERR(plane)) {
-		dev_err(dev->dev, "failed to construct plane for RGB2\n");
-		ret = PTR_ERR(plane);
-		goto fail;
-	}
+		break;
+	case DRM_MODE_ENCODER_TMDS:
+		encoder = mdp4_dtv_encoder_init(dev);
+		if (IS_ERR(encoder)) {
+			dev_err(dev->dev, "failed to construct DTV encoder\n");
+			return PTR_ERR(encoder);
+		}
 
-	crtc  = mdp4_crtc_init(dev, plane, priv->num_crtcs, 0, DMA_P);
-	if (IS_ERR(crtc)) {
-		dev_err(dev->dev, "failed to construct crtc for DMA_P\n");
-		ret = PTR_ERR(crtc);
-		goto fail;
-	}
+		/* DTV can be hooked to DMA_E: */
+		encoder->possible_crtcs = 1 << 1;
 
-	encoder = mdp4_lcdc_encoder_init(dev, panel);
-	if (IS_ERR(encoder)) {
-		dev_err(dev->dev, "failed to construct LCDC encoder\n");
-		ret = PTR_ERR(encoder);
-		goto fail;
-	}
+		if (priv->hdmi) {
+			/* Construct bridge/connector for HDMI: */
+			ret = hdmi_modeset_init(priv->hdmi, dev, encoder);
+			if (ret) {
+				dev_err(dev->dev, "failed to initialize HDMI: %d\n", ret);
+				return ret;
+			}
+		}
 
-	/* LCDC can be hooked to DMA_P: */
-	encoder->possible_crtcs = 1 << priv->num_crtcs;
+		priv->encoders[priv->num_encoders++] = encoder;
 
-	priv->crtcs[priv->num_crtcs++] = crtc;
-	priv->encoders[priv->num_encoders++] = encoder;
+		break;
+	case DRM_MODE_ENCODER_DSI:
+		/* only DSI1 supported for now */
+		dsi_id = 0;
 
-	connector = mdp4_lvds_connector_init(dev, panel, encoder);
-	if (IS_ERR(connector)) {
-		ret = PTR_ERR(connector);
-		dev_err(dev->dev, "failed to initialize LVDS connector: %d\n", ret);
-		goto fail;
-	}
+		if (!priv->dsi[dsi_id])
+			break;
 
-	priv->connectors[priv->num_connectors++] = connector;
+		for (i = 0; i < MSM_DSI_ENCODER_NUM; i++) {
+			dsi_encs[i] = mdp4_dsi_encoder_init(dev);
+			if (IS_ERR(dsi_encs[i])) {
+				ret = PTR_ERR(dsi_encs[i]);
+				dev_err(dev->dev,
+					"failed to construct DSI encoder: %d\n",
+					ret);
+				return ret;
+			}
 
-	/*
-	 * Setup DTV/HDMI path: RGB1 -> DMA_E -> DTV -> HDMI:
-	 */
+			/* TODO: Add DMA_S later? */
+			dsi_encs[i]->possible_crtcs = 1 << DMA_P;
+			priv->encoders[priv->num_encoders++] = dsi_encs[i];
+		}
 
-	plane = mdp4_plane_init(dev, RGB1, true);
-	if (IS_ERR(plane)) {
-		dev_err(dev->dev, "failed to construct plane for RGB1\n");
-		ret = PTR_ERR(plane);
-		goto fail;
-	}
+		ret = msm_dsi_modeset_init(priv->dsi[dsi_id], dev, dsi_encs);
+		if (ret) {
+			dev_err(dev->dev, "failed to initialize DSI: %d\n",
+				ret);
+			return ret;
+		}
 
-	crtc  = mdp4_crtc_init(dev, plane, priv->num_crtcs, 1, DMA_E);
-	if (IS_ERR(crtc)) {
-		dev_err(dev->dev, "failed to construct crtc for DMA_E\n");
-		ret = PTR_ERR(crtc);
-		goto fail;
+		break;
+	default:
+		dev_err(dev->dev, "Invalid or unsupported interface\n");
+		return -EINVAL;
 	}
 
-	encoder = mdp4_dtv_encoder_init(dev);
-	if (IS_ERR(encoder)) {
-		dev_err(dev->dev, "failed to construct DTV encoder\n");
-		ret = PTR_ERR(encoder);
-		goto fail;
+	return 0;
+}
+
+static int modeset_init(struct mdp4_kms *mdp4_kms)
+{
+	struct drm_device *dev = mdp4_kms->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	int i, ret;
+	static const enum mdp4_pipe rgb_planes[] = {
+		RGB1, RGB2,
+	};
+	static const enum mdp4_pipe vg_planes[] = {
+		VG1, VG2,
+	};
+	static const enum mdp4_dma mdp4_crtcs[] = {
+		DMA_P, DMA_E,
+	};
+	static const char * const mdp4_crtc_names[] = {
+		"DMA_P", "DMA_E",
+	};
+	static const int mdp4_intfs[] = {
+		DRM_MODE_ENCODER_LVDS,
+		DRM_MODE_ENCODER_DSI,
+		DRM_MODE_ENCODER_TMDS,
+	};
+
+	/* construct non-private planes: */
+	for (i = 0; i < ARRAY_SIZE(vg_planes); i++) {
+		plane = mdp4_plane_init(dev, vg_planes[i], false);
+		if (IS_ERR(plane)) {
+			dev_err(dev->dev,
+				"failed to construct plane for VG%d\n", i + 1);
+			ret = PTR_ERR(plane);
+			goto fail;
+		}
+		priv->planes[priv->num_planes++] = plane;
 	}
 
-	/* DTV can be hooked to DMA_E: */
-	encoder->possible_crtcs = 1 << priv->num_crtcs;
+	for (i = 0; i < ARRAY_SIZE(mdp4_crtcs); i++) {
+		plane = mdp4_plane_init(dev, rgb_planes[i], true);
+		if (IS_ERR(plane)) {
+			dev_err(dev->dev,
+				"failed to construct plane for RGB%d\n", i + 1);
+			ret = PTR_ERR(plane);
+			goto fail;
+		}
+
+		crtc  = mdp4_crtc_init(dev, plane, priv->num_crtcs, i,
+				mdp4_crtcs[i]);
+		if (IS_ERR(crtc)) {
+			dev_err(dev->dev, "failed to construct crtc for %s\n",
+				mdp4_crtc_names[i]);
+			ret = PTR_ERR(crtc);
+			goto fail;
+		}
+
+		priv->crtcs[priv->num_crtcs++] = crtc;
+	}
 
-	priv->crtcs[priv->num_crtcs++] = crtc;
-	priv->encoders[priv->num_encoders++] = encoder;
+	/*
+	 * we currently set up two relatively fixed paths:
+	 *
+	 * LCDC/LVDS path: RGB1 -> DMA_P -> LCDC -> LVDS
+	 *			or
+	 * DSI path: RGB1 -> DMA_P -> DSI1 -> DSI Panel
+	 *
+	 * DTV/HDMI path: RGB2 -> DMA_E -> DTV -> HDMI
+	 */
 
-	if (priv->hdmi) {
-		/* Construct bridge/connector for HDMI: */
-		ret = hdmi_modeset_init(priv->hdmi, dev, encoder);
+	for (i = 0; i < ARRAY_SIZE(mdp4_intfs); i++) {
+		ret = mdp4_modeset_init_intf(mdp4_kms, mdp4_intfs[i]);
 		if (ret) {
-			dev_err(dev->dev, "failed to initialize HDMI: %d\n", ret);
+			dev_err(dev->dev, "failed to initialize intf: %d, %d\n",
+				i, ret);
 			goto fail;
 		}
 	}
@@ -558,17 +617,10 @@ fail:
 static struct mdp4_platform_config *mdp4_get_config(struct platform_device *dev)
 {
 	static struct mdp4_platform_config config = {};
-#ifdef CONFIG_OF
-	/* TODO */
+
+	/* TODO: Chips that aren't apq8064 have a 200 Mhz max_clk */
 	config.max_clk = 266667000;
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
-#else
-	if (cpu_is_apq8064())
-		config.max_clk = 266667000;
-	else
-		config.max_clk = 200000000;
-
-	config.iommu = msm_get_iommu_domain(DISPLAY_READ_DOMAIN);
-#endif
+
 	return &config;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
index 8a7f6e1e2bca..d2c96ef431f4 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
@@ -157,7 +157,7 @@ static inline uint32_t mixercfg(uint32_t mixer_cfg, int mixer,
 			COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE6_MIXER1);
 		break;
 	default:
-		WARN_ON("invalid pipe");
+		WARN(1, "invalid pipe");
 		break;
 	}
 
@@ -212,10 +212,19 @@ struct drm_encoder *mdp4_dtv_encoder_init(struct drm_device *dev);
 
 long mdp4_lcdc_round_pixclk(struct drm_encoder *encoder, unsigned long rate);
 struct drm_encoder *mdp4_lcdc_encoder_init(struct drm_device *dev,
-		struct drm_panel *panel);
+		struct device_node *panel_node);
 
 struct drm_connector *mdp4_lvds_connector_init(struct drm_device *dev,
-		struct drm_panel *panel, struct drm_encoder *encoder);
+		struct device_node *panel_node, struct drm_encoder *encoder);
+
+#ifdef CONFIG_DRM_MSM_DSI
+struct drm_encoder *mdp4_dsi_encoder_init(struct drm_device *dev);
+#else
+static inline struct drm_encoder *mdp4_dsi_encoder_init(struct drm_device *dev)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
 
 #ifdef CONFIG_COMMON_CLK
 struct clk *mpd4_lvds_pll_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
index 4cd6e721aa0a..cd63fedb67cc 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
@@ -23,6 +23,7 @@
 
 struct mdp4_lcdc_encoder {
 	struct drm_encoder base;
+	struct device_node *panel_node;
 	struct drm_panel *panel;
 	struct clk *lcdc_clk;
 	unsigned long int pixclock;
@@ -338,7 +339,7 @@ static void mdp4_lcdc_encoder_disable(struct drm_encoder *encoder)
 	struct mdp4_lcdc_encoder *mdp4_lcdc_encoder =
 			to_mdp4_lcdc_encoder(encoder);
 	struct mdp4_kms *mdp4_kms = get_kms(encoder);
-	struct drm_panel *panel = mdp4_lcdc_encoder->panel;
+	struct drm_panel *panel;
 	int i, ret;
 
 	if (WARN_ON(!mdp4_lcdc_encoder->enabled))
@@ -346,6 +347,7 @@ static void mdp4_lcdc_encoder_disable(struct drm_encoder *encoder)
 
 	mdp4_write(mdp4_kms, REG_MDP4_LCDC_ENABLE, 0);
 
+	panel = of_drm_find_panel(mdp4_lcdc_encoder->panel_node);
 	if (panel) {
 		drm_panel_disable(panel);
 		drm_panel_unprepare(panel);
@@ -381,7 +383,7 @@ static void mdp4_lcdc_encoder_enable(struct drm_encoder *encoder)
 			to_mdp4_lcdc_encoder(encoder);
 	unsigned long pc = mdp4_lcdc_encoder->pixclock;
 	struct mdp4_kms *mdp4_kms = get_kms(encoder);
-	struct drm_panel *panel = mdp4_lcdc_encoder->panel;
+	struct drm_panel *panel;
 	int i, ret;
 
 	if (WARN_ON(mdp4_lcdc_encoder->enabled))
@@ -414,6 +416,7 @@ static void mdp4_lcdc_encoder_enable(struct drm_encoder *encoder)
 	if (ret)
 		dev_err(dev->dev, "failed to enable lcdc_clk: %d\n", ret);
 
+	panel = of_drm_find_panel(mdp4_lcdc_encoder->panel_node);
 	if (panel) {
 		drm_panel_prepare(panel);
 		drm_panel_enable(panel);
@@ -442,7 +445,7 @@ long mdp4_lcdc_round_pixclk(struct drm_encoder *encoder, unsigned long rate)
 
 /* initialize encoder */
 struct drm_encoder *mdp4_lcdc_encoder_init(struct drm_device *dev,
-		struct drm_panel *panel)
+		struct device_node *panel_node)
 {
 	struct drm_encoder *encoder = NULL;
 	struct mdp4_lcdc_encoder *mdp4_lcdc_encoder;
@@ -455,12 +458,12 @@ struct drm_encoder *mdp4_lcdc_encoder_init(struct drm_device *dev,
 		goto fail;
 	}
 
-	mdp4_lcdc_encoder->panel = panel;
+	mdp4_lcdc_encoder->panel_node = panel_node;
 
 	encoder = &mdp4_lcdc_encoder->base;
 
 	drm_encoder_init(dev, encoder, &mdp4_lcdc_encoder_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 	drm_encoder_helper_add(encoder, &mdp4_lcdc_encoder_helper_funcs);
 
 	/* TODO: do we need different pll in other cases? */
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
index 921185133d38..e73e1742b250 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
@@ -23,6 +23,7 @@
 struct mdp4_lvds_connector {
 	struct drm_connector base;
 	struct drm_encoder *encoder;
+	struct device_node *panel_node;
 	struct drm_panel *panel;
 };
 #define to_mdp4_lvds_connector(x) container_of(x, struct mdp4_lvds_connector, base)
@@ -33,6 +34,10 @@ static enum drm_connector_status mdp4_lvds_connector_detect(
 	struct mdp4_lvds_connector *mdp4_lvds_connector =
 			to_mdp4_lvds_connector(connector);
 
+	if (!mdp4_lvds_connector->panel)
+		mdp4_lvds_connector->panel =
+			of_drm_find_panel(mdp4_lvds_connector->panel_node);
+
 	return mdp4_lvds_connector->panel ?
 			connector_status_connected :
 			connector_status_disconnected;
@@ -42,10 +47,6 @@ static void mdp4_lvds_connector_destroy(struct drm_connector *connector)
 {
 	struct mdp4_lvds_connector *mdp4_lvds_connector =
 			to_mdp4_lvds_connector(connector);
-	struct drm_panel *panel = mdp4_lvds_connector->panel;
-
-	if (panel)
-		drm_panel_detach(panel);
 
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
@@ -60,9 +61,14 @@ static int mdp4_lvds_connector_get_modes(struct drm_connector *connector)
 	struct drm_panel *panel = mdp4_lvds_connector->panel;
 	int ret = 0;
 
-	if (panel)
+	if (panel) {
+		drm_panel_attach(panel, connector);
+
 		ret = panel->funcs->get_modes(panel);
 
+		drm_panel_detach(panel);
+	}
+
 	return ret;
 }
 
@@ -111,7 +117,7 @@ static const struct drm_connector_helper_funcs mdp4_lvds_connector_helper_funcs
 
 /* initialize connector */
 struct drm_connector *mdp4_lvds_connector_init(struct drm_device *dev,
-		struct drm_panel *panel, struct drm_encoder *encoder)
+		struct device_node *panel_node, struct drm_encoder *encoder)
 {
 	struct drm_connector *connector = NULL;
 	struct mdp4_lvds_connector *mdp4_lvds_connector;
@@ -124,7 +130,7 @@ struct drm_connector *mdp4_lvds_connector_init(struct drm_device *dev,
 	}
 
 	mdp4_lvds_connector->encoder = encoder;
-	mdp4_lvds_connector->panel = panel;
+	mdp4_lvds_connector->panel_node = panel_node;
 
 	connector = &mdp4_lvds_connector->base;
 
@@ -141,9 +147,6 @@ struct drm_connector *mdp4_lvds_connector_init(struct drm_device *dev,
 
 	drm_mode_connector_attach_encoder(connector, encoder);
 
-	if (panel)
-		drm_panel_attach(panel, connector);
-
 	return connector;
 
 fail:
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
index 30d57e74c42f..9f96dfe67769 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
@@ -397,7 +397,8 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
 
 	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
-				 mdp4_plane->formats, mdp4_plane->nformats, type);
+				 mdp4_plane->formats, mdp4_plane->nformats,
+				 type, NULL);
 	if (ret)
 		goto fail;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
index bb1225aa2f75..57f73f0c120d 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
@@ -553,9 +553,7 @@ fail:
 static struct mdp5_cfg_platform *mdp5_get_config(struct platform_device *dev)
 {
 	static struct mdp5_cfg_platform config = {};
-#ifdef CONFIG_OF
-	/* TODO */
-#endif
+
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
 
 	return &config;
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
index 8e6c9b598a57..1aa21dba663d 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
@@ -326,7 +326,7 @@ struct drm_encoder *mdp5_cmd_encoder_init(struct drm_device *dev,
 	mdp5_cmd_enc->ctl = ctl;
 
 	drm_encoder_init(dev, encoder, &mdp5_cmd_encoder_funcs,
-			DRM_MODE_ENCODER_DSI);
+			DRM_MODE_ENCODER_DSI, NULL);
 
 	drm_encoder_helper_add(encoder, &mdp5_cmd_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
index 7f9f4ac88029..20cee5ce4071 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
@@ -797,7 +797,8 @@ struct drm_crtc *mdp5_crtc_init(struct drm_device *dev,
 	snprintf(mdp5_crtc->name, sizeof(mdp5_crtc->name), "%s:%d",
 			pipe2name(mdp5_plane_pipe(plane)), id);
 
-	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp5_crtc_funcs);
+	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp5_crtc_funcs,
+				  NULL);
 
 	drm_flip_work_init(&mdp5_crtc->unref_cursor_work,
 			"unref cursor", unref_cursor_worker);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
index c9e32b08a7a0..0d737cad03a6 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
@@ -293,6 +293,24 @@ static const struct drm_encoder_helper_funcs mdp5_encoder_helper_funcs = {
 	.enable = mdp5_encoder_enable,
 };
 
+int mdp5_encoder_get_linecount(struct drm_encoder *encoder)
+{
+	struct mdp5_encoder *mdp5_encoder = to_mdp5_encoder(encoder);
+	struct mdp5_kms *mdp5_kms = get_kms(encoder);
+	int intf = mdp5_encoder->intf.num;
+
+	return mdp5_read(mdp5_kms, REG_MDP5_INTF_LINE_COUNT(intf));
+}
+
+u32 mdp5_encoder_get_framecount(struct drm_encoder *encoder)
+{
+	struct mdp5_encoder *mdp5_encoder = to_mdp5_encoder(encoder);
+	struct mdp5_kms *mdp5_kms = get_kms(encoder);
+	int intf = mdp5_encoder->intf.num;
+
+	return mdp5_read(mdp5_kms, REG_MDP5_INTF_FRAME_COUNT(intf));
+}
+
 int mdp5_encoder_set_split_display(struct drm_encoder *encoder,
 					struct drm_encoder *slave_encoder)
 {
@@ -354,7 +372,7 @@ struct drm_encoder *mdp5_encoder_init(struct drm_device *dev,
 
 	spin_lock_init(&mdp5_encoder->intf_lock);
 
-	drm_encoder_init(dev, encoder, &mdp5_encoder_funcs, enc_type);
+	drm_encoder_init(dev, encoder, &mdp5_encoder_funcs, enc_type, NULL);
 
 	drm_encoder_helper_add(encoder, &mdp5_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
index b0d4b53b97f4..73bc3e312fd4 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
@@ -31,7 +31,7 @@ void mdp5_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
 
 static void mdp5_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
 {
-	DRM_ERROR("errors: %08x\n", irqstatus);
+	DRM_ERROR_RATELIMITED("errors: %08x\n", irqstatus);
 }
 
 void mdp5_irq_preinstall(struct msm_kms *kms)
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
index b532faa8026d..e115318402bd 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
@@ -468,6 +468,127 @@ static int get_clk(struct platform_device *pdev, struct clk **clkp,
 	return 0;
 }
 
+static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_encoder *encoder;
+
+	drm_for_each_encoder(encoder, dev)
+		if (encoder->crtc == crtc)
+			return encoder;
+
+	return NULL;
+}
+
+static int mdp5_get_scanoutpos(struct drm_device *dev, unsigned int pipe,
+			       unsigned int flags, int *vpos, int *hpos,
+			       ktime_t *stime, ktime_t *etime,
+			       const struct drm_display_mode *mode)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
+	int ret = 0;
+
+	crtc = priv->crtcs[pipe];
+	if (!crtc) {
+		DRM_ERROR("Invalid crtc %d\n", pipe);
+		return 0;
+	}
+
+	encoder = get_encoder_from_crtc(crtc);
+	if (!encoder) {
+		DRM_ERROR("no encoder found for crtc %d\n", pipe);
+		return 0;
+	}
+
+	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
+
+	vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
+	vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
+
+	/*
+	 * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
+	 * the end of VFP. Translate the porch values relative to the line
+	 * counter positions.
+	 */
+
+	vactive_start = vsw + vbp + 1;
+
+	vactive_end = vactive_start + mode->crtc_vdisplay;
+
+	/* last scan line before VSYNC */
+	vfp_end = mode->crtc_vtotal;
+
+	if (stime)
+		*stime = ktime_get();
+
+	line = mdp5_encoder_get_linecount(encoder);
+
+	if (line < vactive_start) {
+		line -= vactive_start;
+		ret |= DRM_SCANOUTPOS_IN_VBLANK;
+	} else if (line > vactive_end) {
+		line = line - vfp_end - vactive_start;
+		ret |= DRM_SCANOUTPOS_IN_VBLANK;
+	} else {
+		line -= vactive_start;
+	}
+
+	*vpos = line;
+	*hpos = 0;
+
+	if (etime)
+		*etime = ktime_get();
+
+	return ret;
+}
+
+static int mdp5_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
+				     int *max_error,
+				     struct timeval *vblank_time,
+				     unsigned flags)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct drm_crtc *crtc;
+
+	if (pipe < 0 || pipe >= priv->num_crtcs) {
+		DRM_ERROR("Invalid crtc %d\n", pipe);
+		return -EINVAL;
+	}
+
+	crtc = priv->crtcs[pipe];
+	if (!crtc) {
+		DRM_ERROR("Invalid crtc %d\n", pipe);
+		return -EINVAL;
+	}
+
+	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
+						     vblank_time, flags,
+						     &crtc->mode);
+}
+
+static u32 mdp5_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+
+	if (pipe < 0 || pipe >= priv->num_crtcs)
+		return 0;
+
+	crtc = priv->crtcs[pipe];
+	if (!crtc)
+		return 0;
+
+	encoder = get_encoder_from_crtc(crtc);
+	if (!encoder)
+		return 0;
+
+	return mdp5_encoder_get_framecount(encoder);
+}
+
 struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 {
 	struct platform_device *pdev = dev->platformdev;
@@ -590,6 +711,8 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 				!config->hw->intf.base[i])
 			continue;
 		mdp5_write(mdp5_kms, REG_MDP5_INTF_TIMING_ENGINE_EN(i), 0);
+
+		mdp5_write(mdp5_kms, REG_MDP5_INTF_FRAME_LINE_COUNT_EN(i), 0x3);
 	}
 	mdp5_disable(mdp5_kms);
 	mdelay(16);
@@ -635,6 +758,12 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	dev->mode_config.max_width = config->hw->lm.max_width;
 	dev->mode_config.max_height = config->hw->lm.max_height;
 
+	dev->driver->get_vblank_timestamp = mdp5_get_vblank_timestamp;
+	dev->driver->get_scanout_position = mdp5_get_scanoutpos;
+	dev->driver->get_vblank_counter = mdp5_get_vblank_counter;
+	dev->max_vblank_count = 0xffffffff;
+	dev->vblank_disable_immediate = true;
+
 	return kms;
 
 fail:
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
index 84f65d415598..00730ba08a60 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
@@ -222,6 +222,8 @@ struct drm_encoder *mdp5_encoder_init(struct drm_device *dev,
 		struct mdp5_interface *intf, struct mdp5_ctl *ctl);
 int mdp5_encoder_set_split_display(struct drm_encoder *encoder,
 					struct drm_encoder *slave_encoder);
+int mdp5_encoder_get_linecount(struct drm_encoder *encoder);
+u32 mdp5_encoder_get_framecount(struct drm_encoder *encoder);
 
 #ifdef CONFIG_DRM_MSM_DSI
 struct drm_encoder *mdp5_cmd_encoder_init(struct drm_device *dev,
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index 81cd49045ffc..432c09836b0e 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -904,7 +904,7 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
 	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp5_plane_funcs,
 				 mdp5_plane->formats, mdp5_plane->nformats,
-				 type);
+				 type, NULL);
 	if (ret)
 		goto fail;
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index b88ce514eb8e..9a30807b900b 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -237,20 +237,9 @@ static int msm_unload(struct drm_device *dev)
 
 static int get_mdp_ver(struct platform_device *pdev)
 {
-#ifdef CONFIG_OF
-	static const struct of_device_id match_types[] = { {
-		.compatible = "qcom,mdss_mdp",
-		.data	= (void	*)5,
-	}, {
-		/* end node */
-	} };
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *match;
-	match = of_match_node(match_types, dev->of_node);
-	if (match)
-		return (int)(unsigned long)match->data;
-#endif
-	return 4;
+
+	return (int) (unsigned long) of_device_get_match_data(dev);
 }
 
 #include <linux/of_address.h>
@@ -258,10 +247,10 @@ static int get_mdp_ver(struct platform_device *pdev)
 static int msm_init_vram(struct drm_device *dev)
 {
 	struct msm_drm_private *priv = dev->dev_private;
+	struct device_node *node;
 	unsigned long size = 0;
 	int ret = 0;
 
-#ifdef CONFIG_OF
 	/* In the device-tree world, we could have a 'memory-region'
 	 * phandle, which gives us a link to our "vram".  Allocating
 	 * is all nicely abstracted behind the dma api, but we need
@@ -278,7 +267,6 @@ static int msm_init_vram(struct drm_device *dev)
 	 *     as corruption on screen before we have a chance to
 	 *     load and do initial modeset)
 	 */
-	struct device_node *node;
 
 	node = of_parse_phandle(dev->dev->of_node, "memory-region", 0);
 	if (node) {
@@ -288,14 +276,12 @@ static int msm_init_vram(struct drm_device *dev)
 			return ret;
 		size = r.end - r.start;
 		DRM_INFO("using VRAM carveout: %lx@%pa\n", size, &r.start);
-	} else
-#endif
 
-	/* if we have no IOMMU, then we need to use carveout allocator.
-	 * Grab the entire CMA chunk carved out in early startup in
-	 * mach-msm:
-	 */
-	if (!iommu_present(&platform_bus_type)) {
+		/* if we have no IOMMU, then we need to use carveout allocator.
+		 * Grab the entire CMA chunk carved out in early startup in
+		 * mach-msm:
+		 */
+	} else if (!iommu_present(&platform_bus_type)) {
 		DRM_INFO("using %s VRAM carveout\n", vram);
 		size = memparse(vram, NULL);
 	}
@@ -1035,9 +1021,9 @@ static const struct dev_pm_ops msm_pm_ops = {
  * Componentized driver support:
  */
 
-#ifdef CONFIG_OF
-/* NOTE: the CONFIG_OF case duplicates the same code as exynos or imx
- * (or probably any other).. so probably some room for some helpers
+/*
+ * NOTE: duplication of the same code as exynos or imx (or probably any other).
+ * so probably some room for some helpers
  */
 static int compare_of(struct device *dev, void *data)
 {
@@ -1062,12 +1048,6 @@ static int add_components(struct device *dev, struct component_match **matchptr,
 
 	return 0;
 }
-#else
-static int compare_dev(struct device *dev, void *data)
-{
-	return dev == data;
-}
-#endif
 
 static int msm_drm_bind(struct device *dev)
 {
@@ -1091,35 +1071,9 @@ static const struct component_master_ops msm_drm_ops = {
 static int msm_pdev_probe(struct platform_device *pdev)
 {
 	struct component_match *match = NULL;
-#ifdef CONFIG_OF
+
 	add_components(&pdev->dev, &match, "connectors");
 	add_components(&pdev->dev, &match, "gpus");
-#else
-	/* For non-DT case, it kinda sucks.  We don't actually have a way
-	 * to know whether or not we are waiting for certain devices (or if
-	 * they are simply not present).  But for non-DT we only need to
-	 * care about apq8064/apq8060/etc (all mdp4/a3xx):
-	 */
-	static const char *devnames[] = {
-			"hdmi_msm.0", "kgsl-3d0.0",
-	};
-	int i;
-
-	DBG("Adding components..");
-
-	for (i = 0; i < ARRAY_SIZE(devnames); i++) {
-		struct device *dev;
-
-		dev = bus_find_device_by_name(&platform_bus_type,
-				NULL, devnames[i]);
-		if (!dev) {
-			dev_info(&pdev->dev, "still waiting for %s\n", devnames[i]);
-			return -EPROBE_DEFER;
-		}
-
-		component_match_add(&pdev->dev, &match, compare_dev, dev);
-	}
-#endif
 
 	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
 	return component_master_add_with_match(&pdev->dev, &msm_drm_ops, match);
@@ -1138,8 +1092,10 @@ static const struct platform_device_id msm_id[] = {
 };
 
 static const struct of_device_id dt_match[] = {
-	{ .compatible = "qcom,mdp" },      /* mdp4 */
-	{ .compatible = "qcom,mdss_mdp" }, /* mdp5 */
+	{ .compatible = "qcom,mdp4", .data = (void *) 4 },	/* mdp4 */
+	{ .compatible = "qcom,mdp5", .data = (void *) 5 },	/* mdp5 */
+	/* to support downstream DT files */
+	{ .compatible = "qcom,mdss_mdp", .data = (void *) 5 },  /* mdp5 */
 	{}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 3be7a56b14f1..c1e7bba2fdb7 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -31,14 +31,9 @@
 #include <linux/iommu.h>
 #include <linux/types.h>
 #include <linux/of_graph.h>
+#include <linux/of_device.h>
 #include <asm/sizes.h>
 
-#ifndef CONFIG_OF
-#include <mach/board.h>
-#include <mach/socinfo.h>
-#include <mach/iommu_domains.h>
-#endif
-
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
@@ -240,9 +235,9 @@ uint32_t msm_framebuffer_iova(struct drm_framebuffer *fb, int id, int plane);
 struct drm_gem_object *msm_framebuffer_bo(struct drm_framebuffer *fb, int plane);
 const struct msm_format *msm_framebuffer_format(struct drm_framebuffer *fb);
 struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
-		struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
 struct drm_framebuffer *msm_framebuffer_create(struct drm_device *dev,
-		struct drm_file *file, struct drm_mode_fb_cmd2 *mode_cmd);
+		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
 
 struct drm_fb_helper *msm_fbdev_init(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 121713281417..a474d6cf5d9f 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -138,7 +138,7 @@ const struct msm_format *msm_framebuffer_format(struct drm_framebuffer *fb)
 }
 
 struct drm_framebuffer *msm_framebuffer_create(struct drm_device *dev,
-		struct drm_file *file, struct drm_mode_fb_cmd2 *mode_cmd)
+		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *bos[4] = {0};
 	struct drm_framebuffer *fb;
@@ -168,7 +168,7 @@ out_unref:
 }
 
 struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
-		struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos)
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos)
 {
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_kms *kms = priv->kms;
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index 3f6ec077b51d..d95af6eba602 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -121,7 +121,7 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 		/* note: if fb creation failed, we can't rely on fb destroy
 		 * to unref the bo:
 		 */
-		drm_gem_object_unreference(fbdev->bo);
+		drm_gem_object_unreference_unlocked(fbdev->bo);
 		ret = PTR_ERR(fb);
 		goto fail;
 	}
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index c76cc853b08a..3cedb8d5c855 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -18,6 +18,7 @@
 #include <linux/spinlock.h>
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
+#include <linux/pfn_t.h>
 
 #include "msm_drv.h"
 #include "msm_gem.h"
@@ -222,7 +223,8 @@ int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
 			pfn, pfn << PAGE_SHIFT);
 
-	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
+	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
+			__pfn_to_pfn_t(pfn, PFN_DEV));
 
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild
index a34b437dbc8f..2527bf4ca5d9 100644
--- a/drivers/gpu/drm/nouveau/Kbuild
+++ b/drivers/gpu/drm/nouveau/Kbuild
@@ -24,7 +24,6 @@ nouveau-y += nouveau_hwmon.o
 nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
 nouveau-y += nouveau_nvif.o
 nouveau-$(CONFIG_NOUVEAU_PLATFORM_DRIVER) += nouveau_platform.o
-nouveau-y += nouveau_sysfs.o
 nouveau-y += nouveau_usif.o # userspace <-> nvif
 nouveau-y += nouveau_vga.o
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 3d96b49fe662..6f04397d43a7 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -1081,8 +1081,6 @@ nouveau_crtc_set_config(struct drm_mode_set *set)
 }
 
 static const struct drm_crtc_funcs nv04_crtc_funcs = {
-	.save = nv_crtc_save,
-	.restore = nv_crtc_restore,
 	.cursor_set = nv04_crtc_cursor_set,
 	.cursor_move = nv04_crtc_cursor_move,
 	.gamma_set = nv_crtc_gamma_set,
@@ -1123,6 +1121,9 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
 	nv_crtc->index = crtc_num;
 	nv_crtc->last_dpms = NV_DPMS_CLEARED;
 
+	nv_crtc->save = nv_crtc_save;
+	nv_crtc->restore = nv_crtc_restore;
+
 	drm_crtc_init(dev, &nv_crtc->base, &nv04_crtc_funcs);
 	drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
 	drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dac.c b/drivers/gpu/drm/nouveau/dispnv04/dac.c
index 78cb033bc015..b48eec395f07 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dac.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dac.c
@@ -504,8 +504,6 @@ static void nv04_dac_destroy(struct drm_encoder *encoder)
 
 static const struct drm_encoder_helper_funcs nv04_dac_helper_funcs = {
 	.dpms = nv04_dac_dpms,
-	.save = nv04_dac_save,
-	.restore = nv04_dac_restore,
 	.mode_fixup = nv04_dac_mode_fixup,
 	.prepare = nv04_dac_prepare,
 	.commit = nv04_dac_commit,
@@ -515,8 +513,6 @@ static const struct drm_encoder_helper_funcs nv04_dac_helper_funcs = {
 
 static const struct drm_encoder_helper_funcs nv17_dac_helper_funcs = {
 	.dpms = nv04_dac_dpms,
-	.save = nv04_dac_save,
-	.restore = nv04_dac_restore,
 	.mode_fixup = nv04_dac_mode_fixup,
 	.prepare = nv04_dac_prepare,
 	.commit = nv04_dac_commit,
@@ -545,12 +541,16 @@ nv04_dac_create(struct drm_connector *connector, struct dcb_output *entry)
 	nv_encoder->dcb = entry;
 	nv_encoder->or = ffs(entry->or) - 1;
 
+	nv_encoder->enc_save = nv04_dac_save;
+	nv_encoder->enc_restore = nv04_dac_restore;
+
 	if (nv_gf4_disp_arch(dev))
 		helper = &nv17_dac_helper_funcs;
 	else
 		helper = &nv04_dac_helper_funcs;
 
-	drm_encoder_init(dev, encoder, &nv04_dac_funcs, DRM_MODE_ENCODER_DAC);
+	drm_encoder_init(dev, encoder, &nv04_dac_funcs, DRM_MODE_ENCODER_DAC,
+			 NULL);
 	drm_encoder_helper_add(encoder, helper);
 
 	encoder->possible_crtcs = entry->heads;
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
index 429ab5e3025a..05bfd151d1d8 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
@@ -652,8 +652,6 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
 
 static const struct drm_encoder_helper_funcs nv04_lvds_helper_funcs = {
 	.dpms = nv04_lvds_dpms,
-	.save = nv04_dfp_save,
-	.restore = nv04_dfp_restore,
 	.mode_fixup = nv04_dfp_mode_fixup,
 	.prepare = nv04_dfp_prepare,
 	.commit = nv04_dfp_commit,
@@ -663,8 +661,6 @@ static const struct drm_encoder_helper_funcs nv04_lvds_helper_funcs = {
 
 static const struct drm_encoder_helper_funcs nv04_tmds_helper_funcs = {
 	.dpms = nv04_tmds_dpms,
-	.save = nv04_dfp_save,
-	.restore = nv04_dfp_restore,
 	.mode_fixup = nv04_dfp_mode_fixup,
 	.prepare = nv04_dfp_prepare,
 	.commit = nv04_dfp_commit,
@@ -701,12 +697,15 @@ nv04_dfp_create(struct drm_connector *connector, struct dcb_output *entry)
 	if (!nv_encoder)
 		return -ENOMEM;
 
+	nv_encoder->enc_save = nv04_dfp_save;
+	nv_encoder->enc_restore = nv04_dfp_restore;
+
 	encoder = to_drm_encoder(nv_encoder);
 
 	nv_encoder->dcb = entry;
 	nv_encoder->or = ffs(entry->or) - 1;
 
-	drm_encoder_init(connector->dev, encoder, &nv04_dfp_funcs, type);
+	drm_encoder_init(connector->dev, encoder, &nv04_dfp_funcs, type, NULL);
 	drm_encoder_helper_add(encoder, helper);
 
 	encoder->possible_crtcs = entry->heads;
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
index 9e650081c357..b4a6bc433ef5 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
@@ -39,7 +39,8 @@ nv04_display_create(struct drm_device *dev)
 	struct dcb_table *dcb = &drm->vbios.dcb;
 	struct drm_connector *connector, *ct;
 	struct drm_encoder *encoder;
-	struct drm_crtc *crtc;
+	struct nouveau_encoder *nv_encoder;
+	struct nouveau_crtc *crtc;
 	struct nv04_display *disp;
 	int i, ret;
 
@@ -107,14 +108,11 @@ nv04_display_create(struct drm_device *dev)
 	}
 
 	/* Save previous state */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-		crtc->funcs->save(crtc);
-
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		const struct drm_encoder_helper_funcs *func = encoder->helper_private;
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head)
+		crtc->save(&crtc->base);
 
-		func->save(encoder);
-	}
+	list_for_each_entry(nv_encoder, &dev->mode_config.encoder_list, base.base.head)
+		nv_encoder->enc_save(&nv_encoder->base.base);
 
 	nouveau_overlay_init(dev);
 
@@ -126,8 +124,9 @@ nv04_display_destroy(struct drm_device *dev)
 {
 	struct nv04_display *disp = nv04_display(dev);
 	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct drm_encoder *encoder;
+	struct nouveau_encoder *encoder;
 	struct drm_crtc *crtc;
+	struct nouveau_crtc *nv_crtc;
 
 	/* Turn every CRTC off. */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -139,14 +138,11 @@ nv04_display_destroy(struct drm_device *dev)
 	}
 
 	/* Restore state */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		const struct drm_encoder_helper_funcs *func = encoder->helper_private;
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head)
+		encoder->enc_restore(&encoder->base.base);
 
-		func->restore(encoder);
-	}
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-		crtc->funcs->restore(crtc);
+	list_for_each_entry(nv_crtc, &dev->mode_config.crtc_list, base.head)
+		nv_crtc->restore(&nv_crtc->base);
 
 	nouveau_hw_save_vga_fonts(dev, 0);
 
@@ -159,8 +155,8 @@ nv04_display_destroy(struct drm_device *dev)
 int
 nv04_display_init(struct drm_device *dev)
 {
-	struct drm_encoder *encoder;
-	struct drm_crtc *crtc;
+	struct nouveau_encoder *encoder;
+	struct nouveau_crtc *crtc;
 
 	/* meh.. modeset apparently doesn't setup all the regs and depends
 	 * on pre-existing state, for now load the state of the card *before*
@@ -170,14 +166,11 @@ nv04_display_init(struct drm_device *dev)
 	 * save/restore "pre-load" state, but more general so we can save
 	 * on suspend too.
 	 */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		const struct drm_encoder_helper_funcs *func = encoder->helper_private;
-
-		func->restore(encoder);
-	}
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head)
+		crtc->save(&crtc->base);
 
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-		crtc->funcs->restore(crtc);
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head)
+		encoder->enc_save(&encoder->base.base);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
index 5345eb5378a8..54e9fb9eb5c0 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
@@ -192,8 +192,6 @@ static const struct drm_encoder_funcs nv04_tv_funcs = {
 
 static const struct drm_encoder_helper_funcs nv04_tv_helper_funcs = {
 	.dpms = nv04_tv_dpms,
-	.save = drm_i2c_encoder_save,
-	.restore = drm_i2c_encoder_restore,
 	.mode_fixup = drm_i2c_encoder_mode_fixup,
 	.prepare = nv04_tv_prepare,
 	.commit = nv04_tv_commit,
@@ -225,9 +223,13 @@ nv04_tv_create(struct drm_connector *connector, struct dcb_output *entry)
 	/* Initialize the common members */
 	encoder = to_drm_encoder(nv_encoder);
 
-	drm_encoder_init(dev, encoder, &nv04_tv_funcs, DRM_MODE_ENCODER_TVDAC);
+	drm_encoder_init(dev, encoder, &nv04_tv_funcs, DRM_MODE_ENCODER_TVDAC,
+			 NULL);
 	drm_encoder_helper_add(encoder, &nv04_tv_helper_funcs);
 
+	nv_encoder->enc_save = drm_i2c_encoder_save;
+	nv_encoder->enc_restore = drm_i2c_encoder_restore;
+
 	encoder->possible_crtcs = entry->heads;
 	encoder->possible_clones = 0;
 	nv_encoder->dcb = entry;
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
index b734195d80a0..163317d26de9 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
@@ -769,10 +769,8 @@ static void nv17_tv_destroy(struct drm_encoder *encoder)
 	kfree(tv_enc);
 }
 
-static struct drm_encoder_helper_funcs nv17_tv_helper_funcs = {
+static const struct drm_encoder_helper_funcs nv17_tv_helper_funcs = {
 	.dpms = nv17_tv_dpms,
-	.save = nv17_tv_save,
-	.restore = nv17_tv_restore,
 	.mode_fixup = nv17_tv_mode_fixup,
 	.prepare = nv17_tv_prepare,
 	.commit = nv17_tv_commit,
@@ -780,14 +778,14 @@ static struct drm_encoder_helper_funcs nv17_tv_helper_funcs = {
 	.detect = nv17_tv_detect,
 };
 
-static struct drm_encoder_slave_funcs nv17_tv_slave_funcs = {
+static const struct drm_encoder_slave_funcs nv17_tv_slave_funcs = {
 	.get_modes = nv17_tv_get_modes,
 	.mode_valid = nv17_tv_mode_valid,
 	.create_resources = nv17_tv_create_resources,
 	.set_property = nv17_tv_set_property,
 };
 
-static struct drm_encoder_funcs nv17_tv_funcs = {
+static const struct drm_encoder_funcs nv17_tv_funcs = {
 	.destroy = nv17_tv_destroy,
 };
 
@@ -816,10 +814,14 @@ nv17_tv_create(struct drm_connector *connector, struct dcb_output *entry)
 	tv_enc->base.dcb = entry;
 	tv_enc->base.or = ffs(entry->or) - 1;
 
-	drm_encoder_init(dev, encoder, &nv17_tv_funcs, DRM_MODE_ENCODER_TVDAC);
+	drm_encoder_init(dev, encoder, &nv17_tv_funcs, DRM_MODE_ENCODER_TVDAC,
+			 NULL);
 	drm_encoder_helper_add(encoder, &nv17_tv_helper_funcs);
 	to_encoder_slave(encoder)->slave_funcs = &nv17_tv_slave_funcs;
 
+	tv_enc->base.enc_save = nv17_tv_save;
+	tv_enc->base.enc_restore = nv17_tv_restore;
+
 	encoder->possible_crtcs = entry->heads;
 	encoder->possible_clones = 0;
 
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0002.h b/drivers/gpu/drm/nouveau/include/nvif/cl0002.h
new file mode 100644
index 000000000000..6d72ed38da32
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0002.h
@@ -0,0 +1,66 @@
+#ifndef __NVIF_CL0002_H__
+#define __NVIF_CL0002_H__
+
+struct nv_dma_v0 {
+	__u8  version;
+#define NV_DMA_V0_TARGET_VM                                                0x00
+#define NV_DMA_V0_TARGET_VRAM                                              0x01
+#define NV_DMA_V0_TARGET_PCI                                               0x02
+#define NV_DMA_V0_TARGET_PCI_US                                            0x03
+#define NV_DMA_V0_TARGET_AGP                                               0x04
+	__u8  target;
+#define NV_DMA_V0_ACCESS_VM                                                0x00
+#define NV_DMA_V0_ACCESS_RD                                                0x01
+#define NV_DMA_V0_ACCESS_WR                                                0x02
+#define NV_DMA_V0_ACCESS_RDWR                 (NV_DMA_V0_ACCESS_RD | NV_DMA_V0_ACCESS_WR)
+	__u8  access;
+	__u8  pad03[5];
+	__u64 start;
+	__u64 limit;
+	/* ... chipset-specific class data */
+};
+
+struct nv50_dma_v0 {
+	__u8  version;
+#define NV50_DMA_V0_PRIV_VM                                                0x00
+#define NV50_DMA_V0_PRIV_US                                                0x01
+#define NV50_DMA_V0_PRIV__S                                                0x02
+	__u8  priv;
+#define NV50_DMA_V0_PART_VM                                                0x00
+#define NV50_DMA_V0_PART_256                                               0x01
+#define NV50_DMA_V0_PART_1KB                                               0x02
+	__u8  part;
+#define NV50_DMA_V0_COMP_NONE                                              0x00
+#define NV50_DMA_V0_COMP_1                                                 0x01
+#define NV50_DMA_V0_COMP_2                                                 0x02
+#define NV50_DMA_V0_COMP_VM                                                0x03
+	__u8  comp;
+#define NV50_DMA_V0_KIND_PITCH                                             0x00
+#define NV50_DMA_V0_KIND_VM                                                0x7f
+	__u8  kind;
+	__u8  pad05[3];
+};
+
+struct gf100_dma_v0 {
+	__u8  version;
+#define GF100_DMA_V0_PRIV_VM                                               0x00
+#define GF100_DMA_V0_PRIV_US                                               0x01
+#define GF100_DMA_V0_PRIV__S                                               0x02
+	__u8  priv;
+#define GF100_DMA_V0_KIND_PITCH                                            0x00
+#define GF100_DMA_V0_KIND_VM                                               0xff
+	__u8  kind;
+	__u8  pad03[5];
+};
+
+struct gf119_dma_v0 {
+	__u8  version;
+#define GF119_DMA_V0_PAGE_LP                                               0x00
+#define GF119_DMA_V0_PAGE_SP                                               0x01
+	__u8  page;
+#define GF119_DMA_V0_KIND_PITCH                                            0x00
+#define GF119_DMA_V0_KIND_VM                                               0xff
+	__u8  kind;
+	__u8  pad03[5];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0046.h b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h
new file mode 100644
index 000000000000..a6a71f4ad91e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h
@@ -0,0 +1,28 @@
+#ifndef __NVIF_CL0046_H__
+#define __NVIF_CL0046_H__
+
+#define NV04_DISP_NTFY_VBLANK                                              0x00
+#define NV04_DISP_NTFY_CONN                                                0x01
+
+struct nv04_disp_mthd_v0 {
+	__u8  version;
+#define NV04_DISP_SCANOUTPOS                                               0x00
+	__u8  method;
+	__u8  head;
+	__u8  pad03[5];
+};
+
+struct nv04_disp_scanoutpos_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__s64 time[2];
+	__u16 vblanks;
+	__u16 vblanke;
+	__u16 vtotal;
+	__u16 vline;
+	__u16 hblanks;
+	__u16 hblanke;
+	__u16 htotal;
+	__u16 hline;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl006b.h b/drivers/gpu/drm/nouveau/include/nvif/cl006b.h
new file mode 100644
index 000000000000..309ab8a3d9e8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl006b.h
@@ -0,0 +1,11 @@
+#ifndef __NVIF_CL006B_H__
+#define __NVIF_CL006B_H__
+
+struct nv03_channel_dma_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[2];
+	__u32 offset;
+	__u64 pushbuf;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
new file mode 100644
index 000000000000..331620a52afa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
@@ -0,0 +1,45 @@
+#ifndef __NVIF_CL0080_H__
+#define __NVIF_CL0080_H__
+
+struct nv_device_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 device;	/* device identifier, ~0 for client default */
+};
+
+#define NV_DEVICE_V0_INFO                                                  0x00
+#define NV_DEVICE_V0_TIME                                                  0x01
+
+struct nv_device_info_v0 {
+	__u8  version;
+#define NV_DEVICE_INFO_V0_IGP                                              0x00
+#define NV_DEVICE_INFO_V0_PCI                                              0x01
+#define NV_DEVICE_INFO_V0_AGP                                              0x02
+#define NV_DEVICE_INFO_V0_PCIE                                             0x03
+#define NV_DEVICE_INFO_V0_SOC                                              0x04
+	__u8  platform;
+	__u16 chipset;	/* from NV_PMC_BOOT_0 */
+	__u8  revision;	/* from NV_PMC_BOOT_0 */
+#define NV_DEVICE_INFO_V0_TNT                                              0x01
+#define NV_DEVICE_INFO_V0_CELSIUS                                          0x02
+#define NV_DEVICE_INFO_V0_KELVIN                                           0x03
+#define NV_DEVICE_INFO_V0_RANKINE                                          0x04
+#define NV_DEVICE_INFO_V0_CURIE                                            0x05
+#define NV_DEVICE_INFO_V0_TESLA                                            0x06
+#define NV_DEVICE_INFO_V0_FERMI                                            0x07
+#define NV_DEVICE_INFO_V0_KEPLER                                           0x08
+#define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
+	__u8  family;
+	__u8  pad06[2];
+	__u64 ram_size;
+	__u64 ram_user;
+	char  chip[16];
+	char  name[64];
+};
+
+struct nv_device_time_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 time;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506e.h b/drivers/gpu/drm/nouveau/include/nvif/cl506e.h
new file mode 100644
index 000000000000..aa94b8cf9679
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl506e.h
@@ -0,0 +1,12 @@
+#ifndef __NVIF_CL506E_H__
+#define __NVIF_CL506E_H__
+
+struct nv50_channel_dma_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[6];
+	__u64 vm;
+	__u64 pushbuf;
+	__u64 offset;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506f.h b/drivers/gpu/drm/nouveau/include/nvif/cl506f.h
new file mode 100644
index 000000000000..3b7101966de4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl506f.h
@@ -0,0 +1,13 @@
+#ifndef __NVIF_CL506F_H__
+#define __NVIF_CL506F_H__
+
+struct nv50_channel_gpfifo_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[2];
+	__u32 ilength;
+	__u64 ioffset;
+	__u64 pushbuf;
+	__u64 vm;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
new file mode 100644
index 000000000000..d15c296b5f33
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
@@ -0,0 +1,99 @@
+#ifndef __NVIF_CL5070_H__
+#define __NVIF_CL5070_H__
+
+#define NV50_DISP_MTHD                                                     0x00
+
+struct nv50_disp_mthd_v0 {
+	__u8  version;
+#define NV50_DISP_SCANOUTPOS                                               0x00
+	__u8  method;
+	__u8  head;
+	__u8  pad03[5];
+};
+
+struct nv50_disp_scanoutpos_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__s64 time[2];
+	__u16 vblanks;
+	__u16 vblanke;
+	__u16 vtotal;
+	__u16 vline;
+	__u16 hblanks;
+	__u16 hblanke;
+	__u16 htotal;
+	__u16 hline;
+};
+
+struct nv50_disp_mthd_v1 {
+	__u8  version;
+#define NV50_DISP_MTHD_V1_DAC_PWR                                          0x10
+#define NV50_DISP_MTHD_V1_DAC_LOAD                                         0x11
+#define NV50_DISP_MTHD_V1_SOR_PWR                                          0x20
+#define NV50_DISP_MTHD_V1_SOR_HDA_ELD                                      0x21
+#define NV50_DISP_MTHD_V1_SOR_HDMI_PWR                                     0x22
+#define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT                                  0x23
+#define NV50_DISP_MTHD_V1_SOR_DP_PWR                                       0x24
+#define NV50_DISP_MTHD_V1_PIOR_PWR                                         0x30
+	__u8  method;
+	__u16 hasht;
+	__u16 hashm;
+	__u8  pad06[2];
+};
+
+struct nv50_disp_dac_pwr_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  data;
+	__u8  vsync;
+	__u8  hsync;
+	__u8  pad05[3];
+};
+
+struct nv50_disp_dac_load_v0 {
+	__u8  version;
+	__u8  load;
+	__u8  pad02[2];
+	__u32 data;
+};
+
+struct nv50_disp_sor_pwr_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  pad02[6];
+};
+
+struct nv50_disp_sor_hda_eld_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u8  data[];
+};
+
+struct nv50_disp_sor_hdmi_pwr_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  max_ac_packet;
+	__u8  rekey;
+	__u8  pad04[4];
+};
+
+struct nv50_disp_sor_lvds_script_v0 {
+	__u8  version;
+	__u8  pad01[1];
+	__u16 script;
+	__u8  pad04[4];
+};
+
+struct nv50_disp_sor_dp_pwr_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  pad02[6];
+};
+
+struct nv50_disp_pior_pwr_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  type;
+	__u8  pad03[5];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl507a.h b/drivers/gpu/drm/nouveau/include/nvif/cl507a.h
new file mode 100644
index 000000000000..12e0643b78bd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl507a.h
@@ -0,0 +1,11 @@
+#ifndef __NVIF_CL507A_H__
+#define __NVIF_CL507A_H__
+
+struct nv50_disp_cursor_v0 {
+	__u8  version;
+	__u8  head;
+	__u8  pad02[6];
+};
+
+#define NV50_DISP_CURSOR_V0_NTFY_UEVENT                                    0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl507b.h b/drivers/gpu/drm/nouveau/include/nvif/cl507b.h
new file mode 100644
index 000000000000..99e9d8c47f60
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl507b.h
@@ -0,0 +1,11 @@
+#ifndef __NVIF_CL507B_H__
+#define __NVIF_CL507B_H__
+
+struct nv50_disp_overlay_v0 {
+	__u8  version;
+	__u8  head;
+	__u8  pad02[6];
+};
+
+#define NV50_DISP_OVERLAY_V0_NTFY_UEVENT                                   0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl507c.h b/drivers/gpu/drm/nouveau/include/nvif/cl507c.h
new file mode 100644
index 000000000000..6af70dbdfd9f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl507c.h
@@ -0,0 +1,12 @@
+#ifndef __NVIF_CL507C_H__
+#define __NVIF_CL507C_H__
+
+struct nv50_disp_base_channel_dma_v0 {
+	__u8  version;
+	__u8  head;
+	__u8  pad02[6];
+	__u64 pushbuf;
+};
+
+#define NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT                          0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl507d.h b/drivers/gpu/drm/nouveau/include/nvif/cl507d.h
new file mode 100644
index 000000000000..5ab0c9e4c6a3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl507d.h
@@ -0,0 +1,11 @@
+#ifndef __NVIF_CL507D_H__
+#define __NVIF_CL507D_H__
+
+struct nv50_disp_core_channel_dma_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 pushbuf;
+};
+
+#define NV50_DISP_CORE_CHANNEL_DMA_V0_NTFY_UEVENT                          0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl507e.h b/drivers/gpu/drm/nouveau/include/nvif/cl507e.h
new file mode 100644
index 000000000000..c06209f3cac4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl507e.h
@@ -0,0 +1,12 @@
+#ifndef __NVIF_CL507E_H__
+#define __NVIF_CL507E_H__
+
+struct nv50_disp_overlay_channel_dma_v0 {
+	__u8  version;
+	__u8  head;
+	__u8  pad02[6];
+	__u64 pushbuf;
+};
+
+#define NV50_DISP_OVERLAY_CHANNEL_DMA_V0_NTFY_UEVENT                       0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h
new file mode 100644
index 000000000000..05e6ef7cd190
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h
@@ -0,0 +1,14 @@
+#ifndef __NVIF_CL826E_H__
+#define __NVIF_CL826E_H__
+
+struct g82_channel_dma_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[6];
+	__u64 vm;
+	__u64 pushbuf;
+	__u64 offset;
+};
+
+#define G82_CHANNEL_DMA_V0_NTFY_UEVENT                                     0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h
new file mode 100644
index 000000000000..cecafcb1e954
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h
@@ -0,0 +1,15 @@
+#ifndef __NVIF_CL826F_H__
+#define __NVIF_CL826F_H__
+
+struct g82_channel_gpfifo_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[2];
+	__u32 ilength;
+	__u64 ioffset;
+	__u64 pushbuf;
+	__u64 vm;
+};
+
+#define G82_CHANNEL_GPFIFO_V0_NTFY_UEVENT                                  0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h
new file mode 100644
index 000000000000..2caf0838fcfd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h
@@ -0,0 +1,14 @@
+#ifndef __NVIF_CL906F_H__
+#define __NVIF_CL906F_H__
+
+struct fermi_channel_gpfifo_v0 {
+	__u8  version;
+	__u8  chid;
+	__u8  pad02[2];
+	__u32 ilength;
+	__u64 ioffset;
+	__u64 vm;
+};
+
+#define FERMI_CHANNEL_GPFIFO_V0_NTFY_UEVENT                                0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl9097.h b/drivers/gpu/drm/nouveau/include/nvif/cl9097.h
new file mode 100644
index 000000000000..4057676d2981
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl9097.h
@@ -0,0 +1,44 @@
+#ifndef __NVIF_CL9097_H__
+#define __NVIF_CL9097_H__
+
+#define FERMI_A_ZBC_COLOR                                                  0x00
+#define FERMI_A_ZBC_DEPTH                                                  0x01
+
+struct fermi_a_zbc_color_v0 {
+	__u8  version;
+#define FERMI_A_ZBC_COLOR_V0_FMT_ZERO                                      0x01
+#define FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE                                 0x02
+#define FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32                       0x04
+#define FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16                           0x08
+#define FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16                       0x0c
+#define FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16                       0x10
+#define FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16                       0x14
+#define FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16                       0x16
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8                                  0x18
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8                               0x1c
+#define FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10                               0x20
+#define FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10                           0x24
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8                                  0x28
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8                               0x2c
+#define FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8                              0x30
+#define FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8                              0x34
+#define FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8                              0x38
+#define FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10                               0x3c
+#define FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11                              0x40
+	__u8  format;
+	__u8  index;
+	__u8  pad03[5];
+	__u32 ds[4];
+	__u32 l2[4];
+};
+
+struct fermi_a_zbc_depth_v0 {
+	__u8  version;
+#define FERMI_A_ZBC_DEPTH_V0_FMT_FP32                                      0x01
+	__u8  format;
+	__u8  index;
+	__u8  pad03[5];
+	__u32 ds;
+	__u32 l2;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h
new file mode 100644
index 000000000000..85b7827eb782
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h
@@ -0,0 +1,21 @@
+#ifndef __NVIF_CLA06F_H__
+#define __NVIF_CLA06F_H__
+
+struct kepler_channel_gpfifo_a_v0 {
+	__u8  version;
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR                               0x01
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSPDEC                           0x02
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSPPP                            0x04
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSVLD                            0x08
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE0                              0x10
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE1                              0x20
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_ENC                              0x40
+	__u8  engine;
+	__u16 chid;
+	__u32 ilength;
+	__u64 ioffset;
+	__u64 vm;
+};
+
+#define KEPLER_CHANNEL_GPFIFO_A_V0_NTFY_UEVENT                             0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h
index 95a64d89547c..4179cd65ac0a 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/class.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/class.h
@@ -1,16 +1,21 @@
 #ifndef __NVIF_CLASS_H__
 #define __NVIF_CLASS_H__
 
-/*******************************************************************************
- * class identifiers
- ******************************************************************************/
+/* these class numbers are made up by us, and not nvidia-assigned */
+#define NVIF_CLASS_CONTROL                                    /* if0001.h */ -1
+#define NVIF_CLASS_PERFMON                                    /* if0002.h */ -2
+#define NVIF_CLASS_PERFDOM                                    /* if0003.h */ -3
+#define NVIF_CLASS_SW_NV04                                    /* if0004.h */ -4
+#define NVIF_CLASS_SW_NV10                                    /* if0005.h */ -5
+#define NVIF_CLASS_SW_NV50                                    /* if0005.h */ -6
+#define NVIF_CLASS_SW_GF100                                   /* if0005.h */ -7
 
 /* the below match nvidia-assigned (either in hw, or sw) class numbers */
-#define NV_DEVICE                                                    0x00000080
+#define NV_DEVICE                                     /* cl0080.h */ 0x00000080
 
-#define NV_DMA_FROM_MEMORY                                           0x00000002
-#define NV_DMA_TO_MEMORY                                             0x00000003
-#define NV_DMA_IN_MEMORY                                             0x0000003d
+#define NV_DMA_FROM_MEMORY                            /* cl0002.h */ 0x00000002
+#define NV_DMA_TO_MEMORY                              /* cl0002.h */ 0x00000003
+#define NV_DMA_IN_MEMORY                              /* cl0002.h */ 0x0000003d
 
 #define FERMI_TWOD_A                                                 0x0000902d
 
@@ -19,85 +24,85 @@
 #define KEPLER_INLINE_TO_MEMORY_A                                    0x0000a040
 #define KEPLER_INLINE_TO_MEMORY_B                                    0x0000a140
 
-#define NV04_DISP                                                    0x00000046
-
-#define NV03_CHANNEL_DMA                                             0x0000006b
-#define NV10_CHANNEL_DMA                                             0x0000006e
-#define NV17_CHANNEL_DMA                                             0x0000176e
-#define NV40_CHANNEL_DMA                                             0x0000406e
-#define NV50_CHANNEL_DMA                                             0x0000506e
-#define G82_CHANNEL_DMA                                              0x0000826e
-
-#define NV50_CHANNEL_GPFIFO                                          0x0000506f
-#define G82_CHANNEL_GPFIFO                                           0x0000826f
-#define FERMI_CHANNEL_GPFIFO                                         0x0000906f
-#define KEPLER_CHANNEL_GPFIFO_A                                      0x0000a06f
-#define MAXWELL_CHANNEL_GPFIFO_A                                     0x0000b06f
-
-#define NV50_DISP                                                    0x00005070
-#define G82_DISP                                                     0x00008270
-#define GT200_DISP                                                   0x00008370
-#define GT214_DISP                                                   0x00008570
-#define GT206_DISP                                                   0x00008870
-#define GF110_DISP                                                   0x00009070
-#define GK104_DISP                                                   0x00009170
-#define GK110_DISP                                                   0x00009270
-#define GM107_DISP                                                   0x00009470
-#define GM204_DISP                                                   0x00009570
+#define NV04_DISP                                     /* cl0046.h */ 0x00000046
+
+#define NV03_CHANNEL_DMA                              /* cl506b.h */ 0x0000006b
+#define NV10_CHANNEL_DMA                              /* cl506b.h */ 0x0000006e
+#define NV17_CHANNEL_DMA                              /* cl506b.h */ 0x0000176e
+#define NV40_CHANNEL_DMA                              /* cl506b.h */ 0x0000406e
+#define NV50_CHANNEL_DMA                              /* cl506e.h */ 0x0000506e
+#define G82_CHANNEL_DMA                               /* cl826e.h */ 0x0000826e
+
+#define NV50_CHANNEL_GPFIFO                           /* cl506f.h */ 0x0000506f
+#define G82_CHANNEL_GPFIFO                            /* cl826f.h */ 0x0000826f
+#define FERMI_CHANNEL_GPFIFO                          /* cl906f.h */ 0x0000906f
+#define KEPLER_CHANNEL_GPFIFO_A                       /* cla06f.h */ 0x0000a06f
+#define MAXWELL_CHANNEL_GPFIFO_A                      /* cla06f.h */ 0x0000b06f
+
+#define NV50_DISP                                     /* cl5070.h */ 0x00005070
+#define G82_DISP                                      /* cl5070.h */ 0x00008270
+#define GT200_DISP                                    /* cl5070.h */ 0x00008370
+#define GT214_DISP                                    /* cl5070.h */ 0x00008570
+#define GT206_DISP                                    /* cl5070.h */ 0x00008870
+#define GF110_DISP                                    /* cl5070.h */ 0x00009070
+#define GK104_DISP                                    /* cl5070.h */ 0x00009170
+#define GK110_DISP                                    /* cl5070.h */ 0x00009270
+#define GM107_DISP                                    /* cl5070.h */ 0x00009470
+#define GM204_DISP                                    /* cl5070.h */ 0x00009570
 
 #define NV31_MPEG                                                    0x00003174
 #define G82_MPEG                                                     0x00008274
 
 #define NV74_VP2                                                     0x00007476
 
-#define NV50_DISP_CURSOR                                             0x0000507a
-#define G82_DISP_CURSOR                                              0x0000827a
-#define GT214_DISP_CURSOR                                            0x0000857a
-#define GF110_DISP_CURSOR                                            0x0000907a
-#define GK104_DISP_CURSOR                                            0x0000917a
-
-#define NV50_DISP_OVERLAY                                            0x0000507b
-#define G82_DISP_OVERLAY                                             0x0000827b
-#define GT214_DISP_OVERLAY                                           0x0000857b
-#define GF110_DISP_OVERLAY                                           0x0000907b
-#define GK104_DISP_OVERLAY                                           0x0000917b
-
-#define NV50_DISP_BASE_CHANNEL_DMA                                   0x0000507c
-#define G82_DISP_BASE_CHANNEL_DMA                                    0x0000827c
-#define GT200_DISP_BASE_CHANNEL_DMA                                  0x0000837c
-#define GT214_DISP_BASE_CHANNEL_DMA                                  0x0000857c
-#define GF110_DISP_BASE_CHANNEL_DMA                                  0x0000907c
-#define GK104_DISP_BASE_CHANNEL_DMA                                  0x0000917c
-#define GK110_DISP_BASE_CHANNEL_DMA                                  0x0000927c
-
-#define NV50_DISP_CORE_CHANNEL_DMA                                   0x0000507d
-#define G82_DISP_CORE_CHANNEL_DMA                                    0x0000827d
-#define GT200_DISP_CORE_CHANNEL_DMA                                  0x0000837d
-#define GT214_DISP_CORE_CHANNEL_DMA                                  0x0000857d
-#define GT206_DISP_CORE_CHANNEL_DMA                                  0x0000887d
-#define GF110_DISP_CORE_CHANNEL_DMA                                  0x0000907d
-#define GK104_DISP_CORE_CHANNEL_DMA                                  0x0000917d
-#define GK110_DISP_CORE_CHANNEL_DMA                                  0x0000927d
-#define GM107_DISP_CORE_CHANNEL_DMA                                  0x0000947d
-#define GM204_DISP_CORE_CHANNEL_DMA                                  0x0000957d
-
-#define NV50_DISP_OVERLAY_CHANNEL_DMA                                0x0000507e
-#define G82_DISP_OVERLAY_CHANNEL_DMA                                 0x0000827e
-#define GT200_DISP_OVERLAY_CHANNEL_DMA                               0x0000837e
-#define GT214_DISP_OVERLAY_CHANNEL_DMA                               0x0000857e
-#define GF110_DISP_OVERLAY_CONTROL_DMA                               0x0000907e
-#define GK104_DISP_OVERLAY_CONTROL_DMA                               0x0000917e
-
-#define FERMI_A                                                      0x00009097
-#define FERMI_B                                                      0x00009197
-#define FERMI_C                                                      0x00009297
-
-#define KEPLER_A                                                     0x0000a097
-#define KEPLER_B                                                     0x0000a197
-#define KEPLER_C                                                     0x0000a297
-
-#define MAXWELL_A                                                    0x0000b097
-#define MAXWELL_B                                                    0x0000b197
+#define NV50_DISP_CURSOR                              /* cl507a.h */ 0x0000507a
+#define G82_DISP_CURSOR                               /* cl507a.h */ 0x0000827a
+#define GT214_DISP_CURSOR                             /* cl507a.h */ 0x0000857a
+#define GF110_DISP_CURSOR                             /* cl507a.h */ 0x0000907a
+#define GK104_DISP_CURSOR                             /* cl507a.h */ 0x0000917a
+
+#define NV50_DISP_OVERLAY                             /* cl507b.h */ 0x0000507b
+#define G82_DISP_OVERLAY                              /* cl507b.h */ 0x0000827b
+#define GT214_DISP_OVERLAY                            /* cl507b.h */ 0x0000857b
+#define GF110_DISP_OVERLAY                            /* cl507b.h */ 0x0000907b
+#define GK104_DISP_OVERLAY                            /* cl507b.h */ 0x0000917b
+
+#define NV50_DISP_BASE_CHANNEL_DMA                    /* cl507c.h */ 0x0000507c
+#define G82_DISP_BASE_CHANNEL_DMA                     /* cl507c.h */ 0x0000827c
+#define GT200_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000837c
+#define GT214_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000857c
+#define GF110_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000907c
+#define GK104_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000917c
+#define GK110_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000927c
+
+#define NV50_DISP_CORE_CHANNEL_DMA                    /* cl507d.h */ 0x0000507d
+#define G82_DISP_CORE_CHANNEL_DMA                     /* cl507d.h */ 0x0000827d
+#define GT200_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000837d
+#define GT214_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000857d
+#define GT206_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000887d
+#define GF110_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000907d
+#define GK104_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000917d
+#define GK110_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000927d
+#define GM107_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000947d
+#define GM204_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000957d
+
+#define NV50_DISP_OVERLAY_CHANNEL_DMA                 /* cl507e.h */ 0x0000507e
+#define G82_DISP_OVERLAY_CHANNEL_DMA                  /* cl507e.h */ 0x0000827e
+#define GT200_DISP_OVERLAY_CHANNEL_DMA                /* cl507e.h */ 0x0000837e
+#define GT214_DISP_OVERLAY_CHANNEL_DMA                /* cl507e.h */ 0x0000857e
+#define GF110_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000907e
+#define GK104_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000917e
+
+#define FERMI_A                                       /* cl9097.h */ 0x00009097
+#define FERMI_B                                       /* cl9097.h */ 0x00009197
+#define FERMI_C                                       /* cl9097.h */ 0x00009297
+
+#define KEPLER_A                                      /* cl9097.h */ 0x0000a097
+#define KEPLER_B                                      /* cl9097.h */ 0x0000a197
+#define KEPLER_C                                      /* cl9097.h */ 0x0000a297
+
+#define MAXWELL_A                                     /* cl9097.h */ 0x0000b097
+#define MAXWELL_B                                     /* cl9097.h */ 0x0000b197
 
 #define NV74_BSP                                                     0x000074b0
 
@@ -133,540 +138,4 @@
 #define MAXWELL_COMPUTE_B                                            0x0000b1c0
 
 #define NV74_CIPHER                                                  0x000074c1
-
-/*******************************************************************************
- * client
- ******************************************************************************/
-
-#define NV_CLIENT_DEVLIST                                                  0x00
-
-struct nv_client_devlist_v0 {
-	__u8  version;
-	__u8  count;
-	__u8  pad02[6];
-	__u64 device[];
-};
-
-
-/*******************************************************************************
- * device
- ******************************************************************************/
-
-struct nv_device_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__u64 device;	/* device identifier, ~0 for client default */
-};
-
-#define NV_DEVICE_V0_INFO                                                  0x00
-#define NV_DEVICE_V0_TIME                                                  0x01
-
-struct nv_device_info_v0 {
-	__u8  version;
-#define NV_DEVICE_INFO_V0_IGP                                              0x00
-#define NV_DEVICE_INFO_V0_PCI                                              0x01
-#define NV_DEVICE_INFO_V0_AGP                                              0x02
-#define NV_DEVICE_INFO_V0_PCIE                                             0x03
-#define NV_DEVICE_INFO_V0_SOC                                              0x04
-	__u8  platform;
-	__u16 chipset;	/* from NV_PMC_BOOT_0 */
-	__u8  revision;	/* from NV_PMC_BOOT_0 */
-#define NV_DEVICE_INFO_V0_TNT                                              0x01
-#define NV_DEVICE_INFO_V0_CELSIUS                                          0x02
-#define NV_DEVICE_INFO_V0_KELVIN                                           0x03
-#define NV_DEVICE_INFO_V0_RANKINE                                          0x04
-#define NV_DEVICE_INFO_V0_CURIE                                            0x05
-#define NV_DEVICE_INFO_V0_TESLA                                            0x06
-#define NV_DEVICE_INFO_V0_FERMI                                            0x07
-#define NV_DEVICE_INFO_V0_KEPLER                                           0x08
-#define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
-	__u8  family;
-	__u8  pad06[2];
-	__u64 ram_size;
-	__u64 ram_user;
-	char  chip[16];
-	char  name[64];
-};
-
-struct nv_device_time_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__u64 time;
-};
-
-
-/*******************************************************************************
- * context dma
- ******************************************************************************/
-
-struct nv_dma_v0 {
-	__u8  version;
-#define NV_DMA_V0_TARGET_VM                                                0x00
-#define NV_DMA_V0_TARGET_VRAM                                              0x01
-#define NV_DMA_V0_TARGET_PCI                                               0x02
-#define NV_DMA_V0_TARGET_PCI_US                                            0x03
-#define NV_DMA_V0_TARGET_AGP                                               0x04
-	__u8  target;
-#define NV_DMA_V0_ACCESS_VM                                                0x00
-#define NV_DMA_V0_ACCESS_RD                                                0x01
-#define NV_DMA_V0_ACCESS_WR                                                0x02
-#define NV_DMA_V0_ACCESS_RDWR                 (NV_DMA_V0_ACCESS_RD | NV_DMA_V0_ACCESS_WR)
-	__u8  access;
-	__u8  pad03[5];
-	__u64 start;
-	__u64 limit;
-	/* ... chipset-specific class data */
-};
-
-struct nv50_dma_v0 {
-	__u8  version;
-#define NV50_DMA_V0_PRIV_VM                                                0x00
-#define NV50_DMA_V0_PRIV_US                                                0x01
-#define NV50_DMA_V0_PRIV__S                                                0x02
-	__u8  priv;
-#define NV50_DMA_V0_PART_VM                                                0x00
-#define NV50_DMA_V0_PART_256                                               0x01
-#define NV50_DMA_V0_PART_1KB                                               0x02
-	__u8  part;
-#define NV50_DMA_V0_COMP_NONE                                              0x00
-#define NV50_DMA_V0_COMP_1                                                 0x01
-#define NV50_DMA_V0_COMP_2                                                 0x02
-#define NV50_DMA_V0_COMP_VM                                                0x03
-	__u8  comp;
-#define NV50_DMA_V0_KIND_PITCH                                             0x00
-#define NV50_DMA_V0_KIND_VM                                                0x7f
-	__u8  kind;
-	__u8  pad05[3];
-};
-
-struct gf100_dma_v0 {
-	__u8  version;
-#define GF100_DMA_V0_PRIV_VM                                               0x00
-#define GF100_DMA_V0_PRIV_US                                               0x01
-#define GF100_DMA_V0_PRIV__S                                               0x02
-	__u8  priv;
-#define GF100_DMA_V0_KIND_PITCH                                            0x00
-#define GF100_DMA_V0_KIND_VM                                               0xff
-	__u8  kind;
-	__u8  pad03[5];
-};
-
-struct gf119_dma_v0 {
-	__u8  version;
-#define GF119_DMA_V0_PAGE_LP                                               0x00
-#define GF119_DMA_V0_PAGE_SP                                               0x01
-	__u8  page;
-#define GF119_DMA_V0_KIND_PITCH                                            0x00
-#define GF119_DMA_V0_KIND_VM                                               0xff
-	__u8  kind;
-	__u8  pad03[5];
-};
-
-
-/*******************************************************************************
- * perfmon
- ******************************************************************************/
-
-#define NVIF_PERFMON_V0_QUERY_DOMAIN                                       0x00
-#define NVIF_PERFMON_V0_QUERY_SIGNAL                                       0x01
-#define NVIF_PERFMON_V0_QUERY_SOURCE                                       0x02
-
-struct nvif_perfmon_query_domain_v0 {
-	__u8  version;
-	__u8  id;
-	__u8  counter_nr;
-	__u8  iter;
-	__u16 signal_nr;
-	__u8  pad05[2];
-	char  name[64];
-};
-
-struct nvif_perfmon_query_signal_v0 {
-	__u8  version;
-	__u8  domain;
-	__u16 iter;
-	__u8  signal;
-	__u8  source_nr;
-	__u8  pad05[2];
-	char  name[64];
-};
-
-struct nvif_perfmon_query_source_v0 {
-	__u8  version;
-	__u8  domain;
-	__u8  signal;
-	__u8  iter;
-	__u8  pad04[4];
-	__u32 source;
-	__u32 mask;
-	char  name[64];
-};
-
-
-/*******************************************************************************
- * perfdom
- ******************************************************************************/
-
-struct nvif_perfdom_v0 {
-	__u8  version;
-	__u8  domain;
-	__u8  mode;
-	__u8  pad03[1];
-	struct {
-		__u8  signal[4];
-		__u64 source[4][8];
-		__u16 logic_op;
-	} ctr[4];
-};
-
-#define NVIF_PERFDOM_V0_INIT                                               0x00
-#define NVIF_PERFDOM_V0_SAMPLE                                             0x01
-#define NVIF_PERFDOM_V0_READ                                               0x02
-
-struct nvif_perfdom_init {
-};
-
-struct nvif_perfdom_sample {
-};
-
-struct nvif_perfdom_read_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__u32 ctr[4];
-	__u32 clk;
-	__u8  pad04[4];
-};
-
-
-/*******************************************************************************
- * device control
- ******************************************************************************/
-
-#define NVIF_CONTROL_PSTATE_INFO                                           0x00
-#define NVIF_CONTROL_PSTATE_ATTR                                           0x01
-#define NVIF_CONTROL_PSTATE_USER                                           0x02
-
-struct nvif_control_pstate_info_v0 {
-	__u8  version;
-	__u8  count; /* out: number of power states */
-#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_DISABLE                         (-1)
-#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_PERFMON                         (-2)
-	__s8  ustate_ac; /* out: target pstate index */
-	__s8  ustate_dc; /* out: target pstate index */
-	__s8  pwrsrc; /* out: current power source */
-#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_UNKNOWN                         (-1)
-#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_PERFMON                         (-2)
-	__s8  pstate; /* out: current pstate index */
-	__u8  pad06[2];
-};
-
-struct nvif_control_pstate_attr_v0 {
-	__u8  version;
-#define NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT                          (-1)
-	__s8  state; /*  in: index of pstate to query
-		      * out: pstate identifier
-		      */
-	__u8  index; /*  in: index of attribute to query
-		      * out: index of next attribute, or 0 if no more
-		      */
-	__u8  pad03[5];
-	__u32 min;
-	__u32 max;
-	char  name[32];
-	char  unit[16];
-};
-
-struct nvif_control_pstate_user_v0 {
-	__u8  version;
-#define NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN                          (-1)
-#define NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON                          (-2)
-	__s8  ustate; /*  in: pstate identifier */
-	__s8  pwrsrc; /*  in: target power source */
-	__u8  pad03[5];
-};
-
-
-/*******************************************************************************
- * DMA FIFO channels
- ******************************************************************************/
-
-struct nv03_channel_dma_v0 {
-	__u8  version;
-	__u8  chid;
-	__u8  pad02[2];
-	__u32 offset;
-	__u64 pushbuf;
-};
-
-struct nv50_channel_dma_v0 {
-	__u8  version;
-	__u8  chid;
-	__u8  pad02[6];
-	__u64 vm;
-	__u64 pushbuf;
-	__u64 offset;
-};
-
-#define G82_CHANNEL_DMA_V0_NTFY_UEVENT                                     0x00
-
-/*******************************************************************************
- * GPFIFO channels
- ******************************************************************************/
-
-struct nv50_channel_gpfifo_v0 {
-	__u8  version;
-	__u8  chid;
-	__u8  pad02[2];
-	__u32 ilength;
-	__u64 ioffset;
-	__u64 pushbuf;
-	__u64 vm;
-};
-
-struct fermi_channel_gpfifo_v0 {
-	__u8  version;
-	__u8  chid;
-	__u8  pad02[2];
-	__u32 ilength;
-	__u64 ioffset;
-	__u64 vm;
-};
-
-struct kepler_channel_gpfifo_a_v0 {
-	__u8  version;
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR                               0x01
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSPDEC                           0x02
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSPPP                            0x04
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_MSVLD                            0x08
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE0                              0x10
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE1                              0x20
-#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_ENC                              0x40
-	__u8  engine;
-	__u16 chid;
-	__u32 ilength;
-	__u64 ioffset;
-	__u64 vm;
-};
-
-/*******************************************************************************
- * legacy display
- ******************************************************************************/
-
-#define NV04_DISP_NTFY_VBLANK                                              0x00
-#define NV04_DISP_NTFY_CONN                                                0x01
-
-struct nv04_disp_mthd_v0 {
-	__u8  version;
-#define NV04_DISP_SCANOUTPOS                                               0x00
-	__u8  method;
-	__u8  head;
-	__u8  pad03[5];
-};
-
-struct nv04_disp_scanoutpos_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__s64 time[2];
-	__u16 vblanks;
-	__u16 vblanke;
-	__u16 vtotal;
-	__u16 vline;
-	__u16 hblanks;
-	__u16 hblanke;
-	__u16 htotal;
-	__u16 hline;
-};
-
-/*******************************************************************************
- * display
- ******************************************************************************/
-
-#define NV50_DISP_MTHD                                                     0x00
-
-struct nv50_disp_mthd_v0 {
-	__u8  version;
-#define NV50_DISP_SCANOUTPOS                                               0x00
-	__u8  method;
-	__u8  head;
-	__u8  pad03[5];
-};
-
-struct nv50_disp_mthd_v1 {
-	__u8  version;
-#define NV50_DISP_MTHD_V1_DAC_PWR                                          0x10
-#define NV50_DISP_MTHD_V1_DAC_LOAD                                         0x11
-#define NV50_DISP_MTHD_V1_SOR_PWR                                          0x20
-#define NV50_DISP_MTHD_V1_SOR_HDA_ELD                                      0x21
-#define NV50_DISP_MTHD_V1_SOR_HDMI_PWR                                     0x22
-#define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT                                  0x23
-#define NV50_DISP_MTHD_V1_SOR_DP_PWR                                       0x24
-#define NV50_DISP_MTHD_V1_PIOR_PWR                                         0x30
-	__u8  method;
-	__u16 hasht;
-	__u16 hashm;
-	__u8  pad06[2];
-};
-
-struct nv50_disp_dac_pwr_v0 {
-	__u8  version;
-	__u8  state;
-	__u8  data;
-	__u8  vsync;
-	__u8  hsync;
-	__u8  pad05[3];
-};
-
-struct nv50_disp_dac_load_v0 {
-	__u8  version;
-	__u8  load;
-	__u8  pad02[2];
-	__u32 data;
-};
-
-struct nv50_disp_sor_pwr_v0 {
-	__u8  version;
-	__u8  state;
-	__u8  pad02[6];
-};
-
-struct nv50_disp_sor_hda_eld_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__u8  data[];
-};
-
-struct nv50_disp_sor_hdmi_pwr_v0 {
-	__u8  version;
-	__u8  state;
-	__u8  max_ac_packet;
-	__u8  rekey;
-	__u8  pad04[4];
-};
-
-struct nv50_disp_sor_lvds_script_v0 {
-	__u8  version;
-	__u8  pad01[1];
-	__u16 script;
-	__u8  pad04[4];
-};
-
-struct nv50_disp_sor_dp_pwr_v0 {
-	__u8  version;
-	__u8  state;
-	__u8  pad02[6];
-};
-
-struct nv50_disp_pior_pwr_v0 {
-	__u8  version;
-	__u8  state;
-	__u8  type;
-	__u8  pad03[5];
-};
-
-/* core */
-struct nv50_disp_core_channel_dma_v0 {
-	__u8  version;
-	__u8  pad01[7];
-	__u64 pushbuf;
-};
-
-#define NV50_DISP_CORE_CHANNEL_DMA_V0_NTFY_UEVENT                          0x00
-
-/* cursor immediate */
-struct nv50_disp_cursor_v0 {
-	__u8  version;
-	__u8  head;
-	__u8  pad02[6];
-};
-
-#define NV50_DISP_CURSOR_V0_NTFY_UEVENT                                    0x00
-
-/* base */
-struct nv50_disp_base_channel_dma_v0 {
-	__u8  version;
-	__u8  head;
-	__u8  pad02[6];
-	__u64 pushbuf;
-};
-
-#define NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT                          0x00
-
-/* overlay */
-struct nv50_disp_overlay_channel_dma_v0 {
-	__u8  version;
-	__u8  head;
-	__u8  pad02[6];
-	__u64 pushbuf;
-};
-
-#define NV50_DISP_OVERLAY_CHANNEL_DMA_V0_NTFY_UEVENT                       0x00
-
-/* overlay immediate */
-struct nv50_disp_overlay_v0 {
-	__u8  version;
-	__u8  head;
-	__u8  pad02[6];
-};
-
-#define NV50_DISP_OVERLAY_V0_NTFY_UEVENT                                   0x00
-
-/*******************************************************************************
- * software
- ******************************************************************************/
-
-#define NVSW_NTFY_UEVENT                                                   0x00
-
-#define NV04_NVSW_GET_REF                                                  0x00
-
-struct nv04_nvsw_get_ref_v0 {
-	__u8  version;
-	__u8  pad01[3];
-	__u32 ref;
-};
-
-/*******************************************************************************
- * fermi
- ******************************************************************************/
-
-#define FERMI_A_ZBC_COLOR                                                  0x00
-#define FERMI_A_ZBC_DEPTH                                                  0x01
-
-struct fermi_a_zbc_color_v0 {
-	__u8  version;
-#define FERMI_A_ZBC_COLOR_V0_FMT_ZERO                                      0x01
-#define FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE                                 0x02
-#define FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32                       0x04
-#define FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16                           0x08
-#define FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16                       0x0c
-#define FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16                       0x10
-#define FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16                       0x14
-#define FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16                       0x16
-#define FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8                                  0x18
-#define FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8                               0x1c
-#define FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10                               0x20
-#define FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10                           0x24
-#define FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8                                  0x28
-#define FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8                               0x2c
-#define FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8                              0x30
-#define FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8                              0x34
-#define FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8                              0x38
-#define FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10                               0x3c
-#define FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11                              0x40
-	__u8  format;
-	__u8  index;
-	__u8  pad03[5];
-	__u32 ds[4];
-	__u32 l2[4];
-};
-
-struct fermi_a_zbc_depth_v0 {
-	__u8  version;
-#define FERMI_A_ZBC_DEPTH_V0_FMT_FP32                                      0x01
-	__u8  format;
-	__u8  index;
-	__u8  pad03[5];
-	__u32 ds;
-	__u32 l2;
-};
-
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h
index 700a9b206726..e0ed2f4b2f43 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/device.h
@@ -2,7 +2,7 @@
 #define __NVIF_DEVICE_H__
 
 #include <nvif/object.h>
-#include <nvif/class.h>
+#include <nvif/cl0080.h>
 
 struct nvif_device {
 	struct nvif_object object;
@@ -63,6 +63,7 @@ u64  nvif_device_time(struct nvif_device *);
 #define nvxx_clk(a) nvxx_device(a)->clk
 #define nvxx_i2c(a) nvxx_device(a)->i2c
 #define nvxx_therm(a) nvxx_device(a)->therm
+#define nvxx_volt(a) nvxx_device(a)->volt
 
 #include <core/device.h>
 #include <engine/fifo.h>
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0000.h b/drivers/gpu/drm/nouveau/include/nvif/if0000.h
new file mode 100644
index 000000000000..85c44e8a1201
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0000.h
@@ -0,0 +1,12 @@
+#ifndef __NVIF_IF0000_H__
+#define __NVIF_IF0000_H__
+
+#define NV_CLIENT_DEVLIST                                                  0x00
+
+struct nv_client_devlist_v0 {
+	__u8  version;
+	__u8  count;
+	__u8  pad02[6];
+	__u64 device[];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0001.h b/drivers/gpu/drm/nouveau/include/nvif/if0001.h
new file mode 100644
index 000000000000..bd5b64125eed
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0001.h
@@ -0,0 +1,46 @@
+#ifndef __NVIF_IF0001_H__
+#define __NVIF_IF0001_H__
+
+#define NVIF_CONTROL_PSTATE_INFO                                           0x00
+#define NVIF_CONTROL_PSTATE_ATTR                                           0x01
+#define NVIF_CONTROL_PSTATE_USER                                           0x02
+
+struct nvif_control_pstate_info_v0 {
+	__u8  version;
+	__u8  count; /* out: number of power states */
+#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_DISABLE                         (-1)
+#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_PERFMON                         (-2)
+	__s8  ustate_ac; /* out: target pstate index */
+	__s8  ustate_dc; /* out: target pstate index */
+	__s8  pwrsrc; /* out: current power source */
+#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_UNKNOWN                         (-1)
+#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_PERFMON                         (-2)
+	__s8  pstate; /* out: current pstate index */
+	__u8  pad06[2];
+};
+
+struct nvif_control_pstate_attr_v0 {
+	__u8  version;
+#define NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT                          (-1)
+	__s8  state; /*  in: index of pstate to query
+		      * out: pstate identifier
+		      */
+	__u8  index; /*  in: index of attribute to query
+		      * out: index of next attribute, or 0 if no more
+		      */
+	__u8  pad03[5];
+	__u32 min;
+	__u32 max;
+	char  name[32];
+	char  unit[16];
+};
+
+struct nvif_control_pstate_user_v0 {
+	__u8  version;
+#define NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN                          (-1)
+#define NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON                          (-2)
+	__s8  ustate; /*  in: pstate identifier */
+	__s8  pwrsrc; /*  in: target power source */
+	__u8  pad03[5];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0002.h b/drivers/gpu/drm/nouveau/include/nvif/if0002.h
new file mode 100644
index 000000000000..c04c91d0b818
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0002.h
@@ -0,0 +1,38 @@
+#ifndef __NVIF_IF0002_H__
+#define __NVIF_IF0002_H__
+
+#define NVIF_PERFMON_V0_QUERY_DOMAIN                                       0x00
+#define NVIF_PERFMON_V0_QUERY_SIGNAL                                       0x01
+#define NVIF_PERFMON_V0_QUERY_SOURCE                                       0x02
+
+struct nvif_perfmon_query_domain_v0 {
+	__u8  version;
+	__u8  id;
+	__u8  counter_nr;
+	__u8  iter;
+	__u16 signal_nr;
+	__u8  pad05[2];
+	char  name[64];
+};
+
+struct nvif_perfmon_query_signal_v0 {
+	__u8  version;
+	__u8  domain;
+	__u16 iter;
+	__u8  signal;
+	__u8  source_nr;
+	__u8  pad05[2];
+	char  name[64];
+};
+
+struct nvif_perfmon_query_source_v0 {
+	__u8  version;
+	__u8  domain;
+	__u8  signal;
+	__u8  iter;
+	__u8  pad04[4];
+	__u32 source;
+	__u32 mask;
+	char  name[64];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0003.h b/drivers/gpu/drm/nouveau/include/nvif/if0003.h
new file mode 100644
index 000000000000..0cd03efb80a1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0003.h
@@ -0,0 +1,33 @@
+#ifndef __NVIF_IF0003_H__
+#define __NVIF_IF0003_H__
+
+struct nvif_perfdom_v0 {
+	__u8  version;
+	__u8  domain;
+	__u8  mode;
+	__u8  pad03[1];
+	struct {
+		__u8  signal[4];
+		__u64 source[4][8];
+		__u16 logic_op;
+	} ctr[4];
+};
+
+#define NVIF_PERFDOM_V0_INIT                                               0x00
+#define NVIF_PERFDOM_V0_SAMPLE                                             0x01
+#define NVIF_PERFDOM_V0_READ                                               0x02
+
+struct nvif_perfdom_init {
+};
+
+struct nvif_perfdom_sample {
+};
+
+struct nvif_perfdom_read_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u32 ctr[4];
+	__u32 clk;
+	__u8  pad04[4];
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0004.h b/drivers/gpu/drm/nouveau/include/nvif/if0004.h
new file mode 100644
index 000000000000..bd5cd428cfd7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0004.h
@@ -0,0 +1,13 @@
+#ifndef __NVIF_IF0004_H__
+#define __NVIF_IF0004_H__
+
+#define NV04_NVSW_NTFY_UEVENT                                              0x00
+
+#define NV04_NVSW_GET_REF                                                  0x00
+
+struct nv04_nvsw_get_ref_v0 {
+	__u8  version;
+	__u8  pad01[3];
+	__u32 ref;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0005.h b/drivers/gpu/drm/nouveau/include/nvif/if0005.h
new file mode 100644
index 000000000000..abfd373bb68b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/if0005.h
@@ -0,0 +1,4 @@
+#ifndef __NVIF_IF0005_H__
+#define __NVIF_IF0005_H__
+#define NV10_NVSW_NTFY_UEVENT                                              0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h
index b0ac0215ebf9..c5f5eb83a594 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h
@@ -55,14 +55,6 @@ struct nvif_ioctl_new_v0 {
 	__u64 token;
 	__u64 object;
 	__u32 handle;
-/* these class numbers are made up by us, and not nvidia-assigned */
-#define NVIF_IOCTL_NEW_V0_CONTROL                                            -1
-#define NVIF_IOCTL_NEW_V0_PERFMON                                            -2
-#define NVIF_IOCTL_NEW_V0_PERFDOM                                            -3
-#define NVIF_IOCTL_NEW_V0_SW_NV04                                            -4
-#define NVIF_IOCTL_NEW_V0_SW_NV10                                            -5
-#define NVIF_IOCTL_NEW_V0_SW_NV50                                            -6
-#define NVIF_IOCTL_NEW_V0_SW_GF100                                           -7
 	__s32 oclass;
 	__u8  data[];		/* class data (class.h) */
 };
diff --git a/drivers/gpu/drm/nouveau/include/nvif/unpack.h b/drivers/gpu/drm/nouveau/include/nvif/unpack.h
index 5933188b4a77..751bcf4930a7 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/unpack.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/unpack.h
@@ -1,24 +1,28 @@
 #ifndef __NVIF_UNPACK_H__
 #define __NVIF_UNPACK_H__
 
-#define nvif_unvers(d) ({                                                      \
-	ret = (size == sizeof(d)) ? 0 : -ENOSYS;                               \
-	(ret == 0);                                                            \
+#define nvif_unvers(r,d,s,m) ({                                                \
+	void **_data = (d); __u32 *_size = (s); int _ret = (r);                \
+	if (_ret == -ENOSYS && *_size == sizeof(m)) {                          \
+		*_data = NULL;                                                 \
+		*_size = _ret = 0;                                             \
+	}                                                                      \
+	_ret;                                                                  \
 })
 
-#define nvif_unpack(d,vl,vh,m) ({                                              \
-	if ((vl) == 0 || ret == -ENOSYS) {                                     \
-		int _size = sizeof(d);                                         \
-		if (_size <= size && (d).version >= (vl) &&                    \
-				     (d).version <= (vh)) {                    \
-			data = (u8 *)data + _size;                             \
-			size = size - _size;                                   \
-			ret = ((m) || !size) ? 0 : -E2BIG;                     \
-		} else {                                                       \
-			ret = -ENOSYS;                                         \
+#define nvif_unpack(r,d,s,m,vl,vh,x) ({                                        \
+	void **_data = (d); __u32 *_size = (s);                                \
+	int _ret = (r), _vl = (vl), _vh = (vh);                                \
+	if (_ret == -ENOSYS && *_size >= sizeof(m) &&                          \
+	    (m).version >= _vl && (m).version <= _vh) {                        \
+		*_data = (__u8 *)*_data + sizeof(m);                           \
+		*_size = *_size - sizeof(m);                                   \
+		if (_ret = 0, !(x)) {                                          \
+			_ret = *_size ? -E2BIG : 0;                            \
+			*_data = NULL;                                         \
+			*_size = 0;                                            \
 		}                                                              \
 	}                                                                      \
-	(ret == 0);                                                            \
+	_ret;                                                                  \
 })
-
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
index 7cc2becabc69..d3bd250103d5 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
@@ -13,6 +13,8 @@ struct nvbios_perfE {
 	u32 vdec;
 	u32 disp;
 	u32 script;
+	u8  pcie_speed;
+	u8  pcie_width;
 };
 
 u16 nvbios_perf_entry(struct nvkm_bios *, int idx,
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
index 8708f0a4e188..6b33bc058924 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
@@ -2,6 +2,7 @@
 #define __NVKM_CLK_H__
 #include <core/subdev.h>
 #include <core/notify.h>
+#include <subdev/pci.h>
 struct nvbios_pll;
 struct nvkm_pll_vals;
 
@@ -38,7 +39,7 @@ enum nv_clk_src {
 	nv_clk_src_hubk06,
 	nv_clk_src_hubk07,
 	nv_clk_src_copy,
-	nv_clk_src_daemon,
+	nv_clk_src_pmu,
 	nv_clk_src_disp,
 	nv_clk_src_vdec,
 
@@ -59,6 +60,8 @@ struct nvkm_pstate {
 	struct nvkm_cstate base;
 	u8 pstate;
 	u8 fanspeed;
+	enum nvkm_pcie_speed pcie_speed;
+	u8 pcie_width;
 };
 
 struct nvkm_domain {
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ibus.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ibus.h
index c4dcd2680fe1..ea23e24a246c 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ibus.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ibus.h
@@ -6,4 +6,5 @@ int gf100_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
 int gf117_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
 int gk104_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
 int gk20a_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
+int gm204_ibus_new(struct nvkm_device *, int, struct nvkm_subdev **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
index 3d4dbbf9aab3..0ffa2ec106d6 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
@@ -37,4 +37,5 @@ int gf100_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gk104_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gk20a_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gm107_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
+int gm204_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
index fee0a97c44c5..ddb913889d7e 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
@@ -2,6 +2,12 @@
 #define __NVKM_PCI_H__
 #include <core/subdev.h>
 
+enum nvkm_pcie_speed {
+	NVKM_PCIE_SPEED_2_5,
+	NVKM_PCIE_SPEED_5_0,
+	NVKM_PCIE_SPEED_8_0,
+};
+
 struct nvkm_pci {
 	const struct nvkm_pci_func *func;
 	struct nvkm_subdev subdev;
@@ -18,6 +24,11 @@ struct nvkm_pci {
 		bool acquired;
 	} agp;
 
+	struct {
+		enum nvkm_pcie_speed speed;
+		u8 width;
+	} pcie;
+
 	bool msi;
 };
 
@@ -34,4 +45,9 @@ int nv4c_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int g84_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int g94_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int gf100_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int gf106_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int gk104_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+
+/* pcie functions */
+int nvkm_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8 width);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 7f50cf5f929e..50f52ffe5b0c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -25,6 +25,8 @@
 #include <nvif/driver.h>
 #include <nvif/ioctl.h>
 #include <nvif/class.h>
+#include <nvif/cl0002.h>
+#include <nvif/cla06f.h>
 #include <nvif/unpack.h>
 
 #include "nouveau_drm.h"
@@ -87,18 +89,18 @@ nouveau_abi16_swclass(struct nouveau_drm *drm)
 {
 	switch (drm->device.info.family) {
 	case NV_DEVICE_INFO_V0_TNT:
-		return NVIF_IOCTL_NEW_V0_SW_NV04;
+		return NVIF_CLASS_SW_NV04;
 	case NV_DEVICE_INFO_V0_CELSIUS:
 	case NV_DEVICE_INFO_V0_KELVIN:
 	case NV_DEVICE_INFO_V0_RANKINE:
 	case NV_DEVICE_INFO_V0_CURIE:
-		return NVIF_IOCTL_NEW_V0_SW_NV10;
+		return NVIF_CLASS_SW_NV10;
 	case NV_DEVICE_INFO_V0_TESLA:
-		return NVIF_IOCTL_NEW_V0_SW_NV50;
+		return NVIF_CLASS_SW_NV50;
 	case NV_DEVICE_INFO_V0_FERMI:
 	case NV_DEVICE_INFO_V0_KEPLER:
 	case NV_DEVICE_INFO_V0_MAXWELL:
-		return NVIF_IOCTL_NEW_V0_SW_GF100;
+		return NVIF_CLASS_SW_GF100;
 	}
 
 	return 0x0000;
@@ -355,9 +357,9 @@ nouveau_abi16_usif(struct drm_file *file_priv, void *data, u32 size)
 	} *args = data;
 	struct nouveau_abi16_chan *chan;
 	struct nouveau_abi16 *abi16;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		switch (args->v0.type) {
 		case NVIF_IOCTL_V0_NEW:
 		case NVIF_IOCTL_V0_MTHD:
@@ -433,10 +435,10 @@ nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
 		/* nvsw: compatibility with older 0x*6e class identifier */
 		for (i = 0; !oclass && i < ret; i++) {
 			switch (sclass[i].oclass) {
-			case NVIF_IOCTL_NEW_V0_SW_NV04:
-			case NVIF_IOCTL_NEW_V0_SW_NV10:
-			case NVIF_IOCTL_NEW_V0_SW_NV50:
-			case NVIF_IOCTL_NEW_V0_SW_GF100:
+			case NVIF_CLASS_SW_NV04:
+			case NVIF_CLASS_SW_NV10:
+			case NVIF_CLASS_SW_NV50:
+			case NVIF_CLASS_SW_GF100:
 				oclass = sclass[i].oclass;
 				break;
 			default:
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index 1860f389f21f..3f804a8c590c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -24,6 +24,11 @@
 
 #include <nvif/os.h>
 #include <nvif/class.h>
+#include <nvif/cl0002.h>
+#include <nvif/cl006b.h>
+#include <nvif/cl506f.h>
+#include <nvif/cl906f.h>
+#include <nvif/cla06f.h>
 #include <nvif/ioctl.h>
 
 /*XXX*/
@@ -378,7 +383,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
 	/* allocate software object class (used for fences on <= nv05) */
 	if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) {
 		ret = nvif_object_init(&chan->user, 0x006e,
-				       NVIF_IOCTL_NEW_V0_SW_NV04,
+				       NVIF_CLASS_SW_NV04,
 				       NULL, 0, &chan->nvsw);
 		if (ret)
 			return ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 2e7cbe933533..fcebfae5d426 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -42,6 +42,8 @@
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 
+#include <nvif/class.h>
+#include <nvif/cl0046.h>
 #include <nvif/event.h>
 
 MODULE_PARM_DESC(tv_disable, "Disable TV-out detection");
@@ -56,6 +58,10 @@ MODULE_PARM_DESC(duallink, "Allow dual-link TMDS (default: enabled)");
 int nouveau_duallink = 1;
 module_param_named(duallink, nouveau_duallink, int, 0400);
 
+MODULE_PARM_DESC(hdmimhz, "Force a maximum HDMI pixel clock (in MHz)");
+int nouveau_hdmimhz = 0;
+module_param_named(hdmimhz, nouveau_hdmimhz, int, 0400);
+
 struct nouveau_encoder *
 find_encoder(struct drm_connector *connector, int type)
 {
@@ -809,12 +815,23 @@ nouveau_connector_get_modes(struct drm_connector *connector)
 }
 
 static unsigned
-get_tmds_link_bandwidth(struct drm_connector *connector)
+get_tmds_link_bandwidth(struct drm_connector *connector, bool hdmi)
 {
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct dcb_output *dcb = nv_connector->detected_encoder->dcb;
 
+	if (hdmi) {
+		if (nouveau_hdmimhz > 0)
+			return nouveau_hdmimhz * 1000;
+		/* Note: these limits are conservative, some Fermi's
+		 * can do 297 MHz. Unclear how this can be determined.
+		 */
+		if (drm->device.info.family >= NV_DEVICE_INFO_V0_KEPLER)
+			return 297000;
+		if (drm->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
+			return 225000;
+	}
 	if (dcb->location != DCB_LOC_ON_CHIP ||
 	    drm->device.info.chipset >= 0x46)
 		return 165000;
@@ -835,6 +852,7 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
 	struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
 	unsigned min_clock = 25000, max_clock = min_clock;
 	unsigned clock = mode->clock;
+	bool hdmi;
 
 	switch (nv_encoder->dcb->type) {
 	case DCB_OUTPUT_LVDS:
@@ -847,8 +865,10 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
 		max_clock = 400000;
 		break;
 	case DCB_OUTPUT_TMDS:
-		max_clock = get_tmds_link_bandwidth(connector);
-		if (nouveau_duallink && nv_encoder->dcb->duallink_possible)
+		hdmi = drm_detect_hdmi_monitor(nv_connector->edid);
+		max_clock = get_tmds_link_bandwidth(connector, hdmi);
+		if (!hdmi && nouveau_duallink &&
+		    nv_encoder->dcb->duallink_possible)
 			max_clock *= 2;
 		break;
 	case DCB_OUTPUT_ANALOG:
@@ -898,8 +918,6 @@ nouveau_connector_helper_funcs = {
 static const struct drm_connector_funcs
 nouveau_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = NULL,
-	.restore = NULL,
 	.detect = nouveau_connector_detect,
 	.destroy = nouveau_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -910,8 +928,6 @@ nouveau_connector_funcs = {
 static const struct drm_connector_funcs
 nouveau_connector_funcs_lvds = {
 	.dpms = drm_helper_connector_dpms,
-	.save = NULL,
-	.restore = NULL,
 	.detect = nouveau_connector_detect_lvds,
 	.destroy = nouveau_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -944,8 +960,6 @@ nouveau_connector_dp_dpms(struct drm_connector *connector, int mode)
 static const struct drm_connector_funcs
 nouveau_connector_funcs_dp = {
 	.dpms = nouveau_connector_dp_dpms,
-	.save = NULL,
-	.restore = NULL,
 	.detect = nouveau_connector_detect,
 	.destroy = nouveau_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -969,10 +983,13 @@ nouveau_connector_hotplug(struct nvif_notify *notify)
 
 		NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", name);
 
+		mutex_lock(&drm->dev->mode_config.mutex);
 		if (plugged)
 			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
 		else
 			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+		mutex_unlock(&drm->dev->mode_config.mutex);
+
 		drm_helper_hpd_irq_event(connector->dev);
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h
index f19cb1c5fc5a..863f10b8d818 100644
--- a/drivers/gpu/drm/nouveau/nouveau_crtc.h
+++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h
@@ -73,6 +73,9 @@ struct nouveau_crtc {
 	int (*set_dither)(struct nouveau_crtc *crtc, bool update);
 	int (*set_scale)(struct nouveau_crtc *crtc, bool update);
 	int (*set_color_vibrance)(struct nouveau_crtc *crtc, bool update);
+
+	void (*save)(struct drm_crtc *crtc);
+	void (*restore)(struct drm_crtc *crtc);
 };
 
 static inline struct nouveau_crtc *nouveau_crtc(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
index 5392e07edfc6..3d0dc199b253 100644
--- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c
+++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
@@ -28,6 +28,9 @@
  *  Ben Skeggs <bskeggs@redhat.com>
  */
 
+#include <linux/debugfs.h>
+#include <nvif/class.h>
+#include <nvif/if0001.h>
 #include "nouveau_debugfs.h"
 #include "nouveau_drm.h"
 
@@ -43,22 +46,233 @@ nouveau_debugfs_vbios_image(struct seq_file *m, void *data)
 	return 0;
 }
 
+static int
+nouveau_debugfs_pstate_get(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct nouveau_debugfs *debugfs = nouveau_debugfs(node->minor->dev);
+	struct nvif_object *ctrl = &debugfs->ctrl;
+	struct nvif_control_pstate_info_v0 info = {};
+	int ret, i;
+
+	if (!debugfs)
+		return -ENODEV;
+
+	ret = nvif_mthd(ctrl, NVIF_CONTROL_PSTATE_INFO, &info, sizeof(info));
+	if (ret)
+		return ret;
+
+	for (i = 0; i < info.count + 1; i++) {
+		const s32 state = i < info.count ? i :
+			NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT;
+		struct nvif_control_pstate_attr_v0 attr = {
+			.state = state,
+			.index = 0,
+		};
+
+		ret = nvif_mthd(ctrl, NVIF_CONTROL_PSTATE_ATTR,
+				&attr, sizeof(attr));
+		if (ret)
+			return ret;
+
+		if (i < info.count)
+			seq_printf(m, "%02x:", attr.state);
+		else
+			seq_printf(m, "%s:", info.pwrsrc == 0 ? "DC" :
+					     info.pwrsrc == 1 ? "AC" : "--");
+
+		attr.index = 0;
+		do {
+			attr.state = state;
+			ret = nvif_mthd(ctrl, NVIF_CONTROL_PSTATE_ATTR,
+					&attr, sizeof(attr));
+			if (ret)
+				return ret;
+
+			seq_printf(m, " %s %d", attr.name, attr.min);
+			if (attr.min != attr.max)
+				seq_printf(m, "-%d", attr.max);
+			seq_printf(m, " %s", attr.unit);
+		} while (attr.index);
+
+		if (state >= 0) {
+			if (info.ustate_ac == state)
+				seq_printf(m, " AC");
+			if (info.ustate_dc == state)
+				seq_printf(m, " DC");
+			if (info.pstate == state)
+				seq_printf(m, " *");
+		} else {
+			if (info.ustate_ac < -1)
+				seq_printf(m, " AC");
+			if (info.ustate_dc < -1)
+				seq_printf(m, " DC");
+		}
+
+		seq_printf(m, "\n");
+	}
+
+	return 0;
+}
+
+static ssize_t
+nouveau_debugfs_pstate_set(struct file *file, const char __user *ubuf,
+			   size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct nouveau_debugfs *debugfs = nouveau_debugfs(node->minor->dev);
+	struct nvif_object *ctrl = &debugfs->ctrl;
+	struct nvif_control_pstate_user_v0 args = { .pwrsrc = -EINVAL };
+	char buf[32] = {}, *tmp, *cur = buf;
+	long value, ret;
+
+	if (!debugfs)
+		return -ENODEV;
+
+	if (len >= sizeof(buf))
+		return -EINVAL;
+
+	if (copy_from_user(buf, ubuf, len))
+		return -EFAULT;
+
+	if ((tmp = strchr(buf, '\n')))
+		*tmp = '\0';
+
+	if (!strncasecmp(cur, "dc:", 3)) {
+		args.pwrsrc = 0;
+		cur += 3;
+	} else
+	if (!strncasecmp(cur, "ac:", 3)) {
+		args.pwrsrc = 1;
+		cur += 3;
+	}
+
+	if (!strcasecmp(cur, "none"))
+		args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN;
+	else
+	if (!strcasecmp(cur, "auto"))
+		args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON;
+	else {
+		ret = kstrtol(cur, 16, &value);
+		if (ret)
+			return ret;
+		args.ustate = value;
+	}
+
+	ret = nvif_mthd(ctrl, NVIF_CONTROL_PSTATE_USER, &args, sizeof(args));
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static int
+nouveau_debugfs_pstate_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, nouveau_debugfs_pstate_get, inode->i_private);
+}
+
+static const struct file_operations nouveau_pstate_fops = {
+	.owner = THIS_MODULE,
+	.open = nouveau_debugfs_pstate_open,
+	.read = seq_read,
+	.write = nouveau_debugfs_pstate_set,
+};
+
 static struct drm_info_list nouveau_debugfs_list[] = {
 	{ "vbios.rom", nouveau_debugfs_vbios_image, 0, NULL },
 };
 #define NOUVEAU_DEBUGFS_ENTRIES ARRAY_SIZE(nouveau_debugfs_list)
 
-int
-nouveau_debugfs_init(struct drm_minor *minor)
+static const struct nouveau_debugfs_files {
+	const char *name;
+	const struct file_operations *fops;
+} nouveau_debugfs_files[] = {
+	{"pstate", &nouveau_pstate_fops},
+};
+
+static int
+nouveau_debugfs_create_file(struct drm_minor *minor,
+		const struct nouveau_debugfs_files *ndf)
 {
-	drm_debugfs_create_files(nouveau_debugfs_list, NOUVEAU_DEBUGFS_ENTRIES,
-				 minor->debugfs_root, minor);
+	struct drm_info_node *node;
+
+	node = kmalloc(sizeof(*node), GFP_KERNEL);
+	if (node == NULL)
+		return -ENOMEM;
+
+	node->minor = minor;
+	node->info_ent = (const void *)ndf->fops;
+	node->dent = debugfs_create_file(ndf->name, S_IRUGO | S_IWUSR,
+					 minor->debugfs_root, node, ndf->fops);
+	if (!node->dent) {
+		kfree(node);
+		return -ENOMEM;
+	}
+
+	mutex_lock(&minor->debugfs_lock);
+	list_add(&node->list, &minor->debugfs_list);
+	mutex_unlock(&minor->debugfs_lock);
 	return 0;
 }
 
+int
+nouveau_drm_debugfs_init(struct drm_minor *minor)
+{
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(nouveau_debugfs_files); i++) {
+		ret = nouveau_debugfs_create_file(minor,
+						  &nouveau_debugfs_files[i]);
+
+		if (ret)
+			return ret;
+	}
+
+	return drm_debugfs_create_files(nouveau_debugfs_list,
+					NOUVEAU_DEBUGFS_ENTRIES,
+					minor->debugfs_root, minor);
+}
+
 void
-nouveau_debugfs_takedown(struct drm_minor *minor)
+nouveau_drm_debugfs_cleanup(struct drm_minor *minor)
 {
+	int i;
+
 	drm_debugfs_remove_files(nouveau_debugfs_list, NOUVEAU_DEBUGFS_ENTRIES,
 				 minor);
+
+	for (i = 0; i < ARRAY_SIZE(nouveau_debugfs_files); i++) {
+		drm_debugfs_remove_files((struct drm_info_list *)
+					 nouveau_debugfs_files[i].fops,
+					 1, minor);
+	}
+}
+
+int
+nouveau_debugfs_init(struct nouveau_drm *drm)
+{
+	int ret;
+
+	drm->debugfs = kzalloc(sizeof(*drm->debugfs), GFP_KERNEL);
+	if (!drm->debugfs)
+		return -ENOMEM;
+
+	ret = nvif_object_init(&drm->device.object, 0, NVIF_CLASS_CONTROL,
+			       NULL, 0, &drm->debugfs->ctrl);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void
+nouveau_debugfs_fini(struct nouveau_drm *drm)
+{
+	if (drm->debugfs && drm->debugfs->ctrl.priv)
+		nvif_object_fini(&drm->debugfs->ctrl);
+
+	kfree(drm->debugfs);
+	drm->debugfs = NULL;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.h b/drivers/gpu/drm/nouveau/nouveau_debugfs.h
index a62af6fb5f99..b8c03ff5bf05 100644
--- a/drivers/gpu/drm/nouveau/nouveau_debugfs.h
+++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.h
@@ -4,16 +4,43 @@
 #include <drm/drmP.h>
 
 #if defined(CONFIG_DEBUG_FS)
-extern int  nouveau_debugfs_init(struct drm_minor *);
-extern void nouveau_debugfs_takedown(struct drm_minor *);
+
+#include "nouveau_drm.h"
+
+struct nouveau_debugfs {
+	struct nvif_object ctrl;
+};
+
+static inline struct nouveau_debugfs *
+nouveau_debugfs(struct drm_device *dev)
+{
+	return nouveau_drm(dev)->debugfs;
+}
+
+extern int  nouveau_drm_debugfs_init(struct drm_minor *);
+extern void nouveau_drm_debugfs_cleanup(struct drm_minor *);
+extern int  nouveau_debugfs_init(struct nouveau_drm *);
+extern void nouveau_debugfs_fini(struct nouveau_drm *);
 #else
 static inline int
-nouveau_debugfs_init(struct drm_minor *minor)
+nouveau_drm_debugfs_init(struct drm_minor *minor)
 {
        return 0;
 }
 
-static inline void nouveau_debugfs_takedown(struct drm_minor *minor)
+static inline void
+nouveau_drm_debugfs_cleanup(struct drm_minor *minor)
+{
+}
+
+static inline int
+nouveau_debugfs_init(struct nouveau_drm *drm)
+{
+	return 0;
+}
+
+static inline void
+nouveau_debugfs_fini(struct nouveau_drm *drm)
 {
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 64c8d932d5f1..24be27d3cd18 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -39,6 +39,7 @@
 
 #include "nouveau_fence.h"
 
+#include <nvif/cl0046.h>
 #include <nvif/event.h>
 
 static int
@@ -246,7 +247,7 @@ static const struct drm_framebuffer_funcs nouveau_framebuffer_funcs = {
 int
 nouveau_framebuffer_init(struct drm_device *dev,
 			 struct nouveau_framebuffer *nv_fb,
-			 struct drm_mode_fb_cmd2 *mode_cmd,
+			 const struct drm_mode_fb_cmd2 *mode_cmd,
 			 struct nouveau_bo *nvbo)
 {
 	struct nouveau_display *disp = nouveau_display(dev);
@@ -272,7 +273,7 @@ nouveau_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 nouveau_user_framebuffer_create(struct drm_device *dev,
 				struct drm_file *file_priv,
-				struct drm_mode_fb_cmd2 *mode_cmd)
+				const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct nouveau_framebuffer *nouveau_fb;
 	struct drm_gem_object *gem;
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 856abe0f070d..5a57d8b472c4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -23,7 +23,7 @@ nouveau_framebuffer(struct drm_framebuffer *fb)
 }
 
 int nouveau_framebuffer_init(struct drm_device *, struct nouveau_framebuffer *,
-			     struct drm_mode_fb_cmd2 *, struct nouveau_bo *);
+			     const struct drm_mode_fb_cmd2 *, struct nouveau_bo *);
 
 struct nouveau_page_flip_state {
 	struct list_head head;
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 1d3ee5179ab8..2f2f252e3fb6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -37,12 +37,16 @@
 #include <core/pci.h>
 #include <core/tegra.h>
 
+#include <nvif/class.h>
+#include <nvif/cl0002.h>
+#include <nvif/cla06f.h>
+#include <nvif/if0004.h>
+
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_ttm.h"
 #include "nouveau_gem.h"
 #include "nouveau_vga.h"
-#include "nouveau_sysfs.h"
 #include "nouveau_hwmon.h"
 #include "nouveau_acpi.h"
 #include "nouveau_bios.h"
@@ -256,8 +260,8 @@ nouveau_accel_init(struct nouveau_drm *drm)
 		}
 
 		ret = nvif_notify_init(&drm->nvsw, nouveau_flip_complete,
-				       false, NVSW_NTFY_UEVENT, NULL, 0, 0,
-				       &drm->flip);
+				       false, NV04_NVSW_NTFY_UEVENT,
+				       NULL, 0, 0, &drm->flip);
 		if (ret == 0)
 			ret = nvif_notify_get(&drm->flip);
 		if (ret) {
@@ -448,7 +452,7 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
 			goto fail_dispinit;
 	}
 
-	nouveau_sysfs_init(dev);
+	nouveau_debugfs_init(drm);
 	nouveau_hwmon_init(dev);
 	nouveau_accel_init(drm);
 	nouveau_fbcon_init(dev);
@@ -486,7 +490,7 @@ nouveau_drm_unload(struct drm_device *dev)
 	nouveau_fbcon_fini(dev);
 	nouveau_accel_fini(drm);
 	nouveau_hwmon_fini(dev);
-	nouveau_sysfs_fini(dev);
+	nouveau_debugfs_fini(drm);
 
 	if (dev->mode_config.num_crtc)
 		nouveau_display_fini(dev);
@@ -928,8 +932,8 @@ driver_stub = {
 	.lastclose = nouveau_vga_lastclose,
 
 #if defined(CONFIG_DEBUG_FS)
-	.debugfs_init = nouveau_debugfs_init,
-	.debugfs_cleanup = nouveau_debugfs_takedown,
+	.debugfs_init = nouveau_drm_debugfs_init,
+	.debugfs_cleanup = nouveau_drm_debugfs_cleanup,
 #endif
 
 	.get_vblank_counter = drm_vblank_no_hw_counter,
@@ -1003,7 +1007,6 @@ static void nouveau_display_options(void)
 	DRM_DEBUG_DRIVER("... modeset      : %d\n", nouveau_modeset);
 	DRM_DEBUG_DRIVER("... runpm        : %d\n", nouveau_runtime_pm);
 	DRM_DEBUG_DRIVER("... vram_pushbuf : %d\n", nouveau_vram_pushbuf);
-	DRM_DEBUG_DRIVER("... pstate       : %d\n", nouveau_pstate);
 }
 
 static const struct dev_pm_ops nouveau_pm_ops = {
@@ -1046,10 +1049,6 @@ nouveau_platform_device_create(const struct nvkm_device_tegra_func *func,
 		goto err_free;
 	}
 
-	err = drm_dev_set_unique(drm, "%s", dev_name(&pdev->dev));
-	if (err < 0)
-		goto err_free;
-
 	drm->platformdev = pdev;
 	platform_set_drvdata(pdev, drm);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.h b/drivers/gpu/drm/nouveau/nouveau_drm.h
index a02813e994ec..5c363ed1c842 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.h
@@ -164,7 +164,7 @@ struct nouveau_drm {
 
 	/* power management */
 	struct nouveau_hwmon *hwmon;
-	struct nouveau_sysfs *sysfs;
+	struct nouveau_debugfs *debugfs;
 
 	/* display power reference */
 	bool have_disp_power_ref;
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index b37da95105b0..ee6a6d3fc80f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -63,6 +63,9 @@ struct nouveau_encoder {
 			u32 datarate;
 		} dp;
 	};
+
+	void (*enc_save)(struct drm_encoder *encoder);
+	void (*enc_restore)(struct drm_encoder *encoder);
 };
 
 struct nouveau_encoder *
@@ -80,7 +83,7 @@ static inline struct drm_encoder *to_drm_encoder(struct nouveau_encoder *enc)
 	return &enc->base.base;
 }
 
-static inline struct drm_encoder_slave_funcs *
+static inline const struct drm_encoder_slave_funcs *
 get_slave_funcs(struct drm_encoder *enc)
 {
 	return to_encoder_slave(enc)->slave_funcs;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index 1e2e9e27a03b..ca77ad001978 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -34,7 +34,6 @@
 struct nouveau_fbdev {
 	struct drm_fb_helper helper;
 	struct nouveau_framebuffer nouveau_fb;
-	struct list_head fbdev_list;
 	struct drm_device *dev;
 	unsigned int saved_flags;
 	struct nvif_object surf2d;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 574c36b492ee..9a8c5b727f59 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -30,6 +30,7 @@
 #include <linux/hrtimer.h>
 #include <trace/events/fence.h>
 
+#include <nvif/cl826e.h>
 #include <nvif/notify.h>
 #include <nvif/event.h>
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 491c7149d197..8e13467d0ddb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -34,6 +34,8 @@
 #include "nouveau_drm.h"
 #include "nouveau_hwmon.h"
 
+#include <nvkm/subdev/volt.h>
+
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 static ssize_t
 nouveau_hwmon_show_temp(struct device *d, struct device_attribute *a, char *buf)
@@ -512,6 +514,35 @@ static SENSOR_DEVICE_ATTR(pwm1_max, S_IRUGO | S_IWUSR,
 			  nouveau_hwmon_get_pwm1_max,
 			  nouveau_hwmon_set_pwm1_max, 0);
 
+static ssize_t
+nouveau_hwmon_get_in0_input(struct device *d,
+			    struct device_attribute *a, char *buf)
+{
+	struct drm_device *dev = dev_get_drvdata(d);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvkm_volt *volt = nvxx_volt(&drm->device);
+	int ret;
+
+	ret = nvkm_volt_get(volt);
+	if (ret < 0)
+		return ret;
+
+	return sprintf(buf, "%i\n", ret / 1000);
+}
+
+static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO,
+			  nouveau_hwmon_get_in0_input, NULL, 0);
+
+static ssize_t
+nouveau_hwmon_get_in0_label(struct device *d,
+			    struct device_attribute *a, char *buf)
+{
+	return sprintf(buf, "GPU core\n");
+}
+
+static SENSOR_DEVICE_ATTR(in0_label, S_IRUGO,
+			  nouveau_hwmon_get_in0_label, NULL, 0);
+
 static struct attribute *hwmon_default_attributes[] = {
 	&sensor_dev_attr_name.dev_attr.attr,
 	&sensor_dev_attr_update_rate.dev_attr.attr,
@@ -542,6 +573,12 @@ static struct attribute *hwmon_pwm_fan_attributes[] = {
 	NULL
 };
 
+static struct attribute *hwmon_in0_attributes[] = {
+	&sensor_dev_attr_in0_input.dev_attr.attr,
+	&sensor_dev_attr_in0_label.dev_attr.attr,
+	NULL
+};
+
 static const struct attribute_group hwmon_default_attrgroup = {
 	.attrs = hwmon_default_attributes,
 };
@@ -554,6 +591,9 @@ static const struct attribute_group hwmon_fan_rpm_attrgroup = {
 static const struct attribute_group hwmon_pwm_fan_attrgroup = {
 	.attrs = hwmon_pwm_fan_attributes,
 };
+static const struct attribute_group hwmon_in0_attrgroup = {
+	.attrs = hwmon_in0_attributes,
+};
 #endif
 
 int
@@ -562,6 +602,7 @@ nouveau_hwmon_init(struct drm_device *dev)
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvkm_therm *therm = nvxx_therm(&drm->device);
+	struct nvkm_volt *volt = nvxx_volt(&drm->device);
 	struct nouveau_hwmon *hwmon;
 	struct device *hwmon_dev;
 	int ret = 0;
@@ -613,6 +654,14 @@ nouveau_hwmon_init(struct drm_device *dev)
 			goto error;
 	}
 
+	if (volt && nvkm_volt_get(volt) >= 0) {
+		ret = sysfs_create_group(&hwmon_dev->kobj,
+					 &hwmon_in0_attrgroup);
+
+		if (ret)
+			goto error;
+	}
+
 	hwmon->hwmon = hwmon_dev;
 
 	return 0;
@@ -638,6 +687,7 @@ nouveau_hwmon_fini(struct drm_device *dev)
 		sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_temp_attrgroup);
 		sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_pwm_fan_attrgroup);
 		sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_fan_rpm_attrgroup);
+		sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_in0_attrgroup);
 
 		hwmon_device_unregister(hwmon->hwmon);
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.c b/drivers/gpu/drm/nouveau/nouveau_platform.c
index 60e32c4e4e49..8a70cec59bcd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_platform.c
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.c
@@ -78,3 +78,14 @@ struct platform_driver nouveau_platform_driver = {
 	.probe = nouveau_platform_probe,
 	.remove = nouveau_platform_remove,
 };
+
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC) || IS_ENABLED(CONFIG_ARCH_TEGRA_132_SOC)
+MODULE_FIRMWARE("nvidia/gk20a/fecs_data.bin");
+MODULE_FIRMWARE("nvidia/gk20a/fecs_inst.bin");
+MODULE_FIRMWARE("nvidia/gk20a/gpccs_data.bin");
+MODULE_FIRMWARE("nvidia/gk20a/gpccs_inst.bin");
+MODULE_FIRMWARE("nvidia/gk20a/sw_bundle_init.bin");
+MODULE_FIRMWARE("nvidia/gk20a/sw_ctx.bin");
+MODULE_FIRMWARE("nvidia/gk20a/sw_method_init.bin");
+MODULE_FIRMWARE("nvidia/gk20a/sw_nonctx.bin");
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.c b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
deleted file mode 100644
index 5dac3546c1b8..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_sysfs.c
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Copyright 2013 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-
-#include <nvif/os.h>
-#include <nvif/class.h>
-#include <nvif/ioctl.h>
-
-#include "nouveau_sysfs.h"
-
-MODULE_PARM_DESC(pstate, "enable sysfs pstate file, which will be moved in the future");
-int nouveau_pstate;
-module_param_named(pstate, nouveau_pstate, int, 0400);
-
-static inline struct drm_device *
-drm_device(struct device *d)
-{
-	return dev_get_drvdata(d);
-}
-
-#define snappendf(p,r,f,a...) do {                                             \
-	snprintf(p, r, f, ##a);                                                \
-	r -= strlen(p);                                                        \
-	p += strlen(p);                                                        \
-} while(0)
-
-static ssize_t
-nouveau_sysfs_pstate_get(struct device *d, struct device_attribute *a, char *b)
-{
-	struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
-	struct nvif_control_pstate_info_v0 info = {};
-	size_t cnt = PAGE_SIZE;
-	char *buf = b;
-	int ret, i;
-
-	ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_INFO,
-			&info, sizeof(info));
-	if (ret)
-		return ret;
-
-	for (i = 0; i < info.count + 1; i++) {
-		const s32 state = i < info.count ? i :
-			NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT;
-		struct nvif_control_pstate_attr_v0 attr = {
-			.state = state,
-			.index = 0,
-		};
-
-		ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_ATTR,
-				&attr, sizeof(attr));
-		if (ret)
-			return ret;
-
-		if (i < info.count)
-			snappendf(buf, cnt, "%02x:", attr.state);
-		else
-			snappendf(buf, cnt, "%s:", info.pwrsrc == 0 ? "DC" :
-						   info.pwrsrc == 1 ? "AC" :
-						   "--");
-
-		attr.index = 0;
-		do {
-			attr.state = state;
-			ret = nvif_mthd(&sysfs->ctrl,
-					NVIF_CONTROL_PSTATE_ATTR,
-					&attr, sizeof(attr));
-			if (ret)
-				return ret;
-
-			snappendf(buf, cnt, " %s %d", attr.name, attr.min);
-			if (attr.min != attr.max)
-				snappendf(buf, cnt, "-%d", attr.max);
-			snappendf(buf, cnt, " %s", attr.unit);
-		} while (attr.index);
-
-		if (state >= 0) {
-			if (info.ustate_ac == state)
-				snappendf(buf, cnt, " AC");
-			if (info.ustate_dc == state)
-				snappendf(buf, cnt, " DC");
-			if (info.pstate == state)
-				snappendf(buf, cnt, " *");
-		} else {
-			if (info.ustate_ac < -1)
-				snappendf(buf, cnt, " AC");
-			if (info.ustate_dc < -1)
-				snappendf(buf, cnt, " DC");
-		}
-
-		snappendf(buf, cnt, "\n");
-	}
-
-	return strlen(b);
-}
-
-static ssize_t
-nouveau_sysfs_pstate_set(struct device *d, struct device_attribute *a,
-			 const char *buf, size_t count)
-{
-	struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
-	struct nvif_control_pstate_user_v0 args = { .pwrsrc = -EINVAL };
-	long value, ret;
-	char *tmp;
-
-	if ((tmp = strchr(buf, '\n')))
-		*tmp = '\0';
-
-	if (!strncasecmp(buf, "dc:", 3)) {
-		args.pwrsrc = 0;
-		buf += 3;
-	} else
-	if (!strncasecmp(buf, "ac:", 3)) {
-		args.pwrsrc = 1;
-		buf += 3;
-	}
-
-	if (!strcasecmp(buf, "none"))
-		args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN;
-	else
-	if (!strcasecmp(buf, "auto"))
-		args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON;
-	else {
-		ret = kstrtol(buf, 16, &value);
-		if (ret)
-			return ret;
-		args.ustate = value;
-	}
-
-	ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_USER,
-			&args, sizeof(args));
-	if (ret < 0)
-		return ret;
-
-	return count;
-}
-
-static DEVICE_ATTR(pstate, S_IRUGO | S_IWUSR,
-		   nouveau_sysfs_pstate_get, nouveau_sysfs_pstate_set);
-
-void
-nouveau_sysfs_fini(struct drm_device *dev)
-{
-	struct nouveau_sysfs *sysfs = nouveau_sysfs(dev);
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nvif_device *device = &drm->device;
-
-	if (sysfs && sysfs->ctrl.priv) {
-		device_remove_file(nvxx_device(device)->dev, &dev_attr_pstate);
-		nvif_object_fini(&sysfs->ctrl);
-	}
-
-	drm->sysfs = NULL;
-	kfree(sysfs);
-}
-
-int
-nouveau_sysfs_init(struct drm_device *dev)
-{
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nvif_device *device = &drm->device;
-	struct nouveau_sysfs *sysfs;
-	int ret;
-
-	if (!nouveau_pstate)
-		return 0;
-
-	sysfs = drm->sysfs = kzalloc(sizeof(*sysfs), GFP_KERNEL);
-	if (!sysfs)
-		return -ENOMEM;
-
-	ret = nvif_object_init(&device->object, 0, NVIF_IOCTL_NEW_V0_CONTROL,
-			       NULL, 0, &sysfs->ctrl);
-	if (ret == 0)
-		device_create_file(nvxx_device(device)->dev, &dev_attr_pstate);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.h b/drivers/gpu/drm/nouveau/nouveau_sysfs.h
deleted file mode 100644
index 4e5ea9241b28..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_sysfs.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef __NOUVEAU_SYSFS_H__
-#define __NOUVEAU_SYSFS_H__
-
-#include "nouveau_drm.h"
-
-struct nouveau_sysfs {
-	struct nvif_object ctrl;
-};
-
-static inline struct nouveau_sysfs *
-nouveau_sysfs(struct drm_device *dev)
-{
-	return nouveau_drm(dev)->sysfs;
-}
-
-int  nouveau_sysfs_init(struct drm_device *);
-void nouveau_sysfs_fini(struct drm_device *);
-
-extern int nouveau_pstate;
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.c b/drivers/gpu/drm/nouveau/nouveau_usif.c
index 6ae1b3494bcd..e9f52ef0be83 100644
--- a/drivers/gpu/drm/nouveau/nouveau_usif.c
+++ b/drivers/gpu/drm/nouveau/nouveau_usif.c
@@ -130,20 +130,21 @@ usif_notify_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
 		struct nvif_notify_req_v0 v0;
 	} *req;
 	struct usif_notify *ntfy;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		if (usif_notify_find(f, args->v0.index))
 			return -EEXIST;
 	} else
 		return ret;
 	req = data;
+	ret = -ENOSYS;
 
 	if (!(ntfy = kmalloc(sizeof(*ntfy), GFP_KERNEL)))
 		return -ENOMEM;
 	atomic_set(&ntfy->enabled, 0);
 
-	if (nvif_unpack(req->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, true))) {
 		ntfy->reply = sizeof(struct nvif_notify_rep_v0) + req->v0.reply;
 		ntfy->route = req->v0.route;
 		ntfy->token = req->v0.token;
@@ -171,9 +172,9 @@ usif_notify_del(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
 		struct nvif_ioctl_ntfy_del_v0 v0;
 	} *args = data;
 	struct usif_notify *ntfy;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		if (!(ntfy = usif_notify_find(f, args->v0.index)))
 			return -ENOENT;
 	} else
@@ -194,9 +195,9 @@ usif_notify_get(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
 		struct nvif_ioctl_ntfy_del_v0 v0;
 	} *args = data;
 	struct usif_notify *ntfy;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		if (!(ntfy = usif_notify_find(f, args->v0.index)))
 			return -ENOENT;
 	} else
@@ -233,9 +234,9 @@ usif_notify_put(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
 		struct nvif_ioctl_ntfy_put_v0 v0;
 	} *args = data;
 	struct usif_notify *ntfy;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		if (!(ntfy = usif_notify_find(f, args->v0.index)))
 			return -ENOENT;
 	} else
@@ -270,13 +271,13 @@ usif_object_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
 		struct nvif_ioctl_new_v0 v0;
 	} *args = data;
 	struct usif_object *object;
-	int ret;
+	int ret = -ENOSYS;
 
 	if (!(object = kmalloc(sizeof(*object), GFP_KERNEL)))
 		return -ENOMEM;
 	list_add(&object->head, &cli->objects);
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		object->route = args->v0.route;
 		object->token = args->v0.token;
 		args->v0.route = NVDRM_OBJECT_USIF;
@@ -310,7 +311,7 @@ usif_ioctl(struct drm_file *filp, void __user *user, u32 argc)
 	if (ret = -EFAULT, copy_from_user(argv, user, size))
 		goto done;
 
-	if (nvif_unpack(argv->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(-ENOSYS, &data, &size, argv->v0, 0, 0, true))) {
 		/* block access to objects not created via this interface */
 		owner = argv->v0.owner;
 		if (argv->v0.object == 0ULL)
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index f3d705d67738..3022d24ed88b 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -26,6 +26,8 @@
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
 
+#include <nvif/if0004.h>
+
 struct nv04_fence_chan {
 	struct nouveau_fence_chan base;
 };
diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c
index 80b6eb8b3d02..6a141c9bf5b7 100644
--- a/drivers/gpu/drm/nouveau/nv17_fence.c
+++ b/drivers/gpu/drm/nouveau/nv17_fence.c
@@ -24,6 +24,7 @@
 
 #include <nvif/os.h>
 #include <nvif/class.h>
+#include <nvif/cl0002.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index c053c50b346a..ea3921652449 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -28,8 +28,16 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_dp_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include <nvif/class.h>
+#include <nvif/cl0002.h>
+#include <nvif/cl5070.h>
+#include <nvif/cl507a.h>
+#include <nvif/cl507b.h>
+#include <nvif/cl507c.h>
+#include <nvif/cl507d.h>
+#include <nvif/cl507e.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
@@ -773,7 +781,6 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
 	 */
 	if (nv_connector && ( nv_connector->underscan == UNDERSCAN_ON ||
 			     (nv_connector->underscan == UNDERSCAN_AUTO &&
-			      nv_connector->edid &&
 			      drm_detect_hdmi_monitor(nv_connector->edid)))) {
 		u32 bX = nv_connector->underscan_hborder;
 		u32 bY = nv_connector->underscan_vborder;
@@ -1717,7 +1724,7 @@ nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
 	encoder = to_drm_encoder(nv_encoder);
 	encoder->possible_crtcs = dcbe->heads;
 	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type);
+	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type, NULL);
 	drm_encoder_helper_add(encoder, &nv50_dac_hfunc);
 
 	drm_mode_connector_attach_encoder(connector, encoder);
@@ -1961,10 +1968,17 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 	switch (nv_encoder->dcb->type) {
 	case DCB_OUTPUT_TMDS:
 		if (nv_encoder->dcb->sorconf.link & 1) {
-			if (mode->clock < 165000)
-				proto = 0x1;
-			else
-				proto = 0x5;
+			proto = 0x1;
+			/* Only enable dual-link if:
+			 *  - Need to (i.e. rate > 165MHz)
+			 *  - DCB says we can
+			 *  - Not an HDMI monitor, since there's no dual-link
+			 *    on HDMI.
+			 */
+			if (mode->clock >= 165000 &&
+			    nv_encoder->dcb->duallink_possible &&
+			    !drm_detect_hdmi_monitor(nv_connector->edid))
+				proto |= 0x4;
 		} else {
 			proto = 0x2;
 		}
@@ -2125,7 +2139,7 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 	encoder = to_drm_encoder(nv_encoder);
 	encoder->possible_crtcs = dcbe->heads;
 	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type);
+	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type, NULL);
 	drm_encoder_helper_add(encoder, &nv50_sor_hfunc);
 
 	drm_mode_connector_attach_encoder(connector, encoder);
@@ -2305,7 +2319,7 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
 	encoder = to_drm_encoder(nv_encoder);
 	encoder->possible_crtcs = dcbe->heads;
 	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type);
+	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type, NULL);
 	drm_encoder_helper_add(encoder, &nv50_pior_hfunc);
 
 	drm_mode_connector_attach_encoder(connector, encoder);
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
index f0d96e5da6b4..3695ccce68c7 100644
--- a/drivers/gpu/drm/nouveau/nv50_fence.c
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -24,6 +24,7 @@
 
 #include <nvif/os.h>
 #include <nvif/class.h>
+#include <nvif/cl0002.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c
index 297e1e953fa6..e1943910858e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/client.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c
@@ -28,6 +28,7 @@
 
 #include <nvif/class.h>
 #include <nvif/event.h>
+#include <nvif/if0000.h>
 #include <nvif/unpack.h>
 
 struct nvkm_client_notify {
@@ -96,7 +97,7 @@ nvkm_client_notify_new(struct nvkm_object *object,
 		struct nvif_notify_req_v0 v0;
 	} *req = data;
 	u8  index, reply;
-	int ret;
+	int ret = -ENOSYS;
 
 	for (index = 0; index < ARRAY_SIZE(client->notify); index++) {
 		if (!client->notify[index])
@@ -111,7 +112,7 @@ nvkm_client_notify_new(struct nvkm_object *object,
 		return -ENOMEM;
 
 	nvif_ioctl(object, "notify new size %d\n", size);
-	if (nvif_unpack(req->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, true))) {
 		nvif_ioctl(object, "notify new vers %d reply %d route %02x "
 				   "token %llx\n", req->v0.version,
 			   req->v0.reply, req->v0.route, req->v0.token);
@@ -143,10 +144,10 @@ nvkm_client_mthd_devlist(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nv_client_devlist_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "client devlist size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "client devlist vers %d count %d\n",
 			   args->v0.version, args->v0.count);
 		if (size == sizeof(args->v0.device[0]) * args->v0.count) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
index d87d6ab03cc7..b0db51847c36 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
@@ -34,10 +34,10 @@ nvkm_ioctl_nop(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_nop_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "nop size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "nop vers %lld\n", args->v0.version);
 		args->v0.version = NVIF_VERSION_LATEST;
 	}
@@ -52,10 +52,10 @@ nvkm_ioctl_sclass(struct nvkm_object *object, void *data, u32 size)
 		struct nvif_ioctl_sclass_v0 v0;
 	} *args = data;
 	struct nvkm_oclass oclass;
-	int ret, i = 0;
+	int ret = -ENOSYS, i = 0;
 
 	nvif_ioctl(object, "sclass size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "sclass vers %d count %d\n",
 			   args->v0.version, args->v0.count);
 		if (size != args->v0.count * sizeof(args->v0.oclass[0]))
@@ -86,10 +86,10 @@ nvkm_ioctl_new(struct nvkm_object *parent, void *data, u32 size)
 	struct nvkm_client *client = parent->client;
 	struct nvkm_object *object = NULL;
 	struct nvkm_oclass oclass;
-	int ret, i = 0;
+	int ret = -ENOSYS, i = 0;
 
 	nvif_ioctl(parent, "new size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(parent, "new vers %d handle %08x class %08x "
 				   "route %02x token %llx object %016llx\n",
 			   args->v0.version, args->v0.handle, args->v0.oclass,
@@ -147,10 +147,10 @@ nvkm_ioctl_del(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_del none;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "delete size %d\n", size);
-	if (nvif_unvers(args->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		nvif_ioctl(object, "delete\n");
 		nvkm_object_fini(object, false);
 		nvkm_object_del(&object);
@@ -165,10 +165,10 @@ nvkm_ioctl_mthd(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_mthd_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "mthd size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "mthd vers %d mthd %02x\n",
 			   args->v0.version, args->v0.method);
 		ret = nvkm_object_mthd(object, args->v0.method, data, size);
@@ -189,10 +189,10 @@ nvkm_ioctl_rd(struct nvkm_object *object, void *data, u32 size)
 		u16 b16;
 		u32 b32;
 	} v;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "rd size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "rd vers %d size %d addr %016llx\n",
 			   args->v0.version, args->v0.size, args->v0.addr);
 		switch (args->v0.size) {
@@ -223,10 +223,10 @@ nvkm_ioctl_wr(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_wr_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "wr size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object,
 			   "wr vers %d size %d addr %016llx data %08x\n",
 			   args->v0.version, args->v0.size, args->v0.addr,
@@ -251,10 +251,10 @@ nvkm_ioctl_map(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_map_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "map size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "map vers %d\n", args->v0.version);
 		ret = nvkm_object_map(object, &args->v0.handle,
 					      &args->v0.length);
@@ -269,10 +269,10 @@ nvkm_ioctl_unmap(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_unmap none;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "unmap size %d\n", size);
-	if (nvif_unvers(args->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		nvif_ioctl(object, "unmap\n");
 	}
 
@@ -286,10 +286,10 @@ nvkm_ioctl_ntfy_new(struct nvkm_object *object, void *data, u32 size)
 		struct nvif_ioctl_ntfy_new_v0 v0;
 	} *args = data;
 	struct nvkm_event *event;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "ntfy new size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "ntfy new vers %d event %02x\n",
 			   args->v0.version, args->v0.event);
 		ret = nvkm_object_ntfy(object, args->v0.event, &event);
@@ -312,10 +312,10 @@ nvkm_ioctl_ntfy_del(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_ntfy_del_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "ntfy del size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "ntfy del vers %d index %d\n",
 			   args->v0.version, args->v0.index);
 		ret = nvkm_client_notify_del(client, args->v0.index);
@@ -331,10 +331,10 @@ nvkm_ioctl_ntfy_get(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_ntfy_get_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "ntfy get size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "ntfy get vers %d index %d\n",
 			   args->v0.version, args->v0.index);
 		ret = nvkm_client_notify_get(client, args->v0.index);
@@ -350,10 +350,10 @@ nvkm_ioctl_ntfy_put(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct nvif_ioctl_ntfy_put_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "ntfy put size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "ntfy put vers %d index %d\n",
 			   args->v0.version, args->v0.index);
 		ret = nvkm_client_notify_put(client, args->v0.index);
@@ -421,12 +421,12 @@ nvkm_ioctl(struct nvkm_client *client, bool supervisor,
 	union {
 		struct nvif_ioctl_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	client->super = supervisor;
 	nvif_ioctl(object, "size %d\n", size);
 
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object,
 			   "vers %d type %02x object %016llx owner %02x\n",
 			   args->v0.version, args->v0.type, args->v0.object,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gk104.c
index c541a1c012dc..e2b944dce9b8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gk104.c
@@ -22,19 +22,65 @@
  * Authors: Ben Skeggs
  */
 #include "priv.h"
+#include <core/enum.h>
 
 #include <nvif/class.h>
 
+static const struct nvkm_enum
+gk104_ce_launcherr_report[] = {
+	{ 0x0, "NO_ERR" },
+	{ 0x1, "2D_LAYER_EXCEEDS_DEPTH" },
+	{ 0x2, "INVALID_ARGUMENT" },
+	{ 0x3, "MEM2MEM_RECT_OUT_OF_BOUNDS" },
+	{ 0x4, "SRC_LINE_EXCEEDS_PITCH" },
+	{ 0x5, "SRC_LINE_EXCEEDS_NEG_PITCH" },
+	{ 0x6, "DST_LINE_EXCEEDS_PITCH" },
+	{ 0x7, "DST_LINE_EXCEEDS_NEG_PITCH" },
+	{ 0x8, "BAD_SRC_PIXEL_COMP_REF" },
+	{ 0x9, "INVALID_VALUE" },
+	{ 0xa, "UNUSED_FIELD" },
+	{ 0xb, "INVALID_OPERATION" },
+	{}
+};
+
+static void
+gk104_ce_intr_launcherr(struct nvkm_engine *ce, const u32 base)
+{
+	struct nvkm_subdev *subdev = &ce->subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x104f14 + base);
+	const struct nvkm_enum *en =
+		nvkm_enum_find(gk104_ce_launcherr_report, stat & 0x0000000f);
+	nvkm_warn(subdev, "LAUNCHERR %08x [%s]\n", stat, en ? en->name : "");
+	nvkm_wr32(device, 0x104f14 + base, 0x00000000);
+}
+
 void
 gk104_ce_intr(struct nvkm_engine *ce)
 {
 	const u32 base = (ce->subdev.index - NVKM_ENGINE_CE0) * 0x1000;
 	struct nvkm_subdev *subdev = &ce->subdev;
 	struct nvkm_device *device = subdev->device;
-	u32 stat = nvkm_rd32(device, 0x104908 + base);
-	if (stat) {
-		nvkm_warn(subdev, "intr %08x\n", stat);
-		nvkm_wr32(device, 0x104908 + base, stat);
+	u32 mask = nvkm_rd32(device, 0x104904 + base);
+	u32 intr = nvkm_rd32(device, 0x104908 + base) & mask;
+	if (intr & 0x00000001) {
+		nvkm_warn(subdev, "BLOCKPIPE\n");
+		nvkm_wr32(device, 0x104908 + base, 0x00000001);
+		intr &= ~0x00000001;
+	}
+	if (intr & 0x00000002) {
+		nvkm_warn(subdev, "NONBLOCKPIPE\n");
+		nvkm_wr32(device, 0x104908 + base, 0x00000002);
+		intr &= ~0x00000002;
+	}
+	if (intr & 0x00000004) {
+		gk104_ce_intr_launcherr(ce, base);
+		nvkm_wr32(device, 0x104908 + base, 0x00000004);
+		intr &= ~0x00000004;
+	}
+	if (intr) {
+		nvkm_warn(subdev, "intr %08x\n", intr);
+		nvkm_wr32(device, 0x104908 + base, intr);
 	}
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index bbc9824af6e0..b1ba1c782a2b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -1388,7 +1388,7 @@ nvc1_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gf106_pci_new,
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
@@ -1423,7 +1423,7 @@ nvc3_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gf106_pci_new,
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
@@ -1566,7 +1566,7 @@ nvcf_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gf106_pci_new,
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
@@ -1601,7 +1601,7 @@ nvd7_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gf106_pci_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
 	.ce[0] = gf100_ce_new,
@@ -1634,7 +1634,7 @@ nvd9_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gf106_pci_new,
 	.pmu = gf119_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1669,7 +1669,7 @@ nve4_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1706,7 +1706,7 @@ nve6_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1743,7 +1743,7 @@ nve7_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1804,7 +1804,7 @@ nvf0_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk110_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1840,7 +1840,7 @@ nvf1_chipset = {
 	.mc = gf100_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk110_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1876,7 +1876,7 @@ nv106_chipset = {
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk208_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1912,7 +1912,7 @@ nv108_chipset = {
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gk208_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
@@ -1948,7 +1948,7 @@ nv117_chipset = {
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
 	.therm = gm107_therm_new,
 	.timer = gk20a_timer_new,
@@ -1973,13 +1973,13 @@ nv124_chipset = {
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm204_i2c_new,
-	.ibus = gk104_ibus_new,
+	.ibus = gm204_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gm107_ltc_new,
+	.ltc = gm204_ltc_new,
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
 	.timer = gk20a_timer_new,
 	.volt = gk104_volt_new,
@@ -2004,13 +2004,13 @@ nv126_chipset = {
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm204_i2c_new,
-	.ibus = gk104_ibus_new,
+	.ibus = gm204_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gm107_ltc_new,
+	.ltc = gm204_ltc_new,
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.mxm = nv50_mxm_new,
-	.pci = g94_pci_new,
+	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
 	.timer = gk20a_timer_new,
 	.volt = gk104_volt_new,
@@ -2033,7 +2033,7 @@ nv12b_chipset = {
 	.fuse = gm107_fuse_new,
 	.ibus = gk20a_ibus_new,
 	.imem = gk20a_instmem_new,
-	.ltc = gm107_ltc_new,
+	.ltc = gm204_ltc_new,
 	.mc = gk20a_mc_new,
 	.mmu = gf100_mmu_new,
 	.timer = gk20a_timer_new,
@@ -2494,7 +2494,8 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		device->pri = ioremap(mmio_base, mmio_size);
 		if (!device->pri) {
 			nvdev_error(device, "unable to map PRI\n");
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto done;
 		}
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
index cf8bc068e9b7..b0ece71aefde 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
@@ -27,6 +27,7 @@
 #include <subdev/clk.h>
 
 #include <nvif/class.h>
+#include <nvif/if0001.h>
 #include <nvif/ioctl.h>
 #include <nvif/unpack.h>
 
@@ -37,10 +38,10 @@ nvkm_control_mthd_pstate_info(struct nvkm_control *ctrl, void *data, u32 size)
 		struct nvif_control_pstate_info_v0 v0;
 	} *args = data;
 	struct nvkm_clk *clk = ctrl->device->clk;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(&ctrl->object, "control pstate info size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(&ctrl->object, "control pstate info vers %d\n",
 			   args->v0.version);
 	} else
@@ -75,10 +76,10 @@ nvkm_control_mthd_pstate_attr(struct nvkm_control *ctrl, void *data, u32 size)
 	struct nvkm_cstate *cstate;
 	int i = 0, j = -1;
 	u32 lo, hi;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(&ctrl->object, "control pstate attr size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(&ctrl->object,
 			   "control pstate attr vers %d state %d index %d\n",
 			   args->v0.version, args->v0.state, args->v0.index);
@@ -143,10 +144,10 @@ nvkm_control_mthd_pstate_user(struct nvkm_control *ctrl, void *data, u32 size)
 		struct nvif_control_pstate_user_v0 v0;
 	} *args = data;
 	struct nvkm_clk *clk = ctrl->device->clk;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(&ctrl->object, "control pstate user size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(&ctrl->object,
 			   "control pstate user vers %d ustate %d pwrsrc %d\n",
 			   args->v0.version, args->v0.ustate, args->v0.pwrsrc);
@@ -204,7 +205,7 @@ nvkm_control_new(struct nvkm_device *device, const struct nvkm_oclass *oclass,
 
 const struct nvkm_device_oclass
 nvkm_control_oclass = {
-	.base.oclass = NVIF_IOCTL_NEW_V0_CONTROL,
+	.base.oclass = NVIF_CLASS_CONTROL,
 	.base.minver = -1,
 	.base.maxver = -1,
 	.ctor = nvkm_control_new,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 1ae48f27029d..137066426ed7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -31,6 +31,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cl0080.h>
 #include <nvif/unpack.h>
 
 struct nvkm_udevice {
@@ -48,10 +49,10 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
 	union {
 		struct nv_device_info_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "device info size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "device info vers %d\n", args->v0.version);
 	} else
 		return ret;
@@ -123,13 +124,16 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
 static int
 nvkm_udevice_time(struct nvkm_udevice *udev, void *data, u32 size)
 {
+	struct nvkm_object *object = &udev->object;
 	struct nvkm_device *device = udev->device;
 	union {
 		struct nv_device_time_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	nvif_ioctl(object, "device time size %d\n", size);
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+		nvif_ioctl(object, "device time vers %d\n", args->v0.version);
 		args->v0.time = nvkm_timer_read(device->timer);
 	}
 
@@ -140,6 +144,7 @@ static int
 nvkm_udevice_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 {
 	struct nvkm_udevice *udev = nvkm_udevice(object);
+	nvif_ioctl(object, "device mthd %08x\n", mthd);
 	switch (mthd) {
 	case NV_DEVICE_V0_INFO:
 		return nvkm_udevice_info(udev, data, size);
@@ -331,10 +336,10 @@ nvkm_udevice_new(const struct nvkm_oclass *oclass, void *data, u32 size,
 	struct nvkm_object *parent = &client->object;
 	const struct nvkm_object_func *func;
 	struct nvkm_udevice *udev;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create device size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create device v%d device %016llx\n",
 			   args->v0.version, args->v0.device);
 	} else
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
index 44b67719f64d..785fa76d0fbf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
@@ -32,6 +32,7 @@
 #include <subdev/bios/dcb.h>
 
 #include <nvif/class.h>
+#include <nvif/cl0046.h>
 #include <nvif/event.h>
 #include <nvif/unpack.h>
 
@@ -58,9 +59,9 @@ nvkm_disp_vblank_ctor(struct nvkm_object *object, void *data, u32 size,
 	union {
 		struct nvif_notify_head_req_v0 v0;
 	} *req = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(req->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, false))) {
 		notify->size = sizeof(struct nvif_notify_head_rep_v0);
 		if (ret = -ENXIO, req->v0.head <= disp->vblank.index_nr) {
 			notify->types = 1;
@@ -96,9 +97,9 @@ nvkm_disp_hpd_ctor(struct nvkm_object *object, void *data, u32 size,
 		struct nvif_notify_conn_req_v0 v0;
 	} *req = data;
 	struct nvkm_output *outp;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(req->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, false))) {
 		notify->size = sizeof(struct nvif_notify_conn_rep_v0);
 		list_for_each_entry(outp, &disp->outp, head) {
 			if (ret = -ENXIO, outp->conn->index == req->v0.conn) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
index 1fd89edefc26..83f152300ec0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
@@ -27,6 +27,7 @@
 #include <core/client.h>
 
 #include <nvif/class.h>
+#include <nvif/cl507c.h>
 #include <nvif/unpack.h>
 
 int
@@ -41,11 +42,11 @@ nv50_disp_base_new(const struct nv50_disp_dmac_func *func,
 	} *args = data;
 	struct nvkm_object *parent = oclass->parent;
 	struct nv50_disp *disp = root->disp;
-	int head, ret;
+	int head, ret = -ENOSYS;
 	u64 push;
 
 	nvif_ioctl(parent, "create disp base channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp base channel dma vers %d "
 				   "pushbuf %016llx head %d\n",
 			   args->v0.version, args->v0.pushbuf, args->v0.head);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
index 01803c0679b6..dd2953bc9264 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
@@ -28,7 +28,7 @@
 #include <core/ramht.h>
 #include <engine/dma.h>
 
-#include <nvif/class.h>
+#include <nvif/cl507d.h>
 #include <nvif/event.h>
 #include <nvif/unpack.h>
 
@@ -134,9 +134,9 @@ nv50_disp_chan_uevent_ctor(struct nvkm_object *object, void *data, u32 size,
 	union {
 		struct nvif_notify_uevent_req none;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unvers(args->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		notify->size  = sizeof(struct nvif_notify_uevent_rep);
 		notify->types = 1;
 		notify->index = chan->chid;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
index db4a9b3e0e09..b547c8b833ca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
@@ -28,6 +28,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cl507d.h>
 #include <nvif/unpack.h>
 
 int
@@ -42,10 +43,10 @@ nv50_disp_core_new(const struct nv50_disp_dmac_func *func,
 	} *args = data;
 	struct nvkm_object *parent = oclass->parent;
 	u64 push;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create disp core channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp core channel dma vers %d "
 				   "pushbuf %016llx\n",
 			   args->v0.version, args->v0.pushbuf);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
index 225858e62cf6..8b1320499a0f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
@@ -27,6 +27,7 @@
 #include <core/client.h>
 
 #include <nvif/class.h>
+#include <nvif/cl507a.h>
 #include <nvif/unpack.h>
 
 int
@@ -41,10 +42,10 @@ nv50_disp_curs_new(const struct nv50_disp_chan_func *func,
 	} *args = data;
 	struct nvkm_object *parent = oclass->parent;
 	struct nv50_disp *disp = root->disp;
-	int head, ret;
+	int head, ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create disp cursor size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp cursor vers %d head %d\n",
 			   args->v0.version, args->v0.head);
 		if (args->v0.head > disp->base.head.nr)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
index 9bfa9e7dc161..c9b78b8f9c87 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
@@ -27,7 +27,7 @@
 #include <core/client.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -39,10 +39,10 @@ nv50_dac_power(NV50_DISP_MTHD_V1)
 		struct nv50_disp_dac_pwr_v0 v0;
 	} *args = data;
 	u32 stat;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp dac pwr size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp dac pwr vers %d state %d data %d "
 				   "vsync %d hsync %d\n",
 			   args->v0.version, args->v0.state, args->v0.data,
@@ -76,10 +76,10 @@ nv50_dac_sense(NV50_DISP_MTHD_V1)
 	} *args = data;
 	const u32 doff = outp->or * 0x800;
 	u32 loadval;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp dac load size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp dac load vers %d data %08x\n",
 			   args->v0.version, args->v0.data);
 		if (args->v0.data & 0xfff00000)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
index 186fd3ac78f6..f0314664349c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
@@ -158,7 +158,7 @@ exec_clkcmp(struct nv50_disp *disp, int head, int id, u32 pclk, u32 *conf)
 	switch (outp->info.type) {
 	case DCB_OUTPUT_TMDS:
 		*conf = (ctrl & 0x00000f00) >> 8;
-		if (pclk >= 165000)
+		if (*conf == 5)
 			*conf |= 0x0100;
 		break;
 	case DCB_OUTPUT_LVDS:
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagf119.c
index af99efbd63f7..da6129b2b78f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagf119.c
@@ -29,7 +29,7 @@
 #include <subdev/bios/dcb.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -41,10 +41,10 @@ gf119_hda_eld(NV50_DISP_MTHD_V1)
 	} *args = data;
 	const u32 soff = outp->or * 0x030;
 	const u32 hoff = head * 0x800;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(object, "disp sor hda eld size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "disp sor hda eld vers %d\n",
 			   args->v0.version);
 		if (size > 0x60)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c
index c1590b746f13..6f0436df0219 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c
@@ -27,7 +27,7 @@
 #include <core/client.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -38,10 +38,10 @@ gt215_hda_eld(NV50_DISP_MTHD_V1)
 		struct nv50_disp_sor_hda_eld_v0 v0;
 	} *args = data;
 	const u32 soff = outp->or * 0x800;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(object, "disp sor hda eld size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "disp sor hda eld vers %d\n",
 			   args->v0.version);
 		if (size > 0x60)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmig84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmig84.c
index ee9e800a8f06..1c4256e8cbd6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmig84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmig84.c
@@ -25,7 +25,7 @@
 
 #include <core/client.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -37,10 +37,10 @@ g84_hdmi_ctrl(NV50_DISP_MTHD_V1)
 		struct nv50_disp_sor_hdmi_pwr_v0 v0;
 	} *args = data;
 	u32 ctrl;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
 				   "max_ac_packet %d rekey %d\n",
 			   args->v0.version, args->v0.state,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigf119.c
index b5af025d3b04..632f02da1382 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigf119.c
@@ -25,7 +25,7 @@
 
 #include <core/client.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -37,10 +37,10 @@ gf119_hdmi_ctrl(NV50_DISP_MTHD_V1)
 		struct nv50_disp_sor_hdmi_pwr_v0 v0;
 	} *args = data;
 	u32 ctrl;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
 				   "max_ac_packet %d rekey %d\n",
 			   args->v0.version, args->v0.state,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigk104.c
index 110dc19e4f67..4e8067d511d7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigk104.c
@@ -25,7 +25,7 @@
 
 #include <core/client.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -38,10 +38,10 @@ gk104_hdmi_ctrl(NV50_DISP_MTHD_V1)
 		struct nv50_disp_sor_hdmi_pwr_v0 v0;
 	} *args = data;
 	u32 ctrl;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
 				   "max_ac_packet %d rekey %d\n",
 			   args->v0.version, args->v0.state,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigt215.c
index 61237dbfa35a..f1afc16494b6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigt215.c
@@ -26,7 +26,7 @@
 
 #include <core/client.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -38,10 +38,10 @@ gt215_hdmi_ctrl(NV50_DISP_MTHD_V1)
 		struct nv50_disp_sor_hdmi_pwr_v0 v0;
 	} *args = data;
 	u32 ctrl;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
 				   "max_ac_packet %d rekey %d\n",
 			   args->v0.version, args->v0.state,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
index 32e73a975b58..4226d2153b9c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
@@ -391,7 +391,7 @@ exec_clkcmp(struct nv50_disp *disp, int head, int id, u32 pclk, u32 *conf)
 		switch (outp->info.type) {
 		case DCB_OUTPUT_TMDS:
 			*conf = (ctrl & 0x00000f00) >> 8;
-			if (pclk >= 165000)
+			if (*conf == 5)
 				*conf |= 0x0100;
 			break;
 		case DCB_OUTPUT_LVDS:
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
index cd888a1e443c..3940b9c966ec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
@@ -27,6 +27,7 @@
 #include <core/client.h>
 
 #include <nvif/class.h>
+#include <nvif/cl507b.h>
 #include <nvif/unpack.h>
 
 int
@@ -41,10 +42,10 @@ nv50_disp_oimm_new(const struct nv50_disp_chan_func *func,
 	} *args = data;
 	struct nvkm_object *parent = oclass->parent;
 	struct nv50_disp *disp = root->disp;
-	int head, ret;
+	int head, ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create disp overlay size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp overlay vers %d head %d\n",
 			   args->v0.version, args->v0.head);
 		if (args->v0.head > disp->base.head.nr)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
index 6fa296c047b8..2a49c46425cd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
@@ -27,6 +27,7 @@
 #include <core/client.h>
 
 #include <nvif/class.h>
+#include <nvif/cl507e.h>
 #include <nvif/unpack.h>
 
 int
@@ -41,11 +42,11 @@ nv50_disp_ovly_new(const struct nv50_disp_dmac_func *func,
 	} *args = data;
 	struct nvkm_object *parent = oclass->parent;
 	struct nv50_disp *disp = root->disp;
-	int head, ret;
+	int head, ret = -ENOSYS;
 	u64 push;
 
 	nvif_ioctl(parent, "create disp overlay channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp overlay channel dma vers %d "
 				   "pushbuf %016llx head %d\n",
 			   args->v0.version, args->v0.pushbuf, args->v0.head);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
index ab524bde7795..6c532eadba17 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
@@ -28,7 +28,7 @@
 #include <subdev/i2c.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -40,10 +40,10 @@ nv50_pior_power(NV50_DISP_MTHD_V1)
 		struct nv50_disp_pior_pwr_v0 v0;
 	} *args = data;
 	u32 ctrl, type;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp pior pwr size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp pior pwr vers %d state %d type %x\n",
 			   args->v0.version, args->v0.state, args->v0.type);
 		if (args->v0.type > 0x0f)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
index 8591726871ac..335d88823c22 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
@@ -29,6 +29,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -39,12 +40,12 @@ gf119_disp_root_scanoutpos(NV50_DISP_MTHD_V0)
 	const u32 blanke = nvkm_rd32(device, 0x64041c + (head * 0x300));
 	const u32 blanks = nvkm_rd32(device, 0x640420 + (head * 0x300));
 	union {
-		struct nv04_disp_scanoutpos_v0 v0;
+		struct nv50_disp_scanoutpos_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp scanoutpos size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp scanoutpos vers %d\n",
 			   args->v0.version);
 		args->v0.vblanke = (blanke & 0xffff0000) >> 16;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c
index 2be846374d39..f535f43231e2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c
@@ -27,6 +27,7 @@
 #include <core/client.h>
 
 #include <nvif/class.h>
+#include <nvif/cl0046.h>
 #include <nvif/unpack.h>
 
 struct nv04_disp_root {
@@ -45,10 +46,10 @@ nv04_disp_scanoutpos(struct nv04_disp_root *root,
 		struct nv04_disp_scanoutpos_v0 v0;
 	} *args = data;
 	u32 line;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp scanoutpos size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp scanoutpos vers %d\n",
 			   args->v0.version);
 		args->v0.vblanks = nvkm_rd32(device, 0x680800 + hoff) & 0xffff;
@@ -85,10 +86,10 @@ nv04_disp_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 	union {
 		struct nv04_disp_mthd_v0 v0;
 	} *args = data;
-	int head, ret;
+	int head, ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp mthd size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "disp mthd vers %d mthd %02x head %d\n",
 			   args->v0.version, args->v0.method, args->v0.head);
 		mthd = args->v0.method;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
index 06fb24d88702..2f9cecd81d04 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
@@ -29,6 +29,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -39,12 +40,12 @@ nv50_disp_root_scanoutpos(NV50_DISP_MTHD_V0)
 	const u32 blanks = nvkm_rd32(device, 0x610af4 + (head * 0x540));
 	const u32 total  = nvkm_rd32(device, 0x610afc + (head * 0x540));
 	union {
-		struct nv04_disp_scanoutpos_v0 v0;
+		struct nv50_disp_scanoutpos_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp scanoutpos size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp scanoutpos vers %d\n",
 			   args->v0.version);
 		args->v0.vblanke = (blanke & 0xffff0000) >> 16;
@@ -78,19 +79,19 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 	struct nvkm_output *outp = NULL;
 	struct nvkm_output *temp;
 	u16 type, mask = 0;
-	int head, ret;
+	int head, ret = -ENOSYS;
 
 	if (mthd != NV50_DISP_MTHD)
 		return -EINVAL;
 
 	nvif_ioctl(object, "disp mthd size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(object, "disp mthd vers %d mthd %02x head %d\n",
 			   args->v0.version, args->v0.method, args->v0.head);
 		mthd = args->v0.method;
 		head = args->v0.head;
 	} else
-	if (nvif_unpack(args->v1, 1, 1, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v1, 1, 1, true))) {
 		nvif_ioctl(object, "disp mthd vers %d mthd %02x "
 				   "type %04x mask %04x\n",
 			   args->v1.version, args->v1.method,
@@ -143,8 +144,9 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 		union {
 			struct nv50_disp_sor_lvds_script_v0 v0;
 		} *args = data;
+		int ret = -ENOSYS;
 		nvif_ioctl(object, "disp sor lvds script size %d\n", size);
-		if (nvif_unpack(args->v0, 0, 0, false)) {
+		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 			nvif_ioctl(object, "disp sor lvds script "
 					   "vers %d name %04x\n",
 				   args->v0.version, args->v0.script);
@@ -159,8 +161,9 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 		union {
 			struct nv50_disp_sor_dp_pwr_v0 v0;
 		} *args = data;
+		int ret = -ENOSYS;
 		nvif_ioctl(object, "disp sor dp pwr size %d\n", size);
-		if (nvif_unpack(args->v0, 0, 0, false)) {
+		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 			nvif_ioctl(object, "disp sor dp pwr vers %d state %d\n",
 				   args->v0.version, args->v0.state);
 			if (args->v0.state == 0) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
index 29e0d2a9a839..53596bed3c36 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
@@ -27,7 +27,7 @@
 #include <core/client.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl5070.h>
 #include <nvif/unpack.h>
 
 int
@@ -39,10 +39,10 @@ nv50_sor_power(NV50_DISP_MTHD_V1)
 	} *args = data;
 	const u32 soff = outp->or * 0x800;
 	u32 stat;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "disp sor pwr size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "disp sor pwr vers %d state %d\n",
 			   args->v0.version, args->v0.state);
 		stat = !!args->v0.state;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c
index 45ab062661a4..13c661b1ef14 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c
@@ -28,7 +28,7 @@
 #include <subdev/fb.h>
 #include <subdev/instmem.h>
 
-#include <nvif/class.h>
+#include <nvif/cl0002.h>
 #include <nvif/unpack.h>
 
 static int
@@ -69,7 +69,7 @@ nvkm_dmaobj_ctor(const struct nvkm_dmaobj_func *func, struct nvkm_dma *dma,
 	struct nvkm_fb *fb = device->fb;
 	void *data = *pdata;
 	u32 size = *psize;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvkm_object_ctor(&nvkm_dmaobj_func, oclass, &dmaobj->object);
 	dmaobj->func = func;
@@ -77,7 +77,7 @@ nvkm_dmaobj_ctor(const struct nvkm_dmaobj_func *func, struct nvkm_dma *dma,
 	RB_CLEAR_NODE(&dmaobj->rb);
 
 	nvif_ioctl(parent, "create dma size %d\n", *psize);
-	if (nvif_unpack(args->v0, 0, 0, true)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
 		nvif_ioctl(parent, "create dma vers %d target %d access %d "
 				   "start %016llx limit %016llx\n",
 			   args->v0.version, args->v0.target, args->v0.access,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf100.c
index 13e341cc4e32..ef7ac360101e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf100.c
@@ -28,7 +28,7 @@
 #include <core/gpuobj.h>
 #include <subdev/fb.h>
 
-#include <nvif/class.h>
+#include <nvif/cl0002.h>
 #include <nvif/unpack.h>
 
 struct gf100_dmaobj {
@@ -87,10 +87,11 @@ gf100_dmaobj_new(struct nvkm_dma *dma, const struct nvkm_oclass *oclass,
 	if (ret)
 		return ret;
 
+	ret  = -ENOSYS;
 	args = data;
 
 	nvif_ioctl(parent, "create gf100 dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent,
 			   "create gf100 dma vers %d priv %d kind %02x\n",
 			   args->v0.version, args->v0.priv, args->v0.kind);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf119.c
index 0e1af8b4db84..c068cee34588 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergf119.c
@@ -28,7 +28,7 @@
 #include <core/gpuobj.h>
 #include <subdev/fb.h>
 
-#include <nvif/class.h>
+#include <nvif/cl0002.h>
 #include <nvif/unpack.h>
 
 struct gf119_dmaobj {
@@ -85,10 +85,11 @@ gf119_dmaobj_new(struct nvkm_dma *dma, const struct nvkm_oclass *oclass,
 	if (ret)
 		return ret;
 
+	ret  = -ENOSYS;
 	args = data;
 
 	nvif_ioctl(parent, "create gf119 dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent,
 			   "create gf100 dma vers %d page %d kind %02x\n",
 			   args->v0.version, args->v0.page, args->v0.kind);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv50.c
index 5b7ce313ea14..6a85b5dea643 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv50.c
@@ -28,7 +28,7 @@
 #include <core/gpuobj.h>
 #include <subdev/fb.h>
 
-#include <nvif/class.h>
+#include <nvif/cl0002.h>
 #include <nvif/unpack.h>
 
 struct nv50_dmaobj {
@@ -87,10 +87,11 @@ nv50_dmaobj_new(struct nvkm_dma *dma, const struct nvkm_oclass *oclass,
 	if (ret)
 		return ret;
 
+	ret  = -ENOSYS;
 	args = data;
 
 	nvif_ioctl(parent, "create nv50 dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create nv50 dma vers %d priv %d part %d "
 				   "comp %d kind %02x\n", args->v0.version,
 			   args->v0.priv, args->v0.part, args->v0.comp,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 1fbbfbe6ca9c..cfc7d5725a61 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -129,9 +129,9 @@ nvkm_fifo_uevent_ctor(struct nvkm_object *object, void *data, u32 size,
 	union {
 		struct nvif_notify_uevent_req none;
 	} *req = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unvers(req->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, req->none))) {
 		notify->size  = sizeof(struct nvif_notify_uevent_rep);
 		notify->types = 1;
 		notify->index = 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
index 04305241ceed..aeb3387a3fb0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
@@ -28,7 +28,7 @@
 #include <subdev/mmu.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
+#include <nvif/cl826e.h>
 
 int
 g84_fifo_chan_ntfy(struct nvkm_fifo_chan *chan, u32 type,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
index a5ca52c7b74f..caa914074752 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
@@ -27,6 +27,7 @@
 #include <core/ramht.h>
 
 #include <nvif/class.h>
+#include <nvif/cl826e.h>
 #include <nvif/unpack.h>
 
 static int
@@ -35,14 +36,14 @@ g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 {
 	struct nvkm_object *parent = oclass->parent;
 	union {
-		struct nv50_channel_dma_v0 v0;
+		struct g82_channel_dma_v0 v0;
 	} *args = data;
 	struct nv50_fifo *fifo = nv50_fifo(base);
 	struct nv50_fifo_chan *chan;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d vm %llx "
 				   "pushbuf %llx offset %016llx\n",
 			   args->v0.version, args->v0.vm, args->v0.pushbuf,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
index bfcc6408a772..edec30fd3ecd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
@@ -29,6 +29,7 @@
 #include <subdev/instmem.h>
 
 #include <nvif/class.h>
+#include <nvif/cl006b.h>
 #include <nvif/unpack.h>
 
 void
@@ -167,10 +168,10 @@ nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nv04_fifo_chan *chan = NULL;
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct nvkm_instmem *imem = device->imem;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx "
 				   "offset %08x\n", args->v0.version,
 			   args->v0.pushbuf, args->v0.offset);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c
index 34f68e5bd040..f5f355ff005d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c
@@ -29,6 +29,7 @@
 #include <subdev/instmem.h>
 
 #include <nvif/class.h>
+#include <nvif/cl006b.h>
 #include <nvif/unpack.h>
 
 static int
@@ -43,10 +44,10 @@ nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nv04_fifo_chan *chan = NULL;
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct nvkm_instmem *imem = device->imem;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx "
 				   "offset %08x\n", args->v0.version,
 			   args->v0.pushbuf, args->v0.offset);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c
index ed7cc9f2b540..7edc6a564b5d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c
@@ -29,6 +29,7 @@
 #include <subdev/instmem.h>
 
 #include <nvif/class.h>
+#include <nvif/cl006b.h>
 #include <nvif/unpack.h>
 
 static int
@@ -43,10 +44,10 @@ nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nv04_fifo_chan *chan = NULL;
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct nvkm_instmem *imem = device->imem;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx "
 				   "offset %08x\n", args->v0.version,
 			   args->v0.pushbuf, args->v0.offset);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c
index 043b6c325949..0ec179fc40a1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c
@@ -29,6 +29,7 @@
 #include <subdev/instmem.h>
 
 #include <nvif/class.h>
+#include <nvif/cl006b.h>
 #include <nvif/unpack.h>
 
 static bool
@@ -188,10 +189,10 @@ nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nv04_fifo_chan *chan = NULL;
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct nvkm_instmem *imem = device->imem;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx "
 				   "offset %08x\n", args->v0.version,
 			   args->v0.pushbuf, args->v0.offset);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
index 6b3b15f12c39..480bc3777be5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
@@ -27,6 +27,7 @@
 #include <core/ramht.h>
 
 #include <nvif/class.h>
+#include <nvif/cl506e.h>
 #include <nvif/unpack.h>
 
 static int
@@ -39,10 +40,10 @@ nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	} *args = data;
 	struct nv50_fifo *fifo = nv50_fifo(base);
 	struct nv50_fifo_chan *chan;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel dma size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel dma vers %d vm %llx "
 				   "pushbuf %llx offset %016llx\n",
 			   args->v0.version, args->v0.vm, args->v0.pushbuf,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index ff6fcbda615b..36a39c7fd8d2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -47,7 +47,7 @@ gf100_fifo_uevent_fini(struct nvkm_fifo *fifo)
 }
 
 void
-gf100_fifo_runlist_update(struct gf100_fifo *fifo)
+gf100_fifo_runlist_commit(struct gf100_fifo *fifo)
 {
 	struct gf100_fifo_chan *chan;
 	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
@@ -77,6 +77,22 @@ gf100_fifo_runlist_update(struct gf100_fifo *fifo)
 	mutex_unlock(&subdev->mutex);
 }
 
+void
+gf100_fifo_runlist_remove(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan)
+{
+	mutex_lock(&fifo->base.engine.subdev.mutex);
+	list_del_init(&chan->head);
+	mutex_unlock(&fifo->base.engine.subdev.mutex);
+}
+
+void
+gf100_fifo_runlist_insert(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan)
+{
+	mutex_lock(&fifo->base.engine.subdev.mutex);
+	list_add_tail(&chan->head, &fifo->chan);
+	mutex_unlock(&fifo->base.engine.subdev.mutex);
+}
+
 static inline int
 gf100_fifo_engidx(struct gf100_fifo *fifo, u32 engn)
 {
@@ -139,7 +155,7 @@ gf100_fifo_recover_work(struct work_struct *work)
 		}
 	}
 
-	gf100_fifo_runlist_update(fifo);
+	gf100_fifo_runlist_commit(fifo);
 	nvkm_wr32(device, 0x00262c, engm);
 	nvkm_mask(device, 0x002630, engm, 0x00000000);
 }
@@ -239,7 +255,7 @@ gf100_fifo_fault_engine[] = {
 	{ 0x14, "PMSPDEC", NULL, NVKM_ENGINE_MSPDEC },
 	{ 0x15, "PCE0", NULL, NVKM_ENGINE_CE0 },
 	{ 0x16, "PCE1", NULL, NVKM_ENGINE_CE1 },
-	{ 0x17, "PDAEMON" },
+	{ 0x17, "PMU" },
 	{}
 };
 
@@ -270,7 +286,7 @@ gf100_fifo_fault_hubclient[] = {
 	{ 0x0c, "PMSPPP" },
 	{ 0x0d, "PMSVLD" },
 	{ 0x11, "PCOUNTER" },
-	{ 0x12, "PDAEMON" },
+	{ 0x12, "PMU" },
 	{ 0x14, "CCACHE" },
 	{ 0x15, "CCACHE_POST" },
 	{}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
index c649ca9b53e3..08c33c3ceaf7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
@@ -5,6 +5,7 @@
 
 #include <subdev/mmu.h>
 
+struct gf100_fifo_chan;
 struct gf100_fifo {
 	struct nvkm_fifo base;
 
@@ -27,5 +28,7 @@ struct gf100_fifo {
 };
 
 void gf100_fifo_intr_engine(struct gf100_fifo *);
-void gf100_fifo_runlist_update(struct gf100_fifo *);
+void gf100_fifo_runlist_insert(struct gf100_fifo *, struct gf100_fifo_chan *);
+void gf100_fifo_runlist_remove(struct gf100_fifo *, struct gf100_fifo_chan *);
+void gf100_fifo_runlist_commit(struct gf100_fifo *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 98970a0b7a66..4fcd147d43c8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -47,7 +47,7 @@ gk104_fifo_uevent_init(struct nvkm_fifo *fifo)
 }
 
 void
-gk104_fifo_runlist_update(struct gk104_fifo *fifo, u32 engine)
+gk104_fifo_runlist_commit(struct gk104_fifo *fifo, u32 engine)
 {
 	struct gk104_fifo_engn *engn = &fifo->engine[engine];
 	struct gk104_fifo_chan *chan;
@@ -78,6 +78,22 @@ gk104_fifo_runlist_update(struct gk104_fifo *fifo, u32 engine)
 	mutex_unlock(&subdev->mutex);
 }
 
+void
+gk104_fifo_runlist_remove(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan)
+{
+	mutex_lock(&fifo->base.engine.subdev.mutex);
+	list_del_init(&chan->head);
+	mutex_unlock(&fifo->base.engine.subdev.mutex);
+}
+
+void
+gk104_fifo_runlist_insert(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan)
+{
+	mutex_lock(&fifo->base.engine.subdev.mutex);
+	list_add_tail(&chan->head, &fifo->engine[chan->engine].chan);
+	mutex_unlock(&fifo->base.engine.subdev.mutex);
+}
+
 static inline struct nvkm_engine *
 gk104_fifo_engine(struct gk104_fifo *fifo, u32 engn)
 {
@@ -112,7 +128,7 @@ gk104_fifo_recover_work(struct work_struct *work)
 			nvkm_subdev_fini(&engine->subdev, false);
 			WARN_ON(nvkm_subdev_init(&engine->subdev));
 		}
-		gk104_fifo_runlist_update(fifo, gk104_fifo_subdev_engine(engn));
+		gk104_fifo_runlist_commit(fifo, gk104_fifo_subdev_engine(engn));
 	}
 
 	nvkm_wr32(device, 0x00262c, engm);
@@ -180,7 +196,7 @@ gk104_fifo_intr_sched_ctxsw(struct gk104_fifo *fifo)
 
 	spin_lock_irqsave(&fifo->base.lock, flags);
 	for (engn = 0; engn < ARRAY_SIZE(fifo->engine); engn++) {
-		u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x04));
+		u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x08));
 		u32 busy = (stat & 0x80000000);
 		u32 next = (stat & 0x07ff0000) >> 16;
 		u32 chsw = (stat & 0x00008000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index 5afd9b5ec5d1..bec519d8f91e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -5,6 +5,7 @@
 
 #include <subdev/mmu.h>
 
+struct gk104_fifo_chan;
 struct gk104_fifo_engn {
 	struct nvkm_memory *runlist[2];
 	int cur_runlist;
@@ -35,7 +36,9 @@ void gk104_fifo_fini(struct nvkm_fifo *);
 void gk104_fifo_intr(struct nvkm_fifo *);
 void gk104_fifo_uevent_init(struct nvkm_fifo *);
 void gk104_fifo_uevent_fini(struct nvkm_fifo *);
-void gk104_fifo_runlist_update(struct gk104_fifo *, u32 engine);
+void gk104_fifo_runlist_insert(struct gk104_fifo *, struct gk104_fifo_chan *);
+void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *);
+void gk104_fifo_runlist_commit(struct gk104_fifo *, u32 engine);
 
 static inline u64
 gk104_fifo_engine_subdev(int engine)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
index 820132363f68..77c2f2a28bf3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
@@ -27,6 +27,7 @@
 #include <core/ramht.h>
 
 #include <nvif/class.h>
+#include <nvif/cl826f.h>
 #include <nvif/unpack.h>
 
 static int
@@ -35,15 +36,15 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 {
 	struct nvkm_object *parent = oclass->parent;
 	union {
-		struct nv50_channel_gpfifo_v0 v0;
+		struct g82_channel_gpfifo_v0 v0;
 	} *args = data;
 	struct nv50_fifo *fifo = nv50_fifo(base);
 	struct nv50_fifo_chan *chan;
 	u64 ioffset, ilength;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
 				   "pushbuf %llx ioffset %016llx "
 				   "ilength %08x\n",
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
index e7cbc139c1d4..cbc67f262322 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
@@ -29,6 +29,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cl906f.h>
 #include <nvif/unpack.h>
 
 static u32
@@ -138,9 +139,9 @@ gf100_fifo_gpfifo_fini(struct nvkm_fifo_chan *base)
 	u32 coff = chan->base.chid * 8;
 
 	if (!list_empty(&chan->head) && !chan->killed) {
-		list_del_init(&chan->head);
+		gf100_fifo_runlist_remove(fifo, chan);
 		nvkm_mask(device, 0x003004 + coff, 0x00000001, 0x00000000);
-		gf100_fifo_runlist_update(fifo);
+		gf100_fifo_runlist_commit(fifo);
 	}
 
 	gf100_fifo_intr_engine(fifo);
@@ -160,9 +161,9 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
 	nvkm_wr32(device, 0x003000 + coff, 0xc0000000 | addr);
 
 	if (list_empty(&chan->head) && !chan->killed) {
-		list_add_tail(&chan->head, &fifo->chan);
+		gf100_fifo_runlist_insert(fifo, chan);
 		nvkm_wr32(device, 0x003004 + coff, 0x001f0001);
-		gf100_fifo_runlist_update(fifo);
+		gf100_fifo_runlist_commit(fifo);
 	}
 }
 
@@ -199,10 +200,10 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nvkm_object *parent = oclass->parent;
 	struct gf100_fifo_chan *chan;
 	u64 usermem, ioffset, ilength;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
 				   "ioffset %016llx ilength %08x\n",
 			   args->v0.version, args->v0.vm, args->v0.ioffset,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
index 0b817540a9e4..2e1df01bd928 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
@@ -30,6 +30,7 @@
 #include <subdev/timer.h>
 
 #include <nvif/class.h>
+#include <nvif/cla06f.h>
 #include <nvif/unpack.h>
 
 static int
@@ -151,9 +152,9 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base)
 	u32 coff = chan->base.chid * 8;
 
 	if (!list_empty(&chan->head)) {
-		list_del_init(&chan->head);
+		gk104_fifo_runlist_remove(fifo, chan);
 		nvkm_mask(device, 0x800004 + coff, 0x00000800, 0x00000800);
-		gk104_fifo_runlist_update(fifo, chan->engine);
+		gk104_fifo_runlist_commit(fifo, chan->engine);
 	}
 
 	nvkm_wr32(device, 0x800000 + coff, 0x00000000);
@@ -172,9 +173,9 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
 	nvkm_wr32(device, 0x800000 + coff, 0x80000000 | addr);
 
 	if (list_empty(&chan->head) && !chan->killed) {
-		list_add_tail(&chan->head, &fifo->engine[chan->engine].chan);
+		gk104_fifo_runlist_insert(fifo, chan);
 		nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400);
-		gk104_fifo_runlist_update(fifo, chan->engine);
+		gk104_fifo_runlist_commit(fifo, chan->engine);
 		nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400);
 	}
 }
@@ -213,10 +214,10 @@ gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct gk104_fifo_chan *chan;
 	u64 usermem, ioffset, ilength;
 	u32 engines;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
 				   "ioffset %016llx ilength %08x engine %08x\n",
 			   args->v0.version, args->v0.vm, args->v0.ioffset,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
index a8c69f878221..c5a7de9db259 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
@@ -27,6 +27,7 @@
 #include <core/ramht.h>
 
 #include <nvif/class.h>
+#include <nvif/cl506f.h>
 #include <nvif/unpack.h>
 
 static int
@@ -40,10 +41,10 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 	struct nv50_fifo *fifo = nv50_fifo(base);
 	struct nv50_fifo_chan *chan;
 	u64 ioffset, ilength;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
 				   "pushbuf %llx ioffset %016llx "
 				   "ilength %08x\n",
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
index ddaa16a71c84..ad0a6cfe7580 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
@@ -55,7 +55,7 @@ gk20a_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 
 	gk104_grctx_generate_rop_active_fbps(gr);
 
-	nvkm_mask(device, 0x5044b0, 0x8000000, 0x8000000);
+	nvkm_mask(device, 0x5044b0, 0x08000000, 0x08000000);
 
 	gf100_gr_wait_idle(gr);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpc.fuc b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpc.fuc
index 7dacb3cc0668..e168b83a10c9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpc.fuc
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpc.fuc
@@ -247,10 +247,7 @@ init:
 	tpc_strand_info(-1);
 
 	ld b32 $r4 D[$r0 + #tpc_count]
-	mov $r5 NV_PGRAPH_GPC0_TPC0
-	ld b32 $r6 D[$r0 + #gpc_id]
-	shl b32 $r6 15
-	add b32 $r5 $r6
+	gpc_addr($r5, NV_PGRAPH_GPC0_TPC0)
 	tpc_strand_init_tpc_loop:
 		add b32 $r14 $r5 NV_TPC_STRAND_CNT
 		call(nv_rd32)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
index 11bf363a6ae9..5136f9161706 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
@@ -289,7 +289,7 @@ uint32_t gm107_grgpc_code[] = {
 	0x020014fe,
 	0x12004002,
 	0xbd0002f6,
-	0x05b34104,
+	0x05ad4104,
 	0x400010fe,
 	0x00f60700,
 	0x0204bd00,
@@ -387,180 +387,180 @@ uint32_t gm107_grgpc_code[] = {
 	0x7e00008f,
 	0x98000314,
 	0x00850504,
-	0x06985040,
-	0x0f64b604,
-/* 0x04e3: tpc_strand_init_tpc_loop */
-	0xb80056bb,
-	0x0005705e,
-	0x0000657e,
-	0x74bdf6b2,
-/* 0x04f0: tpc_strand_init_idx_loop */
-	0x05605eb8,
-	0x7e7fb200,
-	0xb800008f,
-	0x0005885e,
-	0x7e082f95,
-	0xb800008f,
-	0x00058c5e,
-	0x7e082f95,
-	0xb800008f,
-	0x0005905e,
-	0x0000657e,
-	0xb606f5b6,
-	0xf4b601f0,
-	0x002fbb08,
-	0xb6003fbb,
-	0x62b60170,
-	0xbf1bf401,
-	0x080050b7,
-	0xf40142b6,
-	0x3f0fa81b,
-	0x501d608e,
-	0xb201e5f0,
-	0x008f7eff,
-	0x8e0d0f00,
-	0xf0501da8,
-	0xffb201e5,
-	0x00008f7e,
-	0x0003147e,
-	0x02010080,
-	0xbd0003f6,
-	0xf024bd04,
-	0x00801f29,
-	0x02f60230,
-/* 0x0577: main */
-	0xf404bd00,
-	0x28f40031,
-	0x7e240d00,
-	0xf4000037,
-	0xe4b0f401,
-	0x1d18f404,
-	0x020181fe,
-	0xfd20bd06,
-	0xe4b60412,
-	0x051efd01,
-	0x7e0018fe,
-	0xf400064a,
-/* 0x05a6: main_not_ctx_xfer */
-	0xef94d40e,
-	0x01f5f010,
-	0x0002f87e,
-/* 0x05b3: ih */
-	0xf9c70ef4,
-	0x0188fe80,
-	0x90f980f9,
-	0xb0f9a0f9,
-	0xe0f9d0f9,
-	0x04bdf0f9,
-	0xcf02004a,
-	0xabc400aa,
-	0x1f0bf404,
-	0x004e240d,
-	0x00eecf1a,
-	0xcf19004f,
-	0x047e00ff,
-	0x010e0000,
-	0xf61d0040,
-	0x04bd000e,
-/* 0x05f0: ih_no_fifo */
-	0xf6010040,
-	0x04bd000a,
-	0xe0fcf0fc,
-	0xb0fcd0fc,
-	0x90fca0fc,
-	0x88fe80fc,
-	0xf480fc00,
-	0x01f80032,
-/* 0x0610: hub_barrier_done */
-	0x0e98010f,
-	0x04febb04,
-	0x188effb2,
-	0x8f7e4094,
-	0x00f80000,
-/* 0x0624: ctx_redswitch */
-	0x0080200f,
-	0x0ff60185,
-	0x0e04bd00,
-/* 0x0631: ctx_redswitch_delay */
-	0x01e2b608,
-	0xf1fd1bf4,
-	0xf10800f5,
-	0x800200f5,
-	0xf6018500,
-	0x04bd000f,
-/* 0x064a: ctx_xfer */
-	0x008000f8,
-	0x0ff60281,
-	0x8e04bd00,
-	0xf0501dc4,
-	0xffb201e5,
-	0x00008f7e,
-	0x7e0711f4,
-/* 0x0667: ctx_xfer_not_load */
-	0x7e000624,
-	0xbd000216,
-	0x47fc8024,
-	0x0002f602,
-	0x2cf004bd,
-	0x0320b601,
-	0x024afc80,
-	0xbd0002f6,
-	0x8e0c0f04,
-	0xf0501da8,
-	0xffb201e5,
-	0x00008f7e,
-	0x0003147e,
+	0x55f05040,
+/* 0x04dd: tpc_strand_init_tpc_loop */
+	0x705eb801,
+	0x657e0005,
+	0xf6b20000,
+/* 0x04ea: tpc_strand_init_idx_loop */
+	0x5eb874bd,
+	0xb2000560,
+	0x008f7e7f,
+	0x885eb800,
+	0x2f950005,
+	0x008f7e08,
+	0x8c5eb800,
+	0x2f950005,
+	0x008f7e08,
+	0x905eb800,
+	0x657e0005,
+	0xf5b60000,
+	0x01f0b606,
+	0xbb08f4b6,
+	0x3fbb002f,
+	0x0170b600,
+	0xf40162b6,
+	0x50b7bf1b,
+	0x42b60800,
+	0xa81bf401,
 	0x608e3f0f,
 	0xe5f0501d,
 	0x7effb201,
 	0x0f00008f,
-	0x1d9c8e00,
+	0x1da88e0d,
 	0x01e5f050,
 	0x8f7effb2,
-	0x010f0000,
-	0x0003147e,
-	0xb601fcf0,
-	0xa88e03f0,
-	0xe5f0501d,
-	0x7effb201,
-	0xf000008f,
-	0xa5f001ac,
-	0x00008b02,
-	0x040c9850,
-	0xbb0fc4b6,
-	0x0c9800bc,
-	0x010d9800,
-	0x3d7e000e,
-	0xacf00001,
-	0x40008b01,
-	0x040c9850,
-	0xbb0fc4b6,
-	0x0c9800bc,
-	0x020d9801,
-	0x4e060f98,
-	0x3d7e0800,
-	0xacf00001,
-	0x04a5f001,
-	0x5030008b,
-	0xb6040c98,
-	0xbcbb0fc4,
-	0x020c9800,
-	0x98030d98,
-	0x004e080f,
-	0x013d7e02,
-	0x020a7e00,
-	0x03147e00,
-	0x0601f400,
-/* 0x073f: ctx_xfer_post */
-	0x7e1a12f4,
-	0x0f000227,
-	0x1da88e0d,
+	0x147e0000,
+	0x00800003,
+	0x03f60201,
+	0xbd04bd00,
+	0x1f29f024,
+	0x02300080,
+	0xbd0002f6,
+/* 0x0571: main */
+	0x0031f404,
+	0x0d0028f4,
+	0x00377e24,
+	0xf401f400,
+	0xf404e4b0,
+	0x81fe1d18,
+	0xbd060201,
+	0x0412fd20,
+	0xfd01e4b6,
+	0x18fe051e,
+	0x06447e00,
+	0xd40ef400,
+/* 0x05a0: main_not_ctx_xfer */
+	0xf010ef94,
+	0xf87e01f5,
+	0x0ef40002,
+/* 0x05ad: ih */
+	0xfe80f9c7,
+	0x80f90188,
+	0xa0f990f9,
+	0xd0f9b0f9,
+	0xf0f9e0f9,
+	0x004a04bd,
+	0x00aacf02,
+	0xf404abc4,
+	0x240d1f0b,
+	0xcf1a004e,
+	0x004f00ee,
+	0x00ffcf19,
+	0x0000047e,
+	0x0040010e,
+	0x000ef61d,
+/* 0x05ea: ih_no_fifo */
+	0x004004bd,
+	0x000af601,
+	0xf0fc04bd,
+	0xd0fce0fc,
+	0xa0fcb0fc,
+	0x80fc90fc,
+	0xfc0088fe,
+	0x0032f480,
+/* 0x060a: hub_barrier_done */
+	0x010f01f8,
+	0xbb040e98,
+	0xffb204fe,
+	0x4094188e,
+	0x00008f7e,
+/* 0x061e: ctx_redswitch */
+	0x200f00f8,
+	0x01850080,
+	0xbd000ff6,
+/* 0x062b: ctx_redswitch_delay */
+	0xb6080e04,
+	0x1bf401e2,
+	0x00f5f1fd,
+	0x00f5f108,
+	0x85008002,
+	0x000ff601,
+	0x00f804bd,
+/* 0x0644: ctx_xfer */
+	0x02810080,
+	0xbd000ff6,
+	0x1dc48e04,
+	0x01e5f050,
+	0x8f7effb2,
+	0x11f40000,
+	0x061e7e07,
+/* 0x0661: ctx_xfer_not_load */
+	0x02167e00,
+	0x8024bd00,
+	0xf60247fc,
+	0x04bd0002,
+	0xb6012cf0,
+	0xfc800320,
+	0x02f6024a,
+	0x0f04bd00,
+	0x1da88e0c,
 	0x01e5f050,
 	0x8f7effb2,
 	0x147e0000,
-/* 0x0756: ctx_xfer_done */
-	0x107e0003,
-	0x00f80006,
+	0x3f0f0003,
+	0x501d608e,
+	0xb201e5f0,
+	0x008f7eff,
+	0x8e000f00,
+	0xf0501d9c,
+	0xffb201e5,
+	0x00008f7e,
+	0x147e010f,
+	0xfcf00003,
+	0x03f0b601,
+	0x501da88e,
+	0xb201e5f0,
+	0x008f7eff,
+	0x01acf000,
+	0x8b02a5f0,
+	0x98500000,
+	0xc4b6040c,
+	0x00bcbb0f,
+	0x98000c98,
+	0x000e010d,
+	0x00013d7e,
+	0x8b01acf0,
+	0x98504000,
+	0xc4b6040c,
+	0x00bcbb0f,
+	0x98010c98,
+	0x0f98020d,
+	0x08004e06,
+	0x00013d7e,
+	0xf001acf0,
+	0x008b04a5,
+	0x0c985030,
+	0x0fc4b604,
+	0x9800bcbb,
+	0x0d98020c,
+	0x080f9803,
+	0x7e02004e,
+	0x7e00013d,
+	0x7e00020a,
+	0xf4000314,
+	0x12f40601,
+/* 0x0739: ctx_xfer_post */
+	0x02277e1a,
+	0x8e0d0f00,
+	0xf0501da8,
+	0xffb201e5,
+	0x00008f7e,
+	0x0003147e,
+/* 0x0750: ctx_xfer_done */
+	0x00060a7e,
+	0x000000f8,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
index 9f5dfc85147a..1f81069edc58 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
@@ -34,6 +34,7 @@
 #include <engine/fifo.h>
 
 #include <nvif/class.h>
+#include <nvif/cl9097.h>
 #include <nvif/unpack.h>
 
 /*******************************************************************************
@@ -139,6 +140,12 @@ gf100_gr_zbc_depth_get(struct gf100_gr *gr, int format,
 /*******************************************************************************
  * Graphics object classes
  ******************************************************************************/
+#define gf100_gr_object(p) container_of((p), struct gf100_gr_object, object)
+
+struct gf100_gr_object {
+	struct nvkm_object object;
+	struct gf100_gr_chan *chan;
+};
 
 static int
 gf100_fermi_mthd_zbc_color(struct nvkm_object *object, void *data, u32 size)
@@ -147,9 +154,9 @@ gf100_fermi_mthd_zbc_color(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct fermi_a_zbc_color_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		switch (args->v0.format) {
 		case FERMI_A_ZBC_COLOR_V0_FMT_ZERO:
 		case FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE:
@@ -193,9 +200,9 @@ gf100_fermi_mthd_zbc_depth(struct nvkm_object *object, void *data, u32 size)
 	union {
 		struct fermi_a_zbc_depth_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		switch (args->v0.format) {
 		case FERMI_A_ZBC_DEPTH_V0_FMT_FP32:
 			ret = gf100_gr_zbc_depth_get(gr, args->v0.format,
@@ -213,6 +220,7 @@ gf100_fermi_mthd_zbc_depth(struct nvkm_object *object, void *data, u32 size)
 static int
 gf100_fermi_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 {
+	nvif_ioctl(object, "fermi mthd %08x\n", mthd);
 	switch (mthd) {
 	case FERMI_A_ZBC_COLOR:
 		return gf100_fermi_mthd_zbc_color(object, data, size);
@@ -256,6 +264,27 @@ gf100_gr_mthd_sw(struct nvkm_device *device, u16 class, u32 mthd, u32 data)
 	return false;
 }
 
+static const struct nvkm_object_func
+gf100_gr_object_func = {
+};
+
+static int
+gf100_gr_object_new(const struct nvkm_oclass *oclass, void *data, u32 size,
+		    struct nvkm_object **pobject)
+{
+	struct gf100_gr_chan *chan = gf100_gr_chan(oclass->parent);
+	struct gf100_gr_object *object;
+
+	if (!(object = kzalloc(sizeof(*object), GFP_KERNEL)))
+		return -ENOMEM;
+	*pobject = &object->object;
+
+	nvkm_object_ctor(oclass->base.func ? oclass->base.func :
+			 &gf100_gr_object_func, oclass, &object->object);
+	object->chan = chan;
+	return 0;
+}
+
 static int
 gf100_gr_object_get(struct nvkm_gr *base, int index, struct nvkm_sclass *sclass)
 {
@@ -265,6 +294,7 @@ gf100_gr_object_get(struct nvkm_gr *base, int index, struct nvkm_sclass *sclass)
 	while (gr->func->sclass[c].oclass) {
 		if (c++ == index) {
 			*sclass = gr->func->sclass[index];
+			sclass->ctor = gf100_gr_object_new;
 			return index;
 		}
 	}
@@ -826,7 +856,41 @@ gf100_gr_units(struct nvkm_gr *base)
 	return cfg;
 }
 
+static const struct nvkm_bitfield gf100_dispatch_error[] = {
+	{ 0x00000001, "INJECTED_BUNDLE_ERROR" },
+	{ 0x00000002, "CLASS_SUBCH_MISMATCH" },
+	{ 0x00000004, "SUBCHSW_DURING_NOTIFY" },
+	{}
+};
+
+static const struct nvkm_bitfield gf100_m2mf_error[] = {
+	{ 0x00000001, "PUSH_TOO_MUCH_DATA" },
+	{ 0x00000002, "PUSH_NOT_ENOUGH_DATA" },
+	{}
+};
+
+static const struct nvkm_bitfield gf100_unk6_error[] = {
+	{ 0x00000001, "TEMP_TOO_SMALL" },
+	{}
+};
+
+static const struct nvkm_bitfield gf100_ccache_error[] = {
+	{ 0x00000001, "INTR" },
+	{ 0x00000002, "LDCONST_OOB" },
+	{}
+};
+
+static const struct nvkm_bitfield gf100_macro_error[] = {
+	{ 0x00000001, "TOO_FEW_PARAMS" },
+	{ 0x00000002, "TOO_MANY_PARAMS" },
+	{ 0x00000004, "ILLEGAL_OPCODE" },
+	{ 0x00000008, "DOUBLE_BRANCH" },
+	{ 0x00000010, "WATCHDOG" },
+	{}
+};
+
 static const struct nvkm_bitfield gk104_sked_error[] = {
+	{ 0x00000040, "CTA_RESUME" },
 	{ 0x00000080, "CONSTANT_BUFFER_SIZE" },
 	{ 0x00000200, "LOCAL_MEMORY_SIZE_POS" },
 	{ 0x00000400, "LOCAL_MEMORY_SIZE_NEG" },
@@ -836,6 +900,8 @@ static const struct nvkm_bitfield gk104_sked_error[] = {
 	{ 0x00040000, "TOTAL_THREADS" },
 	{ 0x00100000, "PROGRAM_OFFSET" },
 	{ 0x00200000, "SHARED_MEMORY_SIZE" },
+	{ 0x00800000, "CTA_THREAD_DIMENSION_ZERO" },
+	{ 0x01000000, "MEMORY_WINDOW_OVERLAP" },
 	{ 0x02000000, "SHARED_CONFIG_TOO_SMALL" },
 	{ 0x04000000, "TOTAL_REGISTER_COUNT" },
 	{}
@@ -1005,12 +1071,16 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 {
 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
+	char error[128];
 	u32 trap = nvkm_rd32(device, 0x400108);
 	int rop, gpc;
 
 	if (trap & 0x00000001) {
 		u32 stat = nvkm_rd32(device, 0x404000);
-		nvkm_error(subdev, "DISPATCH %08x\n", stat);
+
+		nvkm_snprintbf(error, sizeof(error), gf100_dispatch_error,
+			       stat & 0x3fffffff);
+		nvkm_error(subdev, "DISPATCH %08x [%s]\n", stat, error);
 		nvkm_wr32(device, 0x404000, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000001);
 		trap &= ~0x00000001;
@@ -1018,7 +1088,11 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000002) {
 		u32 stat = nvkm_rd32(device, 0x404600);
-		nvkm_error(subdev, "M2MF %08x\n", stat);
+
+		nvkm_snprintbf(error, sizeof(error), gf100_m2mf_error,
+			       stat & 0x3fffffff);
+		nvkm_error(subdev, "M2MF %08x [%s]\n", stat, error);
+
 		nvkm_wr32(device, 0x404600, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000002);
 		trap &= ~0x00000002;
@@ -1026,7 +1100,10 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000008) {
 		u32 stat = nvkm_rd32(device, 0x408030);
-		nvkm_error(subdev, "CCACHE %08x\n", stat);
+
+		nvkm_snprintbf(error, sizeof(error), gf100_m2mf_error,
+			       stat & 0x3fffffff);
+		nvkm_error(subdev, "CCACHE %08x [%s]\n", stat, error);
 		nvkm_wr32(device, 0x408030, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000008);
 		trap &= ~0x00000008;
@@ -1034,7 +1111,8 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000010) {
 		u32 stat = nvkm_rd32(device, 0x405840);
-		nvkm_error(subdev, "SHADER %08x\n", stat);
+		nvkm_error(subdev, "SHADER %08x, sph: 0x%06x, stage: 0x%02x\n",
+			   stat, stat & 0xffffff, (stat >> 24) & 0x3f);
 		nvkm_wr32(device, 0x405840, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000010);
 		trap &= ~0x00000010;
@@ -1042,7 +1120,11 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000040) {
 		u32 stat = nvkm_rd32(device, 0x40601c);
-		nvkm_error(subdev, "UNK6 %08x\n", stat);
+
+		nvkm_snprintbf(error, sizeof(error), gf100_unk6_error,
+			       stat & 0x3fffffff);
+		nvkm_error(subdev, "UNK6 %08x [%s]\n", stat, error);
+
 		nvkm_wr32(device, 0x40601c, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000040);
 		trap &= ~0x00000040;
@@ -1050,7 +1132,16 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000080) {
 		u32 stat = nvkm_rd32(device, 0x404490);
-		nvkm_error(subdev, "MACRO %08x\n", stat);
+		u32 pc = nvkm_rd32(device, 0x404494);
+		u32 op = nvkm_rd32(device, 0x40449c);
+
+		nvkm_snprintbf(error, sizeof(error), gf100_macro_error,
+			       stat & 0x1fffffff);
+		nvkm_error(subdev, "MACRO %08x [%s], pc: 0x%03x%s, op: 0x%08x\n",
+			   stat, error, pc & 0x7ff,
+			   (pc & 0x10000000) ? "" : " (invalid)",
+			   op);
+
 		nvkm_wr32(device, 0x404490, 0xc0000000);
 		nvkm_wr32(device, 0x400108, 0x00000080);
 		trap &= ~0x00000080;
@@ -1058,10 +1149,9 @@ gf100_gr_trap_intr(struct gf100_gr *gr)
 
 	if (trap & 0x00000100) {
 		u32 stat = nvkm_rd32(device, 0x407020) & 0x3fffffff;
-		char sked[128];
 
-		nvkm_snprintbf(sked, sizeof(sked), gk104_sked_error, stat);
-		nvkm_error(subdev, "SKED: %08x [%s]\n", stat, sked);
+		nvkm_snprintbf(error, sizeof(error), gk104_sked_error, stat);
+		nvkm_error(subdev, "SKED: %08x [%s]\n", stat, error);
 
 		if (stat)
 			nvkm_wr32(device, 0x407020, 0x40000000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
index 2721592d3031..f19fabef8d73 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
@@ -27,6 +27,8 @@
 #include <core/option.h>
 
 #include <nvif/class.h>
+#include <nvif/if0002.h>
+#include <nvif/if0003.h>
 #include <nvif/ioctl.h>
 #include <nvif/unpack.h>
 
@@ -210,10 +212,10 @@ nvkm_perfdom_init(struct nvkm_perfdom *dom, void *data, u32 size)
 	} *args = data;
 	struct nvkm_object *object = &dom->object;
 	struct nvkm_pm *pm = dom->perfmon->pm;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(object, "perfdom init size %d\n", size);
-	if (nvif_unvers(args->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		nvif_ioctl(object, "perfdom init\n");
 	} else
 		return ret;
@@ -240,10 +242,10 @@ nvkm_perfdom_sample(struct nvkm_perfdom *dom, void *data, u32 size)
 	} *args = data;
 	struct nvkm_object *object = &dom->object;
 	struct nvkm_pm *pm = dom->perfmon->pm;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(object, "perfdom sample size %d\n", size);
-	if (nvif_unvers(args->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		nvif_ioctl(object, "perfdom sample\n");
 	} else
 		return ret;
@@ -264,10 +266,10 @@ nvkm_perfdom_read(struct nvkm_perfdom *dom, void *data, u32 size)
 	} *args = data;
 	struct nvkm_object *object = &dom->object;
 	struct nvkm_pm *pm = dom->perfmon->pm;
-	int ret, i;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(object, "perfdom read size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "perfdom read vers %d\n", args->v0.version);
 	} else
 		return ret;
@@ -374,10 +376,10 @@ nvkm_perfdom_new_(struct nvkm_perfmon *perfmon,
 	struct nvkm_perfctr *ctr[4] = {};
 	struct nvkm_perfdom *dom;
 	int c, s, m;
-	int ret;
+	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create perfdom size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create perfdom vers %d dom %d mode %02x\n",
 			   args->v0.version, args->v0.domain, args->v0.mode);
 	} else
@@ -439,10 +441,10 @@ nvkm_perfmon_mthd_query_domain(struct nvkm_perfmon *perfmon,
 	struct nvkm_pm *pm = perfmon->pm;
 	struct nvkm_perfdom *dom;
 	u8 domain_nr;
-	int di, ret;
+	int di, ret = -ENOSYS;
 
 	nvif_ioctl(object, "perfmon query domain size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "perfmon domain vers %d iter %02x\n",
 			   args->v0.version, args->v0.iter);
 		di = (args->v0.iter & 0xff) - 1;
@@ -490,10 +492,10 @@ nvkm_perfmon_mthd_query_signal(struct nvkm_perfmon *perfmon,
 	struct nvkm_perfsig *sig;
 	const bool all = nvkm_boolopt(device->cfgopt, "NvPmShowAll", false);
 	const bool raw = nvkm_boolopt(device->cfgopt, "NvPmUnnamed", all);
-	int ret, si;
+	int ret = -ENOSYS, si;
 
 	nvif_ioctl(object, "perfmon query signal size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object,
 			   "perfmon query signal vers %d dom %d iter %04x\n",
 			   args->v0.version, args->v0.domain, args->v0.iter);
@@ -543,10 +545,10 @@ nvkm_perfmon_mthd_query_source(struct nvkm_perfmon *perfmon,
 	struct nvkm_perfsig *sig;
 	struct nvkm_perfsrc *src;
 	u8 source_nr = 0;
-	int si, ret;
+	int si, ret = -ENOSYS;
 
 	nvif_ioctl(object, "perfmon query source size %d\n", size);
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object,
 			   "perfmon source vers %d dom %d sig %02x iter %02x\n",
 			   args->v0.version, args->v0.domain, args->v0.signal,
@@ -612,7 +614,7 @@ nvkm_perfmon_child_get(struct nvkm_object *object, int index,
 		       struct nvkm_oclass *oclass)
 {
 	if (index == 0) {
-		oclass->base.oclass = NVIF_IOCTL_NEW_V0_PERFDOM;
+		oclass->base.oclass = NVIF_CLASS_PERFDOM;
 		oclass->base.minver = 0;
 		oclass->base.maxver = 0;
 		oclass->ctor = nvkm_perfmon_child_new;
@@ -679,7 +681,7 @@ nvkm_pm_oclass_new(struct nvkm_device *device, const struct nvkm_oclass *oclass,
 
 static const struct nvkm_device_oclass
 nvkm_pm_oclass = {
-	.base.oclass = NVIF_IOCTL_NEW_V0_PERFMON,
+	.base.oclass = NVIF_CLASS_PERFMON,
 	.base.minver = -1,
 	.base.maxver = -1,
 	.ctor = nvkm_pm_oclass_new,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c
index d082f4f73a80..f28967065639 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c
@@ -53,9 +53,9 @@ nvkm_sw_chan_event_ctor(struct nvkm_object *object, void *data, u32 size,
 	union {
 		struct nvif_notify_uevent_req none;
 	} *req = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unvers(req->none)) {
+	if (!(ret = nvif_unvers(ret, &data, &size, req->none))) {
 		notify->size  = sizeof(struct nvif_notify_uevent_rep);
 		notify->types = 1;
 		notify->index = 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c
index b01ef7eca906..ea8f4247b628 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c
@@ -28,8 +28,8 @@
 #include <engine/disp.h>
 #include <engine/fifo.h>
 
+#include <nvif/class.h>
 #include <nvif/event.h>
-#include <nvif/ioctl.h>
 
 /*******************************************************************************
  * software context
@@ -143,7 +143,7 @@ static const struct nvkm_sw_func
 gf100_sw = {
 	.chan_new = gf100_sw_chan_new,
 	.sclass = {
-		{ nvkm_nvsw_new, { -1, -1, NVIF_IOCTL_NEW_V0_SW_GF100 } },
+		{ nvkm_nvsw_new, { -1, -1, NVIF_CLASS_SW_GF100 } },
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv04.c
index 445217ffa791..b6675fe1b0ce 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv04.c
@@ -27,6 +27,7 @@
 #include "nvsw.h"
 
 #include <nvif/class.h>
+#include <nvif/if0004.h>
 #include <nvif/ioctl.h>
 #include <nvif/unpack.h>
 
@@ -46,9 +47,9 @@ nv04_nvsw_mthd_get_ref(struct nvkm_nvsw *nvsw, void *data, u32 size)
 	union {
 		struct nv04_nvsw_get_ref_v0 v0;
 	} *args = data;
-	int ret;
+	int ret = -ENOSYS;
 
-	if (nvif_unpack(args->v0, 0, 0, false)) {
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		args->v0.ref = atomic_read(&chan->ref);
 	}
 
@@ -126,7 +127,7 @@ static const struct nvkm_sw_func
 nv04_sw = {
 	.chan_new = nv04_sw_chan_new,
 	.sclass = {
-		{ nv04_nvsw_new, { -1, -1, NVIF_IOCTL_NEW_V0_SW_NV04 } },
+		{ nv04_nvsw_new, { -1, -1, NVIF_CLASS_SW_NV04 } },
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv10.c
index adf70d92b244..09d22fcd194c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv10.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv10.c
@@ -25,7 +25,7 @@
 #include "chan.h"
 #include "nvsw.h"
 
-#include <nvif/ioctl.h>
+#include <nvif/class.h>
 
 /*******************************************************************************
  * software context
@@ -56,7 +56,7 @@ static const struct nvkm_sw_func
 nv10_sw = {
 	.chan_new = nv10_sw_chan_new,
 	.sclass = {
-		{ nvkm_nvsw_new, { -1, -1, NVIF_IOCTL_NEW_V0_SW_NV10 } },
+		{ nvkm_nvsw_new, { -1, -1, NVIF_CLASS_SW_NV10 } },
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c
index a381196af69d..01573d187f2c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c
@@ -28,8 +28,8 @@
 #include <engine/fifo/chan.h>
 #include <subdev/bar.h>
 
+#include <nvif/class.h>
 #include <nvif/event.h>
-#include <nvif/ioctl.h>
 
 /*******************************************************************************
  * software context
@@ -136,7 +136,7 @@ static const struct nvkm_sw_func
 nv50_sw = {
 	.chan_new = nv50_sw_chan_new,
 	.sclass = {
-		{ nvkm_nvsw_new, { -1, -1, NVIF_IOCTL_NEW_V0_SW_NV50 } },
+		{ nvkm_nvsw_new, { -1, -1, NVIF_CLASS_SW_NV50 } },
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c
index 66cf986b9572..33dd03fff3c4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c
@@ -24,7 +24,7 @@
 #include "nvsw.h"
 #include "chan.h"
 
-#include <nvif/class.h>
+#include <nvif/if0004.h>
 
 static int
 nvkm_nvsw_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
@@ -41,7 +41,7 @@ nvkm_nvsw_ntfy_(struct nvkm_object *object, u32 mthd,
 {
 	struct nvkm_nvsw *nvsw = nvkm_nvsw(object);
 	switch (mthd) {
-	case NVSW_NTFY_UEVENT:
+	case NV04_NVSW_NTFY_UEVENT:
 		*pevent = &nvsw->chan->event;
 		return 0;
 	default:
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dcb.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dcb.c
index 8304b806f2a6..a8d5d67feeaf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dcb.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dcb.c
@@ -143,16 +143,19 @@ dcb_outp_parse(struct nvkm_bios *bios, u8 idx, u8 *ver, u8 *len,
 			switch (outp->type) {
 			case DCB_OUTPUT_DP:
 				switch (conf & 0x00e00000) {
-				case 0x00000000:
+				case 0x00000000: /* 1.62 */
 					outp->dpconf.link_bw = 0x06;
 					break;
-				case 0x00200000:
+				case 0x00200000: /* 2.7 */
 					outp->dpconf.link_bw = 0x0a;
 					break;
-				case 0x00400000:
-				default:
+				case 0x00400000: /* 5.4 */
 					outp->dpconf.link_bw = 0x14;
 					break;
+				case 0x00600000: /* 8.1 */
+				default:
+					outp->dpconf.link_bw = 0x1e;
+					break;
 				}
 
 				switch ((conf & 0x0f000000) >> 24) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
index aa7e33b42b30..636bfb665bb9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
@@ -24,6 +24,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/perf.h>
+#include <subdev/pci.h>
 
 u16
 nvbios_perf_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr,
@@ -145,6 +146,21 @@ nvbios_perfEp(struct nvkm_bios *bios, int idx,
 		break;
 	case 0x40:
 		info->voltage  = nvbios_rd08(bios, perf + 0x02);
+		switch (nvbios_rd08(bios, perf + 0xb) & 0x3) {
+		case 0:
+			info->pcie_speed = NVKM_PCIE_SPEED_5_0;
+			break;
+		case 3:
+		case 1:
+			info->pcie_speed = NVKM_PCIE_SPEED_2_5;
+			break;
+		case 2:
+			info->pcie_speed = NVKM_PCIE_SPEED_8_0;
+			break;
+		default:
+			break;
+		}
+		info->pcie_width = 0xff;
 		break;
 	default:
 		return 0x0000;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
index dc8682c91cc7..889cce2eb727 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
@@ -176,6 +176,7 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 {
 	struct nvkm_subdev *subdev = &clk->subdev;
 	struct nvkm_ram *ram = subdev->device->fb->ram;
+	struct nvkm_pci *pci = subdev->device->pci;
 	struct nvkm_pstate *pstate;
 	int ret, idx = 0;
 
@@ -187,6 +188,8 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 	nvkm_debug(subdev, "setting performance state %d\n", pstatei);
 	clk->pstate = pstatei;
 
+	nvkm_pcie_set_link(pci, pstate->pcie_speed, pstate->pcie_width);
+
 	if (ram && ram->func->calc) {
 		int khz = pstate->base.domain[nv_clk_src_mem];
 		do {
@@ -330,6 +333,8 @@ nvkm_pstate_new(struct nvkm_clk *clk, int idx)
 
 	pstate->pstate = perfE.pstate;
 	pstate->fanspeed = perfE.fanspeed;
+	pstate->pcie_speed = perfE.pcie_speed;
+	pstate->pcie_width = perfE.pcie_width;
 	cstate->voltage = perfE.voltage;
 	cstate->domain[nv_clk_src_core] = perfE.core;
 	cstate->domain[nv_clk_src_shader] = perfE.shader;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
index a52b7e7fce41..78c449b417b7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
@@ -188,7 +188,7 @@ gf100_clk_read(struct nvkm_clk *base, enum nv_clk_src src)
 		return read_clk(clk, 0x08);
 	case nv_clk_src_copy:
 		return read_clk(clk, 0x09);
-	case nv_clk_src_daemon:
+	case nv_clk_src_pmu:
 		return read_clk(clk, 0x0c);
 	case nv_clk_src_vdec:
 		return read_clk(clk, 0x0e);
@@ -325,7 +325,7 @@ gf100_clk_calc(struct nvkm_clk *base, struct nvkm_cstate *cstate)
 	    (ret = calc_clk(clk, cstate, 0x07, nv_clk_src_hubk06)) ||
 	    (ret = calc_clk(clk, cstate, 0x08, nv_clk_src_hubk01)) ||
 	    (ret = calc_clk(clk, cstate, 0x09, nv_clk_src_copy)) ||
-	    (ret = calc_clk(clk, cstate, 0x0c, nv_clk_src_daemon)) ||
+	    (ret = calc_clk(clk, cstate, 0x0c, nv_clk_src_pmu)) ||
 	    (ret = calc_clk(clk, cstate, 0x0e, nv_clk_src_vdec)))
 		return ret;
 
@@ -447,7 +447,7 @@ gf100_clk = {
 		{ nv_clk_src_rop    , 0x04 },
 		{ nv_clk_src_mem    , 0x05, 0, "memory", 1000 },
 		{ nv_clk_src_vdec   , 0x06 },
-		{ nv_clk_src_daemon , 0x0a },
+		{ nv_clk_src_pmu    , 0x0a },
 		{ nv_clk_src_hubk07 , 0x0b },
 		{ nv_clk_src_max }
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
index 396f7e4dad0a..975c401bccab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
@@ -209,7 +209,7 @@ gk104_clk_read(struct nvkm_clk *base, enum nv_clk_src src)
 		return read_clk(clk, 0x07);
 	case nv_clk_src_hubk01:
 		return read_clk(clk, 0x08);
-	case nv_clk_src_daemon:
+	case nv_clk_src_pmu:
 		return read_clk(clk, 0x0c);
 	case nv_clk_src_vdec:
 		return read_clk(clk, 0x0e);
@@ -346,7 +346,7 @@ gk104_clk_calc(struct nvkm_clk *base, struct nvkm_cstate *cstate)
 	    (ret = calc_clk(clk, cstate, 0x02, nv_clk_src_hubk07)) ||
 	    (ret = calc_clk(clk, cstate, 0x07, nv_clk_src_hubk06)) ||
 	    (ret = calc_clk(clk, cstate, 0x08, nv_clk_src_hubk01)) ||
-	    (ret = calc_clk(clk, cstate, 0x0c, nv_clk_src_daemon)) ||
+	    (ret = calc_clk(clk, cstate, 0x0c, nv_clk_src_pmu)) ||
 	    (ret = calc_clk(clk, cstate, 0x0e, nv_clk_src_vdec)))
 		return ret;
 
@@ -492,7 +492,7 @@ gk104_clk = {
 		{ nv_clk_src_hubk06 , 0x04, NVKM_CLK_DOM_FLAG_CORE },
 		{ nv_clk_src_hubk01 , 0x05 },
 		{ nv_clk_src_vdec   , 0x06 },
-		{ nv_clk_src_daemon , 0x07 },
+		{ nv_clk_src_pmu    , 0x07 },
 		{ nv_clk_src_max }
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
index 254094ab7fb8..5da2aa8cc333 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
@@ -141,9 +141,8 @@ gk20a_pllg_calc_rate(struct gk20a_clk *clk)
 
 	rate = clk->parent_rate * clk->n;
 	divider = clk->m * pl_to_div[clk->pl];
-	do_div(rate, divider);
 
-	return rate / 2;
+	return rate / divider / 2;
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
index c233e3f653ce..056702ef69aa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
@@ -158,7 +158,7 @@ gt215_clk_read(struct nvkm_clk *base, enum nv_clk_src src)
 		return read_clk(clk, 0x20, false);
 	case nv_clk_src_vdec:
 		return read_clk(clk, 0x21, false);
-	case nv_clk_src_daemon:
+	case nv_clk_src_pmu:
 		return read_clk(clk, 0x25, false);
 	case nv_clk_src_host:
 		hsrc = (nvkm_rd32(device, 0xc040) & 0x30000000) >> 28;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
index f5edfadb5b46..1b5fb02eab2a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
@@ -132,7 +132,7 @@ static const struct nvkm_enum vm_engine[] = {
 	{ 0x0000000b, "PCOUNTER" },
 	{ 0x0000000c, "SEMAPHORE_BG" },
 	{ 0x0000000d, "PCE0" },
-	{ 0x0000000e, "PDAEMON" },
+	{ 0x0000000e, "PMU" },
 	{}
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
index 9df45030ff9f..1fa3ade468ae 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
@@ -216,11 +216,11 @@ r1373f4_fini(struct gk104_ramfuc *fuc)
 	ram_wr32(fuc, 0x1373ec, tmp | (v1 << 16));
 	ram_mask(fuc, 0x1373f0, (~ram->mode & 3), 0x00000000);
 	if (ram->mode == 2) {
-		ram_mask(fuc, 0x1373f4, 0x00000003, 0x000000002);
-		ram_mask(fuc, 0x1373f4, 0x00001100, 0x000000000);
+		ram_mask(fuc, 0x1373f4, 0x00000003, 0x00000002);
+		ram_mask(fuc, 0x1373f4, 0x00001100, 0x00000000);
 	} else {
-		ram_mask(fuc, 0x1373f4, 0x00000003, 0x000000001);
-		ram_mask(fuc, 0x1373f4, 0x00010000, 0x000000000);
+		ram_mask(fuc, 0x1373f4, 0x00000003, 0x00000001);
+		ram_mask(fuc, 0x1373f4, 0x00010000, 0x00000000);
 	}
 	ram_mask(fuc, 0x10f800, 0x00000030, (v0 ^ v1) << 4);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/Kbuild
index de888fa62b3e..7e77a7466992 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/Kbuild
@@ -2,3 +2,4 @@ nvkm-y += nvkm/subdev/ibus/gf100.o
 nvkm-y += nvkm/subdev/ibus/gf117.o
 nvkm-y += nvkm/subdev/ibus/gk104.o
 nvkm-y += nvkm/subdev/ibus/gk20a.o
+nvkm-y += nvkm/subdev/ibus/gm204.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
index ba33609f643c..b5cee3f89aaa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk104.c
@@ -21,7 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include <subdev/ibus.h>
+#include "priv.h"
 
 static void
 gk104_ibus_intr_hub(struct nvkm_subdev *ibus, int i)
@@ -56,7 +56,7 @@ gk104_ibus_intr_gpc(struct nvkm_subdev *ibus, int i)
 	nvkm_mask(device, 0x128128 + (i * 0x0800), 0x00000200, 0x00000000);
 }
 
-static void
+void
 gk104_ibus_intr(struct nvkm_subdev *ibus)
 {
 	struct nvkm_device *device = ibus->device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gm204.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gm204.c
new file mode 100644
index 000000000000..b3839dc254ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gm204.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+static const struct nvkm_subdev_func
+gm204_ibus = {
+	.intr = gk104_ibus_intr,
+};
+
+int
+gm204_ibus_new(struct nvkm_device *device, int index,
+	       struct nvkm_subdev **pibus)
+{
+	struct nvkm_subdev *ibus;
+	if (!(ibus = *pibus = kzalloc(sizeof(*ibus), GFP_KERNEL)))
+		return -ENOMEM;
+	nvkm_subdev_ctor(&gm204_ibus, device, index, 0, ibus);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/priv.h
index 48e1b6365ce6..01caf798cf31 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/priv.h
@@ -4,4 +4,5 @@
 #include <subdev/ibus.h>
 
 void gf100_ibus_intr(struct nvkm_subdev *);
+void gk104_ibus_intr(struct nvkm_subdev *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
index 14107b5b7811..4c20fec64d96 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
@@ -56,7 +56,6 @@ struct gk20a_instobj {
 
 	/* CPU mapping */
 	u32 *vaddr;
-	struct list_head vaddr_node;
 };
 #define gk20a_instobj(p) container_of((p), struct gk20a_instobj, memory)
 
@@ -66,7 +65,6 @@ struct gk20a_instobj {
 struct gk20a_instobj_dma {
 	struct gk20a_instobj base;
 
-	u32 *cpuaddr;
 	dma_addr_t handle;
 	struct nvkm_mm_node r;
 };
@@ -79,6 +77,11 @@ struct gk20a_instobj_dma {
 struct gk20a_instobj_iommu {
 	struct gk20a_instobj base;
 
+	/* to link into gk20a_instmem::vaddr_lru */
+	struct list_head vaddr_node;
+	/* how many clients are using vaddr? */
+	u32 use_cpt;
+
 	/* will point to the higher half of pages */
 	dma_addr_t *dma_addrs;
 	/* array of base.mem->size pages (+ dma_addr_ts) */
@@ -107,8 +110,6 @@ struct gk20a_instmem {
 
 	/* Only used by DMA API */
 	struct dma_attrs attrs;
-
-	void __iomem * (*cpu_map)(struct nvkm_memory *);
 };
 #define gk20a_instmem(p) container_of((p), struct gk20a_instmem, base)
 
@@ -130,70 +131,58 @@ gk20a_instobj_size(struct nvkm_memory *memory)
 	return (u64)gk20a_instobj(memory)->mem.size << 12;
 }
 
-static void __iomem *
-gk20a_instobj_cpu_map_dma(struct nvkm_memory *memory)
-{
-#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-	struct gk20a_instobj_dma *node = gk20a_instobj_dma(memory);
-	struct device *dev = node->base.imem->base.subdev.device->dev;
-	int npages = nvkm_memory_size(memory) >> 12;
-	struct page *pages[npages];
-	int i;
-
-	/* we shouldn't see a gk20a on anything but arm/arm64 anyways */
-	/* phys_to_page does not exist on all platforms... */
-	pages[0] = pfn_to_page(dma_to_phys(dev, node->handle) >> PAGE_SHIFT);
-	for (i = 1; i < npages; i++)
-		pages[i] = pages[0] + i;
-
-	return vmap(pages, npages, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
-#else
-	BUG();
-	return NULL;
-#endif
-}
-
-static void __iomem *
-gk20a_instobj_cpu_map_iommu(struct nvkm_memory *memory)
+/*
+ * Recycle the vaddr of obj. Must be called with gk20a_instmem::lock held.
+ */
+static void
+gk20a_instobj_iommu_recycle_vaddr(struct gk20a_instobj_iommu *obj)
 {
-	struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
-	int npages = nvkm_memory_size(memory) >> 12;
-
-	return vmap(node->pages, npages, VM_MAP,
-		    pgprot_writecombine(PAGE_KERNEL));
+	struct gk20a_instmem *imem = obj->base.imem;
+	/* there should not be any user left... */
+	WARN_ON(obj->use_cpt);
+	list_del(&obj->vaddr_node);
+	vunmap(obj->base.vaddr);
+	obj->base.vaddr = NULL;
+	imem->vaddr_use -= nvkm_memory_size(&obj->base.memory);
+	nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", imem->vaddr_use,
+		   imem->vaddr_max);
 }
 
 /*
- * Must be called while holding gk20a_instmem_lock
+ * Must be called while holding gk20a_instmem::lock
  */
 static void
 gk20a_instmem_vaddr_gc(struct gk20a_instmem *imem, const u64 size)
 {
 	while (imem->vaddr_use + size > imem->vaddr_max) {
-		struct gk20a_instobj *obj;
-
 		/* no candidate that can be unmapped, abort... */
 		if (list_empty(&imem->vaddr_lru))
 			break;
 
-		obj = list_first_entry(&imem->vaddr_lru, struct gk20a_instobj,
-				       vaddr_node);
-		list_del(&obj->vaddr_node);
-		vunmap(obj->vaddr);
-		obj->vaddr = NULL;
-		imem->vaddr_use -= nvkm_memory_size(&obj->memory);
-		nvkm_debug(&imem->base.subdev, "(GC) vaddr used: %x/%x\n",
-			   imem->vaddr_use, imem->vaddr_max);
-
+		gk20a_instobj_iommu_recycle_vaddr(
+				list_first_entry(&imem->vaddr_lru,
+				struct gk20a_instobj_iommu, vaddr_node));
 	}
 }
 
 static void __iomem *
-gk20a_instobj_acquire(struct nvkm_memory *memory)
+gk20a_instobj_acquire_dma(struct nvkm_memory *memory)
 {
 	struct gk20a_instobj *node = gk20a_instobj(memory);
 	struct gk20a_instmem *imem = node->imem;
 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
+
+	nvkm_ltc_flush(ltc);
+
+	return node->vaddr;
+}
+
+static void __iomem *
+gk20a_instobj_acquire_iommu(struct nvkm_memory *memory)
+{
+	struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
+	struct gk20a_instmem *imem = node->base.imem;
+	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
 	const u64 size = nvkm_memory_size(memory);
 	unsigned long flags;
 
@@ -201,19 +190,21 @@ gk20a_instobj_acquire(struct nvkm_memory *memory)
 
 	spin_lock_irqsave(&imem->lock, flags);
 
-	if (node->vaddr) {
-		/* remove us from the LRU list since we cannot be unmapped */
-		list_del(&node->vaddr_node);
-
+	if (node->base.vaddr) {
+		if (!node->use_cpt) {
+			/* remove from LRU list since mapping in use again */
+			list_del(&node->vaddr_node);
+		}
 		goto out;
 	}
 
 	/* try to free some address space if we reached the limit */
 	gk20a_instmem_vaddr_gc(imem, size);
 
-	node->vaddr = imem->cpu_map(memory);
-
-	if (!node->vaddr) {
+	/* map the pages */
+	node->base.vaddr = vmap(node->pages, size >> PAGE_SHIFT, VM_MAP,
+				pgprot_writecombine(PAGE_KERNEL));
+	if (!node->base.vaddr) {
 		nvkm_error(&imem->base.subdev, "cannot map instobj - "
 			   "this is not going to end well...\n");
 		goto out;
@@ -224,24 +215,41 @@ gk20a_instobj_acquire(struct nvkm_memory *memory)
 		   imem->vaddr_use, imem->vaddr_max);
 
 out:
+	node->use_cpt++;
 	spin_unlock_irqrestore(&imem->lock, flags);
 
-	return node->vaddr;
+	return node->base.vaddr;
 }
 
 static void
-gk20a_instobj_release(struct nvkm_memory *memory)
+gk20a_instobj_release_dma(struct nvkm_memory *memory)
 {
 	struct gk20a_instobj *node = gk20a_instobj(memory);
 	struct gk20a_instmem *imem = node->imem;
 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
+
+	nvkm_ltc_invalidate(ltc);
+}
+
+static void
+gk20a_instobj_release_iommu(struct nvkm_memory *memory)
+{
+	struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
+	struct gk20a_instmem *imem = node->base.imem;
+	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;
 	unsigned long flags;
 
 	spin_lock_irqsave(&imem->lock, flags);
 
-	/* add ourselves to the LRU list so our CPU mapping can be freed */
-	list_add_tail(&node->vaddr_node, &imem->vaddr_lru);
+	/* we should at least have one user to release... */
+	if (WARN_ON(node->use_cpt == 0))
+		goto out;
+
+	/* add unused objs to the LRU list to recycle their mapping */
+	if (--node->use_cpt == 0)
+		list_add_tail(&node->vaddr_node, &imem->vaddr_lru);
 
+out:
 	spin_unlock_irqrestore(&imem->lock, flags);
 
 	wmb();
@@ -272,37 +280,6 @@ gk20a_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
 	nvkm_vm_map_at(vma, offset, &node->mem);
 }
 
-/*
- * Clear the CPU mapping of an instobj if it exists
- */
-static void
-gk20a_instobj_dtor(struct gk20a_instobj *node)
-{
-	struct gk20a_instmem *imem = node->imem;
-	struct gk20a_instobj *obj;
-	unsigned long flags;
-
-	spin_lock_irqsave(&imem->lock, flags);
-
-	if (!node->vaddr)
-		goto out;
-
-	list_for_each_entry(obj, &imem->vaddr_lru, vaddr_node) {
-		if (obj == node) {
-			list_del(&obj->vaddr_node);
-			break;
-		}
-	}
-	vunmap(node->vaddr);
-	node->vaddr = NULL;
-	imem->vaddr_use -= nvkm_memory_size(&node->memory);
-	nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n",
-		   imem->vaddr_use, imem->vaddr_max);
-
-out:
-	spin_unlock_irqrestore(&imem->lock, flags);
-}
-
 static void *
 gk20a_instobj_dtor_dma(struct nvkm_memory *memory)
 {
@@ -310,12 +287,10 @@ gk20a_instobj_dtor_dma(struct nvkm_memory *memory)
 	struct gk20a_instmem *imem = node->base.imem;
 	struct device *dev = imem->base.subdev.device->dev;
 
-	gk20a_instobj_dtor(&node->base);
-
-	if (unlikely(!node->cpuaddr))
+	if (unlikely(!node->base.vaddr))
 		goto out;
 
-	dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->cpuaddr,
+	dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->base.vaddr,
 		       node->handle, &imem->attrs);
 
 out:
@@ -329,13 +304,20 @@ gk20a_instobj_dtor_iommu(struct nvkm_memory *memory)
 	struct gk20a_instmem *imem = node->base.imem;
 	struct device *dev = imem->base.subdev.device->dev;
 	struct nvkm_mm_node *r;
+	unsigned long flags;
 	int i;
 
-	gk20a_instobj_dtor(&node->base);
-
 	if (unlikely(list_empty(&node->base.mem.regions)))
 		goto out;
 
+	spin_lock_irqsave(&imem->lock, flags);
+
+	/* vaddr has already been recycled */
+	if (node->base.vaddr)
+		gk20a_instobj_iommu_recycle_vaddr(node);
+
+	spin_unlock_irqrestore(&imem->lock, flags);
+
 	r = list_first_entry(&node->base.mem.regions, struct nvkm_mm_node,
 			     rl_entry);
 
@@ -366,8 +348,8 @@ gk20a_instobj_func_dma = {
 	.target = gk20a_instobj_target,
 	.addr = gk20a_instobj_addr,
 	.size = gk20a_instobj_size,
-	.acquire = gk20a_instobj_acquire,
-	.release = gk20a_instobj_release,
+	.acquire = gk20a_instobj_acquire_dma,
+	.release = gk20a_instobj_release_dma,
 	.rd32 = gk20a_instobj_rd32,
 	.wr32 = gk20a_instobj_wr32,
 	.map = gk20a_instobj_map,
@@ -379,8 +361,8 @@ gk20a_instobj_func_iommu = {
 	.target = gk20a_instobj_target,
 	.addr = gk20a_instobj_addr,
 	.size = gk20a_instobj_size,
-	.acquire = gk20a_instobj_acquire,
-	.release = gk20a_instobj_release,
+	.acquire = gk20a_instobj_acquire_iommu,
+	.release = gk20a_instobj_release_iommu,
 	.rd32 = gk20a_instobj_rd32,
 	.wr32 = gk20a_instobj_wr32,
 	.map = gk20a_instobj_map,
@@ -400,10 +382,10 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align,
 
 	nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.memory);
 
-	node->cpuaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT,
-					&node->handle, GFP_KERNEL,
-					&imem->attrs);
-	if (!node->cpuaddr) {
+	node->base.vaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT,
+					   &node->handle, GFP_KERNEL,
+					   &imem->attrs);
+	if (!node->base.vaddr) {
 		nvkm_error(subdev, "cannot allocate DMA memory\n");
 		return -ENOMEM;
 	}
@@ -609,18 +591,14 @@ gk20a_instmem_new(struct nvkm_device *device, int index,
 		imem->mm = &tdev->iommu.mm;
 		imem->domain = tdev->iommu.domain;
 		imem->iommu_pgshift = tdev->iommu.pgshift;
-		imem->cpu_map = gk20a_instobj_cpu_map_iommu;
 		imem->iommu_bit = tdev->func->iommu_bit;
 
 		nvkm_info(&imem->base.subdev, "using IOMMU\n");
 	} else {
 		init_dma_attrs(&imem->attrs);
-		/* We will access the memory through our own mapping */
 		dma_set_attr(DMA_ATTR_NON_CONSISTENT, &imem->attrs);
 		dma_set_attr(DMA_ATTR_WEAK_ORDERING, &imem->attrs);
 		dma_set_attr(DMA_ATTR_WRITE_COMBINE, &imem->attrs);
-		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &imem->attrs);
-		imem->cpu_map = gk20a_instobj_cpu_map_dma;
 
 		nvkm_info(&imem->base.subdev, "using DMA API\n");
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
index e5df3d865f0c..f8108df3cb38 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
@@ -2,3 +2,4 @@ nvkm-y += nvkm/subdev/ltc/base.o
 nvkm-y += nvkm/subdev/ltc/gf100.o
 nvkm-y += nvkm/subdev/ltc/gk104.o
 nvkm-y += nvkm/subdev/ltc/gm107.o
+nvkm-y += nvkm/subdev/ltc/gm204.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
index 3043bbfd7384..2af1f9e100fc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
@@ -26,16 +26,16 @@
 #include <subdev/fb.h>
 #include <subdev/timer.h>
 
-static void
+void
 gm107_ltc_cbc_clear(struct nvkm_ltc *ltc, u32 start, u32 limit)
 {
 	struct nvkm_device *device = ltc->subdev.device;
 	nvkm_wr32(device, 0x17e270, start);
 	nvkm_wr32(device, 0x17e274, limit);
-	nvkm_wr32(device, 0x17e26c, 0x00000004);
+	nvkm_mask(device, 0x17e26c, 0x00000000, 0x00000004);
 }
 
-static void
+void
 gm107_ltc_cbc_wait(struct nvkm_ltc *ltc)
 {
 	struct nvkm_device *device = ltc->subdev.device;
@@ -51,7 +51,7 @@ gm107_ltc_cbc_wait(struct nvkm_ltc *ltc)
 	}
 }
 
-static void
+void
 gm107_ltc_zbc_clear_color(struct nvkm_ltc *ltc, int i, const u32 color[4])
 {
 	struct nvkm_device *device = ltc->subdev.device;
@@ -62,7 +62,7 @@ gm107_ltc_zbc_clear_color(struct nvkm_ltc *ltc, int i, const u32 color[4])
 	nvkm_wr32(device, 0x17e348, color[3]);
 }
 
-static void
+void
 gm107_ltc_zbc_clear_depth(struct nvkm_ltc *ltc, int i, const u32 depth)
 {
 	struct nvkm_device *device = ltc->subdev.device;
@@ -84,7 +84,7 @@ gm107_ltc_lts_isr(struct nvkm_ltc *ltc, int c, int s)
 	}
 }
 
-static void
+void
 gm107_ltc_intr(struct nvkm_ltc *ltc)
 {
 	struct nvkm_device *device = ltc->subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm204.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm204.c
new file mode 100644
index 000000000000..5ad6fb9d022d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm204.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+
+static int
+gm204_ltc_oneinit(struct nvkm_ltc *ltc)
+{
+	struct nvkm_device *device = ltc->subdev.device;
+
+	ltc->ltc_nr = nvkm_rd32(device, 0x12006c);
+	ltc->lts_nr = nvkm_rd32(device, 0x17e280) >> 28;
+
+	return gf100_ltc_oneinit_tag_ram(ltc);
+}
+static void
+gm204_ltc_init(struct nvkm_ltc *ltc)
+{
+	nvkm_wr32(ltc->subdev.device, 0x17e278, ltc->tag_base);
+}
+
+static const struct nvkm_ltc_func
+gm204_ltc = {
+	.oneinit = gm204_ltc_oneinit,
+	.init = gm204_ltc_init,
+	.intr = gm107_ltc_intr, /*XXX: not validated */
+	.cbc_clear = gm107_ltc_cbc_clear,
+	.cbc_wait = gm107_ltc_cbc_wait,
+	.zbc = 16,
+	.zbc_clear_color = gm107_ltc_zbc_clear_color,
+	.zbc_clear_depth = gm107_ltc_zbc_clear_depth,
+	.invalidate = gf100_ltc_invalidate,
+	.flush = gf100_ltc_flush,
+};
+
+int
+gm204_ltc_new(struct nvkm_device *device, int index, struct nvkm_ltc **pltc)
+{
+	return nvkm_ltc_new_(&gm204_ltc, device, index, pltc);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
index 4e3755b82769..6d81c695ed0d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
@@ -31,4 +31,10 @@ void gf100_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
 void gf100_ltc_zbc_clear_depth(struct nvkm_ltc *, int, const u32);
 void gf100_ltc_invalidate(struct nvkm_ltc *);
 void gf100_ltc_flush(struct nvkm_ltc *);
+
+void gm107_ltc_intr(struct nvkm_ltc *);
+void gm107_ltc_cbc_clear(struct nvkm_ltc *, u32, u32);
+void gm107_ltc_cbc_wait(struct nvkm_ltc *);
+void gm107_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
+void gm107_ltc_zbc_clear_depth(struct nvkm_ltc *, int, const u32);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
index 4476ef75acd6..3c2519fdeb81 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
@@ -1,5 +1,6 @@
 nvkm-y += nvkm/subdev/pci/agp.o
 nvkm-y += nvkm/subdev/pci/base.o
+nvkm-y += nvkm/subdev/pci/pcie.o
 nvkm-y += nvkm/subdev/pci/nv04.o
 nvkm-y += nvkm/subdev/pci/nv40.o
 nvkm-y += nvkm/subdev/pci/nv46.o
@@ -7,3 +8,5 @@ nvkm-y += nvkm/subdev/pci/nv4c.o
 nvkm-y += nvkm/subdev/pci/g84.o
 nvkm-y += nvkm/subdev/pci/g94.o
 nvkm-y += nvkm/subdev/pci/gf100.o
+nvkm-y += nvkm/subdev/pci/gf106.o
+nvkm-y += nvkm/subdev/pci/gk104.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
index d671dcfaff3c..65057c8310a2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
@@ -107,6 +107,15 @@ nvkm_pci_preinit(struct nvkm_subdev *subdev)
 }
 
 static int
+nvkm_pci_oneinit(struct nvkm_subdev *subdev)
+{
+	struct nvkm_pci *pci = nvkm_pci(subdev);
+	if (pci_is_pcie(pci->pdev))
+		return nvkm_pcie_oneinit(pci);
+	return 0;
+}
+
+static int
 nvkm_pci_init(struct nvkm_subdev *subdev)
 {
 	struct nvkm_pci *pci = nvkm_pci(subdev);
@@ -117,6 +126,8 @@ nvkm_pci_init(struct nvkm_subdev *subdev)
 		ret = nvkm_agp_init(pci);
 		if (ret)
 			return ret;
+	} else if (pci_is_pcie(pci->pdev)) {
+		nvkm_pcie_init(pci);
 	}
 
 	if (pci->func->init)
@@ -143,6 +154,7 @@ nvkm_pci_dtor(struct nvkm_subdev *subdev)
 static const struct nvkm_subdev_func
 nvkm_pci_func = {
 	.dtor = nvkm_pci_dtor,
+	.oneinit = nvkm_pci_oneinit,
 	.preinit = nvkm_pci_preinit,
 	.init = nvkm_pci_init,
 	.fini = nvkm_pci_fini,
@@ -160,6 +172,8 @@ nvkm_pci_new_(const struct nvkm_pci_func *func, struct nvkm_device *device,
 	pci->func = func;
 	pci->pdev = device->func->pci(device)->pdev;
 	pci->irq = -1;
+	pci->pcie.speed = -1;
+	pci->pcie.width = -1;
 
 	if (device->type == NVKM_DEVICE_AGP)
 		nvkm_agp_ctor(pci);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g84.c
index 3faa6bfb895b..62438d892f42 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g84.c
@@ -25,6 +25,80 @@
 
 #include <core/pci.h>
 
+static int
+g84_pcie_version_supported(struct nvkm_pci *pci)
+{
+	/* g84 and g86 report wrong information about what they support */
+	return 1;
+}
+
+int
+g84_pcie_version(struct nvkm_pci *pci)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	return (nvkm_rd32(device, 0x00154c) & 0x1) + 1;
+}
+
+void
+g84_pcie_set_version(struct nvkm_pci *pci, u8 ver)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	nvkm_mask(device, 0x00154c, 0x1, (ver >= 2 ? 0x1 : 0x0));
+}
+
+static void
+g84_pcie_set_cap_speed(struct nvkm_pci *pci, bool full_speed)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	nvkm_mask(device, 0x00154c, 0x80, full_speed ? 0x80 : 0x0);
+}
+
+enum nvkm_pcie_speed
+g84_pcie_cur_speed(struct nvkm_pci *pci)
+{
+	u32 reg_v = nvkm_pci_rd32(pci, 0x88) & 0x30000;
+	switch (reg_v) {
+	case 0x30000:
+		return NVKM_PCIE_SPEED_8_0;
+	case 0x20000:
+		return NVKM_PCIE_SPEED_5_0;
+	case 0x10000:
+	default:
+		return NVKM_PCIE_SPEED_2_5;
+	}
+}
+
+enum nvkm_pcie_speed
+g84_pcie_max_speed(struct nvkm_pci *pci)
+{
+	u32 reg_v = nvkm_pci_rd32(pci, 0x460) & 0x3300;
+	if (reg_v == 0x2200)
+		return NVKM_PCIE_SPEED_5_0;
+	return NVKM_PCIE_SPEED_2_5;
+}
+
+void
+g84_pcie_set_link_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
+{
+	u32 mask_value;
+
+	if (speed == NVKM_PCIE_SPEED_5_0)
+		mask_value = 0x20;
+	else
+		mask_value = 0x10;
+
+	nvkm_pci_mask(pci, 0x460, 0x30, mask_value);
+	nvkm_pci_mask(pci, 0x460, 0x1, 0x1);
+}
+
+int
+g84_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
+{
+	g84_pcie_set_cap_speed(pci, speed == NVKM_PCIE_SPEED_5_0);
+	g84_pcie_set_link_speed(pci, speed);
+	return 0;
+}
+
 void
 g84_pci_init(struct nvkm_pci *pci)
 {
@@ -48,6 +122,14 @@ g84_pci_init(struct nvkm_pci *pci)
 		nvkm_pci_mask(pci, 0x041c, 0x00000060, 0x00000000);
 }
 
+int
+g84_pcie_init(struct nvkm_pci *pci)
+{
+	bool full_speed = g84_pcie_cur_speed(pci) == NVKM_PCIE_SPEED_5_0;
+	g84_pcie_set_cap_speed(pci, full_speed);
+	return 0;
+}
+
 static const struct nvkm_pci_func
 g84_pci_func = {
 	.init = g84_pci_init,
@@ -55,6 +137,16 @@ g84_pci_func = {
 	.wr08 = nv40_pci_wr08,
 	.wr32 = nv40_pci_wr32,
 	.msi_rearm = nv46_pci_msi_rearm,
+
+	.pcie.init = g84_pcie_init,
+	.pcie.set_link = g84_pcie_set_link,
+
+	.pcie.max_speed = g84_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = g84_pcie_set_version,
+	.pcie.version = g84_pcie_version,
+	.pcie.version_supported = g84_pcie_version_supported,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g94.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g94.c
index cd311ee311cc..43444123bc04 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/g94.c
@@ -23,6 +23,14 @@
  */
 #include "priv.h"
 
+int
+g94_pcie_version_supported(struct nvkm_pci *pci)
+{
+	if ((nvkm_pci_rd32(pci, 0x460) & 0x200) == 0x200)
+		return 2;
+	return 1;
+}
+
 static const struct nvkm_pci_func
 g94_pci_func = {
 	.init = g84_pci_init,
@@ -30,6 +38,16 @@ g94_pci_func = {
 	.wr08 = nv40_pci_wr08,
 	.wr32 = nv40_pci_wr32,
 	.msi_rearm = nv40_pci_msi_rearm,
+
+	.pcie.init = g84_pcie_init,
+	.pcie.set_link = g84_pcie_set_link,
+
+	.pcie.max_speed = g84_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = g84_pcie_set_version,
+	.pcie.version = g84_pcie_version,
+	.pcie.version_supported = g94_pcie_version_supported,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf100.c
index 25e1ae70867f..e30ea676baf6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf100.c
@@ -29,6 +29,53 @@ gf100_pci_msi_rearm(struct nvkm_pci *pci)
 	nvkm_pci_wr08(pci, 0x0704, 0xff);
 }
 
+void
+gf100_pcie_set_version(struct nvkm_pci *pci, u8 ver)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	nvkm_mask(device, 0x02241c, 0x1, ver > 1 ? 1 : 0);
+}
+
+int
+gf100_pcie_version(struct nvkm_pci *pci)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	return (nvkm_rd32(device, 0x02241c) & 0x1) + 1;
+}
+
+void
+gf100_pcie_set_cap_speed(struct nvkm_pci *pci, bool full_speed)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	nvkm_mask(device, 0x02241c, 0x80, full_speed ? 0x80 : 0x0);
+}
+
+int
+gf100_pcie_cap_speed(struct nvkm_pci *pci)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	u8 punits_pci_cap_speed = nvkm_rd32(device, 0x02241c) & 0x80;
+	if (punits_pci_cap_speed == 0x80)
+		return 1;
+	return 0;
+}
+
+int
+gf100_pcie_init(struct nvkm_pci *pci)
+{
+	bool full_speed = g84_pcie_cur_speed(pci) == NVKM_PCIE_SPEED_5_0;
+	gf100_pcie_set_cap_speed(pci, full_speed);
+	return 0;
+}
+
+int
+gf100_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
+{
+	gf100_pcie_set_cap_speed(pci, speed == NVKM_PCIE_SPEED_5_0);
+	g84_pcie_set_link_speed(pci, speed);
+	return 0;
+}
+
 static const struct nvkm_pci_func
 gf100_pci_func = {
 	.init = g84_pci_init,
@@ -36,6 +83,16 @@ gf100_pci_func = {
 	.wr08 = nv40_pci_wr08,
 	.wr32 = nv40_pci_wr32,
 	.msi_rearm = gf100_pci_msi_rearm,
+
+	.pcie.init = gf100_pcie_init,
+	.pcie.set_link = gf100_pcie_set_link,
+
+	.pcie.max_speed = g84_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = gf100_pcie_set_version,
+	.pcie.version = gf100_pcie_version,
+	.pcie.version_supported = g94_pcie_version_supported,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf106.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf106.c
new file mode 100644
index 000000000000..c3b798c5c6dd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gf106.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2015 Karol Herbst <nouveau@karolherbst.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst <nouveau@karolherbst.de>
+ */
+#include "priv.h"
+
+static const struct nvkm_pci_func
+gf106_pci_func = {
+	.init = g84_pci_init,
+	.rd32 = nv40_pci_rd32,
+	.wr08 = nv40_pci_wr08,
+	.wr32 = nv40_pci_wr32,
+	.msi_rearm = nv40_pci_msi_rearm,
+
+	.pcie.init = gf100_pcie_init,
+	.pcie.set_link = gf100_pcie_set_link,
+
+	.pcie.max_speed = g84_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = gf100_pcie_set_version,
+	.pcie.version = gf100_pcie_version,
+	.pcie.version_supported = g94_pcie_version_supported,
+};
+
+int
+gf106_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+	return nvkm_pci_new_(&gf106_pci_func, device, index, ppci);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gk104.c
new file mode 100644
index 000000000000..e68030507d88
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gk104.c
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2015 Karol Herbst <nouveau@karolherbst.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst <nouveau@karolherbst.de>
+ */
+#include "priv.h"
+
+static int
+gk104_pcie_version_supported(struct nvkm_pci *pci)
+{
+	return (nvkm_rd32(pci->subdev.device, 0x8c1c0) & 0x4) == 0x4 ? 2 : 1;
+}
+
+static void
+gk104_pcie_set_cap_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
+{
+	struct nvkm_device *device = pci->subdev.device;
+
+	switch (speed) {
+	case NVKM_PCIE_SPEED_2_5:
+		gf100_pcie_set_cap_speed(pci, false);
+		nvkm_mask(device, 0x8c1c0, 0x30000, 0x10000);
+		break;
+	case NVKM_PCIE_SPEED_5_0:
+		gf100_pcie_set_cap_speed(pci, true);
+		nvkm_mask(device, 0x8c1c0, 0x30000, 0x20000);
+		break;
+	case NVKM_PCIE_SPEED_8_0:
+		gf100_pcie_set_cap_speed(pci, true);
+		nvkm_mask(device, 0x8c1c0, 0x30000, 0x30000);
+		break;
+	}
+}
+
+static enum nvkm_pcie_speed
+gk104_pcie_cap_speed(struct nvkm_pci *pci)
+{
+	int speed = gf100_pcie_cap_speed(pci);
+
+	if (speed == 0)
+		return NVKM_PCIE_SPEED_2_5;
+
+	if (speed >= 1) {
+		int speed2 = nvkm_rd32(pci->subdev.device, 0x8c1c0) & 0x30000;
+		switch (speed2) {
+		case 0x00000:
+		case 0x10000:
+			return NVKM_PCIE_SPEED_2_5;
+		case 0x20000:
+			return NVKM_PCIE_SPEED_5_0;
+		case 0x30000:
+			return NVKM_PCIE_SPEED_8_0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static void
+gk104_pcie_set_lnkctl_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
+{
+	u8 reg_v = 0;
+	switch (speed) {
+	case NVKM_PCIE_SPEED_2_5:
+		reg_v = 1;
+		break;
+	case NVKM_PCIE_SPEED_5_0:
+		reg_v = 2;
+		break;
+	case NVKM_PCIE_SPEED_8_0:
+		reg_v = 3;
+		break;
+	}
+	nvkm_pci_mask(pci, 0xa8, 0x3, reg_v);
+}
+
+static enum nvkm_pcie_speed
+gk104_pcie_lnkctl_speed(struct nvkm_pci *pci)
+{
+	u8 reg_v = nvkm_pci_rd32(pci, 0xa8) & 0x3;
+	switch (reg_v) {
+	case 0:
+	case 1:
+		return NVKM_PCIE_SPEED_2_5;
+	case 2:
+		return NVKM_PCIE_SPEED_5_0;
+	case 3:
+		return NVKM_PCIE_SPEED_8_0;
+	}
+	return -1;
+}
+
+static enum nvkm_pcie_speed
+gk104_pcie_max_speed(struct nvkm_pci *pci)
+{
+	u32 max_speed = nvkm_rd32(pci->subdev.device, 0x8c1c0) & 0x300000;
+	switch (max_speed) {
+	case 0x000000:
+		return NVKM_PCIE_SPEED_8_0;
+	case 0x100000:
+		return NVKM_PCIE_SPEED_5_0;
+	case 0x200000:
+		return NVKM_PCIE_SPEED_2_5;
+	}
+	return NVKM_PCIE_SPEED_2_5;
+}
+
+static void
+gk104_pcie_set_link_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
+{
+	struct nvkm_device *device = pci->subdev.device;
+	u32 mask_value;
+
+	switch (speed) {
+	case NVKM_PCIE_SPEED_8_0:
+		mask_value = 0x00000;
+		break;
+	case NVKM_PCIE_SPEED_5_0:
+		mask_value = 0x40000;
+		break;
+	case NVKM_PCIE_SPEED_2_5:
+	default:
+		mask_value = 0x80000;
+		break;
+	}
+
+	nvkm_mask(device, 0x8c040, 0xc0000, mask_value);
+	nvkm_mask(device, 0x8c040, 0x1, 0x1);
+}
+
+static int
+gk104_pcie_init(struct nvkm_pci * pci)
+{
+	enum nvkm_pcie_speed lnkctl_speed, max_speed, cap_speed;
+	struct nvkm_subdev *subdev = &pci->subdev;
+
+	if (gf100_pcie_version(pci) < 2)
+		return 0;
+
+	lnkctl_speed = gk104_pcie_lnkctl_speed(pci);
+	max_speed = gk104_pcie_max_speed(pci);
+	cap_speed = gk104_pcie_cap_speed(pci);
+
+	if (cap_speed != max_speed) {
+		nvkm_trace(subdev, "adjusting cap to max speed\n");
+		gk104_pcie_set_cap_speed(pci, max_speed);
+		cap_speed = gk104_pcie_cap_speed(pci);
+		if (cap_speed != max_speed)
+			nvkm_warn(subdev, "failed to adjust cap speed\n");
+	}
+
+	if (lnkctl_speed != max_speed) {
+		nvkm_debug(subdev, "adjusting lnkctl to max speed\n");
+		gk104_pcie_set_lnkctl_speed(pci, max_speed);
+		lnkctl_speed = gk104_pcie_lnkctl_speed(pci);
+		if (lnkctl_speed != max_speed)
+			nvkm_error(subdev, "failed to adjust lnkctl speed\n");
+	}
+
+	return 0;
+}
+
+static int
+gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
+{
+	struct nvkm_subdev *subdev = &pci->subdev;
+	enum nvkm_pcie_speed lnk_ctl_speed = gk104_pcie_lnkctl_speed(pci);
+	enum nvkm_pcie_speed lnk_cap_speed = gk104_pcie_cap_speed(pci);
+
+	if (speed > lnk_cap_speed) {
+		speed = lnk_cap_speed;
+		nvkm_warn(subdev, "dropping requested speed due too low cap"
+			  " speed\n");
+	}
+
+	if (speed > lnk_ctl_speed) {
+		speed = lnk_ctl_speed;
+		nvkm_warn(subdev, "dropping requested speed due too low"
+			  " lnkctl speed\n");
+	}
+
+	gk104_pcie_set_link_speed(pci, speed);
+	return 0;
+}
+
+
+static const struct nvkm_pci_func
+gk104_pci_func = {
+	.init = g84_pci_init,
+	.rd32 = nv40_pci_rd32,
+	.wr08 = nv40_pci_wr08,
+	.wr32 = nv40_pci_wr32,
+	.msi_rearm = nv40_pci_msi_rearm,
+
+	.pcie.init = gk104_pcie_init,
+	.pcie.set_link = gk104_pcie_set_link,
+
+	.pcie.max_speed = gk104_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = gf100_pcie_set_version,
+	.pcie.version = gf100_pcie_version,
+	.pcie.version_supported = gk104_pcie_version_supported,
+};
+
+int
+gk104_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+	return nvkm_pci_new_(&gk104_pci_func, device, index, ppci);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/pcie.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/pcie.c
new file mode 100644
index 000000000000..d71e5db5028a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/pcie.c
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2015 Karol Herbst <nouveau@karolherbst.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst <git@karolherbst.de>
+ */
+#include "priv.h"
+
+static char *nvkm_pcie_speeds[] = {
+	"2.5GT/s",
+	"5.0GT/s",
+	"8.0GT/s",
+};
+
+static enum nvkm_pcie_speed
+nvkm_pcie_speed(enum pci_bus_speed speed)
+{
+	switch (speed) {
+	case PCIE_SPEED_2_5GT:
+		return NVKM_PCIE_SPEED_2_5;
+	case PCIE_SPEED_5_0GT:
+		return NVKM_PCIE_SPEED_5_0;
+	case PCIE_SPEED_8_0GT:
+		return NVKM_PCIE_SPEED_8_0;
+	default:
+		/* XXX 0x16 is 8_0, assume 0x17 will be 16_0 for now */
+		if (speed == 0x17)
+			return NVKM_PCIE_SPEED_8_0;
+		return -1;
+	}
+}
+
+static int
+nvkm_pcie_get_version(struct nvkm_pci *pci)
+{
+	if (!pci->func->pcie.version)
+		return -ENOSYS;
+
+	return pci->func->pcie.version(pci);
+}
+
+static int
+nvkm_pcie_get_max_version(struct nvkm_pci *pci)
+{
+	if (!pci->func->pcie.version_supported)
+		return -ENOSYS;
+
+	return pci->func->pcie.version_supported(pci);
+}
+
+static int
+nvkm_pcie_set_version(struct nvkm_pci *pci, int version)
+{
+	if (!pci->func->pcie.set_version)
+		return -ENOSYS;
+
+	nvkm_trace(&pci->subdev, "set to version %i\n", version);
+	pci->func->pcie.set_version(pci, version);
+	return nvkm_pcie_get_version(pci);
+}
+
+int
+nvkm_pcie_oneinit(struct nvkm_pci *pci)
+{
+	if (pci->func->pcie.max_speed)
+		nvkm_debug(&pci->subdev, "pcie max speed: %s\n",
+			   nvkm_pcie_speeds[pci->func->pcie.max_speed(pci)]);
+	return 0;
+}
+
+int
+nvkm_pcie_init(struct nvkm_pci *pci)
+{
+	struct nvkm_subdev *subdev = &pci->subdev;
+	int ret;
+
+	/* raise pcie version first */
+	ret = nvkm_pcie_get_version(pci);
+	if (ret > 0) {
+		int max_version = nvkm_pcie_get_max_version(pci);
+		if (max_version > 0 && max_version > ret)
+			ret = nvkm_pcie_set_version(pci, max_version);
+
+		if (ret < max_version)
+			nvkm_error(subdev, "couldn't raise version: %i\n", ret);
+	}
+
+	if (pci->func->pcie.init)
+		pci->func->pcie.init(pci);
+
+	if (pci->pcie.speed != -1)
+		nvkm_pcie_set_link(pci, pci->pcie.speed, pci->pcie.width);
+
+	return 0;
+}
+
+int
+nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
+{
+	struct nvkm_subdev *subdev = &pci->subdev;
+	enum nvkm_pcie_speed cur_speed, max_speed;
+	struct pci_bus *pbus;
+	int ret;
+
+	if (!pci || !pci_is_pcie(pci->pdev))
+		return 0;
+	pbus = pci->pdev->bus;
+
+	if (!pci->func->pcie.set_link)
+		return -ENOSYS;
+
+	nvkm_trace(subdev, "requested %s\n", nvkm_pcie_speeds[speed]);
+
+	if (pci->func->pcie.version(pci) < 2) {
+		nvkm_error(subdev, "setting link failed due to low version\n");
+		return -ENODEV;
+	}
+
+	cur_speed = pci->func->pcie.cur_speed(pci);
+	max_speed = min(nvkm_pcie_speed(pbus->max_bus_speed),
+			pci->func->pcie.max_speed(pci));
+
+	nvkm_trace(subdev, "current speed: %s\n", nvkm_pcie_speeds[cur_speed]);
+
+	if (speed > max_speed) {
+		nvkm_debug(subdev, "%s not supported by bus or card, dropping"
+			   "requested speed to %s", nvkm_pcie_speeds[speed],
+			   nvkm_pcie_speeds[max_speed]);
+		speed = max_speed;
+	}
+
+	pci->pcie.speed = speed;
+	pci->pcie.width = width;
+
+	if (speed == cur_speed) {
+		nvkm_debug(subdev, "requested matches current speed\n");
+		return speed;
+	}
+
+	nvkm_debug(subdev, "set link to %s x%i\n",
+		   nvkm_pcie_speeds[speed], width);
+
+	ret = pci->func->pcie.set_link(pci, speed, width);
+	if (ret < 0)
+		nvkm_error(subdev, "setting link failed: %i\n", ret);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/priv.h
index cf46d38d0b0a..23de3180aae5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/priv.h
@@ -12,6 +12,18 @@ struct nvkm_pci_func {
 	void (*wr08)(struct nvkm_pci *, u16 addr, u8 data);
 	void (*wr32)(struct nvkm_pci *, u16 addr, u32 data);
 	void (*msi_rearm)(struct nvkm_pci *);
+
+	struct {
+		int (*init)(struct nvkm_pci *);
+		int (*set_link)(struct nvkm_pci *, enum nvkm_pcie_speed, u8);
+
+		enum nvkm_pcie_speed (*max_speed)(struct nvkm_pci *);
+		enum nvkm_pcie_speed (*cur_speed)(struct nvkm_pci *);
+
+		void (*set_version)(struct nvkm_pci *, u8);
+		int (*version)(struct nvkm_pci *);
+		int (*version_supported)(struct nvkm_pci *);
+	} pcie;
 };
 
 u32 nv40_pci_rd32(struct nvkm_pci *, u16);
@@ -22,4 +34,25 @@ void nv40_pci_msi_rearm(struct nvkm_pci *);
 void nv46_pci_msi_rearm(struct nvkm_pci *);
 
 void g84_pci_init(struct nvkm_pci *pci);
+
+/* pcie functions */
+void g84_pcie_set_version(struct nvkm_pci *, u8);
+int g84_pcie_version(struct nvkm_pci *);
+void g84_pcie_set_link_speed(struct nvkm_pci *, enum nvkm_pcie_speed);
+enum nvkm_pcie_speed g84_pcie_cur_speed(struct nvkm_pci *);
+enum nvkm_pcie_speed g84_pcie_max_speed(struct nvkm_pci *);
+int g84_pcie_init(struct nvkm_pci *);
+int g84_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8);
+
+int g94_pcie_version_supported(struct nvkm_pci *);
+
+void gf100_pcie_set_version(struct nvkm_pci *, u8);
+int gf100_pcie_version(struct nvkm_pci *);
+void gf100_pcie_set_cap_speed(struct nvkm_pci *, bool);
+int gf100_pcie_cap_speed(struct nvkm_pci *);
+int gf100_pcie_init(struct nvkm_pci *);
+int gf100_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8);
+
+int nvkm_pcie_oneinit(struct nvkm_pci *);
+int nvkm_pcie_init(struct nvkm_pci *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
index 302557c52d03..770294457274 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
@@ -24,8 +24,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 /* 0x0058: proc_list_head */
 	0x54534f48,
-	0x00000512,
-	0x000004af,
+	0x00000507,
+	0x000004a4,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -46,8 +46,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x0000075e,
-	0x00000750,
+	0x00000753,
+	0x00000745,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -68,8 +68,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x00000762,
-	0x00000760,
+	0x00000757,
+	0x00000755,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -90,8 +90,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000b92,
-	0x00000a35,
+	0x00000b87,
+	0x00000a2a,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -112,8 +112,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000bbb,
-	0x00000b94,
+	0x00000bb0,
+	0x00000b89,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -134,8 +134,8 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000bc7,
-	0x00000bc5,
+	0x00000bbc,
+	0x00000bba,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -229,26 +229,26 @@ uint32_t gf100_pmu_data[] = {
 /* 0x0370: memx_func_head */
 	0x00000001,
 	0x00000000,
-	0x00000551,
+	0x00000546,
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x000005db,
+	0x000005d0,
 	0x00000003,
 	0x00000002,
-	0x000006a5,
+	0x0000069a,
 	0x00040004,
 	0x00000000,
-	0x000006c1,
+	0x000006b6,
 	0x00010005,
 	0x00000000,
-	0x000006de,
+	0x000006d3,
 	0x00010006,
 	0x00000000,
-	0x00000663,
+	0x00000658,
 	0x00000007,
 	0x00000000,
-	0x000006e9,
+	0x000006de,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -917,7 +917,7 @@ uint32_t gf100_pmu_data[] = {
 };
 
 uint32_t gf100_pmu_code[] = {
-	0x039e0ef5,
+	0x03930ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
 	0xd00604b6,
@@ -987,7 +987,7 @@ uint32_t gf100_pmu_code[] = {
 	0xbb9a0a98,
 	0x1cf4029a,
 	0x01d7f00f,
-	0x02dd21f5,
+	0x02d221f5,
 	0x0ef494bd,
 /* 0x00f9: intr_watchdog_next_time */
 	0x9b0a9815,
@@ -1039,7 +1039,7 @@ uint32_t gf100_pmu_code[] = {
 	0x48e7f1c0,
 	0x53e3f14f,
 	0x00d7f054,
-	0x034221f5,
+	0x033721f5,
 	0x07f1c0fc,
 	0x04b604c0,
 	0x000cd006,
@@ -1048,760 +1048,758 @@ uint32_t gf100_pmu_code[] = {
 	0x04b60688,
 	0x0009d006,
 /* 0x01ca: intr_skip_subintr */
-	0x89c404bd,
-	0x070bf420,
-	0xffbfa4f1,
-/* 0x01d4: intr_skip_pause */
-	0xf44089c4,
-	0xa4f1070b,
-/* 0x01de: intr_skip_user0 */
-	0x07f0ffbf,
-	0x0604b604,
-	0xbd0008d0,
-	0xfe80fc04,
-	0xf0fc0088,
-	0xd0fce0fc,
-	0xb0fcc0fc,
-	0x90fca0fc,
-	0x00fc80fc,
-	0xf80032f4,
-/* 0x0205: ticks_from_ns */
-	0xf9c0f901,
-	0xcbd7f1b0,
-	0x00d3f000,
-	0x041321f5,
-	0x03e8ccec,
-	0xf400b4b0,
-	0xeeec120b,
-	0xd7f103e8,
-	0xd3f000cb,
-	0x1321f500,
-/* 0x022d: ticks_from_ns_quit */
-	0x02ceb904,
-	0xc0fcb0fc,
-/* 0x0236: ticks_from_us */
-	0xc0f900f8,
+	0x97f104bd,
+	0x90bd00e0,
+	0xf00489fd,
+	0x04b60407,
+	0x0008d006,
+	0x80fc04bd,
+	0xfc0088fe,
+	0xfce0fcf0,
+	0xfcc0fcd0,
+	0xfca0fcb0,
+	0xfc80fc90,
+	0x0032f400,
+/* 0x01fa: ticks_from_ns */
+	0xc0f901f8,
 	0xd7f1b0f9,
 	0xd3f000cb,
-	0x1321f500,
-	0x02ceb904,
-	0xf400b4b0,
-	0xe4bd050b,
-/* 0x0250: ticks_from_us_quit */
-	0xc0fcb0fc,
-/* 0x0256: ticks_to_us */
-	0xd7f100f8,
-	0xd3f000cb,
-	0xecedff00,
-/* 0x0262: timer */
-	0x90f900f8,
-	0x32f480f9,
-	0x03f89810,
-	0xf40086b0,
-	0x84bd651c,
-	0xb63807f0,
-	0x08d00604,
-	0xf004bd00,
-	0x84b63487,
-	0x0088cf06,
-	0xbb9a0998,
-	0xe9bb0298,
-	0x03fe8000,
-	0xb60887f0,
-	0x88cf0684,
-	0x0284f000,
-	0xf0261bf4,
-	0x84b63487,
-	0x0088cf06,
-	0xf406e0b8,
-	0xe8b8090b,
-	0x111cf406,
-/* 0x02b8: timer_reset */
-	0xb63407f0,
-	0x0ed00604,
-	0x8004bd00,
-/* 0x02c6: timer_enable */
-	0x87f09a0e,
-	0x3807f001,
+	0x0821f500,
+	0xe8ccec04,
+	0x00b4b003,
+	0xec120bf4,
+	0xf103e8ee,
+	0xf000cbd7,
+	0x21f500d3,
+/* 0x0222: ticks_from_ns_quit */
+	0xceb90408,
+	0xfcb0fc02,
+/* 0x022b: ticks_from_us */
+	0xf900f8c0,
+	0xf1b0f9c0,
+	0xf000cbd7,
+	0x21f500d3,
+	0xceb90408,
+	0x00b4b002,
+	0xbd050bf4,
+/* 0x0245: ticks_from_us_quit */
+	0xfcb0fce4,
+/* 0x024b: ticks_to_us */
+	0xf100f8c0,
+	0xf000cbd7,
+	0xedff00d3,
+/* 0x0257: timer */
+	0xf900f8ec,
+	0xf480f990,
+	0xf8981032,
+	0x0086b003,
+	0xbd651cf4,
+	0x3807f084,
 	0xd00604b6,
 	0x04bd0008,
-/* 0x02d4: timer_done */
-	0xfc1031f4,
-	0xf890fc80,
-/* 0x02dd: send_proc */
-	0xf980f900,
-	0x05e89890,
-	0xf004e998,
-	0x89b80486,
-	0x2a0bf406,
-	0x940398c4,
-	0x80b60488,
-	0x008ebb18,
-	0x8000fa98,
-	0x8d80008a,
-	0x028c8001,
-	0xb6038b80,
-	0x94f00190,
-	0x04e98007,
-/* 0x0317: send_done */
-	0xfc0231f4,
-	0xf880fc90,
-/* 0x031d: find */
-	0xf080f900,
-	0x31f45887,
-/* 0x0325: find_loop */
-	0x008a9801,
-	0xf406aeb8,
-	0x80b6100b,
-	0x6886b158,
-	0xf01bf402,
-/* 0x033b: find_done */
-	0xb90132f4,
-	0x80fc028e,
-/* 0x0342: send */
-	0x21f500f8,
-	0x01f4031d,
-/* 0x034b: recv */
-	0xf900f897,
-	0x9880f990,
-	0xe99805e8,
-	0x0132f404,
-	0xf40689b8,
-	0x89c43d0b,
-	0x0180b603,
-	0x800784f0,
-	0xea9805e8,
-	0xfef0f902,
-	0xf0f9018f,
-	0x9402efb9,
-	0xe9bb0499,
-	0x18e0b600,
-	0x9803eb98,
-	0xed9802ec,
-	0x00ee9801,
-	0xf0fca5f9,
-	0xf400f8fe,
-	0xf0fc0131,
-/* 0x0398: recv_done */
-	0x90fc80fc,
-/* 0x039e: init */
-	0x17f100f8,
-	0x14b60108,
-	0x0011cf06,
-	0x010911e7,
-	0xfe0814b6,
-	0x17f10014,
-	0x13f000e0,
-	0x1c07f000,
-	0xd00604b6,
-	0x04bd0001,
-	0xf0ff17f0,
-	0x04b61407,
-	0x0001d006,
-	0x17f004bd,
-	0x0015f102,
-	0x1007f008,
+	0xb63487f0,
+	0x88cf0684,
+	0x9a099800,
+	0xbb0298bb,
+	0xfe8000e9,
+	0x0887f003,
+	0xcf0684b6,
+	0x84f00088,
+	0x261bf402,
+	0xb63487f0,
+	0x88cf0684,
+	0x06e0b800,
+	0xb8090bf4,
+	0x1cf406e8,
+/* 0x02ad: timer_reset */
+	0x3407f011,
 	0xd00604b6,
-	0x04bd0001,
-	0x011a17f1,
-	0xfe0013f0,
-	0x31f40010,
-	0x0117f010,
-	0xb63807f0,
+	0x04bd000e,
+/* 0x02bb: timer_enable */
+	0xf09a0e80,
+	0x07f00187,
+	0x0604b638,
+	0xbd0008d0,
+/* 0x02c9: timer_done */
+	0x1031f404,
+	0x90fc80fc,
+/* 0x02d2: send_proc */
+	0x80f900f8,
+	0xe89890f9,
+	0x04e99805,
+	0xb80486f0,
+	0x0bf40689,
+	0x0398c42a,
+	0xb6048894,
+	0x8ebb1880,
+	0x00fa9800,
+	0x80008a80,
+	0x8c80018d,
+	0x038b8002,
+	0xf00190b6,
+	0xe9800794,
+	0x0231f404,
+/* 0x030c: send_done */
+	0x80fc90fc,
+/* 0x0312: find */
+	0x80f900f8,
+	0xf45887f0,
+/* 0x031a: find_loop */
+	0x8a980131,
+	0x06aeb800,
+	0xb6100bf4,
+	0x86b15880,
+	0x1bf40268,
+	0x0132f4f0,
+/* 0x0330: find_done */
+	0xfc028eb9,
+/* 0x0337: send */
+	0xf500f880,
+	0xf4031221,
+	0x00f89701,
+/* 0x0340: recv */
+	0x80f990f9,
+	0x9805e898,
+	0x32f404e9,
+	0x0689b801,
+	0xc43d0bf4,
+	0x80b60389,
+	0x0784f001,
+	0x9805e880,
+	0xf0f902ea,
+	0xf9018ffe,
+	0x02efb9f0,
+	0xbb049994,
+	0xe0b600e9,
+	0x03eb9818,
+	0x9802ec98,
+	0xee9801ed,
+	0xfca5f900,
+	0x00f8fef0,
+	0xfc0131f4,
+/* 0x038d: recv_done */
+	0xfc80fcf0,
+/* 0x0393: init */
+	0xf100f890,
+	0xb6010817,
+	0x11cf0614,
+	0x0911e700,
+	0x0814b601,
+	0xf10014fe,
+	0xf000e017,
+	0x07f00013,
+	0x0604b61c,
+	0xbd0001d0,
+	0xff17f004,
+	0xb61407f0,
 	0x01d00604,
 	0xf004bd00,
-/* 0x0402: init_proc */
-	0xf19858f7,
-	0x0016b001,
-	0xf9fa0bf4,
-	0x58f0b615,
-/* 0x0413: mulu32_32_64 */
-	0xf9f20ef4,
-	0xf920f910,
-	0x9540f930,
-	0xd29510e1,
-	0xbdc4bd10,
-	0xc0edffb4,
-	0xb9301dff,
-	0x34f10234,
-	0x34b6ffff,
-	0x1045b610,
-	0xbb00c3bb,
-	0xe2ff01b4,
-	0x0234b930,
-	0xffff34f1,
-	0xb61034b6,
-	0xc3bb1045,
-	0x01b4bb00,
-	0xbb3012ff,
-	0x40fc00b3,
-	0x20fc30fc,
-	0x00f810fc,
-/* 0x0464: host_send */
-	0x04b017f1,
-	0xcf0614b6,
-	0x27f10011,
-	0x24b604a0,
-	0x0022cf06,
-	0xf40612b8,
-	0x1ec4320b,
-	0x04ee9407,
-	0x0270e0b7,
-	0x9803eb98,
-	0xed9802ec,
-	0x00ee9801,
-	0x034221f5,
-	0xc40110b6,
-	0x07f10f1e,
-	0x04b604b0,
-	0x000ed006,
-	0x0ef404bd,
-/* 0x04ad: host_send_done */
-/* 0x04af: host_recv */
-	0xf100f8ba,
-	0xf14e4917,
-	0xb8525413,
-	0x0bf406e1,
-/* 0x04bd: host_recv_wait */
-	0xcc17f1aa,
+	0x15f10217,
+	0x07f00800,
+	0x0604b610,
+	0xbd0001d0,
+	0x1a17f104,
+	0x0013f001,
+	0xf40010fe,
+	0x17f01031,
+	0x3807f001,
+	0xd00604b6,
+	0x04bd0001,
+/* 0x03f7: init_proc */
+	0x9858f7f0,
+	0x16b001f1,
+	0xfa0bf400,
+	0xf0b615f9,
+	0xf20ef458,
+/* 0x0408: mulu32_32_64 */
+	0x20f910f9,
+	0x40f930f9,
+	0x9510e195,
+	0xc4bd10d2,
+	0xedffb4bd,
+	0x301dffc0,
+	0xf10234b9,
+	0xb6ffff34,
+	0x45b61034,
+	0x00c3bb10,
+	0xff01b4bb,
+	0x34b930e2,
+	0xff34f102,
+	0x1034b6ff,
+	0xbb1045b6,
+	0xb4bb00c3,
+	0x3012ff01,
+	0xfc00b3bb,
+	0xfc30fc40,
+	0xf810fc20,
+/* 0x0459: host_send */
+	0xb017f100,
 	0x0614b604,
 	0xf10011cf,
-	0xb604c827,
+	0xb604a027,
 	0x22cf0624,
-	0x0816f000,
-	0xf40612b8,
-	0x23c4e60b,
-	0x0434b607,
-	0x02f030b7,
-	0x80033b80,
-	0x3d80023c,
-	0x003e8001,
-	0xf00120b6,
-	0x07f10f24,
-	0x04b604c8,
-	0x0002d006,
-	0x27f004bd,
-	0x0007f040,
-	0xd00604b6,
-	0x04bd0002,
-/* 0x0512: host_init */
-	0x17f100f8,
+	0x0612b800,
+	0xc4320bf4,
+	0xee94071e,
+	0x70e0b704,
+	0x03eb9802,
+	0x9802ec98,
+	0xee9801ed,
+	0x3721f500,
+	0x0110b603,
+	0xf10f1ec4,
+	0xb604b007,
+	0x0ed00604,
+	0xf404bd00,
+/* 0x04a2: host_send_done */
+	0x00f8ba0e,
+/* 0x04a4: host_recv */
+	0x4e4917f1,
+	0x525413f1,
+	0xf406e1b8,
+/* 0x04b2: host_recv_wait */
+	0x17f1aa0b,
+	0x14b604cc,
+	0x0011cf06,
+	0x04c827f1,
+	0xcf0624b6,
+	0x16f00022,
+	0x0612b808,
+	0xc4e60bf4,
+	0x34b60723,
+	0xf030b704,
+	0x033b8002,
+	0x80023c80,
+	0x3e80013d,
+	0x0120b600,
+	0xf10f24f0,
+	0xb604c807,
+	0x02d00604,
+	0xf004bd00,
+	0x07f04027,
+	0x0604b600,
+	0xbd0002d0,
+/* 0x0507: host_init */
+	0xf100f804,
+	0xb6008017,
+	0x15f11014,
+	0x07f10270,
+	0x04b604d0,
+	0x0001d006,
+	0x17f104bd,
 	0x14b60080,
-	0x7015f110,
-	0xd007f102,
+	0xf015f110,
+	0xdc07f102,
 	0x0604b604,
 	0xbd0001d0,
-	0x8017f104,
-	0x1014b600,
-	0x02f015f1,
-	0x04dc07f1,
+	0x0117f004,
+	0x04c407f1,
 	0xd00604b6,
 	0x04bd0001,
-	0xf10117f0,
-	0xb604c407,
-	0x01d00604,
-	0xf804bd00,
-/* 0x0551: memx_func_enter */
-	0x2067f100,
-	0x5d77f116,
-	0xff73f1f5,
+/* 0x0546: memx_func_enter */
+	0x67f100f8,
+	0x77f11620,
+	0x73f1f55d,
+	0x6eb9ffff,
+	0x0421f402,
+	0xfd02d8b9,
+	0x60f90487,
+	0xd0fc80f9,
+	0x21f4e0fc,
+	0xfe77f13f,
+	0xff73f1ff,
 	0x026eb9ff,
 	0xb90421f4,
 	0x87fd02d8,
 	0xf960f904,
 	0xfcd0fc80,
 	0x3f21f4e0,
-	0xfffe77f1,
-	0xffff73f1,
+	0x26f067f1,
 	0xf4026eb9,
 	0xd8b90421,
 	0x0487fd02,
 	0x80f960f9,
 	0xe0fcd0fc,
-	0xf13f21f4,
-	0xb926f067,
-	0x21f4026e,
-	0x02d8b904,
-	0xf90487fd,
-	0xfc80f960,
-	0xf4e0fcd0,
-	0x67f03f21,
-	0xe007f104,
-	0x0604b607,
-	0xbd0006d0,
-/* 0x05bd: memx_func_enter_wait */
-	0xc067f104,
-	0x0664b607,
-	0xf00066cf,
-	0x0bf40464,
-	0x2c67f0f3,
-	0xcf0664b6,
-	0x06800066,
-/* 0x05db: memx_func_leave */
-	0xf000f8f1,
-	0x64b62c67,
-	0x0066cf06,
-	0xf0f20680,
+	0xf03f21f4,
 	0x07f10467,
-	0x04b607e4,
+	0x04b607e0,
 	0x0006d006,
-/* 0x05f6: memx_func_leave_wait */
+/* 0x05b2: memx_func_enter_wait */
 	0x67f104bd,
 	0x64b607c0,
 	0x0066cf06,
 	0xf40464f0,
-	0x67f1f31b,
-	0x77f126f0,
-	0x73f00001,
-	0x026eb900,
-	0xb90421f4,
-	0x87fd02d8,
-	0xf960f905,
-	0xfcd0fc80,
-	0x3f21f4e0,
-	0x162067f1,
-	0xf4026eb9,
-	0xd8b90421,
-	0x0587fd02,
-	0x80f960f9,
-	0xe0fcd0fc,
-	0xf13f21f4,
-	0xf00aa277,
+	0x67f0f30b,
+	0x0664b62c,
+	0x800066cf,
+	0x00f8f106,
+/* 0x05d0: memx_func_leave */
+	0xb62c67f0,
+	0x66cf0664,
+	0xf2068000,
+	0xf10467f0,
+	0xb607e407,
+	0x06d00604,
+/* 0x05eb: memx_func_leave_wait */
+	0xf104bd00,
+	0xb607c067,
+	0x66cf0664,
+	0x0464f000,
+	0xf1f31bf4,
+	0xf126f067,
+	0xf0000177,
 	0x6eb90073,
 	0x0421f402,
 	0xfd02d8b9,
 	0x60f90587,
 	0xd0fc80f9,
 	0x21f4e0fc,
-/* 0x0663: memx_func_wait_vblank */
-	0x9800f83f,
-	0x66b00016,
-	0x130bf400,
-	0xf40166b0,
-	0x0ef4060b,
-/* 0x0675: memx_func_wait_vblank_head1 */
-	0x2077f12e,
-	0x070ef400,
-/* 0x067c: memx_func_wait_vblank_head0 */
-	0x000877f1,
-/* 0x0680: memx_func_wait_vblank_0 */
-	0x07c467f1,
-	0xcf0664b6,
-	0x67fd0066,
-	0xf31bf404,
-/* 0x0690: memx_func_wait_vblank_1 */
-	0x07c467f1,
-	0xcf0664b6,
-	0x67fd0066,
-	0xf30bf404,
-/* 0x06a0: memx_func_wait_vblank_fini */
-	0xf80410b6,
-/* 0x06a5: memx_func_wr32 */
-	0x00169800,
-	0xb6011598,
-	0x60f90810,
-	0xd0fc50f9,
-	0x21f4e0fc,
-	0x0242b63f,
-	0xf8e91bf4,
-/* 0x06c1: memx_func_wait */
-	0x2c87f000,
-	0xcf0684b6,
-	0x1e980088,
-	0x011d9800,
-	0x98021c98,
-	0x10b6031b,
-	0xa421f410,
-/* 0x06de: memx_func_delay */
-	0x1e9800f8,
+	0x2067f13f,
+	0x026eb916,
+	0xb90421f4,
+	0x87fd02d8,
+	0xf960f905,
+	0xfcd0fc80,
+	0x3f21f4e0,
+	0x0aa277f1,
+	0xb90073f0,
+	0x21f4026e,
+	0x02d8b904,
+	0xf90587fd,
+	0xfc80f960,
+	0xf4e0fcd0,
+	0x00f83f21,
+/* 0x0658: memx_func_wait_vblank */
+	0xb0001698,
+	0x0bf40066,
+	0x0166b013,
+	0xf4060bf4,
+/* 0x066a: memx_func_wait_vblank_head1 */
+	0x77f12e0e,
+	0x0ef40020,
+/* 0x0671: memx_func_wait_vblank_head0 */
+	0x0877f107,
+/* 0x0675: memx_func_wait_vblank_0 */
+	0xc467f100,
+	0x0664b607,
+	0xfd0066cf,
+	0x1bf40467,
+/* 0x0685: memx_func_wait_vblank_1 */
+	0xc467f1f3,
+	0x0664b607,
+	0xfd0066cf,
+	0x0bf40467,
+/* 0x0695: memx_func_wait_vblank_fini */
+	0x0410b6f3,
+/* 0x069a: memx_func_wr32 */
+	0x169800f8,
+	0x01159800,
+	0xf90810b6,
+	0xfc50f960,
+	0xf4e0fcd0,
+	0x42b63f21,
+	0xe91bf402,
+/* 0x06b6: memx_func_wait */
+	0x87f000f8,
+	0x0684b62c,
+	0x980088cf,
+	0x1d98001e,
+	0x021c9801,
+	0xb6031b98,
+	0x21f41010,
+/* 0x06d3: memx_func_delay */
+	0x9800f8a4,
+	0x10b6001e,
+	0x7f21f404,
+/* 0x06de: memx_func_train */
+	0x00f800f8,
+/* 0x06e0: memx_exec */
+	0xd0f9e0f9,
+	0xb902c1b9,
+/* 0x06ea: memx_exec_next */
+	0x139802b2,
 	0x0410b600,
+	0x01f034e7,
+	0x01e033e7,
+	0xf00132b6,
+	0x35980c30,
+	0xb855f9de,
+	0x1ef40612,
+	0xf10b98e4,
+	0xbbf20c98,
+	0xb7f102cb,
+	0xb4b607c4,
+	0x00bbcf06,
+	0xe0fcd0fc,
+	0x033721f5,
+/* 0x0726: memx_info */
+	0xc67000f8,
+	0x0e0bf401,
+/* 0x072c: memx_info_data */
+	0x03ccc7f1,
+	0x0800b7f1,
+/* 0x0737: memx_info_train */
+	0xf10b0ef4,
+	0xf10bccc7,
+/* 0x073f: memx_info_send */
+	0xf50100b7,
+	0xf8033721,
+/* 0x0745: memx_recv */
+	0x01d6b000,
+	0xb0980bf4,
+	0x0bf400d6,
+/* 0x0753: memx_init */
+	0xf800f8d8,
+/* 0x0755: perf_recv */
+/* 0x0757: perf_init */
+	0xf800f800,
+/* 0x0759: i2c_drive_scl */
+	0x0036b000,
+	0xf1110bf4,
+	0xb607e007,
+	0x01d00604,
+	0xf804bd00,
+/* 0x076d: i2c_drive_scl_lo */
+	0xe407f100,
+	0x0604b607,
+	0xbd0001d0,
+/* 0x077b: i2c_drive_sda */
+	0xb000f804,
+	0x0bf40036,
+	0xe007f111,
+	0x0604b607,
+	0xbd0002d0,
+/* 0x078f: i2c_drive_sda_lo */
+	0xf100f804,
+	0xb607e407,
+	0x02d00604,
+	0xf804bd00,
+/* 0x079d: i2c_sense_scl */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x31fd0033,
+	0x060bf404,
+/* 0x07b3: i2c_sense_scl_done */
+	0xf80131f4,
+/* 0x07b5: i2c_sense_sda */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x32fd0033,
+	0x060bf404,
+/* 0x07cb: i2c_sense_sda_done */
+	0xf80131f4,
+/* 0x07cd: i2c_raise_scl */
+	0xf140f900,
+	0xf0089847,
+	0x21f50137,
+/* 0x07da: i2c_raise_scl_wait */
+	0xe7f10759,
+	0x21f403e8,
+	0x9d21f57f,
+	0x0901f407,
+	0xf40142b6,
+/* 0x07ee: i2c_raise_scl_done */
+	0x40fcef1b,
+/* 0x07f2: i2c_start */
+	0x21f500f8,
+	0x11f4079d,
+	0xb521f50d,
+	0x0611f407,
+/* 0x0803: i2c_start_rep */
+	0xf0300ef4,
+	0x21f50037,
+	0x37f00759,
+	0x7b21f501,
+	0x0076bb07,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b607cd,
+	0x1f11f404,
+/* 0x0830: i2c_start_send */
+	0xf50037f0,
+	0xf1077b21,
+	0xf41388e7,
+	0x37f07f21,
+	0x5921f500,
+	0x88e7f107,
+	0x7f21f413,
+/* 0x084c: i2c_start_out */
+/* 0x084e: i2c_stop */
+	0x37f000f8,
+	0x5921f500,
+	0x0037f007,
+	0x077b21f5,
+	0x03e8e7f1,
+	0xf07f21f4,
+	0x21f50137,
+	0xe7f10759,
+	0x21f41388,
+	0x0137f07f,
+	0x077b21f5,
+	0x1388e7f1,
 	0xf87f21f4,
-/* 0x06e9: memx_func_train */
-/* 0x06eb: memx_exec */
-	0xf900f800,
-	0xb9d0f9e0,
-	0xb2b902c1,
-/* 0x06f5: memx_exec_next */
-	0x00139802,
-	0xe70410b6,
-	0xe701f034,
-	0xb601e033,
-	0x30f00132,
-	0xde35980c,
-	0x12b855f9,
-	0xe41ef406,
-	0x98f10b98,
-	0xcbbbf20c,
-	0xc4b7f102,
-	0x06b4b607,
-	0xfc00bbcf,
-	0xf5e0fcd0,
-	0xf8034221,
-/* 0x0731: memx_info */
-	0x01c67000,
-/* 0x0737: memx_info_data */
-	0xf10e0bf4,
-	0xf103ccc7,
-	0xf40800b7,
-/* 0x0742: memx_info_train */
-	0xc7f10b0e,
-	0xb7f10bcc,
-/* 0x074a: memx_info_send */
-	0x21f50100,
-	0x00f80342,
-/* 0x0750: memx_recv */
-	0xf401d6b0,
-	0xd6b0980b,
-	0xd80bf400,
-/* 0x075e: memx_init */
-	0x00f800f8,
-/* 0x0760: perf_recv */
-/* 0x0762: perf_init */
-	0x00f800f8,
-/* 0x0764: i2c_drive_scl */
-	0xf40036b0,
-	0x07f1110b,
-	0x04b607e0,
-	0x0001d006,
-	0x00f804bd,
-/* 0x0778: i2c_drive_scl_lo */
-	0x07e407f1,
-	0xd00604b6,
-	0x04bd0001,
-/* 0x0786: i2c_drive_sda */
-	0x36b000f8,
-	0x110bf400,
-	0x07e007f1,
-	0xd00604b6,
-	0x04bd0002,
-/* 0x079a: i2c_drive_sda_lo */
-	0x07f100f8,
-	0x04b607e4,
-	0x0002d006,
-	0x00f804bd,
-/* 0x07a8: i2c_sense_scl */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0431fd00,
-	0xf4060bf4,
-/* 0x07be: i2c_sense_scl_done */
-	0x00f80131,
-/* 0x07c0: i2c_sense_sda */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0432fd00,
-	0xf4060bf4,
-/* 0x07d6: i2c_sense_sda_done */
-	0x00f80131,
-/* 0x07d8: i2c_raise_scl */
-	0x47f140f9,
-	0x37f00898,
-	0x6421f501,
-/* 0x07e5: i2c_raise_scl_wait */
+/* 0x0881: i2c_bitw */
+	0x7b21f500,
 	0xe8e7f107,
 	0x7f21f403,
-	0x07a821f5,
-	0xb60901f4,
-	0x1bf40142,
-/* 0x07f9: i2c_raise_scl_done */
-	0xf840fcef,
-/* 0x07fd: i2c_start */
-	0xa821f500,
-	0x0d11f407,
-	0x07c021f5,
-	0xf40611f4,
-/* 0x080e: i2c_start_rep */
-	0x37f0300e,
-	0x6421f500,
-	0x0137f007,
-	0x078621f5,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xd821f550,
+	0xcd21f550,
 	0x0464b607,
-/* 0x083b: i2c_start_send */
-	0xf01f11f4,
-	0x21f50037,
-	0xe7f10786,
-	0x21f41388,
-	0x0037f07f,
-	0x076421f5,
-	0x1388e7f1,
-/* 0x0857: i2c_start_out */
-	0xf87f21f4,
-/* 0x0859: i2c_stop */
-	0x0037f000,
-	0x076421f5,
-	0xf50037f0,
-	0xf1078621,
-	0xf403e8e7,
+	0xf11811f4,
+	0xf41388e7,
 	0x37f07f21,
-	0x6421f501,
+	0x5921f500,
 	0x88e7f107,
 	0x7f21f413,
-	0xf50137f0,
-	0xf1078621,
-	0xf41388e7,
-	0x00f87f21,
-/* 0x088c: i2c_bitw */
-	0x078621f5,
-	0x03e8e7f1,
-	0xbb7f21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x07d821f5,
-	0xf40464b6,
-	0xe7f11811,
+/* 0x08c0: i2c_bitw_out */
+/* 0x08c2: i2c_bitr */
+	0x37f000f8,
+	0x7b21f501,
+	0xe8e7f107,
+	0x7f21f403,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xcd21f550,
+	0x0464b607,
+	0xf51b11f4,
+	0xf007b521,
+	0x21f50037,
+	0xe7f10759,
 	0x21f41388,
-	0x0037f07f,
-	0x076421f5,
-	0x1388e7f1,
-/* 0x08cb: i2c_bitw_out */
-	0xf87f21f4,
-/* 0x08cd: i2c_bitr */
-	0x0137f000,
-	0x078621f5,
-	0x03e8e7f1,
-	0xbb7f21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x07d821f5,
-	0xf40464b6,
-	0x21f51b11,
-	0x37f007c0,
-	0x6421f500,
-	0x88e7f107,
-	0x7f21f413,
-	0xf4013cf0,
-/* 0x0912: i2c_bitr_done */
-	0x00f80131,
-/* 0x0914: i2c_get_byte */
-	0xf00057f0,
-/* 0x091a: i2c_get_byte_next */
-	0x54b60847,
+	0x013cf07f,
+/* 0x0907: i2c_bitr_done */
+	0xf80131f4,
+/* 0x0909: i2c_get_byte */
+	0x0057f000,
+/* 0x090f: i2c_get_byte_next */
+	0xb60847f0,
+	0x76bb0154,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb608c221,
+	0x11f40464,
+	0x0553fd2b,
+	0xf40142b6,
+	0x37f0d81b,
 	0x0076bb01,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b608cd,
-	0x2b11f404,
-	0xb60553fd,
-	0x1bf40142,
-	0x0137f0d8,
+	0x64b60881,
+/* 0x0959: i2c_get_byte_done */
+/* 0x095b: i2c_put_byte */
+	0xf000f804,
+/* 0x095e: i2c_put_byte_next */
+	0x42b60847,
+	0x3854ff01,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x8c21f550,
+	0x8121f550,
 	0x0464b608,
-/* 0x0964: i2c_get_byte_done */
-/* 0x0966: i2c_put_byte */
-	0x47f000f8,
-/* 0x0969: i2c_put_byte_next */
-	0x0142b608,
-	0xbb3854ff,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x088c21f5,
-	0xf40464b6,
-	0x46b03411,
-	0xd81bf400,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0xcd21f550,
-	0x0464b608,
-	0xbb0f11f4,
-	0x36b00076,
-	0x061bf401,
-/* 0x09bf: i2c_put_byte_done */
-	0xf80132f4,
-/* 0x09c1: i2c_addr */
-	0x0076bb00,
+	0xb03411f4,
+	0x1bf40046,
+	0x0076bbd8,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b607fd,
-	0x2911f404,
-	0x012ec3e7,
-	0xfd0134b6,
-	0x76bb0553,
+	0x64b608c2,
+	0x0f11f404,
+	0xb00076bb,
+	0x1bf40136,
+	0x0132f406,
+/* 0x09b4: i2c_put_byte_done */
+/* 0x09b6: i2c_addr */
+	0x76bb00f8,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb6096621,
-/* 0x0a06: i2c_addr_done */
-	0x00f80464,
-/* 0x0a08: i2c_acquire_addr */
-	0xb6f8cec7,
-	0xe0b702e4,
-	0xee980d1c,
-/* 0x0a17: i2c_acquire */
-	0xf500f800,
-	0xf40a0821,
-	0xd9f00421,
-	0x3f21f403,
-/* 0x0a26: i2c_release */
-	0x21f500f8,
-	0x21f40a08,
-	0x03daf004,
-	0xf83f21f4,
-/* 0x0a35: i2c_recv */
-	0x0132f400,
-	0xb6f8c1c7,
-	0x16b00214,
-	0x3a1ff528,
-	0xf413a001,
-	0x0032980c,
-	0x0ccc13a0,
-	0xf4003198,
-	0xd0f90231,
-	0xd0f9e0f9,
-	0x000067f1,
-	0x100063f1,
-	0xbb016792,
+	0xb607f221,
+	0x11f40464,
+	0x2ec3e729,
+	0x0134b601,
+	0xbb0553fd,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0a1721f5,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b31bf5,
-	0xbb0057f0,
+	0x095b21f5,
+/* 0x09fb: i2c_addr_done */
+	0xf80464b6,
+/* 0x09fd: i2c_acquire_addr */
+	0xf8cec700,
+	0xb702e4b6,
+	0x980d1ce0,
+	0x00f800ee,
+/* 0x0a0c: i2c_acquire */
+	0x09fd21f5,
+	0xf00421f4,
+	0x21f403d9,
+/* 0x0a1b: i2c_release */
+	0xf500f83f,
+	0xf409fd21,
+	0xdaf00421,
+	0x3f21f403,
+/* 0x0a2a: i2c_recv */
+	0x32f400f8,
+	0xf8c1c701,
+	0xb00214b6,
+	0x1ff52816,
+	0x13a0013a,
+	0x32980cf4,
+	0xcc13a000,
+	0x0031980c,
+	0xf90231f4,
+	0xf9e0f9d0,
+	0x0067f1d0,
+	0x0063f100,
+	0x01679210,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x0c21f550,
+	0x0464b60a,
+	0xd6b0d0fc,
+	0xb31bf500,
+	0x0057f000,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xb621f550,
+	0x0464b609,
+	0x00d011f5,
+	0xbbe0c5c7,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x09c121f5,
+	0x095b21f5,
 	0xf50464b6,
-	0xc700d011,
-	0x76bbe0c5,
+	0xf000ad11,
+	0x76bb0157,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb6096621,
+	0xb609b621,
 	0x11f50464,
-	0x57f000ad,
-	0x0076bb01,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b609c1,
-	0x8a11f504,
-	0x0076bb00,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b60914,
-	0x6a11f404,
-	0xbbe05bcb,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x085921f5,
-	0xb90464b6,
-	0x74bd025b,
-/* 0x0b3b: i2c_recv_not_rd08 */
-	0xb0430ef4,
-	0x1bf401d6,
-	0x0057f03d,
-	0x09c121f5,
-	0xc73311f4,
-	0x21f5e0c5,
-	0x11f40966,
-	0x0057f029,
-	0x09c121f5,
-	0xc71f11f4,
-	0x21f5e0b5,
-	0x11f40966,
-	0x5921f515,
-	0xc774bd08,
-	0x1bf408c5,
-	0x0232f409,
-/* 0x0b7b: i2c_recv_not_wr08 */
-/* 0x0b7b: i2c_recv_done */
-	0xc7030ef4,
-	0x21f5f8ce,
-	0xe0fc0a26,
-	0x12f4d0fc,
-	0x027cb90a,
-	0x034221f5,
-/* 0x0b90: i2c_recv_exit */
-/* 0x0b92: i2c_init */
+	0x76bb008a,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb6090921,
+	0x11f40464,
+	0xe05bcb6a,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x4e21f550,
+	0x0464b608,
+	0xbd025bb9,
+	0x430ef474,
+/* 0x0b30: i2c_recv_not_rd08 */
+	0xf401d6b0,
+	0x57f03d1b,
+	0xb621f500,
+	0x3311f409,
+	0xf5e0c5c7,
+	0xf4095b21,
+	0x57f02911,
+	0xb621f500,
+	0x1f11f409,
+	0xf5e0b5c7,
+	0xf4095b21,
+	0x21f51511,
+	0x74bd084e,
+	0xf408c5c7,
+	0x32f4091b,
+	0x030ef402,
+/* 0x0b70: i2c_recv_not_wr08 */
+/* 0x0b70: i2c_recv_done */
+	0xf5f8cec7,
+	0xfc0a1b21,
+	0xf4d0fce0,
+	0x7cb90a12,
+	0x3721f502,
+/* 0x0b85: i2c_recv_exit */
+/* 0x0b87: i2c_init */
+	0xf800f803,
+/* 0x0b89: test_recv */
+	0xd817f100,
+	0x0614b605,
+	0xb60011cf,
+	0x07f10110,
+	0x04b605d8,
+	0x0001d006,
+	0xe7f104bd,
+	0xe3f1d900,
+	0x21f5134f,
+	0x00f80257,
+/* 0x0bb0: test_init */
+	0x0800e7f1,
+	0x025721f5,
+/* 0x0bba: idle_recv */
 	0x00f800f8,
-/* 0x0b94: test_recv */
-	0x05d817f1,
-	0xcf0614b6,
-	0x10b60011,
-	0xd807f101,
-	0x0604b605,
-	0xbd0001d0,
-	0x00e7f104,
-	0x4fe3f1d9,
-	0x6221f513,
-/* 0x0bbb: test_init */
-	0xf100f802,
-	0xf50800e7,
-	0xf8026221,
-/* 0x0bc5: idle_recv */
-/* 0x0bc7: idle */
-	0xf400f800,
-	0x17f10031,
-	0x14b605d4,
-	0x0011cf06,
-	0xf10110b6,
-	0xb605d407,
-	0x01d00604,
-/* 0x0be3: idle_loop */
-	0xf004bd00,
-	0x32f45817,
-/* 0x0be9: idle_proc */
-/* 0x0be9: idle_proc_exec */
-	0xb910f902,
-	0x21f5021e,
-	0x10fc034b,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0bfd: idle_proc_next */
-	0x5810b6ef,
-	0xf4061fb8,
-	0x02f4e61b,
-	0x0028f4dd,
-	0x00bb0ef4,
+/* 0x0bbc: idle */
+	0xf10031f4,
+	0xb605d417,
+	0x11cf0614,
+	0x0110b600,
+	0x05d407f1,
+	0xd00604b6,
+	0x04bd0001,
+/* 0x0bd8: idle_loop */
+	0xf45817f0,
+/* 0x0bde: idle_proc */
+/* 0x0bde: idle_proc_exec */
+	0x10f90232,
+	0xf5021eb9,
+	0xfc034021,
+	0x0911f410,
+	0xf40231f4,
+/* 0x0bf2: idle_proc_next */
+	0x10b6ef0e,
+	0x061fb858,
+	0xf4e61bf4,
+	0x28f4dd02,
+	0xbb0ef400,
+	0x00000000,
+	0x00000000,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
index 31552af9b06e..7bf6b39ed205 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
@@ -24,8 +24,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 /* 0x0058: proc_list_head */
 	0x54534f48,
-	0x0000049d,
-	0x00000446,
+	0x00000492,
+	0x0000043b,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -46,8 +46,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x0000068b,
-	0x0000067d,
+	0x00000680,
+	0x00000672,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -68,8 +68,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x0000068f,
-	0x0000068d,
+	0x00000684,
+	0x00000682,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -90,8 +90,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000aaa,
-	0x0000094d,
+	0x00000a9f,
+	0x00000942,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -112,8 +112,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000acd,
-	0x00000aac,
+	0x00000ac2,
+	0x00000aa1,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -134,8 +134,8 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000ad9,
-	0x00000ad7,
+	0x00000ace,
+	0x00000acc,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -229,26 +229,26 @@ uint32_t gf119_pmu_data[] = {
 /* 0x0370: memx_func_head */
 	0x00000001,
 	0x00000000,
-	0x000004d3,
+	0x000004c8,
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x00000554,
+	0x00000549,
 	0x00000003,
 	0x00000002,
-	0x000005d8,
+	0x000005cd,
 	0x00040004,
 	0x00000000,
-	0x000005f4,
+	0x000005e9,
 	0x00010005,
 	0x00000000,
-	0x0000060e,
+	0x00000603,
 	0x00010006,
 	0x00000000,
-	0x000005d3,
+	0x000005c8,
 	0x00000007,
 	0x00000000,
-	0x00000619,
+	0x0000060e,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -916,7 +916,7 @@ uint32_t gf119_pmu_data[] = {
 };
 
 uint32_t gf119_pmu_code[] = {
-	0x034d0ef5,
+	0x03420ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
 	0xbd000ed0,
@@ -977,7 +977,7 @@ uint32_t gf119_pmu_code[] = {
 	0xbb9a0a98,
 	0x1cf4029a,
 	0x01d7f00f,
-	0x028c21f5,
+	0x028121f5,
 	0x0ef494bd,
 /* 0x00d5: intr_watchdog_next_time */
 	0x9b0a9815,
@@ -1025,716 +1025,714 @@ uint32_t gf119_pmu_code[] = {
 	0xf14f48e7,
 	0xf05453e3,
 	0x21f500d7,
-	0xc0fc02f1,
+	0xc0fc02e6,
 	0x04c007f1,
 	0xbd000cd0,
 /* 0x0185: intr_subintr_skip_fifo */
 	0x8807f104,
 	0x0009d006,
 /* 0x018e: intr_skip_subintr */
-	0x89c404bd,
-	0x070bf420,
-	0xffbfa4f1,
-/* 0x0198: intr_skip_pause */
-	0xf44089c4,
-	0xa4f1070b,
-/* 0x01a2: intr_skip_user0 */
-	0x07f0ffbf,
-	0x0008d004,
-	0x80fc04bd,
-	0xfc0088fe,
-	0xfce0fcf0,
-	0xfcc0fcd0,
-	0xfca0fcb0,
-	0xfc80fc90,
-	0x0032f400,
-/* 0x01c6: ticks_from_ns */
-	0xc0f901f8,
-	0xd7f1b0f9,
-	0xd3f00144,
-	0xb321f500,
-	0xe8ccec03,
-	0x00b4b003,
-	0xec120bf4,
-	0xf103e8ee,
-	0xf00144d7,
-	0x21f500d3,
-/* 0x01ee: ticks_from_ns_quit */
-	0xceb903b3,
-	0xfcb0fc02,
-/* 0x01f7: ticks_from_us */
-	0xf900f8c0,
+	0x97f104bd,
+	0x90bd00e0,
+	0xf00489fd,
+	0x08d00407,
+	0xfc04bd00,
+	0x0088fe80,
+	0xe0fcf0fc,
+	0xc0fcd0fc,
+	0xa0fcb0fc,
+	0x80fc90fc,
+	0x32f400fc,
+/* 0x01bb: ticks_from_ns */
+	0xf901f800,
 	0xf1b0f9c0,
 	0xf00144d7,
 	0x21f500d3,
-	0xceb903b3,
-	0x00b4b002,
-	0xbd050bf4,
-/* 0x0211: ticks_from_us_quit */
-	0xfcb0fce4,
-/* 0x0217: ticks_to_us */
-	0xf100f8c0,
-	0xf00144d7,
-	0xedff00d3,
-/* 0x0223: timer */
-	0xf900f8ec,
-	0xf480f990,
-	0xf8981032,
-	0x0086b003,
-	0xbd531cf4,
-	0x3807f084,
-	0xbd0008d0,
-	0x3487f004,
-	0x980088cf,
-	0x98bb9a09,
-	0x00e9bb02,
-	0xf003fe80,
-	0x88cf0887,
-	0x0284f000,
-	0xf0201bf4,
-	0x88cf3487,
-	0x06e0b800,
-	0xb8090bf4,
-	0x1cf406e8,
-/* 0x026d: timer_reset */
-	0x3407f00e,
-	0xbd000ed0,
-	0x9a0e8004,
-/* 0x0278: timer_enable */
-	0xf00187f0,
-	0x08d03807,
-/* 0x0283: timer_done */
-	0xf404bd00,
-	0x80fc1031,
-	0x00f890fc,
-/* 0x028c: send_proc */
-	0x90f980f9,
-	0x9805e898,
-	0x86f004e9,
-	0x0689b804,
-	0xc42a0bf4,
-	0x88940398,
-	0x1880b604,
-	0x98008ebb,
-	0x8a8000fa,
-	0x018d8000,
-	0x80028c80,
-	0x90b6038b,
-	0x0794f001,
-	0xf404e980,
-/* 0x02c6: send_done */
-	0x90fc0231,
-	0x00f880fc,
-/* 0x02cc: find */
-	0x87f080f9,
-	0x0131f458,
-/* 0x02d4: find_loop */
-	0xb8008a98,
-	0x0bf406ae,
-	0x5880b610,
-	0x026886b1,
-	0xf4f01bf4,
-/* 0x02ea: find_done */
-	0x8eb90132,
-	0xf880fc02,
-/* 0x02f1: send */
-	0xcc21f500,
-	0x9701f402,
-/* 0x02fa: recv */
-	0x90f900f8,
-	0xe89880f9,
-	0x04e99805,
-	0xb80132f4,
-	0x0bf40689,
-	0x0389c43d,
-	0xf00180b6,
-	0xe8800784,
-	0x02ea9805,
-	0x8ffef0f9,
-	0xb9f0f901,
-	0x999402ef,
-	0x00e9bb04,
-	0x9818e0b6,
-	0xec9803eb,
-	0x01ed9802,
-	0xf900ee98,
-	0xfef0fca5,
-	0x31f400f8,
-/* 0x0347: recv_done */
-	0xfcf0fc01,
+	0xccec03a8,
+	0xb4b003e8,
+	0x120bf400,
+	0x03e8eeec,
+	0x0144d7f1,
+	0xf500d3f0,
+/* 0x01e3: ticks_from_ns_quit */
+	0xb903a821,
+	0xb0fc02ce,
+	0x00f8c0fc,
+/* 0x01ec: ticks_from_us */
+	0xb0f9c0f9,
+	0x0144d7f1,
+	0xf500d3f0,
+	0xb903a821,
+	0xb4b002ce,
+	0x050bf400,
+/* 0x0206: ticks_from_us_quit */
+	0xb0fce4bd,
+	0x00f8c0fc,
+/* 0x020c: ticks_to_us */
+	0x0144d7f1,
+	0xff00d3f0,
+	0x00f8eced,
+/* 0x0218: timer */
+	0x80f990f9,
+	0x981032f4,
+	0x86b003f8,
+	0x531cf400,
+	0x07f084bd,
+	0x0008d038,
+	0x87f004bd,
+	0x0088cf34,
+	0xbb9a0998,
+	0xe9bb0298,
+	0x03fe8000,
+	0xcf0887f0,
+	0x84f00088,
+	0x201bf402,
+	0xcf3487f0,
+	0xe0b80088,
+	0x090bf406,
+	0xf406e8b8,
+/* 0x0262: timer_reset */
+	0x07f00e1c,
+	0x000ed034,
+	0x0e8004bd,
+/* 0x026d: timer_enable */
+	0x0187f09a,
+	0xd03807f0,
+	0x04bd0008,
+/* 0x0278: timer_done */
+	0xfc1031f4,
 	0xf890fc80,
-/* 0x034d: init */
-	0x0817f100,
-	0x0011cf01,
-	0x010911e7,
-	0xfe0814b6,
-	0x17f10014,
-	0x13f000e0,
-	0x1c07f000,
+/* 0x0281: send_proc */
+	0xf980f900,
+	0x05e89890,
+	0xf004e998,
+	0x89b80486,
+	0x2a0bf406,
+	0x940398c4,
+	0x80b60488,
+	0x008ebb18,
+	0x8000fa98,
+	0x8d80008a,
+	0x028c8001,
+	0xb6038b80,
+	0x94f00190,
+	0x04e98007,
+/* 0x02bb: send_done */
+	0xfc0231f4,
+	0xf880fc90,
+/* 0x02c1: find */
+	0xf080f900,
+	0x31f45887,
+/* 0x02c9: find_loop */
+	0x008a9801,
+	0xf406aeb8,
+	0x80b6100b,
+	0x6886b158,
+	0xf01bf402,
+/* 0x02df: find_done */
+	0xb90132f4,
+	0x80fc028e,
+/* 0x02e6: send */
+	0x21f500f8,
+	0x01f402c1,
+/* 0x02ef: recv */
+	0xf900f897,
+	0x9880f990,
+	0xe99805e8,
+	0x0132f404,
+	0xf40689b8,
+	0x89c43d0b,
+	0x0180b603,
+	0x800784f0,
+	0xea9805e8,
+	0xfef0f902,
+	0xf0f9018f,
+	0x9402efb9,
+	0xe9bb0499,
+	0x18e0b600,
+	0x9803eb98,
+	0xed9802ec,
+	0x00ee9801,
+	0xf0fca5f9,
+	0xf400f8fe,
+	0xf0fc0131,
+/* 0x033c: recv_done */
+	0x90fc80fc,
+/* 0x0342: init */
+	0x17f100f8,
+	0x11cf0108,
+	0x0911e700,
+	0x0814b601,
+	0xf10014fe,
+	0xf000e017,
+	0x07f00013,
+	0x0001d01c,
+	0x17f004bd,
+	0x1407f0ff,
 	0xbd0001d0,
-	0xff17f004,
-	0xd01407f0,
+	0x0217f004,
+	0x080015f1,
+	0xd01007f0,
 	0x04bd0001,
-	0xf10217f0,
-	0xf0080015,
-	0x01d01007,
-	0xf104bd00,
-	0xf000f617,
-	0x10fe0013,
-	0x1031f400,
-	0xf00117f0,
-	0x01d03807,
-	0xf004bd00,
-/* 0x03a2: init_proc */
-	0xf19858f7,
-	0x0016b001,
-	0xf9fa0bf4,
-	0x58f0b615,
-/* 0x03b3: mulu32_32_64 */
-	0xf9f20ef4,
-	0xf920f910,
-	0x9540f930,
-	0xd29510e1,
-	0xbdc4bd10,
-	0xc0edffb4,
-	0xb9301dff,
-	0x34f10234,
-	0x34b6ffff,
-	0x1045b610,
-	0xbb00c3bb,
-	0xe2ff01b4,
-	0x0234b930,
-	0xffff34f1,
-	0xb61034b6,
-	0xc3bb1045,
-	0x01b4bb00,
-	0xbb3012ff,
-	0x40fc00b3,
-	0x20fc30fc,
-	0x00f810fc,
-/* 0x0404: host_send */
-	0x04b017f1,
-	0xf10011cf,
-	0xcf04a027,
-	0x12b80022,
-	0x2f0bf406,
-	0x94071ec4,
-	0xe0b704ee,
-	0xeb980270,
-	0x02ec9803,
-	0x9801ed98,
-	0x21f500ee,
-	0x10b602f1,
-	0x0f1ec401,
-	0x04b007f1,
-	0xbd000ed0,
-	0xc30ef404,
-/* 0x0444: host_send_done */
-/* 0x0446: host_recv */
-	0x17f100f8,
-	0x13f14e49,
-	0xe1b85254,
-	0xb30bf406,
-/* 0x0454: host_recv_wait */
-	0x04cc17f1,
-	0xf10011cf,
-	0xcf04c827,
-	0x16f00022,
-	0x0612b808,
-	0xc4ec0bf4,
-	0x34b60723,
-	0xf030b704,
-	0x033b8002,
-	0x80023c80,
-	0x3e80013d,
-	0x0120b600,
-	0xf10f24f0,
-	0xd004c807,
+	0x00f617f1,
+	0xfe0013f0,
+	0x31f40010,
+	0x0117f010,
+	0xd03807f0,
+	0x04bd0001,
+/* 0x0397: init_proc */
+	0x9858f7f0,
+	0x16b001f1,
+	0xfa0bf400,
+	0xf0b615f9,
+	0xf20ef458,
+/* 0x03a8: mulu32_32_64 */
+	0x20f910f9,
+	0x40f930f9,
+	0x9510e195,
+	0xc4bd10d2,
+	0xedffb4bd,
+	0x301dffc0,
+	0xf10234b9,
+	0xb6ffff34,
+	0x45b61034,
+	0x00c3bb10,
+	0xff01b4bb,
+	0x34b930e2,
+	0xff34f102,
+	0x1034b6ff,
+	0xbb1045b6,
+	0xb4bb00c3,
+	0x3012ff01,
+	0xfc00b3bb,
+	0xfc30fc40,
+	0xf810fc20,
+/* 0x03f9: host_send */
+	0xb017f100,
+	0x0011cf04,
+	0x04a027f1,
+	0xb80022cf,
+	0x0bf40612,
+	0x071ec42f,
+	0xb704ee94,
+	0x980270e0,
+	0xec9803eb,
+	0x01ed9802,
+	0xf500ee98,
+	0xb602e621,
+	0x1ec40110,
+	0xb007f10f,
+	0x000ed004,
+	0x0ef404bd,
+/* 0x0439: host_send_done */
+/* 0x043b: host_recv */
+	0xf100f8c3,
+	0xf14e4917,
+	0xb8525413,
+	0x0bf406e1,
+/* 0x0449: host_recv_wait */
+	0xcc17f1b3,
+	0x0011cf04,
+	0x04c827f1,
+	0xf00022cf,
+	0x12b80816,
+	0xec0bf406,
+	0xb60723c4,
+	0x30b70434,
+	0x3b8002f0,
+	0x023c8003,
+	0x80013d80,
+	0x20b6003e,
+	0x0f24f001,
+	0x04c807f1,
+	0xbd0002d0,
+	0x4027f004,
+	0xd00007f0,
 	0x04bd0002,
-	0xf04027f0,
-	0x02d00007,
-	0xf804bd00,
-/* 0x049d: host_init */
-	0x8017f100,
-	0x1014b600,
-	0x027015f1,
-	0x04d007f1,
-	0xbd0001d0,
-	0x8017f104,
-	0x1014b600,
-	0x02f015f1,
-	0x04dc07f1,
-	0xbd0001d0,
-	0x0117f004,
-	0x04c407f1,
-	0xbd0001d0,
-/* 0x04d3: memx_func_enter */
-	0xf100f804,
-	0xf1162067,
-	0xf1f55d77,
+/* 0x0492: host_init */
+	0x17f100f8,
+	0x14b60080,
+	0x7015f110,
+	0xd007f102,
+	0x0001d004,
+	0x17f104bd,
+	0x14b60080,
+	0xf015f110,
+	0xdc07f102,
+	0x0001d004,
+	0x17f004bd,
+	0xc407f101,
+	0x0001d004,
+	0x00f804bd,
+/* 0x04c8: memx_func_enter */
+	0x162067f1,
+	0xf55d77f1,
+	0xffff73f1,
+	0xf4026eb9,
+	0xd8b90421,
+	0x0487fd02,
+	0x80f960f9,
+	0xe0fcd0fc,
+	0xf13321f4,
+	0xf1fffe77,
 	0xb9ffff73,
 	0x21f4026e,
 	0x02d8b904,
 	0xf90487fd,
 	0xfc80f960,
 	0xf4e0fcd0,
-	0x77f13321,
-	0x73f1fffe,
-	0x6eb9ffff,
+	0x67f13321,
+	0x6eb926f0,
 	0x0421f402,
 	0xfd02d8b9,
 	0x60f90487,
 	0xd0fc80f9,
 	0x21f4e0fc,
-	0xf067f133,
-	0x026eb926,
-	0xb90421f4,
-	0x87fd02d8,
-	0xf960f904,
-	0xfcd0fc80,
-	0x3321f4e0,
-	0xf10467f0,
-	0xd007e007,
-	0x04bd0006,
-/* 0x053c: memx_func_enter_wait */
-	0x07c067f1,
-	0xf00066cf,
-	0x0bf40464,
-	0x2c67f0f6,
-	0x800066cf,
-	0x00f8f106,
-/* 0x0554: memx_func_leave */
-	0xcf2c67f0,
-	0x06800066,
-	0x0467f0f2,
-	0x07e407f1,
+	0x0467f033,
+	0x07e007f1,
 	0xbd0006d0,
-/* 0x0569: memx_func_leave_wait */
+/* 0x0531: memx_func_enter_wait */
 	0xc067f104,
 	0x0066cf07,
 	0xf40464f0,
-	0x67f1f61b,
-	0x77f126f0,
-	0x73f00001,
-	0x026eb900,
-	0xb90421f4,
-	0x87fd02d8,
-	0xf960f905,
-	0xfcd0fc80,
-	0x3321f4e0,
-	0x162067f1,
-	0xf4026eb9,
-	0xd8b90421,
-	0x0587fd02,
-	0x80f960f9,
-	0xe0fcd0fc,
-	0xf13321f4,
-	0xf00aa277,
+	0x67f0f60b,
+	0x0066cf2c,
+	0xf8f10680,
+/* 0x0549: memx_func_leave */
+	0x2c67f000,
+	0x800066cf,
+	0x67f0f206,
+	0xe407f104,
+	0x0006d007,
+/* 0x055e: memx_func_leave_wait */
+	0x67f104bd,
+	0x66cf07c0,
+	0x0464f000,
+	0xf1f61bf4,
+	0xf126f067,
+	0xf0000177,
 	0x6eb90073,
 	0x0421f402,
 	0xfd02d8b9,
 	0x60f90587,
 	0xd0fc80f9,
 	0x21f4e0fc,
-/* 0x05d3: memx_func_wait_vblank */
-	0xb600f833,
-	0x00f80410,
-/* 0x05d8: memx_func_wr32 */
-	0x98001698,
-	0x10b60115,
-	0xf960f908,
-	0xfcd0fc50,
+	0x2067f133,
+	0x026eb916,
+	0xb90421f4,
+	0x87fd02d8,
+	0xf960f905,
+	0xfcd0fc80,
 	0x3321f4e0,
-	0xf40242b6,
-	0x00f8e91b,
-/* 0x05f4: memx_func_wait */
-	0xcf2c87f0,
-	0x1e980088,
-	0x011d9800,
-	0x98021c98,
-	0x10b6031b,
-	0x8621f410,
-/* 0x060e: memx_func_delay */
-	0x1e9800f8,
+	0x0aa277f1,
+	0xb90073f0,
+	0x21f4026e,
+	0x02d8b904,
+	0xf90587fd,
+	0xfc80f960,
+	0xf4e0fcd0,
+	0x00f83321,
+/* 0x05c8: memx_func_wait_vblank */
+	0xf80410b6,
+/* 0x05cd: memx_func_wr32 */
+	0x00169800,
+	0xb6011598,
+	0x60f90810,
+	0xd0fc50f9,
+	0x21f4e0fc,
+	0x0242b633,
+	0xf8e91bf4,
+/* 0x05e9: memx_func_wait */
+	0x2c87f000,
+	0x980088cf,
+	0x1d98001e,
+	0x021c9801,
+	0xb6031b98,
+	0x21f41010,
+/* 0x0603: memx_func_delay */
+	0x9800f886,
+	0x10b6001e,
+	0x6721f404,
+/* 0x060e: memx_func_train */
+	0x00f800f8,
+/* 0x0610: memx_exec */
+	0xd0f9e0f9,
+	0xb902c1b9,
+/* 0x061a: memx_exec_next */
+	0x139802b2,
 	0x0410b600,
-	0xf86721f4,
-/* 0x0619: memx_func_train */
-/* 0x061b: memx_exec */
-	0xf900f800,
-	0xb9d0f9e0,
-	0xb2b902c1,
-/* 0x0625: memx_exec_next */
-	0x00139802,
-	0xe70410b6,
-	0xe701f034,
-	0xb601e033,
-	0x30f00132,
-	0xde35980c,
-	0x12b855f9,
-	0xe41ef406,
-	0x98f10b98,
-	0xcbbbf20c,
-	0xc4b7f102,
-	0x00bbcf07,
-	0xe0fcd0fc,
-	0x02f121f5,
-/* 0x065e: memx_info */
-	0xc67000f8,
-	0x0e0bf401,
-/* 0x0664: memx_info_data */
-	0x03ccc7f1,
-	0x0800b7f1,
-/* 0x066f: memx_info_train */
-	0xf10b0ef4,
-	0xf10bccc7,
-/* 0x0677: memx_info_send */
-	0xf50100b7,
-	0xf802f121,
-/* 0x067d: memx_recv */
-	0x01d6b000,
-	0xb09b0bf4,
-	0x0bf400d6,
-/* 0x068b: memx_init */
-	0xf800f8d8,
-/* 0x068d: perf_recv */
-/* 0x068f: perf_init */
-	0xf800f800,
-/* 0x0691: i2c_drive_scl */
-	0x0036b000,
-	0xf10e0bf4,
-	0xd007e007,
+	0x01f034e7,
+	0x01e033e7,
+	0xf00132b6,
+	0x35980c30,
+	0xb855f9de,
+	0x1ef40612,
+	0xf10b98e4,
+	0xbbf20c98,
+	0xb7f102cb,
+	0xbbcf07c4,
+	0xfcd0fc00,
+	0xe621f5e0,
+/* 0x0653: memx_info */
+	0x7000f802,
+	0x0bf401c6,
+/* 0x0659: memx_info_data */
+	0xccc7f10e,
+	0x00b7f103,
+	0x0b0ef408,
+/* 0x0664: memx_info_train */
+	0x0bccc7f1,
+	0x0100b7f1,
+/* 0x066c: memx_info_send */
+	0x02e621f5,
+/* 0x0672: memx_recv */
+	0xd6b000f8,
+	0x9b0bf401,
+	0xf400d6b0,
+	0x00f8d80b,
+/* 0x0680: memx_init */
+/* 0x0682: perf_recv */
+	0x00f800f8,
+/* 0x0684: perf_init */
+/* 0x0686: i2c_drive_scl */
+	0x36b000f8,
+	0x0e0bf400,
+	0x07e007f1,
+	0xbd0001d0,
+/* 0x0697: i2c_drive_scl_lo */
+	0xf100f804,
+	0xd007e407,
 	0x04bd0001,
-/* 0x06a2: i2c_drive_scl_lo */
-	0x07f100f8,
-	0x01d007e4,
-	0xf804bd00,
-/* 0x06ad: i2c_drive_sda */
-	0x0036b000,
-	0xf10e0bf4,
-	0xd007e007,
+/* 0x06a2: i2c_drive_sda */
+	0x36b000f8,
+	0x0e0bf400,
+	0x07e007f1,
+	0xbd0002d0,
+/* 0x06b3: i2c_drive_sda_lo */
+	0xf100f804,
+	0xd007e407,
 	0x04bd0002,
-/* 0x06be: i2c_drive_sda_lo */
-	0x07f100f8,
-	0x02d007e4,
-	0xf804bd00,
-/* 0x06c9: i2c_sense_scl */
-	0x0132f400,
-	0x07c437f1,
-	0xfd0033cf,
-	0x0bf40431,
-	0x0131f406,
-/* 0x06dc: i2c_sense_scl_done */
-/* 0x06de: i2c_sense_sda */
+/* 0x06be: i2c_sense_scl */
 	0x32f400f8,
 	0xc437f101,
 	0x0033cf07,
-	0xf40432fd,
+	0xf40431fd,
 	0x31f4060b,
-/* 0x06f1: i2c_sense_sda_done */
-/* 0x06f3: i2c_raise_scl */
-	0xf900f801,
-	0x9847f140,
-	0x0137f008,
-	0x069121f5,
-/* 0x0700: i2c_raise_scl_wait */
-	0x03e8e7f1,
-	0xf56721f4,
-	0xf406c921,
-	0x42b60901,
-	0xef1bf401,
-/* 0x0714: i2c_raise_scl_done */
-	0x00f840fc,
-/* 0x0718: i2c_start */
-	0x06c921f5,
-	0xf50d11f4,
-	0xf406de21,
-	0x0ef40611,
-/* 0x0729: i2c_start_rep */
-	0x0037f030,
-	0x069121f5,
+/* 0x06d1: i2c_sense_scl_done */
+/* 0x06d3: i2c_sense_sda */
+	0xf400f801,
+	0x37f10132,
+	0x33cf07c4,
+	0x0432fd00,
+	0xf4060bf4,
+/* 0x06e6: i2c_sense_sda_done */
+	0x00f80131,
+/* 0x06e8: i2c_raise_scl */
+	0x47f140f9,
+	0x37f00898,
+	0x8621f501,
+/* 0x06f5: i2c_raise_scl_wait */
+	0xe8e7f106,
+	0x6721f403,
+	0x06be21f5,
+	0xb60901f4,
+	0x1bf40142,
+/* 0x0709: i2c_raise_scl_done */
+	0xf840fcef,
+/* 0x070d: i2c_start */
+	0xbe21f500,
+	0x0d11f406,
+	0x06d321f5,
+	0xf40611f4,
+/* 0x071e: i2c_start_rep */
+	0x37f0300e,
+	0x8621f500,
+	0x0137f006,
+	0x06a221f5,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xe821f550,
+	0x0464b606,
+/* 0x074b: i2c_start_send */
+	0xf01f11f4,
+	0x21f50037,
+	0xe7f106a2,
+	0x21f41388,
+	0x0037f067,
+	0x068621f5,
+	0x1388e7f1,
+/* 0x0767: i2c_start_out */
+	0xf86721f4,
+/* 0x0769: i2c_stop */
+	0x0037f000,
+	0x068621f5,
+	0xf50037f0,
+	0xf106a221,
+	0xf403e8e7,
+	0x37f06721,
+	0x8621f501,
+	0x88e7f106,
+	0x6721f413,
 	0xf50137f0,
-	0xbb06ad21,
+	0xf106a221,
+	0xf41388e7,
+	0x00f86721,
+/* 0x079c: i2c_bitw */
+	0x06a221f5,
+	0x03e8e7f1,
+	0xbb6721f4,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x06f321f5,
+	0x06e821f5,
 	0xf40464b6,
-/* 0x0756: i2c_start_send */
-	0x37f01f11,
-	0xad21f500,
-	0x88e7f106,
-	0x6721f413,
-	0xf50037f0,
-	0xf1069121,
-	0xf41388e7,
-/* 0x0772: i2c_start_out */
-	0x00f86721,
-/* 0x0774: i2c_stop */
-	0xf50037f0,
-	0xf0069121,
-	0x21f50037,
-	0xe7f106ad,
-	0x21f403e8,
-	0x0137f067,
-	0x069121f5,
-	0x1388e7f1,
-	0xf06721f4,
-	0x21f50137,
-	0xe7f106ad,
+	0xe7f11811,
 	0x21f41388,
-/* 0x07a7: i2c_bitw */
-	0xf500f867,
-	0xf106ad21,
-	0xf403e8e7,
-	0x76bb6721,
-	0x0465b600,
-	0x659450f9,
-	0x0256bb04,
-	0x75fd50bd,
-	0xf550fc04,
-	0xb606f321,
-	0x11f40464,
-	0x88e7f118,
-	0x6721f413,
-	0xf50037f0,
-	0xf1069121,
-	0xf41388e7,
-/* 0x07e6: i2c_bitw_out */
-	0x00f86721,
-/* 0x07e8: i2c_bitr */
-	0xf50137f0,
-	0xf106ad21,
-	0xf403e8e7,
-	0x76bb6721,
-	0x0465b600,
-	0x659450f9,
-	0x0256bb04,
-	0x75fd50bd,
-	0xf550fc04,
-	0xb606f321,
-	0x11f40464,
-	0xde21f51b,
-	0x0037f006,
-	0x069121f5,
+	0x0037f067,
+	0x068621f5,
 	0x1388e7f1,
-	0xf06721f4,
-	0x31f4013c,
-/* 0x082d: i2c_bitr_done */
-/* 0x082f: i2c_get_byte */
-	0xf000f801,
-	0x47f00057,
-/* 0x0835: i2c_get_byte_next */
-	0x0154b608,
+/* 0x07db: i2c_bitw_out */
+	0xf86721f4,
+/* 0x07dd: i2c_bitr */
+	0x0137f000,
+	0x06a221f5,
+	0x03e8e7f1,
+	0xbb6721f4,
+	0x65b60076,
+	0x9450f904,
+	0x56bb0465,
+	0xfd50bd02,
+	0x50fc0475,
+	0x06e821f5,
+	0xf40464b6,
+	0x21f51b11,
+	0x37f006d3,
+	0x8621f500,
+	0x88e7f106,
+	0x6721f413,
+	0xf4013cf0,
+/* 0x0822: i2c_bitr_done */
+	0x00f80131,
+/* 0x0824: i2c_get_byte */
+	0xf00057f0,
+/* 0x082a: i2c_get_byte_next */
+	0x54b60847,
+	0x0076bb01,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b607dd,
+	0x2b11f404,
+	0xb60553fd,
+	0x1bf40142,
+	0x0137f0d8,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xe821f550,
+	0x9c21f550,
 	0x0464b607,
-	0xfd2b11f4,
-	0x42b60553,
-	0xd81bf401,
-	0xbb0137f0,
+/* 0x0874: i2c_get_byte_done */
+/* 0x0876: i2c_put_byte */
+	0x47f000f8,
+/* 0x0879: i2c_put_byte_next */
+	0x0142b608,
+	0xbb3854ff,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x07a721f5,
-/* 0x087f: i2c_get_byte_done */
-	0xf80464b6,
-/* 0x0881: i2c_put_byte */
-	0x0847f000,
-/* 0x0884: i2c_put_byte_next */
-	0xff0142b6,
-	0x76bb3854,
-	0x0465b600,
-	0x659450f9,
-	0x0256bb04,
-	0x75fd50bd,
-	0xf550fc04,
-	0xb607a721,
-	0x11f40464,
-	0x0046b034,
-	0xbbd81bf4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x07e821f5,
+	0x079c21f5,
 	0xf40464b6,
-	0x76bb0f11,
-	0x0136b000,
-	0xf4061bf4,
-/* 0x08da: i2c_put_byte_done */
-	0x00f80132,
-/* 0x08dc: i2c_addr */
+	0x46b03411,
+	0xd81bf400,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x1821f550,
+	0xdd21f550,
 	0x0464b607,
-	0xe72911f4,
-	0xb6012ec3,
-	0x53fd0134,
-	0x0076bb05,
+	0xbb0f11f4,
+	0x36b00076,
+	0x061bf401,
+/* 0x08cf: i2c_put_byte_done */
+	0xf80132f4,
+/* 0x08d1: i2c_addr */
+	0x0076bb00,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b60881,
-/* 0x0921: i2c_addr_done */
-/* 0x0923: i2c_acquire_addr */
-	0xc700f804,
-	0xe4b6f8ce,
-	0x14e0b705,
-/* 0x092f: i2c_acquire */
-	0xf500f8d0,
-	0xf4092321,
-	0xd9f00421,
+	0x64b6070d,
+	0x2911f404,
+	0x012ec3e7,
+	0xfd0134b6,
+	0x76bb0553,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb6087621,
+/* 0x0916: i2c_addr_done */
+	0x00f80464,
+/* 0x0918: i2c_acquire_addr */
+	0xb6f8cec7,
+	0xe0b705e4,
+	0x00f8d014,
+/* 0x0924: i2c_acquire */
+	0x091821f5,
+	0xf00421f4,
+	0x21f403d9,
+/* 0x0933: i2c_release */
+	0xf500f833,
+	0xf4091821,
+	0xdaf00421,
 	0x3321f403,
-/* 0x093e: i2c_release */
-	0x21f500f8,
-	0x21f40923,
-	0x03daf004,
-	0xf83321f4,
-/* 0x094d: i2c_recv */
-	0x0132f400,
-	0xb6f8c1c7,
-	0x16b00214,
-	0x3a1ff528,
-	0xf413a001,
-	0x0032980c,
-	0x0ccc13a0,
-	0xf4003198,
-	0xd0f90231,
-	0xd0f9e0f9,
-	0x000067f1,
-	0x100063f1,
-	0xbb016792,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x092f21f5,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b31bf5,
-	0xbb0057f0,
+/* 0x0942: i2c_recv */
+	0x32f400f8,
+	0xf8c1c701,
+	0xb00214b6,
+	0x1ff52816,
+	0x13a0013a,
+	0x32980cf4,
+	0xcc13a000,
+	0x0031980c,
+	0xf90231f4,
+	0xf9e0f9d0,
+	0x0067f1d0,
+	0x0063f100,
+	0x01679210,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x2421f550,
+	0x0464b609,
+	0xd6b0d0fc,
+	0xb31bf500,
+	0x0057f000,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xd121f550,
+	0x0464b608,
+	0x00d011f5,
+	0xbbe0c5c7,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x08dc21f5,
+	0x087621f5,
 	0xf50464b6,
-	0xc700d011,
-	0x76bbe0c5,
+	0xf000ad11,
+	0x76bb0157,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb6088121,
+	0xb608d121,
 	0x11f50464,
-	0x57f000ad,
-	0x0076bb01,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b608dc,
-	0x8a11f504,
-	0x0076bb00,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b6082f,
-	0x6a11f404,
-	0xbbe05bcb,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x077421f5,
-	0xb90464b6,
-	0x74bd025b,
-/* 0x0a53: i2c_recv_not_rd08 */
-	0xb0430ef4,
-	0x1bf401d6,
-	0x0057f03d,
-	0x08dc21f5,
-	0xc73311f4,
-	0x21f5e0c5,
-	0x11f40881,
-	0x0057f029,
-	0x08dc21f5,
-	0xc71f11f4,
-	0x21f5e0b5,
-	0x11f40881,
-	0x7421f515,
-	0xc774bd07,
-	0x1bf408c5,
-	0x0232f409,
-/* 0x0a93: i2c_recv_not_wr08 */
-/* 0x0a93: i2c_recv_done */
-	0xc7030ef4,
-	0x21f5f8ce,
-	0xe0fc093e,
-	0x12f4d0fc,
-	0x027cb90a,
-	0x02f121f5,
-/* 0x0aa8: i2c_recv_exit */
-/* 0x0aaa: i2c_init */
-	0x00f800f8,
-/* 0x0aac: test_recv */
-	0x05d817f1,
-	0xb60011cf,
-	0x07f10110,
-	0x01d005d8,
-	0xf104bd00,
-	0xf1d900e7,
-	0xf5134fe3,
-	0xf8022321,
-/* 0x0acd: test_init */
-	0x00e7f100,
-	0x2321f508,
-/* 0x0ad7: idle_recv */
+	0x76bb008a,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb6082421,
+	0x11f40464,
+	0xe05bcb6a,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x6921f550,
+	0x0464b607,
+	0xbd025bb9,
+	0x430ef474,
+/* 0x0a48: i2c_recv_not_rd08 */
+	0xf401d6b0,
+	0x57f03d1b,
+	0xd121f500,
+	0x3311f408,
+	0xf5e0c5c7,
+	0xf4087621,
+	0x57f02911,
+	0xd121f500,
+	0x1f11f408,
+	0xf5e0b5c7,
+	0xf4087621,
+	0x21f51511,
+	0x74bd0769,
+	0xf408c5c7,
+	0x32f4091b,
+	0x030ef402,
+/* 0x0a88: i2c_recv_not_wr08 */
+/* 0x0a88: i2c_recv_done */
+	0xf5f8cec7,
+	0xfc093321,
+	0xf4d0fce0,
+	0x7cb90a12,
+	0xe621f502,
+/* 0x0a9d: i2c_recv_exit */
+/* 0x0a9f: i2c_init */
 	0xf800f802,
-/* 0x0ad9: idle */
-	0x0031f400,
-	0x05d417f1,
-	0xb60011cf,
-	0x07f10110,
-	0x01d005d4,
-/* 0x0aef: idle_loop */
-	0xf004bd00,
-	0x32f45817,
-/* 0x0af5: idle_proc */
-/* 0x0af5: idle_proc_exec */
-	0xb910f902,
-	0x21f5021e,
-	0x10fc02fa,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0b09: idle_proc_next */
-	0x5810b6ef,
-	0xf4061fb8,
-	0x02f4e61b,
-	0x0028f4dd,
-	0x00c10ef4,
+/* 0x0aa1: test_recv */
+	0xd817f100,
+	0x0011cf05,
+	0xf10110b6,
+	0xd005d807,
+	0x04bd0001,
+	0xd900e7f1,
+	0x134fe3f1,
+	0x021821f5,
+/* 0x0ac2: test_init */
+	0xe7f100f8,
+	0x21f50800,
+	0x00f80218,
+/* 0x0acc: idle_recv */
+/* 0x0ace: idle */
+	0x31f400f8,
+	0xd417f100,
+	0x0011cf05,
+	0xf10110b6,
+	0xd005d407,
+	0x04bd0001,
+/* 0x0ae4: idle_loop */
+	0xf45817f0,
+/* 0x0aea: idle_proc */
+/* 0x0aea: idle_proc_exec */
+	0x10f90232,
+	0xf5021eb9,
+	0xfc02ef21,
+	0x0911f410,
+	0xf40231f4,
+/* 0x0afe: idle_proc_next */
+	0x10b6ef0e,
+	0x061fb858,
+	0xf4e61bf4,
+	0x28f4dd02,
+	0xc10ef400,
+	0x00000000,
+	0x00000000,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
index fe4f63deeaab..8a2b628642ac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
@@ -24,8 +24,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 /* 0x0058: proc_list_head */
 	0x54534f48,
-	0x00000453,
-	0x00000404,
+	0x00000447,
+	0x000003f8,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -46,8 +46,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x0000062d,
-	0x0000061f,
+	0x00000621,
+	0x00000613,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -68,8 +68,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x00000631,
-	0x0000062f,
+	0x00000625,
+	0x00000623,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -90,8 +90,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000a35,
-	0x000008dc,
+	0x00000a29,
+	0x000008d0,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -112,8 +112,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000a56,
-	0x00000a37,
+	0x00000a4a,
+	0x00000a2b,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -134,8 +134,8 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000a61,
-	0x00000a5f,
+	0x00000a55,
+	0x00000a53,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -229,26 +229,26 @@ uint32_t gk208_pmu_data[] = {
 /* 0x0370: memx_func_head */
 	0x00000001,
 	0x00000000,
-	0x00000483,
+	0x00000477,
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x00000500,
+	0x000004f4,
 	0x00000003,
 	0x00000002,
-	0x00000580,
+	0x00000574,
 	0x00040004,
 	0x00000000,
-	0x0000059d,
+	0x00000591,
 	0x00010005,
 	0x00000000,
-	0x000005b7,
+	0x000005ab,
 	0x00010006,
 	0x00000000,
-	0x0000057b,
+	0x0000056f,
 	0x00000007,
 	0x00000000,
-	0x000005c3,
+	0x000005b7,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -916,7 +916,7 @@ uint32_t gk208_pmu_data[] = {
 };
 
 uint32_t gk208_pmu_code[] = {
-	0x031c0ef5,
+	0x03100ef5,
 /* 0x0004: rd32 */
 	0xf607a040,
 	0x04bd000e,
@@ -972,7 +972,7 @@ uint32_t gk208_pmu_code[] = {
 	0x0a98280b,
 	0x029abb9a,
 	0x0d0e1cf4,
-	0x02617e01,
+	0x02557e01,
 	0xf494bd00,
 /* 0x00c2: intr_watchdog_next_time */
 	0x0a98140e,
@@ -1017,21 +1017,16 @@ uint32_t gk208_pmu_code[] = {
 	0xc0f900cc,
 	0xf14f484e,
 	0x0d5453e3,
-	0x02c27e00,
+	0x02b67e00,
 	0x40c0fc00,
 	0x0cf604c0,
 /* 0x0167: intr_subintr_skip_fifo */
 	0x4004bd00,
 	0x09f60688,
 /* 0x016f: intr_skip_subintr */
-	0xc404bd00,
-	0x0bf42089,
-	0xbfa4f107,
-/* 0x0179: intr_skip_pause */
-	0x4089c4ff,
-	0xf1070bf4,
-/* 0x0183: intr_skip_user0 */
-	0x00ffbfa4,
+	0x4904bd00,
+	0x90bd00e0,
+	0x000489fd,
 	0x0008f604,
 	0x80fc04bd,
 	0xfc0088fe,
@@ -1040,35 +1035,35 @@ uint32_t gk208_pmu_code[] = {
 	0xfca0fcb0,
 	0xfc80fc90,
 	0x0032f400,
-/* 0x01a6: ticks_from_ns */
+/* 0x019a: ticks_from_ns */
 	0xc0f901f8,
 	0xd7f1b0f9,
 	0xd3f00144,
-	0x7721f500,
+	0x6b21f500,
 	0xe8ccec03,
 	0x00b4b003,
 	0xec120bf4,
 	0xf103e8ee,
 	0xf00144d7,
 	0x21f500d3,
-/* 0x01ce: ticks_from_ns_quit */
-	0xceb20377,
+/* 0x01c2: ticks_from_ns_quit */
+	0xceb2036b,
 	0xc0fcb0fc,
-/* 0x01d6: ticks_from_us */
+/* 0x01ca: ticks_from_us */
 	0xc0f900f8,
 	0xd7f1b0f9,
 	0xd3f00144,
-	0x7721f500,
+	0x6b21f500,
 	0xb0ceb203,
 	0x0bf400b4,
-/* 0x01ef: ticks_from_us_quit */
+/* 0x01e3: ticks_from_us_quit */
 	0xfce4bd05,
 	0xf8c0fcb0,
-/* 0x01f5: ticks_to_us */
+/* 0x01e9: ticks_to_us */
 	0x44d7f100,
 	0x00d3f001,
 	0xf8ecedff,
-/* 0x0201: timer */
+/* 0x01f5: timer */
 	0xf990f900,
 	0x1032f480,
 	0xb003f898,
@@ -1086,17 +1081,17 @@ uint32_t gk208_pmu_code[] = {
 	0xa60088cf,
 	0x080bf4e0,
 	0x1cf4e8a6,
-/* 0x0245: timer_reset */
+/* 0x0239: timer_reset */
 	0xf634000d,
 	0x04bd000e,
-/* 0x024f: timer_enable */
+/* 0x0243: timer_enable */
 	0x089a0eb5,
 	0xf6380001,
 	0x04bd0008,
-/* 0x0258: timer_done */
+/* 0x024c: timer_done */
 	0xfc1031f4,
 	0xf890fc80,
-/* 0x0261: send_proc */
+/* 0x0255: send_proc */
 	0xf980f900,
 	0x05e89890,
 	0xf004e998,
@@ -1111,24 +1106,24 @@ uint32_t gk208_pmu_code[] = {
 	0x90b6038b,
 	0x0794f001,
 	0xf404e9b5,
-/* 0x029a: send_done */
+/* 0x028e: send_done */
 	0x90fc0231,
 	0x00f880fc,
-/* 0x02a0: find */
+/* 0x0294: find */
 	0x580880f9,
-/* 0x02a7: find_loop */
+/* 0x029b: find_loop */
 	0x980131f4,
 	0xaea6008a,
 	0xb6100bf4,
 	0x86b15880,
 	0x1bf40268,
 	0x0132f4f1,
-/* 0x02bc: find_done */
+/* 0x02b0: find_done */
 	0x80fc8eb2,
-/* 0x02c2: send */
-	0xa07e00f8,
+/* 0x02b6: send */
+	0x947e00f8,
 	0x01f40002,
-/* 0x02cb: recv */
+/* 0x02bf: recv */
 	0xf900f89b,
 	0x9880f990,
 	0xe99805e8,
@@ -1148,10 +1143,10 @@ uint32_t gk208_pmu_code[] = {
 	0xa5f900ee,
 	0xf8fef0fc,
 	0x0131f400,
-/* 0x0316: recv_done */
+/* 0x030a: recv_done */
 	0x80fcf0fc,
 	0x00f890fc,
-/* 0x031c: init */
+/* 0x0310: init */
 	0xcf010841,
 	0x11e70011,
 	0x14b60109,
@@ -1170,12 +1165,12 @@ uint32_t gk208_pmu_code[] = {
 	0x011031f4,
 	0xf6380001,
 	0x04bd0001,
-/* 0x0366: init_proc */
+/* 0x035a: init_proc */
 	0xf198580f,
 	0x0016b001,
 	0xf9fa0bf4,
 	0x58f0b615,
-/* 0x0377: mulu32_32_64 */
+/* 0x036b: mulu32_32_64 */
 	0xf9f20ef4,
 	0xf920f910,
 	0x9540f930,
@@ -1196,7 +1191,7 @@ uint32_t gk208_pmu_code[] = {
 	0x00b3bb30,
 	0x30fc40fc,
 	0x10fc20fc,
-/* 0x03c6: host_send */
+/* 0x03ba: host_send */
 	0xb04100f8,
 	0x0011cf04,
 	0xcf04a042,
@@ -1207,18 +1202,18 @@ uint32_t gk208_pmu_code[] = {
 	0x03eb9802,
 	0x9802ec98,
 	0xee9801ed,
-	0x02c27e00,
+	0x02b67e00,
 	0x0110b600,
 	0x400f1ec4,
 	0x0ef604b0,
 	0xf404bd00,
-/* 0x0402: host_send_done */
+/* 0x03f6: host_send_done */
 	0x00f8c70e,
-/* 0x0404: host_recv */
+/* 0x03f8: host_recv */
 	0xf14e4941,
 	0xa6525413,
 	0xb90bf4e1,
-/* 0x0410: host_recv_wait */
+/* 0x0404: host_recv_wait */
 	0xcf04cc41,
 	0xc8420011,
 	0x0022cf04,
@@ -1235,7 +1230,7 @@ uint32_t gk208_pmu_code[] = {
 	0x04bd0002,
 	0x00004002,
 	0xbd0002f6,
-/* 0x0453: host_init */
+/* 0x0447: host_init */
 	0x4100f804,
 	0x14b60080,
 	0x7015f110,
@@ -1248,7 +1243,7 @@ uint32_t gk208_pmu_code[] = {
 	0x0104bd00,
 	0x04c44001,
 	0xbd0001f6,
-/* 0x0483: memx_func_enter */
+/* 0x0477: memx_func_enter */
 	0xf100f804,
 	0xf1162067,
 	0xf1f55d77,
@@ -1275,19 +1270,19 @@ uint32_t gk208_pmu_code[] = {
 	0x00002e7e,
 	0xe0400406,
 	0x0006f607,
-/* 0x04ea: memx_func_enter_wait */
+/* 0x04de: memx_func_enter_wait */
 	0xc04604bd,
 	0x0066cf07,
 	0xf40464f0,
 	0x2c06f70b,
 	0xb50066cf,
 	0x00f8f106,
-/* 0x0500: memx_func_leave */
+/* 0x04f4: memx_func_leave */
 	0x66cf2c06,
 	0xf206b500,
 	0xe4400406,
 	0x0006f607,
-/* 0x0512: memx_func_leave_wait */
+/* 0x0506: memx_func_leave_wait */
 	0xc04604bd,
 	0x0066cf07,
 	0xf40464f0,
@@ -1314,10 +1309,10 @@ uint32_t gk208_pmu_code[] = {
 	0xf960f905,
 	0xfcd0fc80,
 	0x002e7ee0,
-/* 0x057b: memx_func_wait_vblank */
+/* 0x056f: memx_func_wait_vblank */
 	0xb600f800,
 	0x00f80410,
-/* 0x0580: memx_func_wr32 */
+/* 0x0574: memx_func_wr32 */
 	0x98001698,
 	0x10b60115,
 	0xf960f908,
@@ -1325,23 +1320,23 @@ uint32_t gk208_pmu_code[] = {
 	0x002e7ee0,
 	0x0242b600,
 	0xf8e81bf4,
-/* 0x059d: memx_func_wait */
+/* 0x0591: memx_func_wait */
 	0xcf2c0800,
 	0x1e980088,
 	0x011d9800,
 	0x98021c98,
 	0x10b6031b,
 	0x00797e10,
-/* 0x05b7: memx_func_delay */
+/* 0x05ab: memx_func_delay */
 	0x9800f800,
 	0x10b6001e,
 	0x005d7e04,
-/* 0x05c3: memx_func_train */
+/* 0x05b7: memx_func_train */
 	0xf800f800,
-/* 0x05c5: memx_exec */
+/* 0x05b9: memx_exec */
 	0xf9e0f900,
 	0xb2c1b2d0,
-/* 0x05cd: memx_exec_next */
+/* 0x05c1: memx_exec_next */
 	0x001398b2,
 	0xe70410b6,
 	0xe701f034,
@@ -1354,111 +1349,111 @@ uint32_t gk208_pmu_code[] = {
 	0x02cbbbf2,
 	0xcf07c44b,
 	0xd0fc00bb,
-	0xc27ee0fc,
+	0xb67ee0fc,
 	0x00f80002,
-/* 0x0604: memx_info */
+/* 0x05f8: memx_info */
 	0xf401c670,
-/* 0x060a: memx_info_data */
+/* 0x05fe: memx_info_data */
 	0xcc4c0c0b,
 	0x08004b03,
-/* 0x0613: memx_info_train */
+/* 0x0607: memx_info_train */
 	0x4c090ef4,
 	0x004b0bcc,
-/* 0x0619: memx_info_send */
-	0x02c27e01,
-/* 0x061f: memx_recv */
+/* 0x060d: memx_info_send */
+	0x02b67e01,
+/* 0x0613: memx_recv */
 	0xb000f800,
 	0x0bf401d6,
 	0x00d6b0a3,
 	0xf8dc0bf4,
-/* 0x062d: memx_init */
-/* 0x062f: perf_recv */
+/* 0x0621: memx_init */
+/* 0x0623: perf_recv */
 	0xf800f800,
-/* 0x0631: perf_init */
-/* 0x0633: i2c_drive_scl */
+/* 0x0625: perf_init */
+/* 0x0627: i2c_drive_scl */
 	0xb000f800,
 	0x0bf40036,
 	0x07e0400d,
 	0xbd0001f6,
-/* 0x0643: i2c_drive_scl_lo */
+/* 0x0637: i2c_drive_scl_lo */
 	0x4000f804,
 	0x01f607e4,
 	0xf804bd00,
-/* 0x064d: i2c_drive_sda */
+/* 0x0641: i2c_drive_sda */
 	0x0036b000,
 	0x400d0bf4,
 	0x02f607e0,
 	0xf804bd00,
-/* 0x065d: i2c_drive_sda_lo */
+/* 0x0651: i2c_drive_sda_lo */
 	0x07e44000,
 	0xbd0002f6,
-/* 0x0667: i2c_sense_scl */
+/* 0x065b: i2c_sense_scl */
 	0xf400f804,
 	0xc4430132,
 	0x0033cf07,
 	0xf40431fd,
 	0x31f4060b,
-/* 0x0679: i2c_sense_scl_done */
-/* 0x067b: i2c_sense_sda */
+/* 0x066d: i2c_sense_scl_done */
+/* 0x066f: i2c_sense_sda */
 	0xf400f801,
 	0xc4430132,
 	0x0033cf07,
 	0xf40432fd,
 	0x31f4060b,
-/* 0x068d: i2c_sense_sda_done */
-/* 0x068f: i2c_raise_scl */
+/* 0x0681: i2c_sense_sda_done */
+/* 0x0683: i2c_raise_scl */
 	0xf900f801,
 	0x08984440,
-	0x337e0103,
-/* 0x069a: i2c_raise_scl_wait */
+	0x277e0103,
+/* 0x068e: i2c_raise_scl_wait */
 	0xe84e0006,
 	0x005d7e03,
-	0x06677e00,
+	0x065b7e00,
 	0x0901f400,
 	0xf40142b6,
-/* 0x06ae: i2c_raise_scl_done */
+/* 0x06a2: i2c_raise_scl_done */
 	0x40fcef1b,
-/* 0x06b2: i2c_start */
-	0x677e00f8,
+/* 0x06a6: i2c_start */
+	0x5b7e00f8,
 	0x11f40006,
-	0x067b7e0d,
+	0x066f7e0d,
 	0x0611f400,
-/* 0x06c3: i2c_start_rep */
+/* 0x06b7: i2c_start_rep */
 	0x032e0ef4,
-	0x06337e00,
+	0x06277e00,
 	0x7e010300,
-	0xbb00064d,
+	0xbb000641,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x00068f7e,
+	0x0006837e,
 	0xf40464b6,
-/* 0x06ee: i2c_start_send */
+/* 0x06e2: i2c_start_send */
 	0x00031d11,
-	0x00064d7e,
+	0x0006417e,
 	0x7e13884e,
 	0x0300005d,
-	0x06337e00,
+	0x06277e00,
 	0x13884e00,
 	0x00005d7e,
-/* 0x0708: i2c_start_out */
-/* 0x070a: i2c_stop */
+/* 0x06fc: i2c_start_out */
+/* 0x06fe: i2c_stop */
 	0x000300f8,
-	0x0006337e,
-	0x4d7e0003,
+	0x0006277e,
+	0x417e0003,
 	0xe84e0006,
 	0x005d7e03,
 	0x7e010300,
-	0x4e000633,
+	0x4e000627,
 	0x5d7e1388,
 	0x01030000,
-	0x00064d7e,
+	0x0006417e,
 	0x7e13884e,
 	0xf800005d,
-/* 0x0739: i2c_bitw */
-	0x064d7e00,
+/* 0x072d: i2c_bitw */
+	0x06417e00,
 	0x03e84e00,
 	0x00005d7e,
 	0xb60076bb,
@@ -1466,18 +1461,18 @@ uint32_t gk208_pmu_code[] = {
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x068f7e50,
+	0x06837e50,
 	0x0464b600,
 	0x4e1711f4,
 	0x5d7e1388,
 	0x00030000,
-	0x0006337e,
+	0x0006277e,
 	0x7e13884e,
-/* 0x0777: i2c_bitw_out */
+/* 0x076b: i2c_bitw_out */
 	0xf800005d,
-/* 0x0779: i2c_bitr */
+/* 0x076d: i2c_bitr */
 	0x7e010300,
-	0x4e00064d,
+	0x4e000641,
 	0x5d7e03e8,
 	0x76bb0000,
 	0x0465b600,
@@ -1485,25 +1480,25 @@ uint32_t gk208_pmu_code[] = {
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb600068f,
+	0xb6000683,
 	0x11f40464,
-	0x067b7e1a,
+	0x066f7e1a,
 	0x7e000300,
-	0x4e000633,
+	0x4e000627,
 	0x5d7e1388,
 	0x3cf00000,
 	0x0131f401,
-/* 0x07bc: i2c_bitr_done */
-/* 0x07be: i2c_get_byte */
+/* 0x07b0: i2c_bitr_done */
+/* 0x07b2: i2c_get_byte */
 	0x000500f8,
-/* 0x07c2: i2c_get_byte_next */
+/* 0x07b6: i2c_get_byte_next */
 	0x54b60804,
 	0x0076bb01,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
-	0x797e50fc,
+	0x6d7e50fc,
 	0x64b60007,
 	0x2a11f404,
 	0xb60553fd,
@@ -1514,11 +1509,11 @@ uint32_t gk208_pmu_code[] = {
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0007397e,
-/* 0x080b: i2c_get_byte_done */
+	0x00072d7e,
+/* 0x07ff: i2c_get_byte_done */
 	0xf80464b6,
-/* 0x080d: i2c_put_byte */
-/* 0x080f: i2c_put_byte_next */
+/* 0x0801: i2c_put_byte */
+/* 0x0803: i2c_put_byte_next */
 	0xb6080400,
 	0x54ff0142,
 	0x0076bb38,
@@ -1526,7 +1521,7 @@ uint32_t gk208_pmu_code[] = {
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
-	0x397e50fc,
+	0x2d7e50fc,
 	0x64b60007,
 	0x3411f404,
 	0xf40046b0,
@@ -1536,20 +1531,20 @@ uint32_t gk208_pmu_code[] = {
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb6000779,
+	0xb600076d,
 	0x11f40464,
 	0x0076bb0f,
 	0xf40136b0,
 	0x32f4061b,
-/* 0x0865: i2c_put_byte_done */
-/* 0x0867: i2c_addr */
+/* 0x0859: i2c_put_byte_done */
+/* 0x085b: i2c_addr */
 	0xbb00f801,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0006b27e,
+	0x0006a67e,
 	0xf40464b6,
 	0xc3e72911,
 	0x34b6012e,
@@ -1559,25 +1554,25 @@ uint32_t gk208_pmu_code[] = {
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x080d7e50,
+	0x08017e50,
 	0x0464b600,
-/* 0x08ac: i2c_addr_done */
-/* 0x08ae: i2c_acquire_addr */
+/* 0x08a0: i2c_addr_done */
+/* 0x08a2: i2c_acquire_addr */
 	0xcec700f8,
 	0x05e4b6f8,
 	0xd014e0b7,
-/* 0x08ba: i2c_acquire */
-	0xae7e00f8,
+/* 0x08ae: i2c_acquire */
+	0xa27e00f8,
 	0x047e0008,
 	0xd9f00000,
 	0x002e7e03,
-/* 0x08cb: i2c_release */
+/* 0x08bf: i2c_release */
 	0x7e00f800,
-	0x7e0008ae,
+	0x7e0008a2,
 	0xf0000004,
 	0x2e7e03da,
 	0x00f80000,
-/* 0x08dc: i2c_recv */
+/* 0x08d0: i2c_recv */
 	0xc70132f4,
 	0x14b6f8c1,
 	0x2816b002,
@@ -1596,7 +1591,7 @@ uint32_t gk208_pmu_code[] = {
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x08ba7e50,
+	0x08ae7e50,
 	0x0464b600,
 	0xd6b0d0fc,
 	0xb01bf500,
@@ -1606,7 +1601,7 @@ uint32_t gk208_pmu_code[] = {
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0008677e,
+	0x00085b7e,
 	0xf50464b6,
 	0xc700cc11,
 	0x76bbe0c5,
@@ -1615,7 +1610,7 @@ uint32_t gk208_pmu_code[] = {
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb600080d,
+	0xb6000801,
 	0x11f50464,
 	0x010500a9,
 	0xb60076bb,
@@ -1623,7 +1618,7 @@ uint32_t gk208_pmu_code[] = {
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x08677e50,
+	0x085b7e50,
 	0x0464b600,
 	0x008711f5,
 	0xb60076bb,
@@ -1631,7 +1626,7 @@ uint32_t gk208_pmu_code[] = {
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x07be7e50,
+	0x07b27e50,
 	0x0464b600,
 	0xcb6711f4,
 	0x76bbe05b,
@@ -1640,36 +1635,36 @@ uint32_t gk208_pmu_code[] = {
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb600070a,
+	0xb60006fe,
 	0x5bb20464,
 	0x0ef474bd,
-/* 0x09e1: i2c_recv_not_rd08 */
+/* 0x09d5: i2c_recv_not_rd08 */
 	0x01d6b041,
 	0x053b1bf4,
-	0x08677e00,
+	0x085b7e00,
 	0x3211f400,
 	0x7ee0c5c7,
-	0xf400080d,
+	0xf4000801,
 	0x00052811,
-	0x0008677e,
+	0x00085b7e,
 	0xc71f11f4,
-	0x0d7ee0b5,
+	0x017ee0b5,
 	0x11f40008,
-	0x070a7e15,
+	0x06fe7e15,
 	0xc774bd00,
 	0x1bf408c5,
 	0x0232f409,
-/* 0x0a1f: i2c_recv_not_wr08 */
-/* 0x0a1f: i2c_recv_done */
+/* 0x0a13: i2c_recv_not_wr08 */
+/* 0x0a13: i2c_recv_done */
 	0xc7030ef4,
-	0xcb7ef8ce,
+	0xbf7ef8ce,
 	0xe0fc0008,
 	0x12f4d0fc,
 	0x7e7cb209,
-/* 0x0a33: i2c_recv_exit */
-	0xf80002c2,
-/* 0x0a35: i2c_init */
-/* 0x0a37: test_recv */
+/* 0x0a27: i2c_recv_exit */
+	0xf80002b6,
+/* 0x0a29: i2c_init */
+/* 0x0a2b: test_recv */
 	0x4100f800,
 	0x11cf0458,
 	0x0110b600,
@@ -1677,28 +1672,28 @@ uint32_t gk208_pmu_code[] = {
 	0x04bd0001,
 	0xd900e7f1,
 	0x134fe3f1,
-	0x0002017e,
-/* 0x0a56: test_init */
+	0x0001f57e,
+/* 0x0a4a: test_init */
 	0x004e00f8,
-	0x02017e08,
-/* 0x0a5f: idle_recv */
+	0x01f57e08,
+/* 0x0a53: idle_recv */
 	0xf800f800,
-/* 0x0a61: idle */
+/* 0x0a55: idle */
 	0x0031f400,
 	0xcf045441,
 	0x10b60011,
 	0x04544001,
 	0xbd0001f6,
-/* 0x0a75: idle_loop */
+/* 0x0a69: idle_loop */
 	0xf4580104,
-/* 0x0a7a: idle_proc */
-/* 0x0a7a: idle_proc_exec */
+/* 0x0a6e: idle_proc */
+/* 0x0a6e: idle_proc_exec */
 	0x10f90232,
-	0xcb7e1eb2,
+	0xbf7e1eb2,
 	0x10fc0002,
 	0xf40911f4,
 	0x0ef40231,
-/* 0x0a8d: idle_proc_next */
+/* 0x0a81: idle_proc_next */
 	0x5810b6f0,
 	0x1bf41fa6,
 	0xe002f4e8,
@@ -1728,4 +1723,7 @@ uint32_t gk208_pmu_code[] = {
 	0x00000000,
 	0x00000000,
 	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
index 2686f8fad0f5..516569270bac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
@@ -24,8 +24,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 /* 0x0058: proc_list_head */
 	0x54534f48,
-	0x00000512,
-	0x000004af,
+	0x00000507,
+	0x000004a4,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -46,8 +46,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x00000842,
-	0x00000834,
+	0x00000837,
+	0x00000829,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -68,8 +68,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x00000846,
-	0x00000844,
+	0x0000083b,
+	0x00000839,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -90,8 +90,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000c76,
-	0x00000b19,
+	0x00000c6b,
+	0x00000b0e,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -112,8 +112,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000c9f,
-	0x00000c78,
+	0x00000c94,
+	0x00000c6d,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -134,8 +134,8 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000cab,
-	0x00000ca9,
+	0x00000ca0,
+	0x00000c9e,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -229,26 +229,26 @@ uint32_t gt215_pmu_data[] = {
 /* 0x0370: memx_func_head */
 	0x00000001,
 	0x00000000,
-	0x00000551,
+	0x00000546,
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x000005a8,
+	0x0000059d,
 	0x00000003,
 	0x00000002,
-	0x0000063a,
+	0x0000062f,
 	0x00040004,
 	0x00000000,
-	0x00000656,
+	0x0000064b,
 	0x00010005,
 	0x00000000,
-	0x00000673,
+	0x00000668,
 	0x00010006,
 	0x00000000,
-	0x000005f8,
+	0x000005ed,
 	0x00000007,
 	0x00000000,
-	0x0000067e,
+	0x00000673,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -917,7 +917,7 @@ uint32_t gt215_pmu_data[] = {
 };
 
 uint32_t gt215_pmu_code[] = {
-	0x039e0ef5,
+	0x03930ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
 	0xd00604b6,
@@ -987,7 +987,7 @@ uint32_t gt215_pmu_code[] = {
 	0xbb9a0a98,
 	0x1cf4029a,
 	0x01d7f00f,
-	0x02dd21f5,
+	0x02d221f5,
 	0x0ef494bd,
 /* 0x00f9: intr_watchdog_next_time */
 	0x9b0a9815,
@@ -1039,7 +1039,7 @@ uint32_t gt215_pmu_code[] = {
 	0x48e7f1c0,
 	0x53e3f14f,
 	0x00d7f054,
-	0x034221f5,
+	0x033721f5,
 	0x07f1c0fc,
 	0x04b604c0,
 	0x000cd006,
@@ -1048,820 +1048,818 @@ uint32_t gt215_pmu_code[] = {
 	0x04b60688,
 	0x0009d006,
 /* 0x01ca: intr_skip_subintr */
-	0x89c404bd,
-	0x070bf420,
-	0xffbfa4f1,
-/* 0x01d4: intr_skip_pause */
-	0xf44089c4,
-	0xa4f1070b,
-/* 0x01de: intr_skip_user0 */
-	0x07f0ffbf,
-	0x0604b604,
-	0xbd0008d0,
-	0xfe80fc04,
-	0xf0fc0088,
-	0xd0fce0fc,
-	0xb0fcc0fc,
-	0x90fca0fc,
-	0x00fc80fc,
-	0xf80032f4,
-/* 0x0205: ticks_from_ns */
-	0xf9c0f901,
-	0xcbd7f1b0,
-	0x00d3f000,
-	0x041321f5,
-	0x03e8ccec,
-	0xf400b4b0,
-	0xeeec120b,
-	0xd7f103e8,
-	0xd3f000cb,
-	0x1321f500,
-/* 0x022d: ticks_from_ns_quit */
-	0x02ceb904,
-	0xc0fcb0fc,
-/* 0x0236: ticks_from_us */
-	0xc0f900f8,
+	0x97f104bd,
+	0x90bd00e0,
+	0xf00489fd,
+	0x04b60407,
+	0x0008d006,
+	0x80fc04bd,
+	0xfc0088fe,
+	0xfce0fcf0,
+	0xfcc0fcd0,
+	0xfca0fcb0,
+	0xfc80fc90,
+	0x0032f400,
+/* 0x01fa: ticks_from_ns */
+	0xc0f901f8,
 	0xd7f1b0f9,
 	0xd3f000cb,
-	0x1321f500,
-	0x02ceb904,
-	0xf400b4b0,
-	0xe4bd050b,
-/* 0x0250: ticks_from_us_quit */
-	0xc0fcb0fc,
-/* 0x0256: ticks_to_us */
-	0xd7f100f8,
-	0xd3f000cb,
-	0xecedff00,
-/* 0x0262: timer */
-	0x90f900f8,
-	0x32f480f9,
-	0x03f89810,
-	0xf40086b0,
-	0x84bd651c,
-	0xb63807f0,
-	0x08d00604,
-	0xf004bd00,
-	0x84b63487,
-	0x0088cf06,
-	0xbb9a0998,
-	0xe9bb0298,
-	0x03fe8000,
-	0xb60887f0,
-	0x88cf0684,
-	0x0284f000,
-	0xf0261bf4,
-	0x84b63487,
-	0x0088cf06,
-	0xf406e0b8,
-	0xe8b8090b,
-	0x111cf406,
-/* 0x02b8: timer_reset */
-	0xb63407f0,
-	0x0ed00604,
-	0x8004bd00,
-/* 0x02c6: timer_enable */
-	0x87f09a0e,
-	0x3807f001,
+	0x0821f500,
+	0xe8ccec04,
+	0x00b4b003,
+	0xec120bf4,
+	0xf103e8ee,
+	0xf000cbd7,
+	0x21f500d3,
+/* 0x0222: ticks_from_ns_quit */
+	0xceb90408,
+	0xfcb0fc02,
+/* 0x022b: ticks_from_us */
+	0xf900f8c0,
+	0xf1b0f9c0,
+	0xf000cbd7,
+	0x21f500d3,
+	0xceb90408,
+	0x00b4b002,
+	0xbd050bf4,
+/* 0x0245: ticks_from_us_quit */
+	0xfcb0fce4,
+/* 0x024b: ticks_to_us */
+	0xf100f8c0,
+	0xf000cbd7,
+	0xedff00d3,
+/* 0x0257: timer */
+	0xf900f8ec,
+	0xf480f990,
+	0xf8981032,
+	0x0086b003,
+	0xbd651cf4,
+	0x3807f084,
 	0xd00604b6,
 	0x04bd0008,
-/* 0x02d4: timer_done */
-	0xfc1031f4,
-	0xf890fc80,
-/* 0x02dd: send_proc */
-	0xf980f900,
-	0x05e89890,
-	0xf004e998,
-	0x89b80486,
-	0x2a0bf406,
-	0x940398c4,
-	0x80b60488,
-	0x008ebb18,
-	0x8000fa98,
-	0x8d80008a,
-	0x028c8001,
-	0xb6038b80,
-	0x94f00190,
-	0x04e98007,
-/* 0x0317: send_done */
-	0xfc0231f4,
-	0xf880fc90,
-/* 0x031d: find */
-	0xf080f900,
-	0x31f45887,
-/* 0x0325: find_loop */
-	0x008a9801,
-	0xf406aeb8,
-	0x80b6100b,
-	0x6886b158,
-	0xf01bf402,
-/* 0x033b: find_done */
-	0xb90132f4,
-	0x80fc028e,
-/* 0x0342: send */
-	0x21f500f8,
-	0x01f4031d,
-/* 0x034b: recv */
-	0xf900f897,
-	0x9880f990,
-	0xe99805e8,
-	0x0132f404,
-	0xf40689b8,
-	0x89c43d0b,
-	0x0180b603,
-	0x800784f0,
-	0xea9805e8,
-	0xfef0f902,
-	0xf0f9018f,
-	0x9402efb9,
-	0xe9bb0499,
-	0x18e0b600,
-	0x9803eb98,
-	0xed9802ec,
-	0x00ee9801,
-	0xf0fca5f9,
-	0xf400f8fe,
-	0xf0fc0131,
-/* 0x0398: recv_done */
-	0x90fc80fc,
-/* 0x039e: init */
-	0x17f100f8,
-	0x14b60108,
-	0x0011cf06,
-	0x010911e7,
-	0xfe0814b6,
-	0x17f10014,
-	0x13f000e0,
-	0x1c07f000,
-	0xd00604b6,
-	0x04bd0001,
-	0xf0ff17f0,
-	0x04b61407,
-	0x0001d006,
-	0x17f004bd,
-	0x0015f102,
-	0x1007f008,
+	0xb63487f0,
+	0x88cf0684,
+	0x9a099800,
+	0xbb0298bb,
+	0xfe8000e9,
+	0x0887f003,
+	0xcf0684b6,
+	0x84f00088,
+	0x261bf402,
+	0xb63487f0,
+	0x88cf0684,
+	0x06e0b800,
+	0xb8090bf4,
+	0x1cf406e8,
+/* 0x02ad: timer_reset */
+	0x3407f011,
 	0xd00604b6,
-	0x04bd0001,
-	0x011a17f1,
-	0xfe0013f0,
-	0x31f40010,
-	0x0117f010,
-	0xb63807f0,
+	0x04bd000e,
+/* 0x02bb: timer_enable */
+	0xf09a0e80,
+	0x07f00187,
+	0x0604b638,
+	0xbd0008d0,
+/* 0x02c9: timer_done */
+	0x1031f404,
+	0x90fc80fc,
+/* 0x02d2: send_proc */
+	0x80f900f8,
+	0xe89890f9,
+	0x04e99805,
+	0xb80486f0,
+	0x0bf40689,
+	0x0398c42a,
+	0xb6048894,
+	0x8ebb1880,
+	0x00fa9800,
+	0x80008a80,
+	0x8c80018d,
+	0x038b8002,
+	0xf00190b6,
+	0xe9800794,
+	0x0231f404,
+/* 0x030c: send_done */
+	0x80fc90fc,
+/* 0x0312: find */
+	0x80f900f8,
+	0xf45887f0,
+/* 0x031a: find_loop */
+	0x8a980131,
+	0x06aeb800,
+	0xb6100bf4,
+	0x86b15880,
+	0x1bf40268,
+	0x0132f4f0,
+/* 0x0330: find_done */
+	0xfc028eb9,
+/* 0x0337: send */
+	0xf500f880,
+	0xf4031221,
+	0x00f89701,
+/* 0x0340: recv */
+	0x80f990f9,
+	0x9805e898,
+	0x32f404e9,
+	0x0689b801,
+	0xc43d0bf4,
+	0x80b60389,
+	0x0784f001,
+	0x9805e880,
+	0xf0f902ea,
+	0xf9018ffe,
+	0x02efb9f0,
+	0xbb049994,
+	0xe0b600e9,
+	0x03eb9818,
+	0x9802ec98,
+	0xee9801ed,
+	0xfca5f900,
+	0x00f8fef0,
+	0xfc0131f4,
+/* 0x038d: recv_done */
+	0xfc80fcf0,
+/* 0x0393: init */
+	0xf100f890,
+	0xb6010817,
+	0x11cf0614,
+	0x0911e700,
+	0x0814b601,
+	0xf10014fe,
+	0xf000e017,
+	0x07f00013,
+	0x0604b61c,
+	0xbd0001d0,
+	0xff17f004,
+	0xb61407f0,
 	0x01d00604,
 	0xf004bd00,
-/* 0x0402: init_proc */
-	0xf19858f7,
-	0x0016b001,
-	0xf9fa0bf4,
-	0x58f0b615,
-/* 0x0413: mulu32_32_64 */
-	0xf9f20ef4,
-	0xf920f910,
-	0x9540f930,
-	0xd29510e1,
-	0xbdc4bd10,
-	0xc0edffb4,
-	0xb9301dff,
-	0x34f10234,
-	0x34b6ffff,
-	0x1045b610,
-	0xbb00c3bb,
-	0xe2ff01b4,
-	0x0234b930,
-	0xffff34f1,
-	0xb61034b6,
-	0xc3bb1045,
-	0x01b4bb00,
-	0xbb3012ff,
-	0x40fc00b3,
-	0x20fc30fc,
-	0x00f810fc,
-/* 0x0464: host_send */
-	0x04b017f1,
-	0xcf0614b6,
-	0x27f10011,
-	0x24b604a0,
-	0x0022cf06,
-	0xf40612b8,
-	0x1ec4320b,
-	0x04ee9407,
-	0x0270e0b7,
-	0x9803eb98,
-	0xed9802ec,
-	0x00ee9801,
-	0x034221f5,
-	0xc40110b6,
-	0x07f10f1e,
-	0x04b604b0,
-	0x000ed006,
-	0x0ef404bd,
-/* 0x04ad: host_send_done */
-/* 0x04af: host_recv */
-	0xf100f8ba,
-	0xf14e4917,
-	0xb8525413,
-	0x0bf406e1,
-/* 0x04bd: host_recv_wait */
-	0xcc17f1aa,
+	0x15f10217,
+	0x07f00800,
+	0x0604b610,
+	0xbd0001d0,
+	0x1a17f104,
+	0x0013f001,
+	0xf40010fe,
+	0x17f01031,
+	0x3807f001,
+	0xd00604b6,
+	0x04bd0001,
+/* 0x03f7: init_proc */
+	0x9858f7f0,
+	0x16b001f1,
+	0xfa0bf400,
+	0xf0b615f9,
+	0xf20ef458,
+/* 0x0408: mulu32_32_64 */
+	0x20f910f9,
+	0x40f930f9,
+	0x9510e195,
+	0xc4bd10d2,
+	0xedffb4bd,
+	0x301dffc0,
+	0xf10234b9,
+	0xb6ffff34,
+	0x45b61034,
+	0x00c3bb10,
+	0xff01b4bb,
+	0x34b930e2,
+	0xff34f102,
+	0x1034b6ff,
+	0xbb1045b6,
+	0xb4bb00c3,
+	0x3012ff01,
+	0xfc00b3bb,
+	0xfc30fc40,
+	0xf810fc20,
+/* 0x0459: host_send */
+	0xb017f100,
 	0x0614b604,
 	0xf10011cf,
-	0xb604c827,
+	0xb604a027,
 	0x22cf0624,
-	0x0816f000,
-	0xf40612b8,
-	0x23c4e60b,
-	0x0434b607,
-	0x02f030b7,
-	0x80033b80,
-	0x3d80023c,
-	0x003e8001,
-	0xf00120b6,
-	0x07f10f24,
-	0x04b604c8,
-	0x0002d006,
-	0x27f004bd,
-	0x0007f040,
-	0xd00604b6,
-	0x04bd0002,
-/* 0x0512: host_init */
-	0x17f100f8,
+	0x0612b800,
+	0xc4320bf4,
+	0xee94071e,
+	0x70e0b704,
+	0x03eb9802,
+	0x9802ec98,
+	0xee9801ed,
+	0x3721f500,
+	0x0110b603,
+	0xf10f1ec4,
+	0xb604b007,
+	0x0ed00604,
+	0xf404bd00,
+/* 0x04a2: host_send_done */
+	0x00f8ba0e,
+/* 0x04a4: host_recv */
+	0x4e4917f1,
+	0x525413f1,
+	0xf406e1b8,
+/* 0x04b2: host_recv_wait */
+	0x17f1aa0b,
+	0x14b604cc,
+	0x0011cf06,
+	0x04c827f1,
+	0xcf0624b6,
+	0x16f00022,
+	0x0612b808,
+	0xc4e60bf4,
+	0x34b60723,
+	0xf030b704,
+	0x033b8002,
+	0x80023c80,
+	0x3e80013d,
+	0x0120b600,
+	0xf10f24f0,
+	0xb604c807,
+	0x02d00604,
+	0xf004bd00,
+	0x07f04027,
+	0x0604b600,
+	0xbd0002d0,
+/* 0x0507: host_init */
+	0xf100f804,
+	0xb6008017,
+	0x15f11014,
+	0x07f10270,
+	0x04b604d0,
+	0x0001d006,
+	0x17f104bd,
 	0x14b60080,
-	0x7015f110,
-	0xd007f102,
+	0xf015f110,
+	0xdc07f102,
 	0x0604b604,
 	0xbd0001d0,
-	0x8017f104,
-	0x1014b600,
-	0x02f015f1,
-	0x04dc07f1,
+	0x0117f004,
+	0x04c407f1,
 	0xd00604b6,
 	0x04bd0001,
-	0xf10117f0,
-	0xb604c407,
-	0x01d00604,
-	0xf804bd00,
-/* 0x0551: memx_func_enter */
-	0x1087f100,
-	0x028eb916,
-	0xb90421f4,
-	0x67f102d7,
-	0x63f1fffc,
-	0x76fdffff,
-	0x0267f104,
-	0x0576fd00,
-	0x70f980f9,
-	0xe0fcd0fc,
-	0xf03f21f4,
-	0x07f10467,
-	0x04b607e0,
-	0x0006d006,
-/* 0x058a: memx_func_enter_wait */
-	0x67f104bd,
-	0x64b607c0,
-	0x0066cf06,
-	0xf40464f0,
-	0x67f0f30b,
-	0x0664b62c,
-	0x800066cf,
-	0x00f8f106,
-/* 0x05a8: memx_func_leave */
-	0xb62c67f0,
-	0x66cf0664,
-	0xf2068000,
+/* 0x0546: memx_func_enter */
+	0x87f100f8,
+	0x8eb91610,
+	0x0421f402,
+	0xf102d7b9,
+	0xf1fffc67,
+	0xfdffff63,
+	0x67f10476,
+	0x76fd0002,
+	0xf980f905,
+	0xfcd0fc70,
+	0x3f21f4e0,
 	0xf10467f0,
-	0xb607e407,
+	0xb607e007,
 	0x06d00604,
-/* 0x05c3: memx_func_leave_wait */
+/* 0x057f: memx_func_enter_wait */
 	0xf104bd00,
 	0xb607c067,
 	0x66cf0664,
 	0x0464f000,
-	0xf1f31bf4,
-	0xb9161087,
-	0x21f4028e,
-	0x02d7b904,
-	0xffcc67f1,
-	0xffff63f1,
-	0xf90476fd,
-	0xfc70f980,
-	0xf4e0fcd0,
-	0x00f83f21,
-/* 0x05f8: memx_func_wait_vblank */
-	0xb0001698,
-	0x0bf40066,
-	0x0166b013,
-	0xf4060bf4,
-/* 0x060a: memx_func_wait_vblank_head1 */
-	0x77f12e0e,
-	0x0ef40020,
-/* 0x0611: memx_func_wait_vblank_head0 */
-	0x0877f107,
-/* 0x0615: memx_func_wait_vblank_0 */
-	0xc467f100,
-	0x0664b607,
-	0xfd0066cf,
-	0x1bf40467,
-/* 0x0625: memx_func_wait_vblank_1 */
-	0xc467f1f3,
-	0x0664b607,
-	0xfd0066cf,
-	0x0bf40467,
-/* 0x0635: memx_func_wait_vblank_fini */
-	0x0410b6f3,
-/* 0x063a: memx_func_wr32 */
-	0x169800f8,
-	0x01159800,
-	0xf90810b6,
-	0xfc50f960,
-	0xf4e0fcd0,
-	0x42b63f21,
-	0xe91bf402,
-/* 0x0656: memx_func_wait */
-	0x87f000f8,
-	0x0684b62c,
-	0x980088cf,
-	0x1d98001e,
-	0x021c9801,
-	0xb6031b98,
-	0x21f41010,
-/* 0x0673: memx_func_delay */
-	0x9800f8a4,
-	0x10b6001e,
-	0x7f21f404,
-/* 0x067e: memx_func_train */
-	0x57f100f8,
-	0x77f10003,
-	0x97f10000,
-	0x93f00000,
-	0x029eb970,
-	0xb90421f4,
-	0xe7f102d8,
-	0x21f42710,
-/* 0x069d: memx_func_train_loop_outer */
-	0x0158e07f,
-	0x0083f101,
-	0xe097f102,
-	0x1193f011,
-	0x80f990f9,
+	0xf0f30bf4,
+	0x64b62c67,
+	0x0066cf06,
+	0xf8f10680,
+/* 0x059d: memx_func_leave */
+	0x2c67f000,
+	0xcf0664b6,
+	0x06800066,
+	0x0467f0f2,
+	0x07e407f1,
+	0xd00604b6,
+	0x04bd0006,
+/* 0x05b8: memx_func_leave_wait */
+	0x07c067f1,
+	0xcf0664b6,
+	0x64f00066,
+	0xf31bf404,
+	0x161087f1,
+	0xf4028eb9,
+	0xd7b90421,
+	0xcc67f102,
+	0xff63f1ff,
+	0x0476fdff,
+	0x70f980f9,
 	0xe0fcd0fc,
-	0xf93f21f4,
-	0x0067f150,
-/* 0x06bd: memx_func_train_loop_inner */
-	0x1187f100,
-	0x9068ff11,
-	0xfd109894,
-	0x97f10589,
-	0x93f00720,
-	0xf990f910,
-	0xfcd0fc80,
-	0x3f21f4e0,
-	0x008097f1,
-	0xb91093f0,
-	0x21f4029e,
-	0x02d8b904,
-	0xf92088c5,
-	0xfc80f990,
-	0xf4e0fcd0,
-	0x97f13f21,
-	0x93f0053c,
-	0x0287f110,
-	0x0083f130,
-	0xf990f980,
-	0xfcd0fc80,
-	0x3f21f4e0,
-	0x0560e7f1,
-	0xf110e3f0,
-	0xf10000d7,
-	0x908000d3,
-	0xb7f100dc,
-	0xb3f08480,
-	0xa421f41e,
-	0x000057f1,
-	0xffff97f1,
-	0x830093f1,
-/* 0x073c: memx_func_train_loop_4x */
-	0x0080a7f1,
-	0xb910a3f0,
-	0x21f402ae,
-	0x02d8b904,
-	0xffdfb7f1,
-	0xffffb3f1,
-	0xf9048bfd,
-	0xfc80f9a0,
-	0xf4e0fcd0,
-	0xa7f13f21,
-	0xa3f0053c,
-	0x0287f110,
-	0x0083f130,
-	0xf9a0f980,
+	0xf83f21f4,
+/* 0x05ed: memx_func_wait_vblank */
+	0x00169800,
+	0xf40066b0,
+	0x66b0130b,
+	0x060bf401,
+/* 0x05ff: memx_func_wait_vblank_head1 */
+	0xf12e0ef4,
+	0xf4002077,
+/* 0x0606: memx_func_wait_vblank_head0 */
+	0x77f1070e,
+/* 0x060a: memx_func_wait_vblank_0 */
+	0x67f10008,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x061a: memx_func_wait_vblank_1 */
+	0x67f1f31b,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x062a: memx_func_wait_vblank_fini */
+	0x10b6f30b,
+/* 0x062f: memx_func_wr32 */
+	0x9800f804,
+	0x15980016,
+	0x0810b601,
+	0x50f960f9,
+	0xe0fcd0fc,
+	0xb63f21f4,
+	0x1bf40242,
+/* 0x064b: memx_func_wait */
+	0xf000f8e9,
+	0x84b62c87,
+	0x0088cf06,
+	0x98001e98,
+	0x1c98011d,
+	0x031b9802,
+	0xf41010b6,
+	0x00f8a421,
+/* 0x0668: memx_func_delay */
+	0xb6001e98,
+	0x21f40410,
+/* 0x0673: memx_func_train */
+	0xf100f87f,
+	0xf1000357,
+	0xf1000077,
+	0xf0000097,
+	0x9eb97093,
+	0x0421f402,
+	0xf102d8b9,
+	0xf42710e7,
+/* 0x0692: memx_func_train_loop_outer */
+	0x58e07f21,
+	0x83f10101,
+	0x97f10200,
+	0x93f011e0,
+	0xf990f911,
 	0xfcd0fc80,
 	0x3f21f4e0,
-	0x0560e7f1,
-	0xf110e3f0,
-	0xf10000d7,
-	0xb98000d3,
-	0xb7f102dc,
-	0xb3f02710,
-	0xa421f400,
-	0xf402eeb9,
-	0xddb90421,
-	0x949dff02,
+	0x67f150f9,
+/* 0x06b2: memx_func_train_loop_inner */
+	0x87f10000,
+	0x68ff1111,
+	0x10989490,
+	0xf10589fd,
+	0xf0072097,
+	0x90f91093,
+	0xd0fc80f9,
+	0x21f4e0fc,
+	0x8097f13f,
+	0x1093f000,
+	0xf4029eb9,
+	0xd8b90421,
+	0x2088c502,
+	0x80f990f9,
+	0xe0fcd0fc,
+	0xf13f21f4,
+	0xf0053c97,
+	0x87f11093,
+	0x83f13002,
+	0x90f98000,
+	0xd0fc80f9,
+	0x21f4e0fc,
+	0x60e7f13f,
+	0x10e3f005,
+	0x0000d7f1,
+	0x8000d3f1,
+	0xf100dc90,
+	0xf08480b7,
+	0x21f41eb3,
+	0x0057f1a4,
+	0xff97f100,
+	0x0093f1ff,
+/* 0x0731: memx_func_train_loop_4x */
+	0x80a7f183,
+	0x10a3f000,
+	0xf402aeb9,
+	0xd8b90421,
+	0xdfb7f102,
+	0xffb3f1ff,
+	0x048bfdff,
+	0x80f9a0f9,
+	0xe0fcd0fc,
+	0xf13f21f4,
+	0xf0053ca7,
+	0x87f110a3,
+	0x83f13002,
+	0xa0f98000,
+	0xd0fc80f9,
+	0x21f4e0fc,
+	0x60e7f13f,
+	0x10e3f005,
+	0x0000d7f1,
+	0x8000d3f1,
+	0xf102dcb9,
+	0xf02710b7,
+	0x21f400b3,
+	0x02eeb9a4,
+	0xb90421f4,
+	0x9dff02dd,
+	0x0150b694,
+	0xf4045670,
+	0x7aa0921e,
+	0xa9800bcc,
+	0x0160b600,
+	0x700470b6,
+	0x1ef51066,
+	0x50fcff00,
 	0x700150b6,
-	0x1ef40456,
-	0xcc7aa092,
-	0x00a9800b,
-	0xb60160b6,
-	0x66700470,
-	0x001ef510,
-	0xb650fcff,
-	0x56700150,
-	0xd41ef507,
-/* 0x07cf: memx_exec */
-	0xf900f8fe,
-	0xb9d0f9e0,
-	0xb2b902c1,
-/* 0x07d9: memx_exec_next */
-	0x00139802,
-	0xe70410b6,
-	0xe701f034,
-	0xb601e033,
-	0x30f00132,
-	0xde35980c,
-	0x12b855f9,
-	0xe41ef406,
-	0x98f10b98,
-	0xcbbbf20c,
-	0xc4b7f102,
-	0x06b4b607,
-	0xfc00bbcf,
-	0xf5e0fcd0,
-	0xf8034221,
-/* 0x0815: memx_info */
-	0x01c67000,
-/* 0x081b: memx_info_data */
-	0xf10e0bf4,
-	0xf103ccc7,
-	0xf40800b7,
-/* 0x0826: memx_info_train */
-	0xc7f10b0e,
-	0xb7f10bcc,
-/* 0x082e: memx_info_send */
-	0x21f50100,
-	0x00f80342,
-/* 0x0834: memx_recv */
-	0xf401d6b0,
-	0xd6b0980b,
-	0xd80bf400,
-/* 0x0842: memx_init */
-	0x00f800f8,
-/* 0x0844: perf_recv */
-/* 0x0846: perf_init */
-	0x00f800f8,
-/* 0x0848: i2c_drive_scl */
-	0xf40036b0,
-	0x07f1110b,
-	0x04b607e0,
-	0x0001d006,
-	0x00f804bd,
-/* 0x085c: i2c_drive_scl_lo */
-	0x07e407f1,
-	0xd00604b6,
-	0x04bd0001,
-/* 0x086a: i2c_drive_sda */
-	0x36b000f8,
-	0x110bf400,
-	0x07e007f1,
-	0xd00604b6,
-	0x04bd0002,
-/* 0x087e: i2c_drive_sda_lo */
-	0x07f100f8,
-	0x04b607e4,
-	0x0002d006,
-	0x00f804bd,
-/* 0x088c: i2c_sense_scl */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0431fd00,
-	0xf4060bf4,
-/* 0x08a2: i2c_sense_scl_done */
-	0x00f80131,
-/* 0x08a4: i2c_sense_sda */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0432fd00,
-	0xf4060bf4,
-/* 0x08ba: i2c_sense_sda_done */
-	0x00f80131,
-/* 0x08bc: i2c_raise_scl */
-	0x47f140f9,
-	0x37f00898,
-	0x4821f501,
-/* 0x08c9: i2c_raise_scl_wait */
+	0x1ef50756,
+	0x00f8fed4,
+/* 0x07c4: memx_exec */
+	0xd0f9e0f9,
+	0xb902c1b9,
+/* 0x07ce: memx_exec_next */
+	0x139802b2,
+	0x0410b600,
+	0x01f034e7,
+	0x01e033e7,
+	0xf00132b6,
+	0x35980c30,
+	0xb855f9de,
+	0x1ef40612,
+	0xf10b98e4,
+	0xbbf20c98,
+	0xb7f102cb,
+	0xb4b607c4,
+	0x00bbcf06,
+	0xe0fcd0fc,
+	0x033721f5,
+/* 0x080a: memx_info */
+	0xc67000f8,
+	0x0e0bf401,
+/* 0x0810: memx_info_data */
+	0x03ccc7f1,
+	0x0800b7f1,
+/* 0x081b: memx_info_train */
+	0xf10b0ef4,
+	0xf10bccc7,
+/* 0x0823: memx_info_send */
+	0xf50100b7,
+	0xf8033721,
+/* 0x0829: memx_recv */
+	0x01d6b000,
+	0xb0980bf4,
+	0x0bf400d6,
+/* 0x0837: memx_init */
+	0xf800f8d8,
+/* 0x0839: perf_recv */
+/* 0x083b: perf_init */
+	0xf800f800,
+/* 0x083d: i2c_drive_scl */
+	0x0036b000,
+	0xf1110bf4,
+	0xb607e007,
+	0x01d00604,
+	0xf804bd00,
+/* 0x0851: i2c_drive_scl_lo */
+	0xe407f100,
+	0x0604b607,
+	0xbd0001d0,
+/* 0x085f: i2c_drive_sda */
+	0xb000f804,
+	0x0bf40036,
+	0xe007f111,
+	0x0604b607,
+	0xbd0002d0,
+/* 0x0873: i2c_drive_sda_lo */
+	0xf100f804,
+	0xb607e407,
+	0x02d00604,
+	0xf804bd00,
+/* 0x0881: i2c_sense_scl */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x31fd0033,
+	0x060bf404,
+/* 0x0897: i2c_sense_scl_done */
+	0xf80131f4,
+/* 0x0899: i2c_sense_sda */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x32fd0033,
+	0x060bf404,
+/* 0x08af: i2c_sense_sda_done */
+	0xf80131f4,
+/* 0x08b1: i2c_raise_scl */
+	0xf140f900,
+	0xf0089847,
+	0x21f50137,
+/* 0x08be: i2c_raise_scl_wait */
+	0xe7f1083d,
+	0x21f403e8,
+	0x8121f57f,
+	0x0901f408,
+	0xf40142b6,
+/* 0x08d2: i2c_raise_scl_done */
+	0x40fcef1b,
+/* 0x08d6: i2c_start */
+	0x21f500f8,
+	0x11f40881,
+	0x9921f50d,
+	0x0611f408,
+/* 0x08e7: i2c_start_rep */
+	0xf0300ef4,
+	0x21f50037,
+	0x37f0083d,
+	0x5f21f501,
+	0x0076bb08,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b608b1,
+	0x1f11f404,
+/* 0x0914: i2c_start_send */
+	0xf50037f0,
+	0xf1085f21,
+	0xf41388e7,
+	0x37f07f21,
+	0x3d21f500,
+	0x88e7f108,
+	0x7f21f413,
+/* 0x0930: i2c_start_out */
+/* 0x0932: i2c_stop */
+	0x37f000f8,
+	0x3d21f500,
+	0x0037f008,
+	0x085f21f5,
+	0x03e8e7f1,
+	0xf07f21f4,
+	0x21f50137,
+	0xe7f1083d,
+	0x21f41388,
+	0x0137f07f,
+	0x085f21f5,
+	0x1388e7f1,
+	0xf87f21f4,
+/* 0x0965: i2c_bitw */
+	0x5f21f500,
 	0xe8e7f108,
 	0x7f21f403,
-	0x088c21f5,
-	0xb60901f4,
-	0x1bf40142,
-/* 0x08dd: i2c_raise_scl_done */
-	0xf840fcef,
-/* 0x08e1: i2c_start */
-	0x8c21f500,
-	0x0d11f408,
-	0x08a421f5,
-	0xf40611f4,
-/* 0x08f2: i2c_start_rep */
-	0x37f0300e,
-	0x4821f500,
-	0x0137f008,
-	0x086a21f5,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xbc21f550,
+	0xb121f550,
 	0x0464b608,
-/* 0x091f: i2c_start_send */
-	0xf01f11f4,
-	0x21f50037,
-	0xe7f1086a,
-	0x21f41388,
-	0x0037f07f,
-	0x084821f5,
-	0x1388e7f1,
-/* 0x093b: i2c_start_out */
-	0xf87f21f4,
-/* 0x093d: i2c_stop */
-	0x0037f000,
-	0x084821f5,
-	0xf50037f0,
-	0xf1086a21,
-	0xf403e8e7,
+	0xf11811f4,
+	0xf41388e7,
 	0x37f07f21,
-	0x4821f501,
+	0x3d21f500,
 	0x88e7f108,
 	0x7f21f413,
-	0xf50137f0,
-	0xf1086a21,
-	0xf41388e7,
-	0x00f87f21,
-/* 0x0970: i2c_bitw */
-	0x086a21f5,
-	0x03e8e7f1,
-	0xbb7f21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x08bc21f5,
-	0xf40464b6,
-	0xe7f11811,
+/* 0x09a4: i2c_bitw_out */
+/* 0x09a6: i2c_bitr */
+	0x37f000f8,
+	0x5f21f501,
+	0xe8e7f108,
+	0x7f21f403,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xb121f550,
+	0x0464b608,
+	0xf51b11f4,
+	0xf0089921,
+	0x21f50037,
+	0xe7f1083d,
 	0x21f41388,
-	0x0037f07f,
-	0x084821f5,
-	0x1388e7f1,
-/* 0x09af: i2c_bitw_out */
-	0xf87f21f4,
-/* 0x09b1: i2c_bitr */
-	0x0137f000,
-	0x086a21f5,
-	0x03e8e7f1,
-	0xbb7f21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x08bc21f5,
-	0xf40464b6,
-	0x21f51b11,
-	0x37f008a4,
-	0x4821f500,
-	0x88e7f108,
-	0x7f21f413,
-	0xf4013cf0,
-/* 0x09f6: i2c_bitr_done */
-	0x00f80131,
-/* 0x09f8: i2c_get_byte */
-	0xf00057f0,
-/* 0x09fe: i2c_get_byte_next */
-	0x54b60847,
+	0x013cf07f,
+/* 0x09eb: i2c_bitr_done */
+	0xf80131f4,
+/* 0x09ed: i2c_get_byte */
+	0x0057f000,
+/* 0x09f3: i2c_get_byte_next */
+	0xb60847f0,
+	0x76bb0154,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb609a621,
+	0x11f40464,
+	0x0553fd2b,
+	0xf40142b6,
+	0x37f0d81b,
 	0x0076bb01,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b609b1,
-	0x2b11f404,
-	0xb60553fd,
-	0x1bf40142,
-	0x0137f0d8,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0x7021f550,
-	0x0464b609,
-/* 0x0a48: i2c_get_byte_done */
-/* 0x0a4a: i2c_put_byte */
-	0x47f000f8,
-/* 0x0a4d: i2c_put_byte_next */
-	0x0142b608,
-	0xbb3854ff,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x097021f5,
-	0xf40464b6,
-	0x46b03411,
-	0xd81bf400,
+	0x64b60965,
+/* 0x0a3d: i2c_get_byte_done */
+/* 0x0a3f: i2c_put_byte */
+	0xf000f804,
+/* 0x0a42: i2c_put_byte_next */
+	0x42b60847,
+	0x3854ff01,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xb121f550,
+	0x6521f550,
 	0x0464b609,
-	0xbb0f11f4,
-	0x36b00076,
-	0x061bf401,
-/* 0x0aa3: i2c_put_byte_done */
-	0xf80132f4,
-/* 0x0aa5: i2c_addr */
-	0x0076bb00,
+	0xb03411f4,
+	0x1bf40046,
+	0x0076bbd8,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b608e1,
-	0x2911f404,
-	0x012ec3e7,
-	0xfd0134b6,
-	0x76bb0553,
+	0x64b609a6,
+	0x0f11f404,
+	0xb00076bb,
+	0x1bf40136,
+	0x0132f406,
+/* 0x0a98: i2c_put_byte_done */
+/* 0x0a9a: i2c_addr */
+	0x76bb00f8,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb60a4a21,
-/* 0x0aea: i2c_addr_done */
-	0x00f80464,
-/* 0x0aec: i2c_acquire_addr */
-	0xb6f8cec7,
-	0xe0b702e4,
-	0xee980d1c,
-/* 0x0afb: i2c_acquire */
-	0xf500f800,
-	0xf40aec21,
-	0xd9f00421,
-	0x3f21f403,
-/* 0x0b0a: i2c_release */
-	0x21f500f8,
-	0x21f40aec,
-	0x03daf004,
-	0xf83f21f4,
-/* 0x0b19: i2c_recv */
-	0x0132f400,
-	0xb6f8c1c7,
-	0x16b00214,
-	0x3a1ff528,
-	0xf413a001,
-	0x0032980c,
-	0x0ccc13a0,
-	0xf4003198,
-	0xd0f90231,
-	0xd0f9e0f9,
-	0x000067f1,
-	0x100063f1,
-	0xbb016792,
+	0xb608d621,
+	0x11f40464,
+	0x2ec3e729,
+	0x0134b601,
+	0xbb0553fd,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0afb21f5,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b31bf5,
-	0xbb0057f0,
+	0x0a3f21f5,
+/* 0x0adf: i2c_addr_done */
+	0xf80464b6,
+/* 0x0ae1: i2c_acquire_addr */
+	0xf8cec700,
+	0xb702e4b6,
+	0x980d1ce0,
+	0x00f800ee,
+/* 0x0af0: i2c_acquire */
+	0x0ae121f5,
+	0xf00421f4,
+	0x21f403d9,
+/* 0x0aff: i2c_release */
+	0xf500f83f,
+	0xf40ae121,
+	0xdaf00421,
+	0x3f21f403,
+/* 0x0b0e: i2c_recv */
+	0x32f400f8,
+	0xf8c1c701,
+	0xb00214b6,
+	0x1ff52816,
+	0x13a0013a,
+	0x32980cf4,
+	0xcc13a000,
+	0x0031980c,
+	0xf90231f4,
+	0xf9e0f9d0,
+	0x0067f1d0,
+	0x0063f100,
+	0x01679210,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xf021f550,
+	0x0464b60a,
+	0xd6b0d0fc,
+	0xb31bf500,
+	0x0057f000,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x9a21f550,
+	0x0464b60a,
+	0x00d011f5,
+	0xbbe0c5c7,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0aa521f5,
+	0x0a3f21f5,
 	0xf50464b6,
-	0xc700d011,
-	0x76bbe0c5,
+	0xf000ad11,
+	0x76bb0157,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb60a4a21,
+	0xb60a9a21,
 	0x11f50464,
-	0x57f000ad,
-	0x0076bb01,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b60aa5,
-	0x8a11f504,
-	0x0076bb00,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b609f8,
-	0x6a11f404,
-	0xbbe05bcb,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x093d21f5,
-	0xb90464b6,
-	0x74bd025b,
-/* 0x0c1f: i2c_recv_not_rd08 */
-	0xb0430ef4,
-	0x1bf401d6,
-	0x0057f03d,
-	0x0aa521f5,
-	0xc73311f4,
-	0x21f5e0c5,
-	0x11f40a4a,
-	0x0057f029,
-	0x0aa521f5,
-	0xc71f11f4,
-	0x21f5e0b5,
-	0x11f40a4a,
-	0x3d21f515,
-	0xc774bd09,
-	0x1bf408c5,
-	0x0232f409,
-/* 0x0c5f: i2c_recv_not_wr08 */
-/* 0x0c5f: i2c_recv_done */
-	0xc7030ef4,
-	0x21f5f8ce,
-	0xe0fc0b0a,
-	0x12f4d0fc,
-	0x027cb90a,
-	0x034221f5,
-/* 0x0c74: i2c_recv_exit */
-/* 0x0c76: i2c_init */
+	0x76bb008a,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb609ed21,
+	0x11f40464,
+	0xe05bcb6a,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x3221f550,
+	0x0464b609,
+	0xbd025bb9,
+	0x430ef474,
+/* 0x0c14: i2c_recv_not_rd08 */
+	0xf401d6b0,
+	0x57f03d1b,
+	0x9a21f500,
+	0x3311f40a,
+	0xf5e0c5c7,
+	0xf40a3f21,
+	0x57f02911,
+	0x9a21f500,
+	0x1f11f40a,
+	0xf5e0b5c7,
+	0xf40a3f21,
+	0x21f51511,
+	0x74bd0932,
+	0xf408c5c7,
+	0x32f4091b,
+	0x030ef402,
+/* 0x0c54: i2c_recv_not_wr08 */
+/* 0x0c54: i2c_recv_done */
+	0xf5f8cec7,
+	0xfc0aff21,
+	0xf4d0fce0,
+	0x7cb90a12,
+	0x3721f502,
+/* 0x0c69: i2c_recv_exit */
+/* 0x0c6b: i2c_init */
+	0xf800f803,
+/* 0x0c6d: test_recv */
+	0xd817f100,
+	0x0614b605,
+	0xb60011cf,
+	0x07f10110,
+	0x04b605d8,
+	0x0001d006,
+	0xe7f104bd,
+	0xe3f1d900,
+	0x21f5134f,
+	0x00f80257,
+/* 0x0c94: test_init */
+	0x0800e7f1,
+	0x025721f5,
+/* 0x0c9e: idle_recv */
 	0x00f800f8,
-/* 0x0c78: test_recv */
-	0x05d817f1,
-	0xcf0614b6,
-	0x10b60011,
-	0xd807f101,
-	0x0604b605,
-	0xbd0001d0,
-	0x00e7f104,
-	0x4fe3f1d9,
-	0x6221f513,
-/* 0x0c9f: test_init */
-	0xf100f802,
-	0xf50800e7,
-	0xf8026221,
-/* 0x0ca9: idle_recv */
-/* 0x0cab: idle */
-	0xf400f800,
-	0x17f10031,
-	0x14b605d4,
-	0x0011cf06,
-	0xf10110b6,
-	0xb605d407,
-	0x01d00604,
-/* 0x0cc7: idle_loop */
-	0xf004bd00,
-	0x32f45817,
-/* 0x0ccd: idle_proc */
-/* 0x0ccd: idle_proc_exec */
-	0xb910f902,
-	0x21f5021e,
-	0x10fc034b,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0ce1: idle_proc_next */
-	0x5810b6ef,
-	0xf4061fb8,
-	0x02f4e61b,
-	0x0028f4dd,
-	0x00bb0ef4,
+/* 0x0ca0: idle */
+	0xf10031f4,
+	0xb605d417,
+	0x11cf0614,
+	0x0110b600,
+	0x05d407f1,
+	0xd00604b6,
+	0x04bd0001,
+/* 0x0cbc: idle_loop */
+	0xf45817f0,
+/* 0x0cc2: idle_proc */
+/* 0x0cc2: idle_proc_exec */
+	0x10f90232,
+	0xf5021eb9,
+	0xfc034021,
+	0x0911f410,
+	0xf40231f4,
+/* 0x0cd6: idle_proc_next */
+	0x10b6ef0e,
+	0x061fb858,
+	0xf4e61bf4,
+	0x28f4dd02,
+	0xbb0ef400,
+	0x00000000,
+	0x00000000,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/kernel.fuc b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/kernel.fuc
index 5cf5be63cbef..ad35fa57be94 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/kernel.fuc
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/kernel.fuc
@@ -225,17 +225,11 @@ intr:
 		nv_iowr(NV_PPWR_SUBINTR, $r9)
 
 	intr_skip_subintr:
-	and $r9 $r8 NV_PPWR_INTR_PAUSE
-	bra z #intr_skip_pause
-		and $r10 0xffbf
-
-	intr_skip_pause:
-	and $r9 $r8 NV_PPWR_INTR_USER0
-	bra z #intr_skip_user0
-		and $r10 0xffbf
-
-	intr_skip_user0:
+	mov $r9 (NV_PPWR_INTR_USER0 | NV_PPWR_INTR_USER1 | NV_PPWR_INTR_PAUSE)
+	not b32 $r9
+	and $r8 $r9
 	nv_iowr(NV_PPWR_INTR_ACK, $r8)
+
 	pop $r8
 	mov $flags $r8
 	pop $r15
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/nv40.c
index 6326fdc5a48d..2c92ffb5f9d0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/nv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/nv40.c
@@ -107,7 +107,7 @@ nv40_fan_pwm_ctrl(struct nvkm_therm *therm, int line, bool enable)
 {
 	struct nvkm_subdev *subdev = &therm->subdev;
 	struct nvkm_device *device = subdev->device;
-	u32 mask = enable ? 0x80000000 : 0x0000000;
+	u32 mask = enable ? 0x80000000 : 0x00000000;
 	if      (line == 2) nvkm_mask(device, 0x0010f0, 0x80000000, mask);
 	else if (line == 9) nvkm_mask(device, 0x0015f4, 0x80000000, mask);
 	else {
diff --git a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
index 6c220cd3497a..336ad4de9981 100644
--- a/drivers/gpu/drm/omapdrm/Kconfig
+++ b/drivers/gpu/drm/omapdrm/Kconfig
@@ -1,9 +1,8 @@
-
 config DRM_OMAP
 	tristate "OMAP DRM"
 	depends on DRM
 	depends on ARCH_OMAP2PLUS || ARCH_MULTIPLATFORM
-	depends on OMAP2_DSS
+	select OMAP2_DSS
 	select DRM_KMS_HELPER
 	select DRM_KMS_FB_HELPER
 	select FB_SYS_FILLRECT
@@ -14,13 +13,18 @@ config DRM_OMAP
 	help
 	  DRM display driver for OMAP2/3/4 based boards.
 
+if DRM_OMAP
+
 config DRM_OMAP_NUM_CRTCS
 	int "Number of CRTCs"
 	range 1 10
 	default 1  if ARCH_OMAP2 || ARCH_OMAP3
 	default 2  if ARCH_OMAP4
-	depends on DRM_OMAP
 	help
 	  Select the number of video overlays which can be used as framebuffers.
 	  The remaining overlays are reserved for video.
 
+source "drivers/gpu/drm/omapdrm/dss/Kconfig"
+source "drivers/gpu/drm/omapdrm/displays/Kconfig"
+
+endif
diff --git a/drivers/gpu/drm/omapdrm/Makefile b/drivers/gpu/drm/omapdrm/Makefile
index 778372b062ad..fe4c2228bc18 100644
--- a/drivers/gpu/drm/omapdrm/Makefile
+++ b/drivers/gpu/drm/omapdrm/Makefile
@@ -3,6 +3,9 @@
 # Direct Rendering Infrastructure (DRI)
 #
 
+obj-y += dss/
+obj-y += displays/
+
 ccflags-y := -Iinclude/drm -Werror
 omapdrm-y := omap_drv.o \
 	omap_irq.o \
@@ -12,10 +15,11 @@ omapdrm-y := omap_drv.o \
 	omap_encoder.o \
 	omap_connector.o \
 	omap_fb.o \
-	omap_fbdev.o \
 	omap_gem.o \
 	omap_gem_dmabuf.o \
 	omap_dmm_tiler.o \
 	tcm-sita.o
 
+omapdrm-$(CONFIG_DRM_FBDEV_EMULATION) += omap_fbdev.o
+
 obj-$(CONFIG_DRM_OMAP)	+= omapdrm.o
diff --git a/drivers/video/fbdev/omap2/displays-new/Kconfig b/drivers/gpu/drm/omapdrm/displays/Kconfig
index 574710141a61..2a618afe0f53 100644
--- a/drivers/video/fbdev/omap2/displays-new/Kconfig
+++ b/drivers/gpu/drm/omapdrm/displays/Kconfig
@@ -1,5 +1,4 @@
-menu "OMAP Display Device Drivers (new device model)"
-        depends on OMAP2_DSS
+menu "OMAPDRM External Display Device Drivers"
 
 config DISPLAY_ENCODER_OPA362
 	tristate "OPA362 external analog amplifier"
diff --git a/drivers/video/fbdev/omap2/displays-new/Makefile b/drivers/gpu/drm/omapdrm/displays/Makefile
index 9aa176bfbf2e..9aa176bfbf2e 100644
--- a/drivers/video/fbdev/omap2/displays-new/Makefile
+++ b/drivers/gpu/drm/omapdrm/displays/Makefile
diff --git a/drivers/video/fbdev/omap2/displays-new/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index 8511c648a15c..8511c648a15c 100644
--- a/drivers/video/fbdev/omap2/displays-new/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
diff --git a/drivers/video/fbdev/omap2/displays-new/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
index d811e6dcaef7..d811e6dcaef7 100644
--- a/drivers/video/fbdev/omap2/displays-new/connector-dvi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
diff --git a/drivers/video/fbdev/omap2/displays-new/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index 6ee4129bc0c0..6ee4129bc0c0 100644
--- a/drivers/video/fbdev/omap2/displays-new/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
diff --git a/drivers/video/fbdev/omap2/displays-new/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
index 8c246c213e06..8c246c213e06 100644
--- a/drivers/video/fbdev/omap2/displays-new/encoder-opa362.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
diff --git a/drivers/video/fbdev/omap2/displays-new/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
index d9048b3df495..d9048b3df495 100644
--- a/drivers/video/fbdev/omap2/displays-new/encoder-tfp410.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
diff --git a/drivers/video/fbdev/omap2/displays-new/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index 990af6baeb0f..990af6baeb0f 100644
--- a/drivers/video/fbdev/omap2/displays-new/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
index f7be3489f744..e780fd4f8b46 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-dpi.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
@@ -83,8 +83,7 @@ static int panel_dpi_enable(struct omap_dss_device *dssdev)
 	if (r)
 		return r;
 
-	if (ddata->enable_gpio)
-		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
+	gpiod_set_value_cansleep(ddata->enable_gpio, 1);
 
 	if (gpio_is_valid(ddata->backlight_gpio))
 		gpio_set_value_cansleep(ddata->backlight_gpio, 1);
@@ -102,12 +101,11 @@ static void panel_dpi_disable(struct omap_dss_device *dssdev)
 	if (!omapdss_device_is_enabled(dssdev))
 		return;
 
-	if (ddata->enable_gpio)
-		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-
 	if (gpio_is_valid(ddata->backlight_gpio))
 		gpio_set_value_cansleep(ddata->backlight_gpio, 0);
 
+	gpiod_set_value_cansleep(ddata->enable_gpio, 0);
+
 	in->ops.dpi->disable(in);
 
 	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 3414c2609320..3414c2609320 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index 18eb60e9c9ec..18eb60e9c9ec 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index 8a928c9a2fc9..8a928c9a2fc9 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
index abfd1f6e3327..abfd1f6e3327 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-sharp-ls037v7dw01.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index 31efcca801bd..31efcca801bd 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index 4d657f3ab679..4d657f3ab679 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
diff --git a/drivers/video/fbdev/omap2/displays-new/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
index 68e3b68a2920..68e3b68a2920 100644
--- a/drivers/video/fbdev/omap2/displays-new/panel-tpo-td043mtea1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
diff --git a/drivers/video/fbdev/omap2/dss/Kconfig b/drivers/gpu/drm/omapdrm/dss/Kconfig
index d1fa730c7d54..d1fa730c7d54 100644
--- a/drivers/video/fbdev/omap2/dss/Kconfig
+++ b/drivers/gpu/drm/omapdrm/dss/Kconfig
diff --git a/drivers/video/fbdev/omap2/dss/Makefile b/drivers/gpu/drm/omapdrm/dss/Makefile
index b5136d3d4b77..b5136d3d4b77 100644
--- a/drivers/video/fbdev/omap2/dss/Makefile
+++ b/drivers/gpu/drm/omapdrm/dss/Makefile
diff --git a/drivers/video/fbdev/omap2/dss/apply.c b/drivers/gpu/drm/omapdrm/dss/apply.c
index 663ccc3bf4e5..663ccc3bf4e5 100644
--- a/drivers/video/fbdev/omap2/dss/apply.c
+++ b/drivers/gpu/drm/omapdrm/dss/apply.c
diff --git a/drivers/video/fbdev/omap2/dss/core.c b/drivers/gpu/drm/omapdrm/dss/core.c
index 54eeb507f9b3..54eeb507f9b3 100644
--- a/drivers/video/fbdev/omap2/dss/core.c
+++ b/drivers/gpu/drm/omapdrm/dss/core.c
diff --git a/drivers/video/fbdev/omap2/dss/dispc-compat.c b/drivers/gpu/drm/omapdrm/dss/dispc-compat.c
index 633c461fbc6e..0918b3bfe82a 100644
--- a/drivers/video/fbdev/omap2/dss/dispc-compat.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc-compat.c
@@ -476,7 +476,7 @@ static void dispc_mgr_disable_lcd_out(enum omap_channel channel)
 	int r;
 	u32 irq;
 
-	if (dispc_mgr_is_enabled(channel) == false)
+	if (!dispc_mgr_is_enabled(channel))
 		return;
 
 	/*
@@ -524,7 +524,7 @@ static void dispc_mgr_enable_digit_out(void)
 	int r;
 	u32 irq_mask;
 
-	if (dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT) == true)
+	if (dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT))
 		return;
 
 	/*
@@ -562,7 +562,7 @@ static void dispc_mgr_disable_digit_out(void)
 	u32 irq_mask;
 	int num_irqs;
 
-	if (dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT) == false)
+	if (!dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT))
 		return;
 
 	/*
diff --git a/drivers/video/fbdev/omap2/dss/dispc-compat.h b/drivers/gpu/drm/omapdrm/dss/dispc-compat.h
index 14a69b3d4fb0..14a69b3d4fb0 100644
--- a/drivers/video/fbdev/omap2/dss/dispc-compat.h
+++ b/drivers/gpu/drm/omapdrm/dss/dispc-compat.h
diff --git a/drivers/video/fbdev/omap2/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index be716c9ffb88..6b50476ec669 100644
--- a/drivers/video/fbdev/omap2/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -99,6 +99,11 @@ struct dispc_features {
 
 	/* PIXEL_INC is not added to the last pixel of a line */
 	bool last_pixel_inc_missing:1;
+
+	/* POL_FREQ has ALIGN bit */
+	bool supports_sync_align:1;
+
+	bool has_writeback:1;
 };
 
 #define DISPC_MAX_NR_FIFOS 5
@@ -246,6 +251,11 @@ struct color_conv_coef {
 	int full_range;
 };
 
+static unsigned long dispc_fclk_rate(void);
+static unsigned long dispc_core_clk_rate(void);
+static unsigned long dispc_mgr_lclk_rate(enum omap_channel channel);
+static unsigned long dispc_mgr_pclk_rate(enum omap_channel channel);
+
 static unsigned long dispc_plane_pclk_rate(enum omap_plane plane);
 static unsigned long dispc_plane_lclk_rate(enum omap_plane plane);
 
@@ -571,7 +581,7 @@ EXPORT_SYMBOL(dispc_mgr_go_busy);
 
 void dispc_mgr_go(enum omap_channel channel)
 {
-	WARN_ON(dispc_mgr_is_enabled(channel) == false);
+	WARN_ON(!dispc_mgr_is_enabled(channel));
 	WARN_ON(dispc_mgr_go_busy(channel));
 
 	DSSDBG("GO %s\n", mgr_desc[channel].name);
@@ -707,19 +717,20 @@ static void dispc_setup_color_conv_coef(void)
 {
 	int i;
 	int num_ovl = dss_feat_get_num_ovls();
-	int num_wb = dss_feat_get_num_wbs();
 	const struct color_conv_coef ctbl_bt601_5_ovl = {
+		/* YUV -> RGB */
 		298, 409, 0, 298, -208, -100, 298, 0, 517, 0,
 	};
 	const struct color_conv_coef ctbl_bt601_5_wb = {
-		66, 112, -38, 129, -94, -74, 25, -18, 112, 0,
+		/* RGB -> YUV */
+		66, 129, 25, 112, -94, -18, -38, -74, 112, 0,
 	};
 
 	for (i = 1; i < num_ovl; i++)
 		dispc_ovl_write_color_conv_coef(i, &ctbl_bt601_5_ovl);
 
-	for (; i < num_wb; i++)
-		dispc_ovl_write_color_conv_coef(i, &ctbl_bt601_5_wb);
+	if (dispc.feat->has_writeback)
+		dispc_ovl_write_color_conv_coef(OMAP_DSS_WB, &ctbl_bt601_5_wb);
 }
 
 static void dispc_ovl_set_ba0(enum omap_plane plane, u32 paddr)
@@ -970,6 +981,10 @@ void dispc_ovl_set_channel_out(enum omap_plane plane, enum omap_channel channel)
 				return;
 			}
 			break;
+		case OMAP_DSS_CHANNEL_WB:
+			chan = 0;
+			chan2 = 3;
+			break;
 		default:
 			BUG();
 			return;
@@ -988,7 +1003,6 @@ static enum omap_channel dispc_ovl_get_channel_out(enum omap_plane plane)
 {
 	int shift;
 	u32 val;
-	enum omap_channel channel;
 
 	switch (plane) {
 	case OMAP_DSS_GFX:
@@ -1006,23 +1020,23 @@ static enum omap_channel dispc_ovl_get_channel_out(enum omap_plane plane)
 
 	val = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
 
-	if (dss_has_feature(FEAT_MGR_LCD3)) {
-		if (FLD_GET(val, 31, 30) == 0)
-			channel = FLD_GET(val, shift, shift);
-		else if (FLD_GET(val, 31, 30) == 1)
-			channel = OMAP_DSS_CHANNEL_LCD2;
-		else
-			channel = OMAP_DSS_CHANNEL_LCD3;
-	} else if (dss_has_feature(FEAT_MGR_LCD2)) {
-		if (FLD_GET(val, 31, 30) == 0)
-			channel = FLD_GET(val, shift, shift);
-		else
-			channel = OMAP_DSS_CHANNEL_LCD2;
-	} else {
-		channel = FLD_GET(val, shift, shift);
-	}
+	if (FLD_GET(val, shift, shift) == 1)
+		return OMAP_DSS_CHANNEL_DIGIT;
+
+	if (!dss_has_feature(FEAT_MGR_LCD2))
+		return OMAP_DSS_CHANNEL_LCD;
 
-	return channel;
+	switch (FLD_GET(val, 31, 30)) {
+	case 0:
+	default:
+		return OMAP_DSS_CHANNEL_LCD;
+	case 1:
+		return OMAP_DSS_CHANNEL_LCD2;
+	case 2:
+		return OMAP_DSS_CHANNEL_LCD3;
+	case 3:
+		return OMAP_DSS_CHANNEL_WB;
+	}
 }
 
 void dispc_wb_set_channel_in(enum dss_writeback_channel channel)
@@ -1050,6 +1064,8 @@ static void dispc_configure_burst_sizes(void)
 	/* Configure burst size always to maximum size */
 	for (i = 0; i < dss_feat_get_num_ovls(); ++i)
 		dispc_ovl_set_burst_size(i, burst_size);
+	if (dispc.feat->has_writeback)
+		dispc_ovl_set_burst_size(OMAP_DSS_WB, burst_size);
 }
 
 static u32 dispc_ovl_get_burst_size(enum omap_plane plane)
@@ -1196,6 +1212,17 @@ static void dispc_init_fifos(void)
 
 		dispc_ovl_set_fifo_threshold(i, low, high);
 	}
+
+	if (dispc.feat->has_writeback) {
+		u32 low, high;
+		const bool use_fifomerge = false;
+		const bool manual_update = false;
+
+		dispc_ovl_compute_fifo_thresholds(OMAP_DSS_WB, &low, &high,
+			use_fifomerge, manual_update);
+
+		dispc_ovl_set_fifo_threshold(OMAP_DSS_WB, low, high);
+	}
 }
 
 static u32 dispc_ovl_get_fifo_size(enum omap_plane plane)
@@ -1248,7 +1275,6 @@ void dispc_ovl_set_fifo_threshold(enum omap_plane plane, u32 low, u32 high)
 			plane != OMAP_DSS_WB)
 		dispc_write_reg(DISPC_OVL_PRELOAD(plane), min(high, 0xfffu));
 }
-EXPORT_SYMBOL(dispc_ovl_set_fifo_threshold);
 
 void dispc_enable_fifomerge(bool enable)
 {
@@ -1307,7 +1333,6 @@ void dispc_ovl_compute_fifo_thresholds(enum omap_plane plane,
 		*fifo_high = total_fifo_size - buf_unit;
 	}
 }
-EXPORT_SYMBOL(dispc_ovl_compute_fifo_thresholds);
 
 static void dispc_ovl_set_mflag(enum omap_plane plane, bool enable)
 {
@@ -1364,6 +1389,25 @@ static void dispc_init_mflag(void)
 
 		dispc_ovl_set_mflag_threshold(i, low, high);
 	}
+
+	if (dispc.feat->has_writeback) {
+		u32 size = dispc_ovl_get_fifo_size(OMAP_DSS_WB);
+		u32 unit = dss_feat_get_buffer_size_unit();
+		u32 low, high;
+
+		dispc_ovl_set_mflag(OMAP_DSS_WB, true);
+
+		/*
+		 * Simulation team suggests below thesholds:
+		 * HT = fifosize * 5 / 8;
+		 * LT = fifosize * 4 / 8;
+		 */
+
+		low = size * 4 / 8 / unit;
+		high = size * 5 / 8 / unit;
+
+		dispc_ovl_set_mflag_threshold(OMAP_DSS_WB, low, high);
+	}
 }
 
 static void dispc_ovl_set_fir(enum omap_plane plane,
@@ -2438,7 +2482,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 	if (width == out_width && height == out_height)
 		return 0;
 
-	if (pclk == 0 || mgr_timings->pixelclock == 0) {
+	if (!mem_to_mem && (pclk == 0 || mgr_timings->pixelclock == 0)) {
 		DSSERR("cannot calculate scaling settings: pclk is zero\n");
 		return -EINVAL;
 	}
@@ -2816,8 +2860,25 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
 	l = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
 	l = FLD_MOD(l, truncation, 10, 10);	/* TRUNCATIONENABLE */
 	l = FLD_MOD(l, mem_to_mem, 19, 19);	/* WRITEBACKMODE */
+	if (mem_to_mem)
+		l = FLD_MOD(l, 1, 26, 24);	/* CAPTUREMODE */
+	else
+		l = FLD_MOD(l, 0, 26, 24);	/* CAPTUREMODE */
 	dispc_write_reg(DISPC_OVL_ATTRIBUTES(plane), l);
 
+	if (mem_to_mem) {
+		/* WBDELAYCOUNT */
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), 0, 7, 0);
+	} else {
+		int wbdelay;
+
+		wbdelay = min(mgr_timings->vfp + mgr_timings->vsw +
+			mgr_timings->vbp, 255);
+
+		/* WBDELAYCOUNT */
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), wbdelay, 7, 0);
+	}
+
 	return r;
 }
 
@@ -2896,7 +2957,7 @@ static void dispc_mgr_set_lcd_type_tft(enum omap_channel channel)
 	mgr_fld_write(channel, DISPC_MGR_FLD_STNTFT, 1);
 }
 
-void dispc_set_loadmode(enum omap_dss_load_mode mode)
+static void dispc_set_loadmode(enum omap_dss_load_mode mode)
 {
 	REG_FLD_MOD(DISPC_CONFIG, mode, 2, 1);
 }
@@ -3163,6 +3224,10 @@ static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw,
 		FLD_VAL(hs, 13, 13) |
 		FLD_VAL(vs, 12, 12);
 
+	/* always set ALIGN bit when available */
+	if (dispc.feat->supports_sync_align)
+		l |= (1 << 18);
+
 	dispc_write_reg(DISPC_POL_FREQ(channel), l);
 
 	if (dispc.syscon_pol) {
@@ -3220,7 +3285,7 @@ void dispc_mgr_set_timings(enum omap_channel channel,
 
 		DSSDBG("hsync %luHz, vsync %luHz\n", ht, vt);
 	} else {
-		if (t.interlace == true)
+		if (t.interlace)
 			t.y_res /= 2;
 	}
 
@@ -3237,7 +3302,7 @@ static void dispc_mgr_set_lcd_divisor(enum omap_channel channel, u16 lck_div,
 	dispc_write_reg(DISPC_DIVISORo(channel),
 			FLD_VAL(lck_div, 23, 16) | FLD_VAL(pck_div, 7, 0));
 
-	if (dss_has_feature(FEAT_CORE_CLK_DIV) == false &&
+	if (!dss_has_feature(FEAT_CORE_CLK_DIV) &&
 			channel == OMAP_DSS_CHANNEL_LCD)
 		dispc.core_clk_rate = dispc_fclk_rate() / lck_div;
 }
@@ -3251,7 +3316,7 @@ static void dispc_mgr_get_lcd_divisor(enum omap_channel channel, int *lck_div,
 	*pck_div = FLD_GET(l, 7, 0);
 }
 
-unsigned long dispc_fclk_rate(void)
+static unsigned long dispc_fclk_rate(void)
 {
 	struct dss_pll *pll;
 	unsigned long r = 0;
@@ -3282,7 +3347,7 @@ unsigned long dispc_fclk_rate(void)
 	return r;
 }
 
-unsigned long dispc_mgr_lclk_rate(enum omap_channel channel)
+static unsigned long dispc_mgr_lclk_rate(enum omap_channel channel)
 {
 	struct dss_pll *pll;
 	int lcd;
@@ -3323,7 +3388,7 @@ unsigned long dispc_mgr_lclk_rate(enum omap_channel channel)
 	}
 }
 
-unsigned long dispc_mgr_pclk_rate(enum omap_channel channel)
+static unsigned long dispc_mgr_pclk_rate(enum omap_channel channel)
 {
 	unsigned long r;
 
@@ -3348,7 +3413,7 @@ void dispc_set_tv_pclk(unsigned long pclk)
 	dispc.tv_pclk_rate = pclk;
 }
 
-unsigned long dispc_core_clk_rate(void)
+static unsigned long dispc_core_clk_rate(void)
 {
 	return dispc.core_clk_rate;
 }
@@ -3448,6 +3513,7 @@ static void dispc_dump_regs(struct seq_file *s)
 		[OMAP_DSS_VIDEO1]	= "VID1",
 		[OMAP_DSS_VIDEO2]	= "VID2",
 		[OMAP_DSS_VIDEO3]	= "VID3",
+		[OMAP_DSS_WB]		= "WB",
 	};
 	const char **p_names;
 
@@ -3554,6 +3620,35 @@ static void dispc_dump_regs(struct seq_file *s)
 			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
 	}
 
+	if (dispc.feat->has_writeback) {
+		i = OMAP_DSS_WB;
+		DUMPREG(i, DISPC_OVL_BA0);
+		DUMPREG(i, DISPC_OVL_BA1);
+		DUMPREG(i, DISPC_OVL_SIZE);
+		DUMPREG(i, DISPC_OVL_ATTRIBUTES);
+		DUMPREG(i, DISPC_OVL_FIFO_THRESHOLD);
+		DUMPREG(i, DISPC_OVL_FIFO_SIZE_STATUS);
+		DUMPREG(i, DISPC_OVL_ROW_INC);
+		DUMPREG(i, DISPC_OVL_PIXEL_INC);
+
+		if (dss_has_feature(FEAT_MFLAG))
+			DUMPREG(i, DISPC_OVL_MFLAG_THRESHOLD);
+
+		DUMPREG(i, DISPC_OVL_FIR);
+		DUMPREG(i, DISPC_OVL_PICTURE_SIZE);
+		DUMPREG(i, DISPC_OVL_ACCU0);
+		DUMPREG(i, DISPC_OVL_ACCU1);
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			DUMPREG(i, DISPC_OVL_BA0_UV);
+			DUMPREG(i, DISPC_OVL_BA1_UV);
+			DUMPREG(i, DISPC_OVL_FIR2);
+			DUMPREG(i, DISPC_OVL_ACCU2_0);
+			DUMPREG(i, DISPC_OVL_ACCU2_1);
+		}
+		if (dss_has_feature(FEAT_ATTR2))
+			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
+	}
+
 #undef DISPC_REG
 #undef DUMPREG
 
@@ -3854,6 +3949,8 @@ static const struct dispc_features omap44xx_dispc_feats = {
 	.num_fifos		=	5,
 	.gfx_fifo_workaround	=	true,
 	.set_max_preload	=	true,
+	.supports_sync_align	=	true,
+	.has_writeback		=	true,
 };
 
 static const struct dispc_features omap54xx_dispc_feats = {
@@ -3875,6 +3972,8 @@ static const struct dispc_features omap54xx_dispc_feats = {
 	.gfx_fifo_workaround	=	true,
 	.mstandby_workaround	=	true,
 	.set_max_preload	=	true,
+	.supports_sync_align	=	true,
+	.has_writeback		=	true,
 };
 
 static int dispc_init_features(struct platform_device *pdev)
diff --git a/drivers/video/fbdev/omap2/dss/dispc.h b/drivers/gpu/drm/omapdrm/dss/dispc.h
index 3043d6e0a5f9..483744223dd1 100644
--- a/drivers/video/fbdev/omap2/dss/dispc.h
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.h
@@ -908,6 +908,8 @@ static inline u16 DISPC_MFLAG_THRESHOLD_OFFSET(enum omap_plane plane)
 		return 0x0868;
 	case OMAP_DSS_VIDEO3:
 		return 0x086c;
+	case OMAP_DSS_WB:
+		return 0x0870;
 	default:
 		BUG();
 		return 0;
diff --git a/drivers/video/fbdev/omap2/dss/dispc_coefs.c b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
index 038c15b04215..038c15b04215 100644
--- a/drivers/video/fbdev/omap2/dss/dispc_coefs.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
diff --git a/drivers/video/fbdev/omap2/dss/display-sysfs.c b/drivers/gpu/drm/omapdrm/dss/display-sysfs.c
index 6ad0991f8259..6ad0991f8259 100644
--- a/drivers/video/fbdev/omap2/dss/display-sysfs.c
+++ b/drivers/gpu/drm/omapdrm/dss/display-sysfs.c
diff --git a/drivers/video/fbdev/omap2/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
index ef5b9027985d..ef5b9027985d 100644
--- a/drivers/video/fbdev/omap2/dss/display.c
+++ b/drivers/gpu/drm/omapdrm/dss/display.c
diff --git a/drivers/video/fbdev/omap2/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index fb45b6432968..7953e6a52346 100644
--- a/drivers/video/fbdev/omap2/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -395,7 +395,7 @@ static int dpi_display_enable(struct omap_dss_device *dssdev)
 		goto err_no_reg;
 	}
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		r = -ENODEV;
 		goto err_no_out_mgr;
diff --git a/drivers/video/fbdev/omap2/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index b3606def5b7b..43be4b2a7b05 100644
--- a/drivers/video/fbdev/omap2/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -144,7 +144,7 @@ struct dsi_reg { u16 module; u16 idx; };
 #define DSI_IRQ_TA_TIMEOUT	(1 << 20)
 #define DSI_IRQ_ERROR_MASK \
 	(DSI_IRQ_HS_TX_TIMEOUT | DSI_IRQ_LP_RX_TIMEOUT | DSI_IRQ_SYNC_LOST | \
-	DSI_IRQ_TA_TIMEOUT | DSI_IRQ_SYNC_LOST)
+	DSI_IRQ_TA_TIMEOUT)
 #define DSI_IRQ_CHANNEL_MASK	0xf
 
 /* Virtual channel interrupts */
@@ -3833,7 +3833,7 @@ static int dsi_enable_video_output(struct omap_dss_device *dssdev, int channel)
 	u16 word_count;
 	int r;
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		return -ENODEV;
 	}
diff --git a/drivers/video/fbdev/omap2/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
index bf407b6ba15c..bf407b6ba15c 100644
--- a/drivers/video/fbdev/omap2/dss/dss-of.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c
diff --git a/drivers/video/fbdev/omap2/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index 9200a8668b49..f95ff319e68e 100644
--- a/drivers/video/fbdev/omap2/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -1264,12 +1264,18 @@ static int dss_runtime_suspend(struct device *dev)
 {
 	dss_save_context();
 	dss_set_min_bus_tput(dev, 0);
+
+	pinctrl_pm_select_sleep_state(dev);
+
 	return 0;
 }
 
 static int dss_runtime_resume(struct device *dev)
 {
 	int r;
+
+	pinctrl_pm_select_default_state(dev);
+
 	/*
 	 * Set an arbitrarily high tput request to ensure OPP100.
 	 * What we should really do is to make a request to stay in OPP100,
diff --git a/drivers/video/fbdev/omap2/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
index 2406bcdb831a..9a6453235585 100644
--- a/drivers/video/fbdev/omap2/dss/dss.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h
@@ -343,7 +343,8 @@ u8 dsi_get_pixel_size(enum omap_dss_dsi_pixel_format fmt);
 #else
 static inline u8 dsi_get_pixel_size(enum omap_dss_dsi_pixel_format fmt)
 {
-	WARN("%s: DSI not compiled in, returning pixel_size as 0\n", __func__);
+	WARN(1, "%s: DSI not compiled in, returning pixel_size as 0\n",
+	     __func__);
 	return 0;
 }
 #endif
@@ -378,7 +379,6 @@ void dispc_lcd_enable_signal(bool enable);
 void dispc_pck_free_enable(bool enable);
 void dispc_enable_fifomerge(bool enable);
 void dispc_enable_gamma_table(bool enable);
-void dispc_set_loadmode(enum omap_dss_load_mode mode);
 
 typedef bool (*dispc_div_calc_func)(int lckd, int pckd, unsigned long lck,
 		unsigned long pck, void *data);
@@ -388,7 +388,6 @@ bool dispc_div_calc(unsigned long dispc,
 
 bool dispc_mgr_timings_ok(enum omap_channel channel,
 		const struct omap_video_timings *timings);
-unsigned long dispc_fclk_rate(void);
 int dispc_calc_clock_rates(unsigned long dispc_fclk_rate,
 		struct dispc_clock_info *cinfo);
 
@@ -398,9 +397,6 @@ void dispc_ovl_compute_fifo_thresholds(enum omap_plane plane,
 		u32 *fifo_low, u32 *fifo_high, bool use_fifomerge,
 		bool manual_update);
 
-unsigned long dispc_mgr_lclk_rate(enum omap_channel channel);
-unsigned long dispc_mgr_pclk_rate(enum omap_channel channel);
-unsigned long dispc_core_clk_rate(void);
 void dispc_mgr_set_clock_div(enum omap_channel channel,
 		const struct dispc_clock_info *cinfo);
 int dispc_mgr_get_clock_div(enum omap_channel channel,
diff --git a/drivers/video/fbdev/omap2/dss/dss_features.c b/drivers/gpu/drm/omapdrm/dss/dss_features.c
index b0b6dfd657bf..c886a2927f73 100644
--- a/drivers/video/fbdev/omap2/dss/dss_features.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss_features.c
@@ -46,7 +46,6 @@ struct omap_dss_features {
 
 	const int num_mgrs;
 	const int num_ovls;
-	const int num_wbs;
 	const enum omap_display_type *supported_displays;
 	const enum omap_dss_output_id *supported_outputs;
 	const enum omap_color_mode *supported_color_modes;
@@ -735,7 +734,6 @@ static const struct omap_dss_features omap4430_es1_0_dss_features  = {
 
 	.num_mgrs = 3,
 	.num_ovls = 4,
-	.num_wbs = 1,
 	.supported_displays = omap4_dss_supported_displays,
 	.supported_outputs = omap4_dss_supported_outputs,
 	.supported_color_modes = omap4_dss_supported_color_modes,
@@ -757,7 +755,6 @@ static const struct omap_dss_features omap4430_es2_0_1_2_dss_features = {
 
 	.num_mgrs = 3,
 	.num_ovls = 4,
-	.num_wbs = 1,
 	.supported_displays = omap4_dss_supported_displays,
 	.supported_outputs = omap4_dss_supported_outputs,
 	.supported_color_modes = omap4_dss_supported_color_modes,
@@ -779,7 +776,6 @@ static const struct omap_dss_features omap4_dss_features = {
 
 	.num_mgrs = 3,
 	.num_ovls = 4,
-	.num_wbs = 1,
 	.supported_displays = omap4_dss_supported_displays,
 	.supported_outputs = omap4_dss_supported_outputs,
 	.supported_color_modes = omap4_dss_supported_color_modes,
@@ -825,11 +821,6 @@ int dss_feat_get_num_ovls(void)
 }
 EXPORT_SYMBOL(dss_feat_get_num_ovls);
 
-int dss_feat_get_num_wbs(void)
-{
-	return omap_current_dss_features->num_wbs;
-}
-
 unsigned long dss_feat_get_param_min(enum dss_range_param param)
 {
 	return omap_current_dss_features->dss_params[param].min;
@@ -844,13 +835,11 @@ enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel
 {
 	return omap_current_dss_features->supported_displays[channel];
 }
-EXPORT_SYMBOL(dss_feat_get_supported_displays);
 
 enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel)
 {
 	return omap_current_dss_features->supported_outputs[channel];
 }
-EXPORT_SYMBOL(dss_feat_get_supported_outputs);
 
 enum omap_color_mode dss_feat_get_supported_color_modes(enum omap_plane plane)
 {
diff --git a/drivers/video/fbdev/omap2/dss/dss_features.h b/drivers/gpu/drm/omapdrm/dss/dss_features.h
index 100f7a2d0638..3d67d39f192f 100644
--- a/drivers/video/fbdev/omap2/dss/dss_features.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss_features.h
@@ -86,7 +86,6 @@ enum dss_range_param {
 };
 
 /* DSS Feature Functions */
-int dss_feat_get_num_wbs(void);
 unsigned long dss_feat_get_param_min(enum dss_range_param param);
 unsigned long dss_feat_get_param_max(enum dss_range_param param);
 enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane plane);
@@ -102,4 +101,8 @@ bool dss_feat_rotation_type_supported(enum omap_dss_rotation_type rot_type);
 bool dss_has_feature(enum dss_feat_id id);
 void dss_feat_get_reg_field(enum dss_feat_reg_field id, u8 *start, u8 *end);
 void dss_features_init(enum omapdss_version version);
+
+enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel);
+enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel);
+
 #endif
diff --git a/drivers/video/fbdev/omap2/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
index 53616b02b613..53616b02b613 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h
diff --git a/drivers/video/fbdev/omap2/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index 94c8d5549b4c..7103c659a534 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -343,7 +343,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_lock(&hdmi.lock);
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		r = -ENODEV;
 		goto err0;
diff --git a/drivers/video/fbdev/omap2/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
index fa72e735dad2..fa72e735dad2 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi4_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
diff --git a/drivers/video/fbdev/omap2/dss/hdmi4_core.h b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.h
index a069f96ec6f6..a069f96ec6f6 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi4_core.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.h
diff --git a/drivers/video/fbdev/omap2/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index b59ba7902be1..a955a2c4c061 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -373,7 +373,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_lock(&hdmi.lock);
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		r = -ENODEV;
 		goto err0;
diff --git a/drivers/video/fbdev/omap2/dss/hdmi5_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
index 8ea531d2652c..8ea531d2652c 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi5_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
diff --git a/drivers/video/fbdev/omap2/dss/hdmi5_core.h b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.h
index f2f1022c5516..f2f1022c5516 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi5_core.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.h
diff --git a/drivers/video/fbdev/omap2/dss/hdmi_common.c b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
index 1b8fcc6c4ba1..1b8fcc6c4ba1 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi_common.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
diff --git a/drivers/video/fbdev/omap2/dss/hdmi_phy.c b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
index 1f5d19c119ce..1f5d19c119ce 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi_phy.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
diff --git a/drivers/video/fbdev/omap2/dss/hdmi_pll.c b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
index 06e23a7c432c..06e23a7c432c 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi_pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
diff --git a/drivers/video/fbdev/omap2/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
index 7c544bc56fb5..7c544bc56fb5 100644
--- a/drivers/video/fbdev/omap2/dss/hdmi_wp.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
diff --git a/drivers/video/fbdev/omap2/dss/manager-sysfs.c b/drivers/gpu/drm/omapdrm/dss/manager-sysfs.c
index a7414fb12830..a7414fb12830 100644
--- a/drivers/video/fbdev/omap2/dss/manager-sysfs.c
+++ b/drivers/gpu/drm/omapdrm/dss/manager-sysfs.c
diff --git a/drivers/video/fbdev/omap2/dss/manager.c b/drivers/gpu/drm/omapdrm/dss/manager.c
index 1aac9b4191a9..08a67f4f6a20 100644
--- a/drivers/video/fbdev/omap2/dss/manager.c
+++ b/drivers/gpu/drm/omapdrm/dss/manager.c
@@ -210,7 +210,7 @@ static int dss_mgr_check_lcd_config(struct omap_overlay_manager *mgr,
 		return -EINVAL;
 
 	/* fifohandcheck should be used only with stallmode */
-	if (stallmode == false && fifohandcheck == true)
+	if (!stallmode && fifohandcheck)
 		return -EINVAL;
 
 	/*
diff --git a/drivers/video/fbdev/omap2/dss/omapdss-boot-init.c b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
index 8b6f6d5fdd68..136d30484d02 100644
--- a/drivers/video/fbdev/omap2/dss/omapdss-boot-init.c
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
@@ -199,10 +199,8 @@ static int __init omapdss_boot_init(void)
 	omapdss_walk_device(dss, true);
 
 	for_each_available_child_of_node(dss, child) {
-		if (!of_find_property(child, "compatible", NULL)) {
-			of_node_put(child);
+		if (!of_find_property(child, "compatible", NULL))
 			continue;
-		}
 
 		omapdss_walk_device(child, true);
 	}
diff --git a/drivers/video/fbdev/omap2/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 16072159bd24..16072159bd24 100644
--- a/drivers/video/fbdev/omap2/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
diff --git a/drivers/video/fbdev/omap2/dss/overlay-sysfs.c b/drivers/gpu/drm/omapdrm/dss/overlay-sysfs.c
index 4cc5ddebfb34..4cc5ddebfb34 100644
--- a/drivers/video/fbdev/omap2/dss/overlay-sysfs.c
+++ b/drivers/gpu/drm/omapdrm/dss/overlay-sysfs.c
diff --git a/drivers/video/fbdev/omap2/dss/overlay.c b/drivers/gpu/drm/omapdrm/dss/overlay.c
index 2f7cee985cdd..2f7cee985cdd 100644
--- a/drivers/video/fbdev/omap2/dss/overlay.c
+++ b/drivers/gpu/drm/omapdrm/dss/overlay.c
diff --git a/drivers/video/fbdev/omap2/dss/pll.c b/drivers/gpu/drm/omapdrm/dss/pll.c
index f974ddcd3b6e..f974ddcd3b6e 100644
--- a/drivers/video/fbdev/omap2/dss/pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/pll.c
diff --git a/drivers/video/fbdev/omap2/dss/rfbi.c b/drivers/gpu/drm/omapdrm/dss/rfbi.c
index 1525a494d057..aea6a1d0fb20 100644
--- a/drivers/video/fbdev/omap2/dss/rfbi.c
+++ b/drivers/gpu/drm/omapdrm/dss/rfbi.c
@@ -880,7 +880,7 @@ static int rfbi_display_enable(struct omap_dss_device *dssdev)
 	struct omap_dss_device *out = &rfbi.output;
 	int r;
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		return -ENODEV;
 	}
diff --git a/drivers/video/fbdev/omap2/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index 5843580a1deb..d747cc6b59e1 100644
--- a/drivers/video/fbdev/omap2/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -136,7 +136,7 @@ static int sdi_display_enable(struct omap_dss_device *dssdev)
 	unsigned long pck;
 	int r;
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("failed to enable display: no output/manager\n");
 		return -ENODEV;
 	}
diff --git a/drivers/video/fbdev/omap2/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index d05a54922ba6..08f9def76e27 100644
--- a/drivers/video/fbdev/omap2/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -515,7 +515,7 @@ static int venc_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_lock(&venc.venc_lock);
 
-	if (out == NULL || out->manager == NULL) {
+	if (out->manager == NULL) {
 		DSSERR("Failed to enable display: no output/manager\n");
 		r = -ENODEV;
 		goto err0;
diff --git a/drivers/video/fbdev/omap2/dss/video-pll.c b/drivers/gpu/drm/omapdrm/dss/video-pll.c
index b1ec59e42940..b1ec59e42940 100644
--- a/drivers/video/fbdev/omap2/dss/video-pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/video-pll.c
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index ad09590e8a46..2ed0754ed19e 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -524,7 +524,7 @@ struct drm_crtc *omap_crtc_init(struct drm_device *dev,
 	omap_crtc->mgr = omap_dss_get_overlay_manager(channel);
 
 	ret = drm_crtc_init_with_planes(dev, crtc, plane, NULL,
-					&omap_crtc_funcs);
+					&omap_crtc_funcs, NULL);
 	if (ret < 0) {
 		kfree(omap_crtc);
 		return NULL;
diff --git a/drivers/gpu/drm/omapdrm/omap_debugfs.c b/drivers/gpu/drm/omapdrm/omap_debugfs.c
index ee91a25127f9..6f5fc14fc015 100644
--- a/drivers/gpu/drm/omapdrm/omap_debugfs.c
+++ b/drivers/gpu/drm/omapdrm/omap_debugfs.c
@@ -51,6 +51,7 @@ static int mm_show(struct seq_file *m, void *arg)
 	return drm_mm_dump_table(m, &dev->vma_offset_manager->vm_addr_space_mm);
 }
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 static int fb_show(struct seq_file *m, void *arg)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -73,12 +74,15 @@ static int fb_show(struct seq_file *m, void *arg)
 
 	return 0;
 }
+#endif
 
 /* list of debufs files that are applicable to all devices */
 static struct drm_info_list omap_debugfs_list[] = {
 	{"gem", gem_show, 0},
 	{"mm", mm_show, 0},
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	{"fb", fb_show, 0},
+#endif
 };
 
 /* list of debugfs files that are specific to devices with dmm/tiler */
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
index 7841970de48d..dfebdc4aa0f2 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
@@ -363,6 +363,7 @@ struct tiler_block *tiler_reserve_2d(enum tiler_fmt fmt, uint16_t w,
 	u32 min_align = 128;
 	int ret;
 	unsigned long flags;
+	size_t slot_bytes;
 
 	BUG_ON(!validfmt(fmt));
 
@@ -371,13 +372,15 @@ struct tiler_block *tiler_reserve_2d(enum tiler_fmt fmt, uint16_t w,
 	h = DIV_ROUND_UP(h, geom[fmt].slot_h);
 
 	/* convert alignment to slots */
-	min_align = max(min_align, (geom[fmt].slot_w * geom[fmt].cpp));
-	align = ALIGN(align, min_align);
-	align /= geom[fmt].slot_w * geom[fmt].cpp;
+	slot_bytes = geom[fmt].slot_w * geom[fmt].cpp;
+	min_align = max(min_align, slot_bytes);
+	align = (align > min_align) ? ALIGN(align, min_align) : min_align;
+	align /= slot_bytes;
 
 	block->fmt = fmt;
 
-	ret = tcm_reserve_2d(containers[fmt], w, h, align, &block->area);
+	ret = tcm_reserve_2d(containers[fmt], w, h, align, -1, slot_bytes,
+			&block->area);
 	if (ret) {
 		kfree(block);
 		return ERR_PTR(-ENOMEM);
@@ -739,8 +742,7 @@ static int omap_dmm_probe(struct platform_device *dev)
 	   programming during reill operations */
 	for (i = 0; i < omap_dmm->num_lut; i++) {
 		omap_dmm->tcm[i] = sita_init(omap_dmm->container_width,
-						omap_dmm->container_height,
-						NULL);
+						omap_dmm->container_height);
 
 		if (!omap_dmm->tcm[i]) {
 			dev_err(&dev->dev, "failed to allocate container\n");
@@ -1030,4 +1032,3 @@ struct platform_driver omap_dmm_driver = {
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Andy Gross <andy.gross@ti.com>");
 MODULE_DESCRIPTION("OMAP DMM/Tiler Driver");
-MODULE_ALIAS("platform:" DMM_DRIVER_NAME);
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 5c6609cbb6a2..dfafdb602ad2 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -547,14 +547,19 @@ static int ioctl_set_param(struct drm_device *dev, void *data,
 	return 0;
 }
 
+#define OMAP_BO_USER_MASK	0x00ffffff	/* flags settable by userspace */
+
 static int ioctl_gem_new(struct drm_device *dev, void *data,
 		struct drm_file *file_priv)
 {
 	struct drm_omap_gem_new *args = data;
+	u32 flags = args->flags & OMAP_BO_USER_MASK;
+
 	VERB("%p:%p: size=0x%08x, flags=%08x", dev, file_priv,
-			args->size.bytes, args->flags);
-	return omap_gem_new_handle(dev, file_priv, args->size,
-			args->flags, &args->handle);
+	     args->size.bytes, flags);
+
+	return omap_gem_new_handle(dev, file_priv, args->size, flags,
+				   &args->handle);
 }
 
 static int ioctl_gem_cpu_prep(struct drm_device *dev, void *data,
@@ -692,10 +697,6 @@ static int dev_load(struct drm_device *dev, unsigned long flags)
 		drm_crtc_vblank_off(priv->crtcs[i]);
 
 	priv->fbdev = omap_fbdev_init(dev);
-	if (!priv->fbdev) {
-		dev_warn(dev->dev, "omap_fbdev_init failed\n");
-		/* well, limp along without an fbdev.. maybe X11 will work? */
-	}
 
 	/* store off drm_device for use in pm ops */
 	dev_set_drvdata(dev->dev, dev);
@@ -831,7 +832,8 @@ static const struct file_operations omapdriver_fops = {
 };
 
 static struct drm_driver omap_drm_driver = {
-	.driver_features = DRIVER_MODESET | DRIVER_GEM  | DRIVER_PRIME,
+	.driver_features = DRIVER_MODESET | DRIVER_GEM  | DRIVER_PRIME |
+		DRIVER_ATOMIC,
 	.load = dev_load,
 	.unload = dev_unload,
 	.open = dev_open,
@@ -928,35 +930,23 @@ static struct platform_driver pdev = {
 	.remove = pdev_remove,
 };
 
+static struct platform_driver * const drivers[] = {
+	&omap_dmm_driver,
+	&pdev,
+};
+
 static int __init omap_drm_init(void)
 {
-	int r;
-
 	DBG("init");
 
-	r = platform_driver_register(&omap_dmm_driver);
-	if (r) {
-		pr_err("DMM driver registration failed\n");
-		return r;
-	}
-
-	r = platform_driver_register(&pdev);
-	if (r) {
-		pr_err("omapdrm driver registration failed\n");
-		platform_driver_unregister(&omap_dmm_driver);
-		return r;
-	}
-
-	return 0;
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
 static void __exit omap_drm_fini(void)
 {
 	DBG("fini");
 
-	platform_driver_unregister(&pdev);
-
-	platform_driver_unregister(&omap_dmm_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
 /* need late_initcall() so we load after dss_driver's are loaded */
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index 5c367aad8a6e..9e0030731c37 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -36,11 +36,7 @@
 
 #define MODULE_NAME     "omapdrm"
 
-/* max # of mapper-id's that can be assigned.. todo, come up with a better
- * (but still inexpensive) way to store/access per-buffer mapper private
- * data..
- */
-#define MAX_MAPPERS 2
+struct omap_drm_usergart;
 
 /* parameters which describe (unrotated) coordinates of scanout within a fb: */
 struct omap_drm_window {
@@ -97,6 +93,7 @@ struct omap_drm_private {
 	/* list of GEM objects: */
 	struct list_head obj_list;
 
+	struct omap_drm_usergart *usergart;
 	bool has_dmm;
 
 	/* properties: */
@@ -138,8 +135,18 @@ void omap_irq_unregister(struct drm_device *dev, struct omap_drm_irq *irq);
 void omap_drm_irq_uninstall(struct drm_device *dev);
 int omap_drm_irq_install(struct drm_device *dev);
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev);
 void omap_fbdev_free(struct drm_device *dev);
+#else
+static inline struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev)
+{
+	return NULL;
+}
+static inline void omap_fbdev_free(struct drm_device *dev)
+{
+}
+#endif
 
 struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc);
 enum omap_channel omap_crtc_channel(struct drm_crtc *crtc);
@@ -172,9 +179,9 @@ void copy_timings_drm_to_omap(struct omap_video_timings *timings,
 uint32_t omap_framebuffer_get_formats(uint32_t *pixel_formats,
 		uint32_t max_formats, enum omap_color_mode supported_modes);
 struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
-		struct drm_file *file, struct drm_mode_fb_cmd2 *mode_cmd);
+		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
 struct drm_framebuffer *omap_framebuffer_init(struct drm_device *dev,
-		struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
 struct drm_gem_object *omap_framebuffer_bo(struct drm_framebuffer *fb, int p);
 int omap_framebuffer_pin(struct drm_framebuffer *fb);
 void omap_framebuffer_unpin(struct drm_framebuffer *fb);
@@ -248,7 +255,7 @@ struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder);
 
 static inline int objects_lookup(struct drm_device *dev,
 		struct drm_file *filp, uint32_t pixel_format,
-		struct drm_gem_object **bos, uint32_t *handles)
+		struct drm_gem_object **bos, const uint32_t *handles)
 {
 	int i, n = drm_format_num_planes(pixel_format);
 
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
index 7d9b32a0eb43..61714e9670ae 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c
@@ -110,8 +110,6 @@ static int omap_encoder_update(struct drm_encoder *encoder,
 	struct omap_dss_driver *dssdrv = dssdev->driver;
 	int ret;
 
-	dssdev->src->manager = omap_dss_get_overlay_manager(channel);
-
 	if (dssdrv->check_timings) {
 		ret = dssdrv->check_timings(dssdev, timings);
 	} else {
@@ -178,7 +176,7 @@ struct drm_encoder *omap_encoder_init(struct drm_device *dev,
 	encoder = &omap_encoder->base;
 
 	drm_encoder_init(dev, encoder, &omap_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 	drm_encoder_helper_add(encoder, &omap_encoder_helper_funcs);
 
 	return encoder;
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index 636a1f921569..ad202dfc1a49 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -364,7 +364,7 @@ void omap_framebuffer_describe(struct drm_framebuffer *fb, struct seq_file *m)
 #endif
 
 struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
-		struct drm_file *file, struct drm_mode_fb_cmd2 *mode_cmd)
+		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *bos[4];
 	struct drm_framebuffer *fb;
@@ -386,7 +386,7 @@ struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
 }
 
 struct drm_framebuffer *omap_framebuffer_init(struct drm_device *dev,
-		struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos)
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos)
 {
 	struct omap_framebuffer *omap_fb = NULL;
 	struct drm_framebuffer *fb = NULL;
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index 24f92bea39c7..3cb16f0cf381 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -295,6 +295,10 @@ fini:
 	drm_fb_helper_fini(helper);
 fail:
 	kfree(fbdev);
+
+	dev_warn(dev->dev, "omap_fbdev_init failed\n");
+	/* well, limp along without an fbdev.. maybe X11 will work? */
+
 	return NULL;
 }
 
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 7ed08fdc4c42..8495a1a4b617 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -19,30 +19,22 @@
 
 #include <linux/shmem_fs.h>
 #include <linux/spinlock.h>
+#include <linux/pfn_t.h>
 
 #include <drm/drm_vma_manager.h>
 
 #include "omap_drv.h"
 #include "omap_dmm_tiler.h"
 
-/* remove these once drm core helpers are merged */
-struct page **_drm_gem_get_pages(struct drm_gem_object *obj, gfp_t gfpmask);
-void _drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
-		bool dirty, bool accessed);
-int _drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size);
-
 /*
  * GEM buffer object implementation.
  */
 
-#define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
-
 /* note: we use upper 8 bits of flags for driver-internal flags: */
-#define OMAP_BO_DMA			0x01000000	/* actually is physically contiguous */
+#define OMAP_BO_DMA		0x01000000	/* actually is physically contiguous */
 #define OMAP_BO_EXT_SYNC	0x02000000	/* externally allocated sync object */
 #define OMAP_BO_EXT_MEM		0x04000000	/* externally allocated memory */
 
-
 struct omap_gem_object {
 	struct drm_gem_object base;
 
@@ -119,8 +111,7 @@ struct omap_gem_object {
 	} *sync;
 };
 
-static int get_pages(struct drm_gem_object *obj, struct page ***pages);
-static uint64_t mmap_offset(struct drm_gem_object *obj);
+#define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
 
 /* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
  * not necessarily pinned in TILER all the time, and (b) when they are
@@ -134,27 +125,69 @@ static uint64_t mmap_offset(struct drm_gem_object *obj);
  * for later..
  */
 #define NUM_USERGART_ENTRIES 2
-struct usergart_entry {
+struct omap_drm_usergart_entry {
 	struct tiler_block *block;	/* the reserved tiler block */
 	dma_addr_t paddr;
 	struct drm_gem_object *obj;	/* the current pinned obj */
 	pgoff_t obj_pgoff;		/* page offset of obj currently
 					   mapped in */
 };
-static struct {
-	struct usergart_entry entry[NUM_USERGART_ENTRIES];
+
+struct omap_drm_usergart {
+	struct omap_drm_usergart_entry entry[NUM_USERGART_ENTRIES];
 	int height;				/* height in rows */
 	int height_shift;		/* ilog2(height in rows) */
 	int slot_shift;			/* ilog2(width per slot) */
 	int stride_pfn;			/* stride in pages */
 	int last;				/* index of last used entry */
-} *usergart;
+};
+
+/* -----------------------------------------------------------------------------
+ * Helpers
+ */
+
+/** get mmap offset */
+static uint64_t mmap_offset(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	int ret;
+	size_t size;
+
+	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+	/* Make it mmapable */
+	size = omap_gem_mmap_size(obj);
+	ret = drm_gem_create_mmap_offset_size(obj, size);
+	if (ret) {
+		dev_err(dev->dev, "could not allocate mmap offset\n");
+		return 0;
+	}
+
+	return drm_vma_node_offset_addr(&obj->vma_node);
+}
+
+/* GEM objects can either be allocated from contiguous memory (in which
+ * case obj->filp==NULL), or w/ shmem backing (obj->filp!=NULL).  But non
+ * contiguous buffers can be remapped in TILER/DMM if they need to be
+ * contiguous... but we don't do this all the time to reduce pressure
+ * on TILER/DMM space when we know at allocation time that the buffer
+ * will need to be scanned out.
+ */
+static inline bool is_shmem(struct drm_gem_object *obj)
+{
+	return obj->filp != NULL;
+}
+
+/* -----------------------------------------------------------------------------
+ * Eviction
+ */
 
 static void evict_entry(struct drm_gem_object *obj,
-		enum tiler_fmt fmt, struct usergart_entry *entry)
+		enum tiler_fmt fmt, struct omap_drm_usergart_entry *entry)
 {
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
-	int n = usergart[fmt].height;
+	struct omap_drm_private *priv = obj->dev->dev_private;
+	int n = priv->usergart[fmt].height;
 	size_t size = PAGE_SIZE * n;
 	loff_t off = mmap_offset(obj) +
 			(entry->obj_pgoff << PAGE_SHIFT);
@@ -180,46 +213,25 @@ static void evict_entry(struct drm_gem_object *obj,
 static void evict(struct drm_gem_object *obj)
 {
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
+	struct omap_drm_private *priv = obj->dev->dev_private;
 
 	if (omap_obj->flags & OMAP_BO_TILED) {
 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 		int i;
 
-		if (!usergart)
-			return;
-
 		for (i = 0; i < NUM_USERGART_ENTRIES; i++) {
-			struct usergart_entry *entry = &usergart[fmt].entry[i];
+			struct omap_drm_usergart_entry *entry =
+				&priv->usergart[fmt].entry[i];
+
 			if (entry->obj == obj)
 				evict_entry(obj, fmt, entry);
 		}
 	}
 }
 
-/* GEM objects can either be allocated from contiguous memory (in which
- * case obj->filp==NULL), or w/ shmem backing (obj->filp!=NULL).  But non
- * contiguous buffers can be remapped in TILER/DMM if they need to be
- * contiguous... but we don't do this all the time to reduce pressure
- * on TILER/DMM space when we know at allocation time that the buffer
- * will need to be scanned out.
- */
-static inline bool is_shmem(struct drm_gem_object *obj)
-{
-	return obj->filp != NULL;
-}
-
-/**
- * shmem buffers that are mapped cached can simulate coherency via using
- * page faulting to keep track of dirty pages
+/* -----------------------------------------------------------------------------
+ * Page Management
  */
-static inline bool is_cached_coherent(struct drm_gem_object *obj)
-{
-	struct omap_gem_object *omap_obj = to_omap_bo(obj);
-	return is_shmem(obj) &&
-		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED);
-}
-
-static DEFINE_SPINLOCK(sync_lock);
 
 /** ensure backing pages are allocated */
 static int omap_gem_attach_pages(struct drm_gem_object *obj)
@@ -272,6 +284,28 @@ free_pages:
 	return ret;
 }
 
+/* acquire pages when needed (for example, for DMA where physically
+ * contiguous buffer is not required
+ */
+static int get_pages(struct drm_gem_object *obj, struct page ***pages)
+{
+	struct omap_gem_object *omap_obj = to_omap_bo(obj);
+	int ret = 0;
+
+	if (is_shmem(obj) && !omap_obj->pages) {
+		ret = omap_gem_attach_pages(obj);
+		if (ret) {
+			dev_err(obj->dev->dev, "could not attach pages\n");
+			return ret;
+		}
+	}
+
+	/* TODO: even phys-contig.. we should have a list of pages? */
+	*pages = omap_obj->pages;
+
+	return 0;
+}
+
 /** release backing pages */
 static void omap_gem_detach_pages(struct drm_gem_object *obj)
 {
@@ -301,26 +335,6 @@ uint32_t omap_gem_flags(struct drm_gem_object *obj)
 	return to_omap_bo(obj)->flags;
 }
 
-/** get mmap offset */
-static uint64_t mmap_offset(struct drm_gem_object *obj)
-{
-	struct drm_device *dev = obj->dev;
-	int ret;
-	size_t size;
-
-	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
-	/* Make it mmapable */
-	size = omap_gem_mmap_size(obj);
-	ret = drm_gem_create_mmap_offset_size(obj, size);
-	if (ret) {
-		dev_err(dev->dev, "could not allocate mmap offset\n");
-		return 0;
-	}
-
-	return drm_vma_node_offset_addr(&obj->vma_node);
-}
-
 uint64_t omap_gem_mmap_offset(struct drm_gem_object *obj)
 {
 	uint64_t offset;
@@ -362,6 +376,10 @@ int omap_gem_tiled_size(struct drm_gem_object *obj, uint16_t *w, uint16_t *h)
 	return -EINVAL;
 }
 
+/* -----------------------------------------------------------------------------
+ * Fault Handling
+ */
+
 /* Normal handling for the case of faulting in non-tiled buffers */
 static int fault_1d(struct drm_gem_object *obj,
 		struct vm_area_struct *vma, struct vm_fault *vmf)
@@ -385,7 +403,8 @@ static int fault_1d(struct drm_gem_object *obj,
 	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
 			pfn, pfn << PAGE_SHIFT);
 
-	return vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
+	return vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
+			__pfn_to_pfn_t(pfn, PFN_DEV));
 }
 
 /* Special handling for the case of faulting in 2d tiled buffers */
@@ -393,7 +412,8 @@ static int fault_2d(struct drm_gem_object *obj,
 		struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct omap_gem_object *omap_obj = to_omap_bo(obj);
-	struct usergart_entry *entry;
+	struct omap_drm_private *priv = obj->dev->dev_private;
+	struct omap_drm_usergart_entry *entry;
 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 	struct page *pages[64];  /* XXX is this too much to have on stack? */
 	unsigned long pfn;
@@ -406,8 +426,8 @@ static int fault_2d(struct drm_gem_object *obj,
 	 * that need to be mapped in to fill 4kb wide CPU page.  If the slot
 	 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
 	 */
-	const int n = usergart[fmt].height;
-	const int n_shift = usergart[fmt].height_shift;
+	const int n = priv->usergart[fmt].height;
+	const int n_shift = priv->usergart[fmt].height_shift;
 
 	/*
 	 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
@@ -428,11 +448,11 @@ static int fault_2d(struct drm_gem_object *obj,
 	base_pgoff = round_down(pgoff, m << n_shift);
 
 	/* figure out buffer width in slots */
-	slots = omap_obj->width >> usergart[fmt].slot_shift;
+	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;
 
 	vaddr = vmf->virtual_address - ((pgoff - base_pgoff) << PAGE_SHIFT);
 
-	entry = &usergart[fmt].entry[usergart[fmt].last];
+	entry = &priv->usergart[fmt].entry[priv->usergart[fmt].last];
 
 	/* evict previous buffer using this usergart entry, if any: */
 	if (entry->obj)
@@ -478,13 +498,15 @@ static int fault_2d(struct drm_gem_object *obj,
 			pfn, pfn << PAGE_SHIFT);
 
 	for (i = n; i > 0; i--) {
-		vm_insert_mixed(vma, (unsigned long)vaddr, pfn);
-		pfn += usergart[fmt].stride_pfn;
+		vm_insert_mixed(vma, (unsigned long)vaddr,
+				__pfn_to_pfn_t(pfn, PFN_DEV));
+		pfn += priv->usergart[fmt].stride_pfn;
 		vaddr += PAGE_SIZE * m;
 	}
 
 	/* simple round-robin: */
-	usergart[fmt].last = (usergart[fmt].last + 1) % NUM_USERGART_ENTRIES;
+	priv->usergart[fmt].last = (priv->usergart[fmt].last + 1)
+				 % NUM_USERGART_ENTRIES;
 
 	return 0;
 }
@@ -596,6 +618,9 @@ int omap_gem_mmap_obj(struct drm_gem_object *obj,
 	return 0;
 }
 
+/* -----------------------------------------------------------------------------
+ * Dumb Buffers
+ */
 
 /**
  * omap_gem_dumb_create	-	create a dumb buffer
@@ -653,6 +678,7 @@ fail:
 	return ret;
 }
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 /* Set scrolling position.  This allows us to implement fast scrolling
  * for console.
  *
@@ -689,6 +715,22 @@ fail:
 
 	return ret;
 }
+#endif
+
+/* -----------------------------------------------------------------------------
+ * Memory Management & DMA Sync
+ */
+
+/**
+ * shmem buffers that are mapped cached can simulate coherency via using
+ * page faulting to keep track of dirty pages
+ */
+static inline bool is_cached_coherent(struct drm_gem_object *obj)
+{
+	struct omap_gem_object *omap_obj = to_omap_bo(obj);
+	return is_shmem(obj) &&
+		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED);
+}
 
 /* Sync the buffer for CPU access.. note pages should already be
  * attached, ie. omap_gem_get_pages()
@@ -865,28 +907,6 @@ int omap_gem_tiled_stride(struct drm_gem_object *obj, uint32_t orient)
 	return ret;
 }
 
-/* acquire pages when needed (for example, for DMA where physically
- * contiguous buffer is not required
- */
-static int get_pages(struct drm_gem_object *obj, struct page ***pages)
-{
-	struct omap_gem_object *omap_obj = to_omap_bo(obj);
-	int ret = 0;
-
-	if (is_shmem(obj) && !omap_obj->pages) {
-		ret = omap_gem_attach_pages(obj);
-		if (ret) {
-			dev_err(obj->dev->dev, "could not attach pages\n");
-			return ret;
-		}
-	}
-
-	/* TODO: even phys-contig.. we should have a list of pages? */
-	*pages = omap_obj->pages;
-
-	return 0;
-}
-
 /* if !remap, and we don't have pages backing, then fail, rather than
  * increasing the pin count (which we don't really do yet anyways,
  * because we don't support swapping pages back out).  And 'remap'
@@ -924,6 +944,7 @@ int omap_gem_put_pages(struct drm_gem_object *obj)
 	return 0;
 }
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 /* Get kernel virtual address for CPU access.. this more or less only
  * exists for omap_fbdev.  This should be called with struct_mutex
  * held.
@@ -942,6 +963,11 @@ void *omap_gem_vaddr(struct drm_gem_object *obj)
 	}
 	return omap_obj->vaddr;
 }
+#endif
+
+/* -----------------------------------------------------------------------------
+ * Power Management
+ */
 
 #ifdef CONFIG_PM
 /* re-pin objects in DMM in resume path: */
@@ -971,6 +997,10 @@ int omap_gem_resume(struct device *dev)
 }
 #endif
 
+/* -----------------------------------------------------------------------------
+ * DebugFS
+ */
+
 #ifdef CONFIG_DEBUG_FS
 void omap_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 {
@@ -1017,9 +1047,12 @@ void omap_gem_describe_objects(struct list_head *list, struct seq_file *m)
 }
 #endif
 
-/* Buffer Synchronization:
+/* -----------------------------------------------------------------------------
+ * Buffer Synchronization
  */
 
+static DEFINE_SPINLOCK(sync_lock);
+
 struct omap_gem_sync_waiter {
 	struct list_head list;
 	struct omap_gem_object *omap_obj;
@@ -1265,6 +1298,10 @@ unlock:
 	return ret;
 }
 
+/* -----------------------------------------------------------------------------
+ * Constructor & Destructor
+ */
+
 /* don't call directly.. called from GEM core when it is time to actually
  * free the object..
  */
@@ -1282,8 +1319,6 @@ void omap_gem_free_object(struct drm_gem_object *obj)
 	list_del(&omap_obj->mm_list);
 	spin_unlock(&priv->list_lock);
 
-	drm_gem_free_mmap_offset(obj);
-
 	/* this means the object is still pinned.. which really should
 	 * not happen.  I think..
 	 */
@@ -1308,31 +1343,7 @@ void omap_gem_free_object(struct drm_gem_object *obj)
 
 	drm_gem_object_release(obj);
 
-	kfree(obj);
-}
-
-/* convenience method to construct a GEM buffer object, and userspace handle */
-int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
-		union omap_gem_size gsize, uint32_t flags, uint32_t *handle)
-{
-	struct drm_gem_object *obj;
-	int ret;
-
-	obj = omap_gem_new(dev, gsize, flags);
-	if (!obj)
-		return -ENOMEM;
-
-	ret = drm_gem_handle_create(file, obj, handle);
-	if (ret) {
-		drm_gem_object_release(obj);
-		kfree(obj); /* TODO isn't there a dtor to call? just copying i915 */
-		return ret;
-	}
-
-	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(obj);
-
-	return 0;
+	kfree(omap_obj);
 }
 
 /* GEM buffer object constructor */
@@ -1341,15 +1352,15 @@ struct drm_gem_object *omap_gem_new(struct drm_device *dev,
 {
 	struct omap_drm_private *priv = dev->dev_private;
 	struct omap_gem_object *omap_obj;
-	struct drm_gem_object *obj = NULL;
+	struct drm_gem_object *obj;
 	struct address_space *mapping;
 	size_t size;
 	int ret;
 
 	if (flags & OMAP_BO_TILED) {
-		if (!usergart) {
+		if (!priv->usergart) {
 			dev_err(dev->dev, "Tiled buffers require DMM\n");
-			goto fail;
+			return NULL;
 		}
 
 		/* tiled buffers are always shmem paged backed.. when they are
@@ -1420,16 +1431,42 @@ struct drm_gem_object *omap_gem_new(struct drm_device *dev,
 	return obj;
 
 fail:
-	if (obj)
+	omap_gem_free_object(obj);
+	return NULL;
+}
+
+/* convenience method to construct a GEM buffer object, and userspace handle */
+int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
+		union omap_gem_size gsize, uint32_t flags, uint32_t *handle)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	obj = omap_gem_new(dev, gsize, flags);
+	if (!obj)
+		return -ENOMEM;
+
+	ret = drm_gem_handle_create(file, obj, handle);
+	if (ret) {
 		omap_gem_free_object(obj);
+		return ret;
+	}
 
-	return NULL;
+	/* drop reference from allocate - handle holds it now */
+	drm_gem_object_unreference_unlocked(obj);
+
+	return 0;
 }
 
-/* init/cleanup.. if DMM is used, we need to set some stuff up.. */
+/* -----------------------------------------------------------------------------
+ * Init & Cleanup
+ */
+
+/* If DMM is used, we need to set some stuff up.. */
 void omap_gem_init(struct drm_device *dev)
 {
 	struct omap_drm_private *priv = dev->dev_private;
+	struct omap_drm_usergart *usergart;
 	const enum tiler_fmt fmts[] = {
 			TILFMT_8BIT, TILFMT_16BIT, TILFMT_32BIT
 	};
@@ -1458,10 +1495,11 @@ void omap_gem_init(struct drm_device *dev)
 		usergart[i].stride_pfn = tiler_stride(fmts[i], 0) >> PAGE_SHIFT;
 		usergart[i].slot_shift = ilog2((PAGE_SIZE / h) >> i);
 		for (j = 0; j < NUM_USERGART_ENTRIES; j++) {
-			struct usergart_entry *entry = &usergart[i].entry[j];
-			struct tiler_block *block =
-					tiler_reserve_2d(fmts[i], w, h,
-							PAGE_SIZE);
+			struct omap_drm_usergart_entry *entry;
+			struct tiler_block *block;
+
+			entry = &usergart[i].entry[j];
+			block = tiler_reserve_2d(fmts[i], w, h, PAGE_SIZE);
 			if (IS_ERR(block)) {
 				dev_err(dev->dev,
 						"reserve failed: %d, %d, %ld\n",
@@ -1477,13 +1515,16 @@ void omap_gem_init(struct drm_device *dev)
 		}
 	}
 
+	priv->usergart = usergart;
 	priv->has_dmm = true;
 }
 
 void omap_gem_deinit(struct drm_device *dev)
 {
+	struct omap_drm_private *priv = dev->dev_private;
+
 	/* I believe we can rely on there being no more outstanding GEM
 	 * objects which could depend on usergart/dmm at this point.
 	 */
-	kfree(usergart);
+	kfree(priv->usergart);
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 3054bda72688..d75b197eff46 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -188,33 +188,6 @@ static const struct drm_plane_helper_funcs omap_plane_helper_funcs = {
 	.atomic_disable = omap_plane_atomic_disable,
 };
 
-static void omap_plane_reset(struct drm_plane *plane)
-{
-	struct omap_plane *omap_plane = to_omap_plane(plane);
-	struct omap_plane_state *omap_state;
-
-	if (plane->state && plane->state->fb)
-		drm_framebuffer_unreference(plane->state->fb);
-
-	kfree(plane->state);
-	plane->state = NULL;
-
-	omap_state = kzalloc(sizeof(*omap_state), GFP_KERNEL);
-	if (omap_state == NULL)
-		return;
-
-	/*
-	 * Set defaults depending on whether we are a primary or overlay
-	 * plane.
-	 */
-	omap_state->zorder = plane->type == DRM_PLANE_TYPE_PRIMARY
-			   ? 0 : omap_plane->id;
-	omap_state->base.rotation = BIT(DRM_ROTATE_0);
-
-	plane->state = &omap_state->base;
-	plane->state->plane = plane;
-}
-
 static void omap_plane_destroy(struct drm_plane *plane)
 {
 	struct omap_plane *omap_plane = to_omap_plane(plane);
@@ -270,6 +243,32 @@ static void omap_plane_atomic_destroy_state(struct drm_plane *plane,
 	kfree(to_omap_plane_state(state));
 }
 
+static void omap_plane_reset(struct drm_plane *plane)
+{
+	struct omap_plane *omap_plane = to_omap_plane(plane);
+	struct omap_plane_state *omap_state;
+
+	if (plane->state) {
+		omap_plane_atomic_destroy_state(plane, plane->state);
+		plane->state = NULL;
+	}
+
+	omap_state = kzalloc(sizeof(*omap_state), GFP_KERNEL);
+	if (omap_state == NULL)
+		return;
+
+	/*
+	 * Set defaults depending on whether we are a primary or overlay
+	 * plane.
+	 */
+	omap_state->zorder = plane->type == DRM_PLANE_TYPE_PRIMARY
+			   ? 0 : omap_plane->id;
+	omap_state->base.rotation = BIT(DRM_ROTATE_0);
+
+	plane->state = &omap_state->base;
+	plane->state->plane = plane;
+}
+
 static int omap_plane_atomic_set_property(struct drm_plane *plane,
 					  struct drm_plane_state *state,
 					  struct drm_property *property,
@@ -366,7 +365,7 @@ struct drm_plane *omap_plane_init(struct drm_device *dev,
 
 	ret = drm_universal_plane_init(dev, plane, (1 << priv->num_crtcs) - 1,
 				       &omap_plane_funcs, omap_plane->formats,
-				       omap_plane->nformats, type);
+				       omap_plane->nformats, type, NULL);
 	if (ret < 0)
 		goto error;
 
diff --git a/drivers/gpu/drm/omapdrm/tcm-sita.c b/drivers/gpu/drm/omapdrm/tcm-sita.c
index efb609510540..c10fdfc0930f 100644
--- a/drivers/gpu/drm/omapdrm/tcm-sita.c
+++ b/drivers/gpu/drm/omapdrm/tcm-sita.c
@@ -5,8 +5,9 @@
  *
  * Authors: Ravi Ramachandra <r.ramachandra@ti.com>,
  *          Lajos Molnar <molnar@ti.com>
+ *          Andy Gross <andy.gross@ti.com>
  *
- * Copyright (C) 2009-2010 Texas Instruments, Inc.
+ * Copyright (C) 2012 Texas Instruments, Inc.
  *
  * This package is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -17,687 +18,244 @@
  * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
  *
  */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/bitmap.h>
 #include <linux/slab.h>
-#include <linux/spinlock.h>
+#include "tcm.h"
 
-#include "tcm-sita.h"
-
-#define ALIGN_DOWN(value, align) ((value) & ~((align) - 1))
-
-/* Individual selection criteria for different scan areas */
-static s32 CR_L2R_T2B = CR_BIAS_HORIZONTAL;
-static s32 CR_R2L_T2B = CR_DIAGONAL_BALANCE;
-
-/*********************************************
- *	TCM API - Sita Implementation
- *********************************************/
-static s32 sita_reserve_2d(struct tcm *tcm, u16 h, u16 w, u8 align,
-			   struct tcm_area *area);
-static s32 sita_reserve_1d(struct tcm *tcm, u32 slots, struct tcm_area *area);
-static s32 sita_free(struct tcm *tcm, struct tcm_area *area);
-static void sita_deinit(struct tcm *tcm);
-
-/*********************************************
- *	Main Scanner functions
- *********************************************/
-static s32 scan_areas_and_find_fit(struct tcm *tcm, u16 w, u16 h, u16 align,
-				   struct tcm_area *area);
-
-static s32 scan_l2r_t2b(struct tcm *tcm, u16 w, u16 h, u16 align,
-			struct tcm_area *field, struct tcm_area *area);
-
-static s32 scan_r2l_t2b(struct tcm *tcm, u16 w, u16 h, u16 align,
-			struct tcm_area *field, struct tcm_area *area);
-
-static s32 scan_r2l_b2t_one_dim(struct tcm *tcm, u32 num_slots,
-			struct tcm_area *field, struct tcm_area *area);
-
-/*********************************************
- *	Support Infrastructure Methods
- *********************************************/
-static s32 is_area_free(struct tcm_area ***map, u16 x0, u16 y0, u16 w, u16 h);
-
-static s32 update_candidate(struct tcm *tcm, u16 x0, u16 y0, u16 w, u16 h,
-			    struct tcm_area *field, s32 criteria,
-			    struct score *best);
-
-static void get_nearness_factor(struct tcm_area *field,
-				struct tcm_area *candidate,
-				struct nearness_factor *nf);
-
-static void get_neighbor_stats(struct tcm *tcm, struct tcm_area *area,
-			       struct neighbor_stats *stat);
-
-static void fill_area(struct tcm *tcm,
-				struct tcm_area *area, struct tcm_area *parent);
-
-
-/*********************************************/
-
-/*********************************************
- *	Utility Methods
- *********************************************/
-struct tcm *sita_init(u16 width, u16 height, struct tcm_pt *attr)
-{
-	struct tcm *tcm;
-	struct sita_pvt *pvt;
-	struct tcm_area area = {0};
-	s32 i;
-
-	if (width == 0 || height == 0)
-		return NULL;
-
-	tcm = kmalloc(sizeof(*tcm), GFP_KERNEL);
-	pvt = kmalloc(sizeof(*pvt), GFP_KERNEL);
-	if (!tcm || !pvt)
-		goto error;
-
-	memset(tcm, 0, sizeof(*tcm));
-	memset(pvt, 0, sizeof(*pvt));
-
-	/* Updating the pointers to SiTA implementation APIs */
-	tcm->height = height;
-	tcm->width = width;
-	tcm->reserve_2d = sita_reserve_2d;
-	tcm->reserve_1d = sita_reserve_1d;
-	tcm->free = sita_free;
-	tcm->deinit = sita_deinit;
-	tcm->pvt = (void *)pvt;
-
-	spin_lock_init(&(pvt->lock));
-
-	/* Creating tam map */
-	pvt->map = kmalloc(sizeof(*pvt->map) * tcm->width, GFP_KERNEL);
-	if (!pvt->map)
-		goto error;
-
-	for (i = 0; i < tcm->width; i++) {
-		pvt->map[i] =
-			kmalloc(sizeof(**pvt->map) * tcm->height,
-								GFP_KERNEL);
-		if (pvt->map[i] == NULL) {
-			while (i--)
-				kfree(pvt->map[i]);
-			kfree(pvt->map);
-			goto error;
-		}
-	}
-
-	if (attr && attr->x <= tcm->width && attr->y <= tcm->height) {
-		pvt->div_pt.x = attr->x;
-		pvt->div_pt.y = attr->y;
-
-	} else {
-		/* Defaulting to 3:1 ratio on width for 2D area split */
-		/* Defaulting to 3:1 ratio on height for 2D and 1D split */
-		pvt->div_pt.x = (tcm->width * 3) / 4;
-		pvt->div_pt.y = (tcm->height * 3) / 4;
-	}
-
-	spin_lock(&(pvt->lock));
-	assign(&area, 0, 0, width - 1, height - 1);
-	fill_area(tcm, &area, NULL);
-	spin_unlock(&(pvt->lock));
-	return tcm;
-
-error:
-	kfree(tcm);
-	kfree(pvt);
-	return NULL;
-}
-
-static void sita_deinit(struct tcm *tcm)
-{
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-	struct tcm_area area = {0};
-	s32 i;
-
-	area.p1.x = tcm->width - 1;
-	area.p1.y = tcm->height - 1;
-
-	spin_lock(&(pvt->lock));
-	fill_area(tcm, &area, NULL);
-	spin_unlock(&(pvt->lock));
-
-	for (i = 0; i < tcm->height; i++)
-		kfree(pvt->map[i]);
-	kfree(pvt->map);
-	kfree(pvt);
-}
-
-/**
- * Reserve a 1D area in the container
- *
- * @param num_slots	size of 1D area
- * @param area		pointer to the area that will be populated with the
- *			reserved area
- *
- * @return 0 on success, non-0 error value on failure.
+static unsigned long mask[8];
+/*
+ * pos		position in bitmap
+ * w		width in slots
+ * h		height in slots
+ * map		ptr to bitmap
+ * stride		slots in a row
  */
-static s32 sita_reserve_1d(struct tcm *tcm, u32 num_slots,
-			   struct tcm_area *area)
+static void free_slots(unsigned long pos, uint16_t w, uint16_t h,
+		unsigned long *map, uint16_t stride)
 {
-	s32 ret;
-	struct tcm_area field = {0};
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-
-	spin_lock(&(pvt->lock));
-
-	/* Scanning entire container */
-	assign(&field, tcm->width - 1, tcm->height - 1, 0, 0);
+	int i;
 
-	ret = scan_r2l_b2t_one_dim(tcm, num_slots, &field, area);
-	if (!ret)
-		/* update map */
-		fill_area(tcm, area, area);
-
-	spin_unlock(&(pvt->lock));
-	return ret;
+	for (i = 0; i < h; i++, pos += stride)
+		bitmap_clear(map, pos, w);
 }
 
-/**
- * Reserve a 2D area in the container
- *
- * @param w	width
- * @param h	height
- * @param area	pointer to the area that will be populated with the reserved
- *		area
- *
- * @return 0 on success, non-0 error value on failure.
+/*
+ * w		width in slots
+ * pos		ptr to position
+ * map		ptr to bitmap
+ * num_bits	number of bits in bitmap
  */
-static s32 sita_reserve_2d(struct tcm *tcm, u16 h, u16 w, u8 align,
-			   struct tcm_area *area)
+static int r2l_b2t_1d(uint16_t w, unsigned long *pos, unsigned long *map,
+		size_t num_bits)
 {
-	s32 ret;
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
+	unsigned long search_count = 0;
+	unsigned long bit;
+	bool area_found = false;
 
-	/* not supporting more than 64 as alignment */
-	if (align > 64)
-		return -EINVAL;
+	*pos = num_bits - w;
 
-	/* we prefer 1, 32 and 64 as alignment */
-	align = align <= 1 ? 1 : align <= 32 ? 32 : 64;
+	while (search_count < num_bits) {
+		bit = find_next_bit(map, num_bits, *pos);
 
-	spin_lock(&(pvt->lock));
-	ret = scan_areas_and_find_fit(tcm, w, h, align, area);
-	if (!ret)
-		/* update map */
-		fill_area(tcm, area, area);
+		if (bit - *pos >= w) {
+			/* found a long enough free area */
+			bitmap_set(map, *pos, w);
+			area_found = true;
+			break;
+		}
 
-	spin_unlock(&(pvt->lock));
-	return ret;
+		search_count = num_bits - bit + w;
+		*pos = bit - w;
+	}
+
+	return (area_found) ? 0 : -ENOMEM;
 }
 
-/**
- * Unreserve a previously allocated 2D or 1D area
- * @param area	area to be freed
- * @return 0 - success
+/*
+ * w = width in slots
+ * h = height in slots
+ * a = align in slots	(mask, 2^n-1, 0 is unaligned)
+ * offset = offset in bytes from 4KiB
+ * pos = position in bitmap for buffer
+ * map = bitmap ptr
+ * num_bits = size of bitmap
+ * stride = bits in one row of container
  */
-static s32 sita_free(struct tcm *tcm, struct tcm_area *area)
+static int l2r_t2b(uint16_t w, uint16_t h, uint16_t a, int16_t offset,
+		unsigned long *pos, unsigned long slot_bytes,
+		unsigned long *map, size_t num_bits, size_t slot_stride)
 {
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-
-	spin_lock(&(pvt->lock));
+	int i;
+	unsigned long index;
+	bool area_free;
+	unsigned long slots_per_band = PAGE_SIZE / slot_bytes;
+	unsigned long bit_offset = (offset > 0) ? offset / slot_bytes : 0;
+	unsigned long curr_bit = bit_offset;
+
+	/* reset alignment to 1 if we are matching a specific offset */
+	/* adjust alignment - 1 to get to the format expected in bitmaps */
+	a = (offset > 0) ? 0 : a - 1;
+
+	/* FIXME Return error if slots_per_band > stride */
+
+	while (curr_bit < num_bits) {
+		*pos = bitmap_find_next_zero_area(map, num_bits, curr_bit, w,
+				a);
+
+		/* skip forward if we are not at right offset */
+		if (bit_offset > 0 && (*pos % slots_per_band != bit_offset)) {
+			curr_bit = ALIGN(*pos, slots_per_band) + bit_offset;
+			continue;
+		}
 
-	/* check that this is in fact an existing area */
-	WARN_ON(pvt->map[area->p0.x][area->p0.y] != area ||
-		pvt->map[area->p1.x][area->p1.y] != area);
+		/* skip forward to next row if we overlap end of row */
+		if ((*pos % slot_stride) + w > slot_stride) {
+			curr_bit = ALIGN(*pos, slot_stride) + bit_offset;
+			continue;
+		}
 
-	/* Clear the contents of the associated tiles in the map */
-	fill_area(tcm, area, NULL);
+		/* TODO: Handle overlapping 4K boundaries */
 
-	spin_unlock(&(pvt->lock));
+		/* break out of look if we will go past end of container */
+		if ((*pos + slot_stride * h) > num_bits)
+			break;
 
-	return 0;
-}
+		/* generate mask that represents out matching pattern */
+		bitmap_clear(mask, 0, slot_stride);
+		bitmap_set(mask, (*pos % BITS_PER_LONG), w);
 
-/**
- * Note: In general the cordinates in the scan field area relevant to the can
- * sweep directions. The scan origin (e.g. top-left corner) will always be
- * the p0 member of the field.  Therfore, for a scan from top-left p0.x <= p1.x
- * and p0.y <= p1.y; whereas, for a scan from bottom-right p1.x <= p0.x and p1.y
- * <= p0.y
- */
+		/* assume the area is free until we find an overlap */
+		area_free = true;
 
-/**
- * Raster scan horizontally right to left from top to bottom to find a place for
- * a 2D area of given size inside a scan field.
- *
- * @param w	width of desired area
- * @param h	height of desired area
- * @param align	desired area alignment
- * @param area	pointer to the area that will be set to the best position
- * @param field	area to scan (inclusive)
- *
- * @return 0 on success, non-0 error value on failure.
- */
-static s32 scan_r2l_t2b(struct tcm *tcm, u16 w, u16 h, u16 align,
-			struct tcm_area *field, struct tcm_area *area)
-{
-	s32 x, y;
-	s16 start_x, end_x, start_y, end_y, found_x = -1;
-	struct tcm_area ***map = ((struct sita_pvt *)tcm->pvt)->map;
-	struct score best = {{0}, {0}, {0}, 0};
-
-	start_x = field->p0.x;
-	end_x = field->p1.x;
-	start_y = field->p0.y;
-	end_y = field->p1.y;
-
-	/* check scan area co-ordinates */
-	if (field->p0.x < field->p1.x ||
-	    field->p1.y < field->p0.y)
-		return -EINVAL;
-
-	/* check if allocation would fit in scan area */
-	if (w > LEN(start_x, end_x) || h > LEN(end_y, start_y))
-		return -ENOSPC;
-
-	/* adjust start_x and end_y, as allocation would not fit beyond */
-	start_x = ALIGN_DOWN(start_x - w + 1, align); /* - 1 to be inclusive */
-	end_y = end_y - h + 1;
-
-	/* check if allocation would still fit in scan area */
-	if (start_x < end_x)
-		return -ENOSPC;
-
-	/* scan field top-to-bottom, right-to-left */
-	for (y = start_y; y <= end_y; y++) {
-		for (x = start_x; x >= end_x; x -= align) {
-			if (is_area_free(map, x, y, w, h)) {
-				found_x = x;
-
-				/* update best candidate */
-				if (update_candidate(tcm, x, y, w, h, field,
-							CR_R2L_T2B, &best))
-					goto done;
-
-				/* change upper x bound */
-				end_x = x + 1;
+		/* check subsequent rows to see if complete area is free */
+		for (i = 1; i < h; i++) {
+			index = *pos / BITS_PER_LONG + i * 8;
+			if (bitmap_intersects(&map[index], mask,
+				(*pos % BITS_PER_LONG) + w)) {
+				area_free = false;
 				break;
-			} else if (map[x][y] && map[x][y]->is2d) {
-				/* step over 2D areas */
-				x = ALIGN(map[x][y]->p0.x - w + 1, align);
 			}
 		}
 
-		/* break if you find a free area shouldering the scan field */
-		if (found_x == start_x)
+		if (area_free)
 			break;
-	}
-
-	if (!best.a.tcm)
-		return -ENOSPC;
-done:
-	assign(area, best.a.p0.x, best.a.p0.y, best.a.p1.x, best.a.p1.y);
-	return 0;
-}
-
-/**
- * Raster scan horizontally left to right from top to bottom to find a place for
- * a 2D area of given size inside a scan field.
- *
- * @param w	width of desired area
- * @param h	height of desired area
- * @param align	desired area alignment
- * @param area	pointer to the area that will be set to the best position
- * @param field	area to scan (inclusive)
- *
- * @return 0 on success, non-0 error value on failure.
- */
-static s32 scan_l2r_t2b(struct tcm *tcm, u16 w, u16 h, u16 align,
-			struct tcm_area *field, struct tcm_area *area)
-{
-	s32 x, y;
-	s16 start_x, end_x, start_y, end_y, found_x = -1;
-	struct tcm_area ***map = ((struct sita_pvt *)tcm->pvt)->map;
-	struct score best = {{0}, {0}, {0}, 0};
-
-	start_x = field->p0.x;
-	end_x = field->p1.x;
-	start_y = field->p0.y;
-	end_y = field->p1.y;
-
-	/* check scan area co-ordinates */
-	if (field->p1.x < field->p0.x ||
-	    field->p1.y < field->p0.y)
-		return -EINVAL;
-
-	/* check if allocation would fit in scan area */
-	if (w > LEN(end_x, start_x) || h > LEN(end_y, start_y))
-		return -ENOSPC;
-
-	start_x = ALIGN(start_x, align);
-
-	/* check if allocation would still fit in scan area */
-	if (w > LEN(end_x, start_x))
-		return -ENOSPC;
-
-	/* adjust end_x and end_y, as allocation would not fit beyond */
-	end_x = end_x - w + 1; /* + 1 to be inclusive */
-	end_y = end_y - h + 1;
-
-	/* scan field top-to-bottom, left-to-right */
-	for (y = start_y; y <= end_y; y++) {
-		for (x = start_x; x <= end_x; x += align) {
-			if (is_area_free(map, x, y, w, h)) {
-				found_x = x;
-
-				/* update best candidate */
-				if (update_candidate(tcm, x, y, w, h, field,
-							CR_L2R_T2B, &best))
-					goto done;
-				/* change upper x bound */
-				end_x = x - 1;
 
-				break;
-			} else if (map[x][y] && map[x][y]->is2d) {
-				/* step over 2D areas */
-				x = ALIGN_DOWN(map[x][y]->p1.x, align);
-			}
-		}
+		/* go forward past this match */
+		if (bit_offset > 0)
+			curr_bit = ALIGN(*pos, slots_per_band) + bit_offset;
+		else
+			curr_bit = *pos + a + 1;
+	}
 
-		/* break if you find a free area shouldering the scan field */
-		if (found_x == start_x)
-			break;
+	if (area_free) {
+		/* set area as in-use. iterate over rows */
+		for (i = 0, index = *pos; i < h; i++, index += slot_stride)
+			bitmap_set(map, index, w);
 	}
 
-	if (!best.a.tcm)
-		return -ENOSPC;
-done:
-	assign(area, best.a.p0.x, best.a.p0.y, best.a.p1.x, best.a.p1.y);
-	return 0;
+	return (area_free) ? 0 : -ENOMEM;
 }
 
-/**
- * Raster scan horizontally right to left from bottom to top to find a place
- * for a 1D area of given size inside a scan field.
- *
- * @param num_slots	size of desired area
- * @param align		desired area alignment
- * @param area		pointer to the area that will be set to the best
- *			position
- * @param field		area to scan (inclusive)
- *
- * @return 0 on success, non-0 error value on failure.
- */
-static s32 scan_r2l_b2t_one_dim(struct tcm *tcm, u32 num_slots,
-				struct tcm_area *field, struct tcm_area *area)
+static s32 sita_reserve_1d(struct tcm *tcm, u32 num_slots,
+			   struct tcm_area *area)
 {
-	s32 found = 0;
-	s16 x, y;
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-	struct tcm_area *p;
-
-	/* check scan area co-ordinates */
-	if (field->p0.y < field->p1.y)
-		return -EINVAL;
-
-	/**
-	 * Currently we only support full width 1D scan field, which makes sense
-	 * since 1D slot-ordering spans the full container width.
-	 */
-	if (tcm->width != field->p0.x - field->p1.x + 1)
-		return -EINVAL;
-
-	/* check if allocation would fit in scan area */
-	if (num_slots > tcm->width * LEN(field->p0.y, field->p1.y))
-		return -ENOSPC;
-
-	x = field->p0.x;
-	y = field->p0.y;
-
-	/* find num_slots consecutive free slots to the left */
-	while (found < num_slots) {
-		if (y < 0)
-			return -ENOSPC;
-
-		/* remember bottom-right corner */
-		if (found == 0) {
-			area->p1.x = x;
-			area->p1.y = y;
-		}
-
-		/* skip busy regions */
-		p = pvt->map[x][y];
-		if (p) {
-			/* move to left of 2D areas, top left of 1D */
-			x = p->p0.x;
-			if (!p->is2d)
-				y = p->p0.y;
-
-			/* start over */
-			found = 0;
-		} else {
-			/* count consecutive free slots */
-			found++;
-			if (found == num_slots)
-				break;
-		}
-
-		/* move to the left */
-		if (x == 0)
-			y--;
-		x = (x ? : tcm->width) - 1;
-
+	unsigned long pos;
+	int ret;
+
+	spin_lock(&(tcm->lock));
+	ret = r2l_b2t_1d(num_slots, &pos, tcm->bitmap, tcm->map_size);
+	if (!ret) {
+		area->p0.x = pos % tcm->width;
+		area->p0.y = pos / tcm->width;
+		area->p1.x = (pos + num_slots - 1) % tcm->width;
+		area->p1.y = (pos + num_slots - 1) / tcm->width;
 	}
+	spin_unlock(&(tcm->lock));
 
-	/* set top-left corner */
-	area->p0.x = x;
-	area->p0.y = y;
-	return 0;
+	return ret;
 }
 
-/**
- * Find a place for a 2D area of given size inside a scan field based on its
- * alignment needs.
- *
- * @param w	width of desired area
- * @param h	height of desired area
- * @param align	desired area alignment
- * @param area	pointer to the area that will be set to the best position
- *
- * @return 0 on success, non-0 error value on failure.
- */
-static s32 scan_areas_and_find_fit(struct tcm *tcm, u16 w, u16 h, u16 align,
-				   struct tcm_area *area)
+static s32 sita_reserve_2d(struct tcm *tcm, u16 h, u16 w, u16 align,
+				int16_t offset, uint16_t slot_bytes,
+				struct tcm_area *area)
 {
-	s32 ret = 0;
-	struct tcm_area field = {0};
-	u16 boundary_x, boundary_y;
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-
-	if (align > 1) {
-		/* prefer top-left corner */
-		boundary_x = pvt->div_pt.x - 1;
-		boundary_y = pvt->div_pt.y - 1;
-
-		/* expand width and height if needed */
-		if (w > pvt->div_pt.x)
-			boundary_x = tcm->width - 1;
-		if (h > pvt->div_pt.y)
-			boundary_y = tcm->height - 1;
-
-		assign(&field, 0, 0, boundary_x, boundary_y);
-		ret = scan_l2r_t2b(tcm, w, h, align, &field, area);
-
-		/* scan whole container if failed, but do not scan 2x */
-		if (ret != 0 && (boundary_x != tcm->width - 1 ||
-				 boundary_y != tcm->height - 1)) {
-			/* scan the entire container if nothing found */
-			assign(&field, 0, 0, tcm->width - 1, tcm->height - 1);
-			ret = scan_l2r_t2b(tcm, w, h, align, &field, area);
-		}
-	} else if (align == 1) {
-		/* prefer top-right corner */
-		boundary_x = pvt->div_pt.x;
-		boundary_y = pvt->div_pt.y - 1;
-
-		/* expand width and height if needed */
-		if (w > (tcm->width - pvt->div_pt.x))
-			boundary_x = 0;
-		if (h > pvt->div_pt.y)
-			boundary_y = tcm->height - 1;
-
-		assign(&field, tcm->width - 1, 0, boundary_x, boundary_y);
-		ret = scan_r2l_t2b(tcm, w, h, align, &field, area);
-
-		/* scan whole container if failed, but do not scan 2x */
-		if (ret != 0 && (boundary_x != 0 ||
-				 boundary_y != tcm->height - 1)) {
-			/* scan the entire container if nothing found */
-			assign(&field, tcm->width - 1, 0, 0, tcm->height - 1);
-			ret = scan_r2l_t2b(tcm, w, h, align, &field,
-					   area);
-		}
+	unsigned long pos;
+	int ret;
+
+	spin_lock(&(tcm->lock));
+	ret = l2r_t2b(w, h, align, offset, &pos, slot_bytes, tcm->bitmap,
+			tcm->map_size, tcm->width);
+
+	if (!ret) {
+		area->p0.x = pos % tcm->width;
+		area->p0.y = pos / tcm->width;
+		area->p1.x = area->p0.x + w - 1;
+		area->p1.y = area->p0.y + h - 1;
 	}
+	spin_unlock(&(tcm->lock));
 
 	return ret;
 }
 
-/* check if an entire area is free */
-static s32 is_area_free(struct tcm_area ***map, u16 x0, u16 y0, u16 w, u16 h)
+static void sita_deinit(struct tcm *tcm)
 {
-	u16 x = 0, y = 0;
-	for (y = y0; y < y0 + h; y++) {
-		for (x = x0; x < x0 + w; x++) {
-			if (map[x][y])
-				return false;
-		}
-	}
-	return true;
+	kfree(tcm);
 }
 
-/* fills an area with a parent tcm_area */
-static void fill_area(struct tcm *tcm, struct tcm_area *area,
-			struct tcm_area *parent)
+static s32 sita_free(struct tcm *tcm, struct tcm_area *area)
 {
-	s32 x, y;
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-	struct tcm_area a, a_;
-
-	/* set area's tcm; otherwise, enumerator considers it invalid */
-	area->tcm = tcm;
-
-	tcm_for_each_slice(a, *area, a_) {
-		for (x = a.p0.x; x <= a.p1.x; ++x)
-			for (y = a.p0.y; y <= a.p1.y; ++y)
-				pvt->map[x][y] = parent;
+	unsigned long pos;
+	uint16_t w, h;
 
+	pos = area->p0.x + area->p0.y * tcm->width;
+	if (area->is2d) {
+		w = area->p1.x - area->p0.x + 1;
+		h = area->p1.y - area->p0.y + 1;
+	} else {
+		w = area->p1.x + area->p1.y * tcm->width - pos + 1;
+		h = 1;
 	}
+
+	spin_lock(&(tcm->lock));
+	free_slots(pos, w, h, tcm->bitmap, tcm->width);
+	spin_unlock(&(tcm->lock));
+	return 0;
 }
 
-/**
- * Compares a candidate area to the current best area, and if it is a better
- * fit, it updates the best to this one.
- *
- * @param x0, y0, w, h		top, left, width, height of candidate area
- * @param field			scan field
- * @param criteria		scan criteria
- * @param best			best candidate and its scores
- *
- * @return 1 (true) if the candidate area is known to be the final best, so no
- * more searching should be performed
- */
-static s32 update_candidate(struct tcm *tcm, u16 x0, u16 y0, u16 w, u16 h,
-			    struct tcm_area *field, s32 criteria,
-			    struct score *best)
+struct tcm *sita_init(u16 width, u16 height)
 {
-	struct score me;	/* score for area */
-
-	/*
-	 * NOTE: For horizontal bias we always give the first found, because our
-	 * scan is horizontal-raster-based and the first candidate will always
-	 * have the horizontal bias.
-	 */
-	bool first = criteria & CR_BIAS_HORIZONTAL;
-
-	assign(&me.a, x0, y0, x0 + w - 1, y0 + h - 1);
-
-	/* calculate score for current candidate */
-	if (!first) {
-		get_neighbor_stats(tcm, &me.a, &me.n);
-		me.neighs = me.n.edge + me.n.busy;
-		get_nearness_factor(field, &me.a, &me.f);
-	}
-
-	/* the 1st candidate is always the best */
-	if (!best->a.tcm)
-		goto better;
+	struct tcm *tcm;
+	size_t map_size = BITS_TO_LONGS(width*height) * sizeof(unsigned long);
 
-	BUG_ON(first);
+	if (width == 0 || height == 0)
+		return NULL;
 
-	/* diagonal balance check */
-	if ((criteria & CR_DIAGONAL_BALANCE) &&
-		best->neighs <= me.neighs &&
-		(best->neighs < me.neighs ||
-		 /* this implies that neighs and occupied match */
-		 best->n.busy < me.n.busy ||
-		 (best->n.busy == me.n.busy &&
-		  /* check the nearness factor */
-		  best->f.x + best->f.y > me.f.x + me.f.y)))
-		goto better;
+	tcm = kzalloc(sizeof(*tcm) + map_size, GFP_KERNEL);
+	if (!tcm)
+		goto error;
 
-	/* not better, keep going */
-	return 0;
+	/* Updating the pointers to SiTA implementation APIs */
+	tcm->height = height;
+	tcm->width = width;
+	tcm->reserve_2d = sita_reserve_2d;
+	tcm->reserve_1d = sita_reserve_1d;
+	tcm->free = sita_free;
+	tcm->deinit = sita_deinit;
 
-better:
-	/* save current area as best */
-	memcpy(best, &me, sizeof(me));
-	best->a.tcm = tcm;
-	return first;
-}
+	spin_lock_init(&tcm->lock);
+	tcm->bitmap = (unsigned long *)(tcm + 1);
+	bitmap_clear(tcm->bitmap, 0, width*height);
 
-/**
- * Calculate the nearness factor of an area in a search field.  The nearness
- * factor is smaller if the area is closer to the search origin.
- */
-static void get_nearness_factor(struct tcm_area *field, struct tcm_area *area,
-				struct nearness_factor *nf)
-{
-	/**
-	 * Using signed math as field coordinates may be reversed if
-	 * search direction is right-to-left or bottom-to-top.
-	 */
-	nf->x = (s32)(area->p0.x - field->p0.x) * 1000 /
-		(field->p1.x - field->p0.x);
-	nf->y = (s32)(area->p0.y - field->p0.y) * 1000 /
-		(field->p1.y - field->p0.y);
-}
+	tcm->map_size = width*height;
 
-/* get neighbor statistics */
-static void get_neighbor_stats(struct tcm *tcm, struct tcm_area *area,
-			 struct neighbor_stats *stat)
-{
-	s16 x = 0, y = 0;
-	struct sita_pvt *pvt = (struct sita_pvt *)tcm->pvt;
-
-	/* Clearing any exisiting values */
-	memset(stat, 0, sizeof(*stat));
-
-	/* process top & bottom edges */
-	for (x = area->p0.x; x <= area->p1.x; x++) {
-		if (area->p0.y == 0)
-			stat->edge++;
-		else if (pvt->map[x][area->p0.y - 1])
-			stat->busy++;
-
-		if (area->p1.y == tcm->height - 1)
-			stat->edge++;
-		else if (pvt->map[x][area->p1.y + 1])
-			stat->busy++;
-	}
+	return tcm;
 
-	/* process left & right edges */
-	for (y = area->p0.y; y <= area->p1.y; ++y) {
-		if (area->p0.x == 0)
-			stat->edge++;
-		else if (pvt->map[area->p0.x - 1][y])
-			stat->busy++;
-
-		if (area->p1.x == tcm->width - 1)
-			stat->edge++;
-		else if (pvt->map[area->p1.x + 1][y])
-			stat->busy++;
-	}
+error:
+	kfree(tcm);
+	return NULL;
 }
diff --git a/drivers/gpu/drm/omapdrm/tcm.h b/drivers/gpu/drm/omapdrm/tcm.h
index a8d5ce47686f..ef7df7d6fc84 100644
--- a/drivers/gpu/drm/omapdrm/tcm.h
+++ b/drivers/gpu/drm/omapdrm/tcm.h
@@ -61,18 +61,17 @@ struct tcm {
 
 	unsigned int y_offset;	/* offset to use for y coordinates */
 
-	/* 'pvt' structure shall contain any tcm details (attr) along with
-	linked list of allocated areas and mutex for mutually exclusive access
-	to the list.  It may also contain copies of width and height to notice
-	any changes to the publicly available width and height fields. */
-	void *pvt;
+	spinlock_t lock;
+	unsigned long *bitmap;
+	size_t map_size;
 
 	/* function table */
-	s32 (*reserve_2d)(struct tcm *tcm, u16 height, u16 width, u8 align,
+	s32 (*reserve_2d)(struct tcm *tcm, u16 height, u16 width, u16 align,
+			  int16_t offset, uint16_t slot_bytes,
 			  struct tcm_area *area);
 	s32 (*reserve_1d)(struct tcm *tcm, u32 slots, struct tcm_area *area);
-	s32 (*free)      (struct tcm *tcm, struct tcm_area *area);
-	void (*deinit)   (struct tcm *tcm);
+	s32 (*free)(struct tcm *tcm, struct tcm_area *area);
+	void (*deinit)(struct tcm *tcm);
 };
 
 /*=============================================================================
@@ -91,7 +90,7 @@ struct tcm {
  *
  */
 
-struct tcm *sita_init(u16 width, u16 height, struct tcm_pt *attr);
+struct tcm *sita_init(u16 width, u16 height);
 
 
 /**
@@ -120,6 +119,9 @@ static inline void tcm_deinit(struct tcm *tcm)
  *			all values may be supported by the container manager,
  *			but it must support 0 (1), 32 and 64.
  *			0 value is equivalent to 1.
+ * @param offset	Offset requirement, in bytes.  This is the offset
+ *			from a 4KiB aligned virtual address.
+ * @param slot_bytes	Width of slot in bytes
  * @param area		Pointer to where the reserved area should be stored.
  *
  * @return 0 on success.  Non-0 error code on failure.  Also,
@@ -129,7 +131,8 @@ static inline void tcm_deinit(struct tcm *tcm)
  *	    allocation.
  */
 static inline s32 tcm_reserve_2d(struct tcm *tcm, u16 width, u16 height,
-				 u16 align, struct tcm_area *area)
+				u16 align, int16_t offset, uint16_t slot_bytes,
+				struct tcm_area *area)
 {
 	/* perform rudimentary error checking */
 	s32 res = tcm  == NULL ? -ENODEV :
@@ -140,7 +143,8 @@ static inline s32 tcm_reserve_2d(struct tcm *tcm, u16 width, u16 height,
 
 	if (!res) {
 		area->is2d = true;
-		res = tcm->reserve_2d(tcm, height, width, align, area);
+		res = tcm->reserve_2d(tcm, height, width, align, offset,
+					slot_bytes, area);
 		area->tcm = res ? NULL : tcm;
 	}
 
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 7d4704b1292b..1500ab99f548 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -31,6 +31,16 @@ config DRM_PANEL_LG_LG4573
 	  Say Y here if you want to enable support for LG4573 RGB panel.
 	  To compile this driver as a module, choose M here.
 
+config DRM_PANEL_PANASONIC_VVX10F034N00
+	tristate "Panasonic VVX10F034N00 1920x1200 video mode panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Panasonic VVX10F034N00
+	  WUXGA (1920x1200) Novatek NT1397-based DSI panel as found in some
+	  Xperia Z2 tablets
+
 config DRM_PANEL_SAMSUNG_S6E8AA0
 	tristate "Samsung S6E8AA0 DSI video mode panel"
 	depends on OF
@@ -51,4 +61,13 @@ config DRM_PANEL_SHARP_LQ101R1SX01
 	  To compile this driver as a module, choose M here: the module
 	  will be called panel-sharp-lq101r1sx01.
 
+config DRM_PANEL_SHARP_LS043T1LE01
+	tristate "Sharp LS043T1LE01 qHD video mode panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Sharp LS043T1LE01 qHD
+	  (540x960) DSI panel as found on the Qualcomm APQ8074 Dragonboard
+
 endmenu
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index d0f016dd7ddb..f277eed933d6 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -1,5 +1,7 @@
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
+obj-$(CONFIG_DRM_PANEL_PANASONIC_VVX10F034N00) += panel-panasonic-vvx10f034n00.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_LD9040) += panel-samsung-ld9040.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0) += panel-samsung-s6e8aa0.o
 obj-$(CONFIG_DRM_PANEL_SHARP_LQ101R1SX01) += panel-sharp-lq101r1sx01.o
+obj-$(CONFIG_DRM_PANEL_SHARP_LS043T1LE01) += panel-sharp-ls043t1le01.o
diff --git a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
new file mode 100644
index 000000000000..7f915f706fa6
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
@@ -0,0 +1,334 @@
+/*
+ * Copyright (C) 2015 Red Hat
+ * Copyright (C) 2015 Sony Mobile Communications Inc.
+ * Author: Werner Johansson <werner.johansson@sonymobile.com>
+ *
+ * Based on AUO panel driver by Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/backlight.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <video/mipi_display.h>
+
+/*
+ * When power is turned off to this panel a minimum off time of 500ms has to be
+ * observed before powering back on as there's no external reset pin. Keep
+ * track of earliest wakeup time and delay subsequent prepare call accordingly
+ */
+#define MIN_POFF_MS (500)
+
+struct wuxga_nt_panel {
+	struct drm_panel base;
+	struct mipi_dsi_device *dsi;
+
+	struct backlight_device *backlight;
+	struct regulator *supply;
+
+	bool prepared;
+	bool enabled;
+
+	ktime_t earliest_wake;
+
+	const struct drm_display_mode *mode;
+};
+
+static inline struct wuxga_nt_panel *to_wuxga_nt_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct wuxga_nt_panel, base);
+}
+
+static int wuxga_nt_panel_on(struct wuxga_nt_panel *wuxga_nt)
+{
+	struct mipi_dsi_device *dsi = wuxga_nt->dsi;
+	int ret;
+
+	ret = mipi_dsi_turn_on_peripheral(dsi);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int wuxga_nt_panel_disable(struct drm_panel *panel)
+{
+	struct wuxga_nt_panel *wuxga_nt = to_wuxga_nt_panel(panel);
+
+	if (!wuxga_nt->enabled)
+		return 0;
+
+	mipi_dsi_shutdown_peripheral(wuxga_nt->dsi);
+
+	if (wuxga_nt->backlight) {
+		wuxga_nt->backlight->props.power = FB_BLANK_POWERDOWN;
+		wuxga_nt->backlight->props.state |= BL_CORE_FBBLANK;
+		backlight_update_status(wuxga_nt->backlight);
+	}
+
+	wuxga_nt->enabled = false;
+
+	return 0;
+}
+
+static int wuxga_nt_panel_unprepare(struct drm_panel *panel)
+{
+	struct wuxga_nt_panel *wuxga_nt = to_wuxga_nt_panel(panel);
+
+	if (!wuxga_nt->prepared)
+		return 0;
+
+	regulator_disable(wuxga_nt->supply);
+	wuxga_nt->earliest_wake = ktime_add_ms(ktime_get_real(), MIN_POFF_MS);
+	wuxga_nt->prepared = false;
+
+	return 0;
+}
+
+static int wuxga_nt_panel_prepare(struct drm_panel *panel)
+{
+	struct wuxga_nt_panel *wuxga_nt = to_wuxga_nt_panel(panel);
+	int ret;
+	s64 enablewait;
+
+	if (wuxga_nt->prepared)
+		return 0;
+
+	/*
+	 * If the user re-enabled the panel before the required off-time then
+	 * we need to wait the remaining period before re-enabling regulator
+	 */
+	enablewait = ktime_ms_delta(wuxga_nt->earliest_wake, ktime_get_real());
+
+	/* Sanity check, this should never happen */
+	if (enablewait > MIN_POFF_MS)
+		enablewait = MIN_POFF_MS;
+
+	if (enablewait > 0)
+		msleep(enablewait);
+
+	ret = regulator_enable(wuxga_nt->supply);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * A minimum delay of 250ms is required after power-up until commands
+	 * can be sent
+	 */
+	msleep(250);
+
+	ret = wuxga_nt_panel_on(wuxga_nt);
+	if (ret < 0) {
+		dev_err(panel->dev, "failed to set panel on: %d\n", ret);
+		goto poweroff;
+	}
+
+	wuxga_nt->prepared = true;
+
+	return 0;
+
+poweroff:
+	regulator_disable(wuxga_nt->supply);
+
+	return ret;
+}
+
+static int wuxga_nt_panel_enable(struct drm_panel *panel)
+{
+	struct wuxga_nt_panel *wuxga_nt = to_wuxga_nt_panel(panel);
+
+	if (wuxga_nt->enabled)
+		return 0;
+
+	if (wuxga_nt->backlight) {
+		wuxga_nt->backlight->props.power = FB_BLANK_UNBLANK;
+		wuxga_nt->backlight->props.state &= ~BL_CORE_FBBLANK;
+		backlight_update_status(wuxga_nt->backlight);
+	}
+
+	wuxga_nt->enabled = true;
+
+	return 0;
+}
+
+static const struct drm_display_mode default_mode = {
+	.clock = 164402,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 152,
+	.hsync_end = 1920 + 152 + 52,
+	.htotal = 1920 + 152 + 52 + 20,
+	.vdisplay = 1200,
+	.vsync_start = 1200 + 24,
+	.vsync_end = 1200 + 24 + 6,
+	.vtotal = 1200 + 24 + 6 + 48,
+	.vrefresh = 60,
+};
+
+static int wuxga_nt_panel_get_modes(struct drm_panel *panel)
+{
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(panel->drm, &default_mode);
+	if (!mode) {
+		dev_err(panel->drm->dev, "failed to add mode %ux%ux@%u\n",
+				default_mode.hdisplay, default_mode.vdisplay,
+				default_mode.vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(panel->connector, mode);
+
+	panel->connector->display_info.width_mm = 217;
+	panel->connector->display_info.height_mm = 136;
+
+	return 1;
+}
+
+static const struct drm_panel_funcs wuxga_nt_panel_funcs = {
+	.disable = wuxga_nt_panel_disable,
+	.unprepare = wuxga_nt_panel_unprepare,
+	.prepare = wuxga_nt_panel_prepare,
+	.enable = wuxga_nt_panel_enable,
+	.get_modes = wuxga_nt_panel_get_modes,
+};
+
+static const struct of_device_id wuxga_nt_of_match[] = {
+	{ .compatible = "panasonic,vvx10f034n00", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, wuxga_nt_of_match);
+
+static int wuxga_nt_panel_add(struct wuxga_nt_panel *wuxga_nt)
+{
+	struct device *dev = &wuxga_nt->dsi->dev;
+	struct device_node *np;
+	int ret;
+
+	wuxga_nt->mode = &default_mode;
+
+	wuxga_nt->supply = devm_regulator_get(dev, "power");
+	if (IS_ERR(wuxga_nt->supply))
+		return PTR_ERR(wuxga_nt->supply);
+
+	np = of_parse_phandle(dev->of_node, "backlight", 0);
+	if (np) {
+		wuxga_nt->backlight = of_find_backlight_by_node(np);
+		of_node_put(np);
+
+		if (!wuxga_nt->backlight)
+			return -EPROBE_DEFER;
+	}
+
+	drm_panel_init(&wuxga_nt->base);
+	wuxga_nt->base.funcs = &wuxga_nt_panel_funcs;
+	wuxga_nt->base.dev = &wuxga_nt->dsi->dev;
+
+	ret = drm_panel_add(&wuxga_nt->base);
+	if (ret < 0)
+		goto put_backlight;
+
+	return 0;
+
+put_backlight:
+	if (wuxga_nt->backlight)
+		put_device(&wuxga_nt->backlight->dev);
+
+	return ret;
+}
+
+static void wuxga_nt_panel_del(struct wuxga_nt_panel *wuxga_nt)
+{
+	if (wuxga_nt->base.dev)
+		drm_panel_remove(&wuxga_nt->base);
+
+	if (wuxga_nt->backlight)
+		put_device(&wuxga_nt->backlight->dev);
+}
+
+static int wuxga_nt_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct wuxga_nt_panel *wuxga_nt;
+	int ret;
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+			MIPI_DSI_MODE_VIDEO_HSE |
+			MIPI_DSI_CLOCK_NON_CONTINUOUS |
+			MIPI_DSI_MODE_LPM;
+
+	wuxga_nt = devm_kzalloc(&dsi->dev, sizeof(*wuxga_nt), GFP_KERNEL);
+	if (!wuxga_nt)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, wuxga_nt);
+
+	wuxga_nt->dsi = dsi;
+
+	ret = wuxga_nt_panel_add(wuxga_nt);
+	if (ret < 0)
+		return ret;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static int wuxga_nt_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct wuxga_nt_panel *wuxga_nt = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = wuxga_nt_panel_disable(&wuxga_nt->base);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to disable panel: %d\n", ret);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_detach(&wuxga_nt->base);
+	wuxga_nt_panel_del(wuxga_nt);
+
+	return 0;
+}
+
+static void wuxga_nt_panel_shutdown(struct mipi_dsi_device *dsi)
+{
+	struct wuxga_nt_panel *wuxga_nt = mipi_dsi_get_drvdata(dsi);
+
+	wuxga_nt_panel_disable(&wuxga_nt->base);
+}
+
+static struct mipi_dsi_driver wuxga_nt_panel_driver = {
+	.driver = {
+		.name = "panel-panasonic-vvx10f034n00",
+		.of_match_table = wuxga_nt_of_match,
+	},
+	.probe = wuxga_nt_panel_probe,
+	.remove = wuxga_nt_panel_remove,
+	.shutdown = wuxga_nt_panel_shutdown,
+};
+module_mipi_dsi_driver(wuxga_nt_panel_driver);
+
+MODULE_AUTHOR("Werner Johansson <werner.johansson@sonymobile.com>");
+MODULE_DESCRIPTION("Panasonic VVX10F034N00 Novatek NT1397-based WUXGA (1920x1200) video mode panel driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
new file mode 100644
index 000000000000..3aeb0bda4947
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
@@ -0,0 +1,387 @@
+/*
+ * Copyright (C) 2015 Red Hat
+ * Copyright (C) 2015 Sony Mobile Communications Inc.
+ * Author: Werner Johansson <werner.johansson@sonymobile.com>
+ *
+ * Based on AUO panel driver by Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/backlight.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <video/mipi_display.h>
+
+struct sharp_nt_panel {
+	struct drm_panel base;
+	struct mipi_dsi_device *dsi;
+
+	struct backlight_device *backlight;
+	struct regulator *supply;
+	struct gpio_desc *reset_gpio;
+
+	bool prepared;
+	bool enabled;
+
+	const struct drm_display_mode *mode;
+};
+
+static inline struct sharp_nt_panel *to_sharp_nt_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct sharp_nt_panel, base);
+}
+
+static int sharp_nt_panel_init(struct sharp_nt_panel *sharp_nt)
+{
+	struct mipi_dsi_device *dsi = sharp_nt->dsi;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0)
+		return ret;
+
+	msleep(120);
+
+	/* Novatek two-lane operation */
+	ret = mipi_dsi_dcs_write(dsi, 0xae, (u8[]){ 0x03 }, 1);
+	if (ret < 0)
+		return ret;
+
+	/* Set both MCU and RGB I/F to 24bpp */
+	ret = mipi_dsi_dcs_set_pixel_format(dsi, MIPI_DCS_PIXEL_FMT_24BIT |
+					(MIPI_DCS_PIXEL_FMT_24BIT << 4));
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int sharp_nt_panel_on(struct sharp_nt_panel *sharp_nt)
+{
+	struct mipi_dsi_device *dsi = sharp_nt->dsi;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int sharp_nt_panel_off(struct sharp_nt_panel *sharp_nt)
+{
+	struct mipi_dsi_device *dsi = sharp_nt->dsi;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0)
+		return ret;
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+
+static int sharp_nt_panel_disable(struct drm_panel *panel)
+{
+	struct sharp_nt_panel *sharp_nt = to_sharp_nt_panel(panel);
+
+	if (!sharp_nt->enabled)
+		return 0;
+
+	if (sharp_nt->backlight) {
+		sharp_nt->backlight->props.power = FB_BLANK_POWERDOWN;
+		backlight_update_status(sharp_nt->backlight);
+	}
+
+	sharp_nt->enabled = false;
+
+	return 0;
+}
+
+static int sharp_nt_panel_unprepare(struct drm_panel *panel)
+{
+	struct sharp_nt_panel *sharp_nt = to_sharp_nt_panel(panel);
+	int ret;
+
+	if (!sharp_nt->prepared)
+		return 0;
+
+	ret = sharp_nt_panel_off(sharp_nt);
+	if (ret < 0) {
+		dev_err(panel->dev, "failed to set panel off: %d\n", ret);
+		return ret;
+	}
+
+	regulator_disable(sharp_nt->supply);
+	if (sharp_nt->reset_gpio)
+		gpiod_set_value(sharp_nt->reset_gpio, 0);
+
+	sharp_nt->prepared = false;
+
+	return 0;
+}
+
+static int sharp_nt_panel_prepare(struct drm_panel *panel)
+{
+	struct sharp_nt_panel *sharp_nt = to_sharp_nt_panel(panel);
+	int ret;
+
+	if (sharp_nt->prepared)
+		return 0;
+
+	ret = regulator_enable(sharp_nt->supply);
+	if (ret < 0)
+		return ret;
+
+	msleep(20);
+
+	if (sharp_nt->reset_gpio) {
+		gpiod_set_value(sharp_nt->reset_gpio, 1);
+		msleep(1);
+		gpiod_set_value(sharp_nt->reset_gpio, 0);
+		msleep(1);
+		gpiod_set_value(sharp_nt->reset_gpio, 1);
+		msleep(10);
+	}
+
+	ret = sharp_nt_panel_init(sharp_nt);
+	if (ret < 0) {
+		dev_err(panel->dev, "failed to init panel: %d\n", ret);
+		goto poweroff;
+	}
+
+	ret = sharp_nt_panel_on(sharp_nt);
+	if (ret < 0) {
+		dev_err(panel->dev, "failed to set panel on: %d\n", ret);
+		goto poweroff;
+	}
+
+	sharp_nt->prepared = true;
+
+	return 0;
+
+poweroff:
+	regulator_disable(sharp_nt->supply);
+	if (sharp_nt->reset_gpio)
+		gpiod_set_value(sharp_nt->reset_gpio, 0);
+	return ret;
+}
+
+static int sharp_nt_panel_enable(struct drm_panel *panel)
+{
+	struct sharp_nt_panel *sharp_nt = to_sharp_nt_panel(panel);
+
+	if (sharp_nt->enabled)
+		return 0;
+
+	if (sharp_nt->backlight) {
+		sharp_nt->backlight->props.power = FB_BLANK_UNBLANK;
+		backlight_update_status(sharp_nt->backlight);
+	}
+
+	sharp_nt->enabled = true;
+
+	return 0;
+}
+
+static const struct drm_display_mode default_mode = {
+	.clock = 41118,
+	.hdisplay = 540,
+	.hsync_start = 540 + 48,
+	.hsync_end = 540 + 48 + 80,
+	.htotal = 540 + 48 + 80 + 32,
+	.vdisplay = 960,
+	.vsync_start = 960 + 3,
+	.vsync_end = 960 + 3 + 15,
+	.vtotal = 960 + 3 + 15 + 1,
+	.vrefresh = 60,
+};
+
+static int sharp_nt_panel_get_modes(struct drm_panel *panel)
+{
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(panel->drm, &default_mode);
+	if (!mode) {
+		dev_err(panel->drm->dev, "failed to add mode %ux%ux@%u\n",
+				default_mode.hdisplay, default_mode.vdisplay,
+				default_mode.vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(panel->connector, mode);
+
+	panel->connector->display_info.width_mm = 54;
+	panel->connector->display_info.height_mm = 95;
+
+	return 1;
+}
+
+static const struct drm_panel_funcs sharp_nt_panel_funcs = {
+	.disable = sharp_nt_panel_disable,
+	.unprepare = sharp_nt_panel_unprepare,
+	.prepare = sharp_nt_panel_prepare,
+	.enable = sharp_nt_panel_enable,
+	.get_modes = sharp_nt_panel_get_modes,
+};
+
+static int sharp_nt_panel_add(struct sharp_nt_panel *sharp_nt)
+{
+	struct device *dev = &sharp_nt->dsi->dev;
+	struct device_node *np;
+	int ret;
+
+	sharp_nt->mode = &default_mode;
+
+	sharp_nt->supply = devm_regulator_get(dev, "avdd");
+	if (IS_ERR(sharp_nt->supply))
+		return PTR_ERR(sharp_nt->supply);
+
+	sharp_nt->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(sharp_nt->reset_gpio)) {
+		dev_err(dev, "cannot get reset-gpios %ld\n",
+			PTR_ERR(sharp_nt->reset_gpio));
+		sharp_nt->reset_gpio = NULL;
+	} else {
+		gpiod_set_value(sharp_nt->reset_gpio, 0);
+	}
+
+	np = of_parse_phandle(dev->of_node, "backlight", 0);
+	if (np) {
+		sharp_nt->backlight = of_find_backlight_by_node(np);
+		of_node_put(np);
+
+		if (!sharp_nt->backlight)
+			return -EPROBE_DEFER;
+	}
+
+	drm_panel_init(&sharp_nt->base);
+	sharp_nt->base.funcs = &sharp_nt_panel_funcs;
+	sharp_nt->base.dev = &sharp_nt->dsi->dev;
+
+	ret = drm_panel_add(&sharp_nt->base);
+	if (ret < 0)
+		goto put_backlight;
+
+	return 0;
+
+put_backlight:
+	if (sharp_nt->backlight)
+		put_device(&sharp_nt->backlight->dev);
+
+	return ret;
+}
+
+static void sharp_nt_panel_del(struct sharp_nt_panel *sharp_nt)
+{
+	if (sharp_nt->base.dev)
+		drm_panel_remove(&sharp_nt->base);
+
+	if (sharp_nt->backlight)
+		put_device(&sharp_nt->backlight->dev);
+}
+
+static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct sharp_nt_panel *sharp_nt;
+	int ret;
+
+	dsi->lanes = 2;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+			MIPI_DSI_MODE_VIDEO_HSE |
+			MIPI_DSI_CLOCK_NON_CONTINUOUS |
+			MIPI_DSI_MODE_EOT_PACKET;
+
+	sharp_nt = devm_kzalloc(&dsi->dev, sizeof(*sharp_nt), GFP_KERNEL);
+	if (!sharp_nt)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, sharp_nt);
+
+	sharp_nt->dsi = dsi;
+
+	ret = sharp_nt_panel_add(sharp_nt);
+	if (ret < 0)
+		return ret;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static int sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct sharp_nt_panel *sharp_nt = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = sharp_nt_panel_disable(&sharp_nt->base);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to disable panel: %d\n", ret);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_detach(&sharp_nt->base);
+	sharp_nt_panel_del(sharp_nt);
+
+	return 0;
+}
+
+static void sharp_nt_panel_shutdown(struct mipi_dsi_device *dsi)
+{
+	struct sharp_nt_panel *sharp_nt = mipi_dsi_get_drvdata(dsi);
+
+	sharp_nt_panel_disable(&sharp_nt->base);
+}
+
+static const struct of_device_id sharp_nt_of_match[] = {
+	{ .compatible = "sharp,ls043t1le01-qhd", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sharp_nt_of_match);
+
+static struct mipi_dsi_driver sharp_nt_panel_driver = {
+	.driver = {
+		.name = "panel-sharp-ls043t1le01-qhd",
+		.of_match_table = sharp_nt_of_match,
+	},
+	.probe = sharp_nt_panel_probe,
+	.remove = sharp_nt_panel_remove,
+	.shutdown = sharp_nt_panel_shutdown,
+};
+module_mipi_dsi_driver(sharp_nt_panel_driver);
+
+MODULE_AUTHOR("Werner Johansson <werner.johansson@sonymobile.com>");
+MODULE_DESCRIPTION("Sharp LS043T1LE01 NT35565-based qHD (540x960) video mode panel driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index f97b73ec4713..f88a631c43ab 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -44,6 +44,10 @@ struct panel_desc {
 
 	unsigned int bpc;
 
+	/**
+	 * @width: width (in millimeters) of the panel's active display area
+	 * @height: height (in millimeters) of the panel's active display area
+	 */
 	struct {
 		unsigned int width;
 		unsigned int height;
@@ -832,6 +836,34 @@ static const struct panel_desc innolux_g121i1_l01 = {
 	},
 };
 
+static const struct drm_display_mode innolux_g121x1_l03_mode = {
+	.clock = 65000,
+	.hdisplay = 1024,
+	.hsync_start = 1024 + 0,
+	.hsync_end = 1024 + 1,
+	.htotal = 1024 + 0 + 1 + 320,
+	.vdisplay = 768,
+	.vsync_start = 768 + 38,
+	.vsync_end = 768 + 38 + 1,
+	.vtotal = 768 + 38 + 1 + 0,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc innolux_g121x1_l03 = {
+	.modes = &innolux_g121x1_l03_mode,
+	.num_modes = 1,
+	.bpc = 6,
+	.size = {
+		.width = 246,
+		.height = 185,
+	},
+	.delay = {
+		.enable = 200,
+		.unprepare = 200,
+		.disable = 400,
+	},
+};
+
 static const struct drm_display_mode innolux_n116bge_mode = {
 	.clock = 76420,
 	.hdisplay = 1366,
@@ -902,6 +934,30 @@ static const struct panel_desc innolux_zj070na_01p = {
 	},
 };
 
+static const struct display_timing kyo_tcg121xglp_timing = {
+	.pixelclock = { 52000000, 65000000, 71000000 },
+	.hactive = { 1024, 1024, 1024 },
+	.hfront_porch = { 2, 2, 2 },
+	.hback_porch = { 2, 2, 2 },
+	.hsync_len = { 86, 124, 244 },
+	.vactive = { 768, 768, 768 },
+	.vfront_porch = { 2, 2, 2 },
+	.vback_porch = { 2, 2, 2 },
+	.vsync_len = { 6, 34, 73 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc kyo_tcg121xglp = {
+	.timings = &kyo_tcg121xglp_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 246,
+		.height = 184,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
 static const struct drm_display_mode lg_lb070wv8_mode = {
 	.clock = 33246,
 	.hdisplay = 800,
@@ -1027,6 +1083,30 @@ static const struct panel_desc ortustech_com43h4m85ulc = {
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
 };
 
+static const struct drm_display_mode qd43003c0_40_mode = {
+	.clock = 9000,
+	.hdisplay = 480,
+	.hsync_start = 480 + 8,
+	.hsync_end = 480 + 8 + 4,
+	.htotal = 480 + 8 + 4 + 39,
+	.vdisplay = 272,
+	.vsync_start = 272 + 4,
+	.vsync_end = 272 + 4 + 10,
+	.vtotal = 272 + 4 + 10 + 2,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc qd43003c0_40 = {
+	.modes = &qd43003c0_40_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 95,
+		.height = 53,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+};
+
 static const struct drm_display_mode samsung_ltn101nt05_mode = {
 	.clock = 54030,
 	.hdisplay = 1024,
@@ -1158,6 +1238,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible ="innolux,g121i1-l01",
 		.data = &innolux_g121i1_l01
 	}, {
+		.compatible = "innolux,g121x1-l03",
+		.data = &innolux_g121x1_l03,
+	}, {
 		.compatible = "innolux,n116bge",
 		.data = &innolux_n116bge,
 	}, {
@@ -1167,6 +1250,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "innolux,zj070na-01p",
 		.data = &innolux_zj070na_01p,
 	}, {
+		.compatible = "kyo,tcg121xglp",
+		.data = &kyo_tcg121xglp,
+	}, {
 		.compatible = "lg,lb070wv8",
 		.data = &lg_lb070wv8,
 	}, {
@@ -1182,6 +1268,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "ortustech,com43h4m85ulc",
 		.data = &ortustech_com43h4m85ulc,
 	}, {
+		.compatible = "qiaodian,qd43003c0-40",
+		.data = &qd43003c0_40,
+	}, {
 		.compatible = "samsung,ltn101nt05",
 		.data = &samsung_ltn101nt05,
 	}, {
@@ -1263,6 +1352,36 @@ static const struct panel_desc_dsi auo_b080uan01 = {
 	.lanes = 4,
 };
 
+static const struct drm_display_mode boe_tv080wum_nl0_mode = {
+	.clock = 160000,
+	.hdisplay = 1200,
+	.hsync_start = 1200 + 120,
+	.hsync_end = 1200 + 120 + 20,
+	.htotal = 1200 + 120 + 20 + 21,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 21,
+	.vsync_end = 1920 + 21 + 3,
+	.vtotal = 1920 + 21 + 3 + 18,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc_dsi boe_tv080wum_nl0 = {
+	.desc = {
+		.modes = &boe_tv080wum_nl0_mode,
+		.num_modes = 1,
+		.size = {
+			.width = 107,
+			.height = 172,
+		},
+	},
+	.flags = MIPI_DSI_MODE_VIDEO |
+		 MIPI_DSI_MODE_VIDEO_BURST |
+		 MIPI_DSI_MODE_VIDEO_SYNC_PULSE,
+	.format = MIPI_DSI_FMT_RGB888,
+	.lanes = 4,
+};
+
 static const struct drm_display_mode lg_ld070wx3_sl01_mode = {
 	.clock = 71000,
 	.hdisplay = 800,
@@ -1348,11 +1467,15 @@ static const struct panel_desc_dsi panasonic_vvx10f004b00 = {
 	.lanes = 4,
 };
 
+
 static const struct of_device_id dsi_of_match[] = {
 	{
 		.compatible = "auo,b080uan01",
 		.data = &auo_b080uan01
 	}, {
+		.compatible = "boe,tv080wum-nl0",
+		.data = &boe_tv080wum_nl0
+	}, {
 		.compatible = "lg,ld070wx3-sl01",
 		.data = &lg_ld070wx3_sl01
 	}, {
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 183aea1abebc..86276519b2ef 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -521,7 +521,7 @@ static const struct drm_framebuffer_funcs qxl_fb_funcs = {
 int
 qxl_framebuffer_init(struct drm_device *dev,
 		     struct qxl_framebuffer *qfb,
-		     struct drm_mode_fb_cmd2 *mode_cmd,
+		     const struct drm_mode_fb_cmd2 *mode_cmd,
 		     struct drm_gem_object *obj)
 {
 	int ret;
@@ -876,16 +876,6 @@ static const struct drm_connector_helper_funcs qxl_connector_helper_funcs = {
 	.best_encoder = qxl_best_encoder,
 };
 
-static void qxl_conn_save(struct drm_connector *connector)
-{
-	DRM_DEBUG("\n");
-}
-
-static void qxl_conn_restore(struct drm_connector *connector)
-{
-	DRM_DEBUG("\n");
-}
-
 static enum drm_connector_status qxl_conn_detect(
 			struct drm_connector *connector,
 			bool force)
@@ -932,10 +922,8 @@ static void qxl_conn_destroy(struct drm_connector *connector)
 
 static const struct drm_connector_funcs qxl_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.save = qxl_conn_save,
-	.restore = qxl_conn_restore,
 	.detect = qxl_conn_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes_nomerge,
+	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = qxl_conn_set_property,
 	.destroy = qxl_conn_destroy,
 };
@@ -980,7 +968,7 @@ static int qdev_output_init(struct drm_device *dev, int num_output)
 			   &qxl_connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL);
 
 	drm_encoder_init(dev, &qxl_output->enc, &qxl_enc_funcs,
-			 DRM_MODE_ENCODER_VIRTUAL);
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
 
 	/* we get HPD via client monitors config */
 	connector->polled = DRM_CONNECTOR_POLL_HPD;
@@ -1003,7 +991,7 @@ static int qdev_output_init(struct drm_device *dev, int num_output)
 static struct drm_framebuffer *
 qxl_user_framebuffer_create(struct drm_device *dev,
 			    struct drm_file *file_priv,
-			    struct drm_mode_fb_cmd2 *mode_cmd)
+			    const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct qxl_framebuffer *qxl_fb;
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 01a86948eb8c..6e6b9b1519b8 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -390,7 +390,7 @@ void qxl_fbdev_set_suspend(struct qxl_device *qdev, int state);
 int
 qxl_framebuffer_init(struct drm_device *dev,
 		     struct qxl_framebuffer *rfb,
-		     struct drm_mode_fb_cmd2 *mode_cmd,
+		     const struct drm_mode_fb_cmd2 *mode_cmd,
 		     struct drm_gem_object *obj);
 void qxl_display_read_client_monitors_config(struct qxl_device *qdev);
 void qxl_send_monitors_config(struct qxl_device *qdev);
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index c4a552637c93..7136e521e6db 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -40,7 +40,6 @@
 struct qxl_fbdev {
 	struct drm_fb_helper helper;
 	struct qxl_framebuffer	qfb;
-	struct list_head	fbdev_list;
 	struct qxl_device	*qdev;
 
 	spinlock_t delayed_ops_lock;
@@ -283,7 +282,7 @@ int qxl_get_handle_for_primary_fb(struct qxl_device *qdev,
 }
 
 static int qxlfb_create_pinned_object(struct qxl_fbdev *qfbdev,
-				      struct drm_mode_fb_cmd2 *mode_cmd,
+				      const struct drm_mode_fb_cmd2 *mode_cmd,
 				      struct drm_gem_object **gobj_p)
 {
 	struct qxl_device *qdev = qfbdev->qdev;
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index b28370e014c6..5e1d7899dd72 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -32,7 +32,7 @@ static void qxl_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 	struct qxl_bo *bo;
 	struct qxl_device *qdev;
 
-	bo = container_of(tbo, struct qxl_bo, tbo);
+	bo = to_qxl_bo(tbo);
 	qdev = (struct qxl_device *)bo->gem_base.dev->dev_private;
 
 	qxl_surface_evict(qdev, bo, false);
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 0cbc4c987164..953412766416 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -201,7 +201,7 @@ static void qxl_evict_flags(struct ttm_buffer_object *bo,
 		placement->num_busy_placement = 1;
 		return;
 	}
-	qbo = container_of(bo, struct qxl_bo, tbo);
+	qbo = to_qxl_bo(bo);
 	qxl_ttm_placement_from_domain(qbo, QXL_GEM_DOMAIN_CPU, false);
 	*placement = qbo->placement;
 }
@@ -365,7 +365,7 @@ static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
 
 	if (!qxl_ttm_bo_is_qxl_bo(bo))
 		return;
-	qbo = container_of(bo, struct qxl_bo, tbo);
+	qbo = to_qxl_bo(bo);
 	qdev = qbo->gem_base.dev->dev_private;
 
 	if (bo->mem.mem_type == TTM_PL_PRIV0 && qbo->surface_id)
diff --git a/drivers/gpu/drm/radeon/Kconfig b/drivers/gpu/drm/radeon/Kconfig
index 421ae130809b..9909f5c68d76 100644
--- a/drivers/gpu/drm/radeon/Kconfig
+++ b/drivers/gpu/drm/radeon/Kconfig
@@ -5,12 +5,3 @@ config DRM_RADEON_USERPTR
 	help
 	  This option selects CONFIG_MMU_NOTIFIER if it isn't already
 	  selected to enabled full userptr support.
-
-config DRM_RADEON_UMS
-	bool "Enable userspace modesetting on radeon (DEPRECATED)"
-	depends on DRM_RADEON
-	help
-	  Choose this option if you still need userspace modesetting.
-
-	  Userspace modesetting is deprecated for quite some time now, so
-	  enable this only if you have ancient versions of the DDX drivers.
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index dea53e36a2ef..08bd17d3925c 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -58,10 +58,6 @@ $(obj)/evergreen_cs.o: $(obj)/evergreen_reg_safe.h $(obj)/cayman_reg_safe.h
 
 radeon-y := radeon_drv.o
 
-# add UMS driver
-radeon-$(CONFIG_DRM_RADEON_UMS)+= radeon_cp.o radeon_state.o radeon_mem.o \
-	radeon_irq.o r300_cmdbuf.o r600_cp.o r600_blit.o drm_buffer.o
-
 # add KMS driver
 radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
 	radeon_atombios.o radeon_agp.o atombios_crtc.o radeon_combios.o \
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index dac78ad24b31..801dd60ac192 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -25,6 +25,7 @@
  */
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/radeon_drm.h>
 #include <drm/drm_fixed.h>
 #include "radeon.h"
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index bd73b4069069..44ee72e04df9 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -302,77 +302,31 @@ static int convert_bpc_to_bpp(int bpc)
 		return bpc * 3;
 }
 
-/* get the max pix clock supported by the link rate and lane num */
-static int dp_get_max_dp_pix_clock(int link_rate,
-				   int lane_num,
-				   int bpp)
-{
-	return (link_rate * lane_num * 8) / bpp;
-}
-
 /***** radeon specific DP functions *****/
 
-int radeon_dp_get_max_link_rate(struct drm_connector *connector,
-				const u8 dpcd[DP_DPCD_SIZE])
-{
-	int max_link_rate;
-
-	if (radeon_connector_is_dp12_capable(connector))
-		max_link_rate = min(drm_dp_max_link_rate(dpcd), 540000);
-	else
-		max_link_rate = min(drm_dp_max_link_rate(dpcd), 270000);
-
-	return max_link_rate;
-}
-
-/* First get the min lane# when low rate is used according to pixel clock
- * (prefer low rate), second check max lane# supported by DP panel,
- * if the max lane# < low rate lane# then use max lane# instead.
- */
-static int radeon_dp_get_dp_lane_number(struct drm_connector *connector,
-					const u8 dpcd[DP_DPCD_SIZE],
-					int pix_clock)
-{
-	int bpp = convert_bpc_to_bpp(radeon_get_monitor_bpc(connector));
-	int max_link_rate = radeon_dp_get_max_link_rate(connector, dpcd);
-	int max_lane_num = drm_dp_max_lane_count(dpcd);
-	int lane_num;
-	int max_dp_pix_clock;
-
-	for (lane_num = 1; lane_num < max_lane_num; lane_num <<= 1) {
-		max_dp_pix_clock = dp_get_max_dp_pix_clock(max_link_rate, lane_num, bpp);
-		if (pix_clock <= max_dp_pix_clock)
-			break;
-	}
-
-	return lane_num;
-}
-
-static int radeon_dp_get_dp_link_clock(struct drm_connector *connector,
-				       const u8 dpcd[DP_DPCD_SIZE],
-				       int pix_clock)
+int radeon_dp_get_dp_link_config(struct drm_connector *connector,
+				 const u8 dpcd[DP_DPCD_SIZE],
+				 unsigned pix_clock,
+				 unsigned *dp_lanes, unsigned *dp_rate)
 {
 	int bpp = convert_bpc_to_bpp(radeon_get_monitor_bpc(connector));
-	int lane_num, max_pix_clock;
-
-	if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) ==
-	    ENCODER_OBJECT_ID_NUTMEG)
-		return 270000;
-
-	lane_num = radeon_dp_get_dp_lane_number(connector, dpcd, pix_clock);
-	max_pix_clock = dp_get_max_dp_pix_clock(162000, lane_num, bpp);
-	if (pix_clock <= max_pix_clock)
-		return 162000;
-	max_pix_clock = dp_get_max_dp_pix_clock(270000, lane_num, bpp);
-	if (pix_clock <= max_pix_clock)
-		return 270000;
-	if (radeon_connector_is_dp12_capable(connector)) {
-		max_pix_clock = dp_get_max_dp_pix_clock(540000, lane_num, bpp);
-		if (pix_clock <= max_pix_clock)
-			return 540000;
+	static const unsigned link_rates[3] = { 162000, 270000, 540000 };
+	unsigned max_link_rate = drm_dp_max_link_rate(dpcd);
+	unsigned max_lane_num = drm_dp_max_lane_count(dpcd);
+	unsigned lane_num, i, max_pix_clock;
+
+	for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) {
+		for (i = 0; i < ARRAY_SIZE(link_rates) && link_rates[i] <= max_link_rate; i++) {
+			max_pix_clock = (lane_num * link_rates[i] * 8) / bpp;
+			if (max_pix_clock >= pix_clock) {
+				*dp_lanes = lane_num;
+				*dp_rate = link_rates[i];
+				return 0;
+			}
+		}
 	}
 
-	return radeon_dp_get_max_link_rate(connector, dpcd);
+	return -EINVAL;
 }
 
 static u8 radeon_dp_encoder_service(struct radeon_device *rdev,
@@ -491,6 +445,7 @@ void radeon_dp_set_link_config(struct drm_connector *connector,
 {
 	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 	struct radeon_connector_atom_dig *dig_connector;
+	int ret;
 
 	if (!radeon_connector->con_priv)
 		return;
@@ -498,10 +453,14 @@ void radeon_dp_set_link_config(struct drm_connector *connector,
 
 	if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
 	    (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) {
-		dig_connector->dp_clock =
-			radeon_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
-		dig_connector->dp_lane_count =
-			radeon_dp_get_dp_lane_number(connector, dig_connector->dpcd, mode->clock);
+		ret = radeon_dp_get_dp_link_config(connector, dig_connector->dpcd,
+						   mode->clock,
+						   &dig_connector->dp_lane_count,
+						   &dig_connector->dp_clock);
+		if (ret) {
+			dig_connector->dp_clock = 0;
+			dig_connector->dp_lane_count = 0;
+		}
 	}
 }
 
@@ -510,7 +469,8 @@ int radeon_dp_mode_valid_helper(struct drm_connector *connector,
 {
 	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 	struct radeon_connector_atom_dig *dig_connector;
-	int dp_clock;
+	unsigned dp_clock, dp_lanes;
+	int ret;
 
 	if ((mode->clock > 340000) &&
 	    (!radeon_connector_is_dp12_capable(connector)))
@@ -520,8 +480,12 @@ int radeon_dp_mode_valid_helper(struct drm_connector *connector,
 		return MODE_CLOCK_HIGH;
 	dig_connector = radeon_connector->con_priv;
 
-	dp_clock =
-		radeon_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock);
+	ret = radeon_dp_get_dp_link_config(connector, dig_connector->dpcd,
+					   mode->clock,
+					   &dp_lanes,
+					   &dp_clock);
+	if (ret)
+		return MODE_CLOCK_HIGH;
 
 	if ((dp_clock == 540000) &&
 	    (!radeon_connector_is_dp12_capable(connector)))
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index bb292143997e..01b20e14a247 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -2767,23 +2767,27 @@ radeon_add_atom_encoder(struct drm_device *dev,
 	case ENCODER_OBJECT_ID_INTERNAL_LVTM1:
 		if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 			radeon_encoder->rmx_type = RMX_FULL;
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_LVDS, NULL);
 			radeon_encoder->enc_priv = radeon_atombios_get_lvds_info(radeon_encoder);
 		} else {
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_TMDS, NULL);
 			radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
 		}
 		drm_encoder_helper_add(encoder, &radeon_atom_dig_helper_funcs);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_DAC1:
-		drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_DAC);
+		drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+				 DRM_MODE_ENCODER_DAC, NULL);
 		radeon_encoder->enc_priv = radeon_atombios_set_dac_info(radeon_encoder);
 		drm_encoder_helper_add(encoder, &radeon_atom_dac_helper_funcs);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_DAC2:
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
 	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
-		drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TVDAC);
+		drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+				 DRM_MODE_ENCODER_TVDAC, NULL);
 		radeon_encoder->enc_priv = radeon_atombios_set_dac_info(radeon_encoder);
 		drm_encoder_helper_add(encoder, &radeon_atom_dac_helper_funcs);
 		break;
@@ -2797,13 +2801,16 @@ radeon_add_atom_encoder(struct drm_device *dev,
 	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
 		if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 			radeon_encoder->rmx_type = RMX_FULL;
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_LVDS, NULL);
 			radeon_encoder->enc_priv = radeon_atombios_get_lvds_info(radeon_encoder);
 		} else if (radeon_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_DAC);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_DAC, NULL);
 			radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
 		} else {
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_TMDS, NULL);
 			radeon_encoder->enc_priv = radeon_atombios_set_dig_info(radeon_encoder);
 		}
 		drm_encoder_helper_add(encoder, &radeon_atom_dig_helper_funcs);
@@ -2820,11 +2827,14 @@ radeon_add_atom_encoder(struct drm_device *dev,
 		/* these are handled by the primary encoders */
 		radeon_encoder->is_ext_encoder = true;
 		if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_LVDS, NULL);
 		else if (radeon_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_DAC);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_DAC, NULL);
 		else
-			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_TMDS);
+			drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs,
+					 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &radeon_atom_ext_helper_funcs);
 		break;
 	}
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index f81fb2641097..4c30d8c65558 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -4132,10 +4132,10 @@ struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
  * @rdev: radeon_device pointer
  * @ib: radeon indirect buffer object
  *
- * Emits an DE (drawing engine) or CE (constant engine) IB
+ * Emits a DE (drawing engine) or CE (constant engine) IB
  * on the gfx ring.  IBs are usually generated by userspace
  * acceleration drivers and submitted to the kernel for
- * sheduling on the ring.  This function schedules the IB
+ * scheduling on the ring.  This function schedules the IB
  * on the gfx ring for execution by the GPU.
  */
 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
diff --git a/drivers/gpu/drm/radeon/dce6_afmt.c b/drivers/gpu/drm/radeon/dce6_afmt.c
index 752072771388..367a916f364e 100644
--- a/drivers/gpu/drm/radeon/dce6_afmt.c
+++ b/drivers/gpu/drm/radeon/dce6_afmt.c
@@ -301,6 +301,14 @@ void dce6_dp_audio_set_dto(struct radeon_device *rdev,
 	 * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator
 	 */
 	if (ASIC_IS_DCE8(rdev)) {
+		unsigned int div = (RREG32(DENTIST_DISPCLK_CNTL) &
+			DENTIST_DPREFCLK_WDIVIDER_MASK) >>
+			DENTIST_DPREFCLK_WDIVIDER_SHIFT;
+		div = radeon_audio_decode_dfs_div(div);
+
+		if (div)
+			clock = clock * 100 / div;
+
 		WREG32(DCE8_DCCG_AUDIO_DTO1_PHASE, 24000);
 		WREG32(DCE8_DCCG_AUDIO_DTO1_MODULE, clock);
 	} else {
diff --git a/drivers/gpu/drm/radeon/drm_buffer.c b/drivers/gpu/drm/radeon/drm_buffer.c
deleted file mode 100644
index f4e0f3a3d7b1..000000000000
--- a/drivers/gpu/drm/radeon/drm_buffer.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/**************************************************************************
- *
- * Copyright 2010 Pauli Nieminen.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- *
- **************************************************************************/
-/*
- * Multipart buffer for coping data which is larger than the page size.
- *
- * Authors:
- * Pauli Nieminen <suokkos-at-gmail-dot-com>
- */
-
-#include <linux/export.h>
-#include "drm_buffer.h"
-
-/**
- * Allocate the drm buffer object.
- *
- *   buf: Pointer to a pointer where the object is stored.
- *   size: The number of bytes to allocate.
- */
-int drm_buffer_alloc(struct drm_buffer **buf, int size)
-{
-	int nr_pages = size / PAGE_SIZE + 1;
-	int idx;
-
-	/* Allocating pointer table to end of structure makes drm_buffer
-	 * variable sized */
-	*buf = kzalloc(sizeof(struct drm_buffer) + nr_pages*sizeof(char *),
-			GFP_KERNEL);
-
-	if (*buf == NULL) {
-		DRM_ERROR("Failed to allocate drm buffer object to hold"
-				" %d bytes in %d pages.\n",
-				size, nr_pages);
-		return -ENOMEM;
-	}
-
-	(*buf)->size = size;
-
-	for (idx = 0; idx < nr_pages; ++idx) {
-
-		(*buf)->data[idx] =
-			kmalloc(min(PAGE_SIZE, size - idx * PAGE_SIZE),
-				GFP_KERNEL);
-
-
-		if ((*buf)->data[idx] == NULL) {
-			DRM_ERROR("Failed to allocate %dth page for drm"
-					" buffer with %d bytes and %d pages.\n",
-					idx + 1, size, nr_pages);
-			goto error_out;
-		}
-
-	}
-
-	return 0;
-
-error_out:
-
-	for (; idx >= 0; --idx)
-		kfree((*buf)->data[idx]);
-
-	kfree(*buf);
-	return -ENOMEM;
-}
-
-/**
- * Copy the user data to the begin of the buffer and reset the processing
- * iterator.
- *
- *   user_data: A pointer the data that is copied to the buffer.
- *   size: The Number of bytes to copy.
- */
-int drm_buffer_copy_from_user(struct drm_buffer *buf,
-			      void __user *user_data, int size)
-{
-	int nr_pages = size / PAGE_SIZE + 1;
-	int idx;
-
-	if (size > buf->size) {
-		DRM_ERROR("Requesting to copy %d bytes to a drm buffer with"
-				" %d bytes space\n",
-				size, buf->size);
-		return -EFAULT;
-	}
-
-	for (idx = 0; idx < nr_pages; ++idx) {
-
-		if (copy_from_user(buf->data[idx],
-			user_data + idx * PAGE_SIZE,
-			min(PAGE_SIZE, size - idx * PAGE_SIZE))) {
-			DRM_ERROR("Failed to copy user data (%p) to drm buffer"
-					" (%p) %dth page.\n",
-					user_data, buf, idx);
-			return -EFAULT;
-
-		}
-	}
-	buf->iterator = 0;
-	return 0;
-}
-
-/**
- * Free the drm buffer object
- */
-void drm_buffer_free(struct drm_buffer *buf)
-{
-
-	if (buf != NULL) {
-
-		int nr_pages = buf->size / PAGE_SIZE + 1;
-		int idx;
-		for (idx = 0; idx < nr_pages; ++idx)
-			kfree(buf->data[idx]);
-
-		kfree(buf);
-	}
-}
-
-/**
- * Read an object from buffer that may be split to multiple parts. If object
- * is not split function just returns the pointer to object in buffer. But in
- * case of split object data is copied to given stack object that is suplied
- * by caller.
- *
- * The processing location of the buffer is also advanced to the next byte
- * after the object.
- *
- *   objsize: The size of the objet in bytes.
- *   stack_obj: A pointer to a memory location where object can be copied.
- */
-void *drm_buffer_read_object(struct drm_buffer *buf,
-		int objsize, void *stack_obj)
-{
-	int idx = drm_buffer_index(buf);
-	int page = drm_buffer_page(buf);
-	void *obj = NULL;
-
-	if (idx + objsize <= PAGE_SIZE) {
-		obj = &buf->data[page][idx];
-	} else {
-		/* The object is split which forces copy to temporary object.*/
-		int beginsz = PAGE_SIZE - idx;
-		memcpy(stack_obj, &buf->data[page][idx], beginsz);
-
-		memcpy(stack_obj + beginsz, &buf->data[page + 1][0],
-				objsize - beginsz);
-
-		obj = stack_obj;
-	}
-
-	drm_buffer_advance(buf, objsize);
-	return obj;
-}
diff --git a/drivers/gpu/drm/radeon/drm_buffer.h b/drivers/gpu/drm/radeon/drm_buffer.h
deleted file mode 100644
index c80d3a340b94..000000000000
--- a/drivers/gpu/drm/radeon/drm_buffer.h
+++ /dev/null
@@ -1,148 +0,0 @@
-/**************************************************************************
- *
- * Copyright 2010 Pauli Nieminen.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- *
- **************************************************************************/
-/*
- * Multipart buffer for coping data which is larger than the page size.
- *
- * Authors:
- * Pauli Nieminen <suokkos-at-gmail-dot-com>
- */
-
-#ifndef _DRM_BUFFER_H_
-#define _DRM_BUFFER_H_
-
-#include <drm/drmP.h>
-
-struct drm_buffer {
-	int iterator;
-	int size;
-	char *data[];
-};
-
-
-/**
- * Return the index of page that buffer is currently pointing at.
- */
-static inline int drm_buffer_page(struct drm_buffer *buf)
-{
-	return buf->iterator / PAGE_SIZE;
-}
-/**
- * Return the index of the current byte in the page
- */
-static inline int drm_buffer_index(struct drm_buffer *buf)
-{
-	return buf->iterator & (PAGE_SIZE - 1);
-}
-/**
- * Return number of bytes that is left to process
- */
-static inline int drm_buffer_unprocessed(struct drm_buffer *buf)
-{
-	return buf->size - buf->iterator;
-}
-
-/**
- * Advance the buffer iterator number of bytes that is given.
- */
-static inline void drm_buffer_advance(struct drm_buffer *buf, int bytes)
-{
-	buf->iterator += bytes;
-}
-
-/**
- * Allocate the drm buffer object.
- *
- *   buf: A pointer to a pointer where the object is stored.
- *   size: The number of bytes to allocate.
- */
-extern int drm_buffer_alloc(struct drm_buffer **buf, int size);
-
-/**
- * Copy the user data to the begin of the buffer and reset the processing
- * iterator.
- *
- *   user_data: A pointer the data that is copied to the buffer.
- *   size: The Number of bytes to copy.
- */
-extern int drm_buffer_copy_from_user(struct drm_buffer *buf,
-		void __user *user_data, int size);
-
-/**
- * Free the drm buffer object
- */
-extern void drm_buffer_free(struct drm_buffer *buf);
-
-/**
- * Read an object from buffer that may be split to multiple parts. If object
- * is not split function just returns the pointer to object in buffer. But in
- * case of split object data is copied to given stack object that is suplied
- * by caller.
- *
- * The processing location of the buffer is also advanced to the next byte
- * after the object.
- *
- *   objsize: The size of the objet in bytes.
- *   stack_obj: A pointer to a memory location where object can be copied.
- */
-extern void *drm_buffer_read_object(struct drm_buffer *buf,
-		int objsize, void *stack_obj);
-
-/**
- * Returns the pointer to the dword which is offset number of elements from the
- * current processing location.
- *
- * Caller must make sure that dword is not split in the buffer. This
- * requirement is easily met if all the sizes of objects in buffer are
- * multiples of dword and PAGE_SIZE is multiple dword.
- *
- * Call to this function doesn't change the processing location.
- *
- *   offset: The index of the dword relative to the internat iterator.
- */
-static inline void *drm_buffer_pointer_to_dword(struct drm_buffer *buffer,
-		int offset)
-{
-	int iter = buffer->iterator + offset * 4;
-	return &buffer->data[iter / PAGE_SIZE][iter & (PAGE_SIZE - 1)];
-}
-/**
- * Returns the pointer to the dword which is offset number of elements from
- * the current processing location.
- *
- * Call to this function doesn't change the processing location.
- *
- *   offset: The index of the byte relative to the internat iterator.
- */
-static inline void *drm_buffer_pointer_to_byte(struct drm_buffer *buffer,
-		int offset)
-{
-	int iter = buffer->iterator + offset;
-	return &buffer->data[iter / PAGE_SIZE][iter & (PAGE_SIZE - 1)];
-}
-
-#endif
diff --git a/drivers/gpu/drm/radeon/evergreen_hdmi.c b/drivers/gpu/drm/radeon/evergreen_hdmi.c
index 9953356fe263..3cf04a2f44bb 100644
--- a/drivers/gpu/drm/radeon/evergreen_hdmi.c
+++ b/drivers/gpu/drm/radeon/evergreen_hdmi.c
@@ -289,6 +289,16 @@ void dce4_dp_audio_set_dto(struct radeon_device *rdev,
 	 * number (coefficient of two integer numbers.  DCCG_AUDIO_DTOx_PHASE
 	 * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator
 	 */
+	if (ASIC_IS_DCE41(rdev)) {
+		unsigned int div = (RREG32(DCE41_DENTIST_DISPCLK_CNTL) &
+			DENTIST_DPREFCLK_WDIVIDER_MASK) >>
+			DENTIST_DPREFCLK_WDIVIDER_SHIFT;
+		div = radeon_audio_decode_dfs_div(div);
+
+		if (div)
+			clock = 100 * clock / div;
+	}
+
 	WREG32(DCCG_AUDIO_DTO1_PHASE, 24000);
 	WREG32(DCCG_AUDIO_DTO1_MODULE, clock);
 }
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 4aa5f755572b..13b6029d65cc 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -511,6 +511,11 @@
 #define DCCG_AUDIO_DTO1_CNTL              0x05cc
 #       define DCCG_AUDIO_DTO1_USE_512FBR_DTO (1 << 3)
 
+#define DCE41_DENTIST_DISPCLK_CNTL			0x049c
+#       define DENTIST_DPREFCLK_WDIVIDER(x)		(((x) & 0x7f) << 24)
+#       define DENTIST_DPREFCLK_WDIVIDER_MASK		(0x7f << 24)
+#       define DENTIST_DPREFCLK_WDIVIDER_SHIFT		24
+
 /* DCE 4.0 AFMT */
 #define HDMI_CONTROL                         0x7030
 #       define HDMI_KEEPOUT_MODE             (1 << 0)
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 9e7e2bf03b81..5eae0a88dd3e 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -3150,7 +3150,8 @@ void r100_bandwidth_update(struct radeon_device *rdev)
 {
 	fixed20_12 trcd_ff, trp_ff, tras_ff, trbs_ff, tcas_ff;
 	fixed20_12 sclk_ff, mclk_ff, sclk_eff_ff, sclk_delay_ff;
-	fixed20_12 peak_disp_bw, mem_bw, pix_clk, pix_clk2, temp_ff, crit_point_ff;
+	fixed20_12 peak_disp_bw, mem_bw, pix_clk, pix_clk2, temp_ff;
+	fixed20_12 crit_point_ff = {0};
 	uint32_t temp, data, mem_trcd, mem_trp, mem_tras;
 	fixed20_12 memtcas_ff[8] = {
 		dfixed_init(1),
@@ -3204,7 +3205,7 @@ void r100_bandwidth_update(struct radeon_device *rdev)
 	fixed20_12 min_mem_eff;
 	fixed20_12 mc_latency_sclk, mc_latency_mclk, k1;
 	fixed20_12 cur_latency_mclk, cur_latency_sclk;
-	fixed20_12 disp_latency, disp_latency_overhead, disp_drain_rate,
+	fixed20_12 disp_latency, disp_latency_overhead, disp_drain_rate = {0},
 		disp_drain_rate2, read_return_rate;
 	fixed20_12 time_disp1_drop_priority;
 	int c;
diff --git a/drivers/gpu/drm/radeon/r300_cmdbuf.c b/drivers/gpu/drm/radeon/r300_cmdbuf.c
deleted file mode 100644
index 9418e388b045..000000000000
--- a/drivers/gpu/drm/radeon/r300_cmdbuf.c
+++ /dev/null
@@ -1,1186 +0,0 @@
-/* r300_cmdbuf.c -- Command buffer emission for R300 -*- linux-c -*-
- *
- * Copyright (C) The Weather Channel, Inc.  2002.
- * Copyright (C) 2004 Nicolai Haehnle.
- * All Rights Reserved.
- *
- * The Weather Channel (TM) funded Tungsten Graphics to develop the
- * initial release of the Radeon 8500 driver under the XFree86 license.
- * This notice must be preserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Nicolai Haehnle <prefect_@gmx.net>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-#include "r300_reg.h"
-#include "drm_buffer.h"
-
-#include <asm/unaligned.h>
-
-#define R300_SIMULTANEOUS_CLIPRECTS		4
-
-/* Values for R300_RE_CLIPRECT_CNTL depending on the number of cliprects
- */
-static const int r300_cliprect_cntl[4] = {
-	0xAAAA,
-	0xEEEE,
-	0xFEFE,
-	0xFFFE
-};
-
-/**
- * Emit up to R300_SIMULTANEOUS_CLIPRECTS cliprects from the given command
- * buffer, starting with index n.
- */
-static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
-			       drm_radeon_kcmd_buffer_t *cmdbuf, int n)
-{
-	struct drm_clip_rect box;
-	int nr;
-	int i;
-	RING_LOCALS;
-
-	nr = cmdbuf->nbox - n;
-	if (nr > R300_SIMULTANEOUS_CLIPRECTS)
-		nr = R300_SIMULTANEOUS_CLIPRECTS;
-
-	DRM_DEBUG("%i cliprects\n", nr);
-
-	if (nr) {
-		BEGIN_RING(6 + nr * 2);
-		OUT_RING(CP_PACKET0(R300_RE_CLIPRECT_TL_0, nr * 2 - 1));
-
-		for (i = 0; i < nr; ++i) {
-			if (copy_from_user
-			    (&box, &cmdbuf->boxes[n + i], sizeof(box))) {
-				DRM_ERROR("copy cliprect faulted\n");
-				return -EFAULT;
-			}
-
-			box.x2--; /* Hardware expects inclusive bottom-right corner */
-			box.y2--;
-
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
-				box.x1 = (box.x1) &
-					R300_CLIPRECT_MASK;
-				box.y1 = (box.y1) &
-					R300_CLIPRECT_MASK;
-				box.x2 = (box.x2) &
-					R300_CLIPRECT_MASK;
-				box.y2 = (box.y2) &
-					R300_CLIPRECT_MASK;
-			} else {
-				box.x1 = (box.x1 + R300_CLIPRECT_OFFSET) &
-					R300_CLIPRECT_MASK;
-				box.y1 = (box.y1 + R300_CLIPRECT_OFFSET) &
-					R300_CLIPRECT_MASK;
-				box.x2 = (box.x2 + R300_CLIPRECT_OFFSET) &
-					R300_CLIPRECT_MASK;
-				box.y2 = (box.y2 + R300_CLIPRECT_OFFSET) &
-					R300_CLIPRECT_MASK;
-			}
-
-			OUT_RING((box.x1 << R300_CLIPRECT_X_SHIFT) |
-				 (box.y1 << R300_CLIPRECT_Y_SHIFT));
-			OUT_RING((box.x2 << R300_CLIPRECT_X_SHIFT) |
-				 (box.y2 << R300_CLIPRECT_Y_SHIFT));
-
-		}
-
-		OUT_RING_REG(R300_RE_CLIPRECT_CNTL, r300_cliprect_cntl[nr - 1]);
-
-		/* TODO/SECURITY: Force scissors to a safe value, otherwise the
-		 * client might be able to trample over memory.
-		 * The impact should be very limited, but I'd rather be safe than
-		 * sorry.
-		 */
-		OUT_RING(CP_PACKET0(R300_RE_SCISSORS_TL, 1));
-		OUT_RING(0);
-		OUT_RING(R300_SCISSORS_X_MASK | R300_SCISSORS_Y_MASK);
-		ADVANCE_RING();
-	} else {
-		/* Why we allow zero cliprect rendering:
-		 * There are some commands in a command buffer that must be submitted
-		 * even when there are no cliprects, e.g. DMA buffer discard
-		 * or state setting (though state setting could be avoided by
-		 * simulating a loss of context).
-		 *
-		 * Now since the cmdbuf interface is so chaotic right now (and is
-		 * bound to remain that way for a bit until things settle down),
-		 * it is basically impossible to filter out the commands that are
-		 * necessary and those that aren't.
-		 *
-		 * So I choose the safe way and don't do any filtering at all;
-		 * instead, I simply set up the engine so that all rendering
-		 * can't produce any fragments.
-		 */
-		BEGIN_RING(2);
-		OUT_RING_REG(R300_RE_CLIPRECT_CNTL, 0);
-		ADVANCE_RING();
-	}
-
-	/* flus cache and wait idle clean after cliprect change */
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(R300_RB3D_DC_FLUSH);
-	ADVANCE_RING();
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
-	ADVANCE_RING();
-	/* set flush flag */
-	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
-
-	return 0;
-}
-
-static u8 r300_reg_flags[0x10000 >> 2];
-
-void r300_init_reg_flags(struct drm_device *dev)
-{
-	int i;
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	memset(r300_reg_flags, 0, 0x10000 >> 2);
-#define ADD_RANGE_MARK(reg, count,mark) \
-		for(i=((reg)>>2);i<((reg)>>2)+(count);i++)\
-			r300_reg_flags[i]|=(mark);
-
-#define MARK_SAFE		1
-#define MARK_CHECK_OFFSET	2
-
-#define ADD_RANGE(reg, count)	ADD_RANGE_MARK(reg, count, MARK_SAFE)
-
-	/* these match cmducs() command in r300_driver/r300/r300_cmdbuf.c */
-	ADD_RANGE(R300_SE_VPORT_XSCALE, 6);
-	ADD_RANGE(R300_VAP_CNTL, 1);
-	ADD_RANGE(R300_SE_VTE_CNTL, 2);
-	ADD_RANGE(0x2134, 2);
-	ADD_RANGE(R300_VAP_CNTL_STATUS, 1);
-	ADD_RANGE(R300_VAP_INPUT_CNTL_0, 2);
-	ADD_RANGE(0x21DC, 1);
-	ADD_RANGE(R300_VAP_UNKNOWN_221C, 1);
-	ADD_RANGE(R300_VAP_CLIP_X_0, 4);
-	ADD_RANGE(R300_VAP_PVS_STATE_FLUSH_REG, 1);
-	ADD_RANGE(R300_VAP_UNKNOWN_2288, 1);
-	ADD_RANGE(R300_VAP_OUTPUT_VTX_FMT_0, 2);
-	ADD_RANGE(R300_VAP_PVS_CNTL_1, 3);
-	ADD_RANGE(R300_GB_ENABLE, 1);
-	ADD_RANGE(R300_GB_MSPOS0, 5);
-	ADD_RANGE(R300_TX_INVALTAGS, 1);
-	ADD_RANGE(R300_TX_ENABLE, 1);
-	ADD_RANGE(0x4200, 4);
-	ADD_RANGE(0x4214, 1);
-	ADD_RANGE(R300_RE_POINTSIZE, 1);
-	ADD_RANGE(0x4230, 3);
-	ADD_RANGE(R300_RE_LINE_CNT, 1);
-	ADD_RANGE(R300_RE_UNK4238, 1);
-	ADD_RANGE(0x4260, 3);
-	ADD_RANGE(R300_RE_SHADE, 4);
-	ADD_RANGE(R300_RE_POLYGON_MODE, 5);
-	ADD_RANGE(R300_RE_ZBIAS_CNTL, 1);
-	ADD_RANGE(R300_RE_ZBIAS_T_FACTOR, 4);
-	ADD_RANGE(R300_RE_OCCLUSION_CNTL, 1);
-	ADD_RANGE(R300_RE_CULL_CNTL, 1);
-	ADD_RANGE(0x42C0, 2);
-	ADD_RANGE(R300_RS_CNTL_0, 2);
-
-	ADD_RANGE(R300_SU_REG_DEST, 1);
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530)
-		ADD_RANGE(RV530_FG_ZBREG_DEST, 1);
-
-	ADD_RANGE(R300_SC_HYPERZ, 2);
-	ADD_RANGE(0x43E8, 1);
-
-	ADD_RANGE(0x46A4, 5);
-
-	ADD_RANGE(R300_RE_FOG_STATE, 1);
-	ADD_RANGE(R300_FOG_COLOR_R, 3);
-	ADD_RANGE(R300_PP_ALPHA_TEST, 2);
-	ADD_RANGE(0x4BD8, 1);
-	ADD_RANGE(R300_PFS_PARAM_0_X, 64);
-	ADD_RANGE(0x4E00, 1);
-	ADD_RANGE(R300_RB3D_CBLEND, 2);
-	ADD_RANGE(R300_RB3D_COLORMASK, 1);
-	ADD_RANGE(R300_RB3D_BLEND_COLOR, 3);
-	ADD_RANGE_MARK(R300_RB3D_COLOROFFSET0, 1, MARK_CHECK_OFFSET);	/* check offset */
-	ADD_RANGE(R300_RB3D_COLORPITCH0, 1);
-	ADD_RANGE(0x4E50, 9);
-	ADD_RANGE(0x4E88, 1);
-	ADD_RANGE(0x4EA0, 2);
-	ADD_RANGE(R300_ZB_CNTL, 3);
-	ADD_RANGE(R300_ZB_FORMAT, 4);
-	ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);	/* check offset */
-	ADD_RANGE(R300_ZB_DEPTHPITCH, 1);
-	ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1);
-	ADD_RANGE(R300_ZB_ZMASK_OFFSET, 13);
-	ADD_RANGE(R300_ZB_ZPASS_DATA, 2); /* ZB_ZPASS_DATA, ZB_ZPASS_ADDR */
-
-	ADD_RANGE(R300_TX_FILTER_0, 16);
-	ADD_RANGE(R300_TX_FILTER1_0, 16);
-	ADD_RANGE(R300_TX_SIZE_0, 16);
-	ADD_RANGE(R300_TX_FORMAT_0, 16);
-	ADD_RANGE(R300_TX_PITCH_0, 16);
-	/* Texture offset is dangerous and needs more checking */
-	ADD_RANGE_MARK(R300_TX_OFFSET_0, 16, MARK_CHECK_OFFSET);
-	ADD_RANGE(R300_TX_CHROMA_KEY_0, 16);
-	ADD_RANGE(R300_TX_BORDER_COLOR_0, 16);
-
-	/* Sporadic registers used as primitives are emitted */
-	ADD_RANGE(R300_ZB_ZCACHE_CTLSTAT, 1);
-	ADD_RANGE(R300_RB3D_DSTCACHE_CTLSTAT, 1);
-	ADD_RANGE(R300_VAP_INPUT_ROUTE_0_0, 8);
-	ADD_RANGE(R300_VAP_INPUT_ROUTE_1_0, 8);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
-		ADD_RANGE(R500_VAP_INDEX_OFFSET, 1);
-		ADD_RANGE(R500_US_CONFIG, 2);
-		ADD_RANGE(R500_US_CODE_ADDR, 3);
-		ADD_RANGE(R500_US_FC_CTRL, 1);
-		ADD_RANGE(R500_RS_IP_0, 16);
-		ADD_RANGE(R500_RS_INST_0, 16);
-		ADD_RANGE(R500_RB3D_COLOR_CLEAR_VALUE_AR, 2);
-		ADD_RANGE(R500_RB3D_CONSTANT_COLOR_AR, 2);
-		ADD_RANGE(R500_ZB_FIFO_SIZE, 2);
-	} else {
-		ADD_RANGE(R300_PFS_CNTL_0, 3);
-		ADD_RANGE(R300_PFS_NODE_0, 4);
-		ADD_RANGE(R300_PFS_TEXI_0, 64);
-		ADD_RANGE(R300_PFS_INSTR0_0, 64);
-		ADD_RANGE(R300_PFS_INSTR1_0, 64);
-		ADD_RANGE(R300_PFS_INSTR2_0, 64);
-		ADD_RANGE(R300_PFS_INSTR3_0, 64);
-		ADD_RANGE(R300_RS_INTERP_0, 8);
-		ADD_RANGE(R300_RS_ROUTE_0, 8);
-
-	}
-}
-
-static __inline__ int r300_check_range(unsigned reg, int count)
-{
-	int i;
-	if (reg & ~0xffff)
-		return -1;
-	for (i = (reg >> 2); i < (reg >> 2) + count; i++)
-		if (r300_reg_flags[i] != MARK_SAFE)
-			return 1;
-	return 0;
-}
-
-static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
-							  dev_priv,
-							  drm_radeon_kcmd_buffer_t
-							  * cmdbuf,
-							  drm_r300_cmd_header_t
-							  header)
-{
-	int reg;
-	int sz;
-	int i;
-	u32 *value;
-	RING_LOCALS;
-
-	sz = header.packet0.count;
-	reg = (header.packet0.reghi << 8) | header.packet0.reglo;
-
-	if ((sz > 64) || (sz < 0)) {
-		DRM_ERROR("Cannot emit more than 64 values at a time (reg=%04x sz=%d)\n",
-			 reg, sz);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < sz; i++) {
-		switch (r300_reg_flags[(reg >> 2) + i]) {
-		case MARK_SAFE:
-			break;
-		case MARK_CHECK_OFFSET:
-			value = drm_buffer_pointer_to_dword(cmdbuf->buffer, i);
-			if (!radeon_check_offset(dev_priv, *value)) {
-				DRM_ERROR("Offset failed range check (reg=%04x sz=%d)\n",
-					 reg, sz);
-				return -EINVAL;
-			}
-			break;
-		default:
-			DRM_ERROR("Register %04x failed check as flag=%02x\n",
-				reg + i * 4, r300_reg_flags[(reg >> 2) + i]);
-			return -EINVAL;
-		}
-	}
-
-	BEGIN_RING(1 + sz);
-	OUT_RING(CP_PACKET0(reg, sz - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-/**
- * Emits a packet0 setting arbitrary registers.
- * Called by r300_do_cp_cmdbuf.
- *
- * Note that checks are performed on contents and addresses of the registers
- */
-static __inline__ int r300_emit_packet0(drm_radeon_private_t *dev_priv,
-					drm_radeon_kcmd_buffer_t *cmdbuf,
-					drm_r300_cmd_header_t header)
-{
-	int reg;
-	int sz;
-	RING_LOCALS;
-
-	sz = header.packet0.count;
-	reg = (header.packet0.reghi << 8) | header.packet0.reglo;
-
-	if (!sz)
-		return 0;
-
-	if (sz * 4 > drm_buffer_unprocessed(cmdbuf->buffer))
-		return -EINVAL;
-
-	if (reg + sz * 4 >= 0x10000) {
-		DRM_ERROR("No such registers in hardware reg=%04x sz=%d\n", reg,
-			  sz);
-		return -EINVAL;
-	}
-
-	if (r300_check_range(reg, sz)) {
-		/* go and check everything */
-		return r300_emit_carefully_checked_packet0(dev_priv, cmdbuf,
-							   header);
-	}
-	/* the rest of the data is safe to emit, whatever the values the user passed */
-
-	BEGIN_RING(1 + sz);
-	OUT_RING(CP_PACKET0(reg, sz - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-/**
- * Uploads user-supplied vertex program instructions or parameters onto
- * the graphics card.
- * Called by r300_do_cp_cmdbuf.
- */
-static __inline__ int r300_emit_vpu(drm_radeon_private_t *dev_priv,
-				    drm_radeon_kcmd_buffer_t *cmdbuf,
-				    drm_r300_cmd_header_t header)
-{
-	int sz;
-	int addr;
-	RING_LOCALS;
-
-	sz = header.vpu.count;
-	addr = (header.vpu.adrhi << 8) | header.vpu.adrlo;
-
-	if (!sz)
-		return 0;
-	if (sz * 16 > drm_buffer_unprocessed(cmdbuf->buffer))
-		return -EINVAL;
-
-	/* VAP is very sensitive so we purge cache before we program it
-	 * and we also flush its state before & after */
-	BEGIN_RING(6);
-	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(R300_RB3D_DC_FLUSH);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
-	OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
-	OUT_RING(0);
-	ADVANCE_RING();
-	/* set flush flag */
-	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
-
-	BEGIN_RING(3 + sz * 4);
-	OUT_RING_REG(R300_VAP_PVS_UPLOAD_ADDRESS, addr);
-	OUT_RING(CP_PACKET0_TABLE(R300_VAP_PVS_UPLOAD_DATA, sz * 4 - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz * 4);
-	ADVANCE_RING();
-
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
-	OUT_RING(0);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-/**
- * Emit a clear packet from userspace.
- * Called by r300_emit_packet3.
- */
-static __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv,
-				      drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	RING_LOCALS;
-
-	if (8 * 4 > drm_buffer_unprocessed(cmdbuf->buffer))
-		return -EINVAL;
-
-	BEGIN_RING(10);
-	OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 8));
-	OUT_RING(R300_PRIM_TYPE_POINT | R300_PRIM_WALK_RING |
-		 (1 << R300_PRIM_NUM_VERTICES_SHIFT));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, 8);
-	ADVANCE_RING();
-
-	BEGIN_RING(4);
-	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(R300_RB3D_DC_FLUSH);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
-	ADVANCE_RING();
-	/* set flush flag */
-	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
-
-	return 0;
-}
-
-static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
-					       drm_radeon_kcmd_buffer_t *cmdbuf,
-					       u32 header)
-{
-	int count, i, k;
-#define MAX_ARRAY_PACKET  64
-	u32 *data;
-	u32 narrays;
-	RING_LOCALS;
-
-	count = (header & RADEON_CP_PACKET_COUNT_MASK) >> 16;
-
-	if ((count + 1) > MAX_ARRAY_PACKET) {
-		DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
-			  count);
-		return -EINVAL;
-	}
-	/* carefully check packet contents */
-
-	/* We have already read the header so advance the buffer. */
-	drm_buffer_advance(cmdbuf->buffer, 4);
-
-	narrays = *(u32 *)drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-	k = 0;
-	i = 1;
-	while ((k < narrays) && (i < (count + 1))) {
-		i++;		/* skip attribute field */
-		data = drm_buffer_pointer_to_dword(cmdbuf->buffer, i);
-		if (!radeon_check_offset(dev_priv, *data)) {
-			DRM_ERROR
-			    ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
-			     k, i);
-			return -EINVAL;
-		}
-		k++;
-		i++;
-		if (k == narrays)
-			break;
-		/* have one more to process, they come in pairs */
-		data = drm_buffer_pointer_to_dword(cmdbuf->buffer, i);
-		if (!radeon_check_offset(dev_priv, *data)) {
-			DRM_ERROR
-			    ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
-			     k, i);
-			return -EINVAL;
-		}
-		k++;
-		i++;
-	}
-	/* do the counts match what we expect ? */
-	if ((k != narrays) || (i != (count + 1))) {
-		DRM_ERROR
-		    ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
-		     k, i, narrays, count + 1);
-		return -EINVAL;
-	}
-
-	/* all clear, output packet */
-
-	BEGIN_RING(count + 2);
-	OUT_RING(header);
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 1);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv,
-					     drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	u32 *cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-	int count, ret;
-	RING_LOCALS;
-
-
-	count = (*cmd & RADEON_CP_PACKET_COUNT_MASK) >> 16;
-
-	if (*cmd & 0x8000) {
-		u32 offset;
-		u32 *cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-		if (*cmd1 & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
-			      | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
-
-			u32 *cmd2 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2);
-			offset = *cmd2 << 10;
-			ret = !radeon_check_offset(dev_priv, offset);
-			if (ret) {
-				DRM_ERROR("Invalid bitblt first offset is %08X\n", offset);
-				return -EINVAL;
-			}
-		}
-
-		if ((*cmd1 & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
-		    (*cmd1 & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
-			u32 *cmd3 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 3);
-			offset = *cmd3 << 10;
-			ret = !radeon_check_offset(dev_priv, offset);
-			if (ret) {
-				DRM_ERROR("Invalid bitblt second offset is %08X\n", offset);
-				return -EINVAL;
-			}
-
-		}
-	}
-
-	BEGIN_RING(count+2);
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static __inline__ int r300_emit_draw_indx_2(drm_radeon_private_t *dev_priv,
-					    drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	u32 *cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-	u32 *cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-	int count;
-	int expected_count;
-	RING_LOCALS;
-
-	count = (*cmd & RADEON_CP_PACKET_COUNT_MASK) >> 16;
-
-	expected_count = *cmd1 >> 16;
-	if (!(*cmd1 & R300_VAP_VF_CNTL__INDEX_SIZE_32bit))
-		expected_count = (expected_count+1)/2;
-
-	if (count && count != expected_count) {
-		DRM_ERROR("3D_DRAW_INDX_2: packet size %i, expected %i\n",
-			count, expected_count);
-		return -EINVAL;
-	}
-
-	BEGIN_RING(count+2);
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2);
-	ADVANCE_RING();
-
-	if (!count) {
-		drm_r300_cmd_header_t stack_header, *header;
-		u32 *cmd1, *cmd2, *cmd3;
-
-		if (drm_buffer_unprocessed(cmdbuf->buffer)
-				< 4*4 + sizeof(stack_header)) {
-			DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER, but stream is too short.\n");
-			return -EINVAL;
-		}
-
-		header = drm_buffer_read_object(cmdbuf->buffer,
-				sizeof(stack_header), &stack_header);
-
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-		cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-		cmd2 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2);
-		cmd3 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 3);
-
-		if (header->header.cmd_type != R300_CMD_PACKET3 ||
-		    header->packet3.packet != R300_CMD_PACKET3_RAW ||
-		    *cmd != CP_PACKET3(RADEON_CP_INDX_BUFFER, 2)) {
-			DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER.\n");
-			return -EINVAL;
-		}
-
-		if ((*cmd1 & 0x8000ffff) != 0x80000810) {
-			DRM_ERROR("Invalid indx_buffer reg address %08X\n",
-					*cmd1);
-			return -EINVAL;
-		}
-		if (!radeon_check_offset(dev_priv, *cmd2)) {
-			DRM_ERROR("Invalid indx_buffer offset is %08X\n",
-					*cmd2);
-			return -EINVAL;
-		}
-		if (*cmd3 != expected_count) {
-			DRM_ERROR("INDX_BUFFER: buffer size %i, expected %i\n",
-				*cmd3, expected_count);
-			return -EINVAL;
-		}
-
-		BEGIN_RING(4);
-		OUT_RING_DRM_BUFFER(cmdbuf->buffer, 4);
-		ADVANCE_RING();
-	}
-
-	return 0;
-}
-
-static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
-					    drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	u32 *header;
-	int count;
-	RING_LOCALS;
-
-	if (4 > drm_buffer_unprocessed(cmdbuf->buffer))
-		return -EINVAL;
-
-	/* Fixme !! This simply emits a packet without much checking.
-	   We need to be smarter. */
-
-	/* obtain first word - actual packet3 header */
-	header = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-
-	/* Is it packet 3 ? */
-	if ((*header >> 30) != 0x3) {
-		DRM_ERROR("Not a packet3 header (0x%08x)\n", *header);
-		return -EINVAL;
-	}
-
-	count = (*header >> 16) & 0x3fff;
-
-	/* Check again now that we know how much data to expect */
-	if ((count + 2) * 4 > drm_buffer_unprocessed(cmdbuf->buffer)) {
-		DRM_ERROR
-		    ("Expected packet3 of length %d but have only %d bytes left\n",
-		     (count + 2) * 4, drm_buffer_unprocessed(cmdbuf->buffer));
-		return -EINVAL;
-	}
-
-	/* Is it a packet type we know about ? */
-	switch (*header & 0xff00) {
-	case RADEON_3D_LOAD_VBPNTR:	/* load vertex array pointers */
-		return r300_emit_3d_load_vbpntr(dev_priv, cmdbuf, *header);
-
-	case RADEON_CNTL_BITBLT_MULTI:
-		return r300_emit_bitblt_multi(dev_priv, cmdbuf);
-
-	case RADEON_CP_INDX_BUFFER:
-		DRM_ERROR("packet3 INDX_BUFFER without preceding 3D_DRAW_INDX_2 is illegal.\n");
-		return -EINVAL;
-	case RADEON_CP_3D_DRAW_IMMD_2:
-		/* triggers drawing using in-packet vertex data */
-	case RADEON_CP_3D_DRAW_VBUF_2:
-		/* triggers drawing of vertex buffers setup elsewhere */
-		dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
-					   RADEON_PURGE_EMITED);
-		break;
-	case RADEON_CP_3D_DRAW_INDX_2:
-		/* triggers drawing using indices to vertex buffer */
-		/* whenever we send vertex we clear flush & purge */
-		dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
-					   RADEON_PURGE_EMITED);
-		return r300_emit_draw_indx_2(dev_priv, cmdbuf);
-	case RADEON_WAIT_FOR_IDLE:
-	case RADEON_CP_NOP:
-		/* these packets are safe */
-		break;
-	default:
-		DRM_ERROR("Unknown packet3 header (0x%08x)\n", *header);
-		return -EINVAL;
-	}
-
-	BEGIN_RING(count + 2);
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-/**
- * Emit a rendering packet3 from userspace.
- * Called by r300_do_cp_cmdbuf.
- */
-static __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv,
-					drm_radeon_kcmd_buffer_t *cmdbuf,
-					drm_r300_cmd_header_t header)
-{
-	int n;
-	int ret;
-	int orig_iter = cmdbuf->buffer->iterator;
-
-	/* This is a do-while-loop so that we run the interior at least once,
-	 * even if cmdbuf->nbox is 0. Compare r300_emit_cliprects for rationale.
-	 */
-	n = 0;
-	do {
-		if (cmdbuf->nbox > R300_SIMULTANEOUS_CLIPRECTS) {
-			ret = r300_emit_cliprects(dev_priv, cmdbuf, n);
-			if (ret)
-				return ret;
-
-			cmdbuf->buffer->iterator = orig_iter;
-		}
-
-		switch (header.packet3.packet) {
-		case R300_CMD_PACKET3_CLEAR:
-			DRM_DEBUG("R300_CMD_PACKET3_CLEAR\n");
-			ret = r300_emit_clear(dev_priv, cmdbuf);
-			if (ret) {
-				DRM_ERROR("r300_emit_clear failed\n");
-				return ret;
-			}
-			break;
-
-		case R300_CMD_PACKET3_RAW:
-			DRM_DEBUG("R300_CMD_PACKET3_RAW\n");
-			ret = r300_emit_raw_packet3(dev_priv, cmdbuf);
-			if (ret) {
-				DRM_ERROR("r300_emit_raw_packet3 failed\n");
-				return ret;
-			}
-			break;
-
-		default:
-			DRM_ERROR("bad packet3 type %i at byte %d\n",
-				  header.packet3.packet,
-				  cmdbuf->buffer->iterator - (int)sizeof(header));
-			return -EINVAL;
-		}
-
-		n += R300_SIMULTANEOUS_CLIPRECTS;
-	} while (n < cmdbuf->nbox);
-
-	return 0;
-}
-
-/* Some of the R300 chips seem to be extremely touchy about the two registers
- * that are configured in r300_pacify.
- * Among the worst offenders seems to be the R300 ND (0x4E44): When userspace
- * sends a command buffer that contains only state setting commands and a
- * vertex program/parameter upload sequence, this will eventually lead to a
- * lockup, unless the sequence is bracketed by calls to r300_pacify.
- * So we should take great care to *always* call r300_pacify before
- * *anything* 3D related, and again afterwards. This is what the
- * call bracket in r300_do_cp_cmdbuf is for.
- */
-
-/**
- * Emit the sequence to pacify R300.
- */
-static void r300_pacify(drm_radeon_private_t *dev_priv)
-{
-	uint32_t cache_z, cache_3d, cache_2d;
-	RING_LOCALS;
-
-	cache_z = R300_ZC_FLUSH;
-	cache_2d = R300_RB2D_DC_FLUSH;
-	cache_3d = R300_RB3D_DC_FLUSH;
-	if (!(dev_priv->track_flush & RADEON_PURGE_EMITED)) {
-		/* we can purge, primitive where draw since last purge */
-		cache_z |= R300_ZC_FREE;
-		cache_2d |= R300_RB2D_DC_FREE;
-		cache_3d |= R300_RB3D_DC_FREE;
-	}
-
-	/* flush & purge zbuffer */
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));
-	OUT_RING(cache_z);
-	ADVANCE_RING();
-	/* flush & purge 3d */
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(cache_3d);
-	ADVANCE_RING();
-	/* flush & purge texture */
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_TX_INVALTAGS, 0));
-	OUT_RING(0);
-	ADVANCE_RING();
-	/* FIXME: is this one really needed ? */
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(R300_RB3D_AARESOLVE_CTL, 0));
-	OUT_RING(0);
-	ADVANCE_RING();
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
-	ADVANCE_RING();
-	/* flush & purge 2d through E2 as RB2D will trigger lockup */
-	BEGIN_RING(4);
-	OUT_RING(CP_PACKET0(R300_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(cache_2d);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(RADEON_WAIT_2D_IDLECLEAN |
-		 RADEON_WAIT_HOST_IDLECLEAN);
-	ADVANCE_RING();
-	/* set flush & purge flags */
-	dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
-}
-
-/**
- * Called by r300_do_cp_cmdbuf to update the internal buffer age and state.
- * The actual age emit is done by r300_do_cp_cmdbuf, which is why you must
- * be careful about how this function is called.
- */
-static void r300_discard_buffer(struct drm_device *dev, struct drm_master *master, struct drm_buf *buf)
-{
-	drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-
-	buf_priv->age = ++master_priv->sarea_priv->last_dispatch;
-	buf->pending = 1;
-	buf->used = 0;
-}
-
-static void r300_cmd_wait(drm_radeon_private_t * dev_priv,
-			  drm_r300_cmd_header_t header)
-{
-	u32 wait_until;
-	RING_LOCALS;
-
-	if (!header.wait.flags)
-		return;
-
-	wait_until = 0;
-
-	switch(header.wait.flags) {
-	case R300_WAIT_2D:
-		wait_until = RADEON_WAIT_2D_IDLE;
-		break;
-	case R300_WAIT_3D:
-		wait_until = RADEON_WAIT_3D_IDLE;
-		break;
-	case R300_NEW_WAIT_2D_3D:
-		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_3D_IDLE;
-		break;
-	case R300_NEW_WAIT_2D_2D_CLEAN:
-		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
-		break;
-	case R300_NEW_WAIT_3D_3D_CLEAN:
-		wait_until = RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
-		break;
-	case R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN:
-		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
-		wait_until |= RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
-		break;
-	default:
-		return;
-	}
-
-	BEGIN_RING(2);
-	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-	OUT_RING(wait_until);
-	ADVANCE_RING();
-}
-
-static int r300_scratch(drm_radeon_private_t *dev_priv,
-			drm_radeon_kcmd_buffer_t *cmdbuf,
-			drm_r300_cmd_header_t header)
-{
-	u32 *ref_age_base;
-	u32 i, *buf_idx, h_pending;
-	u64 *ptr_addr;
-	u64 stack_ptr_addr;
-	RING_LOCALS;
-
-	if (drm_buffer_unprocessed(cmdbuf->buffer) <
-	    (sizeof(u64) + header.scratch.n_bufs * sizeof(*buf_idx))) {
-		return -EINVAL;
-	}
-
-	if (header.scratch.reg >= 5) {
-		return -EINVAL;
-	}
-
-	dev_priv->scratch_ages[header.scratch.reg]++;
-
-	ptr_addr = drm_buffer_read_object(cmdbuf->buffer,
-			sizeof(stack_ptr_addr), &stack_ptr_addr);
-	ref_age_base = (u32 *)(unsigned long)get_unaligned(ptr_addr);
-
-	for (i=0; i < header.scratch.n_bufs; i++) {
-		buf_idx = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-		*buf_idx *= 2; /* 8 bytes per buf */
-
-		if (copy_to_user(ref_age_base + *buf_idx,
-				&dev_priv->scratch_ages[header.scratch.reg],
-				sizeof(u32)))
-			return -EINVAL;
-
-		if (copy_from_user(&h_pending,
-				ref_age_base + *buf_idx + 1,
-				sizeof(u32)))
-			return -EINVAL;
-
-		if (h_pending == 0)
-			return -EINVAL;
-
-		h_pending--;
-
-		if (copy_to_user(ref_age_base + *buf_idx + 1,
-					&h_pending,
-					sizeof(u32)))
-			return -EINVAL;
-
-		drm_buffer_advance(cmdbuf->buffer, sizeof(*buf_idx));
-	}
-
-	BEGIN_RING(2);
-	OUT_RING( CP_PACKET0( RADEON_SCRATCH_REG0 + header.scratch.reg * 4, 0 ) );
-	OUT_RING( dev_priv->scratch_ages[header.scratch.reg] );
-	ADVANCE_RING();
-
-	return 0;
-}
-
-/**
- * Uploads user-supplied vertex program instructions or parameters onto
- * the graphics card.
- * Called by r300_do_cp_cmdbuf.
- */
-static inline int r300_emit_r500fp(drm_radeon_private_t *dev_priv,
-				       drm_radeon_kcmd_buffer_t *cmdbuf,
-				       drm_r300_cmd_header_t header)
-{
-	int sz;
-	int addr;
-	int type;
-	int isclamp;
-	int stride;
-	RING_LOCALS;
-
-	sz = header.r500fp.count;
-	/* address is 9 bits 0 - 8, bit 1 of flags is part of address */
-	addr = ((header.r500fp.adrhi_flags & 1) << 8) | header.r500fp.adrlo;
-
-	type = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_TYPE);
-	isclamp = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_CLAMP);
-
-	addr |= (type << 16);
-	addr |= (isclamp << 17);
-
-	stride = type ? 4 : 6;
-
-	DRM_DEBUG("r500fp %d %d type: %d\n", sz, addr, type);
-	if (!sz)
-		return 0;
-	if (sz * stride * 4 > drm_buffer_unprocessed(cmdbuf->buffer))
-		return -EINVAL;
-
-	BEGIN_RING(3 + sz * stride);
-	OUT_RING_REG(R500_GA_US_VECTOR_INDEX, addr);
-	OUT_RING(CP_PACKET0_TABLE(R500_GA_US_VECTOR_DATA, sz * stride - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz * stride);
-
-	ADVANCE_RING();
-
-	return 0;
-}
-
-
-/**
- * Parses and validates a user-supplied command buffer and emits appropriate
- * commands on the DMA ring buffer.
- * Called by the ioctl handler function radeon_cp_cmdbuf.
- */
-int r300_do_cp_cmdbuf(struct drm_device *dev,
-		      struct drm_file *file_priv,
-		      drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf = NULL;
-	int emit_dispatch_age = 0;
-	int ret = 0;
-
-	DRM_DEBUG("\n");
-
-	/* pacify */
-	r300_pacify(dev_priv);
-
-	if (cmdbuf->nbox <= R300_SIMULTANEOUS_CLIPRECTS) {
-		ret = r300_emit_cliprects(dev_priv, cmdbuf, 0);
-		if (ret)
-			goto cleanup;
-	}
-
-	while (drm_buffer_unprocessed(cmdbuf->buffer)
-			>= sizeof(drm_r300_cmd_header_t)) {
-		int idx;
-		drm_r300_cmd_header_t *header, stack_header;
-
-		header = drm_buffer_read_object(cmdbuf->buffer,
-				sizeof(stack_header), &stack_header);
-
-		switch (header->header.cmd_type) {
-		case R300_CMD_PACKET0:
-			DRM_DEBUG("R300_CMD_PACKET0\n");
-			ret = r300_emit_packet0(dev_priv, cmdbuf, *header);
-			if (ret) {
-				DRM_ERROR("r300_emit_packet0 failed\n");
-				goto cleanup;
-			}
-			break;
-
-		case R300_CMD_VPU:
-			DRM_DEBUG("R300_CMD_VPU\n");
-			ret = r300_emit_vpu(dev_priv, cmdbuf, *header);
-			if (ret) {
-				DRM_ERROR("r300_emit_vpu failed\n");
-				goto cleanup;
-			}
-			break;
-
-		case R300_CMD_PACKET3:
-			DRM_DEBUG("R300_CMD_PACKET3\n");
-			ret = r300_emit_packet3(dev_priv, cmdbuf, *header);
-			if (ret) {
-				DRM_ERROR("r300_emit_packet3 failed\n");
-				goto cleanup;
-			}
-			break;
-
-		case R300_CMD_END3D:
-			DRM_DEBUG("R300_CMD_END3D\n");
-			/* TODO:
-			   Ideally userspace driver should not need to issue this call,
-			   i.e. the drm driver should issue it automatically and prevent
-			   lockups.
-
-			   In practice, we do not understand why this call is needed and what
-			   it does (except for some vague guesses that it has to do with cache
-			   coherence) and so the user space driver does it.
-
-			   Once we are sure which uses prevent lockups the code could be moved
-			   into the kernel and the userspace driver will not
-			   need to use this command.
-
-			   Note that issuing this command does not hurt anything
-			   except, possibly, performance */
-			r300_pacify(dev_priv);
-			break;
-
-		case R300_CMD_CP_DELAY:
-			/* simple enough, we can do it here */
-			DRM_DEBUG("R300_CMD_CP_DELAY\n");
-			{
-				int i;
-				RING_LOCALS;
-
-				BEGIN_RING(header->delay.count);
-				for (i = 0; i < header->delay.count; i++)
-					OUT_RING(RADEON_CP_PACKET2);
-				ADVANCE_RING();
-			}
-			break;
-
-		case R300_CMD_DMA_DISCARD:
-			DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
-			idx = header->dma.buf_idx;
-			if (idx < 0 || idx >= dma->buf_count) {
-				DRM_ERROR("buffer index %d (of %d max)\n",
-					  idx, dma->buf_count - 1);
-				ret = -EINVAL;
-				goto cleanup;
-			}
-
-			buf = dma->buflist[idx];
-			if (buf->file_priv != file_priv || buf->pending) {
-				DRM_ERROR("bad buffer %p %p %d\n",
-					  buf->file_priv, file_priv,
-					  buf->pending);
-				ret = -EINVAL;
-				goto cleanup;
-			}
-
-			emit_dispatch_age = 1;
-			r300_discard_buffer(dev, file_priv->master, buf);
-			break;
-
-		case R300_CMD_WAIT:
-			DRM_DEBUG("R300_CMD_WAIT\n");
-			r300_cmd_wait(dev_priv, *header);
-			break;
-
-		case R300_CMD_SCRATCH:
-			DRM_DEBUG("R300_CMD_SCRATCH\n");
-			ret = r300_scratch(dev_priv, cmdbuf, *header);
-			if (ret) {
-				DRM_ERROR("r300_scratch failed\n");
-				goto cleanup;
-			}
-			break;
-
-		case R300_CMD_R500FP:
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV515) {
-				DRM_ERROR("Calling r500 command on r300 card\n");
-				ret = -EINVAL;
-				goto cleanup;
-			}
-			DRM_DEBUG("R300_CMD_R500FP\n");
-			ret = r300_emit_r500fp(dev_priv, cmdbuf, *header);
-			if (ret) {
-				DRM_ERROR("r300_emit_r500fp failed\n");
-				goto cleanup;
-			}
-			break;
-		default:
-			DRM_ERROR("bad cmd_type %i at byte %d\n",
-				  header->header.cmd_type,
-				  cmdbuf->buffer->iterator - (int)sizeof(*header));
-			ret = -EINVAL;
-			goto cleanup;
-		}
-	}
-
-	DRM_DEBUG("END\n");
-
-      cleanup:
-	r300_pacify(dev_priv);
-
-	/* We emit the vertex buffer age here, outside the pacifier "brackets"
-	 * for two reasons:
-	 *  (1) This may coalesce multiple age emissions into a single one and
-	 *  (2) more importantly, some chips lock up hard when scratch registers
-	 *      are written inside the pacifier bracket.
-	 */
-	if (emit_dispatch_age) {
-		RING_LOCALS;
-
-		/* Emit the vertex buffer age */
-		BEGIN_RING(2);
-		RADEON_DISPATCH_AGE(master_priv->sarea_priv->last_dispatch);
-		ADVANCE_RING();
-	}
-
-	COMMIT_RING();
-
-	return ret;
-}
diff --git a/drivers/gpu/drm/radeon/r600_blit.c b/drivers/gpu/drm/radeon/r600_blit.c
deleted file mode 100644
index daf7572be976..000000000000
--- a/drivers/gpu/drm/radeon/r600_blit.c
+++ /dev/null
@@ -1,874 +0,0 @@
-/*
- * Copyright 2009 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *     Alex Deucher <alexander.deucher@amd.com>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-
-#include "r600_blit_shaders.h"
-
-/* 23 bits of float fractional data */
-#define I2F_FRAC_BITS  23
-#define I2F_MASK ((1 << I2F_FRAC_BITS) - 1)
-
-/*
- * Converts unsigned integer into 32-bit IEEE floating point representation.
- * Will be exact from 0 to 2^24.  Above that, we round towards zero
- * as the fractional bits will not fit in a float.  (It would be better to
- * round towards even as the fpu does, but that is slower.)
- */
-static __pure uint32_t int2float(uint32_t x)
-{
-	uint32_t msb, exponent, fraction;
-
-	/* Zero is special */
-	if (!x) return 0;
-
-	/* Get location of the most significant bit */
-	msb = __fls(x);
-
-	/*
-	 * Use a rotate instead of a shift because that works both leftwards
-	 * and rightwards due to the mod(32) behaviour.  This means we don't
-	 * need to check to see if we are above 2^24 or not.
-	 */
-	fraction = ror32(x, (msb - I2F_FRAC_BITS) & 0x1f) & I2F_MASK;
-	exponent = (127 + msb) << I2F_FRAC_BITS;
-
-	return fraction + exponent;
-}
-
-#define DI_PT_RECTLIST        0x11
-#define DI_INDEX_SIZE_16_BIT  0x0
-#define DI_SRC_SEL_AUTO_INDEX 0x2
-
-#define FMT_8                 0x1
-#define FMT_5_6_5             0x8
-#define FMT_8_8_8_8           0x1a
-#define COLOR_8               0x1
-#define COLOR_5_6_5           0x8
-#define COLOR_8_8_8_8         0x1a
-
-static void
-set_render_target(drm_radeon_private_t *dev_priv, int format, int w, int h, u64 gpu_addr)
-{
-	u32 cb_color_info;
-	int pitch, slice;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	h = ALIGN(h, 8);
-	if (h < 8)
-		h = 8;
-
-	cb_color_info = ((format << 2) | (1 << 27));
-	pitch = (w / 8) - 1;
-	slice = ((w * h) / 64) - 1;
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_R600) &&
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV770)) {
-		BEGIN_RING(21 + 2);
-		OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-		OUT_RING((R600_CB_COLOR0_BASE - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-		OUT_RING(gpu_addr >> 8);
-		OUT_RING(CP_PACKET3(R600_IT_SURFACE_BASE_UPDATE, 0));
-		OUT_RING(2 << 0);
-	} else {
-		BEGIN_RING(21);
-		OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-		OUT_RING((R600_CB_COLOR0_BASE - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-		OUT_RING(gpu_addr >> 8);
-	}
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_SIZE - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING((pitch << 0) | (slice << 10));
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_VIEW - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_INFO - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(cb_color_info);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_TILE - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_FRAG - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_CB_COLOR0_MASK - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-
-	ADVANCE_RING();
-}
-
-static void
-cp_set_surface_sync(drm_radeon_private_t *dev_priv,
-		    u32 sync_type, u32 size, u64 mc_addr)
-{
-	u32 cp_coher_size;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	if (size == 0xffffffff)
-		cp_coher_size = 0xffffffff;
-	else
-		cp_coher_size = ((size + 255) >> 8);
-
-	BEGIN_RING(5);
-	OUT_RING(CP_PACKET3(R600_IT_SURFACE_SYNC, 3));
-	OUT_RING(sync_type);
-	OUT_RING(cp_coher_size);
-	OUT_RING((mc_addr >> 8));
-	OUT_RING(10); /* poll interval */
-	ADVANCE_RING();
-}
-
-static void
-set_shaders(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	u64 gpu_addr;
-	int i;
-	u32 *vs, *ps;
-	uint32_t sq_pgm_resources;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	/* load shaders */
-	vs = (u32 *) ((char *)dev->agp_buffer_map->handle + dev_priv->blit_vb->offset);
-	ps = (u32 *) ((char *)dev->agp_buffer_map->handle + dev_priv->blit_vb->offset + 256);
-
-	for (i = 0; i < r6xx_vs_size; i++)
-		vs[i] = cpu_to_le32(r6xx_vs[i]);
-	for (i = 0; i < r6xx_ps_size; i++)
-		ps[i] = cpu_to_le32(r6xx_ps[i]);
-
-	dev_priv->blit_vb->used = 512;
-
-	gpu_addr = dev_priv->gart_buffers_offset + dev_priv->blit_vb->offset;
-
-	/* setup shader regs */
-	sq_pgm_resources = (1 << 0);
-
-	BEGIN_RING(9 + 12);
-	/* VS */
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_START_VS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(gpu_addr >> 8);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_RESOURCES_VS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(sq_pgm_resources);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_CF_OFFSET_VS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-
-	/* PS */
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_START_PS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING((gpu_addr + 256) >> 8);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_RESOURCES_PS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(sq_pgm_resources | (1 << 28));
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_EXPORTS_PS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(2);
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 1));
-	OUT_RING((R600_SQ_PGM_CF_OFFSET_PS - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING(0);
-	ADVANCE_RING();
-
-	cp_set_surface_sync(dev_priv,
-			    R600_SH_ACTION_ENA, 512, gpu_addr);
-}
-
-static void
-set_vtx_resource(drm_radeon_private_t *dev_priv, u64 gpu_addr)
-{
-	uint32_t sq_vtx_constant_word2;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	sq_vtx_constant_word2 = (((gpu_addr >> 32) & 0xff) | (16 << 8));
-#ifdef __BIG_ENDIAN
-	sq_vtx_constant_word2 |= (2 << 30);
-#endif
-
-	BEGIN_RING(9);
-	OUT_RING(CP_PACKET3(R600_IT_SET_RESOURCE, 7));
-	OUT_RING(0x460);
-	OUT_RING(gpu_addr & 0xffffffff);
-	OUT_RING(48 - 1);
-	OUT_RING(sq_vtx_constant_word2);
-	OUT_RING(1 << 0);
-	OUT_RING(0);
-	OUT_RING(0);
-	OUT_RING(R600_SQ_TEX_VTX_VALID_BUFFER << 30);
-	ADVANCE_RING();
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710))
-		cp_set_surface_sync(dev_priv,
-				    R600_TC_ACTION_ENA, 48, gpu_addr);
-	else
-		cp_set_surface_sync(dev_priv,
-				    R600_VC_ACTION_ENA, 48, gpu_addr);
-}
-
-static void
-set_tex_resource(drm_radeon_private_t *dev_priv,
-		 int format, int w, int h, int pitch, u64 gpu_addr)
-{
-	uint32_t sq_tex_resource_word0, sq_tex_resource_word1, sq_tex_resource_word4;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	if (h < 1)
-		h = 1;
-
-	sq_tex_resource_word0 = (1 << 0);
-	sq_tex_resource_word0 |= ((((pitch >> 3) - 1) << 8) |
-				  ((w - 1) << 19));
-
-	sq_tex_resource_word1 = (format << 26);
-	sq_tex_resource_word1 |= ((h - 1) << 0);
-
-	sq_tex_resource_word4 = ((1 << 14) |
-				 (0 << 16) |
-				 (1 << 19) |
-				 (2 << 22) |
-				 (3 << 25));
-
-	BEGIN_RING(9);
-	OUT_RING(CP_PACKET3(R600_IT_SET_RESOURCE, 7));
-	OUT_RING(0);
-	OUT_RING(sq_tex_resource_word0);
-	OUT_RING(sq_tex_resource_word1);
-	OUT_RING(gpu_addr >> 8);
-	OUT_RING(gpu_addr >> 8);
-	OUT_RING(sq_tex_resource_word4);
-	OUT_RING(0);
-	OUT_RING(R600_SQ_TEX_VTX_VALID_TEXTURE << 30);
-	ADVANCE_RING();
-
-}
-
-static void
-set_scissors(drm_radeon_private_t *dev_priv, int x1, int y1, int x2, int y2)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(12);
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 2));
-	OUT_RING((R600_PA_SC_SCREEN_SCISSOR_TL - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING((x1 << 0) | (y1 << 16));
-	OUT_RING((x2 << 0) | (y2 << 16));
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 2));
-	OUT_RING((R600_PA_SC_GENERIC_SCISSOR_TL - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING((x1 << 0) | (y1 << 16) | (1 << 31));
-	OUT_RING((x2 << 0) | (y2 << 16));
-
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONTEXT_REG, 2));
-	OUT_RING((R600_PA_SC_WINDOW_SCISSOR_TL - R600_SET_CONTEXT_REG_OFFSET) >> 2);
-	OUT_RING((x1 << 0) | (y1 << 16) | (1 << 31));
-	OUT_RING((x2 << 0) | (y2 << 16));
-	ADVANCE_RING();
-}
-
-static void
-draw_auto(drm_radeon_private_t *dev_priv)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(10);
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
-	OUT_RING((R600_VGT_PRIMITIVE_TYPE - R600_SET_CONFIG_REG_OFFSET) >> 2);
-	OUT_RING(DI_PT_RECTLIST);
-
-	OUT_RING(CP_PACKET3(R600_IT_INDEX_TYPE, 0));
-#ifdef __BIG_ENDIAN
-	OUT_RING((2 << 2) | DI_INDEX_SIZE_16_BIT);
-#else
-	OUT_RING(DI_INDEX_SIZE_16_BIT);
-#endif
-
-	OUT_RING(CP_PACKET3(R600_IT_NUM_INSTANCES, 0));
-	OUT_RING(1);
-
-	OUT_RING(CP_PACKET3(R600_IT_DRAW_INDEX_AUTO, 1));
-	OUT_RING(3);
-	OUT_RING(DI_SRC_SEL_AUTO_INDEX);
-
-	ADVANCE_RING();
-	COMMIT_RING();
-}
-
-static void
-set_default_state(drm_radeon_private_t *dev_priv)
-{
-	int i;
-	u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2;
-	u32 sq_thread_resource_mgmt, sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2;
-	int num_ps_gprs, num_vs_gprs, num_temp_gprs, num_gs_gprs, num_es_gprs;
-	int num_ps_threads, num_vs_threads, num_gs_threads, num_es_threads;
-	int num_ps_stack_entries, num_vs_stack_entries, num_gs_stack_entries, num_es_stack_entries;
-	RING_LOCALS;
-
-	switch ((dev_priv->flags & RADEON_FAMILY_MASK)) {
-	case CHIP_R600:
-		num_ps_gprs = 192;
-		num_vs_gprs = 56;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 136;
-		num_vs_threads = 48;
-		num_gs_threads = 4;
-		num_es_threads = 4;
-		num_ps_stack_entries = 128;
-		num_vs_stack_entries = 128;
-		num_gs_stack_entries = 0;
-		num_es_stack_entries = 0;
-		break;
-	case CHIP_RV630:
-	case CHIP_RV635:
-		num_ps_gprs = 84;
-		num_vs_gprs = 36;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 144;
-		num_vs_threads = 40;
-		num_gs_threads = 4;
-		num_es_threads = 4;
-		num_ps_stack_entries = 40;
-		num_vs_stack_entries = 40;
-		num_gs_stack_entries = 32;
-		num_es_stack_entries = 16;
-		break;
-	case CHIP_RV610:
-	case CHIP_RV620:
-	case CHIP_RS780:
-	case CHIP_RS880:
-	default:
-		num_ps_gprs = 84;
-		num_vs_gprs = 36;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 136;
-		num_vs_threads = 48;
-		num_gs_threads = 4;
-		num_es_threads = 4;
-		num_ps_stack_entries = 40;
-		num_vs_stack_entries = 40;
-		num_gs_stack_entries = 32;
-		num_es_stack_entries = 16;
-		break;
-	case CHIP_RV670:
-		num_ps_gprs = 144;
-		num_vs_gprs = 40;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 136;
-		num_vs_threads = 48;
-		num_gs_threads = 4;
-		num_es_threads = 4;
-		num_ps_stack_entries = 40;
-		num_vs_stack_entries = 40;
-		num_gs_stack_entries = 32;
-		num_es_stack_entries = 16;
-		break;
-	case CHIP_RV770:
-		num_ps_gprs = 192;
-		num_vs_gprs = 56;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 188;
-		num_vs_threads = 60;
-		num_gs_threads = 0;
-		num_es_threads = 0;
-		num_ps_stack_entries = 256;
-		num_vs_stack_entries = 256;
-		num_gs_stack_entries = 0;
-		num_es_stack_entries = 0;
-		break;
-	case CHIP_RV730:
-	case CHIP_RV740:
-		num_ps_gprs = 84;
-		num_vs_gprs = 36;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 188;
-		num_vs_threads = 60;
-		num_gs_threads = 0;
-		num_es_threads = 0;
-		num_ps_stack_entries = 128;
-		num_vs_stack_entries = 128;
-		num_gs_stack_entries = 0;
-		num_es_stack_entries = 0;
-		break;
-	case CHIP_RV710:
-		num_ps_gprs = 192;
-		num_vs_gprs = 56;
-		num_temp_gprs = 4;
-		num_gs_gprs = 0;
-		num_es_gprs = 0;
-		num_ps_threads = 144;
-		num_vs_threads = 48;
-		num_gs_threads = 0;
-		num_es_threads = 0;
-		num_ps_stack_entries = 128;
-		num_vs_stack_entries = 128;
-		num_gs_stack_entries = 0;
-		num_es_stack_entries = 0;
-		break;
-	}
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710))
-		sq_config = 0;
-	else
-		sq_config = R600_VC_ENABLE;
-
-	sq_config |= (R600_DX9_CONSTS |
-		      R600_ALU_INST_PREFER_VECTOR |
-		      R600_PS_PRIO(0) |
-		      R600_VS_PRIO(1) |
-		      R600_GS_PRIO(2) |
-		      R600_ES_PRIO(3));
-
-	sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(num_ps_gprs) |
-				  R600_NUM_VS_GPRS(num_vs_gprs) |
-				  R600_NUM_CLAUSE_TEMP_GPRS(num_temp_gprs));
-	sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(num_gs_gprs) |
-				  R600_NUM_ES_GPRS(num_es_gprs));
-	sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(num_ps_threads) |
-				   R600_NUM_VS_THREADS(num_vs_threads) |
-				   R600_NUM_GS_THREADS(num_gs_threads) |
-				   R600_NUM_ES_THREADS(num_es_threads));
-	sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(num_ps_stack_entries) |
-				    R600_NUM_VS_STACK_ENTRIES(num_vs_stack_entries));
-	sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(num_gs_stack_entries) |
-				    R600_NUM_ES_STACK_ENTRIES(num_es_stack_entries));
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
-		BEGIN_RING(r7xx_default_size + 10);
-		for (i = 0; i < r7xx_default_size; i++)
-			OUT_RING(r7xx_default_state[i]);
-	} else {
-		BEGIN_RING(r6xx_default_size + 10);
-		for (i = 0; i < r6xx_default_size; i++)
-			OUT_RING(r6xx_default_state[i]);
-	}
-	OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
-	OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
-	/* SQ config */
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 6));
-	OUT_RING((R600_SQ_CONFIG - R600_SET_CONFIG_REG_OFFSET) >> 2);
-	OUT_RING(sq_config);
-	OUT_RING(sq_gpr_resource_mgmt_1);
-	OUT_RING(sq_gpr_resource_mgmt_2);
-	OUT_RING(sq_thread_resource_mgmt);
-	OUT_RING(sq_stack_resource_mgmt_1);
-	OUT_RING(sq_stack_resource_mgmt_2);
-	ADVANCE_RING();
-}
-
-static int r600_nomm_get_vb(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	dev_priv->blit_vb = radeon_freelist_get(dev);
-	if (!dev_priv->blit_vb) {
-		DRM_ERROR("Unable to allocate vertex buffer for blit\n");
-		return -EAGAIN;
-	}
-	return 0;
-}
-
-static void r600_nomm_put_vb(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	dev_priv->blit_vb->used = 0;
-	radeon_cp_discard_buffer(dev, dev_priv->blit_vb->file_priv->master, dev_priv->blit_vb);
-}
-
-static void *r600_nomm_get_vb_ptr(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	return (((char *)dev->agp_buffer_map->handle +
-		 dev_priv->blit_vb->offset + dev_priv->blit_vb->used));
-}
-
-int
-r600_prepare_blit_copy(struct drm_device *dev, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int ret;
-	DRM_DEBUG("\n");
-
-	ret = r600_nomm_get_vb(dev);
-	if (ret)
-		return ret;
-
-	dev_priv->blit_vb->file_priv = file_priv;
-
-	set_default_state(dev_priv);
-	set_shaders(dev);
-
-	return 0;
-}
-
-
-void
-r600_done_blit_copy(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(5);
-	OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
-	OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
-	/* wait for 3D idle clean */
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
-	OUT_RING((R600_WAIT_UNTIL - R600_SET_CONFIG_REG_OFFSET) >> 2);
-	OUT_RING(RADEON_WAIT_3D_IDLE | RADEON_WAIT_3D_IDLECLEAN);
-
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	r600_nomm_put_vb(dev);
-}
-
-void
-r600_blit_copy(struct drm_device *dev,
-	       uint64_t src_gpu_addr, uint64_t dst_gpu_addr,
-	       int size_bytes)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int max_bytes;
-	u64 vb_addr;
-	u32 *vb;
-
-	vb = r600_nomm_get_vb_ptr(dev);
-
-	if ((size_bytes & 3) || (src_gpu_addr & 3) || (dst_gpu_addr & 3)) {
-		max_bytes = 8192;
-
-		while (size_bytes) {
-			int cur_size = size_bytes;
-			int src_x = src_gpu_addr & 255;
-			int dst_x = dst_gpu_addr & 255;
-			int h = 1;
-			src_gpu_addr = src_gpu_addr & ~255;
-			dst_gpu_addr = dst_gpu_addr & ~255;
-
-			if (!src_x && !dst_x) {
-				h = (cur_size / max_bytes);
-				if (h > 8192)
-					h = 8192;
-				if (h == 0)
-					h = 1;
-				else
-					cur_size = max_bytes;
-			} else {
-				if (cur_size > max_bytes)
-					cur_size = max_bytes;
-				if (cur_size > (max_bytes - dst_x))
-					cur_size = (max_bytes - dst_x);
-				if (cur_size > (max_bytes - src_x))
-					cur_size = (max_bytes - src_x);
-			}
-
-			if ((dev_priv->blit_vb->used + 48) > dev_priv->blit_vb->total) {
-
-				r600_nomm_put_vb(dev);
-				r600_nomm_get_vb(dev);
-				if (!dev_priv->blit_vb)
-					return;
-				set_shaders(dev);
-				vb = r600_nomm_get_vb_ptr(dev);
-			}
-
-			vb[0] = int2float(dst_x);
-			vb[1] = 0;
-			vb[2] = int2float(src_x);
-			vb[3] = 0;
-
-			vb[4] = int2float(dst_x);
-			vb[5] = int2float(h);
-			vb[6] = int2float(src_x);
-			vb[7] = int2float(h);
-
-			vb[8] = int2float(dst_x + cur_size);
-			vb[9] = int2float(h);
-			vb[10] = int2float(src_x + cur_size);
-			vb[11] = int2float(h);
-
-			/* src */
-			set_tex_resource(dev_priv, FMT_8,
-					 src_x + cur_size, h, src_x + cur_size,
-					 src_gpu_addr);
-
-			cp_set_surface_sync(dev_priv,
-					    R600_TC_ACTION_ENA, (src_x + cur_size * h), src_gpu_addr);
-
-			/* dst */
-			set_render_target(dev_priv, COLOR_8,
-					  dst_x + cur_size, h,
-					  dst_gpu_addr);
-
-			/* scissors */
-			set_scissors(dev_priv, dst_x, 0, dst_x + cur_size, h);
-
-			/* Vertex buffer setup */
-			vb_addr = dev_priv->gart_buffers_offset +
-				dev_priv->blit_vb->offset +
-				dev_priv->blit_vb->used;
-			set_vtx_resource(dev_priv, vb_addr);
-
-			/* draw */
-			draw_auto(dev_priv);
-
-			cp_set_surface_sync(dev_priv,
-					    R600_CB_ACTION_ENA | R600_CB0_DEST_BASE_ENA,
-					    cur_size * h, dst_gpu_addr);
-
-			vb += 12;
-			dev_priv->blit_vb->used += 12 * 4;
-
-			src_gpu_addr += cur_size * h;
-			dst_gpu_addr += cur_size * h;
-			size_bytes -= cur_size * h;
-		}
-	} else {
-		max_bytes = 8192 * 4;
-
-		while (size_bytes) {
-			int cur_size = size_bytes;
-			int src_x = (src_gpu_addr & 255);
-			int dst_x = (dst_gpu_addr & 255);
-			int h = 1;
-			src_gpu_addr = src_gpu_addr & ~255;
-			dst_gpu_addr = dst_gpu_addr & ~255;
-
-			if (!src_x && !dst_x) {
-				h = (cur_size / max_bytes);
-				if (h > 8192)
-					h = 8192;
-				if (h == 0)
-					h = 1;
-				else
-					cur_size = max_bytes;
-			} else {
-				if (cur_size > max_bytes)
-					cur_size = max_bytes;
-				if (cur_size > (max_bytes - dst_x))
-					cur_size = (max_bytes - dst_x);
-				if (cur_size > (max_bytes - src_x))
-					cur_size = (max_bytes - src_x);
-			}
-
-			if ((dev_priv->blit_vb->used + 48) > dev_priv->blit_vb->total) {
-				r600_nomm_put_vb(dev);
-				r600_nomm_get_vb(dev);
-				if (!dev_priv->blit_vb)
-					return;
-
-				set_shaders(dev);
-				vb = r600_nomm_get_vb_ptr(dev);
-			}
-
-			vb[0] = int2float(dst_x / 4);
-			vb[1] = 0;
-			vb[2] = int2float(src_x / 4);
-			vb[3] = 0;
-
-			vb[4] = int2float(dst_x / 4);
-			vb[5] = int2float(h);
-			vb[6] = int2float(src_x / 4);
-			vb[7] = int2float(h);
-
-			vb[8] = int2float((dst_x + cur_size) / 4);
-			vb[9] = int2float(h);
-			vb[10] = int2float((src_x + cur_size) / 4);
-			vb[11] = int2float(h);
-
-			/* src */
-			set_tex_resource(dev_priv, FMT_8_8_8_8,
-					 (src_x + cur_size) / 4,
-					 h, (src_x + cur_size) / 4,
-					 src_gpu_addr);
-
-			cp_set_surface_sync(dev_priv,
-					    R600_TC_ACTION_ENA, (src_x + cur_size * h), src_gpu_addr);
-
-			/* dst */
-			set_render_target(dev_priv, COLOR_8_8_8_8,
-					  (dst_x + cur_size) / 4, h,
-					  dst_gpu_addr);
-
-			/* scissors */
-			set_scissors(dev_priv, (dst_x / 4), 0, (dst_x + cur_size / 4), h);
-
-			/* Vertex buffer setup */
-			vb_addr = dev_priv->gart_buffers_offset +
-				dev_priv->blit_vb->offset +
-				dev_priv->blit_vb->used;
-			set_vtx_resource(dev_priv, vb_addr);
-
-			/* draw */
-			draw_auto(dev_priv);
-
-			cp_set_surface_sync(dev_priv,
-					    R600_CB_ACTION_ENA | R600_CB0_DEST_BASE_ENA,
-					    cur_size * h, dst_gpu_addr);
-
-			vb += 12;
-			dev_priv->blit_vb->used += 12 * 4;
-
-			src_gpu_addr += cur_size * h;
-			dst_gpu_addr += cur_size * h;
-			size_bytes -= cur_size * h;
-		}
-	}
-}
-
-void
-r600_blit_swap(struct drm_device *dev,
-	       uint64_t src_gpu_addr, uint64_t dst_gpu_addr,
-	       int sx, int sy, int dx, int dy,
-	       int w, int h, int src_pitch, int dst_pitch, int cpp)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int cb_format, tex_format;
-	int sx2, sy2, dx2, dy2;
-	u64 vb_addr;
-	u32 *vb;
-
-	if ((dev_priv->blit_vb->used + 48) > dev_priv->blit_vb->total) {
-
-		r600_nomm_put_vb(dev);
-		r600_nomm_get_vb(dev);
-		if (!dev_priv->blit_vb)
-			return;
-
-		set_shaders(dev);
-	}
-	vb = r600_nomm_get_vb_ptr(dev);
-
-	sx2 = sx + w;
-	sy2 = sy + h;
-	dx2 = dx + w;
-	dy2 = dy + h;
-
-	vb[0] = int2float(dx);
-	vb[1] = int2float(dy);
-	vb[2] = int2float(sx);
-	vb[3] = int2float(sy);
-
-	vb[4] = int2float(dx);
-	vb[5] = int2float(dy2);
-	vb[6] = int2float(sx);
-	vb[7] = int2float(sy2);
-
-	vb[8] = int2float(dx2);
-	vb[9] = int2float(dy2);
-	vb[10] = int2float(sx2);
-	vb[11] = int2float(sy2);
-
-	switch(cpp) {
-	case 4:
-		cb_format = COLOR_8_8_8_8;
-		tex_format = FMT_8_8_8_8;
-		break;
-	case 2:
-		cb_format = COLOR_5_6_5;
-		tex_format = FMT_5_6_5;
-		break;
-	default:
-		cb_format = COLOR_8;
-		tex_format = FMT_8;
-		break;
-	}
-
-	/* src */
-	set_tex_resource(dev_priv, tex_format,
-			 src_pitch / cpp,
-			 sy2, src_pitch / cpp,
-			 src_gpu_addr);
-
-	cp_set_surface_sync(dev_priv,
-			    R600_TC_ACTION_ENA, src_pitch * sy2, src_gpu_addr);
-
-	/* dst */
-	set_render_target(dev_priv, cb_format,
-			  dst_pitch / cpp, dy2,
-			  dst_gpu_addr);
-
-	/* scissors */
-	set_scissors(dev_priv, dx, dy, dx2, dy2);
-
-	/* Vertex buffer setup */
-	vb_addr = dev_priv->gart_buffers_offset +
-		dev_priv->blit_vb->offset +
-		dev_priv->blit_vb->used;
-	set_vtx_resource(dev_priv, vb_addr);
-
-	/* draw */
-	draw_auto(dev_priv);
-
-	cp_set_surface_sync(dev_priv,
-			    R600_CB_ACTION_ENA | R600_CB0_DEST_BASE_ENA,
-			    dst_pitch * dy2, dst_gpu_addr);
-
-	dev_priv->blit_vb->used += 12 * 4;
-}
diff --git a/drivers/gpu/drm/radeon/r600_cp.c b/drivers/gpu/drm/radeon/r600_cp.c
deleted file mode 100644
index e231eeafef23..000000000000
--- a/drivers/gpu/drm/radeon/r600_cp.c
+++ /dev/null
@@ -1,2660 +0,0 @@
-/*
- * Copyright 2008-2009 Advanced Micro Devices, Inc.
- * Copyright 2008 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *     Dave Airlie <airlied@redhat.com>
- *     Alex Deucher <alexander.deucher@amd.com>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <linux/module.h>
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-
-#define PFP_UCODE_SIZE 576
-#define PM4_UCODE_SIZE 1792
-#define R700_PFP_UCODE_SIZE 848
-#define R700_PM4_UCODE_SIZE 1360
-
-/* Firmware Names */
-MODULE_FIRMWARE("radeon/R600_pfp.bin");
-MODULE_FIRMWARE("radeon/R600_me.bin");
-MODULE_FIRMWARE("radeon/RV610_pfp.bin");
-MODULE_FIRMWARE("radeon/RV610_me.bin");
-MODULE_FIRMWARE("radeon/RV630_pfp.bin");
-MODULE_FIRMWARE("radeon/RV630_me.bin");
-MODULE_FIRMWARE("radeon/RV620_pfp.bin");
-MODULE_FIRMWARE("radeon/RV620_me.bin");
-MODULE_FIRMWARE("radeon/RV635_pfp.bin");
-MODULE_FIRMWARE("radeon/RV635_me.bin");
-MODULE_FIRMWARE("radeon/RV670_pfp.bin");
-MODULE_FIRMWARE("radeon/RV670_me.bin");
-MODULE_FIRMWARE("radeon/RS780_pfp.bin");
-MODULE_FIRMWARE("radeon/RS780_me.bin");
-MODULE_FIRMWARE("radeon/RV770_pfp.bin");
-MODULE_FIRMWARE("radeon/RV770_me.bin");
-MODULE_FIRMWARE("radeon/RV730_pfp.bin");
-MODULE_FIRMWARE("radeon/RV730_me.bin");
-MODULE_FIRMWARE("radeon/RV710_pfp.bin");
-MODULE_FIRMWARE("radeon/RV710_me.bin");
-
-
-int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp,
-			unsigned family, u32 *ib, int *l);
-void r600_cs_legacy_init(void);
-
-
-# define ATI_PCIGART_PAGE_SIZE		4096	/**< PCI GART page size */
-# define ATI_PCIGART_PAGE_MASK		(~(ATI_PCIGART_PAGE_SIZE-1))
-
-#define R600_PTE_VALID     (1 << 0)
-#define R600_PTE_SYSTEM    (1 << 1)
-#define R600_PTE_SNOOPED   (1 << 2)
-#define R600_PTE_READABLE  (1 << 5)
-#define R600_PTE_WRITEABLE (1 << 6)
-
-/* MAX values used for gfx init */
-#define R6XX_MAX_SH_GPRS           256
-#define R6XX_MAX_TEMP_GPRS         16
-#define R6XX_MAX_SH_THREADS        256
-#define R6XX_MAX_SH_STACK_ENTRIES  4096
-#define R6XX_MAX_BACKENDS          8
-#define R6XX_MAX_BACKENDS_MASK     0xff
-#define R6XX_MAX_SIMDS             8
-#define R6XX_MAX_SIMDS_MASK        0xff
-#define R6XX_MAX_PIPES             8
-#define R6XX_MAX_PIPES_MASK        0xff
-
-#define R7XX_MAX_SH_GPRS           256
-#define R7XX_MAX_TEMP_GPRS         16
-#define R7XX_MAX_SH_THREADS        256
-#define R7XX_MAX_SH_STACK_ENTRIES  4096
-#define R7XX_MAX_BACKENDS          8
-#define R7XX_MAX_BACKENDS_MASK     0xff
-#define R7XX_MAX_SIMDS             16
-#define R7XX_MAX_SIMDS_MASK        0xffff
-#define R7XX_MAX_PIPES             8
-#define R7XX_MAX_PIPES_MASK        0xff
-
-static int r600_do_wait_for_fifo(drm_radeon_private_t *dev_priv, int entries)
-{
-	int i;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		int slots;
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
-			slots = (RADEON_READ(R600_GRBM_STATUS)
-				 & R700_CMDFIFO_AVAIL_MASK);
-		else
-			slots = (RADEON_READ(R600_GRBM_STATUS)
-				 & R600_CMDFIFO_AVAIL_MASK);
-		if (slots >= entries)
-			return 0;
-		DRM_UDELAY(1);
-	}
-	DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
-		 RADEON_READ(R600_GRBM_STATUS),
-		 RADEON_READ(R600_GRBM_STATUS2));
-
-	return -EBUSY;
-}
-
-static int r600_do_wait_for_idle(drm_radeon_private_t *dev_priv)
-{
-	int i, ret;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
-		ret = r600_do_wait_for_fifo(dev_priv, 8);
-	else
-		ret = r600_do_wait_for_fifo(dev_priv, 16);
-	if (ret)
-		return ret;
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		if (!(RADEON_READ(R600_GRBM_STATUS) & R600_GUI_ACTIVE))
-			return 0;
-		DRM_UDELAY(1);
-	}
-	DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
-		 RADEON_READ(R600_GRBM_STATUS),
-		 RADEON_READ(R600_GRBM_STATUS2));
-
-	return -EBUSY;
-}
-
-void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info)
-{
-	struct drm_sg_mem *entry = dev->sg;
-	int max_pages;
-	int pages;
-	int i;
-
-	if (!entry)
-		return;
-
-	if (gart_info->bus_addr) {
-		max_pages = (gart_info->table_size / sizeof(u64));
-		pages = (entry->pages <= max_pages)
-		  ? entry->pages : max_pages;
-
-		for (i = 0; i < pages; i++) {
-			if (!entry->busaddr[i])
-				break;
-			pci_unmap_page(dev->pdev, entry->busaddr[i],
-				       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-		}
-		if (gart_info->gart_table_location == DRM_ATI_GART_MAIN)
-			gart_info->bus_addr = 0;
-	}
-}
-
-/* R600 has page table setup */
-int r600_page_table_init(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
-	struct drm_local_map *map = &gart_info->mapping;
-	struct drm_sg_mem *entry = dev->sg;
-	int ret = 0;
-	int i, j;
-	int pages;
-	u64 page_base;
-	dma_addr_t entry_addr;
-	int max_ati_pages, max_real_pages, gart_idx;
-
-	/* okay page table is available - lets rock */
-	max_ati_pages = (gart_info->table_size / sizeof(u64));
-	max_real_pages = max_ati_pages / (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE);
-
-	pages = (entry->pages <= max_real_pages) ?
-		entry->pages : max_real_pages;
-
-	memset_io((void __iomem *)map->handle, 0, max_ati_pages * sizeof(u64));
-
-	gart_idx = 0;
-	for (i = 0; i < pages; i++) {
-		entry->busaddr[i] = pci_map_page(dev->pdev,
-						 entry->pagelist[i], 0,
-						 PAGE_SIZE,
-						 PCI_DMA_BIDIRECTIONAL);
-		if (pci_dma_mapping_error(dev->pdev, entry->busaddr[i])) {
-			DRM_ERROR("unable to map PCIGART pages!\n");
-			r600_page_table_cleanup(dev, gart_info);
-			goto done;
-		}
-		entry_addr = entry->busaddr[i];
-		for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
-			page_base = (u64) entry_addr & ATI_PCIGART_PAGE_MASK;
-			page_base |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED;
-			page_base |= R600_PTE_READABLE | R600_PTE_WRITEABLE;
-
-			DRM_WRITE64(map, gart_idx * sizeof(u64), page_base);
-
-			gart_idx++;
-
-			if ((i % 128) == 0)
-				DRM_DEBUG("page entry %d: 0x%016llx\n",
-				    i, (unsigned long long)page_base);
-			entry_addr += ATI_PCIGART_PAGE_SIZE;
-		}
-	}
-	ret = 1;
-done:
-	return ret;
-}
-
-static void r600_vm_flush_gart_range(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	u32 resp, countdown = 1000;
-	RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR, dev_priv->gart_vm_start >> 12);
-	RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
-	RADEON_WRITE(R600_VM_CONTEXT0_REQUEST_RESPONSE, 2);
-
-	do {
-		resp = RADEON_READ(R600_VM_CONTEXT0_REQUEST_RESPONSE);
-		countdown--;
-		DRM_UDELAY(1);
-	} while (((resp & 0xf0) == 0) && countdown);
-}
-
-static void r600_vm_init(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	/* initialise the VM to use the page table we constructed up there */
-	u32 vm_c0, i;
-	u32 mc_rd_a;
-	u32 vm_l2_cntl, vm_l2_cntl3;
-	/* okay set up the PCIE aperture type thingo */
-	RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
-	RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
-	RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
-
-	/* setup MC RD a */
-	mc_rd_a = R600_MCD_L1_TLB | R600_MCD_L1_FRAG_PROC | R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS |
-		R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | R600_MCD_EFFECTIVE_L1_TLB_SIZE(5) |
-		R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(5) | R600_MCD_WAIT_L2_QUERY;
-
-	RADEON_WRITE(R600_MCD_RD_A_CNTL, mc_rd_a);
-	RADEON_WRITE(R600_MCD_RD_B_CNTL, mc_rd_a);
-
-	RADEON_WRITE(R600_MCD_WR_A_CNTL, mc_rd_a);
-	RADEON_WRITE(R600_MCD_WR_B_CNTL, mc_rd_a);
-
-	RADEON_WRITE(R600_MCD_RD_GFX_CNTL, mc_rd_a);
-	RADEON_WRITE(R600_MCD_WR_GFX_CNTL, mc_rd_a);
-
-	RADEON_WRITE(R600_MCD_RD_SYS_CNTL, mc_rd_a);
-	RADEON_WRITE(R600_MCD_WR_SYS_CNTL, mc_rd_a);
-
-	RADEON_WRITE(R600_MCD_RD_HDP_CNTL, mc_rd_a | R600_MCD_L1_STRICT_ORDERING);
-	RADEON_WRITE(R600_MCD_WR_HDP_CNTL, mc_rd_a /*| R600_MCD_L1_STRICT_ORDERING*/);
-
-	RADEON_WRITE(R600_MCD_RD_PDMA_CNTL, mc_rd_a);
-	RADEON_WRITE(R600_MCD_WR_PDMA_CNTL, mc_rd_a);
-
-	RADEON_WRITE(R600_MCD_RD_SEM_CNTL, mc_rd_a | R600_MCD_SEMAPHORE_MODE);
-	RADEON_WRITE(R600_MCD_WR_SEM_CNTL, mc_rd_a);
-
-	vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
-	vm_l2_cntl |= R600_VM_L2_CNTL_QUEUE_SIZE(7);
-	RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
-
-	RADEON_WRITE(R600_VM_L2_CNTL2, 0);
-	vm_l2_cntl3 = (R600_VM_L2_CNTL3_BANK_SELECT_0(0) |
-		       R600_VM_L2_CNTL3_BANK_SELECT_1(1) |
-		       R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(2));
-	RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
-
-	vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
-
-	RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
-
-	vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
-
-	/* disable all other contexts */
-	for (i = 1; i < 8; i++)
-		RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
-
-	RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
-	RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
-	RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
-
-	r600_vm_flush_gart_range(dev);
-}
-
-static int r600_cp_init_microcode(drm_radeon_private_t *dev_priv)
-{
-	struct platform_device *pdev;
-	const char *chip_name;
-	size_t pfp_req_size, me_req_size;
-	char fw_name[30];
-	int err;
-
-	pdev = platform_device_register_simple("r600_cp", 0, NULL, 0);
-	err = IS_ERR(pdev);
-	if (err) {
-		printk(KERN_ERR "r600_cp: Failed to register firmware\n");
-		return -EINVAL;
-	}
-
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_R600:  chip_name = "R600";  break;
-	case CHIP_RV610: chip_name = "RV610"; break;
-	case CHIP_RV630: chip_name = "RV630"; break;
-	case CHIP_RV620: chip_name = "RV620"; break;
-	case CHIP_RV635: chip_name = "RV635"; break;
-	case CHIP_RV670: chip_name = "RV670"; break;
-	case CHIP_RS780:
-	case CHIP_RS880: chip_name = "RS780"; break;
-	case CHIP_RV770: chip_name = "RV770"; break;
-	case CHIP_RV730:
-	case CHIP_RV740: chip_name = "RV730"; break;
-	case CHIP_RV710: chip_name = "RV710"; break;
-	default:         BUG();
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
-		pfp_req_size = R700_PFP_UCODE_SIZE * 4;
-		me_req_size = R700_PM4_UCODE_SIZE * 4;
-	} else {
-		pfp_req_size = PFP_UCODE_SIZE * 4;
-		me_req_size = PM4_UCODE_SIZE * 12;
-	}
-
-	DRM_INFO("Loading %s CP Microcode\n", chip_name);
-
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
-	err = request_firmware(&dev_priv->pfp_fw, fw_name, &pdev->dev);
-	if (err)
-		goto out;
-	if (dev_priv->pfp_fw->size != pfp_req_size) {
-		printk(KERN_ERR
-		       "r600_cp: Bogus length %zu in firmware \"%s\"\n",
-		       dev_priv->pfp_fw->size, fw_name);
-		err = -EINVAL;
-		goto out;
-	}
-
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
-	err = request_firmware(&dev_priv->me_fw, fw_name, &pdev->dev);
-	if (err)
-		goto out;
-	if (dev_priv->me_fw->size != me_req_size) {
-		printk(KERN_ERR
-		       "r600_cp: Bogus length %zu in firmware \"%s\"\n",
-		       dev_priv->me_fw->size, fw_name);
-		err = -EINVAL;
-	}
-out:
-	platform_device_unregister(pdev);
-
-	if (err) {
-		if (err != -EINVAL)
-			printk(KERN_ERR
-			       "r600_cp: Failed to load firmware \"%s\"\n",
-			       fw_name);
-		release_firmware(dev_priv->pfp_fw);
-		dev_priv->pfp_fw = NULL;
-		release_firmware(dev_priv->me_fw);
-		dev_priv->me_fw = NULL;
-	}
-	return err;
-}
-
-static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
-{
-	const __be32 *fw_data;
-	int i;
-
-	if (!dev_priv->me_fw || !dev_priv->pfp_fw)
-		return;
-
-	r600_do_cp_stop(dev_priv);
-
-	RADEON_WRITE(R600_CP_RB_CNTL,
-#ifdef __BIG_ENDIAN
-		     R600_BUF_SWAP_32BIT |
-#endif
-		     R600_RB_NO_UPDATE |
-		     R600_RB_BLKSZ(15) |
-		     R600_RB_BUFSZ(3));
-
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
-	RADEON_READ(R600_GRBM_SOFT_RESET);
-	mdelay(15);
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
-
-	fw_data = (const __be32 *)dev_priv->me_fw->data;
-	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
-		RADEON_WRITE(R600_CP_ME_RAM_DATA,
-			     be32_to_cpup(fw_data++));
-
-	fw_data = (const __be32 *)dev_priv->pfp_fw->data;
-	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-	for (i = 0; i < PFP_UCODE_SIZE; i++)
-		RADEON_WRITE(R600_CP_PFP_UCODE_DATA,
-			     be32_to_cpup(fw_data++));
-
-	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-	RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
-
-}
-
-static void r700_vm_init(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	/* initialise the VM to use the page table we constructed up there */
-	u32 vm_c0, i;
-	u32 mc_vm_md_l1;
-	u32 vm_l2_cntl, vm_l2_cntl3;
-	/* okay set up the PCIE aperture type thingo */
-	RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
-	RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
-	RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
-
-	mc_vm_md_l1 = R700_ENABLE_L1_TLB |
-	    R700_ENABLE_L1_FRAGMENT_PROCESSING |
-	    R700_SYSTEM_ACCESS_MODE_IN_SYS |
-	    R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
-	    R700_EFFECTIVE_L1_TLB_SIZE(5) |
-	    R700_EFFECTIVE_L1_QUEUE_SIZE(5);
-
-	RADEON_WRITE(R700_MC_VM_MD_L1_TLB0_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MD_L1_TLB1_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MD_L1_TLB2_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MB_L1_TLB0_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MB_L1_TLB1_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MB_L1_TLB2_CNTL, mc_vm_md_l1);
-	RADEON_WRITE(R700_MC_VM_MB_L1_TLB3_CNTL, mc_vm_md_l1);
-
-	vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
-	vm_l2_cntl |= R700_VM_L2_CNTL_QUEUE_SIZE(7);
-	RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
-
-	RADEON_WRITE(R600_VM_L2_CNTL2, 0);
-	vm_l2_cntl3 = R700_VM_L2_CNTL3_BANK_SELECT(0) | R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(2);
-	RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
-
-	vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
-
-	RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
-
-	vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
-
-	/* disable all other contexts */
-	for (i = 1; i < 8; i++)
-		RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
-
-	RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
-	RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
-	RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
-
-	r600_vm_flush_gart_range(dev);
-}
-
-static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
-{
-	const __be32 *fw_data;
-	int i;
-
-	if (!dev_priv->me_fw || !dev_priv->pfp_fw)
-		return;
-
-	r600_do_cp_stop(dev_priv);
-
-	RADEON_WRITE(R600_CP_RB_CNTL,
-#ifdef __BIG_ENDIAN
-		     R600_BUF_SWAP_32BIT |
-#endif
-		     R600_RB_NO_UPDATE |
-		     R600_RB_BLKSZ(15) |
-		     R600_RB_BUFSZ(3));
-
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
-	RADEON_READ(R600_GRBM_SOFT_RESET);
-	mdelay(15);
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
-
-	fw_data = (const __be32 *)dev_priv->pfp_fw->data;
-	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
-		RADEON_WRITE(R600_CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
-	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-
-	fw_data = (const __be32 *)dev_priv->me_fw->data;
-	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
-		RADEON_WRITE(R600_CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
-	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-
-	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-	RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
-
-}
-
-static void r600_test_writeback(drm_radeon_private_t *dev_priv)
-{
-	u32 tmp;
-
-	/* Start with assuming that writeback doesn't work */
-	dev_priv->writeback_works = 0;
-
-	/* Writeback doesn't seem to work everywhere, test it here and possibly
-	 * enable it if it appears to work
-	 */
-	radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
-
-	RADEON_WRITE(R600_SCRATCH_REG1, 0xdeadbeef);
-
-	for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
-		u32 val;
-
-		val = radeon_read_ring_rptr(dev_priv, R600_SCRATCHOFF(1));
-		if (val == 0xdeadbeef)
-			break;
-		DRM_UDELAY(1);
-	}
-
-	if (tmp < dev_priv->usec_timeout) {
-		dev_priv->writeback_works = 1;
-		DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
-	} else {
-		dev_priv->writeback_works = 0;
-		DRM_INFO("writeback test failed\n");
-	}
-	if (radeon_no_wb == 1) {
-		dev_priv->writeback_works = 0;
-		DRM_INFO("writeback forced off\n");
-	}
-
-	if (!dev_priv->writeback_works) {
-		/* Disable writeback to avoid unnecessary bus master transfer */
-		RADEON_WRITE(R600_CP_RB_CNTL,
-#ifdef __BIG_ENDIAN
-			     R600_BUF_SWAP_32BIT |
-#endif
-			     RADEON_READ(R600_CP_RB_CNTL) |
-			     R600_RB_NO_UPDATE);
-		RADEON_WRITE(R600_SCRATCH_UMSK, 0);
-	}
-}
-
-int r600_do_engine_reset(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	u32 cp_ptr, cp_me_cntl, cp_rb_cntl;
-
-	DRM_INFO("Resetting GPU\n");
-
-	cp_ptr = RADEON_READ(R600_CP_RB_WPTR);
-	cp_me_cntl = RADEON_READ(R600_CP_ME_CNTL);
-	RADEON_WRITE(R600_CP_ME_CNTL, R600_CP_ME_HALT);
-
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0x7fff);
-	RADEON_READ(R600_GRBM_SOFT_RESET);
-	DRM_UDELAY(50);
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
-	RADEON_READ(R600_GRBM_SOFT_RESET);
-
-	RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
-	cp_rb_cntl = RADEON_READ(R600_CP_RB_CNTL);
-	RADEON_WRITE(R600_CP_RB_CNTL,
-#ifdef __BIG_ENDIAN
-		     R600_BUF_SWAP_32BIT |
-#endif
-		     R600_RB_RPTR_WR_ENA);
-
-	RADEON_WRITE(R600_CP_RB_RPTR_WR, cp_ptr);
-	RADEON_WRITE(R600_CP_RB_WPTR, cp_ptr);
-	RADEON_WRITE(R600_CP_RB_CNTL, cp_rb_cntl);
-	RADEON_WRITE(R600_CP_ME_CNTL, cp_me_cntl);
-
-	/* Reset the CP ring */
-	r600_do_cp_reset(dev_priv);
-
-	/* The CP is no longer running after an engine reset */
-	dev_priv->cp_running = 0;
-
-	/* Reset any pending vertex, indirect buffers */
-	radeon_freelist_reset(dev);
-
-	return 0;
-
-}
-
-static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
-					     u32 num_backends,
-					     u32 backend_disable_mask)
-{
-	u32 backend_map = 0;
-	u32 enabled_backends_mask;
-	u32 enabled_backends_count;
-	u32 cur_pipe;
-	u32 swizzle_pipe[R6XX_MAX_PIPES];
-	u32 cur_backend;
-	u32 i;
-
-	if (num_tile_pipes > R6XX_MAX_PIPES)
-		num_tile_pipes = R6XX_MAX_PIPES;
-	if (num_tile_pipes < 1)
-		num_tile_pipes = 1;
-	if (num_backends > R6XX_MAX_BACKENDS)
-		num_backends = R6XX_MAX_BACKENDS;
-	if (num_backends < 1)
-		num_backends = 1;
-
-	enabled_backends_mask = 0;
-	enabled_backends_count = 0;
-	for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
-		if (((backend_disable_mask >> i) & 1) == 0) {
-			enabled_backends_mask |= (1 << i);
-			++enabled_backends_count;
-		}
-		if (enabled_backends_count == num_backends)
-			break;
-	}
-
-	if (enabled_backends_count == 0) {
-		enabled_backends_mask = 1;
-		enabled_backends_count = 1;
-	}
-
-	if (enabled_backends_count != num_backends)
-		num_backends = enabled_backends_count;
-
-	memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
-	switch (num_tile_pipes) {
-	case 1:
-		swizzle_pipe[0] = 0;
-		break;
-	case 2:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 1;
-		break;
-	case 3:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 1;
-		swizzle_pipe[2] = 2;
-		break;
-	case 4:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 1;
-		swizzle_pipe[2] = 2;
-		swizzle_pipe[3] = 3;
-		break;
-	case 5:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 1;
-		swizzle_pipe[2] = 2;
-		swizzle_pipe[3] = 3;
-		swizzle_pipe[4] = 4;
-		break;
-	case 6:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 2;
-		swizzle_pipe[2] = 4;
-		swizzle_pipe[3] = 5;
-		swizzle_pipe[4] = 1;
-		swizzle_pipe[5] = 3;
-		break;
-	case 7:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 2;
-		swizzle_pipe[2] = 4;
-		swizzle_pipe[3] = 6;
-		swizzle_pipe[4] = 1;
-		swizzle_pipe[5] = 3;
-		swizzle_pipe[6] = 5;
-		break;
-	case 8:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 2;
-		swizzle_pipe[2] = 4;
-		swizzle_pipe[3] = 6;
-		swizzle_pipe[4] = 1;
-		swizzle_pipe[5] = 3;
-		swizzle_pipe[6] = 5;
-		swizzle_pipe[7] = 7;
-		break;
-	}
-
-	cur_backend = 0;
-	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
-		while (((1 << cur_backend) & enabled_backends_mask) == 0)
-			cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
-
-		backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
-
-		cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
-	}
-
-	return backend_map;
-}
-
-static int r600_count_pipe_bits(uint32_t val)
-{
-	return hweight32(val);
-}
-
-static void r600_gfx_init(struct drm_device *dev,
-			  drm_radeon_private_t *dev_priv)
-{
-	int i, j, num_qd_pipes;
-	u32 sx_debug_1;
-	u32 tc_cntl;
-	u32 arb_pop;
-	u32 num_gs_verts_per_thread;
-	u32 vgt_gs_per_es;
-	u32 gs_prim_buffer_depth = 0;
-	u32 sq_ms_fifo_sizes;
-	u32 sq_config;
-	u32 sq_gpr_resource_mgmt_1 = 0;
-	u32 sq_gpr_resource_mgmt_2 = 0;
-	u32 sq_thread_resource_mgmt = 0;
-	u32 sq_stack_resource_mgmt_1 = 0;
-	u32 sq_stack_resource_mgmt_2 = 0;
-	u32 hdp_host_path_cntl;
-	u32 backend_map;
-	u32 gb_tiling_config = 0;
-	u32 cc_rb_backend_disable;
-	u32 cc_gc_shader_pipe_config;
-	u32 ramcfg;
-
-	/* setup chip specs */
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_R600:
-		dev_priv->r600_max_pipes = 4;
-		dev_priv->r600_max_tile_pipes = 8;
-		dev_priv->r600_max_simds = 4;
-		dev_priv->r600_max_backends = 4;
-		dev_priv->r600_max_gprs = 256;
-		dev_priv->r600_max_threads = 192;
-		dev_priv->r600_max_stack_entries = 256;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 16;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 128;
-		dev_priv->r600_sq_num_cf_insts = 2;
-		break;
-	case CHIP_RV630:
-	case CHIP_RV635:
-		dev_priv->r600_max_pipes = 2;
-		dev_priv->r600_max_tile_pipes = 2;
-		dev_priv->r600_max_simds = 3;
-		dev_priv->r600_max_backends = 1;
-		dev_priv->r600_max_gprs = 128;
-		dev_priv->r600_max_threads = 192;
-		dev_priv->r600_max_stack_entries = 128;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 4;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 128;
-		dev_priv->r600_sq_num_cf_insts = 2;
-		break;
-	case CHIP_RV610:
-	case CHIP_RS780:
-	case CHIP_RS880:
-	case CHIP_RV620:
-		dev_priv->r600_max_pipes = 1;
-		dev_priv->r600_max_tile_pipes = 1;
-		dev_priv->r600_max_simds = 2;
-		dev_priv->r600_max_backends = 1;
-		dev_priv->r600_max_gprs = 128;
-		dev_priv->r600_max_threads = 192;
-		dev_priv->r600_max_stack_entries = 128;
-		dev_priv->r600_max_hw_contexts = 4;
-		dev_priv->r600_max_gs_threads = 4;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 128;
-		dev_priv->r600_sq_num_cf_insts = 1;
-		break;
-	case CHIP_RV670:
-		dev_priv->r600_max_pipes = 4;
-		dev_priv->r600_max_tile_pipes = 4;
-		dev_priv->r600_max_simds = 4;
-		dev_priv->r600_max_backends = 4;
-		dev_priv->r600_max_gprs = 192;
-		dev_priv->r600_max_threads = 192;
-		dev_priv->r600_max_stack_entries = 256;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 16;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 128;
-		dev_priv->r600_sq_num_cf_insts = 2;
-		break;
-	default:
-		break;
-	}
-
-	/* Initialize HDP */
-	j = 0;
-	for (i = 0; i < 32; i++) {
-		RADEON_WRITE((0x2c14 + j), 0x00000000);
-		RADEON_WRITE((0x2c18 + j), 0x00000000);
-		RADEON_WRITE((0x2c1c + j), 0x00000000);
-		RADEON_WRITE((0x2c20 + j), 0x00000000);
-		RADEON_WRITE((0x2c24 + j), 0x00000000);
-		j += 0x18;
-	}
-
-	RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
-
-	/* setup tiling, simd, pipe config */
-	ramcfg = RADEON_READ(R600_RAMCFG);
-
-	switch (dev_priv->r600_max_tile_pipes) {
-	case 1:
-		gb_tiling_config |= R600_PIPE_TILING(0);
-		break;
-	case 2:
-		gb_tiling_config |= R600_PIPE_TILING(1);
-		break;
-	case 4:
-		gb_tiling_config |= R600_PIPE_TILING(2);
-		break;
-	case 8:
-		gb_tiling_config |= R600_PIPE_TILING(3);
-		break;
-	default:
-		break;
-	}
-
-	gb_tiling_config |= R600_BANK_TILING((ramcfg >> R600_NOOFBANK_SHIFT) & R600_NOOFBANK_MASK);
-
-	gb_tiling_config |= R600_GROUP_SIZE(0);
-
-	if (((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK) > 3) {
-		gb_tiling_config |= R600_ROW_TILING(3);
-		gb_tiling_config |= R600_SAMPLE_SPLIT(3);
-	} else {
-		gb_tiling_config |=
-			R600_ROW_TILING(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
-		gb_tiling_config |=
-			R600_SAMPLE_SPLIT(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
-	}
-
-	gb_tiling_config |= R600_BANK_SWAPS(1);
-
-	cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
-	cc_rb_backend_disable |=
-		R600_BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R6XX_MAX_BACKENDS_MASK);
-
-	cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
-	cc_gc_shader_pipe_config |=
-		R600_INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R6XX_MAX_PIPES_MASK);
-	cc_gc_shader_pipe_config |=
-		R600_INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R6XX_MAX_SIMDS_MASK);
-
-	backend_map = r600_get_tile_pipe_to_backend_map(dev_priv->r600_max_tile_pipes,
-							(R6XX_MAX_BACKENDS -
-							 r600_count_pipe_bits((cc_rb_backend_disable &
-									       R6XX_MAX_BACKENDS_MASK) >> 16)),
-							(cc_rb_backend_disable >> 16));
-	gb_tiling_config |= R600_BACKEND_MAP(backend_map);
-
-	RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
-	RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
-	RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
-	if (gb_tiling_config & 0xc0) {
-		dev_priv->r600_group_size = 512;
-	} else {
-		dev_priv->r600_group_size = 256;
-	}
-	dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
-	if (gb_tiling_config & 0x30) {
-		dev_priv->r600_nbanks = 8;
-	} else {
-		dev_priv->r600_nbanks = 4;
-	}
-
-	RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
-	RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
-	RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
-
-	num_qd_pipes =
-		R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
-	RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
-	RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
-
-	/* set HW defaults for 3D engine */
-	RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
-						R600_ROQ_IB2_START(0x2b)));
-
-	RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, (R600_MEQ_END(0x40) |
-					      R600_ROQ_END(0x40)));
-
-	RADEON_WRITE(R600_TA_CNTL_AUX, (R600_DISABLE_CUBE_ANISO |
-					R600_SYNC_GRADIENT |
-					R600_SYNC_WALKER |
-					R600_SYNC_ALIGNER));
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)
-		RADEON_WRITE(R600_ARB_GDEC_RD_CNTL, 0x00000021);
-
-	sx_debug_1 = RADEON_READ(R600_SX_DEBUG_1);
-	sx_debug_1 |= R600_SMX_EVENT_RELEASE;
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_R600))
-		sx_debug_1 |= R600_ENABLE_NEW_SMX_ADDRESS;
-	RADEON_WRITE(R600_SX_DEBUG_1, sx_debug_1);
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
-		RADEON_WRITE(R600_DB_DEBUG, R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE);
-	else
-		RADEON_WRITE(R600_DB_DEBUG, 0);
-
-	RADEON_WRITE(R600_DB_WATERMARKS, (R600_DEPTH_FREE(4) |
-					  R600_DEPTH_FLUSH(16) |
-					  R600_DEPTH_PENDING_FREE(4) |
-					  R600_DEPTH_CACHELINE_FREE(16)));
-	RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
-	RADEON_WRITE(R600_VGT_NUM_INSTANCES, 0);
-
-	RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
-	RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(0));
-
-	sq_ms_fifo_sizes = RADEON_READ(R600_SQ_MS_FIFO_SIZES);
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
-		sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(0xa) |
-				    R600_FETCH_FIFO_HIWATER(0xa) |
-				    R600_DONE_FIFO_HIWATER(0xe0) |
-				    R600_ALU_UPDATE_FIFO_HIWATER(0x8));
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
-		sq_ms_fifo_sizes &= ~R600_DONE_FIFO_HIWATER(0xff);
-		sq_ms_fifo_sizes |= R600_DONE_FIFO_HIWATER(0x4);
-	}
-	RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
-
-	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
-	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
-	 */
-	sq_config = RADEON_READ(R600_SQ_CONFIG);
-	sq_config &= ~(R600_PS_PRIO(3) |
-		       R600_VS_PRIO(3) |
-		       R600_GS_PRIO(3) |
-		       R600_ES_PRIO(3));
-	sq_config |= (R600_DX9_CONSTS |
-		      R600_VC_ENABLE |
-		      R600_PS_PRIO(0) |
-		      R600_VS_PRIO(1) |
-		      R600_GS_PRIO(2) |
-		      R600_ES_PRIO(3));
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) {
-		sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(124) |
-					  R600_NUM_VS_GPRS(124) |
-					  R600_NUM_CLAUSE_TEMP_GPRS(4));
-		sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(0) |
-					  R600_NUM_ES_GPRS(0));
-		sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(136) |
-					   R600_NUM_VS_THREADS(48) |
-					   R600_NUM_GS_THREADS(4) |
-					   R600_NUM_ES_THREADS(4));
-		sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(128) |
-					    R600_NUM_VS_STACK_ENTRIES(128));
-		sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(0) |
-					    R600_NUM_ES_STACK_ENTRIES(0));
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
-		/* no vertex cache */
-		sq_config &= ~R600_VC_ENABLE;
-
-		sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
-					  R600_NUM_VS_GPRS(44) |
-					  R600_NUM_CLAUSE_TEMP_GPRS(2));
-		sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
-					  R600_NUM_ES_GPRS(17));
-		sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
-					   R600_NUM_VS_THREADS(78) |
-					   R600_NUM_GS_THREADS(4) |
-					   R600_NUM_ES_THREADS(31));
-		sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
-					    R600_NUM_VS_STACK_ENTRIES(40));
-		sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
-					    R600_NUM_ES_STACK_ENTRIES(16));
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
-		sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
-					  R600_NUM_VS_GPRS(44) |
-					  R600_NUM_CLAUSE_TEMP_GPRS(2));
-		sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(18) |
-					  R600_NUM_ES_GPRS(18));
-		sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
-					   R600_NUM_VS_THREADS(78) |
-					   R600_NUM_GS_THREADS(4) |
-					   R600_NUM_ES_THREADS(31));
-		sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
-					    R600_NUM_VS_STACK_ENTRIES(40));
-		sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
-					    R600_NUM_ES_STACK_ENTRIES(16));
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670) {
-		sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
-					  R600_NUM_VS_GPRS(44) |
-					  R600_NUM_CLAUSE_TEMP_GPRS(2));
-		sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
-					  R600_NUM_ES_GPRS(17));
-		sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
-					   R600_NUM_VS_THREADS(78) |
-					   R600_NUM_GS_THREADS(4) |
-					   R600_NUM_ES_THREADS(31));
-		sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(64) |
-					    R600_NUM_VS_STACK_ENTRIES(64));
-		sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(64) |
-					    R600_NUM_ES_STACK_ENTRIES(64));
-	}
-
-	RADEON_WRITE(R600_SQ_CONFIG, sq_config);
-	RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
-	RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
-	RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
-	RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
-	RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
-		RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_TC_ONLY));
-	else
-		RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_VC_AND_TC));
-
-	RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_2S, (R600_S0_X(0xc) |
-						    R600_S0_Y(0x4) |
-						    R600_S1_X(0x4) |
-						    R600_S1_Y(0xc)));
-	RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_4S, (R600_S0_X(0xe) |
-						    R600_S0_Y(0xe) |
-						    R600_S1_X(0x2) |
-						    R600_S1_Y(0x2) |
-						    R600_S2_X(0xa) |
-						    R600_S2_Y(0x6) |
-						    R600_S3_X(0x6) |
-						    R600_S3_Y(0xa)));
-	RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0, (R600_S0_X(0xe) |
-							R600_S0_Y(0xb) |
-							R600_S1_X(0x4) |
-							R600_S1_Y(0xc) |
-							R600_S2_X(0x1) |
-							R600_S2_Y(0x6) |
-							R600_S3_X(0xa) |
-							R600_S3_Y(0xe)));
-	RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1, (R600_S4_X(0x6) |
-							R600_S4_Y(0x1) |
-							R600_S5_X(0x0) |
-							R600_S5_Y(0x0) |
-							R600_S6_X(0xb) |
-							R600_S6_Y(0x4) |
-							R600_S7_X(0x7) |
-							R600_S7_Y(0x8)));
-
-
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_R600:
-	case CHIP_RV630:
-	case CHIP_RV635:
-		gs_prim_buffer_depth = 0;
-		break;
-	case CHIP_RV610:
-	case CHIP_RS780:
-	case CHIP_RS880:
-	case CHIP_RV620:
-		gs_prim_buffer_depth = 32;
-		break;
-	case CHIP_RV670:
-		gs_prim_buffer_depth = 128;
-		break;
-	default:
-		break;
-	}
-
-	num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
-	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
-	/* Max value for this is 256 */
-	if (vgt_gs_per_es > 256)
-		vgt_gs_per_es = 256;
-
-	RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
-	RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
-	RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
-	RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
-
-	/* more default values. 2D/3D driver should adjust as needed */
-	RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
-	RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
-	RADEON_WRITE(R600_SX_MISC, 0);
-	RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
-	RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
-	RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
-	RADEON_WRITE(R600_SPI_INPUT_Z, 0);
-	RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
-	RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
-
-	/* clear render buffer base addresses */
-	RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
-
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV610:
-	case CHIP_RS780:
-	case CHIP_RS880:
-	case CHIP_RV620:
-		tc_cntl = R600_TC_L2_SIZE(8);
-		break;
-	case CHIP_RV630:
-	case CHIP_RV635:
-		tc_cntl = R600_TC_L2_SIZE(4);
-		break;
-	case CHIP_R600:
-		tc_cntl = R600_TC_L2_SIZE(0) | R600_L2_DISABLE_LATE_HIT;
-		break;
-	default:
-		tc_cntl = R600_TC_L2_SIZE(0);
-		break;
-	}
-
-	RADEON_WRITE(R600_TC_CNTL, tc_cntl);
-
-	hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
-	RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
-
-	arb_pop = RADEON_READ(R600_ARB_POP);
-	arb_pop |= R600_ENABLE_TC128;
-	RADEON_WRITE(R600_ARB_POP, arb_pop);
-
-	RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
-	RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
-					  R600_NUM_CLIP_SEQ(3)));
-	RADEON_WRITE(R600_PA_SC_ENHANCE, R600_FORCE_EOV_MAX_CLK_CNT(4095));
-
-}
-
-static u32 r700_get_tile_pipe_to_backend_map(drm_radeon_private_t *dev_priv,
-					     u32 num_tile_pipes,
-					     u32 num_backends,
-					     u32 backend_disable_mask)
-{
-	u32 backend_map = 0;
-	u32 enabled_backends_mask;
-	u32 enabled_backends_count;
-	u32 cur_pipe;
-	u32 swizzle_pipe[R7XX_MAX_PIPES];
-	u32 cur_backend;
-	u32 i;
-	bool force_no_swizzle;
-
-	if (num_tile_pipes > R7XX_MAX_PIPES)
-		num_tile_pipes = R7XX_MAX_PIPES;
-	if (num_tile_pipes < 1)
-		num_tile_pipes = 1;
-	if (num_backends > R7XX_MAX_BACKENDS)
-		num_backends = R7XX_MAX_BACKENDS;
-	if (num_backends < 1)
-		num_backends = 1;
-
-	enabled_backends_mask = 0;
-	enabled_backends_count = 0;
-	for (i = 0; i < R7XX_MAX_BACKENDS; ++i) {
-		if (((backend_disable_mask >> i) & 1) == 0) {
-			enabled_backends_mask |= (1 << i);
-			++enabled_backends_count;
-		}
-		if (enabled_backends_count == num_backends)
-			break;
-	}
-
-	if (enabled_backends_count == 0) {
-		enabled_backends_mask = 1;
-		enabled_backends_count = 1;
-	}
-
-	if (enabled_backends_count != num_backends)
-		num_backends = enabled_backends_count;
-
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV770:
-	case CHIP_RV730:
-		force_no_swizzle = false;
-		break;
-	case CHIP_RV710:
-	case CHIP_RV740:
-	default:
-		force_no_swizzle = true;
-		break;
-	}
-
-	memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R7XX_MAX_PIPES);
-	switch (num_tile_pipes) {
-	case 1:
-		swizzle_pipe[0] = 0;
-		break;
-	case 2:
-		swizzle_pipe[0] = 0;
-		swizzle_pipe[1] = 1;
-		break;
-	case 3:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 1;
-		}
-		break;
-	case 4:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-			swizzle_pipe[3] = 3;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 3;
-			swizzle_pipe[3] = 1;
-		}
-		break;
-	case 5:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-			swizzle_pipe[3] = 3;
-			swizzle_pipe[4] = 4;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 4;
-			swizzle_pipe[3] = 1;
-			swizzle_pipe[4] = 3;
-		}
-		break;
-	case 6:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-			swizzle_pipe[3] = 3;
-			swizzle_pipe[4] = 4;
-			swizzle_pipe[5] = 5;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 4;
-			swizzle_pipe[3] = 5;
-			swizzle_pipe[4] = 3;
-			swizzle_pipe[5] = 1;
-		}
-		break;
-	case 7:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-			swizzle_pipe[3] = 3;
-			swizzle_pipe[4] = 4;
-			swizzle_pipe[5] = 5;
-			swizzle_pipe[6] = 6;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 4;
-			swizzle_pipe[3] = 6;
-			swizzle_pipe[4] = 3;
-			swizzle_pipe[5] = 1;
-			swizzle_pipe[6] = 5;
-		}
-		break;
-	case 8:
-		if (force_no_swizzle) {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 1;
-			swizzle_pipe[2] = 2;
-			swizzle_pipe[3] = 3;
-			swizzle_pipe[4] = 4;
-			swizzle_pipe[5] = 5;
-			swizzle_pipe[6] = 6;
-			swizzle_pipe[7] = 7;
-		} else {
-			swizzle_pipe[0] = 0;
-			swizzle_pipe[1] = 2;
-			swizzle_pipe[2] = 4;
-			swizzle_pipe[3] = 6;
-			swizzle_pipe[4] = 3;
-			swizzle_pipe[5] = 1;
-			swizzle_pipe[6] = 7;
-			swizzle_pipe[7] = 5;
-		}
-		break;
-	}
-
-	cur_backend = 0;
-	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
-		while (((1 << cur_backend) & enabled_backends_mask) == 0)
-			cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
-
-		backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
-
-		cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
-	}
-
-	return backend_map;
-}
-
-static void r700_gfx_init(struct drm_device *dev,
-			  drm_radeon_private_t *dev_priv)
-{
-	int i, j, num_qd_pipes;
-	u32 ta_aux_cntl;
-	u32 sx_debug_1;
-	u32 smx_dc_ctl0;
-	u32 db_debug3;
-	u32 num_gs_verts_per_thread;
-	u32 vgt_gs_per_es;
-	u32 gs_prim_buffer_depth = 0;
-	u32 sq_ms_fifo_sizes;
-	u32 sq_config;
-	u32 sq_thread_resource_mgmt;
-	u32 hdp_host_path_cntl;
-	u32 sq_dyn_gpr_size_simd_ab_0;
-	u32 backend_map;
-	u32 gb_tiling_config = 0;
-	u32 cc_rb_backend_disable;
-	u32 cc_gc_shader_pipe_config;
-	u32 mc_arb_ramcfg;
-	u32 db_debug4;
-
-	/* setup chip specs */
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV770:
-		dev_priv->r600_max_pipes = 4;
-		dev_priv->r600_max_tile_pipes = 8;
-		dev_priv->r600_max_simds = 10;
-		dev_priv->r600_max_backends = 4;
-		dev_priv->r600_max_gprs = 256;
-		dev_priv->r600_max_threads = 248;
-		dev_priv->r600_max_stack_entries = 512;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 16 * 2;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 112;
-		dev_priv->r600_sq_num_cf_insts = 2;
-
-		dev_priv->r700_sx_num_of_sets = 7;
-		dev_priv->r700_sc_prim_fifo_size = 0xF9;
-		dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
-		dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
-		break;
-	case CHIP_RV730:
-		dev_priv->r600_max_pipes = 2;
-		dev_priv->r600_max_tile_pipes = 4;
-		dev_priv->r600_max_simds = 8;
-		dev_priv->r600_max_backends = 2;
-		dev_priv->r600_max_gprs = 128;
-		dev_priv->r600_max_threads = 248;
-		dev_priv->r600_max_stack_entries = 256;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 16 * 2;
-		dev_priv->r600_sx_max_export_size = 256;
-		dev_priv->r600_sx_max_export_pos_size = 32;
-		dev_priv->r600_sx_max_export_smx_size = 224;
-		dev_priv->r600_sq_num_cf_insts = 2;
-
-		dev_priv->r700_sx_num_of_sets = 7;
-		dev_priv->r700_sc_prim_fifo_size = 0xf9;
-		dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
-		dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
-		if (dev_priv->r600_sx_max_export_pos_size > 16) {
-			dev_priv->r600_sx_max_export_pos_size -= 16;
-			dev_priv->r600_sx_max_export_smx_size += 16;
-		}
-		break;
-	case CHIP_RV710:
-		dev_priv->r600_max_pipes = 2;
-		dev_priv->r600_max_tile_pipes = 2;
-		dev_priv->r600_max_simds = 2;
-		dev_priv->r600_max_backends = 1;
-		dev_priv->r600_max_gprs = 256;
-		dev_priv->r600_max_threads = 192;
-		dev_priv->r600_max_stack_entries = 256;
-		dev_priv->r600_max_hw_contexts = 4;
-		dev_priv->r600_max_gs_threads = 8 * 2;
-		dev_priv->r600_sx_max_export_size = 128;
-		dev_priv->r600_sx_max_export_pos_size = 16;
-		dev_priv->r600_sx_max_export_smx_size = 112;
-		dev_priv->r600_sq_num_cf_insts = 1;
-
-		dev_priv->r700_sx_num_of_sets = 7;
-		dev_priv->r700_sc_prim_fifo_size = 0x40;
-		dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
-		dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
-		break;
-	case CHIP_RV740:
-		dev_priv->r600_max_pipes = 4;
-		dev_priv->r600_max_tile_pipes = 4;
-		dev_priv->r600_max_simds = 8;
-		dev_priv->r600_max_backends = 4;
-		dev_priv->r600_max_gprs = 256;
-		dev_priv->r600_max_threads = 248;
-		dev_priv->r600_max_stack_entries = 512;
-		dev_priv->r600_max_hw_contexts = 8;
-		dev_priv->r600_max_gs_threads = 16 * 2;
-		dev_priv->r600_sx_max_export_size = 256;
-		dev_priv->r600_sx_max_export_pos_size = 32;
-		dev_priv->r600_sx_max_export_smx_size = 224;
-		dev_priv->r600_sq_num_cf_insts = 2;
-
-		dev_priv->r700_sx_num_of_sets = 7;
-		dev_priv->r700_sc_prim_fifo_size = 0x100;
-		dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
-		dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
-
-		if (dev_priv->r600_sx_max_export_pos_size > 16) {
-			dev_priv->r600_sx_max_export_pos_size -= 16;
-			dev_priv->r600_sx_max_export_smx_size += 16;
-		}
-		break;
-	default:
-		break;
-	}
-
-	/* Initialize HDP */
-	j = 0;
-	for (i = 0; i < 32; i++) {
-		RADEON_WRITE((0x2c14 + j), 0x00000000);
-		RADEON_WRITE((0x2c18 + j), 0x00000000);
-		RADEON_WRITE((0x2c1c + j), 0x00000000);
-		RADEON_WRITE((0x2c20 + j), 0x00000000);
-		RADEON_WRITE((0x2c24 + j), 0x00000000);
-		j += 0x18;
-	}
-
-	RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
-
-	/* setup tiling, simd, pipe config */
-	mc_arb_ramcfg = RADEON_READ(R700_MC_ARB_RAMCFG);
-
-	switch (dev_priv->r600_max_tile_pipes) {
-	case 1:
-		gb_tiling_config |= R600_PIPE_TILING(0);
-		break;
-	case 2:
-		gb_tiling_config |= R600_PIPE_TILING(1);
-		break;
-	case 4:
-		gb_tiling_config |= R600_PIPE_TILING(2);
-		break;
-	case 8:
-		gb_tiling_config |= R600_PIPE_TILING(3);
-		break;
-	default:
-		break;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)
-		gb_tiling_config |= R600_BANK_TILING(1);
-	else
-		gb_tiling_config |= R600_BANK_TILING((mc_arb_ramcfg >> R700_NOOFBANK_SHIFT) & R700_NOOFBANK_MASK);
-
-	gb_tiling_config |= R600_GROUP_SIZE(0);
-
-	if (((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK) > 3) {
-		gb_tiling_config |= R600_ROW_TILING(3);
-		gb_tiling_config |= R600_SAMPLE_SPLIT(3);
-	} else {
-		gb_tiling_config |=
-			R600_ROW_TILING(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
-		gb_tiling_config |=
-			R600_SAMPLE_SPLIT(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
-	}
-
-	gb_tiling_config |= R600_BANK_SWAPS(1);
-
-	cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
-	cc_rb_backend_disable |=
-		R600_BACKEND_DISABLE((R7XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R7XX_MAX_BACKENDS_MASK);
-
-	cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
-	cc_gc_shader_pipe_config |=
-		R600_INACTIVE_QD_PIPES((R7XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R7XX_MAX_PIPES_MASK);
-	cc_gc_shader_pipe_config |=
-		R600_INACTIVE_SIMDS((R7XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R7XX_MAX_SIMDS_MASK);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV740)
-		backend_map = 0x28;
-	else
-		backend_map = r700_get_tile_pipe_to_backend_map(dev_priv,
-								dev_priv->r600_max_tile_pipes,
-								(R7XX_MAX_BACKENDS -
-								 r600_count_pipe_bits((cc_rb_backend_disable &
-										       R7XX_MAX_BACKENDS_MASK) >> 16)),
-								(cc_rb_backend_disable >> 16));
-	gb_tiling_config |= R600_BACKEND_MAP(backend_map);
-
-	RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
-	RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
-	RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
-	if (gb_tiling_config & 0xc0) {
-		dev_priv->r600_group_size = 512;
-	} else {
-		dev_priv->r600_group_size = 256;
-	}
-	dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
-	if (gb_tiling_config & 0x30) {
-		dev_priv->r600_nbanks = 8;
-	} else {
-		dev_priv->r600_nbanks = 4;
-	}
-
-	RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
-	RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
-	RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
-
-	RADEON_WRITE(R700_CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
-	RADEON_WRITE(R700_CGTS_SYS_TCC_DISABLE, 0);
-	RADEON_WRITE(R700_CGTS_TCC_DISABLE, 0);
-	RADEON_WRITE(R700_CGTS_USER_SYS_TCC_DISABLE, 0);
-	RADEON_WRITE(R700_CGTS_USER_TCC_DISABLE, 0);
-
-	num_qd_pipes =
-		R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
-	RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
-	RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
-
-	/* set HW defaults for 3D engine */
-	RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
-						R600_ROQ_IB2_START(0x2b)));
-
-	RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, R700_STQ_SPLIT(0x30));
-
-	ta_aux_cntl = RADEON_READ(R600_TA_CNTL_AUX);
-	RADEON_WRITE(R600_TA_CNTL_AUX, ta_aux_cntl | R600_DISABLE_CUBE_ANISO);
-
-	sx_debug_1 = RADEON_READ(R700_SX_DEBUG_1);
-	sx_debug_1 |= R700_ENABLE_NEW_SMX_ADDRESS;
-	RADEON_WRITE(R700_SX_DEBUG_1, sx_debug_1);
-
-	smx_dc_ctl0 = RADEON_READ(R600_SMX_DC_CTL0);
-	smx_dc_ctl0 &= ~R700_CACHE_DEPTH(0x1ff);
-	smx_dc_ctl0 |= R700_CACHE_DEPTH((dev_priv->r700_sx_num_of_sets * 64) - 1);
-	RADEON_WRITE(R600_SMX_DC_CTL0, smx_dc_ctl0);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV740)
-		RADEON_WRITE(R700_SMX_EVENT_CTL, (R700_ES_FLUSH_CTL(4) |
-						  R700_GS_FLUSH_CTL(4) |
-						  R700_ACK_FLUSH_CTL(3) |
-						  R700_SYNC_FLUSH_CTL));
-
-	db_debug3 = RADEON_READ(R700_DB_DEBUG3);
-	db_debug3 &= ~R700_DB_CLK_OFF_DELAY(0x1f);
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV770:
-	case CHIP_RV740:
-		db_debug3 |= R700_DB_CLK_OFF_DELAY(0x1f);
-		break;
-	case CHIP_RV710:
-	case CHIP_RV730:
-	default:
-		db_debug3 |= R700_DB_CLK_OFF_DELAY(2);
-		break;
-	}
-	RADEON_WRITE(R700_DB_DEBUG3, db_debug3);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV770) {
-		db_debug4 = RADEON_READ(RV700_DB_DEBUG4);
-		db_debug4 |= RV700_DISABLE_TILE_COVERED_FOR_PS_ITER;
-		RADEON_WRITE(RV700_DB_DEBUG4, db_debug4);
-	}
-
-	RADEON_WRITE(R600_SX_EXPORT_BUFFER_SIZES, (R600_COLOR_BUFFER_SIZE((dev_priv->r600_sx_max_export_size / 4) - 1) |
-						   R600_POSITION_BUFFER_SIZE((dev_priv->r600_sx_max_export_pos_size / 4) - 1) |
-						   R600_SMX_BUFFER_SIZE((dev_priv->r600_sx_max_export_smx_size / 4) - 1)));
-
-	RADEON_WRITE(R700_PA_SC_FIFO_SIZE_R7XX, (R700_SC_PRIM_FIFO_SIZE(dev_priv->r700_sc_prim_fifo_size) |
-						 R700_SC_HIZ_TILE_FIFO_SIZE(dev_priv->r700_sc_hiz_tile_fifo_size) |
-						 R700_SC_EARLYZ_TILE_FIFO_SIZE(dev_priv->r700_sc_earlyz_tile_fifo_fize)));
-
-	RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
-
-	RADEON_WRITE(R600_VGT_NUM_INSTANCES, 1);
-
-	RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
-
-	RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(4));
-
-	RADEON_WRITE(R600_CP_PERFMON_CNTL, 0);
-
-	sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(16 * dev_priv->r600_sq_num_cf_insts) |
-			    R600_DONE_FIFO_HIWATER(0xe0) |
-			    R600_ALU_UPDATE_FIFO_HIWATER(0x8));
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV770:
-	case CHIP_RV730:
-	case CHIP_RV710:
-		sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x1);
-		break;
-	case CHIP_RV740:
-	default:
-		sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x4);
-		break;
-	}
-	RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
-
-	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
-	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
-	 */
-	sq_config = RADEON_READ(R600_SQ_CONFIG);
-	sq_config &= ~(R600_PS_PRIO(3) |
-		       R600_VS_PRIO(3) |
-		       R600_GS_PRIO(3) |
-		       R600_ES_PRIO(3));
-	sq_config |= (R600_DX9_CONSTS |
-		      R600_VC_ENABLE |
-		      R600_EXPORT_SRC_C |
-		      R600_PS_PRIO(0) |
-		      R600_VS_PRIO(1) |
-		      R600_GS_PRIO(2) |
-		      R600_ES_PRIO(3));
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
-		/* no vertex cache */
-		sq_config &= ~R600_VC_ENABLE;
-
-	RADEON_WRITE(R600_SQ_CONFIG, sq_config);
-
-	RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  (R600_NUM_PS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
-						    R600_NUM_VS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
-						    R600_NUM_CLAUSE_TEMP_GPRS(((dev_priv->r600_max_gprs * 24)/64)/2)));
-
-	RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  (R600_NUM_GS_GPRS((dev_priv->r600_max_gprs * 7)/64) |
-						    R600_NUM_ES_GPRS((dev_priv->r600_max_gprs * 7)/64)));
-
-	sq_thread_resource_mgmt = (R600_NUM_PS_THREADS((dev_priv->r600_max_threads * 4)/8) |
-				   R600_NUM_VS_THREADS((dev_priv->r600_max_threads * 2)/8) |
-				   R600_NUM_ES_THREADS((dev_priv->r600_max_threads * 1)/8));
-	if (((dev_priv->r600_max_threads * 1) / 8) > dev_priv->r600_max_gs_threads)
-		sq_thread_resource_mgmt |= R600_NUM_GS_THREADS(dev_priv->r600_max_gs_threads);
-	else
-		sq_thread_resource_mgmt |= R600_NUM_GS_THREADS((dev_priv->r600_max_gs_threads * 1)/8);
-	RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
-
-	RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, (R600_NUM_PS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
-						     R600_NUM_VS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
-
-	RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, (R600_NUM_GS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
-						     R600_NUM_ES_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
-
-	sq_dyn_gpr_size_simd_ab_0 = (R700_SIMDA_RING0((dev_priv->r600_max_gprs * 38)/64) |
-				     R700_SIMDA_RING1((dev_priv->r600_max_gprs * 38)/64) |
-				     R700_SIMDB_RING0((dev_priv->r600_max_gprs * 38)/64) |
-				     R700_SIMDB_RING1((dev_priv->r600_max_gprs * 38)/64));
-
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
-	RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
-
-	RADEON_WRITE(R700_PA_SC_FORCE_EOV_MAX_CNTS, (R700_FORCE_EOV_MAX_CLK_CNT(4095) |
-						     R700_FORCE_EOV_MAX_REZ_CNT(255)));
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
-		RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_TC_ONLY) |
-							   R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
-	else
-		RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_VC_AND_TC) |
-							   R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
-
-	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
-	case CHIP_RV770:
-	case CHIP_RV730:
-	case CHIP_RV740:
-		gs_prim_buffer_depth = 384;
-		break;
-	case CHIP_RV710:
-		gs_prim_buffer_depth = 128;
-		break;
-	default:
-		break;
-	}
-
-	num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
-	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
-	/* Max value for this is 256 */
-	if (vgt_gs_per_es > 256)
-		vgt_gs_per_es = 256;
-
-	RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
-	RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
-	RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
-
-	/* more default values. 2D/3D driver should adjust as needed */
-	RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
-	RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
-	RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
-	RADEON_WRITE(R600_SX_MISC, 0);
-	RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
-	RADEON_WRITE(R700_PA_SC_EDGERULE, 0xaaaaaaaa);
-	RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
-	RADEON_WRITE(R600_PA_SC_CLIPRECT_RULE, 0xffff);
-	RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
-	RADEON_WRITE(R600_SPI_INPUT_Z, 0);
-	RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
-	RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
-
-	/* clear render buffer base addresses */
-	RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
-	RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
-
-	RADEON_WRITE(R700_TCP_CNTL, 0);
-
-	hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
-	RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
-
-	RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
-
-	RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
-					  R600_NUM_CLIP_SEQ(3)));
-
-}
-
-static void r600_cp_init_ring_buffer(struct drm_device *dev,
-				       drm_radeon_private_t *dev_priv,
-				       struct drm_file *file_priv)
-{
-	struct drm_radeon_master_private *master_priv;
-	u32 ring_start;
-	u64 rptr_addr;
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
-		r700_gfx_init(dev, dev_priv);
-	else
-		r600_gfx_init(dev, dev_priv);
-
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
-	RADEON_READ(R600_GRBM_SOFT_RESET);
-	mdelay(15);
-	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
-
-
-	/* Set ring buffer size */
-#ifdef __BIG_ENDIAN
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     R600_BUF_SWAP_32BIT |
-		     R600_RB_NO_UPDATE |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#else
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     RADEON_RB_NO_UPDATE |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#endif
-
-	RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x0);
-
-	/* Set the write pointer delay */
-	RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
-
-#ifdef __BIG_ENDIAN
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     R600_BUF_SWAP_32BIT |
-		     R600_RB_NO_UPDATE |
-		     R600_RB_RPTR_WR_ENA |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#else
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     R600_RB_NO_UPDATE |
-		     R600_RB_RPTR_WR_ENA |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#endif
-
-	/* Initialize the ring buffer's read and write pointers */
-	RADEON_WRITE(R600_CP_RB_RPTR_WR, 0);
-	RADEON_WRITE(R600_CP_RB_WPTR, 0);
-	SET_RING_HEAD(dev_priv, 0);
-	dev_priv->ring.tail = 0;
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		rptr_addr = dev_priv->ring_rptr->offset
-			- dev->agp->base +
-			dev_priv->gart_vm_start;
-	} else
-#endif
-	{
-		rptr_addr = dev_priv->ring_rptr->offset
-			- ((unsigned long) dev->sg->virtual)
-			+ dev_priv->gart_vm_start;
-	}
-	RADEON_WRITE(R600_CP_RB_RPTR_ADDR, (rptr_addr & 0xfffffffc));
-	RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI, upper_32_bits(rptr_addr));
-
-#ifdef __BIG_ENDIAN
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     RADEON_BUF_SWAP_32BIT |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#else
-	RADEON_WRITE(R600_CP_RB_CNTL,
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#endif
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		/* XXX */
-		radeon_write_agp_base(dev_priv, dev->agp->base);
-
-		/* XXX */
-		radeon_write_agp_location(dev_priv,
-			     (((dev_priv->gart_vm_start - 1 +
-				dev_priv->gart_size) & 0xffff0000) |
-			      (dev_priv->gart_vm_start >> 16)));
-
-		ring_start = (dev_priv->cp_ring->offset
-			      - dev->agp->base
-			      + dev_priv->gart_vm_start);
-	} else
-#endif
-		ring_start = (dev_priv->cp_ring->offset
-			      - (unsigned long)dev->sg->virtual
-			      + dev_priv->gart_vm_start);
-
-	RADEON_WRITE(R600_CP_RB_BASE, ring_start >> 8);
-
-	RADEON_WRITE(R600_CP_ME_CNTL, 0xff);
-
-	RADEON_WRITE(R600_CP_DEBUG, (1 << 27) | (1 << 28));
-
-	/* Initialize the scratch register pointer.  This will cause
-	 * the scratch register values to be written out to memory
-	 * whenever they are updated.
-	 *
-	 * We simply put this behind the ring read pointer, this works
-	 * with PCI GART as well as (whatever kind of) AGP GART
-	 */
-	{
-		u64 scratch_addr;
-
-		scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR) & 0xFFFFFFFC;
-		scratch_addr |= ((u64)RADEON_READ(R600_CP_RB_RPTR_ADDR_HI)) << 32;
-		scratch_addr += R600_SCRATCH_REG_OFFSET;
-		scratch_addr >>= 8;
-		scratch_addr &= 0xffffffff;
-
-		RADEON_WRITE(R600_SCRATCH_ADDR, (uint32_t)scratch_addr);
-	}
-
-	RADEON_WRITE(R600_SCRATCH_UMSK, 0x7);
-
-	/* Turn on bus mastering */
-	radeon_enable_bm(dev_priv);
-
-	radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(0), 0);
-	RADEON_WRITE(R600_LAST_FRAME_REG, 0);
-
-	radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
-	RADEON_WRITE(R600_LAST_DISPATCH_REG, 0);
-
-	radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(2), 0);
-	RADEON_WRITE(R600_LAST_CLEAR_REG, 0);
-
-	/* reset sarea copies of these */
-	master_priv = file_priv->master->driver_priv;
-	if (master_priv->sarea_priv) {
-		master_priv->sarea_priv->last_frame = 0;
-		master_priv->sarea_priv->last_dispatch = 0;
-		master_priv->sarea_priv->last_clear = 0;
-	}
-
-	r600_do_wait_for_idle(dev_priv);
-
-}
-
-int r600_do_cleanup_cp(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	/* Make sure interrupts are disabled here because the uninstall ioctl
-	 * may not have been called from userspace and after dev_private
-	 * is freed, it's too late.
-	 */
-	if (dev->irq_enabled)
-		drm_irq_uninstall(dev);
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		if (dev_priv->cp_ring != NULL) {
-			drm_legacy_ioremapfree(dev_priv->cp_ring, dev);
-			dev_priv->cp_ring = NULL;
-		}
-		if (dev_priv->ring_rptr != NULL) {
-			drm_legacy_ioremapfree(dev_priv->ring_rptr, dev);
-			dev_priv->ring_rptr = NULL;
-		}
-		if (dev->agp_buffer_map != NULL) {
-			drm_legacy_ioremapfree(dev->agp_buffer_map, dev);
-			dev->agp_buffer_map = NULL;
-		}
-	} else
-#endif
-	{
-
-		if (dev_priv->gart_info.bus_addr)
-			r600_page_table_cleanup(dev, &dev_priv->gart_info);
-
-		if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB) {
-			drm_legacy_ioremapfree(&dev_priv->gart_info.mapping, dev);
-			dev_priv->gart_info.addr = NULL;
-		}
-	}
-	/* only clear to the start of flags */
-	memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
-
-	return 0;
-}
-
-int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
-		    struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-
-	DRM_DEBUG("\n");
-
-	mutex_init(&dev_priv->cs_mutex);
-	r600_cs_legacy_init();
-	/* if we require new memory map but we don't have it fail */
-	if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
-		DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
-		DRM_DEBUG("Forcing AGP card to PCI mode\n");
-		dev_priv->flags &= ~RADEON_IS_AGP;
-		/* The writeback test succeeds, but when writeback is enabled,
-		 * the ring buffer read ptr update fails after first 128 bytes.
-		 */
-		radeon_no_wb = 1;
-	} else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
-		 && !init->is_pci) {
-		DRM_DEBUG("Restoring AGP flag\n");
-		dev_priv->flags |= RADEON_IS_AGP;
-	}
-
-	dev_priv->usec_timeout = init->usec_timeout;
-	if (dev_priv->usec_timeout < 1 ||
-	    dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
-		DRM_DEBUG("TIMEOUT problem!\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	/* Enable vblank on CRTC1 for older X servers
-	 */
-	dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
-	dev_priv->do_boxes = 0;
-	dev_priv->cp_mode = init->cp_mode;
-
-	/* We don't support anything other than bus-mastering ring mode,
-	 * but the ring can be in either AGP or PCI space for the ring
-	 * read pointer.
-	 */
-	if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
-	    (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
-		DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	switch (init->fb_bpp) {
-	case 16:
-		dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
-		break;
-	case 32:
-	default:
-		dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
-		break;
-	}
-	dev_priv->front_offset = init->front_offset;
-	dev_priv->front_pitch = init->front_pitch;
-	dev_priv->back_offset = init->back_offset;
-	dev_priv->back_pitch = init->back_pitch;
-
-	dev_priv->ring_offset = init->ring_offset;
-	dev_priv->ring_rptr_offset = init->ring_rptr_offset;
-	dev_priv->buffers_offset = init->buffers_offset;
-	dev_priv->gart_textures_offset = init->gart_textures_offset;
-
-	master_priv->sarea = drm_legacy_getsarea(dev);
-	if (!master_priv->sarea) {
-		DRM_ERROR("could not find sarea!\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	dev_priv->cp_ring = drm_legacy_findmap(dev, init->ring_offset);
-	if (!dev_priv->cp_ring) {
-		DRM_ERROR("could not find cp ring region!\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-	dev_priv->ring_rptr = drm_legacy_findmap(dev, init->ring_rptr_offset);
-	if (!dev_priv->ring_rptr) {
-		DRM_ERROR("could not find ring read pointer!\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-	dev->agp_buffer_token = init->buffers_offset;
-	dev->agp_buffer_map = drm_legacy_findmap(dev, init->buffers_offset);
-	if (!dev->agp_buffer_map) {
-		DRM_ERROR("could not find dma buffer region!\n");
-		r600_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	if (init->gart_textures_offset) {
-		dev_priv->gart_textures =
-		    drm_legacy_findmap(dev, init->gart_textures_offset);
-		if (!dev_priv->gart_textures) {
-			DRM_ERROR("could not find GART texture region!\n");
-			r600_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-	}
-
-#if IS_ENABLED(CONFIG_AGP)
-	/* XXX */
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		drm_legacy_ioremap_wc(dev_priv->cp_ring, dev);
-		drm_legacy_ioremap_wc(dev_priv->ring_rptr, dev);
-		drm_legacy_ioremap_wc(dev->agp_buffer_map, dev);
-		if (!dev_priv->cp_ring->handle ||
-		    !dev_priv->ring_rptr->handle ||
-		    !dev->agp_buffer_map->handle) {
-			DRM_ERROR("could not find ioremap agp regions!\n");
-			r600_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-	} else
-#endif
-	{
-		dev_priv->cp_ring->handle = (void *)(unsigned long)dev_priv->cp_ring->offset;
-		dev_priv->ring_rptr->handle =
-			(void *)(unsigned long)dev_priv->ring_rptr->offset;
-		dev->agp_buffer_map->handle =
-			(void *)(unsigned long)dev->agp_buffer_map->offset;
-
-		DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
-			  dev_priv->cp_ring->handle);
-		DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
-			  dev_priv->ring_rptr->handle);
-		DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
-			  dev->agp_buffer_map->handle);
-	}
-
-	dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 24;
-	dev_priv->fb_size =
-		(((radeon_read_fb_location(dev_priv) & 0xffff0000u) << 8) + 0x1000000)
-		- dev_priv->fb_location;
-
-	dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
-					((dev_priv->front_offset
-					  + dev_priv->fb_location) >> 10));
-
-	dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
-				       ((dev_priv->back_offset
-					 + dev_priv->fb_location) >> 10));
-
-	dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
-					((dev_priv->depth_offset
-					  + dev_priv->fb_location) >> 10));
-
-	dev_priv->gart_size = init->gart_size;
-
-	/* New let's set the memory map ... */
-	if (dev_priv->new_memmap) {
-		u32 base = 0;
-
-		DRM_INFO("Setting GART location based on new memory map\n");
-
-		/* If using AGP, try to locate the AGP aperture at the same
-		 * location in the card and on the bus, though we have to
-		 * align it down.
-		 */
-#if IS_ENABLED(CONFIG_AGP)
-		/* XXX */
-		if (dev_priv->flags & RADEON_IS_AGP) {
-			base = dev->agp->base;
-			/* Check if valid */
-			if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
-			    base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
-				DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
-					 dev->agp->base);
-				base = 0;
-			}
-		}
-#endif
-		/* If not or if AGP is at 0 (Macs), try to put it elsewhere */
-		if (base == 0) {
-			base = dev_priv->fb_location + dev_priv->fb_size;
-			if (base < dev_priv->fb_location ||
-			    ((base + dev_priv->gart_size) & 0xfffffffful) < base)
-				base = dev_priv->fb_location
-					- dev_priv->gart_size;
-		}
-		dev_priv->gart_vm_start = base & 0xffc00000u;
-		if (dev_priv->gart_vm_start != base)
-			DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
-				 base, dev_priv->gart_vm_start);
-	}
-
-#if IS_ENABLED(CONFIG_AGP)
-	/* XXX */
-	if (dev_priv->flags & RADEON_IS_AGP)
-		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-						 - dev->agp->base
-						 + dev_priv->gart_vm_start);
-	else
-#endif
-		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-						 - (unsigned long)dev->sg->virtual
-						 + dev_priv->gart_vm_start);
-
-	DRM_DEBUG("fb 0x%08x size %d\n",
-		  (unsigned int) dev_priv->fb_location,
-		  (unsigned int) dev_priv->fb_size);
-	DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
-	DRM_DEBUG("dev_priv->gart_vm_start 0x%08x\n",
-		  (unsigned int) dev_priv->gart_vm_start);
-	DRM_DEBUG("dev_priv->gart_buffers_offset 0x%08lx\n",
-		  dev_priv->gart_buffers_offset);
-
-	dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
-	dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
-			      + init->ring_size / sizeof(u32));
-	dev_priv->ring.size = init->ring_size;
-	dev_priv->ring.size_l2qw = order_base_2(init->ring_size / 8);
-
-	dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
-	dev_priv->ring.rptr_update_l2qw = order_base_2(/* init->rptr_update */ 4096 / 8);
-
-	dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
-	dev_priv->ring.fetch_size_l2ow = order_base_2(/* init->fetch_size */ 32 / 16);
-
-	dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
-
-	dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		/* XXX turn off pcie gart */
-	} else
-#endif
-	{
-		dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
-		/* if we have an offset set from userspace */
-		if (!dev_priv->pcigart_offset_set) {
-			DRM_ERROR("Need gart offset from userspace\n");
-			r600_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-
-		DRM_DEBUG("Using gart offset 0x%08lx\n", dev_priv->pcigart_offset);
-
-		dev_priv->gart_info.bus_addr =
-			dev_priv->pcigart_offset + dev_priv->fb_location;
-		dev_priv->gart_info.mapping.offset =
-			dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
-		dev_priv->gart_info.mapping.size =
-			dev_priv->gart_info.table_size;
-
-		drm_legacy_ioremap_wc(&dev_priv->gart_info.mapping, dev);
-		if (!dev_priv->gart_info.mapping.handle) {
-			DRM_ERROR("ioremap failed.\n");
-			r600_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-
-		dev_priv->gart_info.addr =
-			dev_priv->gart_info.mapping.handle;
-
-		DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
-			  dev_priv->gart_info.addr,
-			  dev_priv->pcigart_offset);
-
-		if (!r600_page_table_init(dev)) {
-			DRM_ERROR("Failed to init GART table\n");
-			r600_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-
-		if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
-			r700_vm_init(dev);
-		else
-			r600_vm_init(dev);
-	}
-
-	if (!dev_priv->me_fw || !dev_priv->pfp_fw) {
-		int err = r600_cp_init_microcode(dev_priv);
-		if (err) {
-			DRM_ERROR("Failed to load firmware!\n");
-			r600_do_cleanup_cp(dev);
-			return err;
-		}
-	}
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
-		r700_cp_load_microcode(dev_priv);
-	else
-		r600_cp_load_microcode(dev_priv);
-
-	r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
-
-	dev_priv->last_buf = 0;
-
-	r600_do_engine_reset(dev);
-	r600_test_writeback(dev_priv);
-
-	return 0;
-}
-
-int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	DRM_DEBUG("\n");
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)) {
-		r700_vm_init(dev);
-		r700_cp_load_microcode(dev_priv);
-	} else {
-		r600_vm_init(dev);
-		r600_cp_load_microcode(dev_priv);
-	}
-	r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
-	r600_do_engine_reset(dev);
-
-	return 0;
-}
-
-/* Wait for the CP to go idle.
- */
-int r600_do_cp_idle(drm_radeon_private_t *dev_priv)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(5);
-	OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
-	OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
-	/* wait for 3D idle clean */
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
-	OUT_RING((R600_WAIT_UNTIL - R600_SET_CONFIG_REG_OFFSET) >> 2);
-	OUT_RING(RADEON_WAIT_3D_IDLE | RADEON_WAIT_3D_IDLECLEAN);
-
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	return r600_do_wait_for_idle(dev_priv);
-}
-
-/* Start the Command Processor.
- */
-void r600_do_cp_start(drm_radeon_private_t *dev_priv)
-{
-	u32 cp_me;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(7);
-	OUT_RING(CP_PACKET3(R600_IT_ME_INITIALIZE, 5));
-	OUT_RING(0x00000001);
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV770))
-		OUT_RING(0x00000003);
-	else
-		OUT_RING(0x00000000);
-	OUT_RING((dev_priv->r600_max_hw_contexts - 1));
-	OUT_RING(R600_ME_INITIALIZE_DEVICE_ID(1));
-	OUT_RING(0x00000000);
-	OUT_RING(0x00000000);
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	/* set the mux and reset the halt bit */
-	cp_me = 0xff;
-	RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
-
-	dev_priv->cp_running = 1;
-
-}
-
-void r600_do_cp_reset(drm_radeon_private_t *dev_priv)
-{
-	u32 cur_read_ptr;
-	DRM_DEBUG("\n");
-
-	cur_read_ptr = RADEON_READ(R600_CP_RB_RPTR);
-	RADEON_WRITE(R600_CP_RB_WPTR, cur_read_ptr);
-	SET_RING_HEAD(dev_priv, cur_read_ptr);
-	dev_priv->ring.tail = cur_read_ptr;
-}
-
-void r600_do_cp_stop(drm_radeon_private_t *dev_priv)
-{
-	uint32_t cp_me;
-
-	DRM_DEBUG("\n");
-
-	cp_me = 0xff | R600_CP_ME_HALT;
-
-	RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
-
-	dev_priv->cp_running = 0;
-}
-
-int r600_cp_dispatch_indirect(struct drm_device *dev,
-			      struct drm_buf *buf, int start, int end)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	RING_LOCALS;
-
-	if (start != end) {
-		unsigned long offset = (dev_priv->gart_buffers_offset
-					+ buf->offset + start);
-		int dwords = (end - start + 3) / sizeof(u32);
-
-		DRM_DEBUG("dwords:%d\n", dwords);
-		DRM_DEBUG("offset 0x%lx\n", offset);
-
-
-		/* Indirect buffer data must be a multiple of 16 dwords.
-		 * pad the data with a Type-2 CP packet.
-		 */
-		while (dwords & 0xf) {
-			u32 *data = (u32 *)
-			    ((char *)dev->agp_buffer_map->handle
-			     + buf->offset + start);
-			data[dwords++] = RADEON_CP_PACKET2;
-		}
-
-		/* Fire off the indirect buffer */
-		BEGIN_RING(4);
-		OUT_RING(CP_PACKET3(R600_IT_INDIRECT_BUFFER, 2));
-		OUT_RING((offset & 0xfffffffc));
-		OUT_RING((upper_32_bits(offset) & 0xff));
-		OUT_RING(dwords);
-		ADVANCE_RING();
-	}
-
-	return 0;
-}
-
-void r600_cp_dispatch_swap(struct drm_device *dev, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_master *master = file_priv->master;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	int nbox = sarea_priv->nbox;
-	struct drm_clip_rect *pbox = sarea_priv->boxes;
-	int i, cpp, src_pitch, dst_pitch;
-	uint64_t src, dst;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	if (dev_priv->color_fmt == RADEON_COLOR_FORMAT_ARGB8888)
-		cpp = 4;
-	else
-		cpp = 2;
-
-	if (sarea_priv->pfCurrentPage == 0) {
-		src_pitch = dev_priv->back_pitch;
-		dst_pitch = dev_priv->front_pitch;
-		src = dev_priv->back_offset + dev_priv->fb_location;
-		dst = dev_priv->front_offset + dev_priv->fb_location;
-	} else {
-		src_pitch = dev_priv->front_pitch;
-		dst_pitch = dev_priv->back_pitch;
-		src = dev_priv->front_offset + dev_priv->fb_location;
-		dst = dev_priv->back_offset + dev_priv->fb_location;
-	}
-
-	if (r600_prepare_blit_copy(dev, file_priv)) {
-		DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
-		return;
-	}
-	for (i = 0; i < nbox; i++) {
-		int x = pbox[i].x1;
-		int y = pbox[i].y1;
-		int w = pbox[i].x2 - x;
-		int h = pbox[i].y2 - y;
-
-		DRM_DEBUG("%d,%d-%d,%d\n", x, y, w, h);
-
-		r600_blit_swap(dev,
-			       src, dst,
-			       x, y, x, y, w, h,
-			       src_pitch, dst_pitch, cpp);
-	}
-	r600_done_blit_copy(dev);
-
-	/* Increment the frame counter.  The client-side 3D driver must
-	 * throttle the framerate by waiting for this value before
-	 * performing the swapbuffer ioctl.
-	 */
-	sarea_priv->last_frame++;
-
-	BEGIN_RING(3);
-	R600_FRAME_AGE(sarea_priv->last_frame);
-	ADVANCE_RING();
-}
-
-int r600_cp_dispatch_texture(struct drm_device *dev,
-			     struct drm_file *file_priv,
-			     drm_radeon_texture_t *tex,
-			     drm_radeon_tex_image_t *image)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_buf *buf;
-	u32 *buffer;
-	const u8 __user *data;
-	unsigned int size, pass_size;
-	u64 src_offset, dst_offset;
-
-	if (!radeon_check_offset(dev_priv, tex->offset)) {
-		DRM_ERROR("Invalid destination offset\n");
-		return -EINVAL;
-	}
-
-	/* this might fail for zero-sized uploads - are those illegal? */
-	if (!radeon_check_offset(dev_priv, tex->offset + tex->height * tex->pitch - 1)) {
-		DRM_ERROR("Invalid final destination offset\n");
-		return -EINVAL;
-	}
-
-	size = tex->height * tex->pitch;
-
-	if (size == 0)
-		return 0;
-
-	dst_offset = tex->offset;
-
-	if (r600_prepare_blit_copy(dev, file_priv)) {
-		DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
-		return -EAGAIN;
-	}
-	do {
-		data = (const u8 __user *)image->data;
-		pass_size = size;
-
-		buf = radeon_freelist_get(dev);
-		if (!buf) {
-			DRM_DEBUG("EAGAIN\n");
-			if (copy_to_user(tex->image, image, sizeof(*image)))
-				return -EFAULT;
-			return -EAGAIN;
-		}
-
-		if (pass_size > buf->total)
-			pass_size = buf->total;
-
-		/* Dispatch the indirect buffer.
-		 */
-		buffer =
-		    (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
-
-		if (copy_from_user(buffer, data, pass_size)) {
-			DRM_ERROR("EFAULT on pad, %d bytes\n", pass_size);
-			return -EFAULT;
-		}
-
-		buf->file_priv = file_priv;
-		buf->used = pass_size;
-		src_offset = dev_priv->gart_buffers_offset + buf->offset;
-
-		r600_blit_copy(dev, src_offset, dst_offset, pass_size);
-
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-
-		/* Update the input parameters for next time */
-		image->data = (const u8 __user *)image->data + pass_size;
-		dst_offset += pass_size;
-		size -= pass_size;
-	} while (size > 0);
-	r600_done_blit_copy(dev);
-
-	return 0;
-}
-
-/*
- * Legacy cs ioctl
- */
-static u32 radeon_cs_id_get(struct drm_radeon_private *radeon)
-{
-	/* FIXME: check if wrap affect last reported wrap & sequence */
-	radeon->cs_id_scnt = (radeon->cs_id_scnt + 1) & 0x00FFFFFF;
-	if (!radeon->cs_id_scnt) {
-		/* increment wrap counter */
-		radeon->cs_id_wcnt += 0x01000000;
-		/* valid sequence counter start at 1 */
-		radeon->cs_id_scnt = 1;
-	}
-	return (radeon->cs_id_scnt | radeon->cs_id_wcnt);
-}
-
-static void r600_cs_id_emit(drm_radeon_private_t *dev_priv, u32 *id)
-{
-	RING_LOCALS;
-
-	*id = radeon_cs_id_get(dev_priv);
-
-	/* SCRATCH 2 */
-	BEGIN_RING(3);
-	R600_CLEAR_AGE(*id);
-	ADVANCE_RING();
-	COMMIT_RING();
-}
-
-static int r600_ib_get(struct drm_device *dev,
-			struct drm_file *fpriv,
-			struct drm_buf **buffer)
-{
-	struct drm_buf *buf;
-
-	*buffer = NULL;
-	buf = radeon_freelist_get(dev);
-	if (!buf) {
-		return -EBUSY;
-	}
-	buf->file_priv = fpriv;
-	*buffer = buf;
-	return 0;
-}
-
-static void r600_ib_free(struct drm_device *dev, struct drm_buf *buf,
-			struct drm_file *fpriv, int l, int r)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if (buf) {
-		if (!r)
-			r600_cp_dispatch_indirect(dev, buf, 0, l * 4);
-		radeon_cp_discard_buffer(dev, fpriv->master, buf);
-		COMMIT_RING();
-	}
-}
-
-int r600_cs_legacy_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
-{
-	struct drm_radeon_private *dev_priv = dev->dev_private;
-	struct drm_radeon_cs *cs = data;
-	struct drm_buf *buf;
-	unsigned family;
-	int l, r = 0;
-	u32 *ib, cs_id = 0;
-
-	if (dev_priv == NULL) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-	family = dev_priv->flags & RADEON_FAMILY_MASK;
-	if (family < CHIP_R600) {
-		DRM_ERROR("cs ioctl valid only for R6XX & R7XX in legacy mode\n");
-		return -EINVAL;
-	}
-	mutex_lock(&dev_priv->cs_mutex);
-	/* get ib */
-	r = r600_ib_get(dev, fpriv, &buf);
-	if (r) {
-		DRM_ERROR("ib_get failed\n");
-		goto out;
-	}
-	ib = dev->agp_buffer_map->handle + buf->offset;
-	/* now parse command stream */
-	r = r600_cs_legacy(dev, data,  fpriv, family, ib, &l);
-	if (r) {
-		goto out;
-	}
-
-out:
-	r600_ib_free(dev, buf, fpriv, l, r);
-	/* emit cs id sequence */
-	r600_cs_id_emit(dev_priv, &cs_id);
-	cs->cs_id = cs_id;
-	mutex_unlock(&dev_priv->cs_mutex);
-	return r;
-}
-
-void r600_cs_legacy_get_tiling_conf(struct drm_device *dev, u32 *npipes, u32 *nbanks, u32 *group_size)
-{
-	struct drm_radeon_private *dev_priv = dev->dev_private;
-
-	*npipes = dev_priv->r600_npipes;
-	*nbanks = dev_priv->r600_nbanks;
-	*group_size = dev_priv->r600_group_size;
-}
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index acc1f99c84d9..2f36fa1576e0 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -2328,101 +2328,6 @@ int r600_cs_parse(struct radeon_cs_parser *p)
 	return 0;
 }
 
-#ifdef CONFIG_DRM_RADEON_UMS
-
-/**
- * cs_parser_fini() - clean parser states
- * @parser:	parser structure holding parsing context.
- * @error:	error number
- *
- * If error is set than unvalidate buffer, otherwise just free memory
- * used by parsing context.
- **/
-static void r600_cs_parser_fini(struct radeon_cs_parser *parser, int error)
-{
-	unsigned i;
-
-	kfree(parser->relocs);
-	for (i = 0; i < parser->nchunks; i++)
-		drm_free_large(parser->chunks[i].kdata);
-	kfree(parser->chunks);
-	kfree(parser->chunks_array);
-}
-
-static int r600_cs_parser_relocs_legacy(struct radeon_cs_parser *p)
-{
-	if (p->chunk_relocs == NULL) {
-		return 0;
-	}
-	p->relocs = kzalloc(sizeof(struct radeon_bo_list), GFP_KERNEL);
-	if (p->relocs == NULL) {
-		return -ENOMEM;
-	}
-	return 0;
-}
-
-int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp,
-			unsigned family, u32 *ib, int *l)
-{
-	struct radeon_cs_parser parser;
-	struct radeon_cs_chunk *ib_chunk;
-	struct r600_cs_track *track;
-	int r;
-
-	/* initialize tracker */
-	track = kzalloc(sizeof(*track), GFP_KERNEL);
-	if (track == NULL)
-		return -ENOMEM;
-	r600_cs_track_init(track);
-	r600_cs_legacy_get_tiling_conf(dev, &track->npipes, &track->nbanks, &track->group_size);
-	/* initialize parser */
-	memset(&parser, 0, sizeof(struct radeon_cs_parser));
-	parser.filp = filp;
-	parser.dev = &dev->pdev->dev;
-	parser.rdev = NULL;
-	parser.family = family;
-	parser.track = track;
-	parser.ib.ptr = ib;
-	r = radeon_cs_parser_init(&parser, data);
-	if (r) {
-		DRM_ERROR("Failed to initialize parser !\n");
-		r600_cs_parser_fini(&parser, r);
-		return r;
-	}
-	r = r600_cs_parser_relocs_legacy(&parser);
-	if (r) {
-		DRM_ERROR("Failed to parse relocation !\n");
-		r600_cs_parser_fini(&parser, r);
-		return r;
-	}
-	/* Copy the packet into the IB, the parser will read from the
-	 * input memory (cached) and write to the IB (which can be
-	 * uncached). */
-	ib_chunk = parser.chunk_ib;
-	parser.ib.length_dw = ib_chunk->length_dw;
-	*l = parser.ib.length_dw;
-	if (copy_from_user(ib, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) {
-		r = -EFAULT;
-		r600_cs_parser_fini(&parser, r);
-		return r;
-	}
-	r = r600_cs_parse(&parser);
-	if (r) {
-		DRM_ERROR("Invalid command stream !\n");
-		r600_cs_parser_fini(&parser, r);
-		return r;
-	}
-	r600_cs_parser_fini(&parser, r);
-	return r;
-}
-
-void r600_cs_legacy_init(void)
-{
-	r600_nomm = 1;
-}
-
-#endif
-
 /*
  *  DMA
  */
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 87db64983ea8..78a51b3eda10 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -268,6 +268,7 @@ struct radeon_clock {
 	uint32_t current_dispclk;
 	uint32_t dp_extclk;
 	uint32_t max_pixel_clock;
+	uint32_t vco_freq;
 };
 
 /*
@@ -1889,7 +1890,7 @@ struct radeon_asic {
 		void (*pad_ib)(struct radeon_ib *ib);
 	} vm;
 	/* ring specific callbacks */
-	struct radeon_asic_ring *ring[RADEON_NUM_RINGS];
+	const struct radeon_asic_ring *ring[RADEON_NUM_RINGS];
 	/* irqs */
 	struct {
 		int (*set)(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 1d4d4520a0ac..7d5a36dd5094 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -179,7 +179,7 @@ void radeon_agp_disable(struct radeon_device *rdev)
  * ASIC
  */
 
-static struct radeon_asic_ring r100_gfx_ring = {
+static const struct radeon_asic_ring r100_gfx_ring = {
 	.ib_execute = &r100_ring_ib_execute,
 	.emit_fence = &r100_fence_ring_emit,
 	.emit_semaphore = &r100_semaphore_ring_emit,
@@ -329,7 +329,7 @@ static struct radeon_asic r200_asic = {
 	},
 };
 
-static struct radeon_asic_ring r300_gfx_ring = {
+static const struct radeon_asic_ring r300_gfx_ring = {
 	.ib_execute = &r100_ring_ib_execute,
 	.emit_fence = &r300_fence_ring_emit,
 	.emit_semaphore = &r100_semaphore_ring_emit,
@@ -343,7 +343,7 @@ static struct radeon_asic_ring r300_gfx_ring = {
 	.set_wptr = &r100_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring rv515_gfx_ring = {
+static const struct radeon_asic_ring rv515_gfx_ring = {
 	.ib_execute = &r100_ring_ib_execute,
 	.emit_fence = &r300_fence_ring_emit,
 	.emit_semaphore = &r100_semaphore_ring_emit,
@@ -901,7 +901,7 @@ static struct radeon_asic r520_asic = {
 	},
 };
 
-static struct radeon_asic_ring r600_gfx_ring = {
+static const struct radeon_asic_ring r600_gfx_ring = {
 	.ib_execute = &r600_ring_ib_execute,
 	.emit_fence = &r600_fence_ring_emit,
 	.emit_semaphore = &r600_semaphore_ring_emit,
@@ -914,7 +914,7 @@ static struct radeon_asic_ring r600_gfx_ring = {
 	.set_wptr = &r600_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring r600_dma_ring = {
+static const struct radeon_asic_ring r600_dma_ring = {
 	.ib_execute = &r600_dma_ring_ib_execute,
 	.emit_fence = &r600_dma_fence_ring_emit,
 	.emit_semaphore = &r600_dma_semaphore_ring_emit,
@@ -999,7 +999,7 @@ static struct radeon_asic r600_asic = {
 	},
 };
 
-static struct radeon_asic_ring rv6xx_uvd_ring = {
+static const struct radeon_asic_ring rv6xx_uvd_ring = {
 	.ib_execute = &uvd_v1_0_ib_execute,
 	.emit_fence = &uvd_v1_0_fence_emit,
 	.emit_semaphore = &uvd_v1_0_semaphore_emit,
@@ -1198,7 +1198,7 @@ static struct radeon_asic rs780_asic = {
 	},
 };
 
-static struct radeon_asic_ring rv770_uvd_ring = {
+static const struct radeon_asic_ring rv770_uvd_ring = {
 	.ib_execute = &uvd_v1_0_ib_execute,
 	.emit_fence = &uvd_v2_2_fence_emit,
 	.emit_semaphore = &uvd_v2_2_semaphore_emit,
@@ -1305,7 +1305,7 @@ static struct radeon_asic rv770_asic = {
 	},
 };
 
-static struct radeon_asic_ring evergreen_gfx_ring = {
+static const struct radeon_asic_ring evergreen_gfx_ring = {
 	.ib_execute = &evergreen_ring_ib_execute,
 	.emit_fence = &r600_fence_ring_emit,
 	.emit_semaphore = &r600_semaphore_ring_emit,
@@ -1318,7 +1318,7 @@ static struct radeon_asic_ring evergreen_gfx_ring = {
 	.set_wptr = &r600_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring evergreen_dma_ring = {
+static const struct radeon_asic_ring evergreen_dma_ring = {
 	.ib_execute = &evergreen_dma_ring_ib_execute,
 	.emit_fence = &evergreen_dma_fence_ring_emit,
 	.emit_semaphore = &r600_dma_semaphore_ring_emit,
@@ -1612,7 +1612,7 @@ static struct radeon_asic btc_asic = {
 	},
 };
 
-static struct radeon_asic_ring cayman_gfx_ring = {
+static const struct radeon_asic_ring cayman_gfx_ring = {
 	.ib_execute = &cayman_ring_ib_execute,
 	.ib_parse = &evergreen_ib_parse,
 	.emit_fence = &cayman_fence_ring_emit,
@@ -1627,7 +1627,7 @@ static struct radeon_asic_ring cayman_gfx_ring = {
 	.set_wptr = &cayman_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring cayman_dma_ring = {
+static const struct radeon_asic_ring cayman_dma_ring = {
 	.ib_execute = &cayman_dma_ring_ib_execute,
 	.ib_parse = &evergreen_dma_ib_parse,
 	.emit_fence = &evergreen_dma_fence_ring_emit,
@@ -1642,7 +1642,7 @@ static struct radeon_asic_ring cayman_dma_ring = {
 	.set_wptr = &cayman_dma_set_wptr
 };
 
-static struct radeon_asic_ring cayman_uvd_ring = {
+static const struct radeon_asic_ring cayman_uvd_ring = {
 	.ib_execute = &uvd_v1_0_ib_execute,
 	.emit_fence = &uvd_v2_2_fence_emit,
 	.emit_semaphore = &uvd_v3_1_semaphore_emit,
@@ -1760,7 +1760,7 @@ static struct radeon_asic cayman_asic = {
 	},
 };
 
-static struct radeon_asic_ring trinity_vce_ring = {
+static const struct radeon_asic_ring trinity_vce_ring = {
 	.ib_execute = &radeon_vce_ib_execute,
 	.emit_fence = &radeon_vce_fence_emit,
 	.emit_semaphore = &radeon_vce_semaphore_emit,
@@ -1881,7 +1881,7 @@ static struct radeon_asic trinity_asic = {
 	},
 };
 
-static struct radeon_asic_ring si_gfx_ring = {
+static const struct radeon_asic_ring si_gfx_ring = {
 	.ib_execute = &si_ring_ib_execute,
 	.ib_parse = &si_ib_parse,
 	.emit_fence = &si_fence_ring_emit,
@@ -1896,7 +1896,7 @@ static struct radeon_asic_ring si_gfx_ring = {
 	.set_wptr = &cayman_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring si_dma_ring = {
+static const struct radeon_asic_ring si_dma_ring = {
 	.ib_execute = &cayman_dma_ring_ib_execute,
 	.ib_parse = &evergreen_dma_ib_parse,
 	.emit_fence = &evergreen_dma_fence_ring_emit,
@@ -2023,7 +2023,7 @@ static struct radeon_asic si_asic = {
 	},
 };
 
-static struct radeon_asic_ring ci_gfx_ring = {
+static const struct radeon_asic_ring ci_gfx_ring = {
 	.ib_execute = &cik_ring_ib_execute,
 	.ib_parse = &cik_ib_parse,
 	.emit_fence = &cik_fence_gfx_ring_emit,
@@ -2038,7 +2038,7 @@ static struct radeon_asic_ring ci_gfx_ring = {
 	.set_wptr = &cik_gfx_set_wptr,
 };
 
-static struct radeon_asic_ring ci_cp_ring = {
+static const struct radeon_asic_ring ci_cp_ring = {
 	.ib_execute = &cik_ring_ib_execute,
 	.ib_parse = &cik_ib_parse,
 	.emit_fence = &cik_fence_compute_ring_emit,
@@ -2053,7 +2053,7 @@ static struct radeon_asic_ring ci_cp_ring = {
 	.set_wptr = &cik_compute_set_wptr,
 };
 
-static struct radeon_asic_ring ci_dma_ring = {
+static const struct radeon_asic_ring ci_dma_ring = {
 	.ib_execute = &cik_sdma_ring_ib_execute,
 	.ib_parse = &cik_ib_parse,
 	.emit_fence = &cik_sdma_fence_ring_emit,
@@ -2068,7 +2068,7 @@ static struct radeon_asic_ring ci_dma_ring = {
 	.set_wptr = &cik_sdma_set_wptr,
 };
 
-static struct radeon_asic_ring ci_vce_ring = {
+static const struct radeon_asic_ring ci_vce_ring = {
 	.ib_execute = &radeon_vce_ib_execute,
 	.emit_fence = &radeon_vce_fence_emit,
 	.emit_semaphore = &radeon_vce_semaphore_emit,
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index 8f285244c839..de9a2ffcf5f7 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -437,7 +437,9 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
 	}
 
 	/* Fujitsu D3003-S2 board lists DVI-I as DVI-D and VGA */
-	if (((dev->pdev->device == 0x9802) || (dev->pdev->device == 0x9806)) &&
+	if (((dev->pdev->device == 0x9802) ||
+	     (dev->pdev->device == 0x9805) ||
+	     (dev->pdev->device == 0x9806)) &&
 	    (dev->pdev->subsystem_vendor == 0x1734) &&
 	    (dev->pdev->subsystem_device == 0x11bd)) {
 		if (*connector_type == DRM_MODE_CONNECTOR_VGA) {
@@ -448,14 +450,6 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
 		}
 	}
 
-	/* Fujitsu D3003-S2 board lists DVI-I as DVI-I and VGA */
-	if ((dev->pdev->device == 0x9805) &&
-	    (dev->pdev->subsystem_vendor == 0x1734) &&
-	    (dev->pdev->subsystem_device == 0x11bd)) {
-		if (*connector_type == DRM_MODE_CONNECTOR_VGA)
-			return false;
-	}
-
 	return true;
 }
 
@@ -1112,6 +1106,31 @@ union firmware_info {
 	ATOM_FIRMWARE_INFO_V2_2 info_22;
 };
 
+union igp_info {
+	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
+	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
+	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
+	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
+	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8;
+};
+
+static void radeon_atombios_get_dentist_vco_freq(struct radeon_device *rdev)
+{
+	struct radeon_mode_info *mode_info = &rdev->mode_info;
+	int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
+	union igp_info *igp_info;
+	u8 frev, crev;
+	u16 data_offset;
+
+	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+			&frev, &crev, &data_offset)) {
+		igp_info = (union igp_info *)(mode_info->atom_context->bios +
+			data_offset);
+		rdev->clock.vco_freq =
+			le32_to_cpu(igp_info->info_6.ulDentistVCOFreq);
+	}
+}
+
 bool radeon_atom_get_clock_info(struct drm_device *dev)
 {
 	struct radeon_device *rdev = dev->dev_private;
@@ -1263,20 +1282,25 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
 		rdev->mode_info.firmware_flags =
 			le16_to_cpu(firmware_info->info.usFirmwareCapability.susAccess);
 
+		if (ASIC_IS_DCE8(rdev))
+			rdev->clock.vco_freq =
+				le32_to_cpu(firmware_info->info_22.ulGPUPLL_OutputFreq);
+		else if (ASIC_IS_DCE5(rdev))
+			rdev->clock.vco_freq = rdev->clock.current_dispclk;
+		else if (ASIC_IS_DCE41(rdev))
+			radeon_atombios_get_dentist_vco_freq(rdev);
+		else
+			rdev->clock.vco_freq = rdev->clock.current_dispclk;
+
+		if (rdev->clock.vco_freq == 0)
+			rdev->clock.vco_freq = 360000;	/* 3.6 GHz */
+
 		return true;
 	}
 
 	return false;
 }
 
-union igp_info {
-	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
-	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
-	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
-	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
-	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8;
-};
-
 bool radeon_atombios_sideport_present(struct radeon_device *rdev)
 {
 	struct radeon_mode_info *mode_info = &rdev->mode_info;
diff --git a/drivers/gpu/drm/radeon/radeon_audio.c b/drivers/gpu/drm/radeon/radeon_audio.c
index 2c02e99b5f95..b214663b370d 100644
--- a/drivers/gpu/drm/radeon/radeon_audio.c
+++ b/drivers/gpu/drm/radeon/radeon_audio.c
@@ -739,9 +739,6 @@ static void radeon_audio_dp_mode_set(struct drm_encoder *encoder,
 	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
 	struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
 	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
-	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-	struct radeon_connector_atom_dig *dig_connector =
-		radeon_connector->con_priv;
 
 	if (!dig || !dig->afmt)
 		return;
@@ -753,10 +750,7 @@ static void radeon_audio_dp_mode_set(struct drm_encoder *encoder,
 		radeon_audio_write_speaker_allocation(encoder);
 		radeon_audio_write_sad_regs(encoder);
 		radeon_audio_write_latency_fields(encoder, mode);
-		if (rdev->clock.dp_extclk || ASIC_IS_DCE5(rdev))
-			radeon_audio_set_dto(encoder, rdev->clock.default_dispclk * 10);
-		else
-			radeon_audio_set_dto(encoder, dig_connector->dp_clock);
+		radeon_audio_set_dto(encoder, rdev->clock.vco_freq * 10);
 		radeon_audio_set_audio_packet(encoder);
 		radeon_audio_select_pin(encoder);
 
@@ -781,3 +775,15 @@ void radeon_audio_dpms(struct drm_encoder *encoder, int mode)
 	if (radeon_encoder->audio && radeon_encoder->audio->dpms)
 		radeon_encoder->audio->dpms(encoder, mode == DRM_MODE_DPMS_ON);
 }
+
+unsigned int radeon_audio_decode_dfs_div(unsigned int div)
+{
+	if (div >= 8 && div < 64)
+		return (div - 8) * 25 + 200;
+	else if (div >= 64 && div < 96)
+		return (div - 64) * 50 + 1600;
+	else if (div >= 96 && div < 128)
+		return (div - 96) * 100 + 3200;
+	else
+		return 0;
+}
diff --git a/drivers/gpu/drm/radeon/radeon_audio.h b/drivers/gpu/drm/radeon/radeon_audio.h
index 059cc3012062..5c70cceaa4a6 100644
--- a/drivers/gpu/drm/radeon/radeon_audio.h
+++ b/drivers/gpu/drm/radeon/radeon_audio.h
@@ -79,5 +79,6 @@ void radeon_audio_fini(struct radeon_device *rdev);
 void radeon_audio_mode_set(struct drm_encoder *encoder,
 	struct drm_display_mode *mode);
 void radeon_audio_dpms(struct drm_encoder *encoder, int mode);
+unsigned int radeon_audio_decode_dfs_div(unsigned int div);
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
deleted file mode 100644
index 500287eff55d..000000000000
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ /dev/null
@@ -1,2243 +0,0 @@
-/* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
-/*
- * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
- * Copyright 2007 Advanced Micro Devices, Inc.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Kevin E. Martin <martin@valinux.com>
- *    Gareth Hughes <gareth@valinux.com>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <linux/module.h>
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-#include "r300_reg.h"
-
-#define RADEON_FIFO_DEBUG	0
-
-/* Firmware Names */
-#define FIRMWARE_R100		"radeon/R100_cp.bin"
-#define FIRMWARE_R200		"radeon/R200_cp.bin"
-#define FIRMWARE_R300		"radeon/R300_cp.bin"
-#define FIRMWARE_R420		"radeon/R420_cp.bin"
-#define FIRMWARE_RS690		"radeon/RS690_cp.bin"
-#define FIRMWARE_RS600		"radeon/RS600_cp.bin"
-#define FIRMWARE_R520		"radeon/R520_cp.bin"
-
-MODULE_FIRMWARE(FIRMWARE_R100);
-MODULE_FIRMWARE(FIRMWARE_R200);
-MODULE_FIRMWARE(FIRMWARE_R300);
-MODULE_FIRMWARE(FIRMWARE_R420);
-MODULE_FIRMWARE(FIRMWARE_RS690);
-MODULE_FIRMWARE(FIRMWARE_RS600);
-MODULE_FIRMWARE(FIRMWARE_R520);
-
-static int radeon_do_cleanup_cp(struct drm_device * dev);
-static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
-
-u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off)
-{
-	u32 val;
-
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		val = DRM_READ32(dev_priv->ring_rptr, off);
-	} else {
-		val = *(((volatile u32 *)
-			 dev_priv->ring_rptr->handle) +
-			(off / sizeof(u32)));
-		val = le32_to_cpu(val);
-	}
-	return val;
-}
-
-u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv)
-{
-	if (dev_priv->writeback_works)
-		return radeon_read_ring_rptr(dev_priv, 0);
-	else {
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			return RADEON_READ(R600_CP_RB_RPTR);
-		else
-			return RADEON_READ(RADEON_CP_RB_RPTR);
-	}
-}
-
-void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val)
-{
-	if (dev_priv->flags & RADEON_IS_AGP)
-		DRM_WRITE32(dev_priv->ring_rptr, off, val);
-	else
-		*(((volatile u32 *) dev_priv->ring_rptr->handle) +
-		  (off / sizeof(u32))) = cpu_to_le32(val);
-}
-
-void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val)
-{
-	radeon_write_ring_rptr(dev_priv, 0, val);
-}
-
-u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index)
-{
-	if (dev_priv->writeback_works) {
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			return radeon_read_ring_rptr(dev_priv,
-						     R600_SCRATCHOFF(index));
-		else
-			return radeon_read_ring_rptr(dev_priv,
-						     RADEON_SCRATCHOFF(index));
-	} else {
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			return RADEON_READ(R600_SCRATCH_REG0 + 4*index);
-		else
-			return RADEON_READ(RADEON_SCRATCH_REG0 + 4*index);
-	}
-}
-
-static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
-{
-	u32 ret;
-	RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
-	ret = RADEON_READ(R520_MC_IND_DATA);
-	RADEON_WRITE(R520_MC_IND_INDEX, 0);
-	return ret;
-}
-
-static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
-{
-	u32 ret;
-	RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
-	ret = RADEON_READ(RS480_NB_MC_DATA);
-	RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
-	return ret;
-}
-
-static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
-{
-	u32 ret;
-	RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
-	ret = RADEON_READ(RS690_MC_DATA);
-	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
-	return ret;
-}
-
-static u32 RS600_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
-{
-	u32 ret;
-	RADEON_WRITE(RS600_MC_INDEX, ((addr & RS600_MC_ADDR_MASK) |
-				      RS600_MC_IND_CITF_ARB0));
-	ret = RADEON_READ(RS600_MC_DATA);
-	return ret;
-}
-
-static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
-{
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
-		return RS690_READ_MCIND(dev_priv, addr);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-		return RS600_READ_MCIND(dev_priv, addr);
-	else
-		return RS480_READ_MCIND(dev_priv, addr);
-}
-
-u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
-{
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
-		return RADEON_READ(R700_MC_VM_FB_LOCATION);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return RADEON_READ(R600_MC_VM_FB_LOCATION);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
-		return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
-	else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
-		return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-		return RS600_READ_MCIND(dev_priv, RS600_MC_FB_LOCATION);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
-		return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
-	else
-		return RADEON_READ(RADEON_MC_FB_LOCATION);
-}
-
-static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
-{
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
-		RADEON_WRITE(R700_MC_VM_FB_LOCATION, fb_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		RADEON_WRITE(R600_MC_VM_FB_LOCATION, fb_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
-		R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
-	else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
-		RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-		RS600_WRITE_MCIND(RS600_MC_FB_LOCATION, fb_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
-		R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
-	else
-		RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
-}
-
-void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
-{
-	/*R6xx/R7xx: AGP_TOP and BOT are actually 18 bits each */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
-		RADEON_WRITE(R700_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
-		RADEON_WRITE(R700_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
-		RADEON_WRITE(R600_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
-		RADEON_WRITE(R600_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
-		R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
-	else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
-		RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-		RS600_WRITE_MCIND(RS600_MC_AGP_LOCATION, agp_loc);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
-		R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
-	else
-		RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
-}
-
-void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
-{
-	u32 agp_base_hi = upper_32_bits(agp_base);
-	u32 agp_base_lo = agp_base & 0xffffffff;
-	u32 r6xx_agp_base = (agp_base >> 22) & 0x3ffff;
-
-	/* R6xx/R7xx must be aligned to a 4MB boundary */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
-		RADEON_WRITE(R700_MC_VM_AGP_BASE, r6xx_agp_base);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		RADEON_WRITE(R600_MC_VM_AGP_BASE, r6xx_agp_base);
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
-		R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
-		R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
-		RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
-		RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
-		RS600_WRITE_MCIND(RS600_AGP_BASE, agp_base_lo);
-		RS600_WRITE_MCIND(RS600_AGP_BASE_2, agp_base_hi);
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
-		R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
-		R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
-		RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
-		RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
-	} else {
-		RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
-			RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
-	}
-}
-
-void radeon_enable_bm(struct drm_radeon_private *dev_priv)
-{
-	u32 tmp;
-	/* Turn on bus mastering */
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
-		/* rs600/rs690/rs740 */
-		tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
-		RADEON_WRITE(RADEON_BUS_CNTL, tmp);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV350) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
-		/* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
-		tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
-		RADEON_WRITE(RADEON_BUS_CNTL, tmp);
-	} /* PCIE cards appears to not need this */
-}
-
-static int RADEON_READ_PLL(struct drm_device * dev, int addr)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
-	return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
-}
-
-static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
-{
-	RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
-	return RADEON_READ(RADEON_PCIE_DATA);
-}
-
-#if RADEON_FIFO_DEBUG
-static void radeon_status(drm_radeon_private_t * dev_priv)
-{
-	printk("%s:\n", __func__);
-	printk("RBBM_STATUS = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
-	printk("CP_RB_RTPR = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
-	printk("CP_RB_WTPR = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
-	printk("AIC_CNTL = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
-	printk("AIC_STAT = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_AIC_STAT));
-	printk("AIC_PT_BASE = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
-	printk("TLB_ADDR = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
-	printk("TLB_DATA = 0x%08x\n",
-	       (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
-}
-#endif
-
-/* ================================================================
- * Engine, FIFO control
- */
-
-static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
-{
-	u32 tmp;
-	int i;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
-		tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
-		tmp |= RADEON_RB3D_DC_FLUSH_ALL;
-		RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
-
-		for (i = 0; i < dev_priv->usec_timeout; i++) {
-			if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
-			      & RADEON_RB3D_DC_BUSY)) {
-				return 0;
-			}
-			DRM_UDELAY(1);
-		}
-	} else {
-		/* don't flush or purge cache here or lockup */
-		return 0;
-	}
-
-#if RADEON_FIFO_DEBUG
-	DRM_ERROR("failed!\n");
-	radeon_status(dev_priv);
-#endif
-	return -EBUSY;
-}
-
-static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
-{
-	int i;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		int slots = (RADEON_READ(RADEON_RBBM_STATUS)
-			     & RADEON_RBBM_FIFOCNT_MASK);
-		if (slots >= entries)
-			return 0;
-		DRM_UDELAY(1);
-	}
-	DRM_DEBUG("wait for fifo failed status : 0x%08X 0x%08X\n",
-		 RADEON_READ(RADEON_RBBM_STATUS),
-		 RADEON_READ(R300_VAP_CNTL_STATUS));
-
-#if RADEON_FIFO_DEBUG
-	DRM_ERROR("failed!\n");
-	radeon_status(dev_priv);
-#endif
-	return -EBUSY;
-}
-
-static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
-{
-	int i, ret;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	ret = radeon_do_wait_for_fifo(dev_priv, 64);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		if (!(RADEON_READ(RADEON_RBBM_STATUS)
-		      & RADEON_RBBM_ACTIVE)) {
-			radeon_do_pixcache_flush(dev_priv);
-			return 0;
-		}
-		DRM_UDELAY(1);
-	}
-	DRM_DEBUG("wait idle failed status : 0x%08X 0x%08X\n",
-		 RADEON_READ(RADEON_RBBM_STATUS),
-		 RADEON_READ(R300_VAP_CNTL_STATUS));
-
-#if RADEON_FIFO_DEBUG
-	DRM_ERROR("failed!\n");
-	radeon_status(dev_priv);
-#endif
-	return -EBUSY;
-}
-
-static void radeon_init_pipes(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	uint32_t gb_tile_config, gb_pipe_sel = 0;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) {
-		uint32_t z_pipe_sel = RADEON_READ(RV530_GB_PIPE_SELECT2);
-		if ((z_pipe_sel & 3) == 3)
-			dev_priv->num_z_pipes = 2;
-		else
-			dev_priv->num_z_pipes = 1;
-	} else
-		dev_priv->num_z_pipes = 1;
-
-	/* RS4xx/RS6xx/R4xx/R5xx */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
-		gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
-		dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
-		/* SE cards have 1 pipe */
-		if ((dev->pdev->device == 0x5e4c) ||
-		    (dev->pdev->device == 0x5e4f))
-			dev_priv->num_gb_pipes = 1;
-	} else {
-		/* R3xx */
-		if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300 &&
-		     dev->pdev->device != 0x4144) ||
-		    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350 &&
-		     dev->pdev->device != 0x4148)) {
-			dev_priv->num_gb_pipes = 2;
-		} else {
-			/* RV3xx/R300 AD/R350 AH */
-			dev_priv->num_gb_pipes = 1;
-		}
-	}
-	DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
-
-	gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
-
-	switch (dev_priv->num_gb_pipes) {
-	case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
-	case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
-	case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
-	default:
-	case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
-		RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
-		RADEON_WRITE(R300_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
-	}
-	RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
-	radeon_do_wait_for_idle(dev_priv);
-	RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
-	RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
-					       R300_DC_AUTOFLUSH_ENABLE |
-					       R300_DC_DC_DISABLE_IGNORE_PE));
-
-
-}
-
-/* ================================================================
- * CP control, initialization
- */
-
-/* Load the microcode for the CP */
-static int radeon_cp_init_microcode(drm_radeon_private_t *dev_priv)
-{
-	struct platform_device *pdev;
-	const char *fw_name = NULL;
-	int err;
-
-	DRM_DEBUG("\n");
-
-	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
-	err = IS_ERR(pdev);
-	if (err) {
-		printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
-		return -EINVAL;
-	}
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
-		DRM_INFO("Loading R100 Microcode\n");
-		fw_name = FIRMWARE_R100;
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
-		DRM_INFO("Loading R200 Microcode\n");
-		fw_name = FIRMWARE_R200;
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
-		DRM_INFO("Loading R300 Microcode\n");
-		fw_name = FIRMWARE_R300;
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R423) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
-		DRM_INFO("Loading R400 Microcode\n");
-		fw_name = FIRMWARE_R420;
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
-		DRM_INFO("Loading RS690/RS740 Microcode\n");
-		fw_name = FIRMWARE_RS690;
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
-		DRM_INFO("Loading RS600 Microcode\n");
-		fw_name = FIRMWARE_RS600;
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
-		DRM_INFO("Loading R500 Microcode\n");
-		fw_name = FIRMWARE_R520;
-	}
-
-	err = request_firmware(&dev_priv->me_fw, fw_name, &pdev->dev);
-	platform_device_unregister(pdev);
-	if (err) {
-		printk(KERN_ERR "radeon_cp: Failed to load firmware \"%s\"\n",
-		       fw_name);
-	} else if (dev_priv->me_fw->size % 8) {
-		printk(KERN_ERR
-		       "radeon_cp: Bogus length %zu in firmware \"%s\"\n",
-		       dev_priv->me_fw->size, fw_name);
-		err = -EINVAL;
-		release_firmware(dev_priv->me_fw);
-		dev_priv->me_fw = NULL;
-	}
-	return err;
-}
-
-static void radeon_cp_load_microcode(drm_radeon_private_t *dev_priv)
-{
-	const __be32 *fw_data;
-	int i, size;
-
-	radeon_do_wait_for_idle(dev_priv);
-
-	if (dev_priv->me_fw) {
-		size = dev_priv->me_fw->size / 4;
-		fw_data = (const __be32 *)&dev_priv->me_fw->data[0];
-		RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
-		for (i = 0; i < size; i += 2) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     be32_to_cpup(&fw_data[i]));
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     be32_to_cpup(&fw_data[i + 1]));
-		}
-	}
-}
-
-/* Flush any pending commands to the CP.  This should only be used just
- * prior to a wait for idle, as it informs the engine that the command
- * stream is ending.
- */
-static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
-{
-	DRM_DEBUG("\n");
-#if 0
-	u32 tmp;
-
-	tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
-	RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
-#endif
-}
-
-/* Wait for the CP to go idle.
- */
-int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(6);
-
-	RADEON_PURGE_CACHE();
-	RADEON_PURGE_ZCACHE();
-	RADEON_WAIT_UNTIL_IDLE();
-
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	return radeon_do_wait_for_idle(dev_priv);
-}
-
-/* Start the Command Processor.
- */
-static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	radeon_do_wait_for_idle(dev_priv);
-
-	RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
-
-	dev_priv->cp_running = 1;
-
-	/* on r420, any DMA from CP to system memory while 2D is active
-	 * can cause a hang.  workaround is to queue a CP RESYNC token
-	 */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) {
-		BEGIN_RING(3);
-		OUT_RING(CP_PACKET0(R300_CP_RESYNC_ADDR, 1));
-		OUT_RING(5); /* scratch reg 5 */
-		OUT_RING(0xdeadbeef);
-		ADVANCE_RING();
-		COMMIT_RING();
-	}
-
-	BEGIN_RING(8);
-	/* isync can only be written through cp on r5xx write it here */
-	OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
-	OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
-		 RADEON_ISYNC_ANY3D_IDLE2D |
-		 RADEON_ISYNC_WAIT_IDLEGUI |
-		 RADEON_ISYNC_CPSCRATCH_IDLEGUI);
-	RADEON_PURGE_CACHE();
-	RADEON_PURGE_ZCACHE();
-	RADEON_WAIT_UNTIL_IDLE();
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
-}
-
-/* Reset the Command Processor.  This will not flush any pending
- * commands, so you must wait for the CP command stream to complete
- * before calling this routine.
- */
-static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
-{
-	u32 cur_read_ptr;
-	DRM_DEBUG("\n");
-
-	cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
-	RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
-	SET_RING_HEAD(dev_priv, cur_read_ptr);
-	dev_priv->ring.tail = cur_read_ptr;
-}
-
-/* Stop the Command Processor.  This will not flush any pending
- * commands, so you must flush the command stream and wait for the CP
- * to go idle before calling this routine.
- */
-static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
-{
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	/* finish the pending CP_RESYNC token */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) {
-		BEGIN_RING(2);
-		OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-		OUT_RING(R300_RB3D_DC_FINISH);
-		ADVANCE_RING();
-		COMMIT_RING();
-		radeon_do_wait_for_idle(dev_priv);
-	}
-
-	RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
-
-	dev_priv->cp_running = 0;
-}
-
-/* Reset the engine.  This will stop the CP if it is running.
- */
-static int radeon_do_engine_reset(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
-	DRM_DEBUG("\n");
-
-	radeon_do_pixcache_flush(dev_priv);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
-		/* may need something similar for newer chips */
-		clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
-		mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
-
-		RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
-						    RADEON_FORCEON_MCLKA |
-						    RADEON_FORCEON_MCLKB |
-						    RADEON_FORCEON_YCLKA |
-						    RADEON_FORCEON_YCLKB |
-						    RADEON_FORCEON_MC |
-						    RADEON_FORCEON_AIC));
-	}
-
-	rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
-
-	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
-					      RADEON_SOFT_RESET_CP |
-					      RADEON_SOFT_RESET_HI |
-					      RADEON_SOFT_RESET_SE |
-					      RADEON_SOFT_RESET_RE |
-					      RADEON_SOFT_RESET_PP |
-					      RADEON_SOFT_RESET_E2 |
-					      RADEON_SOFT_RESET_RB));
-	RADEON_READ(RADEON_RBBM_SOFT_RESET);
-	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
-					      ~(RADEON_SOFT_RESET_CP |
-						RADEON_SOFT_RESET_HI |
-						RADEON_SOFT_RESET_SE |
-						RADEON_SOFT_RESET_RE |
-						RADEON_SOFT_RESET_PP |
-						RADEON_SOFT_RESET_E2 |
-						RADEON_SOFT_RESET_RB)));
-	RADEON_READ(RADEON_RBBM_SOFT_RESET);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
-		RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
-		RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
-		RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
-	}
-
-	/* setup the raster pipes */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
-	    radeon_init_pipes(dev);
-
-	/* Reset the CP ring */
-	radeon_do_cp_reset(dev_priv);
-
-	/* The CP is no longer running after an engine reset */
-	dev_priv->cp_running = 0;
-
-	/* Reset any pending vertex, indirect buffers */
-	radeon_freelist_reset(dev);
-
-	return 0;
-}
-
-static void radeon_cp_init_ring_buffer(struct drm_device * dev,
-				       drm_radeon_private_t *dev_priv,
-				       struct drm_file *file_priv)
-{
-	struct drm_radeon_master_private *master_priv;
-	u32 ring_start, cur_read_ptr;
-
-	/* Initialize the memory controller. With new memory map, the fb location
-	 * is not changed, it should have been properly initialized already. Part
-	 * of the problem is that the code below is bogus, assuming the GART is
-	 * always appended to the fb which is not necessarily the case
-	 */
-	if (!dev_priv->new_memmap)
-		radeon_write_fb_location(dev_priv,
-			     ((dev_priv->gart_vm_start - 1) & 0xffff0000)
-			     | (dev_priv->fb_location >> 16));
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		radeon_write_agp_base(dev_priv, dev->agp->base);
-
-		radeon_write_agp_location(dev_priv,
-			     (((dev_priv->gart_vm_start - 1 +
-				dev_priv->gart_size) & 0xffff0000) |
-			      (dev_priv->gart_vm_start >> 16)));
-
-		ring_start = (dev_priv->cp_ring->offset
-			      - dev->agp->base
-			      + dev_priv->gart_vm_start);
-	} else
-#endif
-		ring_start = (dev_priv->cp_ring->offset
-			      - (unsigned long)dev->sg->virtual
-			      + dev_priv->gart_vm_start);
-
-	RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
-
-	/* Set the write pointer delay */
-	RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
-
-	/* Initialize the ring buffer's read and write pointers */
-	cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
-	RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
-	SET_RING_HEAD(dev_priv, cur_read_ptr);
-	dev_priv->ring.tail = cur_read_ptr;
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
-			     dev_priv->ring_rptr->offset
-			     - dev->agp->base + dev_priv->gart_vm_start);
-	} else
-#endif
-	{
-		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
-			     dev_priv->ring_rptr->offset
-			     - ((unsigned long) dev->sg->virtual)
-			     + dev_priv->gart_vm_start);
-	}
-
-	/* Set ring buffer size */
-#ifdef __BIG_ENDIAN
-	RADEON_WRITE(RADEON_CP_RB_CNTL,
-		     RADEON_BUF_SWAP_32BIT |
-		     (dev_priv->ring.fetch_size_l2ow << 18) |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#else
-	RADEON_WRITE(RADEON_CP_RB_CNTL,
-		     (dev_priv->ring.fetch_size_l2ow << 18) |
-		     (dev_priv->ring.rptr_update_l2qw << 8) |
-		     dev_priv->ring.size_l2qw);
-#endif
-
-
-	/* Initialize the scratch register pointer.  This will cause
-	 * the scratch register values to be written out to memory
-	 * whenever they are updated.
-	 *
-	 * We simply put this behind the ring read pointer, this works
-	 * with PCI GART as well as (whatever kind of) AGP GART
-	 */
-	RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
-		     + RADEON_SCRATCH_REG_OFFSET);
-
-	RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
-
-	radeon_enable_bm(dev_priv);
-
-	radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(0), 0);
-	RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
-
-	radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
-	RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
-
-	radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(2), 0);
-	RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
-
-	/* reset sarea copies of these */
-	master_priv = file_priv->master->driver_priv;
-	if (master_priv->sarea_priv) {
-		master_priv->sarea_priv->last_frame = 0;
-		master_priv->sarea_priv->last_dispatch = 0;
-		master_priv->sarea_priv->last_clear = 0;
-	}
-
-	radeon_do_wait_for_idle(dev_priv);
-
-	/* Sync everything up */
-	RADEON_WRITE(RADEON_ISYNC_CNTL,
-		     (RADEON_ISYNC_ANY2D_IDLE3D |
-		      RADEON_ISYNC_ANY3D_IDLE2D |
-		      RADEON_ISYNC_WAIT_IDLEGUI |
-		      RADEON_ISYNC_CPSCRATCH_IDLEGUI));
-
-}
-
-static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
-{
-	u32 tmp;
-
-	/* Start with assuming that writeback doesn't work */
-	dev_priv->writeback_works = 0;
-
-	/* Writeback doesn't seem to work everywhere, test it here and possibly
-	 * enable it if it appears to work
-	 */
-	radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
-
-	RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
-
-	for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
-		u32 val;
-
-		val = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
-		if (val == 0xdeadbeef)
-			break;
-		DRM_UDELAY(1);
-	}
-
-	if (tmp < dev_priv->usec_timeout) {
-		dev_priv->writeback_works = 1;
-		DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
-	} else {
-		dev_priv->writeback_works = 0;
-		DRM_INFO("writeback test failed\n");
-	}
-	if (radeon_no_wb == 1) {
-		dev_priv->writeback_works = 0;
-		DRM_INFO("writeback forced off\n");
-	}
-
-	if (!dev_priv->writeback_works) {
-		/* Disable writeback to avoid unnecessary bus master transfer */
-		RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) |
-			     RADEON_RB_NO_UPDATE);
-		RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
-	}
-}
-
-/* Enable or disable IGP GART on the chip */
-static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
-{
-	u32 temp;
-
-	if (on) {
-		DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
-			  dev_priv->gart_vm_start,
-			  (long)dev_priv->gart_info.bus_addr,
-			  dev_priv->gart_size);
-
-		temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
-		if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
-			IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
-							     RS690_BLOCK_GFX_D3_EN));
-		else
-			IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
-
-		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
-							       RS480_VA_SIZE_32MB));
-
-		temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
-		IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
-							RS480_TLB_ENABLE |
-							RS480_GTW_LAC_EN |
-							RS480_1LEVEL_GART));
-
-		temp = dev_priv->gart_info.bus_addr & 0xfffff000;
-		temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
-		IGP_WRITE_MCIND(RS480_GART_BASE, temp);
-
-		temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
-		IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
-						      RS480_REQ_TYPE_SNOOP_DIS));
-
-		radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
-
-		dev_priv->gart_size = 32*1024*1024;
-		temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
-			 0xffff0000) | (dev_priv->gart_vm_start >> 16));
-
-		radeon_write_agp_location(dev_priv, temp);
-
-		temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
-		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
-							       RS480_VA_SIZE_32MB));
-
-		do {
-			temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
-			if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
-				break;
-			DRM_UDELAY(1);
-		} while (1);
-
-		IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
-				RS480_GART_CACHE_INVALIDATE);
-
-		do {
-			temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
-			if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
-				break;
-			DRM_UDELAY(1);
-		} while (1);
-
-		IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
-	} else {
-		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
-	}
-}
-
-/* Enable or disable IGP GART on the chip */
-static void rs600_set_igpgart(drm_radeon_private_t *dev_priv, int on)
-{
-	u32 temp;
-	int i;
-
-	if (on) {
-		DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
-			 dev_priv->gart_vm_start,
-			 (long)dev_priv->gart_info.bus_addr,
-			 dev_priv->gart_size);
-
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (RS600_EFFECTIVE_L2_CACHE_SIZE(6) |
-						    RS600_EFFECTIVE_L2_QUEUE_SIZE(6)));
-
-		for (i = 0; i < 19; i++)
-			IGP_WRITE_MCIND(RS600_MC_PT0_CLIENT0_CNTL + i,
-					(RS600_ENABLE_TRANSLATION_MODE_OVERRIDE |
-					 RS600_SYSTEM_ACCESS_MODE_IN_SYS |
-					 RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASSTHROUGH |
-					 RS600_EFFECTIVE_L1_CACHE_SIZE(3) |
-					 RS600_ENABLE_FRAGMENT_PROCESSING |
-					 RS600_EFFECTIVE_L1_QUEUE_SIZE(3)));
-
-		IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL, (RS600_ENABLE_PAGE_TABLE |
-							     RS600_PAGE_TABLE_TYPE_FLAT));
-
-		/* disable all other contexts */
-		for (i = 1; i < 8; i++)
-			IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL + i, 0);
-
-		/* setup the page table aperture */
-		IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_BASE_ADDR,
-				dev_priv->gart_info.bus_addr);
-		IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_START_ADDR,
-				dev_priv->gart_vm_start);
-		IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_END_ADDR,
-				(dev_priv->gart_vm_start + dev_priv->gart_size - 1));
-		IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_DEFAULT_READ_ADDR, 0);
-
-		/* setup the system aperture */
-		IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_LOW_ADDR,
-				dev_priv->gart_vm_start);
-		IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_HIGH_ADDR,
-				(dev_priv->gart_vm_start + dev_priv->gart_size - 1));
-
-		/* enable page tables */
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (temp | RS600_ENABLE_PT));
-
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
-		IGP_WRITE_MCIND(RS600_MC_CNTL1, (temp | RS600_ENABLE_PAGE_TABLES));
-
-		/* invalidate the cache */
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
-
-		temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
-
-		temp |= RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE;
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
-
-		temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
-
-	} else {
-		IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, 0);
-		temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
-		temp &= ~RS600_ENABLE_PAGE_TABLES;
-		IGP_WRITE_MCIND(RS600_MC_CNTL1, temp);
-	}
-}
-
-static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
-{
-	u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
-	if (on) {
-
-		DRM_DEBUG("programming pcie %08X %08lX %08X\n",
-			  dev_priv->gart_vm_start,
-			  (long)dev_priv->gart_info.bus_addr,
-			  dev_priv->gart_size);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
-				  dev_priv->gart_vm_start);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
-				  dev_priv->gart_info.bus_addr);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
-				  dev_priv->gart_vm_start);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
-				  dev_priv->gart_vm_start +
-				  dev_priv->gart_size - 1);
-
-		radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
-
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
-				  RADEON_PCIE_TX_GART_EN);
-	} else {
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
-				  tmp & ~RADEON_PCIE_TX_GART_EN);
-	}
-}
-
-/* Enable or disable PCI GART on the chip */
-static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
-{
-	u32 tmp;
-
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740) ||
-	    (dev_priv->flags & RADEON_IS_IGPGART)) {
-		radeon_set_igpgart(dev_priv, on);
-		return;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
-		rs600_set_igpgart(dev_priv, on);
-		return;
-	}
-
-	if (dev_priv->flags & RADEON_IS_PCIE) {
-		radeon_set_pciegart(dev_priv, on);
-		return;
-	}
-
-	tmp = RADEON_READ(RADEON_AIC_CNTL);
-
-	if (on) {
-		RADEON_WRITE(RADEON_AIC_CNTL,
-			     tmp | RADEON_PCIGART_TRANSLATE_EN);
-
-		/* set PCI GART page-table base address
-		 */
-		RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
-
-		/* set address range for PCI address translate
-		 */
-		RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
-		RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
-			     + dev_priv->gart_size - 1);
-
-		/* Turn off AGP aperture -- is this required for PCI GART?
-		 */
-		radeon_write_agp_location(dev_priv, 0xffffffc0);
-		RADEON_WRITE(RADEON_AGP_COMMAND, 0);	/* clear AGP_COMMAND */
-	} else {
-		RADEON_WRITE(RADEON_AIC_CNTL,
-			     tmp & ~RADEON_PCIGART_TRANSLATE_EN);
-	}
-}
-
-static int radeon_setup_pcigart_surface(drm_radeon_private_t *dev_priv)
-{
-	struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
-	struct radeon_virt_surface *vp;
-	int i;
-
-	for (i = 0; i < RADEON_MAX_SURFACES * 2; i++) {
-		if (!dev_priv->virt_surfaces[i].file_priv ||
-		    dev_priv->virt_surfaces[i].file_priv == PCIGART_FILE_PRIV)
-			break;
-	}
-	if (i >= 2 * RADEON_MAX_SURFACES)
-		return -ENOMEM;
-	vp = &dev_priv->virt_surfaces[i];
-
-	for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-		struct radeon_surface *sp = &dev_priv->surfaces[i];
-		if (sp->refcount)
-			continue;
-
-		vp->surface_index = i;
-		vp->lower = gart_info->bus_addr;
-		vp->upper = vp->lower + gart_info->table_size;
-		vp->flags = 0;
-		vp->file_priv = PCIGART_FILE_PRIV;
-
-		sp->refcount = 1;
-		sp->lower = vp->lower;
-		sp->upper = vp->upper;
-		sp->flags = 0;
-
-		RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, sp->flags);
-		RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * i, sp->lower);
-		RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * i, sp->upper);
-		return 0;
-	}
-
-	return -ENOMEM;
-}
-
-static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
-			     struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-
-	DRM_DEBUG("\n");
-
-	/* if we require new memory map but we don't have it fail */
-	if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
-		DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
-		DRM_DEBUG("Forcing AGP card to PCI mode\n");
-		dev_priv->flags &= ~RADEON_IS_AGP;
-	} else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
-		   && !init->is_pci) {
-		DRM_DEBUG("Restoring AGP flag\n");
-		dev_priv->flags |= RADEON_IS_AGP;
-	}
-
-	if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
-		DRM_ERROR("PCI GART memory not allocated!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	dev_priv->usec_timeout = init->usec_timeout;
-	if (dev_priv->usec_timeout < 1 ||
-	    dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
-		DRM_DEBUG("TIMEOUT problem!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	/* Enable vblank on CRTC1 for older X servers
-	 */
-	dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
-
-	switch(init->func) {
-	case RADEON_INIT_R200_CP:
-		dev_priv->microcode_version = UCODE_R200;
-		break;
-	case RADEON_INIT_R300_CP:
-		dev_priv->microcode_version = UCODE_R300;
-		break;
-	default:
-		dev_priv->microcode_version = UCODE_R100;
-	}
-
-	dev_priv->do_boxes = 0;
-	dev_priv->cp_mode = init->cp_mode;
-
-	/* We don't support anything other than bus-mastering ring mode,
-	 * but the ring can be in either AGP or PCI space for the ring
-	 * read pointer.
-	 */
-	if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
-	    (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
-		DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	switch (init->fb_bpp) {
-	case 16:
-		dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
-		break;
-	case 32:
-	default:
-		dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
-		break;
-	}
-	dev_priv->front_offset = init->front_offset;
-	dev_priv->front_pitch = init->front_pitch;
-	dev_priv->back_offset = init->back_offset;
-	dev_priv->back_pitch = init->back_pitch;
-
-	switch (init->depth_bpp) {
-	case 16:
-		dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
-		break;
-	case 32:
-	default:
-		dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
-		break;
-	}
-	dev_priv->depth_offset = init->depth_offset;
-	dev_priv->depth_pitch = init->depth_pitch;
-
-	/* Hardware state for depth clears.  Remove this if/when we no
-	 * longer clear the depth buffer with a 3D rectangle.  Hard-code
-	 * all values to prevent unwanted 3D state from slipping through
-	 * and screwing with the clear operation.
-	 */
-	dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
-					   (dev_priv->color_fmt << 10) |
-					   (dev_priv->microcode_version ==
-					    UCODE_R100 ? RADEON_ZBLOCK16 : 0));
-
-	dev_priv->depth_clear.rb3d_zstencilcntl =
-	    (dev_priv->depth_fmt |
-	     RADEON_Z_TEST_ALWAYS |
-	     RADEON_STENCIL_TEST_ALWAYS |
-	     RADEON_STENCIL_S_FAIL_REPLACE |
-	     RADEON_STENCIL_ZPASS_REPLACE |
-	     RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
-
-	dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
-					 RADEON_BFACE_SOLID |
-					 RADEON_FFACE_SOLID |
-					 RADEON_FLAT_SHADE_VTX_LAST |
-					 RADEON_DIFFUSE_SHADE_FLAT |
-					 RADEON_ALPHA_SHADE_FLAT |
-					 RADEON_SPECULAR_SHADE_FLAT |
-					 RADEON_FOG_SHADE_FLAT |
-					 RADEON_VTX_PIX_CENTER_OGL |
-					 RADEON_ROUND_MODE_TRUNC |
-					 RADEON_ROUND_PREC_8TH_PIX);
-
-
-	dev_priv->ring_offset = init->ring_offset;
-	dev_priv->ring_rptr_offset = init->ring_rptr_offset;
-	dev_priv->buffers_offset = init->buffers_offset;
-	dev_priv->gart_textures_offset = init->gart_textures_offset;
-
-	master_priv->sarea = drm_legacy_getsarea(dev);
-	if (!master_priv->sarea) {
-		DRM_ERROR("could not find sarea!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	dev_priv->cp_ring = drm_legacy_findmap(dev, init->ring_offset);
-	if (!dev_priv->cp_ring) {
-		DRM_ERROR("could not find cp ring region!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-	dev_priv->ring_rptr = drm_legacy_findmap(dev, init->ring_rptr_offset);
-	if (!dev_priv->ring_rptr) {
-		DRM_ERROR("could not find ring read pointer!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-	dev->agp_buffer_token = init->buffers_offset;
-	dev->agp_buffer_map = drm_legacy_findmap(dev, init->buffers_offset);
-	if (!dev->agp_buffer_map) {
-		DRM_ERROR("could not find dma buffer region!\n");
-		radeon_do_cleanup_cp(dev);
-		return -EINVAL;
-	}
-
-	if (init->gart_textures_offset) {
-		dev_priv->gart_textures =
-		    drm_legacy_findmap(dev, init->gart_textures_offset);
-		if (!dev_priv->gart_textures) {
-			DRM_ERROR("could not find GART texture region!\n");
-			radeon_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-	}
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		drm_legacy_ioremap_wc(dev_priv->cp_ring, dev);
-		drm_legacy_ioremap_wc(dev_priv->ring_rptr, dev);
-		drm_legacy_ioremap_wc(dev->agp_buffer_map, dev);
-		if (!dev_priv->cp_ring->handle ||
-		    !dev_priv->ring_rptr->handle ||
-		    !dev->agp_buffer_map->handle) {
-			DRM_ERROR("could not find ioremap agp regions!\n");
-			radeon_do_cleanup_cp(dev);
-			return -EINVAL;
-		}
-	} else
-#endif
-	{
-		dev_priv->cp_ring->handle =
-			(void *)(unsigned long)dev_priv->cp_ring->offset;
-		dev_priv->ring_rptr->handle =
-			(void *)(unsigned long)dev_priv->ring_rptr->offset;
-		dev->agp_buffer_map->handle =
-			(void *)(unsigned long)dev->agp_buffer_map->offset;
-
-		DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
-			  dev_priv->cp_ring->handle);
-		DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
-			  dev_priv->ring_rptr->handle);
-		DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
-			  dev->agp_buffer_map->handle);
-	}
-
-	dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
-	dev_priv->fb_size =
-		((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
-		- dev_priv->fb_location;
-
-	dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
-					((dev_priv->front_offset
-					  + dev_priv->fb_location) >> 10));
-
-	dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
-				       ((dev_priv->back_offset
-					 + dev_priv->fb_location) >> 10));
-
-	dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
-					((dev_priv->depth_offset
-					  + dev_priv->fb_location) >> 10));
-
-	dev_priv->gart_size = init->gart_size;
-
-	/* New let's set the memory map ... */
-	if (dev_priv->new_memmap) {
-		u32 base = 0;
-
-		DRM_INFO("Setting GART location based on new memory map\n");
-
-		/* If using AGP, try to locate the AGP aperture at the same
-		 * location in the card and on the bus, though we have to
-		 * align it down.
-		 */
-#if IS_ENABLED(CONFIG_AGP)
-		if (dev_priv->flags & RADEON_IS_AGP) {
-			base = dev->agp->base;
-			/* Check if valid */
-			if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
-			    base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
-				DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
-					 dev->agp->base);
-				base = 0;
-			}
-		}
-#endif
-		/* If not or if AGP is at 0 (Macs), try to put it elsewhere */
-		if (base == 0) {
-			base = dev_priv->fb_location + dev_priv->fb_size;
-			if (base < dev_priv->fb_location ||
-			    ((base + dev_priv->gart_size) & 0xfffffffful) < base)
-				base = dev_priv->fb_location
-					- dev_priv->gart_size;
-		}
-		dev_priv->gart_vm_start = base & 0xffc00000u;
-		if (dev_priv->gart_vm_start != base)
-			DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
-				 base, dev_priv->gart_vm_start);
-	} else {
-		DRM_INFO("Setting GART location based on old memory map\n");
-		dev_priv->gart_vm_start = dev_priv->fb_location +
-			RADEON_READ(RADEON_CONFIG_APER_SIZE);
-	}
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP)
-		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-						 - dev->agp->base
-						 + dev_priv->gart_vm_start);
-	else
-#endif
-		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-					- (unsigned long)dev->sg->virtual
-					+ dev_priv->gart_vm_start);
-
-	DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
-	DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
-	DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
-		  dev_priv->gart_buffers_offset);
-
-	dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
-	dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
-			      + init->ring_size / sizeof(u32));
-	dev_priv->ring.size = init->ring_size;
-	dev_priv->ring.size_l2qw = order_base_2(init->ring_size / 8);
-
-	dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
-	dev_priv->ring.rptr_update_l2qw = order_base_2( /* init->rptr_update */ 4096 / 8);
-
-	dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
-	dev_priv->ring.fetch_size_l2ow = order_base_2( /* init->fetch_size */ 32 / 16);
-	dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
-
-	dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		/* Turn off PCI GART */
-		radeon_set_pcigart(dev_priv, 0);
-	} else
-#endif
-	{
-		u32 sctrl;
-		int ret;
-
-		dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
-		/* if we have an offset set from userspace */
-		if (dev_priv->pcigart_offset_set) {
-			dev_priv->gart_info.bus_addr =
-				(resource_size_t)dev_priv->pcigart_offset + dev_priv->fb_location;
-			dev_priv->gart_info.mapping.offset =
-			    dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
-			dev_priv->gart_info.mapping.size =
-			    dev_priv->gart_info.table_size;
-
-			drm_legacy_ioremap_wc(&dev_priv->gart_info.mapping, dev);
-			dev_priv->gart_info.addr =
-			    dev_priv->gart_info.mapping.handle;
-
-			if (dev_priv->flags & RADEON_IS_PCIE)
-				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
-			else
-				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
-			dev_priv->gart_info.gart_table_location =
-			    DRM_ATI_GART_FB;
-
-			DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
-				  dev_priv->gart_info.addr,
-				  dev_priv->pcigart_offset);
-		} else {
-			if (dev_priv->flags & RADEON_IS_IGPGART)
-				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
-			else
-				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
-			dev_priv->gart_info.gart_table_location =
-			    DRM_ATI_GART_MAIN;
-			dev_priv->gart_info.addr = NULL;
-			dev_priv->gart_info.bus_addr = 0;
-			if (dev_priv->flags & RADEON_IS_PCIE) {
-				DRM_ERROR
-				    ("Cannot use PCI Express without GART in FB memory\n");
-				radeon_do_cleanup_cp(dev);
-				return -EINVAL;
-			}
-		}
-
-		sctrl = RADEON_READ(RADEON_SURFACE_CNTL);
-		RADEON_WRITE(RADEON_SURFACE_CNTL, 0);
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-			ret = r600_page_table_init(dev);
-		else
-			ret = drm_ati_pcigart_init(dev, &dev_priv->gart_info);
-		RADEON_WRITE(RADEON_SURFACE_CNTL, sctrl);
-
-		if (!ret) {
-			DRM_ERROR("failed to init PCI GART!\n");
-			radeon_do_cleanup_cp(dev);
-			return -ENOMEM;
-		}
-
-		ret = radeon_setup_pcigart_surface(dev_priv);
-		if (ret) {
-			DRM_ERROR("failed to setup GART surface!\n");
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-				r600_page_table_cleanup(dev, &dev_priv->gart_info);
-			else
-				drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info);
-			radeon_do_cleanup_cp(dev);
-			return ret;
-		}
-
-		/* Turn on PCI GART */
-		radeon_set_pcigart(dev_priv, 1);
-	}
-
-	if (!dev_priv->me_fw) {
-		int err = radeon_cp_init_microcode(dev_priv);
-		if (err) {
-			DRM_ERROR("Failed to load firmware!\n");
-			radeon_do_cleanup_cp(dev);
-			return err;
-		}
-	}
-	radeon_cp_load_microcode(dev_priv);
-	radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
-
-	dev_priv->last_buf = 0;
-
-	radeon_do_engine_reset(dev);
-	radeon_test_writeback(dev_priv);
-
-	return 0;
-}
-
-static int radeon_do_cleanup_cp(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	/* Make sure interrupts are disabled here because the uninstall ioctl
-	 * may not have been called from userspace and after dev_private
-	 * is freed, it's too late.
-	 */
-	if (dev->irq_enabled)
-		drm_irq_uninstall(dev);
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		if (dev_priv->cp_ring != NULL) {
-			drm_legacy_ioremapfree(dev_priv->cp_ring, dev);
-			dev_priv->cp_ring = NULL;
-		}
-		if (dev_priv->ring_rptr != NULL) {
-			drm_legacy_ioremapfree(dev_priv->ring_rptr, dev);
-			dev_priv->ring_rptr = NULL;
-		}
-		if (dev->agp_buffer_map != NULL) {
-			drm_legacy_ioremapfree(dev->agp_buffer_map, dev);
-			dev->agp_buffer_map = NULL;
-		}
-	} else
-#endif
-	{
-
-		if (dev_priv->gart_info.bus_addr) {
-			/* Turn off PCI GART */
-			radeon_set_pcigart(dev_priv, 0);
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
-				r600_page_table_cleanup(dev, &dev_priv->gart_info);
-			else {
-				if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
-					DRM_ERROR("failed to cleanup PCI GART!\n");
-			}
-		}
-
-		if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
-		{
-			drm_legacy_ioremapfree(&dev_priv->gart_info.mapping, dev);
-			dev_priv->gart_info.addr = NULL;
-		}
-	}
-	/* only clear to the start of flags */
-	memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
-
-	return 0;
-}
-
-/* This code will reinit the Radeon CP hardware after a resume from disc.
- * AFAIK, it would be very difficult to pickle the state at suspend time, so
- * here we make sure that all Radeon hardware initialisation is re-done without
- * affecting running applications.
- *
- * Charl P. Botha <http://cpbotha.net>
- */
-static int radeon_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if (!dev_priv) {
-		DRM_ERROR("Called with no initialization\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG("Starting radeon_do_resume_cp()\n");
-
-#if IS_ENABLED(CONFIG_AGP)
-	if (dev_priv->flags & RADEON_IS_AGP) {
-		/* Turn off PCI GART */
-		radeon_set_pcigart(dev_priv, 0);
-	} else
-#endif
-	{
-		/* Turn on PCI GART */
-		radeon_set_pcigart(dev_priv, 1);
-	}
-
-	radeon_cp_load_microcode(dev_priv);
-	radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
-
-	dev_priv->have_z_offset = 0;
-	radeon_do_engine_reset(dev);
-	radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
-
-	DRM_DEBUG("radeon_do_resume_cp() complete\n");
-
-	return 0;
-}
-
-int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_init_t *init = data;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (init->func == RADEON_INIT_R300_CP)
-		r300_init_reg_flags(dev);
-
-	switch (init->func) {
-	case RADEON_INIT_CP:
-	case RADEON_INIT_R200_CP:
-	case RADEON_INIT_R300_CP:
-		return radeon_do_init_cp(dev, init, file_priv);
-	case RADEON_INIT_R600_CP:
-		return r600_do_init_cp(dev, init, file_priv);
-	case RADEON_CLEANUP_CP:
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			return r600_do_cleanup_cp(dev);
-		else
-			return radeon_do_cleanup_cp(dev);
-	}
-
-	return -EINVAL;
-}
-
-int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (dev_priv->cp_running) {
-		DRM_DEBUG("while CP running\n");
-		return 0;
-	}
-	if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
-		DRM_DEBUG("called with bogus CP mode (%d)\n",
-			  dev_priv->cp_mode);
-		return 0;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_do_cp_start(dev_priv);
-	else
-		radeon_do_cp_start(dev_priv);
-
-	return 0;
-}
-
-/* Stop the CP.  The engine must have been idled before calling this
- * routine.
- */
-int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_cp_stop_t *stop = data;
-	int ret;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (!dev_priv->cp_running)
-		return 0;
-
-	/* Flush any pending CP commands.  This ensures any outstanding
-	 * commands are exectuted by the engine before we turn it off.
-	 */
-	if (stop->flush) {
-		radeon_do_cp_flush(dev_priv);
-	}
-
-	/* If we fail to make the engine go idle, we return an error
-	 * code so that the DRM ioctl wrapper can try again.
-	 */
-	if (stop->idle) {
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			ret = r600_do_cp_idle(dev_priv);
-		else
-			ret = radeon_do_cp_idle(dev_priv);
-		if (ret)
-			return ret;
-	}
-
-	/* Finally, we can turn off the CP.  If the engine isn't idle,
-	 * we will get some dropped triangles as they won't be fully
-	 * rendered before the CP is shut down.
-	 */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_do_cp_stop(dev_priv);
-	else
-		radeon_do_cp_stop(dev_priv);
-
-	/* Reset the engine */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_do_engine_reset(dev);
-	else
-		radeon_do_engine_reset(dev);
-
-	return 0;
-}
-
-void radeon_do_release(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int i, ret;
-
-	if (dev_priv) {
-		if (dev_priv->cp_running) {
-			/* Stop the cp */
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
-				while ((ret = r600_do_cp_idle(dev_priv)) != 0) {
-					DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
-#ifdef __linux__
-					schedule();
-#else
-					tsleep(&ret, PZERO, "rdnrel", 1);
-#endif
-				}
-			} else {
-				while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
-					DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
-#ifdef __linux__
-					schedule();
-#else
-					tsleep(&ret, PZERO, "rdnrel", 1);
-#endif
-				}
-			}
-			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
-				r600_do_cp_stop(dev_priv);
-				r600_do_engine_reset(dev);
-			} else {
-				radeon_do_cp_stop(dev_priv);
-				radeon_do_engine_reset(dev);
-			}
-		}
-
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_R600) {
-			/* Disable *all* interrupts */
-			if (dev_priv->mmio)	/* remove this after permanent addmaps */
-				RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
-
-			if (dev_priv->mmio) {	/* remove all surfaces */
-				for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-					RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
-					RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
-						     16 * i, 0);
-					RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
-						     16 * i, 0);
-				}
-			}
-		}
-
-		/* Free memory heap structures */
-		radeon_mem_takedown(&(dev_priv->gart_heap));
-		radeon_mem_takedown(&(dev_priv->fb_heap));
-
-		/* deallocate kernel resources */
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			r600_do_cleanup_cp(dev);
-		else
-			radeon_do_cleanup_cp(dev);
-		release_firmware(dev_priv->me_fw);
-		dev_priv->me_fw = NULL;
-		release_firmware(dev_priv->pfp_fw);
-		dev_priv->pfp_fw = NULL;
-	}
-}
-
-/* Just reset the CP ring.  Called as part of an X Server engine reset.
- */
-int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (!dev_priv) {
-		DRM_DEBUG("called before init done\n");
-		return -EINVAL;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_do_cp_reset(dev_priv);
-	else
-		radeon_do_cp_reset(dev_priv);
-
-	/* The CP is no longer running after an engine reset */
-	dev_priv->cp_running = 0;
-
-	return 0;
-}
-
-int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return r600_do_cp_idle(dev_priv);
-	else
-		return radeon_do_cp_idle(dev_priv);
-}
-
-/* Added by Charl P. Botha to call radeon_do_resume_cp().
- */
-int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return r600_do_resume_cp(dev, file_priv);
-	else
-		return radeon_do_resume_cp(dev, file_priv);
-}
-
-int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return r600_do_engine_reset(dev);
-	else
-		return radeon_do_engine_reset(dev);
-}
-
-/* ================================================================
- * Fullscreen mode
- */
-
-/* KW: Deprecated to say the least:
- */
-int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	return 0;
-}
-
-/* ================================================================
- * Freelist management
- */
-
-/* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
- *   bufs until freelist code is used.  Note this hides a problem with
- *   the scratch register * (used to keep track of last buffer
- *   completed) being written to before * the last buffer has actually
- *   completed rendering.
- *
- * KW:  It's also a good way to find free buffers quickly.
- *
- * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
- * sleep.  However, bugs in older versions of radeon_accel.c mean that
- * we essentially have to do this, else old clients will break.
- *
- * However, it does leave open a potential deadlock where all the
- * buffers are held by other clients, which can't release them because
- * they can't get the lock.
- */
-
-struct drm_buf *radeon_freelist_get(struct drm_device * dev)
-{
-	struct drm_device_dma *dma = dev->dma;
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_buf_priv_t *buf_priv;
-	struct drm_buf *buf;
-	int i, t;
-	int start;
-
-	if (++dev_priv->last_buf >= dma->buf_count)
-		dev_priv->last_buf = 0;
-
-	start = dev_priv->last_buf;
-
-	for (t = 0; t < dev_priv->usec_timeout; t++) {
-		u32 done_age = GET_SCRATCH(dev_priv, 1);
-		DRM_DEBUG("done_age = %d\n", done_age);
-		for (i = 0; i < dma->buf_count; i++) {
-			buf = dma->buflist[start];
-			buf_priv = buf->dev_private;
-			if (buf->file_priv == NULL || (buf->pending &&
-						       buf_priv->age <=
-						       done_age)) {
-				dev_priv->stats.requested_bufs++;
-				buf->pending = 0;
-				return buf;
-			}
-			if (++start >= dma->buf_count)
-				start = 0;
-		}
-
-		if (t) {
-			DRM_UDELAY(1);
-			dev_priv->stats.freelist_loops++;
-		}
-	}
-
-	return NULL;
-}
-
-void radeon_freelist_reset(struct drm_device * dev)
-{
-	struct drm_device_dma *dma = dev->dma;
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int i;
-
-	dev_priv->last_buf = 0;
-	for (i = 0; i < dma->buf_count; i++) {
-		struct drm_buf *buf = dma->buflist[i];
-		drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
-		buf_priv->age = 0;
-	}
-}
-
-/* ================================================================
- * CP command submission
- */
-
-int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
-{
-	drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
-	int i;
-	u32 last_head = GET_RING_HEAD(dev_priv);
-
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		u32 head = GET_RING_HEAD(dev_priv);
-
-		ring->space = (head - ring->tail) * sizeof(u32);
-		if (ring->space <= 0)
-			ring->space += ring->size;
-		if (ring->space > n)
-			return 0;
-
-		dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-		if (head != last_head)
-			i = 0;
-		last_head = head;
-
-		DRM_UDELAY(1);
-	}
-
-	/* FIXME: This return value is ignored in the BEGIN_RING macro! */
-#if RADEON_FIFO_DEBUG
-	radeon_status(dev_priv);
-	DRM_ERROR("failed!\n");
-#endif
-	return -EBUSY;
-}
-
-static int radeon_cp_get_buffers(struct drm_device *dev,
-				 struct drm_file *file_priv,
-				 struct drm_dma * d)
-{
-	int i;
-	struct drm_buf *buf;
-
-	for (i = d->granted_count; i < d->request_count; i++) {
-		buf = radeon_freelist_get(dev);
-		if (!buf)
-			return -EBUSY;	/* NOTE: broken client */
-
-		buf->file_priv = file_priv;
-
-		if (copy_to_user(&d->request_indices[i], &buf->idx,
-				     sizeof(buf->idx)))
-			return -EFAULT;
-		if (copy_to_user(&d->request_sizes[i], &buf->total,
-				     sizeof(buf->total)))
-			return -EFAULT;
-
-		d->granted_count++;
-	}
-	return 0;
-}
-
-int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	struct drm_device_dma *dma = dev->dma;
-	int ret = 0;
-	struct drm_dma *d = data;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	/* Please don't send us buffers.
-	 */
-	if (d->send_count != 0) {
-		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d->send_count);
-		return -EINVAL;
-	}
-
-	/* We'll send you buffers.
-	 */
-	if (d->request_count < 0 || d->request_count > dma->buf_count) {
-		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d->request_count, dma->buf_count);
-		return -EINVAL;
-	}
-
-	d->granted_count = 0;
-
-	if (d->request_count) {
-		ret = radeon_cp_get_buffers(dev, file_priv, d);
-	}
-
-	return ret;
-}
-
-int radeon_driver_load(struct drm_device *dev, unsigned long flags)
-{
-	drm_radeon_private_t *dev_priv;
-	int ret = 0;
-
-	dev_priv = kzalloc(sizeof(drm_radeon_private_t), GFP_KERNEL);
-	if (dev_priv == NULL)
-		return -ENOMEM;
-
-	dev->dev_private = (void *)dev_priv;
-	dev_priv->flags = flags;
-
-	switch (flags & RADEON_FAMILY_MASK) {
-	case CHIP_R100:
-	case CHIP_RV200:
-	case CHIP_R200:
-	case CHIP_R300:
-	case CHIP_R350:
-	case CHIP_R420:
-	case CHIP_R423:
-	case CHIP_RV410:
-	case CHIP_RV515:
-	case CHIP_R520:
-	case CHIP_RV570:
-	case CHIP_R580:
-		dev_priv->flags |= RADEON_HAS_HIERZ;
-		break;
-	default:
-		/* all other chips have no hierarchical z buffer */
-		break;
-	}
-
-	pci_set_master(dev->pdev);
-
-	if (drm_pci_device_is_agp(dev))
-		dev_priv->flags |= RADEON_IS_AGP;
-	else if (pci_is_pcie(dev->pdev))
-		dev_priv->flags |= RADEON_IS_PCIE;
-	else
-		dev_priv->flags |= RADEON_IS_PCI;
-
-	ret = drm_legacy_addmap(dev, pci_resource_start(dev->pdev, 2),
-				pci_resource_len(dev->pdev, 2), _DRM_REGISTERS,
-				_DRM_READ_ONLY | _DRM_DRIVER, &dev_priv->mmio);
-	if (ret != 0)
-		return ret;
-
-	ret = drm_vblank_init(dev, 2);
-	if (ret) {
-		radeon_driver_unload(dev);
-		return ret;
-	}
-
-	DRM_DEBUG("%s card detected\n",
-		  ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
-	return ret;
-}
-
-int radeon_master_create(struct drm_device *dev, struct drm_master *master)
-{
-	struct drm_radeon_master_private *master_priv;
-	unsigned long sareapage;
-	int ret;
-
-	master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL);
-	if (!master_priv)
-		return -ENOMEM;
-
-	/* prebuild the SAREA */
-	sareapage = max_t(unsigned long, SAREA_MAX, PAGE_SIZE);
-	ret = drm_legacy_addmap(dev, 0, sareapage, _DRM_SHM, _DRM_CONTAINS_LOCK,
-				&master_priv->sarea);
-	if (ret) {
-		DRM_ERROR("SAREA setup failed\n");
-		kfree(master_priv);
-		return ret;
-	}
-	master_priv->sarea_priv = master_priv->sarea->handle + sizeof(struct drm_sarea);
-	master_priv->sarea_priv->pfCurrentPage = 0;
-
-	master->driver_priv = master_priv;
-	return 0;
-}
-
-void radeon_master_destroy(struct drm_device *dev, struct drm_master *master)
-{
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-
-	if (!master_priv)
-		return;
-
-	if (master_priv->sarea_priv &&
-	    master_priv->sarea_priv->pfCurrentPage != 0)
-		radeon_cp_dispatch_flip(dev, master);
-
-	master_priv->sarea_priv = NULL;
-	if (master_priv->sarea)
-		drm_legacy_rmmap_locked(dev, master_priv->sarea);
-
-	kfree(master_priv);
-
-	master->driver_priv = NULL;
-}
-
-/* Create mappings for registers and framebuffer so userland doesn't necessarily
- * have to find them.
- */
-int radeon_driver_firstopen(struct drm_device *dev)
-{
-	int ret;
-	drm_local_map_t *map;
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
-
-	dev_priv->fb_aper_offset = pci_resource_start(dev->pdev, 0);
-	ret = drm_legacy_addmap(dev, dev_priv->fb_aper_offset,
-				pci_resource_len(dev->pdev, 0),
-				_DRM_FRAME_BUFFER, _DRM_WRITE_COMBINING, &map);
-	if (ret != 0)
-		return ret;
-
-	return 0;
-}
-
-int radeon_driver_unload(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	DRM_DEBUG("\n");
-
-	drm_legacy_rmmap(dev, dev_priv->mmio);
-
-	kfree(dev_priv);
-
-	dev->dev_private = NULL;
-	return 0;
-}
-
-void radeon_commit_ring(drm_radeon_private_t *dev_priv)
-{
-	int i;
-	u32 *ring;
-	int tail_aligned;
-
-	/* check if the ring is padded out to 16-dword alignment */
-
-	tail_aligned = dev_priv->ring.tail & (RADEON_RING_ALIGN-1);
-	if (tail_aligned) {
-		int num_p2 = RADEON_RING_ALIGN - tail_aligned;
-
-		ring = dev_priv->ring.start;
-		/* pad with some CP_PACKET2 */
-		for (i = 0; i < num_p2; i++)
-			ring[dev_priv->ring.tail + i] = CP_PACKET2();
-
-		dev_priv->ring.tail += i;
-
-		dev_priv->ring.space -= num_p2 * sizeof(u32);
-	}
-
-	dev_priv->ring.tail &= dev_priv->ring.tail_mask;
-
-	mb();
-	GET_RING_HEAD( dev_priv );
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
-		RADEON_WRITE(R600_CP_RB_WPTR, dev_priv->ring.tail);
-		/* read from PCI bus to ensure correct posting */
-		RADEON_READ(R600_CP_RB_RPTR);
-	} else {
-		RADEON_WRITE(RADEON_CP_RB_WPTR, dev_priv->ring.tail);
-		/* read from PCI bus to ensure correct posting */
-		RADEON_READ(RADEON_CP_RB_RPTR);
-	}
-}
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index c566993a2ec3..902b59cebac5 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1150,7 +1150,7 @@ static void radeon_check_arguments(struct radeon_device *rdev)
 	}
 
 	if (radeon_vm_size < 1) {
-		dev_warn(rdev->dev, "VM size (%d) to small, min is 1GB\n",
+		dev_warn(rdev->dev, "VM size (%d) too small, min is 1GB\n",
 			 radeon_vm_size);
 		radeon_vm_size = 4;
 	}
@@ -1744,6 +1744,7 @@ int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
 	}
 
 	drm_kms_helper_poll_enable(dev);
+	drm_helper_hpd_irq_event(dev);
 
 	/* set the power state here in case we are a PX system or headless */
 	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 1eca0acac016..298ea1c453c3 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -1331,7 +1331,7 @@ static const struct drm_framebuffer_funcs radeon_fb_funcs = {
 int
 radeon_framebuffer_init(struct drm_device *dev,
 			struct radeon_framebuffer *rfb,
-			struct drm_mode_fb_cmd2 *mode_cmd,
+			const struct drm_mode_fb_cmd2 *mode_cmd,
 			struct drm_gem_object *obj)
 {
 	int ret;
@@ -1348,7 +1348,7 @@ radeon_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 radeon_user_framebuffer_create(struct drm_device *dev,
 			       struct drm_file *file_priv,
-			       struct drm_mode_fb_cmd2 *mode_cmd)
+			       const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct radeon_framebuffer *radeon_fb;
@@ -1670,8 +1670,10 @@ int radeon_modeset_init(struct radeon_device *rdev)
 	/* setup afmt */
 	radeon_afmt_init(rdev);
 
-	radeon_fbdev_init(rdev);
-	drm_kms_helper_poll_init(rdev->ddev);
+	if (!list_empty(&rdev->ddev->mode_config.connector_list)) {
+		radeon_fbdev_init(rdev);
+		drm_kms_helper_poll_init(rdev->ddev);
+	}
 
 	/* do pm late init */
 	ret = radeon_pm_late_init(rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c
index 744f5c49c664..df7a1719c841 100644
--- a/drivers/gpu/drm/radeon/radeon_dp_mst.c
+++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c
@@ -329,7 +329,7 @@ static void radeon_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
 	drm_kms_helper_hotplug_event(dev);
 }
 
-struct drm_dp_mst_topology_cbs mst_cbs = {
+const struct drm_dp_mst_topology_cbs mst_cbs = {
 	.add_connector = radeon_dp_add_mst_connector,
 	.register_connector = radeon_dp_register_mst_connector,
 	.destroy_connector = radeon_dp_destroy_mst_connector,
@@ -525,11 +525,17 @@ static bool radeon_mst_mode_fixup(struct drm_encoder *encoder,
 	drm_mode_set_crtcinfo(adjusted_mode, 0);
 	{
 	  struct radeon_connector_atom_dig *dig_connector;
+	  int ret;
 
 	  dig_connector = mst_enc->connector->con_priv;
-	  dig_connector->dp_lane_count = drm_dp_max_lane_count(dig_connector->dpcd);
-	  dig_connector->dp_clock = radeon_dp_get_max_link_rate(&mst_enc->connector->base,
-								dig_connector->dpcd);
+	  ret = radeon_dp_get_dp_link_config(&mst_enc->connector->base,
+					     dig_connector->dpcd, adjusted_mode->clock,
+					     &dig_connector->dp_lane_count,
+					     &dig_connector->dp_clock);
+	  if (ret) {
+		  dig_connector->dp_lane_count = 0;
+		  dig_connector->dp_clock = 0;
+	  }
 	  DRM_DEBUG_KMS("dig clock %p %d %d\n", dig_connector,
 			dig_connector->dp_lane_count, dig_connector->dp_clock);
 	}
@@ -641,7 +647,7 @@ radeon_dp_create_fake_mst_encoder(struct radeon_connector *connector)
 	}
 
 	drm_encoder_init(dev, &radeon_encoder->base, &radeon_dp_mst_enc_funcs,
-			 DRM_MODE_ENCODER_DPMST);
+			 DRM_MODE_ENCODER_DPMST, NULL);
 	drm_encoder_helper_add(encoder, &radeon_mst_helper_funcs);
 
 	mst_enc = radeon_encoder->enc_priv;
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 5b6a6f5b3619..e266ffc520d2 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -291,88 +291,6 @@ static struct pci_device_id pciidlist[] = {
 
 MODULE_DEVICE_TABLE(pci, pciidlist);
 
-#ifdef CONFIG_DRM_RADEON_UMS
-
-static int radeon_suspend(struct drm_device *dev, pm_message_t state)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return 0;
-
-	/* Disable *all* interrupts */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600)
-		RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
-	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
-	return 0;
-}
-
-static int radeon_resume(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return 0;
-
-	/* Restore interrupt registers */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600)
-		RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
-	RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
-	return 0;
-}
-
-
-static const struct file_operations radeon_driver_old_fops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.mmap = drm_legacy_mmap,
-	.poll = drm_poll,
-	.read = drm_read,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl = radeon_compat_ioctl,
-#endif
-	.llseek = noop_llseek,
-};
-
-static struct drm_driver driver_old = {
-	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG |
-	    DRIVER_HAVE_IRQ | DRIVER_HAVE_DMA | DRIVER_IRQ_SHARED,
-	.dev_priv_size = sizeof(drm_radeon_buf_priv_t),
-	.load = radeon_driver_load,
-	.firstopen = radeon_driver_firstopen,
-	.open = radeon_driver_open,
-	.preclose = radeon_driver_preclose,
-	.postclose = radeon_driver_postclose,
-	.lastclose = radeon_driver_lastclose,
-	.set_busid = drm_pci_set_busid,
-	.unload = radeon_driver_unload,
-	.suspend = radeon_suspend,
-	.resume = radeon_resume,
-	.get_vblank_counter = radeon_get_vblank_counter,
-	.enable_vblank = radeon_enable_vblank,
-	.disable_vblank = radeon_disable_vblank,
-	.master_create = radeon_master_create,
-	.master_destroy = radeon_master_destroy,
-	.irq_preinstall = radeon_driver_irq_preinstall,
-	.irq_postinstall = radeon_driver_irq_postinstall,
-	.irq_uninstall = radeon_driver_irq_uninstall,
-	.irq_handler = radeon_driver_irq_handler,
-	.ioctls = radeon_ioctls,
-	.dma_ioctl = radeon_cp_buffers,
-	.fops = &radeon_driver_old_fops,
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.date = DRIVER_DATE,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
-};
-
-#endif
-
 static struct drm_driver kms_driver;
 
 static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
@@ -619,13 +537,6 @@ static struct drm_driver kms_driver = {
 static struct drm_driver *driver;
 static struct pci_driver *pdriver;
 
-#ifdef CONFIG_DRM_RADEON_UMS
-static struct pci_driver radeon_pci_driver = {
-	.name = DRIVER_NAME,
-	.id_table = pciidlist,
-};
-#endif
-
 static struct pci_driver radeon_kms_pci_driver = {
 	.name = DRIVER_NAME,
 	.id_table = pciidlist,
@@ -655,16 +566,8 @@ static int __init radeon_init(void)
 		radeon_register_atpx_handler();
 
 	} else {
-#ifdef CONFIG_DRM_RADEON_UMS
-		DRM_INFO("radeon userspace modesetting enabled.\n");
-		driver = &driver_old;
-		pdriver = &radeon_pci_driver;
-		driver->driver_features &= ~DRIVER_MODESET;
-		driver->num_ioctls = radeon_max_ioctl;
-#else
 		DRM_ERROR("No UMS support in radeon module!\n");
 		return -EINVAL;
-#endif
 	}
 
 	radeon_kfd_init();
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index 0caafc7a6e17..afef2d9fccd8 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -119,2052 +119,4 @@
 long radeon_drm_ioctl(struct file *filp,
 		      unsigned int cmd, unsigned long arg);
 
-/* The rest of the file is DEPRECATED! */
-#ifdef CONFIG_DRM_RADEON_UMS
-
-enum radeon_cp_microcode_version {
-	UCODE_R100,
-	UCODE_R200,
-	UCODE_R300,
-};
-
-typedef struct drm_radeon_freelist {
-	unsigned int age;
-	struct drm_buf *buf;
-	struct drm_radeon_freelist *next;
-	struct drm_radeon_freelist *prev;
-} drm_radeon_freelist_t;
-
-typedef struct drm_radeon_ring_buffer {
-	u32 *start;
-	u32 *end;
-	int size;
-	int size_l2qw;
-
-	int rptr_update; /* Double Words */
-	int rptr_update_l2qw; /* log2 Quad Words */
-
-	int fetch_size; /* Double Words */
-	int fetch_size_l2ow; /* log2 Oct Words */
-
-	u32 tail;
-	u32 tail_mask;
-	int space;
-
-	int high_mark;
-} drm_radeon_ring_buffer_t;
-
-typedef struct drm_radeon_depth_clear_t {
-	u32 rb3d_cntl;
-	u32 rb3d_zstencilcntl;
-	u32 se_cntl;
-} drm_radeon_depth_clear_t;
-
-struct drm_radeon_driver_file_fields {
-	int64_t radeon_fb_delta;
-};
-
-struct mem_block {
-	struct mem_block *next;
-	struct mem_block *prev;
-	int start;
-	int size;
-	struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */
-};
-
-struct radeon_surface {
-	int refcount;
-	u32 lower;
-	u32 upper;
-	u32 flags;
-};
-
-struct radeon_virt_surface {
-	int surface_index;
-	u32 lower;
-	u32 upper;
-	u32 flags;
-	struct drm_file *file_priv;
-#define PCIGART_FILE_PRIV	((void *) -1L)
-};
-
-#define RADEON_FLUSH_EMITED	(1 << 0)
-#define RADEON_PURGE_EMITED	(1 << 1)
-
-struct drm_radeon_master_private {
-	drm_local_map_t *sarea;
-	drm_radeon_sarea_t *sarea_priv;
-};
-
-typedef struct drm_radeon_private {
-	drm_radeon_ring_buffer_t ring;
-
-	u32 fb_location;
-	u32 fb_size;
-	int new_memmap;
-
-	int gart_size;
-	u32 gart_vm_start;
-	unsigned long gart_buffers_offset;
-
-	int cp_mode;
-	int cp_running;
-
-	drm_radeon_freelist_t *head;
-	drm_radeon_freelist_t *tail;
-	int last_buf;
-	int writeback_works;
-
-	int usec_timeout;
-
-	int microcode_version;
-
-	struct {
-		u32 boxes;
-		int freelist_timeouts;
-		int freelist_loops;
-		int requested_bufs;
-		int last_frame_reads;
-		int last_clear_reads;
-		int clears;
-		int texture_uploads;
-	} stats;
-
-	int do_boxes;
-	int page_flipping;
-
-	u32 color_fmt;
-	unsigned int front_offset;
-	unsigned int front_pitch;
-	unsigned int back_offset;
-	unsigned int back_pitch;
-
-	u32 depth_fmt;
-	unsigned int depth_offset;
-	unsigned int depth_pitch;
-
-	u32 front_pitch_offset;
-	u32 back_pitch_offset;
-	u32 depth_pitch_offset;
-
-	drm_radeon_depth_clear_t depth_clear;
-
-	unsigned long ring_offset;
-	unsigned long ring_rptr_offset;
-	unsigned long buffers_offset;
-	unsigned long gart_textures_offset;
-
-	drm_local_map_t *sarea;
-	drm_local_map_t *cp_ring;
-	drm_local_map_t *ring_rptr;
-	drm_local_map_t *gart_textures;
-
-	struct mem_block *gart_heap;
-	struct mem_block *fb_heap;
-
-	/* SW interrupt */
-	wait_queue_head_t swi_queue;
-	atomic_t swi_emitted;
-	int vblank_crtc;
-	uint32_t irq_enable_reg;
-	uint32_t r500_disp_irq_reg;
-
-	struct radeon_surface surfaces[RADEON_MAX_SURFACES];
-	struct radeon_virt_surface virt_surfaces[2 * RADEON_MAX_SURFACES];
-
-	unsigned long pcigart_offset;
-	unsigned int pcigart_offset_set;
-	struct drm_ati_pcigart_info gart_info;
-
-	u32 scratch_ages[5];
-
-	int have_z_offset;
-
-	/* starting from here on, data is preserved across an open */
-	uint32_t flags;		/* see radeon_chip_flags */
-	resource_size_t fb_aper_offset;
-
-	int num_gb_pipes;
-	int num_z_pipes;
-	int track_flush;
-	drm_local_map_t *mmio;
-
-	/* r6xx/r7xx pipe/shader config */
-	int r600_max_pipes;
-	int r600_max_tile_pipes;
-	int r600_max_simds;
-	int r600_max_backends;
-	int r600_max_gprs;
-	int r600_max_threads;
-	int r600_max_stack_entries;
-	int r600_max_hw_contexts;
-	int r600_max_gs_threads;
-	int r600_sx_max_export_size;
-	int r600_sx_max_export_pos_size;
-	int r600_sx_max_export_smx_size;
-	int r600_sq_num_cf_insts;
-	int r700_sx_num_of_sets;
-	int r700_sc_prim_fifo_size;
-	int r700_sc_hiz_tile_fifo_size;
-	int r700_sc_earlyz_tile_fifo_fize;
-	int r600_group_size;
-	int r600_npipes;
-	int r600_nbanks;
-
-	struct mutex cs_mutex;
-	u32 cs_id_scnt;
-	u32 cs_id_wcnt;
-	/* r6xx/r7xx drm blit vertex buffer */
-	struct drm_buf *blit_vb;
-
-	/* firmware */
-	const struct firmware *me_fw, *pfp_fw;
-} drm_radeon_private_t;
-
-typedef struct drm_radeon_buf_priv {
-	u32 age;
-} drm_radeon_buf_priv_t;
-
-struct drm_buffer;
-
-typedef struct drm_radeon_kcmd_buffer {
-	int bufsz;
-	struct drm_buffer *buffer;
-	int nbox;
-	struct drm_clip_rect __user *boxes;
-} drm_radeon_kcmd_buffer_t;
-
-extern int radeon_no_wb;
-extern struct drm_ioctl_desc radeon_ioctls[];
-extern int radeon_max_ioctl;
-
-extern u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv);
-extern void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val);
-
-#define GET_RING_HEAD(dev_priv)	radeon_get_ring_head(dev_priv)
-#define SET_RING_HEAD(dev_priv, val) radeon_set_ring_head(dev_priv, val)
-
-/* Check whether the given hardware address is inside the framebuffer or the
- * GART area.
- */
-static __inline__ int radeon_check_offset(drm_radeon_private_t *dev_priv,
-					  u64 off)
-{
-	u32 fb_start = dev_priv->fb_location;
-	u32 fb_end = fb_start + dev_priv->fb_size - 1;
-	u32 gart_start = dev_priv->gart_vm_start;
-	u32 gart_end = gart_start + dev_priv->gart_size - 1;
-
-	return ((off >= fb_start && off <= fb_end) ||
-		(off >= gart_start && off <= gart_end));
-}
-
-/* radeon_state.c */
-extern void radeon_cp_discard_buffer(struct drm_device *dev, struct drm_master *master, struct drm_buf *buf);
-
-				/* radeon_cp.c */
-extern int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv);
-extern void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc);
-extern void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base);
-
-extern void radeon_freelist_reset(struct drm_device * dev);
-extern struct drm_buf *radeon_freelist_get(struct drm_device * dev);
-
-extern int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n);
-
-extern int radeon_do_cp_idle(drm_radeon_private_t * dev_priv);
-
-extern int radeon_driver_preinit(struct drm_device *dev, unsigned long flags);
-extern int radeon_presetup(struct drm_device *dev);
-extern int radeon_driver_postcleanup(struct drm_device *dev);
-
-extern int radeon_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_mem_init_heap(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern void radeon_mem_takedown(struct mem_block **heap);
-extern void radeon_mem_release(struct drm_file *file_priv,
-			       struct mem_block *heap);
-
-extern void radeon_enable_bm(struct drm_radeon_private *dev_priv);
-extern u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off);
-extern void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val);
-
-				/* radeon_irq.c */
-extern void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state);
-extern int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv);
-
-extern void radeon_do_release(struct drm_device * dev);
-extern u32 radeon_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
-extern int radeon_enable_vblank(struct drm_device *dev, unsigned int pipe);
-extern void radeon_disable_vblank(struct drm_device *dev, unsigned int pipe);
-extern irqreturn_t radeon_driver_irq_handler(int irq, void *arg);
-extern void radeon_driver_irq_preinstall(struct drm_device * dev);
-extern int radeon_driver_irq_postinstall(struct drm_device *dev);
-extern void radeon_driver_irq_uninstall(struct drm_device * dev);
-extern void radeon_enable_interrupt(struct drm_device *dev);
-extern int radeon_vblank_crtc_get(struct drm_device *dev);
-extern int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value);
-
-extern int radeon_driver_load(struct drm_device *dev, unsigned long flags);
-extern int radeon_driver_unload(struct drm_device *dev);
-extern int radeon_driver_firstopen(struct drm_device *dev);
-extern void radeon_driver_preclose(struct drm_device *dev,
-				   struct drm_file *file_priv);
-extern void radeon_driver_postclose(struct drm_device *dev,
-				    struct drm_file *file_priv);
-extern void radeon_driver_lastclose(struct drm_device * dev);
-extern int radeon_driver_open(struct drm_device *dev,
-			      struct drm_file *file_priv);
-extern long radeon_compat_ioctl(struct file *filp, unsigned int cmd,
-				unsigned long arg);
-
-extern int radeon_master_create(struct drm_device *dev, struct drm_master *master);
-extern void radeon_master_destroy(struct drm_device *dev, struct drm_master *master);
-extern void radeon_cp_dispatch_flip(struct drm_device *dev, struct drm_master *master);
-/* r300_cmdbuf.c */
-extern void r300_init_reg_flags(struct drm_device *dev);
-
-extern int r300_do_cp_cmdbuf(struct drm_device *dev,
-			     struct drm_file *file_priv,
-			     drm_radeon_kcmd_buffer_t *cmdbuf);
-
-/* r600_cp.c */
-extern int r600_do_engine_reset(struct drm_device *dev);
-extern int r600_do_cleanup_cp(struct drm_device *dev);
-extern int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
-			   struct drm_file *file_priv);
-extern int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv);
-extern int r600_do_cp_idle(drm_radeon_private_t *dev_priv);
-extern void r600_do_cp_start(drm_radeon_private_t *dev_priv);
-extern void r600_do_cp_reset(drm_radeon_private_t *dev_priv);
-extern void r600_do_cp_stop(drm_radeon_private_t *dev_priv);
-extern int r600_cp_dispatch_indirect(struct drm_device *dev,
-				     struct drm_buf *buf, int start, int end);
-extern int r600_page_table_init(struct drm_device *dev);
-extern void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info);
-extern int r600_cs_legacy_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv);
-extern void r600_cp_dispatch_swap(struct drm_device *dev, struct drm_file *file_priv);
-extern int r600_cp_dispatch_texture(struct drm_device *dev,
-				    struct drm_file *file_priv,
-				    drm_radeon_texture_t *tex,
-				    drm_radeon_tex_image_t *image);
-/* r600_blit.c */
-extern int r600_prepare_blit_copy(struct drm_device *dev, struct drm_file *file_priv);
-extern void r600_done_blit_copy(struct drm_device *dev);
-extern void r600_blit_copy(struct drm_device *dev,
-			   uint64_t src_gpu_addr, uint64_t dst_gpu_addr,
-			   int size_bytes);
-extern void r600_blit_swap(struct drm_device *dev,
-			   uint64_t src_gpu_addr, uint64_t dst_gpu_addr,
-			   int sx, int sy, int dx, int dy,
-			   int w, int h, int src_pitch, int dst_pitch, int cpp);
-
-/* Flags for stats.boxes
- */
-#define RADEON_BOX_DMA_IDLE      0x1
-#define RADEON_BOX_RING_FULL     0x2
-#define RADEON_BOX_FLIP          0x4
-#define RADEON_BOX_WAIT_IDLE     0x8
-#define RADEON_BOX_TEXTURE_LOAD  0x10
-
-/* Register definitions, register access macros and drmAddMap constants
- * for Radeon kernel driver.
- */
-#define RADEON_MM_INDEX		        0x0000
-#define RADEON_MM_DATA		        0x0004
-
-#define RADEON_AGP_COMMAND		0x0f60
-#define RADEON_AGP_COMMAND_PCI_CONFIG   0x0060	/* offset in PCI config */
-#	define RADEON_AGP_ENABLE	(1<<8)
-#define RADEON_AUX_SCISSOR_CNTL		0x26f0
-#	define RADEON_EXCLUSIVE_SCISSOR_0	(1 << 24)
-#	define RADEON_EXCLUSIVE_SCISSOR_1	(1 << 25)
-#	define RADEON_EXCLUSIVE_SCISSOR_2	(1 << 26)
-#	define RADEON_SCISSOR_0_ENABLE		(1 << 28)
-#	define RADEON_SCISSOR_1_ENABLE		(1 << 29)
-#	define RADEON_SCISSOR_2_ENABLE		(1 << 30)
-
-/*
- * PCIE radeons (rv370/rv380, rv410, r423/r430/r480, r5xx)
- * don't have an explicit bus mastering disable bit.  It's handled
- * by the PCI D-states.  PMI_BM_DIS disables D-state bus master
- * handling, not bus mastering itself.
- */
-#define RADEON_BUS_CNTL			0x0030
-/* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
-#	define RADEON_BUS_MASTER_DIS		(1 << 6)
-/* rs600/rs690/rs740 */
-#	define RS600_BUS_MASTER_DIS		(1 << 14)
-#	define RS600_MSI_REARM		        (1 << 20)
-/* see RS400_MSI_REARM in AIC_CNTL for rs480 */
-
-#define RADEON_BUS_CNTL1		0x0034
-#	define RADEON_PMI_BM_DIS		(1 << 2)
-#	define RADEON_PMI_INT_DIS		(1 << 3)
-
-#define RV370_BUS_CNTL			0x004c
-#	define RV370_PMI_BM_DIS		        (1 << 5)
-#	define RV370_PMI_INT_DIS		(1 << 6)
-
-#define RADEON_MSI_REARM_EN		0x0160
-/* rv370/rv380, rv410, r423/r430/r480, r5xx */
-#	define RV370_MSI_REARM_EN		(1 << 0)
-
-#define RADEON_CLOCK_CNTL_DATA		0x000c
-#	define RADEON_PLL_WR_EN			(1 << 7)
-#define RADEON_CLOCK_CNTL_INDEX		0x0008
-#define RADEON_CONFIG_APER_SIZE		0x0108
-#define RADEON_CONFIG_MEMSIZE		0x00f8
-#define RADEON_CRTC_OFFSET		0x0224
-#define RADEON_CRTC_OFFSET_CNTL		0x0228
-#	define RADEON_CRTC_TILE_EN		(1 << 15)
-#	define RADEON_CRTC_OFFSET_FLIP_CNTL	(1 << 16)
-#define RADEON_CRTC2_OFFSET		0x0324
-#define RADEON_CRTC2_OFFSET_CNTL	0x0328
-
-#define RADEON_PCIE_INDEX               0x0030
-#define RADEON_PCIE_DATA                0x0034
-#define RADEON_PCIE_TX_GART_CNTL	0x10
-#	define RADEON_PCIE_TX_GART_EN		(1 << 0)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_PASS_THRU (0 << 1)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_CLAMP_LO  (1 << 1)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD   (3 << 1)
-#	define RADEON_PCIE_TX_GART_MODE_32_128_CACHE	(0 << 3)
-#	define RADEON_PCIE_TX_GART_MODE_8_4_128_CACHE	(1 << 3)
-#	define RADEON_PCIE_TX_GART_CHK_RW_VALID_EN      (1 << 5)
-#	define RADEON_PCIE_TX_GART_INVALIDATE_TLB	(1 << 8)
-#define RADEON_PCIE_TX_DISCARD_RD_ADDR_LO 0x11
-#define RADEON_PCIE_TX_DISCARD_RD_ADDR_HI 0x12
-#define RADEON_PCIE_TX_GART_BASE	0x13
-#define RADEON_PCIE_TX_GART_START_LO	0x14
-#define RADEON_PCIE_TX_GART_START_HI	0x15
-#define RADEON_PCIE_TX_GART_END_LO	0x16
-#define RADEON_PCIE_TX_GART_END_HI	0x17
-
-#define RS480_NB_MC_INDEX               0x168
-#	define RS480_NB_MC_IND_WR_EN	(1 << 8)
-#define RS480_NB_MC_DATA                0x16c
-
-#define RS690_MC_INDEX                  0x78
-#   define RS690_MC_INDEX_MASK          0x1ff
-#   define RS690_MC_INDEX_WR_EN         (1 << 9)
-#   define RS690_MC_INDEX_WR_ACK        0x7f
-#define RS690_MC_DATA                   0x7c
-
-/* MC indirect registers */
-#define RS480_MC_MISC_CNTL              0x18
-#	define RS480_DISABLE_GTW	(1 << 1)
-/* switch between MCIND GART and MM GART registers. 0 = mmgart, 1 = mcind gart */
-#	define RS480_GART_INDEX_REG_EN	(1 << 12)
-#	define RS690_BLOCK_GFX_D3_EN	(1 << 14)
-#define RS480_K8_FB_LOCATION            0x1e
-#define RS480_GART_FEATURE_ID           0x2b
-#	define RS480_HANG_EN	        (1 << 11)
-#	define RS480_TLB_ENABLE	        (1 << 18)
-#	define RS480_P2P_ENABLE	        (1 << 19)
-#	define RS480_GTW_LAC_EN	        (1 << 25)
-#	define RS480_2LEVEL_GART	(0 << 30)
-#	define RS480_1LEVEL_GART	(1 << 30)
-#	define RS480_PDC_EN	        (1 << 31)
-#define RS480_GART_BASE                 0x2c
-#define RS480_GART_CACHE_CNTRL          0x2e
-#	define RS480_GART_CACHE_INVALIDATE (1 << 0) /* wait for it to clear */
-#define RS480_AGP_ADDRESS_SPACE_SIZE    0x38
-#	define RS480_GART_EN	        (1 << 0)
-#	define RS480_VA_SIZE_32MB	(0 << 1)
-#	define RS480_VA_SIZE_64MB	(1 << 1)
-#	define RS480_VA_SIZE_128MB	(2 << 1)
-#	define RS480_VA_SIZE_256MB	(3 << 1)
-#	define RS480_VA_SIZE_512MB	(4 << 1)
-#	define RS480_VA_SIZE_1GB	(5 << 1)
-#	define RS480_VA_SIZE_2GB	(6 << 1)
-#define RS480_AGP_MODE_CNTL             0x39
-#	define RS480_POST_GART_Q_SIZE	(1 << 18)
-#	define RS480_NONGART_SNOOP	(1 << 19)
-#	define RS480_AGP_RD_BUF_SIZE	(1 << 20)
-#	define RS480_REQ_TYPE_SNOOP_SHIFT 22
-#	define RS480_REQ_TYPE_SNOOP_MASK  0x3
-#	define RS480_REQ_TYPE_SNOOP_DIS	(1 << 24)
-#define RS480_MC_MISC_UMA_CNTL          0x5f
-#define RS480_MC_MCLK_CNTL              0x7a
-#define RS480_MC_UMA_DUALCH_CNTL        0x86
-
-#define RS690_MC_FB_LOCATION            0x100
-#define RS690_MC_AGP_LOCATION           0x101
-#define RS690_MC_AGP_BASE               0x102
-#define RS690_MC_AGP_BASE_2             0x103
-
-#define RS600_MC_INDEX                          0x70
-#       define RS600_MC_ADDR_MASK               0xffff
-#       define RS600_MC_IND_SEQ_RBS_0           (1 << 16)
-#       define RS600_MC_IND_SEQ_RBS_1           (1 << 17)
-#       define RS600_MC_IND_SEQ_RBS_2           (1 << 18)
-#       define RS600_MC_IND_SEQ_RBS_3           (1 << 19)
-#       define RS600_MC_IND_AIC_RBS             (1 << 20)
-#       define RS600_MC_IND_CITF_ARB0           (1 << 21)
-#       define RS600_MC_IND_CITF_ARB1           (1 << 22)
-#       define RS600_MC_IND_WR_EN               (1 << 23)
-#define RS600_MC_DATA                           0x74
-
-#define RS600_MC_STATUS                         0x0
-#       define RS600_MC_IDLE                    (1 << 1)
-#define RS600_MC_FB_LOCATION                    0x4
-#define RS600_MC_AGP_LOCATION                   0x5
-#define RS600_AGP_BASE                          0x6
-#define RS600_AGP_BASE_2                        0x7
-#define RS600_MC_CNTL1                          0x9
-#       define RS600_ENABLE_PAGE_TABLES         (1 << 26)
-#define RS600_MC_PT0_CNTL                       0x100
-#       define RS600_ENABLE_PT                  (1 << 0)
-#       define RS600_EFFECTIVE_L2_CACHE_SIZE(x) ((x) << 15)
-#       define RS600_EFFECTIVE_L2_QUEUE_SIZE(x) ((x) << 21)
-#       define RS600_INVALIDATE_ALL_L1_TLBS     (1 << 28)
-#       define RS600_INVALIDATE_L2_CACHE        (1 << 29)
-#define RS600_MC_PT0_CONTEXT0_CNTL              0x102
-#       define RS600_ENABLE_PAGE_TABLE          (1 << 0)
-#       define RS600_PAGE_TABLE_TYPE_FLAT       (0 << 1)
-#define RS600_MC_PT0_SYSTEM_APERTURE_LOW_ADDR   0x112
-#define RS600_MC_PT0_SYSTEM_APERTURE_HIGH_ADDR  0x114
-#define RS600_MC_PT0_CONTEXT0_DEFAULT_READ_ADDR 0x11c
-#define RS600_MC_PT0_CONTEXT0_FLAT_BASE_ADDR    0x12c
-#define RS600_MC_PT0_CONTEXT0_FLAT_START_ADDR   0x13c
-#define RS600_MC_PT0_CONTEXT0_FLAT_END_ADDR     0x14c
-#define RS600_MC_PT0_CLIENT0_CNTL               0x16c
-#       define RS600_ENABLE_TRANSLATION_MODE_OVERRIDE       (1 << 0)
-#       define RS600_TRANSLATION_MODE_OVERRIDE              (1 << 1)
-#       define RS600_SYSTEM_ACCESS_MODE_MASK                (3 << 8)
-#       define RS600_SYSTEM_ACCESS_MODE_PA_ONLY             (0 << 8)
-#       define RS600_SYSTEM_ACCESS_MODE_USE_SYS_MAP         (1 << 8)
-#       define RS600_SYSTEM_ACCESS_MODE_IN_SYS              (2 << 8)
-#       define RS600_SYSTEM_ACCESS_MODE_NOT_IN_SYS          (3 << 8)
-#       define RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASSTHROUGH        (0 << 10)
-#       define RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_DEFAULT_PAGE       (1 << 10)
-#       define RS600_EFFECTIVE_L1_CACHE_SIZE(x) ((x) << 11)
-#       define RS600_ENABLE_FRAGMENT_PROCESSING (1 << 14)
-#       define RS600_EFFECTIVE_L1_QUEUE_SIZE(x) ((x) << 15)
-#       define RS600_INVALIDATE_L1_TLB          (1 << 20)
-
-#define R520_MC_IND_INDEX 0x70
-#define R520_MC_IND_WR_EN (1 << 24)
-#define R520_MC_IND_DATA  0x74
-
-#define RV515_MC_FB_LOCATION 0x01
-#define RV515_MC_AGP_LOCATION 0x02
-#define RV515_MC_AGP_BASE     0x03
-#define RV515_MC_AGP_BASE_2   0x04
-
-#define R520_MC_FB_LOCATION 0x04
-#define R520_MC_AGP_LOCATION 0x05
-#define R520_MC_AGP_BASE     0x06
-#define R520_MC_AGP_BASE_2   0x07
-
-#define RADEON_MPP_TB_CONFIG		0x01c0
-#define RADEON_MEM_CNTL			0x0140
-#define RADEON_MEM_SDRAM_MODE_REG	0x0158
-#define RADEON_AGP_BASE_2		0x015c /* r200+ only */
-#define RS480_AGP_BASE_2		0x0164
-#define RADEON_AGP_BASE			0x0170
-
-/* pipe config regs */
-#define R400_GB_PIPE_SELECT             0x402c
-#define RV530_GB_PIPE_SELECT2           0x4124
-#define R500_DYN_SCLK_PWMEM_PIPE        0x000d /* PLL */
-#define R300_GB_TILE_CONFIG             0x4018
-#       define R300_ENABLE_TILING       (1 << 0)
-#       define R300_PIPE_COUNT_RV350    (0 << 1)
-#       define R300_PIPE_COUNT_R300     (3 << 1)
-#       define R300_PIPE_COUNT_R420_3P  (6 << 1)
-#       define R300_PIPE_COUNT_R420     (7 << 1)
-#       define R300_TILE_SIZE_8         (0 << 4)
-#       define R300_TILE_SIZE_16        (1 << 4)
-#       define R300_TILE_SIZE_32        (2 << 4)
-#       define R300_SUBPIXEL_1_12       (0 << 16)
-#       define R300_SUBPIXEL_1_16       (1 << 16)
-#define R300_DST_PIPE_CONFIG            0x170c
-#       define R300_PIPE_AUTO_CONFIG    (1 << 31)
-#define R300_RB2D_DSTCACHE_MODE         0x3428
-#       define R300_DC_AUTOFLUSH_ENABLE (1 << 8)
-#       define R300_DC_DC_DISABLE_IGNORE_PE (1 << 17)
-
-#define RADEON_RB3D_COLOROFFSET		0x1c40
-#define RADEON_RB3D_COLORPITCH		0x1c48
-
-#define	RADEON_SRC_X_Y			0x1590
-
-#define RADEON_DP_GUI_MASTER_CNTL	0x146c
-#	define RADEON_GMC_SRC_PITCH_OFFSET_CNTL	(1 << 0)
-#	define RADEON_GMC_DST_PITCH_OFFSET_CNTL	(1 << 1)
-#	define RADEON_GMC_BRUSH_SOLID_COLOR	(13 << 4)
-#	define RADEON_GMC_BRUSH_NONE		(15 << 4)
-#	define RADEON_GMC_DST_16BPP		(4 << 8)
-#	define RADEON_GMC_DST_24BPP		(5 << 8)
-#	define RADEON_GMC_DST_32BPP		(6 << 8)
-#	define RADEON_GMC_DST_DATATYPE_SHIFT	8
-#	define RADEON_GMC_SRC_DATATYPE_COLOR	(3 << 12)
-#	define RADEON_DP_SRC_SOURCE_MEMORY	(2 << 24)
-#	define RADEON_DP_SRC_SOURCE_HOST_DATA	(3 << 24)
-#	define RADEON_GMC_CLR_CMP_CNTL_DIS	(1 << 28)
-#	define RADEON_GMC_WR_MSK_DIS		(1 << 30)
-#	define RADEON_ROP3_S			0x00cc0000
-#	define RADEON_ROP3_P			0x00f00000
-#define RADEON_DP_WRITE_MASK		0x16cc
-#define RADEON_SRC_PITCH_OFFSET		0x1428
-#define RADEON_DST_PITCH_OFFSET		0x142c
-#define RADEON_DST_PITCH_OFFSET_C	0x1c80
-#	define RADEON_DST_TILE_LINEAR		(0 << 30)
-#	define RADEON_DST_TILE_MACRO		(1 << 30)
-#	define RADEON_DST_TILE_MICRO		(2 << 30)
-#	define RADEON_DST_TILE_BOTH		(3 << 30)
-
-#define RADEON_SCRATCH_REG0		0x15e0
-#define RADEON_SCRATCH_REG1		0x15e4
-#define RADEON_SCRATCH_REG2		0x15e8
-#define RADEON_SCRATCH_REG3		0x15ec
-#define RADEON_SCRATCH_REG4		0x15f0
-#define RADEON_SCRATCH_REG5		0x15f4
-#define RADEON_SCRATCH_UMSK		0x0770
-#define RADEON_SCRATCH_ADDR		0x0774
-
-#define RADEON_SCRATCHOFF( x )		(RADEON_SCRATCH_REG_OFFSET + 4*(x))
-
-extern u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index);
-
-#define GET_SCRATCH(dev_priv, x) radeon_get_scratch(dev_priv, x)
-
-#define R600_SCRATCH_REG0		0x8500
-#define R600_SCRATCH_REG1		0x8504
-#define R600_SCRATCH_REG2		0x8508
-#define R600_SCRATCH_REG3		0x850c
-#define R600_SCRATCH_REG4		0x8510
-#define R600_SCRATCH_REG5		0x8514
-#define R600_SCRATCH_REG6		0x8518
-#define R600_SCRATCH_REG7		0x851c
-#define R600_SCRATCH_UMSK		0x8540
-#define R600_SCRATCH_ADDR		0x8544
-
-#define R600_SCRATCHOFF(x)		(R600_SCRATCH_REG_OFFSET + 4*(x))
-
-#define RADEON_GEN_INT_CNTL		0x0040
-#	define RADEON_CRTC_VBLANK_MASK		(1 << 0)
-#	define RADEON_CRTC2_VBLANK_MASK		(1 << 9)
-#	define RADEON_GUI_IDLE_INT_ENABLE	(1 << 19)
-#	define RADEON_SW_INT_ENABLE		(1 << 25)
-
-#define RADEON_GEN_INT_STATUS		0x0044
-#	define RADEON_CRTC_VBLANK_STAT		(1 << 0)
-#	define RADEON_CRTC_VBLANK_STAT_ACK	(1 << 0)
-#	define RADEON_CRTC2_VBLANK_STAT		(1 << 9)
-#	define RADEON_CRTC2_VBLANK_STAT_ACK	(1 << 9)
-#	define RADEON_GUI_IDLE_INT_TEST_ACK     (1 << 19)
-#	define RADEON_SW_INT_TEST		(1 << 25)
-#	define RADEON_SW_INT_TEST_ACK		(1 << 25)
-#	define RADEON_SW_INT_FIRE		(1 << 26)
-#       define R500_DISPLAY_INT_STATUS          (1 << 0)
-
-#define RADEON_HOST_PATH_CNTL		0x0130
-#	define RADEON_HDP_SOFT_RESET		(1 << 26)
-#	define RADEON_HDP_WC_TIMEOUT_MASK	(7 << 28)
-#	define RADEON_HDP_WC_TIMEOUT_28BCLK	(7 << 28)
-
-#define RADEON_ISYNC_CNTL		0x1724
-#	define RADEON_ISYNC_ANY2D_IDLE3D	(1 << 0)
-#	define RADEON_ISYNC_ANY3D_IDLE2D	(1 << 1)
-#	define RADEON_ISYNC_TRIG2D_IDLE3D	(1 << 2)
-#	define RADEON_ISYNC_TRIG3D_IDLE2D	(1 << 3)
-#	define RADEON_ISYNC_WAIT_IDLEGUI	(1 << 4)
-#	define RADEON_ISYNC_CPSCRATCH_IDLEGUI	(1 << 5)
-
-#define RADEON_RBBM_GUICNTL		0x172c
-#	define RADEON_HOST_DATA_SWAP_NONE	(0 << 0)
-#	define RADEON_HOST_DATA_SWAP_16BIT	(1 << 0)
-#	define RADEON_HOST_DATA_SWAP_32BIT	(2 << 0)
-#	define RADEON_HOST_DATA_SWAP_HDW	(3 << 0)
-
-#define RADEON_MC_AGP_LOCATION		0x014c
-#define RADEON_MC_FB_LOCATION		0x0148
-#define RADEON_MCLK_CNTL		0x0012
-#	define RADEON_FORCEON_MCLKA		(1 << 16)
-#	define RADEON_FORCEON_MCLKB		(1 << 17)
-#	define RADEON_FORCEON_YCLKA		(1 << 18)
-#	define RADEON_FORCEON_YCLKB		(1 << 19)
-#	define RADEON_FORCEON_MC		(1 << 20)
-#	define RADEON_FORCEON_AIC		(1 << 21)
-
-#define RADEON_PP_BORDER_COLOR_0	0x1d40
-#define RADEON_PP_BORDER_COLOR_1	0x1d44
-#define RADEON_PP_BORDER_COLOR_2	0x1d48
-#define RADEON_PP_CNTL			0x1c38
-#	define RADEON_SCISSOR_ENABLE		(1 <<  1)
-#define RADEON_PP_LUM_MATRIX		0x1d00
-#define RADEON_PP_MISC			0x1c14
-#define RADEON_PP_ROT_MATRIX_0		0x1d58
-#define RADEON_PP_TXFILTER_0		0x1c54
-#define RADEON_PP_TXOFFSET_0		0x1c5c
-#define RADEON_PP_TXFILTER_1		0x1c6c
-#define RADEON_PP_TXFILTER_2		0x1c84
-
-#define R300_RB2D_DSTCACHE_CTLSTAT	0x342c /* use R300_DSTCACHE_CTLSTAT */
-#define R300_DSTCACHE_CTLSTAT		0x1714
-#	define R300_RB2D_DC_FLUSH		(3 << 0)
-#	define R300_RB2D_DC_FREE		(3 << 2)
-#	define R300_RB2D_DC_FLUSH_ALL		0xf
-#	define R300_RB2D_DC_BUSY		(1 << 31)
-#define RADEON_RB3D_CNTL		0x1c3c
-#	define RADEON_ALPHA_BLEND_ENABLE	(1 << 0)
-#	define RADEON_PLANE_MASK_ENABLE		(1 << 1)
-#	define RADEON_DITHER_ENABLE		(1 << 2)
-#	define RADEON_ROUND_ENABLE		(1 << 3)
-#	define RADEON_SCALE_DITHER_ENABLE	(1 << 4)
-#	define RADEON_DITHER_INIT		(1 << 5)
-#	define RADEON_ROP_ENABLE		(1 << 6)
-#	define RADEON_STENCIL_ENABLE		(1 << 7)
-#	define RADEON_Z_ENABLE			(1 << 8)
-#	define RADEON_ZBLOCK16			(1 << 15)
-#define RADEON_RB3D_DEPTHOFFSET		0x1c24
-#define RADEON_RB3D_DEPTHCLEARVALUE	0x3230
-#define RADEON_RB3D_DEPTHPITCH		0x1c28
-#define RADEON_RB3D_PLANEMASK		0x1d84
-#define RADEON_RB3D_STENCILREFMASK	0x1d7c
-#define RADEON_RB3D_ZCACHE_MODE		0x3250
-#define RADEON_RB3D_ZCACHE_CTLSTAT	0x3254
-#	define RADEON_RB3D_ZC_FLUSH		(1 << 0)
-#	define RADEON_RB3D_ZC_FREE		(1 << 2)
-#	define RADEON_RB3D_ZC_FLUSH_ALL		0x5
-#	define RADEON_RB3D_ZC_BUSY		(1 << 31)
-#define R300_ZB_ZCACHE_CTLSTAT                  0x4f18
-#	define R300_ZC_FLUSH		        (1 << 0)
-#	define R300_ZC_FREE		        (1 << 1)
-#	define R300_ZC_BUSY		        (1 << 31)
-#define RADEON_RB3D_DSTCACHE_CTLSTAT	0x325c
-#	define RADEON_RB3D_DC_FLUSH		(3 << 0)
-#	define RADEON_RB3D_DC_FREE		(3 << 2)
-#	define RADEON_RB3D_DC_FLUSH_ALL		0xf
-#	define RADEON_RB3D_DC_BUSY		(1 << 31)
-#define R300_RB3D_DSTCACHE_CTLSTAT              0x4e4c
-#	define R300_RB3D_DC_FLUSH		(2 << 0)
-#	define R300_RB3D_DC_FREE		(2 << 2)
-#	define R300_RB3D_DC_FINISH		(1 << 4)
-#define RADEON_RB3D_ZSTENCILCNTL	0x1c2c
-#	define RADEON_Z_TEST_MASK		(7 << 4)
-#	define RADEON_Z_TEST_ALWAYS		(7 << 4)
-#	define RADEON_Z_HIERARCHY_ENABLE	(1 << 8)
-#	define RADEON_STENCIL_TEST_ALWAYS	(7 << 12)
-#	define RADEON_STENCIL_S_FAIL_REPLACE	(2 << 16)
-#	define RADEON_STENCIL_ZPASS_REPLACE	(2 << 20)
-#	define RADEON_STENCIL_ZFAIL_REPLACE	(2 << 24)
-#	define RADEON_Z_COMPRESSION_ENABLE	(1 << 28)
-#	define RADEON_FORCE_Z_DIRTY		(1 << 29)
-#	define RADEON_Z_WRITE_ENABLE		(1 << 30)
-#	define RADEON_Z_DECOMPRESSION_ENABLE	(1 << 31)
-#define RADEON_RBBM_SOFT_RESET		0x00f0
-#	define RADEON_SOFT_RESET_CP		(1 <<  0)
-#	define RADEON_SOFT_RESET_HI		(1 <<  1)
-#	define RADEON_SOFT_RESET_SE		(1 <<  2)
-#	define RADEON_SOFT_RESET_RE		(1 <<  3)
-#	define RADEON_SOFT_RESET_PP		(1 <<  4)
-#	define RADEON_SOFT_RESET_E2		(1 <<  5)
-#	define RADEON_SOFT_RESET_RB		(1 <<  6)
-#	define RADEON_SOFT_RESET_HDP		(1 <<  7)
-/*
- *   6:0  Available slots in the FIFO
- *   8    Host Interface active
- *   9    CP request active
- *   10   FIFO request active
- *   11   Host Interface retry active
- *   12   CP retry active
- *   13   FIFO retry active
- *   14   FIFO pipeline busy
- *   15   Event engine busy
- *   16   CP command stream busy
- *   17   2D engine busy
- *   18   2D portion of render backend busy
- *   20   3D setup engine busy
- *   26   GA engine busy
- *   27   CBA 2D engine busy
- *   31   2D engine busy or 3D engine busy or FIFO not empty or CP busy or
- *           command stream queue not empty or Ring Buffer not empty
- */
-#define RADEON_RBBM_STATUS		0x0e40
-/* Same as the previous RADEON_RBBM_STATUS; this is a mirror of that register.  */
-/* #define RADEON_RBBM_STATUS		0x1740 */
-/* bits 6:0 are dword slots available in the cmd fifo */
-#	define RADEON_RBBM_FIFOCNT_MASK		0x007f
-#	define RADEON_HIRQ_ON_RBB	(1 <<  8)
-#	define RADEON_CPRQ_ON_RBB	(1 <<  9)
-#	define RADEON_CFRQ_ON_RBB	(1 << 10)
-#	define RADEON_HIRQ_IN_RTBUF	(1 << 11)
-#	define RADEON_CPRQ_IN_RTBUF	(1 << 12)
-#	define RADEON_CFRQ_IN_RTBUF	(1 << 13)
-#	define RADEON_PIPE_BUSY		(1 << 14)
-#	define RADEON_ENG_EV_BUSY	(1 << 15)
-#	define RADEON_CP_CMDSTRM_BUSY	(1 << 16)
-#	define RADEON_E2_BUSY		(1 << 17)
-#	define RADEON_RB2D_BUSY		(1 << 18)
-#	define RADEON_RB3D_BUSY		(1 << 19) /* not used on r300 */
-#	define RADEON_VAP_BUSY		(1 << 20)
-#	define RADEON_RE_BUSY		(1 << 21) /* not used on r300 */
-#	define RADEON_TAM_BUSY		(1 << 22) /* not used on r300 */
-#	define RADEON_TDM_BUSY		(1 << 23) /* not used on r300 */
-#	define RADEON_PB_BUSY		(1 << 24) /* not used on r300 */
-#	define RADEON_TIM_BUSY		(1 << 25) /* not used on r300 */
-#	define RADEON_GA_BUSY		(1 << 26)
-#	define RADEON_CBA2D_BUSY	(1 << 27)
-#	define RADEON_RBBM_ACTIVE	(1 << 31)
-#define RADEON_RE_LINE_PATTERN		0x1cd0
-#define RADEON_RE_MISC			0x26c4
-#define RADEON_RE_TOP_LEFT		0x26c0
-#define RADEON_RE_WIDTH_HEIGHT		0x1c44
-#define RADEON_RE_STIPPLE_ADDR		0x1cc8
-#define RADEON_RE_STIPPLE_DATA		0x1ccc
-
-#define RADEON_SCISSOR_TL_0		0x1cd8
-#define RADEON_SCISSOR_BR_0		0x1cdc
-#define RADEON_SCISSOR_TL_1		0x1ce0
-#define RADEON_SCISSOR_BR_1		0x1ce4
-#define RADEON_SCISSOR_TL_2		0x1ce8
-#define RADEON_SCISSOR_BR_2		0x1cec
-#define RADEON_SE_COORD_FMT		0x1c50
-#define RADEON_SE_CNTL			0x1c4c
-#	define RADEON_FFACE_CULL_CW		(0 << 0)
-#	define RADEON_BFACE_SOLID		(3 << 1)
-#	define RADEON_FFACE_SOLID		(3 << 3)
-#	define RADEON_FLAT_SHADE_VTX_LAST	(3 << 6)
-#	define RADEON_DIFFUSE_SHADE_FLAT	(1 << 8)
-#	define RADEON_DIFFUSE_SHADE_GOURAUD	(2 << 8)
-#	define RADEON_ALPHA_SHADE_FLAT		(1 << 10)
-#	define RADEON_ALPHA_SHADE_GOURAUD	(2 << 10)
-#	define RADEON_SPECULAR_SHADE_FLAT	(1 << 12)
-#	define RADEON_SPECULAR_SHADE_GOURAUD	(2 << 12)
-#	define RADEON_FOG_SHADE_FLAT		(1 << 14)
-#	define RADEON_FOG_SHADE_GOURAUD		(2 << 14)
-#	define RADEON_VPORT_XY_XFORM_ENABLE	(1 << 24)
-#	define RADEON_VPORT_Z_XFORM_ENABLE	(1 << 25)
-#	define RADEON_VTX_PIX_CENTER_OGL	(1 << 27)
-#	define RADEON_ROUND_MODE_TRUNC		(0 << 28)
-#	define RADEON_ROUND_PREC_8TH_PIX	(1 << 30)
-#define RADEON_SE_CNTL_STATUS		0x2140
-#define RADEON_SE_LINE_WIDTH		0x1db8
-#define RADEON_SE_VPORT_XSCALE		0x1d98
-#define RADEON_SE_ZBIAS_FACTOR		0x1db0
-#define RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED 0x2210
-#define RADEON_SE_TCL_OUTPUT_VTX_FMT         0x2254
-#define RADEON_SE_TCL_VECTOR_INDX_REG        0x2200
-#       define RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT  16
-#       define RADEON_VEC_INDX_DWORD_COUNT_SHIFT     28
-#define RADEON_SE_TCL_VECTOR_DATA_REG       0x2204
-#define RADEON_SE_TCL_SCALAR_INDX_REG       0x2208
-#       define RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT  16
-#define RADEON_SE_TCL_SCALAR_DATA_REG       0x220C
-#define RADEON_SURFACE_ACCESS_FLAGS	0x0bf8
-#define RADEON_SURFACE_ACCESS_CLR	0x0bfc
-#define RADEON_SURFACE_CNTL		0x0b00
-#	define RADEON_SURF_TRANSLATION_DIS	(1 << 8)
-#	define RADEON_NONSURF_AP0_SWP_MASK	(3 << 20)
-#	define RADEON_NONSURF_AP0_SWP_LITTLE	(0 << 20)
-#	define RADEON_NONSURF_AP0_SWP_BIG16	(1 << 20)
-#	define RADEON_NONSURF_AP0_SWP_BIG32	(2 << 20)
-#	define RADEON_NONSURF_AP1_SWP_MASK	(3 << 22)
-#	define RADEON_NONSURF_AP1_SWP_LITTLE	(0 << 22)
-#	define RADEON_NONSURF_AP1_SWP_BIG16	(1 << 22)
-#	define RADEON_NONSURF_AP1_SWP_BIG32	(2 << 22)
-#define RADEON_SURFACE0_INFO		0x0b0c
-#	define RADEON_SURF_PITCHSEL_MASK	(0x1ff << 0)
-#	define RADEON_SURF_TILE_MODE_MASK	(3 << 16)
-#	define RADEON_SURF_TILE_MODE_MACRO	(0 << 16)
-#	define RADEON_SURF_TILE_MODE_MICRO	(1 << 16)
-#	define RADEON_SURF_TILE_MODE_32BIT_Z	(2 << 16)
-#	define RADEON_SURF_TILE_MODE_16BIT_Z	(3 << 16)
-#define RADEON_SURFACE0_LOWER_BOUND	0x0b04
-#define RADEON_SURFACE0_UPPER_BOUND	0x0b08
-#	define RADEON_SURF_ADDRESS_FIXED_MASK	(0x3ff << 0)
-#define RADEON_SURFACE1_INFO		0x0b1c
-#define RADEON_SURFACE1_LOWER_BOUND	0x0b14
-#define RADEON_SURFACE1_UPPER_BOUND	0x0b18
-#define RADEON_SURFACE2_INFO		0x0b2c
-#define RADEON_SURFACE2_LOWER_BOUND	0x0b24
-#define RADEON_SURFACE2_UPPER_BOUND	0x0b28
-#define RADEON_SURFACE3_INFO		0x0b3c
-#define RADEON_SURFACE3_LOWER_BOUND	0x0b34
-#define RADEON_SURFACE3_UPPER_BOUND	0x0b38
-#define RADEON_SURFACE4_INFO		0x0b4c
-#define RADEON_SURFACE4_LOWER_BOUND	0x0b44
-#define RADEON_SURFACE4_UPPER_BOUND	0x0b48
-#define RADEON_SURFACE5_INFO		0x0b5c
-#define RADEON_SURFACE5_LOWER_BOUND	0x0b54
-#define RADEON_SURFACE5_UPPER_BOUND	0x0b58
-#define RADEON_SURFACE6_INFO		0x0b6c
-#define RADEON_SURFACE6_LOWER_BOUND	0x0b64
-#define RADEON_SURFACE6_UPPER_BOUND	0x0b68
-#define RADEON_SURFACE7_INFO		0x0b7c
-#define RADEON_SURFACE7_LOWER_BOUND	0x0b74
-#define RADEON_SURFACE7_UPPER_BOUND	0x0b78
-#define RADEON_SW_SEMAPHORE		0x013c
-
-#define RADEON_WAIT_UNTIL		0x1720
-#	define RADEON_WAIT_CRTC_PFLIP		(1 << 0)
-#	define RADEON_WAIT_2D_IDLE		(1 << 14)
-#	define RADEON_WAIT_3D_IDLE		(1 << 15)
-#	define RADEON_WAIT_2D_IDLECLEAN		(1 << 16)
-#	define RADEON_WAIT_3D_IDLECLEAN		(1 << 17)
-#	define RADEON_WAIT_HOST_IDLECLEAN	(1 << 18)
-
-#define RADEON_RB3D_ZMASKOFFSET		0x3234
-#define RADEON_RB3D_ZSTENCILCNTL	0x1c2c
-#	define RADEON_DEPTH_FORMAT_16BIT_INT_Z	(0 << 0)
-#	define RADEON_DEPTH_FORMAT_24BIT_INT_Z	(2 << 0)
-
-/* CP registers */
-#define RADEON_CP_ME_RAM_ADDR		0x07d4
-#define RADEON_CP_ME_RAM_RADDR		0x07d8
-#define RADEON_CP_ME_RAM_DATAH		0x07dc
-#define RADEON_CP_ME_RAM_DATAL		0x07e0
-
-#define RADEON_CP_RB_BASE		0x0700
-#define RADEON_CP_RB_CNTL		0x0704
-#	define RADEON_BUF_SWAP_32BIT		(2 << 16)
-#	define RADEON_RB_NO_UPDATE		(1 << 27)
-#	define RADEON_RB_RPTR_WR_ENA		(1 << 31)
-#define RADEON_CP_RB_RPTR_ADDR		0x070c
-#define RADEON_CP_RB_RPTR		0x0710
-#define RADEON_CP_RB_WPTR		0x0714
-
-#define RADEON_CP_RB_WPTR_DELAY		0x0718
-#	define RADEON_PRE_WRITE_TIMER_SHIFT	0
-#	define RADEON_PRE_WRITE_LIMIT_SHIFT	23
-
-#define RADEON_CP_IB_BASE		0x0738
-
-#define RADEON_CP_CSQ_CNTL		0x0740
-#	define RADEON_CSQ_CNT_PRIMARY_MASK	(0xff << 0)
-#	define RADEON_CSQ_PRIDIS_INDDIS		(0 << 28)
-#	define RADEON_CSQ_PRIPIO_INDDIS		(1 << 28)
-#	define RADEON_CSQ_PRIBM_INDDIS		(2 << 28)
-#	define RADEON_CSQ_PRIPIO_INDBM		(3 << 28)
-#	define RADEON_CSQ_PRIBM_INDBM		(4 << 28)
-#	define RADEON_CSQ_PRIPIO_INDPIO		(15 << 28)
-
-#define R300_CP_RESYNC_ADDR		0x0778
-#define R300_CP_RESYNC_DATA		0x077c
-
-#define RADEON_AIC_CNTL			0x01d0
-#	define RADEON_PCIGART_TRANSLATE_EN	(1 << 0)
-#	define RS400_MSI_REARM	                (1 << 3)
-#define RADEON_AIC_STAT			0x01d4
-#define RADEON_AIC_PT_BASE		0x01d8
-#define RADEON_AIC_LO_ADDR		0x01dc
-#define RADEON_AIC_HI_ADDR		0x01e0
-#define RADEON_AIC_TLB_ADDR		0x01e4
-#define RADEON_AIC_TLB_DATA		0x01e8
-
-/* CP command packets */
-#define RADEON_CP_PACKET0		0x00000000
-#	define RADEON_ONE_REG_WR		(1 << 15)
-#define RADEON_CP_PACKET1		0x40000000
-#define RADEON_CP_PACKET2		0x80000000
-#define RADEON_CP_PACKET3		0xC0000000
-#       define RADEON_CP_NOP                    0x00001000
-#       define RADEON_CP_NEXT_CHAR              0x00001900
-#       define RADEON_CP_PLY_NEXTSCAN           0x00001D00
-#       define RADEON_CP_SET_SCISSORS           0x00001E00
-	     /* GEN_INDX_PRIM is unsupported starting with R300 */
-#	define RADEON_3D_RNDR_GEN_INDX_PRIM	0x00002300
-#	define RADEON_WAIT_FOR_IDLE		0x00002600
-#	define RADEON_3D_DRAW_VBUF		0x00002800
-#	define RADEON_3D_DRAW_IMMD		0x00002900
-#	define RADEON_3D_DRAW_INDX		0x00002A00
-#       define RADEON_CP_LOAD_PALETTE           0x00002C00
-#	define RADEON_3D_LOAD_VBPNTR		0x00002F00
-#	define RADEON_MPEG_IDCT_MACROBLOCK	0x00003000
-#	define RADEON_MPEG_IDCT_MACROBLOCK_REV	0x00003100
-#	define RADEON_3D_CLEAR_ZMASK		0x00003200
-#	define RADEON_CP_INDX_BUFFER		0x00003300
-#       define RADEON_CP_3D_DRAW_VBUF_2         0x00003400
-#       define RADEON_CP_3D_DRAW_IMMD_2         0x00003500
-#       define RADEON_CP_3D_DRAW_INDX_2         0x00003600
-#	define RADEON_3D_CLEAR_HIZ		0x00003700
-#       define RADEON_CP_3D_CLEAR_CMASK         0x00003802
-#	define RADEON_CNTL_HOSTDATA_BLT		0x00009400
-#	define RADEON_CNTL_PAINT_MULTI		0x00009A00
-#	define RADEON_CNTL_BITBLT_MULTI		0x00009B00
-#	define RADEON_CNTL_SET_SCISSORS		0xC0001E00
-
-#       define R600_IT_INDIRECT_BUFFER_END      0x00001700
-#       define R600_IT_SET_PREDICATION          0x00002000
-#       define R600_IT_REG_RMW                  0x00002100
-#       define R600_IT_COND_EXEC                0x00002200
-#       define R600_IT_PRED_EXEC                0x00002300
-#       define R600_IT_START_3D_CMDBUF          0x00002400
-#       define R600_IT_DRAW_INDEX_2             0x00002700
-#       define R600_IT_CONTEXT_CONTROL          0x00002800
-#       define R600_IT_DRAW_INDEX_IMMD_BE       0x00002900
-#       define R600_IT_INDEX_TYPE               0x00002A00
-#       define R600_IT_DRAW_INDEX               0x00002B00
-#       define R600_IT_DRAW_INDEX_AUTO          0x00002D00
-#       define R600_IT_DRAW_INDEX_IMMD          0x00002E00
-#       define R600_IT_NUM_INSTANCES            0x00002F00
-#       define R600_IT_STRMOUT_BUFFER_UPDATE    0x00003400
-#       define R600_IT_INDIRECT_BUFFER_MP       0x00003800
-#       define R600_IT_MEM_SEMAPHORE            0x00003900
-#       define R600_IT_MPEG_INDEX               0x00003A00
-#       define R600_IT_WAIT_REG_MEM             0x00003C00
-#       define R600_IT_MEM_WRITE                0x00003D00
-#       define R600_IT_INDIRECT_BUFFER          0x00003200
-#       define R600_IT_SURFACE_SYNC             0x00004300
-#              define R600_CB0_DEST_BASE_ENA    (1 << 6)
-#              define R600_TC_ACTION_ENA        (1 << 23)
-#              define R600_VC_ACTION_ENA        (1 << 24)
-#              define R600_CB_ACTION_ENA        (1 << 25)
-#              define R600_DB_ACTION_ENA        (1 << 26)
-#              define R600_SH_ACTION_ENA        (1 << 27)
-#              define R600_SMX_ACTION_ENA       (1 << 28)
-#       define R600_IT_ME_INITIALIZE            0x00004400
-#	       define R600_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
-#       define R600_IT_COND_WRITE               0x00004500
-#       define R600_IT_EVENT_WRITE              0x00004600
-#       define R600_IT_EVENT_WRITE_EOP          0x00004700
-#       define R600_IT_ONE_REG_WRITE            0x00005700
-#       define R600_IT_SET_CONFIG_REG           0x00006800
-#              define R600_SET_CONFIG_REG_OFFSET 0x00008000
-#              define R600_SET_CONFIG_REG_END   0x0000ac00
-#       define R600_IT_SET_CONTEXT_REG          0x00006900
-#              define R600_SET_CONTEXT_REG_OFFSET 0x00028000
-#              define R600_SET_CONTEXT_REG_END  0x00029000
-#       define R600_IT_SET_ALU_CONST            0x00006A00
-#              define R600_SET_ALU_CONST_OFFSET 0x00030000
-#              define R600_SET_ALU_CONST_END    0x00032000
-#       define R600_IT_SET_BOOL_CONST           0x00006B00
-#              define R600_SET_BOOL_CONST_OFFSET 0x0003e380
-#              define R600_SET_BOOL_CONST_END   0x00040000
-#       define R600_IT_SET_LOOP_CONST           0x00006C00
-#              define R600_SET_LOOP_CONST_OFFSET 0x0003e200
-#              define R600_SET_LOOP_CONST_END   0x0003e380
-#       define R600_IT_SET_RESOURCE             0x00006D00
-#              define R600_SET_RESOURCE_OFFSET  0x00038000
-#              define R600_SET_RESOURCE_END     0x0003c000
-#              define R600_SQ_TEX_VTX_INVALID_TEXTURE  0x0
-#              define R600_SQ_TEX_VTX_INVALID_BUFFER   0x1
-#              define R600_SQ_TEX_VTX_VALID_TEXTURE    0x2
-#              define R600_SQ_TEX_VTX_VALID_BUFFER     0x3
-#       define R600_IT_SET_SAMPLER              0x00006E00
-#              define R600_SET_SAMPLER_OFFSET   0x0003c000
-#              define R600_SET_SAMPLER_END      0x0003cff0
-#       define R600_IT_SET_CTL_CONST            0x00006F00
-#              define R600_SET_CTL_CONST_OFFSET 0x0003cff0
-#              define R600_SET_CTL_CONST_END    0x0003e200
-#       define R600_IT_SURFACE_BASE_UPDATE      0x00007300
-
-#define RADEON_CP_PACKET_MASK		0xC0000000
-#define RADEON_CP_PACKET_COUNT_MASK	0x3fff0000
-#define RADEON_CP_PACKET0_REG_MASK	0x000007ff
-#define RADEON_CP_PACKET1_REG0_MASK	0x000007ff
-#define RADEON_CP_PACKET1_REG1_MASK	0x003ff800
-
-#define RADEON_VTX_Z_PRESENT			(1 << 31)
-#define RADEON_VTX_PKCOLOR_PRESENT		(1 << 3)
-
-#define RADEON_PRIM_TYPE_NONE			(0 << 0)
-#define RADEON_PRIM_TYPE_POINT			(1 << 0)
-#define RADEON_PRIM_TYPE_LINE			(2 << 0)
-#define RADEON_PRIM_TYPE_LINE_STRIP		(3 << 0)
-#define RADEON_PRIM_TYPE_TRI_LIST		(4 << 0)
-#define RADEON_PRIM_TYPE_TRI_FAN		(5 << 0)
-#define RADEON_PRIM_TYPE_TRI_STRIP		(6 << 0)
-#define RADEON_PRIM_TYPE_TRI_TYPE2		(7 << 0)
-#define RADEON_PRIM_TYPE_RECT_LIST		(8 << 0)
-#define RADEON_PRIM_TYPE_3VRT_POINT_LIST	(9 << 0)
-#define RADEON_PRIM_TYPE_3VRT_LINE_LIST		(10 << 0)
-#define RADEON_PRIM_TYPE_MASK                   0xf
-#define RADEON_PRIM_WALK_IND			(1 << 4)
-#define RADEON_PRIM_WALK_LIST			(2 << 4)
-#define RADEON_PRIM_WALK_RING			(3 << 4)
-#define RADEON_COLOR_ORDER_BGRA			(0 << 6)
-#define RADEON_COLOR_ORDER_RGBA			(1 << 6)
-#define RADEON_MAOS_ENABLE			(1 << 7)
-#define RADEON_VTX_FMT_R128_MODE		(0 << 8)
-#define RADEON_VTX_FMT_RADEON_MODE		(1 << 8)
-#define RADEON_NUM_VERTICES_SHIFT		16
-
-#define RADEON_COLOR_FORMAT_CI8		2
-#define RADEON_COLOR_FORMAT_ARGB1555	3
-#define RADEON_COLOR_FORMAT_RGB565	4
-#define RADEON_COLOR_FORMAT_ARGB8888	6
-#define RADEON_COLOR_FORMAT_RGB332	7
-#define RADEON_COLOR_FORMAT_RGB8	9
-#define RADEON_COLOR_FORMAT_ARGB4444	15
-
-#define RADEON_TXFORMAT_I8		0
-#define RADEON_TXFORMAT_AI88		1
-#define RADEON_TXFORMAT_RGB332		2
-#define RADEON_TXFORMAT_ARGB1555	3
-#define RADEON_TXFORMAT_RGB565		4
-#define RADEON_TXFORMAT_ARGB4444	5
-#define RADEON_TXFORMAT_ARGB8888	6
-#define RADEON_TXFORMAT_RGBA8888	7
-#define RADEON_TXFORMAT_Y8		8
-#define RADEON_TXFORMAT_VYUY422         10
-#define RADEON_TXFORMAT_YVYU422         11
-#define RADEON_TXFORMAT_DXT1            12
-#define RADEON_TXFORMAT_DXT23           14
-#define RADEON_TXFORMAT_DXT45           15
-
-#define R200_PP_TXCBLEND_0                0x2f00
-#define R200_PP_TXCBLEND_1                0x2f10
-#define R200_PP_TXCBLEND_2                0x2f20
-#define R200_PP_TXCBLEND_3                0x2f30
-#define R200_PP_TXCBLEND_4                0x2f40
-#define R200_PP_TXCBLEND_5                0x2f50
-#define R200_PP_TXCBLEND_6                0x2f60
-#define R200_PP_TXCBLEND_7                0x2f70
-#define R200_SE_TCL_LIGHT_MODEL_CTL_0     0x2268
-#define R200_PP_TFACTOR_0                 0x2ee0
-#define R200_SE_VTX_FMT_0                 0x2088
-#define R200_SE_VAP_CNTL                  0x2080
-#define R200_SE_TCL_MATRIX_SEL_0          0x2230
-#define R200_SE_TCL_TEX_PROC_CTL_2        0x22a8
-#define R200_SE_TCL_UCP_VERT_BLEND_CTL    0x22c0
-#define R200_PP_TXFILTER_5                0x2ca0
-#define R200_PP_TXFILTER_4                0x2c80
-#define R200_PP_TXFILTER_3                0x2c60
-#define R200_PP_TXFILTER_2                0x2c40
-#define R200_PP_TXFILTER_1                0x2c20
-#define R200_PP_TXFILTER_0                0x2c00
-#define R200_PP_TXOFFSET_5                0x2d78
-#define R200_PP_TXOFFSET_4                0x2d60
-#define R200_PP_TXOFFSET_3                0x2d48
-#define R200_PP_TXOFFSET_2                0x2d30
-#define R200_PP_TXOFFSET_1                0x2d18
-#define R200_PP_TXOFFSET_0                0x2d00
-
-#define R200_PP_CUBIC_FACES_0             0x2c18
-#define R200_PP_CUBIC_FACES_1             0x2c38
-#define R200_PP_CUBIC_FACES_2             0x2c58
-#define R200_PP_CUBIC_FACES_3             0x2c78
-#define R200_PP_CUBIC_FACES_4             0x2c98
-#define R200_PP_CUBIC_FACES_5             0x2cb8
-#define R200_PP_CUBIC_OFFSET_F1_0         0x2d04
-#define R200_PP_CUBIC_OFFSET_F2_0         0x2d08
-#define R200_PP_CUBIC_OFFSET_F3_0         0x2d0c
-#define R200_PP_CUBIC_OFFSET_F4_0         0x2d10
-#define R200_PP_CUBIC_OFFSET_F5_0         0x2d14
-#define R200_PP_CUBIC_OFFSET_F1_1         0x2d1c
-#define R200_PP_CUBIC_OFFSET_F2_1         0x2d20
-#define R200_PP_CUBIC_OFFSET_F3_1         0x2d24
-#define R200_PP_CUBIC_OFFSET_F4_1         0x2d28
-#define R200_PP_CUBIC_OFFSET_F5_1         0x2d2c
-#define R200_PP_CUBIC_OFFSET_F1_2         0x2d34
-#define R200_PP_CUBIC_OFFSET_F2_2         0x2d38
-#define R200_PP_CUBIC_OFFSET_F3_2         0x2d3c
-#define R200_PP_CUBIC_OFFSET_F4_2         0x2d40
-#define R200_PP_CUBIC_OFFSET_F5_2         0x2d44
-#define R200_PP_CUBIC_OFFSET_F1_3         0x2d4c
-#define R200_PP_CUBIC_OFFSET_F2_3         0x2d50
-#define R200_PP_CUBIC_OFFSET_F3_3         0x2d54
-#define R200_PP_CUBIC_OFFSET_F4_3         0x2d58
-#define R200_PP_CUBIC_OFFSET_F5_3         0x2d5c
-#define R200_PP_CUBIC_OFFSET_F1_4         0x2d64
-#define R200_PP_CUBIC_OFFSET_F2_4         0x2d68
-#define R200_PP_CUBIC_OFFSET_F3_4         0x2d6c
-#define R200_PP_CUBIC_OFFSET_F4_4         0x2d70
-#define R200_PP_CUBIC_OFFSET_F5_4         0x2d74
-#define R200_PP_CUBIC_OFFSET_F1_5         0x2d7c
-#define R200_PP_CUBIC_OFFSET_F2_5         0x2d80
-#define R200_PP_CUBIC_OFFSET_F3_5         0x2d84
-#define R200_PP_CUBIC_OFFSET_F4_5         0x2d88
-#define R200_PP_CUBIC_OFFSET_F5_5         0x2d8c
-
-#define R200_RE_AUX_SCISSOR_CNTL          0x26f0
-#define R200_SE_VTE_CNTL                  0x20b0
-#define R200_SE_TCL_OUTPUT_VTX_COMP_SEL   0x2250
-#define R200_PP_TAM_DEBUG3                0x2d9c
-#define R200_PP_CNTL_X                    0x2cc4
-#define R200_SE_VAP_CNTL_STATUS           0x2140
-#define R200_RE_SCISSOR_TL_0              0x1cd8
-#define R200_RE_SCISSOR_TL_1              0x1ce0
-#define R200_RE_SCISSOR_TL_2              0x1ce8
-#define R200_RB3D_DEPTHXY_OFFSET          0x1d60
-#define R200_RE_AUX_SCISSOR_CNTL          0x26f0
-#define R200_SE_VTX_STATE_CNTL            0x2180
-#define R200_RE_POINTSIZE                 0x2648
-#define R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0 0x2254
-
-#define RADEON_PP_TEX_SIZE_0                0x1d04	/* NPOT */
-#define RADEON_PP_TEX_SIZE_1                0x1d0c
-#define RADEON_PP_TEX_SIZE_2                0x1d14
-
-#define RADEON_PP_CUBIC_FACES_0             0x1d24
-#define RADEON_PP_CUBIC_FACES_1             0x1d28
-#define RADEON_PP_CUBIC_FACES_2             0x1d2c
-#define RADEON_PP_CUBIC_OFFSET_T0_0         0x1dd0	/* bits [31:5] */
-#define RADEON_PP_CUBIC_OFFSET_T1_0         0x1e00
-#define RADEON_PP_CUBIC_OFFSET_T2_0         0x1e14
-
-#define RADEON_SE_TCL_STATE_FLUSH           0x2284
-
-#define SE_VAP_CNTL__TCL_ENA_MASK                          0x00000001
-#define SE_VAP_CNTL__FORCE_W_TO_ONE_MASK                   0x00010000
-#define SE_VAP_CNTL__VF_MAX_VTX_NUM__SHIFT                 0x00000012
-#define SE_VTE_CNTL__VTX_XY_FMT_MASK                       0x00000100
-#define SE_VTE_CNTL__VTX_Z_FMT_MASK                        0x00000200
-#define SE_VTX_FMT_0__VTX_Z0_PRESENT_MASK                  0x00000001
-#define SE_VTX_FMT_0__VTX_W0_PRESENT_MASK                  0x00000002
-#define SE_VTX_FMT_0__VTX_COLOR_0_FMT__SHIFT               0x0000000b
-#define R200_3D_DRAW_IMMD_2      0xC0003500
-#define R200_SE_VTX_FMT_1                 0x208c
-#define R200_RE_CNTL                      0x1c50
-
-#define R200_RB3D_BLENDCOLOR              0x3218
-
-#define R200_SE_TCL_POINT_SPRITE_CNTL     0x22c4
-
-#define R200_PP_TRI_PERF 0x2cf8
-
-#define R200_PP_AFS_0                     0x2f80
-#define R200_PP_AFS_1                     0x2f00	/* same as txcblend_0 */
-
-#define R200_VAP_PVS_CNTL_1               0x22D0
-
-#define RADEON_CRTC_CRNT_FRAME 0x0214
-#define RADEON_CRTC2_CRNT_FRAME 0x0314
-
-#define R500_D1CRTC_STATUS 0x609c
-#define R500_D2CRTC_STATUS 0x689c
-#define R500_CRTC_V_BLANK (1<<0)
-
-#define R500_D1CRTC_FRAME_COUNT 0x60a4
-#define R500_D2CRTC_FRAME_COUNT 0x68a4
-
-#define R500_D1MODE_V_COUNTER 0x6530
-#define R500_D2MODE_V_COUNTER 0x6d30
-
-#define R500_D1MODE_VBLANK_STATUS 0x6534
-#define R500_D2MODE_VBLANK_STATUS 0x6d34
-#define R500_VBLANK_OCCURED (1<<0)
-#define R500_VBLANK_ACK     (1<<4)
-#define R500_VBLANK_STAT    (1<<12)
-#define R500_VBLANK_INT     (1<<16)
-
-#define R500_DxMODE_INT_MASK 0x6540
-#define R500_D1MODE_INT_MASK (1<<0)
-#define R500_D2MODE_INT_MASK (1<<8)
-
-#define R500_DISP_INTERRUPT_STATUS 0x7edc
-#define R500_D1_VBLANK_INTERRUPT (1 << 4)
-#define R500_D2_VBLANK_INTERRUPT (1 << 5)
-
-/* R6xx/R7xx registers */
-#define R600_MC_VM_FB_LOCATION                                 0x2180
-#define R600_MC_VM_AGP_TOP                                     0x2184
-#define R600_MC_VM_AGP_BOT                                     0x2188
-#define R600_MC_VM_AGP_BASE                                    0x218c
-#define R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR                    0x2190
-#define R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR                   0x2194
-#define R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR                0x2198
-
-#define R700_MC_VM_FB_LOCATION                                 0x2024
-#define R700_MC_VM_AGP_TOP                                     0x2028
-#define R700_MC_VM_AGP_BOT                                     0x202c
-#define R700_MC_VM_AGP_BASE                                    0x2030
-#define R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR                    0x2034
-#define R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR                   0x2038
-#define R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR                0x203c
-
-#define R600_MCD_RD_A_CNTL                                     0x219c
-#define R600_MCD_RD_B_CNTL                                     0x21a0
-
-#define R600_MCD_WR_A_CNTL                                     0x21a4
-#define R600_MCD_WR_B_CNTL                                     0x21a8
-
-#define R600_MCD_RD_SYS_CNTL                                   0x2200
-#define R600_MCD_WR_SYS_CNTL                                   0x2214
-
-#define R600_MCD_RD_GFX_CNTL                                   0x21fc
-#define R600_MCD_RD_HDP_CNTL                                   0x2204
-#define R600_MCD_RD_PDMA_CNTL                                  0x2208
-#define R600_MCD_RD_SEM_CNTL                                   0x220c
-#define R600_MCD_WR_GFX_CNTL                                   0x2210
-#define R600_MCD_WR_HDP_CNTL                                   0x2218
-#define R600_MCD_WR_PDMA_CNTL                                  0x221c
-#define R600_MCD_WR_SEM_CNTL                                   0x2220
-
-#       define R600_MCD_L1_TLB                                 (1 << 0)
-#       define R600_MCD_L1_FRAG_PROC                           (1 << 1)
-#       define R600_MCD_L1_STRICT_ORDERING                     (1 << 2)
-
-#       define R600_MCD_SYSTEM_ACCESS_MODE_MASK                (3 << 6)
-#       define R600_MCD_SYSTEM_ACCESS_MODE_PA_ONLY             (0 << 6)
-#       define R600_MCD_SYSTEM_ACCESS_MODE_USE_SYS_MAP         (1 << 6)
-#       define R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS              (2 << 6)
-#       define R600_MCD_SYSTEM_ACCESS_MODE_NOT_IN_SYS          (3 << 6)
-
-#       define R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU    (0 << 8)
-#       define R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_DEFAULT_PAGE (1 << 8)
-
-#       define R600_MCD_SEMAPHORE_MODE                         (1 << 10)
-#       define R600_MCD_WAIT_L2_QUERY                          (1 << 11)
-#       define R600_MCD_EFFECTIVE_L1_TLB_SIZE(x)               ((x) << 12)
-#       define R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(x)             ((x) << 15)
-
-#define R700_MC_VM_MD_L1_TLB0_CNTL                             0x2654
-#define R700_MC_VM_MD_L1_TLB1_CNTL                             0x2658
-#define R700_MC_VM_MD_L1_TLB2_CNTL                             0x265c
-
-#define R700_MC_VM_MB_L1_TLB0_CNTL                             0x2234
-#define R700_MC_VM_MB_L1_TLB1_CNTL                             0x2238
-#define R700_MC_VM_MB_L1_TLB2_CNTL                             0x223c
-#define R700_MC_VM_MB_L1_TLB3_CNTL                             0x2240
-
-#       define R700_ENABLE_L1_TLB                              (1 << 0)
-#       define R700_ENABLE_L1_FRAGMENT_PROCESSING              (1 << 1)
-#       define R700_SYSTEM_ACCESS_MODE_IN_SYS                  (2 << 3)
-#       define R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU  (0 << 5)
-#       define R700_EFFECTIVE_L1_TLB_SIZE(x)                   ((x) << 15)
-#       define R700_EFFECTIVE_L1_QUEUE_SIZE(x)                 ((x) << 18)
-
-#define R700_MC_ARB_RAMCFG                                     0x2760
-#       define R700_NOOFBANK_SHIFT                             0
-#       define R700_NOOFBANK_MASK                              0x3
-#       define R700_NOOFRANK_SHIFT                             2
-#       define R700_NOOFRANK_MASK                              0x1
-#       define R700_NOOFROWS_SHIFT                             3
-#       define R700_NOOFROWS_MASK                              0x7
-#       define R700_NOOFCOLS_SHIFT                             6
-#       define R700_NOOFCOLS_MASK                              0x3
-#       define R700_CHANSIZE_SHIFT                             8
-#       define R700_CHANSIZE_MASK                              0x1
-#       define R700_BURSTLENGTH_SHIFT                          9
-#       define R700_BURSTLENGTH_MASK                           0x1
-#define R600_RAMCFG                                            0x2408
-#       define R600_NOOFBANK_SHIFT                             0
-#       define R600_NOOFBANK_MASK                              0x1
-#       define R600_NOOFRANK_SHIFT                             1
-#       define R600_NOOFRANK_MASK                              0x1
-#       define R600_NOOFROWS_SHIFT                             2
-#       define R600_NOOFROWS_MASK                              0x7
-#       define R600_NOOFCOLS_SHIFT                             5
-#       define R600_NOOFCOLS_MASK                              0x3
-#       define R600_CHANSIZE_SHIFT                             7
-#       define R600_CHANSIZE_MASK                              0x1
-#       define R600_BURSTLENGTH_SHIFT                          8
-#       define R600_BURSTLENGTH_MASK                           0x1
-
-#define R600_VM_L2_CNTL                                        0x1400
-#       define R600_VM_L2_CACHE_EN                             (1 << 0)
-#       define R600_VM_L2_FRAG_PROC                            (1 << 1)
-#       define R600_VM_ENABLE_PTE_CACHE_LRU_W                  (1 << 9)
-#       define R600_VM_L2_CNTL_QUEUE_SIZE(x)                   ((x) << 13)
-#       define R700_VM_L2_CNTL_QUEUE_SIZE(x)                   ((x) << 14)
-
-#define R600_VM_L2_CNTL2                                       0x1404
-#       define R600_VM_L2_CNTL2_INVALIDATE_ALL_L1_TLBS         (1 << 0)
-#       define R600_VM_L2_CNTL2_INVALIDATE_L2_CACHE            (1 << 1)
-#define R600_VM_L2_CNTL3                                       0x1408
-#       define R600_VM_L2_CNTL3_BANK_SELECT_0(x)               ((x) << 0)
-#       define R600_VM_L2_CNTL3_BANK_SELECT_1(x)               ((x) << 5)
-#       define R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(x)           ((x) << 10)
-#       define R700_VM_L2_CNTL3_BANK_SELECT(x)                 ((x) << 0)
-#       define R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(x)           ((x) << 6)
-
-#define R600_VM_L2_STATUS                                      0x140c
-
-#define R600_VM_CONTEXT0_CNTL                                  0x1410
-#       define R600_VM_ENABLE_CONTEXT                          (1 << 0)
-#       define R600_VM_PAGE_TABLE_DEPTH_FLAT                   (0 << 1)
-
-#define R600_VM_CONTEXT0_CNTL2                                 0x1430
-#define R600_VM_CONTEXT0_REQUEST_RESPONSE                      0x1470
-#define R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR                 0x1490
-#define R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR                0x14b0
-#define R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR                  0x1574
-#define R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR                 0x1594
-#define R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR                   0x15b4
-
-#define R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR                  0x153c
-#define R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR                 0x155c
-#define R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR                   0x157c
-
-#define R600_HDP_HOST_PATH_CNTL                                0x2c00
-
-#define R600_GRBM_CNTL                                         0x8000
-#       define R600_GRBM_READ_TIMEOUT(x)                       ((x) << 0)
-
-#define R600_GRBM_STATUS                                       0x8010
-#       define R600_CMDFIFO_AVAIL_MASK                         0x1f
-#       define R700_CMDFIFO_AVAIL_MASK                         0xf
-#       define R600_GUI_ACTIVE                                 (1 << 31)
-#define R600_GRBM_STATUS2                                      0x8014
-#define R600_GRBM_SOFT_RESET                                   0x8020
-#       define R600_SOFT_RESET_CP                              (1 << 0)
-#define R600_WAIT_UNTIL		                               0x8040
-
-#define R600_CP_SEM_WAIT_TIMER                                 0x85bc
-#define R600_CP_ME_CNTL                                        0x86d8
-#       define R600_CP_ME_HALT                                 (1 << 28)
-#define R600_CP_QUEUE_THRESHOLDS                               0x8760
-#       define R600_ROQ_IB1_START(x)                           ((x) << 0)
-#       define R600_ROQ_IB2_START(x)                           ((x) << 8)
-#define R600_CP_MEQ_THRESHOLDS                                 0x8764
-#       define R700_STQ_SPLIT(x)                               ((x) << 0)
-#       define R600_MEQ_END(x)                                 ((x) << 16)
-#       define R600_ROQ_END(x)                                 ((x) << 24)
-#define R600_CP_PERFMON_CNTL                                   0x87fc
-#define R600_CP_RB_BASE                                        0xc100
-#define R600_CP_RB_CNTL                                        0xc104
-#       define R600_RB_BUFSZ(x)                                ((x) << 0)
-#       define R600_RB_BLKSZ(x)                                ((x) << 8)
-#	define R600_BUF_SWAP_32BIT		               (2 << 16)
-#       define R600_RB_NO_UPDATE                               (1 << 27)
-#       define R600_RB_RPTR_WR_ENA                             (1 << 31)
-#define R600_CP_RB_RPTR_WR                                     0xc108
-#define R600_CP_RB_RPTR_ADDR                                   0xc10c
-#define R600_CP_RB_RPTR_ADDR_HI                                0xc110
-#define R600_CP_RB_WPTR                                        0xc114
-#define R600_CP_RB_WPTR_ADDR                                   0xc118
-#define R600_CP_RB_WPTR_ADDR_HI                                0xc11c
-#define R600_CP_RB_RPTR                                        0x8700
-#define R600_CP_RB_WPTR_DELAY                                  0x8704
-#define R600_CP_PFP_UCODE_ADDR                                 0xc150
-#define R600_CP_PFP_UCODE_DATA                                 0xc154
-#define R600_CP_ME_RAM_RADDR                                   0xc158
-#define R600_CP_ME_RAM_WADDR                                   0xc15c
-#define R600_CP_ME_RAM_DATA                                    0xc160
-#define R600_CP_DEBUG                                          0xc1fc
-
-#define R600_PA_CL_ENHANCE                                     0x8a14
-#       define R600_CLIP_VTX_REORDER_ENA                       (1 << 0)
-#       define R600_NUM_CLIP_SEQ(x)                            ((x) << 1)
-#define R600_PA_SC_LINE_STIPPLE_STATE                          0x8b10
-#define R600_PA_SC_MULTI_CHIP_CNTL                             0x8b20
-#define R700_PA_SC_FORCE_EOV_MAX_CNTS                          0x8b24
-#       define R700_FORCE_EOV_MAX_CLK_CNT(x)                   ((x) << 0)
-#       define R700_FORCE_EOV_MAX_REZ_CNT(x)                   ((x) << 16)
-#define R600_PA_SC_AA_SAMPLE_LOCS_2S                           0x8b40
-#define R600_PA_SC_AA_SAMPLE_LOCS_4S                           0x8b44
-#define R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0                       0x8b48
-#define R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1                       0x8b4c
-#       define R600_S0_X(x)                                    ((x) << 0)
-#       define R600_S0_Y(x)                                    ((x) << 4)
-#       define R600_S1_X(x)                                    ((x) << 8)
-#       define R600_S1_Y(x)                                    ((x) << 12)
-#       define R600_S2_X(x)                                    ((x) << 16)
-#       define R600_S2_Y(x)                                    ((x) << 20)
-#       define R600_S3_X(x)                                    ((x) << 24)
-#       define R600_S3_Y(x)                                    ((x) << 28)
-#       define R600_S4_X(x)                                    ((x) << 0)
-#       define R600_S4_Y(x)                                    ((x) << 4)
-#       define R600_S5_X(x)                                    ((x) << 8)
-#       define R600_S5_Y(x)                                    ((x) << 12)
-#       define R600_S6_X(x)                                    ((x) << 16)
-#       define R600_S6_Y(x)                                    ((x) << 20)
-#       define R600_S7_X(x)                                    ((x) << 24)
-#       define R600_S7_Y(x)                                    ((x) << 28)
-#define R600_PA_SC_FIFO_SIZE                                   0x8bd0
-#       define R600_SC_PRIM_FIFO_SIZE(x)                       ((x) << 0)
-#       define R600_SC_HIZ_TILE_FIFO_SIZE(x)                   ((x) << 8)
-#       define R600_SC_EARLYZ_TILE_FIFO_SIZE(x)                ((x) << 16)
-#define R700_PA_SC_FIFO_SIZE_R7XX                              0x8bcc
-#       define R700_SC_PRIM_FIFO_SIZE(x)                       ((x) << 0)
-#       define R700_SC_HIZ_TILE_FIFO_SIZE(x)                   ((x) << 12)
-#       define R700_SC_EARLYZ_TILE_FIFO_SIZE(x)                ((x) << 20)
-#define R600_PA_SC_ENHANCE                                     0x8bf0
-#       define R600_FORCE_EOV_MAX_CLK_CNT(x)                   ((x) << 0)
-#       define R600_FORCE_EOV_MAX_TILE_CNT(x)                  ((x) << 12)
-#define R600_PA_SC_CLIPRECT_RULE                               0x2820c
-#define R700_PA_SC_EDGERULE                                    0x28230
-#define R600_PA_SC_LINE_STIPPLE                                0x28a0c
-#define R600_PA_SC_MODE_CNTL                                   0x28a4c
-#define R600_PA_SC_AA_CONFIG                                   0x28c04
-
-#define R600_SX_EXPORT_BUFFER_SIZES                            0x900c
-#       define R600_COLOR_BUFFER_SIZE(x)                       ((x) << 0)
-#       define R600_POSITION_BUFFER_SIZE(x)                    ((x) << 8)
-#       define R600_SMX_BUFFER_SIZE(x)                         ((x) << 16)
-#define R600_SX_DEBUG_1                                        0x9054
-#       define R600_SMX_EVENT_RELEASE                          (1 << 0)
-#       define R600_ENABLE_NEW_SMX_ADDRESS                     (1 << 16)
-#define R700_SX_DEBUG_1                                        0x9058
-#       define R700_ENABLE_NEW_SMX_ADDRESS                     (1 << 16)
-#define R600_SX_MISC                                           0x28350
-
-#define R600_DB_DEBUG                                          0x9830
-#       define R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE              (1 << 31)
-#define R600_DB_WATERMARKS                                     0x9838
-#       define R600_DEPTH_FREE(x)                              ((x) << 0)
-#       define R600_DEPTH_FLUSH(x)                             ((x) << 5)
-#       define R600_DEPTH_PENDING_FREE(x)                      ((x) << 15)
-#       define R600_DEPTH_CACHELINE_FREE(x)                    ((x) << 20)
-#define R700_DB_DEBUG3                                         0x98b0
-#       define R700_DB_CLK_OFF_DELAY(x)                        ((x) << 11)
-#define RV700_DB_DEBUG4                                        0x9b8c
-#       define RV700_DISABLE_TILE_COVERED_FOR_PS_ITER          (1 << 6)
-
-#define R600_VGT_CACHE_INVALIDATION                            0x88c4
-#       define R600_CACHE_INVALIDATION(x)                      ((x) << 0)
-#       define R600_VC_ONLY                                    0
-#       define R600_TC_ONLY                                    1
-#       define R600_VC_AND_TC                                  2
-#       define R700_AUTO_INVLD_EN(x)                           ((x) << 6)
-#       define R700_NO_AUTO                                    0
-#       define R700_ES_AUTO                                    1
-#       define R700_GS_AUTO                                    2
-#       define R700_ES_AND_GS_AUTO                             3
-#define R600_VGT_GS_PER_ES                                     0x88c8
-#define R600_VGT_ES_PER_GS                                     0x88cc
-#define R600_VGT_GS_PER_VS                                     0x88e8
-#define R600_VGT_GS_VERTEX_REUSE                               0x88d4
-#define R600_VGT_NUM_INSTANCES                                 0x8974
-#define R600_VGT_STRMOUT_EN                                    0x28ab0
-#define R600_VGT_EVENT_INITIATOR                               0x28a90
-#       define R600_CACHE_FLUSH_AND_INV_EVENT                  (0x16 << 0)
-#define R600_VGT_VERTEX_REUSE_BLOCK_CNTL                       0x28c58
-#       define R600_VTX_REUSE_DEPTH_MASK                       0xff
-#define R600_VGT_OUT_DEALLOC_CNTL                              0x28c5c
-#       define R600_DEALLOC_DIST_MASK                          0x7f
-
-#define R600_CB_COLOR0_BASE                                    0x28040
-#define R600_CB_COLOR1_BASE                                    0x28044
-#define R600_CB_COLOR2_BASE                                    0x28048
-#define R600_CB_COLOR3_BASE                                    0x2804c
-#define R600_CB_COLOR4_BASE                                    0x28050
-#define R600_CB_COLOR5_BASE                                    0x28054
-#define R600_CB_COLOR6_BASE                                    0x28058
-#define R600_CB_COLOR7_BASE                                    0x2805c
-#define R600_CB_COLOR7_FRAG                                    0x280fc
-
-#define R600_CB_COLOR0_SIZE                                    0x28060
-#define R600_CB_COLOR0_VIEW                                    0x28080
-#define R600_CB_COLOR0_INFO                                    0x280a0
-#define R600_CB_COLOR0_TILE                                    0x280c0
-#define R600_CB_COLOR0_FRAG                                    0x280e0
-#define R600_CB_COLOR0_MASK                                    0x28100
-
-#define AVIVO_D1MODE_VLINE_START_END                           0x6538
-#define AVIVO_D2MODE_VLINE_START_END                           0x6d38
-#define R600_CP_COHER_BASE                                     0x85f8
-#define R600_DB_DEPTH_BASE                                     0x2800c
-#define R600_SQ_PGM_START_FS                                   0x28894
-#define R600_SQ_PGM_START_ES                                   0x28880
-#define R600_SQ_PGM_START_VS                                   0x28858
-#define R600_SQ_PGM_RESOURCES_VS                               0x28868
-#define R600_SQ_PGM_CF_OFFSET_VS                               0x288d0
-#define R600_SQ_PGM_START_GS                                   0x2886c
-#define R600_SQ_PGM_START_PS                                   0x28840
-#define R600_SQ_PGM_RESOURCES_PS                               0x28850
-#define R600_SQ_PGM_EXPORTS_PS                                 0x28854
-#define R600_SQ_PGM_CF_OFFSET_PS                               0x288cc
-#define R600_VGT_DMA_BASE                                      0x287e8
-#define R600_VGT_DMA_BASE_HI                                   0x287e4
-#define R600_VGT_STRMOUT_BASE_OFFSET_0                         0x28b10
-#define R600_VGT_STRMOUT_BASE_OFFSET_1                         0x28b14
-#define R600_VGT_STRMOUT_BASE_OFFSET_2                         0x28b18
-#define R600_VGT_STRMOUT_BASE_OFFSET_3                         0x28b1c
-#define R600_VGT_STRMOUT_BASE_OFFSET_HI_0                      0x28b44
-#define R600_VGT_STRMOUT_BASE_OFFSET_HI_1                      0x28b48
-#define R600_VGT_STRMOUT_BASE_OFFSET_HI_2                      0x28b4c
-#define R600_VGT_STRMOUT_BASE_OFFSET_HI_3                      0x28b50
-#define R600_VGT_STRMOUT_BUFFER_BASE_0                         0x28ad8
-#define R600_VGT_STRMOUT_BUFFER_BASE_1                         0x28ae8
-#define R600_VGT_STRMOUT_BUFFER_BASE_2                         0x28af8
-#define R600_VGT_STRMOUT_BUFFER_BASE_3                         0x28b08
-#define R600_VGT_STRMOUT_BUFFER_OFFSET_0                       0x28adc
-#define R600_VGT_STRMOUT_BUFFER_OFFSET_1                       0x28aec
-#define R600_VGT_STRMOUT_BUFFER_OFFSET_2                       0x28afc
-#define R600_VGT_STRMOUT_BUFFER_OFFSET_3                       0x28b0c
-
-#define R600_VGT_PRIMITIVE_TYPE                                0x8958
-
-#define R600_PA_SC_SCREEN_SCISSOR_TL                           0x28030
-#define R600_PA_SC_GENERIC_SCISSOR_TL                          0x28240
-#define R600_PA_SC_WINDOW_SCISSOR_TL                           0x28204
-
-#define R600_TC_CNTL                                           0x9608
-#       define R600_TC_L2_SIZE(x)                              ((x) << 5)
-#       define R600_L2_DISABLE_LATE_HIT                        (1 << 9)
-
-#define R600_ARB_POP                                           0x2418
-#       define R600_ENABLE_TC128                               (1 << 30)
-#define R600_ARB_GDEC_RD_CNTL                                  0x246c
-
-#define R600_TA_CNTL_AUX                                       0x9508
-#       define R600_DISABLE_CUBE_WRAP                          (1 << 0)
-#       define R600_DISABLE_CUBE_ANISO                         (1 << 1)
-#       define R700_GETLOD_SELECT(x)                           ((x) << 2)
-#       define R600_SYNC_GRADIENT                              (1 << 24)
-#       define R600_SYNC_WALKER                                (1 << 25)
-#       define R600_SYNC_ALIGNER                               (1 << 26)
-#       define R600_BILINEAR_PRECISION_6_BIT                   (0 << 31)
-#       define R600_BILINEAR_PRECISION_8_BIT                   (1 << 31)
-
-#define R700_TCP_CNTL                                          0x9610
-
-#define R600_SMX_DC_CTL0                                       0xa020
-#       define R700_USE_HASH_FUNCTION                          (1 << 0)
-#       define R700_CACHE_DEPTH(x)                             ((x) << 1)
-#       define R700_FLUSH_ALL_ON_EVENT                         (1 << 10)
-#       define R700_STALL_ON_EVENT                             (1 << 11)
-#define R700_SMX_EVENT_CTL                                     0xa02c
-#       define R700_ES_FLUSH_CTL(x)                            ((x) << 0)
-#       define R700_GS_FLUSH_CTL(x)                            ((x) << 3)
-#       define R700_ACK_FLUSH_CTL(x)                           ((x) << 6)
-#       define R700_SYNC_FLUSH_CTL                             (1 << 8)
-
-#define R600_SQ_CONFIG                                         0x8c00
-#       define R600_VC_ENABLE                                  (1 << 0)
-#       define R600_EXPORT_SRC_C                               (1 << 1)
-#       define R600_DX9_CONSTS                                 (1 << 2)
-#       define R600_ALU_INST_PREFER_VECTOR                     (1 << 3)
-#       define R600_DX10_CLAMP                                 (1 << 4)
-#       define R600_CLAUSE_SEQ_PRIO(x)                         ((x) << 8)
-#       define R600_PS_PRIO(x)                                 ((x) << 24)
-#       define R600_VS_PRIO(x)                                 ((x) << 26)
-#       define R600_GS_PRIO(x)                                 ((x) << 28)
-#       define R600_ES_PRIO(x)                                 ((x) << 30)
-#define R600_SQ_GPR_RESOURCE_MGMT_1                            0x8c04
-#       define R600_NUM_PS_GPRS(x)                             ((x) << 0)
-#       define R600_NUM_VS_GPRS(x)                             ((x) << 16)
-#       define R700_DYN_GPR_ENABLE                             (1 << 27)
-#       define R600_NUM_CLAUSE_TEMP_GPRS(x)                    ((x) << 28)
-#define R600_SQ_GPR_RESOURCE_MGMT_2                            0x8c08
-#       define R600_NUM_GS_GPRS(x)                             ((x) << 0)
-#       define R600_NUM_ES_GPRS(x)                             ((x) << 16)
-#define R600_SQ_THREAD_RESOURCE_MGMT                           0x8c0c
-#       define R600_NUM_PS_THREADS(x)                          ((x) << 0)
-#       define R600_NUM_VS_THREADS(x)                          ((x) << 8)
-#       define R600_NUM_GS_THREADS(x)                          ((x) << 16)
-#       define R600_NUM_ES_THREADS(x)                          ((x) << 24)
-#define R600_SQ_STACK_RESOURCE_MGMT_1                          0x8c10
-#       define R600_NUM_PS_STACK_ENTRIES(x)                    ((x) << 0)
-#       define R600_NUM_VS_STACK_ENTRIES(x)                    ((x) << 16)
-#define R600_SQ_STACK_RESOURCE_MGMT_2                          0x8c14
-#       define R600_NUM_GS_STACK_ENTRIES(x)                    ((x) << 0)
-#       define R600_NUM_ES_STACK_ENTRIES(x)                    ((x) << 16)
-#define R600_SQ_MS_FIFO_SIZES                                  0x8cf0
-#       define R600_CACHE_FIFO_SIZE(x)                         ((x) << 0)
-#       define R600_FETCH_FIFO_HIWATER(x)                      ((x) << 8)
-#       define R600_DONE_FIFO_HIWATER(x)                       ((x) << 16)
-#       define R600_ALU_UPDATE_FIFO_HIWATER(x)                 ((x) << 24)
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_0                         0x8db0
-#       define R700_SIMDA_RING0(x)                             ((x) << 0)
-#       define R700_SIMDA_RING1(x)                             ((x) << 8)
-#       define R700_SIMDB_RING0(x)                             ((x) << 16)
-#       define R700_SIMDB_RING1(x)                             ((x) << 24)
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_1                         0x8db4
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_2                         0x8db8
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_3                         0x8dbc
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_4                         0x8dc0
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_5                         0x8dc4
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_6                         0x8dc8
-#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_7                         0x8dcc
-
-#define R600_SPI_PS_IN_CONTROL_0                               0x286cc
-#       define R600_NUM_INTERP(x)                              ((x) << 0)
-#       define R600_POSITION_ENA                               (1 << 8)
-#       define R600_POSITION_CENTROID                          (1 << 9)
-#       define R600_POSITION_ADDR(x)                           ((x) << 10)
-#       define R600_PARAM_GEN(x)                               ((x) << 15)
-#       define R600_PARAM_GEN_ADDR(x)                          ((x) << 19)
-#       define R600_BARYC_SAMPLE_CNTL(x)                       ((x) << 26)
-#       define R600_PERSP_GRADIENT_ENA                         (1 << 28)
-#       define R600_LINEAR_GRADIENT_ENA                        (1 << 29)
-#       define R600_POSITION_SAMPLE                            (1 << 30)
-#       define R600_BARYC_AT_SAMPLE_ENA                        (1 << 31)
-#define R600_SPI_PS_IN_CONTROL_1                               0x286d0
-#       define R600_GEN_INDEX_PIX                              (1 << 0)
-#       define R600_GEN_INDEX_PIX_ADDR(x)                      ((x) << 1)
-#       define R600_FRONT_FACE_ENA                             (1 << 8)
-#       define R600_FRONT_FACE_CHAN(x)                         ((x) << 9)
-#       define R600_FRONT_FACE_ALL_BITS                        (1 << 11)
-#       define R600_FRONT_FACE_ADDR(x)                         ((x) << 12)
-#       define R600_FOG_ADDR(x)                                ((x) << 17)
-#       define R600_FIXED_PT_POSITION_ENA                      (1 << 24)
-#       define R600_FIXED_PT_POSITION_ADDR(x)                  ((x) << 25)
-#       define R700_POSITION_ULC                               (1 << 30)
-#define R600_SPI_INPUT_Z                                       0x286d8
-
-#define R600_SPI_CONFIG_CNTL                                   0x9100
-#       define R600_GPR_WRITE_PRIORITY(x)                      ((x) << 0)
-#       define R600_DISABLE_INTERP_1                           (1 << 5)
-#define R600_SPI_CONFIG_CNTL_1                                 0x913c
-#       define R600_VTX_DONE_DELAY(x)                          ((x) << 0)
-#       define R600_INTERP_ONE_PRIM_PER_ROW                    (1 << 4)
-
-#define R600_GB_TILING_CONFIG                                  0x98f0
-#       define R600_PIPE_TILING(x)                             ((x) << 1)
-#       define R600_BANK_TILING(x)                             ((x) << 4)
-#       define R600_GROUP_SIZE(x)                              ((x) << 6)
-#       define R600_ROW_TILING(x)                              ((x) << 8)
-#       define R600_BANK_SWAPS(x)                              ((x) << 11)
-#       define R600_SAMPLE_SPLIT(x)                            ((x) << 14)
-#       define R600_BACKEND_MAP(x)                             ((x) << 16)
-#define R600_DCP_TILING_CONFIG                                 0x6ca0
-#define R600_HDP_TILING_CONFIG                                 0x2f3c
-
-#define R600_CC_RB_BACKEND_DISABLE                             0x98f4
-#define R700_CC_SYS_RB_BACKEND_DISABLE                         0x3f88
-#       define R600_BACKEND_DISABLE(x)                         ((x) << 16)
-
-#define R600_CC_GC_SHADER_PIPE_CONFIG                          0x8950
-#define R600_GC_USER_SHADER_PIPE_CONFIG                        0x8954
-#       define R600_INACTIVE_QD_PIPES(x)                       ((x) << 8)
-#       define R600_INACTIVE_QD_PIPES_MASK                     (0xff << 8)
-#       define R600_INACTIVE_SIMDS(x)                          ((x) << 16)
-#       define R600_INACTIVE_SIMDS_MASK                        (0xff << 16)
-
-#define R700_CGTS_SYS_TCC_DISABLE                              0x3f90
-#define R700_CGTS_USER_SYS_TCC_DISABLE                         0x3f94
-#define R700_CGTS_TCC_DISABLE                                  0x9148
-#define R700_CGTS_USER_TCC_DISABLE                             0x914c
-
-/* Constants */
-#define RADEON_MAX_USEC_TIMEOUT		100000	/* 100 ms */
-
-#define RADEON_LAST_FRAME_REG		RADEON_SCRATCH_REG0
-#define RADEON_LAST_DISPATCH_REG	RADEON_SCRATCH_REG1
-#define RADEON_LAST_CLEAR_REG		RADEON_SCRATCH_REG2
-#define RADEON_LAST_SWI_REG		RADEON_SCRATCH_REG3
-#define RADEON_LAST_DISPATCH		1
-
-#define R600_LAST_FRAME_REG		R600_SCRATCH_REG0
-#define R600_LAST_DISPATCH_REG	        R600_SCRATCH_REG1
-#define R600_LAST_CLEAR_REG		R600_SCRATCH_REG2
-#define R600_LAST_SWI_REG		R600_SCRATCH_REG3
-
-#define RADEON_MAX_VB_AGE		0x7fffffff
-#define RADEON_MAX_VB_VERTS		(0xffff)
-
-#define RADEON_RING_HIGH_MARK		128
-
-#define RADEON_PCIGART_TABLE_SIZE      (32*1024)
-
-#define RADEON_READ(reg)	DRM_READ32(  dev_priv->mmio, (reg) )
-#define RADEON_WRITE(reg, val)                                          \
-do {									\
-	if (reg < 0x10000) {				                \
-		DRM_WRITE32(dev_priv->mmio, (reg), (val));		\
-	} else {                                                        \
-		DRM_WRITE32(dev_priv->mmio, RADEON_MM_INDEX, (reg));	\
-		DRM_WRITE32(dev_priv->mmio, RADEON_MM_DATA, (val));	\
-	}                                                               \
-} while (0)
-#define RADEON_READ8(reg)	DRM_READ8(  dev_priv->mmio, (reg) )
-#define RADEON_WRITE8(reg,val)	DRM_WRITE8( dev_priv->mmio, (reg), (val) )
-
-#define RADEON_WRITE_PLL(addr, val)					\
-do {									\
-	RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX,				\
-		       ((addr) & 0x1f) | RADEON_PLL_WR_EN );		\
-	RADEON_WRITE(RADEON_CLOCK_CNTL_DATA, (val));			\
-} while (0)
-
-#define RADEON_WRITE_PCIE(addr, val)					\
-do {									\
-	RADEON_WRITE8(RADEON_PCIE_INDEX,				\
-			((addr) & 0xff));				\
-	RADEON_WRITE(RADEON_PCIE_DATA, (val));			\
-} while (0)
-
-#define R500_WRITE_MCIND(addr, val)					\
-do {								\
-	RADEON_WRITE(R520_MC_IND_INDEX, 0xff0000 | ((addr) & 0xff));	\
-	RADEON_WRITE(R520_MC_IND_DATA, (val));			\
-	RADEON_WRITE(R520_MC_IND_INDEX, 0);	\
-} while (0)
-
-#define RS480_WRITE_MCIND(addr, val)				\
-do {									\
-	RADEON_WRITE(RS480_NB_MC_INDEX,				\
-			((addr) & 0xff) | RS480_NB_MC_IND_WR_EN);	\
-	RADEON_WRITE(RS480_NB_MC_DATA, (val));			\
-	RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);			\
-} while (0)
-
-#define RS690_WRITE_MCIND(addr, val)					\
-do {								\
-	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_EN | ((addr) & RS690_MC_INDEX_MASK));	\
-	RADEON_WRITE(RS690_MC_DATA, val);			\
-	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_ACK);	\
-} while (0)
-
-#define RS600_WRITE_MCIND(addr, val)				\
-do {							        \
-	RADEON_WRITE(RS600_MC_INDEX, RS600_MC_IND_WR_EN | RS600_MC_IND_CITF_ARB0 | ((addr) & RS600_MC_ADDR_MASK)); \
-	RADEON_WRITE(RS600_MC_DATA, val);                       \
-} while (0)
-
-#define IGP_WRITE_MCIND(addr, val)				\
-do {									\
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||   \
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))      \
-		RS690_WRITE_MCIND(addr, val);				\
-	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)  \
-		RS600_WRITE_MCIND(addr, val);				\
-	else								\
-		RS480_WRITE_MCIND(addr, val);				\
-} while (0)
-
-#define CP_PACKET0( reg, n )						\
-	(RADEON_CP_PACKET0 | ((n) << 16) | ((reg) >> 2))
-#define CP_PACKET0_TABLE( reg, n )					\
-	(RADEON_CP_PACKET0 | RADEON_ONE_REG_WR | ((n) << 16) | ((reg) >> 2))
-#define CP_PACKET1( reg0, reg1 )					\
-	(RADEON_CP_PACKET1 | (((reg1) >> 2) << 15) | ((reg0) >> 2))
-#define CP_PACKET2()							\
-	(RADEON_CP_PACKET2)
-#define CP_PACKET3( pkt, n )						\
-	(RADEON_CP_PACKET3 | (pkt) | ((n) << 16))
-
-/* ================================================================
- * Engine control helper macros
- */
-
-#define RADEON_WAIT_UNTIL_2D_IDLE() do {				\
-	OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) );			\
-	OUT_RING( (RADEON_WAIT_2D_IDLECLEAN |				\
-		   RADEON_WAIT_HOST_IDLECLEAN) );			\
-} while (0)
-
-#define RADEON_WAIT_UNTIL_3D_IDLE() do {				\
-	OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) );			\
-	OUT_RING( (RADEON_WAIT_3D_IDLECLEAN |				\
-		   RADEON_WAIT_HOST_IDLECLEAN) );			\
-} while (0)
-
-#define RADEON_WAIT_UNTIL_IDLE() do {					\
-	OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) );			\
-	OUT_RING( (RADEON_WAIT_2D_IDLECLEAN |				\
-		   RADEON_WAIT_3D_IDLECLEAN |				\
-		   RADEON_WAIT_HOST_IDLECLEAN) );			\
-} while (0)
-
-#define RADEON_WAIT_UNTIL_PAGE_FLIPPED() do {				\
-	OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) );			\
-	OUT_RING( RADEON_WAIT_CRTC_PFLIP );				\
-} while (0)
-
-#define RADEON_FLUSH_CACHE() do {					\
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
-		OUT_RING(CP_PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0));	\
-		OUT_RING(RADEON_RB3D_DC_FLUSH);				\
-	} else {                                                        \
-		OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));	\
-		OUT_RING(R300_RB3D_DC_FLUSH);				\
-	}                                                               \
-} while (0)
-
-#define RADEON_PURGE_CACHE() do {					\
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
-		OUT_RING(CP_PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0));	\
-		OUT_RING(RADEON_RB3D_DC_FLUSH | RADEON_RB3D_DC_FREE);	\
-	} else {                                                        \
-		OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));	\
-		OUT_RING(R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE);	\
-	}                                                               \
-} while (0)
-
-#define RADEON_FLUSH_ZCACHE() do {					\
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
-		OUT_RING(CP_PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0));	\
-		OUT_RING(RADEON_RB3D_ZC_FLUSH);				\
-	} else {                                                        \
-		OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));	\
-		OUT_RING(R300_ZC_FLUSH);				\
-	}                                                               \
-} while (0)
-
-#define RADEON_PURGE_ZCACHE() do {					\
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
-		OUT_RING(CP_PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0));	\
-		OUT_RING(RADEON_RB3D_ZC_FLUSH | RADEON_RB3D_ZC_FREE);			\
-	} else {                                                        \
-		OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));	\
-		OUT_RING(R300_ZC_FLUSH | R300_ZC_FREE);				\
-	}                                                               \
-} while (0)
-
-/* ================================================================
- * Misc helper macros
- */
-
-/* Perfbox functionality only.
- */
-#define RING_SPACE_TEST_WITH_RETURN( dev_priv )				\
-do {									\
-	if (!(dev_priv->stats.boxes & RADEON_BOX_DMA_IDLE)) {		\
-		u32 head = GET_RING_HEAD( dev_priv );			\
-		if (head == dev_priv->ring.tail)			\
-			dev_priv->stats.boxes |= RADEON_BOX_DMA_IDLE;	\
-	}								\
-} while (0)
-
-#define VB_AGE_TEST_WITH_RETURN( dev_priv )				\
-do {								\
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;	\
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;	\
-	if ( sarea_priv->last_dispatch >= RADEON_MAX_VB_AGE ) {		\
-		int __ret;						\
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) \
-			__ret = r600_do_cp_idle(dev_priv);		\
-		else							\
-			__ret = radeon_do_cp_idle(dev_priv);		\
-		if ( __ret ) return __ret;				\
-		sarea_priv->last_dispatch = 0;				\
-		radeon_freelist_reset( dev );				\
-	}								\
-} while (0)
-
-#define RADEON_DISPATCH_AGE( age ) do {					\
-	OUT_RING( CP_PACKET0( RADEON_LAST_DISPATCH_REG, 0 ) );		\
-	OUT_RING( age );						\
-} while (0)
-
-#define RADEON_FRAME_AGE( age ) do {					\
-	OUT_RING( CP_PACKET0( RADEON_LAST_FRAME_REG, 0 ) );		\
-	OUT_RING( age );						\
-} while (0)
-
-#define RADEON_CLEAR_AGE( age ) do {					\
-	OUT_RING( CP_PACKET0( RADEON_LAST_CLEAR_REG, 0 ) );		\
-	OUT_RING( age );						\
-} while (0)
-
-#define R600_DISPATCH_AGE(age) do {					\
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));		\
-	OUT_RING((R600_LAST_DISPATCH_REG - R600_SET_CONFIG_REG_OFFSET) >> 2);  \
-	OUT_RING(age);							\
-} while (0)
-
-#define R600_FRAME_AGE(age) do {					\
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));		\
-	OUT_RING((R600_LAST_FRAME_REG - R600_SET_CONFIG_REG_OFFSET) >> 2);  \
-	OUT_RING(age);							\
-} while (0)
-
-#define R600_CLEAR_AGE(age) do {					\
-	OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));		\
-	OUT_RING((R600_LAST_CLEAR_REG - R600_SET_CONFIG_REG_OFFSET) >> 2);  \
-	OUT_RING(age);							\
-} while (0)
-
-/* ================================================================
- * Ring control
- */
-
-#define RADEON_VERBOSE	0
-
-#define RING_LOCALS	int write, _nr, _align_nr; unsigned int mask; u32 *ring;
-
-#define RADEON_RING_ALIGN 16
-
-#define BEGIN_RING( n ) do {						\
-	if ( RADEON_VERBOSE ) {						\
-		DRM_INFO( "BEGIN_RING( %d )\n", (n));			\
-	}								\
-	_align_nr = RADEON_RING_ALIGN - ((dev_priv->ring.tail + n) & (RADEON_RING_ALIGN-1));	\
-	_align_nr += n;							\
-	if (dev_priv->ring.space <= (_align_nr * sizeof(u32))) {	\
-                COMMIT_RING();						\
-		radeon_wait_ring( dev_priv, _align_nr * sizeof(u32));	\
-	}								\
-	_nr = n; dev_priv->ring.space -= (n) * sizeof(u32);		\
-	ring = dev_priv->ring.start;					\
-	write = dev_priv->ring.tail;					\
-	mask = dev_priv->ring.tail_mask;				\
-} while (0)
-
-#define ADVANCE_RING() do {						\
-	if ( RADEON_VERBOSE ) {						\
-		DRM_INFO( "ADVANCE_RING() wr=0x%06x tail=0x%06x\n",	\
-			  write, dev_priv->ring.tail );			\
-	}								\
-	if (((dev_priv->ring.tail + _nr) & mask) != write) {		\
-		DRM_ERROR(						\
-			"ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n",	\
-			((dev_priv->ring.tail + _nr) & mask),		\
-			write, __LINE__);				\
-	} else								\
-		dev_priv->ring.tail = write;				\
-} while (0)
-
-extern void radeon_commit_ring(drm_radeon_private_t *dev_priv);
-
-#define COMMIT_RING() do {						\
-		radeon_commit_ring(dev_priv);				\
-	} while(0)
-
-#define OUT_RING( x ) do {						\
-	if ( RADEON_VERBOSE ) {						\
-		DRM_INFO( "   OUT_RING( 0x%08x ) at 0x%x\n",		\
-			   (unsigned int)(x), write );			\
-	}								\
-	ring[write++] = (x);						\
-	write &= mask;							\
-} while (0)
-
-#define OUT_RING_REG( reg, val ) do {					\
-	OUT_RING( CP_PACKET0( reg, 0 ) );				\
-	OUT_RING( val );						\
-} while (0)
-
-#define OUT_RING_TABLE( tab, sz ) do {					\
-	int _size = (sz);					\
-	int *_tab = (int *)(tab);				\
-								\
-	if (write + _size > mask) {				\
-		int _i = (mask+1) - write;			\
-		_size -= _i;					\
-		while (_i > 0 ) {				\
-			*(int *)(ring + write) = *_tab++;	\
-			write++;				\
-			_i--;					\
-		}						\
-		write = 0;					\
-		_tab += _i;					\
-	}							\
-	while (_size > 0) {					\
-		*(ring + write) = *_tab++;			\
-		write++;					\
-		_size--;					\
-	}							\
-	write &= mask;						\
-} while (0)
-
-/**
- * Copy given number of dwords from drm buffer to the ring buffer.
- */
-#define OUT_RING_DRM_BUFFER(buf, sz) do {				\
-	int _size = (sz) * 4;						\
-	struct drm_buffer *_buf = (buf);				\
-	int _part_size;							\
-	while (_size > 0) {						\
-		_part_size = _size;					\
-									\
-		if (write + _part_size/4 > mask)			\
-			_part_size = ((mask + 1) - write)*4;		\
-									\
-		if (drm_buffer_index(_buf) + _part_size > PAGE_SIZE)	\
-			_part_size = PAGE_SIZE - drm_buffer_index(_buf);\
-									\
-									\
-									\
-		memcpy(ring + write, &_buf->data[drm_buffer_page(_buf)]	\
-			[drm_buffer_index(_buf)], _part_size);		\
-									\
-		_size -= _part_size;					\
-		write = (write + _part_size/4) & mask;			\
-		drm_buffer_advance(_buf, _part_size);			\
-	}								\
-} while (0)
-
-
-#endif				/* CONFIG_DRM_RADEON_UMS */
-
 #endif				/* __RADEON_DRV_H__ */
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 26da2f4d7b4f..d2e628eea53d 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -44,7 +44,6 @@
 struct radeon_fbdev {
 	struct drm_fb_helper helper;
 	struct radeon_framebuffer rfb;
-	struct list_head fbdev_list;
 	struct radeon_device *rdev;
 };
 
@@ -283,7 +282,7 @@ out_unref:
 
 	}
 	if (fb && ret) {
-		drm_gem_object_unreference(gobj);
+		drm_gem_object_unreference_unlocked(gobj);
 		drm_framebuffer_unregister_private(fb);
 		drm_framebuffer_cleanup(fb);
 		kfree(fb);
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index df09ca7c4889..05815c47b246 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -130,7 +130,7 @@ int radeon_fence_emit(struct radeon_device *rdev,
 		      struct radeon_fence **fence,
 		      int ring)
 {
-	u64 seq = ++rdev->fence_drv[ring].sync_seq[ring];
+	u64 seq;
 
 	/* we are protected by the ring emission mutex */
 	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
@@ -138,7 +138,7 @@ int radeon_fence_emit(struct radeon_device *rdev,
 		return -ENOMEM;
 	}
 	(*fence)->rdev = rdev;
-	(*fence)->seq = seq;
+	(*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];
 	(*fence)->ring = ring;
 	(*fence)->is_vm_update = false;
 	fence_init(&(*fence)->base, &radeon_fence_ops,
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 3dcc5733ff69..e26c963f2e93 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -663,6 +663,7 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
 	bo_va = radeon_vm_bo_find(&fpriv->vm, rbo);
 	if (!bo_va) {
 		args->operation = RADEON_VA_RESULT_ERROR;
+		radeon_bo_unreserve(rbo);
 		drm_gem_object_unreference_unlocked(gobj);
 		return -ENOENT;
 	}
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
deleted file mode 100644
index 688afb62f7c4..000000000000
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ /dev/null
@@ -1,402 +0,0 @@
-/* radeon_irq.c -- IRQ handling for radeon -*- linux-c -*- */
-/*
- * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
- *
- * The Weather Channel (TM) funded Tungsten Graphics to develop the
- * initial release of the Radeon 8500 driver under the XFree86 license.
- * This notice must be preserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Keith Whitwell <keith@tungstengraphics.com>
- *    Michel D�zer <michel@daenzer.net>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-
-void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if (state)
-		dev_priv->irq_enable_reg |= mask;
-	else
-		dev_priv->irq_enable_reg &= ~mask;
-
-	if (dev->irq_enabled)
-		RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
-}
-
-static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if (state)
-		dev_priv->r500_disp_irq_reg |= mask;
-	else
-		dev_priv->r500_disp_irq_reg &= ~mask;
-
-	if (dev->irq_enabled)
-		RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
-}
-
-int radeon_enable_vblank(struct drm_device *dev, unsigned int pipe)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
-		switch (pipe) {
-		case 0:
-			r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 1);
-			break;
-		case 1:
-			r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 1);
-			break;
-		default:
-			DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
-				  pipe);
-			return -EINVAL;
-		}
-	} else {
-		switch (pipe) {
-		case 0:
-			radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 1);
-			break;
-		case 1:
-			radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 1);
-			break;
-		default:
-			DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
-				  pipe);
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
-void radeon_disable_vblank(struct drm_device *dev, unsigned int pipe)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
-		switch (pipe) {
-		case 0:
-			r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 0);
-			break;
-		case 1:
-			r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 0);
-			break;
-		default:
-			DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
-				  pipe);
-			break;
-		}
-	} else {
-		switch (pipe) {
-		case 0:
-			radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 0);
-			break;
-		case 1:
-			radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 0);
-			break;
-		default:
-			DRM_ERROR("tried to enable vblank on non-existent crtc %u\n",
-				  pipe);
-			break;
-		}
-	}
-}
-
-static u32 radeon_acknowledge_irqs(drm_radeon_private_t *dev_priv, u32 *r500_disp_int)
-{
-	u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS);
-	u32 irq_mask = RADEON_SW_INT_TEST;
-
-	*r500_disp_int = 0;
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
-		/* vbl interrupts in a different place */
-
-		if (irqs & R500_DISPLAY_INT_STATUS) {
-			/* if a display interrupt */
-			u32 disp_irq;
-
-			disp_irq = RADEON_READ(R500_DISP_INTERRUPT_STATUS);
-
-			*r500_disp_int = disp_irq;
-			if (disp_irq & R500_D1_VBLANK_INTERRUPT)
-				RADEON_WRITE(R500_D1MODE_VBLANK_STATUS, R500_VBLANK_ACK);
-			if (disp_irq & R500_D2_VBLANK_INTERRUPT)
-				RADEON_WRITE(R500_D2MODE_VBLANK_STATUS, R500_VBLANK_ACK);
-		}
-		irq_mask |= R500_DISPLAY_INT_STATUS;
-	} else
-		irq_mask |= RADEON_CRTC_VBLANK_STAT | RADEON_CRTC2_VBLANK_STAT;
-
-	irqs &=	irq_mask;
-
-	if (irqs)
-		RADEON_WRITE(RADEON_GEN_INT_STATUS, irqs);
-
-	return irqs;
-}
-
-/* Interrupts - Used for device synchronization and flushing in the
- * following circumstances:
- *
- * - Exclusive FB access with hw idle:
- *    - Wait for GUI Idle (?) interrupt, then do normal flush.
- *
- * - Frame throttling, NV_fence:
- *    - Drop marker irq's into command stream ahead of time.
- *    - Wait on irq's with lock *not held*
- *    - Check each for termination condition
- *
- * - Internally in cp_getbuffer, etc:
- *    - as above, but wait with lock held???
- *
- * NOTE: These functions are misleadingly named -- the irq's aren't
- * tied to dma at all, this is just a hangover from dri prehistory.
- */
-
-irqreturn_t radeon_driver_irq_handler(int irq, void *arg)
-{
-	struct drm_device *dev = (struct drm_device *) arg;
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-	u32 stat;
-	u32 r500_disp_int;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return IRQ_NONE;
-
-	/* Only consider the bits we're interested in - others could be used
-	 * outside the DRM
-	 */
-	stat = radeon_acknowledge_irqs(dev_priv, &r500_disp_int);
-	if (!stat)
-		return IRQ_NONE;
-
-	stat &= dev_priv->irq_enable_reg;
-
-	/* SW interrupt */
-	if (stat & RADEON_SW_INT_TEST)
-		wake_up(&dev_priv->swi_queue);
-
-	/* VBLANK interrupt */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
-		if (r500_disp_int & R500_D1_VBLANK_INTERRUPT)
-			drm_handle_vblank(dev, 0);
-		if (r500_disp_int & R500_D2_VBLANK_INTERRUPT)
-			drm_handle_vblank(dev, 1);
-	} else {
-		if (stat & RADEON_CRTC_VBLANK_STAT)
-			drm_handle_vblank(dev, 0);
-		if (stat & RADEON_CRTC2_VBLANK_STAT)
-			drm_handle_vblank(dev, 1);
-	}
-	return IRQ_HANDLED;
-}
-
-static int radeon_emit_irq(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	unsigned int ret;
-	RING_LOCALS;
-
-	atomic_inc(&dev_priv->swi_emitted);
-	ret = atomic_read(&dev_priv->swi_emitted);
-
-	BEGIN_RING(4);
-	OUT_RING_REG(RADEON_LAST_SWI_REG, ret);
-	OUT_RING_REG(RADEON_GEN_INT_STATUS, RADEON_SW_INT_FIRE);
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	return ret;
-}
-
-static int radeon_wait_irq(struct drm_device * dev, int swi_nr)
-{
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-	int ret = 0;
-
-	if (RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr)
-		return 0;
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	DRM_WAIT_ON(ret, dev_priv->swi_queue, 3 * HZ,
-		    RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr);
-
-	return ret;
-}
-
-u32 radeon_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	if (pipe > 1) {
-		DRM_ERROR("Invalid crtc %u\n", pipe);
-		return -EINVAL;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) {
-		if (pipe == 0)
-			return RADEON_READ(R500_D1CRTC_FRAME_COUNT);
-		else
-			return RADEON_READ(R500_D2CRTC_FRAME_COUNT);
-	} else {
-		if (pipe == 0)
-			return RADEON_READ(RADEON_CRTC_CRNT_FRAME);
-		else
-			return RADEON_READ(RADEON_CRTC2_CRNT_FRAME);
-	}
-}
-
-/* Needs the lock as it touches the ring.
- */
-int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_irq_emit_t *emit = data;
-	int result;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return -EINVAL;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	result = radeon_emit_irq(dev);
-
-	if (copy_to_user(emit->irq_seq, &result, sizeof(int))) {
-		DRM_ERROR("copy_to_user\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-/* Doesn't need the hardware lock.
- */
-int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_irq_wait_t *irqwait = data;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return -EINVAL;
-
-	return radeon_wait_irq(dev, irqwait->irq_seq);
-}
-
-/* drm_dma.h hooks
-*/
-void radeon_driver_irq_preinstall(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-	u32 dummy;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return;
-
-	/* Disable *all* interrupts */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600)
-		RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
-	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
-
-	/* Clear bits if they're already high */
-	radeon_acknowledge_irqs(dev_priv, &dummy);
-}
-
-int radeon_driver_irq_postinstall(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-
-	atomic_set(&dev_priv->swi_emitted, 0);
-	init_waitqueue_head(&dev_priv->swi_queue);
-
-	dev->max_vblank_count = 0x001fffff;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return 0;
-
-	radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
-
-	return 0;
-}
-
-void radeon_driver_irq_uninstall(struct drm_device * dev)
-{
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-	if (!dev_priv)
-		return;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		return;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600)
-		RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
-	/* Disable *all* interrupts */
-	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
-}
-
-
-int radeon_vblank_crtc_get(struct drm_device *dev)
-{
-	drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
-
-	return dev_priv->vblank_crtc;
-}
-
-int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value)
-{
-	drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
-	if (value & ~(DRM_RADEON_VBLANK_CRTC1 | DRM_RADEON_VBLANK_CRTC2)) {
-		DRM_ERROR("called with invalid crtc 0x%x\n", (unsigned int)value);
-		return -EINVAL;
-	}
-	dev_priv->vblank_crtc = (unsigned int)value;
-	return 0;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index d290a8a09036..414953c46a38 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -748,19 +748,19 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
  * radeon_get_vblank_counter_kms - get frame count
  *
  * @dev: drm dev pointer
- * @crtc: crtc to get the frame count from
+ * @pipe: crtc to get the frame count from
  *
  * Gets the frame count on the requested crtc (all asics).
  * Returns frame count on success, -EINVAL on failure.
  */
-u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
+u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe)
 {
 	int vpos, hpos, stat;
 	u32 count;
 	struct radeon_device *rdev = dev->dev_private;
 
-	if (crtc < 0 || crtc >= rdev->num_crtc) {
-		DRM_ERROR("Invalid crtc %d\n", crtc);
+	if (pipe >= rdev->num_crtc) {
+		DRM_ERROR("Invalid crtc %u\n", pipe);
 		return -EINVAL;
 	}
 
@@ -772,29 +772,29 @@ u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
 	 * and start of vsync, so vpos >= 0 means to bump the hw frame counter
 	 * result by 1 to give the proper appearance to caller.
 	 */
-	if (rdev->mode_info.crtcs[crtc]) {
+	if (rdev->mode_info.crtcs[pipe]) {
 		/* Repeat readout if needed to provide stable result if
 		 * we cross start of vsync during the queries.
 		 */
 		do {
-			count = radeon_get_vblank_counter(rdev, crtc);
+			count = radeon_get_vblank_counter(rdev, pipe);
 			/* Ask radeon_get_crtc_scanoutpos to return vpos as
 			 * distance to start of vblank, instead of regular
 			 * vertical scanout pos.
 			 */
 			stat = radeon_get_crtc_scanoutpos(
-				dev, crtc, GET_DISTANCE_TO_VBLANKSTART,
+				dev, pipe, GET_DISTANCE_TO_VBLANKSTART,
 				&vpos, &hpos, NULL, NULL,
-				&rdev->mode_info.crtcs[crtc]->base.hwmode);
-		} while (count != radeon_get_vblank_counter(rdev, crtc));
+				&rdev->mode_info.crtcs[pipe]->base.hwmode);
+		} while (count != radeon_get_vblank_counter(rdev, pipe));
 
 		if (((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) !=
 		    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE))) {
 			DRM_DEBUG_VBL("Query failed! stat %d\n", stat);
 		}
 		else {
-			DRM_DEBUG_VBL("crtc %d: dist from vblank start %d\n",
-				      crtc, vpos);
+			DRM_DEBUG_VBL("crtc %u: dist from vblank start %d\n",
+				      pipe, vpos);
 
 			/* Bump counter if we are at >= leading edge of vblank,
 			 * but before vsync where vpos would turn negative and
@@ -806,7 +806,7 @@ u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
 	}
 	else {
 	    /* Fallback to use value as is. */
-	    count = radeon_get_vblank_counter(rdev, crtc);
+	    count = radeon_get_vblank_counter(rdev, pipe);
 	    DRM_DEBUG_VBL("NULL mode info! Returned count may be wrong.\n");
 	}
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 678b4386540d..32b338ff436b 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -25,6 +25,7 @@
  */
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/radeon_drm.h>
 #include <drm/drm_fixed.h>
 #include "radeon.h"
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 30de43366eae..88dc973fb209 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -1772,7 +1772,8 @@ radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_
 	switch (radeon_encoder->encoder_id) {
 	case ENCODER_OBJECT_ID_INTERNAL_LVDS:
 		encoder->possible_crtcs = 0x1;
-		drm_encoder_init(dev, encoder, &radeon_legacy_lvds_enc_funcs, DRM_MODE_ENCODER_LVDS);
+		drm_encoder_init(dev, encoder, &radeon_legacy_lvds_enc_funcs,
+				 DRM_MODE_ENCODER_LVDS, NULL);
 		drm_encoder_helper_add(encoder, &radeon_legacy_lvds_helper_funcs);
 		if (rdev->is_atom_bios)
 			radeon_encoder->enc_priv = radeon_atombios_get_lvds_info(radeon_encoder);
@@ -1781,12 +1782,14 @@ radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_
 		radeon_encoder->rmx_type = RMX_FULL;
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_TMDS1:
-		drm_encoder_init(dev, encoder, &radeon_legacy_tmds_int_enc_funcs, DRM_MODE_ENCODER_TMDS);
+		drm_encoder_init(dev, encoder, &radeon_legacy_tmds_int_enc_funcs,
+				 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &radeon_legacy_tmds_int_helper_funcs);
 		radeon_encoder->enc_priv = radeon_legacy_get_tmds_info(radeon_encoder);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_DAC1:
-		drm_encoder_init(dev, encoder, &radeon_legacy_primary_dac_enc_funcs, DRM_MODE_ENCODER_DAC);
+		drm_encoder_init(dev, encoder, &radeon_legacy_primary_dac_enc_funcs,
+				 DRM_MODE_ENCODER_DAC, NULL);
 		drm_encoder_helper_add(encoder, &radeon_legacy_primary_dac_helper_funcs);
 		if (rdev->is_atom_bios)
 			radeon_encoder->enc_priv = radeon_atombios_get_primary_dac_info(radeon_encoder);
@@ -1794,7 +1797,8 @@ radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_
 			radeon_encoder->enc_priv = radeon_combios_get_primary_dac_info(radeon_encoder);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_DAC2:
-		drm_encoder_init(dev, encoder, &radeon_legacy_tv_dac_enc_funcs, DRM_MODE_ENCODER_TVDAC);
+		drm_encoder_init(dev, encoder, &radeon_legacy_tv_dac_enc_funcs,
+				 DRM_MODE_ENCODER_TVDAC, NULL);
 		drm_encoder_helper_add(encoder, &radeon_legacy_tv_dac_helper_funcs);
 		if (rdev->is_atom_bios)
 			radeon_encoder->enc_priv = radeon_atombios_get_tv_dac_info(radeon_encoder);
@@ -1802,7 +1806,8 @@ radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_
 			radeon_encoder->enc_priv = radeon_combios_get_tv_dac_info(radeon_encoder);
 		break;
 	case ENCODER_OBJECT_ID_INTERNAL_DVO1:
-		drm_encoder_init(dev, encoder, &radeon_legacy_tmds_ext_enc_funcs, DRM_MODE_ENCODER_TMDS);
+		drm_encoder_init(dev, encoder, &radeon_legacy_tmds_ext_enc_funcs,
+				 DRM_MODE_ENCODER_TMDS, NULL);
 		drm_encoder_helper_add(encoder, &radeon_legacy_tmds_ext_helper_funcs);
 		if (!rdev->is_atom_bios)
 			radeon_encoder->enc_priv = radeon_legacy_get_ext_tmds_info(radeon_encoder);
diff --git a/drivers/gpu/drm/radeon/radeon_mem.c b/drivers/gpu/drm/radeon/radeon_mem.c
deleted file mode 100644
index 146d253f1131..000000000000
--- a/drivers/gpu/drm/radeon/radeon_mem.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/* radeon_mem.c -- Simple GART/fb memory manager for radeon -*- linux-c -*- */
-/*
- * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
- *
- * The Weather Channel (TM) funded Tungsten Graphics to develop the
- * initial release of the Radeon 8500 driver under the XFree86 license.
- * This notice must be preserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Keith Whitwell <keith@tungstengraphics.com>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-
-/* Very simple allocator for GART memory, working on a static range
- * already mapped into each client's address space.
- */
-
-static struct mem_block *split_block(struct mem_block *p, int start, int size,
-				     struct drm_file *file_priv)
-{
-	/* Maybe cut off the start of an existing block */
-	if (start > p->start) {
-		struct mem_block *newblock = kmalloc(sizeof(*newblock),
-						     GFP_KERNEL);
-		if (!newblock)
-			goto out;
-		newblock->start = start;
-		newblock->size = p->size - (start - p->start);
-		newblock->file_priv = NULL;
-		newblock->next = p->next;
-		newblock->prev = p;
-		p->next->prev = newblock;
-		p->next = newblock;
-		p->size -= newblock->size;
-		p = newblock;
-	}
-
-	/* Maybe cut off the end of an existing block */
-	if (size < p->size) {
-		struct mem_block *newblock = kmalloc(sizeof(*newblock),
-						     GFP_KERNEL);
-		if (!newblock)
-			goto out;
-		newblock->start = start + size;
-		newblock->size = p->size - size;
-		newblock->file_priv = NULL;
-		newblock->next = p->next;
-		newblock->prev = p;
-		p->next->prev = newblock;
-		p->next = newblock;
-		p->size = size;
-	}
-
-      out:
-	/* Our block is in the middle */
-	p->file_priv = file_priv;
-	return p;
-}
-
-static struct mem_block *alloc_block(struct mem_block *heap, int size,
-				     int align2, struct drm_file *file_priv)
-{
-	struct mem_block *p;
-	int mask = (1 << align2) - 1;
-
-	list_for_each(p, heap) {
-		int start = (p->start + mask) & ~mask;
-		if (p->file_priv == NULL && start + size <= p->start + p->size)
-			return split_block(p, start, size, file_priv);
-	}
-
-	return NULL;
-}
-
-static struct mem_block *find_block(struct mem_block *heap, int start)
-{
-	struct mem_block *p;
-
-	list_for_each(p, heap)
-	    if (p->start == start)
-		return p;
-
-	return NULL;
-}
-
-static void free_block(struct mem_block *p)
-{
-	p->file_priv = NULL;
-
-	/* Assumes a single contiguous range.  Needs a special file_priv in
-	 * 'heap' to stop it being subsumed.
-	 */
-	if (p->next->file_priv == NULL) {
-		struct mem_block *q = p->next;
-		p->size += q->size;
-		p->next = q->next;
-		p->next->prev = p;
-		kfree(q);
-	}
-
-	if (p->prev->file_priv == NULL) {
-		struct mem_block *q = p->prev;
-		q->size += p->size;
-		q->next = p->next;
-		q->next->prev = q;
-		kfree(p);
-	}
-}
-
-/* Initialize.  How to check for an uninitialized heap?
- */
-static int init_heap(struct mem_block **heap, int start, int size)
-{
-	struct mem_block *blocks = kmalloc(sizeof(*blocks), GFP_KERNEL);
-
-	if (!blocks)
-		return -ENOMEM;
-
-	*heap = kzalloc(sizeof(**heap), GFP_KERNEL);
-	if (!*heap) {
-		kfree(blocks);
-		return -ENOMEM;
-	}
-
-	blocks->start = start;
-	blocks->size = size;
-	blocks->file_priv = NULL;
-	blocks->next = blocks->prev = *heap;
-
-	(*heap)->file_priv = (struct drm_file *) - 1;
-	(*heap)->next = (*heap)->prev = blocks;
-	return 0;
-}
-
-/* Free all blocks associated with the releasing file.
- */
-void radeon_mem_release(struct drm_file *file_priv, struct mem_block *heap)
-{
-	struct mem_block *p;
-
-	if (!heap || !heap->next)
-		return;
-
-	list_for_each(p, heap) {
-		if (p->file_priv == file_priv)
-			p->file_priv = NULL;
-	}
-
-	/* Assumes a single contiguous range.  Needs a special file_priv in
-	 * 'heap' to stop it being subsumed.
-	 */
-	list_for_each(p, heap) {
-		while (p->file_priv == NULL && p->next->file_priv == NULL) {
-			struct mem_block *q = p->next;
-			p->size += q->size;
-			p->next = q->next;
-			p->next->prev = p;
-			kfree(q);
-		}
-	}
-}
-
-/* Shutdown.
- */
-void radeon_mem_takedown(struct mem_block **heap)
-{
-	struct mem_block *p;
-
-	if (!*heap)
-		return;
-
-	for (p = (*heap)->next; p != *heap;) {
-		struct mem_block *q = p;
-		p = p->next;
-		kfree(q);
-	}
-
-	kfree(*heap);
-	*heap = NULL;
-}
-
-/* IOCTL HANDLERS */
-
-static struct mem_block **get_heap(drm_radeon_private_t * dev_priv, int region)
-{
-	switch (region) {
-	case RADEON_MEM_REGION_GART:
-		return &dev_priv->gart_heap;
-	case RADEON_MEM_REGION_FB:
-		return &dev_priv->fb_heap;
-	default:
-		return NULL;
-	}
-}
-
-int radeon_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_alloc_t *alloc = data;
-	struct mem_block *block, **heap;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	heap = get_heap(dev_priv, alloc->region);
-	if (!heap || !*heap)
-		return -EFAULT;
-
-	/* Make things easier on ourselves: all allocations at least
-	 * 4k aligned.
-	 */
-	if (alloc->alignment < 12)
-		alloc->alignment = 12;
-
-	block = alloc_block(*heap, alloc->size, alloc->alignment, file_priv);
-
-	if (!block)
-		return -ENOMEM;
-
-	if (copy_to_user(alloc->region_offset, &block->start,
-			     sizeof(int))) {
-		DRM_ERROR("copy_to_user\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-int radeon_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_free_t *memfree = data;
-	struct mem_block *block, **heap;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	heap = get_heap(dev_priv, memfree->region);
-	if (!heap || !*heap)
-		return -EFAULT;
-
-	block = find_block(*heap, memfree->region_offset);
-	if (!block)
-		return -EFAULT;
-
-	if (block->file_priv != file_priv)
-		return -EPERM;
-
-	free_block(block);
-	return 0;
-}
-
-int radeon_mem_init_heap(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_init_heap_t *initheap = data;
-	struct mem_block **heap;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	heap = get_heap(dev_priv, initheap->region);
-	if (!heap)
-		return -EFAULT;
-
-	if (*heap) {
-		DRM_ERROR("heap already initialized?");
-		return -EFAULT;
-	}
-
-	return init_heap(heap, initheap->start, initheap->size);
-}
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index bba112628b47..bb75201a24ba 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -757,8 +757,10 @@ extern u8 radeon_dp_getsinktype(struct radeon_connector *radeon_connector);
 extern bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector);
 extern int radeon_dp_get_panel_mode(struct drm_encoder *encoder,
 				    struct drm_connector *connector);
-int radeon_dp_get_max_link_rate(struct drm_connector *connector,
-				const u8 *dpcd);
+extern int radeon_dp_get_dp_link_config(struct drm_connector *connector,
+					const u8 *dpcd,
+					unsigned pix_clock,
+					unsigned *dp_lanes, unsigned *dp_rate);
 extern void radeon_dp_set_rx_power_state(struct drm_connector *connector,
 					 u8 power_state);
 extern void radeon_dp_aux_init(struct radeon_connector *radeon_connector);
@@ -934,7 +936,7 @@ extern void radeon_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green
 				     u16 *blue, int regno);
 int radeon_framebuffer_init(struct drm_device *dev,
 			     struct radeon_framebuffer *rfb,
-			     struct drm_mode_fb_cmd2 *mode_cmd,
+			     const struct drm_mode_fb_cmd2 *mode_cmd,
 			     struct drm_gem_object *obj);
 
 int radeonfb_remove(struct drm_device *dev, struct drm_framebuffer *fb);
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 59abebd6b5dc..460c8f2989da 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -713,7 +713,7 @@ static struct attribute *hwmon_attributes[] = {
 static umode_t hwmon_attributes_visible(struct kobject *kobj,
 					struct attribute *attr, int index)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct radeon_device *rdev = dev_get_drvdata(dev);
 	umode_t effective_mode = attr->mode;
 
diff --git a/drivers/gpu/drm/radeon/radeon_state.c b/drivers/gpu/drm/radeon/radeon_state.c
deleted file mode 100644
index 15aee723db77..000000000000
--- a/drivers/gpu/drm/radeon/radeon_state.c
+++ /dev/null
@@ -1,3261 +0,0 @@
-/* radeon_state.c -- State support for Radeon -*- linux-c -*- */
-/*
- * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Gareth Hughes <gareth@valinux.com>
- *    Kevin E. Martin <martin@valinux.com>
- *
- * ------------------------ This file is DEPRECATED! -------------------------
- */
-
-#include <drm/drmP.h>
-#include <drm/radeon_drm.h>
-#include "radeon_drv.h"
-#include "drm_buffer.h"
-
-/* ================================================================
- * Helper functions for client state checking and fixup
- */
-
-static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
-						    dev_priv,
-						    struct drm_file * file_priv,
-						    u32 *offset)
-{
-	u64 off = *offset;
-	u32 fb_end = dev_priv->fb_location + dev_priv->fb_size - 1;
-	struct drm_radeon_driver_file_fields *radeon_priv;
-
-	/* Hrm ... the story of the offset ... So this function converts
-	 * the various ideas of what userland clients might have for an
-	 * offset in the card address space into an offset into the card
-	 * address space :) So with a sane client, it should just keep
-	 * the value intact and just do some boundary checking. However,
-	 * not all clients are sane. Some older clients pass us 0 based
-	 * offsets relative to the start of the framebuffer and some may
-	 * assume the AGP aperture it appended to the framebuffer, so we
-	 * try to detect those cases and fix them up.
-	 *
-	 * Note: It might be a good idea here to make sure the offset lands
-	 * in some "allowed" area to protect things like the PCIE GART...
-	 */
-
-	/* First, the best case, the offset already lands in either the
-	 * framebuffer or the GART mapped space
-	 */
-	if (radeon_check_offset(dev_priv, off))
-		return 0;
-
-	/* Ok, that didn't happen... now check if we have a zero based
-	 * offset that fits in the framebuffer + gart space, apply the
-	 * magic offset we get from SETPARAM or calculated from fb_location
-	 */
-	if (off < (dev_priv->fb_size + dev_priv->gart_size)) {
-		radeon_priv = file_priv->driver_priv;
-		off += radeon_priv->radeon_fb_delta;
-	}
-
-	/* Finally, assume we aimed at a GART offset if beyond the fb */
-	if (off > fb_end)
-		off = off - fb_end - 1 + dev_priv->gart_vm_start;
-
-	/* Now recheck and fail if out of bounds */
-	if (radeon_check_offset(dev_priv, off)) {
-		DRM_DEBUG("offset fixed up to 0x%x\n", (unsigned int)off);
-		*offset = off;
-		return 0;
-	}
-	return -EINVAL;
-}
-
-static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
-						     dev_priv,
-						     struct drm_file *file_priv,
-						     int id, struct drm_buffer *buf)
-{
-	u32 *data;
-	switch (id) {
-
-	case RADEON_EMIT_PP_MISC:
-		data = drm_buffer_pointer_to_dword(buf,
-			(RADEON_RB3D_DEPTHOFFSET - RADEON_PP_MISC) / 4);
-
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, data)) {
-			DRM_ERROR("Invalid depth buffer offset\n");
-			return -EINVAL;
-		}
-		dev_priv->have_z_offset = 1;
-		break;
-
-	case RADEON_EMIT_PP_CNTL:
-		data = drm_buffer_pointer_to_dword(buf,
-			(RADEON_RB3D_COLOROFFSET - RADEON_PP_CNTL) / 4);
-
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, data)) {
-			DRM_ERROR("Invalid colour buffer offset\n");
-			return -EINVAL;
-		}
-		break;
-
-	case R200_EMIT_PP_TXOFFSET_0:
-	case R200_EMIT_PP_TXOFFSET_1:
-	case R200_EMIT_PP_TXOFFSET_2:
-	case R200_EMIT_PP_TXOFFSET_3:
-	case R200_EMIT_PP_TXOFFSET_4:
-	case R200_EMIT_PP_TXOFFSET_5:
-		data = drm_buffer_pointer_to_dword(buf, 0);
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, data)) {
-			DRM_ERROR("Invalid R200 texture offset\n");
-			return -EINVAL;
-		}
-		break;
-
-	case RADEON_EMIT_PP_TXFILTER_0:
-	case RADEON_EMIT_PP_TXFILTER_1:
-	case RADEON_EMIT_PP_TXFILTER_2:
-		data = drm_buffer_pointer_to_dword(buf,
-			(RADEON_PP_TXOFFSET_0 - RADEON_PP_TXFILTER_0) / 4);
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, data)) {
-			DRM_ERROR("Invalid R100 texture offset\n");
-			return -EINVAL;
-		}
-		break;
-
-	case R200_EMIT_PP_CUBIC_OFFSETS_0:
-	case R200_EMIT_PP_CUBIC_OFFSETS_1:
-	case R200_EMIT_PP_CUBIC_OFFSETS_2:
-	case R200_EMIT_PP_CUBIC_OFFSETS_3:
-	case R200_EMIT_PP_CUBIC_OFFSETS_4:
-	case R200_EMIT_PP_CUBIC_OFFSETS_5:{
-			int i;
-			for (i = 0; i < 5; i++) {
-				data = drm_buffer_pointer_to_dword(buf, i);
-				if (radeon_check_and_fixup_offset(dev_priv,
-								  file_priv,
-								  data)) {
-					DRM_ERROR
-					    ("Invalid R200 cubic texture offset\n");
-					return -EINVAL;
-				}
-			}
-			break;
-		}
-
-	case RADEON_EMIT_PP_CUBIC_OFFSETS_T0:
-	case RADEON_EMIT_PP_CUBIC_OFFSETS_T1:
-	case RADEON_EMIT_PP_CUBIC_OFFSETS_T2:{
-			int i;
-			for (i = 0; i < 5; i++) {
-				data = drm_buffer_pointer_to_dword(buf, i);
-				if (radeon_check_and_fixup_offset(dev_priv,
-								  file_priv,
-								  data)) {
-					DRM_ERROR
-					    ("Invalid R100 cubic texture offset\n");
-					return -EINVAL;
-				}
-			}
-		}
-		break;
-
-	case R200_EMIT_VAP_CTL:{
-			RING_LOCALS;
-			BEGIN_RING(2);
-			OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
-			ADVANCE_RING();
-		}
-		break;
-
-	case RADEON_EMIT_RB3D_COLORPITCH:
-	case RADEON_EMIT_RE_LINE_PATTERN:
-	case RADEON_EMIT_SE_LINE_WIDTH:
-	case RADEON_EMIT_PP_LUM_MATRIX:
-	case RADEON_EMIT_PP_ROT_MATRIX_0:
-	case RADEON_EMIT_RB3D_STENCILREFMASK:
-	case RADEON_EMIT_SE_VPORT_XSCALE:
-	case RADEON_EMIT_SE_CNTL:
-	case RADEON_EMIT_SE_CNTL_STATUS:
-	case RADEON_EMIT_RE_MISC:
-	case RADEON_EMIT_PP_BORDER_COLOR_0:
-	case RADEON_EMIT_PP_BORDER_COLOR_1:
-	case RADEON_EMIT_PP_BORDER_COLOR_2:
-	case RADEON_EMIT_SE_ZBIAS_FACTOR:
-	case RADEON_EMIT_SE_TCL_OUTPUT_VTX_FMT:
-	case RADEON_EMIT_SE_TCL_MATERIAL_EMMISSIVE_RED:
-	case R200_EMIT_PP_TXCBLEND_0:
-	case R200_EMIT_PP_TXCBLEND_1:
-	case R200_EMIT_PP_TXCBLEND_2:
-	case R200_EMIT_PP_TXCBLEND_3:
-	case R200_EMIT_PP_TXCBLEND_4:
-	case R200_EMIT_PP_TXCBLEND_5:
-	case R200_EMIT_PP_TXCBLEND_6:
-	case R200_EMIT_PP_TXCBLEND_7:
-	case R200_EMIT_TCL_LIGHT_MODEL_CTL_0:
-	case R200_EMIT_TFACTOR_0:
-	case R200_EMIT_VTX_FMT_0:
-	case R200_EMIT_MATRIX_SELECT_0:
-	case R200_EMIT_TEX_PROC_CTL_2:
-	case R200_EMIT_TCL_UCP_VERT_BLEND_CTL:
-	case R200_EMIT_PP_TXFILTER_0:
-	case R200_EMIT_PP_TXFILTER_1:
-	case R200_EMIT_PP_TXFILTER_2:
-	case R200_EMIT_PP_TXFILTER_3:
-	case R200_EMIT_PP_TXFILTER_4:
-	case R200_EMIT_PP_TXFILTER_5:
-	case R200_EMIT_VTE_CNTL:
-	case R200_EMIT_OUTPUT_VTX_COMP_SEL:
-	case R200_EMIT_PP_TAM_DEBUG3:
-	case R200_EMIT_PP_CNTL_X:
-	case R200_EMIT_RB3D_DEPTHXY_OFFSET:
-	case R200_EMIT_RE_AUX_SCISSOR_CNTL:
-	case R200_EMIT_RE_SCISSOR_TL_0:
-	case R200_EMIT_RE_SCISSOR_TL_1:
-	case R200_EMIT_RE_SCISSOR_TL_2:
-	case R200_EMIT_SE_VAP_CNTL_STATUS:
-	case R200_EMIT_SE_VTX_STATE_CNTL:
-	case R200_EMIT_RE_POINTSIZE:
-	case R200_EMIT_TCL_INPUT_VTX_VECTOR_ADDR_0:
-	case R200_EMIT_PP_CUBIC_FACES_0:
-	case R200_EMIT_PP_CUBIC_FACES_1:
-	case R200_EMIT_PP_CUBIC_FACES_2:
-	case R200_EMIT_PP_CUBIC_FACES_3:
-	case R200_EMIT_PP_CUBIC_FACES_4:
-	case R200_EMIT_PP_CUBIC_FACES_5:
-	case RADEON_EMIT_PP_TEX_SIZE_0:
-	case RADEON_EMIT_PP_TEX_SIZE_1:
-	case RADEON_EMIT_PP_TEX_SIZE_2:
-	case R200_EMIT_RB3D_BLENDCOLOR:
-	case R200_EMIT_TCL_POINT_SPRITE_CNTL:
-	case RADEON_EMIT_PP_CUBIC_FACES_0:
-	case RADEON_EMIT_PP_CUBIC_FACES_1:
-	case RADEON_EMIT_PP_CUBIC_FACES_2:
-	case R200_EMIT_PP_TRI_PERF_CNTL:
-	case R200_EMIT_PP_AFS_0:
-	case R200_EMIT_PP_AFS_1:
-	case R200_EMIT_ATF_TFACTOR:
-	case R200_EMIT_PP_TXCTLALL_0:
-	case R200_EMIT_PP_TXCTLALL_1:
-	case R200_EMIT_PP_TXCTLALL_2:
-	case R200_EMIT_PP_TXCTLALL_3:
-	case R200_EMIT_PP_TXCTLALL_4:
-	case R200_EMIT_PP_TXCTLALL_5:
-	case R200_EMIT_VAP_PVS_CNTL:
-		/* These packets don't contain memory offsets */
-		break;
-
-	default:
-		DRM_ERROR("Unknown state packet ID %d\n", id);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int radeon_check_and_fixup_packet3(drm_radeon_private_t *
-					  dev_priv,
-					  struct drm_file *file_priv,
-					  drm_radeon_kcmd_buffer_t *
-					  cmdbuf,
-					  unsigned int *cmdsz)
-{
-	u32 *cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0);
-	u32 offset, narrays;
-	int count, i, k;
-
-	count = ((*cmd & RADEON_CP_PACKET_COUNT_MASK) >> 16);
-	*cmdsz = 2 + count;
-
-	if ((*cmd & 0xc0000000) != RADEON_CP_PACKET3) {
-		DRM_ERROR("Not a type 3 packet\n");
-		return -EINVAL;
-	}
-
-	if (4 * *cmdsz > drm_buffer_unprocessed(cmdbuf->buffer)) {
-		DRM_ERROR("Packet size larger than size of data provided\n");
-		return -EINVAL;
-	}
-
-	switch (*cmd & 0xff00) {
-	/* XXX Are there old drivers needing other packets? */
-
-	case RADEON_3D_DRAW_IMMD:
-	case RADEON_3D_DRAW_VBUF:
-	case RADEON_3D_DRAW_INDX:
-	case RADEON_WAIT_FOR_IDLE:
-	case RADEON_CP_NOP:
-	case RADEON_3D_CLEAR_ZMASK:
-/*	case RADEON_CP_NEXT_CHAR:
-	case RADEON_CP_PLY_NEXTSCAN:
-	case RADEON_CP_SET_SCISSORS: */ /* probably safe but will never need them? */
-		/* these packets are safe */
-		break;
-
-	case RADEON_CP_3D_DRAW_IMMD_2:
-	case RADEON_CP_3D_DRAW_VBUF_2:
-	case RADEON_CP_3D_DRAW_INDX_2:
-	case RADEON_3D_CLEAR_HIZ:
-		/* safe but r200 only */
-		if (dev_priv->microcode_version != UCODE_R200) {
-			DRM_ERROR("Invalid 3d packet for r100-class chip\n");
-			return -EINVAL;
-		}
-		break;
-
-	case RADEON_3D_LOAD_VBPNTR:
-
-		if (count > 18) { /* 12 arrays max */
-			DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
-				  count);
-			return -EINVAL;
-		}
-
-		/* carefully check packet contents */
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-
-		narrays = *cmd & ~0xc000;
-		k = 0;
-		i = 2;
-		while ((k < narrays) && (i < (count + 2))) {
-			i++;		/* skip attribute field */
-			cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, i);
-			if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-							  cmd)) {
-				DRM_ERROR
-				    ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
-				     k, i);
-				return -EINVAL;
-			}
-			k++;
-			i++;
-			if (k == narrays)
-				break;
-			/* have one more to process, they come in pairs */
-			cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, i);
-
-			if (radeon_check_and_fixup_offset(dev_priv,
-							  file_priv, cmd))
-			{
-				DRM_ERROR
-				    ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
-				     k, i);
-				return -EINVAL;
-			}
-			k++;
-			i++;
-		}
-		/* do the counts match what we expect ? */
-		if ((k != narrays) || (i != (count + 2))) {
-			DRM_ERROR
-			    ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
-			      k, i, narrays, count + 1);
-			return -EINVAL;
-		}
-		break;
-
-	case RADEON_3D_RNDR_GEN_INDX_PRIM:
-		if (dev_priv->microcode_version != UCODE_R100) {
-			DRM_ERROR("Invalid 3d packet for r200-class chip\n");
-			return -EINVAL;
-		}
-
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, cmd)) {
-				DRM_ERROR("Invalid rndr_gen_indx offset\n");
-				return -EINVAL;
-		}
-		break;
-
-	case RADEON_CP_INDX_BUFFER:
-		if (dev_priv->microcode_version != UCODE_R200) {
-			DRM_ERROR("Invalid 3d packet for r100-class chip\n");
-			return -EINVAL;
-		}
-
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-		if ((*cmd & 0x8000ffff) != 0x80000810) {
-			DRM_ERROR("Invalid indx_buffer reg address %08X\n", *cmd);
-			return -EINVAL;
-		}
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2);
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv, cmd)) {
-			DRM_ERROR("Invalid indx_buffer offset is %08X\n", *cmd);
-			return -EINVAL;
-		}
-		break;
-
-	case RADEON_CNTL_HOSTDATA_BLT:
-	case RADEON_CNTL_PAINT_MULTI:
-	case RADEON_CNTL_BITBLT_MULTI:
-		/* MSB of opcode: next DWORD GUI_CNTL */
-		cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1);
-		if (*cmd & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
-			      | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
-			u32 *cmd2 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2);
-			offset = *cmd2 << 10;
-			if (radeon_check_and_fixup_offset
-			    (dev_priv, file_priv, &offset)) {
-				DRM_ERROR("Invalid first packet offset\n");
-				return -EINVAL;
-			}
-			*cmd2 = (*cmd2 & 0xffc00000) | offset >> 10;
-		}
-
-		if ((*cmd & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
-		    (*cmd & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
-			u32 *cmd3 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 3);
-			offset = *cmd3 << 10;
-			if (radeon_check_and_fixup_offset
-			    (dev_priv, file_priv, &offset)) {
-				DRM_ERROR("Invalid second packet offset\n");
-				return -EINVAL;
-			}
-			*cmd3 = (*cmd3 & 0xffc00000) | offset >> 10;
-		}
-		break;
-
-	default:
-		DRM_ERROR("Invalid packet type %x\n", *cmd & 0xff00);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* ================================================================
- * CP hardware state programming functions
- */
-
-static void radeon_emit_clip_rect(drm_radeon_private_t * dev_priv,
-				  struct drm_clip_rect * box)
-{
-	RING_LOCALS;
-
-	DRM_DEBUG("   box:  x1=%d y1=%d  x2=%d y2=%d\n",
-		  box->x1, box->y1, box->x2, box->y2);
-
-	BEGIN_RING(4);
-	OUT_RING(CP_PACKET0(RADEON_RE_TOP_LEFT, 0));
-	OUT_RING((box->y1 << 16) | box->x1);
-	OUT_RING(CP_PACKET0(RADEON_RE_WIDTH_HEIGHT, 0));
-	OUT_RING(((box->y2 - 1) << 16) | (box->x2 - 1));
-	ADVANCE_RING();
-}
-
-/* Emit 1.1 state
- */
-static int radeon_emit_state(drm_radeon_private_t * dev_priv,
-			     struct drm_file *file_priv,
-			     drm_radeon_context_regs_t * ctx,
-			     drm_radeon_texture_regs_t * tex,
-			     unsigned int dirty)
-{
-	RING_LOCALS;
-	DRM_DEBUG("dirty=0x%08x\n", dirty);
-
-	if (dirty & RADEON_UPLOAD_CONTEXT) {
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-						  &ctx->rb3d_depthoffset)) {
-			DRM_ERROR("Invalid depth buffer offset\n");
-			return -EINVAL;
-		}
-
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-						  &ctx->rb3d_coloroffset)) {
-			DRM_ERROR("Invalid depth buffer offset\n");
-			return -EINVAL;
-		}
-
-		BEGIN_RING(14);
-		OUT_RING(CP_PACKET0(RADEON_PP_MISC, 6));
-		OUT_RING(ctx->pp_misc);
-		OUT_RING(ctx->pp_fog_color);
-		OUT_RING(ctx->re_solid_color);
-		OUT_RING(ctx->rb3d_blendcntl);
-		OUT_RING(ctx->rb3d_depthoffset);
-		OUT_RING(ctx->rb3d_depthpitch);
-		OUT_RING(ctx->rb3d_zstencilcntl);
-		OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 2));
-		OUT_RING(ctx->pp_cntl);
-		OUT_RING(ctx->rb3d_cntl);
-		OUT_RING(ctx->rb3d_coloroffset);
-		OUT_RING(CP_PACKET0(RADEON_RB3D_COLORPITCH, 0));
-		OUT_RING(ctx->rb3d_colorpitch);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_VERTFMT) {
-		BEGIN_RING(2);
-		OUT_RING(CP_PACKET0(RADEON_SE_COORD_FMT, 0));
-		OUT_RING(ctx->se_coord_fmt);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_LINE) {
-		BEGIN_RING(5);
-		OUT_RING(CP_PACKET0(RADEON_RE_LINE_PATTERN, 1));
-		OUT_RING(ctx->re_line_pattern);
-		OUT_RING(ctx->re_line_state);
-		OUT_RING(CP_PACKET0(RADEON_SE_LINE_WIDTH, 0));
-		OUT_RING(ctx->se_line_width);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_BUMPMAP) {
-		BEGIN_RING(5);
-		OUT_RING(CP_PACKET0(RADEON_PP_LUM_MATRIX, 0));
-		OUT_RING(ctx->pp_lum_matrix);
-		OUT_RING(CP_PACKET0(RADEON_PP_ROT_MATRIX_0, 1));
-		OUT_RING(ctx->pp_rot_matrix_0);
-		OUT_RING(ctx->pp_rot_matrix_1);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_MASKS) {
-		BEGIN_RING(4);
-		OUT_RING(CP_PACKET0(RADEON_RB3D_STENCILREFMASK, 2));
-		OUT_RING(ctx->rb3d_stencilrefmask);
-		OUT_RING(ctx->rb3d_ropcntl);
-		OUT_RING(ctx->rb3d_planemask);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_VIEWPORT) {
-		BEGIN_RING(7);
-		OUT_RING(CP_PACKET0(RADEON_SE_VPORT_XSCALE, 5));
-		OUT_RING(ctx->se_vport_xscale);
-		OUT_RING(ctx->se_vport_xoffset);
-		OUT_RING(ctx->se_vport_yscale);
-		OUT_RING(ctx->se_vport_yoffset);
-		OUT_RING(ctx->se_vport_zscale);
-		OUT_RING(ctx->se_vport_zoffset);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_SETUP) {
-		BEGIN_RING(4);
-		OUT_RING(CP_PACKET0(RADEON_SE_CNTL, 0));
-		OUT_RING(ctx->se_cntl);
-		OUT_RING(CP_PACKET0(RADEON_SE_CNTL_STATUS, 0));
-		OUT_RING(ctx->se_cntl_status);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_MISC) {
-		BEGIN_RING(2);
-		OUT_RING(CP_PACKET0(RADEON_RE_MISC, 0));
-		OUT_RING(ctx->re_misc);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_TEX0) {
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-						  &tex[0].pp_txoffset)) {
-			DRM_ERROR("Invalid texture offset for unit 0\n");
-			return -EINVAL;
-		}
-
-		BEGIN_RING(9);
-		OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_0, 5));
-		OUT_RING(tex[0].pp_txfilter);
-		OUT_RING(tex[0].pp_txformat);
-		OUT_RING(tex[0].pp_txoffset);
-		OUT_RING(tex[0].pp_txcblend);
-		OUT_RING(tex[0].pp_txablend);
-		OUT_RING(tex[0].pp_tfactor);
-		OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_0, 0));
-		OUT_RING(tex[0].pp_border_color);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_TEX1) {
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-						  &tex[1].pp_txoffset)) {
-			DRM_ERROR("Invalid texture offset for unit 1\n");
-			return -EINVAL;
-		}
-
-		BEGIN_RING(9);
-		OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_1, 5));
-		OUT_RING(tex[1].pp_txfilter);
-		OUT_RING(tex[1].pp_txformat);
-		OUT_RING(tex[1].pp_txoffset);
-		OUT_RING(tex[1].pp_txcblend);
-		OUT_RING(tex[1].pp_txablend);
-		OUT_RING(tex[1].pp_tfactor);
-		OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_1, 0));
-		OUT_RING(tex[1].pp_border_color);
-		ADVANCE_RING();
-	}
-
-	if (dirty & RADEON_UPLOAD_TEX2) {
-		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
-						  &tex[2].pp_txoffset)) {
-			DRM_ERROR("Invalid texture offset for unit 2\n");
-			return -EINVAL;
-		}
-
-		BEGIN_RING(9);
-		OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_2, 5));
-		OUT_RING(tex[2].pp_txfilter);
-		OUT_RING(tex[2].pp_txformat);
-		OUT_RING(tex[2].pp_txoffset);
-		OUT_RING(tex[2].pp_txcblend);
-		OUT_RING(tex[2].pp_txablend);
-		OUT_RING(tex[2].pp_tfactor);
-		OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_2, 0));
-		OUT_RING(tex[2].pp_border_color);
-		ADVANCE_RING();
-	}
-
-	return 0;
-}
-
-/* Emit 1.2 state
- */
-static int radeon_emit_state2(drm_radeon_private_t * dev_priv,
-			      struct drm_file *file_priv,
-			      drm_radeon_state_t * state)
-{
-	RING_LOCALS;
-
-	if (state->dirty & RADEON_UPLOAD_ZBIAS) {
-		BEGIN_RING(3);
-		OUT_RING(CP_PACKET0(RADEON_SE_ZBIAS_FACTOR, 1));
-		OUT_RING(state->context2.se_zbias_factor);
-		OUT_RING(state->context2.se_zbias_constant);
-		ADVANCE_RING();
-	}
-
-	return radeon_emit_state(dev_priv, file_priv, &state->context,
-				 state->tex, state->dirty);
-}
-
-/* New (1.3) state mechanism.  3 commands (packet, scalar, vector) in
- * 1.3 cmdbuffers allow all previous state to be updated as well as
- * the tcl scalar and vector areas.
- */
-static struct {
-	int start;
-	int len;
-	const char *name;
-} packet[RADEON_MAX_STATE_PACKETS] = {
-	{RADEON_PP_MISC, 7, "RADEON_PP_MISC"},
-	{RADEON_PP_CNTL, 3, "RADEON_PP_CNTL"},
-	{RADEON_RB3D_COLORPITCH, 1, "RADEON_RB3D_COLORPITCH"},
-	{RADEON_RE_LINE_PATTERN, 2, "RADEON_RE_LINE_PATTERN"},
-	{RADEON_SE_LINE_WIDTH, 1, "RADEON_SE_LINE_WIDTH"},
-	{RADEON_PP_LUM_MATRIX, 1, "RADEON_PP_LUM_MATRIX"},
-	{RADEON_PP_ROT_MATRIX_0, 2, "RADEON_PP_ROT_MATRIX_0"},
-	{RADEON_RB3D_STENCILREFMASK, 3, "RADEON_RB3D_STENCILREFMASK"},
-	{RADEON_SE_VPORT_XSCALE, 6, "RADEON_SE_VPORT_XSCALE"},
-	{RADEON_SE_CNTL, 2, "RADEON_SE_CNTL"},
-	{RADEON_SE_CNTL_STATUS, 1, "RADEON_SE_CNTL_STATUS"},
-	{RADEON_RE_MISC, 1, "RADEON_RE_MISC"},
-	{RADEON_PP_TXFILTER_0, 6, "RADEON_PP_TXFILTER_0"},
-	{RADEON_PP_BORDER_COLOR_0, 1, "RADEON_PP_BORDER_COLOR_0"},
-	{RADEON_PP_TXFILTER_1, 6, "RADEON_PP_TXFILTER_1"},
-	{RADEON_PP_BORDER_COLOR_1, 1, "RADEON_PP_BORDER_COLOR_1"},
-	{RADEON_PP_TXFILTER_2, 6, "RADEON_PP_TXFILTER_2"},
-	{RADEON_PP_BORDER_COLOR_2, 1, "RADEON_PP_BORDER_COLOR_2"},
-	{RADEON_SE_ZBIAS_FACTOR, 2, "RADEON_SE_ZBIAS_FACTOR"},
-	{RADEON_SE_TCL_OUTPUT_VTX_FMT, 11, "RADEON_SE_TCL_OUTPUT_VTX_FMT"},
-	{RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 17,
-		    "RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED"},
-	{R200_PP_TXCBLEND_0, 4, "R200_PP_TXCBLEND_0"},
-	{R200_PP_TXCBLEND_1, 4, "R200_PP_TXCBLEND_1"},
-	{R200_PP_TXCBLEND_2, 4, "R200_PP_TXCBLEND_2"},
-	{R200_PP_TXCBLEND_3, 4, "R200_PP_TXCBLEND_3"},
-	{R200_PP_TXCBLEND_4, 4, "R200_PP_TXCBLEND_4"},
-	{R200_PP_TXCBLEND_5, 4, "R200_PP_TXCBLEND_5"},
-	{R200_PP_TXCBLEND_6, 4, "R200_PP_TXCBLEND_6"},
-	{R200_PP_TXCBLEND_7, 4, "R200_PP_TXCBLEND_7"},
-	{R200_SE_TCL_LIGHT_MODEL_CTL_0, 6, "R200_SE_TCL_LIGHT_MODEL_CTL_0"},
-	{R200_PP_TFACTOR_0, 6, "R200_PP_TFACTOR_0"},
-	{R200_SE_VTX_FMT_0, 4, "R200_SE_VTX_FMT_0"},
-	{R200_SE_VAP_CNTL, 1, "R200_SE_VAP_CNTL"},
-	{R200_SE_TCL_MATRIX_SEL_0, 5, "R200_SE_TCL_MATRIX_SEL_0"},
-	{R200_SE_TCL_TEX_PROC_CTL_2, 5, "R200_SE_TCL_TEX_PROC_CTL_2"},
-	{R200_SE_TCL_UCP_VERT_BLEND_CTL, 1, "R200_SE_TCL_UCP_VERT_BLEND_CTL"},
-	{R200_PP_TXFILTER_0, 6, "R200_PP_TXFILTER_0"},
-	{R200_PP_TXFILTER_1, 6, "R200_PP_TXFILTER_1"},
-	{R200_PP_TXFILTER_2, 6, "R200_PP_TXFILTER_2"},
-	{R200_PP_TXFILTER_3, 6, "R200_PP_TXFILTER_3"},
-	{R200_PP_TXFILTER_4, 6, "R200_PP_TXFILTER_4"},
-	{R200_PP_TXFILTER_5, 6, "R200_PP_TXFILTER_5"},
-	{R200_PP_TXOFFSET_0, 1, "R200_PP_TXOFFSET_0"},
-	{R200_PP_TXOFFSET_1, 1, "R200_PP_TXOFFSET_1"},
-	{R200_PP_TXOFFSET_2, 1, "R200_PP_TXOFFSET_2"},
-	{R200_PP_TXOFFSET_3, 1, "R200_PP_TXOFFSET_3"},
-	{R200_PP_TXOFFSET_4, 1, "R200_PP_TXOFFSET_4"},
-	{R200_PP_TXOFFSET_5, 1, "R200_PP_TXOFFSET_5"},
-	{R200_SE_VTE_CNTL, 1, "R200_SE_VTE_CNTL"},
-	{R200_SE_TCL_OUTPUT_VTX_COMP_SEL, 1,
-	 "R200_SE_TCL_OUTPUT_VTX_COMP_SEL"},
-	{R200_PP_TAM_DEBUG3, 1, "R200_PP_TAM_DEBUG3"},
-	{R200_PP_CNTL_X, 1, "R200_PP_CNTL_X"},
-	{R200_RB3D_DEPTHXY_OFFSET, 1, "R200_RB3D_DEPTHXY_OFFSET"},
-	{R200_RE_AUX_SCISSOR_CNTL, 1, "R200_RE_AUX_SCISSOR_CNTL"},
-	{R200_RE_SCISSOR_TL_0, 2, "R200_RE_SCISSOR_TL_0"},
-	{R200_RE_SCISSOR_TL_1, 2, "R200_RE_SCISSOR_TL_1"},
-	{R200_RE_SCISSOR_TL_2, 2, "R200_RE_SCISSOR_TL_2"},
-	{R200_SE_VAP_CNTL_STATUS, 1, "R200_SE_VAP_CNTL_STATUS"},
-	{R200_SE_VTX_STATE_CNTL, 1, "R200_SE_VTX_STATE_CNTL"},
-	{R200_RE_POINTSIZE, 1, "R200_RE_POINTSIZE"},
-	{R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0, 4,
-		    "R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0"},
-	{R200_PP_CUBIC_FACES_0, 1, "R200_PP_CUBIC_FACES_0"},	/* 61 */
-	{R200_PP_CUBIC_OFFSET_F1_0, 5, "R200_PP_CUBIC_OFFSET_F1_0"}, /* 62 */
-	{R200_PP_CUBIC_FACES_1, 1, "R200_PP_CUBIC_FACES_1"},
-	{R200_PP_CUBIC_OFFSET_F1_1, 5, "R200_PP_CUBIC_OFFSET_F1_1"},
-	{R200_PP_CUBIC_FACES_2, 1, "R200_PP_CUBIC_FACES_2"},
-	{R200_PP_CUBIC_OFFSET_F1_2, 5, "R200_PP_CUBIC_OFFSET_F1_2"},
-	{R200_PP_CUBIC_FACES_3, 1, "R200_PP_CUBIC_FACES_3"},
-	{R200_PP_CUBIC_OFFSET_F1_3, 5, "R200_PP_CUBIC_OFFSET_F1_3"},
-	{R200_PP_CUBIC_FACES_4, 1, "R200_PP_CUBIC_FACES_4"},
-	{R200_PP_CUBIC_OFFSET_F1_4, 5, "R200_PP_CUBIC_OFFSET_F1_4"},
-	{R200_PP_CUBIC_FACES_5, 1, "R200_PP_CUBIC_FACES_5"},
-	{R200_PP_CUBIC_OFFSET_F1_5, 5, "R200_PP_CUBIC_OFFSET_F1_5"},
-	{RADEON_PP_TEX_SIZE_0, 2, "RADEON_PP_TEX_SIZE_0"},
-	{RADEON_PP_TEX_SIZE_1, 2, "RADEON_PP_TEX_SIZE_1"},
-	{RADEON_PP_TEX_SIZE_2, 2, "RADEON_PP_TEX_SIZE_2"},
-	{R200_RB3D_BLENDCOLOR, 3, "R200_RB3D_BLENDCOLOR"},
-	{R200_SE_TCL_POINT_SPRITE_CNTL, 1, "R200_SE_TCL_POINT_SPRITE_CNTL"},
-	{RADEON_PP_CUBIC_FACES_0, 1, "RADEON_PP_CUBIC_FACES_0"},
-	{RADEON_PP_CUBIC_OFFSET_T0_0, 5, "RADEON_PP_CUBIC_OFFSET_T0_0"},
-	{RADEON_PP_CUBIC_FACES_1, 1, "RADEON_PP_CUBIC_FACES_1"},
-	{RADEON_PP_CUBIC_OFFSET_T1_0, 5, "RADEON_PP_CUBIC_OFFSET_T1_0"},
-	{RADEON_PP_CUBIC_FACES_2, 1, "RADEON_PP_CUBIC_FACES_2"},
-	{RADEON_PP_CUBIC_OFFSET_T2_0, 5, "RADEON_PP_CUBIC_OFFSET_T2_0"},
-	{R200_PP_TRI_PERF, 2, "R200_PP_TRI_PERF"},
-	{R200_PP_AFS_0, 32, "R200_PP_AFS_0"},     /* 85 */
-	{R200_PP_AFS_1, 32, "R200_PP_AFS_1"},
-	{R200_PP_TFACTOR_0, 8, "R200_ATF_TFACTOR"},
-	{R200_PP_TXFILTER_0, 8, "R200_PP_TXCTLALL_0"},
-	{R200_PP_TXFILTER_1, 8, "R200_PP_TXCTLALL_1"},
-	{R200_PP_TXFILTER_2, 8, "R200_PP_TXCTLALL_2"},
-	{R200_PP_TXFILTER_3, 8, "R200_PP_TXCTLALL_3"},
-	{R200_PP_TXFILTER_4, 8, "R200_PP_TXCTLALL_4"},
-	{R200_PP_TXFILTER_5, 8, "R200_PP_TXCTLALL_5"},
-	{R200_VAP_PVS_CNTL_1, 2, "R200_VAP_PVS_CNTL"},
-};
-
-/* ================================================================
- * Performance monitoring functions
- */
-
-static void radeon_clear_box(drm_radeon_private_t * dev_priv,
-			     struct drm_radeon_master_private *master_priv,
-			     int x, int y, int w, int h, int r, int g, int b)
-{
-	u32 color;
-	RING_LOCALS;
-
-	x += master_priv->sarea_priv->boxes[0].x1;
-	y += master_priv->sarea_priv->boxes[0].y1;
-
-	switch (dev_priv->color_fmt) {
-	case RADEON_COLOR_FORMAT_RGB565:
-		color = (((r & 0xf8) << 8) |
-			 ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
-		break;
-	case RADEON_COLOR_FORMAT_ARGB8888:
-	default:
-		color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
-		break;
-	}
-
-	BEGIN_RING(4);
-	RADEON_WAIT_UNTIL_3D_IDLE();
-	OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
-	OUT_RING(0xffffffff);
-	ADVANCE_RING();
-
-	BEGIN_RING(6);
-
-	OUT_RING(CP_PACKET3(RADEON_CNTL_PAINT_MULTI, 4));
-	OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
-		 RADEON_GMC_BRUSH_SOLID_COLOR |
-		 (dev_priv->color_fmt << 8) |
-		 RADEON_GMC_SRC_DATATYPE_COLOR |
-		 RADEON_ROP3_P | RADEON_GMC_CLR_CMP_CNTL_DIS);
-
-	if (master_priv->sarea_priv->pfCurrentPage == 1) {
-		OUT_RING(dev_priv->front_pitch_offset);
-	} else {
-		OUT_RING(dev_priv->back_pitch_offset);
-	}
-
-	OUT_RING(color);
-
-	OUT_RING((x << 16) | y);
-	OUT_RING((w << 16) | h);
-
-	ADVANCE_RING();
-}
-
-static void radeon_cp_performance_boxes(drm_radeon_private_t *dev_priv, struct drm_radeon_master_private *master_priv)
-{
-	/* Collapse various things into a wait flag -- trying to
-	 * guess if userspase slept -- better just to have them tell us.
-	 */
-	if (dev_priv->stats.last_frame_reads > 1 ||
-	    dev_priv->stats.last_clear_reads > dev_priv->stats.clears) {
-		dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-	}
-
-	if (dev_priv->stats.freelist_loops) {
-		dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-	}
-
-	/* Purple box for page flipping
-	 */
-	if (dev_priv->stats.boxes & RADEON_BOX_FLIP)
-		radeon_clear_box(dev_priv, master_priv, 4, 4, 8, 8, 255, 0, 255);
-
-	/* Red box if we have to wait for idle at any point
-	 */
-	if (dev_priv->stats.boxes & RADEON_BOX_WAIT_IDLE)
-		radeon_clear_box(dev_priv, master_priv, 16, 4, 8, 8, 255, 0, 0);
-
-	/* Blue box: lost context?
-	 */
-
-	/* Yellow box for texture swaps
-	 */
-	if (dev_priv->stats.boxes & RADEON_BOX_TEXTURE_LOAD)
-		radeon_clear_box(dev_priv, master_priv, 40, 4, 8, 8, 255, 255, 0);
-
-	/* Green box if hardware never idles (as far as we can tell)
-	 */
-	if (!(dev_priv->stats.boxes & RADEON_BOX_DMA_IDLE))
-		radeon_clear_box(dev_priv, master_priv, 64, 4, 8, 8, 0, 255, 0);
-
-	/* Draw bars indicating number of buffers allocated
-	 * (not a great measure, easily confused)
-	 */
-	if (dev_priv->stats.requested_bufs) {
-		if (dev_priv->stats.requested_bufs > 100)
-			dev_priv->stats.requested_bufs = 100;
-
-		radeon_clear_box(dev_priv, master_priv, 4, 16,
-				 dev_priv->stats.requested_bufs, 4,
-				 196, 128, 128);
-	}
-
-	memset(&dev_priv->stats, 0, sizeof(dev_priv->stats));
-
-}
-
-/* ================================================================
- * CP command dispatch functions
- */
-
-static void radeon_cp_dispatch_clear(struct drm_device * dev,
-				     struct drm_master *master,
-				     drm_radeon_clear_t * clear,
-				     drm_radeon_clear_rect_t * depth_boxes)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	drm_radeon_depth_clear_t *depth_clear = &dev_priv->depth_clear;
-	int nbox = sarea_priv->nbox;
-	struct drm_clip_rect *pbox = sarea_priv->boxes;
-	unsigned int flags = clear->flags;
-	u32 rb3d_cntl = 0, rb3d_stencilrefmask = 0;
-	int i;
-	RING_LOCALS;
-	DRM_DEBUG("flags = 0x%x\n", flags);
-
-	dev_priv->stats.clears++;
-
-	if (sarea_priv->pfCurrentPage == 1) {
-		unsigned int tmp = flags;
-
-		flags &= ~(RADEON_FRONT | RADEON_BACK);
-		if (tmp & RADEON_FRONT)
-			flags |= RADEON_BACK;
-		if (tmp & RADEON_BACK)
-			flags |= RADEON_FRONT;
-	}
-	if (flags & (RADEON_DEPTH|RADEON_STENCIL)) {
-		if (!dev_priv->have_z_offset) {
-			printk_once(KERN_ERR "radeon: illegal depth clear request. Buggy mesa detected - please update.\n");
-			flags &= ~(RADEON_DEPTH | RADEON_STENCIL);
-		}
-	}
-
-	if (flags & (RADEON_FRONT | RADEON_BACK)) {
-
-		BEGIN_RING(4);
-
-		/* Ensure the 3D stream is idle before doing a
-		 * 2D fill to clear the front or back buffer.
-		 */
-		RADEON_WAIT_UNTIL_3D_IDLE();
-
-		OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
-		OUT_RING(clear->color_mask);
-
-		ADVANCE_RING();
-
-		/* Make sure we restore the 3D state next time.
-		 */
-		sarea_priv->ctx_owner = 0;
-
-		for (i = 0; i < nbox; i++) {
-			int x = pbox[i].x1;
-			int y = pbox[i].y1;
-			int w = pbox[i].x2 - x;
-			int h = pbox[i].y2 - y;
-
-			DRM_DEBUG("%d,%d-%d,%d flags 0x%x\n",
-				  x, y, w, h, flags);
-
-			if (flags & RADEON_FRONT) {
-				BEGIN_RING(6);
-
-				OUT_RING(CP_PACKET3
-					 (RADEON_CNTL_PAINT_MULTI, 4));
-				OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
-					 RADEON_GMC_BRUSH_SOLID_COLOR |
-					 (dev_priv->
-					  color_fmt << 8) |
-					 RADEON_GMC_SRC_DATATYPE_COLOR |
-					 RADEON_ROP3_P |
-					 RADEON_GMC_CLR_CMP_CNTL_DIS);
-
-				OUT_RING(dev_priv->front_pitch_offset);
-				OUT_RING(clear->clear_color);
-
-				OUT_RING((x << 16) | y);
-				OUT_RING((w << 16) | h);
-
-				ADVANCE_RING();
-			}
-
-			if (flags & RADEON_BACK) {
-				BEGIN_RING(6);
-
-				OUT_RING(CP_PACKET3
-					 (RADEON_CNTL_PAINT_MULTI, 4));
-				OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
-					 RADEON_GMC_BRUSH_SOLID_COLOR |
-					 (dev_priv->
-					  color_fmt << 8) |
-					 RADEON_GMC_SRC_DATATYPE_COLOR |
-					 RADEON_ROP3_P |
-					 RADEON_GMC_CLR_CMP_CNTL_DIS);
-
-				OUT_RING(dev_priv->back_pitch_offset);
-				OUT_RING(clear->clear_color);
-
-				OUT_RING((x << 16) | y);
-				OUT_RING((w << 16) | h);
-
-				ADVANCE_RING();
-			}
-		}
-	}
-
-	/* hyper z clear */
-	/* no docs available, based on reverse engineering by Stephane Marchesin */
-	if ((flags & (RADEON_DEPTH | RADEON_STENCIL))
-	    && (flags & RADEON_CLEAR_FASTZ)) {
-
-		int i;
-		int depthpixperline =
-		    dev_priv->depth_fmt ==
-		    RADEON_DEPTH_FORMAT_16BIT_INT_Z ? (dev_priv->depth_pitch /
-						       2) : (dev_priv->
-							     depth_pitch / 4);
-
-		u32 clearmask;
-
-		u32 tempRB3D_DEPTHCLEARVALUE = clear->clear_depth |
-		    ((clear->depth_mask & 0xff) << 24);
-
-		/* Make sure we restore the 3D state next time.
-		 * we haven't touched any "normal" state - still need this?
-		 */
-		sarea_priv->ctx_owner = 0;
-
-		if ((dev_priv->flags & RADEON_HAS_HIERZ)
-		    && (flags & RADEON_USE_HIERZ)) {
-			/* FIXME : reverse engineer that for Rx00 cards */
-			/* FIXME : the mask supposedly contains low-res z values. So can't set
-			   just to the max (0xff? or actually 0x3fff?), need to take z clear
-			   value into account? */
-			/* pattern seems to work for r100, though get slight
-			   rendering errors with glxgears. If hierz is not enabled for r100,
-			   only 4 bits which indicate clear (15,16,31,32, all zero) matter, the
-			   other ones are ignored, and the same clear mask can be used. That's
-			   very different behaviour than R200 which needs different clear mask
-			   and different number of tiles to clear if hierz is enabled or not !?!
-			 */
-			clearmask = (0xff << 22) | (0xff << 6) | 0x003f003f;
-		} else {
-			/* clear mask : chooses the clearing pattern.
-			   rv250: could be used to clear only parts of macrotiles
-			   (but that would get really complicated...)?
-			   bit 0 and 1 (either or both of them ?!?!) are used to
-			   not clear tile (or maybe one of the bits indicates if the tile is
-			   compressed or not), bit 2 and 3 to not clear tile 1,...,.
-			   Pattern is as follows:
-			   | 0,1 | 4,5 | 8,9 |12,13|16,17|20,21|24,25|28,29|
-			   bits -------------------------------------------------
-			   | 2,3 | 6,7 |10,11|14,15|18,19|22,23|26,27|30,31|
-			   rv100: clearmask covers 2x8 4x1 tiles, but one clear still
-			   covers 256 pixels ?!?
-			 */
-			clearmask = 0x0;
-		}
-
-		BEGIN_RING(8);
-		RADEON_WAIT_UNTIL_2D_IDLE();
-		OUT_RING_REG(RADEON_RB3D_DEPTHCLEARVALUE,
-			     tempRB3D_DEPTHCLEARVALUE);
-		/* what offset is this exactly ? */
-		OUT_RING_REG(RADEON_RB3D_ZMASKOFFSET, 0);
-		/* need ctlstat, otherwise get some strange black flickering */
-		OUT_RING_REG(RADEON_RB3D_ZCACHE_CTLSTAT,
-			     RADEON_RB3D_ZC_FLUSH_ALL);
-		ADVANCE_RING();
-
-		for (i = 0; i < nbox; i++) {
-			int tileoffset, nrtilesx, nrtilesy, j;
-			/* it looks like r200 needs rv-style clears, at least if hierz is not enabled? */
-			if ((dev_priv->flags & RADEON_HAS_HIERZ)
-			    && !(dev_priv->microcode_version == UCODE_R200)) {
-				/* FIXME : figure this out for r200 (when hierz is enabled). Or
-				   maybe r200 actually doesn't need to put the low-res z value into
-				   the tile cache like r100, but just needs to clear the hi-level z-buffer?
-				   Works for R100, both with hierz and without.
-				   R100 seems to operate on 2x1 8x8 tiles, but...
-				   odd: offset/nrtiles need to be 64 pix (4 block) aligned? Potentially
-				   problematic with resolutions which are not 64 pix aligned? */
-				tileoffset =
-				    ((pbox[i].y1 >> 3) * depthpixperline +
-				     pbox[i].x1) >> 6;
-				nrtilesx =
-				    ((pbox[i].x2 & ~63) -
-				     (pbox[i].x1 & ~63)) >> 4;
-				nrtilesy =
-				    (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
-				for (j = 0; j <= nrtilesy; j++) {
-					BEGIN_RING(4);
-					OUT_RING(CP_PACKET3
-						 (RADEON_3D_CLEAR_ZMASK, 2));
-					/* first tile */
-					OUT_RING(tileoffset * 8);
-					/* the number of tiles to clear */
-					OUT_RING(nrtilesx + 4);
-					/* clear mask : chooses the clearing pattern. */
-					OUT_RING(clearmask);
-					ADVANCE_RING();
-					tileoffset += depthpixperline >> 6;
-				}
-			} else if (dev_priv->microcode_version == UCODE_R200) {
-				/* works for rv250. */
-				/* find first macro tile (8x2 4x4 z-pixels on rv250) */
-				tileoffset =
-				    ((pbox[i].y1 >> 3) * depthpixperline +
-				     pbox[i].x1) >> 5;
-				nrtilesx =
-				    (pbox[i].x2 >> 5) - (pbox[i].x1 >> 5);
-				nrtilesy =
-				    (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
-				for (j = 0; j <= nrtilesy; j++) {
-					BEGIN_RING(4);
-					OUT_RING(CP_PACKET3
-						 (RADEON_3D_CLEAR_ZMASK, 2));
-					/* first tile */
-					/* judging by the first tile offset needed, could possibly
-					   directly address/clear 4x4 tiles instead of 8x2 * 4x4
-					   macro tiles, though would still need clear mask for
-					   right/bottom if truly 4x4 granularity is desired ? */
-					OUT_RING(tileoffset * 16);
-					/* the number of tiles to clear */
-					OUT_RING(nrtilesx + 1);
-					/* clear mask : chooses the clearing pattern. */
-					OUT_RING(clearmask);
-					ADVANCE_RING();
-					tileoffset += depthpixperline >> 5;
-				}
-			} else {	/* rv 100 */
-				/* rv100 might not need 64 pix alignment, who knows */
-				/* offsets are, hmm, weird */
-				tileoffset =
-				    ((pbox[i].y1 >> 4) * depthpixperline +
-				     pbox[i].x1) >> 6;
-				nrtilesx =
-				    ((pbox[i].x2 & ~63) -
-				     (pbox[i].x1 & ~63)) >> 4;
-				nrtilesy =
-				    (pbox[i].y2 >> 4) - (pbox[i].y1 >> 4);
-				for (j = 0; j <= nrtilesy; j++) {
-					BEGIN_RING(4);
-					OUT_RING(CP_PACKET3
-						 (RADEON_3D_CLEAR_ZMASK, 2));
-					OUT_RING(tileoffset * 128);
-					/* the number of tiles to clear */
-					OUT_RING(nrtilesx + 4);
-					/* clear mask : chooses the clearing pattern. */
-					OUT_RING(clearmask);
-					ADVANCE_RING();
-					tileoffset += depthpixperline >> 6;
-				}
-			}
-		}
-
-		/* TODO don't always clear all hi-level z tiles */
-		if ((dev_priv->flags & RADEON_HAS_HIERZ)
-		    && (dev_priv->microcode_version == UCODE_R200)
-		    && (flags & RADEON_USE_HIERZ))
-			/* r100 and cards without hierarchical z-buffer have no high-level z-buffer */
-			/* FIXME : the mask supposedly contains low-res z values. So can't set
-			   just to the max (0xff? or actually 0x3fff?), need to take z clear
-			   value into account? */
-		{
-			BEGIN_RING(4);
-			OUT_RING(CP_PACKET3(RADEON_3D_CLEAR_HIZ, 2));
-			OUT_RING(0x0);	/* First tile */
-			OUT_RING(0x3cc0);
-			OUT_RING((0xff << 22) | (0xff << 6) | 0x003f003f);
-			ADVANCE_RING();
-		}
-	}
-
-	/* We have to clear the depth and/or stencil buffers by
-	 * rendering a quad into just those buffers.  Thus, we have to
-	 * make sure the 3D engine is configured correctly.
-	 */
-	else if ((dev_priv->microcode_version == UCODE_R200) &&
-		(flags & (RADEON_DEPTH | RADEON_STENCIL))) {
-
-		int tempPP_CNTL;
-		int tempRE_CNTL;
-		int tempRB3D_CNTL;
-		int tempRB3D_ZSTENCILCNTL;
-		int tempRB3D_STENCILREFMASK;
-		int tempRB3D_PLANEMASK;
-		int tempSE_CNTL;
-		int tempSE_VTE_CNTL;
-		int tempSE_VTX_FMT_0;
-		int tempSE_VTX_FMT_1;
-		int tempSE_VAP_CNTL;
-		int tempRE_AUX_SCISSOR_CNTL;
-
-		tempPP_CNTL = 0;
-		tempRE_CNTL = 0;
-
-		tempRB3D_CNTL = depth_clear->rb3d_cntl;
-
-		tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
-		tempRB3D_STENCILREFMASK = 0x0;
-
-		tempSE_CNTL = depth_clear->se_cntl;
-
-		/* Disable TCL */
-
-		tempSE_VAP_CNTL = (	/* SE_VAP_CNTL__FORCE_W_TO_ONE_MASK |  */
-					  (0x9 <<
-					   SE_VAP_CNTL__VF_MAX_VTX_NUM__SHIFT));
-
-		tempRB3D_PLANEMASK = 0x0;
-
-		tempRE_AUX_SCISSOR_CNTL = 0x0;
-
-		tempSE_VTE_CNTL =
-		    SE_VTE_CNTL__VTX_XY_FMT_MASK | SE_VTE_CNTL__VTX_Z_FMT_MASK;
-
-		/* Vertex format (X, Y, Z, W) */
-		tempSE_VTX_FMT_0 =
-		    SE_VTX_FMT_0__VTX_Z0_PRESENT_MASK |
-		    SE_VTX_FMT_0__VTX_W0_PRESENT_MASK;
-		tempSE_VTX_FMT_1 = 0x0;
-
-		/*
-		 * Depth buffer specific enables
-		 */
-		if (flags & RADEON_DEPTH) {
-			/* Enable depth buffer */
-			tempRB3D_CNTL |= RADEON_Z_ENABLE;
-		} else {
-			/* Disable depth buffer */
-			tempRB3D_CNTL &= ~RADEON_Z_ENABLE;
-		}
-
-		/*
-		 * Stencil buffer specific enables
-		 */
-		if (flags & RADEON_STENCIL) {
-			tempRB3D_CNTL |= RADEON_STENCIL_ENABLE;
-			tempRB3D_STENCILREFMASK = clear->depth_mask;
-		} else {
-			tempRB3D_CNTL &= ~RADEON_STENCIL_ENABLE;
-			tempRB3D_STENCILREFMASK = 0x00000000;
-		}
-
-		if (flags & RADEON_USE_COMP_ZBUF) {
-			tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
-			    RADEON_Z_DECOMPRESSION_ENABLE;
-		}
-		if (flags & RADEON_USE_HIERZ) {
-			tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
-		}
-
-		BEGIN_RING(26);
-		RADEON_WAIT_UNTIL_2D_IDLE();
-
-		OUT_RING_REG(RADEON_PP_CNTL, tempPP_CNTL);
-		OUT_RING_REG(R200_RE_CNTL, tempRE_CNTL);
-		OUT_RING_REG(RADEON_RB3D_CNTL, tempRB3D_CNTL);
-		OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
-		OUT_RING_REG(RADEON_RB3D_STENCILREFMASK,
-			     tempRB3D_STENCILREFMASK);
-		OUT_RING_REG(RADEON_RB3D_PLANEMASK, tempRB3D_PLANEMASK);
-		OUT_RING_REG(RADEON_SE_CNTL, tempSE_CNTL);
-		OUT_RING_REG(R200_SE_VTE_CNTL, tempSE_VTE_CNTL);
-		OUT_RING_REG(R200_SE_VTX_FMT_0, tempSE_VTX_FMT_0);
-		OUT_RING_REG(R200_SE_VTX_FMT_1, tempSE_VTX_FMT_1);
-		OUT_RING_REG(R200_SE_VAP_CNTL, tempSE_VAP_CNTL);
-		OUT_RING_REG(R200_RE_AUX_SCISSOR_CNTL, tempRE_AUX_SCISSOR_CNTL);
-		ADVANCE_RING();
-
-		/* Make sure we restore the 3D state next time.
-		 */
-		sarea_priv->ctx_owner = 0;
-
-		for (i = 0; i < nbox; i++) {
-
-			/* Funny that this should be required --
-			 *  sets top-left?
-			 */
-			radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
-
-			BEGIN_RING(14);
-			OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 12));
-			OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
-				  RADEON_PRIM_WALK_RING |
-				  (3 << RADEON_NUM_VERTICES_SHIFT)));
-			OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x3f800000);
-			OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x3f800000);
-			OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x3f800000);
-			ADVANCE_RING();
-		}
-	} else if ((flags & (RADEON_DEPTH | RADEON_STENCIL))) {
-
-		int tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
-
-		rb3d_cntl = depth_clear->rb3d_cntl;
-
-		if (flags & RADEON_DEPTH) {
-			rb3d_cntl |= RADEON_Z_ENABLE;
-		} else {
-			rb3d_cntl &= ~RADEON_Z_ENABLE;
-		}
-
-		if (flags & RADEON_STENCIL) {
-			rb3d_cntl |= RADEON_STENCIL_ENABLE;
-			rb3d_stencilrefmask = clear->depth_mask;	/* misnamed field */
-		} else {
-			rb3d_cntl &= ~RADEON_STENCIL_ENABLE;
-			rb3d_stencilrefmask = 0x00000000;
-		}
-
-		if (flags & RADEON_USE_COMP_ZBUF) {
-			tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
-			    RADEON_Z_DECOMPRESSION_ENABLE;
-		}
-		if (flags & RADEON_USE_HIERZ) {
-			tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
-		}
-
-		BEGIN_RING(13);
-		RADEON_WAIT_UNTIL_2D_IDLE();
-
-		OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 1));
-		OUT_RING(0x00000000);
-		OUT_RING(rb3d_cntl);
-
-		OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
-		OUT_RING_REG(RADEON_RB3D_STENCILREFMASK, rb3d_stencilrefmask);
-		OUT_RING_REG(RADEON_RB3D_PLANEMASK, 0x00000000);
-		OUT_RING_REG(RADEON_SE_CNTL, depth_clear->se_cntl);
-		ADVANCE_RING();
-
-		/* Make sure we restore the 3D state next time.
-		 */
-		sarea_priv->ctx_owner = 0;
-
-		for (i = 0; i < nbox; i++) {
-
-			/* Funny that this should be required --
-			 *  sets top-left?
-			 */
-			radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
-
-			BEGIN_RING(15);
-
-			OUT_RING(CP_PACKET3(RADEON_3D_DRAW_IMMD, 13));
-			OUT_RING(RADEON_VTX_Z_PRESENT |
-				 RADEON_VTX_PKCOLOR_PRESENT);
-			OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
-				  RADEON_PRIM_WALK_RING |
-				  RADEON_MAOS_ENABLE |
-				  RADEON_VTX_FMT_RADEON_MODE |
-				  (3 << RADEON_NUM_VERTICES_SHIFT)));
-
-			OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x0);
-
-			OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x0);
-
-			OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
-			OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
-			OUT_RING(0x0);
-
-			ADVANCE_RING();
-		}
-	}
-
-	/* Increment the clear counter.  The client-side 3D driver must
-	 * wait on this value before performing the clear ioctl.  We
-	 * need this because the card's so damned fast...
-	 */
-	sarea_priv->last_clear++;
-
-	BEGIN_RING(4);
-
-	RADEON_CLEAR_AGE(sarea_priv->last_clear);
-	RADEON_WAIT_UNTIL_IDLE();
-
-	ADVANCE_RING();
-}
-
-static void radeon_cp_dispatch_swap(struct drm_device *dev, struct drm_master *master)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	int nbox = sarea_priv->nbox;
-	struct drm_clip_rect *pbox = sarea_priv->boxes;
-	int i;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	/* Do some trivial performance monitoring...
-	 */
-	if (dev_priv->do_boxes)
-		radeon_cp_performance_boxes(dev_priv, master_priv);
-
-	/* Wait for the 3D stream to idle before dispatching the bitblt.
-	 * This will prevent data corruption between the two streams.
-	 */
-	BEGIN_RING(2);
-
-	RADEON_WAIT_UNTIL_3D_IDLE();
-
-	ADVANCE_RING();
-
-	for (i = 0; i < nbox; i++) {
-		int x = pbox[i].x1;
-		int y = pbox[i].y1;
-		int w = pbox[i].x2 - x;
-		int h = pbox[i].y2 - y;
-
-		DRM_DEBUG("%d,%d-%d,%d\n", x, y, w, h);
-
-		BEGIN_RING(9);
-
-		OUT_RING(CP_PACKET0(RADEON_DP_GUI_MASTER_CNTL, 0));
-		OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
-			 RADEON_GMC_DST_PITCH_OFFSET_CNTL |
-			 RADEON_GMC_BRUSH_NONE |
-			 (dev_priv->color_fmt << 8) |
-			 RADEON_GMC_SRC_DATATYPE_COLOR |
-			 RADEON_ROP3_S |
-			 RADEON_DP_SRC_SOURCE_MEMORY |
-			 RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
-
-		/* Make this work even if front & back are flipped:
-		 */
-		OUT_RING(CP_PACKET0(RADEON_SRC_PITCH_OFFSET, 1));
-		if (sarea_priv->pfCurrentPage == 0) {
-			OUT_RING(dev_priv->back_pitch_offset);
-			OUT_RING(dev_priv->front_pitch_offset);
-		} else {
-			OUT_RING(dev_priv->front_pitch_offset);
-			OUT_RING(dev_priv->back_pitch_offset);
-		}
-
-		OUT_RING(CP_PACKET0(RADEON_SRC_X_Y, 2));
-		OUT_RING((x << 16) | y);
-		OUT_RING((x << 16) | y);
-		OUT_RING((w << 16) | h);
-
-		ADVANCE_RING();
-	}
-
-	/* Increment the frame counter.  The client-side 3D driver must
-	 * throttle the framerate by waiting for this value before
-	 * performing the swapbuffer ioctl.
-	 */
-	sarea_priv->last_frame++;
-
-	BEGIN_RING(4);
-
-	RADEON_FRAME_AGE(sarea_priv->last_frame);
-	RADEON_WAIT_UNTIL_2D_IDLE();
-
-	ADVANCE_RING();
-}
-
-void radeon_cp_dispatch_flip(struct drm_device *dev, struct drm_master *master)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	struct drm_sarea *sarea = (struct drm_sarea *)master_priv->sarea->handle;
-	int offset = (master_priv->sarea_priv->pfCurrentPage == 1)
-	    ? dev_priv->front_offset : dev_priv->back_offset;
-	RING_LOCALS;
-	DRM_DEBUG("pfCurrentPage=%d\n",
-		  master_priv->sarea_priv->pfCurrentPage);
-
-	/* Do some trivial performance monitoring...
-	 */
-	if (dev_priv->do_boxes) {
-		dev_priv->stats.boxes |= RADEON_BOX_FLIP;
-		radeon_cp_performance_boxes(dev_priv, master_priv);
-	}
-
-	/* Update the frame offsets for both CRTCs
-	 */
-	BEGIN_RING(6);
-
-	RADEON_WAIT_UNTIL_3D_IDLE();
-	OUT_RING_REG(RADEON_CRTC_OFFSET,
-		     ((sarea->frame.y * dev_priv->front_pitch +
-		       sarea->frame.x * (dev_priv->color_fmt - 2)) & ~7)
-		     + offset);
-	OUT_RING_REG(RADEON_CRTC2_OFFSET, master_priv->sarea_priv->crtc2_base
-		     + offset);
-
-	ADVANCE_RING();
-
-	/* Increment the frame counter.  The client-side 3D driver must
-	 * throttle the framerate by waiting for this value before
-	 * performing the swapbuffer ioctl.
-	 */
-	master_priv->sarea_priv->last_frame++;
-	master_priv->sarea_priv->pfCurrentPage =
-		1 - master_priv->sarea_priv->pfCurrentPage;
-
-	BEGIN_RING(2);
-
-	RADEON_FRAME_AGE(master_priv->sarea_priv->last_frame);
-
-	ADVANCE_RING();
-}
-
-static int bad_prim_vertex_nr(int primitive, int nr)
-{
-	switch (primitive & RADEON_PRIM_TYPE_MASK) {
-	case RADEON_PRIM_TYPE_NONE:
-	case RADEON_PRIM_TYPE_POINT:
-		return nr < 1;
-	case RADEON_PRIM_TYPE_LINE:
-		return (nr & 1) || nr == 0;
-	case RADEON_PRIM_TYPE_LINE_STRIP:
-		return nr < 2;
-	case RADEON_PRIM_TYPE_TRI_LIST:
-	case RADEON_PRIM_TYPE_3VRT_POINT_LIST:
-	case RADEON_PRIM_TYPE_3VRT_LINE_LIST:
-	case RADEON_PRIM_TYPE_RECT_LIST:
-		return nr % 3 || nr == 0;
-	case RADEON_PRIM_TYPE_TRI_FAN:
-	case RADEON_PRIM_TYPE_TRI_STRIP:
-		return nr < 3;
-	default:
-		return 1;
-	}
-}
-
-typedef struct {
-	unsigned int start;
-	unsigned int finish;
-	unsigned int prim;
-	unsigned int numverts;
-	unsigned int offset;
-	unsigned int vc_format;
-} drm_radeon_tcl_prim_t;
-
-static void radeon_cp_dispatch_vertex(struct drm_device * dev,
-				      struct drm_file *file_priv,
-				      struct drm_buf * buf,
-				      drm_radeon_tcl_prim_t * prim)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	int offset = dev_priv->gart_buffers_offset + buf->offset + prim->start;
-	int numverts = (int)prim->numverts;
-	int nbox = sarea_priv->nbox;
-	int i = 0;
-	RING_LOCALS;
-
-	DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d %d verts\n",
-		  prim->prim,
-		  prim->vc_format, prim->start, prim->finish, prim->numverts);
-
-	if (bad_prim_vertex_nr(prim->prim, prim->numverts)) {
-		DRM_ERROR("bad prim %x numverts %d\n",
-			  prim->prim, prim->numverts);
-		return;
-	}
-
-	do {
-		/* Emit the next cliprect */
-		if (i < nbox) {
-			radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
-		}
-
-		/* Emit the vertex buffer rendering commands */
-		BEGIN_RING(5);
-
-		OUT_RING(CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, 3));
-		OUT_RING(offset);
-		OUT_RING(numverts);
-		OUT_RING(prim->vc_format);
-		OUT_RING(prim->prim | RADEON_PRIM_WALK_LIST |
-			 RADEON_COLOR_ORDER_RGBA |
-			 RADEON_VTX_FMT_RADEON_MODE |
-			 (numverts << RADEON_NUM_VERTICES_SHIFT));
-
-		ADVANCE_RING();
-
-		i++;
-	} while (i < nbox);
-}
-
-void radeon_cp_discard_buffer(struct drm_device *dev, struct drm_master *master, struct drm_buf *buf)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
-	RING_LOCALS;
-
-	buf_priv->age = ++master_priv->sarea_priv->last_dispatch;
-
-	/* Emit the vertex buffer age */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
-		BEGIN_RING(3);
-		R600_DISPATCH_AGE(buf_priv->age);
-		ADVANCE_RING();
-	} else {
-		BEGIN_RING(2);
-		RADEON_DISPATCH_AGE(buf_priv->age);
-		ADVANCE_RING();
-	}
-
-	buf->pending = 1;
-	buf->used = 0;
-}
-
-static void radeon_cp_dispatch_indirect(struct drm_device * dev,
-					struct drm_buf * buf, int start, int end)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	RING_LOCALS;
-	DRM_DEBUG("buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
-
-	if (start != end) {
-		int offset = (dev_priv->gart_buffers_offset
-			      + buf->offset + start);
-		int dwords = (end - start + 3) / sizeof(u32);
-
-		/* Indirect buffer data must be an even number of
-		 * dwords, so if we've been given an odd number we must
-		 * pad the data with a Type-2 CP packet.
-		 */
-		if (dwords & 1) {
-			u32 *data = (u32 *)
-			    ((char *)dev->agp_buffer_map->handle
-			     + buf->offset + start);
-			data[dwords++] = RADEON_CP_PACKET2;
-		}
-
-		/* Fire off the indirect buffer */
-		BEGIN_RING(3);
-
-		OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
-		OUT_RING(offset);
-		OUT_RING(dwords);
-
-		ADVANCE_RING();
-	}
-}
-
-static void radeon_cp_dispatch_indices(struct drm_device *dev,
-				       struct drm_master *master,
-				       struct drm_buf * elt_buf,
-				       drm_radeon_tcl_prim_t * prim)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	int offset = dev_priv->gart_buffers_offset + prim->offset;
-	u32 *data;
-	int dwords;
-	int i = 0;
-	int start = prim->start + RADEON_INDEX_PRIM_OFFSET;
-	int count = (prim->finish - start) / sizeof(u16);
-	int nbox = sarea_priv->nbox;
-
-	DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d offset: %x nr %d\n",
-		  prim->prim,
-		  prim->vc_format,
-		  prim->start, prim->finish, prim->offset, prim->numverts);
-
-	if (bad_prim_vertex_nr(prim->prim, count)) {
-		DRM_ERROR("bad prim %x count %d\n", prim->prim, count);
-		return;
-	}
-
-	if (start >= prim->finish || (prim->start & 0x7)) {
-		DRM_ERROR("buffer prim %d\n", prim->prim);
-		return;
-	}
-
-	dwords = (prim->finish - prim->start + 3) / sizeof(u32);
-
-	data = (u32 *) ((char *)dev->agp_buffer_map->handle +
-			elt_buf->offset + prim->start);
-
-	data[0] = CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, dwords - 2);
-	data[1] = offset;
-	data[2] = prim->numverts;
-	data[3] = prim->vc_format;
-	data[4] = (prim->prim |
-		   RADEON_PRIM_WALK_IND |
-		   RADEON_COLOR_ORDER_RGBA |
-		   RADEON_VTX_FMT_RADEON_MODE |
-		   (count << RADEON_NUM_VERTICES_SHIFT));
-
-	do {
-		if (i < nbox)
-			radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
-
-		radeon_cp_dispatch_indirect(dev, elt_buf,
-					    prim->start, prim->finish);
-
-		i++;
-	} while (i < nbox);
-
-}
-
-#define RADEON_MAX_TEXTURE_SIZE RADEON_BUFFER_SIZE
-
-static int radeon_cp_dispatch_texture(struct drm_device * dev,
-				      struct drm_file *file_priv,
-				      drm_radeon_texture_t * tex,
-				      drm_radeon_tex_image_t * image)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_buf *buf;
-	u32 format;
-	u32 *buffer;
-	const u8 __user *data;
-	unsigned int size, dwords, tex_width, blit_width, spitch;
-	u32 height;
-	int i;
-	u32 texpitch, microtile;
-	u32 offset, byte_offset;
-	RING_LOCALS;
-
-	if (radeon_check_and_fixup_offset(dev_priv, file_priv, &tex->offset)) {
-		DRM_ERROR("Invalid destination offset\n");
-		return -EINVAL;
-	}
-
-	dev_priv->stats.boxes |= RADEON_BOX_TEXTURE_LOAD;
-
-	/* Flush the pixel cache.  This ensures no pixel data gets mixed
-	 * up with the texture data from the host data blit, otherwise
-	 * part of the texture image may be corrupted.
-	 */
-	BEGIN_RING(4);
-	RADEON_FLUSH_CACHE();
-	RADEON_WAIT_UNTIL_IDLE();
-	ADVANCE_RING();
-
-	/* The compiler won't optimize away a division by a variable,
-	 * even if the only legal values are powers of two.  Thus, we'll
-	 * use a shift instead.
-	 */
-	switch (tex->format) {
-	case RADEON_TXFORMAT_ARGB8888:
-	case RADEON_TXFORMAT_RGBA8888:
-		format = RADEON_COLOR_FORMAT_ARGB8888;
-		tex_width = tex->width * 4;
-		blit_width = image->width * 4;
-		break;
-	case RADEON_TXFORMAT_AI88:
-	case RADEON_TXFORMAT_ARGB1555:
-	case RADEON_TXFORMAT_RGB565:
-	case RADEON_TXFORMAT_ARGB4444:
-	case RADEON_TXFORMAT_VYUY422:
-	case RADEON_TXFORMAT_YVYU422:
-		format = RADEON_COLOR_FORMAT_RGB565;
-		tex_width = tex->width * 2;
-		blit_width = image->width * 2;
-		break;
-	case RADEON_TXFORMAT_I8:
-	case RADEON_TXFORMAT_RGB332:
-		format = RADEON_COLOR_FORMAT_CI8;
-		tex_width = tex->width * 1;
-		blit_width = image->width * 1;
-		break;
-	default:
-		DRM_ERROR("invalid texture format %d\n", tex->format);
-		return -EINVAL;
-	}
-	spitch = blit_width >> 6;
-	if (spitch == 0 && image->height > 1)
-		return -EINVAL;
-
-	texpitch = tex->pitch;
-	if ((texpitch << 22) & RADEON_DST_TILE_MICRO) {
-		microtile = 1;
-		if (tex_width < 64) {
-			texpitch &= ~(RADEON_DST_TILE_MICRO >> 22);
-			/* we got tiled coordinates, untile them */
-			image->x *= 2;
-		}
-	} else
-		microtile = 0;
-
-	/* this might fail for zero-sized uploads - are those illegal? */
-	if (!radeon_check_offset(dev_priv, tex->offset + image->height *
-				blit_width - 1)) {
-		DRM_ERROR("Invalid final destination offset\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG("tex=%dx%d blit=%d\n", tex_width, tex->height, blit_width);
-
-	do {
-		DRM_DEBUG("tex: ofs=0x%x p=%d f=%d x=%hd y=%hd w=%hd h=%hd\n",
-			  tex->offset >> 10, tex->pitch, tex->format,
-			  image->x, image->y, image->width, image->height);
-
-		/* Make a copy of some parameters in case we have to
-		 * update them for a multi-pass texture blit.
-		 */
-		height = image->height;
-		data = (const u8 __user *)image->data;
-
-		size = height * blit_width;
-
-		if (size > RADEON_MAX_TEXTURE_SIZE) {
-			height = RADEON_MAX_TEXTURE_SIZE / blit_width;
-			size = height * blit_width;
-		} else if (size < 4 && size > 0) {
-			size = 4;
-		} else if (size == 0) {
-			return 0;
-		}
-
-		buf = radeon_freelist_get(dev);
-		if (0 && !buf) {
-			radeon_do_cp_idle(dev_priv);
-			buf = radeon_freelist_get(dev);
-		}
-		if (!buf) {
-			DRM_DEBUG("EAGAIN\n");
-			if (copy_to_user(tex->image, image, sizeof(*image)))
-				return -EFAULT;
-			return -EAGAIN;
-		}
-
-		/* Dispatch the indirect buffer.
-		 */
-		buffer =
-		    (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
-		dwords = size / 4;
-
-#define RADEON_COPY_MT(_buf, _data, _width) \
-	do { \
-		if (copy_from_user(_buf, _data, (_width))) {\
-			DRM_ERROR("EFAULT on pad, %d bytes\n", (_width)); \
-			return -EFAULT; \
-		} \
-	} while(0)
-
-		if (microtile) {
-			/* texture micro tiling in use, minimum texture width is thus 16 bytes.
-			   however, we cannot use blitter directly for texture width < 64 bytes,
-			   since minimum tex pitch is 64 bytes and we need this to match
-			   the texture width, otherwise the blitter will tile it wrong.
-			   Thus, tiling manually in this case. Additionally, need to special
-			   case tex height = 1, since our actual image will have height 2
-			   and we need to ensure we don't read beyond the texture size
-			   from user space. */
-			if (tex->height == 1) {
-				if (tex_width >= 64 || tex_width <= 16) {
-					RADEON_COPY_MT(buffer, data,
-						(int)(tex_width * sizeof(u32)));
-				} else if (tex_width == 32) {
-					RADEON_COPY_MT(buffer, data, 16);
-					RADEON_COPY_MT(buffer + 8,
-						       data + 16, 16);
-				}
-			} else if (tex_width >= 64 || tex_width == 16) {
-				RADEON_COPY_MT(buffer, data,
-					       (int)(dwords * sizeof(u32)));
-			} else if (tex_width < 16) {
-				for (i = 0; i < tex->height; i++) {
-					RADEON_COPY_MT(buffer, data, tex_width);
-					buffer += 4;
-					data += tex_width;
-				}
-			} else if (tex_width == 32) {
-				/* TODO: make sure this works when not fitting in one buffer
-				   (i.e. 32bytes x 2048...) */
-				for (i = 0; i < tex->height; i += 2) {
-					RADEON_COPY_MT(buffer, data, 16);
-					data += 16;
-					RADEON_COPY_MT(buffer + 8, data, 16);
-					data += 16;
-					RADEON_COPY_MT(buffer + 4, data, 16);
-					data += 16;
-					RADEON_COPY_MT(buffer + 12, data, 16);
-					data += 16;
-					buffer += 16;
-				}
-			}
-		} else {
-			if (tex_width >= 32) {
-				/* Texture image width is larger than the minimum, so we
-				 * can upload it directly.
-				 */
-				RADEON_COPY_MT(buffer, data,
-					       (int)(dwords * sizeof(u32)));
-			} else {
-				/* Texture image width is less than the minimum, so we
-				 * need to pad out each image scanline to the minimum
-				 * width.
-				 */
-				for (i = 0; i < tex->height; i++) {
-					RADEON_COPY_MT(buffer, data, tex_width);
-					buffer += 8;
-					data += tex_width;
-				}
-			}
-		}
-
-#undef RADEON_COPY_MT
-		byte_offset = (image->y & ~2047) * blit_width;
-		buf->file_priv = file_priv;
-		buf->used = size;
-		offset = dev_priv->gart_buffers_offset + buf->offset;
-		BEGIN_RING(9);
-		OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 5));
-		OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
-			 RADEON_GMC_DST_PITCH_OFFSET_CNTL |
-			 RADEON_GMC_BRUSH_NONE |
-			 (format << 8) |
-			 RADEON_GMC_SRC_DATATYPE_COLOR |
-			 RADEON_ROP3_S |
-			 RADEON_DP_SRC_SOURCE_MEMORY |
-			 RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
-		OUT_RING((spitch << 22) | (offset >> 10));
-		OUT_RING((texpitch << 22) | ((tex->offset >> 10) + (byte_offset >> 10)));
-		OUT_RING(0);
-		OUT_RING((image->x << 16) | (image->y % 2048));
-		OUT_RING((image->width << 16) | height);
-		RADEON_WAIT_UNTIL_2D_IDLE();
-		ADVANCE_RING();
-		COMMIT_RING();
-
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-
-		/* Update the input parameters for next time */
-		image->y += height;
-		image->height -= height;
-		image->data = (const u8 __user *)image->data + size;
-	} while (image->height > 0);
-
-	/* Flush the pixel cache after the blit completes.  This ensures
-	 * the texture data is written out to memory before rendering
-	 * continues.
-	 */
-	BEGIN_RING(4);
-	RADEON_FLUSH_CACHE();
-	RADEON_WAIT_UNTIL_2D_IDLE();
-	ADVANCE_RING();
-	COMMIT_RING();
-
-	return 0;
-}
-
-static void radeon_cp_dispatch_stipple(struct drm_device * dev, u32 * stipple)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	int i;
-	RING_LOCALS;
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(35);
-
-	OUT_RING(CP_PACKET0(RADEON_RE_STIPPLE_ADDR, 0));
-	OUT_RING(0x00000000);
-
-	OUT_RING(CP_PACKET0_TABLE(RADEON_RE_STIPPLE_DATA, 31));
-	for (i = 0; i < 32; i++) {
-		OUT_RING(stipple[i]);
-	}
-
-	ADVANCE_RING();
-}
-
-static void radeon_apply_surface_regs(int surf_index,
-				      drm_radeon_private_t *dev_priv)
-{
-	if (!dev_priv->mmio)
-		return;
-
-	radeon_do_cp_idle(dev_priv);
-
-	RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * surf_index,
-		     dev_priv->surfaces[surf_index].flags);
-	RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * surf_index,
-		     dev_priv->surfaces[surf_index].lower);
-	RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * surf_index,
-		     dev_priv->surfaces[surf_index].upper);
-}
-
-/* Allocates a virtual surface
- * doesn't always allocate a real surface, will stretch an existing
- * surface when possible.
- *
- * Note that refcount can be at most 2, since during a free refcount=3
- * might mean we have to allocate a new surface which might not always
- * be available.
- * For example : we allocate three contiguous surfaces ABC. If B is
- * freed, we suddenly need two surfaces to store A and C, which might
- * not always be available.
- */
-static int alloc_surface(drm_radeon_surface_alloc_t *new,
-			 drm_radeon_private_t *dev_priv,
-			 struct drm_file *file_priv)
-{
-	struct radeon_virt_surface *s;
-	int i;
-	int virt_surface_index;
-	uint32_t new_upper, new_lower;
-
-	new_lower = new->address;
-	new_upper = new_lower + new->size - 1;
-
-	/* sanity check */
-	if ((new_lower >= new_upper) || (new->flags == 0) || (new->size == 0) ||
-	    ((new_upper & RADEON_SURF_ADDRESS_FIXED_MASK) !=
-	     RADEON_SURF_ADDRESS_FIXED_MASK)
-	    || ((new_lower & RADEON_SURF_ADDRESS_FIXED_MASK) != 0))
-		return -1;
-
-	/* make sure there is no overlap with existing surfaces */
-	for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-		if ((dev_priv->surfaces[i].refcount != 0) &&
-		    (((new_lower >= dev_priv->surfaces[i].lower) &&
-		      (new_lower < dev_priv->surfaces[i].upper)) ||
-		     ((new_lower < dev_priv->surfaces[i].lower) &&
-		      (new_upper > dev_priv->surfaces[i].lower)))) {
-			return -1;
-		}
-	}
-
-	/* find a virtual surface */
-	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++)
-		if (dev_priv->virt_surfaces[i].file_priv == NULL)
-			break;
-	if (i == 2 * RADEON_MAX_SURFACES) {
-		return -1;
-	}
-	virt_surface_index = i;
-
-	/* try to reuse an existing surface */
-	for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-		/* extend before */
-		if ((dev_priv->surfaces[i].refcount == 1) &&
-		    (new->flags == dev_priv->surfaces[i].flags) &&
-		    (new_upper + 1 == dev_priv->surfaces[i].lower)) {
-			s = &(dev_priv->virt_surfaces[virt_surface_index]);
-			s->surface_index = i;
-			s->lower = new_lower;
-			s->upper = new_upper;
-			s->flags = new->flags;
-			s->file_priv = file_priv;
-			dev_priv->surfaces[i].refcount++;
-			dev_priv->surfaces[i].lower = s->lower;
-			radeon_apply_surface_regs(s->surface_index, dev_priv);
-			return virt_surface_index;
-		}
-
-		/* extend after */
-		if ((dev_priv->surfaces[i].refcount == 1) &&
-		    (new->flags == dev_priv->surfaces[i].flags) &&
-		    (new_lower == dev_priv->surfaces[i].upper + 1)) {
-			s = &(dev_priv->virt_surfaces[virt_surface_index]);
-			s->surface_index = i;
-			s->lower = new_lower;
-			s->upper = new_upper;
-			s->flags = new->flags;
-			s->file_priv = file_priv;
-			dev_priv->surfaces[i].refcount++;
-			dev_priv->surfaces[i].upper = s->upper;
-			radeon_apply_surface_regs(s->surface_index, dev_priv);
-			return virt_surface_index;
-		}
-	}
-
-	/* okay, we need a new one */
-	for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-		if (dev_priv->surfaces[i].refcount == 0) {
-			s = &(dev_priv->virt_surfaces[virt_surface_index]);
-			s->surface_index = i;
-			s->lower = new_lower;
-			s->upper = new_upper;
-			s->flags = new->flags;
-			s->file_priv = file_priv;
-			dev_priv->surfaces[i].refcount = 1;
-			dev_priv->surfaces[i].lower = s->lower;
-			dev_priv->surfaces[i].upper = s->upper;
-			dev_priv->surfaces[i].flags = s->flags;
-			radeon_apply_surface_regs(s->surface_index, dev_priv);
-			return virt_surface_index;
-		}
-	}
-
-	/* we didn't find anything */
-	return -1;
-}
-
-static int free_surface(struct drm_file *file_priv,
-			drm_radeon_private_t * dev_priv,
-			int lower)
-{
-	struct radeon_virt_surface *s;
-	int i;
-	/* find the virtual surface */
-	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
-		s = &(dev_priv->virt_surfaces[i]);
-		if (s->file_priv) {
-			if ((lower == s->lower) && (file_priv == s->file_priv))
-			{
-				if (dev_priv->surfaces[s->surface_index].
-				    lower == s->lower)
-					dev_priv->surfaces[s->surface_index].
-					    lower = s->upper;
-
-				if (dev_priv->surfaces[s->surface_index].
-				    upper == s->upper)
-					dev_priv->surfaces[s->surface_index].
-					    upper = s->lower;
-
-				dev_priv->surfaces[s->surface_index].refcount--;
-				if (dev_priv->surfaces[s->surface_index].
-				    refcount == 0)
-					dev_priv->surfaces[s->surface_index].
-					    flags = 0;
-				s->file_priv = NULL;
-				radeon_apply_surface_regs(s->surface_index,
-							  dev_priv);
-				return 0;
-			}
-		}
-	}
-	return 1;
-}
-
-static void radeon_surfaces_release(struct drm_file *file_priv,
-				    drm_radeon_private_t * dev_priv)
-{
-	int i;
-	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
-		if (dev_priv->virt_surfaces[i].file_priv == file_priv)
-			free_surface(file_priv, dev_priv,
-				     dev_priv->virt_surfaces[i].lower);
-	}
-}
-
-/* ================================================================
- * IOCTL functions
- */
-static int radeon_surface_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_surface_alloc_t *alloc = data;
-
-	if (alloc_surface(alloc, dev_priv, file_priv) == -1)
-		return -EINVAL;
-	else
-		return 0;
-}
-
-static int radeon_surface_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_surface_free_t *memfree = data;
-
-	if (free_surface(file_priv, dev_priv, memfree->address))
-		return -EINVAL;
-	else
-		return 0;
-}
-
-static int radeon_cp_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-	drm_radeon_clear_t *clear = data;
-	drm_radeon_clear_rect_t depth_boxes[RADEON_NR_SAREA_CLIPRECTS];
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-
-	if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
-		sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
-
-	if (copy_from_user(&depth_boxes, clear->depth_boxes,
-			       sarea_priv->nbox * sizeof(depth_boxes[0])))
-		return -EFAULT;
-
-	radeon_cp_dispatch_clear(dev, file_priv->master, clear, depth_boxes);
-
-	COMMIT_RING();
-	return 0;
-}
-
-/* Not sure why this isn't set all the time:
- */
-static int radeon_do_init_pageflip(struct drm_device *dev, struct drm_master *master)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = master->driver_priv;
-	RING_LOCALS;
-
-	DRM_DEBUG("\n");
-
-	BEGIN_RING(6);
-	RADEON_WAIT_UNTIL_3D_IDLE();
-	OUT_RING(CP_PACKET0(RADEON_CRTC_OFFSET_CNTL, 0));
-	OUT_RING(RADEON_READ(RADEON_CRTC_OFFSET_CNTL) |
-		 RADEON_CRTC_OFFSET_FLIP_CNTL);
-	OUT_RING(CP_PACKET0(RADEON_CRTC2_OFFSET_CNTL, 0));
-	OUT_RING(RADEON_READ(RADEON_CRTC2_OFFSET_CNTL) |
-		 RADEON_CRTC_OFFSET_FLIP_CNTL);
-	ADVANCE_RING();
-
-	dev_priv->page_flipping = 1;
-
-	if (master_priv->sarea_priv->pfCurrentPage != 1)
-		master_priv->sarea_priv->pfCurrentPage = 0;
-
-	return 0;
-}
-
-/* Swapping and flipping are different operations, need different ioctls.
- * They can & should be intermixed to support multiple 3d windows.
- */
-static int radeon_cp_flip(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-
-	if (!dev_priv->page_flipping)
-		radeon_do_init_pageflip(dev, file_priv->master);
-
-	radeon_cp_dispatch_flip(dev, file_priv->master);
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_swap(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
-
-	DRM_DEBUG("\n");
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-
-	if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
-		sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_cp_dispatch_swap(dev, file_priv);
-	else
-		radeon_cp_dispatch_swap(dev, file_priv->master);
-	sarea_priv->ctx_owner = 0;
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_vertex(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf;
-	drm_radeon_vertex_t *vertex = data;
-	drm_radeon_tcl_prim_t prim;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	sarea_priv = master_priv->sarea_priv;
-
-	DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
-		  DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard);
-
-	if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
-		DRM_ERROR("buffer index %d (of %d max)\n",
-			  vertex->idx, dma->buf_count - 1);
-		return -EINVAL;
-	}
-	if (vertex->prim < 0 || vertex->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
-		DRM_ERROR("buffer prim %d\n", vertex->prim);
-		return -EINVAL;
-	}
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	buf = dma->buflist[vertex->idx];
-
-	if (buf->file_priv != file_priv) {
-		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->file_priv);
-		return -EINVAL;
-	}
-	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", vertex->idx);
-		return -EINVAL;
-	}
-
-	/* Build up a prim_t record:
-	 */
-	if (vertex->count) {
-		buf->used = vertex->count;	/* not used? */
-
-		if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
-			if (radeon_emit_state(dev_priv, file_priv,
-					      &sarea_priv->context_state,
-					      sarea_priv->tex_state,
-					      sarea_priv->dirty)) {
-				DRM_ERROR("radeon_emit_state failed\n");
-				return -EINVAL;
-			}
-
-			sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
-					       RADEON_UPLOAD_TEX1IMAGES |
-					       RADEON_UPLOAD_TEX2IMAGES |
-					       RADEON_REQUIRE_QUIESCENCE);
-		}
-
-		prim.start = 0;
-		prim.finish = vertex->count;	/* unused */
-		prim.prim = vertex->prim;
-		prim.numverts = vertex->count;
-		prim.vc_format = sarea_priv->vc_format;
-
-		radeon_cp_dispatch_vertex(dev, file_priv, buf, &prim);
-	}
-
-	if (vertex->discard) {
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-	}
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_indices(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf;
-	drm_radeon_indices_t *elts = data;
-	drm_radeon_tcl_prim_t prim;
-	int count;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	sarea_priv = master_priv->sarea_priv;
-
-	DRM_DEBUG("pid=%d index=%d start=%d end=%d discard=%d\n",
-		  DRM_CURRENTPID, elts->idx, elts->start, elts->end,
-		  elts->discard);
-
-	if (elts->idx < 0 || elts->idx >= dma->buf_count) {
-		DRM_ERROR("buffer index %d (of %d max)\n",
-			  elts->idx, dma->buf_count - 1);
-		return -EINVAL;
-	}
-	if (elts->prim < 0 || elts->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
-		DRM_ERROR("buffer prim %d\n", elts->prim);
-		return -EINVAL;
-	}
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	buf = dma->buflist[elts->idx];
-
-	if (buf->file_priv != file_priv) {
-		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->file_priv);
-		return -EINVAL;
-	}
-	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", elts->idx);
-		return -EINVAL;
-	}
-
-	count = (elts->end - elts->start) / sizeof(u16);
-	elts->start -= RADEON_INDEX_PRIM_OFFSET;
-
-	if (elts->start & 0x7) {
-		DRM_ERROR("misaligned buffer 0x%x\n", elts->start);
-		return -EINVAL;
-	}
-	if (elts->start < buf->used) {
-		DRM_ERROR("no header 0x%x - 0x%x\n", elts->start, buf->used);
-		return -EINVAL;
-	}
-
-	buf->used = elts->end;
-
-	if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
-		if (radeon_emit_state(dev_priv, file_priv,
-				      &sarea_priv->context_state,
-				      sarea_priv->tex_state,
-				      sarea_priv->dirty)) {
-			DRM_ERROR("radeon_emit_state failed\n");
-			return -EINVAL;
-		}
-
-		sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
-				       RADEON_UPLOAD_TEX1IMAGES |
-				       RADEON_UPLOAD_TEX2IMAGES |
-				       RADEON_REQUIRE_QUIESCENCE);
-	}
-
-	/* Build up a prim_t record:
-	 */
-	prim.start = elts->start;
-	prim.finish = elts->end;
-	prim.prim = elts->prim;
-	prim.offset = 0;	/* offset from start of dma buffers */
-	prim.numverts = RADEON_MAX_VB_VERTS;	/* duh */
-	prim.vc_format = sarea_priv->vc_format;
-
-	radeon_cp_dispatch_indices(dev, file_priv->master, buf, &prim);
-	if (elts->discard) {
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-	}
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_texture(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_texture_t *tex = data;
-	drm_radeon_tex_image_t image;
-	int ret;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (tex->image == NULL) {
-		DRM_ERROR("null texture image!\n");
-		return -EINVAL;
-	}
-
-	if (copy_from_user(&image,
-			       (drm_radeon_tex_image_t __user *) tex->image,
-			       sizeof(image)))
-		return -EFAULT;
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		ret = r600_cp_dispatch_texture(dev, file_priv, tex, &image);
-	else
-		ret = radeon_cp_dispatch_texture(dev, file_priv, tex, &image);
-
-	return ret;
-}
-
-static int radeon_cp_stipple(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_stipple_t *stipple = data;
-	u32 mask[32];
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	if (copy_from_user(&mask, stipple->mask, 32 * sizeof(u32)))
-		return -EFAULT;
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-
-	radeon_cp_dispatch_stipple(dev, mask);
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_indirect(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf;
-	drm_radeon_indirect_t *indirect = data;
-	RING_LOCALS;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	DRM_DEBUG("idx=%d s=%d e=%d d=%d\n",
-		  indirect->idx, indirect->start, indirect->end,
-		  indirect->discard);
-
-	if (indirect->idx < 0 || indirect->idx >= dma->buf_count) {
-		DRM_ERROR("buffer index %d (of %d max)\n",
-			  indirect->idx, dma->buf_count - 1);
-		return -EINVAL;
-	}
-
-	buf = dma->buflist[indirect->idx];
-
-	if (buf->file_priv != file_priv) {
-		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->file_priv);
-		return -EINVAL;
-	}
-	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", indirect->idx);
-		return -EINVAL;
-	}
-
-	if (indirect->start < buf->used) {
-		DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
-			  indirect->start, buf->used);
-		return -EINVAL;
-	}
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	buf->used = indirect->end;
-
-	/* Dispatch the indirect buffer full of commands from the
-	 * X server.  This is insecure and is thus only available to
-	 * privileged clients.
-	 */
-	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-		r600_cp_dispatch_indirect(dev, buf, indirect->start, indirect->end);
-	else {
-		/* Wait for the 3D stream to idle before the indirect buffer
-		 * containing 2D acceleration commands is processed.
-		 */
-		BEGIN_RING(2);
-		RADEON_WAIT_UNTIL_3D_IDLE();
-		ADVANCE_RING();
-		radeon_cp_dispatch_indirect(dev, buf, indirect->start, indirect->end);
-	}
-
-	if (indirect->discard) {
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-	}
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_cp_vertex2(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_sarea_t *sarea_priv;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf;
-	drm_radeon_vertex2_t *vertex = data;
-	int i;
-	unsigned char laststate;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	sarea_priv = master_priv->sarea_priv;
-
-	DRM_DEBUG("pid=%d index=%d discard=%d\n",
-		  DRM_CURRENTPID, vertex->idx, vertex->discard);
-
-	if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
-		DRM_ERROR("buffer index %d (of %d max)\n",
-			  vertex->idx, dma->buf_count - 1);
-		return -EINVAL;
-	}
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	buf = dma->buflist[vertex->idx];
-
-	if (buf->file_priv != file_priv) {
-		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->file_priv);
-		return -EINVAL;
-	}
-
-	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", vertex->idx);
-		return -EINVAL;
-	}
-
-	if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
-		return -EINVAL;
-
-	for (laststate = 0xff, i = 0; i < vertex->nr_prims; i++) {
-		drm_radeon_prim_t prim;
-		drm_radeon_tcl_prim_t tclprim;
-
-		if (copy_from_user(&prim, &vertex->prim[i], sizeof(prim)))
-			return -EFAULT;
-
-		if (prim.stateidx != laststate) {
-			drm_radeon_state_t state;
-
-			if (copy_from_user(&state,
-					       &vertex->state[prim.stateidx],
-					       sizeof(state)))
-				return -EFAULT;
-
-			if (radeon_emit_state2(dev_priv, file_priv, &state)) {
-				DRM_ERROR("radeon_emit_state2 failed\n");
-				return -EINVAL;
-			}
-
-			laststate = prim.stateidx;
-		}
-
-		tclprim.start = prim.start;
-		tclprim.finish = prim.finish;
-		tclprim.prim = prim.prim;
-		tclprim.vc_format = prim.vc_format;
-
-		if (prim.prim & RADEON_PRIM_WALK_IND) {
-			tclprim.offset = prim.numverts * 64;
-			tclprim.numverts = RADEON_MAX_VB_VERTS;	/* duh */
-
-			radeon_cp_dispatch_indices(dev, file_priv->master, buf, &tclprim);
-		} else {
-			tclprim.numverts = prim.numverts;
-			tclprim.offset = 0;	/* not used */
-
-			radeon_cp_dispatch_vertex(dev, file_priv, buf, &tclprim);
-		}
-
-		if (sarea_priv->nbox == 1)
-			sarea_priv->nbox = 0;
-	}
-
-	if (vertex->discard) {
-		radeon_cp_discard_buffer(dev, file_priv->master, buf);
-	}
-
-	COMMIT_RING();
-	return 0;
-}
-
-static int radeon_emit_packets(drm_radeon_private_t * dev_priv,
-			       struct drm_file *file_priv,
-			       drm_radeon_cmd_header_t header,
-			       drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	int id = (int)header.packet.packet_id;
-	int sz, reg;
-	RING_LOCALS;
-
-	if (id >= RADEON_MAX_STATE_PACKETS)
-		return -EINVAL;
-
-	sz = packet[id].len;
-	reg = packet[id].start;
-
-	if (sz * sizeof(u32) > drm_buffer_unprocessed(cmdbuf->buffer)) {
-		DRM_ERROR("Packet size provided larger than data provided\n");
-		return -EINVAL;
-	}
-
-	if (radeon_check_and_fixup_packets(dev_priv, file_priv, id,
-				cmdbuf->buffer)) {
-		DRM_ERROR("Packet verification failed\n");
-		return -EINVAL;
-	}
-
-	BEGIN_RING(sz + 1);
-	OUT_RING(CP_PACKET0(reg, (sz - 1)));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static __inline__ int radeon_emit_scalars(drm_radeon_private_t *dev_priv,
-					  drm_radeon_cmd_header_t header,
-					  drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	int sz = header.scalars.count;
-	int start = header.scalars.offset;
-	int stride = header.scalars.stride;
-	RING_LOCALS;
-
-	BEGIN_RING(3 + sz);
-	OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
-	OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
-	OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-	return 0;
-}
-
-/* God this is ugly
- */
-static __inline__ int radeon_emit_scalars2(drm_radeon_private_t *dev_priv,
-					   drm_radeon_cmd_header_t header,
-					   drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	int sz = header.scalars.count;
-	int start = ((unsigned int)header.scalars.offset) + 0x100;
-	int stride = header.scalars.stride;
-	RING_LOCALS;
-
-	BEGIN_RING(3 + sz);
-	OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
-	OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
-	OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-	return 0;
-}
-
-static __inline__ int radeon_emit_vectors(drm_radeon_private_t *dev_priv,
-					  drm_radeon_cmd_header_t header,
-					  drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	int sz = header.vectors.count;
-	int start = header.vectors.offset;
-	int stride = header.vectors.stride;
-	RING_LOCALS;
-
-	BEGIN_RING(5 + sz);
-	OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
-	OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
-	OUT_RING(start | (stride << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
-	OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static __inline__ int radeon_emit_veclinear(drm_radeon_private_t *dev_priv,
-					  drm_radeon_cmd_header_t header,
-					  drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	int sz = header.veclinear.count * 4;
-	int start = header.veclinear.addr_lo | (header.veclinear.addr_hi << 8);
-	RING_LOCALS;
-
-        if (!sz)
-                return 0;
-	if (sz * 4 > drm_buffer_unprocessed(cmdbuf->buffer))
-                return -EINVAL;
-
-	BEGIN_RING(5 + sz);
-	OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
-	OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
-	OUT_RING(start | (1 << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
-	OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static int radeon_emit_packet3(struct drm_device * dev,
-			       struct drm_file *file_priv,
-			       drm_radeon_kcmd_buffer_t *cmdbuf)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	unsigned int cmdsz;
-	int ret;
-	RING_LOCALS;
-
-	DRM_DEBUG("\n");
-
-	if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
-						  cmdbuf, &cmdsz))) {
-		DRM_ERROR("Packet verification failed\n");
-		return ret;
-	}
-
-	BEGIN_RING(cmdsz);
-	OUT_RING_DRM_BUFFER(cmdbuf->buffer, cmdsz);
-	ADVANCE_RING();
-
-	return 0;
-}
-
-static int radeon_emit_packet3_cliprect(struct drm_device *dev,
-					struct drm_file *file_priv,
-					drm_radeon_kcmd_buffer_t *cmdbuf,
-					int orig_nbox)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_clip_rect box;
-	unsigned int cmdsz;
-	int ret;
-	struct drm_clip_rect __user *boxes = cmdbuf->boxes;
-	int i = 0;
-	RING_LOCALS;
-
-	DRM_DEBUG("\n");
-
-	if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
-						  cmdbuf, &cmdsz))) {
-		DRM_ERROR("Packet verification failed\n");
-		return ret;
-	}
-
-	if (!orig_nbox)
-		goto out;
-
-	do {
-		if (i < cmdbuf->nbox) {
-			if (copy_from_user(&box, &boxes[i], sizeof(box)))
-				return -EFAULT;
-			/* FIXME The second and subsequent times round
-			 * this loop, send a WAIT_UNTIL_3D_IDLE before
-			 * calling emit_clip_rect(). This fixes a
-			 * lockup on fast machines when sending
-			 * several cliprects with a cmdbuf, as when
-			 * waving a 2D window over a 3D
-			 * window. Something in the commands from user
-			 * space seems to hang the card when they're
-			 * sent several times in a row. That would be
-			 * the correct place to fix it but this works
-			 * around it until I can figure that out - Tim
-			 * Smith */
-			if (i) {
-				BEGIN_RING(2);
-				RADEON_WAIT_UNTIL_3D_IDLE();
-				ADVANCE_RING();
-			}
-			radeon_emit_clip_rect(dev_priv, &box);
-		}
-
-		BEGIN_RING(cmdsz);
-		OUT_RING_DRM_BUFFER(cmdbuf->buffer, cmdsz);
-		ADVANCE_RING();
-
-	} while (++i < cmdbuf->nbox);
-	if (cmdbuf->nbox == 1)
-		cmdbuf->nbox = 0;
-
-	return 0;
-      out:
-	drm_buffer_advance(cmdbuf->buffer, cmdsz * 4);
-	return 0;
-}
-
-static int radeon_emit_wait(struct drm_device * dev, int flags)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	RING_LOCALS;
-
-	DRM_DEBUG("%x\n", flags);
-	switch (flags) {
-	case RADEON_WAIT_2D:
-		BEGIN_RING(2);
-		RADEON_WAIT_UNTIL_2D_IDLE();
-		ADVANCE_RING();
-		break;
-	case RADEON_WAIT_3D:
-		BEGIN_RING(2);
-		RADEON_WAIT_UNTIL_3D_IDLE();
-		ADVANCE_RING();
-		break;
-	case RADEON_WAIT_2D | RADEON_WAIT_3D:
-		BEGIN_RING(2);
-		RADEON_WAIT_UNTIL_IDLE();
-		ADVANCE_RING();
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int radeon_cp_cmdbuf(struct drm_device *dev, void *data,
-		struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_device_dma *dma = dev->dma;
-	struct drm_buf *buf = NULL;
-	drm_radeon_cmd_header_t stack_header;
-	int idx;
-	drm_radeon_kcmd_buffer_t *cmdbuf = data;
-	int orig_nbox;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	RING_SPACE_TEST_WITH_RETURN(dev_priv);
-	VB_AGE_TEST_WITH_RETURN(dev_priv);
-
-	if (cmdbuf->bufsz > 64 * 1024 || cmdbuf->bufsz < 0) {
-		return -EINVAL;
-	}
-
-	/* Allocate an in-kernel area and copy in the cmdbuf.  Do this to avoid
-	 * races between checking values and using those values in other code,
-	 * and simply to avoid a lot of function calls to copy in data.
-	 */
-	if (cmdbuf->bufsz != 0) {
-		int rv;
-		void __user *buffer = cmdbuf->buffer;
-		rv = drm_buffer_alloc(&cmdbuf->buffer, cmdbuf->bufsz);
-		if (rv)
-			return rv;
-		rv = drm_buffer_copy_from_user(cmdbuf->buffer, buffer,
-						cmdbuf->bufsz);
-		if (rv) {
-			drm_buffer_free(cmdbuf->buffer);
-			return rv;
-		}
-	} else
-		goto done;
-
-	orig_nbox = cmdbuf->nbox;
-
-	if (dev_priv->microcode_version == UCODE_R300) {
-		int temp;
-		temp = r300_do_cp_cmdbuf(dev, file_priv, cmdbuf);
-
-		drm_buffer_free(cmdbuf->buffer);
-
-		return temp;
-	}
-
-	/* microcode_version != r300 */
-	while (drm_buffer_unprocessed(cmdbuf->buffer) >= sizeof(stack_header)) {
-
-		drm_radeon_cmd_header_t *header;
-		header = drm_buffer_read_object(cmdbuf->buffer,
-				sizeof(stack_header), &stack_header);
-
-		switch (header->header.cmd_type) {
-		case RADEON_CMD_PACKET:
-			DRM_DEBUG("RADEON_CMD_PACKET\n");
-			if (radeon_emit_packets
-			    (dev_priv, file_priv, *header, cmdbuf)) {
-				DRM_ERROR("radeon_emit_packets failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_SCALARS:
-			DRM_DEBUG("RADEON_CMD_SCALARS\n");
-			if (radeon_emit_scalars(dev_priv, *header, cmdbuf)) {
-				DRM_ERROR("radeon_emit_scalars failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_VECTORS:
-			DRM_DEBUG("RADEON_CMD_VECTORS\n");
-			if (radeon_emit_vectors(dev_priv, *header, cmdbuf)) {
-				DRM_ERROR("radeon_emit_vectors failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_DMA_DISCARD:
-			DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
-			idx = header->dma.buf_idx;
-			if (idx < 0 || idx >= dma->buf_count) {
-				DRM_ERROR("buffer index %d (of %d max)\n",
-					  idx, dma->buf_count - 1);
-				goto err;
-			}
-
-			buf = dma->buflist[idx];
-			if (buf->file_priv != file_priv || buf->pending) {
-				DRM_ERROR("bad buffer %p %p %d\n",
-					  buf->file_priv, file_priv,
-					  buf->pending);
-				goto err;
-			}
-
-			radeon_cp_discard_buffer(dev, file_priv->master, buf);
-			break;
-
-		case RADEON_CMD_PACKET3:
-			DRM_DEBUG("RADEON_CMD_PACKET3\n");
-			if (radeon_emit_packet3(dev, file_priv, cmdbuf)) {
-				DRM_ERROR("radeon_emit_packet3 failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_PACKET3_CLIP:
-			DRM_DEBUG("RADEON_CMD_PACKET3_CLIP\n");
-			if (radeon_emit_packet3_cliprect
-			    (dev, file_priv, cmdbuf, orig_nbox)) {
-				DRM_ERROR("radeon_emit_packet3_clip failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_SCALARS2:
-			DRM_DEBUG("RADEON_CMD_SCALARS2\n");
-			if (radeon_emit_scalars2(dev_priv, *header, cmdbuf)) {
-				DRM_ERROR("radeon_emit_scalars2 failed\n");
-				goto err;
-			}
-			break;
-
-		case RADEON_CMD_WAIT:
-			DRM_DEBUG("RADEON_CMD_WAIT\n");
-			if (radeon_emit_wait(dev, header->wait.flags)) {
-				DRM_ERROR("radeon_emit_wait failed\n");
-				goto err;
-			}
-			break;
-		case RADEON_CMD_VECLINEAR:
-			DRM_DEBUG("RADEON_CMD_VECLINEAR\n");
-			if (radeon_emit_veclinear(dev_priv, *header, cmdbuf)) {
-				DRM_ERROR("radeon_emit_veclinear failed\n");
-				goto err;
-			}
-			break;
-
-		default:
-			DRM_ERROR("bad cmd_type %d at byte %d\n",
-				  header->header.cmd_type,
-				  cmdbuf->buffer->iterator);
-			goto err;
-		}
-	}
-
-	drm_buffer_free(cmdbuf->buffer);
-
-      done:
-	DRM_DEBUG("DONE\n");
-	COMMIT_RING();
-	return 0;
-
-      err:
-	drm_buffer_free(cmdbuf->buffer);
-	return -EINVAL;
-}
-
-static int radeon_cp_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_getparam_t *param = data;
-	int value;
-
-	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
-
-	switch (param->param) {
-	case RADEON_PARAM_GART_BUFFER_OFFSET:
-		value = dev_priv->gart_buffers_offset;
-		break;
-	case RADEON_PARAM_LAST_FRAME:
-		dev_priv->stats.last_frame_reads++;
-		value = GET_SCRATCH(dev_priv, 0);
-		break;
-	case RADEON_PARAM_LAST_DISPATCH:
-		value = GET_SCRATCH(dev_priv, 1);
-		break;
-	case RADEON_PARAM_LAST_CLEAR:
-		dev_priv->stats.last_clear_reads++;
-		value = GET_SCRATCH(dev_priv, 2);
-		break;
-	case RADEON_PARAM_IRQ_NR:
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			value = 0;
-		else
-			value = dev->pdev->irq;
-		break;
-	case RADEON_PARAM_GART_BASE:
-		value = dev_priv->gart_vm_start;
-		break;
-	case RADEON_PARAM_REGISTER_HANDLE:
-		value = dev_priv->mmio->offset;
-		break;
-	case RADEON_PARAM_STATUS_HANDLE:
-		value = dev_priv->ring_rptr_offset;
-		break;
-#if BITS_PER_LONG == 32
-		/*
-		 * This ioctl() doesn't work on 64-bit platforms because hw_lock is a
-		 * pointer which can't fit into an int-sized variable.  According to
-		 * Michel Dänzer, the ioctl() is only used on embedded platforms, so
-		 * not supporting it shouldn't be a problem.  If the same functionality
-		 * is needed on 64-bit platforms, a new ioctl() would have to be added,
-		 * so backwards-compatibility for the embedded platforms can be
-		 * maintained.  --davidm 4-Feb-2004.
-		 */
-	case RADEON_PARAM_SAREA_HANDLE:
-		/* The lock is the first dword in the sarea. */
-		/* no users of this parameter */
-		break;
-#endif
-	case RADEON_PARAM_GART_TEX_HANDLE:
-		value = dev_priv->gart_textures_offset;
-		break;
-	case RADEON_PARAM_SCRATCH_OFFSET:
-		if (!dev_priv->writeback_works)
-			return -EINVAL;
-		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
-			value = R600_SCRATCH_REG_OFFSET;
-		else
-			value = RADEON_SCRATCH_REG_OFFSET;
-		break;
-	case RADEON_PARAM_CARD_TYPE:
-		if (dev_priv->flags & RADEON_IS_PCIE)
-			value = RADEON_CARD_PCIE;
-		else if (dev_priv->flags & RADEON_IS_AGP)
-			value = RADEON_CARD_AGP;
-		else
-			value = RADEON_CARD_PCI;
-		break;
-	case RADEON_PARAM_VBLANK_CRTC:
-		value = radeon_vblank_crtc_get(dev);
-		break;
-	case RADEON_PARAM_FB_LOCATION:
-		value = radeon_read_fb_location(dev_priv);
-		break;
-	case RADEON_PARAM_NUM_GB_PIPES:
-		value = dev_priv->num_gb_pipes;
-		break;
-	case RADEON_PARAM_NUM_Z_PIPES:
-		value = dev_priv->num_z_pipes;
-		break;
-	default:
-		DRM_DEBUG("Invalid parameter %d\n", param->param);
-		return -EINVAL;
-	}
-
-	if (copy_to_user(param->value, &value, sizeof(int))) {
-		DRM_ERROR("copy_to_user\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static int radeon_cp_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
-	drm_radeon_setparam_t *sp = data;
-	struct drm_radeon_driver_file_fields *radeon_priv;
-
-	switch (sp->param) {
-	case RADEON_SETPARAM_FB_LOCATION:
-		radeon_priv = file_priv->driver_priv;
-		radeon_priv->radeon_fb_delta = dev_priv->fb_location -
-		    sp->value;
-		break;
-	case RADEON_SETPARAM_SWITCH_TILING:
-		if (sp->value == 0) {
-			DRM_DEBUG("color tiling disabled\n");
-			dev_priv->front_pitch_offset &= ~RADEON_DST_TILE_MACRO;
-			dev_priv->back_pitch_offset &= ~RADEON_DST_TILE_MACRO;
-			if (master_priv->sarea_priv)
-				master_priv->sarea_priv->tiling_enabled = 0;
-		} else if (sp->value == 1) {
-			DRM_DEBUG("color tiling enabled\n");
-			dev_priv->front_pitch_offset |= RADEON_DST_TILE_MACRO;
-			dev_priv->back_pitch_offset |= RADEON_DST_TILE_MACRO;
-			if (master_priv->sarea_priv)
-				master_priv->sarea_priv->tiling_enabled = 1;
-		}
-		break;
-	case RADEON_SETPARAM_PCIGART_LOCATION:
-		dev_priv->pcigart_offset = sp->value;
-		dev_priv->pcigart_offset_set = 1;
-		break;
-	case RADEON_SETPARAM_NEW_MEMMAP:
-		dev_priv->new_memmap = sp->value;
-		break;
-	case RADEON_SETPARAM_PCIGART_TABLE_SIZE:
-		dev_priv->gart_info.table_size = sp->value;
-		if (dev_priv->gart_info.table_size < RADEON_PCIGART_TABLE_SIZE)
-			dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
-		break;
-	case RADEON_SETPARAM_VBLANK_CRTC:
-		return radeon_vblank_crtc_set(dev, sp->value);
-		break;
-	default:
-		DRM_DEBUG("Invalid parameter %d\n", sp->param);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* When a client dies:
- *    - Check for and clean up flipped page state
- *    - Free any alloced GART memory.
- *    - Free any alloced radeon surfaces.
- *
- * DRM infrastructure takes care of reclaiming dma buffers.
- */
-void radeon_driver_preclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-	if (dev->dev_private) {
-		drm_radeon_private_t *dev_priv = dev->dev_private;
-		dev_priv->page_flipping = 0;
-		radeon_mem_release(file_priv, dev_priv->gart_heap);
-		radeon_mem_release(file_priv, dev_priv->fb_heap);
-		radeon_surfaces_release(file_priv, dev_priv);
-	}
-}
-
-void radeon_driver_lastclose(struct drm_device *dev)
-{
-	radeon_surfaces_release(PCIGART_FILE_PRIV, dev->dev_private);
-	radeon_do_release(dev);
-}
-
-int radeon_driver_open(struct drm_device *dev, struct drm_file *file_priv)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	struct drm_radeon_driver_file_fields *radeon_priv;
-
-	DRM_DEBUG("\n");
-	radeon_priv = kmalloc(sizeof(*radeon_priv), GFP_KERNEL);
-
-	if (!radeon_priv)
-		return -ENOMEM;
-
-	file_priv->driver_priv = radeon_priv;
-
-	if (dev_priv)
-		radeon_priv->radeon_fb_delta = dev_priv->fb_location;
-	else
-		radeon_priv->radeon_fb_delta = 0;
-	return 0;
-}
-
-void radeon_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-	struct drm_radeon_driver_file_fields *radeon_priv =
-	    file_priv->driver_priv;
-
-	kfree(radeon_priv);
-}
-
-struct drm_ioctl_desc radeon_ioctls[] = {
-	DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_CP_START, radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, radeon_cp_idle, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, radeon_cp_resume, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_RESET, radeon_engine_reset, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, radeon_fullscreen, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_SWAP, radeon_cp_swap, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_CLEAR, radeon_cp_clear, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_VERTEX, radeon_cp_vertex, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_INDICES, radeon_cp_indices, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, radeon_cp_texture, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, radeon_cp_stipple, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, radeon_cp_vertex2, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, radeon_cp_cmdbuf, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, radeon_cp_getparam, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_FLIP, radeon_cp_flip, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_ALLOC, radeon_mem_alloc, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_FREE, radeon_mem_free, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, radeon_irq_emit, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, radeon_irq_wait, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, radeon_cp_setparam, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, radeon_surface_alloc, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, radeon_surface_free, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(RADEON_CS, r600_cs_legacy_ioctl, DRM_AUTH)
-};
-
-int radeon_max_ioctl = ARRAY_SIZE(radeon_ioctls);
diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c
index 48d97c040f49..3979632b9225 100644
--- a/drivers/gpu/drm/radeon/radeon_vm.c
+++ b/drivers/gpu/drm/radeon/radeon_vm.c
@@ -455,15 +455,15 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev,
 
 	if (soffset) {
 		/* make sure object fit at this offset */
-		eoffset = soffset + size;
+		eoffset = soffset + size - 1;
 		if (soffset >= eoffset) {
 			r = -EINVAL;
 			goto error_unreserve;
 		}
 
 		last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
-		if (last_pfn > rdev->vm_manager.max_pfn) {
-			dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
+		if (last_pfn >= rdev->vm_manager.max_pfn) {
+			dev_err(rdev->dev, "va above limit (0x%08X >= 0x%08X)\n",
 				last_pfn, rdev->vm_manager.max_pfn);
 			r = -EINVAL;
 			goto error_unreserve;
@@ -478,7 +478,7 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev,
 	eoffset /= RADEON_GPU_PAGE_SIZE;
 	if (soffset || eoffset) {
 		struct interval_tree_node *it;
-		it = interval_tree_iter_first(&vm->va, soffset, eoffset - 1);
+		it = interval_tree_iter_first(&vm->va, soffset, eoffset);
 		if (it && it != &bo_va->it) {
 			struct radeon_bo_va *tmp;
 			tmp = container_of(it, struct radeon_bo_va, it);
@@ -518,7 +518,7 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev,
 	if (soffset || eoffset) {
 		spin_lock(&vm->status_lock);
 		bo_va->it.start = soffset;
-		bo_va->it.last = eoffset - 1;
+		bo_va->it.last = eoffset;
 		list_add(&bo_va->vm_status, &vm->cleared);
 		spin_unlock(&vm->status_lock);
 		interval_tree_insert(&bo_va->it, &vm->va);
@@ -888,7 +888,7 @@ static void radeon_vm_fence_pts(struct radeon_vm *vm,
 	unsigned i;
 
 	start >>= radeon_vm_block_size;
-	end >>= radeon_vm_block_size;
+	end = (end - 1) >> radeon_vm_block_size;
 
 	for (i = start; i <= end; ++i)
 		radeon_bo_fence(vm->page_tables[i].bo, fence, true);
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index 4c4a7218a3bd..d1a7b58dd291 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -915,6 +915,11 @@
 #define DCCG_AUDIO_DTO1_PHASE                           0x05c0
 #define DCCG_AUDIO_DTO1_MODULE                          0x05c4
 
+#define DENTIST_DISPCLK_CNTL				0x0490
+#	define DENTIST_DPREFCLK_WDIVIDER(x)		(((x) & 0x7f) << 24)
+#	define DENTIST_DPREFCLK_WDIVIDER_MASK		(0x7f << 24)
+#	define DENTIST_DPREFCLK_WDIVIDER_SHIFT		24
+
 #define AFMT_AUDIO_SRC_CONTROL                          0x713c
 #define		AFMT_AUDIO_SRC_SELECT(x)		(((x) & 7) << 0)
 /* AFMT_AUDIO_SRC_SELECT
diff --git a/drivers/gpu/drm/radeon/vce_v1_0.c b/drivers/gpu/drm/radeon/vce_v1_0.c
index 07a0d378e122..a01efe39a820 100644
--- a/drivers/gpu/drm/radeon/vce_v1_0.c
+++ b/drivers/gpu/drm/radeon/vce_v1_0.c
@@ -178,12 +178,12 @@ int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data)
 		return -EINVAL;
 	}
 
-	for (i = 0; i < sign->num; ++i) {
-		if (sign->val[i].chip_id == chip_id)
+	for (i = 0; i < le32_to_cpu(sign->num); ++i) {
+		if (le32_to_cpu(sign->val[i].chip_id) == chip_id)
 			break;
 	}
 
-	if (i == sign->num)
+	if (i == le32_to_cpu(sign->num))
 		return -EINVAL;
 
 	data += (256 - 64) / 4;
@@ -191,18 +191,18 @@ int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data)
 	data[1] = sign->val[i].nonce[1];
 	data[2] = sign->val[i].nonce[2];
 	data[3] = sign->val[i].nonce[3];
-	data[4] = sign->len + 64;
+	data[4] = cpu_to_le32(le32_to_cpu(sign->len) + 64);
 
 	memset(&data[5], 0, 44);
 	memcpy(&data[16], &sign[1], rdev->vce_fw->size - sizeof(*sign));
 
-	data += data[4] / 4;
+	data += le32_to_cpu(data[4]) / 4;
 	data[0] = sign->val[i].sigval[0];
 	data[1] = sign->val[i].sigval[1];
 	data[2] = sign->val[i].sigval[2];
 	data[3] = sign->val[i].sigval[3];
 
-	rdev->vce.keyselect = sign->val[i].keyselect;
+	rdev->vce.keyselect = le32_to_cpu(sign->val[i].keyselect);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig
index d4e0a39568f6..96dcd4a78951 100644
--- a/drivers/gpu/drm/rcar-du/Kconfig
+++ b/drivers/gpu/drm/rcar-du/Kconfig
@@ -1,6 +1,6 @@
 config DRM_RCAR_DU
 	tristate "DRM Support for R-Car Display Unit"
-	depends on DRM && ARM && HAVE_DMA_ATTRS && OF
+	depends on DRM && ARM && OF
 	depends on ARCH_SHMOBILE || COMPILE_TEST
 	select DRM_KMS_HELPER
 	select DRM_KMS_CMA_HELPER
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 48cb19949ca3..88a4b706be16 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -613,7 +613,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 
 	ret = drm_crtc_init_with_planes(rcdu->ddev, crtc,
 					&rgrp->planes[index % 2].plane,
-					NULL, &crtc_funcs);
+					NULL, &crtc_funcs, NULL);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index d0ae1e8009c6..c08700757feb 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -173,7 +173,7 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 			goto done;
 	} else {
 		ret = drm_encoder_init(rcdu->ddev, encoder, &encoder_funcs,
-				       encoder_type);
+				       encoder_type, NULL);
 		if (ret < 0)
 			goto done;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c
index 96f2eb43713c..a37b6e2fe51a 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c
@@ -28,7 +28,7 @@ static int rcar_du_hdmi_connector_get_modes(struct drm_connector *connector)
 {
 	struct rcar_du_connector *con = to_rcar_connector(connector);
 	struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (sfuncs->get_modes == NULL)
 		return 0;
@@ -41,7 +41,7 @@ static int rcar_du_hdmi_connector_mode_valid(struct drm_connector *connector,
 {
 	struct rcar_du_connector *con = to_rcar_connector(connector);
 	struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (sfuncs->mode_valid == NULL)
 		return MODE_OK;
@@ -66,7 +66,7 @@ rcar_du_hdmi_connector_detect(struct drm_connector *connector, bool force)
 {
 	struct rcar_du_connector *con = to_rcar_connector(connector);
 	struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (sfuncs->detect == NULL)
 		return connector_status_unknown;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
index 81da8419282b..2567efcbee36 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
@@ -35,7 +35,7 @@ struct rcar_du_hdmienc {
 static void rcar_du_hdmienc_disable(struct drm_encoder *encoder)
 {
 	struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (sfuncs->dpms)
 		sfuncs->dpms(encoder, DRM_MODE_DPMS_OFF);
@@ -50,7 +50,7 @@ static void rcar_du_hdmienc_disable(struct drm_encoder *encoder)
 static void rcar_du_hdmienc_enable(struct drm_encoder *encoder)
 {
 	struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (hdmienc->renc->lvds)
 		rcar_du_lvdsenc_enable(hdmienc->renc->lvds, encoder->crtc,
@@ -67,7 +67,7 @@ static int rcar_du_hdmienc_atomic_check(struct drm_encoder *encoder,
 					struct drm_connector_state *conn_state)
 {
 	struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
 	const struct drm_display_mode *mode = &crtc_state->mode;
 
@@ -89,7 +89,7 @@ static void rcar_du_hdmienc_mode_set(struct drm_encoder *encoder,
 				     struct drm_display_mode *adjusted_mode)
 {
 	struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-	struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
+	const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
 	if (sfuncs->mode_set)
 		sfuncs->mode_set(encoder, mode, adjusted_mode);
@@ -151,7 +151,7 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
 		goto error;
 
 	ret = drm_encoder_init(rcdu->ddev, encoder, &encoder_funcs,
-			       DRM_MODE_ENCODER_TMDS);
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret < 0)
 		goto error;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index ca12e8ca5552..43bce69d8560 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -136,7 +136,7 @@ int rcar_du_dumb_create(struct drm_file *file, struct drm_device *dev,
 
 static struct drm_framebuffer *
 rcar_du_fb_create(struct drm_device *dev, struct drm_file *file_priv,
-		  struct drm_mode_fb_cmd2 *mode_cmd)
+		  const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct rcar_du_device *rcdu = dev->dev_private;
 	const struct rcar_du_format_info *format;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index ffa583712cd9..c3ed9522c0e1 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -410,7 +410,8 @@ int rcar_du_planes_init(struct rcar_du_group *rgrp)
 
 		ret = drm_universal_plane_init(rcdu->ddev, &plane->plane, crtcs,
 					       &rcar_du_plane_funcs, formats,
-					       ARRAY_SIZE(formats), type);
+					       ARRAY_SIZE(formats), type,
+					       NULL);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 35215f6867d3..85739859dffc 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -25,3 +25,13 @@ config ROCKCHIP_DW_HDMI
 	  for the Synopsys DesignWare HDMI driver. If you want to
 	  enable HDMI on RK3288 based SoC, you should selet this
 	  option.
+
+config ROCKCHIP_DW_MIPI_DSI
+	tristate "Rockchip specific extensions for Synopsys DW MIPI DSI"
+	depends on DRM_ROCKCHIP
+	select DRM_MIPI_DSI
+	help
+	 This selects support for Rockchip SoC specific extensions
+	 for the Synopsys DesignWare HDMI driver. If you want to
+	 enable MIPI DSI on RK3288 based SoC, you should selet this
+	 option.
diff --git a/drivers/gpu/drm/rockchip/Makefile b/drivers/gpu/drm/rockchip/Makefile
index f3d8a19c641f..f6a809afceec 100644
--- a/drivers/gpu/drm/rockchip/Makefile
+++ b/drivers/gpu/drm/rockchip/Makefile
@@ -2,9 +2,11 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-rockchipdrm-y := rockchip_drm_drv.o rockchip_drm_fb.o rockchip_drm_fbdev.o \
-		rockchip_drm_gem.o
+rockchipdrm-y := rockchip_drm_drv.o rockchip_drm_fb.o \
+		rockchip_drm_gem.o rockchip_drm_vop.o
+rockchipdrm-$(CONFIG_DRM_FBDEV_EMULATION) += rockchip_drm_fbdev.o
 
 obj-$(CONFIG_ROCKCHIP_DW_HDMI) += dw_hdmi-rockchip.o
+obj-$(CONFIG_ROCKCHIP_DW_MIPI_DSI) += dw-mipi-dsi.o
 
-obj-$(CONFIG_DRM_ROCKCHIP) += rockchipdrm.o rockchip_drm_vop.o
+obj-$(CONFIG_DRM_ROCKCHIP) += rockchipdrm.o rockchip_vop_reg.o
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
new file mode 100644
index 000000000000..f8f8f29fb7c3
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
@@ -0,0 +1,1195 @@
+/*
+ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/iopoll.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drmP.h>
+#include <video/mipi_display.h>
+
+#include "rockchip_drm_drv.h"
+#include "rockchip_drm_vop.h"
+
+#define DRIVER_NAME    "dw-mipi-dsi"
+
+#define GRF_SOC_CON6                    0x025c
+#define DSI0_SEL_VOP_LIT                (1 << 6)
+#define DSI1_SEL_VOP_LIT                (1 << 9)
+
+#define DSI_VERSION			0x00
+#define DSI_PWR_UP			0x04
+#define RESET				0
+#define POWERUP				BIT(0)
+
+#define DSI_CLKMGR_CFG			0x08
+#define TO_CLK_DIVIDSION(div)		(((div) & 0xff) << 8)
+#define TX_ESC_CLK_DIVIDSION(div)	(((div) & 0xff) << 0)
+
+#define DSI_DPI_VCID			0x0c
+#define DPI_VID(vid)			(((vid) & 0x3) << 0)
+
+#define DSI_DPI_COLOR_CODING		0x10
+#define EN18_LOOSELY			BIT(8)
+#define DPI_COLOR_CODING_16BIT_1	0x0
+#define DPI_COLOR_CODING_16BIT_2	0x1
+#define DPI_COLOR_CODING_16BIT_3	0x2
+#define DPI_COLOR_CODING_18BIT_1	0x3
+#define DPI_COLOR_CODING_18BIT_2	0x4
+#define DPI_COLOR_CODING_24BIT		0x5
+
+#define DSI_DPI_CFG_POL			0x14
+#define COLORM_ACTIVE_LOW		BIT(4)
+#define SHUTD_ACTIVE_LOW		BIT(3)
+#define HSYNC_ACTIVE_LOW		BIT(2)
+#define VSYNC_ACTIVE_LOW		BIT(1)
+#define DATAEN_ACTIVE_LOW		BIT(0)
+
+#define DSI_DPI_LP_CMD_TIM		0x18
+#define OUTVACT_LPCMD_TIME(p)		(((p) & 0xff) << 16)
+#define INVACT_LPCMD_TIME(p)		((p) & 0xff)
+
+#define DSI_DBI_CFG			0x20
+#define DSI_DBI_CMDSIZE			0x28
+
+#define DSI_PCKHDL_CFG			0x2c
+#define EN_CRC_RX			BIT(4)
+#define EN_ECC_RX			BIT(3)
+#define EN_BTA				BIT(2)
+#define EN_EOTP_RX			BIT(1)
+#define EN_EOTP_TX			BIT(0)
+
+#define DSI_MODE_CFG			0x34
+#define ENABLE_VIDEO_MODE		0
+#define ENABLE_CMD_MODE			BIT(0)
+
+#define DSI_VID_MODE_CFG		0x38
+#define FRAME_BTA_ACK			BIT(14)
+#define ENABLE_LOW_POWER		(0x3f << 8)
+#define ENABLE_LOW_POWER_MASK		(0x3f << 8)
+#define VID_MODE_TYPE_BURST_SYNC_PULSES		0x2
+#define VID_MODE_TYPE_MASK			0x3
+
+#define DSI_VID_PKT_SIZE		0x3c
+#define VID_PKT_SIZE(p)			(((p) & 0x3fff) << 0)
+#define VID_PKT_MAX_SIZE		0x3fff
+
+#define DSI_VID_HSA_TIME		0x48
+#define DSI_VID_HBP_TIME		0x4c
+#define DSI_VID_HLINE_TIME		0x50
+#define DSI_VID_VSA_LINES		0x54
+#define DSI_VID_VBP_LINES		0x58
+#define DSI_VID_VFP_LINES		0x5c
+#define DSI_VID_VACTIVE_LINES		0x60
+#define DSI_CMD_MODE_CFG		0x68
+#define MAX_RD_PKT_SIZE_LP		BIT(24)
+#define DCS_LW_TX_LP			BIT(19)
+#define DCS_SR_0P_TX_LP			BIT(18)
+#define DCS_SW_1P_TX_LP			BIT(17)
+#define DCS_SW_0P_TX_LP			BIT(16)
+#define GEN_LW_TX_LP			BIT(14)
+#define GEN_SR_2P_TX_LP			BIT(13)
+#define GEN_SR_1P_TX_LP			BIT(12)
+#define GEN_SR_0P_TX_LP			BIT(11)
+#define GEN_SW_2P_TX_LP			BIT(10)
+#define GEN_SW_1P_TX_LP			BIT(9)
+#define GEN_SW_0P_TX_LP			BIT(8)
+#define EN_ACK_RQST			BIT(1)
+#define EN_TEAR_FX			BIT(0)
+
+#define CMD_MODE_ALL_LP			(MAX_RD_PKT_SIZE_LP | \
+					 DCS_LW_TX_LP | \
+					 DCS_SR_0P_TX_LP | \
+					 DCS_SW_1P_TX_LP | \
+					 DCS_SW_0P_TX_LP | \
+					 GEN_LW_TX_LP | \
+					 GEN_SR_2P_TX_LP | \
+					 GEN_SR_1P_TX_LP | \
+					 GEN_SR_0P_TX_LP | \
+					 GEN_SW_2P_TX_LP | \
+					 GEN_SW_1P_TX_LP | \
+					 GEN_SW_0P_TX_LP)
+
+#define DSI_GEN_HDR			0x6c
+#define GEN_HDATA(data)			(((data) & 0xffff) << 8)
+#define GEN_HDATA_MASK			(0xffff << 8)
+#define GEN_HTYPE(type)			(((type) & 0xff) << 0)
+#define GEN_HTYPE_MASK			0xff
+
+#define DSI_GEN_PLD_DATA		0x70
+
+#define DSI_CMD_PKT_STATUS		0x74
+#define GEN_CMD_EMPTY			BIT(0)
+#define GEN_CMD_FULL			BIT(1)
+#define GEN_PLD_W_EMPTY			BIT(2)
+#define GEN_PLD_W_FULL			BIT(3)
+#define GEN_PLD_R_EMPTY			BIT(4)
+#define GEN_PLD_R_FULL			BIT(5)
+#define GEN_RD_CMD_BUSY			BIT(6)
+
+#define DSI_TO_CNT_CFG			0x78
+#define HSTX_TO_CNT(p)			(((p) & 0xffff) << 16)
+#define LPRX_TO_CNT(p)			((p) & 0xffff)
+
+#define DSI_BTA_TO_CNT			0x8c
+
+#define DSI_LPCLK_CTRL			0x94
+#define AUTO_CLKLANE_CTRL		BIT(1)
+#define PHY_TXREQUESTCLKHS		BIT(0)
+
+#define DSI_PHY_TMR_LPCLK_CFG		0x98
+#define PHY_CLKHS2LP_TIME(lbcc)		(((lbcc) & 0x3ff) << 16)
+#define PHY_CLKLP2HS_TIME(lbcc)		((lbcc) & 0x3ff)
+
+#define DSI_PHY_TMR_CFG			0x9c
+#define PHY_HS2LP_TIME(lbcc)		(((lbcc) & 0xff) << 24)
+#define PHY_LP2HS_TIME(lbcc)		(((lbcc) & 0xff) << 16)
+#define MAX_RD_TIME(lbcc)		((lbcc) & 0x7fff)
+
+#define DSI_PHY_RSTZ			0xa0
+#define PHY_DISFORCEPLL			0
+#define PHY_ENFORCEPLL			BIT(3)
+#define PHY_DISABLECLK			0
+#define PHY_ENABLECLK			BIT(2)
+#define PHY_RSTZ			0
+#define PHY_UNRSTZ			BIT(1)
+#define PHY_SHUTDOWNZ			0
+#define PHY_UNSHUTDOWNZ			BIT(0)
+
+#define DSI_PHY_IF_CFG			0xa4
+#define N_LANES(n)			((((n) - 1) & 0x3) << 0)
+#define PHY_STOP_WAIT_TIME(cycle)	(((cycle) & 0xff) << 8)
+
+#define DSI_PHY_STATUS			0xb0
+#define LOCK				BIT(0)
+#define STOP_STATE_CLK_LANE		BIT(2)
+
+#define DSI_PHY_TST_CTRL0		0xb4
+#define PHY_TESTCLK			BIT(1)
+#define PHY_UNTESTCLK			0
+#define PHY_TESTCLR			BIT(0)
+#define PHY_UNTESTCLR			0
+
+#define DSI_PHY_TST_CTRL1		0xb8
+#define PHY_TESTEN			BIT(16)
+#define PHY_UNTESTEN			0
+#define PHY_TESTDOUT(n)			(((n) & 0xff) << 8)
+#define PHY_TESTDIN(n)			(((n) & 0xff) << 0)
+
+#define DSI_INT_ST0			0xbc
+#define DSI_INT_ST1			0xc0
+#define DSI_INT_MSK0			0xc4
+#define DSI_INT_MSK1			0xc8
+
+#define PHY_STATUS_TIMEOUT_US		10000
+#define CMD_PKT_STATUS_TIMEOUT_US	20000
+
+#define BYPASS_VCO_RANGE	BIT(7)
+#define VCO_RANGE_CON_SEL(val)	(((val) & 0x7) << 3)
+#define VCO_IN_CAP_CON_DEFAULT	(0x0 << 1)
+#define VCO_IN_CAP_CON_LOW	(0x1 << 1)
+#define VCO_IN_CAP_CON_HIGH	(0x2 << 1)
+#define REF_BIAS_CUR_SEL	BIT(0)
+
+#define CP_CURRENT_3MA		BIT(3)
+#define CP_PROGRAM_EN		BIT(7)
+#define LPF_PROGRAM_EN		BIT(6)
+#define LPF_RESISTORS_20_KOHM	0
+
+#define HSFREQRANGE_SEL(val)	(((val) & 0x3f) << 1)
+
+#define INPUT_DIVIDER(val)	((val - 1) & 0x7f)
+#define LOW_PROGRAM_EN		0
+#define HIGH_PROGRAM_EN		BIT(7)
+#define LOOP_DIV_LOW_SEL(val)	((val - 1) & 0x1f)
+#define LOOP_DIV_HIGH_SEL(val)	(((val - 1) >> 5) & 0x1f)
+#define PLL_LOOP_DIV_EN		BIT(5)
+#define PLL_INPUT_DIV_EN	BIT(4)
+
+#define POWER_CONTROL		BIT(6)
+#define INTERNAL_REG_CURRENT	BIT(3)
+#define BIAS_BLOCK_ON		BIT(2)
+#define BANDGAP_ON		BIT(0)
+
+#define TER_RESISTOR_HIGH	BIT(7)
+#define	TER_RESISTOR_LOW	0
+#define LEVEL_SHIFTERS_ON	BIT(6)
+#define TER_CAL_DONE		BIT(5)
+#define SETRD_MAX		(0x7 << 2)
+#define POWER_MANAGE		BIT(1)
+#define TER_RESISTORS_ON	BIT(0)
+
+#define BIASEXTR_SEL(val)	((val) & 0x7)
+#define BANDGAP_SEL(val)	((val) & 0x7)
+#define TLP_PROGRAM_EN		BIT(7)
+#define THS_PRE_PROGRAM_EN	BIT(7)
+#define THS_ZERO_PROGRAM_EN	BIT(6)
+
+enum {
+	BANDGAP_97_07,
+	BANDGAP_98_05,
+	BANDGAP_99_02,
+	BANDGAP_100_00,
+	BANDGAP_93_17,
+	BANDGAP_94_15,
+	BANDGAP_95_12,
+	BANDGAP_96_10,
+};
+
+enum {
+	BIASEXTR_87_1,
+	BIASEXTR_91_5,
+	BIASEXTR_95_9,
+	BIASEXTR_100,
+	BIASEXTR_105_94,
+	BIASEXTR_111_88,
+	BIASEXTR_118_8,
+	BIASEXTR_127_7,
+};
+
+struct dw_mipi_dsi_plat_data {
+	unsigned int max_data_lanes;
+	enum drm_mode_status (*mode_valid)(struct drm_connector *connector,
+					   struct drm_display_mode *mode);
+};
+
+struct dw_mipi_dsi {
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+	struct mipi_dsi_host dsi_host;
+	struct drm_panel *panel;
+	struct device *dev;
+	struct regmap *grf_regmap;
+	void __iomem *base;
+
+	struct clk *pllref_clk;
+	struct clk *pclk;
+
+	unsigned int lane_mbps; /* per lane */
+	u32 channel;
+	u32 lanes;
+	u32 format;
+	u16 input_div;
+	u16 feedback_div;
+	struct drm_display_mode *mode;
+
+	const struct dw_mipi_dsi_plat_data *pdata;
+};
+
+enum dw_mipi_dsi_mode {
+	DW_MIPI_DSI_CMD_MODE,
+	DW_MIPI_DSI_VID_MODE,
+};
+
+struct dphy_pll_testdin_map {
+	unsigned int max_mbps;
+	u8 testdin;
+};
+
+/* The table is based on 27MHz DPHY pll reference clock. */
+static const struct dphy_pll_testdin_map dptdin_map[] = {
+	{  90, 0x00}, { 100, 0x10}, { 110, 0x20}, { 130, 0x01},
+	{ 140, 0x11}, { 150, 0x21}, { 170, 0x02}, { 180, 0x12},
+	{ 200, 0x22}, { 220, 0x03}, { 240, 0x13}, { 250, 0x23},
+	{ 270, 0x04}, { 300, 0x14}, { 330, 0x05}, { 360, 0x15},
+	{ 400, 0x25}, { 450, 0x06}, { 500, 0x16}, { 550, 0x07},
+	{ 600, 0x17}, { 650, 0x08}, { 700, 0x18}, { 750, 0x09},
+	{ 800, 0x19}, { 850, 0x29}, { 900, 0x39}, { 950, 0x0a},
+	{1000, 0x1a}, {1050, 0x2a}, {1100, 0x3a}, {1150, 0x0b},
+	{1200, 0x1b}, {1250, 0x2b}, {1300, 0x3b}, {1350, 0x0c},
+	{1400, 0x1c}, {1450, 0x2c}, {1500, 0x3c}
+};
+
+static int max_mbps_to_testdin(unsigned int max_mbps)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dptdin_map); i++)
+		if (dptdin_map[i].max_mbps > max_mbps)
+			return dptdin_map[i].testdin;
+
+	return -EINVAL;
+}
+
+/*
+ * The controller should generate 2 frames before
+ * preparing the peripheral.
+ */
+static void dw_mipi_dsi_wait_for_two_frames(struct dw_mipi_dsi *dsi)
+{
+	int refresh, two_frames;
+
+	refresh = drm_mode_vrefresh(dsi->mode);
+	two_frames = DIV_ROUND_UP(MSEC_PER_SEC, refresh) * 2;
+	msleep(two_frames);
+}
+
+static inline struct dw_mipi_dsi *host_to_dsi(struct mipi_dsi_host *host)
+{
+	return container_of(host, struct dw_mipi_dsi, dsi_host);
+}
+
+static inline struct dw_mipi_dsi *con_to_dsi(struct drm_connector *con)
+{
+	return container_of(con, struct dw_mipi_dsi, connector);
+}
+
+static inline struct dw_mipi_dsi *encoder_to_dsi(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct dw_mipi_dsi, encoder);
+}
+static inline void dsi_write(struct dw_mipi_dsi *dsi, u32 reg, u32 val)
+{
+	writel(val, dsi->base + reg);
+}
+
+static inline u32 dsi_read(struct dw_mipi_dsi *dsi, u32 reg)
+{
+	return readl(dsi->base + reg);
+}
+
+static void dw_mipi_dsi_phy_write(struct dw_mipi_dsi *dsi, u8 test_code,
+				 u8 test_data)
+{
+	/*
+	 * With the falling edge on TESTCLK, the TESTDIN[7:0] signal content
+	 * is latched internally as the current test code. Test data is
+	 * programmed internally by rising edge on TESTCLK.
+	 */
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1, PHY_TESTEN | PHY_TESTDOUT(0) |
+					  PHY_TESTDIN(test_code));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLK | PHY_UNTESTCLR);
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL1, PHY_UNTESTEN | PHY_TESTDOUT(0) |
+					  PHY_TESTDIN(test_data));
+
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLK | PHY_UNTESTCLR);
+}
+
+static int dw_mipi_dsi_phy_init(struct dw_mipi_dsi *dsi)
+{
+	int ret, testdin, vco, val;
+
+	vco = (dsi->lane_mbps < 200) ? 0 : (dsi->lane_mbps + 100) / 200;
+
+	testdin = max_mbps_to_testdin(dsi->lane_mbps);
+	if (testdin < 0) {
+		dev_err(dsi->dev,
+			"failed to get testdin for %dmbps lane clock\n",
+			dsi->lane_mbps);
+		return testdin;
+	}
+
+	dsi_write(dsi, DSI_PWR_UP, POWERUP);
+
+	dw_mipi_dsi_phy_write(dsi, 0x10, BYPASS_VCO_RANGE |
+					 VCO_RANGE_CON_SEL(vco) |
+					 VCO_IN_CAP_CON_LOW |
+					 REF_BIAS_CUR_SEL);
+
+	dw_mipi_dsi_phy_write(dsi, 0x11, CP_CURRENT_3MA);
+	dw_mipi_dsi_phy_write(dsi, 0x12, CP_PROGRAM_EN | LPF_PROGRAM_EN |
+					 LPF_RESISTORS_20_KOHM);
+
+	dw_mipi_dsi_phy_write(dsi, 0x44, HSFREQRANGE_SEL(testdin));
+
+	dw_mipi_dsi_phy_write(dsi, 0x19, PLL_LOOP_DIV_EN | PLL_INPUT_DIV_EN);
+	dw_mipi_dsi_phy_write(dsi, 0x17, INPUT_DIVIDER(dsi->input_div));
+	dw_mipi_dsi_phy_write(dsi, 0x18, LOOP_DIV_LOW_SEL(dsi->feedback_div) |
+					 LOW_PROGRAM_EN);
+	dw_mipi_dsi_phy_write(dsi, 0x18, LOOP_DIV_HIGH_SEL(dsi->feedback_div) |
+					 HIGH_PROGRAM_EN);
+
+	dw_mipi_dsi_phy_write(dsi, 0x20, POWER_CONTROL | INTERNAL_REG_CURRENT |
+					 BIAS_BLOCK_ON | BANDGAP_ON);
+
+	dw_mipi_dsi_phy_write(dsi, 0x21, TER_RESISTOR_LOW | TER_CAL_DONE |
+					 SETRD_MAX | TER_RESISTORS_ON);
+	dw_mipi_dsi_phy_write(dsi, 0x21, TER_RESISTOR_HIGH | LEVEL_SHIFTERS_ON |
+					 SETRD_MAX | POWER_MANAGE |
+					 TER_RESISTORS_ON);
+
+	dw_mipi_dsi_phy_write(dsi, 0x22, LOW_PROGRAM_EN |
+					 BIASEXTR_SEL(BIASEXTR_127_7));
+	dw_mipi_dsi_phy_write(dsi, 0x22, HIGH_PROGRAM_EN |
+					 BANDGAP_SEL(BANDGAP_96_10));
+
+	dw_mipi_dsi_phy_write(dsi, 0x70, TLP_PROGRAM_EN | 0xf);
+	dw_mipi_dsi_phy_write(dsi, 0x71, THS_PRE_PROGRAM_EN | 0x55);
+	dw_mipi_dsi_phy_write(dsi, 0x72, THS_ZERO_PROGRAM_EN | 0xa);
+
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_ENFORCEPLL | PHY_ENABLECLK |
+				     PHY_UNRSTZ | PHY_UNSHUTDOWNZ);
+
+
+	ret = readx_poll_timeout(readl, dsi->base + DSI_PHY_STATUS,
+				 val, val & LOCK, 1000, PHY_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to wait for phy lock state\n");
+		return ret;
+	}
+
+	ret = readx_poll_timeout(readl, dsi->base + DSI_PHY_STATUS,
+				 val, val & STOP_STATE_CLK_LANE, 1000,
+				 PHY_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev,
+			"failed to wait for phy clk lane stop state\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static int dw_mipi_dsi_get_lane_bps(struct dw_mipi_dsi *dsi)
+{
+	unsigned int i, pre;
+	unsigned long mpclk, pllref, tmp;
+	unsigned int m = 1, n = 1, target_mbps = 1000;
+	unsigned int max_mbps = dptdin_map[ARRAY_SIZE(dptdin_map) - 1].max_mbps;
+	int bpp;
+
+	bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
+	if (bpp < 0) {
+		dev_err(dsi->dev, "failed to get bpp for pixel format %d\n",
+			dsi->format);
+		return bpp;
+	}
+
+	mpclk = DIV_ROUND_UP(dsi->mode->clock, MSEC_PER_SEC);
+	if (mpclk) {
+		/* take 1 / 0.9, since mbps must big than bandwidth of RGB */
+		tmp = mpclk * (bpp / dsi->lanes) * 10 / 9;
+		if (tmp < max_mbps)
+			target_mbps = tmp;
+		else
+			dev_err(dsi->dev, "DPHY clock frequency is out of range\n");
+	}
+
+	pllref = DIV_ROUND_UP(clk_get_rate(dsi->pllref_clk), USEC_PER_SEC);
+	tmp = pllref;
+
+	for (i = 1; i < 6; i++) {
+		pre = pllref / i;
+		if ((tmp > (target_mbps % pre)) && (target_mbps / pre < 512)) {
+			tmp = target_mbps % pre;
+			n = i;
+			m = target_mbps / pre;
+		}
+		if (tmp == 0)
+			break;
+	}
+
+	dsi->lane_mbps = pllref / n * m;
+	dsi->input_div = n;
+	dsi->feedback_div = m;
+
+	return 0;
+}
+
+static int dw_mipi_dsi_host_attach(struct mipi_dsi_host *host,
+				   struct mipi_dsi_device *device)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+
+	if (device->lanes > dsi->pdata->max_data_lanes) {
+		dev_err(dsi->dev, "the number of data lanes(%u) is too many\n",
+				device->lanes);
+		return -EINVAL;
+	}
+
+	if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO_BURST) ||
+	    !(device->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)) {
+		dev_err(dsi->dev, "device mode is unsupported\n");
+		return -EINVAL;
+	}
+
+	dsi->lanes = device->lanes;
+	dsi->channel = device->channel;
+	dsi->format = device->format;
+	dsi->panel = of_drm_find_panel(device->dev.of_node);
+	if (dsi->panel)
+		return drm_panel_attach(dsi->panel, &dsi->connector);
+
+	return -EINVAL;
+}
+
+static int dw_mipi_dsi_host_detach(struct mipi_dsi_host *host,
+				   struct mipi_dsi_device *device)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+
+	drm_panel_detach(dsi->panel);
+
+	return 0;
+}
+
+static int dw_mipi_dsi_gen_pkt_hdr_write(struct dw_mipi_dsi *dsi, u32 val)
+{
+	int ret;
+
+	ret = readx_poll_timeout(readl, dsi->base + DSI_CMD_PKT_STATUS,
+				 val, !(val & GEN_CMD_FULL), 1000,
+				 CMD_PKT_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to get available command FIFO\n");
+		return ret;
+	}
+
+	dsi_write(dsi, DSI_GEN_HDR, val);
+
+	ret = readx_poll_timeout(readl, dsi->base + DSI_CMD_PKT_STATUS,
+				 val, val & (GEN_CMD_EMPTY | GEN_PLD_W_EMPTY),
+				 1000, CMD_PKT_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to write command FIFO\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dw_mipi_dsi_dcs_short_write(struct dw_mipi_dsi *dsi,
+				       const struct mipi_dsi_msg *msg)
+{
+	const u16 *tx_buf = msg->tx_buf;
+	u32 val = GEN_HDATA(*tx_buf) | GEN_HTYPE(msg->type);
+
+	if (msg->tx_len > 2) {
+		dev_err(dsi->dev, "too long tx buf length %zu for short write\n",
+			msg->tx_len);
+		return -EINVAL;
+	}
+
+	return dw_mipi_dsi_gen_pkt_hdr_write(dsi, val);
+}
+
+static int dw_mipi_dsi_dcs_long_write(struct dw_mipi_dsi *dsi,
+				      const struct mipi_dsi_msg *msg)
+{
+	const u32 *tx_buf = msg->tx_buf;
+	int len = msg->tx_len, pld_data_bytes = sizeof(*tx_buf), ret;
+	u32 val = GEN_HDATA(msg->tx_len) | GEN_HTYPE(msg->type);
+	u32 remainder = 0;
+
+	if (msg->tx_len < 3) {
+		dev_err(dsi->dev, "wrong tx buf length %zu for long write\n",
+			msg->tx_len);
+		return -EINVAL;
+	}
+
+	while (DIV_ROUND_UP(len, pld_data_bytes)) {
+		if (len < pld_data_bytes) {
+			memcpy(&remainder, tx_buf, len);
+			dsi_write(dsi, DSI_GEN_PLD_DATA, remainder);
+			len = 0;
+		} else {
+			dsi_write(dsi, DSI_GEN_PLD_DATA, *tx_buf);
+			tx_buf++;
+			len -= pld_data_bytes;
+		}
+
+		ret = readx_poll_timeout(readl, dsi->base + DSI_CMD_PKT_STATUS,
+					 val, !(val & GEN_PLD_W_FULL), 1000,
+					 CMD_PKT_STATUS_TIMEOUT_US);
+		if (ret < 0) {
+			dev_err(dsi->dev,
+				"failed to get available write payload FIFO\n");
+			return ret;
+		}
+	}
+
+	return dw_mipi_dsi_gen_pkt_hdr_write(dsi, val);
+}
+
+static ssize_t dw_mipi_dsi_host_transfer(struct mipi_dsi_host *host,
+					 const struct mipi_dsi_msg *msg)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+	int ret;
+
+	switch (msg->type) {
+	case MIPI_DSI_DCS_SHORT_WRITE:
+	case MIPI_DSI_DCS_SHORT_WRITE_PARAM:
+	case MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE:
+		ret = dw_mipi_dsi_dcs_short_write(dsi, msg);
+		break;
+	case MIPI_DSI_DCS_LONG_WRITE:
+		ret = dw_mipi_dsi_dcs_long_write(dsi, msg);
+		break;
+	default:
+		dev_err(dsi->dev, "unsupported message type\n");
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct mipi_dsi_host_ops dw_mipi_dsi_host_ops = {
+	.attach = dw_mipi_dsi_host_attach,
+	.detach = dw_mipi_dsi_host_detach,
+	.transfer = dw_mipi_dsi_host_transfer,
+};
+
+static void dw_mipi_dsi_video_mode_config(struct dw_mipi_dsi *dsi)
+{
+	u32 val;
+
+	val = VID_MODE_TYPE_BURST_SYNC_PULSES | ENABLE_LOW_POWER;
+
+	dsi_write(dsi, DSI_VID_MODE_CFG, val);
+}
+
+static void dw_mipi_dsi_set_mode(struct dw_mipi_dsi *dsi,
+				 enum dw_mipi_dsi_mode mode)
+{
+	if (mode == DW_MIPI_DSI_CMD_MODE) {
+		dsi_write(dsi, DSI_PWR_UP, RESET);
+		dsi_write(dsi, DSI_MODE_CFG, ENABLE_CMD_MODE);
+		dsi_write(dsi, DSI_PWR_UP, POWERUP);
+	} else {
+		dsi_write(dsi, DSI_PWR_UP, RESET);
+		dsi_write(dsi, DSI_MODE_CFG, ENABLE_VIDEO_MODE);
+		dw_mipi_dsi_video_mode_config(dsi);
+		dsi_write(dsi, DSI_PWR_UP, POWERUP);
+	}
+}
+
+static void dw_mipi_dsi_disable(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PWR_UP, RESET);
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_RSTZ);
+}
+
+static void dw_mipi_dsi_init(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PWR_UP, RESET);
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_DISFORCEPLL | PHY_DISABLECLK
+		  | PHY_RSTZ | PHY_SHUTDOWNZ);
+	dsi_write(dsi, DSI_CLKMGR_CFG, TO_CLK_DIVIDSION(10) |
+		  TX_ESC_CLK_DIVIDSION(7));
+	dsi_write(dsi, DSI_LPCLK_CTRL, PHY_TXREQUESTCLKHS);
+}
+
+static void dw_mipi_dsi_dpi_config(struct dw_mipi_dsi *dsi,
+				   struct drm_display_mode *mode)
+{
+	u32 val = 0, color = 0;
+
+	switch (dsi->format) {
+	case MIPI_DSI_FMT_RGB888:
+		color = DPI_COLOR_CODING_24BIT;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		color = DPI_COLOR_CODING_18BIT_2 | EN18_LOOSELY;
+		break;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		color = DPI_COLOR_CODING_18BIT_1;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		color = DPI_COLOR_CODING_16BIT_1;
+		break;
+	}
+
+	if (!(mode->flags & DRM_MODE_FLAG_PVSYNC))
+		val |= VSYNC_ACTIVE_LOW;
+	if (!(mode->flags & DRM_MODE_FLAG_PHSYNC))
+		val |= HSYNC_ACTIVE_LOW;
+
+	dsi_write(dsi, DSI_DPI_VCID, DPI_VID(dsi->channel));
+	dsi_write(dsi, DSI_DPI_COLOR_CODING, color);
+	dsi_write(dsi, DSI_DPI_CFG_POL, val);
+	dsi_write(dsi, DSI_DPI_LP_CMD_TIM, OUTVACT_LPCMD_TIME(4)
+		  | INVACT_LPCMD_TIME(4));
+}
+
+static void dw_mipi_dsi_packet_handler_config(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PCKHDL_CFG, EN_CRC_RX | EN_ECC_RX | EN_BTA);
+}
+
+static void dw_mipi_dsi_video_packet_config(struct dw_mipi_dsi *dsi,
+					    struct drm_display_mode *mode)
+{
+	dsi_write(dsi, DSI_VID_PKT_SIZE, VID_PKT_SIZE(mode->hdisplay));
+}
+
+static void dw_mipi_dsi_command_mode_config(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_TO_CNT_CFG, HSTX_TO_CNT(1000) | LPRX_TO_CNT(1000));
+	dsi_write(dsi, DSI_BTA_TO_CNT, 0xd00);
+	dsi_write(dsi, DSI_CMD_MODE_CFG, CMD_MODE_ALL_LP);
+	dsi_write(dsi, DSI_MODE_CFG, ENABLE_CMD_MODE);
+}
+
+/* Get lane byte clock cycles. */
+static u32 dw_mipi_dsi_get_hcomponent_lbcc(struct dw_mipi_dsi *dsi,
+					   u32 hcomponent)
+{
+	u32 frac, lbcc;
+
+	lbcc = hcomponent * dsi->lane_mbps * MSEC_PER_SEC / 8;
+
+	frac = lbcc % dsi->mode->clock;
+	lbcc = lbcc / dsi->mode->clock;
+	if (frac)
+		lbcc++;
+
+	return lbcc;
+}
+
+static void dw_mipi_dsi_line_timer_config(struct dw_mipi_dsi *dsi)
+{
+	u32 htotal, hsa, hbp, lbcc;
+	struct drm_display_mode *mode = dsi->mode;
+
+	htotal = mode->htotal;
+	hsa = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, htotal);
+	dsi_write(dsi, DSI_VID_HLINE_TIME, lbcc);
+
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, hsa);
+	dsi_write(dsi, DSI_VID_HSA_TIME, lbcc);
+
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, hbp);
+	dsi_write(dsi, DSI_VID_HBP_TIME, lbcc);
+}
+
+static void dw_mipi_dsi_vertical_timing_config(struct dw_mipi_dsi *dsi)
+{
+	u32 vactive, vsa, vfp, vbp;
+	struct drm_display_mode *mode = dsi->mode;
+
+	vactive = mode->vdisplay;
+	vsa = mode->vsync_end - mode->vsync_start;
+	vfp = mode->vsync_start - mode->vdisplay;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	dsi_write(dsi, DSI_VID_VACTIVE_LINES, vactive);
+	dsi_write(dsi, DSI_VID_VSA_LINES, vsa);
+	dsi_write(dsi, DSI_VID_VFP_LINES, vfp);
+	dsi_write(dsi, DSI_VID_VBP_LINES, vbp);
+}
+
+static void dw_mipi_dsi_dphy_timing_config(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PHY_TMR_CFG, PHY_HS2LP_TIME(0x40)
+		  | PHY_LP2HS_TIME(0x40) | MAX_RD_TIME(10000));
+
+	dsi_write(dsi, DSI_PHY_TMR_LPCLK_CFG, PHY_CLKHS2LP_TIME(0x40)
+		  | PHY_CLKLP2HS_TIME(0x40));
+}
+
+static void dw_mipi_dsi_dphy_interface_config(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PHY_IF_CFG, PHY_STOP_WAIT_TIME(0x20) |
+		  N_LANES(dsi->lanes));
+}
+
+static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi)
+{
+	dsi_read(dsi, DSI_INT_ST0);
+	dsi_read(dsi, DSI_INT_ST1);
+	dsi_write(dsi, DSI_INT_MSK0, 0);
+	dsi_write(dsi, DSI_INT_MSK1, 0);
+}
+
+static void dw_mipi_dsi_encoder_mode_set(struct drm_encoder *encoder,
+					struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	struct dw_mipi_dsi *dsi = encoder_to_dsi(encoder);
+	int ret;
+
+	dsi->mode = adjusted_mode;
+
+	ret = dw_mipi_dsi_get_lane_bps(dsi);
+	if (ret < 0)
+		return;
+
+	if (clk_prepare_enable(dsi->pclk)) {
+		dev_err(dsi->dev, "%s: Failed to enable pclk\n", __func__);
+		return;
+	}
+
+	dw_mipi_dsi_init(dsi);
+	dw_mipi_dsi_dpi_config(dsi, mode);
+	dw_mipi_dsi_packet_handler_config(dsi);
+	dw_mipi_dsi_video_mode_config(dsi);
+	dw_mipi_dsi_video_packet_config(dsi, mode);
+	dw_mipi_dsi_command_mode_config(dsi);
+	dw_mipi_dsi_line_timer_config(dsi);
+	dw_mipi_dsi_vertical_timing_config(dsi);
+	dw_mipi_dsi_dphy_timing_config(dsi);
+	dw_mipi_dsi_dphy_interface_config(dsi);
+	dw_mipi_dsi_clear_err(dsi);
+	if (drm_panel_prepare(dsi->panel))
+		dev_err(dsi->dev, "failed to prepare panel\n");
+
+	clk_disable_unprepare(dsi->pclk);
+}
+
+static void dw_mipi_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct dw_mipi_dsi *dsi = encoder_to_dsi(encoder);
+
+	drm_panel_disable(dsi->panel);
+
+	if (clk_prepare_enable(dsi->pclk)) {
+		dev_err(dsi->dev, "%s: Failed to enable pclk\n", __func__);
+		return;
+	}
+
+	dw_mipi_dsi_set_mode(dsi, DW_MIPI_DSI_CMD_MODE);
+	drm_panel_unprepare(dsi->panel);
+	dw_mipi_dsi_set_mode(dsi, DW_MIPI_DSI_VID_MODE);
+
+	/*
+	 * This is necessary to make sure the peripheral will be driven
+	 * normally when the display is enabled again later.
+	 */
+	msleep(120);
+
+	dw_mipi_dsi_set_mode(dsi, DW_MIPI_DSI_CMD_MODE);
+	dw_mipi_dsi_disable(dsi);
+	clk_disable_unprepare(dsi->pclk);
+}
+
+static bool dw_mipi_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void dw_mipi_dsi_encoder_commit(struct drm_encoder *encoder)
+{
+	struct dw_mipi_dsi *dsi = encoder_to_dsi(encoder);
+	int mux  = rockchip_drm_encoder_get_mux_id(dsi->dev->of_node, encoder);
+	u32 interface_pix_fmt;
+	u32 val;
+
+	if (clk_prepare_enable(dsi->pclk)) {
+		dev_err(dsi->dev, "%s: Failed to enable pclk\n", __func__);
+		return;
+	}
+
+	dw_mipi_dsi_phy_init(dsi);
+	dw_mipi_dsi_wait_for_two_frames(dsi);
+
+	dw_mipi_dsi_set_mode(dsi, DW_MIPI_DSI_VID_MODE);
+	drm_panel_enable(dsi->panel);
+
+	clk_disable_unprepare(dsi->pclk);
+
+	switch (dsi->format) {
+	case MIPI_DSI_FMT_RGB888:
+		interface_pix_fmt = ROCKCHIP_OUT_MODE_P888;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		interface_pix_fmt = ROCKCHIP_OUT_MODE_P666;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		interface_pix_fmt = ROCKCHIP_OUT_MODE_P565;
+		break;
+	default:
+		WARN_ON(1);
+		return;
+	}
+
+	rockchip_drm_crtc_mode_config(encoder->crtc, DRM_MODE_CONNECTOR_DSI,
+				      interface_pix_fmt);
+
+	if (mux)
+		val = DSI0_SEL_VOP_LIT | (DSI0_SEL_VOP_LIT << 16);
+	else
+		val = DSI0_SEL_VOP_LIT << 16;
+
+	regmap_write(dsi->grf_regmap, GRF_SOC_CON6, val);
+	dev_dbg(dsi->dev, "vop %s output to dsi0\n", (mux) ? "LIT" : "BIG");
+}
+
+static struct drm_encoder_helper_funcs
+dw_mipi_dsi_encoder_helper_funcs = {
+	.mode_fixup = dw_mipi_dsi_encoder_mode_fixup,
+	.commit = dw_mipi_dsi_encoder_commit,
+	.mode_set = dw_mipi_dsi_encoder_mode_set,
+	.disable = dw_mipi_dsi_encoder_disable,
+};
+
+static struct drm_encoder_funcs dw_mipi_dsi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int dw_mipi_dsi_connector_get_modes(struct drm_connector *connector)
+{
+	struct dw_mipi_dsi *dsi = con_to_dsi(connector);
+
+	return drm_panel_get_modes(dsi->panel);
+}
+
+static enum drm_mode_status dw_mipi_dsi_mode_valid(
+					struct drm_connector *connector,
+					struct drm_display_mode *mode)
+{
+	struct dw_mipi_dsi *dsi = con_to_dsi(connector);
+
+	enum drm_mode_status mode_status = MODE_OK;
+
+	if (dsi->pdata->mode_valid)
+		mode_status = dsi->pdata->mode_valid(connector, mode);
+
+	return mode_status;
+}
+
+static struct drm_encoder *dw_mipi_dsi_connector_best_encoder(
+					struct drm_connector *connector)
+{
+	struct dw_mipi_dsi *dsi = con_to_dsi(connector);
+
+	return &dsi->encoder;
+}
+
+static struct drm_connector_helper_funcs dw_mipi_dsi_connector_helper_funcs = {
+	.get_modes = dw_mipi_dsi_connector_get_modes,
+	.mode_valid = dw_mipi_dsi_mode_valid,
+	.best_encoder = dw_mipi_dsi_connector_best_encoder,
+};
+
+static enum drm_connector_status
+dw_mipi_dsi_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static void dw_mipi_dsi_drm_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static struct drm_connector_funcs dw_mipi_dsi_atomic_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = dw_mipi_dsi_detect,
+	.destroy = dw_mipi_dsi_drm_connector_destroy,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int dw_mipi_dsi_register(struct drm_device *drm,
+				      struct dw_mipi_dsi *dsi)
+{
+	struct drm_encoder *encoder = &dsi->encoder;
+	struct drm_connector *connector = &dsi->connector;
+	struct device *dev = dsi->dev;
+	int ret;
+
+	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm,
+							     dev->of_node);
+	/*
+	 * If we failed to find the CRTC(s) which this encoder is
+	 * supposed to be connected to, it's because the CRTC has
+	 * not been registered yet.  Defer probing, and hope that
+	 * the required CRTC is added later.
+	 */
+	if (encoder->possible_crtcs == 0)
+		return -EPROBE_DEFER;
+
+	drm_encoder_helper_add(&dsi->encoder,
+			       &dw_mipi_dsi_encoder_helper_funcs);
+	ret = drm_encoder_init(drm, &dsi->encoder, &dw_mipi_dsi_encoder_funcs,
+			 DRM_MODE_ENCODER_DSI, NULL);
+	if (ret) {
+		dev_err(dev, "Failed to initialize encoder with drm\n");
+		return ret;
+	}
+
+	drm_connector_helper_add(connector,
+			&dw_mipi_dsi_connector_helper_funcs);
+
+	drm_connector_init(drm, &dsi->connector,
+			   &dw_mipi_dsi_atomic_connector_funcs,
+			   DRM_MODE_CONNECTOR_DSI);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
+
+	return 0;
+}
+
+static int rockchip_mipi_parse_dt(struct dw_mipi_dsi *dsi)
+{
+	struct device_node *np = dsi->dev->of_node;
+
+	dsi->grf_regmap = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
+	if (IS_ERR(dsi->grf_regmap)) {
+		dev_err(dsi->dev, "Unable to get rockchip,grf\n");
+		return PTR_ERR(dsi->grf_regmap);
+	}
+
+	return 0;
+}
+
+static enum drm_mode_status rk3288_mipi_dsi_mode_valid(
+					struct drm_connector *connector,
+					struct drm_display_mode *mode)
+{
+	/*
+	 * The VID_PKT_SIZE field in the DSI_VID_PKT_CFG
+	 * register is 11-bit.
+	 */
+	if (mode->hdisplay > 0x7ff)
+		return MODE_BAD_HVALUE;
+
+	/*
+	 * The V_ACTIVE_LINES field in the DSI_VTIMING_CFG
+	 * register is 11-bit.
+	 */
+	if (mode->vdisplay > 0x7ff)
+		return MODE_BAD_VVALUE;
+
+	return MODE_OK;
+}
+
+static struct dw_mipi_dsi_plat_data rk3288_mipi_dsi_drv_data = {
+	.max_data_lanes = 4,
+	.mode_valid = rk3288_mipi_dsi_mode_valid,
+};
+
+static const struct of_device_id dw_mipi_dsi_dt_ids[] = {
+	{
+	 .compatible = "rockchip,rk3288-mipi-dsi",
+	 .data = &rk3288_mipi_dsi_drv_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, dw_mipi_dsi_dt_ids);
+
+static int dw_mipi_dsi_bind(struct device *dev, struct device *master,
+			     void *data)
+{
+	const struct of_device_id *of_id =
+			of_match_device(dw_mipi_dsi_dt_ids, dev);
+	const struct dw_mipi_dsi_plat_data *pdata = of_id->data;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = data;
+	struct dw_mipi_dsi *dsi;
+	struct resource *res;
+	int ret;
+
+	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+
+	dsi->dev = dev;
+	dsi->pdata = pdata;
+
+	ret = rockchip_mipi_parse_dt(dsi);
+	if (ret)
+		return ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	dsi->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(dsi->base))
+		return PTR_ERR(dsi->base);
+
+	dsi->pllref_clk = devm_clk_get(dev, "ref");
+	if (IS_ERR(dsi->pllref_clk)) {
+		ret = PTR_ERR(dsi->pllref_clk);
+		dev_err(dev, "Unable to get pll reference clock: %d\n", ret);
+		return ret;
+	}
+
+	dsi->pclk = devm_clk_get(dev, "pclk");
+	if (IS_ERR(dsi->pclk)) {
+		ret = PTR_ERR(dsi->pclk);
+		dev_err(dev, "Unable to get pclk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(dsi->pllref_clk);
+	if (ret) {
+		dev_err(dev, "%s: Failed to enable pllref_clk\n", __func__);
+		return ret;
+	}
+
+	ret = dw_mipi_dsi_register(drm, dsi);
+	if (ret) {
+		dev_err(dev, "Failed to register mipi_dsi: %d\n", ret);
+		goto err_pllref;
+	}
+
+	dev_set_drvdata(dev, dsi);
+
+	dsi->dsi_host.ops = &dw_mipi_dsi_host_ops;
+	dsi->dsi_host.dev = dev;
+	return mipi_dsi_host_register(&dsi->dsi_host);
+
+err_pllref:
+	clk_disable_unprepare(dsi->pllref_clk);
+	return ret;
+}
+
+static void dw_mipi_dsi_unbind(struct device *dev, struct device *master,
+	void *data)
+{
+	struct dw_mipi_dsi *dsi = dev_get_drvdata(dev);
+
+	mipi_dsi_host_unregister(&dsi->dsi_host);
+	clk_disable_unprepare(dsi->pllref_clk);
+}
+
+static const struct component_ops dw_mipi_dsi_ops = {
+	.bind	= dw_mipi_dsi_bind,
+	.unbind	= dw_mipi_dsi_unbind,
+};
+
+static int dw_mipi_dsi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dw_mipi_dsi_ops);
+}
+
+static int dw_mipi_dsi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dw_mipi_dsi_ops);
+	return 0;
+}
+
+static struct platform_driver dw_mipi_dsi_driver = {
+	.probe		= dw_mipi_dsi_probe,
+	.remove		= dw_mipi_dsi_remove,
+	.driver		= {
+		.of_match_table = dw_mipi_dsi_dt_ids,
+		.name	= DRIVER_NAME,
+	},
+};
+module_platform_driver(dw_mipi_dsi_driver);
+
+MODULE_DESCRIPTION("ROCKCHIP MIPI DSI host controller driver");
+MODULE_AUTHOR("Chris Zhong <zyw@rock-chips.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index 80d6fc8a5cee..c65ce8cb30d3 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -173,7 +173,7 @@ dw_hdmi_rockchip_mode_valid(struct drm_connector *connector,
 	return (valid) ? MODE_OK : MODE_BAD;
 }
 
-static struct drm_encoder_funcs dw_hdmi_rockchip_encoder_funcs = {
+static const struct drm_encoder_funcs dw_hdmi_rockchip_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
@@ -195,12 +195,15 @@ static void dw_hdmi_rockchip_encoder_mode_set(struct drm_encoder *encoder,
 {
 }
 
-static void dw_hdmi_rockchip_encoder_commit(struct drm_encoder *encoder)
+static void dw_hdmi_rockchip_encoder_enable(struct drm_encoder *encoder)
 {
 	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
 	u32 val;
 	int mux;
 
+	rockchip_drm_crtc_mode_config(encoder->crtc, DRM_MODE_CONNECTOR_HDMIA,
+				      ROCKCHIP_OUT_MODE_AAAA);
+
 	mux = rockchip_drm_encoder_get_mux_id(hdmi->dev->of_node, encoder);
 	if (mux)
 		val = HDMI_SEL_VOP_LIT | (HDMI_SEL_VOP_LIT << 16);
@@ -212,17 +215,10 @@ static void dw_hdmi_rockchip_encoder_commit(struct drm_encoder *encoder)
 		(mux) ? "LIT" : "BIG");
 }
 
-static void dw_hdmi_rockchip_encoder_prepare(struct drm_encoder *encoder)
-{
-	rockchip_drm_crtc_mode_config(encoder->crtc, DRM_MODE_CONNECTOR_HDMIA,
-				      ROCKCHIP_OUT_MODE_AAAA);
-}
-
-static struct drm_encoder_helper_funcs dw_hdmi_rockchip_encoder_helper_funcs = {
+static const struct drm_encoder_helper_funcs dw_hdmi_rockchip_encoder_helper_funcs = {
 	.mode_fixup = dw_hdmi_rockchip_encoder_mode_fixup,
 	.mode_set   = dw_hdmi_rockchip_encoder_mode_set,
-	.prepare    = dw_hdmi_rockchip_encoder_prepare,
-	.commit     = dw_hdmi_rockchip_encoder_commit,
+	.enable     = dw_hdmi_rockchip_encoder_enable,
 	.disable    = dw_hdmi_rockchip_encoder_disable,
 };
 
@@ -295,7 +291,7 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 
 	drm_encoder_helper_add(encoder, &dw_hdmi_rockchip_encoder_helper_funcs);
 	drm_encoder_init(drm, encoder, &dw_hdmi_rockchip_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	return dw_hdmi_bind(dev, master, data, encoder, iores, irq, plat_data);
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index f22e1e1ee64a..a0d51ccb6ea4 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -55,20 +55,18 @@ int rockchip_drm_dma_attach_device(struct drm_device *drm_dev,
 
 	return arm_iommu_attach_device(dev, mapping);
 }
-EXPORT_SYMBOL_GPL(rockchip_drm_dma_attach_device);
 
 void rockchip_drm_dma_detach_device(struct drm_device *drm_dev,
 				    struct device *dev)
 {
 	arm_iommu_detach_device(dev);
 }
-EXPORT_SYMBOL_GPL(rockchip_drm_dma_detach_device);
 
-int rockchip_register_crtc_funcs(struct drm_device *dev,
-				 const struct rockchip_crtc_funcs *crtc_funcs,
-				 int pipe)
+int rockchip_register_crtc_funcs(struct drm_crtc *crtc,
+				 const struct rockchip_crtc_funcs *crtc_funcs)
 {
-	struct rockchip_drm_private *priv = dev->dev_private;
+	int pipe = drm_crtc_index(crtc);
+	struct rockchip_drm_private *priv = crtc->dev->dev_private;
 
 	if (pipe > ROCKCHIP_MAX_CRTC)
 		return -EINVAL;
@@ -77,18 +75,17 @@ int rockchip_register_crtc_funcs(struct drm_device *dev,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(rockchip_register_crtc_funcs);
 
-void rockchip_unregister_crtc_funcs(struct drm_device *dev, int pipe)
+void rockchip_unregister_crtc_funcs(struct drm_crtc *crtc)
 {
-	struct rockchip_drm_private *priv = dev->dev_private;
+	int pipe = drm_crtc_index(crtc);
+	struct rockchip_drm_private *priv = crtc->dev->dev_private;
 
 	if (pipe > ROCKCHIP_MAX_CRTC)
 		return;
 
 	priv->crtc_funcs[pipe] = NULL;
 }
-EXPORT_SYMBOL_GPL(rockchip_unregister_crtc_funcs);
 
 static struct drm_crtc *rockchip_crtc_from_pipe(struct drm_device *drm,
 						int pipe)
@@ -139,6 +136,9 @@ static int rockchip_drm_load(struct drm_device *drm_dev, unsigned long flags)
 	if (!private)
 		return -ENOMEM;
 
+	mutex_init(&private->commit.lock);
+	INIT_WORK(&private->commit.work, rockchip_drm_atomic_work);
+
 	drm_dev->dev_private = private;
 
 	drm_mode_config_init(drm_dev);
@@ -212,6 +212,8 @@ static int rockchip_drm_load(struct drm_device *drm_dev, unsigned long flags)
 	 */
 	drm_dev->vblank_disable_allowed = true;
 
+	drm_mode_config_reset(drm_dev);
+
 	ret = rockchip_drm_fbdev_init(drm_dev);
 	if (ret)
 		goto err_vblank_cleanup;
@@ -275,7 +277,8 @@ const struct vm_operations_struct rockchip_drm_vm_ops = {
 };
 
 static struct drm_driver rockchip_drm_driver = {
-	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME,
+	.driver_features	= DRIVER_MODESET | DRIVER_GEM |
+				  DRIVER_PRIME | DRIVER_ATOMIC,
 	.load			= rockchip_drm_load,
 	.unload			= rockchip_drm_unload,
 	.lastclose		= rockchip_drm_lastclose,
@@ -450,10 +453,6 @@ static int rockchip_drm_bind(struct device *dev)
 	if (!drm)
 		return -ENOMEM;
 
-	ret = drm_dev_set_unique(drm, "%s", dev_name(dev));
-	if (ret)
-		goto err_free;
-
 	ret = drm_dev_register(drm, 0);
 	if (ret)
 		goto err_free;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index dc4e5f03ac79..bb8b076f1dbb 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -18,6 +18,7 @@
 #define _ROCKCHIP_DRM_DRV_H
 
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_gem.h>
 
 #include <linux/module.h>
@@ -38,6 +39,14 @@ struct drm_connector;
 struct rockchip_crtc_funcs {
 	int (*enable_vblank)(struct drm_crtc *crtc);
 	void (*disable_vblank)(struct drm_crtc *crtc);
+	void (*wait_for_update)(struct drm_crtc *crtc);
+};
+
+struct rockchip_atomic_commit {
+	struct work_struct	work;
+	struct drm_atomic_state *state;
+	struct drm_device *dev;
+	struct mutex lock;
 };
 
 /*
@@ -50,12 +59,14 @@ struct rockchip_drm_private {
 	struct drm_fb_helper fbdev_helper;
 	struct drm_gem_object *fbdev_bo;
 	const struct rockchip_crtc_funcs *crtc_funcs[ROCKCHIP_MAX_CRTC];
+
+	struct rockchip_atomic_commit commit;
 };
 
-int rockchip_register_crtc_funcs(struct drm_device *dev,
-				 const struct rockchip_crtc_funcs *crtc_funcs,
-				 int pipe);
-void rockchip_unregister_crtc_funcs(struct drm_device *dev, int pipe);
+void rockchip_drm_atomic_work(struct work_struct *work);
+int rockchip_register_crtc_funcs(struct drm_crtc *crtc,
+				 const struct rockchip_crtc_funcs *crtc_funcs);
+void rockchip_unregister_crtc_funcs(struct drm_crtc *crtc);
 int rockchip_drm_encoder_get_mux_id(struct device_node *node,
 				    struct drm_encoder *encoder);
 int rockchip_drm_crtc_mode_config(struct drm_crtc *crtc, int connector_type,
@@ -64,5 +75,4 @@ int rockchip_drm_dma_attach_device(struct drm_device *drm_dev,
 				   struct device *dev);
 void rockchip_drm_dma_detach_device(struct drm_device *drm_dev,
 				    struct device *dev);
-
 #endif /* _ROCKCHIP_DRM_DRV_H_ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index 002645bb5bbf..3b8f652698f8 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <drm/drm.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 
@@ -38,7 +39,6 @@ struct drm_gem_object *rockchip_fb_get_gem_obj(struct drm_framebuffer *fb,
 
 	return rk_fb->obj[plane];
 }
-EXPORT_SYMBOL_GPL(rockchip_fb_get_gem_obj);
 
 static void rockchip_drm_fb_destroy(struct drm_framebuffer *fb)
 {
@@ -66,13 +66,13 @@ static int rockchip_drm_fb_create_handle(struct drm_framebuffer *fb,
 				     rockchip_fb->obj[0], handle);
 }
 
-static struct drm_framebuffer_funcs rockchip_drm_fb_funcs = {
+static const struct drm_framebuffer_funcs rockchip_drm_fb_funcs = {
 	.destroy	= rockchip_drm_fb_destroy,
 	.create_handle	= rockchip_drm_fb_create_handle,
 };
 
 static struct rockchip_drm_fb *
-rockchip_fb_alloc(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd,
+rockchip_fb_alloc(struct drm_device *dev, const struct drm_mode_fb_cmd2 *mode_cmd,
 		  struct drm_gem_object **obj, unsigned int num_planes)
 {
 	struct rockchip_drm_fb *rockchip_fb;
@@ -102,7 +102,7 @@ rockchip_fb_alloc(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd,
 
 static struct drm_framebuffer *
 rockchip_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
-			struct drm_mode_fb_cmd2 *mode_cmd)
+			const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct rockchip_drm_fb *rockchip_fb;
 	struct drm_gem_object *objs[ROCKCHIP_MAX_FB_BUFFER];
@@ -166,14 +166,155 @@ static void rockchip_drm_output_poll_changed(struct drm_device *dev)
 		drm_fb_helper_hotplug_event(fb_helper);
 }
 
+static void rockchip_crtc_wait_for_update(struct drm_crtc *crtc)
+{
+	struct rockchip_drm_private *priv = crtc->dev->dev_private;
+	int pipe = drm_crtc_index(crtc);
+	const struct rockchip_crtc_funcs *crtc_funcs = priv->crtc_funcs[pipe];
+
+	if (crtc_funcs && crtc_funcs->wait_for_update)
+		crtc_funcs->wait_for_update(crtc);
+}
+
+/*
+ * We can't use drm_atomic_helper_wait_for_vblanks() because rk3288 and rk3066
+ * have hardware counters for neither vblanks nor scanlines, which results in
+ * a race where:
+ *				| <-- HW vsync irq and reg take effect
+ *	       plane_commit --> |
+ *	get_vblank and wait --> |
+ *				| <-- handle_vblank, vblank->count + 1
+ *		 cleanup_fb --> |
+ *		iommu crash --> |
+ *				| <-- HW vsync irq and reg take effect
+ *
+ * This function is equivalent but uses rockchip_crtc_wait_for_update() instead
+ * of waiting for vblank_count to change.
+ */
+static void
+rockchip_atomic_wait_for_complete(struct drm_device *dev, struct drm_atomic_state *old_state)
+{
+	struct drm_crtc_state *old_crtc_state;
+	struct drm_crtc *crtc;
+	int i, ret;
+
+	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
+		/* No one cares about the old state, so abuse it for tracking
+		 * and store whether we hold a vblank reference (and should do a
+		 * vblank wait) in the ->enable boolean.
+		 */
+		old_crtc_state->enable = false;
+
+		if (!crtc->state->active)
+			continue;
+
+		if (!drm_atomic_helper_framebuffer_changed(dev,
+				old_state, crtc))
+			continue;
+
+		ret = drm_crtc_vblank_get(crtc);
+		if (ret != 0)
+			continue;
+
+		old_crtc_state->enable = true;
+	}
+
+	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
+		if (!old_crtc_state->enable)
+			continue;
+
+		rockchip_crtc_wait_for_update(crtc);
+		drm_crtc_vblank_put(crtc);
+	}
+}
+
+static void
+rockchip_atomic_commit_complete(struct rockchip_atomic_commit *commit)
+{
+	struct drm_atomic_state *state = commit->state;
+	struct drm_device *dev = commit->dev;
+
+	/*
+	 * TODO: do fence wait here.
+	 */
+
+	/*
+	 * Rockchip crtc support runtime PM, can't update display planes
+	 * when crtc is disabled.
+	 *
+	 * drm_atomic_helper_commit comments detail that:
+	 *     For drivers supporting runtime PM the recommended sequence is
+	 *
+	 *     drm_atomic_helper_commit_modeset_disables(dev, state);
+	 *
+	 *     drm_atomic_helper_commit_modeset_enables(dev, state);
+	 *
+	 *     drm_atomic_helper_commit_planes(dev, state, true);
+	 *
+	 * See the kerneldoc entries for these three functions for more details.
+	 */
+	drm_atomic_helper_commit_modeset_disables(dev, state);
+
+	drm_atomic_helper_commit_modeset_enables(dev, state);
+
+	drm_atomic_helper_commit_planes(dev, state, true);
+
+	rockchip_atomic_wait_for_complete(dev, state);
+
+	drm_atomic_helper_cleanup_planes(dev, state);
+
+	drm_atomic_state_free(state);
+}
+
+void rockchip_drm_atomic_work(struct work_struct *work)
+{
+	struct rockchip_atomic_commit *commit = container_of(work,
+					struct rockchip_atomic_commit, work);
+
+	rockchip_atomic_commit_complete(commit);
+}
+
+int rockchip_drm_atomic_commit(struct drm_device *dev,
+			       struct drm_atomic_state *state,
+			       bool async)
+{
+	struct rockchip_drm_private *private = dev->dev_private;
+	struct rockchip_atomic_commit *commit = &private->commit;
+	int ret;
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (ret)
+		return ret;
+
+	/* serialize outstanding asynchronous commits */
+	mutex_lock(&commit->lock);
+	flush_work(&commit->work);
+
+	drm_atomic_helper_swap_state(dev, state);
+
+	commit->dev = dev;
+	commit->state = state;
+
+	if (async)
+		schedule_work(&commit->work);
+	else
+		rockchip_atomic_commit_complete(commit);
+
+	mutex_unlock(&commit->lock);
+
+	return 0;
+}
+
 static const struct drm_mode_config_funcs rockchip_drm_mode_config_funcs = {
 	.fb_create = rockchip_user_fb_create,
 	.output_poll_changed = rockchip_drm_output_poll_changed,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = rockchip_drm_atomic_commit,
 };
 
 struct drm_framebuffer *
 rockchip_drm_framebuffer_init(struct drm_device *dev,
-			      struct drm_mode_fb_cmd2 *mode_cmd,
+			      const struct drm_mode_fb_cmd2 *mode_cmd,
 			      struct drm_gem_object *obj)
 {
 	struct rockchip_drm_fb *rockchip_fb;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.h b/drivers/gpu/drm/rockchip/rockchip_drm_fb.h
index 09574d48226f..2fe47f1ee98f 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.h
@@ -17,7 +17,7 @@
 
 struct drm_framebuffer *
 rockchip_drm_framebuffer_init(struct drm_device *dev,
-			      struct drm_mode_fb_cmd2 *mode_cmd,
+			      const struct drm_mode_fb_cmd2 *mode_cmd,
 			      struct drm_gem_object *obj);
 void rockchip_drm_framebuffer_fini(struct drm_framebuffer *fb);
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.h b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.h
index 50432e9b5b37..73718c5f5bbf 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.h
@@ -15,7 +15,18 @@
 #ifndef _ROCKCHIP_DRM_FBDEV_H
 #define _ROCKCHIP_DRM_FBDEV_H
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 int rockchip_drm_fbdev_init(struct drm_device *dev);
 void rockchip_drm_fbdev_fini(struct drm_device *dev);
+#else
+static inline int rockchip_drm_fbdev_init(struct drm_device *dev)
+{
+	return 0;
+}
+
+static inline void rockchip_drm_fbdev_fini(struct drm_device *dev)
+{
+}
+#endif
 
 #endif /* _ROCKCHIP_DRM_FBDEV_H */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index d908321b94ce..18e07338c6e5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -234,13 +234,8 @@ int rockchip_gem_dumb_create(struct drm_file *file_priv,
 	/*
 	 * align to 64 bytes since Mali requires it.
 	 */
-	min_pitch = ALIGN(min_pitch, 64);
-
-	if (args->pitch < min_pitch)
-		args->pitch = min_pitch;
-
-	if (args->size < args->pitch * args->height)
-		args->size = args->pitch * args->height;
+	args->pitch = ALIGN(min_pitch, 64);
+	args->size = args->pitch * args->height;
 
 	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,
 						 &args->handle);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 03c47eeadc81..fd370548d7d7 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -14,6 +14,7 @@
 
 #include <drm/drm.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
@@ -35,11 +36,6 @@
 #include "rockchip_drm_fb.h"
 #include "rockchip_drm_vop.h"
 
-#define VOP_REG(off, _mask, s) \
-		{.offset = off, \
-		 .mask = _mask, \
-		 .shift = s,}
-
 #define __REG_SET_RELAXED(x, off, mask, shift, v) \
 		vop_mask_write_relaxed(x, off, (mask) << shift, (v) << shift)
 #define __REG_SET_NORMAL(x, off, mask, shift, v) \
@@ -47,14 +43,37 @@
 
 #define REG_SET(x, base, reg, v, mode) \
 		__REG_SET_##mode(x, base + reg.offset, reg.mask, reg.shift, v)
+#define REG_SET_MASK(x, base, reg, mask, v, mode) \
+		__REG_SET_##mode(x, base + reg.offset, mask, reg.shift, v)
 
 #define VOP_WIN_SET(x, win, name, v) \
 		REG_SET(x, win->base, win->phy->name, v, RELAXED)
 #define VOP_SCL_SET(x, win, name, v) \
 		REG_SET(x, win->base, win->phy->scl->name, v, RELAXED)
+#define VOP_SCL_SET_EXT(x, win, name, v) \
+		REG_SET(x, win->base, win->phy->scl->ext->name, v, RELAXED)
 #define VOP_CTRL_SET(x, name, v) \
 		REG_SET(x, 0, (x)->data->ctrl->name, v, NORMAL)
 
+#define VOP_INTR_GET(vop, name) \
+		vop_read_reg(vop, 0, &vop->data->ctrl->name)
+
+#define VOP_INTR_SET(vop, name, mask, v) \
+		REG_SET_MASK(vop, 0, vop->data->intr->name, mask, v, NORMAL)
+#define VOP_INTR_SET_TYPE(vop, name, type, v) \
+	do { \
+		int i, reg = 0, mask = 0; \
+		for (i = 0; i < vop->data->intr->nintrs; i++) { \
+			if (vop->data->intr->intrs[i] & type) { \
+				reg |= (v) << i; \
+				mask |= 1 << i; \
+			} \
+		} \
+		VOP_INTR_SET(vop, name, mask, reg); \
+	} while (0)
+#define VOP_INTR_GET_TYPE(vop, name, type) \
+		vop_get_intr_type(vop, &vop->data->intr->name, type)
+
 #define VOP_WIN_GET(x, win, name) \
 		vop_read_reg(x, win->base, &win->phy->name)
 
@@ -63,12 +82,15 @@
 
 #define to_vop(x) container_of(x, struct vop, crtc)
 #define to_vop_win(x) container_of(x, struct vop_win, base)
+#define to_vop_plane_state(x) container_of(x, struct vop_plane_state, base)
 
-struct vop_win_state {
-	struct list_head head;
-	struct drm_framebuffer *fb;
+struct vop_plane_state {
+	struct drm_plane_state base;
+	int format;
+	struct drm_rect src;
+	struct drm_rect dest;
 	dma_addr_t yrgb_mst;
-	struct drm_pending_vblank_event *event;
+	bool enable;
 };
 
 struct vop_win {
@@ -76,8 +98,7 @@ struct vop_win {
 	const struct vop_win_data *data;
 	struct vop *vop;
 
-	struct list_head pending;
-	struct vop_win_state *active;
+	struct vop_plane_state state;
 };
 
 struct vop {
@@ -86,13 +107,12 @@ struct vop {
 	struct drm_device *drm_dev;
 	bool is_enabled;
 
-	int connector_type;
-	int connector_out_mode;
-
 	/* mutex vsync_ work */
 	struct mutex vsync_mutex;
 	bool vsync_work_pending;
 	struct completion dsp_hold_completion;
+	struct completion wait_update_complete;
+	struct drm_pending_vblank_event *event;
 
 	const struct vop_data *data;
 
@@ -119,263 +139,9 @@ struct vop {
 	/* vop dclk reset */
 	struct reset_control *dclk_rst;
 
-	int pipe;
-
 	struct vop_win win[];
 };
 
-enum vop_data_format {
-	VOP_FMT_ARGB8888 = 0,
-	VOP_FMT_RGB888,
-	VOP_FMT_RGB565,
-	VOP_FMT_YUV420SP = 4,
-	VOP_FMT_YUV422SP,
-	VOP_FMT_YUV444SP,
-};
-
-struct vop_reg_data {
-	uint32_t offset;
-	uint32_t value;
-};
-
-struct vop_reg {
-	uint32_t offset;
-	uint32_t shift;
-	uint32_t mask;
-};
-
-struct vop_ctrl {
-	struct vop_reg standby;
-	struct vop_reg data_blank;
-	struct vop_reg gate_en;
-	struct vop_reg mmu_en;
-	struct vop_reg rgb_en;
-	struct vop_reg edp_en;
-	struct vop_reg hdmi_en;
-	struct vop_reg mipi_en;
-	struct vop_reg out_mode;
-	struct vop_reg dither_down;
-	struct vop_reg dither_up;
-	struct vop_reg pin_pol;
-
-	struct vop_reg htotal_pw;
-	struct vop_reg hact_st_end;
-	struct vop_reg vtotal_pw;
-	struct vop_reg vact_st_end;
-	struct vop_reg hpost_st_end;
-	struct vop_reg vpost_st_end;
-};
-
-struct vop_scl_regs {
-	struct vop_reg cbcr_vsd_mode;
-	struct vop_reg cbcr_vsu_mode;
-	struct vop_reg cbcr_hsd_mode;
-	struct vop_reg cbcr_ver_scl_mode;
-	struct vop_reg cbcr_hor_scl_mode;
-	struct vop_reg yrgb_vsd_mode;
-	struct vop_reg yrgb_vsu_mode;
-	struct vop_reg yrgb_hsd_mode;
-	struct vop_reg yrgb_ver_scl_mode;
-	struct vop_reg yrgb_hor_scl_mode;
-	struct vop_reg line_load_mode;
-	struct vop_reg cbcr_axi_gather_num;
-	struct vop_reg yrgb_axi_gather_num;
-	struct vop_reg vsd_cbcr_gt2;
-	struct vop_reg vsd_cbcr_gt4;
-	struct vop_reg vsd_yrgb_gt2;
-	struct vop_reg vsd_yrgb_gt4;
-	struct vop_reg bic_coe_sel;
-	struct vop_reg cbcr_axi_gather_en;
-	struct vop_reg yrgb_axi_gather_en;
-
-	struct vop_reg lb_mode;
-	struct vop_reg scale_yrgb_x;
-	struct vop_reg scale_yrgb_y;
-	struct vop_reg scale_cbcr_x;
-	struct vop_reg scale_cbcr_y;
-};
-
-struct vop_win_phy {
-	const struct vop_scl_regs *scl;
-	const uint32_t *data_formats;
-	uint32_t nformats;
-
-	struct vop_reg enable;
-	struct vop_reg format;
-	struct vop_reg rb_swap;
-	struct vop_reg act_info;
-	struct vop_reg dsp_info;
-	struct vop_reg dsp_st;
-	struct vop_reg yrgb_mst;
-	struct vop_reg uv_mst;
-	struct vop_reg yrgb_vir;
-	struct vop_reg uv_vir;
-
-	struct vop_reg dst_alpha_ctl;
-	struct vop_reg src_alpha_ctl;
-};
-
-struct vop_win_data {
-	uint32_t base;
-	const struct vop_win_phy *phy;
-	enum drm_plane_type type;
-};
-
-struct vop_data {
-	const struct vop_reg_data *init_table;
-	unsigned int table_size;
-	const struct vop_ctrl *ctrl;
-	const struct vop_win_data *win;
-	unsigned int win_size;
-};
-
-static const uint32_t formats_01[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ABGR8888,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_BGR888,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_BGR565,
-	DRM_FORMAT_NV12,
-	DRM_FORMAT_NV16,
-	DRM_FORMAT_NV24,
-};
-
-static const uint32_t formats_234[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ABGR8888,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_BGR888,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_BGR565,
-};
-
-static const struct vop_scl_regs win_full_scl = {
-	.cbcr_vsd_mode = VOP_REG(WIN0_CTRL1, 0x1, 31),
-	.cbcr_vsu_mode = VOP_REG(WIN0_CTRL1, 0x1, 30),
-	.cbcr_hsd_mode = VOP_REG(WIN0_CTRL1, 0x3, 28),
-	.cbcr_ver_scl_mode = VOP_REG(WIN0_CTRL1, 0x3, 26),
-	.cbcr_hor_scl_mode = VOP_REG(WIN0_CTRL1, 0x3, 24),
-	.yrgb_vsd_mode = VOP_REG(WIN0_CTRL1, 0x1, 23),
-	.yrgb_vsu_mode = VOP_REG(WIN0_CTRL1, 0x1, 22),
-	.yrgb_hsd_mode = VOP_REG(WIN0_CTRL1, 0x3, 20),
-	.yrgb_ver_scl_mode = VOP_REG(WIN0_CTRL1, 0x3, 18),
-	.yrgb_hor_scl_mode = VOP_REG(WIN0_CTRL1, 0x3, 16),
-	.line_load_mode = VOP_REG(WIN0_CTRL1, 0x1, 15),
-	.cbcr_axi_gather_num = VOP_REG(WIN0_CTRL1, 0x7, 12),
-	.yrgb_axi_gather_num = VOP_REG(WIN0_CTRL1, 0xf, 8),
-	.vsd_cbcr_gt2 = VOP_REG(WIN0_CTRL1, 0x1, 7),
-	.vsd_cbcr_gt4 = VOP_REG(WIN0_CTRL1, 0x1, 6),
-	.vsd_yrgb_gt2 = VOP_REG(WIN0_CTRL1, 0x1, 5),
-	.vsd_yrgb_gt4 = VOP_REG(WIN0_CTRL1, 0x1, 4),
-	.bic_coe_sel = VOP_REG(WIN0_CTRL1, 0x3, 2),
-	.cbcr_axi_gather_en = VOP_REG(WIN0_CTRL1, 0x1, 1),
-	.yrgb_axi_gather_en = VOP_REG(WIN0_CTRL1, 0x1, 0),
-	.lb_mode = VOP_REG(WIN0_CTRL0, 0x7, 5),
-	.scale_yrgb_x = VOP_REG(WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
-	.scale_yrgb_y = VOP_REG(WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
-	.scale_cbcr_x = VOP_REG(WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
-	.scale_cbcr_y = VOP_REG(WIN0_SCL_FACTOR_CBR, 0xffff, 16),
-};
-
-static const struct vop_win_phy win01_data = {
-	.scl = &win_full_scl,
-	.data_formats = formats_01,
-	.nformats = ARRAY_SIZE(formats_01),
-	.enable = VOP_REG(WIN0_CTRL0, 0x1, 0),
-	.format = VOP_REG(WIN0_CTRL0, 0x7, 1),
-	.rb_swap = VOP_REG(WIN0_CTRL0, 0x1, 12),
-	.act_info = VOP_REG(WIN0_ACT_INFO, 0x1fff1fff, 0),
-	.dsp_info = VOP_REG(WIN0_DSP_INFO, 0x0fff0fff, 0),
-	.dsp_st = VOP_REG(WIN0_DSP_ST, 0x1fff1fff, 0),
-	.yrgb_mst = VOP_REG(WIN0_YRGB_MST, 0xffffffff, 0),
-	.uv_mst = VOP_REG(WIN0_CBR_MST, 0xffffffff, 0),
-	.yrgb_vir = VOP_REG(WIN0_VIR, 0x3fff, 0),
-	.uv_vir = VOP_REG(WIN0_VIR, 0x3fff, 16),
-	.src_alpha_ctl = VOP_REG(WIN0_SRC_ALPHA_CTRL, 0xff, 0),
-	.dst_alpha_ctl = VOP_REG(WIN0_DST_ALPHA_CTRL, 0xff, 0),
-};
-
-static const struct vop_win_phy win23_data = {
-	.data_formats = formats_234,
-	.nformats = ARRAY_SIZE(formats_234),
-	.enable = VOP_REG(WIN2_CTRL0, 0x1, 0),
-	.format = VOP_REG(WIN2_CTRL0, 0x7, 1),
-	.rb_swap = VOP_REG(WIN2_CTRL0, 0x1, 12),
-	.dsp_info = VOP_REG(WIN2_DSP_INFO0, 0x0fff0fff, 0),
-	.dsp_st = VOP_REG(WIN2_DSP_ST0, 0x1fff1fff, 0),
-	.yrgb_mst = VOP_REG(WIN2_MST0, 0xffffffff, 0),
-	.yrgb_vir = VOP_REG(WIN2_VIR0_1, 0x1fff, 0),
-	.src_alpha_ctl = VOP_REG(WIN2_SRC_ALPHA_CTRL, 0xff, 0),
-	.dst_alpha_ctl = VOP_REG(WIN2_DST_ALPHA_CTRL, 0xff, 0),
-};
-
-static const struct vop_ctrl ctrl_data = {
-	.standby = VOP_REG(SYS_CTRL, 0x1, 22),
-	.gate_en = VOP_REG(SYS_CTRL, 0x1, 23),
-	.mmu_en = VOP_REG(SYS_CTRL, 0x1, 20),
-	.rgb_en = VOP_REG(SYS_CTRL, 0x1, 12),
-	.hdmi_en = VOP_REG(SYS_CTRL, 0x1, 13),
-	.edp_en = VOP_REG(SYS_CTRL, 0x1, 14),
-	.mipi_en = VOP_REG(SYS_CTRL, 0x1, 15),
-	.dither_down = VOP_REG(DSP_CTRL1, 0xf, 1),
-	.dither_up = VOP_REG(DSP_CTRL1, 0x1, 6),
-	.data_blank = VOP_REG(DSP_CTRL0, 0x1, 19),
-	.out_mode = VOP_REG(DSP_CTRL0, 0xf, 0),
-	.pin_pol = VOP_REG(DSP_CTRL0, 0xf, 4),
-	.htotal_pw = VOP_REG(DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
-	.hact_st_end = VOP_REG(DSP_HACT_ST_END, 0x1fff1fff, 0),
-	.vtotal_pw = VOP_REG(DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
-	.vact_st_end = VOP_REG(DSP_VACT_ST_END, 0x1fff1fff, 0),
-	.hpost_st_end = VOP_REG(POST_DSP_HACT_INFO, 0x1fff1fff, 0),
-	.vpost_st_end = VOP_REG(POST_DSP_VACT_INFO, 0x1fff1fff, 0),
-};
-
-static const struct vop_reg_data vop_init_reg_table[] = {
-	{SYS_CTRL, 0x00c00000},
-	{DSP_CTRL0, 0x00000000},
-	{WIN0_CTRL0, 0x00000080},
-	{WIN1_CTRL0, 0x00000080},
-	/* TODO: Win2/3 support multiple area function, but we haven't found
-	 * a suitable way to use it yet, so let's just use them as other windows
-	 * with only area 0 enabled.
-	 */
-	{WIN2_CTRL0, 0x00000010},
-	{WIN3_CTRL0, 0x00000010},
-};
-
-/*
- * Note: rk3288 has a dedicated 'cursor' window, however, that window requires
- * special support to get alpha blending working.  For now, just use overlay
- * window 3 for the drm cursor.
- *
- */
-static const struct vop_win_data rk3288_vop_win_data[] = {
-	{ .base = 0x00, .phy = &win01_data, .type = DRM_PLANE_TYPE_PRIMARY },
-	{ .base = 0x40, .phy = &win01_data, .type = DRM_PLANE_TYPE_OVERLAY },
-	{ .base = 0x00, .phy = &win23_data, .type = DRM_PLANE_TYPE_OVERLAY },
-	{ .base = 0x50, .phy = &win23_data, .type = DRM_PLANE_TYPE_CURSOR },
-};
-
-static const struct vop_data rk3288_vop = {
-	.init_table = vop_init_reg_table,
-	.table_size = ARRAY_SIZE(vop_init_reg_table),
-	.ctrl = &ctrl_data,
-	.win = rk3288_vop_win_data,
-	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
-};
-
-static const struct of_device_id vop_driver_dt_match[] = {
-	{ .compatible = "rockchip,rk3288-vop",
-	  .data = &rk3288_vop },
-	{},
-};
-MODULE_DEVICE_TABLE(of, vop_driver_dt_match);
-
 static inline void vop_writel(struct vop *vop, uint32_t offset, uint32_t v)
 {
 	writel(v, vop->regs + offset);
@@ -393,11 +159,6 @@ static inline uint32_t vop_read_reg(struct vop *vop, uint32_t base,
 	return (vop_readl(vop, base + reg->offset) >> reg->shift) & reg->mask;
 }
 
-static inline void vop_cfg_done(struct vop *vop)
-{
-	writel(0x01, vop->regs + REG_CFG_DONE);
-}
-
 static inline void vop_mask_write(struct vop *vop, uint32_t offset,
 				  uint32_t mask, uint32_t v)
 {
@@ -422,6 +183,25 @@ static inline void vop_mask_write_relaxed(struct vop *vop, uint32_t offset,
 	}
 }
 
+static inline uint32_t vop_get_intr_type(struct vop *vop,
+					 const struct vop_reg *reg, int type)
+{
+	uint32_t i, ret = 0;
+	uint32_t regs = vop_read_reg(vop, 0, reg);
+
+	for (i = 0; i < vop->data->intr->nintrs; i++) {
+		if ((type & vop->data->intr->intrs[i]) && (regs & 1 << i))
+			ret |= vop->data->intr->intrs[i];
+	}
+
+	return ret;
+}
+
+static inline void vop_cfg_done(struct vop *vop)
+{
+	VOP_CTRL_SET(vop, cfg_done, 1);
+}
+
 static bool has_rb_swapped(uint32_t format)
 {
 	switch (format) {
@@ -537,6 +317,20 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 		return;
 	}
 
+	if (!win->phy->scl->ext) {
+		VOP_SCL_SET(vop, win, scale_yrgb_x,
+			    scl_cal_scale2(src_w, dst_w));
+		VOP_SCL_SET(vop, win, scale_yrgb_y,
+			    scl_cal_scale2(src_h, dst_h));
+		if (is_yuv) {
+			VOP_SCL_SET(vop, win, scale_cbcr_x,
+				    scl_cal_scale2(src_w, dst_w));
+			VOP_SCL_SET(vop, win, scale_cbcr_y,
+				    scl_cal_scale2(src_h, dst_h));
+		}
+		return;
+	}
+
 	yrgb_hor_scl_mode = scl_get_scl_mode(src_w, dst_w);
 	yrgb_ver_scl_mode = scl_get_scl_mode(src_h, dst_h);
 
@@ -554,7 +348,7 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 			lb_mode = scl_vop_cal_lb_mode(src_w, false);
 	}
 
-	VOP_SCL_SET(vop, win, lb_mode, lb_mode);
+	VOP_SCL_SET_EXT(vop, win, lb_mode, lb_mode);
 	if (lb_mode == LB_RGB_3840X2) {
 		if (yrgb_ver_scl_mode != SCALE_NONE) {
 			DRM_ERROR("ERROR : not allow yrgb ver scale\n");
@@ -578,14 +372,14 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 				false, vsu_mode, &vskiplines);
 	VOP_SCL_SET(vop, win, scale_yrgb_y, val);
 
-	VOP_SCL_SET(vop, win, vsd_yrgb_gt4, vskiplines == 4);
-	VOP_SCL_SET(vop, win, vsd_yrgb_gt2, vskiplines == 2);
+	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt4, vskiplines == 4);
+	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt2, vskiplines == 2);
 
-	VOP_SCL_SET(vop, win, yrgb_hor_scl_mode, yrgb_hor_scl_mode);
-	VOP_SCL_SET(vop, win, yrgb_ver_scl_mode, yrgb_ver_scl_mode);
-	VOP_SCL_SET(vop, win, yrgb_hsd_mode, SCALE_DOWN_BIL);
-	VOP_SCL_SET(vop, win, yrgb_vsd_mode, SCALE_DOWN_BIL);
-	VOP_SCL_SET(vop, win, yrgb_vsu_mode, vsu_mode);
+	VOP_SCL_SET_EXT(vop, win, yrgb_hor_scl_mode, yrgb_hor_scl_mode);
+	VOP_SCL_SET_EXT(vop, win, yrgb_ver_scl_mode, yrgb_ver_scl_mode);
+	VOP_SCL_SET_EXT(vop, win, yrgb_hsd_mode, SCALE_DOWN_BIL);
+	VOP_SCL_SET_EXT(vop, win, yrgb_vsd_mode, SCALE_DOWN_BIL);
+	VOP_SCL_SET_EXT(vop, win, yrgb_vsu_mode, vsu_mode);
 	if (is_yuv) {
 		val = scl_vop_cal_scale(cbcr_hor_scl_mode, cbcr_src_w,
 					dst_w, true, 0, NULL);
@@ -594,13 +388,13 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 					dst_h, false, vsu_mode, &vskiplines);
 		VOP_SCL_SET(vop, win, scale_cbcr_y, val);
 
-		VOP_SCL_SET(vop, win, vsd_cbcr_gt4, vskiplines == 4);
-		VOP_SCL_SET(vop, win, vsd_cbcr_gt2, vskiplines == 2);
-		VOP_SCL_SET(vop, win, cbcr_hor_scl_mode, cbcr_hor_scl_mode);
-		VOP_SCL_SET(vop, win, cbcr_ver_scl_mode, cbcr_ver_scl_mode);
-		VOP_SCL_SET(vop, win, cbcr_hsd_mode, SCALE_DOWN_BIL);
-		VOP_SCL_SET(vop, win, cbcr_vsd_mode, SCALE_DOWN_BIL);
-		VOP_SCL_SET(vop, win, cbcr_vsu_mode, vsu_mode);
+		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt4, vskiplines == 4);
+		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt2, vskiplines == 2);
+		VOP_SCL_SET_EXT(vop, win, cbcr_hor_scl_mode, cbcr_hor_scl_mode);
+		VOP_SCL_SET_EXT(vop, win, cbcr_ver_scl_mode, cbcr_ver_scl_mode);
+		VOP_SCL_SET_EXT(vop, win, cbcr_hsd_mode, SCALE_DOWN_BIL);
+		VOP_SCL_SET_EXT(vop, win, cbcr_vsd_mode, SCALE_DOWN_BIL);
+		VOP_SCL_SET_EXT(vop, win, cbcr_vsu_mode, vsu_mode);
 	}
 }
 
@@ -613,8 +407,7 @@ static void vop_dsp_hold_valid_irq_enable(struct vop *vop)
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
 
-	vop_mask_write(vop, INTR_CTRL0, DSP_HOLD_VALID_INTR_MASK,
-		       DSP_HOLD_VALID_INTR_EN(1));
+	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 1);
 
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
@@ -628,8 +421,7 @@ static void vop_dsp_hold_valid_irq_disable(struct vop *vop)
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
 
-	vop_mask_write(vop, INTR_CTRL0, DSP_HOLD_VALID_INTR_MASK,
-		       DSP_HOLD_VALID_INTR_EN(0));
+	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 0);
 
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
@@ -692,7 +484,7 @@ static void vop_enable(struct drm_crtc *crtc)
 
 	enable_irq(vop->irq);
 
-	drm_vblank_on(vop->drm_dev, vop->pipe);
+	drm_crtc_vblank_on(crtc);
 
 	return;
 
@@ -704,14 +496,14 @@ err_disable_hclk:
 	clk_disable(vop->hclk);
 }
 
-static void vop_disable(struct drm_crtc *crtc)
+static void vop_crtc_disable(struct drm_crtc *crtc)
 {
 	struct vop *vop = to_vop(crtc);
 
 	if (!vop->is_enabled)
 		return;
 
-	drm_vblank_off(crtc->dev, vop->pipe);
+	drm_crtc_vblank_off(crtc);
 
 	/*
 	 * Vop standby will take effect at end of current frame,
@@ -748,224 +540,188 @@ static void vop_disable(struct drm_crtc *crtc)
 	pm_runtime_put(vop->dev);
 }
 
-/*
- * Caller must hold vsync_mutex.
- */
-static struct drm_framebuffer *vop_win_last_pending_fb(struct vop_win *vop_win)
-{
-	struct vop_win_state *last;
-	struct vop_win_state *active = vop_win->active;
-
-	if (list_empty(&vop_win->pending))
-		return active ? active->fb : NULL;
-
-	last = list_last_entry(&vop_win->pending, struct vop_win_state, head);
-	return last ? last->fb : NULL;
-}
-
-/*
- * Caller must hold vsync_mutex.
- */
-static int vop_win_queue_fb(struct vop_win *vop_win,
-			    struct drm_framebuffer *fb, dma_addr_t yrgb_mst,
-			    struct drm_pending_vblank_event *event)
+static void vop_plane_destroy(struct drm_plane *plane)
 {
-	struct vop_win_state *state;
-
-	state = kzalloc(sizeof(*state), GFP_KERNEL);
-	if (!state)
-		return -ENOMEM;
-
-	state->fb = fb;
-	state->yrgb_mst = yrgb_mst;
-	state->event = event;
-
-	list_add_tail(&state->head, &vop_win->pending);
-
-	return 0;
+	drm_plane_cleanup(plane);
 }
 
-static int vop_update_plane_event(struct drm_plane *plane,
-				  struct drm_crtc *crtc,
-				  struct drm_framebuffer *fb, int crtc_x,
-				  int crtc_y, unsigned int crtc_w,
-				  unsigned int crtc_h, uint32_t src_x,
-				  uint32_t src_y, uint32_t src_w,
-				  uint32_t src_h,
-				  struct drm_pending_vblank_event *event)
+static int vop_plane_atomic_check(struct drm_plane *plane,
+			   struct drm_plane_state *state)
 {
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_framebuffer *fb = state->fb;
 	struct vop_win *vop_win = to_vop_win(plane);
+	struct vop_plane_state *vop_plane_state = to_vop_plane_state(state);
 	const struct vop_win_data *win = vop_win->data;
-	struct vop *vop = to_vop(crtc);
-	struct drm_gem_object *obj;
-	struct rockchip_gem_object *rk_obj;
-	struct drm_gem_object *uv_obj;
-	struct rockchip_gem_object *rk_uv_obj;
-	unsigned long offset;
-	unsigned int actual_w;
-	unsigned int actual_h;
-	unsigned int dsp_stx;
-	unsigned int dsp_sty;
-	unsigned int y_vir_stride;
-	unsigned int uv_vir_stride = 0;
-	dma_addr_t yrgb_mst;
-	dma_addr_t uv_mst = 0;
-	enum vop_data_format format;
-	uint32_t val;
-	bool is_alpha;
-	bool rb_swap;
-	bool is_yuv;
 	bool visible;
 	int ret;
-	struct drm_rect dest = {
-		.x1 = crtc_x,
-		.y1 = crtc_y,
-		.x2 = crtc_x + crtc_w,
-		.y2 = crtc_y + crtc_h,
-	};
-	struct drm_rect src = {
-		/* 16.16 fixed point */
-		.x1 = src_x,
-		.y1 = src_y,
-		.x2 = src_x + src_w,
-		.y2 = src_y + src_h,
-	};
-	const struct drm_rect clip = {
-		.x2 = crtc->mode.hdisplay,
-		.y2 = crtc->mode.vdisplay,
-	};
-	bool can_position = plane->type != DRM_PLANE_TYPE_PRIMARY;
+	struct drm_rect *dest = &vop_plane_state->dest;
+	struct drm_rect *src = &vop_plane_state->src;
+	struct drm_rect clip;
 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
 					DRM_PLANE_HELPER_NO_SCALING;
 	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
 					DRM_PLANE_HELPER_NO_SCALING;
 
-	ret = drm_plane_helper_check_update(plane, crtc, fb,
-					    &src, &dest, &clip,
+	crtc = crtc ? crtc : plane->state->crtc;
+	/*
+	 * Both crtc or plane->state->crtc can be null.
+	 */
+	if (!crtc || !fb)
+		goto out_disable;
+	src->x1 = state->src_x;
+	src->y1 = state->src_y;
+	src->x2 = state->src_x + state->src_w;
+	src->y2 = state->src_y + state->src_h;
+	dest->x1 = state->crtc_x;
+	dest->y1 = state->crtc_y;
+	dest->x2 = state->crtc_x + state->crtc_w;
+	dest->y2 = state->crtc_y + state->crtc_h;
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc->mode.hdisplay;
+	clip.y2 = crtc->mode.vdisplay;
+
+	ret = drm_plane_helper_check_update(plane, crtc, state->fb,
+					    src, dest, &clip,
 					    min_scale,
 					    max_scale,
-					    can_position, false, &visible);
+					    true, true, &visible);
 	if (ret)
 		return ret;
 
 	if (!visible)
-		return 0;
-
-	is_alpha = is_alpha_support(fb->pixel_format);
-	rb_swap = has_rb_swapped(fb->pixel_format);
-	is_yuv = is_yuv_support(fb->pixel_format);
+		goto out_disable;
 
-	format = vop_convert_format(fb->pixel_format);
-	if (format < 0)
-		return format;
+	vop_plane_state->format = vop_convert_format(fb->pixel_format);
+	if (vop_plane_state->format < 0)
+		return vop_plane_state->format;
 
-	obj = rockchip_fb_get_gem_obj(fb, 0);
-	if (!obj) {
-		DRM_ERROR("fail to get rockchip gem object from framebuffer\n");
+	/*
+	 * Src.x1 can be odd when do clip, but yuv plane start point
+	 * need align with 2 pixel.
+	 */
+	if (is_yuv_support(fb->pixel_format) && ((src->x1 >> 16) % 2))
 		return -EINVAL;
-	}
 
-	rk_obj = to_rockchip_obj(obj);
+	vop_plane_state->enable = true;
 
-	if (is_yuv) {
-		/*
-		 * Src.x1 can be odd when do clip, but yuv plane start point
-		 * need align with 2 pixel.
-		 */
-		val = (src.x1 >> 16) % 2;
-		src.x1 += val << 16;
-		src.x2 += val << 16;
-	}
+	return 0;
 
-	actual_w = (src.x2 - src.x1) >> 16;
-	actual_h = (src.y2 - src.y1) >> 16;
+out_disable:
+	vop_plane_state->enable = false;
+	return 0;
+}
 
-	dsp_stx = dest.x1 + crtc->mode.htotal - crtc->mode.hsync_start;
-	dsp_sty = dest.y1 + crtc->mode.vtotal - crtc->mode.vsync_start;
+static void vop_plane_atomic_disable(struct drm_plane *plane,
+				     struct drm_plane_state *old_state)
+{
+	struct vop_plane_state *vop_plane_state = to_vop_plane_state(old_state);
+	struct vop_win *vop_win = to_vop_win(plane);
+	const struct vop_win_data *win = vop_win->data;
+	struct vop *vop = to_vop(old_state->crtc);
 
-	offset = (src.x1 >> 16) * drm_format_plane_cpp(fb->pixel_format, 0);
-	offset += (src.y1 >> 16) * fb->pitches[0];
+	if (!old_state->crtc)
+		return;
 
-	yrgb_mst = rk_obj->dma_addr + offset + fb->offsets[0];
-	y_vir_stride = fb->pitches[0] >> 2;
+	spin_lock(&vop->reg_lock);
 
-	if (is_yuv) {
-		int hsub = drm_format_horz_chroma_subsampling(fb->pixel_format);
-		int vsub = drm_format_vert_chroma_subsampling(fb->pixel_format);
-		int bpp = drm_format_plane_cpp(fb->pixel_format, 1);
+	VOP_WIN_SET(vop, win, enable, 0);
 
-		uv_obj = rockchip_fb_get_gem_obj(fb, 1);
-		if (!uv_obj) {
-			DRM_ERROR("fail to get uv object from framebuffer\n");
-			return -EINVAL;
-		}
-		rk_uv_obj = to_rockchip_obj(uv_obj);
-		uv_vir_stride = fb->pitches[1] >> 2;
+	spin_unlock(&vop->reg_lock);
 
-		offset = (src.x1 >> 16) * bpp / hsub;
-		offset += (src.y1 >> 16) * fb->pitches[1] / vsub;
+	vop_plane_state->enable = false;
+}
 
-		uv_mst = rk_uv_obj->dma_addr + offset + fb->offsets[1];
-	}
+static void vop_plane_atomic_update(struct drm_plane *plane,
+		struct drm_plane_state *old_state)
+{
+	struct drm_plane_state *state = plane->state;
+	struct drm_crtc *crtc = state->crtc;
+	struct vop_win *vop_win = to_vop_win(plane);
+	struct vop_plane_state *vop_plane_state = to_vop_plane_state(state);
+	const struct vop_win_data *win = vop_win->data;
+	struct vop *vop = to_vop(state->crtc);
+	struct drm_framebuffer *fb = state->fb;
+	unsigned int actual_w, actual_h;
+	unsigned int dsp_stx, dsp_sty;
+	uint32_t act_info, dsp_info, dsp_st;
+	struct drm_rect *src = &vop_plane_state->src;
+	struct drm_rect *dest = &vop_plane_state->dest;
+	struct drm_gem_object *obj, *uv_obj;
+	struct rockchip_gem_object *rk_obj, *rk_uv_obj;
+	unsigned long offset;
+	dma_addr_t dma_addr;
+	uint32_t val;
+	bool rb_swap;
 
 	/*
-	 * If this plane update changes the plane's framebuffer, (or more
-	 * precisely, if this update has a different framebuffer than the last
-	 * update), enqueue it so we can track when it completes.
-	 *
-	 * Only when we discover that this update has completed, can we
-	 * unreference any previous framebuffers.
+	 * can't update plane when vop is disabled.
 	 */
-	mutex_lock(&vop->vsync_mutex);
-	if (fb != vop_win_last_pending_fb(vop_win)) {
-		ret = drm_vblank_get(plane->dev, vop->pipe);
-		if (ret) {
-			DRM_ERROR("failed to get vblank, %d\n", ret);
-			mutex_unlock(&vop->vsync_mutex);
-			return ret;
-		}
-
-		drm_framebuffer_reference(fb);
+	if (!crtc)
+		return;
 
-		ret = vop_win_queue_fb(vop_win, fb, yrgb_mst, event);
-		if (ret) {
-			drm_vblank_put(plane->dev, vop->pipe);
-			mutex_unlock(&vop->vsync_mutex);
-			return ret;
-		}
+	if (WARN_ON(!vop->is_enabled))
+		return;
 
-		vop->vsync_work_pending = true;
+	if (!vop_plane_state->enable) {
+		vop_plane_atomic_disable(plane, old_state);
+		return;
 	}
-	mutex_unlock(&vop->vsync_mutex);
+
+	obj = rockchip_fb_get_gem_obj(fb, 0);
+	rk_obj = to_rockchip_obj(obj);
+
+	actual_w = drm_rect_width(src) >> 16;
+	actual_h = drm_rect_height(src) >> 16;
+	act_info = (actual_h - 1) << 16 | ((actual_w - 1) & 0xffff);
+
+	dsp_info = (drm_rect_height(dest) - 1) << 16;
+	dsp_info |= (drm_rect_width(dest) - 1) & 0xffff;
+
+	dsp_stx = dest->x1 + crtc->mode.htotal - crtc->mode.hsync_start;
+	dsp_sty = dest->y1 + crtc->mode.vtotal - crtc->mode.vsync_start;
+	dsp_st = dsp_sty << 16 | (dsp_stx & 0xffff);
+
+	offset = (src->x1 >> 16) * drm_format_plane_cpp(fb->pixel_format, 0);
+	offset += (src->y1 >> 16) * fb->pitches[0];
+	vop_plane_state->yrgb_mst = rk_obj->dma_addr + offset + fb->offsets[0];
 
 	spin_lock(&vop->reg_lock);
 
-	VOP_WIN_SET(vop, win, format, format);
-	VOP_WIN_SET(vop, win, yrgb_vir, y_vir_stride);
-	VOP_WIN_SET(vop, win, yrgb_mst, yrgb_mst);
-	if (is_yuv) {
-		VOP_WIN_SET(vop, win, uv_vir, uv_vir_stride);
-		VOP_WIN_SET(vop, win, uv_mst, uv_mst);
+	VOP_WIN_SET(vop, win, format, vop_plane_state->format);
+	VOP_WIN_SET(vop, win, yrgb_vir, fb->pitches[0] >> 2);
+	VOP_WIN_SET(vop, win, yrgb_mst, vop_plane_state->yrgb_mst);
+	if (is_yuv_support(fb->pixel_format)) {
+		int hsub = drm_format_horz_chroma_subsampling(fb->pixel_format);
+		int vsub = drm_format_vert_chroma_subsampling(fb->pixel_format);
+		int bpp = drm_format_plane_cpp(fb->pixel_format, 1);
+
+		uv_obj = rockchip_fb_get_gem_obj(fb, 1);
+		rk_uv_obj = to_rockchip_obj(uv_obj);
+
+		offset = (src->x1 >> 16) * bpp / hsub;
+		offset += (src->y1 >> 16) * fb->pitches[1] / vsub;
+
+		dma_addr = rk_uv_obj->dma_addr + offset + fb->offsets[1];
+		VOP_WIN_SET(vop, win, uv_vir, fb->pitches[1] >> 2);
+		VOP_WIN_SET(vop, win, uv_mst, dma_addr);
 	}
 
 	if (win->phy->scl)
 		scl_vop_cal_scl_fac(vop, win, actual_w, actual_h,
-				    dest.x2 - dest.x1, dest.y2 - dest.y1,
+				    drm_rect_width(dest), drm_rect_height(dest),
 				    fb->pixel_format);
 
-	val = (actual_h - 1) << 16;
-	val |= (actual_w - 1) & 0xffff;
-	VOP_WIN_SET(vop, win, act_info, val);
+	VOP_WIN_SET(vop, win, act_info, act_info);
+	VOP_WIN_SET(vop, win, dsp_info, dsp_info);
+	VOP_WIN_SET(vop, win, dsp_st, dsp_st);
 
-	val = (dest.y2 - dest.y1 - 1) << 16;
-	val |= (dest.x2 - dest.x1 - 1) & 0xffff;
-	VOP_WIN_SET(vop, win, dsp_info, val);
-	val = dsp_sty << 16;
-	val |= dsp_stx & 0xffff;
-	VOP_WIN_SET(vop, win, dsp_st, val);
+	rb_swap = has_rb_swapped(fb->pixel_format);
 	VOP_WIN_SET(vop, win, rb_swap, rb_swap);
 
-	if (is_alpha) {
+	if (is_alpha_support(fb->pixel_format)) {
 		VOP_WIN_SET(vop, win, dst_alpha_ctl,
 			    DST_FACTOR_M0(ALPHA_SRC_INVERSE));
 		val = SRC_ALPHA_EN(1) | SRC_COLOR_M0(ALPHA_SRC_PRE_MUL) |
@@ -979,86 +735,70 @@ static int vop_update_plane_event(struct drm_plane *plane,
 	}
 
 	VOP_WIN_SET(vop, win, enable, 1);
-
-	vop_cfg_done(vop);
 	spin_unlock(&vop->reg_lock);
-
-	return 0;
 }
 
-static int vop_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
-			    struct drm_framebuffer *fb, int crtc_x, int crtc_y,
-			    unsigned int crtc_w, unsigned int crtc_h,
-			    uint32_t src_x, uint32_t src_y, uint32_t src_w,
-			    uint32_t src_h)
-{
-	return vop_update_plane_event(plane, crtc, fb, crtc_x, crtc_y, crtc_w,
-				      crtc_h, src_x, src_y, src_w, src_h,
-				      NULL);
-}
+static const struct drm_plane_helper_funcs plane_helper_funcs = {
+	.atomic_check = vop_plane_atomic_check,
+	.atomic_update = vop_plane_atomic_update,
+	.atomic_disable = vop_plane_atomic_disable,
+};
 
-static int vop_update_primary_plane(struct drm_crtc *crtc,
-				    struct drm_pending_vblank_event *event)
+void vop_atomic_plane_reset(struct drm_plane *plane)
 {
-	unsigned int crtc_w, crtc_h;
+	struct vop_plane_state *vop_plane_state =
+					to_vop_plane_state(plane->state);
 
-	crtc_w = crtc->primary->fb->width - crtc->x;
-	crtc_h = crtc->primary->fb->height - crtc->y;
+	if (plane->state && plane->state->fb)
+		drm_framebuffer_unreference(plane->state->fb);
 
-	return vop_update_plane_event(crtc->primary, crtc, crtc->primary->fb,
-				      0, 0, crtc_w, crtc_h, crtc->x << 16,
-				      crtc->y << 16, crtc_w << 16,
-				      crtc_h << 16, event);
+	kfree(vop_plane_state);
+	vop_plane_state = kzalloc(sizeof(*vop_plane_state), GFP_KERNEL);
+	if (!vop_plane_state)
+		return;
+
+	plane->state = &vop_plane_state->base;
+	plane->state->plane = plane;
 }
 
-static int vop_disable_plane(struct drm_plane *plane)
+struct drm_plane_state *
+vop_atomic_plane_duplicate_state(struct drm_plane *plane)
 {
-	struct vop_win *vop_win = to_vop_win(plane);
-	const struct vop_win_data *win = vop_win->data;
-	struct vop *vop;
-	int ret;
+	struct vop_plane_state *old_vop_plane_state;
+	struct vop_plane_state *vop_plane_state;
 
-	if (!plane->crtc)
-		return 0;
+	if (WARN_ON(!plane->state))
+		return NULL;
 
-	vop = to_vop(plane->crtc);
+	old_vop_plane_state = to_vop_plane_state(plane->state);
+	vop_plane_state = kmemdup(old_vop_plane_state,
+				  sizeof(*vop_plane_state), GFP_KERNEL);
+	if (!vop_plane_state)
+		return NULL;
 
-	ret = drm_vblank_get(plane->dev, vop->pipe);
-	if (ret) {
-		DRM_ERROR("failed to get vblank, %d\n", ret);
-		return ret;
-	}
+	__drm_atomic_helper_plane_duplicate_state(plane,
+						  &vop_plane_state->base);
 
-	mutex_lock(&vop->vsync_mutex);
-
-	ret = vop_win_queue_fb(vop_win, NULL, 0, NULL);
-	if (ret) {
-		drm_vblank_put(plane->dev, vop->pipe);
-		mutex_unlock(&vop->vsync_mutex);
-		return ret;
-	}
-
-	vop->vsync_work_pending = true;
-	mutex_unlock(&vop->vsync_mutex);
-
-	spin_lock(&vop->reg_lock);
-	VOP_WIN_SET(vop, win, enable, 0);
-	vop_cfg_done(vop);
-	spin_unlock(&vop->reg_lock);
-
-	return 0;
+	return &vop_plane_state->base;
 }
 
-static void vop_plane_destroy(struct drm_plane *plane)
+static void vop_atomic_plane_destroy_state(struct drm_plane *plane,
+					   struct drm_plane_state *state)
 {
-	vop_disable_plane(plane);
-	drm_plane_cleanup(plane);
+	struct vop_plane_state *vop_state = to_vop_plane_state(state);
+
+	__drm_atomic_helper_plane_destroy_state(plane, state);
+
+	kfree(vop_state);
 }
 
 static const struct drm_plane_funcs vop_plane_funcs = {
-	.update_plane = vop_update_plane,
-	.disable_plane = vop_disable_plane,
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = vop_plane_destroy,
+	.reset = vop_atomic_plane_reset,
+	.atomic_duplicate_state = vop_atomic_plane_duplicate_state,
+	.atomic_destroy_state = vop_atomic_plane_destroy_state,
 };
 
 int rockchip_drm_crtc_mode_config(struct drm_crtc *crtc,
@@ -1067,8 +807,27 @@ int rockchip_drm_crtc_mode_config(struct drm_crtc *crtc,
 {
 	struct vop *vop = to_vop(crtc);
 
-	vop->connector_type = connector_type;
-	vop->connector_out_mode = out_mode;
+	if (WARN_ON(!vop->is_enabled))
+		return -EINVAL;
+
+	switch (connector_type) {
+	case DRM_MODE_CONNECTOR_LVDS:
+		VOP_CTRL_SET(vop, rgb_en, 1);
+		break;
+	case DRM_MODE_CONNECTOR_eDP:
+		VOP_CTRL_SET(vop, edp_en, 1);
+		break;
+	case DRM_MODE_CONNECTOR_HDMIA:
+		VOP_CTRL_SET(vop, hdmi_en, 1);
+		break;
+	case DRM_MODE_CONNECTOR_DSI:
+		VOP_CTRL_SET(vop, mipi_en, 1);
+		break;
+	default:
+		DRM_ERROR("unsupport connector_type[%d]\n", connector_type);
+		return -EINVAL;
+	};
+	VOP_CTRL_SET(vop, out_mode, out_mode);
 
 	return 0;
 }
@@ -1079,12 +838,12 @@ static int vop_crtc_enable_vblank(struct drm_crtc *crtc)
 	struct vop *vop = to_vop(crtc);
 	unsigned long flags;
 
-	if (!vop->is_enabled)
+	if (WARN_ON(!vop->is_enabled))
 		return -EPERM;
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
 
-	vop_mask_write(vop, INTR_CTRL0, FS_INTR_MASK, FS_INTR_EN(1));
+	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 1);
 
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 
@@ -1096,76 +855,49 @@ static void vop_crtc_disable_vblank(struct drm_crtc *crtc)
 	struct vop *vop = to_vop(crtc);
 	unsigned long flags;
 
-	if (!vop->is_enabled)
+	if (WARN_ON(!vop->is_enabled))
 		return;
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
-	vop_mask_write(vop, INTR_CTRL0, FS_INTR_MASK, FS_INTR_EN(0));
+
+	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 0);
+
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
 
-static const struct rockchip_crtc_funcs private_crtc_funcs = {
-	.enable_vblank = vop_crtc_enable_vblank,
-	.disable_vblank = vop_crtc_disable_vblank,
-};
-
-static void vop_crtc_dpms(struct drm_crtc *crtc, int mode)
+static void vop_crtc_wait_for_update(struct drm_crtc *crtc)
 {
-	DRM_DEBUG_KMS("crtc[%d] mode[%d]\n", crtc->base.id, mode);
+	struct vop *vop = to_vop(crtc);
 
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		vop_enable(crtc);
-		break;
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-	case DRM_MODE_DPMS_OFF:
-		vop_disable(crtc);
-		break;
-	default:
-		DRM_DEBUG_KMS("unspecified mode %d\n", mode);
-		break;
-	}
+	reinit_completion(&vop->wait_update_complete);
+	WARN_ON(!wait_for_completion_timeout(&vop->wait_update_complete, 100));
 }
 
-static void vop_crtc_prepare(struct drm_crtc *crtc)
-{
-	vop_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
-}
+static const struct rockchip_crtc_funcs private_crtc_funcs = {
+	.enable_vblank = vop_crtc_enable_vblank,
+	.disable_vblank = vop_crtc_disable_vblank,
+	.wait_for_update = vop_crtc_wait_for_update,
+};
 
 static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
 				const struct drm_display_mode *mode,
 				struct drm_display_mode *adjusted_mode)
 {
+	struct vop *vop = to_vop(crtc);
+
 	if (adjusted_mode->htotal == 0 || adjusted_mode->vtotal == 0)
 		return false;
 
-	return true;
-}
-
-static int vop_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-				  struct drm_framebuffer *old_fb)
-{
-	int ret;
-
-	crtc->x = x;
-	crtc->y = y;
+	adjusted_mode->clock =
+		clk_round_rate(vop->dclk, mode->clock * 1000) / 1000;
 
-	ret = vop_update_primary_plane(crtc, NULL);
-	if (ret < 0) {
-		DRM_ERROR("fail to update plane\n");
-		return ret;
-	}
-
-	return 0;
+	return true;
 }
 
-static int vop_crtc_mode_set(struct drm_crtc *crtc,
-			     struct drm_display_mode *mode,
-			     struct drm_display_mode *adjusted_mode,
-			     int x, int y, struct drm_framebuffer *fb)
+static void vop_crtc_enable(struct drm_crtc *crtc)
 {
 	struct vop *vop = to_vop(crtc);
+	struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode;
 	u16 hsync_len = adjusted_mode->hsync_end - adjusted_mode->hsync_start;
 	u16 hdisplay = adjusted_mode->hdisplay;
 	u16 htotal = adjusted_mode->htotal;
@@ -1176,32 +908,44 @@ static int vop_crtc_mode_set(struct drm_crtc *crtc,
 	u16 vsync_len = adjusted_mode->vsync_end - adjusted_mode->vsync_start;
 	u16 vact_st = adjusted_mode->vtotal - adjusted_mode->vsync_start;
 	u16 vact_end = vact_st + vdisplay;
-	int ret, ret_clk;
 	uint32_t val;
 
+	vop_enable(crtc);
 	/*
-	 * disable dclk to stop frame scan, so that we can safe config mode and
-	 * enable iommu.
+	 * If dclk rate is zero, mean that scanout is stop,
+	 * we don't need wait any more.
 	 */
-	clk_disable(vop->dclk);
+	if (clk_get_rate(vop->dclk)) {
+		/*
+		 * Rk3288 vop timing register is immediately, when configure
+		 * display timing on display time, may cause tearing.
+		 *
+		 * Vop standby will take effect at end of current frame,
+		 * if dsp hold valid irq happen, it means standby complete.
+		 *
+		 * mode set:
+		 *    standby and wait complete --> |----
+		 *                                  | display time
+		 *                                  |----
+		 *                                  |---> dsp hold irq
+		 *     configure display timing --> |
+		 *         standby exit             |
+		 *                                  | new frame start.
+		 */
 
-	switch (vop->connector_type) {
-	case DRM_MODE_CONNECTOR_LVDS:
-		VOP_CTRL_SET(vop, rgb_en, 1);
-		break;
-	case DRM_MODE_CONNECTOR_eDP:
-		VOP_CTRL_SET(vop, edp_en, 1);
-		break;
-	case DRM_MODE_CONNECTOR_HDMIA:
-		VOP_CTRL_SET(vop, hdmi_en, 1);
-		break;
-	default:
-		DRM_ERROR("unsupport connector_type[%d]\n",
-			  vop->connector_type);
-		ret = -EINVAL;
-		goto out;
-	};
-	VOP_CTRL_SET(vop, out_mode, vop->connector_out_mode);
+		reinit_completion(&vop->dsp_hold_completion);
+		vop_dsp_hold_valid_irq_enable(vop);
+
+		spin_lock(&vop->reg_lock);
+
+		VOP_CTRL_SET(vop, standby, 1);
+
+		spin_unlock(&vop->reg_lock);
+
+		wait_for_completion(&vop->dsp_hold_completion);
+
+		vop_dsp_hold_valid_irq_disable(vop);
+	}
 
 	val = 0x8;
 	val |= (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC) ? 0 : 1;
@@ -1220,211 +964,119 @@ static int vop_crtc_mode_set(struct drm_crtc *crtc,
 	VOP_CTRL_SET(vop, vact_st_end, val);
 	VOP_CTRL_SET(vop, vpost_st_end, val);
 
-	ret = vop_crtc_mode_set_base(crtc, x, y, fb);
-	if (ret)
-		goto out;
-
-	/*
-	 * reset dclk, take all mode config affect, so the clk would run in
-	 * correct frame.
-	 */
-	reset_control_assert(vop->dclk_rst);
-	usleep_range(10, 20);
-	reset_control_deassert(vop->dclk_rst);
-
 	clk_set_rate(vop->dclk, adjusted_mode->clock * 1000);
-out:
-	ret_clk = clk_enable(vop->dclk);
-	if (ret_clk < 0) {
-		dev_err(vop->dev, "failed to enable dclk - %d\n", ret_clk);
-		return ret_clk;
-	}
 
-	return ret;
-}
-
-static void vop_crtc_commit(struct drm_crtc *crtc)
-{
+	VOP_CTRL_SET(vop, standby, 0);
 }
 
-static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
-	.dpms = vop_crtc_dpms,
-	.prepare = vop_crtc_prepare,
-	.mode_fixup = vop_crtc_mode_fixup,
-	.mode_set = vop_crtc_mode_set,
-	.mode_set_base = vop_crtc_mode_set_base,
-	.commit = vop_crtc_commit,
-};
-
-static int vop_crtc_page_flip(struct drm_crtc *crtc,
-			      struct drm_framebuffer *fb,
-			      struct drm_pending_vblank_event *event,
-			      uint32_t page_flip_flags)
+static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_crtc_state)
 {
 	struct vop *vop = to_vop(crtc);
-	struct drm_framebuffer *old_fb = crtc->primary->fb;
-	int ret;
 
-	/* when the page flip is requested, crtc should be on */
-	if (!vop->is_enabled) {
-		DRM_DEBUG("page flip request rejected because crtc is off.\n");
-		return 0;
-	}
+	if (WARN_ON(!vop->is_enabled))
+		return;
 
-	crtc->primary->fb = fb;
+	spin_lock(&vop->reg_lock);
 
-	ret = vop_update_primary_plane(crtc, event);
-	if (ret)
-		crtc->primary->fb = old_fb;
+	vop_cfg_done(vop);
 
-	return ret;
+	spin_unlock(&vop->reg_lock);
 }
 
-static void vop_win_state_complete(struct vop_win *vop_win,
-				   struct vop_win_state *state)
+static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_crtc_state)
 {
-	struct vop *vop = vop_win->vop;
-	struct drm_crtc *crtc = &vop->crtc;
-	struct drm_device *drm = crtc->dev;
-	unsigned long flags;
+	struct vop *vop = to_vop(crtc);
 
-	if (state->event) {
-		spin_lock_irqsave(&drm->event_lock, flags);
-		drm_crtc_send_vblank_event(crtc, state->event);
-		spin_unlock_irqrestore(&drm->event_lock, flags);
-	}
+	if (crtc->state->event) {
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
 
-	list_del(&state->head);
-	drm_vblank_put(crtc->dev, vop->pipe);
+		vop->event = crtc->state->event;
+		crtc->state->event = NULL;
+	}
 }
 
+static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
+	.enable = vop_crtc_enable,
+	.disable = vop_crtc_disable,
+	.mode_fixup = vop_crtc_mode_fixup,
+	.atomic_flush = vop_crtc_atomic_flush,
+	.atomic_begin = vop_crtc_atomic_begin,
+};
+
 static void vop_crtc_destroy(struct drm_crtc *crtc)
 {
 	drm_crtc_cleanup(crtc);
 }
 
 static const struct drm_crtc_funcs vop_crtc_funcs = {
-	.set_config = drm_crtc_helper_set_config,
-	.page_flip = vop_crtc_page_flip,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
 	.destroy = vop_crtc_destroy,
+	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 };
 
-static bool vop_win_state_is_active(struct vop_win *vop_win,
-				    struct vop_win_state *state)
+static bool vop_win_pending_is_complete(struct vop_win *vop_win)
 {
-	bool active = false;
-
-	if (state->fb) {
-		dma_addr_t yrgb_mst;
-
-		/* check yrgb_mst to tell if pending_fb is now front */
-		yrgb_mst = VOP_WIN_GET_YRGBADDR(vop_win->vop, vop_win->data);
-
-		active = (yrgb_mst == state->yrgb_mst);
-	} else {
-		bool enabled;
-
-		/* if enable bit is clear, plane is now disabled */
-		enabled = VOP_WIN_GET(vop_win->vop, vop_win->data, enable);
-
-		active = (enabled == 0);
-	}
+	struct drm_plane *plane = &vop_win->base;
+	struct vop_plane_state *state = to_vop_plane_state(plane->state);
+	dma_addr_t yrgb_mst;
 
-	return active;
-}
+	if (!state->enable)
+		return VOP_WIN_GET(vop_win->vop, vop_win->data, enable) == 0;
 
-static void vop_win_state_destroy(struct vop_win_state *state)
-{
-	struct drm_framebuffer *fb = state->fb;
+	yrgb_mst = VOP_WIN_GET_YRGBADDR(vop_win->vop, vop_win->data);
 
-	if (fb)
-		drm_framebuffer_unreference(fb);
-
-	kfree(state);
+	return yrgb_mst == state->yrgb_mst;
 }
 
-static void vop_win_update_state(struct vop_win *vop_win)
+static void vop_handle_vblank(struct vop *vop)
 {
-	struct vop_win_state *state, *n, *new_active = NULL;
-
-	/* Check if any pending states are now active */
-	list_for_each_entry(state, &vop_win->pending, head)
-		if (vop_win_state_is_active(vop_win, state)) {
-			new_active = state;
-			break;
-		}
-
-	if (!new_active)
-		return;
+	struct drm_device *drm = vop->drm_dev;
+	struct drm_crtc *crtc = &vop->crtc;
+	unsigned long flags;
+	int i;
 
-	/*
-	 * Destroy any 'skipped' pending states - states that were queued
-	 * before the newly active state.
-	 */
-	list_for_each_entry_safe(state, n, &vop_win->pending, head) {
-		if (state == new_active)
-			break;
-		vop_win_state_complete(vop_win, state);
-		vop_win_state_destroy(state);
+	for (i = 0; i < vop->data->win_size; i++) {
+		if (!vop_win_pending_is_complete(&vop->win[i]))
+			return;
 	}
 
-	vop_win_state_complete(vop_win, new_active);
-
-	if (vop_win->active)
-		vop_win_state_destroy(vop_win->active);
-	vop_win->active = new_active;
-}
-
-static bool vop_win_has_pending_state(struct vop_win *vop_win)
-{
-	return !list_empty(&vop_win->pending);
-}
-
-static irqreturn_t vop_isr_thread(int irq, void *data)
-{
-	struct vop *vop = data;
-	const struct vop_data *vop_data = vop->data;
-	unsigned int i;
-
-	mutex_lock(&vop->vsync_mutex);
-
-	if (!vop->vsync_work_pending)
-		goto done;
+	if (vop->event) {
+		spin_lock_irqsave(&drm->event_lock, flags);
 
-	vop->vsync_work_pending = false;
+		drm_crtc_send_vblank_event(crtc, vop->event);
+		drm_crtc_vblank_put(crtc);
+		vop->event = NULL;
 
-	for (i = 0; i < vop_data->win_size; i++) {
-		struct vop_win *vop_win = &vop->win[i];
-
-		vop_win_update_state(vop_win);
-		if (vop_win_has_pending_state(vop_win))
-			vop->vsync_work_pending = true;
+		spin_unlock_irqrestore(&drm->event_lock, flags);
 	}
-
-done:
-	mutex_unlock(&vop->vsync_mutex);
-
-	return IRQ_HANDLED;
+	if (!completion_done(&vop->wait_update_complete))
+		complete(&vop->wait_update_complete);
 }
 
 static irqreturn_t vop_isr(int irq, void *data)
 {
 	struct vop *vop = data;
-	uint32_t intr0_reg, active_irqs;
+	struct drm_crtc *crtc = &vop->crtc;
+	uint32_t active_irqs;
 	unsigned long flags;
 	int ret = IRQ_NONE;
 
 	/*
-	 * INTR_CTRL0 register has interrupt status, enable and clear bits, we
+	 * interrupt register has interrupt status, enable and clear bits, we
 	 * must hold irq_lock to avoid a race with enable/disable_vblank().
 	*/
 	spin_lock_irqsave(&vop->irq_lock, flags);
-	intr0_reg = vop_readl(vop, INTR_CTRL0);
-	active_irqs = intr0_reg & INTR_MASK;
+
+	active_irqs = VOP_INTR_GET_TYPE(vop, status, INTR_MASK);
 	/* Clear all active interrupt sources */
 	if (active_irqs)
-		vop_writel(vop, INTR_CTRL0,
-			   intr0_reg | (active_irqs << INTR_CLR_SHIFT));
+		VOP_INTR_SET_TYPE(vop, clear, active_irqs, 1);
+
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 
 	/* This is expected for vop iommu irqs, since the irq is shared */
@@ -1438,9 +1090,10 @@ static irqreturn_t vop_isr(int irq, void *data)
 	}
 
 	if (active_irqs & FS_INTR) {
-		drm_handle_vblank(vop->drm_dev, vop->pipe);
+		drm_crtc_handle_vblank(crtc);
+		vop_handle_vblank(vop);
 		active_irqs &= ~FS_INTR;
-		ret = (vop->vsync_work_pending) ? IRQ_WAKE_THREAD : IRQ_HANDLED;
+		ret = IRQ_HANDLED;
 	}
 
 	/* Unhandled irqs are spurious. */
@@ -1478,13 +1131,14 @@ static int vop_create_crtc(struct vop *vop)
 					       0, &vop_plane_funcs,
 					       win_data->phy->data_formats,
 					       win_data->phy->nformats,
-					       win_data->type);
+					       win_data->type, NULL);
 		if (ret) {
 			DRM_ERROR("failed to initialize plane\n");
 			goto err_cleanup_planes;
 		}
 
 		plane = &vop_win->base;
+		drm_plane_helper_add(plane, &plane_helper_funcs);
 		if (plane->type == DRM_PLANE_TYPE_PRIMARY)
 			primary = plane;
 		else if (plane->type == DRM_PLANE_TYPE_CURSOR)
@@ -1492,7 +1146,7 @@ static int vop_create_crtc(struct vop *vop)
 	}
 
 	ret = drm_crtc_init_with_planes(drm_dev, crtc, primary, cursor,
-					&vop_crtc_funcs);
+					&vop_crtc_funcs, NULL);
 	if (ret)
 		return ret;
 
@@ -1515,11 +1169,12 @@ static int vop_create_crtc(struct vop *vop)
 					       &vop_plane_funcs,
 					       win_data->phy->data_formats,
 					       win_data->phy->nformats,
-					       win_data->type);
+					       win_data->type, NULL);
 		if (ret) {
 			DRM_ERROR("failed to initialize overlay plane\n");
 			goto err_cleanup_crtc;
 		}
+		drm_plane_helper_add(&vop_win->base, &plane_helper_funcs);
 	}
 
 	port = of_get_child_by_name(dev->of_node, "port");
@@ -1530,9 +1185,9 @@ static int vop_create_crtc(struct vop *vop)
 	}
 
 	init_completion(&vop->dsp_hold_completion);
+	init_completion(&vop->wait_update_complete);
 	crtc->port = port;
-	vop->pipe = drm_crtc_index(crtc);
-	rockchip_register_crtc_funcs(drm_dev, &private_crtc_funcs, vop->pipe);
+	rockchip_register_crtc_funcs(crtc, &private_crtc_funcs);
 
 	return 0;
 
@@ -1548,7 +1203,7 @@ static void vop_destroy_crtc(struct vop *vop)
 {
 	struct drm_crtc *crtc = &vop->crtc;
 
-	rockchip_unregister_crtc_funcs(vop->drm_dev, vop->pipe);
+	rockchip_unregister_crtc_funcs(crtc);
 	of_node_put(crtc->port);
 	drm_crtc_cleanup(crtc);
 }
@@ -1664,14 +1319,12 @@ static void vop_win_init(struct vop *vop)
 
 		vop_win->data = win_data;
 		vop_win->vop = vop;
-		INIT_LIST_HEAD(&vop_win->pending);
 	}
 }
 
 static int vop_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
-	const struct of_device_id *of_id;
 	const struct vop_data *vop_data;
 	struct drm_device *drm_dev = data;
 	struct vop *vop;
@@ -1679,8 +1332,7 @@ static int vop_bind(struct device *dev, struct device *master, void *data)
 	size_t alloc_size;
 	int ret, irq;
 
-	of_id = of_match_device(vop_driver_dt_match, dev);
-	vop_data = of_id->data;
+	vop_data = of_device_get_match_data(dev);
 	if (!vop_data)
 		return -ENODEV;
 
@@ -1725,8 +1377,8 @@ static int vop_bind(struct device *dev, struct device *master, void *data)
 
 	mutex_init(&vop->vsync_mutex);
 
-	ret = devm_request_threaded_irq(dev, vop->irq, vop_isr, vop_isr_thread,
-					IRQF_SHARED, dev_name(dev), vop);
+	ret = devm_request_irq(dev, vop->irq, vop_isr,
+			       IRQF_SHARED, dev_name(dev), vop);
 	if (ret)
 		return ret;
 
@@ -1749,42 +1401,8 @@ static void vop_unbind(struct device *dev, struct device *master, void *data)
 	vop_destroy_crtc(vop);
 }
 
-static const struct component_ops vop_component_ops = {
+const struct component_ops vop_component_ops = {
 	.bind = vop_bind,
 	.unbind = vop_unbind,
 };
-
-static int vop_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-
-	if (!dev->of_node) {
-		dev_err(dev, "can't find vop devices\n");
-		return -ENODEV;
-	}
-
-	return component_add(dev, &vop_component_ops);
-}
-
-static int vop_remove(struct platform_device *pdev)
-{
-	component_del(&pdev->dev, &vop_component_ops);
-
-	return 0;
-}
-
-struct platform_driver vop_platform_driver = {
-	.probe = vop_probe,
-	.remove = vop_remove,
-	.driver = {
-		.name = "rockchip-vop",
-		.owner = THIS_MODULE,
-		.of_match_table = of_match_ptr(vop_driver_dt_match),
-	},
-};
-
-module_platform_driver(vop_platform_driver);
-
-MODULE_AUTHOR("Mark Yao <mark.yao@rock-chips.com>");
-MODULE_DESCRIPTION("ROCKCHIP VOP Driver");
-MODULE_LICENSE("GPL v2");
+EXPORT_SYMBOL_GPL(vop_component_ops);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index a2d4ddb896fa..071ff0be7a95 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -15,111 +15,125 @@
 #ifndef _ROCKCHIP_DRM_VOP_H
 #define _ROCKCHIP_DRM_VOP_H
 
-/* register definition */
-#define REG_CFG_DONE			0x0000
-#define VERSION_INFO			0x0004
-#define SYS_CTRL			0x0008
-#define SYS_CTRL1			0x000c
-#define DSP_CTRL0			0x0010
-#define DSP_CTRL1			0x0014
-#define DSP_BG				0x0018
-#define MCU_CTRL			0x001c
-#define INTR_CTRL0			0x0020
-#define INTR_CTRL1			0x0024
-#define WIN0_CTRL0			0x0030
-#define WIN0_CTRL1			0x0034
-#define WIN0_COLOR_KEY			0x0038
-#define WIN0_VIR			0x003c
-#define WIN0_YRGB_MST			0x0040
-#define WIN0_CBR_MST			0x0044
-#define WIN0_ACT_INFO			0x0048
-#define WIN0_DSP_INFO			0x004c
-#define WIN0_DSP_ST			0x0050
-#define WIN0_SCL_FACTOR_YRGB		0x0054
-#define WIN0_SCL_FACTOR_CBR		0x0058
-#define WIN0_SCL_OFFSET			0x005c
-#define WIN0_SRC_ALPHA_CTRL		0x0060
-#define WIN0_DST_ALPHA_CTRL		0x0064
-#define WIN0_FADING_CTRL		0x0068
-/* win1 register */
-#define WIN1_CTRL0			0x0070
-#define WIN1_CTRL1			0x0074
-#define WIN1_COLOR_KEY			0x0078
-#define WIN1_VIR			0x007c
-#define WIN1_YRGB_MST			0x0080
-#define WIN1_CBR_MST			0x0084
-#define WIN1_ACT_INFO			0x0088
-#define WIN1_DSP_INFO			0x008c
-#define WIN1_DSP_ST			0x0090
-#define WIN1_SCL_FACTOR_YRGB		0x0094
-#define WIN1_SCL_FACTOR_CBR		0x0098
-#define WIN1_SCL_OFFSET			0x009c
-#define WIN1_SRC_ALPHA_CTRL		0x00a0
-#define WIN1_DST_ALPHA_CTRL		0x00a4
-#define WIN1_FADING_CTRL		0x00a8
-/* win2 register */
-#define WIN2_CTRL0			0x00b0
-#define WIN2_CTRL1			0x00b4
-#define WIN2_VIR0_1			0x00b8
-#define WIN2_VIR2_3			0x00bc
-#define WIN2_MST0			0x00c0
-#define WIN2_DSP_INFO0			0x00c4
-#define WIN2_DSP_ST0			0x00c8
-#define WIN2_COLOR_KEY			0x00cc
-#define WIN2_MST1			0x00d0
-#define WIN2_DSP_INFO1			0x00d4
-#define WIN2_DSP_ST1			0x00d8
-#define WIN2_SRC_ALPHA_CTRL		0x00dc
-#define WIN2_MST2			0x00e0
-#define WIN2_DSP_INFO2			0x00e4
-#define WIN2_DSP_ST2			0x00e8
-#define WIN2_DST_ALPHA_CTRL		0x00ec
-#define WIN2_MST3			0x00f0
-#define WIN2_DSP_INFO3			0x00f4
-#define WIN2_DSP_ST3			0x00f8
-#define WIN2_FADING_CTRL		0x00fc
-/* win3 register */
-#define WIN3_CTRL0			0x0100
-#define WIN3_CTRL1			0x0104
-#define WIN3_VIR0_1			0x0108
-#define WIN3_VIR2_3			0x010c
-#define WIN3_MST0			0x0110
-#define WIN3_DSP_INFO0			0x0114
-#define WIN3_DSP_ST0			0x0118
-#define WIN3_COLOR_KEY			0x011c
-#define WIN3_MST1			0x0120
-#define WIN3_DSP_INFO1			0x0124
-#define WIN3_DSP_ST1			0x0128
-#define WIN3_SRC_ALPHA_CTRL		0x012c
-#define WIN3_MST2			0x0130
-#define WIN3_DSP_INFO2			0x0134
-#define WIN3_DSP_ST2			0x0138
-#define WIN3_DST_ALPHA_CTRL		0x013c
-#define WIN3_MST3			0x0140
-#define WIN3_DSP_INFO3			0x0144
-#define WIN3_DSP_ST3			0x0148
-#define WIN3_FADING_CTRL		0x014c
-/* hwc register */
-#define HWC_CTRL0			0x0150
-#define HWC_CTRL1			0x0154
-#define HWC_MST				0x0158
-#define HWC_DSP_ST			0x015c
-#define HWC_SRC_ALPHA_CTRL		0x0160
-#define HWC_DST_ALPHA_CTRL		0x0164
-#define HWC_FADING_CTRL			0x0168
-/* post process register */
-#define POST_DSP_HACT_INFO		0x0170
-#define POST_DSP_VACT_INFO		0x0174
-#define POST_SCL_FACTOR_YRGB		0x0178
-#define POST_SCL_CTRL			0x0180
-#define POST_DSP_VACT_INFO_F1		0x0184
-#define DSP_HTOTAL_HS_END		0x0188
-#define DSP_HACT_ST_END			0x018c
-#define DSP_VTOTAL_VS_END		0x0190
-#define DSP_VACT_ST_END			0x0194
-#define DSP_VS_ST_END_F1		0x0198
-#define DSP_VACT_ST_END_F1		0x019c
-/* register definition end */
+enum vop_data_format {
+	VOP_FMT_ARGB8888 = 0,
+	VOP_FMT_RGB888,
+	VOP_FMT_RGB565,
+	VOP_FMT_YUV420SP = 4,
+	VOP_FMT_YUV422SP,
+	VOP_FMT_YUV444SP,
+};
+
+struct vop_reg_data {
+	uint32_t offset;
+	uint32_t value;
+};
+
+struct vop_reg {
+	uint32_t offset;
+	uint32_t shift;
+	uint32_t mask;
+};
+
+struct vop_ctrl {
+	struct vop_reg standby;
+	struct vop_reg data_blank;
+	struct vop_reg gate_en;
+	struct vop_reg mmu_en;
+	struct vop_reg rgb_en;
+	struct vop_reg edp_en;
+	struct vop_reg hdmi_en;
+	struct vop_reg mipi_en;
+	struct vop_reg out_mode;
+	struct vop_reg dither_down;
+	struct vop_reg dither_up;
+	struct vop_reg pin_pol;
+
+	struct vop_reg htotal_pw;
+	struct vop_reg hact_st_end;
+	struct vop_reg vtotal_pw;
+	struct vop_reg vact_st_end;
+	struct vop_reg hpost_st_end;
+	struct vop_reg vpost_st_end;
+
+	struct vop_reg cfg_done;
+};
+
+struct vop_intr {
+	const int *intrs;
+	uint32_t nintrs;
+	struct vop_reg enable;
+	struct vop_reg clear;
+	struct vop_reg status;
+};
+
+struct vop_scl_extension {
+	struct vop_reg cbcr_vsd_mode;
+	struct vop_reg cbcr_vsu_mode;
+	struct vop_reg cbcr_hsd_mode;
+	struct vop_reg cbcr_ver_scl_mode;
+	struct vop_reg cbcr_hor_scl_mode;
+	struct vop_reg yrgb_vsd_mode;
+	struct vop_reg yrgb_vsu_mode;
+	struct vop_reg yrgb_hsd_mode;
+	struct vop_reg yrgb_ver_scl_mode;
+	struct vop_reg yrgb_hor_scl_mode;
+	struct vop_reg line_load_mode;
+	struct vop_reg cbcr_axi_gather_num;
+	struct vop_reg yrgb_axi_gather_num;
+	struct vop_reg vsd_cbcr_gt2;
+	struct vop_reg vsd_cbcr_gt4;
+	struct vop_reg vsd_yrgb_gt2;
+	struct vop_reg vsd_yrgb_gt4;
+	struct vop_reg bic_coe_sel;
+	struct vop_reg cbcr_axi_gather_en;
+	struct vop_reg yrgb_axi_gather_en;
+	struct vop_reg lb_mode;
+};
+
+struct vop_scl_regs {
+	const struct vop_scl_extension *ext;
+
+	struct vop_reg scale_yrgb_x;
+	struct vop_reg scale_yrgb_y;
+	struct vop_reg scale_cbcr_x;
+	struct vop_reg scale_cbcr_y;
+};
+
+struct vop_win_phy {
+	const struct vop_scl_regs *scl;
+	const uint32_t *data_formats;
+	uint32_t nformats;
+
+	struct vop_reg enable;
+	struct vop_reg format;
+	struct vop_reg rb_swap;
+	struct vop_reg act_info;
+	struct vop_reg dsp_info;
+	struct vop_reg dsp_st;
+	struct vop_reg yrgb_mst;
+	struct vop_reg uv_mst;
+	struct vop_reg yrgb_vir;
+	struct vop_reg uv_vir;
+
+	struct vop_reg dst_alpha_ctl;
+	struct vop_reg src_alpha_ctl;
+};
+
+struct vop_win_data {
+	uint32_t base;
+	const struct vop_win_phy *phy;
+	enum drm_plane_type type;
+};
+
+struct vop_data {
+	const struct vop_reg_data *init_table;
+	unsigned int table_size;
+	const struct vop_ctrl *ctrl;
+	const struct vop_intr *intr;
+	const struct vop_win_data *win;
+	unsigned int win_size;
+};
 
 /* interrupt define */
 #define DSP_HOLD_VALID_INTR		(1 << 0)
@@ -233,6 +247,11 @@ static inline uint16_t scl_cal_scale(int src, int dst, int shift)
 	return ((src * 2 - 3) << (shift - 1)) / (dst - 1);
 }
 
+static inline uint16_t scl_cal_scale2(int src, int dst)
+{
+	return ((src - 1) << 12) / (dst - 1);
+}
+
 #define GET_SCL_FT_BILI_DN(src, dst)	scl_cal_scale(src, dst, 12)
 #define GET_SCL_FT_BILI_UP(src, dst)	scl_cal_scale(src, dst, 16)
 #define GET_SCL_FT_BIC(src, dst)	scl_cal_scale(src, dst, 16)
@@ -286,4 +305,5 @@ static inline int scl_vop_cal_lb_mode(int width, bool is_yuv)
 	return lb_mode;
 }
 
+extern const struct component_ops vop_component_ops;
 #endif /* _ROCKCHIP_DRM_VOP_H */
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
new file mode 100644
index 000000000000..3166b46a5893
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -0,0 +1,316 @@
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author:Mark Yao <mark.yao@rock-chips.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <drm/drmP.h>
+
+#include <linux/kernel.h>
+#include <linux/component.h>
+
+#include "rockchip_drm_vop.h"
+#include "rockchip_vop_reg.h"
+
+#define VOP_REG(off, _mask, s) \
+		{.offset = off, \
+		 .mask = _mask, \
+		 .shift = s,}
+
+static const uint32_t formats_win_full[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV16,
+	DRM_FORMAT_NV24,
+};
+
+static const uint32_t formats_win_lite[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+};
+
+static const struct vop_scl_extension rk3288_win_full_scl_ext = {
+	.cbcr_vsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 31),
+	.cbcr_vsu_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 30),
+	.cbcr_hsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 28),
+	.cbcr_ver_scl_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 26),
+	.cbcr_hor_scl_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 24),
+	.yrgb_vsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 23),
+	.yrgb_vsu_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 22),
+	.yrgb_hsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 20),
+	.yrgb_ver_scl_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 18),
+	.yrgb_hor_scl_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 16),
+	.line_load_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 15),
+	.cbcr_axi_gather_num = VOP_REG(RK3288_WIN0_CTRL1, 0x7, 12),
+	.yrgb_axi_gather_num = VOP_REG(RK3288_WIN0_CTRL1, 0xf, 8),
+	.vsd_cbcr_gt2 = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 7),
+	.vsd_cbcr_gt4 = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 6),
+	.vsd_yrgb_gt2 = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 5),
+	.vsd_yrgb_gt4 = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 4),
+	.bic_coe_sel = VOP_REG(RK3288_WIN0_CTRL1, 0x3, 2),
+	.cbcr_axi_gather_en = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 1),
+	.yrgb_axi_gather_en = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 0),
+	.lb_mode = VOP_REG(RK3288_WIN0_CTRL0, 0x7, 5),
+};
+
+static const struct vop_scl_regs rk3288_win_full_scl = {
+	.ext = &rk3288_win_full_scl_ext,
+	.scale_yrgb_x = VOP_REG(RK3288_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
+	.scale_yrgb_y = VOP_REG(RK3288_WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
+	.scale_cbcr_x = VOP_REG(RK3288_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
+	.scale_cbcr_y = VOP_REG(RK3288_WIN0_SCL_FACTOR_CBR, 0xffff, 16),
+};
+
+static const struct vop_win_phy rk3288_win01_data = {
+	.scl = &rk3288_win_full_scl,
+	.data_formats = formats_win_full,
+	.nformats = ARRAY_SIZE(formats_win_full),
+	.enable = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 0),
+	.format = VOP_REG(RK3288_WIN0_CTRL0, 0x7, 1),
+	.rb_swap = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 12),
+	.act_info = VOP_REG(RK3288_WIN0_ACT_INFO, 0x1fff1fff, 0),
+	.dsp_info = VOP_REG(RK3288_WIN0_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3288_WIN0_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3288_WIN0_YRGB_MST, 0xffffffff, 0),
+	.uv_mst = VOP_REG(RK3288_WIN0_CBR_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3288_WIN0_VIR, 0x3fff, 0),
+	.uv_vir = VOP_REG(RK3288_WIN0_VIR, 0x3fff, 16),
+	.src_alpha_ctl = VOP_REG(RK3288_WIN0_SRC_ALPHA_CTRL, 0xff, 0),
+	.dst_alpha_ctl = VOP_REG(RK3288_WIN0_DST_ALPHA_CTRL, 0xff, 0),
+};
+
+static const struct vop_win_phy rk3288_win23_data = {
+	.data_formats = formats_win_lite,
+	.nformats = ARRAY_SIZE(formats_win_lite),
+	.enable = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 0),
+	.format = VOP_REG(RK3288_WIN2_CTRL0, 0x7, 1),
+	.rb_swap = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 12),
+	.dsp_info = VOP_REG(RK3288_WIN2_DSP_INFO0, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3288_WIN2_DSP_ST0, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3288_WIN2_MST0, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3288_WIN2_VIR0_1, 0x1fff, 0),
+	.src_alpha_ctl = VOP_REG(RK3288_WIN2_SRC_ALPHA_CTRL, 0xff, 0),
+	.dst_alpha_ctl = VOP_REG(RK3288_WIN2_DST_ALPHA_CTRL, 0xff, 0),
+};
+
+static const struct vop_ctrl rk3288_ctrl_data = {
+	.standby = VOP_REG(RK3288_SYS_CTRL, 0x1, 22),
+	.gate_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 23),
+	.mmu_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 20),
+	.rgb_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 15),
+	.dither_down = VOP_REG(RK3288_DSP_CTRL1, 0xf, 1),
+	.dither_up = VOP_REG(RK3288_DSP_CTRL1, 0x1, 6),
+	.data_blank = VOP_REG(RK3288_DSP_CTRL0, 0x1, 19),
+	.out_mode = VOP_REG(RK3288_DSP_CTRL0, 0xf, 0),
+	.pin_pol = VOP_REG(RK3288_DSP_CTRL0, 0xf, 4),
+	.htotal_pw = VOP_REG(RK3288_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
+	.hact_st_end = VOP_REG(RK3288_DSP_HACT_ST_END, 0x1fff1fff, 0),
+	.vtotal_pw = VOP_REG(RK3288_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
+	.vact_st_end = VOP_REG(RK3288_DSP_VACT_ST_END, 0x1fff1fff, 0),
+	.hpost_st_end = VOP_REG(RK3288_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
+	.vpost_st_end = VOP_REG(RK3288_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
+	.cfg_done = VOP_REG(RK3288_REG_CFG_DONE, 0x1, 0),
+};
+
+static const struct vop_reg_data rk3288_init_reg_table[] = {
+	{RK3288_SYS_CTRL, 0x00c00000},
+	{RK3288_DSP_CTRL0, 0x00000000},
+	{RK3288_WIN0_CTRL0, 0x00000080},
+	{RK3288_WIN1_CTRL0, 0x00000080},
+	/* TODO: Win2/3 support multiple area function, but we haven't found
+	 * a suitable way to use it yet, so let's just use them as other windows
+	 * with only area 0 enabled.
+	 */
+	{RK3288_WIN2_CTRL0, 0x00000010},
+	{RK3288_WIN3_CTRL0, 0x00000010},
+};
+
+/*
+ * Note: rk3288 has a dedicated 'cursor' window, however, that window requires
+ * special support to get alpha blending working.  For now, just use overlay
+ * window 3 for the drm cursor.
+ *
+ */
+static const struct vop_win_data rk3288_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x40, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_OVERLAY },
+	{ .base = 0x00, .phy = &rk3288_win23_data,
+	  .type = DRM_PLANE_TYPE_OVERLAY },
+	{ .base = 0x50, .phy = &rk3288_win23_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const int rk3288_vop_intrs[] = {
+	DSP_HOLD_VALID_INTR,
+	FS_INTR,
+	LINE_FLAG_INTR,
+	BUS_ERROR_INTR,
+};
+
+static const struct vop_intr rk3288_vop_intr = {
+	.intrs = rk3288_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3288_vop_intrs),
+	.status = VOP_REG(RK3288_INTR_CTRL0, 0xf, 0),
+	.enable = VOP_REG(RK3288_INTR_CTRL0, 0xf, 4),
+	.clear = VOP_REG(RK3288_INTR_CTRL0, 0xf, 8),
+};
+
+static const struct vop_data rk3288_vop = {
+	.init_table = rk3288_init_reg_table,
+	.table_size = ARRAY_SIZE(rk3288_init_reg_table),
+	.intr = &rk3288_vop_intr,
+	.ctrl = &rk3288_ctrl_data,
+	.win = rk3288_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
+};
+
+static const struct vop_scl_regs rk3066_win_scl = {
+	.scale_yrgb_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
+	.scale_yrgb_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
+	.scale_cbcr_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
+	.scale_cbcr_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 16),
+};
+
+static const struct vop_win_phy rk3036_win0_data = {
+	.scl = &rk3066_win_scl,
+	.data_formats = formats_win_full,
+	.nformats = ARRAY_SIZE(formats_win_full),
+	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 0),
+	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 3),
+	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 15),
+	.act_info = VOP_REG(RK3036_WIN0_ACT_INFO, 0x1fff1fff, 0),
+	.dsp_info = VOP_REG(RK3036_WIN0_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3036_WIN0_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3036_WIN0_YRGB_MST, 0xffffffff, 0),
+	.uv_mst = VOP_REG(RK3036_WIN0_CBR_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3036_WIN0_VIR, 0xffff, 0),
+};
+
+static const struct vop_win_phy rk3036_win1_data = {
+	.data_formats = formats_win_lite,
+	.nformats = ARRAY_SIZE(formats_win_lite),
+	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 1),
+	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 6),
+	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 19),
+	.act_info = VOP_REG(RK3036_WIN1_ACT_INFO, 0x1fff1fff, 0),
+	.dsp_info = VOP_REG(RK3036_WIN1_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3036_WIN1_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3036_WIN1_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0),
+};
+
+static const struct vop_win_data rk3036_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3036_win0_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x00, .phy = &rk3036_win1_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const int rk3036_vop_intrs[] = {
+	DSP_HOLD_VALID_INTR,
+	FS_INTR,
+	LINE_FLAG_INTR,
+	BUS_ERROR_INTR,
+};
+
+static const struct vop_intr rk3036_intr = {
+	.intrs = rk3036_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3036_vop_intrs),
+	.status = VOP_REG(RK3036_INT_STATUS, 0xf, 0),
+	.enable = VOP_REG(RK3036_INT_STATUS, 0xf, 4),
+	.clear = VOP_REG(RK3036_INT_STATUS, 0xf, 8),
+};
+
+static const struct vop_ctrl rk3036_ctrl_data = {
+	.standby = VOP_REG(RK3036_SYS_CTRL, 0x1, 30),
+	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
+	.pin_pol = VOP_REG(RK3036_DSP_CTRL0, 0xf, 4),
+	.htotal_pw = VOP_REG(RK3036_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
+	.hact_st_end = VOP_REG(RK3036_DSP_HACT_ST_END, 0x1fff1fff, 0),
+	.vtotal_pw = VOP_REG(RK3036_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
+	.vact_st_end = VOP_REG(RK3036_DSP_VACT_ST_END, 0x1fff1fff, 0),
+	.cfg_done = VOP_REG(RK3036_REG_CFG_DONE, 0x1, 0),
+};
+
+static const struct vop_reg_data rk3036_vop_init_reg_table[] = {
+	{RK3036_DSP_CTRL1, 0x00000000},
+};
+
+static const struct vop_data rk3036_vop = {
+	.init_table = rk3036_vop_init_reg_table,
+	.table_size = ARRAY_SIZE(rk3036_vop_init_reg_table),
+	.ctrl = &rk3036_ctrl_data,
+	.intr = &rk3036_intr,
+	.win = rk3036_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
+};
+
+static const struct of_device_id vop_driver_dt_match[] = {
+	{ .compatible = "rockchip,rk3288-vop",
+	  .data = &rk3288_vop },
+	{ .compatible = "rockchip,rk3036-vop",
+	  .data = &rk3036_vop },
+	{},
+};
+MODULE_DEVICE_TABLE(of, vop_driver_dt_match);
+
+static int vop_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	if (!dev->of_node) {
+		dev_err(dev, "can't find vop devices\n");
+		return -ENODEV;
+	}
+
+	return component_add(dev, &vop_component_ops);
+}
+
+static int vop_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vop_component_ops);
+
+	return 0;
+}
+
+struct platform_driver vop_platform_driver = {
+	.probe = vop_probe,
+	.remove = vop_remove,
+	.driver = {
+		.name = "rockchip-vop",
+		.owner = THIS_MODULE,
+		.of_match_table = of_match_ptr(vop_driver_dt_match),
+	},
+};
+
+module_platform_driver(vop_platform_driver);
+
+MODULE_AUTHOR("Mark Yao <mark.yao@rock-chips.com>");
+MODULE_DESCRIPTION("ROCKCHIP VOP Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
new file mode 100644
index 000000000000..d4b46cba2f26
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author:Mark Yao <mark.yao@rock-chips.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _ROCKCHIP_VOP_REG_H
+#define _ROCKCHIP_VOP_REG_H
+
+/* rk3288 register definition */
+#define RK3288_REG_CFG_DONE			0x0000
+#define RK3288_VERSION_INFO			0x0004
+#define RK3288_SYS_CTRL				0x0008
+#define RK3288_SYS_CTRL1			0x000c
+#define RK3288_DSP_CTRL0			0x0010
+#define RK3288_DSP_CTRL1			0x0014
+#define RK3288_DSP_BG				0x0018
+#define RK3288_MCU_CTRL				0x001c
+#define RK3288_INTR_CTRL0			0x0020
+#define RK3288_INTR_CTRL1			0x0024
+#define RK3288_WIN0_CTRL0			0x0030
+#define RK3288_WIN0_CTRL1			0x0034
+#define RK3288_WIN0_COLOR_KEY			0x0038
+#define RK3288_WIN0_VIR				0x003c
+#define RK3288_WIN0_YRGB_MST			0x0040
+#define RK3288_WIN0_CBR_MST			0x0044
+#define RK3288_WIN0_ACT_INFO			0x0048
+#define RK3288_WIN0_DSP_INFO			0x004c
+#define RK3288_WIN0_DSP_ST			0x0050
+#define RK3288_WIN0_SCL_FACTOR_YRGB		0x0054
+#define RK3288_WIN0_SCL_FACTOR_CBR		0x0058
+#define RK3288_WIN0_SCL_OFFSET			0x005c
+#define RK3288_WIN0_SRC_ALPHA_CTRL		0x0060
+#define RK3288_WIN0_DST_ALPHA_CTRL		0x0064
+#define RK3288_WIN0_FADING_CTRL			0x0068
+
+/* win1 register */
+#define RK3288_WIN1_CTRL0			0x0070
+#define RK3288_WIN1_CTRL1			0x0074
+#define RK3288_WIN1_COLOR_KEY			0x0078
+#define RK3288_WIN1_VIR				0x007c
+#define RK3288_WIN1_YRGB_MST			0x0080
+#define RK3288_WIN1_CBR_MST			0x0084
+#define RK3288_WIN1_ACT_INFO			0x0088
+#define RK3288_WIN1_DSP_INFO			0x008c
+#define RK3288_WIN1_DSP_ST			0x0090
+#define RK3288_WIN1_SCL_FACTOR_YRGB		0x0094
+#define RK3288_WIN1_SCL_FACTOR_CBR		0x0098
+#define RK3288_WIN1_SCL_OFFSET			0x009c
+#define RK3288_WIN1_SRC_ALPHA_CTRL		0x00a0
+#define RK3288_WIN1_DST_ALPHA_CTRL		0x00a4
+#define RK3288_WIN1_FADING_CTRL			0x00a8
+/* win2 register */
+#define RK3288_WIN2_CTRL0			0x00b0
+#define RK3288_WIN2_CTRL1			0x00b4
+#define RK3288_WIN2_VIR0_1			0x00b8
+#define RK3288_WIN2_VIR2_3			0x00bc
+#define RK3288_WIN2_MST0			0x00c0
+#define RK3288_WIN2_DSP_INFO0			0x00c4
+#define RK3288_WIN2_DSP_ST0			0x00c8
+#define RK3288_WIN2_COLOR_KEY			0x00cc
+#define RK3288_WIN2_MST1			0x00d0
+#define RK3288_WIN2_DSP_INFO1			0x00d4
+#define RK3288_WIN2_DSP_ST1			0x00d8
+#define RK3288_WIN2_SRC_ALPHA_CTRL		0x00dc
+#define RK3288_WIN2_MST2			0x00e0
+#define RK3288_WIN2_DSP_INFO2			0x00e4
+#define RK3288_WIN2_DSP_ST2			0x00e8
+#define RK3288_WIN2_DST_ALPHA_CTRL		0x00ec
+#define RK3288_WIN2_MST3			0x00f0
+#define RK3288_WIN2_DSP_INFO3			0x00f4
+#define RK3288_WIN2_DSP_ST3			0x00f8
+#define RK3288_WIN2_FADING_CTRL			0x00fc
+/* win3 register */
+#define RK3288_WIN3_CTRL0			0x0100
+#define RK3288_WIN3_CTRL1			0x0104
+#define RK3288_WIN3_VIR0_1			0x0108
+#define RK3288_WIN3_VIR2_3			0x010c
+#define RK3288_WIN3_MST0			0x0110
+#define RK3288_WIN3_DSP_INFO0			0x0114
+#define RK3288_WIN3_DSP_ST0			0x0118
+#define RK3288_WIN3_COLOR_KEY			0x011c
+#define RK3288_WIN3_MST1			0x0120
+#define RK3288_WIN3_DSP_INFO1			0x0124
+#define RK3288_WIN3_DSP_ST1			0x0128
+#define RK3288_WIN3_SRC_ALPHA_CTRL		0x012c
+#define RK3288_WIN3_MST2			0x0130
+#define RK3288_WIN3_DSP_INFO2			0x0134
+#define RK3288_WIN3_DSP_ST2			0x0138
+#define RK3288_WIN3_DST_ALPHA_CTRL		0x013c
+#define RK3288_WIN3_MST3			0x0140
+#define RK3288_WIN3_DSP_INFO3			0x0144
+#define RK3288_WIN3_DSP_ST3			0x0148
+#define RK3288_WIN3_FADING_CTRL			0x014c
+/* hwc register */
+#define RK3288_HWC_CTRL0			0x0150
+#define RK3288_HWC_CTRL1			0x0154
+#define RK3288_HWC_MST				0x0158
+#define RK3288_HWC_DSP_ST			0x015c
+#define RK3288_HWC_SRC_ALPHA_CTRL		0x0160
+#define RK3288_HWC_DST_ALPHA_CTRL		0x0164
+#define RK3288_HWC_FADING_CTRL			0x0168
+/* post process register */
+#define RK3288_POST_DSP_HACT_INFO		0x0170
+#define RK3288_POST_DSP_VACT_INFO		0x0174
+#define RK3288_POST_SCL_FACTOR_YRGB		0x0178
+#define RK3288_POST_SCL_CTRL			0x0180
+#define RK3288_POST_DSP_VACT_INFO_F1		0x0184
+#define RK3288_DSP_HTOTAL_HS_END		0x0188
+#define RK3288_DSP_HACT_ST_END			0x018c
+#define RK3288_DSP_VTOTAL_VS_END		0x0190
+#define RK3288_DSP_VACT_ST_END			0x0194
+#define RK3288_DSP_VS_ST_END_F1			0x0198
+#define RK3288_DSP_VACT_ST_END_F1		0x019c
+/* register definition end */
+
+/* rk3036 register definition */
+#define RK3036_SYS_CTRL			0x00
+#define RK3036_DSP_CTRL0		0x04
+#define RK3036_DSP_CTRL1		0x08
+#define RK3036_INT_STATUS		0x10
+#define RK3036_ALPHA_CTRL		0x14
+#define RK3036_WIN0_COLOR_KEY		0x18
+#define RK3036_WIN1_COLOR_KEY		0x1c
+#define RK3036_WIN0_YRGB_MST		0x20
+#define RK3036_WIN0_CBR_MST		0x24
+#define RK3036_WIN1_VIR			0x28
+#define RK3036_AXI_BUS_CTRL		0x2c
+#define RK3036_WIN0_VIR			0x30
+#define RK3036_WIN0_ACT_INFO		0x34
+#define RK3036_WIN0_DSP_INFO		0x38
+#define RK3036_WIN0_DSP_ST		0x3c
+#define RK3036_WIN0_SCL_FACTOR_YRGB	0x40
+#define RK3036_WIN0_SCL_FACTOR_CBR	0x44
+#define RK3036_WIN0_SCL_OFFSET		0x48
+#define RK3036_HWC_MST			0x58
+#define RK3036_HWC_DSP_ST		0x5c
+#define RK3036_DSP_HTOTAL_HS_END	0x6c
+#define RK3036_DSP_HACT_ST_END		0x70
+#define RK3036_DSP_VTOTAL_VS_END	0x74
+#define RK3036_DSP_VACT_ST_END		0x78
+#define RK3036_DSP_VS_ST_END_F1		0x7c
+#define RK3036_DSP_VACT_ST_END_F1	0x80
+#define RK3036_GATHER_TRANSFER		0x84
+#define RK3036_VERSION_INFO		0x94
+#define RK3036_REG_CFG_DONE		0x90
+#define RK3036_WIN1_MST			0xa0
+#define RK3036_WIN1_ACT_INFO		0xb4
+#define RK3036_WIN1_DSP_INFO		0xb8
+#define RK3036_WIN1_DSP_ST		0xbc
+#define RK3036_WIN1_SCL_FACTOR_YRGB	0xc0
+#define RK3036_WIN1_SCL_OFFSET		0xc8
+#define RK3036_BCSH_CTRL		0xd0
+#define RK3036_BCSH_COLOR_BAR		0xd4
+#define RK3036_BCSH_BCS			0xd8
+#define RK3036_BCSH_H			0xdc
+#define RK3036_WIN1_LUT_ADDR		0x400
+#define RK3036_HWC_LUT_ADDR		0x800
+/* rk3036 register definition end */
+
+#endif /* _ROCKCHIP_VOP_REG_H */
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index b9202aa6f8ab..8d17d00ddb4b 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -1,6 +1,6 @@
 config DRM_SHMOBILE
 	tristate "DRM Support for SH Mobile"
-	depends on DRM && ARM && HAVE_DMA_ATTRS
+	depends on DRM && ARM
 	depends on ARCH_SHMOBILE || COMPILE_TEST
 	depends on FB_SH_MOBILE_MERAM || !FB_SH_MOBILE_MERAM
 	select BACKLIGHT_CLASS_DEVICE
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index e9272b0a8592..db0763794edc 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -613,7 +613,7 @@ int shmob_drm_encoder_create(struct shmob_drm_device *sdev)
 	encoder->possible_crtcs = 1;
 
 	ret = drm_encoder_init(sdev->ddev, encoder, &encoder_funcs,
-			       DRM_MODE_ENCODER_LVDS);
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret < 0)
 		return ret;
 
@@ -739,8 +739,6 @@ int shmob_drm_connector_create(struct shmob_drm_device *sdev,
 	if (ret < 0)
 		goto err_backlight;
 
-	connector->encoder = encoder;
-
 	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
 	drm_object_property_set_value(&connector->base,
 		sdev->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.c b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
index aaf98ace4a90..388a0fc13564 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
@@ -104,7 +104,7 @@ const struct shmob_drm_format_info *shmob_drm_format_info(u32 fourcc)
 
 static struct drm_framebuffer *
 shmob_drm_fb_create(struct drm_device *dev, struct drm_file *file_priv,
-		    struct drm_mode_fb_cmd2 *mode_cmd)
+		    const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	const struct shmob_drm_format_info *format;
 
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index 10c1b1926e6f..5ad43a1bb260 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -1,6 +1,6 @@
 config DRM_STI
 	tristate "DRM Support for STMicroelectronics SoC stiH41x Series"
-	depends on DRM && (SOC_STIH415 || SOC_STIH416 || ARCH_MULTIPLATFORM) && HAVE_DMA_ATTRS
+	depends on DRM && (SOC_STIH415 || SOC_STIH416 || ARCH_MULTIPLATFORM)
 	select RESET_CONTROLLER
 	select DRM_KMS_HELPER
 	select DRM_GEM_CMA_HELPER
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index 3ae09dcd4fd8..de11c7cfb02f 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -367,7 +367,7 @@ int sti_crtc_init(struct drm_device *drm_dev, struct sti_mixer *mixer,
 	int res;
 
 	res = drm_crtc_init_with_planes(drm_dev, crtc, primary, cursor,
-					&sti_crtc_funcs);
+					&sti_crtc_funcs, NULL);
 	if (res) {
 		DRM_ERROR("Can't initialze CRTC\n");
 		return -EINVAL;
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index dd1032195051..807863106b8d 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -272,7 +272,7 @@ struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
 				       &sti_plane_helpers_funcs,
 				       cursor_supported_formats,
 				       ARRAY_SIZE(cursor_supported_formats),
-				       DRM_PLANE_TYPE_CURSOR);
+				       DRM_PLANE_TYPE_CURSOR, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		goto err_plane;
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 1469987949d8..506b5626f3ed 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -160,6 +160,7 @@ static int sti_load(struct drm_device *dev, unsigned long flags)
 
 	drm_mode_config_reset(dev);
 
+	drm_helper_disable_unused_functions(dev);
 	drm_fbdev_cma_init(dev, 32,
 			   dev->mode_config.num_crtc,
 			   dev->mode_config.num_connector);
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index c85dc7d6b005..f9a1d92c9d95 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -630,7 +630,7 @@ struct drm_plane *sti_gdp_create(struct drm_device *drm_dev,
 				       &sti_plane_helpers_funcs,
 				       gdp_supported_formats,
 				       ARRAY_SIZE(gdp_supported_formats),
-				       type);
+				       type, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		goto err;
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index d735daccd458..49cce833f2c8 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -543,8 +543,6 @@ static int sti_hda_connector_get_modes(struct drm_connector *connector)
 		count++;
 	}
 
-	drm_mode_sort(&connector->modes);
-
 	return count;
 }
 
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index ea0690bc77d5..43861b52261d 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -973,7 +973,7 @@ static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
 				       &sti_plane_helpers_funcs,
 				       hqvdp_supported_formats,
 				       ARRAY_SIZE(hqvdp_supported_formats),
-				       DRM_PLANE_TYPE_OVERLAY);
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		return NULL;
diff --git a/drivers/gpu/drm/sti/sti_tvout.c b/drivers/gpu/drm/sti/sti_tvout.c
index c8a4c5dae2b6..f2afcf5438b8 100644
--- a/drivers/gpu/drm/sti/sti_tvout.c
+++ b/drivers/gpu/drm/sti/sti_tvout.c
@@ -512,7 +512,8 @@ sti_tvout_create_dvo_encoder(struct drm_device *dev,
 	drm_encoder->possible_clones = 1 << 0;
 
 	drm_encoder_init(dev, drm_encoder,
-			 &sti_tvout_encoder_funcs, DRM_MODE_ENCODER_LVDS);
+			 &sti_tvout_encoder_funcs, DRM_MODE_ENCODER_LVDS,
+			 NULL);
 
 	drm_encoder_helper_add(drm_encoder, &sti_dvo_encoder_helper_funcs);
 
@@ -564,7 +565,7 @@ static struct drm_encoder *sti_tvout_create_hda_encoder(struct drm_device *dev,
 	drm_encoder->possible_clones = 1 << 0;
 
 	drm_encoder_init(dev, drm_encoder,
-			&sti_tvout_encoder_funcs, DRM_MODE_ENCODER_DAC);
+			&sti_tvout_encoder_funcs, DRM_MODE_ENCODER_DAC, NULL);
 
 	drm_encoder_helper_add(drm_encoder, &sti_hda_encoder_helper_funcs);
 
@@ -613,7 +614,7 @@ static struct drm_encoder *sti_tvout_create_hdmi_encoder(struct drm_device *dev,
 	drm_encoder->possible_clones = 1 << 1;
 
 	drm_encoder_init(dev, drm_encoder,
-			&sti_tvout_encoder_funcs, DRM_MODE_ENCODER_TMDS);
+			&sti_tvout_encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(drm_encoder, &sti_hdmi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index 74d9d621453d..63ebb154b9b5 100644
--- a/drivers/gpu/drm/tegra/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -16,18 +16,6 @@ config DRM_TEGRA
 
 if DRM_TEGRA
 
-config DRM_TEGRA_FBDEV
-	bool "Enable legacy fbdev support"
-	select DRM_KMS_FB_HELPER
-	select FB_SYS_FILLRECT
-	select FB_SYS_COPYAREA
-	select FB_SYS_IMAGEBLIT
-	default y
-	help
-	  Choose this option if you have a need for the legacy fbdev support.
-	  Note that this support also provides the Linux console on top of
-	  the Tegra modesetting driver.
-
 config DRM_TEGRA_DEBUG
 	bool "NVIDIA Tegra DRM debug support"
 	help
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index e9f24a85a103..dde6f208c347 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -660,7 +660,8 @@ static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, possible_crtcs,
 				       &tegra_primary_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_PRIMARY);
+				       num_formats, DRM_PLANE_TYPE_PRIMARY,
+				       NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -827,7 +828,8 @@ static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
 				       &tegra_cursor_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_CURSOR);
+				       num_formats, DRM_PLANE_TYPE_CURSOR,
+				       NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -890,7 +892,8 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
 				       &tegra_overlay_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_OVERLAY);
+				       num_formats, DRM_PLANE_TYPE_OVERLAY,
+				       NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -1732,7 +1735,7 @@ static int tegra_dc_init(struct host1x_client *client)
 	}
 
 	err = drm_crtc_init_with_planes(drm, &dc->base, primary, cursor,
-					&tegra_crtc_funcs);
+					&tegra_crtc_funcs, NULL);
 	if (err < 0)
 		goto cleanup;
 
@@ -1952,8 +1955,10 @@ static int tegra_dc_parse_dt(struct tegra_dc *dc)
 		 * cases where only a single display controller is used.
 		 */
 		for_each_matching_node(np, tegra_dc_of_match) {
-			if (np == dc->dev->of_node)
+			if (np == dc->dev->of_node) {
+				of_node_put(np);
 				break;
+			}
 
 			value++;
 		}
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index 6aecb6647313..b24a0f14821a 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -436,7 +436,7 @@ struct platform_driver tegra_dpaux_driver = {
 	.remove = tegra_dpaux_remove,
 };
 
-struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np)
+struct drm_dp_aux *drm_dp_aux_find_by_of_node(struct device_node *np)
 {
 	struct tegra_dpaux *dpaux;
 
@@ -445,7 +445,7 @@ struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np)
 	list_for_each_entry(dpaux, &dpaux_list, list)
 		if (np == dpaux->dev->of_node) {
 			mutex_unlock(&dpaux_lock);
-			return dpaux;
+			return &dpaux->aux;
 		}
 
 	mutex_unlock(&dpaux_lock);
@@ -453,8 +453,9 @@ struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np)
 	return NULL;
 }
 
-int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output)
+int drm_dp_aux_attach(struct drm_dp_aux *aux, struct tegra_output *output)
 {
+	struct tegra_dpaux *dpaux = to_dpaux(aux);
 	unsigned long timeout;
 	int err;
 
@@ -470,7 +471,7 @@ int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output)
 	while (time_before(jiffies, timeout)) {
 		enum drm_connector_status status;
 
-		status = tegra_dpaux_detect(dpaux);
+		status = drm_dp_aux_detect(aux);
 		if (status == connector_status_connected) {
 			enable_irq(dpaux->irq);
 			return 0;
@@ -482,8 +483,9 @@ int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output)
 	return -ETIMEDOUT;
 }
 
-int tegra_dpaux_detach(struct tegra_dpaux *dpaux)
+int drm_dp_aux_detach(struct drm_dp_aux *aux)
 {
+	struct tegra_dpaux *dpaux = to_dpaux(aux);
 	unsigned long timeout;
 	int err;
 
@@ -498,7 +500,7 @@ int tegra_dpaux_detach(struct tegra_dpaux *dpaux)
 	while (time_before(jiffies, timeout)) {
 		enum drm_connector_status status;
 
-		status = tegra_dpaux_detect(dpaux);
+		status = drm_dp_aux_detect(aux);
 		if (status == connector_status_disconnected) {
 			dpaux->output = NULL;
 			return 0;
@@ -510,8 +512,9 @@ int tegra_dpaux_detach(struct tegra_dpaux *dpaux)
 	return -ETIMEDOUT;
 }
 
-enum drm_connector_status tegra_dpaux_detect(struct tegra_dpaux *dpaux)
+enum drm_connector_status drm_dp_aux_detect(struct drm_dp_aux *aux)
 {
+	struct tegra_dpaux *dpaux = to_dpaux(aux);
 	u32 value;
 
 	value = tegra_dpaux_readl(dpaux, DPAUX_DP_AUXSTAT);
@@ -522,8 +525,9 @@ enum drm_connector_status tegra_dpaux_detect(struct tegra_dpaux *dpaux)
 	return connector_status_disconnected;
 }
 
-int tegra_dpaux_enable(struct tegra_dpaux *dpaux)
+int drm_dp_aux_enable(struct drm_dp_aux *aux)
 {
+	struct tegra_dpaux *dpaux = to_dpaux(aux);
 	u32 value;
 
 	value = DPAUX_HYBRID_PADCTL_AUX_CMH(2) |
@@ -540,8 +544,9 @@ int tegra_dpaux_enable(struct tegra_dpaux *dpaux)
 	return 0;
 }
 
-int tegra_dpaux_disable(struct tegra_dpaux *dpaux)
+int drm_dp_aux_disable(struct drm_dp_aux *aux)
 {
+	struct tegra_dpaux *dpaux = to_dpaux(aux);
 	u32 value;
 
 	value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
@@ -551,11 +556,11 @@ int tegra_dpaux_disable(struct tegra_dpaux *dpaux)
 	return 0;
 }
 
-int tegra_dpaux_prepare(struct tegra_dpaux *dpaux, u8 encoding)
+int drm_dp_aux_prepare(struct drm_dp_aux *aux, u8 encoding)
 {
 	int err;
 
-	err = drm_dp_dpcd_writeb(&dpaux->aux, DP_MAIN_LINK_CHANNEL_CODING_SET,
+	err = drm_dp_dpcd_writeb(aux, DP_MAIN_LINK_CHANNEL_CODING_SET,
 				 encoding);
 	if (err < 0)
 		return err;
@@ -563,15 +568,15 @@ int tegra_dpaux_prepare(struct tegra_dpaux *dpaux, u8 encoding)
 	return 0;
 }
 
-int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
-		      u8 pattern)
+int drm_dp_aux_train(struct drm_dp_aux *aux, struct drm_dp_link *link,
+		     u8 pattern)
 {
 	u8 tp = pattern & DP_TRAINING_PATTERN_MASK;
 	u8 status[DP_LINK_STATUS_SIZE], values[4];
 	unsigned int i;
 	int err;
 
-	err = drm_dp_dpcd_writeb(&dpaux->aux, DP_TRAINING_PATTERN_SET, pattern);
+	err = drm_dp_dpcd_writeb(aux, DP_TRAINING_PATTERN_SET, pattern);
 	if (err < 0)
 		return err;
 
@@ -584,14 +589,14 @@ int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
 			    DP_TRAIN_MAX_SWING_REACHED |
 			    DP_TRAIN_VOLTAGE_SWING_LEVEL_0;
 
-	err = drm_dp_dpcd_write(&dpaux->aux, DP_TRAINING_LANE0_SET, values,
+	err = drm_dp_dpcd_write(aux, DP_TRAINING_LANE0_SET, values,
 				link->num_lanes);
 	if (err < 0)
 		return err;
 
 	usleep_range(500, 1000);
 
-	err = drm_dp_dpcd_read_link_status(&dpaux->aux, status);
+	err = drm_dp_dpcd_read_link_status(aux, status);
 	if (err < 0)
 		return err;
 
@@ -609,11 +614,11 @@ int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
 		break;
 
 	default:
-		dev_err(dpaux->dev, "unsupported training pattern %u\n", tp);
+		dev_err(aux->dev, "unsupported training pattern %u\n", tp);
 		return -EINVAL;
 	}
 
-	err = drm_dp_dpcd_writeb(&dpaux->aux, DP_EDP_CONFIGURATION_SET, 0);
+	err = drm_dp_dpcd_writeb(aux, DP_EDP_CONFIGURATION_SET, 0);
 	if (err < 0)
 		return err;
 
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 159ef515cab1..c5c856a0879d 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -106,7 +106,7 @@ static int tegra_atomic_commit(struct drm_device *drm,
 
 static const struct drm_mode_config_funcs tegra_drm_mode_funcs = {
 	.fb_create = tegra_fb_create,
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	.output_poll_changed = tegra_fb_output_poll_changed,
 #endif
 	.atomic_check = drm_atomic_helper_check,
@@ -137,8 +137,8 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 		start = geometry->aperture_start;
 		end = geometry->aperture_end;
 
-		DRM_DEBUG("IOMMU context initialized (aperture: %#llx-%#llx)\n",
-			  start, end);
+		DRM_DEBUG_DRIVER("IOMMU aperture initialized (%#llx-%#llx)\n",
+				 start, end);
 		drm_mm_init(&tegra->mm, start, end - start + 1);
 	}
 
@@ -260,7 +260,7 @@ static void tegra_drm_context_free(struct tegra_drm_context *context)
 
 static void tegra_drm_lastclose(struct drm_device *drm)
 {
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_drm *tegra = drm->dev_private;
 
 	tegra_fbdev_restore_mode(tegra->fbdev);
@@ -277,9 +277,7 @@ host1x_bo_lookup(struct drm_device *drm, struct drm_file *file, u32 handle)
 	if (!gem)
 		return NULL;
 
-	mutex_lock(&drm->struct_mutex);
-	drm_gem_object_unreference(gem);
-	mutex_unlock(&drm->struct_mutex);
+	drm_gem_object_unreference_unlocked(gem);
 
 	bo = to_tegra_bo(gem);
 	return &bo->base;
@@ -473,7 +471,7 @@ static int tegra_gem_mmap(struct drm_device *drm, void *data,
 
 	args->offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 
-	drm_gem_object_unreference(gem);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return 0;
 }
@@ -683,7 +681,7 @@ static int tegra_gem_set_tiling(struct drm_device *drm, void *data,
 	bo->tiling.mode = mode;
 	bo->tiling.value = value;
 
-	drm_gem_object_unreference(gem);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return 0;
 }
@@ -723,7 +721,7 @@ static int tegra_gem_get_tiling(struct drm_device *drm, void *data,
 		break;
 	}
 
-	drm_gem_object_unreference(gem);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return err;
 }
@@ -748,7 +746,7 @@ static int tegra_gem_set_flags(struct drm_device *drm, void *data,
 	if (args->flags & DRM_TEGRA_GEM_BOTTOM_UP)
 		bo->flags |= TEGRA_BO_BOTTOM_UP;
 
-	drm_gem_object_unreference(gem);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return 0;
 }
@@ -770,7 +768,7 @@ static int tegra_gem_get_flags(struct drm_device *drm, void *data,
 	if (bo->flags & TEGRA_BO_BOTTOM_UP)
 		args->flags |= DRM_TEGRA_GEM_BOTTOM_UP;
 
-	drm_gem_object_unreference(gem);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return 0;
 }
@@ -921,7 +919,8 @@ static void tegra_debugfs_cleanup(struct drm_minor *minor)
 #endif
 
 static struct drm_driver tegra_drm_driver = {
-	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME,
+	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
+			   DRIVER_ATOMIC,
 	.load = tegra_drm_load,
 	.unload = tegra_drm_unload,
 	.open = tegra_drm_open,
@@ -991,7 +990,6 @@ static int host1x_drm_probe(struct host1x_device *dev)
 	if (!drm)
 		return -ENOMEM;
 
-	drm_dev_set_unique(drm, dev_name(&dev->dev));
 	dev_set_drvdata(&dev->dev, drm);
 
 	err = drm_dev_register(drm, 0);
@@ -1023,8 +1021,17 @@ static int host1x_drm_remove(struct host1x_device *dev)
 static int host1x_drm_suspend(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
+	struct tegra_drm *tegra = drm->dev_private;
 
 	drm_kms_helper_poll_disable(drm);
+	tegra_drm_fb_suspend(drm);
+
+	tegra->state = drm_atomic_helper_suspend(drm);
+	if (IS_ERR(tegra->state)) {
+		tegra_drm_fb_resume(drm);
+		drm_kms_helper_poll_enable(drm);
+		return PTR_ERR(tegra->state);
+	}
 
 	return 0;
 }
@@ -1032,7 +1039,10 @@ static int host1x_drm_suspend(struct device *dev)
 static int host1x_drm_resume(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
+	struct tegra_drm *tegra = drm->dev_private;
 
+	drm_atomic_helper_resume(drm, tegra->state);
+	tegra_drm_fb_resume(drm);
 	drm_kms_helper_poll_enable(drm);
 
 	return 0;
@@ -1076,6 +1086,16 @@ static struct host1x_driver host1x_drm_driver = {
 	.subdevs = host1x_drm_subdevs,
 };
 
+static struct platform_driver * const drivers[] = {
+	&tegra_dc_driver,
+	&tegra_hdmi_driver,
+	&tegra_dsi_driver,
+	&tegra_dpaux_driver,
+	&tegra_sor_driver,
+	&tegra_gr2d_driver,
+	&tegra_gr3d_driver,
+};
+
 static int __init host1x_drm_init(void)
 {
 	int err;
@@ -1084,48 +1104,12 @@ static int __init host1x_drm_init(void)
 	if (err < 0)
 		return err;
 
-	err = platform_driver_register(&tegra_dc_driver);
+	err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 	if (err < 0)
 		goto unregister_host1x;
 
-	err = platform_driver_register(&tegra_dsi_driver);
-	if (err < 0)
-		goto unregister_dc;
-
-	err = platform_driver_register(&tegra_sor_driver);
-	if (err < 0)
-		goto unregister_dsi;
-
-	err = platform_driver_register(&tegra_hdmi_driver);
-	if (err < 0)
-		goto unregister_sor;
-
-	err = platform_driver_register(&tegra_dpaux_driver);
-	if (err < 0)
-		goto unregister_hdmi;
-
-	err = platform_driver_register(&tegra_gr2d_driver);
-	if (err < 0)
-		goto unregister_dpaux;
-
-	err = platform_driver_register(&tegra_gr3d_driver);
-	if (err < 0)
-		goto unregister_gr2d;
-
 	return 0;
 
-unregister_gr2d:
-	platform_driver_unregister(&tegra_gr2d_driver);
-unregister_dpaux:
-	platform_driver_unregister(&tegra_dpaux_driver);
-unregister_hdmi:
-	platform_driver_unregister(&tegra_hdmi_driver);
-unregister_sor:
-	platform_driver_unregister(&tegra_sor_driver);
-unregister_dsi:
-	platform_driver_unregister(&tegra_dsi_driver);
-unregister_dc:
-	platform_driver_unregister(&tegra_dc_driver);
 unregister_host1x:
 	host1x_driver_unregister(&host1x_drm_driver);
 	return err;
@@ -1134,13 +1118,7 @@ module_init(host1x_drm_init);
 
 static void __exit host1x_drm_exit(void)
 {
-	platform_driver_unregister(&tegra_gr3d_driver);
-	platform_driver_unregister(&tegra_gr2d_driver);
-	platform_driver_unregister(&tegra_dpaux_driver);
-	platform_driver_unregister(&tegra_hdmi_driver);
-	platform_driver_unregister(&tegra_sor_driver);
-	platform_driver_unregister(&tegra_dsi_driver);
-	platform_driver_unregister(&tegra_dc_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 	host1x_driver_unregister(&host1x_drm_driver);
 }
 module_exit(host1x_drm_exit);
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index ec49275ffb24..c088f2f67eda 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -30,7 +30,7 @@ struct tegra_fb {
 	unsigned int num_planes;
 };
 
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 struct tegra_fbdev {
 	struct drm_fb_helper base;
 	struct tegra_fb *fb;
@@ -46,7 +46,7 @@ struct tegra_drm {
 	struct mutex clients_lock;
 	struct list_head clients;
 
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_fbdev *fbdev;
 #endif
 
@@ -57,6 +57,8 @@ struct tegra_drm {
 		struct work_struct work;
 		struct mutex lock;
 	} commit;
+
+	struct drm_atomic_state *state;
 };
 
 struct tegra_drm_client;
@@ -247,18 +249,17 @@ void tegra_output_connector_destroy(struct drm_connector *connector);
 void tegra_output_encoder_destroy(struct drm_encoder *encoder);
 
 /* from dpaux.c */
-struct tegra_dpaux;
 struct drm_dp_link;
 
-struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np);
-enum drm_connector_status tegra_dpaux_detect(struct tegra_dpaux *dpaux);
-int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output);
-int tegra_dpaux_detach(struct tegra_dpaux *dpaux);
-int tegra_dpaux_enable(struct tegra_dpaux *dpaux);
-int tegra_dpaux_disable(struct tegra_dpaux *dpaux);
-int tegra_dpaux_prepare(struct tegra_dpaux *dpaux, u8 encoding);
-int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
-		      u8 pattern);
+struct drm_dp_aux *drm_dp_aux_find_by_of_node(struct device_node *np);
+enum drm_connector_status drm_dp_aux_detect(struct drm_dp_aux *aux);
+int drm_dp_aux_attach(struct drm_dp_aux *aux, struct tegra_output *output);
+int drm_dp_aux_detach(struct drm_dp_aux *aux);
+int drm_dp_aux_enable(struct drm_dp_aux *aux);
+int drm_dp_aux_disable(struct drm_dp_aux *aux);
+int drm_dp_aux_prepare(struct drm_dp_aux *aux, u8 encoding);
+int drm_dp_aux_train(struct drm_dp_aux *aux, struct drm_dp_link *link,
+		     u8 pattern);
 
 /* from fb.c */
 struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
@@ -268,21 +269,23 @@ int tegra_fb_get_tiling(struct drm_framebuffer *framebuffer,
 			struct tegra_bo_tiling *tiling);
 struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 					struct drm_file *file,
-					struct drm_mode_fb_cmd2 *cmd);
+					const struct drm_mode_fb_cmd2 *cmd);
 int tegra_drm_fb_prepare(struct drm_device *drm);
 void tegra_drm_fb_free(struct drm_device *drm);
 int tegra_drm_fb_init(struct drm_device *drm);
 void tegra_drm_fb_exit(struct drm_device *drm);
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+void tegra_drm_fb_suspend(struct drm_device *drm);
+void tegra_drm_fb_resume(struct drm_device *drm);
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev);
 void tegra_fb_output_poll_changed(struct drm_device *drm);
 #endif
 
 extern struct platform_driver tegra_dc_driver;
-extern struct platform_driver tegra_dsi_driver;
-extern struct platform_driver tegra_sor_driver;
 extern struct platform_driver tegra_hdmi_driver;
+extern struct platform_driver tegra_dsi_driver;
 extern struct platform_driver tegra_dpaux_driver;
+extern struct platform_driver tegra_sor_driver;
 extern struct platform_driver tegra_gr2d_driver;
 extern struct platform_driver tegra_gr3d_driver;
 
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index f0a138ef68ce..44e102799195 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -745,14 +745,13 @@ static void tegra_dsi_soft_reset(struct tegra_dsi *dsi)
 
 static void tegra_dsi_connector_reset(struct drm_connector *connector)
 {
-	struct tegra_dsi_state *state;
-
-	kfree(connector->state);
-	connector->state = NULL;
+	struct tegra_dsi_state *state =
+		kzalloc(sizeof(*state), GFP_KERNEL);
 
-	state = kzalloc(sizeof(*state), GFP_KERNEL);
-	if (state)
-		connector->state = &state->base;
+	if (state) {
+		kfree(connector->state);
+		__drm_atomic_helper_connector_reset(connector, &state->base);
+	}
 }
 
 static struct drm_connector_state *
@@ -1023,7 +1022,7 @@ static int tegra_dsi_init(struct host1x_client *client)
 
 		drm_encoder_init(drm, &dsi->output.encoder,
 				 &tegra_dsi_encoder_funcs,
-				 DRM_MODE_ENCODER_DSI);
+				 DRM_MODE_ENCODER_DSI, NULL);
 		drm_encoder_helper_add(&dsi->output.encoder,
 				       &tegra_dsi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index 1004075fd088..ca84de9ccb51 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -10,6 +10,8 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/console.h>
+
 #include "drm.h"
 #include "gem.h"
 
@@ -18,7 +20,7 @@ static inline struct tegra_fb *to_tegra_fb(struct drm_framebuffer *fb)
 	return container_of(fb, struct tegra_fb, base);
 }
 
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 static inline struct tegra_fbdev *to_tegra_fbdev(struct drm_fb_helper *helper)
 {
 	return container_of(helper, struct tegra_fbdev, base);
@@ -86,13 +88,13 @@ static int tegra_fb_create_handle(struct drm_framebuffer *framebuffer,
 	return drm_gem_handle_create(file, &fb->planes[0]->gem, handle);
 }
 
-static struct drm_framebuffer_funcs tegra_fb_funcs = {
+static const struct drm_framebuffer_funcs tegra_fb_funcs = {
 	.destroy = tegra_fb_destroy,
 	.create_handle = tegra_fb_create_handle,
 };
 
 static struct tegra_fb *tegra_fb_alloc(struct drm_device *drm,
-				       struct drm_mode_fb_cmd2 *mode_cmd,
+				       const struct drm_mode_fb_cmd2 *mode_cmd,
 				       struct tegra_bo **planes,
 				       unsigned int num_planes)
 {
@@ -131,7 +133,7 @@ static struct tegra_fb *tegra_fb_alloc(struct drm_device *drm,
 
 struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 					struct drm_file *file,
-					struct drm_mode_fb_cmd2 *cmd)
+					const struct drm_mode_fb_cmd2 *cmd)
 {
 	unsigned int hsub, vsub, i;
 	struct tegra_bo *planes[4];
@@ -181,7 +183,7 @@ unreference:
 	return ERR_PTR(err);
 }
 
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 static struct fb_ops tegra_fb_ops = {
 	.owner = THIS_MODULE,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
@@ -370,7 +372,7 @@ void tegra_fb_output_poll_changed(struct drm_device *drm)
 
 int tegra_drm_fb_prepare(struct drm_device *drm)
 {
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_drm *tegra = drm->dev_private;
 
 	tegra->fbdev = tegra_fbdev_create(drm);
@@ -383,7 +385,7 @@ int tegra_drm_fb_prepare(struct drm_device *drm)
 
 void tegra_drm_fb_free(struct drm_device *drm)
 {
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_drm *tegra = drm->dev_private;
 
 	tegra_fbdev_free(tegra->fbdev);
@@ -392,7 +394,7 @@ void tegra_drm_fb_free(struct drm_device *drm)
 
 int tegra_drm_fb_init(struct drm_device *drm)
 {
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_drm *tegra = drm->dev_private;
 	int err;
 
@@ -407,9 +409,31 @@ int tegra_drm_fb_init(struct drm_device *drm)
 
 void tegra_drm_fb_exit(struct drm_device *drm)
 {
-#ifdef CONFIG_DRM_TEGRA_FBDEV
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct tegra_drm *tegra = drm->dev_private;
 
 	tegra_fbdev_exit(tegra->fbdev);
 #endif
 }
+
+void tegra_drm_fb_suspend(struct drm_device *drm)
+{
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+	struct tegra_drm *tegra = drm->dev_private;
+
+	console_lock();
+	drm_fb_helper_set_suspend(&tegra->fbdev->base, 1);
+	console_unlock();
+#endif
+}
+
+void tegra_drm_fb_resume(struct drm_device *drm)
+{
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+	struct tegra_drm *tegra = drm->dev_private;
+
+	console_lock();
+	drm_fb_helper_set_suspend(&tegra->fbdev->base, 0);
+	console_unlock();
+#endif
+}
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 01e16e146bfe..33add93b4ed9 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -28,11 +28,8 @@ static inline struct tegra_bo *host1x_to_tegra_bo(struct host1x_bo *bo)
 static void tegra_bo_put(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
-	struct drm_device *drm = obj->gem.dev;
 
-	mutex_lock(&drm->struct_mutex);
-	drm_gem_object_unreference(&obj->gem);
-	mutex_unlock(&drm->struct_mutex);
+	drm_gem_object_unreference_unlocked(&obj->gem);
 }
 
 static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
@@ -72,11 +69,8 @@ static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page,
 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
-	struct drm_device *drm = obj->gem.dev;
 
-	mutex_lock(&drm->struct_mutex);
 	drm_gem_object_reference(&obj->gem);
-	mutex_unlock(&drm->struct_mutex);
 
 	return bo;
 }
@@ -408,12 +402,9 @@ int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm,
 	struct drm_gem_object *gem;
 	struct tegra_bo *bo;
 
-	mutex_lock(&drm->struct_mutex);
-
 	gem = drm_gem_object_lookup(drm, file, handle);
 	if (!gem) {
 		dev_err(drm->dev, "failed to lookup GEM object\n");
-		mutex_unlock(&drm->struct_mutex);
 		return -EINVAL;
 	}
 
@@ -421,9 +412,7 @@ int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm,
 
 	*offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 
-	drm_gem_object_unreference(gem);
-
-	mutex_unlock(&drm->struct_mutex);
+	drm_gem_object_unreference_unlocked(gem);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 52b32cbd9de6..b7ef4929e347 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -1320,7 +1320,7 @@ static int tegra_hdmi_init(struct host1x_client *client)
 	hdmi->output.connector.dpms = DRM_MODE_DPMS_OFF;
 
 	drm_encoder_init(drm, &hdmi->output.encoder, &tegra_hdmi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 	drm_encoder_helper_add(&hdmi->output.encoder,
 			       &tegra_hdmi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index bc9735b4ad60..e246334e0252 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -287,7 +287,7 @@ int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc)
 	output->connector.dpms = DRM_MODE_DPMS_OFF;
 
 	drm_encoder_init(drm, &output->encoder, &tegra_rgb_encoder_funcs,
-			 DRM_MODE_ENCODER_LVDS);
+			 DRM_MODE_ENCODER_LVDS, NULL);
 	drm_encoder_helper_add(&output->encoder,
 			       &tegra_rgb_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 3eff7cf75d25..757c6e8603af 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -173,7 +173,7 @@ struct tegra_sor {
 	struct clk *clk_dp;
 	struct clk *clk;
 
-	struct tegra_dpaux *dpaux;
+	struct drm_dp_aux *aux;
 
 	struct drm_info_list *debugfs_files;
 	struct drm_minor *minor;
@@ -273,7 +273,7 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 		   SOR_DP_PADCTL_CM_TXD_1 | SOR_DP_PADCTL_CM_TXD_0);
 	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
 
-	err = tegra_dpaux_prepare(sor->dpaux, DP_SET_ANSI_8B10B);
+	err = drm_dp_aux_prepare(sor->aux, DP_SET_ANSI_8B10B);
 	if (err < 0)
 		return err;
 
@@ -288,7 +288,7 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 
 	pattern = DP_TRAINING_PATTERN_1;
 
-	err = tegra_dpaux_train(sor->dpaux, link, pattern);
+	err = drm_dp_aux_train(sor->aux, link, pattern);
 	if (err < 0)
 		return err;
 
@@ -309,7 +309,7 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 
 	pattern = DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_2;
 
-	err = tegra_dpaux_train(sor->dpaux, link, pattern);
+	err = drm_dp_aux_train(sor->aux, link, pattern);
 	if (err < 0)
 		return err;
 
@@ -324,7 +324,7 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 
 	pattern = DP_TRAINING_PATTERN_DISABLE;
 
-	err = tegra_dpaux_train(sor->dpaux, link, pattern);
+	err = drm_dp_aux_train(sor->aux, link, pattern);
 	if (err < 0)
 		return err;
 
@@ -1044,8 +1044,8 @@ tegra_sor_connector_detect(struct drm_connector *connector, bool force)
 	struct tegra_output *output = connector_to_output(connector);
 	struct tegra_sor *sor = to_sor(output);
 
-	if (sor->dpaux)
-		return tegra_dpaux_detect(sor->dpaux);
+	if (sor->aux)
+		return drm_dp_aux_detect(sor->aux);
 
 	return tegra_output_connector_detect(connector, force);
 }
@@ -1066,13 +1066,13 @@ static int tegra_sor_connector_get_modes(struct drm_connector *connector)
 	struct tegra_sor *sor = to_sor(output);
 	int err;
 
-	if (sor->dpaux)
-		tegra_dpaux_enable(sor->dpaux);
+	if (sor->aux)
+		drm_dp_aux_enable(sor->aux);
 
 	err = tegra_output_connector_get_modes(connector);
 
-	if (sor->dpaux)
-		tegra_dpaux_disable(sor->dpaux);
+	if (sor->aux)
+		drm_dp_aux_disable(sor->aux);
 
 	return err;
 }
@@ -1128,8 +1128,8 @@ static void tegra_sor_edp_disable(struct drm_encoder *encoder)
 	if (err < 0)
 		dev_err(sor->dev, "failed to power down SOR: %d\n", err);
 
-	if (sor->dpaux) {
-		err = tegra_dpaux_disable(sor->dpaux);
+	if (sor->aux) {
+		err = drm_dp_aux_disable(sor->aux);
 		if (err < 0)
 			dev_err(sor->dev, "failed to disable DP: %d\n", err);
 	}
@@ -1196,7 +1196,7 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	struct tegra_sor *sor = to_sor(output);
 	struct tegra_sor_config config;
 	struct drm_dp_link link;
-	struct drm_dp_aux *aux;
+	u8 rate, lanes;
 	int err = 0;
 	u32 value;
 
@@ -1209,20 +1209,14 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	if (output->panel)
 		drm_panel_prepare(output->panel);
 
-	/* FIXME: properly convert to struct drm_dp_aux */
-	aux = (struct drm_dp_aux *)sor->dpaux;
-
-	if (sor->dpaux) {
-		err = tegra_dpaux_enable(sor->dpaux);
-		if (err < 0)
-			dev_err(sor->dev, "failed to enable DP: %d\n", err);
+	err = drm_dp_aux_enable(sor->aux);
+	if (err < 0)
+		dev_err(sor->dev, "failed to enable DP: %d\n", err);
 
-		err = drm_dp_link_probe(aux, &link);
-		if (err < 0) {
-			dev_err(sor->dev, "failed to probe eDP link: %d\n",
-				err);
-			return;
-		}
+	err = drm_dp_link_probe(sor->aux, &link);
+	if (err < 0) {
+		dev_err(sor->dev, "failed to probe eDP link: %d\n", err);
+		return;
 	}
 
 	err = clk_set_parent(sor->clk, sor->clk_safe);
@@ -1434,60 +1428,51 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	value |= SOR_DP_PADCTL_PAD_CAL_PD;
 	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
 
-	if (sor->dpaux) {
-		u8 rate, lanes;
-
-		err = drm_dp_link_probe(aux, &link);
-		if (err < 0)
-			dev_err(sor->dev, "failed to probe eDP link: %d\n",
-				err);
+	err = drm_dp_link_probe(sor->aux, &link);
+	if (err < 0)
+		dev_err(sor->dev, "failed to probe eDP link: %d\n", err);
 
-		err = drm_dp_link_power_up(aux, &link);
-		if (err < 0)
-			dev_err(sor->dev, "failed to power up eDP link: %d\n",
-				err);
+	err = drm_dp_link_power_up(sor->aux, &link);
+	if (err < 0)
+		dev_err(sor->dev, "failed to power up eDP link: %d\n", err);
 
-		err = drm_dp_link_configure(aux, &link);
-		if (err < 0)
-			dev_err(sor->dev, "failed to configure eDP link: %d\n",
-				err);
+	err = drm_dp_link_configure(sor->aux, &link);
+	if (err < 0)
+		dev_err(sor->dev, "failed to configure eDP link: %d\n", err);
 
-		rate = drm_dp_link_rate_to_bw_code(link.rate);
-		lanes = link.num_lanes;
+	rate = drm_dp_link_rate_to_bw_code(link.rate);
+	lanes = link.num_lanes;
 
-		value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
-		value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
-		value |= SOR_CLK_CNTRL_DP_LINK_SPEED(rate);
-		tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+	value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+	value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
+	value |= SOR_CLK_CNTRL_DP_LINK_SPEED(rate);
+	tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
 
-		value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
-		value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
-		value |= SOR_DP_LINKCTL_LANE_COUNT(lanes);
+	value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
+	value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
+	value |= SOR_DP_LINKCTL_LANE_COUNT(lanes);
 
-		if (link.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
-			value |= SOR_DP_LINKCTL_ENHANCED_FRAME;
+	if (link.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
+		value |= SOR_DP_LINKCTL_ENHANCED_FRAME;
 
-		tegra_sor_writel(sor, value, SOR_DP_LINKCTL0);
+	tegra_sor_writel(sor, value, SOR_DP_LINKCTL0);
 
-		/* disable training pattern generator */
+	/* disable training pattern generator */
 
-		for (i = 0; i < link.num_lanes; i++) {
-			unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
-					     SOR_DP_TPG_SCRAMBLER_GALIOS |
-					     SOR_DP_TPG_PATTERN_NONE;
-			value = (value << 8) | lane;
-		}
+	for (i = 0; i < link.num_lanes; i++) {
+		unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+				     SOR_DP_TPG_SCRAMBLER_GALIOS |
+				     SOR_DP_TPG_PATTERN_NONE;
+		value = (value << 8) | lane;
+	}
 
-		tegra_sor_writel(sor, value, SOR_DP_TPG);
+	tegra_sor_writel(sor, value, SOR_DP_TPG);
 
-		err = tegra_sor_dp_train_fast(sor, &link);
-		if (err < 0) {
-			dev_err(sor->dev, "DP fast link training failed: %d\n",
-				err);
-		}
+	err = tegra_sor_dp_train_fast(sor, &link);
+	if (err < 0)
+		dev_err(sor->dev, "DP fast link training failed: %d\n", err);
 
-		dev_dbg(sor->dev, "fast link training succeeded\n");
-	}
+	dev_dbg(sor->dev, "fast link training succeeded\n");
 
 	err = tegra_sor_power_up(sor, 250);
 	if (err < 0)
@@ -1961,9 +1946,9 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 
 	/* production settings */
 	settings = tegra_sor_hdmi_find_settings(sor, mode->clock * 1000);
-	if (IS_ERR(settings)) {
-		dev_err(sor->dev, "no settings for pixel clock %d Hz: %ld\n",
-			mode->clock * 1000, PTR_ERR(settings));
+	if (!settings) {
+		dev_err(sor->dev, "no settings for pixel clock %d Hz\n",
+			mode->clock * 1000);
 		return;
 	}
 
@@ -2148,7 +2133,7 @@ static int tegra_sor_init(struct host1x_client *client)
 	int encoder = DRM_MODE_ENCODER_NONE;
 	int err;
 
-	if (!sor->dpaux) {
+	if (!sor->aux) {
 		if (sor->soc->supports_hdmi) {
 			connector = DRM_MODE_CONNECTOR_HDMIA;
 			encoder = DRM_MODE_ENCODER_TMDS;
@@ -2178,7 +2163,7 @@ static int tegra_sor_init(struct host1x_client *client)
 	sor->output.connector.dpms = DRM_MODE_DPMS_OFF;
 
 	drm_encoder_init(drm, &sor->output.encoder, &tegra_sor_encoder_funcs,
-			 encoder);
+			 encoder, NULL);
 	drm_encoder_helper_add(&sor->output.encoder, helpers);
 
 	drm_mode_connector_attach_encoder(&sor->output.connector,
@@ -2199,8 +2184,8 @@ static int tegra_sor_init(struct host1x_client *client)
 			dev_err(sor->dev, "debugfs setup failed: %d\n", err);
 	}
 
-	if (sor->dpaux) {
-		err = tegra_dpaux_attach(sor->dpaux, &sor->output);
+	if (sor->aux) {
+		err = drm_dp_aux_attach(sor->aux, &sor->output);
 		if (err < 0) {
 			dev_err(sor->dev, "failed to attach DP: %d\n", err);
 			return err;
@@ -2249,8 +2234,8 @@ static int tegra_sor_exit(struct host1x_client *client)
 
 	tegra_output_exit(&sor->output);
 
-	if (sor->dpaux) {
-		err = tegra_dpaux_detach(sor->dpaux);
+	if (sor->aux) {
+		err = drm_dp_aux_detach(sor->aux);
 		if (err < 0) {
 			dev_err(sor->dev, "failed to detach DP: %d\n", err);
 			return err;
@@ -2399,14 +2384,14 @@ static int tegra_sor_probe(struct platform_device *pdev)
 
 	np = of_parse_phandle(pdev->dev.of_node, "nvidia,dpaux", 0);
 	if (np) {
-		sor->dpaux = tegra_dpaux_find_by_of_node(np);
+		sor->aux = drm_dp_aux_find_by_of_node(np);
 		of_node_put(np);
 
-		if (!sor->dpaux)
+		if (!sor->aux)
 			return -EPROBE_DEFER;
 	}
 
-	if (!sor->dpaux) {
+	if (!sor->aux) {
 		if (sor->soc->supports_hdmi) {
 			sor->ops = &tegra_sor_hdmi_ops;
 		} else if (sor->soc->supports_lvds) {
diff --git a/drivers/gpu/drm/tilcdc/Kconfig b/drivers/gpu/drm/tilcdc/Kconfig
index 78beafb0742c..f60a1ec84fa4 100644
--- a/drivers/gpu/drm/tilcdc/Kconfig
+++ b/drivers/gpu/drm/tilcdc/Kconfig
@@ -1,6 +1,6 @@
 config DRM_TILCDC
 	tristate "DRM Support for TI LCDC Display Controller"
-	depends on DRM && OF && ARM && HAVE_DMA_ATTRS
+	depends on DRM && OF && ARM
 	select DRM_KMS_HELPER
 	select DRM_KMS_FB_HELPER
 	select DRM_KMS_CMA_HELPER
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 876cad58b1f9..d7f5b897c6c5 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -46,7 +46,7 @@ void tilcdc_module_cleanup(struct tilcdc_module *mod)
 static struct of_device_id tilcdc_of_match[];
 
 static struct drm_framebuffer *tilcdc_fb_create(struct drm_device *dev,
-		struct drm_file *file_priv, struct drm_mode_fb_cmd2 *mode_cmd)
+		struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	return drm_fb_cma_create(dev, file_priv, mode_cmd);
 }
@@ -294,6 +294,7 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 			break;
 	}
 
+	drm_helper_disable_unused_functions(dev);
 	priv->fbdev = drm_fbdev_cma_init(dev, bpp,
 			dev->mode_config.num_crtc,
 			dev->mode_config.num_connector);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index 0af8bed7ce1e..4dda6e2f464b 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -128,7 +128,7 @@ static struct drm_encoder *panel_encoder_create(struct drm_device *dev,
 	encoder->possible_crtcs = 1;
 
 	ret = drm_encoder_init(dev, encoder, &panel_encoder_funcs,
-			DRM_MODE_ENCODER_LVDS);
+			DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret < 0)
 		goto fail;
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
index 354c47ca6374..5052a8af7ecb 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
@@ -138,7 +138,7 @@ static struct drm_encoder *tfp410_encoder_create(struct drm_device *dev,
 	encoder->possible_crtcs = 1;
 
 	ret = drm_encoder_init(dev, encoder, &tfp410_encoder_funcs,
-			DRM_MODE_ENCODER_TMDS);
+			DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret < 0)
 		goto fail;
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 745e996d2dbc..4cbf26555093 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -176,7 +176,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 		list_add_tail(&bo->lru, &man->lru);
 		kref_get(&bo->list_kref);
 
-		if (bo->ttm != NULL) {
+		if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
 			list_add_tail(&bo->swap, &bo->glob->swap_lru);
 			kref_get(&bo->list_kref);
 		}
@@ -228,6 +228,27 @@ void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 }
 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 
+void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
+{
+	struct ttm_bo_device *bdev = bo->bdev;
+	struct ttm_mem_type_manager *man;
+
+	lockdep_assert_held(&bo->resv->lock.base);
+
+	if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
+		list_del_init(&bo->swap);
+		list_del_init(&bo->lru);
+
+	} else {
+		if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG))
+			list_move_tail(&bo->swap, &bo->glob->swap_lru);
+
+		man = &bdev->man[bo->mem.mem_type];
+		list_move_tail(&bo->lru, &man->lru);
+	}
+}
+EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
+
 /*
  * Call bo->mutex locked.
  */
@@ -1170,9 +1191,15 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
 	if (likely(!ret))
 		ret = ttm_bo_validate(bo, placement, interruptible, false);
 
-	if (!resv)
+	if (!resv) {
 		ttm_bo_unreserve(bo);
 
+	} else if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
+		spin_lock(&bo->glob->lru_lock);
+		ttm_bo_add_to_lru(bo);
+		spin_unlock(&bo->glob->lru_lock);
+	}
+
 	if (unlikely(ret))
 		ttm_bo_unref(&bo);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 8fb7213277cc..06d26dc438b2 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -35,6 +35,7 @@
 #include <ttm/ttm_placement.h>
 #include <drm/drm_vma_manager.h>
 #include <linux/mm.h>
+#include <linux/pfn_t.h>
 #include <linux/rbtree.h>
 #include <linux/module.h>
 #include <linux/uaccess.h>
@@ -229,7 +230,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		}
 
 		if (vma->vm_flags & VM_MIXEDMAP)
-			ret = vm_insert_mixed(&cvma, address, pfn);
+			ret = vm_insert_mixed(&cvma, address,
+					__pfn_to_pfn_t(pfn, PFN_DEV));
 		else
 			ret = vm_insert_pfn(&cvma, address, pfn);
 
diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
index 0110d95522f3..4709b54c204c 100644
--- a/drivers/gpu/drm/udl/udl_connector.c
+++ b/drivers/gpu/drm/udl/udl_connector.c
@@ -122,13 +122,13 @@ static void udl_connector_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-static struct drm_connector_helper_funcs udl_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs udl_connector_helper_funcs = {
 	.get_modes = udl_get_modes,
 	.mode_valid = udl_mode_valid,
 	.best_encoder = udl_best_single_encoder,
 };
 
-static struct drm_connector_funcs udl_connector_funcs = {
+static const struct drm_connector_funcs udl_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = udl_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index 80adbac82bde..4a064efcea58 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -108,7 +108,7 @@ void udl_fbdev_unplug(struct drm_device *dev);
 struct drm_framebuffer *
 udl_fb_user_fb_create(struct drm_device *dev,
 		      struct drm_file *file,
-		      struct drm_mode_fb_cmd2 *mode_cmd);
+		      const struct drm_mode_fb_cmd2 *mode_cmd);
 
 int udl_render_hline(struct drm_device *dev, int bpp, struct urb **urb_ptr,
 		     const char *front, char **urb_buf_ptr,
diff --git a/drivers/gpu/drm/udl/udl_encoder.c b/drivers/gpu/drm/udl/udl_encoder.c
index 4052c4656498..a181a647fcf9 100644
--- a/drivers/gpu/drm/udl/udl_encoder.c
+++ b/drivers/gpu/drm/udl/udl_encoder.c
@@ -73,7 +73,8 @@ struct drm_encoder *udl_encoder_init(struct drm_device *dev)
 	if (!encoder)
 		return NULL;
 
-	drm_encoder_init(dev, encoder, &udl_enc_funcs, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(dev, encoder, &udl_enc_funcs, DRM_MODE_ENCODER_TMDS,
+			 NULL);
 	drm_encoder_helper_add(encoder, &udl_helper_funcs);
 	encoder->possible_crtcs = 1;
 	return encoder;
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index 62c7b1dafaa4..200419d4d43c 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -33,7 +33,6 @@ module_param(fb_defio, int, S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP);
 struct udl_fbdev {
 	struct drm_fb_helper helper;
 	struct udl_framebuffer ufb;
-	struct list_head fbdev_list;
 	int fb_count;
 };
 
@@ -456,7 +455,7 @@ static const struct drm_framebuffer_funcs udlfb_funcs = {
 static int
 udl_framebuffer_init(struct drm_device *dev,
 		     struct udl_framebuffer *ufb,
-		     struct drm_mode_fb_cmd2 *mode_cmd,
+		     const struct drm_mode_fb_cmd2 *mode_cmd,
 		     struct udl_gem_object *obj)
 {
 	int ret;
@@ -624,7 +623,7 @@ void udl_fbdev_unplug(struct drm_device *dev)
 struct drm_framebuffer *
 udl_fb_user_fb_create(struct drm_device *dev,
 		   struct drm_file *file,
-		   struct drm_mode_fb_cmd2 *mode_cmd)
+		   const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
 	struct udl_framebuffer *ufb;
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 677190a65e82..160ef2a08b89 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -400,7 +400,7 @@ static void udl_crtc_commit(struct drm_crtc *crtc)
 	udl_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
 }
 
-static struct drm_crtc_helper_funcs udl_helper_funcs = {
+static const struct drm_crtc_helper_funcs udl_helper_funcs = {
 	.dpms = udl_crtc_dpms,
 	.mode_fixup = udl_crtc_mode_fixup,
 	.mode_set = udl_crtc_mode_set,
diff --git a/drivers/gpu/drm/vc4/Kconfig b/drivers/gpu/drm/vc4/Kconfig
index 2d7d115ddf3f..584810474e5b 100644
--- a/drivers/gpu/drm/vc4/Kconfig
+++ b/drivers/gpu/drm/vc4/Kconfig
@@ -1,7 +1,7 @@
 config DRM_VC4
 	tristate "Broadcom VC4 Graphics"
 	depends on ARCH_BCM2835 || COMPILE_TEST
-	depends on DRM && HAVE_DMA_ATTRS
+	depends on DRM
 	select DRM_KMS_HELPER
 	select DRM_KMS_CMA_HELPER
 	select DRM_GEM_CMA_HELPER
diff --git a/drivers/gpu/drm/vc4/Makefile b/drivers/gpu/drm/vc4/Makefile
index 32b4f9cd8f52..4c6a99f0398c 100644
--- a/drivers/gpu/drm/vc4/Makefile
+++ b/drivers/gpu/drm/vc4/Makefile
@@ -8,10 +8,19 @@ vc4-y := \
 	vc4_crtc.o \
 	vc4_drv.o \
 	vc4_kms.o \
+	vc4_gem.o \
 	vc4_hdmi.o \
 	vc4_hvs.o \
-	vc4_plane.o
+	vc4_irq.o \
+	vc4_plane.o \
+	vc4_render_cl.o \
+	vc4_trace_points.o \
+	vc4_v3d.o \
+	vc4_validate.o \
+	vc4_validate_shaders.o
 
 vc4-$(CONFIG_DEBUG_FS) += vc4_debugfs.o
 
 obj-$(CONFIG_DRM_VC4)  += vc4.o
+
+CFLAGS_vc4_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index ab9f5108ae1a..18dfe3ec9a62 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -12,19 +12,236 @@
  * access to system memory with no MMU in between.  To support it, we
  * use the GEM CMA helper functions to allocate contiguous ranges of
  * physical memory for our BOs.
+ *
+ * Since the CMA allocator is very slow, we keep a cache of recently
+ * freed BOs around so that the kernel's allocation of objects for 3D
+ * rendering can return quickly.
  */
 
 #include "vc4_drv.h"
+#include "uapi/drm/vc4_drm.h"
+
+static void vc4_bo_stats_dump(struct vc4_dev *vc4)
+{
+	DRM_INFO("num bos allocated: %d\n",
+		 vc4->bo_stats.num_allocated);
+	DRM_INFO("size bos allocated: %dkb\n",
+		 vc4->bo_stats.size_allocated / 1024);
+	DRM_INFO("num bos used: %d\n",
+		 vc4->bo_stats.num_allocated - vc4->bo_stats.num_cached);
+	DRM_INFO("size bos used: %dkb\n",
+		 (vc4->bo_stats.size_allocated -
+		  vc4->bo_stats.size_cached) / 1024);
+	DRM_INFO("num bos cached: %d\n",
+		 vc4->bo_stats.num_cached);
+	DRM_INFO("size bos cached: %dkb\n",
+		 vc4->bo_stats.size_cached / 1024);
+}
+
+#ifdef CONFIG_DEBUG_FS
+int vc4_bo_stats_debugfs(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_bo_stats stats;
+
+	/* Take a snapshot of the current stats with the lock held. */
+	mutex_lock(&vc4->bo_lock);
+	stats = vc4->bo_stats;
+	mutex_unlock(&vc4->bo_lock);
+
+	seq_printf(m, "num bos allocated: %d\n",
+		   stats.num_allocated);
+	seq_printf(m, "size bos allocated: %dkb\n",
+		   stats.size_allocated / 1024);
+	seq_printf(m, "num bos used: %d\n",
+		   stats.num_allocated - stats.num_cached);
+	seq_printf(m, "size bos used: %dkb\n",
+		   (stats.size_allocated - stats.size_cached) / 1024);
+	seq_printf(m, "num bos cached: %d\n",
+		   stats.num_cached);
+	seq_printf(m, "size bos cached: %dkb\n",
+		   stats.size_cached / 1024);
+
+	return 0;
+}
+#endif
+
+static uint32_t bo_page_index(size_t size)
+{
+	return (size / PAGE_SIZE) - 1;
+}
+
+/* Must be called with bo_lock held. */
+static void vc4_bo_destroy(struct vc4_bo *bo)
+{
+	struct drm_gem_object *obj = &bo->base.base;
+	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);
+
+	if (bo->validated_shader) {
+		kfree(bo->validated_shader->texture_samples);
+		kfree(bo->validated_shader);
+		bo->validated_shader = NULL;
+	}
+
+	vc4->bo_stats.num_allocated--;
+	vc4->bo_stats.size_allocated -= obj->size;
+	drm_gem_cma_free_object(obj);
+}
+
+/* Must be called with bo_lock held. */
+static void vc4_bo_remove_from_cache(struct vc4_bo *bo)
+{
+	struct drm_gem_object *obj = &bo->base.base;
+	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);
+
+	vc4->bo_stats.num_cached--;
+	vc4->bo_stats.size_cached -= obj->size;
+
+	list_del(&bo->unref_head);
+	list_del(&bo->size_head);
+}
+
+static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev,
+						     size_t size)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint32_t page_index = bo_page_index(size);
+
+	if (vc4->bo_cache.size_list_size <= page_index) {
+		uint32_t new_size = max(vc4->bo_cache.size_list_size * 2,
+					page_index + 1);
+		struct list_head *new_list;
+		uint32_t i;
+
+		new_list = kmalloc_array(new_size, sizeof(struct list_head),
+					 GFP_KERNEL);
+		if (!new_list)
+			return NULL;
+
+		/* Rebase the old cached BO lists to their new list
+		 * head locations.
+		 */
+		for (i = 0; i < vc4->bo_cache.size_list_size; i++) {
+			struct list_head *old_list =
+				&vc4->bo_cache.size_list[i];
+
+			if (list_empty(old_list))
+				INIT_LIST_HEAD(&new_list[i]);
+			else
+				list_replace(old_list, &new_list[i]);
+		}
+		/* And initialize the brand new BO list heads. */
+		for (i = vc4->bo_cache.size_list_size; i < new_size; i++)
+			INIT_LIST_HEAD(&new_list[i]);
+
+		kfree(vc4->bo_cache.size_list);
+		vc4->bo_cache.size_list = new_list;
+		vc4->bo_cache.size_list_size = new_size;
+	}
+
+	return &vc4->bo_cache.size_list[page_index];
+}
+
+void vc4_bo_cache_purge(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	mutex_lock(&vc4->bo_lock);
+	while (!list_empty(&vc4->bo_cache.time_list)) {
+		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,
+						    struct vc4_bo, unref_head);
+		vc4_bo_remove_from_cache(bo);
+		vc4_bo_destroy(bo);
+	}
+	mutex_unlock(&vc4->bo_lock);
+}
+
+static struct vc4_bo *vc4_bo_get_from_cache(struct drm_device *dev,
+					    uint32_t size)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint32_t page_index = bo_page_index(size);
+	struct vc4_bo *bo = NULL;
+
+	size = roundup(size, PAGE_SIZE);
+
+	mutex_lock(&vc4->bo_lock);
+	if (page_index >= vc4->bo_cache.size_list_size)
+		goto out;
 
-struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size)
+	if (list_empty(&vc4->bo_cache.size_list[page_index]))
+		goto out;
+
+	bo = list_first_entry(&vc4->bo_cache.size_list[page_index],
+			      struct vc4_bo, size_head);
+	vc4_bo_remove_from_cache(bo);
+	kref_init(&bo->base.base.refcount);
+
+out:
+	mutex_unlock(&vc4->bo_lock);
+	return bo;
+}
+
+/**
+ * vc4_gem_create_object - Implementation of driver->gem_create_object.
+ *
+ * This lets the CMA helpers allocate object structs for us, and keep
+ * our BO stats correct.
+ */
+struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 {
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_bo *bo;
+
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&vc4->bo_lock);
+	vc4->bo_stats.num_allocated++;
+	vc4->bo_stats.size_allocated += size;
+	mutex_unlock(&vc4->bo_lock);
+
+	return &bo->base.base;
+}
+
+struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
+			     bool from_cache)
+{
+	size_t size = roundup(unaligned_size, PAGE_SIZE);
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct drm_gem_cma_object *cma_obj;
 
-	cma_obj = drm_gem_cma_create(dev, size);
-	if (IS_ERR(cma_obj))
+	if (size == 0)
 		return NULL;
-	else
-		return to_vc4_bo(&cma_obj->base);
+
+	/* First, try to get a vc4_bo from the kernel BO cache. */
+	if (from_cache) {
+		struct vc4_bo *bo = vc4_bo_get_from_cache(dev, size);
+
+		if (bo)
+			return bo;
+	}
+
+	cma_obj = drm_gem_cma_create(dev, size);
+	if (IS_ERR(cma_obj)) {
+		/*
+		 * If we've run out of CMA memory, kill the cache of
+		 * CMA allocations we've got laying around and try again.
+		 */
+		vc4_bo_cache_purge(dev);
+
+		cma_obj = drm_gem_cma_create(dev, size);
+		if (IS_ERR(cma_obj)) {
+			DRM_ERROR("Failed to allocate from CMA:\n");
+			vc4_bo_stats_dump(vc4);
+			return NULL;
+		}
+	}
+
+	return to_vc4_bo(&cma_obj->base);
 }
 
 int vc4_dumb_create(struct drm_file *file_priv,
@@ -41,7 +258,191 @@ int vc4_dumb_create(struct drm_file *file_priv,
 	if (args->size < args->pitch * args->height)
 		args->size = args->pitch * args->height;
 
-	bo = vc4_bo_create(dev, roundup(args->size, PAGE_SIZE));
+	bo = vc4_bo_create(dev, args->size, false);
+	if (!bo)
+		return -ENOMEM;
+
+	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
+	drm_gem_object_unreference_unlocked(&bo->base.base);
+
+	return ret;
+}
+
+/* Must be called with bo_lock held. */
+static void vc4_bo_cache_free_old(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	unsigned long expire_time = jiffies - msecs_to_jiffies(1000);
+
+	while (!list_empty(&vc4->bo_cache.time_list)) {
+		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,
+						    struct vc4_bo, unref_head);
+		if (time_before(expire_time, bo->free_time)) {
+			mod_timer(&vc4->bo_cache.time_timer,
+				  round_jiffies_up(jiffies +
+						   msecs_to_jiffies(1000)));
+			return;
+		}
+
+		vc4_bo_remove_from_cache(bo);
+		vc4_bo_destroy(bo);
+	}
+}
+
+/* Called on the last userspace/kernel unreference of the BO.  Returns
+ * it to the BO cache if possible, otherwise frees it.
+ *
+ * Note that this is called with the struct_mutex held.
+ */
+void vc4_free_object(struct drm_gem_object *gem_bo)
+{
+	struct drm_device *dev = gem_bo->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_bo *bo = to_vc4_bo(gem_bo);
+	struct list_head *cache_list;
+
+	mutex_lock(&vc4->bo_lock);
+	/* If the object references someone else's memory, we can't cache it.
+	 */
+	if (gem_bo->import_attach) {
+		vc4_bo_destroy(bo);
+		goto out;
+	}
+
+	/* Don't cache if it was publicly named. */
+	if (gem_bo->name) {
+		vc4_bo_destroy(bo);
+		goto out;
+	}
+
+	cache_list = vc4_get_cache_list_for_size(dev, gem_bo->size);
+	if (!cache_list) {
+		vc4_bo_destroy(bo);
+		goto out;
+	}
+
+	if (bo->validated_shader) {
+		kfree(bo->validated_shader->texture_samples);
+		kfree(bo->validated_shader);
+		bo->validated_shader = NULL;
+	}
+
+	bo->free_time = jiffies;
+	list_add(&bo->size_head, cache_list);
+	list_add(&bo->unref_head, &vc4->bo_cache.time_list);
+
+	vc4->bo_stats.num_cached++;
+	vc4->bo_stats.size_cached += gem_bo->size;
+
+	vc4_bo_cache_free_old(dev);
+
+out:
+	mutex_unlock(&vc4->bo_lock);
+}
+
+static void vc4_bo_cache_time_work(struct work_struct *work)
+{
+	struct vc4_dev *vc4 =
+		container_of(work, struct vc4_dev, bo_cache.time_work);
+	struct drm_device *dev = vc4->dev;
+
+	mutex_lock(&vc4->bo_lock);
+	vc4_bo_cache_free_old(dev);
+	mutex_unlock(&vc4->bo_lock);
+}
+
+static void vc4_bo_cache_time_timer(unsigned long data)
+{
+	struct drm_device *dev = (struct drm_device *)data;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	schedule_work(&vc4->bo_cache.time_work);
+}
+
+struct dma_buf *
+vc4_prime_export(struct drm_device *dev, struct drm_gem_object *obj, int flags)
+{
+	struct vc4_bo *bo = to_vc4_bo(obj);
+
+	if (bo->validated_shader) {
+		DRM_ERROR("Attempting to export shader BO\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	return drm_gem_prime_export(dev, obj, flags);
+}
+
+int vc4_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gem_obj;
+	struct vc4_bo *bo;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	gem_obj = vma->vm_private_data;
+	bo = to_vc4_bo(gem_obj);
+
+	if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
+		DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
+	 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
+	 * the whole buffer.
+	 */
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_pgoff = 0;
+
+	ret = dma_mmap_writecombine(bo->base.base.dev->dev, vma,
+				    bo->base.vaddr, bo->base.paddr,
+				    vma->vm_end - vma->vm_start);
+	if (ret)
+		drm_gem_vm_close(vma);
+
+	return ret;
+}
+
+int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	struct vc4_bo *bo = to_vc4_bo(obj);
+
+	if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
+		DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
+		return -EINVAL;
+	}
+
+	return drm_gem_cma_prime_mmap(obj, vma);
+}
+
+void *vc4_prime_vmap(struct drm_gem_object *obj)
+{
+	struct vc4_bo *bo = to_vc4_bo(obj);
+
+	if (bo->validated_shader) {
+		DRM_ERROR("mmaping of shader BOs not allowed.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	return drm_gem_cma_prime_vmap(obj);
+}
+
+int vc4_create_bo_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
+{
+	struct drm_vc4_create_bo *args = data;
+	struct vc4_bo *bo = NULL;
+	int ret;
+
+	/*
+	 * We can't allocate from the BO cache, because the BOs don't
+	 * get zeroed, and that might leak data between users.
+	 */
+	bo = vc4_bo_create(dev, args->size, false);
 	if (!bo)
 		return -ENOMEM;
 
@@ -50,3 +451,107 @@ int vc4_dumb_create(struct drm_file *file_priv,
 
 	return ret;
 }
+
+int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_vc4_mmap_bo *args = data;
+	struct drm_gem_object *gem_obj;
+
+	gem_obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		return -EINVAL;
+	}
+
+	/* The mmap offset was set up at BO allocation time. */
+	args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
+
+	drm_gem_object_unreference_unlocked(gem_obj);
+	return 0;
+}
+
+int
+vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv)
+{
+	struct drm_vc4_create_shader_bo *args = data;
+	struct vc4_bo *bo = NULL;
+	int ret;
+
+	if (args->size == 0)
+		return -EINVAL;
+
+	if (args->size % sizeof(u64) != 0)
+		return -EINVAL;
+
+	if (args->flags != 0) {
+		DRM_INFO("Unknown flags set: 0x%08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	if (args->pad != 0) {
+		DRM_INFO("Pad set: 0x%08x\n", args->pad);
+		return -EINVAL;
+	}
+
+	bo = vc4_bo_create(dev, args->size, true);
+	if (!bo)
+		return -ENOMEM;
+
+	ret = copy_from_user(bo->base.vaddr,
+			     (void __user *)(uintptr_t)args->data,
+			     args->size);
+	if (ret != 0)
+		goto fail;
+	/* Clear the rest of the memory from allocating from the BO
+	 * cache.
+	 */
+	memset(bo->base.vaddr + args->size, 0,
+	       bo->base.base.size - args->size);
+
+	bo->validated_shader = vc4_validate_shader(&bo->base);
+	if (!bo->validated_shader) {
+		ret = -EINVAL;
+		goto fail;
+	}
+
+	/* We have to create the handle after validation, to avoid
+	 * races for users to do doing things like mmap the shader BO.
+	 */
+	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
+
+ fail:
+	drm_gem_object_unreference_unlocked(&bo->base.base);
+
+	return ret;
+}
+
+void vc4_bo_cache_init(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	mutex_init(&vc4->bo_lock);
+
+	INIT_LIST_HEAD(&vc4->bo_cache.time_list);
+
+	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);
+	setup_timer(&vc4->bo_cache.time_timer,
+		    vc4_bo_cache_time_timer,
+		    (unsigned long)dev);
+}
+
+void vc4_bo_cache_destroy(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	del_timer(&vc4->bo_cache.time_timer);
+	cancel_work_sync(&vc4->bo_cache.time_work);
+
+	vc4_bo_cache_purge(dev);
+
+	if (vc4->bo_stats.num_allocated) {
+		DRM_ERROR("Destroying BO cache while BOs still allocated:\n");
+		vc4_bo_stats_dump(vc4);
+	}
+}
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 265064c62d49..018145e0b87d 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -35,6 +35,7 @@
 #include "drm_atomic_helper.h"
 #include "drm_crtc_helper.h"
 #include "linux/clk.h"
+#include "drm_fb_cma_helper.h"
 #include "linux/component.h"
 #include "linux/of_device.h"
 #include "vc4_drv.h"
@@ -327,7 +328,7 @@ static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
 	/* The pixelvalve can only feed one encoder (and encoders are
 	 * 1:1 with connectors.)
 	 */
-	if (drm_atomic_connectors_for_crtc(state->state, crtc) > 1)
+	if (hweight32(state->connector_mask) > 1)
 		return -EINVAL;
 
 	drm_atomic_crtc_state_for_each_plane(plane, state) {
@@ -476,10 +477,106 @@ static irqreturn_t vc4_crtc_irq_handler(int irq, void *data)
 	return ret;
 }
 
+struct vc4_async_flip_state {
+	struct drm_crtc *crtc;
+	struct drm_framebuffer *fb;
+	struct drm_pending_vblank_event *event;
+
+	struct vc4_seqno_cb cb;
+};
+
+/* Called when the V3D execution for the BO being flipped to is done, so that
+ * we can actually update the plane's address to point to it.
+ */
+static void
+vc4_async_page_flip_complete(struct vc4_seqno_cb *cb)
+{
+	struct vc4_async_flip_state *flip_state =
+		container_of(cb, struct vc4_async_flip_state, cb);
+	struct drm_crtc *crtc = flip_state->crtc;
+	struct drm_device *dev = crtc->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_plane *plane = crtc->primary;
+
+	vc4_plane_async_set_fb(plane, flip_state->fb);
+	if (flip_state->event) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, flip_state->event);
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
+
+	drm_framebuffer_unreference(flip_state->fb);
+	kfree(flip_state);
+
+	up(&vc4->async_modeset);
+}
+
+/* Implements async (non-vblank-synced) page flips.
+ *
+ * The page flip ioctl needs to return immediately, so we grab the
+ * modeset semaphore on the pipe, and queue the address update for
+ * when V3D is done with the BO being flipped to.
+ */
+static int vc4_async_page_flip(struct drm_crtc *crtc,
+			       struct drm_framebuffer *fb,
+			       struct drm_pending_vblank_event *event,
+			       uint32_t flags)
+{
+	struct drm_device *dev = crtc->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_plane *plane = crtc->primary;
+	int ret = 0;
+	struct vc4_async_flip_state *flip_state;
+	struct drm_gem_cma_object *cma_bo = drm_fb_cma_get_gem_obj(fb, 0);
+	struct vc4_bo *bo = to_vc4_bo(&cma_bo->base);
+
+	flip_state = kzalloc(sizeof(*flip_state), GFP_KERNEL);
+	if (!flip_state)
+		return -ENOMEM;
+
+	drm_framebuffer_reference(fb);
+	flip_state->fb = fb;
+	flip_state->crtc = crtc;
+	flip_state->event = event;
+
+	/* Make sure all other async modesetes have landed. */
+	ret = down_interruptible(&vc4->async_modeset);
+	if (ret) {
+		kfree(flip_state);
+		return ret;
+	}
+
+	/* Immediately update the plane's legacy fb pointer, so that later
+	 * modeset prep sees the state that will be present when the semaphore
+	 * is released.
+	 */
+	drm_atomic_set_fb_for_plane(plane->state, fb);
+	plane->fb = fb;
+
+	vc4_queue_seqno_cb(dev, &flip_state->cb, bo->seqno,
+			   vc4_async_page_flip_complete);
+
+	/* Driver takes ownership of state on successful async commit. */
+	return 0;
+}
+
+static int vc4_page_flip(struct drm_crtc *crtc,
+			 struct drm_framebuffer *fb,
+			 struct drm_pending_vblank_event *event,
+			 uint32_t flags)
+{
+	if (flags & DRM_MODE_PAGE_FLIP_ASYNC)
+		return vc4_async_page_flip(crtc, fb, event, flags);
+	else
+		return drm_atomic_helper_page_flip(crtc, fb, event, flags);
+}
+
 static const struct drm_crtc_funcs vc4_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = vc4_crtc_destroy,
-	.page_flip = drm_atomic_helper_page_flip,
+	.page_flip = vc4_page_flip,
 	.set_property = NULL,
 	.cursor_set = NULL, /* handled by drm_mode_cursor_universal */
 	.cursor_move = NULL, /* handled by drm_mode_cursor_universal */
@@ -606,7 +703,7 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 	}
 
 	drm_crtc_init_with_planes(drm, crtc, primary_plane, cursor_plane,
-				  &vc4_crtc_funcs);
+				  &vc4_crtc_funcs, NULL);
 	drm_crtc_helper_add(crtc, &vc4_crtc_helper_funcs);
 	primary_plane->crtc = crtc;
 	cursor_plane->crtc = crtc;
diff --git a/drivers/gpu/drm/vc4/vc4_debugfs.c b/drivers/gpu/drm/vc4/vc4_debugfs.c
index 4297b0a5b74e..d76ad10b07fd 100644
--- a/drivers/gpu/drm/vc4/vc4_debugfs.c
+++ b/drivers/gpu/drm/vc4/vc4_debugfs.c
@@ -16,11 +16,14 @@
 #include "vc4_regs.h"
 
 static const struct drm_info_list vc4_debugfs_list[] = {
+	{"bo_stats", vc4_bo_stats_debugfs, 0},
 	{"hdmi_regs", vc4_hdmi_debugfs_regs, 0},
 	{"hvs_regs", vc4_hvs_debugfs_regs, 0},
 	{"crtc0_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)0},
 	{"crtc1_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)1},
 	{"crtc2_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)2},
+	{"v3d_ident", vc4_v3d_debugfs_ident, 0},
+	{"v3d_regs", vc4_v3d_debugfs_regs, 0},
 };
 
 #define VC4_DEBUGFS_ENTRIES ARRAY_SIZE(vc4_debugfs_list)
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index d5db9e0f3b73..f1655fff8425 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include "drm_fb_cma_helper.h"
 
+#include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
@@ -63,7 +64,7 @@ static const struct file_operations vc4_drm_fops = {
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
-	.mmap = drm_gem_cma_mmap,
+	.mmap = vc4_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
 #ifdef CONFIG_COMPAT
@@ -73,16 +74,30 @@ static const struct file_operations vc4_drm_fops = {
 };
 
 static const struct drm_ioctl_desc vc4_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(VC4_SUBMIT_CL, vc4_submit_cl_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_WAIT_SEQNO, vc4_wait_seqno_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_WAIT_BO, vc4_wait_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_CREATE_BO, vc4_create_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_MMAP_BO, vc4_mmap_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_CREATE_SHADER_BO, vc4_create_shader_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(VC4_GET_HANG_STATE, vc4_get_hang_state_ioctl,
+			  DRM_ROOT_ONLY),
 };
 
 static struct drm_driver vc4_drm_driver = {
 	.driver_features = (DRIVER_MODESET |
 			    DRIVER_ATOMIC |
 			    DRIVER_GEM |
+			    DRIVER_HAVE_IRQ |
 			    DRIVER_PRIME),
 	.lastclose = vc4_lastclose,
 	.preclose = vc4_drm_preclose,
 
+	.irq_handler = vc4_irq,
+	.irq_preinstall = vc4_irq_preinstall,
+	.irq_postinstall = vc4_irq_postinstall,
+	.irq_uninstall = vc4_irq_uninstall,
+
 	.enable_vblank = vc4_enable_vblank,
 	.disable_vblank = vc4_disable_vblank,
 	.get_vblank_counter = drm_vblank_count,
@@ -92,18 +107,19 @@ static struct drm_driver vc4_drm_driver = {
 	.debugfs_cleanup = vc4_debugfs_cleanup,
 #endif
 
-	.gem_free_object = drm_gem_cma_free_object,
+	.gem_create_object = vc4_create_object,
+	.gem_free_object = vc4_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_import = drm_gem_prime_import,
-	.gem_prime_export = drm_gem_prime_export,
+	.gem_prime_export = vc4_prime_export,
 	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
 	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
-	.gem_prime_vmap = drm_gem_cma_prime_vmap,
+	.gem_prime_vmap = vc4_prime_vmap,
 	.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
-	.gem_prime_mmap = drm_gem_cma_prime_mmap,
+	.gem_prime_mmap = vc4_prime_mmap,
 
 	.dumb_create = vc4_dumb_create,
 	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
@@ -168,15 +184,17 @@ static int vc4_drm_bind(struct device *dev)
 	vc4->dev = drm;
 	drm->dev_private = vc4;
 
-	drm_dev_set_unique(drm, dev_name(dev));
+	vc4_bo_cache_init(drm);
 
 	drm_mode_config_init(drm);
 	if (ret)
 		goto unref;
 
+	vc4_gem_init(drm);
+
 	ret = component_bind_all(dev, drm);
 	if (ret)
-		goto unref;
+		goto gem_destroy;
 
 	ret = drm_dev_register(drm, 0);
 	if (ret < 0)
@@ -200,8 +218,11 @@ unregister:
 	drm_dev_unregister(drm);
 unbind_all:
 	component_unbind_all(dev, drm);
+gem_destroy:
+	vc4_gem_destroy(drm);
 unref:
 	drm_dev_unref(drm);
+	vc4_bo_cache_destroy(drm);
 	return ret;
 }
 
@@ -228,6 +249,7 @@ static struct platform_driver *const component_drivers[] = {
 	&vc4_hdmi_driver,
 	&vc4_crtc_driver,
 	&vc4_hvs_driver,
+	&vc4_v3d_driver,
 };
 
 static int vc4_platform_drm_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index fd8319fa682e..080865ec2bae 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -15,8 +15,89 @@ struct vc4_dev {
 	struct vc4_hdmi *hdmi;
 	struct vc4_hvs *hvs;
 	struct vc4_crtc *crtc[3];
+	struct vc4_v3d *v3d;
 
 	struct drm_fbdev_cma *fbdev;
+
+	struct vc4_hang_state *hang_state;
+
+	/* The kernel-space BO cache.  Tracks buffers that have been
+	 * unreferenced by all other users (refcounts of 0!) but not
+	 * yet freed, so we can do cheap allocations.
+	 */
+	struct vc4_bo_cache {
+		/* Array of list heads for entries in the BO cache,
+		 * based on number of pages, so we can do O(1) lookups
+		 * in the cache when allocating.
+		 */
+		struct list_head *size_list;
+		uint32_t size_list_size;
+
+		/* List of all BOs in the cache, ordered by age, so we
+		 * can do O(1) lookups when trying to free old
+		 * buffers.
+		 */
+		struct list_head time_list;
+		struct work_struct time_work;
+		struct timer_list time_timer;
+	} bo_cache;
+
+	struct vc4_bo_stats {
+		u32 num_allocated;
+		u32 size_allocated;
+		u32 num_cached;
+		u32 size_cached;
+	} bo_stats;
+
+	/* Protects bo_cache and the BO stats. */
+	struct mutex bo_lock;
+
+	/* Sequence number for the last job queued in job_list.
+	 * Starts at 0 (no jobs emitted).
+	 */
+	uint64_t emit_seqno;
+
+	/* Sequence number for the last completed job on the GPU.
+	 * Starts at 0 (no jobs completed).
+	 */
+	uint64_t finished_seqno;
+
+	/* List of all struct vc4_exec_info for jobs to be executed.
+	 * The first job in the list is the one currently programmed
+	 * into ct0ca/ct1ca for execution.
+	 */
+	struct list_head job_list;
+	/* List of the finished vc4_exec_infos waiting to be freed by
+	 * job_done_work.
+	 */
+	struct list_head job_done_list;
+	/* Spinlock used to synchronize the job_list and seqno
+	 * accesses between the IRQ handler and GEM ioctls.
+	 */
+	spinlock_t job_lock;
+	wait_queue_head_t job_wait_queue;
+	struct work_struct job_done_work;
+
+	/* List of struct vc4_seqno_cb for callbacks to be made from a
+	 * workqueue when the given seqno is passed.
+	 */
+	struct list_head seqno_cb_list;
+
+	/* The binner overflow memory that's currently set up in
+	 * BPOA/BPOS registers.  When overflow occurs and a new one is
+	 * allocated, the previous one will be moved to
+	 * vc4->current_exec's free list.
+	 */
+	struct vc4_bo *overflow_mem;
+	struct work_struct overflow_mem_work;
+
+	struct {
+		uint32_t last_ct0ca, last_ct1ca;
+		struct timer_list timer;
+		struct work_struct reset_work;
+	} hangcheck;
+
+	struct semaphore async_modeset;
 };
 
 static inline struct vc4_dev *
@@ -27,6 +108,25 @@ to_vc4_dev(struct drm_device *dev)
 
 struct vc4_bo {
 	struct drm_gem_cma_object base;
+
+	/* seqno of the last job to render to this BO. */
+	uint64_t seqno;
+
+	/* List entry for the BO's position in either
+	 * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list
+	 */
+	struct list_head unref_head;
+
+	/* Time in jiffies when the BO was put in vc4->bo_cache. */
+	unsigned long free_time;
+
+	/* List entry for the BO's position in vc4_dev->bo_cache.size_list */
+	struct list_head size_head;
+
+	/* Struct for shader validation state, if created by
+	 * DRM_IOCTL_VC4_CREATE_SHADER_BO.
+	 */
+	struct vc4_validated_shader_info *validated_shader;
 };
 
 static inline struct vc4_bo *
@@ -35,6 +135,17 @@ to_vc4_bo(struct drm_gem_object *bo)
 	return (struct vc4_bo *)bo;
 }
 
+struct vc4_seqno_cb {
+	struct work_struct work;
+	uint64_t seqno;
+	void (*func)(struct vc4_seqno_cb *cb);
+};
+
+struct vc4_v3d {
+	struct platform_device *pdev;
+	void __iomem *regs;
+};
+
 struct vc4_hvs {
 	struct platform_device *pdev;
 	void __iomem *regs;
@@ -72,9 +183,142 @@ to_vc4_encoder(struct drm_encoder *encoder)
 	return container_of(encoder, struct vc4_encoder, base);
 }
 
+#define V3D_READ(offset) readl(vc4->v3d->regs + offset)
+#define V3D_WRITE(offset, val) writel(val, vc4->v3d->regs + offset)
 #define HVS_READ(offset) readl(vc4->hvs->regs + offset)
 #define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset)
 
+struct vc4_exec_info {
+	/* Sequence number for this bin/render job. */
+	uint64_t seqno;
+
+	/* Kernel-space copy of the ioctl arguments */
+	struct drm_vc4_submit_cl *args;
+
+	/* This is the array of BOs that were looked up at the start of exec.
+	 * Command validation will use indices into this array.
+	 */
+	struct drm_gem_cma_object **bo;
+	uint32_t bo_count;
+
+	/* Pointers for our position in vc4->job_list */
+	struct list_head head;
+
+	/* List of other BOs used in the job that need to be released
+	 * once the job is complete.
+	 */
+	struct list_head unref_list;
+
+	/* Current unvalidated indices into @bo loaded by the non-hardware
+	 * VC4_PACKET_GEM_HANDLES.
+	 */
+	uint32_t bo_index[2];
+
+	/* This is the BO where we store the validated command lists, shader
+	 * records, and uniforms.
+	 */
+	struct drm_gem_cma_object *exec_bo;
+
+	/**
+	 * This tracks the per-shader-record state (packet 64) that
+	 * determines the length of the shader record and the offset
+	 * it's expected to be found at.  It gets read in from the
+	 * command lists.
+	 */
+	struct vc4_shader_state {
+		uint32_t addr;
+		/* Maximum vertex index referenced by any primitive using this
+		 * shader state.
+		 */
+		uint32_t max_index;
+	} *shader_state;
+
+	/** How many shader states the user declared they were using. */
+	uint32_t shader_state_size;
+	/** How many shader state records the validator has seen. */
+	uint32_t shader_state_count;
+
+	bool found_tile_binning_mode_config_packet;
+	bool found_start_tile_binning_packet;
+	bool found_increment_semaphore_packet;
+	bool found_flush;
+	uint8_t bin_tiles_x, bin_tiles_y;
+	struct drm_gem_cma_object *tile_bo;
+	uint32_t tile_alloc_offset;
+
+	/**
+	 * Computed addresses pointing into exec_bo where we start the
+	 * bin thread (ct0) and render thread (ct1).
+	 */
+	uint32_t ct0ca, ct0ea;
+	uint32_t ct1ca, ct1ea;
+
+	/* Pointer to the unvalidated bin CL (if present). */
+	void *bin_u;
+
+	/* Pointers to the shader recs.  These paddr gets incremented as CL
+	 * packets are relocated in validate_gl_shader_state, and the vaddrs
+	 * (u and v) get incremented and size decremented as the shader recs
+	 * themselves are validated.
+	 */
+	void *shader_rec_u;
+	void *shader_rec_v;
+	uint32_t shader_rec_p;
+	uint32_t shader_rec_size;
+
+	/* Pointers to the uniform data.  These pointers are incremented, and
+	 * size decremented, as each batch of uniforms is uploaded.
+	 */
+	void *uniforms_u;
+	void *uniforms_v;
+	uint32_t uniforms_p;
+	uint32_t uniforms_size;
+};
+
+static inline struct vc4_exec_info *
+vc4_first_job(struct vc4_dev *vc4)
+{
+	if (list_empty(&vc4->job_list))
+		return NULL;
+	return list_first_entry(&vc4->job_list, struct vc4_exec_info, head);
+}
+
+/**
+ * struct vc4_texture_sample_info - saves the offsets into the UBO for texture
+ * setup parameters.
+ *
+ * This will be used at draw time to relocate the reference to the texture
+ * contents in p0, and validate that the offset combined with
+ * width/height/stride/etc. from p1 and p2/p3 doesn't sample outside the BO.
+ * Note that the hardware treats unprovided config parameters as 0, so not all
+ * of them need to be set up for every texure sample, and we'll store ~0 as
+ * the offset to mark the unused ones.
+ *
+ * See the VC4 3D architecture guide page 41 ("Texture and Memory Lookup Unit
+ * Setup") for definitions of the texture parameters.
+ */
+struct vc4_texture_sample_info {
+	bool is_direct;
+	uint32_t p_offset[4];
+};
+
+/**
+ * struct vc4_validated_shader_info - information about validated shaders that
+ * needs to be used from command list validation.
+ *
+ * For a given shader, each time a shader state record references it, we need
+ * to verify that the shader doesn't read more uniforms than the shader state
+ * record's uniform BO pointer can provide, and we need to apply relocations
+ * and validate the shader state record's uniforms that define the texture
+ * samples.
+ */
+struct vc4_validated_shader_info {
+	uint32_t uniforms_size;
+	uint32_t uniforms_src_size;
+	uint32_t num_texture_samples;
+	struct vc4_texture_sample_info *texture_samples;
+};
+
 /**
  * _wait_for - magic (register) wait macro
  *
@@ -104,13 +348,29 @@ to_vc4_encoder(struct drm_encoder *encoder)
 #define wait_for(COND, MS) _wait_for(COND, MS, 1)
 
 /* vc4_bo.c */
+struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size);
 void vc4_free_object(struct drm_gem_object *gem_obj);
-struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size);
+struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size,
+			     bool from_cache);
 int vc4_dumb_create(struct drm_file *file_priv,
 		    struct drm_device *dev,
 		    struct drm_mode_create_dumb *args);
 struct dma_buf *vc4_prime_export(struct drm_device *dev,
 				 struct drm_gem_object *obj, int flags);
+int vc4_create_bo_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file_priv);
+int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
+int vc4_mmap(struct file *filp, struct vm_area_struct *vma);
+int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+void *vc4_prime_vmap(struct drm_gem_object *obj);
+void vc4_bo_cache_init(struct drm_device *dev);
+void vc4_bo_cache_destroy(struct drm_device *dev);
+int vc4_bo_stats_debugfs(struct seq_file *m, void *arg);
 
 /* vc4_crtc.c */
 extern struct platform_driver vc4_crtc_driver;
@@ -126,10 +386,34 @@ void vc4_debugfs_cleanup(struct drm_minor *minor);
 /* vc4_drv.c */
 void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index);
 
+/* vc4_gem.c */
+void vc4_gem_init(struct drm_device *dev);
+void vc4_gem_destroy(struct drm_device *dev);
+int vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+int vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+void vc4_submit_next_job(struct drm_device *dev);
+int vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno,
+		       uint64_t timeout_ns, bool interruptible);
+void vc4_job_handle_completed(struct vc4_dev *vc4);
+int vc4_queue_seqno_cb(struct drm_device *dev,
+		       struct vc4_seqno_cb *cb, uint64_t seqno,
+		       void (*func)(struct vc4_seqno_cb *cb));
+
 /* vc4_hdmi.c */
 extern struct platform_driver vc4_hdmi_driver;
 int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused);
 
+/* vc4_irq.c */
+irqreturn_t vc4_irq(int irq, void *arg);
+void vc4_irq_preinstall(struct drm_device *dev);
+int vc4_irq_postinstall(struct drm_device *dev);
+void vc4_irq_uninstall(struct drm_device *dev);
+void vc4_irq_reset(struct drm_device *dev);
+
 /* vc4_hvs.c */
 extern struct platform_driver vc4_hvs_driver;
 void vc4_hvs_dump_state(struct drm_device *dev);
@@ -143,3 +427,35 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 				 enum drm_plane_type type);
 u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist);
 u32 vc4_plane_dlist_size(struct drm_plane_state *state);
+void vc4_plane_async_set_fb(struct drm_plane *plane,
+			    struct drm_framebuffer *fb);
+
+/* vc4_v3d.c */
+extern struct platform_driver vc4_v3d_driver;
+int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused);
+int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused);
+int vc4_v3d_set_power(struct vc4_dev *vc4, bool on);
+
+/* vc4_validate.c */
+int
+vc4_validate_bin_cl(struct drm_device *dev,
+		    void *validated,
+		    void *unvalidated,
+		    struct vc4_exec_info *exec);
+
+int
+vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec);
+
+struct drm_gem_cma_object *vc4_use_bo(struct vc4_exec_info *exec,
+				      uint32_t hindex);
+
+int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec);
+
+bool vc4_check_tex_size(struct vc4_exec_info *exec,
+			struct drm_gem_cma_object *fbo,
+			uint32_t offset, uint8_t tiling_format,
+			uint32_t width, uint32_t height, uint8_t cpp);
+
+/* vc4_validate_shader.c */
+struct vc4_validated_shader_info *
+vc4_validate_shader(struct drm_gem_cma_object *shader_obj);
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
new file mode 100644
index 000000000000..48ce30a6f4b5
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -0,0 +1,866 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/io.h>
+
+#include "uapi/drm/vc4_drm.h"
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+#include "vc4_trace.h"
+
+static void
+vc4_queue_hangcheck(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	mod_timer(&vc4->hangcheck.timer,
+		  round_jiffies_up(jiffies + msecs_to_jiffies(100)));
+}
+
+struct vc4_hang_state {
+	struct drm_vc4_get_hang_state user_state;
+
+	u32 bo_count;
+	struct drm_gem_object **bo;
+};
+
+static void
+vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)
+{
+	unsigned int i;
+
+	mutex_lock(&dev->struct_mutex);
+	for (i = 0; i < state->user_state.bo_count; i++)
+		drm_gem_object_unreference(state->bo[i]);
+	mutex_unlock(&dev->struct_mutex);
+
+	kfree(state);
+}
+
+int
+vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	struct drm_vc4_get_hang_state *get_state = data;
+	struct drm_vc4_get_hang_state_bo *bo_state;
+	struct vc4_hang_state *kernel_state;
+	struct drm_vc4_get_hang_state *state;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	unsigned long irqflags;
+	u32 i;
+	int ret = 0;
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	kernel_state = vc4->hang_state;
+	if (!kernel_state) {
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		return -ENOENT;
+	}
+	state = &kernel_state->user_state;
+
+	/* If the user's array isn't big enough, just return the
+	 * required array size.
+	 */
+	if (get_state->bo_count < state->bo_count) {
+		get_state->bo_count = state->bo_count;
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		return 0;
+	}
+
+	vc4->hang_state = NULL;
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+
+	/* Save the user's BO pointer, so we don't stomp it with the memcpy. */
+	state->bo = get_state->bo;
+	memcpy(get_state, state, sizeof(*state));
+
+	bo_state = kcalloc(state->bo_count, sizeof(*bo_state), GFP_KERNEL);
+	if (!bo_state) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+
+	for (i = 0; i < state->bo_count; i++) {
+		struct vc4_bo *vc4_bo = to_vc4_bo(kernel_state->bo[i]);
+		u32 handle;
+
+		ret = drm_gem_handle_create(file_priv, kernel_state->bo[i],
+					    &handle);
+
+		if (ret) {
+			state->bo_count = i - 1;
+			goto err;
+		}
+		bo_state[i].handle = handle;
+		bo_state[i].paddr = vc4_bo->base.paddr;
+		bo_state[i].size = vc4_bo->base.base.size;
+	}
+
+	if (copy_to_user((void __user *)(uintptr_t)get_state->bo,
+			 bo_state,
+			 state->bo_count * sizeof(*bo_state)))
+		ret = -EFAULT;
+
+	kfree(bo_state);
+
+err_free:
+
+	vc4_free_hang_state(dev, kernel_state);
+
+err:
+	return ret;
+}
+
+static void
+vc4_save_hang_state(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_vc4_get_hang_state *state;
+	struct vc4_hang_state *kernel_state;
+	struct vc4_exec_info *exec;
+	struct vc4_bo *bo;
+	unsigned long irqflags;
+	unsigned int i, unref_list_count;
+
+	kernel_state = kcalloc(1, sizeof(*kernel_state), GFP_KERNEL);
+	if (!kernel_state)
+		return;
+
+	state = &kernel_state->user_state;
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	exec = vc4_first_job(vc4);
+	if (!exec) {
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		return;
+	}
+
+	unref_list_count = 0;
+	list_for_each_entry(bo, &exec->unref_list, unref_head)
+		unref_list_count++;
+
+	state->bo_count = exec->bo_count + unref_list_count;
+	kernel_state->bo = kcalloc(state->bo_count, sizeof(*kernel_state->bo),
+				   GFP_ATOMIC);
+	if (!kernel_state->bo) {
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		return;
+	}
+
+	for (i = 0; i < exec->bo_count; i++) {
+		drm_gem_object_reference(&exec->bo[i]->base);
+		kernel_state->bo[i] = &exec->bo[i]->base;
+	}
+
+	list_for_each_entry(bo, &exec->unref_list, unref_head) {
+		drm_gem_object_reference(&bo->base.base);
+		kernel_state->bo[i] = &bo->base.base;
+		i++;
+	}
+
+	state->start_bin = exec->ct0ca;
+	state->start_render = exec->ct1ca;
+
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+
+	state->ct0ca = V3D_READ(V3D_CTNCA(0));
+	state->ct0ea = V3D_READ(V3D_CTNEA(0));
+
+	state->ct1ca = V3D_READ(V3D_CTNCA(1));
+	state->ct1ea = V3D_READ(V3D_CTNEA(1));
+
+	state->ct0cs = V3D_READ(V3D_CTNCS(0));
+	state->ct1cs = V3D_READ(V3D_CTNCS(1));
+
+	state->ct0ra0 = V3D_READ(V3D_CT00RA0);
+	state->ct1ra0 = V3D_READ(V3D_CT01RA0);
+
+	state->bpca = V3D_READ(V3D_BPCA);
+	state->bpcs = V3D_READ(V3D_BPCS);
+	state->bpoa = V3D_READ(V3D_BPOA);
+	state->bpos = V3D_READ(V3D_BPOS);
+
+	state->vpmbase = V3D_READ(V3D_VPMBASE);
+
+	state->dbge = V3D_READ(V3D_DBGE);
+	state->fdbgo = V3D_READ(V3D_FDBGO);
+	state->fdbgb = V3D_READ(V3D_FDBGB);
+	state->fdbgr = V3D_READ(V3D_FDBGR);
+	state->fdbgs = V3D_READ(V3D_FDBGS);
+	state->errstat = V3D_READ(V3D_ERRSTAT);
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	if (vc4->hang_state) {
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		vc4_free_hang_state(dev, kernel_state);
+	} else {
+		vc4->hang_state = kernel_state;
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+	}
+}
+
+static void
+vc4_reset(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	DRM_INFO("Resetting GPU.\n");
+	vc4_v3d_set_power(vc4, false);
+	vc4_v3d_set_power(vc4, true);
+
+	vc4_irq_reset(dev);
+
+	/* Rearm the hangcheck -- another job might have been waiting
+	 * for our hung one to get kicked off, and vc4_irq_reset()
+	 * would have started it.
+	 */
+	vc4_queue_hangcheck(dev);
+}
+
+static void
+vc4_reset_work(struct work_struct *work)
+{
+	struct vc4_dev *vc4 =
+		container_of(work, struct vc4_dev, hangcheck.reset_work);
+
+	vc4_save_hang_state(vc4->dev);
+
+	vc4_reset(vc4->dev);
+}
+
+static void
+vc4_hangcheck_elapsed(unsigned long data)
+{
+	struct drm_device *dev = (struct drm_device *)data;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint32_t ct0ca, ct1ca;
+
+	/* If idle, we can stop watching for hangs. */
+	if (list_empty(&vc4->job_list))
+		return;
+
+	ct0ca = V3D_READ(V3D_CTNCA(0));
+	ct1ca = V3D_READ(V3D_CTNCA(1));
+
+	/* If we've made any progress in execution, rearm the timer
+	 * and wait.
+	 */
+	if (ct0ca != vc4->hangcheck.last_ct0ca ||
+	    ct1ca != vc4->hangcheck.last_ct1ca) {
+		vc4->hangcheck.last_ct0ca = ct0ca;
+		vc4->hangcheck.last_ct1ca = ct1ca;
+		vc4_queue_hangcheck(dev);
+		return;
+	}
+
+	/* We've gone too long with no progress, reset.  This has to
+	 * be done from a work struct, since resetting can sleep and
+	 * this timer hook isn't allowed to.
+	 */
+	schedule_work(&vc4->hangcheck.reset_work);
+}
+
+static void
+submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Set the current and end address of the control list.
+	 * Writing the end register is what starts the job.
+	 */
+	V3D_WRITE(V3D_CTNCA(thread), start);
+	V3D_WRITE(V3D_CTNEA(thread), end);
+}
+
+int
+vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns,
+		   bool interruptible)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int ret = 0;
+	unsigned long timeout_expire;
+	DEFINE_WAIT(wait);
+
+	if (vc4->finished_seqno >= seqno)
+		return 0;
+
+	if (timeout_ns == 0)
+		return -ETIME;
+
+	timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns);
+
+	trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns);
+	for (;;) {
+		prepare_to_wait(&vc4->job_wait_queue, &wait,
+				interruptible ? TASK_INTERRUPTIBLE :
+				TASK_UNINTERRUPTIBLE);
+
+		if (interruptible && signal_pending(current)) {
+			ret = -ERESTARTSYS;
+			break;
+		}
+
+		if (vc4->finished_seqno >= seqno)
+			break;
+
+		if (timeout_ns != ~0ull) {
+			if (time_after_eq(jiffies, timeout_expire)) {
+				ret = -ETIME;
+				break;
+			}
+			schedule_timeout(timeout_expire - jiffies);
+		} else {
+			schedule();
+		}
+	}
+
+	finish_wait(&vc4->job_wait_queue, &wait);
+	trace_vc4_wait_for_seqno_end(dev, seqno);
+
+	if (ret && ret != -ERESTARTSYS) {
+		DRM_ERROR("timeout waiting for render thread idle\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void
+vc4_flush_caches(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Flush the GPU L2 caches.  These caches sit on top of system
+	 * L3 (the 128kb or so shared with the CPU), and are
+	 * non-allocating in the L3.
+	 */
+	V3D_WRITE(V3D_L2CACTL,
+		  V3D_L2CACTL_L2CCLR);
+
+	V3D_WRITE(V3D_SLCACTL,
+		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |
+		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) |
+		  VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) |
+		  VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC));
+}
+
+/* Sets the registers for the next job to be actually be executed in
+ * the hardware.
+ *
+ * The job_lock should be held during this.
+ */
+void
+vc4_submit_next_job(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_exec_info *exec = vc4_first_job(vc4);
+
+	if (!exec)
+		return;
+
+	vc4_flush_caches(dev);
+
+	/* Disable the binner's pre-loaded overflow memory address */
+	V3D_WRITE(V3D_BPOA, 0);
+	V3D_WRITE(V3D_BPOS, 0);
+
+	if (exec->ct0ca != exec->ct0ea)
+		submit_cl(dev, 0, exec->ct0ca, exec->ct0ea);
+	submit_cl(dev, 1, exec->ct1ca, exec->ct1ea);
+}
+
+static void
+vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
+{
+	struct vc4_bo *bo;
+	unsigned i;
+
+	for (i = 0; i < exec->bo_count; i++) {
+		bo = to_vc4_bo(&exec->bo[i]->base);
+		bo->seqno = seqno;
+	}
+
+	list_for_each_entry(bo, &exec->unref_list, unref_head) {
+		bo->seqno = seqno;
+	}
+}
+
+/* Queues a struct vc4_exec_info for execution.  If no job is
+ * currently executing, then submits it.
+ *
+ * Unlike most GPUs, our hardware only handles one command list at a
+ * time.  To queue multiple jobs at once, we'd need to edit the
+ * previous command list to have a jump to the new one at the end, and
+ * then bump the end address.  That's a change for a later date,
+ * though.
+ */
+static void
+vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint64_t seqno;
+	unsigned long irqflags;
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+
+	seqno = ++vc4->emit_seqno;
+	exec->seqno = seqno;
+	vc4_update_bo_seqnos(exec, seqno);
+
+	list_add_tail(&exec->head, &vc4->job_list);
+
+	/* If no job was executing, kick ours off.  Otherwise, it'll
+	 * get started when the previous job's frame done interrupt
+	 * occurs.
+	 */
+	if (vc4_first_job(vc4) == exec) {
+		vc4_submit_next_job(dev);
+		vc4_queue_hangcheck(dev);
+	}
+
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+}
+
+/**
+ * Looks up a bunch of GEM handles for BOs and stores the array for
+ * use in the command validator that actually writes relocated
+ * addresses pointing to them.
+ */
+static int
+vc4_cl_lookup_bos(struct drm_device *dev,
+		  struct drm_file *file_priv,
+		  struct vc4_exec_info *exec)
+{
+	struct drm_vc4_submit_cl *args = exec->args;
+	uint32_t *handles;
+	int ret = 0;
+	int i;
+
+	exec->bo_count = args->bo_handle_count;
+
+	if (!exec->bo_count) {
+		/* See comment on bo_index for why we have to check
+		 * this.
+		 */
+		DRM_ERROR("Rendering requires BOs to validate\n");
+		return -EINVAL;
+	}
+
+	exec->bo = kcalloc(exec->bo_count, sizeof(struct drm_gem_cma_object *),
+			   GFP_KERNEL);
+	if (!exec->bo) {
+		DRM_ERROR("Failed to allocate validated BO pointers\n");
+		return -ENOMEM;
+	}
+
+	handles = drm_malloc_ab(exec->bo_count, sizeof(uint32_t));
+	if (!handles) {
+		DRM_ERROR("Failed to allocate incoming GEM handles\n");
+		goto fail;
+	}
+
+	ret = copy_from_user(handles,
+			     (void __user *)(uintptr_t)args->bo_handles,
+			     exec->bo_count * sizeof(uint32_t));
+	if (ret) {
+		DRM_ERROR("Failed to copy in GEM handles\n");
+		goto fail;
+	}
+
+	spin_lock(&file_priv->table_lock);
+	for (i = 0; i < exec->bo_count; i++) {
+		struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
+						     handles[i]);
+		if (!bo) {
+			DRM_ERROR("Failed to look up GEM BO %d: %d\n",
+				  i, handles[i]);
+			ret = -EINVAL;
+			spin_unlock(&file_priv->table_lock);
+			goto fail;
+		}
+		drm_gem_object_reference(bo);
+		exec->bo[i] = (struct drm_gem_cma_object *)bo;
+	}
+	spin_unlock(&file_priv->table_lock);
+
+fail:
+	kfree(handles);
+	return 0;
+}
+
+static int
+vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
+{
+	struct drm_vc4_submit_cl *args = exec->args;
+	void *temp = NULL;
+	void *bin;
+	int ret = 0;
+	uint32_t bin_offset = 0;
+	uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,
+					     16);
+	uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;
+	uint32_t exec_size = uniforms_offset + args->uniforms_size;
+	uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *
+					  args->shader_rec_count);
+	struct vc4_bo *bo;
+
+	if (uniforms_offset < shader_rec_offset ||
+	    exec_size < uniforms_offset ||
+	    args->shader_rec_count >= (UINT_MAX /
+					  sizeof(struct vc4_shader_state)) ||
+	    temp_size < exec_size) {
+		DRM_ERROR("overflow in exec arguments\n");
+		goto fail;
+	}
+
+	/* Allocate space where we'll store the copied in user command lists
+	 * and shader records.
+	 *
+	 * We don't just copy directly into the BOs because we need to
+	 * read the contents back for validation, and I think the
+	 * bo->vaddr is uncached access.
+	 */
+	temp = kmalloc(temp_size, GFP_KERNEL);
+	if (!temp) {
+		DRM_ERROR("Failed to allocate storage for copying "
+			  "in bin/render CLs.\n");
+		ret = -ENOMEM;
+		goto fail;
+	}
+	bin = temp + bin_offset;
+	exec->shader_rec_u = temp + shader_rec_offset;
+	exec->uniforms_u = temp + uniforms_offset;
+	exec->shader_state = temp + exec_size;
+	exec->shader_state_size = args->shader_rec_count;
+
+	if (copy_from_user(bin,
+			   (void __user *)(uintptr_t)args->bin_cl,
+			   args->bin_cl_size)) {
+		ret = -EFAULT;
+		goto fail;
+	}
+
+	if (copy_from_user(exec->shader_rec_u,
+			   (void __user *)(uintptr_t)args->shader_rec,
+			   args->shader_rec_size)) {
+		ret = -EFAULT;
+		goto fail;
+	}
+
+	if (copy_from_user(exec->uniforms_u,
+			   (void __user *)(uintptr_t)args->uniforms,
+			   args->uniforms_size)) {
+		ret = -EFAULT;
+		goto fail;
+	}
+
+	bo = vc4_bo_create(dev, exec_size, true);
+	if (!bo) {
+		DRM_ERROR("Couldn't allocate BO for binning\n");
+		ret = -ENOMEM;
+		goto fail;
+	}
+	exec->exec_bo = &bo->base;
+
+	list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,
+		      &exec->unref_list);
+
+	exec->ct0ca = exec->exec_bo->paddr + bin_offset;
+
+	exec->bin_u = bin;
+
+	exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;
+	exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset;
+	exec->shader_rec_size = args->shader_rec_size;
+
+	exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;
+	exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset;
+	exec->uniforms_size = args->uniforms_size;
+
+	ret = vc4_validate_bin_cl(dev,
+				  exec->exec_bo->vaddr + bin_offset,
+				  bin,
+				  exec);
+	if (ret)
+		goto fail;
+
+	ret = vc4_validate_shader_recs(dev, exec);
+
+fail:
+	kfree(temp);
+	return ret;
+}
+
+static void
+vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
+{
+	unsigned i;
+
+	/* Need the struct lock for drm_gem_object_unreference(). */
+	mutex_lock(&dev->struct_mutex);
+	if (exec->bo) {
+		for (i = 0; i < exec->bo_count; i++)
+			drm_gem_object_unreference(&exec->bo[i]->base);
+		kfree(exec->bo);
+	}
+
+	while (!list_empty(&exec->unref_list)) {
+		struct vc4_bo *bo = list_first_entry(&exec->unref_list,
+						     struct vc4_bo, unref_head);
+		list_del(&bo->unref_head);
+		drm_gem_object_unreference(&bo->base.base);
+	}
+	mutex_unlock(&dev->struct_mutex);
+
+	kfree(exec);
+}
+
+void
+vc4_job_handle_completed(struct vc4_dev *vc4)
+{
+	unsigned long irqflags;
+	struct vc4_seqno_cb *cb, *cb_temp;
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	while (!list_empty(&vc4->job_done_list)) {
+		struct vc4_exec_info *exec =
+			list_first_entry(&vc4->job_done_list,
+					 struct vc4_exec_info, head);
+		list_del(&exec->head);
+
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+		vc4_complete_exec(vc4->dev, exec);
+		spin_lock_irqsave(&vc4->job_lock, irqflags);
+	}
+
+	list_for_each_entry_safe(cb, cb_temp, &vc4->seqno_cb_list, work.entry) {
+		if (cb->seqno <= vc4->finished_seqno) {
+			list_del_init(&cb->work.entry);
+			schedule_work(&cb->work);
+		}
+	}
+
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+}
+
+static void vc4_seqno_cb_work(struct work_struct *work)
+{
+	struct vc4_seqno_cb *cb = container_of(work, struct vc4_seqno_cb, work);
+
+	cb->func(cb);
+}
+
+int vc4_queue_seqno_cb(struct drm_device *dev,
+		       struct vc4_seqno_cb *cb, uint64_t seqno,
+		       void (*func)(struct vc4_seqno_cb *cb))
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int ret = 0;
+	unsigned long irqflags;
+
+	cb->func = func;
+	INIT_WORK(&cb->work, vc4_seqno_cb_work);
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	if (seqno > vc4->finished_seqno) {
+		cb->seqno = seqno;
+		list_add_tail(&cb->work.entry, &vc4->seqno_cb_list);
+	} else {
+		schedule_work(&cb->work);
+	}
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+
+	return ret;
+}
+
+/* Scheduled when any job has been completed, this walks the list of
+ * jobs that had completed and unrefs their BOs and frees their exec
+ * structs.
+ */
+static void
+vc4_job_done_work(struct work_struct *work)
+{
+	struct vc4_dev *vc4 =
+		container_of(work, struct vc4_dev, job_done_work);
+
+	vc4_job_handle_completed(vc4);
+}
+
+static int
+vc4_wait_for_seqno_ioctl_helper(struct drm_device *dev,
+				uint64_t seqno,
+				uint64_t *timeout_ns)
+{
+	unsigned long start = jiffies;
+	int ret = vc4_wait_for_seqno(dev, seqno, *timeout_ns, true);
+
+	if ((ret == -EINTR || ret == -ERESTARTSYS) && *timeout_ns != ~0ull) {
+		uint64_t delta = jiffies_to_nsecs(jiffies - start);
+
+		if (*timeout_ns >= delta)
+			*timeout_ns -= delta;
+	}
+
+	return ret;
+}
+
+int
+vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
+{
+	struct drm_vc4_wait_seqno *args = data;
+
+	return vc4_wait_for_seqno_ioctl_helper(dev, args->seqno,
+					       &args->timeout_ns);
+}
+
+int
+vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
+{
+	int ret;
+	struct drm_vc4_wait_bo *args = data;
+	struct drm_gem_object *gem_obj;
+	struct vc4_bo *bo;
+
+	gem_obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		return -EINVAL;
+	}
+	bo = to_vc4_bo(gem_obj);
+
+	ret = vc4_wait_for_seqno_ioctl_helper(dev, bo->seqno,
+					      &args->timeout_ns);
+
+	drm_gem_object_unreference_unlocked(gem_obj);
+	return ret;
+}
+
+/**
+ * Submits a command list to the VC4.
+ *
+ * This is what is called batchbuffer emitting on other hardware.
+ */
+int
+vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_vc4_submit_cl *args = data;
+	struct vc4_exec_info *exec;
+	int ret;
+
+	if ((args->flags & ~VC4_SUBMIT_CL_USE_CLEAR_COLOR) != 0) {
+		DRM_ERROR("Unknown flags: 0x%02x\n", args->flags);
+		return -EINVAL;
+	}
+
+	exec = kcalloc(1, sizeof(*exec), GFP_KERNEL);
+	if (!exec) {
+		DRM_ERROR("malloc failure on exec struct\n");
+		return -ENOMEM;
+	}
+
+	exec->args = args;
+	INIT_LIST_HEAD(&exec->unref_list);
+
+	ret = vc4_cl_lookup_bos(dev, file_priv, exec);
+	if (ret)
+		goto fail;
+
+	if (exec->args->bin_cl_size != 0) {
+		ret = vc4_get_bcl(dev, exec);
+		if (ret)
+			goto fail;
+	} else {
+		exec->ct0ca = 0;
+		exec->ct0ea = 0;
+	}
+
+	ret = vc4_get_rcl(dev, exec);
+	if (ret)
+		goto fail;
+
+	/* Clear this out of the struct we'll be putting in the queue,
+	 * since it's part of our stack.
+	 */
+	exec->args = NULL;
+
+	vc4_queue_submit(dev, exec);
+
+	/* Return the seqno for our job. */
+	args->seqno = vc4->emit_seqno;
+
+	return 0;
+
+fail:
+	vc4_complete_exec(vc4->dev, exec);
+
+	return ret;
+}
+
+void
+vc4_gem_init(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	INIT_LIST_HEAD(&vc4->job_list);
+	INIT_LIST_HEAD(&vc4->job_done_list);
+	INIT_LIST_HEAD(&vc4->seqno_cb_list);
+	spin_lock_init(&vc4->job_lock);
+
+	INIT_WORK(&vc4->hangcheck.reset_work, vc4_reset_work);
+	setup_timer(&vc4->hangcheck.timer,
+		    vc4_hangcheck_elapsed,
+		    (unsigned long)dev);
+
+	INIT_WORK(&vc4->job_done_work, vc4_job_done_work);
+}
+
+void
+vc4_gem_destroy(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Waiting for exec to finish would need to be done before
+	 * unregistering V3D.
+	 */
+	WARN_ON(vc4->emit_seqno != vc4->finished_seqno);
+
+	/* V3D should already have disabled its interrupt and cleared
+	 * the overflow allocation registers.  Now free the object.
+	 */
+	if (vc4->overflow_mem) {
+		drm_gem_object_unreference_unlocked(&vc4->overflow_mem->base.base);
+		vc4->overflow_mem = NULL;
+	}
+
+	vc4_bo_cache_destroy(dev);
+
+	if (vc4->hang_state)
+		vc4_free_hang_state(dev, vc4->hang_state);
+}
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index da9a36d6e1d1..c69c0460196b 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -519,7 +519,7 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	WARN_ON_ONCE((HD_READ(VC4_HD_M_CTL) & VC4_HD_M_ENABLE) == 0);
 
 	drm_encoder_init(drm, hdmi->encoder, &vc4_hdmi_encoder_funcs,
-			 DRM_MODE_ENCODER_TMDS);
+			 DRM_MODE_ENCODER_TMDS, NULL);
 	drm_encoder_helper_add(hdmi->encoder, &vc4_hdmi_encoder_helper_funcs);
 
 	hdmi->connector = vc4_hdmi_connector_init(drm, hdmi->encoder);
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
new file mode 100644
index 000000000000..b68060e758db
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -0,0 +1,210 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/** DOC: Interrupt management for the V3D engine.
+ *
+ * We have an interrupt status register (V3D_INTCTL) which reports
+ * interrupts, and where writing 1 bits clears those interrupts.
+ * There are also a pair of interrupt registers
+ * (V3D_INTENA/V3D_INTDIS) where writing a 1 to their bits enables or
+ * disables that specific interrupt, and 0s written are ignored
+ * (reading either one returns the set of enabled interrupts).
+ *
+ * When we take a render frame interrupt, we need to wake the
+ * processes waiting for some frame to be done, and get the next frame
+ * submitted ASAP (so the hardware doesn't sit idle when there's work
+ * to do).
+ *
+ * When we take the binner out of memory interrupt, we need to
+ * allocate some new memory and pass it to the binner so that the
+ * current job can make progress.
+ */
+
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+#define V3D_DRIVER_IRQS (V3D_INT_OUTOMEM | \
+			 V3D_INT_FRDONE)
+
+DECLARE_WAIT_QUEUE_HEAD(render_wait);
+
+static void
+vc4_overflow_mem_work(struct work_struct *work)
+{
+	struct vc4_dev *vc4 =
+		container_of(work, struct vc4_dev, overflow_mem_work);
+	struct drm_device *dev = vc4->dev;
+	struct vc4_bo *bo;
+
+	bo = vc4_bo_create(dev, 256 * 1024, true);
+	if (!bo) {
+		DRM_ERROR("Couldn't allocate binner overflow mem\n");
+		return;
+	}
+
+	/* If there's a job executing currently, then our previous
+	 * overflow allocation is getting used in that job and we need
+	 * to queue it to be released when the job is done.  But if no
+	 * job is executing at all, then we can free the old overflow
+	 * object direcctly.
+	 *
+	 * No lock necessary for this pointer since we're the only
+	 * ones that update the pointer, and our workqueue won't
+	 * reenter.
+	 */
+	if (vc4->overflow_mem) {
+		struct vc4_exec_info *current_exec;
+		unsigned long irqflags;
+
+		spin_lock_irqsave(&vc4->job_lock, irqflags);
+		current_exec = vc4_first_job(vc4);
+		if (current_exec) {
+			vc4->overflow_mem->seqno = vc4->finished_seqno + 1;
+			list_add_tail(&vc4->overflow_mem->unref_head,
+				      &current_exec->unref_list);
+			vc4->overflow_mem = NULL;
+		}
+		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+	}
+
+	if (vc4->overflow_mem)
+		drm_gem_object_unreference_unlocked(&vc4->overflow_mem->base.base);
+	vc4->overflow_mem = bo;
+
+	V3D_WRITE(V3D_BPOA, bo->base.paddr);
+	V3D_WRITE(V3D_BPOS, bo->base.base.size);
+	V3D_WRITE(V3D_INTCTL, V3D_INT_OUTOMEM);
+	V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM);
+}
+
+static void
+vc4_irq_finish_job(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_exec_info *exec = vc4_first_job(vc4);
+
+	if (!exec)
+		return;
+
+	vc4->finished_seqno++;
+	list_move_tail(&exec->head, &vc4->job_done_list);
+	vc4_submit_next_job(dev);
+
+	wake_up_all(&vc4->job_wait_queue);
+	schedule_work(&vc4->job_done_work);
+}
+
+irqreturn_t
+vc4_irq(int irq, void *arg)
+{
+	struct drm_device *dev = arg;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint32_t intctl;
+	irqreturn_t status = IRQ_NONE;
+
+	barrier();
+	intctl = V3D_READ(V3D_INTCTL);
+
+	/* Acknowledge the interrupts we're handling here. The render
+	 * frame done interrupt will be cleared, while OUTOMEM will
+	 * stay high until the underlying cause is cleared.
+	 */
+	V3D_WRITE(V3D_INTCTL, intctl);
+
+	if (intctl & V3D_INT_OUTOMEM) {
+		/* Disable OUTOMEM until the work is done. */
+		V3D_WRITE(V3D_INTDIS, V3D_INT_OUTOMEM);
+		schedule_work(&vc4->overflow_mem_work);
+		status = IRQ_HANDLED;
+	}
+
+	if (intctl & V3D_INT_FRDONE) {
+		spin_lock(&vc4->job_lock);
+		vc4_irq_finish_job(dev);
+		spin_unlock(&vc4->job_lock);
+		status = IRQ_HANDLED;
+	}
+
+	return status;
+}
+
+void
+vc4_irq_preinstall(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	init_waitqueue_head(&vc4->job_wait_queue);
+	INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work);
+
+	/* Clear any pending interrupts someone might have left around
+	 * for us.
+	 */
+	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
+}
+
+int
+vc4_irq_postinstall(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Enable both the render done and out of memory interrupts. */
+	V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS);
+
+	return 0;
+}
+
+void
+vc4_irq_uninstall(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Disable sending interrupts for our driver's IRQs. */
+	V3D_WRITE(V3D_INTDIS, V3D_DRIVER_IRQS);
+
+	/* Clear any pending interrupts we might have left. */
+	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
+
+	cancel_work_sync(&vc4->overflow_mem_work);
+}
+
+/** Reinitializes interrupt registers when a GPU reset is performed. */
+void vc4_irq_reset(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	unsigned long irqflags;
+
+	/* Acknowledge any stale IRQs. */
+	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
+
+	/*
+	 * Turn all our interrupts on.  Binner out of memory is the
+	 * only one we expect to trigger at this point, since we've
+	 * just come from poweron and haven't supplied any overflow
+	 * memory yet.
+	 */
+	V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS);
+
+	spin_lock_irqsave(&vc4->job_lock, irqflags);
+	vc4_irq_finish_job(dev);
+	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
+}
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 2e5597d10cc6..f95f2df5f8d1 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -15,6 +15,7 @@
  */
 
 #include "drm_crtc.h"
+#include "drm_atomic.h"
 #include "drm_atomic_helper.h"
 #include "drm_crtc_helper.h"
 #include "drm_plane_helper.h"
@@ -29,10 +30,152 @@ static void vc4_output_poll_changed(struct drm_device *dev)
 		drm_fbdev_cma_hotplug_event(vc4->fbdev);
 }
 
+struct vc4_commit {
+	struct drm_device *dev;
+	struct drm_atomic_state *state;
+	struct vc4_seqno_cb cb;
+};
+
+static void
+vc4_atomic_complete_commit(struct vc4_commit *c)
+{
+	struct drm_atomic_state *state = c->state;
+	struct drm_device *dev = state->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	drm_atomic_helper_commit_modeset_disables(dev, state);
+
+	drm_atomic_helper_commit_planes(dev, state, false);
+
+	drm_atomic_helper_commit_modeset_enables(dev, state);
+
+	drm_atomic_helper_wait_for_vblanks(dev, state);
+
+	drm_atomic_helper_cleanup_planes(dev, state);
+
+	drm_atomic_state_free(state);
+
+	up(&vc4->async_modeset);
+
+	kfree(c);
+}
+
+static void
+vc4_atomic_complete_commit_seqno_cb(struct vc4_seqno_cb *cb)
+{
+	struct vc4_commit *c = container_of(cb, struct vc4_commit, cb);
+
+	vc4_atomic_complete_commit(c);
+}
+
+static struct vc4_commit *commit_init(struct drm_atomic_state *state)
+{
+	struct vc4_commit *c = kzalloc(sizeof(*c), GFP_KERNEL);
+
+	if (!c)
+		return NULL;
+	c->dev = state->dev;
+	c->state = state;
+
+	return c;
+}
+
+/**
+ * vc4_atomic_commit - commit validated state object
+ * @dev: DRM device
+ * @state: the driver state object
+ * @async: asynchronous commit
+ *
+ * This function commits a with drm_atomic_helper_check() pre-validated state
+ * object. This can still fail when e.g. the framebuffer reservation fails. For
+ * now this doesn't implement asynchronous commits.
+ *
+ * RETURNS
+ * Zero for success or -errno.
+ */
+static int vc4_atomic_commit(struct drm_device *dev,
+			     struct drm_atomic_state *state,
+			     bool async)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int ret;
+	int i;
+	uint64_t wait_seqno = 0;
+	struct vc4_commit *c;
+
+	c = commit_init(state);
+	if (!c)
+		return -ENOMEM;
+
+	/* Make sure that any outstanding modesets have finished. */
+	ret = down_interruptible(&vc4->async_modeset);
+	if (ret) {
+		kfree(c);
+		return ret;
+	}
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (ret) {
+		kfree(c);
+		up(&vc4->async_modeset);
+		return ret;
+	}
+
+	for (i = 0; i < dev->mode_config.num_total_plane; i++) {
+		struct drm_plane *plane = state->planes[i];
+		struct drm_plane_state *new_state = state->plane_states[i];
+
+		if (!plane)
+			continue;
+
+		if ((plane->state->fb != new_state->fb) && new_state->fb) {
+			struct drm_gem_cma_object *cma_bo =
+				drm_fb_cma_get_gem_obj(new_state->fb, 0);
+			struct vc4_bo *bo = to_vc4_bo(&cma_bo->base);
+
+			wait_seqno = max(bo->seqno, wait_seqno);
+		}
+	}
+
+	/*
+	 * This is the point of no return - everything below never fails except
+	 * when the hw goes bonghits. Which means we can commit the new state on
+	 * the software side now.
+	 */
+
+	drm_atomic_helper_swap_state(dev, state);
+
+	/*
+	 * Everything below can be run asynchronously without the need to grab
+	 * any modeset locks at all under one condition: It must be guaranteed
+	 * that the asynchronous work has either been cancelled (if the driver
+	 * supports it, which at least requires that the framebuffers get
+	 * cleaned up with drm_atomic_helper_cleanup_planes()) or completed
+	 * before the new state gets committed on the software side with
+	 * drm_atomic_helper_swap_state().
+	 *
+	 * This scheme allows new atomic state updates to be prepared and
+	 * checked in parallel to the asynchronous completion of the previous
+	 * update. Which is important since compositors need to figure out the
+	 * composition of the next frame right after having submitted the
+	 * current layout.
+	 */
+
+	if (async) {
+		vc4_queue_seqno_cb(dev, &c->cb, wait_seqno,
+				   vc4_atomic_complete_commit_seqno_cb);
+	} else {
+		vc4_wait_for_seqno(dev, wait_seqno, ~0ull, false);
+		vc4_atomic_complete_commit(c);
+	}
+
+	return 0;
+}
+
 static const struct drm_mode_config_funcs vc4_mode_funcs = {
 	.output_poll_changed = vc4_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = drm_atomic_helper_commit,
+	.atomic_commit = vc4_atomic_commit,
 	.fb_create = drm_fb_cma_create,
 };
 
@@ -41,6 +184,8 @@ int vc4_kms_load(struct drm_device *dev)
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	int ret;
 
+	sema_init(&vc4->async_modeset, 1);
+
 	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
 	if (ret < 0) {
 		dev_err(dev->dev, "failed to initialize vblank\n");
@@ -51,6 +196,8 @@ int vc4_kms_load(struct drm_device *dev)
 	dev->mode_config.max_height = 2048;
 	dev->mode_config.funcs = &vc4_mode_funcs;
 	dev->mode_config.preferred_depth = 24;
+	dev->mode_config.async_page_flip = true;
+
 	dev->vblank_disable_allowed = true;
 
 	drm_mode_config_reset(dev);
diff --git a/drivers/gpu/drm/vc4/vc4_packet.h b/drivers/gpu/drm/vc4/vc4_packet.h
new file mode 100644
index 000000000000..0f31cc06500f
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_packet.h
@@ -0,0 +1,399 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef VC4_PACKET_H
+#define VC4_PACKET_H
+
+#include "vc4_regs.h" /* for VC4_MASK, VC4_GET_FIELD, VC4_SET_FIELD */
+
+enum vc4_packet {
+	VC4_PACKET_HALT = 0,
+	VC4_PACKET_NOP = 1,
+
+	VC4_PACKET_FLUSH = 4,
+	VC4_PACKET_FLUSH_ALL = 5,
+	VC4_PACKET_START_TILE_BINNING = 6,
+	VC4_PACKET_INCREMENT_SEMAPHORE = 7,
+	VC4_PACKET_WAIT_ON_SEMAPHORE = 8,
+
+	VC4_PACKET_BRANCH = 16,
+	VC4_PACKET_BRANCH_TO_SUB_LIST = 17,
+
+	VC4_PACKET_STORE_MS_TILE_BUFFER = 24,
+	VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF = 25,
+	VC4_PACKET_STORE_FULL_RES_TILE_BUFFER = 26,
+	VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER = 27,
+	VC4_PACKET_STORE_TILE_BUFFER_GENERAL = 28,
+	VC4_PACKET_LOAD_TILE_BUFFER_GENERAL = 29,
+
+	VC4_PACKET_GL_INDEXED_PRIMITIVE = 32,
+	VC4_PACKET_GL_ARRAY_PRIMITIVE = 33,
+
+	VC4_PACKET_COMPRESSED_PRIMITIVE = 48,
+	VC4_PACKET_CLIPPED_COMPRESSED_PRIMITIVE = 49,
+
+	VC4_PACKET_PRIMITIVE_LIST_FORMAT = 56,
+
+	VC4_PACKET_GL_SHADER_STATE = 64,
+	VC4_PACKET_NV_SHADER_STATE = 65,
+	VC4_PACKET_VG_SHADER_STATE = 66,
+
+	VC4_PACKET_CONFIGURATION_BITS = 96,
+	VC4_PACKET_FLAT_SHADE_FLAGS = 97,
+	VC4_PACKET_POINT_SIZE = 98,
+	VC4_PACKET_LINE_WIDTH = 99,
+	VC4_PACKET_RHT_X_BOUNDARY = 100,
+	VC4_PACKET_DEPTH_OFFSET = 101,
+	VC4_PACKET_CLIP_WINDOW = 102,
+	VC4_PACKET_VIEWPORT_OFFSET = 103,
+	VC4_PACKET_Z_CLIPPING = 104,
+	VC4_PACKET_CLIPPER_XY_SCALING = 105,
+	VC4_PACKET_CLIPPER_Z_SCALING = 106,
+
+	VC4_PACKET_TILE_BINNING_MODE_CONFIG = 112,
+	VC4_PACKET_TILE_RENDERING_MODE_CONFIG = 113,
+	VC4_PACKET_CLEAR_COLORS = 114,
+	VC4_PACKET_TILE_COORDINATES = 115,
+
+	/* Not an actual hardware packet -- this is what we use to put
+	 * references to GEM bos in the command stream, since we need the u32
+	 * int the actual address packet in order to store the offset from the
+	 * start of the BO.
+	 */
+	VC4_PACKET_GEM_HANDLES = 254,
+} __attribute__ ((__packed__));
+
+#define VC4_PACKET_HALT_SIZE						1
+#define VC4_PACKET_NOP_SIZE						1
+#define VC4_PACKET_FLUSH_SIZE						1
+#define VC4_PACKET_FLUSH_ALL_SIZE					1
+#define VC4_PACKET_START_TILE_BINNING_SIZE				1
+#define VC4_PACKET_INCREMENT_SEMAPHORE_SIZE				1
+#define VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE				1
+#define VC4_PACKET_BRANCH_SIZE						5
+#define VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE				5
+#define VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE				1
+#define VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF_SIZE			1
+#define VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE			5
+#define VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE			5
+#define VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE			7
+#define VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE			7
+#define VC4_PACKET_GL_INDEXED_PRIMITIVE_SIZE				14
+#define VC4_PACKET_GL_ARRAY_PRIMITIVE_SIZE				10
+#define VC4_PACKET_COMPRESSED_PRIMITIVE_SIZE				1
+#define VC4_PACKET_CLIPPED_COMPRESSED_PRIMITIVE_SIZE			1
+#define VC4_PACKET_PRIMITIVE_LIST_FORMAT_SIZE				2
+#define VC4_PACKET_GL_SHADER_STATE_SIZE					5
+#define VC4_PACKET_NV_SHADER_STATE_SIZE					5
+#define VC4_PACKET_VG_SHADER_STATE_SIZE					5
+#define VC4_PACKET_CONFIGURATION_BITS_SIZE				4
+#define VC4_PACKET_FLAT_SHADE_FLAGS_SIZE				5
+#define VC4_PACKET_POINT_SIZE_SIZE					5
+#define VC4_PACKET_LINE_WIDTH_SIZE					5
+#define VC4_PACKET_RHT_X_BOUNDARY_SIZE					3
+#define VC4_PACKET_DEPTH_OFFSET_SIZE					5
+#define VC4_PACKET_CLIP_WINDOW_SIZE					9
+#define VC4_PACKET_VIEWPORT_OFFSET_SIZE					5
+#define VC4_PACKET_Z_CLIPPING_SIZE					9
+#define VC4_PACKET_CLIPPER_XY_SCALING_SIZE				9
+#define VC4_PACKET_CLIPPER_Z_SCALING_SIZE				9
+#define VC4_PACKET_TILE_BINNING_MODE_CONFIG_SIZE			16
+#define VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE			11
+#define VC4_PACKET_CLEAR_COLORS_SIZE					14
+#define VC4_PACKET_TILE_COORDINATES_SIZE				3
+#define VC4_PACKET_GEM_HANDLES_SIZE					9
+
+/* Number of multisamples supported. */
+#define VC4_MAX_SAMPLES							4
+/* Size of a full resolution color or Z tile buffer load/store. */
+#define VC4_TILE_BUFFER_SIZE			(64 * 64 * 4)
+
+/** @{
+ * Bits used by packets like VC4_PACKET_STORE_TILE_BUFFER_GENERAL and
+ * VC4_PACKET_TILE_RENDERING_MODE_CONFIG.
+*/
+#define VC4_TILING_FORMAT_LINEAR    0
+#define VC4_TILING_FORMAT_T         1
+#define VC4_TILING_FORMAT_LT        2
+/** @} */
+
+/** @{
+ *
+ * low bits of VC4_PACKET_STORE_FULL_RES_TILE_BUFFER and
+ * VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER.
+ */
+#define VC4_LOADSTORE_FULL_RES_EOF                     BIT(3)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL       BIT(2)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_ZS              BIT(1)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_COLOR           BIT(0)
+
+/** @{
+ *
+ * low bits of VC4_PACKET_STORE_FULL_RES_TILE_BUFFER and
+ * VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER.
+ */
+#define VC4_LOADSTORE_FULL_RES_EOF                     BIT(3)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL       BIT(2)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_ZS              BIT(1)
+#define VC4_LOADSTORE_FULL_RES_DISABLE_COLOR           BIT(0)
+
+/** @{
+ *
+ * byte 2 of VC4_PACKET_STORE_TILE_BUFFER_GENERAL and
+ * VC4_PACKET_LOAD_TILE_BUFFER_GENERAL (low bits of the address)
+ */
+
+#define VC4_LOADSTORE_TILE_BUFFER_EOF                  BIT(3)
+#define VC4_LOADSTORE_TILE_BUFFER_DISABLE_FULL_VG_MASK BIT(2)
+#define VC4_LOADSTORE_TILE_BUFFER_DISABLE_FULL_ZS      BIT(1)
+#define VC4_LOADSTORE_TILE_BUFFER_DISABLE_FULL_COLOR   BIT(0)
+
+/** @} */
+
+/** @{
+ *
+ * byte 0-1 of VC4_PACKET_STORE_TILE_BUFFER_GENERAL and
+ * VC4_PACKET_LOAD_TILE_BUFFER_GENERAL
+ */
+#define VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR BIT(15)
+#define VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR     BIT(14)
+#define VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR  BIT(13)
+#define VC4_STORE_TILE_BUFFER_DISABLE_SWAP         BIT(12)
+
+#define VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK      VC4_MASK(9, 8)
+#define VC4_LOADSTORE_TILE_BUFFER_FORMAT_SHIFT     8
+#define VC4_LOADSTORE_TILE_BUFFER_RGBA8888         0
+#define VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER    1
+#define VC4_LOADSTORE_TILE_BUFFER_BGR565           2
+/** @} */
+
+/** @{
+ *
+ * byte 0 of VC4_PACKET_STORE_TILE_BUFFER_GENERAL and
+ * VC4_PACKET_LOAD_TILE_BUFFER_GENERAL
+ */
+#define VC4_STORE_TILE_BUFFER_MODE_MASK            VC4_MASK(7, 6)
+#define VC4_STORE_TILE_BUFFER_MODE_SHIFT           6
+#define VC4_STORE_TILE_BUFFER_MODE_SAMPLE0         (0 << 6)
+#define VC4_STORE_TILE_BUFFER_MODE_DECIMATE_X4     (1 << 6)
+#define VC4_STORE_TILE_BUFFER_MODE_DECIMATE_X16    (2 << 6)
+
+/** The values of the field are VC4_TILING_FORMAT_* */
+#define VC4_LOADSTORE_TILE_BUFFER_TILING_MASK      VC4_MASK(5, 4)
+#define VC4_LOADSTORE_TILE_BUFFER_TILING_SHIFT     4
+
+#define VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK      VC4_MASK(2, 0)
+#define VC4_LOADSTORE_TILE_BUFFER_BUFFER_SHIFT     0
+#define VC4_LOADSTORE_TILE_BUFFER_NONE             0
+#define VC4_LOADSTORE_TILE_BUFFER_COLOR            1
+#define VC4_LOADSTORE_TILE_BUFFER_ZS               2
+#define VC4_LOADSTORE_TILE_BUFFER_Z                3
+#define VC4_LOADSTORE_TILE_BUFFER_VG_MASK          4
+#define VC4_LOADSTORE_TILE_BUFFER_FULL             5
+/** @} */
+
+#define VC4_INDEX_BUFFER_U8                        (0 << 4)
+#define VC4_INDEX_BUFFER_U16                       (1 << 4)
+
+/* This flag is only present in NV shader state. */
+#define VC4_SHADER_FLAG_SHADED_CLIP_COORDS         BIT(3)
+#define VC4_SHADER_FLAG_ENABLE_CLIPPING            BIT(2)
+#define VC4_SHADER_FLAG_VS_POINT_SIZE              BIT(1)
+#define VC4_SHADER_FLAG_FS_SINGLE_THREAD           BIT(0)
+
+/** @{ byte 2 of config bits. */
+#define VC4_CONFIG_BITS_EARLY_Z_UPDATE             BIT(1)
+#define VC4_CONFIG_BITS_EARLY_Z                    BIT(0)
+/** @} */
+
+/** @{ byte 1 of config bits. */
+#define VC4_CONFIG_BITS_Z_UPDATE                   BIT(7)
+/** same values in this 3-bit field as PIPE_FUNC_* */
+#define VC4_CONFIG_BITS_DEPTH_FUNC_SHIFT           4
+#define VC4_CONFIG_BITS_COVERAGE_READ_LEAVE        BIT(3)
+
+#define VC4_CONFIG_BITS_COVERAGE_UPDATE_NONZERO    (0 << 1)
+#define VC4_CONFIG_BITS_COVERAGE_UPDATE_ODD        (1 << 1)
+#define VC4_CONFIG_BITS_COVERAGE_UPDATE_OR         (2 << 1)
+#define VC4_CONFIG_BITS_COVERAGE_UPDATE_ZERO       (3 << 1)
+
+#define VC4_CONFIG_BITS_COVERAGE_PIPE_SELECT       BIT(0)
+/** @} */
+
+/** @{ byte 0 of config bits. */
+#define VC4_CONFIG_BITS_RASTERIZER_OVERSAMPLE_NONE (0 << 6)
+#define VC4_CONFIG_BITS_RASTERIZER_OVERSAMPLE_4X   (1 << 6)
+#define VC4_CONFIG_BITS_RASTERIZER_OVERSAMPLE_16X  (2 << 6)
+
+#define VC4_CONFIG_BITS_AA_POINTS_AND_LINES        BIT(4)
+#define VC4_CONFIG_BITS_ENABLE_DEPTH_OFFSET        BIT(3)
+#define VC4_CONFIG_BITS_CW_PRIMITIVES              BIT(2)
+#define VC4_CONFIG_BITS_ENABLE_PRIM_BACK           BIT(1)
+#define VC4_CONFIG_BITS_ENABLE_PRIM_FRONT          BIT(0)
+/** @} */
+
+/** @{ bits in the last u8 of VC4_PACKET_TILE_BINNING_MODE_CONFIG */
+#define VC4_BIN_CONFIG_DB_NON_MS                   BIT(7)
+
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_MASK       VC4_MASK(6, 5)
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_SHIFT      5
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_32         0
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_64         1
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_128        2
+#define VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_256        3
+
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_MASK  VC4_MASK(4, 3)
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_SHIFT 3
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_32    0
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_64    1
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_128   2
+#define VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_256   3
+
+#define VC4_BIN_CONFIG_AUTO_INIT_TSDA              BIT(2)
+#define VC4_BIN_CONFIG_TILE_BUFFER_64BIT           BIT(1)
+#define VC4_BIN_CONFIG_MS_MODE_4X                  BIT(0)
+/** @} */
+
+/** @{ bits in the last u16 of VC4_PACKET_TILE_RENDERING_MODE_CONFIG */
+#define VC4_RENDER_CONFIG_DB_NON_MS                BIT(12)
+#define VC4_RENDER_CONFIG_EARLY_Z_COVERAGE_DISABLE BIT(11)
+#define VC4_RENDER_CONFIG_EARLY_Z_DIRECTION_G      BIT(10)
+#define VC4_RENDER_CONFIG_COVERAGE_MODE            BIT(9)
+#define VC4_RENDER_CONFIG_ENABLE_VG_MASK           BIT(8)
+
+/** The values of the field are VC4_TILING_FORMAT_* */
+#define VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK       VC4_MASK(7, 6)
+#define VC4_RENDER_CONFIG_MEMORY_FORMAT_SHIFT      6
+
+#define VC4_RENDER_CONFIG_DECIMATE_MODE_1X         (0 << 4)
+#define VC4_RENDER_CONFIG_DECIMATE_MODE_4X         (1 << 4)
+#define VC4_RENDER_CONFIG_DECIMATE_MODE_16X        (2 << 4)
+
+#define VC4_RENDER_CONFIG_FORMAT_MASK              VC4_MASK(3, 2)
+#define VC4_RENDER_CONFIG_FORMAT_SHIFT             2
+#define VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED   0
+#define VC4_RENDER_CONFIG_FORMAT_RGBA8888          1
+#define VC4_RENDER_CONFIG_FORMAT_BGR565            2
+
+#define VC4_RENDER_CONFIG_TILE_BUFFER_64BIT        BIT(1)
+#define VC4_RENDER_CONFIG_MS_MODE_4X               BIT(0)
+
+#define VC4_PRIMITIVE_LIST_FORMAT_16_INDEX         (1 << 4)
+#define VC4_PRIMITIVE_LIST_FORMAT_32_XY            (3 << 4)
+#define VC4_PRIMITIVE_LIST_FORMAT_TYPE_POINTS      (0 << 0)
+#define VC4_PRIMITIVE_LIST_FORMAT_TYPE_LINES       (1 << 0)
+#define VC4_PRIMITIVE_LIST_FORMAT_TYPE_TRIANGLES   (2 << 0)
+#define VC4_PRIMITIVE_LIST_FORMAT_TYPE_RHT         (3 << 0)
+
+enum vc4_texture_data_type {
+	VC4_TEXTURE_TYPE_RGBA8888 = 0,
+	VC4_TEXTURE_TYPE_RGBX8888 = 1,
+	VC4_TEXTURE_TYPE_RGBA4444 = 2,
+	VC4_TEXTURE_TYPE_RGBA5551 = 3,
+	VC4_TEXTURE_TYPE_RGB565 = 4,
+	VC4_TEXTURE_TYPE_LUMINANCE = 5,
+	VC4_TEXTURE_TYPE_ALPHA = 6,
+	VC4_TEXTURE_TYPE_LUMALPHA = 7,
+	VC4_TEXTURE_TYPE_ETC1 = 8,
+	VC4_TEXTURE_TYPE_S16F = 9,
+	VC4_TEXTURE_TYPE_S8 = 10,
+	VC4_TEXTURE_TYPE_S16 = 11,
+	VC4_TEXTURE_TYPE_BW1 = 12,
+	VC4_TEXTURE_TYPE_A4 = 13,
+	VC4_TEXTURE_TYPE_A1 = 14,
+	VC4_TEXTURE_TYPE_RGBA64 = 15,
+	VC4_TEXTURE_TYPE_RGBA32R = 16,
+	VC4_TEXTURE_TYPE_YUV422R = 17,
+};
+
+#define VC4_TEX_P0_OFFSET_MASK                     VC4_MASK(31, 12)
+#define VC4_TEX_P0_OFFSET_SHIFT                    12
+#define VC4_TEX_P0_CSWIZ_MASK                      VC4_MASK(11, 10)
+#define VC4_TEX_P0_CSWIZ_SHIFT                     10
+#define VC4_TEX_P0_CMMODE_MASK                     VC4_MASK(9, 9)
+#define VC4_TEX_P0_CMMODE_SHIFT                    9
+#define VC4_TEX_P0_FLIPY_MASK                      VC4_MASK(8, 8)
+#define VC4_TEX_P0_FLIPY_SHIFT                     8
+#define VC4_TEX_P0_TYPE_MASK                       VC4_MASK(7, 4)
+#define VC4_TEX_P0_TYPE_SHIFT                      4
+#define VC4_TEX_P0_MIPLVLS_MASK                    VC4_MASK(3, 0)
+#define VC4_TEX_P0_MIPLVLS_SHIFT                   0
+
+#define VC4_TEX_P1_TYPE4_MASK                      VC4_MASK(31, 31)
+#define VC4_TEX_P1_TYPE4_SHIFT                     31
+#define VC4_TEX_P1_HEIGHT_MASK                     VC4_MASK(30, 20)
+#define VC4_TEX_P1_HEIGHT_SHIFT                    20
+#define VC4_TEX_P1_ETCFLIP_MASK                    VC4_MASK(19, 19)
+#define VC4_TEX_P1_ETCFLIP_SHIFT                   19
+#define VC4_TEX_P1_WIDTH_MASK                      VC4_MASK(18, 8)
+#define VC4_TEX_P1_WIDTH_SHIFT                     8
+
+#define VC4_TEX_P1_MAGFILT_MASK                    VC4_MASK(7, 7)
+#define VC4_TEX_P1_MAGFILT_SHIFT                   7
+# define VC4_TEX_P1_MAGFILT_LINEAR                 0
+# define VC4_TEX_P1_MAGFILT_NEAREST                1
+
+#define VC4_TEX_P1_MINFILT_MASK                    VC4_MASK(6, 4)
+#define VC4_TEX_P1_MINFILT_SHIFT                   4
+# define VC4_TEX_P1_MINFILT_LINEAR                 0
+# define VC4_TEX_P1_MINFILT_NEAREST                1
+# define VC4_TEX_P1_MINFILT_NEAR_MIP_NEAR          2
+# define VC4_TEX_P1_MINFILT_NEAR_MIP_LIN           3
+# define VC4_TEX_P1_MINFILT_LIN_MIP_NEAR           4
+# define VC4_TEX_P1_MINFILT_LIN_MIP_LIN            5
+
+#define VC4_TEX_P1_WRAP_T_MASK                     VC4_MASK(3, 2)
+#define VC4_TEX_P1_WRAP_T_SHIFT                    2
+#define VC4_TEX_P1_WRAP_S_MASK                     VC4_MASK(1, 0)
+#define VC4_TEX_P1_WRAP_S_SHIFT                    0
+# define VC4_TEX_P1_WRAP_REPEAT                    0
+# define VC4_TEX_P1_WRAP_CLAMP                     1
+# define VC4_TEX_P1_WRAP_MIRROR                    2
+# define VC4_TEX_P1_WRAP_BORDER                    3
+
+#define VC4_TEX_P2_PTYPE_MASK                      VC4_MASK(31, 30)
+#define VC4_TEX_P2_PTYPE_SHIFT                     30
+# define VC4_TEX_P2_PTYPE_IGNORED                  0
+# define VC4_TEX_P2_PTYPE_CUBE_MAP_STRIDE          1
+# define VC4_TEX_P2_PTYPE_CHILD_IMAGE_DIMENSIONS   2
+# define VC4_TEX_P2_PTYPE_CHILD_IMAGE_OFFSETS      3
+
+/* VC4_TEX_P2_PTYPE_CUBE_MAP_STRIDE bits */
+#define VC4_TEX_P2_CMST_MASK                       VC4_MASK(29, 12)
+#define VC4_TEX_P2_CMST_SHIFT                      12
+#define VC4_TEX_P2_BSLOD_MASK                      VC4_MASK(0, 0)
+#define VC4_TEX_P2_BSLOD_SHIFT                     0
+
+/* VC4_TEX_P2_PTYPE_CHILD_IMAGE_DIMENSIONS */
+#define VC4_TEX_P2_CHEIGHT_MASK                    VC4_MASK(22, 12)
+#define VC4_TEX_P2_CHEIGHT_SHIFT                   12
+#define VC4_TEX_P2_CWIDTH_MASK                     VC4_MASK(10, 0)
+#define VC4_TEX_P2_CWIDTH_SHIFT                    0
+
+/* VC4_TEX_P2_PTYPE_CHILD_IMAGE_OFFSETS */
+#define VC4_TEX_P2_CYOFF_MASK                      VC4_MASK(22, 12)
+#define VC4_TEX_P2_CYOFF_SHIFT                     12
+#define VC4_TEX_P2_CXOFF_MASK                      VC4_MASK(10, 0)
+#define VC4_TEX_P2_CXOFF_SHIFT                     0
+
+#endif /* VC4_PACKET_H */
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 887f3caad0be..0addbad15832 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -29,6 +29,14 @@ struct vc4_plane_state {
 	u32 *dlist;
 	u32 dlist_size; /* Number of dwords in allocated for the display list */
 	u32 dlist_count; /* Number of used dwords in the display list. */
+
+	/* Offset in the dlist to pointer word 0. */
+	u32 pw0_offset;
+
+	/* Offset where the plane's dlist was last stored in the
+	   hardware at vc4_crtc_atomic_flush() time.
+	*/
+	u32 *hw_dlist;
 };
 
 static inline struct vc4_plane_state *
@@ -207,6 +215,8 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 	/* Position Word 3: Context.  Written by the HVS. */
 	vc4_dlist_write(vc4_state, 0xc0c0c0c0);
 
+	vc4_state->pw0_offset = vc4_state->dlist_count;
+
 	/* Pointer Word 0: RGB / Y Pointer */
 	vc4_dlist_write(vc4_state, bo->paddr + offset);
 
@@ -258,6 +268,8 @@ u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist)
 	struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
 	int i;
 
+	vc4_state->hw_dlist = dlist;
+
 	/* Can't memcpy_toio() because it needs to be 32-bit writes. */
 	for (i = 0; i < vc4_state->dlist_count; i++)
 		writel(vc4_state->dlist[i], &dlist[i]);
@@ -272,6 +284,34 @@ u32 vc4_plane_dlist_size(struct drm_plane_state *state)
 	return vc4_state->dlist_count;
 }
 
+/* Updates the plane to immediately (well, once the FIFO needs
+ * refilling) scan out from at a new framebuffer.
+ */
+void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
+{
+	struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
+	struct drm_gem_cma_object *bo = drm_fb_cma_get_gem_obj(fb, 0);
+	uint32_t addr;
+
+	/* We're skipping the address adjustment for negative origin,
+	 * because this is only called on the primary plane.
+	 */
+	WARN_ON_ONCE(plane->state->crtc_x < 0 || plane->state->crtc_y < 0);
+	addr = bo->paddr + fb->offsets[0];
+
+	/* Write the new address into the hardware immediately.  The
+	 * scanout will start from this address as soon as the FIFO
+	 * needs to refill with pixels.
+	 */
+	writel(addr, &vc4_state->hw_dlist[vc4_state->pw0_offset]);
+
+	/* Also update the CPU-side dlist copy, so that any later
+	 * atomic updates that don't do a new modeset on our plane
+	 * also use our updated address.
+	 */
+	vc4_state->dlist[vc4_state->pw0_offset] = addr;
+}
+
 static const struct drm_plane_helper_funcs vc4_plane_helper_funcs = {
 	.prepare_fb = NULL,
 	.cleanup_fb = NULL,
@@ -317,7 +357,7 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 	ret = drm_universal_plane_init(dev, plane, 0xff,
 				       &vc4_plane_funcs,
 				       formats, ARRAY_SIZE(formats),
-				       type);
+				       type, NULL);
 
 	drm_plane_helper_add(plane, &vc4_plane_helper_funcs);
 
diff --git a/drivers/gpu/drm/vc4/vc4_qpu_defines.h b/drivers/gpu/drm/vc4/vc4_qpu_defines.h
new file mode 100644
index 000000000000..d5c2f3c85ebb
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_qpu_defines.h
@@ -0,0 +1,264 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef VC4_QPU_DEFINES_H
+#define VC4_QPU_DEFINES_H
+
+enum qpu_op_add {
+	QPU_A_NOP,
+	QPU_A_FADD,
+	QPU_A_FSUB,
+	QPU_A_FMIN,
+	QPU_A_FMAX,
+	QPU_A_FMINABS,
+	QPU_A_FMAXABS,
+	QPU_A_FTOI,
+	QPU_A_ITOF,
+	QPU_A_ADD = 12,
+	QPU_A_SUB,
+	QPU_A_SHR,
+	QPU_A_ASR,
+	QPU_A_ROR,
+	QPU_A_SHL,
+	QPU_A_MIN,
+	QPU_A_MAX,
+	QPU_A_AND,
+	QPU_A_OR,
+	QPU_A_XOR,
+	QPU_A_NOT,
+	QPU_A_CLZ,
+	QPU_A_V8ADDS = 30,
+	QPU_A_V8SUBS = 31,
+};
+
+enum qpu_op_mul {
+	QPU_M_NOP,
+	QPU_M_FMUL,
+	QPU_M_MUL24,
+	QPU_M_V8MULD,
+	QPU_M_V8MIN,
+	QPU_M_V8MAX,
+	QPU_M_V8ADDS,
+	QPU_M_V8SUBS,
+};
+
+enum qpu_raddr {
+	QPU_R_FRAG_PAYLOAD_ZW = 15, /* W for A file, Z for B file */
+	/* 0-31 are the plain regfile a or b fields */
+	QPU_R_UNIF = 32,
+	QPU_R_VARY = 35,
+	QPU_R_ELEM_QPU = 38,
+	QPU_R_NOP,
+	QPU_R_XY_PIXEL_COORD = 41,
+	QPU_R_MS_REV_FLAGS = 41,
+	QPU_R_VPM = 48,
+	QPU_R_VPM_LD_BUSY,
+	QPU_R_VPM_LD_WAIT,
+	QPU_R_MUTEX_ACQUIRE,
+};
+
+enum qpu_waddr {
+	/* 0-31 are the plain regfile a or b fields */
+	QPU_W_ACC0 = 32, /* aka r0 */
+	QPU_W_ACC1,
+	QPU_W_ACC2,
+	QPU_W_ACC3,
+	QPU_W_TMU_NOSWAP,
+	QPU_W_ACC5,
+	QPU_W_HOST_INT,
+	QPU_W_NOP,
+	QPU_W_UNIFORMS_ADDRESS,
+	QPU_W_QUAD_XY, /* X for regfile a, Y for regfile b */
+	QPU_W_MS_FLAGS = 42,
+	QPU_W_REV_FLAG = 42,
+	QPU_W_TLB_STENCIL_SETUP = 43,
+	QPU_W_TLB_Z,
+	QPU_W_TLB_COLOR_MS,
+	QPU_W_TLB_COLOR_ALL,
+	QPU_W_TLB_ALPHA_MASK,
+	QPU_W_VPM,
+	QPU_W_VPMVCD_SETUP, /* LD for regfile a, ST for regfile b */
+	QPU_W_VPM_ADDR, /* LD for regfile a, ST for regfile b */
+	QPU_W_MUTEX_RELEASE,
+	QPU_W_SFU_RECIP,
+	QPU_W_SFU_RECIPSQRT,
+	QPU_W_SFU_EXP,
+	QPU_W_SFU_LOG,
+	QPU_W_TMU0_S,
+	QPU_W_TMU0_T,
+	QPU_W_TMU0_R,
+	QPU_W_TMU0_B,
+	QPU_W_TMU1_S,
+	QPU_W_TMU1_T,
+	QPU_W_TMU1_R,
+	QPU_W_TMU1_B,
+};
+
+enum qpu_sig_bits {
+	QPU_SIG_SW_BREAKPOINT,
+	QPU_SIG_NONE,
+	QPU_SIG_THREAD_SWITCH,
+	QPU_SIG_PROG_END,
+	QPU_SIG_WAIT_FOR_SCOREBOARD,
+	QPU_SIG_SCOREBOARD_UNLOCK,
+	QPU_SIG_LAST_THREAD_SWITCH,
+	QPU_SIG_COVERAGE_LOAD,
+	QPU_SIG_COLOR_LOAD,
+	QPU_SIG_COLOR_LOAD_END,
+	QPU_SIG_LOAD_TMU0,
+	QPU_SIG_LOAD_TMU1,
+	QPU_SIG_ALPHA_MASK_LOAD,
+	QPU_SIG_SMALL_IMM,
+	QPU_SIG_LOAD_IMM,
+	QPU_SIG_BRANCH
+};
+
+enum qpu_mux {
+	/* hardware mux values */
+	QPU_MUX_R0,
+	QPU_MUX_R1,
+	QPU_MUX_R2,
+	QPU_MUX_R3,
+	QPU_MUX_R4,
+	QPU_MUX_R5,
+	QPU_MUX_A,
+	QPU_MUX_B,
+
+	/* non-hardware mux values */
+	QPU_MUX_IMM,
+};
+
+enum qpu_cond {
+	QPU_COND_NEVER,
+	QPU_COND_ALWAYS,
+	QPU_COND_ZS,
+	QPU_COND_ZC,
+	QPU_COND_NS,
+	QPU_COND_NC,
+	QPU_COND_CS,
+	QPU_COND_CC,
+};
+
+enum qpu_pack_mul {
+	QPU_PACK_MUL_NOP,
+	/* replicated to each 8 bits of the 32-bit dst. */
+	QPU_PACK_MUL_8888 = 3,
+	QPU_PACK_MUL_8A,
+	QPU_PACK_MUL_8B,
+	QPU_PACK_MUL_8C,
+	QPU_PACK_MUL_8D,
+};
+
+enum qpu_pack_a {
+	QPU_PACK_A_NOP,
+	/* convert to 16 bit float if float input, or to int16. */
+	QPU_PACK_A_16A,
+	QPU_PACK_A_16B,
+	/* replicated to each 8 bits of the 32-bit dst. */
+	QPU_PACK_A_8888,
+	/* Convert to 8-bit unsigned int. */
+	QPU_PACK_A_8A,
+	QPU_PACK_A_8B,
+	QPU_PACK_A_8C,
+	QPU_PACK_A_8D,
+
+	/* Saturating variants of the previous instructions. */
+	QPU_PACK_A_32_SAT, /* int-only */
+	QPU_PACK_A_16A_SAT, /* int or float */
+	QPU_PACK_A_16B_SAT,
+	QPU_PACK_A_8888_SAT,
+	QPU_PACK_A_8A_SAT,
+	QPU_PACK_A_8B_SAT,
+	QPU_PACK_A_8C_SAT,
+	QPU_PACK_A_8D_SAT,
+};
+
+enum qpu_unpack_r4 {
+	QPU_UNPACK_R4_NOP,
+	QPU_UNPACK_R4_F16A_TO_F32,
+	QPU_UNPACK_R4_F16B_TO_F32,
+	QPU_UNPACK_R4_8D_REP,
+	QPU_UNPACK_R4_8A,
+	QPU_UNPACK_R4_8B,
+	QPU_UNPACK_R4_8C,
+	QPU_UNPACK_R4_8D,
+};
+
+#define QPU_MASK(high, low) \
+	((((uint64_t)1 << ((high) - (low) + 1)) - 1) << (low))
+
+#define QPU_GET_FIELD(word, field) \
+	((uint32_t)(((word)  & field ## _MASK) >> field ## _SHIFT))
+
+#define QPU_SIG_SHIFT                   60
+#define QPU_SIG_MASK                    QPU_MASK(63, 60)
+
+#define QPU_UNPACK_SHIFT                57
+#define QPU_UNPACK_MASK                 QPU_MASK(59, 57)
+
+/**
+ * If set, the pack field means PACK_MUL or R4 packing, instead of normal
+ * regfile a packing.
+ */
+#define QPU_PM                          ((uint64_t)1 << 56)
+
+#define QPU_PACK_SHIFT                  52
+#define QPU_PACK_MASK                   QPU_MASK(55, 52)
+
+#define QPU_COND_ADD_SHIFT              49
+#define QPU_COND_ADD_MASK               QPU_MASK(51, 49)
+#define QPU_COND_MUL_SHIFT              46
+#define QPU_COND_MUL_MASK               QPU_MASK(48, 46)
+
+#define QPU_SF                          ((uint64_t)1 << 45)
+
+#define QPU_WADDR_ADD_SHIFT             38
+#define QPU_WADDR_ADD_MASK              QPU_MASK(43, 38)
+#define QPU_WADDR_MUL_SHIFT             32
+#define QPU_WADDR_MUL_MASK              QPU_MASK(37, 32)
+
+#define QPU_OP_MUL_SHIFT                29
+#define QPU_OP_MUL_MASK                 QPU_MASK(31, 29)
+
+#define QPU_RADDR_A_SHIFT               18
+#define QPU_RADDR_A_MASK                QPU_MASK(23, 18)
+#define QPU_RADDR_B_SHIFT               12
+#define QPU_RADDR_B_MASK                QPU_MASK(17, 12)
+#define QPU_SMALL_IMM_SHIFT             12
+#define QPU_SMALL_IMM_MASK              QPU_MASK(17, 12)
+
+#define QPU_ADD_A_SHIFT                 9
+#define QPU_ADD_A_MASK                  QPU_MASK(11, 9)
+#define QPU_ADD_B_SHIFT                 6
+#define QPU_ADD_B_MASK                  QPU_MASK(8, 6)
+#define QPU_MUL_A_SHIFT                 3
+#define QPU_MUL_A_MASK                  QPU_MASK(5, 3)
+#define QPU_MUL_B_SHIFT                 0
+#define QPU_MUL_B_MASK                  QPU_MASK(2, 0)
+
+#define QPU_WS                          ((uint64_t)1 << 44)
+
+#define QPU_OP_ADD_SHIFT                24
+#define QPU_OP_ADD_MASK                 QPU_MASK(28, 24)
+
+#endif /* VC4_QPU_DEFINES_H */
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index 9e4e904c668e..4e52a0a88551 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -154,7 +154,7 @@
 #define V3D_PCTRS14  0x006f4
 #define V3D_PCTR15   0x006f8
 #define V3D_PCTRS15  0x006fc
-#define V3D_BGE      0x00f00
+#define V3D_DBGE     0x00f00
 #define V3D_FDBGO    0x00f04
 #define V3D_FDBGB    0x00f08
 #define V3D_FDBGR    0x00f0c
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
new file mode 100644
index 000000000000..8a2a312e2c1b
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -0,0 +1,634 @@
+/*
+ * Copyright © 2014-2015 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * DOC: Render command list generation
+ *
+ * In the VC4 driver, render command list generation is performed by the
+ * kernel instead of userspace.  We do this because validating a
+ * user-submitted command list is hard to get right and has high CPU overhead,
+ * while the number of valid configurations for render command lists is
+ * actually fairly low.
+ */
+
+#include "uapi/drm/vc4_drm.h"
+#include "vc4_drv.h"
+#include "vc4_packet.h"
+
+struct vc4_rcl_setup {
+	struct drm_gem_cma_object *color_read;
+	struct drm_gem_cma_object *color_write;
+	struct drm_gem_cma_object *zs_read;
+	struct drm_gem_cma_object *zs_write;
+	struct drm_gem_cma_object *msaa_color_write;
+	struct drm_gem_cma_object *msaa_zs_write;
+
+	struct drm_gem_cma_object *rcl;
+	u32 next_offset;
+};
+
+static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
+{
+	*(u8 *)(setup->rcl->vaddr + setup->next_offset) = val;
+	setup->next_offset += 1;
+}
+
+static inline void rcl_u16(struct vc4_rcl_setup *setup, u16 val)
+{
+	*(u16 *)(setup->rcl->vaddr + setup->next_offset) = val;
+	setup->next_offset += 2;
+}
+
+static inline void rcl_u32(struct vc4_rcl_setup *setup, u32 val)
+{
+	*(u32 *)(setup->rcl->vaddr + setup->next_offset) = val;
+	setup->next_offset += 4;
+}
+
+/*
+ * Emits a no-op STORE_TILE_BUFFER_GENERAL.
+ *
+ * If we emit a PACKET_TILE_COORDINATES, it must be followed by a store of
+ * some sort before another load is triggered.
+ */
+static void vc4_store_before_load(struct vc4_rcl_setup *setup)
+{
+	rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
+	rcl_u16(setup,
+		VC4_SET_FIELD(VC4_LOADSTORE_TILE_BUFFER_NONE,
+			      VC4_LOADSTORE_TILE_BUFFER_BUFFER) |
+		VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR |
+		VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR |
+		VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR);
+	rcl_u32(setup, 0); /* no address, since we're in None mode */
+}
+
+/*
+ * Calculates the physical address of the start of a tile in a RCL surface.
+ *
+ * Unlike the other load/store packets,
+ * VC4_PACKET_LOAD/STORE_FULL_RES_TILE_BUFFER don't look at the tile
+ * coordinates packet, and instead just store to the address given.
+ */
+static uint32_t vc4_full_res_offset(struct vc4_exec_info *exec,
+				    struct drm_gem_cma_object *bo,
+				    struct drm_vc4_submit_rcl_surface *surf,
+				    uint8_t x, uint8_t y)
+{
+	return bo->paddr + surf->offset + VC4_TILE_BUFFER_SIZE *
+		(DIV_ROUND_UP(exec->args->width, 32) * y + x);
+}
+
+/*
+ * Emits a PACKET_TILE_COORDINATES if one isn't already pending.
+ *
+ * The tile coordinates packet triggers a pending load if there is one, are
+ * used for clipping during rendering, and determine where loads/stores happen
+ * relative to their base address.
+ */
+static void vc4_tile_coordinates(struct vc4_rcl_setup *setup,
+				 uint32_t x, uint32_t y)
+{
+	rcl_u8(setup, VC4_PACKET_TILE_COORDINATES);
+	rcl_u8(setup, x);
+	rcl_u8(setup, y);
+}
+
+static void emit_tile(struct vc4_exec_info *exec,
+		      struct vc4_rcl_setup *setup,
+		      uint8_t x, uint8_t y, bool first, bool last)
+{
+	struct drm_vc4_submit_cl *args = exec->args;
+	bool has_bin = args->bin_cl_size != 0;
+
+	/* Note that the load doesn't actually occur until the
+	 * tile coords packet is processed, and only one load
+	 * may be outstanding at a time.
+	 */
+	if (setup->color_read) {
+		if (args->color_read.flags &
+		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+			rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
+			rcl_u32(setup,
+				vc4_full_res_offset(exec, setup->color_read,
+						    &args->color_read, x, y) |
+				VC4_LOADSTORE_FULL_RES_DISABLE_ZS);
+		} else {
+			rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
+			rcl_u16(setup, args->color_read.bits);
+			rcl_u32(setup, setup->color_read->paddr +
+				args->color_read.offset);
+		}
+	}
+
+	if (setup->zs_read) {
+		if (args->zs_read.flags &
+		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+			rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
+			rcl_u32(setup,
+				vc4_full_res_offset(exec, setup->zs_read,
+						    &args->zs_read, x, y) |
+				VC4_LOADSTORE_FULL_RES_DISABLE_COLOR);
+		} else {
+			if (setup->color_read) {
+				/* Exec previous load. */
+				vc4_tile_coordinates(setup, x, y);
+				vc4_store_before_load(setup);
+			}
+
+			rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
+			rcl_u16(setup, args->zs_read.bits);
+			rcl_u32(setup, setup->zs_read->paddr +
+				args->zs_read.offset);
+		}
+	}
+
+	/* Clipping depends on tile coordinates having been
+	 * emitted, so we always need one here.
+	 */
+	vc4_tile_coordinates(setup, x, y);
+
+	/* Wait for the binner before jumping to the first
+	 * tile's lists.
+	 */
+	if (first && has_bin)
+		rcl_u8(setup, VC4_PACKET_WAIT_ON_SEMAPHORE);
+
+	if (has_bin) {
+		rcl_u8(setup, VC4_PACKET_BRANCH_TO_SUB_LIST);
+		rcl_u32(setup, (exec->tile_bo->paddr +
+				exec->tile_alloc_offset +
+				(y * exec->bin_tiles_x + x) * 32));
+	}
+
+	if (setup->msaa_color_write) {
+		bool last_tile_write = (!setup->msaa_zs_write &&
+					!setup->zs_write &&
+					!setup->color_write);
+		uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_ZS;
+
+		if (!last_tile_write)
+			bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
+		else if (last)
+			bits |= VC4_LOADSTORE_FULL_RES_EOF;
+		rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
+		rcl_u32(setup,
+			vc4_full_res_offset(exec, setup->msaa_color_write,
+					    &args->msaa_color_write, x, y) |
+			bits);
+	}
+
+	if (setup->msaa_zs_write) {
+		bool last_tile_write = (!setup->zs_write &&
+					!setup->color_write);
+		uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_COLOR;
+
+		if (setup->msaa_color_write)
+			vc4_tile_coordinates(setup, x, y);
+		if (!last_tile_write)
+			bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
+		else if (last)
+			bits |= VC4_LOADSTORE_FULL_RES_EOF;
+		rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
+		rcl_u32(setup,
+			vc4_full_res_offset(exec, setup->msaa_zs_write,
+					    &args->msaa_zs_write, x, y) |
+			bits);
+	}
+
+	if (setup->zs_write) {
+		bool last_tile_write = !setup->color_write;
+
+		if (setup->msaa_color_write || setup->msaa_zs_write)
+			vc4_tile_coordinates(setup, x, y);
+
+		rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
+		rcl_u16(setup, args->zs_write.bits |
+			(last_tile_write ?
+			 0 : VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR));
+		rcl_u32(setup,
+			(setup->zs_write->paddr + args->zs_write.offset) |
+			((last && last_tile_write) ?
+			 VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
+	}
+
+	if (setup->color_write) {
+		if (setup->msaa_color_write || setup->msaa_zs_write ||
+		    setup->zs_write) {
+			vc4_tile_coordinates(setup, x, y);
+		}
+
+		if (last)
+			rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF);
+		else
+			rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER);
+	}
+}
+
+static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
+			     struct vc4_rcl_setup *setup)
+{
+	struct drm_vc4_submit_cl *args = exec->args;
+	bool has_bin = args->bin_cl_size != 0;
+	uint8_t min_x_tile = args->min_x_tile;
+	uint8_t min_y_tile = args->min_y_tile;
+	uint8_t max_x_tile = args->max_x_tile;
+	uint8_t max_y_tile = args->max_y_tile;
+	uint8_t xtiles = max_x_tile - min_x_tile + 1;
+	uint8_t ytiles = max_y_tile - min_y_tile + 1;
+	uint8_t x, y;
+	uint32_t size, loop_body_size;
+
+	size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
+	loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
+
+	if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
+		size += VC4_PACKET_CLEAR_COLORS_SIZE +
+			VC4_PACKET_TILE_COORDINATES_SIZE +
+			VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
+	}
+
+	if (setup->color_read) {
+		if (args->color_read.flags &
+		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+			loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
+		} else {
+			loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
+		}
+	}
+	if (setup->zs_read) {
+		if (args->zs_read.flags &
+		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+			loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
+		} else {
+			if (setup->color_read &&
+			    !(args->color_read.flags &
+			      VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) {
+				loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
+				loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
+			}
+			loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
+		}
+	}
+
+	if (has_bin) {
+		size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
+		loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
+	}
+
+	if (setup->msaa_color_write)
+		loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
+	if (setup->msaa_zs_write)
+		loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
+
+	if (setup->zs_write)
+		loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
+	if (setup->color_write)
+		loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
+
+	/* We need a VC4_PACKET_TILE_COORDINATES in between each store. */
+	loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE *
+		((setup->msaa_color_write != NULL) +
+		 (setup->msaa_zs_write != NULL) +
+		 (setup->color_write != NULL) +
+		 (setup->zs_write != NULL) - 1);
+
+	size += xtiles * ytiles * loop_body_size;
+
+	setup->rcl = &vc4_bo_create(dev, size, true)->base;
+	if (!setup->rcl)
+		return -ENOMEM;
+	list_add_tail(&to_vc4_bo(&setup->rcl->base)->unref_head,
+		      &exec->unref_list);
+
+	rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
+	rcl_u32(setup,
+		(setup->color_write ? (setup->color_write->paddr +
+				       args->color_write.offset) :
+		 0));
+	rcl_u16(setup, args->width);
+	rcl_u16(setup, args->height);
+	rcl_u16(setup, args->color_write.bits);
+
+	/* The tile buffer gets cleared when the previous tile is stored.  If
+	 * the clear values changed between frames, then the tile buffer has
+	 * stale clear values in it, so we have to do a store in None mode (no
+	 * writes) so that we trigger the tile buffer clear.
+	 */
+	if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
+		rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
+		rcl_u32(setup, args->clear_color[0]);
+		rcl_u32(setup, args->clear_color[1]);
+		rcl_u32(setup, args->clear_z);
+		rcl_u8(setup, args->clear_s);
+
+		vc4_tile_coordinates(setup, 0, 0);
+
+		rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
+		rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
+		rcl_u32(setup, 0); /* no address, since we're in None mode */
+	}
+
+	for (y = min_y_tile; y <= max_y_tile; y++) {
+		for (x = min_x_tile; x <= max_x_tile; x++) {
+			bool first = (x == min_x_tile && y == min_y_tile);
+			bool last = (x == max_x_tile && y == max_y_tile);
+
+			emit_tile(exec, setup, x, y, first, last);
+		}
+	}
+
+	BUG_ON(setup->next_offset != size);
+	exec->ct1ca = setup->rcl->paddr;
+	exec->ct1ea = setup->rcl->paddr + setup->next_offset;
+
+	return 0;
+}
+
+static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
+				     struct drm_gem_cma_object *obj,
+				     struct drm_vc4_submit_rcl_surface *surf)
+{
+	struct drm_vc4_submit_cl *args = exec->args;
+	u32 render_tiles_stride = DIV_ROUND_UP(exec->args->width, 32);
+
+	if (surf->offset > obj->base.size) {
+		DRM_ERROR("surface offset %d > BO size %zd\n",
+			  surf->offset, obj->base.size);
+		return -EINVAL;
+	}
+
+	if ((obj->base.size - surf->offset) / VC4_TILE_BUFFER_SIZE <
+	    render_tiles_stride * args->max_y_tile + args->max_x_tile) {
+		DRM_ERROR("MSAA tile %d, %d out of bounds "
+			  "(bo size %zd, offset %d).\n",
+			  args->max_x_tile, args->max_y_tile,
+			  obj->base.size,
+			  surf->offset);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
+				      struct drm_gem_cma_object **obj,
+				      struct drm_vc4_submit_rcl_surface *surf)
+{
+	if (surf->flags != 0 || surf->bits != 0) {
+		DRM_ERROR("MSAA surface had nonzero flags/bits\n");
+		return -EINVAL;
+	}
+
+	if (surf->hindex == ~0)
+		return 0;
+
+	*obj = vc4_use_bo(exec, surf->hindex);
+	if (!*obj)
+		return -EINVAL;
+
+	if (surf->offset & 0xf) {
+		DRM_ERROR("MSAA write must be 16b aligned.\n");
+		return -EINVAL;
+	}
+
+	return vc4_full_res_bounds_check(exec, *obj, surf);
+}
+
+static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
+				 struct drm_gem_cma_object **obj,
+				 struct drm_vc4_submit_rcl_surface *surf)
+{
+	uint8_t tiling = VC4_GET_FIELD(surf->bits,
+				       VC4_LOADSTORE_TILE_BUFFER_TILING);
+	uint8_t buffer = VC4_GET_FIELD(surf->bits,
+				       VC4_LOADSTORE_TILE_BUFFER_BUFFER);
+	uint8_t format = VC4_GET_FIELD(surf->bits,
+				       VC4_LOADSTORE_TILE_BUFFER_FORMAT);
+	int cpp;
+	int ret;
+
+	if (surf->flags & ~VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+		DRM_ERROR("Extra flags set\n");
+		return -EINVAL;
+	}
+
+	if (surf->hindex == ~0)
+		return 0;
+
+	*obj = vc4_use_bo(exec, surf->hindex);
+	if (!*obj)
+		return -EINVAL;
+
+	if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
+		if (surf == &exec->args->zs_write) {
+			DRM_ERROR("general zs write may not be a full-res.\n");
+			return -EINVAL;
+		}
+
+		if (surf->bits != 0) {
+			DRM_ERROR("load/store general bits set with "
+				  "full res load/store.\n");
+			return -EINVAL;
+		}
+
+		ret = vc4_full_res_bounds_check(exec, *obj, surf);
+		if (!ret)
+			return ret;
+
+		return 0;
+	}
+
+	if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
+			   VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
+			   VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
+		DRM_ERROR("Unknown bits in load/store: 0x%04x\n",
+			  surf->bits);
+		return -EINVAL;
+	}
+
+	if (tiling > VC4_TILING_FORMAT_LT) {
+		DRM_ERROR("Bad tiling format\n");
+		return -EINVAL;
+	}
+
+	if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
+		if (format != 0) {
+			DRM_ERROR("No color format should be set for ZS\n");
+			return -EINVAL;
+		}
+		cpp = 4;
+	} else if (buffer == VC4_LOADSTORE_TILE_BUFFER_COLOR) {
+		switch (format) {
+		case VC4_LOADSTORE_TILE_BUFFER_BGR565:
+		case VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER:
+			cpp = 2;
+			break;
+		case VC4_LOADSTORE_TILE_BUFFER_RGBA8888:
+			cpp = 4;
+			break;
+		default:
+			DRM_ERROR("Bad tile buffer format\n");
+			return -EINVAL;
+		}
+	} else {
+		DRM_ERROR("Bad load/store buffer %d.\n", buffer);
+		return -EINVAL;
+	}
+
+	if (surf->offset & 0xf) {
+		DRM_ERROR("load/store buffer must be 16b aligned.\n");
+		return -EINVAL;
+	}
+
+	if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
+				exec->args->width, exec->args->height, cpp)) {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
+				    struct vc4_rcl_setup *setup,
+				    struct drm_gem_cma_object **obj,
+				    struct drm_vc4_submit_rcl_surface *surf)
+{
+	uint8_t tiling = VC4_GET_FIELD(surf->bits,
+				       VC4_RENDER_CONFIG_MEMORY_FORMAT);
+	uint8_t format = VC4_GET_FIELD(surf->bits,
+				       VC4_RENDER_CONFIG_FORMAT);
+	int cpp;
+
+	if (surf->flags != 0) {
+		DRM_ERROR("No flags supported on render config.\n");
+		return -EINVAL;
+	}
+
+	if (surf->bits & ~(VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK |
+			   VC4_RENDER_CONFIG_FORMAT_MASK |
+			   VC4_RENDER_CONFIG_MS_MODE_4X |
+			   VC4_RENDER_CONFIG_DECIMATE_MODE_4X)) {
+		DRM_ERROR("Unknown bits in render config: 0x%04x\n",
+			  surf->bits);
+		return -EINVAL;
+	}
+
+	if (surf->hindex == ~0)
+		return 0;
+
+	*obj = vc4_use_bo(exec, surf->hindex);
+	if (!*obj)
+		return -EINVAL;
+
+	if (tiling > VC4_TILING_FORMAT_LT) {
+		DRM_ERROR("Bad tiling format\n");
+		return -EINVAL;
+	}
+
+	switch (format) {
+	case VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED:
+	case VC4_RENDER_CONFIG_FORMAT_BGR565:
+		cpp = 2;
+		break;
+	case VC4_RENDER_CONFIG_FORMAT_RGBA8888:
+		cpp = 4;
+		break;
+	default:
+		DRM_ERROR("Bad tile buffer format\n");
+		return -EINVAL;
+	}
+
+	if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
+				exec->args->width, exec->args->height, cpp)) {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
+{
+	struct vc4_rcl_setup setup = {0};
+	struct drm_vc4_submit_cl *args = exec->args;
+	bool has_bin = args->bin_cl_size != 0;
+	int ret;
+
+	if (args->min_x_tile > args->max_x_tile ||
+	    args->min_y_tile > args->max_y_tile) {
+		DRM_ERROR("Bad render tile set (%d,%d)-(%d,%d)\n",
+			  args->min_x_tile, args->min_y_tile,
+			  args->max_x_tile, args->max_y_tile);
+		return -EINVAL;
+	}
+
+	if (has_bin &&
+	    (args->max_x_tile > exec->bin_tiles_x ||
+	     args->max_y_tile > exec->bin_tiles_y)) {
+		DRM_ERROR("Render tiles (%d,%d) outside of bin config "
+			  "(%d,%d)\n",
+			  args->max_x_tile, args->max_y_tile,
+			  exec->bin_tiles_x, exec->bin_tiles_y);
+		return -EINVAL;
+	}
+
+	ret = vc4_rcl_render_config_surface_setup(exec, &setup,
+						  &setup.color_write,
+						  &args->color_write);
+	if (ret)
+		return ret;
+
+	ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read);
+	if (ret)
+		return ret;
+
+	ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read);
+	if (ret)
+		return ret;
+
+	ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write);
+	if (ret)
+		return ret;
+
+	ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_color_write,
+					 &args->msaa_color_write);
+	if (ret)
+		return ret;
+
+	ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_zs_write,
+					 &args->msaa_zs_write);
+	if (ret)
+		return ret;
+
+	/* We shouldn't even have the job submitted to us if there's no
+	 * surface to write out.
+	 */
+	if (!setup.color_write && !setup.zs_write &&
+	    !setup.msaa_color_write && !setup.msaa_zs_write) {
+		DRM_ERROR("RCL requires color or Z/S write\n");
+		return -EINVAL;
+	}
+
+	return vc4_create_rcl_bo(dev, exec, &setup);
+}
diff --git a/drivers/gpu/drm/vc4/vc4_trace.h b/drivers/gpu/drm/vc4/vc4_trace.h
new file mode 100644
index 000000000000..ad7b1ea720c2
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_trace.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#if !defined(_VC4_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _VC4_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM vc4
+#define TRACE_INCLUDE_FILE vc4_trace
+
+TRACE_EVENT(vc4_wait_for_seqno_begin,
+	    TP_PROTO(struct drm_device *dev, uint64_t seqno, uint64_t timeout),
+	    TP_ARGS(dev, seqno, timeout),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     __field(u64, seqno)
+			     __field(u64, timeout)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->dev = dev->primary->index;
+			   __entry->seqno = seqno;
+			   __entry->timeout = timeout;
+			   ),
+
+	    TP_printk("dev=%u, seqno=%llu, timeout=%llu",
+		      __entry->dev, __entry->seqno, __entry->timeout)
+);
+
+TRACE_EVENT(vc4_wait_for_seqno_end,
+	    TP_PROTO(struct drm_device *dev, uint64_t seqno),
+	    TP_ARGS(dev, seqno),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     __field(u64, seqno)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->dev = dev->primary->index;
+			   __entry->seqno = seqno;
+			   ),
+
+	    TP_printk("dev=%u, seqno=%llu",
+		      __entry->dev, __entry->seqno)
+);
+
+#endif /* _VC4_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/vc4/vc4_trace_points.c b/drivers/gpu/drm/vc4/vc4_trace_points.c
new file mode 100644
index 000000000000..e6278f25716b
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_trace_points.c
@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2015 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "vc4_drv.h"
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "vc4_trace.h"
+#endif
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
new file mode 100644
index 000000000000..314ff71db978
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2014 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "linux/component.h"
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+#ifdef CONFIG_DEBUG_FS
+#define REGDEF(reg) { reg, #reg }
+static const struct {
+	uint32_t reg;
+	const char *name;
+} vc4_reg_defs[] = {
+	REGDEF(V3D_IDENT0),
+	REGDEF(V3D_IDENT1),
+	REGDEF(V3D_IDENT2),
+	REGDEF(V3D_SCRATCH),
+	REGDEF(V3D_L2CACTL),
+	REGDEF(V3D_SLCACTL),
+	REGDEF(V3D_INTCTL),
+	REGDEF(V3D_INTENA),
+	REGDEF(V3D_INTDIS),
+	REGDEF(V3D_CT0CS),
+	REGDEF(V3D_CT1CS),
+	REGDEF(V3D_CT0EA),
+	REGDEF(V3D_CT1EA),
+	REGDEF(V3D_CT0CA),
+	REGDEF(V3D_CT1CA),
+	REGDEF(V3D_CT00RA0),
+	REGDEF(V3D_CT01RA0),
+	REGDEF(V3D_CT0LC),
+	REGDEF(V3D_CT1LC),
+	REGDEF(V3D_CT0PC),
+	REGDEF(V3D_CT1PC),
+	REGDEF(V3D_PCS),
+	REGDEF(V3D_BFC),
+	REGDEF(V3D_RFC),
+	REGDEF(V3D_BPCA),
+	REGDEF(V3D_BPCS),
+	REGDEF(V3D_BPOA),
+	REGDEF(V3D_BPOS),
+	REGDEF(V3D_BXCF),
+	REGDEF(V3D_SQRSV0),
+	REGDEF(V3D_SQRSV1),
+	REGDEF(V3D_SQCNTL),
+	REGDEF(V3D_SRQPC),
+	REGDEF(V3D_SRQUA),
+	REGDEF(V3D_SRQUL),
+	REGDEF(V3D_SRQCS),
+	REGDEF(V3D_VPACNTL),
+	REGDEF(V3D_VPMBASE),
+	REGDEF(V3D_PCTRC),
+	REGDEF(V3D_PCTRE),
+	REGDEF(V3D_PCTR0),
+	REGDEF(V3D_PCTRS0),
+	REGDEF(V3D_PCTR1),
+	REGDEF(V3D_PCTRS1),
+	REGDEF(V3D_PCTR2),
+	REGDEF(V3D_PCTRS2),
+	REGDEF(V3D_PCTR3),
+	REGDEF(V3D_PCTRS3),
+	REGDEF(V3D_PCTR4),
+	REGDEF(V3D_PCTRS4),
+	REGDEF(V3D_PCTR5),
+	REGDEF(V3D_PCTRS5),
+	REGDEF(V3D_PCTR6),
+	REGDEF(V3D_PCTRS6),
+	REGDEF(V3D_PCTR7),
+	REGDEF(V3D_PCTRS7),
+	REGDEF(V3D_PCTR8),
+	REGDEF(V3D_PCTRS8),
+	REGDEF(V3D_PCTR9),
+	REGDEF(V3D_PCTRS9),
+	REGDEF(V3D_PCTR10),
+	REGDEF(V3D_PCTRS10),
+	REGDEF(V3D_PCTR11),
+	REGDEF(V3D_PCTRS11),
+	REGDEF(V3D_PCTR12),
+	REGDEF(V3D_PCTRS12),
+	REGDEF(V3D_PCTR13),
+	REGDEF(V3D_PCTRS13),
+	REGDEF(V3D_PCTR14),
+	REGDEF(V3D_PCTRS14),
+	REGDEF(V3D_PCTR15),
+	REGDEF(V3D_PCTRS15),
+	REGDEF(V3D_DBGE),
+	REGDEF(V3D_FDBGO),
+	REGDEF(V3D_FDBGB),
+	REGDEF(V3D_FDBGR),
+	REGDEF(V3D_FDBGS),
+	REGDEF(V3D_ERRSTAT),
+};
+
+int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vc4_reg_defs); i++) {
+		seq_printf(m, "%s (0x%04x): 0x%08x\n",
+			   vc4_reg_defs[i].name, vc4_reg_defs[i].reg,
+			   V3D_READ(vc4_reg_defs[i].reg));
+	}
+
+	return 0;
+}
+
+int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	uint32_t ident1 = V3D_READ(V3D_IDENT1);
+	uint32_t nslc = VC4_GET_FIELD(ident1, V3D_IDENT1_NSLC);
+	uint32_t tups = VC4_GET_FIELD(ident1, V3D_IDENT1_TUPS);
+	uint32_t qups = VC4_GET_FIELD(ident1, V3D_IDENT1_QUPS);
+
+	seq_printf(m, "Revision:   %d\n",
+		   VC4_GET_FIELD(ident1, V3D_IDENT1_REV));
+	seq_printf(m, "Slices:     %d\n", nslc);
+	seq_printf(m, "TMUs:       %d\n", nslc * tups);
+	seq_printf(m, "QPUs:       %d\n", nslc * qups);
+	seq_printf(m, "Semaphores: %d\n",
+		   VC4_GET_FIELD(ident1, V3D_IDENT1_NSEM));
+
+	return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
+int
+vc4_v3d_set_power(struct vc4_dev *vc4, bool on)
+{
+	/* XXX: This interface is needed for GPU reset, and the way to
+	 * do it is to turn our power domain off and back on.  We
+	 * can't just reset from within the driver, because the reset
+	 * bits are in the power domain's register area, and get set
+	 * during the poweron process.
+	 */
+	return 0;
+}
+
+static void vc4_v3d_init_hw(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	/* Take all the memory that would have been reserved for user
+	 * QPU programs, since we don't have an interface for running
+	 * them, anyway.
+	 */
+	V3D_WRITE(V3D_VPMBASE, 0);
+}
+
+static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = dev_get_drvdata(master);
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	struct vc4_v3d *v3d = NULL;
+	int ret;
+
+	v3d = devm_kzalloc(&pdev->dev, sizeof(*v3d), GFP_KERNEL);
+	if (!v3d)
+		return -ENOMEM;
+
+	v3d->pdev = pdev;
+
+	v3d->regs = vc4_ioremap_regs(pdev, 0);
+	if (IS_ERR(v3d->regs))
+		return PTR_ERR(v3d->regs);
+
+	vc4->v3d = v3d;
+
+	if (V3D_READ(V3D_IDENT0) != V3D_EXPECTED_IDENT0) {
+		DRM_ERROR("V3D_IDENT0 read 0x%08x instead of 0x%08x\n",
+			  V3D_READ(V3D_IDENT0), V3D_EXPECTED_IDENT0);
+		return -EINVAL;
+	}
+
+	/* Reset the binner overflow address/size at setup, to be sure
+	 * we don't reuse an old one.
+	 */
+	V3D_WRITE(V3D_BPOA, 0);
+	V3D_WRITE(V3D_BPOS, 0);
+
+	vc4_v3d_init_hw(drm);
+
+	ret = drm_irq_install(drm, platform_get_irq(pdev, 0));
+	if (ret) {
+		DRM_ERROR("Failed to install IRQ handler\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void vc4_v3d_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct drm_device *drm = dev_get_drvdata(master);
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+
+	drm_irq_uninstall(drm);
+
+	/* Disable the binner's overflow memory address, so the next
+	 * driver probe (if any) doesn't try to reuse our old
+	 * allocation.
+	 */
+	V3D_WRITE(V3D_BPOA, 0);
+	V3D_WRITE(V3D_BPOS, 0);
+
+	vc4->v3d = NULL;
+}
+
+static const struct component_ops vc4_v3d_ops = {
+	.bind   = vc4_v3d_bind,
+	.unbind = vc4_v3d_unbind,
+};
+
+static int vc4_v3d_dev_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vc4_v3d_ops);
+}
+
+static int vc4_v3d_dev_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vc4_v3d_ops);
+	return 0;
+}
+
+static const struct of_device_id vc4_v3d_dt_match[] = {
+	{ .compatible = "brcm,vc4-v3d" },
+	{}
+};
+
+struct platform_driver vc4_v3d_driver = {
+	.probe = vc4_v3d_dev_probe,
+	.remove = vc4_v3d_dev_remove,
+	.driver = {
+		.name = "vc4_v3d",
+		.of_match_table = vc4_v3d_dt_match,
+	},
+};
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
new file mode 100644
index 000000000000..e26d9f6face3
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -0,0 +1,900 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * Command list validator for VC4.
+ *
+ * The VC4 has no IOMMU between it and system memory.  So, a user with
+ * access to execute command lists could escalate privilege by
+ * overwriting system memory (drawing to it as a framebuffer) or
+ * reading system memory it shouldn't (reading it as a texture, or
+ * uniform data, or vertex data).
+ *
+ * This validates command lists to ensure that all accesses are within
+ * the bounds of the GEM objects referenced.  It explicitly whitelists
+ * packets, and looks at the offsets in any address fields to make
+ * sure they're constrained within the BOs they reference.
+ *
+ * Note that because of the validation that's happening anyway, this
+ * is where GEM relocation processing happens.
+ */
+
+#include "uapi/drm/vc4_drm.h"
+#include "vc4_drv.h"
+#include "vc4_packet.h"
+
+#define VALIDATE_ARGS \
+	struct vc4_exec_info *exec,			\
+	void *validated,				\
+	void *untrusted
+
+/** Return the width in pixels of a 64-byte microtile. */
+static uint32_t
+utile_width(int cpp)
+{
+	switch (cpp) {
+	case 1:
+	case 2:
+		return 8;
+	case 4:
+		return 4;
+	case 8:
+		return 2;
+	default:
+		DRM_ERROR("unknown cpp: %d\n", cpp);
+		return 1;
+	}
+}
+
+/** Return the height in pixels of a 64-byte microtile. */
+static uint32_t
+utile_height(int cpp)
+{
+	switch (cpp) {
+	case 1:
+		return 8;
+	case 2:
+	case 4:
+	case 8:
+		return 4;
+	default:
+		DRM_ERROR("unknown cpp: %d\n", cpp);
+		return 1;
+	}
+}
+
+/**
+ * The texture unit decides what tiling format a particular miplevel is using
+ * this function, so we lay out our miptrees accordingly.
+ */
+static bool
+size_is_lt(uint32_t width, uint32_t height, int cpp)
+{
+	return (width <= 4 * utile_width(cpp) ||
+		height <= 4 * utile_height(cpp));
+}
+
+struct drm_gem_cma_object *
+vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
+{
+	struct drm_gem_cma_object *obj;
+	struct vc4_bo *bo;
+
+	if (hindex >= exec->bo_count) {
+		DRM_ERROR("BO index %d greater than BO count %d\n",
+			  hindex, exec->bo_count);
+		return NULL;
+	}
+	obj = exec->bo[hindex];
+	bo = to_vc4_bo(&obj->base);
+
+	if (bo->validated_shader) {
+		DRM_ERROR("Trying to use shader BO as something other than "
+			  "a shader\n");
+		return NULL;
+	}
+
+	return obj;
+}
+
+static struct drm_gem_cma_object *
+vc4_use_handle(struct vc4_exec_info *exec, uint32_t gem_handles_packet_index)
+{
+	return vc4_use_bo(exec, exec->bo_index[gem_handles_packet_index]);
+}
+
+static bool
+validate_bin_pos(struct vc4_exec_info *exec, void *untrusted, uint32_t pos)
+{
+	/* Note that the untrusted pointer passed to these functions is
+	 * incremented past the packet byte.
+	 */
+	return (untrusted - 1 == exec->bin_u + pos);
+}
+
+static uint32_t
+gl_shader_rec_size(uint32_t pointer_bits)
+{
+	uint32_t attribute_count = pointer_bits & 7;
+	bool extended = pointer_bits & 8;
+
+	if (attribute_count == 0)
+		attribute_count = 8;
+
+	if (extended)
+		return 100 + attribute_count * 4;
+	else
+		return 36 + attribute_count * 8;
+}
+
+bool
+vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_cma_object *fbo,
+		   uint32_t offset, uint8_t tiling_format,
+		   uint32_t width, uint32_t height, uint8_t cpp)
+{
+	uint32_t aligned_width, aligned_height, stride, size;
+	uint32_t utile_w = utile_width(cpp);
+	uint32_t utile_h = utile_height(cpp);
+
+	/* The shaded vertex format stores signed 12.4 fixed point
+	 * (-2048,2047) offsets from the viewport center, so we should
+	 * never have a render target larger than 4096.  The texture
+	 * unit can only sample from 2048x2048, so it's even more
+	 * restricted.  This lets us avoid worrying about overflow in
+	 * our math.
+	 */
+	if (width > 4096 || height > 4096) {
+		DRM_ERROR("Surface dimesions (%d,%d) too large", width, height);
+		return false;
+	}
+
+	switch (tiling_format) {
+	case VC4_TILING_FORMAT_LINEAR:
+		aligned_width = round_up(width, utile_w);
+		aligned_height = height;
+		break;
+	case VC4_TILING_FORMAT_T:
+		aligned_width = round_up(width, utile_w * 8);
+		aligned_height = round_up(height, utile_h * 8);
+		break;
+	case VC4_TILING_FORMAT_LT:
+		aligned_width = round_up(width, utile_w);
+		aligned_height = round_up(height, utile_h);
+		break;
+	default:
+		DRM_ERROR("buffer tiling %d unsupported\n", tiling_format);
+		return false;
+	}
+
+	stride = aligned_width * cpp;
+	size = stride * aligned_height;
+
+	if (size + offset < size ||
+	    size + offset > fbo->base.size) {
+		DRM_ERROR("Overflow in %dx%d (%dx%d) fbo size (%d + %d > %zd)\n",
+			  width, height,
+			  aligned_width, aligned_height,
+			  size, offset, fbo->base.size);
+		return false;
+	}
+
+	return true;
+}
+
+static int
+validate_flush(VALIDATE_ARGS)
+{
+	if (!validate_bin_pos(exec, untrusted, exec->args->bin_cl_size - 1)) {
+		DRM_ERROR("Bin CL must end with VC4_PACKET_FLUSH\n");
+		return -EINVAL;
+	}
+	exec->found_flush = true;
+
+	return 0;
+}
+
+static int
+validate_start_tile_binning(VALIDATE_ARGS)
+{
+	if (exec->found_start_tile_binning_packet) {
+		DRM_ERROR("Duplicate VC4_PACKET_START_TILE_BINNING\n");
+		return -EINVAL;
+	}
+	exec->found_start_tile_binning_packet = true;
+
+	if (!exec->found_tile_binning_mode_config_packet) {
+		DRM_ERROR("missing VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+validate_increment_semaphore(VALIDATE_ARGS)
+{
+	if (!validate_bin_pos(exec, untrusted, exec->args->bin_cl_size - 2)) {
+		DRM_ERROR("Bin CL must end with "
+			  "VC4_PACKET_INCREMENT_SEMAPHORE\n");
+		return -EINVAL;
+	}
+	exec->found_increment_semaphore_packet = true;
+
+	return 0;
+}
+
+static int
+validate_indexed_prim_list(VALIDATE_ARGS)
+{
+	struct drm_gem_cma_object *ib;
+	uint32_t length = *(uint32_t *)(untrusted + 1);
+	uint32_t offset = *(uint32_t *)(untrusted + 5);
+	uint32_t max_index = *(uint32_t *)(untrusted + 9);
+	uint32_t index_size = (*(uint8_t *)(untrusted + 0) >> 4) ? 2 : 1;
+	struct vc4_shader_state *shader_state;
+
+	/* Check overflow condition */
+	if (exec->shader_state_count == 0) {
+		DRM_ERROR("shader state must precede primitives\n");
+		return -EINVAL;
+	}
+	shader_state = &exec->shader_state[exec->shader_state_count - 1];
+
+	if (max_index > shader_state->max_index)
+		shader_state->max_index = max_index;
+
+	ib = vc4_use_handle(exec, 0);
+	if (!ib)
+		return -EINVAL;
+
+	if (offset > ib->base.size ||
+	    (ib->base.size - offset) / index_size < length) {
+		DRM_ERROR("IB access overflow (%d + %d*%d > %zd)\n",
+			  offset, length, index_size, ib->base.size);
+		return -EINVAL;
+	}
+
+	*(uint32_t *)(validated + 5) = ib->paddr + offset;
+
+	return 0;
+}
+
+static int
+validate_gl_array_primitive(VALIDATE_ARGS)
+{
+	uint32_t length = *(uint32_t *)(untrusted + 1);
+	uint32_t base_index = *(uint32_t *)(untrusted + 5);
+	uint32_t max_index;
+	struct vc4_shader_state *shader_state;
+
+	/* Check overflow condition */
+	if (exec->shader_state_count == 0) {
+		DRM_ERROR("shader state must precede primitives\n");
+		return -EINVAL;
+	}
+	shader_state = &exec->shader_state[exec->shader_state_count - 1];
+
+	if (length + base_index < length) {
+		DRM_ERROR("primitive vertex count overflow\n");
+		return -EINVAL;
+	}
+	max_index = length + base_index - 1;
+
+	if (max_index > shader_state->max_index)
+		shader_state->max_index = max_index;
+
+	return 0;
+}
+
+static int
+validate_gl_shader_state(VALIDATE_ARGS)
+{
+	uint32_t i = exec->shader_state_count++;
+
+	if (i >= exec->shader_state_size) {
+		DRM_ERROR("More requests for shader states than declared\n");
+		return -EINVAL;
+	}
+
+	exec->shader_state[i].addr = *(uint32_t *)untrusted;
+	exec->shader_state[i].max_index = 0;
+
+	if (exec->shader_state[i].addr & ~0xf) {
+		DRM_ERROR("high bits set in GL shader rec reference\n");
+		return -EINVAL;
+	}
+
+	*(uint32_t *)validated = (exec->shader_rec_p +
+				  exec->shader_state[i].addr);
+
+	exec->shader_rec_p +=
+		roundup(gl_shader_rec_size(exec->shader_state[i].addr), 16);
+
+	return 0;
+}
+
+static int
+validate_tile_binning_config(VALIDATE_ARGS)
+{
+	struct drm_device *dev = exec->exec_bo->base.dev;
+	struct vc4_bo *tile_bo;
+	uint8_t flags;
+	uint32_t tile_state_size, tile_alloc_size;
+	uint32_t tile_count;
+
+	if (exec->found_tile_binning_mode_config_packet) {
+		DRM_ERROR("Duplicate VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
+		return -EINVAL;
+	}
+	exec->found_tile_binning_mode_config_packet = true;
+
+	exec->bin_tiles_x = *(uint8_t *)(untrusted + 12);
+	exec->bin_tiles_y = *(uint8_t *)(untrusted + 13);
+	tile_count = exec->bin_tiles_x * exec->bin_tiles_y;
+	flags = *(uint8_t *)(untrusted + 14);
+
+	if (exec->bin_tiles_x == 0 ||
+	    exec->bin_tiles_y == 0) {
+		DRM_ERROR("Tile binning config of %dx%d too small\n",
+			  exec->bin_tiles_x, exec->bin_tiles_y);
+		return -EINVAL;
+	}
+
+	if (flags & (VC4_BIN_CONFIG_DB_NON_MS |
+		     VC4_BIN_CONFIG_TILE_BUFFER_64BIT)) {
+		DRM_ERROR("unsupported binning config flags 0x%02x\n", flags);
+		return -EINVAL;
+	}
+
+	/* The tile state data array is 48 bytes per tile, and we put it at
+	 * the start of a BO containing both it and the tile alloc.
+	 */
+	tile_state_size = 48 * tile_count;
+
+	/* Since the tile alloc array will follow us, align. */
+	exec->tile_alloc_offset = roundup(tile_state_size, 4096);
+
+	*(uint8_t *)(validated + 14) =
+		((flags & ~(VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_MASK |
+			    VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_MASK)) |
+		 VC4_BIN_CONFIG_AUTO_INIT_TSDA |
+		 VC4_SET_FIELD(VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_32,
+			       VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE) |
+		 VC4_SET_FIELD(VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE_128,
+			       VC4_BIN_CONFIG_ALLOC_BLOCK_SIZE));
+
+	/* Initial block size. */
+	tile_alloc_size = 32 * tile_count;
+
+	/*
+	 * The initial allocation gets rounded to the next 256 bytes before
+	 * the hardware starts fulfilling further allocations.
+	 */
+	tile_alloc_size = roundup(tile_alloc_size, 256);
+
+	/* Add space for the extra allocations.  This is what gets used first,
+	 * before overflow memory.  It must have at least 4096 bytes, but we
+	 * want to avoid overflow memory usage if possible.
+	 */
+	tile_alloc_size += 1024 * 1024;
+
+	tile_bo = vc4_bo_create(dev, exec->tile_alloc_offset + tile_alloc_size,
+				true);
+	exec->tile_bo = &tile_bo->base;
+	if (!exec->tile_bo)
+		return -ENOMEM;
+	list_add_tail(&tile_bo->unref_head, &exec->unref_list);
+
+	/* tile alloc address. */
+	*(uint32_t *)(validated + 0) = (exec->tile_bo->paddr +
+					exec->tile_alloc_offset);
+	/* tile alloc size. */
+	*(uint32_t *)(validated + 4) = tile_alloc_size;
+	/* tile state address. */
+	*(uint32_t *)(validated + 8) = exec->tile_bo->paddr;
+
+	return 0;
+}
+
+static int
+validate_gem_handles(VALIDATE_ARGS)
+{
+	memcpy(exec->bo_index, untrusted, sizeof(exec->bo_index));
+	return 0;
+}
+
+#define VC4_DEFINE_PACKET(packet, func) \
+	[packet] = { packet ## _SIZE, #packet, func }
+
+static const struct cmd_info {
+	uint16_t len;
+	const char *name;
+	int (*func)(struct vc4_exec_info *exec, void *validated,
+		    void *untrusted);
+} cmd_info[] = {
+	VC4_DEFINE_PACKET(VC4_PACKET_HALT, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_NOP, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_FLUSH, validate_flush),
+	VC4_DEFINE_PACKET(VC4_PACKET_FLUSH_ALL, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_START_TILE_BINNING,
+			  validate_start_tile_binning),
+	VC4_DEFINE_PACKET(VC4_PACKET_INCREMENT_SEMAPHORE,
+			  validate_increment_semaphore),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_GL_INDEXED_PRIMITIVE,
+			  validate_indexed_prim_list),
+	VC4_DEFINE_PACKET(VC4_PACKET_GL_ARRAY_PRIMITIVE,
+			  validate_gl_array_primitive),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_PRIMITIVE_LIST_FORMAT, NULL),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_GL_SHADER_STATE, validate_gl_shader_state),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_CONFIGURATION_BITS, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_FLAT_SHADE_FLAGS, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_POINT_SIZE, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_LINE_WIDTH, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_RHT_X_BOUNDARY, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_DEPTH_OFFSET, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_CLIP_WINDOW, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_VIEWPORT_OFFSET, NULL),
+	VC4_DEFINE_PACKET(VC4_PACKET_CLIPPER_XY_SCALING, NULL),
+	/* Note: The docs say this was also 105, but it was 106 in the
+	 * initial userland code drop.
+	 */
+	VC4_DEFINE_PACKET(VC4_PACKET_CLIPPER_Z_SCALING, NULL),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_TILE_BINNING_MODE_CONFIG,
+			  validate_tile_binning_config),
+
+	VC4_DEFINE_PACKET(VC4_PACKET_GEM_HANDLES, validate_gem_handles),
+};
+
+int
+vc4_validate_bin_cl(struct drm_device *dev,
+		    void *validated,
+		    void *unvalidated,
+		    struct vc4_exec_info *exec)
+{
+	uint32_t len = exec->args->bin_cl_size;
+	uint32_t dst_offset = 0;
+	uint32_t src_offset = 0;
+
+	while (src_offset < len) {
+		void *dst_pkt = validated + dst_offset;
+		void *src_pkt = unvalidated + src_offset;
+		u8 cmd = *(uint8_t *)src_pkt;
+		const struct cmd_info *info;
+
+		if (cmd >= ARRAY_SIZE(cmd_info)) {
+			DRM_ERROR("0x%08x: packet %d out of bounds\n",
+				  src_offset, cmd);
+			return -EINVAL;
+		}
+
+		info = &cmd_info[cmd];
+		if (!info->name) {
+			DRM_ERROR("0x%08x: packet %d invalid\n",
+				  src_offset, cmd);
+			return -EINVAL;
+		}
+
+		if (src_offset + info->len > len) {
+			DRM_ERROR("0x%08x: packet %d (%s) length 0x%08x "
+				  "exceeds bounds (0x%08x)\n",
+				  src_offset, cmd, info->name, info->len,
+				  src_offset + len);
+			return -EINVAL;
+		}
+
+		if (cmd != VC4_PACKET_GEM_HANDLES)
+			memcpy(dst_pkt, src_pkt, info->len);
+
+		if (info->func && info->func(exec,
+					     dst_pkt + 1,
+					     src_pkt + 1)) {
+			DRM_ERROR("0x%08x: packet %d (%s) failed to validate\n",
+				  src_offset, cmd, info->name);
+			return -EINVAL;
+		}
+
+		src_offset += info->len;
+		/* GEM handle loading doesn't produce HW packets. */
+		if (cmd != VC4_PACKET_GEM_HANDLES)
+			dst_offset += info->len;
+
+		/* When the CL hits halt, it'll stop reading anything else. */
+		if (cmd == VC4_PACKET_HALT)
+			break;
+	}
+
+	exec->ct0ea = exec->ct0ca + dst_offset;
+
+	if (!exec->found_start_tile_binning_packet) {
+		DRM_ERROR("Bin CL missing VC4_PACKET_START_TILE_BINNING\n");
+		return -EINVAL;
+	}
+
+	/* The bin CL must be ended with INCREMENT_SEMAPHORE and FLUSH.  The
+	 * semaphore is used to trigger the render CL to start up, and the
+	 * FLUSH is what caps the bin lists with
+	 * VC4_PACKET_RETURN_FROM_SUB_LIST (so they jump back to the main
+	 * render CL when they get called to) and actually triggers the queued
+	 * semaphore increment.
+	 */
+	if (!exec->found_increment_semaphore_packet || !exec->found_flush) {
+		DRM_ERROR("Bin CL missing VC4_PACKET_INCREMENT_SEMAPHORE + "
+			  "VC4_PACKET_FLUSH\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static bool
+reloc_tex(struct vc4_exec_info *exec,
+	  void *uniform_data_u,
+	  struct vc4_texture_sample_info *sample,
+	  uint32_t texture_handle_index)
+
+{
+	struct drm_gem_cma_object *tex;
+	uint32_t p0 = *(uint32_t *)(uniform_data_u + sample->p_offset[0]);
+	uint32_t p1 = *(uint32_t *)(uniform_data_u + sample->p_offset[1]);
+	uint32_t p2 = (sample->p_offset[2] != ~0 ?
+		       *(uint32_t *)(uniform_data_u + sample->p_offset[2]) : 0);
+	uint32_t p3 = (sample->p_offset[3] != ~0 ?
+		       *(uint32_t *)(uniform_data_u + sample->p_offset[3]) : 0);
+	uint32_t *validated_p0 = exec->uniforms_v + sample->p_offset[0];
+	uint32_t offset = p0 & VC4_TEX_P0_OFFSET_MASK;
+	uint32_t miplevels = VC4_GET_FIELD(p0, VC4_TEX_P0_MIPLVLS);
+	uint32_t width = VC4_GET_FIELD(p1, VC4_TEX_P1_WIDTH);
+	uint32_t height = VC4_GET_FIELD(p1, VC4_TEX_P1_HEIGHT);
+	uint32_t cpp, tiling_format, utile_w, utile_h;
+	uint32_t i;
+	uint32_t cube_map_stride = 0;
+	enum vc4_texture_data_type type;
+
+	tex = vc4_use_bo(exec, texture_handle_index);
+	if (!tex)
+		return false;
+
+	if (sample->is_direct) {
+		uint32_t remaining_size = tex->base.size - p0;
+
+		if (p0 > tex->base.size - 4) {
+			DRM_ERROR("UBO offset greater than UBO size\n");
+			goto fail;
+		}
+		if (p1 > remaining_size - 4) {
+			DRM_ERROR("UBO clamp would allow reads "
+				  "outside of UBO\n");
+			goto fail;
+		}
+		*validated_p0 = tex->paddr + p0;
+		return true;
+	}
+
+	if (width == 0)
+		width = 2048;
+	if (height == 0)
+		height = 2048;
+
+	if (p0 & VC4_TEX_P0_CMMODE_MASK) {
+		if (VC4_GET_FIELD(p2, VC4_TEX_P2_PTYPE) ==
+		    VC4_TEX_P2_PTYPE_CUBE_MAP_STRIDE)
+			cube_map_stride = p2 & VC4_TEX_P2_CMST_MASK;
+		if (VC4_GET_FIELD(p3, VC4_TEX_P2_PTYPE) ==
+		    VC4_TEX_P2_PTYPE_CUBE_MAP_STRIDE) {
+			if (cube_map_stride) {
+				DRM_ERROR("Cube map stride set twice\n");
+				goto fail;
+			}
+
+			cube_map_stride = p3 & VC4_TEX_P2_CMST_MASK;
+		}
+		if (!cube_map_stride) {
+			DRM_ERROR("Cube map stride not set\n");
+			goto fail;
+		}
+	}
+
+	type = (VC4_GET_FIELD(p0, VC4_TEX_P0_TYPE) |
+		(VC4_GET_FIELD(p1, VC4_TEX_P1_TYPE4) << 4));
+
+	switch (type) {
+	case VC4_TEXTURE_TYPE_RGBA8888:
+	case VC4_TEXTURE_TYPE_RGBX8888:
+	case VC4_TEXTURE_TYPE_RGBA32R:
+		cpp = 4;
+		break;
+	case VC4_TEXTURE_TYPE_RGBA4444:
+	case VC4_TEXTURE_TYPE_RGBA5551:
+	case VC4_TEXTURE_TYPE_RGB565:
+	case VC4_TEXTURE_TYPE_LUMALPHA:
+	case VC4_TEXTURE_TYPE_S16F:
+	case VC4_TEXTURE_TYPE_S16:
+		cpp = 2;
+		break;
+	case VC4_TEXTURE_TYPE_LUMINANCE:
+	case VC4_TEXTURE_TYPE_ALPHA:
+	case VC4_TEXTURE_TYPE_S8:
+		cpp = 1;
+		break;
+	case VC4_TEXTURE_TYPE_ETC1:
+	case VC4_TEXTURE_TYPE_BW1:
+	case VC4_TEXTURE_TYPE_A4:
+	case VC4_TEXTURE_TYPE_A1:
+	case VC4_TEXTURE_TYPE_RGBA64:
+	case VC4_TEXTURE_TYPE_YUV422R:
+	default:
+		DRM_ERROR("Texture format %d unsupported\n", type);
+		goto fail;
+	}
+	utile_w = utile_width(cpp);
+	utile_h = utile_height(cpp);
+
+	if (type == VC4_TEXTURE_TYPE_RGBA32R) {
+		tiling_format = VC4_TILING_FORMAT_LINEAR;
+	} else {
+		if (size_is_lt(width, height, cpp))
+			tiling_format = VC4_TILING_FORMAT_LT;
+		else
+			tiling_format = VC4_TILING_FORMAT_T;
+	}
+
+	if (!vc4_check_tex_size(exec, tex, offset + cube_map_stride * 5,
+				tiling_format, width, height, cpp)) {
+		goto fail;
+	}
+
+	/* The mipmap levels are stored before the base of the texture.  Make
+	 * sure there is actually space in the BO.
+	 */
+	for (i = 1; i <= miplevels; i++) {
+		uint32_t level_width = max(width >> i, 1u);
+		uint32_t level_height = max(height >> i, 1u);
+		uint32_t aligned_width, aligned_height;
+		uint32_t level_size;
+
+		/* Once the levels get small enough, they drop from T to LT. */
+		if (tiling_format == VC4_TILING_FORMAT_T &&
+		    size_is_lt(level_width, level_height, cpp)) {
+			tiling_format = VC4_TILING_FORMAT_LT;
+		}
+
+		switch (tiling_format) {
+		case VC4_TILING_FORMAT_T:
+			aligned_width = round_up(level_width, utile_w * 8);
+			aligned_height = round_up(level_height, utile_h * 8);
+			break;
+		case VC4_TILING_FORMAT_LT:
+			aligned_width = round_up(level_width, utile_w);
+			aligned_height = round_up(level_height, utile_h);
+			break;
+		default:
+			aligned_width = round_up(level_width, utile_w);
+			aligned_height = level_height;
+			break;
+		}
+
+		level_size = aligned_width * cpp * aligned_height;
+
+		if (offset < level_size) {
+			DRM_ERROR("Level %d (%dx%d -> %dx%d) size %db "
+				  "overflowed buffer bounds (offset %d)\n",
+				  i, level_width, level_height,
+				  aligned_width, aligned_height,
+				  level_size, offset);
+			goto fail;
+		}
+
+		offset -= level_size;
+	}
+
+	*validated_p0 = tex->paddr + p0;
+
+	return true;
+ fail:
+	DRM_INFO("Texture p0 at %d: 0x%08x\n", sample->p_offset[0], p0);
+	DRM_INFO("Texture p1 at %d: 0x%08x\n", sample->p_offset[1], p1);
+	DRM_INFO("Texture p2 at %d: 0x%08x\n", sample->p_offset[2], p2);
+	DRM_INFO("Texture p3 at %d: 0x%08x\n", sample->p_offset[3], p3);
+	return false;
+}
+
+static int
+validate_gl_shader_rec(struct drm_device *dev,
+		       struct vc4_exec_info *exec,
+		       struct vc4_shader_state *state)
+{
+	uint32_t *src_handles;
+	void *pkt_u, *pkt_v;
+	static const uint32_t shader_reloc_offsets[] = {
+		4, /* fs */
+		16, /* vs */
+		28, /* cs */
+	};
+	uint32_t shader_reloc_count = ARRAY_SIZE(shader_reloc_offsets);
+	struct drm_gem_cma_object *bo[shader_reloc_count + 8];
+	uint32_t nr_attributes, nr_relocs, packet_size;
+	int i;
+
+	nr_attributes = state->addr & 0x7;
+	if (nr_attributes == 0)
+		nr_attributes = 8;
+	packet_size = gl_shader_rec_size(state->addr);
+
+	nr_relocs = ARRAY_SIZE(shader_reloc_offsets) + nr_attributes;
+	if (nr_relocs * 4 > exec->shader_rec_size) {
+		DRM_ERROR("overflowed shader recs reading %d handles "
+			  "from %d bytes left\n",
+			  nr_relocs, exec->shader_rec_size);
+		return -EINVAL;
+	}
+	src_handles = exec->shader_rec_u;
+	exec->shader_rec_u += nr_relocs * 4;
+	exec->shader_rec_size -= nr_relocs * 4;
+
+	if (packet_size > exec->shader_rec_size) {
+		DRM_ERROR("overflowed shader recs copying %db packet "
+			  "from %d bytes left\n",
+			  packet_size, exec->shader_rec_size);
+		return -EINVAL;
+	}
+	pkt_u = exec->shader_rec_u;
+	pkt_v = exec->shader_rec_v;
+	memcpy(pkt_v, pkt_u, packet_size);
+	exec->shader_rec_u += packet_size;
+	/* Shader recs have to be aligned to 16 bytes (due to the attribute
+	 * flags being in the low bytes), so round the next validated shader
+	 * rec address up.  This should be safe, since we've got so many
+	 * relocations in a shader rec packet.
+	 */
+	BUG_ON(roundup(packet_size, 16) - packet_size > nr_relocs * 4);
+	exec->shader_rec_v += roundup(packet_size, 16);
+	exec->shader_rec_size -= packet_size;
+
+	if (!(*(uint16_t *)pkt_u & VC4_SHADER_FLAG_FS_SINGLE_THREAD)) {
+		DRM_ERROR("Multi-threaded fragment shaders not supported.\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < shader_reloc_count; i++) {
+		if (src_handles[i] > exec->bo_count) {
+			DRM_ERROR("Shader handle %d too big\n", src_handles[i]);
+			return -EINVAL;
+		}
+
+		bo[i] = exec->bo[src_handles[i]];
+		if (!bo[i])
+			return -EINVAL;
+	}
+	for (i = shader_reloc_count; i < nr_relocs; i++) {
+		bo[i] = vc4_use_bo(exec, src_handles[i]);
+		if (!bo[i])
+			return -EINVAL;
+	}
+
+	for (i = 0; i < shader_reloc_count; i++) {
+		struct vc4_validated_shader_info *validated_shader;
+		uint32_t o = shader_reloc_offsets[i];
+		uint32_t src_offset = *(uint32_t *)(pkt_u + o);
+		uint32_t *texture_handles_u;
+		void *uniform_data_u;
+		uint32_t tex;
+
+		*(uint32_t *)(pkt_v + o) = bo[i]->paddr + src_offset;
+
+		if (src_offset != 0) {
+			DRM_ERROR("Shaders must be at offset 0 of "
+				  "the BO.\n");
+			return -EINVAL;
+		}
+
+		validated_shader = to_vc4_bo(&bo[i]->base)->validated_shader;
+		if (!validated_shader)
+			return -EINVAL;
+
+		if (validated_shader->uniforms_src_size >
+		    exec->uniforms_size) {
+			DRM_ERROR("Uniforms src buffer overflow\n");
+			return -EINVAL;
+		}
+
+		texture_handles_u = exec->uniforms_u;
+		uniform_data_u = (texture_handles_u +
+				  validated_shader->num_texture_samples);
+
+		memcpy(exec->uniforms_v, uniform_data_u,
+		       validated_shader->uniforms_size);
+
+		for (tex = 0;
+		     tex < validated_shader->num_texture_samples;
+		     tex++) {
+			if (!reloc_tex(exec,
+				       uniform_data_u,
+				       &validated_shader->texture_samples[tex],
+				       texture_handles_u[tex])) {
+				return -EINVAL;
+			}
+		}
+
+		*(uint32_t *)(pkt_v + o + 4) = exec->uniforms_p;
+
+		exec->uniforms_u += validated_shader->uniforms_src_size;
+		exec->uniforms_v += validated_shader->uniforms_size;
+		exec->uniforms_p += validated_shader->uniforms_size;
+	}
+
+	for (i = 0; i < nr_attributes; i++) {
+		struct drm_gem_cma_object *vbo =
+			bo[ARRAY_SIZE(shader_reloc_offsets) + i];
+		uint32_t o = 36 + i * 8;
+		uint32_t offset = *(uint32_t *)(pkt_u + o + 0);
+		uint32_t attr_size = *(uint8_t *)(pkt_u + o + 4) + 1;
+		uint32_t stride = *(uint8_t *)(pkt_u + o + 5);
+		uint32_t max_index;
+
+		if (state->addr & 0x8)
+			stride |= (*(uint32_t *)(pkt_u + 100 + i * 4)) & ~0xff;
+
+		if (vbo->base.size < offset ||
+		    vbo->base.size - offset < attr_size) {
+			DRM_ERROR("BO offset overflow (%d + %d > %zu)\n",
+				  offset, attr_size, vbo->base.size);
+			return -EINVAL;
+		}
+
+		if (stride != 0) {
+			max_index = ((vbo->base.size - offset - attr_size) /
+				     stride);
+			if (state->max_index > max_index) {
+				DRM_ERROR("primitives use index %d out of "
+					  "supplied %d\n",
+					  state->max_index, max_index);
+				return -EINVAL;
+			}
+		}
+
+		*(uint32_t *)(pkt_v + o) = vbo->paddr + offset;
+	}
+
+	return 0;
+}
+
+int
+vc4_validate_shader_recs(struct drm_device *dev,
+			 struct vc4_exec_info *exec)
+{
+	uint32_t i;
+	int ret = 0;
+
+	for (i = 0; i < exec->shader_state_count; i++) {
+		ret = validate_gl_shader_rec(dev, exec, &exec->shader_state[i]);
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
new file mode 100644
index 000000000000..f67124b4c534
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -0,0 +1,513 @@
+/*
+ * Copyright © 2014 Broadcom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * DOC: Shader validator for VC4.
+ *
+ * The VC4 has no IOMMU between it and system memory, so a user with
+ * access to execute shaders could escalate privilege by overwriting
+ * system memory (using the VPM write address register in the
+ * general-purpose DMA mode) or reading system memory it shouldn't
+ * (reading it as a texture, or uniform data, or vertex data).
+ *
+ * This walks over a shader BO, ensuring that its accesses are
+ * appropriately bounded, and recording how many texture accesses are
+ * made and where so that we can do relocations for them in the
+ * uniform stream.
+ */
+
+#include "vc4_drv.h"
+#include "vc4_qpu_defines.h"
+
+struct vc4_shader_validation_state {
+	struct vc4_texture_sample_info tmu_setup[2];
+	int tmu_write_count[2];
+
+	/* For registers that were last written to by a MIN instruction with
+	 * one argument being a uniform, the address of the uniform.
+	 * Otherwise, ~0.
+	 *
+	 * This is used for the validation of direct address memory reads.
+	 */
+	uint32_t live_min_clamp_offsets[32 + 32 + 4];
+	bool live_max_clamp_regs[32 + 32 + 4];
+};
+
+static uint32_t
+waddr_to_live_reg_index(uint32_t waddr, bool is_b)
+{
+	if (waddr < 32) {
+		if (is_b)
+			return 32 + waddr;
+		else
+			return waddr;
+	} else if (waddr <= QPU_W_ACC3) {
+		return 64 + waddr - QPU_W_ACC0;
+	} else {
+		return ~0;
+	}
+}
+
+static uint32_t
+raddr_add_a_to_live_reg_index(uint64_t inst)
+{
+	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+	uint32_t add_a = QPU_GET_FIELD(inst, QPU_ADD_A);
+	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
+	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
+
+	if (add_a == QPU_MUX_A)
+		return raddr_a;
+	else if (add_a == QPU_MUX_B && sig != QPU_SIG_SMALL_IMM)
+		return 32 + raddr_b;
+	else if (add_a <= QPU_MUX_R3)
+		return 64 + add_a;
+	else
+		return ~0;
+}
+
+static bool
+is_tmu_submit(uint32_t waddr)
+{
+	return (waddr == QPU_W_TMU0_S ||
+		waddr == QPU_W_TMU1_S);
+}
+
+static bool
+is_tmu_write(uint32_t waddr)
+{
+	return (waddr >= QPU_W_TMU0_S &&
+		waddr <= QPU_W_TMU1_B);
+}
+
+static bool
+record_texture_sample(struct vc4_validated_shader_info *validated_shader,
+		      struct vc4_shader_validation_state *validation_state,
+		      int tmu)
+{
+	uint32_t s = validated_shader->num_texture_samples;
+	int i;
+	struct vc4_texture_sample_info *temp_samples;
+
+	temp_samples = krealloc(validated_shader->texture_samples,
+				(s + 1) * sizeof(*temp_samples),
+				GFP_KERNEL);
+	if (!temp_samples)
+		return false;
+
+	memcpy(&temp_samples[s],
+	       &validation_state->tmu_setup[tmu],
+	       sizeof(*temp_samples));
+
+	validated_shader->num_texture_samples = s + 1;
+	validated_shader->texture_samples = temp_samples;
+
+	for (i = 0; i < 4; i++)
+		validation_state->tmu_setup[tmu].p_offset[i] = ~0;
+
+	return true;
+}
+
+static bool
+check_tmu_write(uint64_t inst,
+		struct vc4_validated_shader_info *validated_shader,
+		struct vc4_shader_validation_state *validation_state,
+		bool is_mul)
+{
+	uint32_t waddr = (is_mul ?
+			  QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
+			  QPU_GET_FIELD(inst, QPU_WADDR_ADD));
+	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
+	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
+	int tmu = waddr > QPU_W_TMU0_B;
+	bool submit = is_tmu_submit(waddr);
+	bool is_direct = submit && validation_state->tmu_write_count[tmu] == 0;
+	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+
+	if (is_direct) {
+		uint32_t add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
+		uint32_t clamp_reg, clamp_offset;
+
+		if (sig == QPU_SIG_SMALL_IMM) {
+			DRM_ERROR("direct TMU read used small immediate\n");
+			return false;
+		}
+
+		/* Make sure that this texture load is an add of the base
+		 * address of the UBO to a clamped offset within the UBO.
+		 */
+		if (is_mul ||
+		    QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
+			DRM_ERROR("direct TMU load wasn't an add\n");
+			return false;
+		}
+
+		/* We assert that the the clamped address is the first
+		 * argument, and the UBO base address is the second argument.
+		 * This is arbitrary, but simpler than supporting flipping the
+		 * two either way.
+		 */
+		clamp_reg = raddr_add_a_to_live_reg_index(inst);
+		if (clamp_reg == ~0) {
+			DRM_ERROR("direct TMU load wasn't clamped\n");
+			return false;
+		}
+
+		clamp_offset = validation_state->live_min_clamp_offsets[clamp_reg];
+		if (clamp_offset == ~0) {
+			DRM_ERROR("direct TMU load wasn't clamped\n");
+			return false;
+		}
+
+		/* Store the clamp value's offset in p1 (see reloc_tex() in
+		 * vc4_validate.c).
+		 */
+		validation_state->tmu_setup[tmu].p_offset[1] =
+			clamp_offset;
+
+		if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
+		    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
+			DRM_ERROR("direct TMU load didn't add to a uniform\n");
+			return false;
+		}
+
+		validation_state->tmu_setup[tmu].is_direct = true;
+	} else {
+		if (raddr_a == QPU_R_UNIF || (sig != QPU_SIG_SMALL_IMM &&
+					      raddr_b == QPU_R_UNIF)) {
+			DRM_ERROR("uniform read in the same instruction as "
+				  "texture setup.\n");
+			return false;
+		}
+	}
+
+	if (validation_state->tmu_write_count[tmu] >= 4) {
+		DRM_ERROR("TMU%d got too many parameters before dispatch\n",
+			  tmu);
+		return false;
+	}
+	validation_state->tmu_setup[tmu].p_offset[validation_state->tmu_write_count[tmu]] =
+		validated_shader->uniforms_size;
+	validation_state->tmu_write_count[tmu]++;
+	/* Since direct uses a RADDR uniform reference, it will get counted in
+	 * check_instruction_reads()
+	 */
+	if (!is_direct)
+		validated_shader->uniforms_size += 4;
+
+	if (submit) {
+		if (!record_texture_sample(validated_shader,
+					   validation_state, tmu)) {
+			return false;
+		}
+
+		validation_state->tmu_write_count[tmu] = 0;
+	}
+
+	return true;
+}
+
+static bool
+check_reg_write(uint64_t inst,
+		struct vc4_validated_shader_info *validated_shader,
+		struct vc4_shader_validation_state *validation_state,
+		bool is_mul)
+{
+	uint32_t waddr = (is_mul ?
+			  QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
+			  QPU_GET_FIELD(inst, QPU_WADDR_ADD));
+
+	switch (waddr) {
+	case QPU_W_UNIFORMS_ADDRESS:
+		/* XXX: We'll probably need to support this for reladdr, but
+		 * it's definitely a security-related one.
+		 */
+		DRM_ERROR("uniforms address load unsupported\n");
+		return false;
+
+	case QPU_W_TLB_COLOR_MS:
+	case QPU_W_TLB_COLOR_ALL:
+	case QPU_W_TLB_Z:
+		/* These only interact with the tile buffer, not main memory,
+		 * so they're safe.
+		 */
+		return true;
+
+	case QPU_W_TMU0_S:
+	case QPU_W_TMU0_T:
+	case QPU_W_TMU0_R:
+	case QPU_W_TMU0_B:
+	case QPU_W_TMU1_S:
+	case QPU_W_TMU1_T:
+	case QPU_W_TMU1_R:
+	case QPU_W_TMU1_B:
+		return check_tmu_write(inst, validated_shader, validation_state,
+				       is_mul);
+
+	case QPU_W_HOST_INT:
+	case QPU_W_TMU_NOSWAP:
+	case QPU_W_TLB_ALPHA_MASK:
+	case QPU_W_MUTEX_RELEASE:
+		/* XXX: I haven't thought about these, so don't support them
+		 * for now.
+		 */
+		DRM_ERROR("Unsupported waddr %d\n", waddr);
+		return false;
+
+	case QPU_W_VPM_ADDR:
+		DRM_ERROR("General VPM DMA unsupported\n");
+		return false;
+
+	case QPU_W_VPM:
+	case QPU_W_VPMVCD_SETUP:
+		/* We allow VPM setup in general, even including VPM DMA
+		 * configuration setup, because the (unsafe) DMA can only be
+		 * triggered by QPU_W_VPM_ADDR writes.
+		 */
+		return true;
+
+	case QPU_W_TLB_STENCIL_SETUP:
+		return true;
+	}
+
+	return true;
+}
+
+static void
+track_live_clamps(uint64_t inst,
+		  struct vc4_validated_shader_info *validated_shader,
+		  struct vc4_shader_validation_state *validation_state)
+{
+	uint32_t op_add = QPU_GET_FIELD(inst, QPU_OP_ADD);
+	uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
+	uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
+	uint32_t cond_add = QPU_GET_FIELD(inst, QPU_COND_ADD);
+	uint32_t add_a = QPU_GET_FIELD(inst, QPU_ADD_A);
+	uint32_t add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
+	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
+	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
+	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+	bool ws = inst & QPU_WS;
+	uint32_t lri_add_a, lri_add, lri_mul;
+	bool add_a_is_min_0;
+
+	/* Check whether OP_ADD's A argumennt comes from a live MAX(x, 0),
+	 * before we clear previous live state.
+	 */
+	lri_add_a = raddr_add_a_to_live_reg_index(inst);
+	add_a_is_min_0 = (lri_add_a != ~0 &&
+			  validation_state->live_max_clamp_regs[lri_add_a]);
+
+	/* Clear live state for registers written by our instruction. */
+	lri_add = waddr_to_live_reg_index(waddr_add, ws);
+	lri_mul = waddr_to_live_reg_index(waddr_mul, !ws);
+	if (lri_mul != ~0) {
+		validation_state->live_max_clamp_regs[lri_mul] = false;
+		validation_state->live_min_clamp_offsets[lri_mul] = ~0;
+	}
+	if (lri_add != ~0) {
+		validation_state->live_max_clamp_regs[lri_add] = false;
+		validation_state->live_min_clamp_offsets[lri_add] = ~0;
+	} else {
+		/* Nothing further to do for live tracking, since only ADDs
+		 * generate new live clamp registers.
+		 */
+		return;
+	}
+
+	/* Now, handle remaining live clamp tracking for the ADD operation. */
+
+	if (cond_add != QPU_COND_ALWAYS)
+		return;
+
+	if (op_add == QPU_A_MAX) {
+		/* Track live clamps of a value to a minimum of 0 (in either
+		 * arg).
+		 */
+		if (sig != QPU_SIG_SMALL_IMM || raddr_b != 0 ||
+		    (add_a != QPU_MUX_B && add_b != QPU_MUX_B)) {
+			return;
+		}
+
+		validation_state->live_max_clamp_regs[lri_add] = true;
+	} else if (op_add == QPU_A_MIN) {
+		/* Track live clamps of a value clamped to a minimum of 0 and
+		 * a maximum of some uniform's offset.
+		 */
+		if (!add_a_is_min_0)
+			return;
+
+		if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
+		    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF &&
+		      sig != QPU_SIG_SMALL_IMM)) {
+			return;
+		}
+
+		validation_state->live_min_clamp_offsets[lri_add] =
+			validated_shader->uniforms_size;
+	}
+}
+
+static bool
+check_instruction_writes(uint64_t inst,
+			 struct vc4_validated_shader_info *validated_shader,
+			 struct vc4_shader_validation_state *validation_state)
+{
+	uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
+	uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
+	bool ok;
+
+	if (is_tmu_write(waddr_add) && is_tmu_write(waddr_mul)) {
+		DRM_ERROR("ADD and MUL both set up textures\n");
+		return false;
+	}
+
+	ok = (check_reg_write(inst, validated_shader, validation_state,
+			      false) &&
+	      check_reg_write(inst, validated_shader, validation_state,
+			      true));
+
+	track_live_clamps(inst, validated_shader, validation_state);
+
+	return ok;
+}
+
+static bool
+check_instruction_reads(uint64_t inst,
+			struct vc4_validated_shader_info *validated_shader)
+{
+	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
+	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
+	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+
+	if (raddr_a == QPU_R_UNIF ||
+	    (raddr_b == QPU_R_UNIF && sig != QPU_SIG_SMALL_IMM)) {
+		/* This can't overflow the uint32_t, because we're reading 8
+		 * bytes of instruction to increment by 4 here, so we'd
+		 * already be OOM.
+		 */
+		validated_shader->uniforms_size += 4;
+	}
+
+	return true;
+}
+
+struct vc4_validated_shader_info *
+vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
+{
+	bool found_shader_end = false;
+	int shader_end_ip = 0;
+	uint32_t ip, max_ip;
+	uint64_t *shader;
+	struct vc4_validated_shader_info *validated_shader;
+	struct vc4_shader_validation_state validation_state;
+	int i;
+
+	memset(&validation_state, 0, sizeof(validation_state));
+
+	for (i = 0; i < 8; i++)
+		validation_state.tmu_setup[i / 4].p_offset[i % 4] = ~0;
+	for (i = 0; i < ARRAY_SIZE(validation_state.live_min_clamp_offsets); i++)
+		validation_state.live_min_clamp_offsets[i] = ~0;
+
+	shader = shader_obj->vaddr;
+	max_ip = shader_obj->base.size / sizeof(uint64_t);
+
+	validated_shader = kcalloc(1, sizeof(*validated_shader), GFP_KERNEL);
+	if (!validated_shader)
+		return NULL;
+
+	for (ip = 0; ip < max_ip; ip++) {
+		uint64_t inst = shader[ip];
+		uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+
+		switch (sig) {
+		case QPU_SIG_NONE:
+		case QPU_SIG_WAIT_FOR_SCOREBOARD:
+		case QPU_SIG_SCOREBOARD_UNLOCK:
+		case QPU_SIG_COLOR_LOAD:
+		case QPU_SIG_LOAD_TMU0:
+		case QPU_SIG_LOAD_TMU1:
+		case QPU_SIG_PROG_END:
+		case QPU_SIG_SMALL_IMM:
+			if (!check_instruction_writes(inst, validated_shader,
+						      &validation_state)) {
+				DRM_ERROR("Bad write at ip %d\n", ip);
+				goto fail;
+			}
+
+			if (!check_instruction_reads(inst, validated_shader))
+				goto fail;
+
+			if (sig == QPU_SIG_PROG_END) {
+				found_shader_end = true;
+				shader_end_ip = ip;
+			}
+
+			break;
+
+		case QPU_SIG_LOAD_IMM:
+			if (!check_instruction_writes(inst, validated_shader,
+						      &validation_state)) {
+				DRM_ERROR("Bad LOAD_IMM write at ip %d\n", ip);
+				goto fail;
+			}
+			break;
+
+		default:
+			DRM_ERROR("Unsupported QPU signal %d at "
+				  "instruction %d\n", sig, ip);
+			goto fail;
+		}
+
+		/* There are two delay slots after program end is signaled
+		 * that are still executed, then we're finished.
+		 */
+		if (found_shader_end && ip == shader_end_ip + 2)
+			break;
+	}
+
+	if (ip == max_ip) {
+		DRM_ERROR("shader failed to terminate before "
+			  "shader BO end at %zd\n",
+			  shader_obj->base.size);
+		goto fail;
+	}
+
+	/* Again, no chance of integer overflow here because the worst case
+	 * scenario is 8 bytes of uniforms plus handles per 8-byte
+	 * instruction.
+	 */
+	validated_shader->uniforms_src_size =
+		(validated_shader->uniforms_size +
+		 4 * validated_shader->num_texture_samples);
+
+	return validated_shader;
+
+fail:
+	if (validated_shader) {
+		kfree(validated_shader->texture_samples);
+		kfree(validated_shader);
+	}
+	return NULL;
+}
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 578fe0a9324c..a165f03eaa79 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -215,7 +215,7 @@ static const struct drm_framebuffer_funcs virtio_gpu_fb_funcs = {
 int
 virtio_gpu_framebuffer_init(struct drm_device *dev,
 			    struct virtio_gpu_framebuffer *vgfb,
-			    struct drm_mode_fb_cmd2 *mode_cmd,
+			    const struct drm_mode_fb_cmd2 *mode_cmd,
 			    struct drm_gem_object *obj)
 {
 	int ret;
@@ -374,16 +374,6 @@ static const struct drm_connector_helper_funcs virtio_gpu_conn_helper_funcs = {
 	.best_encoder = virtio_gpu_best_encoder,
 };
 
-static void virtio_gpu_conn_save(struct drm_connector *connector)
-{
-	DRM_DEBUG("\n");
-}
-
-static void virtio_gpu_conn_restore(struct drm_connector *connector)
-{
-	DRM_DEBUG("\n");
-}
-
 static enum drm_connector_status virtio_gpu_conn_detect(
 			struct drm_connector *connector,
 			bool force)
@@ -409,10 +399,8 @@ static void virtio_gpu_conn_destroy(struct drm_connector *connector)
 
 static const struct drm_connector_funcs virtio_gpu_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
-	.save = virtio_gpu_conn_save,
-	.restore = virtio_gpu_conn_restore,
 	.detect = virtio_gpu_conn_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes_nomerge,
+	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = virtio_gpu_conn_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -443,7 +431,7 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
 	if (IS_ERR(plane))
 		return PTR_ERR(plane);
 	drm_crtc_init_with_planes(dev, crtc, plane, NULL,
-				  &virtio_gpu_crtc_funcs);
+				  &virtio_gpu_crtc_funcs, NULL);
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 	drm_crtc_helper_add(crtc, &virtio_gpu_crtc_helper_funcs);
 	plane->crtc = crtc;
@@ -453,7 +441,7 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
 	drm_connector_helper_add(connector, &virtio_gpu_conn_helper_funcs);
 
 	drm_encoder_init(dev, encoder, &virtio_gpu_enc_funcs,
-			 DRM_MODE_ENCODER_VIRTUAL);
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
 	drm_encoder_helper_add(encoder, &virtio_gpu_enc_helper_funcs);
 	encoder->possible_crtcs = 1 << index;
 
@@ -465,7 +453,7 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
 static struct drm_framebuffer *
 virtio_gpu_user_framebuffer_create(struct drm_device *dev,
 				   struct drm_file *file_priv,
-				   struct drm_mode_fb_cmd2 *mode_cmd)
+				   const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj = NULL;
 	struct virtio_gpu_framebuffer *virtio_gpu_fb;
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 79f0abe69b64..8f486f4c7023 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -328,7 +328,7 @@ void virtio_gpu_dequeue_fence_func(struct work_struct *work);
 /* virtio_gpu_display.c */
 int virtio_gpu_framebuffer_init(struct drm_device *dev,
 				struct virtio_gpu_framebuffer *vgfb,
-				struct drm_mode_fb_cmd2 *mode_cmd,
+				const struct drm_mode_fb_cmd2 *mode_cmd,
 				struct drm_gem_object *obj);
 int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev);
 void virtio_gpu_modeset_fini(struct virtio_gpu_device *vgdev);
diff --git a/drivers/gpu/drm/virtio/virtgpu_fb.c b/drivers/gpu/drm/virtio/virtgpu_fb.c
index 6a81e084593b..2242a80866a9 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fb.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fb.c
@@ -32,7 +32,6 @@
 struct virtio_gpu_fbdev {
 	struct drm_fb_helper           helper;
 	struct virtio_gpu_framebuffer  vgfb;
-	struct list_head	       fbdev_list;
 	struct virtio_gpu_device       *vgdev;
 	struct delayed_work            work;
 };
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 06496a128162..4150873d432e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -130,7 +130,7 @@ int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags)
 	static vq_callback_t *callbacks[] = {
 		virtio_gpu_ctrl_ack, virtio_gpu_cursor_ack
 	};
-	static const char *names[] = { "control", "cursor" };
+	static const char * const names[] = { "control", "cursor" };
 
 	struct virtio_gpu_device *vgdev;
 	/* this will expand later */
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 4a74129c5708..572fb351feab 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -107,7 +107,7 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 				       &virtio_gpu_plane_funcs,
 				       virtio_gpu_formats,
 				       ARRAY_SIZE(virtio_gpu_formats),
-				       DRM_PLANE_TYPE_PRIMARY);
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		goto err_plane_init;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
index 6377e8151000..67cebb23c940 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
@@ -247,7 +247,7 @@ static void __vmw_cmdbuf_header_free(struct vmw_cmdbuf_header *header)
 {
 	struct vmw_cmdbuf_man *man = header->man;
 
-	BUG_ON(!spin_is_locked(&man->lock));
+	lockdep_assert_held_once(&man->lock);
 
 	if (header->inline_space) {
 		vmw_cmdbuf_header_inline_free(header);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index c49812b80dd0..24fb348a44e1 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -25,6 +25,7 @@
  *
  **************************************************************************/
 #include <linux/module.h>
+#include <linux/console.h>
 
 #include <drm/drmP.h>
 #include "vmwgfx_drv.h"
@@ -1538,6 +1539,12 @@ static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 static int __init vmwgfx_init(void)
 {
 	int ret;
+
+#ifdef CONFIG_VGA_CONSOLE
+	if (vgacon_text_force())
+		return -EINVAL;
+#endif
+
 	ret = drm_pci_init(&driver, &vmw_pci_driver);
 	if (ret)
 		DRM_ERROR("Failed initializing DRM.\n");
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 9b4bb9e74d73..b221a8c40282 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -508,7 +508,7 @@ int vmw_kms_readback(struct vmw_private *dev_priv,
 }
 
 
-static struct drm_framebuffer_funcs vmw_framebuffer_surface_funcs = {
+static const struct drm_framebuffer_funcs vmw_framebuffer_surface_funcs = {
 	.destroy = vmw_framebuffer_surface_destroy,
 	.dirty = vmw_framebuffer_surface_dirty,
 };
@@ -685,7 +685,7 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
 	return ret;
 }
 
-static struct drm_framebuffer_funcs vmw_framebuffer_dmabuf_funcs = {
+static const struct drm_framebuffer_funcs vmw_framebuffer_dmabuf_funcs = {
 	.destroy = vmw_framebuffer_dmabuf_destroy,
 	.dirty = vmw_framebuffer_dmabuf_dirty,
 };
@@ -763,21 +763,25 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 	uint32_t format;
 	struct drm_vmw_size content_base_size;
 	struct vmw_resource *res;
+	unsigned int bytes_pp;
 	int ret;
 
 	switch (mode_cmd->depth) {
 	case 32:
 	case 24:
 		format = SVGA3D_X8R8G8B8;
+		bytes_pp = 4;
 		break;
 
 	case 16:
 	case 15:
 		format = SVGA3D_R5G6B5;
+		bytes_pp = 2;
 		break;
 
 	case 8:
 		format = SVGA3D_P8;
+		bytes_pp = 1;
 		break;
 
 	default:
@@ -785,7 +789,7 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	content_base_size.width  = mode_cmd->width;
+	content_base_size.width  = mode_cmd->pitch / bytes_pp;
 	content_base_size.height = mode_cmd->height;
 	content_base_size.depth  = 1;
 
@@ -968,7 +972,7 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
 
 static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 						 struct drm_file *file_priv,
-						 struct drm_mode_fb_cmd2 *mode_cmd2)
+						 const struct drm_mode_fb_cmd2 *mode_cmd2)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
@@ -1369,14 +1373,6 @@ static int vmw_du_update_layout(struct vmw_private *dev_priv, unsigned num,
 	return 0;
 }
 
-void vmw_du_crtc_save(struct drm_crtc *crtc)
-{
-}
-
-void vmw_du_crtc_restore(struct drm_crtc *crtc)
-{
-}
-
 void vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
 			   u16 *r, u16 *g, u16 *b,
 			   uint32_t start, uint32_t size)
@@ -1398,14 +1394,6 @@ int vmw_du_connector_dpms(struct drm_connector *connector, int mode)
 	return 0;
 }
 
-void vmw_du_connector_save(struct drm_connector *connector)
-{
-}
-
-void vmw_du_connector_restore(struct drm_connector *connector)
-{
-}
-
 enum drm_connector_status
 vmw_du_connector_detect(struct drm_connector *connector, bool force)
 {
@@ -1592,7 +1580,7 @@ int vmw_du_connector_fill_modes(struct drm_connector *connector,
 		drm_mode_probed_add(connector, mode);
 	}
 
-	drm_mode_connector_list_update(connector, true);
+	drm_mode_connector_list_update(connector);
 	/* Move the prefered mode first, help apps pick the right mode. */
 	drm_mode_sort(&connector->modes);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 52caecb4502e..b6fa44fe8929 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -294,9 +294,7 @@ static int vmw_ldu_crtc_set_config(struct drm_mode_set *set)
 	return vmw_ldu_commit_list(dev_priv);
 }
 
-static struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
-	.save = vmw_du_crtc_save,
-	.restore = vmw_du_crtc_restore,
+static const struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
 	.cursor_set2 = vmw_du_crtc_cursor_set2,
 	.cursor_move = vmw_du_crtc_cursor_move,
 	.gamma_set = vmw_du_crtc_gamma_set,
@@ -314,7 +312,7 @@ static void vmw_ldu_encoder_destroy(struct drm_encoder *encoder)
 	vmw_ldu_destroy(vmw_encoder_to_ldu(encoder));
 }
 
-static struct drm_encoder_funcs vmw_legacy_encoder_funcs = {
+static const struct drm_encoder_funcs vmw_legacy_encoder_funcs = {
 	.destroy = vmw_ldu_encoder_destroy,
 };
 
@@ -327,10 +325,8 @@ static void vmw_ldu_connector_destroy(struct drm_connector *connector)
 	vmw_ldu_destroy(vmw_connector_to_ldu(connector));
 }
 
-static struct drm_connector_funcs vmw_legacy_connector_funcs = {
+static const struct drm_connector_funcs vmw_legacy_connector_funcs = {
 	.dpms = vmw_du_connector_dpms,
-	.save = vmw_du_connector_save,
-	.restore = vmw_du_connector_restore,
 	.detect = vmw_du_connector_detect,
 	.fill_modes = vmw_du_connector_fill_modes,
 	.set_property = vmw_du_connector_set_property,
@@ -367,7 +363,7 @@ static int vmw_ldu_init(struct vmw_private *dev_priv, unsigned unit)
 	connector->status = vmw_du_connector_detect(connector, true);
 
 	drm_encoder_init(dev, encoder, &vmw_legacy_encoder_funcs,
-			 DRM_MODE_ENCODER_VIRTUAL);
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
 	drm_mode_connector_attach_encoder(connector, encoder);
 	encoder->possible_crtcs = (1 << unit);
 	encoder->possible_clones = 0;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 13926ff192e3..db082bea8daf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -530,9 +530,7 @@ out_no_fence:
 	return ret;
 }
 
-static struct drm_crtc_funcs vmw_screen_object_crtc_funcs = {
-	.save = vmw_du_crtc_save,
-	.restore = vmw_du_crtc_restore,
+static const struct drm_crtc_funcs vmw_screen_object_crtc_funcs = {
 	.cursor_set2 = vmw_du_crtc_cursor_set2,
 	.cursor_move = vmw_du_crtc_cursor_move,
 	.gamma_set = vmw_du_crtc_gamma_set,
@@ -550,7 +548,7 @@ static void vmw_sou_encoder_destroy(struct drm_encoder *encoder)
 	vmw_sou_destroy(vmw_encoder_to_sou(encoder));
 }
 
-static struct drm_encoder_funcs vmw_screen_object_encoder_funcs = {
+static const struct drm_encoder_funcs vmw_screen_object_encoder_funcs = {
 	.destroy = vmw_sou_encoder_destroy,
 };
 
@@ -563,12 +561,8 @@ static void vmw_sou_connector_destroy(struct drm_connector *connector)
 	vmw_sou_destroy(vmw_connector_to_sou(connector));
 }
 
-static struct drm_connector_funcs vmw_sou_connector_funcs = {
+static const struct drm_connector_funcs vmw_sou_connector_funcs = {
 	.dpms = vmw_du_connector_dpms,
-	.save = vmw_du_connector_save,
-	.restore = vmw_du_connector_restore,
-	.detect = vmw_du_connector_detect,
-	.fill_modes = vmw_du_connector_fill_modes,
 	.set_property = vmw_du_connector_set_property,
 	.destroy = vmw_sou_connector_destroy,
 };
@@ -603,7 +597,7 @@ static int vmw_sou_init(struct vmw_private *dev_priv, unsigned unit)
 	connector->status = vmw_du_connector_detect(connector, true);
 
 	drm_encoder_init(dev, encoder, &vmw_screen_object_encoder_funcs,
-			 DRM_MODE_ENCODER_VIRTUAL);
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
 	drm_mode_connector_attach_encoder(connector, encoder);
 	encoder->possible_crtcs = (1 << unit);
 	encoder->possible_clones = 0;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index f823fc3efed7..4ef5ffd7189d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -1040,9 +1040,7 @@ out_finish:
 /*
  *  Screen Target CRTC dispatch table
  */
-static struct drm_crtc_funcs vmw_stdu_crtc_funcs = {
-	.save = vmw_du_crtc_save,
-	.restore = vmw_du_crtc_restore,
+static const struct drm_crtc_funcs vmw_stdu_crtc_funcs = {
 	.cursor_set2 = vmw_du_crtc_cursor_set2,
 	.cursor_move = vmw_du_crtc_cursor_move,
 	.gamma_set = vmw_du_crtc_gamma_set,
@@ -1072,7 +1070,7 @@ static void vmw_stdu_encoder_destroy(struct drm_encoder *encoder)
 	vmw_stdu_destroy(vmw_encoder_to_stdu(encoder));
 }
 
-static struct drm_encoder_funcs vmw_stdu_encoder_funcs = {
+static const struct drm_encoder_funcs vmw_stdu_encoder_funcs = {
 	.destroy = vmw_stdu_encoder_destroy,
 };
 
@@ -1099,10 +1097,8 @@ static void vmw_stdu_connector_destroy(struct drm_connector *connector)
 
 
 
-static struct drm_connector_funcs vmw_stdu_connector_funcs = {
+static const struct drm_connector_funcs vmw_stdu_connector_funcs = {
 	.dpms = vmw_du_connector_dpms,
-	.save = vmw_du_connector_save,
-	.restore = vmw_du_connector_restore,
 	.detect = vmw_du_connector_detect,
 	.fill_modes = vmw_du_connector_fill_modes,
 	.set_property = vmw_du_connector_set_property,
@@ -1149,7 +1145,7 @@ static int vmw_stdu_init(struct vmw_private *dev_priv, unsigned unit)
 	connector->status = vmw_du_connector_detect(connector, false);
 
 	drm_encoder_init(dev, encoder, &vmw_stdu_encoder_funcs,
-			 DRM_MODE_ENCODER_VIRTUAL);
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
 	drm_mode_connector_attach_encoder(connector, encoder);
 	encoder->possible_crtcs = (1 << unit);
 	encoder->possible_clones = 0;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index 7d620e82e000..c2a721a8cef9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -771,7 +771,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	}
 	srf->offsets = kmalloc(srf->num_sizes * sizeof(*srf->offsets),
 			       GFP_KERNEL);
-	if (unlikely(srf->sizes == NULL)) {
+	if (unlikely(srf->offsets == NULL)) {
 		ret = -ENOMEM;
 		goto out_no_offsets;
 	}
@@ -815,11 +815,8 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	    srf->sizes[0].height == 64 &&
 	    srf->format == SVGA3D_A8R8G8B8) {
 
-		srf->snooper.image = kmalloc(64 * 64 * 4, GFP_KERNEL);
-		/* clear the image */
-		if (srf->snooper.image) {
-			memset(srf->snooper.image, 0x00, 64 * 64 * 4);
-		} else {
+		srf->snooper.image = kzalloc(64 * 64 * 4, GFP_KERNEL);
+		if (!srf->snooper.image) {
 			DRM_ERROR("Failed to allocate cursor_image\n");
 			ret = -ENOMEM;
 			goto out_no_copy;
diff --git a/drivers/gpu/host1x/Makefile b/drivers/gpu/host1x/Makefile
index c1189f004441..a1d9974cfcb5 100644
--- a/drivers/gpu/host1x/Makefile
+++ b/drivers/gpu/host1x/Makefile
@@ -10,6 +10,7 @@ host1x-y = \
 	mipi.o \
 	hw/host1x01.o \
 	hw/host1x02.o \
-	hw/host1x04.o
+	hw/host1x04.o \
+	hw/host1x05.o
 
 obj-$(CONFIG_TEGRA_HOST1X) += host1x.o
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index 4a99c6416e6a..da462afcb225 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -538,6 +538,8 @@ EXPORT_SYMBOL(host1x_driver_register_full);
 
 void host1x_driver_unregister(struct host1x_driver *driver)
 {
+	driver_unregister(&driver->driver);
+
 	mutex_lock(&drivers_lock);
 	list_del_init(&driver->list);
 	mutex_unlock(&drivers_lock);
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 53d3d1d45b48..314bf3718cc7 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -35,6 +35,7 @@
 #include "hw/host1x01.h"
 #include "hw/host1x02.h"
 #include "hw/host1x04.h"
+#include "hw/host1x05.h"
 
 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r)
 {
@@ -87,7 +88,17 @@ static const struct host1x_info host1x04_info = {
 	.sync_offset = 0x2100,
 };
 
+static const struct host1x_info host1x05_info = {
+	.nb_channels = 14,
+	.nb_pts = 192,
+	.nb_mlocks = 16,
+	.nb_bases = 64,
+	.init = host1x05_init,
+	.sync_offset = 0x2100,
+};
+
 static struct of_device_id host1x_of_match[] = {
+	{ .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, },
 	{ .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, },
 	{ .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, },
 	{ .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, },
@@ -212,6 +223,11 @@ static struct platform_driver tegra_host1x_driver = {
 	.remove = host1x_remove,
 };
 
+static struct platform_driver * const drivers[] = {
+	&tegra_host1x_driver,
+	&tegra_mipi_driver,
+};
+
 static int __init tegra_host1x_init(void)
 {
 	int err;
@@ -220,28 +236,17 @@ static int __init tegra_host1x_init(void)
 	if (err < 0)
 		return err;
 
-	err = platform_driver_register(&tegra_host1x_driver);
-	if (err < 0)
-		goto unregister_bus;
-
-	err = platform_driver_register(&tegra_mipi_driver);
+	err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 	if (err < 0)
-		goto unregister_host1x;
+		bus_unregister(&host1x_bus_type);
 
-	return 0;
-
-unregister_host1x:
-	platform_driver_unregister(&tegra_host1x_driver);
-unregister_bus:
-	bus_unregister(&host1x_bus_type);
 	return err;
 }
 module_init(tegra_host1x_init);
 
 static void __exit tegra_host1x_exit(void)
 {
-	platform_driver_unregister(&tegra_mipi_driver);
-	platform_driver_unregister(&tegra_host1x_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 	bus_unregister(&host1x_bus_type);
 }
 module_exit(tegra_host1x_exit);
diff --git a/drivers/gpu/host1x/hw/host1x05.c b/drivers/gpu/host1x/hw/host1x05.c
new file mode 100644
index 000000000000..047097ce3bad
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x05.c
@@ -0,0 +1,42 @@
+/*
+ * Host1x init for Tegra210 SoCs
+ *
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* include hw specification */
+#include "host1x05.h"
+#include "host1x05_hardware.h"
+
+/* include code */
+#include "cdma_hw.c"
+#include "channel_hw.c"
+#include "debug_hw.c"
+#include "intr_hw.c"
+#include "syncpt_hw.c"
+
+#include "../dev.h"
+
+int host1x05_init(struct host1x *host)
+{
+	host->channel_op = &host1x_channel_ops;
+	host->cdma_op = &host1x_cdma_ops;
+	host->cdma_pb_op = &host1x_pushbuffer_ops;
+	host->syncpt_op = &host1x_syncpt_ops;
+	host->intr_op = &host1x_intr_ops;
+	host->debug_op = &host1x_debug_ops;
+
+	return 0;
+}
diff --git a/drivers/gpu/host1x/hw/host1x05.h b/drivers/gpu/host1x/hw/host1x05.h
new file mode 100644
index 000000000000..a306d9c05cd5
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x05.h
@@ -0,0 +1,26 @@
+/*
+ * Host1x init for Tegra210 SoCs
+ *
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HOST1X_HOST1X05_H
+#define HOST1X_HOST1X05_H
+
+struct host1x;
+
+int host1x05_init(struct host1x *host);
+
+#endif
diff --git a/drivers/gpu/host1x/hw/host1x05_hardware.h b/drivers/gpu/host1x/hw/host1x05_hardware.h
new file mode 100644
index 000000000000..2937ebb6be11
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x05_hardware.h
@@ -0,0 +1,142 @@
+/*
+ * Tegra host1x Register Offsets for Tegra210
+ *
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __HOST1X_HOST1X05_HARDWARE_H
+#define __HOST1X_HOST1X05_HARDWARE_H
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+#include "hw_host1x05_channel.h"
+#include "hw_host1x05_sync.h"
+#include "hw_host1x05_uclass.h"
+
+static inline u32 host1x_class_host_wait_syncpt(
+	unsigned indx, unsigned threshold)
+{
+	return host1x_uclass_wait_syncpt_indx_f(indx)
+		| host1x_uclass_wait_syncpt_thresh_f(threshold);
+}
+
+static inline u32 host1x_class_host_load_syncpt_base(
+	unsigned indx, unsigned threshold)
+{
+	return host1x_uclass_load_syncpt_base_base_indx_f(indx)
+		| host1x_uclass_load_syncpt_base_value_f(threshold);
+}
+
+static inline u32 host1x_class_host_wait_syncpt_base(
+	unsigned indx, unsigned base_indx, unsigned offset)
+{
+	return host1x_uclass_wait_syncpt_base_indx_f(indx)
+		| host1x_uclass_wait_syncpt_base_base_indx_f(base_indx)
+		| host1x_uclass_wait_syncpt_base_offset_f(offset);
+}
+
+static inline u32 host1x_class_host_incr_syncpt_base(
+	unsigned base_indx, unsigned offset)
+{
+	return host1x_uclass_incr_syncpt_base_base_indx_f(base_indx)
+		| host1x_uclass_incr_syncpt_base_offset_f(offset);
+}
+
+static inline u32 host1x_class_host_incr_syncpt(
+	unsigned cond, unsigned indx)
+{
+	return host1x_uclass_incr_syncpt_cond_f(cond)
+		| host1x_uclass_incr_syncpt_indx_f(indx);
+}
+
+static inline u32 host1x_class_host_indoff_reg_write(
+	unsigned mod_id, unsigned offset, bool auto_inc)
+{
+	u32 v = host1x_uclass_indoff_indbe_f(0xf)
+		| host1x_uclass_indoff_indmodid_f(mod_id)
+		| host1x_uclass_indoff_indroffset_f(offset);
+	if (auto_inc)
+		v |= host1x_uclass_indoff_autoinc_f(1);
+	return v;
+}
+
+static inline u32 host1x_class_host_indoff_reg_read(
+	unsigned mod_id, unsigned offset, bool auto_inc)
+{
+	u32 v = host1x_uclass_indoff_indmodid_f(mod_id)
+		| host1x_uclass_indoff_indroffset_f(offset)
+		| host1x_uclass_indoff_rwn_read_v();
+	if (auto_inc)
+		v |= host1x_uclass_indoff_autoinc_f(1);
+	return v;
+}
+
+/* cdma opcodes */
+static inline u32 host1x_opcode_setclass(
+	unsigned class_id, unsigned offset, unsigned mask)
+{
+	return (0 << 28) | (offset << 16) | (class_id << 6) | mask;
+}
+
+static inline u32 host1x_opcode_incr(unsigned offset, unsigned count)
+{
+	return (1 << 28) | (offset << 16) | count;
+}
+
+static inline u32 host1x_opcode_nonincr(unsigned offset, unsigned count)
+{
+	return (2 << 28) | (offset << 16) | count;
+}
+
+static inline u32 host1x_opcode_mask(unsigned offset, unsigned mask)
+{
+	return (3 << 28) | (offset << 16) | mask;
+}
+
+static inline u32 host1x_opcode_imm(unsigned offset, unsigned value)
+{
+	return (4 << 28) | (offset << 16) | value;
+}
+
+static inline u32 host1x_opcode_imm_incr_syncpt(unsigned cond, unsigned indx)
+{
+	return host1x_opcode_imm(host1x_uclass_incr_syncpt_r(),
+		host1x_class_host_incr_syncpt(cond, indx));
+}
+
+static inline u32 host1x_opcode_restart(unsigned address)
+{
+	return (5 << 28) | (address >> 4);
+}
+
+static inline u32 host1x_opcode_gather(unsigned count)
+{
+	return (6 << 28) | count;
+}
+
+static inline u32 host1x_opcode_gather_nonincr(unsigned offset,	unsigned count)
+{
+	return (6 << 28) | (offset << 16) | BIT(15) | count;
+}
+
+static inline u32 host1x_opcode_gather_incr(unsigned offset, unsigned count)
+{
+	return (6 << 28) | (offset << 16) | BIT(15) | BIT(14) | count;
+}
+
+#define HOST1X_OPCODE_NOP host1x_opcode_nonincr(0, 0)
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x05_channel.h b/drivers/gpu/host1x/hw/hw_host1x05_channel.h
new file mode 100644
index 000000000000..fce6e2c1ff4c
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x05_channel.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X05_CHANNEL_H
+#define HOST1X_HW_HOST1X05_CHANNEL_H
+
+static inline u32 host1x_channel_fifostat_r(void)
+{
+	return 0x0;
+}
+#define HOST1X_CHANNEL_FIFOSTAT \
+	host1x_channel_fifostat_r()
+static inline u32 host1x_channel_fifostat_cfempty_v(u32 r)
+{
+	return (r >> 11) & 0x1;
+}
+#define HOST1X_CHANNEL_FIFOSTAT_CFEMPTY_V(r) \
+	host1x_channel_fifostat_cfempty_v(r)
+static inline u32 host1x_channel_dmastart_r(void)
+{
+	return 0x14;
+}
+#define HOST1X_CHANNEL_DMASTART \
+	host1x_channel_dmastart_r()
+static inline u32 host1x_channel_dmaput_r(void)
+{
+	return 0x18;
+}
+#define HOST1X_CHANNEL_DMAPUT \
+	host1x_channel_dmaput_r()
+static inline u32 host1x_channel_dmaget_r(void)
+{
+	return 0x1c;
+}
+#define HOST1X_CHANNEL_DMAGET \
+	host1x_channel_dmaget_r()
+static inline u32 host1x_channel_dmaend_r(void)
+{
+	return 0x20;
+}
+#define HOST1X_CHANNEL_DMAEND \
+	host1x_channel_dmaend_r()
+static inline u32 host1x_channel_dmactrl_r(void)
+{
+	return 0x24;
+}
+#define HOST1X_CHANNEL_DMACTRL \
+	host1x_channel_dmactrl_r()
+static inline u32 host1x_channel_dmactrl_dmastop(void)
+{
+	return 1 << 0;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMASTOP \
+	host1x_channel_dmactrl_dmastop()
+static inline u32 host1x_channel_dmactrl_dmastop_v(u32 r)
+{
+	return (r >> 0) & 0x1;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMASTOP_V(r) \
+	host1x_channel_dmactrl_dmastop_v(r)
+static inline u32 host1x_channel_dmactrl_dmagetrst(void)
+{
+	return 1 << 1;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMAGETRST \
+	host1x_channel_dmactrl_dmagetrst()
+static inline u32 host1x_channel_dmactrl_dmainitget(void)
+{
+	return 1 << 2;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMAINITGET \
+	host1x_channel_dmactrl_dmainitget()
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x05_sync.h b/drivers/gpu/host1x/hw/hw_host1x05_sync.h
new file mode 100644
index 000000000000..ca10eee5045c
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x05_sync.h
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X05_SYNC_H
+#define HOST1X_HW_HOST1X05_SYNC_H
+
+#define REGISTER_STRIDE	4
+
+static inline u32 host1x_sync_syncpt_r(unsigned int id)
+{
+	return 0xf80 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT(id) \
+	host1x_sync_syncpt_r(id)
+static inline u32 host1x_sync_syncpt_thresh_cpu0_int_status_r(unsigned int id)
+{
+	return 0xe80 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(id) \
+	host1x_sync_syncpt_thresh_cpu0_int_status_r(id)
+static inline u32 host1x_sync_syncpt_thresh_int_disable_r(unsigned int id)
+{
+	return 0xf00 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(id) \
+	host1x_sync_syncpt_thresh_int_disable_r(id)
+static inline u32 host1x_sync_syncpt_thresh_int_enable_cpu0_r(unsigned int id)
+{
+	return 0xf20 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(id) \
+	host1x_sync_syncpt_thresh_int_enable_cpu0_r(id)
+static inline u32 host1x_sync_cf_setup_r(unsigned int channel)
+{
+	return 0xc00 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CF_SETUP(channel) \
+	host1x_sync_cf_setup_r(channel)
+static inline u32 host1x_sync_cf_setup_base_v(u32 r)
+{
+	return (r >> 0) & 0x3ff;
+}
+#define HOST1X_SYNC_CF_SETUP_BASE_V(r) \
+	host1x_sync_cf_setup_base_v(r)
+static inline u32 host1x_sync_cf_setup_limit_v(u32 r)
+{
+	return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CF_SETUP_LIMIT_V(r) \
+	host1x_sync_cf_setup_limit_v(r)
+static inline u32 host1x_sync_cmdproc_stop_r(void)
+{
+	return 0xac;
+}
+#define HOST1X_SYNC_CMDPROC_STOP \
+	host1x_sync_cmdproc_stop_r()
+static inline u32 host1x_sync_ch_teardown_r(void)
+{
+	return 0xb0;
+}
+#define HOST1X_SYNC_CH_TEARDOWN \
+	host1x_sync_ch_teardown_r()
+static inline u32 host1x_sync_usec_clk_r(void)
+{
+	return 0x1a4;
+}
+#define HOST1X_SYNC_USEC_CLK \
+	host1x_sync_usec_clk_r()
+static inline u32 host1x_sync_ctxsw_timeout_cfg_r(void)
+{
+	return 0x1a8;
+}
+#define HOST1X_SYNC_CTXSW_TIMEOUT_CFG \
+	host1x_sync_ctxsw_timeout_cfg_r()
+static inline u32 host1x_sync_ip_busy_timeout_r(void)
+{
+	return 0x1bc;
+}
+#define HOST1X_SYNC_IP_BUSY_TIMEOUT \
+	host1x_sync_ip_busy_timeout_r()
+static inline u32 host1x_sync_mlock_owner_r(unsigned int id)
+{
+	return 0x340 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_MLOCK_OWNER(id) \
+	host1x_sync_mlock_owner_r(id)
+static inline u32 host1x_sync_mlock_owner_chid_v(u32 r)
+{
+	return (r >> 8) & 0xf;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CHID_V(v) \
+	host1x_sync_mlock_owner_chid_v(v)
+static inline u32 host1x_sync_mlock_owner_cpu_owns_v(u32 r)
+{
+	return (r >> 1) & 0x1;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CPU_OWNS_V(r) \
+	host1x_sync_mlock_owner_cpu_owns_v(r)
+static inline u32 host1x_sync_mlock_owner_ch_owns_v(u32 r)
+{
+	return (r >> 0) & 0x1;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CH_OWNS_V(r) \
+	host1x_sync_mlock_owner_ch_owns_v(r)
+static inline u32 host1x_sync_syncpt_int_thresh_r(unsigned int id)
+{
+	return 0x1380 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_INT_THRESH(id) \
+	host1x_sync_syncpt_int_thresh_r(id)
+static inline u32 host1x_sync_syncpt_base_r(unsigned int id)
+{
+	return 0x600 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_BASE(id) \
+	host1x_sync_syncpt_base_r(id)
+static inline u32 host1x_sync_syncpt_cpu_incr_r(unsigned int id)
+{
+	return 0xf60 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_CPU_INCR(id) \
+	host1x_sync_syncpt_cpu_incr_r(id)
+static inline u32 host1x_sync_cbread_r(unsigned int channel)
+{
+	return 0xc80 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CBREAD(channel) \
+	host1x_sync_cbread_r(channel)
+static inline u32 host1x_sync_cfpeek_ctrl_r(void)
+{
+	return 0x74c;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL \
+	host1x_sync_cfpeek_ctrl_r()
+static inline u32 host1x_sync_cfpeek_ctrl_addr_f(u32 v)
+{
+	return (v & 0x3ff) << 0;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_ADDR_F(v) \
+	host1x_sync_cfpeek_ctrl_addr_f(v)
+static inline u32 host1x_sync_cfpeek_ctrl_channr_f(u32 v)
+{
+	return (v & 0xf) << 16;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_CHANNR_F(v) \
+	host1x_sync_cfpeek_ctrl_channr_f(v)
+static inline u32 host1x_sync_cfpeek_ctrl_ena_f(u32 v)
+{
+	return (v & 0x1) << 31;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_ENA_F(v) \
+	host1x_sync_cfpeek_ctrl_ena_f(v)
+static inline u32 host1x_sync_cfpeek_read_r(void)
+{
+	return 0x750;
+}
+#define HOST1X_SYNC_CFPEEK_READ \
+	host1x_sync_cfpeek_read_r()
+static inline u32 host1x_sync_cfpeek_ptrs_r(void)
+{
+	return 0x754;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS \
+	host1x_sync_cfpeek_ptrs_r()
+static inline u32 host1x_sync_cfpeek_ptrs_cf_rd_ptr_v(u32 r)
+{
+	return (r >> 0) & 0x3ff;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS_CF_RD_PTR_V(r) \
+	host1x_sync_cfpeek_ptrs_cf_rd_ptr_v(r)
+static inline u32 host1x_sync_cfpeek_ptrs_cf_wr_ptr_v(u32 r)
+{
+	return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS_CF_WR_PTR_V(r) \
+	host1x_sync_cfpeek_ptrs_cf_wr_ptr_v(r)
+static inline u32 host1x_sync_cbstat_r(unsigned int channel)
+{
+	return 0xcc0 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CBSTAT(channel) \
+	host1x_sync_cbstat_r(channel)
+static inline u32 host1x_sync_cbstat_cboffset_v(u32 r)
+{
+	return (r >> 0) & 0xffff;
+}
+#define HOST1X_SYNC_CBSTAT_CBOFFSET_V(r) \
+	host1x_sync_cbstat_cboffset_v(r)
+static inline u32 host1x_sync_cbstat_cbclass_v(u32 r)
+{
+	return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CBSTAT_CBCLASS_V(r) \
+	host1x_sync_cbstat_cbclass_v(r)
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x05_uclass.h b/drivers/gpu/host1x/hw/hw_host1x05_uclass.h
new file mode 100644
index 000000000000..0c411da6bc41
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x05_uclass.h
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2015 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X05_UCLASS_H
+#define HOST1X_HW_HOST1X05_UCLASS_H
+
+static inline u32 host1x_uclass_incr_syncpt_r(void)
+{
+	return 0x0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT \
+	host1x_uclass_incr_syncpt_r()
+static inline u32 host1x_uclass_incr_syncpt_cond_f(u32 v)
+{
+	return (v & 0xff) << 8;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_COND_F(v) \
+	host1x_uclass_incr_syncpt_cond_f(v)
+static inline u32 host1x_uclass_incr_syncpt_indx_f(u32 v)
+{
+	return (v & 0xff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_INDX_F(v) \
+	host1x_uclass_incr_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_r(void)
+{
+	return 0x8;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT \
+	host1x_uclass_wait_syncpt_r()
+static inline u32 host1x_uclass_wait_syncpt_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_INDX_F(v) \
+	host1x_uclass_wait_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_thresh_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_THRESH_F(v) \
+	host1x_uclass_wait_syncpt_thresh_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_r(void)
+{
+	return 0x9;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE \
+	host1x_uclass_wait_syncpt_base_r()
+static inline u32 host1x_uclass_wait_syncpt_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_INDX_F(v) \
+	host1x_uclass_wait_syncpt_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 16;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_wait_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_offset_f(u32 v)
+{
+	return (v & 0xffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_OFFSET_F(v) \
+	host1x_uclass_wait_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_r(void)
+{
+	return 0xb;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE \
+	host1x_uclass_load_syncpt_base_r()
+static inline u32 host1x_uclass_load_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_load_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_value_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(v) \
+	host1x_uclass_load_syncpt_base_value_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_incr_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_offset_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_OFFSET_F(v) \
+	host1x_uclass_incr_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_indoff_r(void)
+{
+	return 0x2d;
+}
+#define HOST1X_UCLASS_INDOFF \
+	host1x_uclass_indoff_r()
+static inline u32 host1x_uclass_indoff_indbe_f(u32 v)
+{
+	return (v & 0xf) << 28;
+}
+#define HOST1X_UCLASS_INDOFF_INDBE_F(v) \
+	host1x_uclass_indoff_indbe_f(v)
+static inline u32 host1x_uclass_indoff_autoinc_f(u32 v)
+{
+	return (v & 0x1) << 27;
+}
+#define HOST1X_UCLASS_INDOFF_AUTOINC_F(v) \
+	host1x_uclass_indoff_autoinc_f(v)
+static inline u32 host1x_uclass_indoff_indmodid_f(u32 v)
+{
+	return (v & 0xff) << 18;
+}
+#define HOST1X_UCLASS_INDOFF_INDMODID_F(v) \
+	host1x_uclass_indoff_indmodid_f(v)
+static inline u32 host1x_uclass_indoff_indroffset_f(u32 v)
+{
+	return (v & 0xffff) << 2;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+	host1x_uclass_indoff_indroffset_f(v)
+static inline u32 host1x_uclass_indoff_rwn_read_v(void)
+{
+	return 1;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+	host1x_uclass_indoff_indroffset_f(v)
+
+#endif
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index 41edd5a3f100..665ab9fd0e01 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -36,6 +36,7 @@
 #include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
@@ -63,7 +64,7 @@
  * for the inactive GPU.) Also, muxes are often used to cut power to the
  * discrete GPU while it is not used.
  *
- * DRM drivers register GPUs with vga_switcheroo, these are heretoforth called
+ * DRM drivers register GPUs with vga_switcheroo, these are henceforth called
  * clients. The mux is called the handler. Muxless machines also register a
  * handler to control the power state of the discrete GPU, its ->switchto
  * callback is a no-op for obvious reasons. The discrete GPU is often equipped
@@ -918,17 +919,17 @@ int vga_switcheroo_init_domain_pm_ops(struct device *dev,
 		domain->ops.runtime_suspend = vga_switcheroo_runtime_suspend;
 		domain->ops.runtime_resume = vga_switcheroo_runtime_resume;
 
-		dev->pm_domain = domain;
+		dev_pm_domain_set(dev, domain);
 		return 0;
 	}
-	dev->pm_domain = NULL;
+	dev_pm_domain_set(dev, NULL);
 	return -EINVAL;
 }
 EXPORT_SYMBOL(vga_switcheroo_init_domain_pm_ops);
 
 void vga_switcheroo_fini_domain_pm_ops(struct device *dev)
 {
-	dev->pm_domain = NULL;
+	dev_pm_domain_set(dev, NULL);
 }
 EXPORT_SYMBOL(vga_switcheroo_fini_domain_pm_ops);
 
@@ -989,10 +990,10 @@ vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev,
 		domain->ops.runtime_resume =
 			vga_switcheroo_runtime_resume_hdmi_audio;
 
-		dev->pm_domain = domain;
+		dev_pm_domain_set(dev, domain);
 		return 0;
 	}
-	dev->pm_domain = NULL;
+	dev_pm_domain_set(dev, NULL);
 	return -EINVAL;
 }
 EXPORT_SYMBOL(vga_switcheroo_init_domain_pm_optimus_hdmi_audio);
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index c6f7a694f67a..7e89288b1537 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -625,7 +625,7 @@ static void hid_close_report(struct hid_device *device)
 
 static void hid_device_release(struct device *dev)
 {
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 
 	hid_close_report(hid);
 	kfree(hid->dev_rdesc);
@@ -1571,8 +1571,8 @@ read_report_descriptor(struct file *filp, struct kobject *kobj,
 		struct bin_attribute *attr,
 		char *buf, loff_t off, size_t count)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct device *dev = kobj_to_dev(kobj);
+	struct hid_device *hdev = to_hid_device(dev);
 
 	if (off >= hdev->rsize)
 		return 0;
@@ -1589,7 +1589,7 @@ static ssize_t
 show_country(struct device *dev, struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 
 	return sprintf(buf, "%02x\n", hdev->country & 0xff);
 }
@@ -1691,11 +1691,6 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask)
 		hid_warn(hdev,
 			 "can't create sysfs country code attribute err: %d\n", ret);
 
-	ret = device_create_bin_file(&hdev->dev, &dev_bin_attr_report_desc);
-	if (ret)
-		hid_warn(hdev,
-			 "can't create sysfs report descriptor attribute err: %d\n", ret);
-
 	hid_info(hdev, "%s: %s HID v%x.%02x %s [%s] on %s\n",
 		 buf, bus, hdev->version >> 8, hdev->version & 0xff,
 		 type, hdev->name, hdev->phys);
@@ -1707,7 +1702,6 @@ EXPORT_SYMBOL_GPL(hid_connect);
 void hid_disconnect(struct hid_device *hdev)
 {
 	device_remove_file(&hdev->dev, &dev_attr_country);
-	device_remove_bin_file(&hdev->dev, &dev_bin_attr_report_desc);
 	if (hdev->claimed & HID_CLAIMED_INPUT)
 		hidinput_disconnect(hdev);
 	if (hdev->claimed & HID_CLAIMED_HIDDEV)
@@ -1902,6 +1896,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2_2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G29_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G920_WHEEL) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_F3D) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_FFG ) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FORCE3D_PRO) },
@@ -2076,7 +2071,7 @@ struct hid_dynid {
 static ssize_t store_new_id(struct device_driver *drv, const char *buf,
 		size_t count)
 {
-	struct hid_driver *hdrv = container_of(drv, struct hid_driver, driver);
+	struct hid_driver *hdrv = to_hid_driver(drv);
 	struct hid_dynid *dynid;
 	__u32 bus, vendor, product;
 	unsigned long driver_data = 0;
@@ -2138,17 +2133,16 @@ static const struct hid_device_id *hid_match_device(struct hid_device *hdev,
 
 static int hid_bus_match(struct device *dev, struct device_driver *drv)
 {
-	struct hid_driver *hdrv = container_of(drv, struct hid_driver, driver);
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_driver *hdrv = to_hid_driver(drv);
+	struct hid_device *hdev = to_hid_device(dev);
 
 	return hid_match_device(hdev, hdrv) != NULL;
 }
 
 static int hid_device_probe(struct device *dev)
 {
-	struct hid_driver *hdrv = container_of(dev->driver,
-			struct hid_driver, driver);
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_driver *hdrv = to_hid_driver(dev->driver);
+	struct hid_device *hdev = to_hid_device(dev);
 	const struct hid_device_id *id;
 	int ret = 0;
 
@@ -2190,7 +2184,7 @@ unlock_driver_lock:
 
 static int hid_device_remove(struct device *dev)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct hid_driver *hdrv;
 	int ret = 0;
 
@@ -2223,12 +2217,9 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *a,
 			     char *buf)
 {
 	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
-	int len;
-
-	len = snprintf(buf, PAGE_SIZE, "hid:b%04Xg%04Xv%08Xp%08X\n",
-		       hdev->bus, hdev->group, hdev->vendor, hdev->product);
 
-	return (len >= PAGE_SIZE) ? (PAGE_SIZE - 1) : len;
+	return scnprintf(buf, PAGE_SIZE, "hid:b%04Xg%04Xv%08Xp%08X\n",
+			 hdev->bus, hdev->group, hdev->vendor, hdev->product);
 }
 static DEVICE_ATTR_RO(modalias);
 
@@ -2236,11 +2227,19 @@ static struct attribute *hid_dev_attrs[] = {
 	&dev_attr_modalias.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(hid_dev);
+static struct bin_attribute *hid_dev_bin_attrs[] = {
+	&dev_bin_attr_report_desc,
+	NULL
+};
+static const struct attribute_group hid_dev_group = {
+	.attrs = hid_dev_attrs,
+	.bin_attrs = hid_dev_bin_attrs,
+};
+__ATTRIBUTE_GROUPS(hid_dev);
 
 static int hid_uevent(struct device *dev, struct kobj_uevent_env *env)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);	
 
 	if (add_uevent_var(env, "HID_ID=%04X:%08X:%08X",
 			hdev->bus, hdev->vendor, hdev->product))
@@ -2408,6 +2407,7 @@ static const struct hid_device_id hid_ignore_list[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT1) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454_V2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_NATIONAL_SEMICONDUCTOR, USB_DEVICE_ID_N_S_HARMONY) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 20) },
@@ -2660,6 +2660,7 @@ struct hid_device *hid_allocate_device(void)
 	device_initialize(&hdev->dev);
 	hdev->dev.release = hid_device_release;
 	hdev->dev.bus = &hid_bus_type;
+	device_enable_async_suspend(&hdev->dev);
 
 	hid_close_report(hdev);
 
diff --git a/drivers/hid/hid-corsair.c b/drivers/hid/hid-corsair.c
index bcefb9ebb026..58551964ce86 100644
--- a/drivers/hid/hid-corsair.c
+++ b/drivers/hid/hid-corsair.c
@@ -655,18 +655,7 @@ static struct hid_driver corsair_driver = {
 	.input_mapping = corsair_input_mapping,
 };
 
-static int __init corsair_init(void)
-{
-	return hid_register_driver(&corsair_driver);
-}
-
-static void corsair_exit(void)
-{
-	hid_unregister_driver(&corsair_driver);
-}
-
-module_init(corsair_init);
-module_exit(corsair_exit);
+module_hid_driver(corsair_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Clement Vuchener");
diff --git a/drivers/hid/hid-cp2112.c b/drivers/hid/hid-cp2112.c
index 7afc3fcc122c..086d8a507157 100644
--- a/drivers/hid/hid-cp2112.c
+++ b/drivers/hid/hid-cp2112.c
@@ -24,7 +24,7 @@
  *   http://www.silabs.com/Support%20Documents/TechnicalDocs/AN495.pdf
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/hid.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
@@ -169,8 +169,7 @@ MODULE_PARM_DESC(gpio_push_pull, "GPIO push-pull configuration bitmask");
 
 static int cp2112_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct cp2112_device *dev = container_of(chip, struct cp2112_device,
-						 gc);
+	struct cp2112_device *dev = gpiochip_get_data(chip);
 	struct hid_device *hdev = dev->hdev;
 	u8 buf[5];
 	int ret;
@@ -198,8 +197,7 @@ static int cp2112_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct cp2112_device *dev = container_of(chip, struct cp2112_device,
-						 gc);
+	struct cp2112_device *dev = gpiochip_get_data(chip);
 	struct hid_device *hdev = dev->hdev;
 	u8 buf[3];
 	int ret;
@@ -216,8 +214,7 @@ static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct cp2112_device *dev = container_of(chip, struct cp2112_device,
-						 gc);
+	struct cp2112_device *dev = gpiochip_get_data(chip);
 	struct hid_device *hdev = dev->hdev;
 	u8 buf[2];
 	int ret;
@@ -235,8 +232,7 @@ static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset)
 static int cp2112_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset, int value)
 {
-	struct cp2112_device *dev = container_of(chip, struct cp2112_device,
-						 gc);
+	struct cp2112_device *dev = gpiochip_get_data(chip);
 	struct hid_device *hdev = dev->hdev;
 	u8 buf[5];
 	int ret;
@@ -807,7 +803,7 @@ static ssize_t name##_store(struct device *kdev, \
 			    struct device_attribute *attr, const char *buf, \
 			    size_t count) \
 { \
-	struct hid_device *hdev = container_of(kdev, struct hid_device, dev); \
+	struct hid_device *hdev = to_hid_device(kdev); \
 	struct cp2112_usb_config_report cfg; \
 	int ret = cp2112_get_usb_config(hdev, &cfg); \
 	if (ret) \
@@ -822,7 +818,7 @@ static ssize_t name##_store(struct device *kdev, \
 static ssize_t name##_show(struct device *kdev, \
 			   struct device_attribute *attr, char *buf) \
 { \
-	struct hid_device *hdev = container_of(kdev, struct hid_device, dev); \
+	struct hid_device *hdev = to_hid_device(kdev); \
 	struct cp2112_usb_config_report cfg; \
 	int ret = cp2112_get_usb_config(hdev, &cfg); \
 	if (ret) \
@@ -887,7 +883,7 @@ static ssize_t pstr_store(struct device *kdev,
 			  struct device_attribute *kattr, const char *buf,
 			  size_t count)
 {
-	struct hid_device *hdev = container_of(kdev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(kdev);
 	struct cp2112_pstring_attribute *attr =
 		container_of(kattr, struct cp2112_pstring_attribute, attr);
 	struct cp2112_string_report report;
@@ -918,7 +914,7 @@ static ssize_t pstr_store(struct device *kdev,
 static ssize_t pstr_show(struct device *kdev,
 			 struct device_attribute *kattr, char *buf)
 {
-	struct hid_device *hdev = container_of(kdev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(kdev);
 	struct cp2112_pstring_attribute *attr =
 		container_of(kattr, struct cp2112_pstring_attribute, attr);
 	struct cp2112_string_report report;
@@ -1104,9 +1100,9 @@ static int cp2112_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	dev->gc.base			= -1;
 	dev->gc.ngpio			= 8;
 	dev->gc.can_sleep		= 1;
-	dev->gc.dev			= &hdev->dev;
+	dev->gc.parent			= &hdev->dev;
 
-	ret = gpiochip_add(&dev->gc);
+	ret = gpiochip_add_data(&dev->gc, dev);
 	if (ret < 0) {
 		hid_err(hdev, "error registering gpio chip\n");
 		goto err_free_i2c;
diff --git a/drivers/hid/hid-debug.c b/drivers/hid/hid-debug.c
index 2886b645ced7..acfb522a432a 100644
--- a/drivers/hid/hid-debug.c
+++ b/drivers/hid/hid-debug.c
@@ -659,13 +659,13 @@ EXPORT_SYMBOL_GPL(hid_dump_device);
 /* enqueue string to 'events' ring buffer */
 void hid_debug_event(struct hid_device *hdev, char *buf)
 {
-	int i;
+	unsigned i;
 	struct hid_debug_list *list;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hdev->debug_list_lock, flags);
 	list_for_each_entry(list, &hdev->debug_list, node) {
-		for (i = 0; i < strlen(buf); i++)
+		for (i = 0; buf[i]; i++)
 			list->hid_debug_buf[(list->tail + i) % HID_DEBUG_BUFSIZE] =
 				buf[i];
 		list->tail = (list->tail + i) % HID_DEBUG_BUFSIZE;
diff --git a/drivers/hid/hid-gt683r.c b/drivers/hid/hid-gt683r.c
index 0d6f135e266c..a298fbd8db6b 100644
--- a/drivers/hid/hid-gt683r.c
+++ b/drivers/hid/hid-gt683r.c
@@ -70,7 +70,7 @@ static void gt683r_brightness_set(struct led_classdev *led_cdev,
 {
 	int i;
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct gt683r_led *led = hid_get_drvdata(hdev);
 
 	for (i = 0; i < GT683R_LED_COUNT; i++) {
@@ -89,8 +89,7 @@ static ssize_t mode_show(struct device *dev,
 				char *buf)
 {
 	u8 sysfs_mode;
-	struct hid_device *hdev = container_of(dev->parent,
-					struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev->parent);
 	struct gt683r_led *led = hid_get_drvdata(hdev);
 
 	if (led->mode == GT683R_LED_NORMAL)
@@ -108,8 +107,7 @@ static ssize_t mode_store(struct device *dev,
 				const char *buf, size_t count)
 {
 	u8 sysfs_mode;
-	struct hid_device *hdev = container_of(dev->parent,
-					struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev->parent);
 	struct gt683r_led *led = hid_get_drvdata(hdev);
 
 
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 8b78a7f1f779..b6ff6e78ac54 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -510,6 +510,7 @@
 #define USB_VENDOR_ID_ITE               0x048d
 #define USB_DEVICE_ID_ITE_LENOVO_YOGA   0x8386
 #define USB_DEVICE_ID_ITE_LENOVO_YOGA2  0x8350
+#define USB_DEVICE_ID_ITE_LENOVO_YOGA900	0x8396
 
 #define USB_VENDOR_ID_JABRA		0x0b0e
 #define USB_DEVICE_ID_JABRA_SPEAK_410	0x0412
@@ -615,6 +616,7 @@
 #define USB_DEVICE_ID_LOGITECH_RUMBLEPAD2	0xc218
 #define USB_DEVICE_ID_LOGITECH_RUMBLEPAD2_2	0xc219
 #define USB_DEVICE_ID_LOGITECH_G29_WHEEL	0xc24f
+#define USB_DEVICE_ID_LOGITECH_G920_WHEEL	0xc262
 #define USB_DEVICE_ID_LOGITECH_WINGMAN_F3D	0xc283
 #define USB_DEVICE_ID_LOGITECH_FORCE3D_PRO	0xc286
 #define USB_DEVICE_ID_LOGITECH_FLIGHT_SYSTEM_G940	0xc287
@@ -664,6 +666,7 @@
 #define USB_DEVICE_ID_PICOLCD		0xc002
 #define USB_DEVICE_ID_PICOLCD_BOOTLOADER	0xf002
 #define USB_DEVICE_ID_PICK16F1454	0x0042
+#define USB_DEVICE_ID_PICK16F1454_V2	0xf2f7
 
 #define USB_VENDOR_ID_MICROSOFT		0x045e
 #define USB_DEVICE_ID_SIDEWINDER_GV	0x003b
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index 2ba6bf69b7d0..bcfaf32d9e5e 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -303,6 +303,7 @@ static enum power_supply_property hidinput_battery_props[] = {
 
 #define HID_BATTERY_QUIRK_PERCENT	(1 << 0) /* always reports percent */
 #define HID_BATTERY_QUIRK_FEATURE	(1 << 1) /* ask for feature report */
+#define HID_BATTERY_QUIRK_IGNORE	(1 << 2) /* completely ignore the battery */
 
 static const struct hid_device_id hid_battery_quirks[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
@@ -320,6 +321,9 @@ static const struct hid_device_id hid_battery_quirks[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
 		USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI),
 	  HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM,
+		USB_DEVICE_ID_ELECOM_BM084),
+	  HID_BATTERY_QUIRK_IGNORE },
 	{}
 };
 
@@ -408,6 +412,14 @@ static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 	if (dev->battery != NULL)
 		goto out;	/* already initialized? */
 
+	quirks = find_battery_quirk(dev);
+
+	hid_dbg(dev, "device %x:%x:%x %d quirks %d\n",
+		dev->bus, dev->vendor, dev->product, dev->version, quirks);
+
+	if (quirks & HID_BATTERY_QUIRK_IGNORE)
+		goto out;
+
 	psy_desc = kzalloc(sizeof(*psy_desc), GFP_KERNEL);
 	if (psy_desc == NULL)
 		goto out;
@@ -424,11 +436,6 @@ static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 	psy_desc->use_for_apm = 0;
 	psy_desc->get_property = hidinput_get_battery_property;
 
-	quirks = find_battery_quirk(dev);
-
-	hid_dbg(dev, "device %x:%x:%x %d quirks %d\n",
-		dev->bus, dev->vendor, dev->product, dev->version, quirks);
-
 	min = field->logical_minimum;
 	max = field->logical_maximum;
 
@@ -960,6 +967,10 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 		goto ignore;
 
 	case HID_UP_LOGIVENDOR:
+		/* intentional fallback */
+	case HID_UP_LOGIVENDOR2:
+		/* intentional fallback */
+	case HID_UP_LOGIVENDOR3:
 		goto ignore;
 
 	case HID_UP_PID:
diff --git a/drivers/hid/hid-lenovo.c b/drivers/hid/hid-lenovo.c
index 8979f1fd5208..0125e356bd8d 100644
--- a/drivers/hid/hid-lenovo.c
+++ b/drivers/hid/hid-lenovo.c
@@ -220,7 +220,7 @@ static ssize_t attr_fn_lock_show_cptkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_cptkbd *cptkbd_data = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", cptkbd_data->fn_lock);
@@ -231,7 +231,7 @@ static ssize_t attr_fn_lock_store_cptkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_cptkbd *cptkbd_data = hid_get_drvdata(hdev);
 	int value;
 
@@ -250,7 +250,7 @@ static ssize_t attr_sensitivity_show_cptkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_cptkbd *cptkbd_data = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n",
@@ -262,7 +262,7 @@ static ssize_t attr_sensitivity_store_cptkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_cptkbd *cptkbd_data = hid_get_drvdata(hdev);
 	int value;
 
@@ -387,7 +387,7 @@ static ssize_t attr_press_to_select_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", data_pointer->press_to_select);
@@ -398,7 +398,7 @@ static ssize_t attr_press_to_select_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -417,7 +417,7 @@ static ssize_t attr_dragging_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", data_pointer->dragging);
@@ -428,7 +428,7 @@ static ssize_t attr_dragging_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -447,7 +447,7 @@ static ssize_t attr_release_to_select_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", data_pointer->release_to_select);
@@ -458,7 +458,7 @@ static ssize_t attr_release_to_select_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -477,7 +477,7 @@ static ssize_t attr_select_right_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", data_pointer->select_right);
@@ -488,7 +488,7 @@ static ssize_t attr_select_right_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -507,7 +507,7 @@ static ssize_t attr_sensitivity_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n",
@@ -519,7 +519,7 @@ static ssize_t attr_sensitivity_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -536,7 +536,7 @@ static ssize_t attr_press_speed_show_tpkbd(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%u\n",
@@ -548,7 +548,7 @@ static ssize_t attr_press_speed_store_tpkbd(struct device *dev,
 		const char *buf,
 		size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int value;
 
@@ -609,7 +609,7 @@ static enum led_brightness lenovo_led_brightness_get_tpkbd(
 			struct led_classdev *led_cdev)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	int led_nr = 0;
 
@@ -625,7 +625,7 @@ static void lenovo_led_brightness_set_tpkbd(struct led_classdev *led_cdev,
 			enum led_brightness value)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct lenovo_drvdata_tpkbd *data_pointer = hid_get_drvdata(hdev);
 	struct hid_report *report;
 	int led_nr = 0;
diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c
index fbddcb37ae98..af3a8ec8a746 100644
--- a/drivers/hid/hid-lg4ff.c
+++ b/drivers/hid/hid-lg4ff.c
@@ -33,8 +33,6 @@
 #include "hid-lg4ff.h"
 #include "hid-ids.h"
 
-#define to_hid_device(pdev) container_of(pdev, struct hid_device, dev)
-
 #define LG4FF_MMODE_IS_MULTIMODE 0
 #define LG4FF_MMODE_SWITCHED 1
 #define LG4FF_MMODE_NOT_MULTIMODE 2
@@ -1020,7 +1018,7 @@ static void lg4ff_led_set_brightness(struct led_classdev *led_cdev,
 			enum led_brightness value)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct lg_drv_data *drv_data = hid_get_drvdata(hid);
 	struct lg4ff_device_entry *entry;
 	int i, state = 0;
@@ -1055,7 +1053,7 @@ static void lg4ff_led_set_brightness(struct led_classdev *led_cdev,
 static enum led_brightness lg4ff_led_get_brightness(struct led_classdev *led_cdev)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct lg_drv_data *drv_data = hid_get_drvdata(hid);
 	struct lg4ff_device_entry *entry;
 	int i, value = 0;
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index 5fd97860aec4..bd2ab476c65e 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -40,18 +40,22 @@ MODULE_PARM_DESC(disable_tap_to_click,
 
 #define REPORT_ID_HIDPP_SHORT			0x10
 #define REPORT_ID_HIDPP_LONG			0x11
+#define REPORT_ID_HIDPP_VERY_LONG		0x12
 
 #define HIDPP_REPORT_SHORT_LENGTH		7
 #define HIDPP_REPORT_LONG_LENGTH		20
+#define HIDPP_REPORT_VERY_LONG_LENGTH		64
 
 #define HIDPP_QUIRK_CLASS_WTP			BIT(0)
 #define HIDPP_QUIRK_CLASS_M560			BIT(1)
 #define HIDPP_QUIRK_CLASS_K400			BIT(2)
+#define HIDPP_QUIRK_CLASS_G920			BIT(3)
 
 /* bits 2..20 are reserved for classes */
 #define HIDPP_QUIRK_CONNECT_EVENTS		BIT(21)
 #define HIDPP_QUIRK_WTP_PHYSICAL_BUTTONS	BIT(22)
 #define HIDPP_QUIRK_NO_HIDINPUT			BIT(23)
+#define HIDPP_QUIRK_FORCE_OUTPUT_REPORTS	BIT(24)
 
 #define HIDPP_QUIRK_DELAYED_INIT		(HIDPP_QUIRK_NO_HIDINPUT | \
 						 HIDPP_QUIRK_CONNECT_EVENTS)
@@ -81,13 +85,13 @@ MODULE_PARM_DESC(disable_tap_to_click,
 struct fap {
 	u8 feature_index;
 	u8 funcindex_clientid;
-	u8 params[HIDPP_REPORT_LONG_LENGTH - 4U];
+	u8 params[HIDPP_REPORT_VERY_LONG_LENGTH - 4U];
 };
 
 struct rap {
 	u8 sub_id;
 	u8 reg_address;
-	u8 params[HIDPP_REPORT_LONG_LENGTH - 4U];
+	u8 params[HIDPP_REPORT_VERY_LONG_LENGTH - 4U];
 };
 
 struct hidpp_report {
@@ -144,8 +148,11 @@ static void hidpp_connect_event(struct hidpp_device *hidpp_dev);
 static int __hidpp_send_report(struct hid_device *hdev,
 				struct hidpp_report *hidpp_report)
 {
+	struct hidpp_device *hidpp = hid_get_drvdata(hdev);
 	int fields_count, ret;
 
+	hidpp = hid_get_drvdata(hdev);
+
 	switch (hidpp_report->report_id) {
 	case REPORT_ID_HIDPP_SHORT:
 		fields_count = HIDPP_REPORT_SHORT_LENGTH;
@@ -153,6 +160,9 @@ static int __hidpp_send_report(struct hid_device *hdev,
 	case REPORT_ID_HIDPP_LONG:
 		fields_count = HIDPP_REPORT_LONG_LENGTH;
 		break;
+	case REPORT_ID_HIDPP_VERY_LONG:
+		fields_count = HIDPP_REPORT_VERY_LONG_LENGTH;
+		break;
 	default:
 		return -ENODEV;
 	}
@@ -163,9 +173,13 @@ static int __hidpp_send_report(struct hid_device *hdev,
 	 */
 	hidpp_report->device_index = 0xff;
 
-	ret = hid_hw_raw_request(hdev, hidpp_report->report_id,
-		(u8 *)hidpp_report, fields_count, HID_OUTPUT_REPORT,
-		HID_REQ_SET_REPORT);
+	if (hidpp->quirks & HIDPP_QUIRK_FORCE_OUTPUT_REPORTS) {
+		ret = hid_hw_output_report(hdev, (u8 *)hidpp_report, fields_count);
+	} else {
+		ret = hid_hw_raw_request(hdev, hidpp_report->report_id,
+			(u8 *)hidpp_report, fields_count, HID_OUTPUT_REPORT,
+			HID_REQ_SET_REPORT);
+	}
 
 	return ret == fields_count ? 0 : -1;
 }
@@ -217,8 +231,9 @@ static int hidpp_send_message_sync(struct hidpp_device *hidpp,
 		goto exit;
 	}
 
-	if (response->report_id == REPORT_ID_HIDPP_LONG &&
-	    response->fap.feature_index == HIDPP20_ERROR) {
+	if ((response->report_id == REPORT_ID_HIDPP_LONG ||
+			response->report_id == REPORT_ID_HIDPP_VERY_LONG) &&
+			response->fap.feature_index == HIDPP20_ERROR) {
 		ret = response->fap.params[1];
 		dbg_hid("%s:got hidpp 2.0 error %02X\n", __func__, ret);
 		goto exit;
@@ -243,7 +258,11 @@ static int hidpp_send_fap_command_sync(struct hidpp_device *hidpp,
 	message = kzalloc(sizeof(struct hidpp_report), GFP_KERNEL);
 	if (!message)
 		return -ENOMEM;
-	message->report_id = REPORT_ID_HIDPP_LONG;
+
+	if (param_count > (HIDPP_REPORT_LONG_LENGTH - 4))
+		message->report_id = REPORT_ID_HIDPP_VERY_LONG;
+	else
+		message->report_id = REPORT_ID_HIDPP_LONG;
 	message->fap.feature_index = feat_index;
 	message->fap.funcindex_clientid = funcindex_clientid;
 	memcpy(&message->fap.params, params, param_count);
@@ -258,13 +277,23 @@ static int hidpp_send_rap_command_sync(struct hidpp_device *hidpp_dev,
 	struct hidpp_report *response)
 {
 	struct hidpp_report *message;
-	int ret;
+	int ret, max_count;
 
-	if ((report_id != REPORT_ID_HIDPP_SHORT) &&
-	    (report_id != REPORT_ID_HIDPP_LONG))
+	switch (report_id) {
+	case REPORT_ID_HIDPP_SHORT:
+		max_count = HIDPP_REPORT_SHORT_LENGTH - 4;
+		break;
+	case REPORT_ID_HIDPP_LONG:
+		max_count = HIDPP_REPORT_LONG_LENGTH - 4;
+		break;
+	case REPORT_ID_HIDPP_VERY_LONG:
+		max_count = HIDPP_REPORT_VERY_LONG_LENGTH - 4;
+		break;
+	default:
 		return -EINVAL;
+	}
 
-	if (param_count > sizeof(message->rap.params))
+	if (param_count > max_count)
 		return -EINVAL;
 
 	message = kzalloc(sizeof(struct hidpp_report), GFP_KERNEL);
@@ -508,10 +537,19 @@ static int hidpp_devicenametype_get_device_name(struct hidpp_device *hidpp,
 	if (ret)
 		return ret;
 
-	if (response.report_id == REPORT_ID_HIDPP_LONG)
+	switch (response.report_id) {
+	case REPORT_ID_HIDPP_VERY_LONG:
+		count = HIDPP_REPORT_VERY_LONG_LENGTH - 4;
+		break;
+	case REPORT_ID_HIDPP_LONG:
 		count = HIDPP_REPORT_LONG_LENGTH - 4;
-	else
+		break;
+	case REPORT_ID_HIDPP_SHORT:
 		count = HIDPP_REPORT_SHORT_LENGTH - 4;
+		break;
+	default:
+		return -EPROTO;
+	}
 
 	if (len_buf < count)
 		count = len_buf;
@@ -1257,6 +1295,131 @@ static int k400_connect(struct hid_device *hdev, bool connected)
 	return k400_disable_tap_to_click(hidpp);
 }
 
+/* ------------------------------------------------------------------------- */
+/* Logitech G920 Driving Force Racing Wheel for Xbox One                     */
+/* ------------------------------------------------------------------------- */
+
+#define HIDPP_PAGE_G920_FORCE_FEEDBACK			0x8123
+
+/* Using session ID = 1 */
+#define CMD_G920_FORCE_GET_APERTURE			0x51
+#define CMD_G920_FORCE_SET_APERTURE			0x61
+
+struct g920_private_data {
+	u8 force_feature;
+	u16 range;
+};
+
+static ssize_t g920_range_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct hid_device *hid = to_hid_device(dev);
+	struct hidpp_device *hidpp = hid_get_drvdata(hid);
+	struct g920_private_data *pdata;
+
+	pdata = hidpp->private_data;
+	if (!pdata) {
+		hid_err(hid, "Private driver data not found!\n");
+		return -EINVAL;
+	}
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", pdata->range);
+}
+
+static ssize_t g920_range_store(struct device *dev, struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct hid_device *hid = to_hid_device(dev);
+	struct hidpp_device *hidpp = hid_get_drvdata(hid);
+	struct g920_private_data *pdata;
+	struct hidpp_report response;
+	u8 params[2];
+	int ret;
+	u16 range = simple_strtoul(buf, NULL, 10);
+
+	pdata = hidpp->private_data;
+	if (!pdata) {
+		hid_err(hid, "Private driver data not found!\n");
+		return -EINVAL;
+	}
+
+	if (range < 180)
+		range = 180;
+	else if (range > 900)
+		range = 900;
+
+	params[0] = range >> 8;
+	params[1] = range & 0x00FF;
+
+	ret = hidpp_send_fap_command_sync(hidpp, pdata->force_feature,
+		CMD_G920_FORCE_SET_APERTURE, params, 2, &response);
+	if (ret)
+		return ret;
+
+	pdata->range = range;
+	return count;
+}
+
+static DEVICE_ATTR(range, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, g920_range_show, g920_range_store);
+
+static int g920_allocate(struct hid_device *hdev)
+{
+	struct hidpp_device *hidpp = hid_get_drvdata(hdev);
+	struct g920_private_data *pdata;
+
+	pdata = devm_kzalloc(&hdev->dev, sizeof(struct g920_private_data),
+			GFP_KERNEL);
+	if (!pdata)
+		return -ENOMEM;
+
+	hidpp->private_data = pdata;
+
+	return 0;
+}
+
+static int g920_get_config(struct hidpp_device *hidpp)
+{
+	struct g920_private_data *pdata = hidpp->private_data;
+	struct hidpp_report response;
+	u8 feature_type;
+	u8 feature_index;
+	int ret;
+
+	pdata = hidpp->private_data;
+	if (!pdata) {
+		hid_err(hidpp->hid_dev, "Private driver data not found!\n");
+		return -EINVAL;
+	}
+
+	/* Find feature and store for later use */
+	ret = hidpp_root_get_feature(hidpp, HIDPP_PAGE_G920_FORCE_FEEDBACK,
+		&feature_index, &feature_type);
+	if (ret)
+		return ret;
+
+	pdata->force_feature = feature_index;
+
+	/* Read current Range */
+	ret = hidpp_send_fap_command_sync(hidpp, feature_index,
+		CMD_G920_FORCE_GET_APERTURE, NULL, 0, &response);
+	if (ret > 0) {
+		hid_err(hidpp->hid_dev, "%s: received protocol error 0x%02x\n",
+			__func__, ret);
+		return -EPROTO;
+	}
+	if (ret)
+		return ret;
+
+	pdata->range = get_unaligned_be16(&response.fap.params[0]);
+
+	/* Create sysfs interface */
+	ret = device_create_file(&(hidpp->hid_dev->dev), &dev_attr_range);
+	if (ret)
+		hid_warn(hidpp->hid_dev, "Unable to create sysfs interface for \"range\", errno %d\n", ret);
+
+	return 0;
+}
+
 /* -------------------------------------------------------------------------- */
 /* Generic HID++ devices                                                      */
 /* -------------------------------------------------------------------------- */
@@ -1276,6 +1439,25 @@ static int hidpp_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 	return 0;
 }
 
+static int hidpp_input_mapped(struct hid_device *hdev, struct hid_input *hi,
+		struct hid_field *field, struct hid_usage *usage,
+		unsigned long **bit, int *max)
+{
+	struct hidpp_device *hidpp = hid_get_drvdata(hdev);
+
+	/* Ensure that Logitech G920 is not given a default fuzz/flat value */
+	if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) {
+		if (usage->type == EV_ABS && (usage->code == ABS_X ||
+				usage->code == ABS_Y || usage->code == ABS_Z ||
+				usage->code == ABS_RZ)) {
+			field->application = HID_GD_MULTIAXIS;
+		}
+	}
+
+	return 0;
+}
+
+
 static void hidpp_populate_input(struct hidpp_device *hidpp,
 		struct input_dev *input, bool origin_is_hid_core)
 {
@@ -1347,6 +1529,14 @@ static int hidpp_raw_event(struct hid_device *hdev, struct hid_report *report,
 
 	/* Generic HID++ processing. */
 	switch (data[0]) {
+	case REPORT_ID_HIDPP_VERY_LONG:
+		if (size != HIDPP_REPORT_VERY_LONG_LENGTH) {
+			hid_err(hdev, "received hid++ report of bad size (%d)",
+				size);
+			return 1;
+		}
+		ret = hidpp_raw_hidpp_event(hidpp, data, size);
+		break;
 	case REPORT_ID_HIDPP_LONG:
 		if (size != HIDPP_REPORT_LONG_LENGTH) {
 			hid_err(hdev, "received hid++ report of bad size (%d)",
@@ -1393,10 +1583,12 @@ static void hidpp_overwrite_name(struct hid_device *hdev, bool use_unifying)
 	else
 		name = hidpp_get_device_name(hidpp);
 
-	if (!name)
+	if (!name) {
 		hid_err(hdev, "unable to retrieve the name of the device");
-	else
+	} else {
+		dbg_hid("HID++: Got name: %s\n", name);
 		snprintf(hdev->name, sizeof(hdev->name), "%s", name);
+	}
 
 	kfree(name);
 }
@@ -1547,6 +1739,10 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		ret = k400_allocate(hdev);
 		if (ret)
 			goto allocate_fail;
+	} else if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) {
+		ret = g920_allocate(hdev);
+		if (ret)
+			goto allocate_fail;
 	}
 
 	INIT_WORK(&hidpp->work, delayed_work_cb);
@@ -1559,6 +1755,25 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		goto hid_parse_fail;
 	}
 
+	if (hidpp->quirks & HIDPP_QUIRK_NO_HIDINPUT)
+		connect_mask &= ~HID_CONNECT_HIDINPUT;
+
+	if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) {
+		ret = hid_hw_start(hdev, connect_mask);
+		if (ret) {
+			hid_err(hdev, "hw start failed\n");
+			goto hid_hw_start_fail;
+		}
+		ret = hid_hw_open(hdev);
+		if (ret < 0) {
+			dev_err(&hdev->dev, "%s:hid_hw_open returned error:%d\n",
+				__func__, ret);
+			hid_hw_stop(hdev);
+			goto hid_hw_start_fail;
+		}
+	}
+
+
 	/* Allow incoming packets */
 	hid_device_io_start(hdev);
 
@@ -1567,8 +1782,7 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		if (!connected) {
 			ret = -ENODEV;
 			hid_err(hdev, "Device not connected");
-			hid_device_io_stop(hdev);
-			goto hid_parse_fail;
+			goto hid_hw_open_failed;
 		}
 
 		hid_info(hdev, "HID++ %u.%u device connected.\n",
@@ -1581,19 +1795,22 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	if (connected && (hidpp->quirks & HIDPP_QUIRK_CLASS_WTP)) {
 		ret = wtp_get_config(hidpp);
 		if (ret)
-			goto hid_parse_fail;
+			goto hid_hw_open_failed;
+	} else if (connected && (hidpp->quirks & HIDPP_QUIRK_CLASS_G920)) {
+		ret = g920_get_config(hidpp);
+		if (ret)
+			goto hid_hw_open_failed;
 	}
 
 	/* Block incoming packets */
 	hid_device_io_stop(hdev);
 
-	if (hidpp->quirks & HIDPP_QUIRK_NO_HIDINPUT)
-		connect_mask &= ~HID_CONNECT_HIDINPUT;
-
-	ret = hid_hw_start(hdev, connect_mask);
-	if (ret) {
-		hid_err(hdev, "%s:hid_hw_start returned error\n", __func__);
-		goto hid_hw_start_fail;
+	if (!(hidpp->quirks & HIDPP_QUIRK_CLASS_G920)) {
+		ret = hid_hw_start(hdev, connect_mask);
+		if (ret) {
+			hid_err(hdev, "%s:hid_hw_start returned error\n", __func__);
+			goto hid_hw_start_fail;
+		}
 	}
 
 	if (hidpp->quirks & HIDPP_QUIRK_CONNECT_EVENTS) {
@@ -1605,6 +1822,13 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 	return ret;
 
+hid_hw_open_failed:
+	hid_device_io_stop(hdev);
+	if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) {
+		device_remove_file(&hdev->dev, &dev_attr_range);
+		hid_hw_close(hdev);
+		hid_hw_stop(hdev);
+	}
 hid_hw_start_fail:
 hid_parse_fail:
 	cancel_work_sync(&hidpp->work);
@@ -1618,9 +1842,13 @@ static void hidpp_remove(struct hid_device *hdev)
 {
 	struct hidpp_device *hidpp = hid_get_drvdata(hdev);
 
+	if (hidpp->quirks & HIDPP_QUIRK_CLASS_G920) {
+		device_remove_file(&hdev->dev, &dev_attr_range);
+		hid_hw_close(hdev);
+	}
+	hid_hw_stop(hdev);
 	cancel_work_sync(&hidpp->work);
 	mutex_destroy(&hidpp->send_mutex);
-	hid_hw_stop(hdev);
 }
 
 static const struct hid_device_id hidpp_devices[] = {
@@ -1648,6 +1876,9 @@ static const struct hid_device_id hidpp_devices[] = {
 
 	{ HID_DEVICE(BUS_USB, HID_GROUP_LOGITECH_DJ_DEVICE,
 		USB_VENDOR_ID_LOGITECH, HID_ANY_ID)},
+
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G920_WHEEL),
+		.driver_data = HIDPP_QUIRK_CLASS_G920 | HIDPP_QUIRK_FORCE_OUTPUT_REPORTS},
 	{}
 };
 
@@ -1661,6 +1892,7 @@ static struct hid_driver hidpp_driver = {
 	.raw_event = hidpp_raw_event,
 	.input_configured = hidpp_input_configured,
 	.input_mapping = hidpp_input_mapping,
+	.input_mapped = hidpp_input_mapped,
 };
 
 module_hid_driver(hidpp_driver);
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index 3d664d01305e..296d4991560e 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -272,7 +272,7 @@ static ssize_t mt_show_quirks(struct device *dev,
 			   struct device_attribute *attr,
 			   char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct mt_device *td = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%u\n", td->mtclass.quirks);
@@ -282,7 +282,7 @@ static ssize_t mt_set_quirks(struct device *dev,
 			  struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct mt_device *td = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -357,8 +357,19 @@ static void mt_feature_mapping(struct hid_device *hdev,
 			break;
 		}
 
-		td->inputmode = field->report->id;
-		td->inputmode_index = usage->usage_index;
+		if (td->inputmode < 0) {
+			td->inputmode = field->report->id;
+			td->inputmode_index = usage->usage_index;
+		} else {
+			/*
+			 * Some elan panels wrongly declare 2 input mode
+			 * features, and silently ignore when we set the
+			 * value in the second field. Skip the second feature
+			 * and hope for the best.
+			 */
+			dev_info(&hdev->dev,
+				 "Ignoring the extra HID_DG_INPUTMODE\n");
+		}
 
 		break;
 	case HID_DG_CONTACTMAX:
@@ -486,6 +497,11 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 			mt_store_field(usage, td, hi);
 			return 1;
 		case HID_DG_CONFIDENCE:
+			if (cls->name == MT_CLS_WIN_8 &&
+				field->application == HID_DG_TOUCHPAD) {
+				cls->quirks &= ~MT_QUIRK_ALWAYS_VALID;
+				cls->quirks |= MT_QUIRK_VALID_IS_CONFIDENCE;
+			}
 			mt_store_field(usage, td, hi);
 			return 1;
 		case HID_DG_TIPSWITCH:
diff --git a/drivers/hid/hid-ntrig.c b/drivers/hid/hid-ntrig.c
index 756d1ef9bd99..1b0084d4af2e 100644
--- a/drivers/hid/hid-ntrig.c
+++ b/drivers/hid/hid-ntrig.c
@@ -173,7 +173,7 @@ static ssize_t show_phys_width(struct device *dev,
 			       struct device_attribute *attr,
 			       char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->sensor_physical_width);
@@ -185,7 +185,7 @@ static ssize_t show_phys_height(struct device *dev,
 				struct device_attribute *attr,
 				char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->sensor_physical_height);
@@ -197,7 +197,7 @@ static ssize_t show_log_width(struct device *dev,
 			      struct device_attribute *attr,
 			      char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->sensor_logical_width);
@@ -209,7 +209,7 @@ static ssize_t show_log_height(struct device *dev,
 			       struct device_attribute *attr,
 			       char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->sensor_logical_height);
@@ -221,7 +221,7 @@ static ssize_t show_min_width(struct device *dev,
 			      struct device_attribute *attr,
 			      char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->min_width *
@@ -233,7 +233,7 @@ static ssize_t set_min_width(struct device *dev,
 			     struct device_attribute *attr,
 			     const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -256,7 +256,7 @@ static ssize_t show_min_height(struct device *dev,
 			       struct device_attribute *attr,
 			       char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->min_height *
@@ -268,7 +268,7 @@ static ssize_t set_min_height(struct device *dev,
 			      struct device_attribute *attr,
 			      const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -292,7 +292,7 @@ static ssize_t show_activate_slack(struct device *dev,
 				   struct device_attribute *attr,
 				   char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->activate_slack);
@@ -302,7 +302,7 @@ static ssize_t set_activate_slack(struct device *dev,
 				  struct device_attribute *attr,
 				  const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -325,7 +325,7 @@ static ssize_t show_activation_width(struct device *dev,
 				     struct device_attribute *attr,
 				     char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->activation_width *
@@ -337,7 +337,7 @@ static ssize_t set_activation_width(struct device *dev,
 				    struct device_attribute *attr,
 				    const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -361,7 +361,7 @@ static ssize_t show_activation_height(struct device *dev,
 				      struct device_attribute *attr,
 				      char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", nd->activation_height *
@@ -373,7 +373,7 @@ static ssize_t set_activation_height(struct device *dev,
 				     struct device_attribute *attr,
 				     const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
@@ -397,7 +397,7 @@ static ssize_t show_deactivate_slack(struct device *dev,
 				     struct device_attribute *attr,
 				     char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	return sprintf(buf, "%d\n", -nd->deactivate_slack);
@@ -407,7 +407,7 @@ static ssize_t set_deactivate_slack(struct device *dev,
 				    struct device_attribute *attr,
 				    const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct ntrig_data *nd = hid_get_drvdata(hdev);
 
 	unsigned long val;
diff --git a/drivers/hid/hid-picolcd_leds.c b/drivers/hid/hid-picolcd_leds.c
index e994f9c29012..a802b4f49c7b 100644
--- a/drivers/hid/hid-picolcd_leds.c
+++ b/drivers/hid/hid-picolcd_leds.c
@@ -66,7 +66,7 @@ static void picolcd_led_set_brightness(struct led_classdev *led_cdev,
 	int i, state = 0;
 
 	dev  = led_cdev->dev->parent;
-	hdev = container_of(dev, struct hid_device, dev);
+	hdev = to_hid_device(dev);
 	data = hid_get_drvdata(hdev);
 	if (!data)
 		return;
@@ -93,7 +93,7 @@ static enum led_brightness picolcd_led_get_brightness(struct led_classdev *led_c
 	int i, value = 0;
 
 	dev  = led_cdev->dev->parent;
-	hdev = container_of(dev, struct hid_device, dev);
+	hdev = to_hid_device(dev);
 	data = hid_get_drvdata(hdev);
 	for (i = 0; i < 8; i++)
 		if (led_cdev == data->led[i]) {
diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c
index 3a207c0ac0e3..f095bf8a3aa9 100644
--- a/drivers/hid/hid-prodikeys.c
+++ b/drivers/hid/hid-prodikeys.c
@@ -103,7 +103,7 @@ MODULE_PARM_DESC(enable, "Enable for the PC-MIDI virtual audio driver");
 static ssize_t show_channel(struct device *dev,
 	struct device_attribute *attr, char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	dbg_hid("pcmidi sysfs read channel=%u\n", pk->pm->midi_channel);
@@ -116,7 +116,7 @@ static ssize_t show_channel(struct device *dev,
 static ssize_t store_channel(struct device *dev,
 	struct device_attribute *attr, const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	unsigned channel = 0;
@@ -140,7 +140,7 @@ static struct device_attribute *sysfs_device_attr_channel = {
 static ssize_t show_sustain(struct device *dev,
  struct device_attribute *attr, char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	dbg_hid("pcmidi sysfs read sustain=%u\n", pk->pm->midi_sustain);
@@ -153,7 +153,7 @@ static ssize_t show_sustain(struct device *dev,
 static ssize_t store_sustain(struct device *dev,
 	struct device_attribute *attr, const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	unsigned sustain = 0;
@@ -179,7 +179,7 @@ static struct device_attribute *sysfs_device_attr_sustain = {
 static ssize_t show_octave(struct device *dev,
 	struct device_attribute *attr, char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	dbg_hid("pcmidi sysfs read octave=%d\n", pk->pm->midi_octave);
@@ -192,7 +192,7 @@ static ssize_t show_octave(struct device *dev,
 static ssize_t store_octave(struct device *dev,
 	struct device_attribute *attr, const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct pk_device *pk = hid_get_drvdata(hdev);
 
 	int octave = 0;
diff --git a/drivers/hid/hid-roccat-arvo.c b/drivers/hid/hid-roccat-arvo.c
index 1948208fe038..329c5d1270f9 100644
--- a/drivers/hid/hid-roccat-arvo.c
+++ b/drivers/hid/hid-roccat-arvo.c
@@ -191,8 +191,7 @@ static ssize_t arvo_sysfs_write(struct file *fp,
 		struct kobject *kobj, void const *buf,
 		loff_t off, size_t count, size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct arvo_device *arvo = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -211,8 +210,7 @@ static ssize_t arvo_sysfs_read(struct file *fp,
 		struct kobject *kobj, void *buf, loff_t off,
 		size_t count, size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct arvo_device *arvo = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
diff --git a/drivers/hid/hid-roccat-common.c b/drivers/hid/hid-roccat-common.c
index 02e28e9f4ea7..8155ac5fede2 100644
--- a/drivers/hid/hid-roccat-common.c
+++ b/drivers/hid/hid-roccat-common.c
@@ -134,8 +134,7 @@ ssize_t roccat_common2_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct roccat_common2_device *roccat_dev = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -158,8 +157,7 @@ ssize_t roccat_common2_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct roccat_common2_device *roccat_dev = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
diff --git a/drivers/hid/hid-roccat-isku.c b/drivers/hid/hid-roccat-isku.c
index bc62ed91e451..02db537f8f3e 100644
--- a/drivers/hid/hid-roccat-isku.c
+++ b/drivers/hid/hid-roccat-isku.c
@@ -121,8 +121,7 @@ static ssize_t isku_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct isku_device *isku = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -144,8 +143,7 @@ static ssize_t isku_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct isku_device *isku = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c
index c29265055ac1..bf4675a27396 100644
--- a/drivers/hid/hid-roccat-kone.c
+++ b/drivers/hid/hid-roccat-kone.c
@@ -269,8 +269,7 @@ static int kone_get_firmware_version(struct usb_device *usb_dev, int *result)
 static ssize_t kone_sysfs_read_settings(struct file *fp, struct kobject *kobj,
 		struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count) {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kone_device *kone = hid_get_drvdata(dev_get_drvdata(dev));
 
 	if (off >= sizeof(struct kone_settings))
@@ -294,8 +293,7 @@ static ssize_t kone_sysfs_read_settings(struct file *fp, struct kobject *kobj,
 static ssize_t kone_sysfs_write_settings(struct file *fp, struct kobject *kobj,
 		struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count) {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kone_device *kone = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval = 0, difference, old_profile;
@@ -332,8 +330,7 @@ static BIN_ATTR(settings, 0660, kone_sysfs_read_settings,
 static ssize_t kone_sysfs_read_profilex(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr,
 		char *buf, loff_t off, size_t count) {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kone_device *kone = hid_get_drvdata(dev_get_drvdata(dev));
 
 	if (off >= sizeof(struct kone_profile))
@@ -353,8 +350,7 @@ static ssize_t kone_sysfs_read_profilex(struct file *fp,
 static ssize_t kone_sysfs_write_profilex(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr,
 		char *buf, loff_t off, size_t count) {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kone_device *kone = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	struct kone_profile *profile;
diff --git a/drivers/hid/hid-roccat-koneplus.c b/drivers/hid/hid-roccat-koneplus.c
index 5e99fcdc71b9..09e8fc72aa1d 100644
--- a/drivers/hid/hid-roccat-koneplus.c
+++ b/drivers/hid/hid-roccat-koneplus.c
@@ -87,8 +87,7 @@ static ssize_t koneplus_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct koneplus_device *koneplus = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -113,8 +112,7 @@ static ssize_t koneplus_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct koneplus_device *koneplus = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -193,8 +191,7 @@ static ssize_t koneplus_sysfs_read_profilex_settings(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
@@ -212,8 +209,7 @@ static ssize_t koneplus_sysfs_read_profilex_buttons(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
diff --git a/drivers/hid/hid-roccat-kovaplus.c b/drivers/hid/hid-roccat-kovaplus.c
index 966047711fbf..43617fb28b87 100644
--- a/drivers/hid/hid-roccat-kovaplus.c
+++ b/drivers/hid/hid-roccat-kovaplus.c
@@ -128,8 +128,7 @@ static ssize_t kovaplus_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kovaplus_device *kovaplus = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -154,8 +153,7 @@ static ssize_t kovaplus_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct kovaplus_device *kovaplus = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -221,8 +219,7 @@ static ssize_t kovaplus_sysfs_read_profilex_settings(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
@@ -240,8 +237,7 @@ static ssize_t kovaplus_sysfs_read_profilex_buttons(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
diff --git a/drivers/hid/hid-roccat-lua.c b/drivers/hid/hid-roccat-lua.c
index 65e2e76bf2fe..ac1a7313e259 100644
--- a/drivers/hid/hid-roccat-lua.c
+++ b/drivers/hid/hid-roccat-lua.c
@@ -30,7 +30,7 @@ static ssize_t lua_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct lua_device *lua = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -52,7 +52,7 @@ static ssize_t lua_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct lua_device *lua = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
diff --git a/drivers/hid/hid-roccat-pyra.c b/drivers/hid/hid-roccat-pyra.c
index 47d7e74231e5..b30aa7b82bf8 100644
--- a/drivers/hid/hid-roccat-pyra.c
+++ b/drivers/hid/hid-roccat-pyra.c
@@ -90,8 +90,7 @@ static ssize_t pyra_sysfs_read(struct file *fp, struct kobject *kobj,
 		char *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct pyra_device *pyra = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -116,8 +115,7 @@ static ssize_t pyra_sysfs_write(struct file *fp, struct kobject *kobj,
 		void const *buf, loff_t off, size_t count,
 		size_t real_size, uint command)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct pyra_device *pyra = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval;
@@ -191,8 +189,7 @@ static ssize_t pyra_sysfs_read_profilex_settings(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
@@ -210,8 +207,7 @@ static ssize_t pyra_sysfs_read_profilex_buttons(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	ssize_t retval;
 
@@ -248,8 +244,7 @@ static ssize_t pyra_sysfs_write_settings(struct file *fp,
 		struct kobject *kobj, struct bin_attribute *attr, char *buf,
 		loff_t off, size_t count)
 {
-	struct device *dev =
-			container_of(kobj, struct device, kobj)->parent->parent;
+	struct device *dev = kobj_to_dev(kobj)->parent->parent;
 	struct pyra_device *pyra = hid_get_drvdata(dev_get_drvdata(dev));
 	struct usb_device *usb_dev = interface_to_usbdev(to_usb_interface(dev));
 	int retval = 0;
diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c
index 92870cdb52d9..3d5ba5b51af3 100644
--- a/drivers/hid/hid-sensor-hub.c
+++ b/drivers/hid/hid-sensor-hub.c
@@ -218,7 +218,8 @@ int sensor_hub_set_feature(struct hid_sensor_hub_device *hsdev, u32 report_id,
 		goto done_proc;
 	}
 
-	remaining_bytes = do_div(buffer_size, sizeof(__s32));
+	remaining_bytes = buffer_size % sizeof(__s32);
+	buffer_size = buffer_size / sizeof(__s32);
 	if (buffer_size) {
 		for (i = 0; i < buffer_size; ++i) {
 			hid_set_field(report->field[field_index], i,
@@ -794,6 +795,9 @@ static const struct hid_device_id sensor_hub_devices[] = {
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
 			USB_DEVICE_ID_ITE_LENOVO_YOGA2),
 			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
+			USB_DEVICE_ID_ITE_LENOVO_YOGA900),
+			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, HID_ANY_ID,
 		     HID_ANY_ID) },
 	{ }
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index 774cd2210566..9b8db0e0ef1c 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -1028,6 +1028,7 @@ struct sony_sc {
 	struct led_classdev *leds[MAX_LEDS];
 	unsigned long quirks;
 	struct work_struct state_worker;
+	void(*send_output_report)(struct sony_sc*);
 	struct power_supply *battery;
 	struct power_supply_desc battery_desc;
 	int device_id;
@@ -1044,6 +1045,7 @@ struct sony_sc {
 	__u8 battery_charging;
 	__u8 battery_capacity;
 	__u8 led_state[MAX_LEDS];
+	__u8 resume_led_state[MAX_LEDS];
 	__u8 led_delay_on[MAX_LEDS];
 	__u8 led_delay_off[MAX_LEDS];
 	__u8 led_count;
@@ -1137,11 +1139,11 @@ static __u8 *sony_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 	 * the gyroscope values to corresponding axes so we need a
 	 * modified one.
 	 */
-	if ((sc->quirks & DUALSHOCK4_CONTROLLER_USB) && *rsize == 467) {
+	if (sc->quirks & DUALSHOCK4_CONTROLLER_USB) {
 		hid_info(hdev, "Using modified Dualshock 4 report descriptor with gyroscope axes\n");
 		rdesc = dualshock4_usb_rdesc;
 		*rsize = sizeof(dualshock4_usb_rdesc);
-	} else if ((sc->quirks & DUALSHOCK4_CONTROLLER_BT) && *rsize == 357) {
+	} else if (sc->quirks & DUALSHOCK4_CONTROLLER_BT) {
 		hid_info(hdev, "Using modified Dualshock 4 Bluetooth report descriptor\n");
 		rdesc = dualshock4_bt_rdesc;
 		*rsize = sizeof(dualshock4_bt_rdesc);
@@ -1549,7 +1551,7 @@ static void sony_led_set_brightness(struct led_classdev *led,
 				    enum led_brightness value)
 {
 	struct device *dev = led->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct sony_sc *drv_data;
 
 	int n;
@@ -1591,7 +1593,7 @@ static void sony_led_set_brightness(struct led_classdev *led,
 static enum led_brightness sony_led_get_brightness(struct led_classdev *led)
 {
 	struct device *dev = led->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct sony_sc *drv_data;
 
 	int n;
@@ -1614,7 +1616,7 @@ static int sony_led_blink_set(struct led_classdev *led, unsigned long *delay_on,
 				unsigned long *delay_off)
 {
 	struct device *dev = led->dev->parent;
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct sony_sc *drv_data = hid_get_drvdata(hdev);
 	int n;
 	__u8 new_on, new_off;
@@ -1789,7 +1791,7 @@ error_leds:
 	return ret;
 }
 
-static void sixaxis_state_worker(struct work_struct *work)
+static void sixaxis_send_output_report(struct sony_sc *sc)
 {
 	static const union sixaxis_output_report_01 default_report = {
 		.buf = {
@@ -1803,7 +1805,6 @@ static void sixaxis_state_worker(struct work_struct *work)
 			0x00, 0x00, 0x00, 0x00, 0x00
 		}
 	};
-	struct sony_sc *sc = container_of(work, struct sony_sc, state_worker);
 	struct sixaxis_output_report *report =
 		(struct sixaxis_output_report *)sc->output_report_dmabuf;
 	int n;
@@ -1846,9 +1847,8 @@ static void sixaxis_state_worker(struct work_struct *work)
 			HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
 }
 
-static void dualshock4_state_worker(struct work_struct *work)
+static void dualshock4_send_output_report(struct sony_sc *sc)
 {
-	struct sony_sc *sc = container_of(work, struct sony_sc, state_worker);
 	struct hid_device *hdev = sc->hdev;
 	__u8 *buf = sc->output_report_dmabuf;
 	int offset;
@@ -1893,9 +1893,8 @@ static void dualshock4_state_worker(struct work_struct *work)
 				HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
 }
 
-static void motion_state_worker(struct work_struct *work)
+static void motion_send_output_report(struct sony_sc *sc)
 {
-	struct sony_sc *sc = container_of(work, struct sony_sc, state_worker);
 	struct hid_device *hdev = sc->hdev;
 	struct motion_output_report_02 *report =
 		(struct motion_output_report_02 *)sc->output_report_dmabuf;
@@ -1914,6 +1913,18 @@ static void motion_state_worker(struct work_struct *work)
 	hid_hw_output_report(hdev, (__u8 *)report, MOTION_REPORT_0x02_SIZE);
 }
 
+static inline void sony_send_output_report(struct sony_sc *sc)
+{
+	if (sc->send_output_report)
+		sc->send_output_report(sc);
+}
+
+static void sony_state_worker(struct work_struct *work)
+{
+	struct sony_sc *sc = container_of(work, struct sony_sc, state_worker);
+	sc->send_output_report(sc);
+}
+
 static int sony_allocate_output_report(struct sony_sc *sc)
 {
 	if ((sc->quirks & SIXAXIS_CONTROLLER) ||
@@ -2241,11 +2252,13 @@ static void sony_release_device_id(struct sony_sc *sc)
 	}
 }
 
-static inline void sony_init_work(struct sony_sc *sc,
-					void (*worker)(struct work_struct *))
+static inline void sony_init_output_report(struct sony_sc *sc,
+				void(*send_output_report)(struct sony_sc*))
 {
+	sc->send_output_report = send_output_report;
+
 	if (!sc->worker_initialized)
-		INIT_WORK(&sc->state_worker, worker);
+		INIT_WORK(&sc->state_worker, sony_state_worker);
 
 	sc->worker_initialized = 1;
 }
@@ -2319,7 +2332,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		hdev->quirks |= HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP;
 		hdev->quirks |= HID_QUIRK_SKIP_OUTPUT_REPORT_ID;
 		ret = sixaxis_set_operational_usb(hdev);
-		sony_init_work(sc, sixaxis_state_worker);
+		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if ((sc->quirks & SIXAXIS_CONTROLLER_BT) ||
 			(sc->quirks & NAVIGATION_CONTROLLER_BT)) {
 		/*
@@ -2328,7 +2341,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		 */
 		hdev->quirks |= HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP;
 		ret = sixaxis_set_operational_bt(hdev);
-		sony_init_work(sc, sixaxis_state_worker);
+		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & DUALSHOCK4_CONTROLLER) {
 		if (sc->quirks & DUALSHOCK4_CONTROLLER_BT) {
 			/*
@@ -2343,9 +2356,9 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 			}
 		}
 
-		sony_init_work(sc, dualshock4_state_worker);
+		sony_init_output_report(sc, dualshock4_send_output_report);
 	} else if (sc->quirks & MOTION_CONTROLLER) {
-		sony_init_work(sc, motion_state_worker);
+		sony_init_output_report(sc, motion_send_output_report);
 	} else {
 		ret = 0;
 	}
@@ -2421,6 +2434,56 @@ static void sony_remove(struct hid_device *hdev)
 	hid_hw_stop(hdev);
 }
 
+#ifdef CONFIG_PM
+
+static int sony_suspend(struct hid_device *hdev, pm_message_t message)
+{
+	/*
+	 * On suspend save the current LED state,
+	 * stop running force-feedback and blank the LEDS.
+         */
+	if (SONY_LED_SUPPORT || SONY_FF_SUPPORT) {
+		struct sony_sc *sc = hid_get_drvdata(hdev);
+
+#ifdef CONFIG_SONY_FF
+		sc->left = sc->right = 0;
+#endif
+
+		memcpy(sc->resume_led_state, sc->led_state,
+			sizeof(sc->resume_led_state));
+		memset(sc->led_state, 0, sizeof(sc->led_state));
+
+		sony_send_output_report(sc);
+	}
+
+	return 0;
+}
+
+static int sony_resume(struct hid_device *hdev)
+{
+	/* Restore the state of controller LEDs on resume */
+	if (SONY_LED_SUPPORT) {
+		struct sony_sc *sc = hid_get_drvdata(hdev);
+
+		memcpy(sc->led_state, sc->resume_led_state,
+			sizeof(sc->led_state));
+
+		/*
+		 * The Sixaxis and navigation controllers on USB need to be
+		 * reinitialized on resume or they won't behave properly.
+		 */
+		if ((sc->quirks & SIXAXIS_CONTROLLER_USB) ||
+			(sc->quirks & NAVIGATION_CONTROLLER_USB))
+			sixaxis_set_operational_usb(sc->hdev);
+
+		sony_set_leds(sc);
+	}
+
+	return 0;
+}
+
+#endif
+
 static const struct hid_device_id sony_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER),
 		.driver_data = SIXAXIS_CONTROLLER_USB },
@@ -2470,7 +2533,13 @@ static struct hid_driver sony_driver = {
 	.probe            = sony_probe,
 	.remove           = sony_remove,
 	.report_fixup     = sony_report_fixup,
-	.raw_event        = sony_raw_event
+	.raw_event        = sony_raw_event,
+
+#ifdef CONFIG_PM
+	.suspend          = sony_suspend,
+	.resume	          = sony_resume,
+	.reset_resume     = sony_resume,
+#endif
 };
 
 static int __init sony_init(void)
diff --git a/drivers/hid/hid-steelseries.c b/drivers/hid/hid-steelseries.c
index 3edd4ac36494..ec18768b124a 100644
--- a/drivers/hid/hid-steelseries.c
+++ b/drivers/hid/hid-steelseries.c
@@ -141,7 +141,7 @@ static void steelseries_srws1_led_all_set_brightness(struct led_classdev *led_cd
 			enum led_brightness value)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct steelseries_srws1_data *drv_data = hid_get_drvdata(hid);
 
 	if (!drv_data) {
@@ -160,7 +160,7 @@ static void steelseries_srws1_led_all_set_brightness(struct led_classdev *led_cd
 static enum led_brightness steelseries_srws1_led_all_get_brightness(struct led_classdev *led_cdev)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct steelseries_srws1_data *drv_data;
 
 	drv_data = hid_get_drvdata(hid);
@@ -177,7 +177,7 @@ static void steelseries_srws1_led_set_brightness(struct led_classdev *led_cdev,
 			enum led_brightness value)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct steelseries_srws1_data *drv_data = hid_get_drvdata(hid);
 	int i, state = 0;
 
@@ -205,7 +205,7 @@ static void steelseries_srws1_led_set_brightness(struct led_classdev *led_cdev,
 static enum led_brightness steelseries_srws1_led_get_brightness(struct led_classdev *led_cdev)
 {
 	struct device *dev = led_cdev->dev->parent;
-	struct hid_device *hid = container_of(dev, struct hid_device, dev);
+	struct hid_device *hid = to_hid_device(dev);
 	struct steelseries_srws1_data *drv_data;
 	int i, value = 0;
 
diff --git a/drivers/hid/hid-wiimote-modules.c b/drivers/hid/hid-wiimote-modules.c
index 05e23c417d50..4390eee2ce84 100644
--- a/drivers/hid/hid-wiimote-modules.c
+++ b/drivers/hid/hid-wiimote-modules.c
@@ -296,14 +296,12 @@ static const struct wiimod_ops wiimod_battery = {
 
 static enum led_brightness wiimod_led_get(struct led_classdev *led_dev)
 {
-	struct wiimote_data *wdata;
 	struct device *dev = led_dev->dev->parent;
+	struct wiimote_data *wdata = dev_to_wii(dev);
 	int i;
 	unsigned long flags;
 	bool value = false;
 
-	wdata = hid_get_drvdata(container_of(dev, struct hid_device, dev));
-
 	for (i = 0; i < 4; ++i) {
 		if (wdata->leds[i] == led_dev) {
 			spin_lock_irqsave(&wdata->state.lock, flags);
@@ -319,14 +317,12 @@ static enum led_brightness wiimod_led_get(struct led_classdev *led_dev)
 static void wiimod_led_set(struct led_classdev *led_dev,
 			   enum led_brightness value)
 {
-	struct wiimote_data *wdata;
 	struct device *dev = led_dev->dev->parent;
+	struct wiimote_data *wdata = dev_to_wii(dev);
 	int i;
 	unsigned long flags;
 	__u8 state, flag;
 
-	wdata = hid_get_drvdata(container_of(dev, struct hid_device, dev));
-
 	for (i = 0; i < 4; ++i) {
 		if (wdata->leds[i] == led_dev) {
 			flag = WIIPROTO_FLAG_LED(i + 1);
diff --git a/drivers/hid/hid-wiimote.h b/drivers/hid/hid-wiimote.h
index 875694d43e4d..510ca77fe14e 100644
--- a/drivers/hid/hid-wiimote.h
+++ b/drivers/hid/hid-wiimote.h
@@ -256,8 +256,7 @@ enum wiiproto_reqs {
 	WIIPROTO_REQ_MAX
 };
 
-#define dev_to_wii(pdev) hid_get_drvdata(container_of(pdev, struct hid_device, \
-									dev))
+#define dev_to_wii(pdev) hid_get_drvdata(to_hid_device(pdev))
 
 void __wiimote_schedule(struct wiimote_data *wdata);
 
diff --git a/drivers/hid/i2c-hid/i2c-hid.c b/drivers/hid/i2c-hid/i2c-hid.c
index 10bd8e6e4c9c..b9216938a718 100644
--- a/drivers/hid/i2c-hid/i2c-hid.c
+++ b/drivers/hid/i2c-hid/i2c-hid.c
@@ -151,6 +151,7 @@ struct i2c_hid {
 	struct i2c_hid_platform_data pdata;
 
 	bool			irq_wake_enabled;
+	struct mutex		reset_lock;
 };
 
 static int __i2c_hid_command(struct i2c_client *client,
@@ -356,9 +357,16 @@ static int i2c_hid_hwreset(struct i2c_client *client)
 
 	i2c_hid_dbg(ihid, "%s\n", __func__);
 
+	/*
+	 * This prevents sending feature reports while the device is
+	 * being reset. Otherwise we may lose the reset complete
+	 * interrupt.
+	 */
+	mutex_lock(&ihid->reset_lock);
+
 	ret = i2c_hid_set_power(client, I2C_HID_PWR_ON);
 	if (ret)
-		return ret;
+		goto out_unlock;
 
 	i2c_hid_dbg(ihid, "resetting...\n");
 
@@ -366,10 +374,11 @@ static int i2c_hid_hwreset(struct i2c_client *client)
 	if (ret) {
 		dev_err(&client->dev, "failed to reset device.\n");
 		i2c_hid_set_power(client, I2C_HID_PWR_SLEEP);
-		return ret;
 	}
 
-	return 0;
+out_unlock:
+	mutex_unlock(&ihid->reset_lock);
+	return ret;
 }
 
 static void i2c_hid_get_input(struct i2c_hid *ihid)
@@ -587,12 +596,15 @@ static int i2c_hid_output_raw_report(struct hid_device *hid, __u8 *buf,
 		size_t count, unsigned char report_type, bool use_data)
 {
 	struct i2c_client *client = hid->driver_data;
+	struct i2c_hid *ihid = i2c_get_clientdata(client);
 	int report_id = buf[0];
 	int ret;
 
 	if (report_type == HID_INPUT_REPORT)
 		return -EINVAL;
 
+	mutex_lock(&ihid->reset_lock);
+
 	if (report_id) {
 		buf++;
 		count--;
@@ -605,6 +617,8 @@ static int i2c_hid_output_raw_report(struct hid_device *hid, __u8 *buf,
 	if (report_id && ret >= 0)
 		ret++; /* add report_id to the number of transfered bytes */
 
+	mutex_unlock(&ihid->reset_lock);
+
 	return ret;
 }
 
@@ -990,6 +1004,7 @@ static int i2c_hid_probe(struct i2c_client *client,
 	ihid->wHIDDescRegister = cpu_to_le16(hidRegister);
 
 	init_waitqueue_head(&ihid->wait);
+	mutex_init(&ihid->reset_lock);
 
 	/* we need to allocate the command buffer without knowing the maximum
 	 * size of the reports. Let's use HID_MIN_BUFFER_SIZE, then we do the
@@ -1184,7 +1199,6 @@ MODULE_DEVICE_TABLE(i2c, i2c_hid_id_table);
 static struct i2c_driver i2c_hid_driver = {
 	.driver = {
 		.name	= "i2c_hid",
-		.owner	= THIS_MODULE,
 		.pm	= &i2c_hid_pm,
 		.acpi_match_table = ACPI_PTR(i2c_hid_acpi_match),
 		.of_match_table = of_match_ptr(i2c_hid_of_match),
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index 36712e9f56c2..ad71160b9ea4 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -274,10 +274,10 @@ static void hid_irq_in(struct urb *urb)
 
 	switch (urb->status) {
 	case 0:			/* success */
-		usbhid_mark_busy(usbhid);
 		usbhid->retry_delay = 0;
 		if ((hid->quirks & HID_QUIRK_ALWAYS_POLL) && !hid->open)
 			break;
+		usbhid_mark_busy(usbhid);
 		if (!test_bit(HID_RESUME_RUNNING, &usbhid->iofl)) {
 			hid_input_report(urb->context, HID_INPUT_REPORT,
 					 urb->transfer_buffer,
@@ -477,8 +477,6 @@ static void hid_ctrl(struct urb *urb)
 	struct usbhid_device *usbhid = hid->driver_data;
 	int unplug = 0, status = urb->status;
 
-	spin_lock(&usbhid->lock);
-
 	switch (status) {
 	case 0:			/* success */
 		if (usbhid->ctrl[usbhid->ctrltail].dir == USB_DIR_IN)
@@ -498,6 +496,8 @@ static void hid_ctrl(struct urb *urb)
 		hid_warn(urb->dev, "ctrl urb status %d received\n", status);
 	}
 
+	spin_lock(&usbhid->lock);
+
 	if (unplug) {
 		usbhid->ctrltail = usbhid->ctrlhead;
 	} else {
diff --git a/drivers/hid/usbhid/usbhid.h b/drivers/hid/usbhid/usbhid.h
index 807922b49aa4..fa47d666cfcf 100644
--- a/drivers/hid/usbhid/usbhid.h
+++ b/drivers/hid/usbhid/usbhid.h
@@ -96,7 +96,7 @@ struct usbhid_device {
 };
 
 #define	hid_to_usb_dev(hid_dev) \
-	container_of(hid_dev->dev.parent->parent, struct usb_device, dev)
+	to_usb_device(hid_dev->dev.parent->parent)
 
 #endif
 
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index e06af5b9f59e..5cb21dd91094 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -686,7 +686,7 @@ out:
 static ssize_t wacom_led_select_store(struct device *dev, int set_id,
 				      const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	unsigned int id;
 	int err;
@@ -714,7 +714,7 @@ static ssize_t wacom_led##SET_ID##_select_store(struct device *dev,	\
 static ssize_t wacom_led##SET_ID##_select_show(struct device *dev,	\
 	struct device_attribute *attr, char *buf)			\
 {									\
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);\
+	struct hid_device *hdev = to_hid_device(dev);\
 	struct wacom *wacom = hid_get_drvdata(hdev);			\
 	return scnprintf(buf, PAGE_SIZE, "%d\n",			\
 			 wacom->led.select[SET_ID]);			\
@@ -750,7 +750,7 @@ static ssize_t wacom_luminance_store(struct wacom *wacom, u8 *dest,
 static ssize_t wacom_##name##_luminance_store(struct device *dev,	\
 	struct device_attribute *attr, const char *buf, size_t count)	\
 {									\
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);\
+	struct hid_device *hdev = to_hid_device(dev);\
 	struct wacom *wacom = hid_get_drvdata(hdev);			\
 									\
 	return wacom_luminance_store(wacom, &wacom->led.field,		\
@@ -773,7 +773,7 @@ DEVICE_LUMINANCE_ATTR(buttons, img_lum);
 static ssize_t wacom_button_image_store(struct device *dev, int button_id,
 					const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	int err;
 	unsigned len;
@@ -1097,7 +1097,7 @@ static ssize_t wacom_show_speed(struct device *dev,
 				struct device_attribute
 				*attr, char *buf)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 
 	return snprintf(buf, PAGE_SIZE, "%i\n", wacom->wacom_wac.bt_high_speed);
@@ -1107,7 +1107,7 @@ static ssize_t wacom_store_speed(struct device *dev,
 				struct device_attribute *attr,
 				const char *buf, size_t count)
 {
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	u8 new_speed;
 
@@ -1130,8 +1130,8 @@ static ssize_t wacom_show_remote_mode(struct kobject *kobj,
 				      struct kobj_attribute *kattr,
 				      char *buf, int index)
 {
-	struct device *dev = container_of(kobj->parent, struct device, kobj);
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct device *dev = kobj_to_dev(kobj->parent);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	u8 mode;
 
@@ -1241,8 +1241,8 @@ static ssize_t wacom_store_unpair_remote(struct kobject *kobj,
 					 const char *buf, size_t count)
 {
 	unsigned char selector = 0;
-	struct device *dev = container_of(kobj->parent, struct device, kobj);
-	struct hid_device *hdev = container_of(dev, struct hid_device, dev);
+	struct device *dev = kobj_to_dev(kobj->parent);
+	struct hid_device *hdev = to_hid_device(dev);
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	int err;
 
@@ -1353,8 +1353,7 @@ static void wacom_clean_inputs(struct wacom *wacom)
 		else
 			input_free_device(wacom->wacom_wac.pad_input);
 	}
-	if (wacom->remote_dir)
-		kobject_put(wacom->remote_dir);
+	kobject_put(wacom->remote_dir);
 	wacom->wacom_wac.pen_input = NULL;
 	wacom->wacom_wac.touch_input = NULL;
 	wacom->wacom_wac.pad_input = NULL;
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 01a4f05c1642..99ef77fcfb80 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -34,6 +34,9 @@
  */
 #define WACOM_CONTACT_AREA_SCALE 2607
 
+static void wacom_report_numbered_buttons(struct input_dev *input_dev,
+				int button_count, int mask);
+
 /*
  * Percent of battery capacity for Graphire.
  * 8th value means AC online and show 100% capacity.
@@ -436,16 +439,142 @@ exit:
 static void wacom_intuos_schedule_prox_event(struct wacom_wac *wacom_wac)
 {
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
+	struct wacom_features *features = &wacom_wac->features;
 	struct hid_report *r;
 	struct hid_report_enum *re;
 
 	re = &(wacom->hdev->report_enum[HID_FEATURE_REPORT]);
-	r = re->report_id_hash[WACOM_REPORT_INTUOSREAD];
+	if (features->type == INTUOSHT2)
+		r = re->report_id_hash[WACOM_REPORT_INTUOSHT2_ID];
+	else
+		r = re->report_id_hash[WACOM_REPORT_INTUOS_ID1];
 	if (r) {
 		hid_hw_request(wacom->hdev, r, HID_REQ_GET_REPORT);
 	}
 }
 
+static int wacom_intuos_pad(struct wacom_wac *wacom)
+{
+	struct wacom_features *features = &wacom->features;
+	unsigned char *data = wacom->data;
+	struct input_dev *input = wacom->pad_input;
+	int i;
+	int buttons = 0, nbuttons = features->numbered_buttons;
+	int keys = 0, nkeys = 0;
+	int ring1 = 0, ring2 = 0;
+	int strip1 = 0, strip2 = 0;
+	bool prox = false;
+
+	/* pad packets. Works as a second tool and is always in prox */
+	if (!(data[0] == WACOM_REPORT_INTUOSPAD || data[0] == WACOM_REPORT_INTUOS5PAD ||
+	      data[0] == WACOM_REPORT_CINTIQPAD))
+		return 0;
+
+	if (features->type >= INTUOS4S && features->type <= INTUOS4L) {
+		buttons = (data[3] << 1) | (data[2] & 0x01);
+		ring1 = data[1];
+	} else if (features->type == DTK) {
+		buttons = data[6];
+	} else if (features->type == WACOM_13HD) {
+		buttons = (data[4] << 1) | (data[3] & 0x01);
+	} else if (features->type == WACOM_24HD) {
+		buttons = (data[8] << 8) | data[6];
+		ring1 = data[1];
+		ring2 = data[2];
+
+		/*
+		 * Three "buttons" are available on the 24HD which are
+		 * physically implemented as a touchstrip. Each button
+		 * is approximately 3 bits wide with a 2 bit spacing.
+		 * The raw touchstrip bits are stored at:
+		 *    ((data[3] & 0x1f) << 8) | data[4])
+		 */
+		nkeys = 3;
+		keys = ((data[3] & 0x1C) ? 1<<2 : 0) |
+		       ((data[4] & 0xE0) ? 1<<1 : 0) |
+		       ((data[4] & 0x07) ? 1<<0 : 0);
+	} else if (features->type == WACOM_27QHD) {
+		nkeys = 3;
+		keys = data[2] & 0x07;
+
+		input_report_abs(input, ABS_X, be16_to_cpup((__be16 *)&data[4]));
+		input_report_abs(input, ABS_Y, be16_to_cpup((__be16 *)&data[6]));
+		input_report_abs(input, ABS_Z, be16_to_cpup((__be16 *)&data[8]));
+	} else if (features->type == CINTIQ_HYBRID) {
+		/*
+		 * Do not send hardware buttons under Android. They
+		 * are already sent to the system through GPIO (and
+		 * have different meaning).
+		 *
+		 * d-pad right  -> data[4] & 0x10
+		 * d-pad up     -> data[4] & 0x20
+		 * d-pad left   -> data[4] & 0x40
+		 * d-pad down   -> data[4] & 0x80
+		 * d-pad center -> data[3] & 0x01
+		 */
+		buttons = (data[4] << 1) | (data[3] & 0x01);
+	} else if (features->type == CINTIQ_COMPANION_2) {
+		/* d-pad right  -> data[4] & 0x10
+		 * d-pad up     -> data[4] & 0x20
+		 * d-pad left   -> data[4] & 0x40
+		 * d-pad down   -> data[4] & 0x80
+		 * d-pad center -> data[3] & 0x01
+		 */
+		buttons = ((data[2] >> 4) << 7) |
+		          ((data[1] & 0x04) << 6) |
+		          ((data[2] & 0x0F) << 2) |
+		          (data[1] & 0x03);
+	} else if (features->type >= INTUOS5S && features->type <= INTUOSPL) {
+		/*
+		 * ExpressKeys on Intuos5/Intuos Pro have a capacitive sensor in
+		 * addition to the mechanical switch. Switch data is
+		 * stored in data[4], capacitive data in data[5].
+		 *
+		 * Touch ring mode switch (data[3]) has no capacitive sensor
+		 */
+		buttons = (data[4] << 1) | (data[3] & 0x01);
+		ring1 = data[2];
+	} else {
+		if (features->type == WACOM_21UX2 || features->type == WACOM_22HD) {
+			buttons = (data[8] << 10) | ((data[7] & 0x01) << 9) |
+			          (data[6] << 1) | (data[5] & 0x01);
+
+			if (features->type == WACOM_22HD) {
+				nkeys = 3;
+				keys = data[9] & 0x07;
+			}
+		} else {
+			buttons = ((data[6] & 0x10) << 10) |
+			          ((data[5] & 0x10) << 9)  |
+			          ((data[6] & 0x0F) << 4)  |
+			          (data[5] & 0x0F);
+		}
+		strip1 = ((data[1] & 0x1f) << 8) | data[2];
+		strip2 = ((data[3] & 0x1f) << 8) | data[4];
+	}
+
+	prox = (buttons & ~(~0 << nbuttons)) | (keys & ~(~0 << nkeys)) |
+	       (ring1 & 0x80) | (ring2 & 0x80) | strip1 | strip2;
+
+	wacom_report_numbered_buttons(input, nbuttons, buttons);
+
+	for (i = 0; i < nkeys; i++)
+		input_report_key(input, KEY_PROG1 + i, keys & (1 << i));
+
+	input_report_abs(input, ABS_RX, strip1);
+	input_report_abs(input, ABS_RY, strip2);
+
+	input_report_abs(input, ABS_WHEEL,    (ring1 & 0x80) ? (ring1 & 0x7f) : 0);
+	input_report_abs(input, ABS_THROTTLE, (ring2 & 0x80) ? (ring2 & 0x7f) : 0);
+
+	input_report_key(input, wacom->tool[1], prox ? 1 : 0);
+	input_report_abs(input, ABS_MISC, prox ? PAD_DEVICE_ID : 0);
+
+	input_event(input, EV_MSC, MSC_SERIAL, 0xffffffff);
+
+	return 1;
+}
+
 static int wacom_intuos_inout(struct wacom_wac *wacom)
 {
 	struct wacom_features *features = &wacom->features;
@@ -755,19 +884,40 @@ static int wacom_remote_status_irq(struct wacom_wac *wacom_wac, size_t len)
 	return 0;
 }
 
-static void wacom_intuos_general(struct wacom_wac *wacom)
+static int wacom_intuos_general(struct wacom_wac *wacom)
 {
 	struct wacom_features *features = &wacom->features;
 	unsigned char *data = wacom->data;
 	struct input_dev *input = wacom->pen_input;
-	unsigned int t;
+	int idx = (features->type == INTUOS) ? (data[1] & 0x01) : 0;
+	unsigned char type = (data[1] >> 1) & 0x0F;
+	unsigned int x, y, distance, t;
 
-	/* general pen packet */
-	if ((data[1] & 0xb8) == 0xa0) {
-		t = (data[6] << 2) | ((data[7] >> 6) & 3);
-		if (features->pressure_max == 2047) {
-			t = (t << 1) | (data[1] & 1);
-		}
+	if (data[0] != WACOM_REPORT_PENABLED && data[0] != WACOM_REPORT_CINTIQ &&
+		data[0] != WACOM_REPORT_INTUOS_PEN)
+		return 0;
+
+	x = (be16_to_cpup((__be16 *)&data[2]) << 1) | ((data[9] >> 1) & 1);
+	y = (be16_to_cpup((__be16 *)&data[4]) << 1) | (data[9] & 1);
+	distance = data[9] >> 2;
+	if (features->type < INTUOS3S) {
+		x >>= 1;
+		y >>= 1;
+		distance >>= 1;
+	}
+	input_report_abs(input, ABS_X, x);
+	input_report_abs(input, ABS_Y, y);
+	input_report_abs(input, ABS_DISTANCE, distance);
+
+	switch (type) {
+	case 0x00:
+	case 0x01:
+	case 0x02:
+	case 0x03:
+		/* general pen packet */
+		t = (data[6] << 3) | ((data[7] & 0xC0) >> 5) | (data[1] & 1);
+		if (features->pressure_max < 2047)
+			t >>= 1;
 		input_report_abs(input, ABS_PRESSURE, t);
 		if (features->type != INTUOSHT2) {
 		    input_report_abs(input, ABS_TILT_X,
@@ -777,29 +927,112 @@ static void wacom_intuos_general(struct wacom_wac *wacom)
 		input_report_key(input, BTN_STYLUS, data[1] & 2);
 		input_report_key(input, BTN_STYLUS2, data[1] & 4);
 		input_report_key(input, BTN_TOUCH, t > 10);
-	}
+		break;
 
-	/* airbrush second packet */
-	if ((data[1] & 0xbc) == 0xb4) {
+	case 0x0a:
+		/* airbrush second packet */
 		input_report_abs(input, ABS_WHEEL,
 				(data[6] << 2) | ((data[7] >> 6) & 3));
 		input_report_abs(input, ABS_TILT_X,
 				 (((data[7] << 1) & 0x7e) | (data[8] >> 7)) - 64);
 		input_report_abs(input, ABS_TILT_Y, (data[8] & 0x7f) - 64);
+		break;
+
+	case 0x05:
+		/* Rotation packet */
+		if (features->type >= INTUOS3S) {
+			/* I3 marker pen rotation */
+			t = (data[6] << 3) | ((data[7] >> 5) & 7);
+			t = (data[7] & 0x20) ? ((t > 900) ? ((t-1) / 2 - 1350) :
+				((t-1) / 2 + 450)) : (450 - t / 2) ;
+			input_report_abs(input, ABS_Z, t);
+		} else {
+			/* 4D mouse 2nd packet */
+			t = (data[6] << 3) | ((data[7] >> 5) & 7);
+			input_report_abs(input, ABS_RZ, (data[7] & 0x20) ?
+				((t - 1) / 2) : -t / 2);
+		}
+		break;
+
+	case 0x04:
+		/* 4D mouse 1st packet */
+		input_report_key(input, BTN_LEFT,   data[8] & 0x01);
+		input_report_key(input, BTN_MIDDLE, data[8] & 0x02);
+		input_report_key(input, BTN_RIGHT,  data[8] & 0x04);
+
+		input_report_key(input, BTN_SIDE,   data[8] & 0x20);
+		input_report_key(input, BTN_EXTRA,  data[8] & 0x10);
+		t = (data[6] << 2) | ((data[7] >> 6) & 3);
+		input_report_abs(input, ABS_THROTTLE, (data[8] & 0x08) ? -t : t);
+		break;
+
+	case 0x06:
+		/* I4 mouse */
+		input_report_key(input, BTN_LEFT,   data[6] & 0x01);
+		input_report_key(input, BTN_MIDDLE, data[6] & 0x02);
+		input_report_key(input, BTN_RIGHT,  data[6] & 0x04);
+		input_report_rel(input, REL_WHEEL, ((data[7] & 0x80) >> 7)
+				 - ((data[7] & 0x40) >> 6));
+		input_report_key(input, BTN_SIDE,   data[6] & 0x08);
+		input_report_key(input, BTN_EXTRA,  data[6] & 0x10);
+
+		input_report_abs(input, ABS_TILT_X,
+			(((data[7] << 1) & 0x7e) | (data[8] >> 7)) - 64);
+		input_report_abs(input, ABS_TILT_Y, (data[8] & 0x7f) - 64);
+		break;
+
+	case 0x08:
+		if (wacom->tool[idx] == BTN_TOOL_MOUSE) {
+			/* 2D mouse packet */
+			input_report_key(input, BTN_LEFT,   data[8] & 0x04);
+			input_report_key(input, BTN_MIDDLE, data[8] & 0x08);
+			input_report_key(input, BTN_RIGHT,  data[8] & 0x10);
+			input_report_rel(input, REL_WHEEL, (data[8] & 0x01)
+					 - ((data[8] & 0x02) >> 1));
+
+			/* I3 2D mouse side buttons */
+			if (features->type >= INTUOS3S && features->type <= INTUOS3L) {
+				input_report_key(input, BTN_SIDE,   data[8] & 0x40);
+				input_report_key(input, BTN_EXTRA,  data[8] & 0x20);
+			}
+		}
+		else if (wacom->tool[idx] == BTN_TOOL_LENS) {
+			/* Lens cursor packets */
+			input_report_key(input, BTN_LEFT,   data[8] & 0x01);
+			input_report_key(input, BTN_MIDDLE, data[8] & 0x02);
+			input_report_key(input, BTN_RIGHT,  data[8] & 0x04);
+			input_report_key(input, BTN_SIDE,   data[8] & 0x10);
+			input_report_key(input, BTN_EXTRA,  data[8] & 0x08);
+		}
+		break;
+
+	case 0x07:
+	case 0x09:
+	case 0x0b:
+	case 0x0c:
+	case 0x0d:
+	case 0x0e:
+	case 0x0f:
+		/* unhandled */
+		break;
 	}
+
+	input_report_abs(input, ABS_MISC, wacom->id[idx]); /* report tool id */
+	input_report_key(input, wacom->tool[idx], 1);
+	input_event(input, EV_MSC, MSC_SERIAL, wacom->serial[idx]);
+	wacom->reporting_data = true;
+	return 2;
 }
 
 static int wacom_intuos_irq(struct wacom_wac *wacom)
 {
-	struct wacom_features *features = &wacom->features;
 	unsigned char *data = wacom->data;
 	struct input_dev *input = wacom->pen_input;
-	unsigned int t;
-	int idx = 0, result;
+	int result;
 
 	if (data[0] != WACOM_REPORT_PENABLED &&
-	    data[0] != WACOM_REPORT_INTUOSREAD &&
-	    data[0] != WACOM_REPORT_INTUOSWRITE &&
+	    data[0] != WACOM_REPORT_INTUOS_ID1 &&
+	    data[0] != WACOM_REPORT_INTUOS_ID2 &&
 	    data[0] != WACOM_REPORT_INTUOSPAD &&
 	    data[0] != WACOM_REPORT_INTUOS_PEN &&
 	    data[0] != WACOM_REPORT_CINTIQ &&
@@ -810,339 +1043,22 @@ static int wacom_intuos_irq(struct wacom_wac *wacom)
                 return 0;
 	}
 
-	/* tool number */
-	if (features->type == INTUOS)
-		idx = data[1] & 0x01;
-
-	/* pad packets. Works as a second tool and is always in prox */
-	if (data[0] == WACOM_REPORT_INTUOSPAD || data[0] == WACOM_REPORT_INTUOS5PAD ||
-	    data[0] == WACOM_REPORT_CINTIQPAD) {
-		input = wacom->pad_input;
-		if (features->type >= INTUOS4S && features->type <= INTUOS4L) {
-			input_report_key(input, BTN_0, (data[2] & 0x01));
-			input_report_key(input, BTN_1, (data[3] & 0x01));
-			input_report_key(input, BTN_2, (data[3] & 0x02));
-			input_report_key(input, BTN_3, (data[3] & 0x04));
-			input_report_key(input, BTN_4, (data[3] & 0x08));
-			input_report_key(input, BTN_5, (data[3] & 0x10));
-			input_report_key(input, BTN_6, (data[3] & 0x20));
-			if (data[1] & 0x80) {
-				input_report_abs(input, ABS_WHEEL, (data[1] & 0x7f));
-			} else {
-				/* Out of proximity, clear wheel value. */
-				input_report_abs(input, ABS_WHEEL, 0);
-			}
-			if (features->type != INTUOS4S) {
-				input_report_key(input, BTN_7, (data[3] & 0x40));
-				input_report_key(input, BTN_8, (data[3] & 0x80));
-			}
-			if (data[1] | (data[2] & 0x01) | data[3]) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else if (features->type == DTK) {
-			input_report_key(input, BTN_0, (data[6] & 0x01));
-			input_report_key(input, BTN_1, (data[6] & 0x02));
-			input_report_key(input, BTN_2, (data[6] & 0x04));
-			input_report_key(input, BTN_3, (data[6] & 0x08));
-			input_report_key(input, BTN_4, (data[6] & 0x10));
-			input_report_key(input, BTN_5, (data[6] & 0x20));
-			if (data[6] & 0x3f) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else if (features->type == WACOM_13HD) {
-			input_report_key(input, BTN_0, (data[3] & 0x01));
-			input_report_key(input, BTN_1, (data[4] & 0x01));
-			input_report_key(input, BTN_2, (data[4] & 0x02));
-			input_report_key(input, BTN_3, (data[4] & 0x04));
-			input_report_key(input, BTN_4, (data[4] & 0x08));
-			input_report_key(input, BTN_5, (data[4] & 0x10));
-			input_report_key(input, BTN_6, (data[4] & 0x20));
-			input_report_key(input, BTN_7, (data[4] & 0x40));
-			input_report_key(input, BTN_8, (data[4] & 0x80));
-			if ((data[3] & 0x01) | data[4]) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else if (features->type == WACOM_24HD) {
-			input_report_key(input, BTN_0, (data[6] & 0x01));
-			input_report_key(input, BTN_1, (data[6] & 0x02));
-			input_report_key(input, BTN_2, (data[6] & 0x04));
-			input_report_key(input, BTN_3, (data[6] & 0x08));
-			input_report_key(input, BTN_4, (data[6] & 0x10));
-			input_report_key(input, BTN_5, (data[6] & 0x20));
-			input_report_key(input, BTN_6, (data[6] & 0x40));
-			input_report_key(input, BTN_7, (data[6] & 0x80));
-			input_report_key(input, BTN_8, (data[8] & 0x01));
-			input_report_key(input, BTN_9, (data[8] & 0x02));
-			input_report_key(input, BTN_A, (data[8] & 0x04));
-			input_report_key(input, BTN_B, (data[8] & 0x08));
-			input_report_key(input, BTN_C, (data[8] & 0x10));
-			input_report_key(input, BTN_X, (data[8] & 0x20));
-			input_report_key(input, BTN_Y, (data[8] & 0x40));
-			input_report_key(input, BTN_Z, (data[8] & 0x80));
-
-			/*
-			 * Three "buttons" are available on the 24HD which are
-			 * physically implemented as a touchstrip. Each button
-			 * is approximately 3 bits wide with a 2 bit spacing.
-			 * The raw touchstrip bits are stored at:
-			 *    ((data[3] & 0x1f) << 8) | data[4])
-			 */
-			input_report_key(input, KEY_PROG1, data[4] & 0x07);
-			input_report_key(input, KEY_PROG2, data[4] & 0xE0);
-			input_report_key(input, KEY_PROG3, data[3] & 0x1C);
-
-			if (data[1] & 0x80) {
-				input_report_abs(input, ABS_WHEEL, (data[1] & 0x7f));
-			} else {
-				/* Out of proximity, clear wheel value. */
-				input_report_abs(input, ABS_WHEEL, 0);
-			}
-
-			if (data[2] & 0x80) {
-				input_report_abs(input, ABS_THROTTLE, (data[2] & 0x7f));
-			} else {
-				/* Out of proximity, clear second wheel value. */
-				input_report_abs(input, ABS_THROTTLE, 0);
-			}
-
-			if (data[1] | data[2] | (data[3] & 0x1f) | data[4] | data[6] | data[8]) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else if (features->type == WACOM_27QHD) {
-			input_report_key(input, KEY_PROG1, data[2] & 0x01);
-			input_report_key(input, KEY_PROG2, data[2] & 0x02);
-			input_report_key(input, KEY_PROG3, data[2] & 0x04);
-
-			input_report_abs(input, ABS_X, be16_to_cpup((__be16 *)&data[4]));
-			input_report_abs(input, ABS_Y, be16_to_cpup((__be16 *)&data[6]));
-			input_report_abs(input, ABS_Z, be16_to_cpup((__be16 *)&data[8]));
-			if ((data[2] & 0x07) | data[4] | data[5] | data[6] | data[7] | data[8] | data[9]) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else if (features->type == CINTIQ_HYBRID) {
-			/*
-			 * Do not send hardware buttons under Android. They
-			 * are already sent to the system through GPIO (and
-			 * have different meaning).
-			 */
-			input_report_key(input, BTN_1, (data[4] & 0x01));
-			input_report_key(input, BTN_2, (data[4] & 0x02));
-			input_report_key(input, BTN_3, (data[4] & 0x04));
-			input_report_key(input, BTN_4, (data[4] & 0x08));
-
-			input_report_key(input, BTN_5, (data[4] & 0x10));  /* Right  */
-			input_report_key(input, BTN_6, (data[4] & 0x20));  /* Up     */
-			input_report_key(input, BTN_7, (data[4] & 0x40));  /* Left   */
-			input_report_key(input, BTN_8, (data[4] & 0x80));  /* Down   */
-			input_report_key(input, BTN_0, (data[3] & 0x01));  /* Center */
-
-			if (data[4] | (data[3] & 0x01)) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-
-		} else if (features->type == CINTIQ_COMPANION_2) {
-			input_report_key(input, BTN_1, (data[1] & 0x02));
-			input_report_key(input, BTN_2, (data[2] & 0x01));
-			input_report_key(input, BTN_3, (data[2] & 0x02));
-			input_report_key(input, BTN_4, (data[2] & 0x04));
-			input_report_key(input, BTN_5, (data[2] & 0x08));
-			input_report_key(input, BTN_6, (data[1] & 0x04));
-
-			input_report_key(input, BTN_7, (data[2] & 0x10));  /* Right  */
-			input_report_key(input, BTN_8, (data[2] & 0x20));  /* Up	 */
-			input_report_key(input, BTN_9, (data[2] & 0x40));  /* Left   */
-			input_report_key(input, BTN_A, (data[2] & 0x80));  /* Down   */
-			input_report_key(input, BTN_0, (data[1] & 0x01));  /* Center */
-
-			if (data[2] | (data[1] & 0x07)) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-
-		} else if (features->type >= INTUOS5S && features->type <= INTUOSPL) {
-			int i;
-
-			/* Touch ring mode switch has no capacitive sensor */
-			input_report_key(input, BTN_0, (data[3] & 0x01));
-
-			/*
-			 * ExpressKeys on Intuos5/Intuos Pro have a capacitive sensor in
-			 * addition to the mechanical switch. Switch data is
-			 * stored in data[4], capacitive data in data[5].
-			 */
-			for (i = 0; i < 8; i++)
-				input_report_key(input, BTN_1 + i, data[4] & (1 << i));
-
-			if (data[2] & 0x80) {
-				input_report_abs(input, ABS_WHEEL, (data[2] & 0x7f));
-			} else {
-				/* Out of proximity, clear wheel value. */
-				input_report_abs(input, ABS_WHEEL, 0);
-			}
-
-			if (data[2] | (data[3] & 0x01) | data[4] | data[5]) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		} else {
-			if (features->type == WACOM_21UX2 || features->type == WACOM_22HD) {
-				input_report_key(input, BTN_0, (data[5] & 0x01));
-				input_report_key(input, BTN_1, (data[6] & 0x01));
-				input_report_key(input, BTN_2, (data[6] & 0x02));
-				input_report_key(input, BTN_3, (data[6] & 0x04));
-				input_report_key(input, BTN_4, (data[6] & 0x08));
-				input_report_key(input, BTN_5, (data[6] & 0x10));
-				input_report_key(input, BTN_6, (data[6] & 0x20));
-				input_report_key(input, BTN_7, (data[6] & 0x40));
-				input_report_key(input, BTN_8, (data[6] & 0x80));
-				input_report_key(input, BTN_9, (data[7] & 0x01));
-				input_report_key(input, BTN_A, (data[8] & 0x01));
-				input_report_key(input, BTN_B, (data[8] & 0x02));
-				input_report_key(input, BTN_C, (data[8] & 0x04));
-				input_report_key(input, BTN_X, (data[8] & 0x08));
-				input_report_key(input, BTN_Y, (data[8] & 0x10));
-				input_report_key(input, BTN_Z, (data[8] & 0x20));
-				input_report_key(input, BTN_BASE, (data[8] & 0x40));
-				input_report_key(input, BTN_BASE2, (data[8] & 0x80));
-
-				if (features->type == WACOM_22HD) {
-					input_report_key(input, KEY_PROG1, data[9] & 0x01);
-					input_report_key(input, KEY_PROG2, data[9] & 0x02);
-					input_report_key(input, KEY_PROG3, data[9] & 0x04);
-				}
-			} else {
-				input_report_key(input, BTN_0, (data[5] & 0x01));
-				input_report_key(input, BTN_1, (data[5] & 0x02));
-				input_report_key(input, BTN_2, (data[5] & 0x04));
-				input_report_key(input, BTN_3, (data[5] & 0x08));
-				input_report_key(input, BTN_4, (data[6] & 0x01));
-				input_report_key(input, BTN_5, (data[6] & 0x02));
-				input_report_key(input, BTN_6, (data[6] & 0x04));
-				input_report_key(input, BTN_7, (data[6] & 0x08));
-				input_report_key(input, BTN_8, (data[5] & 0x10));
-				input_report_key(input, BTN_9, (data[6] & 0x10));
-			}
-			input_report_abs(input, ABS_RX, ((data[1] & 0x1f) << 8) | data[2]);
-			input_report_abs(input, ABS_RY, ((data[3] & 0x1f) << 8) | data[4]);
-
-			if ((data[5] & 0x1f) | data[6] | (data[1] & 0x1f) |
-				data[2] | (data[3] & 0x1f) | data[4] | data[8] |
-				(data[7] & 0x01)) {
-				input_report_abs(input, ABS_MISC, PAD_DEVICE_ID);
-			} else {
-				input_report_abs(input, ABS_MISC, 0);
-			}
-		}
-                return 1;
-	}
+	/* process pad events */
+	result = wacom_intuos_pad(wacom);
+	if (result)
+		return result;
 
 	/* process in/out prox events */
 	result = wacom_intuos_inout(wacom);
 	if (result)
-                return result - 1;
-
-	if (features->type >= INTUOS3S) {
-		input_report_abs(input, ABS_X, (data[2] << 9) | (data[3] << 1) | ((data[9] >> 1) & 1));
-		input_report_abs(input, ABS_Y, (data[4] << 9) | (data[5] << 1) | (data[9] & 1));
-		input_report_abs(input, ABS_DISTANCE, ((data[9] >> 2) & 0x3f));
-	} else {
-		input_report_abs(input, ABS_X, be16_to_cpup((__be16 *)&data[2]));
-		input_report_abs(input, ABS_Y, be16_to_cpup((__be16 *)&data[4]));
-		input_report_abs(input, ABS_DISTANCE, ((data[9] >> 3) & 0x1f));
-	}
+		return result - 1;
 
 	/* process general packets */
-	wacom_intuos_general(wacom);
-
-	/* 4D mouse, 2D mouse, marker pen rotation, tilt mouse, or Lens cursor packets */
-	if ((data[1] & 0xbc) == 0xa8 || (data[1] & 0xbe) == 0xb0 || (data[1] & 0xbc) == 0xac) {
-
-		if (data[1] & 0x02) {
-			/* Rotation packet */
-			if (features->type >= INTUOS3S) {
-				/* I3 marker pen rotation */
-				t = (data[6] << 3) | ((data[7] >> 5) & 7);
-				t = (data[7] & 0x20) ? ((t > 900) ? ((t-1) / 2 - 1350) :
-					((t-1) / 2 + 450)) : (450 - t / 2) ;
-				input_report_abs(input, ABS_Z, t);
-			} else {
-				/* 4D mouse rotation packet */
-				t = (data[6] << 3) | ((data[7] >> 5) & 7);
-				input_report_abs(input, ABS_RZ, (data[7] & 0x20) ?
-					((t - 1) / 2) : -t / 2);
-			}
-
-		} else if (!(data[1] & 0x10) && features->type < INTUOS3S) {
-			/* 4D mouse packet */
-			input_report_key(input, BTN_LEFT,   data[8] & 0x01);
-			input_report_key(input, BTN_MIDDLE, data[8] & 0x02);
-			input_report_key(input, BTN_RIGHT,  data[8] & 0x04);
-
-			input_report_key(input, BTN_SIDE,   data[8] & 0x20);
-			input_report_key(input, BTN_EXTRA,  data[8] & 0x10);
-			t = (data[6] << 2) | ((data[7] >> 6) & 3);
-			input_report_abs(input, ABS_THROTTLE, (data[8] & 0x08) ? -t : t);
-
-		} else if (wacom->tool[idx] == BTN_TOOL_MOUSE) {
-			/* I4 mouse */
-			if (features->type >= INTUOS4S && features->type <= INTUOSPL) {
-				input_report_key(input, BTN_LEFT,   data[6] & 0x01);
-				input_report_key(input, BTN_MIDDLE, data[6] & 0x02);
-				input_report_key(input, BTN_RIGHT,  data[6] & 0x04);
-				input_report_rel(input, REL_WHEEL, ((data[7] & 0x80) >> 7)
-						 - ((data[7] & 0x40) >> 6));
-				input_report_key(input, BTN_SIDE,   data[6] & 0x08);
-				input_report_key(input, BTN_EXTRA,  data[6] & 0x10);
-
-				input_report_abs(input, ABS_TILT_X,
-					(((data[7] << 1) & 0x7e) | (data[8] >> 7)) - 64);
-				input_report_abs(input, ABS_TILT_Y, (data[8] & 0x7f) - 64);
-			} else {
-				/* 2D mouse packet */
-				input_report_key(input, BTN_LEFT,   data[8] & 0x04);
-				input_report_key(input, BTN_MIDDLE, data[8] & 0x08);
-				input_report_key(input, BTN_RIGHT,  data[8] & 0x10);
-				input_report_rel(input, REL_WHEEL, (data[8] & 0x01)
-						 - ((data[8] & 0x02) >> 1));
-
-				/* I3 2D mouse side buttons */
-				if (features->type >= INTUOS3S && features->type <= INTUOS3L) {
-					input_report_key(input, BTN_SIDE,   data[8] & 0x40);
-					input_report_key(input, BTN_EXTRA,  data[8] & 0x20);
-				}
-			}
-		} else if ((features->type < INTUOS3S || features->type == INTUOS3L ||
-				features->type == INTUOS4L || features->type == INTUOS5L ||
-				features->type == INTUOSPL) &&
-			   wacom->tool[idx] == BTN_TOOL_LENS) {
-			/* Lens cursor packets */
-			input_report_key(input, BTN_LEFT,   data[8] & 0x01);
-			input_report_key(input, BTN_MIDDLE, data[8] & 0x02);
-			input_report_key(input, BTN_RIGHT,  data[8] & 0x04);
-			input_report_key(input, BTN_SIDE,   data[8] & 0x10);
-			input_report_key(input, BTN_EXTRA,  data[8] & 0x08);
-		}
-	}
+	result = wacom_intuos_general(wacom);
+	if (result)
+		return result - 1;
 
-	input_report_abs(input, ABS_MISC, wacom->id[idx]); /* report tool id */
-	input_report_key(input, wacom->tool[idx], 1);
-	input_event(input, EV_MSC, MSC_SERIAL, wacom->serial[idx]);
-	wacom->reporting_data = true;
-	return 1;
+	return 0;
 }
 
 static int int_dist(int x1, int y1, int x2, int y2)
@@ -2509,7 +2425,7 @@ void wacom_setup_device_quirks(struct wacom *wacom)
 		features->quirks |= WACOM_QUIRK_BATTERY;
 
 	/* quirk for bamboo touch with 2 low res touches */
-	if (features->type == BAMBOO_PT &&
+	if ((features->type == BAMBOO_PT || features->type == BAMBOO_TOUCH) &&
 	    features->pktlen == WACOM_PKGLEN_BBTOUCH) {
 		features->x_max <<= 5;
 		features->y_max <<= 5;
@@ -2806,6 +2722,19 @@ static void wacom_setup_numbered_buttons(struct input_dev *input_dev,
 		__set_bit(BTN_BASE + (i-16), input_dev->keybit);
 }
 
+static void wacom_report_numbered_buttons(struct input_dev *input_dev,
+				int button_count, int mask)
+{
+	int i;
+
+	for (i = 0; i < button_count && i < 10; i++)
+		input_report_key(input_dev, BTN_0 + i, mask & (1 << i));
+	for (i = 10; i < button_count && i < 16; i++)
+		input_report_key(input_dev, BTN_A + (i-10), mask & (1 << i));
+	for (i = 16; i < button_count && i < 18; i++)
+		input_report_key(input_dev, BTN_BASE + (i-16), mask & (1 << i));
+}
+
 int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
 				   struct wacom_wac *wacom_wac)
 {
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 877c24a5df94..25baa7f29599 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -47,8 +47,8 @@
 /* wacom data packet report IDs */
 #define WACOM_REPORT_PENABLED		2
 #define WACOM_REPORT_PENABLED_BT	3
-#define WACOM_REPORT_INTUOSREAD		5
-#define WACOM_REPORT_INTUOSWRITE	6
+#define WACOM_REPORT_INTUOS_ID1		5
+#define WACOM_REPORT_INTUOS_ID2		6
 #define WACOM_REPORT_INTUOSPAD		12
 #define WACOM_REPORT_INTUOS5PAD		3
 #define WACOM_REPORT_DTUSPAD		21
@@ -70,6 +70,7 @@
 #define WACOM_REPORT_DEVICE_LIST	16
 #define WACOM_REPORT_INTUOS_PEN		16
 #define WACOM_REPORT_REMOTE		17
+#define WACOM_REPORT_INTUOSHT2_ID	8
 
 /* device quirks */
 #define WACOM_QUIRK_BBTOUCH_LOWRES	0x0001
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index c8487894b312..c43318d3416e 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -932,6 +932,17 @@ MODULE_DEVICE_TABLE(dmi, i8k_dmi_table);
 static struct dmi_system_id i8k_blacklist_dmi_table[] __initdata = {
 	{
 		/*
+		 * CPU fan speed going up and down on Dell Studio XPS 8000
+		 * for unknown reasons.
+		 */
+		.ident = "Dell Studio XPS 8000",
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Studio XPS 8000"),
+		},
+	},
+	{
+		/*
 		 * CPU fan speed going up and down on Dell Studio XPS 8100
 		 * for unknown reasons.
 		 */
diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
index f77eb971ce95..4f695d8fcafa 100644
--- a/drivers/hwmon/fam15h_power.c
+++ b/drivers/hwmon/fam15h_power.c
@@ -90,7 +90,15 @@ static ssize_t show_power(struct device *dev,
 	pci_bus_read_config_dword(f4->bus, PCI_DEVFN(PCI_SLOT(f4->devfn), 5),
 				  REG_TDP_LIMIT3, &val);
 
-	tdp_limit = val >> 16;
+	/*
+	 * On Carrizo and later platforms, ApmTdpLimit bit field
+	 * is extended to 16:31 from 16:28.
+	 */
+	if (boot_cpu_data.x86 == 0x15 && boot_cpu_data.x86_model >= 0x60)
+		tdp_limit = val >> 16;
+	else
+		tdp_limit = (val >> 16) & 0x1fff;
+
 	curr_pwr_watts = ((u64)(tdp_limit +
 				data->base_tdp)) << running_avg_range;
 	curr_pwr_watts -= running_avg_capture;
diff --git a/drivers/hwmon/sch56xx-common.c b/drivers/hwmon/sch56xx-common.c
index 738681983284..68c350c704fb 100644
--- a/drivers/hwmon/sch56xx-common.c
+++ b/drivers/hwmon/sch56xx-common.c
@@ -30,7 +30,6 @@
 #include <linux/watchdog.h>
 #include <linux/miscdevice.h>
 #include <linux/uaccess.h>
-#include <linux/kref.h>
 #include <linux/slab.h>
 #include "sch56xx-common.h"
 
@@ -67,7 +66,6 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
 struct sch56xx_watchdog_data {
 	u16 addr;
 	struct mutex *io_lock;
-	struct kref kref;
 	struct watchdog_info wdinfo;
 	struct watchdog_device wddev;
 	u8 watchdog_preset;
@@ -258,15 +256,6 @@ EXPORT_SYMBOL(sch56xx_read_virtual_reg12);
  * Watchdog routines
  */
 
-/* Release our data struct when we're unregistered *and*
-   all references to our watchdog device are released */
-static void watchdog_release_resources(struct kref *r)
-{
-	struct sch56xx_watchdog_data *data =
-		container_of(r, struct sch56xx_watchdog_data, kref);
-	kfree(data);
-}
-
 static int watchdog_set_timeout(struct watchdog_device *wddev,
 				unsigned int timeout)
 {
@@ -395,28 +384,12 @@ static int watchdog_stop(struct watchdog_device *wddev)
 	return 0;
 }
 
-static void watchdog_ref(struct watchdog_device *wddev)
-{
-	struct sch56xx_watchdog_data *data = watchdog_get_drvdata(wddev);
-
-	kref_get(&data->kref);
-}
-
-static void watchdog_unref(struct watchdog_device *wddev)
-{
-	struct sch56xx_watchdog_data *data = watchdog_get_drvdata(wddev);
-
-	kref_put(&data->kref, watchdog_release_resources);
-}
-
 static const struct watchdog_ops watchdog_ops = {
 	.owner		= THIS_MODULE,
 	.start		= watchdog_start,
 	.stop		= watchdog_stop,
 	.ping		= watchdog_trigger,
 	.set_timeout	= watchdog_set_timeout,
-	.ref		= watchdog_ref,
-	.unref		= watchdog_unref,
 };
 
 struct sch56xx_watchdog_data *sch56xx_watchdog_register(struct device *parent,
@@ -448,7 +421,6 @@ struct sch56xx_watchdog_data *sch56xx_watchdog_register(struct device *parent,
 
 	data->addr = addr;
 	data->io_lock = io_lock;
-	kref_init(&data->kref);
 
 	strlcpy(data->wdinfo.identity, "sch56xx watchdog",
 		sizeof(data->wdinfo.identity));
@@ -494,8 +466,7 @@ EXPORT_SYMBOL(sch56xx_watchdog_register);
 void sch56xx_watchdog_unregister(struct sch56xx_watchdog_data *data)
 {
 	watchdog_unregister_device(&data->wddev);
-	kref_put(&data->kref, watchdog_release_resources);
-	/* Don't touch data after this it may have been free-ed! */
+	kfree(data);
 }
 EXPORT_SYMBOL(sch56xx_watchdog_unregister);
 
diff --git a/drivers/i2c/busses/i2c-designware-core.c b/drivers/i2c/busses/i2c-designware-core.c
index ba9732c236c5..10fbd6d841e0 100644
--- a/drivers/i2c/busses/i2c-designware-core.c
+++ b/drivers/i2c/busses/i2c-designware-core.c
@@ -874,7 +874,8 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 	i2c_set_adapdata(adap, dev);
 
 	i2c_dw_disable_int(dev);
-	r = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr, IRQF_SHARED,
+	r = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr,
+			     IRQF_SHARED | IRQF_COND_SUSPEND,
 			     dev_name(dev->dev), dev);
 	if (r) {
 		dev_err(dev->dev, "failure requesting irq %i: %d\n",
diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
index e04598595073..93f2895383ee 100644
--- a/drivers/i2c/busses/i2c-piix4.c
+++ b/drivers/i2c/busses/i2c-piix4.c
@@ -137,10 +137,11 @@ static const struct dmi_system_id piix4_dmi_ibm[] = {
 };
 
 /* SB800 globals */
+static DEFINE_MUTEX(piix4_mutex_sb800);
 static const char *piix4_main_port_names_sb800[PIIX4_MAX_ADAPTERS] = {
-	"SDA0", "SDA2", "SDA3", "SDA4"
+	" port 0", " port 2", " port 3", " port 4"
 };
-static const char *piix4_aux_port_name_sb800 = "SDA1";
+static const char *piix4_aux_port_name_sb800 = " port 1";
 
 struct i2c_piix4_adapdata {
 	unsigned short smba;
@@ -148,7 +149,6 @@ struct i2c_piix4_adapdata {
 	/* SB800 */
 	bool sb800_main;
 	unsigned short port;
-	struct mutex *mutex;
 };
 
 static int piix4_setup(struct pci_dev *PIIX4_dev,
@@ -275,10 +275,12 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
 	else
 		smb_en = (aux) ? 0x28 : 0x2c;
 
+	mutex_lock(&piix4_mutex_sb800);
 	outb_p(smb_en, SB800_PIIX4_SMB_IDX);
 	smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
 	outb_p(smb_en + 1, SB800_PIIX4_SMB_IDX);
 	smba_en_hi = inb_p(SB800_PIIX4_SMB_IDX + 1);
+	mutex_unlock(&piix4_mutex_sb800);
 
 	if (!smb_en) {
 		smb_en_status = smba_en_lo & 0x10;
@@ -559,7 +561,7 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr,
 	u8 port;
 	int retval;
 
-	mutex_lock(adapdata->mutex);
+	mutex_lock(&piix4_mutex_sb800);
 
 	outb_p(SB800_PIIX4_PORT_IDX, SB800_PIIX4_SMB_IDX);
 	smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
@@ -574,7 +576,7 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr,
 
 	outb_p(smba_en_lo, SB800_PIIX4_SMB_IDX + 1);
 
-	mutex_unlock(adapdata->mutex);
+	mutex_unlock(&piix4_mutex_sb800);
 
 	return retval;
 }
@@ -625,6 +627,7 @@ static struct i2c_adapter *piix4_main_adapters[PIIX4_MAX_ADAPTERS];
 static struct i2c_adapter *piix4_aux_adapter;
 
 static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
+			     bool sb800_main, unsigned short port,
 			     const char *name, struct i2c_adapter **padap)
 {
 	struct i2c_adapter *adap;
@@ -639,7 +642,8 @@ static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
 
 	adap->owner = THIS_MODULE;
 	adap->class = I2C_CLASS_HWMON | I2C_CLASS_SPD;
-	adap->algo = &smbus_algorithm;
+	adap->algo = sb800_main ? &piix4_smbus_algorithm_sb800
+				: &smbus_algorithm;
 
 	adapdata = kzalloc(sizeof(*adapdata), GFP_KERNEL);
 	if (adapdata == NULL) {
@@ -649,12 +653,14 @@ static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
 	}
 
 	adapdata->smba = smba;
+	adapdata->sb800_main = sb800_main;
+	adapdata->port = port;
 
 	/* set up the sysfs linkage to our parent device */
 	adap->dev.parent = &dev->dev;
 
 	snprintf(adap->name, sizeof(adap->name),
-		"SMBus PIIX4 adapter %s at %04x", name, smba);
+		"SMBus PIIX4 adapter%s at %04x", name, smba);
 
 	i2c_set_adapdata(adap, adapdata);
 
@@ -673,30 +679,16 @@ static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
 
 static int piix4_add_adapters_sb800(struct pci_dev *dev, unsigned short smba)
 {
-	struct mutex *mutex;
 	struct i2c_piix4_adapdata *adapdata;
 	int port;
 	int retval;
 
-	mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
-	if (mutex == NULL)
-		return -ENOMEM;
-
-	mutex_init(mutex);
-
 	for (port = 0; port < PIIX4_MAX_ADAPTERS; port++) {
-		retval = piix4_add_adapter(dev, smba,
+		retval = piix4_add_adapter(dev, smba, true, port,
 					   piix4_main_port_names_sb800[port],
 					   &piix4_main_adapters[port]);
 		if (retval < 0)
 			goto error;
-
-		piix4_main_adapters[port]->algo = &piix4_smbus_algorithm_sb800;
-
-		adapdata = i2c_get_adapdata(piix4_main_adapters[port]);
-		adapdata->sb800_main = true;
-		adapdata->port = port;
-		adapdata->mutex = mutex;
 	}
 
 	return retval;
@@ -714,19 +706,20 @@ error:
 		}
 	}
 
-	kfree(mutex);
-
 	return retval;
 }
 
 static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	int retval;
+	bool is_sb800 = false;
 
 	if ((dev->vendor == PCI_VENDOR_ID_ATI &&
 	     dev->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
 	     dev->revision >= 0x40) ||
 	    dev->vendor == PCI_VENDOR_ID_AMD) {
+		is_sb800 = true;
+
 		if (!request_region(SB800_PIIX4_SMB_IDX, 2, "smba_idx")) {
 			dev_err(&dev->dev,
 			"SMBus base address index region 0x%x already in use!\n",
@@ -756,7 +749,7 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id)
 			return retval;
 
 		/* Try to register main SMBus adapter, give up if we can't */
-		retval = piix4_add_adapter(dev, retval, "main",
+		retval = piix4_add_adapter(dev, retval, false, 0, "",
 					   &piix4_main_adapters[0]);
 		if (retval < 0)
 			return retval;
@@ -783,7 +776,8 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	if (retval > 0) {
 		/* Try to add the aux adapter if it exists,
 		 * piix4_add_adapter will clean up if this fails */
-		piix4_add_adapter(dev, retval, piix4_aux_port_name_sb800,
+		piix4_add_adapter(dev, retval, false, 0,
+				  is_sb800 ? piix4_aux_port_name_sb800 : "",
 				  &piix4_aux_adapter);
 	}
 
@@ -798,10 +792,8 @@ static void piix4_adap_remove(struct i2c_adapter *adap)
 		i2c_del_adapter(adap);
 		if (adapdata->port == 0) {
 			release_region(adapdata->smba, SMBIOSIZE);
-			if (adapdata->sb800_main) {
-				kfree(adapdata->mutex);
+			if (adapdata->sb800_main)
 				release_region(SB800_PIIX4_SMB_IDX, 2);
-			}
 		}
 		kfree(adapdata);
 		kfree(adap);
diff --git a/drivers/ide/ide-scan-pci.c b/drivers/ide/ide-scan-pci.c
index c3da53e7bb2b..86aa88aeb3a6 100644
--- a/drivers/ide/ide-scan-pci.c
+++ b/drivers/ide/ide-scan-pci.c
@@ -107,5 +107,4 @@ static int __init ide_scan_pcibus(void)
 
 	return 0;
 }
-
-module_init(ide_scan_pcibus);
+device_initcall(ide_scan_pcibus);
diff --git a/drivers/ide/ide.c b/drivers/ide/ide.c
index f086ef387475..d127ace6aa57 100644
--- a/drivers/ide/ide.c
+++ b/drivers/ide/ide.c
@@ -178,17 +178,17 @@ MODULE_PARM_DESC(pci_clock, "PCI bus clock frequency (in MHz)");
 
 static int ide_set_dev_param_mask(const char *s, const struct kernel_param *kp)
 {
-	int a, b, i, j = 1;
+	unsigned int a, b, i, j = 1;
 	unsigned int *dev_param_mask = (unsigned int *)kp->arg;
 
 	/* controller . device (0 or 1) [ : 1 (set) | 0 (clear) ] */
-	if (sscanf(s, "%d.%d:%d", &a, &b, &j) != 3 &&
-	    sscanf(s, "%d.%d", &a, &b) != 2)
+	if (sscanf(s, "%u.%u:%u", &a, &b, &j) != 3 &&
+	    sscanf(s, "%u.%u", &a, &b) != 2)
 		return -EINVAL;
 
 	i = a * MAX_DRIVES + b;
 
-	if (i >= MAX_HWIFS * MAX_DRIVES || j < 0 || j > 1)
+	if (i >= MAX_HWIFS * MAX_DRIVES || j > 1)
 		return -EINVAL;
 
 	if (j)
@@ -246,17 +246,17 @@ static struct chs_geom ide_disks_chs[MAX_HWIFS * MAX_DRIVES];
 
 static int ide_set_disk_chs(const char *str, struct kernel_param *kp)
 {
-	int a, b, c = 0, h = 0, s = 0, i, j = 1;
+	unsigned int a, b, c = 0, h = 0, s = 0, i, j = 1;
 
 	/* controller . device (0 or 1) : Cylinders , Heads , Sectors */
 	/* controller . device (0 or 1) : 1 (use CHS) | 0 (ignore CHS) */
-	if (sscanf(str, "%d.%d:%d,%d,%d", &a, &b, &c, &h, &s) != 5 &&
-	    sscanf(str, "%d.%d:%d", &a, &b, &j) != 3)
+	if (sscanf(str, "%u.%u:%u,%u,%u", &a, &b, &c, &h, &s) != 5 &&
+	    sscanf(str, "%u.%u:%u", &a, &b, &j) != 3)
 		return -EINVAL;
 
 	i = a * MAX_DRIVES + b;
 
-	if (i >= MAX_HWIFS * MAX_DRIVES || j < 0 || j > 1)
+	if (i >= MAX_HWIFS * MAX_DRIVES || j > 1)
 		return -EINVAL;
 
 	if (c > INT_MAX || h > 255 || s > 255)
diff --git a/drivers/ide/it821x.c b/drivers/ide/it821x.c
index f01ba4606be0..04029d18a696 100644
--- a/drivers/ide/it821x.c
+++ b/drivers/ide/it821x.c
@@ -508,7 +508,7 @@ static void it821x_quirkproc(ide_drive_t *drive)
 
 }
 
-static struct ide_dma_ops it821x_pass_through_dma_ops = {
+static const struct ide_dma_ops it821x_pass_through_dma_ops = {
 	.dma_host_set		= ide_dma_host_set,
 	.dma_setup		= ide_dma_setup,
 	.dma_start		= it821x_dma_start,
diff --git a/drivers/ide/trm290.c b/drivers/ide/trm290.c
index 0069f6ce74cf..d550b379b0f1 100644
--- a/drivers/ide/trm290.c
+++ b/drivers/ide/trm290.c
@@ -314,7 +314,7 @@ static const struct ide_tp_ops trm290_tp_ops = {
 	.output_data		= ide_output_data,
 };
 
-static struct ide_dma_ops trm290_dma_ops = {
+static const struct ide_dma_ops trm290_dma_ops = {
 	.dma_host_set		= trm290_dma_host_set,
 	.dma_setup 		= trm290_dma_setup,
 	.dma_start 		= trm290_dma_start,
diff --git a/drivers/iio/adc/exynos_adc.c b/drivers/iio/adc/exynos_adc.c
index 3a2dbb3b4926..c15756d7bf7f 100644
--- a/drivers/iio/adc/exynos_adc.c
+++ b/drivers/iio/adc/exynos_adc.c
@@ -35,6 +35,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/of_platform.h>
 #include <linux/err.h>
+#include <linux/input.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/machine.h>
@@ -42,12 +43,18 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 
+#include <linux/platform_data/touchscreen-s3c2410.h>
+
 /* S3C/EXYNOS4412/5250 ADC_V1 registers definitions */
 #define ADC_V1_CON(x)		((x) + 0x00)
+#define ADC_V1_TSC(x)		((x) + 0x04)
 #define ADC_V1_DLY(x)		((x) + 0x08)
 #define ADC_V1_DATX(x)		((x) + 0x0C)
+#define ADC_V1_DATY(x)		((x) + 0x10)
+#define ADC_V1_UPDN(x)		((x) + 0x14)
 #define ADC_V1_INTCLR(x)	((x) + 0x18)
 #define ADC_V1_MUX(x)		((x) + 0x1c)
+#define ADC_V1_CLRINTPNDNUP(x)	((x) + 0x20)
 
 /* S3C2410 ADC registers definitions */
 #define ADC_S3C2410_MUX(x)	((x) + 0x18)
@@ -71,6 +78,30 @@
 #define ADC_S3C2410_DATX_MASK	0x3FF
 #define ADC_S3C2416_CON_RES_SEL	(1u << 3)
 
+/* touch screen always uses channel 0 */
+#define ADC_S3C2410_MUX_TS	0
+
+/* ADCTSC Register Bits */
+#define ADC_S3C2443_TSC_UD_SEN		(1u << 8)
+#define ADC_S3C2410_TSC_YM_SEN		(1u << 7)
+#define ADC_S3C2410_TSC_YP_SEN		(1u << 6)
+#define ADC_S3C2410_TSC_XM_SEN		(1u << 5)
+#define ADC_S3C2410_TSC_XP_SEN		(1u << 4)
+#define ADC_S3C2410_TSC_PULL_UP_DISABLE	(1u << 3)
+#define ADC_S3C2410_TSC_AUTO_PST	(1u << 2)
+#define ADC_S3C2410_TSC_XY_PST(x)	(((x) & 0x3) << 0)
+
+#define ADC_TSC_WAIT4INT (ADC_S3C2410_TSC_YM_SEN | \
+			 ADC_S3C2410_TSC_YP_SEN | \
+			 ADC_S3C2410_TSC_XP_SEN | \
+			 ADC_S3C2410_TSC_XY_PST(3))
+
+#define ADC_TSC_AUTOPST	(ADC_S3C2410_TSC_YM_SEN | \
+			 ADC_S3C2410_TSC_YP_SEN | \
+			 ADC_S3C2410_TSC_XP_SEN | \
+			 ADC_S3C2410_TSC_AUTO_PST | \
+			 ADC_S3C2410_TSC_XY_PST(0))
+
 /* Bit definitions for ADC_V2 */
 #define ADC_V2_CON1_SOFT_RESET	(1u << 2)
 
@@ -88,7 +119,9 @@
 /* Bit definitions common for ADC_V1 and ADC_V2 */
 #define ADC_CON_EN_START	(1u << 0)
 #define ADC_CON_EN_START_MASK	(0x3 << 0)
+#define ADC_DATX_PRESSED	(1u << 15)
 #define ADC_DATX_MASK		0xFFF
+#define ADC_DATY_MASK		0xFFF
 
 #define EXYNOS_ADC_TIMEOUT	(msecs_to_jiffies(100))
 
@@ -98,17 +131,24 @@
 struct exynos_adc {
 	struct exynos_adc_data	*data;
 	struct device		*dev;
+	struct input_dev	*input;
 	void __iomem		*regs;
 	struct regmap		*pmu_map;
 	struct clk		*clk;
 	struct clk		*sclk;
 	unsigned int		irq;
+	unsigned int		tsirq;
+	unsigned int		delay;
 	struct regulator	*vdd;
 
 	struct completion	completion;
 
 	u32			value;
 	unsigned int            version;
+
+	bool			read_ts;
+	u32			ts_x;
+	u32			ts_y;
 };
 
 struct exynos_adc_data {
@@ -197,6 +237,9 @@ static void exynos_adc_v1_init_hw(struct exynos_adc *info)
 	/* Enable 12-bit ADC resolution */
 	con1 |= ADC_V1_CON_RES;
 	writel(con1, ADC_V1_CON(info->regs));
+
+	/* set touchscreen delay */
+	writel(info->delay, ADC_V1_DLY(info->regs));
 }
 
 static void exynos_adc_v1_exit_hw(struct exynos_adc *info)
@@ -480,8 +523,8 @@ static int exynos_read_raw(struct iio_dev *indio_dev,
 	if (info->data->start_conv)
 		info->data->start_conv(info, chan->address);
 
-	timeout = wait_for_completion_timeout
-			(&info->completion, EXYNOS_ADC_TIMEOUT);
+	timeout = wait_for_completion_timeout(&info->completion,
+					      EXYNOS_ADC_TIMEOUT);
 	if (timeout == 0) {
 		dev_warn(&indio_dev->dev, "Conversion timed out! Resetting\n");
 		if (info->data->init_hw)
@@ -498,13 +541,55 @@ static int exynos_read_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
+static int exynos_read_s3c64xx_ts(struct iio_dev *indio_dev, int *x, int *y)
+{
+	struct exynos_adc *info = iio_priv(indio_dev);
+	unsigned long timeout;
+	int ret;
+
+	mutex_lock(&indio_dev->mlock);
+	info->read_ts = true;
+
+	reinit_completion(&info->completion);
+
+	writel(ADC_S3C2410_TSC_PULL_UP_DISABLE | ADC_TSC_AUTOPST,
+	       ADC_V1_TSC(info->regs));
+
+	/* Select the ts channel to be used and Trigger conversion */
+	info->data->start_conv(info, ADC_S3C2410_MUX_TS);
+
+	timeout = wait_for_completion_timeout(&info->completion,
+					      EXYNOS_ADC_TIMEOUT);
+	if (timeout == 0) {
+		dev_warn(&indio_dev->dev, "Conversion timed out! Resetting\n");
+		if (info->data->init_hw)
+			info->data->init_hw(info);
+		ret = -ETIMEDOUT;
+	} else {
+		*x = info->ts_x;
+		*y = info->ts_y;
+		ret = 0;
+	}
+
+	info->read_ts = false;
+	mutex_unlock(&indio_dev->mlock);
+
+	return ret;
+}
+
 static irqreturn_t exynos_adc_isr(int irq, void *dev_id)
 {
 	struct exynos_adc *info = (struct exynos_adc *)dev_id;
 	u32 mask = info->data->mask;
 
 	/* Read value */
-	info->value = readl(ADC_V1_DATX(info->regs)) & mask;
+	if (info->read_ts) {
+		info->ts_x = readl(ADC_V1_DATX(info->regs));
+		info->ts_y = readl(ADC_V1_DATY(info->regs));
+		writel(ADC_TSC_WAIT4INT | ADC_S3C2443_TSC_UD_SEN, ADC_V1_TSC(info->regs));
+	} else {
+		info->value = readl(ADC_V1_DATX(info->regs)) & mask;
+	}
 
 	/* clear irq */
 	if (info->data->clear_irq)
@@ -515,6 +600,46 @@ static irqreturn_t exynos_adc_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+/*
+ * Here we (ab)use a threaded interrupt handler to stay running
+ * for as long as the touchscreen remains pressed, we report
+ * a new event with the latest data and then sleep until the
+ * next timer tick. This mirrors the behavior of the old
+ * driver, with much less code.
+ */
+static irqreturn_t exynos_ts_isr(int irq, void *dev_id)
+{
+	struct exynos_adc *info = dev_id;
+	struct iio_dev *dev = dev_get_drvdata(info->dev);
+	u32 x, y;
+	bool pressed;
+	int ret;
+
+	while (info->input->users) {
+		ret = exynos_read_s3c64xx_ts(dev, &x, &y);
+		if (ret == -ETIMEDOUT)
+			break;
+
+		pressed = x & y & ADC_DATX_PRESSED;
+		if (!pressed) {
+			input_report_key(info->input, BTN_TOUCH, 0);
+			input_sync(info->input);
+			break;
+		}
+
+		input_report_abs(info->input, ABS_X, x & ADC_DATX_MASK);
+		input_report_abs(info->input, ABS_Y, y & ADC_DATY_MASK);
+		input_report_key(info->input, BTN_TOUCH, 1);
+		input_sync(info->input);
+
+		msleep(1);
+	};
+
+	writel(0, ADC_V1_CLRINTPNDNUP(info->regs));
+
+	return IRQ_HANDLED;
+}
+
 static int exynos_adc_reg_access(struct iio_dev *indio_dev,
 			      unsigned reg, unsigned writeval,
 			      unsigned *readval)
@@ -566,18 +691,72 @@ static int exynos_adc_remove_devices(struct device *dev, void *c)
 	return 0;
 }
 
+static int exynos_adc_ts_open(struct input_dev *dev)
+{
+	struct exynos_adc *info = input_get_drvdata(dev);
+
+	enable_irq(info->tsirq);
+
+	return 0;
+}
+
+static void exynos_adc_ts_close(struct input_dev *dev)
+{
+	struct exynos_adc *info = input_get_drvdata(dev);
+
+	disable_irq(info->tsirq);
+}
+
+static int exynos_adc_ts_init(struct exynos_adc *info)
+{
+	int ret;
+
+	if (info->tsirq <= 0)
+		return -ENODEV;
+
+	info->input = input_allocate_device();
+	if (!info->input)
+		return -ENOMEM;
+
+	info->input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
+	info->input->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH);
+
+	input_set_abs_params(info->input, ABS_X, 0, 0x3FF, 0, 0);
+	input_set_abs_params(info->input, ABS_Y, 0, 0x3FF, 0, 0);
+
+	info->input->name = "S3C24xx TouchScreen";
+	info->input->id.bustype = BUS_HOST;
+	info->input->open = exynos_adc_ts_open;
+	info->input->close = exynos_adc_ts_close;
+
+	input_set_drvdata(info->input, info);
+
+	ret = input_register_device(info->input);
+	if (ret) {
+		input_free_device(info->input);
+		return ret;
+	}
+
+	disable_irq(info->tsirq);
+	ret = request_threaded_irq(info->tsirq, NULL, exynos_ts_isr,
+				   IRQF_ONESHOT, "touchscreen", info);
+	if (ret)
+		input_unregister_device(info->input);
+
+	return ret;
+}
+
 static int exynos_adc_probe(struct platform_device *pdev)
 {
 	struct exynos_adc *info = NULL;
 	struct device_node *np = pdev->dev.of_node;
+	struct s3c2410_ts_mach_info *pdata = dev_get_platdata(&pdev->dev);
 	struct iio_dev *indio_dev = NULL;
 	struct resource	*mem;
+	bool has_ts = false;
 	int ret = -ENODEV;
 	int irq;
 
-	if (!np)
-		return ret;
-
 	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct exynos_adc));
 	if (!indio_dev) {
 		dev_err(&pdev->dev, "failed allocating iio device\n");
@@ -613,8 +792,14 @@ static int exynos_adc_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "no irq resource?\n");
 		return irq;
 	}
-
 	info->irq = irq;
+
+	irq = platform_get_irq(pdev, 1);
+	if (irq == -EPROBE_DEFER)
+		return irq;
+
+	info->tsirq = irq;
+
 	info->dev = &pdev->dev;
 
 	init_completion(&info->completion);
@@ -680,6 +865,22 @@ static int exynos_adc_probe(struct platform_device *pdev)
 	if (info->data->init_hw)
 		info->data->init_hw(info);
 
+	/* leave out any TS related code if unreachable */
+	if (IS_REACHABLE(CONFIG_INPUT)) {
+		has_ts = of_property_read_bool(pdev->dev.of_node,
+					       "has-touchscreen") || pdata;
+	}
+
+	if (pdata)
+		info->delay = pdata->delay;
+	else
+		info->delay = 10000;
+
+	if (has_ts)
+		ret = exynos_adc_ts_init(info);
+	if (ret)
+		goto err_iio;
+
 	ret = of_platform_populate(np, exynos_adc_match, NULL, &indio_dev->dev);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed adding child nodes\n");
@@ -691,6 +892,11 @@ static int exynos_adc_probe(struct platform_device *pdev)
 err_of_populate:
 	device_for_each_child(&indio_dev->dev, NULL,
 				exynos_adc_remove_devices);
+	if (has_ts) {
+		input_unregister_device(info->input);
+		free_irq(info->tsirq, info);
+	}
+err_iio:
 	iio_device_unregister(indio_dev);
 err_irq:
 	free_irq(info->irq, info);
@@ -710,6 +916,10 @@ static int exynos_adc_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct exynos_adc *info = iio_priv(indio_dev);
 
+	if (IS_REACHABLE(CONFIG_INPUT)) {
+		free_irq(info->tsirq, info);
+		input_unregister_device(info->input);
+	}
 	device_for_each_child(&indio_dev->dev, NULL,
 				exynos_adc_remove_devices);
 	iio_device_unregister(indio_dev);
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index fd01f3493fc7..af7cc1e65656 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -433,16 +433,15 @@ ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals)
 		scale_db = true;
 	case IIO_VAL_INT_PLUS_MICRO:
 		if (vals[1] < 0)
-			return sprintf(buf, "-%ld.%06u%s\n", abs(vals[0]),
-					-vals[1],
-				scale_db ? " dB" : "");
+			return sprintf(buf, "-%d.%06u%s\n", abs(vals[0]),
+				       -vals[1], scale_db ? " dB" : "");
 		else
 			return sprintf(buf, "%d.%06u%s\n", vals[0], vals[1],
 				scale_db ? " dB" : "");
 	case IIO_VAL_INT_PLUS_NANO:
 		if (vals[1] < 0)
-			return sprintf(buf, "-%ld.%09u\n", abs(vals[0]),
-					-vals[1]);
+			return sprintf(buf, "-%d.%09u\n", abs(vals[0]),
+				       -vals[1]);
 		else
 			return sprintf(buf, "%d.%09u\n", vals[0], vals[1]);
 	case IIO_VAL_FRACTIONAL:
diff --git a/drivers/iio/industrialio-sw-trigger.c b/drivers/iio/industrialio-sw-trigger.c
index 311f9fe5aa34..8d24fb159cc9 100644
--- a/drivers/iio/industrialio-sw-trigger.c
+++ b/drivers/iio/industrialio-sw-trigger.c
@@ -167,9 +167,7 @@ static int __init iio_sw_trigger_init(void)
 		configfs_register_default_group(&iio_configfs_subsys.su_group,
 						"triggers",
 						&iio_triggers_group_type);
-	if (IS_ERR(iio_triggers_group))
-		return PTR_ERR(iio_triggers_group);
-	return 0;
+	return PTR_ERR_OR_ZERO(iio_triggers_group);
 }
 module_init(iio_sw_trigger_init);
 
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index aa26f3c3416b..8a8440c0eed1 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -5,6 +5,7 @@ menuconfig INFINIBAND
 	depends on NET
 	depends on INET
 	depends on m || IPV6 != m
+	select IRQ_POLL
 	---help---
 	  Core support for InfiniBand (IB).  Make sure to also select
 	  any protocols you wish to use as well as drivers for your
@@ -54,6 +55,15 @@ config INFINIBAND_ADDR_TRANS
 	depends on INFINIBAND
 	default y
 
+config INFINIBAND_ADDR_TRANS_CONFIGFS
+	bool
+	depends on INFINIBAND_ADDR_TRANS && CONFIGFS_FS && !(INFINIBAND=y && CONFIGFS_FS=m)
+	default y
+	---help---
+	  ConfigFS support for RDMA communication manager (CM).
+	  This allows the user to config the default GID type that the CM
+	  uses for each device, when initiaing new connections.
+
 source "drivers/infiniband/hw/mthca/Kconfig"
 source "drivers/infiniband/hw/qib/Kconfig"
 source "drivers/infiniband/hw/cxgb3/Kconfig"
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index d43a8994ac5c..f818538a7f4e 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -8,7 +8,7 @@ obj-$(CONFIG_INFINIBAND_USER_MAD) +=	ib_umad.o
 obj-$(CONFIG_INFINIBAND_USER_ACCESS) +=	ib_uverbs.o ib_ucm.o \
 					$(user_access-y)
 
-ib_core-y :=			packer.o ud_header.o verbs.o sysfs.o \
+ib_core-y :=			packer.o ud_header.o verbs.o cq.o sysfs.o \
 				device.o fmr_pool.o cache.o netlink.o \
 				roce_gid_mgmt.o
 ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
@@ -24,6 +24,8 @@ iw_cm-y :=			iwcm.o iwpm_util.o iwpm_msg.o
 
 rdma_cm-y :=			cma.o
 
+rdma_cm-$(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS) += cma_configfs.o
+
 rdma_ucm-y :=			ucma.o
 
 ib_addr-y :=			addr.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 34b1adad07aa..337353d86cfa 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -121,7 +121,8 @@ int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
 }
 EXPORT_SYMBOL(rdma_copy_addr);
 
-int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr,
+int rdma_translate_ip(const struct sockaddr *addr,
+		      struct rdma_dev_addr *dev_addr,
 		      u16 *vlan_id)
 {
 	struct net_device *dev;
@@ -139,7 +140,7 @@ int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr,
 	switch (addr->sa_family) {
 	case AF_INET:
 		dev = ip_dev_find(dev_addr->net,
-			((struct sockaddr_in *) addr)->sin_addr.s_addr);
+			((const struct sockaddr_in *)addr)->sin_addr.s_addr);
 
 		if (!dev)
 			return ret;
@@ -154,7 +155,7 @@ int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr,
 		rcu_read_lock();
 		for_each_netdev_rcu(dev_addr->net, dev) {
 			if (ipv6_chk_addr(dev_addr->net,
-					  &((struct sockaddr_in6 *) addr)->sin6_addr,
+					  &((const struct sockaddr_in6 *)addr)->sin6_addr,
 					  dev, 1)) {
 				ret = rdma_copy_addr(dev_addr, dev, NULL);
 				if (vlan_id)
@@ -198,7 +199,8 @@ static void queue_req(struct addr_req *req)
 	mutex_unlock(&lock);
 }
 
-static int dst_fetch_ha(struct dst_entry *dst, struct rdma_dev_addr *dev_addr, void *daddr)
+static int dst_fetch_ha(struct dst_entry *dst, struct rdma_dev_addr *dev_addr,
+			const void *daddr)
 {
 	struct neighbour *n;
 	int ret;
@@ -222,8 +224,9 @@ static int dst_fetch_ha(struct dst_entry *dst, struct rdma_dev_addr *dev_addr, v
 }
 
 static int addr4_resolve(struct sockaddr_in *src_in,
-			 struct sockaddr_in *dst_in,
-			 struct rdma_dev_addr *addr)
+			 const struct sockaddr_in *dst_in,
+			 struct rdma_dev_addr *addr,
+			 struct rtable **prt)
 {
 	__be32 src_ip = src_in->sin_addr.s_addr;
 	__be32 dst_ip = dst_in->sin_addr.s_addr;
@@ -243,33 +246,29 @@ static int addr4_resolve(struct sockaddr_in *src_in,
 	src_in->sin_family = AF_INET;
 	src_in->sin_addr.s_addr = fl4.saddr;
 
-	if (rt->dst.dev->flags & IFF_LOOPBACK) {
-		ret = rdma_translate_ip((struct sockaddr *)dst_in, addr, NULL);
-		if (!ret)
-			memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN);
-		goto put;
-	}
+	/* If there's a gateway, we're definitely in RoCE v2 (as RoCE v1 isn't
+	 * routable) and we could set the network type accordingly.
+	 */
+	if (rt->rt_uses_gateway)
+		addr->network = RDMA_NETWORK_IPV4;
 
-	/* If the device does ARP internally, return 'done' */
-	if (rt->dst.dev->flags & IFF_NOARP) {
-		ret = rdma_copy_addr(addr, rt->dst.dev, NULL);
-		goto put;
-	}
+	addr->hoplimit = ip4_dst_hoplimit(&rt->dst);
 
-	ret = dst_fetch_ha(&rt->dst, addr, &fl4.daddr);
-put:
-	ip_rt_put(rt);
+	*prt = rt;
+	return 0;
 out:
 	return ret;
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
 static int addr6_resolve(struct sockaddr_in6 *src_in,
-			 struct sockaddr_in6 *dst_in,
-			 struct rdma_dev_addr *addr)
+			 const struct sockaddr_in6 *dst_in,
+			 struct rdma_dev_addr *addr,
+			 struct dst_entry **pdst)
 {
 	struct flowi6 fl6;
 	struct dst_entry *dst;
+	struct rt6_info *rt;
 	int ret;
 
 	memset(&fl6, 0, sizeof fl6);
@@ -281,6 +280,7 @@ static int addr6_resolve(struct sockaddr_in6 *src_in,
 	if ((ret = dst->error))
 		goto put;
 
+	rt = (struct rt6_info *)dst;
 	if (ipv6_addr_any(&fl6.saddr)) {
 		ret = ipv6_dev_get_saddr(addr->net, ip6_dst_idev(dst)->dev,
 					 &fl6.daddr, 0, &fl6.saddr);
@@ -291,43 +291,111 @@ static int addr6_resolve(struct sockaddr_in6 *src_in,
 		src_in->sin6_addr = fl6.saddr;
 	}
 
-	if (dst->dev->flags & IFF_LOOPBACK) {
-		ret = rdma_translate_ip((struct sockaddr *)dst_in, addr, NULL);
-		if (!ret)
-			memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN);
-		goto put;
-	}
+	/* If there's a gateway, we're definitely in RoCE v2 (as RoCE v1 isn't
+	 * routable) and we could set the network type accordingly.
+	 */
+	if (rt->rt6i_flags & RTF_GATEWAY)
+		addr->network = RDMA_NETWORK_IPV6;
 
-	/* If the device does ARP internally, return 'done' */
-	if (dst->dev->flags & IFF_NOARP) {
-		ret = rdma_copy_addr(addr, dst->dev, NULL);
-		goto put;
-	}
+	addr->hoplimit = ip6_dst_hoplimit(dst);
 
-	ret = dst_fetch_ha(dst, addr, &fl6.daddr);
+	*pdst = dst;
+	return 0;
 put:
 	dst_release(dst);
 	return ret;
 }
 #else
 static int addr6_resolve(struct sockaddr_in6 *src_in,
-			 struct sockaddr_in6 *dst_in,
-			 struct rdma_dev_addr *addr)
+			 const struct sockaddr_in6 *dst_in,
+			 struct rdma_dev_addr *addr,
+			 struct dst_entry **pdst)
 {
 	return -EADDRNOTAVAIL;
 }
 #endif
 
+static int addr_resolve_neigh(struct dst_entry *dst,
+			      const struct sockaddr *dst_in,
+			      struct rdma_dev_addr *addr)
+{
+	if (dst->dev->flags & IFF_LOOPBACK) {
+		int ret;
+
+		ret = rdma_translate_ip(dst_in, addr, NULL);
+		if (!ret)
+			memcpy(addr->dst_dev_addr, addr->src_dev_addr,
+			       MAX_ADDR_LEN);
+
+		return ret;
+	}
+
+	/* If the device doesn't do ARP internally */
+	if (!(dst->dev->flags & IFF_NOARP)) {
+		const struct sockaddr_in *dst_in4 =
+			(const struct sockaddr_in *)dst_in;
+		const struct sockaddr_in6 *dst_in6 =
+			(const struct sockaddr_in6 *)dst_in;
+
+		return dst_fetch_ha(dst, addr,
+				    dst_in->sa_family == AF_INET ?
+				    (const void *)&dst_in4->sin_addr.s_addr :
+				    (const void *)&dst_in6->sin6_addr);
+	}
+
+	return rdma_copy_addr(addr, dst->dev, NULL);
+}
+
 static int addr_resolve(struct sockaddr *src_in,
-			struct sockaddr *dst_in,
-			struct rdma_dev_addr *addr)
+			const struct sockaddr *dst_in,
+			struct rdma_dev_addr *addr,
+			bool resolve_neigh)
 {
+	struct net_device *ndev;
+	struct dst_entry *dst;
+	int ret;
+
 	if (src_in->sa_family == AF_INET) {
-		return addr4_resolve((struct sockaddr_in *) src_in,
-			(struct sockaddr_in *) dst_in, addr);
-	} else
-		return addr6_resolve((struct sockaddr_in6 *) src_in,
-			(struct sockaddr_in6 *) dst_in, addr);
+		struct rtable *rt = NULL;
+		const struct sockaddr_in *dst_in4 =
+			(const struct sockaddr_in *)dst_in;
+
+		ret = addr4_resolve((struct sockaddr_in *)src_in,
+				    dst_in4, addr, &rt);
+		if (ret)
+			return ret;
+
+		if (resolve_neigh)
+			ret = addr_resolve_neigh(&rt->dst, dst_in, addr);
+
+		ndev = rt->dst.dev;
+		dev_hold(ndev);
+
+		ip_rt_put(rt);
+	} else {
+		const struct sockaddr_in6 *dst_in6 =
+			(const struct sockaddr_in6 *)dst_in;
+
+		ret = addr6_resolve((struct sockaddr_in6 *)src_in,
+				    dst_in6, addr,
+				    &dst);
+		if (ret)
+			return ret;
+
+		if (resolve_neigh)
+			ret = addr_resolve_neigh(dst, dst_in, addr);
+
+		ndev = dst->dev;
+		dev_hold(ndev);
+
+		dst_release(dst);
+	}
+
+	addr->bound_dev_if = ndev->ifindex;
+	addr->net = dev_net(ndev);
+	dev_put(ndev);
+
+	return ret;
 }
 
 static void process_req(struct work_struct *work)
@@ -343,7 +411,8 @@ static void process_req(struct work_struct *work)
 		if (req->status == -ENODATA) {
 			src_in = (struct sockaddr *) &req->src_addr;
 			dst_in = (struct sockaddr *) &req->dst_addr;
-			req->status = addr_resolve(src_in, dst_in, req->addr);
+			req->status = addr_resolve(src_in, dst_in, req->addr,
+						   true);
 			if (req->status && time_after_eq(jiffies, req->timeout))
 				req->status = -ETIMEDOUT;
 			else if (req->status == -ENODATA)
@@ -403,7 +472,7 @@ int rdma_resolve_ip(struct rdma_addr_client *client,
 	req->client = client;
 	atomic_inc(&client->refcount);
 
-	req->status = addr_resolve(src_in, dst_in, addr);
+	req->status = addr_resolve(src_in, dst_in, addr, true);
 	switch (req->status) {
 	case 0:
 		req->timeout = jiffies;
@@ -425,6 +494,26 @@ err:
 }
 EXPORT_SYMBOL(rdma_resolve_ip);
 
+int rdma_resolve_ip_route(struct sockaddr *src_addr,
+			  const struct sockaddr *dst_addr,
+			  struct rdma_dev_addr *addr)
+{
+	struct sockaddr_storage ssrc_addr = {};
+	struct sockaddr *src_in = (struct sockaddr *)&ssrc_addr;
+
+	if (src_addr) {
+		if (src_addr->sa_family != dst_addr->sa_family)
+			return -EINVAL;
+
+		memcpy(src_in, src_addr, rdma_addr_size(src_addr));
+	} else {
+		src_in->sa_family = dst_addr->sa_family;
+	}
+
+	return addr_resolve(src_in, dst_addr, addr, false);
+}
+EXPORT_SYMBOL(rdma_resolve_ip_route);
+
 void rdma_addr_cancel(struct rdma_dev_addr *addr)
 {
 	struct addr_req *req, *temp_req;
@@ -456,8 +545,10 @@ static void resolve_cb(int status, struct sockaddr *src_addr,
 	complete(&((struct resolve_cb_context *)context)->comp);
 }
 
-int rdma_addr_find_dmac_by_grh(const union ib_gid *sgid, const union ib_gid *dgid,
-			       u8 *dmac, u16 *vlan_id, int if_index)
+int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
+				 const union ib_gid *dgid,
+				 u8 *dmac, u16 *vlan_id, int *if_index,
+				 int *hoplimit)
 {
 	int ret = 0;
 	struct rdma_dev_addr dev_addr;
@@ -475,7 +566,8 @@ int rdma_addr_find_dmac_by_grh(const union ib_gid *sgid, const union ib_gid *dgi
 	rdma_gid2ip(&dgid_addr._sockaddr, dgid);
 
 	memset(&dev_addr, 0, sizeof(dev_addr));
-	dev_addr.bound_dev_if = if_index;
+	if (if_index)
+		dev_addr.bound_dev_if = *if_index;
 	dev_addr.net = &init_net;
 
 	ctx.addr = &dev_addr;
@@ -491,12 +583,16 @@ int rdma_addr_find_dmac_by_grh(const union ib_gid *sgid, const union ib_gid *dgi
 	dev = dev_get_by_index(&init_net, dev_addr.bound_dev_if);
 	if (!dev)
 		return -ENODEV;
+	if (if_index)
+		*if_index = dev_addr.bound_dev_if;
 	if (vlan_id)
 		*vlan_id = rdma_vlan_dev_vlan_id(dev);
+	if (hoplimit)
+		*hoplimit = dev_addr.hoplimit;
 	dev_put(dev);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_addr_find_dmac_by_grh);
+EXPORT_SYMBOL(rdma_addr_find_l2_eth_by_grh);
 
 int rdma_addr_find_smac_by_sgid(union ib_gid *sgid, u8 *smac, u16 *vlan_id)
 {
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 89bebeada38b..53343ffbff7a 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -64,6 +64,7 @@ enum gid_attr_find_mask {
 	GID_ATTR_FIND_MASK_GID          = 1UL << 0,
 	GID_ATTR_FIND_MASK_NETDEV	= 1UL << 1,
 	GID_ATTR_FIND_MASK_DEFAULT	= 1UL << 2,
+	GID_ATTR_FIND_MASK_GID_TYPE	= 1UL << 3,
 };
 
 enum gid_table_entry_props {
@@ -81,10 +82,6 @@ enum gid_table_write_action {
 };
 
 struct ib_gid_table_entry {
-	/* This lock protects an entry from being
-	 * read and written simultaneously.
-	 */
-	rwlock_t	    lock;
 	unsigned long	    props;
 	union ib_gid        gid;
 	struct ib_gid_attr  attr;
@@ -109,28 +106,86 @@ struct ib_gid_table {
 	 * are locked by this lock.
 	 **/
 	struct mutex         lock;
+	/* This lock protects the table entries from being
+	 * read and written simultaneously.
+	 */
+	rwlock_t	     rwlock;
 	struct ib_gid_table_entry *data_vec;
 };
 
+static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)
+{
+	if (rdma_cap_roce_gid_table(ib_dev, port)) {
+		struct ib_event event;
+
+		event.device		= ib_dev;
+		event.element.port_num	= port;
+		event.event		= IB_EVENT_GID_CHANGE;
+
+		ib_dispatch_event(&event);
+	}
+}
+
+static const char * const gid_type_str[] = {
+	[IB_GID_TYPE_IB]	= "IB/RoCE v1",
+	[IB_GID_TYPE_ROCE_UDP_ENCAP]	= "RoCE v2",
+};
+
+const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
+{
+	if (gid_type < ARRAY_SIZE(gid_type_str) && gid_type_str[gid_type])
+		return gid_type_str[gid_type];
+
+	return "Invalid GID type";
+}
+EXPORT_SYMBOL(ib_cache_gid_type_str);
+
+int ib_cache_gid_parse_type_str(const char *buf)
+{
+	unsigned int i;
+	size_t len;
+	int err = -EINVAL;
+
+	len = strlen(buf);
+	if (len == 0)
+		return -EINVAL;
+
+	if (buf[len - 1] == '\n')
+		len--;
+
+	for (i = 0; i < ARRAY_SIZE(gid_type_str); ++i)
+		if (gid_type_str[i] && !strncmp(buf, gid_type_str[i], len) &&
+		    len == strlen(gid_type_str[i])) {
+			err = i;
+			break;
+		}
+
+	return err;
+}
+EXPORT_SYMBOL(ib_cache_gid_parse_type_str);
+
+/* This function expects that rwlock will be write locked in all
+ * scenarios and that lock will be locked in sleep-able (RoCE)
+ * scenarios.
+ */
 static int write_gid(struct ib_device *ib_dev, u8 port,
 		     struct ib_gid_table *table, int ix,
 		     const union ib_gid *gid,
 		     const struct ib_gid_attr *attr,
 		     enum gid_table_write_action action,
 		     bool  default_gid)
+	__releases(&table->rwlock) __acquires(&table->rwlock)
 {
 	int ret = 0;
 	struct net_device *old_net_dev;
-	unsigned long flags;
 
 	/* in rdma_cap_roce_gid_table, this funciton should be protected by a
 	 * sleep-able lock.
 	 */
-	write_lock_irqsave(&table->data_vec[ix].lock, flags);
 
 	if (rdma_cap_roce_gid_table(ib_dev, port)) {
 		table->data_vec[ix].props |= GID_TABLE_ENTRY_INVALID;
-		write_unlock_irqrestore(&table->data_vec[ix].lock, flags);
+		write_unlock_irq(&table->rwlock);
 		/* GID_TABLE_WRITE_ACTION_MODIFY currently isn't supported by
 		 * RoCE providers and thus only updates the cache.
 		 */
@@ -140,7 +195,7 @@ static int write_gid(struct ib_device *ib_dev, u8 port,
 		else if (action == GID_TABLE_WRITE_ACTION_DEL)
 			ret = ib_dev->del_gid(ib_dev, port, ix,
 					      &table->data_vec[ix].context);
-		write_lock_irqsave(&table->data_vec[ix].lock, flags);
+		write_lock_irq(&table->rwlock);
 	}
 
 	old_net_dev = table->data_vec[ix].attr.ndev;
@@ -162,17 +217,6 @@ static int write_gid(struct ib_device *ib_dev, u8 port,
 
 	table->data_vec[ix].props &= ~GID_TABLE_ENTRY_INVALID;
 
-	write_unlock_irqrestore(&table->data_vec[ix].lock, flags);
-
-	if (!ret && rdma_cap_roce_gid_table(ib_dev, port)) {
-		struct ib_event event;
-
-		event.device		= ib_dev;
-		event.element.port_num	= port;
-		event.event		= IB_EVENT_GID_CHANGE;
-
-		ib_dispatch_event(&event);
-	}
 	return ret;
 }
 
@@ -201,41 +245,58 @@ static int del_gid(struct ib_device *ib_dev, u8 port,
 			 GID_TABLE_WRITE_ACTION_DEL, default_gid);
 }
 
+/* rwlock should be read locked */
 static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
 		    const struct ib_gid_attr *val, bool default_gid,
-		    unsigned long mask)
+		    unsigned long mask, int *pempty)
 {
-	int i;
+	int i = 0;
+	int found = -1;
+	int empty = pempty ? -1 : 0;
 
-	for (i = 0; i < table->sz; i++) {
-		unsigned long flags;
-		struct ib_gid_attr *attr = &table->data_vec[i].attr;
+	while (i < table->sz && (found < 0 || empty < 0)) {
+		struct ib_gid_table_entry *data = &table->data_vec[i];
+		struct ib_gid_attr *attr = &data->attr;
+		int curr_index = i;
 
-		read_lock_irqsave(&table->data_vec[i].lock, flags);
+		i++;
 
-		if (table->data_vec[i].props & GID_TABLE_ENTRY_INVALID)
-			goto next;
+		if (data->props & GID_TABLE_ENTRY_INVALID)
+			continue;
+
+		if (empty < 0)
+			if (!memcmp(&data->gid, &zgid, sizeof(*gid)) &&
+			    !memcmp(attr, &zattr, sizeof(*attr)) &&
+			    !data->props)
+				empty = curr_index;
+
+		if (found >= 0)
+			continue;
+
+		if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
+		    attr->gid_type != val->gid_type)
+			continue;
 
 		if (mask & GID_ATTR_FIND_MASK_GID &&
-		    memcmp(gid, &table->data_vec[i].gid, sizeof(*gid)))
-			goto next;
+		    memcmp(gid, &data->gid, sizeof(*gid)))
+			continue;
 
 		if (mask & GID_ATTR_FIND_MASK_NETDEV &&
 		    attr->ndev != val->ndev)
-			goto next;
+			continue;
 
 		if (mask & GID_ATTR_FIND_MASK_DEFAULT &&
-		    !!(table->data_vec[i].props & GID_TABLE_ENTRY_DEFAULT) !=
+		    !!(data->props & GID_TABLE_ENTRY_DEFAULT) !=
 		    default_gid)
-			goto next;
+			continue;
 
-		read_unlock_irqrestore(&table->data_vec[i].lock, flags);
-		return i;
-next:
-		read_unlock_irqrestore(&table->data_vec[i].lock, flags);
+		found = curr_index;
 	}
 
-	return -1;
+	if (pempty)
+		*pempty = empty;
+
+	return found;
 }
 
 static void make_default_gid(struct  net_device *dev, union ib_gid *gid)
@@ -252,6 +313,7 @@ int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
 	int ix;
 	int ret = 0;
 	struct net_device *idev;
+	int empty;
 
 	table = ports_table[port - rdma_start_port(ib_dev)];
 
@@ -275,22 +337,25 @@ int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
 	}
 
 	mutex_lock(&table->lock);
+	write_lock_irq(&table->rwlock);
 
 	ix = find_gid(table, gid, attr, false, GID_ATTR_FIND_MASK_GID |
-		      GID_ATTR_FIND_MASK_NETDEV);
+		      GID_ATTR_FIND_MASK_GID_TYPE |
+		      GID_ATTR_FIND_MASK_NETDEV, &empty);
 	if (ix >= 0)
 		goto out_unlock;
 
-	ix = find_gid(table, &zgid, NULL, false, GID_ATTR_FIND_MASK_GID |
-		      GID_ATTR_FIND_MASK_DEFAULT);
-	if (ix < 0) {
+	if (empty < 0) {
 		ret = -ENOSPC;
 		goto out_unlock;
 	}
 
-	add_gid(ib_dev, port, table, ix, gid, attr, false);
+	ret = add_gid(ib_dev, port, table, empty, gid, attr, false);
+	if (!ret)
+		dispatch_gid_change_event(ib_dev, port);
 
 out_unlock:
+	write_unlock_irq(&table->rwlock);
 	mutex_unlock(&table->lock);
 	return ret;
 }
@@ -305,17 +370,22 @@ int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
 	table = ports_table[port - rdma_start_port(ib_dev)];
 
 	mutex_lock(&table->lock);
+	write_lock_irq(&table->rwlock);
 
 	ix = find_gid(table, gid, attr, false,
 		      GID_ATTR_FIND_MASK_GID	  |
+		      GID_ATTR_FIND_MASK_GID_TYPE |
 		      GID_ATTR_FIND_MASK_NETDEV	  |
-		      GID_ATTR_FIND_MASK_DEFAULT);
+		      GID_ATTR_FIND_MASK_DEFAULT,
+		      NULL);
 	if (ix < 0)
 		goto out_unlock;
 
-	del_gid(ib_dev, port, table, ix, false);
+	if (!del_gid(ib_dev, port, table, ix, false))
+		dispatch_gid_change_event(ib_dev, port);
 
 out_unlock:
+	write_unlock_irq(&table->rwlock);
 	mutex_unlock(&table->lock);
 	return 0;
 }
@@ -326,16 +396,24 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
 	struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
 	struct ib_gid_table *table;
 	int ix;
+	bool deleted = false;
 
 	table  = ports_table[port - rdma_start_port(ib_dev)];
 
 	mutex_lock(&table->lock);
+	write_lock_irq(&table->rwlock);
 
 	for (ix = 0; ix < table->sz; ix++)
 		if (table->data_vec[ix].attr.ndev == ndev)
-			del_gid(ib_dev, port, table, ix, false);
+			if (!del_gid(ib_dev, port, table, ix, false))
+				deleted = true;
 
+	write_unlock_irq(&table->rwlock);
 	mutex_unlock(&table->lock);
+
+	if (deleted)
+		dispatch_gid_change_event(ib_dev, port);
+
 	return 0;
 }
 
@@ -344,18 +422,14 @@ static int __ib_cache_gid_get(struct ib_device *ib_dev, u8 port, int index,
 {
 	struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
 	struct ib_gid_table *table;
-	unsigned long flags;
 
 	table = ports_table[port - rdma_start_port(ib_dev)];
 
 	if (index < 0 || index >= table->sz)
 		return -EINVAL;
 
-	read_lock_irqsave(&table->data_vec[index].lock, flags);
-	if (table->data_vec[index].props & GID_TABLE_ENTRY_INVALID) {
-		read_unlock_irqrestore(&table->data_vec[index].lock, flags);
+	if (table->data_vec[index].props & GID_TABLE_ENTRY_INVALID)
 		return -EAGAIN;
-	}
 
 	memcpy(gid, &table->data_vec[index].gid, sizeof(*gid));
 	if (attr) {
@@ -364,7 +438,6 @@ static int __ib_cache_gid_get(struct ib_device *ib_dev, u8 port, int index,
 			dev_hold(attr->ndev);
 	}
 
-	read_unlock_irqrestore(&table->data_vec[index].lock, flags);
 	return 0;
 }
 
@@ -378,17 +451,21 @@ static int _ib_cache_gid_table_find(struct ib_device *ib_dev,
 	struct ib_gid_table *table;
 	u8 p;
 	int local_index;
+	unsigned long flags;
 
 	for (p = 0; p < ib_dev->phys_port_cnt; p++) {
 		table = ports_table[p];
-		local_index = find_gid(table, gid, val, false, mask);
+		read_lock_irqsave(&table->rwlock, flags);
+		local_index = find_gid(table, gid, val, false, mask, NULL);
 		if (local_index >= 0) {
 			if (index)
 				*index = local_index;
 			if (port)
 				*port = p + rdma_start_port(ib_dev);
+			read_unlock_irqrestore(&table->rwlock, flags);
 			return 0;
 		}
+		read_unlock_irqrestore(&table->rwlock, flags);
 	}
 
 	return -ENOENT;
@@ -396,11 +473,13 @@ static int _ib_cache_gid_table_find(struct ib_device *ib_dev,
 
 static int ib_cache_gid_find(struct ib_device *ib_dev,
 			     const union ib_gid *gid,
+			     enum ib_gid_type gid_type,
 			     struct net_device *ndev, u8 *port,
 			     u16 *index)
 {
-	unsigned long mask = GID_ATTR_FIND_MASK_GID;
-	struct ib_gid_attr gid_attr_val = {.ndev = ndev};
+	unsigned long mask = GID_ATTR_FIND_MASK_GID |
+			     GID_ATTR_FIND_MASK_GID_TYPE;
+	struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};
 
 	if (ndev)
 		mask |= GID_ATTR_FIND_MASK_NETDEV;
@@ -411,14 +490,17 @@ static int ib_cache_gid_find(struct ib_device *ib_dev,
 
 int ib_find_cached_gid_by_port(struct ib_device *ib_dev,
 			       const union ib_gid *gid,
+			       enum ib_gid_type gid_type,
 			       u8 port, struct net_device *ndev,
 			       u16 *index)
 {
 	int local_index;
 	struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
 	struct ib_gid_table *table;
-	unsigned long mask = GID_ATTR_FIND_MASK_GID;
-	struct ib_gid_attr val = {.ndev = ndev};
+	unsigned long mask = GID_ATTR_FIND_MASK_GID |
+			     GID_ATTR_FIND_MASK_GID_TYPE;
+	struct ib_gid_attr val = {.ndev = ndev, .gid_type = gid_type};
+	unsigned long flags;
 
 	if (port < rdma_start_port(ib_dev) ||
 	    port > rdma_end_port(ib_dev))
@@ -429,13 +511,16 @@ int ib_find_cached_gid_by_port(struct ib_device *ib_dev,
 	if (ndev)
 		mask |= GID_ATTR_FIND_MASK_NETDEV;
 
-	local_index = find_gid(table, gid, &val, false, mask);
+	read_lock_irqsave(&table->rwlock, flags);
+	local_index = find_gid(table, gid, &val, false, mask, NULL);
 	if (local_index >= 0) {
 		if (index)
 			*index = local_index;
+		read_unlock_irqrestore(&table->rwlock, flags);
 		return 0;
 	}
 
+	read_unlock_irqrestore(&table->rwlock, flags);
 	return -ENOENT;
 }
 EXPORT_SYMBOL(ib_find_cached_gid_by_port);
@@ -472,6 +557,7 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev,
 	struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
 	struct ib_gid_table *table;
 	unsigned int i;
+	unsigned long flags;
 	bool found = false;
 
 	if (!ports_table)
@@ -484,11 +570,10 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev,
 
 	table = ports_table[port - rdma_start_port(ib_dev)];
 
+	read_lock_irqsave(&table->rwlock, flags);
 	for (i = 0; i < table->sz; i++) {
 		struct ib_gid_attr attr;
-		unsigned long flags;
 
-		read_lock_irqsave(&table->data_vec[i].lock, flags);
 		if (table->data_vec[i].props & GID_TABLE_ENTRY_INVALID)
 			goto next;
 
@@ -501,11 +586,10 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev,
 			found = true;
 
 next:
-		read_unlock_irqrestore(&table->data_vec[i].lock, flags);
-
 		if (found)
 			break;
 	}
+	read_unlock_irqrestore(&table->rwlock, flags);
 
 	if (!found)
 		return -ENOENT;
@@ -517,9 +601,9 @@ next:
 
 static struct ib_gid_table *alloc_gid_table(int sz)
 {
-	unsigned int i;
 	struct ib_gid_table *table =
 		kzalloc(sizeof(struct ib_gid_table), GFP_KERNEL);
+
 	if (!table)
 		return NULL;
 
@@ -530,9 +614,7 @@ static struct ib_gid_table *alloc_gid_table(int sz)
 	mutex_init(&table->lock);
 
 	table->sz = sz;
-
-	for (i = 0; i < sz; i++)
-		rwlock_init(&table->data_vec[i].lock);
+	rwlock_init(&table->rwlock);
 
 	return table;
 
@@ -553,30 +635,37 @@ static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
 				   struct ib_gid_table *table)
 {
 	int i;
+	bool deleted = false;
 
 	if (!table)
 		return;
 
+	write_lock_irq(&table->rwlock);
 	for (i = 0; i < table->sz; ++i) {
 		if (memcmp(&table->data_vec[i].gid, &zgid,
 			   sizeof(table->data_vec[i].gid)))
-			del_gid(ib_dev, port, table, i,
-				table->data_vec[i].props &
-				GID_ATTR_FIND_MASK_DEFAULT);
+			if (!del_gid(ib_dev, port, table, i,
+				     table->data_vec[i].props &
+				     GID_ATTR_FIND_MASK_DEFAULT))
+				deleted = true;
 	}
+	write_unlock_irq(&table->rwlock);
+
+	if (deleted)
+		dispatch_gid_change_event(ib_dev, port);
 }
 
 void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
 				  struct net_device *ndev,
+				  unsigned long gid_type_mask,
 				  enum ib_cache_gid_default_mode mode)
 {
 	struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
 	union ib_gid gid;
 	struct ib_gid_attr gid_attr;
+	struct ib_gid_attr zattr_type = zattr;
 	struct ib_gid_table *table;
-	int ix;
-	union ib_gid current_gid;
-	struct ib_gid_attr current_gid_attr = {};
+	unsigned int gid_type;
 
 	table  = ports_table[port - rdma_start_port(ib_dev)];
 
@@ -584,46 +673,82 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
 	memset(&gid_attr, 0, sizeof(gid_attr));
 	gid_attr.ndev = ndev;
 
-	mutex_lock(&table->lock);
-	ix = find_gid(table, NULL, NULL, true, GID_ATTR_FIND_MASK_DEFAULT);
-
-	/* Coudn't find default GID location */
-	WARN_ON(ix < 0);
-
-	if (!__ib_cache_gid_get(ib_dev, port, ix,
-				&current_gid, &current_gid_attr) &&
-	    mode == IB_CACHE_GID_DEFAULT_MODE_SET &&
-	    !memcmp(&gid, &current_gid, sizeof(gid)) &&
-	    !memcmp(&gid_attr, &current_gid_attr, sizeof(gid_attr)))
-		goto unlock;
-
-	if ((memcmp(&current_gid, &zgid, sizeof(current_gid)) ||
-	     memcmp(&current_gid_attr, &zattr,
-		    sizeof(current_gid_attr))) &&
-	    del_gid(ib_dev, port, table, ix, true)) {
-		pr_warn("ib_cache_gid: can't delete index %d for default gid %pI6\n",
-			ix, gid.raw);
-		goto unlock;
-	}
+	for (gid_type = 0; gid_type < IB_GID_TYPE_SIZE; ++gid_type) {
+		int ix;
+		union ib_gid current_gid;
+		struct ib_gid_attr current_gid_attr = {};
+
+		if (1UL << gid_type & ~gid_type_mask)
+			continue;
+
+		gid_attr.gid_type = gid_type;
+
+		mutex_lock(&table->lock);
+		write_lock_irq(&table->rwlock);
+		ix = find_gid(table, NULL, &gid_attr, true,
+			      GID_ATTR_FIND_MASK_GID_TYPE |
+			      GID_ATTR_FIND_MASK_DEFAULT,
+			      NULL);
+
+		/* Coudn't find default GID location */
+		WARN_ON(ix < 0);
+
+		zattr_type.gid_type = gid_type;
+
+		if (!__ib_cache_gid_get(ib_dev, port, ix,
+					&current_gid, &current_gid_attr) &&
+		    mode == IB_CACHE_GID_DEFAULT_MODE_SET &&
+		    !memcmp(&gid, &current_gid, sizeof(gid)) &&
+		    !memcmp(&gid_attr, &current_gid_attr, sizeof(gid_attr)))
+			goto release;
+
+		if (memcmp(&current_gid, &zgid, sizeof(current_gid)) ||
+		    memcmp(&current_gid_attr, &zattr_type,
+			   sizeof(current_gid_attr))) {
+			if (del_gid(ib_dev, port, table, ix, true)) {
+				pr_warn("ib_cache_gid: can't delete index %d for default gid %pI6\n",
+					ix, gid.raw);
+				goto release;
+			} else {
+				dispatch_gid_change_event(ib_dev, port);
+			}
+		}
 
-	if (mode == IB_CACHE_GID_DEFAULT_MODE_SET)
-		if (add_gid(ib_dev, port, table, ix, &gid, &gid_attr, true))
-			pr_warn("ib_cache_gid: unable to add default gid %pI6\n",
-				gid.raw);
+		if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
+			if (add_gid(ib_dev, port, table, ix, &gid, &gid_attr, true))
+				pr_warn("ib_cache_gid: unable to add default gid %pI6\n",
+					gid.raw);
+			else
+				dispatch_gid_change_event(ib_dev, port);
+		}
 
-unlock:
-	if (current_gid_attr.ndev)
-		dev_put(current_gid_attr.ndev);
-	mutex_unlock(&table->lock);
+release:
+		if (current_gid_attr.ndev)
+			dev_put(current_gid_attr.ndev);
+		write_unlock_irq(&table->rwlock);
+		mutex_unlock(&table->lock);
+	}
 }
 
 static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
 				     struct ib_gid_table *table)
 {
-	if (rdma_protocol_roce(ib_dev, port)) {
-		struct ib_gid_table_entry *entry = &table->data_vec[0];
+	unsigned int i;
+	unsigned long roce_gid_type_mask;
+	unsigned int num_default_gids;
+	unsigned int current_gid = 0;
+
+	roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
+	num_default_gids = hweight_long(roce_gid_type_mask);
+	for (i = 0; i < num_default_gids && i < table->sz; i++) {
+		struct ib_gid_table_entry *entry =
+			&table->data_vec[i];
 
 		entry->props |= GID_TABLE_ENTRY_DEFAULT;
+		current_gid = find_next_bit(&roce_gid_type_mask,
+					    BITS_PER_LONG,
+					    current_gid);
+		entry->attr.gid_type = current_gid++;
 	}
 
 	return 0;
@@ -728,20 +853,30 @@ int ib_get_cached_gid(struct ib_device *device,
 		      union ib_gid     *gid,
 		      struct ib_gid_attr *gid_attr)
 {
+	int res;
+	unsigned long flags;
+	struct ib_gid_table **ports_table = device->cache.gid_cache;
+	struct ib_gid_table *table = ports_table[port_num - rdma_start_port(device)];
+
 	if (port_num < rdma_start_port(device) || port_num > rdma_end_port(device))
 		return -EINVAL;
 
-	return __ib_cache_gid_get(device, port_num, index, gid, gid_attr);
+	read_lock_irqsave(&table->rwlock, flags);
+	res = __ib_cache_gid_get(device, port_num, index, gid, gid_attr);
+	read_unlock_irqrestore(&table->rwlock, flags);
+
+	return res;
 }
 EXPORT_SYMBOL(ib_get_cached_gid);
 
 int ib_find_cached_gid(struct ib_device *device,
 		       const union ib_gid *gid,
+		       enum ib_gid_type gid_type,
 		       struct net_device *ndev,
 		       u8               *port_num,
 		       u16              *index)
 {
-	return ib_cache_gid_find(device, gid, ndev, port_num, index);
+	return ib_cache_gid_find(device, gid, gid_type, ndev, port_num, index);
 }
 EXPORT_SYMBOL(ib_find_cached_gid);
 
@@ -956,10 +1091,12 @@ static void ib_cache_update(struct ib_device *device,
 
 	device->cache.pkey_cache[port - rdma_start_port(device)] = pkey_cache;
 	if (!use_roce_gid_table) {
+		write_lock(&table->rwlock);
 		for (i = 0; i < gid_cache->table_len; i++) {
 			modify_gid(device, port, table, i, gid_cache->table + i,
 				   &zattr, false);
 		}
+		write_unlock(&table->rwlock);
 	}
 
 	device->cache.lmc_cache[port - rdma_start_port(device)] = tprops->lmc;
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 0a26dd6d9b19..1d92e091e22e 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -364,7 +364,7 @@ static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
 	read_lock_irqsave(&cm.device_lock, flags);
 	list_for_each_entry(cm_dev, &cm.device_list, list) {
 		if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
-					ndev, &p, NULL)) {
+					path->gid_type, ndev, &p, NULL)) {
 			port = cm_dev->port[p-1];
 			break;
 		}
@@ -782,11 +782,11 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 	wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 
 	/* Check if the device started its remove_one */
-	spin_lock_irq(&cm.lock);
+	spin_lock_irqsave(&cm.lock, flags);
 	if (!cm_dev->going_down)
 		queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
 				   msecs_to_jiffies(wait_time));
-	spin_unlock_irq(&cm.lock);
+	spin_unlock_irqrestore(&cm.lock, flags);
 
 	cm_id_priv->timewait_info = NULL;
 }
@@ -1600,6 +1600,8 @@ static int cm_req_handler(struct cm_work *work)
 	struct ib_cm_id *cm_id;
 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
 	struct cm_req_msg *req_msg;
+	union ib_gid gid;
+	struct ib_gid_attr gid_attr;
 	int ret;
 
 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
@@ -1639,11 +1641,31 @@ static int cm_req_handler(struct cm_work *work)
 	cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
 
 	memcpy(work->path[0].dmac, cm_id_priv->av.ah_attr.dmac, ETH_ALEN);
-	ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
+	work->path[0].hop_limit = cm_id_priv->av.ah_attr.grh.hop_limit;
+	ret = ib_get_cached_gid(work->port->cm_dev->ib_device,
+				work->port->port_num,
+				cm_id_priv->av.ah_attr.grh.sgid_index,
+				&gid, &gid_attr);
+	if (!ret) {
+		if (gid_attr.ndev) {
+			work->path[0].ifindex = gid_attr.ndev->ifindex;
+			work->path[0].net = dev_net(gid_attr.ndev);
+			dev_put(gid_attr.ndev);
+		}
+		work->path[0].gid_type = gid_attr.gid_type;
+		ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
+	}
 	if (ret) {
-		ib_get_cached_gid(work->port->cm_dev->ib_device,
-				  work->port->port_num, 0, &work->path[0].sgid,
-				  NULL);
+		int err = ib_get_cached_gid(work->port->cm_dev->ib_device,
+					    work->port->port_num, 0,
+					    &work->path[0].sgid,
+					    &gid_attr);
+		if (!err && gid_attr.ndev) {
+			work->path[0].ifindex = gid_attr.ndev->ifindex;
+			work->path[0].net = dev_net(gid_attr.ndev);
+			dev_put(gid_attr.ndev);
+		}
+		work->path[0].gid_type = gid_attr.gid_type;
 		ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
 			       &work->path[0].sgid, sizeof work->path[0].sgid,
 			       NULL, 0);
@@ -3482,6 +3504,7 @@ int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
 EXPORT_SYMBOL(ib_cm_notify);
 
 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
+			    struct ib_mad_send_buf *send_buf,
 			    struct ib_mad_recv_wc *mad_recv_wc)
 {
 	struct cm_port *port = mad_agent->context;
@@ -3731,16 +3754,6 @@ int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
 }
 EXPORT_SYMBOL(ib_cm_init_qp_attr);
 
-static void cm_get_ack_delay(struct cm_device *cm_dev)
-{
-	struct ib_device_attr attr;
-
-	if (ib_query_device(cm_dev->ib_device, &attr))
-		cm_dev->ack_delay = 0; /* acks will rely on packet life time */
-	else
-		cm_dev->ack_delay = attr.local_ca_ack_delay;
-}
-
 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
 			       char *buf)
 {
@@ -3852,7 +3865,7 @@ static void cm_add_one(struct ib_device *ib_device)
 		return;
 
 	cm_dev->ib_device = ib_device;
-	cm_get_ack_delay(cm_dev);
+	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
 	cm_dev->going_down = 0;
 	cm_dev->device = device_create(&cm_class, &ib_device->dev,
 				       MKDEV(0, 0), NULL,
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 2d762a2ecd81..9729639df407 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -38,6 +38,7 @@
 #include <linux/in6.h>
 #include <linux/mutex.h>
 #include <linux/random.h>
+#include <linux/igmp.h>
 #include <linux/idr.h>
 #include <linux/inetdevice.h>
 #include <linux/slab.h>
@@ -60,6 +61,8 @@
 #include <rdma/ib_sa.h>
 #include <rdma/iw_cm.h>
 
+#include "core_priv.h"
+
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -150,6 +153,7 @@ struct cma_device {
 	struct completion	comp;
 	atomic_t		refcount;
 	struct list_head	id_list;
+	enum ib_gid_type	*default_gid_type;
 };
 
 struct rdma_bind_list {
@@ -185,6 +189,67 @@ enum {
 	CMA_OPTION_AFONLY,
 };
 
+void cma_ref_dev(struct cma_device *cma_dev)
+{
+	atomic_inc(&cma_dev->refcount);
+}
+
+struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,
+					     void		*cookie)
+{
+	struct cma_device *cma_dev;
+	struct cma_device *found_cma_dev = NULL;
+
+	mutex_lock(&lock);
+
+	list_for_each_entry(cma_dev, &dev_list, list)
+		if (filter(cma_dev->device, cookie)) {
+			found_cma_dev = cma_dev;
+			break;
+		}
+
+	if (found_cma_dev)
+		cma_ref_dev(found_cma_dev);
+	mutex_unlock(&lock);
+	return found_cma_dev;
+}
+
+int cma_get_default_gid_type(struct cma_device *cma_dev,
+			     unsigned int port)
+{
+	if (port < rdma_start_port(cma_dev->device) ||
+	    port > rdma_end_port(cma_dev->device))
+		return -EINVAL;
+
+	return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
+}
+
+int cma_set_default_gid_type(struct cma_device *cma_dev,
+			     unsigned int port,
+			     enum ib_gid_type default_gid_type)
+{
+	unsigned long supported_gids;
+
+	if (port < rdma_start_port(cma_dev->device) ||
+	    port > rdma_end_port(cma_dev->device))
+		return -EINVAL;
+
+	supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
+
+	if (!(supported_gids & 1 << default_gid_type))
+		return -EINVAL;
+
+	cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
+		default_gid_type;
+
+	return 0;
+}
+
+struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
+{
+	return cma_dev->device;
+}
+
 /*
  * Device removal can occur at anytime, so we need extra handling to
  * serialize notifying the user of device removal with other callbacks.
@@ -228,6 +293,7 @@ struct rdma_id_private {
 	u8			tos;
 	u8			reuseaddr;
 	u8			afonly;
+	enum ib_gid_type	gid_type;
 };
 
 struct cma_multicast {
@@ -239,6 +305,7 @@ struct cma_multicast {
 	void			*context;
 	struct sockaddr_storage	addr;
 	struct kref		mcref;
+	bool			igmp_joined;
 };
 
 struct cma_work {
@@ -335,18 +402,48 @@ static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 	hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 }
 
-static void cma_attach_to_dev(struct rdma_id_private *id_priv,
-			      struct cma_device *cma_dev)
+static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
 {
-	atomic_inc(&cma_dev->refcount);
+	struct in_device *in_dev = NULL;
+
+	if (ndev) {
+		rtnl_lock();
+		in_dev = __in_dev_get_rtnl(ndev);
+		if (in_dev) {
+			if (join)
+				ip_mc_inc_group(in_dev,
+						*(__be32 *)(mgid->raw + 12));
+			else
+				ip_mc_dec_group(in_dev,
+						*(__be32 *)(mgid->raw + 12));
+		}
+		rtnl_unlock();
+	}
+	return (in_dev) ? 0 : -ENODEV;
+}
+
+static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
+			       struct cma_device *cma_dev)
+{
+	cma_ref_dev(cma_dev);
 	id_priv->cma_dev = cma_dev;
+	id_priv->gid_type = 0;
 	id_priv->id.device = cma_dev->device;
 	id_priv->id.route.addr.dev_addr.transport =
 		rdma_node_get_transport(cma_dev->device->node_type);
 	list_add_tail(&id_priv->list, &cma_dev->id_list);
 }
 
-static inline void cma_deref_dev(struct cma_device *cma_dev)
+static void cma_attach_to_dev(struct rdma_id_private *id_priv,
+			      struct cma_device *cma_dev)
+{
+	_cma_attach_to_dev(id_priv, cma_dev);
+	id_priv->gid_type =
+		cma_dev->default_gid_type[id_priv->id.port_num -
+					  rdma_start_port(cma_dev->device)];
+}
+
+void cma_deref_dev(struct cma_device *cma_dev)
 {
 	if (atomic_dec_and_test(&cma_dev->refcount))
 		complete(&cma_dev->comp);
@@ -441,6 +538,7 @@ static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_a
 }
 
 static inline int cma_validate_port(struct ib_device *device, u8 port,
+				    enum ib_gid_type gid_type,
 				      union ib_gid *gid, int dev_type,
 				      int bound_if_index)
 {
@@ -453,10 +551,25 @@ static inline int cma_validate_port(struct ib_device *device, u8 port,
 	if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 		return ret;
 
-	if (dev_type == ARPHRD_ETHER)
+	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
 		ndev = dev_get_by_index(&init_net, bound_if_index);
+		if (ndev && ndev->flags & IFF_LOOPBACK) {
+			pr_info("detected loopback device\n");
+			dev_put(ndev);
 
-	ret = ib_find_cached_gid_by_port(device, gid, port, ndev, NULL);
+			if (!device->get_netdev)
+				return -EOPNOTSUPP;
+
+			ndev = device->get_netdev(device, port);
+			if (!ndev)
+				return -ENODEV;
+		}
+	} else {
+		gid_type = IB_GID_TYPE_IB;
+	}
+
+	ret = ib_find_cached_gid_by_port(device, gid, gid_type, port,
+					 ndev, NULL);
 
 	if (ndev)
 		dev_put(ndev);
@@ -490,7 +603,10 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv,
 		gidp = rdma_protocol_roce(cma_dev->device, port) ?
 		       &iboe_gid : &gid;
 
-		ret = cma_validate_port(cma_dev->device, port, gidp,
+		ret = cma_validate_port(cma_dev->device, port,
+					rdma_protocol_ib(cma_dev->device, port) ?
+					IB_GID_TYPE_IB :
+					listen_id_priv->gid_type, gidp,
 					dev_addr->dev_type,
 					dev_addr->bound_dev_if);
 		if (!ret) {
@@ -509,8 +625,11 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv,
 			gidp = rdma_protocol_roce(cma_dev->device, port) ?
 			       &iboe_gid : &gid;
 
-			ret = cma_validate_port(cma_dev->device, port, gidp,
-						dev_addr->dev_type,
+			ret = cma_validate_port(cma_dev->device, port,
+						rdma_protocol_ib(cma_dev->device, port) ?
+						IB_GID_TYPE_IB :
+						cma_dev->default_gid_type[port - 1],
+						gidp, dev_addr->dev_type,
 						dev_addr->bound_dev_if);
 			if (!ret) {
 				id_priv->id.port_num = port;
@@ -1437,8 +1556,24 @@ static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 				      id_priv->id.port_num)) {
 			ib_sa_free_multicast(mc->multicast.ib);
 			kfree(mc);
-		} else
+		} else {
+			if (mc->igmp_joined) {
+				struct rdma_dev_addr *dev_addr =
+					&id_priv->id.route.addr.dev_addr;
+				struct net_device *ndev = NULL;
+
+				if (dev_addr->bound_dev_if)
+					ndev = dev_get_by_index(&init_net,
+								dev_addr->bound_dev_if);
+				if (ndev) {
+					cma_igmp_send(ndev,
+						      &mc->multicast.ib->rec.mgid,
+						      false);
+					dev_put(ndev);
+				}
+			}
 			kref_put(&mc->mcref, release_mc);
+		}
 	}
 }
 
@@ -1896,7 +2031,6 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
 	struct rdma_id_private *listen_id, *conn_id;
 	struct rdma_cm_event event;
 	int ret;
-	struct ib_device_attr attr;
 	struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
 	struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
 
@@ -1938,13 +2072,6 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
 	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
 	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
 
-	ret = ib_query_device(conn_id->id.device, &attr);
-	if (ret) {
-		mutex_unlock(&conn_id->handler_mutex);
-		rdma_destroy_id(new_cm_id);
-		goto out;
-	}
-
 	memset(&event, 0, sizeof event);
 	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
 	event.param.conn.private_data = iw_event->private_data;
@@ -2051,7 +2178,7 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv,
 	memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
 	       rdma_addr_size(cma_src_addr(id_priv)));
 
-	cma_attach_to_dev(dev_id_priv, cma_dev);
+	_cma_attach_to_dev(dev_id_priv, cma_dev);
 	list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
 	atomic_inc(&id_priv->refcount);
 	dev_id_priv->internal_id = 1;
@@ -2321,8 +2448,23 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 
 	if (addr->dev_addr.bound_dev_if) {
 		ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
+		if (!ndev)
+			return -ENODEV;
+
+		if (ndev->flags & IFF_LOOPBACK) {
+			dev_put(ndev);
+			if (!id_priv->id.device->get_netdev)
+				return -EOPNOTSUPP;
+
+			ndev = id_priv->id.device->get_netdev(id_priv->id.device,
+							      id_priv->id.port_num);
+			if (!ndev)
+				return -ENODEV;
+		}
+
 		route->path_rec->net = &init_net;
-		route->path_rec->ifindex = addr->dev_addr.bound_dev_if;
+		route->path_rec->ifindex = ndev->ifindex;
+		route->path_rec->gid_type = id_priv->gid_type;
 	}
 	if (!ndev) {
 		ret = -ENODEV;
@@ -2336,7 +2478,14 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
 		    &route->path_rec->dgid);
 
-	route->path_rec->hop_limit = 1;
+	/* Use the hint from IP Stack to select GID Type */
+	if (route->path_rec->gid_type < ib_network_to_gid_type(addr->dev_addr.network))
+		route->path_rec->gid_type = ib_network_to_gid_type(addr->dev_addr.network);
+	if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
+		/* TODO: get the hoplimit from the inet/inet6 device */
+		route->path_rec->hop_limit = addr->dev_addr.hoplimit;
+	else
+		route->path_rec->hop_limit = 1;
 	route->path_rec->reversible = 1;
 	route->path_rec->pkey = cpu_to_be16(0xffff);
 	route->path_rec->mtu_selector = IB_SA_EQ;
@@ -3534,12 +3683,23 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
 	event.status = status;
 	event.param.ud.private_data = mc->context;
 	if (!status) {
+		struct rdma_dev_addr *dev_addr =
+			&id_priv->id.route.addr.dev_addr;
+		struct net_device *ndev =
+			dev_get_by_index(&init_net, dev_addr->bound_dev_if);
+		enum ib_gid_type gid_type =
+			id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
+			rdma_start_port(id_priv->cma_dev->device)];
+
 		event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
 		ib_init_ah_from_mcmember(id_priv->id.device,
 					 id_priv->id.port_num, &multicast->rec,
+					 ndev, gid_type,
 					 &event.param.ud.ah_attr);
 		event.param.ud.qp_num = 0xFFFFFF;
 		event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
+		if (ndev)
+			dev_put(ndev);
 	} else
 		event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
 
@@ -3672,9 +3832,10 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
 {
 	struct iboe_mcast_work *work;
 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
-	int err;
+	int err = 0;
 	struct sockaddr *addr = (struct sockaddr *)&mc->addr;
 	struct net_device *ndev = NULL;
+	enum ib_gid_type gid_type;
 
 	if (cma_zero_addr((struct sockaddr *)&mc->addr))
 		return -EINVAL;
@@ -3704,9 +3865,25 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
 	mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
 	mc->multicast.ib->rec.hop_limit = 1;
 	mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
+
+	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
+		   rdma_start_port(id_priv->cma_dev->device)];
+	if (addr->sa_family == AF_INET) {
+		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
+			err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid,
+					    true);
+		if (!err) {
+			mc->igmp_joined = true;
+			mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
+		}
+	} else {
+		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
+			err = -ENOTSUPP;
+	}
 	dev_put(ndev);
-	if (!mc->multicast.ib->rec.mtu) {
-		err = -EINVAL;
+	if (err || !mc->multicast.ib->rec.mtu) {
+		if (!err)
+			err = -EINVAL;
 		goto out2;
 	}
 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
@@ -3745,7 +3922,7 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
 	memcpy(&mc->addr, addr, rdma_addr_size(addr));
 	mc->context = context;
 	mc->id_priv = id_priv;
-
+	mc->igmp_joined = false;
 	spin_lock(&id_priv->lock);
 	list_add(&mc->list, &id_priv->mc_list);
 	spin_unlock(&id_priv->lock);
@@ -3790,9 +3967,25 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
 			if (rdma_cap_ib_mcast(id->device, id->port_num)) {
 				ib_sa_free_multicast(mc->multicast.ib);
 				kfree(mc);
-			} else if (rdma_protocol_roce(id->device, id->port_num))
+			} else if (rdma_protocol_roce(id->device, id->port_num)) {
+				if (mc->igmp_joined) {
+					struct rdma_dev_addr *dev_addr =
+						&id->route.addr.dev_addr;
+					struct net_device *ndev = NULL;
+
+					if (dev_addr->bound_dev_if)
+						ndev = dev_get_by_index(&init_net,
+									dev_addr->bound_dev_if);
+					if (ndev) {
+						cma_igmp_send(ndev,
+							      &mc->multicast.ib->rec.mgid,
+							      false);
+						dev_put(ndev);
+					}
+					mc->igmp_joined = false;
+				}
 				kref_put(&mc->mcref, release_mc);
-
+			}
 			return;
 		}
 	}
@@ -3861,12 +4054,27 @@ static void cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
 	struct rdma_id_private *id_priv;
+	unsigned int i;
+	unsigned long supported_gids = 0;
 
 	cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
 	if (!cma_dev)
 		return;
 
 	cma_dev->device = device;
+	cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
+					    sizeof(*cma_dev->default_gid_type),
+					    GFP_KERNEL);
+	if (!cma_dev->default_gid_type) {
+		kfree(cma_dev);
+		return;
+	}
+	for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) {
+		supported_gids = roce_gid_type_mask_support(device, i);
+		WARN_ON(!supported_gids);
+		cma_dev->default_gid_type[i - rdma_start_port(device)] =
+			find_first_bit(&supported_gids, BITS_PER_LONG);
+	}
 
 	init_completion(&cma_dev->comp);
 	atomic_set(&cma_dev->refcount, 1);
@@ -3946,6 +4154,7 @@ static void cma_remove_one(struct ib_device *device, void *client_data)
 	mutex_unlock(&lock);
 
 	cma_process_remove(cma_dev);
+	kfree(cma_dev->default_gid_type);
 	kfree(cma_dev);
 }
 
@@ -4079,6 +4288,7 @@ static int __init cma_init(void)
 
 	if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
 		printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
+	cma_configfs_init();
 
 	return 0;
 
@@ -4093,6 +4303,7 @@ err_wq:
 
 static void __exit cma_cleanup(void)
 {
+	cma_configfs_exit();
 	ibnl_remove_client(RDMA_NL_RDMA_CM);
 	ib_unregister_client(&cma_client);
 	unregister_netdevice_notifier(&cma_nb);
diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c
new file mode 100644
index 000000000000..18b112aa577e
--- /dev/null
+++ b/drivers/infiniband/core/cma_configfs.c
@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2015, Mellanox Technologies inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/configfs.h>
+#include <rdma/ib_verbs.h>
+#include "core_priv.h"
+
+struct cma_device;
+
+struct cma_dev_group;
+
+struct cma_dev_port_group {
+	unsigned int		port_num;
+	struct cma_dev_group	*cma_dev_group;
+	struct config_group	group;
+};
+
+struct cma_dev_group {
+	char				name[IB_DEVICE_NAME_MAX];
+	struct config_group		device_group;
+	struct config_group		ports_group;
+	struct config_group		*default_dev_group[2];
+	struct config_group		**default_ports_group;
+	struct cma_dev_port_group	*ports;
+};
+
+static struct cma_dev_port_group *to_dev_port_group(struct config_item *item)
+{
+	struct config_group *group;
+
+	if (!item)
+		return NULL;
+
+	group = container_of(item, struct config_group, cg_item);
+	return container_of(group, struct cma_dev_port_group, group);
+}
+
+static bool filter_by_name(struct ib_device *ib_dev, void *cookie)
+{
+	return !strcmp(ib_dev->name, cookie);
+}
+
+static int cma_configfs_params_get(struct config_item *item,
+				   struct cma_device **pcma_dev,
+				   struct cma_dev_port_group **pgroup)
+{
+	struct cma_dev_port_group *group = to_dev_port_group(item);
+	struct cma_device *cma_dev;
+
+	if (!group)
+		return -ENODEV;
+
+	cma_dev = cma_enum_devices_by_ibdev(filter_by_name,
+					    group->cma_dev_group->name);
+	if (!cma_dev)
+		return -ENODEV;
+
+	*pcma_dev = cma_dev;
+	*pgroup = group;
+
+	return 0;
+}
+
+static void cma_configfs_params_put(struct cma_device *cma_dev)
+{
+	cma_deref_dev(cma_dev);
+}
+
+static ssize_t default_roce_mode_show(struct config_item *item,
+				      char *buf)
+{
+	struct cma_device *cma_dev;
+	struct cma_dev_port_group *group;
+	int gid_type;
+	ssize_t ret;
+
+	ret = cma_configfs_params_get(item, &cma_dev, &group);
+	if (ret)
+		return ret;
+
+	gid_type = cma_get_default_gid_type(cma_dev, group->port_num);
+	cma_configfs_params_put(cma_dev);
+
+	if (gid_type < 0)
+		return gid_type;
+
+	return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_type));
+}
+
+static ssize_t default_roce_mode_store(struct config_item *item,
+				       const char *buf, size_t count)
+{
+	struct cma_device *cma_dev;
+	struct cma_dev_port_group *group;
+	int gid_type = ib_cache_gid_parse_type_str(buf);
+	ssize_t ret;
+
+	if (gid_type < 0)
+		return -EINVAL;
+
+	ret = cma_configfs_params_get(item, &cma_dev, &group);
+	if (ret)
+		return ret;
+
+	ret = cma_set_default_gid_type(cma_dev, group->port_num, gid_type);
+
+	cma_configfs_params_put(cma_dev);
+
+	return !ret ? strnlen(buf, count) : ret;
+}
+
+CONFIGFS_ATTR(, default_roce_mode);
+
+static struct configfs_attribute *cma_configfs_attributes[] = {
+	&attr_default_roce_mode,
+	NULL,
+};
+
+static struct config_item_type cma_port_group_type = {
+	.ct_attrs	= cma_configfs_attributes,
+	.ct_owner	= THIS_MODULE
+};
+
+static int make_cma_ports(struct cma_dev_group *cma_dev_group,
+			  struct cma_device *cma_dev)
+{
+	struct ib_device *ibdev;
+	unsigned int i;
+	unsigned int ports_num;
+	struct cma_dev_port_group *ports;
+	struct config_group **ports_group;
+	int err;
+
+	ibdev = cma_get_ib_dev(cma_dev);
+
+	if (!ibdev)
+		return -ENODEV;
+
+	ports_num = ibdev->phys_port_cnt;
+	ports = kcalloc(ports_num, sizeof(*cma_dev_group->ports),
+			GFP_KERNEL);
+	ports_group = kcalloc(ports_num + 1, sizeof(*ports_group), GFP_KERNEL);
+
+	if (!ports || !ports_group) {
+		err = -ENOMEM;
+		goto free;
+	}
+
+	for (i = 0; i < ports_num; i++) {
+		char port_str[10];
+
+		ports[i].port_num = i + 1;
+		snprintf(port_str, sizeof(port_str), "%u", i + 1);
+		ports[i].cma_dev_group = cma_dev_group;
+		config_group_init_type_name(&ports[i].group,
+					    port_str,
+					    &cma_port_group_type);
+		ports_group[i] = &ports[i].group;
+	}
+	ports_group[i] = NULL;
+	cma_dev_group->default_ports_group = ports_group;
+	cma_dev_group->ports = ports;
+
+	return 0;
+free:
+	kfree(ports);
+	kfree(ports_group);
+	cma_dev_group->ports = NULL;
+	cma_dev_group->default_ports_group = NULL;
+	return err;
+}
+
+static void release_cma_dev(struct config_item  *item)
+{
+	struct config_group *group = container_of(item, struct config_group,
+						  cg_item);
+	struct cma_dev_group *cma_dev_group = container_of(group,
+							   struct cma_dev_group,
+							   device_group);
+
+	kfree(cma_dev_group);
+};
+
+static void release_cma_ports_group(struct config_item  *item)
+{
+	struct config_group *group = container_of(item, struct config_group,
+						  cg_item);
+	struct cma_dev_group *cma_dev_group = container_of(group,
+							   struct cma_dev_group,
+							   ports_group);
+
+	kfree(cma_dev_group->ports);
+	kfree(cma_dev_group->default_ports_group);
+	cma_dev_group->ports = NULL;
+	cma_dev_group->default_ports_group = NULL;
+};
+
+static struct configfs_item_operations cma_ports_item_ops = {
+	.release = release_cma_ports_group
+};
+
+static struct config_item_type cma_ports_group_type = {
+	.ct_item_ops	= &cma_ports_item_ops,
+	.ct_owner	= THIS_MODULE
+};
+
+static struct configfs_item_operations cma_device_item_ops = {
+	.release = release_cma_dev
+};
+
+static struct config_item_type cma_device_group_type = {
+	.ct_item_ops	= &cma_device_item_ops,
+	.ct_owner	= THIS_MODULE
+};
+
+static struct config_group *make_cma_dev(struct config_group *group,
+					 const char *name)
+{
+	int err = -ENODEV;
+	struct cma_device *cma_dev = cma_enum_devices_by_ibdev(filter_by_name,
+							       (void *)name);
+	struct cma_dev_group *cma_dev_group = NULL;
+
+	if (!cma_dev)
+		goto fail;
+
+	cma_dev_group = kzalloc(sizeof(*cma_dev_group), GFP_KERNEL);
+
+	if (!cma_dev_group) {
+		err = -ENOMEM;
+		goto fail;
+	}
+
+	strncpy(cma_dev_group->name, name, sizeof(cma_dev_group->name));
+
+	err = make_cma_ports(cma_dev_group, cma_dev);
+	if (err)
+		goto fail;
+
+	cma_dev_group->ports_group.default_groups =
+		cma_dev_group->default_ports_group;
+	config_group_init_type_name(&cma_dev_group->ports_group, "ports",
+				    &cma_ports_group_type);
+
+	cma_dev_group->device_group.default_groups
+		= cma_dev_group->default_dev_group;
+	cma_dev_group->default_dev_group[0] = &cma_dev_group->ports_group;
+	cma_dev_group->default_dev_group[1] = NULL;
+
+	config_group_init_type_name(&cma_dev_group->device_group, name,
+				    &cma_device_group_type);
+
+	cma_deref_dev(cma_dev);
+	return &cma_dev_group->device_group;
+
+fail:
+	if (cma_dev)
+		cma_deref_dev(cma_dev);
+	kfree(cma_dev_group);
+	return ERR_PTR(err);
+}
+
+static struct configfs_group_operations cma_subsys_group_ops = {
+	.make_group	= make_cma_dev,
+};
+
+static struct config_item_type cma_subsys_type = {
+	.ct_group_ops	= &cma_subsys_group_ops,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct configfs_subsystem cma_subsys = {
+	.su_group	= {
+		.cg_item	= {
+			.ci_namebuf	= "rdma_cm",
+			.ci_type	= &cma_subsys_type,
+		},
+	},
+};
+
+int __init cma_configfs_init(void)
+{
+	config_group_init(&cma_subsys.su_group);
+	mutex_init(&cma_subsys.su_mutex);
+	return configfs_register_subsystem(&cma_subsys);
+}
+
+void __exit cma_configfs_exit(void)
+{
+	configfs_unregister_subsystem(&cma_subsys);
+}
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 5cf6eb716f00..eab32215756b 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -38,6 +38,32 @@
 
 #include <rdma/ib_verbs.h>
 
+#if IS_ENABLED(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS)
+int cma_configfs_init(void);
+void cma_configfs_exit(void);
+#else
+static inline int cma_configfs_init(void)
+{
+	return 0;
+}
+
+static inline void cma_configfs_exit(void)
+{
+}
+#endif
+struct cma_device;
+void cma_ref_dev(struct cma_device *cma_dev);
+void cma_deref_dev(struct cma_device *cma_dev);
+typedef bool (*cma_device_filter)(struct ib_device *, void *);
+struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,
+					     void		*cookie);
+int cma_get_default_gid_type(struct cma_device *cma_dev,
+			     unsigned int port);
+int cma_set_default_gid_type(struct cma_device *cma_dev,
+			     unsigned int port,
+			     enum ib_gid_type default_gid_type);
+struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev);
+
 int  ib_device_register_sysfs(struct ib_device *device,
 			      int (*port_callback)(struct ib_device *,
 						   u8, struct kobject *));
@@ -70,8 +96,13 @@ enum ib_cache_gid_default_mode {
 	IB_CACHE_GID_DEFAULT_MODE_DELETE
 };
 
+int ib_cache_gid_parse_type_str(const char *buf);
+
+const char *ib_cache_gid_type_str(enum ib_gid_type gid_type);
+
 void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
 				  struct net_device *ndev,
+				  unsigned long gid_type_mask,
 				  enum ib_cache_gid_default_mode mode);
 
 int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
@@ -87,9 +118,23 @@ int roce_gid_mgmt_init(void);
 void roce_gid_mgmt_cleanup(void);
 
 int roce_rescan_device(struct ib_device *ib_dev);
+unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port);
 
 int ib_cache_setup_one(struct ib_device *device);
 void ib_cache_cleanup_one(struct ib_device *device);
 void ib_cache_release_one(struct ib_device *device);
 
+static inline bool rdma_is_upper_dev_rcu(struct net_device *dev,
+					 struct net_device *upper)
+{
+	struct net_device *_upper = NULL;
+	struct list_head *iter;
+
+	netdev_for_each_all_upper_dev_rcu(dev, _upper, iter)
+		if (_upper == upper)
+			break;
+
+	return _upper == upper;
+}
+
 #endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
new file mode 100644
index 000000000000..a754fc727de5
--- /dev/null
+++ b/drivers/infiniband/core/cq.c
@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2015 HGST, a Western Digital Company.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <rdma/ib_verbs.h>
+
+/* # of WCs to poll for with a single call to ib_poll_cq */
+#define IB_POLL_BATCH			16
+
+/* # of WCs to iterate over before yielding */
+#define IB_POLL_BUDGET_IRQ		256
+#define IB_POLL_BUDGET_WORKQUEUE	65536
+
+#define IB_POLL_FLAGS \
+	(IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
+
+static int __ib_process_cq(struct ib_cq *cq, int budget)
+{
+	int i, n, completed = 0;
+
+	while ((n = ib_poll_cq(cq, IB_POLL_BATCH, cq->wc)) > 0) {
+		for (i = 0; i < n; i++) {
+			struct ib_wc *wc = &cq->wc[i];
+
+			if (wc->wr_cqe)
+				wc->wr_cqe->done(cq, wc);
+			else
+				WARN_ON_ONCE(wc->status == IB_WC_SUCCESS);
+		}
+
+		completed += n;
+
+		if (n != IB_POLL_BATCH ||
+		    (budget != -1 && completed >= budget))
+			break;
+	}
+
+	return completed;
+}
+
+/**
+ * ib_process_direct_cq - process a CQ in caller context
+ * @cq:		CQ to process
+ * @budget:	number of CQEs to poll for
+ *
+ * This function is used to process all outstanding CQ entries on a
+ * %IB_POLL_DIRECT CQ.  It does not offload CQ processing to a different
+ * context and does not ask for completion interrupts from the HCA.
+ *
+ * Note: for compatibility reasons -1 can be passed in %budget for unlimited
+ * polling.  Do not use this feature in new code, it will be removed soon.
+ */
+int ib_process_cq_direct(struct ib_cq *cq, int budget)
+{
+	WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT);
+
+	return __ib_process_cq(cq, budget);
+}
+EXPORT_SYMBOL(ib_process_cq_direct);
+
+static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
+{
+	WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);
+}
+
+static int ib_poll_handler(struct irq_poll *iop, int budget)
+{
+	struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
+	int completed;
+
+	completed = __ib_process_cq(cq, budget);
+	if (completed < budget) {
+		irq_poll_complete(&cq->iop);
+		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
+			irq_poll_sched(&cq->iop);
+	}
+
+	return completed;
+}
+
+static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
+{
+	irq_poll_sched(&cq->iop);
+}
+
+static void ib_cq_poll_work(struct work_struct *work)
+{
+	struct ib_cq *cq = container_of(work, struct ib_cq, work);
+	int completed;
+
+	completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE);
+	if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
+	    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
+		queue_work(ib_comp_wq, &cq->work);
+}
+
+static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
+{
+	queue_work(ib_comp_wq, &cq->work);
+}
+
+/**
+ * ib_alloc_cq - allocate a completion queue
+ * @dev:		device to allocate the CQ for
+ * @private:		driver private data, accessible from cq->cq_context
+ * @nr_cqe:		number of CQEs to allocate
+ * @comp_vector:	HCA completion vectors for this CQ
+ * @poll_ctx:		context to poll the CQ from.
+ *
+ * This is the proper interface to allocate a CQ for in-kernel users. A
+ * CQ allocated with this interface will automatically be polled from the
+ * specified context.  The ULP needs must use wr->wr_cqe instead of wr->wr_id
+ * to use this CQ abstraction.
+ */
+struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
+		int nr_cqe, int comp_vector, enum ib_poll_context poll_ctx)
+{
+	struct ib_cq_init_attr cq_attr = {
+		.cqe		= nr_cqe,
+		.comp_vector	= comp_vector,
+	};
+	struct ib_cq *cq;
+	int ret = -ENOMEM;
+
+	cq = dev->create_cq(dev, &cq_attr, NULL, NULL);
+	if (IS_ERR(cq))
+		return cq;
+
+	cq->device = dev;
+	cq->uobject = NULL;
+	cq->event_handler = NULL;
+	cq->cq_context = private;
+	cq->poll_ctx = poll_ctx;
+	atomic_set(&cq->usecnt, 0);
+
+	cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
+	if (!cq->wc)
+		goto out_destroy_cq;
+
+	switch (cq->poll_ctx) {
+	case IB_POLL_DIRECT:
+		cq->comp_handler = ib_cq_completion_direct;
+		break;
+	case IB_POLL_SOFTIRQ:
+		cq->comp_handler = ib_cq_completion_softirq;
+
+		irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
+		ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+		break;
+	case IB_POLL_WORKQUEUE:
+		cq->comp_handler = ib_cq_completion_workqueue;
+		INIT_WORK(&cq->work, ib_cq_poll_work);
+		ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+		break;
+	default:
+		ret = -EINVAL;
+		goto out_free_wc;
+	}
+
+	return cq;
+
+out_free_wc:
+	kfree(cq->wc);
+out_destroy_cq:
+	cq->device->destroy_cq(cq);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(ib_alloc_cq);
+
+/**
+ * ib_free_cq - free a completion queue
+ * @cq:		completion queue to free.
+ */
+void ib_free_cq(struct ib_cq *cq)
+{
+	int ret;
+
+	if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
+		return;
+
+	switch (cq->poll_ctx) {
+	case IB_POLL_DIRECT:
+		break;
+	case IB_POLL_SOFTIRQ:
+		irq_poll_disable(&cq->iop);
+		break;
+	case IB_POLL_WORKQUEUE:
+		flush_work(&cq->work);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+	}
+
+	kfree(cq->wc);
+	ret = cq->device->destroy_cq(cq);
+	WARN_ON_ONCE(ret);
+}
+EXPORT_SYMBOL(ib_free_cq);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 179e8134d57f..00da80e02154 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -58,6 +58,7 @@ struct ib_client_data {
 	bool		  going_down;
 };
 
+struct workqueue_struct *ib_comp_wq;
 struct workqueue_struct *ib_wq;
 EXPORT_SYMBOL_GPL(ib_wq);
 
@@ -325,6 +326,7 @@ int ib_register_device(struct ib_device *device,
 {
 	int ret;
 	struct ib_client *client;
+	struct ib_udata uhw = {.outlen = 0, .inlen = 0};
 
 	mutex_lock(&device_mutex);
 
@@ -352,6 +354,13 @@ int ib_register_device(struct ib_device *device,
 		goto out;
 	}
 
+	memset(&device->attrs, 0, sizeof(device->attrs));
+	ret = device->query_device(device, &device->attrs, &uhw);
+	if (ret) {
+		printk(KERN_WARNING "Couldn't query the device attributes\n");
+		goto out;
+	}
+
 	ret = ib_device_register_sysfs(device, port_callback);
 	if (ret) {
 		printk(KERN_WARNING "Couldn't register device %s with driver model\n",
@@ -628,25 +637,6 @@ void ib_dispatch_event(struct ib_event *event)
 EXPORT_SYMBOL(ib_dispatch_event);
 
 /**
- * ib_query_device - Query IB device attributes
- * @device:Device to query
- * @device_attr:Device attributes
- *
- * ib_query_device() returns the attributes of a device through the
- * @device_attr pointer.
- */
-int ib_query_device(struct ib_device *device,
-		    struct ib_device_attr *device_attr)
-{
-	struct ib_udata uhw = {.outlen = 0, .inlen = 0};
-
-	memset(device_attr, 0, sizeof(*device_attr));
-
-	return device->query_device(device, device_attr, &uhw);
-}
-EXPORT_SYMBOL(ib_query_device);
-
-/**
  * ib_query_port - Query IB port attributes
  * @device:Device to query
  * @port_num:Port number to query
@@ -825,26 +815,31 @@ EXPORT_SYMBOL(ib_modify_port);
  *   a specified GID value occurs.
  * @device: The device to query.
  * @gid: The GID value to search for.
+ * @gid_type: Type of GID.
  * @ndev: The ndev related to the GID to search for.
  * @port_num: The port number of the device where the GID value was found.
  * @index: The index into the GID table where the GID was found.  This
  *   parameter may be NULL.
  */
 int ib_find_gid(struct ib_device *device, union ib_gid *gid,
-		struct net_device *ndev, u8 *port_num, u16 *index)
+		enum ib_gid_type gid_type, struct net_device *ndev,
+		u8 *port_num, u16 *index)
 {
 	union ib_gid tmp_gid;
 	int ret, port, i;
 
 	for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) {
 		if (rdma_cap_roce_gid_table(device, port)) {
-			if (!ib_find_cached_gid_by_port(device, gid, port,
+			if (!ib_find_cached_gid_by_port(device, gid, gid_type, port,
 							ndev, index)) {
 				*port_num = port;
 				return 0;
 			}
 		}
 
+		if (gid_type != IB_GID_TYPE_IB)
+			continue;
+
 		for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) {
 			ret = ib_query_gid(device, port, i, &tmp_gid, NULL);
 			if (ret)
@@ -954,10 +949,18 @@ static int __init ib_core_init(void)
 	if (!ib_wq)
 		return -ENOMEM;
 
+	ib_comp_wq = alloc_workqueue("ib-comp-wq",
+			WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
+			WQ_UNBOUND_MAX_ACTIVE);
+	if (!ib_comp_wq) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
 	ret = class_register(&ib_class);
 	if (ret) {
 		printk(KERN_WARNING "Couldn't create InfiniBand device class\n");
-		goto err;
+		goto err_comp;
 	}
 
 	ret = ibnl_init();
@@ -972,7 +975,8 @@ static int __init ib_core_init(void)
 
 err_sysfs:
 	class_unregister(&ib_class);
-
+err_comp:
+	destroy_workqueue(ib_comp_wq);
 err:
 	destroy_workqueue(ib_wq);
 	return ret;
@@ -983,6 +987,7 @@ static void __exit ib_core_cleanup(void)
 	ib_cache_cleanup();
 	ibnl_cleanup();
 	class_unregister(&ib_class);
+	destroy_workqueue(ib_comp_wq);
 	/* Make sure that any pending umem accounting work is done. */
 	destroy_workqueue(ib_wq);
 }
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
index 9f5ad7cc33c8..6ac3683c144b 100644
--- a/drivers/infiniband/core/fmr_pool.c
+++ b/drivers/infiniband/core/fmr_pool.c
@@ -212,7 +212,6 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,
 {
 	struct ib_device   *device;
 	struct ib_fmr_pool *pool;
-	struct ib_device_attr *attr;
 	int i;
 	int ret;
 	int max_remaps;
@@ -228,25 +227,10 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,
 		return ERR_PTR(-ENOSYS);
 	}
 
-	attr = kmalloc(sizeof *attr, GFP_KERNEL);
-	if (!attr) {
-		printk(KERN_WARNING PFX "couldn't allocate device attr struct\n");
-		return ERR_PTR(-ENOMEM);
-	}
-
-	ret = ib_query_device(device, attr);
-	if (ret) {
-		printk(KERN_WARNING PFX "couldn't query device: %d\n", ret);
-		kfree(attr);
-		return ERR_PTR(ret);
-	}
-
-	if (!attr->max_map_per_fmr)
+	if (!device->attrs.max_map_per_fmr)
 		max_remaps = IB_FMR_MAX_REMAPS;
 	else
-		max_remaps = attr->max_map_per_fmr;
-
-	kfree(attr);
+		max_remaps = device->attrs.max_map_per_fmr;
 
 	pool = kmalloc(sizeof *pool, GFP_KERNEL);
 	if (!pool) {
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 2281de122038..9fa5bf33f5a3 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -84,6 +84,9 @@ static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 			      u8 mgmt_class);
 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 			   struct ib_mad_agent_private *agent_priv);
+static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
+			      struct ib_wc *wc);
+static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);
 
 /*
  * Returns a ib_mad_port_private structure or NULL for a device/port
@@ -681,7 +684,7 @@ static void snoop_recv(struct ib_mad_qp_info *qp_info,
 
 		atomic_inc(&mad_snoop_priv->refcount);
 		spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-		mad_snoop_priv->agent.recv_handler(&mad_snoop_priv->agent,
+		mad_snoop_priv->agent.recv_handler(&mad_snoop_priv->agent, NULL,
 						   mad_recv_wc);
 		deref_snoop_agent(mad_snoop_priv);
 		spin_lock_irqsave(&qp_info->snoop_lock, flags);
@@ -689,12 +692,11 @@ static void snoop_recv(struct ib_mad_qp_info *qp_info,
 	spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
 }
 
-static void build_smp_wc(struct ib_qp *qp,
-			 u64 wr_id, u16 slid, u16 pkey_index, u8 port_num,
-			 struct ib_wc *wc)
+static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
+		u16 pkey_index, u8 port_num, struct ib_wc *wc)
 {
 	memset(wc, 0, sizeof *wc);
-	wc->wr_id = wr_id;
+	wc->wr_cqe = cqe;
 	wc->status = IB_WC_SUCCESS;
 	wc->opcode = IB_WC_RECV;
 	wc->pkey_index = pkey_index;
@@ -832,7 +834,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	}
 
 	build_smp_wc(mad_agent_priv->agent.qp,
-		     send_wr->wr.wr_id, drslid,
+		     send_wr->wr.wr_cqe, drslid,
 		     send_wr->pkey_index,
 		     send_wr->port_num, &mad_wc);
 
@@ -1039,7 +1041,9 @@ struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
 
 	mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;
 
-	mad_send_wr->send_wr.wr.wr_id = (unsigned long) mad_send_wr;
+	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
+
+	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
 	mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;
 	mad_send_wr->send_wr.wr.num_sge = 2;
 	mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;
@@ -1151,8 +1155,9 @@ int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
 
 	/* Set WR ID to find mad_send_wr upon completion */
 	qp_info = mad_send_wr->mad_agent_priv->qp_info;
-	mad_send_wr->send_wr.wr.wr_id = (unsigned long)&mad_send_wr->mad_list;
 	mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
+	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
+	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
 
 	mad_agent = mad_send_wr->send_buf.mad_agent;
 	sge = mad_send_wr->sg_list;
@@ -1982,9 +1987,9 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 				/* user rmpp is in effect
 				 * and this is an active RMPP MAD
 				 */
-				mad_recv_wc->wc->wr_id = 0;
-				mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent,
-								   mad_recv_wc);
+				mad_agent_priv->agent.recv_handler(
+						&mad_agent_priv->agent, NULL,
+						mad_recv_wc);
 				atomic_dec(&mad_agent_priv->refcount);
 			} else {
 				/* not user rmpp, revert to normal behavior and
@@ -1998,9 +2003,10 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
 			/* Defined behavior is to complete response before request */
-			mad_recv_wc->wc->wr_id = (unsigned long) &mad_send_wr->send_buf;
-			mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent,
-							   mad_recv_wc);
+			mad_agent_priv->agent.recv_handler(
+					&mad_agent_priv->agent,
+					&mad_send_wr->send_buf,
+					mad_recv_wc);
 			atomic_dec(&mad_agent_priv->refcount);
 
 			mad_send_wc.status = IB_WC_SUCCESS;
@@ -2009,7 +2015,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 			ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
 		}
 	} else {
-		mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent,
+		mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
 						   mad_recv_wc);
 		deref_mad_agent(mad_agent_priv);
 	}
@@ -2172,13 +2178,14 @@ handle_smi(struct ib_mad_port_private *port_priv,
 	return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response);
 }
 
-static void ib_mad_recv_done_handler(struct ib_mad_port_private *port_priv,
-				     struct ib_wc *wc)
+static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 {
+	struct ib_mad_port_private *port_priv = cq->cq_context;
+	struct ib_mad_list_head *mad_list =
+		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
 	struct ib_mad_qp_info *qp_info;
 	struct ib_mad_private_header *mad_priv_hdr;
 	struct ib_mad_private *recv, *response = NULL;
-	struct ib_mad_list_head *mad_list;
 	struct ib_mad_agent_private *mad_agent;
 	int port_num;
 	int ret = IB_MAD_RESULT_SUCCESS;
@@ -2186,7 +2193,17 @@ static void ib_mad_recv_done_handler(struct ib_mad_port_private *port_priv,
 	u16 resp_mad_pkey_index = 0;
 	bool opa;
 
-	mad_list = (struct ib_mad_list_head *)(unsigned long)wc->wr_id;
+	if (list_empty_careful(&port_priv->port_list))
+		return;
+
+	if (wc->status != IB_WC_SUCCESS) {
+		/*
+		 * Receive errors indicate that the QP has entered the error
+		 * state - error handling/shutdown code will cleanup
+		 */
+		return;
+	}
+
 	qp_info = mad_list->mad_queue->qp_info;
 	dequeue_mad(mad_list);
 
@@ -2227,7 +2244,7 @@ static void ib_mad_recv_done_handler(struct ib_mad_port_private *port_priv,
 	response = alloc_mad_private(mad_size, GFP_KERNEL);
 	if (!response) {
 		dev_err(&port_priv->device->dev,
-			"ib_mad_recv_done_handler no memory for response buffer\n");
+			"%s: no memory for response buffer\n", __func__);
 		goto out;
 	}
 
@@ -2413,11 +2430,12 @@ done:
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 }
 
-static void ib_mad_send_done_handler(struct ib_mad_port_private *port_priv,
-				     struct ib_wc *wc)
+static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
 {
+	struct ib_mad_port_private *port_priv = cq->cq_context;
+	struct ib_mad_list_head *mad_list =
+		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
 	struct ib_mad_send_wr_private	*mad_send_wr, *queued_send_wr;
-	struct ib_mad_list_head		*mad_list;
 	struct ib_mad_qp_info		*qp_info;
 	struct ib_mad_queue		*send_queue;
 	struct ib_send_wr		*bad_send_wr;
@@ -2425,7 +2443,14 @@ static void ib_mad_send_done_handler(struct ib_mad_port_private *port_priv,
 	unsigned long flags;
 	int ret;
 
-	mad_list = (struct ib_mad_list_head *)(unsigned long)wc->wr_id;
+	if (list_empty_careful(&port_priv->port_list))
+		return;
+
+	if (wc->status != IB_WC_SUCCESS) {
+		if (!ib_mad_send_error(port_priv, wc))
+			return;
+	}
+
 	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
 				   mad_list);
 	send_queue = mad_list->mad_queue;
@@ -2490,24 +2515,15 @@ static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info)
 	spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
 }
 
-static void mad_error_handler(struct ib_mad_port_private *port_priv,
-			      struct ib_wc *wc)
+static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
+		struct ib_wc *wc)
 {
-	struct ib_mad_list_head *mad_list;
-	struct ib_mad_qp_info *qp_info;
+	struct ib_mad_list_head *mad_list =
+		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
+	struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info;
 	struct ib_mad_send_wr_private *mad_send_wr;
 	int ret;
 
-	/* Determine if failure was a send or receive */
-	mad_list = (struct ib_mad_list_head *)(unsigned long)wc->wr_id;
-	qp_info = mad_list->mad_queue->qp_info;
-	if (mad_list->mad_queue == &qp_info->recv_queue)
-		/*
-		 * Receive errors indicate that the QP has entered the error
-		 * state - error handling/shutdown code will cleanup
-		 */
-		return;
-
 	/*
 	 * Send errors will transition the QP to SQE - move
 	 * QP to RTS and repost flushed work requests
@@ -2522,10 +2538,9 @@ static void mad_error_handler(struct ib_mad_port_private *port_priv,
 			mad_send_wr->retry = 0;
 			ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
 					&bad_send_wr);
-			if (ret)
-				ib_mad_send_done_handler(port_priv, wc);
-		} else
-			ib_mad_send_done_handler(port_priv, wc);
+			if (!ret)
+				return false;
+		}
 	} else {
 		struct ib_qp_attr *attr;
 
@@ -2539,42 +2554,14 @@ static void mad_error_handler(struct ib_mad_port_private *port_priv,
 			kfree(attr);
 			if (ret)
 				dev_err(&port_priv->device->dev,
-					"mad_error_handler - ib_modify_qp to RTS : %d\n",
-					ret);
+					"%s - ib_modify_qp to RTS: %d\n",
+					__func__, ret);
 			else
 				mark_sends_for_retry(qp_info);
 		}
-		ib_mad_send_done_handler(port_priv, wc);
 	}
-}
 
-/*
- * IB MAD completion callback
- */
-static void ib_mad_completion_handler(struct work_struct *work)
-{
-	struct ib_mad_port_private *port_priv;
-	struct ib_wc wc;
-
-	port_priv = container_of(work, struct ib_mad_port_private, work);
-	ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
-
-	while (ib_poll_cq(port_priv->cq, 1, &wc) == 1) {
-		if (wc.status == IB_WC_SUCCESS) {
-			switch (wc.opcode) {
-			case IB_WC_SEND:
-				ib_mad_send_done_handler(port_priv, &wc);
-				break;
-			case IB_WC_RECV:
-				ib_mad_recv_done_handler(port_priv, &wc);
-				break;
-			default:
-				BUG_ON(1);
-				break;
-			}
-		} else
-			mad_error_handler(port_priv, &wc);
-	}
+	return true;
 }
 
 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
@@ -2716,7 +2703,7 @@ static void local_completions(struct work_struct *work)
 			 * before request
 			 */
 			build_smp_wc(recv_mad_agent->agent.qp,
-				     (unsigned long) local->mad_send_wr,
+				     local->mad_send_wr->send_wr.wr.wr_cqe,
 				     be16_to_cpu(IB_LID_PERMISSIVE),
 				     local->mad_send_wr->send_wr.pkey_index,
 				     recv_mad_agent->agent.port_num, &wc);
@@ -2744,6 +2731,7 @@ static void local_completions(struct work_struct *work)
 					   IB_MAD_SNOOP_RECVS);
 			recv_mad_agent->agent.recv_handler(
 						&recv_mad_agent->agent,
+						&local->mad_send_wr->send_buf,
 						&local->mad_priv->header.recv_wc);
 			spin_lock_irqsave(&recv_mad_agent->lock, flags);
 			atomic_dec(&recv_mad_agent->refcount);
@@ -2855,17 +2843,6 @@ static void timeout_sends(struct work_struct *work)
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 }
 
-static void ib_mad_thread_completion_handler(struct ib_cq *cq, void *arg)
-{
-	struct ib_mad_port_private *port_priv = cq->cq_context;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
-	if (!list_empty(&port_priv->port_list))
-		queue_work(port_priv->wq, &port_priv->work);
-	spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
-}
-
 /*
  * Allocate receive MADs and post receive WRs for them
  */
@@ -2913,8 +2890,9 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
 			break;
 		}
 		mad_priv->header.mapping = sg_list.addr;
-		recv_wr.wr_id = (unsigned long)&mad_priv->header.mad_list;
 		mad_priv->header.mad_list.mad_queue = recv_queue;
+		mad_priv->header.mad_list.cqe.done = ib_mad_recv_done;
+		recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe;
 
 		/* Post receive WR */
 		spin_lock_irqsave(&recv_queue->lock, flags);
@@ -3151,7 +3129,6 @@ static int ib_mad_port_open(struct ib_device *device,
 	unsigned long flags;
 	char name[sizeof "ib_mad123"];
 	int has_smi;
-	struct ib_cq_init_attr cq_attr = {};
 
 	if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE))
 		return -EFAULT;
@@ -3179,10 +3156,8 @@ static int ib_mad_port_open(struct ib_device *device,
 	if (has_smi)
 		cq_size *= 2;
 
-	cq_attr.cqe = cq_size;
-	port_priv->cq = ib_create_cq(port_priv->device,
-				     ib_mad_thread_completion_handler,
-				     NULL, port_priv, &cq_attr);
+	port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0,
+			IB_POLL_WORKQUEUE);
 	if (IS_ERR(port_priv->cq)) {
 		dev_err(&device->dev, "Couldn't create ib_mad CQ\n");
 		ret = PTR_ERR(port_priv->cq);
@@ -3211,7 +3186,6 @@ static int ib_mad_port_open(struct ib_device *device,
 		ret = -ENOMEM;
 		goto error8;
 	}
-	INIT_WORK(&port_priv->work, ib_mad_completion_handler);
 
 	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
 	list_add_tail(&port_priv->port_list, &ib_mad_port_list);
@@ -3238,7 +3212,7 @@ error7:
 error6:
 	ib_dealloc_pd(port_priv->pd);
 error4:
-	ib_destroy_cq(port_priv->cq);
+	ib_free_cq(port_priv->cq);
 	cleanup_recv_queue(&port_priv->qp_info[1]);
 	cleanup_recv_queue(&port_priv->qp_info[0]);
 error3:
@@ -3271,7 +3245,7 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)
 	destroy_mad_qp(&port_priv->qp_info[1]);
 	destroy_mad_qp(&port_priv->qp_info[0]);
 	ib_dealloc_pd(port_priv->pd);
-	ib_destroy_cq(port_priv->cq);
+	ib_free_cq(port_priv->cq);
 	cleanup_recv_queue(&port_priv->qp_info[1]);
 	cleanup_recv_queue(&port_priv->qp_info[0]);
 	/* XXX: Handle deallocation of MAD registration tables */
diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
index 990698a6ab4b..28669f6419e1 100644
--- a/drivers/infiniband/core/mad_priv.h
+++ b/drivers/infiniband/core/mad_priv.h
@@ -64,6 +64,7 @@
 
 struct ib_mad_list_head {
 	struct list_head list;
+	struct ib_cqe cqe;
 	struct ib_mad_queue *mad_queue;
 };
 
@@ -204,7 +205,6 @@ struct ib_mad_port_private {
 	struct ib_mad_mgmt_version_table version[MAX_MGMT_VERSION];
 	struct list_head agent_list;
 	struct workqueue_struct *wq;
-	struct work_struct work;
 	struct ib_mad_qp_info qp_info[IB_MAD_QPS_CORE];
 };
 
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index bb6685fb08c6..250937cb9a1a 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -723,14 +723,27 @@ EXPORT_SYMBOL(ib_sa_get_mcmember_rec);
 
 int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
 			     struct ib_sa_mcmember_rec *rec,
+			     struct net_device *ndev,
+			     enum ib_gid_type gid_type,
 			     struct ib_ah_attr *ah_attr)
 {
 	int ret;
 	u16 gid_index;
 	u8 p;
 
-	ret = ib_find_cached_gid(device, &rec->port_gid,
-				 NULL, &p, &gid_index);
+	if (rdma_protocol_roce(device, port_num)) {
+		ret = ib_find_cached_gid_by_port(device, &rec->port_gid,
+						 gid_type, port_num,
+						 ndev,
+						 &gid_index);
+	} else if (rdma_protocol_ib(device, port_num)) {
+		ret = ib_find_cached_gid(device, &rec->port_gid,
+					 IB_GID_TYPE_IB, NULL, &p,
+					 &gid_index);
+	} else {
+		ret = -EINVAL;
+	}
+
 	if (ret)
 		return ret;
 
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index 178f98482e13..06556c34606d 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -67,17 +67,53 @@ struct netdev_event_work {
 	struct netdev_event_work_cmd	cmds[ROCE_NETDEV_CALLBACK_SZ];
 };
 
+static const struct {
+	bool (*is_supported)(const struct ib_device *device, u8 port_num);
+	enum ib_gid_type gid_type;
+} PORT_CAP_TO_GID_TYPE[] = {
+	{rdma_protocol_roce_eth_encap, IB_GID_TYPE_ROCE},
+	{rdma_protocol_roce_udp_encap, IB_GID_TYPE_ROCE_UDP_ENCAP},
+};
+
+#define CAP_TO_GID_TABLE_SIZE	ARRAY_SIZE(PORT_CAP_TO_GID_TYPE)
+
+unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port)
+{
+	int i;
+	unsigned int ret_flags = 0;
+
+	if (!rdma_protocol_roce(ib_dev, port))
+		return 1UL << IB_GID_TYPE_IB;
+
+	for (i = 0; i < CAP_TO_GID_TABLE_SIZE; i++)
+		if (PORT_CAP_TO_GID_TYPE[i].is_supported(ib_dev, port))
+			ret_flags |= 1UL << PORT_CAP_TO_GID_TYPE[i].gid_type;
+
+	return ret_flags;
+}
+EXPORT_SYMBOL(roce_gid_type_mask_support);
+
 static void update_gid(enum gid_op_type gid_op, struct ib_device *ib_dev,
 		       u8 port, union ib_gid *gid,
 		       struct ib_gid_attr *gid_attr)
 {
-	switch (gid_op) {
-	case GID_ADD:
-		ib_cache_gid_add(ib_dev, port, gid, gid_attr);
-		break;
-	case GID_DEL:
-		ib_cache_gid_del(ib_dev, port, gid, gid_attr);
-		break;
+	int i;
+	unsigned long gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
+
+	for (i = 0; i < IB_GID_TYPE_SIZE; i++) {
+		if ((1UL << i) & gid_type_mask) {
+			gid_attr->gid_type = i;
+			switch (gid_op) {
+			case GID_ADD:
+				ib_cache_gid_add(ib_dev, port,
+						 gid, gid_attr);
+				break;
+			case GID_DEL:
+				ib_cache_gid_del(ib_dev, port,
+						 gid, gid_attr);
+				break;
+			}
+		}
 	}
 }
 
@@ -103,18 +139,6 @@ static enum bonding_slave_state is_eth_active_slave_of_bonding_rcu(struct net_de
 	return BONDING_SLAVE_STATE_NA;
 }
 
-static bool is_upper_dev_rcu(struct net_device *dev, struct net_device *upper)
-{
-	struct net_device *_upper = NULL;
-	struct list_head *iter;
-
-	netdev_for_each_all_upper_dev_rcu(dev, _upper, iter)
-		if (_upper == upper)
-			break;
-
-	return _upper == upper;
-}
-
 #define REQUIRED_BOND_STATES		(BONDING_SLAVE_STATE_ACTIVE |	\
 					 BONDING_SLAVE_STATE_NA)
 static int is_eth_port_of_netdev(struct ib_device *ib_dev, u8 port,
@@ -132,7 +156,7 @@ static int is_eth_port_of_netdev(struct ib_device *ib_dev, u8 port,
 	if (!real_dev)
 		real_dev = event_ndev;
 
-	res = ((is_upper_dev_rcu(rdma_ndev, event_ndev) &&
+	res = ((rdma_is_upper_dev_rcu(rdma_ndev, event_ndev) &&
 	       (is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev) &
 		REQUIRED_BOND_STATES)) ||
 	       real_dev == rdma_ndev);
@@ -178,7 +202,7 @@ static int upper_device_filter(struct ib_device *ib_dev, u8 port,
 		return 1;
 
 	rcu_read_lock();
-	res = is_upper_dev_rcu(rdma_ndev, event_ndev);
+	res = rdma_is_upper_dev_rcu(rdma_ndev, event_ndev);
 	rcu_read_unlock();
 
 	return res;
@@ -203,10 +227,12 @@ static void enum_netdev_default_gids(struct ib_device *ib_dev,
 				     u8 port, struct net_device *event_ndev,
 				     struct net_device *rdma_ndev)
 {
+	unsigned long gid_type_mask;
+
 	rcu_read_lock();
 	if (!rdma_ndev ||
 	    ((rdma_ndev != event_ndev &&
-	      !is_upper_dev_rcu(rdma_ndev, event_ndev)) ||
+	      !rdma_is_upper_dev_rcu(rdma_ndev, event_ndev)) ||
 	     is_eth_active_slave_of_bonding_rcu(rdma_ndev,
 						netdev_master_upper_dev_get_rcu(rdma_ndev)) ==
 	     BONDING_SLAVE_STATE_INACTIVE)) {
@@ -215,7 +241,9 @@ static void enum_netdev_default_gids(struct ib_device *ib_dev,
 	}
 	rcu_read_unlock();
 
-	ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev,
+	gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
+
+	ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev, gid_type_mask,
 				     IB_CACHE_GID_DEFAULT_MODE_SET);
 }
 
@@ -234,12 +262,17 @@ static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,
 
 	rcu_read_lock();
 
-	if (is_upper_dev_rcu(rdma_ndev, event_ndev) &&
+	if (rdma_is_upper_dev_rcu(rdma_ndev, event_ndev) &&
 	    is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev) ==
 	    BONDING_SLAVE_STATE_INACTIVE) {
+		unsigned long gid_type_mask;
+
 		rcu_read_unlock();
 
+		gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
+
 		ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev,
+					     gid_type_mask,
 					     IB_CACHE_GID_DEFAULT_MODE_DELETE);
 	} else {
 		rcu_read_unlock();
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index a95a32ba596e..f334090bb612 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -49,7 +49,9 @@
 #include <net/netlink.h>
 #include <uapi/rdma/ib_user_sa.h>
 #include <rdma/ib_marshall.h>
+#include <rdma/ib_addr.h>
 #include "sa.h"
+#include "core_priv.h"
 
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("InfiniBand subnet administration query support");
@@ -715,7 +717,9 @@ static int ib_nl_handle_set_timeout(struct sk_buff *skb,
 	struct nlattr *tb[LS_NLA_TYPE_MAX];
 	int ret;
 
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
+	if (!(nlh->nlmsg_flags & NLM_F_REQUEST) ||
+	    !(NETLINK_CB(skb).sk) ||
+	    !netlink_capable(skb, CAP_NET_ADMIN))
 		return -EPERM;
 
 	ret = nla_parse(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
@@ -789,7 +793,9 @@ static int ib_nl_handle_resolve_resp(struct sk_buff *skb,
 	int found = 0;
 	int ret;
 
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
+	if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
+	    !(NETLINK_CB(skb).sk) ||
+	    !netlink_capable(skb, CAP_NET_ADMIN))
 		return -EPERM;
 
 	spin_lock_irqsave(&ib_nl_request_lock, flags);
@@ -996,7 +1002,8 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 {
 	int ret;
 	u16 gid_index;
-	int force_grh;
+	int use_roce;
+	struct net_device *ndev = NULL;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
 	ah_attr->dlid = be16_to_cpu(rec->dlid);
@@ -1006,16 +1013,71 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 	ah_attr->port_num = port_num;
 	ah_attr->static_rate = rec->rate;
 
-	force_grh = rdma_cap_eth_ah(device, port_num);
+	use_roce = rdma_cap_eth_ah(device, port_num);
+
+	if (use_roce) {
+		struct net_device *idev;
+		struct net_device *resolved_dev;
+		struct rdma_dev_addr dev_addr = {.bound_dev_if = rec->ifindex,
+						 .net = rec->net ? rec->net :
+							 &init_net};
+		union {
+			struct sockaddr     _sockaddr;
+			struct sockaddr_in  _sockaddr_in;
+			struct sockaddr_in6 _sockaddr_in6;
+		} sgid_addr, dgid_addr;
+
+		if (!device->get_netdev)
+			return -EOPNOTSUPP;
+
+		rdma_gid2ip(&sgid_addr._sockaddr, &rec->sgid);
+		rdma_gid2ip(&dgid_addr._sockaddr, &rec->dgid);
+
+		/* validate the route */
+		ret = rdma_resolve_ip_route(&sgid_addr._sockaddr,
+					    &dgid_addr._sockaddr, &dev_addr);
+		if (ret)
+			return ret;
 
-	if (rec->hop_limit > 1 || force_grh) {
-		struct net_device *ndev = ib_get_ndev_from_path(rec);
+		if ((dev_addr.network == RDMA_NETWORK_IPV4 ||
+		     dev_addr.network == RDMA_NETWORK_IPV6) &&
+		    rec->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
+			return -EINVAL;
+
+		idev = device->get_netdev(device, port_num);
+		if (!idev)
+			return -ENODEV;
+
+		resolved_dev = dev_get_by_index(dev_addr.net,
+						dev_addr.bound_dev_if);
+		if (resolved_dev->flags & IFF_LOOPBACK) {
+			dev_put(resolved_dev);
+			resolved_dev = idev;
+			dev_hold(resolved_dev);
+		}
+		ndev = ib_get_ndev_from_path(rec);
+		rcu_read_lock();
+		if ((ndev && ndev != resolved_dev) ||
+		    (resolved_dev != idev &&
+		     !rdma_is_upper_dev_rcu(idev, resolved_dev)))
+			ret = -EHOSTUNREACH;
+		rcu_read_unlock();
+		dev_put(idev);
+		dev_put(resolved_dev);
+		if (ret) {
+			if (ndev)
+				dev_put(ndev);
+			return ret;
+		}
+	}
 
+	if (rec->hop_limit > 1 || use_roce) {
 		ah_attr->ah_flags = IB_AH_GRH;
 		ah_attr->grh.dgid = rec->dgid;
 
-		ret = ib_find_cached_gid(device, &rec->sgid, ndev, &port_num,
-					 &gid_index);
+		ret = ib_find_cached_gid_by_port(device, &rec->sgid,
+						 rec->gid_type, port_num, ndev,
+						 &gid_index);
 		if (ret) {
 			if (ndev)
 				dev_put(ndev);
@@ -1029,9 +1091,10 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 		if (ndev)
 			dev_put(ndev);
 	}
-	if (force_grh) {
+
+	if (use_roce)
 		memcpy(ah_attr->dmac, rec->dmac, ETH_ALEN);
-	}
+
 	return 0;
 }
 EXPORT_SYMBOL(ib_init_ah_from_path);
@@ -1157,6 +1220,7 @@ static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
 			  mad->data, &rec);
 		rec.net = NULL;
 		rec.ifindex = 0;
+		rec.gid_type = IB_GID_TYPE_IB;
 		memset(rec.dmac, 0, ETH_ALEN);
 		query->callback(status, &rec, query->context);
 	} else
@@ -1609,14 +1673,15 @@ static void send_handler(struct ib_mad_agent *agent,
 }
 
 static void recv_handler(struct ib_mad_agent *mad_agent,
+			 struct ib_mad_send_buf *send_buf,
 			 struct ib_mad_recv_wc *mad_recv_wc)
 {
 	struct ib_sa_query *query;
-	struct ib_mad_send_buf *mad_buf;
 
-	mad_buf = (void *) (unsigned long) mad_recv_wc->wc->wr_id;
-	query = mad_buf->context[0];
+	if (!send_buf)
+		return;
 
+	query = send_buf->context[0];
 	if (query->callback) {
 		if (mad_recv_wc->wc->status == IB_WC_SUCCESS)
 			query->callback(query,
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index b1f37d4095fa..3de93517efe4 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -37,15 +37,27 @@
 #include <linux/slab.h>
 #include <linux/stat.h>
 #include <linux/string.h>
+#include <linux/netdevice.h>
 
 #include <rdma/ib_mad.h>
+#include <rdma/ib_pma.h>
 
+struct ib_port;
+
+struct gid_attr_group {
+	struct ib_port		*port;
+	struct kobject		kobj;
+	struct attribute_group	ndev;
+	struct attribute_group	type;
+};
 struct ib_port {
 	struct kobject         kobj;
 	struct ib_device      *ibdev;
+	struct gid_attr_group *gid_attr_group;
 	struct attribute_group gid_group;
 	struct attribute_group pkey_group;
 	u8                     port_num;
+	struct attribute_group *pma_table;
 };
 
 struct port_attribute {
@@ -65,6 +77,7 @@ struct port_table_attribute {
 	struct port_attribute	attr;
 	char			name[8];
 	int			index;
+	__be16			attr_id;
 };
 
 static ssize_t port_attr_show(struct kobject *kobj,
@@ -84,6 +97,24 @@ static const struct sysfs_ops port_sysfs_ops = {
 	.show = port_attr_show
 };
 
+static ssize_t gid_attr_show(struct kobject *kobj,
+			     struct attribute *attr, char *buf)
+{
+	struct port_attribute *port_attr =
+		container_of(attr, struct port_attribute, attr);
+	struct ib_port *p = container_of(kobj, struct gid_attr_group,
+					 kobj)->port;
+
+	if (!port_attr->show)
+		return -EIO;
+
+	return port_attr->show(p, port_attr, buf);
+}
+
+static const struct sysfs_ops gid_attr_sysfs_ops = {
+	.show = gid_attr_show
+};
+
 static ssize_t state_show(struct ib_port *p, struct port_attribute *unused,
 			  char *buf)
 {
@@ -281,6 +312,46 @@ static struct attribute *port_default_attrs[] = {
 	NULL
 };
 
+static size_t print_ndev(struct ib_gid_attr *gid_attr, char *buf)
+{
+	if (!gid_attr->ndev)
+		return -EINVAL;
+
+	return sprintf(buf, "%s\n", gid_attr->ndev->name);
+}
+
+static size_t print_gid_type(struct ib_gid_attr *gid_attr, char *buf)
+{
+	return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_attr->gid_type));
+}
+
+static ssize_t _show_port_gid_attr(struct ib_port *p,
+				   struct port_attribute *attr,
+				   char *buf,
+				   size_t (*print)(struct ib_gid_attr *gid_attr,
+						   char *buf))
+{
+	struct port_table_attribute *tab_attr =
+		container_of(attr, struct port_table_attribute, attr);
+	union ib_gid gid;
+	struct ib_gid_attr gid_attr = {};
+	ssize_t ret;
+	va_list args;
+
+	ret = ib_query_gid(p->ibdev, p->port_num, tab_attr->index, &gid,
+			   &gid_attr);
+	if (ret)
+		goto err;
+
+	ret = print(&gid_attr, buf);
+
+err:
+	if (gid_attr.ndev)
+		dev_put(gid_attr.ndev);
+	va_end(args);
+	return ret;
+}
+
 static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr,
 			     char *buf)
 {
@@ -296,6 +367,19 @@ static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr,
 	return sprintf(buf, "%pI6\n", gid.raw);
 }
 
+static ssize_t show_port_gid_attr_ndev(struct ib_port *p,
+				       struct port_attribute *attr, char *buf)
+{
+	return _show_port_gid_attr(p, attr, buf, print_ndev);
+}
+
+static ssize_t show_port_gid_attr_gid_type(struct ib_port *p,
+					   struct port_attribute *attr,
+					   char *buf)
+{
+	return _show_port_gid_attr(p, attr, buf, print_gid_type);
+}
+
 static ssize_t show_port_pkey(struct ib_port *p, struct port_attribute *attr,
 			      char *buf)
 {
@@ -314,24 +398,32 @@ static ssize_t show_port_pkey(struct ib_port *p, struct port_attribute *attr,
 #define PORT_PMA_ATTR(_name, _counter, _width, _offset)			\
 struct port_table_attribute port_pma_attr_##_name = {			\
 	.attr  = __ATTR(_name, S_IRUGO, show_pma_counter, NULL),	\
-	.index = (_offset) | ((_width) << 16) | ((_counter) << 24)	\
+	.index = (_offset) | ((_width) << 16) | ((_counter) << 24),	\
+	.attr_id = IB_PMA_PORT_COUNTERS ,				\
 }
 
-static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr,
-				char *buf)
+#define PORT_PMA_ATTR_EXT(_name, _width, _offset)			\
+struct port_table_attribute port_pma_attr_ext_##_name = {		\
+	.attr  = __ATTR(_name, S_IRUGO, show_pma_counter, NULL),	\
+	.index = (_offset) | ((_width) << 16),				\
+	.attr_id = IB_PMA_PORT_COUNTERS_EXT ,				\
+}
+
+/*
+ * Get a Perfmgmt MAD block of data.
+ * Returns error code or the number of bytes retrieved.
+ */
+static int get_perf_mad(struct ib_device *dev, int port_num, __be16 attr,
+		void *data, int offset, size_t size)
 {
-	struct port_table_attribute *tab_attr =
-		container_of(attr, struct port_table_attribute, attr);
-	int offset = tab_attr->index & 0xffff;
-	int width  = (tab_attr->index >> 16) & 0xff;
-	struct ib_mad *in_mad  = NULL;
-	struct ib_mad *out_mad = NULL;
+	struct ib_mad *in_mad;
+	struct ib_mad *out_mad;
 	size_t mad_size = sizeof(*out_mad);
 	u16 out_mad_pkey_index = 0;
 	ssize_t ret;
 
-	if (!p->ibdev->process_mad)
-		return sprintf(buf, "N/A (no PMA)\n");
+	if (!dev->process_mad)
+		return -ENOSYS;
 
 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
@@ -344,12 +436,13 @@ static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr,
 	in_mad->mad_hdr.mgmt_class    = IB_MGMT_CLASS_PERF_MGMT;
 	in_mad->mad_hdr.class_version = 1;
 	in_mad->mad_hdr.method        = IB_MGMT_METHOD_GET;
-	in_mad->mad_hdr.attr_id       = cpu_to_be16(0x12); /* PortCounters */
+	in_mad->mad_hdr.attr_id       = attr;
 
-	in_mad->data[41] = p->port_num;	/* PortSelect field */
+	if (attr != IB_PMA_CLASS_PORT_INFO)
+		in_mad->data[41] = port_num;	/* PortSelect field */
 
-	if ((p->ibdev->process_mad(p->ibdev, IB_MAD_IGNORE_MKEY,
-		 p->port_num, NULL, NULL,
+	if ((dev->process_mad(dev, IB_MAD_IGNORE_MKEY,
+		 port_num, NULL, NULL,
 		 (const struct ib_mad_hdr *)in_mad, mad_size,
 		 (struct ib_mad_hdr *)out_mad, &mad_size,
 		 &out_mad_pkey_index) &
@@ -358,31 +451,54 @@ static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr,
 		ret = -EINVAL;
 		goto out;
 	}
+	memcpy(data, out_mad->data + offset, size);
+	ret = size;
+out:
+	kfree(in_mad);
+	kfree(out_mad);
+	return ret;
+}
+
+static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr,
+				char *buf)
+{
+	struct port_table_attribute *tab_attr =
+		container_of(attr, struct port_table_attribute, attr);
+	int offset = tab_attr->index & 0xffff;
+	int width  = (tab_attr->index >> 16) & 0xff;
+	ssize_t ret;
+	u8 data[8];
+
+	ret = get_perf_mad(p->ibdev, p->port_num, tab_attr->attr_id, &data,
+			40 + offset / 8, sizeof(data));
+	if (ret < 0)
+		return sprintf(buf, "N/A (no PMA)\n");
 
 	switch (width) {
 	case 4:
-		ret = sprintf(buf, "%u\n", (out_mad->data[40 + offset / 8] >>
+		ret = sprintf(buf, "%u\n", (*data >>
 					    (4 - (offset % 8))) & 0xf);
 		break;
 	case 8:
-		ret = sprintf(buf, "%u\n", out_mad->data[40 + offset / 8]);
+		ret = sprintf(buf, "%u\n", *data);
 		break;
 	case 16:
 		ret = sprintf(buf, "%u\n",
-			      be16_to_cpup((__be16 *)(out_mad->data + 40 + offset / 8)));
+			      be16_to_cpup((__be16 *)data));
 		break;
 	case 32:
 		ret = sprintf(buf, "%u\n",
-			      be32_to_cpup((__be32 *)(out_mad->data + 40 + offset / 8)));
+			      be32_to_cpup((__be32 *)data));
+		break;
+	case 64:
+		ret = sprintf(buf, "%llu\n",
+				be64_to_cpup((__be64 *)data));
 		break;
+
 	default:
 		ret = 0;
 	}
 
-out:
-	kfree(in_mad);
-	kfree(out_mad);
-
 	return ret;
 }
 
@@ -403,6 +519,18 @@ static PORT_PMA_ATTR(port_rcv_data		    , 13, 32, 224);
 static PORT_PMA_ATTR(port_xmit_packets		    , 14, 32, 256);
 static PORT_PMA_ATTR(port_rcv_packets		    , 15, 32, 288);
 
+/*
+ * Counters added by extended set
+ */
+static PORT_PMA_ATTR_EXT(port_xmit_data		    , 64,  64);
+static PORT_PMA_ATTR_EXT(port_rcv_data		    , 64, 128);
+static PORT_PMA_ATTR_EXT(port_xmit_packets	    , 64, 192);
+static PORT_PMA_ATTR_EXT(port_rcv_packets	    , 64, 256);
+static PORT_PMA_ATTR_EXT(unicast_xmit_packets	    , 64, 320);
+static PORT_PMA_ATTR_EXT(unicast_rcv_packets	    , 64, 384);
+static PORT_PMA_ATTR_EXT(multicast_xmit_packets	    , 64, 448);
+static PORT_PMA_ATTR_EXT(multicast_rcv_packets	    , 64, 512);
+
 static struct attribute *pma_attrs[] = {
 	&port_pma_attr_symbol_error.attr.attr,
 	&port_pma_attr_link_error_recovery.attr.attr,
@@ -423,11 +551,65 @@ static struct attribute *pma_attrs[] = {
 	NULL
 };
 
+static struct attribute *pma_attrs_ext[] = {
+	&port_pma_attr_symbol_error.attr.attr,
+	&port_pma_attr_link_error_recovery.attr.attr,
+	&port_pma_attr_link_downed.attr.attr,
+	&port_pma_attr_port_rcv_errors.attr.attr,
+	&port_pma_attr_port_rcv_remote_physical_errors.attr.attr,
+	&port_pma_attr_port_rcv_switch_relay_errors.attr.attr,
+	&port_pma_attr_port_xmit_discards.attr.attr,
+	&port_pma_attr_port_xmit_constraint_errors.attr.attr,
+	&port_pma_attr_port_rcv_constraint_errors.attr.attr,
+	&port_pma_attr_local_link_integrity_errors.attr.attr,
+	&port_pma_attr_excessive_buffer_overrun_errors.attr.attr,
+	&port_pma_attr_VL15_dropped.attr.attr,
+	&port_pma_attr_ext_port_xmit_data.attr.attr,
+	&port_pma_attr_ext_port_rcv_data.attr.attr,
+	&port_pma_attr_ext_port_xmit_packets.attr.attr,
+	&port_pma_attr_ext_port_rcv_packets.attr.attr,
+	&port_pma_attr_ext_unicast_rcv_packets.attr.attr,
+	&port_pma_attr_ext_unicast_xmit_packets.attr.attr,
+	&port_pma_attr_ext_multicast_rcv_packets.attr.attr,
+	&port_pma_attr_ext_multicast_xmit_packets.attr.attr,
+	NULL
+};
+
+static struct attribute *pma_attrs_noietf[] = {
+	&port_pma_attr_symbol_error.attr.attr,
+	&port_pma_attr_link_error_recovery.attr.attr,
+	&port_pma_attr_link_downed.attr.attr,
+	&port_pma_attr_port_rcv_errors.attr.attr,
+	&port_pma_attr_port_rcv_remote_physical_errors.attr.attr,
+	&port_pma_attr_port_rcv_switch_relay_errors.attr.attr,
+	&port_pma_attr_port_xmit_discards.attr.attr,
+	&port_pma_attr_port_xmit_constraint_errors.attr.attr,
+	&port_pma_attr_port_rcv_constraint_errors.attr.attr,
+	&port_pma_attr_local_link_integrity_errors.attr.attr,
+	&port_pma_attr_excessive_buffer_overrun_errors.attr.attr,
+	&port_pma_attr_VL15_dropped.attr.attr,
+	&port_pma_attr_ext_port_xmit_data.attr.attr,
+	&port_pma_attr_ext_port_rcv_data.attr.attr,
+	&port_pma_attr_ext_port_xmit_packets.attr.attr,
+	&port_pma_attr_ext_port_rcv_packets.attr.attr,
+	NULL
+};
+
 static struct attribute_group pma_group = {
 	.name  = "counters",
 	.attrs  = pma_attrs
 };
 
+static struct attribute_group pma_group_ext = {
+	.name  = "counters",
+	.attrs  = pma_attrs_ext
+};
+
+static struct attribute_group pma_group_noietf = {
+	.name  = "counters",
+	.attrs  = pma_attrs_noietf
+};
+
 static void ib_port_release(struct kobject *kobj)
 {
 	struct ib_port *p = container_of(kobj, struct ib_port, kobj);
@@ -451,12 +633,41 @@ static void ib_port_release(struct kobject *kobj)
 	kfree(p);
 }
 
+static void ib_port_gid_attr_release(struct kobject *kobj)
+{
+	struct gid_attr_group *g = container_of(kobj, struct gid_attr_group,
+						kobj);
+	struct attribute *a;
+	int i;
+
+	if (g->ndev.attrs) {
+		for (i = 0; (a = g->ndev.attrs[i]); ++i)
+			kfree(a);
+
+		kfree(g->ndev.attrs);
+	}
+
+	if (g->type.attrs) {
+		for (i = 0; (a = g->type.attrs[i]); ++i)
+			kfree(a);
+
+		kfree(g->type.attrs);
+	}
+
+	kfree(g);
+}
+
 static struct kobj_type port_type = {
 	.release       = ib_port_release,
 	.sysfs_ops     = &port_sysfs_ops,
 	.default_attrs = port_default_attrs
 };
 
+static struct kobj_type gid_attr_type = {
+	.sysfs_ops      = &gid_attr_sysfs_ops,
+	.release        = ib_port_gid_attr_release
+};
+
 static struct attribute **
 alloc_group_attrs(ssize_t (*show)(struct ib_port *,
 				  struct port_attribute *, char *buf),
@@ -500,6 +711,31 @@ err:
 	return NULL;
 }
 
+/*
+ * Figure out which counter table to use depending on
+ * the device capabilities.
+ */
+static struct attribute_group *get_counter_table(struct ib_device *dev,
+						 int port_num)
+{
+	struct ib_class_port_info cpi;
+
+	if (get_perf_mad(dev, port_num, IB_PMA_CLASS_PORT_INFO,
+				&cpi, 40, sizeof(cpi)) >= 0) {
+
+		if (cpi.capability_mask && IB_PMA_CLASS_CAP_EXT_WIDTH)
+			/* We have extended counters */
+			return &pma_group_ext;
+
+		if (cpi.capability_mask && IB_PMA_CLASS_CAP_EXT_WIDTH_NOIETF)
+			/* But not the IETF ones */
+			return &pma_group_noietf;
+	}
+
+	/* Fall back to normal counters */
+	return &pma_group;
+}
+
 static int add_port(struct ib_device *device, int port_num,
 		    int (*port_callback)(struct ib_device *,
 					 u8, struct kobject *))
@@ -528,9 +764,24 @@ static int add_port(struct ib_device *device, int port_num,
 		return ret;
 	}
 
-	ret = sysfs_create_group(&p->kobj, &pma_group);
-	if (ret)
+	p->gid_attr_group = kzalloc(sizeof(*p->gid_attr_group), GFP_KERNEL);
+	if (!p->gid_attr_group) {
+		ret = -ENOMEM;
 		goto err_put;
+	}
+
+	p->gid_attr_group->port = p;
+	ret = kobject_init_and_add(&p->gid_attr_group->kobj, &gid_attr_type,
+				   &p->kobj, "gid_attrs");
+	if (ret) {
+		kfree(p->gid_attr_group);
+		goto err_put;
+	}
+
+	p->pma_table = get_counter_table(device, port_num);
+	ret = sysfs_create_group(&p->kobj, p->pma_table);
+	if (ret)
+		goto err_put_gid_attrs;
 
 	p->gid_group.name  = "gids";
 	p->gid_group.attrs = alloc_group_attrs(show_port_gid, attr.gid_tbl_len);
@@ -543,12 +794,38 @@ static int add_port(struct ib_device *device, int port_num,
 	if (ret)
 		goto err_free_gid;
 
+	p->gid_attr_group->ndev.name = "ndevs";
+	p->gid_attr_group->ndev.attrs = alloc_group_attrs(show_port_gid_attr_ndev,
+							  attr.gid_tbl_len);
+	if (!p->gid_attr_group->ndev.attrs) {
+		ret = -ENOMEM;
+		goto err_remove_gid;
+	}
+
+	ret = sysfs_create_group(&p->gid_attr_group->kobj,
+				 &p->gid_attr_group->ndev);
+	if (ret)
+		goto err_free_gid_ndev;
+
+	p->gid_attr_group->type.name = "types";
+	p->gid_attr_group->type.attrs = alloc_group_attrs(show_port_gid_attr_gid_type,
+							  attr.gid_tbl_len);
+	if (!p->gid_attr_group->type.attrs) {
+		ret = -ENOMEM;
+		goto err_remove_gid_ndev;
+	}
+
+	ret = sysfs_create_group(&p->gid_attr_group->kobj,
+				 &p->gid_attr_group->type);
+	if (ret)
+		goto err_free_gid_type;
+
 	p->pkey_group.name  = "pkeys";
 	p->pkey_group.attrs = alloc_group_attrs(show_port_pkey,
 						attr.pkey_tbl_len);
 	if (!p->pkey_group.attrs) {
 		ret = -ENOMEM;
-		goto err_remove_gid;
+		goto err_remove_gid_type;
 	}
 
 	ret = sysfs_create_group(&p->kobj, &p->pkey_group);
@@ -576,6 +853,28 @@ err_free_pkey:
 	kfree(p->pkey_group.attrs);
 	p->pkey_group.attrs = NULL;
 
+err_remove_gid_type:
+	sysfs_remove_group(&p->gid_attr_group->kobj,
+			   &p->gid_attr_group->type);
+
+err_free_gid_type:
+	for (i = 0; i < attr.gid_tbl_len; ++i)
+		kfree(p->gid_attr_group->type.attrs[i]);
+
+	kfree(p->gid_attr_group->type.attrs);
+	p->gid_attr_group->type.attrs = NULL;
+
+err_remove_gid_ndev:
+	sysfs_remove_group(&p->gid_attr_group->kobj,
+			   &p->gid_attr_group->ndev);
+
+err_free_gid_ndev:
+	for (i = 0; i < attr.gid_tbl_len; ++i)
+		kfree(p->gid_attr_group->ndev.attrs[i]);
+
+	kfree(p->gid_attr_group->ndev.attrs);
+	p->gid_attr_group->ndev.attrs = NULL;
+
 err_remove_gid:
 	sysfs_remove_group(&p->kobj, &p->gid_group);
 
@@ -587,7 +886,10 @@ err_free_gid:
 	p->gid_group.attrs = NULL;
 
 err_remove_pma:
-	sysfs_remove_group(&p->kobj, &pma_group);
+	sysfs_remove_group(&p->kobj, p->pma_table);
+
+err_put_gid_attrs:
+	kobject_put(&p->gid_attr_group->kobj);
 
 err_put:
 	kobject_put(&p->kobj);
@@ -614,18 +916,12 @@ static ssize_t show_sys_image_guid(struct device *device,
 				   struct device_attribute *dev_attr, char *buf)
 {
 	struct ib_device *dev = container_of(device, struct ib_device, dev);
-	struct ib_device_attr attr;
-	ssize_t ret;
-
-	ret = ib_query_device(dev, &attr);
-	if (ret)
-		return ret;
 
 	return sprintf(buf, "%04x:%04x:%04x:%04x\n",
-		       be16_to_cpu(((__be16 *) &attr.sys_image_guid)[0]),
-		       be16_to_cpu(((__be16 *) &attr.sys_image_guid)[1]),
-		       be16_to_cpu(((__be16 *) &attr.sys_image_guid)[2]),
-		       be16_to_cpu(((__be16 *) &attr.sys_image_guid)[3]));
+		       be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[0]),
+		       be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[1]),
+		       be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[2]),
+		       be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[3]));
 }
 
 static ssize_t show_node_guid(struct device *device,
@@ -800,9 +1096,14 @@ static void free_port_list_attributes(struct ib_device *device)
 	list_for_each_entry_safe(p, t, &device->port_list, entry) {
 		struct ib_port *port = container_of(p, struct ib_port, kobj);
 		list_del(&p->entry);
-		sysfs_remove_group(p, &pma_group);
+		sysfs_remove_group(p, port->pma_table);
 		sysfs_remove_group(p, &port->pkey_group);
 		sysfs_remove_group(p, &port->gid_group);
+		sysfs_remove_group(&port->gid_attr_group->kobj,
+				   &port->gid_attr_group->ndev);
+		sysfs_remove_group(&port->gid_attr_group->kobj,
+				   &port->gid_attr_group->type);
+		kobject_put(&port->gid_attr_group->kobj);
 		kobject_put(p);
 	}
 
diff --git a/drivers/infiniband/core/ud_header.c b/drivers/infiniband/core/ud_header.c
index 72feee620ebf..19837d270278 100644
--- a/drivers/infiniband/core/ud_header.c
+++ b/drivers/infiniband/core/ud_header.c
@@ -35,6 +35,7 @@
 #include <linux/string.h>
 #include <linux/export.h>
 #include <linux/if_ether.h>
+#include <linux/ip.h>
 
 #include <rdma/ib_pack.h>
 
@@ -116,6 +117,72 @@ static const struct ib_field vlan_table[]  = {
 	  .size_bits    = 16 }
 };
 
+static const struct ib_field ip4_table[]  = {
+	{ STRUCT_FIELD(ip4, ver),
+	  .offset_words = 0,
+	  .offset_bits  = 0,
+	  .size_bits    = 4 },
+	{ STRUCT_FIELD(ip4, hdr_len),
+	  .offset_words = 0,
+	  .offset_bits  = 4,
+	  .size_bits    = 4 },
+	{ STRUCT_FIELD(ip4, tos),
+	  .offset_words = 0,
+	  .offset_bits  = 8,
+	  .size_bits    = 8 },
+	{ STRUCT_FIELD(ip4, tot_len),
+	  .offset_words = 0,
+	  .offset_bits  = 16,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(ip4, id),
+	  .offset_words = 1,
+	  .offset_bits  = 0,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(ip4, frag_off),
+	  .offset_words = 1,
+	  .offset_bits  = 16,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(ip4, ttl),
+	  .offset_words = 2,
+	  .offset_bits  = 0,
+	  .size_bits    = 8 },
+	{ STRUCT_FIELD(ip4, protocol),
+	  .offset_words = 2,
+	  .offset_bits  = 8,
+	  .size_bits    = 8 },
+	{ STRUCT_FIELD(ip4, check),
+	  .offset_words = 2,
+	  .offset_bits  = 16,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(ip4, saddr),
+	  .offset_words = 3,
+	  .offset_bits  = 0,
+	  .size_bits    = 32 },
+	{ STRUCT_FIELD(ip4, daddr),
+	  .offset_words = 4,
+	  .offset_bits  = 0,
+	  .size_bits    = 32 }
+};
+
+static const struct ib_field udp_table[]  = {
+	{ STRUCT_FIELD(udp, sport),
+	  .offset_words = 0,
+	  .offset_bits  = 0,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(udp, dport),
+	  .offset_words = 0,
+	  .offset_bits  = 16,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(udp, length),
+	  .offset_words = 1,
+	  .offset_bits  = 0,
+	  .size_bits    = 16 },
+	{ STRUCT_FIELD(udp, csum),
+	  .offset_words = 1,
+	  .offset_bits  = 16,
+	  .size_bits    = 16 }
+};
+
 static const struct ib_field grh_table[]  = {
 	{ STRUCT_FIELD(grh, ip_version),
 	  .offset_words = 0,
@@ -213,26 +280,57 @@ static const struct ib_field deth_table[] = {
 	  .size_bits    = 24 }
 };
 
+__sum16 ib_ud_ip4_csum(struct ib_ud_header *header)
+{
+	struct iphdr iph;
+
+	iph.ihl		= 5;
+	iph.version	= 4;
+	iph.tos		= header->ip4.tos;
+	iph.tot_len	= header->ip4.tot_len;
+	iph.id		= header->ip4.id;
+	iph.frag_off	= header->ip4.frag_off;
+	iph.ttl		= header->ip4.ttl;
+	iph.protocol	= header->ip4.protocol;
+	iph.check	= 0;
+	iph.saddr	= header->ip4.saddr;
+	iph.daddr	= header->ip4.daddr;
+
+	return ip_fast_csum((u8 *)&iph, iph.ihl);
+}
+EXPORT_SYMBOL(ib_ud_ip4_csum);
+
 /**
  * ib_ud_header_init - Initialize UD header structure
  * @payload_bytes:Length of packet payload
  * @lrh_present: specify if LRH is present
  * @eth_present: specify if Eth header is present
  * @vlan_present: packet is tagged vlan
- * @grh_present:GRH flag (if non-zero, GRH will be included)
+ * @grh_present: GRH flag (if non-zero, GRH will be included)
+ * @ip_version: if non-zero, IP header, V4 or V6, will be included
+ * @udp_present :if non-zero, UDP header will be included
  * @immediate_present: specify if immediate data is present
  * @header:Structure to initialize
  */
-void ib_ud_header_init(int     		    payload_bytes,
-		       int		    lrh_present,
-		       int		    eth_present,
-		       int		    vlan_present,
-		       int    		    grh_present,
-		       int		    immediate_present,
-		       struct ib_ud_header *header)
+int ib_ud_header_init(int     payload_bytes,
+		      int    lrh_present,
+		      int    eth_present,
+		      int    vlan_present,
+		      int    grh_present,
+		      int    ip_version,
+		      int    udp_present,
+		      int    immediate_present,
+		      struct ib_ud_header *header)
 {
+	grh_present = grh_present && !ip_version;
 	memset(header, 0, sizeof *header);
 
+	/*
+	 * UDP header without IP header doesn't make sense
+	 */
+	if (udp_present && ip_version != 4 && ip_version != 6)
+		return -EINVAL;
+
 	if (lrh_present) {
 		u16 packet_length;
 
@@ -252,7 +350,7 @@ void ib_ud_header_init(int     		    payload_bytes,
 	if (vlan_present)
 		header->eth.type = cpu_to_be16(ETH_P_8021Q);
 
-	if (grh_present) {
+	if (ip_version == 6 || grh_present) {
 		header->grh.ip_version      = 6;
 		header->grh.payload_length  =
 			cpu_to_be16((IB_BTH_BYTES     +
@@ -260,8 +358,30 @@ void ib_ud_header_init(int     		    payload_bytes,
 				     payload_bytes    +
 				     4                + /* ICRC     */
 				     3) & ~3);          /* round up */
-		header->grh.next_header     = 0x1b;
+		header->grh.next_header     = udp_present ? IPPROTO_UDP : 0x1b;
+	}
+
+	if (ip_version == 4) {
+		int udp_bytes = udp_present ? IB_UDP_BYTES : 0;
+
+		header->ip4.ver = 4; /* version 4 */
+		header->ip4.hdr_len = 5; /* 5 words */
+		header->ip4.tot_len =
+			cpu_to_be16(IB_IP4_BYTES   +
+				     udp_bytes     +
+				     IB_BTH_BYTES  +
+				     IB_DETH_BYTES +
+				     payload_bytes +
+				     4);     /* ICRC     */
+		header->ip4.protocol = IPPROTO_UDP;
 	}
+	if (udp_present && ip_version)
+		header->udp.length =
+			cpu_to_be16(IB_UDP_BYTES   +
+				     IB_BTH_BYTES  +
+				     IB_DETH_BYTES +
+				     payload_bytes +
+				     4);     /* ICRC     */
 
 	if (immediate_present)
 		header->bth.opcode           = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
@@ -273,8 +393,11 @@ void ib_ud_header_init(int     		    payload_bytes,
 	header->lrh_present = lrh_present;
 	header->eth_present = eth_present;
 	header->vlan_present = vlan_present;
-	header->grh_present = grh_present;
+	header->grh_present = grh_present || (ip_version == 6);
+	header->ipv4_present = ip_version == 4;
+	header->udp_present = udp_present;
 	header->immediate_present = immediate_present;
+	return 0;
 }
 EXPORT_SYMBOL(ib_ud_header_init);
 
@@ -311,6 +434,16 @@ int ib_ud_header_pack(struct ib_ud_header *header,
 			&header->grh, buf + len);
 		len += IB_GRH_BYTES;
 	}
+	if (header->ipv4_present) {
+		ib_pack(ip4_table, ARRAY_SIZE(ip4_table),
+			&header->ip4, buf + len);
+		len += IB_IP4_BYTES;
+	}
+	if (header->udp_present) {
+		ib_pack(udp_table, ARRAY_SIZE(udp_table),
+			&header->udp, buf + len);
+		len += IB_UDP_BYTES;
+	}
 
 	ib_pack(bth_table, ARRAY_SIZE(bth_table),
 		&header->bth, buf + len);
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 40becdb3196e..e69bf266049d 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -232,7 +232,7 @@ static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
 	ib_ucontext_notifier_end_account(context);
 }
 
-static struct mmu_notifier_ops ib_umem_notifiers = {
+static const struct mmu_notifier_ops ib_umem_notifiers = {
 	.release                    = ib_umem_notifier_release,
 	.invalidate_page            = ib_umem_notifier_invalidate_page,
 	.invalidate_range_start     = ib_umem_notifier_invalidate_range_start,
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 57f281f8d686..415a3185cde7 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -210,6 +210,7 @@ static void send_handler(struct ib_mad_agent *agent,
 }
 
 static void recv_handler(struct ib_mad_agent *agent,
+			 struct ib_mad_send_buf *send_buf,
 			 struct ib_mad_recv_wc *mad_recv_wc)
 {
 	struct ib_umad_file *file = agent->context;
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 94bbd8c155fc..612ccfd39bf9 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -204,6 +204,8 @@ void ib_uverbs_event_handler(struct ib_event_handler *handler,
 			     struct ib_event *event);
 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, struct ib_xrcd *xrcd);
 
+int uverbs_dealloc_mw(struct ib_mw *mw);
+
 struct ib_uverbs_flow_spec {
 	union {
 		union {
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 1c02deab068f..6ffc9c4e93af 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -291,9 +291,6 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 	struct ib_uverbs_get_context      cmd;
 	struct ib_uverbs_get_context_resp resp;
 	struct ib_udata                   udata;
-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
-	struct ib_device_attr		  dev_attr;
-#endif
 	struct ib_ucontext		 *ucontext;
 	struct file			 *filp;
 	int ret;
@@ -342,10 +339,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 	ucontext->odp_mrs_count = 0;
 	INIT_LIST_HEAD(&ucontext->no_private_counters);
 
-	ret = ib_query_device(ib_dev, &dev_attr);
-	if (ret)
-		goto err_free;
-	if (!(dev_attr.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
+	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
 		ucontext->invalidate_range = NULL;
 
 #endif
@@ -447,8 +441,6 @@ ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
 {
 	struct ib_uverbs_query_device      cmd;
 	struct ib_uverbs_query_device_resp resp;
-	struct ib_device_attr              attr;
-	int                                ret;
 
 	if (out_len < sizeof resp)
 		return -ENOSPC;
@@ -456,12 +448,8 @@ ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	ret = ib_query_device(ib_dev, &attr);
-	if (ret)
-		return ret;
-
 	memset(&resp, 0, sizeof resp);
-	copy_query_dev_fields(file, ib_dev, &resp, &attr);
+	copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs);
 
 	if (copy_to_user((void __user *) (unsigned long) cmd.response,
 			 &resp, sizeof resp))
@@ -986,11 +974,8 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
 	}
 
 	if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
-		struct ib_device_attr attr;
-
-		ret = ib_query_device(pd->device, &attr);
-		if (ret || !(attr.device_cap_flags &
-				IB_DEVICE_ON_DEMAND_PAGING)) {
+		if (!(pd->device->attrs.device_cap_flags &
+		      IB_DEVICE_ON_DEMAND_PAGING)) {
 			pr_debug("ODP support not available\n");
 			ret = -EINVAL;
 			goto err_put;
@@ -1008,7 +993,6 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
 	mr->pd      = pd;
 	mr->uobject = uobj;
 	atomic_inc(&pd->usecnt);
-	atomic_set(&mr->usecnt, 0);
 
 	uobj->object = mr;
 	ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj);
@@ -1106,11 +1090,6 @@ ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
 		}
 	}
 
-	if (atomic_read(&mr->usecnt)) {
-		ret = -EBUSY;
-		goto put_uobj_pd;
-	}
-
 	old_pd = mr->pd;
 	ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
 					cmd.length, cmd.hca_va,
@@ -1258,7 +1237,7 @@ err_copy:
 	idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
 
 err_unalloc:
-	ib_dealloc_mw(mw);
+	uverbs_dealloc_mw(mw);
 
 err_put:
 	put_pd_read(pd);
@@ -1287,7 +1266,7 @@ ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
 
 	mw = uobj->object;
 
-	ret = ib_dealloc_mw(mw);
+	ret = uverbs_dealloc_mw(mw);
 	if (!ret)
 		uobj->live = 0;
 
@@ -1845,7 +1824,10 @@ static int create_qp(struct ib_uverbs_file *file,
 		      sizeof(cmd->create_flags))
 		attr.create_flags = cmd->create_flags;
 
-	if (attr.create_flags & ~IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
+	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
+				IB_QP_CREATE_CROSS_CHANNEL |
+				IB_QP_CREATE_MANAGED_SEND |
+				IB_QP_CREATE_MANAGED_RECV)) {
 		ret = -EINVAL;
 		goto err_put;
 	}
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index e3ef28861be6..39680aed99dd 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -133,6 +133,17 @@ static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file,
 static void ib_uverbs_add_one(struct ib_device *device);
 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
 
+int uverbs_dealloc_mw(struct ib_mw *mw)
+{
+	struct ib_pd *pd = mw->pd;
+	int ret;
+
+	ret = mw->device->dealloc_mw(mw);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+	return ret;
+}
+
 static void ib_uverbs_release_dev(struct kobject *kobj)
 {
 	struct ib_uverbs_device *dev =
@@ -224,7 +235,7 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
 		struct ib_mw *mw = uobj->object;
 
 		idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
-		ib_dealloc_mw(mw);
+		uverbs_dealloc_mw(mw);
 		kfree(uobj);
 	}
 
diff --git a/drivers/infiniband/core/uverbs_marshall.c b/drivers/infiniband/core/uverbs_marshall.c
index 7d2f14c9bbef..af020f80d50f 100644
--- a/drivers/infiniband/core/uverbs_marshall.c
+++ b/drivers/infiniband/core/uverbs_marshall.c
@@ -144,5 +144,6 @@ void ib_copy_path_rec_from_user(struct ib_sa_path_rec *dst,
 	memset(dst->dmac, 0, sizeof(dst->dmac));
 	dst->net = NULL;
 	dst->ifindex = 0;
+	dst->gid_type = IB_GID_TYPE_IB;
 }
 EXPORT_SYMBOL(ib_copy_path_rec_from_user);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 545906dec26d..5af6d024e053 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -229,12 +229,6 @@ EXPORT_SYMBOL(rdma_port_get_link_layer);
 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 {
 	struct ib_pd *pd;
-	struct ib_device_attr devattr;
-	int rc;
-
-	rc = ib_query_device(device, &devattr);
-	if (rc)
-		return ERR_PTR(rc);
 
 	pd = device->alloc_pd(device, NULL, NULL);
 	if (IS_ERR(pd))
@@ -245,7 +239,7 @@ struct ib_pd *ib_alloc_pd(struct ib_device *device)
 	pd->local_mr = NULL;
 	atomic_set(&pd->usecnt, 0);
 
-	if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
+	if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
 		pd->local_dma_lkey = device->local_dma_lkey;
 	else {
 		struct ib_mr *mr;
@@ -311,8 +305,61 @@ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 }
 EXPORT_SYMBOL(ib_create_ah);
 
+static int ib_get_header_version(const union rdma_network_hdr *hdr)
+{
+	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
+	struct iphdr ip4h_checked;
+	const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
+
+	/* If it's IPv6, the version must be 6, otherwise, the first
+	 * 20 bytes (before the IPv4 header) are garbled.
+	 */
+	if (ip6h->version != 6)
+		return (ip4h->version == 4) ? 4 : 0;
+	/* version may be 6 or 4 because the first 20 bytes could be garbled */
+
+	/* RoCE v2 requires no options, thus header length
+	 * must be 5 words
+	 */
+	if (ip4h->ihl != 5)
+		return 6;
+
+	/* Verify checksum.
+	 * We can't write on scattered buffers so we need to copy to
+	 * temp buffer.
+	 */
+	memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
+	ip4h_checked.check = 0;
+	ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5);
+	/* if IPv4 header checksum is OK, believe it */
+	if (ip4h->check == ip4h_checked.check)
+		return 4;
+	return 6;
+}
+
+static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
+						     u8 port_num,
+						     const struct ib_grh *grh)
+{
+	int grh_version;
+
+	if (rdma_protocol_ib(device, port_num))
+		return RDMA_NETWORK_IB;
+
+	grh_version = ib_get_header_version((union rdma_network_hdr *)grh);
+
+	if (grh_version == 4)
+		return RDMA_NETWORK_IPV4;
+
+	if (grh->next_hdr == IPPROTO_UDP)
+		return RDMA_NETWORK_IPV6;
+
+	return RDMA_NETWORK_ROCE_V1;
+}
+
 struct find_gid_index_context {
 	u16 vlan_id;
+	enum ib_gid_type gid_type;
 };
 
 static bool find_gid_index(const union ib_gid *gid,
@@ -322,6 +369,9 @@ static bool find_gid_index(const union ib_gid *gid,
 	struct find_gid_index_context *ctx =
 		(struct find_gid_index_context *)context;
 
+	if (ctx->gid_type != gid_attr->gid_type)
+		return false;
+
 	if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) ||
 	    (is_vlan_dev(gid_attr->ndev) &&
 	     vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id))
@@ -332,14 +382,49 @@ static bool find_gid_index(const union ib_gid *gid,
 
 static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num,
 				   u16 vlan_id, const union ib_gid *sgid,
+				   enum ib_gid_type gid_type,
 				   u16 *gid_index)
 {
-	struct find_gid_index_context context = {.vlan_id = vlan_id};
+	struct find_gid_index_context context = {.vlan_id = vlan_id,
+						 .gid_type = gid_type};
 
 	return ib_find_gid_by_filter(device, sgid, port_num, find_gid_index,
 				     &context, gid_index);
 }
 
+static int get_gids_from_rdma_hdr(union rdma_network_hdr *hdr,
+				  enum rdma_network_type net_type,
+				  union ib_gid *sgid, union ib_gid *dgid)
+{
+	struct sockaddr_in  src_in;
+	struct sockaddr_in  dst_in;
+	__be32 src_saddr, dst_saddr;
+
+	if (!sgid || !dgid)
+		return -EINVAL;
+
+	if (net_type == RDMA_NETWORK_IPV4) {
+		memcpy(&src_in.sin_addr.s_addr,
+		       &hdr->roce4grh.saddr, 4);
+		memcpy(&dst_in.sin_addr.s_addr,
+		       &hdr->roce4grh.daddr, 4);
+		src_saddr = src_in.sin_addr.s_addr;
+		dst_saddr = dst_in.sin_addr.s_addr;
+		ipv6_addr_set_v4mapped(src_saddr,
+				       (struct in6_addr *)sgid);
+		ipv6_addr_set_v4mapped(dst_saddr,
+				       (struct in6_addr *)dgid);
+		return 0;
+	} else if (net_type == RDMA_NETWORK_IPV6 ||
+		   net_type == RDMA_NETWORK_IB) {
+		*dgid = hdr->ibgrh.dgid;
+		*sgid = hdr->ibgrh.sgid;
+		return 0;
+	} else {
+		return -EINVAL;
+	}
+}
+
 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 		       const struct ib_wc *wc, const struct ib_grh *grh,
 		       struct ib_ah_attr *ah_attr)
@@ -347,33 +432,72 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 	u32 flow_class;
 	u16 gid_index;
 	int ret;
+	enum rdma_network_type net_type = RDMA_NETWORK_IB;
+	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
+	int hoplimit = 0xff;
+	union ib_gid dgid;
+	union ib_gid sgid;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
 	if (rdma_cap_eth_ah(device, port_num)) {
+		if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
+			net_type = wc->network_hdr_type;
+		else
+			net_type = ib_get_net_type_by_grh(device, port_num, grh);
+		gid_type = ib_network_to_gid_type(net_type);
+	}
+	ret = get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
+				     &sgid, &dgid);
+	if (ret)
+		return ret;
+
+	if (rdma_protocol_roce(device, port_num)) {
+		int if_index = 0;
 		u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
 				wc->vlan_id : 0xffff;
+		struct net_device *idev;
+		struct net_device *resolved_dev;
 
 		if (!(wc->wc_flags & IB_WC_GRH))
 			return -EPROTOTYPE;
 
-		if (!(wc->wc_flags & IB_WC_WITH_SMAC) ||
-		    !(wc->wc_flags & IB_WC_WITH_VLAN)) {
-			ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
-							 ah_attr->dmac,
-							 wc->wc_flags & IB_WC_WITH_VLAN ?
-							 NULL : &vlan_id,
-							 0);
-			if (ret)
-				return ret;
+		if (!device->get_netdev)
+			return -EOPNOTSUPP;
+
+		idev = device->get_netdev(device, port_num);
+		if (!idev)
+			return -ENODEV;
+
+		ret = rdma_addr_find_l2_eth_by_grh(&dgid, &sgid,
+						   ah_attr->dmac,
+						   wc->wc_flags & IB_WC_WITH_VLAN ?
+						   NULL : &vlan_id,
+						   &if_index, &hoplimit);
+		if (ret) {
+			dev_put(idev);
+			return ret;
 		}
 
-		ret = get_sgid_index_from_eth(device, port_num, vlan_id,
-					      &grh->dgid, &gid_index);
+		resolved_dev = dev_get_by_index(&init_net, if_index);
+		if (resolved_dev->flags & IFF_LOOPBACK) {
+			dev_put(resolved_dev);
+			resolved_dev = idev;
+			dev_hold(resolved_dev);
+		}
+		rcu_read_lock();
+		if (resolved_dev != idev && !rdma_is_upper_dev_rcu(idev,
+								   resolved_dev))
+			ret = -EHOSTUNREACH;
+		rcu_read_unlock();
+		dev_put(idev);
+		dev_put(resolved_dev);
 		if (ret)
 			return ret;
 
-		if (wc->wc_flags & IB_WC_WITH_SMAC)
-			memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
+		ret = get_sgid_index_from_eth(device, port_num, vlan_id,
+					      &dgid, gid_type, &gid_index);
+		if (ret)
+			return ret;
 	}
 
 	ah_attr->dlid = wc->slid;
@@ -383,10 +507,11 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 
 	if (wc->wc_flags & IB_WC_GRH) {
 		ah_attr->ah_flags = IB_AH_GRH;
-		ah_attr->grh.dgid = grh->sgid;
+		ah_attr->grh.dgid = sgid;
 
 		if (!rdma_cap_eth_ah(device, port_num)) {
-			ret = ib_find_cached_gid_by_port(device, &grh->dgid,
+			ret = ib_find_cached_gid_by_port(device, &dgid,
+							 IB_GID_TYPE_IB,
 							 port_num, NULL,
 							 &gid_index);
 			if (ret)
@@ -396,7 +521,7 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 		ah_attr->grh.sgid_index = (u8) gid_index;
 		flow_class = be32_to_cpu(grh->version_tclass_flow);
 		ah_attr->grh.flow_label = flow_class & 0xFFFFF;
-		ah_attr->grh.hop_limit = 0xFF;
+		ah_attr->grh.hop_limit = hoplimit;
 		ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 	}
 	return 0;
@@ -1014,6 +1139,7 @@ int ib_resolve_eth_dmac(struct ib_qp *qp,
 			union ib_gid		sgid;
 			struct ib_gid_attr	sgid_attr;
 			int			ifindex;
+			int			hop_limit;
 
 			ret = ib_query_gid(qp->device,
 					   qp_attr->ah_attr.port_num,
@@ -1028,12 +1154,14 @@ int ib_resolve_eth_dmac(struct ib_qp *qp,
 
 			ifindex = sgid_attr.ndev->ifindex;
 
-			ret = rdma_addr_find_dmac_by_grh(&sgid,
-							 &qp_attr->ah_attr.grh.dgid,
-							 qp_attr->ah_attr.dmac,
-							 NULL, ifindex);
+			ret = rdma_addr_find_l2_eth_by_grh(&sgid,
+							   &qp_attr->ah_attr.grh.dgid,
+							   qp_attr->ah_attr.dmac,
+							   NULL, &ifindex, &hop_limit);
 
 			dev_put(sgid_attr.ndev);
+
+			qp_attr->ah_attr.grh.hop_limit = hop_limit;
 		}
 	}
 out:
@@ -1215,29 +1343,17 @@ struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
 		mr->pd      = pd;
 		mr->uobject = NULL;
 		atomic_inc(&pd->usecnt);
-		atomic_set(&mr->usecnt, 0);
 	}
 
 	return mr;
 }
 EXPORT_SYMBOL(ib_get_dma_mr);
 
-int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
-{
-	return mr->device->query_mr ?
-		mr->device->query_mr(mr, mr_attr) : -ENOSYS;
-}
-EXPORT_SYMBOL(ib_query_mr);
-
 int ib_dereg_mr(struct ib_mr *mr)
 {
-	struct ib_pd *pd;
+	struct ib_pd *pd = mr->pd;
 	int ret;
 
-	if (atomic_read(&mr->usecnt))
-		return -EBUSY;
-
-	pd = mr->pd;
 	ret = mr->device->dereg_mr(mr);
 	if (!ret)
 		atomic_dec(&pd->usecnt);
@@ -1273,49 +1389,12 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
 		mr->pd      = pd;
 		mr->uobject = NULL;
 		atomic_inc(&pd->usecnt);
-		atomic_set(&mr->usecnt, 0);
 	}
 
 	return mr;
 }
 EXPORT_SYMBOL(ib_alloc_mr);
 
-/* Memory windows */
-
-struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
-{
-	struct ib_mw *mw;
-
-	if (!pd->device->alloc_mw)
-		return ERR_PTR(-ENOSYS);
-
-	mw = pd->device->alloc_mw(pd, type);
-	if (!IS_ERR(mw)) {
-		mw->device  = pd->device;
-		mw->pd      = pd;
-		mw->uobject = NULL;
-		mw->type    = type;
-		atomic_inc(&pd->usecnt);
-	}
-
-	return mw;
-}
-EXPORT_SYMBOL(ib_alloc_mw);
-
-int ib_dealloc_mw(struct ib_mw *mw)
-{
-	struct ib_pd *pd;
-	int ret;
-
-	pd = mw->pd;
-	ret = mw->device->dealloc_mw(mw);
-	if (!ret)
-		atomic_dec(&pd->usecnt);
-
-	return ret;
-}
-EXPORT_SYMBOL(ib_dealloc_mw);
-
 /* "Fast" memory regions */
 
 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
@@ -1530,7 +1609,7 @@ int ib_sg_to_pages(struct ib_mr *mr,
 		   int (*set_page)(struct ib_mr *, u64))
 {
 	struct scatterlist *sg;
-	u64 last_end_dma_addr = 0, last_page_addr = 0;
+	u64 last_end_dma_addr = 0;
 	unsigned int last_page_off = 0;
 	u64 page_mask = ~((u64)mr->page_size - 1);
 	int i, ret;
@@ -1572,7 +1651,6 @@ next_page:
 
 		mr->length += dma_len;
 		last_end_dma_addr = end_dma_addr;
-		last_page_addr = end_dma_addr & page_mask;
 		last_page_off = end_dma_addr & ~page_mask;
 	}
 
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index cb78b1e9bcd9..f504ba73e5dc 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -149,7 +149,7 @@ static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_en
 	error = l2t_send(tdev, skb, l2e);
 	if (error < 0)
 		kfree_skb(skb);
-	return error;
+	return error < 0 ? error : 0;
 }
 
 int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb)
@@ -165,7 +165,7 @@ int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb)
 	error = cxgb3_ofld_send(tdev, skb);
 	if (error < 0)
 		kfree_skb(skb);
-	return error;
+	return error < 0 ? error : 0;
 }
 
 static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cq.c b/drivers/infiniband/hw/cxgb3/iwch_cq.c
index cfe404925a39..97fbfd2c298e 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cq.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cq.c
@@ -115,10 +115,6 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
 		case T3_SEND_WITH_SE_INV:
 			wc->opcode = IB_WC_SEND;
 			break;
-		case T3_BIND_MW:
-			wc->opcode = IB_WC_BIND_MW;
-			break;
-
 		case T3_LOCAL_INV:
 			wc->opcode = IB_WC_LOCAL_INV;
 			break;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_mem.c b/drivers/infiniband/hw/cxgb3/iwch_mem.c
index 5c36ee2809ac..1d04c872c9d5 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_mem.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_mem.c
@@ -75,37 +75,6 @@ int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
 	return ret;
 }
 
-int iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php,
-					struct iwch_mr *mhp,
-					int shift,
-					int npages)
-{
-	u32 stag;
-	int ret;
-
-	/* We could support this... */
-	if (npages > mhp->attr.pbl_size)
-		return -ENOMEM;
-
-	stag = mhp->attr.stag;
-	if (cxio_reregister_phys_mem(&rhp->rdev,
-				   &stag, mhp->attr.pdid,
-				   mhp->attr.perms,
-				   mhp->attr.zbva,
-				   mhp->attr.va_fbo,
-				   mhp->attr.len,
-				   shift - 12,
-				   mhp->attr.pbl_size, mhp->attr.pbl_addr))
-		return -ENOMEM;
-
-	ret = iwch_finish_mem_reg(mhp, stag);
-	if (ret)
-		cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
-		       mhp->attr.pbl_addr);
-
-	return ret;
-}
-
 int iwch_alloc_pbl(struct iwch_mr *mhp, int npages)
 {
 	mhp->attr.pbl_addr = cxio_hal_pblpool_alloc(&mhp->rhp->rdev,
@@ -130,74 +99,3 @@ int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset)
 	return cxio_write_pbl(&mhp->rhp->rdev, pages,
 			      mhp->attr.pbl_addr + (offset << 3), npages);
 }
-
-int build_phys_page_list(struct ib_phys_buf *buffer_list,
-					int num_phys_buf,
-					u64 *iova_start,
-					u64 *total_size,
-					int *npages,
-					int *shift,
-					__be64 **page_list)
-{
-	u64 mask;
-	int i, j, n;
-
-	mask = 0;
-	*total_size = 0;
-	for (i = 0; i < num_phys_buf; ++i) {
-		if (i != 0 && buffer_list[i].addr & ~PAGE_MASK)
-			return -EINVAL;
-		if (i != 0 && i != num_phys_buf - 1 &&
-		    (buffer_list[i].size & ~PAGE_MASK))
-			return -EINVAL;
-		*total_size += buffer_list[i].size;
-		if (i > 0)
-			mask |= buffer_list[i].addr;
-		else
-			mask |= buffer_list[i].addr & PAGE_MASK;
-		if (i != num_phys_buf - 1)
-			mask |= buffer_list[i].addr + buffer_list[i].size;
-		else
-			mask |= (buffer_list[i].addr + buffer_list[i].size +
-				PAGE_SIZE - 1) & PAGE_MASK;
-	}
-
-	if (*total_size > 0xFFFFFFFFULL)
-		return -ENOMEM;
-
-	/* Find largest page shift we can use to cover buffers */
-	for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift))
-		if ((1ULL << *shift) & mask)
-			break;
-
-	buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1);
-	buffer_list[0].addr &= ~0ull << *shift;
-
-	*npages = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		*npages += (buffer_list[i].size +
-			(1ULL << *shift) - 1) >> *shift;
-
-	if (!*npages)
-		return -EINVAL;
-
-	*page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL);
-	if (!*page_list)
-		return -ENOMEM;
-
-	n = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		for (j = 0;
-		     j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift;
-		     ++j)
-			(*page_list)[n++] = cpu_to_be64(buffer_list[i].addr +
-			    ((u64) j << *shift));
-
-	PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n",
-	     __func__, (unsigned long long) *iova_start,
-	     (unsigned long long) mask, *shift, (unsigned long long) *total_size,
-	     *npages);
-
-	return 0;
-
-}
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index c34725ca0bb4..2734820d291b 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -458,9 +458,6 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr)
 	u32 mmid;
 
 	PDBG("%s ib_mr %p\n", __func__, ib_mr);
-	/* There can be no memory windows */
-	if (atomic_read(&ib_mr->usecnt))
-		return -EINVAL;
 
 	mhp = to_iwch_mr(ib_mr);
 	kfree(mhp->pages);
@@ -479,24 +476,25 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr)
 	return 0;
 }
 
-static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
-					struct ib_phys_buf *buffer_list,
-					int num_phys_buf,
-					int acc,
-					u64 *iova_start)
+static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
 {
-	__be64 *page_list;
-	int shift;
-	u64 total_size;
-	int npages;
-	struct iwch_dev *rhp;
-	struct iwch_pd *php;
+	const u64 total_size = 0xffffffff;
+	const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
+	struct iwch_pd *php = to_iwch_pd(pd);
+	struct iwch_dev *rhp = php->rhp;
 	struct iwch_mr *mhp;
-	int ret;
+	__be64 *page_list;
+	int shift = 26, npages, ret, i;
 
 	PDBG("%s ib_pd %p\n", __func__, pd);
-	php = to_iwch_pd(pd);
-	rhp = php->rhp;
+
+	/*
+	 * T3 only supports 32 bits of size.
+	 */
+	if (sizeof(phys_addr_t) > 4) {
+		pr_warn_once(MOD "Cannot support dma_mrs on this platform.\n");
+		return ERR_PTR(-ENOTSUPP);
+	}
 
 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 	if (!mhp)
@@ -504,22 +502,23 @@ static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
 
 	mhp->rhp = rhp;
 
-	/* First check that we have enough alignment */
-	if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
+	npages = (total_size + (1ULL << shift) - 1) >> shift;
+	if (!npages) {
 		ret = -EINVAL;
 		goto err;
 	}
 
-	if (num_phys_buf > 1 &&
-	    ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
-		ret = -EINVAL;
+	page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
+	if (!page_list) {
+		ret = -ENOMEM;
 		goto err;
 	}
 
-	ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
-				   &total_size, &npages, &shift, &page_list);
-	if (ret)
-		goto err;
+	for (i = 0; i < npages; i++)
+		page_list[i] = cpu_to_be64((u64)i << shift);
+
+	PDBG("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
+		__func__, mask, shift, total_size, npages);
 
 	ret = iwch_alloc_pbl(mhp, npages);
 	if (ret) {
@@ -536,7 +535,7 @@ static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
 	mhp->attr.zbva = 0;
 
 	mhp->attr.perms = iwch_ib_to_tpt_access(acc);
-	mhp->attr.va_fbo = *iova_start;
+	mhp->attr.va_fbo = 0;
 	mhp->attr.page_size = shift - 12;
 
 	mhp->attr.len = (u32) total_size;
@@ -553,76 +552,8 @@ err_pbl:
 err:
 	kfree(mhp);
 	return ERR_PTR(ret);
-
-}
-
-static int iwch_reregister_phys_mem(struct ib_mr *mr,
-				     int mr_rereg_mask,
-				     struct ib_pd *pd,
-	                             struct ib_phys_buf *buffer_list,
-	                             int num_phys_buf,
-	                             int acc, u64 * iova_start)
-{
-
-	struct iwch_mr mh, *mhp;
-	struct iwch_pd *php;
-	struct iwch_dev *rhp;
-	__be64 *page_list = NULL;
-	int shift = 0;
-	u64 total_size;
-	int npages = 0;
-	int ret;
-
-	PDBG("%s ib_mr %p ib_pd %p\n", __func__, mr, pd);
-
-	/* There can be no memory windows */
-	if (atomic_read(&mr->usecnt))
-		return -EINVAL;
-
-	mhp = to_iwch_mr(mr);
-	rhp = mhp->rhp;
-	php = to_iwch_pd(mr->pd);
-
-	/* make sure we are on the same adapter */
-	if (rhp != php->rhp)
-		return -EINVAL;
-
-	memcpy(&mh, mhp, sizeof *mhp);
-
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		php = to_iwch_pd(pd);
-	if (mr_rereg_mask & IB_MR_REREG_ACCESS)
-		mh.attr.perms = iwch_ib_to_tpt_access(acc);
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
-		ret = build_phys_page_list(buffer_list, num_phys_buf,
-					   iova_start,
-					   &total_size, &npages,
-					   &shift, &page_list);
-		if (ret)
-			return ret;
-	}
-
-	ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
-	kfree(page_list);
-	if (ret) {
-		return ret;
-	}
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		mhp->attr.pdid = php->pdid;
-	if (mr_rereg_mask & IB_MR_REREG_ACCESS)
-		mhp->attr.perms = iwch_ib_to_tpt_access(acc);
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
-		mhp->attr.zbva = 0;
-		mhp->attr.va_fbo = *iova_start;
-		mhp->attr.page_size = shift - 12;
-		mhp->attr.len = (u32) total_size;
-		mhp->attr.pbl_size = npages;
-	}
-
-	return 0;
 }
 
-
 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				      u64 virt, int acc, struct ib_udata *udata)
 {
@@ -726,28 +657,6 @@ err:
 	return ERR_PTR(err);
 }
 
-static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
-{
-	struct ib_phys_buf bl;
-	u64 kva;
-	struct ib_mr *ibmr;
-
-	PDBG("%s ib_pd %p\n", __func__, pd);
-
-	/*
-	 * T3 only supports 32 bits of size.
-	 */
-	if (sizeof(phys_addr_t) > 4) {
-		pr_warn_once(MOD "Cannot support dma_mrs on this platform.\n");
-		return ERR_PTR(-ENOTSUPP);
-	}
-	bl.size = 0xffffffff;
-	bl.addr = 0;
-	kva = 0;
-	ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
-	return ibmr;
-}
-
 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
 {
 	struct iwch_dev *rhp;
@@ -1452,12 +1361,9 @@ int iwch_register_device(struct iwch_dev *dev)
 	dev->ibdev.resize_cq = iwch_resize_cq;
 	dev->ibdev.poll_cq = iwch_poll_cq;
 	dev->ibdev.get_dma_mr = iwch_get_dma_mr;
-	dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
-	dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
 	dev->ibdev.reg_user_mr = iwch_reg_user_mr;
 	dev->ibdev.dereg_mr = iwch_dereg_mr;
 	dev->ibdev.alloc_mw = iwch_alloc_mw;
-	dev->ibdev.bind_mw = iwch_bind_mw;
 	dev->ibdev.dealloc_mw = iwch_dealloc_mw;
 	dev->ibdev.alloc_mr = iwch_alloc_mr;
 	dev->ibdev.map_mr_sg = iwch_map_mr_sg;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.h b/drivers/infiniband/hw/cxgb3/iwch_provider.h
index 2ac85b86a680..252c464a09f6 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.h
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.h
@@ -330,9 +330,6 @@ int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		      struct ib_send_wr **bad_wr);
 int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		      struct ib_recv_wr **bad_wr);
-int iwch_bind_mw(struct ib_qp *qp,
-			     struct ib_mw *mw,
-			     struct ib_mw_bind *mw_bind);
 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg);
 int iwch_post_zb_read(struct iwch_ep *ep);
@@ -341,21 +338,9 @@ void iwch_unregister_device(struct iwch_dev *dev);
 void stop_read_rep_timer(struct iwch_qp *qhp);
 int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
 		      struct iwch_mr *mhp, int shift);
-int iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php,
-					struct iwch_mr *mhp,
-					int shift,
-					int npages);
 int iwch_alloc_pbl(struct iwch_mr *mhp, int npages);
 void iwch_free_pbl(struct iwch_mr *mhp);
 int iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset);
-int build_phys_page_list(struct ib_phys_buf *buffer_list,
-					int num_phys_buf,
-					u64 *iova_start,
-					u64 *total_size,
-					int *npages,
-					int *shift,
-					__be64 **page_list);
-
 
 #define IWCH_NODE_DESC "cxgb3 Chelsio Communications"
 
diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c
index d0548fc6395e..d939980a708f 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_qp.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c
@@ -526,88 +526,6 @@ out:
 	return err;
 }
 
-int iwch_bind_mw(struct ib_qp *qp,
-			     struct ib_mw *mw,
-			     struct ib_mw_bind *mw_bind)
-{
-	struct iwch_dev *rhp;
-	struct iwch_mw *mhp;
-	struct iwch_qp *qhp;
-	union t3_wr *wqe;
-	u32 pbl_addr;
-	u8 page_size;
-	u32 num_wrs;
-	unsigned long flag;
-	struct ib_sge sgl;
-	int err=0;
-	enum t3_wr_flags t3_wr_flags;
-	u32 idx;
-	struct t3_swsq *sqp;
-
-	qhp = to_iwch_qp(qp);
-	mhp = to_iwch_mw(mw);
-	rhp = qhp->rhp;
-
-	spin_lock_irqsave(&qhp->lock, flag);
-	if (qhp->attr.state > IWCH_QP_STATE_RTS) {
-		spin_unlock_irqrestore(&qhp->lock, flag);
-		return -EINVAL;
-	}
-	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,
-			    qhp->wq.sq_size_log2);
-	if (num_wrs == 0) {
-		spin_unlock_irqrestore(&qhp->lock, flag);
-		return -ENOMEM;
-	}
-	idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
-	PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __func__, idx,
-	     mw, mw_bind);
-	wqe = (union t3_wr *) (qhp->wq.queue + idx);
-
-	t3_wr_flags = 0;
-	if (mw_bind->send_flags & IB_SEND_SIGNALED)
-		t3_wr_flags = T3_COMPLETION_FLAG;
-
-	sgl.addr = mw_bind->bind_info.addr;
-	sgl.lkey = mw_bind->bind_info.mr->lkey;
-	sgl.length = mw_bind->bind_info.length;
-	wqe->bind.reserved = 0;
-	wqe->bind.type = TPT_VATO;
-
-	/* TBD: check perms */
-	wqe->bind.perms = iwch_ib_to_tpt_bind_access(
-		mw_bind->bind_info.mw_access_flags);
-	wqe->bind.mr_stag = cpu_to_be32(mw_bind->bind_info.mr->lkey);
-	wqe->bind.mw_stag = cpu_to_be32(mw->rkey);
-	wqe->bind.mw_len = cpu_to_be32(mw_bind->bind_info.length);
-	wqe->bind.mw_va = cpu_to_be64(mw_bind->bind_info.addr);
-	err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size);
-	if (err) {
-		spin_unlock_irqrestore(&qhp->lock, flag);
-		return err;
-	}
-	wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
-	sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
-	sqp->wr_id = mw_bind->wr_id;
-	sqp->opcode = T3_BIND_MW;
-	sqp->sq_wptr = qhp->wq.sq_wptr;
-	sqp->complete = 0;
-	sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED);
-	wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr);
-	wqe->bind.mr_pagesz = page_size;
-	build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags,
-		       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0,
-		       sizeof(struct t3_bind_mw_wr) >> 3, T3_SOPEOP);
-	++(qhp->wq.wptr);
-	++(qhp->wq.sq_wptr);
-	spin_unlock_irqrestore(&qhp->lock, flag);
-
-	if (cxio_wq_db_enabled(&qhp->wq))
-		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);
-
-	return err;
-}
-
 static inline void build_term_codes(struct respQ_msg_t *rsp_msg,
 				    u8 *layer_type, u8 *ecode)
 {
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index c9cffced00ca..cd2ff5f9518a 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -449,7 +449,7 @@ static void act_open_req_arp_failure(void *handle, struct sk_buff *skb)
 {
 	struct c4iw_ep *ep = handle;
 
-	printk(KERN_ERR MOD "ARP failure duing connect\n");
+	printk(KERN_ERR MOD "ARP failure during connect\n");
 	kfree_skb(skb);
 	connect_reply_upcall(ep, -EHOSTUNREACH);
 	state_set(&ep->com, DEAD);
@@ -3271,6 +3271,12 @@ static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep)
 	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
 				    &ep->com.mapped_local_addr;
 
+	if (ipv6_addr_type(&sin6->sin6_addr) != IPV6_ADDR_ANY) {
+		err = cxgb4_clip_get(ep->com.dev->rdev.lldi.ports[0],
+				     (const u32 *)&sin6->sin6_addr.s6_addr, 1);
+		if (err)
+			return err;
+	}
 	c4iw_init_wr_wait(&ep->com.wr_wait);
 	err = cxgb4_create_server6(ep->com.dev->rdev.lldi.ports[0],
 				   ep->stid, &sin6->sin6_addr,
@@ -3282,13 +3288,13 @@ static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep)
 					  0, 0, __func__);
 	else if (err > 0)
 		err = net_xmit_errno(err);
-	if (err)
+	if (err) {
+		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
+				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 		pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n",
 		       err, ep->stid,
 		       sin6->sin6_addr.s6_addr, ntohs(sin6->sin6_port));
-	else
-		cxgb4_clip_get(ep->com.dev->rdev.lldi.ports[0],
-			       (const u32 *)&sin6->sin6_addr.s6_addr, 1);
+	}
 	return err;
 }
 
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index de9cd6901752..cf21df4a8bf5 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -744,9 +744,6 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
 		case FW_RI_SEND_WITH_SE:
 			wc->opcode = IB_WC_SEND;
 			break;
-		case FW_RI_BIND_MW:
-			wc->opcode = IB_WC_BIND_MW;
-			break;
 
 		case FW_RI_LOCAL_INV:
 			wc->opcode = IB_WC_LOCAL_INV;
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 58fce1742b8d..8024ea4417b8 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -315,14 +315,12 @@ static int qp_release(struct inode *inode, struct file *file)
 static int qp_open(struct inode *inode, struct file *file)
 {
 	struct c4iw_debugfs_data *qpd;
-	int ret = 0;
 	int count = 1;
 
 	qpd = kmalloc(sizeof *qpd, GFP_KERNEL);
-	if (!qpd) {
-		ret = -ENOMEM;
-		goto out;
-	}
+	if (!qpd)
+		return -ENOMEM;
+
 	qpd->devp = inode->i_private;
 	qpd->pos = 0;
 
@@ -333,8 +331,8 @@ static int qp_open(struct inode *inode, struct file *file)
 	qpd->bufsize = count * 128;
 	qpd->buf = vmalloc(qpd->bufsize);
 	if (!qpd->buf) {
-		ret = -ENOMEM;
-		goto err1;
+		kfree(qpd);
+		return -ENOMEM;
 	}
 
 	spin_lock_irq(&qpd->devp->lock);
@@ -343,11 +341,7 @@ static int qp_open(struct inode *inode, struct file *file)
 
 	qpd->buf[qpd->pos++] = 0;
 	file->private_data = qpd;
-	goto out;
-err1:
-	kfree(qpd);
-out:
-	return ret;
+	return 0;
 }
 
 static const struct file_operations qp_debugfs_fops = {
@@ -781,8 +775,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 		pr_err(MOD "%s: unsupported udb/ucq densities %u/%u\n",
 		       pci_name(rdev->lldi.pdev), rdev->lldi.udb_density,
 		       rdev->lldi.ucq_density);
-		err = -EINVAL;
-		goto err1;
+		return -EINVAL;
 	}
 	if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start ||
 	    rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) {
@@ -791,8 +784,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 		       pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start,
 		       rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size,
 		       rdev->lldi.vr->cq.size);
-		err = -EINVAL;
-		goto err1;
+		return -EINVAL;
 	}
 
 	rdev->qpmask = rdev->lldi.udb_density - 1;
@@ -816,10 +808,8 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 	     rdev->lldi.db_reg, rdev->lldi.gts_reg,
 	     rdev->qpmask, rdev->cqmask);
 
-	if (c4iw_num_stags(rdev) == 0) {
-		err = -EINVAL;
-		goto err1;
-	}
+	if (c4iw_num_stags(rdev) == 0)
+		return -EINVAL;
 
 	rdev->stats.pd.total = T4_MAX_NUM_PD;
 	rdev->stats.stag.total = rdev->lldi.vr->stag.size;
@@ -831,29 +821,31 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 	err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD);
 	if (err) {
 		printk(KERN_ERR MOD "error %d initializing resources\n", err);
-		goto err1;
+		return err;
 	}
 	err = c4iw_pblpool_create(rdev);
 	if (err) {
 		printk(KERN_ERR MOD "error %d initializing pbl pool\n", err);
-		goto err2;
+		goto destroy_resource;
 	}
 	err = c4iw_rqtpool_create(rdev);
 	if (err) {
 		printk(KERN_ERR MOD "error %d initializing rqt pool\n", err);
-		goto err3;
+		goto destroy_pblpool;
 	}
 	err = c4iw_ocqp_pool_create(rdev);
 	if (err) {
 		printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err);
-		goto err4;
+		goto destroy_rqtpool;
 	}
 	rdev->status_page = (struct t4_dev_status_page *)
 			    __get_free_page(GFP_KERNEL);
-	if (!rdev->status_page) {
-		pr_err(MOD "error allocating status page\n");
-		goto err4;
-	}
+	if (!rdev->status_page)
+		goto destroy_ocqp_pool;
+	rdev->status_page->qp_start = rdev->lldi.vr->qp.start;
+	rdev->status_page->qp_size = rdev->lldi.vr->qp.size;
+	rdev->status_page->cq_start = rdev->lldi.vr->cq.start;
+	rdev->status_page->cq_size = rdev->lldi.vr->cq.size;
 
 	if (c4iw_wr_log) {
 		rdev->wr_log = kzalloc((1 << c4iw_wr_log_size_order) *
@@ -869,13 +861,14 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 	rdev->status_page->db_off = 0;
 
 	return 0;
-err4:
+destroy_ocqp_pool:
+	c4iw_ocqp_pool_destroy(rdev);
+destroy_rqtpool:
 	c4iw_rqtpool_destroy(rdev);
-err3:
+destroy_pblpool:
 	c4iw_pblpool_destroy(rdev);
-err2:
+destroy_resource:
 	c4iw_destroy_resource(&rdev->resource);
-err1:
 	return err;
 }
 
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 00e55faa086a..fb2de75a0392 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -947,8 +947,6 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		      struct ib_send_wr **bad_wr);
 int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		      struct ib_recv_wr **bad_wr);
-int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
-		 struct ib_mw_bind *mw_bind);
 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
 int c4iw_destroy_listen(struct iw_cm_id *cm_id);
@@ -968,17 +966,6 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
 					   u64 length, u64 virt, int acc,
 					   struct ib_udata *udata);
 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
-struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
-					struct ib_phys_buf *buffer_list,
-					int num_phys_buf,
-					int acc,
-					u64 *iova_start);
-int c4iw_reregister_phys_mem(struct ib_mr *mr,
-				     int mr_rereg_mask,
-				     struct ib_pd *pd,
-				     struct ib_phys_buf *buffer_list,
-				     int num_phys_buf,
-				     int acc, u64 *iova_start);
 int c4iw_dereg_mr(struct ib_mr *ib_mr);
 int c4iw_destroy_cq(struct ib_cq *ib_cq);
 struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index e1629ab58db7..7849890c4781 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -392,32 +392,6 @@ static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
 	return ret;
 }
 
-static int reregister_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
-			  struct c4iw_mr *mhp, int shift, int npages)
-{
-	u32 stag;
-	int ret;
-
-	if (npages > mhp->attr.pbl_size)
-		return -ENOMEM;
-
-	stag = mhp->attr.stag;
-	ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid,
-			      FW_RI_STAG_NSMR, mhp->attr.perms,
-			      mhp->attr.mw_bind_enable, mhp->attr.zbva,
-			      mhp->attr.va_fbo, mhp->attr.len, shift - 12,
-			      mhp->attr.pbl_size, mhp->attr.pbl_addr);
-	if (ret)
-		return ret;
-
-	ret = finish_mem_reg(mhp, stag);
-	if (ret)
-		dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
-		       mhp->attr.pbl_addr);
-
-	return ret;
-}
-
 static int alloc_pbl(struct c4iw_mr *mhp, int npages)
 {
 	mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev,
@@ -431,228 +405,6 @@ static int alloc_pbl(struct c4iw_mr *mhp, int npages)
 	return 0;
 }
 
-static int build_phys_page_list(struct ib_phys_buf *buffer_list,
-				int num_phys_buf, u64 *iova_start,
-				u64 *total_size, int *npages,
-				int *shift, __be64 **page_list)
-{
-	u64 mask;
-	int i, j, n;
-
-	mask = 0;
-	*total_size = 0;
-	for (i = 0; i < num_phys_buf; ++i) {
-		if (i != 0 && buffer_list[i].addr & ~PAGE_MASK)
-			return -EINVAL;
-		if (i != 0 && i != num_phys_buf - 1 &&
-		    (buffer_list[i].size & ~PAGE_MASK))
-			return -EINVAL;
-		*total_size += buffer_list[i].size;
-		if (i > 0)
-			mask |= buffer_list[i].addr;
-		else
-			mask |= buffer_list[i].addr & PAGE_MASK;
-		if (i != num_phys_buf - 1)
-			mask |= buffer_list[i].addr + buffer_list[i].size;
-		else
-			mask |= (buffer_list[i].addr + buffer_list[i].size +
-				PAGE_SIZE - 1) & PAGE_MASK;
-	}
-
-	if (*total_size > 0xFFFFFFFFULL)
-		return -ENOMEM;
-
-	/* Find largest page shift we can use to cover buffers */
-	for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift))
-		if ((1ULL << *shift) & mask)
-			break;
-
-	buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1);
-	buffer_list[0].addr &= ~0ull << *shift;
-
-	*npages = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		*npages += (buffer_list[i].size +
-			(1ULL << *shift) - 1) >> *shift;
-
-	if (!*npages)
-		return -EINVAL;
-
-	*page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL);
-	if (!*page_list)
-		return -ENOMEM;
-
-	n = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		for (j = 0;
-		     j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift;
-		     ++j)
-			(*page_list)[n++] = cpu_to_be64(buffer_list[i].addr +
-			    ((u64) j << *shift));
-
-	PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n",
-	     __func__, (unsigned long long)*iova_start,
-	     (unsigned long long)mask, *shift, (unsigned long long)*total_size,
-	     *npages);
-
-	return 0;
-
-}
-
-int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask,
-			     struct ib_pd *pd, struct ib_phys_buf *buffer_list,
-			     int num_phys_buf, int acc, u64 *iova_start)
-{
-
-	struct c4iw_mr mh, *mhp;
-	struct c4iw_pd *php;
-	struct c4iw_dev *rhp;
-	__be64 *page_list = NULL;
-	int shift = 0;
-	u64 total_size;
-	int npages;
-	int ret;
-
-	PDBG("%s ib_mr %p ib_pd %p\n", __func__, mr, pd);
-
-	/* There can be no memory windows */
-	if (atomic_read(&mr->usecnt))
-		return -EINVAL;
-
-	mhp = to_c4iw_mr(mr);
-	rhp = mhp->rhp;
-	php = to_c4iw_pd(mr->pd);
-
-	/* make sure we are on the same adapter */
-	if (rhp != php->rhp)
-		return -EINVAL;
-
-	memcpy(&mh, mhp, sizeof *mhp);
-
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		php = to_c4iw_pd(pd);
-	if (mr_rereg_mask & IB_MR_REREG_ACCESS) {
-		mh.attr.perms = c4iw_ib_to_tpt_access(acc);
-		mh.attr.mw_bind_enable = (acc & IB_ACCESS_MW_BIND) ==
-					 IB_ACCESS_MW_BIND;
-	}
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
-		ret = build_phys_page_list(buffer_list, num_phys_buf,
-						iova_start,
-						&total_size, &npages,
-						&shift, &page_list);
-		if (ret)
-			return ret;
-	}
-
-	if (mr_exceeds_hw_limits(rhp, total_size)) {
-		kfree(page_list);
-		return -EINVAL;
-	}
-
-	ret = reregister_mem(rhp, php, &mh, shift, npages);
-	kfree(page_list);
-	if (ret)
-		return ret;
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		mhp->attr.pdid = php->pdid;
-	if (mr_rereg_mask & IB_MR_REREG_ACCESS)
-		mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
-		mhp->attr.zbva = 0;
-		mhp->attr.va_fbo = *iova_start;
-		mhp->attr.page_size = shift - 12;
-		mhp->attr.len = (u32) total_size;
-		mhp->attr.pbl_size = npages;
-	}
-
-	return 0;
-}
-
-struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
-				     struct ib_phys_buf *buffer_list,
-				     int num_phys_buf, int acc, u64 *iova_start)
-{
-	__be64 *page_list;
-	int shift;
-	u64 total_size;
-	int npages;
-	struct c4iw_dev *rhp;
-	struct c4iw_pd *php;
-	struct c4iw_mr *mhp;
-	int ret;
-
-	PDBG("%s ib_pd %p\n", __func__, pd);
-	php = to_c4iw_pd(pd);
-	rhp = php->rhp;
-
-	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
-	if (!mhp)
-		return ERR_PTR(-ENOMEM);
-
-	mhp->rhp = rhp;
-
-	/* First check that we have enough alignment */
-	if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
-		ret = -EINVAL;
-		goto err;
-	}
-
-	if (num_phys_buf > 1 &&
-	    ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
-		ret = -EINVAL;
-		goto err;
-	}
-
-	ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
-					&total_size, &npages, &shift,
-					&page_list);
-	if (ret)
-		goto err;
-
-	if (mr_exceeds_hw_limits(rhp, total_size)) {
-		kfree(page_list);
-		ret = -EINVAL;
-		goto err;
-	}
-
-	ret = alloc_pbl(mhp, npages);
-	if (ret) {
-		kfree(page_list);
-		goto err;
-	}
-
-	ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr,
-			     npages);
-	kfree(page_list);
-	if (ret)
-		goto err_pbl;
-
-	mhp->attr.pdid = php->pdid;
-	mhp->attr.zbva = 0;
-
-	mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
-	mhp->attr.va_fbo = *iova_start;
-	mhp->attr.page_size = shift - 12;
-
-	mhp->attr.len = (u32) total_size;
-	mhp->attr.pbl_size = npages;
-	ret = register_mem(rhp, php, mhp, shift);
-	if (ret)
-		goto err_pbl;
-
-	return &mhp->ibmr;
-
-err_pbl:
-	c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
-			      mhp->attr.pbl_size << 3);
-
-err:
-	kfree(mhp);
-	return ERR_PTR(ret);
-
-}
-
 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
 {
 	struct c4iw_dev *rhp;
@@ -952,9 +704,6 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr)
 	u32 mmid;
 
 	PDBG("%s ib_mr %p\n", __func__, ib_mr);
-	/* There can be no memory windows */
-	if (atomic_read(&ib_mr->usecnt))
-		return -EINVAL;
 
 	mhp = to_c4iw_mr(ib_mr);
 	rhp = mhp->rhp;
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 0a7d99818b17..ec04272fbdc2 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -549,12 +549,9 @@ int c4iw_register_device(struct c4iw_dev *dev)
 	dev->ibdev.resize_cq = c4iw_resize_cq;
 	dev->ibdev.poll_cq = c4iw_poll_cq;
 	dev->ibdev.get_dma_mr = c4iw_get_dma_mr;
-	dev->ibdev.reg_phys_mr = c4iw_register_phys_mem;
-	dev->ibdev.rereg_phys_mr = c4iw_reregister_phys_mem;
 	dev->ibdev.reg_user_mr = c4iw_reg_user_mr;
 	dev->ibdev.dereg_mr = c4iw_dereg_mr;
 	dev->ibdev.alloc_mw = c4iw_alloc_mw;
-	dev->ibdev.bind_mw = c4iw_bind_mw;
 	dev->ibdev.dealloc_mw = c4iw_dealloc_mw;
 	dev->ibdev.alloc_mr = c4iw_alloc_mr;
 	dev->ibdev.map_mr_sg = c4iw_map_mr_sg;
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index aa515afee724..e99345eb875a 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -933,11 +933,6 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 	return err;
 }
 
-int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw, struct ib_mw_bind *mw_bind)
-{
-	return -ENOSYS;
-}
-
 static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type,
 				    u8 *ecode)
 {
diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index 1092a2d1f607..6126bbe36095 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -699,4 +699,11 @@ static inline void t4_set_cq_in_error(struct t4_cq *cq)
 
 struct t4_dev_status_page {
 	u8 db_off;
+	u8 pad1;
+	u16 pad2;
+	u32 pad3;
+	u64 qp_start;
+	u64 qp_size;
+	u64 cq_start;
+	u64 cq_size;
 };
diff --git a/drivers/infiniband/hw/cxgb4/user.h b/drivers/infiniband/hw/cxgb4/user.h
index cbd0ce170728..295f422b9a3a 100644
--- a/drivers/infiniband/hw/cxgb4/user.h
+++ b/drivers/infiniband/hw/cxgb4/user.h
@@ -32,7 +32,7 @@
 #ifndef __C4IW_USER_H__
 #define __C4IW_USER_H__
 
-#define C4IW_UVERBS_ABI_VERSION	2
+#define C4IW_UVERBS_ABI_VERSION	3
 
 /*
  * Make sure that all structs defined in this file remain laid out so
diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
index 86af71351d9a..105246fba2e7 100644
--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -92,7 +92,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
 				ah_attr->grh.sgid_index, &sgid, &gid_attr);
 	if (ret)
 		return ERR_PTR(ret);
-	memset(ah->av.eth.s_mac, 0, ETH_ALEN);
+	eth_zero_addr(ah->av.eth.s_mac);
 	if (gid_attr.ndev) {
 		if (is_vlan_dev(gid_attr.ndev))
 			vlan_tag = vlan_dev_vlan_id(gid_attr.ndev);
@@ -104,6 +104,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
 	ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24));
 	ah->av.eth.gid_index = mlx4_ib_gid_index_to_real_index(ibdev, ah_attr->port_num, ah_attr->grh.sgid_index);
 	ah->av.eth.vlan = cpu_to_be16(vlan_tag);
+	ah->av.eth.hop_limit = ah_attr->grh.hop_limit;
 	if (ah_attr->static_rate) {
 		ah->av.eth.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET;
 		while (ah->av.eth.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index b88fc8f5ab18..9f8b516eb2b0 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -811,9 +811,6 @@ repoll:
 			wc->opcode    = IB_WC_MASKED_FETCH_ADD;
 			wc->byte_len  = 8;
 			break;
-		case MLX4_OPCODE_BIND_MW:
-			wc->opcode    = IB_WC_BIND_MW;
-			break;
 		case MLX4_OPCODE_LSO:
 			wc->opcode    = IB_WC_LSO;
 			break;
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 97d6878f9938..1c7ab6cabbb8 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -154,9 +154,9 @@ static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_n
 	return dev;
 }
 
-static int mlx4_ib_update_gids(struct gid_entry *gids,
-			       struct mlx4_ib_dev *ibdev,
-			       u8 port_num)
+static int mlx4_ib_update_gids_v1(struct gid_entry *gids,
+				  struct mlx4_ib_dev *ibdev,
+				  u8 port_num)
 {
 	struct mlx4_cmd_mailbox *mailbox;
 	int err;
@@ -187,6 +187,63 @@ static int mlx4_ib_update_gids(struct gid_entry *gids,
 	return err;
 }
 
+static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids,
+				     struct mlx4_ib_dev *ibdev,
+				     u8 port_num)
+{
+	struct mlx4_cmd_mailbox *mailbox;
+	int err;
+	struct mlx4_dev *dev = ibdev->dev;
+	int i;
+	struct {
+		union ib_gid	gid;
+		__be32		rsrvd1[2];
+		__be16		rsrvd2;
+		u8		type;
+		u8		version;
+		__be32		rsrvd3;
+	} *gid_tbl;
+
+	mailbox = mlx4_alloc_cmd_mailbox(dev);
+	if (IS_ERR(mailbox))
+		return -ENOMEM;
+
+	gid_tbl = mailbox->buf;
+	for (i = 0; i < MLX4_MAX_PORT_GIDS; ++i) {
+		memcpy(&gid_tbl[i].gid, &gids[i].gid, sizeof(union ib_gid));
+		if (gids[i].gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
+			gid_tbl[i].version = 2;
+			if (!ipv6_addr_v4mapped((struct in6_addr *)&gids[i].gid))
+				gid_tbl[i].type = 1;
+			else
+				memset(&gid_tbl[i].gid, 0, 12);
+		}
+	}
+
+	err = mlx4_cmd(dev, mailbox->dma,
+		       MLX4_SET_PORT_ROCE_ADDR << 8 | port_num,
+		       1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
+		       MLX4_CMD_WRAPPED);
+	if (mlx4_is_bonded(dev))
+		err += mlx4_cmd(dev, mailbox->dma,
+				MLX4_SET_PORT_ROCE_ADDR << 8 | 2,
+				1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
+				MLX4_CMD_WRAPPED);
+
+	mlx4_free_cmd_mailbox(dev, mailbox);
+	return err;
+}
+
+static int mlx4_ib_update_gids(struct gid_entry *gids,
+			       struct mlx4_ib_dev *ibdev,
+			       u8 port_num)
+{
+	if (ibdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2)
+		return mlx4_ib_update_gids_v1_v2(gids, ibdev, port_num);
+
+	return mlx4_ib_update_gids_v1(gids, ibdev, port_num);
+}
+
 static int mlx4_ib_add_gid(struct ib_device *device,
 			   u8 port_num,
 			   unsigned int index,
@@ -215,7 +272,8 @@ static int mlx4_ib_add_gid(struct ib_device *device,
 	port_gid_table = &iboe->gids[port_num - 1];
 	spin_lock_bh(&iboe->lock);
 	for (i = 0; i < MLX4_MAX_PORT_GIDS; ++i) {
-		if (!memcmp(&port_gid_table->gids[i].gid, gid, sizeof(*gid))) {
+		if (!memcmp(&port_gid_table->gids[i].gid, gid, sizeof(*gid)) &&
+		    (port_gid_table->gids[i].gid_type == attr->gid_type))  {
 			found = i;
 			break;
 		}
@@ -233,6 +291,7 @@ static int mlx4_ib_add_gid(struct ib_device *device,
 			} else {
 				*context = port_gid_table->gids[free].ctx;
 				memcpy(&port_gid_table->gids[free].gid, gid, sizeof(*gid));
+				port_gid_table->gids[free].gid_type = attr->gid_type;
 				port_gid_table->gids[free].ctx->real_index = free;
 				port_gid_table->gids[free].ctx->refcount = 1;
 				hw_update = 1;
@@ -248,8 +307,10 @@ static int mlx4_ib_add_gid(struct ib_device *device,
 		if (!gids) {
 			ret = -ENOMEM;
 		} else {
-			for (i = 0; i < MLX4_MAX_PORT_GIDS; i++)
+			for (i = 0; i < MLX4_MAX_PORT_GIDS; i++) {
 				memcpy(&gids[i].gid, &port_gid_table->gids[i].gid, sizeof(union ib_gid));
+				gids[i].gid_type = port_gid_table->gids[i].gid_type;
+			}
 		}
 	}
 	spin_unlock_bh(&iboe->lock);
@@ -325,6 +386,7 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
 	int i;
 	int ret;
 	unsigned long flags;
+	struct ib_gid_attr attr;
 
 	if (port_num > MLX4_MAX_PORTS)
 		return -EINVAL;
@@ -335,10 +397,13 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
 	if (!rdma_cap_roce_gid_table(&ibdev->ib_dev, port_num))
 		return index;
 
-	ret = ib_get_cached_gid(&ibdev->ib_dev, port_num, index, &gid, NULL);
+	ret = ib_get_cached_gid(&ibdev->ib_dev, port_num, index, &gid, &attr);
 	if (ret)
 		return ret;
 
+	if (attr.ndev)
+		dev_put(attr.ndev);
+
 	if (!memcmp(&gid, &zgid, sizeof(gid)))
 		return -EINVAL;
 
@@ -346,7 +411,8 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
 	port_gid_table = &iboe->gids[port_num - 1];
 
 	for (i = 0; i < MLX4_MAX_PORT_GIDS; ++i)
-		if (!memcmp(&port_gid_table->gids[i].gid, &gid, sizeof(gid))) {
+		if (!memcmp(&port_gid_table->gids[i].gid, &gid, sizeof(gid)) &&
+		    attr.gid_type == port_gid_table->gids[i].gid_type) {
 			ctx = port_gid_table->gids[i].ctx;
 			break;
 		}
@@ -2119,6 +2185,7 @@ static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num,
 			       struct ib_port_immutable *immutable)
 {
 	struct ib_port_attr attr;
+	struct mlx4_ib_dev *mdev = to_mdev(ibdev);
 	int err;
 
 	err = mlx4_ib_query_port(ibdev, port_num, &attr);
@@ -2128,10 +2195,15 @@ static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num,
 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 
-	if (mlx4_ib_port_link_layer(ibdev, port_num) == IB_LINK_LAYER_INFINIBAND)
+	if (mlx4_ib_port_link_layer(ibdev, port_num) == IB_LINK_LAYER_INFINIBAND) {
 		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
-	else
-		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	} else {
+		if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE)
+			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+		if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2)
+			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
+				RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+	}
 
 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 
@@ -2283,7 +2355,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
 	    dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
 		ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw;
-		ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw;
 		ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw;
 
 		ibdev->ib_dev.uverbs_cmd_mask |=
@@ -2423,7 +2494,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 	if (mlx4_ib_init_sriov(ibdev))
 		goto err_mad;
 
-	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) {
+	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE ||
+	    dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
 		if (!iboe->nb.notifier_call) {
 			iboe->nb.notifier_call = mlx4_ib_netdev_event;
 			err = register_netdevice_notifier(&iboe->nb);
@@ -2432,6 +2504,12 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 				goto err_notif;
 			}
 		}
+		if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
+			err = mlx4_config_roce_v2_port(dev, ROCE_V2_UDP_DPORT);
+			if (err) {
+				goto err_notif;
+			}
+		}
 	}
 
 	for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) {
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 1caa11edac03..52ce7b000044 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -177,11 +177,18 @@ struct mlx4_ib_wq {
 	unsigned		tail;
 };
 
+enum {
+	MLX4_IB_QP_CREATE_ROCE_V2_GSI = IB_QP_CREATE_RESERVED_START
+};
+
 enum mlx4_ib_qp_flags {
 	MLX4_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
 	MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
 	MLX4_IB_QP_NETIF = IB_QP_CREATE_NETIF_QP,
 	MLX4_IB_QP_CREATE_USE_GFP_NOIO = IB_QP_CREATE_USE_GFP_NOIO,
+
+	/* Mellanox specific flags start from IB_QP_CREATE_RESERVED_START */
+	MLX4_IB_ROCE_V2_GSI_QP = MLX4_IB_QP_CREATE_ROCE_V2_GSI,
 	MLX4_IB_SRIOV_TUNNEL_QP = 1 << 30,
 	MLX4_IB_SRIOV_SQP = 1 << 31,
 };
@@ -478,6 +485,7 @@ struct gid_cache_context {
 
 struct gid_entry {
 	union ib_gid	gid;
+	enum ib_gid_type gid_type;
 	struct gid_cache_context *ctx;
 };
 
@@ -704,8 +712,6 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				  struct ib_udata *udata);
 int mlx4_ib_dereg_mr(struct ib_mr *mr);
 struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
-int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
-		    struct ib_mw_bind *mw_bind);
 int mlx4_ib_dealloc_mw(struct ib_mw *mw);
 struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
 			       enum ib_mr_type mr_type,
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 4d1e1c632603..242b94ec105b 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -366,28 +366,6 @@ err_free:
 	return ERR_PTR(err);
 }
 
-int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
-		    struct ib_mw_bind *mw_bind)
-{
-	struct ib_bind_mw_wr  wr;
-	struct ib_send_wr *bad_wr;
-	int ret;
-
-	memset(&wr, 0, sizeof(wr));
-	wr.wr.opcode		= IB_WR_BIND_MW;
-	wr.wr.wr_id		= mw_bind->wr_id;
-	wr.wr.send_flags	= mw_bind->send_flags;
-	wr.mw			= mw;
-	wr.bind_info		= mw_bind->bind_info;
-	wr.rkey			= ib_inc_rkey(mw->rkey);
-
-	ret = mlx4_ib_post_send(qp, &wr.wr, &bad_wr);
-	if (!ret)
-		mw->rkey = wr.rkey;
-
-	return ret;
-}
-
 int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)
 {
 	struct mlx4_ib_mw *mw = to_mmw(ibmw);
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 13eaaf45288f..bc5536f00b6c 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -32,6 +32,8 @@
  */
 
 #include <linux/log2.h>
+#include <linux/etherdevice.h>
+#include <net/ip.h>
 #include <linux/slab.h>
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
@@ -85,6 +87,7 @@ struct mlx4_ib_sqp {
 	u32			send_psn;
 	struct ib_ud_header	ud_header;
 	u8			header_buf[MLX4_IB_UD_HEADER_SIZE];
+	struct ib_qp		*roce_v2_gsi;
 };
 
 enum {
@@ -115,7 +118,6 @@ static const __be32 mlx4_ib_opcode[] = {
 	[IB_WR_REG_MR]				= cpu_to_be32(MLX4_OPCODE_FMR),
 	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
 	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
-	[IB_WR_BIND_MW]				= cpu_to_be32(MLX4_OPCODE_BIND_MW),
 };
 
 static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
@@ -154,7 +156,10 @@ static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 			}
 		}
 	}
-	return proxy_sqp;
+	if (proxy_sqp)
+		return 1;
+
+	return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
 }
 
 /* used for INIT/CLOSE port logic */
@@ -796,11 +801,13 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 		if (err)
 			goto err_mtt;
 
-		qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(u64), gfp);
+		qp->sq.wrid = kmalloc_array(qp->sq.wqe_cnt, sizeof(u64),
+					gfp | __GFP_NOWARN);
 		if (!qp->sq.wrid)
 			qp->sq.wrid = __vmalloc(qp->sq.wqe_cnt * sizeof(u64),
 						gfp, PAGE_KERNEL);
-		qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(u64), gfp);
+		qp->rq.wrid = kmalloc_array(qp->rq.wqe_cnt, sizeof(u64),
+					gfp | __GFP_NOWARN);
 		if (!qp->rq.wrid)
 			qp->rq.wrid = __vmalloc(qp->rq.wqe_cnt * sizeof(u64),
 						gfp, PAGE_KERNEL);
@@ -1099,9 +1106,9 @@ static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
 		return dev->dev->caps.qp1_proxy[attr->port_num - 1];
 }
 
-struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
-				struct ib_qp_init_attr *init_attr,
-				struct ib_udata *udata)
+static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
+					struct ib_qp_init_attr *init_attr,
+					struct ib_udata *udata)
 {
 	struct mlx4_ib_qp *qp = NULL;
 	int err;
@@ -1120,6 +1127,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 					MLX4_IB_SRIOV_TUNNEL_QP |
 					MLX4_IB_SRIOV_SQP |
 					MLX4_IB_QP_NETIF |
+					MLX4_IB_QP_CREATE_ROCE_V2_GSI |
 					MLX4_IB_QP_CREATE_USE_GFP_NOIO))
 		return ERR_PTR(-EINVAL);
 
@@ -1128,15 +1136,21 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 			return ERR_PTR(-EINVAL);
 	}
 
-	if (init_attr->create_flags &&
-	    ((udata && init_attr->create_flags & ~(sup_u_create_flags)) ||
-	     ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
-					   MLX4_IB_QP_CREATE_USE_GFP_NOIO |
-					   MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)) &&
-	      init_attr->qp_type != IB_QPT_UD) ||
-	     ((init_attr->create_flags & MLX4_IB_SRIOV_SQP) &&
-	      init_attr->qp_type > IB_QPT_GSI)))
-		return ERR_PTR(-EINVAL);
+	if (init_attr->create_flags) {
+		if (udata && init_attr->create_flags & ~(sup_u_create_flags))
+			return ERR_PTR(-EINVAL);
+
+		if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
+						 MLX4_IB_QP_CREATE_USE_GFP_NOIO |
+						 MLX4_IB_QP_CREATE_ROCE_V2_GSI  |
+						 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
+		     init_attr->qp_type != IB_QPT_UD) ||
+		    (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
+		     init_attr->qp_type > IB_QPT_GSI) ||
+		    (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
+		     init_attr->qp_type != IB_QPT_GSI))
+			return ERR_PTR(-EINVAL);
+	}
 
 	switch (init_attr->qp_type) {
 	case IB_QPT_XRC_TGT:
@@ -1173,19 +1187,29 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 	case IB_QPT_SMI:
 	case IB_QPT_GSI:
 	{
+		int sqpn;
+
 		/* Userspace is not allowed to create special QPs: */
 		if (udata)
 			return ERR_PTR(-EINVAL);
+		if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
+			int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0);
+
+			if (res)
+				return ERR_PTR(res);
+		} else {
+			sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
+		}
 
 		err = create_qp_common(to_mdev(pd->device), pd, init_attr, udata,
-				       get_sqp_num(to_mdev(pd->device), init_attr),
+				       sqpn,
 				       &qp, gfp);
 		if (err)
 			return ERR_PTR(err);
 
 		qp->port	= init_attr->port_num;
-		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
-
+		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
+			init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
 		break;
 	}
 	default:
@@ -1196,7 +1220,41 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 	return &qp->ibqp;
 }
 
-int mlx4_ib_destroy_qp(struct ib_qp *qp)
+struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
+				struct ib_qp_init_attr *init_attr,
+				struct ib_udata *udata) {
+	struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
+	struct ib_qp *ibqp;
+	struct mlx4_ib_dev *dev = to_mdev(device);
+
+	ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
+
+	if (!IS_ERR(ibqp) &&
+	    (init_attr->qp_type == IB_QPT_GSI) &&
+	    !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
+		struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
+		int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
+
+		if (is_eth &&
+		    dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
+			init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
+			sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
+
+			if (IS_ERR(sqp->roce_v2_gsi)) {
+				pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
+				sqp->roce_v2_gsi = NULL;
+			} else {
+				sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
+				sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;
+			}
+
+			init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
+		}
+	}
+	return ibqp;
+}
+
+static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 {
 	struct mlx4_ib_dev *dev = to_mdev(qp->device);
 	struct mlx4_ib_qp *mqp = to_mqp(qp);
@@ -1225,6 +1283,20 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp)
 	return 0;
 }
 
+int mlx4_ib_destroy_qp(struct ib_qp *qp)
+{
+	struct mlx4_ib_qp *mqp = to_mqp(qp);
+
+	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
+		struct mlx4_ib_sqp *sqp = to_msqp(mqp);
+
+		if (sqp->roce_v2_gsi)
+			ib_destroy_qp(sqp->roce_v2_gsi);
+	}
+
+	return _mlx4_ib_destroy_qp(qp);
+}
+
 static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
 {
 	switch (type) {
@@ -1507,6 +1579,24 @@ static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 	return 0;
 }
 
+enum {
+	MLX4_QPC_ROCE_MODE_1 = 0,
+	MLX4_QPC_ROCE_MODE_2 = 2,
+	MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
+};
+
+static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
+{
+	switch (gid_type) {
+	case IB_GID_TYPE_ROCE:
+		return MLX4_QPC_ROCE_MODE_1;
+	case IB_GID_TYPE_ROCE_UDP_ENCAP:
+		return MLX4_QPC_ROCE_MODE_2;
+	default:
+		return MLX4_QPC_ROCE_MODE_UNDEFINED;
+	}
+}
+
 static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 			       const struct ib_qp_attr *attr, int attr_mask,
 			       enum ib_qp_state cur_state, enum ib_qp_state new_state)
@@ -1633,6 +1723,14 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 			mlx4_ib_steer_qp_reg(dev, qp, 1);
 			steer_qp = 1;
 		}
+
+		if (ibqp->qp_type == IB_QPT_GSI) {
+			enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
+				IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
+			u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
+
+			context->rlkey_roce_mode |= (qpc_roce_mode << 6);
+		}
 	}
 
 	if (attr_mask & IB_QP_PKEY_INDEX) {
@@ -1650,9 +1748,10 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		u16 vlan = 0xffff;
 		u8 smac[ETH_ALEN];
 		int status = 0;
+		int is_eth = rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
+			attr->ah_attr.ah_flags & IB_AH_GRH;
 
-		if (rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
-		    attr->ah_attr.ah_flags & IB_AH_GRH) {
+		if (is_eth) {
 			int index = attr->ah_attr.grh.sgid_index;
 
 			status = ib_get_cached_gid(ibqp->device, port_num,
@@ -1674,6 +1773,18 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 
 		optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
 			   MLX4_QP_OPTPAR_SCHED_QUEUE);
+
+		if (is_eth &&
+		    (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
+			u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
+
+			if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
+				err = -EINVAL;
+				goto out;
+			}
+			context->rlkey_roce_mode |= (qpc_roce_mode << 6);
+		}
+
 	}
 
 	if (attr_mask & IB_QP_TIMEOUT) {
@@ -1845,7 +1956,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		sqd_event = 0;
 
 	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
-		context->rlkey |= (1 << 4);
+		context->rlkey_roce_mode |= (1 << 4);
 
 	/*
 	 * Before passing a kernel QP to the HW, make sure that the
@@ -2022,8 +2133,8 @@ out:
 	return err;
 }
 
-int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
-		      int attr_mask, struct ib_udata *udata)
+static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			      int attr_mask, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
@@ -2126,6 +2237,27 @@ out:
 	return err;
 }
 
+int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		      int attr_mask, struct ib_udata *udata)
+{
+	struct mlx4_ib_qp *mqp = to_mqp(ibqp);
+	int ret;
+
+	ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
+
+	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
+		struct mlx4_ib_sqp *sqp = to_msqp(mqp);
+		int err = 0;
+
+		if (sqp->roce_v2_gsi)
+			err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
+		if (err)
+			pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
+			       err);
+	}
+	return ret;
+}
+
 static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
 {
 	int i;
@@ -2168,7 +2300,7 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
 	if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
 		send_size += sizeof (struct mlx4_ib_tunnel_header);
 
-	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, &sqp->ud_header);
+	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
 
 	if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
 		sqp->ud_header.lrh.service_level =
@@ -2252,16 +2384,7 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
 	return 0;
 }
 
-static void mlx4_u64_to_smac(u8 *dst_mac, u64 src_mac)
-{
-	int i;
-
-	for (i = ETH_ALEN; i; i--) {
-		dst_mac[i - 1] = src_mac & 0xff;
-		src_mac >>= 8;
-	}
-}
-
+#define MLX4_ROCEV2_QP1_SPORT 0xC000
 static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 			    void *wqe, unsigned *mlx_seg_len)
 {
@@ -2281,6 +2404,8 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 	bool is_eth;
 	bool is_vlan = false;
 	bool is_grh;
+	bool is_udp = false;
+	int ip_version = 0;
 
 	send_size = 0;
 	for (i = 0; i < wr->wr.num_sge; ++i)
@@ -2289,6 +2414,8 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 	is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
 	is_grh = mlx4_ib_ah_grh_present(ah);
 	if (is_eth) {
+		struct ib_gid_attr gid_attr;
+
 		if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
 			/* When multi-function is enabled, the ib_core gid
 			 * indexes don't necessarily match the hw ones, so
@@ -2302,19 +2429,35 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 			err = ib_get_cached_gid(ib_dev,
 						be32_to_cpu(ah->av.ib.port_pd) >> 24,
 						ah->av.ib.gid_index, &sgid,
-						NULL);
-			if (!err && !memcmp(&sgid, &zgid, sizeof(sgid)))
-				err = -ENOENT;
-			if (err)
+						&gid_attr);
+			if (!err) {
+				if (gid_attr.ndev)
+					dev_put(gid_attr.ndev);
+				if (!memcmp(&sgid, &zgid, sizeof(sgid)))
+					err = -ENOENT;
+			}
+			if (!err) {
+				is_udp = gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
+				if (is_udp) {
+					if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
+						ip_version = 4;
+					else
+						ip_version = 6;
+					is_grh = false;
+				}
+			} else {
 				return err;
+			}
 		}
-
 		if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
 			vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
 			is_vlan = 1;
 		}
 	}
-	ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header);
+	err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
+			  ip_version, is_udp, 0, &sqp->ud_header);
+	if (err)
+		return err;
 
 	if (!is_eth) {
 		sqp->ud_header.lrh.service_level =
@@ -2323,7 +2466,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 		sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
 	}
 
-	if (is_grh) {
+	if (is_grh || (ip_version == 6)) {
 		sqp->ud_header.grh.traffic_class =
 			(be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
 		sqp->ud_header.grh.flow_label    =
@@ -2352,6 +2495,25 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 		       ah->av.ib.dgid, 16);
 	}
 
+	if (ip_version == 4) {
+		sqp->ud_header.ip4.tos =
+			(be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
+		sqp->ud_header.ip4.id = 0;
+		sqp->ud_header.ip4.frag_off = htons(IP_DF);
+		sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
+
+		memcpy(&sqp->ud_header.ip4.saddr,
+		       sgid.raw + 12, 4);
+		memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
+		sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
+	}
+
+	if (is_udp) {
+		sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
+		sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
+		sqp->ud_header.udp.csum = 0;
+	}
+
 	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
 
 	if (!is_eth) {
@@ -2380,34 +2542,27 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 
 	if (is_eth) {
 		struct in6_addr in6;
-
+		u16 ether_type;
 		u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
 
+		ether_type = (!is_udp) ? MLX4_IB_IBOE_ETHERTYPE :
+			(ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
+
 		mlx->sched_prio = cpu_to_be16(pcp);
 
+		ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
 		memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
-		/* FIXME: cache smac value? */
 		memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
 		memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
 		memcpy(&in6, sgid.raw, sizeof(in6));
 
-		if (!mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
-			u64 mac = atomic64_read(&to_mdev(ib_dev)->iboe.mac[sqp->qp.port - 1]);
-			u8 smac[ETH_ALEN];
-
-			mlx4_u64_to_smac(smac, mac);
-			memcpy(sqp->ud_header.eth.smac_h, smac, ETH_ALEN);
-		} else {
-			/* use the src mac of the tunnel */
-			memcpy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac, ETH_ALEN);
-		}
 
 		if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
 			mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
 		if (!is_vlan) {
-			sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
+			sqp->ud_header.eth.type = cpu_to_be16(ether_type);
 		} else {
-			sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
+			sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
 			sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
 		}
 	} else {
@@ -2528,25 +2683,6 @@ static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
 	fseg->reserved[1]	= 0;
 }
 
-static void set_bind_seg(struct mlx4_wqe_bind_seg *bseg,
-		struct ib_bind_mw_wr *wr)
-{
-	bseg->flags1 =
-		convert_access(wr->bind_info.mw_access_flags) &
-		cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ  |
-			    MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE |
-			    MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC);
-	bseg->flags2 = 0;
-	if (wr->mw->type == IB_MW_TYPE_2)
-		bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_TYPE_2);
-	if (wr->bind_info.mw_access_flags & IB_ZERO_BASED)
-		bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_ZERO_BASED);
-	bseg->new_rkey = cpu_to_be32(wr->rkey);
-	bseg->lkey = cpu_to_be32(wr->bind_info.mr->lkey);
-	bseg->addr = cpu_to_be64(wr->bind_info.addr);
-	bseg->length = cpu_to_be64(wr->bind_info.length);
-}
-
 static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
 {
 	memset(iseg, 0, sizeof(*iseg));
@@ -2766,6 +2902,29 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 	int i;
 	struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
 
+	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
+		struct mlx4_ib_sqp *sqp = to_msqp(qp);
+
+		if (sqp->roce_v2_gsi) {
+			struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
+			struct ib_gid_attr gid_attr;
+			union ib_gid gid;
+
+			if (!ib_get_cached_gid(ibqp->device,
+					       be32_to_cpu(ah->av.ib.port_pd) >> 24,
+					       ah->av.ib.gid_index, &gid,
+					       &gid_attr)) {
+				if (gid_attr.ndev)
+					dev_put(gid_attr.ndev);
+				qp = (gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
+					to_mqp(sqp->roce_v2_gsi) : qp;
+			} else {
+				pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
+				       ah->av.ib.gid_index);
+			}
+		}
+	}
+
 	spin_lock_irqsave(&qp->sq.lock, flags);
 	if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
 		err = -EIO;
@@ -2867,13 +3026,6 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
 				break;
 
-			case IB_WR_BIND_MW:
-				ctrl->srcrb_flags |=
-					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
-				set_bind_seg(wqe, bind_mw_wr(wr));
-				wqe  += sizeof(struct mlx4_wqe_bind_seg);
-				size += sizeof(struct mlx4_wqe_bind_seg) / 16;
-				break;
 			default:
 				/* No extra segments required for sends */
 				break;
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index c394376ebe06..0597f3eef5d0 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -171,7 +171,8 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 		if (err)
 			goto err_mtt;
 
-		srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL);
+		srq->wrid = kmalloc_array(srq->msrq.max, sizeof(u64),
+					GFP_KERNEL | __GFP_NOWARN);
 		if (!srq->wrid) {
 			srq->wrid = __vmalloc(srq->msrq.max * sizeof(u64),
 					      GFP_KERNEL, PAGE_KERNEL);
diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c
index 66080580e24d..745efa4cfc71 100644
--- a/drivers/infiniband/hw/mlx5/ah.c
+++ b/drivers/infiniband/hw/mlx5/ah.c
@@ -32,8 +32,10 @@
 
 #include "mlx5_ib.h"
 
-struct ib_ah *create_ib_ah(struct ib_ah_attr *ah_attr,
-			   struct mlx5_ib_ah *ah)
+static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
+				  struct mlx5_ib_ah *ah,
+				  struct ib_ah_attr *ah_attr,
+				  enum rdma_link_layer ll)
 {
 	if (ah_attr->ah_flags & IB_AH_GRH) {
 		memcpy(ah->av.rgid, &ah_attr->grh.dgid, 16);
@@ -44,9 +46,20 @@ struct ib_ah *create_ib_ah(struct ib_ah_attr *ah_attr,
 		ah->av.tclass = ah_attr->grh.traffic_class;
 	}
 
-	ah->av.rlid = cpu_to_be16(ah_attr->dlid);
-	ah->av.fl_mlid = ah_attr->src_path_bits & 0x7f;
-	ah->av.stat_rate_sl = (ah_attr->static_rate << 4) | (ah_attr->sl & 0xf);
+	ah->av.stat_rate_sl = (ah_attr->static_rate << 4);
+
+	if (ll == IB_LINK_LAYER_ETHERNET) {
+		memcpy(ah->av.rmac, ah_attr->dmac, sizeof(ah_attr->dmac));
+		ah->av.udp_sport =
+			mlx5_get_roce_udp_sport(dev,
+						ah_attr->port_num,
+						ah_attr->grh.sgid_index);
+		ah->av.stat_rate_sl |= (ah_attr->sl & 0x7) << 1;
+	} else {
+		ah->av.rlid = cpu_to_be16(ah_attr->dlid);
+		ah->av.fl_mlid = ah_attr->src_path_bits & 0x7f;
+		ah->av.stat_rate_sl |= (ah_attr->sl & 0xf);
+	}
 
 	return &ah->ibah;
 }
@@ -54,12 +67,19 @@ struct ib_ah *create_ib_ah(struct ib_ah_attr *ah_attr,
 struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 {
 	struct mlx5_ib_ah *ah;
+	struct mlx5_ib_dev *dev = to_mdev(pd->device);
+	enum rdma_link_layer ll;
+
+	ll = pd->device->get_link_layer(pd->device, ah_attr->port_num);
+
+	if (ll == IB_LINK_LAYER_ETHERNET && !(ah_attr->ah_flags & IB_AH_GRH))
+		return ERR_PTR(-EINVAL);
 
 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
 	if (!ah)
 		return ERR_PTR(-ENOMEM);
 
-	return create_ib_ah(ah_attr, ah); /* never fails */
+	return create_ib_ah(dev, ah, ah_attr, ll); /* never fails */
 }
 
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 3dfd287256d6..fd1de31e0611 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -154,9 +154,6 @@ static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 		wc->opcode    = IB_WC_MASKED_FETCH_ADD;
 		wc->byte_len  = 8;
 		break;
-	case MLX5_OPCODE_BIND_MW:
-		wc->opcode    = IB_WC_BIND_MW;
-		break;
 	case MLX5_OPCODE_UMR:
 		wc->opcode = get_umr_comp(wq, idx);
 		break;
@@ -171,6 +168,7 @@ enum {
 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 			     struct mlx5_ib_qp *qp)
 {
+	enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1);
 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
 	struct mlx5_ib_srq *srq;
 	struct mlx5_ib_wq *wq;
@@ -236,6 +234,22 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 	} else {
 		wc->pkey_index = 0;
 	}
+
+	if (ll != IB_LINK_LAYER_ETHERNET)
+		return;
+
+	switch (wc->sl & 0x3) {
+	case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH:
+		wc->network_hdr_type = RDMA_NETWORK_IB;
+		break;
+	case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6:
+		wc->network_hdr_type = RDMA_NETWORK_IPV6;
+		break;
+	case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV4:
+		wc->network_hdr_type = RDMA_NETWORK_IPV4;
+		break;
+	}
+	wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
 }
 
 static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)
@@ -756,16 +770,16 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 	int uninitialized_var(index);
 	int uninitialized_var(inlen);
 	int cqe_size;
-	int irqn;
+	unsigned int irqn;
 	int eqn;
 	int err;
 
-	if (attr->flags)
-		return ERR_PTR(-EINVAL);
-
 	if (entries < 0)
 		return ERR_PTR(-EINVAL);
 
+	if (check_cq_create_flags(attr->flags))
+		return ERR_PTR(-EOPNOTSUPP);
+
 	entries = roundup_pow_of_two(entries + 1);
 	if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
 		return ERR_PTR(-EINVAL);
@@ -779,6 +793,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 	spin_lock_init(&cq->lock);
 	cq->resize_buf = NULL;
 	cq->resize_umem = NULL;
+	cq->create_flags = attr->flags;
 
 	if (context) {
 		err = create_cq_user(dev, udata, context, cq, entries,
@@ -796,6 +811,10 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 
 	cq->cqe_size = cqe_size;
 	cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
+
+	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)
+		cqb->ctx.cqe_sz_flags |= (1 << 1);
+
 	cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index);
 	err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
 	if (err)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index b0ec175cc6ba..ec737e2287fe 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -40,6 +40,8 @@
 #include <linux/io-mapping.h>
 #include <linux/sched.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
 #include <linux/mlx5/vport.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_umem.h>
@@ -66,12 +68,14 @@ static char mlx5_version[] =
 	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
 	DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
 
+enum {
+	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
+};
+
 static enum rdma_link_layer
-mlx5_ib_port_link_layer(struct ib_device *device)
+mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
 {
-	struct mlx5_ib_dev *dev = to_mdev(device);
-
-	switch (MLX5_CAP_GEN(dev->mdev, port_type)) {
+	switch (port_type_cap) {
 	case MLX5_CAP_PORT_TYPE_IB:
 		return IB_LINK_LAYER_INFINIBAND;
 	case MLX5_CAP_PORT_TYPE_ETH:
@@ -81,6 +85,202 @@ mlx5_ib_port_link_layer(struct ib_device *device)
 	}
 }
 
+static enum rdma_link_layer
+mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
+{
+	struct mlx5_ib_dev *dev = to_mdev(device);
+	int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
+
+	return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
+}
+
+static int mlx5_netdev_event(struct notifier_block *this,
+			     unsigned long event, void *ptr)
+{
+	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
+	struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
+						 roce.nb);
+
+	if ((event != NETDEV_UNREGISTER) && (event != NETDEV_REGISTER))
+		return NOTIFY_DONE;
+
+	write_lock(&ibdev->roce.netdev_lock);
+	if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
+		ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ? NULL : ndev;
+	write_unlock(&ibdev->roce.netdev_lock);
+
+	return NOTIFY_DONE;
+}
+
+static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
+					     u8 port_num)
+{
+	struct mlx5_ib_dev *ibdev = to_mdev(device);
+	struct net_device *ndev;
+
+	/* Ensure ndev does not disappear before we invoke dev_hold()
+	 */
+	read_lock(&ibdev->roce.netdev_lock);
+	ndev = ibdev->roce.netdev;
+	if (ndev)
+		dev_hold(ndev);
+	read_unlock(&ibdev->roce.netdev_lock);
+
+	return ndev;
+}
+
+static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
+				struct ib_port_attr *props)
+{
+	struct mlx5_ib_dev *dev = to_mdev(device);
+	struct net_device *ndev;
+	enum ib_mtu ndev_ib_mtu;
+	u16 qkey_viol_cntr;
+
+	memset(props, 0, sizeof(*props));
+
+	props->port_cap_flags  |= IB_PORT_CM_SUP;
+	props->port_cap_flags  |= IB_PORT_IP_BASED_GIDS;
+
+	props->gid_tbl_len      = MLX5_CAP_ROCE(dev->mdev,
+						roce_address_table_size);
+	props->max_mtu          = IB_MTU_4096;
+	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
+	props->pkey_tbl_len     = 1;
+	props->state            = IB_PORT_DOWN;
+	props->phys_state       = 3;
+
+	mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev, &qkey_viol_cntr);
+	props->qkey_viol_cntr = qkey_viol_cntr;
+
+	ndev = mlx5_ib_get_netdev(device, port_num);
+	if (!ndev)
+		return 0;
+
+	if (netif_running(ndev) && netif_carrier_ok(ndev)) {
+		props->state      = IB_PORT_ACTIVE;
+		props->phys_state = 5;
+	}
+
+	ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
+
+	dev_put(ndev);
+
+	props->active_mtu	= min(props->max_mtu, ndev_ib_mtu);
+
+	props->active_width	= IB_WIDTH_4X;  /* TODO */
+	props->active_speed	= IB_SPEED_QDR; /* TODO */
+
+	return 0;
+}
+
+static void ib_gid_to_mlx5_roce_addr(const union ib_gid *gid,
+				     const struct ib_gid_attr *attr,
+				     void *mlx5_addr)
+{
+#define MLX5_SET_RA(p, f, v) MLX5_SET(roce_addr_layout, p, f, v)
+	char *mlx5_addr_l3_addr	= MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
+					       source_l3_address);
+	void *mlx5_addr_mac	= MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
+					       source_mac_47_32);
+
+	if (!gid)
+		return;
+
+	ether_addr_copy(mlx5_addr_mac, attr->ndev->dev_addr);
+
+	if (is_vlan_dev(attr->ndev)) {
+		MLX5_SET_RA(mlx5_addr, vlan_valid, 1);
+		MLX5_SET_RA(mlx5_addr, vlan_id, vlan_dev_vlan_id(attr->ndev));
+	}
+
+	switch (attr->gid_type) {
+	case IB_GID_TYPE_IB:
+		MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_1);
+		break;
+	case IB_GID_TYPE_ROCE_UDP_ENCAP:
+		MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_2);
+		break;
+
+	default:
+		WARN_ON(true);
+	}
+
+	if (attr->gid_type != IB_GID_TYPE_IB) {
+		if (ipv6_addr_v4mapped((void *)gid))
+			MLX5_SET_RA(mlx5_addr, roce_l3_type,
+				    MLX5_ROCE_L3_TYPE_IPV4);
+		else
+			MLX5_SET_RA(mlx5_addr, roce_l3_type,
+				    MLX5_ROCE_L3_TYPE_IPV6);
+	}
+
+	if ((attr->gid_type == IB_GID_TYPE_IB) ||
+	    !ipv6_addr_v4mapped((void *)gid))
+		memcpy(mlx5_addr_l3_addr, gid, sizeof(*gid));
+	else
+		memcpy(&mlx5_addr_l3_addr[12], &gid->raw[12], 4);
+}
+
+static int set_roce_addr(struct ib_device *device, u8 port_num,
+			 unsigned int index,
+			 const union ib_gid *gid,
+			 const struct ib_gid_attr *attr)
+{
+	struct mlx5_ib_dev *dev	= to_mdev(device);
+	u32  in[MLX5_ST_SZ_DW(set_roce_address_in)];
+	u32 out[MLX5_ST_SZ_DW(set_roce_address_out)];
+	void *in_addr = MLX5_ADDR_OF(set_roce_address_in, in, roce_address);
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(device, port_num);
+
+	if (ll != IB_LINK_LAYER_ETHERNET)
+		return -EINVAL;
+
+	memset(in, 0, sizeof(in));
+
+	ib_gid_to_mlx5_roce_addr(gid, attr, in_addr);
+
+	MLX5_SET(set_roce_address_in, in, roce_address_index, index);
+	MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
+			   unsigned int index, const union ib_gid *gid,
+			   const struct ib_gid_attr *attr,
+			   __always_unused void **context)
+{
+	return set_roce_addr(device, port_num, index, gid, attr);
+}
+
+static int mlx5_ib_del_gid(struct ib_device *device, u8 port_num,
+			   unsigned int index, __always_unused void **context)
+{
+	return set_roce_addr(device, port_num, index, NULL, NULL);
+}
+
+__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
+			       int index)
+{
+	struct ib_gid_attr attr;
+	union ib_gid gid;
+
+	if (ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr))
+		return 0;
+
+	if (!attr.ndev)
+		return 0;
+
+	dev_put(attr.ndev);
+
+	if (attr.gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
+		return 0;
+
+	return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
+}
+
 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
 {
 	return !dev->mdev->issi;
@@ -97,13 +297,35 @@ static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 	if (mlx5_use_mad_ifc(to_mdev(ibdev)))
 		return MLX5_VPORT_ACCESS_METHOD_MAD;
 
-	if (mlx5_ib_port_link_layer(ibdev) ==
+	if (mlx5_ib_port_link_layer(ibdev, 1) ==
 	    IB_LINK_LAYER_ETHERNET)
 		return MLX5_VPORT_ACCESS_METHOD_NIC;
 
 	return MLX5_VPORT_ACCESS_METHOD_HCA;
 }
 
+static void get_atomic_caps(struct mlx5_ib_dev *dev,
+			    struct ib_device_attr *props)
+{
+	u8 tmp;
+	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
+	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
+	u8 atomic_req_8B_endianness_mode =
+		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianess_mode);
+
+	/* Check if HW supports 8 bytes standard atomic operations and capable
+	 * of host endianness respond
+	 */
+	tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
+	if (((atomic_operations & tmp) == tmp) &&
+	    (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
+	    (atomic_req_8B_endianness_mode)) {
+		props->atomic_cap = IB_ATOMIC_HCA;
+	} else {
+		props->atomic_cap = IB_ATOMIC_NONE;
+	}
+}
+
 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 					__be64 *sys_image_guid)
 {
@@ -119,13 +341,21 @@ static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 
 	case MLX5_VPORT_ACCESS_METHOD_HCA:
 		err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
-		if (!err)
-			*sys_image_guid = cpu_to_be64(tmp);
-		return err;
+		break;
+
+	case MLX5_VPORT_ACCESS_METHOD_NIC:
+		err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
+		break;
 
 	default:
 		return -EINVAL;
 	}
+
+	if (!err)
+		*sys_image_guid = cpu_to_be64(tmp);
+
+	return err;
+
 }
 
 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
@@ -179,13 +409,20 @@ static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
 
 	case MLX5_VPORT_ACCESS_METHOD_HCA:
 		err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
-		if (!err)
-			*node_guid = cpu_to_be64(tmp);
-		return err;
+		break;
+
+	case MLX5_VPORT_ACCESS_METHOD_NIC:
+		err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
+		break;
 
 	default:
 		return -EINVAL;
 	}
+
+	if (!err)
+		*node_guid = cpu_to_be64(tmp);
+
+	return err;
 }
 
 struct mlx5_reg_node_desc {
@@ -263,6 +500,10 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (MLX5_CAP_GEN(mdev, block_lb_mc))
 		props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 
+	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
+	    (MLX5_CAP_ETH(dev->mdev, csum_cap)))
+			props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
+
 	props->vendor_part_id	   = mdev->pdev->device;
 	props->hw_ver		   = mdev->pdev->revision;
 
@@ -278,7 +519,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_sge = min(max_rq_sg, max_sq_sg);
 	props->max_sge_rd = props->max_sge;
 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
-	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_eq_sz)) - 1;
+	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
 	props->max_mr		   = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
 	props->max_pd		   = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
 	props->max_qp_rd_atom	   = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
@@ -289,13 +530,15 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
 	props->max_srq_sge	   = max_rq_sg - 1;
 	props->max_fast_reg_page_list_len = (unsigned int)-1;
-	props->atomic_cap	   = IB_ATOMIC_NONE;
+	get_atomic_caps(dev, props);
 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 					   props->max_mcast_grp;
 	props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
+	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
+	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 	if (MLX5_CAP_GEN(mdev, pg))
@@ -303,6 +546,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->odp_caps = dev->odp_caps;
 #endif
 
+	if (MLX5_CAP_GEN(mdev, cd))
+		props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
+
 	return 0;
 }
 
@@ -483,6 +729,9 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 	case MLX5_VPORT_ACCESS_METHOD_HCA:
 		return mlx5_query_hca_port(ibdev, port, props);
 
+	case MLX5_VPORT_ACCESS_METHOD_NIC:
+		return mlx5_query_port_roce(ibdev, port, props);
+
 	default:
 		return -EINVAL;
 	}
@@ -583,8 +832,8 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 						  struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
-	struct mlx5_ib_alloc_ucontext_req_v2 req;
-	struct mlx5_ib_alloc_ucontext_resp resp;
+	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
+	struct mlx5_ib_alloc_ucontext_resp resp = {};
 	struct mlx5_ib_ucontext *context;
 	struct mlx5_uuar_info *uuari;
 	struct mlx5_uar *uars;
@@ -599,20 +848,22 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	if (!dev->ib_active)
 		return ERR_PTR(-EAGAIN);
 
-	memset(&req, 0, sizeof(req));
+	if (udata->inlen < sizeof(struct ib_uverbs_cmd_hdr))
+		return ERR_PTR(-EINVAL);
+
 	reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
 	if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
 		ver = 0;
-	else if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
+	else if (reqlen >= sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
 		ver = 2;
 	else
 		return ERR_PTR(-EINVAL);
 
-	err = ib_copy_from_udata(&req, udata, reqlen);
+	err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
 	if (err)
 		return ERR_PTR(err);
 
-	if (req.flags || req.reserved)
+	if (req.flags)
 		return ERR_PTR(-EINVAL);
 
 	if (req.total_num_uuars > MLX5_MAX_UUARS)
@@ -621,6 +872,14 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	if (req.total_num_uuars == 0)
 		return ERR_PTR(-EINVAL);
 
+	if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (reqlen > sizeof(req) &&
+	    !ib_is_udata_cleared(udata, sizeof(req),
+				 reqlen - sizeof(req)))
+		return ERR_PTR(-EOPNOTSUPP);
+
 	req.total_num_uuars = ALIGN(req.total_num_uuars,
 				    MLX5_NON_FP_BF_REGS_PER_PAGE);
 	if (req.num_low_latency_uuars > req.total_num_uuars - 1)
@@ -636,6 +895,11 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
 	resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
 	resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
+	resp.cqe_version = min_t(__u8,
+				 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
+				 req.max_cqe_version);
+	resp.response_length = min(offsetof(typeof(resp), response_length) +
+				   sizeof(resp.response_length), udata->outlen);
 
 	context = kzalloc(sizeof(*context), GFP_KERNEL);
 	if (!context)
@@ -681,22 +945,49 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
 #endif
 
+	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
+		err = mlx5_core_alloc_transport_domain(dev->mdev,
+						       &context->tdn);
+		if (err)
+			goto out_uars;
+	}
+
 	INIT_LIST_HEAD(&context->db_page_list);
 	mutex_init(&context->db_page_mutex);
 
 	resp.tot_uuars = req.total_num_uuars;
 	resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
-	err = ib_copy_to_udata(udata, &resp,
-			       sizeof(resp) - sizeof(resp.reserved));
+
+	if (field_avail(typeof(resp), cqe_version, udata->outlen))
+		resp.response_length += sizeof(resp.cqe_version);
+
+	if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
+		resp.comp_mask |=
+			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
+		resp.hca_core_clock_offset =
+			offsetof(struct mlx5_init_seg, internal_timer_h) %
+			PAGE_SIZE;
+		resp.response_length += sizeof(resp.hca_core_clock_offset) +
+					sizeof(resp.reserved2) +
+					sizeof(resp.reserved3);
+	}
+
+	err = ib_copy_to_udata(udata, &resp, resp.response_length);
 	if (err)
-		goto out_uars;
+		goto out_td;
 
 	uuari->ver = ver;
 	uuari->num_low_latency_uuars = req.num_low_latency_uuars;
 	uuari->uars = uars;
 	uuari->num_uars = num_uars;
+	context->cqe_version = resp.cqe_version;
+
 	return &context->ibucontext;
 
+out_td:
+	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
+		mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
+
 out_uars:
 	for (i--; i >= 0; i--)
 		mlx5_cmd_free_uar(dev->mdev, uars[i].index);
@@ -721,6 +1012,9 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 	struct mlx5_uuar_info *uuari = &context->uuari;
 	int i;
 
+	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
+		mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
+
 	for (i = 0; i < uuari->num_uars; i++) {
 		if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index))
 			mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
@@ -790,6 +1084,30 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
 		return -ENOSYS;
 
+	case MLX5_IB_MMAP_CORE_CLOCK:
+		if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+			return -EINVAL;
+
+		if (vma->vm_flags & (VM_WRITE | VM_EXEC))
+			return -EPERM;
+
+		/* Don't expose to user-space information it shouldn't have */
+		if (PAGE_SIZE > 4096)
+			return -EOPNOTSUPP;
+
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		pfn = (dev->mdev->iseg_base +
+		       offsetof(struct mlx5_init_seg, internal_timer_h)) >>
+			PAGE_SHIFT;
+		if (io_remap_pfn_range(vma, vma->vm_start, pfn,
+				       PAGE_SIZE, vma->vm_page_prot))
+			return -EAGAIN;
+
+		mlx5_ib_dbg(dev, "mapped internal timer at 0x%lx, PA 0x%llx\n",
+			    vma->vm_start,
+			    (unsigned long long)pfn << PAGE_SHIFT);
+		break;
+
 	default:
 		return -EINVAL;
 	}
@@ -1758,6 +2076,32 @@ static void destroy_dev_resources(struct mlx5_ib_resources *devr)
 	mlx5_ib_dealloc_pd(devr->p0);
 }
 
+static u32 get_core_cap_flags(struct ib_device *ibdev)
+{
+	struct mlx5_ib_dev *dev = to_mdev(ibdev);
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
+	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
+	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
+	u32 ret = 0;
+
+	if (ll == IB_LINK_LAYER_INFINIBAND)
+		return RDMA_CORE_PORT_IBA_IB;
+
+	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
+		return 0;
+
+	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
+		return 0;
+
+	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
+		ret |= RDMA_CORE_PORT_IBA_ROCE;
+
+	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
+		ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+
+	return ret;
+}
+
 static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
 			       struct ib_port_immutable *immutable)
 {
@@ -1770,20 +2114,50 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
 
 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	immutable->gid_tbl_len = attr.gid_tbl_len;
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
+	immutable->core_cap_flags = get_core_cap_flags(ibdev);
 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 
 	return 0;
 }
 
+static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
+{
+	int err;
+
+	dev->roce.nb.notifier_call = mlx5_netdev_event;
+	err = register_netdevice_notifier(&dev->roce.nb);
+	if (err)
+		return err;
+
+	err = mlx5_nic_vport_enable_roce(dev->mdev);
+	if (err)
+		goto err_unregister_netdevice_notifier;
+
+	return 0;
+
+err_unregister_netdevice_notifier:
+	unregister_netdevice_notifier(&dev->roce.nb);
+	return err;
+}
+
+static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
+{
+	mlx5_nic_vport_disable_roce(dev->mdev);
+	unregister_netdevice_notifier(&dev->roce.nb);
+}
+
 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_ib_dev *dev;
+	enum rdma_link_layer ll;
+	int port_type_cap;
 	int err;
 	int i;
 
-	/* don't create IB instance over Eth ports, no RoCE yet! */
-	if (MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH)
+	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
+	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
+
+	if ((ll == IB_LINK_LAYER_ETHERNET) && !MLX5_CAP_GEN(mdev, roce))
 		return NULL;
 
 	printk_once(KERN_INFO "%s", mlx5_version);
@@ -1794,6 +2168,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	dev->mdev = mdev;
 
+	rwlock_init(&dev->roce.netdev_lock);
 	err = get_port_caps(dev);
 	if (err)
 		goto err_dealloc;
@@ -1843,7 +2218,12 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	dev->ib_dev.query_device	= mlx5_ib_query_device;
 	dev->ib_dev.query_port		= mlx5_ib_query_port;
+	dev->ib_dev.get_link_layer	= mlx5_ib_port_link_layer;
+	if (ll == IB_LINK_LAYER_ETHERNET)
+		dev->ib_dev.get_netdev	= mlx5_ib_get_netdev;
 	dev->ib_dev.query_gid		= mlx5_ib_query_gid;
+	dev->ib_dev.add_gid		= mlx5_ib_add_gid;
+	dev->ib_dev.del_gid		= mlx5_ib_del_gid;
 	dev->ib_dev.query_pkey		= mlx5_ib_query_pkey;
 	dev->ib_dev.modify_device	= mlx5_ib_modify_device;
 	dev->ib_dev.modify_port		= mlx5_ib_modify_port;
@@ -1893,7 +2273,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
 	}
 
-	if (mlx5_ib_port_link_layer(&dev->ib_dev) ==
+	if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
 	    IB_LINK_LAYER_ETHERNET) {
 		dev->ib_dev.create_flow	= mlx5_ib_create_flow;
 		dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
@@ -1908,9 +2288,15 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	mutex_init(&dev->flow_db.lock);
 	mutex_init(&dev->cap_mask_mutex);
 
+	if (ll == IB_LINK_LAYER_ETHERNET) {
+		err = mlx5_enable_roce(dev);
+		if (err)
+			goto err_dealloc;
+	}
+
 	err = create_dev_resources(&dev->devr);
 	if (err)
-		goto err_dealloc;
+		goto err_disable_roce;
 
 	err = mlx5_ib_odp_init_one(dev);
 	if (err)
@@ -1947,6 +2333,10 @@ err_odp:
 err_rsrc:
 	destroy_dev_resources(&dev->devr);
 
+err_disable_roce:
+	if (ll == IB_LINK_LAYER_ETHERNET)
+		mlx5_disable_roce(dev);
+
 err_dealloc:
 	ib_dealloc_device((struct ib_device *)dev);
 
@@ -1956,11 +2346,14 @@ err_dealloc:
 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 {
 	struct mlx5_ib_dev *dev = context;
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
 
 	ib_unregister_device(&dev->ib_dev);
 	destroy_umrc_res(dev);
 	mlx5_ib_odp_remove_one(dev);
 	destroy_dev_resources(&dev->devr);
+	if (ll == IB_LINK_LAYER_ETHERNET)
+		mlx5_disable_roce(dev);
 	ib_dealloc_device(&dev->ib_dev);
 }
 
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 1474cccd1e0f..d2b9737baa36 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -42,6 +42,7 @@
 #include <linux/mlx5/qp.h>
 #include <linux/mlx5/srq.h>
 #include <linux/types.h>
+#include <linux/mlx5/transobj.h>
 
 #define mlx5_ib_dbg(dev, format, arg...)				\
 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
@@ -55,6 +56,11 @@ pr_err("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
 pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
 	__LINE__, current->pid, ##arg)
 
+#define field_avail(type, fld, sz) (offsetof(type, fld) +		\
+				    sizeof(((type *)0)->fld) <= (sz))
+#define MLX5_IB_DEFAULT_UIDX 0xffffff
+#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
+
 enum {
 	MLX5_IB_MMAP_CMD_SHIFT	= 8,
 	MLX5_IB_MMAP_CMD_MASK	= 0xff,
@@ -62,7 +68,9 @@ enum {
 
 enum mlx5_ib_mmap_cmd {
 	MLX5_IB_MMAP_REGULAR_PAGE		= 0,
-	MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES	= 1, /* always last */
+	MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES	= 1,
+	/* 5 is chosen in order to be compatible with old versions of libmlx5 */
+	MLX5_IB_MMAP_CORE_CLOCK			= 5,
 };
 
 enum {
@@ -85,6 +93,15 @@ enum mlx5_ib_mad_ifc_flags {
 	MLX5_MAD_IFC_NET_VIEW		= 4,
 };
 
+enum {
+	MLX5_CROSS_CHANNEL_UUAR         = 0,
+};
+
+enum {
+	MLX5_CQE_VERSION_V0,
+	MLX5_CQE_VERSION_V1,
+};
+
 struct mlx5_ib_ucontext {
 	struct ib_ucontext	ibucontext;
 	struct list_head	db_page_list;
@@ -93,6 +110,9 @@ struct mlx5_ib_ucontext {
 	 */
 	struct mutex		db_page_mutex;
 	struct mlx5_uuar_info	uuari;
+	u8			cqe_version;
+	/* Transport Domain number */
+	u32			tdn;
 };
 
 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
@@ -201,47 +221,70 @@ struct mlx5_ib_pfault {
 	struct mlx5_pagefault	mpfault;
 };
 
+struct mlx5_ib_ubuffer {
+	struct ib_umem	       *umem;
+	int			buf_size;
+	u64			buf_addr;
+};
+
+struct mlx5_ib_qp_base {
+	struct mlx5_ib_qp	*container_mibqp;
+	struct mlx5_core_qp	mqp;
+	struct mlx5_ib_ubuffer	ubuffer;
+};
+
+struct mlx5_ib_qp_trans {
+	struct mlx5_ib_qp_base	base;
+	u16			xrcdn;
+	u8			alt_port;
+	u8			atomic_rd_en;
+	u8			resp_depth;
+};
+
 struct mlx5_ib_rq {
+	struct mlx5_ib_qp_base base;
+	struct mlx5_ib_wq	*rq;
+	struct mlx5_ib_ubuffer	ubuffer;
+	struct mlx5_db		*doorbell;
 	u32			tirn;
+	u8			state;
+};
+
+struct mlx5_ib_sq {
+	struct mlx5_ib_qp_base base;
+	struct mlx5_ib_wq	*sq;
+	struct mlx5_ib_ubuffer  ubuffer;
+	struct mlx5_db		*doorbell;
+	u32			tisn;
+	u8			state;
 };
 
 struct mlx5_ib_raw_packet_qp {
+	struct mlx5_ib_sq sq;
 	struct mlx5_ib_rq rq;
 };
 
 struct mlx5_ib_qp {
 	struct ib_qp		ibqp;
 	union {
-		struct mlx5_core_qp		mqp;
-		struct mlx5_ib_raw_packet_qp	raw_packet_qp;
+		struct mlx5_ib_qp_trans trans_qp;
+		struct mlx5_ib_raw_packet_qp raw_packet_qp;
 	};
-
 	struct mlx5_buf		buf;
 
 	struct mlx5_db		db;
 	struct mlx5_ib_wq	rq;
 
-	u32			doorbell_qpn;
 	u8			sq_signal_bits;
 	u8			fm_cache;
-	int			sq_max_wqes_per_wr;
-	int			sq_spare_wqes;
 	struct mlx5_ib_wq	sq;
 
-	struct ib_umem	       *umem;
-	int			buf_size;
-
 	/* serialize qp state modifications
 	 */
 	struct mutex		mutex;
-	u16			xrcdn;
 	u32			flags;
 	u8			port;
-	u8			alt_port;
-	u8			atomic_rd_en;
-	u8			resp_depth;
 	u8			state;
-	int			mlx_type;
 	int			wq_sig;
 	int			scat_cqe;
 	int			max_inline_data;
@@ -284,6 +327,9 @@ struct mlx5_ib_cq_buf {
 enum mlx5_ib_qp_flags {
 	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = 1 << 0,
 	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 << 1,
+	MLX5_IB_QP_CROSS_CHANNEL		= 1 << 2,
+	MLX5_IB_QP_MANAGED_SEND			= 1 << 3,
+	MLX5_IB_QP_MANAGED_RECV			= 1 << 4,
 };
 
 struct mlx5_umr_wr {
@@ -326,6 +372,7 @@ struct mlx5_ib_cq {
 	struct mlx5_ib_cq_buf  *resize_buf;
 	struct ib_umem	       *resize_umem;
 	int			cqe_size;
+	u32			create_flags;
 };
 
 struct mlx5_ib_srq {
@@ -449,9 +496,19 @@ struct mlx5_ib_resources {
 	struct ib_srq	*s1;
 };
 
+struct mlx5_roce {
+	/* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
+	 * netdev pointer
+	 */
+	rwlock_t		netdev_lock;
+	struct net_device	*netdev;
+	struct notifier_block	nb;
+};
+
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
+	struct mlx5_roce		roce;
 	MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
 	int				num_ports;
 	/* serialize update of capability mask
@@ -498,7 +555,7 @@ static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
 
 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
 {
-	return container_of(mqp, struct mlx5_ib_qp, mqp);
+	return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
 }
 
 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmr)
@@ -550,8 +607,6 @@ void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
 		 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
 		 const void *in_mad, void *response_mad);
-struct ib_ah *create_ib_ah(struct ib_ah_attr *ah_attr,
-			   struct mlx5_ib_ah *ah);
 struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
 int mlx5_ib_destroy_ah(struct ib_ah *ah);
@@ -578,7 +633,8 @@ int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		      struct ib_recv_wr **bad_wr);
 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
-			  void *buffer, u32 length);
+			  void *buffer, u32 length,
+			  struct mlx5_ib_qp_base *base);
 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
 				struct ib_ucontext *context,
@@ -680,6 +736,9 @@ static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp)  {}
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
+			       int index);
+
 static inline void init_query_mad(struct ib_smp *mad)
 {
 	mad->base_version  = 1;
@@ -705,4 +764,28 @@ static inline int is_qp1(enum ib_qp_type qp_type)
 #define MLX5_MAX_UMR_SHIFT 16
 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
 
+static inline u32 check_cq_create_flags(u32 flags)
+{
+	/*
+	 * It returns non-zero value for unsupported CQ
+	 * create flags, otherwise it returns zero.
+	 */
+	return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
+			  IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
+}
+
+static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
+				     u32 *user_index)
+{
+	if (cqe_version) {
+		if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
+		    (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
+			return -EINVAL;
+		*user_index = cmd_uidx;
+	} else {
+		*user_index = MLX5_IB_DEFAULT_UIDX;
+	}
+
+	return 0;
+}
 #endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index aa8391e75385..b8d76361a48d 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -153,14 +153,16 @@ static struct mlx5_ib_mr *mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev *dev,
 
 static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp,
 				      struct mlx5_ib_pfault *pfault,
-				      int error) {
+				      int error)
+{
 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);
-	int ret = mlx5_core_page_fault_resume(dev->mdev, qp->mqp.qpn,
+	u32 qpn = qp->trans_qp.base.mqp.qpn;
+	int ret = mlx5_core_page_fault_resume(dev->mdev,
+					      qpn,
 					      pfault->mpfault.flags,
 					      error);
 	if (ret)
-		pr_err("Failed to resolve the page fault on QP 0x%x\n",
-		       qp->mqp.qpn);
+		pr_err("Failed to resolve the page fault on QP 0x%x\n", qpn);
 }
 
 /*
@@ -391,6 +393,7 @@ static int mlx5_ib_mr_initiator_pfault_handler(
 #if defined(DEBUG)
 	u32 ctrl_wqe_index, ctrl_qpn;
 #endif
+	u32 qpn = qp->trans_qp.base.mqp.qpn;
 
 	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 	if (ds * MLX5_WQE_DS_UNITS > wqe_length) {
@@ -401,7 +404,7 @@ static int mlx5_ib_mr_initiator_pfault_handler(
 
 	if (ds == 0) {
 		mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
-			    wqe_index, qp->mqp.qpn);
+			    wqe_index, qpn);
 		return -EFAULT;
 	}
 
@@ -411,16 +414,16 @@ static int mlx5_ib_mr_initiator_pfault_handler(
 			MLX5_WQE_CTRL_WQE_INDEX_SHIFT;
 	if (wqe_index != ctrl_wqe_index) {
 		mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n",
-			    wqe_index, qp->mqp.qpn,
+			    wqe_index, qpn,
 			    ctrl_wqe_index);
 		return -EFAULT;
 	}
 
 	ctrl_qpn = (be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_QPN_MASK) >>
 		MLX5_WQE_CTRL_QPN_SHIFT;
-	if (qp->mqp.qpn != ctrl_qpn) {
+	if (qpn != ctrl_qpn) {
 		mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n",
-			    wqe_index, qp->mqp.qpn,
+			    wqe_index, qpn,
 			    ctrl_qpn);
 		return -EFAULT;
 	}
@@ -537,6 +540,7 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_qp *qp,
 	int resume_with_error = 0;
 	u16 wqe_index = pfault->mpfault.wqe.wqe_index;
 	int requestor = pfault->mpfault.flags & MLX5_PFAULT_REQUESTOR;
+	u32 qpn = qp->trans_qp.base.mqp.qpn;
 
 	buffer = (char *)__get_free_page(GFP_KERNEL);
 	if (!buffer) {
@@ -546,10 +550,10 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_qp *qp,
 	}
 
 	ret = mlx5_ib_read_user_wqe(qp, requestor, wqe_index, buffer,
-				    PAGE_SIZE);
+				    PAGE_SIZE, &qp->trans_qp.base);
 	if (ret < 0) {
 		mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%x, wqe_index=%x, qpn=%x\n",
-			    -ret, wqe_index, qp->mqp.qpn);
+			    -ret, wqe_index, qpn);
 		resume_with_error = 1;
 		goto resolve_page_fault;
 	}
@@ -586,7 +590,8 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_qp *qp,
 resolve_page_fault:
 	mlx5_ib_page_fault_resume(qp, pfault, resume_with_error);
 	mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, flags: 0x%x\n",
-		    qp->mqp.qpn, resume_with_error, pfault->mpfault.flags);
+		    qpn, resume_with_error,
+		    pfault->mpfault.flags);
 
 	free_page((unsigned long)buffer);
 }
@@ -753,7 +758,7 @@ void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp)
 	qp->disable_page_faults = 1;
 	spin_lock_init(&qp->disable_page_faults_lock);
 
-	qp->mqp.pfault_handler	= mlx5_ib_pfault_handler;
+	qp->trans_qp.base.mqp.pfault_handler = mlx5_ib_pfault_handler;
 
 	for (i = 0; i < MLX5_IB_PAGEFAULT_CONTEXTS; ++i)
 		INIT_WORK(&qp->pagefaults[i].work, mlx5_ib_qp_pfault_action);
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 307bdbca8938..8fb9c27485e1 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -32,6 +32,8 @@
 
 #include <linux/module.h>
 #include <rdma/ib_umem.h>
+#include <rdma/ib_cache.h>
+#include <rdma/ib_user_verbs.h>
 #include "mlx5_ib.h"
 #include "user.h"
 
@@ -114,14 +116,15 @@ void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n)
  * Return: the number of bytes copied, or an error code.
  */
 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
-			  void *buffer, u32 length)
+			  void *buffer, u32 length,
+			  struct mlx5_ib_qp_base *base)
 {
 	struct ib_device *ibdev = qp->ibqp.device;
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq;
 	size_t offset;
 	size_t wq_end;
-	struct ib_umem *umem = qp->umem;
+	struct ib_umem *umem = base->ubuffer.umem;
 	u32 first_copy_length;
 	int wqe_length;
 	int ret;
@@ -172,8 +175,10 @@ static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
 	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 	struct ib_event event;
 
-	if (type == MLX5_EVENT_TYPE_PATH_MIG)
-		to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
+	if (type == MLX5_EVENT_TYPE_PATH_MIG) {
+		/* This event is only valid for trans_qps */
+		to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
+	}
 
 	if (ibqp->event_handler) {
 		event.device     = ibqp->device;
@@ -366,7 +371,9 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 
 static int set_user_buf_size(struct mlx5_ib_dev *dev,
 			    struct mlx5_ib_qp *qp,
-			    struct mlx5_ib_create_qp *ucmd)
+			    struct mlx5_ib_create_qp *ucmd,
+			    struct mlx5_ib_qp_base *base,
+			    struct ib_qp_init_attr *attr)
 {
 	int desc_sz = 1 << qp->sq.wqe_shift;
 
@@ -391,8 +398,13 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev,
 		return -EINVAL;
 	}
 
-	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
-		(qp->sq.wqe_cnt << 6);
+	if (attr->qp_type == IB_QPT_RAW_PACKET) {
+		base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
+		qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
+	} else {
+		base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
+					 (qp->sq.wqe_cnt << 6);
+	}
 
 	return 0;
 }
@@ -578,8 +590,8 @@ static int to_mlx5_st(enum ib_qp_type type)
 	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;
 	case IB_QPT_GSI:		return MLX5_QP_ST_QP1;
 	case IB_QPT_RAW_IPV6:		return MLX5_QP_ST_RAW_IPV6;
-	case IB_QPT_RAW_ETHERTYPE:	return MLX5_QP_ST_RAW_ETHERTYPE;
 	case IB_QPT_RAW_PACKET:
+	case IB_QPT_RAW_ETHERTYPE:	return MLX5_QP_ST_RAW_ETHERTYPE;
 	case IB_QPT_MAX:
 	default:		return -EINVAL;
 	}
@@ -590,13 +602,51 @@ static int uuarn_to_uar_index(struct mlx5_uuar_info *uuari, int uuarn)
 	return uuari->uars[uuarn / MLX5_BF_REGS_PER_PAGE].index;
 }
 
+static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
+			    struct ib_pd *pd,
+			    unsigned long addr, size_t size,
+			    struct ib_umem **umem,
+			    int *npages, int *page_shift, int *ncont,
+			    u32 *offset)
+{
+	int err;
+
+	*umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0);
+	if (IS_ERR(*umem)) {
+		mlx5_ib_dbg(dev, "umem_get failed\n");
+		return PTR_ERR(*umem);
+	}
+
+	mlx5_ib_cont_pages(*umem, addr, npages, page_shift, ncont, NULL);
+
+	err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
+	if (err) {
+		mlx5_ib_warn(dev, "bad offset\n");
+		goto err_umem;
+	}
+
+	mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
+		    addr, size, *npages, *page_shift, *ncont, *offset);
+
+	return 0;
+
+err_umem:
+	ib_umem_release(*umem);
+	*umem = NULL;
+
+	return err;
+}
+
 static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 			  struct mlx5_ib_qp *qp, struct ib_udata *udata,
+			  struct ib_qp_init_attr *attr,
 			  struct mlx5_create_qp_mbox_in **in,
-			  struct mlx5_ib_create_qp_resp *resp, int *inlen)
+			  struct mlx5_ib_create_qp_resp *resp, int *inlen,
+			  struct mlx5_ib_qp_base *base)
 {
 	struct mlx5_ib_ucontext *context;
 	struct mlx5_ib_create_qp ucmd;
+	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 	int page_shift = 0;
 	int uar_index;
 	int npages;
@@ -615,18 +665,23 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	/*
 	 * TBD: should come from the verbs when we have the API
 	 */
-	uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);
-	if (uuarn < 0) {
-		mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
-		mlx5_ib_dbg(dev, "reverting to medium latency\n");
-		uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM);
+	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+		/* In CROSS_CHANNEL CQ and QP must use the same UAR */
+		uuarn = MLX5_CROSS_CHANNEL_UUAR;
+	else {
+		uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);
 		if (uuarn < 0) {
-			mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n");
-			mlx5_ib_dbg(dev, "reverting to high latency\n");
-			uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW);
+			mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
+			mlx5_ib_dbg(dev, "reverting to medium latency\n");
+			uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM);
 			if (uuarn < 0) {
-				mlx5_ib_warn(dev, "uuar allocation failed\n");
-				return uuarn;
+				mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n");
+				mlx5_ib_dbg(dev, "reverting to high latency\n");
+				uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW);
+				if (uuarn < 0) {
+					mlx5_ib_warn(dev, "uuar allocation failed\n");
+					return uuarn;
+				}
 			}
 		}
 	}
@@ -638,32 +693,20 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 
-	err = set_user_buf_size(dev, qp, &ucmd);
+	err = set_user_buf_size(dev, qp, &ucmd, base, attr);
 	if (err)
 		goto err_uuar;
 
-	if (ucmd.buf_addr && qp->buf_size) {
-		qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
-				       qp->buf_size, 0, 0);
-		if (IS_ERR(qp->umem)) {
-			mlx5_ib_dbg(dev, "umem_get failed\n");
-			err = PTR_ERR(qp->umem);
+	if (ucmd.buf_addr && ubuffer->buf_size) {
+		ubuffer->buf_addr = ucmd.buf_addr;
+		err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr,
+				       ubuffer->buf_size,
+				       &ubuffer->umem, &npages, &page_shift,
+				       &ncont, &offset);
+		if (err)
 			goto err_uuar;
-		}
 	} else {
-		qp->umem = NULL;
-	}
-
-	if (qp->umem) {
-		mlx5_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift,
-				   &ncont, NULL);
-		err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset);
-		if (err) {
-			mlx5_ib_warn(dev, "bad offset\n");
-			goto err_umem;
-		}
-		mlx5_ib_dbg(dev, "addr 0x%llx, size %d, npages %d, page_shift %d, ncont %d, offset %d\n",
-			    ucmd.buf_addr, qp->buf_size, npages, page_shift, ncont, offset);
+		ubuffer->umem = NULL;
 	}
 
 	*inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont;
@@ -672,8 +715,9 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		err = -ENOMEM;
 		goto err_umem;
 	}
-	if (qp->umem)
-		mlx5_ib_populate_pas(dev, qp->umem, page_shift, (*in)->pas, 0);
+	if (ubuffer->umem)
+		mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift,
+				     (*in)->pas, 0);
 	(*in)->ctx.log_pg_sz_remote_qpn =
 		cpu_to_be32((page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24);
 	(*in)->ctx.params2 = cpu_to_be32(offset << 6);
@@ -704,29 +748,31 @@ err_free:
 	kvfree(*in);
 
 err_umem:
-	if (qp->umem)
-		ib_umem_release(qp->umem);
+	if (ubuffer->umem)
+		ib_umem_release(ubuffer->umem);
 
 err_uuar:
 	free_uuar(&context->uuari, uuarn);
 	return err;
 }
 
-static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp)
+static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp,
+			    struct mlx5_ib_qp_base *base)
 {
 	struct mlx5_ib_ucontext *context;
 
 	context = to_mucontext(pd->uobject->context);
 	mlx5_ib_db_unmap_user(context, &qp->db);
-	if (qp->umem)
-		ib_umem_release(qp->umem);
+	if (base->ubuffer.umem)
+		ib_umem_release(base->ubuffer.umem);
 	free_uuar(&context->uuari, qp->uuarn);
 }
 
 static int create_kernel_qp(struct mlx5_ib_dev *dev,
 			    struct ib_qp_init_attr *init_attr,
 			    struct mlx5_ib_qp *qp,
-			    struct mlx5_create_qp_mbox_in **in, int *inlen)
+			    struct mlx5_create_qp_mbox_in **in, int *inlen,
+			    struct mlx5_ib_qp_base *base)
 {
 	enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
 	struct mlx5_uuar_info *uuari;
@@ -758,9 +804,9 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 
 	qp->rq.offset = 0;
 	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
-	qp->buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
+	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
 
-	err = mlx5_buf_alloc(dev->mdev, qp->buf_size, &qp->buf);
+	err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
 	if (err) {
 		mlx5_ib_dbg(dev, "err %d\n", err);
 		goto err_uuar;
@@ -853,19 +899,304 @@ static int is_connected(enum ib_qp_type qp_type)
 	return 0;
 }
 
+static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
+				    struct mlx5_ib_sq *sq, u32 tdn)
+{
+	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
+	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(tisc, tisc, transport_domain, tdn);
+
+	return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
+}
+
+static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
+				      struct mlx5_ib_sq *sq)
+{
+	mlx5_core_destroy_tis(dev->mdev, sq->tisn);
+}
+
+static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
+				   struct mlx5_ib_sq *sq, void *qpin,
+				   struct ib_pd *pd)
+{
+	struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
+	__be64 *pas;
+	void *in;
+	void *sqc;
+	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
+	void *wq;
+	int inlen;
+	int err;
+	int page_shift = 0;
+	int npages;
+	int ncont = 0;
+	u32 offset = 0;
+
+	err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size,
+			       &sq->ubuffer.umem, &npages, &page_shift,
+			       &ncont, &offset);
+	if (err)
+		return err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		err = -ENOMEM;
+		goto err_umem;
+	}
+
+	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
+	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
+	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
+	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
+	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
+	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
+	MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
+
+	wq = MLX5_ADDR_OF(sqc, sqc, wq);
+	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
+	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
+	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
+	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
+	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
+	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
+	MLX5_SET(wq, wq, log_wq_pg_sz,  page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+	MLX5_SET(wq, wq, page_offset, offset);
+
+	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
+	mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
+
+	err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp);
+
+	kvfree(in);
+
+	if (err)
+		goto err_umem;
+
+	return 0;
+
+err_umem:
+	ib_umem_release(sq->ubuffer.umem);
+	sq->ubuffer.umem = NULL;
+
+	return err;
+}
+
+static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
+				     struct mlx5_ib_sq *sq)
+{
+	mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
+	ib_umem_release(sq->ubuffer.umem);
+}
+
+static int get_rq_pas_size(void *qpc)
+{
+	u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
+	u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
+	u32 log_rq_size   = MLX5_GET(qpc, qpc, log_rq_size);
+	u32 page_offset   = MLX5_GET(qpc, qpc, page_offset);
+	u32 po_quanta	  = 1 << (log_page_size - 6);
+	u32 rq_sz	  = 1 << (log_rq_size + 4 + log_rq_stride);
+	u32 page_size	  = 1 << log_page_size;
+	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
+	u32 rq_num_pas	  = (rq_sz_po + page_size - 1) / page_size;
+
+	return rq_num_pas * sizeof(u64);
+}
+
+static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
+				   struct mlx5_ib_rq *rq, void *qpin)
+{
+	__be64 *pas;
+	__be64 *qp_pas;
+	void *in;
+	void *rqc;
+	void *wq;
+	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
+	int inlen;
+	int err;
+	u32 rq_pas_size = get_rq_pas_size(qpc);
+
+	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
+	MLX5_SET(rqc, rqc, vsd, 1);
+	MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
+	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
+	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
+	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
+	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
+
+	wq = MLX5_ADDR_OF(rqc, rqc, wq);
+	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
+	MLX5_SET(wq, wq, end_padding_mode,
+		 MLX5_GET64(qpc, qpc, end_padding_mode));
+	MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
+	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
+	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
+	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
+	MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
+	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
+
+	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
+	qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
+	memcpy(pas, qp_pas, rq_pas_size);
+
+	err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp);
+
+	kvfree(in);
+
+	return err;
+}
+
+static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
+				     struct mlx5_ib_rq *rq)
+{
+	mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp);
+}
+
+static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
+				    struct mlx5_ib_rq *rq, u32 tdn)
+{
+	u32 *in;
+	void *tirc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
+	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
+	MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
+	MLX5_SET(tirc, tirc, transport_domain, tdn);
+
+	err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
+
+	kvfree(in);
+
+	return err;
+}
+
+static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
+				      struct mlx5_ib_rq *rq)
+{
+	mlx5_core_destroy_tir(dev->mdev, rq->tirn);
+}
+
+static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+				struct mlx5_create_qp_mbox_in *in,
+				struct ib_pd *pd)
+{
+	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
+	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
+	struct ib_uobject *uobj = pd->uobject;
+	struct ib_ucontext *ucontext = uobj->context;
+	struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
+	int err;
+	u32 tdn = mucontext->tdn;
+
+	if (qp->sq.wqe_cnt) {
+		err = create_raw_packet_qp_tis(dev, sq, tdn);
+		if (err)
+			return err;
+
+		err = create_raw_packet_qp_sq(dev, sq, in, pd);
+		if (err)
+			goto err_destroy_tis;
+
+		sq->base.container_mibqp = qp;
+	}
+
+	if (qp->rq.wqe_cnt) {
+		err = create_raw_packet_qp_rq(dev, rq, in);
+		if (err)
+			goto err_destroy_sq;
+
+		rq->base.container_mibqp = qp;
+
+		err = create_raw_packet_qp_tir(dev, rq, tdn);
+		if (err)
+			goto err_destroy_rq;
+	}
+
+	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
+						     rq->base.mqp.qpn;
+
+	return 0;
+
+err_destroy_rq:
+	destroy_raw_packet_qp_rq(dev, rq);
+err_destroy_sq:
+	if (!qp->sq.wqe_cnt)
+		return err;
+	destroy_raw_packet_qp_sq(dev, sq);
+err_destroy_tis:
+	destroy_raw_packet_qp_tis(dev, sq);
+
+	return err;
+}
+
+static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
+				  struct mlx5_ib_qp *qp)
+{
+	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
+	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
+
+	if (qp->rq.wqe_cnt) {
+		destroy_raw_packet_qp_tir(dev, rq);
+		destroy_raw_packet_qp_rq(dev, rq);
+	}
+
+	if (qp->sq.wqe_cnt) {
+		destroy_raw_packet_qp_sq(dev, sq);
+		destroy_raw_packet_qp_tis(dev, sq);
+	}
+}
+
+static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
+				    struct mlx5_ib_raw_packet_qp *raw_packet_qp)
+{
+	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
+
+	sq->sq = &qp->sq;
+	rq->rq = &qp->rq;
+	sq->doorbell = &qp->db;
+	rq->doorbell = &qp->db;
+}
+
 static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 			    struct ib_qp_init_attr *init_attr,
 			    struct ib_udata *udata, struct mlx5_ib_qp *qp)
 {
 	struct mlx5_ib_resources *devr = &dev->devr;
 	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_ib_qp_base *base;
 	struct mlx5_ib_create_qp_resp resp;
 	struct mlx5_create_qp_mbox_in *in;
 	struct mlx5_ib_create_qp ucmd;
 	int inlen = sizeof(*in);
 	int err;
+	u32 uidx = MLX5_IB_DEFAULT_UIDX;
+	void *qpc;
+
+	base = init_attr->qp_type == IB_QPT_RAW_PACKET ?
+	       &qp->raw_packet_qp.rq.base :
+	       &qp->trans_qp.base;
 
-	mlx5_ib_odp_create_qp(qp);
+	if (init_attr->qp_type != IB_QPT_RAW_PACKET)
+		mlx5_ib_odp_create_qp(qp);
 
 	mutex_init(&qp->mutex);
 	spin_lock_init(&qp->sq.lock);
@@ -880,6 +1211,21 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		}
 	}
 
+	if (init_attr->create_flags &
+			(IB_QP_CREATE_CROSS_CHANNEL |
+			 IB_QP_CREATE_MANAGED_SEND |
+			 IB_QP_CREATE_MANAGED_RECV)) {
+		if (!MLX5_CAP_GEN(mdev, cd)) {
+			mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
+			return -EINVAL;
+		}
+		if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
+			qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;
+		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
+			qp->flags |= MLX5_IB_QP_MANAGED_SEND;
+		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
+			qp->flags |= MLX5_IB_QP_MANAGED_RECV;
+	}
 	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
 		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
 
@@ -889,6 +1235,11 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 			return -EFAULT;
 		}
 
+		err = get_qp_user_index(to_mucontext(pd->uobject->context),
+					&ucmd, udata->inlen, &uidx);
+		if (err)
+			return err;
+
 		qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
 		qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
 	} else {
@@ -918,11 +1269,13 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 					    ucmd.sq_wqe_count, max_wqes);
 				return -EINVAL;
 			}
-			err = create_user_qp(dev, pd, qp, udata, &in, &resp, &inlen);
+			err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
+					     &resp, &inlen, base);
 			if (err)
 				mlx5_ib_dbg(dev, "err %d\n", err);
 		} else {
-			err = create_kernel_qp(dev, init_attr, qp, &in, &inlen);
+			err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
+					       base);
 			if (err)
 				mlx5_ib_dbg(dev, "err %d\n", err);
 		}
@@ -954,6 +1307,13 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
 		in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_BLOCK_MCAST);
 
+	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_MASTER);
+	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
+		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_SEND);
+	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
+		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_RECV);
+
 	if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
 		int rcqe_sz;
 		int scqe_sz;
@@ -1018,26 +1378,35 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 	in->ctx.db_rec_addr = cpu_to_be64(qp->db.dma);
 
-	err = mlx5_core_create_qp(dev->mdev, &qp->mqp, in, inlen);
+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1) {
+		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
+		/* 0xffffff means we ask to work with cqe version 0 */
+		MLX5_SET(qpc, qpc, user_index, uidx);
+	}
+
+	if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
+		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
+		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
+		err = create_raw_packet_qp(dev, qp, in, pd);
+	} else {
+		err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
+	}
+
 	if (err) {
 		mlx5_ib_dbg(dev, "create qp failed\n");
 		goto err_create;
 	}
 
 	kvfree(in);
-	/* Hardware wants QPN written in big-endian order (after
-	 * shifting) for send doorbell.  Precompute this value to save
-	 * a little bit when posting sends.
-	 */
-	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
 
-	qp->mqp.event = mlx5_ib_qp_event;
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 
 	return 0;
 
 err_create:
 	if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(pd, qp);
+		destroy_qp_user(pd, qp, base);
 	else if (qp->create_type == MLX5_QP_KERNEL)
 		destroy_qp_kernel(dev, qp);
 
@@ -1129,11 +1498,11 @@ static void get_cqs(struct mlx5_ib_qp *qp,
 	case IB_QPT_UD:
 	case IB_QPT_RAW_IPV6:
 	case IB_QPT_RAW_ETHERTYPE:
+	case IB_QPT_RAW_PACKET:
 		*send_cq = to_mcq(qp->ibqp.send_cq);
 		*recv_cq = to_mcq(qp->ibqp.recv_cq);
 		break;
 
-	case IB_QPT_RAW_PACKET:
 	case IB_QPT_MAX:
 	default:
 		*send_cq = NULL;
@@ -1142,45 +1511,66 @@ static void get_cqs(struct mlx5_ib_qp *qp,
 	}
 }
 
+static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+				u16 operation);
+
 static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 {
 	struct mlx5_ib_cq *send_cq, *recv_cq;
+	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 	struct mlx5_modify_qp_mbox_in *in;
 	int err;
 
+	base = qp->ibqp.qp_type == IB_QPT_RAW_PACKET ?
+	       &qp->raw_packet_qp.rq.base :
+	       &qp->trans_qp.base;
+
 	in = kzalloc(sizeof(*in), GFP_KERNEL);
 	if (!in)
 		return;
 
 	if (qp->state != IB_QPS_RESET) {
-		mlx5_ib_qp_disable_pagefaults(qp);
-		if (mlx5_core_qp_modify(dev->mdev, to_mlx5_state(qp->state),
-					MLX5_QP_STATE_RST, in, 0, &qp->mqp))
-			mlx5_ib_warn(dev, "mlx5_ib: modify QP %06x to RESET failed\n",
-				     qp->mqp.qpn);
+		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) {
+			mlx5_ib_qp_disable_pagefaults(qp);
+			err = mlx5_core_qp_modify(dev->mdev,
+						  MLX5_CMD_OP_2RST_QP, in, 0,
+						  &base->mqp);
+		} else {
+			err = modify_raw_packet_qp(dev, qp,
+						   MLX5_CMD_OP_2RST_QP);
+		}
+		if (err)
+			mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
+				     base->mqp.qpn);
 	}
 
 	get_cqs(qp, &send_cq, &recv_cq);
 
 	if (qp->create_type == MLX5_QP_KERNEL) {
 		mlx5_ib_lock_cqs(send_cq, recv_cq);
-		__mlx5_ib_cq_clean(recv_cq, qp->mqp.qpn,
+		__mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
 				   qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
 		if (send_cq != recv_cq)
-			__mlx5_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
+			__mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
+					   NULL);
 		mlx5_ib_unlock_cqs(send_cq, recv_cq);
 	}
 
-	err = mlx5_core_destroy_qp(dev->mdev, &qp->mqp);
-	if (err)
-		mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n", qp->mqp.qpn);
-	kfree(in);
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+		destroy_raw_packet_qp(dev, qp);
+	} else {
+		err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
+		if (err)
+			mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
+				     base->mqp.qpn);
+	}
 
+	kfree(in);
 
 	if (qp->create_type == MLX5_QP_KERNEL)
 		destroy_qp_kernel(dev, qp);
 	else if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(&get_pd(qp)->ibpd, qp);
+		destroy_qp_user(&get_pd(qp)->ibpd, qp, base);
 }
 
 static const char *ib_qp_type_str(enum ib_qp_type type)
@@ -1234,6 +1624,16 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 			return ERR_PTR(-EINVAL);
 		}
 		dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
+
+		if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
+			if (!pd->uobject) {
+				mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
+				return ERR_PTR(-EINVAL);
+			} else if (!to_mucontext(pd->uobject->context)->cqe_version) {
+				mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
+				return ERR_PTR(-EINVAL);
+			}
+		}
 	}
 
 	switch (init_attr->qp_type) {
@@ -1250,6 +1650,7 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 		}
 
 		/* fall through */
+	case IB_QPT_RAW_PACKET:
 	case IB_QPT_RC:
 	case IB_QPT_UC:
 	case IB_QPT_UD:
@@ -1272,19 +1673,19 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 		else if (is_qp1(init_attr->qp_type))
 			qp->ibqp.qp_num = 1;
 		else
-			qp->ibqp.qp_num = qp->mqp.qpn;
+			qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
 
 		mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
-			    qp->ibqp.qp_num, qp->mqp.qpn, to_mcq(init_attr->recv_cq)->mcq.cqn,
+			    qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
+			    to_mcq(init_attr->recv_cq)->mcq.cqn,
 			    to_mcq(init_attr->send_cq)->mcq.cqn);
 
-		qp->xrcdn = xrcdn;
+		qp->trans_qp.xrcdn = xrcdn;
 
 		break;
 
 	case IB_QPT_RAW_IPV6:
 	case IB_QPT_RAW_ETHERTYPE:
-	case IB_QPT_RAW_PACKET:
 	case IB_QPT_MAX:
 	default:
 		mlx5_ib_dbg(dev, "unsupported qp type %d\n",
@@ -1318,12 +1719,12 @@ static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_att
 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
 		dest_rd_atomic = attr->max_dest_rd_atomic;
 	else
-		dest_rd_atomic = qp->resp_depth;
+		dest_rd_atomic = qp->trans_qp.resp_depth;
 
 	if (attr_mask & IB_QP_ACCESS_FLAGS)
 		access_flags = attr->qp_access_flags;
 	else
-		access_flags = qp->atomic_rd_en;
+		access_flags = qp->trans_qp.atomic_rd_en;
 
 	if (!dest_rd_atomic)
 		access_flags &= IB_ACCESS_REMOTE_WRITE;
@@ -1360,21 +1761,42 @@ static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
 	return rate + MLX5_STAT_RATE_OFFSET;
 }
 
-static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah,
+static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
+				      struct mlx5_ib_sq *sq, u8 sl)
+{
+	void *in;
+	void *tisc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
+
+	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
+	MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
+
+	err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+
+static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			 const struct ib_ah_attr *ah,
 			 struct mlx5_qp_path *path, u8 port, int attr_mask,
 			 u32 path_flags, const struct ib_qp_attr *attr)
 {
+	enum rdma_link_layer ll = rdma_port_get_link_layer(&dev->ib_dev, port);
 	int err;
 
-	path->fl = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
-	path->free_ar = (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x80 : 0;
-
 	if (attr_mask & IB_QP_PKEY_INDEX)
 		path->pkey_index = attr->pkey_index;
 
-	path->grh_mlid	= ah->src_path_bits & 0x7f;
-	path->rlid	= cpu_to_be16(ah->dlid);
-
 	if (ah->ah_flags & IB_AH_GRH) {
 		if (ah->grh.sgid_index >=
 		    dev->mdev->port_caps[port - 1].gid_table_len) {
@@ -1383,7 +1805,27 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah,
 			       dev->mdev->port_caps[port - 1].gid_table_len);
 			return -EINVAL;
 		}
-		path->grh_mlid |= 1 << 7;
+	}
+
+	if (ll == IB_LINK_LAYER_ETHERNET) {
+		if (!(ah->ah_flags & IB_AH_GRH))
+			return -EINVAL;
+		memcpy(path->rmac, ah->dmac, sizeof(ah->dmac));
+		path->udp_sport = mlx5_get_roce_udp_sport(dev, port,
+							  ah->grh.sgid_index);
+		path->dci_cfi_prio_sl = (ah->sl & 0x7) << 4;
+	} else {
+		path->fl = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
+		path->free_ar = (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x80 :
+									0;
+		path->rlid = cpu_to_be16(ah->dlid);
+		path->grh_mlid = ah->src_path_bits & 0x7f;
+		if (ah->ah_flags & IB_AH_GRH)
+			path->grh_mlid	|= 1 << 7;
+		path->dci_cfi_prio_sl = ah->sl & 0xf;
+	}
+
+	if (ah->ah_flags & IB_AH_GRH) {
 		path->mgid_index = ah->grh.sgid_index;
 		path->hop_limit  = ah->grh.hop_limit;
 		path->tclass_flowlabel =
@@ -1401,7 +1843,10 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah,
 	if (attr_mask & IB_QP_TIMEOUT)
 		path->ackto_lt = attr->timeout << 3;
 
-	path->sl = ah->sl & 0xf;
+	if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
+		return modify_raw_packet_eth_prio(dev->mdev,
+						  &qp->raw_packet_qp.sq,
+						  ah->sl & 0xf);
 
 	return 0;
 }
@@ -1549,12 +1994,154 @@ static int ib_mask_to_mlx5_opt(int ib_mask)
 	return result;
 }
 
+static int modify_raw_packet_qp_rq(struct mlx5_core_dev *dev,
+				   struct mlx5_ib_rq *rq, int new_state)
+{
+	void *in;
+	void *rqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
+
+	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
+	MLX5_SET(rqc, rqc, state, new_state);
+
+	err = mlx5_core_modify_rq(dev, rq->base.mqp.qpn, in, inlen);
+	if (err)
+		goto out;
+
+	rq->state = new_state;
+
+out:
+	kvfree(in);
+	return err;
+}
+
+static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
+				   struct mlx5_ib_sq *sq, int new_state)
+{
+	void *in;
+	void *sqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_sq_in, in, sq_state, sq->state);
+
+	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
+	MLX5_SET(sqc, sqc, state, new_state);
+
+	err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
+	if (err)
+		goto out;
+
+	sq->state = new_state;
+
+out:
+	kvfree(in);
+	return err;
+}
+
+static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+				u16 operation)
+{
+	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
+	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
+	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	int rq_state;
+	int sq_state;
+	int err;
+
+	switch (operation) {
+	case MLX5_CMD_OP_RST2INIT_QP:
+		rq_state = MLX5_RQC_STATE_RDY;
+		sq_state = MLX5_SQC_STATE_RDY;
+		break;
+	case MLX5_CMD_OP_2ERR_QP:
+		rq_state = MLX5_RQC_STATE_ERR;
+		sq_state = MLX5_SQC_STATE_ERR;
+		break;
+	case MLX5_CMD_OP_2RST_QP:
+		rq_state = MLX5_RQC_STATE_RST;
+		sq_state = MLX5_SQC_STATE_RST;
+		break;
+	case MLX5_CMD_OP_INIT2INIT_QP:
+	case MLX5_CMD_OP_INIT2RTR_QP:
+	case MLX5_CMD_OP_RTR2RTS_QP:
+	case MLX5_CMD_OP_RTS2RTS_QP:
+		/* Nothing to do here... */
+		return 0;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	if (qp->rq.wqe_cnt) {
+		err =  modify_raw_packet_qp_rq(dev->mdev, rq, rq_state);
+		if (err)
+			return err;
+	}
+
+	if (qp->sq.wqe_cnt)
+		return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state);
+
+	return 0;
+}
+
 static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 			       const struct ib_qp_attr *attr, int attr_mask,
 			       enum ib_qp_state cur_state, enum ib_qp_state new_state)
 {
+	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
+		[MLX5_QP_STATE_RST] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
+		},
+		[MLX5_QP_STATE_INIT]  = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
+			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
+		},
+		[MLX5_QP_STATE_RTR]   = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
+		},
+		[MLX5_QP_STATE_RTS]   = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
+		},
+		[MLX5_QP_STATE_SQD] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+		},
+		[MLX5_QP_STATE_SQER] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
+		},
+		[MLX5_QP_STATE_ERR] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+		}
+	};
+
 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
 	struct mlx5_ib_qp *qp = to_mqp(ibqp);
+	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 	struct mlx5_ib_cq *send_cq, *recv_cq;
 	struct mlx5_qp_context *context;
 	struct mlx5_modify_qp_mbox_in *in;
@@ -1564,6 +2151,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	int sqd_event;
 	int mlx5_st;
 	int err;
+	u16 op;
 
 	in = kzalloc(sizeof(*in), GFP_KERNEL);
 	if (!in)
@@ -1623,7 +2211,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 		context->pri_path.port = attr->port_num;
 
 	if (attr_mask & IB_QP_AV) {
-		err = mlx5_set_path(dev, &attr->ah_attr, &context->pri_path,
+		err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
 				    attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
 				    attr_mask, 0, attr);
 		if (err)
@@ -1634,7 +2222,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 		context->pri_path.ackto_lt |= attr->timeout << 3;
 
 	if (attr_mask & IB_QP_ALT_PATH) {
-		err = mlx5_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
+		err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
+				    &context->alt_path,
 				    attr->alt_port_num, attr_mask, 0, attr);
 		if (err)
 			goto out;
@@ -1706,41 +2295,51 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	 * again to RTS, and may cause the driver and the device to get out of
 	 * sync. */
 	if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
-	    (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
+	    (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) &&
+	    (qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
 		mlx5_ib_qp_disable_pagefaults(qp);
 
+	if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
+	    !optab[mlx5_cur][mlx5_new])
+		goto out;
+
+	op = optab[mlx5_cur][mlx5_new];
 	optpar = ib_mask_to_mlx5_opt(attr_mask);
 	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
 	in->optparam = cpu_to_be32(optpar);
-	err = mlx5_core_qp_modify(dev->mdev, to_mlx5_state(cur_state),
-				  to_mlx5_state(new_state), in, sqd_event,
-				  &qp->mqp);
+
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET)
+		err = modify_raw_packet_qp(dev, qp, op);
+	else
+		err = mlx5_core_qp_modify(dev->mdev, op, in, sqd_event,
+					  &base->mqp);
 	if (err)
 		goto out;
 
-	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT &&
+	    (qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
 		mlx5_ib_qp_enable_pagefaults(qp);
 
 	qp->state = new_state;
 
 	if (attr_mask & IB_QP_ACCESS_FLAGS)
-		qp->atomic_rd_en = attr->qp_access_flags;
+		qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
-		qp->resp_depth = attr->max_dest_rd_atomic;
+		qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
 	if (attr_mask & IB_QP_PORT)
 		qp->port = attr->port_num;
 	if (attr_mask & IB_QP_ALT_PATH)
-		qp->alt_port = attr->alt_port_num;
+		qp->trans_qp.alt_port = attr->alt_port_num;
 
 	/*
 	 * If we moved a kernel QP to RESET, clean up all old CQ
 	 * entries and reinitialize the QP.
 	 */
 	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
-		mlx5_ib_cq_clean(recv_cq, qp->mqp.qpn,
+		mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
 				 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
 		if (send_cq != recv_cq)
-			mlx5_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
+			mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
 
 		qp->rq.head = 0;
 		qp->rq.tail = 0;
@@ -1765,15 +2364,21 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	enum ib_qp_state cur_state, new_state;
 	int err = -EINVAL;
 	int port;
+	enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
 
 	mutex_lock(&qp->mutex);
 
 	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
 
+	if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
+		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
+		ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
+	}
+
 	if (ibqp->qp_type != MLX5_IB_QPT_REG_UMR &&
 	    !ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask,
-				IB_LINK_LAYER_UNSPECIFIED))
+				ll))
 		goto out;
 
 	if ((attr_mask & IB_QP_PORT) &&
@@ -2570,7 +3175,7 @@ static void finish_wqe(struct mlx5_ib_qp *qp,
 
 	ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
 					     mlx5_opcode | ((u32)opmod << 24));
-	ctrl->qpn_ds = cpu_to_be32(size | (qp->mqp.qpn << 8));
+	ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
 	ctrl->fm_ce_se |= fence;
 	qp->fm_cache = next_fence;
 	if (unlikely(qp->wq_sig))
@@ -3003,7 +3608,7 @@ static void to_ib_ah_attr(struct mlx5_ib_dev *ibdev, struct ib_ah_attr *ib_ah_at
 	    ib_ah_attr->port_num > MLX5_CAP_GEN(dev, num_ports))
 		return;
 
-	ib_ah_attr->sl = path->sl & 0xf;
+	ib_ah_attr->sl = path->dci_cfi_prio_sl & 0xf;
 
 	ib_ah_attr->dlid	  = be16_to_cpu(path->rlid);
 	ib_ah_attr->src_path_bits = path->grh_mlid & 0x7f;
@@ -3021,39 +3626,153 @@ static void to_ib_ah_attr(struct mlx5_ib_dev *ibdev, struct ib_ah_attr *ib_ah_at
 	}
 }
 
-int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
-		     struct ib_qp_init_attr *qp_init_attr)
+static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
+					struct mlx5_ib_sq *sq,
+					u8 *sq_state)
+{
+	void *out;
+	void *sqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(query_sq_out);
+	out = mlx5_vzalloc(inlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_core_query_sq(dev->mdev, sq->base.mqp.qpn, out);
+	if (err)
+		goto out;
+
+	sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
+	*sq_state = MLX5_GET(sqc, sqc, state);
+	sq->state = *sq_state;
+
+out:
+	kvfree(out);
+	return err;
+}
+
+static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
+					struct mlx5_ib_rq *rq,
+					u8 *rq_state)
+{
+	void *out;
+	void *rqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(query_rq_out);
+	out = mlx5_vzalloc(inlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
+	if (err)
+		goto out;
+
+	rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
+	*rq_state = MLX5_GET(rqc, rqc, state);
+	rq->state = *rq_state;
+
+out:
+	kvfree(out);
+	return err;
+}
+
+static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
+				  struct mlx5_ib_qp *qp, u8 *qp_state)
+{
+	static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
+		[MLX5_RQC_STATE_RST] = {
+			[MLX5_SQC_STATE_RST]	= IB_QPS_RESET,
+			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
+			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE_BAD,
+			[MLX5_SQ_STATE_NA]	= IB_QPS_RESET,
+		},
+		[MLX5_RQC_STATE_RDY] = {
+			[MLX5_SQC_STATE_RST]	= MLX5_QP_STATE_BAD,
+			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
+			[MLX5_SQC_STATE_ERR]	= IB_QPS_SQE,
+			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE,
+		},
+		[MLX5_RQC_STATE_ERR] = {
+			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE_BAD,
+			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
+			[MLX5_SQC_STATE_ERR]	= IB_QPS_ERR,
+			[MLX5_SQ_STATE_NA]	= IB_QPS_ERR,
+		},
+		[MLX5_RQ_STATE_NA] = {
+			[MLX5_SQC_STATE_RST]    = IB_QPS_RESET,
+			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
+			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE,
+			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE_BAD,
+		},
+	};
+
+	*qp_state = sqrq_trans[rq_state][sq_state];
+
+	if (*qp_state == MLX5_QP_STATE_BAD) {
+		WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
+		     qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
+		     qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
+		return -EINVAL;
+	}
+
+	if (*qp_state == MLX5_QP_STATE)
+		*qp_state = qp->state;
+
+	return 0;
+}
+
+static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
+				     struct mlx5_ib_qp *qp,
+				     u8 *raw_packet_qp_state)
+{
+	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
+	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
+	int err;
+	u8 sq_state = MLX5_SQ_STATE_NA;
+	u8 rq_state = MLX5_RQ_STATE_NA;
+
+	if (qp->sq.wqe_cnt) {
+		err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
+		if (err)
+			return err;
+	}
+
+	if (qp->rq.wqe_cnt) {
+		err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
+		if (err)
+			return err;
+	}
+
+	return sqrq_state_to_qp_state(sq_state, rq_state, qp,
+				      raw_packet_qp_state);
+}
+
+static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			 struct ib_qp_attr *qp_attr)
 {
-	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
-	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 	struct mlx5_query_qp_mbox_out *outb;
 	struct mlx5_qp_context *context;
 	int mlx5_state;
 	int err = 0;
 
-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
-	/*
-	 * Wait for any outstanding page faults, in case the user frees memory
-	 * based upon this query's result.
-	 */
-	flush_workqueue(mlx5_ib_page_fault_wq);
-#endif
-
-	mutex_lock(&qp->mutex);
 	outb = kzalloc(sizeof(*outb), GFP_KERNEL);
-	if (!outb) {
-		err = -ENOMEM;
-		goto out;
-	}
+	if (!outb)
+		return -ENOMEM;
+
 	context = &outb->ctx;
-	err = mlx5_core_qp_query(dev->mdev, &qp->mqp, outb, sizeof(*outb));
+	err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
+				 sizeof(*outb));
 	if (err)
-		goto out_free;
+		goto out;
 
 	mlx5_state = be32_to_cpu(context->flags) >> 28;
 
 	qp->state		     = to_ib_qp_state(mlx5_state);
-	qp_attr->qp_state	     = qp->state;
 	qp_attr->path_mtu	     = context->mtu_msgmax >> 5;
 	qp_attr->path_mig_state	     =
 		to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
@@ -3087,6 +3806,43 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr
 	qp_attr->retry_cnt	    = (be32_to_cpu(context->params1) >> 16) & 0x7;
 	qp_attr->rnr_retry	    = (be32_to_cpu(context->params1) >> 13) & 0x7;
 	qp_attr->alt_timeout	    = context->alt_path.ackto_lt >> 3;
+
+out:
+	kfree(outb);
+	return err;
+}
+
+int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	struct mlx5_ib_qp *qp = to_mqp(ibqp);
+	int err = 0;
+	u8 raw_packet_qp_state;
+
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+	/*
+	 * Wait for any outstanding page faults, in case the user frees memory
+	 * based upon this query's result.
+	 */
+	flush_workqueue(mlx5_ib_page_fault_wq);
+#endif
+
+	mutex_lock(&qp->mutex);
+
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+		err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
+		if (err)
+			goto out;
+		qp->state = raw_packet_qp_state;
+		qp_attr->port_num = 1;
+	} else {
+		err = query_qp_attr(dev, qp, qp_attr);
+		if (err)
+			goto out;
+	}
+
+	qp_attr->qp_state	     = qp->state;
 	qp_attr->cur_qp_state	     = qp_attr->qp_state;
 	qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
 	qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
@@ -3110,12 +3866,16 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr
 	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
 		qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
 
+	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+		qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;
+	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
+		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;
+	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
+		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;
+
 	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
 		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
 
-out_free:
-	kfree(outb);
-
 out:
 	mutex_unlock(&qp->mutex);
 	return err;
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index e008505e96e9..4659256cd95e 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -78,28 +78,41 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 			   struct ib_udata *udata, int buf_size, int *inlen)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5_ib_create_srq ucmd;
+	struct mlx5_ib_create_srq ucmd = {};
 	size_t ucmdlen;
+	void *xsrqc;
 	int err;
 	int npages;
 	int page_shift;
 	int ncont;
 	u32 offset;
+	u32 uidx = MLX5_IB_DEFAULT_UIDX;
+	int drv_data = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
 
-	ucmdlen =
-		(udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
-		 sizeof(ucmd)) ? (sizeof(ucmd) -
-				  sizeof(ucmd.reserved)) : sizeof(ucmd);
+	if (drv_data < 0)
+		return -EINVAL;
+
+	ucmdlen = (drv_data < sizeof(ucmd)) ?
+		  drv_data : sizeof(ucmd);
 
 	if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
 		mlx5_ib_dbg(dev, "failed copy udata\n");
 		return -EFAULT;
 	}
 
-	if (ucmdlen == sizeof(ucmd) &&
-	    ucmd.reserved != 0)
+	if (ucmd.reserved0 || ucmd.reserved1)
 		return -EINVAL;
 
+	if (drv_data > sizeof(ucmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(ucmd),
+				 drv_data - sizeof(ucmd)))
+		return -EINVAL;
+
+	err = get_srq_user_index(to_mucontext(pd->uobject->context),
+				 &ucmd, udata->inlen, &uidx);
+	if (err)
+		return err;
+
 	srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
 
 	srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
@@ -138,6 +151,12 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 	(*in)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 	(*in)->ctx.pgoff_cqn = cpu_to_be32(offset << 26);
 
+	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1) {
+		xsrqc = MLX5_ADDR_OF(create_xrc_srq_in, *in,
+				     xrc_srq_context_entry);
+		MLX5_SET(xrc_srqc, xsrqc, user_index, uidx);
+	}
+
 	return 0;
 
 err_in:
@@ -158,6 +177,7 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
 	struct mlx5_wqe_srq_next_seg *next;
 	int page_shift;
 	int npages;
+	void *xsrqc;
 
 	err = mlx5_db_alloc(dev->mdev, &srq->db);
 	if (err) {
@@ -204,6 +224,13 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
 
 	(*in)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 
+	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1) {
+		xsrqc = MLX5_ADDR_OF(create_xrc_srq_in, *in,
+				     xrc_srq_context_entry);
+		/* 0xffffff means we ask to work with cqe version 0 */
+		MLX5_SET(xrc_srqc, xsrqc, user_index, MLX5_IB_DEFAULT_UIDX);
+	}
+
 	return 0;
 
 err_in:
diff --git a/drivers/infiniband/hw/mlx5/user.h b/drivers/infiniband/hw/mlx5/user.h
index 76fb7b927d37..b94a55404a59 100644
--- a/drivers/infiniband/hw/mlx5/user.h
+++ b/drivers/infiniband/hw/mlx5/user.h
@@ -35,6 +35,8 @@
 
 #include <linux/types.h>
 
+#include "mlx5_ib.h"
+
 enum {
 	MLX5_QP_FLAG_SIGNATURE		= 1 << 0,
 	MLX5_QP_FLAG_SCATTER_CQE	= 1 << 1,
@@ -66,7 +68,15 @@ struct mlx5_ib_alloc_ucontext_req_v2 {
 	__u32	total_num_uuars;
 	__u32	num_low_latency_uuars;
 	__u32	flags;
-	__u32	reserved;
+	__u32	comp_mask;
+	__u8	max_cqe_version;
+	__u8	reserved0;
+	__u16	reserved1;
+	__u32	reserved2;
+};
+
+enum mlx5_ib_alloc_ucontext_resp_mask {
+	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET = 1UL << 0,
 };
 
 struct mlx5_ib_alloc_ucontext_resp {
@@ -80,7 +90,13 @@ struct mlx5_ib_alloc_ucontext_resp {
 	__u32	max_recv_wr;
 	__u32	max_srq_recv_wr;
 	__u16	num_ports;
-	__u16	reserved;
+	__u16	reserved1;
+	__u32	comp_mask;
+	__u32	response_length;
+	__u8	cqe_version;
+	__u8	reserved2;
+	__u16	reserved3;
+	__u64	hca_core_clock_offset;
 };
 
 struct mlx5_ib_alloc_pd_resp {
@@ -110,7 +126,9 @@ struct mlx5_ib_create_srq {
 	__u64	buf_addr;
 	__u64	db_addr;
 	__u32	flags;
-	__u32	reserved; /* explicit padding (optional on i386) */
+	__u32	reserved0; /* explicit padding (optional on i386) */
+	__u32	uidx;
+	__u32	reserved1;
 };
 
 struct mlx5_ib_create_srq_resp {
@@ -125,9 +143,48 @@ struct mlx5_ib_create_qp {
 	__u32	rq_wqe_count;
 	__u32	rq_wqe_shift;
 	__u32	flags;
+	__u32	uidx;
+	__u32	reserved0;
+	__u64	sq_buf_addr;
 };
 
 struct mlx5_ib_create_qp_resp {
 	__u32	uuar_index;
 };
+
+static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
+				    struct mlx5_ib_create_qp *ucmd,
+				    int inlen,
+				    u32 *user_index)
+{
+	u8 cqe_version = ucontext->cqe_version;
+
+	if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
+	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
+		return 0;
+
+	if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
+	       !!cqe_version))
+		return -EINVAL;
+
+	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
+}
+
+static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
+				     struct mlx5_ib_create_srq *ucmd,
+				     int inlen,
+				     u32 *user_index)
+{
+	u8 cqe_version = ucontext->cqe_version;
+
+	if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
+	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
+		return 0;
+
+	if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
+	       !!cqe_version))
+		return -EINVAL;
+
+	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
+}
 #endif /* MLX5_IB_USER_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 40ba83338155..a6531ffe29a6 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -608,9 +608,6 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
 			entry->opcode    = IB_WC_FETCH_ADD;
 			entry->byte_len  = MTHCA_ATOMIC_BYTE_LEN;
 			break;
-		case MTHCA_OPCODE_BIND_MW:
-			entry->opcode    = IB_WC_BIND_MW;
-			break;
 		default:
 			entry->opcode    = MTHCA_OPCODE_INVALID;
 			break;
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index dc2d48c59e62..9866c35cc977 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -898,89 +898,6 @@ static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
 	return &mr->ibmr;
 }
 
-static struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
-				       struct ib_phys_buf *buffer_list,
-				       int                 num_phys_buf,
-				       int                 acc,
-				       u64                *iova_start)
-{
-	struct mthca_mr *mr;
-	u64 *page_list;
-	u64 total_size;
-	unsigned long mask;
-	int shift;
-	int npages;
-	int err;
-	int i, j, n;
-
-	mask = buffer_list[0].addr ^ *iova_start;
-	total_size = 0;
-	for (i = 0; i < num_phys_buf; ++i) {
-		if (i != 0)
-			mask |= buffer_list[i].addr;
-		if (i != num_phys_buf - 1)
-			mask |= buffer_list[i].addr + buffer_list[i].size;
-
-		total_size += buffer_list[i].size;
-	}
-
-	if (mask & ~PAGE_MASK)
-		return ERR_PTR(-EINVAL);
-
-	shift = __ffs(mask | 1 << 31);
-
-	buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1);
-	buffer_list[0].addr &= ~0ull << shift;
-
-	mr = kmalloc(sizeof *mr, GFP_KERNEL);
-	if (!mr)
-		return ERR_PTR(-ENOMEM);
-
-	npages = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
-
-	if (!npages)
-		return &mr->ibmr;
-
-	page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
-	if (!page_list) {
-		kfree(mr);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	n = 0;
-	for (i = 0; i < num_phys_buf; ++i)
-		for (j = 0;
-		     j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
-		     ++j)
-			page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
-
-	mthca_dbg(to_mdev(pd->device), "Registering memory at %llx (iova %llx) "
-		  "in PD %x; shift %d, npages %d.\n",
-		  (unsigned long long) buffer_list[0].addr,
-		  (unsigned long long) *iova_start,
-		  to_mpd(pd)->pd_num,
-		  shift, npages);
-
-	err = mthca_mr_alloc_phys(to_mdev(pd->device),
-				  to_mpd(pd)->pd_num,
-				  page_list, shift, npages,
-				  *iova_start, total_size,
-				  convert_access(acc), mr);
-
-	if (err) {
-		kfree(page_list);
-		kfree(mr);
-		return ERR_PTR(err);
-	}
-
-	kfree(page_list);
-	mr->umem = NULL;
-
-	return &mr->ibmr;
-}
-
 static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				       u64 virt, int acc, struct ib_udata *udata)
 {
@@ -1346,7 +1263,6 @@ int mthca_register_device(struct mthca_dev *dev)
 	dev->ib_dev.destroy_cq           = mthca_destroy_cq;
 	dev->ib_dev.poll_cq              = mthca_poll_cq;
 	dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
-	dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
 	dev->ib_dev.reg_user_mr          = mthca_reg_user_mr;
 	dev->ib_dev.dereg_mr             = mthca_dereg_mr;
 	dev->ib_dev.get_port_immutable   = mthca_port_immutable;
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 35fe506e2cfa..96e5fb91fb48 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -1485,7 +1485,7 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,
 	u16 pkey;
 
 	ib_ud_header_init(256, /* assume a MAD */ 1, 0, 0,
-			  mthca_ah_grh_present(to_mah(wr->ah)), 0,
+			  mthca_ah_grh_present(to_mah(wr->ah)), 0, 0, 0,
 			  &sqp->ud_header);
 
 	err = mthca_read_ah(dev, to_mah(wr->ah), &sqp->ud_header);
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 8a3ad170d790..cb9f0f27308d 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -134,7 +134,7 @@ static void record_ird_ord(struct nes_cm_node *, u16, u16);
 /* External CM API Interface */
 /* instance of function pointers for client API */
 /* set address of this instance to cm_core->cm_ops at cm_core alloc */
-static struct nes_cm_ops nes_cm_api = {
+static const struct nes_cm_ops nes_cm_api = {
 	mini_cm_accelerated,
 	mini_cm_listen,
 	mini_cm_del_listen,
@@ -3232,7 +3232,6 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	int passive_state;
 	struct nes_ib_device *nesibdev;
 	struct ib_mr *ibmr = NULL;
-	struct ib_phys_buf ibphysbuf;
 	struct nes_pd *nespd;
 	u64 tagged_offset;
 	u8 mpa_frame_offset = 0;
@@ -3316,21 +3315,19 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		u64temp = (unsigned long)nesqp;
 		nesibdev = nesvnic->nesibdev;
 		nespd = nesqp->nespd;
-		ibphysbuf.addr = nesqp->ietf_frame_pbase + mpa_frame_offset;
-		ibphysbuf.size = buff_len;
 		tagged_offset = (u64)(unsigned long)*start_buff;
-		ibmr = nesibdev->ibdev.reg_phys_mr((struct ib_pd *)nespd,
-						   &ibphysbuf, 1,
-						   IB_ACCESS_LOCAL_WRITE,
-						   &tagged_offset);
-		if (!ibmr) {
+		ibmr = nes_reg_phys_mr(&nespd->ibpd,
+				nesqp->ietf_frame_pbase + mpa_frame_offset,
+				buff_len, IB_ACCESS_LOCAL_WRITE,
+				&tagged_offset);
+		if (IS_ERR(ibmr)) {
 			nes_debug(NES_DBG_CM, "Unable to register memory region"
 				  "for lSMM for cm_node = %p \n",
 				  cm_node);
 			pci_free_consistent(nesdev->pcidev,
 					    nesqp->private_data_len + nesqp->ietf_frame_size,
 					    nesqp->ietf_frame, nesqp->ietf_frame_pbase);
-			return -ENOMEM;
+			return PTR_ERR(ibmr);
 		}
 
 		ibmr->pd = &nespd->ibpd;
diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h
index 32a6420c2940..147c2c884227 100644
--- a/drivers/infiniband/hw/nes/nes_cm.h
+++ b/drivers/infiniband/hw/nes/nes_cm.h
@@ -423,7 +423,7 @@ struct nes_cm_core {
 
 	struct timer_list       tcp_timer;
 
-	struct nes_cm_ops       *api;
+	const struct nes_cm_ops *api;
 
 	int (*post_event)(struct nes_cm_event *event);
 	atomic_t                events_posted;
diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c
index 2042c0f29759..6d3a169c049b 100644
--- a/drivers/infiniband/hw/nes/nes_utils.c
+++ b/drivers/infiniband/hw/nes/nes_utils.c
@@ -727,7 +727,7 @@ int nes_arp_table(struct nes_device *nesdev, u32 ip_addr, u8 *mac_addr, u32 acti
 	if (action == NES_ARP_DELETE) {
 		nes_debug(NES_DBG_NETDEV, "DELETE, arp_index=%d\n", arp_index);
 		nesadapter->arp_table[arp_index].ip_addr = 0;
-		memset(nesadapter->arp_table[arp_index].mac_addr, 0x00, ETH_ALEN);
+		eth_zero_addr(nesadapter->arp_table[arp_index].mac_addr);
 		nes_free_resource(nesadapter, nesadapter->allocated_arps, arp_index);
 		return arp_index;
 	}
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 137880a19ebe..8c4daf7f22ec 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -206,80 +206,6 @@ static int nes_dealloc_mw(struct ib_mw *ibmw)
 }
 
 
-/**
- * nes_bind_mw
- */
-static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw,
-		struct ib_mw_bind *ibmw_bind)
-{
-	u64 u64temp;
-	struct nes_vnic *nesvnic = to_nesvnic(ibqp->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	/* struct nes_mr *nesmr = to_nesmw(ibmw); */
-	struct nes_qp *nesqp = to_nesqp(ibqp);
-	struct nes_hw_qp_wqe *wqe;
-	unsigned long flags = 0;
-	u32 head;
-	u32 wqe_misc = 0;
-	u32 qsize;
-
-	if (nesqp->ibqp_state > IB_QPS_RTS)
-		return -EINVAL;
-
-	spin_lock_irqsave(&nesqp->lock, flags);
-
-	head = nesqp->hwqp.sq_head;
-	qsize = nesqp->hwqp.sq_tail;
-
-	/* Check for SQ overflow */
-	if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {
-		spin_unlock_irqrestore(&nesqp->lock, flags);
-		return -ENOMEM;
-	}
-
-	wqe = &nesqp->hwqp.sq_vbase[head];
-	/* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */
-	nes_fill_init_qp_wqe(wqe, nesqp, head);
-	u64temp = ibmw_bind->wr_id;
-	set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, u64temp);
-	wqe_misc = NES_IWARP_SQ_OP_BIND;
-
-	wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
-
-	if (ibmw_bind->send_flags & IB_SEND_SIGNALED)
-		wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
-
-	if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_WRITE)
-		wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE;
-	if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_READ)
-		wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_READ;
-
-	set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_MISC_IDX, wqe_misc);
-	set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX,
-			    ibmw_bind->bind_info.mr->lkey);
-	set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MW_IDX, ibmw->rkey);
-	set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX,
-			ibmw_bind->bind_info.length);
-	wqe->wqe_words[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX] = 0;
-	u64temp = (u64)ibmw_bind->bind_info.addr;
-	set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX, u64temp);
-
-	head++;
-	if (head >= qsize)
-		head = 0;
-
-	nesqp->hwqp.sq_head = head;
-	barrier();
-
-	nes_write32(nesdev->regs+NES_WQE_ALLOC,
-			(1 << 24) | 0x00800000 | nesqp->hwqp.qp_id);
-
-	spin_unlock_irqrestore(&nesqp->lock, flags);
-
-	return 0;
-}
-
-
 /*
  * nes_alloc_fast_mr
  */
@@ -2074,9 +2000,8 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
 /**
  * nes_reg_phys_mr
  */
-static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
-		struct ib_phys_buf *buffer_list, int num_phys_buf, int acc,
-		u64 * iova_start)
+struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
+		int acc, u64 *iova_start)
 {
 	u64 region_length;
 	struct nes_pd *nespd = to_nespd(ib_pd);
@@ -2088,13 +2013,10 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
 	struct nes_vpbl vpbl;
 	struct nes_root_vpbl root_vpbl;
 	u32 stag;
-	u32 i;
 	unsigned long mask;
 	u32 stag_index = 0;
 	u32 next_stag_index = 0;
 	u32 driver_key = 0;
-	u32 root_pbl_index = 0;
-	u32 cur_pbl_index = 0;
 	int err = 0;
 	int ret = 0;
 	u16 pbl_count = 0;
@@ -2113,11 +2035,8 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
 
 	next_stag_index >>= 8;
 	next_stag_index %= nesadapter->max_mr;
-	if (num_phys_buf > (1024*512)) {
-		return ERR_PTR(-E2BIG);
-	}
 
-	if ((buffer_list[0].addr ^ *iova_start) & ~PAGE_MASK)
+	if ((addr ^ *iova_start) & ~PAGE_MASK)
 		return ERR_PTR(-EINVAL);
 
 	err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr,
@@ -2132,84 +2051,33 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	for (i = 0; i < num_phys_buf; i++) {
+	/* Allocate a 4K buffer for the PBL */
+	vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
+			&vpbl.pbl_pbase);
+	nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
+			vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase);
+	if (!vpbl.pbl_vbase) {
+		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
+		ibmr = ERR_PTR(-ENOMEM);
+		kfree(nesmr);
+		goto reg_phys_err;
+	}
 
-		if ((i & 0x01FF) == 0) {
-			if (root_pbl_index == 1) {
-				/* Allocate the root PBL */
-				root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192,
-						&root_vpbl.pbl_pbase);
-				nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n",
-						root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase);
-				if (!root_vpbl.pbl_vbase) {
-					pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
-							vpbl.pbl_pbase);
-					nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-					kfree(nesmr);
-					return ERR_PTR(-ENOMEM);
-				}
-				root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, GFP_KERNEL);
-				if (!root_vpbl.leaf_vpbl) {
-					pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
-							root_vpbl.pbl_pbase);
-					pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase,
-							vpbl.pbl_pbase);
-					nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-					kfree(nesmr);
-					return ERR_PTR(-ENOMEM);
-				}
-				root_vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase);
-				root_vpbl.pbl_vbase[0].pa_high =
-						cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
-				root_vpbl.leaf_vpbl[0] = vpbl;
-			}
-			/* Allocate a 4K buffer for the PBL */
-			vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
-					&vpbl.pbl_pbase);
-			nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
-					vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase);
-			if (!vpbl.pbl_vbase) {
-				nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-				ibmr = ERR_PTR(-ENOMEM);
-				kfree(nesmr);
-				goto reg_phys_err;
-			}
-			/* Fill in the root table */
-			if (1 <= root_pbl_index) {
-				root_vpbl.pbl_vbase[root_pbl_index].pa_low =
-						cpu_to_le32((u32)vpbl.pbl_pbase);
-				root_vpbl.pbl_vbase[root_pbl_index].pa_high =
-						cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32)));
-				root_vpbl.leaf_vpbl[root_pbl_index] = vpbl;
-			}
-			root_pbl_index++;
-			cur_pbl_index = 0;
-		}
 
-		mask = !buffer_list[i].size;
-		if (i != 0)
-			mask |= buffer_list[i].addr;
-		if (i != num_phys_buf - 1)
-			mask |= buffer_list[i].addr + buffer_list[i].size;
-
-		if (mask & ~PAGE_MASK) {
-			nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-			nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n");
-			ibmr = ERR_PTR(-EINVAL);
-			kfree(nesmr);
-			goto reg_phys_err;
-		}
+	mask = !size;
 
-		region_length += buffer_list[i].size;
-		if ((i != 0) && (single_page)) {
-			if ((buffer_list[i-1].addr+PAGE_SIZE) != buffer_list[i].addr)
-				single_page = 0;
-		}
-		vpbl.pbl_vbase[cur_pbl_index].pa_low = cpu_to_le32((u32)buffer_list[i].addr & PAGE_MASK);
-		vpbl.pbl_vbase[cur_pbl_index++].pa_high =
-				cpu_to_le32((u32)((((u64)buffer_list[i].addr) >> 32)));
+	if (mask & ~PAGE_MASK) {
+		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
+		nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n");
+		ibmr = ERR_PTR(-EINVAL);
+		kfree(nesmr);
+		goto reg_phys_err;
 	}
 
+	region_length += size;
+	vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)addr & PAGE_MASK);
+	vpbl.pbl_vbase[0].pa_high = cpu_to_le32((u32)((((u64)addr) >> 32)));
+
 	stag = stag_index << 8;
 	stag |= driver_key;
 	stag += (u32)stag_key;
@@ -2219,17 +2087,15 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
 			stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index);
 
 	/* Make the leaf PBL the root if only one PBL */
-	if (root_pbl_index == 1) {
-		root_vpbl.pbl_pbase = vpbl.pbl_pbase;
-	}
+	root_vpbl.pbl_pbase = vpbl.pbl_pbase;
 
 	if (single_page) {
 		pbl_count = 0;
 	} else {
-		pbl_count = root_pbl_index;
+		pbl_count = 1;
 	}
 	ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
-			buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start,
+			addr, pbl_count, 1, acc, iova_start,
 			&nesmr->pbls_used, &nesmr->pbl_4k);
 
 	if (ret == 0) {
@@ -2242,21 +2108,9 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
 		ibmr = ERR_PTR(-ENOMEM);
 	}
 
-	reg_phys_err:
-	/* free the resources */
-	if (root_pbl_index == 1) {
-		/* single PBL case */
-		pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase);
-	} else {
-		for (i=0; i<root_pbl_index; i++) {
-			pci_free_consistent(nesdev->pcidev, 4096, root_vpbl.leaf_vpbl[i].pbl_vbase,
-					root_vpbl.leaf_vpbl[i].pbl_pbase);
-		}
-		kfree(root_vpbl.leaf_vpbl);
-		pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase,
-				root_vpbl.pbl_pbase);
-	}
-
+reg_phys_err:
+	/* single PBL case */
+	pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase);
 	return ibmr;
 }
 
@@ -2266,17 +2120,13 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
  */
 static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc)
 {
-	struct ib_phys_buf bl;
 	u64 kva = 0;
 
 	nes_debug(NES_DBG_MR, "\n");
 
-	bl.size = (u64)0xffffffffffULL;
-	bl.addr = 0;
-	return nes_reg_phys_mr(pd, &bl, 1, acc, &kva);
+	return nes_reg_phys_mr(pd, 0, 0xffffffffffULL, acc, &kva);
 }
 
-
 /**
  * nes_reg_user_mr
  */
@@ -3888,12 +3738,10 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
 	nesibdev->ibdev.destroy_cq = nes_destroy_cq;
 	nesibdev->ibdev.poll_cq = nes_poll_cq;
 	nesibdev->ibdev.get_dma_mr = nes_get_dma_mr;
-	nesibdev->ibdev.reg_phys_mr = nes_reg_phys_mr;
 	nesibdev->ibdev.reg_user_mr = nes_reg_user_mr;
 	nesibdev->ibdev.dereg_mr = nes_dereg_mr;
 	nesibdev->ibdev.alloc_mw = nes_alloc_mw;
 	nesibdev->ibdev.dealloc_mw = nes_dealloc_mw;
-	nesibdev->ibdev.bind_mw = nes_bind_mw;
 
 	nesibdev->ibdev.alloc_mr = nes_alloc_mr;
 	nesibdev->ibdev.map_mr_sg = nes_map_mr_sg;
diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h
index a204b677af22..70290883d067 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.h
+++ b/drivers/infiniband/hw/nes/nes_verbs.h
@@ -190,4 +190,8 @@ struct nes_qp {
 	u8                    pau_state;
 	__u64                 nesuqp_addr;
 };
+
+struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd,
+		u64 addr, u64 size, int acc, u64 *iova_start);
+
 #endif			/* NES_VERBS_H */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
index 9820074be59d..3790771f2baa 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
@@ -152,9 +152,10 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
 	if ((pd->uctx) &&
 	    (!rdma_is_multicast_addr((struct in6_addr *)attr->grh.dgid.raw)) &&
 	    (!rdma_link_local_addr((struct in6_addr *)attr->grh.dgid.raw))) {
-		status = rdma_addr_find_dmac_by_grh(&sgid, &attr->grh.dgid,
-						    attr->dmac, &vlan_tag,
-						    sgid_attr.ndev->ifindex);
+		status = rdma_addr_find_l2_eth_by_grh(&sgid, &attr->grh.dgid,
+						      attr->dmac, &vlan_tag,
+						      &sgid_attr.ndev->ifindex,
+						      NULL);
 		if (status) {
 			pr_err("%s(): Failed to resolve dmac from gid." 
 				"status = %d\n", __func__, status);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index 3afb40b85159..573849354cb9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -175,7 +175,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
 	dev->ibdev.req_notify_cq = ocrdma_arm_cq;
 
 	dev->ibdev.get_dma_mr = ocrdma_get_dma_mr;
-	dev->ibdev.reg_phys_mr = ocrdma_reg_kernel_mr;
 	dev->ibdev.dereg_mr = ocrdma_dereg_mr;
 	dev->ibdev.reg_user_mr = ocrdma_reg_user_mr;
 
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 76e96f97b3f6..d4c687b548d8 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -3066,169 +3066,6 @@ pl_err:
 	return ERR_PTR(-ENOMEM);
 }
 
-#define MAX_KERNEL_PBE_SIZE 65536
-static inline int count_kernel_pbes(struct ib_phys_buf *buf_list,
-				    int buf_cnt, u32 *pbe_size)
-{
-	u64 total_size = 0;
-	u64 buf_size = 0;
-	int i;
-	*pbe_size = roundup(buf_list[0].size, PAGE_SIZE);
-	*pbe_size = roundup_pow_of_two(*pbe_size);
-
-	/* find the smallest PBE size that we can have */
-	for (i = 0; i < buf_cnt; i++) {
-		/* first addr may not be page aligned, so ignore checking */
-		if ((i != 0) && ((buf_list[i].addr & ~PAGE_MASK) ||
-				 (buf_list[i].size & ~PAGE_MASK))) {
-			return 0;
-		}
-
-		/* if configured PBE size is greater then the chosen one,
-		 * reduce the PBE size.
-		 */
-		buf_size = roundup(buf_list[i].size, PAGE_SIZE);
-		/* pbe_size has to be even multiple of 4K 1,2,4,8...*/
-		buf_size = roundup_pow_of_two(buf_size);
-		if (*pbe_size > buf_size)
-			*pbe_size = buf_size;
-
-		total_size += buf_size;
-	}
-	*pbe_size = *pbe_size > MAX_KERNEL_PBE_SIZE ?
-	    (MAX_KERNEL_PBE_SIZE) : (*pbe_size);
-
-	/* num_pbes = total_size / (*pbe_size);  this is implemented below. */
-
-	return total_size >> ilog2(*pbe_size);
-}
-
-static void build_kernel_pbes(struct ib_phys_buf *buf_list, int ib_buf_cnt,
-			      u32 pbe_size, struct ocrdma_pbl *pbl_tbl,
-			      struct ocrdma_hw_mr *hwmr)
-{
-	int i;
-	int idx;
-	int pbes_per_buf = 0;
-	u64 buf_addr = 0;
-	int num_pbes;
-	struct ocrdma_pbe *pbe;
-	int total_num_pbes = 0;
-
-	if (!hwmr->num_pbes)
-		return;
-
-	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
-	num_pbes = 0;
-
-	/* go through the OS phy regions & fill hw pbe entries into pbls. */
-	for (i = 0; i < ib_buf_cnt; i++) {
-		buf_addr = buf_list[i].addr;
-		pbes_per_buf =
-		    roundup_pow_of_two(roundup(buf_list[i].size, PAGE_SIZE)) /
-		    pbe_size;
-		hwmr->len += buf_list[i].size;
-		/* number of pbes can be more for one OS buf, when
-		 * buffers are of different sizes.
-		 * split the ib_buf to one or more pbes.
-		 */
-		for (idx = 0; idx < pbes_per_buf; idx++) {
-			/* we program always page aligned addresses,
-			 * first unaligned address is taken care by fbo.
-			 */
-			if (i == 0) {
-				/* for non zero fbo, assign the
-				 * start of the page.
-				 */
-				pbe->pa_lo =
-				    cpu_to_le32((u32) (buf_addr & PAGE_MASK));
-				pbe->pa_hi =
-				    cpu_to_le32((u32) upper_32_bits(buf_addr));
-			} else {
-				pbe->pa_lo =
-				    cpu_to_le32((u32) (buf_addr & 0xffffffff));
-				pbe->pa_hi =
-				    cpu_to_le32((u32) upper_32_bits(buf_addr));
-			}
-			buf_addr += pbe_size;
-			num_pbes += 1;
-			total_num_pbes += 1;
-			pbe++;
-
-			if (total_num_pbes == hwmr->num_pbes)
-				goto mr_tbl_done;
-			/* if the pbl is full storing the pbes,
-			 * move to next pbl.
-			 */
-			if (num_pbes == (hwmr->pbl_size/sizeof(u64))) {
-				pbl_tbl++;
-				pbe = (struct ocrdma_pbe *)pbl_tbl->va;
-				num_pbes = 0;
-			}
-		}
-	}
-mr_tbl_done:
-	return;
-}
-
-struct ib_mr *ocrdma_reg_kernel_mr(struct ib_pd *ibpd,
-				   struct ib_phys_buf *buf_list,
-				   int buf_cnt, int acc, u64 *iova_start)
-{
-	int status = -ENOMEM;
-	struct ocrdma_mr *mr;
-	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
-	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
-	u32 num_pbes;
-	u32 pbe_size = 0;
-
-	if ((acc & IB_ACCESS_REMOTE_WRITE) && !(acc & IB_ACCESS_LOCAL_WRITE))
-		return ERR_PTR(-EINVAL);
-
-	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-	if (!mr)
-		return ERR_PTR(status);
-
-	num_pbes = count_kernel_pbes(buf_list, buf_cnt, &pbe_size);
-	if (num_pbes == 0) {
-		status = -EINVAL;
-		goto pbl_err;
-	}
-	status = ocrdma_get_pbl_info(dev, mr, num_pbes);
-	if (status)
-		goto pbl_err;
-
-	mr->hwmr.pbe_size = pbe_size;
-	mr->hwmr.fbo = *iova_start - (buf_list[0].addr & PAGE_MASK);
-	mr->hwmr.va = *iova_start;
-	mr->hwmr.local_rd = 1;
-	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
-	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
-	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
-	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
-	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
-
-	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
-	if (status)
-		goto pbl_err;
-	build_kernel_pbes(buf_list, buf_cnt, pbe_size, mr->hwmr.pbl_table,
-			  &mr->hwmr);
-	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
-	if (status)
-		goto mbx_err;
-
-	mr->ibmr.lkey = mr->hwmr.lkey;
-	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
-		mr->ibmr.rkey = mr->hwmr.lkey;
-	return &mr->ibmr;
-
-mbx_err:
-	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
-pbl_err:
-	kfree(mr);
-	return ERR_PTR(status);
-}
-
 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
 {
 	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index a2f3b4dc20b0..8b517fd36779 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -117,9 +117,6 @@ int ocrdma_post_srq_recv(struct ib_srq *, struct ib_recv_wr *,
 
 int ocrdma_dereg_mr(struct ib_mr *);
 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *, int acc);
-struct ib_mr *ocrdma_reg_kernel_mr(struct ib_pd *,
-				   struct ib_phys_buf *buffer_list,
-				   int num_phys_buf, int acc, u64 *iova_start);
 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length,
 				 u64 virt, int acc, struct ib_udata *);
 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd,
diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c
index 13ef22bd9459..fcdf37913a26 100644
--- a/drivers/infiniband/hw/qib/qib_fs.c
+++ b/drivers/infiniband/hw/qib/qib_fs.c
@@ -89,14 +89,14 @@ static int create_file(const char *name, umode_t mode,
 {
 	int error;
 
-	mutex_lock(&d_inode(parent)->i_mutex);
+	inode_lock(d_inode(parent));
 	*dentry = lookup_one_len(name, parent, strlen(name));
 	if (!IS_ERR(*dentry))
 		error = qibfs_mknod(d_inode(parent), *dentry,
 				    mode, fops, data);
 	else
 		error = PTR_ERR(*dentry);
-	mutex_unlock(&d_inode(parent)->i_mutex);
+	inode_unlock(d_inode(parent));
 
 	return error;
 }
@@ -481,7 +481,7 @@ static int remove_device_files(struct super_block *sb,
 	int ret, i;
 
 	root = dget(sb->s_root);
-	mutex_lock(&d_inode(root)->i_mutex);
+	inode_lock(d_inode(root));
 	snprintf(unit, sizeof(unit), "%u", dd->unit);
 	dir = lookup_one_len(unit, root, strlen(unit));
 
@@ -491,7 +491,7 @@ static int remove_device_files(struct super_block *sb,
 		goto bail;
 	}
 
-	mutex_lock(&d_inode(dir)->i_mutex);
+	inode_lock(d_inode(dir));
 	remove_file(dir, "counters");
 	remove_file(dir, "counter_names");
 	remove_file(dir, "portcounter_names");
@@ -506,13 +506,13 @@ static int remove_device_files(struct super_block *sb,
 		}
 	}
 	remove_file(dir, "flash");
-	mutex_unlock(&d_inode(dir)->i_mutex);
+	inode_unlock(d_inode(dir));
 	ret = simple_rmdir(d_inode(root), dir);
 	d_delete(dir);
 	dput(dir);
 
 bail:
-	mutex_unlock(&d_inode(root)->i_mutex);
+	inode_unlock(d_inode(root));
 	dput(root);
 	return ret;
 }
diff --git a/drivers/infiniband/hw/qib/qib_mr.c b/drivers/infiniband/hw/qib/qib_mr.c
index 294f5c706be9..5f53304e8a9b 100644
--- a/drivers/infiniband/hw/qib/qib_mr.c
+++ b/drivers/infiniband/hw/qib/qib_mr.c
@@ -150,10 +150,7 @@ static struct qib_mr *alloc_mr(int count, struct ib_pd *pd)
 	rval = init_qib_mregion(&mr->mr, pd, count);
 	if (rval)
 		goto bail;
-	/*
-	 * ib_reg_phys_mr() will initialize mr->ibmr except for
-	 * lkey and rkey.
-	 */
+
 	rval = qib_alloc_lkey(&mr->mr, 0);
 	if (rval)
 		goto bail_mregion;
@@ -171,52 +168,6 @@ bail:
 }
 
 /**
- * qib_reg_phys_mr - register a physical memory region
- * @pd: protection domain for this memory region
- * @buffer_list: pointer to the list of physical buffers to register
- * @num_phys_buf: the number of physical buffers to register
- * @iova_start: the starting address passed over IB which maps to this MR
- *
- * Returns the memory region on success, otherwise returns an errno.
- */
-struct ib_mr *qib_reg_phys_mr(struct ib_pd *pd,
-			      struct ib_phys_buf *buffer_list,
-			      int num_phys_buf, int acc, u64 *iova_start)
-{
-	struct qib_mr *mr;
-	int n, m, i;
-	struct ib_mr *ret;
-
-	mr = alloc_mr(num_phys_buf, pd);
-	if (IS_ERR(mr)) {
-		ret = (struct ib_mr *)mr;
-		goto bail;
-	}
-
-	mr->mr.user_base = *iova_start;
-	mr->mr.iova = *iova_start;
-	mr->mr.access_flags = acc;
-
-	m = 0;
-	n = 0;
-	for (i = 0; i < num_phys_buf; i++) {
-		mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr;
-		mr->mr.map[m]->segs[n].length = buffer_list[i].size;
-		mr->mr.length += buffer_list[i].size;
-		n++;
-		if (n == QIB_SEGSZ) {
-			m++;
-			n = 0;
-		}
-	}
-
-	ret = &mr->ibmr;
-
-bail:
-	return ret;
-}
-
-/**
  * qib_reg_user_mr - register a userspace memory region
  * @pd: protection domain for this memory region
  * @start: starting userspace address
diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c
index 40f85bb3e0d3..3eff35c2d453 100644
--- a/drivers/infiniband/hw/qib/qib_qp.c
+++ b/drivers/infiniband/hw/qib/qib_qp.c
@@ -100,9 +100,10 @@ static u32 credit_table[31] = {
 	32768                   /* 1E */
 };
 
-static void get_map_page(struct qib_qpn_table *qpt, struct qpn_map *map)
+static void get_map_page(struct qib_qpn_table *qpt, struct qpn_map *map,
+			 gfp_t gfp)
 {
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
+	unsigned long page = get_zeroed_page(gfp);
 
 	/*
 	 * Free the page if someone raced with us installing it.
@@ -121,7 +122,7 @@ static void get_map_page(struct qib_qpn_table *qpt, struct qpn_map *map)
  * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI.
  */
 static int alloc_qpn(struct qib_devdata *dd, struct qib_qpn_table *qpt,
-		     enum ib_qp_type type, u8 port)
+		     enum ib_qp_type type, u8 port, gfp_t gfp)
 {
 	u32 i, offset, max_scan, qpn;
 	struct qpn_map *map;
@@ -151,7 +152,7 @@ static int alloc_qpn(struct qib_devdata *dd, struct qib_qpn_table *qpt,
 	max_scan = qpt->nmaps - !offset;
 	for (i = 0;;) {
 		if (unlikely(!map->page)) {
-			get_map_page(qpt, map);
+			get_map_page(qpt, map, gfp);
 			if (unlikely(!map->page))
 				break;
 		}
@@ -983,13 +984,21 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
 	size_t sz;
 	size_t sg_list_sz;
 	struct ib_qp *ret;
+	gfp_t gfp;
+
 
 	if (init_attr->cap.max_send_sge > ib_qib_max_sges ||
 	    init_attr->cap.max_send_wr > ib_qib_max_qp_wrs ||
-	    init_attr->create_flags) {
-		ret = ERR_PTR(-EINVAL);
-		goto bail;
-	}
+	    init_attr->create_flags & ~(IB_QP_CREATE_USE_GFP_NOIO))
+		return ERR_PTR(-EINVAL);
+
+	/* GFP_NOIO is applicable in RC QPs only */
+	if (init_attr->create_flags & IB_QP_CREATE_USE_GFP_NOIO &&
+	    init_attr->qp_type != IB_QPT_RC)
+		return ERR_PTR(-EINVAL);
+
+	gfp = init_attr->create_flags & IB_QP_CREATE_USE_GFP_NOIO ?
+			GFP_NOIO : GFP_KERNEL;
 
 	/* Check receive queue parameters if no SRQ is specified. */
 	if (!init_attr->srq) {
@@ -1021,7 +1030,8 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
 		sz = sizeof(struct qib_sge) *
 			init_attr->cap.max_send_sge +
 			sizeof(struct qib_swqe);
-		swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz);
+		swq = __vmalloc((init_attr->cap.max_send_wr + 1) * sz,
+				gfp, PAGE_KERNEL);
 		if (swq == NULL) {
 			ret = ERR_PTR(-ENOMEM);
 			goto bail;
@@ -1037,13 +1047,13 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
 		} else if (init_attr->cap.max_recv_sge > 1)
 			sg_list_sz = sizeof(*qp->r_sg_list) *
 				(init_attr->cap.max_recv_sge - 1);
-		qp = kzalloc(sz + sg_list_sz, GFP_KERNEL);
+		qp = kzalloc(sz + sg_list_sz, gfp);
 		if (!qp) {
 			ret = ERR_PTR(-ENOMEM);
 			goto bail_swq;
 		}
 		RCU_INIT_POINTER(qp->next, NULL);
-		qp->s_hdr = kzalloc(sizeof(*qp->s_hdr), GFP_KERNEL);
+		qp->s_hdr = kzalloc(sizeof(*qp->s_hdr), gfp);
 		if (!qp->s_hdr) {
 			ret = ERR_PTR(-ENOMEM);
 			goto bail_qp;
@@ -1058,8 +1068,16 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
 			qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
 			sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +
 				sizeof(struct qib_rwqe);
-			qp->r_rq.wq = vmalloc_user(sizeof(struct qib_rwq) +
-						   qp->r_rq.size * sz);
+			if (gfp != GFP_NOIO)
+				qp->r_rq.wq = vmalloc_user(
+						sizeof(struct qib_rwq) +
+						qp->r_rq.size * sz);
+			else
+				qp->r_rq.wq = __vmalloc(
+						sizeof(struct qib_rwq) +
+						qp->r_rq.size * sz,
+						gfp, PAGE_KERNEL);
+
 			if (!qp->r_rq.wq) {
 				ret = ERR_PTR(-ENOMEM);
 				goto bail_qp;
@@ -1090,7 +1108,7 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
 		dev = to_idev(ibpd->device);
 		dd = dd_from_dev(dev);
 		err = alloc_qpn(dd, &dev->qpn_table, init_attr->qp_type,
-				init_attr->port_num);
+				init_attr->port_num, gfp);
 		if (err < 0) {
 			ret = ERR_PTR(err);
 			vfree(qp->r_rq.wq);
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index de6cb6fcda8d..baf1e42b6896 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -346,6 +346,7 @@ static int qib_post_one_send(struct qib_qp *qp, struct ib_send_wr *wr,
 	unsigned long flags;
 	struct qib_lkey_table *rkt;
 	struct qib_pd *pd;
+	int avoid_schedule = 0;
 
 	spin_lock_irqsave(&qp->s_lock, flags);
 
@@ -438,11 +439,15 @@ static int qib_post_one_send(struct qib_qp *qp, struct ib_send_wr *wr,
 	    qp->ibqp.qp_type == IB_QPT_RC) {
 		if (wqe->length > 0x80000000U)
 			goto bail_inval_free;
+		if (wqe->length <= qp->pmtu)
+			avoid_schedule = 1;
 	} else if (wqe->length > (dd_from_ibdev(qp->ibqp.device)->pport +
-				  qp->port_num - 1)->ibmtu)
+				  qp->port_num - 1)->ibmtu) {
 		goto bail_inval_free;
-	else
+	} else {
 		atomic_inc(&to_iah(ud_wr(wr)->ah)->refcount);
+		avoid_schedule = 1;
+	}
 	wqe->ssn = qp->s_ssn++;
 	qp->s_head = next;
 
@@ -458,7 +463,7 @@ bail_inval_free:
 bail_inval:
 	ret = -EINVAL;
 bail:
-	if (!ret && !wr->next &&
+	if (!ret && !wr->next && !avoid_schedule &&
 	 !qib_sdma_empty(
 	   dd_from_ibdev(qp->ibqp.device)->pport + qp->port_num - 1)) {
 		qib_schedule_send(qp);
@@ -2256,7 +2261,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
 	ibdev->poll_cq = qib_poll_cq;
 	ibdev->req_notify_cq = qib_req_notify_cq;
 	ibdev->get_dma_mr = qib_get_dma_mr;
-	ibdev->reg_phys_mr = qib_reg_phys_mr;
 	ibdev->reg_user_mr = qib_reg_user_mr;
 	ibdev->dereg_mr = qib_dereg_mr;
 	ibdev->alloc_mr = qib_alloc_mr;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index bc803f33d5f6..6c5e77753d85 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -1032,10 +1032,6 @@ int qib_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 
 struct ib_mr *qib_get_dma_mr(struct ib_pd *pd, int acc);
 
-struct ib_mr *qib_reg_phys_mr(struct ib_pd *pd,
-			      struct ib_phys_buf *buffer_list,
-			      int num_phys_buf, int acc, u64 *iova_start);
-
 struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			      u64 virt_addr, int mr_access_flags,
 			      struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/qib/qib_verbs_mcast.c b/drivers/infiniband/hw/qib/qib_verbs_mcast.c
index f8ea069a3eaf..b2fb5286dbd9 100644
--- a/drivers/infiniband/hw/qib/qib_verbs_mcast.c
+++ b/drivers/infiniband/hw/qib/qib_verbs_mcast.c
@@ -286,15 +286,13 @@ int qib_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 	struct qib_ibdev *dev = to_idev(ibqp->device);
 	struct qib_ibport *ibp = to_iport(ibqp->device, qp->port_num);
 	struct qib_mcast *mcast = NULL;
-	struct qib_mcast_qp *p, *tmp;
+	struct qib_mcast_qp *p, *tmp, *delp = NULL;
 	struct rb_node *n;
 	int last = 0;
 	int ret;
 
-	if (ibqp->qp_num <= 1 || qp->state == IB_QPS_RESET) {
-		ret = -EINVAL;
-		goto bail;
-	}
+	if (ibqp->qp_num <= 1 || qp->state == IB_QPS_RESET)
+		return -EINVAL;
 
 	spin_lock_irq(&ibp->lock);
 
@@ -303,8 +301,7 @@ int qib_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 	while (1) {
 		if (n == NULL) {
 			spin_unlock_irq(&ibp->lock);
-			ret = -EINVAL;
-			goto bail;
+			return -EINVAL;
 		}
 
 		mcast = rb_entry(n, struct qib_mcast, rb_node);
@@ -328,6 +325,7 @@ int qib_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 		 */
 		list_del_rcu(&p->list);
 		mcast->n_attached--;
+		delp = p;
 
 		/* If this was the last attached QP, remove the GID too. */
 		if (list_empty(&mcast->qp_list)) {
@@ -338,15 +336,16 @@ int qib_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 	}
 
 	spin_unlock_irq(&ibp->lock);
+	/* QP not attached */
+	if (!delp)
+		return -EINVAL;
+	/*
+	 * Wait for any list walkers to finish before freeing the
+	 * list element.
+	 */
+	wait_event(mcast->wait, atomic_read(&mcast->refcount) <= 1);
+	qib_mcast_qp_free(delp);
 
-	if (p) {
-		/*
-		 * Wait for any list walkers to finish before freeing the
-		 * list element.
-		 */
-		wait_event(mcast->wait, atomic_read(&mcast->refcount) <= 1);
-		qib_mcast_qp_free(p);
-	}
 	if (last) {
 		atomic_dec(&mcast->refcount);
 		wait_event(mcast->wait, !atomic_read(&mcast->refcount));
@@ -355,11 +354,7 @@ int qib_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 		dev->n_mcast_grps_allocated--;
 		spin_unlock_irq(&dev->n_mcast_grps_lock);
 	}
-
-	ret = 0;
-
-bail:
-	return ret;
+	return 0;
 }
 
 int qib_mcast_tree_empty(struct qib_ibport *ibp)
diff --git a/drivers/infiniband/hw/usnic/usnic_debugfs.c b/drivers/infiniband/hw/usnic/usnic_debugfs.c
index 5e55b8bc6fe4..92dc66cc2d50 100644
--- a/drivers/infiniband/hw/usnic/usnic_debugfs.c
+++ b/drivers/infiniband/hw/usnic/usnic_debugfs.c
@@ -157,8 +157,9 @@ void usnic_debugfs_flow_add(struct usnic_ib_qp_grp_flow *qp_flow)
 							qp_flow,
 							&flowinfo_ops);
 	if (IS_ERR_OR_NULL(qp_flow->dbgfs_dentry)) {
-		usnic_err("Failed to create dbg fs entry for flow %u\n",
-				qp_flow->flow->flow_id);
+		usnic_err("Failed to create dbg fs entry for flow %u with error %ld\n",
+				qp_flow->flow->flow_id,
+				PTR_ERR(qp_flow->dbgfs_dentry));
 	}
 }
 
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
index fcea3a24d3eb..5f44b66ccb86 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
@@ -521,7 +521,7 @@ int usnic_ib_qp_grp_modify(struct usnic_ib_qp_grp *qp_grp,
 
 	if (!status) {
 		qp_grp->state = new_state;
-		usnic_info("Transistioned %u from %s to %s",
+		usnic_info("Transitioned %u from %s to %s",
 		qp_grp->grp_id,
 		usnic_ib_qp_grp_state_to_string(old_state),
 		usnic_ib_qp_grp_state_to_string(new_state));
@@ -575,7 +575,7 @@ alloc_res_chunk_list(struct usnic_vnic *vnic,
 	return res_chunk_list;
 
 out_free_res:
-	for (i--; i > 0; i--)
+	for (i--; i >= 0; i--)
 		usnic_vnic_put_resources(res_chunk_list[i]);
 	kfree(res_chunk_list);
 	return ERR_PTR(err);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index f8e3211689a3..6cdb4d23f78f 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -51,7 +51,7 @@
 
 static void usnic_ib_fw_string_to_u64(char *fw_ver_str, u64 *fw_ver)
 {
-	*fw_ver = (u64) *fw_ver_str;
+	*fw_ver = *((u64 *)fw_ver_str);
 }
 
 static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
@@ -571,20 +571,20 @@ int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 	qp_grp = to_uqp_grp(ibqp);
 
-	/* TODO: Future Support All States */
 	mutex_lock(&qp_grp->vf->pf->usdev_lock);
-	if ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT) {
-		status = usnic_ib_qp_grp_modify(qp_grp, IB_QPS_INIT, NULL);
-	} else if ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_RTR) {
-		status = usnic_ib_qp_grp_modify(qp_grp, IB_QPS_RTR, NULL);
-	} else if ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_RTS) {
-		status = usnic_ib_qp_grp_modify(qp_grp, IB_QPS_RTS, NULL);
+	if ((attr_mask & IB_QP_PORT) && attr->port_num != 1) {
+		/* usnic devices only have one port */
+		status = -EINVAL;
+		goto out_unlock;
+	}
+	if (attr_mask & IB_QP_STATE) {
+		status = usnic_ib_qp_grp_modify(qp_grp, attr->qp_state, NULL);
 	} else {
-		usnic_err("Unexpected combination mask: %u state: %u\n",
-				attr_mask & IB_QP_STATE, attr->qp_state);
+		usnic_err("Unhandled request, attr_mask=0x%x\n", attr_mask);
 		status = -EINVAL;
 	}
 
+out_unlock:
 	mutex_unlock(&qp_grp->vf->pf->usdev_lock);
 	return status;
 }
@@ -625,8 +625,8 @@ struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length,
 			virt_addr, length);
 
 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-	if (IS_ERR_OR_NULL(mr))
-		return ERR_PTR(mr ? PTR_ERR(mr) : -ENOMEM);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
 
 	mr->umem = usnic_uiom_reg_get(to_upd(pd)->umem_pd, start, length,
 					access_flags, 0);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
index 414eaa566bd9..0d9d2e6a14d5 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
@@ -43,8 +43,6 @@ int usnic_ib_query_device(struct ib_device *ibdev,
 			  struct ib_udata *uhw);
 int usnic_ib_query_port(struct ib_device *ibdev, u8 port,
 				struct ib_port_attr *props);
-enum rdma_protocol_type
-usnic_ib_query_protocol(struct ib_device *device, u8 port_num);
 int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
 				int qp_attr_mask,
 				struct ib_qp_init_attr *qp_init_attr);
diff --git a/drivers/infiniband/hw/usnic/usnic_vnic.c b/drivers/infiniband/hw/usnic/usnic_vnic.c
index 66de93fb8ea9..887510718690 100644
--- a/drivers/infiniband/hw/usnic/usnic_vnic.c
+++ b/drivers/infiniband/hw/usnic/usnic_vnic.c
@@ -237,7 +237,7 @@ usnic_vnic_get_resources(struct usnic_vnic *vnic, enum usnic_vnic_res_type type,
 	struct usnic_vnic_res *res;
 	int i;
 
-	if (usnic_vnic_res_free_cnt(vnic, type) < cnt || cnt < 1 || !owner)
+	if (usnic_vnic_res_free_cnt(vnic, type) < cnt || cnt < 0 || !owner)
 		return ERR_PTR(-EINVAL);
 
 	ret = kzalloc(sizeof(*ret), GFP_ATOMIC);
@@ -247,26 +247,28 @@ usnic_vnic_get_resources(struct usnic_vnic *vnic, enum usnic_vnic_res_type type,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	ret->res = kzalloc(sizeof(*(ret->res))*cnt, GFP_ATOMIC);
-	if (!ret->res) {
-		usnic_err("Failed to allocate resources for %s. Out of memory\n",
-				usnic_vnic_pci_name(vnic));
-		kfree(ret);
-		return ERR_PTR(-ENOMEM);
-	}
+	if (cnt > 0) {
+		ret->res = kcalloc(cnt, sizeof(*(ret->res)), GFP_ATOMIC);
+		if (!ret->res) {
+			usnic_err("Failed to allocate resources for %s. Out of memory\n",
+					usnic_vnic_pci_name(vnic));
+			kfree(ret);
+			return ERR_PTR(-ENOMEM);
+		}
 
-	spin_lock(&vnic->res_lock);
-	src = &vnic->chunks[type];
-	for (i = 0; i < src->cnt && ret->cnt < cnt; i++) {
-		res = src->res[i];
-		if (!res->owner) {
-			src->free_cnt--;
-			res->owner = owner;
-			ret->res[ret->cnt++] = res;
+		spin_lock(&vnic->res_lock);
+		src = &vnic->chunks[type];
+		for (i = 0; i < src->cnt && ret->cnt < cnt; i++) {
+			res = src->res[i];
+			if (!res->owner) {
+				src->free_cnt--;
+				res->owner = owner;
+				ret->res[ret->cnt++] = res;
+			}
 		}
-	}
 
-	spin_unlock(&vnic->res_lock);
+		spin_unlock(&vnic->res_lock);
+	}
 	ret->type = type;
 	ret->vnic = vnic;
 	WARN_ON(ret->cnt != cnt);
@@ -281,14 +283,16 @@ void usnic_vnic_put_resources(struct usnic_vnic_res_chunk *chunk)
 	int i;
 	struct usnic_vnic *vnic = chunk->vnic;
 
-	spin_lock(&vnic->res_lock);
-	while ((i = --chunk->cnt) >= 0) {
-		res = chunk->res[i];
-		chunk->res[i] = NULL;
-		res->owner = NULL;
-		vnic->chunks[res->type].free_cnt++;
+	if (chunk->cnt > 0) {
+		spin_lock(&vnic->res_lock);
+		while ((i = --chunk->cnt) >= 0) {
+			res = chunk->res[i];
+			chunk->res[i] = NULL;
+			res->owner = NULL;
+			vnic->chunks[res->type].free_cnt++;
+		}
+		spin_unlock(&vnic->res_lock);
 	}
-	spin_unlock(&vnic->res_lock);
 
 	kfree(chunk->res);
 	kfree(chunk);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 3ede10309754..a6f3eab0f350 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -495,7 +495,6 @@ void ipoib_dev_cleanup(struct net_device *dev);
 void ipoib_mcast_join_task(struct work_struct *work);
 void ipoib_mcast_carrier_on_task(struct work_struct *work);
 void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb);
-void ipoib_mcast_free(struct ipoib_mcast *mc);
 
 void ipoib_mcast_restart_task(struct work_struct *work);
 int ipoib_mcast_start_thread(struct net_device *dev);
@@ -549,8 +548,9 @@ void ipoib_path_iter_read(struct ipoib_path_iter *iter,
 
 int ipoib_mcast_attach(struct net_device *dev, u16 mlid,
 		       union ib_gid *mgid, int set_qkey);
-int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast);
-struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid);
+void ipoib_mcast_remove_list(struct list_head *remove_list);
+void ipoib_check_and_add_mcast_sendonly(struct ipoib_dev_priv *priv, u8 *mgid,
+				struct list_head *remove_list);
 
 int ipoib_init_qp(struct net_device *dev);
 int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 3ae9726efb98..917e46ea3bf6 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -70,7 +70,6 @@ static struct ib_qp_attr ipoib_cm_err_attr = {
 #define IPOIB_CM_RX_DRAIN_WRID 0xffffffff
 
 static struct ib_send_wr ipoib_cm_rx_drain_wr = {
-	.wr_id = IPOIB_CM_RX_DRAIN_WRID,
 	.opcode = IB_WR_SEND,
 };
 
@@ -223,6 +222,7 @@ static void ipoib_cm_start_rx_drain(struct ipoib_dev_priv *priv)
 	 * error" WC will be immediately generated for each WR we post.
 	 */
 	p = list_entry(priv->cm.rx_flush_list.next, typeof(*p), list);
+	ipoib_cm_rx_drain_wr.wr_id = IPOIB_CM_RX_DRAIN_WRID;
 	if (ib_post_send(p->qp, &ipoib_cm_rx_drain_wr, &bad_wr))
 		ipoib_warn(priv, "failed to post drain wr\n");
 
@@ -1522,8 +1522,7 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge)
 int ipoib_cm_dev_init(struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
-	int i, ret;
-	struct ib_device_attr attr;
+	int max_srq_sge, i;
 
 	INIT_LIST_HEAD(&priv->cm.passive_ids);
 	INIT_LIST_HEAD(&priv->cm.reap_list);
@@ -1540,19 +1539,13 @@ int ipoib_cm_dev_init(struct net_device *dev)
 
 	skb_queue_head_init(&priv->cm.skb_queue);
 
-	ret = ib_query_device(priv->ca, &attr);
-	if (ret) {
-		printk(KERN_WARNING "ib_query_device() failed with %d\n", ret);
-		return ret;
-	}
-
-	ipoib_dbg(priv, "max_srq_sge=%d\n", attr.max_srq_sge);
+	ipoib_dbg(priv, "max_srq_sge=%d\n", priv->ca->attrs.max_srq_sge);
 
-	attr.max_srq_sge = min_t(int, IPOIB_CM_RX_SG, attr.max_srq_sge);
-	ipoib_cm_create_srq(dev, attr.max_srq_sge);
+	max_srq_sge = min_t(int, IPOIB_CM_RX_SG, priv->ca->attrs.max_srq_sge);
+	ipoib_cm_create_srq(dev, max_srq_sge);
 	if (ipoib_cm_has_srq(dev)) {
-		priv->cm.max_cm_mtu = attr.max_srq_sge * PAGE_SIZE - 0x10;
-		priv->cm.num_frags  = attr.max_srq_sge;
+		priv->cm.max_cm_mtu = max_srq_sge * PAGE_SIZE - 0x10;
+		priv->cm.num_frags  = max_srq_sge;
 		ipoib_dbg(priv, "max_cm_mtu = 0x%x, num_frags=%d\n",
 			  priv->cm.max_cm_mtu, priv->cm.num_frags);
 	} else {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
index 078cadd6c797..a53fa5fc0dec 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
@@ -40,15 +40,11 @@ static void ipoib_get_drvinfo(struct net_device *netdev,
 			      struct ethtool_drvinfo *drvinfo)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(netdev);
-	struct ib_device_attr *attr;
-
-	attr = kmalloc(sizeof(*attr), GFP_KERNEL);
-	if (attr && !ib_query_device(priv->ca, attr))
-		snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
-			 "%d.%d.%d", (int)(attr->fw_ver >> 32),
-			 (int)(attr->fw_ver >> 16) & 0xffff,
-			 (int)attr->fw_ver & 0xffff);
-	kfree(attr);
+
+	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
+		 "%d.%d.%d", (int)(priv->ca->attrs.fw_ver >> 32),
+		 (int)(priv->ca->attrs.fw_ver >> 16) & 0xffff,
+		 (int)priv->ca->attrs.fw_ver & 0xffff);
 
 	strlcpy(drvinfo->bus_info, dev_name(priv->ca->dma_device),
 		sizeof(drvinfo->bus_info));
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 7d3281866ffc..25509bbd4a05 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1150,8 +1150,6 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
 	unsigned long flags;
 	int i;
 	LIST_HEAD(remove_list);
-	struct ipoib_mcast *mcast, *tmcast;
-	struct net_device *dev = priv->dev;
 
 	if (test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags))
 		return;
@@ -1179,18 +1177,8 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
 							  lockdep_is_held(&priv->lock))) != NULL) {
 			/* was the neigh idle for two GC periods */
 			if (time_after(neigh_obsolete, neigh->alive)) {
-				u8 *mgid = neigh->daddr + 4;
 
-				/* Is this multicast ? */
-				if (*mgid == 0xff) {
-					mcast = __ipoib_mcast_find(dev, mgid);
-
-					if (mcast && test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags)) {
-						list_del(&mcast->list);
-						rb_erase(&mcast->rb_node, &priv->multicast_tree);
-						list_add_tail(&mcast->list, &remove_list);
-					}
-				}
+				ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);
 
 				rcu_assign_pointer(*np,
 						   rcu_dereference_protected(neigh->hnext,
@@ -1207,10 +1195,7 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
 
 out_unlock:
 	spin_unlock_irqrestore(&priv->lock, flags);
-	list_for_each_entry_safe(mcast, tmcast, &remove_list, list) {
-		ipoib_mcast_leave(dev, mcast);
-		ipoib_mcast_free(mcast);
-	}
+	ipoib_mcast_remove_list(&remove_list);
 }
 
 static void ipoib_reap_neigh(struct work_struct *work)
@@ -1777,26 +1762,7 @@ int ipoib_add_pkey_attr(struct net_device *dev)
 
 int ipoib_set_dev_features(struct ipoib_dev_priv *priv, struct ib_device *hca)
 {
-	struct ib_device_attr *device_attr;
-	int result = -ENOMEM;
-
-	device_attr = kmalloc(sizeof *device_attr, GFP_KERNEL);
-	if (!device_attr) {
-		printk(KERN_WARNING "%s: allocation of %zu bytes failed\n",
-		       hca->name, sizeof *device_attr);
-		return result;
-	}
-
-	result = ib_query_device(hca, device_attr);
-	if (result) {
-		printk(KERN_WARNING "%s: ib_query_device failed (ret = %d)\n",
-		       hca->name, result);
-		kfree(device_attr);
-		return result;
-	}
-	priv->hca_caps = device_attr->device_cap_flags;
-
-	kfree(device_attr);
+	priv->hca_caps = hca->attrs.device_cap_flags;
 
 	if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
 		priv->dev->hw_features = NETIF_F_SG |
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index f357ca67a41c..050dfa175d16 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -106,7 +106,7 @@ static void __ipoib_mcast_schedule_join_thread(struct ipoib_dev_priv *priv,
 		queue_delayed_work(priv->wq, &priv->mcast_task, 0);
 }
 
-void ipoib_mcast_free(struct ipoib_mcast *mcast)
+static void ipoib_mcast_free(struct ipoib_mcast *mcast)
 {
 	struct net_device *dev = mcast->dev;
 	int tx_dropped = 0;
@@ -153,7 +153,7 @@ static struct ipoib_mcast *ipoib_mcast_alloc(struct net_device *dev,
 	return mcast;
 }
 
-struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid)
+static struct ipoib_mcast *__ipoib_mcast_find(struct net_device *dev, void *mgid)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
 	struct rb_node *n = priv->multicast_tree.rb_node;
@@ -677,7 +677,7 @@ int ipoib_mcast_stop_thread(struct net_device *dev)
 	return 0;
 }
 
-int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
+static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
 	int ret = 0;
@@ -704,6 +704,35 @@ int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
 	return 0;
 }
 
+/*
+ * Check if the multicast group is sendonly. If so remove it from the maps
+ * and add to the remove list
+ */
+void ipoib_check_and_add_mcast_sendonly(struct ipoib_dev_priv *priv, u8 *mgid,
+				struct list_head *remove_list)
+{
+	/* Is this multicast ? */
+	if (*mgid == 0xff) {
+		struct ipoib_mcast *mcast = __ipoib_mcast_find(priv->dev, mgid);
+
+		if (mcast && test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags)) {
+			list_del(&mcast->list);
+			rb_erase(&mcast->rb_node, &priv->multicast_tree);
+			list_add_tail(&mcast->list, remove_list);
+		}
+	}
+}
+
+void ipoib_mcast_remove_list(struct list_head *remove_list)
+{
+	struct ipoib_mcast *mcast, *tmcast;
+
+	list_for_each_entry_safe(mcast, tmcast, remove_list, list) {
+		ipoib_mcast_leave(mcast->dev, mcast);
+		ipoib_mcast_free(mcast);
+	}
+}
+
 void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -810,10 +839,7 @@ void ipoib_mcast_dev_flush(struct net_device *dev)
 		if (test_bit(IPOIB_MCAST_FLAG_BUSY, &mcast->flags))
 			wait_for_completion(&mcast->done);
 
-	list_for_each_entry_safe(mcast, tmcast, &remove_list, list) {
-		ipoib_mcast_leave(dev, mcast);
-		ipoib_mcast_free(mcast);
-	}
+	ipoib_mcast_remove_list(&remove_list);
 }
 
 static int ipoib_mcast_addr_is_valid(const u8 *addr, const u8 *broadcast)
@@ -939,10 +965,7 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 		if (test_bit(IPOIB_MCAST_FLAG_BUSY, &mcast->flags))
 			wait_for_completion(&mcast->done);
 
-	list_for_each_entry_safe(mcast, tmcast, &remove_list, list) {
-		ipoib_mcast_leave(mcast->dev, mcast);
-		ipoib_mcast_free(mcast);
-	}
+	ipoib_mcast_remove_list(&remove_list);
 
 	/*
 	 * Double check that we are still up
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 9080161e01af..c827c93f46c5 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -644,7 +644,7 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
 
 		ib_conn = &iser_conn->ib_conn;
 		if (ib_conn->pi_support) {
-			u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap;
+			u32 sig_caps = ib_conn->device->ib_device->attrs.sig_prot_cap;
 
 			scsi_host_set_prot(shost, iser_dif_prot_caps(sig_caps));
 			scsi_host_set_guard(shost, SHOST_DIX_GUARD_IP |
@@ -656,7 +656,7 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
 		 * max fastreg page list length.
 		 */
 		shost->sg_tablesize = min_t(unsigned short, shost->sg_tablesize,
-			ib_conn->device->dev_attr.max_fast_reg_page_list_len);
+			ib_conn->device->ib_device->attrs.max_fast_reg_page_list_len);
 		shost->max_sectors = min_t(unsigned int,
 			1024, (shost->sg_tablesize * PAGE_SIZE) >> 9);
 
@@ -1059,7 +1059,8 @@ static int __init iser_init(void)
 	release_wq = alloc_workqueue("release workqueue", 0, 0);
 	if (!release_wq) {
 		iser_err("failed to allocate release workqueue\n");
-		return -ENOMEM;
+		err = -ENOMEM;
+		goto err_alloc_wq;
 	}
 
 	iscsi_iser_scsi_transport = iscsi_register_transport(
@@ -1067,12 +1068,14 @@ static int __init iser_init(void)
 	if (!iscsi_iser_scsi_transport) {
 		iser_err("iscsi_register_transport failed\n");
 		err = -EINVAL;
-		goto register_transport_failure;
+		goto err_reg;
 	}
 
 	return 0;
 
-register_transport_failure:
+err_reg:
+	destroy_workqueue(release_wq);
+err_alloc_wq:
 	kmem_cache_destroy(ig.desc_cache);
 
 	return err;
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 8a5998e6a407..95f0a64e076b 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -48,6 +48,7 @@
 #include <scsi/scsi_transport_iscsi.h>
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
+#include <scsi/iser.h>
 
 #include <linux/interrupt.h>
 #include <linux/wait.h>
@@ -151,46 +152,10 @@
 					 - ISER_MAX_RX_MISC_PDUS) /	\
 					 (1 + ISER_INFLIGHT_DATAOUTS))
 
-#define ISER_WC_BATCH_COUNT   16
 #define ISER_SIGNAL_CMD_COUNT 32
 
-#define ISER_VER			0x10
-#define ISER_WSV			0x08
-#define ISER_RSV			0x04
-
-#define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
-#define ISER_BEACON_WRID		0xfffffffffffffffeULL
-
-/**
- * struct iser_hdr - iSER header
- *
- * @flags:        flags support (zbva, remote_inv)
- * @rsvd:         reserved
- * @write_stag:   write rkey
- * @write_va:     write virtual address
- * @reaf_stag:    read rkey
- * @read_va:      read virtual address
- */
-struct iser_hdr {
-	u8      flags;
-	u8      rsvd[3];
-	__be32  write_stag;
-	__be64  write_va;
-	__be32  read_stag;
-	__be64  read_va;
-} __attribute__((packed));
-
-
-#define ISER_ZBVA_NOT_SUPPORTED		0x80
-#define ISER_SEND_W_INV_NOT_SUPPORTED	0x40
-
-struct iser_cm_hdr {
-	u8      flags;
-	u8      rsvd[3];
-} __packed;
-
 /* Constant PDU lengths calculations */
-#define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
+#define ISER_HEADERS_LEN	(sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
 
 #define ISER_RECV_DATA_SEG_LEN	128
 #define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
@@ -269,7 +234,7 @@ enum iser_desc_type {
 #define ISER_MAX_WRS 7
 
 /**
- * struct iser_tx_desc - iSER TX descriptor (for send wr_id)
+ * struct iser_tx_desc - iSER TX descriptor
  *
  * @iser_header:   iser header
  * @iscsi_header:  iscsi header
@@ -287,12 +252,13 @@ enum iser_desc_type {
  * @sig_attrs:     Signature attributes
  */
 struct iser_tx_desc {
-	struct iser_hdr              iser_header;
+	struct iser_ctrl             iser_header;
 	struct iscsi_hdr             iscsi_header;
 	enum   iser_desc_type        type;
 	u64		             dma_addr;
 	struct ib_sge		     tx_sg[2];
 	int                          num_sge;
+	struct ib_cqe		     cqe;
 	bool			     mapped;
 	u8                           wr_idx;
 	union iser_wr {
@@ -306,9 +272,10 @@ struct iser_tx_desc {
 };
 
 #define ISER_RX_PAD_SIZE	(256 - (ISER_RX_PAYLOAD_SIZE + \
-					sizeof(u64) + sizeof(struct ib_sge)))
+				 sizeof(u64) + sizeof(struct ib_sge) + \
+				 sizeof(struct ib_cqe)))
 /**
- * struct iser_rx_desc - iSER RX descriptor (for recv wr_id)
+ * struct iser_rx_desc - iSER RX descriptor
  *
  * @iser_header:   iser header
  * @iscsi_header:  iscsi header
@@ -318,12 +285,32 @@ struct iser_tx_desc {
  * @pad:           for sense data TODO: Modify to maximum sense length supported
  */
 struct iser_rx_desc {
-	struct iser_hdr              iser_header;
+	struct iser_ctrl             iser_header;
 	struct iscsi_hdr             iscsi_header;
 	char		             data[ISER_RECV_DATA_SEG_LEN];
 	u64		             dma_addr;
 	struct ib_sge		     rx_sg;
+	struct ib_cqe		     cqe;
 	char		             pad[ISER_RX_PAD_SIZE];
+} __packed;
+
+/**
+ * struct iser_login_desc - iSER login descriptor
+ *
+ * @req:           pointer to login request buffer
+ * @resp:          pointer to login response buffer
+ * @req_dma:       DMA address of login request buffer
+ * @rsp_dma:      DMA address of login response buffer
+ * @sge:           IB sge for login post recv
+ * @cqe:           completion handler
+ */
+struct iser_login_desc {
+	void                         *req;
+	void                         *rsp;
+	u64                          req_dma;
+	u64                          rsp_dma;
+	struct ib_sge                sge;
+	struct ib_cqe		     cqe;
 } __attribute__((packed));
 
 struct iser_conn;
@@ -333,18 +320,12 @@ struct iscsi_iser_task;
 /**
  * struct iser_comp - iSER completion context
  *
- * @device:     pointer to device handle
  * @cq:         completion queue
- * @wcs:        work completion array
- * @tasklet:    Tasklet handle
  * @active_qps: Number of active QPs attached
  *              to completion context
  */
 struct iser_comp {
-	struct iser_device      *device;
 	struct ib_cq		*cq;
-	struct ib_wc		 wcs[ISER_WC_BATCH_COUNT];
-	struct tasklet_struct	 tasklet;
 	int                      active_qps;
 };
 
@@ -380,7 +361,6 @@ struct iser_reg_ops {
  *
  * @ib_device:     RDMA device
  * @pd:            Protection Domain for this device
- * @dev_attr:      Device attributes container
  * @mr:            Global DMA memory region
  * @event_handler: IB events handle routine
  * @ig_list:	   entry in devices list
@@ -389,18 +369,19 @@ struct iser_reg_ops {
  *                 cpus and device max completion vectors
  * @comps:         Dinamically allocated array of completion handlers
  * @reg_ops:       Registration ops
+ * @remote_inv_sup: Remote invalidate is supported on this device
  */
 struct iser_device {
 	struct ib_device             *ib_device;
 	struct ib_pd	             *pd;
-	struct ib_device_attr	     dev_attr;
 	struct ib_mr	             *mr;
 	struct ib_event_handler      event_handler;
 	struct list_head             ig_list;
 	int                          refcount;
 	int			     comps_used;
 	struct iser_comp	     *comps;
-	struct iser_reg_ops          *reg_ops;
+	const struct iser_reg_ops    *reg_ops;
+	bool                         remote_inv_sup;
 };
 
 #define ISER_CHECK_GUARD	0xc0
@@ -475,10 +456,11 @@ struct iser_fr_pool {
  * @rx_wr:               receive work request for batch posts
  * @device:              reference to iser device
  * @comp:                iser completion context
- * @pi_support:          Indicate device T10-PI support
- * @beacon:              beacon send wr to signal all flush errors were drained
- * @flush_comp:          completes when all connection completions consumed
  * @fr_pool:             connection fast registration poool
+ * @pi_support:          Indicate device T10-PI support
+ * @last:                last send wr to signal all flush errors were drained
+ * @last_cqe:            cqe handler for last wr
+ * @last_comp:           completes when all connection completions consumed
  */
 struct ib_conn {
 	struct rdma_cm_id           *cma_id;
@@ -488,10 +470,12 @@ struct ib_conn {
 	struct ib_recv_wr	     rx_wr[ISER_MIN_POSTED_RX];
 	struct iser_device          *device;
 	struct iser_comp	    *comp;
-	bool			     pi_support;
-	struct ib_send_wr	     beacon;
-	struct completion	     flush_comp;
 	struct iser_fr_pool          fr_pool;
+	bool			     pi_support;
+	struct ib_send_wr	     last;
+	struct ib_cqe		     last_cqe;
+	struct ib_cqe		     reg_cqe;
+	struct completion	     last_comp;
 };
 
 /**
@@ -514,11 +498,7 @@ struct ib_conn {
  * @up_completion:    connection establishment completed
  *                    (state is ISER_CONN_UP)
  * @conn_list:        entry in ig conn list
- * @login_buf:        login data buffer (stores login parameters)
- * @login_req_buf:    login request buffer
- * @login_req_dma:    login request buffer dma address
- * @login_resp_buf:   login response buffer
- * @login_resp_dma:   login response buffer dma address
+ * @login_desc:       login descriptor
  * @rx_desc_head:     head of rx_descs cyclic buffer
  * @rx_descs:         rx buffers array (cyclic buffer)
  * @num_rx_descs:     number of rx descriptors
@@ -541,15 +521,13 @@ struct iser_conn {
 	struct completion	     ib_completion;
 	struct completion	     up_completion;
 	struct list_head	     conn_list;
-
-	char  			     *login_buf;
-	char			     *login_req_buf, *login_resp_buf;
-	u64			     login_req_dma, login_resp_dma;
+	struct iser_login_desc       login_desc;
 	unsigned int 		     rx_desc_head;
 	struct iser_rx_desc	     *rx_descs;
 	u32                          num_rx_descs;
 	unsigned short               scsi_sg_tablesize;
 	unsigned int                 scsi_max_sectors;
+	bool			     snd_w_inv;
 };
 
 /**
@@ -579,9 +557,8 @@ struct iscsi_iser_task {
 
 struct iser_page_vec {
 	u64 *pages;
-	int length;
-	int offset;
-	int data_size;
+	int npages;
+	struct ib_mr fake_mr;
 };
 
 /**
@@ -633,12 +610,14 @@ int iser_conn_terminate(struct iser_conn *iser_conn);
 
 void iser_release_work(struct work_struct *work);
 
-void iser_rcv_completion(struct iser_rx_desc *desc,
-			 unsigned long dto_xfer_len,
-			 struct ib_conn *ib_conn);
-
-void iser_snd_completion(struct iser_tx_desc *desc,
-			 struct ib_conn *ib_conn);
+void iser_err_comp(struct ib_wc *wc, const char *type);
+void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc);
+void iser_last_comp(struct ib_cq *cq, struct ib_wc *wc);
 
 void iser_task_rdma_init(struct iscsi_iser_task *task);
 
@@ -651,7 +630,8 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
 				     enum iser_data_dir cmd_dir);
 
 int iser_reg_rdma_mem(struct iscsi_iser_task *task,
-		      enum iser_data_dir dir);
+		      enum iser_data_dir dir,
+		      bool all_imm);
 void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
 			 enum iser_data_dir dir);
 
@@ -719,4 +699,28 @@ iser_tx_next_wr(struct iser_tx_desc *tx_desc)
 	return cur_wr;
 }
 
+static inline struct iser_conn *
+to_iser_conn(struct ib_conn *ib_conn)
+{
+	return container_of(ib_conn, struct iser_conn, ib_conn);
+}
+
+static inline struct iser_rx_desc *
+iser_rx(struct ib_cqe *cqe)
+{
+	return container_of(cqe, struct iser_rx_desc, cqe);
+}
+
+static inline struct iser_tx_desc *
+iser_tx(struct ib_cqe *cqe)
+{
+	return container_of(cqe, struct iser_tx_desc, cqe);
+}
+
+static inline struct iser_login_desc *
+iser_login(struct ib_cqe *cqe)
+{
+	return container_of(cqe, struct iser_login_desc, cqe);
+}
+
 #endif
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index ffd00c420729..ed54b388e7ad 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -51,7 +51,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
 	struct iscsi_iser_task *iser_task = task->dd_data;
 	struct iser_mem_reg *mem_reg;
 	int err;
-	struct iser_hdr *hdr = &iser_task->desc.iser_header;
+	struct iser_ctrl *hdr = &iser_task->desc.iser_header;
 	struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN];
 
 	err = iser_dma_map_task_data(iser_task,
@@ -72,7 +72,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
 			return err;
 	}
 
-	err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
+	err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN, false);
 	if (err) {
 		iser_err("Failed to set up Data-IN RDMA\n");
 		return err;
@@ -104,7 +104,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
 	struct iscsi_iser_task *iser_task = task->dd_data;
 	struct iser_mem_reg *mem_reg;
 	int err;
-	struct iser_hdr *hdr = &iser_task->desc.iser_header;
+	struct iser_ctrl *hdr = &iser_task->desc.iser_header;
 	struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT];
 	struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1];
 
@@ -126,7 +126,8 @@ iser_prepare_write_cmd(struct iscsi_task *task,
 			return err;
 	}
 
-	err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
+	err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT,
+				buf_out->data_len == imm_sz);
 	if (err != 0) {
 		iser_err("Failed to register write cmd RDMA mem\n");
 		return err;
@@ -166,7 +167,7 @@ static void iser_create_send_desc(struct iser_conn	*iser_conn,
 	ib_dma_sync_single_for_cpu(device->ib_device,
 		tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
 
-	memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
+	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
 	tx_desc->iser_header.flags = ISER_VER;
 	tx_desc->num_sge = 1;
 }
@@ -174,73 +175,63 @@ static void iser_create_send_desc(struct iser_conn	*iser_conn,
 static void iser_free_login_buf(struct iser_conn *iser_conn)
 {
 	struct iser_device *device = iser_conn->ib_conn.device;
+	struct iser_login_desc *desc = &iser_conn->login_desc;
 
-	if (!iser_conn->login_buf)
+	if (!desc->req)
 		return;
 
-	if (iser_conn->login_req_dma)
-		ib_dma_unmap_single(device->ib_device,
-				    iser_conn->login_req_dma,
-				    ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
+	ib_dma_unmap_single(device->ib_device, desc->req_dma,
+			    ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
 
-	if (iser_conn->login_resp_dma)
-		ib_dma_unmap_single(device->ib_device,
-				    iser_conn->login_resp_dma,
-				    ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
+	ib_dma_unmap_single(device->ib_device, desc->rsp_dma,
+			    ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
 
-	kfree(iser_conn->login_buf);
+	kfree(desc->req);
+	kfree(desc->rsp);
 
 	/* make sure we never redo any unmapping */
-	iser_conn->login_req_dma = 0;
-	iser_conn->login_resp_dma = 0;
-	iser_conn->login_buf = NULL;
+	desc->req = NULL;
+	desc->rsp = NULL;
 }
 
 static int iser_alloc_login_buf(struct iser_conn *iser_conn)
 {
 	struct iser_device *device = iser_conn->ib_conn.device;
-	int			req_err, resp_err;
-
-	BUG_ON(device == NULL);
-
-	iser_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
-				     ISER_RX_LOGIN_SIZE, GFP_KERNEL);
-	if (!iser_conn->login_buf)
-		goto out_err;
-
-	iser_conn->login_req_buf  = iser_conn->login_buf;
-	iser_conn->login_resp_buf = iser_conn->login_buf +
-						ISCSI_DEF_MAX_RECV_SEG_LEN;
-
-	iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
-						     iser_conn->login_req_buf,
-						     ISCSI_DEF_MAX_RECV_SEG_LEN,
-						     DMA_TO_DEVICE);
-
-	iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
-						      iser_conn->login_resp_buf,
-						      ISER_RX_LOGIN_SIZE,
-						      DMA_FROM_DEVICE);
-
-	req_err  = ib_dma_mapping_error(device->ib_device,
-					iser_conn->login_req_dma);
-	resp_err = ib_dma_mapping_error(device->ib_device,
-					iser_conn->login_resp_dma);
-
-	if (req_err || resp_err) {
-		if (req_err)
-			iser_conn->login_req_dma = 0;
-		if (resp_err)
-			iser_conn->login_resp_dma = 0;
-		goto free_login_buf;
-	}
+	struct iser_login_desc *desc = &iser_conn->login_desc;
+
+	desc->req = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN, GFP_KERNEL);
+	if (!desc->req)
+		return -ENOMEM;
+
+	desc->req_dma = ib_dma_map_single(device->ib_device, desc->req,
+					  ISCSI_DEF_MAX_RECV_SEG_LEN,
+					  DMA_TO_DEVICE);
+	if (ib_dma_mapping_error(device->ib_device,
+				desc->req_dma))
+		goto free_req;
+
+	desc->rsp = kmalloc(ISER_RX_LOGIN_SIZE, GFP_KERNEL);
+	if (!desc->rsp)
+		goto unmap_req;
+
+	desc->rsp_dma = ib_dma_map_single(device->ib_device, desc->rsp,
+					   ISER_RX_LOGIN_SIZE,
+					   DMA_FROM_DEVICE);
+	if (ib_dma_mapping_error(device->ib_device,
+				desc->rsp_dma))
+		goto free_rsp;
+
 	return 0;
 
-free_login_buf:
-	iser_free_login_buf(iser_conn);
+free_rsp:
+	kfree(desc->rsp);
+unmap_req:
+	ib_dma_unmap_single(device->ib_device, desc->req_dma,
+			    ISCSI_DEF_MAX_RECV_SEG_LEN,
+			    DMA_TO_DEVICE);
+free_req:
+	kfree(desc->req);
 
-out_err:
-	iser_err("unable to alloc or map login buf\n");
 	return -ENOMEM;
 }
 
@@ -280,11 +271,11 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
 			goto rx_desc_dma_map_failed;
 
 		rx_desc->dma_addr = dma_addr;
-
+		rx_desc->cqe.done = iser_task_rsp;
 		rx_sg = &rx_desc->rx_sg;
-		rx_sg->addr   = rx_desc->dma_addr;
+		rx_sg->addr = rx_desc->dma_addr;
 		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
-		rx_sg->lkey   = device->pd->local_dma_lkey;
+		rx_sg->lkey = device->pd->local_dma_lkey;
 	}
 
 	iser_conn->rx_desc_head = 0;
@@ -383,6 +374,7 @@ int iser_send_command(struct iscsi_conn *conn,
 
 	/* build the tx desc regd header and add it to the tx desc dto */
 	tx_desc->type = ISCSI_TX_SCSI_COMMAND;
+	tx_desc->cqe.done = iser_cmd_comp;
 	iser_create_send_desc(iser_conn, tx_desc);
 
 	if (hdr->flags & ISCSI_FLAG_CMD_READ) {
@@ -464,6 +456,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
 	}
 
 	tx_desc->type = ISCSI_TX_DATAOUT;
+	tx_desc->cqe.done = iser_dataout_comp;
 	tx_desc->iser_header.flags = ISER_VER;
 	memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr));
 
@@ -513,6 +506,7 @@ int iser_send_control(struct iscsi_conn *conn,
 
 	/* build the tx desc regd header and add it to the tx desc dto */
 	mdesc->type = ISCSI_TX_CONTROL;
+	mdesc->cqe.done = iser_ctrl_comp;
 	iser_create_send_desc(iser_conn, mdesc);
 
 	device = iser_conn->ib_conn.device;
@@ -520,25 +514,25 @@ int iser_send_control(struct iscsi_conn *conn,
 	data_seg_len = ntoh24(task->hdr->dlength);
 
 	if (data_seg_len > 0) {
+		struct iser_login_desc *desc = &iser_conn->login_desc;
 		struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
+
 		if (task != conn->login_task) {
 			iser_err("data present on non login task!!!\n");
 			goto send_control_error;
 		}
 
-		ib_dma_sync_single_for_cpu(device->ib_device,
-			iser_conn->login_req_dma, task->data_count,
-			DMA_TO_DEVICE);
+		ib_dma_sync_single_for_cpu(device->ib_device, desc->req_dma,
+					   task->data_count, DMA_TO_DEVICE);
 
-		memcpy(iser_conn->login_req_buf, task->data, task->data_count);
+		memcpy(desc->req, task->data, task->data_count);
 
-		ib_dma_sync_single_for_device(device->ib_device,
-			iser_conn->login_req_dma, task->data_count,
-			DMA_TO_DEVICE);
+		ib_dma_sync_single_for_device(device->ib_device, desc->req_dma,
+					      task->data_count, DMA_TO_DEVICE);
 
-		tx_dsg->addr    = iser_conn->login_req_dma;
-		tx_dsg->length  = task->data_count;
-		tx_dsg->lkey    = device->pd->local_dma_lkey;
+		tx_dsg->addr = desc->req_dma;
+		tx_dsg->length = task->data_count;
+		tx_dsg->lkey = device->pd->local_dma_lkey;
 		mdesc->num_sge = 2;
 	}
 
@@ -562,41 +556,126 @@ send_control_error:
 	return err;
 }
 
-/**
- * iser_rcv_dto_completion - recv DTO completion
- */
-void iser_rcv_completion(struct iser_rx_desc *rx_desc,
-			 unsigned long rx_xfer_len,
-			 struct ib_conn *ib_conn)
+void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
-						   ib_conn);
+	struct ib_conn *ib_conn = wc->qp->qp_context;
+	struct iser_conn *iser_conn = to_iser_conn(ib_conn);
+	struct iser_login_desc *desc = iser_login(wc->wr_cqe);
 	struct iscsi_hdr *hdr;
-	u64 rx_dma;
-	int rx_buflen, outstanding, count, err;
+	char *data;
+	int length;
 
-	/* differentiate between login to all other PDUs */
-	if ((char *)rx_desc == iser_conn->login_resp_buf) {
-		rx_dma = iser_conn->login_resp_dma;
-		rx_buflen = ISER_RX_LOGIN_SIZE;
-	} else {
-		rx_dma = rx_desc->dma_addr;
-		rx_buflen = ISER_RX_PAYLOAD_SIZE;
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		iser_err_comp(wc, "login_rsp");
+		return;
+	}
+
+	ib_dma_sync_single_for_cpu(ib_conn->device->ib_device,
+				   desc->rsp_dma, ISER_RX_LOGIN_SIZE,
+				   DMA_FROM_DEVICE);
+
+	hdr = desc->rsp + sizeof(struct iser_ctrl);
+	data = desc->rsp + ISER_HEADERS_LEN;
+	length = wc->byte_len - ISER_HEADERS_LEN;
+
+	iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode,
+		 hdr->itt, length);
+
+	iscsi_iser_recv(iser_conn->iscsi_conn, hdr, data, length);
+
+	ib_dma_sync_single_for_device(ib_conn->device->ib_device,
+				      desc->rsp_dma, ISER_RX_LOGIN_SIZE,
+				      DMA_FROM_DEVICE);
+
+	ib_conn->post_recv_buf_count--;
+}
+
+static inline void
+iser_inv_desc(struct iser_fr_desc *desc, u32 rkey)
+{
+	if (likely(rkey == desc->rsc.mr->rkey))
+		desc->rsc.mr_valid = 0;
+	else if (likely(rkey == desc->pi_ctx->sig_mr->rkey))
+		desc->pi_ctx->sig_mr_valid = 0;
+}
+
+static int
+iser_check_remote_inv(struct iser_conn *iser_conn,
+		      struct ib_wc *wc,
+		      struct iscsi_hdr *hdr)
+{
+	if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {
+		struct iscsi_task *task;
+		u32 rkey = wc->ex.invalidate_rkey;
+
+		iser_dbg("conn %p: remote invalidation for rkey %#x\n",
+			 iser_conn, rkey);
+
+		if (unlikely(!iser_conn->snd_w_inv)) {
+			iser_err("conn %p: unexepected remote invalidation, "
+				 "terminating connection\n", iser_conn);
+			return -EPROTO;
+		}
+
+		task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt);
+		if (likely(task)) {
+			struct iscsi_iser_task *iser_task = task->dd_data;
+			struct iser_fr_desc *desc;
+
+			if (iser_task->dir[ISER_DIR_IN]) {
+				desc = iser_task->rdma_reg[ISER_DIR_IN].mem_h;
+				iser_inv_desc(desc, rkey);
+			}
+
+			if (iser_task->dir[ISER_DIR_OUT]) {
+				desc = iser_task->rdma_reg[ISER_DIR_OUT].mem_h;
+				iser_inv_desc(desc, rkey);
+			}
+		} else {
+			iser_err("failed to get task for itt=%d\n", hdr->itt);
+			return -EINVAL;
+		}
 	}
 
-	ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
-				   rx_buflen, DMA_FROM_DEVICE);
+	return 0;
+}
 
-	hdr = &rx_desc->iscsi_header;
+
+void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct ib_conn *ib_conn = wc->qp->qp_context;
+	struct iser_conn *iser_conn = to_iser_conn(ib_conn);
+	struct iser_rx_desc *desc = iser_rx(wc->wr_cqe);
+	struct iscsi_hdr *hdr;
+	int length;
+	int outstanding, count, err;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		iser_err_comp(wc, "task_rsp");
+		return;
+	}
+
+	ib_dma_sync_single_for_cpu(ib_conn->device->ib_device,
+				   desc->dma_addr, ISER_RX_PAYLOAD_SIZE,
+				   DMA_FROM_DEVICE);
+
+	hdr = &desc->iscsi_header;
+	length = wc->byte_len - ISER_HEADERS_LEN;
 
 	iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode,
-			hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN));
+		 hdr->itt, length);
+
+	if (iser_check_remote_inv(iser_conn, wc, hdr)) {
+		iscsi_conn_failure(iser_conn->iscsi_conn,
+				   ISCSI_ERR_CONN_FAILED);
+		return;
+	}
 
-	iscsi_iser_recv(iser_conn->iscsi_conn, hdr, rx_desc->data,
-			rx_xfer_len - ISER_HEADERS_LEN);
+	iscsi_iser_recv(iser_conn->iscsi_conn, hdr, desc->data, length);
 
-	ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
-				      rx_buflen, DMA_FROM_DEVICE);
+	ib_dma_sync_single_for_device(ib_conn->device->ib_device,
+				      desc->dma_addr, ISER_RX_PAYLOAD_SIZE,
+				      DMA_FROM_DEVICE);
 
 	/* decrementing conn->post_recv_buf_count only --after-- freeing the   *
 	 * task eliminates the need to worry on tasks which are completed in   *
@@ -604,9 +683,6 @@ void iser_rcv_completion(struct iser_rx_desc *rx_desc,
 	 * for the posted rx bufs refcount to become zero handles everything   */
 	ib_conn->post_recv_buf_count--;
 
-	if (rx_dma == iser_conn->login_resp_dma)
-		return;
-
 	outstanding = ib_conn->post_recv_buf_count;
 	if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
 		count = min(iser_conn->qp_max_recv_dtos - outstanding,
@@ -617,26 +693,47 @@ void iser_rcv_completion(struct iser_rx_desc *rx_desc,
 	}
 }
 
-void iser_snd_completion(struct iser_tx_desc *tx_desc,
-			struct ib_conn *ib_conn)
+void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc)
 {
+	if (unlikely(wc->status != IB_WC_SUCCESS))
+		iser_err_comp(wc, "command");
+}
+
+void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct iser_tx_desc *desc = iser_tx(wc->wr_cqe);
 	struct iscsi_task *task;
-	struct iser_device *device = ib_conn->device;
 
-	if (tx_desc->type == ISCSI_TX_DATAOUT) {
-		ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
-					ISER_HEADERS_LEN, DMA_TO_DEVICE);
-		kmem_cache_free(ig.desc_cache, tx_desc);
-		tx_desc = NULL;
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		iser_err_comp(wc, "control");
+		return;
 	}
 
-	if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) {
-		/* this arithmetic is legal by libiscsi dd_data allocation */
-		task = (void *) ((long)(void *)tx_desc -
-				  sizeof(struct iscsi_task));
-		if (task->hdr->itt == RESERVED_ITT)
-			iscsi_put_task(task);
-	}
+	/* this arithmetic is legal by libiscsi dd_data allocation */
+	task = (void *)desc - sizeof(struct iscsi_task);
+	if (task->hdr->itt == RESERVED_ITT)
+		iscsi_put_task(task);
+}
+
+void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct iser_tx_desc *desc = iser_tx(wc->wr_cqe);
+	struct ib_conn *ib_conn = wc->qp->qp_context;
+	struct iser_device *device = ib_conn->device;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS))
+		iser_err_comp(wc, "dataout");
+
+	ib_dma_unmap_single(device->ib_device, desc->dma_addr,
+			    ISER_HEADERS_LEN, DMA_TO_DEVICE);
+	kmem_cache_free(ig.desc_cache, desc);
+}
+
+void iser_last_comp(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct ib_conn *ib_conn = wc->qp->qp_context;
+
+	complete(&ib_conn->last_comp);
 }
 
 void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index ea765fb9664d..9a391cc5b9b3 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -49,7 +49,7 @@ int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 		     struct iser_reg_resources *rsc,
 		     struct iser_mem_reg *mem_reg);
 
-static struct iser_reg_ops fastreg_ops = {
+static const struct iser_reg_ops fastreg_ops = {
 	.alloc_reg_res	= iser_alloc_fastreg_pool,
 	.free_reg_res	= iser_free_fastreg_pool,
 	.reg_mem	= iser_fast_reg_mr,
@@ -58,7 +58,7 @@ static struct iser_reg_ops fastreg_ops = {
 	.reg_desc_put	= iser_reg_desc_put_fr,
 };
 
-static struct iser_reg_ops fmr_ops = {
+static const struct iser_reg_ops fmr_ops = {
 	.alloc_reg_res	= iser_alloc_fmr_pool,
 	.free_reg_res	= iser_free_fmr_pool,
 	.reg_mem	= iser_fast_reg_fmr,
@@ -67,19 +67,24 @@ static struct iser_reg_ops fmr_ops = {
 	.reg_desc_put	= iser_reg_desc_put_fmr,
 };
 
+void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc)
+{
+	iser_err_comp(wc, "memreg");
+}
+
 int iser_assign_reg_ops(struct iser_device *device)
 {
-	struct ib_device_attr *dev_attr = &device->dev_attr;
+	struct ib_device *ib_dev = device->ib_device;
 
 	/* Assign function handles  - based on FMR support */
-	if (device->ib_device->alloc_fmr && device->ib_device->dealloc_fmr &&
-	    device->ib_device->map_phys_fmr && device->ib_device->unmap_fmr) {
+	if (ib_dev->alloc_fmr && ib_dev->dealloc_fmr &&
+	    ib_dev->map_phys_fmr && ib_dev->unmap_fmr) {
 		iser_info("FMR supported, using FMR for registration\n");
 		device->reg_ops = &fmr_ops;
-	} else
-	if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
+	} else if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
 		iser_info("FastReg supported, using FastReg for registration\n");
 		device->reg_ops = &fastreg_ops;
+		device->remote_inv_sup = iser_always_reg;
 	} else {
 		iser_err("IB device does not support FMRs nor FastRegs, can't register memory\n");
 		return -1;
@@ -131,67 +136,6 @@ iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
 {
 }
 
-#define IS_4K_ALIGNED(addr)	((((unsigned long)addr) & ~MASK_4K) == 0)
-
-/**
- * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
- * and returns the length of resulting physical address array (may be less than
- * the original due to possible compaction).
- *
- * we build a "page vec" under the assumption that the SG meets the RDMA
- * alignment requirements. Other then the first and last SG elements, all
- * the "internal" elements can be compacted into a list whose elements are
- * dma addresses of physical pages. The code supports also the weird case
- * where --few fragments of the same page-- are present in the SG as
- * consecutive elements. Also, it handles one entry SG.
- */
-
-static int iser_sg_to_page_vec(struct iser_data_buf *data,
-			       struct ib_device *ibdev, u64 *pages,
-			       int *offset, int *data_size)
-{
-	struct scatterlist *sg, *sgl = data->sg;
-	u64 start_addr, end_addr, page, chunk_start = 0;
-	unsigned long total_sz = 0;
-	unsigned int dma_len;
-	int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
-
-	/* compute the offset of first element */
-	*offset = (u64) sgl[0].offset & ~MASK_4K;
-
-	new_chunk = 1;
-	cur_page  = 0;
-	for_each_sg(sgl, sg, data->dma_nents, i) {
-		start_addr = ib_sg_dma_address(ibdev, sg);
-		if (new_chunk)
-			chunk_start = start_addr;
-		dma_len = ib_sg_dma_len(ibdev, sg);
-		end_addr = start_addr + dma_len;
-		total_sz += dma_len;
-
-		/* collect page fragments until aligned or end of SG list */
-		if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
-			new_chunk = 0;
-			continue;
-		}
-		new_chunk = 1;
-
-		/* address of the first page in the contiguous chunk;
-		   masking relevant for the very first SG entry,
-		   which might be unaligned */
-		page = chunk_start & MASK_4K;
-		do {
-			pages[cur_page++] = page;
-			page += SIZE_4K;
-		} while (page < end_addr);
-	}
-
-	*data_size = total_sz;
-	iser_dbg("page_vec->data_size:%d cur_page %d\n",
-		 *data_size, cur_page);
-	return cur_page;
-}
-
 static void iser_data_buf_dump(struct iser_data_buf *data,
 			       struct ib_device *ibdev)
 {
@@ -210,10 +154,10 @@ static void iser_dump_page_vec(struct iser_page_vec *page_vec)
 {
 	int i;
 
-	iser_err("page vec length %d data size %d\n",
-		 page_vec->length, page_vec->data_size);
-	for (i = 0; i < page_vec->length; i++)
-		iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]);
+	iser_err("page vec npages %d data length %d\n",
+		 page_vec->npages, page_vec->fake_mr.length);
+	for (i = 0; i < page_vec->npages; i++)
+		iser_err("vec[%d]: %llx\n", i, page_vec->pages[i]);
 }
 
 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
@@ -251,7 +195,11 @@ iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
 	struct scatterlist *sg = mem->sg;
 
 	reg->sge.lkey = device->pd->local_dma_lkey;
-	reg->rkey = device->mr->rkey;
+	/*
+	 * FIXME: rework the registration code path to differentiate
+	 * rkey/lkey use cases
+	 */
+	reg->rkey = device->mr ? device->mr->rkey : 0;
 	reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]);
 	reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]);
 
@@ -262,11 +210,16 @@ iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
 	return 0;
 }
 
-/**
- * iser_reg_page_vec - Register physical memory
- *
- * returns: 0 on success, errno code on failure
- */
+static int iser_set_page(struct ib_mr *mr, u64 addr)
+{
+	struct iser_page_vec *page_vec =
+		container_of(mr, struct iser_page_vec, fake_mr);
+
+	page_vec->pages[page_vec->npages++] = addr;
+
+	return 0;
+}
+
 static
 int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
 		      struct iser_data_buf *mem,
@@ -280,22 +233,19 @@ int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
 	struct ib_pool_fmr *fmr;
 	int ret, plen;
 
-	plen = iser_sg_to_page_vec(mem, device->ib_device,
-				   page_vec->pages,
-				   &page_vec->offset,
-				   &page_vec->data_size);
-	page_vec->length = plen;
-	if (plen * SIZE_4K < page_vec->data_size) {
+	page_vec->npages = 0;
+	page_vec->fake_mr.page_size = SIZE_4K;
+	plen = ib_sg_to_pages(&page_vec->fake_mr, mem->sg,
+			      mem->size, iser_set_page);
+	if (unlikely(plen < mem->size)) {
 		iser_err("page vec too short to hold this SG\n");
 		iser_data_buf_dump(mem, device->ib_device);
 		iser_dump_page_vec(page_vec);
 		return -EINVAL;
 	}
 
-	fmr  = ib_fmr_pool_map_phys(fmr_pool,
-				    page_vec->pages,
-				    page_vec->length,
-				    page_vec->pages[0]);
+	fmr  = ib_fmr_pool_map_phys(fmr_pool, page_vec->pages,
+				    page_vec->npages, page_vec->pages[0]);
 	if (IS_ERR(fmr)) {
 		ret = PTR_ERR(fmr);
 		iser_err("ib_fmr_pool_map_phys failed: %d\n", ret);
@@ -304,8 +254,8 @@ int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
 
 	reg->sge.lkey = fmr->fmr->lkey;
 	reg->rkey = fmr->fmr->rkey;
-	reg->sge.addr = page_vec->pages[0] + page_vec->offset;
-	reg->sge.length = page_vec->data_size;
+	reg->sge.addr = page_vec->fake_mr.iova;
+	reg->sge.length = page_vec->fake_mr.length;
 	reg->mem_h = fmr;
 
 	iser_dbg("fmr reg: lkey=0x%x, rkey=0x%x, addr=0x%llx,"
@@ -413,19 +363,16 @@ iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
 		*mask |= ISER_CHECK_GUARD;
 }
 
-static void
-iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
+static inline void
+iser_inv_rkey(struct ib_send_wr *inv_wr,
+	      struct ib_mr *mr,
+	      struct ib_cqe *cqe)
 {
-	u32 rkey;
-
 	inv_wr->opcode = IB_WR_LOCAL_INV;
-	inv_wr->wr_id = ISER_FASTREG_LI_WRID;
+	inv_wr->wr_cqe = cqe;
 	inv_wr->ex.invalidate_rkey = mr->rkey;
 	inv_wr->send_flags = 0;
 	inv_wr->num_sge = 0;
-
-	rkey = ib_inc_rkey(mr->rkey);
-	ib_update_fast_reg_key(mr, rkey);
 }
 
 static int
@@ -437,7 +384,9 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
 {
 	struct iser_tx_desc *tx_desc = &iser_task->desc;
 	struct ib_sig_attrs *sig_attrs = &tx_desc->sig_attrs;
+	struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
 	struct ib_sig_handover_wr *wr;
+	struct ib_mr *mr = pi_ctx->sig_mr;
 	int ret;
 
 	memset(sig_attrs, 0, sizeof(*sig_attrs));
@@ -447,17 +396,19 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
 
 	iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask);
 
-	if (!pi_ctx->sig_mr_valid)
-		iser_inv_rkey(iser_tx_next_wr(tx_desc), pi_ctx->sig_mr);
+	if (pi_ctx->sig_mr_valid)
+		iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
+
+	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
 
 	wr = sig_handover_wr(iser_tx_next_wr(tx_desc));
 	wr->wr.opcode = IB_WR_REG_SIG_MR;
-	wr->wr.wr_id = ISER_FASTREG_LI_WRID;
+	wr->wr.wr_cqe = cqe;
 	wr->wr.sg_list = &data_reg->sge;
 	wr->wr.num_sge = 1;
 	wr->wr.send_flags = 0;
 	wr->sig_attrs = sig_attrs;
-	wr->sig_mr = pi_ctx->sig_mr;
+	wr->sig_mr = mr;
 	if (scsi_prot_sg_count(iser_task->sc))
 		wr->prot = &prot_reg->sge;
 	else
@@ -465,10 +416,10 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
 	wr->access_flags = IB_ACCESS_LOCAL_WRITE |
 			   IB_ACCESS_REMOTE_READ |
 			   IB_ACCESS_REMOTE_WRITE;
-	pi_ctx->sig_mr_valid = 0;
+	pi_ctx->sig_mr_valid = 1;
 
-	sig_reg->sge.lkey = pi_ctx->sig_mr->lkey;
-	sig_reg->rkey = pi_ctx->sig_mr->rkey;
+	sig_reg->sge.lkey = mr->lkey;
+	sig_reg->rkey = mr->rkey;
 	sig_reg->sge.addr = 0;
 	sig_reg->sge.length = scsi_transfer_length(iser_task->sc);
 
@@ -485,12 +436,15 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 			    struct iser_mem_reg *reg)
 {
 	struct iser_tx_desc *tx_desc = &iser_task->desc;
+	struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
 	struct ib_mr *mr = rsc->mr;
 	struct ib_reg_wr *wr;
 	int n;
 
-	if (!rsc->mr_valid)
-		iser_inv_rkey(iser_tx_next_wr(tx_desc), mr);
+	if (rsc->mr_valid)
+		iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
+
+	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
 
 	n = ib_map_mr_sg(mr, mem->sg, mem->size, SIZE_4K);
 	if (unlikely(n != mem->size)) {
@@ -501,7 +455,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 
 	wr = reg_wr(iser_tx_next_wr(tx_desc));
 	wr->wr.opcode = IB_WR_REG_MR;
-	wr->wr.wr_id = ISER_FASTREG_LI_WRID;
+	wr->wr.wr_cqe = cqe;
 	wr->wr.send_flags = 0;
 	wr->wr.num_sge = 0;
 	wr->mr = mr;
@@ -510,7 +464,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 		     IB_ACCESS_REMOTE_WRITE |
 		     IB_ACCESS_REMOTE_READ;
 
-	rsc->mr_valid = 0;
+	rsc->mr_valid = 1;
 
 	reg->sge.lkey = mr->lkey;
 	reg->rkey = mr->rkey;
@@ -554,7 +508,8 @@ iser_reg_data_sg(struct iscsi_iser_task *task,
 }
 
 int iser_reg_rdma_mem(struct iscsi_iser_task *task,
-		      enum iser_data_dir dir)
+		      enum iser_data_dir dir,
+		      bool all_imm)
 {
 	struct ib_conn *ib_conn = &task->iser_conn->ib_conn;
 	struct iser_device *device = ib_conn->device;
@@ -565,8 +520,8 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
 	bool use_dma_key;
 	int err;
 
-	use_dma_key = (mem->dma_nents == 1 && !iser_always_reg &&
-		       scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL);
+	use_dma_key = mem->dma_nents == 1 && (all_imm || !iser_always_reg) &&
+		      scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL;
 
 	if (!use_dma_key) {
 		desc = device->reg_ops->reg_desc_get(ib_conn);
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 42f4da620f2e..40c0f4978e2f 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -44,17 +44,6 @@
 #define ISER_MAX_CQ_LEN		(ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
 				 ISCSI_ISER_MAX_CONN)
 
-static int iser_cq_poll_limit = 512;
-
-static void iser_cq_tasklet_fn(unsigned long data);
-static void iser_cq_callback(struct ib_cq *cq, void *cq_context);
-
-static void iser_cq_event_callback(struct ib_event *cause, void *context)
-{
-	iser_err("cq event %s (%d)\n",
-		 ib_event_msg(cause->event), cause->event);
-}
-
 static void iser_qp_event_callback(struct ib_event *cause, void *context)
 {
 	iser_err("qp event %s (%d)\n",
@@ -78,59 +67,40 @@ static void iser_event_handler(struct ib_event_handler *handler,
  */
 static int iser_create_device_ib_res(struct iser_device *device)
 {
-	struct ib_device_attr *dev_attr = &device->dev_attr;
+	struct ib_device *ib_dev = device->ib_device;
 	int ret, i, max_cqe;
 
-	ret = ib_query_device(device->ib_device, dev_attr);
-	if (ret) {
-		pr_warn("Query device failed for %s\n", device->ib_device->name);
-		return ret;
-	}
-
 	ret = iser_assign_reg_ops(device);
 	if (ret)
 		return ret;
 
 	device->comps_used = min_t(int, num_online_cpus(),
-				 device->ib_device->num_comp_vectors);
+				 ib_dev->num_comp_vectors);
 
 	device->comps = kcalloc(device->comps_used, sizeof(*device->comps),
 				GFP_KERNEL);
 	if (!device->comps)
 		goto comps_err;
 
-	max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe);
+	max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe);
 
 	iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
-		  device->comps_used, device->ib_device->name,
-		  device->ib_device->num_comp_vectors, max_cqe);
+		  device->comps_used, ib_dev->name,
+		  ib_dev->num_comp_vectors, max_cqe);
 
-	device->pd = ib_alloc_pd(device->ib_device);
+	device->pd = ib_alloc_pd(ib_dev);
 	if (IS_ERR(device->pd))
 		goto pd_err;
 
 	for (i = 0; i < device->comps_used; i++) {
-		struct ib_cq_init_attr cq_attr = {};
 		struct iser_comp *comp = &device->comps[i];
 
-		comp->device = device;
-		cq_attr.cqe = max_cqe;
-		cq_attr.comp_vector = i;
-		comp->cq = ib_create_cq(device->ib_device,
-					iser_cq_callback,
-					iser_cq_event_callback,
-					(void *)comp,
-					&cq_attr);
+		comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i,
+				       IB_POLL_SOFTIRQ);
 		if (IS_ERR(comp->cq)) {
 			comp->cq = NULL;
 			goto cq_err;
 		}
-
-		if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP))
-			goto cq_err;
-
-		tasklet_init(&comp->tasklet, iser_cq_tasklet_fn,
-			     (unsigned long)comp);
 	}
 
 	if (!iser_always_reg) {
@@ -140,11 +110,11 @@ static int iser_create_device_ib_res(struct iser_device *device)
 
 		device->mr = ib_get_dma_mr(device->pd, access);
 		if (IS_ERR(device->mr))
-			goto dma_mr_err;
+			goto cq_err;
 	}
 
-	INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device,
-				iser_event_handler);
+	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
+			      iser_event_handler);
 	if (ib_register_event_handler(&device->event_handler))
 		goto handler_err;
 
@@ -153,15 +123,12 @@ static int iser_create_device_ib_res(struct iser_device *device)
 handler_err:
 	if (device->mr)
 		ib_dereg_mr(device->mr);
-dma_mr_err:
-	for (i = 0; i < device->comps_used; i++)
-		tasklet_kill(&device->comps[i].tasklet);
 cq_err:
 	for (i = 0; i < device->comps_used; i++) {
 		struct iser_comp *comp = &device->comps[i];
 
 		if (comp->cq)
-			ib_destroy_cq(comp->cq);
+			ib_free_cq(comp->cq);
 	}
 	ib_dealloc_pd(device->pd);
 pd_err:
@@ -182,8 +149,7 @@ static void iser_free_device_ib_res(struct iser_device *device)
 	for (i = 0; i < device->comps_used; i++) {
 		struct iser_comp *comp = &device->comps[i];
 
-		tasklet_kill(&comp->tasklet);
-		ib_destroy_cq(comp->cq);
+		ib_free_cq(comp->cq);
 		comp->cq = NULL;
 	}
 
@@ -299,7 +265,7 @@ iser_alloc_reg_res(struct ib_device *ib_device,
 		iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret);
 		return ret;
 	}
-	res->mr_valid = 1;
+	res->mr_valid = 0;
 
 	return 0;
 }
@@ -336,7 +302,7 @@ iser_alloc_pi_ctx(struct ib_device *ib_device,
 		ret = PTR_ERR(pi_ctx->sig_mr);
 		goto sig_mr_failure;
 	}
-	pi_ctx->sig_mr_valid = 1;
+	pi_ctx->sig_mr_valid = 0;
 	desc->pi_ctx->sig_protected = 0;
 
 	return 0;
@@ -461,10 +427,9 @@ void iser_free_fastreg_pool(struct ib_conn *ib_conn)
  */
 static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
 {
-	struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
-						   ib_conn);
+	struct iser_conn *iser_conn = to_iser_conn(ib_conn);
 	struct iser_device	*device;
-	struct ib_device_attr *dev_attr;
+	struct ib_device	*ib_dev;
 	struct ib_qp_init_attr	init_attr;
 	int			ret = -ENOMEM;
 	int index, min_index = 0;
@@ -472,7 +437,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
 	BUG_ON(ib_conn->device == NULL);
 
 	device = ib_conn->device;
-	dev_attr = &device->dev_attr;
+	ib_dev = device->ib_device;
 
 	memset(&init_attr, 0, sizeof init_attr);
 
@@ -503,16 +468,16 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
 		iser_conn->max_cmds =
 			ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
 	} else {
-		if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
+		if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
 			init_attr.cap.max_send_wr  = ISER_QP_MAX_REQ_DTOS + 1;
 			iser_conn->max_cmds =
 				ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
 		} else {
-			init_attr.cap.max_send_wr = dev_attr->max_qp_wr;
+			init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr;
 			iser_conn->max_cmds =
-				ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr);
+				ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr);
 			iser_dbg("device %s supports max_send_wr %d\n",
-				 device->ib_device->name, dev_attr->max_qp_wr);
+				 device->ib_device->name, ib_dev->attrs.max_qp_wr);
 		}
 	}
 
@@ -724,13 +689,13 @@ int iser_conn_terminate(struct iser_conn *iser_conn)
 				 iser_conn, err);
 
 		/* post an indication that all flush errors were consumed */
-		err = ib_post_send(ib_conn->qp, &ib_conn->beacon, &bad_wr);
+		err = ib_post_send(ib_conn->qp, &ib_conn->last, &bad_wr);
 		if (err) {
-			iser_err("conn %p failed to post beacon", ib_conn);
+			iser_err("conn %p failed to post last wr", ib_conn);
 			return 1;
 		}
 
-		wait_for_completion(&ib_conn->flush_comp);
+		wait_for_completion(&ib_conn->last_comp);
 	}
 
 	return 1;
@@ -756,7 +721,7 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
 
 	sg_tablesize = DIV_ROUND_UP(max_sectors * 512, SIZE_4K);
 	sup_sg_tablesize = min_t(unsigned, ISCSI_ISER_MAX_SG_TABLESIZE,
-				 device->dev_attr.max_fast_reg_page_list_len);
+				 device->ib_device->attrs.max_fast_reg_page_list_len);
 
 	if (sg_tablesize > sup_sg_tablesize) {
 		sg_tablesize = sup_sg_tablesize;
@@ -799,7 +764,7 @@ static void iser_addr_handler(struct rdma_cm_id *cma_id)
 
 	/* connection T10-PI support */
 	if (iser_pi_enable) {
-		if (!(device->dev_attr.device_cap_flags &
+		if (!(device->ib_device->attrs.device_cap_flags &
 		      IB_DEVICE_SIGNATURE_HANDOVER)) {
 			iser_warn("T10-PI requested but not supported on %s, "
 				  "continue without T10-PI\n",
@@ -841,16 +806,17 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
 		goto failure;
 
 	memset(&conn_param, 0, sizeof conn_param);
-	conn_param.responder_resources = device->dev_attr.max_qp_rd_atom;
+	conn_param.responder_resources = device->ib_device->attrs.max_qp_rd_atom;
 	conn_param.initiator_depth     = 1;
 	conn_param.retry_count	       = 7;
 	conn_param.rnr_retry_count     = 6;
 
 	memset(&req_hdr, 0, sizeof(req_hdr));
-	req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED |
-			ISER_SEND_W_INV_NOT_SUPPORTED);
-	conn_param.private_data		= (void *)&req_hdr;
-	conn_param.private_data_len	= sizeof(struct iser_cm_hdr);
+	req_hdr.flags = ISER_ZBVA_NOT_SUP;
+	if (!device->remote_inv_sup)
+		req_hdr.flags |= ISER_SEND_W_INV_NOT_SUP;
+	conn_param.private_data	= (void *)&req_hdr;
+	conn_param.private_data_len = sizeof(struct iser_cm_hdr);
 
 	ret = rdma_connect(cma_id, &conn_param);
 	if (ret) {
@@ -863,7 +829,8 @@ failure:
 	iser_connect_error(cma_id);
 }
 
-static void iser_connected_handler(struct rdma_cm_id *cma_id)
+static void iser_connected_handler(struct rdma_cm_id *cma_id,
+				   const void *private_data)
 {
 	struct iser_conn *iser_conn;
 	struct ib_qp_attr attr;
@@ -877,6 +844,15 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id)
 	(void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);
 	iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num);
 
+	if (private_data) {
+		u8 flags = *(u8 *)private_data;
+
+		iser_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP);
+	}
+
+	iser_info("conn %p: negotiated %s invalidation\n",
+		  iser_conn, iser_conn->snd_w_inv ? "remote" : "local");
+
 	iser_conn->state = ISER_CONN_UP;
 	complete(&iser_conn->up_completion);
 }
@@ -928,7 +904,7 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
 		iser_route_handler(cma_id);
 		break;
 	case RDMA_CM_EVENT_ESTABLISHED:
-		iser_connected_handler(cma_id);
+		iser_connected_handler(cma_id, event->param.conn.private_data);
 		break;
 	case RDMA_CM_EVENT_ADDR_ERROR:
 	case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -967,14 +943,21 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
 
 void iser_conn_init(struct iser_conn *iser_conn)
 {
+	struct ib_conn *ib_conn = &iser_conn->ib_conn;
+
 	iser_conn->state = ISER_CONN_INIT;
-	iser_conn->ib_conn.post_recv_buf_count = 0;
-	init_completion(&iser_conn->ib_conn.flush_comp);
 	init_completion(&iser_conn->stop_completion);
 	init_completion(&iser_conn->ib_completion);
 	init_completion(&iser_conn->up_completion);
 	INIT_LIST_HEAD(&iser_conn->conn_list);
 	mutex_init(&iser_conn->state_mutex);
+
+	ib_conn->post_recv_buf_count = 0;
+	ib_conn->reg_cqe.done = iser_reg_comp;
+	ib_conn->last_cqe.done = iser_last_comp;
+	ib_conn->last.wr_cqe = &ib_conn->last_cqe;
+	ib_conn->last.opcode = IB_WR_SEND;
+	init_completion(&ib_conn->last_comp);
 }
 
  /**
@@ -1000,9 +983,6 @@ int iser_connect(struct iser_conn   *iser_conn,
 
 	iser_conn->state = ISER_CONN_PENDING;
 
-	ib_conn->beacon.wr_id = ISER_BEACON_WRID;
-	ib_conn->beacon.opcode = IB_WR_SEND;
-
 	ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler,
 					 (void *)iser_conn,
 					 RDMA_PS_TCP, IB_QPT_RC);
@@ -1045,56 +1025,60 @@ connect_failure:
 
 int iser_post_recvl(struct iser_conn *iser_conn)
 {
-	struct ib_recv_wr rx_wr, *rx_wr_failed;
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
-	struct ib_sge	  sge;
+	struct iser_login_desc *desc = &iser_conn->login_desc;
+	struct ib_recv_wr wr, *wr_failed;
 	int ib_ret;
 
-	sge.addr   = iser_conn->login_resp_dma;
-	sge.length = ISER_RX_LOGIN_SIZE;
-	sge.lkey   = ib_conn->device->pd->local_dma_lkey;
+	desc->sge.addr = desc->rsp_dma;
+	desc->sge.length = ISER_RX_LOGIN_SIZE;
+	desc->sge.lkey = ib_conn->device->pd->local_dma_lkey;
 
-	rx_wr.wr_id   = (uintptr_t)iser_conn->login_resp_buf;
-	rx_wr.sg_list = &sge;
-	rx_wr.num_sge = 1;
-	rx_wr.next    = NULL;
+	desc->cqe.done = iser_login_rsp;
+	wr.wr_cqe = &desc->cqe;
+	wr.sg_list = &desc->sge;
+	wr.num_sge = 1;
+	wr.next = NULL;
 
 	ib_conn->post_recv_buf_count++;
-	ib_ret	= ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed);
+	ib_ret = ib_post_recv(ib_conn->qp, &wr, &wr_failed);
 	if (ib_ret) {
 		iser_err("ib_post_recv failed ret=%d\n", ib_ret);
 		ib_conn->post_recv_buf_count--;
 	}
+
 	return ib_ret;
 }
 
 int iser_post_recvm(struct iser_conn *iser_conn, int count)
 {
-	struct ib_recv_wr *rx_wr, *rx_wr_failed;
-	int i, ib_ret;
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
 	unsigned int my_rx_head = iser_conn->rx_desc_head;
 	struct iser_rx_desc *rx_desc;
+	struct ib_recv_wr *wr, *wr_failed;
+	int i, ib_ret;
 
-	for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
-		rx_desc		= &iser_conn->rx_descs[my_rx_head];
-		rx_wr->wr_id	= (uintptr_t)rx_desc;
-		rx_wr->sg_list	= &rx_desc->rx_sg;
-		rx_wr->num_sge	= 1;
-		rx_wr->next	= rx_wr + 1;
+	for (wr = ib_conn->rx_wr, i = 0; i < count; i++, wr++) {
+		rx_desc = &iser_conn->rx_descs[my_rx_head];
+		rx_desc->cqe.done = iser_task_rsp;
+		wr->wr_cqe = &rx_desc->cqe;
+		wr->sg_list = &rx_desc->rx_sg;
+		wr->num_sge = 1;
+		wr->next = wr + 1;
 		my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask;
 	}
 
-	rx_wr--;
-	rx_wr->next = NULL; /* mark end of work requests list */
+	wr--;
+	wr->next = NULL; /* mark end of work requests list */
 
 	ib_conn->post_recv_buf_count += count;
-	ib_ret	= ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed);
+	ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &wr_failed);
 	if (ib_ret) {
 		iser_err("ib_post_recv failed ret=%d\n", ib_ret);
 		ib_conn->post_recv_buf_count -= count;
 	} else
 		iser_conn->rx_desc_head = my_rx_head;
+
 	return ib_ret;
 }
 
@@ -1115,7 +1099,7 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
 				      DMA_TO_DEVICE);
 
 	wr->next = NULL;
-	wr->wr_id = (uintptr_t)tx_desc;
+	wr->wr_cqe = &tx_desc->cqe;
 	wr->sg_list = tx_desc->tx_sg;
 	wr->num_sge = tx_desc->num_sge;
 	wr->opcode = IB_WR_SEND;
@@ -1129,149 +1113,6 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
 	return ib_ret;
 }
 
-/**
- * is_iser_tx_desc - Indicate if the completion wr_id
- *     is a TX descriptor or not.
- * @iser_conn: iser connection
- * @wr_id: completion WR identifier
- *
- * Since we cannot rely on wc opcode in FLUSH errors
- * we must work around it by checking if the wr_id address
- * falls in the iser connection rx_descs buffer. If so
- * it is an RX descriptor, otherwize it is a TX.
- */
-static inline bool
-is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id)
-{
-	void *start = iser_conn->rx_descs;
-	int len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs);
-
-	if (wr_id >= start && wr_id < start + len)
-		return false;
-
-	return true;
-}
-
-/**
- * iser_handle_comp_error() - Handle error completion
- * @ib_conn:   connection RDMA resources
- * @wc:        work completion
- *
- * Notes: We may handle a FLUSH error completion and in this case
- *        we only cleanup in case TX type was DATAOUT. For non-FLUSH
- *        error completion we should also notify iscsi layer that
- *        connection is failed (in case we passed bind stage).
- */
-static void
-iser_handle_comp_error(struct ib_conn *ib_conn,
-		       struct ib_wc *wc)
-{
-	void *wr_id = (void *)(uintptr_t)wc->wr_id;
-	struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
-						   ib_conn);
-
-	if (wc->status != IB_WC_WR_FLUSH_ERR)
-		if (iser_conn->iscsi_conn)
-			iscsi_conn_failure(iser_conn->iscsi_conn,
-					   ISCSI_ERR_CONN_FAILED);
-
-	if (wc->wr_id == ISER_FASTREG_LI_WRID)
-		return;
-
-	if (is_iser_tx_desc(iser_conn, wr_id)) {
-		struct iser_tx_desc *desc = wr_id;
-
-		if (desc->type == ISCSI_TX_DATAOUT)
-			kmem_cache_free(ig.desc_cache, desc);
-	} else {
-		ib_conn->post_recv_buf_count--;
-	}
-}
-
-/**
- * iser_handle_wc - handle a single work completion
- * @wc: work completion
- *
- * Soft-IRQ context, work completion can be either
- * SEND or RECV, and can turn out successful or
- * with error (or flush error).
- */
-static void iser_handle_wc(struct ib_wc *wc)
-{
-	struct ib_conn *ib_conn;
-	struct iser_tx_desc *tx_desc;
-	struct iser_rx_desc *rx_desc;
-
-	ib_conn = wc->qp->qp_context;
-	if (likely(wc->status == IB_WC_SUCCESS)) {
-		if (wc->opcode == IB_WC_RECV) {
-			rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id;
-			iser_rcv_completion(rx_desc, wc->byte_len,
-					    ib_conn);
-		} else
-		if (wc->opcode == IB_WC_SEND) {
-			tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
-			iser_snd_completion(tx_desc, ib_conn);
-		} else {
-			iser_err("Unknown wc opcode %d\n", wc->opcode);
-		}
-	} else {
-		if (wc->status != IB_WC_WR_FLUSH_ERR)
-			iser_err("%s (%d): wr id %llx vend_err %x\n",
-				 ib_wc_status_msg(wc->status), wc->status,
-				 wc->wr_id, wc->vendor_err);
-		else
-			iser_dbg("%s (%d): wr id %llx\n",
-				 ib_wc_status_msg(wc->status), wc->status,
-				 wc->wr_id);
-
-		if (wc->wr_id == ISER_BEACON_WRID)
-			/* all flush errors were consumed */
-			complete(&ib_conn->flush_comp);
-		else
-			iser_handle_comp_error(ib_conn, wc);
-	}
-}
-
-/**
- * iser_cq_tasklet_fn - iSER completion polling loop
- * @data: iSER completion context
- *
- * Soft-IRQ context, polling connection CQ until
- * either CQ was empty or we exausted polling budget
- */
-static void iser_cq_tasklet_fn(unsigned long data)
-{
-	struct iser_comp *comp = (struct iser_comp *)data;
-	struct ib_cq *cq = comp->cq;
-	struct ib_wc *const wcs = comp->wcs;
-	int i, n, completed = 0;
-
-	while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
-		for (i = 0; i < n; i++)
-			iser_handle_wc(&wcs[i]);
-
-		completed += n;
-		if (completed >= iser_cq_poll_limit)
-			break;
-	}
-
-	/*
-	 * It is assumed here that arming CQ only once its empty
-	 * would not cause interrupts to be missed.
-	 */
-	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
-
-	iser_dbg("got %d completions\n", completed);
-}
-
-static void iser_cq_callback(struct ib_cq *cq, void *cq_context)
-{
-	struct iser_comp *comp = cq_context;
-
-	tasklet_schedule(&comp->tasklet);
-}
-
 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
 			     enum iser_data_dir cmd_dir, sector_t *sector)
 {
@@ -1319,3 +1160,21 @@ err:
 	/* Not alot we can do here, return ambiguous guard error */
 	return 0x1;
 }
+
+void iser_err_comp(struct ib_wc *wc, const char *type)
+{
+	if (wc->status != IB_WC_WR_FLUSH_ERR) {
+		struct iser_conn *iser_conn = to_iser_conn(wc->qp->qp_context);
+
+		iser_err("%s failure: %s (%d) vend_err %x\n", type,
+			 ib_wc_status_msg(wc->status), wc->status,
+			 wc->vendor_err);
+
+		if (iser_conn->iscsi_conn)
+			iscsi_conn_failure(iser_conn->iscsi_conn,
+					   ISCSI_ERR_CONN_FAILED);
+	} else {
+		iser_dbg("%s failure: %s (%d)\n", type,
+			 ib_wc_status_msg(wc->status), wc->status);
+	}
+}
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 8a51c3b5d657..f121e6129339 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -29,7 +29,6 @@
 #include <target/iscsi/iscsi_transport.h>
 #include <linux/semaphore.h>
 
-#include "isert_proto.h"
 #include "ib_isert.h"
 
 #define	ISERT_MAX_CONN		8
@@ -95,22 +94,6 @@ isert_qp_event_callback(struct ib_event *e, void *context)
 	}
 }
 
-static int
-isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
-{
-	int ret;
-
-	ret = ib_query_device(ib_dev, devattr);
-	if (ret) {
-		isert_err("ib_query_device() failed: %d\n", ret);
-		return ret;
-	}
-	isert_dbg("devattr->max_sge: %d\n", devattr->max_sge);
-	isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
-
-	return 0;
-}
-
 static struct isert_comp *
 isert_comp_get(struct isert_conn *isert_conn)
 {
@@ -157,9 +140,9 @@ isert_create_qp(struct isert_conn *isert_conn,
 	attr.recv_cq = comp->cq;
 	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
 	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
-	attr.cap.max_send_sge = device->dev_attr.max_sge;
-	isert_conn->max_sge = min(device->dev_attr.max_sge,
-				  device->dev_attr.max_sge_rd);
+	attr.cap.max_send_sge = device->ib_device->attrs.max_sge;
+	isert_conn->max_sge = min(device->ib_device->attrs.max_sge,
+				  device->ib_device->attrs.max_sge_rd);
 	attr.cap.max_recv_sge = 1;
 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 	attr.qp_type = IB_QPT_RC;
@@ -287,8 +270,7 @@ isert_free_comps(struct isert_device *device)
 }
 
 static int
-isert_alloc_comps(struct isert_device *device,
-		  struct ib_device_attr *attr)
+isert_alloc_comps(struct isert_device *device)
 {
 	int i, max_cqe, ret = 0;
 
@@ -308,7 +290,7 @@ isert_alloc_comps(struct isert_device *device,
 		return -ENOMEM;
 	}
 
-	max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe);
+	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
 
 	for (i = 0; i < device->comps_used; i++) {
 		struct ib_cq_init_attr cq_attr = {};
@@ -344,17 +326,15 @@ out_cq:
 static int
 isert_create_device_ib_res(struct isert_device *device)
 {
-	struct ib_device_attr *dev_attr;
+	struct ib_device *ib_dev = device->ib_device;
 	int ret;
 
-	dev_attr = &device->dev_attr;
-	ret = isert_query_device(device->ib_device, dev_attr);
-	if (ret)
-		return ret;
+	isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge);
+	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
 
 	/* asign function handlers */
-	if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS &&
-	    dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) {
+	if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS &&
+	    ib_dev->attrs.device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) {
 		device->use_fastreg = 1;
 		device->reg_rdma_mem = isert_reg_rdma;
 		device->unreg_rdma_mem = isert_unreg_rdma;
@@ -364,11 +344,11 @@ isert_create_device_ib_res(struct isert_device *device)
 		device->unreg_rdma_mem = isert_unmap_cmd;
 	}
 
-	ret = isert_alloc_comps(device, dev_attr);
+	ret = isert_alloc_comps(device);
 	if (ret)
-		return ret;
+		goto out;
 
-	device->pd = ib_alloc_pd(device->ib_device);
+	device->pd = ib_alloc_pd(ib_dev);
 	if (IS_ERR(device->pd)) {
 		ret = PTR_ERR(device->pd);
 		isert_err("failed to allocate pd, device %p, ret=%d\n",
@@ -377,13 +357,16 @@ isert_create_device_ib_res(struct isert_device *device)
 	}
 
 	/* Check signature cap */
-	device->pi_capable = dev_attr->device_cap_flags &
+	device->pi_capable = ib_dev->attrs.device_cap_flags &
 			     IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
 
 	return 0;
 
 out_cq:
 	isert_free_comps(device);
+out:
+	if (ret > 0)
+		ret = -EINVAL;
 	return ret;
 }
 
@@ -673,6 +656,32 @@ out_login_buf:
 	return ret;
 }
 
+static void
+isert_set_nego_params(struct isert_conn *isert_conn,
+		      struct rdma_conn_param *param)
+{
+	struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
+
+	/* Set max inflight RDMA READ requests */
+	isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
+				attr->max_qp_init_rd_atom);
+	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
+
+	if (param->private_data) {
+		u8 flags = *(u8 *)param->private_data;
+
+		/*
+		 * use remote invalidation if the both initiator
+		 * and the HCA support it
+		 */
+		isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
+					  (attr->device_cap_flags &
+					   IB_DEVICE_MEM_MGT_EXTENSIONS);
+		if (isert_conn->snd_w_inv)
+			isert_info("Using remote invalidation\n");
+	}
+}
+
 static int
 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 {
@@ -711,11 +720,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 	}
 	isert_conn->device = device;
 
-	/* Set max inflight RDMA READ requests */
-	isert_conn->initiator_depth = min_t(u8,
-				event->param.conn.initiator_depth,
-				device->dev_attr.max_qp_init_rd_atom);
-	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
+	isert_set_nego_params(isert_conn, &event->param.conn);
 
 	ret = isert_conn_setup_qp(isert_conn, cma_id);
 	if (ret)
@@ -1047,8 +1052,8 @@ isert_create_send_desc(struct isert_conn *isert_conn,
 	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
 				   ISER_HEADERS_LEN, DMA_TO_DEVICE);
 
-	memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
-	tx_desc->iser_header.flags = ISER_VER;
+	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
+	tx_desc->iser_header.flags = ISCSI_CTRL;
 
 	tx_desc->num_sge = 1;
 	tx_desc->isert_cmd = isert_cmd;
@@ -1094,7 +1099,14 @@ isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
 
 	isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
 	send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
-	send_wr->opcode = IB_WR_SEND;
+
+	if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
+		send_wr->opcode  = IB_WR_SEND_WITH_INV;
+		send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
+	} else {
+		send_wr->opcode = IB_WR_SEND;
+	}
+
 	send_wr->sg_list = &tx_desc->tx_sg[0];
 	send_wr->num_sge = isert_cmd->tx_desc.num_sge;
 	send_wr->send_flags = IB_SEND_SIGNALED;
@@ -1483,6 +1495,7 @@ isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
 		isert_cmd->read_va = read_va;
 		isert_cmd->write_stag = write_stag;
 		isert_cmd->write_va = write_va;
+		isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
 
 		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
 					rx_desc, (unsigned char *)hdr);
@@ -1540,21 +1553,21 @@ isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
 static void
 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
 {
-	struct iser_hdr *iser_hdr = &rx_desc->iser_header;
+	struct iser_ctrl *iser_ctrl = &rx_desc->iser_header;
 	uint64_t read_va = 0, write_va = 0;
 	uint32_t read_stag = 0, write_stag = 0;
 
-	switch (iser_hdr->flags & 0xF0) {
+	switch (iser_ctrl->flags & 0xF0) {
 	case ISCSI_CTRL:
-		if (iser_hdr->flags & ISER_RSV) {
-			read_stag = be32_to_cpu(iser_hdr->read_stag);
-			read_va = be64_to_cpu(iser_hdr->read_va);
+		if (iser_ctrl->flags & ISER_RSV) {
+			read_stag = be32_to_cpu(iser_ctrl->read_stag);
+			read_va = be64_to_cpu(iser_ctrl->read_va);
 			isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
 				  read_stag, (unsigned long long)read_va);
 		}
-		if (iser_hdr->flags & ISER_WSV) {
-			write_stag = be32_to_cpu(iser_hdr->write_stag);
-			write_va = be64_to_cpu(iser_hdr->write_va);
+		if (iser_ctrl->flags & ISER_WSV) {
+			write_stag = be32_to_cpu(iser_ctrl->write_stag);
+			write_va = be64_to_cpu(iser_ctrl->write_va);
 			isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
 				  write_stag, (unsigned long long)write_va);
 		}
@@ -1565,7 +1578,7 @@ isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
 		isert_err("iSER Hello message\n");
 		break;
 	default:
-		isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
+		isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
 		break;
 	}
 
@@ -3092,12 +3105,20 @@ isert_rdma_accept(struct isert_conn *isert_conn)
 	struct rdma_cm_id *cm_id = isert_conn->cm_id;
 	struct rdma_conn_param cp;
 	int ret;
+	struct iser_cm_hdr rsp_hdr;
 
 	memset(&cp, 0, sizeof(struct rdma_conn_param));
 	cp.initiator_depth = isert_conn->initiator_depth;
 	cp.retry_count = 7;
 	cp.rnr_retry_count = 7;
 
+	memset(&rsp_hdr, 0, sizeof(rsp_hdr));
+	rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
+	if (!isert_conn->snd_w_inv)
+		rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
+	cp.private_data = (void *)&rsp_hdr;
+	cp.private_data_len = sizeof(rsp_hdr);
+
 	ret = rdma_accept(cm_id, &cp);
 	if (ret) {
 		isert_err("rdma_accept() failed with: %d\n", ret);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index 3d7fbc47c343..8d50453eef66 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -3,6 +3,8 @@
 #include <linux/in6.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
+#include <scsi/iser.h>
+
 
 #define DRV_NAME	"isert"
 #define PFX		DRV_NAME ": "
@@ -31,6 +33,38 @@
 #define isert_err(fmt, arg...) \
 	pr_err(PFX "%s: " fmt, __func__ , ## arg)
 
+/* Constant PDU lengths calculations */
+#define ISER_HEADERS_LEN	(sizeof(struct iser_ctrl) + \
+				 sizeof(struct iscsi_hdr))
+#define ISER_RECV_DATA_SEG_LEN	8192
+#define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
+#define ISER_RX_LOGIN_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
+
+/* QP settings */
+/* Maximal bounds on received asynchronous PDUs */
+#define ISERT_MAX_TX_MISC_PDUS	4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
+
+#define ISERT_MAX_RX_MISC_PDUS	6 /*
+				   * NOOP_OUT(2), TEXT(1),
+				   * SCSI_TMFUNC(2), LOGOUT(1)
+				   */
+
+#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
+
+#define ISERT_QP_MAX_RECV_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX)
+
+#define ISERT_MIN_POSTED_RX	(ISCSI_DEF_XMIT_CMDS_MAX >> 2)
+
+#define ISERT_INFLIGHT_DATAOUTS	8
+
+#define ISERT_QP_MAX_REQ_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX *    \
+				(1 + ISERT_INFLIGHT_DATAOUTS) + \
+				ISERT_MAX_TX_MISC_PDUS	+ \
+				ISERT_MAX_RX_MISC_PDUS)
+
+#define ISER_RX_PAD_SIZE	(ISER_RECV_DATA_SEG_LEN + 4096 - \
+		(ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge)))
+
 #define ISCSI_ISER_SG_TABLESIZE		256
 #define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
 #define ISER_BEACON_WRID               0xfffffffffffffffeULL
@@ -56,7 +90,7 @@ enum iser_conn_state {
 };
 
 struct iser_rx_desc {
-	struct iser_hdr iser_header;
+	struct iser_ctrl iser_header;
 	struct iscsi_hdr iscsi_header;
 	char		data[ISER_RECV_DATA_SEG_LEN];
 	u64		dma_addr;
@@ -65,7 +99,7 @@ struct iser_rx_desc {
 } __packed;
 
 struct iser_tx_desc {
-	struct iser_hdr iser_header;
+	struct iser_ctrl iser_header;
 	struct iscsi_hdr iscsi_header;
 	enum isert_desc_type type;
 	u64		dma_addr;
@@ -129,6 +163,7 @@ struct isert_cmd {
 	uint32_t		write_stag;
 	uint64_t		read_va;
 	uint64_t		write_va;
+	uint32_t		inv_rkey;
 	u64			pdu_buf_dma;
 	u32			pdu_buf_len;
 	struct isert_conn	*conn;
@@ -176,6 +211,7 @@ struct isert_conn {
 	struct work_struct	release_work;
 	struct ib_recv_wr       beacon;
 	bool                    logout_posted;
+	bool                    snd_w_inv;
 };
 
 #define ISERT_MAX_CQ 64
@@ -207,7 +243,6 @@ struct isert_device {
 	struct isert_comp	*comps;
 	int                     comps_used;
 	struct list_head	dev_node;
-	struct ib_device_attr	dev_attr;
 	int			(*reg_rdma_mem)(struct iscsi_conn *conn,
 						    struct iscsi_cmd *cmd,
 						    struct isert_rdma_wr *wr);
diff --git a/drivers/infiniband/ulp/isert/isert_proto.h b/drivers/infiniband/ulp/isert/isert_proto.h
deleted file mode 100644
index 4dccd313b777..000000000000
--- a/drivers/infiniband/ulp/isert/isert_proto.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* From iscsi_iser.h */
-
-struct iser_hdr {
-	u8	flags;
-	u8	rsvd[3];
-	__be32	write_stag; /* write rkey */
-	__be64	write_va;
-	__be32	read_stag;  /* read rkey */
-	__be64	read_va;
-} __packed;
-
-/*Constant PDU lengths calculations */
-#define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
-
-#define ISER_RECV_DATA_SEG_LEN  8192
-#define ISER_RX_PAYLOAD_SIZE    (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
-#define ISER_RX_LOGIN_SIZE      (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
-
-/* QP settings */
-/* Maximal bounds on received asynchronous PDUs */
-#define ISERT_MAX_TX_MISC_PDUS	4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
-
-#define ISERT_MAX_RX_MISC_PDUS	6 /* NOOP_OUT(2), TEXT(1),         *
-				   * SCSI_TMFUNC(2), LOGOUT(1) */
-
-#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
-
-#define ISERT_QP_MAX_RECV_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX)
-
-#define ISERT_MIN_POSTED_RX	(ISCSI_DEF_XMIT_CMDS_MAX >> 2)
-
-#define ISERT_INFLIGHT_DATAOUTS	8
-
-#define ISERT_QP_MAX_REQ_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX *    \
-				(1 + ISERT_INFLIGHT_DATAOUTS) + \
-				ISERT_MAX_TX_MISC_PDUS	+ \
-				ISERT_MAX_RX_MISC_PDUS)
-
-#define ISER_RX_PAD_SIZE	(ISER_RECV_DATA_SEG_LEN + 4096 - \
-		(ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge)))
-
-#define ISER_VER	0x10
-#define ISER_WSV	0x08
-#define ISER_RSV	0x04
-#define ISCSI_CTRL	0x10
-#define ISER_HELLO	0x20
-#define ISER_HELLORPLY	0x30
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 3db9a659719b..03022f6420d7 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -132,8 +132,9 @@ MODULE_PARM_DESC(ch_count,
 
 static void srp_add_one(struct ib_device *device);
 static void srp_remove_one(struct ib_device *device, void *client_data);
-static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
-static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
+static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
+static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
+		const char *opname);
 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 
 static struct scsi_transport_template *ib_srp_transport_template;
@@ -445,6 +446,17 @@ static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 				  dev->max_pages_per_mr);
 }
 
+static void srp_drain_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct srp_rdma_ch *ch = cq->cq_context;
+
+	complete(&ch->done);
+}
+
+static struct ib_cqe srp_drain_cqe = {
+	.done		= srp_drain_done,
+};
+
 /**
  * srp_destroy_qp() - destroy an RDMA queue pair
  * @ch: SRP RDMA channel.
@@ -457,10 +469,11 @@ static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 static void srp_destroy_qp(struct srp_rdma_ch *ch)
 {
 	static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
-	static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
+	static struct ib_recv_wr wr = { 0 };
 	struct ib_recv_wr *bad_wr;
 	int ret;
 
+	wr.wr_cqe = &srp_drain_cqe;
 	/* Destroying a QP and reusing ch->done is only safe if not connected */
 	WARN_ON_ONCE(ch->connected);
 
@@ -489,34 +502,27 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	struct ib_fmr_pool *fmr_pool = NULL;
 	struct srp_fr_pool *fr_pool = NULL;
 	const int m = dev->use_fast_reg ? 3 : 1;
-	struct ib_cq_init_attr cq_attr = {};
 	int ret;
 
 	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 	if (!init_attr)
 		return -ENOMEM;
 
-	/* + 1 for SRP_LAST_WR_ID */
-	cq_attr.cqe = target->queue_size + 1;
-	cq_attr.comp_vector = ch->comp_vector;
-	recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
-			       &cq_attr);
+	/* queue_size + 1 for ib_drain_qp */
+	recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
+				ch->comp_vector, IB_POLL_SOFTIRQ);
 	if (IS_ERR(recv_cq)) {
 		ret = PTR_ERR(recv_cq);
 		goto err;
 	}
 
-	cq_attr.cqe = m * target->queue_size;
-	cq_attr.comp_vector = ch->comp_vector;
-	send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
-			       &cq_attr);
+	send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
+				ch->comp_vector, IB_POLL_DIRECT);
 	if (IS_ERR(send_cq)) {
 		ret = PTR_ERR(send_cq);
 		goto err_recv_cq;
 	}
 
-	ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
-
 	init_attr->event_handler       = srp_qp_event;
 	init_attr->cap.max_send_wr     = m * target->queue_size;
 	init_attr->cap.max_recv_wr     = target->queue_size + 1;
@@ -558,9 +564,9 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	if (ch->qp)
 		srp_destroy_qp(ch);
 	if (ch->recv_cq)
-		ib_destroy_cq(ch->recv_cq);
+		ib_free_cq(ch->recv_cq);
 	if (ch->send_cq)
-		ib_destroy_cq(ch->send_cq);
+		ib_free_cq(ch->send_cq);
 
 	ch->qp = qp;
 	ch->recv_cq = recv_cq;
@@ -580,13 +586,13 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	return 0;
 
 err_qp:
-	ib_destroy_qp(qp);
+	srp_destroy_qp(ch);
 
 err_send_cq:
-	ib_destroy_cq(send_cq);
+	ib_free_cq(send_cq);
 
 err_recv_cq:
-	ib_destroy_cq(recv_cq);
+	ib_free_cq(recv_cq);
 
 err:
 	kfree(init_attr);
@@ -622,9 +628,10 @@ static void srp_free_ch_ib(struct srp_target_port *target,
 		if (ch->fmr_pool)
 			ib_destroy_fmr_pool(ch->fmr_pool);
 	}
+
 	srp_destroy_qp(ch);
-	ib_destroy_cq(ch->send_cq);
-	ib_destroy_cq(ch->recv_cq);
+	ib_free_cq(ch->send_cq);
+	ib_free_cq(ch->recv_cq);
 
 	/*
 	 * Avoid that the SCSI error handler tries to use this channel after
@@ -1041,18 +1048,25 @@ out:
 	return ret <= 0 ? ret : -ENODEV;
 }
 
-static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
+static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	srp_handle_qp_err(cq, wc, "INV RKEY");
+}
+
+static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
+		u32 rkey)
 {
 	struct ib_send_wr *bad_wr;
 	struct ib_send_wr wr = {
 		.opcode		    = IB_WR_LOCAL_INV,
-		.wr_id		    = LOCAL_INV_WR_ID_MASK,
 		.next		    = NULL,
 		.num_sge	    = 0,
 		.send_flags	    = 0,
 		.ex.invalidate_rkey = rkey,
 	};
 
+	wr.wr_cqe = &req->reg_cqe;
+	req->reg_cqe.done = srp_inv_rkey_err_done;
 	return ib_post_send(ch->qp, &wr, &bad_wr);
 }
 
@@ -1074,7 +1088,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
 		struct srp_fr_desc **pfr;
 
 		for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
-			res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
+			res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
 			if (res < 0) {
 				shost_printk(KERN_ERR, target->scsi_host, PFX
 				  "Queueing INV WR for rkey %#x failed (%d)\n",
@@ -1312,7 +1326,13 @@ reset_state:
 	return 0;
 }
 
+static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	srp_handle_qp_err(cq, wc, "FAST REG");
+}
+
 static int srp_map_finish_fr(struct srp_map_state *state,
+			     struct srp_request *req,
 			     struct srp_rdma_ch *ch, int sg_nents)
 {
 	struct srp_target_port *target = ch->target;
@@ -1349,9 +1369,11 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 	if (unlikely(n < 0))
 		return n;
 
+	req->reg_cqe.done = srp_reg_mr_err_done;
+
 	wr.wr.next = NULL;
 	wr.wr.opcode = IB_WR_REG_MR;
-	wr.wr.wr_id = FAST_REG_WR_ID_MASK;
+	wr.wr.wr_cqe = &req->reg_cqe;
 	wr.wr.num_sge = 0;
 	wr.wr.send_flags = 0;
 	wr.mr = desc->mr;
@@ -1455,7 +1477,7 @@ static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
 	while (count) {
 		int i, n;
 
-		n = srp_map_finish_fr(state, ch, count);
+		n = srp_map_finish_fr(state, req, ch, count);
 		if (unlikely(n < 0))
 			return n;
 
@@ -1524,7 +1546,7 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 		idb_sg->dma_length = idb_sg->length;	      /* hack^2 */
 #endif
-		ret = srp_map_finish_fr(&state, ch, 1);
+		ret = srp_map_finish_fr(&state, req, ch, 1);
 		if (ret < 0)
 			return ret;
 	} else if (dev->use_fmr) {
@@ -1719,7 +1741,7 @@ static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
 	s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
 	struct srp_iu *iu;
 
-	srp_send_completion(ch->send_cq, ch);
+	ib_process_cq_direct(ch->send_cq, -1);
 
 	if (list_empty(&ch->free_tx))
 		return NULL;
@@ -1739,6 +1761,19 @@ static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
 	return iu;
 }
 
+static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
+	struct srp_rdma_ch *ch = cq->cq_context;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		srp_handle_qp_err(cq, wc, "SEND");
+		return;
+	}
+
+	list_add(&iu->list, &ch->free_tx);
+}
+
 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
 {
 	struct srp_target_port *target = ch->target;
@@ -1749,8 +1784,10 @@ static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
 	list.length = len;
 	list.lkey   = target->lkey;
 
+	iu->cqe.done = srp_send_done;
+
 	wr.next       = NULL;
-	wr.wr_id      = (uintptr_t) iu;
+	wr.wr_cqe     = &iu->cqe;
 	wr.sg_list    = &list;
 	wr.num_sge    = 1;
 	wr.opcode     = IB_WR_SEND;
@@ -1769,8 +1806,10 @@ static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
 	list.length = iu->size;
 	list.lkey   = target->lkey;
 
+	iu->cqe.done = srp_recv_done;
+
 	wr.next     = NULL;
-	wr.wr_id    = (uintptr_t) iu;
+	wr.wr_cqe   = &iu->cqe;
 	wr.sg_list  = &list;
 	wr.num_sge  = 1;
 
@@ -1902,14 +1941,20 @@ static void srp_process_aer_req(struct srp_rdma_ch *ch,
 			     "problems processing SRP_AER_REQ\n");
 }
 
-static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
+static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 {
+	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
+	struct srp_rdma_ch *ch = cq->cq_context;
 	struct srp_target_port *target = ch->target;
 	struct ib_device *dev = target->srp_host->srp_dev->dev;
-	struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
 	int res;
 	u8 opcode;
 
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		srp_handle_qp_err(cq, wc, "RECV");
+		return;
+	}
+
 	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
 				   DMA_FROM_DEVICE);
 
@@ -1972,68 +2017,22 @@ static void srp_tl_err_work(struct work_struct *work)
 		srp_start_tl_fail_timers(target->rport);
 }
 
-static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
-			      bool send_err, struct srp_rdma_ch *ch)
+static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
+		const char *opname)
 {
+	struct srp_rdma_ch *ch = cq->cq_context;
 	struct srp_target_port *target = ch->target;
 
-	if (wr_id == SRP_LAST_WR_ID) {
-		complete(&ch->done);
-		return;
-	}
-
 	if (ch->connected && !target->qp_in_error) {
-		if (wr_id & LOCAL_INV_WR_ID_MASK) {
-			shost_printk(KERN_ERR, target->scsi_host, PFX
-				     "LOCAL_INV failed with status %s (%d)\n",
-				     ib_wc_status_msg(wc_status), wc_status);
-		} else if (wr_id & FAST_REG_WR_ID_MASK) {
-			shost_printk(KERN_ERR, target->scsi_host, PFX
-				     "FAST_REG_MR failed status %s (%d)\n",
-				     ib_wc_status_msg(wc_status), wc_status);
-		} else {
-			shost_printk(KERN_ERR, target->scsi_host,
-				     PFX "failed %s status %s (%d) for iu %p\n",
-				     send_err ? "send" : "receive",
-				     ib_wc_status_msg(wc_status), wc_status,
-				     (void *)(uintptr_t)wr_id);
-		}
+		shost_printk(KERN_ERR, target->scsi_host,
+			     PFX "failed %s status %s (%d) for CQE %p\n",
+			     opname, ib_wc_status_msg(wc->status), wc->status,
+			     wc->wr_cqe);
 		queue_work(system_long_wq, &target->tl_err_work);
 	}
 	target->qp_in_error = true;
 }
 
-static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
-{
-	struct srp_rdma_ch *ch = ch_ptr;
-	struct ib_wc wc;
-
-	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
-	while (ib_poll_cq(cq, 1, &wc) > 0) {
-		if (likely(wc.status == IB_WC_SUCCESS)) {
-			srp_handle_recv(ch, &wc);
-		} else {
-			srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
-		}
-	}
-}
-
-static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
-{
-	struct srp_rdma_ch *ch = ch_ptr;
-	struct ib_wc wc;
-	struct srp_iu *iu;
-
-	while (ib_poll_cq(cq, 1, &wc) > 0) {
-		if (likely(wc.status == IB_WC_SUCCESS)) {
-			iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
-			list_add(&iu->list, &ch->free_tx);
-		} else {
-			srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
-		}
-	}
-}
-
 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
 {
 	struct srp_target_port *target = host_to_target(shost);
@@ -3439,27 +3438,17 @@ free_host:
 static void srp_add_one(struct ib_device *device)
 {
 	struct srp_device *srp_dev;
-	struct ib_device_attr *dev_attr;
 	struct srp_host *host;
 	int mr_page_shift, p;
 	u64 max_pages_per_mr;
 
-	dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
-	if (!dev_attr)
-		return;
-
-	if (ib_query_device(device, dev_attr)) {
-		pr_warn("Query device failed for %s\n", device->name);
-		goto free_attr;
-	}
-
 	srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
 	if (!srp_dev)
-		goto free_attr;
+		return;
 
 	srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
 			    device->map_phys_fmr && device->unmap_fmr);
-	srp_dev->has_fr = (dev_attr->device_cap_flags &
+	srp_dev->has_fr = (device->attrs.device_cap_flags &
 			   IB_DEVICE_MEM_MGT_EXTENSIONS);
 	if (!srp_dev->has_fmr && !srp_dev->has_fr)
 		dev_warn(&device->dev, "neither FMR nor FR is supported\n");
@@ -3473,23 +3462,23 @@ static void srp_add_one(struct ib_device *device)
 	 * minimum of 4096 bytes. We're unlikely to build large sglists
 	 * out of smaller entries.
 	 */
-	mr_page_shift		= max(12, ffs(dev_attr->page_size_cap) - 1);
+	mr_page_shift		= max(12, ffs(device->attrs.page_size_cap) - 1);
 	srp_dev->mr_page_size	= 1 << mr_page_shift;
 	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
-	max_pages_per_mr	= dev_attr->max_mr_size;
+	max_pages_per_mr	= device->attrs.max_mr_size;
 	do_div(max_pages_per_mr, srp_dev->mr_page_size);
 	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
 					  max_pages_per_mr);
 	if (srp_dev->use_fast_reg) {
 		srp_dev->max_pages_per_mr =
 			min_t(u32, srp_dev->max_pages_per_mr,
-			      dev_attr->max_fast_reg_page_list_len);
+			      device->attrs.max_fast_reg_page_list_len);
 	}
 	srp_dev->mr_max_size	= srp_dev->mr_page_size *
 				   srp_dev->max_pages_per_mr;
-	pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
-		 device->name, mr_page_shift, dev_attr->max_mr_size,
-		 dev_attr->max_fast_reg_page_list_len,
+	pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
+		 device->name, mr_page_shift, device->attrs.max_mr_size,
+		 device->attrs.max_fast_reg_page_list_len,
 		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
 
 	INIT_LIST_HEAD(&srp_dev->dev_list);
@@ -3517,17 +3506,13 @@ static void srp_add_one(struct ib_device *device)
 	}
 
 	ib_set_client_data(device, &srp_client, srp_dev);
-
-	goto free_attr;
+	return;
 
 err_pd:
 	ib_dealloc_pd(srp_dev->pd);
 
 free_dev:
 	kfree(srp_dev);
-
-free_attr:
-	kfree(dev_attr);
 }
 
 static void srp_remove_one(struct ib_device *device, void *client_data)
@@ -3587,8 +3572,6 @@ static int __init srp_init_module(void)
 {
 	int ret;
 
-	BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
-
 	if (srp_sg_tablesize) {
 		pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
 		if (!cmd_sg_entries)
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index f6af531f9f32..9e05ce4a04fd 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -66,11 +66,6 @@ enum {
 	SRP_TAG_TSK_MGMT	= 1U << 31,
 
 	SRP_MAX_PAGES_PER_MR	= 512,
-
-	LOCAL_INV_WR_ID_MASK	= 1,
-	FAST_REG_WR_ID_MASK	= 2,
-
-	SRP_LAST_WR_ID		= 0xfffffffcU,
 };
 
 enum srp_target_state {
@@ -128,6 +123,7 @@ struct srp_request {
 	struct srp_direct_buf  *indirect_desc;
 	dma_addr_t		indirect_dma_addr;
 	short			nmdesc;
+	struct ib_cqe		reg_cqe;
 };
 
 /**
@@ -231,6 +227,7 @@ struct srp_iu {
 	void		       *buf;
 	size_t			size;
 	enum dma_data_direction	direction;
+	struct ib_cqe		cqe;
 };
 
 /**
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 2e2fe818ca9f..0c37fee363b1 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -93,6 +93,8 @@ MODULE_PARM_DESC(srpt_service_guid,
 static struct ib_client srpt_client;
 static void srpt_release_channel(struct srpt_rdma_ch *ch);
 static int srpt_queue_status(struct se_cmd *cmd);
+static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
+static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
 
 /**
  * opposite_dma_dir() - Swap DMA_TO_DEVICE and DMA_FROM_DEVICE.
@@ -341,10 +343,10 @@ static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
 	memset(iocp, 0, sizeof *iocp);
 	strcpy(iocp->id_string, SRPT_ID_STRING);
 	iocp->guid = cpu_to_be64(srpt_service_guid);
-	iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
-	iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
-	iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
-	iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
+	iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
+	iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
+	iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
+	iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
 	iocp->subsys_device_id = 0x0;
 	iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
 	iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
@@ -453,6 +455,7 @@ static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
  * srpt_mad_recv_handler() - MAD reception callback function.
  */
 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
+				  struct ib_mad_send_buf *send_buf,
 				  struct ib_mad_recv_wc *mad_wc)
 {
 	struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
@@ -778,12 +781,12 @@ static int srpt_post_recv(struct srpt_device *sdev,
 	struct ib_recv_wr wr, *bad_wr;
 
 	BUG_ON(!sdev);
-	wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index);
-
 	list.addr = ioctx->ioctx.dma;
 	list.length = srp_max_req_size;
 	list.lkey = sdev->pd->local_dma_lkey;
 
+	ioctx->ioctx.cqe.done = srpt_recv_done;
+	wr.wr_cqe = &ioctx->ioctx.cqe;
 	wr.next = NULL;
 	wr.sg_list = &list;
 	wr.num_sge = 1;
@@ -819,8 +822,9 @@ static int srpt_post_send(struct srpt_rdma_ch *ch,
 	list.length = len;
 	list.lkey = sdev->pd->local_dma_lkey;
 
+	ioctx->ioctx.cqe.done = srpt_send_done;
 	wr.next = NULL;
-	wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index);
+	wr.wr_cqe = &ioctx->ioctx.cqe;
 	wr.sg_list = &list;
 	wr.num_sge = 1;
 	wr.opcode = IB_WR_SEND;
@@ -1052,13 +1056,13 @@ static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 
 	BUG_ON(!ch);
 	BUG_ON(!ioctx);
-	BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius);
+	BUG_ON(ioctx->n_rdma && !ioctx->rdma_wrs);
 
 	while (ioctx->n_rdma)
-		kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge);
+		kfree(ioctx->rdma_wrs[--ioctx->n_rdma].wr.sg_list);
 
-	kfree(ioctx->rdma_ius);
-	ioctx->rdma_ius = NULL;
+	kfree(ioctx->rdma_wrs);
+	ioctx->rdma_wrs = NULL;
 
 	if (ioctx->mapped_sg_count) {
 		sg = ioctx->sg;
@@ -1082,7 +1086,7 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 	struct scatterlist *sg, *sg_orig;
 	int sg_cnt;
 	enum dma_data_direction dir;
-	struct rdma_iu *riu;
+	struct ib_rdma_wr *riu;
 	struct srp_direct_buf *db;
 	dma_addr_t dma_addr;
 	struct ib_sge *sge;
@@ -1109,23 +1113,24 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 
 	ioctx->mapped_sg_count = count;
 
-	if (ioctx->rdma_ius && ioctx->n_rdma_ius)
-		nrdma = ioctx->n_rdma_ius;
+	if (ioctx->rdma_wrs && ioctx->n_rdma_wrs)
+		nrdma = ioctx->n_rdma_wrs;
 	else {
 		nrdma = (count + SRPT_DEF_SG_PER_WQE - 1) / SRPT_DEF_SG_PER_WQE
 			+ ioctx->n_rbuf;
 
-		ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu, GFP_KERNEL);
-		if (!ioctx->rdma_ius)
+		ioctx->rdma_wrs = kcalloc(nrdma, sizeof(*ioctx->rdma_wrs),
+				GFP_KERNEL);
+		if (!ioctx->rdma_wrs)
 			goto free_mem;
 
-		ioctx->n_rdma_ius = nrdma;
+		ioctx->n_rdma_wrs = nrdma;
 	}
 
 	db = ioctx->rbufs;
 	tsize = cmd->data_length;
 	dma_len = ib_sg_dma_len(dev, &sg[0]);
-	riu = ioctx->rdma_ius;
+	riu = ioctx->rdma_wrs;
 
 	/*
 	 * For each remote desc - calculate the #ib_sge.
@@ -1139,9 +1144,9 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 	     j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
 		rsize = be32_to_cpu(db->len);
 		raddr = be64_to_cpu(db->va);
-		riu->raddr = raddr;
+		riu->remote_addr = raddr;
 		riu->rkey = be32_to_cpu(db->key);
-		riu->sge_cnt = 0;
+		riu->wr.num_sge = 0;
 
 		/* calculate how many sge required for this remote_buf */
 		while (rsize > 0 && tsize > 0) {
@@ -1165,33 +1170,35 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 				rsize = 0;
 			}
 
-			++riu->sge_cnt;
+			++riu->wr.num_sge;
 
-			if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
+			if (rsize > 0 &&
+			    riu->wr.num_sge == SRPT_DEF_SG_PER_WQE) {
 				++ioctx->n_rdma;
-				riu->sge =
-				    kmalloc(riu->sge_cnt * sizeof *riu->sge,
-					    GFP_KERNEL);
-				if (!riu->sge)
+				riu->wr.sg_list = kmalloc_array(riu->wr.num_sge,
+						sizeof(*riu->wr.sg_list),
+						GFP_KERNEL);
+				if (!riu->wr.sg_list)
 					goto free_mem;
 
 				++riu;
-				riu->sge_cnt = 0;
-				riu->raddr = raddr;
+				riu->wr.num_sge = 0;
+				riu->remote_addr = raddr;
 				riu->rkey = be32_to_cpu(db->key);
 			}
 		}
 
 		++ioctx->n_rdma;
-		riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
-				   GFP_KERNEL);
-		if (!riu->sge)
+		riu->wr.sg_list = kmalloc_array(riu->wr.num_sge,
+					sizeof(*riu->wr.sg_list),
+					GFP_KERNEL);
+		if (!riu->wr.sg_list)
 			goto free_mem;
 	}
 
 	db = ioctx->rbufs;
 	tsize = cmd->data_length;
-	riu = ioctx->rdma_ius;
+	riu = ioctx->rdma_wrs;
 	sg = sg_orig;
 	dma_len = ib_sg_dma_len(dev, &sg[0]);
 	dma_addr = ib_sg_dma_address(dev, &sg[0]);
@@ -1200,7 +1207,7 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 	for (i = 0, j = 0;
 	     j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
 		rsize = be32_to_cpu(db->len);
-		sge = riu->sge;
+		sge = riu->wr.sg_list;
 		k = 0;
 
 		while (rsize > 0 && tsize > 0) {
@@ -1232,9 +1239,9 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 			}
 
 			++k;
-			if (k == riu->sge_cnt && rsize > 0 && tsize > 0) {
+			if (k == riu->wr.num_sge && rsize > 0 && tsize > 0) {
 				++riu;
-				sge = riu->sge;
+				sge = riu->wr.sg_list;
 				k = 0;
 			} else if (rsize > 0 && tsize > 0)
 				++sge;
@@ -1277,8 +1284,8 @@ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
 	ioctx->n_rbuf = 0;
 	ioctx->rbufs = NULL;
 	ioctx->n_rdma = 0;
-	ioctx->n_rdma_ius = 0;
-	ioctx->rdma_ius = NULL;
+	ioctx->n_rdma_wrs = 0;
+	ioctx->rdma_wrs = NULL;
 	ioctx->mapped_sg_count = 0;
 	init_completion(&ioctx->tx_done);
 	ioctx->queue_status_only = false;
@@ -1380,118 +1387,44 @@ out:
 }
 
 /**
- * srpt_handle_send_err_comp() - Process an IB_WC_SEND error completion.
- */
-static void srpt_handle_send_err_comp(struct srpt_rdma_ch *ch, u64 wr_id)
-{
-	struct srpt_send_ioctx *ioctx;
-	enum srpt_command_state state;
-	u32 index;
-
-	atomic_inc(&ch->sq_wr_avail);
-
-	index = idx_from_wr_id(wr_id);
-	ioctx = ch->ioctx_ring[index];
-	state = srpt_get_cmd_state(ioctx);
-
-	WARN_ON(state != SRPT_STATE_CMD_RSP_SENT
-		&& state != SRPT_STATE_MGMT_RSP_SENT
-		&& state != SRPT_STATE_NEED_DATA
-		&& state != SRPT_STATE_DONE);
-
-	/* If SRP_RSP sending failed, undo the ch->req_lim change. */
-	if (state == SRPT_STATE_CMD_RSP_SENT
-	    || state == SRPT_STATE_MGMT_RSP_SENT)
-		atomic_dec(&ch->req_lim);
-
-	srpt_abort_cmd(ioctx);
-}
-
-/**
- * srpt_handle_send_comp() - Process an IB send completion notification.
- */
-static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
-				  struct srpt_send_ioctx *ioctx)
-{
-	enum srpt_command_state state;
-
-	atomic_inc(&ch->sq_wr_avail);
-
-	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
-
-	if (WARN_ON(state != SRPT_STATE_CMD_RSP_SENT
-		    && state != SRPT_STATE_MGMT_RSP_SENT
-		    && state != SRPT_STATE_DONE))
-		pr_debug("state = %d\n", state);
-
-	if (state != SRPT_STATE_DONE) {
-		srpt_unmap_sg_to_ib_sge(ch, ioctx);
-		transport_generic_free_cmd(&ioctx->cmd, 0);
-	} else {
-		pr_err("IB completion has been received too late for"
-		       " wr_id = %u.\n", ioctx->ioctx.index);
-	}
-}
-
-/**
- * srpt_handle_rdma_comp() - Process an IB RDMA completion notification.
- *
  * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
  * the data that has been transferred via IB RDMA had to be postponed until the
  * check_stop_free() callback.  None of this is necessary anymore and needs to
  * be cleaned up.
  */
-static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
-				  struct srpt_send_ioctx *ioctx,
-				  enum srpt_opcode opcode)
+static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
 {
+	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_send_ioctx *ioctx =
+		container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
+
 	WARN_ON(ioctx->n_rdma <= 0);
 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
 
-	if (opcode == SRPT_RDMA_READ_LAST) {
-		if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
-						SRPT_STATE_DATA_IN))
-			target_execute_cmd(&ioctx->cmd);
-		else
-			pr_err("%s[%d]: wrong state = %d\n", __func__,
-			       __LINE__, srpt_get_cmd_state(ioctx));
-	} else if (opcode == SRPT_RDMA_ABORT) {
-		ioctx->rdma_aborted = true;
-	} else {
-		WARN(true, "unexpected opcode %d\n", opcode);
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
+			ioctx, wc->status);
+		srpt_abort_cmd(ioctx);
+		return;
 	}
+
+	if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
+					SRPT_STATE_DATA_IN))
+		target_execute_cmd(&ioctx->cmd);
+	else
+		pr_err("%s[%d]: wrong state = %d\n", __func__,
+		       __LINE__, srpt_get_cmd_state(ioctx));
 }
 
-/**
- * srpt_handle_rdma_err_comp() - Process an IB RDMA error completion.
- */
-static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch,
-				      struct srpt_send_ioctx *ioctx,
-				      enum srpt_opcode opcode)
+static void srpt_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	enum srpt_command_state state;
+	struct srpt_send_ioctx *ioctx =
+		container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
 
-	state = srpt_get_cmd_state(ioctx);
-	switch (opcode) {
-	case SRPT_RDMA_READ_LAST:
-		if (ioctx->n_rdma <= 0) {
-			pr_err("Received invalid RDMA read"
-			       " error completion with idx %d\n",
-			       ioctx->ioctx.index);
-			break;
-		}
-		atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
-		if (state == SRPT_STATE_NEED_DATA)
-			srpt_abort_cmd(ioctx);
-		else
-			pr_err("%s[%d]: wrong state = %d\n",
-			       __func__, __LINE__, state);
-		break;
-	case SRPT_RDMA_WRITE_LAST:
-		break;
-	default:
-		pr_err("%s[%d]: opcode = %u\n", __func__, __LINE__, opcode);
-		break;
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		pr_info("RDMA_WRITE for ioctx 0x%p failed with status %d\n",
+			ioctx, wc->status);
+		srpt_abort_cmd(ioctx);
 	}
 }
 
@@ -1926,32 +1859,26 @@ out:
 	return;
 }
 
-static void srpt_process_rcv_completion(struct ib_cq *cq,
-					struct srpt_rdma_ch *ch,
-					struct ib_wc *wc)
+static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_device *sdev = ch->sport->sdev;
-	struct srpt_recv_ioctx *ioctx;
-	u32 index;
+	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_recv_ioctx *ioctx =
+		container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
 
-	index = idx_from_wr_id(wc->wr_id);
 	if (wc->status == IB_WC_SUCCESS) {
 		int req_lim;
 
 		req_lim = atomic_dec_return(&ch->req_lim);
 		if (unlikely(req_lim < 0))
 			pr_err("req_lim = %d < 0\n", req_lim);
-		ioctx = sdev->ioctx_ring[index];
 		srpt_handle_new_iu(ch, ioctx, NULL);
 	} else {
-		pr_info("receiving failed for idx %u with status %d\n",
-			index, wc->status);
+		pr_info("receiving failed for ioctx %p with status %d\n",
+			ioctx, wc->status);
 	}
 }
 
 /**
- * srpt_process_send_completion() - Process an IB send completion.
- *
  * Note: Although this has not yet been observed during tests, at least in
  * theory it is possible that the srpt_get_send_ioctx() call invoked by
  * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
@@ -1964,109 +1891,52 @@ static void srpt_process_rcv_completion(struct ib_cq *cq,
  * are queued on cmd_wait_list. The code below processes these delayed
  * requests one at a time.
  */
-static void srpt_process_send_completion(struct ib_cq *cq,
-					 struct srpt_rdma_ch *ch,
-					 struct ib_wc *wc)
+static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_send_ioctx *send_ioctx;
-	uint32_t index;
-	enum srpt_opcode opcode;
+	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_send_ioctx *ioctx =
+		container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
+	enum srpt_command_state state;
 
-	index = idx_from_wr_id(wc->wr_id);
-	opcode = opcode_from_wr_id(wc->wr_id);
-	send_ioctx = ch->ioctx_ring[index];
-	if (wc->status == IB_WC_SUCCESS) {
-		if (opcode == SRPT_SEND)
-			srpt_handle_send_comp(ch, send_ioctx);
-		else {
-			WARN_ON(opcode != SRPT_RDMA_ABORT &&
-				wc->opcode != IB_WC_RDMA_READ);
-			srpt_handle_rdma_comp(ch, send_ioctx, opcode);
-		}
+	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
+
+	WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
+		state != SRPT_STATE_MGMT_RSP_SENT);
+
+	atomic_inc(&ch->sq_wr_avail);
+
+	if (wc->status != IB_WC_SUCCESS) {
+		pr_info("sending response for ioctx 0x%p failed"
+			" with status %d\n", ioctx, wc->status);
+
+		atomic_dec(&ch->req_lim);
+		srpt_abort_cmd(ioctx);
+		goto out;
+	}
+
+	if (state != SRPT_STATE_DONE) {
+		srpt_unmap_sg_to_ib_sge(ch, ioctx);
+		transport_generic_free_cmd(&ioctx->cmd, 0);
 	} else {
-		if (opcode == SRPT_SEND) {
-			pr_info("sending response for idx %u failed"
-				" with status %d\n", index, wc->status);
-			srpt_handle_send_err_comp(ch, wc->wr_id);
-		} else if (opcode != SRPT_RDMA_MID) {
-			pr_info("RDMA t %d for idx %u failed with"
-				" status %d\n", opcode, index, wc->status);
-			srpt_handle_rdma_err_comp(ch, send_ioctx, opcode);
-		}
+		pr_err("IB completion has been received too late for"
+		       " wr_id = %u.\n", ioctx->ioctx.index);
 	}
 
-	while (unlikely(opcode == SRPT_SEND
-			&& !list_empty(&ch->cmd_wait_list)
-			&& srpt_get_ch_state(ch) == CH_LIVE
-			&& (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) {
+out:
+	while (!list_empty(&ch->cmd_wait_list) &&
+	       srpt_get_ch_state(ch) == CH_LIVE &&
+	       (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
 		struct srpt_recv_ioctx *recv_ioctx;
 
 		recv_ioctx = list_first_entry(&ch->cmd_wait_list,
 					      struct srpt_recv_ioctx,
 					      wait_list);
 		list_del(&recv_ioctx->wait_list);
-		srpt_handle_new_iu(ch, recv_ioctx, send_ioctx);
-	}
-}
-
-static void srpt_process_completion(struct ib_cq *cq, struct srpt_rdma_ch *ch)
-{
-	struct ib_wc *const wc = ch->wc;
-	int i, n;
-
-	WARN_ON(cq != ch->cq);
-
-	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
-	while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) {
-		for (i = 0; i < n; i++) {
-			if (opcode_from_wr_id(wc[i].wr_id) == SRPT_RECV)
-				srpt_process_rcv_completion(cq, ch, &wc[i]);
-			else
-				srpt_process_send_completion(cq, ch, &wc[i]);
-		}
+		srpt_handle_new_iu(ch, recv_ioctx, ioctx);
 	}
 }
 
 /**
- * srpt_completion() - IB completion queue callback function.
- *
- * Notes:
- * - It is guaranteed that a completion handler will never be invoked
- *   concurrently on two different CPUs for the same completion queue. See also
- *   Documentation/infiniband/core_locking.txt and the implementation of
- *   handle_edge_irq() in kernel/irq/chip.c.
- * - When threaded IRQs are enabled, completion handlers are invoked in thread
- *   context instead of interrupt context.
- */
-static void srpt_completion(struct ib_cq *cq, void *ctx)
-{
-	struct srpt_rdma_ch *ch = ctx;
-
-	wake_up_interruptible(&ch->wait_queue);
-}
-
-static int srpt_compl_thread(void *arg)
-{
-	struct srpt_rdma_ch *ch;
-
-	/* Hibernation / freezing of the SRPT kernel thread is not supported. */
-	current->flags |= PF_NOFREEZE;
-
-	ch = arg;
-	BUG_ON(!ch);
-	pr_info("Session %s: kernel thread %s (PID %d) started\n",
-		ch->sess_name, ch->thread->comm, current->pid);
-	while (!kthread_should_stop()) {
-		wait_event_interruptible(ch->wait_queue,
-			(srpt_process_completion(ch->cq, ch),
-			 kthread_should_stop()));
-	}
-	pr_info("Session %s: kernel thread %s (PID %d) stopped\n",
-		ch->sess_name, ch->thread->comm, current->pid);
-	return 0;
-}
-
-/**
  * srpt_create_ch_ib() - Create receive and send completion queues.
  */
 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
@@ -2075,7 +1945,6 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	struct srpt_port *sport = ch->sport;
 	struct srpt_device *sdev = sport->sdev;
 	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
-	struct ib_cq_init_attr cq_attr = {};
 	int ret;
 
 	WARN_ON(ch->rq_size < 1);
@@ -2086,9 +1955,8 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 		goto out;
 
 retry:
-	cq_attr.cqe = ch->rq_size + srp_sq_size;
-	ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch,
-			      &cq_attr);
+	ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
+			0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
 	if (IS_ERR(ch->cq)) {
 		ret = PTR_ERR(ch->cq);
 		pr_err("failed to create CQ cqe= %d ret= %d\n",
@@ -2131,18 +1999,6 @@ retry:
 	if (ret)
 		goto err_destroy_qp;
 
-	init_waitqueue_head(&ch->wait_queue);
-
-	pr_debug("creating thread for session %s\n", ch->sess_name);
-
-	ch->thread = kthread_run(srpt_compl_thread, ch, "ib_srpt_compl");
-	if (IS_ERR(ch->thread)) {
-		pr_err("failed to create kernel thread %ld\n",
-		       PTR_ERR(ch->thread));
-		ch->thread = NULL;
-		goto err_destroy_qp;
-	}
-
 out:
 	kfree(qp_init);
 	return ret;
@@ -2150,17 +2006,14 @@ out:
 err_destroy_qp:
 	ib_destroy_qp(ch->qp);
 err_destroy_cq:
-	ib_destroy_cq(ch->cq);
+	ib_free_cq(ch->cq);
 	goto out;
 }
 
 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
 {
-	if (ch->thread)
-		kthread_stop(ch->thread);
-
 	ib_destroy_qp(ch->qp);
-	ib_destroy_cq(ch->cq);
+	ib_free_cq(ch->cq);
 }
 
 /**
@@ -2370,31 +2223,6 @@ static void srpt_release_channel_work(struct work_struct *w)
 	kfree(ch);
 }
 
-static struct srpt_node_acl *__srpt_lookup_acl(struct srpt_port *sport,
-					       u8 i_port_id[16])
-{
-	struct srpt_node_acl *nacl;
-
-	list_for_each_entry(nacl, &sport->port_acl_list, list)
-		if (memcmp(nacl->i_port_id, i_port_id,
-			   sizeof(nacl->i_port_id)) == 0)
-			return nacl;
-
-	return NULL;
-}
-
-static struct srpt_node_acl *srpt_lookup_acl(struct srpt_port *sport,
-					     u8 i_port_id[16])
-{
-	struct srpt_node_acl *nacl;
-
-	spin_lock_irq(&sport->port_acl_lock);
-	nacl = __srpt_lookup_acl(sport, i_port_id);
-	spin_unlock_irq(&sport->port_acl_lock);
-
-	return nacl;
-}
-
 /**
  * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
  *
@@ -2412,10 +2240,10 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 	struct srp_login_rej *rej;
 	struct ib_cm_rep_param *rep_param;
 	struct srpt_rdma_ch *ch, *tmp_ch;
-	struct srpt_node_acl *nacl;
+	struct se_node_acl *se_acl;
 	u32 it_iu_len;
-	int i;
-	int ret = 0;
+	int i, ret = 0;
+	unsigned char *p;
 
 	WARN_ON_ONCE(irqs_disabled());
 
@@ -2565,33 +2393,47 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 		       " RTR failed (error code = %d)\n", ret);
 		goto destroy_ib;
 	}
+
 	/*
-	 * Use the initator port identifier as the session name.
+	 * Use the initator port identifier as the session name, when
+	 * checking against se_node_acl->initiatorname[] this can be
+	 * with or without preceeding '0x'.
 	 */
 	snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
 			be64_to_cpu(*(__be64 *)ch->i_port_id),
 			be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
 
 	pr_debug("registering session %s\n", ch->sess_name);
+	p = &ch->sess_name[0];
 
-	nacl = srpt_lookup_acl(sport, ch->i_port_id);
-	if (!nacl) {
-		pr_info("Rejected login because no ACL has been"
-			" configured yet for initiator %s.\n", ch->sess_name);
+	ch->sess = transport_init_session(TARGET_PROT_NORMAL);
+	if (IS_ERR(ch->sess)) {
 		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
+				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_debug("Failed to create session\n");
 		goto destroy_ib;
 	}
 
-	ch->sess = transport_init_session(TARGET_PROT_NORMAL);
-	if (IS_ERR(ch->sess)) {
+try_again:
+	se_acl = core_tpg_get_initiator_node_acl(&sport->port_tpg_1, p);
+	if (!se_acl) {
+		pr_info("Rejected login because no ACL has been"
+			" configured yet for initiator %s.\n", ch->sess_name);
+		/*
+		 * XXX: Hack to retry of ch->i_port_id without leading '0x'
+		 */
+		if (p == &ch->sess_name[0]) {
+			p += 2;
+			goto try_again;
+		}
 		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
-		pr_debug("Failed to create session\n");
-		goto deregister_session;
+				SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
+		transport_free_session(ch->sess);
+		goto destroy_ib;
 	}
-	ch->sess->se_node_acl = &nacl->nacl;
-	transport_register_session(&sport->port_tpg_1, &nacl->nacl, ch->sess, ch);
+	ch->sess->se_node_acl = se_acl;
+
+	transport_register_session(&sport->port_tpg_1, se_acl, ch->sess, ch);
 
 	pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
 		 ch->sess_name, ch->cm_id);
@@ -2635,8 +2477,6 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 release_channel:
 	srpt_set_ch_state(ch, CH_RELEASING);
 	transport_deregister_session_configfs(ch->sess);
-
-deregister_session:
 	transport_deregister_session(ch->sess);
 	ch->sess = NULL;
 
@@ -2821,12 +2661,8 @@ static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch,
 			      struct srpt_send_ioctx *ioctx)
 {
-	struct ib_rdma_wr wr;
 	struct ib_send_wr *bad_wr;
-	struct rdma_iu *riu;
-	int i;
-	int ret;
-	int sq_wr_avail;
+	int sq_wr_avail, ret, i;
 	enum dma_data_direction dir;
 	const int n_rdma = ioctx->n_rdma;
 
@@ -2842,59 +2678,32 @@ static int srpt_perform_rdmas(struct srpt_rdma_ch *ch,
 		}
 	}
 
-	ioctx->rdma_aborted = false;
-	ret = 0;
-	riu = ioctx->rdma_ius;
-	memset(&wr, 0, sizeof wr);
-
-	for (i = 0; i < n_rdma; ++i, ++riu) {
-		if (dir == DMA_FROM_DEVICE) {
-			wr.wr.opcode = IB_WR_RDMA_WRITE;
-			wr.wr.wr_id = encode_wr_id(i == n_rdma - 1 ?
-						SRPT_RDMA_WRITE_LAST :
-						SRPT_RDMA_MID,
-						ioctx->ioctx.index);
-		} else {
-			wr.wr.opcode = IB_WR_RDMA_READ;
-			wr.wr.wr_id = encode_wr_id(i == n_rdma - 1 ?
-						SRPT_RDMA_READ_LAST :
-						SRPT_RDMA_MID,
-						ioctx->ioctx.index);
-		}
-		wr.wr.next = NULL;
-		wr.remote_addr = riu->raddr;
-		wr.rkey = riu->rkey;
-		wr.wr.num_sge = riu->sge_cnt;
-		wr.wr.sg_list = riu->sge;
+	for (i = 0; i < n_rdma; i++) {
+		struct ib_send_wr *wr = &ioctx->rdma_wrs[i].wr;
 
-		/* only get completion event for the last rdma write */
-		if (i == (n_rdma - 1) && dir == DMA_TO_DEVICE)
-			wr.wr.send_flags = IB_SEND_SIGNALED;
+		wr->opcode = (dir == DMA_FROM_DEVICE) ?
+				IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
 
-		ret = ib_post_send(ch->qp, &wr.wr, &bad_wr);
-		if (ret)
-			break;
+		if (i == n_rdma - 1) {
+			/* only get completion event for the last rdma read */
+			if (dir == DMA_TO_DEVICE) {
+				wr->send_flags = IB_SEND_SIGNALED;
+				ioctx->rdma_cqe.done = srpt_rdma_read_done;
+			} else {
+				ioctx->rdma_cqe.done = srpt_rdma_write_done;
+			}
+			wr->wr_cqe = &ioctx->rdma_cqe;
+			wr->next = NULL;
+		} else {
+			wr->wr_cqe = NULL;
+			wr->next = &ioctx->rdma_wrs[i + 1].wr;
+		}
 	}
 
+	ret = ib_post_send(ch->qp, &ioctx->rdma_wrs->wr, &bad_wr);
 	if (ret)
 		pr_err("%s[%d]: ib_post_send() returned %d for %d/%d\n",
 				 __func__, __LINE__, ret, i, n_rdma);
-	if (ret && i > 0) {
-		wr.wr.num_sge = 0;
-		wr.wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index);
-		wr.wr.send_flags = IB_SEND_SIGNALED;
-		while (ch->state == CH_LIVE &&
-			ib_post_send(ch->qp, &wr.wr, &bad_wr) != 0) {
-			pr_info("Trying to abort failed RDMA transfer [%d]\n",
-				ioctx->ioctx.index);
-			msleep(1000);
-		}
-		while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) {
-			pr_info("Waiting until RDMA abort finished [%d]\n",
-				ioctx->ioctx.index);
-			msleep(1000);
-		}
-	}
 out:
 	if (unlikely(dir == DMA_TO_DEVICE && ret < 0))
 		atomic_add(n_rdma, &ch->sq_wr_avail);
@@ -3203,14 +3012,11 @@ static void srpt_add_one(struct ib_device *device)
 	init_waitqueue_head(&sdev->ch_releaseQ);
 	spin_lock_init(&sdev->spinlock);
 
-	if (ib_query_device(device, &sdev->dev_attr))
-		goto free_dev;
-
 	sdev->pd = ib_alloc_pd(device);
 	if (IS_ERR(sdev->pd))
 		goto free_dev;
 
-	sdev->srq_size = min(srpt_srq_size, sdev->dev_attr.max_srq_wr);
+	sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
 
 	srq_attr.event_handler = srpt_srq_event;
 	srq_attr.srq_context = (void *)sdev;
@@ -3224,7 +3030,7 @@ static void srpt_add_one(struct ib_device *device)
 		goto err_pd;
 
 	pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
-		 __func__, sdev->srq_size, sdev->dev_attr.max_srq_wr,
+		 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
 		 device->name);
 
 	if (!srpt_service_guid)
@@ -3273,8 +3079,6 @@ static void srpt_add_one(struct ib_device *device)
 		sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
 		sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
 		INIT_WORK(&sport->work, srpt_refresh_port_work);
-		INIT_LIST_HEAD(&sport->port_acl_list);
-		spin_lock_init(&sport->port_acl_lock);
 
 		if (srpt_refresh_port(sport)) {
 			pr_err("MAD registration failed for %s-%d.\n",
@@ -3508,42 +3312,15 @@ out:
  */
 static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
 {
-	struct srpt_port *sport =
-		container_of(se_nacl->se_tpg, struct srpt_port, port_tpg_1);
-	struct srpt_node_acl *nacl =
-		container_of(se_nacl, struct srpt_node_acl, nacl);
 	u8 i_port_id[16];
 
 	if (srpt_parse_i_port_id(i_port_id, name) < 0) {
 		pr_err("invalid initiator port ID %s\n", name);
 		return -EINVAL;
 	}
-
-	memcpy(&nacl->i_port_id[0], &i_port_id[0], 16);
-	nacl->sport = sport;
-
-	spin_lock_irq(&sport->port_acl_lock);
-	list_add_tail(&nacl->list, &sport->port_acl_list);
-	spin_unlock_irq(&sport->port_acl_lock);
-
 	return 0;
 }
 
-/*
- * configfs callback function invoked for
- * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
- */
-static void srpt_cleanup_nodeacl(struct se_node_acl *se_nacl)
-{
-	struct srpt_node_acl *nacl =
-		container_of(se_nacl, struct srpt_node_acl, nacl);
-	struct srpt_port *sport = nacl->sport;
-
-	spin_lock_irq(&sport->port_acl_lock);
-	list_del(&nacl->list);
-	spin_unlock_irq(&sport->port_acl_lock);
-}
-
 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
 		char *page)
 {
@@ -3820,7 +3597,6 @@ static const struct target_core_fabric_ops srpt_template = {
 	.fabric_make_tpg		= srpt_make_tpg,
 	.fabric_drop_tpg		= srpt_drop_tpg,
 	.fabric_init_nodeacl		= srpt_init_nodeacl,
-	.fabric_cleanup_nodeacl		= srpt_cleanup_nodeacl,
 
 	.tfc_wwn_attrs			= srpt_wwn_attrs,
 	.tfc_tpg_base_attrs		= srpt_tpg_attrs,
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 5faad8acd789..09037f2b0b51 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -128,36 +128,6 @@ enum {
 	DEFAULT_MAX_RDMA_SIZE = 65536,
 };
 
-enum srpt_opcode {
-	SRPT_RECV,
-	SRPT_SEND,
-	SRPT_RDMA_MID,
-	SRPT_RDMA_ABORT,
-	SRPT_RDMA_READ_LAST,
-	SRPT_RDMA_WRITE_LAST,
-};
-
-static inline u64 encode_wr_id(u8 opcode, u32 idx)
-{
-	return ((u64)opcode << 32) | idx;
-}
-static inline enum srpt_opcode opcode_from_wr_id(u64 wr_id)
-{
-	return wr_id >> 32;
-}
-static inline u32 idx_from_wr_id(u64 wr_id)
-{
-	return (u32)wr_id;
-}
-
-struct rdma_iu {
-	u64		raddr;
-	u32		rkey;
-	struct ib_sge	*sge;
-	u32		sge_cnt;
-	int		mem_id;
-};
-
 /**
  * enum srpt_command_state - SCSI command state managed by SRPT.
  * @SRPT_STATE_NEW:           New command arrived and is being processed.
@@ -189,6 +159,7 @@ enum srpt_command_state {
  * @index: Index of the I/O context in its ioctx_ring array.
  */
 struct srpt_ioctx {
+	struct ib_cqe		cqe;
 	void			*buf;
 	dma_addr_t		dma;
 	uint32_t		index;
@@ -215,32 +186,30 @@ struct srpt_recv_ioctx {
  * @sg:          Pointer to sg-list associated with this I/O context.
  * @sg_cnt:      SG-list size.
  * @mapped_sg_count: ib_dma_map_sg() return value.
- * @n_rdma_ius:  Number of elements in the rdma_ius array.
- * @rdma_ius:    Array with information about the RDMA mapping.
+ * @n_rdma_wrs:  Number of elements in the rdma_wrs array.
+ * @rdma_wrs:    Array with information about the RDMA mapping.
  * @tag:         Tag of the received SRP information unit.
  * @spinlock:    Protects 'state'.
  * @state:       I/O context state.
- * @rdma_aborted: If initiating a multipart RDMA transfer failed, whether
- * 		 the already initiated transfers have finished.
  * @cmd:         Target core command data structure.
  * @sense_data:  SCSI sense data.
  */
 struct srpt_send_ioctx {
 	struct srpt_ioctx	ioctx;
 	struct srpt_rdma_ch	*ch;
-	struct rdma_iu		*rdma_ius;
+	struct ib_rdma_wr	*rdma_wrs;
+	struct ib_cqe		rdma_cqe;
 	struct srp_direct_buf	*rbufs;
 	struct srp_direct_buf	single_rbuf;
 	struct scatterlist	*sg;
 	struct list_head	free_list;
 	spinlock_t		spinlock;
 	enum srpt_command_state	state;
-	bool			rdma_aborted;
 	struct se_cmd		cmd;
 	struct completion	tx_done;
 	int			sg_cnt;
 	int			mapped_sg_count;
-	u16			n_rdma_ius;
+	u16			n_rdma_wrs;
 	u8			n_rdma;
 	u8			n_rbuf;
 	bool			queue_status_only;
@@ -267,9 +236,6 @@ enum rdma_ch_state {
 
 /**
  * struct srpt_rdma_ch - RDMA channel.
- * @wait_queue:    Allows the kernel thread to wait for more work.
- * @thread:        Kernel thread that processes the IB queues associated with
- *                 the channel.
  * @cm_id:         IB CM ID associated with the channel.
  * @qp:            IB queue pair used for communicating over this channel.
  * @cq:            IB completion queue for this channel.
@@ -288,7 +254,6 @@ enum rdma_ch_state {
  * @free_list:     Head of list with free send I/O contexts.
  * @state:         channel state. See also enum rdma_ch_state.
  * @ioctx_ring:    Send ring.
- * @wc:            IB work completion array for srpt_process_completion().
  * @list:          Node for insertion in the srpt_device.rch_list list.
  * @cmd_wait_list: List of SCSI commands that arrived before the RTU event. This
  *                 list contains struct srpt_ioctx elements and is protected
@@ -299,8 +264,6 @@ enum rdma_ch_state {
  * @release_done:  Enables waiting for srpt_release_channel() completion.
  */
 struct srpt_rdma_ch {
-	wait_queue_head_t	wait_queue;
-	struct task_struct	*thread;
 	struct ib_cm_id		*cm_id;
 	struct ib_qp		*qp;
 	struct ib_cq		*cq;
@@ -317,7 +280,6 @@ struct srpt_rdma_ch {
 	struct list_head	free_list;
 	enum rdma_ch_state	state;
 	struct srpt_send_ioctx	**ioctx_ring;
-	struct ib_wc		wc[16];
 	struct list_head	list;
 	struct list_head	cmd_wait_list;
 	struct se_session	*sess;
@@ -364,11 +326,9 @@ struct srpt_port {
 	u16			sm_lid;
 	u16			lid;
 	union ib_gid		gid;
-	spinlock_t		port_acl_lock;
 	struct work_struct	work;
 	struct se_portal_group	port_tpg_1;
 	struct se_wwn		port_wwn;
-	struct list_head	port_acl_list;
 	struct srpt_port_attrib port_attrib;
 };
 
@@ -379,8 +339,6 @@ struct srpt_port {
  * @mr:            L_Key (local key) with write access to all local memory.
  * @srq:           Per-HCA SRQ (shared receive queue).
  * @cm_id:         Connection identifier.
- * @dev_attr:      Attributes of the InfiniBand device as obtained during the
- *                 ib_client.add() callback.
  * @srq_size:      SRQ size.
  * @ioctx_ring:    Per-HCA SRQ.
  * @rch_list:      Per-device channel list -- see also srpt_rdma_ch.list.
@@ -395,7 +353,6 @@ struct srpt_device {
 	struct ib_pd		*pd;
 	struct ib_srq		*srq;
 	struct ib_cm_id		*cm_id;
-	struct ib_device_attr	dev_attr;
 	int			srq_size;
 	struct srpt_recv_ioctx	**ioctx_ring;
 	struct list_head	rch_list;
@@ -409,15 +366,9 @@ struct srpt_device {
 /**
  * struct srpt_node_acl - Per-initiator ACL data (managed via configfs).
  * @nacl:      Target core node ACL information.
- * @i_port_id: 128-bit SRP initiator port ID.
- * @sport:     port information.
- * @list:      Element of the per-HCA ACL list.
  */
 struct srpt_node_acl {
 	struct se_node_acl	nacl;
-	u8			i_port_id[16];
-	struct srpt_port	*sport;
-	struct list_head	list;
 };
 
 #endif				/* IB_SRPT_H */
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index fd4100d56d8c..6727954ab74b 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -76,10 +76,13 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/input.h>
+#include <linux/rcupdate.h>
 #include <linux/slab.h>
 #include <linux/stat.h>
 #include <linux/module.h>
 #include <linux/usb/input.h>
+#include <linux/usb/quirks.h>
 
 #define DRIVER_AUTHOR "Marko Friedemann <mfr@bmx-chemnitz.de>"
 #define DRIVER_DESC "X-Box pad driver"
@@ -125,7 +128,7 @@ static const struct xpad_device {
 	{ 0x045e, 0x0289, "Microsoft X-Box pad v2 (US)", 0, XTYPE_XBOX },
 	{ 0x045e, 0x028e, "Microsoft X-Box 360 pad", 0, XTYPE_XBOX360 },
 	{ 0x045e, 0x02d1, "Microsoft X-Box One pad", 0, XTYPE_XBOXONE },
-	{ 0x045e, 0x02dd, "Microsoft X-Box One pad (Covert Forces)", 0, XTYPE_XBOXONE },
+	{ 0x045e, 0x02dd, "Microsoft X-Box One pad (Firmware 2015)", 0, XTYPE_XBOXONE },
 	{ 0x045e, 0x0291, "Xbox 360 Wireless Receiver (XBOX)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W },
 	{ 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W },
 	{ 0x044f, 0x0f07, "Thrustmaster, Inc. Controller", 0, XTYPE_XBOX },
@@ -317,21 +320,42 @@ static struct usb_device_id xpad_table[] = {
 
 MODULE_DEVICE_TABLE(usb, xpad_table);
 
+struct xpad_output_packet {
+	u8 data[XPAD_PKT_LEN];
+	u8 len;
+	bool pending;
+};
+
+#define XPAD_OUT_CMD_IDX	0
+#define XPAD_OUT_FF_IDX		1
+#define XPAD_OUT_LED_IDX	(1 + IS_ENABLED(CONFIG_JOYSTICK_XPAD_FF))
+#define XPAD_NUM_OUT_PACKETS	(1 + \
+				 IS_ENABLED(CONFIG_JOYSTICK_XPAD_FF) + \
+				 IS_ENABLED(CONFIG_JOYSTICK_XPAD_LEDS))
+
 struct usb_xpad {
 	struct input_dev *dev;		/* input device interface */
+	struct input_dev __rcu *x360w_dev;
 	struct usb_device *udev;	/* usb device */
 	struct usb_interface *intf;	/* usb interface */
 
-	int pad_present;
+	bool pad_present;
+	bool input_created;
 
 	struct urb *irq_in;		/* urb for interrupt in report */
 	unsigned char *idata;		/* input data */
 	dma_addr_t idata_dma;
 
 	struct urb *irq_out;		/* urb for interrupt out report */
+	struct usb_anchor irq_out_anchor;
+	bool irq_out_active;		/* we must not use an active URB */
+	u8 odata_serial;		/* serial number for xbox one protocol */
 	unsigned char *odata;		/* output data */
 	dma_addr_t odata_dma;
-	struct mutex odata_mutex;
+	spinlock_t odata_lock;
+
+	struct xpad_output_packet out_packets[XPAD_NUM_OUT_PACKETS];
+	int last_out_packet;
 
 #if defined(CONFIG_JOYSTICK_XPAD_LEDS)
 	struct xpad_led *led;
@@ -343,8 +367,12 @@ struct usb_xpad {
 	int xtype;			/* type of xbox device */
 	int pad_nr;			/* the order x360 pads were attached */
 	const char *name;		/* name of the device */
+	struct work_struct work;	/* init/remove device from callback */
 };
 
+static int xpad_init_input(struct usb_xpad *xpad);
+static void xpad_deinit_input(struct usb_xpad *xpad);
+
 /*
  *	xpad_process_packet
  *
@@ -424,11 +452,9 @@ static void xpad_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *d
  *		http://www.free60.org/wiki/Gamepad
  */
 
-static void xpad360_process_packet(struct usb_xpad *xpad,
+static void xpad360_process_packet(struct usb_xpad *xpad, struct input_dev *dev,
 				   u16 cmd, unsigned char *data)
 {
-	struct input_dev *dev = xpad->dev;
-
 	/* digital pad */
 	if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
 		/* dpad as buttons (left, right, up, down) */
@@ -495,7 +521,30 @@ static void xpad360_process_packet(struct usb_xpad *xpad,
 	input_sync(dev);
 }
 
-static void xpad_identify_controller(struct usb_xpad *xpad);
+static void xpad_presence_work(struct work_struct *work)
+{
+	struct usb_xpad *xpad = container_of(work, struct usb_xpad, work);
+	int error;
+
+	if (xpad->pad_present) {
+		error = xpad_init_input(xpad);
+		if (error) {
+			/* complain only, not much else we can do here */
+			dev_err(&xpad->dev->dev,
+				"unable to init device: %d\n", error);
+		} else {
+			rcu_assign_pointer(xpad->x360w_dev, xpad->dev);
+		}
+	} else {
+		RCU_INIT_POINTER(xpad->x360w_dev, NULL);
+		synchronize_rcu();
+		/*
+		 * Now that we are sure xpad360w_process_packet is not
+		 * using input device we can get rid of it.
+		 */
+		xpad_deinit_input(xpad);
+	}
+}
 
 /*
  * xpad360w_process_packet
@@ -513,24 +562,28 @@ static void xpad_identify_controller(struct usb_xpad *xpad);
  */
 static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data)
 {
+	struct input_dev *dev;
+	bool present;
+
 	/* Presence change */
 	if (data[0] & 0x08) {
-		if (data[1] & 0x80) {
-			xpad->pad_present = 1;
-			/*
-			 * Light up the segment corresponding to
-			 * controller number.
-			 */
-			xpad_identify_controller(xpad);
-		} else
-			xpad->pad_present = 0;
+		present = (data[1] & 0x80) != 0;
+
+		if (xpad->pad_present != present) {
+			xpad->pad_present = present;
+			schedule_work(&xpad->work);
+		}
 	}
 
 	/* Valid pad data */
-	if (!(data[1] & 0x1))
+	if (data[1] != 0x1)
 		return;
 
-	xpad360_process_packet(xpad, cmd, &data[4]);
+	rcu_read_lock();
+	dev = rcu_dereference(xpad->x360w_dev);
+	if (dev)
+		xpad360_process_packet(xpad, dev, cmd, &data[4]);
+	rcu_read_unlock();
 }
 
 /*
@@ -659,7 +712,7 @@ static void xpad_irq_in(struct urb *urb)
 
 	switch (xpad->xtype) {
 	case XTYPE_XBOX360:
-		xpad360_process_packet(xpad, 0, xpad->idata);
+		xpad360_process_packet(xpad, xpad->dev, 0, xpad->idata);
 		break;
 	case XTYPE_XBOX360W:
 		xpad360w_process_packet(xpad, 0, xpad->idata);
@@ -678,18 +731,73 @@ exit:
 			__func__, retval);
 }
 
+/* Callers must hold xpad->odata_lock spinlock */
+static bool xpad_prepare_next_out_packet(struct usb_xpad *xpad)
+{
+	struct xpad_output_packet *pkt, *packet = NULL;
+	int i;
+
+	for (i = 0; i < XPAD_NUM_OUT_PACKETS; i++) {
+		if (++xpad->last_out_packet >= XPAD_NUM_OUT_PACKETS)
+			xpad->last_out_packet = 0;
+
+		pkt = &xpad->out_packets[xpad->last_out_packet];
+		if (pkt->pending) {
+			dev_dbg(&xpad->intf->dev,
+				"%s - found pending output packet %d\n",
+				__func__, xpad->last_out_packet);
+			packet = pkt;
+			break;
+		}
+	}
+
+	if (packet) {
+		memcpy(xpad->odata, packet->data, packet->len);
+		xpad->irq_out->transfer_buffer_length = packet->len;
+		return true;
+	}
+
+	return false;
+}
+
+/* Callers must hold xpad->odata_lock spinlock */
+static int xpad_try_sending_next_out_packet(struct usb_xpad *xpad)
+{
+	int error;
+
+	if (!xpad->irq_out_active && xpad_prepare_next_out_packet(xpad)) {
+		usb_anchor_urb(xpad->irq_out, &xpad->irq_out_anchor);
+		error = usb_submit_urb(xpad->irq_out, GFP_ATOMIC);
+		if (error) {
+			dev_err(&xpad->intf->dev,
+				"%s - usb_submit_urb failed with result %d\n",
+				__func__, error);
+			usb_unanchor_urb(xpad->irq_out);
+			return -EIO;
+		}
+
+		xpad->irq_out_active = true;
+	}
+
+	return 0;
+}
+
 static void xpad_irq_out(struct urb *urb)
 {
 	struct usb_xpad *xpad = urb->context;
 	struct device *dev = &xpad->intf->dev;
-	int retval, status;
+	int status = urb->status;
+	int error;
+	unsigned long flags;
 
-	status = urb->status;
+	spin_lock_irqsave(&xpad->odata_lock, flags);
 
 	switch (status) {
 	case 0:
 		/* success */
-		return;
+		xpad->out_packets[xpad->last_out_packet].pending = false;
+		xpad->irq_out_active = xpad_prepare_next_out_packet(xpad);
+		break;
 
 	case -ECONNRESET:
 	case -ENOENT:
@@ -697,19 +805,28 @@ static void xpad_irq_out(struct urb *urb)
 		/* this urb is terminated, clean up */
 		dev_dbg(dev, "%s - urb shutting down with status: %d\n",
 			__func__, status);
-		return;
+		xpad->irq_out_active = false;
+		break;
 
 	default:
 		dev_dbg(dev, "%s - nonzero urb status received: %d\n",
 			__func__, status);
-		goto exit;
+		break;
 	}
 
-exit:
-	retval = usb_submit_urb(urb, GFP_ATOMIC);
-	if (retval)
-		dev_err(dev, "%s - usb_submit_urb failed with result %d\n",
-			__func__, retval);
+	if (xpad->irq_out_active) {
+		usb_anchor_urb(urb, &xpad->irq_out_anchor);
+		error = usb_submit_urb(urb, GFP_ATOMIC);
+		if (error) {
+			dev_err(dev,
+				"%s - usb_submit_urb failed with result %d\n",
+				__func__, error);
+			usb_unanchor_urb(urb);
+			xpad->irq_out_active = false;
+		}
+	}
+
+	spin_unlock_irqrestore(&xpad->odata_lock, flags);
 }
 
 static int xpad_init_output(struct usb_interface *intf, struct usb_xpad *xpad)
@@ -721,6 +838,8 @@ static int xpad_init_output(struct usb_interface *intf, struct usb_xpad *xpad)
 	if (xpad->xtype == XTYPE_UNKNOWN)
 		return 0;
 
+	init_usb_anchor(&xpad->irq_out_anchor);
+
 	xpad->odata = usb_alloc_coherent(xpad->udev, XPAD_PKT_LEN,
 					 GFP_KERNEL, &xpad->odata_dma);
 	if (!xpad->odata) {
@@ -728,7 +847,7 @@ static int xpad_init_output(struct usb_interface *intf, struct usb_xpad *xpad)
 		goto fail1;
 	}
 
-	mutex_init(&xpad->odata_mutex);
+	spin_lock_init(&xpad->odata_lock);
 
 	xpad->irq_out = usb_alloc_urb(0, GFP_KERNEL);
 	if (!xpad->irq_out) {
@@ -755,8 +874,14 @@ static int xpad_init_output(struct usb_interface *intf, struct usb_xpad *xpad)
 
 static void xpad_stop_output(struct usb_xpad *xpad)
 {
-	if (xpad->xtype != XTYPE_UNKNOWN)
-		usb_kill_urb(xpad->irq_out);
+	if (xpad->xtype != XTYPE_UNKNOWN) {
+		if (!usb_wait_anchor_empty_timeout(&xpad->irq_out_anchor,
+						   5000)) {
+			dev_warn(&xpad->intf->dev,
+				 "timed out waiting for output URB to complete, killing\n");
+			usb_kill_anchored_urbs(&xpad->irq_out_anchor);
+		}
+	}
 }
 
 static void xpad_deinit_output(struct usb_xpad *xpad)
@@ -770,27 +895,60 @@ static void xpad_deinit_output(struct usb_xpad *xpad)
 
 static int xpad_inquiry_pad_presence(struct usb_xpad *xpad)
 {
+	struct xpad_output_packet *packet =
+			&xpad->out_packets[XPAD_OUT_CMD_IDX];
+	unsigned long flags;
 	int retval;
 
-	mutex_lock(&xpad->odata_mutex);
+	spin_lock_irqsave(&xpad->odata_lock, flags);
+
+	packet->data[0] = 0x08;
+	packet->data[1] = 0x00;
+	packet->data[2] = 0x0F;
+	packet->data[3] = 0xC0;
+	packet->data[4] = 0x00;
+	packet->data[5] = 0x00;
+	packet->data[6] = 0x00;
+	packet->data[7] = 0x00;
+	packet->data[8] = 0x00;
+	packet->data[9] = 0x00;
+	packet->data[10] = 0x00;
+	packet->data[11] = 0x00;
+	packet->len = 12;
+	packet->pending = true;
+
+	/* Reset the sequence so we send out presence first */
+	xpad->last_out_packet = -1;
+	retval = xpad_try_sending_next_out_packet(xpad);
+
+	spin_unlock_irqrestore(&xpad->odata_lock, flags);
 
-	xpad->odata[0] = 0x08;
-	xpad->odata[1] = 0x00;
-	xpad->odata[2] = 0x0F;
-	xpad->odata[3] = 0xC0;
-	xpad->odata[4] = 0x00;
-	xpad->odata[5] = 0x00;
-	xpad->odata[6] = 0x00;
-	xpad->odata[7] = 0x00;
-	xpad->odata[8] = 0x00;
-	xpad->odata[9] = 0x00;
-	xpad->odata[10] = 0x00;
-	xpad->odata[11] = 0x00;
-	xpad->irq_out->transfer_buffer_length = 12;
+	return retval;
+}
+
+static int xpad_start_xbox_one(struct usb_xpad *xpad)
+{
+	struct xpad_output_packet *packet =
+			&xpad->out_packets[XPAD_OUT_CMD_IDX];
+	unsigned long flags;
+	int retval;
 
-	retval = usb_submit_urb(xpad->irq_out, GFP_KERNEL);
+	spin_lock_irqsave(&xpad->odata_lock, flags);
 
-	mutex_unlock(&xpad->odata_mutex);
+	/* Xbox one controller needs to be initialized. */
+	packet->data[0] = 0x05;
+	packet->data[1] = 0x20;
+	packet->data[2] = xpad->odata_serial++; /* packet serial */
+	packet->data[3] = 0x01; /* rumble bit enable?  */
+	packet->data[4] = 0x00;
+	packet->len = 5;
+	packet->pending = true;
+
+	/* Reset the sequence so we send out start packet first */
+	xpad->last_out_packet = -1;
+	retval = xpad_try_sending_next_out_packet(xpad);
+
+	spin_unlock_irqrestore(&xpad->odata_lock, flags);
 
 	return retval;
 }
@@ -799,8 +957,11 @@ static int xpad_inquiry_pad_presence(struct usb_xpad *xpad)
 static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect *effect)
 {
 	struct usb_xpad *xpad = input_get_drvdata(dev);
+	struct xpad_output_packet *packet = &xpad->out_packets[XPAD_OUT_FF_IDX];
 	__u16 strong;
 	__u16 weak;
+	int retval;
+	unsigned long flags;
 
 	if (effect->type != FF_RUMBLE)
 		return 0;
@@ -808,69 +969,81 @@ static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect
 	strong = effect->u.rumble.strong_magnitude;
 	weak = effect->u.rumble.weak_magnitude;
 
+	spin_lock_irqsave(&xpad->odata_lock, flags);
+
 	switch (xpad->xtype) {
 	case XTYPE_XBOX:
-		xpad->odata[0] = 0x00;
-		xpad->odata[1] = 0x06;
-		xpad->odata[2] = 0x00;
-		xpad->odata[3] = strong / 256;	/* left actuator */
-		xpad->odata[4] = 0x00;
-		xpad->odata[5] = weak / 256;	/* right actuator */
-		xpad->irq_out->transfer_buffer_length = 6;
+		packet->data[0] = 0x00;
+		packet->data[1] = 0x06;
+		packet->data[2] = 0x00;
+		packet->data[3] = strong / 256;	/* left actuator */
+		packet->data[4] = 0x00;
+		packet->data[5] = weak / 256;	/* right actuator */
+		packet->len = 6;
+		packet->pending = true;
 		break;
 
 	case XTYPE_XBOX360:
-		xpad->odata[0] = 0x00;
-		xpad->odata[1] = 0x08;
-		xpad->odata[2] = 0x00;
-		xpad->odata[3] = strong / 256;  /* left actuator? */
-		xpad->odata[4] = weak / 256;	/* right actuator? */
-		xpad->odata[5] = 0x00;
-		xpad->odata[6] = 0x00;
-		xpad->odata[7] = 0x00;
-		xpad->irq_out->transfer_buffer_length = 8;
+		packet->data[0] = 0x00;
+		packet->data[1] = 0x08;
+		packet->data[2] = 0x00;
+		packet->data[3] = strong / 256;  /* left actuator? */
+		packet->data[4] = weak / 256;	/* right actuator? */
+		packet->data[5] = 0x00;
+		packet->data[6] = 0x00;
+		packet->data[7] = 0x00;
+		packet->len = 8;
+		packet->pending = true;
 		break;
 
 	case XTYPE_XBOX360W:
-		xpad->odata[0] = 0x00;
-		xpad->odata[1] = 0x01;
-		xpad->odata[2] = 0x0F;
-		xpad->odata[3] = 0xC0;
-		xpad->odata[4] = 0x00;
-		xpad->odata[5] = strong / 256;
-		xpad->odata[6] = weak / 256;
-		xpad->odata[7] = 0x00;
-		xpad->odata[8] = 0x00;
-		xpad->odata[9] = 0x00;
-		xpad->odata[10] = 0x00;
-		xpad->odata[11] = 0x00;
-		xpad->irq_out->transfer_buffer_length = 12;
+		packet->data[0] = 0x00;
+		packet->data[1] = 0x01;
+		packet->data[2] = 0x0F;
+		packet->data[3] = 0xC0;
+		packet->data[4] = 0x00;
+		packet->data[5] = strong / 256;
+		packet->data[6] = weak / 256;
+		packet->data[7] = 0x00;
+		packet->data[8] = 0x00;
+		packet->data[9] = 0x00;
+		packet->data[10] = 0x00;
+		packet->data[11] = 0x00;
+		packet->len = 12;
+		packet->pending = true;
 		break;
 
 	case XTYPE_XBOXONE:
-		xpad->odata[0] = 0x09; /* activate rumble */
-		xpad->odata[1] = 0x08;
-		xpad->odata[2] = 0x00;
-		xpad->odata[3] = 0x08; /* continuous effect */
-		xpad->odata[4] = 0x00; /* simple rumble mode */
-		xpad->odata[5] = 0x03; /* L and R actuator only */
-		xpad->odata[6] = 0x00; /* TODO: LT actuator */
-		xpad->odata[7] = 0x00; /* TODO: RT actuator */
-		xpad->odata[8] = strong / 256;	/* left actuator */
-		xpad->odata[9] = weak / 256;	/* right actuator */
-		xpad->odata[10] = 0x80;	/* length of pulse */
-		xpad->odata[11] = 0x00;	/* stop period of pulse */
-		xpad->irq_out->transfer_buffer_length = 12;
+		packet->data[0] = 0x09; /* activate rumble */
+		packet->data[1] = 0x08;
+		packet->data[2] = xpad->odata_serial++;
+		packet->data[3] = 0x08; /* continuous effect */
+		packet->data[4] = 0x00; /* simple rumble mode */
+		packet->data[5] = 0x03; /* L and R actuator only */
+		packet->data[6] = 0x00; /* TODO: LT actuator */
+		packet->data[7] = 0x00; /* TODO: RT actuator */
+		packet->data[8] = strong / 512;	/* left actuator */
+		packet->data[9] = weak / 512;	/* right actuator */
+		packet->data[10] = 0x80;	/* length of pulse */
+		packet->data[11] = 0x00;	/* stop period of pulse */
+		packet->data[12] = 0x00;
+		packet->len = 13;
+		packet->pending = true;
 		break;
 
 	default:
 		dev_dbg(&xpad->dev->dev,
 			"%s - rumble command sent to unsupported xpad type: %d\n",
 			__func__, xpad->xtype);
-		return -EINVAL;
+		retval = -EINVAL;
+		goto out;
 	}
 
-	return usb_submit_urb(xpad->irq_out, GFP_ATOMIC);
+	retval = xpad_try_sending_next_out_packet(xpad);
+
+out:
+	spin_unlock_irqrestore(&xpad->odata_lock, flags);
+	return retval;
 }
 
 static int xpad_init_ff(struct usb_xpad *xpad)
@@ -921,36 +1094,44 @@ struct xpad_led {
  */
 static void xpad_send_led_command(struct usb_xpad *xpad, int command)
 {
+	struct xpad_output_packet *packet =
+			&xpad->out_packets[XPAD_OUT_LED_IDX];
+	unsigned long flags;
+
 	command %= 16;
 
-	mutex_lock(&xpad->odata_mutex);
+	spin_lock_irqsave(&xpad->odata_lock, flags);
 
 	switch (xpad->xtype) {
 	case XTYPE_XBOX360:
-		xpad->odata[0] = 0x01;
-		xpad->odata[1] = 0x03;
-		xpad->odata[2] = command;
-		xpad->irq_out->transfer_buffer_length = 3;
+		packet->data[0] = 0x01;
+		packet->data[1] = 0x03;
+		packet->data[2] = command;
+		packet->len = 3;
+		packet->pending = true;
 		break;
+
 	case XTYPE_XBOX360W:
-		xpad->odata[0] = 0x00;
-		xpad->odata[1] = 0x00;
-		xpad->odata[2] = 0x08;
-		xpad->odata[3] = 0x40 + command;
-		xpad->odata[4] = 0x00;
-		xpad->odata[5] = 0x00;
-		xpad->odata[6] = 0x00;
-		xpad->odata[7] = 0x00;
-		xpad->odata[8] = 0x00;
-		xpad->odata[9] = 0x00;
-		xpad->odata[10] = 0x00;
-		xpad->odata[11] = 0x00;
-		xpad->irq_out->transfer_buffer_length = 12;
+		packet->data[0] = 0x00;
+		packet->data[1] = 0x00;
+		packet->data[2] = 0x08;
+		packet->data[3] = 0x40 + command;
+		packet->data[4] = 0x00;
+		packet->data[5] = 0x00;
+		packet->data[6] = 0x00;
+		packet->data[7] = 0x00;
+		packet->data[8] = 0x00;
+		packet->data[9] = 0x00;
+		packet->data[10] = 0x00;
+		packet->data[11] = 0x00;
+		packet->len = 12;
+		packet->pending = true;
 		break;
 	}
 
-	usb_submit_urb(xpad->irq_out, GFP_KERNEL);
-	mutex_unlock(&xpad->odata_mutex);
+	xpad_try_sending_next_out_packet(xpad);
+
+	spin_unlock_irqrestore(&xpad->odata_lock, flags);
 }
 
 /*
@@ -959,7 +1140,7 @@ static void xpad_send_led_command(struct usb_xpad *xpad, int command)
  */
 static void xpad_identify_controller(struct usb_xpad *xpad)
 {
-	xpad_send_led_command(xpad, (xpad->pad_nr % 4) + 2);
+	led_set_brightness(&xpad->led->led_cdev, (xpad->pad_nr % 4) + 2);
 }
 
 static void xpad_led_set(struct led_classdev *led_cdev,
@@ -1001,14 +1182,7 @@ static int xpad_led_probe(struct usb_xpad *xpad)
 	if (error)
 		goto err_free_id;
 
-	if (xpad->xtype == XTYPE_XBOX360) {
-		/*
-		 * Light up the segment corresponding to controller
-		 * number on wired devices. On wireless we'll do that
-		 * when they respond to "presence" packet.
-		 */
-		xpad_identify_controller(xpad);
-	}
+	xpad_identify_controller(xpad);
 
 	return 0;
 
@@ -1036,37 +1210,73 @@ static void xpad_led_disconnect(struct usb_xpad *xpad) { }
 static void xpad_identify_controller(struct usb_xpad *xpad) { }
 #endif
 
-static int xpad_open(struct input_dev *dev)
+static int xpad_start_input(struct usb_xpad *xpad)
 {
-	struct usb_xpad *xpad = input_get_drvdata(dev);
-
-	/* URB was submitted in probe */
-	if (xpad->xtype == XTYPE_XBOX360W)
-		return 0;
+	int error;
 
-	xpad->irq_in->dev = xpad->udev;
 	if (usb_submit_urb(xpad->irq_in, GFP_KERNEL))
 		return -EIO;
 
 	if (xpad->xtype == XTYPE_XBOXONE) {
-		/* Xbox one controller needs to be initialized. */
-		xpad->odata[0] = 0x05;
-		xpad->odata[1] = 0x20;
-		xpad->irq_out->transfer_buffer_length = 2;
-		return usb_submit_urb(xpad->irq_out, GFP_KERNEL);
+		error = xpad_start_xbox_one(xpad);
+		if (error) {
+			usb_kill_urb(xpad->irq_in);
+			return error;
+		}
 	}
 
 	return 0;
 }
 
-static void xpad_close(struct input_dev *dev)
+static void xpad_stop_input(struct usb_xpad *xpad)
 {
-	struct usb_xpad *xpad = input_get_drvdata(dev);
+	usb_kill_urb(xpad->irq_in);
+}
+
+static int xpad360w_start_input(struct usb_xpad *xpad)
+{
+	int error;
 
-	if (xpad->xtype != XTYPE_XBOX360W)
+	error = usb_submit_urb(xpad->irq_in, GFP_KERNEL);
+	if (error)
+		return -EIO;
+
+	/*
+	 * Send presence packet.
+	 * This will force the controller to resend connection packets.
+	 * This is useful in the case we activate the module after the
+	 * adapter has been plugged in, as it won't automatically
+	 * send us info about the controllers.
+	 */
+	error = xpad_inquiry_pad_presence(xpad);
+	if (error) {
 		usb_kill_urb(xpad->irq_in);
+		return error;
+	}
 
-	xpad_stop_output(xpad);
+	return 0;
+}
+
+static void xpad360w_stop_input(struct usb_xpad *xpad)
+{
+	usb_kill_urb(xpad->irq_in);
+
+	/* Make sure we are done with presence work if it was scheduled */
+	flush_work(&xpad->work);
+}
+
+static int xpad_open(struct input_dev *dev)
+{
+	struct usb_xpad *xpad = input_get_drvdata(dev);
+
+	return xpad_start_input(xpad);
+}
+
+static void xpad_close(struct input_dev *dev)
+{
+	struct usb_xpad *xpad = input_get_drvdata(dev);
+
+	xpad_stop_input(xpad);
 }
 
 static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs)
@@ -1097,8 +1307,11 @@ static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs)
 
 static void xpad_deinit_input(struct usb_xpad *xpad)
 {
-	xpad_led_disconnect(xpad);
-	input_unregister_device(xpad->dev);
+	if (xpad->input_created) {
+		xpad->input_created = false;
+		xpad_led_disconnect(xpad);
+		input_unregister_device(xpad->dev);
+	}
 }
 
 static int xpad_init_input(struct usb_xpad *xpad)
@@ -1118,8 +1331,10 @@ static int xpad_init_input(struct usb_xpad *xpad)
 
 	input_set_drvdata(input_dev, xpad);
 
-	input_dev->open = xpad_open;
-	input_dev->close = xpad_close;
+	if (xpad->xtype != XTYPE_XBOX360W) {
+		input_dev->open = xpad_open;
+		input_dev->close = xpad_close;
+	}
 
 	__set_bit(EV_KEY, input_dev->evbit);
 
@@ -1181,6 +1396,7 @@ static int xpad_init_input(struct usb_xpad *xpad)
 	if (error)
 		goto err_disconnect_led;
 
+	xpad->input_created = true;
 	return 0;
 
 err_disconnect_led:
@@ -1241,6 +1457,7 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id
 	xpad->mapping = xpad_device[i].mapping;
 	xpad->xtype = xpad_device[i].xtype;
 	xpad->name = xpad_device[i].name;
+	INIT_WORK(&xpad->work, xpad_presence_work);
 
 	if (xpad->xtype == XTYPE_UNKNOWN) {
 		if (intf->cur_altsetting->desc.bInterfaceClass == USB_CLASS_VENDOR_SPEC) {
@@ -1277,10 +1494,6 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id
 
 	usb_set_intfdata(intf, xpad);
 
-	error = xpad_init_input(xpad);
-	if (error)
-		goto err_deinit_output;
-
 	if (xpad->xtype == XTYPE_XBOX360W) {
 		/*
 		 * Submit the int URB immediately rather than waiting for open
@@ -1289,28 +1502,24 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id
 		 * exactly the message that a controller has arrived that
 		 * we're waiting for.
 		 */
-		xpad->irq_in->dev = xpad->udev;
-		error = usb_submit_urb(xpad->irq_in, GFP_KERNEL);
+		error = xpad360w_start_input(xpad);
 		if (error)
-			goto err_deinit_input;
-
+			goto err_deinit_output;
 		/*
-		 * Send presence packet.
-		 * This will force the controller to resend connection packets.
-		 * This is useful in the case we activate the module after the
-		 * adapter has been plugged in, as it won't automatically
-		 * send us info about the controllers.
+		 * Wireless controllers require RESET_RESUME to work properly
+		 * after suspend. Ideally this quirk should be in usb core
+		 * quirk list, but we have too many vendors producing these
+		 * controllers and we'd need to maintain 2 identical lists
+		 * here in this driver and in usb core.
 		 */
-		error = xpad_inquiry_pad_presence(xpad);
+		udev->quirks |= USB_QUIRK_RESET_RESUME;
+	} else {
+		error = xpad_init_input(xpad);
 		if (error)
-			goto err_kill_in_urb;
+			goto err_deinit_output;
 	}
 	return 0;
 
-err_kill_in_urb:
-	usb_kill_urb(xpad->irq_in);
-err_deinit_input:
-	xpad_deinit_input(xpad);
 err_deinit_output:
 	xpad_deinit_output(xpad);
 err_free_in_urb:
@@ -1320,19 +1529,24 @@ err_free_idata:
 err_free_mem:
 	kfree(xpad);
 	return error;
-
 }
 
 static void xpad_disconnect(struct usb_interface *intf)
 {
-	struct usb_xpad *xpad = usb_get_intfdata (intf);
+	struct usb_xpad *xpad = usb_get_intfdata(intf);
+
+	if (xpad->xtype == XTYPE_XBOX360W)
+		xpad360w_stop_input(xpad);
 
 	xpad_deinit_input(xpad);
-	xpad_deinit_output(xpad);
 
-	if (xpad->xtype == XTYPE_XBOX360W) {
-		usb_kill_urb(xpad->irq_in);
-	}
+	/*
+	 * Now that both input device and LED device are gone we can
+	 * stop output URB.
+	 */
+	xpad_stop_output(xpad);
+
+	xpad_deinit_output(xpad);
 
 	usb_free_urb(xpad->irq_in);
 	usb_free_coherent(xpad->udev, XPAD_PKT_LEN,
@@ -1343,10 +1557,55 @@ static void xpad_disconnect(struct usb_interface *intf)
 	usb_set_intfdata(intf, NULL);
 }
 
+static int xpad_suspend(struct usb_interface *intf, pm_message_t message)
+{
+	struct usb_xpad *xpad = usb_get_intfdata(intf);
+	struct input_dev *input = xpad->dev;
+
+	if (xpad->xtype == XTYPE_XBOX360W) {
+		/*
+		 * Wireless controllers always listen to input so
+		 * they are notified when controller shows up
+		 * or goes away.
+		 */
+		xpad360w_stop_input(xpad);
+	} else {
+		mutex_lock(&input->mutex);
+		if (input->users)
+			xpad_stop_input(xpad);
+		mutex_unlock(&input->mutex);
+	}
+
+	xpad_stop_output(xpad);
+
+	return 0;
+}
+
+static int xpad_resume(struct usb_interface *intf)
+{
+	struct usb_xpad *xpad = usb_get_intfdata(intf);
+	struct input_dev *input = xpad->dev;
+	int retval = 0;
+
+	if (xpad->xtype == XTYPE_XBOX360W) {
+		retval = xpad360w_start_input(xpad);
+	} else {
+		mutex_lock(&input->mutex);
+		if (input->users)
+			retval = xpad_start_input(xpad);
+		mutex_unlock(&input->mutex);
+	}
+
+	return retval;
+}
+
 static struct usb_driver xpad_driver = {
 	.name		= "xpad",
 	.probe		= xpad_probe,
 	.disconnect	= xpad_disconnect,
+	.suspend	= xpad_suspend,
+	.resume		= xpad_resume,
+	.reset_resume	= xpad_resume,
 	.id_table	= xpad_table,
 };
 
diff --git a/drivers/input/keyboard/gpio_keys.c b/drivers/input/keyboard/gpio_keys.c
index b9f01bd1b7ef..29093657f2ef 100644
--- a/drivers/input/keyboard/gpio_keys.c
+++ b/drivers/input/keyboard/gpio_keys.c
@@ -630,7 +630,7 @@ gpio_keys_get_devtree_pdata(struct device *dev)
 	if (!node)
 		return ERR_PTR(-ENODEV);
 
-	nbuttons = of_get_child_count(node);
+	nbuttons = of_get_available_child_count(node);
 	if (nbuttons == 0)
 		return ERR_PTR(-ENODEV);
 
@@ -645,8 +645,10 @@ gpio_keys_get_devtree_pdata(struct device *dev)
 
 	pdata->rep = !!of_get_property(node, "autorepeat", NULL);
 
+	of_property_read_string(node, "label", &pdata->name);
+
 	i = 0;
-	for_each_child_of_node(node, pp) {
+	for_each_available_child_of_node(node, pp) {
 		enum of_gpio_flags flags;
 
 		button = &pdata->buttons[i++];
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index 53a97b379c9f..66c62641b59a 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -376,7 +376,7 @@ config TOUCHSCREEN_IPROC
 config TOUCHSCREEN_S3C2410
 	tristate "Samsung S3C2410/generic touchscreen input driver"
 	depends on ARCH_S3C24XX || SAMSUNG_DEV_TS
-	select S3C_ADC
+	depends on S3C_ADC
 	help
 	  Say Y here if you have the s3c2410 touchscreen.
 
diff --git a/drivers/input/touchscreen/ad7879.c b/drivers/input/touchscreen/ad7879.c
index fec66ad80513..16b5cc2196f2 100644
--- a/drivers/input/touchscreen/ad7879.c
+++ b/drivers/input/touchscreen/ad7879.c
@@ -454,7 +454,7 @@ static int ad7879_gpio_add(struct ad7879 *ts,
 		ts->gc.ngpio = 1;
 		ts->gc.label = "AD7879-GPIO";
 		ts->gc.owner = THIS_MODULE;
-		ts->gc.dev = ts->dev;
+		ts->gc.parent = ts->dev;
 
 		ret = gpiochip_add(&ts->gc);
 		if (ret)
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index 2d5794ec338b..2160512e861a 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -113,8 +113,8 @@ struct t7_config {
 #define MXT_T9_DETECT		(1 << 7)
 
 struct t9_range {
-	u16 x;
-	u16 y;
+	__le16 x;
+	__le16 y;
 } __packed;
 
 /* MXT_TOUCH_MULTI_T9 orient */
@@ -216,6 +216,7 @@ struct mxt_data {
 	unsigned int irq;
 	unsigned int max_x;
 	unsigned int max_y;
+	bool xy_switch;
 	bool in_bootloader;
 	u16 mem_size;
 	u8 t100_aux_ampl;
@@ -1665,8 +1666,8 @@ static int mxt_read_t9_resolution(struct mxt_data *data)
 	if (error)
 		return error;
 
-	le16_to_cpus(&range.x);
-	le16_to_cpus(&range.y);
+	data->max_x = get_unaligned_le16(&range.x);
+	data->max_y = get_unaligned_le16(&range.y);
 
 	error =  __mxt_read_reg(client,
 				object->start_address + MXT_T9_ORIENT,
@@ -1674,23 +1675,7 @@ static int mxt_read_t9_resolution(struct mxt_data *data)
 	if (error)
 		return error;
 
-	/* Handle default values */
-	if (range.x == 0)
-		range.x = 1023;
-
-	if (range.y == 0)
-		range.y = 1023;
-
-	if (orient & MXT_T9_ORIENT_SWITCH) {
-		data->max_x = range.y;
-		data->max_y = range.x;
-	} else {
-		data->max_x = range.x;
-		data->max_y = range.y;
-	}
-
-	dev_dbg(&client->dev,
-		"Touchscreen size X%uY%u\n", data->max_x, data->max_y);
+	data->xy_switch = orient & MXT_T9_ORIENT_SWITCH;
 
 	return 0;
 }
@@ -1708,13 +1693,14 @@ static int mxt_read_t100_config(struct mxt_data *data)
 	if (!object)
 		return -EINVAL;
 
+	/* read touchscreen dimensions */
 	error = __mxt_read_reg(client,
 			       object->start_address + MXT_T100_XRANGE,
 			       sizeof(range_x), &range_x);
 	if (error)
 		return error;
 
-	le16_to_cpus(&range_x);
+	data->max_x = get_unaligned_le16(&range_x);
 
 	error = __mxt_read_reg(client,
 			       object->start_address + MXT_T100_YRANGE,
@@ -1722,36 +1708,24 @@ static int mxt_read_t100_config(struct mxt_data *data)
 	if (error)
 		return error;
 
-	le16_to_cpus(&range_y);
+	data->max_y = get_unaligned_le16(&range_y);
 
+	/* read orientation config */
 	error =  __mxt_read_reg(client,
 				object->start_address + MXT_T100_CFG1,
 				1, &cfg);
 	if (error)
 		return error;
 
+	data->xy_switch = cfg & MXT_T100_CFG_SWITCHXY;
+
+	/* allocate aux bytes */
 	error =  __mxt_read_reg(client,
 				object->start_address + MXT_T100_TCHAUX,
 				1, &tchaux);
 	if (error)
 		return error;
 
-	/* Handle default values */
-	if (range_x == 0)
-		range_x = 1023;
-
-	if (range_y == 0)
-		range_y = 1023;
-
-	if (cfg & MXT_T100_CFG_SWITCHXY) {
-		data->max_x = range_y;
-		data->max_y = range_x;
-	} else {
-		data->max_x = range_x;
-		data->max_y = range_y;
-	}
-
-	/* allocate aux bytes */
 	aux = 6;
 
 	if (tchaux & MXT_T100_TCHAUX_VECT)
@@ -1767,9 +1741,6 @@ static int mxt_read_t100_config(struct mxt_data *data)
 		"T100 aux mappings vect:%u ampl:%u area:%u\n",
 		data->t100_aux_vect, data->t100_aux_ampl, data->t100_aux_area);
 
-	dev_info(&client->dev,
-		 "T100 Touchscreen size X%uY%u\n", data->max_x, data->max_y);
-
 	return 0;
 }
 
@@ -1828,6 +1799,19 @@ static int mxt_initialize_input_device(struct mxt_data *data)
 		return -EINVAL;
 	}
 
+	/* Handle default values and orientation switch */
+	if (data->max_x == 0)
+		data->max_x = 1023;
+
+	if (data->max_y == 0)
+		data->max_y = 1023;
+
+	if (data->xy_switch)
+		swap(data->max_x, data->max_y);
+
+	dev_info(dev, "Touchscreen size X%uY%u\n", data->max_x, data->max_y);
+
+	/* Register input device */
 	input_dev = input_allocate_device();
 	if (!input_dev) {
 		dev_err(dev, "Failed to allocate memory\n");
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index b9094e9da537..a1e75cba18e0 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -263,81 +263,6 @@ config EXYNOS_IOMMU_DEBUG
 
 	  Say N unless you need kernel log message for IOMMU debugging.
 
-config SHMOBILE_IPMMU
-	bool
-
-config SHMOBILE_IPMMU_TLB
-	bool
-
-config SHMOBILE_IOMMU
-	bool "IOMMU for Renesas IPMMU/IPMMUI"
-	default n
-	depends on ARM && MMU
-	depends on ARCH_SHMOBILE || COMPILE_TEST
-	select IOMMU_API
-	select ARM_DMA_USE_IOMMU
-	select SHMOBILE_IPMMU
-	select SHMOBILE_IPMMU_TLB
-	help
-	  Support for Renesas IPMMU/IPMMUI. This option enables
-	  remapping of DMA memory accesses from all of the IP blocks
-	  on the ICB.
-
-	  Warning: Drivers (including userspace drivers of UIO
-	  devices) of the IP blocks on the ICB *must* use addresses
-	  allocated from the IPMMU (iova) for DMA with this option
-	  enabled.
-
-	  If unsure, say N.
-
-choice
-	prompt "IPMMU/IPMMUI address space size"
-	default SHMOBILE_IOMMU_ADDRSIZE_2048MB
-	depends on SHMOBILE_IOMMU
-	help
-	  This option sets IPMMU/IPMMUI address space size by
-	  adjusting the 1st level page table size. The page table size
-	  is calculated as follows:
-
-	      page table size = number of page table entries * 4 bytes
-	      number of page table entries = address space size / 1 MiB
-
-	  For example, when the address space size is 2048 MiB, the
-	  1st level page table size is 8192 bytes.
-
-	config SHMOBILE_IOMMU_ADDRSIZE_2048MB
-		bool "2 GiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_1024MB
-		bool "1 GiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_512MB
-		bool "512 MiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_256MB
-		bool "256 MiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_128MB
-		bool "128 MiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_64MB
-		bool "64 MiB"
-
-	config SHMOBILE_IOMMU_ADDRSIZE_32MB
-		bool "32 MiB"
-
-endchoice
-
-config SHMOBILE_IOMMU_L1SIZE
-	int
-	default 8192 if SHMOBILE_IOMMU_ADDRSIZE_2048MB
-	default 4096 if SHMOBILE_IOMMU_ADDRSIZE_1024MB
-	default 2048 if SHMOBILE_IOMMU_ADDRSIZE_512MB
-	default 1024 if SHMOBILE_IOMMU_ADDRSIZE_256MB
-	default 512 if SHMOBILE_IOMMU_ADDRSIZE_128MB
-	default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB
-	default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB
-
 config IPMMU_VMSA
 	bool "Renesas VMSA-compatible IPMMU"
 	depends on ARM_LPAE
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 68faca02225d..42fc0c25cf1a 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -22,7 +22,5 @@ obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
 obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
 obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
 obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
-obj-$(CONFIG_SHMOBILE_IOMMU) += shmobile-iommu.o
-obj-$(CONFIG_SHMOBILE_IPMMU) += shmobile-ipmmu.o
 obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
 obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 8b2be1e7714f..e5e223938eec 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -35,6 +35,7 @@
 #include <linux/msi.h>
 #include <linux/dma-contiguous.h>
 #include <linux/irqdomain.h>
+#include <linux/percpu.h>
 #include <asm/irq_remapping.h>
 #include <asm/io_apic.h>
 #include <asm/apic.h>
@@ -114,6 +115,45 @@ struct kmem_cache *amd_iommu_irq_cache;
 static void update_domain(struct protection_domain *domain);
 static int protection_domain_init(struct protection_domain *domain);
 
+/*
+ * For dynamic growth the aperture size is split into ranges of 128MB of
+ * DMA address space each. This struct represents one such range.
+ */
+struct aperture_range {
+
+	spinlock_t bitmap_lock;
+
+	/* address allocation bitmap */
+	unsigned long *bitmap;
+	unsigned long offset;
+	unsigned long next_bit;
+
+	/*
+	 * Array of PTE pages for the aperture. In this array we save all the
+	 * leaf pages of the domain page table used for the aperture. This way
+	 * we don't need to walk the page table to find a specific PTE. We can
+	 * just calculate its address in constant time.
+	 */
+	u64 *pte_pages[64];
+};
+
+/*
+ * Data container for a dma_ops specific protection domain
+ */
+struct dma_ops_domain {
+	/* generic protection domain information */
+	struct protection_domain domain;
+
+	/* size of the aperture for the mappings */
+	unsigned long aperture_size;
+
+	/* aperture index we start searching for free addresses */
+	u32 __percpu *next_index;
+
+	/* address space relevant data */
+	struct aperture_range *aperture[APERTURE_MAX_RANGES];
+};
+
 /****************************************************************************
  *
  * Helper functions
@@ -1167,11 +1207,21 @@ static u64 *alloc_pte(struct protection_domain *domain,
 	end_lvl = PAGE_SIZE_LEVEL(page_size);
 
 	while (level > end_lvl) {
-		if (!IOMMU_PTE_PRESENT(*pte)) {
+		u64 __pte, __npte;
+
+		__pte = *pte;
+
+		if (!IOMMU_PTE_PRESENT(__pte)) {
 			page = (u64 *)get_zeroed_page(gfp);
 			if (!page)
 				return NULL;
-			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
+
+			__npte = PM_LEVEL_PDE(level, virt_to_phys(page));
+
+			if (cmpxchg64(pte, __pte, __npte)) {
+				free_page((unsigned long)page);
+				continue;
+			}
 		}
 
 		/* No level skipping support yet */
@@ -1376,8 +1426,10 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 			   bool populate, gfp_t gfp)
 {
 	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
-	struct amd_iommu *iommu;
 	unsigned long i, old_size, pte_pgsize;
+	struct aperture_range *range;
+	struct amd_iommu *iommu;
+	unsigned long flags;
 
 #ifdef CONFIG_IOMMU_STRESS
 	populate = false;
@@ -1386,15 +1438,17 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 	if (index >= APERTURE_MAX_RANGES)
 		return -ENOMEM;
 
-	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
-	if (!dma_dom->aperture[index])
+	range = kzalloc(sizeof(struct aperture_range), gfp);
+	if (!range)
 		return -ENOMEM;
 
-	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
-	if (!dma_dom->aperture[index]->bitmap)
+	range->bitmap = (void *)get_zeroed_page(gfp);
+	if (!range->bitmap)
 		goto out_free;
 
-	dma_dom->aperture[index]->offset = dma_dom->aperture_size;
+	range->offset = dma_dom->aperture_size;
+
+	spin_lock_init(&range->bitmap_lock);
 
 	if (populate) {
 		unsigned long address = dma_dom->aperture_size;
@@ -1407,14 +1461,20 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 			if (!pte)
 				goto out_free;
 
-			dma_dom->aperture[index]->pte_pages[i] = pte_page;
+			range->pte_pages[i] = pte_page;
 
 			address += APERTURE_RANGE_SIZE / 64;
 		}
 	}
 
-	old_size                = dma_dom->aperture_size;
-	dma_dom->aperture_size += APERTURE_RANGE_SIZE;
+	spin_lock_irqsave(&dma_dom->domain.lock, flags);
+
+	/* First take the bitmap_lock and then publish the range */
+	spin_lock(&range->bitmap_lock);
+
+	old_size                 = dma_dom->aperture_size;
+	dma_dom->aperture[index] = range;
+	dma_dom->aperture_size  += APERTURE_RANGE_SIZE;
 
 	/* Reserve address range used for MSI messages */
 	if (old_size < MSI_ADDR_BASE_LO &&
@@ -1461,62 +1521,123 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 
 	update_domain(&dma_dom->domain);
 
+	spin_unlock(&range->bitmap_lock);
+
+	spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
+
 	return 0;
 
 out_free:
 	update_domain(&dma_dom->domain);
 
-	free_page((unsigned long)dma_dom->aperture[index]->bitmap);
+	free_page((unsigned long)range->bitmap);
 
-	kfree(dma_dom->aperture[index]);
-	dma_dom->aperture[index] = NULL;
+	kfree(range);
 
 	return -ENOMEM;
 }
 
+static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom,
+					 struct aperture_range *range,
+					 unsigned long pages,
+					 unsigned long dma_mask,
+					 unsigned long boundary_size,
+					 unsigned long align_mask,
+					 bool trylock)
+{
+	unsigned long offset, limit, flags;
+	dma_addr_t address;
+	bool flush = false;
+
+	offset = range->offset >> PAGE_SHIFT;
+	limit  = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
+					dma_mask >> PAGE_SHIFT);
+
+	if (trylock) {
+		if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
+			return -1;
+	} else {
+		spin_lock_irqsave(&range->bitmap_lock, flags);
+	}
+
+	address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
+				   pages, offset, boundary_size, align_mask);
+	if (address == -1) {
+		/* Nothing found, retry one time */
+		address = iommu_area_alloc(range->bitmap, limit,
+					   0, pages, offset, boundary_size,
+					   align_mask);
+		flush = true;
+	}
+
+	if (address != -1)
+		range->next_bit = address + pages;
+
+	spin_unlock_irqrestore(&range->bitmap_lock, flags);
+
+	if (flush) {
+		domain_flush_tlb(&dom->domain);
+		domain_flush_complete(&dom->domain);
+	}
+
+	return address;
+}
+
 static unsigned long dma_ops_area_alloc(struct device *dev,
 					struct dma_ops_domain *dom,
 					unsigned int pages,
 					unsigned long align_mask,
-					u64 dma_mask,
-					unsigned long start)
+					u64 dma_mask)
 {
-	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
-	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
-	int i = start >> APERTURE_RANGE_SHIFT;
 	unsigned long boundary_size, mask;
 	unsigned long address = -1;
-	unsigned long limit;
+	bool first = true;
+	u32 start, i;
 
-	next_bit >>= PAGE_SHIFT;
+	preempt_disable();
 
 	mask = dma_get_seg_boundary(dev);
 
+again:
+	start = this_cpu_read(*dom->next_index);
+
+	/* Sanity check - is it really necessary? */
+	if (unlikely(start > APERTURE_MAX_RANGES)) {
+		start = 0;
+		this_cpu_write(*dom->next_index, 0);
+	}
+
 	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
 				   1UL << (BITS_PER_LONG - PAGE_SHIFT);
 
-	for (;i < max_index; ++i) {
-		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
+	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
+		struct aperture_range *range;
+		int index;
 
-		if (dom->aperture[i]->offset >= dma_mask)
-			break;
+		index = (start + i) % APERTURE_MAX_RANGES;
 
-		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
-					       dma_mask >> PAGE_SHIFT);
+		range = dom->aperture[index];
 
-		address = iommu_area_alloc(dom->aperture[i]->bitmap,
-					   limit, next_bit, pages, 0,
-					    boundary_size, align_mask);
+		if (!range || range->offset >= dma_mask)
+			continue;
+
+		address = dma_ops_aperture_alloc(dom, range, pages,
+						 dma_mask, boundary_size,
+						 align_mask, first);
 		if (address != -1) {
-			address = dom->aperture[i]->offset +
-				  (address << PAGE_SHIFT);
-			dom->next_address = address + (pages << PAGE_SHIFT);
+			address = range->offset + (address << PAGE_SHIFT);
+			this_cpu_write(*dom->next_index, index);
 			break;
 		}
+	}
 
-		next_bit = 0;
+	if (address == -1 && first) {
+		first = false;
+		goto again;
 	}
 
+	preempt_enable();
+
 	return address;
 }
 
@@ -1526,21 +1647,14 @@ static unsigned long dma_ops_alloc_addresses(struct device *dev,
 					     unsigned long align_mask,
 					     u64 dma_mask)
 {
-	unsigned long address;
-
-#ifdef CONFIG_IOMMU_STRESS
-	dom->next_address = 0;
-	dom->need_flush = true;
-#endif
+	unsigned long address = -1;
 
-	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
-				     dma_mask, dom->next_address);
+	while (address == -1) {
+		address = dma_ops_area_alloc(dev, dom, pages,
+					     align_mask, dma_mask);
 
-	if (address == -1) {
-		dom->next_address = 0;
-		address = dma_ops_area_alloc(dev, dom, pages, align_mask,
-					     dma_mask, 0);
-		dom->need_flush = true;
+		if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
+			break;
 	}
 
 	if (unlikely(address == -1))
@@ -1562,6 +1676,7 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
 {
 	unsigned i = address >> APERTURE_RANGE_SHIFT;
 	struct aperture_range *range = dom->aperture[i];
+	unsigned long flags;
 
 	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
 
@@ -1570,12 +1685,18 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
 		return;
 #endif
 
-	if (address >= dom->next_address)
-		dom->need_flush = true;
+	if (amd_iommu_unmap_flush) {
+		domain_flush_tlb(&dom->domain);
+		domain_flush_complete(&dom->domain);
+	}
 
 	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
 
+	spin_lock_irqsave(&range->bitmap_lock, flags);
+	if (address + pages > range->next_bit)
+		range->next_bit = address + pages;
 	bitmap_clear(range->bitmap, address, pages);
+	spin_unlock_irqrestore(&range->bitmap_lock, flags);
 
 }
 
@@ -1755,6 +1876,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 	if (!dom)
 		return;
 
+	free_percpu(dom->next_index);
+
 	del_domain_from_list(&dom->domain);
 
 	free_pagetable(&dom->domain);
@@ -1769,6 +1892,23 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 	kfree(dom);
 }
 
+static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
+					  int max_apertures)
+{
+	int ret, i, apertures;
+
+	apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
+	ret       = 0;
+
+	for (i = apertures; i < max_apertures; ++i) {
+		ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
 /*
  * Allocates a new protection domain usable for the dma_ops functions.
  * It also initializes the page table and the address allocator data
@@ -1777,6 +1917,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 static struct dma_ops_domain *dma_ops_domain_alloc(void)
 {
 	struct dma_ops_domain *dma_dom;
+	int cpu;
 
 	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
 	if (!dma_dom)
@@ -1785,6 +1926,10 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	if (protection_domain_init(&dma_dom->domain))
 		goto free_dma_dom;
 
+	dma_dom->next_index = alloc_percpu(u32);
+	if (!dma_dom->next_index)
+		goto free_dma_dom;
+
 	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
 	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
 	dma_dom->domain.flags = PD_DMA_OPS_MASK;
@@ -1792,8 +1937,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	if (!dma_dom->domain.pt_root)
 		goto free_dma_dom;
 
-	dma_dom->need_flush = false;
-
 	add_domain_to_list(&dma_dom->domain);
 
 	if (alloc_new_range(dma_dom, true, GFP_KERNEL))
@@ -1804,8 +1947,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	 * a valid dma-address. So we can use 0 as error value
 	 */
 	dma_dom->aperture[0]->bitmap[0] = 1;
-	dma_dom->next_address = 0;
 
+	for_each_possible_cpu(cpu)
+		*per_cpu_ptr(dma_dom->next_index, cpu) = 0;
 
 	return dma_dom;
 
@@ -1905,7 +2049,7 @@ static void do_attach(struct iommu_dev_data *dev_data,
 	/* Update device table */
 	set_dte_entry(dev_data->devid, domain, ats);
 	if (alias != dev_data->devid)
-		set_dte_entry(dev_data->devid, domain, ats);
+		set_dte_entry(alias, domain, ats);
 
 	device_flush_dte(dev_data);
 }
@@ -2328,7 +2472,7 @@ static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
 	else if (direction == DMA_BIDIRECTIONAL)
 		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
 
-	WARN_ON(*pte);
+	WARN_ON_ONCE(*pte);
 
 	*pte = __pte;
 
@@ -2357,7 +2501,7 @@ static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
 
 	pte += PM_LEVEL_INDEX(0, address);
 
-	WARN_ON(!*pte);
+	WARN_ON_ONCE(!*pte);
 
 	*pte = 0ULL;
 }
@@ -2393,26 +2537,11 @@ static dma_addr_t __map_single(struct device *dev,
 	if (align)
 		align_mask = (1UL << get_order(size)) - 1;
 
-retry:
 	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
 					  dma_mask);
-	if (unlikely(address == DMA_ERROR_CODE)) {
-		/*
-		 * setting next_address here will let the address
-		 * allocator only scan the new allocated range in the
-		 * first run. This is a small optimization.
-		 */
-		dma_dom->next_address = dma_dom->aperture_size;
 
-		if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
-			goto out;
-
-		/*
-		 * aperture was successfully enlarged by 128 MB, try
-		 * allocation again
-		 */
-		goto retry;
-	}
+	if (address == DMA_ERROR_CODE)
+		goto out;
 
 	start = address;
 	for (i = 0; i < pages; ++i) {
@@ -2427,11 +2556,10 @@ retry:
 
 	ADD_STATS_COUNTER(alloced_io_mem, size);
 
-	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
-		domain_flush_tlb(&dma_dom->domain);
-		dma_dom->need_flush = false;
-	} else if (unlikely(amd_iommu_np_cache))
+	if (unlikely(amd_iommu_np_cache)) {
 		domain_flush_pages(&dma_dom->domain, address, size);
+		domain_flush_complete(&dma_dom->domain);
+	}
 
 out:
 	return address;
@@ -2478,11 +2606,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
 	SUB_STATS_COUNTER(alloced_io_mem, size);
 
 	dma_ops_free_addresses(dma_dom, dma_addr, pages);
-
-	if (amd_iommu_unmap_flush || dma_dom->need_flush) {
-		domain_flush_pages(&dma_dom->domain, flush_addr, size);
-		dma_dom->need_flush = false;
-	}
 }
 
 /*
@@ -2493,11 +2616,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
 			   enum dma_data_direction dir,
 			   struct dma_attrs *attrs)
 {
-	unsigned long flags;
+	phys_addr_t paddr = page_to_phys(page) + offset;
 	struct protection_domain *domain;
-	dma_addr_t addr;
 	u64 dma_mask;
-	phys_addr_t paddr = page_to_phys(page) + offset;
 
 	INC_STATS_COUNTER(cnt_map_single);
 
@@ -2509,19 +2630,8 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
 
 	dma_mask = *dev->dma_mask;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
-	addr = __map_single(dev, domain->priv, paddr, size, dir, false,
+	return __map_single(dev, domain->priv, paddr, size, dir, false,
 			    dma_mask);
-	if (addr == DMA_ERROR_CODE)
-		goto out;
-
-	domain_flush_complete(domain);
-
-out:
-	spin_unlock_irqrestore(&domain->lock, flags);
-
-	return addr;
 }
 
 /*
@@ -2530,7 +2640,6 @@ out:
 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
 		       enum dma_data_direction dir, struct dma_attrs *attrs)
 {
-	unsigned long flags;
 	struct protection_domain *domain;
 
 	INC_STATS_COUNTER(cnt_unmap_single);
@@ -2539,13 +2648,7 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
 	if (IS_ERR(domain))
 		return;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
 	__unmap_single(domain->priv, dma_addr, size, dir);
-
-	domain_flush_complete(domain);
-
-	spin_unlock_irqrestore(&domain->lock, flags);
 }
 
 /*
@@ -2556,7 +2659,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
 		  int nelems, enum dma_data_direction dir,
 		  struct dma_attrs *attrs)
 {
-	unsigned long flags;
 	struct protection_domain *domain;
 	int i;
 	struct scatterlist *s;
@@ -2572,8 +2674,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
 
 	dma_mask = *dev->dma_mask;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
 	for_each_sg(sglist, s, nelems, i) {
 		paddr = sg_phys(s);
 
@@ -2588,12 +2688,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
 			goto unmap;
 	}
 
-	domain_flush_complete(domain);
-
-out:
-	spin_unlock_irqrestore(&domain->lock, flags);
-
 	return mapped_elems;
+
 unmap:
 	for_each_sg(sglist, s, mapped_elems, i) {
 		if (s->dma_address)
@@ -2602,9 +2698,7 @@ unmap:
 		s->dma_address = s->dma_length = 0;
 	}
 
-	mapped_elems = 0;
-
-	goto out;
+	return 0;
 }
 
 /*
@@ -2615,7 +2709,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
 		     int nelems, enum dma_data_direction dir,
 		     struct dma_attrs *attrs)
 {
-	unsigned long flags;
 	struct protection_domain *domain;
 	struct scatterlist *s;
 	int i;
@@ -2626,17 +2719,11 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
 	if (IS_ERR(domain))
 		return;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
 	for_each_sg(sglist, s, nelems, i) {
 		__unmap_single(domain->priv, s->dma_address,
 			       s->dma_length, dir);
 		s->dma_address = s->dma_length = 0;
 	}
-
-	domain_flush_complete(domain);
-
-	spin_unlock_irqrestore(&domain->lock, flags);
 }
 
 /*
@@ -2648,7 +2735,6 @@ static void *alloc_coherent(struct device *dev, size_t size,
 {
 	u64 dma_mask = dev->coherent_dma_mask;
 	struct protection_domain *domain;
-	unsigned long flags;
 	struct page *page;
 
 	INC_STATS_COUNTER(cnt_alloc_coherent);
@@ -2680,19 +2766,11 @@ static void *alloc_coherent(struct device *dev, size_t size,
 	if (!dma_mask)
 		dma_mask = *dev->dma_mask;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
 	*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
 				 size, DMA_BIDIRECTIONAL, true, dma_mask);
 
-	if (*dma_addr == DMA_ERROR_CODE) {
-		spin_unlock_irqrestore(&domain->lock, flags);
+	if (*dma_addr == DMA_ERROR_CODE)
 		goto out_free;
-	}
-
-	domain_flush_complete(domain);
-
-	spin_unlock_irqrestore(&domain->lock, flags);
 
 	return page_address(page);
 
@@ -2712,7 +2790,6 @@ static void free_coherent(struct device *dev, size_t size,
 			  struct dma_attrs *attrs)
 {
 	struct protection_domain *domain;
-	unsigned long flags;
 	struct page *page;
 
 	INC_STATS_COUNTER(cnt_free_coherent);
@@ -2724,14 +2801,8 @@ static void free_coherent(struct device *dev, size_t size,
 	if (IS_ERR(domain))
 		goto free_mem;
 
-	spin_lock_irqsave(&domain->lock, flags);
-
 	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
 
-	domain_flush_complete(domain);
-
-	spin_unlock_irqrestore(&domain->lock, flags);
-
 free_mem:
 	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
 		__free_pages(page, get_order(size));
@@ -2746,14 +2817,43 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
 	return check_device(dev);
 }
 
+static int set_dma_mask(struct device *dev, u64 mask)
+{
+	struct protection_domain *domain;
+	int max_apertures = 1;
+
+	domain = get_domain(dev);
+	if (IS_ERR(domain))
+		return PTR_ERR(domain);
+
+	if (mask == DMA_BIT_MASK(64))
+		max_apertures = 8;
+	else if (mask > DMA_BIT_MASK(32))
+		max_apertures = 4;
+
+	/*
+	 * To prevent lock contention it doesn't make sense to allocate more
+	 * apertures than online cpus
+	 */
+	if (max_apertures > num_online_cpus())
+		max_apertures = num_online_cpus();
+
+	if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures))
+		dev_err(dev, "Can't allocate %d iommu apertures\n",
+			max_apertures);
+
+	return 0;
+}
+
 static struct dma_map_ops amd_iommu_dma_ops = {
-	.alloc = alloc_coherent,
-	.free = free_coherent,
-	.map_page = map_page,
-	.unmap_page = unmap_page,
-	.map_sg = map_sg,
-	.unmap_sg = unmap_sg,
-	.dma_supported = amd_iommu_dma_supported,
+	.alloc		= alloc_coherent,
+	.free		= free_coherent,
+	.map_page	= map_page,
+	.unmap_page	= unmap_page,
+	.map_sg		= map_sg,
+	.unmap_sg	= unmap_sg,
+	.dma_supported	= amd_iommu_dma_supported,
+	.set_dma_mask	= set_dma_mask,
 };
 
 int __init amd_iommu_init_api(void)
@@ -3757,11 +3857,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
 	case X86_IRQ_ALLOC_TYPE_MSI:
 	case X86_IRQ_ALLOC_TYPE_MSIX:
 		devid = get_device_id(&info->msi_dev->dev);
-		if (devid >= 0) {
-			iommu = amd_iommu_rlookup_table[devid];
-			if (iommu)
-				return iommu->msi_domain;
-		}
+		iommu = amd_iommu_rlookup_table[devid];
+		if (iommu)
+			return iommu->msi_domain;
 		break;
 	default:
 		break;
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index b08cf57bf455..9d32b20a5e9a 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -425,46 +425,6 @@ struct protection_domain {
 };
 
 /*
- * For dynamic growth the aperture size is split into ranges of 128MB of
- * DMA address space each. This struct represents one such range.
- */
-struct aperture_range {
-
-	/* address allocation bitmap */
-	unsigned long *bitmap;
-
-	/*
-	 * Array of PTE pages for the aperture. In this array we save all the
-	 * leaf pages of the domain page table used for the aperture. This way
-	 * we don't need to walk the page table to find a specific PTE. We can
-	 * just calculate its address in constant time.
-	 */
-	u64 *pte_pages[64];
-
-	unsigned long offset;
-};
-
-/*
- * Data container for a dma_ops specific protection domain
- */
-struct dma_ops_domain {
-	/* generic protection domain information */
-	struct protection_domain domain;
-
-	/* size of the aperture for the mappings */
-	unsigned long aperture_size;
-
-	/* address we start to search for free addresses */
-	unsigned long next_address;
-
-	/* address space relevant data */
-	struct aperture_range *aperture[APERTURE_MAX_RANGES];
-
-	/* This will be set to true when TLB needs to be flushed */
-	bool need_flush;
-};
-
-/*
  * Structure where we save information about one hardware AMD IOMMU in the
  * system.
  */
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 7caf2fa237f2..c865737326e1 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -432,7 +432,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	unbind_pasid(pasid_state);
 }
 
-static struct mmu_notifier_ops iommu_mn = {
+static const struct mmu_notifier_ops iommu_mn = {
 	.release		= mn_release,
 	.clear_flush_young      = mn_clear_flush_young,
 	.invalidate_page        = mn_invalidate_page,
@@ -513,43 +513,39 @@ static bool access_error(struct vm_area_struct *vma, struct fault *fault)
 static void do_fault(struct work_struct *work)
 {
 	struct fault *fault = container_of(work, struct fault, work);
-	struct mm_struct *mm;
 	struct vm_area_struct *vma;
+	int ret = VM_FAULT_ERROR;
+	unsigned int flags = 0;
+	struct mm_struct *mm;
 	u64 address;
-	int ret, write;
-
-	write = !!(fault->flags & PPR_FAULT_WRITE);
 
 	mm = fault->state->mm;
 	address = fault->address;
 
+	if (fault->flags & PPR_FAULT_USER)
+		flags |= FAULT_FLAG_USER;
+	if (fault->flags & PPR_FAULT_WRITE)
+		flags |= FAULT_FLAG_WRITE;
+
 	down_read(&mm->mmap_sem);
 	vma = find_extend_vma(mm, address);
-	if (!vma || address < vma->vm_start) {
+	if (!vma || address < vma->vm_start)
 		/* failed to get a vma in the right range */
-		up_read(&mm->mmap_sem);
-		handle_fault_error(fault);
 		goto out;
-	}
 
 	/* Check if we have the right permissions on the vma */
-	if (access_error(vma, fault)) {
-		up_read(&mm->mmap_sem);
-		handle_fault_error(fault);
+	if (access_error(vma, fault))
 		goto out;
-	}
 
-	ret = handle_mm_fault(mm, vma, address, write);
-	if (ret & VM_FAULT_ERROR) {
-		/* failed to service fault */
-		up_read(&mm->mmap_sem);
-		handle_fault_error(fault);
-		goto out;
-	}
+	ret = handle_mm_fault(mm, vma, address, flags);
 
+out:
 	up_read(&mm->mmap_sem);
 
-out:
+	if (ret & VM_FAULT_ERROR)
+		/* failed to service fault */
+		handle_fault_error(fault);
+
 	finish_pri_tag(fault->dev_state, fault->state, fault->tag);
 
 	put_pasid_state(fault->state);
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 4e5118a4cd30..20875341c865 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -40,7 +40,10 @@
 #define IDR0_ST_LVL_SHIFT		27
 #define IDR0_ST_LVL_MASK		0x3
 #define IDR0_ST_LVL_2LVL		(1 << IDR0_ST_LVL_SHIFT)
-#define IDR0_STALL_MODEL		(3 << 24)
+#define IDR0_STALL_MODEL_SHIFT		24
+#define IDR0_STALL_MODEL_MASK		0x3
+#define IDR0_STALL_MODEL_STALL		(0 << IDR0_STALL_MODEL_SHIFT)
+#define IDR0_STALL_MODEL_FORCE		(2 << IDR0_STALL_MODEL_SHIFT)
 #define IDR0_TTENDIAN_SHIFT		21
 #define IDR0_TTENDIAN_MASK		0x3
 #define IDR0_TTENDIAN_LE		(2 << IDR0_TTENDIAN_SHIFT)
@@ -253,6 +256,9 @@
 #define STRTAB_STE_1_STRW_EL2		2UL
 #define STRTAB_STE_1_STRW_SHIFT		30
 
+#define STRTAB_STE_1_SHCFG_INCOMING	1UL
+#define STRTAB_STE_1_SHCFG_SHIFT	44
+
 #define STRTAB_STE_2_S2VMID_SHIFT	0
 #define STRTAB_STE_2_S2VMID_MASK	0xffffUL
 #define STRTAB_STE_2_VTCR_SHIFT		32
@@ -378,7 +384,6 @@
 #define PRIQ_0_SID_MASK			0xffffffffUL
 #define PRIQ_0_SSID_SHIFT		32
 #define PRIQ_0_SSID_MASK		0xfffffUL
-#define PRIQ_0_OF			(1UL << 57)
 #define PRIQ_0_PERM_PRIV		(1UL << 58)
 #define PRIQ_0_PERM_EXEC		(1UL << 59)
 #define PRIQ_0_PERM_READ		(1UL << 60)
@@ -855,15 +860,17 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
 	};
 
 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
-		cerror_str[idx]);
+		idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
 
 	switch (idx) {
-	case CMDQ_ERR_CERROR_ILL_IDX:
-		break;
 	case CMDQ_ERR_CERROR_ABT_IDX:
 		dev_err(smmu->dev, "retrying command fetch\n");
 	case CMDQ_ERR_CERROR_NONE_IDX:
 		return;
+	case CMDQ_ERR_CERROR_ILL_IDX:
+		/* Fallthrough */
+	default:
+		break;
 	}
 
 	/*
@@ -1042,6 +1049,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
 		val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
 				      : STRTAB_STE_0_CFG_BYPASS;
 		dst[0] = cpu_to_le64(val);
+		dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
+			 << STRTAB_STE_1_SHCFG_SHIFT);
 		dst[2] = 0; /* Nuke the VMID */
 		if (ste_live)
 			arm_smmu_sync_ste_for_sid(smmu, sid);
@@ -1056,12 +1065,14 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
 			 STRTAB_STE_1_S1C_CACHE_WBRA
 			 << STRTAB_STE_1_S1COR_SHIFT |
 			 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
-			 STRTAB_STE_1_S1STALLD |
 #ifdef CONFIG_PCI_ATS
 			 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
 #endif
 			 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
 
+		if (smmu->features & ARM_SMMU_FEAT_STALLS)
+			dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
+
 		val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
 		        << STRTAB_STE_0_S1CTXPTR_SHIFT) |
 			STRTAB_STE_0_CFG_S1_TRANS;
@@ -1123,8 +1134,8 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
 	strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
 
 	desc->span = STRTAB_SPLIT + 1;
-	desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
-					  GFP_KERNEL);
+	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
+					  GFP_KERNEL | __GFP_ZERO);
 	if (!desc->l2ptr) {
 		dev_err(smmu->dev,
 			"failed to allocate l2 stream table for SID %u\n",
@@ -1250,50 +1261,50 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
 
 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
 {
-	u32 gerror, gerrorn;
+	u32 gerror, gerrorn, active;
 	struct arm_smmu_device *smmu = dev;
 
 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
 
-	gerror ^= gerrorn;
-	if (!(gerror & GERROR_ERR_MASK))
+	active = gerror ^ gerrorn;
+	if (!(active & GERROR_ERR_MASK))
 		return IRQ_NONE; /* No errors pending */
 
 	dev_warn(smmu->dev,
 		 "unexpected global error reported (0x%08x), this could be serious\n",
-		 gerror);
+		 active);
 
-	if (gerror & GERROR_SFM_ERR) {
+	if (active & GERROR_SFM_ERR) {
 		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
 		arm_smmu_device_disable(smmu);
 	}
 
-	if (gerror & GERROR_MSI_GERROR_ABT_ERR)
+	if (active & GERROR_MSI_GERROR_ABT_ERR)
 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
 
-	if (gerror & GERROR_MSI_PRIQ_ABT_ERR) {
+	if (active & GERROR_MSI_PRIQ_ABT_ERR) {
 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
 		arm_smmu_priq_handler(irq, smmu->dev);
 	}
 
-	if (gerror & GERROR_MSI_EVTQ_ABT_ERR) {
+	if (active & GERROR_MSI_EVTQ_ABT_ERR) {
 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
 		arm_smmu_evtq_handler(irq, smmu->dev);
 	}
 
-	if (gerror & GERROR_MSI_CMDQ_ABT_ERR) {
+	if (active & GERROR_MSI_CMDQ_ABT_ERR) {
 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
 		arm_smmu_cmdq_sync_handler(irq, smmu->dev);
 	}
 
-	if (gerror & GERROR_PRIQ_ABT_ERR)
+	if (active & GERROR_PRIQ_ABT_ERR)
 		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
 
-	if (gerror & GERROR_EVTQ_ABT_ERR)
+	if (active & GERROR_EVTQ_ABT_ERR)
 		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
 
-	if (gerror & GERROR_CMDQ_ERR)
+	if (active & GERROR_CMDQ_ERR)
 		arm_smmu_cmdq_skip_err(smmu);
 
 	writel(gerror, smmu->base + ARM_SMMU_GERRORN);
@@ -1335,7 +1346,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
 }
 
 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
-					  bool leaf, void *cookie)
+					  size_t granule, bool leaf, void *cookie)
 {
 	struct arm_smmu_domain *smmu_domain = cookie;
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
@@ -1354,7 +1365,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
 	}
 
-	arm_smmu_cmdq_issue_cmd(smmu, &cmd);
+	do {
+		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
+		cmd.tlbi.addr += granule;
+	} while (size -= granule);
 }
 
 static struct iommu_gather_ops arm_smmu_gather_ops = {
@@ -1429,10 +1443,10 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
 
 		if (cfg->cdptr) {
-			dma_free_coherent(smmu_domain->smmu->dev,
-					  CTXDESC_CD_DWORDS << 3,
-					  cfg->cdptr,
-					  cfg->cdptr_dma);
+			dmam_free_coherent(smmu_domain->smmu->dev,
+					   CTXDESC_CD_DWORDS << 3,
+					   cfg->cdptr,
+					   cfg->cdptr_dma);
 
 			arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
 		}
@@ -1457,8 +1471,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	if (IS_ERR_VALUE(asid))
 		return asid;
 
-	cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
-					 &cfg->cdptr_dma, GFP_KERNEL);
+	cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
+					 &cfg->cdptr_dma,
+					 GFP_KERNEL | __GFP_ZERO);
 	if (!cfg->cdptr) {
 		dev_warn(smmu->dev, "failed to allocate context descriptor\n");
 		ret = -ENOMEM;
@@ -1804,13 +1819,13 @@ static int arm_smmu_add_device(struct device *dev)
 		smmu = arm_smmu_get_for_pci_dev(pdev);
 		if (!smmu) {
 			ret = -ENOENT;
-			goto out_put_group;
+			goto out_remove_dev;
 		}
 
 		smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
 		if (!smmu_group) {
 			ret = -ENOMEM;
-			goto out_put_group;
+			goto out_remove_dev;
 		}
 
 		smmu_group->ste.valid	= true;
@@ -1826,20 +1841,20 @@ static int arm_smmu_add_device(struct device *dev)
 	for (i = 0; i < smmu_group->num_sids; ++i) {
 		/* If we already know about this SID, then we're done */
 		if (smmu_group->sids[i] == sid)
-			return 0;
+			goto out_put_group;
 	}
 
 	/* Check the SID is in range of the SMMU and our stream table */
 	if (!arm_smmu_sid_in_range(smmu, sid)) {
 		ret = -ERANGE;
-		goto out_put_group;
+		goto out_remove_dev;
 	}
 
 	/* Ensure l2 strtab is initialised */
 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
 		ret = arm_smmu_init_l2_strtab(smmu, sid);
 		if (ret)
-			goto out_put_group;
+			goto out_remove_dev;
 	}
 
 	/* Resize the SID array for the group */
@@ -1849,16 +1864,20 @@ static int arm_smmu_add_device(struct device *dev)
 	if (!sids) {
 		smmu_group->num_sids--;
 		ret = -ENOMEM;
-		goto out_put_group;
+		goto out_remove_dev;
 	}
 
 	/* Add the new SID */
 	sids[smmu_group->num_sids - 1] = sid;
 	smmu_group->sids = sids;
-	return 0;
 
 out_put_group:
 	iommu_group_put(group);
+	return 0;
+
+out_remove_dev:
+	iommu_group_remove_device(dev);
+	iommu_group_put(group);
 	return ret;
 }
 
@@ -1937,7 +1956,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
 {
 	size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
 
-	q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
+	q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
 	if (!q->base) {
 		dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
 			qsz);
@@ -1957,23 +1976,6 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
 	return 0;
 }
 
-static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu,
-				    struct arm_smmu_queue *q)
-{
-	size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;
-
-	dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);
-}
-
-static void arm_smmu_free_queues(struct arm_smmu_device *smmu)
-{
-	arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
-	arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
-
-	if (smmu->features & ARM_SMMU_FEAT_PRI)
-		arm_smmu_free_one_queue(smmu, &smmu->priq.q);
-}
-
 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
 {
 	int ret;
@@ -1983,49 +1985,20 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
 				      ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
 	if (ret)
-		goto out;
+		return ret;
 
 	/* evtq */
 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
 				      ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
 	if (ret)
-		goto out_free_cmdq;
+		return ret;
 
 	/* priq */
 	if (!(smmu->features & ARM_SMMU_FEAT_PRI))
 		return 0;
 
-	ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
-				      ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
-	if (ret)
-		goto out_free_evtq;
-
-	return 0;
-
-out_free_evtq:
-	arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
-out_free_cmdq:
-	arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
-out:
-	return ret;
-}
-
-static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu)
-{
-	int i;
-	size_t size;
-	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
-
-	size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
-	for (i = 0; i < cfg->num_l1_ents; ++i) {
-		struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i];
-
-		if (!desc->l2ptr)
-			continue;
-
-		dma_free_coherent(smmu->dev, size, desc->l2ptr,
-				  desc->l2ptr_dma);
-	}
+	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
+				       ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
 }
 
 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
@@ -2054,7 +2027,6 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
 	void *strtab;
 	u64 reg;
 	u32 size, l1size;
-	int ret;
 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 
 	/*
@@ -2077,8 +2049,8 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
 			 size, smmu->sid_bits);
 
 	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
-	strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
-				     GFP_KERNEL);
+	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
+				     GFP_KERNEL | __GFP_ZERO);
 	if (!strtab) {
 		dev_err(smmu->dev,
 			"failed to allocate l1 stream table (%u bytes)\n",
@@ -2095,13 +2067,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
 		<< STRTAB_BASE_CFG_SPLIT_SHIFT;
 	cfg->strtab_base_cfg = reg;
 
-	ret = arm_smmu_init_l1_strtab(smmu);
-	if (ret)
-		dma_free_coherent(smmu->dev,
-				  l1size,
-				  strtab,
-				  cfg->strtab_dma);
-	return ret;
+	return arm_smmu_init_l1_strtab(smmu);
 }
 
 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
@@ -2112,8 +2078,8 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 
 	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
-	strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma,
-				     GFP_KERNEL);
+	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
+				     GFP_KERNEL | __GFP_ZERO);
 	if (!strtab) {
 		dev_err(smmu->dev,
 			"failed to allocate linear stream table (%u bytes)\n",
@@ -2157,21 +2123,6 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
 	return 0;
 }
 
-static void arm_smmu_free_strtab(struct arm_smmu_device *smmu)
-{
-	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
-	u32 size = cfg->num_l1_ents;
-
-	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
-		arm_smmu_free_l2_strtab(smmu);
-		size *= STRTAB_L1_DESC_DWORDS << 3;
-	} else {
-		size *= STRTAB_STE_DWORDS * 3;
-	}
-
-	dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma);
-}
-
 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
 {
 	int ret;
@@ -2180,21 +2131,7 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
 	if (ret)
 		return ret;
 
-	ret = arm_smmu_init_strtab(smmu);
-	if (ret)
-		goto out_free_queues;
-
-	return 0;
-
-out_free_queues:
-	arm_smmu_free_queues(smmu);
-	return ret;
-}
-
-static void arm_smmu_free_structures(struct arm_smmu_device *smmu)
-{
-	arm_smmu_free_strtab(smmu);
-	arm_smmu_free_queues(smmu);
+	return arm_smmu_init_strtab(smmu);
 }
 
 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
@@ -2532,8 +2469,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
 		dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
 			 coherent ? "true" : "false");
 
-	if (reg & IDR0_STALL_MODEL)
+	switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
+	case IDR0_STALL_MODEL_STALL:
+		/* Fallthrough */
+	case IDR0_STALL_MODEL_FORCE:
 		smmu->features |= ARM_SMMU_FEAT_STALLS;
+	}
 
 	if (reg & IDR0_S1P)
 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
@@ -2699,15 +2640,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, smmu);
 
 	/* Reset the device */
-	ret = arm_smmu_device_reset(smmu);
-	if (ret)
-		goto out_free_structures;
-
-	return 0;
-
-out_free_structures:
-	arm_smmu_free_structures(smmu);
-	return ret;
+	return arm_smmu_device_reset(smmu);
 }
 
 static int arm_smmu_device_remove(struct platform_device *pdev)
@@ -2715,7 +2648,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
 
 	arm_smmu_device_disable(smmu);
-	arm_smmu_free_structures(smmu);
 	return 0;
 }
 
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 47dc7a793f5c..59ee4b8a3236 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -582,7 +582,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
 }
 
 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
-					  bool leaf, void *cookie)
+					  size_t granule, bool leaf, void *cookie)
 {
 	struct arm_smmu_domain *smmu_domain = cookie;
 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
@@ -597,12 +597,18 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 		if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {
 			iova &= ~12UL;
 			iova |= ARM_SMMU_CB_ASID(cfg);
-			writel_relaxed(iova, reg);
+			do {
+				writel_relaxed(iova, reg);
+				iova += granule;
+			} while (size -= granule);
 #ifdef CONFIG_64BIT
 		} else {
 			iova >>= 12;
 			iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48;
-			writeq_relaxed(iova, reg);
+			do {
+				writeq_relaxed(iova, reg);
+				iova += granule >> 12;
+			} while (size -= granule);
 #endif
 		}
 #ifdef CONFIG_64BIT
@@ -610,7 +616,11 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 		reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 		reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 			      ARM_SMMU_CB_S2_TLBIIPAS2;
-		writeq_relaxed(iova >> 12, reg);
+		iova >>= 12;
+		do {
+			writeq_relaxed(iova, reg);
+			iova += granule >> 12;
+		} while (size -= granule);
 #endif
 	} else {
 		reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
@@ -945,9 +955,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 		free_irq(irq, domain);
 	}
 
-	if (smmu_domain->pgtbl_ops)
-		free_io_pgtable_ops(smmu_domain->pgtbl_ops);
-
+	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 }
 
@@ -1357,6 +1365,7 @@ static int arm_smmu_add_device(struct device *dev)
 	if (IS_ERR(group))
 		return PTR_ERR(group);
 
+	iommu_group_put(group);
 	return 0;
 }
 
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 80e3c176008e..62a400c5ba06 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1063,13 +1063,19 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
 
 	raw_spin_lock_init(&iommu->register_lock);
 
-	drhd->iommu = iommu;
-
-	if (intel_iommu_enabled)
+	if (intel_iommu_enabled) {
 		iommu->iommu_dev = iommu_device_create(NULL, iommu,
 						       intel_iommu_groups,
 						       "%s", iommu->name);
 
+		if (IS_ERR(iommu->iommu_dev)) {
+			err = PTR_ERR(iommu->iommu_dev);
+			goto err_unmap;
+		}
+	}
+
+	drhd->iommu = iommu;
+
 	return 0;
 
 err_unmap:
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index ac7387686ddc..986a53e3eb96 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -1489,7 +1489,7 @@ static void iommu_disable_dev_iotlb(struct device_domain_info *info)
 {
 	struct pci_dev *pdev;
 
-	if (dev_is_pci(info->dev))
+	if (!dev_is_pci(info->dev))
 		return;
 
 	pdev = to_pci_dev(info->dev);
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 7df97777662d..381ca5a37a7b 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -25,6 +25,7 @@
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/barrier.h>
 
@@ -38,9 +39,6 @@
 #define io_pgtable_to_data(x)						\
 	container_of((x), struct arm_lpae_io_pgtable, iop)
 
-#define io_pgtable_ops_to_pgtable(x)					\
-	container_of((x), struct io_pgtable, ops)
-
 #define io_pgtable_ops_to_data(x)					\
 	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
 
@@ -58,8 +56,10 @@
 	((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1))		\
 	  * (d)->bits_per_level) + (d)->pg_shift)
 
+#define ARM_LPAE_GRANULE(d)		(1UL << (d)->pg_shift)
+
 #define ARM_LPAE_PAGES_PER_PGD(d)					\
-	DIV_ROUND_UP((d)->pgd_size, 1UL << (d)->pg_shift)
+	DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d))
 
 /*
  * Calculate the index at level l used to map virtual address a using the
@@ -169,7 +169,7 @@
 /* IOPTE accessors */
 #define iopte_deref(pte,d)					\
 	(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)	\
-	& ~((1ULL << (d)->pg_shift) - 1)))
+	& ~(ARM_LPAE_GRANULE(d) - 1ULL)))
 
 #define iopte_type(pte,l)					\
 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
@@ -326,7 +326,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
 	/* Grab a pointer to the next level */
 	pte = *ptep;
 	if (!pte) {
-		cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
+		cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data),
 					       GFP_ATOMIC, cfg);
 		if (!cptep)
 			return -ENOMEM;
@@ -405,17 +405,18 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
 	arm_lpae_iopte *start, *end;
 	unsigned long table_size;
 
-	/* Only leaf entries at the last level */
-	if (lvl == ARM_LPAE_MAX_LEVELS - 1)
-		return;
-
 	if (lvl == ARM_LPAE_START_LVL(data))
 		table_size = data->pgd_size;
 	else
-		table_size = 1UL << data->pg_shift;
+		table_size = ARM_LPAE_GRANULE(data);
 
 	start = ptep;
-	end = (void *)ptep + table_size;
+
+	/* Only leaf entries at the last level */
+	if (lvl == ARM_LPAE_MAX_LEVELS - 1)
+		end = ptep;
+	else
+		end = (void *)ptep + table_size;
 
 	while (ptep != end) {
 		arm_lpae_iopte pte = *ptep++;
@@ -473,7 +474,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
 
 	__arm_lpae_set_pte(ptep, table, cfg);
 	iova &= ~(blk_size - 1);
-	cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
+	cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie);
 	return size;
 }
 
@@ -486,11 +487,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 	void *cookie = data->iop.cookie;
 	size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
 
+	/* Something went horribly wrong and we ran out of page table */
+	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
+		return 0;
+
 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
 	pte = *ptep;
-
-	/* Something went horribly wrong and we ran out of page table */
-	if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS)))
+	if (WARN_ON(!pte))
 		return 0;
 
 	/* If the size matches this level, we're in the right place */
@@ -499,12 +502,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 
 		if (!iopte_leaf(pte, lvl)) {
 			/* Also flush any partial walks */
-			tlb->tlb_add_flush(iova, size, false, cookie);
+			tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data),
+					   false, cookie);
 			tlb->tlb_sync(cookie);
 			ptep = iopte_deref(pte, data);
 			__arm_lpae_free_pgtable(data, lvl + 1, ptep);
 		} else {
-			tlb->tlb_add_flush(iova, size, true, cookie);
+			tlb->tlb_add_flush(iova, size, size, true, cookie);
 		}
 
 		return size;
@@ -570,7 +574,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
 	return 0;
 
 found_translation:
-	iova &= ((1 << data->pg_shift) - 1);
+	iova &= (ARM_LPAE_GRANULE(data) - 1);
 	return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
 }
 
@@ -668,7 +672,7 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 	      (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
 	      (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
 
-	switch (1 << data->pg_shift) {
+	switch (ARM_LPAE_GRANULE(data)) {
 	case SZ_4K:
 		reg |= ARM_LPAE_TCR_TG0_4K;
 		break;
@@ -769,7 +773,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
 
 	sl = ARM_LPAE_START_LVL(data);
 
-	switch (1 << data->pg_shift) {
+	switch (ARM_LPAE_GRANULE(data)) {
 	case SZ_4K:
 		reg |= ARM_LPAE_TCR_TG0_4K;
 		sl++; /* SL0 format is different for 4K granule size */
@@ -889,8 +893,8 @@ static void dummy_tlb_flush_all(void *cookie)
 	WARN_ON(cookie != cfg_cookie);
 }
 
-static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
-				void *cookie)
+static void dummy_tlb_add_flush(unsigned long iova, size_t size,
+				size_t granule, bool leaf, void *cookie)
 {
 	WARN_ON(cookie != cfg_cookie);
 	WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index ac9e2341a633..36673c83de58 100644
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -26,8 +26,8 @@ enum io_pgtable_fmt {
  */
 struct iommu_gather_ops {
 	void (*tlb_flush_all)(void *cookie);
-	void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf,
-			      void *cookie);
+	void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,
+			      bool leaf, void *cookie);
 	void (*tlb_sync)(void *cookie);
 };
 
@@ -131,6 +131,8 @@ struct io_pgtable {
 	struct io_pgtable_ops	ops;
 };
 
+#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
+
 /**
  * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
  *                              particular format.
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index dfb868e2d129..2fdbac67a77f 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -277,8 +277,8 @@ static void ipmmu_tlb_flush_all(void *cookie)
 	ipmmu_tlb_invalidate(domain);
 }
 
-static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
-				void *cookie)
+static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
+				size_t granule, bool leaf, void *cookie)
 {
 	/* The hardware doesn't support selective TLB flush. */
 }
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
index b6d01f97e537..4b09e815accf 100644
--- a/drivers/iommu/msm_iommu_dev.c
+++ b/drivers/iommu/msm_iommu_dev.c
@@ -359,30 +359,19 @@ static struct platform_driver msm_iommu_ctx_driver = {
 	.remove		= msm_iommu_ctx_remove,
 };
 
+static struct platform_driver * const drivers[] = {
+	&msm_iommu_driver,
+	&msm_iommu_ctx_driver,
+};
+
 static int __init msm_iommu_driver_init(void)
 {
-	int ret;
-	ret = platform_driver_register(&msm_iommu_driver);
-	if (ret != 0) {
-		pr_err("Failed to register IOMMU driver\n");
-		goto error;
-	}
-
-	ret = platform_driver_register(&msm_iommu_ctx_driver);
-	if (ret != 0) {
-		platform_driver_unregister(&msm_iommu_driver);
-		pr_err("Failed to register IOMMU context driver\n");
-		goto error;
-	}
-
-error:
-	return ret;
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
 static void __exit msm_iommu_driver_exit(void)
 {
-	platform_driver_unregister(&msm_iommu_ctx_driver);
-	platform_driver_unregister(&msm_iommu_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
 subsys_initcall(msm_iommu_driver_init);
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 471ee36b9c6e..a04d491cf431 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -49,7 +49,7 @@ static bool s390_iommu_capable(enum iommu_cap cap)
 	}
 }
 
-struct iommu_domain *s390_domain_alloc(unsigned domain_type)
+static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
 {
 	struct s390_domain *s390_domain;
 
@@ -73,7 +73,7 @@ struct iommu_domain *s390_domain_alloc(unsigned domain_type)
 	return &s390_domain->domain;
 }
 
-void s390_domain_free(struct iommu_domain *domain)
+static void s390_domain_free(struct iommu_domain *domain)
 {
 	struct s390_domain *s390_domain = to_s390_domain(domain);
 
diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c
deleted file mode 100644
index a0287519a1d4..000000000000
--- a/drivers/iommu/shmobile-iommu.c
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * IOMMU for IPMMU/IPMMUI
- * Copyright (C) 2012  Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/io.h>
-#include <linux/iommu.h>
-#include <linux/platform_device.h>
-#include <linux/sizes.h>
-#include <linux/slab.h>
-#include <asm/dma-iommu.h>
-#include "shmobile-ipmmu.h"
-
-#define L1_SIZE CONFIG_SHMOBILE_IOMMU_L1SIZE
-#define L1_LEN (L1_SIZE / 4)
-#define L1_ALIGN L1_SIZE
-#define L2_SIZE SZ_1K
-#define L2_LEN (L2_SIZE / 4)
-#define L2_ALIGN L2_SIZE
-
-struct shmobile_iommu_domain_pgtable {
-	uint32_t *pgtable;
-	dma_addr_t handle;
-};
-
-struct shmobile_iommu_archdata {
-	struct list_head attached_list;
-	struct dma_iommu_mapping *iommu_mapping;
-	spinlock_t attach_lock;
-	struct shmobile_iommu_domain *attached;
-	int num_attached_devices;
-	struct shmobile_ipmmu *ipmmu;
-};
-
-struct shmobile_iommu_domain {
-	struct shmobile_iommu_domain_pgtable l1, l2[L1_LEN];
-	spinlock_t map_lock;
-	spinlock_t attached_list_lock;
-	struct list_head attached_list;
-	struct iommu_domain domain;
-};
-
-static struct shmobile_iommu_archdata *ipmmu_archdata;
-static struct kmem_cache *l1cache, *l2cache;
-
-static struct shmobile_iommu_domain *to_sh_domain(struct iommu_domain *dom)
-{
-	return container_of(dom, struct shmobile_iommu_domain, domain);
-}
-
-static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable,
-			 struct kmem_cache *cache, size_t size)
-{
-	pgtable->pgtable = kmem_cache_zalloc(cache, GFP_ATOMIC);
-	if (!pgtable->pgtable)
-		return -ENOMEM;
-	pgtable->handle = dma_map_single(NULL, pgtable->pgtable, size,
-					 DMA_TO_DEVICE);
-	return 0;
-}
-
-static void pgtable_free(struct shmobile_iommu_domain_pgtable *pgtable,
-			 struct kmem_cache *cache, size_t size)
-{
-	dma_unmap_single(NULL, pgtable->handle, size, DMA_TO_DEVICE);
-	kmem_cache_free(cache, pgtable->pgtable);
-}
-
-static uint32_t pgtable_read(struct shmobile_iommu_domain_pgtable *pgtable,
-			     unsigned int index)
-{
-	return pgtable->pgtable[index];
-}
-
-static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable,
-			  unsigned int index, unsigned int count, uint32_t val)
-{
-	unsigned int i;
-
-	for (i = 0; i < count; i++)
-		pgtable->pgtable[index + i] = val;
-	dma_sync_single_for_device(NULL, pgtable->handle + index * sizeof(val),
-				   sizeof(val) * count, DMA_TO_DEVICE);
-}
-
-static struct iommu_domain *shmobile_iommu_domain_alloc(unsigned type)
-{
-	struct shmobile_iommu_domain *sh_domain;
-	int i, ret;
-
-	if (type != IOMMU_DOMAIN_UNMANAGED)
-		return NULL;
-
-	sh_domain = kzalloc(sizeof(*sh_domain), GFP_KERNEL);
-	if (!sh_domain)
-		return NULL;
-	ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE);
-	if (ret < 0) {
-		kfree(sh_domain);
-		return NULL;
-	}
-	for (i = 0; i < L1_LEN; i++)
-		sh_domain->l2[i].pgtable = NULL;
-	spin_lock_init(&sh_domain->map_lock);
-	spin_lock_init(&sh_domain->attached_list_lock);
-	INIT_LIST_HEAD(&sh_domain->attached_list);
-	return &sh_domain->domain;
-}
-
-static void shmobile_iommu_domain_free(struct iommu_domain *domain)
-{
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-	int i;
-
-	for (i = 0; i < L1_LEN; i++) {
-		if (sh_domain->l2[i].pgtable)
-			pgtable_free(&sh_domain->l2[i], l2cache, L2_SIZE);
-	}
-	pgtable_free(&sh_domain->l1, l1cache, L1_SIZE);
-	kfree(sh_domain);
-}
-
-static int shmobile_iommu_attach_device(struct iommu_domain *domain,
-					struct device *dev)
-{
-	struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-	int ret = -EBUSY;
-
-	if (!archdata)
-		return -ENODEV;
-	spin_lock(&sh_domain->attached_list_lock);
-	spin_lock(&archdata->attach_lock);
-	if (archdata->attached != sh_domain) {
-		if (archdata->attached)
-			goto err;
-		ipmmu_tlb_set(archdata->ipmmu, sh_domain->l1.handle, L1_SIZE,
-			      0);
-		ipmmu_tlb_flush(archdata->ipmmu);
-		archdata->attached = sh_domain;
-		archdata->num_attached_devices = 0;
-		list_add(&archdata->attached_list, &sh_domain->attached_list);
-	}
-	archdata->num_attached_devices++;
-	ret = 0;
-err:
-	spin_unlock(&archdata->attach_lock);
-	spin_unlock(&sh_domain->attached_list_lock);
-	return ret;
-}
-
-static void shmobile_iommu_detach_device(struct iommu_domain *domain,
-					 struct device *dev)
-{
-	struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-
-	if (!archdata)
-		return;
-	spin_lock(&sh_domain->attached_list_lock);
-	spin_lock(&archdata->attach_lock);
-	archdata->num_attached_devices--;
-	if (!archdata->num_attached_devices) {
-		ipmmu_tlb_set(archdata->ipmmu, 0, 0, 0);
-		ipmmu_tlb_flush(archdata->ipmmu);
-		archdata->attached = NULL;
-		list_del(&archdata->attached_list);
-	}
-	spin_unlock(&archdata->attach_lock);
-	spin_unlock(&sh_domain->attached_list_lock);
-}
-
-static void domain_tlb_flush(struct shmobile_iommu_domain *sh_domain)
-{
-	struct shmobile_iommu_archdata *archdata;
-
-	spin_lock(&sh_domain->attached_list_lock);
-	list_for_each_entry(archdata, &sh_domain->attached_list, attached_list)
-		ipmmu_tlb_flush(archdata->ipmmu);
-	spin_unlock(&sh_domain->attached_list_lock);
-}
-
-static int l2alloc(struct shmobile_iommu_domain *sh_domain,
-		   unsigned int l1index)
-{
-	int ret;
-
-	if (!sh_domain->l2[l1index].pgtable) {
-		ret = pgtable_alloc(&sh_domain->l2[l1index], l2cache, L2_SIZE);
-		if (ret < 0)
-			return ret;
-	}
-	pgtable_write(&sh_domain->l1, l1index, 1,
-		      sh_domain->l2[l1index].handle | 0x1);
-	return 0;
-}
-
-static void l2realfree(struct shmobile_iommu_domain_pgtable *l2)
-{
-	if (l2->pgtable)
-		pgtable_free(l2, l2cache, L2_SIZE);
-}
-
-static void l2free(struct shmobile_iommu_domain *sh_domain,
-		   unsigned int l1index,
-		   struct shmobile_iommu_domain_pgtable *l2)
-{
-	pgtable_write(&sh_domain->l1, l1index, 1, 0);
-	if (sh_domain->l2[l1index].pgtable) {
-		*l2 = sh_domain->l2[l1index];
-		sh_domain->l2[l1index].pgtable = NULL;
-	}
-}
-
-static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova,
-			      phys_addr_t paddr, size_t size, int prot)
-{
-	struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-	unsigned int l1index, l2index;
-	int ret;
-
-	l1index = iova >> 20;
-	switch (size) {
-	case SZ_4K:
-		l2index = (iova >> 12) & 0xff;
-		spin_lock(&sh_domain->map_lock);
-		ret = l2alloc(sh_domain, l1index);
-		if (!ret)
-			pgtable_write(&sh_domain->l2[l1index], l2index, 1,
-				      paddr | 0xff2);
-		spin_unlock(&sh_domain->map_lock);
-		break;
-	case SZ_64K:
-		l2index = (iova >> 12) & 0xf0;
-		spin_lock(&sh_domain->map_lock);
-		ret = l2alloc(sh_domain, l1index);
-		if (!ret)
-			pgtable_write(&sh_domain->l2[l1index], l2index, 0x10,
-				      paddr | 0xff1);
-		spin_unlock(&sh_domain->map_lock);
-		break;
-	case SZ_1M:
-		spin_lock(&sh_domain->map_lock);
-		l2free(sh_domain, l1index, &l2);
-		pgtable_write(&sh_domain->l1, l1index, 1, paddr | 0xc02);
-		spin_unlock(&sh_domain->map_lock);
-		ret = 0;
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	if (!ret)
-		domain_tlb_flush(sh_domain);
-	l2realfree(&l2);
-	return ret;
-}
-
-static size_t shmobile_iommu_unmap(struct iommu_domain *domain,
-				   unsigned long iova, size_t size)
-{
-	struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-	unsigned int l1index, l2index;
-	uint32_t l2entry = 0;
-	size_t ret = 0;
-
-	l1index = iova >> 20;
-	if (!(iova & 0xfffff) && size >= SZ_1M) {
-		spin_lock(&sh_domain->map_lock);
-		l2free(sh_domain, l1index, &l2);
-		spin_unlock(&sh_domain->map_lock);
-		ret = SZ_1M;
-		goto done;
-	}
-	l2index = (iova >> 12) & 0xff;
-	spin_lock(&sh_domain->map_lock);
-	if (sh_domain->l2[l1index].pgtable)
-		l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
-	switch (l2entry & 3) {
-	case 1:
-		if (l2index & 0xf)
-			break;
-		pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, 0);
-		ret = SZ_64K;
-		break;
-	case 2:
-		pgtable_write(&sh_domain->l2[l1index], l2index, 1, 0);
-		ret = SZ_4K;
-		break;
-	}
-	spin_unlock(&sh_domain->map_lock);
-done:
-	if (ret)
-		domain_tlb_flush(sh_domain);
-	l2realfree(&l2);
-	return ret;
-}
-
-static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain,
-					       dma_addr_t iova)
-{
-	struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-	uint32_t l1entry = 0, l2entry = 0;
-	unsigned int l1index, l2index;
-
-	l1index = iova >> 20;
-	l2index = (iova >> 12) & 0xff;
-	spin_lock(&sh_domain->map_lock);
-	if (sh_domain->l2[l1index].pgtable)
-		l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
-	else
-		l1entry = pgtable_read(&sh_domain->l1, l1index);
-	spin_unlock(&sh_domain->map_lock);
-	switch (l2entry & 3) {
-	case 1:
-		return (l2entry & ~0xffff) | (iova & 0xffff);
-	case 2:
-		return (l2entry & ~0xfff) | (iova & 0xfff);
-	default:
-		if ((l1entry & 3) == 2)
-			return (l1entry & ~0xfffff) | (iova & 0xfffff);
-		return 0;
-	}
-}
-
-static int find_dev_name(struct shmobile_ipmmu *ipmmu, const char *dev_name)
-{
-	unsigned int i, n = ipmmu->num_dev_names;
-
-	for (i = 0; i < n; i++) {
-		if (strcmp(ipmmu->dev_names[i], dev_name) == 0)
-			return 1;
-	}
-	return 0;
-}
-
-static int shmobile_iommu_add_device(struct device *dev)
-{
-	struct shmobile_iommu_archdata *archdata = ipmmu_archdata;
-	struct dma_iommu_mapping *mapping;
-
-	if (!find_dev_name(archdata->ipmmu, dev_name(dev)))
-		return 0;
-	mapping = archdata->iommu_mapping;
-	if (!mapping) {
-		mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
-						   L1_LEN << 20);
-		if (IS_ERR(mapping))
-			return PTR_ERR(mapping);
-		archdata->iommu_mapping = mapping;
-	}
-	dev->archdata.iommu = archdata;
-	if (arm_iommu_attach_device(dev, mapping))
-		pr_err("arm_iommu_attach_device failed\n");
-	return 0;
-}
-
-static const struct iommu_ops shmobile_iommu_ops = {
-	.domain_alloc = shmobile_iommu_domain_alloc,
-	.domain_free = shmobile_iommu_domain_free,
-	.attach_dev = shmobile_iommu_attach_device,
-	.detach_dev = shmobile_iommu_detach_device,
-	.map = shmobile_iommu_map,
-	.unmap = shmobile_iommu_unmap,
-	.map_sg = default_iommu_map_sg,
-	.iova_to_phys = shmobile_iommu_iova_to_phys,
-	.add_device = shmobile_iommu_add_device,
-	.pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K,
-};
-
-int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
-{
-	static struct shmobile_iommu_archdata *archdata;
-
-	l1cache = kmem_cache_create("shmobile-iommu-pgtable1", L1_SIZE,
-				    L1_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
-	if (!l1cache)
-		return -ENOMEM;
-	l2cache = kmem_cache_create("shmobile-iommu-pgtable2", L2_SIZE,
-				    L2_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
-	if (!l2cache) {
-		kmem_cache_destroy(l1cache);
-		return -ENOMEM;
-	}
-	archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
-	if (!archdata) {
-		kmem_cache_destroy(l1cache);
-		kmem_cache_destroy(l2cache);
-		return -ENOMEM;
-	}
-	spin_lock_init(&archdata->attach_lock);
-	archdata->ipmmu = ipmmu;
-	ipmmu_archdata = archdata;
-	bus_set_iommu(&platform_bus_type, &shmobile_iommu_ops);
-	return 0;
-}
diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c
deleted file mode 100644
index 951651a9746b..000000000000
--- a/drivers/iommu/shmobile-ipmmu.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * IPMMU/IPMMUI
- * Copyright (C) 2012  Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#include <linux/err.h>
-#include <linux/export.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/platform_data/sh_ipmmu.h>
-#include "shmobile-ipmmu.h"
-
-#define IMCTR1 0x000
-#define IMCTR2 0x004
-#define IMASID 0x010
-#define IMTTBR 0x014
-#define IMTTBCR 0x018
-
-#define IMCTR1_TLBEN (1 << 0)
-#define IMCTR1_FLUSH (1 << 1)
-
-static void ipmmu_reg_write(struct shmobile_ipmmu *ipmmu, unsigned long reg_off,
-			    unsigned long data)
-{
-	iowrite32(data, ipmmu->ipmmu_base + reg_off);
-}
-
-void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu)
-{
-	if (!ipmmu)
-		return;
-
-	spin_lock(&ipmmu->flush_lock);
-	if (ipmmu->tlb_enabled)
-		ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH | IMCTR1_TLBEN);
-	else
-		ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH);
-	spin_unlock(&ipmmu->flush_lock);
-}
-
-void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
-		   int asid)
-{
-	if (!ipmmu)
-		return;
-
-	spin_lock(&ipmmu->flush_lock);
-	switch (size) {
-	default:
-		ipmmu->tlb_enabled = 0;
-		break;
-	case 0x2000:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 1);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x1000:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 2);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x800:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 3);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x400:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 4);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x200:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 5);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x100:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 6);
-		ipmmu->tlb_enabled = 1;
-		break;
-	case 0x80:
-		ipmmu_reg_write(ipmmu, IMTTBCR, 7);
-		ipmmu->tlb_enabled = 1;
-		break;
-	}
-	ipmmu_reg_write(ipmmu, IMTTBR, phys);
-	ipmmu_reg_write(ipmmu, IMASID, asid);
-	spin_unlock(&ipmmu->flush_lock);
-}
-
-static int ipmmu_probe(struct platform_device *pdev)
-{
-	struct shmobile_ipmmu *ipmmu;
-	struct resource *res;
-	struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data;
-
-	ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL);
-	if (!ipmmu) {
-		dev_err(&pdev->dev, "cannot allocate device data\n");
-		return -ENOMEM;
-	}
-	spin_lock_init(&ipmmu->flush_lock);
-	ipmmu->dev = &pdev->dev;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(ipmmu->ipmmu_base))
-		return PTR_ERR(ipmmu->ipmmu_base);
-
-	ipmmu->dev_names = pdata->dev_names;
-	ipmmu->num_dev_names = pdata->num_dev_names;
-	platform_set_drvdata(pdev, ipmmu);
-	ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */
-	ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */
-	return ipmmu_iommu_init(ipmmu);
-}
-
-static struct platform_driver ipmmu_driver = {
-	.probe = ipmmu_probe,
-	.driver = {
-		.name = "ipmmu",
-	},
-};
-
-static int __init ipmmu_init(void)
-{
-	return platform_driver_register(&ipmmu_driver);
-}
-subsys_initcall(ipmmu_init);
diff --git a/drivers/iommu/shmobile-ipmmu.h b/drivers/iommu/shmobile-ipmmu.h
deleted file mode 100644
index 9524743ca1fb..000000000000
--- a/drivers/iommu/shmobile-ipmmu.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* shmobile-ipmmu.h
- *
- * Copyright (C) 2012  Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#ifndef __SHMOBILE_IPMMU_H__
-#define __SHMOBILE_IPMMU_H__
-
-struct shmobile_ipmmu {
-	struct device *dev;
-	void __iomem *ipmmu_base;
-	int tlb_enabled;
-	spinlock_t flush_lock;
-	const char * const *dev_names;
-	unsigned int num_dev_names;
-};
-
-#ifdef CONFIG_SHMOBILE_IPMMU_TLB
-void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu);
-void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
-		   int asid);
-int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu);
-#else
-static inline int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
-{
-	return -EINVAL;
-}
-#endif
-
-#endif /* __SHMOBILE_IPMMU_H__ */
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 11fc2a27fa2e..fb50911b3940 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -130,6 +130,11 @@ config ORION_IRQCHIP
 	select IRQ_DOMAIN
 	select MULTI_IRQ_HANDLER
 
+config PIC32_EVIC
+	bool
+	select GENERIC_IRQ_CHIP
+	select IRQ_DOMAIN
+
 config RENESAS_INTC_IRQPIN
 	bool
 	select IRQ_DOMAIN
@@ -154,6 +159,7 @@ config TB10X_IRQC
 config TS4800_IRQ
 	tristate "TS-4800 IRQ controller"
 	select IRQ_DOMAIN
+	depends on HAS_IOMEM
 	help
 	  Support for the TS-4800 FPGA IRQ controller
 
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index d4c2e4ebc308..18caacb60d58 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -58,3 +58,4 @@ obj-$(CONFIG_RENESAS_H8S_INTC)		+= irq-renesas-h8s.o
 obj-$(CONFIG_ARCH_SA1100)		+= irq-sa11x0.o
 obj-$(CONFIG_INGENIC_IRQ)		+= irq-ingenic.o
 obj-$(CONFIG_IMX_GPCV2)			+= irq-imx-gpcv2.o
+obj-$(CONFIG_PIC32_EVIC)		+= irq-pic32-evic.o
diff --git a/drivers/irqchip/irq-atmel-aic-common.c b/drivers/irqchip/irq-atmel-aic-common.c
index b12a5d58546f..37199b9b2cfa 100644
--- a/drivers/irqchip/irq-atmel-aic-common.c
+++ b/drivers/irqchip/irq-atmel-aic-common.c
@@ -86,7 +86,7 @@ int aic_common_set_priority(int priority, unsigned *val)
 	    priority > AT91_AIC_IRQ_MAX_PRIORITY)
 		return -EINVAL;
 
-	*val &= AT91_AIC_PRIOR;
+	*val &= ~AT91_AIC_PRIOR;
 	*val |= priority;
 
 	return 0;
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index e23d1d18f9d6..3447549fcc93 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -875,6 +875,7 @@ static int its_alloc_tables(const char *node_name, struct its_node *its)
 		}
 
 		alloc_size = (1 << order) * PAGE_SIZE;
+retry_alloc_baser:
 		alloc_pages = (alloc_size / psz);
 		if (alloc_pages > GITS_BASER_PAGES_MAX) {
 			alloc_pages = GITS_BASER_PAGES_MAX;
@@ -938,13 +939,16 @@ retry_baser:
 			 * size and retry. If we reach 4K, then
 			 * something is horribly wrong...
 			 */
+			free_pages((unsigned long)base, order);
+			its->tables[i] = NULL;
+
 			switch (psz) {
 			case SZ_16K:
 				psz = SZ_4K;
-				goto retry_baser;
+				goto retry_alloc_baser;
 			case SZ_64K:
 				psz = SZ_16K;
-				goto retry_baser;
+				goto retry_alloc_baser;
 			}
 		}
 
diff --git a/drivers/irqchip/irq-mxs.c b/drivers/irqchip/irq-mxs.c
index c22e2d40cb30..efe50845939d 100644
--- a/drivers/irqchip/irq-mxs.c
+++ b/drivers/irqchip/irq-mxs.c
@@ -241,6 +241,7 @@ static int __init asm9260_of_init(struct device_node *np,
 		writel(0, icoll_priv.intr + i);
 
 	icoll_add_domain(np, ASM9260_NUM_IRQS);
+	set_handle_irq(icoll_handle_irq);
 
 	return 0;
 }
diff --git a/drivers/irqchip/irq-pic32-evic.c b/drivers/irqchip/irq-pic32-evic.c
new file mode 100644
index 000000000000..e7155db01d55
--- /dev/null
+++ b/drivers/irqchip/irq-pic32-evic.c
@@ -0,0 +1,324 @@
+/*
+ * Cristian Birsan <cristian.birsan@microchip.com>
+ * Joshua Henderson <joshua.henderson@microchip.com>
+ * Copyright (C) 2016 Microchip Technology Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/irqdomain.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/irq.h>
+
+#include <asm/irq.h>
+#include <asm/traps.h>
+#include <asm/mach-pic32/pic32.h>
+
+#define REG_INTCON	0x0000
+#define REG_INTSTAT	0x0020
+#define REG_IFS_OFFSET	0x0040
+#define REG_IEC_OFFSET	0x00C0
+#define REG_IPC_OFFSET	0x0140
+#define REG_OFF_OFFSET	0x0540
+
+#define MAJPRI_MASK	0x07
+#define SUBPRI_MASK	0x03
+#define PRIORITY_MASK	0x1F
+
+#define PIC32_INT_PRI(pri, subpri)				\
+	((((pri) & MAJPRI_MASK) << 2) | ((subpri) & SUBPRI_MASK))
+
+struct evic_chip_data {
+	u32 irq_types[NR_IRQS];
+	u32 ext_irqs[8];
+};
+
+static struct irq_domain *evic_irq_domain;
+static void __iomem *evic_base;
+
+asmlinkage void __weak plat_irq_dispatch(void)
+{
+	unsigned int irq, hwirq;
+
+	hwirq = readl(evic_base + REG_INTSTAT) & 0xFF;
+	irq = irq_linear_revmap(evic_irq_domain, hwirq);
+	do_IRQ(irq);
+}
+
+static struct evic_chip_data *irqd_to_priv(struct irq_data *data)
+{
+	return (struct evic_chip_data *)data->domain->host_data;
+}
+
+static int pic32_set_ext_polarity(int bit, u32 type)
+{
+	/*
+	 * External interrupts can be either edge rising or edge falling,
+	 * but not both.
+	 */
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+		writel(BIT(bit), evic_base + PIC32_SET(REG_INTCON));
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		writel(BIT(bit), evic_base + PIC32_CLR(REG_INTCON));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int pic32_set_type_edge(struct irq_data *data,
+			       unsigned int flow_type)
+{
+	struct evic_chip_data *priv = irqd_to_priv(data);
+	int ret;
+	int i;
+
+	if (!(flow_type & IRQ_TYPE_EDGE_BOTH))
+		return -EBADR;
+
+	/* set polarity for external interrupts only */
+	for (i = 0; i < ARRAY_SIZE(priv->ext_irqs); i++) {
+		if (priv->ext_irqs[i] == data->hwirq) {
+			ret = pic32_set_ext_polarity(i + 1, flow_type);
+			if (ret)
+				return ret;
+		}
+	}
+
+	irqd_set_trigger_type(data, flow_type);
+
+	return IRQ_SET_MASK_OK;
+}
+
+static void pic32_bind_evic_interrupt(int irq, int set)
+{
+	writel(set, evic_base + REG_OFF_OFFSET + irq * 4);
+}
+
+static void pic32_set_irq_priority(int irq, int priority)
+{
+	u32 reg, shift;
+
+	reg = irq / 4;
+	shift = (irq % 4) * 8;
+
+	writel(PRIORITY_MASK << shift,
+		evic_base + PIC32_CLR(REG_IPC_OFFSET + reg * 0x10));
+	writel(priority << shift,
+		evic_base + PIC32_SET(REG_IPC_OFFSET + reg * 0x10));
+}
+
+#define IRQ_REG_MASK(_hwirq, _reg, _mask)		       \
+	do {						       \
+		_reg = _hwirq / 32;			       \
+		_mask = 1 << (_hwirq % 32);		       \
+	} while (0)
+
+static int pic32_irq_domain_map(struct irq_domain *d, unsigned int virq,
+				irq_hw_number_t hw)
+{
+	struct evic_chip_data *priv = d->host_data;
+	struct irq_data *data;
+	int ret;
+	u32 iecclr, ifsclr;
+	u32 reg, mask;
+
+	ret = irq_map_generic_chip(d, virq, hw);
+	if (ret)
+		return ret;
+
+	/*
+	 * Piggyback on xlate function to move to an alternate chip as necessary
+	 * at time of mapping instead of allowing the flow handler/chip to be
+	 * changed later. This requires all interrupts to be configured through
+	 * DT.
+	 */
+	if (priv->irq_types[hw] & IRQ_TYPE_SENSE_MASK) {
+		data = irq_domain_get_irq_data(d, virq);
+		irqd_set_trigger_type(data, priv->irq_types[hw]);
+		irq_setup_alt_chip(data, priv->irq_types[hw]);
+	}
+
+	IRQ_REG_MASK(hw, reg, mask);
+
+	iecclr = PIC32_CLR(REG_IEC_OFFSET + reg * 0x10);
+	ifsclr = PIC32_CLR(REG_IFS_OFFSET + reg * 0x10);
+
+	/* mask and clear flag */
+	writel(mask, evic_base + iecclr);
+	writel(mask, evic_base + ifsclr);
+
+	/* default priority is required */
+	pic32_set_irq_priority(hw, PIC32_INT_PRI(2, 0));
+
+	return ret;
+}
+
+int pic32_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
+			   const u32 *intspec, unsigned int intsize,
+			   irq_hw_number_t *out_hwirq, unsigned int *out_type)
+{
+	struct evic_chip_data *priv = d->host_data;
+
+	if (WARN_ON(intsize < 2))
+		return -EINVAL;
+
+	if (WARN_ON(intspec[0] >= NR_IRQS))
+		return -EINVAL;
+
+	*out_hwirq = intspec[0];
+	*out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+	priv->irq_types[intspec[0]] = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+	return 0;
+}
+
+static const struct irq_domain_ops pic32_irq_domain_ops = {
+	.map	= pic32_irq_domain_map,
+	.xlate	= pic32_irq_domain_xlate,
+};
+
+static void __init pic32_ext_irq_of_init(struct irq_domain *domain)
+{
+	struct device_node *node = irq_domain_get_of_node(domain);
+	struct evic_chip_data *priv = domain->host_data;
+	struct property *prop;
+	const __le32 *p;
+	u32 hwirq;
+	int i = 0;
+	const char *pname = "microchip,external-irqs";
+
+	of_property_for_each_u32(node, pname, prop, p, hwirq) {
+		if (i >= ARRAY_SIZE(priv->ext_irqs)) {
+			pr_warn("More than %d external irq, skip rest\n",
+				ARRAY_SIZE(priv->ext_irqs));
+			break;
+		}
+
+		priv->ext_irqs[i] = hwirq;
+		i++;
+	}
+}
+
+static int __init pic32_of_init(struct device_node *node,
+				struct device_node *parent)
+{
+	struct irq_chip_generic *gc;
+	struct evic_chip_data *priv;
+	unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
+	int nchips, ret;
+	int i;
+
+	nchips = DIV_ROUND_UP(NR_IRQS, 32);
+
+	evic_base = of_iomap(node, 0);
+	if (!evic_base)
+		return -ENOMEM;
+
+	priv = kcalloc(nchips, sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		ret = -ENOMEM;
+		goto err_iounmap;
+	}
+
+	evic_irq_domain = irq_domain_add_linear(node, nchips * 32,
+						&pic32_irq_domain_ops,
+						priv);
+	if (!evic_irq_domain) {
+		ret = -ENOMEM;
+		goto err_free_priv;
+	}
+
+	/*
+	 * The PIC32 EVIC has a linear list of irqs and the type of each
+	 * irq is determined by the hardware peripheral the EVIC is arbitrating.
+	 * These irq types are defined in the datasheet as "persistent" and
+	 * "non-persistent" which are mapped here to level and edge
+	 * respectively. To manage the different flow handler requirements of
+	 * each irq type, different chip_types are used.
+	 */
+	ret = irq_alloc_domain_generic_chips(evic_irq_domain, 32, 2,
+					     "evic-level", handle_level_irq,
+					     clr, 0, 0);
+	if (ret)
+		goto err_domain_remove;
+
+	board_bind_eic_interrupt = &pic32_bind_evic_interrupt;
+
+	for (i = 0; i < nchips; i++) {
+		u32 ifsclr = PIC32_CLR(REG_IFS_OFFSET + (i * 0x10));
+		u32 iec = REG_IEC_OFFSET + (i * 0x10);
+
+		gc = irq_get_domain_generic_chip(evic_irq_domain, i * 32);
+
+		gc->reg_base = evic_base;
+		gc->unused = 0;
+
+		/*
+		 * Level/persistent interrupts have a special requirement that
+		 * the condition generating the interrupt be cleared before the
+		 * interrupt flag (ifs) can be cleared. chip.irq_eoi is used to
+		 * complete the interrupt with an ack.
+		 */
+		gc->chip_types[0].type			= IRQ_TYPE_LEVEL_MASK;
+		gc->chip_types[0].handler		= handle_fasteoi_irq;
+		gc->chip_types[0].regs.ack		= ifsclr;
+		gc->chip_types[0].regs.mask		= iec;
+		gc->chip_types[0].chip.name		= "evic-level";
+		gc->chip_types[0].chip.irq_eoi		= irq_gc_ack_set_bit;
+		gc->chip_types[0].chip.irq_mask		= irq_gc_mask_clr_bit;
+		gc->chip_types[0].chip.irq_unmask	= irq_gc_mask_set_bit;
+		gc->chip_types[0].chip.flags		= IRQCHIP_SKIP_SET_WAKE;
+
+		/* Edge interrupts */
+		gc->chip_types[1].type			= IRQ_TYPE_EDGE_BOTH;
+		gc->chip_types[1].handler		= handle_edge_irq;
+		gc->chip_types[1].regs.ack		= ifsclr;
+		gc->chip_types[1].regs.mask		= iec;
+		gc->chip_types[1].chip.name		= "evic-edge";
+		gc->chip_types[1].chip.irq_ack		= irq_gc_ack_set_bit;
+		gc->chip_types[1].chip.irq_mask		= irq_gc_mask_clr_bit;
+		gc->chip_types[1].chip.irq_unmask	= irq_gc_mask_set_bit;
+		gc->chip_types[1].chip.irq_set_type	= pic32_set_type_edge;
+		gc->chip_types[1].chip.flags		= IRQCHIP_SKIP_SET_WAKE;
+
+		gc->private = &priv[i];
+	}
+
+	irq_set_default_host(evic_irq_domain);
+
+	/*
+	 * External interrupts have software configurable edge polarity. These
+	 * interrupts are defined in DT allowing polarity to be configured only
+	 * for these interrupts when requested.
+	 */
+	pic32_ext_irq_of_init(evic_irq_domain);
+
+	return 0;
+
+err_domain_remove:
+	irq_domain_remove(evic_irq_domain);
+
+err_free_priv:
+	kfree(priv);
+
+err_iounmap:
+	iounmap(evic_base);
+
+	return ret;
+}
+
+IRQCHIP_DECLARE(pic32_evic, "microchip,pic32mzda-evic", pic32_of_init);
diff --git a/drivers/irqchip/irq-renesas-h8s.c b/drivers/irqchip/irq-renesas-h8s.c
index 8098ead1eb22..af8c6c61c824 100644
--- a/drivers/irqchip/irq-renesas-h8s.c
+++ b/drivers/irqchip/irq-renesas-h8s.c
@@ -40,8 +40,8 @@ static void h8s_disable_irq(struct irq_data *data)
 	addr = IPRA + ((ipr_table[irq - 16] & 0xf0) >> 3);
 	pos = (ipr_table[irq - 16] & 0x0f) * 4;
 	pri = ~(0x000f << pos);
-	pri &= ctrl_inw(addr);
-	ctrl_outw(pri, addr);
+	pri &= readw(addr);
+	writew(pri, addr);
 }
 
 static void h8s_enable_irq(struct irq_data *data)
@@ -54,9 +54,9 @@ static void h8s_enable_irq(struct irq_data *data)
 	addr = IPRA + ((ipr_table[irq - 16] & 0xf0) >> 3);
 	pos = (ipr_table[irq - 16] & 0x0f) * 4;
 	pri = ~(0x000f << pos);
-	pri &= ctrl_inw(addr);
+	pri &= readw(addr);
 	pri |= 1 << pos;
-	ctrl_outw(pri, addr);
+	writew(pri, addr);
 }
 
 struct irq_chip h8s_irq_chip = {
@@ -90,7 +90,7 @@ static int __init h8s_intc_of_init(struct device_node *intc,
 	/* All interrupt priority is 0 (disable) */
 	/* IPRA to IPRK */
 	for (n = 0; n <= 'k' - 'a'; n++)
-		ctrl_outw(0x0000, IPRA + (n * 2));
+		writew(0x0000, IPRA + (n * 2));
 
 	domain = irq_domain_add_linear(intc, NR_IRQS, &irq_ops, NULL);
 	BUG_ON(!domain);
diff --git a/drivers/irqchip/irq-s3c24xx.c b/drivers/irqchip/irq-s3c24xx.c
index c71914e8f596..5dc5a760c723 100644
--- a/drivers/irqchip/irq-s3c24xx.c
+++ b/drivers/irqchip/irq-s3c24xx.c
@@ -605,7 +605,7 @@ err:
 	return ERR_PTR(ret);
 }
 
-static struct s3c_irq_data init_eint[32] = {
+static struct s3c_irq_data __maybe_unused init_eint[32] = {
 	{ .type = S3C_IRQTYPE_NONE, }, /* reserved */
 	{ .type = S3C_IRQTYPE_NONE, }, /* reserved */
 	{ .type = S3C_IRQTYPE_NONE, }, /* reserved */
diff --git a/drivers/irqchip/irq-versatile-fpga.c b/drivers/irqchip/irq-versatile-fpga.c
index cadf104e3074..598ab3f0e0ac 100644
--- a/drivers/irqchip/irq-versatile-fpga.c
+++ b/drivers/irqchip/irq-versatile-fpga.c
@@ -210,12 +210,7 @@ int __init fpga_irq_of_init(struct device_node *node,
 		parent_irq = -1;
 	}
 
-#ifdef CONFIG_ARCH_VERSATILE
-	fpga_irq_init(base, node->name, IRQ_SIC_START, parent_irq, valid_mask,
-				  node);
-#else
 	fpga_irq_init(base, node->name, 0, parent_irq, valid_mask, node);
-#endif
 
 	writel(clear_mask, base + IRQ_ENABLE_CLEAR);
 	writel(clear_mask, base + FIQ_ENABLE_CLEAR);
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index 17c63ec9fb9e..e3d3b1aaa9e0 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -242,7 +242,7 @@ static void pca9532_input_work(struct work_struct *work)
 #ifdef CONFIG_LEDS_PCA9532_GPIO
 static int pca9532_gpio_request_pin(struct gpio_chip *gc, unsigned offset)
 {
-	struct pca9532_data *data = container_of(gc, struct pca9532_data, gpio);
+	struct pca9532_data *data = gpiochip_get_data(gc);
 	struct pca9532_led *led = &data->leds[offset];
 
 	if (led->type == PCA9532_TYPE_GPIO)
@@ -253,7 +253,7 @@ static int pca9532_gpio_request_pin(struct gpio_chip *gc, unsigned offset)
 
 static void pca9532_gpio_set_value(struct gpio_chip *gc, unsigned offset, int val)
 {
-	struct pca9532_data *data = container_of(gc, struct pca9532_data, gpio);
+	struct pca9532_data *data = gpiochip_get_data(gc);
 	struct pca9532_led *led = &data->leds[offset];
 
 	if (val)
@@ -266,7 +266,7 @@ static void pca9532_gpio_set_value(struct gpio_chip *gc, unsigned offset, int va
 
 static int pca9532_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct pca9532_data *data = container_of(gc, struct pca9532_data, gpio);
+	struct pca9532_data *data = gpiochip_get_data(gc);
 	unsigned char reg;
 
 	reg = i2c_smbus_read_byte_data(data->client, PCA9532_REG_INPUT(offset));
@@ -315,7 +315,7 @@ static int pca9532_destroy_devices(struct pca9532_data *data, int n_devs)
 	}
 
 #ifdef CONFIG_LEDS_PCA9532_GPIO
-	if (data->gpio.dev)
+	if (data->gpio.parent)
 		gpiochip_remove(&data->gpio);
 #endif
 
@@ -409,13 +409,13 @@ static int pca9532_configure(struct i2c_client *client,
 		data->gpio.can_sleep = 1;
 		data->gpio.base = pdata->gpio_base;
 		data->gpio.ngpio = data->chip_info->num_leds;
-		data->gpio.dev = &client->dev;
+		data->gpio.parent = &client->dev;
 		data->gpio.owner = THIS_MODULE;
 
-		err = gpiochip_add(&data->gpio);
+		err = gpiochip_add_data(&data->gpio, data);
 		if (err) {
 			/* Use data->gpio.dev as a flag for freeing gpiochip */
-			data->gpio.dev = NULL;
+			data->gpio.parent = NULL;
 			dev_warn(&client->dev, "could not add gpiochip\n");
 		} else {
 			dev_info(&client->dev, "gpios %i...%i\n",
diff --git a/drivers/leds/leds-tca6507.c b/drivers/leds/leds-tca6507.c
index edbecc4ca2da..c548ea10f0f0 100644
--- a/drivers/leds/leds-tca6507.c
+++ b/drivers/leds/leds-tca6507.c
@@ -603,7 +603,7 @@ static int tca6507_blink_set(struct led_classdev *led_cdev,
 static void tca6507_gpio_set_value(struct gpio_chip *gc,
 				   unsigned offset, int val)
 {
-	struct tca6507_chip *tca = container_of(gc, struct tca6507_chip, gpio);
+	struct tca6507_chip *tca = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&tca->lock, flags);
@@ -651,11 +651,11 @@ static int tca6507_probe_gpios(struct i2c_client *client,
 	tca->gpio.owner = THIS_MODULE;
 	tca->gpio.direction_output = tca6507_gpio_direction_output;
 	tca->gpio.set = tca6507_gpio_set_value;
-	tca->gpio.dev = &client->dev;
+	tca->gpio.parent = &client->dev;
 #ifdef CONFIG_OF_GPIO
 	tca->gpio.of_node = of_node_get(client->dev.of_node);
 #endif
-	err = gpiochip_add(&tca->gpio);
+	err = gpiochip_add_data(&tca->gpio, tca);
 	if (err) {
 		tca->gpio.ngpio = 0;
 		return err;
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile
index 7e0f42acb737..a7a0a22cf1a5 100644
--- a/drivers/lightnvm/Makefile
+++ b/drivers/lightnvm/Makefile
@@ -2,6 +2,6 @@
 # Makefile for Open-Channel SSDs.
 #
 
-obj-$(CONFIG_NVM)		:= core.o
+obj-$(CONFIG_NVM)		:= core.o sysblk.o
 obj-$(CONFIG_NVM_GENNVM) 	+= gennvm.o
 obj-$(CONFIG_NVM_RRPC)		+= rrpc.o
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 8f41b245cd55..33224cb91c5b 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -27,6 +27,7 @@
 #include <linux/module.h>
 #include <linux/miscdevice.h>
 #include <linux/lightnvm.h>
+#include <linux/sched/sysctl.h>
 #include <uapi/linux/lightnvm.h>
 
 static LIST_HEAD(nvm_targets);
@@ -105,6 +106,9 @@ struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev)
 	lockdep_assert_held(&nvm_lock);
 
 	list_for_each_entry(mt, &nvm_mgrs, list) {
+		if (strncmp(dev->sb.mmtype, mt->name, NVM_MMTYPE_LEN))
+			continue;
+
 		ret = mt->register_mgr(dev);
 		if (ret < 0) {
 			pr_err("nvm: media mgr failed to init (%d) on dev %s\n",
@@ -166,6 +170,20 @@ static struct nvm_dev *nvm_find_nvm_dev(const char *name)
 	return NULL;
 }
 
+struct nvm_block *nvm_get_blk_unlocked(struct nvm_dev *dev, struct nvm_lun *lun,
+							unsigned long flags)
+{
+	return dev->mt->get_blk_unlocked(dev, lun, flags);
+}
+EXPORT_SYMBOL(nvm_get_blk_unlocked);
+
+/* Assumes that all valid pages have already been moved on release to bm */
+void nvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk)
+{
+	return dev->mt->put_blk_unlocked(dev, blk);
+}
+EXPORT_SYMBOL(nvm_put_blk_unlocked);
+
 struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun,
 							unsigned long flags)
 {
@@ -192,6 +210,206 @@ int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk)
 }
 EXPORT_SYMBOL(nvm_erase_blk);
 
+void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
+{
+	int i;
+
+	if (rqd->nr_pages > 1) {
+		for (i = 0; i < rqd->nr_pages; i++)
+			rqd->ppa_list[i] = dev_to_generic_addr(dev,
+							rqd->ppa_list[i]);
+	} else {
+		rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
+	}
+}
+EXPORT_SYMBOL(nvm_addr_to_generic_mode);
+
+void nvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
+{
+	int i;
+
+	if (rqd->nr_pages > 1) {
+		for (i = 0; i < rqd->nr_pages; i++)
+			rqd->ppa_list[i] = generic_to_dev_addr(dev,
+							rqd->ppa_list[i]);
+	} else {
+		rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
+	}
+}
+EXPORT_SYMBOL(nvm_generic_to_addr_mode);
+
+int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
+					struct ppa_addr *ppas, int nr_ppas)
+{
+	int i, plane_cnt, pl_idx;
+
+	if (dev->plane_mode == NVM_PLANE_SINGLE && nr_ppas == 1) {
+		rqd->nr_pages = 1;
+		rqd->ppa_addr = ppas[0];
+
+		return 0;
+	}
+
+	plane_cnt = (1 << dev->plane_mode);
+	rqd->nr_pages = plane_cnt * nr_ppas;
+
+	if (dev->ops->max_phys_sect < rqd->nr_pages)
+		return -EINVAL;
+
+	rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
+	if (!rqd->ppa_list) {
+		pr_err("nvm: failed to allocate dma memory\n");
+		return -ENOMEM;
+	}
+
+	for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
+		for (i = 0; i < nr_ppas; i++) {
+			ppas[i].g.pl = pl_idx;
+			rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppas[i];
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(nvm_set_rqd_ppalist);
+
+void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
+{
+	if (!rqd->ppa_list)
+		return;
+
+	nvm_dev_dma_free(dev, rqd->ppa_list, rqd->dma_ppa_list);
+}
+EXPORT_SYMBOL(nvm_free_rqd_ppalist);
+
+int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas)
+{
+	struct nvm_rq rqd;
+	int ret;
+
+	if (!dev->ops->erase_block)
+		return 0;
+
+	memset(&rqd, 0, sizeof(struct nvm_rq));
+
+	ret = nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas);
+	if (ret)
+		return ret;
+
+	nvm_generic_to_addr_mode(dev, &rqd);
+
+	ret = dev->ops->erase_block(dev, &rqd);
+
+	nvm_free_rqd_ppalist(dev, &rqd);
+
+	return ret;
+}
+EXPORT_SYMBOL(nvm_erase_ppa);
+
+void nvm_end_io(struct nvm_rq *rqd, int error)
+{
+	rqd->error = error;
+	rqd->end_io(rqd);
+}
+EXPORT_SYMBOL(nvm_end_io);
+
+static void nvm_end_io_sync(struct nvm_rq *rqd)
+{
+	struct completion *waiting = rqd->wait;
+
+	rqd->wait = NULL;
+
+	complete(waiting);
+}
+
+int nvm_submit_ppa(struct nvm_dev *dev, struct ppa_addr *ppa, int nr_ppas,
+				int opcode, int flags, void *buf, int len)
+{
+	DECLARE_COMPLETION_ONSTACK(wait);
+	struct nvm_rq rqd;
+	struct bio *bio;
+	int ret;
+	unsigned long hang_check;
+
+	bio = bio_map_kern(dev->q, buf, len, GFP_KERNEL);
+	if (IS_ERR_OR_NULL(bio))
+		return -ENOMEM;
+
+	memset(&rqd, 0, sizeof(struct nvm_rq));
+	ret = nvm_set_rqd_ppalist(dev, &rqd, ppa, nr_ppas);
+	if (ret) {
+		bio_put(bio);
+		return ret;
+	}
+
+	rqd.opcode = opcode;
+	rqd.bio = bio;
+	rqd.wait = &wait;
+	rqd.dev = dev;
+	rqd.end_io = nvm_end_io_sync;
+	rqd.flags = flags;
+	nvm_generic_to_addr_mode(dev, &rqd);
+
+	ret = dev->ops->submit_io(dev, &rqd);
+
+	/* Prevent hang_check timer from firing at us during very long I/O */
+	hang_check = sysctl_hung_task_timeout_secs;
+	if (hang_check)
+		while (!wait_for_completion_io_timeout(&wait, hang_check * (HZ/2)));
+	else
+		wait_for_completion_io(&wait);
+
+	nvm_free_rqd_ppalist(dev, &rqd);
+
+	return rqd.error;
+}
+EXPORT_SYMBOL(nvm_submit_ppa);
+
+static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
+{
+	int i;
+
+	dev->lps_per_blk = dev->pgs_per_blk;
+	dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
+	if (!dev->lptbl)
+		return -ENOMEM;
+
+	/* Just a linear array */
+	for (i = 0; i < dev->lps_per_blk; i++)
+		dev->lptbl[i] = i;
+
+	return 0;
+}
+
+static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
+{
+	int i, p;
+	struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
+
+	if (!mlc->num_pairs)
+		return 0;
+
+	dev->lps_per_blk = mlc->num_pairs;
+	dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
+	if (!dev->lptbl)
+		return -ENOMEM;
+
+	/* The lower page table encoding consists of a list of bytes, where each
+	 * has a lower and an upper half. The first half byte maintains the
+	 * increment value and every value after is an offset added to the
+	 * previous incrementation value */
+	dev->lptbl[0] = mlc->pairs[0] & 0xF;
+	for (i = 1; i < dev->lps_per_blk; i++) {
+		p = mlc->pairs[i >> 1];
+		if (i & 0x1) /* upper */
+			dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
+		else /* lower */
+			dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
+	}
+
+	return 0;
+}
+
 static int nvm_core_init(struct nvm_dev *dev)
 {
 	struct nvm_id *id = &dev->identity;
@@ -206,6 +424,7 @@ static int nvm_core_init(struct nvm_dev *dev)
 	dev->sec_size = grp->csecs;
 	dev->oob_size = grp->sos;
 	dev->sec_per_pg = grp->fpg_sz / grp->csecs;
+	dev->mccap = grp->mccap;
 	memcpy(&dev->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
 
 	dev->plane_mode = NVM_PLANE_SINGLE;
@@ -216,11 +435,23 @@ static int nvm_core_init(struct nvm_dev *dev)
 		return -EINVAL;
 	}
 
-	if (grp->fmtype != 0 && grp->fmtype != 1) {
+	switch (grp->fmtype) {
+	case NVM_ID_FMTYPE_SLC:
+		if (nvm_init_slc_tbl(dev, grp))
+			return -ENOMEM;
+		break;
+	case NVM_ID_FMTYPE_MLC:
+		if (nvm_init_mlc_tbl(dev, grp))
+			return -ENOMEM;
+		break;
+	default:
 		pr_err("nvm: flash type not supported\n");
 		return -EINVAL;
 	}
 
+	if (!dev->lps_per_blk)
+		pr_info("nvm: lower page programming table missing\n");
+
 	if (grp->mpos & 0x020202)
 		dev->plane_mode = NVM_PLANE_DOUBLE;
 	if (grp->mpos & 0x040404)
@@ -238,6 +469,7 @@ static int nvm_core_init(struct nvm_dev *dev)
 				dev->nr_chnls;
 	dev->total_pages = dev->total_blocks * dev->pgs_per_blk;
 	INIT_LIST_HEAD(&dev->online_targets);
+	mutex_init(&dev->mlock);
 
 	return 0;
 }
@@ -249,6 +481,8 @@ static void nvm_free(struct nvm_dev *dev)
 
 	if (dev->mt)
 		dev->mt->unregister_mgr(dev);
+
+	kfree(dev->lptbl);
 }
 
 static int nvm_init(struct nvm_dev *dev)
@@ -338,9 +572,16 @@ int nvm_register(struct request_queue *q, char *disk_name,
 		}
 	}
 
+	ret = nvm_get_sysblock(dev, &dev->sb);
+	if (!ret)
+		pr_err("nvm: device not initialized.\n");
+	else if (ret < 0)
+		pr_err("nvm: err (%d) on device initialization\n", ret);
+
 	/* register device with a supported media manager */
 	down_write(&nvm_lock);
-	dev->mt = nvm_init_mgr(dev);
+	if (ret > 0)
+		dev->mt = nvm_init_mgr(dev);
 	list_add(&dev->devices, &nvm_devices);
 	up_write(&nvm_lock);
 
@@ -788,6 +1029,97 @@ static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
 	return __nvm_configure_remove(&remove);
 }
 
+static void nvm_setup_nvm_sb_info(struct nvm_sb_info *info)
+{
+	info->seqnr = 1;
+	info->erase_cnt = 0;
+	info->version = 1;
+}
+
+static long __nvm_ioctl_dev_init(struct nvm_ioctl_dev_init *init)
+{
+	struct nvm_dev *dev;
+	struct nvm_sb_info info;
+	int ret;
+
+	down_write(&nvm_lock);
+	dev = nvm_find_nvm_dev(init->dev);
+	up_write(&nvm_lock);
+	if (!dev) {
+		pr_err("nvm: device not found\n");
+		return -EINVAL;
+	}
+
+	nvm_setup_nvm_sb_info(&info);
+
+	strncpy(info.mmtype, init->mmtype, NVM_MMTYPE_LEN);
+	info.fs_ppa.ppa = -1;
+
+	ret = nvm_init_sysblock(dev, &info);
+	if (ret)
+		return ret;
+
+	memcpy(&dev->sb, &info, sizeof(struct nvm_sb_info));
+
+	down_write(&nvm_lock);
+	dev->mt = nvm_init_mgr(dev);
+	up_write(&nvm_lock);
+
+	return 0;
+}
+
+static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
+{
+	struct nvm_ioctl_dev_init init;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
+		return -EFAULT;
+
+	if (init.flags != 0) {
+		pr_err("nvm: no flags supported\n");
+		return -EINVAL;
+	}
+
+	init.dev[DISK_NAME_LEN - 1] = '\0';
+
+	return __nvm_ioctl_dev_init(&init);
+}
+
+static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
+{
+	struct nvm_ioctl_dev_factory fact;
+	struct nvm_dev *dev;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
+		return -EFAULT;
+
+	fact.dev[DISK_NAME_LEN - 1] = '\0';
+
+	if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
+		return -EINVAL;
+
+	down_write(&nvm_lock);
+	dev = nvm_find_nvm_dev(fact.dev);
+	up_write(&nvm_lock);
+	if (!dev) {
+		pr_err("nvm: device not found\n");
+		return -EINVAL;
+	}
+
+	if (dev->mt) {
+		dev->mt->unregister_mgr(dev);
+		dev->mt = NULL;
+	}
+
+	return nvm_dev_factory(dev, fact.flags);
+}
+
 static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
@@ -801,6 +1133,10 @@ static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
 		return nvm_ioctl_dev_create(file, argp);
 	case NVM_DEV_REMOVE:
 		return nvm_ioctl_dev_remove(file, argp);
+	case NVM_DEV_INIT:
+		return nvm_ioctl_dev_init(file, argp);
+	case NVM_DEV_FACTORY:
+		return nvm_ioctl_dev_factory(file, argp);
 	}
 	return 0;
 }
diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c
index a54b339951a3..7fb725b16148 100644
--- a/drivers/lightnvm/gennvm.c
+++ b/drivers/lightnvm/gennvm.c
@@ -60,7 +60,8 @@ static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn)
 		lun->vlun.lun_id = i % dev->luns_per_chnl;
 		lun->vlun.chnl_id = i / dev->luns_per_chnl;
 		lun->vlun.nr_free_blocks = dev->blks_per_lun;
-		lun->vlun.nr_inuse_blocks = 0;
+		lun->vlun.nr_open_blocks = 0;
+		lun->vlun.nr_closed_blocks = 0;
 		lun->vlun.nr_bad_blocks = 0;
 	}
 	return 0;
@@ -89,6 +90,7 @@ static int gennvm_block_bb(struct ppa_addr ppa, int nr_blocks, u8 *blks,
 
 		list_move_tail(&blk->list, &lun->bb_list);
 		lun->vlun.nr_bad_blocks++;
+		lun->vlun.nr_free_blocks--;
 	}
 
 	return 0;
@@ -133,15 +135,15 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
 		pba = pba - (dev->sec_per_lun * lun_id);
 		blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)];
 
-		if (!blk->type) {
+		if (!blk->state) {
 			/* at this point, we don't know anything about the
 			 * block. It's up to the FTL on top to re-etablish the
-			 * block state
+			 * block state. The block is assumed to be open.
 			 */
 			list_move_tail(&blk->list, &lun->used_list);
-			blk->type = 1;
+			blk->state = NVM_BLK_ST_OPEN;
 			lun->vlun.nr_free_blocks--;
-			lun->vlun.nr_inuse_blocks++;
+			lun->vlun.nr_open_blocks++;
 		}
 	}
 
@@ -255,14 +257,14 @@ static void gennvm_unregister(struct nvm_dev *dev)
 	module_put(THIS_MODULE);
 }
 
-static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
+static struct nvm_block *gennvm_get_blk_unlocked(struct nvm_dev *dev,
 				struct nvm_lun *vlun, unsigned long flags)
 {
 	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
 	struct nvm_block *blk = NULL;
 	int is_gc = flags & NVM_IOTYPE_GC;
 
-	spin_lock(&vlun->lock);
+	assert_spin_locked(&vlun->lock);
 
 	if (list_empty(&lun->free_list)) {
 		pr_err_ratelimited("gennvm: lun %u have no free pages available",
@@ -275,83 +277,64 @@ static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
 
 	blk = list_first_entry(&lun->free_list, struct nvm_block, list);
 	list_move_tail(&blk->list, &lun->used_list);
-	blk->type = 1;
+	blk->state = NVM_BLK_ST_OPEN;
 
 	lun->vlun.nr_free_blocks--;
-	lun->vlun.nr_inuse_blocks++;
+	lun->vlun.nr_open_blocks++;
 
 out:
+	return blk;
+}
+
+static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
+				struct nvm_lun *vlun, unsigned long flags)
+{
+	struct nvm_block *blk;
+
+	spin_lock(&vlun->lock);
+	blk = gennvm_get_blk_unlocked(dev, vlun, flags);
 	spin_unlock(&vlun->lock);
 	return blk;
 }
 
-static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
+static void gennvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk)
 {
 	struct nvm_lun *vlun = blk->lun;
 	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
 
-	spin_lock(&vlun->lock);
+	assert_spin_locked(&vlun->lock);
 
-	switch (blk->type) {
-	case 1:
+	if (blk->state & NVM_BLK_ST_OPEN) {
 		list_move_tail(&blk->list, &lun->free_list);
+		lun->vlun.nr_open_blocks--;
 		lun->vlun.nr_free_blocks++;
-		lun->vlun.nr_inuse_blocks--;
-		blk->type = 0;
-		break;
-	case 2:
+		blk->state = NVM_BLK_ST_FREE;
+	} else if (blk->state & NVM_BLK_ST_CLOSED) {
+		list_move_tail(&blk->list, &lun->free_list);
+		lun->vlun.nr_closed_blocks--;
+		lun->vlun.nr_free_blocks++;
+		blk->state = NVM_BLK_ST_FREE;
+	} else if (blk->state & NVM_BLK_ST_BAD) {
 		list_move_tail(&blk->list, &lun->bb_list);
 		lun->vlun.nr_bad_blocks++;
-		lun->vlun.nr_inuse_blocks--;
-		break;
-	default:
+		blk->state = NVM_BLK_ST_BAD;
+	} else {
 		WARN_ON_ONCE(1);
 		pr_err("gennvm: erroneous block type (%lu -> %u)\n",
-							blk->id, blk->type);
+							blk->id, blk->state);
 		list_move_tail(&blk->list, &lun->bb_list);
 		lun->vlun.nr_bad_blocks++;
-		lun->vlun.nr_inuse_blocks--;
-	}
-
-	spin_unlock(&vlun->lock);
-}
-
-static void gennvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
-{
-	int i;
-
-	if (rqd->nr_pages > 1) {
-		for (i = 0; i < rqd->nr_pages; i++)
-			rqd->ppa_list[i] = dev_to_generic_addr(dev,
-							rqd->ppa_list[i]);
-	} else {
-		rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
+		blk->state = NVM_BLK_ST_BAD;
 	}
 }
 
-static void gennvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
-{
-	int i;
-
-	if (rqd->nr_pages > 1) {
-		for (i = 0; i < rqd->nr_pages; i++)
-			rqd->ppa_list[i] = generic_to_dev_addr(dev,
-							rqd->ppa_list[i]);
-	} else {
-		rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
-	}
-}
-
-static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
+static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
 {
-	if (!dev->ops->submit_io)
-		return 0;
-
-	/* Convert address space */
-	gennvm_generic_to_addr_mode(dev, rqd);
+	struct nvm_lun *vlun = blk->lun;
 
-	rqd->dev = dev;
-	return dev->ops->submit_io(dev, rqd);
+	spin_lock(&vlun->lock);
+	gennvm_put_blk_unlocked(dev, blk);
+	spin_unlock(&vlun->lock);
 }
 
 static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
@@ -376,7 +359,7 @@ static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
 	blk = &lun->vlun.blocks[ppa->g.blk];
 
 	/* will be moved to bb list on put_blk from target */
-	blk->type = type;
+	blk->state = type;
 }
 
 /* mark block bad. It is expected the target recover from the error. */
@@ -390,77 +373,51 @@ static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd)
 	if (dev->ops->set_bb_tbl(dev, rqd, 1))
 		return;
 
-	gennvm_addr_to_generic_mode(dev, rqd);
+	nvm_addr_to_generic_mode(dev, rqd);
 
 	/* look up blocks and mark them as bad */
 	if (rqd->nr_pages > 1)
 		for (i = 0; i < rqd->nr_pages; i++)
-			gennvm_blk_set_type(dev, &rqd->ppa_list[i], 2);
+			gennvm_blk_set_type(dev, &rqd->ppa_list[i],
+						NVM_BLK_ST_BAD);
 	else
-		gennvm_blk_set_type(dev, &rqd->ppa_addr, 2);
+		gennvm_blk_set_type(dev, &rqd->ppa_addr, NVM_BLK_ST_BAD);
 }
 
-static int gennvm_end_io(struct nvm_rq *rqd, int error)
+static void gennvm_end_io(struct nvm_rq *rqd)
 {
 	struct nvm_tgt_instance *ins = rqd->ins;
-	int ret = 0;
 
-	switch (error) {
+	switch (rqd->error) {
 	case NVM_RSP_SUCCESS:
-		break;
 	case NVM_RSP_ERR_EMPTYPAGE:
 		break;
 	case NVM_RSP_ERR_FAILWRITE:
 		gennvm_mark_blk_bad(rqd->dev, rqd);
-	default:
-		ret++;
 	}
 
-	ret += ins->tt->end_io(rqd, error);
-
-	return ret;
+	ins->tt->end_io(rqd);
 }
 
-static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
-							unsigned long flags)
+static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 {
-	int plane_cnt = 0, pl_idx, ret;
-	struct ppa_addr addr;
-	struct nvm_rq rqd;
-
-	if (!dev->ops->erase_block)
-		return 0;
-
-	addr = block_to_ppa(dev, blk);
-
-	if (dev->plane_mode == NVM_PLANE_SINGLE) {
-		rqd.nr_pages = 1;
-		rqd.ppa_addr = addr;
-	} else {
-		plane_cnt = (1 << dev->plane_mode);
-		rqd.nr_pages = plane_cnt;
-
-		rqd.ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL,
-							&rqd.dma_ppa_list);
-		if (!rqd.ppa_list) {
-			pr_err("gennvm: failed to allocate dma memory\n");
-			return -ENOMEM;
-		}
-
-		for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
-			addr.g.pl = pl_idx;
-			rqd.ppa_list[pl_idx] = addr;
-		}
-	}
+	if (!dev->ops->submit_io)
+		return -ENODEV;
 
-	gennvm_generic_to_addr_mode(dev, &rqd);
+	/* Convert address space */
+	nvm_generic_to_addr_mode(dev, rqd);
 
-	ret = dev->ops->erase_block(dev, &rqd);
+	rqd->dev = dev;
+	rqd->end_io = gennvm_end_io;
+	return dev->ops->submit_io(dev, rqd);
+}
 
-	if (plane_cnt)
-		nvm_dev_dma_free(dev, rqd.ppa_list, rqd.dma_ppa_list);
+static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
+							unsigned long flags)
+{
+	struct ppa_addr addr = block_to_ppa(dev, blk);
 
-	return ret;
+	return nvm_erase_ppa(dev, &addr, 1);
 }
 
 static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid)
@@ -480,10 +437,11 @@ static void gennvm_lun_info_print(struct nvm_dev *dev)
 	gennvm_for_each_lun(gn, lun, i) {
 		spin_lock(&lun->vlun.lock);
 
-		pr_info("%s: lun%8u\t%u\t%u\t%u\n",
+		pr_info("%s: lun%8u\t%u\t%u\t%u\t%u\n",
 				dev->name, i,
 				lun->vlun.nr_free_blocks,
-				lun->vlun.nr_inuse_blocks,
+				lun->vlun.nr_open_blocks,
+				lun->vlun.nr_closed_blocks,
 				lun->vlun.nr_bad_blocks);
 
 		spin_unlock(&lun->vlun.lock);
@@ -491,21 +449,23 @@ static void gennvm_lun_info_print(struct nvm_dev *dev)
 }
 
 static struct nvmm_type gennvm = {
-	.name		= "gennvm",
-	.version	= {0, 1, 0},
+	.name			= "gennvm",
+	.version		= {0, 1, 0},
+
+	.register_mgr		= gennvm_register,
+	.unregister_mgr		= gennvm_unregister,
 
-	.register_mgr	= gennvm_register,
-	.unregister_mgr	= gennvm_unregister,
+	.get_blk_unlocked	= gennvm_get_blk_unlocked,
+	.put_blk_unlocked	= gennvm_put_blk_unlocked,
 
-	.get_blk	= gennvm_get_blk,
-	.put_blk	= gennvm_put_blk,
+	.get_blk		= gennvm_get_blk,
+	.put_blk		= gennvm_put_blk,
 
-	.submit_io	= gennvm_submit_io,
-	.end_io		= gennvm_end_io,
-	.erase_blk	= gennvm_erase_blk,
+	.submit_io		= gennvm_submit_io,
+	.erase_blk		= gennvm_erase_blk,
 
-	.get_lun	= gennvm_get_lun,
-	.lun_info_print = gennvm_lun_info_print,
+	.get_lun		= gennvm_get_lun,
+	.lun_info_print		= gennvm_lun_info_print,
 };
 
 static int __init gennvm_module_init(void)
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
index 134e4faba482..d8c75958ced3 100644
--- a/drivers/lightnvm/rrpc.c
+++ b/drivers/lightnvm/rrpc.c
@@ -179,16 +179,23 @@ static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk)
 static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
 							unsigned long flags)
 {
+	struct nvm_lun *lun = rlun->parent;
 	struct nvm_block *blk;
 	struct rrpc_block *rblk;
 
-	blk = nvm_get_blk(rrpc->dev, rlun->parent, flags);
-	if (!blk)
+	spin_lock(&lun->lock);
+	blk = nvm_get_blk_unlocked(rrpc->dev, rlun->parent, flags);
+	if (!blk) {
+		pr_err("nvm: rrpc: cannot get new block from media manager\n");
+		spin_unlock(&lun->lock);
 		return NULL;
+	}
 
 	rblk = &rlun->blocks[blk->id];
-	blk->priv = rblk;
+	list_add_tail(&rblk->list, &rlun->open_list);
+	spin_unlock(&lun->lock);
 
+	blk->priv = rblk;
 	bitmap_zero(rblk->invalid_pages, rrpc->dev->pgs_per_blk);
 	rblk->next_page = 0;
 	rblk->nr_invalid_pages = 0;
@@ -199,7 +206,13 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
 
 static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	nvm_put_blk(rrpc->dev, rblk->parent);
+	struct rrpc_lun *rlun = rblk->rlun;
+	struct nvm_lun *lun = rlun->parent;
+
+	spin_lock(&lun->lock);
+	nvm_put_blk_unlocked(rrpc->dev, rblk->parent);
+	list_del(&rblk->list);
+	spin_unlock(&lun->lock);
 }
 
 static void rrpc_put_blks(struct rrpc *rrpc)
@@ -287,6 +300,8 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
 	}
 
 	page = mempool_alloc(rrpc->page_pool, GFP_NOIO);
+	if (!page)
+		return -ENOMEM;
 
 	while ((slot = find_first_zero_bit(rblk->invalid_pages,
 					    nr_pgs_per_blk)) < nr_pgs_per_blk) {
@@ -328,6 +343,10 @@ try:
 			goto finished;
 		}
 		wait_for_completion_io(&wait);
+		if (bio->bi_error) {
+			rrpc_inflight_laddr_release(rrpc, rqd);
+			goto finished;
+		}
 
 		bio_reset(bio);
 		reinit_completion(&wait);
@@ -350,6 +369,8 @@ try:
 		wait_for_completion_io(&wait);
 
 		rrpc_inflight_laddr_release(rrpc, rqd);
+		if (bio->bi_error)
+			goto finished;
 
 		bio_reset(bio);
 	}
@@ -373,16 +394,26 @@ static void rrpc_block_gc(struct work_struct *work)
 	struct rrpc *rrpc = gcb->rrpc;
 	struct rrpc_block *rblk = gcb->rblk;
 	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_lun *lun = rblk->parent->lun;
+	struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset];
 
+	mempool_free(gcb, rrpc->gcb_pool);
 	pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id);
 
 	if (rrpc_move_valid_pages(rrpc, rblk))
-		goto done;
+		goto put_back;
+
+	if (nvm_erase_blk(dev, rblk->parent))
+		goto put_back;
 
-	nvm_erase_blk(dev, rblk->parent);
 	rrpc_put_blk(rrpc, rblk);
-done:
-	mempool_free(gcb, rrpc->gcb_pool);
+
+	return;
+
+put_back:
+	spin_lock(&rlun->lock);
+	list_add_tail(&rblk->prio, &rlun->prio_list);
+	spin_unlock(&rlun->lock);
 }
 
 /* the block with highest number of invalid pages, will be in the beginning
@@ -427,7 +458,7 @@ static void rrpc_lun_gc(struct work_struct *work)
 	if (nr_blocks_need < rrpc->nr_luns)
 		nr_blocks_need = rrpc->nr_luns;
 
-	spin_lock(&lun->lock);
+	spin_lock(&rlun->lock);
 	while (nr_blocks_need > lun->nr_free_blocks &&
 					!list_empty(&rlun->prio_list)) {
 		struct rrpc_block *rblock = block_prio_find_max(rlun);
@@ -436,16 +467,16 @@ static void rrpc_lun_gc(struct work_struct *work)
 		if (!rblock->nr_invalid_pages)
 			break;
 
+		gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
+		if (!gcb)
+			break;
+
 		list_del_init(&rblock->prio);
 
 		BUG_ON(!block_is_full(rrpc, rblock));
 
 		pr_debug("rrpc: selected block '%lu' for GC\n", block->id);
 
-		gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
-		if (!gcb)
-			break;
-
 		gcb->rrpc = rrpc;
 		gcb->rblk = rblock;
 		INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
@@ -454,7 +485,7 @@ static void rrpc_lun_gc(struct work_struct *work)
 
 		nr_blocks_need--;
 	}
-	spin_unlock(&lun->lock);
+	spin_unlock(&rlun->lock);
 
 	/* TODO: Hint that request queue can be started again */
 }
@@ -635,12 +666,24 @@ static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
 		lun = rblk->parent->lun;
 
 		cmnt_size = atomic_inc_return(&rblk->data_cmnt_size);
-		if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk))
+		if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk)) {
+			struct nvm_block *blk = rblk->parent;
+			struct rrpc_lun *rlun = rblk->rlun;
+
+			spin_lock(&lun->lock);
+			lun->nr_open_blocks--;
+			lun->nr_closed_blocks++;
+			blk->state &= ~NVM_BLK_ST_OPEN;
+			blk->state |= NVM_BLK_ST_CLOSED;
+			list_move_tail(&rblk->list, &rlun->closed_list);
+			spin_unlock(&lun->lock);
+
 			rrpc_run_gc(rrpc, rblk);
+		}
 	}
 }
 
-static int rrpc_end_io(struct nvm_rq *rqd, int error)
+static void rrpc_end_io(struct nvm_rq *rqd)
 {
 	struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
 	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
@@ -650,11 +693,12 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error)
 	if (bio_data_dir(rqd->bio) == WRITE)
 		rrpc_end_io_write(rrpc, rrqd, laddr, npages);
 
+	bio_put(rqd->bio);
+
 	if (rrqd->flags & NVM_IOTYPE_GC)
-		return 0;
+		return;
 
 	rrpc_unlock_rq(rrpc, rqd);
-	bio_put(rqd->bio);
 
 	if (npages > 1)
 		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
@@ -662,8 +706,6 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error)
 		nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata);
 
 	mempool_free(rqd, rrpc->rq_pool);
-
-	return 0;
 }
 
 static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
@@ -841,6 +883,13 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
 	err = nvm_submit_io(rrpc->dev, rqd);
 	if (err) {
 		pr_err("rrpc: I/O submission failed: %d\n", err);
+		bio_put(bio);
+		if (!(flags & NVM_IOTYPE_GC)) {
+			rrpc_unlock_rq(rrpc, rqd);
+			if (rqd->nr_pages > 1)
+				nvm_dev_dma_free(rrpc->dev,
+			rqd->ppa_list, rqd->dma_ppa_list);
+		}
 		return NVM_IO_ERR;
 	}
 
@@ -1090,6 +1139,11 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
 	struct rrpc_lun *rlun;
 	int i, j;
 
+	if (dev->pgs_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
+		pr_err("rrpc: number of pages per block too high.");
+		return -EINVAL;
+	}
+
 	spin_lock_init(&rrpc->rev_lock);
 
 	rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun),
@@ -1101,16 +1155,13 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
 	for (i = 0; i < rrpc->nr_luns; i++) {
 		struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i);
 
-		if (dev->pgs_per_blk >
-				MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
-			pr_err("rrpc: number of pages per block too high.");
-			goto err;
-		}
-
 		rlun = &rrpc->luns[i];
 		rlun->rrpc = rrpc;
 		rlun->parent = lun;
 		INIT_LIST_HEAD(&rlun->prio_list);
+		INIT_LIST_HEAD(&rlun->open_list);
+		INIT_LIST_HEAD(&rlun->closed_list);
+
 		INIT_WORK(&rlun->ws_gc, rrpc_lun_gc);
 		spin_lock_init(&rlun->lock);
 
@@ -1127,6 +1178,7 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
 			struct nvm_block *blk = &lun->blocks[j];
 
 			rblk->parent = blk;
+			rblk->rlun = rlun;
 			INIT_LIST_HEAD(&rblk->prio);
 			spin_lock_init(&rblk->lock);
 		}
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
index a9696a06c38c..ef13ac7700c8 100644
--- a/drivers/lightnvm/rrpc.h
+++ b/drivers/lightnvm/rrpc.h
@@ -54,7 +54,9 @@ struct rrpc_rq {
 
 struct rrpc_block {
 	struct nvm_block *parent;
+	struct rrpc_lun *rlun;
 	struct list_head prio;
+	struct list_head list;
 
 #define MAX_INVALID_PAGES_STORAGE 8
 	/* Bitmap for invalid page intries */
@@ -73,7 +75,16 @@ struct rrpc_lun {
 	struct nvm_lun *parent;
 	struct rrpc_block *cur, *gc_cur;
 	struct rrpc_block *blocks;	/* Reference to block allocation */
-	struct list_head prio_list;		/* Blocks that may be GC'ed */
+
+	struct list_head prio_list;	/* Blocks that may be GC'ed */
+	struct list_head open_list;	/* In-use open blocks. These are blocks
+					 * that can be both written to and read
+					 * from
+					 */
+	struct list_head closed_list;	/* In-use closed blocks. These are
+					 * blocks that can _only_ be read from
+					 */
+
 	struct work_struct ws_gc;
 
 	spinlock_t lock;
diff --git a/drivers/lightnvm/sysblk.c b/drivers/lightnvm/sysblk.c
new file mode 100644
index 000000000000..321de1f154c5
--- /dev/null
+++ b/drivers/lightnvm/sysblk.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright (C) 2015 Matias Bjorling. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
+ * USA.
+ *
+ */
+
+#include <linux/lightnvm.h>
+
+#define MAX_SYSBLKS 3	/* remember to update mapping scheme on change */
+#define MAX_BLKS_PR_SYSBLK 2 /* 2 blks with 256 pages and 3000 erases
+			      * enables ~1.5M updates per sysblk unit
+			      */
+
+struct sysblk_scan {
+	/* A row is a collection of flash blocks for a system block. */
+	int nr_rows;
+	int row;
+	int act_blk[MAX_SYSBLKS];
+
+	int nr_ppas;
+	struct ppa_addr ppas[MAX_SYSBLKS * MAX_BLKS_PR_SYSBLK];/* all sysblks */
+};
+
+static inline int scan_ppa_idx(int row, int blkid)
+{
+	return (row * MAX_BLKS_PR_SYSBLK) + blkid;
+}
+
+void nvm_sysblk_to_cpu(struct nvm_sb_info *info, struct nvm_system_block *sb)
+{
+	info->seqnr = be32_to_cpu(sb->seqnr);
+	info->erase_cnt = be32_to_cpu(sb->erase_cnt);
+	info->version = be16_to_cpu(sb->version);
+	strncpy(info->mmtype, sb->mmtype, NVM_MMTYPE_LEN);
+	info->fs_ppa.ppa = be64_to_cpu(sb->fs_ppa);
+}
+
+void nvm_cpu_to_sysblk(struct nvm_system_block *sb, struct nvm_sb_info *info)
+{
+	sb->magic = cpu_to_be32(NVM_SYSBLK_MAGIC);
+	sb->seqnr = cpu_to_be32(info->seqnr);
+	sb->erase_cnt = cpu_to_be32(info->erase_cnt);
+	sb->version = cpu_to_be16(info->version);
+	strncpy(sb->mmtype, info->mmtype, NVM_MMTYPE_LEN);
+	sb->fs_ppa = cpu_to_be64(info->fs_ppa.ppa);
+}
+
+static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
+{
+	int nr_rows = min_t(int, MAX_SYSBLKS, dev->nr_chnls);
+	int i;
+
+	for (i = 0; i < nr_rows; i++)
+		sysblk_ppas[i].ppa = 0;
+
+	/* if possible, place sysblk at first channel, middle channel and last
+	 * channel of the device. If not, create only one or two sys blocks
+	 */
+	switch (dev->nr_chnls) {
+	case 2:
+		sysblk_ppas[1].g.ch = 1;
+		/* fall-through */
+	case 1:
+		sysblk_ppas[0].g.ch = 0;
+		break;
+	default:
+		sysblk_ppas[0].g.ch = 0;
+		sysblk_ppas[1].g.ch = dev->nr_chnls / 2;
+		sysblk_ppas[2].g.ch = dev->nr_chnls - 1;
+		break;
+	}
+
+	return nr_rows;
+}
+
+void nvm_setup_sysblk_scan(struct nvm_dev *dev, struct sysblk_scan *s,
+						struct ppa_addr *sysblk_ppas)
+{
+	memset(s, 0, sizeof(struct sysblk_scan));
+	s->nr_rows = nvm_setup_sysblks(dev, sysblk_ppas);
+}
+
+static int sysblk_get_host_blks(struct ppa_addr ppa, int nr_blks, u8 *blks,
+								void *private)
+{
+	struct sysblk_scan *s = private;
+	int i, nr_sysblk = 0;
+
+	for (i = 0; i < nr_blks; i++) {
+		if (blks[i] != NVM_BLK_T_HOST)
+			continue;
+
+		if (s->nr_ppas == MAX_BLKS_PR_SYSBLK * MAX_SYSBLKS) {
+			pr_err("nvm: too many host blks\n");
+			return -EINVAL;
+		}
+
+		ppa.g.blk = i;
+
+		s->ppas[scan_ppa_idx(s->row, nr_sysblk)] = ppa;
+		s->nr_ppas++;
+		nr_sysblk++;
+	}
+
+	return 0;
+}
+
+static int nvm_get_all_sysblks(struct nvm_dev *dev, struct sysblk_scan *s,
+				struct ppa_addr *ppas, nvm_bb_update_fn *fn)
+{
+	struct ppa_addr dppa;
+	int i, ret;
+
+	s->nr_ppas = 0;
+
+	for (i = 0; i < s->nr_rows; i++) {
+		dppa = generic_to_dev_addr(dev, ppas[i]);
+		s->row = i;
+
+		ret = dev->ops->get_bb_tbl(dev, dppa, dev->blks_per_lun, fn, s);
+		if (ret) {
+			pr_err("nvm: failed bb tbl for ppa (%u %u)\n",
+							ppas[i].g.ch,
+							ppas[i].g.blk);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * scans a block for latest sysblk.
+ * Returns:
+ *	0 - newer sysblk not found. PPA is updated to latest page.
+ *	1 - newer sysblk found and stored in *cur. PPA is updated to
+ *	    next valid page.
+ *	<0- error.
+ */
+static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
+						struct nvm_system_block *sblk)
+{
+	struct nvm_system_block *cur;
+	int pg, cursz, ret, found = 0;
+
+	/* the full buffer for a flash page is allocated. Only the first of it
+	 * contains the system block information
+	 */
+	cursz = dev->sec_size * dev->sec_per_pg * dev->nr_planes;
+	cur = kmalloc(cursz, GFP_KERNEL);
+	if (!cur)
+		return -ENOMEM;
+
+	/* perform linear scan through the block */
+	for (pg = 0; pg < dev->lps_per_blk; pg++) {
+		ppa->g.pg = ppa_to_slc(dev, pg);
+
+		ret = nvm_submit_ppa(dev, ppa, 1, NVM_OP_PREAD, NVM_IO_SLC_MODE,
+								cur, cursz);
+		if (ret) {
+			if (ret == NVM_RSP_ERR_EMPTYPAGE) {
+				pr_debug("nvm: sysblk scan empty ppa (%u %u %u %u)\n",
+							ppa->g.ch,
+							ppa->g.lun,
+							ppa->g.blk,
+							ppa->g.pg);
+				break;
+			}
+			pr_err("nvm: read failed (%x) for ppa (%u %u %u %u)",
+							ret,
+							ppa->g.ch,
+							ppa->g.lun,
+							ppa->g.blk,
+							ppa->g.pg);
+			break; /* if we can't read a page, continue to the
+				* next blk
+				*/
+		}
+
+		if (be32_to_cpu(cur->magic) != NVM_SYSBLK_MAGIC) {
+			pr_debug("nvm: scan break for ppa (%u %u %u %u)\n",
+							ppa->g.ch,
+							ppa->g.lun,
+							ppa->g.blk,
+							ppa->g.pg);
+			break; /* last valid page already found */
+		}
+
+		if (be32_to_cpu(cur->seqnr) < be32_to_cpu(sblk->seqnr))
+			continue;
+
+		memcpy(sblk, cur, sizeof(struct nvm_system_block));
+		found = 1;
+	}
+
+	kfree(cur);
+
+	return found;
+}
+
+static int nvm_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s, int type)
+{
+	struct nvm_rq rqd;
+	int ret;
+
+	if (s->nr_ppas > dev->ops->max_phys_sect) {
+		pr_err("nvm: unable to update all sysblocks atomically\n");
+		return -EINVAL;
+	}
+
+	memset(&rqd, 0, sizeof(struct nvm_rq));
+
+	nvm_set_rqd_ppalist(dev, &rqd, s->ppas, s->nr_ppas);
+	nvm_generic_to_addr_mode(dev, &rqd);
+
+	ret = dev->ops->set_bb_tbl(dev, &rqd, type);
+	nvm_free_rqd_ppalist(dev, &rqd);
+	if (ret) {
+		pr_err("nvm: sysblk failed bb mark\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int sysblk_get_free_blks(struct ppa_addr ppa, int nr_blks, u8 *blks,
+								void *private)
+{
+	struct sysblk_scan *s = private;
+	struct ppa_addr *sppa;
+	int i, blkid = 0;
+
+	for (i = 0; i < nr_blks; i++) {
+		if (blks[i] == NVM_BLK_T_HOST)
+			return -EEXIST;
+
+		if (blks[i] != NVM_BLK_T_FREE)
+			continue;
+
+		sppa = &s->ppas[scan_ppa_idx(s->row, blkid)];
+		sppa->g.ch = ppa.g.ch;
+		sppa->g.lun = ppa.g.lun;
+		sppa->g.blk = i;
+		s->nr_ppas++;
+		blkid++;
+
+		pr_debug("nvm: use (%u %u %u) as sysblk\n",
+					sppa->g.ch, sppa->g.lun, sppa->g.blk);
+		if (blkid > MAX_BLKS_PR_SYSBLK - 1)
+			return 0;
+	}
+
+	pr_err("nvm: sysblk failed get sysblk\n");
+	return -EINVAL;
+}
+
+static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
+							struct sysblk_scan *s)
+{
+	struct nvm_system_block nvmsb;
+	void *buf;
+	int i, sect, ret, bufsz;
+	struct ppa_addr *ppas;
+
+	nvm_cpu_to_sysblk(&nvmsb, info);
+
+	/* buffer for flash page */
+	bufsz = dev->sec_size * dev->sec_per_pg * dev->nr_planes;
+	buf = kzalloc(bufsz, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	memcpy(buf, &nvmsb, sizeof(struct nvm_system_block));
+
+	ppas = kcalloc(dev->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL);
+	if (!ppas) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	/* Write and verify */
+	for (i = 0; i < s->nr_rows; i++) {
+		ppas[0] = s->ppas[scan_ppa_idx(i, s->act_blk[i])];
+
+		pr_debug("nvm: writing sysblk to ppa (%u %u %u %u)\n",
+							ppas[0].g.ch,
+							ppas[0].g.lun,
+							ppas[0].g.blk,
+							ppas[0].g.pg);
+
+		/* Expand to all sectors within a flash page */
+		if (dev->sec_per_pg > 1) {
+			for (sect = 1; sect < dev->sec_per_pg; sect++) {
+				ppas[sect].ppa = ppas[0].ppa;
+				ppas[sect].g.sec = sect;
+			}
+		}
+
+		ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PWRITE,
+						NVM_IO_SLC_MODE, buf, bufsz);
+		if (ret) {
+			pr_err("nvm: sysblk failed program (%u %u %u)\n",
+							ppas[0].g.ch,
+							ppas[0].g.lun,
+							ppas[0].g.blk);
+			break;
+		}
+
+		ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PREAD,
+						NVM_IO_SLC_MODE, buf, bufsz);
+		if (ret) {
+			pr_err("nvm: sysblk failed read (%u %u %u)\n",
+							ppas[0].g.ch,
+							ppas[0].g.lun,
+							ppas[0].g.blk);
+			break;
+		}
+
+		if (memcmp(buf, &nvmsb, sizeof(struct nvm_system_block))) {
+			pr_err("nvm: sysblk failed verify (%u %u %u)\n",
+							ppas[0].g.ch,
+							ppas[0].g.lun,
+							ppas[0].g.blk);
+			ret = -EINVAL;
+			break;
+		}
+	}
+
+	kfree(ppas);
+err:
+	kfree(buf);
+
+	return ret;
+}
+
+static int nvm_prepare_new_sysblks(struct nvm_dev *dev, struct sysblk_scan *s)
+{
+	int i, ret;
+	unsigned long nxt_blk;
+	struct ppa_addr *ppa;
+
+	for (i = 0; i < s->nr_rows; i++) {
+		nxt_blk = (s->act_blk[i] + 1) % MAX_BLKS_PR_SYSBLK;
+		ppa = &s->ppas[scan_ppa_idx(i, nxt_blk)];
+		ppa->g.pg = ppa_to_slc(dev, 0);
+
+		ret = nvm_erase_ppa(dev, ppa, 1);
+		if (ret)
+			return ret;
+
+		s->act_blk[i] = nxt_blk;
+	}
+
+	return 0;
+}
+
+int nvm_get_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
+{
+	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
+	struct sysblk_scan s;
+	struct nvm_system_block *cur;
+	int i, j, found = 0;
+	int ret = -ENOMEM;
+
+	/*
+	 * 1. setup sysblk locations
+	 * 2. get bad block list
+	 * 3. filter on host-specific (type 3)
+	 * 4. iterate through all and find the highest seq nr.
+	 * 5. return superblock information
+	 */
+
+	if (!dev->ops->get_bb_tbl)
+		return -EINVAL;
+
+	nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
+
+	mutex_lock(&dev->mlock);
+	ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, sysblk_get_host_blks);
+	if (ret)
+		goto err_sysblk;
+
+	/* no sysblocks initialized */
+	if (!s.nr_ppas)
+		goto err_sysblk;
+
+	cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
+	if (!cur)
+		goto err_sysblk;
+
+	/* find the latest block across all sysblocks */
+	for (i = 0; i < s.nr_rows; i++) {
+		for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
+			struct ppa_addr ppa = s.ppas[scan_ppa_idx(i, j)];
+
+			ret = nvm_scan_block(dev, &ppa, cur);
+			if (ret > 0)
+				found = 1;
+			else if (ret < 0)
+				break;
+		}
+	}
+
+	nvm_sysblk_to_cpu(info, cur);
+
+	kfree(cur);
+err_sysblk:
+	mutex_unlock(&dev->mlock);
+
+	if (found)
+		return 1;
+	return ret;
+}
+
+int nvm_update_sysblock(struct nvm_dev *dev, struct nvm_sb_info *new)
+{
+	/* 1. for each latest superblock
+	 * 2. if room
+	 *    a. write new flash page entry with the updated information
+	 * 3. if no room
+	 *    a. find next available block on lun (linear search)
+	 *       if none, continue to next lun
+	 *       if none at all, report error. also report that it wasn't
+	 *       possible to write to all superblocks.
+	 *    c. write data to block.
+	 */
+	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
+	struct sysblk_scan s;
+	struct nvm_system_block *cur;
+	int i, j, ppaidx, found = 0;
+	int ret = -ENOMEM;
+
+	if (!dev->ops->get_bb_tbl)
+		return -EINVAL;
+
+	nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
+
+	mutex_lock(&dev->mlock);
+	ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, sysblk_get_host_blks);
+	if (ret)
+		goto err_sysblk;
+
+	cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
+	if (!cur)
+		goto err_sysblk;
+
+	/* Get the latest sysblk for each sysblk row */
+	for (i = 0; i < s.nr_rows; i++) {
+		found = 0;
+		for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
+			ppaidx = scan_ppa_idx(i, j);
+			ret = nvm_scan_block(dev, &s.ppas[ppaidx], cur);
+			if (ret > 0) {
+				s.act_blk[i] = j;
+				found = 1;
+			} else if (ret < 0)
+				break;
+		}
+	}
+
+	if (!found) {
+		pr_err("nvm: no valid sysblks found to update\n");
+		ret = -EINVAL;
+		goto err_cur;
+	}
+
+	/*
+	 * All sysblocks found. Check that they have same page id in their flash
+	 * blocks
+	 */
+	for (i = 1; i < s.nr_rows; i++) {
+		struct ppa_addr l = s.ppas[scan_ppa_idx(0, s.act_blk[0])];
+		struct ppa_addr r = s.ppas[scan_ppa_idx(i, s.act_blk[i])];
+
+		if (l.g.pg != r.g.pg) {
+			pr_err("nvm: sysblks not on same page. Previous update failed.\n");
+			ret = -EINVAL;
+			goto err_cur;
+		}
+	}
+
+	/*
+	 * Check that there haven't been another update to the seqnr since we
+	 * began
+	 */
+	if ((new->seqnr - 1) != be32_to_cpu(cur->seqnr)) {
+		pr_err("nvm: seq is not sequential\n");
+		ret = -EINVAL;
+		goto err_cur;
+	}
+
+	/*
+	 * When all pages in a block has been written, a new block is selected
+	 * and writing is performed on the new block.
+	 */
+	if (s.ppas[scan_ppa_idx(0, s.act_blk[0])].g.pg ==
+						dev->lps_per_blk - 1) {
+		ret = nvm_prepare_new_sysblks(dev, &s);
+		if (ret)
+			goto err_cur;
+	}
+
+	ret = nvm_write_and_verify(dev, new, &s);
+err_cur:
+	kfree(cur);
+err_sysblk:
+	mutex_unlock(&dev->mlock);
+
+	return ret;
+}
+
+int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
+{
+	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
+	struct sysblk_scan s;
+	int ret;
+
+	/*
+	 * 1. select master blocks and select first available blks
+	 * 2. get bad block list
+	 * 3. mark MAX_SYSBLKS block as host-based device allocated.
+	 * 4. write and verify data to block
+	 */
+
+	if (!dev->ops->get_bb_tbl || !dev->ops->set_bb_tbl)
+		return -EINVAL;
+
+	if (!(dev->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) {
+		pr_err("nvm: memory does not support SLC access\n");
+		return -EINVAL;
+	}
+
+	/* Index all sysblocks and mark them as host-driven */
+	nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
+
+	mutex_lock(&dev->mlock);
+	ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, sysblk_get_free_blks);
+	if (ret)
+		goto err_mark;
+
+	ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_HOST);
+	if (ret)
+		goto err_mark;
+
+	/* Write to the first block of each row */
+	ret = nvm_write_and_verify(dev, info, &s);
+err_mark:
+	mutex_unlock(&dev->mlock);
+	return ret;
+}
+
+struct factory_blks {
+	struct nvm_dev *dev;
+	int flags;
+	unsigned long *blks;
+};
+
+static int factory_nblks(int nblks)
+{
+	/* Round up to nearest BITS_PER_LONG */
+	return (nblks + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
+}
+
+static unsigned int factory_blk_offset(struct nvm_dev *dev, int ch, int lun)
+{
+	int nblks = factory_nblks(dev->blks_per_lun);
+
+	return ((ch * dev->luns_per_chnl * nblks) + (lun * nblks)) /
+								BITS_PER_LONG;
+}
+
+static int nvm_factory_blks(struct ppa_addr ppa, int nr_blks, u8 *blks,
+								void *private)
+{
+	struct factory_blks *f = private;
+	struct nvm_dev *dev = f->dev;
+	int i, lunoff;
+
+	lunoff = factory_blk_offset(dev, ppa.g.ch, ppa.g.lun);
+
+	/* non-set bits correspond to the block must be erased */
+	for (i = 0; i < nr_blks; i++) {
+		switch (blks[i]) {
+		case NVM_BLK_T_FREE:
+			if (f->flags & NVM_FACTORY_ERASE_ONLY_USER)
+				set_bit(i, &f->blks[lunoff]);
+			break;
+		case NVM_BLK_T_HOST:
+			if (!(f->flags & NVM_FACTORY_RESET_HOST_BLKS))
+				set_bit(i, &f->blks[lunoff]);
+			break;
+		case NVM_BLK_T_GRWN_BAD:
+			if (!(f->flags & NVM_FACTORY_RESET_GRWN_BBLKS))
+				set_bit(i, &f->blks[lunoff]);
+			break;
+		default:
+			set_bit(i, &f->blks[lunoff]);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list,
+					int max_ppas, struct factory_blks *f)
+{
+	struct ppa_addr ppa;
+	int ch, lun, blkid, idx, done = 0, ppa_cnt = 0;
+	unsigned long *offset;
+
+	while (!done) {
+		done = 1;
+		for (ch = 0; ch < dev->nr_chnls; ch++) {
+			for (lun = 0; lun < dev->luns_per_chnl; lun++) {
+				idx = factory_blk_offset(dev, ch, lun);
+				offset = &f->blks[idx];
+
+				blkid = find_first_zero_bit(offset,
+							dev->blks_per_lun);
+				if (blkid >= dev->blks_per_lun)
+					continue;
+				set_bit(blkid, offset);
+
+				ppa.ppa = 0;
+				ppa.g.ch = ch;
+				ppa.g.lun = lun;
+				ppa.g.blk = blkid;
+				pr_debug("nvm: erase ppa (%u %u %u)\n",
+								ppa.g.ch,
+								ppa.g.lun,
+								ppa.g.blk);
+
+				erase_list[ppa_cnt] = ppa;
+				ppa_cnt++;
+				done = 0;
+
+				if (ppa_cnt == max_ppas)
+					return ppa_cnt;
+			}
+		}
+	}
+
+	return ppa_cnt;
+}
+
+static int nvm_fact_get_bb_tbl(struct nvm_dev *dev, struct ppa_addr ppa,
+					nvm_bb_update_fn *fn, void *priv)
+{
+	struct ppa_addr dev_ppa;
+	int ret;
+
+	dev_ppa = generic_to_dev_addr(dev, ppa);
+
+	ret = dev->ops->get_bb_tbl(dev, dev_ppa, dev->blks_per_lun, fn, priv);
+	if (ret)
+		pr_err("nvm: failed bb tbl for ch%u lun%u\n",
+							ppa.g.ch, ppa.g.blk);
+	return ret;
+}
+
+static int nvm_fact_select_blks(struct nvm_dev *dev, struct factory_blks *f)
+{
+	int ch, lun, ret;
+	struct ppa_addr ppa;
+
+	ppa.ppa = 0;
+	for (ch = 0; ch < dev->nr_chnls; ch++) {
+		for (lun = 0; lun < dev->luns_per_chnl; lun++) {
+			ppa.g.ch = ch;
+			ppa.g.lun = lun;
+
+			ret = nvm_fact_get_bb_tbl(dev, ppa, nvm_factory_blks,
+									f);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+int nvm_dev_factory(struct nvm_dev *dev, int flags)
+{
+	struct factory_blks f;
+	struct ppa_addr *ppas;
+	int ppa_cnt, ret = -ENOMEM;
+	int max_ppas = dev->ops->max_phys_sect / dev->nr_planes;
+	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
+	struct sysblk_scan s;
+
+	f.blks = kzalloc(factory_nblks(dev->blks_per_lun) * dev->nr_luns,
+								GFP_KERNEL);
+	if (!f.blks)
+		return ret;
+
+	ppas = kcalloc(max_ppas, sizeof(struct ppa_addr), GFP_KERNEL);
+	if (!ppas)
+		goto err_blks;
+
+	f.dev = dev;
+	f.flags = flags;
+
+	/* create list of blks to be erased */
+	ret = nvm_fact_select_blks(dev, &f);
+	if (ret)
+		goto err_ppas;
+
+	/* continue to erase until list of blks until empty */
+	while ((ppa_cnt = nvm_fact_get_blks(dev, ppas, max_ppas, &f)) > 0)
+		nvm_erase_ppa(dev, ppas, ppa_cnt);
+
+	/* mark host reserved blocks free */
+	if (flags & NVM_FACTORY_RESET_HOST_BLKS) {
+		nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
+		mutex_lock(&dev->mlock);
+		ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas,
+							sysblk_get_host_blks);
+		if (!ret)
+			ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_FREE);
+		mutex_unlock(&dev->mlock);
+	}
+err_ppas:
+	kfree(ppas);
+err_blks:
+	kfree(f.blks);
+	return ret;
+}
+EXPORT_SYMBOL(nvm_dev_factory);
diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c
index 048901a1111a..caaec654d7ea 100644
--- a/drivers/macintosh/rack-meter.c
+++ b/drivers/macintosh/rack-meter.c
@@ -582,6 +582,7 @@ static struct of_device_id rackmeter_match[] = {
 	{ .name = "i2s" },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, rackmeter_match);
 
 static struct macio_driver rackmeter_driver = {
 	.driver = {
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index f9512bfa6c3c..01ee736fe0ef 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -425,8 +425,9 @@ static int __init via_pmu_start(void)
 			gpio_irq = irq_of_parse_and_map(gpio_node, 0);
 
 		if (gpio_irq != NO_IRQ) {
-			if (request_irq(gpio_irq, gpio1_interrupt, IRQF_TIMER,
-					"GPIO1 ADB", (void *)0))
+			if (request_irq(gpio_irq, gpio1_interrupt,
+					IRQF_NO_SUSPEND, "GPIO1 ADB",
+					(void *)0))
 				printk(KERN_ERR "pmu: can't get irq %d"
 				       " (GPIO1)\n", gpio_irq);
 			else
diff --git a/drivers/mailbox/mailbox-sti.c b/drivers/mailbox/mailbox-sti.c
index 4835817c5365..2394cfe892b6 100644
--- a/drivers/mailbox/mailbox-sti.c
+++ b/drivers/mailbox/mailbox-sti.c
@@ -384,7 +384,7 @@ static struct mbox_chan *sti_mbox_xlate(struct mbox_controller *mbox,
 	return chan;
 }
 
-static struct mbox_chan_ops sti_mbox_ops = {
+static const struct mbox_chan_ops sti_mbox_ops = {
 	.startup	= sti_mbox_startup_chan,
 	.shutdown	= sti_mbox_shutdown_chan,
 	.send_data	= sti_mbox_send_data,
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 83392f856dfd..22b9e34ceb75 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -1741,6 +1741,7 @@ static void bch_btree_gc(struct cache_set *c)
 	do {
 		ret = btree_root(gc_root, c, &op, &writes, &stats);
 		closure_sync(&writes);
+		cond_resched();
 
 		if (ret && ret != -EAGAIN)
 			pr_warn("gc failed!");
@@ -2162,8 +2163,10 @@ int bch_btree_insert_check_key(struct btree *b, struct btree_op *op,
 		rw_lock(true, b, b->level);
 
 		if (b->key.ptr[0] != btree_ptr ||
-		    b->seq != seq + 1)
+                   b->seq != seq + 1) {
+                       op->lock = b->level;
 			goto out;
+               }
 	}
 
 	SET_KEY_PTRS(check_key, 1);
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 679a093a3bf6..8d0ead98eb6e 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -685,6 +685,8 @@ static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
 	WARN(sysfs_create_link(&d->kobj, &c->kobj, "cache") ||
 	     sysfs_create_link(&c->kobj, &d->kobj, d->name),
 	     "Couldn't create device <-> cache set symlinks");
+
+	clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);
 }
 
 static void bcache_device_detach(struct bcache_device *d)
@@ -847,8 +849,11 @@ void bch_cached_dev_run(struct cached_dev *dc)
 	buf[SB_LABEL_SIZE] = '\0';
 	env[2] = kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf);
 
-	if (atomic_xchg(&dc->running, 1))
+	if (atomic_xchg(&dc->running, 1)) {
+		kfree(env[1]);
+		kfree(env[2]);
 		return;
+	}
 
 	if (!d->c &&
 	    BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
@@ -1933,6 +1938,8 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
 			else
 				err = "device busy";
 			mutex_unlock(&bch_register_lock);
+			if (attr == &ksysfs_register_quiet)
+				goto out;
 		}
 		goto err;
 	}
@@ -1971,8 +1978,7 @@ out:
 err_close:
 	blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
 err:
-	if (attr != &ksysfs_register_quiet)
-		pr_info("error opening %s: %s", path, err);
+	pr_info("error opening %s: %s", path, err);
 	ret = -EINVAL;
 	goto out;
 }
@@ -2066,8 +2072,10 @@ static int __init bcache_init(void)
 	closure_debug_init();
 
 	bcache_major = register_blkdev(0, "bcache");
-	if (bcache_major < 0)
+	if (bcache_major < 0) {
+		unregister_reboot_notifier(&reboot);
 		return bcache_major;
+	}
 
 	if (!(bcache_wq = create_workqueue("bcache")) ||
 	    !(bcache_kobj = kobject_create_and_add("bcache", fs_kobj)) ||
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index b23f88d9f18c..b9346cd9cda1 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -323,6 +323,10 @@ void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
 
 static bool dirty_pred(struct keybuf *buf, struct bkey *k)
 {
+	struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);
+
+	BUG_ON(KEY_INODE(k) != dc->disk.id);
+
 	return KEY_DIRTY(k);
 }
 
@@ -372,11 +376,24 @@ next:
 	}
 }
 
+/*
+ * Returns true if we scanned the entire disk
+ */
 static bool refill_dirty(struct cached_dev *dc)
 {
 	struct keybuf *buf = &dc->writeback_keys;
+	struct bkey start = KEY(dc->disk.id, 0, 0);
 	struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
-	bool searched_from_start = false;
+	struct bkey start_pos;
+
+	/*
+	 * make sure keybuf pos is inside the range for this disk - at bringup
+	 * we might not be attached yet so this disk's inode nr isn't
+	 * initialized then
+	 */
+	if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
+	    bkey_cmp(&buf->last_scanned, &end) > 0)
+		buf->last_scanned = start;
 
 	if (dc->partial_stripes_expensive) {
 		refill_full_stripes(dc);
@@ -384,14 +401,20 @@ static bool refill_dirty(struct cached_dev *dc)
 			return false;
 	}
 
-	if (bkey_cmp(&buf->last_scanned, &end) >= 0) {
-		buf->last_scanned = KEY(dc->disk.id, 0, 0);
-		searched_from_start = true;
-	}
-
+	start_pos = buf->last_scanned;
 	bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
 
-	return bkey_cmp(&buf->last_scanned, &end) >= 0 && searched_from_start;
+	if (bkey_cmp(&buf->last_scanned, &end) < 0)
+		return false;
+
+	/*
+	 * If we get to the end start scanning again from the beginning, and
+	 * only scan up to where we initially started scanning from:
+	 */
+	buf->last_scanned = start;
+	bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);
+
+	return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
 }
 
 static int bch_writeback_thread(void *arg)
diff --git a/drivers/md/bcache/writeback.h b/drivers/md/bcache/writeback.h
index 0a9dab187b79..073a042aed24 100644
--- a/drivers/md/bcache/writeback.h
+++ b/drivers/md/bcache/writeback.h
@@ -63,7 +63,8 @@ static inline bool should_writeback(struct cached_dev *dc, struct bio *bio,
 
 static inline void bch_writeback_queue(struct cached_dev *dc)
 {
-	wake_up_process(dc->writeback_thread);
+	if (!IS_ERR_OR_NULL(dc->writeback_thread))
+		wake_up_process(dc->writeback_thread);
 }
 
 static inline void bch_writeback_add(struct cached_dev *dc)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index d6a1126d85ce..0ded8e97751d 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -48,13 +48,29 @@ struct resync_info {
 #define		MD_CLUSTER_SUSPEND_READ_BALANCING	2
 #define		MD_CLUSTER_BEGIN_JOIN_CLUSTER		3
 
+/* Lock the send communication. This is done through
+ * bit manipulation as opposed to a mutex in order to
+ * accomodate lock and hold. See next comment.
+ */
+#define		MD_CLUSTER_SEND_LOCK			4
+/* If cluster operations (such as adding a disk) must lock the
+ * communication channel, so as to perform extra operations
+ * (update metadata) and no other operation is allowed on the
+ * MD. Token needs to be locked and held until the operation
+ * completes witha md_update_sb(), which would eventually release
+ * the lock.
+ */
+#define		MD_CLUSTER_SEND_LOCKED_ALREADY		5
+
 
 struct md_cluster_info {
 	/* dlm lock space and resources for clustered raid. */
 	dlm_lockspace_t *lockspace;
 	int slot_number;
 	struct completion completion;
+	struct mutex recv_mutex;
 	struct dlm_lock_resource *bitmap_lockres;
+	struct dlm_lock_resource **other_bitmap_lockres;
 	struct dlm_lock_resource *resync_lockres;
 	struct list_head suspend_list;
 	spinlock_t suspend_lock;
@@ -67,6 +83,7 @@ struct md_cluster_info {
 	struct dlm_lock_resource *no_new_dev_lockres;
 	struct md_thread *recv_thread;
 	struct completion newdisk_completion;
+	wait_queue_head_t wait;
 	unsigned long state;
 };
 
@@ -431,8 +448,10 @@ static void process_add_new_disk(struct mddev *mddev, struct cluster_msg *cmsg)
 static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
-	md_reload_sb(mddev, le32_to_cpu(msg->raid_slot));
+	mddev->good_device_nr = le32_to_cpu(msg->raid_slot);
+	set_bit(MD_RELOAD_SB, &mddev->flags);
 	dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
+	md_wakeup_thread(mddev->thread);
 }
 
 static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
@@ -440,8 +459,11 @@ static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
 	struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev,
 						   le32_to_cpu(msg->raid_slot));
 
-	if (rdev)
-		md_kick_rdev_from_array(rdev);
+	if (rdev) {
+		set_bit(ClusterRemove, &rdev->flags);
+		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+		md_wakeup_thread(mddev->thread);
+	}
 	else
 		pr_warn("%s: %d Could not find disk(%d) to REMOVE\n",
 			__func__, __LINE__, le32_to_cpu(msg->raid_slot));
@@ -502,9 +524,11 @@ static void recv_daemon(struct md_thread *thread)
 	struct cluster_msg msg;
 	int ret;
 
+	mutex_lock(&cinfo->recv_mutex);
 	/*get CR on Message*/
 	if (dlm_lock_sync(message_lockres, DLM_LOCK_CR)) {
 		pr_err("md/raid1:failed to get CR on MESSAGE\n");
+		mutex_unlock(&cinfo->recv_mutex);
 		return;
 	}
 
@@ -528,33 +552,45 @@ static void recv_daemon(struct md_thread *thread)
 	ret = dlm_unlock_sync(message_lockres);
 	if (unlikely(ret != 0))
 		pr_info("unlock msg failed return %d\n", ret);
+	mutex_unlock(&cinfo->recv_mutex);
 }
 
-/* lock_comm()
+/* lock_token()
  * Takes the lock on the TOKEN lock resource so no other
  * node can communicate while the operation is underway.
- * If called again, and the TOKEN lock is alread in EX mode
- * return success. However, care must be taken that unlock_comm()
- * is called only once.
  */
-static int lock_comm(struct md_cluster_info *cinfo)
+static int lock_token(struct md_cluster_info *cinfo)
 {
 	int error;
 
-	if (cinfo->token_lockres->mode == DLM_LOCK_EX)
-		return 0;
-
 	error = dlm_lock_sync(cinfo->token_lockres, DLM_LOCK_EX);
 	if (error)
 		pr_err("md-cluster(%s:%d): failed to get EX on TOKEN (%d)\n",
 				__func__, __LINE__, error);
+
+	/* Lock the receive sequence */
+	mutex_lock(&cinfo->recv_mutex);
 	return error;
 }
 
+/* lock_comm()
+ * Sets the MD_CLUSTER_SEND_LOCK bit to lock the send channel.
+ */
+static int lock_comm(struct md_cluster_info *cinfo)
+{
+	wait_event(cinfo->wait,
+		   !test_and_set_bit(MD_CLUSTER_SEND_LOCK, &cinfo->state));
+
+	return lock_token(cinfo);
+}
+
 static void unlock_comm(struct md_cluster_info *cinfo)
 {
 	WARN_ON(cinfo->token_lockres->mode != DLM_LOCK_EX);
+	mutex_unlock(&cinfo->recv_mutex);
 	dlm_unlock_sync(cinfo->token_lockres);
+	clear_bit(MD_CLUSTER_SEND_LOCK, &cinfo->state);
+	wake_up(&cinfo->wait);
 }
 
 /* __sendmsg()
@@ -707,6 +743,8 @@ static int join(struct mddev *mddev, int nodes)
 	spin_lock_init(&cinfo->suspend_lock);
 	init_completion(&cinfo->completion);
 	set_bit(MD_CLUSTER_BEGIN_JOIN_CLUSTER, &cinfo->state);
+	init_waitqueue_head(&cinfo->wait);
+	mutex_init(&cinfo->recv_mutex);
 
 	mddev->cluster_info = cinfo;
 
@@ -800,6 +838,7 @@ static void resync_bitmap(struct mddev *mddev)
 			__func__, __LINE__, err);
 }
 
+static void unlock_all_bitmaps(struct mddev *mddev);
 static int leave(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
@@ -820,6 +859,7 @@ static int leave(struct mddev *mddev)
 	lockres_free(cinfo->ack_lockres);
 	lockres_free(cinfo->no_new_dev_lockres);
 	lockres_free(cinfo->bitmap_lockres);
+	unlock_all_bitmaps(mddev);
 	dlm_release_lockspace(cinfo->lockspace, 2);
 	return 0;
 }
@@ -835,9 +875,25 @@ static int slot_number(struct mddev *mddev)
 	return cinfo->slot_number - 1;
 }
 
+/*
+ * Check if the communication is already locked, else lock the communication
+ * channel.
+ * If it is already locked, token is in EX mode, and hence lock_token()
+ * should not be called.
+ */
 static int metadata_update_start(struct mddev *mddev)
 {
-	return lock_comm(mddev->cluster_info);
+	struct md_cluster_info *cinfo = mddev->cluster_info;
+
+	wait_event(cinfo->wait,
+		   !test_and_set_bit(MD_CLUSTER_SEND_LOCK, &cinfo->state) ||
+		   test_and_clear_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state));
+
+	/* If token is already locked, return 0 */
+	if (cinfo->token_lockres->mode == DLM_LOCK_EX)
+		return 0;
+
+	return lock_token(cinfo);
 }
 
 static int metadata_update_finish(struct mddev *mddev)
@@ -862,6 +918,7 @@ static int metadata_update_finish(struct mddev *mddev)
 		ret = __sendmsg(cinfo, &cmsg);
 	} else
 		pr_warn("md-cluster: No good device id found to send\n");
+	clear_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state);
 	unlock_comm(cinfo);
 	return ret;
 }
@@ -869,6 +926,7 @@ static int metadata_update_finish(struct mddev *mddev)
 static void metadata_update_cancel(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
+	clear_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state);
 	unlock_comm(cinfo);
 }
 
@@ -882,8 +940,16 @@ static int resync_start(struct mddev *mddev)
 static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
+	struct resync_info ri;
 	struct cluster_msg cmsg = {0};
 
+	/* do not send zero again, if we have sent before */
+	if (hi == 0) {
+		memcpy(&ri, cinfo->bitmap_lockres->lksb.sb_lvbptr, sizeof(struct resync_info));
+		if (le64_to_cpu(ri.hi) == 0)
+			return 0;
+	}
+
 	add_resync_info(cinfo->bitmap_lockres, lo, hi);
 	/* Re-acquire the lock to refresh LVB */
 	dlm_lock_sync(cinfo->bitmap_lockres, DLM_LOCK_PW);
@@ -954,14 +1020,30 @@ static int add_new_disk(struct mddev *mddev, struct md_rdev *rdev)
 		ret = -ENOENT;
 	if (ret)
 		unlock_comm(cinfo);
-	else
+	else {
 		dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
+		/* Since MD_CHANGE_DEVS will be set in add_bound_rdev which
+		 * will run soon after add_new_disk, the below path will be
+		 * invoked:
+		 *   md_wakeup_thread(mddev->thread)
+		 *	-> conf->thread (raid1d)
+		 *	-> md_check_recovery -> md_update_sb
+		 *	-> metadata_update_start/finish
+		 * MD_CLUSTER_SEND_LOCKED_ALREADY will be cleared eventually.
+		 *
+		 * For other failure cases, metadata_update_cancel and
+		 * add_new_disk_cancel also clear below bit as well.
+		 * */
+		set_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state);
+		wake_up(&cinfo->wait);
+	}
 	return ret;
 }
 
 static void add_new_disk_cancel(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
+	clear_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state);
 	unlock_comm(cinfo);
 }
 
@@ -986,7 +1068,59 @@ static int remove_disk(struct mddev *mddev, struct md_rdev *rdev)
 	struct md_cluster_info *cinfo = mddev->cluster_info;
 	cmsg.type = cpu_to_le32(REMOVE);
 	cmsg.raid_slot = cpu_to_le32(rdev->desc_nr);
-	return __sendmsg(cinfo, &cmsg);
+	return sendmsg(cinfo, &cmsg);
+}
+
+static int lock_all_bitmaps(struct mddev *mddev)
+{
+	int slot, my_slot, ret, held = 1, i = 0;
+	char str[64];
+	struct md_cluster_info *cinfo = mddev->cluster_info;
+
+	cinfo->other_bitmap_lockres = kzalloc((mddev->bitmap_info.nodes - 1) *
+					     sizeof(struct dlm_lock_resource *),
+					     GFP_KERNEL);
+	if (!cinfo->other_bitmap_lockres) {
+		pr_err("md: can't alloc mem for other bitmap locks\n");
+		return 0;
+	}
+
+	my_slot = slot_number(mddev);
+	for (slot = 0; slot < mddev->bitmap_info.nodes; slot++) {
+		if (slot == my_slot)
+			continue;
+
+		memset(str, '\0', 64);
+		snprintf(str, 64, "bitmap%04d", slot);
+		cinfo->other_bitmap_lockres[i] = lockres_init(mddev, str, NULL, 1);
+		if (!cinfo->other_bitmap_lockres[i])
+			return -ENOMEM;
+
+		cinfo->other_bitmap_lockres[i]->flags |= DLM_LKF_NOQUEUE;
+		ret = dlm_lock_sync(cinfo->other_bitmap_lockres[i], DLM_LOCK_PW);
+		if (ret)
+			held = -1;
+		i++;
+	}
+
+	return held;
+}
+
+static void unlock_all_bitmaps(struct mddev *mddev)
+{
+	struct md_cluster_info *cinfo = mddev->cluster_info;
+	int i;
+
+	/* release other node's bitmap lock if they are existed */
+	if (cinfo->other_bitmap_lockres) {
+		for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) {
+			if (cinfo->other_bitmap_lockres[i]) {
+				dlm_unlock_sync(cinfo->other_bitmap_lockres[i]);
+				lockres_free(cinfo->other_bitmap_lockres[i]);
+			}
+		}
+		kfree(cinfo->other_bitmap_lockres);
+	}
 }
 
 static int gather_bitmaps(struct md_rdev *rdev)
@@ -1034,6 +1168,8 @@ static struct md_cluster_operations cluster_ops = {
 	.new_disk_ack = new_disk_ack,
 	.remove_disk = remove_disk,
 	.gather_bitmaps = gather_bitmaps,
+	.lock_all_bitmaps = lock_all_bitmaps,
+	.unlock_all_bitmaps = unlock_all_bitmaps,
 };
 
 static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index e75ea2613184..45ce6c97d8bd 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -24,6 +24,8 @@ struct md_cluster_operations {
 	int (*new_disk_ack)(struct mddev *mddev, bool ack);
 	int (*remove_disk)(struct mddev *mddev, struct md_rdev *rdev);
 	int (*gather_bitmaps)(struct md_rdev *rdev);
+	int (*lock_all_bitmaps)(struct mddev *mddev);
+	void (*unlock_all_bitmaps)(struct mddev *mddev);
 };
 
 #endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 31b595479aa5..e55e6cf9ec17 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -206,15 +206,6 @@ void md_new_event(struct mddev *mddev)
 }
 EXPORT_SYMBOL_GPL(md_new_event);
 
-/* Alternate version that can be called from interrupts
- * when calling sysfs_notify isn't needed.
- */
-static void md_new_event_inintr(struct mddev *mddev)
-{
-	atomic_inc(&md_event_count);
-	wake_up(&md_event_waiters);
-}
-
 /*
  * Enables to iterate over all existing md arrays
  * all_mddevs_lock protects this list.
@@ -260,8 +251,7 @@ static blk_qc_t md_make_request(struct request_queue *q, struct bio *bio)
 
 	blk_queue_split(q, &bio, q->bio_split);
 
-	if (mddev == NULL || mddev->pers == NULL
-	    || !mddev->ready) {
+	if (mddev == NULL || mddev->pers == NULL) {
 		bio_io_error(bio);
 		return BLK_QC_T_NONE;
 	}
@@ -1026,8 +1016,9 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
 	 * (not needed for Linear and RAID0 as metadata doesn't
 	 * record this size)
 	 */
-	if (rdev->sectors >= (2ULL << 32) && sb->level >= 1)
-		rdev->sectors = (2ULL << 32) - 2;
+	if (IS_ENABLED(CONFIG_LBDAF) && (u64)rdev->sectors >= (2ULL << 32) &&
+	    sb->level >= 1)
+		rdev->sectors = (sector_t)(2ULL << 32) - 2;
 
 	if (rdev->sectors < ((sector_t)sb->size) * 2 && sb->level >= 1)
 		/* "this cannot possibly happen" ... */
@@ -1199,13 +1190,13 @@ static void super_90_sync(struct mddev *mddev, struct md_rdev *rdev)
 	memcpy(&sb->set_uuid2, mddev->uuid+8, 4);
 	memcpy(&sb->set_uuid3, mddev->uuid+12,4);
 
-	sb->ctime = mddev->ctime;
+	sb->ctime = clamp_t(time64_t, mddev->ctime, 0, U32_MAX);
 	sb->level = mddev->level;
 	sb->size = mddev->dev_sectors / 2;
 	sb->raid_disks = mddev->raid_disks;
 	sb->md_minor = mddev->md_minor;
 	sb->not_persistent = 0;
-	sb->utime = mddev->utime;
+	sb->utime = clamp_t(time64_t, mddev->utime, 0, U32_MAX);
 	sb->state = 0;
 	sb->events_hi = (mddev->events>>32);
 	sb->events_lo = (u32)mddev->events;
@@ -1320,8 +1311,9 @@ super_90_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
 	/* Limit to 4TB as metadata cannot record more than that.
 	 * 4TB == 2^32 KB, or 2*2^32 sectors.
 	 */
-	if (num_sectors >= (2ULL << 32) && rdev->mddev->level >= 1)
-		num_sectors = (2ULL << 32) - 2;
+	if (IS_ENABLED(CONFIG_LBDAF) && (u64)num_sectors >= (2ULL << 32) &&
+	    rdev->mddev->level >= 1)
+		num_sectors = (sector_t)(2ULL << 32) - 2;
 	md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
 		       rdev->sb_page);
 	md_super_wait(rdev->mddev);
@@ -1542,8 +1534,8 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 		mddev->patch_version = 0;
 		mddev->external = 0;
 		mddev->chunk_sectors = le32_to_cpu(sb->chunksize);
-		mddev->ctime = le64_to_cpu(sb->ctime) & ((1ULL << 32)-1);
-		mddev->utime = le64_to_cpu(sb->utime) & ((1ULL << 32)-1);
+		mddev->ctime = le64_to_cpu(sb->ctime);
+		mddev->utime = le64_to_cpu(sb->utime);
 		mddev->level = le32_to_cpu(sb->level);
 		mddev->clevel[0] = 0;
 		mddev->layout = le32_to_cpu(sb->layout);
@@ -1602,6 +1594,11 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 			mddev->new_chunk_sectors = mddev->chunk_sectors;
 		}
 
+		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL) {
+			set_bit(MD_HAS_JOURNAL, &mddev->flags);
+			if (mddev->recovery_cp == MaxSector)
+				set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
+		}
 	} else if (mddev->pers == NULL) {
 		/* Insist of good event counter while assembling, except for
 		 * spares (which don't need an event count) */
@@ -1648,8 +1645,6 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 			}
 			set_bit(Journal, &rdev->flags);
 			rdev->journal_tail = le64_to_cpu(sb->journal_tail);
-			if (mddev->recovery_cp == MaxSector)
-				set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
 			rdev->raid_disk = 0;
 			break;
 		default:
@@ -1669,8 +1664,6 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 			set_bit(WriteMostly, &rdev->flags);
 		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT)
 			set_bit(Replacement, &rdev->flags);
-		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)
-			set_bit(MD_HAS_JOURNAL, &mddev->flags);
 	} else /* MULTIPATH are always insync */
 		set_bit(In_sync, &rdev->flags);
 
@@ -2014,28 +2007,32 @@ int md_integrity_register(struct mddev *mddev)
 }
 EXPORT_SYMBOL(md_integrity_register);
 
-/* Disable data integrity if non-capable/non-matching disk is being added */
-void md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev)
+/*
+ * Attempt to add an rdev, but only if it is consistent with the current
+ * integrity profile
+ */
+int md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev)
 {
 	struct blk_integrity *bi_rdev;
 	struct blk_integrity *bi_mddev;
+	char name[BDEVNAME_SIZE];
 
 	if (!mddev->gendisk)
-		return;
+		return 0;
 
 	bi_rdev = bdev_get_integrity(rdev->bdev);
 	bi_mddev = blk_get_integrity(mddev->gendisk);
 
 	if (!bi_mddev) /* nothing to do */
-		return;
-	if (rdev->raid_disk < 0) /* skip spares */
-		return;
-	if (bi_rdev && blk_integrity_compare(mddev->gendisk,
-					     rdev->bdev->bd_disk) >= 0)
-		return;
-	WARN_ON_ONCE(!mddev->suspended);
-	printk(KERN_NOTICE "disabling data integrity on %s\n", mdname(mddev));
-	blk_integrity_unregister(mddev->gendisk);
+		return 0;
+
+	if (blk_integrity_compare(mddev->gendisk, rdev->bdev->bd_disk) != 0) {
+		printk(KERN_NOTICE "%s: incompatible integrity profile for %s\n",
+				mdname(mddev), bdevname(rdev->bdev, name));
+		return -ENXIO;
+	}
+
+	return 0;
 }
 EXPORT_SYMBOL(md_integrity_add_rdev);
 
@@ -2050,8 +2047,9 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
 		return -EEXIST;
 
 	/* make sure rdev->sectors exceeds mddev->dev_sectors */
-	if (rdev->sectors && (mddev->dev_sectors == 0 ||
-			rdev->sectors < mddev->dev_sectors)) {
+	if (!test_bit(Journal, &rdev->flags) &&
+	    rdev->sectors &&
+	    (mddev->dev_sectors == 0 || rdev->sectors < mddev->dev_sectors)) {
 		if (mddev->pers) {
 			/* Cannot change size, so fail
 			 * If mddev->level <= 0, then we don't care
@@ -2082,7 +2080,8 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
 		}
 	}
 	rcu_read_unlock();
-	if (mddev->max_disks && rdev->desc_nr >= mddev->max_disks) {
+	if (!test_bit(Journal, &rdev->flags) &&
+	    mddev->max_disks && rdev->desc_nr >= mddev->max_disks) {
 		printk(KERN_WARNING "md: %s: array is limited to %d devices\n",
 		       mdname(mddev), mddev->max_disks);
 		return -EBUSY;
@@ -2331,7 +2330,7 @@ repeat:
 
 	spin_lock(&mddev->lock);
 
-	mddev->utime = get_seconds();
+	mddev->utime = ktime_get_real_seconds();
 
 	if (test_and_clear_bit(MD_CHANGE_DEVS, &mddev->flags))
 		force_change = 1;
@@ -2457,15 +2456,20 @@ static int add_bound_rdev(struct md_rdev *rdev)
 {
 	struct mddev *mddev = rdev->mddev;
 	int err = 0;
+	bool add_journal = test_bit(Journal, &rdev->flags);
 
-	if (!mddev->pers->hot_remove_disk) {
+	if (!mddev->pers->hot_remove_disk || add_journal) {
 		/* If there is hot_add_disk but no hot_remove_disk
 		 * then added disks for geometry changes,
 		 * and should be added immediately.
 		 */
 		super_types[mddev->major_version].
 			validate_super(mddev, rdev);
+		if (add_journal)
+			mddev_suspend(mddev);
 		err = mddev->pers->hot_add_disk(mddev, rdev);
+		if (add_journal)
+			mddev_resume(mddev);
 		if (err) {
 			unbind_rdev_from_array(rdev);
 			export_rdev(rdev);
@@ -5299,7 +5303,6 @@ int md_run(struct mddev *mddev)
 	smp_wmb();
 	spin_lock(&mddev->lock);
 	mddev->pers = pers;
-	mddev->ready = 1;
 	spin_unlock(&mddev->lock);
 	rdev_for_each(rdev, mddev)
 		if (rdev->raid_disk >= 0)
@@ -5499,7 +5502,6 @@ static void __md_stop(struct mddev *mddev)
 	/* Ensure ->event_work is done */
 	flush_workqueue(md_misc_wq);
 	spin_lock(&mddev->lock);
-	mddev->ready = 0;
 	mddev->pers = NULL;
 	spin_unlock(&mddev->lock);
 	pers->free(mddev, mddev->private);
@@ -5837,7 +5839,7 @@ static int get_array_info(struct mddev *mddev, void __user *arg)
 	info.major_version = mddev->major_version;
 	info.minor_version = mddev->minor_version;
 	info.patch_version = MD_PATCHLEVEL_VERSION;
-	info.ctime         = mddev->ctime;
+	info.ctime         = clamp_t(time64_t, mddev->ctime, 0, U32_MAX);
 	info.level         = mddev->level;
 	info.size          = mddev->dev_sectors / 2;
 	if (info.size != mddev->dev_sectors / 2) /* overflow */
@@ -5847,7 +5849,7 @@ static int get_array_info(struct mddev *mddev, void __user *arg)
 	info.md_minor      = mddev->md_minor;
 	info.not_persistent= !mddev->persistent;
 
-	info.utime         = mddev->utime;
+	info.utime         = clamp_t(time64_t, mddev->utime, 0, U32_MAX);
 	info.state         = 0;
 	if (mddev->in_sync)
 		info.state = (1<<MD_SB_CLEAN);
@@ -6038,8 +6040,23 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 		else
 			clear_bit(WriteMostly, &rdev->flags);
 
-		if (info->state & (1<<MD_DISK_JOURNAL))
+		if (info->state & (1<<MD_DISK_JOURNAL)) {
+			struct md_rdev *rdev2;
+			bool has_journal = false;
+
+			/* make sure no existing journal disk */
+			rdev_for_each(rdev2, mddev) {
+				if (test_bit(Journal, &rdev2->flags)) {
+					has_journal = true;
+					break;
+				}
+			}
+			if (has_journal) {
+				export_rdev(rdev);
+				return -EBUSY;
+			}
 			set_bit(Journal, &rdev->flags);
+		}
 		/*
 		 * check whether the device shows up in other nodes
 		 */
@@ -6130,15 +6147,11 @@ static int hot_remove_disk(struct mddev *mddev, dev_t dev)
 {
 	char b[BDEVNAME_SIZE];
 	struct md_rdev *rdev;
-	int ret = -1;
 
 	rdev = find_rdev(mddev, dev);
 	if (!rdev)
 		return -ENXIO;
 
-	if (mddev_is_clustered(mddev))
-		ret = md_cluster_ops->metadata_update_start(mddev);
-
 	if (rdev->raid_disk < 0)
 		goto kick_rdev;
 
@@ -6149,7 +6162,7 @@ static int hot_remove_disk(struct mddev *mddev, dev_t dev)
 		goto busy;
 
 kick_rdev:
-	if (mddev_is_clustered(mddev) && ret == 0)
+	if (mddev_is_clustered(mddev))
 		md_cluster_ops->remove_disk(mddev, rdev);
 
 	md_kick_rdev_from_array(rdev);
@@ -6158,9 +6171,6 @@ kick_rdev:
 
 	return 0;
 busy:
-	if (mddev_is_clustered(mddev) && ret == 0)
-		md_cluster_ops->metadata_update_cancel(mddev);
-
 	printk(KERN_WARNING "md: cannot remove active disk %s from %s ...\n",
 		bdevname(rdev->bdev,b), mdname(mddev));
 	return -EBUSY;
@@ -6354,13 +6364,13 @@ static int set_array_info(struct mddev *mddev, mdu_array_info_t *info)
 		/* ensure mddev_put doesn't delete this now that there
 		 * is some minimal configuration.
 		 */
-		mddev->ctime         = get_seconds();
+		mddev->ctime         = ktime_get_real_seconds();
 		return 0;
 	}
 	mddev->major_version = MD_MAJOR_VERSION;
 	mddev->minor_version = MD_MINOR_VERSION;
 	mddev->patch_version = MD_PATCHLEVEL_VERSION;
-	mddev->ctime         = get_seconds();
+	mddev->ctime         = ktime_get_real_seconds();
 
 	mddev->level         = info->level;
 	mddev->clevel[0]     = 0;
@@ -6602,6 +6612,19 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
 				rv = -EINVAL;
 				goto err;
 			}
+			if (mddev->bitmap_info.nodes) {
+				/* hold PW on all the bitmap lock */
+				if (md_cluster_ops->lock_all_bitmaps(mddev) <= 0) {
+					printk("md: can't change bitmap to none since the"
+					       " array is in use by more than one node\n");
+					rv = -EPERM;
+					md_cluster_ops->unlock_all_bitmaps(mddev);
+					goto err;
+				}
+
+				mddev->bitmap_info.nodes = 0;
+				md_cluster_ops->leave(mddev);
+			}
 			mddev->pers->quiesce(mddev, 1);
 			bitmap_destroy(mddev);
 			mddev->pers->quiesce(mddev, 0);
@@ -7180,7 +7203,7 @@ void md_error(struct mddev *mddev, struct md_rdev *rdev)
 	md_wakeup_thread(mddev->thread);
 	if (mddev->event_work.func)
 		queue_work(md_misc_wq, &mddev->event_work);
-	md_new_event_inintr(mddev);
+	md_new_event(mddev);
 }
 EXPORT_SYMBOL(md_error);
 
@@ -7704,7 +7727,7 @@ EXPORT_SYMBOL(md_write_end);
  * attempting a GFP_KERNEL allocation while holding the mddev lock.
  * Must be called with mddev_lock held.
  *
- * In the ->external case MD_CHANGE_CLEAN can not be cleared until mddev->lock
+ * In the ->external case MD_CHANGE_PENDING can not be cleared until mddev->lock
  * is dropped, so return -EAGAIN after notifying userspace.
  */
 int md_allow_write(struct mddev *mddev)
@@ -8169,19 +8192,20 @@ static int remove_and_add_spares(struct mddev *mddev,
 			continue;
 		if (test_bit(Faulty, &rdev->flags))
 			continue;
-		if (test_bit(Journal, &rdev->flags))
-			continue;
-		if (mddev->ro &&
-		    ! (rdev->saved_raid_disk >= 0 &&
-		       !test_bit(Bitmap_sync, &rdev->flags)))
-			continue;
+		if (!test_bit(Journal, &rdev->flags)) {
+			if (mddev->ro &&
+			    ! (rdev->saved_raid_disk >= 0 &&
+			       !test_bit(Bitmap_sync, &rdev->flags)))
+				continue;
 
-		rdev->recovery_offset = 0;
+			rdev->recovery_offset = 0;
+		}
 		if (mddev->pers->
 		    hot_add_disk(mddev, rdev) == 0) {
 			if (sysfs_link_rdev(mddev, rdev))
 				/* failure here is OK */;
-			spares++;
+			if (!test_bit(Journal, &rdev->flags))
+				spares++;
 			md_new_event(mddev);
 			set_bit(MD_CHANGE_DEVS, &mddev->flags);
 		}
@@ -8276,6 +8300,7 @@ void md_check_recovery(struct mddev *mddev)
 		(mddev->flags & MD_UPDATE_SB_FLAGS & ~ (1<<MD_CHANGE_PENDING)) ||
 		test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) ||
 		test_bit(MD_RECOVERY_DONE, &mddev->recovery) ||
+		test_bit(MD_RELOAD_SB, &mddev->flags) ||
 		(mddev->external == 0 && mddev->safemode == 1) ||
 		(mddev->safemode == 2 && ! atomic_read(&mddev->writes_pending)
 		 && !mddev->in_sync && mddev->recovery_cp == MaxSector)
@@ -8314,6 +8339,21 @@ void md_check_recovery(struct mddev *mddev)
 			goto unlock;
 		}
 
+		if (mddev_is_clustered(mddev)) {
+			struct md_rdev *rdev;
+			/* kick the device if another node issued a
+			 * remove disk.
+			 */
+			rdev_for_each(rdev, mddev) {
+				if (test_and_clear_bit(ClusterRemove, &rdev->flags) &&
+						rdev->raid_disk < 0)
+					md_kick_rdev_from_array(rdev);
+			}
+
+			if (test_and_clear_bit(MD_RELOAD_SB, &mddev->flags))
+				md_reload_sb(mddev, mddev->good_device_nr);
+		}
+
 		if (!mddev->external) {
 			int did_change = 0;
 			spin_lock(&mddev->lock);
@@ -8635,7 +8675,6 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev)
 				ret = remove_and_add_spares(mddev, rdev2);
 				pr_info("Activated spare: %s\n",
 						bdevname(rdev2->bdev,b));
-				continue;
 			}
 			/* device faulty
 			 * We just want to do the minimum to mark the disk
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 75b9aaacb03f..b5c4be73e6e4 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -162,6 +162,7 @@ enum flag_bits {
 				 * Usually, this device should be faster
 				 * than other devices in the array
 				 */
+	ClusterRemove,
 };
 
 static inline int is_badblock(struct md_rdev *rdev, sector_t s, int sectors,
@@ -200,6 +201,9 @@ struct mddev {
 				 */
 #define MD_JOURNAL_CLEAN 5	/* A raid with journal is already clean */
 #define MD_HAS_JOURNAL	6	/* The raid array has journal feature set */
+#define MD_RELOAD_SB	7	/* Reload the superblock because another node
+				 * updated it.
+				 */
 
 	int				suspended;
 	atomic_t			active_io;
@@ -208,8 +212,6 @@ struct mddev {
 						       * are happening, so run/
 						       * takeover/stop are not safe
 						       */
-	int				ready; /* See when safe to pass
-						* IO requests down */
 	struct gendisk			*gendisk;
 
 	struct kobject			kobj;
@@ -226,7 +228,7 @@ struct mddev {
 							 * managed externally */
 	char				metadata_type[17]; /* externally set*/
 	int				chunk_sectors;
-	time_t				ctime, utime;
+	time64_t			ctime, utime;
 	int				level, layout;
 	char				clevel[16];
 	int				raid_disks;
@@ -430,6 +432,7 @@ struct mddev {
 	struct work_struct event_work;	/* used by dm to report failure event */
 	void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);
 	struct md_cluster_info		*cluster_info;
+	unsigned int			good_device_nr;	/* good device num within cluster raid */
 };
 
 static inline int __must_check mddev_lock(struct mddev *mddev)
@@ -623,7 +626,7 @@ extern void md_wait_for_blocked_rdev(struct md_rdev *rdev, struct mddev *mddev);
 extern void md_set_array_sectors(struct mddev *mddev, sector_t array_sectors);
 extern int md_check_no_bitmap(struct mddev *mddev);
 extern int md_integrity_register(struct mddev *mddev);
-extern void md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev);
+extern int md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev);
 extern int strict_strtoul_scaled(const char *cp, unsigned long *res, int scale);
 
 extern void mddev_init(struct mddev *mddev);
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index 7331a80d89f1..0a72ab6e6c20 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -257,6 +257,9 @@ static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 			disk_stack_limits(mddev->gendisk, rdev->bdev,
 					  rdev->data_offset << 9);
 
+			err = md_integrity_add_rdev(rdev, mddev);
+			if (err)
+				break;
 			spin_lock_irq(&conf->device_lock);
 			mddev->degraded--;
 			rdev->raid_disk = path;
@@ -264,9 +267,6 @@ static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 			spin_unlock_irq(&conf->device_lock);
 			rcu_assign_pointer(p->rdev, rdev);
 			err = 0;
-			mddev_suspend(mddev);
-			md_integrity_add_rdev(rdev, mddev);
-			mddev_resume(mddev);
 			break;
 		}
 
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index f8e5db0cb5aa..2ea12c6bf659 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -549,13 +549,13 @@ static void *raid0_takeover_raid10(struct mddev *mddev)
 	 *  - all mirrors must be already degraded
 	 */
 	if (mddev->layout != ((1 << 8) + 2)) {
-		printk(KERN_ERR "md/raid0:%s:: Raid0 cannot takover layout: 0x%x\n",
+		printk(KERN_ERR "md/raid0:%s:: Raid0 cannot takeover layout: 0x%x\n",
 		       mdname(mddev),
 		       mddev->layout);
 		return ERR_PTR(-EINVAL);
 	}
 	if (mddev->raid_disks & 1) {
-		printk(KERN_ERR "md/raid0:%s: Raid0 cannot takover Raid10 with odd disk number.\n",
+		printk(KERN_ERR "md/raid0:%s: Raid0 cannot takeover Raid10 with odd disk number.\n",
 		       mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index e2169ff6e0f0..c4b913409226 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1589,6 +1589,9 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 	if (mddev->recovery_disabled == conf->recovery_disabled)
 		return -EBUSY;
 
+	if (md_integrity_add_rdev(rdev, mddev))
+		return -ENXIO;
+
 	if (rdev->raid_disk >= 0)
 		first = last = rdev->raid_disk;
 
@@ -1632,9 +1635,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 			break;
 		}
 	}
-	mddev_suspend(mddev);
-	md_integrity_add_rdev(rdev, mddev);
-	mddev_resume(mddev);
 	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
 		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
 	print_conf(conf);
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 84e597e1c489..ce959b4ae4df 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1698,6 +1698,9 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
 		return -EINVAL;
 
+	if (md_integrity_add_rdev(rdev, mddev))
+		return -ENXIO;
+
 	if (rdev->raid_disk >= 0)
 		first = last = rdev->raid_disk;
 
@@ -1739,9 +1742,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 		rcu_assign_pointer(p->rdev, rdev);
 		break;
 	}
-	mddev_suspend(mddev);
-	md_integrity_add_rdev(rdev, mddev);
-	mddev_resume(mddev);
 	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
 		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
 
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index b887e04d7e5c..9531f5f05b93 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -34,6 +34,12 @@
 #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
 #define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
 
+/*
+ * We only need 2 bios per I/O unit to make progress, but ensure we
+ * have a few more available to not get too tight.
+ */
+#define R5L_POOL_SIZE	4
+
 struct r5l_log {
 	struct md_rdev *rdev;
 
@@ -69,7 +75,12 @@ struct r5l_log {
 	struct list_head finished_ios;	/* io_units which settle down in log disk */
 	struct bio flush_bio;
 
+	struct list_head no_mem_stripes;   /* pending stripes, -ENOMEM */
+
 	struct kmem_cache *io_kc;
+	mempool_t *io_pool;
+	struct bio_set *bs;
+	mempool_t *meta_pool;
 
 	struct md_thread *reclaim_thread;
 	unsigned long reclaim_target;	/* number of space that need to be
@@ -150,27 +161,6 @@ static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
 	return log->device_size > used_size + size;
 }
 
-static void r5l_free_io_unit(struct r5l_log *log, struct r5l_io_unit *io)
-{
-	__free_page(io->meta_page);
-	kmem_cache_free(log->io_kc, io);
-}
-
-static void r5l_move_io_unit_list(struct list_head *from, struct list_head *to,
-				  enum r5l_io_unit_state state)
-{
-	struct r5l_io_unit *io;
-
-	while (!list_empty(from)) {
-		io = list_first_entry(from, struct r5l_io_unit, log_sibling);
-		/* don't change list order */
-		if (io->state >= state)
-			list_move_tail(&io->log_sibling, to);
-		else
-			break;
-	}
-}
-
 static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
 				    enum r5l_io_unit_state state)
 {
@@ -206,6 +196,20 @@ static void r5l_log_run_stripes(struct r5l_log *log)
 	}
 }
 
+static void r5l_move_to_end_ios(struct r5l_log *log)
+{
+	struct r5l_io_unit *io, *next;
+
+	assert_spin_locked(&log->io_list_lock);
+
+	list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
+		/* don't change list order */
+		if (io->state < IO_UNIT_IO_END)
+			break;
+		list_move_tail(&io->log_sibling, &log->io_end_ios);
+	}
+}
+
 static void r5l_log_endio(struct bio *bio)
 {
 	struct r5l_io_unit *io = bio->bi_private;
@@ -216,12 +220,12 @@ static void r5l_log_endio(struct bio *bio)
 		md_error(log->rdev->mddev, log->rdev);
 
 	bio_put(bio);
+	mempool_free(io->meta_page, log->meta_pool);
 
 	spin_lock_irqsave(&log->io_list_lock, flags);
 	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
 	if (log->need_cache_flush)
-		r5l_move_io_unit_list(&log->running_ios, &log->io_end_ios,
-				      IO_UNIT_IO_END);
+		r5l_move_to_end_ios(log);
 	else
 		r5l_log_run_stripes(log);
 	spin_unlock_irqrestore(&log->io_list_lock, flags);
@@ -255,7 +259,7 @@ static void r5l_submit_current_io(struct r5l_log *log)
 
 static struct bio *r5l_bio_alloc(struct r5l_log *log)
 {
-	struct bio *bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL, BIO_MAX_PAGES);
+	struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
 
 	bio->bi_rw = WRITE;
 	bio->bi_bdev = log->rdev->bdev;
@@ -286,15 +290,19 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 	struct r5l_io_unit *io;
 	struct r5l_meta_block *block;
 
-	/* We can't handle memory allocate failure so far */
-	io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);
+	io = mempool_alloc(log->io_pool, GFP_ATOMIC);
+	if (!io)
+		return NULL;
+	memset(io, 0, sizeof(*io));
+
 	io->log = log;
 	INIT_LIST_HEAD(&io->log_sibling);
 	INIT_LIST_HEAD(&io->stripe_list);
 	io->state = IO_UNIT_RUNNING;
 
-	io->meta_page = alloc_page(GFP_NOIO | __GFP_NOFAIL | __GFP_ZERO);
+	io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
 	block = page_address(io->meta_page);
+	clear_page(block);
 	block->magic = cpu_to_le32(R5LOG_MAGIC);
 	block->version = R5LOG_VERSION;
 	block->seq = cpu_to_le64(log->seq);
@@ -324,8 +332,12 @@ static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
 	    log->current_io->meta_offset + payload_size > PAGE_SIZE)
 		r5l_submit_current_io(log);
 
-	if (!log->current_io)
+	if (!log->current_io) {
 		log->current_io = r5l_new_meta(log);
+		if (!log->current_io)
+			return -ENOMEM;
+	}
+
 	return 0;
 }
 
@@ -370,11 +382,12 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
 	r5_reserve_log_entry(log, io);
 }
 
-static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
+static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 			   int data_pages, int parity_pages)
 {
 	int i;
 	int meta_size;
+	int ret;
 	struct r5l_io_unit *io;
 
 	meta_size =
@@ -383,7 +396,10 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 		sizeof(struct r5l_payload_data_parity) +
 		sizeof(__le32) * parity_pages;
 
-	r5l_get_meta(log, meta_size);
+	ret = r5l_get_meta(log, meta_size);
+	if (ret)
+		return ret;
+
 	io = log->current_io;
 
 	for (i = 0; i < sh->disks; i++) {
@@ -413,6 +429,8 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 	list_add_tail(&sh->log_list, &io->stripe_list);
 	atomic_inc(&io->pending_stripe);
 	sh->log_io = io;
+
+	return 0;
 }
 
 static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
@@ -427,6 +445,7 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 	int meta_size;
 	int reserve;
 	int i;
+	int ret = 0;
 
 	if (!log)
 		return -EAGAIN;
@@ -475,17 +494,22 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 	mutex_lock(&log->io_mutex);
 	/* meta + data */
 	reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
-	if (r5l_has_free_space(log, reserve))
-		r5l_log_stripe(log, sh, data_pages, parity_pages);
-	else {
+	if (!r5l_has_free_space(log, reserve)) {
 		spin_lock(&log->no_space_stripes_lock);
 		list_add_tail(&sh->log_list, &log->no_space_stripes);
 		spin_unlock(&log->no_space_stripes_lock);
 
 		r5l_wake_reclaim(log, reserve);
+	} else {
+		ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
+		if (ret) {
+			spin_lock_irq(&log->io_list_lock);
+			list_add_tail(&sh->log_list, &log->no_mem_stripes);
+			spin_unlock_irq(&log->io_list_lock);
+		}
 	}
-	mutex_unlock(&log->io_mutex);
 
+	mutex_unlock(&log->io_mutex);
 	return 0;
 }
 
@@ -538,6 +562,21 @@ static sector_t r5l_reclaimable_space(struct r5l_log *log)
 				 log->next_checkpoint);
 }
 
+static void r5l_run_no_mem_stripe(struct r5l_log *log)
+{
+	struct stripe_head *sh;
+
+	assert_spin_locked(&log->io_list_lock);
+
+	if (!list_empty(&log->no_mem_stripes)) {
+		sh = list_first_entry(&log->no_mem_stripes,
+				      struct stripe_head, log_list);
+		list_del_init(&sh->log_list);
+		set_bit(STRIPE_HANDLE, &sh->state);
+		raid5_release_stripe(sh);
+	}
+}
+
 static bool r5l_complete_finished_ios(struct r5l_log *log)
 {
 	struct r5l_io_unit *io, *next;
@@ -554,7 +593,8 @@ static bool r5l_complete_finished_ios(struct r5l_log *log)
 		log->next_cp_seq = io->seq;
 
 		list_del(&io->log_sibling);
-		r5l_free_io_unit(log, io);
+		mempool_free(io, log->io_pool);
+		r5l_run_no_mem_stripe(log);
 
 		found = true;
 	}
@@ -787,6 +827,13 @@ void r5l_quiesce(struct r5l_log *log, int state)
 		return;
 	if (state == 0) {
 		log->in_teardown = 0;
+		/*
+		 * This is a special case for hotadd. In suspend, the array has
+		 * no journal. In resume, journal is initialized as well as the
+		 * reclaim thread.
+		 */
+		if (log->reclaim_thread)
+			return;
 		log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
 					log->rdev->mddev, "reclaim");
 	} else if (state == 1) {
@@ -806,10 +853,18 @@ void r5l_quiesce(struct r5l_log *log, int state)
 
 bool r5l_log_disk_error(struct r5conf *conf)
 {
+	struct r5l_log *log;
+	bool ret;
 	/* don't allow write if journal disk is missing */
-	if (!conf->log)
-		return test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
-	return test_bit(Faulty, &conf->log->rdev->flags);
+	rcu_read_lock();
+	log = rcu_dereference(conf->log);
+
+	if (!log)
+		ret = test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
+	else
+		ret = test_bit(Faulty, &log->rdev->flags);
+	rcu_read_unlock();
+	return ret;
 }
 
 struct r5l_recovery_ctx {
@@ -1160,23 +1215,45 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	if (!log->io_kc)
 		goto io_kc;
 
+	log->io_pool = mempool_create_slab_pool(R5L_POOL_SIZE, log->io_kc);
+	if (!log->io_pool)
+		goto io_pool;
+
+	log->bs = bioset_create(R5L_POOL_SIZE, 0);
+	if (!log->bs)
+		goto io_bs;
+
+	log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0);
+	if (!log->meta_pool)
+		goto out_mempool;
+
 	log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
 						 log->rdev->mddev, "reclaim");
 	if (!log->reclaim_thread)
 		goto reclaim_thread;
 	init_waitqueue_head(&log->iounit_wait);
 
+	INIT_LIST_HEAD(&log->no_mem_stripes);
+
 	INIT_LIST_HEAD(&log->no_space_stripes);
 	spin_lock_init(&log->no_space_stripes_lock);
 
 	if (r5l_load_log(log))
 		goto error;
 
-	conf->log = log;
+	rcu_assign_pointer(conf->log, log);
+	set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
 	return 0;
+
 error:
 	md_unregister_thread(&log->reclaim_thread);
 reclaim_thread:
+	mempool_destroy(log->meta_pool);
+out_mempool:
+	bioset_free(log->bs);
+io_bs:
+	mempool_destroy(log->io_pool);
+io_pool:
 	kmem_cache_destroy(log->io_kc);
 io_kc:
 	kfree(log);
@@ -1186,6 +1263,9 @@ io_kc:
 void r5l_exit_log(struct r5l_log *log)
 {
 	md_unregister_thread(&log->reclaim_thread);
+	mempool_destroy(log->meta_pool);
+	bioset_free(log->bs);
+	mempool_destroy(log->io_pool);
 	kmem_cache_destroy(log->io_kc);
 	kfree(log);
 }
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 704ef7fcfbf8..a086014dcd49 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -772,8 +772,6 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh
 	int hash;
 	int dd_idx;
 
-	if (!stripe_can_batch(sh))
-		return;
 	/* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
 	tmp_sec = sh->sector;
 	if (!sector_div(tmp_sec, conf->chunk_sectors))
@@ -7141,14 +7139,19 @@ static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
 	struct disk_info *p = conf->disks + number;
 
 	print_raid5_conf(conf);
-	if (test_bit(Journal, &rdev->flags)) {
+	if (test_bit(Journal, &rdev->flags) && conf->log) {
+		struct r5l_log *log;
 		/*
-		 * journal disk is not removable, but we need give a chance to
-		 * update superblock of other disks. Otherwise journal disk
-		 * will be considered as 'fresh'
+		 * we can't wait pending write here, as this is called in
+		 * raid5d, wait will deadlock.
 		 */
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
-		return -EINVAL;
+		if (atomic_read(&mddev->writes_pending))
+			return -EBUSY;
+		log = conf->log;
+		conf->log = NULL;
+		synchronize_rcu();
+		r5l_exit_log(log);
+		return 0;
 	}
 	if (rdev == p->rdev)
 		rdevp = &p->rdev;
@@ -7212,8 +7215,21 @@ static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 	int first = 0;
 	int last = conf->raid_disks - 1;
 
-	if (test_bit(Journal, &rdev->flags))
-		return -EINVAL;
+	if (test_bit(Journal, &rdev->flags)) {
+		char b[BDEVNAME_SIZE];
+		if (conf->log)
+			return -EBUSY;
+
+		rdev->raid_disk = 0;
+		/*
+		 * The array is in readonly mode if journal is missing, so no
+		 * write requests running. We should be safe
+		 */
+		r5l_init_log(conf, rdev);
+		printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
+		       mdname(mddev), bdevname(rdev->bdev, b));
+		return 0;
+	}
 	if (mddev->recovery_disabled == conf->recovery_disabled)
 		return -EBUSY;
 
diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c
index d5837be3e8cf..ea2f3bf7368b 100644
--- a/drivers/media/common/saa7146/saa7146_video.c
+++ b/drivers/media/common/saa7146/saa7146_video.c
@@ -502,7 +502,7 @@ static int vidioc_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuf
 	/* check if overlay is running */
 	if (IS_OVERLAY_ACTIVE(fh) != 0) {
 		if (vv->video_fh != fh) {
-			DEB_D("refusing to change framebuffer informations while overlay is active in another open\n");
+			DEB_D("refusing to change framebuffer information while overlay is active in another open\n");
 			return -EBUSY;
 		}
 	}
diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c
index def6d21d1445..24a457d9d803 100644
--- a/drivers/media/dvb-frontends/cxd2820r_core.c
+++ b/drivers/media/dvb-frontends/cxd2820r_core.c
@@ -722,7 +722,7 @@ struct dvb_frontend *cxd2820r_attach(const struct cxd2820r_config *cfg,
 #ifdef CONFIG_GPIOLIB
 		/* add GPIOs */
 		priv->gpio_chip.label = KBUILD_MODNAME;
-		priv->gpio_chip.dev = &priv->i2c->dev;
+		priv->gpio_chip.parent = &priv->i2c->dev;
 		priv->gpio_chip.owner = THIS_MODULE;
 		priv->gpio_chip.direction_output =
 				cxd2820r_gpio_direction_output;
diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c
index feeeb70d841e..ce73a5ec6036 100644
--- a/drivers/media/dvb-frontends/m88ds3103.c
+++ b/drivers/media/dvb-frontends/m88ds3103.c
@@ -685,7 +685,7 @@ static int m88ds3103_init(struct dvb_frontend *fe)
 	/* request the firmware, this will block and timeout */
 	ret = request_firmware(&fw, fw_file, &client->dev);
 	if (ret) {
-		dev_err(&client->dev, "firmare file '%s' not found\n", fw_file);
+		dev_err(&client->dev, "firmware file '%s' not found\n", fw_file);
 		goto err;
 	}
 
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index 1cf6e52e0105..2b93241d4bc1 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -541,7 +541,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 					   &offset, block_count);
 	if (ret < 0) {
 		dev_err(&state->i2c->dev,
-			"%s: firmare could not be uploaded\n",
+			"%s: firmware could not be uploaded\n",
 			KBUILD_MODNAME);
 		goto error;
 	}
@@ -565,7 +565,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 
 	if (len != offset) {
 		dev_err(&state->i2c->dev,
-			"%s: firmare len mismatch %04x != %04x\n",
+			"%s: firmware len mismatch %04x != %04x\n",
 			KBUILD_MODNAME, len, offset);
 		ret = -EINVAL;
 		goto error;
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
index 81e1a5e26efb..525ebfefeee8 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
@@ -387,7 +387,7 @@ static int netup_unidvb_dvb_init(struct netup_unidvb_dev *ndev,
 		vb2_dvb_alloc_frontend(
 			&ndev->frontends[num], 3) == NULL) {
 		dev_dbg(&ndev->pci_dev->dev,
-			"%s(): unable to to alllocate vb2_dvb_frontend\n",
+			"%s(): unable to allocate vb2_dvb_frontend\n",
 			__func__);
 		return -ENOMEM;
 	}
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index 0c53805dff0e..526359447ff9 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -216,7 +216,6 @@ config VIDEO_STI_BDISP
 	tristate "STMicroelectronics BDISP 2D blitter driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_STI || COMPILE_TEST
-	depends on HAVE_DMA_ATTRS
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	help
diff --git a/drivers/media/platform/omap/Kconfig b/drivers/media/platform/omap/Kconfig
index 217d613b0fe7..e8e2db181a7a 100644
--- a/drivers/media/platform/omap/Kconfig
+++ b/drivers/media/platform/omap/Kconfig
@@ -5,9 +5,9 @@ config VIDEO_OMAP2_VOUT
 	tristate "OMAP2/OMAP3 V4L2-Display driver"
 	depends on MMU
 	depends on ARCH_OMAP2 || ARCH_OMAP3
+	depends on FB_OMAP2
 	select VIDEOBUF_GEN
 	select VIDEOBUF_DMA_CONTIG
-	select OMAP2_DSS if HAS_IOMEM && ARCH_OMAP2PLUS
 	select OMAP2_VRFB if ARCH_OMAP2 || ARCH_OMAP3
 	select VIDEO_OMAP2_VOUT_VRFB if VIDEO_OMAP2_VOUT && OMAP2_VRFB
 	select FRAME_VECTOR
diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c
index 21e7255e3d96..5a58e440f4a7 100644
--- a/drivers/memory/tegra/tegra124.c
+++ b/drivers/memory/tegra/tegra124.c
@@ -1007,6 +1007,7 @@ static const struct tegra_smmu_soc tegra124_smmu_soc = {
 	.num_swgroups = ARRAY_SIZE(tegra124_swgroups),
 	.supports_round_robin_arbitration = true,
 	.supports_request_limit = true,
+	.num_tlb_lines = 32,
 	.num_asids = 128,
 };
 
diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
index 24f2f8473dee..84abf9d3c24e 100644
--- a/drivers/memstick/core/ms_block.c
+++ b/drivers/memstick/core/ms_block.c
@@ -1909,7 +1909,7 @@ static void msb_io_work(struct work_struct *work)
 		lba = blk_rq_pos(msb->req);
 
 		sector_div(lba, msb->page_size / 512);
-		page = do_div(lba, msb->pages_in_block);
+		page = sector_div(lba, msb->pages_in_block);
 
 		if (rq_data_dir(msb->req) == READ)
 			error = msb_do_read_request(msb, lba, page, sg,
diff --git a/drivers/mfd/dm355evm_msp.c b/drivers/mfd/dm355evm_msp.c
index bf3e0b21b247..ec4438ed2faf 100644
--- a/drivers/mfd/dm355evm_msp.c
+++ b/drivers/mfd/dm355evm_msp.c
@@ -259,7 +259,7 @@ static int add_children(struct i2c_client *client)
 	int		i;
 
 	/* GPIO-ish stuff */
-	dm355evm_msp_gpio.dev = &client->dev;
+	dm355evm_msp_gpio.parent = &client->dev;
 	status = gpiochip_add(&dm355evm_msp_gpio);
 	if (status < 0)
 		return status;
diff --git a/drivers/mfd/htc-egpio.c b/drivers/mfd/htc-egpio.c
index e4f4a31b76d9..c636b5f83cfb 100644
--- a/drivers/mfd/htc-egpio.c
+++ b/drivers/mfd/htc-egpio.c
@@ -321,7 +321,7 @@ static int __init egpio_probe(struct platform_device *pdev)
 		ei->chip[i].dev = &(pdev->dev);
 		chip = &(ei->chip[i].chip);
 		chip->label           = "htc-egpio";
-		chip->dev             = &pdev->dev;
+		chip->parent          = &pdev->dev;
 		chip->owner           = THIS_MODULE;
 		chip->get             = egpio_get;
 		chip->set             = egpio_set;
diff --git a/drivers/mfd/htc-i2cpld.c b/drivers/mfd/htc-i2cpld.c
index 0c6ff727b2ec..bd6b96d07ab8 100644
--- a/drivers/mfd/htc-i2cpld.c
+++ b/drivers/mfd/htc-i2cpld.c
@@ -429,7 +429,7 @@ static int htcpld_register_chip_gpio(
 	/* Setup the GPIO chips */
 	gpio_chip = &(chip->chip_out);
 	gpio_chip->label           = "htcpld-out";
-	gpio_chip->dev             = dev;
+	gpio_chip->parent             = dev;
 	gpio_chip->owner           = THIS_MODULE;
 	gpio_chip->get             = htcpld_chip_get;
 	gpio_chip->set             = htcpld_chip_set;
@@ -440,7 +440,7 @@ static int htcpld_register_chip_gpio(
 
 	gpio_chip = &(chip->chip_in);
 	gpio_chip->label           = "htcpld-in";
-	gpio_chip->dev             = dev;
+	gpio_chip->parent             = dev;
 	gpio_chip->owner           = THIS_MODULE;
 	gpio_chip->get             = htcpld_chip_get;
 	gpio_chip->set             = NULL;
diff --git a/drivers/mfd/tps65010.c b/drivers/mfd/tps65010.c
index 677a127619d4..83e615ed100a 100644
--- a/drivers/mfd/tps65010.c
+++ b/drivers/mfd/tps65010.c
@@ -638,7 +638,7 @@ static int tps65010_probe(struct i2c_client *client,
 		tps->outmask = board->outmask;
 
 		tps->chip.label = client->name;
-		tps->chip.dev = &client->dev;
+		tps->chip.parent = &client->dev;
 		tps->chip.owner = THIS_MODULE;
 
 		tps->chip.set = tps65010_gpio_set;
diff --git a/drivers/mfd/ucb1x00-core.c b/drivers/mfd/ucb1x00-core.c
index e0dd83fb95d3..bcafe1ecd71c 100644
--- a/drivers/mfd/ucb1x00-core.c
+++ b/drivers/mfd/ucb1x00-core.c
@@ -570,7 +570,7 @@ static int ucb1x00_probe(struct mcp *mcp)
 
 	if (pdata && pdata->gpio_base) {
 		ucb->gpio.label = dev_name(&ucb->dev);
-		ucb->gpio.dev = &ucb->dev;
+		ucb->gpio.parent = &ucb->dev;
 		ucb->gpio.owner = THIS_MODULE;
 		ucb->gpio.base = pdata->gpio_base;
 		ucb->gpio.ngpio = 10;
diff --git a/drivers/mfd/vexpress-sysreg.c b/drivers/mfd/vexpress-sysreg.c
index 3e628df9280c..855c0204f09a 100644
--- a/drivers/mfd/vexpress-sysreg.c
+++ b/drivers/mfd/vexpress-sysreg.c
@@ -11,7 +11,7 @@
  * Copyright (C) 2012 ARM Limited
  */
 
-#include <linux/basic_mmio_gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/mfd/core.h>
@@ -164,7 +164,7 @@ static int vexpress_sysreg_probe(struct platform_device *pdev)
 {
 	struct resource *mem;
 	void __iomem *base;
-	struct bgpio_chip *mmc_gpio_chip;
+	struct gpio_chip *mmc_gpio_chip;
 	int master;
 	u32 dt_hbi;
 
@@ -201,8 +201,8 @@ static int vexpress_sysreg_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	bgpio_init(mmc_gpio_chip, &pdev->dev, 0x4, base + SYS_MCI,
 			NULL, NULL, NULL, NULL, 0);
-	mmc_gpio_chip->gc.ngpio = 2;
-	gpiochip_add(&mmc_gpio_chip->gc);
+	mmc_gpio_chip->ngpio = 2;
+	gpiochip_add(mmc_gpio_chip);
 
 	return mfd_add_devices(&pdev->dev, PLATFORM_DEVID_AUTO,
 			vexpress_sysreg_cells,
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 22892c701c63..054fc10cb3b6 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -95,6 +95,7 @@ config DUMMY_IRQ
 config IBM_ASM
 	tristate "Device driver for IBM RSA service processor"
 	depends on X86 && PCI && INPUT
+	depends on SERIAL_8250 || SERIAL_8250=n
 	---help---
 	  This option enables device driver support for in-band access to the
 	  IBM RSA (Condor) service processor in eServer xSeries systems.
diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile
index 6982f603fadc..be2ac5ce349f 100644
--- a/drivers/misc/cxl/Makefile
+++ b/drivers/misc/cxl/Makefile
@@ -1,4 +1,5 @@
-ccflags-y := -Werror -Wno-unused-const-variable
+ccflags-y			:= $(call cc-disable-warning, unused-const-variable)
+ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
 
 cxl-y				+= main.o file.o irq.o fault.o native.o
 cxl-y				+= context.o sysfs.o debugfs.o pci.o trace.o
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 103baf0e0c5b..ea3eeb7011e1 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -25,7 +25,6 @@ struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
 
 	afu = cxl_pci_to_afu(dev);
 
-	get_device(&afu->dev);
 	ctx = cxl_context_alloc();
 	if (IS_ERR(ctx)) {
 		rc = PTR_ERR(ctx);
@@ -61,7 +60,6 @@ err_mapping:
 err_ctx:
 	kfree(ctx);
 err_dev:
-	put_device(&afu->dev);
 	return ERR_PTR(rc);
 }
 EXPORT_SYMBOL_GPL(cxl_dev_context_init);
@@ -87,8 +85,6 @@ int cxl_release_context(struct cxl_context *ctx)
 	if (ctx->status >= STARTED)
 		return -EBUSY;
 
-	put_device(&ctx->afu->dev);
-
 	cxl_context_free(ctx);
 
 	return 0;
@@ -176,7 +172,7 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
 
 	if (task) {
 		ctx->pid = get_task_pid(task, PIDTYPE_PID);
-		get_pid(ctx->pid);
+		ctx->glpid = get_task_pid(task->group_leader, PIDTYPE_PID);
 		kernel = false;
 	}
 
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 2faa1270d085..262b88eac414 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -42,7 +42,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
 	spin_lock_init(&ctx->sste_lock);
 	ctx->afu = afu;
 	ctx->master = master;
-	ctx->pid = NULL; /* Set in start work ioctl */
+	ctx->pid = ctx->glpid = NULL; /* Set in start work ioctl */
 	mutex_init(&ctx->mapping_lock);
 	ctx->mapping = mapping;
 
@@ -97,6 +97,12 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
 	ctx->pe = i;
 	ctx->elem = &ctx->afu->spa[i];
 	ctx->pe_inserted = false;
+
+	/*
+	 * take a ref on the afu so that it stays alive at-least till
+	 * this context is reclaimed inside reclaim_ctx.
+	 */
+	cxl_afu_get(afu);
 	return 0;
 }
 
@@ -211,7 +217,11 @@ int __detach_context(struct cxl_context *ctx)
 	WARN_ON(cxl_detach_process(ctx) &&
 		cxl_adapter_link_ok(ctx->afu->adapter));
 	flush_work(&ctx->fault_work); /* Only needed for dedicated process */
+
+	/* release the reference to the group leader and mm handling pid */
 	put_pid(ctx->pid);
+	put_pid(ctx->glpid);
+
 	cxl_ctx_put();
 	return 0;
 }
@@ -278,6 +288,9 @@ static void reclaim_ctx(struct rcu_head *rcu)
 	if (ctx->irq_bitmap)
 		kfree(ctx->irq_bitmap);
 
+	/* Drop ref to the afu device taken during cxl_context_init */
+	cxl_afu_put(ctx->afu);
+
 	kfree(ctx);
 }
 
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index 0cfb9c129f27..a521bc72cec2 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -403,6 +403,18 @@ struct cxl_afu {
 	bool enabled;
 };
 
+/* AFU refcount management */
+static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu)
+{
+
+	return (get_device(&afu->dev) == NULL) ? NULL : afu;
+}
+
+static inline void  cxl_afu_put(struct cxl_afu *afu)
+{
+	put_device(&afu->dev);
+}
+
 
 struct cxl_irq_name {
 	struct list_head list;
@@ -433,6 +445,9 @@ struct cxl_context {
 	unsigned int sst_size, sst_lru;
 
 	wait_queue_head_t wq;
+	/* pid of the group leader associated with the pid */
+	struct pid *glpid;
+	/* use mm context associated with this pid for ds faults */
 	struct pid *pid;
 	spinlock_t lock; /* Protects pending_irq_mask, pending_fault and fault_addr */
 	/* Only used in PR mode */
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c
index 25a5418c55cb..81c3f75b7330 100644
--- a/drivers/misc/cxl/fault.c
+++ b/drivers/misc/cxl/fault.c
@@ -166,13 +166,92 @@ static void cxl_handle_page_fault(struct cxl_context *ctx,
 	cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
 }
 
+/*
+ * Returns the mm_struct corresponding to the context ctx via ctx->pid
+ * In case the task has exited we use the task group leader accessible
+ * via ctx->glpid to find the next task in the thread group that has a
+ * valid  mm_struct associated with it. If a task with valid mm_struct
+ * is found the ctx->pid is updated to use the task struct for subsequent
+ * translations. In case no valid mm_struct is found in the task group to
+ * service the fault a NULL is returned.
+ */
+static struct mm_struct *get_mem_context(struct cxl_context *ctx)
+{
+	struct task_struct *task = NULL;
+	struct mm_struct *mm = NULL;
+	struct pid *old_pid = ctx->pid;
+
+	if (old_pid == NULL) {
+		pr_warn("%s: Invalid context for pe=%d\n",
+			 __func__, ctx->pe);
+		return NULL;
+	}
+
+	task = get_pid_task(old_pid, PIDTYPE_PID);
+
+	/*
+	 * pid_alive may look racy but this saves us from costly
+	 * get_task_mm when the task is a zombie. In worst case
+	 * we may think a task is alive, which is about to die
+	 * but get_task_mm will return NULL.
+	 */
+	if (task != NULL && pid_alive(task))
+		mm = get_task_mm(task);
+
+	/* release the task struct that was taken earlier */
+	if (task)
+		put_task_struct(task);
+	else
+		pr_devel("%s: Context owning pid=%i for pe=%i dead\n",
+			__func__, pid_nr(old_pid), ctx->pe);
+
+	/*
+	 * If we couldn't find the mm context then use the group
+	 * leader to iterate over the task group and find a task
+	 * that gives us mm_struct.
+	 */
+	if (unlikely(mm == NULL && ctx->glpid != NULL)) {
+
+		rcu_read_lock();
+		task = pid_task(ctx->glpid, PIDTYPE_PID);
+		if (task)
+			do {
+				mm = get_task_mm(task);
+				if (mm) {
+					ctx->pid = get_task_pid(task,
+								PIDTYPE_PID);
+					break;
+				}
+				task = next_thread(task);
+			} while (task && !thread_group_leader(task));
+		rcu_read_unlock();
+
+		/* check if we switched pid */
+		if (ctx->pid != old_pid) {
+			if (mm)
+				pr_devel("%s:pe=%i switch pid %i->%i\n",
+					 __func__, ctx->pe, pid_nr(old_pid),
+					 pid_nr(ctx->pid));
+			else
+				pr_devel("%s:Cannot find mm for pid=%i\n",
+					 __func__, pid_nr(old_pid));
+
+			/* drop the reference to older pid */
+			put_pid(old_pid);
+		}
+	}
+
+	return mm;
+}
+
+
+
 void cxl_handle_fault(struct work_struct *fault_work)
 {
 	struct cxl_context *ctx =
 		container_of(fault_work, struct cxl_context, fault_work);
 	u64 dsisr = ctx->dsisr;
 	u64 dar = ctx->dar;
-	struct task_struct *task = NULL;
 	struct mm_struct *mm = NULL;
 
 	if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||
@@ -195,17 +274,17 @@ void cxl_handle_fault(struct work_struct *fault_work)
 		"DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);
 
 	if (!ctx->kernel) {
-		if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
-			pr_devel("cxl_handle_fault unable to get task %i\n",
-				 pid_nr(ctx->pid));
+
+		mm = get_mem_context(ctx);
+		/* indicates all the thread in task group have exited */
+		if (mm == NULL) {
+			pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
+				 __func__, ctx->pe, pid_nr(ctx->pid));
 			cxl_ack_ae(ctx);
 			return;
-		}
-		if (!(mm = get_task_mm(task))) {
-			pr_devel("cxl_handle_fault unable to get mm %i\n",
-				 pid_nr(ctx->pid));
-			cxl_ack_ae(ctx);
-			goto out;
+		} else {
+			pr_devel("Handling page fault for pe=%d pid=%i\n",
+				 ctx->pe, pid_nr(ctx->pid));
 		}
 	}
 
@@ -218,33 +297,22 @@ void cxl_handle_fault(struct work_struct *fault_work)
 
 	if (mm)
 		mmput(mm);
-out:
-	if (task)
-		put_task_struct(task);
 }
 
 static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)
 {
-	int rc;
-	struct task_struct *task;
 	struct mm_struct *mm;
 
-	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
-		pr_devel("cxl_prefault_one unable to get task %i\n",
-			 pid_nr(ctx->pid));
-		return;
-	}
-	if (!(mm = get_task_mm(task))) {
+	mm = get_mem_context(ctx);
+	if (mm == NULL) {
 		pr_devel("cxl_prefault_one unable to get mm %i\n",
 			 pid_nr(ctx->pid));
-		put_task_struct(task);
 		return;
 	}
 
-	rc = cxl_fault_segment(ctx, mm, ea);
+	cxl_fault_segment(ctx, mm, ea);
 
 	mmput(mm);
-	put_task_struct(task);
 }
 
 static u64 next_segment(u64 ea, u64 vsid)
@@ -263,18 +331,13 @@ static void cxl_prefault_vma(struct cxl_context *ctx)
 	struct copro_slb slb;
 	struct vm_area_struct *vma;
 	int rc;
-	struct task_struct *task;
 	struct mm_struct *mm;
 
-	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
-		pr_devel("cxl_prefault_vma unable to get task %i\n",
-			 pid_nr(ctx->pid));
-		return;
-	}
-	if (!(mm = get_task_mm(task))) {
+	mm = get_mem_context(ctx);
+	if (mm == NULL) {
 		pr_devel("cxl_prefault_vm unable to get mm %i\n",
 			 pid_nr(ctx->pid));
-		goto out1;
+		return;
 	}
 
 	down_read(&mm->mmap_sem);
@@ -295,8 +358,6 @@ static void cxl_prefault_vma(struct cxl_context *ctx)
 	up_read(&mm->mmap_sem);
 
 	mmput(mm);
-out1:
-	put_task_struct(task);
 }
 
 void cxl_prefault(struct cxl_context *ctx, u64 wed)
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index 7ccd2998be92..783337d22f36 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -67,7 +67,13 @@ static int __afu_open(struct inode *inode, struct file *file, bool master)
 		spin_unlock(&adapter->afu_list_lock);
 		goto err_put_adapter;
 	}
-	get_device(&afu->dev);
+
+	/*
+	 * taking a ref to the afu so that it doesn't go away
+	 * for rest of the function. This ref is released before
+	 * we return.
+	 */
+	cxl_afu_get(afu);
 	spin_unlock(&adapter->afu_list_lock);
 
 	if (!afu->current_mode)
@@ -90,13 +96,12 @@ static int __afu_open(struct inode *inode, struct file *file, bool master)
 	file->private_data = ctx;
 	cxl_ctx_get();
 
-	/* Our ref on the AFU will now hold the adapter */
-	put_device(&adapter->dev);
-
-	return 0;
+	/* indicate success */
+	rc = 0;
 
 err_put_afu:
-	put_device(&afu->dev);
+	/* release the ref taken earlier */
+	cxl_afu_put(afu);
 err_put_adapter:
 	put_device(&adapter->dev);
 	return rc;
@@ -131,8 +136,6 @@ int afu_release(struct inode *inode, struct file *file)
 		mutex_unlock(&ctx->mapping_lock);
 	}
 
-	put_device(&ctx->afu->dev);
-
 	/*
 	 * At this this point all bottom halfs have finished and we should be
 	 * getting no more IRQs from the hardware for this context.  Once it's
@@ -198,8 +201,12 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 	 * where a process (master, some daemon, etc) has opened the chardev on
 	 * behalf of another process, so the AFU's mm gets bound to the process
 	 * that performs this ioctl and not the process that opened the file.
+	 * Also we grab the PID of the group leader so that if the task that
+	 * has performed the attach operation exits the mm context of the
+	 * process is still accessible.
 	 */
-	ctx->pid = get_pid(get_task_pid(current, PIDTYPE_PID));
+	ctx->pid = get_task_pid(current, PIDTYPE_PID);
+	ctx->glpid = get_task_pid(current->group_leader, PIDTYPE_PID);
 
 	trace_cxl_attach(ctx, work.work_element_descriptor, work.num_interrupts, amr);
 
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 85761d7eb333..4c1903f781fc 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -138,6 +138,7 @@ static const struct pci_device_id cxl_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0477), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x044b), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x04cf), },
+	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0601), },
 	{ PCI_DEVICE_CLASS(0x120000, ~0), },
 
 	{ }
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index c241e15cacb1..cbd4331fb45c 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -203,7 +203,7 @@ static int cxl_pcie_write_config(struct pci_bus *bus, unsigned int devfn,
 	mask <<= shift;
 	val <<= shift;
 
-	v = (in_le32(ioaddr) & ~mask) || (val & mask);
+	v = (in_le32(ioaddr) & ~mask) | (val & mask);
 
 	out_le32(ioaddr, v);
 	return PCIBIOS_SUCCESSFUL;
diff --git a/drivers/misc/ibmasm/ibmasm.h b/drivers/misc/ibmasm/ibmasm.h
index 9b083448814d..5bd127727d8e 100644
--- a/drivers/misc/ibmasm/ibmasm.h
+++ b/drivers/misc/ibmasm/ibmasm.h
@@ -211,7 +211,7 @@ void ibmasmfs_unregister(void);
 void ibmasmfs_add_sp(struct service_processor *sp);
 
 /* uart */
-#ifdef CONFIG_SERIAL_8250
+#if IS_ENABLED(CONFIG_SERIAL_8250)
 void ibmasm_register_uart(struct service_processor *sp);
 void ibmasm_unregister_uart(struct service_processor *sp);
 #else
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index 27678d8154e0..75fc9c688df8 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -31,6 +31,7 @@
 #include <linux/jiffies.h>
 #include <linux/interrupt.h>
 
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 
 #include <linux/mei.h>
@@ -436,7 +437,7 @@ static inline void mei_me_set_pm_domain(struct mei_device *dev)
 		dev->pg_domain.ops.runtime_resume = mei_me_pm_runtime_resume;
 		dev->pg_domain.ops.runtime_idle = mei_me_pm_runtime_idle;
 
-		pdev->dev.pm_domain = &dev->pg_domain;
+		dev_pm_domain_set(&pdev->dev, &dev->pg_domain);
 	}
 }
 
@@ -448,7 +449,7 @@ static inline void mei_me_set_pm_domain(struct mei_device *dev)
 static inline void mei_me_unset_pm_domain(struct mei_device *dev)
 {
 	/* stop using pm callbacks if any */
-	dev->dev->pm_domain = NULL;
+	dev_pm_domain_set(dev->dev, NULL);
 }
 
 static const struct dev_pm_ops mei_me_pm_ops = {
diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c
index 0882c0201907..71f8a7475717 100644
--- a/drivers/misc/mei/pci-txe.c
+++ b/drivers/misc/mei/pci-txe.c
@@ -27,6 +27,7 @@
 #include <linux/jiffies.h>
 #include <linux/interrupt.h>
 #include <linux/workqueue.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 
 #include <linux/mei.h>
@@ -388,7 +389,7 @@ static inline void mei_txe_set_pm_domain(struct mei_device *dev)
 		dev->pg_domain.ops.runtime_resume = mei_txe_pm_runtime_resume;
 		dev->pg_domain.ops.runtime_idle = mei_txe_pm_runtime_idle;
 
-		pdev->dev.pm_domain = &dev->pg_domain;
+		dev_pm_domain_set(&pdev->dev, &dev->pg_domain);
 	}
 }
 
@@ -400,7 +401,7 @@ static inline void mei_txe_set_pm_domain(struct mei_device *dev)
 static inline void mei_txe_unset_pm_domain(struct mei_device *dev)
 {
 	/* stop using pm callbacks if any */
-	dev->dev->pm_domain = NULL;
+	dev_pm_domain_set(dev->dev, NULL);
 }
 
 static const struct dev_pm_ops mei_txe_pm_ops = {
diff --git a/drivers/misc/mic/card/mic_virtio.c b/drivers/misc/mic/card/mic_virtio.c
index e486a0c26267..f6ed57d3125c 100644
--- a/drivers/misc/mic/card/mic_virtio.c
+++ b/drivers/misc/mic/card/mic_virtio.c
@@ -311,7 +311,7 @@ unmap:
 static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			struct virtqueue *vqs[],
 			vq_callback_t *callbacks[],
-			const char *names[])
+			const char * const names[])
 {
 	struct mic_vdev *mvdev = to_micvdev(vdev);
 	struct mic_device_ctrl __iomem *dc = mvdev->dc;
diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c
index 154aced0b91b..65cc0ac9b82d 100644
--- a/drivers/mmc/core/debugfs.c
+++ b/drivers/mmc/core/debugfs.c
@@ -170,7 +170,7 @@ static int mmc_ios_show(struct seq_file *s, void *data)
 		str = "invalid";
 		break;
 	}
-	seq_printf(s, "signal voltage:\t%u (%s)\n", ios->chip_select, str);
+	seq_printf(s, "signal voltage:\t%u (%s)\n", ios->signal_voltage, str);
 
 	switch (ios->drv_type) {
 	case MMC_SET_DRIVER_TYPE_A:
diff --git a/drivers/mmc/core/pwrseq_simple.c b/drivers/mmc/core/pwrseq_simple.c
index 2b16263458af..aba786daebca 100644
--- a/drivers/mmc/core/pwrseq_simple.c
+++ b/drivers/mmc/core/pwrseq_simple.c
@@ -29,15 +29,18 @@ struct mmc_pwrseq_simple {
 static void mmc_pwrseq_simple_set_gpios_value(struct mmc_pwrseq_simple *pwrseq,
 					      int value)
 {
-	int i;
 	struct gpio_descs *reset_gpios = pwrseq->reset_gpios;
-	int values[reset_gpios->ndescs];
 
-	for (i = 0; i < reset_gpios->ndescs; i++)
-		values[i] = value;
+	if (!IS_ERR(reset_gpios)) {
+		int i;
+		int values[reset_gpios->ndescs];
 
-	gpiod_set_array_value_cansleep(reset_gpios->ndescs, reset_gpios->desc,
-				       values);
+		for (i = 0; i < reset_gpios->ndescs; i++)
+			values[i] = value;
+
+		gpiod_set_array_value_cansleep(
+			reset_gpios->ndescs, reset_gpios->desc, values);
+	}
 }
 
 static void mmc_pwrseq_simple_pre_power_on(struct mmc_host *host)
@@ -79,7 +82,8 @@ static void mmc_pwrseq_simple_free(struct mmc_host *host)
 	struct mmc_pwrseq_simple *pwrseq = container_of(host->pwrseq,
 					struct mmc_pwrseq_simple, pwrseq);
 
-	gpiod_put_array(pwrseq->reset_gpios);
+	if (!IS_ERR(pwrseq->reset_gpios))
+		gpiod_put_array(pwrseq->reset_gpios);
 
 	if (!IS_ERR(pwrseq->ext_clk))
 		clk_put(pwrseq->ext_clk);
@@ -112,7 +116,9 @@ struct mmc_pwrseq *mmc_pwrseq_simple_alloc(struct mmc_host *host,
 	}
 
 	pwrseq->reset_gpios = gpiod_get_array(dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(pwrseq->reset_gpios)) {
+	if (IS_ERR(pwrseq->reset_gpios) &&
+	    PTR_ERR(pwrseq->reset_gpios) != -ENOENT &&
+	    PTR_ERR(pwrseq->reset_gpios) != -ENOSYS) {
 		ret = PTR_ERR(pwrseq->reset_gpios);
 		goto clk_put;
 	}
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index f2b164b214ae..bb39a29b2db6 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -329,6 +329,7 @@ static int mmc_read_switch(struct mmc_card *card)
 		card->sw_caps.sd3_bus_mode = status[13];
 		/* Driver Strengths supported by the card */
 		card->sw_caps.sd3_drv_type = status[9];
+		card->sw_caps.sd3_curr_limit = status[7] | status[6] << 8;
 	}
 
 out:
@@ -545,14 +546,25 @@ static int sd_set_current_limit(struct mmc_card *card, u8 *status)
 	 * when we set current limit to 200ma, the card will draw 200ma, and
 	 * when we set current limit to 400/600/800ma, the card will draw its
 	 * maximum 300ma from the host.
+	 *
+	 * The above is incorrect: if we try to set a current limit that is
+	 * not supported by the card, the card can rightfully error out the
+	 * attempt, and remain at the default current limit.  This results
+	 * in a 300mA card being limited to 200mA even though the host
+	 * supports 800mA. Failures seen with SanDisk 8GB UHS cards with
+	 * an iMX6 host. --rmk
 	 */
-	if (max_current >= 800)
+	if (max_current >= 800 &&
+	    card->sw_caps.sd3_curr_limit & SD_MAX_CURRENT_800)
 		current_limit = SD_SET_CURRENT_LIMIT_800;
-	else if (max_current >= 600)
+	else if (max_current >= 600 &&
+		 card->sw_caps.sd3_curr_limit & SD_MAX_CURRENT_600)
 		current_limit = SD_SET_CURRENT_LIMIT_600;
-	else if (max_current >= 400)
+	else if (max_current >= 400 &&
+		 card->sw_caps.sd3_curr_limit & SD_MAX_CURRENT_400)
 		current_limit = SD_SET_CURRENT_LIMIT_400;
-	else if (max_current >= 200)
+	else if (max_current >= 200 &&
+		 card->sw_caps.sd3_curr_limit & SD_MAX_CURRENT_200)
 		current_limit = SD_SET_CURRENT_LIMIT_200;
 
 	if (current_limit != SD_SET_CURRENT_NO_CHANGE) {
@@ -626,9 +638,9 @@ static int mmc_sd_init_uhs_card(struct mmc_card *card)
 	 * SDR104 mode SD-cards. Note that tuning is mandatory for SDR104.
 	 */
 	if (!mmc_host_is_spi(card->host) &&
-		(card->sd_bus_speed == UHS_SDR50_BUS_SPEED ||
-		 card->sd_bus_speed == UHS_DDR50_BUS_SPEED ||
-		 card->sd_bus_speed == UHS_SDR104_BUS_SPEED)) {
+		(card->host->ios.timing == MMC_TIMING_UHS_SDR50 ||
+		 card->host->ios.timing == MMC_TIMING_UHS_DDR50 ||
+		 card->host->ios.timing == MMC_TIMING_UHS_SDR104)) {
 		err = mmc_execute_tuning(card);
 
 		/*
@@ -638,7 +650,7 @@ static int mmc_sd_init_uhs_card(struct mmc_card *card)
 		 * difference between v3.00 and 3.01 spec means that CMD19
 		 * tuning is also available for DDR50 mode.
 		 */
-		if (err && card->sd_bus_speed == UHS_DDR50_BUS_SPEED) {
+		if (err && card->host->ios.timing == MMC_TIMING_UHS_DDR50) {
 			pr_warn("%s: ddr50 tuning failed\n",
 				mmc_hostname(card->host));
 			err = 0;
diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c
index d61ba1a0495e..467b3cf80c44 100644
--- a/drivers/mmc/core/sdio.c
+++ b/drivers/mmc/core/sdio.c
@@ -535,8 +535,8 @@ static int mmc_sdio_init_uhs_card(struct mmc_card *card)
 	 * SDR104 mode SD-cards. Note that tuning is mandatory for SDR104.
 	 */
 	if (!mmc_host_is_spi(card->host) &&
-	    ((card->sw_caps.sd3_bus_mode & SD_MODE_UHS_SDR50) ||
-	     (card->sw_caps.sd3_bus_mode & SD_MODE_UHS_SDR104)))
+	    ((card->host->ios.timing == MMC_TIMING_UHS_SDR50) ||
+	      (card->host->ios.timing == MMC_TIMING_UHS_SDR104)))
 		err = mmc_execute_tuning(card);
 out:
 	return err;
diff --git a/drivers/mmc/core/sdio_cis.c b/drivers/mmc/core/sdio_cis.c
index 8e94e555b788..6f6fc527a263 100644
--- a/drivers/mmc/core/sdio_cis.c
+++ b/drivers/mmc/core/sdio_cis.c
@@ -223,6 +223,7 @@ static const struct cis_tpl cis_tpl_list[] = {
 	{	0x20,	4,	cistpl_manfid		},
 	{	0x21,	2,	/* cistpl_funcid */	},
 	{	0x22,	0,	cistpl_funce		},
+	{	0x91,	2,	/* cistpl_sdio_std */	},
 };
 
 static int sdio_read_cis(struct mmc_card *card, struct sdio_func *func)
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index a36ebdae2388..851ccd9ac868 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -29,7 +29,6 @@
 #include <linux/slab.h>
 #include <linux/stat.h>
 #include <linux/types.h>
-#include <linux/platform_data/mmc-atmel-mci.h>
 
 #include <linux/mmc/host.h>
 #include <linux/mmc/sdio.h>
@@ -2439,6 +2438,23 @@ static int atmci_configure_dma(struct atmel_mci *host)
 {
 	host->dma.chan = dma_request_slave_channel_reason(&host->pdev->dev,
 							"rxtx");
+
+	if (PTR_ERR(host->dma.chan) == -ENODEV) {
+		struct mci_platform_data *pdata = host->pdev->dev.platform_data;
+		dma_cap_mask_t mask;
+
+		if (!pdata->dma_filter)
+			return -ENODEV;
+
+		dma_cap_zero(mask);
+		dma_cap_set(DMA_SLAVE, mask);
+
+		host->dma.chan = dma_request_channel(mask, pdata->dma_filter,
+						     pdata->dma_slave);
+		if (!host->dma.chan)
+			host->dma.chan = ERR_PTR(-ENODEV);
+	}
+
 	if (IS_ERR(host->dma.chan))
 		return PTR_ERR(host->dma.chan);
 
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index fb266745f824..0d6ca4116f3d 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -151,6 +151,7 @@ static struct variant_data variant_nomadik = {
 	.fifosize		= 16 * 4,
 	.fifohalfsize		= 8 * 4,
 	.clkreg			= MCI_CLK_ENABLE,
+	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 	.datalength_bits	= 24,
 	.datactrl_mask_sdio	= MCI_ST_DPSM_SDIOEN,
 	.st_sdio		= true,
@@ -1886,7 +1887,7 @@ static struct amba_id mmci_ids[] = {
 	{
 		.id     = 0x00280180,
 		.mask   = 0x00ffffff,
-		.data	= &variant_u300,
+		.data	= &variant_nomadik,
 	},
 	{
 		.id     = 0x00480180,
diff --git a/drivers/mmc/host/tmio_mmc_dma.c b/drivers/mmc/host/tmio_mmc_dma.c
index e4b05dbb9ca8..4a0d6b80eaa3 100644
--- a/drivers/mmc/host/tmio_mmc_dma.c
+++ b/drivers/mmc/host/tmio_mmc_dma.c
@@ -94,9 +94,9 @@ static void tmio_mmc_start_dma_rx(struct tmio_mmc_host *host)
 			desc = NULL;
 			ret = cookie;
 		}
+		dev_dbg(&host->pdev->dev, "%s(): mapped %d -> %d, cookie %d, rq %p\n",
+			__func__, host->sg_len, ret, cookie, host->mrq);
 	}
-	dev_dbg(&host->pdev->dev, "%s(): mapped %d -> %d, cookie %d, rq %p\n",
-		__func__, host->sg_len, ret, cookie, host->mrq);
 
 pio:
 	if (!desc) {
@@ -116,8 +116,8 @@ pio:
 			 "DMA failed: %d, falling back to PIO\n", ret);
 	}
 
-	dev_dbg(&host->pdev->dev, "%s(): desc %p, cookie %d, sg[%d]\n", __func__,
-		desc, cookie, host->sg_len);
+	dev_dbg(&host->pdev->dev, "%s(): desc %p, sg[%d]\n", __func__,
+		desc, host->sg_len);
 }
 
 static void tmio_mmc_start_dma_tx(struct tmio_mmc_host *host)
@@ -174,9 +174,9 @@ static void tmio_mmc_start_dma_tx(struct tmio_mmc_host *host)
 			desc = NULL;
 			ret = cookie;
 		}
+		dev_dbg(&host->pdev->dev, "%s(): mapped %d -> %d, cookie %d, rq %p\n",
+			__func__, host->sg_len, ret, cookie, host->mrq);
 	}
-	dev_dbg(&host->pdev->dev, "%s(): mapped %d -> %d, cookie %d, rq %p\n",
-		__func__, host->sg_len, ret, cookie, host->mrq);
 
 pio:
 	if (!desc) {
@@ -196,8 +196,7 @@ pio:
 			 "DMA failed: %d, falling back to PIO\n", ret);
 	}
 
-	dev_dbg(&host->pdev->dev, "%s(): desc %p, cookie %d\n", __func__,
-		desc, cookie);
+	dev_dbg(&host->pdev->dev, "%s(): desc %p\n", __func__, desc);
 }
 
 void tmio_mmc_start_dma(struct tmio_mmc_host *host,
diff --git a/drivers/mtd/bcm63xxpart.c b/drivers/mtd/bcm63xxpart.c
index 440936998593..cec3188a170d 100644
--- a/drivers/mtd/bcm63xxpart.c
+++ b/drivers/mtd/bcm63xxpart.c
@@ -24,6 +24,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/bcm963xx_tag.h>
 #include <linux/crc32.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -34,11 +35,8 @@
 #include <linux/mtd/partitions.h>
 
 #include <asm/mach-bcm63xx/bcm63xx_nvram.h>
-#include <asm/mach-bcm63xx/bcm963xx_tag.h>
 #include <asm/mach-bcm63xx/board_bcm963xx.h>
 
-#define BCM63XX_EXTENDED_SIZE	0xBFC00000	/* Extended flash address */
-
 #define BCM63XX_CFE_BLOCK_SIZE	SZ_64K		/* always at least 64KiB */
 
 #define BCM63XX_CFE_MAGIC_OFFSET 0x4e0
@@ -123,8 +121,8 @@ static int bcm63xx_parse_cfe_partitions(struct mtd_info *master,
 		pr_info("CFE boot tag found with version %s and board type %s\n",
 			tagversion, boardid);
 
-		kerneladdr = kerneladdr - BCM63XX_EXTENDED_SIZE;
-		rootfsaddr = rootfsaddr - BCM63XX_EXTENDED_SIZE;
+		kerneladdr = kerneladdr - BCM963XX_EXTENDED_SIZE;
+		rootfsaddr = rootfsaddr - BCM963XX_EXTENDED_SIZE;
 		spareaddr = roundup(totallen, master->erasesize) + cfelen;
 
 		if (rootfsaddr < kerneladdr) {
diff --git a/drivers/mtd/ubi/cdev.c b/drivers/mtd/ubi/cdev.c
index 54e056d3be02..ee2b74d1d1b5 100644
--- a/drivers/mtd/ubi/cdev.c
+++ b/drivers/mtd/ubi/cdev.c
@@ -174,9 +174,9 @@ static int vol_cdev_fsync(struct file *file, loff_t start, loff_t end,
 	struct ubi_device *ubi = desc->vol->ubi;
 	struct inode *inode = file_inode(file);
 	int err;
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	err = ubi_sync(ubi->ubi_num);
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 	return err;
 }
 
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 1c5f3b273e6a..79e1a0282163 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -2459,8 +2459,13 @@ boomerang_interrupt(int irq, void *dev_id)
 					struct sk_buff *skb = vp->tx_skbuff[entry];
 #if DO_ZEROCOPY
 					int i;
-					for (i=0; i<=skb_shinfo(skb)->nr_frags; i++)
-							pci_unmap_single(VORTEX_PCI(vp),
+					pci_unmap_single(VORTEX_PCI(vp),
+							le32_to_cpu(vp->tx_ring[entry].frag[0].addr),
+							le32_to_cpu(vp->tx_ring[entry].frag[0].length),
+							PCI_DMA_TODEVICE);
+
+					for (i=1; i<=skb_shinfo(skb)->nr_frags; i++)
+							pci_unmap_page(VORTEX_PCI(vp),
 											 le32_to_cpu(vp->tx_ring[entry].frag[i].addr),
 											 le32_to_cpu(vp->tx_ring[entry].frag[i].length)&0xFFF,
 											 PCI_DMA_TODEVICE);
diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
index c7798d360512..06f6cffdfaf5 100644
--- a/drivers/net/ethernet/broadcom/bgmac.c
+++ b/drivers/net/ethernet/broadcom/bgmac.c
@@ -466,6 +466,11 @@ static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
 			len -= ETH_FCS_LEN;
 
 			skb = build_skb(buf, BGMAC_RX_ALLOC_SIZE);
+			if (unlikely(!skb)) {
+				bgmac_err(bgmac, "build_skb failed\n");
+				put_page(virt_to_head_page(buf));
+				break;
+			}
 			skb_put(skb, BGMAC_RX_FRAME_OFFSET +
 				BGMAC_RX_BUF_OFFSET + len);
 			skb_pull(skb, BGMAC_RX_FRAME_OFFSET +
diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c
index 21a0cfc3e7ec..771cc267f217 100644
--- a/drivers/net/ethernet/brocade/bna/bnad.c
+++ b/drivers/net/ethernet/brocade/bna/bnad.c
@@ -542,39 +542,50 @@ bnad_cq_drop_packet(struct bnad *bnad, struct bna_rcb *rcb,
 }
 
 static void
-bnad_cq_setup_skb_frags(struct bna_rcb *rcb, struct sk_buff *skb,
-			u32 sop_ci, u32 nvecs, u32 last_fraglen)
+bnad_cq_setup_skb_frags(struct bna_ccb *ccb, struct sk_buff *skb, u32 nvecs)
 {
+	struct bna_rcb *rcb;
 	struct bnad *bnad;
-	u32 ci, vec, len, totlen = 0;
 	struct bnad_rx_unmap_q *unmap_q;
-	struct bnad_rx_unmap *unmap;
+	struct bna_cq_entry *cq, *cmpl;
+	u32 ci, pi, totlen = 0;
+
+	cq = ccb->sw_q;
+	pi = ccb->producer_index;
+	cmpl = &cq[pi];
 
+	rcb = bna_is_small_rxq(cmpl->rxq_id) ? ccb->rcb[1] : ccb->rcb[0];
 	unmap_q = rcb->unmap_q;
 	bnad = rcb->bnad;
+	ci = rcb->consumer_index;
 
 	/* prefetch header */
-	prefetch(page_address(unmap_q->unmap[sop_ci].page) +
-			unmap_q->unmap[sop_ci].page_offset);
+	prefetch(page_address(unmap_q->unmap[ci].page) +
+		 unmap_q->unmap[ci].page_offset);
+
+	while (nvecs--) {
+		struct bnad_rx_unmap *unmap;
+		u32 len;
 
-	for (vec = 1, ci = sop_ci; vec <= nvecs; vec++) {
 		unmap = &unmap_q->unmap[ci];
 		BNA_QE_INDX_INC(ci, rcb->q_depth);
 
 		dma_unmap_page(&bnad->pcidev->dev,
-				dma_unmap_addr(&unmap->vector, dma_addr),
-				unmap->vector.len, DMA_FROM_DEVICE);
+			       dma_unmap_addr(&unmap->vector, dma_addr),
+			       unmap->vector.len, DMA_FROM_DEVICE);
 
-		len = (vec == nvecs) ?
-			last_fraglen : unmap->vector.len;
+		len = ntohs(cmpl->length);
 		skb->truesize += unmap->vector.len;
 		totlen += len;
 
 		skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
-				unmap->page, unmap->page_offset, len);
+				   unmap->page, unmap->page_offset, len);
 
 		unmap->page = NULL;
 		unmap->vector.len = 0;
+
+		BNA_QE_INDX_INC(pi, ccb->q_depth);
+		cmpl = &cq[pi];
 	}
 
 	skb->len += totlen;
@@ -704,7 +715,7 @@ bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget)
 		if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 			bnad_cq_setup_skb(bnad, skb, unmap, len);
 		else
-			bnad_cq_setup_skb_frags(rcb, skb, sop_ci, nvecs, len);
+			bnad_cq_setup_skb_frags(ccb, skb, nvecs);
 
 		rcb->rxq->rx_packets++;
 		rcb->rxq->rx_bytes += totlen;
diff --git a/drivers/net/ethernet/cadence/macb.c b/drivers/net/ethernet/cadence/macb.c
index c56347536f6b..9d9984a87d42 100644
--- a/drivers/net/ethernet/cadence/macb.c
+++ b/drivers/net/ethernet/cadence/macb.c
@@ -1040,6 +1040,8 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id)
 		/* close possible race with dev_close */
 		if (unlikely(!netif_running(dev))) {
 			queue_writel(queue, IDR, -1);
+			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
+				queue_writel(queue, ISR, -1);
 			break;
 		}
 
@@ -1561,6 +1563,8 @@ static void macb_reset_hw(struct macb *bp)
 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
 		queue_writel(queue, IDR, -1);
 		queue_readl(queue, ISR);
+		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
+			queue_writel(queue, ISR, -1);
 	}
 }
 
diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
index 6b1261c0b1c2..7c92eb854925 100644
--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
@@ -1434,7 +1434,6 @@ struct fman_mac *dtsec_config(struct fman_mac_params *params)
 	dtsec->tbiphy = of_phy_find_device(params->internal_phy_node);
 	if (!dtsec->tbiphy) {
 		pr_err("of_phy_find_device (TBI PHY) failed\n");
-		put_device(&dtsec->tbiphy->mdio.dev);
 		goto err_dtsec_drv_param;
 	}
 
diff --git a/drivers/net/ethernet/freescale/fsl_pq_mdio.c b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
index 622005abf859..f3c63dce1e30 100644
--- a/drivers/net/ethernet/freescale/fsl_pq_mdio.c
+++ b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
@@ -29,7 +29,7 @@
 
 #include <asm/io.h>
 #if IS_ENABLED(CONFIG_UCC_GETH)
-#include <asm/ucc.h>	/* for ucc_set_qe_mux_mii_mng() */
+#include <soc/fsl/qe/ucc.h>
 #endif
 
 #include "gianfar.h"
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index cbddbe2d0429..5bf1ade28315 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -40,10 +40,10 @@
 #include <asm/uaccess.h>
 #include <asm/irq.h>
 #include <asm/io.h>
-#include <asm/immap_qe.h>
-#include <asm/qe.h>
-#include <asm/ucc.h>
-#include <asm/ucc_fast.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_fast.h>
 #include <asm/machdep.h>
 
 #include "ucc_geth.h"
diff --git a/drivers/net/ethernet/freescale/ucc_geth.h b/drivers/net/ethernet/freescale/ucc_geth.h
index 75f337163ce3..5da19b440a6a 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.h
+++ b/drivers/net/ethernet/freescale/ucc_geth.h
@@ -22,11 +22,11 @@
 #include <linux/list.h>
 #include <linux/if_ether.h>
 
-#include <asm/immap_qe.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
 
-#include <asm/ucc.h>
-#include <asm/ucc_fast.h>
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_fast.h>
 
 #define DRV_DESC "QE UCC Gigabit Ethernet Controller"
 #define DRV_NAME "ucc_geth"
diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
index 253f8ed0537a..0c4afe95ef54 100644
--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
+++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
@@ -500,8 +500,10 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget)
 	while (cnt && !last) {
 		buf = priv->rx_buf[priv->rx_head];
 		skb = build_skb(buf, priv->rx_buf_size);
-		if (unlikely(!skb))
+		if (unlikely(!skb)) {
 			net_dbg_ratelimited("build_skb failed\n");
+			goto refill;
+		}
 
 		dma_unmap_single(&ndev->dev, priv->rx_phys[priv->rx_head],
 				 RX_BUF_SIZE, DMA_FROM_DEVICE);
@@ -528,6 +530,7 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget)
 			rx++;
 		}
 
+refill:
 		buf = netdev_alloc_frag(priv->rx_buf_size);
 		if (!buf)
 			goto done;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
index d2263c72bd8a..12188807468c 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
@@ -369,8 +369,17 @@ int hns_rcb_common_init_hw(struct rcb_common_cb *rcb_common)
 	dsaf_write_dev(rcb_common, RCB_COM_CFG_ENDIAN_REG,
 		       HNS_RCB_COMMON_ENDIAN);
 
-	dsaf_write_dev(rcb_common, RCB_COM_CFG_FNA_REG, 0x0);
-	dsaf_write_dev(rcb_common, RCB_COM_CFG_FA_REG, 0x1);
+	if (AE_IS_VER1(rcb_common->dsaf_dev->dsaf_ver)) {
+		dsaf_write_dev(rcb_common, RCB_COM_CFG_FNA_REG, 0x0);
+		dsaf_write_dev(rcb_common, RCB_COM_CFG_FA_REG, 0x1);
+	} else {
+		dsaf_set_dev_bit(rcb_common, RCBV2_COM_CFG_USER_REG,
+				 RCB_COM_CFG_FNA_B, false);
+		dsaf_set_dev_bit(rcb_common, RCBV2_COM_CFG_USER_REG,
+				 RCB_COM_CFG_FA_B, true);
+		dsaf_set_dev_bit(rcb_common, RCBV2_COM_CFG_TSO_MODE_REG,
+				 RCB_COM_TSO_MODE_B, HNS_TSO_MODE_8BD_32K);
+	}
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
index 29041b18741a..81fe9f849973 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
@@ -54,6 +54,9 @@ struct rcb_common_cb;
 #define HNS_DUMP_REG_NUM			500
 #define HNS_STATIC_REG_NUM			12
 
+#define HNS_TSO_MODE_8BD_32K			1
+#define HNS_TSO_MDOE_4BD_16K			0
+
 enum rcb_int_flag {
 	RCB_INT_FLAG_TX = 0x1,
 	RCB_INT_FLAG_RX = (0x1 << 1),
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
index 5d1b746e141d..f0c4f9b09d5b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
@@ -363,6 +363,8 @@
 #define RCB_COM_CFG_FA_REG			0x3C
 #define RCB_COM_CFG_PKT_TC_BP_REG		0x40
 #define RCB_COM_CFG_PPE_TNL_CLKEN_REG		0x44
+#define RCBV2_COM_CFG_USER_REG			0x30
+#define RCBV2_COM_CFG_TSO_MODE_REG		0x50
 
 #define RCB_COM_INTMSK_TX_PKT_REG		0x3A0
 #define RCB_COM_RINT_TX_PKT_REG			0x3A8
@@ -860,6 +862,9 @@
 
 #define PPE_COMMON_CNT_CLR_CE_B	0
 #define PPE_COMMON_CNT_CLR_SNAP_EN_B	1
+#define RCB_COM_TSO_MODE_B	0
+#define RCB_COM_CFG_FNA_B	1
+#define RCB_COM_CFG_FA_B	0
 
 #define GMAC_DUPLEX_TYPE_B 0
 
diff --git a/drivers/net/ethernet/hisilicon/hns_mdio.c b/drivers/net/ethernet/hisilicon/hns_mdio.c
index 58c96c412fe8..765ddb3dcd1a 100644
--- a/drivers/net/ethernet/hisilicon/hns_mdio.c
+++ b/drivers/net/ethernet/hisilicon/hns_mdio.c
@@ -458,7 +458,7 @@ static int hns_mdio_probe(struct platform_device *pdev)
 	}
 
 	mdio_dev->subctrl_vbase =
-		syscon_node_to_regmap(of_parse_phandle(np, "subctrl_vbase", 0));
+		syscon_node_to_regmap(of_parse_phandle(np, "subctrl-vbase", 0));
 	if (IS_ERR(mdio_dev->subctrl_vbase)) {
 		dev_warn(&pdev->dev, "no syscon hisilicon,peri-c-subctrl\n");
 		mdio_dev->subctrl_vbase = NULL;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index a0bbb6b9f5aa..94da913b151d 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -1239,7 +1239,7 @@ static int i40evf_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
 
 	if (adapter->current_op != I40E_VIRTCHNL_OP_UNKNOWN) {
 		/* bail because we already have a command pending */
-		dev_err(&adapter->pdev->dev, "Cannot confiure RSS, command %d pending\n",
+		dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
 			adapter->current_op);
 		return -EBUSY;
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index 2c2baab9d880..d66c690a8597 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -157,6 +157,7 @@ static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags)
 		[29] = "802.1ad offload support",
 		[31] = "Modifying loopback source checks using UPDATE_QP support",
 		[32] = "Loopback source checks support",
+		[33] = "RoCEv2 support"
 	};
 	int i;
 
@@ -626,6 +627,8 @@ out:
 	return err;
 }
 
+static void disable_unsupported_roce_caps(void *buf);
+
 int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 {
 	struct mlx4_cmd_mailbox *mailbox;
@@ -738,6 +741,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 	if (err)
 		goto out;
 
+	if (mlx4_is_mfunc(dev))
+		disable_unsupported_roce_caps(outbox);
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_QP_OFFSET);
 	dev_cap->reserved_qps = 1 << (field & 0xf);
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_OFFSET);
@@ -905,6 +910,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
 	MLX4_GET(dev_cap->bmme_flags, outbox,
 		 QUERY_DEV_CAP_BMME_FLAGS_OFFSET);
+	if (dev_cap->bmme_flags & MLX4_FLAG_ROCE_V1_V2)
+		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ROCE_V1_V2;
 	if (dev_cap->bmme_flags & MLX4_FLAG_PORT_REMAP)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_PORT_REMAP;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_CONFIG_DEV_OFFSET);
@@ -1161,6 +1168,7 @@ int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave,
 	if (err)
 		return err;
 
+	disable_unsupported_roce_caps(outbox->buf);
 	/* add port mng change event capability and disable mw type 1
 	 * unconditionally to slaves
 	 */
@@ -1258,6 +1266,21 @@ int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave,
 	return 0;
 }
 
+static void disable_unsupported_roce_caps(void *buf)
+{
+	u32 flags;
+
+	MLX4_GET(flags, buf, QUERY_DEV_CAP_EXT_FLAGS_OFFSET);
+	flags &= ~(1UL << 31);
+	MLX4_PUT(buf, flags, QUERY_DEV_CAP_EXT_FLAGS_OFFSET);
+	MLX4_GET(flags, buf, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET);
+	flags &= ~(1UL << 24);
+	MLX4_PUT(buf, flags, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET);
+	MLX4_GET(flags, buf, QUERY_DEV_CAP_BMME_FLAGS_OFFSET);
+	flags &= ~(MLX4_FLAG_ROCE_V1_V2);
+	MLX4_PUT(buf, flags, QUERY_DEV_CAP_BMME_FLAGS_OFFSET);
+}
+
 int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave,
 			    struct mlx4_vhcr *vhcr,
 			    struct mlx4_cmd_mailbox *inbox,
@@ -2239,7 +2262,8 @@ struct mlx4_config_dev {
 	__be32	rsvd1[3];
 	__be16	vxlan_udp_dport;
 	__be16	rsvd2;
-	__be32	rsvd3;
+	__be16  roce_v2_entropy;
+	__be16  roce_v2_udp_dport;
 	__be32	roce_flags;
 	__be32	rsvd4[25];
 	__be16	rsvd5;
@@ -2248,6 +2272,7 @@ struct mlx4_config_dev {
 };
 
 #define MLX4_VXLAN_UDP_DPORT (1 << 0)
+#define MLX4_ROCE_V2_UDP_DPORT BIT(3)
 #define MLX4_DISABLE_RX_PORT BIT(18)
 
 static int mlx4_CONFIG_DEV_set(struct mlx4_dev *dev, struct mlx4_config_dev *config_dev)
@@ -2365,6 +2390,18 @@ int mlx4_disable_rx_port_check(struct mlx4_dev *dev, bool dis)
 	return mlx4_CONFIG_DEV_set(dev, &config_dev);
 }
 
+int mlx4_config_roce_v2_port(struct mlx4_dev *dev, u16 udp_port)
+{
+	struct mlx4_config_dev config_dev;
+
+	memset(&config_dev, 0, sizeof(config_dev));
+	config_dev.update_flags    = cpu_to_be32(MLX4_ROCE_V2_UDP_DPORT);
+	config_dev.roce_v2_udp_dport = cpu_to_be16(udp_port);
+
+	return mlx4_CONFIG_DEV_set(dev, &config_dev);
+}
+EXPORT_SYMBOL_GPL(mlx4_config_roce_v2_port);
+
 int mlx4_virt2phy_port_map(struct mlx4_dev *dev, u32 port1, u32 port2)
 {
 	struct mlx4_cmd_mailbox *mailbox;
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index 2404c22ad2b2..7baef52db6b7 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -780,7 +780,10 @@ struct mlx4_set_port_general_context {
 	u16 reserved1;
 	u8 v_ignore_fcs;
 	u8 flags;
-	u8 ignore_fcs;
+	union {
+		u8 ignore_fcs;
+		u8 roce_mode;
+	};
 	u8 reserved2;
 	__be16 mtu;
 	u8 pptx;
diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c
index f2550425c251..787b7bb54d52 100644
--- a/drivers/net/ethernet/mellanox/mlx4/port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/port.c
@@ -1520,6 +1520,8 @@ int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz)
 	return err;
 }
 
+#define SET_PORT_ROCE_2_FLAGS          0x10
+#define MLX4_SET_PORT_ROCE_V1_V2       0x2
 int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu,
 			  u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx)
 {
@@ -1539,6 +1541,11 @@ int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu,
 	context->pprx = (pprx * (!pfcrx)) << 7;
 	context->pfcrx = pfcrx;
 
+	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
+		context->flags |= SET_PORT_ROCE_2_FLAGS;
+		context->roce_mode |=
+			MLX4_SET_PORT_ROCE_V1_V2 << 4;
+	}
 	in_mod = MLX4_SET_PORT_GENERAL << 8 | port;
 	err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE,
 		       MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
diff --git a/drivers/net/ethernet/mellanox/mlx4/qp.c b/drivers/net/ethernet/mellanox/mlx4/qp.c
index 168823dde79f..d1cd9c32a9ae 100644
--- a/drivers/net/ethernet/mellanox/mlx4/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx4/qp.c
@@ -167,6 +167,12 @@ static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 		context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
 	}
 
+	if ((cur_state == MLX4_QP_STATE_RTR) &&
+	    (new_state == MLX4_QP_STATE_RTS) &&
+	    dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2)
+		context->roce_entropy =
+			cpu_to_be16(mlx4_qp_roce_entropy(dev, qp->qpn));
+
 	*(__be32 *) mailbox->buf = cpu_to_be32(optpar);
 	memcpy(mailbox->buf + 8, context, sizeof *context);
 
@@ -921,3 +927,23 @@ int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx4_qp_to_ready);
+
+u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn)
+{
+	struct mlx4_qp_context context;
+	struct mlx4_qp qp;
+	int err;
+
+	qp.qpn = qpn;
+	err = mlx4_qp_query(dev, &qp, &context);
+	if (!err) {
+		u32 dest_qpn = be32_to_cpu(context.remote_qpn) & 0xffffff;
+		u16 folded_dst = folded_qp(dest_qpn);
+		u16 folded_src = folded_qp(qpn);
+
+		return (dest_qpn != qpn) ?
+			((folded_dst ^ folded_src) | 0xC000) :
+			folded_src | 0xC000;
+	}
+	return 0xdead;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 9ea49a893323..aac071a7e830 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -39,8 +39,8 @@
 #include <linux/mlx5/qp.h>
 #include <linux/mlx5/cq.h>
 #include <linux/mlx5/vport.h>
+#include <linux/mlx5/transobj.h>
 #include "wq.h"
-#include "transobj.h"
 #include "mlx5_core.h"
 
 #define MLX5E_MAX_NUM_TC	8
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 5c74a734f158..6a3e430f1062 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -752,7 +752,7 @@ static int mlx5e_create_cq(struct mlx5e_channel *c,
 	struct mlx5_core_dev *mdev = priv->mdev;
 	struct mlx5_core_cq *mcq = &cq->mcq;
 	int eqn_not_used;
-	int irqn;
+	unsigned int irqn;
 	int err;
 	u32 i;
 
@@ -806,7 +806,7 @@ static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 	void *in;
 	void *cqc;
 	int inlen;
-	int irqn_not_used;
+	unsigned int irqn_not_used;
 	int eqn;
 	int err;
 
@@ -1517,7 +1517,7 @@ static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
 	struct mlx5_core_dev *mdev = priv->mdev;
 	struct mlx5_core_cq *mcq = &cq->mcq;
 	int eqn_not_used;
-	int irqn;
+	unsigned int irqn;
 	int err;
 
 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
@@ -2241,7 +2241,7 @@ static void *mlx5e_create_netdev(struct mlx5_core_dev *mdev)
 		goto err_unmap_free_uar;
 	}
 
-	err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
+	err = mlx5_core_alloc_transport_domain(mdev, &priv->tdn);
 	if (err) {
 		mlx5_core_err(mdev, "alloc td failed, %d\n", err);
 		goto err_dealloc_pd;
@@ -2324,7 +2324,7 @@ err_destroy_mkey:
 	mlx5_core_destroy_mkey(mdev, &priv->mr);
 
 err_dealloc_transport_domain:
-	mlx5_dealloc_transport_domain(mdev, priv->tdn);
+	mlx5_core_dealloc_transport_domain(mdev, priv->tdn);
 
 err_dealloc_pd:
 	mlx5_core_dealloc_pd(mdev, priv->pdn);
@@ -2356,7 +2356,7 @@ static void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, void *vpriv)
 	mlx5e_close_drop_rq(priv);
 	mlx5e_destroy_tises(priv);
 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
-	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
+	mlx5_core_dealloc_transport_domain(priv->mdev, priv->tdn);
 	mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
 	mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
 	free_netdev(netdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 23c244a7e5d7..647a3ca2c2a9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -230,6 +230,7 @@ static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 		case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 		case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 			rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+			rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
 			mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n",
 				      eqe_type_str(eqe->type), eqe->type, rsn);
 			mlx5_rsc_event(dev, rsn, eqe->type);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 67676cf0d507..1545a944c309 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -78,6 +78,11 @@ struct mlx5_device_context {
 	void		       *context;
 };
 
+enum {
+	MLX5_ATOMIC_REQ_MODE_BE = 0x0,
+	MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
+};
+
 static struct mlx5_profile profile[] = {
 	[0] = {
 		.mask           = 0,
@@ -387,7 +392,7 @@ query_ex:
 	return err;
 }
 
-static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
+static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod)
 {
 	u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)];
 	int err;
@@ -395,6 +400,7 @@ static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
 	memset(out, 0, sizeof(out));
 
 	MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
+	MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
 	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 	if (err)
 		return err;
@@ -404,6 +410,46 @@ static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
 	return err;
 }
 
+static int handle_hca_cap_atomic(struct mlx5_core_dev *dev)
+{
+	void *set_ctx;
+	void *set_hca_cap;
+	int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
+	int req_endianness;
+	int err;
+
+	if (MLX5_CAP_GEN(dev, atomic)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+	} else {
+		return 0;
+	}
+
+	req_endianness =
+		MLX5_CAP_ATOMIC(dev,
+				supported_atomic_req_8B_endianess_mode_1);
+
+	if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
+		return 0;
+
+	set_ctx = kzalloc(set_sz, GFP_KERNEL);
+	if (!set_ctx)
+		return -ENOMEM;
+
+	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
+
+	/* Set requestor to host endianness */
+	MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode,
+		 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
+
+	err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC);
+
+	kfree(set_ctx);
+	return err;
+}
+
 static int handle_hca_cap(struct mlx5_core_dev *dev)
 {
 	void *set_ctx = NULL;
@@ -445,7 +491,8 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
 
 	MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
 
-	err = set_caps(dev, set_ctx, set_sz);
+	err = set_caps(dev, set_ctx, set_sz,
+		       MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
 
 query_ex:
 	kfree(set_ctx);
@@ -585,7 +632,8 @@ static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev)
 		mlx5_irq_clear_affinity_hint(mdev, i);
 }
 
-int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn)
+int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn,
+		    unsigned int *irqn)
 {
 	struct mlx5_eq_table *table = &dev->priv.eq_table;
 	struct mlx5_eq *eq, *n;
@@ -666,7 +714,6 @@ clean:
 	return err;
 }
 
-#ifdef CONFIG_MLX5_CORE_EN
 static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 {
 	u32 query_in[MLX5_ST_SZ_DW(query_issi_in)];
@@ -719,7 +766,6 @@ static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 
 	return -ENOTSUPP;
 }
-#endif
 
 static int map_bf_area(struct mlx5_core_dev *dev)
 {
@@ -965,13 +1011,11 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 		goto err_pagealloc_cleanup;
 	}
 
-#ifdef CONFIG_MLX5_CORE_EN
 	err = mlx5_core_set_issi(dev);
 	if (err) {
 		dev_err(&pdev->dev, "failed to set issi\n");
 		goto err_disable_hca;
 	}
-#endif
 
 	err = mlx5_satisfy_startup_pages(dev, 1);
 	if (err) {
@@ -991,6 +1035,12 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 		goto reclaim_boot_pages;
 	}
 
+	err = handle_hca_cap_atomic(dev);
+	if (err) {
+		dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n");
+		goto reclaim_boot_pages;
+	}
+
 	err = mlx5_satisfy_startup_pages(dev, 0);
 	if (err) {
 		dev_err(&pdev->dev, "failed to allocate init pages\n");
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qp.c b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
index 30e2ba3f5f16..def289375ecb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
@@ -36,6 +36,7 @@
 #include <linux/mlx5/cmd.h>
 #include <linux/mlx5/qp.h>
 #include <linux/mlx5/driver.h>
+#include <linux/mlx5/transobj.h>
 
 #include "mlx5_core.h"
 
@@ -67,6 +68,52 @@ void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)
 		complete(&common->free);
 }
 
+static u64 qp_allowed_event_types(void)
+{
+	u64 mask;
+
+	mask = BIT(MLX5_EVENT_TYPE_PATH_MIG) |
+	       BIT(MLX5_EVENT_TYPE_COMM_EST) |
+	       BIT(MLX5_EVENT_TYPE_SQ_DRAINED) |
+	       BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
+	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR) |
+	       BIT(MLX5_EVENT_TYPE_PATH_MIG_FAILED) |
+	       BIT(MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) |
+	       BIT(MLX5_EVENT_TYPE_WQ_ACCESS_ERROR);
+
+	return mask;
+}
+
+static u64 rq_allowed_event_types(void)
+{
+	u64 mask;
+
+	mask = BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
+	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
+
+	return mask;
+}
+
+static u64 sq_allowed_event_types(void)
+{
+	return BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
+}
+
+static bool is_event_type_allowed(int rsc_type, int event_type)
+{
+	switch (rsc_type) {
+	case MLX5_EVENT_QUEUE_TYPE_QP:
+		return BIT(event_type) & qp_allowed_event_types();
+	case MLX5_EVENT_QUEUE_TYPE_RQ:
+		return BIT(event_type) & rq_allowed_event_types();
+	case MLX5_EVENT_QUEUE_TYPE_SQ:
+		return BIT(event_type) & sq_allowed_event_types();
+	default:
+		WARN(1, "Event arrived for unknown resource type");
+		return false;
+	}
+}
+
 void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
 {
 	struct mlx5_core_rsc_common *common = mlx5_get_rsc(dev, rsn);
@@ -75,8 +122,16 @@ void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
 	if (!common)
 		return;
 
+	if (!is_event_type_allowed((rsn >> MLX5_USER_INDEX_LEN), event_type)) {
+		mlx5_core_warn(dev, "event 0x%.2x is not allowed on resource 0x%.8x\n",
+			       event_type, rsn);
+		return;
+	}
+
 	switch (common->res) {
 	case MLX5_RES_QP:
+	case MLX5_RES_RQ:
+	case MLX5_RES_SQ:
 		qp = (struct mlx5_core_qp *)common;
 		qp->event(qp, event_type);
 		break;
@@ -177,27 +232,56 @@ void mlx5_eq_pagefault(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
 }
 #endif
 
+static int create_qprqsq_common(struct mlx5_core_dev *dev,
+				struct mlx5_core_qp *qp,
+				int rsc_type)
+{
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+	int err;
+
+	qp->common.res = rsc_type;
+	spin_lock_irq(&table->lock);
+	err = radix_tree_insert(&table->tree,
+				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),
+				qp);
+	spin_unlock_irq(&table->lock);
+	if (err)
+		return err;
+
+	atomic_set(&qp->common.refcount, 1);
+	init_completion(&qp->common.free);
+	qp->pid = current->pid;
+
+	return 0;
+}
+
+static void destroy_qprqsq_common(struct mlx5_core_dev *dev,
+				  struct mlx5_core_qp *qp)
+{
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+	unsigned long flags;
+
+	spin_lock_irqsave(&table->lock, flags);
+	radix_tree_delete(&table->tree,
+			  qp->qpn | (qp->common.res << MLX5_USER_INDEX_LEN));
+	spin_unlock_irqrestore(&table->lock, flags);
+	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
+	wait_for_completion(&qp->common.free);
+}
+
 int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 			struct mlx5_core_qp *qp,
 			struct mlx5_create_qp_mbox_in *in,
 			int inlen)
 {
-	struct mlx5_qp_table *table = &dev->priv.qp_table;
 	struct mlx5_create_qp_mbox_out out;
 	struct mlx5_destroy_qp_mbox_in din;
 	struct mlx5_destroy_qp_mbox_out dout;
 	int err;
-	void *qpc;
 
 	memset(&out, 0, sizeof(out));
 	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_QP);
 
-	if (dev->issi) {
-		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
-		/* 0xffffff means we ask to work with cqe version 0 */
-		MLX5_SET(qpc, qpc, user_index, 0xffffff);
-	}
-
 	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
 	if (err) {
 		mlx5_core_warn(dev, "ret %d\n", err);
@@ -213,24 +297,16 @@ int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 	qp->qpn = be32_to_cpu(out.qpn) & 0xffffff;
 	mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn);
 
-	qp->common.res = MLX5_RES_QP;
-	spin_lock_irq(&table->lock);
-	err = radix_tree_insert(&table->tree, qp->qpn, qp);
-	spin_unlock_irq(&table->lock);
-	if (err) {
-		mlx5_core_warn(dev, "err %d\n", err);
+	err = create_qprqsq_common(dev, qp, MLX5_RES_QP);
+	if (err)
 		goto err_cmd;
-	}
 
 	err = mlx5_debug_qp_add(dev, qp);
 	if (err)
 		mlx5_core_dbg(dev, "failed adding QP 0x%x to debug file system\n",
 			      qp->qpn);
 
-	qp->pid = current->pid;
-	atomic_set(&qp->common.refcount, 1);
 	atomic_inc(&dev->num_qps);
-	init_completion(&qp->common.free);
 
 	return 0;
 
@@ -250,18 +326,11 @@ int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 {
 	struct mlx5_destroy_qp_mbox_in in;
 	struct mlx5_destroy_qp_mbox_out out;
-	struct mlx5_qp_table *table = &dev->priv.qp_table;
-	unsigned long flags;
 	int err;
 
 	mlx5_debug_qp_remove(dev, qp);
 
-	spin_lock_irqsave(&table->lock, flags);
-	radix_tree_delete(&table->tree, qp->qpn);
-	spin_unlock_irqrestore(&table->lock, flags);
-
-	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
-	wait_for_completion(&qp->common.free);
+	destroy_qprqsq_common(dev, qp);
 
 	memset(&in, 0, sizeof(in));
 	memset(&out, 0, sizeof(out));
@@ -279,59 +348,15 @@ int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp);
 
-int mlx5_core_qp_modify(struct mlx5_core_dev *dev, enum mlx5_qp_state cur_state,
-			enum mlx5_qp_state new_state,
+int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 operation,
 			struct mlx5_modify_qp_mbox_in *in, int sqd_event,
 			struct mlx5_core_qp *qp)
 {
-	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
-		[MLX5_QP_STATE_RST] = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
-		},
-		[MLX5_QP_STATE_INIT]  = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
-			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
-		},
-		[MLX5_QP_STATE_RTR]   = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
-		},
-		[MLX5_QP_STATE_RTS]   = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
-		},
-		[MLX5_QP_STATE_SQD] = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-		},
-		[MLX5_QP_STATE_SQER] = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
-		},
-		[MLX5_QP_STATE_ERR] = {
-			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
-			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
-		}
-	};
-
 	struct mlx5_modify_qp_mbox_out out;
 	int err = 0;
-	u16 op;
-
-	if (cur_state >= MLX5_QP_NUM_STATE || new_state >= MLX5_QP_NUM_STATE ||
-	    !optab[cur_state][new_state])
-		return -EINVAL;
 
 	memset(&out, 0, sizeof(out));
-	op = optab[cur_state][new_state];
-	in->hdr.opcode = cpu_to_be16(op);
+	in->hdr.opcode = cpu_to_be16(operation);
 	in->qpn = cpu_to_be32(qp->qpn);
 	err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
 	if (err)
@@ -449,3 +474,67 @@ int mlx5_core_page_fault_resume(struct mlx5_core_dev *dev, u32 qpn,
 }
 EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume);
 #endif
+
+int mlx5_core_create_rq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
+				struct mlx5_core_qp *rq)
+{
+	int err;
+	u32 rqn;
+
+	err = mlx5_core_create_rq(dev, in, inlen, &rqn);
+	if (err)
+		return err;
+
+	rq->qpn = rqn;
+	err = create_qprqsq_common(dev, rq, MLX5_RES_RQ);
+	if (err)
+		goto err_destroy_rq;
+
+	return 0;
+
+err_destroy_rq:
+	mlx5_core_destroy_rq(dev, rq->qpn);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_rq_tracked);
+
+void mlx5_core_destroy_rq_tracked(struct mlx5_core_dev *dev,
+				  struct mlx5_core_qp *rq)
+{
+	destroy_qprqsq_common(dev, rq);
+	mlx5_core_destroy_rq(dev, rq->qpn);
+}
+EXPORT_SYMBOL(mlx5_core_destroy_rq_tracked);
+
+int mlx5_core_create_sq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
+				struct mlx5_core_qp *sq)
+{
+	int err;
+	u32 sqn;
+
+	err = mlx5_core_create_sq(dev, in, inlen, &sqn);
+	if (err)
+		return err;
+
+	sq->qpn = sqn;
+	err = create_qprqsq_common(dev, sq, MLX5_RES_SQ);
+	if (err)
+		goto err_destroy_sq;
+
+	return 0;
+
+err_destroy_sq:
+	mlx5_core_destroy_sq(dev, sq->qpn);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_sq_tracked);
+
+void mlx5_core_destroy_sq_tracked(struct mlx5_core_dev *dev,
+				  struct mlx5_core_qp *sq)
+{
+	destroy_qprqsq_common(dev, sq);
+	mlx5_core_destroy_sq(dev, sq->qpn);
+}
+EXPORT_SYMBOL(mlx5_core_destroy_sq_tracked);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/srq.c b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
index ffada801976b..04bc522605a0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/srq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
@@ -37,7 +37,7 @@
 #include <linux/mlx5/srq.h>
 #include <rdma/ib_verbs.h>
 #include "mlx5_core.h"
-#include "transobj.h"
+#include <linux/mlx5/transobj.h>
 
 void mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type)
 {
@@ -241,8 +241,6 @@ static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
 
 	memcpy(xrc_srqc, srqc, MLX5_ST_SZ_BYTES(srqc));
 	memcpy(pas, in->pas, pas_size);
-	/* 0xffffff means we ask to work with cqe version 0 */
-	MLX5_SET(xrc_srqc,	    xrc_srqc,  user_index, 0xffffff);
 	MLX5_SET(create_xrc_srq_in, create_in, opcode,
 		 MLX5_CMD_OP_CREATE_XRC_SRQ);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
index d7068f54e800..03a5093ffeb7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
@@ -32,9 +32,9 @@
 
 #include <linux/mlx5/driver.h>
 #include "mlx5_core.h"
-#include "transobj.h"
+#include <linux/mlx5/transobj.h>
 
-int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
+int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
 {
 	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)];
 	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)];
@@ -53,8 +53,9 @@ int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
 
 	return err;
 }
+EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
 
-void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
+void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
 {
 	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)];
 	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)];
@@ -68,6 +69,7 @@ void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
 
 	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 }
+EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
 
 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
 {
@@ -94,6 +96,7 @@ int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen)
 	memset(out, 0, sizeof(out));
 	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
 }
+EXPORT_SYMBOL(mlx5_core_modify_rq);
 
 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
 {
@@ -108,6 +111,18 @@ void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
 	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 }
 
+int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
+{
+	u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {0};
+	int outlen = MLX5_ST_SZ_BYTES(query_rq_out);
+
+	MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
+	MLX5_SET(query_rq_in, in, rqn, rqn);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+}
+EXPORT_SYMBOL(mlx5_core_query_rq);
+
 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_sq_out)];
@@ -133,6 +148,7 @@ int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen)
 	memset(out, 0, sizeof(out));
 	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
 }
+EXPORT_SYMBOL(mlx5_core_modify_sq);
 
 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
 {
@@ -147,6 +163,18 @@ void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
 	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 }
 
+int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
+{
+	u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {0};
+	int outlen = MLX5_ST_SZ_BYTES(query_sq_out);
+
+	MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
+	MLX5_SET(query_sq_in, in, sqn, sqn);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+}
+EXPORT_SYMBOL(mlx5_core_query_sq);
+
 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *tirn)
 {
@@ -162,6 +190,7 @@ int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
 
 	return err;
 }
+EXPORT_SYMBOL(mlx5_core_create_tir);
 
 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
 			 int inlen)
@@ -187,6 +216,7 @@ void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
 
 	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 }
+EXPORT_SYMBOL(mlx5_core_destroy_tir);
 
 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *tisn)
@@ -203,6 +233,19 @@ int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
 
 	return err;
 }
+EXPORT_SYMBOL(mlx5_core_create_tis);
+
+int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
+			 int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_tis_out)] = {0};
+
+	MLX5_SET(modify_tis_in, in, tisn, tisn);
+	MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
+
+	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_core_modify_tis);
 
 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
 {
@@ -216,6 +259,7 @@ void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
 
 	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
 }
+EXPORT_SYMBOL(mlx5_core_destroy_tis);
 
 int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *rmpn)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.h b/drivers/net/ethernet/mellanox/mlx5/core/transobj.h
deleted file mode 100644
index 74cae51436e4..000000000000
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef __TRANSOBJ_H__
-#define __TRANSOBJ_H__
-
-int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn);
-void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn);
-int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			u32 *rqn);
-int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen);
-void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn);
-int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			u32 *sqn);
-int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen);
-void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn);
-int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			 u32 *tirn);
-int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
-			 int inlen);
-void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn);
-int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			 u32 *tisn);
-void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn);
-int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			 u32 *rmpn);
-int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen);
-int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn);
-int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
-int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
-int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			  u32 *rmpn);
-int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 rmpn);
-int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
-int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
-
-int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
-			 u32 *rqtn);
-int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
-			 int inlen);
-void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn);
-
-#endif /* __TRANSOBJ_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
index 076197efea9b..c7398b95aecd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
@@ -76,7 +76,7 @@ u8 mlx5_query_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
 
 	return MLX5_GET(query_vport_state_out, out, admin_state);
 }
-EXPORT_SYMBOL(mlx5_query_vport_admin_state);
+EXPORT_SYMBOL_GPL(mlx5_query_vport_admin_state);
 
 int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod,
 				  u16 vport, u8 state)
@@ -104,7 +104,7 @@ int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod,
 
 	return err;
 }
-EXPORT_SYMBOL(mlx5_modify_vport_admin_state);
+EXPORT_SYMBOL_GPL(mlx5_modify_vport_admin_state);
 
 static int mlx5_query_nic_vport_context(struct mlx5_core_dev *mdev, u16 vport,
 					u32 *out, int outlen)
@@ -151,12 +151,9 @@ int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev,
 				nic_vport_context.permanent_address);
 
 	err = mlx5_query_nic_vport_context(mdev, vport, out, outlen);
-	if (err)
-		goto out;
-
-	ether_addr_copy(addr, &out_addr[2]);
+	if (!err)
+		ether_addr_copy(addr, &out_addr[2]);
 
-out:
 	kvfree(out);
 	return err;
 }
@@ -197,7 +194,7 @@ int mlx5_modify_nic_vport_mac_address(struct mlx5_core_dev *mdev,
 
 	return err;
 }
-EXPORT_SYMBOL(mlx5_modify_nic_vport_mac_address);
+EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_mac_address);
 
 int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev,
 				  u32 vport,
@@ -430,6 +427,68 @@ int mlx5_modify_nic_vport_vlans(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_vlans);
 
+int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev,
+					   u64 *system_image_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+
+	*system_image_guid = MLX5_GET64(query_nic_vport_context_out, out,
+					nic_vport_context.system_image_guid);
+
+	kfree(out);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_system_image_guid);
+
+int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+
+	*node_guid = MLX5_GET64(query_nic_vport_context_out, out,
+				nic_vport_context.node_guid);
+
+	kfree(out);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_node_guid);
+
+int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev,
+					u16 *qkey_viol_cntr)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+
+	*qkey_viol_cntr = MLX5_GET(query_nic_vport_context_out, out,
+				   nic_vport_context.qkey_violation_counter);
+
+	kfree(out);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_qkey_viol_cntr);
+
 int mlx5_query_hca_vport_gid(struct mlx5_core_dev *dev, u8 other_vport,
 			     u8 port_num, u16  vf_num, u16 gid_index,
 			     union ib_gid *gid)
@@ -750,3 +809,44 @@ int mlx5_modify_nic_vport_promisc(struct mlx5_core_dev *mdev,
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_promisc);
+
+enum mlx5_vport_roce_state {
+	MLX5_VPORT_ROCE_DISABLED = 0,
+	MLX5_VPORT_ROCE_ENABLED  = 1,
+};
+
+static int mlx5_nic_vport_update_roce_state(struct mlx5_core_dev *mdev,
+					    enum mlx5_vport_roce_state state)
+{
+	void *in;
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(mdev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+
+	MLX5_SET(modify_nic_vport_context_in, in, field_select.roce_en, 1);
+	MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.roce_en,
+		 state);
+
+	err = mlx5_modify_nic_vport_context(mdev, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+
+int mlx5_nic_vport_enable_roce(struct mlx5_core_dev *mdev)
+{
+	return mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_ENABLED);
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_enable_roce);
+
+int mlx5_nic_vport_disable_roce(struct mlx5_core_dev *mdev)
+{
+	return mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_DISABLED);
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_disable_roce);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index ffe894e6d287..45479ef5bcf4 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -1015,6 +1015,7 @@ static int mlxsw_sp_port_obj_del(struct net_device *dev,
 	case SWITCHDEV_OBJ_ID_PORT_MDB:
 		err = mlxsw_sp_port_mdb_del(mlxsw_sp_port,
 					    SWITCHDEV_OBJ_PORT_MDB(obj));
+		break;
 	default:
 		err = -EOPNOTSUPP;
 		break;
diff --git a/drivers/net/ethernet/smsc/Kconfig b/drivers/net/ethernet/smsc/Kconfig
index eb9230e2092f..63aca9f847e1 100644
--- a/drivers/net/ethernet/smsc/Kconfig
+++ b/drivers/net/ethernet/smsc/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SMSC
 	default y
 	depends on ARM || ARM64 || ATARI_ETHERNAT || BLACKFIN || COLDFIRE || \
 		   ISA || M32R || MAC || MIPS || MN10300 || NIOS2 || PCI || \
-		   PCMCIA || SUPERH || XTENSA
+		   PCMCIA || SUPERH || XTENSA || H8300
 	---help---
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
@@ -38,7 +38,7 @@ config SMC91X
 	select MII
 	depends on !OF || GPIOLIB
 	depends on ARM || ARM64 || ATARI_ETHERNAT || BLACKFIN || COLDFIRE || \
-		   M32R || MIPS || MN10300 || NIOS2 || SUPERH || XTENSA
+		   M32R || MIPS || MN10300 || NIOS2 || SUPERH || XTENSA || H8300
 	---help---
 	  This is a driver for SMC's 91x series of Ethernet chipsets,
 	  including the SMC91C94 and the SMC91C111. Say Y if you want it
diff --git a/drivers/net/ethernet/smsc/smc91x.h b/drivers/net/ethernet/smsc/smc91x.h
index a3c129e1e40a..1a55c7976df0 100644
--- a/drivers/net/ethernet/smsc/smc91x.h
+++ b/drivers/net/ethernet/smsc/smc91x.h
@@ -172,6 +172,17 @@ static inline void mcf_outsw(void *a, unsigned char *p, int l)
 
 #define SMC_IRQ_FLAGS		0
 
+#elif defined(CONFIG_H8300)
+#define SMC_CAN_USE_8BIT	1
+#define SMC_CAN_USE_16BIT	0
+#define SMC_CAN_USE_32BIT	0
+#define SMC_NOWAIT		0
+
+#define SMC_inb(a, r)		ioread8((a) + (r))
+#define SMC_outb(v, a, r)	iowrite8(v, (a) + (r))
+#define SMC_insb(a, r, p, l)	ioread8_rep((a) + (r), p, l)
+#define SMC_outsb(a, r, p, l)	iowrite8_rep((a) + (r), p, l)
+
 #else
 
 /*
diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
index 8a8f6fb2880d..2174ec937b4d 100644
--- a/drivers/net/phy/at803x.c
+++ b/drivers/net/phy/at803x.c
@@ -20,10 +20,21 @@
 #include <linux/gpio/consumer.h>
 
 #define AT803X_INTR_ENABLE			0x12
+#define AT803X_INTR_ENABLE_AUTONEG_ERR		BIT(15)
+#define AT803X_INTR_ENABLE_SPEED_CHANGED	BIT(14)
+#define AT803X_INTR_ENABLE_DUPLEX_CHANGED	BIT(13)
+#define AT803X_INTR_ENABLE_PAGE_RECEIVED	BIT(12)
+#define AT803X_INTR_ENABLE_LINK_FAIL		BIT(11)
+#define AT803X_INTR_ENABLE_LINK_SUCCESS		BIT(10)
+#define AT803X_INTR_ENABLE_WIRESPEED_DOWNGRADE	BIT(5)
+#define AT803X_INTR_ENABLE_POLARITY_CHANGED	BIT(1)
+#define AT803X_INTR_ENABLE_WOL			BIT(0)
+
 #define AT803X_INTR_STATUS			0x13
+
 #define AT803X_SMART_SPEED			0x14
 #define AT803X_LED_CONTROL			0x18
-#define AT803X_WOL_ENABLE			0x01
+
 #define AT803X_DEVICE_ADDR			0x03
 #define AT803X_LOC_MAC_ADDR_0_15_OFFSET		0x804C
 #define AT803X_LOC_MAC_ADDR_16_31_OFFSET	0x804B
@@ -31,13 +42,15 @@
 #define AT803X_MMD_ACCESS_CONTROL		0x0D
 #define AT803X_MMD_ACCESS_CONTROL_DATA		0x0E
 #define AT803X_FUNC_DATA			0x4003
-#define AT803X_INER				0x0012
-#define AT803X_INER_INIT			0xec00
-#define AT803X_INSR				0x0013
+
 #define AT803X_DEBUG_ADDR			0x1D
 #define AT803X_DEBUG_DATA			0x1E
-#define AT803X_DEBUG_SYSTEM_MODE_CTRL		0x05
-#define AT803X_DEBUG_RGMII_TX_CLK_DLY		BIT(8)
+
+#define AT803X_DEBUG_REG_0			0x00
+#define AT803X_DEBUG_RX_CLK_DLY_EN		BIT(15)
+
+#define AT803X_DEBUG_REG_5			0x05
+#define AT803X_DEBUG_TX_CLK_DLY_EN		BIT(8)
 
 #define ATH8030_PHY_ID 0x004dd076
 #define ATH8031_PHY_ID 0x004dd074
@@ -61,6 +74,46 @@ struct at803x_context {
 	u16 led_control;
 };
 
+static int at803x_debug_reg_read(struct phy_device *phydev, u16 reg)
+{
+	int ret;
+
+	ret = phy_write(phydev, AT803X_DEBUG_ADDR, reg);
+	if (ret < 0)
+		return ret;
+
+	return phy_read(phydev, AT803X_DEBUG_DATA);
+}
+
+static int at803x_debug_reg_mask(struct phy_device *phydev, u16 reg,
+				 u16 clear, u16 set)
+{
+	u16 val;
+	int ret;
+
+	ret = at803x_debug_reg_read(phydev, reg);
+	if (ret < 0)
+		return ret;
+
+	val = ret & 0xffff;
+	val &= ~clear;
+	val |= set;
+
+	return phy_write(phydev, AT803X_DEBUG_DATA, val);
+}
+
+static inline int at803x_enable_rx_delay(struct phy_device *phydev)
+{
+	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_0, 0,
+					AT803X_DEBUG_RX_CLK_DLY_EN);
+}
+
+static inline int at803x_enable_tx_delay(struct phy_device *phydev)
+{
+	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_5, 0,
+					AT803X_DEBUG_TX_CLK_DLY_EN);
+}
+
 /* save relevant PHY registers to private copy */
 static void at803x_context_save(struct phy_device *phydev,
 				struct at803x_context *context)
@@ -119,14 +172,14 @@ static int at803x_set_wol(struct phy_device *phydev,
 		}
 
 		value = phy_read(phydev, AT803X_INTR_ENABLE);
-		value |= AT803X_WOL_ENABLE;
+		value |= AT803X_INTR_ENABLE_WOL;
 		ret = phy_write(phydev, AT803X_INTR_ENABLE, value);
 		if (ret)
 			return ret;
 		value = phy_read(phydev, AT803X_INTR_STATUS);
 	} else {
 		value = phy_read(phydev, AT803X_INTR_ENABLE);
-		value &= (~AT803X_WOL_ENABLE);
+		value &= (~AT803X_INTR_ENABLE_WOL);
 		ret = phy_write(phydev, AT803X_INTR_ENABLE, value);
 		if (ret)
 			return ret;
@@ -145,7 +198,7 @@ static void at803x_get_wol(struct phy_device *phydev,
 	wol->wolopts = 0;
 
 	value = phy_read(phydev, AT803X_INTR_ENABLE);
-	if (value & AT803X_WOL_ENABLE)
+	if (value & AT803X_INTR_ENABLE_WOL)
 		wol->wolopts |= WAKE_MAGIC;
 }
 
@@ -157,7 +210,7 @@ static int at803x_suspend(struct phy_device *phydev)
 	mutex_lock(&phydev->lock);
 
 	value = phy_read(phydev, AT803X_INTR_ENABLE);
-	wol_enabled = value & AT803X_WOL_ENABLE;
+	wol_enabled = value & AT803X_INTR_ENABLE_WOL;
 
 	value = phy_read(phydev, MII_BMCR);
 
@@ -217,14 +270,17 @@ static int at803x_config_init(struct phy_device *phydev)
 	if (ret < 0)
 		return ret;
 
-	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID) {
-		ret = phy_write(phydev, AT803X_DEBUG_ADDR,
-				AT803X_DEBUG_SYSTEM_MODE_CTRL);
-		if (ret)
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID ||
+			phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) {
+		ret = at803x_enable_rx_delay(phydev);
+		if (ret < 0)
 			return ret;
-		ret = phy_write(phydev, AT803X_DEBUG_DATA,
-				AT803X_DEBUG_RGMII_TX_CLK_DLY);
-		if (ret)
+	}
+
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID ||
+			phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) {
+		ret = at803x_enable_tx_delay(phydev);
+		if (ret < 0)
 			return ret;
 	}
 
@@ -235,7 +291,7 @@ static int at803x_ack_interrupt(struct phy_device *phydev)
 {
 	int err;
 
-	err = phy_read(phydev, AT803X_INSR);
+	err = phy_read(phydev, AT803X_INTR_STATUS);
 
 	return (err < 0) ? err : 0;
 }
@@ -245,13 +301,19 @@ static int at803x_config_intr(struct phy_device *phydev)
 	int err;
 	int value;
 
-	value = phy_read(phydev, AT803X_INER);
+	value = phy_read(phydev, AT803X_INTR_ENABLE);
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		value |= AT803X_INTR_ENABLE_AUTONEG_ERR;
+		value |= AT803X_INTR_ENABLE_SPEED_CHANGED;
+		value |= AT803X_INTR_ENABLE_DUPLEX_CHANGED;
+		value |= AT803X_INTR_ENABLE_LINK_FAIL;
+		value |= AT803X_INTR_ENABLE_LINK_SUCCESS;
 
-	if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
-		err = phy_write(phydev, AT803X_INER,
-				value | AT803X_INER_INIT);
+		err = phy_write(phydev, AT803X_INTR_ENABLE, value);
+	}
 	else
-		err = phy_write(phydev, AT803X_INER, 0);
+		err = phy_write(phydev, AT803X_INTR_ENABLE, 0);
 
 	return err;
 }
@@ -322,7 +384,7 @@ static struct phy_driver at803x_driver[] = {
 	.get_wol		= at803x_get_wol,
 	.suspend		= at803x_suspend,
 	.resume			= at803x_resume,
-	.features		= PHY_GBIT_FEATURES,
+	.features		= PHY_BASIC_FEATURES,
 	.flags			= PHY_HAS_INTERRUPT,
 	.config_aneg		= genphy_config_aneg,
 	.read_status		= genphy_read_status,
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 903737adfc01..bad3f005faee 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -901,6 +901,11 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 
 	phydev->state = PHY_READY;
 
+	/* Initial carrier state is off as the phy is about to be
+	 * (re)initialized.
+	 */
+	netif_carrier_off(phydev->attached_dev);
+
 	/* Do initial configuration here, now that
 	 * we have certain key parameters
 	 * (dev_flags and interface)
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index 2528331de193..718ceeab4dbc 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -1872,10 +1872,10 @@ static int team_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
 	struct team *team = netdev_priv(dev);
 	struct team_port *port;
 
-	rcu_read_lock();
-	list_for_each_entry_rcu(port, &team->port_list, list)
+	mutex_lock(&team->lock);
+	list_for_each_entry(port, &team->port_list, list)
 		vlan_vid_del(port->dev, proto, vid);
-	rcu_read_unlock();
+	mutex_unlock(&team->lock);
 
 	return 0;
 }
diff --git a/drivers/net/wan/x25_asy.c b/drivers/net/wan/x25_asy.c
index cd39025d2abf..1bc5e93d2a34 100644
--- a/drivers/net/wan/x25_asy.c
+++ b/drivers/net/wan/x25_asy.c
@@ -571,8 +571,10 @@ static int x25_asy_open_tty(struct tty_struct *tty)
 
 	/* Perform the low-level X.25 async init */
 	err = x25_asy_open(sl->dev);
-	if (err)
+	if (err) {
+		x25_asy_free(sl);
 		return err;
+	}
 	/* Done.  We have linked the TTY line to a channel. */
 	return 0;
 }
diff --git a/drivers/net/wireless/ath/ath10k/thermal.h b/drivers/net/wireless/ath/ath10k/thermal.h
index b610ea5caae8..c9223e9e962f 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.h
+++ b/drivers/net/wireless/ath/ath10k/thermal.h
@@ -36,7 +36,7 @@ struct ath10k_thermal {
 	int temperature;
 };
 
-#ifdef CONFIG_THERMAL
+#if IS_REACHABLE(CONFIG_THERMAL)
 int ath10k_thermal_register(struct ath10k *ar);
 void ath10k_thermal_unregister(struct ath10k *ar);
 void ath10k_thermal_event_temperature(struct ath10k *ar, int temperature);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
index ef06f57a7a0e..d3c9f0d52ae3 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
@@ -47,8 +47,7 @@ static const char BRCM_ ## fw_nvram_name ## _FIRMWARE_NAME[] = \
 	BRCMF_FW_DEFAULT_PATH fw; \
 static const char BRCM_ ## fw_nvram_name ## _NVRAM_NAME[] = \
 	BRCMF_FW_DEFAULT_PATH nvram; \
-MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw); \
-MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH nvram)
+MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw);
 
 #define BRCMF_FW_DEF(fw_name, fw) \
 static const char BRCM_ ## fw_name ## _FIRMWARE_NAME[] = \
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
index 07a4c644fb9b..e9cef9de9ed8 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
@@ -901,7 +901,7 @@ static void iwlagn_gain_computation(struct iwl_priv *priv,
 		/* bound gain by 2 bits value max, 3rd bit is sign */
 		data->delta_gain_code[i] =
 			min(abs(delta_g),
-			(long) CHAIN_NOISE_MAX_DELTA_GAIN_CODE);
+			(s32) CHAIN_NOISE_MAX_DELTA_GAIN_CODE);
 
 		if (delta_g < 0)
 			/*
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index e7bd63eb2876..f5231a2dd2ac 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -615,6 +615,7 @@ err_tx_unbind:
 	queue->tx_irq = 0;
 err_unmap:
 	xenvif_unmap_frontend_rings(queue);
+	netif_napi_del(&queue->napi);
 err:
 	module_put(THIS_MODULE);
 	return err;
@@ -684,22 +685,16 @@ void xenvif_deinit_queue(struct xenvif_queue *queue)
 
 void xenvif_free(struct xenvif *vif)
 {
-	struct xenvif_queue *queue = NULL;
+	struct xenvif_queue *queues = vif->queues;
 	unsigned int num_queues = vif->num_queues;
 	unsigned int queue_index;
 
 	unregister_netdev(vif->dev);
-
-	for (queue_index = 0; queue_index < num_queues; ++queue_index) {
-		queue = &vif->queues[queue_index];
-		xenvif_deinit_queue(queue);
-	}
-
-	vfree(vif->queues);
-	vif->queues = NULL;
-	vif->num_queues = 0;
-
 	free_netdev(vif->dev);
 
+	for (queue_index = 0; queue_index < num_queues; ++queue_index)
+		xenvif_deinit_queue(&queues[queue_index]);
+	vfree(queues);
+
 	module_put(THIS_MODULE);
 }
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 1049c34e7d43..61b97c34bb3b 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -149,20 +149,19 @@ static inline pending_ring_idx_t pending_index(unsigned i)
 	return i & (MAX_PENDING_REQS-1);
 }
 
-static int xenvif_rx_ring_slots_needed(struct xenvif *vif)
-{
-	if (vif->gso_mask)
-		return DIV_ROUND_UP(vif->dev->gso_max_size, XEN_PAGE_SIZE) + 1;
-	else
-		return DIV_ROUND_UP(vif->dev->mtu, XEN_PAGE_SIZE);
-}
-
 static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
 {
 	RING_IDX prod, cons;
+	struct sk_buff *skb;
 	int needed;
 
-	needed = xenvif_rx_ring_slots_needed(queue->vif);
+	skb = skb_peek(&queue->rx_queue);
+	if (!skb)
+		return false;
+
+	needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
+	if (skb_is_gso(skb))
+		needed++;
 
 	do {
 		prod = queue->rx.sring->req_prod;
@@ -2005,8 +2004,7 @@ static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
 
 static bool xenvif_have_rx_work(struct xenvif_queue *queue)
 {
-	return (!skb_queue_empty(&queue->rx_queue)
-		&& xenvif_rx_ring_slots_available(queue))
+	return xenvif_rx_ring_slots_available(queue)
 		|| (queue->vif->stall_timeout &&
 		    (xenvif_rx_queue_stalled(queue)
 		     || xenvif_rx_queue_ready(queue)))
diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 4d5535c4cddf..7116472b4625 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1 +1,2 @@
+source "drivers/ntb/hw/amd/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 175d7c92a569..532e0859b4a1 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1 +1,2 @@
+obj-$(CONFIG_NTB_AMD)	+= amd/
 obj-$(CONFIG_NTB_INTEL)	+= intel/
diff --git a/drivers/ntb/hw/amd/Kconfig b/drivers/ntb/hw/amd/Kconfig
new file mode 100644
index 000000000000..cfe903cd9514
--- /dev/null
+++ b/drivers/ntb/hw/amd/Kconfig
@@ -0,0 +1,7 @@
+config NTB_AMD
+	tristate "AMD Non-Transparent Bridge support"
+	depends on X86_64
+	help
+	 This driver supports AMD NTB on capable Zeppelin hardware.
+
+	 If unsure, say N.
diff --git a/drivers/ntb/hw/amd/Makefile b/drivers/ntb/hw/amd/Makefile
new file mode 100644
index 000000000000..ad54da917563
--- /dev/null
+++ b/drivers/ntb/hw/amd/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_NTB_AMD) += ntb_hw_amd.o
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
new file mode 100644
index 000000000000..588803ad6847
--- /dev/null
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -0,0 +1,1143 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of AMD Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * AMD PCIe NTB Linux driver
+ *
+ * Contact Information:
+ * Xiangliang Yu <Xiangliang.Yu@amd.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/acpi.h>
+#include <linux/pci.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/ntb.h>
+
+#include "ntb_hw_amd.h"
+
+#define NTB_NAME	"ntb_hw_amd"
+#define NTB_DESC	"AMD(R) PCI-E Non-Transparent Bridge Driver"
+#define NTB_VER		"1.0"
+
+MODULE_DESCRIPTION(NTB_DESC);
+MODULE_VERSION(NTB_VER);
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("AMD Inc.");
+
+static const struct file_operations amd_ntb_debugfs_info;
+static struct dentry *debugfs_dir;
+
+static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
+{
+	if (idx < 0 || idx > ndev->mw_count)
+		return -EINVAL;
+
+	return 1 << idx;
+}
+
+static int amd_ntb_mw_count(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->mw_count;
+}
+
+static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
+				phys_addr_t *base,
+				resource_size_t *size,
+				resource_size_t *align,
+				resource_size_t *align_size)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	int bar;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	if (base)
+		*base = pci_resource_start(ndev->ntb.pdev, bar);
+
+	if (size)
+		*size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (align)
+		*align = SZ_4K;
+
+	if (align_size)
+		*align_size = 1;
+
+	return 0;
+}
+
+static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+				dma_addr_t addr, resource_size_t size)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	unsigned long xlat_reg, limit_reg = 0;
+	resource_size_t mw_size;
+	void __iomem *mmio, *peer_mmio;
+	u64 base_addr, limit, reg_val;
+	int bar;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	mw_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	/* make sure the range fits in the usable mw size */
+	if (size > mw_size)
+		return -EINVAL;
+
+	mmio = ndev->self_mmio;
+	peer_mmio = ndev->peer_mmio;
+
+	base_addr = pci_resource_start(ndev->ntb.pdev, bar);
+
+	if (bar != 1) {
+		xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3);
+		limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3);
+
+		/* Set the limit if supported */
+		limit = base_addr + size;
+
+		/* set and verify setting the translation address */
+		write64(addr, peer_mmio + xlat_reg);
+		reg_val = read64(peer_mmio + xlat_reg);
+		if (reg_val != addr) {
+			write64(0, peer_mmio + xlat_reg);
+			return -EIO;
+		}
+
+		/* set and verify setting the limit */
+		write64(limit, mmio + limit_reg);
+		reg_val = read64(mmio + limit_reg);
+		if (reg_val != limit) {
+			write64(base_addr, mmio + limit_reg);
+			write64(0, peer_mmio + xlat_reg);
+			return -EIO;
+		}
+	} else {
+		xlat_reg = AMD_BAR1XLAT_OFFSET;
+		limit_reg = AMD_BAR1LMT_OFFSET;
+
+		/* split bar addr range must all be 32 bit */
+		if (addr & (~0ull << 32))
+			return -EINVAL;
+		if ((addr + size) & (~0ull << 32))
+			return -EINVAL;
+
+		/* Set the limit if supported */
+		limit = base_addr + size;
+
+		/* set and verify setting the translation address */
+		write64(addr, peer_mmio + xlat_reg);
+		reg_val = read64(peer_mmio + xlat_reg);
+		if (reg_val != addr) {
+			write64(0, peer_mmio + xlat_reg);
+			return -EIO;
+		}
+
+		/* set and verify setting the limit */
+		writel(limit, mmio + limit_reg);
+		reg_val = readl(mmio + limit_reg);
+		if (reg_val != limit) {
+			writel(base_addr, mmio + limit_reg);
+			writel(0, peer_mmio + xlat_reg);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+static int amd_link_is_up(struct amd_ntb_dev *ndev)
+{
+	if (!ndev->peer_sta)
+		return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
+
+	/* If peer_sta is reset or D0 event, the ISR has
+	 * started a timer to check link status of hardware.
+	 * So here just clear status bit. And if peer_sta is
+	 * D3 or PME_TO, D0/reset event will be happened when
+	 * system wakeup/poweron, so do nothing here.
+	 */
+	if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
+		ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
+	else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
+		ndev->peer_sta = 0;
+
+	return 0;
+}
+
+static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+			      enum ntb_speed *speed,
+			      enum ntb_width *width)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	int ret = 0;
+
+	if (amd_link_is_up(ndev)) {
+		if (speed)
+			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+		if (width)
+			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+
+		dev_dbg(ndev_dev(ndev), "link is up.\n");
+
+		ret = 1;
+	} else {
+		if (speed)
+			*speed = NTB_SPEED_NONE;
+		if (width)
+			*width = NTB_WIDTH_NONE;
+
+		dev_dbg(ndev_dev(ndev), "link is down.\n");
+	}
+
+	return ret;
+}
+
+static int amd_ntb_link_enable(struct ntb_dev *ntb,
+			       enum ntb_speed max_speed,
+			       enum ntb_width max_width)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 ntb_ctl;
+
+	/* Enable event interrupt */
+	ndev->int_mask &= ~AMD_EVENT_INTMASK;
+	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+	if (ndev->ntb.topo == NTB_TOPO_SEC)
+		return -EINVAL;
+	dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+	ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
+	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+	return 0;
+}
+
+static int amd_ntb_link_disable(struct ntb_dev *ntb)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 ntb_ctl;
+
+	/* Disable event interrupt */
+	ndev->int_mask |= AMD_EVENT_INTMASK;
+	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+	if (ndev->ntb.topo == NTB_TOPO_SEC)
+		return -EINVAL;
+	dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+	ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
+	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+	return 0;
+}
+
+static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->db_valid_mask;
+}
+
+static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->db_count;
+}
+
+static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+
+	if (db_vector < 0 || db_vector > ndev->db_count)
+		return 0;
+
+	return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
+}
+
+static u64 amd_ntb_db_read(struct ntb_dev *ntb)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+
+	return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
+}
+
+static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+
+	writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
+
+	return 0;
+}
+
+static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	unsigned long flags;
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ndev->db_mask_lock, flags);
+	ndev->db_mask |= db_bits;
+	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
+	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
+
+	return 0;
+}
+
+static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	unsigned long flags;
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ndev->db_mask_lock, flags);
+	ndev->db_mask &= ~db_bits;
+	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
+	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
+
+	return 0;
+}
+
+static int amd_ntb_peer_db_addr(struct ntb_dev *ntb,
+				phys_addr_t *db_addr,
+				resource_size_t *db_size)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+
+	if (db_addr)
+		*db_addr = (phys_addr_t)(ndev->peer_mmio + AMD_DBREQ_OFFSET);
+	if (db_size)
+		*db_size = sizeof(u32);
+
+	return 0;
+}
+
+static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+
+	writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
+
+	return 0;
+}
+
+static int amd_ntb_spad_count(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->spad_count;
+}
+
+static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return 0;
+
+	offset = ndev->self_spad + (idx << 2);
+	return readl(mmio + AMD_SPAD_OFFSET + offset);
+}
+
+static int amd_ntb_spad_write(struct ntb_dev *ntb,
+			      int idx, u32 val)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = ndev->self_spad + (idx << 2);
+	writel(val, mmio + AMD_SPAD_OFFSET + offset);
+
+	return 0;
+}
+
+static int amd_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+				  phys_addr_t *spad_addr)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	if (spad_addr)
+		*spad_addr = (phys_addr_t)(ndev->self_mmio + AMD_SPAD_OFFSET +
+					   ndev->peer_spad + (idx << 2));
+	return 0;
+}
+
+static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = ndev->peer_spad + (idx << 2);
+	return readl(mmio + AMD_SPAD_OFFSET + offset);
+}
+
+static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
+				   int idx, u32 val)
+{
+	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+	void __iomem *mmio = ndev->self_mmio;
+	u32 offset;
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	offset = ndev->peer_spad + (idx << 2);
+	writel(val, mmio + AMD_SPAD_OFFSET + offset);
+
+	return 0;
+}
+
+static const struct ntb_dev_ops amd_ntb_ops = {
+	.mw_count		= amd_ntb_mw_count,
+	.mw_get_range		= amd_ntb_mw_get_range,
+	.mw_set_trans		= amd_ntb_mw_set_trans,
+	.link_is_up		= amd_ntb_link_is_up,
+	.link_enable		= amd_ntb_link_enable,
+	.link_disable		= amd_ntb_link_disable,
+	.db_valid_mask		= amd_ntb_db_valid_mask,
+	.db_vector_count	= amd_ntb_db_vector_count,
+	.db_vector_mask		= amd_ntb_db_vector_mask,
+	.db_read		= amd_ntb_db_read,
+	.db_clear		= amd_ntb_db_clear,
+	.db_set_mask		= amd_ntb_db_set_mask,
+	.db_clear_mask		= amd_ntb_db_clear_mask,
+	.peer_db_addr		= amd_ntb_peer_db_addr,
+	.peer_db_set		= amd_ntb_peer_db_set,
+	.spad_count		= amd_ntb_spad_count,
+	.spad_read		= amd_ntb_spad_read,
+	.spad_write		= amd_ntb_spad_write,
+	.peer_spad_addr		= amd_ntb_peer_spad_addr,
+	.peer_spad_read		= amd_ntb_peer_spad_read,
+	.peer_spad_write	= amd_ntb_peer_spad_write,
+};
+
+static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	int reg;
+
+	reg = readl(mmio + AMD_SMUACK_OFFSET);
+	reg |= bit;
+	writel(reg, mmio + AMD_SMUACK_OFFSET);
+
+	ndev->peer_sta |= bit;
+}
+
+static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	u32 status;
+
+	status = readl(mmio + AMD_INTSTAT_OFFSET);
+	if (!(status & AMD_EVENT_INTMASK))
+		return;
+
+	dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec);
+
+	status &= AMD_EVENT_INTMASK;
+	switch (status) {
+	case AMD_PEER_FLUSH_EVENT:
+		dev_info(ndev_dev(ndev), "Flush is done.\n");
+		break;
+	case AMD_PEER_RESET_EVENT:
+		amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
+
+		/* link down first */
+		ntb_link_event(&ndev->ntb);
+		/* polling peer status */
+		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
+
+		break;
+	case AMD_PEER_D3_EVENT:
+	case AMD_PEER_PMETO_EVENT:
+		amd_ack_smu(ndev, status);
+
+		/* link down */
+		ntb_link_event(&ndev->ntb);
+
+		break;
+	case AMD_PEER_D0_EVENT:
+		mmio = ndev->peer_mmio;
+		status = readl(mmio + AMD_PMESTAT_OFFSET);
+		/* check if this is WAKEUP event */
+		if (status & 0x1)
+			dev_info(ndev_dev(ndev), "Wakeup is done.\n");
+
+		amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
+
+		/* start a timer to poll link status */
+		schedule_delayed_work(&ndev->hb_timer,
+				      AMD_LINK_HB_TIMEOUT);
+		break;
+	default:
+		dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status);
+		break;
+	}
+}
+
+static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
+{
+	dev_dbg(ndev_dev(ndev), "vec %d\n", vec);
+
+	if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
+		amd_handle_event(ndev, vec);
+
+	if (vec < AMD_DB_CNT)
+		ntb_db_event(&ndev->ntb, vec);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ndev_vec_isr(int irq, void *dev)
+{
+	struct amd_ntb_vec *nvec = dev;
+
+	return ndev_interrupt(nvec->ndev, nvec->num);
+}
+
+static irqreturn_t ndev_irq_isr(int irq, void *dev)
+{
+	struct amd_ntb_dev *ndev = dev;
+
+	return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
+}
+
+static int ndev_init_isr(struct amd_ntb_dev *ndev,
+			 int msix_min, int msix_max)
+{
+	struct pci_dev *pdev;
+	int rc, i, msix_count, node;
+
+	pdev = ndev_pdev(ndev);
+
+	node = dev_to_node(&pdev->dev);
+
+	ndev->db_mask = ndev->db_valid_mask;
+
+	/* Try to set up msix irq */
+	ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
+				 GFP_KERNEL, node);
+	if (!ndev->vec)
+		goto err_msix_vec_alloc;
+
+	ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
+				  GFP_KERNEL, node);
+	if (!ndev->msix)
+		goto err_msix_alloc;
+
+	for (i = 0; i < msix_max; ++i)
+		ndev->msix[i].entry = i;
+
+	msix_count = pci_enable_msix_range(pdev, ndev->msix,
+					   msix_min, msix_max);
+	if (msix_count < 0)
+		goto err_msix_enable;
+
+	/* NOTE: Disable MSIX if msix count is less than 16 because of
+	 * hardware limitation.
+	 */
+	if (msix_count < msix_min) {
+		pci_disable_msix(pdev);
+		goto err_msix_enable;
+	}
+
+	for (i = 0; i < msix_count; ++i) {
+		ndev->vec[i].ndev = ndev;
+		ndev->vec[i].num = i;
+		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
+				 "ndev_vec_isr", &ndev->vec[i]);
+		if (rc)
+			goto err_msix_request;
+	}
+
+	dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
+	ndev->db_count = msix_min;
+	ndev->msix_vec_count = msix_max;
+	return 0;
+
+err_msix_request:
+	while (i-- > 0)
+		free_irq(ndev->msix[i].vector, ndev);
+	pci_disable_msix(pdev);
+err_msix_enable:
+	kfree(ndev->msix);
+err_msix_alloc:
+	kfree(ndev->vec);
+err_msix_vec_alloc:
+	ndev->msix = NULL;
+	ndev->vec = NULL;
+
+	/* Try to set up msi irq */
+	rc = pci_enable_msi(pdev);
+	if (rc)
+		goto err_msi_enable;
+
+	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
+			 "ndev_irq_isr", ndev);
+	if (rc)
+		goto err_msi_request;
+
+	dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
+	ndev->db_count = 1;
+	ndev->msix_vec_count = 1;
+	return 0;
+
+err_msi_request:
+	pci_disable_msi(pdev);
+err_msi_enable:
+
+	/* Try to set up intx irq */
+	pci_intx(pdev, 1);
+
+	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
+			 "ndev_irq_isr", ndev);
+	if (rc)
+		goto err_intx_request;
+
+	dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
+	ndev->db_count = 1;
+	ndev->msix_vec_count = 1;
+	return 0;
+
+err_intx_request:
+	return rc;
+}
+
+static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
+{
+	struct pci_dev *pdev;
+	void __iomem *mmio = ndev->self_mmio;
+	int i;
+
+	pdev = ndev_pdev(ndev);
+
+	/* Mask all doorbell interrupts */
+	ndev->db_mask = ndev->db_valid_mask;
+	writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
+
+	if (ndev->msix) {
+		i = ndev->msix_vec_count;
+		while (i--)
+			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
+		pci_disable_msix(pdev);
+		kfree(ndev->msix);
+		kfree(ndev->vec);
+	} else {
+		free_irq(pdev->irq, ndev);
+		if (pci_dev_msi_enabled(pdev))
+			pci_disable_msi(pdev);
+		else
+			pci_intx(pdev, 0);
+	}
+}
+
+static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
+				 size_t count, loff_t *offp)
+{
+	struct amd_ntb_dev *ndev;
+	void __iomem *mmio;
+	char *buf;
+	size_t buf_size;
+	ssize_t ret, off;
+	union { u64 v64; u32 v32; u16 v16; } u;
+
+	ndev = filp->private_data;
+	mmio = ndev->self_mmio;
+
+	buf_size = min(count, 0x800ul);
+
+	buf = kmalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	off = 0;
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB Device Information:\n");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Connection Topology -\t%s\n",
+			 ntb_topo_string(ndev->ntb.topo));
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
+
+	if (!amd_link_is_up(ndev)) {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tDown\n");
+	} else {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tUp\n");
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Speed -\t\tPCI-E Gen %u\n",
+				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Width -\t\tx%u\n",
+				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Memory Window Count -\t%u\n", ndev->mw_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Scratchpad Count -\t%u\n", ndev->spad_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Count -\t%u\n", ndev->db_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
+
+	u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
+
+	u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Incoming XLAT:\n");
+
+	u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "XLAT1 -\t\t%#018llx\n", u.v64);
+
+	u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "XLAT23 -\t\t%#018llx\n", u.v64);
+
+	u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "XLAT45 -\t\t%#018llx\n", u.v64);
+
+	u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LMT1 -\t\t\t%#06x\n", u.v32);
+
+	u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LMT23 -\t\t\t%#018llx\n", u.v64);
+
+	u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LMT45 -\t\t\t%#018llx\n", u.v64);
+
+	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
+	kfree(buf);
+	return ret;
+}
+
+static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
+{
+	if (!debugfs_dir) {
+		ndev->debugfs_dir = NULL;
+		ndev->debugfs_info = NULL;
+	} else {
+		ndev->debugfs_dir =
+			debugfs_create_dir(ndev_name(ndev), debugfs_dir);
+		if (!ndev->debugfs_dir)
+			ndev->debugfs_info = NULL;
+		else
+			ndev->debugfs_info =
+				debugfs_create_file("info", S_IRUSR,
+						    ndev->debugfs_dir, ndev,
+						    &amd_ntb_debugfs_info);
+	}
+}
+
+static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
+{
+	debugfs_remove_recursive(ndev->debugfs_dir);
+}
+
+static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
+				    struct pci_dev *pdev)
+{
+	ndev->ntb.pdev = pdev;
+	ndev->ntb.topo = NTB_TOPO_NONE;
+	ndev->ntb.ops = &amd_ntb_ops;
+	ndev->int_mask = AMD_EVENT_INTMASK;
+	spin_lock_init(&ndev->db_mask_lock);
+}
+
+static int amd_poll_link(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->peer_mmio;
+	u32 reg, stat;
+	int rc;
+
+	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
+	reg &= NTB_LIN_STA_ACTIVE_BIT;
+
+	dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg);
+
+	if (reg == ndev->cntl_sta)
+		return 0;
+
+	ndev->cntl_sta = reg;
+
+	rc = pci_read_config_dword(ndev->ntb.pdev,
+				   AMD_LINK_STATUS_OFFSET, &stat);
+	if (rc)
+		return 0;
+	ndev->lnk_sta = stat;
+
+	return 1;
+}
+
+static void amd_link_hb(struct work_struct *work)
+{
+	struct amd_ntb_dev *ndev = hb_ndev(work);
+
+	if (amd_poll_link(ndev))
+		ntb_link_event(&ndev->ntb);
+
+	if (!amd_link_is_up(ndev))
+		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
+}
+
+static int amd_init_isr(struct amd_ntb_dev *ndev)
+{
+	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
+}
+
+static void amd_init_side_info(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	unsigned int reg;
+
+	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
+	if (!(reg & AMD_SIDE_READY)) {
+		reg |= AMD_SIDE_READY;
+		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
+	}
+}
+
+static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	unsigned int reg;
+
+	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
+	if (reg & AMD_SIDE_READY) {
+		reg &= ~AMD_SIDE_READY;
+		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
+		readl(mmio + AMD_SIDEINFO_OFFSET);
+	}
+}
+
+static int amd_init_ntb(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+
+	ndev->mw_count = AMD_MW_CNT;
+	ndev->spad_count = AMD_SPADS_CNT;
+	ndev->db_count = AMD_DB_CNT;
+
+	switch (ndev->ntb.topo) {
+	case NTB_TOPO_PRI:
+	case NTB_TOPO_SEC:
+		ndev->spad_count >>= 1;
+		if (ndev->ntb.topo == NTB_TOPO_PRI) {
+			ndev->self_spad = 0;
+			ndev->peer_spad = 0x20;
+		} else {
+			ndev->self_spad = 0x20;
+			ndev->peer_spad = 0;
+		}
+
+		INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
+		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
+
+		break;
+	default:
+		dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n");
+		return -EINVAL;
+	}
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+	/* Mask event interrupts */
+	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+	return 0;
+}
+
+static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	u32 info;
+
+	info = readl(mmio + AMD_SIDEINFO_OFFSET);
+	if (info & AMD_SIDE_MASK)
+		return NTB_TOPO_SEC;
+	else
+		return NTB_TOPO_PRI;
+}
+
+static int amd_init_dev(struct amd_ntb_dev *ndev)
+{
+	struct pci_dev *pdev;
+	int rc = 0;
+
+	pdev = ndev_pdev(ndev);
+
+	ndev->ntb.topo = amd_get_topo(ndev);
+	dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n",
+		ntb_topo_string(ndev->ntb.topo));
+
+	rc = amd_init_ntb(ndev);
+	if (rc)
+		return rc;
+
+	rc = amd_init_isr(ndev);
+	if (rc) {
+		dev_err(ndev_dev(ndev), "fail to init isr.\n");
+		return rc;
+	}
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+	return 0;
+}
+
+static void amd_deinit_dev(struct amd_ntb_dev *ndev)
+{
+	cancel_delayed_work_sync(&ndev->hb_timer);
+
+	ndev_deinit_isr(ndev);
+}
+
+static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
+			    struct pci_dev *pdev)
+{
+	int rc;
+
+	pci_set_drvdata(pdev, ndev);
+
+	rc = pci_enable_device(pdev);
+	if (rc)
+		goto err_pci_enable;
+
+	rc = pci_request_regions(pdev, NTB_NAME);
+	if (rc)
+		goto err_pci_regions;
+
+	pci_set_master(pdev);
+
+	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (rc)
+			goto err_dma_mask;
+		dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
+	}
+
+	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (rc)
+			goto err_dma_mask;
+		dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
+	}
+
+	ndev->self_mmio = pci_iomap(pdev, 0, 0);
+	if (!ndev->self_mmio) {
+		rc = -EIO;
+		goto err_dma_mask;
+	}
+	ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
+
+	return 0;
+
+err_dma_mask:
+	pci_clear_master(pdev);
+err_pci_regions:
+	pci_disable_device(pdev);
+err_pci_enable:
+	pci_set_drvdata(pdev, NULL);
+	return rc;
+}
+
+static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
+{
+	struct pci_dev *pdev = ndev_pdev(ndev);
+
+	pci_iounmap(pdev, ndev->self_mmio);
+
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static int amd_ntb_pci_probe(struct pci_dev *pdev,
+			     const struct pci_device_id *id)
+{
+	struct amd_ntb_dev *ndev;
+	int rc, node;
+
+	node = dev_to_node(&pdev->dev);
+
+	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
+	if (!ndev) {
+		rc = -ENOMEM;
+		goto err_ndev;
+	}
+
+	ndev_init_struct(ndev, pdev);
+
+	rc = amd_ntb_init_pci(ndev, pdev);
+	if (rc)
+		goto err_init_pci;
+
+	rc = amd_init_dev(ndev);
+	if (rc)
+		goto err_init_dev;
+
+	/* write side info */
+	amd_init_side_info(ndev);
+
+	amd_poll_link(ndev);
+
+	ndev_init_debugfs(ndev);
+
+	rc = ntb_register_device(&ndev->ntb);
+	if (rc)
+		goto err_register;
+
+	dev_info(&pdev->dev, "NTB device registered.\n");
+
+	return 0;
+
+err_register:
+	ndev_deinit_debugfs(ndev);
+	amd_deinit_dev(ndev);
+err_init_dev:
+	amd_ntb_deinit_pci(ndev);
+err_init_pci:
+	kfree(ndev);
+err_ndev:
+	return rc;
+}
+
+static void amd_ntb_pci_remove(struct pci_dev *pdev)
+{
+	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
+
+	ntb_unregister_device(&ndev->ntb);
+	ndev_deinit_debugfs(ndev);
+	amd_deinit_side_info(ndev);
+	amd_deinit_dev(ndev);
+	amd_ntb_deinit_pci(ndev);
+	kfree(ndev);
+}
+
+static const struct file_operations amd_ntb_debugfs_info = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = ndev_debugfs_read,
+};
+
+static const struct pci_device_id amd_ntb_pci_tbl[] = {
+	{PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
+	{0}
+};
+MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
+
+static struct pci_driver amd_ntb_pci_driver = {
+	.name		= KBUILD_MODNAME,
+	.id_table	= amd_ntb_pci_tbl,
+	.probe		= amd_ntb_pci_probe,
+	.remove		= amd_ntb_pci_remove,
+};
+
+static int __init amd_ntb_pci_driver_init(void)
+{
+	pr_info("%s %s\n", NTB_DESC, NTB_VER);
+
+	if (debugfs_initialized())
+		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+	return pci_register_driver(&amd_ntb_pci_driver);
+}
+module_init(amd_ntb_pci_driver_init);
+
+static void __exit amd_ntb_pci_driver_exit(void)
+{
+	pci_unregister_driver(&amd_ntb_pci_driver);
+	debugfs_remove_recursive(debugfs_dir);
+}
+module_exit(amd_ntb_pci_driver_exit);
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.h b/drivers/ntb/hw/amd/ntb_hw_amd.h
new file mode 100644
index 000000000000..2eac3cd3e646
--- /dev/null
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.h
@@ -0,0 +1,217 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of AMD Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * AMD PCIe NTB Linux driver
+ *
+ * Contact Information:
+ * Xiangliang Yu <Xiangliang.Yu@amd.com>
+ */
+
+#ifndef NTB_HW_AMD_H
+#define NTB_HW_AMD_H
+
+#include <linux/ntb.h>
+#include <linux/pci.h>
+
+#define PCI_DEVICE_ID_AMD_NTB	0x145B
+#define AMD_LINK_HB_TIMEOUT	msecs_to_jiffies(1000)
+#define AMD_LINK_STATUS_OFFSET	0x68
+#define NTB_LIN_STA_ACTIVE_BIT	0x00000002
+#define NTB_LNK_STA_SPEED_MASK	0x000F0000
+#define NTB_LNK_STA_WIDTH_MASK	0x03F00000
+#define NTB_LNK_STA_ACTIVE(x)	(!!((x) & NTB_LIN_STA_ACTIVE_BIT))
+#define NTB_LNK_STA_SPEED(x)	(((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
+#define NTB_LNK_STA_WIDTH(x)	(((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
+
+#ifndef read64
+#ifdef readq
+#define read64 readq
+#else
+#define read64 _read64
+static inline u64 _read64(void __iomem *mmio)
+{
+	u64 low, high;
+
+	low = readl(mmio);
+	high = readl(mmio + sizeof(u32));
+	return low | (high << 32);
+}
+#endif
+#endif
+
+#ifndef write64
+#ifdef writeq
+#define write64 writeq
+#else
+#define write64 _write64
+static inline void _write64(u64 val, void __iomem *mmio)
+{
+	writel(val, mmio);
+	writel(val >> 32, mmio + sizeof(u32));
+}
+#endif
+#endif
+
+enum {
+	/* AMD NTB Capability */
+	AMD_MW_CNT		= 3,
+	AMD_DB_CNT		= 16,
+	AMD_MSIX_VECTOR_CNT	= 24,
+	AMD_SPADS_CNT		= 16,
+
+	/*  AMD NTB register offset */
+	AMD_CNTL_OFFSET		= 0x200,
+
+	/* NTB control register bits */
+	PMM_REG_CTL		= BIT(21),
+	SMM_REG_CTL		= BIT(20),
+	SMM_REG_ACC_PATH	= BIT(18),
+	PMM_REG_ACC_PATH	= BIT(17),
+	NTB_CLK_EN		= BIT(16),
+
+	AMD_STA_OFFSET		= 0x204,
+	AMD_PGSLV_OFFSET	= 0x208,
+	AMD_SPAD_MUX_OFFSET	= 0x20C,
+	AMD_SPAD_OFFSET		= 0x210,
+	AMD_RSMU_HCID		= 0x250,
+	AMD_RSMU_SIID		= 0x254,
+	AMD_PSION_OFFSET	= 0x300,
+	AMD_SSION_OFFSET	= 0x330,
+	AMD_MMINDEX_OFFSET	= 0x400,
+	AMD_MMDATA_OFFSET	= 0x404,
+	AMD_SIDEINFO_OFFSET	= 0x408,
+
+	AMD_SIDE_MASK		= BIT(0),
+	AMD_SIDE_READY		= BIT(1),
+
+	/* limit register */
+	AMD_ROMBARLMT_OFFSET	= 0x410,
+	AMD_BAR1LMT_OFFSET	= 0x414,
+	AMD_BAR23LMT_OFFSET	= 0x418,
+	AMD_BAR45LMT_OFFSET	= 0x420,
+	/* xlat address */
+	AMD_POMBARXLAT_OFFSET	= 0x428,
+	AMD_BAR1XLAT_OFFSET	= 0x430,
+	AMD_BAR23XLAT_OFFSET	= 0x438,
+	AMD_BAR45XLAT_OFFSET	= 0x440,
+	/* doorbell and interrupt */
+	AMD_DBFM_OFFSET		= 0x450,
+	AMD_DBREQ_OFFSET	= 0x454,
+	AMD_MIRRDBSTAT_OFFSET	= 0x458,
+	AMD_DBMASK_OFFSET	= 0x45C,
+	AMD_DBSTAT_OFFSET	= 0x460,
+	AMD_INTMASK_OFFSET	= 0x470,
+	AMD_INTSTAT_OFFSET	= 0x474,
+
+	/* event type */
+	AMD_PEER_FLUSH_EVENT	= BIT(0),
+	AMD_PEER_RESET_EVENT	= BIT(1),
+	AMD_PEER_D3_EVENT	= BIT(2),
+	AMD_PEER_PMETO_EVENT	= BIT(3),
+	AMD_PEER_D0_EVENT	= BIT(4),
+	AMD_EVENT_INTMASK	= (AMD_PEER_FLUSH_EVENT |
+				AMD_PEER_RESET_EVENT | AMD_PEER_D3_EVENT |
+				AMD_PEER_PMETO_EVENT | AMD_PEER_D0_EVENT),
+
+	AMD_PMESTAT_OFFSET	= 0x480,
+	AMD_PMSGTRIG_OFFSET	= 0x490,
+	AMD_LTRLATENCY_OFFSET	= 0x494,
+	AMD_FLUSHTRIG_OFFSET	= 0x498,
+
+	/* SMU register*/
+	AMD_SMUACK_OFFSET	= 0x4A0,
+	AMD_SINRST_OFFSET	= 0x4A4,
+	AMD_RSPNUM_OFFSET	= 0x4A8,
+	AMD_SMU_SPADMUTEX	= 0x4B0,
+	AMD_SMU_SPADOFFSET	= 0x4B4,
+
+	AMD_PEER_OFFSET		= 0x400,
+};
+
+struct amd_ntb_dev;
+
+struct amd_ntb_vec {
+	struct amd_ntb_dev	*ndev;
+	int			num;
+};
+
+struct amd_ntb_dev {
+	struct ntb_dev ntb;
+
+	u32 ntb_side;
+	u32 lnk_sta;
+	u32 cntl_sta;
+	u32 peer_sta;
+
+	unsigned char mw_count;
+	unsigned char spad_count;
+	unsigned char db_count;
+	unsigned char msix_vec_count;
+
+	u64 db_valid_mask;
+	u64 db_mask;
+	u32 int_mask;
+
+	struct msix_entry *msix;
+	struct amd_ntb_vec *vec;
+
+	/* synchronize rmw access of db_mask and hw reg */
+	spinlock_t db_mask_lock;
+
+	void __iomem *self_mmio;
+	void __iomem *peer_mmio;
+	unsigned int self_spad;
+	unsigned int peer_spad;
+
+	struct delayed_work hb_timer;
+
+	struct dentry *debugfs_dir;
+	struct dentry *debugfs_info;
+};
+
+#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
+#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
+#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
+#define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb)
+#define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work)
+
+#endif
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index a198f8298258..40d04ef5da9e 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -875,7 +875,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
 	limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
 
 	if (bar < 4 || !ndev->bar4_split) {
-		base = ioread64(mmio + base_reg);
+		base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
 
 		/* Set the limit if supported, if size is not mw_size */
 		if (limit_reg && size != mw_size)
@@ -906,7 +906,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
 		if ((addr + size) & (~0ull << 32))
 			return -EINVAL;
 
-		base = ioread32(mmio + base_reg);
+		base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
 
 		/* Set the limit if supported, if size is not mw_size */
 		if (limit_reg && size != mw_size)
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 2eb4addd10d0..3ec149cf6562 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -245,6 +245,9 @@
 #define NTB_UNSAFE_DB			BIT_ULL(0)
 #define NTB_UNSAFE_SPAD			BIT_ULL(1)
 
+#define NTB_BAR_MASK_64			~(0xfull)
+#define NTB_BAR_MASK_32			~(0xfu)
+
 struct intel_ntb_dev;
 
 struct intel_ntb_reg {
@@ -334,7 +337,8 @@ struct intel_ntb_dev {
 #define ndev_pdev(ndev) ((ndev)->ntb.pdev)
 #define ndev_name(ndev) pci_name(ndev_pdev(ndev))
 #define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
-#define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb)
-#define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work)
+#define ntb_ndev(__ntb) container_of(__ntb, struct intel_ntb_dev, ntb)
+#define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \
+				     hb_timer.work)
 
 #endif
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 60654d524858..ec4775f0ec16 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -171,12 +171,14 @@ struct ntb_transport_qp {
 	u64 rx_err_ver;
 	u64 rx_memcpy;
 	u64 rx_async;
+	u64 dma_rx_prep_err;
 	u64 tx_bytes;
 	u64 tx_pkts;
 	u64 tx_ring_full;
 	u64 tx_err_no_buf;
 	u64 tx_memcpy;
 	u64 tx_async;
+	u64 dma_tx_prep_err;
 };
 
 struct ntb_transport_mw {
@@ -249,6 +251,8 @@ enum {
 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
 #define NTB_QP_DEF_NUM_ENTRIES	100
 #define NTB_LINK_DOWN_TIMEOUT	10
+#define DMA_RETRIES		20
+#define DMA_OUT_RESOURCE_TO	50
 
 static void ntb_transport_rxc_db(unsigned long data);
 static const struct ntb_ctx_ops ntb_transport_ops;
@@ -501,6 +505,12 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
 			       "free tx - \t%u\n",
 			       ntb_transport_tx_free_entry(qp));
+	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+			       "DMA tx prep err - \t%llu\n",
+			       qp->dma_tx_prep_err);
+	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+			       "DMA rx prep err - \t%llu\n",
+			       qp->dma_rx_prep_err);
 
 	out_offset += snprintf(buf + out_offset, out_count - out_offset,
 			       "\n");
@@ -726,6 +736,8 @@ static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
 	qp->tx_err_no_buf = 0;
 	qp->tx_memcpy = 0;
 	qp->tx_async = 0;
+	qp->dma_tx_prep_err = 0;
+	qp->dma_rx_prep_err = 0;
 }
 
 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
@@ -1228,6 +1240,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 	struct dmaengine_unmap_data *unmap;
 	dma_cookie_t cookie;
 	void *buf = entry->buf;
+	int retries = 0;
 
 	len = entry->len;
 
@@ -1263,11 +1276,21 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 
 	unmap->from_cnt = 1;
 
-	txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
-					     unmap->addr[0], len,
-					     DMA_PREP_INTERRUPT);
-	if (!txd)
+	for (retries = 0; retries < DMA_RETRIES; retries++) {
+		txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
+						     unmap->addr[0], len,
+						     DMA_PREP_INTERRUPT);
+		if (txd)
+			break;
+
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule_timeout(DMA_OUT_RESOURCE_TO);
+	}
+
+	if (!txd) {
+		qp->dma_rx_prep_err++;
 		goto err_get_unmap;
+	}
 
 	txd->callback = ntb_rx_copy_callback;
 	txd->callback_param = entry;
@@ -1460,6 +1483,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 	void __iomem *offset;
 	size_t len = entry->len;
 	void *buf = entry->buf;
+	int retries = 0;
 
 	offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;
 	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
@@ -1494,10 +1518,20 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 
 	unmap->to_cnt = 1;
 
-	txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len,
-					     DMA_PREP_INTERRUPT);
-	if (!txd)
+	for (retries = 0; retries < DMA_RETRIES; retries++) {
+		txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0],
+						     len, DMA_PREP_INTERRUPT);
+		if (txd)
+			break;
+
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule_timeout(DMA_OUT_RESOURCE_TO);
+	}
+
+	if (!txd) {
+		qp->dma_tx_prep_err++;
 		goto err_get_unmap;
+	}
 
 	txd->callback = ntb_tx_copy_callback;
 	txd->callback_param = entry;
@@ -1532,7 +1566,7 @@ static int ntb_process_tx(struct ntb_transport_qp *qp,
 
 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
 		if (qp->tx_handler)
-			qp->tx_handler(qp->cb_data, qp, NULL, -EIO);
+			qp->tx_handler(qp, qp->cb_data, NULL, -EIO);
 
 		ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
 			     &qp->tx_free_q);
diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig
index 01852f98a843..a5d0eda44438 100644
--- a/drivers/ntb/test/Kconfig
+++ b/drivers/ntb/test/Kconfig
@@ -17,3 +17,11 @@ config NTB_TOOL
 	 functioning at a basic level.
 
 	 If unsure, say N.
+
+config NTB_PERF
+	tristate "NTB RAW Perf Measuring Tool"
+	help
+	 This is a tool to measure raw NTB performance by transferring data
+	 to and from the window without additional software interaction.
+
+	 If unsure, say N.
diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile
index 0ea32a324b6c..9e77e0b761c2 100644
--- a/drivers/ntb/test/Makefile
+++ b/drivers/ntb/test/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
 obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
+obj-$(CONFIG_NTB_PERF) += ntb_perf.o
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
new file mode 100644
index 000000000000..c8a37ba4b4f9
--- /dev/null
+++ b/drivers/ntb/test/ntb_perf.c
@@ -0,0 +1,748 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *   PCIe NTB Perf Linux driver
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kthread.h>
+#include <linux/time.h>
+#include <linux/timer.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/debugfs.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/sizes.h>
+#include <linux/ntb.h>
+
+#define DRIVER_NAME		"ntb_perf"
+#define DRIVER_DESCRIPTION	"PCIe NTB Performance Measurement Tool"
+
+#define DRIVER_LICENSE		"Dual BSD/GPL"
+#define DRIVER_VERSION		"1.0"
+#define DRIVER_AUTHOR		"Dave Jiang <dave.jiang@intel.com>"
+
+#define PERF_LINK_DOWN_TIMEOUT	10
+#define PERF_VERSION		0xffff0001
+#define MAX_THREADS		32
+#define MAX_TEST_SIZE		SZ_1M
+#define MAX_SRCS		32
+#define DMA_OUT_RESOURCE_TO	50
+#define DMA_RETRIES		20
+#define SZ_4G			(1ULL << 32)
+#define MAX_SEG_ORDER		20 /* no larger than 1M for kmalloc buffer */
+
+MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+
+static struct dentry *perf_debugfs_dir;
+
+static unsigned int seg_order = 19; /* 512K */
+module_param(seg_order, uint, 0644);
+MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");
+
+static unsigned int run_order = 32; /* 4G */
+module_param(run_order, uint, 0644);
+MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");
+
+static bool use_dma; /* default to 0 */
+module_param(use_dma, bool, 0644);
+MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
+
+struct perf_mw {
+	phys_addr_t	phys_addr;
+	resource_size_t	phys_size;
+	resource_size_t	xlat_align;
+	resource_size_t	xlat_align_size;
+	void __iomem	*vbase;
+	size_t		xlat_size;
+	size_t		buf_size;
+	void		*virt_addr;
+	dma_addr_t	dma_addr;
+};
+
+struct perf_ctx;
+
+struct pthr_ctx {
+	struct task_struct	*thread;
+	struct perf_ctx		*perf;
+	atomic_t		dma_sync;
+	struct dma_chan		*dma_chan;
+	int			dma_prep_err;
+	int			src_idx;
+	void			*srcs[MAX_SRCS];
+};
+
+struct perf_ctx {
+	struct ntb_dev		*ntb;
+	spinlock_t		db_lock;
+	struct perf_mw		mw;
+	bool			link_is_up;
+	struct work_struct	link_cleanup;
+	struct delayed_work	link_work;
+	struct dentry		*debugfs_node_dir;
+	struct dentry		*debugfs_run;
+	struct dentry		*debugfs_threads;
+	u8			perf_threads;
+	bool			run;
+	struct pthr_ctx		pthr_ctx[MAX_THREADS];
+	atomic_t		tsync;
+};
+
+enum {
+	VERSION = 0,
+	MW_SZ_HIGH,
+	MW_SZ_LOW,
+	SPAD_MSG,
+	SPAD_ACK,
+	MAX_SPAD
+};
+
+static void perf_link_event(void *ctx)
+{
+	struct perf_ctx *perf = ctx;
+
+	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1)
+		schedule_delayed_work(&perf->link_work, 2*HZ);
+	else
+		schedule_work(&perf->link_cleanup);
+}
+
+static void perf_db_event(void *ctx, int vec)
+{
+	struct perf_ctx *perf = ctx;
+	u64 db_bits, db_mask;
+
+	db_mask = ntb_db_vector_mask(perf->ntb, vec);
+	db_bits = ntb_db_read(perf->ntb);
+
+	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
+		vec, db_mask, db_bits);
+}
+
+static const struct ntb_ctx_ops perf_ops = {
+	.link_event = perf_link_event,
+	.db_event = perf_db_event,
+};
+
+static void perf_copy_callback(void *data)
+{
+	struct pthr_ctx *pctx = data;
+
+	atomic_dec(&pctx->dma_sync);
+}
+
+static ssize_t perf_copy(struct pthr_ctx *pctx, char *dst,
+			 char *src, size_t size)
+{
+	struct perf_ctx *perf = pctx->perf;
+	struct dma_async_tx_descriptor *txd;
+	struct dma_chan *chan = pctx->dma_chan;
+	struct dma_device *device;
+	struct dmaengine_unmap_data *unmap;
+	dma_cookie_t cookie;
+	size_t src_off, dst_off;
+	struct perf_mw *mw = &perf->mw;
+	u64 vbase, dst_vaddr;
+	dma_addr_t dst_phys;
+	int retries = 0;
+
+	if (!use_dma) {
+		memcpy_toio(dst, src, size);
+		return size;
+	}
+
+	if (!chan) {
+		dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
+		return -EINVAL;
+	}
+
+	device = chan->device;
+	src_off = (size_t)src & ~PAGE_MASK;
+	dst_off = (size_t)dst & ~PAGE_MASK;
+
+	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
+		return -ENODEV;
+
+	vbase = (u64)(u64 *)mw->vbase;
+	dst_vaddr = (u64)(u64 *)dst;
+	dst_phys = mw->phys_addr + (dst_vaddr - vbase);
+
+	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
+	if (!unmap)
+		return -ENOMEM;
+
+	unmap->len = size;
+	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
+				      src_off, size, DMA_TO_DEVICE);
+	if (dma_mapping_error(device->dev, unmap->addr[0]))
+		goto err_get_unmap;
+
+	unmap->to_cnt = 1;
+
+	do {
+		txd = device->device_prep_dma_memcpy(chan, dst_phys,
+						     unmap->addr[0],
+						     size, DMA_PREP_INTERRUPT);
+		if (!txd) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(DMA_OUT_RESOURCE_TO);
+		}
+	} while (!txd && (++retries < DMA_RETRIES));
+
+	if (!txd) {
+		pctx->dma_prep_err++;
+		goto err_get_unmap;
+	}
+
+	txd->callback = perf_copy_callback;
+	txd->callback_param = pctx;
+	dma_set_unmap(txd, unmap);
+
+	cookie = dmaengine_submit(txd);
+	if (dma_submit_error(cookie))
+		goto err_set_unmap;
+
+	atomic_inc(&pctx->dma_sync);
+	dma_async_issue_pending(chan);
+
+	return size;
+
+err_set_unmap:
+	dmaengine_unmap_put(unmap);
+err_get_unmap:
+	dmaengine_unmap_put(unmap);
+	return 0;
+}
+
+static int perf_move_data(struct pthr_ctx *pctx, char *dst, char *src,
+			  u64 buf_size, u64 win_size, u64 total)
+{
+	int chunks, total_chunks, i;
+	int copied_chunks = 0;
+	u64 copied = 0, result;
+	char *tmp = dst;
+	u64 perf, diff_us;
+	ktime_t kstart, kstop, kdiff;
+
+	chunks = div64_u64(win_size, buf_size);
+	total_chunks = div64_u64(total, buf_size);
+	kstart = ktime_get();
+
+	for (i = 0; i < total_chunks; i++) {
+		result = perf_copy(pctx, tmp, src, buf_size);
+		copied += result;
+		copied_chunks++;
+		if (copied_chunks == chunks) {
+			tmp = dst;
+			copied_chunks = 0;
+		} else
+			tmp += buf_size;
+
+		/* Probably should schedule every 4GB to prevent soft hang. */
+		if (((copied % SZ_4G) == 0) && !use_dma) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(1);
+		}
+	}
+
+	if (use_dma) {
+		pr_info("%s: All DMA descriptors submitted\n", current->comm);
+		while (atomic_read(&pctx->dma_sync) != 0)
+			msleep(20);
+	}
+
+	kstop = ktime_get();
+	kdiff = ktime_sub(kstop, kstart);
+	diff_us = ktime_to_us(kdiff);
+
+	pr_info("%s: copied %llu bytes\n", current->comm, copied);
+
+	pr_info("%s: lasted %llu usecs\n", current->comm, diff_us);
+
+	perf = div64_u64(copied, diff_us);
+
+	pr_info("%s: MBytes/s: %llu\n", current->comm, perf);
+
+	return 0;
+}
+
+static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
+{
+	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
+}
+
+static int ntb_perf_thread(void *data)
+{
+	struct pthr_ctx *pctx = data;
+	struct perf_ctx *perf = pctx->perf;
+	struct pci_dev *pdev = perf->ntb->pdev;
+	struct perf_mw *mw = &perf->mw;
+	char *dst;
+	u64 win_size, buf_size, total;
+	void *src;
+	int rc, node, i;
+	struct dma_chan *dma_chan = NULL;
+
+	pr_info("kthread %s starting...\n", current->comm);
+
+	node = dev_to_node(&pdev->dev);
+
+	if (use_dma && !pctx->dma_chan) {
+		dma_cap_mask_t dma_mask;
+
+		dma_cap_zero(dma_mask);
+		dma_cap_set(DMA_MEMCPY, dma_mask);
+		dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
+					       (void *)(unsigned long)node);
+		if (!dma_chan) {
+			pr_warn("%s: cannot acquire DMA channel, quitting\n",
+				current->comm);
+			return -ENODEV;
+		}
+		pctx->dma_chan = dma_chan;
+	}
+
+	for (i = 0; i < MAX_SRCS; i++) {
+		pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
+		if (!pctx->srcs[i]) {
+			rc = -ENOMEM;
+			goto err;
+		}
+	}
+
+	win_size = mw->phys_size;
+	buf_size = 1ULL << seg_order;
+	total = 1ULL << run_order;
+
+	if (buf_size > MAX_TEST_SIZE)
+		buf_size = MAX_TEST_SIZE;
+
+	dst = (char *)mw->vbase;
+
+	atomic_inc(&perf->tsync);
+	while (atomic_read(&perf->tsync) != perf->perf_threads)
+		schedule();
+
+	src = pctx->srcs[pctx->src_idx];
+	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);
+
+	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);
+
+	atomic_dec(&perf->tsync);
+
+	if (rc < 0) {
+		pr_err("%s: failed\n", current->comm);
+		rc = -ENXIO;
+		goto err;
+	}
+
+	for (i = 0; i < MAX_SRCS; i++) {
+		kfree(pctx->srcs[i]);
+		pctx->srcs[i] = NULL;
+	}
+
+	return 0;
+
+err:
+	for (i = 0; i < MAX_SRCS; i++) {
+		kfree(pctx->srcs[i]);
+		pctx->srcs[i] = NULL;
+	}
+
+	if (dma_chan) {
+		dma_release_channel(dma_chan);
+		pctx->dma_chan = NULL;
+	}
+
+	return rc;
+}
+
+static void perf_free_mw(struct perf_ctx *perf)
+{
+	struct perf_mw *mw = &perf->mw;
+	struct pci_dev *pdev = perf->ntb->pdev;
+
+	if (!mw->virt_addr)
+		return;
+
+	ntb_mw_clear_trans(perf->ntb, 0);
+	dma_free_coherent(&pdev->dev, mw->buf_size,
+			  mw->virt_addr, mw->dma_addr);
+	mw->xlat_size = 0;
+	mw->buf_size = 0;
+	mw->virt_addr = NULL;
+}
+
+static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
+{
+	struct perf_mw *mw = &perf->mw;
+	size_t xlat_size, buf_size;
+
+	if (!size)
+		return -EINVAL;
+
+	xlat_size = round_up(size, mw->xlat_align_size);
+	buf_size = round_up(size, mw->xlat_align);
+
+	if (mw->xlat_size == xlat_size)
+		return 0;
+
+	if (mw->buf_size)
+		perf_free_mw(perf);
+
+	mw->xlat_size = xlat_size;
+	mw->buf_size = buf_size;
+
+	mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
+					   &mw->dma_addr, GFP_KERNEL);
+	if (!mw->virt_addr) {
+		mw->xlat_size = 0;
+		mw->buf_size = 0;
+	}
+
+	return 0;
+}
+
+static void perf_link_work(struct work_struct *work)
+{
+	struct perf_ctx *perf =
+		container_of(work, struct perf_ctx, link_work.work);
+	struct ntb_dev *ndev = perf->ntb;
+	struct pci_dev *pdev = ndev->pdev;
+	u32 val;
+	u64 size;
+	int rc;
+
+	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
+
+	size = perf->mw.phys_size;
+	ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
+	ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
+	ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);
+
+	/* now read what peer wrote */
+	val = ntb_spad_read(ndev, VERSION);
+	if (val != PERF_VERSION) {
+		dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
+		goto out;
+	}
+
+	val = ntb_spad_read(ndev, MW_SZ_HIGH);
+	size = (u64)val << 32;
+
+	val = ntb_spad_read(ndev, MW_SZ_LOW);
+	size |= val;
+
+	dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);
+
+	rc = perf_set_mw(perf, size);
+	if (rc)
+		goto out1;
+
+	perf->link_is_up = true;
+
+	return;
+
+out1:
+	perf_free_mw(perf);
+
+out:
+	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
+		schedule_delayed_work(&perf->link_work,
+				      msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
+}
+
+static void perf_link_cleanup(struct work_struct *work)
+{
+	struct perf_ctx *perf = container_of(work,
+					     struct perf_ctx,
+					     link_cleanup);
+
+	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
+
+	if (!perf->link_is_up)
+		cancel_delayed_work_sync(&perf->link_work);
+}
+
+static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf)
+{
+	struct perf_mw *mw;
+	int rc;
+
+	mw = &perf->mw;
+
+	rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size,
+			      &mw->xlat_align, &mw->xlat_align_size);
+	if (rc)
+		return rc;
+
+	perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
+	if (!mw->vbase)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf,
+				size_t count, loff_t *offp)
+{
+	struct perf_ctx *perf = filp->private_data;
+	char *buf;
+	ssize_t ret, out_offset;
+
+	if (!perf)
+		return 0;
+
+	buf = kmalloc(64, GFP_KERNEL);
+	out_offset = snprintf(buf, 64, "%d\n", perf->run);
+	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
+	kfree(buf);
+
+	return ret;
+}
+
+static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf,
+				 size_t count, loff_t *offp)
+{
+	struct perf_ctx *perf = filp->private_data;
+	int node, i;
+
+	if (!perf->link_is_up)
+		return 0;
+
+	if (perf->perf_threads == 0)
+		return 0;
+
+	if (atomic_read(&perf->tsync) == 0)
+		perf->run = false;
+
+	if (perf->run) {
+		/* lets stop the threads */
+		perf->run = false;
+		for (i = 0; i < MAX_THREADS; i++) {
+			if (perf->pthr_ctx[i].thread) {
+				kthread_stop(perf->pthr_ctx[i].thread);
+				perf->pthr_ctx[i].thread = NULL;
+			} else
+				break;
+		}
+	} else {
+		perf->run = true;
+
+		if (perf->perf_threads > MAX_THREADS) {
+			perf->perf_threads = MAX_THREADS;
+			pr_info("Reset total threads to: %u\n", MAX_THREADS);
+		}
+
+		/* no greater than 1M */
+		if (seg_order > MAX_SEG_ORDER) {
+			seg_order = MAX_SEG_ORDER;
+			pr_info("Fix seg_order to %u\n", seg_order);
+		}
+
+		if (run_order < seg_order) {
+			run_order = seg_order;
+			pr_info("Fix run_order to %u\n", run_order);
+		}
+
+		node = dev_to_node(&perf->ntb->pdev->dev);
+		/* launch kernel thread */
+		for (i = 0; i < perf->perf_threads; i++) {
+			struct pthr_ctx *pctx;
+
+			pctx = &perf->pthr_ctx[i];
+			atomic_set(&pctx->dma_sync, 0);
+			pctx->perf = perf;
+			pctx->thread =
+				kthread_create_on_node(ntb_perf_thread,
+						       (void *)pctx,
+						       node, "ntb_perf %d", i);
+			if (pctx->thread)
+				wake_up_process(pctx->thread);
+			else {
+				perf->run = false;
+				for (i = 0; i < MAX_THREADS; i++) {
+					if (pctx->thread) {
+						kthread_stop(pctx->thread);
+						pctx->thread = NULL;
+					}
+				}
+			}
+
+			if (perf->run == false)
+				return -ENXIO;
+		}
+
+	}
+
+	return count;
+}
+
+static const struct file_operations ntb_perf_debugfs_run = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = debugfs_run_read,
+	.write = debugfs_run_write,
+};
+
+static int perf_debugfs_setup(struct perf_ctx *perf)
+{
+	struct pci_dev *pdev = perf->ntb->pdev;
+
+	if (!debugfs_initialized())
+		return -ENODEV;
+
+	if (!perf_debugfs_dir) {
+		perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+		if (!perf_debugfs_dir)
+			return -ENODEV;
+	}
+
+	perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
+						    perf_debugfs_dir);
+	if (!perf->debugfs_node_dir)
+		return -ENODEV;
+
+	perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
+						perf->debugfs_node_dir, perf,
+						&ntb_perf_debugfs_run);
+	if (!perf->debugfs_run)
+		return -ENODEV;
+
+	perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
+						  perf->debugfs_node_dir,
+						  &perf->perf_threads);
+	if (!perf->debugfs_threads)
+		return -ENODEV;
+
+	return 0;
+}
+
+static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
+{
+	struct pci_dev *pdev = ntb->pdev;
+	struct perf_ctx *perf;
+	int node;
+	int rc = 0;
+
+	node = dev_to_node(&pdev->dev);
+
+	perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
+	if (!perf) {
+		rc = -ENOMEM;
+		goto err_perf;
+	}
+
+	perf->ntb = ntb;
+	perf->perf_threads = 1;
+	atomic_set(&perf->tsync, 0);
+	perf->run = false;
+	spin_lock_init(&perf->db_lock);
+	perf_setup_mw(ntb, perf);
+	INIT_DELAYED_WORK(&perf->link_work, perf_link_work);
+	INIT_WORK(&perf->link_cleanup, perf_link_cleanup);
+
+	rc = ntb_set_ctx(ntb, perf, &perf_ops);
+	if (rc)
+		goto err_ctx;
+
+	perf->link_is_up = false;
+	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	ntb_link_event(ntb);
+
+	rc = perf_debugfs_setup(perf);
+	if (rc)
+		goto err_ctx;
+
+	return 0;
+
+err_ctx:
+	cancel_delayed_work_sync(&perf->link_work);
+	cancel_work_sync(&perf->link_cleanup);
+	kfree(perf);
+err_perf:
+	return rc;
+}
+
+static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
+{
+	struct perf_ctx *perf = ntb->ctx;
+	int i;
+
+	dev_dbg(&perf->ntb->dev, "%s called\n", __func__);
+
+	cancel_delayed_work_sync(&perf->link_work);
+	cancel_work_sync(&perf->link_cleanup);
+
+	ntb_clear_ctx(ntb);
+	ntb_link_disable(ntb);
+
+	debugfs_remove_recursive(perf_debugfs_dir);
+	perf_debugfs_dir = NULL;
+
+	if (use_dma) {
+		for (i = 0; i < MAX_THREADS; i++) {
+			struct pthr_ctx *pctx = &perf->pthr_ctx[i];
+
+			if (pctx->dma_chan)
+				dma_release_channel(pctx->dma_chan);
+		}
+	}
+
+	kfree(perf);
+}
+
+static struct ntb_client perf_client = {
+	.ops = {
+		.probe = perf_probe,
+		.remove = perf_remove,
+	},
+};
+module_ntb_client(perf_client);
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index f9b674bc49db..0cc9048b86e2 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -83,8 +83,7 @@ static ssize_t mode_store(struct device *dev,
 
 		if (strncmp(buf, "pmem\n", n) == 0
 				|| strncmp(buf, "pmem", n) == 0) {
-			/* TODO: allocate from PMEM support */
-			rc = -ENOTTY;
+			nd_pfn->mode = PFN_MODE_PMEM;
 		} else if (strncmp(buf, "ram\n", n) == 0
 				|| strncmp(buf, "ram", n) == 0)
 			nd_pfn->mode = PFN_MODE_RAM;
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index b493ff3fccb2..7edf31671dab 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -21,10 +21,11 @@
 #include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
-#include <linux/memory_hotplug.h>
 #include <linux/moduleparam.h>
 #include <linux/badblocks.h>
+#include <linux/memremap.h>
 #include <linux/vmalloc.h>
+#include <linux/pfn_t.h>
 #include <linux/slab.h>
 #include <linux/pmem.h>
 #include <linux/nd.h>
@@ -40,6 +41,7 @@ struct pmem_device {
 	phys_addr_t		phys_addr;
 	/* when non-zero this device is hosting a 'pfn' instance */
 	phys_addr_t		data_offset;
+	unsigned long		pfn_flags;
 	void __pmem		*virt_addr;
 	size_t			size;
 	struct badblocks	bb;
@@ -135,13 +137,13 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector,
 }
 
 static long pmem_direct_access(struct block_device *bdev, sector_t sector,
-		      void __pmem **kaddr, unsigned long *pfn)
+		      void __pmem **kaddr, pfn_t *pfn)
 {
 	struct pmem_device *pmem = bdev->bd_disk->private_data;
 	resource_size_t offset = sector * 512 + pmem->data_offset;
 
 	*kaddr = pmem->virt_addr + offset;
-	*pfn = (pmem->phys_addr + offset) >> PAGE_SHIFT;
+	*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
 
 	return pmem->size - offset;
 }
@@ -157,6 +159,7 @@ static struct pmem_device *pmem_alloc(struct device *dev,
 		struct resource *res, int id)
 {
 	struct pmem_device *pmem;
+	struct request_queue *q;
 
 	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
 	if (!pmem)
@@ -174,16 +177,26 @@ static struct pmem_device *pmem_alloc(struct device *dev,
 		return ERR_PTR(-EBUSY);
 	}
 
-	if (pmem_should_map_pages(dev))
-		pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res);
-	else
+	q = blk_alloc_queue_node(GFP_KERNEL, dev_to_node(dev));
+	if (!q)
+		return ERR_PTR(-ENOMEM);
+
+	pmem->pfn_flags = PFN_DEV;
+	if (pmem_should_map_pages(dev)) {
+		pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res,
+				&q->q_usage_counter, NULL);
+		pmem->pfn_flags |= PFN_MAP;
+	} else
 		pmem->virt_addr = (void __pmem *) devm_memremap(dev,
 				pmem->phys_addr, pmem->size,
 				ARCH_MEMREMAP_PMEM);
 
-	if (IS_ERR(pmem->virt_addr))
+	if (IS_ERR(pmem->virt_addr)) {
+		blk_cleanup_queue(q);
 		return (void __force *) pmem->virt_addr;
+	}
 
+	pmem->pmem_queue = q;
 	return pmem;
 }
 
@@ -203,10 +216,6 @@ static int pmem_attach_disk(struct device *dev,
 	int nid = dev_to_node(dev);
 	struct gendisk *disk;
 
-	pmem->pmem_queue = blk_alloc_queue_node(GFP_KERNEL, nid);
-	if (!pmem->pmem_queue)
-		return -ENOMEM;
-
 	blk_queue_make_request(pmem->pmem_queue, pmem_make_request);
 	blk_queue_physical_block_size(pmem->pmem_queue, PAGE_SIZE);
 	blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);
@@ -352,12 +361,17 @@ static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
 	struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
 	struct nd_pfn *nd_pfn = to_nd_pfn(ndns->claim);
 	struct device *dev = &nd_pfn->dev;
-	struct vmem_altmap *altmap;
 	struct nd_region *nd_region;
+	struct vmem_altmap *altmap;
 	struct nd_pfn_sb *pfn_sb;
 	struct pmem_device *pmem;
+	struct request_queue *q;
 	phys_addr_t offset;
 	int rc;
+	struct vmem_altmap __altmap = {
+		.base_pfn = __phys_to_pfn(nsio->res.start),
+		.reserve = __phys_to_pfn(SZ_8K),
+	};
 
 	if (!nd_pfn->uuid || !nd_pfn->ndns)
 		return -ENODEV;
@@ -375,6 +389,17 @@ static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
 			return -EINVAL;
 		nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
 		altmap = NULL;
+	} else if (nd_pfn->mode == PFN_MODE_PMEM) {
+		nd_pfn->npfns = (resource_size(&nsio->res) - offset)
+			/ PAGE_SIZE;
+		if (le64_to_cpu(nd_pfn->pfn_sb->npfns) > nd_pfn->npfns)
+			dev_info(&nd_pfn->dev,
+					"number of pfns truncated from %lld to %ld\n",
+					le64_to_cpu(nd_pfn->pfn_sb->npfns),
+					nd_pfn->npfns);
+		altmap = & __altmap;
+		altmap->free = __phys_to_pfn(offset - SZ_8K);
+		altmap->alloc = 0;
 	} else {
 		rc = -ENXIO;
 		goto err;
@@ -382,8 +407,11 @@ static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
 
 	/* establish pfn range for lookup, and switch to direct map */
 	pmem = dev_get_drvdata(dev);
+	q = pmem->pmem_queue;
 	devm_memunmap(dev, (void __force *) pmem->virt_addr);
-	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res);
+	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res,
+			&q->q_usage_counter, altmap);
+	pmem->pfn_flags |= PFN_MAP;
 	if (IS_ERR(pmem->virt_addr)) {
 		rc = PTR_ERR(pmem->virt_addr);
 		goto err;
@@ -424,19 +452,22 @@ static int nd_pmem_probe(struct device *dev)
 		return -ENOMEM;
 	nvdimm_namespace_add_poison(ndns, &pmem->bb, 0);
 
-	if (is_nd_btt(dev))
+	if (is_nd_btt(dev)) {
+		/* btt allocates its own request_queue */
+		blk_cleanup_queue(pmem->pmem_queue);
+		pmem->pmem_queue = NULL;
 		return nvdimm_namespace_attach_btt(ndns);
+	}
 
 	if (is_nd_pfn(dev))
 		return nvdimm_namespace_attach_pfn(ndns);
 
-	if (nd_btt_probe(ndns, pmem) == 0) {
-		/* we'll come back as btt-pmem */
-		return -ENXIO;
-	}
-
-	if (nd_pfn_probe(ndns, pmem) == 0) {
-		/* we'll come back as pfn-pmem */
+	if (nd_btt_probe(ndns, pmem) == 0 || nd_pfn_probe(ndns, pmem) == 0) {
+		/*
+		 * We'll come back as either btt-pmem, or pfn-pmem, so
+		 * drop the queue allocation for now.
+		 */
+		blk_cleanup_queue(pmem->pmem_queue);
 		return -ENXIO;
 	}
 
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 002a94abdbc4..5d6237391dcd 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -8,3 +8,14 @@ config BLK_DEV_NVME
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called nvme.
+
+config BLK_DEV_NVME_SCSI
+	bool "SCSI emulation for NVMe device nodes"
+	depends on BLK_DEV_NVME
+	---help---
+	  This adds support for the SG_IO ioctl on the NVMe character
+	  and block devices nodes, as well a a translation for a small
+	  number of selected SCSI commands to NVMe commands to the NVMe
+	  driver.  If you don't know what this means you probably want
+	  to say N here, and if you know what it means you probably
+	  want to say N as well.
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index a5fe23952586..51bf90871549 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -1,5 +1,6 @@
 
 obj-$(CONFIG_BLK_DEV_NVME)     += nvme.o
 
-lightnvm-$(CONFIG_NVM)	:= lightnvm.o
-nvme-y		+= pci.o scsi.o $(lightnvm-y)
+lightnvm-$(CONFIG_NVM)			:= lightnvm.o
+nvme-y					+= core.o pci.o $(lightnvm-y)
+nvme-$(CONFIG_BLK_DEV_NVME_SCSI)        += scsi.o
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
new file mode 100644
index 000000000000..c5bf001af559
--- /dev/null
+++ b/drivers/nvme/host/core.c
@@ -0,0 +1,1472 @@
+/*
+ * NVM Express device driver
+ * Copyright (c) 2011-2014, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/hdreg.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/list_sort.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/pr.h>
+#include <linux/ptrace.h>
+#include <linux/nvme_ioctl.h>
+#include <linux/t10-pi.h>
+#include <scsi/sg.h>
+#include <asm/unaligned.h>
+
+#include "nvme.h"
+
+#define NVME_MINORS		(1U << MINORBITS)
+
+static int nvme_major;
+module_param(nvme_major, int, 0);
+
+static int nvme_char_major;
+module_param(nvme_char_major, int, 0);
+
+static LIST_HEAD(nvme_ctrl_list);
+DEFINE_SPINLOCK(dev_list_lock);
+
+static struct class *nvme_class;
+
+static void nvme_free_ns(struct kref *kref)
+{
+	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
+
+	if (ns->type == NVME_NS_LIGHTNVM)
+		nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
+
+	spin_lock(&dev_list_lock);
+	ns->disk->private_data = NULL;
+	spin_unlock(&dev_list_lock);
+
+	nvme_put_ctrl(ns->ctrl);
+	put_disk(ns->disk);
+	kfree(ns);
+}
+
+static void nvme_put_ns(struct nvme_ns *ns)
+{
+	kref_put(&ns->kref, nvme_free_ns);
+}
+
+static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
+{
+	struct nvme_ns *ns;
+
+	spin_lock(&dev_list_lock);
+	ns = disk->private_data;
+	if (ns && !kref_get_unless_zero(&ns->kref))
+		ns = NULL;
+	spin_unlock(&dev_list_lock);
+
+	return ns;
+}
+
+void nvme_requeue_req(struct request *req)
+{
+	unsigned long flags;
+
+	blk_mq_requeue_request(req);
+	spin_lock_irqsave(req->q->queue_lock, flags);
+	if (!blk_queue_stopped(req->q))
+		blk_mq_kick_requeue_list(req->q);
+	spin_unlock_irqrestore(req->q->queue_lock, flags);
+}
+
+struct request *nvme_alloc_request(struct request_queue *q,
+		struct nvme_command *cmd, unsigned int flags)
+{
+	bool write = cmd->common.opcode & 1;
+	struct request *req;
+
+	req = blk_mq_alloc_request(q, write, flags);
+	if (IS_ERR(req))
+		return req;
+
+	req->cmd_type = REQ_TYPE_DRV_PRIV;
+	req->cmd_flags |= REQ_FAILFAST_DRIVER;
+	req->__data_len = 0;
+	req->__sector = (sector_t) -1;
+	req->bio = req->biotail = NULL;
+
+	req->cmd = (unsigned char *)cmd;
+	req->cmd_len = sizeof(struct nvme_command);
+	req->special = (void *)0;
+
+	return req;
+}
+
+/*
+ * Returns 0 on success.  If the result is negative, it's a Linux error code;
+ * if the result is positive, it's an NVM Express status code
+ */
+int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void *buffer, unsigned bufflen, u32 *result, unsigned timeout)
+{
+	struct request *req;
+	int ret;
+
+	req = nvme_alloc_request(q, cmd, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
+
+	if (buffer && bufflen) {
+		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
+		if (ret)
+			goto out;
+	}
+
+	blk_execute_rq(req->q, NULL, req, 0);
+	if (result)
+		*result = (u32)(uintptr_t)req->special;
+	ret = req->errors;
+ out:
+	blk_mq_free_request(req);
+	return ret;
+}
+
+int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void *buffer, unsigned bufflen)
+{
+	return __nvme_submit_sync_cmd(q, cmd, buffer, bufflen, NULL, 0);
+}
+
+int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void __user *ubuffer, unsigned bufflen,
+		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
+		u32 *result, unsigned timeout)
+{
+	bool write = cmd->common.opcode & 1;
+	struct nvme_ns *ns = q->queuedata;
+	struct gendisk *disk = ns ? ns->disk : NULL;
+	struct request *req;
+	struct bio *bio = NULL;
+	void *meta = NULL;
+	int ret;
+
+	req = nvme_alloc_request(q, cmd, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
+
+	if (ubuffer && bufflen) {
+		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
+				GFP_KERNEL);
+		if (ret)
+			goto out;
+		bio = req->bio;
+
+		if (!disk)
+			goto submit;
+		bio->bi_bdev = bdget_disk(disk, 0);
+		if (!bio->bi_bdev) {
+			ret = -ENODEV;
+			goto out_unmap;
+		}
+
+		if (meta_buffer) {
+			struct bio_integrity_payload *bip;
+
+			meta = kmalloc(meta_len, GFP_KERNEL);
+			if (!meta) {
+				ret = -ENOMEM;
+				goto out_unmap;
+			}
+
+			if (write) {
+				if (copy_from_user(meta, meta_buffer,
+						meta_len)) {
+					ret = -EFAULT;
+					goto out_free_meta;
+				}
+			}
+
+			bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
+			if (IS_ERR(bip)) {
+				ret = PTR_ERR(bip);
+				goto out_free_meta;
+			}
+
+			bip->bip_iter.bi_size = meta_len;
+			bip->bip_iter.bi_sector = meta_seed;
+
+			ret = bio_integrity_add_page(bio, virt_to_page(meta),
+					meta_len, offset_in_page(meta));
+			if (ret != meta_len) {
+				ret = -ENOMEM;
+				goto out_free_meta;
+			}
+		}
+	}
+ submit:
+	blk_execute_rq(req->q, disk, req, 0);
+	ret = req->errors;
+	if (result)
+		*result = (u32)(uintptr_t)req->special;
+	if (meta && !ret && !write) {
+		if (copy_to_user(meta_buffer, meta, meta_len))
+			ret = -EFAULT;
+	}
+ out_free_meta:
+	kfree(meta);
+ out_unmap:
+	if (bio) {
+		if (disk && bio->bi_bdev)
+			bdput(bio->bi_bdev);
+		blk_rq_unmap_user(bio);
+	}
+ out:
+	blk_mq_free_request(req);
+	return ret;
+}
+
+int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void __user *ubuffer, unsigned bufflen, u32 *result,
+		unsigned timeout)
+{
+	return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
+			result, timeout);
+}
+
+int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
+{
+	struct nvme_command c = { };
+	int error;
+
+	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
+	c.identify.opcode = nvme_admin_identify;
+	c.identify.cns = cpu_to_le32(1);
+
+	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
+	if (!*id)
+		return -ENOMEM;
+
+	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
+			sizeof(struct nvme_id_ctrl));
+	if (error)
+		kfree(*id);
+	return error;
+}
+
+static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
+{
+	struct nvme_command c = { };
+
+	c.identify.opcode = nvme_admin_identify;
+	c.identify.cns = cpu_to_le32(2);
+	c.identify.nsid = cpu_to_le32(nsid);
+	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
+}
+
+int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
+		struct nvme_id_ns **id)
+{
+	struct nvme_command c = { };
+	int error;
+
+	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
+	c.identify.opcode = nvme_admin_identify,
+	c.identify.nsid = cpu_to_le32(nsid),
+
+	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
+	if (!*id)
+		return -ENOMEM;
+
+	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
+			sizeof(struct nvme_id_ns));
+	if (error)
+		kfree(*id);
+	return error;
+}
+
+int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
+					dma_addr_t dma_addr, u32 *result)
+{
+	struct nvme_command c;
+
+	memset(&c, 0, sizeof(c));
+	c.features.opcode = nvme_admin_get_features;
+	c.features.nsid = cpu_to_le32(nsid);
+	c.features.prp1 = cpu_to_le64(dma_addr);
+	c.features.fid = cpu_to_le32(fid);
+
+	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0, result, 0);
+}
+
+int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
+					dma_addr_t dma_addr, u32 *result)
+{
+	struct nvme_command c;
+
+	memset(&c, 0, sizeof(c));
+	c.features.opcode = nvme_admin_set_features;
+	c.features.prp1 = cpu_to_le64(dma_addr);
+	c.features.fid = cpu_to_le32(fid);
+	c.features.dword11 = cpu_to_le32(dword11);
+
+	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0, result, 0);
+}
+
+int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
+{
+	struct nvme_command c = { };
+	int error;
+
+	c.common.opcode = nvme_admin_get_log_page,
+	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
+	c.common.cdw10[0] = cpu_to_le32(
+			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
+			 NVME_LOG_SMART),
+
+	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
+	if (!*log)
+		return -ENOMEM;
+
+	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
+			sizeof(struct nvme_smart_log));
+	if (error)
+		kfree(*log);
+	return error;
+}
+
+int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
+{
+	u32 q_count = (*count - 1) | ((*count - 1) << 16);
+	u32 result;
+	int status, nr_io_queues;
+
+	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
+			&result);
+	if (status)
+		return status;
+
+	nr_io_queues = min(result & 0xffff, result >> 16) + 1;
+	*count = min(*count, nr_io_queues);
+	return 0;
+}
+
+static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
+{
+	struct nvme_user_io io;
+	struct nvme_command c;
+	unsigned length, meta_len;
+	void __user *metadata;
+
+	if (copy_from_user(&io, uio, sizeof(io)))
+		return -EFAULT;
+
+	switch (io.opcode) {
+	case nvme_cmd_write:
+	case nvme_cmd_read:
+	case nvme_cmd_compare:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	length = (io.nblocks + 1) << ns->lba_shift;
+	meta_len = (io.nblocks + 1) * ns->ms;
+	metadata = (void __user *)(uintptr_t)io.metadata;
+
+	if (ns->ext) {
+		length += meta_len;
+		meta_len = 0;
+	} else if (meta_len) {
+		if ((io.metadata & 3) || !io.metadata)
+			return -EINVAL;
+	}
+
+	memset(&c, 0, sizeof(c));
+	c.rw.opcode = io.opcode;
+	c.rw.flags = io.flags;
+	c.rw.nsid = cpu_to_le32(ns->ns_id);
+	c.rw.slba = cpu_to_le64(io.slba);
+	c.rw.length = cpu_to_le16(io.nblocks);
+	c.rw.control = cpu_to_le16(io.control);
+	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
+	c.rw.reftag = cpu_to_le32(io.reftag);
+	c.rw.apptag = cpu_to_le16(io.apptag);
+	c.rw.appmask = cpu_to_le16(io.appmask);
+
+	return __nvme_submit_user_cmd(ns->queue, &c,
+			(void __user *)(uintptr_t)io.addr, length,
+			metadata, meta_len, io.slba, NULL, 0);
+}
+
+static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+			struct nvme_passthru_cmd __user *ucmd)
+{
+	struct nvme_passthru_cmd cmd;
+	struct nvme_command c;
+	unsigned timeout = 0;
+	int status;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
+		return -EFAULT;
+
+	memset(&c, 0, sizeof(c));
+	c.common.opcode = cmd.opcode;
+	c.common.flags = cmd.flags;
+	c.common.nsid = cpu_to_le32(cmd.nsid);
+	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
+	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
+	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
+	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
+	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
+	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
+	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
+	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
+
+	if (cmd.timeout_ms)
+		timeout = msecs_to_jiffies(cmd.timeout_ms);
+
+	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
+			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
+			&cmd.result, timeout);
+	if (status >= 0) {
+		if (put_user(cmd.result, &ucmd->result))
+			return -EFAULT;
+	}
+
+	return status;
+}
+
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+		unsigned int cmd, unsigned long arg)
+{
+	struct nvme_ns *ns = bdev->bd_disk->private_data;
+
+	switch (cmd) {
+	case NVME_IOCTL_ID:
+		force_successful_syscall_return();
+		return ns->ns_id;
+	case NVME_IOCTL_ADMIN_CMD:
+		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
+	case NVME_IOCTL_IO_CMD:
+		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
+	case NVME_IOCTL_SUBMIT_IO:
+		return nvme_submit_io(ns, (void __user *)arg);
+#ifdef CONFIG_BLK_DEV_NVME_SCSI
+	case SG_GET_VERSION_NUM:
+		return nvme_sg_get_version_num((void __user *)arg);
+	case SG_IO:
+		return nvme_sg_io(ns, (void __user *)arg);
+#endif
+	default:
+		return -ENOTTY;
+	}
+}
+
+#ifdef CONFIG_COMPAT
+static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	switch (cmd) {
+	case SG_IO:
+		return -ENOIOCTLCMD;
+	}
+	return nvme_ioctl(bdev, mode, cmd, arg);
+}
+#else
+#define nvme_compat_ioctl	NULL
+#endif
+
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+	return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+	nvme_put_ns(disk->private_data);
+}
+
+static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+{
+	/* some standard values */
+	geo->heads = 1 << 6;
+	geo->sectors = 1 << 5;
+	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
+	return 0;
+}
+
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+static void nvme_init_integrity(struct nvme_ns *ns)
+{
+	struct blk_integrity integrity;
+
+	switch (ns->pi_type) {
+	case NVME_NS_DPS_PI_TYPE3:
+		integrity.profile = &t10_pi_type3_crc;
+		break;
+	case NVME_NS_DPS_PI_TYPE1:
+	case NVME_NS_DPS_PI_TYPE2:
+		integrity.profile = &t10_pi_type1_crc;
+		break;
+	default:
+		integrity.profile = NULL;
+		break;
+	}
+	integrity.tuple_size = ns->ms;
+	blk_integrity_register(ns->disk, &integrity);
+	blk_queue_max_integrity_segments(ns->queue, 1);
+}
+#else
+static void nvme_init_integrity(struct nvme_ns *ns)
+{
+}
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
+static void nvme_config_discard(struct nvme_ns *ns)
+{
+	u32 logical_block_size = queue_logical_block_size(ns->queue);
+	ns->queue->limits.discard_zeroes_data = 0;
+	ns->queue->limits.discard_alignment = logical_block_size;
+	ns->queue->limits.discard_granularity = logical_block_size;
+	blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
+	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
+}
+
+static int nvme_revalidate_disk(struct gendisk *disk)
+{
+	struct nvme_ns *ns = disk->private_data;
+	struct nvme_id_ns *id;
+	u8 lbaf, pi_type;
+	u16 old_ms;
+	unsigned short bs;
+
+	if (nvme_identify_ns(ns->ctrl, ns->ns_id, &id)) {
+		dev_warn(ns->ctrl->dev, "%s: Identify failure nvme%dn%d\n",
+				__func__, ns->ctrl->instance, ns->ns_id);
+		return -ENODEV;
+	}
+	if (id->ncap == 0) {
+		kfree(id);
+		return -ENODEV;
+	}
+
+	if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
+		if (nvme_nvm_register(ns->queue, disk->disk_name)) {
+			dev_warn(ns->ctrl->dev,
+				"%s: LightNVM init failure\n", __func__);
+			kfree(id);
+			return -ENODEV;
+		}
+		ns->type = NVME_NS_LIGHTNVM;
+	}
+
+	if (ns->ctrl->vs >= NVME_VS(1, 1))
+		memcpy(ns->eui, id->eui64, sizeof(ns->eui));
+	if (ns->ctrl->vs >= NVME_VS(1, 2))
+		memcpy(ns->uuid, id->nguid, sizeof(ns->uuid));
+
+	old_ms = ns->ms;
+	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
+	ns->lba_shift = id->lbaf[lbaf].ds;
+	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
+	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
+
+	/*
+	 * If identify namespace failed, use default 512 byte block size so
+	 * block layer can use before failing read/write for 0 capacity.
+	 */
+	if (ns->lba_shift == 0)
+		ns->lba_shift = 9;
+	bs = 1 << ns->lba_shift;
+	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
+	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
+					id->dps & NVME_NS_DPS_PI_MASK : 0;
+
+	blk_mq_freeze_queue(disk->queue);
+	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
+				ns->ms != old_ms ||
+				bs != queue_logical_block_size(disk->queue) ||
+				(ns->ms && ns->ext)))
+		blk_integrity_unregister(disk);
+
+	ns->pi_type = pi_type;
+	blk_queue_logical_block_size(ns->queue, bs);
+
+	if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
+		nvme_init_integrity(ns);
+	if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
+		set_capacity(disk, 0);
+	else
+		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
+
+	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
+		nvme_config_discard(ns);
+	blk_mq_unfreeze_queue(disk->queue);
+
+	kfree(id);
+	return 0;
+}
+
+static char nvme_pr_type(enum pr_type type)
+{
+	switch (type) {
+	case PR_WRITE_EXCLUSIVE:
+		return 1;
+	case PR_EXCLUSIVE_ACCESS:
+		return 2;
+	case PR_WRITE_EXCLUSIVE_REG_ONLY:
+		return 3;
+	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
+		return 4;
+	case PR_WRITE_EXCLUSIVE_ALL_REGS:
+		return 5;
+	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
+		return 6;
+	default:
+		return 0;
+	}
+};
+
+static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
+				u64 key, u64 sa_key, u8 op)
+{
+	struct nvme_ns *ns = bdev->bd_disk->private_data;
+	struct nvme_command c;
+	u8 data[16] = { 0, };
+
+	put_unaligned_le64(key, &data[0]);
+	put_unaligned_le64(sa_key, &data[8]);
+
+	memset(&c, 0, sizeof(c));
+	c.common.opcode = op;
+	c.common.nsid = cpu_to_le32(ns->ns_id);
+	c.common.cdw10[0] = cpu_to_le32(cdw10);
+
+	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
+}
+
+static int nvme_pr_register(struct block_device *bdev, u64 old,
+		u64 new, unsigned flags)
+{
+	u32 cdw10;
+
+	if (flags & ~PR_FL_IGNORE_KEY)
+		return -EOPNOTSUPP;
+
+	cdw10 = old ? 2 : 0;
+	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
+	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
+	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_reserve(struct block_device *bdev, u64 key,
+		enum pr_type type, unsigned flags)
+{
+	u32 cdw10;
+
+	if (flags & ~PR_FL_IGNORE_KEY)
+		return -EOPNOTSUPP;
+
+	cdw10 = nvme_pr_type(type) << 8;
+	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
+	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
+		enum pr_type type, bool abort)
+{
+	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
+	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_clear(struct block_device *bdev, u64 key)
+{
+	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
+	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
+{
+	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
+	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
+}
+
+static const struct pr_ops nvme_pr_ops = {
+	.pr_register	= nvme_pr_register,
+	.pr_reserve	= nvme_pr_reserve,
+	.pr_release	= nvme_pr_release,
+	.pr_preempt	= nvme_pr_preempt,
+	.pr_clear	= nvme_pr_clear,
+};
+
+static const struct block_device_operations nvme_fops = {
+	.owner		= THIS_MODULE,
+	.ioctl		= nvme_ioctl,
+	.compat_ioctl	= nvme_compat_ioctl,
+	.open		= nvme_open,
+	.release	= nvme_release,
+	.getgeo		= nvme_getgeo,
+	.revalidate_disk= nvme_revalidate_disk,
+	.pr_ops		= &nvme_pr_ops,
+};
+
+static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
+{
+	unsigned long timeout =
+		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
+	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
+	int ret;
+
+	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
+		if ((csts & NVME_CSTS_RDY) == bit)
+			break;
+
+		msleep(100);
+		if (fatal_signal_pending(current))
+			return -EINTR;
+		if (time_after(jiffies, timeout)) {
+			dev_err(ctrl->dev,
+				"Device not ready; aborting %s\n", enabled ?
+						"initialisation" : "reset");
+			return -ENODEV;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * If the device has been passed off to us in an enabled state, just clear
+ * the enabled bit.  The spec says we should set the 'shutdown notification
+ * bits', but doing so may cause the device to complete commands to the
+ * admin queue ... and we don't know what memory that might be pointing at!
+ */
+int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
+{
+	int ret;
+
+	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
+	ctrl->ctrl_config &= ~NVME_CC_ENABLE;
+
+	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+	if (ret)
+		return ret;
+	return nvme_wait_ready(ctrl, cap, false);
+}
+
+int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
+{
+	/*
+	 * Default to a 4K page size, with the intention to update this
+	 * path in the future to accomodate architectures with differing
+	 * kernel and IO page sizes.
+	 */
+	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
+	int ret;
+
+	if (page_shift < dev_page_min) {
+		dev_err(ctrl->dev,
+			"Minimum device page size %u too large for host (%u)\n",
+			1 << dev_page_min, 1 << page_shift);
+		return -ENODEV;
+	}
+
+	ctrl->page_size = 1 << page_shift;
+
+	ctrl->ctrl_config = NVME_CC_CSS_NVM;
+	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
+	ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
+	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
+	ctrl->ctrl_config |= NVME_CC_ENABLE;
+
+	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+	if (ret)
+		return ret;
+	return nvme_wait_ready(ctrl, cap, true);
+}
+
+int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
+{
+	unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
+	u32 csts;
+	int ret;
+
+	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
+	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
+
+	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+	if (ret)
+		return ret;
+
+	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
+		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
+			break;
+
+		msleep(100);
+		if (fatal_signal_pending(current))
+			return -EINTR;
+		if (time_after(jiffies, timeout)) {
+			dev_err(ctrl->dev,
+				"Device shutdown incomplete; abort shutdown\n");
+			return -ENODEV;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * Initialize the cached copies of the Identify data and various controller
+ * register in our nvme_ctrl structure.  This should be called as soon as
+ * the admin queue is fully up and running.
+ */
+int nvme_init_identify(struct nvme_ctrl *ctrl)
+{
+	struct nvme_id_ctrl *id;
+	u64 cap;
+	int ret, page_shift;
+
+	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
+	if (ret) {
+		dev_err(ctrl->dev, "Reading VS failed (%d)\n", ret);
+		return ret;
+	}
+
+	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
+	if (ret) {
+		dev_err(ctrl->dev, "Reading CAP failed (%d)\n", ret);
+		return ret;
+	}
+	page_shift = NVME_CAP_MPSMIN(cap) + 12;
+
+	if (ctrl->vs >= NVME_VS(1, 1))
+		ctrl->subsystem = NVME_CAP_NSSRC(cap);
+
+	ret = nvme_identify_ctrl(ctrl, &id);
+	if (ret) {
+		dev_err(ctrl->dev, "Identify Controller failed (%d)\n", ret);
+		return -EIO;
+	}
+
+	ctrl->oncs = le16_to_cpup(&id->oncs);
+	atomic_set(&ctrl->abort_limit, id->acl + 1);
+	ctrl->vwc = id->vwc;
+	memcpy(ctrl->serial, id->sn, sizeof(id->sn));
+	memcpy(ctrl->model, id->mn, sizeof(id->mn));
+	memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
+	if (id->mdts)
+		ctrl->max_hw_sectors = 1 << (id->mdts + page_shift - 9);
+	else
+		ctrl->max_hw_sectors = UINT_MAX;
+
+	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && id->vs[3]) {
+		unsigned int max_hw_sectors;
+
+		ctrl->stripe_size = 1 << (id->vs[3] + page_shift);
+		max_hw_sectors = ctrl->stripe_size >> (page_shift - 9);
+		if (ctrl->max_hw_sectors) {
+			ctrl->max_hw_sectors = min(max_hw_sectors,
+							ctrl->max_hw_sectors);
+		} else {
+			ctrl->max_hw_sectors = max_hw_sectors;
+		}
+	}
+
+	kfree(id);
+	return 0;
+}
+
+static int nvme_dev_open(struct inode *inode, struct file *file)
+{
+	struct nvme_ctrl *ctrl;
+	int instance = iminor(inode);
+	int ret = -ENODEV;
+
+	spin_lock(&dev_list_lock);
+	list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
+		if (ctrl->instance != instance)
+			continue;
+
+		if (!ctrl->admin_q) {
+			ret = -EWOULDBLOCK;
+			break;
+		}
+		if (!kref_get_unless_zero(&ctrl->kref))
+			break;
+		file->private_data = ctrl;
+		ret = 0;
+		break;
+	}
+	spin_unlock(&dev_list_lock);
+
+	return ret;
+}
+
+static int nvme_dev_release(struct inode *inode, struct file *file)
+{
+	nvme_put_ctrl(file->private_data);
+	return 0;
+}
+
+static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
+{
+	struct nvme_ns *ns;
+	int ret;
+
+	mutex_lock(&ctrl->namespaces_mutex);
+	if (list_empty(&ctrl->namespaces)) {
+		ret = -ENOTTY;
+		goto out_unlock;
+	}
+
+	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
+	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
+		dev_warn(ctrl->dev,
+			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	dev_warn(ctrl->dev,
+		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
+	kref_get(&ns->kref);
+	mutex_unlock(&ctrl->namespaces_mutex);
+
+	ret = nvme_user_cmd(ctrl, ns, argp);
+	nvme_put_ns(ns);
+	return ret;
+
+out_unlock:
+	mutex_unlock(&ctrl->namespaces_mutex);
+	return ret;
+}
+
+static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
+		unsigned long arg)
+{
+	struct nvme_ctrl *ctrl = file->private_data;
+	void __user *argp = (void __user *)arg;
+
+	switch (cmd) {
+	case NVME_IOCTL_ADMIN_CMD:
+		return nvme_user_cmd(ctrl, NULL, argp);
+	case NVME_IOCTL_IO_CMD:
+		return nvme_dev_user_cmd(ctrl, argp);
+	case NVME_IOCTL_RESET:
+		dev_warn(ctrl->dev, "resetting controller\n");
+		return ctrl->ops->reset_ctrl(ctrl);
+	case NVME_IOCTL_SUBSYS_RESET:
+		return nvme_reset_subsystem(ctrl);
+	default:
+		return -ENOTTY;
+	}
+}
+
+static const struct file_operations nvme_dev_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvme_dev_open,
+	.release	= nvme_dev_release,
+	.unlocked_ioctl	= nvme_dev_ioctl,
+	.compat_ioctl	= nvme_dev_ioctl,
+};
+
+static ssize_t nvme_sysfs_reset(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
+	int ret;
+
+	ret = ctrl->ops->reset_ctrl(ctrl);
+	if (ret < 0)
+		return ret;
+	return count;
+}
+static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
+
+static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
+								char *buf)
+{
+	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	return sprintf(buf, "%pU\n", ns->uuid);
+}
+static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
+
+static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
+								char *buf)
+{
+	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	return sprintf(buf, "%8phd\n", ns->eui);
+}
+static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
+
+static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
+								char *buf)
+{
+	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	return sprintf(buf, "%d\n", ns->ns_id);
+}
+static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
+
+static struct attribute *nvme_ns_attrs[] = {
+	&dev_attr_uuid.attr,
+	&dev_attr_eui.attr,
+	&dev_attr_nsid.attr,
+	NULL,
+};
+
+static umode_t nvme_attrs_are_visible(struct kobject *kobj,
+		struct attribute *a, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+
+	if (a == &dev_attr_uuid.attr) {
+		if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
+			return 0;
+	}
+	if (a == &dev_attr_eui.attr) {
+		if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
+			return 0;
+	}
+	return a->mode;
+}
+
+static const struct attribute_group nvme_ns_attr_group = {
+	.attrs		= nvme_ns_attrs,
+	.is_visible	= nvme_attrs_are_visible,
+};
+
+#define nvme_show_function(field)						\
+static ssize_t  field##_show(struct device *dev,				\
+			    struct device_attribute *attr, char *buf)		\
+{										\
+        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
+        return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field);	\
+}										\
+static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
+
+nvme_show_function(model);
+nvme_show_function(serial);
+nvme_show_function(firmware_rev);
+
+static struct attribute *nvme_dev_attrs[] = {
+	&dev_attr_reset_controller.attr,
+	&dev_attr_model.attr,
+	&dev_attr_serial.attr,
+	&dev_attr_firmware_rev.attr,
+	NULL
+};
+
+static struct attribute_group nvme_dev_attrs_group = {
+	.attrs = nvme_dev_attrs,
+};
+
+static const struct attribute_group *nvme_dev_attr_groups[] = {
+	&nvme_dev_attrs_group,
+	NULL,
+};
+
+static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
+{
+	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
+	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
+
+	return nsa->ns_id - nsb->ns_id;
+}
+
+static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid)
+{
+	struct nvme_ns *ns;
+
+	lockdep_assert_held(&ctrl->namespaces_mutex);
+
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		if (ns->ns_id == nsid)
+			return ns;
+		if (ns->ns_id > nsid)
+			break;
+	}
+	return NULL;
+}
+
+static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
+{
+	struct nvme_ns *ns;
+	struct gendisk *disk;
+	int node = dev_to_node(ctrl->dev);
+
+	lockdep_assert_held(&ctrl->namespaces_mutex);
+
+	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
+	if (!ns)
+		return;
+
+	ns->queue = blk_mq_init_queue(ctrl->tagset);
+	if (IS_ERR(ns->queue))
+		goto out_free_ns;
+	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
+	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
+	ns->queue->queuedata = ns;
+	ns->ctrl = ctrl;
+
+	disk = alloc_disk_node(0, node);
+	if (!disk)
+		goto out_free_queue;
+
+	kref_init(&ns->kref);
+	ns->ns_id = nsid;
+	ns->disk = disk;
+	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
+
+	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
+	if (ctrl->max_hw_sectors) {
+		blk_queue_max_hw_sectors(ns->queue, ctrl->max_hw_sectors);
+		blk_queue_max_segments(ns->queue,
+			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1);
+	}
+	if (ctrl->stripe_size)
+		blk_queue_chunk_sectors(ns->queue, ctrl->stripe_size >> 9);
+	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
+		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
+	blk_queue_virt_boundary(ns->queue, ctrl->page_size - 1);
+
+	disk->major = nvme_major;
+	disk->first_minor = 0;
+	disk->fops = &nvme_fops;
+	disk->private_data = ns;
+	disk->queue = ns->queue;
+	disk->driverfs_dev = ctrl->device;
+	disk->flags = GENHD_FL_EXT_DEVT;
+	sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, nsid);
+
+	if (nvme_revalidate_disk(ns->disk))
+		goto out_free_disk;
+
+	list_add_tail(&ns->list, &ctrl->namespaces);
+	kref_get(&ctrl->kref);
+	if (ns->type == NVME_NS_LIGHTNVM)
+		return;
+
+	add_disk(ns->disk);
+	if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
+					&nvme_ns_attr_group))
+		pr_warn("%s: failed to create sysfs group for identification\n",
+			ns->disk->disk_name);
+	return;
+ out_free_disk:
+	kfree(disk);
+ out_free_queue:
+	blk_cleanup_queue(ns->queue);
+ out_free_ns:
+	kfree(ns);
+}
+
+static void nvme_ns_remove(struct nvme_ns *ns)
+{
+	bool kill = nvme_io_incapable(ns->ctrl) &&
+			!blk_queue_dying(ns->queue);
+
+	lockdep_assert_held(&ns->ctrl->namespaces_mutex);
+
+	if (kill) {
+		blk_set_queue_dying(ns->queue);
+
+		/*
+		 * The controller was shutdown first if we got here through
+		 * device removal. The shutdown may requeue outstanding
+		 * requests. These need to be aborted immediately so
+		 * del_gendisk doesn't block indefinitely for their completion.
+		 */
+		blk_mq_abort_requeue_list(ns->queue);
+	}
+	if (ns->disk->flags & GENHD_FL_UP) {
+		if (blk_get_integrity(ns->disk))
+			blk_integrity_unregister(ns->disk);
+		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
+					&nvme_ns_attr_group);
+		del_gendisk(ns->disk);
+	}
+	if (kill || !blk_queue_dying(ns->queue)) {
+		blk_mq_abort_requeue_list(ns->queue);
+		blk_cleanup_queue(ns->queue);
+	}
+	list_del_init(&ns->list);
+	nvme_put_ns(ns);
+}
+
+static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
+{
+	struct nvme_ns *ns;
+
+	ns = nvme_find_ns(ctrl, nsid);
+	if (ns) {
+		if (revalidate_disk(ns->disk))
+			nvme_ns_remove(ns);
+	} else
+		nvme_alloc_ns(ctrl, nsid);
+}
+
+static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
+{
+	struct nvme_ns *ns;
+	__le32 *ns_list;
+	unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
+	int ret = 0;
+
+	ns_list = kzalloc(0x1000, GFP_KERNEL);
+	if (!ns_list)
+		return -ENOMEM;
+
+	for (i = 0; i < num_lists; i++) {
+		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
+		if (ret)
+			goto out;
+
+		for (j = 0; j < min(nn, 1024U); j++) {
+			nsid = le32_to_cpu(ns_list[j]);
+			if (!nsid)
+				goto out;
+
+			nvme_validate_ns(ctrl, nsid);
+
+			while (++prev < nsid) {
+				ns = nvme_find_ns(ctrl, prev);
+				if (ns)
+					nvme_ns_remove(ns);
+			}
+		}
+		nn -= j;
+	}
+ out:
+	kfree(ns_list);
+	return ret;
+}
+
+static void __nvme_scan_namespaces(struct nvme_ctrl *ctrl, unsigned nn)
+{
+	struct nvme_ns *ns, *next;
+	unsigned i;
+
+	lockdep_assert_held(&ctrl->namespaces_mutex);
+
+	for (i = 1; i <= nn; i++)
+		nvme_validate_ns(ctrl, i);
+
+	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
+		if (ns->ns_id > nn)
+			nvme_ns_remove(ns);
+	}
+}
+
+void nvme_scan_namespaces(struct nvme_ctrl *ctrl)
+{
+	struct nvme_id_ctrl *id;
+	unsigned nn;
+
+	if (nvme_identify_ctrl(ctrl, &id))
+		return;
+
+	mutex_lock(&ctrl->namespaces_mutex);
+	nn = le32_to_cpu(id->nn);
+	if (ctrl->vs >= NVME_VS(1, 1) &&
+	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
+		if (!nvme_scan_ns_list(ctrl, nn))
+			goto done;
+	}
+	__nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
+ done:
+	list_sort(NULL, &ctrl->namespaces, ns_cmp);
+	mutex_unlock(&ctrl->namespaces_mutex);
+	kfree(id);
+}
+
+void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
+{
+	struct nvme_ns *ns, *next;
+
+	mutex_lock(&ctrl->namespaces_mutex);
+	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
+		nvme_ns_remove(ns);
+	mutex_unlock(&ctrl->namespaces_mutex);
+}
+
+static DEFINE_IDA(nvme_instance_ida);
+
+static int nvme_set_instance(struct nvme_ctrl *ctrl)
+{
+	int instance, error;
+
+	do {
+		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
+			return -ENODEV;
+
+		spin_lock(&dev_list_lock);
+		error = ida_get_new(&nvme_instance_ida, &instance);
+		spin_unlock(&dev_list_lock);
+	} while (error == -EAGAIN);
+
+	if (error)
+		return -ENODEV;
+
+	ctrl->instance = instance;
+	return 0;
+}
+
+static void nvme_release_instance(struct nvme_ctrl *ctrl)
+{
+	spin_lock(&dev_list_lock);
+	ida_remove(&nvme_instance_ida, ctrl->instance);
+	spin_unlock(&dev_list_lock);
+}
+
+void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
+ {
+	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
+
+	spin_lock(&dev_list_lock);
+	list_del(&ctrl->node);
+	spin_unlock(&dev_list_lock);
+}
+
+static void nvme_free_ctrl(struct kref *kref)
+{
+	struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
+
+	put_device(ctrl->device);
+	nvme_release_instance(ctrl);
+
+	ctrl->ops->free_ctrl(ctrl);
+}
+
+void nvme_put_ctrl(struct nvme_ctrl *ctrl)
+{
+	kref_put(&ctrl->kref, nvme_free_ctrl);
+}
+
+/*
+ * Initialize a NVMe controller structures.  This needs to be called during
+ * earliest initialization so that we have the initialized structured around
+ * during probing.
+ */
+int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+		const struct nvme_ctrl_ops *ops, unsigned long quirks)
+{
+	int ret;
+
+	INIT_LIST_HEAD(&ctrl->namespaces);
+	mutex_init(&ctrl->namespaces_mutex);
+	kref_init(&ctrl->kref);
+	ctrl->dev = dev;
+	ctrl->ops = ops;
+	ctrl->quirks = quirks;
+
+	ret = nvme_set_instance(ctrl);
+	if (ret)
+		goto out;
+
+	ctrl->device = device_create_with_groups(nvme_class, ctrl->dev,
+				MKDEV(nvme_char_major, ctrl->instance),
+				dev, nvme_dev_attr_groups,
+				"nvme%d", ctrl->instance);
+	if (IS_ERR(ctrl->device)) {
+		ret = PTR_ERR(ctrl->device);
+		goto out_release_instance;
+	}
+	get_device(ctrl->device);
+	dev_set_drvdata(ctrl->device, ctrl);
+
+	spin_lock(&dev_list_lock);
+	list_add_tail(&ctrl->node, &nvme_ctrl_list);
+	spin_unlock(&dev_list_lock);
+
+	return 0;
+out_release_instance:
+	nvme_release_instance(ctrl);
+out:
+	return ret;
+}
+
+void nvme_stop_queues(struct nvme_ctrl *ctrl)
+{
+	struct nvme_ns *ns;
+
+	mutex_lock(&ctrl->namespaces_mutex);
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		spin_lock_irq(ns->queue->queue_lock);
+		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
+		spin_unlock_irq(ns->queue->queue_lock);
+
+		blk_mq_cancel_requeue_work(ns->queue);
+		blk_mq_stop_hw_queues(ns->queue);
+	}
+	mutex_unlock(&ctrl->namespaces_mutex);
+}
+
+void nvme_start_queues(struct nvme_ctrl *ctrl)
+{
+	struct nvme_ns *ns;
+
+	mutex_lock(&ctrl->namespaces_mutex);
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
+		blk_mq_start_stopped_hw_queues(ns->queue, true);
+		blk_mq_kick_requeue_list(ns->queue);
+	}
+	mutex_unlock(&ctrl->namespaces_mutex);
+}
+
+int __init nvme_core_init(void)
+{
+	int result;
+
+	result = register_blkdev(nvme_major, "nvme");
+	if (result < 0)
+		return result;
+	else if (result > 0)
+		nvme_major = result;
+
+	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
+							&nvme_dev_fops);
+	if (result < 0)
+		goto unregister_blkdev;
+	else if (result > 0)
+		nvme_char_major = result;
+
+	nvme_class = class_create(THIS_MODULE, "nvme");
+	if (IS_ERR(nvme_class)) {
+		result = PTR_ERR(nvme_class);
+		goto unregister_chrdev;
+	}
+
+	return 0;
+
+ unregister_chrdev:
+	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
+ unregister_blkdev:
+	unregister_blkdev(nvme_major, "nvme");
+	return result;
+}
+
+void nvme_core_exit(void)
+{
+	unregister_blkdev(nvme_major, "nvme");
+	class_destroy(nvme_class);
+	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
+}
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index 15f2acb4d5cd..5cd3725e2fa4 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -146,6 +146,16 @@ struct nvme_nvm_command {
 	};
 };
 
+struct nvme_nvm_lp_mlc {
+	__u16			num_pairs;
+	__u8			pairs[886];
+};
+
+struct nvme_nvm_lp_tbl {
+	__u8			id[8];
+	struct nvme_nvm_lp_mlc	mlc;
+};
+
 struct nvme_nvm_id_group {
 	__u8			mtype;
 	__u8			fmtype;
@@ -169,7 +179,8 @@ struct nvme_nvm_id_group {
 	__le32			mpos;
 	__le32			mccap;
 	__le16			cpar;
-	__u8			reserved[906];
+	__u8			reserved[10];
+	struct nvme_nvm_lp_tbl lptbl;
 } __packed;
 
 struct nvme_nvm_addr_format {
@@ -266,6 +277,15 @@ static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
 		dst->mccap = le32_to_cpu(src->mccap);
 
 		dst->cpar = le16_to_cpu(src->cpar);
+
+		if (dst->fmtype == NVM_ID_FMTYPE_MLC) {
+			memcpy(dst->lptbl.id, src->lptbl.id, 8);
+			dst->lptbl.mlc.num_pairs =
+					le16_to_cpu(src->lptbl.mlc.num_pairs);
+			/* 4 bits per pair */
+			memcpy(dst->lptbl.mlc.pairs, src->lptbl.mlc.pairs,
+						dst->lptbl.mlc.num_pairs >> 1);
+		}
 	}
 
 	return 0;
@@ -274,7 +294,6 @@ static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
 static int nvme_nvm_identity(struct nvm_dev *nvmdev, struct nvm_id *nvm_id)
 {
 	struct nvme_ns *ns = nvmdev->q->queuedata;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_nvm_id *nvme_nvm_id;
 	struct nvme_nvm_command c = {};
 	int ret;
@@ -287,7 +306,7 @@ static int nvme_nvm_identity(struct nvm_dev *nvmdev, struct nvm_id *nvm_id)
 	if (!nvme_nvm_id)
 		return -ENOMEM;
 
-	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
+	ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, (struct nvme_command *)&c,
 				nvme_nvm_id, sizeof(struct nvme_nvm_id));
 	if (ret) {
 		ret = -EIO;
@@ -312,9 +331,8 @@ static int nvme_nvm_get_l2p_tbl(struct nvm_dev *nvmdev, u64 slba, u32 nlb,
 				nvm_l2p_update_fn *update_l2p, void *priv)
 {
 	struct nvme_ns *ns = nvmdev->q->queuedata;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_nvm_command c = {};
-	u32 len = queue_max_hw_sectors(dev->admin_q) << 9;
+	u32 len = queue_max_hw_sectors(ns->ctrl->admin_q) << 9;
 	u32 nlb_pr_rq = len / sizeof(u64);
 	u64 cmd_slba = slba;
 	void *entries;
@@ -332,10 +350,10 @@ static int nvme_nvm_get_l2p_tbl(struct nvm_dev *nvmdev, u64 slba, u32 nlb,
 		c.l2p.slba = cpu_to_le64(cmd_slba);
 		c.l2p.nlb = cpu_to_le32(cmd_nlb);
 
-		ret = nvme_submit_sync_cmd(dev->admin_q,
+		ret = nvme_submit_sync_cmd(ns->ctrl->admin_q,
 				(struct nvme_command *)&c, entries, len);
 		if (ret) {
-			dev_err(dev->dev, "L2P table transfer failed (%d)\n",
+			dev_err(ns->ctrl->dev, "L2P table transfer failed (%d)\n",
 									ret);
 			ret = -EIO;
 			goto out;
@@ -361,7 +379,7 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 {
 	struct request_queue *q = nvmdev->q;
 	struct nvme_ns *ns = q->queuedata;
-	struct nvme_dev *dev = ns->dev;
+	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_nvm_command c = {};
 	struct nvme_nvm_bb_tbl *bb_tbl;
 	int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blocks;
@@ -375,41 +393,36 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 	if (!bb_tbl)
 		return -ENOMEM;
 
-	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
+	ret = nvme_submit_sync_cmd(ctrl->admin_q, (struct nvme_command *)&c,
 								bb_tbl, tblsz);
 	if (ret) {
-		dev_err(dev->dev, "get bad block table failed (%d)\n", ret);
+		dev_err(ctrl->dev, "get bad block table failed (%d)\n", ret);
 		ret = -EIO;
 		goto out;
 	}
 
 	if (bb_tbl->tblid[0] != 'B' || bb_tbl->tblid[1] != 'B' ||
 		bb_tbl->tblid[2] != 'L' || bb_tbl->tblid[3] != 'T') {
-		dev_err(dev->dev, "bbt format mismatch\n");
+		dev_err(ctrl->dev, "bbt format mismatch\n");
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (le16_to_cpu(bb_tbl->verid) != 1) {
 		ret = -EINVAL;
-		dev_err(dev->dev, "bbt version not supported\n");
+		dev_err(ctrl->dev, "bbt version not supported\n");
 		goto out;
 	}
 
 	if (le32_to_cpu(bb_tbl->tblks) != nr_blocks) {
 		ret = -EINVAL;
-		dev_err(dev->dev, "bbt unsuspected blocks returned (%u!=%u)",
+		dev_err(ctrl->dev, "bbt unsuspected blocks returned (%u!=%u)",
 					le32_to_cpu(bb_tbl->tblks), nr_blocks);
 		goto out;
 	}
 
 	ppa = dev_to_generic_addr(nvmdev, ppa);
 	ret = update_bbtbl(ppa, nr_blocks, bb_tbl->blk, priv);
-	if (ret) {
-		ret = -EINTR;
-		goto out;
-	}
-
 out:
 	kfree(bb_tbl);
 	return ret;
@@ -419,7 +432,6 @@ static int nvme_nvm_set_bb_tbl(struct nvm_dev *nvmdev, struct nvm_rq *rqd,
 								int type)
 {
 	struct nvme_ns *ns = nvmdev->q->queuedata;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_nvm_command c = {};
 	int ret = 0;
 
@@ -429,10 +441,10 @@ static int nvme_nvm_set_bb_tbl(struct nvm_dev *nvmdev, struct nvm_rq *rqd,
 	c.set_bb.nlb = cpu_to_le16(rqd->nr_pages - 1);
 	c.set_bb.value = type;
 
-	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
+	ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, (struct nvme_command *)&c,
 								NULL, 0);
 	if (ret)
-		dev_err(dev->dev, "set bad block table failed (%d)\n", ret);
+		dev_err(ns->ctrl->dev, "set bad block table failed (%d)\n", ret);
 	return ret;
 }
 
@@ -453,11 +465,8 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
 static void nvme_nvm_end_io(struct request *rq, int error)
 {
 	struct nvm_rq *rqd = rq->end_io_data;
-	struct nvm_dev *dev = rqd->dev;
 
-	if (dev->mt && dev->mt->end_io(rqd, error))
-		pr_err("nvme: err status: %x result: %lx\n",
-				rq->errors, (unsigned long)rq->special);
+	nvm_end_io(rqd, error);
 
 	kfree(rq->cmd);
 	blk_mq_free_request(rq);
@@ -471,7 +480,7 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 	struct bio *bio = rqd->bio;
 	struct nvme_nvm_command *cmd;
 
-	rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0);
+	rq = blk_mq_alloc_request(q, bio_rw(bio), 0);
 	if (IS_ERR(rq))
 		return -ENOMEM;
 
@@ -520,9 +529,8 @@ static int nvme_nvm_erase_block(struct nvm_dev *dev, struct nvm_rq *rqd)
 static void *nvme_nvm_create_dma_pool(struct nvm_dev *nvmdev, char *name)
 {
 	struct nvme_ns *ns = nvmdev->q->queuedata;
-	struct nvme_dev *dev = ns->dev;
 
-	return dma_pool_create(name, dev->dev, PAGE_SIZE, PAGE_SIZE, 0);
+	return dma_pool_create(name, ns->ctrl->dev, PAGE_SIZE, PAGE_SIZE, 0);
 }
 
 static void nvme_nvm_destroy_dma_pool(void *pool)
@@ -580,8 +588,9 @@ void nvme_nvm_unregister(struct request_queue *q, char *disk_name)
 
 int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
 {
-	struct nvme_dev *dev = ns->dev;
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	struct nvme_ctrl *ctrl = ns->ctrl;
+	/* XXX: this is poking into PCI structures from generic code! */
+	struct pci_dev *pdev = to_pci_dev(ctrl->dev);
 
 	/* QEMU NVMe simulator - PCI ID + Vendor specific bit */
 	if (pdev->vendor == PCI_VENDOR_ID_CNEX &&
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 044253dca30a..4fb5bb737868 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -19,58 +19,77 @@
 #include <linux/kref.h>
 #include <linux/blk-mq.h>
 
+enum {
+	/*
+	 * Driver internal status code for commands that were cancelled due
+	 * to timeouts or controller shutdown.  The value is negative so
+	 * that it a) doesn't overlap with the unsigned hardware error codes,
+	 * and b) can easily be tested for.
+	 */
+	NVME_SC_CANCELLED		= -EINTR,
+};
+
 extern unsigned char nvme_io_timeout;
 #define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)
 
+extern unsigned char admin_timeout;
+#define ADMIN_TIMEOUT	(admin_timeout * HZ)
+
+extern unsigned char shutdown_timeout;
+#define SHUTDOWN_TIMEOUT	(shutdown_timeout * HZ)
+
 enum {
 	NVME_NS_LBA		= 0,
 	NVME_NS_LIGHTNVM	= 1,
 };
 
 /*
- * Represents an NVM Express device.  Each nvme_dev is a PCI function.
+ * List of workarounds for devices that required behavior not specified in
+ * the standard.
  */
-struct nvme_dev {
-	struct list_head node;
-	struct nvme_queue **queues;
+enum nvme_quirks {
+	/*
+	 * Prefers I/O aligned to a stripe size specified in a vendor
+	 * specific Identify field.
+	 */
+	NVME_QUIRK_STRIPE_SIZE			= (1 << 0),
+
+	/*
+	 * The controller doesn't handle Identify value others than 0 or 1
+	 * correctly.
+	 */
+	NVME_QUIRK_IDENTIFY_CNS			= (1 << 1),
+};
+
+struct nvme_ctrl {
+	const struct nvme_ctrl_ops *ops;
 	struct request_queue *admin_q;
-	struct blk_mq_tag_set tagset;
-	struct blk_mq_tag_set admin_tagset;
-	u32 __iomem *dbs;
 	struct device *dev;
-	struct dma_pool *prp_page_pool;
-	struct dma_pool *prp_small_pool;
+	struct kref kref;
 	int instance;
-	unsigned queue_count;
-	unsigned online_queues;
-	unsigned max_qid;
-	int q_depth;
-	u32 db_stride;
-	u32 ctrl_config;
-	struct msix_entry *entry;
-	struct nvme_bar __iomem *bar;
+	struct blk_mq_tag_set *tagset;
 	struct list_head namespaces;
-	struct kref kref;
-	struct device *device;
-	struct work_struct reset_work;
-	struct work_struct probe_work;
-	struct work_struct scan_work;
+	struct mutex namespaces_mutex;
+	struct device *device;	/* char device */
+	struct list_head node;
+
 	char name[12];
 	char serial[20];
 	char model[40];
 	char firmware_rev[8];
-	bool subsystem;
+
+	u32 ctrl_config;
+
+	u32 page_size;
 	u32 max_hw_sectors;
 	u32 stripe_size;
-	u32 page_size;
-	void __iomem *cmb;
-	dma_addr_t cmb_dma_addr;
-	u64 cmb_size;
-	u32 cmbsz;
 	u16 oncs;
-	u16 abort_limit;
+	atomic_t abort_limit;
 	u8 event_limit;
 	u8 vwc;
+	u32 vs;
+	bool subsystem;
+	unsigned long quirks;
 };
 
 /*
@@ -79,11 +98,14 @@ struct nvme_dev {
 struct nvme_ns {
 	struct list_head list;
 
-	struct nvme_dev *dev;
+	struct nvme_ctrl *ctrl;
 	struct request_queue *queue;
 	struct gendisk *disk;
 	struct kref kref;
 
+	u8 eui[8];
+	u8 uuid[16];
+
 	unsigned ns_id;
 	int lba_shift;
 	u16 ms;
@@ -94,41 +116,156 @@ struct nvme_ns {
 	u32 mode_select_block_len;
 };
 
-/*
- * The nvme_iod describes the data in an I/O, including the list of PRP
- * entries.  You can't see it in this data structure because C doesn't let
- * me express that.  Use nvme_alloc_iod to ensure there's enough space
- * allocated to store the PRP list.
- */
-struct nvme_iod {
-	unsigned long private;	/* For the use of the submitter of the I/O */
-	int npages;		/* In the PRP list. 0 means small pool in use */
-	int offset;		/* Of PRP list */
-	int nents;		/* Used in scatterlist */
-	int length;		/* Of data, in bytes */
-	dma_addr_t first_dma;
-	struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */
-	struct scatterlist sg[0];
+struct nvme_ctrl_ops {
+	int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
+	int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
+	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
+	bool (*io_incapable)(struct nvme_ctrl *ctrl);
+	int (*reset_ctrl)(struct nvme_ctrl *ctrl);
+	void (*free_ctrl)(struct nvme_ctrl *ctrl);
 };
 
+static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
+{
+	u32 val = 0;
+
+	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
+		return false;
+	return val & NVME_CSTS_RDY;
+}
+
+static inline bool nvme_io_incapable(struct nvme_ctrl *ctrl)
+{
+	u32 val = 0;
+
+	if (ctrl->ops->io_incapable(ctrl))
+		return false;
+	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
+		return false;
+	return val & NVME_CSTS_CFS;
+}
+
+static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
+{
+	if (!ctrl->subsystem)
+		return -ENOTTY;
+	return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
+}
+
 static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
 {
 	return (sector >> (ns->lba_shift - 9));
 }
 
+static inline void nvme_setup_flush(struct nvme_ns *ns,
+		struct nvme_command *cmnd)
+{
+	memset(cmnd, 0, sizeof(*cmnd));
+	cmnd->common.opcode = nvme_cmd_flush;
+	cmnd->common.nsid = cpu_to_le32(ns->ns_id);
+}
+
+static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
+		struct nvme_command *cmnd)
+{
+	u16 control = 0;
+	u32 dsmgmt = 0;
+
+	if (req->cmd_flags & REQ_FUA)
+		control |= NVME_RW_FUA;
+	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
+		control |= NVME_RW_LR;
+
+	if (req->cmd_flags & REQ_RAHEAD)
+		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
+
+	memset(cmnd, 0, sizeof(*cmnd));
+	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
+	cmnd->rw.command_id = req->tag;
+	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
+	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
+	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
+
+	if (ns->ms) {
+		switch (ns->pi_type) {
+		case NVME_NS_DPS_PI_TYPE3:
+			control |= NVME_RW_PRINFO_PRCHK_GUARD;
+			break;
+		case NVME_NS_DPS_PI_TYPE1:
+		case NVME_NS_DPS_PI_TYPE2:
+			control |= NVME_RW_PRINFO_PRCHK_GUARD |
+					NVME_RW_PRINFO_PRCHK_REF;
+			cmnd->rw.reftag = cpu_to_le32(
+					nvme_block_nr(ns, blk_rq_pos(req)));
+			break;
+		}
+		if (!blk_integrity_rq(req))
+			control |= NVME_RW_PRINFO_PRACT;
+	}
+
+	cmnd->rw.control = cpu_to_le16(control);
+	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
+}
+
+
+static inline int nvme_error_status(u16 status)
+{
+	switch (status & 0x7ff) {
+	case NVME_SC_SUCCESS:
+		return 0;
+	case NVME_SC_CAP_EXCEEDED:
+		return -ENOSPC;
+	default:
+		return -EIO;
+	}
+}
+
+static inline bool nvme_req_needs_retry(struct request *req, u16 status)
+{
+	return !(status & NVME_SC_DNR || blk_noretry_request(req)) &&
+		(jiffies - req->start_time) < req->timeout;
+}
+
+int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
+int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
+int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
+int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+		const struct nvme_ctrl_ops *ops, unsigned long quirks);
+void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
+void nvme_put_ctrl(struct nvme_ctrl *ctrl);
+int nvme_init_identify(struct nvme_ctrl *ctrl);
+
+void nvme_scan_namespaces(struct nvme_ctrl *ctrl);
+void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
+
+void nvme_stop_queues(struct nvme_ctrl *ctrl);
+void nvme_start_queues(struct nvme_ctrl *ctrl);
+
+struct request *nvme_alloc_request(struct request_queue *q,
+		struct nvme_command *cmd, unsigned int flags);
+void nvme_requeue_req(struct request *req);
 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		void *buf, unsigned bufflen);
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void *buffer, void __user *ubuffer, unsigned bufflen,
+		void *buffer, unsigned bufflen,  u32 *result, unsigned timeout);
+int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void __user *ubuffer, unsigned bufflen, u32 *result,
+		unsigned timeout);
+int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
+		void __user *ubuffer, unsigned bufflen,
+		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
 		u32 *result, unsigned timeout);
-int nvme_identify_ctrl(struct nvme_dev *dev, struct nvme_id_ctrl **id);
-int nvme_identify_ns(struct nvme_dev *dev, unsigned nsid,
+int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id);
+int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
 		struct nvme_id_ns **id);
-int nvme_get_log_page(struct nvme_dev *dev, struct nvme_smart_log **log);
-int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
+int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log);
+int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
 			dma_addr_t dma_addr, u32 *result);
-int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
+int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
 			dma_addr_t dma_addr, u32 *result);
+int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
+
+extern spinlock_t dev_list_lock;
 
 struct sg_io_hdr;
 
@@ -154,4 +291,7 @@ static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *i
 }
 #endif /* CONFIG_NVM */
 
+int __init nvme_core_init(void);
+void nvme_core_exit(void);
+
 #endif /* _NVME_H */
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0c67b57be83c..72ef8322d32a 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -12,6 +12,7 @@
  * more details.
  */
 
+#include <linux/aer.h>
 #include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/blk-mq.h>
@@ -28,10 +29,10 @@
 #include <linux/kdev_t.h>
 #include <linux/kthread.h>
 #include <linux/kernel.h>
-#include <linux/list_sort.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/poison.h>
 #include <linux/ptrace.h>
@@ -39,23 +40,24 @@
 #include <linux/slab.h>
 #include <linux/t10-pi.h>
 #include <linux/types.h>
-#include <linux/pr.h>
-#include <scsi/sg.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <asm/unaligned.h>
 
-#include <uapi/linux/nvme_ioctl.h>
 #include "nvme.h"
 
-#define NVME_MINORS		(1U << MINORBITS)
 #define NVME_Q_DEPTH		1024
 #define NVME_AQ_DEPTH		256
 #define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
 #define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
-#define ADMIN_TIMEOUT		(admin_timeout * HZ)
-#define SHUTDOWN_TIMEOUT	(shutdown_timeout * HZ)
+		
+/*
+ * We handle AEN commands ourselves and don't even let the
+ * block layer know about them.
+ */
+#define NVME_NR_AEN_COMMANDS	1
+#define NVME_AQ_BLKMQ_DEPTH	(NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS)
 
-static unsigned char admin_timeout = 60;
+unsigned char admin_timeout = 60;
 module_param(admin_timeout, byte, 0644);
 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
 
@@ -63,16 +65,10 @@ unsigned char nvme_io_timeout = 30;
 module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
 
-static unsigned char shutdown_timeout = 5;
+unsigned char shutdown_timeout = 5;
 module_param(shutdown_timeout, byte, 0644);
 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
 
-static int nvme_major;
-module_param(nvme_major, int, 0);
-
-static int nvme_char_major;
-module_param(nvme_char_major, int, 0);
-
 static int use_threaded_interrupts;
 module_param(use_threaded_interrupts, int, 0);
 
@@ -80,28 +76,60 @@ static bool use_cmb_sqes = true;
 module_param(use_cmb_sqes, bool, 0644);
 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
 
-static DEFINE_SPINLOCK(dev_list_lock);
 static LIST_HEAD(dev_list);
 static struct task_struct *nvme_thread;
 static struct workqueue_struct *nvme_workq;
 static wait_queue_head_t nvme_kthread_wait;
 
-static struct class *nvme_class;
+struct nvme_dev;
+struct nvme_queue;
 
-static int __nvme_reset(struct nvme_dev *dev);
 static int nvme_reset(struct nvme_dev *dev);
 static void nvme_process_cq(struct nvme_queue *nvmeq);
-static void nvme_dead_ctrl(struct nvme_dev *dev);
+static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
+static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
 
-struct async_cmd_info {
-	struct kthread_work work;
-	struct kthread_worker *worker;
-	struct request *req;
-	u32 result;
-	int status;
-	void *ctx;
+/*
+ * Represents an NVM Express device.  Each nvme_dev is a PCI function.
+ */
+struct nvme_dev {
+	struct list_head node;
+	struct nvme_queue **queues;
+	struct blk_mq_tag_set tagset;
+	struct blk_mq_tag_set admin_tagset;
+	u32 __iomem *dbs;
+	struct device *dev;
+	struct dma_pool *prp_page_pool;
+	struct dma_pool *prp_small_pool;
+	unsigned queue_count;
+	unsigned online_queues;
+	unsigned max_qid;
+	int q_depth;
+	u32 db_stride;
+	struct msix_entry *entry;
+	void __iomem *bar;
+	struct work_struct reset_work;
+	struct work_struct scan_work;
+	struct work_struct remove_work;
+	struct mutex shutdown_lock;
+	bool subsystem;
+	void __iomem *cmb;
+	dma_addr_t cmb_dma_addr;
+	u64 cmb_size;
+	u32 cmbsz;
+	unsigned long flags;
+
+#define NVME_CTRL_RESETTING    0
+
+	struct nvme_ctrl ctrl;
+	struct completion ioq_wait;
 };
 
+static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
+{
+	return container_of(ctrl, struct nvme_dev, ctrl);
+}
+
 /*
  * An NVM Express queue.  Each device has at least two (one for admin
  * commands and one for I/O commands).
@@ -126,7 +154,24 @@ struct nvme_queue {
 	u16 qid;
 	u8 cq_phase;
 	u8 cqe_seen;
-	struct async_cmd_info cmdinfo;
+};
+
+/*
+ * The nvme_iod describes the data in an I/O, including the list of PRP
+ * entries.  You can't see it in this data structure because C doesn't let
+ * me express that.  Use nvme_init_iod to ensure there's enough space
+ * allocated to store the PRP list.
+ */
+struct nvme_iod {
+	struct nvme_queue *nvmeq;
+	int aborted;
+	int npages;		/* In the PRP list. 0 means small pool in use */
+	int nents;		/* Used in scatterlist */
+	int length;		/* Of data, in bytes */
+	dma_addr_t first_dma;
+	struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
+	struct scatterlist *sg;
+	struct scatterlist inline_sg[0];
 };
 
 /*
@@ -148,23 +193,11 @@ static inline void _nvme_check_size(void)
 	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 }
 
-typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
-						struct nvme_completion *);
-
-struct nvme_cmd_info {
-	nvme_completion_fn fn;
-	void *ctx;
-	int aborted;
-	struct nvme_queue *nvmeq;
-	struct nvme_iod iod[0];
-};
-
 /*
  * Max size of iod being embedded in the request payload
  */
 #define NVME_INT_PAGES		2
-#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->page_size)
-#define NVME_INT_MASK		0x01
+#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
 
 /*
  * Will slightly overestimate the number of pages needed.  This is OK
@@ -173,19 +206,22 @@ struct nvme_cmd_info {
  */
 static int nvme_npages(unsigned size, struct nvme_dev *dev)
 {
-	unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
+	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
+				      dev->ctrl.page_size);
 	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 }
 
-static unsigned int nvme_cmd_size(struct nvme_dev *dev)
+static unsigned int nvme_iod_alloc_size(struct nvme_dev *dev,
+		unsigned int size, unsigned int nseg)
 {
-	unsigned int ret = sizeof(struct nvme_cmd_info);
-
-	ret += sizeof(struct nvme_iod);
-	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
-	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;
+	return sizeof(__le64 *) * nvme_npages(size, dev) +
+			sizeof(struct scatterlist) * nseg;
+}
 
-	return ret;
+static unsigned int nvme_cmd_size(struct nvme_dev *dev)
+{
+	return sizeof(struct nvme_iod) +
+		nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES);
 }
 
 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
@@ -215,11 +251,11 @@ static int nvme_admin_init_request(void *data, struct request *req,
 				unsigned int numa_node)
 {
 	struct nvme_dev *dev = data;
-	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	struct nvme_queue *nvmeq = dev->queues[0];
 
 	BUG_ON(!nvmeq);
-	cmd->nvmeq = nvmeq;
+	iod->nvmeq = nvmeq;
 	return 0;
 }
 
@@ -242,148 +278,36 @@ static int nvme_init_request(void *data, struct request *req,
 				unsigned int numa_node)
 {
 	struct nvme_dev *dev = data;
-	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
 
 	BUG_ON(!nvmeq);
-	cmd->nvmeq = nvmeq;
+	iod->nvmeq = nvmeq;
 	return 0;
 }
 
-static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
-				nvme_completion_fn handler)
-{
-	cmd->fn = handler;
-	cmd->ctx = ctx;
-	cmd->aborted = 0;
-	blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
-}
-
-static void *iod_get_private(struct nvme_iod *iod)
-{
-	return (void *) (iod->private & ~0x1UL);
-}
-
-/*
- * If bit 0 is set, the iod is embedded in the request payload.
- */
-static bool iod_should_kfree(struct nvme_iod *iod)
-{
-	return (iod->private & NVME_INT_MASK) == 0;
-}
-
-/* Special values must be less than 0x1000 */
-#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
-#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
-#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
-#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
-
-static void special_completion(struct nvme_queue *nvmeq, void *ctx,
-						struct nvme_completion *cqe)
-{
-	if (ctx == CMD_CTX_CANCELLED)
-		return;
-	if (ctx == CMD_CTX_COMPLETED) {
-		dev_warn(nvmeq->q_dmadev,
-				"completed id %d twice on queue %d\n",
-				cqe->command_id, le16_to_cpup(&cqe->sq_id));
-		return;
-	}
-	if (ctx == CMD_CTX_INVALID) {
-		dev_warn(nvmeq->q_dmadev,
-				"invalid id %d completed on queue %d\n",
-				cqe->command_id, le16_to_cpup(&cqe->sq_id));
-		return;
-	}
-	dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
-}
-
-static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
-{
-	void *ctx;
-
-	if (fn)
-		*fn = cmd->fn;
-	ctx = cmd->ctx;
-	cmd->fn = special_completion;
-	cmd->ctx = CMD_CTX_CANCELLED;
-	return ctx;
-}
-
-static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
-						struct nvme_completion *cqe)
+static void nvme_complete_async_event(struct nvme_dev *dev,
+		struct nvme_completion *cqe)
 {
-	u32 result = le32_to_cpup(&cqe->result);
-	u16 status = le16_to_cpup(&cqe->status) >> 1;
+	u16 status = le16_to_cpu(cqe->status) >> 1;
+	u32 result = le32_to_cpu(cqe->result);
 
 	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
-		++nvmeq->dev->event_limit;
+		++dev->ctrl.event_limit;
 	if (status != NVME_SC_SUCCESS)
 		return;
 
 	switch (result & 0xff07) {
 	case NVME_AER_NOTICE_NS_CHANGED:
-		dev_info(nvmeq->q_dmadev, "rescanning\n");
-		schedule_work(&nvmeq->dev->scan_work);
+		dev_info(dev->dev, "rescanning\n");
+		queue_work(nvme_workq, &dev->scan_work);
 	default:
-		dev_warn(nvmeq->q_dmadev, "async event result %08x\n", result);
+		dev_warn(dev->dev, "async event result %08x\n", result);
 	}
 }
 
-static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
-						struct nvme_completion *cqe)
-{
-	struct request *req = ctx;
-
-	u16 status = le16_to_cpup(&cqe->status) >> 1;
-	u32 result = le32_to_cpup(&cqe->result);
-
-	blk_mq_free_request(req);
-
-	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
-	++nvmeq->dev->abort_limit;
-}
-
-static void async_completion(struct nvme_queue *nvmeq, void *ctx,
-						struct nvme_completion *cqe)
-{
-	struct async_cmd_info *cmdinfo = ctx;
-	cmdinfo->result = le32_to_cpup(&cqe->result);
-	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
-	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
-	blk_mq_free_request(cmdinfo->req);
-}
-
-static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
-				  unsigned int tag)
-{
-	struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
-
-	return blk_mq_rq_to_pdu(req);
-}
-
-/*
- * Called with local interrupts disabled and the q_lock held.  May not sleep.
- */
-static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
-						nvme_completion_fn *fn)
-{
-	struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
-	void *ctx;
-	if (tag >= nvmeq->q_depth) {
-		*fn = special_completion;
-		return CMD_CTX_INVALID;
-	}
-	if (fn)
-		*fn = cmd->fn;
-	ctx = cmd->ctx;
-	cmd->fn = special_completion;
-	cmd->ctx = CMD_CTX_COMPLETED;
-	return ctx;
-}
-
 /**
- * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
+ * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
  * @nvmeq: The queue to use
  * @cmd: The command to send
  *
@@ -405,69 +329,44 @@ static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
 	nvmeq->sq_tail = tail;
 }
 
-static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&nvmeq->q_lock, flags);
-	__nvme_submit_cmd(nvmeq, cmd);
-	spin_unlock_irqrestore(&nvmeq->q_lock, flags);
-}
-
-static __le64 **iod_list(struct nvme_iod *iod)
+static __le64 **iod_list(struct request *req)
 {
-	return ((void *)iod) + iod->offset;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	return (__le64 **)(iod->sg + req->nr_phys_segments);
 }
 
-static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
-			    unsigned nseg, unsigned long private)
+static int nvme_init_iod(struct request *rq, struct nvme_dev *dev)
 {
-	iod->private = private;
-	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
-	iod->npages = -1;
-	iod->length = nbytes;
-	iod->nents = 0;
-}
-
-static struct nvme_iod *
-__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
-		 unsigned long priv, gfp_t gfp)
-{
-	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
-				sizeof(__le64 *) * nvme_npages(bytes, dev) +
-				sizeof(struct scatterlist) * nseg, gfp);
-
-	if (iod)
-		iod_init(iod, bytes, nseg, priv);
-
-	return iod;
-}
-
-static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
-			               gfp_t gfp)
-{
-	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
-                                                sizeof(struct nvme_dsm_range);
-	struct nvme_iod *iod;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
+	int nseg = rq->nr_phys_segments;
+	unsigned size;
 
-	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
-	    size <= NVME_INT_BYTES(dev)) {
-		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
+	if (rq->cmd_flags & REQ_DISCARD)
+		size = sizeof(struct nvme_dsm_range);
+	else
+		size = blk_rq_bytes(rq);
 
-		iod = cmd->iod;
-		iod_init(iod, size, rq->nr_phys_segments,
-				(unsigned long) rq | NVME_INT_MASK);
-		return iod;
+	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
+		iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC);
+		if (!iod->sg)
+			return BLK_MQ_RQ_QUEUE_BUSY;
+	} else {
+		iod->sg = iod->inline_sg;
 	}
 
-	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
-				(unsigned long) rq, gfp);
+	iod->aborted = 0;
+	iod->npages = -1;
+	iod->nents = 0;
+	iod->length = size;
+	return 0;
 }
 
-static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
+static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
 {
-	const int last_prp = dev->page_size / 8 - 1;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	const int last_prp = dev->ctrl.page_size / 8 - 1;
 	int i;
-	__le64 **list = iod_list(iod);
+	__le64 **list = iod_list(req);
 	dma_addr_t prp_dma = iod->first_dma;
 
 	if (iod->npages == 0)
@@ -479,20 +378,8 @@ static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
 		prp_dma = next_prp_dma;
 	}
 
-	if (iod_should_kfree(iod))
-		kfree(iod);
-}
-
-static int nvme_error_status(u16 status)
-{
-	switch (status & 0x7ff) {
-	case NVME_SC_SUCCESS:
-		return 0;
-	case NVME_SC_CAP_EXCEEDED:
-		return -ENOSPC;
-	default:
-		return -EIO;
-	}
+	if (iod->sg != iod->inline_sg)
+		kfree(iod->sg);
 }
 
 #ifdef CONFIG_BLK_DEV_INTEGRITY
@@ -549,27 +436,6 @@ static void nvme_dif_remap(struct request *req,
 	}
 	kunmap_atomic(pmap);
 }
-
-static void nvme_init_integrity(struct nvme_ns *ns)
-{
-	struct blk_integrity integrity;
-
-	switch (ns->pi_type) {
-	case NVME_NS_DPS_PI_TYPE3:
-		integrity.profile = &t10_pi_type3_crc;
-		break;
-	case NVME_NS_DPS_PI_TYPE1:
-	case NVME_NS_DPS_PI_TYPE2:
-		integrity.profile = &t10_pi_type1_crc;
-		break;
-	default:
-		integrity.profile = NULL;
-		break;
-	}
-	integrity.tuple_size = ns->ms;
-	blk_integrity_register(ns->disk, &integrity);
-	blk_queue_max_integrity_segments(ns->queue, 1);
-}
 #else /* CONFIG_BLK_DEV_INTEGRITY */
 static void nvme_dif_remap(struct request *req,
 			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
@@ -581,91 +447,27 @@ static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 {
 }
-static void nvme_init_integrity(struct nvme_ns *ns)
-{
-}
 #endif
 
-static void req_completion(struct nvme_queue *nvmeq, void *ctx,
-						struct nvme_completion *cqe)
-{
-	struct nvme_iod *iod = ctx;
-	struct request *req = iod_get_private(iod);
-	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
-	u16 status = le16_to_cpup(&cqe->status) >> 1;
-	bool requeue = false;
-	int error = 0;
-
-	if (unlikely(status)) {
-		if (!(status & NVME_SC_DNR || blk_noretry_request(req))
-		    && (jiffies - req->start_time) < req->timeout) {
-			unsigned long flags;
-
-			requeue = true;
-			blk_mq_requeue_request(req);
-			spin_lock_irqsave(req->q->queue_lock, flags);
-			if (!blk_queue_stopped(req->q))
-				blk_mq_kick_requeue_list(req->q);
-			spin_unlock_irqrestore(req->q->queue_lock, flags);
-			goto release_iod;
-		}
-
-		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
-			if (cmd_rq->ctx == CMD_CTX_CANCELLED)
-				error = -EINTR;
-			else
-				error = status;
-		} else {
-			error = nvme_error_status(status);
-		}
-	}
-
-	if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
-		u32 result = le32_to_cpup(&cqe->result);
-		req->special = (void *)(uintptr_t)result;
-	}
-
-	if (cmd_rq->aborted)
-		dev_warn(nvmeq->dev->dev,
-			"completing aborted command with status:%04x\n",
-			error);
-
-release_iod:
-	if (iod->nents) {
-		dma_unmap_sg(nvmeq->dev->dev, iod->sg, iod->nents,
-			rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
-		if (blk_integrity_rq(req)) {
-			if (!rq_data_dir(req))
-				nvme_dif_remap(req, nvme_dif_complete);
-			dma_unmap_sg(nvmeq->dev->dev, iod->meta_sg, 1,
-				rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
-		}
-	}
-	nvme_free_iod(nvmeq->dev, iod);
-
-	if (likely(!requeue))
-		blk_mq_complete_request(req, error);
-}
-
-/* length is in bytes.  gfp flags indicates whether we may sleep. */
-static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
-		int total_len, gfp_t gfp)
+static bool nvme_setup_prps(struct nvme_dev *dev, struct request *req,
+		int total_len)
 {
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	struct dma_pool *pool;
 	int length = total_len;
 	struct scatterlist *sg = iod->sg;
 	int dma_len = sg_dma_len(sg);
 	u64 dma_addr = sg_dma_address(sg);
-	u32 page_size = dev->page_size;
+	u32 page_size = dev->ctrl.page_size;
 	int offset = dma_addr & (page_size - 1);
 	__le64 *prp_list;
-	__le64 **list = iod_list(iod);
+	__le64 **list = iod_list(req);
 	dma_addr_t prp_dma;
 	int nprps, i;
 
 	length -= (page_size - offset);
 	if (length <= 0)
-		return total_len;
+		return true;
 
 	dma_len -= (page_size - offset);
 	if (dma_len) {
@@ -678,7 +480,7 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
 
 	if (length <= page_size) {
 		iod->first_dma = dma_addr;
-		return total_len;
+		return true;
 	}
 
 	nprps = DIV_ROUND_UP(length, page_size);
@@ -690,11 +492,11 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
 		iod->npages = 1;
 	}
 
-	prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
+	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 	if (!prp_list) {
 		iod->first_dma = dma_addr;
 		iod->npages = -1;
-		return (total_len - length) + page_size;
+		return false;
 	}
 	list[0] = prp_list;
 	iod->first_dma = prp_dma;
@@ -702,9 +504,9 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
 	for (;;) {
 		if (i == page_size >> 3) {
 			__le64 *old_prp_list = prp_list;
-			prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
+			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 			if (!prp_list)
-				return total_len - length;
+				return false;
 			list[iod->npages++] = prp_list;
 			prp_list[0] = old_prp_list[i - 1];
 			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
@@ -724,115 +526,105 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
 		dma_len = sg_dma_len(sg);
 	}
 
-	return total_len;
+	return true;
 }
 
-static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req,
-		struct nvme_iod *iod)
+static int nvme_map_data(struct nvme_dev *dev, struct request *req,
+		struct nvme_command *cmnd)
 {
-	struct nvme_command cmnd;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct request_queue *q = req->q;
+	enum dma_data_direction dma_dir = rq_data_dir(req) ?
+			DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	int ret = BLK_MQ_RQ_QUEUE_ERROR;
 
-	memcpy(&cmnd, req->cmd, sizeof(cmnd));
-	cmnd.rw.command_id = req->tag;
-	if (req->nr_phys_segments) {
-		cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
-		cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
-	}
+	sg_init_table(iod->sg, req->nr_phys_segments);
+	iod->nents = blk_rq_map_sg(q, req, iod->sg);
+	if (!iod->nents)
+		goto out;
 
-	__nvme_submit_cmd(nvmeq, &cmnd);
-}
+	ret = BLK_MQ_RQ_QUEUE_BUSY;
+	if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
+		goto out;
 
-/*
- * We reuse the small pool to allocate the 16-byte range here as it is not
- * worth having a special pool for these or additional cases to handle freeing
- * the iod.
- */
-static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
-		struct request *req, struct nvme_iod *iod)
-{
-	struct nvme_dsm_range *range =
-				(struct nvme_dsm_range *)iod_list(iod)[0];
-	struct nvme_command cmnd;
+	if (!nvme_setup_prps(dev, req, blk_rq_bytes(req)))
+		goto out_unmap;
 
-	range->cattr = cpu_to_le32(0);
-	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
-	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
+	ret = BLK_MQ_RQ_QUEUE_ERROR;
+	if (blk_integrity_rq(req)) {
+		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
+			goto out_unmap;
 
-	memset(&cmnd, 0, sizeof(cmnd));
-	cmnd.dsm.opcode = nvme_cmd_dsm;
-	cmnd.dsm.command_id = req->tag;
-	cmnd.dsm.nsid = cpu_to_le32(ns->ns_id);
-	cmnd.dsm.prp1 = cpu_to_le64(iod->first_dma);
-	cmnd.dsm.nr = 0;
-	cmnd.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
+		sg_init_table(&iod->meta_sg, 1);
+		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
+			goto out_unmap;
 
-	__nvme_submit_cmd(nvmeq, &cmnd);
-}
+		if (rq_data_dir(req))
+			nvme_dif_remap(req, nvme_dif_prep);
 
-static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
-								int cmdid)
-{
-	struct nvme_command cmnd;
+		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
+			goto out_unmap;
+	}
 
-	memset(&cmnd, 0, sizeof(cmnd));
-	cmnd.common.opcode = nvme_cmd_flush;
-	cmnd.common.command_id = cmdid;
-	cmnd.common.nsid = cpu_to_le32(ns->ns_id);
+	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
+	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
+	if (blk_integrity_rq(req))
+		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
+	return BLK_MQ_RQ_QUEUE_OK;
 
-	__nvme_submit_cmd(nvmeq, &cmnd);
+out_unmap:
+	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
+out:
+	return ret;
 }
 
-static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
-							struct nvme_ns *ns)
+static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 {
-	struct request *req = iod_get_private(iod);
-	struct nvme_command cmnd;
-	u16 control = 0;
-	u32 dsmgmt = 0;
-
-	if (req->cmd_flags & REQ_FUA)
-		control |= NVME_RW_FUA;
-	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
-		control |= NVME_RW_LR;
-
-	if (req->cmd_flags & REQ_RAHEAD)
-		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
-
-	memset(&cmnd, 0, sizeof(cmnd));
-	cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
-	cmnd.rw.command_id = req->tag;
-	cmnd.rw.nsid = cpu_to_le32(ns->ns_id);
-	cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
-	cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
-	cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
-	cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
-
-	if (ns->ms) {
-		switch (ns->pi_type) {
-		case NVME_NS_DPS_PI_TYPE3:
-			control |= NVME_RW_PRINFO_PRCHK_GUARD;
-			break;
-		case NVME_NS_DPS_PI_TYPE1:
-		case NVME_NS_DPS_PI_TYPE2:
-			control |= NVME_RW_PRINFO_PRCHK_GUARD |
-					NVME_RW_PRINFO_PRCHK_REF;
-			cmnd.rw.reftag = cpu_to_le32(
-					nvme_block_nr(ns, blk_rq_pos(req)));
-			break;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	enum dma_data_direction dma_dir = rq_data_dir(req) ?
+			DMA_TO_DEVICE : DMA_FROM_DEVICE;
+
+	if (iod->nents) {
+		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
+		if (blk_integrity_rq(req)) {
+			if (!rq_data_dir(req))
+				nvme_dif_remap(req, nvme_dif_complete);
+			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
 		}
-		if (blk_integrity_rq(req))
-			cmnd.rw.metadata =
-				cpu_to_le64(sg_dma_address(iod->meta_sg));
-		else
-			control |= NVME_RW_PRINFO_PRACT;
 	}
 
-	cmnd.rw.control = cpu_to_le16(control);
-	cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt);
+	nvme_free_iod(dev, req);
+}
 
-	__nvme_submit_cmd(nvmeq, &cmnd);
+/*
+ * We reuse the small pool to allocate the 16-byte range here as it is not
+ * worth having a special pool for these or additional cases to handle freeing
+ * the iod.
+ */
+static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
+		struct request *req, struct nvme_command *cmnd)
+{
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_dsm_range *range;
 
-	return 0;
+	range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
+						&iod->first_dma);
+	if (!range)
+		return BLK_MQ_RQ_QUEUE_BUSY;
+	iod_list(req)[0] = (__le64 *)range;
+	iod->npages = 0;
+
+	range->cattr = cpu_to_le32(0);
+	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
+	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
+
+	memset(cmnd, 0, sizeof(*cmnd));
+	cmnd->dsm.opcode = nvme_cmd_dsm;
+	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
+	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
+	cmnd->dsm.nr = 0;
+	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
+	return BLK_MQ_RQ_QUEUE_OK;
 }
 
 /*
@@ -845,9 +637,8 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	struct nvme_queue *nvmeq = hctx->driver_data;
 	struct nvme_dev *dev = nvmeq->dev;
 	struct request *req = bd->rq;
-	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
-	struct nvme_iod *iod;
-	enum dma_data_direction dma_dir;
+	struct nvme_command cmnd;
+	int ret = BLK_MQ_RQ_QUEUE_OK;
 
 	/*
 	 * If formated with metadata, require the block layer provide a buffer
@@ -857,91 +648,72 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	if (ns && ns->ms && !blk_integrity_rq(req)) {
 		if (!(ns->pi_type && ns->ms == 8) &&
 					req->cmd_type != REQ_TYPE_DRV_PRIV) {
-			blk_mq_complete_request(req, -EFAULT);
+			blk_mq_end_request(req, -EFAULT);
 			return BLK_MQ_RQ_QUEUE_OK;
 		}
 	}
 
-	iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
-	if (!iod)
-		return BLK_MQ_RQ_QUEUE_BUSY;
+	ret = nvme_init_iod(req, dev);
+	if (ret)
+		return ret;
 
 	if (req->cmd_flags & REQ_DISCARD) {
-		void *range;
-		/*
-		 * We reuse the small pool to allocate the 16-byte range here
-		 * as it is not worth having a special pool for these or
-		 * additional cases to handle freeing the iod.
-		 */
-		range = dma_pool_alloc(dev->prp_small_pool, GFP_ATOMIC,
-						&iod->first_dma);
-		if (!range)
-			goto retry_cmd;
-		iod_list(iod)[0] = (__le64 *)range;
-		iod->npages = 0;
-	} else if (req->nr_phys_segments) {
-		dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+		ret = nvme_setup_discard(nvmeq, ns, req, &cmnd);
+	} else {
+		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
+			memcpy(&cmnd, req->cmd, sizeof(cmnd));
+		else if (req->cmd_flags & REQ_FLUSH)
+			nvme_setup_flush(ns, &cmnd);
+		else
+			nvme_setup_rw(ns, req, &cmnd);
 
-		sg_init_table(iod->sg, req->nr_phys_segments);
-		iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
-		if (!iod->nents)
-			goto error_cmd;
+		if (req->nr_phys_segments)
+			ret = nvme_map_data(dev, req, &cmnd);
+	}
 
-		if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
-			goto retry_cmd;
+	if (ret)
+		goto out;
 
-		if (blk_rq_bytes(req) !=
-                    nvme_setup_prps(dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
-			dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
-			goto retry_cmd;
-		}
-		if (blk_integrity_rq(req)) {
-			if (blk_rq_count_integrity_sg(req->q, req->bio) != 1) {
-				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
-						dma_dir);
-				goto error_cmd;
-			}
+	cmnd.common.command_id = req->tag;
+	blk_mq_start_request(req);
 
-			sg_init_table(iod->meta_sg, 1);
-			if (blk_rq_map_integrity_sg(
-					req->q, req->bio, iod->meta_sg) != 1) {
-				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
-						dma_dir);
-				goto error_cmd;
-			}
+	spin_lock_irq(&nvmeq->q_lock);
+	__nvme_submit_cmd(nvmeq, &cmnd);
+	nvme_process_cq(nvmeq);
+	spin_unlock_irq(&nvmeq->q_lock);
+	return BLK_MQ_RQ_QUEUE_OK;
+out:
+	nvme_free_iod(dev, req);
+	return ret;
+}
 
-			if (rq_data_dir(req))
-				nvme_dif_remap(req, nvme_dif_prep);
+static void nvme_complete_rq(struct request *req)
+{
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_dev *dev = iod->nvmeq->dev;
+	int error = 0;
 
-			if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir)) {
-				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
-						dma_dir);
-				goto error_cmd;
-			}
+	nvme_unmap_data(dev, req);
+
+	if (unlikely(req->errors)) {
+		if (nvme_req_needs_retry(req, req->errors)) {
+			nvme_requeue_req(req);
+			return;
 		}
-	}
 
-	nvme_set_info(cmd, iod, req_completion);
-	spin_lock_irq(&nvmeq->q_lock);
-	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
-		nvme_submit_priv(nvmeq, req, iod);
-	else if (req->cmd_flags & REQ_DISCARD)
-		nvme_submit_discard(nvmeq, ns, req, iod);
-	else if (req->cmd_flags & REQ_FLUSH)
-		nvme_submit_flush(nvmeq, ns, req->tag);
-	else
-		nvme_submit_iod(nvmeq, iod, ns);
+		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
+			error = req->errors;
+		else
+			error = nvme_error_status(req->errors);
+	}
 
-	nvme_process_cq(nvmeq);
-	spin_unlock_irq(&nvmeq->q_lock);
-	return BLK_MQ_RQ_QUEUE_OK;
+	if (unlikely(iod->aborted)) {
+		dev_warn(dev->dev,
+			"completing aborted command with status: %04x\n",
+			req->errors);
+	}
 
- error_cmd:
-	nvme_free_iod(dev, iod);
-	return BLK_MQ_RQ_QUEUE_ERROR;
- retry_cmd:
-	nvme_free_iod(dev, iod);
-	return BLK_MQ_RQ_QUEUE_BUSY;
+	blk_mq_end_request(req, error);
 }
 
 static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
@@ -952,20 +724,47 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
 	phase = nvmeq->cq_phase;
 
 	for (;;) {
-		void *ctx;
-		nvme_completion_fn fn;
 		struct nvme_completion cqe = nvmeq->cqes[head];
-		if ((le16_to_cpu(cqe.status) & 1) != phase)
+		u16 status = le16_to_cpu(cqe.status);
+		struct request *req;
+
+		if ((status & 1) != phase)
 			break;
 		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
 		if (++head == nvmeq->q_depth) {
 			head = 0;
 			phase = !phase;
 		}
+
 		if (tag && *tag == cqe.command_id)
 			*tag = -1;
-		ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
-		fn(nvmeq, ctx, &cqe);
+
+		if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
+			dev_warn(nvmeq->q_dmadev,
+				"invalid id %d completed on queue %d\n",
+				cqe.command_id, le16_to_cpu(cqe.sq_id));
+			continue;
+		}
+
+		/*
+		 * AEN requests are special as they don't time out and can
+		 * survive any kind of queue freeze and often don't respond to
+		 * aborts.  We don't even bother to allocate a struct request
+		 * for them but rather special case them here.
+		 */
+		if (unlikely(nvmeq->qid == 0 &&
+				cqe.command_id >= NVME_AQ_BLKMQ_DEPTH)) {
+			nvme_complete_async_event(nvmeq->dev, &cqe);
+			continue;
+		}
+
+		req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
+		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
+			u32 result = le32_to_cpu(cqe.result);
+			req->special = (void *)(uintptr_t)result;
+		}
+		blk_mq_complete_request(req, status >> 1);
+
 	}
 
 	/* If the controller ignores the cq head doorbell and continuously
@@ -1028,111 +827,15 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 	return 0;
 }
 
-/*
- * Returns 0 on success.  If the result is negative, it's a Linux error code;
- * if the result is positive, it's an NVM Express status code
- */
-int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void *buffer, void __user *ubuffer, unsigned bufflen,
-		u32 *result, unsigned timeout)
-{
-	bool write = cmd->common.opcode & 1;
-	struct bio *bio = NULL;
-	struct request *req;
-	int ret;
-
-	req = blk_mq_alloc_request(q, write, GFP_KERNEL, false);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
-
-	req->cmd_type = REQ_TYPE_DRV_PRIV;
-	req->cmd_flags |= REQ_FAILFAST_DRIVER;
-	req->__data_len = 0;
-	req->__sector = (sector_t) -1;
-	req->bio = req->biotail = NULL;
-
-	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
-
-	req->cmd = (unsigned char *)cmd;
-	req->cmd_len = sizeof(struct nvme_command);
-	req->special = (void *)0;
-
-	if (buffer && bufflen) {
-		ret = blk_rq_map_kern(q, req, buffer, bufflen,
-				      __GFP_DIRECT_RECLAIM);
-		if (ret)
-			goto out;
-	} else if (ubuffer && bufflen) {
-		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
-				      __GFP_DIRECT_RECLAIM);
-		if (ret)
-			goto out;
-		bio = req->bio;
-	}
-
-	blk_execute_rq(req->q, NULL, req, 0);
-	if (bio)
-		blk_rq_unmap_user(bio);
-	if (result)
-		*result = (u32)(uintptr_t)req->special;
-	ret = req->errors;
- out:
-	blk_mq_free_request(req);
-	return ret;
-}
-
-int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void *buffer, unsigned bufflen)
-{
-	return __nvme_submit_sync_cmd(q, cmd, buffer, NULL, bufflen, NULL, 0);
-}
-
-static int nvme_submit_async_admin_req(struct nvme_dev *dev)
+static void nvme_submit_async_event(struct nvme_dev *dev)
 {
-	struct nvme_queue *nvmeq = dev->queues[0];
 	struct nvme_command c;
-	struct nvme_cmd_info *cmd_info;
-	struct request *req;
-
-	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, true);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
-
-	req->cmd_flags |= REQ_NO_TIMEOUT;
-	cmd_info = blk_mq_rq_to_pdu(req);
-	nvme_set_info(cmd_info, NULL, async_req_completion);
 
 	memset(&c, 0, sizeof(c));
 	c.common.opcode = nvme_admin_async_event;
-	c.common.command_id = req->tag;
+	c.common.command_id = NVME_AQ_BLKMQ_DEPTH + --dev->ctrl.event_limit;
 
-	blk_mq_free_request(req);
-	__nvme_submit_cmd(nvmeq, &c);
-	return 0;
-}
-
-static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
-			struct nvme_command *cmd,
-			struct async_cmd_info *cmdinfo, unsigned timeout)
-{
-	struct nvme_queue *nvmeq = dev->queues[0];
-	struct request *req;
-	struct nvme_cmd_info *cmd_rq;
-
-	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
-
-	req->timeout = timeout;
-	cmd_rq = blk_mq_rq_to_pdu(req);
-	cmdinfo->req = req;
-	nvme_set_info(cmd_rq, cmdinfo, async_completion);
-	cmdinfo->status = -EINTR;
-
-	cmd->common.command_id = req->tag;
-
-	nvme_submit_cmd(nvmeq, cmd);
-	return 0;
+	__nvme_submit_cmd(dev->queues[0], &c);
 }
 
 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
@@ -1143,7 +846,7 @@ static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 	c.delete_queue.opcode = opcode;
 	c.delete_queue.qid = cpu_to_le16(id);
 
-	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
+	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 }
 
 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
@@ -1164,7 +867,7 @@ static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 	c.create_cq.cq_flags = cpu_to_le16(flags);
 	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 
-	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
+	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 }
 
 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
@@ -1185,7 +888,7 @@ static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 	c.create_sq.sq_flags = cpu_to_le16(flags);
 	c.create_sq.cqid = cpu_to_le16(qid);
 
-	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
+	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 }
 
 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
@@ -1198,195 +901,111 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 }
 
-int nvme_identify_ctrl(struct nvme_dev *dev, struct nvme_id_ctrl **id)
-{
-	struct nvme_command c = { };
-	int error;
-
-	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
-	c.identify.opcode = nvme_admin_identify;
-	c.identify.cns = cpu_to_le32(1);
-
-	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
-	if (!*id)
-		return -ENOMEM;
-
-	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
-			sizeof(struct nvme_id_ctrl));
-	if (error)
-		kfree(*id);
-	return error;
-}
-
-int nvme_identify_ns(struct nvme_dev *dev, unsigned nsid,
-		struct nvme_id_ns **id)
+static void abort_endio(struct request *req, int error)
 {
-	struct nvme_command c = { };
-	int error;
-
-	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
-	c.identify.opcode = nvme_admin_identify,
-	c.identify.nsid = cpu_to_le32(nsid),
-
-	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
-	if (!*id)
-		return -ENOMEM;
-
-	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
-			sizeof(struct nvme_id_ns));
-	if (error)
-		kfree(*id);
-	return error;
-}
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_queue *nvmeq = iod->nvmeq;
+	u32 result = (u32)(uintptr_t)req->special;
+	u16 status = req->errors;
 
-int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
-					dma_addr_t dma_addr, u32 *result)
-{
-	struct nvme_command c;
-
-	memset(&c, 0, sizeof(c));
-	c.features.opcode = nvme_admin_get_features;
-	c.features.nsid = cpu_to_le32(nsid);
-	c.features.prp1 = cpu_to_le64(dma_addr);
-	c.features.fid = cpu_to_le32(fid);
-
-	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
-			result, 0);
-}
-
-int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
-					dma_addr_t dma_addr, u32 *result)
-{
-	struct nvme_command c;
-
-	memset(&c, 0, sizeof(c));
-	c.features.opcode = nvme_admin_set_features;
-	c.features.prp1 = cpu_to_le64(dma_addr);
-	c.features.fid = cpu_to_le32(fid);
-	c.features.dword11 = cpu_to_le32(dword11);
-
-	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
-			result, 0);
-}
-
-int nvme_get_log_page(struct nvme_dev *dev, struct nvme_smart_log **log)
-{
-	struct nvme_command c = { };
-	int error;
-
-	c.common.opcode = nvme_admin_get_log_page,
-	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
-	c.common.cdw10[0] = cpu_to_le32(
-			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
-			 NVME_LOG_SMART),
-
-	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
-	if (!*log)
-		return -ENOMEM;
+	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
+	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
 
-	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
-			sizeof(struct nvme_smart_log));
-	if (error)
-		kfree(*log);
-	return error;
+	blk_mq_free_request(req);
 }
 
-/**
- * nvme_abort_req - Attempt aborting a request
- *
- * Schedule controller reset if the command was already aborted once before and
- * still hasn't been returned to the driver, or if this is the admin queue.
- */
-static void nvme_abort_req(struct request *req)
+static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 {
-	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
-	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_queue *nvmeq = iod->nvmeq;
 	struct nvme_dev *dev = nvmeq->dev;
 	struct request *abort_req;
-	struct nvme_cmd_info *abort_cmd;
 	struct nvme_command cmd;
 
-	if (!nvmeq->qid || cmd_rq->aborted) {
-		spin_lock(&dev_list_lock);
-		if (!__nvme_reset(dev)) {
-			dev_warn(dev->dev,
-				 "I/O %d QID %d timeout, reset controller\n",
-				 req->tag, nvmeq->qid);
-		}
-		spin_unlock(&dev_list_lock);
-		return;
+	/*
+	 * Shutdown immediately if controller times out while starting. The
+	 * reset work will see the pci device disabled when it gets the forced
+	 * cancellation error. All outstanding requests are completed on
+	 * shutdown, so we return BLK_EH_HANDLED.
+	 */
+	if (test_bit(NVME_CTRL_RESETTING, &dev->flags)) {
+		dev_warn(dev->dev,
+			 "I/O %d QID %d timeout, disable controller\n",
+			 req->tag, nvmeq->qid);
+		nvme_dev_disable(dev, false);
+		req->errors = NVME_SC_CANCELLED;
+		return BLK_EH_HANDLED;
 	}
 
-	if (!dev->abort_limit)
-		return;
+	/*
+ 	 * Shutdown the controller immediately and schedule a reset if the
+ 	 * command was already aborted once before and still hasn't been
+ 	 * returned to the driver, or if this is the admin queue.
+	 */
+	if (!nvmeq->qid || iod->aborted) {
+		dev_warn(dev->dev,
+			 "I/O %d QID %d timeout, reset controller\n",
+			 req->tag, nvmeq->qid);
+		nvme_dev_disable(dev, false);
+		queue_work(nvme_workq, &dev->reset_work);
 
-	abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
-									false);
-	if (IS_ERR(abort_req))
-		return;
+		/*
+		 * Mark the request as handled, since the inline shutdown
+		 * forces all outstanding requests to complete.
+		 */
+		req->errors = NVME_SC_CANCELLED;
+		return BLK_EH_HANDLED;
+	}
+
+	iod->aborted = 1;
 
-	abort_cmd = blk_mq_rq_to_pdu(abort_req);
-	nvme_set_info(abort_cmd, abort_req, abort_completion);
+	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
+		atomic_inc(&dev->ctrl.abort_limit);
+		return BLK_EH_RESET_TIMER;
+	}
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abort.opcode = nvme_admin_abort_cmd;
 	cmd.abort.cid = req->tag;
 	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
-	cmd.abort.command_id = abort_req->tag;
 
-	--dev->abort_limit;
-	cmd_rq->aborted = 1;
+	dev_warn(nvmeq->q_dmadev, "I/O %d QID %d timeout, aborting\n",
+				 req->tag, nvmeq->qid);
+
+	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
+			BLK_MQ_REQ_NOWAIT);
+	if (IS_ERR(abort_req)) {
+		atomic_inc(&dev->ctrl.abort_limit);
+		return BLK_EH_RESET_TIMER;
+	}
 
-	dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
-							nvmeq->qid);
-	nvme_submit_cmd(dev->queues[0], &cmd);
+	abort_req->timeout = ADMIN_TIMEOUT;
+	abort_req->end_io_data = NULL;
+	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
+
+	/*
+	 * The aborted req will be completed on receiving the abort req.
+	 * We enable the timer again. If hit twice, it'll cause a device reset,
+	 * as the device then is in a faulty state.
+	 */
+	return BLK_EH_RESET_TIMER;
 }
 
 static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
 {
 	struct nvme_queue *nvmeq = data;
-	void *ctx;
-	nvme_completion_fn fn;
-	struct nvme_cmd_info *cmd;
-	struct nvme_completion cqe;
+	int status;
 
 	if (!blk_mq_request_started(req))
 		return;
 
-	cmd = blk_mq_rq_to_pdu(req);
-
-	if (cmd->ctx == CMD_CTX_CANCELLED)
-		return;
+	dev_warn(nvmeq->q_dmadev,
+		 "Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
 
+	status = NVME_SC_ABORT_REQ;
 	if (blk_queue_dying(req->q))
-		cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
-	else
-		cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
-
-
-	dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
-						req->tag, nvmeq->qid);
-	ctx = cancel_cmd_info(cmd, &fn);
-	fn(nvmeq, ctx, &cqe);
-}
-
-static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
-{
-	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
-	struct nvme_queue *nvmeq = cmd->nvmeq;
-
-	dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
-							nvmeq->qid);
-	spin_lock_irq(&nvmeq->q_lock);
-	nvme_abort_req(req);
-	spin_unlock_irq(&nvmeq->q_lock);
-
-	/*
-	 * The aborted req will be completed on receiving the abort req.
-	 * We enable the timer again. If hit twice, it'll cause a device reset,
-	 * as the device then is in a faulty state.
-	 */
-	return BLK_EH_RESET_TIMER;
+		status |= NVME_SC_DNR;
+	blk_mq_complete_request(req, status);
 }
 
 static void nvme_free_queue(struct nvme_queue *nvmeq)
@@ -1429,8 +1048,8 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
 	nvmeq->cq_vector = -1;
 	spin_unlock_irq(&nvmeq->q_lock);
 
-	if (!nvmeq->qid && nvmeq->dev->admin_q)
-		blk_mq_freeze_queue_start(nvmeq->dev->admin_q);
+	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
+		blk_mq_stop_hw_queues(nvmeq->dev->ctrl.admin_q);
 
 	irq_set_affinity_hint(vector, NULL);
 	free_irq(vector, nvmeq);
@@ -1446,21 +1065,20 @@ static void nvme_clear_queue(struct nvme_queue *nvmeq)
 	spin_unlock_irq(&nvmeq->q_lock);
 }
 
-static void nvme_disable_queue(struct nvme_dev *dev, int qid)
+static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
 {
-	struct nvme_queue *nvmeq = dev->queues[qid];
+	struct nvme_queue *nvmeq = dev->queues[0];
 
 	if (!nvmeq)
 		return;
 	if (nvme_suspend_queue(nvmeq))
 		return;
 
-	/* Don't tell the adapter to delete the admin queue.
-	 * Don't tell a removed adapter to delete IO queues. */
-	if (qid && readl(&dev->bar->csts) != -1) {
-		adapter_delete_sq(dev, qid);
-		adapter_delete_cq(dev, qid);
-	}
+	if (shutdown)
+		nvme_shutdown_ctrl(&dev->ctrl);
+	else
+		nvme_disable_ctrl(&dev->ctrl, lo_hi_readq(
+						dev->bar + NVME_REG_CAP));
 
 	spin_lock_irq(&nvmeq->q_lock);
 	nvme_process_cq(nvmeq);
@@ -1471,11 +1089,12 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
 				int entry_size)
 {
 	int q_depth = dev->q_depth;
-	unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size);
+	unsigned q_size_aligned = roundup(q_depth * entry_size,
+					  dev->ctrl.page_size);
 
 	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
 		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
-		mem_per_q = round_down(mem_per_q, dev->page_size);
+		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
 		q_depth = div_u64(mem_per_q, entry_size);
 
 		/*
@@ -1494,8 +1113,8 @@ static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
 				int qid, int depth)
 {
 	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
-		unsigned offset = (qid - 1) *
-					roundup(SQ_SIZE(depth), dev->page_size);
+		unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
+						      dev->ctrl.page_size);
 		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
 		nvmeq->sq_cmds_io = dev->cmb + offset;
 	} else {
@@ -1526,7 +1145,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
 	nvmeq->q_dmadev = dev->dev;
 	nvmeq->dev = dev;
 	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
-			dev->instance, qid);
+			dev->ctrl.instance, qid);
 	spin_lock_init(&nvmeq->q_lock);
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
@@ -1603,79 +1222,9 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 	return result;
 }
 
-static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
-{
-	unsigned long timeout;
-	u32 bit = enabled ? NVME_CSTS_RDY : 0;
-
-	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
-
-	while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
-		msleep(100);
-		if (fatal_signal_pending(current))
-			return -EINTR;
-		if (time_after(jiffies, timeout)) {
-			dev_err(dev->dev,
-				"Device not ready; aborting %s\n", enabled ?
-						"initialisation" : "reset");
-			return -ENODEV;
-		}
-	}
-
-	return 0;
-}
-
-/*
- * If the device has been passed off to us in an enabled state, just clear
- * the enabled bit.  The spec says we should set the 'shutdown notification
- * bits', but doing so may cause the device to complete commands to the
- * admin queue ... and we don't know what memory that might be pointing at!
- */
-static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
-{
-	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
-	dev->ctrl_config &= ~NVME_CC_ENABLE;
-	writel(dev->ctrl_config, &dev->bar->cc);
-
-	return nvme_wait_ready(dev, cap, false);
-}
-
-static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
-{
-	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
-	dev->ctrl_config |= NVME_CC_ENABLE;
-	writel(dev->ctrl_config, &dev->bar->cc);
-
-	return nvme_wait_ready(dev, cap, true);
-}
-
-static int nvme_shutdown_ctrl(struct nvme_dev *dev)
-{
-	unsigned long timeout;
-
-	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
-	dev->ctrl_config |= NVME_CC_SHN_NORMAL;
-
-	writel(dev->ctrl_config, &dev->bar->cc);
-
-	timeout = SHUTDOWN_TIMEOUT + jiffies;
-	while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
-							NVME_CSTS_SHST_CMPLT) {
-		msleep(100);
-		if (fatal_signal_pending(current))
-			return -EINTR;
-		if (time_after(jiffies, timeout)) {
-			dev_err(dev->dev,
-				"Device shutdown incomplete; abort shutdown\n");
-			return -ENODEV;
-		}
-	}
-
-	return 0;
-}
-
 static struct blk_mq_ops nvme_mq_admin_ops = {
 	.queue_rq	= nvme_queue_rq,
+	.complete	= nvme_complete_rq,
 	.map_queue	= blk_mq_map_queue,
 	.init_hctx	= nvme_admin_init_hctx,
 	.exit_hctx      = nvme_admin_exit_hctx,
@@ -1685,6 +1234,7 @@ static struct blk_mq_ops nvme_mq_admin_ops = {
 
 static struct blk_mq_ops nvme_mq_ops = {
 	.queue_rq	= nvme_queue_rq,
+	.complete	= nvme_complete_rq,
 	.map_queue	= blk_mq_map_queue,
 	.init_hctx	= nvme_init_hctx,
 	.init_request	= nvme_init_request,
@@ -1694,19 +1244,23 @@ static struct blk_mq_ops nvme_mq_ops = {
 
 static void nvme_dev_remove_admin(struct nvme_dev *dev)
 {
-	if (dev->admin_q && !blk_queue_dying(dev->admin_q)) {
-		blk_cleanup_queue(dev->admin_q);
+	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
+		blk_cleanup_queue(dev->ctrl.admin_q);
 		blk_mq_free_tag_set(&dev->admin_tagset);
 	}
 }
 
 static int nvme_alloc_admin_tags(struct nvme_dev *dev)
 {
-	if (!dev->admin_q) {
+	if (!dev->ctrl.admin_q) {
 		dev->admin_tagset.ops = &nvme_mq_admin_ops;
 		dev->admin_tagset.nr_hw_queues = 1;
-		dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
-		dev->admin_tagset.reserved_tags = 1;
+
+		/*
+		 * Subtract one to leave an empty queue entry for 'Full Queue'
+		 * condition. See NVM-Express 1.2 specification, section 4.1.2.
+		 */
+		dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH - 1;
 		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
 		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
 		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
@@ -1715,18 +1269,18 @@ static int nvme_alloc_admin_tags(struct nvme_dev *dev)
 		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
 			return -ENOMEM;
 
-		dev->admin_q = blk_mq_init_queue(&dev->admin_tagset);
-		if (IS_ERR(dev->admin_q)) {
+		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
+		if (IS_ERR(dev->ctrl.admin_q)) {
 			blk_mq_free_tag_set(&dev->admin_tagset);
 			return -ENOMEM;
 		}
-		if (!blk_get_queue(dev->admin_q)) {
+		if (!blk_get_queue(dev->ctrl.admin_q)) {
 			nvme_dev_remove_admin(dev);
-			dev->admin_q = NULL;
+			dev->ctrl.admin_q = NULL;
 			return -ENODEV;
 		}
 	} else
-		blk_mq_unfreeze_queue(dev->admin_q);
+		blk_mq_start_stopped_hw_queues(dev->ctrl.admin_q, true);
 
 	return 0;
 }
@@ -1735,31 +1289,17 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
 {
 	int result;
 	u32 aqa;
-	u64 cap = lo_hi_readq(&dev->bar->cap);
+	u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
 	struct nvme_queue *nvmeq;
-	/*
-	 * default to a 4K page size, with the intention to update this
-	 * path in the future to accomodate architectures with differing
-	 * kernel and IO page sizes.
-	 */
-	unsigned page_shift = 12;
-	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
-
-	if (page_shift < dev_page_min) {
-		dev_err(dev->dev,
-				"Minimum device page size (%u) too large for "
-				"host (%u)\n", 1 << dev_page_min,
-				1 << page_shift);
-		return -ENODEV;
-	}
 
-	dev->subsystem = readl(&dev->bar->vs) >= NVME_VS(1, 1) ?
+	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
 						NVME_CAP_NSSRC(cap) : 0;
 
-	if (dev->subsystem && (readl(&dev->bar->csts) & NVME_CSTS_NSSRO))
-		writel(NVME_CSTS_NSSRO, &dev->bar->csts);
+	if (dev->subsystem &&
+	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
+		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
 
-	result = nvme_disable_ctrl(dev, cap);
+	result = nvme_disable_ctrl(&dev->ctrl, cap);
 	if (result < 0)
 		return result;
 
@@ -1773,18 +1313,11 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
 	aqa = nvmeq->q_depth - 1;
 	aqa |= aqa << 16;
 
-	dev->page_size = 1 << page_shift;
+	writel(aqa, dev->bar + NVME_REG_AQA);
+	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
+	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
 
-	dev->ctrl_config = NVME_CC_CSS_NVM;
-	dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
-	dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
-	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
-
-	writel(aqa, &dev->bar->aqa);
-	lo_hi_writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
-	lo_hi_writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
-
-	result = nvme_enable_ctrl(dev, cap);
+	result = nvme_enable_ctrl(&dev->ctrl, cap);
 	if (result)
 		goto free_nvmeq;
 
@@ -1802,406 +1335,6 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
 	return result;
 }
 
-static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
-{
-	struct nvme_dev *dev = ns->dev;
-	struct nvme_user_io io;
-	struct nvme_command c;
-	unsigned length, meta_len;
-	int status, write;
-	dma_addr_t meta_dma = 0;
-	void *meta = NULL;
-	void __user *metadata;
-
-	if (copy_from_user(&io, uio, sizeof(io)))
-		return -EFAULT;
-
-	switch (io.opcode) {
-	case nvme_cmd_write:
-	case nvme_cmd_read:
-	case nvme_cmd_compare:
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	length = (io.nblocks + 1) << ns->lba_shift;
-	meta_len = (io.nblocks + 1) * ns->ms;
-	metadata = (void __user *)(uintptr_t)io.metadata;
-	write = io.opcode & 1;
-
-	if (ns->ext) {
-		length += meta_len;
-		meta_len = 0;
-	}
-	if (meta_len) {
-		if (((io.metadata & 3) || !io.metadata) && !ns->ext)
-			return -EINVAL;
-
-		meta = dma_alloc_coherent(dev->dev, meta_len,
-						&meta_dma, GFP_KERNEL);
-
-		if (!meta) {
-			status = -ENOMEM;
-			goto unmap;
-		}
-		if (write) {
-			if (copy_from_user(meta, metadata, meta_len)) {
-				status = -EFAULT;
-				goto unmap;
-			}
-		}
-	}
-
-	memset(&c, 0, sizeof(c));
-	c.rw.opcode = io.opcode;
-	c.rw.flags = io.flags;
-	c.rw.nsid = cpu_to_le32(ns->ns_id);
-	c.rw.slba = cpu_to_le64(io.slba);
-	c.rw.length = cpu_to_le16(io.nblocks);
-	c.rw.control = cpu_to_le16(io.control);
-	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
-	c.rw.reftag = cpu_to_le32(io.reftag);
-	c.rw.apptag = cpu_to_le16(io.apptag);
-	c.rw.appmask = cpu_to_le16(io.appmask);
-	c.rw.metadata = cpu_to_le64(meta_dma);
-
-	status = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
-			(void __user *)(uintptr_t)io.addr, length, NULL, 0);
- unmap:
-	if (meta) {
-		if (status == NVME_SC_SUCCESS && !write) {
-			if (copy_to_user(metadata, meta, meta_len))
-				status = -EFAULT;
-		}
-		dma_free_coherent(dev->dev, meta_len, meta, meta_dma);
-	}
-	return status;
-}
-
-static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
-			struct nvme_passthru_cmd __user *ucmd)
-{
-	struct nvme_passthru_cmd cmd;
-	struct nvme_command c;
-	unsigned timeout = 0;
-	int status;
-
-	if (!capable(CAP_SYS_ADMIN))
-		return -EACCES;
-	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
-		return -EFAULT;
-
-	memset(&c, 0, sizeof(c));
-	c.common.opcode = cmd.opcode;
-	c.common.flags = cmd.flags;
-	c.common.nsid = cpu_to_le32(cmd.nsid);
-	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
-	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
-	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
-	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
-	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
-	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
-	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
-	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
-
-	if (cmd.timeout_ms)
-		timeout = msecs_to_jiffies(cmd.timeout_ms);
-
-	status = __nvme_submit_sync_cmd(ns ? ns->queue : dev->admin_q, &c,
-			NULL, (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
-			&cmd.result, timeout);
-	if (status >= 0) {
-		if (put_user(cmd.result, &ucmd->result))
-			return -EFAULT;
-	}
-
-	return status;
-}
-
-static int nvme_subsys_reset(struct nvme_dev *dev)
-{
-	if (!dev->subsystem)
-		return -ENOTTY;
-
-	writel(0x4E564D65, &dev->bar->nssr); /* "NVMe" */
-	return 0;
-}
-
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
-							unsigned long arg)
-{
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-
-	switch (cmd) {
-	case NVME_IOCTL_ID:
-		force_successful_syscall_return();
-		return ns->ns_id;
-	case NVME_IOCTL_ADMIN_CMD:
-		return nvme_user_cmd(ns->dev, NULL, (void __user *)arg);
-	case NVME_IOCTL_IO_CMD:
-		return nvme_user_cmd(ns->dev, ns, (void __user *)arg);
-	case NVME_IOCTL_SUBMIT_IO:
-		return nvme_submit_io(ns, (void __user *)arg);
-	case SG_GET_VERSION_NUM:
-		return nvme_sg_get_version_num((void __user *)arg);
-	case SG_IO:
-		return nvme_sg_io(ns, (void __user *)arg);
-	default:
-		return -ENOTTY;
-	}
-}
-
-#ifdef CONFIG_COMPAT
-static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
-					unsigned int cmd, unsigned long arg)
-{
-	switch (cmd) {
-	case SG_IO:
-		return -ENOIOCTLCMD;
-	}
-	return nvme_ioctl(bdev, mode, cmd, arg);
-}
-#else
-#define nvme_compat_ioctl	NULL
-#endif
-
-static void nvme_free_dev(struct kref *kref);
-static void nvme_free_ns(struct kref *kref)
-{
-	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
-
-	if (ns->type == NVME_NS_LIGHTNVM)
-		nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
-
-	spin_lock(&dev_list_lock);
-	ns->disk->private_data = NULL;
-	spin_unlock(&dev_list_lock);
-
-	kref_put(&ns->dev->kref, nvme_free_dev);
-	put_disk(ns->disk);
-	kfree(ns);
-}
-
-static int nvme_open(struct block_device *bdev, fmode_t mode)
-{
-	int ret = 0;
-	struct nvme_ns *ns;
-
-	spin_lock(&dev_list_lock);
-	ns = bdev->bd_disk->private_data;
-	if (!ns)
-		ret = -ENXIO;
-	else if (!kref_get_unless_zero(&ns->kref))
-		ret = -ENXIO;
-	spin_unlock(&dev_list_lock);
-
-	return ret;
-}
-
-static void nvme_release(struct gendisk *disk, fmode_t mode)
-{
-	struct nvme_ns *ns = disk->private_data;
-	kref_put(&ns->kref, nvme_free_ns);
-}
-
-static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
-{
-	/* some standard values */
-	geo->heads = 1 << 6;
-	geo->sectors = 1 << 5;
-	geo->cylinders = get_capacity(bd->bd_disk) >> 11;
-	return 0;
-}
-
-static void nvme_config_discard(struct nvme_ns *ns)
-{
-	u32 logical_block_size = queue_logical_block_size(ns->queue);
-	ns->queue->limits.discard_zeroes_data = 0;
-	ns->queue->limits.discard_alignment = logical_block_size;
-	ns->queue->limits.discard_granularity = logical_block_size;
-	blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
-	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
-}
-
-static int nvme_revalidate_disk(struct gendisk *disk)
-{
-	struct nvme_ns *ns = disk->private_data;
-	struct nvme_dev *dev = ns->dev;
-	struct nvme_id_ns *id;
-	u8 lbaf, pi_type;
-	u16 old_ms;
-	unsigned short bs;
-
-	if (nvme_identify_ns(dev, ns->ns_id, &id)) {
-		dev_warn(dev->dev, "%s: Identify failure nvme%dn%d\n", __func__,
-						dev->instance, ns->ns_id);
-		return -ENODEV;
-	}
-	if (id->ncap == 0) {
-		kfree(id);
-		return -ENODEV;
-	}
-
-	if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
-		if (nvme_nvm_register(ns->queue, disk->disk_name)) {
-			dev_warn(dev->dev,
-				"%s: LightNVM init failure\n", __func__);
-			kfree(id);
-			return -ENODEV;
-		}
-		ns->type = NVME_NS_LIGHTNVM;
-	}
-
-	old_ms = ns->ms;
-	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
-	ns->lba_shift = id->lbaf[lbaf].ds;
-	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
-	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
-
-	/*
-	 * If identify namespace failed, use default 512 byte block size so
-	 * block layer can use before failing read/write for 0 capacity.
-	 */
-	if (ns->lba_shift == 0)
-		ns->lba_shift = 9;
-	bs = 1 << ns->lba_shift;
-
-	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
-	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
-					id->dps & NVME_NS_DPS_PI_MASK : 0;
-
-	blk_mq_freeze_queue(disk->queue);
-	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
-				ns->ms != old_ms ||
-				bs != queue_logical_block_size(disk->queue) ||
-				(ns->ms && ns->ext)))
-		blk_integrity_unregister(disk);
-
-	ns->pi_type = pi_type;
-	blk_queue_logical_block_size(ns->queue, bs);
-
-	if (ns->ms && !ns->ext)
-		nvme_init_integrity(ns);
-
-	if ((ns->ms && !(ns->ms == 8 && ns->pi_type) &&
-						!blk_get_integrity(disk)) ||
-						ns->type == NVME_NS_LIGHTNVM)
-		set_capacity(disk, 0);
-	else
-		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
-
-	if (dev->oncs & NVME_CTRL_ONCS_DSM)
-		nvme_config_discard(ns);
-	blk_mq_unfreeze_queue(disk->queue);
-
-	kfree(id);
-	return 0;
-}
-
-static char nvme_pr_type(enum pr_type type)
-{
-	switch (type) {
-	case PR_WRITE_EXCLUSIVE:
-		return 1;
-	case PR_EXCLUSIVE_ACCESS:
-		return 2;
-	case PR_WRITE_EXCLUSIVE_REG_ONLY:
-		return 3;
-	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
-		return 4;
-	case PR_WRITE_EXCLUSIVE_ALL_REGS:
-		return 5;
-	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
-		return 6;
-	default:
-		return 0;
-	}
-};
-
-static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
-				u64 key, u64 sa_key, u8 op)
-{
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-	struct nvme_command c;
-	u8 data[16] = { 0, };
-
-	put_unaligned_le64(key, &data[0]);
-	put_unaligned_le64(sa_key, &data[8]);
-
-	memset(&c, 0, sizeof(c));
-	c.common.opcode = op;
-	c.common.nsid = cpu_to_le32(ns->ns_id);
-	c.common.cdw10[0] = cpu_to_le32(cdw10);
-
-	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
-}
-
-static int nvme_pr_register(struct block_device *bdev, u64 old,
-		u64 new, unsigned flags)
-{
-	u32 cdw10;
-
-	if (flags & ~PR_FL_IGNORE_KEY)
-		return -EOPNOTSUPP;
-
-	cdw10 = old ? 2 : 0;
-	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
-	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
-	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
-}
-
-static int nvme_pr_reserve(struct block_device *bdev, u64 key,
-		enum pr_type type, unsigned flags)
-{
-	u32 cdw10;
-
-	if (flags & ~PR_FL_IGNORE_KEY)
-		return -EOPNOTSUPP;
-
-	cdw10 = nvme_pr_type(type) << 8;
-	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
-	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
-}
-
-static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
-		enum pr_type type, bool abort)
-{
-	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
-	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
-}
-
-static int nvme_pr_clear(struct block_device *bdev, u64 key)
-{
-	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
-	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
-}
-
-static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
-{
-	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
-	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
-}
-
-static const struct pr_ops nvme_pr_ops = {
-	.pr_register	= nvme_pr_register,
-	.pr_reserve	= nvme_pr_reserve,
-	.pr_release	= nvme_pr_release,
-	.pr_preempt	= nvme_pr_preempt,
-	.pr_clear	= nvme_pr_clear,
-};
-
-static const struct block_device_operations nvme_fops = {
-	.owner		= THIS_MODULE,
-	.ioctl		= nvme_ioctl,
-	.compat_ioctl	= nvme_compat_ioctl,
-	.open		= nvme_open,
-	.release	= nvme_release,
-	.getgeo		= nvme_getgeo,
-	.revalidate_disk= nvme_revalidate_disk,
-	.pr_ops		= &nvme_pr_ops,
-};
-
 static int nvme_kthread(void *data)
 {
 	struct nvme_dev *dev, *next;
@@ -2211,14 +1344,20 @@ static int nvme_kthread(void *data)
 		spin_lock(&dev_list_lock);
 		list_for_each_entry_safe(dev, next, &dev_list, node) {
 			int i;
-			u32 csts = readl(&dev->bar->csts);
+			u32 csts = readl(dev->bar + NVME_REG_CSTS);
+
+			/*
+			 * Skip controllers currently under reset.
+			 */
+			if (work_pending(&dev->reset_work) || work_busy(&dev->reset_work))
+				continue;
 
 			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
 							csts & NVME_CSTS_CFS) {
-				if (!__nvme_reset(dev)) {
+				if (queue_work(nvme_workq, &dev->reset_work)) {
 					dev_warn(dev->dev,
 						"Failed status: %x, reset controller\n",
-						readl(&dev->bar->csts));
+						readl(dev->bar + NVME_REG_CSTS));
 				}
 				continue;
 			}
@@ -2229,11 +1368,8 @@ static int nvme_kthread(void *data)
 				spin_lock_irq(&nvmeq->q_lock);
 				nvme_process_cq(nvmeq);
 
-				while ((i == 0) && (dev->event_limit > 0)) {
-					if (nvme_submit_async_admin_req(dev))
-						break;
-					dev->event_limit--;
-				}
+				while (i == 0 && dev->ctrl.event_limit > 0)
+					nvme_submit_async_event(dev);
 				spin_unlock_irq(&nvmeq->q_lock);
 			}
 		}
@@ -2243,127 +1379,33 @@ static int nvme_kthread(void *data)
 	return 0;
 }
 
-static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
-{
-	struct nvme_ns *ns;
-	struct gendisk *disk;
-	int node = dev_to_node(dev->dev);
-
-	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
-	if (!ns)
-		return;
-
-	ns->queue = blk_mq_init_queue(&dev->tagset);
-	if (IS_ERR(ns->queue))
-		goto out_free_ns;
-	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
-	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
-	ns->dev = dev;
-	ns->queue->queuedata = ns;
-
-	disk = alloc_disk_node(0, node);
-	if (!disk)
-		goto out_free_queue;
-
-	kref_init(&ns->kref);
-	ns->ns_id = nsid;
-	ns->disk = disk;
-	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
-	list_add_tail(&ns->list, &dev->namespaces);
-
-	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
-	if (dev->max_hw_sectors) {
-		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
-		blk_queue_max_segments(ns->queue,
-			(dev->max_hw_sectors / (dev->page_size >> 9)) + 1);
-	}
-	if (dev->stripe_size)
-		blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
-	if (dev->vwc & NVME_CTRL_VWC_PRESENT)
-		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
-	blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
-
-	disk->major = nvme_major;
-	disk->first_minor = 0;
-	disk->fops = &nvme_fops;
-	disk->private_data = ns;
-	disk->queue = ns->queue;
-	disk->driverfs_dev = dev->device;
-	disk->flags = GENHD_FL_EXT_DEVT;
-	sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
-
-	/*
-	 * Initialize capacity to 0 until we establish the namespace format and
-	 * setup integrity extentions if necessary. The revalidate_disk after
-	 * add_disk allows the driver to register with integrity if the format
-	 * requires it.
-	 */
-	set_capacity(disk, 0);
-	if (nvme_revalidate_disk(ns->disk))
-		goto out_free_disk;
-
-	kref_get(&dev->kref);
-	if (ns->type != NVME_NS_LIGHTNVM) {
-		add_disk(ns->disk);
-		if (ns->ms) {
-			struct block_device *bd = bdget_disk(ns->disk, 0);
-			if (!bd)
-				return;
-			if (blkdev_get(bd, FMODE_READ, NULL)) {
-				bdput(bd);
-				return;
-			}
-			blkdev_reread_part(bd);
-			blkdev_put(bd, FMODE_READ);
-		}
-	}
-	return;
- out_free_disk:
-	kfree(disk);
-	list_del(&ns->list);
- out_free_queue:
-	blk_cleanup_queue(ns->queue);
- out_free_ns:
-	kfree(ns);
-}
-
-/*
- * Create I/O queues.  Failing to create an I/O queue is not an issue,
- * we can continue with less than the desired amount of queues, and
- * even a controller without I/O queues an still be used to issue
- * admin commands.  This might be useful to upgrade a buggy firmware
- * for example.
- */
-static void nvme_create_io_queues(struct nvme_dev *dev)
+static int nvme_create_io_queues(struct nvme_dev *dev)
 {
 	unsigned i;
+	int ret = 0;
 
-	for (i = dev->queue_count; i <= dev->max_qid; i++)
-		if (!nvme_alloc_queue(dev, i, dev->q_depth))
+	for (i = dev->queue_count; i <= dev->max_qid; i++) {
+		if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
+			ret = -ENOMEM;
 			break;
+		}
+	}
 
-	for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
-		if (nvme_create_queue(dev->queues[i], i)) {
+	for (i = dev->online_queues; i <= dev->queue_count - 1; i++) {
+		ret = nvme_create_queue(dev->queues[i], i);
+		if (ret) {
 			nvme_free_queues(dev, i);
 			break;
 		}
-}
-
-static int set_queue_count(struct nvme_dev *dev, int count)
-{
-	int status;
-	u32 result;
-	u32 q_count = (count - 1) | ((count - 1) << 16);
-
-	status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
-								&result);
-	if (status < 0)
-		return status;
-	if (status > 0) {
-		dev_err(dev->dev, "Could not set queue count (%d)\n", status);
-		return 0;
 	}
-	return min(result & 0xffff, result >> 16) + 1;
+
+	/*
+	 * Ignore failing Create SQ/CQ commands, we can continue with less
+	 * than the desired aount of queues, and even a controller without
+	 * I/O queues an still be used to issue admin commands.  This might
+	 * be useful to upgrade a buggy firmware for example.
+	 */
+	return ret >= 0 ? 0 : ret;
 }
 
 static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
@@ -2378,11 +1420,11 @@ static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
 	if (!use_cmb_sqes)
 		return NULL;
 
-	dev->cmbsz = readl(&dev->bar->cmbsz);
+	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
 	if (!(NVME_CMB_SZ(dev->cmbsz)))
 		return NULL;
 
-	cmbloc = readl(&dev->bar->cmbloc);
+	cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
 
 	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
 	size = szu * NVME_CMB_SZ(dev->cmbsz);
@@ -2430,11 +1472,20 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 	int result, i, vecs, nr_io_queues, size;
 
 	nr_io_queues = num_possible_cpus();
-	result = set_queue_count(dev, nr_io_queues);
-	if (result <= 0)
+	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
+	if (result < 0)
 		return result;
-	if (result < nr_io_queues)
-		nr_io_queues = result;
+
+	/*
+	 * Degraded controllers might return an error when setting the queue
+	 * count.  We still want to be able to bring them online and offer
+	 * access to the admin queue, as that might be only way to fix them up.
+	 */
+	if (result > 0) {
+		dev_err(dev->dev, "Could not set queue count (%d)\n", result);
+		nr_io_queues = 0;
+		result = 0;
+	}
 
 	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
 		result = nvme_cmb_qdepth(dev, nr_io_queues,
@@ -2456,7 +1507,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 				return -ENOMEM;
 			size = db_bar_size(dev, nr_io_queues);
 		} while (1);
-		dev->dbs = ((void __iomem *)dev->bar) + 4096;
+		dev->dbs = dev->bar + 4096;
 		adminq->q_db = dev->dbs;
 	}
 
@@ -2500,115 +1551,115 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 
 	/* Free previously allocated queues that are no longer usable */
 	nvme_free_queues(dev, nr_io_queues + 1);
-	nvme_create_io_queues(dev);
-
-	return 0;
+	return nvme_create_io_queues(dev);
 
  free_queues:
 	nvme_free_queues(dev, 1);
 	return result;
 }
 
-static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
+static void nvme_set_irq_hints(struct nvme_dev *dev)
 {
-	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
-	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
+	struct nvme_queue *nvmeq;
+	int i;
 
-	return nsa->ns_id - nsb->ns_id;
-}
+	for (i = 0; i < dev->online_queues; i++) {
+		nvmeq = dev->queues[i];
 
-static struct nvme_ns *nvme_find_ns(struct nvme_dev *dev, unsigned nsid)
-{
-	struct nvme_ns *ns;
+		if (!nvmeq->tags || !(*nvmeq->tags))
+			continue;
 
-	list_for_each_entry(ns, &dev->namespaces, list) {
-		if (ns->ns_id == nsid)
-			return ns;
-		if (ns->ns_id > nsid)
-			break;
+		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
+					blk_mq_tags_cpumask(*nvmeq->tags));
 	}
-	return NULL;
 }
 
-static inline bool nvme_io_incapable(struct nvme_dev *dev)
+static void nvme_dev_scan(struct work_struct *work)
 {
-	return (!dev->bar || readl(&dev->bar->csts) & NVME_CSTS_CFS ||
-							dev->online_queues < 2);
+	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
+
+	if (!dev->tagset.tags)
+		return;
+	nvme_scan_namespaces(&dev->ctrl);
+	nvme_set_irq_hints(dev);
 }
 
-static void nvme_ns_remove(struct nvme_ns *ns)
+static void nvme_del_queue_end(struct request *req, int error)
 {
-	bool kill = nvme_io_incapable(ns->dev) && !blk_queue_dying(ns->queue);
-
-	if (kill) {
-		blk_set_queue_dying(ns->queue);
+	struct nvme_queue *nvmeq = req->end_io_data;
 
-		/*
-		 * The controller was shutdown first if we got here through
-		 * device removal. The shutdown may requeue outstanding
-		 * requests. These need to be aborted immediately so
-		 * del_gendisk doesn't block indefinitely for their completion.
-		 */
-		blk_mq_abort_requeue_list(ns->queue);
-	}
-	if (ns->disk->flags & GENHD_FL_UP)
-		del_gendisk(ns->disk);
-	if (kill || !blk_queue_dying(ns->queue)) {
-		blk_mq_abort_requeue_list(ns->queue);
-		blk_cleanup_queue(ns->queue);
-	}
-	list_del_init(&ns->list);
-	kref_put(&ns->kref, nvme_free_ns);
+	blk_mq_free_request(req);
+	complete(&nvmeq->dev->ioq_wait);
 }
 
-static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
+static void nvme_del_cq_end(struct request *req, int error)
 {
-	struct nvme_ns *ns, *next;
-	unsigned i;
+	struct nvme_queue *nvmeq = req->end_io_data;
 
-	for (i = 1; i <= nn; i++) {
-		ns = nvme_find_ns(dev, i);
-		if (ns) {
-			if (revalidate_disk(ns->disk))
-				nvme_ns_remove(ns);
-		} else
-			nvme_alloc_ns(dev, i);
-	}
-	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
-		if (ns->ns_id > nn)
-			nvme_ns_remove(ns);
+	if (!error) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&nvmeq->q_lock, flags);
+		nvme_process_cq(nvmeq);
+		spin_unlock_irqrestore(&nvmeq->q_lock, flags);
 	}
-	list_sort(NULL, &dev->namespaces, ns_cmp);
+
+	nvme_del_queue_end(req, error);
 }
 
-static void nvme_set_irq_hints(struct nvme_dev *dev)
+static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
 {
-	struct nvme_queue *nvmeq;
-	int i;
+	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
+	struct request *req;
+	struct nvme_command cmd;
 
-	for (i = 0; i < dev->online_queues; i++) {
-		nvmeq = dev->queues[i];
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.delete_queue.opcode = opcode;
+	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
 
-		if (!nvmeq->tags || !(*nvmeq->tags))
-			continue;
+	req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
 
-		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
-					blk_mq_tags_cpumask(*nvmeq->tags));
-	}
+	req->timeout = ADMIN_TIMEOUT;
+	req->end_io_data = nvmeq;
+
+	blk_execute_rq_nowait(q, NULL, req, false,
+			opcode == nvme_admin_delete_cq ?
+				nvme_del_cq_end : nvme_del_queue_end);
+	return 0;
 }
 
-static void nvme_dev_scan(struct work_struct *work)
+static void nvme_disable_io_queues(struct nvme_dev *dev)
 {
-	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
-	struct nvme_id_ctrl *ctrl;
+	int pass;
+	unsigned long timeout;
+	u8 opcode = nvme_admin_delete_sq;
 
-	if (!dev->tagset.tags)
-		return;
-	if (nvme_identify_ctrl(dev, &ctrl))
-		return;
-	nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
-	kfree(ctrl);
-	nvme_set_irq_hints(dev);
+	for (pass = 0; pass < 2; pass++) {
+		int sent = 0, i = dev->queue_count - 1;
+
+		reinit_completion(&dev->ioq_wait);
+ retry:
+		timeout = ADMIN_TIMEOUT;
+		for (; i > 0; i--) {
+			struct nvme_queue *nvmeq = dev->queues[i];
+
+			if (!pass)
+				nvme_suspend_queue(nvmeq);
+			if (nvme_delete_queue(nvmeq, opcode))
+				break;
+			++sent;
+		}
+		while (sent--) {
+			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
+			if (timeout == 0)
+				return;
+			if (i)
+				goto retry;
+		}
+		opcode = nvme_admin_delete_cq;
+	}
 }
 
 /*
@@ -2619,42 +1670,7 @@ static void nvme_dev_scan(struct work_struct *work)
  */
 static int nvme_dev_add(struct nvme_dev *dev)
 {
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	int res;
-	struct nvme_id_ctrl *ctrl;
-	int shift = NVME_CAP_MPSMIN(lo_hi_readq(&dev->bar->cap)) + 12;
-
-	res = nvme_identify_ctrl(dev, &ctrl);
-	if (res) {
-		dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
-		return -EIO;
-	}
-
-	dev->oncs = le16_to_cpup(&ctrl->oncs);
-	dev->abort_limit = ctrl->acl + 1;
-	dev->vwc = ctrl->vwc;
-	memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
-	memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
-	memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
-	if (ctrl->mdts)
-		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
-	else
-		dev->max_hw_sectors = UINT_MAX;
-	if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
-			(pdev->device == 0x0953) && ctrl->vs[3]) {
-		unsigned int max_hw_sectors;
-
-		dev->stripe_size = 1 << (ctrl->vs[3] + shift);
-		max_hw_sectors = dev->stripe_size >> (shift - 9);
-		if (dev->max_hw_sectors) {
-			dev->max_hw_sectors = min(max_hw_sectors,
-							dev->max_hw_sectors);
-		} else
-			dev->max_hw_sectors = max_hw_sectors;
-	}
-	kfree(ctrl);
-
-	if (!dev->tagset.tags) {
+	if (!dev->ctrl.tagset) {
 		dev->tagset.ops = &nvme_mq_ops;
 		dev->tagset.nr_hw_queues = dev->online_queues - 1;
 		dev->tagset.timeout = NVME_IO_TIMEOUT;
@@ -2667,8 +1683,9 @@ static int nvme_dev_add(struct nvme_dev *dev)
 
 		if (blk_mq_alloc_tag_set(&dev->tagset))
 			return 0;
+		dev->ctrl.tagset = &dev->tagset;
 	}
-	schedule_work(&dev->scan_work);
+	queue_work(nvme_workq, &dev->scan_work);
 	return 0;
 }
 
@@ -2698,7 +1715,7 @@ static int nvme_dev_map(struct nvme_dev *dev)
 	if (!dev->bar)
 		goto disable;
 
-	if (readl(&dev->bar->csts) == -1) {
+	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
 		result = -ENODEV;
 		goto unmap;
 	}
@@ -2713,10 +1730,11 @@ static int nvme_dev_map(struct nvme_dev *dev)
 			goto unmap;
 	}
 
-	cap = lo_hi_readq(&dev->bar->cap);
+	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
+
 	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
 	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
-	dev->dbs = ((void __iomem *)dev->bar) + 4096;
+	dev->dbs = dev->bar + 4096;
 
 	/*
 	 * Temporary fix for the Apple controller found in the MacBook8,1 and
@@ -2729,9 +1747,11 @@ static int nvme_dev_map(struct nvme_dev *dev)
 			dev->q_depth);
 	}
 
-	if (readl(&dev->bar->vs) >= NVME_VS(1, 2))
+	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
 		dev->cmb = nvme_map_cmb(dev);
 
+	pci_enable_pcie_error_reporting(pdev);
+	pci_save_state(pdev);
 	return 0;
 
  unmap:
@@ -2759,152 +1779,34 @@ static void nvme_dev_unmap(struct nvme_dev *dev)
 		pci_release_regions(pdev);
 	}
 
-	if (pci_is_enabled(pdev))
+	if (pci_is_enabled(pdev)) {
+		pci_disable_pcie_error_reporting(pdev);
 		pci_disable_device(pdev);
-}
-
-struct nvme_delq_ctx {
-	struct task_struct *waiter;
-	struct kthread_worker *worker;
-	atomic_t refcount;
-};
-
-static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
-{
-	dq->waiter = current;
-	mb();
-
-	for (;;) {
-		set_current_state(TASK_KILLABLE);
-		if (!atomic_read(&dq->refcount))
-			break;
-		if (!schedule_timeout(ADMIN_TIMEOUT) ||
-					fatal_signal_pending(current)) {
-			/*
-			 * Disable the controller first since we can't trust it
-			 * at this point, but leave the admin queue enabled
-			 * until all queue deletion requests are flushed.
-			 * FIXME: This may take a while if there are more h/w
-			 * queues than admin tags.
-			 */
-			set_current_state(TASK_RUNNING);
-			nvme_disable_ctrl(dev, lo_hi_readq(&dev->bar->cap));
-			nvme_clear_queue(dev->queues[0]);
-			flush_kthread_worker(dq->worker);
-			nvme_disable_queue(dev, 0);
-			return;
-		}
 	}
-	set_current_state(TASK_RUNNING);
-}
-
-static void nvme_put_dq(struct nvme_delq_ctx *dq)
-{
-	atomic_dec(&dq->refcount);
-	if (dq->waiter)
-		wake_up_process(dq->waiter);
-}
-
-static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
-{
-	atomic_inc(&dq->refcount);
-	return dq;
-}
-
-static void nvme_del_queue_end(struct nvme_queue *nvmeq)
-{
-	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
-	nvme_put_dq(dq);
-
-	spin_lock_irq(&nvmeq->q_lock);
-	nvme_process_cq(nvmeq);
-	spin_unlock_irq(&nvmeq->q_lock);
-}
-
-static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
-						kthread_work_func_t fn)
-{
-	struct nvme_command c;
-
-	memset(&c, 0, sizeof(c));
-	c.delete_queue.opcode = opcode;
-	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);
-
-	init_kthread_work(&nvmeq->cmdinfo.work, fn);
-	return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
-								ADMIN_TIMEOUT);
-}
-
-static void nvme_del_cq_work_handler(struct kthread_work *work)
-{
-	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
-							cmdinfo.work);
-	nvme_del_queue_end(nvmeq);
 }
 
-static int nvme_delete_cq(struct nvme_queue *nvmeq)
+static int nvme_dev_list_add(struct nvme_dev *dev)
 {
-	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
-						nvme_del_cq_work_handler);
-}
-
-static void nvme_del_sq_work_handler(struct kthread_work *work)
-{
-	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
-							cmdinfo.work);
-	int status = nvmeq->cmdinfo.status;
-
-	if (!status)
-		status = nvme_delete_cq(nvmeq);
-	if (status)
-		nvme_del_queue_end(nvmeq);
-}
-
-static int nvme_delete_sq(struct nvme_queue *nvmeq)
-{
-	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
-						nvme_del_sq_work_handler);
-}
-
-static void nvme_del_queue_start(struct kthread_work *work)
-{
-	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
-							cmdinfo.work);
-	if (nvme_delete_sq(nvmeq))
-		nvme_del_queue_end(nvmeq);
-}
+	bool start_thread = false;
 
-static void nvme_disable_io_queues(struct nvme_dev *dev)
-{
-	int i;
-	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
-	struct nvme_delq_ctx dq;
-	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
-					&worker, "nvme%d", dev->instance);
-
-	if (IS_ERR(kworker_task)) {
-		dev_err(dev->dev,
-			"Failed to create queue del task\n");
-		for (i = dev->queue_count - 1; i > 0; i--)
-			nvme_disable_queue(dev, i);
-		return;
+	spin_lock(&dev_list_lock);
+	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
+		start_thread = true;
+		nvme_thread = NULL;
 	}
+	list_add(&dev->node, &dev_list);
+	spin_unlock(&dev_list_lock);
 
-	dq.waiter = NULL;
-	atomic_set(&dq.refcount, 0);
-	dq.worker = &worker;
-	for (i = dev->queue_count - 1; i > 0; i--) {
-		struct nvme_queue *nvmeq = dev->queues[i];
+	if (start_thread) {
+		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
+		wake_up_all(&nvme_kthread_wait);
+	} else
+		wait_event_killable(nvme_kthread_wait, nvme_thread);
 
-		if (nvme_suspend_queue(nvmeq))
-			continue;
-		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
-		nvmeq->cmdinfo.worker = dq.worker;
-		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
-		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
-	}
-	nvme_wait_dq(&dq, dev);
-	kthread_stop(kworker_task);
+	if (IS_ERR_OR_NULL(nvme_thread))
+		return nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
+
+	return 0;
 }
 
 /*
@@ -2927,44 +1829,17 @@ static void nvme_dev_list_remove(struct nvme_dev *dev)
 		kthread_stop(tmp);
 }
 
-static void nvme_freeze_queues(struct nvme_dev *dev)
-{
-	struct nvme_ns *ns;
-
-	list_for_each_entry(ns, &dev->namespaces, list) {
-		blk_mq_freeze_queue_start(ns->queue);
-
-		spin_lock_irq(ns->queue->queue_lock);
-		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
-		spin_unlock_irq(ns->queue->queue_lock);
-
-		blk_mq_cancel_requeue_work(ns->queue);
-		blk_mq_stop_hw_queues(ns->queue);
-	}
-}
-
-static void nvme_unfreeze_queues(struct nvme_dev *dev)
-{
-	struct nvme_ns *ns;
-
-	list_for_each_entry(ns, &dev->namespaces, list) {
-		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
-		blk_mq_unfreeze_queue(ns->queue);
-		blk_mq_start_stopped_hw_queues(ns->queue, true);
-		blk_mq_kick_requeue_list(ns->queue);
-	}
-}
-
-static void nvme_dev_shutdown(struct nvme_dev *dev)
+static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 {
 	int i;
 	u32 csts = -1;
 
 	nvme_dev_list_remove(dev);
 
+	mutex_lock(&dev->shutdown_lock);
 	if (dev->bar) {
-		nvme_freeze_queues(dev);
-		csts = readl(&dev->bar->csts);
+		nvme_stop_queues(&dev->ctrl);
+		csts = readl(dev->bar + NVME_REG_CSTS);
 	}
 	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
 		for (i = dev->queue_count - 1; i >= 0; i--) {
@@ -2973,30 +1848,13 @@ static void nvme_dev_shutdown(struct nvme_dev *dev)
 		}
 	} else {
 		nvme_disable_io_queues(dev);
-		nvme_shutdown_ctrl(dev);
-		nvme_disable_queue(dev, 0);
+		nvme_disable_admin_queue(dev, shutdown);
 	}
 	nvme_dev_unmap(dev);
 
 	for (i = dev->queue_count - 1; i >= 0; i--)
 		nvme_clear_queue(dev->queues[i]);
-}
-
-static void nvme_dev_remove(struct nvme_dev *dev)
-{
-	struct nvme_ns *ns, *next;
-
-	if (nvme_io_incapable(dev)) {
-		/*
-		 * If the device is not capable of IO (surprise hot-removal,
-		 * for example), we need to quiesce prior to deleting the
-		 * namespaces. This will end outstanding requests and prevent
-		 * attempts to sync dirty data.
-		 */
-		nvme_dev_shutdown(dev);
-	}
-	list_for_each_entry_safe(ns, next, &dev->namespaces, list)
-		nvme_ns_remove(ns);
+	mutex_unlock(&dev->shutdown_lock);
 }
 
 static int nvme_setup_prp_pools(struct nvme_dev *dev)
@@ -3022,119 +1880,36 @@ static void nvme_release_prp_pools(struct nvme_dev *dev)
 	dma_pool_destroy(dev->prp_small_pool);
 }
 
-static DEFINE_IDA(nvme_instance_ida);
-
-static int nvme_set_instance(struct nvme_dev *dev)
+static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
 {
-	int instance, error;
-
-	do {
-		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
-			return -ENODEV;
-
-		spin_lock(&dev_list_lock);
-		error = ida_get_new(&nvme_instance_ida, &instance);
-		spin_unlock(&dev_list_lock);
-	} while (error == -EAGAIN);
-
-	if (error)
-		return -ENODEV;
-
-	dev->instance = instance;
-	return 0;
-}
-
-static void nvme_release_instance(struct nvme_dev *dev)
-{
-	spin_lock(&dev_list_lock);
-	ida_remove(&nvme_instance_ida, dev->instance);
-	spin_unlock(&dev_list_lock);
-}
-
-static void nvme_free_dev(struct kref *kref)
-{
-	struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
+	struct nvme_dev *dev = to_nvme_dev(ctrl);
 
 	put_device(dev->dev);
-	put_device(dev->device);
-	nvme_release_instance(dev);
 	if (dev->tagset.tags)
 		blk_mq_free_tag_set(&dev->tagset);
-	if (dev->admin_q)
-		blk_put_queue(dev->admin_q);
+	if (dev->ctrl.admin_q)
+		blk_put_queue(dev->ctrl.admin_q);
 	kfree(dev->queues);
 	kfree(dev->entry);
 	kfree(dev);
 }
 
-static int nvme_dev_open(struct inode *inode, struct file *f)
-{
-	struct nvme_dev *dev;
-	int instance = iminor(inode);
-	int ret = -ENODEV;
-
-	spin_lock(&dev_list_lock);
-	list_for_each_entry(dev, &dev_list, node) {
-		if (dev->instance == instance) {
-			if (!dev->admin_q) {
-				ret = -EWOULDBLOCK;
-				break;
-			}
-			if (!kref_get_unless_zero(&dev->kref))
-				break;
-			f->private_data = dev;
-			ret = 0;
-			break;
-		}
-	}
-	spin_unlock(&dev_list_lock);
-
-	return ret;
-}
-
-static int nvme_dev_release(struct inode *inode, struct file *f)
+static void nvme_reset_work(struct work_struct *work)
 {
-	struct nvme_dev *dev = f->private_data;
-	kref_put(&dev->kref, nvme_free_dev);
-	return 0;
-}
+	struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
+	int result;
 
-static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
-{
-	struct nvme_dev *dev = f->private_data;
-	struct nvme_ns *ns;
-
-	switch (cmd) {
-	case NVME_IOCTL_ADMIN_CMD:
-		return nvme_user_cmd(dev, NULL, (void __user *)arg);
-	case NVME_IOCTL_IO_CMD:
-		if (list_empty(&dev->namespaces))
-			return -ENOTTY;
-		ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
-		return nvme_user_cmd(dev, ns, (void __user *)arg);
-	case NVME_IOCTL_RESET:
-		dev_warn(dev->dev, "resetting controller\n");
-		return nvme_reset(dev);
-	case NVME_IOCTL_SUBSYS_RESET:
-		return nvme_subsys_reset(dev);
-	default:
-		return -ENOTTY;
-	}
-}
+	if (WARN_ON(test_bit(NVME_CTRL_RESETTING, &dev->flags)))
+		goto out;
 
-static const struct file_operations nvme_dev_fops = {
-	.owner		= THIS_MODULE,
-	.open		= nvme_dev_open,
-	.release	= nvme_dev_release,
-	.unlocked_ioctl	= nvme_dev_ioctl,
-	.compat_ioctl	= nvme_dev_ioctl,
-};
+	/*
+	 * If we're called to reset a live controller first shut it down before
+	 * moving on.
+	 */
+	if (dev->bar)
+		nvme_dev_disable(dev, false);
 
-static void nvme_probe_work(struct work_struct *work)
-{
-	struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
-	bool start_thread = false;
-	int result;
+	set_bit(NVME_CTRL_RESETTING, &dev->flags);
 
 	result = nvme_dev_map(dev);
 	if (result)
@@ -3144,35 +1919,24 @@ static void nvme_probe_work(struct work_struct *work)
 	if (result)
 		goto unmap;
 
-	spin_lock(&dev_list_lock);
-	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
-		start_thread = true;
-		nvme_thread = NULL;
-	}
-	list_add(&dev->node, &dev_list);
-	spin_unlock(&dev_list_lock);
-
-	if (start_thread) {
-		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
-		wake_up_all(&nvme_kthread_wait);
-	} else
-		wait_event_killable(nvme_kthread_wait, nvme_thread);
-
-	if (IS_ERR_OR_NULL(nvme_thread)) {
-		result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
-		goto disable;
-	}
-
 	nvme_init_queue(dev->queues[0], 0);
 	result = nvme_alloc_admin_tags(dev);
 	if (result)
 		goto disable;
 
+	result = nvme_init_identify(&dev->ctrl);
+	if (result)
+		goto free_tags;
+
 	result = nvme_setup_io_queues(dev);
 	if (result)
 		goto free_tags;
 
-	dev->event_limit = 1;
+	dev->ctrl.event_limit = NVME_NR_AEN_COMMANDS;
+
+	result = nvme_dev_list_add(dev);
+	if (result)
+		goto remove;
 
 	/*
 	 * Keep the controller around but remove all namespaces if we don't have
@@ -3180,117 +1944,98 @@ static void nvme_probe_work(struct work_struct *work)
 	 */
 	if (dev->online_queues < 2) {
 		dev_warn(dev->dev, "IO queues not created\n");
-		nvme_dev_remove(dev);
+		nvme_remove_namespaces(&dev->ctrl);
 	} else {
-		nvme_unfreeze_queues(dev);
+		nvme_start_queues(&dev->ctrl);
 		nvme_dev_add(dev);
 	}
 
+	clear_bit(NVME_CTRL_RESETTING, &dev->flags);
 	return;
 
+ remove:
+	nvme_dev_list_remove(dev);
  free_tags:
 	nvme_dev_remove_admin(dev);
-	blk_put_queue(dev->admin_q);
-	dev->admin_q = NULL;
+	blk_put_queue(dev->ctrl.admin_q);
+	dev->ctrl.admin_q = NULL;
 	dev->queues[0]->tags = NULL;
  disable:
-	nvme_disable_queue(dev, 0);
-	nvme_dev_list_remove(dev);
+	nvme_disable_admin_queue(dev, false);
  unmap:
 	nvme_dev_unmap(dev);
  out:
-	if (!work_busy(&dev->reset_work))
-		nvme_dead_ctrl(dev);
+	nvme_remove_dead_ctrl(dev);
 }
 
-static int nvme_remove_dead_ctrl(void *arg)
+static void nvme_remove_dead_ctrl_work(struct work_struct *work)
 {
-	struct nvme_dev *dev = (struct nvme_dev *)arg;
+	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 
 	if (pci_get_drvdata(pdev))
 		pci_stop_and_remove_bus_device_locked(pdev);
-	kref_put(&dev->kref, nvme_free_dev);
-	return 0;
+	nvme_put_ctrl(&dev->ctrl);
 }
 
-static void nvme_dead_ctrl(struct nvme_dev *dev)
+static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
 {
-	dev_warn(dev->dev, "Device failed to resume\n");
-	kref_get(&dev->kref);
-	if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
-						dev->instance))) {
-		dev_err(dev->dev,
-			"Failed to start controller remove task\n");
-		kref_put(&dev->kref, nvme_free_dev);
-	}
+	dev_warn(dev->dev, "Removing after probe failure\n");
+	kref_get(&dev->ctrl.kref);
+	if (!schedule_work(&dev->remove_work))
+		nvme_put_ctrl(&dev->ctrl);
 }
 
-static void nvme_reset_work(struct work_struct *ws)
+static int nvme_reset(struct nvme_dev *dev)
 {
-	struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
-	bool in_probe = work_busy(&dev->probe_work);
-
-	nvme_dev_shutdown(dev);
+	if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
+		return -ENODEV;
 
-	/* Synchronize with device probe so that work will see failure status
-	 * and exit gracefully without trying to schedule another reset */
-	flush_work(&dev->probe_work);
+	if (!queue_work(nvme_workq, &dev->reset_work))
+		return -EBUSY;
 
-	/* Fail this device if reset occured during probe to avoid
-	 * infinite initialization loops. */
-	if (in_probe) {
-		nvme_dead_ctrl(dev);
-		return;
-	}
-	/* Schedule device resume asynchronously so the reset work is available
-	 * to cleanup errors that may occur during reinitialization */
-	schedule_work(&dev->probe_work);
+	flush_work(&dev->reset_work);
+	return 0;
 }
 
-static int __nvme_reset(struct nvme_dev *dev)
+static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 {
-	if (work_pending(&dev->reset_work))
-		return -EBUSY;
-	list_del_init(&dev->node);
-	queue_work(nvme_workq, &dev->reset_work);
+	*val = readl(to_nvme_dev(ctrl)->bar + off);
 	return 0;
 }
 
-static int nvme_reset(struct nvme_dev *dev)
+static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
 {
-	int ret;
-
-	if (!dev->admin_q || blk_queue_dying(dev->admin_q))
-		return -ENODEV;
-
-	spin_lock(&dev_list_lock);
-	ret = __nvme_reset(dev);
-	spin_unlock(&dev_list_lock);
-
-	if (!ret) {
-		flush_work(&dev->reset_work);
-		flush_work(&dev->probe_work);
-		return 0;
-	}
+	writel(val, to_nvme_dev(ctrl)->bar + off);
+	return 0;
+}
 
-	return ret;
+static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
+{
+	*val = readq(to_nvme_dev(ctrl)->bar + off);
+	return 0;
 }
 
-static ssize_t nvme_sysfs_reset(struct device *dev,
-				struct device_attribute *attr, const char *buf,
-				size_t count)
+static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
 {
-	struct nvme_dev *ndev = dev_get_drvdata(dev);
-	int ret;
+	struct nvme_dev *dev = to_nvme_dev(ctrl);
 
-	ret = nvme_reset(ndev);
-	if (ret < 0)
-		return ret;
+	return !dev->bar || dev->online_queues < 2;
+}
 
-	return count;
+static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
+{
+	return nvme_reset(to_nvme_dev(ctrl));
 }
-static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
+
+static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
+	.reg_read32		= nvme_pci_reg_read32,
+	.reg_write32		= nvme_pci_reg_write32,
+	.reg_read64		= nvme_pci_reg_read64,
+	.io_incapable		= nvme_pci_io_incapable,
+	.reset_ctrl		= nvme_pci_reset_ctrl,
+	.free_ctrl		= nvme_pci_free_ctrl,
+};
 
 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
@@ -3313,46 +2058,30 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (!dev->queues)
 		goto free;
 
-	INIT_LIST_HEAD(&dev->namespaces);
-	INIT_WORK(&dev->reset_work, nvme_reset_work);
 	dev->dev = get_device(&pdev->dev);
 	pci_set_drvdata(pdev, dev);
-	result = nvme_set_instance(dev);
-	if (result)
-		goto put_pci;
+
+	INIT_LIST_HEAD(&dev->node);
+	INIT_WORK(&dev->scan_work, nvme_dev_scan);
+	INIT_WORK(&dev->reset_work, nvme_reset_work);
+	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
+	mutex_init(&dev->shutdown_lock);
+	init_completion(&dev->ioq_wait);
 
 	result = nvme_setup_prp_pools(dev);
 	if (result)
-		goto release;
-
-	kref_init(&dev->kref);
-	dev->device = device_create(nvme_class, &pdev->dev,
-				MKDEV(nvme_char_major, dev->instance),
-				dev, "nvme%d", dev->instance);
-	if (IS_ERR(dev->device)) {
-		result = PTR_ERR(dev->device);
-		goto release_pools;
-	}
-	get_device(dev->device);
-	dev_set_drvdata(dev->device, dev);
+		goto put_pci;
 
-	result = device_create_file(dev->device, &dev_attr_reset_controller);
+	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
+			id->driver_data);
 	if (result)
-		goto put_dev;
+		goto release_pools;
 
-	INIT_LIST_HEAD(&dev->node);
-	INIT_WORK(&dev->scan_work, nvme_dev_scan);
-	INIT_WORK(&dev->probe_work, nvme_probe_work);
-	schedule_work(&dev->probe_work);
+	queue_work(nvme_workq, &dev->reset_work);
 	return 0;
 
- put_dev:
-	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
-	put_device(dev->device);
  release_pools:
 	nvme_release_prp_pools(dev);
- release:
-	nvme_release_instance(dev);
  put_pci:
 	put_device(dev->dev);
  free:
@@ -3367,15 +2096,15 @@ static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
 	struct nvme_dev *dev = pci_get_drvdata(pdev);
 
 	if (prepare)
-		nvme_dev_shutdown(dev);
+		nvme_dev_disable(dev, false);
 	else
-		schedule_work(&dev->probe_work);
+		queue_work(nvme_workq, &dev->reset_work);
 }
 
 static void nvme_shutdown(struct pci_dev *pdev)
 {
 	struct nvme_dev *dev = pci_get_drvdata(pdev);
-	nvme_dev_shutdown(dev);
+	nvme_dev_disable(dev, true);
 }
 
 static void nvme_remove(struct pci_dev *pdev)
@@ -3387,34 +2116,25 @@ static void nvme_remove(struct pci_dev *pdev)
 	spin_unlock(&dev_list_lock);
 
 	pci_set_drvdata(pdev, NULL);
-	flush_work(&dev->probe_work);
 	flush_work(&dev->reset_work);
 	flush_work(&dev->scan_work);
-	device_remove_file(dev->device, &dev_attr_reset_controller);
-	nvme_dev_remove(dev);
-	nvme_dev_shutdown(dev);
+	nvme_remove_namespaces(&dev->ctrl);
+	nvme_uninit_ctrl(&dev->ctrl);
+	nvme_dev_disable(dev, true);
 	nvme_dev_remove_admin(dev);
-	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
 	nvme_free_queues(dev, 0);
 	nvme_release_cmb(dev);
 	nvme_release_prp_pools(dev);
-	kref_put(&dev->kref, nvme_free_dev);
+	nvme_put_ctrl(&dev->ctrl);
 }
 
-/* These functions are yet to be implemented */
-#define nvme_error_detected NULL
-#define nvme_dump_registers NULL
-#define nvme_link_reset NULL
-#define nvme_slot_reset NULL
-#define nvme_error_resume NULL
-
 #ifdef CONFIG_PM_SLEEP
 static int nvme_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct nvme_dev *ndev = pci_get_drvdata(pdev);
 
-	nvme_dev_shutdown(ndev);
+	nvme_dev_disable(ndev, true);
 	return 0;
 }
 
@@ -3423,17 +2143,53 @@ static int nvme_resume(struct device *dev)
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct nvme_dev *ndev = pci_get_drvdata(pdev);
 
-	schedule_work(&ndev->probe_work);
+	queue_work(nvme_workq, &ndev->reset_work);
 	return 0;
 }
 #endif
 
 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
 
+static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
+						pci_channel_state_t state)
+{
+	struct nvme_dev *dev = pci_get_drvdata(pdev);
+
+	/*
+	 * A frozen channel requires a reset. When detected, this method will
+	 * shutdown the controller to quiesce. The controller will be restarted
+	 * after the slot reset through driver's slot_reset callback.
+	 */
+	dev_warn(&pdev->dev, "error detected: state:%d\n", state);
+	switch (state) {
+	case pci_channel_io_normal:
+		return PCI_ERS_RESULT_CAN_RECOVER;
+	case pci_channel_io_frozen:
+		nvme_dev_disable(dev, false);
+		return PCI_ERS_RESULT_NEED_RESET;
+	case pci_channel_io_perm_failure:
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
+{
+	struct nvme_dev *dev = pci_get_drvdata(pdev);
+
+	dev_info(&pdev->dev, "restart after slot reset\n");
+	pci_restore_state(pdev);
+	queue_work(nvme_workq, &dev->reset_work);
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void nvme_error_resume(struct pci_dev *pdev)
+{
+	pci_cleanup_aer_uncorrect_error_status(pdev);
+}
+
 static const struct pci_error_handlers nvme_err_handler = {
 	.error_detected	= nvme_error_detected,
-	.mmio_enabled	= nvme_dump_registers,
-	.link_reset	= nvme_link_reset,
 	.slot_reset	= nvme_slot_reset,
 	.resume		= nvme_error_resume,
 	.reset_notify	= nvme_reset_notify,
@@ -3443,6 +2199,10 @@ static const struct pci_error_handlers nvme_err_handler = {
 #define PCI_CLASS_STORAGE_EXPRESS	0x010802
 
 static const struct pci_device_id nvme_id_table[] = {
+	{ PCI_VDEVICE(INTEL, 0x0953),
+		.driver_data = NVME_QUIRK_STRIPE_SIZE, },
+	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
+		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
 	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
 	{ 0, }
@@ -3467,40 +2227,21 @@ static int __init nvme_init(void)
 
 	init_waitqueue_head(&nvme_kthread_wait);
 
-	nvme_workq = create_singlethread_workqueue("nvme");
+	nvme_workq = alloc_workqueue("nvme", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
 	if (!nvme_workq)
 		return -ENOMEM;
 
-	result = register_blkdev(nvme_major, "nvme");
+	result = nvme_core_init();
 	if (result < 0)
 		goto kill_workq;
-	else if (result > 0)
-		nvme_major = result;
-
-	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
-							&nvme_dev_fops);
-	if (result < 0)
-		goto unregister_blkdev;
-	else if (result > 0)
-		nvme_char_major = result;
-
-	nvme_class = class_create(THIS_MODULE, "nvme");
-	if (IS_ERR(nvme_class)) {
-		result = PTR_ERR(nvme_class);
-		goto unregister_chrdev;
-	}
 
 	result = pci_register_driver(&nvme_driver);
 	if (result)
-		goto destroy_class;
+		goto core_exit;
 	return 0;
 
- destroy_class:
-	class_destroy(nvme_class);
- unregister_chrdev:
-	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
- unregister_blkdev:
-	unregister_blkdev(nvme_major, "nvme");
+ core_exit:
+	nvme_core_exit();
  kill_workq:
 	destroy_workqueue(nvme_workq);
 	return result;
@@ -3509,10 +2250,8 @@ static int __init nvme_init(void)
 static void __exit nvme_exit(void)
 {
 	pci_unregister_driver(&nvme_driver);
-	unregister_blkdev(nvme_major, "nvme");
+	nvme_core_exit();
 	destroy_workqueue(nvme_workq);
-	class_destroy(nvme_class);
-	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
 	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
 	_nvme_check_size();
 }
diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c
index c3d8d3887a31..e947e298a737 100644
--- a/drivers/nvme/host/scsi.c
+++ b/drivers/nvme/host/scsi.c
@@ -524,7 +524,7 @@ static int nvme_trans_standard_inquiry_page(struct nvme_ns *ns,
 					struct sg_io_hdr *hdr, u8 *inq_response,
 					int alloc_len)
 {
-	struct nvme_dev *dev = ns->dev;
+	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_id_ns *id_ns;
 	int res;
 	int nvme_sc;
@@ -532,10 +532,10 @@ static int nvme_trans_standard_inquiry_page(struct nvme_ns *ns,
 	u8 resp_data_format = 0x02;
 	u8 protect;
 	u8 cmdque = 0x01 << 1;
-	u8 fw_offset = sizeof(dev->firmware_rev);
+	u8 fw_offset = sizeof(ctrl->firmware_rev);
 
 	/* nvme ns identify - use DPS value for PROTECT field */
-	nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+	nvme_sc = nvme_identify_ns(ctrl, ns->ns_id, &id_ns);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		return res;
@@ -553,12 +553,12 @@ static int nvme_trans_standard_inquiry_page(struct nvme_ns *ns,
 	inq_response[5] = protect;	/* sccs=0 | acc=0 | tpgs=0 | pc3=0 */
 	inq_response[7] = cmdque;	/* wbus16=0 | sync=0 | vs=0 */
 	strncpy(&inq_response[8], "NVMe    ", 8);
-	strncpy(&inq_response[16], dev->model, 16);
+	strncpy(&inq_response[16], ctrl->model, 16);
 
-	while (dev->firmware_rev[fw_offset - 1] == ' ' && fw_offset > 4)
+	while (ctrl->firmware_rev[fw_offset - 1] == ' ' && fw_offset > 4)
 		fw_offset--;
 	fw_offset -= 4;
-	strncpy(&inq_response[32], dev->firmware_rev + fw_offset, 4);
+	strncpy(&inq_response[32], ctrl->firmware_rev + fw_offset, 4);
 
 	xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH);
 	return nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
@@ -588,82 +588,113 @@ static int nvme_trans_unit_serial_page(struct nvme_ns *ns,
 					struct sg_io_hdr *hdr, u8 *inq_response,
 					int alloc_len)
 {
-	struct nvme_dev *dev = ns->dev;
 	int xfer_len;
 
 	memset(inq_response, 0, STANDARD_INQUIRY_LENGTH);
 	inq_response[1] = INQ_UNIT_SERIAL_NUMBER_PAGE; /* Page Code */
 	inq_response[3] = INQ_SERIAL_NUMBER_LENGTH;    /* Page Length */
-	strncpy(&inq_response[4], dev->serial, INQ_SERIAL_NUMBER_LENGTH);
+	strncpy(&inq_response[4], ns->ctrl->serial, INQ_SERIAL_NUMBER_LENGTH);
 
 	xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH);
 	return nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 }
 
-static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
-					u8 *inq_response, int alloc_len)
+static int nvme_fill_device_id_eui64(struct nvme_ns *ns, struct sg_io_hdr *hdr,
+		u8 *inq_response, int alloc_len)
 {
-	struct nvme_dev *dev = ns->dev;
-	int res;
-	int nvme_sc;
-	int xfer_len;
-	__be32 tmp_id = cpu_to_be32(ns->ns_id);
+	struct nvme_id_ns *id_ns;
+	int nvme_sc, res;
+	size_t len;
+	void *eui;
 
-	memset(inq_response, 0, alloc_len);
-	inq_response[1] = INQ_DEVICE_IDENTIFICATION_PAGE;    /* Page Code */
-	if (readl(&dev->bar->vs) >= NVME_VS(1, 1)) {
-		struct nvme_id_ns *id_ns;
-		void *eui;
-		int len;
+	nvme_sc = nvme_identify_ns(ns->ctrl, ns->ns_id, &id_ns);
+	res = nvme_trans_status_code(hdr, nvme_sc);
+	if (res)
+		return res;
 
-		nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
-		res = nvme_trans_status_code(hdr, nvme_sc);
-		if (res)
-			return res;
+	eui = id_ns->eui64;
+	len = sizeof(id_ns->eui64);
 
-		eui = id_ns->eui64;
-		len = sizeof(id_ns->eui64);
-		if (readl(&dev->bar->vs) >= NVME_VS(1, 2)) {
-			if (bitmap_empty(eui, len * 8)) {
-				eui = id_ns->nguid;
-				len = sizeof(id_ns->nguid);
-			}
-		}
+	if (ns->ctrl->vs >= NVME_VS(1, 2)) {
 		if (bitmap_empty(eui, len * 8)) {
-			kfree(id_ns);
-			goto scsi_string;
+			eui = id_ns->nguid;
+			len = sizeof(id_ns->nguid);
 		}
+	}
 
-		inq_response[3] = 4 + len; /* Page Length */
-		/* Designation Descriptor start */
-		inq_response[4] = 0x01;    /* Proto ID=0h | Code set=1h */
-		inq_response[5] = 0x02;    /* PIV=0b | Asso=00b | Designator Type=2h */
-		inq_response[6] = 0x00;    /* Rsvd */
-		inq_response[7] = len;     /* Designator Length */
-		memcpy(&inq_response[8], eui, len);
-		kfree(id_ns);
-	} else {
- scsi_string:
-		if (alloc_len < 72) {
-			return nvme_trans_completion(hdr,
-					SAM_STAT_CHECK_CONDITION,
-					ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
-					SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
-		}
-		inq_response[3] = 0x48;    /* Page Length */
-		/* Designation Descriptor start */
-		inq_response[4] = 0x03;    /* Proto ID=0h | Code set=3h */
-		inq_response[5] = 0x08;    /* PIV=0b | Asso=00b | Designator Type=8h */
-		inq_response[6] = 0x00;    /* Rsvd */
-		inq_response[7] = 0x44;    /* Designator Length */
-
-		sprintf(&inq_response[8], "%04x", to_pci_dev(dev->dev)->vendor);
-		memcpy(&inq_response[12], dev->model, sizeof(dev->model));
-		sprintf(&inq_response[52], "%04x", tmp_id);
-		memcpy(&inq_response[56], dev->serial, sizeof(dev->serial));
+	if (bitmap_empty(eui, len * 8)) {
+		res = -EOPNOTSUPP;
+		goto out_free_id;
 	}
-	xfer_len = alloc_len;
-	return nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
+
+	memset(inq_response, 0, alloc_len);
+	inq_response[1] = INQ_DEVICE_IDENTIFICATION_PAGE;
+	inq_response[3] = 4 + len; /* Page Length */
+
+	/* Designation Descriptor start */
+	inq_response[4] = 0x01;	/* Proto ID=0h | Code set=1h */
+	inq_response[5] = 0x02;	/* PIV=0b | Asso=00b | Designator Type=2h */
+	inq_response[6] = 0x00;	/* Rsvd */
+	inq_response[7] = len;	/* Designator Length */
+	memcpy(&inq_response[8], eui, len);
+
+	res = nvme_trans_copy_to_user(hdr, inq_response, alloc_len);
+out_free_id:
+	kfree(id_ns);
+	return res;
+}
+
+static int nvme_fill_device_id_scsi_string(struct nvme_ns *ns,
+		struct sg_io_hdr *hdr, u8 *inq_response, int alloc_len)
+{
+	struct nvme_ctrl *ctrl = ns->ctrl;
+	struct nvme_id_ctrl *id_ctrl;
+	int nvme_sc, res;
+
+	if (alloc_len < 72) {
+		return nvme_trans_completion(hdr,
+				SAM_STAT_CHECK_CONDITION,
+				ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
+				SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
+	}
+
+	nvme_sc = nvme_identify_ctrl(ctrl, &id_ctrl);
+	res = nvme_trans_status_code(hdr, nvme_sc);
+	if (res)
+		return res;
+
+	memset(inq_response, 0, alloc_len);
+	inq_response[1] = INQ_DEVICE_IDENTIFICATION_PAGE;
+	inq_response[3] = 0x48;	/* Page Length */
+
+	/* Designation Descriptor start */
+	inq_response[4] = 0x03;	/* Proto ID=0h | Code set=3h */
+	inq_response[5] = 0x08;	/* PIV=0b | Asso=00b | Designator Type=8h */
+	inq_response[6] = 0x00;	/* Rsvd */
+	inq_response[7] = 0x44;	/* Designator Length */
+
+	sprintf(&inq_response[8], "%04x", le16_to_cpu(id_ctrl->vid));
+	memcpy(&inq_response[12], ctrl->model, sizeof(ctrl->model));
+	sprintf(&inq_response[52], "%04x", cpu_to_be32(ns->ns_id));
+	memcpy(&inq_response[56], ctrl->serial, sizeof(ctrl->serial));
+
+	res = nvme_trans_copy_to_user(hdr, inq_response, alloc_len);
+	kfree(id_ctrl);
+	return res;
+}
+
+static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
+					u8 *resp, int alloc_len)
+{
+	int res;
+
+	if (ns->ctrl->vs >= NVME_VS(1, 1)) {
+		res = nvme_fill_device_id_eui64(ns, hdr, resp, alloc_len);
+		if (res != -EOPNOTSUPP)
+			return res;
+	}
+
+	return nvme_fill_device_id_scsi_string(ns, hdr, resp, alloc_len);
 }
 
 static int nvme_trans_ext_inq_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
@@ -672,7 +703,7 @@ static int nvme_trans_ext_inq_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	u8 *inq_response;
 	int res;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
+	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_id_ctrl *id_ctrl;
 	struct nvme_id_ns *id_ns;
 	int xfer_len;
@@ -688,7 +719,7 @@ static int nvme_trans_ext_inq_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	if (inq_response == NULL)
 		return -ENOMEM;
 
-	nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+	nvme_sc = nvme_identify_ns(ctrl, ns->ns_id, &id_ns);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		goto out_free_inq;
@@ -704,7 +735,7 @@ static int nvme_trans_ext_inq_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	app_chk = protect << 1;
 	ref_chk = protect;
 
-	nvme_sc = nvme_identify_ctrl(dev, &id_ctrl);
+	nvme_sc = nvme_identify_ctrl(ctrl, &id_ctrl);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		goto out_free_inq;
@@ -815,7 +846,6 @@ static int nvme_trans_log_info_exceptions(struct nvme_ns *ns,
 	int res;
 	int xfer_len;
 	u8 *log_response;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_smart_log *smart_log;
 	u8 temp_c;
 	u16 temp_k;
@@ -824,7 +854,7 @@ static int nvme_trans_log_info_exceptions(struct nvme_ns *ns,
 	if (log_response == NULL)
 		return -ENOMEM;
 
-	res = nvme_get_log_page(dev, &smart_log);
+	res = nvme_get_log_page(ns->ctrl, &smart_log);
 	if (res < 0)
 		goto out_free_response;
 
@@ -862,7 +892,6 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	int res;
 	int xfer_len;
 	u8 *log_response;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_smart_log *smart_log;
 	u32 feature_resp;
 	u8 temp_c_cur, temp_c_thresh;
@@ -872,7 +901,7 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	if (log_response == NULL)
 		return -ENOMEM;
 
-	res = nvme_get_log_page(dev, &smart_log);
+	res = nvme_get_log_page(ns->ctrl, &smart_log);
 	if (res < 0)
 		goto out_free_response;
 
@@ -886,7 +915,7 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	kfree(smart_log);
 
 	/* Get Features for Temp Threshold */
-	res = nvme_get_features(dev, NVME_FEAT_TEMP_THRESH, 0, 0,
+	res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, 0,
 								&feature_resp);
 	if (res != NVME_SC_SUCCESS)
 		temp_c_thresh = LOG_TEMP_UNKNOWN;
@@ -948,7 +977,6 @@ static int nvme_trans_fill_blk_desc(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 {
 	int res;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_id_ns *id_ns;
 	u8 flbas;
 	u32 lba_length;
@@ -958,7 +986,7 @@ static int nvme_trans_fill_blk_desc(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	else if (llbaa > 0 && len < MODE_PAGE_LLBAA_BLK_DES_LEN)
 		return -EINVAL;
 
-	nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+	nvme_sc = nvme_identify_ns(ns->ctrl, ns->ns_id, &id_ns);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		return res;
@@ -1014,14 +1042,13 @@ static int nvme_trans_fill_caching_page(struct nvme_ns *ns,
 {
 	int res = 0;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	u32 feature_resp;
 	u8 vwc;
 
 	if (len < MODE_PAGE_CACHING_LEN)
 		return -EINVAL;
 
-	nvme_sc = nvme_get_features(dev, NVME_FEAT_VOLATILE_WC, 0, 0,
+	nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, 0,
 								&feature_resp);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
@@ -1207,12 +1234,11 @@ static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 {
 	int res;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_id_ctrl *id_ctrl;
 	int lowest_pow_st;	/* max npss = lowest power consumption */
 	unsigned ps_desired = 0;
 
-	nvme_sc = nvme_identify_ctrl(dev, &id_ctrl);
+	nvme_sc = nvme_identify_ctrl(ns->ctrl, &id_ctrl);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		return res;
@@ -1256,7 +1282,7 @@ static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 				SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
 		break;
 	}
-	nvme_sc = nvme_set_features(dev, NVME_FEAT_POWER_MGMT, ps_desired, 0,
+	nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_POWER_MGMT, ps_desired, 0,
 				    NULL);
 	return nvme_trans_status_code(hdr, nvme_sc);
 }
@@ -1280,7 +1306,6 @@ static int nvme_trans_send_download_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr
 					u8 buffer_id)
 {
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_command c;
 
 	if (hdr->iovec_count > 0) {
@@ -1297,7 +1322,7 @@ static int nvme_trans_send_download_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr
 	c.dlfw.numd = cpu_to_le32((tot_len/BYTES_TO_DWORDS) - 1);
 	c.dlfw.offset = cpu_to_le32(offset/BYTES_TO_DWORDS);
 
-	nvme_sc = __nvme_submit_sync_cmd(dev->admin_q, &c, NULL,
+	nvme_sc = nvme_submit_user_cmd(ns->ctrl->admin_q, &c,
 			hdr->dxferp, tot_len, NULL, 0);
 	return nvme_trans_status_code(hdr, nvme_sc);
 }
@@ -1364,14 +1389,13 @@ static int nvme_trans_modesel_get_mp(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 {
 	int res = 0;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	unsigned dword11;
 
 	switch (page_code) {
 	case MODE_PAGE_CACHING:
 		dword11 = ((mode_page[2] & CACHING_MODE_PAGE_WCE_MASK) ? 1 : 0);
-		nvme_sc = nvme_set_features(dev, NVME_FEAT_VOLATILE_WC, dword11,
-					    0, NULL);
+		nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_VOLATILE_WC,
+					    dword11, 0, NULL);
 		res = nvme_trans_status_code(hdr, nvme_sc);
 		break;
 	case MODE_PAGE_CONTROL:
@@ -1473,7 +1497,6 @@ static int nvme_trans_fmt_set_blk_size_count(struct nvme_ns *ns,
 {
 	int res = 0;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	u8 flbas;
 
 	/*
@@ -1486,7 +1509,7 @@ static int nvme_trans_fmt_set_blk_size_count(struct nvme_ns *ns,
 	if (ns->mode_select_num_blocks == 0 || ns->mode_select_block_len == 0) {
 		struct nvme_id_ns *id_ns;
 
-		nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+		nvme_sc = nvme_identify_ns(ns->ctrl, ns->ns_id, &id_ns);
 		res = nvme_trans_status_code(hdr, nvme_sc);
 		if (res)
 			return res;
@@ -1570,7 +1593,6 @@ static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 {
 	int res;
 	int nvme_sc;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_id_ns *id_ns;
 	u8 i;
 	u8 flbas, nlbaf;
@@ -1579,7 +1601,7 @@ static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	struct nvme_command c;
 
 	/* Loop thru LBAF's in id_ns to match reqd lbaf, put in cdw10 */
-	nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+	nvme_sc = nvme_identify_ns(ns->ctrl, ns->ns_id, &id_ns);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		return res;
@@ -1611,7 +1633,7 @@ static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	c.format.nsid = cpu_to_le32(ns->ns_id);
 	c.format.cdw10 = cpu_to_le32(cdw10);
 
-	nvme_sc = nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
+	nvme_sc = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, NULL, 0);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 
 	kfree(id_ns);
@@ -1704,7 +1726,7 @@ static int nvme_trans_do_nvme_io(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 			nvme_sc = NVME_SC_LBA_RANGE;
 			break;
 		}
-		nvme_sc = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
+		nvme_sc = nvme_submit_user_cmd(ns->queue, &c,
 				next_mapping_addr, unit_len, NULL, 0);
 		if (nvme_sc)
 			break;
@@ -2040,7 +2062,6 @@ static int nvme_trans_read_capacity(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	u32 alloc_len;
 	u32 resp_size;
 	u32 xfer_len;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_id_ns *id_ns;
 	u8 *response;
 
@@ -2052,7 +2073,7 @@ static int nvme_trans_read_capacity(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 		resp_size = READ_CAP_10_RESP_SIZE;
 	}
 
-	nvme_sc = nvme_identify_ns(dev, ns->ns_id, &id_ns);
+	nvme_sc = nvme_identify_ns(ns->ctrl, ns->ns_id, &id_ns);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
 		return res;	
@@ -2080,7 +2101,6 @@ static int nvme_trans_report_luns(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	int nvme_sc;
 	u32 alloc_len, xfer_len, resp_size;
 	u8 *response;
-	struct nvme_dev *dev = ns->dev;
 	struct nvme_id_ctrl *id_ctrl;
 	u32 ll_length, lun_id;
 	u8 lun_id_offset = REPORT_LUNS_FIRST_LUN_OFFSET;
@@ -2094,7 +2114,7 @@ static int nvme_trans_report_luns(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	case ALL_LUNS_RETURNED:
 	case ALL_WELL_KNOWN_LUNS_RETURNED:
 	case RESTRICTED_LUNS_RETURNED:
-		nvme_sc = nvme_identify_ctrl(dev, &id_ctrl);
+		nvme_sc = nvme_identify_ctrl(ns->ctrl, &id_ctrl);
 		res = nvme_trans_status_code(hdr, nvme_sc);
 		if (res)
 			return res;
@@ -2295,9 +2315,7 @@ static int nvme_trans_test_unit_ready(struct nvme_ns *ns,
 					struct sg_io_hdr *hdr,
 					u8 *cmd)
 {
-	struct nvme_dev *dev = ns->dev;
-
-	if (!(readl(&dev->bar->csts) & NVME_CSTS_RDY))
+	if (nvme_ctrl_ready(ns->ctrl))
 		return nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
 					    NOT_READY, SCSI_ASC_LUN_NOT_READY,
 					    SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index 706e3ff67f8b..7ee21ae305ae 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -679,18 +679,6 @@ u32 of_msi_map_rid(struct device *dev, struct device_node *msi_np, u32 rid_in)
 	return __of_msi_map_rid(dev, &msi_np, rid_in);
 }
 
-static struct irq_domain *__of_get_msi_domain(struct device_node *np,
-					      enum irq_domain_bus_token token)
-{
-	struct irq_domain *d;
-
-	d = irq_find_matching_host(np, token);
-	if (!d)
-		d = irq_find_host(np);
-
-	return d;
-}
-
 /**
  * of_msi_map_get_device_domain - Use msi-map to find the relevant MSI domain
  * @dev: device for which the mapping is to be done.
@@ -706,7 +694,7 @@ struct irq_domain *of_msi_map_get_device_domain(struct device *dev, u32 rid)
 	struct device_node *np = NULL;
 
 	__of_msi_map_rid(dev, &np, rid);
-	return __of_get_msi_domain(np, DOMAIN_BUS_PCI_MSI);
+	return irq_find_matching_host(np, DOMAIN_BUS_PCI_MSI);
 }
 
 /**
@@ -730,7 +718,7 @@ struct irq_domain *of_msi_get_domain(struct device *dev,
 	/* Check for a single msi-parent property */
 	msi_np = of_parse_phandle(np, "msi-parent", 0);
 	if (msi_np && !of_property_read_bool(msi_np, "#msi-cells")) {
-		d = __of_get_msi_domain(msi_np, token);
+		d = irq_find_matching_host(msi_np, token);
 		if (!d)
 			of_node_put(msi_np);
 		return d;
@@ -744,7 +732,7 @@ struct irq_domain *of_msi_get_domain(struct device *dev,
 		while (!of_parse_phandle_with_args(np, "msi-parent",
 						   "#msi-cells",
 						   index, &args)) {
-			d = __of_get_msi_domain(args.np, token);
+			d = irq_find_matching_host(args.np, token);
 			if (d)
 				return d;
 
diff --git a/drivers/oprofile/oprofilefs.c b/drivers/oprofile/oprofilefs.c
index dd92c5edf219..b48ac6300c79 100644
--- a/drivers/oprofile/oprofilefs.c
+++ b/drivers/oprofile/oprofilefs.c
@@ -138,22 +138,22 @@ static int __oprofilefs_create_file(struct dentry *root, char const *name,
 	struct dentry *dentry;
 	struct inode *inode;
 
-	mutex_lock(&d_inode(root)->i_mutex);
+	inode_lock(d_inode(root));
 	dentry = d_alloc_name(root, name);
 	if (!dentry) {
-		mutex_unlock(&d_inode(root)->i_mutex);
+		inode_unlock(d_inode(root));
 		return -ENOMEM;
 	}
 	inode = oprofilefs_get_inode(root->d_sb, S_IFREG | perm);
 	if (!inode) {
 		dput(dentry);
-		mutex_unlock(&d_inode(root)->i_mutex);
+		inode_unlock(d_inode(root));
 		return -ENOMEM;
 	}
 	inode->i_fop = fops;
 	inode->i_private = priv;
 	d_add(dentry, inode);
-	mutex_unlock(&d_inode(root)->i_mutex);
+	inode_unlock(d_inode(root));
 	return 0;
 }
 
@@ -215,22 +215,22 @@ struct dentry *oprofilefs_mkdir(struct dentry *parent, char const *name)
 	struct dentry *dentry;
 	struct inode *inode;
 
-	mutex_lock(&d_inode(parent)->i_mutex);
+	inode_lock(d_inode(parent));
 	dentry = d_alloc_name(parent, name);
 	if (!dentry) {
-		mutex_unlock(&d_inode(parent)->i_mutex);
+		inode_unlock(d_inode(parent));
 		return NULL;
 	}
 	inode = oprofilefs_get_inode(parent->d_sb, S_IFDIR | 0755);
 	if (!inode) {
 		dput(dentry);
-		mutex_unlock(&d_inode(parent)->i_mutex);
+		inode_unlock(d_inode(parent));
 		return NULL;
 	}
 	inode->i_op = &simple_dir_inode_operations;
 	inode->i_fop = &simple_dir_operations;
 	d_add(dentry, inode);
-	mutex_unlock(&d_inode(parent)->i_mutex);
+	inode_unlock(d_inode(parent));
 	return dentry;
 }
 
diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c
index 8e11fb2831cd..e24b05996a1b 100644
--- a/drivers/parisc/ccio-dma.c
+++ b/drivers/parisc/ccio-dma.c
@@ -786,18 +786,27 @@ ccio_map_single(struct device *dev, void *addr, size_t size,
 	return CCIO_IOVA(iovp, offset);
 }
 
+
+static dma_addr_t
+ccio_map_page(struct device *dev, struct page *page, unsigned long offset,
+		size_t size, enum dma_data_direction direction,
+		struct dma_attrs *attrs)
+{
+	return ccio_map_single(dev, page_address(page) + offset, size,
+			direction);
+}
+
+
 /**
- * ccio_unmap_single - Unmap an address range from the IOMMU.
+ * ccio_unmap_page - Unmap an address range from the IOMMU.
  * @dev: The PCI device.
  * @addr: The start address of the DMA region.
  * @size: The length of the DMA region.
  * @direction: The direction of the DMA transaction (to/from device).
- *
- * This function implements the pci_unmap_single function.
  */
 static void 
-ccio_unmap_single(struct device *dev, dma_addr_t iova, size_t size, 
-		  enum dma_data_direction direction)
+ccio_unmap_page(struct device *dev, dma_addr_t iova, size_t size,
+		enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 	unsigned long flags; 
@@ -826,7 +835,7 @@ ccio_unmap_single(struct device *dev, dma_addr_t iova, size_t size,
 }
 
 /**
- * ccio_alloc_consistent - Allocate a consistent DMA mapping.
+ * ccio_alloc - Allocate a consistent DMA mapping.
  * @dev: The PCI device.
  * @size: The length of the DMA region.
  * @dma_handle: The DMA address handed back to the device (not the cpu).
@@ -834,7 +843,8 @@ ccio_unmap_single(struct device *dev, dma_addr_t iova, size_t size,
  * This function implements the pci_alloc_consistent function.
  */
 static void * 
-ccio_alloc_consistent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag)
+ccio_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag,
+		struct dma_attrs *attrs)
 {
       void *ret;
 #if 0
@@ -858,7 +868,7 @@ ccio_alloc_consistent(struct device *dev, size_t size, dma_addr_t *dma_handle, g
 }
 
 /**
- * ccio_free_consistent - Free a consistent DMA mapping.
+ * ccio_free - Free a consistent DMA mapping.
  * @dev: The PCI device.
  * @size: The length of the DMA region.
  * @cpu_addr: The cpu address returned from the ccio_alloc_consistent.
@@ -867,10 +877,10 @@ ccio_alloc_consistent(struct device *dev, size_t size, dma_addr_t *dma_handle, g
  * This function implements the pci_free_consistent function.
  */
 static void 
-ccio_free_consistent(struct device *dev, size_t size, void *cpu_addr, 
-		     dma_addr_t dma_handle)
+ccio_free(struct device *dev, size_t size, void *cpu_addr,
+		dma_addr_t dma_handle, struct dma_attrs *attrs)
 {
-	ccio_unmap_single(dev, dma_handle, size, 0);
+	ccio_unmap_page(dev, dma_handle, size, 0, NULL);
 	free_pages((unsigned long)cpu_addr, get_order(size));
 }
 
@@ -897,7 +907,7 @@ ccio_free_consistent(struct device *dev, size_t size, void *cpu_addr,
  */
 static int
 ccio_map_sg(struct device *dev, struct scatterlist *sglist, int nents, 
-	    enum dma_data_direction direction)
+	    enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 	int coalesced, filled = 0;
@@ -974,7 +984,7 @@ ccio_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
  */
 static void 
 ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, 
-	      enum dma_data_direction direction)
+	      enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 
@@ -993,27 +1003,22 @@ ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
 #ifdef CCIO_COLLECT_STATS
 		ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT;
 #endif
-		ccio_unmap_single(dev, sg_dma_address(sglist),
-				  sg_dma_len(sglist), direction);
+		ccio_unmap_page(dev, sg_dma_address(sglist),
+				  sg_dma_len(sglist), direction, NULL);
 		++sglist;
 	}
 
 	DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
 }
 
-static struct hppa_dma_ops ccio_ops = {
+static struct dma_map_ops ccio_ops = {
 	.dma_supported =	ccio_dma_supported,
-	.alloc_consistent =	ccio_alloc_consistent,
-	.alloc_noncoherent =	ccio_alloc_consistent,
-	.free_consistent =	ccio_free_consistent,
-	.map_single =		ccio_map_single,
-	.unmap_single =		ccio_unmap_single,
+	.alloc =		ccio_alloc,
+	.free =			ccio_free,
+	.map_page =		ccio_map_page,
+	.unmap_page =		ccio_unmap_page,
 	.map_sg = 		ccio_map_sg,
 	.unmap_sg = 		ccio_unmap_sg,
-	.dma_sync_single_for_cpu =	NULL,	/* NOP for U2/Uturn */
-	.dma_sync_single_for_device =	NULL,	/* NOP for U2/Uturn */
-	.dma_sync_sg_for_cpu =		NULL,	/* ditto */
-	.dma_sync_sg_for_device =		NULL,	/* ditto */
 };
 
 #ifdef CONFIG_PROC_FS
@@ -1062,7 +1067,7 @@ static int ccio_proc_info(struct seq_file *m, void *p)
 			   ioc->msingle_calls, ioc->msingle_pages,
 			   (int)((ioc->msingle_pages * 1000)/ioc->msingle_calls));
 
-		/* KLUGE - unmap_sg calls unmap_single for each mapped page */
+		/* KLUGE - unmap_sg calls unmap_page for each mapped page */
 		min = ioc->usingle_calls - ioc->usg_calls;
 		max = ioc->usingle_pages - ioc->usg_pages;
 		seq_printf(m, "pci_unmap_single: %8ld calls  %8ld pages (avg %d/1000)\n",
diff --git a/drivers/parisc/dino.c b/drivers/parisc/dino.c
index a0580afe1713..1133b5cc88ca 100644
--- a/drivers/parisc/dino.c
+++ b/drivers/parisc/dino.c
@@ -599,8 +599,10 @@ dino_fixup_bus(struct pci_bus *bus)
 		** P2PB's only have 2 BARs, no IRQs.
 		** I'd like to just ignore them for now.
 		*/
-		if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI)
+		if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI)  {
+			pcibios_init_bridge(dev);
 			continue;
+		}
 
 		/* null out the ROM resource if there is one (we don't
 		 * care about an expansion rom on parisc, since it
diff --git a/drivers/parisc/lba_pci.c b/drivers/parisc/lba_pci.c
index 42844c2bc065..2ec2aef4d211 100644
--- a/drivers/parisc/lba_pci.c
+++ b/drivers/parisc/lba_pci.c
@@ -790,8 +790,10 @@ lba_fixup_bus(struct pci_bus *bus)
                 /*
 		** P2PB's have no IRQs. ignore them.
 		*/
-		if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI)
+		if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
+			pcibios_init_bridge(dev);
 			continue;
+		}
 
 		/* Adjust INTERRUPT_LINE for this dev */
 		iosapic_fixup_irq(ldev->iosapic_obj, dev);
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index 225049b492e5..42ec4600b7e4 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -780,8 +780,18 @@ sba_map_single(struct device *dev, void *addr, size_t size,
 }
 
 
+static dma_addr_t
+sba_map_page(struct device *dev, struct page *page, unsigned long offset,
+		size_t size, enum dma_data_direction direction,
+		struct dma_attrs *attrs)
+{
+	return sba_map_single(dev, page_address(page) + offset, size,
+			direction);
+}
+
+
 /**
- * sba_unmap_single - unmap one IOVA and free resources
+ * sba_unmap_page - unmap one IOVA and free resources
  * @dev: instance of PCI owned by the driver that's asking.
  * @iova:  IOVA of driver buffer previously mapped.
  * @size:  number of bytes mapped in driver buffer.
@@ -790,8 +800,8 @@ sba_map_single(struct device *dev, void *addr, size_t size,
  * See Documentation/DMA-API-HOWTO.txt
  */
 static void
-sba_unmap_single(struct device *dev, dma_addr_t iova, size_t size,
-		 enum dma_data_direction direction)
+sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size,
+		enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 #if DELAYED_RESOURCE_CNT > 0
@@ -858,15 +868,15 @@ sba_unmap_single(struct device *dev, dma_addr_t iova, size_t size,
 
 
 /**
- * sba_alloc_consistent - allocate/map shared mem for DMA
+ * sba_alloc - allocate/map shared mem for DMA
  * @hwdev: instance of PCI owned by the driver that's asking.
  * @size:  number of bytes mapped in driver buffer.
  * @dma_handle:  IOVA of new buffer.
  *
  * See Documentation/DMA-API-HOWTO.txt
  */
-static void *sba_alloc_consistent(struct device *hwdev, size_t size,
-					dma_addr_t *dma_handle, gfp_t gfp)
+static void *sba_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
+		gfp_t gfp, struct dma_attrs *attrs)
 {
 	void *ret;
 
@@ -888,7 +898,7 @@ static void *sba_alloc_consistent(struct device *hwdev, size_t size,
 
 
 /**
- * sba_free_consistent - free/unmap shared mem for DMA
+ * sba_free - free/unmap shared mem for DMA
  * @hwdev: instance of PCI owned by the driver that's asking.
  * @size:  number of bytes mapped in driver buffer.
  * @vaddr:  virtual address IOVA of "consistent" buffer.
@@ -897,10 +907,10 @@ static void *sba_alloc_consistent(struct device *hwdev, size_t size,
  * See Documentation/DMA-API-HOWTO.txt
  */
 static void
-sba_free_consistent(struct device *hwdev, size_t size, void *vaddr,
-		    dma_addr_t dma_handle)
+sba_free(struct device *hwdev, size_t size, void *vaddr,
+		    dma_addr_t dma_handle, struct dma_attrs *attrs)
 {
-	sba_unmap_single(hwdev, dma_handle, size, 0);
+	sba_unmap_page(hwdev, dma_handle, size, 0, NULL);
 	free_pages((unsigned long) vaddr, get_order(size));
 }
 
@@ -933,7 +943,7 @@ int dump_run_sg = 0;
  */
 static int
 sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
-	   enum dma_data_direction direction)
+	   enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 	int coalesced, filled = 0;
@@ -1016,7 +1026,7 @@ sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
  */
 static void 
 sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
-	     enum dma_data_direction direction)
+	     enum dma_data_direction direction, struct dma_attrs *attrs)
 {
 	struct ioc *ioc;
 #ifdef ASSERT_PDIR_SANITY
@@ -1040,7 +1050,8 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
 
 	while (sg_dma_len(sglist) && nents--) {
 
-		sba_unmap_single(dev, sg_dma_address(sglist), sg_dma_len(sglist), direction);
+		sba_unmap_page(dev, sg_dma_address(sglist), sg_dma_len(sglist),
+				direction, NULL);
 #ifdef SBA_COLLECT_STATS
 		ioc->usg_pages += ((sg_dma_address(sglist) & ~IOVP_MASK) + sg_dma_len(sglist) + IOVP_SIZE - 1) >> PAGE_SHIFT;
 		ioc->usingle_calls--;	/* kluge since call is unmap_sg() */
@@ -1058,19 +1069,14 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
 
 }
 
-static struct hppa_dma_ops sba_ops = {
+static struct dma_map_ops sba_ops = {
 	.dma_supported =	sba_dma_supported,
-	.alloc_consistent =	sba_alloc_consistent,
-	.alloc_noncoherent =	sba_alloc_consistent,
-	.free_consistent =	sba_free_consistent,
-	.map_single =		sba_map_single,
-	.unmap_single =		sba_unmap_single,
+	.alloc =		sba_alloc,
+	.free =			sba_free,
+	.map_page =		sba_map_page,
+	.unmap_page =		sba_unmap_page,
 	.map_sg =		sba_map_sg,
 	.unmap_sg =		sba_unmap_sg,
-	.dma_sync_single_for_cpu =	NULL,
-	.dma_sync_single_for_device =	NULL,
-	.dma_sync_sg_for_cpu =		NULL,
-	.dma_sync_sg_for_device =	NULL,
 };
 
 
diff --git a/drivers/pci/access.c b/drivers/pci/access.c
index 59ac36fe7c42..8c05b5ceeaec 100644
--- a/drivers/pci/access.c
+++ b/drivers/pci/access.c
@@ -25,7 +25,7 @@ DEFINE_RAW_SPINLOCK(pci_lock);
 #define PCI_word_BAD (pos & 1)
 #define PCI_dword_BAD (pos & 3)
 
-#define PCI_OP_READ(size,type,len) \
+#define PCI_OP_READ(size, type, len) \
 int pci_bus_read_config_##size \
 	(struct pci_bus *bus, unsigned int devfn, int pos, type *value)	\
 {									\
@@ -40,7 +40,7 @@ int pci_bus_read_config_##size \
 	return res;							\
 }
 
-#define PCI_OP_WRITE(size,type,len) \
+#define PCI_OP_WRITE(size, type, len) \
 int pci_bus_write_config_##size \
 	(struct pci_bus *bus, unsigned int devfn, int pos, type value)	\
 {									\
@@ -231,7 +231,7 @@ static noinline void pci_wait_cfg(struct pci_dev *dev)
 }
 
 /* Returns 0 on success, negative values indicate error. */
-#define PCI_USER_READ_CONFIG(size,type)					\
+#define PCI_USER_READ_CONFIG(size, type)					\
 int pci_user_read_config_##size						\
 	(struct pci_dev *dev, int pos, type *val)			\
 {									\
@@ -251,7 +251,7 @@ int pci_user_read_config_##size						\
 EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
 
 /* Returns 0 on success, negative values indicate error. */
-#define PCI_USER_WRITE_CONFIG(size,type)				\
+#define PCI_USER_WRITE_CONFIG(size, type)				\
 int pci_user_write_config_##size					\
 	(struct pci_dev *dev, int pos, type val)			\
 {									\
diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c
index d3346d23963b..89b3befc7155 100644
--- a/drivers/pci/bus.c
+++ b/drivers/pci/bus.c
@@ -140,6 +140,8 @@ static int pci_bus_alloc_from_region(struct pci_bus *bus, struct resource *res,
 	type_mask |= IORESOURCE_TYPE_BITS;
 
 	pci_bus_for_each_resource(bus, r, i) {
+		resource_size_t min_used = min;
+
 		if (!r)
 			continue;
 
@@ -163,12 +165,12 @@ static int pci_bus_alloc_from_region(struct pci_bus *bus, struct resource *res,
 		 * overrides "min".
 		 */
 		if (avail.start)
-			min = avail.start;
+			min_used = avail.start;
 
 		max = avail.end;
 
 		/* Ok, try it out.. */
-		ret = allocate_resource(r, res, size, min, max,
+		ret = allocate_resource(r, res, size, min_used, max,
 					align, alignf, alignf_data);
 		if (ret == 0)
 			return 0;
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index c0ad9aaa16a7..75a605426538 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -49,8 +49,7 @@ config PCI_RCAR_GEN2
 
 config PCI_RCAR_GEN2_PCIE
 	bool "Renesas R-Car PCIe controller"
-	depends on ARM
-	depends on ARCH_SHMOBILE || COMPILE_TEST
+	depends on ARCH_SHMOBILE || (ARM && COMPILE_TEST)
 	help
 	  Say Y here if you want PCIe controller support on R-Car Gen2 SoCs.
 
@@ -119,13 +118,11 @@ config PCI_VERSATILE
 	depends on ARCH_VERSATILE
 
 config PCIE_IPROC
-	tristate "Broadcom iProc PCIe controller"
-	depends on OF && (ARM || ARM64)
-	default n
+	tristate
 	help
 	  This enables the iProc PCIe core controller support for Broadcom's
-	  iProc family of SoCs. An appropriate bus interface driver also needs
-	  to be enabled
+	  iProc family of SoCs. An appropriate bus interface driver needs
+	  to be enabled to select this.
 
 config PCIE_IPROC_PLATFORM
 	tristate "Broadcom iProc PCIe platform bus driver"
@@ -148,6 +145,16 @@ config PCIE_IPROC_BCMA
 	  Say Y here if you want to use the Broadcom iProc PCIe controller
 	  through the BCMA bus interface
 
+config PCIE_IPROC_MSI
+	bool "Broadcom iProc PCIe MSI support"
+	depends on PCIE_IPROC_PLATFORM || PCIE_IPROC_BCMA
+	depends on PCI_MSI
+	select PCI_MSI_IRQ_DOMAIN
+	default ARCH_BCM_IPROC
+	help
+	  Say Y here if you want to enable MSI support for Broadcom's iProc
+	  PCIe controller
+
 config PCIE_ALTERA
 	bool "Altera PCIe controller"
 	depends on ARM || NIOS2
@@ -167,10 +174,21 @@ config PCIE_ALTERA_MSI
 
 config PCI_HISI
 	depends on OF && ARM64
-	bool "HiSilicon SoC HIP05 PCIe controller"
+	bool "HiSilicon Hip05 and Hip06 SoCs PCIe controllers"
 	select PCIEPORTBUS
 	select PCIE_DW
 	help
-	  Say Y here if you want PCIe controller support on HiSilicon HIP05 SoC
+	  Say Y here if you want PCIe controller support on HiSilicon
+	  Hip05 and Hip06 SoCs
+
+config PCIE_QCOM
+	bool "Qualcomm PCIe controller"
+	depends on ARCH_QCOM && OF
+	select PCIE_DW
+	select PCIEPORTBUS
+	help
+	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
+	  PCIe controller uses the Designware core plus Qualcomm-specific
+	  hardware wrappers.
 
 endmenu
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 9d4d3c6924a1..7b2f20c6ccc6 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -15,8 +15,10 @@ obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene-msi.o
 obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o
 obj-$(CONFIG_PCI_VERSATILE) += pci-versatile.o
 obj-$(CONFIG_PCIE_IPROC) += pcie-iproc.o
+obj-$(CONFIG_PCIE_IPROC_MSI) += pcie-iproc-msi.o
 obj-$(CONFIG_PCIE_IPROC_PLATFORM) += pcie-iproc-platform.o
 obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o
 obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o
 obj-$(CONFIG_PCIE_ALTERA_MSI) += pcie-altera-msi.o
 obj-$(CONFIG_PCI_HISI) += pcie-hisi.o
+obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o
diff --git a/drivers/pci/host/pci-dra7xx.c b/drivers/pci/host/pci-dra7xx.c
index 8c3688046c02..923607bdabc5 100644
--- a/drivers/pci/host/pci-dra7xx.c
+++ b/drivers/pci/host/pci-dra7xx.c
@@ -302,7 +302,8 @@ static int __init dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
 	}
 
 	ret = devm_request_irq(&pdev->dev, pp->irq,
-			       dra7xx_pcie_msi_irq_handler, IRQF_SHARED,
+			       dra7xx_pcie_msi_irq_handler,
+			       IRQF_SHARED | IRQF_NO_THREAD,
 			       "dra7-pcie-msi",	pp);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to request irq\n");
diff --git a/drivers/pci/host/pci-exynos.c b/drivers/pci/host/pci-exynos.c
index 01095e1160a4..d997d22d4231 100644
--- a/drivers/pci/host/pci-exynos.c
+++ b/drivers/pci/host/pci-exynos.c
@@ -522,7 +522,8 @@ static int __init exynos_add_pcie_port(struct pcie_port *pp,
 
 		ret = devm_request_irq(&pdev->dev, pp->msi_irq,
 					exynos_pcie_msi_irq_handler,
-					IRQF_SHARED, "exynos-pcie", pp);
+					IRQF_SHARED | IRQF_NO_THREAD,
+					"exynos-pcie", pp);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to request msi irq\n");
 			return ret;
diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
index 5434c90db243..1652bc70b145 100644
--- a/drivers/pci/host/pci-host-generic.c
+++ b/drivers/pci/host/pci-host-generic.c
@@ -38,16 +38,7 @@ struct gen_pci_cfg_windows {
 	struct gen_pci_cfg_bus_ops		*ops;
 };
 
-/*
- * ARM pcibios functions expect the ARM struct pci_sys_data as the PCI
- * sysdata.  Add pci_sys_data as the first element in struct gen_pci so
- * that when we use a gen_pci pointer as sysdata, it is also a pointer to
- * a struct pci_sys_data.
- */
 struct gen_pci {
-#ifdef CONFIG_ARM
-	struct pci_sys_data			sys;
-#endif
 	struct pci_host_bridge			host;
 	struct gen_pci_cfg_windows		cfg;
 	struct list_head			resources;
diff --git a/drivers/pci/host/pci-imx6.c b/drivers/pci/host/pci-imx6.c
index 22e8224126fd..fe600964fa50 100644
--- a/drivers/pci/host/pci-imx6.c
+++ b/drivers/pci/host/pci-imx6.c
@@ -32,7 +32,7 @@
 #define to_imx6_pcie(x)	container_of(x, struct imx6_pcie, pp)
 
 struct imx6_pcie {
-	int			reset_gpio;
+	struct gpio_desc	*reset_gpio;
 	struct clk		*pcie_bus;
 	struct clk		*pcie_phy;
 	struct clk		*pcie;
@@ -122,7 +122,7 @@ static int pcie_phy_wait_ack(void __iomem *dbi_base, int addr)
 }
 
 /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
-static int pcie_phy_read(void __iomem *dbi_base, int addr , int *data)
+static int pcie_phy_read(void __iomem *dbi_base, int addr, int *data)
 {
 	u32 val, phy_ctl;
 	int ret;
@@ -287,10 +287,10 @@ static int imx6_pcie_deassert_core_reset(struct pcie_port *pp)
 	usleep_range(200, 500);
 
 	/* Some boards don't have PCIe reset GPIO. */
-	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
-		gpio_set_value(imx6_pcie->reset_gpio, 0);
+	if (imx6_pcie->reset_gpio) {
+		gpiod_set_value_cansleep(imx6_pcie->reset_gpio, 0);
 		msleep(100);
-		gpio_set_value(imx6_pcie->reset_gpio, 1);
+		gpiod_set_value_cansleep(imx6_pcie->reset_gpio, 1);
 	}
 	return 0;
 
@@ -537,7 +537,8 @@ static int __init imx6_add_pcie_port(struct pcie_port *pp,
 
 		ret = devm_request_irq(&pdev->dev, pp->msi_irq,
 				       imx6_pcie_msi_handler,
-				       IRQF_SHARED, "mx6-pcie-msi", pp);
+				       IRQF_SHARED | IRQF_NO_THREAD,
+				       "mx6-pcie-msi", pp);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to request MSI irq\n");
 			return ret;
@@ -560,7 +561,6 @@ static int __init imx6_pcie_probe(struct platform_device *pdev)
 {
 	struct imx6_pcie *imx6_pcie;
 	struct pcie_port *pp;
-	struct device_node *np = pdev->dev.of_node;
 	struct resource *dbi_base;
 	int ret;
 
@@ -581,15 +581,8 @@ static int __init imx6_pcie_probe(struct platform_device *pdev)
 		return PTR_ERR(pp->dbi_base);
 
 	/* Fetch GPIOs */
-	imx6_pcie->reset_gpio = of_get_named_gpio(np, "reset-gpio", 0);
-	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
-		ret = devm_gpio_request_one(&pdev->dev, imx6_pcie->reset_gpio,
-					    GPIOF_OUT_INIT_LOW, "PCIe reset");
-		if (ret) {
-			dev_err(&pdev->dev, "unable to get reset gpio\n");
-			return ret;
-		}
-	}
+	imx6_pcie->reset_gpio = devm_gpiod_get_optional(&pdev->dev, "reset",
+							GPIOD_OUT_LOW);
 
 	/* Fetch clocks */
 	imx6_pcie->pcie_phy = devm_clk_get(&pdev->dev, "pcie_phy");
diff --git a/drivers/pci/host/pci-rcar-gen2.c b/drivers/pci/host/pci-rcar-gen2.c
index c4f64bfee551..9980a4bdae7e 100644
--- a/drivers/pci/host/pci-rcar-gen2.c
+++ b/drivers/pci/host/pci-rcar-gen2.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
@@ -102,6 +103,8 @@ struct rcar_pci_priv {
 	unsigned busnr;
 	int irq;
 	unsigned long window_size;
+	unsigned long window_addr;
+	unsigned long window_pci;
 };
 
 /* PCI configuration space operations */
@@ -239,8 +242,8 @@ static int rcar_pci_setup(int nr, struct pci_sys_data *sys)
 	       RCAR_PCI_ARBITER_PCIBP_MODE;
 	iowrite32(val, reg + RCAR_PCI_ARBITER_CTR_REG);
 
-	/* PCI-AHB mapping: 0x40000000 base */
-	iowrite32(0x40000000 | RCAR_PCIAHB_PREFETCH16,
+	/* PCI-AHB mapping */
+	iowrite32(priv->window_addr | RCAR_PCIAHB_PREFETCH16,
 		  reg + RCAR_PCIAHB_WIN1_CTR_REG);
 
 	/* AHB-PCI mapping: OHCI/EHCI registers */
@@ -251,7 +254,7 @@ static int rcar_pci_setup(int nr, struct pci_sys_data *sys)
 	iowrite32(RCAR_AHBPCI_WIN1_HOST | RCAR_AHBPCI_WIN_CTR_CFG,
 		  reg + RCAR_AHBPCI_WIN1_CTR_REG);
 	/* Set PCI-AHB Window1 address */
-	iowrite32(0x40000000 | PCI_BASE_ADDRESS_MEM_PREFETCH,
+	iowrite32(priv->window_pci | PCI_BASE_ADDRESS_MEM_PREFETCH,
 		  reg + PCI_BASE_ADDRESS_1);
 	/* Set AHB-PCI bridge PCI communication area address */
 	val = priv->cfg_res->start + RCAR_AHBPCI_PCICOM_OFFSET;
@@ -284,6 +287,64 @@ static struct pci_ops rcar_pci_ops = {
 	.write	= pci_generic_config_write,
 };
 
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+				     struct device_node *node)
+{
+	const int na = 3, ns = 2;
+	int rlen;
+
+	parser->node = node;
+	parser->pna = of_n_addr_cells(node);
+	parser->np = parser->pna + na + ns;
+
+	parser->range = of_get_property(node, "dma-ranges", &rlen);
+	if (!parser->range)
+		return -ENOENT;
+
+	parser->end = parser->range + rlen / sizeof(__be32);
+	return 0;
+}
+
+static int rcar_pci_parse_map_dma_ranges(struct rcar_pci_priv *pci,
+					 struct device_node *np)
+{
+	struct of_pci_range range;
+	struct of_pci_range_parser parser;
+	int index = 0;
+
+	/* Failure to parse is ok as we fall back to defaults */
+	if (pci_dma_range_parser_init(&parser, np))
+		return 0;
+
+	/* Get the dma-ranges from DT */
+	for_each_of_pci_range(&parser, &range) {
+		/* Hardware only allows one inbound 32-bit range */
+		if (index)
+			return -EINVAL;
+
+		pci->window_addr = (unsigned long)range.cpu_addr;
+		pci->window_pci = (unsigned long)range.pci_addr;
+		pci->window_size = (unsigned long)range.size;
+
+		/* Catch HW limitations */
+		if (!(range.flags & IORESOURCE_PREFETCH)) {
+			dev_err(pci->dev, "window must be prefetchable\n");
+			return -EINVAL;
+		}
+		if (pci->window_addr) {
+			u32 lowaddr = 1 << (ffs(pci->window_addr) - 1);
+
+			if (lowaddr < pci->window_size) {
+				dev_err(pci->dev, "invalid window size/addr\n");
+				return -EINVAL;
+			}
+		}
+		index++;
+	}
+
+	return 0;
+}
+
 static int rcar_pci_probe(struct platform_device *pdev)
 {
 	struct resource *cfg_res, *mem_res;
@@ -329,6 +390,9 @@ static int rcar_pci_probe(struct platform_device *pdev)
 		return priv->irq;
 	}
 
+	/* default window addr and size if not specified in DT */
+	priv->window_addr = 0x40000000;
+	priv->window_pci = 0x40000000;
 	priv->window_size = SZ_1G;
 
 	if (pdev->dev.of_node) {
@@ -344,6 +408,12 @@ static int rcar_pci_probe(struct platform_device *pdev)
 		priv->busnr = busnr.start;
 		if (busnr.end != busnr.start)
 			dev_warn(&pdev->dev, "only one bus number supported\n");
+
+		ret = rcar_pci_parse_map_dma_ranges(priv, pdev->dev.of_node);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "failed to parse dma-range\n");
+			return ret;
+		}
 	} else {
 		priv->busnr = pdev->id;
 	}
@@ -360,6 +430,7 @@ static int rcar_pci_probe(struct platform_device *pdev)
 }
 
 static struct of_device_id rcar_pci_of_match[] = {
+	{ .compatible = "renesas,pci-rcar-gen2", },
 	{ .compatible = "renesas,pci-r8a7790", },
 	{ .compatible = "renesas,pci-r8a7791", },
 	{ .compatible = "renesas,pci-r8a7794", },
diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 3018ae52e092..30323114c53c 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -1288,7 +1288,7 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 
 	msi->irq = err;
 
-	err = request_irq(msi->irq, tegra_pcie_msi_irq, 0,
+	err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD,
 			  tegra_msi_irq_chip.name, pcie);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
diff --git a/drivers/pci/host/pci-versatile.c b/drivers/pci/host/pci-versatile.c
index 0863d9cc25f8..f843a72dc51c 100644
--- a/drivers/pci/host/pci-versatile.c
+++ b/drivers/pci/host/pci-versatile.c
@@ -125,9 +125,6 @@ out_release_res:
 	return err;
 }
 
-/* Unused, temporary to satisfy ARM arch code */
-struct pci_sys_data sys;
-
 static int versatile_pci_probe(struct platform_device *pdev)
 {
 	struct resource *res;
@@ -208,7 +205,7 @@ static int versatile_pci_probe(struct platform_device *pdev)
 	pci_add_flags(PCI_ENABLE_PROC_DOMAINS);
 	pci_add_flags(PCI_REASSIGN_ALL_BUS | PCI_REASSIGN_ALL_RSRC);
 
-	bus = pci_scan_root_bus(&pdev->dev, 0, &pci_versatile_ops, &sys, &pci_res);
+	bus = pci_scan_root_bus(&pdev->dev, 0, &pci_versatile_ops, NULL, &pci_res);
 	if (!bus)
 		return -ENOMEM;
 
diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c
index 02a7452bdf23..21716827847a 100644
--- a/drivers/pci/host/pcie-designware.c
+++ b/drivers/pci/host/pcie-designware.c
@@ -128,32 +128,26 @@ static inline void dw_pcie_writel_rc(struct pcie_port *pp, u32 val, u32 reg)
 static int dw_pcie_rd_own_conf(struct pcie_port *pp, int where, int size,
 			       u32 *val)
 {
-	int ret;
-
 	if (pp->ops->rd_own_conf)
-		ret = pp->ops->rd_own_conf(pp, where, size, val);
-	else
-		ret = dw_pcie_cfg_read(pp->dbi_base + where, size, val);
+		return pp->ops->rd_own_conf(pp, where, size, val);
 
-	return ret;
+	return dw_pcie_cfg_read(pp->dbi_base + where, size, val);
 }
 
 static int dw_pcie_wr_own_conf(struct pcie_port *pp, int where, int size,
 			       u32 val)
 {
-	int ret;
-
 	if (pp->ops->wr_own_conf)
-		ret = pp->ops->wr_own_conf(pp, where, size, val);
-	else
-		ret = dw_pcie_cfg_write(pp->dbi_base + where, size, val);
+		return pp->ops->wr_own_conf(pp, where, size, val);
 
-	return ret;
+	return dw_pcie_cfg_write(pp->dbi_base + where, size, val);
 }
 
 static void dw_pcie_prog_outbound_atu(struct pcie_port *pp, int index,
 		int type, u64 cpu_addr, u64 pci_addr, u32 size)
 {
+	u32 val;
+
 	dw_pcie_writel_rc(pp, PCIE_ATU_REGION_OUTBOUND | index,
 			  PCIE_ATU_VIEWPORT);
 	dw_pcie_writel_rc(pp, lower_32_bits(cpu_addr), PCIE_ATU_LOWER_BASE);
@@ -164,6 +158,12 @@ static void dw_pcie_prog_outbound_atu(struct pcie_port *pp, int index,
 	dw_pcie_writel_rc(pp, upper_32_bits(pci_addr), PCIE_ATU_UPPER_TARGET);
 	dw_pcie_writel_rc(pp, type, PCIE_ATU_CR1);
 	dw_pcie_writel_rc(pp, PCIE_ATU_ENABLE, PCIE_ATU_CR2);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	dw_pcie_readl_rc(pp, PCIE_ATU_CR2, &val);
 }
 
 static struct irq_chip dw_msi_irq_chip = {
@@ -384,8 +384,8 @@ int dw_pcie_link_up(struct pcie_port *pp)
 {
 	if (pp->ops->link_up)
 		return pp->ops->link_up(pp);
-	else
-		return 0;
+
+	return 0;
 }
 
 static int dw_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
@@ -571,6 +571,9 @@ static int dw_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus,
 	u64 cpu_addr;
 	void __iomem *va_cfg_base;
 
+	if (pp->ops->rd_other_conf)
+		return pp->ops->rd_other_conf(pp, bus, devfn, where, size, val);
+
 	busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
 		 PCIE_ATU_FUNC(PCI_FUNC(devfn));
 
@@ -605,6 +608,9 @@ static int dw_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus,
 	u64 cpu_addr;
 	void __iomem *va_cfg_base;
 
+	if (pp->ops->wr_other_conf)
+		return pp->ops->wr_other_conf(pp, bus, devfn, where, size, val);
+
 	busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
 		 PCIE_ATU_FUNC(PCI_FUNC(devfn));
 
@@ -658,46 +664,30 @@ static int dw_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
 			int size, u32 *val)
 {
 	struct pcie_port *pp = bus->sysdata;
-	int ret;
 
 	if (dw_pcie_valid_config(pp, bus, PCI_SLOT(devfn)) == 0) {
 		*val = 0xffffffff;
 		return PCIBIOS_DEVICE_NOT_FOUND;
 	}
 
-	if (bus->number != pp->root_bus_nr)
-		if (pp->ops->rd_other_conf)
-			ret = pp->ops->rd_other_conf(pp, bus, devfn,
-						where, size, val);
-		else
-			ret = dw_pcie_rd_other_conf(pp, bus, devfn,
-						where, size, val);
-	else
-		ret = dw_pcie_rd_own_conf(pp, where, size, val);
+	if (bus->number == pp->root_bus_nr)
+		return dw_pcie_rd_own_conf(pp, where, size, val);
 
-	return ret;
+	return dw_pcie_rd_other_conf(pp, bus, devfn, where, size, val);
 }
 
 static int dw_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
 			int where, int size, u32 val)
 {
 	struct pcie_port *pp = bus->sysdata;
-	int ret;
 
 	if (dw_pcie_valid_config(pp, bus, PCI_SLOT(devfn)) == 0)
 		return PCIBIOS_DEVICE_NOT_FOUND;
 
-	if (bus->number != pp->root_bus_nr)
-		if (pp->ops->wr_other_conf)
-			ret = pp->ops->wr_other_conf(pp, bus, devfn,
-						where, size, val);
-		else
-			ret = dw_pcie_wr_other_conf(pp, bus, devfn,
-						where, size, val);
-	else
-		ret = dw_pcie_wr_own_conf(pp, where, size, val);
+	if (bus->number == pp->root_bus_nr)
+		return dw_pcie_wr_own_conf(pp, where, size, val);
 
-	return ret;
+	return dw_pcie_wr_other_conf(pp, bus, devfn, where, size, val);
 }
 
 static struct pci_ops dw_pcie_ops = {
diff --git a/drivers/pci/host/pcie-hisi.c b/drivers/pci/host/pcie-hisi.c
index 77f7c669a1b9..3e98d4edae2d 100644
--- a/drivers/pci/host/pcie-hisi.c
+++ b/drivers/pci/host/pcie-hisi.c
@@ -1,10 +1,11 @@
 /*
- * PCIe host controller driver for HiSilicon Hip05 SoC
+ * PCIe host controller driver for HiSilicon SoCs
  *
  * Copyright (C) 2015 HiSilicon Co., Ltd. http://www.hisilicon.com
  *
- * Author: Zhou Wang <wangzhou1@hisilicon.com>
- *         Dacai Zhu <zhudacai@hisilicon.com>
+ * Authors: Zhou Wang <wangzhou1@hisilicon.com>
+ *          Dacai Zhu <zhudacai@hisilicon.com>
+ *          Gabriele Paoloni <gabriele.paoloni@huawei.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -16,21 +17,31 @@
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/platform_device.h>
+#include <linux/of_device.h>
 #include <linux/regmap.h>
 
 #include "pcie-designware.h"
 
-#define PCIE_SUBCTRL_SYS_STATE4_REG                     0x6818
-#define PCIE_LTSSM_LINKUP_STATE                         0x11
-#define PCIE_LTSSM_STATE_MASK                           0x3F
+#define PCIE_LTSSM_LINKUP_STATE				0x11
+#define PCIE_LTSSM_STATE_MASK				0x3F
+#define PCIE_SUBCTRL_SYS_STATE4_REG			0x6818
+#define PCIE_SYS_STATE4						0x31c
+#define PCIE_HIP06_CTRL_OFF					0x1000
 
 #define to_hisi_pcie(x)	container_of(x, struct hisi_pcie, pp)
 
+struct hisi_pcie;
+
+struct pcie_soc_ops {
+	int (*hisi_pcie_link_up)(struct hisi_pcie *pcie);
+};
+
 struct hisi_pcie {
 	struct regmap *subctrl;
 	void __iomem *reg_base;
 	u32 port_id;
 	struct pcie_port pp;
+	struct pcie_soc_ops *soc_ops;
 };
 
 static inline void hisi_pcie_apb_writel(struct hisi_pcie *pcie,
@@ -44,7 +55,7 @@ static inline u32 hisi_pcie_apb_readl(struct hisi_pcie *pcie, u32 reg)
 	return readl(pcie->reg_base + reg);
 }
 
-/* Hip05 PCIe host only supports 32-bit config access */
+/* HipXX PCIe host only supports 32-bit config access */
 static int hisi_pcie_cfg_read(struct pcie_port *pp, int where, int size,
 			      u32 *val)
 {
@@ -69,7 +80,7 @@ static int hisi_pcie_cfg_read(struct pcie_port *pp, int where, int size,
 	return PCIBIOS_SUCCESSFUL;
 }
 
-/* Hip05 PCIe host only supports 32-bit config access */
+/* HipXX PCIe host only supports 32-bit config access */
 static int hisi_pcie_cfg_write(struct pcie_port *pp, int where, int  size,
 				u32 val)
 {
@@ -96,10 +107,9 @@ static int hisi_pcie_cfg_write(struct pcie_port *pp, int where, int  size,
 	return PCIBIOS_SUCCESSFUL;
 }
 
-static int hisi_pcie_link_up(struct pcie_port *pp)
+static int hisi_pcie_link_up_hip05(struct hisi_pcie *hisi_pcie)
 {
 	u32 val;
-	struct hisi_pcie *hisi_pcie = to_hisi_pcie(pp);
 
 	regmap_read(hisi_pcie->subctrl, PCIE_SUBCTRL_SYS_STATE4_REG +
 		    0x100 * hisi_pcie->port_id, &val);
@@ -107,6 +117,23 @@ static int hisi_pcie_link_up(struct pcie_port *pp)
 	return ((val & PCIE_LTSSM_STATE_MASK) == PCIE_LTSSM_LINKUP_STATE);
 }
 
+static int hisi_pcie_link_up_hip06(struct hisi_pcie *hisi_pcie)
+{
+	u32 val;
+
+	val = hisi_pcie_apb_readl(hisi_pcie, PCIE_HIP06_CTRL_OFF +
+			PCIE_SYS_STATE4);
+
+	return ((val & PCIE_LTSSM_STATE_MASK) == PCIE_LTSSM_LINKUP_STATE);
+}
+
+static int hisi_pcie_link_up(struct pcie_port *pp)
+{
+	struct hisi_pcie *hisi_pcie = to_hisi_pcie(pp);
+
+	return hisi_pcie->soc_ops->hisi_pcie_link_up(hisi_pcie);
+}
+
 static struct pcie_host_ops hisi_pcie_host_ops = {
 	.rd_own_conf = hisi_pcie_cfg_read,
 	.wr_own_conf = hisi_pcie_cfg_write,
@@ -145,7 +172,9 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 {
 	struct hisi_pcie *hisi_pcie;
 	struct pcie_port *pp;
+	const struct of_device_id *match;
 	struct resource *reg;
+	struct device_driver *driver;
 	int ret;
 
 	hisi_pcie = devm_kzalloc(&pdev->dev, sizeof(*hisi_pcie), GFP_KERNEL);
@@ -154,6 +183,10 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 
 	pp = &hisi_pcie->pp;
 	pp->dev = &pdev->dev;
+	driver = (pdev->dev).driver;
+
+	match = of_match_device(driver->of_match_table, &pdev->dev);
+	hisi_pcie->soc_ops = (struct pcie_soc_ops *) match->data;
 
 	hisi_pcie->subctrl =
 	syscon_regmap_lookup_by_compatible("hisilicon,pcie-sas-subctrl");
@@ -182,11 +215,27 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static struct pcie_soc_ops hip05_ops = {
+		&hisi_pcie_link_up_hip05
+};
+
+static struct pcie_soc_ops hip06_ops = {
+		&hisi_pcie_link_up_hip06
+};
+
 static const struct of_device_id hisi_pcie_of_match[] = {
-	{.compatible = "hisilicon,hip05-pcie",},
+	{
+			.compatible = "hisilicon,hip05-pcie",
+			.data	    = (void *) &hip05_ops,
+	},
+	{
+			.compatible = "hisilicon,hip06-pcie",
+			.data	    = (void *) &hip06_ops,
+	},
 	{},
 };
 
+
 MODULE_DEVICE_TABLE(of, hisi_pcie_of_match);
 
 static struct platform_driver hisi_pcie_driver = {
@@ -198,3 +247,8 @@ static struct platform_driver hisi_pcie_driver = {
 };
 
 module_platform_driver(hisi_pcie_driver);
+
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_AUTHOR("Dacai Zhu <zhudacai@hisilicon.com>");
+MODULE_AUTHOR("Gabriele Paoloni <gabriele.paoloni@huawei.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pcie-iproc-bcma.c b/drivers/pci/host/pcie-iproc-bcma.c
index 96a7d999fd5e..0d7bee4a0d26 100644
--- a/drivers/pci/host/pcie-iproc-bcma.c
+++ b/drivers/pci/host/pcie-iproc-bcma.c
@@ -55,6 +55,7 @@ static int iproc_pcie_bcma_probe(struct bcma_device *bdev)
 	bcma_set_drvdata(bdev, pcie);
 
 	pcie->base = bdev->io_addr;
+	pcie->base_addr = bdev->addr;
 
 	res_mem.start = bdev->addr_s[0];
 	res_mem.end = bdev->addr_s[0] + SZ_128M - 1;
diff --git a/drivers/pci/host/pcie-iproc-msi.c b/drivers/pci/host/pcie-iproc-msi.c
new file mode 100644
index 000000000000..9a2973bdc78a
--- /dev/null
+++ b/drivers/pci/host/pcie-iproc-msi.c
@@ -0,0 +1,675 @@
+/*
+ * Copyright (C) 2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+
+#include "pcie-iproc.h"
+
+#define IPROC_MSI_INTR_EN_SHIFT        11
+#define IPROC_MSI_INTR_EN              BIT(IPROC_MSI_INTR_EN_SHIFT)
+#define IPROC_MSI_INT_N_EVENT_SHIFT    1
+#define IPROC_MSI_INT_N_EVENT          BIT(IPROC_MSI_INT_N_EVENT_SHIFT)
+#define IPROC_MSI_EQ_EN_SHIFT          0
+#define IPROC_MSI_EQ_EN                BIT(IPROC_MSI_EQ_EN_SHIFT)
+
+#define IPROC_MSI_EQ_MASK              0x3f
+
+/* Max number of GIC interrupts */
+#define NR_HW_IRQS                     6
+
+/* Number of entries in each event queue */
+#define EQ_LEN                         64
+
+/* Size of each event queue memory region */
+#define EQ_MEM_REGION_SIZE             SZ_4K
+
+/* Size of each MSI address region */
+#define MSI_MEM_REGION_SIZE            SZ_4K
+
+enum iproc_msi_reg {
+	IPROC_MSI_EQ_PAGE = 0,
+	IPROC_MSI_EQ_PAGE_UPPER,
+	IPROC_MSI_PAGE,
+	IPROC_MSI_PAGE_UPPER,
+	IPROC_MSI_CTRL,
+	IPROC_MSI_EQ_HEAD,
+	IPROC_MSI_EQ_TAIL,
+	IPROC_MSI_INTS_EN,
+	IPROC_MSI_REG_SIZE,
+};
+
+struct iproc_msi;
+
+/**
+ * iProc MSI group
+ *
+ * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI
+ * event queue.
+ *
+ * @msi: pointer to iProc MSI data
+ * @gic_irq: GIC interrupt
+ * @eq: Event queue number
+ */
+struct iproc_msi_grp {
+	struct iproc_msi *msi;
+	int gic_irq;
+	unsigned int eq;
+};
+
+/**
+ * iProc event queue based MSI
+ *
+ * Only meant to be used on platforms without MSI support integrated into the
+ * GIC.
+ *
+ * @pcie: pointer to iProc PCIe data
+ * @reg_offsets: MSI register offsets
+ * @grps: MSI groups
+ * @nr_irqs: number of total interrupts connected to GIC
+ * @nr_cpus: number of toal CPUs
+ * @has_inten_reg: indicates the MSI interrupt enable register needs to be
+ * set explicitly (required for some legacy platforms)
+ * @bitmap: MSI vector bitmap
+ * @bitmap_lock: lock to protect access to the MSI bitmap
+ * @nr_msi_vecs: total number of MSI vectors
+ * @inner_domain: inner IRQ domain
+ * @msi_domain: MSI IRQ domain
+ * @nr_eq_region: required number of 4K aligned memory region for MSI event
+ * queues
+ * @nr_msi_region: required number of 4K aligned address region for MSI posted
+ * writes
+ * @eq_cpu: pointer to allocated memory region for MSI event queues
+ * @eq_dma: DMA address of MSI event queues
+ * @msi_addr: MSI address
+ */
+struct iproc_msi {
+	struct iproc_pcie *pcie;
+	const u16 (*reg_offsets)[IPROC_MSI_REG_SIZE];
+	struct iproc_msi_grp *grps;
+	int nr_irqs;
+	int nr_cpus;
+	bool has_inten_reg;
+	unsigned long *bitmap;
+	struct mutex bitmap_lock;
+	unsigned int nr_msi_vecs;
+	struct irq_domain *inner_domain;
+	struct irq_domain *msi_domain;
+	unsigned int nr_eq_region;
+	unsigned int nr_msi_region;
+	void *eq_cpu;
+	dma_addr_t eq_dma;
+	phys_addr_t msi_addr;
+};
+
+static const u16 iproc_msi_reg_paxb[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x210, 0x250, 0x254, 0x208 },
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x214, 0x258, 0x25c, 0x208 },
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x218, 0x260, 0x264, 0x208 },
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x21c, 0x268, 0x26c, 0x208 },
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x220, 0x270, 0x274, 0x208 },
+	{ 0x200, 0x2c0, 0x204, 0x2c4, 0x224, 0x278, 0x27c, 0x208 },
+};
+
+static const u16 iproc_msi_reg_paxc[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
+	{ 0xc00, 0xc04, 0xc08, 0xc0c, 0xc40, 0xc50, 0xc60 },
+	{ 0xc10, 0xc14, 0xc18, 0xc1c, 0xc44, 0xc54, 0xc64 },
+	{ 0xc20, 0xc24, 0xc28, 0xc2c, 0xc48, 0xc58, 0xc68 },
+	{ 0xc30, 0xc34, 0xc38, 0xc3c, 0xc4c, 0xc5c, 0xc6c },
+};
+
+static inline u32 iproc_msi_read_reg(struct iproc_msi *msi,
+				     enum iproc_msi_reg reg,
+				     unsigned int eq)
+{
+	struct iproc_pcie *pcie = msi->pcie;
+
+	return readl_relaxed(pcie->base + msi->reg_offsets[eq][reg]);
+}
+
+static inline void iproc_msi_write_reg(struct iproc_msi *msi,
+				       enum iproc_msi_reg reg,
+				       int eq, u32 val)
+{
+	struct iproc_pcie *pcie = msi->pcie;
+
+	writel_relaxed(val, pcie->base + msi->reg_offsets[eq][reg]);
+}
+
+static inline u32 hwirq_to_group(struct iproc_msi *msi, unsigned long hwirq)
+{
+	return (hwirq % msi->nr_irqs);
+}
+
+static inline unsigned int iproc_msi_addr_offset(struct iproc_msi *msi,
+						 unsigned long hwirq)
+{
+	if (msi->nr_msi_region > 1)
+		return hwirq_to_group(msi, hwirq) * MSI_MEM_REGION_SIZE;
+	else
+		return hwirq_to_group(msi, hwirq) * sizeof(u32);
+}
+
+static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq)
+{
+	if (msi->nr_eq_region > 1)
+		return eq * EQ_MEM_REGION_SIZE;
+	else
+		return eq * EQ_LEN * sizeof(u32);
+}
+
+static struct irq_chip iproc_msi_irq_chip = {
+	.name = "iProc-MSI",
+};
+
+static struct msi_domain_info iproc_msi_domain_info = {
+	.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		MSI_FLAG_PCI_MSIX,
+	.chip = &iproc_msi_irq_chip,
+};
+
+/*
+ * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a
+ * dedicated event queue.  Each MSI group can support up to 64 MSI vectors.
+ *
+ * The number of MSI groups varies between different iProc SoCs.  The total
+ * number of CPU cores also varies.  To support MSI IRQ affinity, we
+ * distribute GIC interrupts across all available CPUs.  MSI vector is moved
+ * from one GIC interrupt to another to steer to the target CPU.
+ *
+ * Assuming:
+ * - the number of MSI groups is M
+ * - the number of CPU cores is N
+ * - M is always a multiple of N
+ *
+ * Total number of raw MSI vectors = M * 64
+ * Total number of supported MSI vectors = (M * 64) / N
+ */
+static inline int hwirq_to_cpu(struct iproc_msi *msi, unsigned long hwirq)
+{
+	return (hwirq % msi->nr_cpus);
+}
+
+static inline unsigned long hwirq_to_canonical_hwirq(struct iproc_msi *msi,
+						     unsigned long hwirq)
+{
+	return (hwirq - hwirq_to_cpu(msi, hwirq));
+}
+
+static int iproc_msi_irq_set_affinity(struct irq_data *data,
+				      const struct cpumask *mask, bool force)
+{
+	struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+	int target_cpu = cpumask_first(mask);
+	int curr_cpu;
+
+	curr_cpu = hwirq_to_cpu(msi, data->hwirq);
+	if (curr_cpu == target_cpu)
+		return IRQ_SET_MASK_OK_DONE;
+
+	/* steer MSI to the target CPU */
+	data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu;
+
+	return IRQ_SET_MASK_OK;
+}
+
+static void iproc_msi_irq_compose_msi_msg(struct irq_data *data,
+					  struct msi_msg *msg)
+{
+	struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+	dma_addr_t addr;
+
+	addr = msi->msi_addr + iproc_msi_addr_offset(msi, data->hwirq);
+	msg->address_lo = lower_32_bits(addr);
+	msg->address_hi = upper_32_bits(addr);
+	msg->data = data->hwirq;
+}
+
+static struct irq_chip iproc_msi_bottom_irq_chip = {
+	.name = "MSI",
+	.irq_set_affinity = iproc_msi_irq_set_affinity,
+	.irq_compose_msi_msg = iproc_msi_irq_compose_msi_msg,
+};
+
+static int iproc_msi_irq_domain_alloc(struct irq_domain *domain,
+				      unsigned int virq, unsigned int nr_irqs,
+				      void *args)
+{
+	struct iproc_msi *msi = domain->host_data;
+	int hwirq;
+
+	mutex_lock(&msi->bitmap_lock);
+
+	/* Allocate 'nr_cpus' number of MSI vectors each time */
+	hwirq = bitmap_find_next_zero_area(msi->bitmap, msi->nr_msi_vecs, 0,
+					   msi->nr_cpus, 0);
+	if (hwirq < msi->nr_msi_vecs) {
+		bitmap_set(msi->bitmap, hwirq, msi->nr_cpus);
+	} else {
+		mutex_unlock(&msi->bitmap_lock);
+		return -ENOSPC;
+	}
+
+	mutex_unlock(&msi->bitmap_lock);
+
+	irq_domain_set_info(domain, virq, hwirq, &iproc_msi_bottom_irq_chip,
+			    domain->host_data, handle_simple_irq, NULL, NULL);
+
+	return 0;
+}
+
+static void iproc_msi_irq_domain_free(struct irq_domain *domain,
+				      unsigned int virq, unsigned int nr_irqs)
+{
+	struct irq_data *data = irq_domain_get_irq_data(domain, virq);
+	struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+	unsigned int hwirq;
+
+	mutex_lock(&msi->bitmap_lock);
+
+	hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq);
+	bitmap_clear(msi->bitmap, hwirq, msi->nr_cpus);
+
+	mutex_unlock(&msi->bitmap_lock);
+
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+	.alloc = iproc_msi_irq_domain_alloc,
+	.free = iproc_msi_irq_domain_free,
+};
+
+static inline u32 decode_msi_hwirq(struct iproc_msi *msi, u32 eq, u32 head)
+{
+	u32 *msg, hwirq;
+	unsigned int offs;
+
+	offs = iproc_msi_eq_offset(msi, eq) + head * sizeof(u32);
+	msg = (u32 *)(msi->eq_cpu + offs);
+	hwirq = *msg & IPROC_MSI_EQ_MASK;
+
+	/*
+	 * Since we have multiple hwirq mapped to a single MSI vector,
+	 * now we need to derive the hwirq at CPU0.  It can then be used to
+	 * mapped back to virq.
+	 */
+	return hwirq_to_canonical_hwirq(msi, hwirq);
+}
+
+static void iproc_msi_handler(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct iproc_msi_grp *grp;
+	struct iproc_msi *msi;
+	struct iproc_pcie *pcie;
+	u32 eq, head, tail, nr_events;
+	unsigned long hwirq;
+	int virq;
+
+	chained_irq_enter(chip, desc);
+
+	grp = irq_desc_get_handler_data(desc);
+	msi = grp->msi;
+	pcie = msi->pcie;
+	eq = grp->eq;
+
+	/*
+	 * iProc MSI event queue is tracked by head and tail pointers.  Head
+	 * pointer indicates the next entry (MSI data) to be consumed by SW in
+	 * the queue and needs to be updated by SW.  iProc MSI core uses the
+	 * tail pointer as the next data insertion point.
+	 *
+	 * Entries between head and tail pointers contain valid MSI data.  MSI
+	 * data is guaranteed to be in the event queue memory before the tail
+	 * pointer is updated by the iProc MSI core.
+	 */
+	head = iproc_msi_read_reg(msi, IPROC_MSI_EQ_HEAD,
+				  eq) & IPROC_MSI_EQ_MASK;
+	do {
+		tail = iproc_msi_read_reg(msi, IPROC_MSI_EQ_TAIL,
+					  eq) & IPROC_MSI_EQ_MASK;
+
+		/*
+		 * Figure out total number of events (MSI data) to be
+		 * processed.
+		 */
+		nr_events = (tail < head) ?
+			(EQ_LEN - (head - tail)) : (tail - head);
+		if (!nr_events)
+			break;
+
+		/* process all outstanding events */
+		while (nr_events--) {
+			hwirq = decode_msi_hwirq(msi, eq, head);
+			virq = irq_find_mapping(msi->inner_domain, hwirq);
+			generic_handle_irq(virq);
+
+			head++;
+			head %= EQ_LEN;
+		}
+
+		/*
+		 * Now all outstanding events have been processed.  Update the
+		 * head pointer.
+		 */
+		iproc_msi_write_reg(msi, IPROC_MSI_EQ_HEAD, eq, head);
+
+		/*
+		 * Now go read the tail pointer again to see if there are new
+		 * oustanding events that came in during the above window.
+		 */
+	} while (true);
+
+	chained_irq_exit(chip, desc);
+}
+
+static void iproc_msi_enable(struct iproc_msi *msi)
+{
+	int i, eq;
+	u32 val;
+
+	/* Program memory region for each event queue */
+	for (i = 0; i < msi->nr_eq_region; i++) {
+		dma_addr_t addr = msi->eq_dma + (i * EQ_MEM_REGION_SIZE);
+
+		iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE, i,
+				    lower_32_bits(addr));
+		iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE_UPPER, i,
+				    upper_32_bits(addr));
+	}
+
+	/* Program address region for MSI posted writes */
+	for (i = 0; i < msi->nr_msi_region; i++) {
+		phys_addr_t addr = msi->msi_addr + (i * MSI_MEM_REGION_SIZE);
+
+		iproc_msi_write_reg(msi, IPROC_MSI_PAGE, i,
+				    lower_32_bits(addr));
+		iproc_msi_write_reg(msi, IPROC_MSI_PAGE_UPPER, i,
+				    upper_32_bits(addr));
+	}
+
+	for (eq = 0; eq < msi->nr_irqs; eq++) {
+		/* Enable MSI event queue */
+		val = IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
+			IPROC_MSI_EQ_EN;
+		iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
+
+		/*
+		 * Some legacy platforms require the MSI interrupt enable
+		 * register to be set explicitly.
+		 */
+		if (msi->has_inten_reg) {
+			val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
+			val |= BIT(eq);
+			iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
+		}
+	}
+}
+
+static void iproc_msi_disable(struct iproc_msi *msi)
+{
+	u32 eq, val;
+
+	for (eq = 0; eq < msi->nr_irqs; eq++) {
+		if (msi->has_inten_reg) {
+			val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
+			val &= ~BIT(eq);
+			iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
+		}
+
+		val = iproc_msi_read_reg(msi, IPROC_MSI_CTRL, eq);
+		val &= ~(IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
+			 IPROC_MSI_EQ_EN);
+		iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
+	}
+}
+
+static int iproc_msi_alloc_domains(struct device_node *node,
+				   struct iproc_msi *msi)
+{
+	msi->inner_domain = irq_domain_add_linear(NULL, msi->nr_msi_vecs,
+						  &msi_domain_ops, msi);
+	if (!msi->inner_domain)
+		return -ENOMEM;
+
+	msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node),
+						    &iproc_msi_domain_info,
+						    msi->inner_domain);
+	if (!msi->msi_domain) {
+		irq_domain_remove(msi->inner_domain);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void iproc_msi_free_domains(struct iproc_msi *msi)
+{
+	if (msi->msi_domain)
+		irq_domain_remove(msi->msi_domain);
+
+	if (msi->inner_domain)
+		irq_domain_remove(msi->inner_domain);
+}
+
+static void iproc_msi_irq_free(struct iproc_msi *msi, unsigned int cpu)
+{
+	int i;
+
+	for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
+		irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
+						 NULL, NULL);
+	}
+}
+
+static int iproc_msi_irq_setup(struct iproc_msi *msi, unsigned int cpu)
+{
+	int i, ret;
+	cpumask_var_t mask;
+	struct iproc_pcie *pcie = msi->pcie;
+
+	for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
+		irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
+						 iproc_msi_handler,
+						 &msi->grps[i]);
+		/* Dedicate GIC interrupt to each CPU core */
+		if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
+			cpumask_clear(mask);
+			cpumask_set_cpu(cpu, mask);
+			ret = irq_set_affinity(msi->grps[i].gic_irq, mask);
+			if (ret)
+				dev_err(pcie->dev,
+					"failed to set affinity for IRQ%d\n",
+					msi->grps[i].gic_irq);
+			free_cpumask_var(mask);
+		} else {
+			dev_err(pcie->dev, "failed to alloc CPU mask\n");
+			ret = -EINVAL;
+		}
+
+		if (ret) {
+			/* Free all configured/unconfigured IRQs */
+			iproc_msi_irq_free(msi, cpu);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
+{
+	struct iproc_msi *msi;
+	int i, ret;
+	unsigned int cpu;
+
+	if (!of_device_is_compatible(node, "brcm,iproc-msi"))
+		return -ENODEV;
+
+	if (!of_find_property(node, "msi-controller", NULL))
+		return -ENODEV;
+
+	if (pcie->msi)
+		return -EBUSY;
+
+	msi = devm_kzalloc(pcie->dev, sizeof(*msi), GFP_KERNEL);
+	if (!msi)
+		return -ENOMEM;
+
+	msi->pcie = pcie;
+	pcie->msi = msi;
+	msi->msi_addr = pcie->base_addr;
+	mutex_init(&msi->bitmap_lock);
+	msi->nr_cpus = num_possible_cpus();
+
+	msi->nr_irqs = of_irq_count(node);
+	if (!msi->nr_irqs) {
+		dev_err(pcie->dev, "found no MSI GIC interrupt\n");
+		return -ENODEV;
+	}
+
+	if (msi->nr_irqs > NR_HW_IRQS) {
+		dev_warn(pcie->dev, "too many MSI GIC interrupts defined %d\n",
+			 msi->nr_irqs);
+		msi->nr_irqs = NR_HW_IRQS;
+	}
+
+	if (msi->nr_irqs < msi->nr_cpus) {
+		dev_err(pcie->dev,
+			"not enough GIC interrupts for MSI affinity\n");
+		return -EINVAL;
+	}
+
+	if (msi->nr_irqs % msi->nr_cpus != 0) {
+		msi->nr_irqs -= msi->nr_irqs % msi->nr_cpus;
+		dev_warn(pcie->dev, "Reducing number of interrupts to %d\n",
+			 msi->nr_irqs);
+	}
+
+	switch (pcie->type) {
+	case IPROC_PCIE_PAXB:
+		msi->reg_offsets = iproc_msi_reg_paxb;
+		msi->nr_eq_region = 1;
+		msi->nr_msi_region = 1;
+		break;
+	case IPROC_PCIE_PAXC:
+		msi->reg_offsets = iproc_msi_reg_paxc;
+		msi->nr_eq_region = msi->nr_irqs;
+		msi->nr_msi_region = msi->nr_irqs;
+		break;
+	default:
+		dev_err(pcie->dev, "incompatible iProc PCIe interface\n");
+		return -EINVAL;
+	}
+
+	if (of_find_property(node, "brcm,pcie-msi-inten", NULL))
+		msi->has_inten_reg = true;
+
+	msi->nr_msi_vecs = msi->nr_irqs * EQ_LEN;
+	msi->bitmap = devm_kcalloc(pcie->dev, BITS_TO_LONGS(msi->nr_msi_vecs),
+				   sizeof(*msi->bitmap), GFP_KERNEL);
+	if (!msi->bitmap)
+		return -ENOMEM;
+
+	msi->grps = devm_kcalloc(pcie->dev, msi->nr_irqs, sizeof(*msi->grps),
+				 GFP_KERNEL);
+	if (!msi->grps)
+		return -ENOMEM;
+
+	for (i = 0; i < msi->nr_irqs; i++) {
+		unsigned int irq = irq_of_parse_and_map(node, i);
+
+		if (!irq) {
+			dev_err(pcie->dev, "unable to parse/map interrupt\n");
+			ret = -ENODEV;
+			goto free_irqs;
+		}
+		msi->grps[i].gic_irq = irq;
+		msi->grps[i].msi = msi;
+		msi->grps[i].eq = i;
+	}
+
+	/* Reserve memory for event queue and make sure memories are zeroed */
+	msi->eq_cpu = dma_zalloc_coherent(pcie->dev,
+					  msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+					  &msi->eq_dma, GFP_KERNEL);
+	if (!msi->eq_cpu) {
+		ret = -ENOMEM;
+		goto free_irqs;
+	}
+
+	ret = iproc_msi_alloc_domains(node, msi);
+	if (ret) {
+		dev_err(pcie->dev, "failed to create MSI domains\n");
+		goto free_eq_dma;
+	}
+
+	for_each_online_cpu(cpu) {
+		ret = iproc_msi_irq_setup(msi, cpu);
+		if (ret)
+			goto free_msi_irq;
+	}
+
+	iproc_msi_enable(msi);
+
+	return 0;
+
+free_msi_irq:
+	for_each_online_cpu(cpu)
+		iproc_msi_irq_free(msi, cpu);
+	iproc_msi_free_domains(msi);
+
+free_eq_dma:
+	dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+			  msi->eq_cpu, msi->eq_dma);
+
+free_irqs:
+	for (i = 0; i < msi->nr_irqs; i++) {
+		if (msi->grps[i].gic_irq)
+			irq_dispose_mapping(msi->grps[i].gic_irq);
+	}
+	pcie->msi = NULL;
+	return ret;
+}
+EXPORT_SYMBOL(iproc_msi_init);
+
+void iproc_msi_exit(struct iproc_pcie *pcie)
+{
+	struct iproc_msi *msi = pcie->msi;
+	unsigned int i, cpu;
+
+	if (!msi)
+		return;
+
+	iproc_msi_disable(msi);
+
+	for_each_online_cpu(cpu)
+		iproc_msi_irq_free(msi, cpu);
+
+	iproc_msi_free_domains(msi);
+
+	dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+			  msi->eq_cpu, msi->eq_dma);
+
+	for (i = 0; i < msi->nr_irqs; i++) {
+		if (msi->grps[i].gic_irq)
+			irq_dispose_mapping(msi->grps[i].gic_irq);
+	}
+}
+EXPORT_SYMBOL(iproc_msi_exit);
diff --git a/drivers/pci/host/pcie-iproc-platform.c b/drivers/pci/host/pcie-iproc-platform.c
index c9550dc8b8ed..1738c5288eb6 100644
--- a/drivers/pci/host/pcie-iproc-platform.c
+++ b/drivers/pci/host/pcie-iproc-platform.c
@@ -26,8 +26,21 @@
 
 #include "pcie-iproc.h"
 
+static const struct of_device_id iproc_pcie_of_match_table[] = {
+	{
+		.compatible = "brcm,iproc-pcie",
+		.data = (int *)IPROC_PCIE_PAXB,
+	}, {
+		.compatible = "brcm,iproc-pcie-paxc",
+		.data = (int *)IPROC_PCIE_PAXC,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, iproc_pcie_of_match_table);
+
 static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *of_id;
 	struct iproc_pcie *pcie;
 	struct device_node *np = pdev->dev.of_node;
 	struct resource reg;
@@ -35,11 +48,16 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 	LIST_HEAD(res);
 	int ret;
 
+	of_id = of_match_device(iproc_pcie_of_match_table, &pdev->dev);
+	if (!of_id)
+		return -EINVAL;
+
 	pcie = devm_kzalloc(&pdev->dev, sizeof(struct iproc_pcie), GFP_KERNEL);
 	if (!pcie)
 		return -ENOMEM;
 
 	pcie->dev = &pdev->dev;
+	pcie->type = (enum iproc_pcie_type)of_id->data;
 	platform_set_drvdata(pdev, pcie);
 
 	ret = of_address_to_resource(np, 0, &reg);
@@ -53,6 +71,7 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 		dev_err(pcie->dev, "unable to map controller registers\n");
 		return -ENOMEM;
 	}
+	pcie->base_addr = reg.start;
 
 	if (of_property_read_bool(np, "brcm,pcie-ob")) {
 		u32 val;
@@ -114,12 +133,6 @@ static int iproc_pcie_pltfm_remove(struct platform_device *pdev)
 	return iproc_pcie_remove(pcie);
 }
 
-static const struct of_device_id iproc_pcie_of_match_table[] = {
-	{ .compatible = "brcm,iproc-pcie", },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, iproc_pcie_of_match_table);
-
 static struct platform_driver iproc_pcie_pltfm_driver = {
 	.driver = {
 		.name = "iproc-pcie",
diff --git a/drivers/pci/host/pcie-iproc.c b/drivers/pci/host/pcie-iproc.c
index eac719af16aa..5816bceddb65 100644
--- a/drivers/pci/host/pcie-iproc.c
+++ b/drivers/pci/host/pcie-iproc.c
@@ -30,20 +30,16 @@
 
 #include "pcie-iproc.h"
 
-#define CLK_CONTROL_OFFSET           0x000
 #define EP_PERST_SOURCE_SELECT_SHIFT 2
 #define EP_PERST_SOURCE_SELECT       BIT(EP_PERST_SOURCE_SELECT_SHIFT)
 #define EP_MODE_SURVIVE_PERST_SHIFT  1
 #define EP_MODE_SURVIVE_PERST        BIT(EP_MODE_SURVIVE_PERST_SHIFT)
 #define RC_PCIE_RST_OUTPUT_SHIFT     0
 #define RC_PCIE_RST_OUTPUT           BIT(RC_PCIE_RST_OUTPUT_SHIFT)
+#define PAXC_RESET_MASK              0x7f
 
-#define CFG_IND_ADDR_OFFSET          0x120
 #define CFG_IND_ADDR_MASK            0x00001ffc
 
-#define CFG_IND_DATA_OFFSET          0x124
-
-#define CFG_ADDR_OFFSET              0x1f8
 #define CFG_ADDR_BUS_NUM_SHIFT       20
 #define CFG_ADDR_BUS_NUM_MASK        0x0ff00000
 #define CFG_ADDR_DEV_NUM_SHIFT       15
@@ -55,12 +51,8 @@
 #define CFG_ADDR_CFG_TYPE_SHIFT      0
 #define CFG_ADDR_CFG_TYPE_MASK       0x00000003
 
-#define CFG_DATA_OFFSET              0x1fc
-
-#define SYS_RC_INTX_EN               0x330
 #define SYS_RC_INTX_MASK             0xf
 
-#define PCIE_LINK_STATUS_OFFSET      0xf0c
 #define PCIE_PHYLINKUP_SHIFT         3
 #define PCIE_PHYLINKUP               BIT(PCIE_PHYLINKUP_SHIFT)
 #define PCIE_DL_ACTIVE_SHIFT         2
@@ -71,12 +63,54 @@
 #define OARR_SIZE_CFG_SHIFT          1
 #define OARR_SIZE_CFG                BIT(OARR_SIZE_CFG_SHIFT)
 
-#define OARR_LO(window)              (0xd20 + (window) * 8)
-#define OARR_HI(window)              (0xd24 + (window) * 8)
-#define OMAP_LO(window)              (0xd40 + (window) * 8)
-#define OMAP_HI(window)              (0xd44 + (window) * 8)
-
 #define MAX_NUM_OB_WINDOWS           2
+#define MAX_NUM_PAXC_PF              4
+
+#define IPROC_PCIE_REG_INVALID 0xffff
+
+enum iproc_pcie_reg {
+	IPROC_PCIE_CLK_CTRL = 0,
+	IPROC_PCIE_CFG_IND_ADDR,
+	IPROC_PCIE_CFG_IND_DATA,
+	IPROC_PCIE_CFG_ADDR,
+	IPROC_PCIE_CFG_DATA,
+	IPROC_PCIE_INTX_EN,
+	IPROC_PCIE_OARR_LO,
+	IPROC_PCIE_OARR_HI,
+	IPROC_PCIE_OMAP_LO,
+	IPROC_PCIE_OMAP_HI,
+	IPROC_PCIE_LINK_STATUS,
+};
+
+/* iProc PCIe PAXB registers */
+static const u16 iproc_pcie_reg_paxb[] = {
+	[IPROC_PCIE_CLK_CTRL]     = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR] = 0x120,
+	[IPROC_PCIE_CFG_IND_DATA] = 0x124,
+	[IPROC_PCIE_CFG_ADDR]     = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]     = 0x1fc,
+	[IPROC_PCIE_INTX_EN]      = 0x330,
+	[IPROC_PCIE_OARR_LO]      = 0xd20,
+	[IPROC_PCIE_OARR_HI]      = 0xd24,
+	[IPROC_PCIE_OMAP_LO]      = 0xd40,
+	[IPROC_PCIE_OMAP_HI]      = 0xd44,
+	[IPROC_PCIE_LINK_STATUS]  = 0xf0c,
+};
+
+/* iProc PCIe PAXC v1 registers */
+static const u16 iproc_pcie_reg_paxc[] = {
+	[IPROC_PCIE_CLK_CTRL]     = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,
+	[IPROC_PCIE_CFG_IND_DATA] = 0x1f4,
+	[IPROC_PCIE_CFG_ADDR]     = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]     = 0x1fc,
+	[IPROC_PCIE_INTX_EN]      = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_OARR_LO]      = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_OARR_HI]      = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_OMAP_LO]      = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_OMAP_HI]      = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_LINK_STATUS]  = IPROC_PCIE_REG_INVALID,
+};
 
 static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
 {
@@ -91,6 +125,65 @@ static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
 	return pcie;
 }
 
+static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset)
+{
+	return !!(reg_offset == IPROC_PCIE_REG_INVALID);
+}
+
+static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,
+					enum iproc_pcie_reg reg)
+{
+	return pcie->reg_offsets[reg];
+}
+
+static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,
+				      enum iproc_pcie_reg reg)
+{
+	u16 offset = iproc_pcie_reg_offset(pcie, reg);
+
+	if (iproc_pcie_reg_is_invalid(offset))
+		return 0;
+
+	return readl(pcie->base + offset);
+}
+
+static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,
+					enum iproc_pcie_reg reg, u32 val)
+{
+	u16 offset = iproc_pcie_reg_offset(pcie, reg);
+
+	if (iproc_pcie_reg_is_invalid(offset))
+		return;
+
+	writel(val, pcie->base + offset);
+}
+
+static inline void iproc_pcie_ob_write(struct iproc_pcie *pcie,
+				       enum iproc_pcie_reg reg,
+				       unsigned window, u32 val)
+{
+	u16 offset = iproc_pcie_reg_offset(pcie, reg);
+
+	if (iproc_pcie_reg_is_invalid(offset))
+		return;
+
+	writel(val, pcie->base + offset + (window * 8));
+}
+
+static inline bool iproc_pcie_device_is_valid(struct iproc_pcie *pcie,
+					      unsigned int slot,
+					      unsigned int fn)
+{
+	if (slot > 0)
+		return false;
+
+	/* PAXC can only support limited number of functions */
+	if (pcie->type == IPROC_PCIE_PAXC && fn >= MAX_NUM_PAXC_PF)
+		return false;
+
+	return true;
+}
+
 /**
  * Note access to the configuration registers are protected at the higher layer
  * by 'pci_lock' in drivers/pci/access.c
@@ -104,28 +197,34 @@ static void __iomem *iproc_pcie_map_cfg_bus(struct pci_bus *bus,
 	unsigned fn = PCI_FUNC(devfn);
 	unsigned busno = bus->number;
 	u32 val;
+	u16 offset;
+
+	if (!iproc_pcie_device_is_valid(pcie, slot, fn))
+		return NULL;
 
 	/* root complex access */
 	if (busno == 0) {
-		if (slot >= 1)
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,
+				     where & CFG_IND_ADDR_MASK);
+		offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);
+		if (iproc_pcie_reg_is_invalid(offset))
 			return NULL;
-		writel(where & CFG_IND_ADDR_MASK,
-		       pcie->base + CFG_IND_ADDR_OFFSET);
-		return (pcie->base + CFG_IND_DATA_OFFSET);
+		else
+			return (pcie->base + offset);
 	}
 
-	if (fn > 1)
-		return NULL;
-
 	/* EP device access */
 	val = (busno << CFG_ADDR_BUS_NUM_SHIFT) |
 		(slot << CFG_ADDR_DEV_NUM_SHIFT) |
 		(fn << CFG_ADDR_FUNC_NUM_SHIFT) |
 		(where & CFG_ADDR_REG_NUM_MASK) |
 		(1 & CFG_ADDR_CFG_TYPE_MASK);
-	writel(val, pcie->base + CFG_ADDR_OFFSET);
-
-	return (pcie->base + CFG_DATA_OFFSET);
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);
+	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);
+	if (iproc_pcie_reg_is_invalid(offset))
+		return NULL;
+	else
+		return (pcie->base + offset);
 }
 
 static struct pci_ops iproc_pcie_ops = {
@@ -138,18 +237,29 @@ static void iproc_pcie_reset(struct iproc_pcie *pcie)
 {
 	u32 val;
 
+	if (pcie->type == IPROC_PCIE_PAXC) {
+		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
+		val &= ~PAXC_RESET_MASK;
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+		udelay(100);
+		val |= PAXC_RESET_MASK;
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+		udelay(100);
+		return;
+	}
+
 	/*
 	 * Select perst_b signal as reset source. Put the device into reset,
 	 * and then bring it out of reset
 	 */
-	val = readl(pcie->base + CLK_CONTROL_OFFSET);
+	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
 	val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &
 		~RC_PCIE_RST_OUTPUT;
-	writel(val, pcie->base + CLK_CONTROL_OFFSET);
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
 	udelay(250);
 
 	val |= RC_PCIE_RST_OUTPUT;
-	writel(val, pcie->base + CLK_CONTROL_OFFSET);
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
 	msleep(100);
 }
 
@@ -160,7 +270,14 @@ static int iproc_pcie_check_link(struct iproc_pcie *pcie, struct pci_bus *bus)
 	u16 pos, link_status;
 	bool link_is_active = false;
 
-	val = readl(pcie->base + PCIE_LINK_STATUS_OFFSET);
+	/*
+	 * PAXC connects to emulated endpoint devices directly and does not
+	 * have a Serdes.  Therefore skip the link detection logic here.
+	 */
+	if (pcie->type == IPROC_PCIE_PAXC)
+		return 0;
+
+	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);
 	if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) {
 		dev_err(pcie->dev, "PHY or data link is INACTIVE!\n");
 		return -ENODEV;
@@ -221,7 +338,7 @@ static int iproc_pcie_check_link(struct iproc_pcie *pcie, struct pci_bus *bus)
 
 static void iproc_pcie_enable(struct iproc_pcie *pcie)
 {
-	writel(SYS_RC_INTX_MASK, pcie->base + SYS_RC_INTX_EN);
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);
 }
 
 /**
@@ -245,7 +362,7 @@ static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
 
 	if (size > max_size) {
 		dev_err(pcie->dev,
-			"res size 0x%pap exceeds max supported size 0x%llx\n",
+			"res size %pap exceeds max supported size 0x%llx\n",
 			&size, max_size);
 		return -EINVAL;
 	}
@@ -272,11 +389,15 @@ static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
 	axi_addr -= ob->axi_offset;
 
 	for (i = 0; i < MAX_NUM_OB_WINDOWS; i++) {
-		writel(lower_32_bits(axi_addr) | OARR_VALID |
-		       (ob->set_oarr_size ? 1 : 0), pcie->base + OARR_LO(i));
-		writel(upper_32_bits(axi_addr), pcie->base + OARR_HI(i));
-		writel(lower_32_bits(pci_addr), pcie->base + OMAP_LO(i));
-		writel(upper_32_bits(pci_addr), pcie->base + OMAP_HI(i));
+		iproc_pcie_ob_write(pcie, IPROC_PCIE_OARR_LO, i,
+				    lower_32_bits(axi_addr) | OARR_VALID |
+				    (ob->set_oarr_size ? 1 : 0));
+		iproc_pcie_ob_write(pcie, IPROC_PCIE_OARR_HI, i,
+				    upper_32_bits(axi_addr));
+		iproc_pcie_ob_write(pcie, IPROC_PCIE_OMAP_LO, i,
+				    lower_32_bits(pci_addr));
+		iproc_pcie_ob_write(pcie, IPROC_PCIE_OMAP_HI, i,
+				    upper_32_bits(pci_addr));
 
 		size -= ob->window_size;
 		if (size == 0)
@@ -319,6 +440,26 @@ static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,
 	return 0;
 }
 
+static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)
+{
+	struct device_node *msi_node;
+
+	msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);
+	if (!msi_node)
+		return -ENODEV;
+
+	/*
+	 * If another MSI controller is being used, the call below should fail
+	 * but that is okay
+	 */
+	return iproc_msi_init(pcie, msi_node);
+}
+
+static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)
+{
+	iproc_msi_exit(pcie);
+}
+
 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 {
 	int ret;
@@ -340,6 +481,19 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 		goto err_exit_phy;
 	}
 
+	switch (pcie->type) {
+	case IPROC_PCIE_PAXB:
+		pcie->reg_offsets = iproc_pcie_reg_paxb;
+		break;
+	case IPROC_PCIE_PAXC:
+		pcie->reg_offsets = iproc_pcie_reg_paxc;
+		break;
+	default:
+		dev_err(pcie->dev, "incompatible iProc PCIe interface\n");
+		ret = -EINVAL;
+		goto err_power_off_phy;
+	}
+
 	iproc_pcie_reset(pcie);
 
 	if (pcie->need_ob_cfg) {
@@ -373,6 +527,10 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 
 	iproc_pcie_enable(pcie);
 
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		if (iproc_pcie_msi_enable(pcie))
+			dev_info(pcie->dev, "not using iProc MSI\n");
+
 	pci_scan_child_bus(bus);
 	pci_assign_unassigned_bus_resources(bus);
 	pci_fixup_irqs(pci_common_swizzle, pcie->map_irq);
@@ -397,6 +555,8 @@ int iproc_pcie_remove(struct iproc_pcie *pcie)
 	pci_stop_root_bus(pcie->root_bus);
 	pci_remove_root_bus(pcie->root_bus);
 
+	iproc_pcie_msi_disable(pcie);
+
 	phy_power_off(pcie->phy);
 	phy_exit(pcie->phy);
 
diff --git a/drivers/pci/host/pcie-iproc.h b/drivers/pci/host/pcie-iproc.h
index d3dc940f773a..e84d93c53c7b 100644
--- a/drivers/pci/host/pcie-iproc.h
+++ b/drivers/pci/host/pcie-iproc.h
@@ -15,6 +15,20 @@
 #define _PCIE_IPROC_H
 
 /**
+ * iProc PCIe interface type
+ *
+ * PAXB is the wrapper used in root complex that can be connected to an
+ * external endpoint device.
+ *
+ * PAXC is the wrapper used in root complex dedicated for internal emulated
+ * endpoint devices.
+ */
+enum iproc_pcie_type {
+	IPROC_PCIE_PAXB = 0,
+	IPROC_PCIE_PAXC,
+};
+
+/**
  * iProc PCIe outbound mapping
  * @set_oarr_size: indicates the OARR size bit needs to be set
  * @axi_offset: offset from the AXI address to the internal address used by
@@ -27,21 +41,30 @@ struct iproc_pcie_ob {
 	resource_size_t window_size;
 };
 
+struct iproc_msi;
+
 /**
  * iProc PCIe device
+ *
  * @dev: pointer to device data structure
+ * @type: iProc PCIe interface type
+ * @reg_offsets: register offsets
  * @base: PCIe host controller I/O register base
+ * @base_addr: PCIe host controller register base physical address
  * @sysdata: Per PCI controller data (ARM-specific)
  * @root_bus: pointer to root bus
  * @phy: optional PHY device that controls the Serdes
- * @irqs: interrupt IDs
  * @map_irq: function callback to map interrupts
- * @need_ob_cfg: indidates SW needs to configure the outbound mapping window
+ * @need_ob_cfg: indicates SW needs to configure the outbound mapping window
  * @ob: outbound mapping parameters
+ * @msi: MSI data
  */
 struct iproc_pcie {
 	struct device *dev;
+	enum iproc_pcie_type type;
+	const u16 *reg_offsets;
 	void __iomem *base;
+	phys_addr_t base_addr;
 #ifdef CONFIG_ARM
 	struct pci_sys_data sysdata;
 #endif
@@ -50,9 +73,24 @@ struct iproc_pcie {
 	int (*map_irq)(const struct pci_dev *, u8, u8);
 	bool need_ob_cfg;
 	struct iproc_pcie_ob ob;
+	struct iproc_msi *msi;
 };
 
 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res);
 int iproc_pcie_remove(struct iproc_pcie *pcie);
 
+#ifdef CONFIG_PCIE_IPROC_MSI
+int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node);
+void iproc_msi_exit(struct iproc_pcie *pcie);
+#else
+static inline int iproc_msi_init(struct iproc_pcie *pcie,
+				 struct device_node *node)
+{
+	return -ENODEV;
+}
+static inline void iproc_msi_exit(struct iproc_pcie *pcie)
+{
+}
+#endif
+
 #endif /* _PCIE_IPROC_H */
diff --git a/drivers/pci/host/pcie-qcom.c b/drivers/pci/host/pcie-qcom.c
new file mode 100644
index 000000000000..e845fba19632
--- /dev/null
+++ b/drivers/pci/host/pcie-qcom.c
@@ -0,0 +1,616 @@
+/*
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
+ * Copyright 2015 Linaro Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "pcie-designware.h"
+
+#define PCIE20_PARF_PHY_CTRL			0x40
+#define PCIE20_PARF_PHY_REFCLK			0x4C
+#define PCIE20_PARF_DBI_BASE_ADDR		0x168
+#define PCIE20_PARF_SLV_ADDR_SPACE_SIZE		0x16c
+#define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT	0x178
+
+#define PCIE20_ELBI_SYS_CTRL			0x04
+#define PCIE20_ELBI_SYS_CTRL_LT_ENABLE		BIT(0)
+
+#define PCIE20_CAP				0x70
+
+#define PERST_DELAY_US				1000
+
+struct qcom_pcie_resources_v0 {
+	struct clk *iface_clk;
+	struct clk *core_clk;
+	struct clk *phy_clk;
+	struct reset_control *pci_reset;
+	struct reset_control *axi_reset;
+	struct reset_control *ahb_reset;
+	struct reset_control *por_reset;
+	struct reset_control *phy_reset;
+	struct regulator *vdda;
+	struct regulator *vdda_phy;
+	struct regulator *vdda_refclk;
+};
+
+struct qcom_pcie_resources_v1 {
+	struct clk *iface;
+	struct clk *aux;
+	struct clk *master_bus;
+	struct clk *slave_bus;
+	struct reset_control *core;
+	struct regulator *vdda;
+};
+
+union qcom_pcie_resources {
+	struct qcom_pcie_resources_v0 v0;
+	struct qcom_pcie_resources_v1 v1;
+};
+
+struct qcom_pcie;
+
+struct qcom_pcie_ops {
+	int (*get_resources)(struct qcom_pcie *pcie);
+	int (*init)(struct qcom_pcie *pcie);
+	void (*deinit)(struct qcom_pcie *pcie);
+};
+
+struct qcom_pcie {
+	struct pcie_port pp;
+	struct device *dev;
+	union qcom_pcie_resources res;
+	void __iomem *parf;
+	void __iomem *dbi;
+	void __iomem *elbi;
+	struct phy *phy;
+	struct gpio_desc *reset;
+	struct qcom_pcie_ops *ops;
+};
+
+#define to_qcom_pcie(x)		container_of(x, struct qcom_pcie, pp)
+
+static void qcom_ep_reset_assert(struct qcom_pcie *pcie)
+{
+	gpiod_set_value(pcie->reset, 1);
+	usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
+}
+
+static void qcom_ep_reset_deassert(struct qcom_pcie *pcie)
+{
+	gpiod_set_value(pcie->reset, 0);
+	usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
+}
+
+static irqreturn_t qcom_pcie_msi_irq_handler(int irq, void *arg)
+{
+	struct pcie_port *pp = arg;
+
+	return dw_handle_msi_irq(pp);
+}
+
+static int qcom_pcie_establish_link(struct qcom_pcie *pcie)
+{
+	struct device *dev = pcie->dev;
+	unsigned int retries = 0;
+	u32 val;
+
+	if (dw_pcie_link_up(&pcie->pp))
+		return 0;
+
+	/* enable link training */
+	val = readl(pcie->elbi + PCIE20_ELBI_SYS_CTRL);
+	val |= PCIE20_ELBI_SYS_CTRL_LT_ENABLE;
+	writel(val, pcie->elbi + PCIE20_ELBI_SYS_CTRL);
+
+	do {
+		if (dw_pcie_link_up(&pcie->pp))
+			return 0;
+		usleep_range(250, 1000);
+	} while (retries < 200);
+
+	dev_warn(dev, "phy link never came up\n");
+
+	return -ETIMEDOUT;
+}
+
+static int qcom_pcie_get_resources_v0(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+	struct device *dev = pcie->dev;
+
+	res->vdda = devm_regulator_get(dev, "vdda");
+	if (IS_ERR(res->vdda))
+		return PTR_ERR(res->vdda);
+
+	res->vdda_phy = devm_regulator_get(dev, "vdda_phy");
+	if (IS_ERR(res->vdda_phy))
+		return PTR_ERR(res->vdda_phy);
+
+	res->vdda_refclk = devm_regulator_get(dev, "vdda_refclk");
+	if (IS_ERR(res->vdda_refclk))
+		return PTR_ERR(res->vdda_refclk);
+
+	res->iface_clk = devm_clk_get(dev, "iface");
+	if (IS_ERR(res->iface_clk))
+		return PTR_ERR(res->iface_clk);
+
+	res->core_clk = devm_clk_get(dev, "core");
+	if (IS_ERR(res->core_clk))
+		return PTR_ERR(res->core_clk);
+
+	res->phy_clk = devm_clk_get(dev, "phy");
+	if (IS_ERR(res->phy_clk))
+		return PTR_ERR(res->phy_clk);
+
+	res->pci_reset = devm_reset_control_get(dev, "pci");
+	if (IS_ERR(res->pci_reset))
+		return PTR_ERR(res->pci_reset);
+
+	res->axi_reset = devm_reset_control_get(dev, "axi");
+	if (IS_ERR(res->axi_reset))
+		return PTR_ERR(res->axi_reset);
+
+	res->ahb_reset = devm_reset_control_get(dev, "ahb");
+	if (IS_ERR(res->ahb_reset))
+		return PTR_ERR(res->ahb_reset);
+
+	res->por_reset = devm_reset_control_get(dev, "por");
+	if (IS_ERR(res->por_reset))
+		return PTR_ERR(res->por_reset);
+
+	res->phy_reset = devm_reset_control_get(dev, "phy");
+	if (IS_ERR(res->phy_reset))
+		return PTR_ERR(res->phy_reset);
+
+	return 0;
+}
+
+static int qcom_pcie_get_resources_v1(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+	struct device *dev = pcie->dev;
+
+	res->vdda = devm_regulator_get(dev, "vdda");
+	if (IS_ERR(res->vdda))
+		return PTR_ERR(res->vdda);
+
+	res->iface = devm_clk_get(dev, "iface");
+	if (IS_ERR(res->iface))
+		return PTR_ERR(res->iface);
+
+	res->aux = devm_clk_get(dev, "aux");
+	if (IS_ERR(res->aux))
+		return PTR_ERR(res->aux);
+
+	res->master_bus = devm_clk_get(dev, "master_bus");
+	if (IS_ERR(res->master_bus))
+		return PTR_ERR(res->master_bus);
+
+	res->slave_bus = devm_clk_get(dev, "slave_bus");
+	if (IS_ERR(res->slave_bus))
+		return PTR_ERR(res->slave_bus);
+
+	res->core = devm_reset_control_get(dev, "core");
+	if (IS_ERR(res->core))
+		return PTR_ERR(res->core);
+
+	return 0;
+}
+
+static void qcom_pcie_deinit_v0(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+
+	reset_control_assert(res->pci_reset);
+	reset_control_assert(res->axi_reset);
+	reset_control_assert(res->ahb_reset);
+	reset_control_assert(res->por_reset);
+	reset_control_assert(res->pci_reset);
+	clk_disable_unprepare(res->iface_clk);
+	clk_disable_unprepare(res->core_clk);
+	clk_disable_unprepare(res->phy_clk);
+	regulator_disable(res->vdda);
+	regulator_disable(res->vdda_phy);
+	regulator_disable(res->vdda_refclk);
+}
+
+static int qcom_pcie_init_v0(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+	struct device *dev = pcie->dev;
+	u32 val;
+	int ret;
+
+	ret = regulator_enable(res->vdda);
+	if (ret) {
+		dev_err(dev, "cannot enable vdda regulator\n");
+		return ret;
+	}
+
+	ret = regulator_enable(res->vdda_refclk);
+	if (ret) {
+		dev_err(dev, "cannot enable vdda_refclk regulator\n");
+		goto err_refclk;
+	}
+
+	ret = regulator_enable(res->vdda_phy);
+	if (ret) {
+		dev_err(dev, "cannot enable vdda_phy regulator\n");
+		goto err_vdda_phy;
+	}
+
+	ret = reset_control_assert(res->ahb_reset);
+	if (ret) {
+		dev_err(dev, "cannot assert ahb reset\n");
+		goto err_assert_ahb;
+	}
+
+	ret = clk_prepare_enable(res->iface_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable iface clock\n");
+		goto err_assert_ahb;
+	}
+
+	ret = clk_prepare_enable(res->phy_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable phy clock\n");
+		goto err_clk_phy;
+	}
+
+	ret = clk_prepare_enable(res->core_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable core clock\n");
+		goto err_clk_core;
+	}
+
+	ret = reset_control_deassert(res->ahb_reset);
+	if (ret) {
+		dev_err(dev, "cannot deassert ahb reset\n");
+		goto err_deassert_ahb;
+	}
+
+	/* enable PCIe clocks and resets */
+	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val &= ~BIT(0);
+	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+
+	/* enable external reference clock */
+	val = readl(pcie->parf + PCIE20_PARF_PHY_REFCLK);
+	val |= BIT(16);
+	writel(val, pcie->parf + PCIE20_PARF_PHY_REFCLK);
+
+	ret = reset_control_deassert(res->phy_reset);
+	if (ret) {
+		dev_err(dev, "cannot deassert phy reset\n");
+		return ret;
+	}
+
+	ret = reset_control_deassert(res->pci_reset);
+	if (ret) {
+		dev_err(dev, "cannot deassert pci reset\n");
+		return ret;
+	}
+
+	ret = reset_control_deassert(res->por_reset);
+	if (ret) {
+		dev_err(dev, "cannot deassert por reset\n");
+		return ret;
+	}
+
+	ret = reset_control_deassert(res->axi_reset);
+	if (ret) {
+		dev_err(dev, "cannot deassert axi reset\n");
+		return ret;
+	}
+
+	/* wait for clock acquisition */
+	usleep_range(1000, 1500);
+
+	return 0;
+
+err_deassert_ahb:
+	clk_disable_unprepare(res->core_clk);
+err_clk_core:
+	clk_disable_unprepare(res->phy_clk);
+err_clk_phy:
+	clk_disable_unprepare(res->iface_clk);
+err_assert_ahb:
+	regulator_disable(res->vdda_phy);
+err_vdda_phy:
+	regulator_disable(res->vdda_refclk);
+err_refclk:
+	regulator_disable(res->vdda);
+
+	return ret;
+}
+
+static void qcom_pcie_deinit_v1(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+
+	reset_control_assert(res->core);
+	clk_disable_unprepare(res->slave_bus);
+	clk_disable_unprepare(res->master_bus);
+	clk_disable_unprepare(res->iface);
+	clk_disable_unprepare(res->aux);
+	regulator_disable(res->vdda);
+}
+
+static int qcom_pcie_init_v1(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+	struct device *dev = pcie->dev;
+	int ret;
+
+	ret = reset_control_deassert(res->core);
+	if (ret) {
+		dev_err(dev, "cannot deassert core reset\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(res->aux);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable aux clock\n");
+		goto err_res;
+	}
+
+	ret = clk_prepare_enable(res->iface);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable iface clock\n");
+		goto err_aux;
+	}
+
+	ret = clk_prepare_enable(res->master_bus);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable master_bus clock\n");
+		goto err_iface;
+	}
+
+	ret = clk_prepare_enable(res->slave_bus);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable slave_bus clock\n");
+		goto err_master;
+	}
+
+	ret = regulator_enable(res->vdda);
+	if (ret) {
+		dev_err(dev, "cannot enable vdda regulator\n");
+		goto err_slave;
+	}
+
+	/* change DBI base address */
+	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		u32 val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
+
+		val |= BIT(31);
+		writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
+	}
+
+	return 0;
+err_slave:
+	clk_disable_unprepare(res->slave_bus);
+err_master:
+	clk_disable_unprepare(res->master_bus);
+err_iface:
+	clk_disable_unprepare(res->iface);
+err_aux:
+	clk_disable_unprepare(res->aux);
+err_res:
+	reset_control_assert(res->core);
+
+	return ret;
+}
+
+static int qcom_pcie_link_up(struct pcie_port *pp)
+{
+	struct qcom_pcie *pcie = to_qcom_pcie(pp);
+	u16 val = readw(pcie->dbi + PCIE20_CAP + PCI_EXP_LNKSTA);
+
+	return !!(val & PCI_EXP_LNKSTA_DLLLA);
+}
+
+static void qcom_pcie_host_init(struct pcie_port *pp)
+{
+	struct qcom_pcie *pcie = to_qcom_pcie(pp);
+	int ret;
+
+	qcom_ep_reset_assert(pcie);
+
+	ret = pcie->ops->init(pcie);
+	if (ret)
+		goto err_deinit;
+
+	ret = phy_power_on(pcie->phy);
+	if (ret)
+		goto err_deinit;
+
+	dw_pcie_setup_rc(pp);
+
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		dw_pcie_msi_init(pp);
+
+	qcom_ep_reset_deassert(pcie);
+
+	ret = qcom_pcie_establish_link(pcie);
+	if (ret)
+		goto err;
+
+	return;
+err:
+	qcom_ep_reset_assert(pcie);
+	phy_power_off(pcie->phy);
+err_deinit:
+	pcie->ops->deinit(pcie);
+}
+
+static int qcom_pcie_rd_own_conf(struct pcie_port *pp, int where, int size,
+				 u32 *val)
+{
+	/* the device class is not reported correctly from the register */
+	if (where == PCI_CLASS_REVISION && size == 4) {
+		*val = readl(pp->dbi_base + PCI_CLASS_REVISION);
+		*val &= 0xff;	/* keep revision id */
+		*val |= PCI_CLASS_BRIDGE_PCI << 16;
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	return dw_pcie_cfg_read(pp->dbi_base + where, size, val);
+}
+
+static struct pcie_host_ops qcom_pcie_dw_ops = {
+	.link_up = qcom_pcie_link_up,
+	.host_init = qcom_pcie_host_init,
+	.rd_own_conf = qcom_pcie_rd_own_conf,
+};
+
+static const struct qcom_pcie_ops ops_v0 = {
+	.get_resources = qcom_pcie_get_resources_v0,
+	.init = qcom_pcie_init_v0,
+	.deinit = qcom_pcie_deinit_v0,
+};
+
+static const struct qcom_pcie_ops ops_v1 = {
+	.get_resources = qcom_pcie_get_resources_v1,
+	.init = qcom_pcie_init_v1,
+	.deinit = qcom_pcie_deinit_v1,
+};
+
+static int qcom_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct qcom_pcie *pcie;
+	struct pcie_port *pp;
+	int ret;
+
+	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+
+	pcie->ops = (struct qcom_pcie_ops *)of_device_get_match_data(dev);
+	pcie->dev = dev;
+
+	pcie->reset = devm_gpiod_get_optional(dev, "perst", GPIOD_OUT_LOW);
+	if (IS_ERR(pcie->reset))
+		return PTR_ERR(pcie->reset);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "parf");
+	pcie->parf = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pcie->parf))
+		return PTR_ERR(pcie->parf);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
+	pcie->dbi = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pcie->dbi))
+		return PTR_ERR(pcie->dbi);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi");
+	pcie->elbi = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pcie->elbi))
+		return PTR_ERR(pcie->elbi);
+
+	pcie->phy = devm_phy_optional_get(dev, "pciephy");
+	if (IS_ERR(pcie->phy))
+		return PTR_ERR(pcie->phy);
+
+	ret = pcie->ops->get_resources(pcie);
+	if (ret)
+		return ret;
+
+	pp = &pcie->pp;
+	pp->dev = dev;
+	pp->dbi_base = pcie->dbi;
+	pp->root_bus_nr = -1;
+	pp->ops = &qcom_pcie_dw_ops;
+
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		pp->msi_irq = platform_get_irq_byname(pdev, "msi");
+		if (pp->msi_irq < 0)
+			return pp->msi_irq;
+
+		ret = devm_request_irq(dev, pp->msi_irq,
+				       qcom_pcie_msi_irq_handler,
+				       IRQF_SHARED, "qcom-pcie-msi", pp);
+		if (ret) {
+			dev_err(dev, "cannot request msi irq\n");
+			return ret;
+		}
+	}
+
+	ret = phy_init(pcie->phy);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_host_init(pp);
+	if (ret) {
+		dev_err(dev, "cannot initialize host\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, pcie);
+
+	return 0;
+}
+
+static int qcom_pcie_remove(struct platform_device *pdev)
+{
+	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
+
+	qcom_ep_reset_assert(pcie);
+	phy_power_off(pcie->phy);
+	phy_exit(pcie->phy);
+	pcie->ops->deinit(pcie);
+
+	return 0;
+}
+
+static const struct of_device_id qcom_pcie_match[] = {
+	{ .compatible = "qcom,pcie-ipq8064", .data = &ops_v0 },
+	{ .compatible = "qcom,pcie-apq8064", .data = &ops_v0 },
+	{ .compatible = "qcom,pcie-apq8084", .data = &ops_v1 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qcom_pcie_match);
+
+static struct platform_driver qcom_pcie_driver = {
+	.probe = qcom_pcie_probe,
+	.remove = qcom_pcie_remove,
+	.driver = {
+		.name = "qcom-pcie",
+		.of_match_table = qcom_pcie_match,
+	},
+};
+
+module_platform_driver(qcom_pcie_driver);
+
+MODULE_AUTHOR("Stanimir Varbanov <svarbanov@mm-sol.com>");
+MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
index f4fa6c537448..4edb5181f4e2 100644
--- a/drivers/pci/host/pcie-rcar.c
+++ b/drivers/pci/host/pcie-rcar.c
@@ -26,6 +26,7 @@
 #include <linux/of_platform.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
 #define DRV_NAME "rcar-pcie"
@@ -94,6 +95,11 @@
 #define H1_PCIEPHYDOUTR		0x040014
 #define H1_PCIEPHYSR		0x040018
 
+/* R-Car Gen2 PHY */
+#define GEN2_PCIEPHYADDR	0x780
+#define GEN2_PCIEPHYDATA	0x784
+#define GEN2_PCIEPHYCTRL	0x78c
+
 #define INT_PCI_MSI_NR	32
 
 #define RCONF(x)	(PCICONF(0)+(x))
@@ -108,8 +114,6 @@
 #define RCAR_PCI_MAX_RESOURCES 4
 #define MAX_NR_INBOUND_MAPS 6
 
-static unsigned long global_io_offset;
-
 struct rcar_msi {
 	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
 	struct irq_domain *domain;
@@ -126,20 +130,10 @@ static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
 }
 
 /* Structure representing the PCIe interface */
-/*
- * ARM pcibios functions expect the ARM struct pci_sys_data as the PCI
- * sysdata.  Add pci_sys_data as the first element in struct gen_pci so
- * that when we use a gen_pci pointer as sysdata, it is also a pointer to
- * a struct pci_sys_data.
- */
 struct rcar_pcie {
-#ifdef CONFIG_ARM
-	struct pci_sys_data	sys;
-#endif
 	struct device		*dev;
 	void __iomem		*base;
-	struct resource		res[RCAR_PCI_MAX_RESOURCES];
-	struct resource		busn;
+	struct list_head	resources;
 	int			root_bus_nr;
 	struct clk		*clk;
 	struct clk		*bus_clk;
@@ -323,10 +317,9 @@ static struct pci_ops rcar_pcie_ops = {
 	.write	= rcar_pcie_write_conf,
 };
 
-static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie)
+static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
+				   struct resource *res)
 {
-	struct resource *res = &pcie->res[win];
-
 	/* Setup PCIe address space mappings for each resource */
 	resource_size_t size;
 	resource_size_t res_start;
@@ -359,31 +352,33 @@ static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie)
 	rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win));
 }
 
-static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pcie)
+static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pci)
 {
-	struct resource *res;
-	int i;
-
-	pcie->root_bus_nr = pcie->busn.start;
+	struct resource_entry *win;
+	int i = 0;
 
 	/* Setup PCI resources */
-	for (i = 0; i < RCAR_PCI_MAX_RESOURCES; i++) {
+	resource_list_for_each_entry(win, &pci->resources) {
+		struct resource *res = win->res;
 
-		res = &pcie->res[i];
 		if (!res->flags)
 			continue;
 
-		rcar_pcie_setup_window(i, pcie);
-
-		if (res->flags & IORESOURCE_IO) {
-			phys_addr_t io_start = pci_pio_to_address(res->start);
-			pci_ioremap_io(global_io_offset, io_start);
-			global_io_offset += SZ_64K;
+		switch (resource_type(res)) {
+		case IORESOURCE_IO:
+		case IORESOURCE_MEM:
+			rcar_pcie_setup_window(i, pci, res);
+			i++;
+			break;
+		case IORESOURCE_BUS:
+			pci->root_bus_nr = res->start;
+			break;
+		default:
+			continue;
 		}
 
 		pci_add_resource(resource, res);
 	}
-	pci_add_resource(resource, &pcie->busn);
 
 	return 1;
 }
@@ -578,6 +573,26 @@ static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
 	return -ETIMEDOUT;
 }
 
+static int rcar_pcie_hw_init_gen2(struct rcar_pcie *pcie)
+{
+	/*
+	 * These settings come from the R-Car Series, 2nd Generation User's
+	 * Manual, section 50.3.1 (2) Initialization of the physical layer.
+	 */
+	rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR);
+	rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA);
+	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+	rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR);
+	/* The following value is for DC connection, no termination resistor */
+	rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA);
+	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+	return rcar_pcie_hw_init(pcie);
+}
+
 static int rcar_msi_alloc(struct rcar_msi *chip)
 {
 	int msi;
@@ -720,14 +735,16 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
 
 	/* Two irqs are for MSI, but they are also used for non-MSI irqs */
 	err = devm_request_irq(&pdev->dev, msi->irq1, rcar_pcie_msi_irq,
-			       IRQF_SHARED, rcar_msi_irq_chip.name, pcie);
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       rcar_msi_irq_chip.name, pcie);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
 		goto err;
 	}
 
 	err = devm_request_irq(&pdev->dev, msi->irq2, rcar_pcie_msi_irq,
-			       IRQF_SHARED, rcar_msi_irq_chip.name, pcie);
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       rcar_msi_irq_chip.name, pcie);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
 		goto err;
@@ -917,20 +934,71 @@ static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
 
 static const struct of_device_id rcar_pcie_of_match[] = {
 	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
-	{ .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init },
-	{ .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init },
+	{ .compatible = "renesas,pcie-rcar-gen2", .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7795", .data = rcar_pcie_hw_init },
 	{},
 };
 MODULE_DEVICE_TABLE(of, rcar_pcie_of_match);
 
+static void rcar_pcie_release_of_pci_ranges(struct rcar_pcie *pci)
+{
+	pci_free_resource_list(&pci->resources);
+}
+
+static int rcar_pcie_parse_request_of_pci_ranges(struct rcar_pcie *pci)
+{
+	int err;
+	struct device *dev = pci->dev;
+	struct device_node *np = dev->of_node;
+	resource_size_t iobase;
+	struct resource_entry *win;
+
+	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources, &iobase);
+	if (err)
+		return err;
+
+	resource_list_for_each_entry(win, &pci->resources) {
+		struct resource *parent, *res = win->res;
+
+		switch (resource_type(res)) {
+		case IORESOURCE_IO:
+			parent = &ioport_resource;
+			err = pci_remap_iospace(res, iobase);
+			if (err) {
+				dev_warn(dev, "error %d: failed to map resource %pR\n",
+					 err, res);
+				continue;
+			}
+			break;
+		case IORESOURCE_MEM:
+			parent = &iomem_resource;
+			break;
+
+		case IORESOURCE_BUS:
+		default:
+			continue;
+		}
+
+		err = devm_request_resource(dev, parent, res);
+		if (err)
+			goto out_release_res;
+	}
+
+	return 0;
+
+out_release_res:
+	rcar_pcie_release_of_pci_ranges(pci);
+	return err;
+}
+
 static int rcar_pcie_probe(struct platform_device *pdev)
 {
 	struct rcar_pcie *pcie;
 	unsigned int data;
-	struct of_pci_range range;
-	struct of_pci_range_parser parser;
 	const struct of_device_id *of_id;
-	int err, win = 0;
+	int err;
 	int (*hw_init_fn)(struct rcar_pcie *);
 
 	pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
@@ -940,16 +1008,9 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 	pcie->dev = &pdev->dev;
 	platform_set_drvdata(pdev, pcie);
 
-	/* Get the bus range */
-	if (of_pci_parse_bus_range(pdev->dev.of_node, &pcie->busn)) {
-		dev_err(&pdev->dev, "failed to parse bus-range property\n");
-		return -EINVAL;
-	}
+	INIT_LIST_HEAD(&pcie->resources);
 
-	if (of_pci_range_parser_init(&parser, pdev->dev.of_node)) {
-		dev_err(&pdev->dev, "missing ranges property\n");
-		return -EINVAL;
-	}
+	rcar_pcie_parse_request_of_pci_ranges(pcie);
 
 	err = rcar_pcie_get_resources(pdev, pcie);
 	if (err < 0) {
@@ -957,46 +1018,55 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	for_each_of_pci_range(&parser, &range) {
-		err = of_pci_range_to_resource(&range, pdev->dev.of_node,
-						&pcie->res[win++]);
-		if (err < 0)
-			return err;
-
-		if (win > RCAR_PCI_MAX_RESOURCES)
-			break;
-	}
-
 	 err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node);
 	 if (err)
 		return err;
 
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		err = rcar_pcie_enable_msi(pcie);
-		if (err < 0) {
-			dev_err(&pdev->dev,
-				"failed to enable MSI support: %d\n",
-				err);
-			return err;
-		}
-	}
-
 	of_id = of_match_device(rcar_pcie_of_match, pcie->dev);
 	if (!of_id || !of_id->data)
 		return -EINVAL;
 	hw_init_fn = of_id->data;
 
+	pm_runtime_enable(pcie->dev);
+	err = pm_runtime_get_sync(pcie->dev);
+	if (err < 0) {
+		dev_err(pcie->dev, "pm_runtime_get_sync failed\n");
+		goto err_pm_disable;
+	}
+
 	/* Failure to get a link might just be that no cards are inserted */
 	err = hw_init_fn(pcie);
 	if (err) {
 		dev_info(&pdev->dev, "PCIe link down\n");
-		return 0;
+		err = 0;
+		goto err_pm_put;
 	}
 
 	data = rcar_pci_read_reg(pcie, MACSR);
 	dev_info(&pdev->dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
 
-	return rcar_pcie_enable(pcie);
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		err = rcar_pcie_enable_msi(pcie);
+		if (err < 0) {
+			dev_err(&pdev->dev,
+				"failed to enable MSI support: %d\n",
+				err);
+			goto err_pm_put;
+		}
+	}
+
+	err = rcar_pcie_enable(pcie);
+	if (err)
+		goto err_pm_put;
+
+	return 0;
+
+err_pm_put:
+	pm_runtime_put(pcie->dev);
+
+err_pm_disable:
+	pm_runtime_disable(pcie->dev);
+	return err;
 }
 
 static struct platform_driver rcar_pcie_driver = {
diff --git a/drivers/pci/host/pcie-spear13xx.c b/drivers/pci/host/pcie-spear13xx.c
index b95b7563c052..a6cd8233e8c0 100644
--- a/drivers/pci/host/pcie-spear13xx.c
+++ b/drivers/pci/host/pcie-spear13xx.c
@@ -279,7 +279,8 @@ static int spear13xx_add_pcie_port(struct pcie_port *pp,
 		return -ENODEV;
 	}
 	ret = devm_request_irq(dev, pp->irq, spear13xx_pcie_irq_handler,
-			       IRQF_SHARED, "spear1340-pcie", pp);
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       "spear1340-pcie", pp);
 	if (ret) {
 		dev_err(dev, "failed to request irq %d\n", pp->irq);
 		return ret;
diff --git a/drivers/pci/host/pcie-xilinx.c b/drivers/pci/host/pcie-xilinx.c
index 3c7a0d580b1e..4cfa46360d12 100644
--- a/drivers/pci/host/pcie-xilinx.c
+++ b/drivers/pci/host/pcie-xilinx.c
@@ -781,7 +781,8 @@ static int xilinx_pcie_parse_dt(struct xilinx_pcie_port *port)
 
 	port->irq = irq_of_parse_and_map(node, 0);
 	err = devm_request_irq(dev, port->irq, xilinx_pcie_intr_handler,
-			       IRQF_SHARED, "xilinx-pcie", port);
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       "xilinx-pcie", port);
 	if (err) {
 		dev_err(dev, "unable to request irq %d\n", port->irq);
 		return err;
diff --git a/drivers/pci/hotplug/acpi_pcihp.c b/drivers/pci/hotplug/acpi_pcihp.c
index 876ccc620440..a5e66df4ad14 100644
--- a/drivers/pci/hotplug/acpi_pcihp.c
+++ b/drivers/pci/hotplug/acpi_pcihp.c
@@ -36,10 +36,10 @@
 
 #define MY_NAME	"acpi_pcihp"
 
-#define dbg(fmt, arg...) do { if (debug_acpi) printk(KERN_DEBUG "%s: %s: " fmt , MY_NAME , __func__ , ## arg); } while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format , MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format , MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format , MY_NAME , ## arg)
+#define dbg(fmt, arg...) do { if (debug_acpi) printk(KERN_DEBUG "%s: %s: " fmt, MY_NAME, __func__, ## arg); } while (0)
+#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 #define	METHOD_NAME__SUN	"_SUN"
 #define	METHOD_NAME_OSHP	"OSHP"
@@ -132,7 +132,7 @@ int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev, u32 flags)
 
 	while (handle) {
 		acpi_get_name(handle, ACPI_FULL_PATHNAME, &string);
-		dbg("Trying to get hotplug control for %s \n",
+		dbg("Trying to get hotplug control for %s\n",
 		    (char *)string.pointer);
 		status = acpi_run_oshp(handle);
 		if (ACPI_SUCCESS(status))
diff --git a/drivers/pci/hotplug/acpiphp.h b/drivers/pci/hotplug/acpiphp.h
index b0e61bf261a7..f0ebc8b9a15a 100644
--- a/drivers/pci/hotplug/acpiphp.h
+++ b/drivers/pci/hotplug/acpiphp.h
@@ -181,7 +181,7 @@ struct acpiphp_attention_info
 /* function prototypes */
 
 /* acpiphp_core.c */
-int acpiphp_register_attention(struct acpiphp_attention_info*info);
+int acpiphp_register_attention(struct acpiphp_attention_info *info);
 int acpiphp_unregister_attention(struct acpiphp_attention_info *info);
 int acpiphp_register_hotplug_slot(struct acpiphp_slot *slot, unsigned int sun);
 void acpiphp_unregister_hotplug_slot(struct acpiphp_slot *slot);
diff --git a/drivers/pci/hotplug/acpiphp_core.c b/drivers/pci/hotplug/acpiphp_core.c
index e291efcd02a2..3c81fc8b0103 100644
--- a/drivers/pci/hotplug/acpiphp_core.c
+++ b/drivers/pci/hotplug/acpiphp_core.c
@@ -63,13 +63,13 @@ MODULE_LICENSE("GPL");
 MODULE_PARM_DESC(disable, "disable acpiphp driver");
 module_param_named(disable, acpiphp_disabled, bool, 0444);
 
-static int enable_slot		(struct hotplug_slot *slot);
-static int disable_slot		(struct hotplug_slot *slot);
-static int set_attention_status (struct hotplug_slot *slot, u8 value);
-static int get_power_status	(struct hotplug_slot *slot, u8 *value);
-static int get_attention_status (struct hotplug_slot *slot, u8 *value);
-static int get_latch_status	(struct hotplug_slot *slot, u8 *value);
-static int get_adapter_status	(struct hotplug_slot *slot, u8 *value);
+static int enable_slot(struct hotplug_slot *slot);
+static int disable_slot(struct hotplug_slot *slot);
+static int set_attention_status(struct hotplug_slot *slot, u8 value);
+static int get_power_status(struct hotplug_slot *slot, u8 *value);
+static int get_attention_status(struct hotplug_slot *slot, u8 *value);
+static int get_latch_status(struct hotplug_slot *slot, u8 *value);
+static int get_adapter_status(struct hotplug_slot *slot, u8 *value);
 
 static struct hotplug_slot_ops acpi_hotplug_slot_ops = {
 	.enable_slot		= enable_slot,
diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c
index ff538568a617..fa49f9143b80 100644
--- a/drivers/pci/hotplug/acpiphp_glue.c
+++ b/drivers/pci/hotplug/acpiphp_glue.c
@@ -707,7 +707,7 @@ static void acpiphp_sanitize_bus(struct pci_bus *bus)
 	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM;
 
 	list_for_each_entry_safe_reverse(dev, tmp, &bus->devices, bus_list) {
-		for (i=0; i<PCI_BRIDGE_RESOURCES; i++) {
+		for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) {
 			struct resource *res = &dev->resource[i];
 			if ((res->flags & type_mask) && !res->start &&
 					res->end) {
@@ -953,8 +953,10 @@ int acpiphp_enable_slot(struct acpiphp_slot *slot)
 {
 	pci_lock_rescan_remove();
 
-	if (slot->flags & SLOT_IS_GOING_AWAY)
+	if (slot->flags & SLOT_IS_GOING_AWAY) {
+		pci_unlock_rescan_remove();
 		return -ENODEV;
+	}
 
 	/* configure all functions */
 	if (!(slot->flags & SLOT_ENABLED))
diff --git a/drivers/pci/hotplug/acpiphp_ibm.c b/drivers/pci/hotplug/acpiphp_ibm.c
index 6ca23998ee8f..2f6d3a1c1726 100644
--- a/drivers/pci/hotplug/acpiphp_ibm.c
+++ b/drivers/pci/hotplug/acpiphp_ibm.c
@@ -154,7 +154,8 @@ static union apci_descriptor *ibm_slot_from_id(int id)
 ibm_slot_done:
 	if (ret) {
 		ret = kmalloc(sizeof(union apci_descriptor), GFP_KERNEL);
-		memcpy(ret, des, sizeof(union apci_descriptor));
+		if (ret)
+			memcpy(ret, des, sizeof(union apci_descriptor));
 	}
 	kfree(table);
 	return ret;
@@ -175,8 +176,13 @@ static int ibm_set_attention_status(struct hotplug_slot *slot, u8 status)
 	acpi_status stat;
 	unsigned long long rc;
 	union apci_descriptor *ibm_slot;
+	int id = hpslot_to_sun(slot);
 
-	ibm_slot = ibm_slot_from_id(hpslot_to_sun(slot));
+	ibm_slot = ibm_slot_from_id(id);
+	if (!ibm_slot) {
+		pr_err("APLS null ACPI descriptor for slot %d\n", id);
+		return -ENODEV;
+	}
 
 	pr_debug("%s: set slot %d (%d) attention status to %d\n", __func__,
 			ibm_slot->slot.slot_num, ibm_slot->slot.slot_id,
@@ -215,8 +221,13 @@ static int ibm_set_attention_status(struct hotplug_slot *slot, u8 status)
 static int ibm_get_attention_status(struct hotplug_slot *slot, u8 *status)
 {
 	union apci_descriptor *ibm_slot;
+	int id = hpslot_to_sun(slot);
 
-	ibm_slot = ibm_slot_from_id(hpslot_to_sun(slot));
+	ibm_slot = ibm_slot_from_id(id);
+	if (!ibm_slot) {
+		pr_err("APLS null ACPI descriptor for slot %d\n", id);
+		return -ENODEV;
+	}
 
 	if (ibm_slot->slot.attn & 0xa0 || ibm_slot->slot.status[1] & 0x08)
 		*status = 1;
@@ -325,7 +336,7 @@ static int ibm_get_table_from_acpi(char **bufp)
 	}
 
 	size = 0;
-	for (i=0; i<package->package.count; i++) {
+	for (i = 0; i < package->package.count; i++) {
 		memcpy(&lbuf[size],
 				package->package.elements[i].buffer.pointer,
 				package->package.elements[i].buffer.length);
diff --git a/drivers/pci/hotplug/cpci_hotplug.h b/drivers/pci/hotplug/cpci_hotplug.h
index 6a0ddf757349..555bcde3b196 100644
--- a/drivers/pci/hotplug/cpci_hotplug.h
+++ b/drivers/pci/hotplug/cpci_hotplug.h
@@ -52,13 +52,13 @@ struct slot {
 };
 
 struct cpci_hp_controller_ops {
-	int (*query_enum) (void);
-	int (*enable_irq) (void);
-	int (*disable_irq) (void);
-	int (*check_irq) (void *dev_id);
-	int (*hardware_test) (struct slot *slot, u32 value);
-	u8  (*get_power) (struct slot *slot);
-	int (*set_power) (struct slot *slot, int value);
+	int (*query_enum)(void);
+	int (*enable_irq)(void);
+	int (*disable_irq)(void);
+	int (*check_irq)(void *dev_id);
+	int (*hardware_test)(struct slot *slot, u32 value);
+	u8  (*get_power)(struct slot *slot);
+	int (*set_power)(struct slot *slot, int value);
 };
 
 struct cpci_hp_controller {
diff --git a/drivers/pci/hotplug/cpci_hotplug_core.c b/drivers/pci/hotplug/cpci_hotplug_core.c
index 46db29395a62..7d3866c47312 100644
--- a/drivers/pci/hotplug/cpci_hotplug_core.c
+++ b/drivers/pci/hotplug/cpci_hotplug_core.c
@@ -45,12 +45,12 @@
 #define dbg(format, arg...)					\
 	do {							\
 		if (cpci_debug)					\
-			printk (KERN_DEBUG "%s: " format "\n",	\
-				MY_NAME , ## arg);		\
+			printk(KERN_DEBUG "%s: " format "\n",	\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME, ## arg)
 
 /* local variables */
 static DECLARE_RWSEM(list_rwsem);
@@ -238,21 +238,21 @@ cpci_hp_register_bus(struct pci_bus *bus, u8 first, u8 last)
 	 * with the pci_hotplug subsystem.
 	 */
 	for (i = first; i <= last; ++i) {
-		slot = kzalloc(sizeof (struct slot), GFP_KERNEL);
+		slot = kzalloc(sizeof(struct slot), GFP_KERNEL);
 		if (!slot) {
 			status = -ENOMEM;
 			goto error;
 		}
 
 		hotplug_slot =
-			kzalloc(sizeof (struct hotplug_slot), GFP_KERNEL);
+			kzalloc(sizeof(struct hotplug_slot), GFP_KERNEL);
 		if (!hotplug_slot) {
 			status = -ENOMEM;
 			goto error_slot;
 		}
 		slot->hotplug_slot = hotplug_slot;
 
-		info = kzalloc(sizeof (struct hotplug_slot_info), GFP_KERNEL);
+		info = kzalloc(sizeof(struct hotplug_slot_info), GFP_KERNEL);
 		if (!info) {
 			status = -ENOMEM;
 			goto error_hpslot;
diff --git a/drivers/pci/hotplug/cpci_hotplug_pci.c b/drivers/pci/hotplug/cpci_hotplug_pci.c
index 788db48dbbad..80c80017197d 100644
--- a/drivers/pci/hotplug/cpci_hotplug_pci.c
+++ b/drivers/pci/hotplug/cpci_hotplug_pci.c
@@ -38,12 +38,12 @@ extern int cpci_debug;
 #define dbg(format, arg...)					\
 	do {							\
 		if (cpci_debug)					\
-			printk (KERN_DEBUG "%s: " format "\n",	\
-				MY_NAME , ## arg);		\
+			printk(KERN_DEBUG "%s: " format "\n",	\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME, ## arg)
 
 
 u8 cpci_get_attention_status(struct slot *slot)
diff --git a/drivers/pci/hotplug/cpcihp_generic.c b/drivers/pci/hotplug/cpcihp_generic.c
index 66b7bbebe493..88a44a707b96 100644
--- a/drivers/pci/hotplug/cpcihp_generic.c
+++ b/drivers/pci/hotplug/cpcihp_generic.c
@@ -54,12 +54,12 @@
 #define dbg(format, arg...)					\
 	do {							\
 		if (debug)					\
-			printk (KERN_DEBUG "%s: " format "\n",	\
-				MY_NAME , ## arg);		\
+			printk(KERN_DEBUG "%s: " format "\n",	\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME, ## arg)
 
 /* local variables */
 static bool debug;
@@ -164,7 +164,7 @@ static int __init cpcihp_generic_init(void)
 	bus = dev->subordinate;
 	pci_dev_put(dev);
 
-	memset(&generic_hpc, 0, sizeof (struct cpci_hp_controller));
+	memset(&generic_hpc, 0, sizeof(struct cpci_hp_controller));
 	generic_hpc_ops.query_enum = query_enum;
 	generic_hpc.ops = &generic_hpc_ops;
 
diff --git a/drivers/pci/hotplug/cpcihp_zt5550.c b/drivers/pci/hotplug/cpcihp_zt5550.c
index 7ecf34e76a61..5f49c3fd736a 100644
--- a/drivers/pci/hotplug/cpcihp_zt5550.c
+++ b/drivers/pci/hotplug/cpcihp_zt5550.c
@@ -49,12 +49,12 @@
 #define dbg(format, arg...)					\
 	do {							\
 		if (debug)					\
-			printk (KERN_DEBUG "%s: " format "\n",	\
-				MY_NAME , ## arg);		\
+			printk(KERN_DEBUG "%s: " format "\n",	\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME, ## arg)
 
 /* local variables */
 static bool debug;
@@ -204,7 +204,7 @@ static int zt5550_hc_disable_irq(void)
 	return 0;
 }
 
-static int zt5550_hc_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
+static int zt5550_hc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	int status;
 
@@ -214,7 +214,7 @@ static int zt5550_hc_init_one (struct pci_dev *pdev, const struct pci_device_id
 
 	dbg("returned from zt5550_hc_config");
 
-	memset(&zt5550_hpc, 0, sizeof (struct cpci_hp_controller));
+	memset(&zt5550_hpc, 0, sizeof(struct cpci_hp_controller));
 	zt5550_hpc_ops.query_enum = zt5550_hc_query_enum;
 	zt5550_hpc.ops = &zt5550_hpc_ops;
 	if (!poll) {
diff --git a/drivers/pci/hotplug/cpqphp.h b/drivers/pci/hotplug/cpqphp.h
index b28b2d2184cd..9103a7b9f3b9 100644
--- a/drivers/pci/hotplug/cpqphp.h
+++ b/drivers/pci/hotplug/cpqphp.h
@@ -36,10 +36,10 @@
 
 #define MY_NAME	"cpqphp"
 
-#define dbg(fmt, arg...) do { if (cpqhp_debug) printk(KERN_DEBUG "%s: " fmt , MY_NAME , ## arg); } while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format , MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format , MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format , MY_NAME , ## arg)
+#define dbg(fmt, arg...) do { if (cpqhp_debug) printk(KERN_DEBUG "%s: " fmt, MY_NAME, ## arg); } while (0)
+#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 
 
@@ -424,7 +424,7 @@ int cpqhp_process_SS(struct controller *ctrl, struct pci_func *func);
 int cpqhp_hardware_test(struct controller *ctrl, int test_num);
 
 /* resource functions */
-int	cpqhp_resource_sort_and_combine	(struct pci_resource **head);
+int	cpqhp_resource_sort_and_combine(struct pci_resource **head);
 
 /* pci functions */
 int cpqhp_set_irq(u8 bus_num, u8 dev_num, u8 int_pin, u8 irq_num);
@@ -685,7 +685,7 @@ static inline int cpq_get_latch_status(struct controller *ctrl,
 	u8 hp_slot;
 
 	hp_slot = slot->device - ctrl->slot_device_offset;
-	dbg("%s: slot->device = %d, ctrl->slot_device_offset = %d \n",
+	dbg("%s: slot->device = %d, ctrl->slot_device_offset = %d\n",
 	    __func__, slot->device, ctrl->slot_device_offset);
 
 	status = (readl(ctrl->hpc_reg + INT_INPUT_CLEAR) & (0x01L << hp_slot));
@@ -712,7 +712,7 @@ static inline int get_presence_status(struct controller *ctrl,
 
 static inline int wait_for_ctrl_irq(struct controller *ctrl)
 {
-        DECLARE_WAITQUEUE(wait, current);
+	DECLARE_WAITQUEUE(wait, current);
 	int retval = 0;
 
 	dbg("%s - start\n", __func__);
diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c
index a53084ddc118..74f3a0695b43 100644
--- a/drivers/pci/hotplug/cpqphp_core.c
+++ b/drivers/pci/hotplug/cpqphp_core.c
@@ -291,7 +291,7 @@ static void release_slot(struct hotplug_slot *hotplug_slot)
 	kfree(slot);
 }
 
-static int ctrl_slot_cleanup (struct controller *ctrl)
+static int ctrl_slot_cleanup(struct controller *ctrl)
 {
 	struct slot *old_slot, *next_slot;
 
@@ -301,7 +301,7 @@ static int ctrl_slot_cleanup (struct controller *ctrl)
 	while (old_slot) {
 		/* memory will be freed by the release_slot callback */
 		next_slot = old_slot->next;
-		pci_hp_deregister (old_slot->hotplug_slot);
+		pci_hp_deregister(old_slot->hotplug_slot);
 		old_slot = next_slot;
 	}
 
@@ -413,9 +413,9 @@ cpqhp_set_attention_status(struct controller *ctrl, struct pci_func *func,
 	mutex_lock(&ctrl->crit_sect);
 
 	if (status == 1)
-		amber_LED_on (ctrl, hp_slot);
+		amber_LED_on(ctrl, hp_slot);
 	else if (status == 0)
-		amber_LED_off (ctrl, hp_slot);
+		amber_LED_off(ctrl, hp_slot);
 	else {
 		/* Done with exclusive hardware access */
 		mutex_unlock(&ctrl->crit_sect);
@@ -425,7 +425,7 @@ cpqhp_set_attention_status(struct controller *ctrl, struct pci_func *func,
 	set_SOGO(ctrl);
 
 	/* Wait for SOBS to be unset */
-	wait_for_ctrl_irq (ctrl);
+	wait_for_ctrl_irq(ctrl);
 
 	/* Done with exclusive hardware access */
 	mutex_unlock(&ctrl->crit_sect);
@@ -439,7 +439,7 @@ cpqhp_set_attention_status(struct controller *ctrl, struct pci_func *func,
  * @hotplug_slot: slot to change LED on
  * @status: LED control flag
  */
-static int set_attention_status (struct hotplug_slot *hotplug_slot, u8 status)
+static int set_attention_status(struct hotplug_slot *hotplug_slot, u8 status)
 {
 	struct pci_func *slot_func;
 	struct slot *slot = hotplug_slot->private;
@@ -610,7 +610,7 @@ static int ctrl_slot_setup(struct controller *ctrl,
 	u8 ctrl_slot;
 	u32 tempdword;
 	char name[SLOT_NAME_SIZE];
-	void __iomem *slot_entry= NULL;
+	void __iomem *slot_entry = NULL;
 	int result;
 
 	dbg("%s\n", __func__);
@@ -755,7 +755,7 @@ static int one_time_init(void)
 	if (cpqhp_debug)
 		pci_print_IRQ_route();
 
-	dbg("Initialize + Start the notification mechanism \n");
+	dbg("Initialize + Start the notification mechanism\n");
 
 	retval = cpqhp_event_start_thread();
 	if (retval)
@@ -772,7 +772,7 @@ static int one_time_init(void)
 	/* Map rom address */
 	cpqhp_rom_start = ioremap(ROM_PHY_ADDR, ROM_PHY_LEN);
 	if (!cpqhp_rom_start) {
-		err ("Could not ioremap memory region for ROM\n");
+		err("Could not ioremap memory region for ROM\n");
 		retval = -EIO;
 		goto error;
 	}
@@ -786,7 +786,7 @@ static int one_time_init(void)
 	smbios_table = detect_SMBIOS_pointer(cpqhp_rom_start,
 					cpqhp_rom_start + ROM_PHY_LEN);
 	if (!smbios_table) {
-		err ("Could not find the SMBIOS pointer in memory\n");
+		err("Could not find the SMBIOS pointer in memory\n");
 		retval = -EIO;
 		goto error_rom_start;
 	}
@@ -794,7 +794,7 @@ static int one_time_init(void)
 	smbios_start = ioremap(readl(smbios_table + ST_ADDRESS),
 					readw(smbios_table + ST_LENGTH));
 	if (!smbios_start) {
-		err ("Could not ioremap memory region taken from SMBIOS values\n");
+		err("Could not ioremap memory region taken from SMBIOS values\n");
 		retval = -EIO;
 		goto error_smbios_start;
 	}
@@ -1181,7 +1181,7 @@ static int cpqhpc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * Finish setting up the hot plug ctrl device
 	 */
 	ctrl->slot_device_offset = readb(ctrl->hpc_reg + SLOT_MASK) >> 4;
-	dbg("NumSlots %d \n", ctrl->slot_device_offset);
+	dbg("NumSlots %d\n", ctrl->slot_device_offset);
 
 	ctrl->next_event = 0;
 
@@ -1198,7 +1198,7 @@ static int cpqhpc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	writel(0xFFFFFFFFL, ctrl->hpc_reg + INT_MASK);
 
 	/* set up the interrupt */
-	dbg("HPC interrupt = %d \n", ctrl->interrupt);
+	dbg("HPC interrupt = %d\n", ctrl->interrupt);
 	if (request_irq(ctrl->interrupt, cpqhp_ctrl_intr,
 			IRQF_SHARED, MY_NAME, ctrl)) {
 		err("Can't get irq %d for the hotplug pci controller\n",
@@ -1321,7 +1321,7 @@ static void __exit unload_cpqphpd(void)
 	while (ctrl) {
 		if (ctrl->hpc_reg) {
 			u16 misc;
-			rc = read_slot_enable (ctrl);
+			rc = read_slot_enable(ctrl);
 
 			writeb(0, ctrl->hpc_reg + SLOT_SERR);
 			writel(0xFFFFFFC0L | ~rc, ctrl->hpc_reg + INT_MASK);
@@ -1361,7 +1361,7 @@ static void __exit unload_cpqphpd(void)
 			kfree(tres);
 		}
 
-		kfree (ctrl->pci_bus);
+		kfree(ctrl->pci_bus);
 
 		tctrl = ctrl;
 		ctrl = ctrl->next;
@@ -1446,7 +1446,7 @@ static int __init cpqhpc_init(void)
 
 	cpqhp_debug = debug;
 
-	info (DRIVER_DESC " version: " DRIVER_VERSION "\n");
+	info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
 	cpqhp_initialize_debugfs();
 	result = pci_register_driver(&cpqhpc_driver);
 	dbg("pci_register_driver = %d\n", result);
diff --git a/drivers/pci/hotplug/cpqphp_ctrl.c b/drivers/pci/hotplug/cpqphp_ctrl.c
index c5cbefee5236..a55653b54eed 100644
--- a/drivers/pci/hotplug/cpqphp_ctrl.c
+++ b/drivers/pci/hotplug/cpqphp_ctrl.c
@@ -155,7 +155,7 @@ static u8 handle_presence_change(u16 change, struct controller *ctrl)
 	 * Presence Change
 	 */
 	dbg("cpqsbd:  Presence/Notify input change.\n");
-	dbg("         Changed bits are 0x%4.4x\n", change );
+	dbg("         Changed bits are 0x%4.4x\n", change);
 
 	for (hp_slot = 0; hp_slot < 6; hp_slot++) {
 		if (change & (0x0101 << hp_slot)) {
@@ -276,9 +276,9 @@ static u8 handle_power_fault(u8 change, struct controller *ctrl)
 				taskInfo->event_type = INT_POWER_FAULT;
 
 				if (ctrl->rev < 4) {
-					amber_LED_on (ctrl, hp_slot);
-					green_LED_off (ctrl, hp_slot);
-					set_SOGO (ctrl);
+					amber_LED_on(ctrl, hp_slot);
+					green_LED_off(ctrl, hp_slot);
+					set_SOGO(ctrl);
 
 					/* this is a fatal condition, we want
 					 * to crash the machine to protect from
@@ -438,7 +438,7 @@ static struct pci_resource *do_pre_bridge_resource_split(struct pci_resource **h
 
 	node = *head;
 
-	if (node->length & (alignment -1)) {
+	if (node->length & (alignment - 1)) {
 		/* this one isn't an aligned length, so we'll make a new entry
 		 * and split it up.
 		 */
@@ -835,13 +835,13 @@ int cpqhp_resource_sort_and_combine(struct pci_resource **head)
 	if (!(*head))
 		return 1;
 
-	dbg("*head->next = %p\n",(*head)->next);
+	dbg("*head->next = %p\n", (*head)->next);
 
 	if (!(*head)->next)
 		return 0;	/* only one item on the list, already sorted! */
 
-	dbg("*head->base = 0x%x\n",(*head)->base);
-	dbg("*head->next->base = 0x%x\n",(*head)->next->base);
+	dbg("*head->base = 0x%x\n", (*head)->base);
+	dbg("*head->next->base = 0x%x\n", (*head)->next->base);
 	while (out_of_order) {
 		out_of_order = 0;
 
@@ -917,7 +917,7 @@ irqreturn_t cpqhp_ctrl_intr(int IRQ, void *data)
 		/* Read to clear posted writes */
 		misc = readw(ctrl->hpc_reg + MISC);
 
-		dbg ("%s - waking up\n", __func__);
+		dbg("%s - waking up\n", __func__);
 		wake_up_interruptible(&ctrl->queue);
 	}
 
@@ -1285,18 +1285,18 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 	/*
 	 * The board is already on
 	 */
-	else if (is_slot_enabled (ctrl, hp_slot))
+	else if (is_slot_enabled(ctrl, hp_slot))
 		rc = CARD_FUNCTIONING;
 	else {
 		mutex_lock(&ctrl->crit_sect);
 
 		/* turn on board without attaching to the bus */
-		enable_slot_power (ctrl, hp_slot);
+		enable_slot_power(ctrl, hp_slot);
 
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		/* Change bits in slot power register to force another shift out
 		 * NOTE: this is to work around the timer bug */
@@ -1307,7 +1307,7 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		adapter_speed = get_adapter_speed(ctrl, hp_slot);
 		if (bus->cur_bus_speed != adapter_speed)
@@ -1315,12 +1315,12 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 				rc = WRONG_BUS_FREQUENCY;
 
 		/* turn off board without attaching to the bus */
-		disable_slot_power (ctrl, hp_slot);
+		disable_slot_power(ctrl, hp_slot);
 
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		mutex_unlock(&ctrl->crit_sect);
 
@@ -1329,15 +1329,15 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 
 		mutex_lock(&ctrl->crit_sect);
 
-		slot_enable (ctrl, hp_slot);
-		green_LED_blink (ctrl, hp_slot);
+		slot_enable(ctrl, hp_slot);
+		green_LED_blink(ctrl, hp_slot);
 
-		amber_LED_off (ctrl, hp_slot);
+		amber_LED_off(ctrl, hp_slot);
 
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		mutex_unlock(&ctrl->crit_sect);
 
@@ -1366,14 +1366,14 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 
 			mutex_lock(&ctrl->crit_sect);
 
-			amber_LED_on (ctrl, hp_slot);
-			green_LED_off (ctrl, hp_slot);
-			slot_disable (ctrl, hp_slot);
+			amber_LED_on(ctrl, hp_slot);
+			green_LED_off(ctrl, hp_slot);
+			slot_disable(ctrl, hp_slot);
 
 			set_SOGO(ctrl);
 
 			/* Wait for SOBS to be unset */
-			wait_for_ctrl_irq (ctrl);
+			wait_for_ctrl_irq(ctrl);
 
 			mutex_unlock(&ctrl->crit_sect);
 
@@ -1392,14 +1392,14 @@ static u32 board_replaced(struct pci_func *func, struct controller *ctrl)
 
 			mutex_lock(&ctrl->crit_sect);
 
-			amber_LED_on (ctrl, hp_slot);
-			green_LED_off (ctrl, hp_slot);
-			slot_disable (ctrl, hp_slot);
+			amber_LED_on(ctrl, hp_slot);
+			green_LED_off(ctrl, hp_slot);
+			slot_disable(ctrl, hp_slot);
 
 			set_SOGO(ctrl);
 
 			/* Wait for SOBS to be unset */
-			wait_for_ctrl_irq (ctrl);
+			wait_for_ctrl_irq(ctrl);
 
 			mutex_unlock(&ctrl->crit_sect);
 		}
@@ -1443,7 +1443,7 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 	set_SOGO(ctrl);
 
 	/* Wait for SOBS to be unset */
-	wait_for_ctrl_irq (ctrl);
+	wait_for_ctrl_irq(ctrl);
 
 	/* Change bits in slot power register to force another shift out
 	 * NOTE: this is to work around the timer bug
@@ -1455,7 +1455,7 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 	set_SOGO(ctrl);
 
 	/* Wait for SOBS to be unset */
-	wait_for_ctrl_irq (ctrl);
+	wait_for_ctrl_irq(ctrl);
 
 	adapter_speed = get_adapter_speed(ctrl, hp_slot);
 	if (bus->cur_bus_speed != adapter_speed)
@@ -1463,7 +1463,7 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 			rc = WRONG_BUS_FREQUENCY;
 
 	/* turn off board without attaching to the bus */
-	disable_slot_power (ctrl, hp_slot);
+	disable_slot_power(ctrl, hp_slot);
 
 	set_SOGO(ctrl);
 
@@ -1484,20 +1484,20 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 	dbg("%s: after down\n", __func__);
 
 	dbg("%s: before slot_enable\n", __func__);
-	slot_enable (ctrl, hp_slot);
+	slot_enable(ctrl, hp_slot);
 
 	dbg("%s: before green_LED_blink\n", __func__);
-	green_LED_blink (ctrl, hp_slot);
+	green_LED_blink(ctrl, hp_slot);
 
 	dbg("%s: before amber_LED_blink\n", __func__);
-	amber_LED_off (ctrl, hp_slot);
+	amber_LED_off(ctrl, hp_slot);
 
 	dbg("%s: before set_SOGO\n", __func__);
 	set_SOGO(ctrl);
 
 	/* Wait for SOBS to be unset */
 	dbg("%s: before wait_for_ctrl_irq\n", __func__);
-	wait_for_ctrl_irq (ctrl);
+	wait_for_ctrl_irq(ctrl);
 	dbg("%s: after wait_for_ctrl_irq\n", __func__);
 
 	dbg("%s: before up\n", __func__);
@@ -1520,7 +1520,7 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 	} else {
 		/* Get vendor/device ID u32 */
 		ctrl->pci_bus->number = func->bus;
-		rc = pci_bus_read_config_dword (ctrl->pci_bus, PCI_DEVFN(func->device, func->function), PCI_VENDOR_ID, &temp_register);
+		rc = pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(func->device, func->function), PCI_VENDOR_ID, &temp_register);
 		dbg("%s: pci_read_config_dword returns %d\n", __func__, rc);
 		dbg("%s: temp_register is %x\n", __func__, temp_register);
 
@@ -1557,14 +1557,14 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 		if (rc) {
 			mutex_lock(&ctrl->crit_sect);
 
-			amber_LED_on (ctrl, hp_slot);
-			green_LED_off (ctrl, hp_slot);
-			slot_disable (ctrl, hp_slot);
+			amber_LED_on(ctrl, hp_slot);
+			green_LED_off(ctrl, hp_slot);
+			slot_disable(ctrl, hp_slot);
 
 			set_SOGO(ctrl);
 
 			/* Wait for SOBS to be unset */
-			wait_for_ctrl_irq (ctrl);
+			wait_for_ctrl_irq(ctrl);
 
 			mutex_unlock(&ctrl->crit_sect);
 			return rc;
@@ -1589,25 +1589,25 @@ static u32 board_added(struct pci_func *func, struct controller *ctrl)
 
 		mutex_lock(&ctrl->crit_sect);
 
-		green_LED_on (ctrl, hp_slot);
+		green_LED_on(ctrl, hp_slot);
 
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		mutex_unlock(&ctrl->crit_sect);
 	} else {
 		mutex_lock(&ctrl->crit_sect);
 
-		amber_LED_on (ctrl, hp_slot);
-		green_LED_off (ctrl, hp_slot);
-		slot_disable (ctrl, hp_slot);
+		amber_LED_on(ctrl, hp_slot);
+		green_LED_off(ctrl, hp_slot);
+		slot_disable(ctrl, hp_slot);
 
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 
 		mutex_unlock(&ctrl->crit_sect);
 
@@ -1672,8 +1672,8 @@ static u32 remove_board(struct pci_func *func, u32 replace_flag, struct controll
 
 	mutex_lock(&ctrl->crit_sect);
 
-	green_LED_off (ctrl, hp_slot);
-	slot_disable (ctrl, hp_slot);
+	green_LED_off(ctrl, hp_slot);
+	slot_disable(ctrl, hp_slot);
 
 	set_SOGO(ctrl);
 
@@ -1683,7 +1683,7 @@ static u32 remove_board(struct pci_func *func, u32 replace_flag, struct controll
 	writeb(temp_byte, ctrl->hpc_reg + SLOT_SERR);
 
 	/* Wait for SOBS to be unset */
-	wait_for_ctrl_irq (ctrl);
+	wait_for_ctrl_irq(ctrl);
 
 	mutex_unlock(&ctrl->crit_sect);
 
@@ -1755,7 +1755,7 @@ static int event_thread(void *data)
 		if (pushbutton_pending)
 			cpqhp_pushbutton_thread(pushbutton_pending);
 		else
-			for (ctrl = cpqhp_ctrl_list; ctrl; ctrl=ctrl->next)
+			for (ctrl = cpqhp_ctrl_list; ctrl; ctrl = ctrl->next)
 				interrupt_event_handler(ctrl);
 	}
 	dbg("event_thread signals exit\n");
@@ -1766,7 +1766,7 @@ int cpqhp_event_start_thread(void)
 {
 	cpqhp_event_thread = kthread_run(event_thread, NULL, "phpd_event");
 	if (IS_ERR(cpqhp_event_thread)) {
-		err ("Can't start up our event thread\n");
+		err("Can't start up our event thread\n");
 		return PTR_ERR(cpqhp_event_thread);
 	}
 
@@ -1794,7 +1794,7 @@ static int update_slot_info(struct controller *ctrl, struct slot *slot)
 	info->latch_status = cpq_get_latch_status(ctrl, slot);
 	info->adapter_status = get_presence_status(ctrl, slot);
 	result = pci_hp_change_slot_info(slot->hotplug_slot, info);
-	kfree (info);
+	kfree(info);
 	return result;
 }
 
@@ -1837,23 +1837,23 @@ static void interrupt_event_handler(struct controller *ctrl)
 					if (p_slot->state == BLINKINGOFF_STATE) {
 						/* slot is on */
 						dbg("turn on green LED\n");
-						green_LED_on (ctrl, hp_slot);
+						green_LED_on(ctrl, hp_slot);
 					} else if (p_slot->state == BLINKINGON_STATE) {
 						/* slot is off */
 						dbg("turn off green LED\n");
-						green_LED_off (ctrl, hp_slot);
+						green_LED_off(ctrl, hp_slot);
 					}
 
 					info(msg_button_cancel, p_slot->number);
 
 					p_slot->state = STATIC_STATE;
 
-					amber_LED_off (ctrl, hp_slot);
+					amber_LED_off(ctrl, hp_slot);
 
 					set_SOGO(ctrl);
 
 					/* Wait for SOBS to be unset */
-					wait_for_ctrl_irq (ctrl);
+					wait_for_ctrl_irq(ctrl);
 
 					mutex_unlock(&ctrl->crit_sect);
 				}
@@ -1861,7 +1861,7 @@ static void interrupt_event_handler(struct controller *ctrl)
 				else if (ctrl->event_queue[loop].event_type == INT_BUTTON_RELEASE) {
 					dbg("button release\n");
 
-					if (is_slot_enabled (ctrl, hp_slot)) {
+					if (is_slot_enabled(ctrl, hp_slot)) {
 						dbg("slot is on\n");
 						p_slot->state = BLINKINGOFF_STATE;
 						info(msg_button_off, p_slot->number);
@@ -1874,13 +1874,13 @@ static void interrupt_event_handler(struct controller *ctrl)
 
 					dbg("blink green LED and turn off amber\n");
 
-					amber_LED_off (ctrl, hp_slot);
-					green_LED_blink (ctrl, hp_slot);
+					amber_LED_off(ctrl, hp_slot);
+					green_LED_blink(ctrl, hp_slot);
 
 					set_SOGO(ctrl);
 
 					/* Wait for SOBS to be unset */
-					wait_for_ctrl_irq (ctrl);
+					wait_for_ctrl_irq(ctrl);
 
 					mutex_unlock(&ctrl->crit_sect);
 					init_timer(&p_slot->task_event);
@@ -1940,7 +1940,7 @@ void cpqhp_pushbutton_thread(unsigned long slot)
 		dbg("In power_down_board, func = %p, ctrl = %p\n", func, ctrl);
 		if (!func) {
 			dbg("Error! func NULL in %s\n", __func__);
-			return ;
+			return;
 		}
 
 		if (cpqhp_process_SS(ctrl, func) != 0) {
@@ -1962,7 +1962,7 @@ void cpqhp_pushbutton_thread(unsigned long slot)
 		dbg("In add_board, func = %p, ctrl = %p\n", func, ctrl);
 		if (!func) {
 			dbg("Error! func NULL in %s\n", __func__);
-			return ;
+			return;
 		}
 
 		if (ctrl != NULL) {
@@ -1973,7 +1973,7 @@ void cpqhp_pushbutton_thread(unsigned long slot)
 				set_SOGO(ctrl);
 
 				/* Wait for SOBS to be unset */
-				wait_for_ctrl_irq (ctrl);
+				wait_for_ctrl_irq(ctrl);
 			}
 		}
 
@@ -2086,7 +2086,7 @@ int cpqhp_process_SS(struct controller *ctrl, struct pci_func *func)
 	unsigned int devfn;
 	struct slot *p_slot;
 	struct pci_bus *pci_bus = ctrl->pci_bus;
-	int physical_slot=0;
+	int physical_slot = 0;
 
 	device = func->device;
 	func = cpqhp_slot_find(ctrl->bus, device, index++);
@@ -2100,7 +2100,7 @@ int cpqhp_process_SS(struct controller *ctrl, struct pci_func *func)
 		devfn = PCI_DEVFN(func->device, func->function);
 
 		/* Check the Class Code */
-		rc = pci_bus_read_config_byte (pci_bus, devfn, 0x0B, &class_code);
+		rc = pci_bus_read_config_byte(pci_bus, devfn, 0x0B, &class_code);
 		if (rc)
 			return rc;
 
@@ -2109,13 +2109,13 @@ int cpqhp_process_SS(struct controller *ctrl, struct pci_func *func)
 			rc = REMOVE_NOT_SUPPORTED;
 		} else {
 			/* See if it's a bridge */
-			rc = pci_bus_read_config_byte (pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
+			rc = pci_bus_read_config_byte(pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
 			if (rc)
 				return rc;
 
 			/* If it's a bridge, check the VGA Enable bit */
 			if ((header_type & 0x7F) == PCI_HEADER_TYPE_BRIDGE) {
-				rc = pci_bus_read_config_byte (pci_bus, devfn, PCI_BRIDGE_CONTROL, &BCR);
+				rc = pci_bus_read_config_byte(pci_bus, devfn, PCI_BRIDGE_CONTROL, &BCR);
 				if (rc)
 					return rc;
 
@@ -2217,7 +2217,7 @@ int cpqhp_hardware_test(struct controller *ctrl, int test_num)
 			set_SOGO(ctrl);
 
 			/* Wait for SOGO interrupt */
-			wait_for_ctrl_irq (ctrl);
+			wait_for_ctrl_irq(ctrl);
 
 			/* Get ready for next iteration */
 			long_delay((3*HZ)/10);
@@ -2227,7 +2227,7 @@ int cpqhp_hardware_test(struct controller *ctrl, int test_num)
 			set_SOGO(ctrl);
 
 			/* Wait for SOGO interrupt */
-			wait_for_ctrl_irq (ctrl);
+			wait_for_ctrl_irq(ctrl);
 
 			/* Get ready for next iteration */
 			long_delay((3*HZ)/10);
@@ -2243,7 +2243,7 @@ int cpqhp_hardware_test(struct controller *ctrl, int test_num)
 		set_SOGO(ctrl);
 
 		/* Wait for SOBS to be unset */
-		wait_for_ctrl_irq (ctrl);
+		wait_for_ctrl_irq(ctrl);
 		break;
 	case 2:
 		/* Do other stuff here! */
@@ -2279,7 +2279,7 @@ static u32 configure_new_device(struct controller  *ctrl, struct pci_func  *func
 	dbg("%s\n", __func__);
 	/* Check for Multi-function device */
 	ctrl->pci_bus->number = func->bus;
-	rc = pci_bus_read_config_byte (ctrl->pci_bus, PCI_DEVFN(func->device, func->function), 0x0E, &temp_byte);
+	rc = pci_bus_read_config_byte(ctrl->pci_bus, PCI_DEVFN(func->device, func->function), 0x0E, &temp_byte);
 	if (rc) {
 		dbg("%s: rc = %d\n", __func__, rc);
 		return rc;
@@ -2296,7 +2296,7 @@ static u32 configure_new_device(struct controller  *ctrl, struct pci_func  *func
 		rc = configure_new_function(ctrl, new_slot, behind_bridge, resources);
 
 		if (rc) {
-			dbg("configure_new_function failed %d\n",rc);
+			dbg("configure_new_function failed %d\n", rc);
 			index = 0;
 
 			while (new_slot) {
@@ -2317,7 +2317,7 @@ static u32 configure_new_device(struct controller  *ctrl, struct pci_func  *func
 		 * and creates a board structure */
 
 		while ((function < max_functions) && (!stop_it)) {
-			pci_bus_read_config_dword (ctrl->pci_bus, PCI_DEVFN(func->device, function), 0x00, &ID);
+			pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(func->device, function), 0x00, &ID);
 
 			if (ID == 0xFFFFFFFF) {
 				function++;
@@ -2543,10 +2543,10 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 
 		/* set Pre Mem base and Limit registers */
 		temp_word = p_mem_node->base >> 16;
-		rc = pci_bus_write_config_word (pci_bus, devfn, PCI_PREF_MEMORY_BASE, temp_word);
+		rc = pci_bus_write_config_word(pci_bus, devfn, PCI_PREF_MEMORY_BASE, temp_word);
 
 		temp_word = (p_mem_node->base + p_mem_node->length - 1) >> 16;
-		rc = pci_bus_write_config_word (pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
+		rc = pci_bus_write_config_word(pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
 
 		/* Adjust this to compensate for extra adjustment in first loop
 		 */
@@ -2560,7 +2560,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 
 			ID = 0xFFFFFFFF;
 			pci_bus->number = hold_bus_node->base;
-			pci_bus_read_config_dword (pci_bus, PCI_DEVFN(device, 0), 0x00, &ID);
+			pci_bus_read_config_dword(pci_bus, PCI_DEVFN(device, 0), 0x00, &ID);
 			pci_bus->number = func->bus;
 
 			if (ID != 0xFFFFFFFF) {	  /*  device present */
@@ -2579,7 +2579,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 				new_slot->status = 0;
 
 				rc = configure_new_device(ctrl, new_slot, 1, &temp_resources);
-				dbg("configure_new_device rc=0x%x\n",rc);
+				dbg("configure_new_device rc=0x%x\n", rc);
 			}	/* End of IF (device in slot?) */
 		}		/* End of FOR loop */
 
@@ -2615,7 +2615,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 			temp_byte = temp_resources.bus_head->base - 1;
 
 			/* set subordinate bus */
-			rc = pci_bus_write_config_byte (pci_bus, devfn, PCI_SUBORDINATE_BUS, temp_byte);
+			rc = pci_bus_write_config_byte(pci_bus, devfn, PCI_SUBORDINATE_BUS, temp_byte);
 
 			if (temp_resources.bus_head->length == 0) {
 				kfree(temp_resources.bus_head);
@@ -2636,7 +2636,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 				hold_IO_node->base = io_node->base + io_node->length;
 
 				temp_byte = (hold_IO_node->base) >> 8;
-				rc = pci_bus_write_config_word (pci_bus, devfn, PCI_IO_BASE, temp_byte);
+				rc = pci_bus_write_config_word(pci_bus, devfn, PCI_IO_BASE, temp_byte);
 
 				return_resource(&(resources->io_head), io_node);
 			}
@@ -2655,13 +2655,13 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 					func->io_head = hold_IO_node;
 
 					temp_byte = (io_node->base - 1) >> 8;
-					rc = pci_bus_write_config_byte (pci_bus, devfn, PCI_IO_LIMIT, temp_byte);
+					rc = pci_bus_write_config_byte(pci_bus, devfn, PCI_IO_LIMIT, temp_byte);
 
 					return_resource(&(resources->io_head), io_node);
 				} else {
 					/* it doesn't need any IO */
 					temp_word = 0x0000;
-					rc = pci_bus_write_config_word (pci_bus, devfn, PCI_IO_LIMIT, temp_word);
+					rc = pci_bus_write_config_word(pci_bus, devfn, PCI_IO_LIMIT, temp_word);
 
 					return_resource(&(resources->io_head), io_node);
 					kfree(hold_IO_node);
@@ -2687,7 +2687,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 				hold_mem_node->base = mem_node->base + mem_node->length;
 
 				temp_word = (hold_mem_node->base) >> 16;
-				rc = pci_bus_write_config_word (pci_bus, devfn, PCI_MEMORY_BASE, temp_word);
+				rc = pci_bus_write_config_word(pci_bus, devfn, PCI_MEMORY_BASE, temp_word);
 
 				return_resource(&(resources->mem_head), mem_node);
 			}
@@ -2706,14 +2706,14 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 
 					/* configure end address */
 					temp_word = (mem_node->base - 1) >> 16;
-					rc = pci_bus_write_config_word (pci_bus, devfn, PCI_MEMORY_LIMIT, temp_word);
+					rc = pci_bus_write_config_word(pci_bus, devfn, PCI_MEMORY_LIMIT, temp_word);
 
 					/* Return unused resources to the pool */
 					return_resource(&(resources->mem_head), mem_node);
 				} else {
 					/* it doesn't need any Mem */
 					temp_word = 0x0000;
-					rc = pci_bus_write_config_word (pci_bus, devfn, PCI_MEMORY_LIMIT, temp_word);
+					rc = pci_bus_write_config_word(pci_bus, devfn, PCI_MEMORY_LIMIT, temp_word);
 
 					return_resource(&(resources->mem_head), mem_node);
 					kfree(hold_mem_node);
@@ -2739,7 +2739,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 				hold_p_mem_node->base = p_mem_node->base + p_mem_node->length;
 
 				temp_word = (hold_p_mem_node->base) >> 16;
-				rc = pci_bus_write_config_word (pci_bus, devfn, PCI_PREF_MEMORY_BASE, temp_word);
+				rc = pci_bus_write_config_word(pci_bus, devfn, PCI_PREF_MEMORY_BASE, temp_word);
 
 				return_resource(&(resources->p_mem_head), p_mem_node);
 			}
@@ -2758,13 +2758,13 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 					func->p_mem_head = hold_p_mem_node;
 
 					temp_word = (p_mem_node->base - 1) >> 16;
-					rc = pci_bus_write_config_word (pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
+					rc = pci_bus_write_config_word(pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
 
 					return_resource(&(resources->p_mem_head), p_mem_node);
 				} else {
 					/* it doesn't need any PMem */
 					temp_word = 0x0000;
-					rc = pci_bus_write_config_word (pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
+					rc = pci_bus_write_config_word(pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, temp_word);
 
 					return_resource(&(resources->p_mem_head), p_mem_node);
 					kfree(hold_p_mem_node);
@@ -2790,16 +2790,16 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 					 *   PCI_COMMAND_INVALIDATE |
 					 *   PCI_COMMAND_PARITY |
 					 *   PCI_COMMAND_SERR */
-		rc = pci_bus_write_config_word (pci_bus, devfn, PCI_COMMAND, command);
+		rc = pci_bus_write_config_word(pci_bus, devfn, PCI_COMMAND, command);
 
 		/* set Bridge Control Register */
 		command = 0x07;		/* = PCI_BRIDGE_CTL_PARITY |
 					 *   PCI_BRIDGE_CTL_SERR |
 					 *   PCI_BRIDGE_CTL_NO_ISA */
-		rc = pci_bus_write_config_word (pci_bus, devfn, PCI_BRIDGE_CONTROL, command);
+		rc = pci_bus_write_config_word(pci_bus, devfn, PCI_BRIDGE_CONTROL, command);
 	} else if ((temp_byte & 0x7F) == PCI_HEADER_TYPE_NORMAL) {
 		/* Standard device */
-		rc = pci_bus_read_config_byte (pci_bus, devfn, 0x0B, &class_code);
+		rc = pci_bus_read_config_byte(pci_bus, devfn, 0x0B, &class_code);
 
 		if (class_code == PCI_BASE_CLASS_DISPLAY) {
 			/* Display (video) adapter (not supported) */
@@ -2810,9 +2810,9 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 			temp_register = 0xFFFFFFFF;
 
 			dbg("CND: bus=%d, devfn=%d, offset=%d\n", pci_bus->number, devfn, cloop);
-			rc = pci_bus_write_config_dword (pci_bus, devfn, cloop, temp_register);
+			rc = pci_bus_write_config_dword(pci_bus, devfn, cloop, temp_register);
 
-			rc = pci_bus_read_config_dword (pci_bus, devfn, cloop, &temp_register);
+			rc = pci_bus_read_config_dword(pci_bus, devfn, cloop, &temp_register);
 			dbg("CND: base = 0x%x\n", temp_register);
 
 			if (temp_register) {	  /* If this register is implemented */
@@ -2891,7 +2891,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 		}		/* End of base register loop */
 		if (cpqhp_legacy_mode) {
 			/* Figure out which interrupt pin this function uses */
-			rc = pci_bus_read_config_byte (pci_bus, devfn,
+			rc = pci_bus_read_config_byte(pci_bus, devfn,
 				PCI_INTERRUPT_PIN, &temp_byte);
 
 			/* If this function needs an interrupt and we are behind
@@ -2905,7 +2905,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 					resources->irqs->barber_pole - 1) & 0x03];
 			} else {
 				/* Program IRQ based on card type */
-				rc = pci_bus_read_config_byte (pci_bus, devfn, 0x0B, &class_code);
+				rc = pci_bus_read_config_byte(pci_bus, devfn, 0x0B, &class_code);
 
 				if (class_code == PCI_BASE_CLASS_STORAGE)
 					IRQ = cpqhp_disk_irq;
@@ -2914,7 +2914,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 			}
 
 			/* IRQ Line */
-			rc = pci_bus_write_config_byte (pci_bus, devfn, PCI_INTERRUPT_LINE, IRQ);
+			rc = pci_bus_write_config_byte(pci_bus, devfn, PCI_INTERRUPT_LINE, IRQ);
 		}
 
 		if (!behind_bridge) {
@@ -2950,7 +2950,7 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 					 *   PCI_COMMAND_INVALIDATE |
 					 *   PCI_COMMAND_PARITY |
 					 *   PCI_COMMAND_SERR */
-		rc = pci_bus_write_config_word (pci_bus, devfn,
+		rc = pci_bus_write_config_word(pci_bus, devfn,
 					PCI_COMMAND, temp_word);
 	} else {		/* End of Not-A-Bridge else */
 		/* It's some strange type of PCI adapter (Cardbus?) */
@@ -2961,11 +2961,11 @@ static int configure_new_function(struct controller *ctrl, struct pci_func *func
 
 	return 0;
 free_and_out:
-	cpqhp_destroy_resource_list (&temp_resources);
+	cpqhp_destroy_resource_list(&temp_resources);
 
-	return_resource(&(resources-> bus_head), hold_bus_node);
-	return_resource(&(resources-> io_head), hold_IO_node);
-	return_resource(&(resources-> mem_head), hold_mem_node);
-	return_resource(&(resources-> p_mem_head), hold_p_mem_node);
+	return_resource(&(resources->bus_head), hold_bus_node);
+	return_resource(&(resources->io_head), hold_IO_node);
+	return_resource(&(resources->mem_head), hold_mem_node);
+	return_resource(&(resources->p_mem_head), hold_p_mem_node);
 	return rc;
 }
diff --git a/drivers/pci/hotplug/cpqphp_nvram.c b/drivers/pci/hotplug/cpqphp_nvram.c
index 1e08ff8c229c..c25fc9061059 100644
--- a/drivers/pci/hotplug/cpqphp_nvram.c
+++ b/drivers/pci/hotplug/cpqphp_nvram.c
@@ -114,10 +114,10 @@ static u32 add_byte(u32 **p_buffer, u8 value, u32 *used, u32 *avail)
 	if ((*used + 1) > *avail)
 		return(1);
 
-	*((u8*)*p_buffer) = value;
-	tByte = (u8**)p_buffer;
+	*((u8 *)*p_buffer) = value;
+	tByte = (u8 **)p_buffer;
 	(*tByte)++;
-	*used+=1;
+	*used += 1;
 	return(0);
 }
 
@@ -129,7 +129,7 @@ static u32 add_dword(u32 **p_buffer, u32 value, u32 *used, u32 *avail)
 
 	**p_buffer = value;
 	(*p_buffer)++;
-	*used+=4;
+	*used += 4;
 	return(0);
 }
 
@@ -141,7 +141,7 @@ static u32 add_dword(u32 **p_buffer, u32 value, u32 *used, u32 *avail)
  *
  * returns 0 for non-Compaq ROM, 1 for Compaq ROM
  */
-static int check_for_compaq_ROM (void __iomem *rom_start)
+static int check_for_compaq_ROM(void __iomem *rom_start)
 {
 	u8 temp1, temp2, temp3, temp4, temp5, temp6;
 	int result = 0;
@@ -160,12 +160,12 @@ static int check_for_compaq_ROM (void __iomem *rom_start)
 	    (temp6 == 'Q')) {
 		result = 1;
 	}
-	dbg ("%s - returned %d\n", __func__, result);
+	dbg("%s - returned %d\n", __func__, result);
 	return result;
 }
 
 
-static u32 access_EV (u16 operation, u8 *ev_name, u8 *buffer, u32 *buf_size)
+static u32 access_EV(u16 operation, u8 *ev_name, u8 *buffer, u32 *buf_size)
 {
 	unsigned long flags;
 	int op = operation;
@@ -197,7 +197,7 @@ static u32 access_EV (u16 operation, u8 *ev_name, u8 *buffer, u32 *buf_size)
  *
  * Read the hot plug Resource Table from NVRAM
  */
-static int load_HRT (void __iomem *rom_start)
+static int load_HRT(void __iomem *rom_start)
 {
 	u32 available;
 	u32 temp_dword;
@@ -232,7 +232,7 @@ static int load_HRT (void __iomem *rom_start)
  *
  * Save the hot plug Resource Table in NVRAM
  */
-static u32 store_HRT (void __iomem *rom_start)
+static u32 store_HRT(void __iomem *rom_start)
 {
 	u32 *buffer;
 	u32 *pFill;
@@ -252,7 +252,7 @@ static u32 store_HRT (void __iomem *rom_start)
 	if (!check_for_compaq_ROM(rom_start))
 		return(1);
 
-	buffer = (u32*) evbuffer;
+	buffer = (u32 *) evbuffer;
 
 	if (!buffer)
 		return(1);
@@ -306,7 +306,7 @@ static u32 store_HRT (void __iomem *rom_start)
 		loop = 0;
 
 		while (resNode) {
-			loop ++;
+			loop++;
 
 			/* base */
 			rc = add_dword(&pFill, resNode->base, &usedbytes, &available);
@@ -331,7 +331,7 @@ static u32 store_HRT (void __iomem *rom_start)
 		loop = 0;
 
 		while (resNode) {
-			loop ++;
+			loop++;
 
 			/* base */
 			rc = add_dword(&pFill, resNode->base, &usedbytes, &available);
@@ -356,7 +356,7 @@ static u32 store_HRT (void __iomem *rom_start)
 		loop = 0;
 
 		while (resNode) {
-			loop ++;
+			loop++;
 
 			/* base */
 			rc = add_dword(&pFill, resNode->base, &usedbytes, &available);
@@ -381,7 +381,7 @@ static u32 store_HRT (void __iomem *rom_start)
 		loop = 0;
 
 		while (resNode) {
-			loop ++;
+			loop++;
 
 			/* base */
 			rc = add_dword(&pFill, resNode->base, &usedbytes, &available);
@@ -408,7 +408,7 @@ static u32 store_HRT (void __iomem *rom_start)
 
 	temp_dword = usedbytes;
 
-	rc = access_EV(WRITE_EV, "CQTHPS", (u8*) buffer, &temp_dword);
+	rc = access_EV(WRITE_EV, "CQTHPS", (u8 *) buffer, &temp_dword);
 
 	dbg("usedbytes = 0x%x, length = 0x%x\n", usedbytes, temp_dword);
 
@@ -423,7 +423,7 @@ static u32 store_HRT (void __iomem *rom_start)
 }
 
 
-void compaq_nvram_init (void __iomem *rom_start)
+void compaq_nvram_init(void __iomem *rom_start)
 {
 	if (rom_start)
 		compaq_int15_entry_point = (rom_start + ROM_INT15_PHY_ADDR - ROM_PHY_ADDR);
@@ -435,7 +435,7 @@ void compaq_nvram_init (void __iomem *rom_start)
 }
 
 
-int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
+int compaq_nvram_load(void __iomem *rom_start, struct controller *ctrl)
 {
 	u8 bus, device, function;
 	u8 nummem, numpmem, numio, numbus;
@@ -451,7 +451,7 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 	if (!evbuffer_init) {
 		/* Read the resource list information in from NVRAM */
 		if (load_HRT(rom_start))
-			memset (evbuffer, 0, 1024);
+			memset(evbuffer, 0, 1024);
 
 		evbuffer_init = 1;
 	}
@@ -472,7 +472,7 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 
 		p_byte += 3;
 
-		if (p_byte > ((u8*)p_EV_header + evbuffer_length))
+		if (p_byte > ((u8 *)p_EV_header + evbuffer_length))
 			return 2;
 
 		bus = p_ev_ctrl->bus;
@@ -489,20 +489,20 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length))
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length))
 				return 2;
 
 			/* Skip forward to the next entry */
 			p_byte += (nummem + numpmem + numio + numbus) * 8;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length))
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length))
 				return 2;
 
 			p_ev_ctrl = (struct ev_hrt_ctrl *) p_byte;
 
 			p_byte += 3;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length))
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length))
 				return 2;
 
 			bus = p_ev_ctrl->bus;
@@ -517,7 +517,7 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 
 		p_byte += 4;
 
-		if (p_byte > ((u8*)p_EV_header + evbuffer_length))
+		if (p_byte > ((u8 *)p_EV_header + evbuffer_length))
 			return 2;
 
 		while (nummem--) {
@@ -526,20 +526,20 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 			if (!mem_node)
 				break;
 
-			mem_node->base = *(u32*)p_byte;
-			dbg("mem base = %8.8x\n",mem_node->base);
+			mem_node->base = *(u32 *)p_byte;
+			dbg("mem base = %8.8x\n", mem_node->base);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(mem_node);
 				return 2;
 			}
 
-			mem_node->length = *(u32*)p_byte;
-			dbg("mem length = %8.8x\n",mem_node->length);
+			mem_node->length = *(u32 *)p_byte;
+			dbg("mem length = %8.8x\n", mem_node->length);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(mem_node);
 				return 2;
 			}
@@ -554,20 +554,20 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 			if (!p_mem_node)
 				break;
 
-			p_mem_node->base = *(u32*)p_byte;
-			dbg("pre-mem base = %8.8x\n",p_mem_node->base);
+			p_mem_node->base = *(u32 *)p_byte;
+			dbg("pre-mem base = %8.8x\n", p_mem_node->base);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(p_mem_node);
 				return 2;
 			}
 
-			p_mem_node->length = *(u32*)p_byte;
-			dbg("pre-mem length = %8.8x\n",p_mem_node->length);
+			p_mem_node->length = *(u32 *)p_byte;
+			dbg("pre-mem length = %8.8x\n", p_mem_node->length);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(p_mem_node);
 				return 2;
 			}
@@ -582,20 +582,20 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 			if (!io_node)
 				break;
 
-			io_node->base = *(u32*)p_byte;
-			dbg("io base = %8.8x\n",io_node->base);
+			io_node->base = *(u32 *)p_byte;
+			dbg("io base = %8.8x\n", io_node->base);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(io_node);
 				return 2;
 			}
 
-			io_node->length = *(u32*)p_byte;
-			dbg("io length = %8.8x\n",io_node->length);
+			io_node->length = *(u32 *)p_byte;
+			dbg("io length = %8.8x\n", io_node->length);
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(io_node);
 				return 2;
 			}
@@ -610,18 +610,18 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 			if (!bus_node)
 				break;
 
-			bus_node->base = *(u32*)p_byte;
+			bus_node->base = *(u32 *)p_byte;
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(bus_node);
 				return 2;
 			}
 
-			bus_node->length = *(u32*)p_byte;
+			bus_node->length = *(u32 *)p_byte;
 			p_byte += 4;
 
-			if (p_byte > ((u8*)p_EV_header + evbuffer_length)) {
+			if (p_byte > ((u8 *)p_EV_header + evbuffer_length)) {
 				kfree(bus_node);
 				return 2;
 			}
@@ -650,7 +650,7 @@ int compaq_nvram_load (void __iomem *rom_start, struct controller *ctrl)
 }
 
 
-int compaq_nvram_store (void __iomem *rom_start)
+int compaq_nvram_store(void __iomem *rom_start)
 {
 	int rc = 1;
 
diff --git a/drivers/pci/hotplug/cpqphp_pci.c b/drivers/pci/hotplug/cpqphp_pci.c
index 1c8c2f130d31..e220d49307bd 100644
--- a/drivers/pci/hotplug/cpqphp_pci.c
+++ b/drivers/pci/hotplug/cpqphp_pci.c
@@ -81,7 +81,7 @@ static void __iomem *detect_HRT_floating_pointer(void __iomem *begin, void __iom
 }
 
 
-int cpqhp_configure_device (struct controller *ctrl, struct pci_func *func)
+int cpqhp_configure_device(struct controller *ctrl, struct pci_func *func)
 {
 	struct pci_bus *child;
 	int num;
@@ -89,7 +89,7 @@ int cpqhp_configure_device (struct controller *ctrl, struct pci_func *func)
 	pci_lock_rescan_remove();
 
 	if (func->pci_dev == NULL)
-		func->pci_dev = pci_get_bus_and_slot(func->bus,PCI_DEVFN(func->device, func->function));
+		func->pci_dev = pci_get_bus_and_slot(func->bus, PCI_DEVFN(func->device, func->function));
 
 	/* No pci device, we need to create it then */
 	if (func->pci_dev == NULL) {
@@ -128,7 +128,7 @@ int cpqhp_unconfigure_device(struct pci_func *func)
 	dbg("%s: bus/dev/func = %x/%x/%x\n", __func__, func->bus, func->device, func->function);
 
 	pci_lock_rescan_remove();
-	for (j=0; j<8 ; j++) {
+	for (j = 0; j < 8 ; j++) {
 		struct pci_dev *temp = pci_get_bus_and_slot(func->bus, PCI_DEVFN(func->device, j));
 		if (temp) {
 			pci_dev_put(temp);
@@ -143,11 +143,11 @@ static int PCI_RefinedAccessConfig(struct pci_bus *bus, unsigned int devfn, u8 o
 {
 	u32 vendID = 0;
 
-	if (pci_bus_read_config_dword (bus, devfn, PCI_VENDOR_ID, &vendID) == -1)
+	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, &vendID) == -1)
 		return -1;
 	if (vendID == 0xffffffff)
 		return -1;
-	return pci_bus_read_config_dword (bus, devfn, offset, value);
+	return pci_bus_read_config_dword(bus, devfn, offset, value);
 }
 
 
@@ -158,7 +158,7 @@ static int PCI_RefinedAccessConfig(struct pci_bus *bus, unsigned int devfn, u8 o
  * @dev_num: device number of PCI device
  * @slot: pointer to u8 where slot number will be returned
  */
-int cpqhp_set_irq (u8 bus_num, u8 dev_num, u8 int_pin, u8 irq_num)
+int cpqhp_set_irq(u8 bus_num, u8 dev_num, u8 int_pin, u8 irq_num)
 {
 	int rc = 0;
 
@@ -230,7 +230,7 @@ static int PCI_ScanBusForNonBridge(struct controller *ctrl, u8 bus_num, u8 *dev_
 		dbg("Looking for bridge bus_num %d dev_num %d\n", bus_num, tdevice);
 		/* Yep we got one. bridge ? */
 		if ((work >> 8) == PCI_TO_PCI_BRIDGE_CLASS) {
-			pci_bus_read_config_byte (ctrl->pci_bus, PCI_DEVFN(tdevice, 0), PCI_SECONDARY_BUS, &tbus);
+			pci_bus_read_config_byte(ctrl->pci_bus, PCI_DEVFN(tdevice, 0), PCI_SECONDARY_BUS, &tbus);
 			/* XXX: no recursion, wtf? */
 			dbg("Recurse on bus_num %d tdevice %d\n", tbus, tdevice);
 			return 0;
@@ -257,16 +257,16 @@ static int PCI_GetBusDevHelper(struct controller *ctrl, u8 *bus_num, u8 *dev_num
 			*bus_num = tbus;
 			*dev_num = tdevice;
 			ctrl->pci_bus->number = tbus;
-			pci_bus_read_config_dword (ctrl->pci_bus, *dev_num, PCI_VENDOR_ID, &work);
+			pci_bus_read_config_dword(ctrl->pci_bus, *dev_num, PCI_VENDOR_ID, &work);
 			if (!nobridge || (work == 0xffffffff))
 				return 0;
 
 			dbg("bus_num %d devfn %d\n", *bus_num, *dev_num);
-			pci_bus_read_config_dword (ctrl->pci_bus, *dev_num, PCI_CLASS_REVISION, &work);
+			pci_bus_read_config_dword(ctrl->pci_bus, *dev_num, PCI_CLASS_REVISION, &work);
 			dbg("work >> 8 (%x) = BRIDGE (%x)\n", work >> 8, PCI_TO_PCI_BRIDGE_CLASS);
 
 			if ((work >> 8) == PCI_TO_PCI_BRIDGE_CLASS) {
-				pci_bus_read_config_byte (ctrl->pci_bus, *dev_num, PCI_SECONDARY_BUS, &tbus);
+				pci_bus_read_config_byte(ctrl->pci_bus, *dev_num, PCI_SECONDARY_BUS, &tbus);
 				dbg("Scan bus for Non Bridge: bus %d\n", tbus);
 				if (PCI_ScanBusForNonBridge(ctrl, tbus, dev_num) == 0) {
 					*bus_num = tbus;
@@ -280,7 +280,7 @@ static int PCI_GetBusDevHelper(struct controller *ctrl, u8 *bus_num, u8 *dev_num
 }
 
 
-int cpqhp_get_bus_dev (struct controller *ctrl, u8 *bus_num, u8 *dev_num, u8 slot)
+int cpqhp_get_bus_dev(struct controller *ctrl, u8 *bus_num, u8 *dev_num, u8 slot)
 {
 	/* plain (bridges allowed) */
 	return PCI_GetBusDevHelper(ctrl, bus_num, dev_num, slot, 0);
@@ -419,7 +419,7 @@ int cpqhp_save_config(struct controller *ctrl, int busnumber, int is_hot_plug)
 			new_slot->pci_dev = pci_get_bus_and_slot(new_slot->bus, (new_slot->device << 3) | new_slot->function);
 
 			for (cloop = 0; cloop < 0x20; cloop++) {
-				rc = pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(device, function), cloop << 2, (u32 *) & (new_slot-> config_space [cloop]));
+				rc = pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(device, function), cloop << 2, (u32 *) &(new_slot->config_space[cloop]));
 				if (rc)
 					return rc;
 			}
@@ -465,7 +465,7 @@ int cpqhp_save_config(struct controller *ctrl, int busnumber, int is_hot_plug)
  *
  * returns 0 if success
  */
-int cpqhp_save_slot_config (struct controller *ctrl, struct pci_func *new_slot)
+int cpqhp_save_slot_config(struct controller *ctrl, struct pci_func *new_slot)
 {
 	long rc;
 	u8 class_code;
@@ -481,7 +481,7 @@ int cpqhp_save_slot_config (struct controller *ctrl, struct pci_func *new_slot)
 	ID = 0xFFFFFFFF;
 
 	ctrl->pci_bus->number = new_slot->bus;
-	pci_bus_read_config_dword (ctrl->pci_bus, PCI_DEVFN(new_slot->device, 0), PCI_VENDOR_ID, &ID);
+	pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(new_slot->device, 0), PCI_VENDOR_ID, &ID);
 
 	if (ID == 0xFFFFFFFF)
 		return 2;
@@ -497,7 +497,7 @@ int cpqhp_save_slot_config (struct controller *ctrl, struct pci_func *new_slot)
 	while (function < max_functions) {
 		if ((header_type & 0x7F) == PCI_HEADER_TYPE_BRIDGE) {
 			/*  Recurse the subordinate bus */
-			pci_bus_read_config_byte (ctrl->pci_bus, PCI_DEVFN(new_slot->device, function), PCI_SECONDARY_BUS, &secondary_bus);
+			pci_bus_read_config_byte(ctrl->pci_bus, PCI_DEVFN(new_slot->device, function), PCI_SECONDARY_BUS, &secondary_bus);
 
 			sub_bus = (int) secondary_bus;
 
@@ -514,7 +514,7 @@ int cpqhp_save_slot_config (struct controller *ctrl, struct pci_func *new_slot)
 		new_slot->status = 0;
 
 		for (cloop = 0; cloop < 0x20; cloop++)
-			pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(new_slot->device, function), cloop << 2, (u32 *) & (new_slot-> config_space [cloop]));
+			pci_bus_read_config_dword(ctrl->pci_bus, PCI_DEVFN(new_slot->device, function), cloop << 2, (u32 *) &(new_slot->config_space[cloop]));
 
 		function++;
 
@@ -571,10 +571,10 @@ int cpqhp_save_base_addr_length(struct controller *ctrl, struct pci_func *func)
 		devfn = PCI_DEVFN(func->device, func->function);
 
 		/* Check for Bridge */
-		pci_bus_read_config_byte (pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
+		pci_bus_read_config_byte(pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
 
 		if ((header_type & 0x7F) == PCI_HEADER_TYPE_BRIDGE) {
-			pci_bus_read_config_byte (pci_bus, devfn, PCI_SECONDARY_BUS, &secondary_bus);
+			pci_bus_read_config_byte(pci_bus, devfn, PCI_SECONDARY_BUS, &secondary_bus);
 
 			sub_bus = (int) secondary_bus;
 
@@ -595,8 +595,8 @@ int cpqhp_save_base_addr_length(struct controller *ctrl, struct pci_func *func)
 			 */
 			for (cloop = 0x10; cloop <= 0x14; cloop += 4) {
 				temp_register = 0xFFFFFFFF;
-				pci_bus_write_config_dword (pci_bus, devfn, cloop, temp_register);
-				pci_bus_read_config_dword (pci_bus, devfn, cloop, &base);
+				pci_bus_write_config_dword(pci_bus, devfn, cloop, temp_register);
+				pci_bus_read_config_dword(pci_bus, devfn, cloop, &base);
 				/* If this register is implemented */
 				if (base) {
 					if (base & 0x01L) {
@@ -631,8 +631,8 @@ int cpqhp_save_base_addr_length(struct controller *ctrl, struct pci_func *func)
 			/* Figure out IO and memory base lengths */
 			for (cloop = 0x10; cloop <= 0x24; cloop += 4) {
 				temp_register = 0xFFFFFFFF;
-				pci_bus_write_config_dword (pci_bus, devfn, cloop, temp_register);
-				pci_bus_read_config_dword (pci_bus, devfn, cloop, &base);
+				pci_bus_write_config_dword(pci_bus, devfn, cloop, temp_register);
+				pci_bus_read_config_dword(pci_bus, devfn, cloop, &base);
 
 				/* If this register is implemented */
 				if (base) {
@@ -686,7 +686,7 @@ int cpqhp_save_base_addr_length(struct controller *ctrl, struct pci_func *func)
  *
  * returns 0 if success
  */
-int cpqhp_save_used_resources (struct controller *ctrl, struct pci_func *func)
+int cpqhp_save_used_resources(struct controller *ctrl, struct pci_func *func)
 {
 	u8 cloop;
 	u8 header_type;
@@ -791,7 +791,7 @@ int cpqhp_save_used_resources (struct controller *ctrl, struct pci_func *func)
 			}
 			/* Figure out IO and memory base lengths */
 			for (cloop = 0x10; cloop <= 0x14; cloop += 4) {
-				pci_bus_read_config_dword (pci_bus, devfn, cloop, &save_base);
+				pci_bus_read_config_dword(pci_bus, devfn, cloop, &save_base);
 
 				temp_register = 0xFFFFFFFF;
 				pci_bus_write_config_dword(pci_bus, devfn, cloop, temp_register);
@@ -972,13 +972,13 @@ int cpqhp_configure_board(struct controller *ctrl, struct pci_func *func)
 		 * registers are programmed last
 		 */
 		for (cloop = 0x3C; cloop > 0; cloop -= 4)
-			pci_bus_write_config_dword (pci_bus, devfn, cloop, func->config_space[cloop >> 2]);
+			pci_bus_write_config_dword(pci_bus, devfn, cloop, func->config_space[cloop >> 2]);
 
-		pci_bus_read_config_byte (pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
+		pci_bus_read_config_byte(pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
 
 		/* If this is a bridge device, restore subordinate devices */
 		if ((header_type & 0x7F) == PCI_HEADER_TYPE_BRIDGE) {
-			pci_bus_read_config_byte (pci_bus, devfn, PCI_SECONDARY_BUS, &secondary_bus);
+			pci_bus_read_config_byte(pci_bus, devfn, PCI_SECONDARY_BUS, &secondary_bus);
 
 			sub_bus = (int) secondary_bus;
 
@@ -998,7 +998,7 @@ int cpqhp_configure_board(struct controller *ctrl, struct pci_func *func)
 			 */
 
 			for (cloop = 16; cloop < 40; cloop += 4) {
-				pci_bus_read_config_dword (pci_bus, devfn, cloop, &temp);
+				pci_bus_read_config_dword(pci_bus, devfn, cloop, &temp);
 
 				if (temp != func->config_space[cloop >> 2]) {
 					dbg("Config space compare failure!!! offset = %x\n", cloop);
@@ -1050,7 +1050,7 @@ int cpqhp_valid_replace(struct controller *ctrl, struct pci_func *func)
 		pci_bus->number = func->bus;
 		devfn = PCI_DEVFN(func->device, func->function);
 
-		pci_bus_read_config_dword (pci_bus, devfn, PCI_VENDOR_ID, &temp_register);
+		pci_bus_read_config_dword(pci_bus, devfn, PCI_VENDOR_ID, &temp_register);
 
 		/* No adapter present */
 		if (temp_register == 0xFFFFFFFF)
@@ -1060,14 +1060,14 @@ int cpqhp_valid_replace(struct controller *ctrl, struct pci_func *func)
 			return(ADAPTER_NOT_SAME);
 
 		/* Check for same revision number and class code */
-		pci_bus_read_config_dword (pci_bus, devfn, PCI_CLASS_REVISION, &temp_register);
+		pci_bus_read_config_dword(pci_bus, devfn, PCI_CLASS_REVISION, &temp_register);
 
 		/* Adapter not the same */
 		if (temp_register != func->config_space[0x08 >> 2])
 			return(ADAPTER_NOT_SAME);
 
 		/* Check for Bridge */
-		pci_bus_read_config_byte (pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
+		pci_bus_read_config_byte(pci_bus, devfn, PCI_HEADER_TYPE, &header_type);
 
 		if ((header_type & 0x7F) == PCI_HEADER_TYPE_BRIDGE) {
 			/* In order to continue checking, we must program the
@@ -1076,7 +1076,7 @@ int cpqhp_valid_replace(struct controller *ctrl, struct pci_func *func)
 			 */
 
 			temp_register = func->config_space[0x18 >> 2];
-			pci_bus_write_config_dword (pci_bus, devfn, PCI_PRIMARY_BUS, temp_register);
+			pci_bus_write_config_dword(pci_bus, devfn, PCI_PRIMARY_BUS, temp_register);
 
 			secondary_bus = (temp_register >> 8) & 0xFF;
 
@@ -1094,7 +1094,7 @@ int cpqhp_valid_replace(struct controller *ctrl, struct pci_func *func)
 		/* Check to see if it is a standard config header */
 		else if ((header_type & 0x7F) == PCI_HEADER_TYPE_NORMAL) {
 			/* Check subsystem vendor and ID */
-			pci_bus_read_config_dword (pci_bus, devfn, PCI_SUBSYSTEM_VENDOR_ID, &temp_register);
+			pci_bus_read_config_dword(pci_bus, devfn, PCI_SUBSYSTEM_VENDOR_ID, &temp_register);
 
 			if (temp_register != func->config_space[0x2C >> 2]) {
 				/* If it's a SMART-2 and the register isn't
@@ -1108,8 +1108,8 @@ int cpqhp_valid_replace(struct controller *ctrl, struct pci_func *func)
 			/* Figure out IO and memory base lengths */
 			for (cloop = 0x10; cloop <= 0x24; cloop += 4) {
 				temp_register = 0xFFFFFFFF;
-				pci_bus_write_config_dword (pci_bus, devfn, cloop, temp_register);
-				pci_bus_read_config_dword (pci_bus, devfn, cloop, &base);
+				pci_bus_write_config_dword(pci_bus, devfn, cloop, temp_register);
+				pci_bus_read_config_dword(pci_bus, devfn, cloop, &base);
 
 				/* If this register is implemented */
 				if (base) {
@@ -1234,7 +1234,7 @@ int cpqhp_find_available_resources(struct controller *ctrl, void __iomem *rom_st
 	if (rc)
 		return rc;
 
-	one_slot = rom_resource_table + sizeof (struct hrt);
+	one_slot = rom_resource_table + sizeof(struct hrt);
 
 	i = readb(rom_resource_table + NUMBER_OF_ENTRIES);
 	dbg("number_of_entries = %d\n", i);
@@ -1263,12 +1263,12 @@ int cpqhp_find_available_resources(struct controller *ctrl, void __iomem *rom_st
 		/* If this entry isn't for our controller's bus, ignore it */
 		if (primary_bus != ctrl->bus) {
 			i--;
-			one_slot += sizeof (struct slot_rt);
+			one_slot += sizeof(struct slot_rt);
 			continue;
 		}
 		/* find out if this entry is for an occupied slot */
 		ctrl->pci_bus->number = primary_bus;
-		pci_bus_read_config_dword (ctrl->pci_bus, dev_func, PCI_VENDOR_ID, &temp_dword);
+		pci_bus_read_config_dword(ctrl->pci_bus, dev_func, PCI_VENDOR_ID, &temp_dword);
 		dbg("temp_D_word = %x\n", temp_dword);
 
 		if (temp_dword != 0xFFFFFFFF) {
@@ -1283,7 +1283,7 @@ int cpqhp_find_available_resources(struct controller *ctrl, void __iomem *rom_st
 			/* If we can't find a match, skip this table entry */
 			if (!func) {
 				i--;
-				one_slot += sizeof (struct slot_rt);
+				one_slot += sizeof(struct slot_rt);
 				continue;
 			}
 			/* this may not work and shouldn't be used */
@@ -1395,7 +1395,7 @@ int cpqhp_find_available_resources(struct controller *ctrl, void __iomem *rom_st
 		}
 
 		i--;
-		one_slot += sizeof (struct slot_rt);
+		one_slot += sizeof(struct slot_rt);
 	}
 
 	/* If all of the following fail, we don't have any resources for
@@ -1475,7 +1475,7 @@ int cpqhp_return_board_resources(struct pci_func *func, struct resource_lists *r
  *
  * Puts node back in the resource list pointed to by head
  */
-void cpqhp_destroy_resource_list (struct resource_lists *resources)
+void cpqhp_destroy_resource_list(struct resource_lists *resources)
 {
 	struct pci_resource *res, *tres;
 
@@ -1522,7 +1522,7 @@ void cpqhp_destroy_resource_list (struct resource_lists *resources)
  *
  * Puts node back in the resource list pointed to by head
  */
-void cpqhp_destroy_board_resources (struct pci_func *func)
+void cpqhp_destroy_board_resources(struct pci_func *func)
 {
 	struct pci_resource *res, *tres;
 
diff --git a/drivers/pci/hotplug/cpqphp_sysfs.c b/drivers/pci/hotplug/cpqphp_sysfs.c
index d81648f71425..775974deda74 100644
--- a/drivers/pci/hotplug/cpqphp_sysfs.c
+++ b/drivers/pci/hotplug/cpqphp_sysfs.c
@@ -39,7 +39,7 @@
 #include "cpqphp.h"
 
 static DEFINE_MUTEX(cpqphp_mutex);
-static int show_ctrl (struct controller *ctrl, char *buf)
+static int show_ctrl(struct controller *ctrl, char *buf)
 {
 	char *out = buf;
 	int index;
@@ -77,7 +77,7 @@ static int show_ctrl (struct controller *ctrl, char *buf)
 	return out - buf;
 }
 
-static int show_dev (struct controller *ctrl, char *buf)
+static int show_dev(struct controller *ctrl, char *buf)
 {
 	char *out = buf;
 	int index;
@@ -119,7 +119,7 @@ static int show_dev (struct controller *ctrl, char *buf)
 			out += sprintf(out, "start = %8.8x, length = %8.8x\n", res->base, res->length);
 			res = res->next;
 		}
-		slot=slot->next;
+		slot = slot->next;
 	}
 
 	return out - buf;
diff --git a/drivers/pci/hotplug/ibmphp.h b/drivers/pci/hotplug/ibmphp.h
index e3e46a7b3ee7..d3256838cb05 100644
--- a/drivers/pci/hotplug/ibmphp.h
+++ b/drivers/pci/hotplug/ibmphp.h
@@ -39,11 +39,11 @@ extern int ibmphp_debug;
 #else
 	#define MY_NAME THIS_MODULE->name
 #endif
-#define debug(fmt, arg...) do { if (ibmphp_debug == 1) printk(KERN_DEBUG "%s: " fmt , MY_NAME , ## arg); } while (0)
-#define debug_pci(fmt, arg...) do { if (ibmphp_debug) printk(KERN_DEBUG "%s: " fmt , MY_NAME , ## arg); } while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format , MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format , MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format , MY_NAME , ## arg)
+#define debug(fmt, arg...) do { if (ibmphp_debug == 1) printk(KERN_DEBUG "%s: " fmt, MY_NAME, ## arg); } while (0)
+#define debug_pci(fmt, arg...) do { if (ibmphp_debug) printk(KERN_DEBUG "%s: " fmt, MY_NAME, ## arg); } while (0)
+#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 
 /* EBDA stuff */
@@ -603,7 +603,7 @@ void ibmphp_hpc_stop_poll_thread(void);
 #define SLOT_CONNECT(s)	((u8) ((s & HPC_SLOT_CONNECT) \
 	? HPC_SLOT_DISCONNECTED : HPC_SLOT_CONNECTED))
 
-#define SLOT_ATTN(s,es)	((u8) ((es & HPC_SLOT_BLINK_ATTN) \
+#define SLOT_ATTN(s, es)	((u8) ((es & HPC_SLOT_BLINK_ATTN) \
 	? HPC_SLOT_ATTN_BLINK \
 	: ((s & HPC_SLOT_ATTN) ? HPC_SLOT_ATTN_ON : HPC_SLOT_ATTN_OFF)))
 
diff --git a/drivers/pci/hotplug/ibmphp_core.c b/drivers/pci/hotplug/ibmphp_core.c
index 15302475f5b7..5efd01d84498 100644
--- a/drivers/pci/hotplug/ibmphp_core.c
+++ b/drivers/pci/hotplug/ibmphp_core.c
@@ -39,11 +39,11 @@
 #include <asm/io_apic.h>
 #include "ibmphp.h"
 
-#define attn_on(sl)  ibmphp_hpc_writeslot (sl, HPC_SLOT_ATTNON)
-#define attn_off(sl) ibmphp_hpc_writeslot (sl, HPC_SLOT_ATTNOFF)
-#define attn_LED_blink(sl) ibmphp_hpc_writeslot (sl, HPC_SLOT_BLINKLED)
-#define get_ctrl_revision(sl, rev) ibmphp_hpc_readslot (sl, READ_REVLEVEL, rev)
-#define get_hpc_options(sl, opt) ibmphp_hpc_readslot (sl, READ_HPCOPTIONS, opt)
+#define attn_on(sl)  ibmphp_hpc_writeslot(sl, HPC_SLOT_ATTNON)
+#define attn_off(sl) ibmphp_hpc_writeslot(sl, HPC_SLOT_ATTNOFF)
+#define attn_LED_blink(sl) ibmphp_hpc_writeslot(sl, HPC_SLOT_BLINKLED)
+#define get_ctrl_revision(sl, rev) ibmphp_hpc_readslot(sl, READ_REVLEVEL, rev)
+#define get_hpc_options(sl, opt) ibmphp_hpc_readslot(sl, READ_HPCOPTIONS, opt)
 
 #define DRIVER_VERSION	"0.6"
 #define DRIVER_DESC	"IBM Hot Plug PCI Controller Driver"
@@ -52,9 +52,9 @@ int ibmphp_debug;
 
 static bool debug;
 module_param(debug, bool, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC (debug, "Debugging mode enabled or not");
-MODULE_LICENSE ("GPL");
-MODULE_DESCRIPTION (DRIVER_DESC);
+MODULE_PARM_DESC(debug, "Debugging mode enabled or not");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION(DRIVER_DESC);
 
 struct pci_bus *ibmphp_pci_bus;
 static int max_slots;
@@ -113,14 +113,12 @@ static inline int slot_update(struct slot **sl)
 	return rc;
 }
 
-static int __init get_max_slots (void)
+static int __init get_max_slots(void)
 {
 	struct slot *slot_cur;
-	struct list_head *tmp;
 	u8 slot_count = 0;
 
-	list_for_each(tmp, &ibmphp_slot_head) {
-		slot_cur = list_entry(tmp, struct slot, ibm_slot_list);
+	list_for_each_entry(slot_cur, &ibmphp_slot_head, ibm_slot_list) {
 		/* sometimes the hot-pluggable slots start with 4 (not always from 1) */
 		slot_count = max(slot_count, slot_cur->number);
 	}
@@ -459,7 +457,7 @@ static int get_max_adapter_speed_1(struct hotplug_slot *hotplug_slot, u8 *value,
 					*value = SLOT_SPEED(myslot.ext_status);
 			} else
 				*value = MAX_ADAPTER_NONE;
-                }
+		}
 	}
 
 	if (flag)
@@ -501,16 +499,10 @@ static int get_bus_name(struct hotplug_slot *hotplug_slot, char *value)
 static int __init init_ops(void)
 {
 	struct slot *slot_cur;
-	struct list_head *tmp;
 	int retval;
 	int rc;
 
-	list_for_each(tmp, &ibmphp_slot_head) {
-		slot_cur = list_entry(tmp, struct slot, ibm_slot_list);
-
-		if (!slot_cur)
-			return -ENODEV;
-
+	list_for_each_entry(slot_cur, &ibmphp_slot_head, ibm_slot_list) {
 		debug("BEFORE GETTING SLOT STATUS, slot # %x\n",
 							slot_cur->number);
 		if (slot_cur->ctrl->revision == 0xFF)
@@ -620,11 +612,11 @@ int ibmphp_update_slot_info(struct slot *slot_cur)
 	info->attention_status = SLOT_ATTN(slot_cur->status,
 						slot_cur->ext_status);
 	info->latch_status = SLOT_LATCH(slot_cur->status);
-        if (!SLOT_PRESENT(slot_cur->status)) {
-                info->adapter_status = 0;
+	if (!SLOT_PRESENT(slot_cur->status)) {
+		info->adapter_status = 0;
 /*		info->max_adapter_speed_status = MAX_ADAPTER_NONE; */
 	} else {
-                info->adapter_status = 1;
+		info->adapter_status = 1;
 /*		get_max_adapter_speed_1(slot_cur->hotplug_slot,
 					&info->max_adapter_speed_status, 0); */
 	}
@@ -669,9 +661,7 @@ static struct pci_func *ibm_slot_find(u8 busno, u8 device, u8 function)
 {
 	struct pci_func *func_cur;
 	struct slot *slot_cur;
-	struct list_head *tmp;
-	list_for_each(tmp, &ibmphp_slot_head) {
-		slot_cur = list_entry(tmp, struct slot, ibm_slot_list);
+	list_for_each_entry(slot_cur, &ibmphp_slot_head, ibm_slot_list) {
 		if (slot_cur->func) {
 			func_cur = slot_cur->func;
 			while (func_cur) {
@@ -693,14 +683,12 @@ static struct pci_func *ibm_slot_find(u8 busno, u8 device, u8 function)
  *************************************************************/
 static void free_slots(void)
 {
-	struct slot *slot_cur;
-	struct list_head *tmp;
-	struct list_head *next;
+	struct slot *slot_cur, *next;
 
 	debug("%s -- enter\n", __func__);
 
-	list_for_each_safe(tmp, next, &ibmphp_slot_head) {
-		slot_cur = list_entry(tmp, struct slot, ibm_slot_list);
+	list_for_each_entry_safe(slot_cur, next, &ibmphp_slot_head,
+				 ibm_slot_list) {
 		pci_hp_deregister(slot_cur->hotplug_slot);
 	}
 	debug("%s -- exit\n", __func__);
@@ -866,7 +854,7 @@ static int set_bus(struct slot *slot_cur)
 	int retval;
 	static struct pci_device_id ciobx[] = {
 		{ PCI_DEVICE(PCI_VENDOR_ID_SERVERWORKS, 0x0101) },
-	        { },
+		{ },
 	};
 
 	debug("%s - entry slot # %d\n", __func__, slot_cur->number);
@@ -1182,7 +1170,7 @@ error_power:
 * HOT REMOVING ADAPTER CARD                                   *
 * INPUT: POINTER TO THE HOTPLUG SLOT STRUCTURE                *
 * OUTPUT: SUCCESS 0 ; FAILURE: UNCONFIGURE , VALIDATE         *
-          DISABLE POWER ,                                    *
+*		DISABLE POWER ,                               *
 **************************************************************/
 static int ibmphp_disable_slot(struct hotplug_slot *hotplug_slot)
 {
diff --git a/drivers/pci/hotplug/ibmphp_ebda.c b/drivers/pci/hotplug/ibmphp_ebda.c
index d9b197d5c6b4..43e345ac296b 100644
--- a/drivers/pci/hotplug/ibmphp_ebda.c
+++ b/drivers/pci/hotplug/ibmphp_ebda.c
@@ -49,32 +49,32 @@
  */
 
 /* Global lists */
-LIST_HEAD (ibmphp_ebda_pci_rsrc_head);
-LIST_HEAD (ibmphp_slot_head);
+LIST_HEAD(ibmphp_ebda_pci_rsrc_head);
+LIST_HEAD(ibmphp_slot_head);
 
 /* Local variables */
 static struct ebda_hpc_list *hpc_list_ptr;
 static struct ebda_rsrc_list *rsrc_list_ptr;
 static struct rio_table_hdr *rio_table_ptr = NULL;
-static LIST_HEAD (ebda_hpc_head);
-static LIST_HEAD (bus_info_head);
-static LIST_HEAD (rio_vg_head);
-static LIST_HEAD (rio_lo_head);
-static LIST_HEAD (opt_vg_head);
-static LIST_HEAD (opt_lo_head);
+static LIST_HEAD(ebda_hpc_head);
+static LIST_HEAD(bus_info_head);
+static LIST_HEAD(rio_vg_head);
+static LIST_HEAD(rio_lo_head);
+static LIST_HEAD(opt_vg_head);
+static LIST_HEAD(opt_lo_head);
 static void __iomem *io_mem;
 
 /* Local functions */
-static int ebda_rsrc_controller (void);
-static int ebda_rsrc_rsrc (void);
-static int ebda_rio_table (void);
+static int ebda_rsrc_controller(void);
+static int ebda_rsrc_rsrc(void);
+static int ebda_rio_table(void);
 
-static struct ebda_hpc_list * __init alloc_ebda_hpc_list (void)
+static struct ebda_hpc_list * __init alloc_ebda_hpc_list(void)
 {
 	return kzalloc(sizeof(struct ebda_hpc_list), GFP_KERNEL);
 }
 
-static struct controller *alloc_ebda_hpc (u32 slot_count, u32 bus_count)
+static struct controller *alloc_ebda_hpc(u32 slot_count, u32 bus_count)
 {
 	struct controller *controller;
 	struct ebda_hpc_slot *slots;
@@ -103,146 +103,146 @@ error:
 	return NULL;
 }
 
-static void free_ebda_hpc (struct controller *controller)
+static void free_ebda_hpc(struct controller *controller)
 {
-	kfree (controller->slots);
-	kfree (controller->buses);
-	kfree (controller);
+	kfree(controller->slots);
+	kfree(controller->buses);
+	kfree(controller);
 }
 
-static struct ebda_rsrc_list * __init alloc_ebda_rsrc_list (void)
+static struct ebda_rsrc_list * __init alloc_ebda_rsrc_list(void)
 {
 	return kzalloc(sizeof(struct ebda_rsrc_list), GFP_KERNEL);
 }
 
-static struct ebda_pci_rsrc *alloc_ebda_pci_rsrc (void)
+static struct ebda_pci_rsrc *alloc_ebda_pci_rsrc(void)
 {
 	return kzalloc(sizeof(struct ebda_pci_rsrc), GFP_KERNEL);
 }
 
-static void __init print_bus_info (void)
+static void __init print_bus_info(void)
 {
 	struct bus_info *ptr;
 
 	list_for_each_entry(ptr, &bus_info_head, bus_info_list) {
-		debug ("%s - slot_min = %x\n", __func__, ptr->slot_min);
-		debug ("%s - slot_max = %x\n", __func__, ptr->slot_max);
-		debug ("%s - slot_count = %x\n", __func__, ptr->slot_count);
-		debug ("%s - bus# = %x\n", __func__, ptr->busno);
-		debug ("%s - current_speed = %x\n", __func__, ptr->current_speed);
-		debug ("%s - controller_id = %x\n", __func__, ptr->controller_id);
-
-		debug ("%s - slots_at_33_conv = %x\n", __func__, ptr->slots_at_33_conv);
-		debug ("%s - slots_at_66_conv = %x\n", __func__, ptr->slots_at_66_conv);
-		debug ("%s - slots_at_66_pcix = %x\n", __func__, ptr->slots_at_66_pcix);
-		debug ("%s - slots_at_100_pcix = %x\n", __func__, ptr->slots_at_100_pcix);
-		debug ("%s - slots_at_133_pcix = %x\n", __func__, ptr->slots_at_133_pcix);
+		debug("%s - slot_min = %x\n", __func__, ptr->slot_min);
+		debug("%s - slot_max = %x\n", __func__, ptr->slot_max);
+		debug("%s - slot_count = %x\n", __func__, ptr->slot_count);
+		debug("%s - bus# = %x\n", __func__, ptr->busno);
+		debug("%s - current_speed = %x\n", __func__, ptr->current_speed);
+		debug("%s - controller_id = %x\n", __func__, ptr->controller_id);
+
+		debug("%s - slots_at_33_conv = %x\n", __func__, ptr->slots_at_33_conv);
+		debug("%s - slots_at_66_conv = %x\n", __func__, ptr->slots_at_66_conv);
+		debug("%s - slots_at_66_pcix = %x\n", __func__, ptr->slots_at_66_pcix);
+		debug("%s - slots_at_100_pcix = %x\n", __func__, ptr->slots_at_100_pcix);
+		debug("%s - slots_at_133_pcix = %x\n", __func__, ptr->slots_at_133_pcix);
 
 	}
 }
 
-static void print_lo_info (void)
+static void print_lo_info(void)
 {
 	struct rio_detail *ptr;
-	debug ("print_lo_info ----\n");
+	debug("print_lo_info ----\n");
 	list_for_each_entry(ptr, &rio_lo_head, rio_detail_list) {
-		debug ("%s - rio_node_id = %x\n", __func__, ptr->rio_node_id);
-		debug ("%s - rio_type = %x\n", __func__, ptr->rio_type);
-		debug ("%s - owner_id = %x\n", __func__, ptr->owner_id);
-		debug ("%s - first_slot_num = %x\n", __func__, ptr->first_slot_num);
-		debug ("%s - wpindex = %x\n", __func__, ptr->wpindex);
-		debug ("%s - chassis_num = %x\n", __func__, ptr->chassis_num);
+		debug("%s - rio_node_id = %x\n", __func__, ptr->rio_node_id);
+		debug("%s - rio_type = %x\n", __func__, ptr->rio_type);
+		debug("%s - owner_id = %x\n", __func__, ptr->owner_id);
+		debug("%s - first_slot_num = %x\n", __func__, ptr->first_slot_num);
+		debug("%s - wpindex = %x\n", __func__, ptr->wpindex);
+		debug("%s - chassis_num = %x\n", __func__, ptr->chassis_num);
 
 	}
 }
 
-static void print_vg_info (void)
+static void print_vg_info(void)
 {
 	struct rio_detail *ptr;
-	debug ("%s ---\n", __func__);
+	debug("%s ---\n", __func__);
 	list_for_each_entry(ptr, &rio_vg_head, rio_detail_list) {
-		debug ("%s - rio_node_id = %x\n", __func__, ptr->rio_node_id);
-		debug ("%s - rio_type = %x\n", __func__, ptr->rio_type);
-		debug ("%s - owner_id = %x\n", __func__, ptr->owner_id);
-		debug ("%s - first_slot_num = %x\n", __func__, ptr->first_slot_num);
-		debug ("%s - wpindex = %x\n", __func__, ptr->wpindex);
-		debug ("%s - chassis_num = %x\n", __func__, ptr->chassis_num);
+		debug("%s - rio_node_id = %x\n", __func__, ptr->rio_node_id);
+		debug("%s - rio_type = %x\n", __func__, ptr->rio_type);
+		debug("%s - owner_id = %x\n", __func__, ptr->owner_id);
+		debug("%s - first_slot_num = %x\n", __func__, ptr->first_slot_num);
+		debug("%s - wpindex = %x\n", __func__, ptr->wpindex);
+		debug("%s - chassis_num = %x\n", __func__, ptr->chassis_num);
 
 	}
 }
 
-static void __init print_ebda_pci_rsrc (void)
+static void __init print_ebda_pci_rsrc(void)
 {
 	struct ebda_pci_rsrc *ptr;
 
 	list_for_each_entry(ptr, &ibmphp_ebda_pci_rsrc_head, ebda_pci_rsrc_list) {
-		debug ("%s - rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
-			__func__, ptr->rsrc_type ,ptr->bus_num, ptr->dev_fun,ptr->start_addr, ptr->end_addr);
+		debug("%s - rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
+			__func__, ptr->rsrc_type, ptr->bus_num, ptr->dev_fun, ptr->start_addr, ptr->end_addr);
 	}
 }
 
-static void __init print_ibm_slot (void)
+static void __init print_ibm_slot(void)
 {
 	struct slot *ptr;
 
 	list_for_each_entry(ptr, &ibmphp_slot_head, ibm_slot_list) {
-		debug ("%s - slot_number: %x\n", __func__, ptr->number);
+		debug("%s - slot_number: %x\n", __func__, ptr->number);
 	}
 }
 
-static void __init print_opt_vg (void)
+static void __init print_opt_vg(void)
 {
 	struct opt_rio *ptr;
-	debug ("%s ---\n", __func__);
+	debug("%s ---\n", __func__);
 	list_for_each_entry(ptr, &opt_vg_head, opt_rio_list) {
-		debug ("%s - rio_type %x\n", __func__, ptr->rio_type);
-		debug ("%s - chassis_num: %x\n", __func__, ptr->chassis_num);
-		debug ("%s - first_slot_num: %x\n", __func__, ptr->first_slot_num);
-		debug ("%s - middle_num: %x\n", __func__, ptr->middle_num);
+		debug("%s - rio_type %x\n", __func__, ptr->rio_type);
+		debug("%s - chassis_num: %x\n", __func__, ptr->chassis_num);
+		debug("%s - first_slot_num: %x\n", __func__, ptr->first_slot_num);
+		debug("%s - middle_num: %x\n", __func__, ptr->middle_num);
 	}
 }
 
-static void __init print_ebda_hpc (void)
+static void __init print_ebda_hpc(void)
 {
 	struct controller *hpc_ptr;
 	u16 index;
 
 	list_for_each_entry(hpc_ptr, &ebda_hpc_head, ebda_hpc_list) {
 		for (index = 0; index < hpc_ptr->slot_count; index++) {
-			debug ("%s - physical slot#: %x\n", __func__, hpc_ptr->slots[index].slot_num);
-			debug ("%s - pci bus# of the slot: %x\n", __func__, hpc_ptr->slots[index].slot_bus_num);
-			debug ("%s - index into ctlr addr: %x\n", __func__, hpc_ptr->slots[index].ctl_index);
-			debug ("%s - cap of the slot: %x\n", __func__, hpc_ptr->slots[index].slot_cap);
+			debug("%s - physical slot#: %x\n", __func__, hpc_ptr->slots[index].slot_num);
+			debug("%s - pci bus# of the slot: %x\n", __func__, hpc_ptr->slots[index].slot_bus_num);
+			debug("%s - index into ctlr addr: %x\n", __func__, hpc_ptr->slots[index].ctl_index);
+			debug("%s - cap of the slot: %x\n", __func__, hpc_ptr->slots[index].slot_cap);
 		}
 
 		for (index = 0; index < hpc_ptr->bus_count; index++)
-			debug ("%s - bus# of each bus controlled by this ctlr: %x\n", __func__, hpc_ptr->buses[index].bus_num);
+			debug("%s - bus# of each bus controlled by this ctlr: %x\n", __func__, hpc_ptr->buses[index].bus_num);
 
-		debug ("%s - type of hpc: %x\n", __func__, hpc_ptr->ctlr_type);
+		debug("%s - type of hpc: %x\n", __func__, hpc_ptr->ctlr_type);
 		switch (hpc_ptr->ctlr_type) {
 		case 1:
-			debug ("%s - bus: %x\n", __func__, hpc_ptr->u.pci_ctlr.bus);
-			debug ("%s - dev_fun: %x\n", __func__, hpc_ptr->u.pci_ctlr.dev_fun);
-			debug ("%s - irq: %x\n", __func__, hpc_ptr->irq);
+			debug("%s - bus: %x\n", __func__, hpc_ptr->u.pci_ctlr.bus);
+			debug("%s - dev_fun: %x\n", __func__, hpc_ptr->u.pci_ctlr.dev_fun);
+			debug("%s - irq: %x\n", __func__, hpc_ptr->irq);
 			break;
 
 		case 0:
-			debug ("%s - io_start: %x\n", __func__, hpc_ptr->u.isa_ctlr.io_start);
-			debug ("%s - io_end: %x\n", __func__, hpc_ptr->u.isa_ctlr.io_end);
-			debug ("%s - irq: %x\n", __func__, hpc_ptr->irq);
+			debug("%s - io_start: %x\n", __func__, hpc_ptr->u.isa_ctlr.io_start);
+			debug("%s - io_end: %x\n", __func__, hpc_ptr->u.isa_ctlr.io_end);
+			debug("%s - irq: %x\n", __func__, hpc_ptr->irq);
 			break;
 
 		case 2:
 		case 4:
-			debug ("%s - wpegbbar: %lx\n", __func__, hpc_ptr->u.wpeg_ctlr.wpegbbar);
-			debug ("%s - i2c_addr: %x\n", __func__, hpc_ptr->u.wpeg_ctlr.i2c_addr);
-			debug ("%s - irq: %x\n", __func__, hpc_ptr->irq);
+			debug("%s - wpegbbar: %lx\n", __func__, hpc_ptr->u.wpeg_ctlr.wpegbbar);
+			debug("%s - i2c_addr: %x\n", __func__, hpc_ptr->u.wpeg_ctlr.i2c_addr);
+			debug("%s - irq: %x\n", __func__, hpc_ptr->irq);
 			break;
 		}
 	}
 }
 
-int __init ibmphp_access_ebda (void)
+int __init ibmphp_access_ebda(void)
 {
 	u8 format, num_ctlrs, rio_complete, hs_complete, ebda_sz;
 	u16 ebda_seg, num_entries, next_offset, offset, blk_id, sub_addr, re, rc_id, re_id, base;
@@ -252,12 +252,12 @@ int __init ibmphp_access_ebda (void)
 	rio_complete = 0;
 	hs_complete = 0;
 
-	io_mem = ioremap ((0x40 << 4) + 0x0e, 2);
-	if (!io_mem )
+	io_mem = ioremap((0x40 << 4) + 0x0e, 2);
+	if (!io_mem)
 		return -ENOMEM;
-	ebda_seg = readw (io_mem);
-	iounmap (io_mem);
-	debug ("returned ebda segment: %x\n", ebda_seg);
+	ebda_seg = readw(io_mem);
+	iounmap(io_mem);
+	debug("returned ebda segment: %x\n", ebda_seg);
 
 	io_mem = ioremap(ebda_seg<<4, 1);
 	if (!io_mem)
@@ -269,7 +269,7 @@ int __init ibmphp_access_ebda (void)
 		return -ENOMEM;
 
 	io_mem = ioremap(ebda_seg<<4, (ebda_sz * 1024));
-	if (!io_mem )
+	if (!io_mem)
 		return -ENOMEM;
 	next_offset = 0x180;
 
@@ -281,12 +281,12 @@ int __init ibmphp_access_ebda (void)
 			 "ibmphp_ebda: next read is beyond ebda_sz\n"))
 			break;
 
-		next_offset = readw (io_mem + offset);	/* offset of next blk */
+		next_offset = readw(io_mem + offset);	/* offset of next blk */
 
 		offset += 2;
 		if (next_offset == 0)	/* 0 indicate it's last blk */
 			break;
-		blk_id = readw (io_mem + offset);	/* this blk id */
+		blk_id = readw(io_mem + offset);	/* this blk id */
 
 		offset += 2;
 		/* check if it is hot swap block or rio block */
@@ -294,31 +294,31 @@ int __init ibmphp_access_ebda (void)
 			continue;
 		/* found hs table */
 		if (blk_id == 0x4853) {
-			debug ("now enter hot swap block---\n");
-			debug ("hot blk id: %x\n", blk_id);
-			format = readb (io_mem + offset);
+			debug("now enter hot swap block---\n");
+			debug("hot blk id: %x\n", blk_id);
+			format = readb(io_mem + offset);
 
 			offset += 1;
 			if (format != 4)
 				goto error_nodev;
-			debug ("hot blk format: %x\n", format);
+			debug("hot blk format: %x\n", format);
 			/* hot swap sub blk */
 			base = offset;
 
 			sub_addr = base;
-			re = readw (io_mem + sub_addr);	/* next sub blk */
+			re = readw(io_mem + sub_addr);	/* next sub blk */
 
 			sub_addr += 2;
-			rc_id = readw (io_mem + sub_addr);	/* sub blk id */
+			rc_id = readw(io_mem + sub_addr);	/* sub blk id */
 
 			sub_addr += 2;
 			if (rc_id != 0x5243)
 				goto error_nodev;
 			/* rc sub blk signature  */
-			num_ctlrs = readb (io_mem + sub_addr);
+			num_ctlrs = readb(io_mem + sub_addr);
 
 			sub_addr += 1;
-			hpc_list_ptr = alloc_ebda_hpc_list ();
+			hpc_list_ptr = alloc_ebda_hpc_list();
 			if (!hpc_list_ptr) {
 				rc = -ENOMEM;
 				goto out;
@@ -326,28 +326,28 @@ int __init ibmphp_access_ebda (void)
 			hpc_list_ptr->format = format;
 			hpc_list_ptr->num_ctlrs = num_ctlrs;
 			hpc_list_ptr->phys_addr = sub_addr;	/*  offset of RSRC_CONTROLLER blk */
-			debug ("info about hpc descriptor---\n");
-			debug ("hot blk format: %x\n", format);
-			debug ("num of controller: %x\n", num_ctlrs);
-			debug ("offset of hpc data structure entries: %x\n ", sub_addr);
+			debug("info about hpc descriptor---\n");
+			debug("hot blk format: %x\n", format);
+			debug("num of controller: %x\n", num_ctlrs);
+			debug("offset of hpc data structure entries: %x\n ", sub_addr);
 
 			sub_addr = base + re;	/* re sub blk */
 			/* FIXME: rc is never used/checked */
-			rc = readw (io_mem + sub_addr);	/* next sub blk */
+			rc = readw(io_mem + sub_addr);	/* next sub blk */
 
 			sub_addr += 2;
-			re_id = readw (io_mem + sub_addr);	/* sub blk id */
+			re_id = readw(io_mem + sub_addr);	/* sub blk id */
 
 			sub_addr += 2;
 			if (re_id != 0x5245)
 				goto error_nodev;
 
 			/* signature of re */
-			num_entries = readw (io_mem + sub_addr);
+			num_entries = readw(io_mem + sub_addr);
 
 			sub_addr += 2;	/* offset of RSRC_ENTRIES blk */
-			rsrc_list_ptr = alloc_ebda_rsrc_list ();
-			if (!rsrc_list_ptr ) {
+			rsrc_list_ptr = alloc_ebda_rsrc_list();
+			if (!rsrc_list_ptr) {
 				rc = -ENOMEM;
 				goto out;
 			}
@@ -355,26 +355,26 @@ int __init ibmphp_access_ebda (void)
 			rsrc_list_ptr->num_entries = num_entries;
 			rsrc_list_ptr->phys_addr = sub_addr;
 
-			debug ("info about rsrc descriptor---\n");
-			debug ("format: %x\n", format);
-			debug ("num of rsrc: %x\n", num_entries);
-			debug ("offset of rsrc data structure entries: %x\n ", sub_addr);
+			debug("info about rsrc descriptor---\n");
+			debug("format: %x\n", format);
+			debug("num of rsrc: %x\n", num_entries);
+			debug("offset of rsrc data structure entries: %x\n ", sub_addr);
 
 			hs_complete = 1;
 		} else {
 		/* found rio table, blk_id == 0x4752 */
-			debug ("now enter io table ---\n");
-			debug ("rio blk id: %x\n", blk_id);
+			debug("now enter io table ---\n");
+			debug("rio blk id: %x\n", blk_id);
 
 			rio_table_ptr = kzalloc(sizeof(struct rio_table_hdr), GFP_KERNEL);
 			if (!rio_table_ptr) {
 				rc = -ENOMEM;
 				goto out;
 			}
-			rio_table_ptr->ver_num = readb (io_mem + offset);
-			rio_table_ptr->scal_count = readb (io_mem + offset + 1);
-			rio_table_ptr->riodev_count = readb (io_mem + offset + 2);
-			rio_table_ptr->offset = offset +3 ;
+			rio_table_ptr->ver_num = readb(io_mem + offset);
+			rio_table_ptr->scal_count = readb(io_mem + offset + 1);
+			rio_table_ptr->riodev_count = readb(io_mem + offset + 2);
+			rio_table_ptr->offset = offset + 3 ;
 
 			debug("info about rio table hdr ---\n");
 			debug("ver_num: %x\nscal_count: %x\nriodev_count: %x\noffset of rio table: %x\n ",
@@ -390,28 +390,28 @@ int __init ibmphp_access_ebda (void)
 
 	if (rio_table_ptr) {
 		if (rio_complete && rio_table_ptr->ver_num == 3) {
-			rc = ebda_rio_table ();
+			rc = ebda_rio_table();
 			if (rc)
 				goto out;
 		}
 	}
-	rc = ebda_rsrc_controller ();
+	rc = ebda_rsrc_controller();
 	if (rc)
 		goto out;
 
-	rc = ebda_rsrc_rsrc ();
+	rc = ebda_rsrc_rsrc();
 	goto out;
 error_nodev:
 	rc = -ENODEV;
 out:
-	iounmap (io_mem);
+	iounmap(io_mem);
 	return rc;
 }
 
 /*
  * map info of scalability details and rio details from physical address
  */
-static int __init ebda_rio_table (void)
+static int __init ebda_rio_table(void)
 {
 	u16 offset;
 	u8 i;
@@ -425,39 +425,39 @@ static int __init ebda_rio_table (void)
 		rio_detail_ptr = kzalloc(sizeof(struct rio_detail), GFP_KERNEL);
 		if (!rio_detail_ptr)
 			return -ENOMEM;
-		rio_detail_ptr->rio_node_id = readb (io_mem + offset);
-		rio_detail_ptr->bbar = readl (io_mem + offset + 1);
-		rio_detail_ptr->rio_type = readb (io_mem + offset + 5);
-		rio_detail_ptr->owner_id = readb (io_mem + offset + 6);
-		rio_detail_ptr->port0_node_connect = readb (io_mem + offset + 7);
-		rio_detail_ptr->port0_port_connect = readb (io_mem + offset + 8);
-		rio_detail_ptr->port1_node_connect = readb (io_mem + offset + 9);
-		rio_detail_ptr->port1_port_connect = readb (io_mem + offset + 10);
-		rio_detail_ptr->first_slot_num = readb (io_mem + offset + 11);
-		rio_detail_ptr->status = readb (io_mem + offset + 12);
-		rio_detail_ptr->wpindex = readb (io_mem + offset + 13);
-		rio_detail_ptr->chassis_num = readb (io_mem + offset + 14);
-//		debug ("rio_node_id: %x\nbbar: %x\nrio_type: %x\nowner_id: %x\nport0_node: %x\nport0_port: %x\nport1_node: %x\nport1_port: %x\nfirst_slot_num: %x\nstatus: %x\n", rio_detail_ptr->rio_node_id, rio_detail_ptr->bbar, rio_detail_ptr->rio_type, rio_detail_ptr->owner_id, rio_detail_ptr->port0_node_connect, rio_detail_ptr->port0_port_connect, rio_detail_ptr->port1_node_connect, rio_detail_ptr->port1_port_connect, rio_detail_ptr->first_slot_num, rio_detail_ptr->status);
+		rio_detail_ptr->rio_node_id = readb(io_mem + offset);
+		rio_detail_ptr->bbar = readl(io_mem + offset + 1);
+		rio_detail_ptr->rio_type = readb(io_mem + offset + 5);
+		rio_detail_ptr->owner_id = readb(io_mem + offset + 6);
+		rio_detail_ptr->port0_node_connect = readb(io_mem + offset + 7);
+		rio_detail_ptr->port0_port_connect = readb(io_mem + offset + 8);
+		rio_detail_ptr->port1_node_connect = readb(io_mem + offset + 9);
+		rio_detail_ptr->port1_port_connect = readb(io_mem + offset + 10);
+		rio_detail_ptr->first_slot_num = readb(io_mem + offset + 11);
+		rio_detail_ptr->status = readb(io_mem + offset + 12);
+		rio_detail_ptr->wpindex = readb(io_mem + offset + 13);
+		rio_detail_ptr->chassis_num = readb(io_mem + offset + 14);
+//		debug("rio_node_id: %x\nbbar: %x\nrio_type: %x\nowner_id: %x\nport0_node: %x\nport0_port: %x\nport1_node: %x\nport1_port: %x\nfirst_slot_num: %x\nstatus: %x\n", rio_detail_ptr->rio_node_id, rio_detail_ptr->bbar, rio_detail_ptr->rio_type, rio_detail_ptr->owner_id, rio_detail_ptr->port0_node_connect, rio_detail_ptr->port0_port_connect, rio_detail_ptr->port1_node_connect, rio_detail_ptr->port1_port_connect, rio_detail_ptr->first_slot_num, rio_detail_ptr->status);
 		//create linked list of chassis
 		if (rio_detail_ptr->rio_type == 4 || rio_detail_ptr->rio_type == 5)
-			list_add (&rio_detail_ptr->rio_detail_list, &rio_vg_head);
+			list_add(&rio_detail_ptr->rio_detail_list, &rio_vg_head);
 		//create linked list of expansion box
 		else if (rio_detail_ptr->rio_type == 6 || rio_detail_ptr->rio_type == 7)
-			list_add (&rio_detail_ptr->rio_detail_list, &rio_lo_head);
+			list_add(&rio_detail_ptr->rio_detail_list, &rio_lo_head);
 		else
 			// not in my concern
-			kfree (rio_detail_ptr);
+			kfree(rio_detail_ptr);
 		offset += 15;
 	}
-	print_lo_info ();
-	print_vg_info ();
+	print_lo_info();
+	print_vg_info();
 	return 0;
 }
 
 /*
  * reorganizing linked list of chassis
  */
-static struct opt_rio *search_opt_vg (u8 chassis_num)
+static struct opt_rio *search_opt_vg(u8 chassis_num)
 {
 	struct opt_rio *ptr;
 	list_for_each_entry(ptr, &opt_vg_head, opt_rio_list) {
@@ -467,13 +467,13 @@ static struct opt_rio *search_opt_vg (u8 chassis_num)
 	return NULL;
 }
 
-static int __init combine_wpg_for_chassis (void)
+static int __init combine_wpg_for_chassis(void)
 {
 	struct opt_rio *opt_rio_ptr = NULL;
 	struct rio_detail *rio_detail_ptr = NULL;
 
 	list_for_each_entry(rio_detail_ptr, &rio_vg_head, rio_detail_list) {
-		opt_rio_ptr = search_opt_vg (rio_detail_ptr->chassis_num);
+		opt_rio_ptr = search_opt_vg(rio_detail_ptr->chassis_num);
 		if (!opt_rio_ptr) {
 			opt_rio_ptr = kzalloc(sizeof(struct opt_rio), GFP_KERNEL);
 			if (!opt_rio_ptr)
@@ -482,20 +482,20 @@ static int __init combine_wpg_for_chassis (void)
 			opt_rio_ptr->chassis_num = rio_detail_ptr->chassis_num;
 			opt_rio_ptr->first_slot_num = rio_detail_ptr->first_slot_num;
 			opt_rio_ptr->middle_num = rio_detail_ptr->first_slot_num;
-			list_add (&opt_rio_ptr->opt_rio_list, &opt_vg_head);
+			list_add(&opt_rio_ptr->opt_rio_list, &opt_vg_head);
 		} else {
-			opt_rio_ptr->first_slot_num = min (opt_rio_ptr->first_slot_num, rio_detail_ptr->first_slot_num);
-			opt_rio_ptr->middle_num = max (opt_rio_ptr->middle_num, rio_detail_ptr->first_slot_num);
+			opt_rio_ptr->first_slot_num = min(opt_rio_ptr->first_slot_num, rio_detail_ptr->first_slot_num);
+			opt_rio_ptr->middle_num = max(opt_rio_ptr->middle_num, rio_detail_ptr->first_slot_num);
 		}
 	}
-	print_opt_vg ();
+	print_opt_vg();
 	return 0;
 }
 
 /*
  * reorganizing linked list of expansion box
  */
-static struct opt_rio_lo *search_opt_lo (u8 chassis_num)
+static struct opt_rio_lo *search_opt_lo(u8 chassis_num)
 {
 	struct opt_rio_lo *ptr;
 	list_for_each_entry(ptr, &opt_lo_head, opt_rio_lo_list) {
@@ -505,13 +505,13 @@ static struct opt_rio_lo *search_opt_lo (u8 chassis_num)
 	return NULL;
 }
 
-static int combine_wpg_for_expansion (void)
+static int combine_wpg_for_expansion(void)
 {
 	struct opt_rio_lo *opt_rio_lo_ptr = NULL;
 	struct rio_detail *rio_detail_ptr = NULL;
 
 	list_for_each_entry(rio_detail_ptr, &rio_lo_head, rio_detail_list) {
-		opt_rio_lo_ptr = search_opt_lo (rio_detail_ptr->chassis_num);
+		opt_rio_lo_ptr = search_opt_lo(rio_detail_ptr->chassis_num);
 		if (!opt_rio_lo_ptr) {
 			opt_rio_lo_ptr = kzalloc(sizeof(struct opt_rio_lo), GFP_KERNEL);
 			if (!opt_rio_lo_ptr)
@@ -522,10 +522,10 @@ static int combine_wpg_for_expansion (void)
 			opt_rio_lo_ptr->middle_num = rio_detail_ptr->first_slot_num;
 			opt_rio_lo_ptr->pack_count = 1;
 
-			list_add (&opt_rio_lo_ptr->opt_rio_lo_list, &opt_lo_head);
+			list_add(&opt_rio_lo_ptr->opt_rio_lo_list, &opt_lo_head);
 		} else {
-			opt_rio_lo_ptr->first_slot_num = min (opt_rio_lo_ptr->first_slot_num, rio_detail_ptr->first_slot_num);
-			opt_rio_lo_ptr->middle_num = max (opt_rio_lo_ptr->middle_num, rio_detail_ptr->first_slot_num);
+			opt_rio_lo_ptr->first_slot_num = min(opt_rio_lo_ptr->first_slot_num, rio_detail_ptr->first_slot_num);
+			opt_rio_lo_ptr->middle_num = max(opt_rio_lo_ptr->middle_num, rio_detail_ptr->first_slot_num);
 			opt_rio_lo_ptr->pack_count = 2;
 		}
 	}
@@ -538,7 +538,7 @@ static int combine_wpg_for_expansion (void)
  * Arguments: slot_num, 1st slot number of the chassis we think we are on,
  * var (0 = chassis, 1 = expansion box)
  */
-static int first_slot_num (u8 slot_num, u8 first_slot, u8 var)
+static int first_slot_num(u8 slot_num, u8 first_slot, u8 var)
 {
 	struct opt_rio *opt_vg_ptr = NULL;
 	struct opt_rio_lo *opt_lo_ptr = NULL;
@@ -562,25 +562,25 @@ static int first_slot_num (u8 slot_num, u8 first_slot, u8 var)
 	return rc;
 }
 
-static struct opt_rio_lo *find_rxe_num (u8 slot_num)
+static struct opt_rio_lo *find_rxe_num(u8 slot_num)
 {
 	struct opt_rio_lo *opt_lo_ptr;
 
 	list_for_each_entry(opt_lo_ptr, &opt_lo_head, opt_rio_lo_list) {
 		//check to see if this slot_num belongs to expansion box
-		if ((slot_num >= opt_lo_ptr->first_slot_num) && (!first_slot_num (slot_num, opt_lo_ptr->first_slot_num, 1)))
+		if ((slot_num >= opt_lo_ptr->first_slot_num) && (!first_slot_num(slot_num, opt_lo_ptr->first_slot_num, 1)))
 			return opt_lo_ptr;
 	}
 	return NULL;
 }
 
-static struct opt_rio *find_chassis_num (u8 slot_num)
+static struct opt_rio *find_chassis_num(u8 slot_num)
 {
 	struct opt_rio *opt_vg_ptr;
 
 	list_for_each_entry(opt_vg_ptr, &opt_vg_head, opt_rio_list) {
 		//check to see if this slot_num belongs to chassis
-		if ((slot_num >= opt_vg_ptr->first_slot_num) && (!first_slot_num (slot_num, opt_vg_ptr->first_slot_num, 0)))
+		if ((slot_num >= opt_vg_ptr->first_slot_num) && (!first_slot_num(slot_num, opt_vg_ptr->first_slot_num, 0)))
 			return opt_vg_ptr;
 	}
 	return NULL;
@@ -589,7 +589,7 @@ static struct opt_rio *find_chassis_num (u8 slot_num)
 /* This routine will find out how many slots are in the chassis, so that
  * the slot numbers for rxe100 would start from 1, and not from 7, or 6 etc
  */
-static u8 calculate_first_slot (u8 slot_num)
+static u8 calculate_first_slot(u8 slot_num)
 {
 	u8 first_slot = 1;
 	struct slot *slot_cur;
@@ -606,7 +606,7 @@ static u8 calculate_first_slot (u8 slot_num)
 
 #define SLOT_NAME_SIZE 30
 
-static char *create_file_name (struct slot *slot_cur)
+static char *create_file_name(struct slot *slot_cur)
 {
 	struct opt_rio *opt_vg_ptr = NULL;
 	struct opt_rio_lo *opt_lo_ptr = NULL;
@@ -618,18 +618,18 @@ static char *create_file_name (struct slot *slot_cur)
 	u8 flag = 0;
 
 	if (!slot_cur) {
-		err ("Structure passed is empty\n");
+		err("Structure passed is empty\n");
 		return NULL;
 	}
 
 	slot_num = slot_cur->number;
 
-	memset (str, 0, sizeof(str));
+	memset(str, 0, sizeof(str));
 
 	if (rio_table_ptr) {
 		if (rio_table_ptr->ver_num == 3) {
-			opt_vg_ptr = find_chassis_num (slot_num);
-			opt_lo_ptr = find_rxe_num (slot_num);
+			opt_vg_ptr = find_chassis_num(slot_num);
+			opt_lo_ptr = find_rxe_num(slot_num);
 		}
 	}
 	if (opt_vg_ptr) {
@@ -662,7 +662,7 @@ static char *create_file_name (struct slot *slot_cur)
 	}
 	if (!flag) {
 		if (slot_cur->ctrl->ctlr_type == 4) {
-			first_slot = calculate_first_slot (slot_num);
+			first_slot = calculate_first_slot(slot_num);
 			which = 1;
 		} else {
 			which = 0;
@@ -698,7 +698,7 @@ static int fillslotinfo(struct hotplug_slot *hotplug_slot)
 	hotplug_slot->info->latch_status = SLOT_LATCH(slot->status);
 
 	// pci board - present:1 not:0
-	if (SLOT_PRESENT (slot->status))
+	if (SLOT_PRESENT(slot->status))
 		hotplug_slot->info->adapter_status = 1;
 	else
 		hotplug_slot->info->adapter_status = 0;
@@ -729,7 +729,7 @@ static void release_slot(struct hotplug_slot *hotplug_slot)
 	/* we don't want to actually remove the resources, since free_resources will do just that */
 	ibmphp_unconfigure_card(&slot, -1);
 
-	kfree (slot);
+	kfree(slot);
 }
 
 static struct pci_driver ibmphp_driver;
@@ -739,7 +739,7 @@ static struct pci_driver ibmphp_driver;
  * each hpc from physical address to a list of hot plug controllers based on
  * hpc descriptors.
  */
-static int __init ebda_rsrc_controller (void)
+static int __init ebda_rsrc_controller(void)
 {
 	u16 addr, addr_slot, addr_bus;
 	u8 ctlr_id, temp, bus_index;
@@ -757,25 +757,25 @@ static int __init ebda_rsrc_controller (void)
 	addr = hpc_list_ptr->phys_addr;
 	for (ctlr = 0; ctlr < hpc_list_ptr->num_ctlrs; ctlr++) {
 		bus_index = 1;
-		ctlr_id = readb (io_mem + addr);
+		ctlr_id = readb(io_mem + addr);
 		addr += 1;
-		slot_num = readb (io_mem + addr);
+		slot_num = readb(io_mem + addr);
 
 		addr += 1;
 		addr_slot = addr;	/* offset of slot structure */
 		addr += (slot_num * 4);
 
-		bus_num = readb (io_mem + addr);
+		bus_num = readb(io_mem + addr);
 
 		addr += 1;
 		addr_bus = addr;	/* offset of bus */
 		addr += (bus_num * 9);	/* offset of ctlr_type */
-		temp = readb (io_mem + addr);
+		temp = readb(io_mem + addr);
 
 		addr += 1;
 		/* init hpc structure */
-		hpc_ptr = alloc_ebda_hpc (slot_num, bus_num);
-		if (!hpc_ptr ) {
+		hpc_ptr = alloc_ebda_hpc(slot_num, bus_num);
+		if (!hpc_ptr) {
 			rc = -ENOMEM;
 			goto error_no_hpc;
 		}
@@ -783,23 +783,23 @@ static int __init ebda_rsrc_controller (void)
 		hpc_ptr->ctlr_relative_id = ctlr;
 		hpc_ptr->slot_count = slot_num;
 		hpc_ptr->bus_count = bus_num;
-		debug ("now enter ctlr data structure ---\n");
-		debug ("ctlr id: %x\n", ctlr_id);
-		debug ("ctlr_relative_id: %x\n", hpc_ptr->ctlr_relative_id);
-		debug ("count of slots controlled by this ctlr: %x\n", slot_num);
-		debug ("count of buses controlled by this ctlr: %x\n", bus_num);
+		debug("now enter ctlr data structure ---\n");
+		debug("ctlr id: %x\n", ctlr_id);
+		debug("ctlr_relative_id: %x\n", hpc_ptr->ctlr_relative_id);
+		debug("count of slots controlled by this ctlr: %x\n", slot_num);
+		debug("count of buses controlled by this ctlr: %x\n", bus_num);
 
 		/* init slot structure, fetch slot, bus, cap... */
 		slot_ptr = hpc_ptr->slots;
 		for (slot = 0; slot < slot_num; slot++) {
-			slot_ptr->slot_num = readb (io_mem + addr_slot);
-			slot_ptr->slot_bus_num = readb (io_mem + addr_slot + slot_num);
-			slot_ptr->ctl_index = readb (io_mem + addr_slot + 2*slot_num);
-			slot_ptr->slot_cap = readb (io_mem + addr_slot + 3*slot_num);
+			slot_ptr->slot_num = readb(io_mem + addr_slot);
+			slot_ptr->slot_bus_num = readb(io_mem + addr_slot + slot_num);
+			slot_ptr->ctl_index = readb(io_mem + addr_slot + 2*slot_num);
+			slot_ptr->slot_cap = readb(io_mem + addr_slot + 3*slot_num);
 
 			// create bus_info lined list --- if only one slot per bus: slot_min = slot_max
 
-			bus_info_ptr2 = ibmphp_find_same_bus_num (slot_ptr->slot_bus_num);
+			bus_info_ptr2 = ibmphp_find_same_bus_num(slot_ptr->slot_bus_num);
 			if (!bus_info_ptr2) {
 				bus_info_ptr1 = kzalloc(sizeof(struct bus_info), GFP_KERNEL);
 				if (!bus_info_ptr1) {
@@ -816,11 +816,11 @@ static int __init ebda_rsrc_controller (void)
 
 				bus_info_ptr1->controller_id = hpc_ptr->ctlr_id;
 
-				list_add_tail (&bus_info_ptr1->bus_info_list, &bus_info_head);
+				list_add_tail(&bus_info_ptr1->bus_info_list, &bus_info_head);
 
 			} else {
-				bus_info_ptr2->slot_min = min (bus_info_ptr2->slot_min, slot_ptr->slot_num);
-				bus_info_ptr2->slot_max = max (bus_info_ptr2->slot_max, slot_ptr->slot_num);
+				bus_info_ptr2->slot_min = min(bus_info_ptr2->slot_min, slot_ptr->slot_num);
+				bus_info_ptr2->slot_max = max(bus_info_ptr2->slot_max, slot_ptr->slot_num);
 				bus_info_ptr2->slot_count += 1;
 
 			}
@@ -834,17 +834,17 @@ static int __init ebda_rsrc_controller (void)
 		/* init bus structure */
 		bus_ptr = hpc_ptr->buses;
 		for (bus = 0; bus < bus_num; bus++) {
-			bus_ptr->bus_num = readb (io_mem + addr_bus + bus);
-			bus_ptr->slots_at_33_conv = readb (io_mem + addr_bus + bus_num + 8 * bus);
-			bus_ptr->slots_at_66_conv = readb (io_mem + addr_bus + bus_num + 8 * bus + 1);
+			bus_ptr->bus_num = readb(io_mem + addr_bus + bus);
+			bus_ptr->slots_at_33_conv = readb(io_mem + addr_bus + bus_num + 8 * bus);
+			bus_ptr->slots_at_66_conv = readb(io_mem + addr_bus + bus_num + 8 * bus + 1);
 
-			bus_ptr->slots_at_66_pcix = readb (io_mem + addr_bus + bus_num + 8 * bus + 2);
+			bus_ptr->slots_at_66_pcix = readb(io_mem + addr_bus + bus_num + 8 * bus + 2);
 
-			bus_ptr->slots_at_100_pcix = readb (io_mem + addr_bus + bus_num + 8 * bus + 3);
+			bus_ptr->slots_at_100_pcix = readb(io_mem + addr_bus + bus_num + 8 * bus + 3);
 
-			bus_ptr->slots_at_133_pcix = readb (io_mem + addr_bus + bus_num + 8 * bus + 4);
+			bus_ptr->slots_at_133_pcix = readb(io_mem + addr_bus + bus_num + 8 * bus + 4);
 
-			bus_info_ptr2 = ibmphp_find_same_bus_num (bus_ptr->bus_num);
+			bus_info_ptr2 = ibmphp_find_same_bus_num(bus_ptr->bus_num);
 			if (bus_info_ptr2) {
 				bus_info_ptr2->slots_at_33_conv = bus_ptr->slots_at_33_conv;
 				bus_info_ptr2->slots_at_66_conv = bus_ptr->slots_at_66_conv;
@@ -859,33 +859,33 @@ static int __init ebda_rsrc_controller (void)
 
 		switch (hpc_ptr->ctlr_type) {
 			case 1:
-				hpc_ptr->u.pci_ctlr.bus = readb (io_mem + addr);
-				hpc_ptr->u.pci_ctlr.dev_fun = readb (io_mem + addr + 1);
-				hpc_ptr->irq = readb (io_mem + addr + 2);
+				hpc_ptr->u.pci_ctlr.bus = readb(io_mem + addr);
+				hpc_ptr->u.pci_ctlr.dev_fun = readb(io_mem + addr + 1);
+				hpc_ptr->irq = readb(io_mem + addr + 2);
 				addr += 3;
-				debug ("ctrl bus = %x, ctlr devfun = %x, irq = %x\n",
+				debug("ctrl bus = %x, ctlr devfun = %x, irq = %x\n",
 					hpc_ptr->u.pci_ctlr.bus,
 					hpc_ptr->u.pci_ctlr.dev_fun, hpc_ptr->irq);
 				break;
 
 			case 0:
-				hpc_ptr->u.isa_ctlr.io_start = readw (io_mem + addr);
-				hpc_ptr->u.isa_ctlr.io_end = readw (io_mem + addr + 2);
-				if (!request_region (hpc_ptr->u.isa_ctlr.io_start,
+				hpc_ptr->u.isa_ctlr.io_start = readw(io_mem + addr);
+				hpc_ptr->u.isa_ctlr.io_end = readw(io_mem + addr + 2);
+				if (!request_region(hpc_ptr->u.isa_ctlr.io_start,
 						     (hpc_ptr->u.isa_ctlr.io_end - hpc_ptr->u.isa_ctlr.io_start + 1),
 						     "ibmphp")) {
 					rc = -ENODEV;
 					goto error_no_hp_slot;
 				}
-				hpc_ptr->irq = readb (io_mem + addr + 4);
+				hpc_ptr->irq = readb(io_mem + addr + 4);
 				addr += 5;
 				break;
 
 			case 2:
 			case 4:
-				hpc_ptr->u.wpeg_ctlr.wpegbbar = readl (io_mem + addr);
-				hpc_ptr->u.wpeg_ctlr.i2c_addr = readb (io_mem + addr + 4);
-				hpc_ptr->irq = readb (io_mem + addr + 5);
+				hpc_ptr->u.wpeg_ctlr.wpegbbar = readl(io_mem + addr);
+				hpc_ptr->u.wpeg_ctlr.i2c_addr = readb(io_mem + addr + 4);
+				hpc_ptr->irq = readb(io_mem + addr + 5);
 				addr += 6;
 				break;
 			default:
@@ -894,8 +894,8 @@ static int __init ebda_rsrc_controller (void)
 		}
 
 		//reorganize chassis' linked list
-		combine_wpg_for_chassis ();
-		combine_wpg_for_expansion ();
+		combine_wpg_for_chassis();
+		combine_wpg_for_expansion();
 		hpc_ptr->revision = 0xff;
 		hpc_ptr->options = 0xff;
 		hpc_ptr->starting_slot_num = hpc_ptr->slots[0].slot_num;
@@ -940,7 +940,7 @@ static int __init ebda_rsrc_controller (void)
 
 			tmp_slot->bus = hpc_ptr->slots[index].slot_bus_num;
 
-			bus_info_ptr1 = ibmphp_find_same_bus_num (hpc_ptr->slots[index].slot_bus_num);
+			bus_info_ptr1 = ibmphp_find_same_bus_num(hpc_ptr->slots[index].slot_bus_num);
 			if (!bus_info_ptr1) {
 				kfree(tmp_slot);
 				rc = -ENODEV;
@@ -961,18 +961,18 @@ static int __init ebda_rsrc_controller (void)
 			if (rc)
 				goto error;
 
-			rc = ibmphp_init_devno ((struct slot **) &hp_slot_ptr->private);
+			rc = ibmphp_init_devno((struct slot **) &hp_slot_ptr->private);
 			if (rc)
 				goto error;
 			hp_slot_ptr->ops = &ibmphp_hotplug_slot_ops;
 
 			// end of registering ibm slot with hotplug core
 
-			list_add (& ((struct slot *)(hp_slot_ptr->private))->ibm_slot_list, &ibmphp_slot_head);
+			list_add(&((struct slot *)(hp_slot_ptr->private))->ibm_slot_list, &ibmphp_slot_head);
 		}
 
-		print_bus_info ();
-		list_add (&hpc_ptr->ebda_hpc_list, &ebda_hpc_head );
+		print_bus_info();
+		list_add(&hpc_ptr->ebda_hpc_list, &ebda_hpc_head);
 
 	}			/* each hpc  */
 
@@ -982,20 +982,20 @@ static int __init ebda_rsrc_controller (void)
 			pci_find_bus(0, tmp_slot->bus), tmp_slot->device, name);
 	}
 
-	print_ebda_hpc ();
-	print_ibm_slot ();
+	print_ebda_hpc();
+	print_ibm_slot();
 	return 0;
 
 error:
-	kfree (hp_slot_ptr->private);
+	kfree(hp_slot_ptr->private);
 error_no_slot:
-	kfree (hp_slot_ptr->info);
+	kfree(hp_slot_ptr->info);
 error_no_hp_info:
-	kfree (hp_slot_ptr);
+	kfree(hp_slot_ptr);
 error_no_hp_slot:
-	free_ebda_hpc (hpc_ptr);
+	free_ebda_hpc(hpc_ptr);
 error_no_hpc:
-	iounmap (io_mem);
+	iounmap(io_mem);
 	return rc;
 }
 
@@ -1003,7 +1003,7 @@ error_no_hpc:
  * map info (bus, devfun, start addr, end addr..) of i/o, memory,
  * pfm from the physical addr to a list of resource.
  */
-static int __init ebda_rsrc_rsrc (void)
+static int __init ebda_rsrc_rsrc(void)
 {
 	u16 addr;
 	short rsrc;
@@ -1011,69 +1011,69 @@ static int __init ebda_rsrc_rsrc (void)
 	struct ebda_pci_rsrc *rsrc_ptr;
 
 	addr = rsrc_list_ptr->phys_addr;
-	debug ("now entering rsrc land\n");
-	debug ("offset of rsrc: %x\n", rsrc_list_ptr->phys_addr);
+	debug("now entering rsrc land\n");
+	debug("offset of rsrc: %x\n", rsrc_list_ptr->phys_addr);
 
 	for (rsrc = 0; rsrc < rsrc_list_ptr->num_entries; rsrc++) {
-		type = readb (io_mem + addr);
+		type = readb(io_mem + addr);
 
 		addr += 1;
 		rsrc_type = type & EBDA_RSRC_TYPE_MASK;
 
 		if (rsrc_type == EBDA_IO_RSRC_TYPE) {
-			rsrc_ptr = alloc_ebda_pci_rsrc ();
+			rsrc_ptr = alloc_ebda_pci_rsrc();
 			if (!rsrc_ptr) {
-				iounmap (io_mem);
+				iounmap(io_mem);
 				return -ENOMEM;
 			}
 			rsrc_ptr->rsrc_type = type;
 
-			rsrc_ptr->bus_num = readb (io_mem + addr);
-			rsrc_ptr->dev_fun = readb (io_mem + addr + 1);
-			rsrc_ptr->start_addr = readw (io_mem + addr + 2);
-			rsrc_ptr->end_addr = readw (io_mem + addr + 4);
+			rsrc_ptr->bus_num = readb(io_mem + addr);
+			rsrc_ptr->dev_fun = readb(io_mem + addr + 1);
+			rsrc_ptr->start_addr = readw(io_mem + addr + 2);
+			rsrc_ptr->end_addr = readw(io_mem + addr + 4);
 			addr += 6;
 
-			debug ("rsrc from io type ----\n");
-			debug ("rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
+			debug("rsrc from io type ----\n");
+			debug("rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
 				rsrc_ptr->rsrc_type, rsrc_ptr->bus_num, rsrc_ptr->dev_fun, rsrc_ptr->start_addr, rsrc_ptr->end_addr);
 
-			list_add (&rsrc_ptr->ebda_pci_rsrc_list, &ibmphp_ebda_pci_rsrc_head);
+			list_add(&rsrc_ptr->ebda_pci_rsrc_list, &ibmphp_ebda_pci_rsrc_head);
 		}
 
 		if (rsrc_type == EBDA_MEM_RSRC_TYPE || rsrc_type == EBDA_PFM_RSRC_TYPE) {
-			rsrc_ptr = alloc_ebda_pci_rsrc ();
-			if (!rsrc_ptr ) {
-				iounmap (io_mem);
+			rsrc_ptr = alloc_ebda_pci_rsrc();
+			if (!rsrc_ptr) {
+				iounmap(io_mem);
 				return -ENOMEM;
 			}
 			rsrc_ptr->rsrc_type = type;
 
-			rsrc_ptr->bus_num = readb (io_mem + addr);
-			rsrc_ptr->dev_fun = readb (io_mem + addr + 1);
-			rsrc_ptr->start_addr = readl (io_mem + addr + 2);
-			rsrc_ptr->end_addr = readl (io_mem + addr + 6);
+			rsrc_ptr->bus_num = readb(io_mem + addr);
+			rsrc_ptr->dev_fun = readb(io_mem + addr + 1);
+			rsrc_ptr->start_addr = readl(io_mem + addr + 2);
+			rsrc_ptr->end_addr = readl(io_mem + addr + 6);
 			addr += 10;
 
-			debug ("rsrc from mem or pfm ---\n");
-			debug ("rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
+			debug("rsrc from mem or pfm ---\n");
+			debug("rsrc type: %x bus#: %x dev_func: %x start addr: %x end addr: %x\n",
 				rsrc_ptr->rsrc_type, rsrc_ptr->bus_num, rsrc_ptr->dev_fun, rsrc_ptr->start_addr, rsrc_ptr->end_addr);
 
-			list_add (&rsrc_ptr->ebda_pci_rsrc_list, &ibmphp_ebda_pci_rsrc_head);
+			list_add(&rsrc_ptr->ebda_pci_rsrc_list, &ibmphp_ebda_pci_rsrc_head);
 		}
 	}
-	kfree (rsrc_list_ptr);
+	kfree(rsrc_list_ptr);
 	rsrc_list_ptr = NULL;
-	print_ebda_pci_rsrc ();
+	print_ebda_pci_rsrc();
 	return 0;
 }
 
-u16 ibmphp_get_total_controllers (void)
+u16 ibmphp_get_total_controllers(void)
 {
 	return hpc_list_ptr->num_ctlrs;
 }
 
-struct slot *ibmphp_get_slot_from_physical_num (u8 physical_num)
+struct slot *ibmphp_get_slot_from_physical_num(u8 physical_num)
 {
 	struct slot *slot;
 
@@ -1090,7 +1090,7 @@ struct slot *ibmphp_get_slot_from_physical_num (u8 physical_num)
  *	- the total number of the slots based on each bus
  *	  (if only one slot per bus slot_min = slot_max )
  */
-struct bus_info *ibmphp_find_same_bus_num (u32 num)
+struct bus_info *ibmphp_find_same_bus_num(u32 num)
 {
 	struct bus_info *ptr;
 
@@ -1104,7 +1104,7 @@ struct bus_info *ibmphp_find_same_bus_num (u32 num)
 /*  Finding relative bus number, in order to map corresponding
  *  bus register
  */
-int ibmphp_get_bus_index (u8 num)
+int ibmphp_get_bus_index(u8 num)
 {
 	struct bus_info *ptr;
 
@@ -1115,45 +1115,39 @@ int ibmphp_get_bus_index (u8 num)
 	return -ENODEV;
 }
 
-void ibmphp_free_bus_info_queue (void)
+void ibmphp_free_bus_info_queue(void)
 {
-	struct bus_info *bus_info;
-	struct list_head *list;
-	struct list_head *next;
+	struct bus_info *bus_info, *next;
 
-	list_for_each_safe (list, next, &bus_info_head ) {
-		bus_info = list_entry (list, struct bus_info, bus_info_list);
+	list_for_each_entry_safe(bus_info, next, &bus_info_head,
+				 bus_info_list) {
 		kfree (bus_info);
 	}
 }
 
-void ibmphp_free_ebda_hpc_queue (void)
+void ibmphp_free_ebda_hpc_queue(void)
 {
-	struct controller *controller = NULL;
-	struct list_head *list;
-	struct list_head *next;
+	struct controller *controller = NULL, *next;
 	int pci_flag = 0;
 
-	list_for_each_safe (list, next, &ebda_hpc_head) {
-		controller = list_entry (list, struct controller, ebda_hpc_list);
+	list_for_each_entry_safe(controller, next, &ebda_hpc_head,
+				 ebda_hpc_list) {
 		if (controller->ctlr_type == 0)
-			release_region (controller->u.isa_ctlr.io_start, (controller->u.isa_ctlr.io_end - controller->u.isa_ctlr.io_start + 1));
+			release_region(controller->u.isa_ctlr.io_start, (controller->u.isa_ctlr.io_end - controller->u.isa_ctlr.io_start + 1));
 		else if ((controller->ctlr_type == 1) && (!pci_flag)) {
 			++pci_flag;
-			pci_unregister_driver (&ibmphp_driver);
+			pci_unregister_driver(&ibmphp_driver);
 		}
-		free_ebda_hpc (controller);
+		free_ebda_hpc(controller);
 	}
 }
 
-void ibmphp_free_ebda_pci_rsrc_queue (void)
+void ibmphp_free_ebda_pci_rsrc_queue(void)
 {
-	struct ebda_pci_rsrc *resource;
-	struct list_head *list;
-	struct list_head *next;
+	struct ebda_pci_rsrc *resource, *next;
 
-	list_for_each_safe (list, next, &ibmphp_ebda_pci_rsrc_head) {
-		resource = list_entry (list, struct ebda_pci_rsrc, ebda_pci_rsrc_list);
+	list_for_each_entry_safe(resource, next, &ibmphp_ebda_pci_rsrc_head,
+				 ebda_pci_rsrc_list) {
 		kfree (resource);
 		resource = NULL;
 	}
@@ -1171,14 +1165,14 @@ static struct pci_device_id id_table[] = {
 
 MODULE_DEVICE_TABLE(pci, id_table);
 
-static int ibmphp_probe (struct pci_dev *, const struct pci_device_id *);
+static int ibmphp_probe(struct pci_dev *, const struct pci_device_id *);
 static struct pci_driver ibmphp_driver = {
 	.name		= "ibmphp",
 	.id_table	= id_table,
 	.probe		= ibmphp_probe,
 };
 
-int ibmphp_register_pci (void)
+int ibmphp_register_pci(void)
 {
 	struct controller *ctrl;
 	int rc = 0;
@@ -1191,18 +1185,18 @@ int ibmphp_register_pci (void)
 	}
 	return rc;
 }
-static int ibmphp_probe (struct pci_dev *dev, const struct pci_device_id *ids)
+static int ibmphp_probe(struct pci_dev *dev, const struct pci_device_id *ids)
 {
 	struct controller *ctrl;
 
-	debug ("inside ibmphp_probe\n");
+	debug("inside ibmphp_probe\n");
 
 	list_for_each_entry(ctrl, &ebda_hpc_head, ebda_hpc_list) {
 		if (ctrl->ctlr_type == 1) {
 			if ((dev->devfn == ctrl->u.pci_ctlr.dev_fun) && (dev->bus->number == ctrl->u.pci_ctlr.bus)) {
 				ctrl->ctrl_dev = dev;
-				debug ("found device!!!\n");
-				debug ("dev->device = %x, dev->subsystem_device = %x\n", dev->device, dev->subsystem_device);
+				debug("found device!!!\n");
+				debug("dev->device = %x, dev->subsystem_device = %x\n", dev->device, dev->subsystem_device);
 				return 0;
 			}
 		}
diff --git a/drivers/pci/hotplug/ibmphp_hpc.c b/drivers/pci/hotplug/ibmphp_hpc.c
index 220876715a08..a6b458e4ab46 100644
--- a/drivers/pci/hotplug/ibmphp_hpc.c
+++ b/drivers/pci/hotplug/ibmphp_hpc.c
@@ -40,7 +40,7 @@
 #include "ibmphp.h"
 
 static int to_debug = 0;
-#define debug_polling(fmt, arg...)	do { if (to_debug) debug (fmt, arg); } while (0)
+#define debug_polling(fmt, arg...)	do { if (to_debug) debug(fmt, arg); } while (0)
 
 //----------------------------------------------------------------------------
 // timeout values
@@ -110,16 +110,16 @@ static struct task_struct *ibmphp_poll_thread;
 //----------------------------------------------------------------------------
 // local function prototypes
 //----------------------------------------------------------------------------
-static u8 i2c_ctrl_read (struct controller *, void __iomem *, u8);
-static u8 i2c_ctrl_write (struct controller *, void __iomem *, u8, u8);
-static u8 hpc_writecmdtoindex (u8, u8);
-static u8 hpc_readcmdtoindex (u8, u8);
-static void get_hpc_access (void);
-static void free_hpc_access (void);
+static u8 i2c_ctrl_read(struct controller *, void __iomem *, u8);
+static u8 i2c_ctrl_write(struct controller *, void __iomem *, u8, u8);
+static u8 hpc_writecmdtoindex(u8, u8);
+static u8 hpc_readcmdtoindex(u8, u8);
+static void get_hpc_access(void);
+static void free_hpc_access(void);
 static int poll_hpc(void *data);
-static int process_changeinstatus (struct slot *, struct slot *);
-static int process_changeinlatch (u8, u8, struct controller *);
-static int hpc_wait_ctlr_notworking (int, struct controller *, void __iomem *, u8 *);
+static int process_changeinstatus(struct slot *, struct slot *);
+static int process_changeinlatch(u8, u8, struct controller *);
+static int hpc_wait_ctlr_notworking(int, struct controller *, void __iomem *, u8 *);
 //----------------------------------------------------------------------------
 
 
@@ -128,16 +128,16 @@ static int hpc_wait_ctlr_notworking (int, struct controller *, void __iomem *, u
 *
 * Action:  initialize semaphores and variables
 *---------------------------------------------------------------------*/
-void __init ibmphp_hpc_initvars (void)
+void __init ibmphp_hpc_initvars(void)
 {
-	debug ("%s - Entry\n", __func__);
+	debug("%s - Entry\n", __func__);
 
 	mutex_init(&sem_hpcaccess);
 	sema_init(&semOperations, 1);
 	sema_init(&sem_exit, 0);
 	to_debug = 0;
 
-	debug ("%s - Exit\n", __func__);
+	debug("%s - Exit\n", __func__);
 }
 
 /*----------------------------------------------------------------------
@@ -146,7 +146,7 @@ void __init ibmphp_hpc_initvars (void)
 * Action:  read from HPC over I2C
 *
 *---------------------------------------------------------------------*/
-static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8 index)
+static u8 i2c_ctrl_read(struct controller *ctlr_ptr, void __iomem *WPGBbar, u8 index)
 {
 	u8 status;
 	int i;
@@ -155,7 +155,7 @@ static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	unsigned long ultemp;
 	unsigned long data;	// actual data HILO format
 
-	debug_polling ("%s - Entry WPGBbar[%p] index[%x] \n", __func__, WPGBbar, index);
+	debug_polling("%s - Entry WPGBbar[%p] index[%x] \n", __func__, WPGBbar, index);
 
 	//--------------------------------------------------------------------
 	// READ - step 1
@@ -178,28 +178,28 @@ static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 		ultemp = ultemp << 8;
 		data |= ultemp;
 	} else {
-		err ("this controller type is not supported \n");
+		err("this controller type is not supported \n");
 		return HPC_ERROR;
 	}
 
-	wpg_data = swab32 (data);	// swap data before writing
+	wpg_data = swab32(data);	// swap data before writing
 	wpg_addr = WPGBbar + WPG_I2CMOSUP_OFFSET;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// READ - step 2 : clear the message buffer
 	data = 0x00000000;
-	wpg_data = swab32 (data);
+	wpg_data = swab32(data);
 	wpg_addr = WPGBbar + WPG_I2CMBUFL_OFFSET;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// READ - step 3 : issue start operation, I2C master control bit 30:ON
 	//                 2020 : [20] OR operation at [20] offset 0x20
 	data = WPG_I2CMCNTL_STARTOP_MASK;
-	wpg_data = swab32 (data);
+	wpg_data = swab32(data);
 	wpg_addr = WPGBbar + WPG_I2CMCNTL_OFFSET + WPG_I2C_OR;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// READ - step 4 : wait until start operation bit clears
@@ -207,14 +207,14 @@ static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	while (i) {
 		msleep(10);
 		wpg_addr = WPGBbar + WPG_I2CMCNTL_OFFSET;
-		wpg_data = readl (wpg_addr);
-		data = swab32 (wpg_data);
+		wpg_data = readl(wpg_addr);
+		data = swab32(wpg_data);
 		if (!(data & WPG_I2CMCNTL_STARTOP_MASK))
 			break;
 		i--;
 	}
 	if (i == 0) {
-		debug ("%s - Error : WPG timeout\n", __func__);
+		debug("%s - Error : WPG timeout\n", __func__);
 		return HPC_ERROR;
 	}
 	//--------------------------------------------------------------------
@@ -223,26 +223,26 @@ static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	while (i) {
 		msleep(10);
 		wpg_addr = WPGBbar + WPG_I2CSTAT_OFFSET;
-		wpg_data = readl (wpg_addr);
-		data = swab32 (wpg_data);
-		if (HPC_I2CSTATUS_CHECK (data))
+		wpg_data = readl(wpg_addr);
+		data = swab32(wpg_data);
+		if (HPC_I2CSTATUS_CHECK(data))
 			break;
 		i--;
 	}
 	if (i == 0) {
-		debug ("ctrl_read - Exit Error:I2C timeout\n");
+		debug("ctrl_read - Exit Error:I2C timeout\n");
 		return HPC_ERROR;
 	}
 
 	//--------------------------------------------------------------------
 	// READ - step 6 : get DATA
 	wpg_addr = WPGBbar + WPG_I2CMBUFL_OFFSET;
-	wpg_data = readl (wpg_addr);
-	data = swab32 (wpg_data);
+	wpg_data = readl(wpg_addr);
+	data = swab32(wpg_data);
 
 	status = (u8) data;
 
-	debug_polling ("%s - Exit index[%x] status[%x]\n", __func__, index, status);
+	debug_polling("%s - Exit index[%x] status[%x]\n", __func__, index, status);
 
 	return (status);
 }
@@ -254,7 +254,7 @@ static u8 i2c_ctrl_read (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 *
 * Return   0 or error codes
 *---------------------------------------------------------------------*/
-static u8 i2c_ctrl_write (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8 index, u8 cmd)
+static u8 i2c_ctrl_write(struct controller *ctlr_ptr, void __iomem *WPGBbar, u8 index, u8 cmd)
 {
 	u8 rc;
 	void __iomem *wpg_addr;	// base addr + offset
@@ -263,7 +263,7 @@ static u8 i2c_ctrl_write (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	unsigned long data;	// actual data HILO format
 	int i;
 
-	debug_polling ("%s - Entry WPGBbar[%p] index[%x] cmd[%x]\n", __func__, WPGBbar, index, cmd);
+	debug_polling("%s - Entry WPGBbar[%p] index[%x] cmd[%x]\n", __func__, WPGBbar, index, cmd);
 
 	rc = 0;
 	//--------------------------------------------------------------------
@@ -289,28 +289,28 @@ static u8 i2c_ctrl_write (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 		ultemp = ultemp << 8;
 		data |= ultemp;
 	} else {
-		err ("this controller type is not supported \n");
+		err("this controller type is not supported \n");
 		return HPC_ERROR;
 	}
 
-	wpg_data = swab32 (data);	// swap data before writing
+	wpg_data = swab32(data);	// swap data before writing
 	wpg_addr = WPGBbar + WPG_I2CMOSUP_OFFSET;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// WRITE - step 2 : clear the message buffer
 	data = 0x00000000 | (unsigned long)cmd;
-	wpg_data = swab32 (data);
+	wpg_data = swab32(data);
 	wpg_addr = WPGBbar + WPG_I2CMBUFL_OFFSET;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// WRITE - step 3 : issue start operation,I2C master control bit 30:ON
 	//                 2020 : [20] OR operation at [20] offset 0x20
 	data = WPG_I2CMCNTL_STARTOP_MASK;
-	wpg_data = swab32 (data);
+	wpg_data = swab32(data);
 	wpg_addr = WPGBbar + WPG_I2CMCNTL_OFFSET + WPG_I2C_OR;
-	writel (wpg_data, wpg_addr);
+	writel(wpg_data, wpg_addr);
 
 	//--------------------------------------------------------------------
 	// WRITE - step 4 : wait until start operation bit clears
@@ -318,14 +318,14 @@ static u8 i2c_ctrl_write (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	while (i) {
 		msleep(10);
 		wpg_addr = WPGBbar + WPG_I2CMCNTL_OFFSET;
-		wpg_data = readl (wpg_addr);
-		data = swab32 (wpg_data);
+		wpg_data = readl(wpg_addr);
+		data = swab32(wpg_data);
 		if (!(data & WPG_I2CMCNTL_STARTOP_MASK))
 			break;
 		i--;
 	}
 	if (i == 0) {
-		debug ("%s - Exit Error:WPG timeout\n", __func__);
+		debug("%s - Exit Error:WPG timeout\n", __func__);
 		rc = HPC_ERROR;
 	}
 
@@ -335,25 +335,25 @@ static u8 i2c_ctrl_write (struct controller *ctlr_ptr, void __iomem *WPGBbar, u8
 	while (i) {
 		msleep(10);
 		wpg_addr = WPGBbar + WPG_I2CSTAT_OFFSET;
-		wpg_data = readl (wpg_addr);
-		data = swab32 (wpg_data);
-		if (HPC_I2CSTATUS_CHECK (data))
+		wpg_data = readl(wpg_addr);
+		data = swab32(wpg_data);
+		if (HPC_I2CSTATUS_CHECK(data))
 			break;
 		i--;
 	}
 	if (i == 0) {
-		debug ("ctrl_read - Error : I2C timeout\n");
+		debug("ctrl_read - Error : I2C timeout\n");
 		rc = HPC_ERROR;
 	}
 
-	debug_polling ("%s Exit rc[%x]\n", __func__, rc);
+	debug_polling("%s Exit rc[%x]\n", __func__, rc);
 	return (rc);
 }
 
 //------------------------------------------------------------
 //  Read from ISA type HPC
 //------------------------------------------------------------
-static u8 isa_ctrl_read (struct controller *ctlr_ptr, u8 offset)
+static u8 isa_ctrl_read(struct controller *ctlr_ptr, u8 offset)
 {
 	u16 start_address;
 	u16 end_address;
@@ -361,56 +361,56 @@ static u8 isa_ctrl_read (struct controller *ctlr_ptr, u8 offset)
 
 	start_address = ctlr_ptr->u.isa_ctlr.io_start;
 	end_address = ctlr_ptr->u.isa_ctlr.io_end;
-	data = inb (start_address + offset);
+	data = inb(start_address + offset);
 	return data;
 }
 
 //--------------------------------------------------------------
 // Write to ISA type HPC
 //--------------------------------------------------------------
-static void isa_ctrl_write (struct controller *ctlr_ptr, u8 offset, u8 data)
+static void isa_ctrl_write(struct controller *ctlr_ptr, u8 offset, u8 data)
 {
 	u16 start_address;
 	u16 port_address;
 
 	start_address = ctlr_ptr->u.isa_ctlr.io_start;
 	port_address = start_address + (u16) offset;
-	outb (data, port_address);
+	outb(data, port_address);
 }
 
-static u8 pci_ctrl_read (struct controller *ctrl, u8 offset)
+static u8 pci_ctrl_read(struct controller *ctrl, u8 offset)
 {
 	u8 data = 0x00;
-	debug ("inside pci_ctrl_read\n");
+	debug("inside pci_ctrl_read\n");
 	if (ctrl->ctrl_dev)
-		pci_read_config_byte (ctrl->ctrl_dev, HPC_PCI_OFFSET + offset, &data);
+		pci_read_config_byte(ctrl->ctrl_dev, HPC_PCI_OFFSET + offset, &data);
 	return data;
 }
 
-static u8 pci_ctrl_write (struct controller *ctrl, u8 offset, u8 data)
+static u8 pci_ctrl_write(struct controller *ctrl, u8 offset, u8 data)
 {
 	u8 rc = -ENODEV;
-	debug ("inside pci_ctrl_write\n");
+	debug("inside pci_ctrl_write\n");
 	if (ctrl->ctrl_dev) {
-		pci_write_config_byte (ctrl->ctrl_dev, HPC_PCI_OFFSET + offset, data);
+		pci_write_config_byte(ctrl->ctrl_dev, HPC_PCI_OFFSET + offset, data);
 		rc = 0;
 	}
 	return rc;
 }
 
-static u8 ctrl_read (struct controller *ctlr, void __iomem *base, u8 offset)
+static u8 ctrl_read(struct controller *ctlr, void __iomem *base, u8 offset)
 {
 	u8 rc;
 	switch (ctlr->ctlr_type) {
 	case 0:
-		rc = isa_ctrl_read (ctlr, offset);
+		rc = isa_ctrl_read(ctlr, offset);
 		break;
 	case 1:
-		rc = pci_ctrl_read (ctlr, offset);
+		rc = pci_ctrl_read(ctlr, offset);
 		break;
 	case 2:
 	case 4:
-		rc = i2c_ctrl_read (ctlr, base, offset);
+		rc = i2c_ctrl_read(ctlr, base, offset);
 		break;
 	default:
 		return -ENODEV;
@@ -418,7 +418,7 @@ static u8 ctrl_read (struct controller *ctlr, void __iomem *base, u8 offset)
 	return rc;
 }
 
-static u8 ctrl_write (struct controller *ctlr, void __iomem *base, u8 offset, u8 data)
+static u8 ctrl_write(struct controller *ctlr, void __iomem *base, u8 offset, u8 data)
 {
 	u8 rc = 0;
 	switch (ctlr->ctlr_type) {
@@ -426,7 +426,7 @@ static u8 ctrl_write (struct controller *ctlr, void __iomem *base, u8 offset, u8
 		isa_ctrl_write(ctlr, offset, data);
 		break;
 	case 1:
-		rc = pci_ctrl_write (ctlr, offset, data);
+		rc = pci_ctrl_write(ctlr, offset, data);
 		break;
 	case 2:
 	case 4:
@@ -444,7 +444,7 @@ static u8 ctrl_write (struct controller *ctlr, void __iomem *base, u8 offset, u8
 *
 * Return   index, HPC_ERROR
 *---------------------------------------------------------------------*/
-static u8 hpc_writecmdtoindex (u8 cmd, u8 index)
+static u8 hpc_writecmdtoindex(u8 cmd, u8 index)
 {
 	u8 rc;
 
@@ -476,7 +476,7 @@ static u8 hpc_writecmdtoindex (u8 cmd, u8 index)
 		break;
 
 	default:
-		err ("hpc_writecmdtoindex - Error invalid cmd[%x]\n", cmd);
+		err("hpc_writecmdtoindex - Error invalid cmd[%x]\n", cmd);
 		rc = HPC_ERROR;
 	}
 
@@ -490,7 +490,7 @@ static u8 hpc_writecmdtoindex (u8 cmd, u8 index)
 *
 * Return   index, HPC_ERROR
 *---------------------------------------------------------------------*/
-static u8 hpc_readcmdtoindex (u8 cmd, u8 index)
+static u8 hpc_readcmdtoindex(u8 cmd, u8 index)
 {
 	u8 rc;
 
@@ -533,78 +533,77 @@ static u8 hpc_readcmdtoindex (u8 cmd, u8 index)
 *
 * Return   0 or error codes
 *---------------------------------------------------------------------*/
-int ibmphp_hpc_readslot (struct slot *pslot, u8 cmd, u8 *pstatus)
+int ibmphp_hpc_readslot(struct slot *pslot, u8 cmd, u8 *pstatus)
 {
 	void __iomem *wpg_bbar = NULL;
 	struct controller *ctlr_ptr;
-	struct list_head *pslotlist;
 	u8 index, status;
 	int rc = 0;
 	int busindex;
 
-	debug_polling ("%s - Entry pslot[%p] cmd[%x] pstatus[%p]\n", __func__, pslot, cmd, pstatus);
+	debug_polling("%s - Entry pslot[%p] cmd[%x] pstatus[%p]\n", __func__, pslot, cmd, pstatus);
 
 	if ((pslot == NULL)
 	    || ((pstatus == NULL) && (cmd != READ_ALLSTAT) && (cmd != READ_BUSSTATUS))) {
 		rc = -EINVAL;
-		err ("%s - Error invalid pointer, rc[%d]\n", __func__, rc);
+		err("%s - Error invalid pointer, rc[%d]\n", __func__, rc);
 		return rc;
 	}
 
 	if (cmd == READ_BUSSTATUS) {
-		busindex = ibmphp_get_bus_index (pslot->bus);
+		busindex = ibmphp_get_bus_index(pslot->bus);
 		if (busindex < 0) {
 			rc = -EINVAL;
-			err ("%s - Exit Error:invalid bus, rc[%d]\n", __func__, rc);
+			err("%s - Exit Error:invalid bus, rc[%d]\n", __func__, rc);
 			return rc;
 		} else
 			index = (u8) busindex;
 	} else
 		index = pslot->ctlr_index;
 
-	index = hpc_readcmdtoindex (cmd, index);
+	index = hpc_readcmdtoindex(cmd, index);
 
 	if (index == HPC_ERROR) {
 		rc = -EINVAL;
-		err ("%s - Exit Error:invalid index, rc[%d]\n", __func__, rc);
+		err("%s - Exit Error:invalid index, rc[%d]\n", __func__, rc);
 		return rc;
 	}
 
 	ctlr_ptr = pslot->ctrl;
 
-	get_hpc_access ();
+	get_hpc_access();
 
 	//--------------------------------------------------------------------
 	// map physical address to logical address
 	//--------------------------------------------------------------------
 	if ((ctlr_ptr->ctlr_type == 2) || (ctlr_ptr->ctlr_type == 4))
-		wpg_bbar = ioremap (ctlr_ptr->u.wpeg_ctlr.wpegbbar, WPG_I2C_IOREMAP_SIZE);
+		wpg_bbar = ioremap(ctlr_ptr->u.wpeg_ctlr.wpegbbar, WPG_I2C_IOREMAP_SIZE);
 
 	//--------------------------------------------------------------------
 	// check controller status before reading
 	//--------------------------------------------------------------------
-	rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar, &status);
+	rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar, &status);
 	if (!rc) {
 		switch (cmd) {
 		case READ_ALLSTAT:
 			// update the slot structure
 			pslot->ctrl->status = status;
-			pslot->status = ctrl_read (ctlr_ptr, wpg_bbar, index);
-			rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar,
+			pslot->status = ctrl_read(ctlr_ptr, wpg_bbar, index);
+			rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar,
 						       &status);
 			if (!rc)
-				pslot->ext_status = ctrl_read (ctlr_ptr, wpg_bbar, index + WPG_1ST_EXTSLOT_INDEX);
+				pslot->ext_status = ctrl_read(ctlr_ptr, wpg_bbar, index + WPG_1ST_EXTSLOT_INDEX);
 
 			break;
 
 		case READ_SLOTSTATUS:
 			// DO NOT update the slot structure
-			*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 
 		case READ_EXTSLOTSTATUS:
 			// DO NOT update the slot structure
-			*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 
 		case READ_CTLRSTATUS:
@@ -613,36 +612,36 @@ int ibmphp_hpc_readslot (struct slot *pslot, u8 cmd, u8 *pstatus)
 			break;
 
 		case READ_BUSSTATUS:
-			pslot->busstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			pslot->busstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 		case READ_REVLEVEL:
-			*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 		case READ_HPCOPTIONS:
-			*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 		case READ_SLOTLATCHLOWREG:
 			// DO NOT update the slot structure
-			*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, index);
+			*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, index);
 			break;
 
 			// Not used
 		case READ_ALLSLOT:
-			list_for_each (pslotlist, &ibmphp_slot_head) {
-				pslot = list_entry (pslotlist, struct slot, ibm_slot_list);
+			list_for_each_entry(pslot, &ibmphp_slot_head,
+					    ibm_slot_list) {
 				index = pslot->ctlr_index;
-				rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT, ctlr_ptr,
+				rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT, ctlr_ptr,
 								wpg_bbar, &status);
 				if (!rc) {
-					pslot->status = ctrl_read (ctlr_ptr, wpg_bbar, index);
-					rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT,
+					pslot->status = ctrl_read(ctlr_ptr, wpg_bbar, index);
+					rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT,
 									ctlr_ptr, wpg_bbar, &status);
 					if (!rc)
 						pslot->ext_status =
-						    ctrl_read (ctlr_ptr, wpg_bbar,
+						    ctrl_read(ctlr_ptr, wpg_bbar,
 								index + WPG_1ST_EXTSLOT_INDEX);
 				} else {
-					err ("%s - Error ctrl_read failed\n", __func__);
+					err("%s - Error ctrl_read failed\n", __func__);
 					rc = -EINVAL;
 					break;
 				}
@@ -659,11 +658,11 @@ int ibmphp_hpc_readslot (struct slot *pslot, u8 cmd, u8 *pstatus)
 
 	// remove physical to logical address mapping
 	if ((ctlr_ptr->ctlr_type == 2) || (ctlr_ptr->ctlr_type == 4))
-		iounmap (wpg_bbar);
+		iounmap(wpg_bbar);
 
-	free_hpc_access ();
+	free_hpc_access();
 
-	debug_polling ("%s - Exit rc[%d]\n", __func__, rc);
+	debug_polling("%s - Exit rc[%d]\n", __func__, rc);
 	return rc;
 }
 
@@ -672,7 +671,7 @@ int ibmphp_hpc_readslot (struct slot *pslot, u8 cmd, u8 *pstatus)
 *
 * Action: issue a WRITE command to HPC
 *---------------------------------------------------------------------*/
-int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
+int ibmphp_hpc_writeslot(struct slot *pslot, u8 cmd)
 {
 	void __iomem *wpg_bbar = NULL;
 	struct controller *ctlr_ptr;
@@ -682,55 +681,55 @@ int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
 	int rc = 0;
 	int timeout;
 
-	debug_polling ("%s - Entry pslot[%p] cmd[%x]\n", __func__, pslot, cmd);
+	debug_polling("%s - Entry pslot[%p] cmd[%x]\n", __func__, pslot, cmd);
 	if (pslot == NULL) {
 		rc = -EINVAL;
-		err ("%s - Error Exit rc[%d]\n", __func__, rc);
+		err("%s - Error Exit rc[%d]\n", __func__, rc);
 		return rc;
 	}
 
 	if ((cmd == HPC_BUS_33CONVMODE) || (cmd == HPC_BUS_66CONVMODE) ||
 		(cmd == HPC_BUS_66PCIXMODE) || (cmd == HPC_BUS_100PCIXMODE) ||
 		(cmd == HPC_BUS_133PCIXMODE)) {
-		busindex = ibmphp_get_bus_index (pslot->bus);
+		busindex = ibmphp_get_bus_index(pslot->bus);
 		if (busindex < 0) {
 			rc = -EINVAL;
-			err ("%s - Exit Error:invalid bus, rc[%d]\n", __func__, rc);
+			err("%s - Exit Error:invalid bus, rc[%d]\n", __func__, rc);
 			return rc;
 		} else
 			index = (u8) busindex;
 	} else
 		index = pslot->ctlr_index;
 
-	index = hpc_writecmdtoindex (cmd, index);
+	index = hpc_writecmdtoindex(cmd, index);
 
 	if (index == HPC_ERROR) {
 		rc = -EINVAL;
-		err ("%s - Error Exit rc[%d]\n", __func__, rc);
+		err("%s - Error Exit rc[%d]\n", __func__, rc);
 		return rc;
 	}
 
 	ctlr_ptr = pslot->ctrl;
 
-	get_hpc_access ();
+	get_hpc_access();
 
 	//--------------------------------------------------------------------
 	// map physical address to logical address
 	//--------------------------------------------------------------------
 	if ((ctlr_ptr->ctlr_type == 2) || (ctlr_ptr->ctlr_type == 4)) {
-		wpg_bbar = ioremap (ctlr_ptr->u.wpeg_ctlr.wpegbbar, WPG_I2C_IOREMAP_SIZE);
+		wpg_bbar = ioremap(ctlr_ptr->u.wpeg_ctlr.wpegbbar, WPG_I2C_IOREMAP_SIZE);
 
-		debug ("%s - ctlr id[%x] physical[%lx] logical[%lx] i2c[%x]\n", __func__,
+		debug("%s - ctlr id[%x] physical[%lx] logical[%lx] i2c[%x]\n", __func__,
 		ctlr_ptr->ctlr_id, (ulong) (ctlr_ptr->u.wpeg_ctlr.wpegbbar), (ulong) wpg_bbar,
 		ctlr_ptr->u.wpeg_ctlr.i2c_addr);
 	}
 	//--------------------------------------------------------------------
 	// check controller status before writing
 	//--------------------------------------------------------------------
-	rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar, &status);
+	rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar, &status);
 	if (!rc) {
 
-		ctrl_write (ctlr_ptr, wpg_bbar, index, cmd);
+		ctrl_write(ctlr_ptr, wpg_bbar, index, cmd);
 
 		//--------------------------------------------------------------------
 		// check controller is still not working on the command
@@ -738,11 +737,11 @@ int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
 		timeout = CMD_COMPLETE_TOUT_SEC;
 		done = 0;
 		while (!done) {
-			rc = hpc_wait_ctlr_notworking (HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar,
+			rc = hpc_wait_ctlr_notworking(HPC_CTLR_WORKING_TOUT, ctlr_ptr, wpg_bbar,
 							&status);
 			if (!rc) {
-				if (NEEDTOCHECK_CMDSTATUS (cmd)) {
-					if (CTLR_FINISHED (status) == HPC_CTLR_FINISHED_YES)
+				if (NEEDTOCHECK_CMDSTATUS(cmd)) {
+					if (CTLR_FINISHED(status) == HPC_CTLR_FINISHED_YES)
 						done = 1;
 				} else
 					done = 1;
@@ -751,7 +750,7 @@ int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
 				msleep(1000);
 				if (timeout < 1) {
 					done = 1;
-					err ("%s - Error command complete timeout\n", __func__);
+					err("%s - Error command complete timeout\n", __func__);
 					rc = -EFAULT;
 				} else
 					timeout--;
@@ -763,10 +762,10 @@ int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
 
 	// remove physical to logical address mapping
 	if ((ctlr_ptr->ctlr_type == 2) || (ctlr_ptr->ctlr_type == 4))
-		iounmap (wpg_bbar);
-	free_hpc_access ();
+		iounmap(wpg_bbar);
+	free_hpc_access();
 
-	debug_polling ("%s - Exit rc[%d]\n", __func__, rc);
+	debug_polling("%s - Exit rc[%d]\n", __func__, rc);
 	return rc;
 }
 
@@ -775,7 +774,7 @@ int ibmphp_hpc_writeslot (struct slot *pslot, u8 cmd)
 *
 * Action: make sure only one process can access HPC at one time
 *---------------------------------------------------------------------*/
-static void get_hpc_access (void)
+static void get_hpc_access(void)
 {
 	mutex_lock(&sem_hpcaccess);
 }
@@ -783,7 +782,7 @@ static void get_hpc_access (void)
 /*----------------------------------------------------------------------
 * Name:    free_hpc_access()
 *---------------------------------------------------------------------*/
-void free_hpc_access (void)
+void free_hpc_access(void)
 {
 	mutex_unlock(&sem_hpcaccess);
 }
@@ -793,21 +792,21 @@ void free_hpc_access (void)
 *
 * Action: make sure only one process can change the data structure
 *---------------------------------------------------------------------*/
-void ibmphp_lock_operations (void)
+void ibmphp_lock_operations(void)
 {
-	down (&semOperations);
+	down(&semOperations);
 	to_debug = 1;
 }
 
 /*----------------------------------------------------------------------
 * Name:    ibmphp_unlock_operations()
 *---------------------------------------------------------------------*/
-void ibmphp_unlock_operations (void)
+void ibmphp_unlock_operations(void)
 {
-	debug ("%s - Entry\n", __func__);
-	up (&semOperations);
+	debug("%s - Entry\n", __func__);
+	up(&semOperations);
 	to_debug = 0;
-	debug ("%s - Exit\n", __func__);
+	debug("%s - Exit\n", __func__);
 }
 
 /*----------------------------------------------------------------------
@@ -820,7 +819,6 @@ static int poll_hpc(void *data)
 {
 	struct slot myslot;
 	struct slot *pslot = NULL;
-	struct list_head *pslotlist;
 	int rc;
 	int poll_state = POLL_LATCH_REGISTER;
 	u8 oldlatchlow = 0x00;
@@ -828,28 +826,28 @@ static int poll_hpc(void *data)
 	int poll_count = 0;
 	u8 ctrl_count = 0x00;
 
-	debug ("%s - Entry\n", __func__);
+	debug("%s - Entry\n", __func__);
 
 	while (!kthread_should_stop()) {
 		/* try to get the lock to do some kind of hardware access */
-		down (&semOperations);
+		down(&semOperations);
 
 		switch (poll_state) {
 		case POLL_LATCH_REGISTER:
 			oldlatchlow = curlatchlow;
 			ctrl_count = 0x00;
-			list_for_each (pslotlist, &ibmphp_slot_head) {
+			list_for_each_entry(pslot, &ibmphp_slot_head,
+					    ibm_slot_list) {
 				if (ctrl_count >= ibmphp_get_total_controllers())
 					break;
-				pslot = list_entry (pslotlist, struct slot, ibm_slot_list);
 				if (pslot->ctrl->ctlr_relative_id == ctrl_count) {
 					ctrl_count++;
-					if (READ_SLOT_LATCH (pslot->ctrl)) {
-						rc = ibmphp_hpc_readslot (pslot,
+					if (READ_SLOT_LATCH(pslot->ctrl)) {
+						rc = ibmphp_hpc_readslot(pslot,
 									  READ_SLOTLATCHLOWREG,
 									  &curlatchlow);
 						if (oldlatchlow != curlatchlow)
-							process_changeinlatch (oldlatchlow,
+							process_changeinlatch(oldlatchlow,
 									       curlatchlow,
 									       pslot->ctrl);
 					}
@@ -859,25 +857,25 @@ static int poll_hpc(void *data)
 			poll_state = POLL_SLEEP;
 			break;
 		case POLL_SLOTS:
-			list_for_each (pslotlist, &ibmphp_slot_head) {
-				pslot = list_entry (pslotlist, struct slot, ibm_slot_list);
+			list_for_each_entry(pslot, &ibmphp_slot_head,
+					    ibm_slot_list) {
 				// make a copy of the old status
-				memcpy ((void *) &myslot, (void *) pslot,
-					sizeof (struct slot));
-				rc = ibmphp_hpc_readslot (pslot, READ_ALLSTAT, NULL);
+				memcpy((void *) &myslot, (void *) pslot,
+					sizeof(struct slot));
+				rc = ibmphp_hpc_readslot(pslot, READ_ALLSTAT, NULL);
 				if ((myslot.status != pslot->status)
 				    || (myslot.ext_status != pslot->ext_status))
-					process_changeinstatus (pslot, &myslot);
+					process_changeinstatus(pslot, &myslot);
 			}
 			ctrl_count = 0x00;
-			list_for_each (pslotlist, &ibmphp_slot_head) {
+			list_for_each_entry(pslot, &ibmphp_slot_head,
+					    ibm_slot_list) {
 				if (ctrl_count >= ibmphp_get_total_controllers())
 					break;
-				pslot = list_entry (pslotlist, struct slot, ibm_slot_list);
 				if (pslot->ctrl->ctlr_relative_id == ctrl_count) {
 					ctrl_count++;
-					if (READ_SLOT_LATCH (pslot->ctrl))
-						rc = ibmphp_hpc_readslot (pslot,
+					if (READ_SLOT_LATCH(pslot->ctrl))
+						rc = ibmphp_hpc_readslot(pslot,
 									  READ_SLOTLATCHLOWREG,
 									  &curlatchlow);
 				}
@@ -887,13 +885,13 @@ static int poll_hpc(void *data)
 			break;
 		case POLL_SLEEP:
 			/* don't sleep with a lock on the hardware */
-			up (&semOperations);
+			up(&semOperations);
 			msleep(POLL_INTERVAL_SEC * 1000);
 
 			if (kthread_should_stop())
 				goto out_sleep;
 
-			down (&semOperations);
+			down(&semOperations);
 
 			if (poll_count >= POLL_LATCH_CNT) {
 				poll_count = 0;
@@ -903,13 +901,13 @@ static int poll_hpc(void *data)
 			break;
 		}
 		/* give up the hardware semaphore */
-		up (&semOperations);
+		up(&semOperations);
 		/* sleep for a short time just for good measure */
 out_sleep:
 		msleep(100);
 	}
-	up (&sem_exit);
-	debug ("%s - Exit\n", __func__);
+	up(&sem_exit);
+	debug("%s - Exit\n", __func__);
 	return 0;
 }
 
@@ -929,14 +927,14 @@ out_sleep:
 *
 * Notes:
 *---------------------------------------------------------------------*/
-static int process_changeinstatus (struct slot *pslot, struct slot *poldslot)
+static int process_changeinstatus(struct slot *pslot, struct slot *poldslot)
 {
 	u8 status;
 	int rc = 0;
 	u8 disable = 0;
 	u8 update = 0;
 
-	debug ("process_changeinstatus - Entry pslot[%p], poldslot[%p]\n", pslot, poldslot);
+	debug("process_changeinstatus - Entry pslot[%p], poldslot[%p]\n", pslot, poldslot);
 
 	// bit 0 - HPC_SLOT_POWER
 	if ((pslot->status & 0x01) != (poldslot->status & 0x01))
@@ -958,7 +956,7 @@ static int process_changeinstatus (struct slot *pslot, struct slot *poldslot)
 	// bit 5 - HPC_SLOT_PWRGD
 	if ((pslot->status & 0x20) != (poldslot->status & 0x20))
 		// OFF -> ON: ignore, ON -> OFF: disable slot
-		if ((poldslot->status & 0x20) && (SLOT_CONNECT (poldslot->status) == HPC_SLOT_CONNECTED) && (SLOT_PRESENT (poldslot->status)))
+		if ((poldslot->status & 0x20) && (SLOT_CONNECT(poldslot->status) == HPC_SLOT_CONNECTED) && (SLOT_PRESENT(poldslot->status)))
 			disable = 1;
 
 	// bit 6 - HPC_SLOT_BUS_SPEED
@@ -969,20 +967,20 @@ static int process_changeinstatus (struct slot *pslot, struct slot *poldslot)
 		update = 1;
 		// OPEN -> CLOSE
 		if (pslot->status & 0x80) {
-			if (SLOT_PWRGD (pslot->status)) {
+			if (SLOT_PWRGD(pslot->status)) {
 				// power goes on and off after closing latch
 				// check again to make sure power is still ON
 				msleep(1000);
-				rc = ibmphp_hpc_readslot (pslot, READ_SLOTSTATUS, &status);
-				if (SLOT_PWRGD (status))
+				rc = ibmphp_hpc_readslot(pslot, READ_SLOTSTATUS, &status);
+				if (SLOT_PWRGD(status))
 					update = 1;
 				else	// overwrite power in pslot to OFF
 					pslot->status &= ~HPC_SLOT_POWER;
 			}
 		}
 		// CLOSE -> OPEN
-		else if ((SLOT_PWRGD (poldslot->status) == HPC_SLOT_PWRGD_GOOD)
-			&& (SLOT_CONNECT (poldslot->status) == HPC_SLOT_CONNECTED) && (SLOT_PRESENT (poldslot->status))) {
+		else if ((SLOT_PWRGD(poldslot->status) == HPC_SLOT_PWRGD_GOOD)
+			&& (SLOT_CONNECT(poldslot->status) == HPC_SLOT_CONNECTED) && (SLOT_PRESENT(poldslot->status))) {
 			disable = 1;
 		}
 		// else - ignore
@@ -992,15 +990,15 @@ static int process_changeinstatus (struct slot *pslot, struct slot *poldslot)
 		update = 1;
 
 	if (disable) {
-		debug ("process_changeinstatus - disable slot\n");
+		debug("process_changeinstatus - disable slot\n");
 		pslot->flag = 0;
-		rc = ibmphp_do_disable_slot (pslot);
+		rc = ibmphp_do_disable_slot(pslot);
 	}
 
 	if (update || disable)
-		ibmphp_update_slot_info (pslot);
+		ibmphp_update_slot_info(pslot);
 
-	debug ("%s - Exit rc[%d] disable[%x] update[%x]\n", __func__, rc, disable, update);
+	debug("%s - Exit rc[%d] disable[%x] update[%x]\n", __func__, rc, disable, update);
 
 	return rc;
 }
@@ -1015,32 +1013,32 @@ static int process_changeinstatus (struct slot *pslot, struct slot *poldslot)
 * Return   0 or error codes
 * Value:
 *---------------------------------------------------------------------*/
-static int process_changeinlatch (u8 old, u8 new, struct controller *ctrl)
+static int process_changeinlatch(u8 old, u8 new, struct controller *ctrl)
 {
 	struct slot myslot, *pslot;
 	u8 i;
 	u8 mask;
 	int rc = 0;
 
-	debug ("%s - Entry old[%x], new[%x]\n", __func__, old, new);
+	debug("%s - Entry old[%x], new[%x]\n", __func__, old, new);
 	// bit 0 reserved, 0 is LSB, check bit 1-6 for 6 slots
 
 	for (i = ctrl->starting_slot_num; i <= ctrl->ending_slot_num; i++) {
 		mask = 0x01 << i;
 		if ((mask & old) != (mask & new)) {
-			pslot = ibmphp_get_slot_from_physical_num (i);
+			pslot = ibmphp_get_slot_from_physical_num(i);
 			if (pslot) {
-				memcpy ((void *) &myslot, (void *) pslot, sizeof (struct slot));
-				rc = ibmphp_hpc_readslot (pslot, READ_ALLSTAT, NULL);
-				debug ("%s - call process_changeinstatus for slot[%d]\n", __func__, i);
-				process_changeinstatus (pslot, &myslot);
+				memcpy((void *) &myslot, (void *) pslot, sizeof(struct slot));
+				rc = ibmphp_hpc_readslot(pslot, READ_ALLSTAT, NULL);
+				debug("%s - call process_changeinstatus for slot[%d]\n", __func__, i);
+				process_changeinstatus(pslot, &myslot);
 			} else {
 				rc = -EINVAL;
-				err ("%s - Error bad pointer for slot[%d]\n", __func__, i);
+				err("%s - Error bad pointer for slot[%d]\n", __func__, i);
 			}
 		}
 	}
-	debug ("%s - Exit rc[%d]\n", __func__, rc);
+	debug("%s - Exit rc[%d]\n", __func__, rc);
 	return rc;
 }
 
@@ -1049,13 +1047,13 @@ static int process_changeinlatch (u8 old, u8 new, struct controller *ctrl)
 *
 * Action:  start polling thread
 *---------------------------------------------------------------------*/
-int __init ibmphp_hpc_start_poll_thread (void)
+int __init ibmphp_hpc_start_poll_thread(void)
 {
-	debug ("%s - Entry\n", __func__);
+	debug("%s - Entry\n", __func__);
 
 	ibmphp_poll_thread = kthread_run(poll_hpc, NULL, "hpc_poll");
 	if (IS_ERR(ibmphp_poll_thread)) {
-		err ("%s - Error, thread not started\n", __func__);
+		err("%s - Error, thread not started\n", __func__);
 		return PTR_ERR(ibmphp_poll_thread);
 	}
 	return 0;
@@ -1066,30 +1064,30 @@ int __init ibmphp_hpc_start_poll_thread (void)
 *
 * Action:  stop polling thread and cleanup
 *---------------------------------------------------------------------*/
-void __exit ibmphp_hpc_stop_poll_thread (void)
+void __exit ibmphp_hpc_stop_poll_thread(void)
 {
-	debug ("%s - Entry\n", __func__);
+	debug("%s - Entry\n", __func__);
 
 	kthread_stop(ibmphp_poll_thread);
-	debug ("before locking operations \n");
-	ibmphp_lock_operations ();
-	debug ("after locking operations \n");
+	debug("before locking operations\n");
+	ibmphp_lock_operations();
+	debug("after locking operations\n");
 
 	// wait for poll thread to exit
-	debug ("before sem_exit down \n");
-	down (&sem_exit);
-	debug ("after sem_exit down \n");
+	debug("before sem_exit down\n");
+	down(&sem_exit);
+	debug("after sem_exit down\n");
 
 	// cleanup
-	debug ("before free_hpc_access \n");
-	free_hpc_access ();
-	debug ("after free_hpc_access \n");
-	ibmphp_unlock_operations ();
-	debug ("after unlock operations \n");
-	up (&sem_exit);
-	debug ("after sem exit up\n");
-
-	debug ("%s - Exit\n", __func__);
+	debug("before free_hpc_access\n");
+	free_hpc_access();
+	debug("after free_hpc_access\n");
+	ibmphp_unlock_operations();
+	debug("after unlock operations\n");
+	up(&sem_exit);
+	debug("after sem exit up\n");
+
+	debug("%s - Exit\n", __func__);
 }
 
 /*----------------------------------------------------------------------
@@ -1100,32 +1098,32 @@ void __exit ibmphp_hpc_stop_poll_thread (void)
 * Return   0, HPC_ERROR
 * Value:
 *---------------------------------------------------------------------*/
-static int hpc_wait_ctlr_notworking (int timeout, struct controller *ctlr_ptr, void __iomem *wpg_bbar,
+static int hpc_wait_ctlr_notworking(int timeout, struct controller *ctlr_ptr, void __iomem *wpg_bbar,
 				    u8 *pstatus)
 {
 	int rc = 0;
 	u8 done = 0;
 
-	debug_polling ("hpc_wait_ctlr_notworking - Entry timeout[%d]\n", timeout);
+	debug_polling("hpc_wait_ctlr_notworking - Entry timeout[%d]\n", timeout);
 
 	while (!done) {
-		*pstatus = ctrl_read (ctlr_ptr, wpg_bbar, WPG_CTLR_INDEX);
+		*pstatus = ctrl_read(ctlr_ptr, wpg_bbar, WPG_CTLR_INDEX);
 		if (*pstatus == HPC_ERROR) {
 			rc = HPC_ERROR;
 			done = 1;
 		}
-		if (CTLR_WORKING (*pstatus) == HPC_CTLR_WORKING_NO)
+		if (CTLR_WORKING(*pstatus) == HPC_CTLR_WORKING_NO)
 			done = 1;
 		if (!done) {
 			msleep(1000);
 			if (timeout < 1) {
 				done = 1;
-				err ("HPCreadslot - Error ctlr timeout\n");
+				err("HPCreadslot - Error ctlr timeout\n");
 				rc = HPC_ERROR;
 			} else
 				timeout--;
 		}
 	}
-	debug_polling ("hpc_wait_ctlr_notworking - Exit rc[%x] status[%x]\n", rc, *pstatus);
+	debug_polling("hpc_wait_ctlr_notworking - Exit rc[%x] status[%x]\n", rc, *pstatus);
 	return rc;
 }
diff --git a/drivers/pci/hotplug/ibmphp_pci.c b/drivers/pci/hotplug/ibmphp_pci.c
index 814cea22a9fa..dc1876feb06f 100644
--- a/drivers/pci/hotplug/ibmphp_pci.c
+++ b/drivers/pci/hotplug/ibmphp_pci.c
@@ -37,8 +37,8 @@
 static int configure_device(struct pci_func *);
 static int configure_bridge(struct pci_func **, u8);
 static struct res_needed *scan_behind_bridge(struct pci_func *, u8);
-static int add_new_bus (struct bus_node *, struct resource_node *, struct resource_node *, struct resource_node *, u8);
-static u8 find_sec_number (u8 primary_busno, u8 slotno);
+static int add_new_bus(struct bus_node *, struct resource_node *, struct resource_node *, struct resource_node *, u8);
+static u8 find_sec_number(u8 primary_busno, u8 slotno);
 
 /*
  * NOTE..... If BIOS doesn't provide default routing, we assign:
@@ -47,7 +47,7 @@ static u8 find_sec_number (u8 primary_busno, u8 slotno);
  * We also assign the same irq numbers for multi function devices.
  * These are PIC mode, so shouldn't matter n.e.ways (hopefully)
  */
-static void assign_alt_irq (struct pci_func *cur_func, u8 class_code)
+static void assign_alt_irq(struct pci_func *cur_func, u8 class_code)
 {
 	int j;
 	for (j = 0; j < 4; j++) {
@@ -78,7 +78,7 @@ static void assign_alt_irq (struct pci_func *cur_func, u8 class_code)
  * if there is an error, will need to go through all previous functions and
  * unconfigure....or can add some code into unconfigure_card....
  */
-int ibmphp_configure_card (struct pci_func *func, u8 slotno)
+int ibmphp_configure_card(struct pci_func *func, u8 slotno)
 {
 	u16 vendor_id;
 	u32 class;
@@ -92,7 +92,7 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 	u8 flag;
 	u8 valid_device = 0x00; /* to see if we are able to read from card any device info at all */
 
-	debug ("inside configure_card, func->busno = %x\n", func->busno);
+	debug("inside configure_card, func->busno = %x\n", func->busno);
 
 	device = func->device;
 	cur_func = func;
@@ -109,15 +109,15 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 
 		cur_func->function = function;
 
-		debug ("inside the loop, cur_func->busno = %x, cur_func->device = %x, cur_func->function = %x\n",
+		debug("inside the loop, cur_func->busno = %x, cur_func->device = %x, cur_func->function = %x\n",
 			cur_func->busno, cur_func->device, cur_func->function);
 
-		pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
+		pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
 
-		debug ("vendor_id is %x\n", vendor_id);
+		debug("vendor_id is %x\n", vendor_id);
 		if (vendor_id != PCI_VENDOR_ID_NOTVALID) {
 			/* found correct device!!! */
-			debug ("found valid device, vendor_id = %x\n", vendor_id);
+			debug("found valid device, vendor_id = %x\n", vendor_id);
 
 			++valid_device;
 
@@ -126,29 +126,29 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 			 *         |_=> 0 = single function device, 1 = multi-function device
 			 */
 
-			pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
-			pci_bus_read_config_dword (ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
+			pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
+			pci_bus_read_config_dword(ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
 
 			class_code = class >> 24;
-			debug ("hrd_type = %x, class = %x, class_code %x\n", hdr_type, class, class_code);
+			debug("hrd_type = %x, class = %x, class_code %x\n", hdr_type, class, class_code);
 			class >>= 8;	/* to take revision out, class = class.subclass.prog i/f */
 			if (class == PCI_CLASS_NOT_DEFINED_VGA) {
-				err ("The device %x is VGA compatible and as is not supported for hot plugging. "
+				err("The device %x is VGA compatible and as is not supported for hot plugging. "
 				     "Please choose another device.\n", cur_func->device);
 				return -ENODEV;
 			} else if (class == PCI_CLASS_DISPLAY_VGA) {
-				err ("The device %x is not supported for hot plugging. Please choose another device.\n",
+				err("The device %x is not supported for hot plugging. Please choose another device.\n",
 				     cur_func->device);
 				return -ENODEV;
 			}
 			switch (hdr_type) {
 				case PCI_HEADER_TYPE_NORMAL:
-					debug ("single device case.... vendor id = %x, hdr_type = %x, class = %x\n", vendor_id, hdr_type, class);
-					assign_alt_irq (cur_func, class_code);
+					debug("single device case.... vendor id = %x, hdr_type = %x, class = %x\n", vendor_id, hdr_type, class);
+					assign_alt_irq(cur_func, class_code);
 					rc = configure_device(cur_func);
 					if (rc < 0) {
 						/* We need to do this in case some other BARs were properly inserted */
-						err ("was not able to configure devfunc %x on bus %x.\n",
+						err("was not able to configure devfunc %x on bus %x.\n",
 						     cur_func->device, cur_func->busno);
 						cleanup_count = 6;
 						goto error;
@@ -157,18 +157,18 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 					function = 0x8;
 					break;
 				case PCI_HEADER_TYPE_MULTIDEVICE:
-					assign_alt_irq (cur_func, class_code);
+					assign_alt_irq(cur_func, class_code);
 					rc = configure_device(cur_func);
 					if (rc < 0) {
 						/* We need to do this in case some other BARs were properly inserted */
-						err ("was not able to configure devfunc %x on bus %x...bailing out\n",
+						err("was not able to configure devfunc %x on bus %x...bailing out\n",
 						     cur_func->device, cur_func->busno);
 						cleanup_count = 6;
 						goto error;
 					}
 					newfunc = kzalloc(sizeof(*newfunc), GFP_KERNEL);
 					if (!newfunc) {
-						err ("out of system memory\n");
+						err("out of system memory\n");
 						return -ENOMEM;
 					}
 					newfunc->busno = cur_func->busno;
@@ -181,32 +181,32 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 				case PCI_HEADER_TYPE_MULTIBRIDGE:
 					class >>= 8;
 					if (class != PCI_CLASS_BRIDGE_PCI) {
-						err ("This %x is not PCI-to-PCI bridge, and as is not supported for hot-plugging.  Please insert another card.\n",
+						err("This %x is not PCI-to-PCI bridge, and as is not supported for hot-plugging.  Please insert another card.\n",
 						     cur_func->device);
 						return -ENODEV;
 					}
-					assign_alt_irq (cur_func, class_code);
-					rc = configure_bridge (&cur_func, slotno);
+					assign_alt_irq(cur_func, class_code);
+					rc = configure_bridge(&cur_func, slotno);
 					if (rc == -ENODEV) {
-						err ("You chose to insert Single Bridge, or nested bridges, this is not supported...\n");
-						err ("Bus %x, devfunc %x\n", cur_func->busno, cur_func->device);
+						err("You chose to insert Single Bridge, or nested bridges, this is not supported...\n");
+						err("Bus %x, devfunc %x\n", cur_func->busno, cur_func->device);
 						return rc;
 					}
 					if (rc) {
 						/* We need to do this in case some other BARs were properly inserted */
-						err ("was not able to hot-add PPB properly.\n");
+						err("was not able to hot-add PPB properly.\n");
 						func->bus = 1; /* To indicate to the unconfigure function that this is a PPB */
 						cleanup_count = 2;
 						goto error;
 					}
 
-					pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
+					pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
 					flag = 0;
 					for (i = 0; i < 32; i++) {
 						if (func->devices[i]) {
 							newfunc = kzalloc(sizeof(*newfunc), GFP_KERNEL);
 							if (!newfunc) {
-								err ("out of system memory\n");
+								err("out of system memory\n");
 								return -ENOMEM;
 							}
 							newfunc->busno = sec_number;
@@ -220,7 +220,7 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 							} else
 								cur_func->next = newfunc;
 
-							rc = ibmphp_configure_card (newfunc, slotno);
+							rc = ibmphp_configure_card(newfunc, slotno);
 							/* This could only happen if kmalloc failed */
 							if (rc) {
 								/* We need to do this in case bridge itself got configured properly, but devices behind it failed */
@@ -234,53 +234,53 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 
 					newfunc = kzalloc(sizeof(*newfunc), GFP_KERNEL);
 					if (!newfunc) {
-						err ("out of system memory\n");
+						err("out of system memory\n");
 						return -ENOMEM;
 					}
 					newfunc->busno = cur_func->busno;
 					newfunc->device = device;
 					for (j = 0; j < 4; j++)
 						newfunc->irq[j] = cur_func->irq[j];
-					for (prev_func = cur_func; prev_func->next; prev_func = prev_func->next) ;
+					for (prev_func = cur_func; prev_func->next; prev_func = prev_func->next);
 					prev_func->next = newfunc;
 					cur_func = newfunc;
 					break;
 				case PCI_HEADER_TYPE_BRIDGE:
 					class >>= 8;
-					debug ("class now is %x\n", class);
+					debug("class now is %x\n", class);
 					if (class != PCI_CLASS_BRIDGE_PCI) {
-						err ("This %x is not PCI-to-PCI bridge, and as is not supported for hot-plugging.  Please insert another card.\n",
+						err("This %x is not PCI-to-PCI bridge, and as is not supported for hot-plugging.  Please insert another card.\n",
 						     cur_func->device);
 						return -ENODEV;
 					}
 
-					assign_alt_irq (cur_func, class_code);
+					assign_alt_irq(cur_func, class_code);
 
-					debug ("cur_func->busno b4 configure_bridge is %x\n", cur_func->busno);
-					rc = configure_bridge (&cur_func, slotno);
+					debug("cur_func->busno b4 configure_bridge is %x\n", cur_func->busno);
+					rc = configure_bridge(&cur_func, slotno);
 					if (rc == -ENODEV) {
-						err ("You chose to insert Single Bridge, or nested bridges, this is not supported...\n");
-						err ("Bus %x, devfunc %x\n", cur_func->busno, cur_func->device);
+						err("You chose to insert Single Bridge, or nested bridges, this is not supported...\n");
+						err("Bus %x, devfunc %x\n", cur_func->busno, cur_func->device);
 						return rc;
 					}
 					if (rc) {
 						/* We need to do this in case some other BARs were properly inserted */
 						func->bus = 1; /* To indicate to the unconfigure function that this is a PPB */
-						err ("was not able to hot-add PPB properly.\n");
+						err("was not able to hot-add PPB properly.\n");
 						cleanup_count = 2;
 						goto error;
 					}
-					debug ("cur_func->busno = %x, device = %x, function = %x\n",
+					debug("cur_func->busno = %x, device = %x, function = %x\n",
 						cur_func->busno, device, function);
-					pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
-					debug ("after configuring bridge..., sec_number = %x\n", sec_number);
+					pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
+					debug("after configuring bridge..., sec_number = %x\n", sec_number);
 					flag = 0;
 					for (i = 0; i < 32; i++) {
 						if (func->devices[i]) {
-							debug ("inside for loop, device is %x\n", i);
+							debug("inside for loop, device is %x\n", i);
 							newfunc = kzalloc(sizeof(*newfunc), GFP_KERNEL);
 							if (!newfunc) {
-								err (" out of system memory\n");
+								err(" out of system memory\n");
 								return -ENOMEM;
 							}
 							newfunc->busno = sec_number;
@@ -289,12 +289,12 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 								newfunc->irq[j] = cur_func->irq[j];
 
 							if (flag) {
-								for (prev_func = cur_func; prev_func->next; prev_func = prev_func->next) ;
+								for (prev_func = cur_func; prev_func->next; prev_func = prev_func->next);
 								prev_func->next = newfunc;
 							} else
 								cur_func->next = newfunc;
 
-							rc = ibmphp_configure_card (newfunc, slotno);
+							rc = ibmphp_configure_card(newfunc, slotno);
 
 							/* Again, this case should not happen... For complete paranoia, will need to call remove_bus */
 							if (rc) {
@@ -310,7 +310,7 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 					function = 0x8;
 					break;
 				default:
-					err ("MAJOR PROBLEM!!!!, header type not supported? %x\n", hdr_type);
+					err("MAJOR PROBLEM!!!!, header type not supported? %x\n", hdr_type);
 					return -ENXIO;
 					break;
 			}	/* end of switch */
@@ -318,7 +318,7 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 	}	/* end of for */
 
 	if (!valid_device) {
-		err ("Cannot find any valid devices on the card.  Or unable to read from card.\n");
+		err("Cannot find any valid devices on the card.  Or unable to read from card.\n");
 		return -ENODEV;
 	}
 
@@ -327,13 +327,13 @@ int ibmphp_configure_card (struct pci_func *func, u8 slotno)
 error:
 	for (i = 0; i < cleanup_count; i++) {
 		if (cur_func->io[i]) {
-			ibmphp_remove_resource (cur_func->io[i]);
+			ibmphp_remove_resource(cur_func->io[i]);
 			cur_func->io[i] = NULL;
 		} else if (cur_func->pfmem[i]) {
-			ibmphp_remove_resource (cur_func->pfmem[i]);
+			ibmphp_remove_resource(cur_func->pfmem[i]);
 			cur_func->pfmem[i] = NULL;
 		} else if (cur_func->mem[i]) {
-			ibmphp_remove_resource (cur_func->mem[i]);
+			ibmphp_remove_resource(cur_func->mem[i]);
 			cur_func->mem[i] = NULL;
 		}
 	}
@@ -345,7 +345,7 @@ error:
  * Input: pointer to the pci_func
  * Output: configured PCI, 0, or error
  */
-static int configure_device (struct pci_func *func)
+static int configure_device(struct pci_func *func)
 {
 	u32 bar[6];
 	u32 address[] = {
@@ -366,7 +366,7 @@ static int configure_device (struct pci_func *func)
 	struct resource_node *pfmem[6];
 	unsigned int devfn;
 
-	debug ("%s - inside\n", __func__);
+	debug("%s - inside\n", __func__);
 
 	devfn = PCI_DEVFN(func->device, func->function);
 	ibmphp_pci_bus->number = func->busno;
@@ -386,27 +386,27 @@ static int configure_device (struct pci_func *func)
 			pcibios_write_config_dword(cur_func->busno, cur_func->device,
 			PCI_BASE_ADDRESS_0 + 4 * count, 0xFFFFFFFF);
 		 */
-		pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
-		pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
+		pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
+		pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
 
 		if (!bar[count])	/* This BAR is not implemented */
 			continue;
 
-		debug ("Device %x BAR %d wants %x\n", func->device, count, bar[count]);
+		debug("Device %x BAR %d wants %x\n", func->device, count, bar[count]);
 
 		if (bar[count] & PCI_BASE_ADDRESS_SPACE_IO) {
 			/* This is IO */
-			debug ("inside IO SPACE\n");
+			debug("inside IO SPACE\n");
 
 			len[count] = bar[count] & 0xFFFFFFFC;
 			len[count] = ~len[count] + 1;
 
-			debug ("len[count] in IO %x, count %d\n", len[count], count);
+			debug("len[count] in IO %x, count %d\n", len[count], count);
 
 			io[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 
 			if (!io[count]) {
-				err ("out of system memory\n");
+				err("out of system memory\n");
 				return -ENOMEM;
 			}
 			io[count]->type = IO;
@@ -414,36 +414,36 @@ static int configure_device (struct pci_func *func)
 			io[count]->devfunc = PCI_DEVFN(func->device, func->function);
 			io[count]->len = len[count];
 			if (ibmphp_check_resource(io[count], 0) == 0) {
-				ibmphp_add_resource (io[count]);
+				ibmphp_add_resource(io[count]);
 				func->io[count] = io[count];
 			} else {
-				err ("cannot allocate requested io for bus %x device %x function %x len %x\n",
+				err("cannot allocate requested io for bus %x device %x function %x len %x\n",
 				     func->busno, func->device, func->function, len[count]);
-				kfree (io[count]);
+				kfree(io[count]);
 				return -EIO;
 			}
-			pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->io[count]->start);
+			pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->io[count]->start);
 
 			/* _______________This is for debugging purposes only_____________________ */
-			debug ("b4 writing, the IO address is %x\n", func->io[count]->start);
-			pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
-			debug ("after writing.... the start address is %x\n", bar[count]);
+			debug("b4 writing, the IO address is %x\n", func->io[count]->start);
+			pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
+			debug("after writing.... the start address is %x\n", bar[count]);
 			/* _________________________________________________________________________*/
 
 		} else {
 			/* This is Memory */
 			if (bar[count] & PCI_BASE_ADDRESS_MEM_PREFETCH) {
 				/* pfmem */
-				debug ("PFMEM SPACE\n");
+				debug("PFMEM SPACE\n");
 
 				len[count] = bar[count] & 0xFFFFFFF0;
 				len[count] = ~len[count] + 1;
 
-				debug ("len[count] in PFMEM %x, count %d\n", len[count], count);
+				debug("len[count] in PFMEM %x, count %d\n", len[count], count);
 
 				pfmem[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 				if (!pfmem[count]) {
-					err ("out of system memory\n");
+					err("out of system memory\n");
 					return -ENOMEM;
 				}
 				pfmem[count]->type = PFMEM;
@@ -452,64 +452,64 @@ static int configure_device (struct pci_func *func)
 							func->function);
 				pfmem[count]->len = len[count];
 				pfmem[count]->fromMem = 0;
-				if (ibmphp_check_resource (pfmem[count], 0) == 0) {
-					ibmphp_add_resource (pfmem[count]);
+				if (ibmphp_check_resource(pfmem[count], 0) == 0) {
+					ibmphp_add_resource(pfmem[count]);
 					func->pfmem[count] = pfmem[count];
 				} else {
 					mem_tmp = kzalloc(sizeof(*mem_tmp), GFP_KERNEL);
 					if (!mem_tmp) {
-						err ("out of system memory\n");
-						kfree (pfmem[count]);
+						err("out of system memory\n");
+						kfree(pfmem[count]);
 						return -ENOMEM;
 					}
 					mem_tmp->type = MEM;
 					mem_tmp->busno = pfmem[count]->busno;
 					mem_tmp->devfunc = pfmem[count]->devfunc;
 					mem_tmp->len = pfmem[count]->len;
-					debug ("there's no pfmem... going into mem.\n");
-					if (ibmphp_check_resource (mem_tmp, 0) == 0) {
-						ibmphp_add_resource (mem_tmp);
+					debug("there's no pfmem... going into mem.\n");
+					if (ibmphp_check_resource(mem_tmp, 0) == 0) {
+						ibmphp_add_resource(mem_tmp);
 						pfmem[count]->fromMem = 1;
 						pfmem[count]->rangeno = mem_tmp->rangeno;
 						pfmem[count]->start = mem_tmp->start;
 						pfmem[count]->end = mem_tmp->end;
-						ibmphp_add_pfmem_from_mem (pfmem[count]);
+						ibmphp_add_pfmem_from_mem(pfmem[count]);
 						func->pfmem[count] = pfmem[count];
 					} else {
-						err ("cannot allocate requested pfmem for bus %x, device %x, len %x\n",
+						err("cannot allocate requested pfmem for bus %x, device %x, len %x\n",
 						     func->busno, func->device, len[count]);
-						kfree (mem_tmp);
-						kfree (pfmem[count]);
+						kfree(mem_tmp);
+						kfree(pfmem[count]);
 						return -EIO;
 					}
 				}
 
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->pfmem[count]->start);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->pfmem[count]->start);
 
 				/*_______________This is for debugging purposes only______________________________*/
-				debug ("b4 writing, start address is %x\n", func->pfmem[count]->start);
-				pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
-				debug ("after writing, start address is %x\n", bar[count]);
+				debug("b4 writing, start address is %x\n", func->pfmem[count]->start);
+				pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
+				debug("after writing, start address is %x\n", bar[count]);
 				/*_________________________________________________________________________________*/
 
 				if (bar[count] & PCI_BASE_ADDRESS_MEM_TYPE_64) {	/* takes up another dword */
-					debug ("inside the mem 64 case, count %d\n", count);
+					debug("inside the mem 64 case, count %d\n", count);
 					count += 1;
 					/* on the 2nd dword, write all 0s, since we can't handle them n.e.ways */
-					pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0x00000000);
+					pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0x00000000);
 				}
 			} else {
 				/* regular memory */
-				debug ("REGULAR MEM SPACE\n");
+				debug("REGULAR MEM SPACE\n");
 
 				len[count] = bar[count] & 0xFFFFFFF0;
 				len[count] = ~len[count] + 1;
 
-				debug ("len[count] in Mem %x, count %d\n", len[count], count);
+				debug("len[count] in Mem %x, count %d\n", len[count], count);
 
 				mem[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 				if (!mem[count]) {
-					err ("out of system memory\n");
+					err("out of system memory\n");
 					return -ENOMEM;
 				}
 				mem[count]->type = MEM;
@@ -517,43 +517,43 @@ static int configure_device (struct pci_func *func)
 				mem[count]->devfunc = PCI_DEVFN(func->device,
 							func->function);
 				mem[count]->len = len[count];
-				if (ibmphp_check_resource (mem[count], 0) == 0) {
-					ibmphp_add_resource (mem[count]);
+				if (ibmphp_check_resource(mem[count], 0) == 0) {
+					ibmphp_add_resource(mem[count]);
 					func->mem[count] = mem[count];
 				} else {
-					err ("cannot allocate requested mem for bus %x, device %x, len %x\n",
+					err("cannot allocate requested mem for bus %x, device %x, len %x\n",
 					     func->busno, func->device, len[count]);
-					kfree (mem[count]);
+					kfree(mem[count]);
 					return -EIO;
 				}
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->mem[count]->start);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->mem[count]->start);
 				/* _______________________This is for debugging purposes only _______________________*/
-				debug ("b4 writing, start address is %x\n", func->mem[count]->start);
-				pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
-				debug ("after writing, the address is %x\n", bar[count]);
+				debug("b4 writing, start address is %x\n", func->mem[count]->start);
+				pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
+				debug("after writing, the address is %x\n", bar[count]);
 				/* __________________________________________________________________________________*/
 
 				if (bar[count] & PCI_BASE_ADDRESS_MEM_TYPE_64) {
 					/* takes up another dword */
-					debug ("inside mem 64 case, reg. mem, count %d\n", count);
+					debug("inside mem 64 case, reg. mem, count %d\n", count);
 					count += 1;
 					/* on the 2nd dword, write all 0s, since we can't handle them n.e.ways */
-					pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0x00000000);
+					pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0x00000000);
 				}
 			}
 		}		/* end of mem */
 	}			/* end of for */
 
 	func->bus = 0;		/* To indicate that this is not a PPB */
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_INTERRUPT_PIN, &irq);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_INTERRUPT_PIN, &irq);
 	if ((irq > 0x00) && (irq < 0x05))
-		pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_INTERRUPT_LINE, func->irq[irq - 1]);
+		pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_INTERRUPT_LINE, func->irq[irq - 1]);
 
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_CACHE_LINE_SIZE, CACHE);
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_LATENCY_TIMER, LATENCY);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_CACHE_LINE_SIZE, CACHE);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_LATENCY_TIMER, LATENCY);
 
-	pci_bus_write_config_dword (ibmphp_pci_bus, devfn, PCI_ROM_ADDRESS, 0x00L);
-	pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_COMMAND, DEVICEENABLE);
+	pci_bus_write_config_dword(ibmphp_pci_bus, devfn, PCI_ROM_ADDRESS, 0x00L);
+	pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_COMMAND, DEVICEENABLE);
 
 	return 0;
 }
@@ -563,7 +563,7 @@ static int configure_device (struct pci_func *func)
  * Parameters: pci_func
  * Returns:
  ******************************************************************************/
-static int configure_bridge (struct pci_func **func_passed, u8 slotno)
+static int configure_bridge(struct pci_func **func_passed, u8 slotno)
 {
 	int count;
 	int i;
@@ -597,7 +597,7 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 	u8 irq;
 	int retval;
 
-	debug ("%s - enter\n", __func__);
+	debug("%s - enter\n", __func__);
 
 	devfn = PCI_DEVFN(func->function, func->device);
 	ibmphp_pci_bus->number = func->busno;
@@ -606,43 +606,43 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 	 * behind it
 	 */
 
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, func->busno);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, func->busno);
 
 	/* _____________________For debugging purposes only __________________________
-	pci_bus_config_byte (ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, &pri_number);
-	debug ("primary # written into the bridge is %x\n", pri_number);
+	pci_bus_config_byte(ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, &pri_number);
+	debug("primary # written into the bridge is %x\n", pri_number);
 	 ___________________________________________________________________________*/
 
 	/* in EBDA, only get allocated 1 additional bus # per slot */
-	sec_number = find_sec_number (func->busno, slotno);
+	sec_number = find_sec_number(func->busno, slotno);
 	if (sec_number == 0xff) {
-		err ("cannot allocate secondary bus number for the bridged device\n");
+		err("cannot allocate secondary bus number for the bridged device\n");
 		return -EINVAL;
 	}
 
-	debug ("after find_sec_number, the number we got is %x\n", sec_number);
-	debug ("AFTER FIND_SEC_NUMBER, func->busno IS %x\n", func->busno);
+	debug("after find_sec_number, the number we got is %x\n", sec_number);
+	debug("AFTER FIND_SEC_NUMBER, func->busno IS %x\n", func->busno);
 
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, sec_number);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, sec_number);
 
 	/* __________________For debugging purposes only __________________________________
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
-	debug ("sec_number after write/read is %x\n", sec_number);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
+	debug("sec_number after write/read is %x\n", sec_number);
 	 ________________________________________________________________________________*/
 
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, sec_number);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, sec_number);
 
 	/* __________________For debugging purposes only ____________________________________
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, &sec_number);
-	debug ("subordinate number after write/read is %x\n", sec_number);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, &sec_number);
+	debug("subordinate number after write/read is %x\n", sec_number);
 	 __________________________________________________________________________________*/
 
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_CACHE_LINE_SIZE, CACHE);
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_LATENCY_TIMER, LATENCY);
-	pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_SEC_LATENCY_TIMER, LATENCY);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_CACHE_LINE_SIZE, CACHE);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_LATENCY_TIMER, LATENCY);
+	pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_SEC_LATENCY_TIMER, LATENCY);
 
-	debug ("func->busno is %x\n", func->busno);
-	debug ("sec_number after writing is %x\n", sec_number);
+	debug("func->busno is %x\n", func->busno);
+	debug("sec_number after writing is %x\n", sec_number);
 
 
 	/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -652,29 +652,29 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 
 	/* First we need to allocate mem/io for the bridge itself in case it needs it */
 	for (count = 0; address[count]; count++) {	/* for 2 BARs */
-		pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
-		pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
+		pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
+		pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
 
 		if (!bar[count]) {
 			/* This BAR is not implemented */
-			debug ("so we come here then, eh?, count = %d\n", count);
+			debug("so we come here then, eh?, count = %d\n", count);
 			continue;
 		}
 		//  tmp_bar = bar[count];
 
-		debug ("Bar %d wants %x\n", count, bar[count]);
+		debug("Bar %d wants %x\n", count, bar[count]);
 
 		if (bar[count] & PCI_BASE_ADDRESS_SPACE_IO) {
 			/* This is IO */
 			len[count] = bar[count] & 0xFFFFFFFC;
 			len[count] = ~len[count] + 1;
 
-			debug ("len[count] in IO = %x\n", len[count]);
+			debug("len[count] in IO = %x\n", len[count]);
 
 			bus_io[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 
 			if (!bus_io[count]) {
-				err ("out of system memory\n");
+				err("out of system memory\n");
 				retval = -ENOMEM;
 				goto error;
 			}
@@ -683,17 +683,17 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 			bus_io[count]->devfunc = PCI_DEVFN(func->device,
 							func->function);
 			bus_io[count]->len = len[count];
-			if (ibmphp_check_resource (bus_io[count], 0) == 0) {
-				ibmphp_add_resource (bus_io[count]);
+			if (ibmphp_check_resource(bus_io[count], 0) == 0) {
+				ibmphp_add_resource(bus_io[count]);
 				func->io[count] = bus_io[count];
 			} else {
-				err ("cannot allocate requested io for bus %x, device %x, len %x\n",
+				err("cannot allocate requested io for bus %x, device %x, len %x\n",
 				     func->busno, func->device, len[count]);
-				kfree (bus_io[count]);
+				kfree(bus_io[count]);
 				return -EIO;
 			}
 
-			pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->io[count]->start);
+			pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->io[count]->start);
 
 		} else {
 			/* This is Memory */
@@ -702,11 +702,11 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 				len[count] = bar[count] & 0xFFFFFFF0;
 				len[count] = ~len[count] + 1;
 
-				debug ("len[count] in PFMEM = %x\n", len[count]);
+				debug("len[count] in PFMEM = %x\n", len[count]);
 
 				bus_pfmem[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 				if (!bus_pfmem[count]) {
-					err ("out of system memory\n");
+					err("out of system memory\n");
 					retval = -ENOMEM;
 					goto error;
 				}
@@ -716,13 +716,13 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 							func->function);
 				bus_pfmem[count]->len = len[count];
 				bus_pfmem[count]->fromMem = 0;
-				if (ibmphp_check_resource (bus_pfmem[count], 0) == 0) {
-					ibmphp_add_resource (bus_pfmem[count]);
+				if (ibmphp_check_resource(bus_pfmem[count], 0) == 0) {
+					ibmphp_add_resource(bus_pfmem[count]);
 					func->pfmem[count] = bus_pfmem[count];
 				} else {
 					mem_tmp = kzalloc(sizeof(*mem_tmp), GFP_KERNEL);
 					if (!mem_tmp) {
-						err ("out of system memory\n");
+						err("out of system memory\n");
 						retval = -ENOMEM;
 						goto error;
 					}
@@ -730,28 +730,28 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 					mem_tmp->busno = bus_pfmem[count]->busno;
 					mem_tmp->devfunc = bus_pfmem[count]->devfunc;
 					mem_tmp->len = bus_pfmem[count]->len;
-					if (ibmphp_check_resource (mem_tmp, 0) == 0) {
-						ibmphp_add_resource (mem_tmp);
+					if (ibmphp_check_resource(mem_tmp, 0) == 0) {
+						ibmphp_add_resource(mem_tmp);
 						bus_pfmem[count]->fromMem = 1;
 						bus_pfmem[count]->rangeno = mem_tmp->rangeno;
-						ibmphp_add_pfmem_from_mem (bus_pfmem[count]);
+						ibmphp_add_pfmem_from_mem(bus_pfmem[count]);
 						func->pfmem[count] = bus_pfmem[count];
 					} else {
-						err ("cannot allocate requested pfmem for bus %x, device %x, len %x\n",
+						err("cannot allocate requested pfmem for bus %x, device %x, len %x\n",
 						     func->busno, func->device, len[count]);
-						kfree (mem_tmp);
-						kfree (bus_pfmem[count]);
+						kfree(mem_tmp);
+						kfree(bus_pfmem[count]);
 						return -EIO;
 					}
 				}
 
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->pfmem[count]->start);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->pfmem[count]->start);
 
 				if (bar[count] & PCI_BASE_ADDRESS_MEM_TYPE_64) {
 					/* takes up another dword */
 					count += 1;
 					/* on the 2nd dword, write all 0s, since we can't handle them n.e.ways */
-					pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0x00000000);
+					pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0x00000000);
 
 				}
 			} else {
@@ -759,11 +759,11 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 				len[count] = bar[count] & 0xFFFFFFF0;
 				len[count] = ~len[count] + 1;
 
-				debug ("len[count] in Memory is %x\n", len[count]);
+				debug("len[count] in Memory is %x\n", len[count]);
 
 				bus_mem[count] = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 				if (!bus_mem[count]) {
-					err ("out of system memory\n");
+					err("out of system memory\n");
 					retval = -ENOMEM;
 					goto error;
 				}
@@ -772,23 +772,23 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 				bus_mem[count]->devfunc = PCI_DEVFN(func->device,
 							func->function);
 				bus_mem[count]->len = len[count];
-				if (ibmphp_check_resource (bus_mem[count], 0) == 0) {
-					ibmphp_add_resource (bus_mem[count]);
+				if (ibmphp_check_resource(bus_mem[count], 0) == 0) {
+					ibmphp_add_resource(bus_mem[count]);
 					func->mem[count] = bus_mem[count];
 				} else {
-					err ("cannot allocate requested mem for bus %x, device %x, len %x\n",
+					err("cannot allocate requested mem for bus %x, device %x, len %x\n",
 					     func->busno, func->device, len[count]);
-					kfree (bus_mem[count]);
+					kfree(bus_mem[count]);
 					return -EIO;
 				}
 
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], func->mem[count]->start);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], func->mem[count]->start);
 
 				if (bar[count] & PCI_BASE_ADDRESS_MEM_TYPE_64) {
 					/* takes up another dword */
 					count += 1;
 					/* on the 2nd dword, write all 0s, since we can't handle them n.e.ways */
-					pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0x00000000);
+					pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0x00000000);
 
 				}
 			}
@@ -796,45 +796,45 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 	}			/* end of for  */
 
 	/* Now need to see how much space the devices behind the bridge needed */
-	amount_needed = scan_behind_bridge (func, sec_number);
+	amount_needed = scan_behind_bridge(func, sec_number);
 	if (amount_needed == NULL)
 		return -ENOMEM;
 
 	ibmphp_pci_bus->number = func->busno;
-	debug ("after coming back from scan_behind_bridge\n");
-	debug ("amount_needed->not_correct = %x\n", amount_needed->not_correct);
-	debug ("amount_needed->io = %x\n", amount_needed->io);
-	debug ("amount_needed->mem = %x\n", amount_needed->mem);
-	debug ("amount_needed->pfmem =  %x\n", amount_needed->pfmem);
+	debug("after coming back from scan_behind_bridge\n");
+	debug("amount_needed->not_correct = %x\n", amount_needed->not_correct);
+	debug("amount_needed->io = %x\n", amount_needed->io);
+	debug("amount_needed->mem = %x\n", amount_needed->mem);
+	debug("amount_needed->pfmem =  %x\n", amount_needed->pfmem);
 
 	if (amount_needed->not_correct) {
-		debug ("amount_needed is not correct\n");
+		debug("amount_needed is not correct\n");
 		for (count = 0; address[count]; count++) {
 			/* for 2 BARs */
 			if (bus_io[count]) {
-				ibmphp_remove_resource (bus_io[count]);
+				ibmphp_remove_resource(bus_io[count]);
 				func->io[count] = NULL;
 			} else if (bus_pfmem[count]) {
-				ibmphp_remove_resource (bus_pfmem[count]);
+				ibmphp_remove_resource(bus_pfmem[count]);
 				func->pfmem[count] = NULL;
 			} else if (bus_mem[count]) {
-				ibmphp_remove_resource (bus_mem[count]);
+				ibmphp_remove_resource(bus_mem[count]);
 				func->mem[count] = NULL;
 			}
 		}
-		kfree (amount_needed);
+		kfree(amount_needed);
 		return -ENODEV;
 	}
 
 	if (!amount_needed->io) {
-		debug ("it doesn't want IO?\n");
+		debug("it doesn't want IO?\n");
 		flag_io = 1;
 	} else {
-		debug ("it wants %x IO behind the bridge\n", amount_needed->io);
+		debug("it wants %x IO behind the bridge\n", amount_needed->io);
 		io = kzalloc(sizeof(*io), GFP_KERNEL);
 
 		if (!io) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			retval = -ENOMEM;
 			goto error;
 		}
@@ -842,21 +842,21 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 		io->busno = func->busno;
 		io->devfunc = PCI_DEVFN(func->device, func->function);
 		io->len = amount_needed->io;
-		if (ibmphp_check_resource (io, 1) == 0) {
-			debug ("were we able to add io\n");
-			ibmphp_add_resource (io);
+		if (ibmphp_check_resource(io, 1) == 0) {
+			debug("were we able to add io\n");
+			ibmphp_add_resource(io);
 			flag_io = 1;
 		}
 	}
 
 	if (!amount_needed->mem) {
-		debug ("it doesn't want n.e.memory?\n");
+		debug("it doesn't want n.e.memory?\n");
 		flag_mem = 1;
 	} else {
-		debug ("it wants %x memory behind the bridge\n", amount_needed->mem);
+		debug("it wants %x memory behind the bridge\n", amount_needed->mem);
 		mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 		if (!mem) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			retval = -ENOMEM;
 			goto error;
 		}
@@ -864,21 +864,21 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 		mem->busno = func->busno;
 		mem->devfunc = PCI_DEVFN(func->device, func->function);
 		mem->len = amount_needed->mem;
-		if (ibmphp_check_resource (mem, 1) == 0) {
-			ibmphp_add_resource (mem);
+		if (ibmphp_check_resource(mem, 1) == 0) {
+			ibmphp_add_resource(mem);
 			flag_mem = 1;
-			debug ("were we able to add mem\n");
+			debug("were we able to add mem\n");
 		}
 	}
 
 	if (!amount_needed->pfmem) {
-		debug ("it doesn't want n.e.pfmem mem?\n");
+		debug("it doesn't want n.e.pfmem mem?\n");
 		flag_pfmem = 1;
 	} else {
-		debug ("it wants %x pfmemory behind the bridge\n", amount_needed->pfmem);
+		debug("it wants %x pfmemory behind the bridge\n", amount_needed->pfmem);
 		pfmem = kzalloc(sizeof(*pfmem), GFP_KERNEL);
 		if (!pfmem) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			retval = -ENOMEM;
 			goto error;
 		}
@@ -887,13 +887,13 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 		pfmem->devfunc = PCI_DEVFN(func->device, func->function);
 		pfmem->len = amount_needed->pfmem;
 		pfmem->fromMem = 0;
-		if (ibmphp_check_resource (pfmem, 1) == 0) {
-			ibmphp_add_resource (pfmem);
+		if (ibmphp_check_resource(pfmem, 1) == 0) {
+			ibmphp_add_resource(pfmem);
 			flag_pfmem = 1;
 		} else {
 			mem_tmp = kzalloc(sizeof(*mem_tmp), GFP_KERNEL);
 			if (!mem_tmp) {
-				err ("out of system memory\n");
+				err("out of system memory\n");
 				retval = -ENOMEM;
 				goto error;
 			}
@@ -901,18 +901,18 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 			mem_tmp->busno = pfmem->busno;
 			mem_tmp->devfunc = pfmem->devfunc;
 			mem_tmp->len = pfmem->len;
-			if (ibmphp_check_resource (mem_tmp, 1) == 0) {
-				ibmphp_add_resource (mem_tmp);
+			if (ibmphp_check_resource(mem_tmp, 1) == 0) {
+				ibmphp_add_resource(mem_tmp);
 				pfmem->fromMem = 1;
 				pfmem->rangeno = mem_tmp->rangeno;
-				ibmphp_add_pfmem_from_mem (pfmem);
+				ibmphp_add_pfmem_from_mem(pfmem);
 				flag_pfmem = 1;
 			}
 		}
 	}
 
-	debug ("b4 if (flag_io && flag_mem && flag_pfmem)\n");
-	debug ("flag_io = %x, flag_mem = %x, flag_pfmem = %x\n", flag_io, flag_mem, flag_pfmem);
+	debug("b4 if (flag_io && flag_mem && flag_pfmem)\n");
+	debug("flag_io = %x, flag_mem = %x, flag_pfmem = %x\n", flag_io, flag_mem, flag_pfmem);
 
 	if (flag_io && flag_mem && flag_pfmem) {
 		/* If on bootup, there was a bridged card in this slot,
@@ -920,127 +920,127 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 		 * back again, there's no way for us to remove the bus
 		 * struct, so no need to kmalloc, can use existing node
 		 */
-		bus = ibmphp_find_res_bus (sec_number);
+		bus = ibmphp_find_res_bus(sec_number);
 		if (!bus) {
 			bus = kzalloc(sizeof(*bus), GFP_KERNEL);
 			if (!bus) {
-				err ("out of system memory\n");
+				err("out of system memory\n");
 				retval = -ENOMEM;
 				goto error;
 			}
 			bus->busno = sec_number;
-			debug ("b4 adding new bus\n");
-			rc = add_new_bus (bus, io, mem, pfmem, func->busno);
+			debug("b4 adding new bus\n");
+			rc = add_new_bus(bus, io, mem, pfmem, func->busno);
 		} else if (!(bus->rangeIO) && !(bus->rangeMem) && !(bus->rangePFMem))
-			rc = add_new_bus (bus, io, mem, pfmem, 0xFF);
+			rc = add_new_bus(bus, io, mem, pfmem, 0xFF);
 		else {
-			err ("expected bus structure not empty?\n");
+			err("expected bus structure not empty?\n");
 			retval = -EIO;
 			goto error;
 		}
 		if (rc) {
 			if (rc == -ENOMEM) {
-				ibmphp_remove_bus (bus, func->busno);
-				kfree (amount_needed);
+				ibmphp_remove_bus(bus, func->busno);
+				kfree(amount_needed);
 				return rc;
 			}
 			retval = rc;
 			goto error;
 		}
-		pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_IO_BASE, &io_base);
-		pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &pfmem_base);
+		pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_IO_BASE, &io_base);
+		pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &pfmem_base);
 
 		if ((io_base & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
-			debug ("io 32\n");
+			debug("io 32\n");
 			need_io_upper = 1;
 		}
 		if ((pfmem_base & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
-			debug ("pfmem 64\n");
+			debug("pfmem 64\n");
 			need_pfmem_upper = 1;
 		}
 
 		if (bus->noIORanges) {
-			pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_IO_BASE, 0x00 | bus->rangeIO->start >> 8);
-			pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_IO_LIMIT, 0x00 | bus->rangeIO->end >> 8);
+			pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_IO_BASE, 0x00 | bus->rangeIO->start >> 8);
+			pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_IO_LIMIT, 0x00 | bus->rangeIO->end >> 8);
 
 			/* _______________This is for debugging purposes only ____________________
-			pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_IO_BASE, &temp);
-			debug ("io_base = %x\n", (temp & PCI_IO_RANGE_TYPE_MASK) << 8);
-			pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_IO_LIMIT, &temp);
-			debug ("io_limit = %x\n", (temp & PCI_IO_RANGE_TYPE_MASK) << 8);
+			pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_IO_BASE, &temp);
+			debug("io_base = %x\n", (temp & PCI_IO_RANGE_TYPE_MASK) << 8);
+			pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_IO_LIMIT, &temp);
+			debug("io_limit = %x\n", (temp & PCI_IO_RANGE_TYPE_MASK) << 8);
 			 ________________________________________________________________________*/
 
 			if (need_io_upper) {	/* since can't support n.e.ways */
-				pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_IO_BASE_UPPER16, 0x0000);
-				pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_IO_LIMIT_UPPER16, 0x0000);
+				pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_IO_BASE_UPPER16, 0x0000);
+				pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_IO_LIMIT_UPPER16, 0x0000);
 			}
 		} else {
-			pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_IO_BASE, 0x00);
-			pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_IO_LIMIT, 0x00);
+			pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_IO_BASE, 0x00);
+			pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_IO_LIMIT, 0x00);
 		}
 
 		if (bus->noMemRanges) {
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, 0x0000 | bus->rangeMem->start >> 16);
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, 0x0000 | bus->rangeMem->end >> 16);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, 0x0000 | bus->rangeMem->start >> 16);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, 0x0000 | bus->rangeMem->end >> 16);
 
 			/* ____________________This is for debugging purposes only ________________________
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, &temp);
-			debug ("mem_base = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, &temp);
-			debug ("mem_limit = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, &temp);
+			debug("mem_base = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, &temp);
+			debug("mem_limit = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
 			 __________________________________________________________________________________*/
 
 		} else {
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, 0xffff);
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, 0x0000);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, 0xffff);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, 0x0000);
 		}
 		if (bus->noPFMemRanges) {
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, 0x0000 | bus->rangePFMem->start >> 16);
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, 0x0000 | bus->rangePFMem->end >> 16);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, 0x0000 | bus->rangePFMem->start >> 16);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, 0x0000 | bus->rangePFMem->end >> 16);
 
 			/* __________________________This is for debugging purposes only _______________________
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &temp);
-			debug ("pfmem_base = %x", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, &temp);
-			debug ("pfmem_limit = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &temp);
+			debug("pfmem_base = %x", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, &temp);
+			debug("pfmem_limit = %x\n", (temp & PCI_MEMORY_RANGE_TYPE_MASK) << 16);
 			 ______________________________________________________________________________________*/
 
 			if (need_pfmem_upper) {	/* since can't support n.e.ways */
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, PCI_PREF_BASE_UPPER32, 0x00000000);
-				pci_bus_write_config_dword (ibmphp_pci_bus, devfn, PCI_PREF_LIMIT_UPPER32, 0x00000000);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, PCI_PREF_BASE_UPPER32, 0x00000000);
+				pci_bus_write_config_dword(ibmphp_pci_bus, devfn, PCI_PREF_LIMIT_UPPER32, 0x00000000);
 			}
 		} else {
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, 0xffff);
-			pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, 0x0000);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, 0xffff);
+			pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, 0x0000);
 		}
 
-		debug ("b4 writing control information\n");
+		debug("b4 writing control information\n");
 
-		pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_INTERRUPT_PIN, &irq);
+		pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_INTERRUPT_PIN, &irq);
 		if ((irq > 0x00) && (irq < 0x05))
-			pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_INTERRUPT_LINE, func->irq[irq - 1]);
+			pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_INTERRUPT_LINE, func->irq[irq - 1]);
 		/*
-		pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, ctrl);
-		pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, PCI_BRIDGE_CTL_PARITY);
-		pci_bus_write_config_byte (ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, PCI_BRIDGE_CTL_SERR);
+		pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, ctrl);
+		pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, PCI_BRIDGE_CTL_PARITY);
+		pci_bus_write_config_byte(ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, PCI_BRIDGE_CTL_SERR);
 		 */
 
-		pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_COMMAND, DEVICEENABLE);
-		pci_bus_write_config_word (ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, 0x07);
+		pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_COMMAND, DEVICEENABLE);
+		pci_bus_write_config_word(ibmphp_pci_bus, devfn, PCI_BRIDGE_CONTROL, 0x07);
 		for (i = 0; i < 32; i++) {
 			if (amount_needed->devices[i]) {
-				debug ("device where devices[i] is 1 = %x\n", i);
+				debug("device where devices[i] is 1 = %x\n", i);
 				func->devices[i] = 1;
 			}
 		}
 		func->bus = 1;	/* For unconfiguring, to indicate it's PPB */
 		func_passed = &func;
-		debug ("func->busno b4 returning is %x\n", func->busno);
-		debug ("func->busno b4 returning in the other structure is %x\n", (*func_passed)->busno);
-		kfree (amount_needed);
+		debug("func->busno b4 returning is %x\n", func->busno);
+		debug("func->busno b4 returning in the other structure is %x\n", (*func_passed)->busno);
+		kfree(amount_needed);
 		return 0;
 	} else {
-		err ("Configuring bridge was unsuccessful...\n");
+		err("Configuring bridge was unsuccessful...\n");
 		mem_tmp = NULL;
 		retval = -EIO;
 		goto error;
@@ -1049,20 +1049,20 @@ static int configure_bridge (struct pci_func **func_passed, u8 slotno)
 error:
 	kfree(amount_needed);
 	if (pfmem)
-		ibmphp_remove_resource (pfmem);
+		ibmphp_remove_resource(pfmem);
 	if (io)
-		ibmphp_remove_resource (io);
+		ibmphp_remove_resource(io);
 	if (mem)
-		ibmphp_remove_resource (mem);
+		ibmphp_remove_resource(mem);
 	for (i = 0; i < 2; i++) {	/* for 2 BARs */
 		if (bus_io[i]) {
-			ibmphp_remove_resource (bus_io[i]);
+			ibmphp_remove_resource(bus_io[i]);
 			func->io[i] = NULL;
 		} else if (bus_pfmem[i]) {
-			ibmphp_remove_resource (bus_pfmem[i]);
+			ibmphp_remove_resource(bus_pfmem[i]);
 			func->pfmem[i] = NULL;
 		} else if (bus_mem[i]) {
-			ibmphp_remove_resource (bus_mem[i]);
+			ibmphp_remove_resource(bus_mem[i]);
 			func->mem[i] = NULL;
 		}
 	}
@@ -1075,7 +1075,7 @@ error:
  * Input: bridge function
  * Output: amount of resources needed
  *****************************************************************************/
-static struct res_needed *scan_behind_bridge (struct pci_func *func, u8 busno)
+static struct res_needed *scan_behind_bridge(struct pci_func *func, u8 busno)
 {
 	int count, len[6];
 	u16 vendor_id;
@@ -1102,36 +1102,36 @@ static struct res_needed *scan_behind_bridge (struct pci_func *func, u8 busno)
 
 	ibmphp_pci_bus->number = busno;
 
-	debug ("the bus_no behind the bridge is %x\n", busno);
-	debug ("scanning devices behind the bridge...\n");
+	debug("the bus_no behind the bridge is %x\n", busno);
+	debug("scanning devices behind the bridge...\n");
 	for (device = 0; device < 32; device++) {
 		amount->devices[device] = 0;
 		for (function = 0; function < 8; function++) {
 			devfn = PCI_DEVFN(device, function);
 
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
 
 			if (vendor_id != PCI_VENDOR_ID_NOTVALID) {
 				/* found correct device!!! */
 				howmany++;
 
-				pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
-				pci_bus_read_config_dword (ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
+				pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
+				pci_bus_read_config_dword(ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
 
-				debug ("hdr_type behind the bridge is %x\n", hdr_type);
-				if (hdr_type & PCI_HEADER_TYPE_BRIDGE) {
-					err ("embedded bridges not supported for hot-plugging.\n");
+				debug("hdr_type behind the bridge is %x\n", hdr_type);
+				if ((hdr_type & 0x7f) == PCI_HEADER_TYPE_BRIDGE) {
+					err("embedded bridges not supported for hot-plugging.\n");
 					amount->not_correct = 1;
 					return amount;
 				}
 
 				class >>= 8;	/* to take revision out, class = class.subclass.prog i/f */
 				if (class == PCI_CLASS_NOT_DEFINED_VGA) {
-					err ("The device %x is VGA compatible and as is not supported for hot plugging.  Please choose another device.\n", device);
+					err("The device %x is VGA compatible and as is not supported for hot plugging.  Please choose another device.\n", device);
 					amount->not_correct = 1;
 					return amount;
 				} else if (class == PCI_CLASS_DISPLAY_VGA) {
-					err ("The device %x is not supported for hot plugging.  Please choose another device.\n", device);
+					err("The device %x is not supported for hot plugging.  Please choose another device.\n", device);
 					amount->not_correct = 1;
 					return amount;
 				}
@@ -1141,23 +1141,23 @@ static struct res_needed *scan_behind_bridge (struct pci_func *func, u8 busno)
 				for (count = 0; address[count]; count++) {
 					/* for 6 BARs */
 					/*
-					pci_bus_read_config_byte (ibmphp_pci_bus, devfn, address[count], &tmp);
+					pci_bus_read_config_byte(ibmphp_pci_bus, devfn, address[count], &tmp);
 					if (tmp & 0x01) // IO
-						pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFD);
+						pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFD);
 					else // MEMORY
-						pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
+						pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
 					*/
-					pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
-					pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &bar[count]);
+					pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
+					pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &bar[count]);
 
-					debug ("what is bar[count]? %x, count = %d\n", bar[count], count);
+					debug("what is bar[count]? %x, count = %d\n", bar[count], count);
 
 					if (!bar[count])	/* This BAR is not implemented */
 						continue;
 
 					//tmp_bar = bar[count];
 
-					debug ("count %d device %x function %x wants %x resources\n", count, device, function, bar[count]);
+					debug("count %d device %x function %x wants %x resources\n", count, device, function, bar[count]);
 
 					if (bar[count] & PCI_BASE_ADDRESS_SPACE_IO) {
 						/* This is IO */
@@ -1211,7 +1211,7 @@ static struct res_needed *scan_behind_bridge (struct pci_func *func, u8 busno)
  * Change: we also call these functions even if we configured the card ourselves (i.e., not
  * the bootup case), since it should work same way
  */
-static int unconfigure_boot_device (u8 busno, u8 device, u8 function)
+static int unconfigure_boot_device(u8 busno, u8 device, u8 function)
 {
 	u32 start_address;
 	u32 address[] = {
@@ -1234,30 +1234,30 @@ static int unconfigure_boot_device (u8 busno, u8 device, u8 function)
 	u32 tmp_address;
 	unsigned int devfn;
 
-	debug ("%s - enter\n", __func__);
+	debug("%s - enter\n", __func__);
 
-	bus = ibmphp_find_res_bus (busno);
+	bus = ibmphp_find_res_bus(busno);
 	if (!bus) {
-		debug ("cannot find corresponding bus.\n");
+		debug("cannot find corresponding bus.\n");
 		return -EINVAL;
 	}
 
 	devfn = PCI_DEVFN(device, function);
 	ibmphp_pci_bus->number = busno;
 	for (count = 0; address[count]; count++) {	/* for 6 BARs */
-		pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &start_address);
+		pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &start_address);
 
 		/* We can do this here, b/c by that time the device driver of the card has been stopped */
 
-		pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
-		pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &size);
-		pci_bus_write_config_dword (ibmphp_pci_bus, devfn, address[count], start_address);
+		pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], 0xFFFFFFFF);
+		pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &size);
+		pci_bus_write_config_dword(ibmphp_pci_bus, devfn, address[count], start_address);
 
-		debug ("start_address is %x\n", start_address);
-		debug ("busno, device, function %x %x %x\n", busno, device, function);
+		debug("start_address is %x\n", start_address);
+		debug("busno, device, function %x %x %x\n", busno, device, function);
 		if (!size) {
 			/* This BAR is not implemented */
-			debug ("is this bar no implemented?, count = %d\n", count);
+			debug("is this bar no implemented?, count = %d\n", count);
 			continue;
 		}
 		tmp_address = start_address;
@@ -1267,24 +1267,24 @@ static int unconfigure_boot_device (u8 busno, u8 device, u8 function)
 			size = size & 0xFFFFFFFC;
 			size = ~size + 1;
 			end_address = start_address + size - 1;
-			if (ibmphp_find_resource (bus, start_address, &io, IO) < 0) {
-				err ("cannot find corresponding IO resource to remove\n");
+			if (ibmphp_find_resource(bus, start_address, &io, IO) < 0) {
+				err("cannot find corresponding IO resource to remove\n");
 				return -EIO;
 			}
-			debug ("io->start = %x\n", io->start);
+			debug("io->start = %x\n", io->start);
 			temp_end = io->end;
 			start_address = io->end + 1;
-			ibmphp_remove_resource (io);
+			ibmphp_remove_resource(io);
 			/* This is needed b/c of the old I/O restrictions in the BIOS */
 			while (temp_end < end_address) {
-				if (ibmphp_find_resource (bus, start_address, &io, IO) < 0) {
-					err ("cannot find corresponding IO resource to remove\n");
+				if (ibmphp_find_resource(bus, start_address, &io, IO) < 0) {
+					err("cannot find corresponding IO resource to remove\n");
 					return -EIO;
 				}
-				debug ("io->start = %x\n", io->start);
+				debug("io->start = %x\n", io->start);
 				temp_end = io->end;
 				start_address = io->end + 1;
-				ibmphp_remove_resource (io);
+				ibmphp_remove_resource(io);
 			}
 
 			/* ????????? DO WE NEED TO WRITE ANYTHING INTO THE PCI CONFIG SPACE BACK ?????????? */
@@ -1292,29 +1292,29 @@ static int unconfigure_boot_device (u8 busno, u8 device, u8 function)
 			/* This is Memory */
 			if (start_address & PCI_BASE_ADDRESS_MEM_PREFETCH) {
 				/* pfmem */
-				debug ("start address of pfmem is %x\n", start_address);
+				debug("start address of pfmem is %x\n", start_address);
 				start_address &= PCI_BASE_ADDRESS_MEM_MASK;
 
-				if (ibmphp_find_resource (bus, start_address, &pfmem, PFMEM) < 0) {
-					err ("cannot find corresponding PFMEM resource to remove\n");
+				if (ibmphp_find_resource(bus, start_address, &pfmem, PFMEM) < 0) {
+					err("cannot find corresponding PFMEM resource to remove\n");
 					return -EIO;
 				}
 				if (pfmem) {
-					debug ("pfmem->start = %x\n", pfmem->start);
+					debug("pfmem->start = %x\n", pfmem->start);
 
 					ibmphp_remove_resource(pfmem);
 				}
 			} else {
 				/* regular memory */
-				debug ("start address of mem is %x\n", start_address);
+				debug("start address of mem is %x\n", start_address);
 				start_address &= PCI_BASE_ADDRESS_MEM_MASK;
 
-				if (ibmphp_find_resource (bus, start_address, &mem, MEM) < 0) {
-					err ("cannot find corresponding MEM resource to remove\n");
+				if (ibmphp_find_resource(bus, start_address, &mem, MEM) < 0) {
+					err("cannot find corresponding MEM resource to remove\n");
 					return -EIO;
 				}
 				if (mem) {
-					debug ("mem->start = %x\n", mem->start);
+					debug("mem->start = %x\n", mem->start);
 
 					ibmphp_remove_resource(mem);
 				}
@@ -1329,7 +1329,7 @@ static int unconfigure_boot_device (u8 busno, u8 device, u8 function)
 	return 0;
 }
 
-static int unconfigure_boot_bridge (u8 busno, u8 device, u8 function)
+static int unconfigure_boot_bridge(u8 busno, u8 device, u8 function)
 {
 	int count;
 	int bus_no, pri_no, sub_no, sec_no = 0;
@@ -1349,40 +1349,40 @@ static int unconfigure_boot_bridge (u8 busno, u8 device, u8 function)
 	devfn = PCI_DEVFN(device, function);
 	ibmphp_pci_bus->number = busno;
 	bus_no = (int) busno;
-	debug ("busno is %x\n", busno);
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, &pri_number);
-	debug ("%s - busno = %x, primary_number = %x\n", __func__, busno, pri_number);
+	debug("busno is %x\n", busno);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_PRIMARY_BUS, &pri_number);
+	debug("%s - busno = %x, primary_number = %x\n", __func__, busno, pri_number);
 
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
-	debug ("sec_number is %x\n", sec_number);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_number);
+	debug("sec_number is %x\n", sec_number);
 	sec_no = (int) sec_number;
 	pri_no = (int) pri_number;
 	if (pri_no != bus_no) {
-		err ("primary numbers in our structures and pci config space don't match.\n");
+		err("primary numbers in our structures and pci config space don't match.\n");
 		return -EINVAL;
 	}
 
-	pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, &sub_number);
+	pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SUBORDINATE_BUS, &sub_number);
 	sub_no = (int) sub_number;
-	debug ("sub_no is %d, sec_no is %d\n", sub_no, sec_no);
+	debug("sub_no is %d, sec_no is %d\n", sub_no, sec_no);
 	if (sec_no != sub_number) {
-		err ("there're more buses behind this bridge.  Hot removal is not supported.  Please choose another card\n");
+		err("there're more buses behind this bridge.  Hot removal is not supported.  Please choose another card\n");
 		return -ENODEV;
 	}
 
-	bus = ibmphp_find_res_bus (sec_number);
+	bus = ibmphp_find_res_bus(sec_number);
 	if (!bus) {
-		err ("cannot find Bus structure for the bridged device\n");
+		err("cannot find Bus structure for the bridged device\n");
 		return -EINVAL;
 	}
 	debug("bus->busno is %x\n", bus->busno);
 	debug("sec_number is %x\n", sec_number);
 
-	ibmphp_remove_bus (bus, busno);
+	ibmphp_remove_bus(bus, busno);
 
 	for (count = 0; address[count]; count++) {
 		/* for 2 BARs */
-		pci_bus_read_config_dword (ibmphp_pci_bus, devfn, address[count], &start_address);
+		pci_bus_read_config_dword(ibmphp_pci_bus, devfn, address[count], &start_address);
 
 		if (!start_address) {
 			/* This BAR is not implemented */
@@ -1394,14 +1394,14 @@ static int unconfigure_boot_bridge (u8 busno, u8 device, u8 function)
 		if (start_address & PCI_BASE_ADDRESS_SPACE_IO) {
 			/* This is IO */
 			start_address &= PCI_BASE_ADDRESS_IO_MASK;
-			if (ibmphp_find_resource (bus, start_address, &io, IO) < 0) {
-				err ("cannot find corresponding IO resource to remove\n");
+			if (ibmphp_find_resource(bus, start_address, &io, IO) < 0) {
+				err("cannot find corresponding IO resource to remove\n");
 				return -EIO;
 			}
 			if (io)
-				debug ("io->start = %x\n", io->start);
+				debug("io->start = %x\n", io->start);
 
-			ibmphp_remove_resource (io);
+			ibmphp_remove_resource(io);
 
 			/* ????????? DO WE NEED TO WRITE ANYTHING INTO THE PCI CONFIG SPACE BACK ?????????? */
 		} else {
@@ -1409,24 +1409,24 @@ static int unconfigure_boot_bridge (u8 busno, u8 device, u8 function)
 			if (start_address & PCI_BASE_ADDRESS_MEM_PREFETCH) {
 				/* pfmem */
 				start_address &= PCI_BASE_ADDRESS_MEM_MASK;
-				if (ibmphp_find_resource (bus, start_address, &pfmem, PFMEM) < 0) {
-					err ("cannot find corresponding PFMEM resource to remove\n");
+				if (ibmphp_find_resource(bus, start_address, &pfmem, PFMEM) < 0) {
+					err("cannot find corresponding PFMEM resource to remove\n");
 					return -EINVAL;
 				}
 				if (pfmem) {
-					debug ("pfmem->start = %x\n", pfmem->start);
+					debug("pfmem->start = %x\n", pfmem->start);
 
 					ibmphp_remove_resource(pfmem);
 				}
 			} else {
 				/* regular memory */
 				start_address &= PCI_BASE_ADDRESS_MEM_MASK;
-				if (ibmphp_find_resource (bus, start_address, &mem, MEM) < 0) {
-					err ("cannot find corresponding MEM resource to remove\n");
+				if (ibmphp_find_resource(bus, start_address, &mem, MEM) < 0) {
+					err("cannot find corresponding MEM resource to remove\n");
 					return -EINVAL;
 				}
 				if (mem) {
-					debug ("mem->start = %x\n", mem->start);
+					debug("mem->start = %x\n", mem->start);
 
 					ibmphp_remove_resource(mem);
 				}
@@ -1437,11 +1437,11 @@ static int unconfigure_boot_bridge (u8 busno, u8 device, u8 function)
 			}
 		}	/* end of mem */
 	}	/* end of for */
-	debug ("%s - exiting, returning success\n", __func__);
+	debug("%s - exiting, returning success\n", __func__);
 	return 0;
 }
 
-static int unconfigure_boot_card (struct slot *slot_cur)
+static int unconfigure_boot_card(struct slot *slot_cur)
 {
 	u16 vendor_id;
 	u32 class;
@@ -1453,57 +1453,57 @@ static int unconfigure_boot_card (struct slot *slot_cur)
 	unsigned int devfn;
 	u8 valid_device = 0x00; /* To see if we are ever able to find valid device and read it */
 
-	debug ("%s - enter\n", __func__);
+	debug("%s - enter\n", __func__);
 
 	device = slot_cur->device;
 	busno = slot_cur->bus;
 
-	debug ("b4 for loop, device is %x\n", device);
+	debug("b4 for loop, device is %x\n", device);
 	/* For every function on the card */
 	for (function = 0x0; function < 0x08; function++) {
 		devfn = PCI_DEVFN(device, function);
 		ibmphp_pci_bus->number = busno;
 
-		pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
+		pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
 
 		if (vendor_id != PCI_VENDOR_ID_NOTVALID) {
 			/* found correct device!!! */
 			++valid_device;
 
-			debug ("%s - found correct device\n", __func__);
+			debug("%s - found correct device\n", __func__);
 
 			/* header: x x x x x x x x
 			 *         | |___________|=> 1=PPB bridge, 0=normal device, 2=CardBus Bridge
 			 *         |_=> 0 = single function device, 1 = multi-function device
 			 */
 
-			pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
-			pci_bus_read_config_dword (ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
+			pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
+			pci_bus_read_config_dword(ibmphp_pci_bus, devfn, PCI_CLASS_REVISION, &class);
 
-			debug ("hdr_type %x, class %x\n", hdr_type, class);
+			debug("hdr_type %x, class %x\n", hdr_type, class);
 			class >>= 8;	/* to take revision out, class = class.subclass.prog i/f */
 			if (class == PCI_CLASS_NOT_DEFINED_VGA) {
-				err ("The device %x function %x is VGA compatible and is not supported for hot removing.  Please choose another device.\n", device, function);
+				err("The device %x function %x is VGA compatible and is not supported for hot removing.  Please choose another device.\n", device, function);
 				return -ENODEV;
 			} else if (class == PCI_CLASS_DISPLAY_VGA) {
-				err ("The device %x function %x is not supported for hot removing.  Please choose another device.\n", device, function);
+				err("The device %x function %x is not supported for hot removing.  Please choose another device.\n", device, function);
 				return -ENODEV;
 			}
 
 			switch (hdr_type) {
 				case PCI_HEADER_TYPE_NORMAL:
-					rc = unconfigure_boot_device (busno, device, function);
+					rc = unconfigure_boot_device(busno, device, function);
 					if (rc) {
-						err ("was not able to unconfigure device %x func %x on bus %x. bailing out...\n",
+						err("was not able to unconfigure device %x func %x on bus %x. bailing out...\n",
 						     device, function, busno);
 						return rc;
 					}
 					function = 0x8;
 					break;
 				case PCI_HEADER_TYPE_MULTIDEVICE:
-					rc = unconfigure_boot_device (busno, device, function);
+					rc = unconfigure_boot_device(busno, device, function);
 					if (rc) {
-						err ("was not able to unconfigure device %x func %x on bus %x. bailing out...\n",
+						err("was not able to unconfigure device %x func %x on bus %x. bailing out...\n",
 						     device, function, busno);
 						return rc;
 					}
@@ -1511,12 +1511,12 @@ static int unconfigure_boot_card (struct slot *slot_cur)
 				case PCI_HEADER_TYPE_BRIDGE:
 					class >>= 8;
 					if (class != PCI_CLASS_BRIDGE_PCI) {
-						err ("This device %x function %x is not PCI-to-PCI bridge, and is not supported for hot-removing.  Please try another card.\n", device, function);
+						err("This device %x function %x is not PCI-to-PCI bridge, and is not supported for hot-removing.  Please try another card.\n", device, function);
 						return -ENODEV;
 					}
-					rc = unconfigure_boot_bridge (busno, device, function);
+					rc = unconfigure_boot_bridge(busno, device, function);
 					if (rc != 0) {
-						err ("was not able to hot-remove PPB properly.\n");
+						err("was not able to hot-remove PPB properly.\n");
 						return rc;
 					}
 
@@ -1525,17 +1525,17 @@ static int unconfigure_boot_card (struct slot *slot_cur)
 				case PCI_HEADER_TYPE_MULTIBRIDGE:
 					class >>= 8;
 					if (class != PCI_CLASS_BRIDGE_PCI) {
-						err ("This device %x function %x is not PCI-to-PCI bridge,  and is not supported for hot-removing.  Please try another card.\n", device, function);
+						err("This device %x function %x is not PCI-to-PCI bridge,  and is not supported for hot-removing.  Please try another card.\n", device, function);
 						return -ENODEV;
 					}
-					rc = unconfigure_boot_bridge (busno, device, function);
+					rc = unconfigure_boot_bridge(busno, device, function);
 					if (rc != 0) {
-						err ("was not able to hot-remove PPB properly.\n");
+						err("was not able to hot-remove PPB properly.\n");
 						return rc;
 					}
 					break;
 				default:
-					err ("MAJOR PROBLEM!!!! Cannot read device's header\n");
+					err("MAJOR PROBLEM!!!! Cannot read device's header\n");
 					return -1;
 					break;
 			}	/* end of switch */
@@ -1543,7 +1543,7 @@ static int unconfigure_boot_card (struct slot *slot_cur)
 	}	/* end of for */
 
 	if (!valid_device) {
-		err ("Could not find device to unconfigure.  Or could not read the card.\n");
+		err("Could not find device to unconfigure.  Or could not read the card.\n");
 		return -1;
 	}
 	return 0;
@@ -1558,7 +1558,7 @@ static int unconfigure_boot_card (struct slot *slot_cur)
  *			!!!!!!!!!!!!!!!!!!!!!!!!!FOR BUSES!!!!!!!!!!!!
  * Returns: 0, -1, -ENODEV
  */
-int ibmphp_unconfigure_card (struct slot **slot_cur, int the_end)
+int ibmphp_unconfigure_card(struct slot **slot_cur, int the_end)
 {
 	int i;
 	int count;
@@ -1567,11 +1567,11 @@ int ibmphp_unconfigure_card (struct slot **slot_cur, int the_end)
 	struct pci_func *cur_func = NULL;
 	struct pci_func *temp_func;
 
-	debug ("%s - enter\n", __func__);
+	debug("%s - enter\n", __func__);
 
 	if (!the_end) {
 		/* Need to unconfigure the card */
-		rc = unconfigure_boot_card (sl);
+		rc = unconfigure_boot_card(sl);
 		if ((rc == -ENODEV) || (rc == -EIO) || (rc == -EINVAL)) {
 			/* In all other cases, will still need to get rid of func structure if it exists */
 			return rc;
@@ -1591,34 +1591,34 @@ int ibmphp_unconfigure_card (struct slot **slot_cur, int the_end)
 
 			for (i = 0; i < count; i++) {
 				if (cur_func->io[i]) {
-					debug ("io[%d] exists\n", i);
+					debug("io[%d] exists\n", i);
 					if (the_end > 0)
-						ibmphp_remove_resource (cur_func->io[i]);
+						ibmphp_remove_resource(cur_func->io[i]);
 					cur_func->io[i] = NULL;
 				}
 				if (cur_func->mem[i]) {
-					debug ("mem[%d] exists\n", i);
+					debug("mem[%d] exists\n", i);
 					if (the_end > 0)
-						ibmphp_remove_resource (cur_func->mem[i]);
+						ibmphp_remove_resource(cur_func->mem[i]);
 					cur_func->mem[i] = NULL;
 				}
 				if (cur_func->pfmem[i]) {
-					debug ("pfmem[%d] exists\n", i);
+					debug("pfmem[%d] exists\n", i);
 					if (the_end > 0)
-						ibmphp_remove_resource (cur_func->pfmem[i]);
+						ibmphp_remove_resource(cur_func->pfmem[i]);
 					cur_func->pfmem[i] = NULL;
 				}
 			}
 
 			temp_func = cur_func->next;
-			kfree (cur_func);
+			kfree(cur_func);
 			cur_func = temp_func;
 		}
 	}
 
 	sl->func = NULL;
 	*slot_cur = sl;
-	debug ("%s - exit\n", __func__);
+	debug("%s - exit\n", __func__);
 	return 0;
 }
 
@@ -1630,7 +1630,7 @@ int ibmphp_unconfigure_card (struct slot **slot_cur, int the_end)
  * Output: bus added to the correct spot
  *         0, -1, error
  */
-static int add_new_bus (struct bus_node *bus, struct resource_node *io, struct resource_node *mem, struct resource_node *pfmem, u8 parent_busno)
+static int add_new_bus(struct bus_node *bus, struct resource_node *io, struct resource_node *mem, struct resource_node *pfmem, u8 parent_busno)
 {
 	struct range_node *io_range = NULL;
 	struct range_node *mem_range = NULL;
@@ -1639,18 +1639,18 @@ static int add_new_bus (struct bus_node *bus, struct resource_node *io, struct r
 
 	/* Trying to find the parent bus number */
 	if (parent_busno != 0xFF) {
-		cur_bus	= ibmphp_find_res_bus (parent_busno);
+		cur_bus	= ibmphp_find_res_bus(parent_busno);
 		if (!cur_bus) {
-			err ("strange, cannot find bus which is supposed to be at the system... something is terribly wrong...\n");
+			err("strange, cannot find bus which is supposed to be at the system... something is terribly wrong...\n");
 			return -ENODEV;
 		}
 
-		list_add (&bus->bus_list, &cur_bus->bus_list);
+		list_add(&bus->bus_list, &cur_bus->bus_list);
 	}
 	if (io) {
 		io_range = kzalloc(sizeof(*io_range), GFP_KERNEL);
 		if (!io_range) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			return -ENOMEM;
 		}
 		io_range->start = io->start;
@@ -1662,7 +1662,7 @@ static int add_new_bus (struct bus_node *bus, struct resource_node *io, struct r
 	if (mem) {
 		mem_range = kzalloc(sizeof(*mem_range), GFP_KERNEL);
 		if (!mem_range) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			return -ENOMEM;
 		}
 		mem_range->start = mem->start;
@@ -1674,7 +1674,7 @@ static int add_new_bus (struct bus_node *bus, struct resource_node *io, struct r
 	if (pfmem) {
 		pfmem_range = kzalloc(sizeof(*pfmem_range), GFP_KERNEL);
 		if (!pfmem_range) {
-			err ("out of system memory\n");
+			err("out of system memory\n");
 			return -ENOMEM;
 		}
 		pfmem_range->start = pfmem->start;
@@ -1691,27 +1691,27 @@ static int add_new_bus (struct bus_node *bus, struct resource_node *io, struct r
  * Parameters: bus_number of the primary bus
  * Returns: bus_number of the secondary bus or 0xff in case of failure
  */
-static u8 find_sec_number (u8 primary_busno, u8 slotno)
+static u8 find_sec_number(u8 primary_busno, u8 slotno)
 {
 	int min, max;
 	u8 busno;
 	struct bus_info *bus;
 	struct bus_node *bus_cur;
 
-	bus = ibmphp_find_same_bus_num (primary_busno);
+	bus = ibmphp_find_same_bus_num(primary_busno);
 	if (!bus) {
-		err ("cannot get slot range of the bus from the BIOS\n");
+		err("cannot get slot range of the bus from the BIOS\n");
 		return 0xff;
 	}
 	max = bus->slot_max;
 	min = bus->slot_min;
 	if ((slotno > max) || (slotno < min)) {
-		err ("got the wrong range\n");
+		err("got the wrong range\n");
 		return 0xff;
 	}
 	busno = (u8) (slotno - (u8) min);
 	busno += primary_busno + 0x01;
-	bus_cur = ibmphp_find_res_bus (busno);
+	bus_cur = ibmphp_find_res_bus(busno);
 	/* either there is no such bus number, or there are no ranges, which
 	 * can only happen if we removed the bridged device in previous load
 	 * of the driver, and now only have the skeleton bus struct
diff --git a/drivers/pci/hotplug/ibmphp_res.c b/drivers/pci/hotplug/ibmphp_res.c
index f279060cf6e2..aee6e41001e1 100644
--- a/drivers/pci/hotplug/ibmphp_res.c
+++ b/drivers/pci/hotplug/ibmphp_res.c
@@ -36,28 +36,28 @@
 
 static int flags = 0;		/* for testing */
 
-static void update_resources (struct bus_node *bus_cur, int type, int rangeno);
-static int once_over (void);
-static int remove_ranges (struct bus_node *, struct bus_node *);
-static int update_bridge_ranges (struct bus_node **);
-static int add_bus_range (int type, struct range_node *, struct bus_node *);
-static void fix_resources (struct bus_node *);
-static struct bus_node *find_bus_wprev (u8, struct bus_node **, u8);
+static void update_resources(struct bus_node *bus_cur, int type, int rangeno);
+static int once_over(void);
+static int remove_ranges(struct bus_node *, struct bus_node *);
+static int update_bridge_ranges(struct bus_node **);
+static int add_bus_range(int type, struct range_node *, struct bus_node *);
+static void fix_resources(struct bus_node *);
+static struct bus_node *find_bus_wprev(u8, struct bus_node **, u8);
 
 static LIST_HEAD(gbuses);
 
-static struct bus_node * __init alloc_error_bus (struct ebda_pci_rsrc *curr, u8 busno, int flag)
+static struct bus_node * __init alloc_error_bus(struct ebda_pci_rsrc *curr, u8 busno, int flag)
 {
 	struct bus_node *newbus;
 
 	if (!(curr) && !(flag)) {
-		err ("NULL pointer passed\n");
+		err("NULL pointer passed\n");
 		return NULL;
 	}
 
 	newbus = kzalloc(sizeof(struct bus_node), GFP_KERNEL);
 	if (!newbus) {
-		err ("out of system memory\n");
+		err("out of system memory\n");
 		return NULL;
 	}
 
@@ -65,22 +65,22 @@ static struct bus_node * __init alloc_error_bus (struct ebda_pci_rsrc *curr, u8
 		newbus->busno = busno;
 	else
 		newbus->busno = curr->bus_num;
-	list_add_tail (&newbus->bus_list, &gbuses);
+	list_add_tail(&newbus->bus_list, &gbuses);
 	return newbus;
 }
 
-static struct resource_node * __init alloc_resources (struct ebda_pci_rsrc *curr)
+static struct resource_node * __init alloc_resources(struct ebda_pci_rsrc *curr)
 {
 	struct resource_node *rs;
 
 	if (!curr) {
-		err ("NULL passed to allocate\n");
+		err("NULL passed to allocate\n");
 		return NULL;
 	}
 
 	rs = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 	if (!rs) {
-		err ("out of system memory\n");
+		err("out of system memory\n");
 		return NULL;
 	}
 	rs->busno = curr->bus_num;
@@ -91,7 +91,7 @@ static struct resource_node * __init alloc_resources (struct ebda_pci_rsrc *curr
 	return rs;
 }
 
-static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node **new_range, struct ebda_pci_rsrc *curr, int flag, u8 first_bus)
+static int __init alloc_bus_range(struct bus_node **new_bus, struct range_node **new_range, struct ebda_pci_rsrc *curr, int flag, u8 first_bus)
 {
 	struct bus_node *newbus;
 	struct range_node *newrange;
@@ -100,7 +100,7 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 	if (first_bus) {
 		newbus = kzalloc(sizeof(struct bus_node), GFP_KERNEL);
 		if (!newbus) {
-			err ("out of system memory.\n");
+			err("out of system memory.\n");
 			return -ENOMEM;
 		}
 		newbus->busno = curr->bus_num;
@@ -122,8 +122,8 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 	newrange = kzalloc(sizeof(struct range_node), GFP_KERNEL);
 	if (!newrange) {
 		if (first_bus)
-			kfree (newbus);
-		err ("out of system memory\n");
+			kfree(newbus);
+		err("out of system memory\n");
 		return -ENOMEM;
 	}
 	newrange->start = curr->start_addr;
@@ -133,8 +133,8 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 		newrange->rangeno = 1;
 	else {
 		/* need to insert our range */
-		add_bus_range (flag, newrange, newbus);
-		debug ("%d resource Primary Bus inserted on bus %x [%x - %x]\n", flag, newbus->busno, newrange->start, newrange->end);
+		add_bus_range(flag, newrange, newbus);
+		debug("%d resource Primary Bus inserted on bus %x [%x - %x]\n", flag, newbus->busno, newrange->start, newrange->end);
 	}
 
 	switch (flag) {
@@ -143,9 +143,9 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 			if (first_bus)
 				newbus->noMemRanges = 1;
 			else {
-				debug ("First Memory Primary on bus %x, [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+				debug("First Memory Primary on bus %x, [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				++newbus->noMemRanges;
-				fix_resources (newbus);
+				fix_resources(newbus);
 			}
 			break;
 		case IO:
@@ -153,9 +153,9 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 			if (first_bus)
 				newbus->noIORanges = 1;
 			else {
-				debug ("First IO Primary on bus %x, [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+				debug("First IO Primary on bus %x, [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				++newbus->noIORanges;
-				fix_resources (newbus);
+				fix_resources(newbus);
 			}
 			break;
 		case PFMEM:
@@ -163,9 +163,9 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
 			if (first_bus)
 				newbus->noPFMemRanges = 1;
 			else {
-				debug ("1st PFMemory Primary on Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+				debug("1st PFMemory Primary on Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				++newbus->noPFMemRanges;
-				fix_resources (newbus);
+				fix_resources(newbus);
 			}
 
 			break;
@@ -183,7 +183,7 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
  * 2. If cannot allocate out of PFMem range, allocate from Mem ranges.  PFmemFromMem
  * are not sorted. (no need since use mem node). To not change the entire code, we
  * also add mem node whenever this case happens so as not to change
- * ibmphp_check_mem_resource etc (and since it really is taking Mem resource)
+ * ibmphp_check_mem_resource etc(and since it really is taking Mem resource)
  */
 
 /*****************************************************************************
@@ -196,25 +196,23 @@ static int __init alloc_bus_range (struct bus_node **new_bus, struct range_node
  * Input: ptr to the head of the resource list from EBDA
  * Output: 0, -1 or error codes
  ***************************************************************************/
-int __init ibmphp_rsrc_init (void)
+int __init ibmphp_rsrc_init(void)
 {
 	struct ebda_pci_rsrc *curr;
 	struct range_node *newrange = NULL;
 	struct bus_node *newbus = NULL;
 	struct bus_node *bus_cur;
 	struct bus_node *bus_prev;
-	struct list_head *tmp;
 	struct resource_node *new_io = NULL;
 	struct resource_node *new_mem = NULL;
 	struct resource_node *new_pfmem = NULL;
 	int rc;
-	struct list_head *tmp_ebda;
 
-	list_for_each (tmp_ebda, &ibmphp_ebda_pci_rsrc_head) {
-		curr = list_entry (tmp_ebda, struct ebda_pci_rsrc, ebda_pci_rsrc_list);
+	list_for_each_entry(curr, &ibmphp_ebda_pci_rsrc_head,
+			    ebda_pci_rsrc_list) {
 		if (!(curr->rsrc_type & PCIDEVMASK)) {
 			/* EBDA still lists non PCI devices, so ignore... */
-			debug ("this is not a PCI DEVICE in rsrc_init, please take care\n");
+			debug("this is not a PCI DEVICE in rsrc_init, please take care\n");
 			// continue;
 		}
 
@@ -223,17 +221,17 @@ int __init ibmphp_rsrc_init (void)
 			/* memory */
 			if ((curr->rsrc_type & RESTYPE) == MMASK) {
 				/* no bus structure exists in place yet */
-				if (list_empty (&gbuses)) {
+				if (list_empty(&gbuses)) {
 					rc = alloc_bus_range(&newbus, &newrange, curr, MEM, 1);
 					if (rc)
 						return rc;
-					list_add_tail (&newbus->bus_list, &gbuses);
-					debug ("gbuses = NULL, Memory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+					list_add_tail(&newbus->bus_list, &gbuses);
+					debug("gbuses = NULL, Memory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				} else {
-					bus_cur = find_bus_wprev (curr->bus_num, &bus_prev, 1);
+					bus_cur = find_bus_wprev(curr->bus_num, &bus_prev, 1);
 					/* found our bus */
 					if (bus_cur) {
-						rc = alloc_bus_range (&bus_cur, &newrange, curr, MEM, 0);
+						rc = alloc_bus_range(&bus_cur, &newrange, curr, MEM, 0);
 						if (rc)
 							return rc;
 					} else {
@@ -242,24 +240,24 @@ int __init ibmphp_rsrc_init (void)
 						if (rc)
 							return rc;
 
-						list_add_tail (&newbus->bus_list, &gbuses);
-						debug ("New Bus, Memory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+						list_add_tail(&newbus->bus_list, &gbuses);
+						debug("New Bus, Memory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 					}
 				}
 			} else if ((curr->rsrc_type & RESTYPE) == PFMASK) {
 				/* prefetchable memory */
-				if (list_empty (&gbuses)) {
+				if (list_empty(&gbuses)) {
 					/* no bus structure exists in place yet */
 					rc = alloc_bus_range(&newbus, &newrange, curr, PFMEM, 1);
 					if (rc)
 						return rc;
-					list_add_tail (&newbus->bus_list, &gbuses);
-					debug ("gbuses = NULL, PFMemory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+					list_add_tail(&newbus->bus_list, &gbuses);
+					debug("gbuses = NULL, PFMemory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				} else {
-					bus_cur = find_bus_wprev (curr->bus_num, &bus_prev, 1);
+					bus_cur = find_bus_wprev(curr->bus_num, &bus_prev, 1);
 					if (bus_cur) {
 						/* found our bus */
-						rc = alloc_bus_range (&bus_cur, &newrange, curr, PFMEM, 0);
+						rc = alloc_bus_range(&bus_cur, &newrange, curr, PFMEM, 0);
 						if (rc)
 							return rc;
 					} else {
@@ -267,23 +265,23 @@ int __init ibmphp_rsrc_init (void)
 						rc = alloc_bus_range(&newbus, &newrange, curr, PFMEM, 1);
 						if (rc)
 							return rc;
-						list_add_tail (&newbus->bus_list, &gbuses);
-						debug ("1st Bus, PFMemory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+						list_add_tail(&newbus->bus_list, &gbuses);
+						debug("1st Bus, PFMemory Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 					}
 				}
 			} else if ((curr->rsrc_type & RESTYPE) == IOMASK) {
 				/* IO */
-				if (list_empty (&gbuses)) {
+				if (list_empty(&gbuses)) {
 					/* no bus structure exists in place yet */
 					rc = alloc_bus_range(&newbus, &newrange, curr, IO, 1);
 					if (rc)
 						return rc;
-					list_add_tail (&newbus->bus_list, &gbuses);
-					debug ("gbuses = NULL, IO Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+					list_add_tail(&newbus->bus_list, &gbuses);
+					debug("gbuses = NULL, IO Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 				} else {
-					bus_cur = find_bus_wprev (curr->bus_num, &bus_prev, 1);
+					bus_cur = find_bus_wprev(curr->bus_num, &bus_prev, 1);
 					if (bus_cur) {
-						rc = alloc_bus_range (&bus_cur, &newrange, curr, IO, 0);
+						rc = alloc_bus_range(&bus_cur, &newrange, curr, IO, 0);
 						if (rc)
 							return rc;
 					} else {
@@ -291,8 +289,8 @@ int __init ibmphp_rsrc_init (void)
 						rc = alloc_bus_range(&newbus, &newrange, curr, IO, 1);
 						if (rc)
 							return rc;
-						list_add_tail (&newbus->bus_list, &gbuses);
-						debug ("1st Bus, IO Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
+						list_add_tail(&newbus->bus_list, &gbuses);
+						debug("1st Bus, IO Primary Bus %x [%x - %x]\n", newbus->busno, newrange->start, newrange->end);
 					}
 				}
 
@@ -304,7 +302,7 @@ int __init ibmphp_rsrc_init (void)
 			/* regular pci device resource */
 			if ((curr->rsrc_type & RESTYPE) == MMASK) {
 				/* Memory resource */
-				new_mem = alloc_resources (curr);
+				new_mem = alloc_resources(curr);
 				if (!new_mem)
 					return -ENOMEM;
 				new_mem->type = MEM;
@@ -315,25 +313,25 @@ int __init ibmphp_rsrc_init (void)
 				 * assign a -1 and then update once the range
 				 * actually appears...
 				 */
-				if (ibmphp_add_resource (new_mem) < 0) {
-					newbus = alloc_error_bus (curr, 0, 0);
+				if (ibmphp_add_resource(new_mem) < 0) {
+					newbus = alloc_error_bus(curr, 0, 0);
 					if (!newbus)
 						return -ENOMEM;
 					newbus->firstMem = new_mem;
 					++newbus->needMemUpdate;
 					new_mem->rangeno = -1;
 				}
-				debug ("Memory resource for device %x, bus %x, [%x - %x]\n", new_mem->devfunc, new_mem->busno, new_mem->start, new_mem->end);
+				debug("Memory resource for device %x, bus %x, [%x - %x]\n", new_mem->devfunc, new_mem->busno, new_mem->start, new_mem->end);
 
 			} else if ((curr->rsrc_type & RESTYPE) == PFMASK) {
 				/* PFMemory resource */
-				new_pfmem = alloc_resources (curr);
+				new_pfmem = alloc_resources(curr);
 				if (!new_pfmem)
 					return -ENOMEM;
 				new_pfmem->type = PFMEM;
 				new_pfmem->fromMem = 0;
-				if (ibmphp_add_resource (new_pfmem) < 0) {
-					newbus = alloc_error_bus (curr, 0, 0);
+				if (ibmphp_add_resource(new_pfmem) < 0) {
+					newbus = alloc_error_bus(curr, 0, 0);
 					if (!newbus)
 						return -ENOMEM;
 					newbus->firstPFMem = new_pfmem;
@@ -341,10 +339,10 @@ int __init ibmphp_rsrc_init (void)
 					new_pfmem->rangeno = -1;
 				}
 
-				debug ("PFMemory resource for device %x, bus %x, [%x - %x]\n", new_pfmem->devfunc, new_pfmem->busno, new_pfmem->start, new_pfmem->end);
+				debug("PFMemory resource for device %x, bus %x, [%x - %x]\n", new_pfmem->devfunc, new_pfmem->busno, new_pfmem->start, new_pfmem->end);
 			} else if ((curr->rsrc_type & RESTYPE) == IOMASK) {
 				/* IO resource */
-				new_io = alloc_resources (curr);
+				new_io = alloc_resources(curr);
 				if (!new_io)
 					return -ENOMEM;
 				new_io->type = IO;
@@ -356,27 +354,26 @@ int __init ibmphp_rsrc_init (void)
 				 * Can assign a -1 and then update once the
 				 * range actually appears...
 				 */
-				if (ibmphp_add_resource (new_io) < 0) {
-					newbus = alloc_error_bus (curr, 0, 0);
+				if (ibmphp_add_resource(new_io) < 0) {
+					newbus = alloc_error_bus(curr, 0, 0);
 					if (!newbus)
 						return -ENOMEM;
 					newbus->firstIO = new_io;
 					++newbus->needIOUpdate;
 					new_io->rangeno = -1;
 				}
-				debug ("IO resource for device %x, bus %x, [%x - %x]\n", new_io->devfunc, new_io->busno, new_io->start, new_io->end);
+				debug("IO resource for device %x, bus %x, [%x - %x]\n", new_io->devfunc, new_io->busno, new_io->start, new_io->end);
 			}
 		}
 	}
 
-	list_for_each (tmp, &gbuses) {
-		bus_cur = list_entry (tmp, struct bus_node, bus_list);
+	list_for_each_entry(bus_cur, &gbuses, bus_list) {
 		/* This is to get info about PPB resources, since EBDA doesn't put this info into the primary bus info */
-		rc = update_bridge_ranges (&bus_cur);
+		rc = update_bridge_ranges(&bus_cur);
 		if (rc)
 			return rc;
 	}
-	return once_over ();	/* This is to align ranges (so no -1) */
+	return once_over();	/* This is to align ranges (so no -1) */
 }
 
 /********************************************************************************
@@ -387,7 +384,7 @@ int __init ibmphp_rsrc_init (void)
  * Input: type of the resource, range to add, current bus
  * Output: 0 or -1, bus and range ptrs
  ********************************************************************************/
-static int add_bus_range (int type, struct range_node *range, struct bus_node *bus_cur)
+static int add_bus_range(int type, struct range_node *range, struct bus_node *bus_cur)
 {
 	struct range_node *range_cur = NULL;
 	struct range_node *range_prev;
@@ -452,7 +449,7 @@ static int add_bus_range (int type, struct range_node *range, struct bus_node *b
 		range_cur = range_cur->next;
 	}
 
-	update_resources (bus_cur, type, i_init + 1);
+	update_resources(bus_cur, type, i_init + 1);
 	return 0;
 }
 
@@ -462,7 +459,7 @@ static int add_bus_range (int type, struct range_node *range, struct bus_node *b
  *
  * Input: bus, type of the resource, the rangeno starting from which to update
  ******************************************************************************/
-static void update_resources (struct bus_node *bus_cur, int type, int rangeno)
+static void update_resources(struct bus_node *bus_cur, int type, int rangeno)
 {
 	struct resource_node *res = NULL;
 	u8 eol = 0;	/* end of list indicator */
@@ -506,9 +503,9 @@ static void update_resources (struct bus_node *bus_cur, int type, int rangeno)
 	}
 }
 
-static void fix_me (struct resource_node *res, struct bus_node *bus_cur, struct range_node *range)
+static void fix_me(struct resource_node *res, struct bus_node *bus_cur, struct range_node *range)
 {
-	char * str = "";
+	char *str = "";
 	switch (res->type) {
 		case IO:
 			str = "io";
@@ -526,7 +523,7 @@ static void fix_me (struct resource_node *res, struct bus_node *bus_cur, struct
 			while (range) {
 				if ((res->start >= range->start) && (res->end <= range->end)) {
 					res->rangeno = range->rangeno;
-					debug ("%s->rangeno in fix_resources is %d\n", str, res->rangeno);
+					debug("%s->rangeno in fix_resources is %d\n", str, res->rangeno);
 					switch (res->type) {
 						case IO:
 							--bus_cur->needIOUpdate;
@@ -561,27 +558,27 @@ static void fix_me (struct resource_node *res, struct bus_node *bus_cur, struct
  * Input: current bus
  * Output: none, list of resources for that bus are fixed if can be
  *******************************************************************************/
-static void fix_resources (struct bus_node *bus_cur)
+static void fix_resources(struct bus_node *bus_cur)
 {
 	struct range_node *range;
 	struct resource_node *res;
 
-	debug ("%s - bus_cur->busno = %d\n", __func__, bus_cur->busno);
+	debug("%s - bus_cur->busno = %d\n", __func__, bus_cur->busno);
 
 	if (bus_cur->needIOUpdate) {
 		res = bus_cur->firstIO;
 		range = bus_cur->rangeIO;
-		fix_me (res, bus_cur, range);
+		fix_me(res, bus_cur, range);
 	}
 	if (bus_cur->needMemUpdate) {
 		res = bus_cur->firstMem;
 		range = bus_cur->rangeMem;
-		fix_me (res, bus_cur, range);
+		fix_me(res, bus_cur, range);
 	}
 	if (bus_cur->needPFMemUpdate) {
 		res = bus_cur->firstPFMem;
 		range = bus_cur->rangePFMem;
-		fix_me (res, bus_cur, range);
+		fix_me(res, bus_cur, range);
 	}
 }
 
@@ -594,7 +591,7 @@ static void fix_resources (struct bus_node *bus_cur)
  * Output: ptrs assigned (to the node)
  * 0 or -1
  *******************************************************************************/
-int ibmphp_add_resource (struct resource_node *res)
+int ibmphp_add_resource(struct resource_node *res)
 {
 	struct resource_node *res_cur;
 	struct resource_node *res_prev;
@@ -602,18 +599,18 @@ int ibmphp_add_resource (struct resource_node *res)
 	struct range_node *range_cur = NULL;
 	struct resource_node *res_start = NULL;
 
-	debug ("%s - enter\n", __func__);
+	debug("%s - enter\n", __func__);
 
 	if (!res) {
-		err ("NULL passed to add\n");
+		err("NULL passed to add\n");
 		return -ENODEV;
 	}
 
-	bus_cur = find_bus_wprev (res->busno, NULL, 0);
+	bus_cur = find_bus_wprev(res->busno, NULL, 0);
 
 	if (!bus_cur) {
 		/* didn't find a bus, something's wrong!!! */
-		debug ("no bus in the system, either pci_dev's wrong or allocation failed\n");
+		debug("no bus in the system, either pci_dev's wrong or allocation failed\n");
 		return -ENODEV;
 	}
 
@@ -632,7 +629,7 @@ int ibmphp_add_resource (struct resource_node *res)
 			res_start = bus_cur->firstPFMem;
 			break;
 		default:
-			err ("cannot read the type of the resource to add... problem\n");
+			err("cannot read the type of the resource to add... problem\n");
 			return -EINVAL;
 	}
 	while (range_cur) {
@@ -663,7 +660,7 @@ int ibmphp_add_resource (struct resource_node *res)
 		res->rangeno = -1;
 	}
 
-	debug ("The range is %d\n", res->rangeno);
+	debug("The range is %d\n", res->rangeno);
 	if (!res_start) {
 		/* no first{IO,Mem,Pfmem} on the bus, 1st IO/Mem/Pfmem resource ever */
 		switch (res->type) {
@@ -683,7 +680,7 @@ int ibmphp_add_resource (struct resource_node *res)
 		res_cur = res_start;
 		res_prev = NULL;
 
-		debug ("res_cur->rangeno is %d\n", res_cur->rangeno);
+		debug("res_cur->rangeno is %d\n", res_cur->rangeno);
 
 		while (res_cur) {
 			if (res_cur->rangeno >= res->rangeno)
@@ -697,7 +694,7 @@ int ibmphp_add_resource (struct resource_node *res)
 
 		if (!res_cur) {
 			/* at the end of the resource list */
-			debug ("i should be here, [%x - %x]\n", res->start, res->end);
+			debug("i should be here, [%x - %x]\n", res->start, res->end);
 			res_prev->nextRange = res;
 			res->next = NULL;
 			res->nextRange = NULL;
@@ -765,7 +762,7 @@ int ibmphp_add_resource (struct resource_node *res)
 		}
 	}
 
-	debug ("%s - exit\n", __func__);
+	debug("%s - exit\n", __func__);
 	return 0;
 }
 
@@ -776,23 +773,23 @@ int ibmphp_add_resource (struct resource_node *res)
  * Output: modified resource list
  *        0 or error code
  ****************************************************************************/
-int ibmphp_remove_resource (struct resource_node *res)
+int ibmphp_remove_resource(struct resource_node *res)
 {
 	struct bus_node *bus_cur;
 	struct resource_node *res_cur = NULL;
 	struct resource_node *res_prev;
 	struct resource_node *mem_cur;
-	char * type = "";
+	char *type = "";
 
 	if (!res)  {
-		err ("resource to remove is NULL\n");
+		err("resource to remove is NULL\n");
 		return -ENODEV;
 	}
 
-	bus_cur = find_bus_wprev (res->busno, NULL, 0);
+	bus_cur = find_bus_wprev(res->busno, NULL, 0);
 
 	if (!bus_cur) {
-		err ("cannot find corresponding bus of the io resource to remove  bailing out...\n");
+		err("cannot find corresponding bus of the io resource to remove  bailing out...\n");
 		return -ENODEV;
 	}
 
@@ -810,7 +807,7 @@ int ibmphp_remove_resource (struct resource_node *res)
 			type = "pfmem";
 			break;
 		default:
-			err ("unknown type for resource to remove\n");
+			err("unknown type for resource to remove\n");
 			return -EINVAL;
 	}
 	res_prev = NULL;
@@ -848,16 +845,16 @@ int ibmphp_remove_resource (struct resource_node *res)
 							mem_cur = mem_cur->nextRange;
 					}
 					if (!mem_cur) {
-						err ("cannot find corresponding mem node for pfmem...\n");
+						err("cannot find corresponding mem node for pfmem...\n");
 						return -EINVAL;
 					}
 
-					ibmphp_remove_resource (mem_cur);
+					ibmphp_remove_resource(mem_cur);
 					if (!res_prev)
 						bus_cur->firstPFMemFromMem = res_cur->next;
 					else
 						res_prev->next = res_cur->next;
-					kfree (res_cur);
+					kfree(res_cur);
 					return 0;
 				}
 				res_prev = res_cur;
@@ -867,11 +864,11 @@ int ibmphp_remove_resource (struct resource_node *res)
 					res_cur = res_cur->nextRange;
 			}
 			if (!res_cur) {
-				err ("cannot find pfmem to delete...\n");
+				err("cannot find pfmem to delete...\n");
 				return -EINVAL;
 			}
 		} else {
-			err ("the %s resource is not in the list to be deleted...\n", type);
+			err("the %s resource is not in the list to be deleted...\n", type);
 			return -EINVAL;
 		}
 	}
@@ -914,7 +911,7 @@ int ibmphp_remove_resource (struct resource_node *res)
 					break;
 			}
 		}
-		kfree (res_cur);
+		kfree(res_cur);
 		return 0;
 	} else {
 		if (res_cur->next) {
@@ -929,14 +926,14 @@ int ibmphp_remove_resource (struct resource_node *res)
 			res_prev->next = NULL;
 			res_prev->nextRange = NULL;
 		}
-		kfree (res_cur);
+		kfree(res_cur);
 		return 0;
 	}
 
 	return 0;
 }
 
-static struct range_node *find_range (struct bus_node *bus_cur, struct resource_node *res)
+static struct range_node *find_range(struct bus_node *bus_cur, struct resource_node *res)
 {
 	struct range_node *range = NULL;
 
@@ -951,7 +948,7 @@ static struct range_node *find_range (struct bus_node *bus_cur, struct resource_
 			range = bus_cur->rangePFMem;
 			break;
 		default:
-			err ("cannot read resource type in find_range\n");
+			err("cannot read resource type in find_range\n");
 	}
 
 	while (range) {
@@ -971,7 +968,7 @@ static struct range_node *find_range (struct bus_node *bus_cur, struct resource_
  * Output: the correct start and end address are inputted into the resource node,
  *        0 or -EINVAL
  *****************************************************************************/
-int ibmphp_check_resource (struct resource_node *res, u8 bridge)
+int ibmphp_check_resource(struct resource_node *res, u8 bridge)
 {
 	struct bus_node *bus_cur;
 	struct range_node *range = NULL;
@@ -995,16 +992,16 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 	} else
 		tmp_divide = res->len;
 
-	bus_cur = find_bus_wprev (res->busno, NULL, 0);
+	bus_cur = find_bus_wprev(res->busno, NULL, 0);
 
 	if (!bus_cur) {
 		/* didn't find a bus, something's wrong!!! */
-		debug ("no bus in the system, either pci_dev's wrong or allocation failed\n");
+		debug("no bus in the system, either pci_dev's wrong or allocation failed\n");
 		return -EINVAL;
 	}
 
-	debug ("%s - enter\n", __func__);
-	debug ("bus_cur->busno is %d\n", bus_cur->busno);
+	debug("%s - enter\n", __func__);
+	debug("bus_cur->busno is %d\n", bus_cur->busno);
 
 	/* This is a quick fix to not mess up with the code very much.  i.e.,
 	 * 2000-2fff, len = 1000, but when we compare, we need it to be fff */
@@ -1024,17 +1021,17 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 			noranges = bus_cur->noPFMemRanges;
 			break;
 		default:
-			err ("wrong type of resource to check\n");
+			err("wrong type of resource to check\n");
 			return -EINVAL;
 	}
 	res_prev = NULL;
 
 	while (res_cur) {
-		range = find_range (bus_cur, res_cur);
-		debug ("%s - rangeno = %d\n", __func__, res_cur->rangeno);
+		range = find_range(bus_cur, res_cur);
+		debug("%s - rangeno = %d\n", __func__, res_cur->rangeno);
 
 		if (!range) {
-			err ("no range for the device exists... bailing out...\n");
+			err("no range for the device exists... bailing out...\n");
 			return -EINVAL;
 		}
 
@@ -1044,7 +1041,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 			len_tmp = res_cur->start - 1 - range->start;
 
 			if ((res_cur->start != range->start) && (len_tmp >= res->len)) {
-				debug ("len_tmp = %x\n", len_tmp);
+				debug("len_tmp = %x\n", len_tmp);
 
 				if ((len_tmp < len_cur) || (len_cur == 0)) {
 
@@ -1072,7 +1069,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 					}
 
 					if (flag && len_cur == res->len) {
-						debug ("but we are not here, right?\n");
+						debug("but we are not here, right?\n");
 						res->start = start_cur;
 						res->len += 1; /* To restore the balance */
 						res->end = res->start + res->len - 1;
@@ -1086,7 +1083,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 			len_tmp = range->end - (res_cur->end + 1);
 
 			if ((range->end != res_cur->end) && (len_tmp >= res->len)) {
-				debug ("len_tmp = %x\n", len_tmp);
+				debug("len_tmp = %x\n", len_tmp);
 				if ((len_tmp < len_cur) || (len_cur == 0)) {
 
 					if (((res_cur->end + 1) % tmp_divide) == 0) {
@@ -1262,7 +1259,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 
 		if ((!range) && (len_cur == 0)) {
 			/* have gone through the list of devices and ranges and haven't found n.e.thing */
-			err ("no appropriate range.. bailing out...\n");
+			err("no appropriate range.. bailing out...\n");
 			return -EINVAL;
 		} else if (len_cur) {
 			res->start = start_cur;
@@ -1273,7 +1270,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 	}
 
 	if (!res_cur) {
-		debug ("prev->rangeno = %d, noranges = %d\n", res_prev->rangeno, noranges);
+		debug("prev->rangeno = %d, noranges = %d\n", res_prev->rangeno, noranges);
 		if (res_prev->rangeno < noranges) {
 			/* if there're more ranges out there to check */
 			switch (res->type) {
@@ -1328,7 +1325,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 
 			if ((!range) && (len_cur == 0)) {
 				/* have gone through the list of devices and ranges and haven't found n.e.thing */
-				err ("no appropriate range.. bailing out...\n");
+				err("no appropriate range.. bailing out...\n");
 				return -EINVAL;
 			} else if (len_cur) {
 				res->start = start_cur;
@@ -1345,7 +1342,7 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
 				return 0;
 			} else {
 				/* have gone through the list of devices and haven't found n.e.thing */
-				err ("no appropriate range.. bailing out...\n");
+				err("no appropriate range.. bailing out...\n");
 				return -EINVAL;
 			}
 		}
@@ -1359,23 +1356,23 @@ int ibmphp_check_resource (struct resource_node *res, u8 bridge)
  * Input: Bus
  * Output: 0, -ENODEV
  ********************************************************************************/
-int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
+int ibmphp_remove_bus(struct bus_node *bus, u8 parent_busno)
 {
 	struct resource_node *res_cur;
 	struct resource_node *res_tmp;
 	struct bus_node *prev_bus;
 	int rc;
 
-	prev_bus = find_bus_wprev (parent_busno, NULL, 0);
+	prev_bus = find_bus_wprev(parent_busno, NULL, 0);
 
 	if (!prev_bus) {
-		debug ("something terribly wrong. Cannot find parent bus to the one to remove\n");
+		debug("something terribly wrong. Cannot find parent bus to the one to remove\n");
 		return -ENODEV;
 	}
 
-	debug ("In ibmphp_remove_bus... prev_bus->busno is %x\n", prev_bus->busno);
+	debug("In ibmphp_remove_bus... prev_bus->busno is %x\n", prev_bus->busno);
 
-	rc = remove_ranges (bus, prev_bus);
+	rc = remove_ranges(bus, prev_bus);
 	if (rc)
 		return rc;
 
@@ -1387,7 +1384,7 @@ int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
 				res_cur = res_cur->next;
 			else
 				res_cur = res_cur->nextRange;
-			kfree (res_tmp);
+			kfree(res_tmp);
 			res_tmp = NULL;
 		}
 		bus->firstIO = NULL;
@@ -1400,7 +1397,7 @@ int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
 				res_cur = res_cur->next;
 			else
 				res_cur = res_cur->nextRange;
-			kfree (res_tmp);
+			kfree(res_tmp);
 			res_tmp = NULL;
 		}
 		bus->firstMem = NULL;
@@ -1413,7 +1410,7 @@ int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
 				res_cur = res_cur->next;
 			else
 				res_cur = res_cur->nextRange;
-			kfree (res_tmp);
+			kfree(res_tmp);
 			res_tmp = NULL;
 		}
 		bus->firstPFMem = NULL;
@@ -1425,14 +1422,14 @@ int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
 			res_tmp = res_cur;
 			res_cur = res_cur->next;
 
-			kfree (res_tmp);
+			kfree(res_tmp);
 			res_tmp = NULL;
 		}
 		bus->firstPFMemFromMem = NULL;
 	}
 
-	list_del (&bus->bus_list);
-	kfree (bus);
+	list_del(&bus->bus_list);
+	kfree(bus);
 	return 0;
 }
 
@@ -1442,7 +1439,7 @@ int ibmphp_remove_bus (struct bus_node *bus, u8 parent_busno)
  * Input: current bus, previous bus
  * Output: 0, -EINVAL
  ******************************************************************************/
-static int remove_ranges (struct bus_node *bus_cur, struct bus_node *bus_prev)
+static int remove_ranges(struct bus_node *bus_cur, struct bus_node *bus_prev)
 {
 	struct range_node *range_cur;
 	struct range_node *range_tmp;
@@ -1452,13 +1449,13 @@ static int remove_ranges (struct bus_node *bus_cur, struct bus_node *bus_prev)
 	if (bus_cur->noIORanges) {
 		range_cur = bus_cur->rangeIO;
 		for (i = 0; i < bus_cur->noIORanges; i++) {
-			if (ibmphp_find_resource (bus_prev, range_cur->start, &res, IO) < 0)
+			if (ibmphp_find_resource(bus_prev, range_cur->start, &res, IO) < 0)
 				return -EINVAL;
-			ibmphp_remove_resource (res);
+			ibmphp_remove_resource(res);
 
 			range_tmp = range_cur;
 			range_cur = range_cur->next;
-			kfree (range_tmp);
+			kfree(range_tmp);
 			range_tmp = NULL;
 		}
 		bus_cur->rangeIO = NULL;
@@ -1466,13 +1463,13 @@ static int remove_ranges (struct bus_node *bus_cur, struct bus_node *bus_prev)
 	if (bus_cur->noMemRanges) {
 		range_cur = bus_cur->rangeMem;
 		for (i = 0; i < bus_cur->noMemRanges; i++) {
-			if (ibmphp_find_resource (bus_prev, range_cur->start, &res, MEM) < 0)
+			if (ibmphp_find_resource(bus_prev, range_cur->start, &res, MEM) < 0)
 				return -EINVAL;
 
-			ibmphp_remove_resource (res);
+			ibmphp_remove_resource(res);
 			range_tmp = range_cur;
 			range_cur = range_cur->next;
-			kfree (range_tmp);
+			kfree(range_tmp);
 			range_tmp = NULL;
 		}
 		bus_cur->rangeMem = NULL;
@@ -1480,13 +1477,13 @@ static int remove_ranges (struct bus_node *bus_cur, struct bus_node *bus_prev)
 	if (bus_cur->noPFMemRanges) {
 		range_cur = bus_cur->rangePFMem;
 		for (i = 0; i < bus_cur->noPFMemRanges; i++) {
-			if (ibmphp_find_resource (bus_prev, range_cur->start, &res, PFMEM) < 0)
+			if (ibmphp_find_resource(bus_prev, range_cur->start, &res, PFMEM) < 0)
 				return -EINVAL;
 
-			ibmphp_remove_resource (res);
+			ibmphp_remove_resource(res);
 			range_tmp = range_cur;
 			range_cur = range_cur->next;
-			kfree (range_tmp);
+			kfree(range_tmp);
 			range_tmp = NULL;
 		}
 		bus_cur->rangePFMem = NULL;
@@ -1498,13 +1495,13 @@ static int remove_ranges (struct bus_node *bus_cur, struct bus_node *bus_prev)
  * find the resource node in the bus
  * Input: Resource needed, start address of the resource, type of resource
  */
-int ibmphp_find_resource (struct bus_node *bus, u32 start_address, struct resource_node **res, int flag)
+int ibmphp_find_resource(struct bus_node *bus, u32 start_address, struct resource_node **res, int flag)
 {
 	struct resource_node *res_cur = NULL;
-	char * type = "";
+	char *type = "";
 
 	if (!bus) {
-		err ("The bus passed in NULL to find resource\n");
+		err("The bus passed in NULL to find resource\n");
 		return -ENODEV;
 	}
 
@@ -1522,7 +1519,7 @@ int ibmphp_find_resource (struct bus_node *bus, u32 start_address, struct resour
 			type = "pfmem";
 			break;
 		default:
-			err ("wrong type of flag\n");
+			err("wrong type of flag\n");
 			return -EINVAL;
 	}
 
@@ -1548,17 +1545,17 @@ int ibmphp_find_resource (struct bus_node *bus, u32 start_address, struct resour
 				res_cur = res_cur->next;
 			}
 			if (!res_cur) {
-				debug ("SOS...cannot find %s resource in the bus.\n", type);
+				debug("SOS...cannot find %s resource in the bus.\n", type);
 				return -EINVAL;
 			}
 		} else {
-			debug ("SOS... cannot find %s resource in the bus.\n", type);
+			debug("SOS... cannot find %s resource in the bus.\n", type);
 			return -EINVAL;
 		}
 	}
 
 	if (*res)
-		debug ("*res->start = %x\n", (*res)->start);
+		debug("*res->start = %x\n", (*res)->start);
 
 	return 0;
 }
@@ -1569,21 +1566,18 @@ int ibmphp_find_resource (struct bus_node *bus, u32 start_address, struct resour
  * Parameters: none
  * Returns: none
  ***********************************************************************/
-void ibmphp_free_resources (void)
+void ibmphp_free_resources(void)
 {
-	struct bus_node *bus_cur = NULL;
+	struct bus_node *bus_cur = NULL, *next;
 	struct bus_node *bus_tmp;
 	struct range_node *range_cur;
 	struct range_node *range_tmp;
 	struct resource_node *res_cur;
 	struct resource_node *res_tmp;
-	struct list_head *tmp;
-	struct list_head *next;
 	int i = 0;
 	flags = 1;
 
-	list_for_each_safe (tmp, next, &gbuses) {
-		bus_cur = list_entry (tmp, struct bus_node, bus_list);
+	list_for_each_entry_safe(bus_cur, next, &gbuses, bus_list) {
 		if (bus_cur->noIORanges) {
 			range_cur = bus_cur->rangeIO;
 			for (i = 0; i < bus_cur->noIORanges; i++) {
@@ -1591,7 +1585,7 @@ void ibmphp_free_resources (void)
 					break;
 				range_tmp = range_cur;
 				range_cur = range_cur->next;
-				kfree (range_tmp);
+				kfree(range_tmp);
 				range_tmp = NULL;
 			}
 		}
@@ -1602,7 +1596,7 @@ void ibmphp_free_resources (void)
 					break;
 				range_tmp = range_cur;
 				range_cur = range_cur->next;
-				kfree (range_tmp);
+				kfree(range_tmp);
 				range_tmp = NULL;
 			}
 		}
@@ -1613,7 +1607,7 @@ void ibmphp_free_resources (void)
 					break;
 				range_tmp = range_cur;
 				range_cur = range_cur->next;
-				kfree (range_tmp);
+				kfree(range_tmp);
 				range_tmp = NULL;
 			}
 		}
@@ -1626,7 +1620,7 @@ void ibmphp_free_resources (void)
 					res_cur = res_cur->next;
 				else
 					res_cur = res_cur->nextRange;
-				kfree (res_tmp);
+				kfree(res_tmp);
 				res_tmp = NULL;
 			}
 			bus_cur->firstIO = NULL;
@@ -1639,7 +1633,7 @@ void ibmphp_free_resources (void)
 					res_cur = res_cur->next;
 				else
 					res_cur = res_cur->nextRange;
-				kfree (res_tmp);
+				kfree(res_tmp);
 				res_tmp = NULL;
 			}
 			bus_cur->firstMem = NULL;
@@ -1652,7 +1646,7 @@ void ibmphp_free_resources (void)
 					res_cur = res_cur->next;
 				else
 					res_cur = res_cur->nextRange;
-				kfree (res_tmp);
+				kfree(res_tmp);
 				res_tmp = NULL;
 			}
 			bus_cur->firstPFMem = NULL;
@@ -1664,15 +1658,15 @@ void ibmphp_free_resources (void)
 				res_tmp = res_cur;
 				res_cur = res_cur->next;
 
-				kfree (res_tmp);
+				kfree(res_tmp);
 				res_tmp = NULL;
 			}
 			bus_cur->firstPFMemFromMem = NULL;
 		}
 
 		bus_tmp = bus_cur;
-		list_del (&bus_cur->bus_list);
-		kfree (bus_tmp);
+		list_del(&bus_cur->bus_list);
+		kfree(bus_tmp);
 		bus_tmp = NULL;
 	}
 }
@@ -1685,16 +1679,14 @@ void ibmphp_free_resources (void)
  * a new Mem node
  * This routine is called right after initialization
  *******************************************************************************/
-static int __init once_over (void)
+static int __init once_over(void)
 {
 	struct resource_node *pfmem_cur;
 	struct resource_node *pfmem_prev;
 	struct resource_node *mem;
 	struct bus_node *bus_cur;
-	struct list_head *tmp;
 
-	list_for_each (tmp, &gbuses) {
-		bus_cur = list_entry (tmp, struct bus_node, bus_list);
+	list_for_each_entry(bus_cur, &gbuses, bus_list) {
 		if ((!bus_cur->rangePFMem) && (bus_cur->firstPFMem)) {
 			for (pfmem_cur = bus_cur->firstPFMem, pfmem_prev = NULL; pfmem_cur; pfmem_prev = pfmem_cur, pfmem_cur = pfmem_cur->next) {
 				pfmem_cur->fromMem = 1;
@@ -1716,7 +1708,7 @@ static int __init once_over (void)
 
 				mem = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 				if (!mem) {
-					err ("out of system memory\n");
+					err("out of system memory\n");
 					return -ENOMEM;
 				}
 				mem->type = MEM;
@@ -1725,8 +1717,8 @@ static int __init once_over (void)
 				mem->start = pfmem_cur->start;
 				mem->end = pfmem_cur->end;
 				mem->len = pfmem_cur->len;
-				if (ibmphp_add_resource (mem) < 0)
-					err ("Trouble...trouble... EBDA allocated pfmem from mem, but system doesn't display it has this space... unless not PCI device...\n");
+				if (ibmphp_add_resource(mem) < 0)
+					err("Trouble...trouble... EBDA allocated pfmem from mem, but system doesn't display it has this space... unless not PCI device...\n");
 				pfmem_cur->rangeno = mem->rangeno;
 			}	/* end for pfmem */
 		}	/* end if */
@@ -1734,12 +1726,12 @@ static int __init once_over (void)
 	return 0;
 }
 
-int ibmphp_add_pfmem_from_mem (struct resource_node *pfmem)
+int ibmphp_add_pfmem_from_mem(struct resource_node *pfmem)
 {
-	struct bus_node *bus_cur = find_bus_wprev (pfmem->busno, NULL, 0);
+	struct bus_node *bus_cur = find_bus_wprev(pfmem->busno, NULL, 0);
 
 	if (!bus_cur) {
-		err ("cannot find bus of pfmem to add...\n");
+		err("cannot find bus of pfmem to add...\n");
 		return -ENODEV;
 	}
 
@@ -1759,22 +1751,18 @@ int ibmphp_add_pfmem_from_mem (struct resource_node *pfmem)
  * Parameters: bus_number
  * Returns: Bus pointer or NULL
  */
-struct bus_node *ibmphp_find_res_bus (u8 bus_number)
+struct bus_node *ibmphp_find_res_bus(u8 bus_number)
 {
-	return find_bus_wprev (bus_number, NULL, 0);
+	return find_bus_wprev(bus_number, NULL, 0);
 }
 
-static struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u8 flag)
+static struct bus_node *find_bus_wprev(u8 bus_number, struct bus_node **prev, u8 flag)
 {
 	struct bus_node *bus_cur;
-	struct list_head *tmp;
-	struct list_head *tmp_prev;
 
-	list_for_each (tmp, &gbuses) {
-		tmp_prev = tmp->prev;
-		bus_cur = list_entry (tmp, struct bus_node, bus_list);
+	list_for_each_entry(bus_cur, &gbuses, bus_list) {
 		if (flag)
-			*prev = list_entry (tmp_prev, struct bus_node, bus_list);
+			*prev = list_prev_entry(bus_cur, bus_list);
 		if (bus_cur->busno == bus_number)
 			return bus_cur;
 	}
@@ -1782,23 +1770,21 @@ static struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u
 	return NULL;
 }
 
-void ibmphp_print_test (void)
+void ibmphp_print_test(void)
 {
 	int i = 0;
 	struct bus_node *bus_cur = NULL;
 	struct range_node *range;
 	struct resource_node *res;
-	struct list_head *tmp;
 
-	debug_pci ("*****************START**********************\n");
+	debug_pci("*****************START**********************\n");
 
 	if ((!list_empty(&gbuses)) && flags) {
-		err ("The GBUSES is not NULL?!?!?!?!?\n");
+		err("The GBUSES is not NULL?!?!?!?!?\n");
 		return;
 	}
 
-	list_for_each (tmp, &gbuses) {
-		bus_cur = list_entry (tmp, struct bus_node, bus_list);
+	list_for_each_entry(bus_cur, &gbuses, bus_list) {
 		debug_pci ("This is bus # %d.  There are\n", bus_cur->busno);
 		debug_pci ("IORanges = %d\t", bus_cur->noIORanges);
 		debug_pci ("MemRanges = %d\t", bus_cur->noMemRanges);
@@ -1807,42 +1793,42 @@ void ibmphp_print_test (void)
 		if (bus_cur->rangeIO) {
 			range = bus_cur->rangeIO;
 			for (i = 0; i < bus_cur->noIORanges; i++) {
-				debug_pci ("rangeno is %d\n", range->rangeno);
-				debug_pci ("[%x - %x]\n", range->start, range->end);
+				debug_pci("rangeno is %d\n", range->rangeno);
+				debug_pci("[%x - %x]\n", range->start, range->end);
 				range = range->next;
 			}
 		}
 
-		debug_pci ("The Mem Ranges are as follows:\n");
+		debug_pci("The Mem Ranges are as follows:\n");
 		if (bus_cur->rangeMem) {
 			range = bus_cur->rangeMem;
 			for (i = 0; i < bus_cur->noMemRanges; i++) {
-				debug_pci ("rangeno is %d\n", range->rangeno);
-				debug_pci ("[%x - %x]\n", range->start, range->end);
+				debug_pci("rangeno is %d\n", range->rangeno);
+				debug_pci("[%x - %x]\n", range->start, range->end);
 				range = range->next;
 			}
 		}
 
-		debug_pci ("The PFMem Ranges are as follows:\n");
+		debug_pci("The PFMem Ranges are as follows:\n");
 
 		if (bus_cur->rangePFMem) {
 			range = bus_cur->rangePFMem;
 			for (i = 0; i < bus_cur->noPFMemRanges; i++) {
-				debug_pci ("rangeno is %d\n", range->rangeno);
-				debug_pci ("[%x - %x]\n", range->start, range->end);
+				debug_pci("rangeno is %d\n", range->rangeno);
+				debug_pci("[%x - %x]\n", range->start, range->end);
 				range = range->next;
 			}
 		}
 
-		debug_pci ("The resources on this bus are as follows\n");
+		debug_pci("The resources on this bus are as follows\n");
 
-		debug_pci ("IO...\n");
+		debug_pci("IO...\n");
 		if (bus_cur->firstIO) {
 			res = bus_cur->firstIO;
 			while (res) {
-				debug_pci ("The range # is %d\n", res->rangeno);
-				debug_pci ("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
-				debug_pci ("[%x - %x], len=%x\n", res->start, res->end, res->len);
+				debug_pci("The range # is %d\n", res->rangeno);
+				debug_pci("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
+				debug_pci("[%x - %x], len=%x\n", res->start, res->end, res->len);
 				if (res->next)
 					res = res->next;
 				else if (res->nextRange)
@@ -1851,13 +1837,13 @@ void ibmphp_print_test (void)
 					break;
 			}
 		}
-		debug_pci ("Mem...\n");
+		debug_pci("Mem...\n");
 		if (bus_cur->firstMem) {
 			res = bus_cur->firstMem;
 			while (res) {
-				debug_pci ("The range # is %d\n", res->rangeno);
-				debug_pci ("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
-				debug_pci ("[%x - %x], len=%x\n", res->start, res->end, res->len);
+				debug_pci("The range # is %d\n", res->rangeno);
+				debug_pci("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
+				debug_pci("[%x - %x], len=%x\n", res->start, res->end, res->len);
 				if (res->next)
 					res = res->next;
 				else if (res->nextRange)
@@ -1866,13 +1852,13 @@ void ibmphp_print_test (void)
 					break;
 			}
 		}
-		debug_pci ("PFMem...\n");
+		debug_pci("PFMem...\n");
 		if (bus_cur->firstPFMem) {
 			res = bus_cur->firstPFMem;
 			while (res) {
-				debug_pci ("The range # is %d\n", res->rangeno);
-				debug_pci ("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
-				debug_pci ("[%x - %x], len=%x\n", res->start, res->end, res->len);
+				debug_pci("The range # is %d\n", res->rangeno);
+				debug_pci("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
+				debug_pci("[%x - %x], len=%x\n", res->start, res->end, res->len);
 				if (res->next)
 					res = res->next;
 				else if (res->nextRange)
@@ -1882,23 +1868,23 @@ void ibmphp_print_test (void)
 			}
 		}
 
-		debug_pci ("PFMemFromMem...\n");
+		debug_pci("PFMemFromMem...\n");
 		if (bus_cur->firstPFMemFromMem) {
 			res = bus_cur->firstPFMemFromMem;
 			while (res) {
-				debug_pci ("The range # is %d\n", res->rangeno);
-				debug_pci ("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
-				debug_pci ("[%x - %x], len=%x\n", res->start, res->end, res->len);
+				debug_pci("The range # is %d\n", res->rangeno);
+				debug_pci("The bus, devfnc is %d, %x\n", res->busno, res->devfunc);
+				debug_pci("[%x - %x], len=%x\n", res->start, res->end, res->len);
 				res = res->next;
 			}
 		}
 	}
-	debug_pci ("***********************END***********************\n");
+	debug_pci("***********************END***********************\n");
 }
 
-static int range_exists_already (struct range_node * range, struct bus_node * bus_cur, u8 type)
+static int range_exists_already(struct range_node *range, struct bus_node *bus_cur, u8 type)
 {
-	struct range_node * range_cur = NULL;
+	struct range_node *range_cur = NULL;
 	switch (type) {
 		case IO:
 			range_cur = bus_cur->rangeIO;
@@ -1910,7 +1896,7 @@ static int range_exists_already (struct range_node * range, struct bus_node * bu
 			range_cur = bus_cur->rangePFMem;
 			break;
 		default:
-			err ("wrong type passed to find out if range already exists\n");
+			err("wrong type passed to find out if range already exists\n");
 			return -ENODEV;
 	}
 
@@ -1937,7 +1923,7 @@ static int range_exists_already (struct range_node * range, struct bus_node * bu
  *	 behind them All these are TO DO.
  *	 Also need to add more error checkings... (from fnc returns etc)
  */
-static int __init update_bridge_ranges (struct bus_node **bus)
+static int __init update_bridge_ranges(struct bus_node **bus)
 {
 	u8 sec_busno, device, function, hdr_type, start_io_address, end_io_address;
 	u16 vendor_id, upper_io_start, upper_io_end, start_mem_address, end_mem_address;
@@ -1955,17 +1941,17 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 		return -ENODEV;
 	ibmphp_pci_bus->number = bus_cur->busno;
 
-	debug ("inside %s\n", __func__);
-	debug ("bus_cur->busno = %x\n", bus_cur->busno);
+	debug("inside %s\n", __func__);
+	debug("bus_cur->busno = %x\n", bus_cur->busno);
 
 	for (device = 0; device < 32; device++) {
 		for (function = 0x00; function < 0x08; function++) {
 			devfn = PCI_DEVFN(device, function);
-			pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
+			pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_VENDOR_ID, &vendor_id);
 
 			if (vendor_id != PCI_VENDOR_ID_NOTVALID) {
 				/* found correct device!!! */
-				pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
+				pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_HEADER_TYPE, &hdr_type);
 
 				switch (hdr_type) {
 					case PCI_HEADER_TYPE_NORMAL:
@@ -1984,18 +1970,18 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 						   temp++;
 						   }
 						 */
-						pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_busno);
-						bus_sec = find_bus_wprev (sec_busno, NULL, 0);
+						pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_SECONDARY_BUS, &sec_busno);
+						bus_sec = find_bus_wprev(sec_busno, NULL, 0);
 						/* this bus structure doesn't exist yet, PPB was configured during previous loading of ibmphp */
 						if (!bus_sec) {
-							bus_sec = alloc_error_bus (NULL, sec_busno, 1);
+							bus_sec = alloc_error_bus(NULL, sec_busno, 1);
 							/* the rest will be populated during NVRAM call */
 							return 0;
 						}
-						pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_IO_BASE, &start_io_address);
-						pci_bus_read_config_byte (ibmphp_pci_bus, devfn, PCI_IO_LIMIT, &end_io_address);
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_IO_BASE_UPPER16, &upper_io_start);
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_IO_LIMIT_UPPER16, &upper_io_end);
+						pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_IO_BASE, &start_io_address);
+						pci_bus_read_config_byte(ibmphp_pci_bus, devfn, PCI_IO_LIMIT, &end_io_address);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_IO_BASE_UPPER16, &upper_io_start);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_IO_LIMIT_UPPER16, &upper_io_end);
 						start_address = (start_io_address & PCI_IO_RANGE_MASK) << 8;
 						start_address |= (upper_io_start << 16);
 						end_address = (end_io_address & PCI_IO_RANGE_MASK) << 8;
@@ -2004,18 +1990,18 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 						if ((start_address) && (start_address <= end_address)) {
 							range = kzalloc(sizeof(struct range_node), GFP_KERNEL);
 							if (!range) {
-								err ("out of system memory\n");
+								err("out of system memory\n");
 								return -ENOMEM;
 							}
 							range->start = start_address;
 							range->end = end_address + 0xfff;
 
 							if (bus_sec->noIORanges > 0) {
-								if (!range_exists_already (range, bus_sec, IO)) {
-									add_bus_range (IO, range, bus_sec);
+								if (!range_exists_already(range, bus_sec, IO)) {
+									add_bus_range(IO, range, bus_sec);
 									++bus_sec->noIORanges;
 								} else {
-									kfree (range);
+									kfree(range);
 									range = NULL;
 								}
 							} else {
@@ -2024,13 +2010,13 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								bus_sec->rangeIO = range;
 								++bus_sec->noIORanges;
 							}
-							fix_resources (bus_sec);
+							fix_resources(bus_sec);
 
-							if (ibmphp_find_resource (bus_cur, start_address, &io, IO)) {
+							if (ibmphp_find_resource(bus_cur, start_address, &io, IO)) {
 								io = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 								if (!io) {
-									kfree (range);
-									err ("out of system memory\n");
+									kfree(range);
+									err("out of system memory\n");
 									return -ENOMEM;
 								}
 								io->type = IO;
@@ -2039,12 +2025,12 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								io->start = start_address;
 								io->end = end_address + 0xfff;
 								io->len = io->end - io->start + 1;
-								ibmphp_add_resource (io);
+								ibmphp_add_resource(io);
 							}
 						}
 
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, &start_mem_address);
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, &end_mem_address);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_BASE, &start_mem_address);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_MEMORY_LIMIT, &end_mem_address);
 
 						start_address = 0x00000000 | (start_mem_address & PCI_MEMORY_RANGE_MASK) << 16;
 						end_address = 0x00000000 | (end_mem_address & PCI_MEMORY_RANGE_MASK) << 16;
@@ -2053,18 +2039,18 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 
 							range = kzalloc(sizeof(struct range_node), GFP_KERNEL);
 							if (!range) {
-								err ("out of system memory\n");
+								err("out of system memory\n");
 								return -ENOMEM;
 							}
 							range->start = start_address;
 							range->end = end_address + 0xfffff;
 
 							if (bus_sec->noMemRanges > 0) {
-								if (!range_exists_already (range, bus_sec, MEM)) {
-									add_bus_range (MEM, range, bus_sec);
+								if (!range_exists_already(range, bus_sec, MEM)) {
+									add_bus_range(MEM, range, bus_sec);
 									++bus_sec->noMemRanges;
 								} else {
-									kfree (range);
+									kfree(range);
 									range = NULL;
 								}
 							} else {
@@ -2074,13 +2060,13 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								++bus_sec->noMemRanges;
 							}
 
-							fix_resources (bus_sec);
+							fix_resources(bus_sec);
 
-							if (ibmphp_find_resource (bus_cur, start_address, &mem, MEM)) {
+							if (ibmphp_find_resource(bus_cur, start_address, &mem, MEM)) {
 								mem = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 								if (!mem) {
-									kfree (range);
-									err ("out of system memory\n");
+									kfree(range);
+									err("out of system memory\n");
 									return -ENOMEM;
 								}
 								mem->type = MEM;
@@ -2089,13 +2075,13 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								mem->start = start_address;
 								mem->end = end_address + 0xfffff;
 								mem->len = mem->end - mem->start + 1;
-								ibmphp_add_resource (mem);
+								ibmphp_add_resource(mem);
 							}
 						}
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &start_mem_address);
-						pci_bus_read_config_word (ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, &end_mem_address);
-						pci_bus_read_config_dword (ibmphp_pci_bus, devfn, PCI_PREF_BASE_UPPER32, &upper_start);
-						pci_bus_read_config_dword (ibmphp_pci_bus, devfn, PCI_PREF_LIMIT_UPPER32, &upper_end);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_BASE, &start_mem_address);
+						pci_bus_read_config_word(ibmphp_pci_bus, devfn, PCI_PREF_MEMORY_LIMIT, &end_mem_address);
+						pci_bus_read_config_dword(ibmphp_pci_bus, devfn, PCI_PREF_BASE_UPPER32, &upper_start);
+						pci_bus_read_config_dword(ibmphp_pci_bus, devfn, PCI_PREF_LIMIT_UPPER32, &upper_end);
 						start_address = 0x00000000 | (start_mem_address & PCI_MEMORY_RANGE_MASK) << 16;
 						end_address = 0x00000000 | (end_mem_address & PCI_MEMORY_RANGE_MASK) << 16;
 #if BITS_PER_LONG == 64
@@ -2107,18 +2093,18 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 
 							range = kzalloc(sizeof(struct range_node), GFP_KERNEL);
 							if (!range) {
-								err ("out of system memory\n");
+								err("out of system memory\n");
 								return -ENOMEM;
 							}
 							range->start = start_address;
 							range->end = end_address + 0xfffff;
 
 							if (bus_sec->noPFMemRanges > 0) {
-								if (!range_exists_already (range, bus_sec, PFMEM)) {
-									add_bus_range (PFMEM, range, bus_sec);
+								if (!range_exists_already(range, bus_sec, PFMEM)) {
+									add_bus_range(PFMEM, range, bus_sec);
 									++bus_sec->noPFMemRanges;
 								} else {
-									kfree (range);
+									kfree(range);
 									range = NULL;
 								}
 							} else {
@@ -2128,12 +2114,12 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								++bus_sec->noPFMemRanges;
 							}
 
-							fix_resources (bus_sec);
-							if (ibmphp_find_resource (bus_cur, start_address, &pfmem, PFMEM)) {
+							fix_resources(bus_sec);
+							if (ibmphp_find_resource(bus_cur, start_address, &pfmem, PFMEM)) {
 								pfmem = kzalloc(sizeof(struct resource_node), GFP_KERNEL);
 								if (!pfmem) {
-									kfree (range);
-									err ("out of system memory\n");
+									kfree(range);
+									err("out of system memory\n");
 									return -ENOMEM;
 								}
 								pfmem->type = PFMEM;
@@ -2144,7 +2130,7 @@ static int __init update_bridge_ranges (struct bus_node **bus)
 								pfmem->len = pfmem->end - pfmem->start + 1;
 								pfmem->fromMem = 0;
 
-								ibmphp_add_resource (pfmem);
+								ibmphp_add_resource(pfmem);
 							}
 						}
 						break;
diff --git a/drivers/pci/hotplug/pci_hotplug_core.c b/drivers/pci/hotplug/pci_hotplug_core.c
index d1fab97d6b01..9acd1997c6fe 100644
--- a/drivers/pci/hotplug/pci_hotplug_core.c
+++ b/drivers/pci/hotplug/pci_hotplug_core.c
@@ -45,10 +45,10 @@
 
 #define MY_NAME	"pci_hotplug"
 
-#define dbg(fmt, arg...) do { if (debug) printk(KERN_DEBUG "%s: %s: " fmt , MY_NAME , __func__ , ## arg); } while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format , MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format , MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format , MY_NAME , ## arg)
+#define dbg(fmt, arg...) do { if (debug) printk(KERN_DEBUG "%s: %s: " fmt, MY_NAME, __func__, ## arg); } while (0)
+#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 
 /* local variables */
@@ -226,7 +226,7 @@ static ssize_t test_write_file(struct pci_slot *pci_slot, const char *buf,
 	u32 test;
 	int retval = 0;
 
-	ltest = simple_strtoul (buf, NULL, 10);
+	ltest = simple_strtoul(buf, NULL, 10);
 	test = (u32)(ltest & 0xffffffff);
 	dbg("test = %d\n", test);
 
@@ -396,10 +396,8 @@ static void fs_remove_slot(struct pci_slot *pci_slot)
 static struct hotplug_slot *get_slot_from_name(const char *name)
 {
 	struct hotplug_slot *slot;
-	struct list_head *tmp;
 
-	list_for_each(tmp, &pci_hotplug_slot_list) {
-		slot = list_entry(tmp, struct hotplug_slot, slot_list);
+	list_for_each_entry(slot, &pci_hotplug_slot_list, slot_list) {
 		if (strcmp(hotplug_slot_name(slot), name) == 0)
 			return slot;
 	}
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index 62d6fe6c3714..e764918641ae 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -47,14 +47,14 @@ extern bool pciehp_debug;
 #define dbg(format, arg...)						\
 do {									\
 	if (pciehp_debug)						\
-		printk(KERN_DEBUG "%s: " format, MY_NAME , ## arg);	\
+		printk(KERN_DEBUG "%s: " format, MY_NAME, ## arg);	\
 } while (0)
 #define err(format, arg...)						\
-	printk(KERN_ERR "%s: " format, MY_NAME , ## arg)
+	printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
 #define info(format, arg...)						\
-	printk(KERN_INFO "%s: " format, MY_NAME , ## arg)
+	printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
 #define warn(format, arg...)						\
-	printk(KERN_WARNING "%s: " format, MY_NAME , ## arg)
+	printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 #define ctrl_dbg(ctrl, format, arg...)					\
 	do {								\
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 612b21a14df5..ac531e674a05 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -62,14 +62,14 @@ MODULE_PARM_DESC(pciehp_force, "Force pciehp, even if OSHP is missing");
 
 #define PCIE_MODULE_NAME "pciehp"
 
-static int set_attention_status (struct hotplug_slot *slot, u8 value);
-static int enable_slot		(struct hotplug_slot *slot);
-static int disable_slot		(struct hotplug_slot *slot);
-static int get_power_status	(struct hotplug_slot *slot, u8 *value);
-static int get_attention_status	(struct hotplug_slot *slot, u8 *value);
-static int get_latch_status	(struct hotplug_slot *slot, u8 *value);
-static int get_adapter_status	(struct hotplug_slot *slot, u8 *value);
-static int reset_slot		(struct hotplug_slot *slot, int probe);
+static int set_attention_status(struct hotplug_slot *slot, u8 value);
+static int enable_slot(struct hotplug_slot *slot);
+static int disable_slot(struct hotplug_slot *slot);
+static int get_power_status(struct hotplug_slot *slot, u8 *value);
+static int get_attention_status(struct hotplug_slot *slot, u8 *value);
+static int get_latch_status(struct hotplug_slot *slot, u8 *value);
+static int get_adapter_status(struct hotplug_slot *slot, u8 *value);
+static int reset_slot(struct hotplug_slot *slot, int probe);
 
 /**
  * release_slot - free up the memory used by a slot
diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c
index 4c8f4cde6854..880978b6d534 100644
--- a/drivers/pci/hotplug/pciehp_ctrl.c
+++ b/drivers/pci/hotplug/pciehp_ctrl.c
@@ -511,7 +511,9 @@ int pciehp_sysfs_disable_slot(struct slot *p_slot)
 	case STATIC_STATE:
 		p_slot->state = POWEROFF_STATE;
 		mutex_unlock(&p_slot->lock);
+		mutex_lock(&p_slot->hotplug_lock);
 		retval = pciehp_disable_slot(p_slot);
+		mutex_unlock(&p_slot->hotplug_lock);
 		mutex_lock(&p_slot->lock);
 		p_slot->state = STATIC_STATE;
 		break;
diff --git a/drivers/pci/hotplug/pcihp_skeleton.c b/drivers/pci/hotplug/pcihp_skeleton.c
index d062c008fc95..172ed89200cd 100644
--- a/drivers/pci/hotplug/pcihp_skeleton.c
+++ b/drivers/pci/hotplug/pcihp_skeleton.c
@@ -52,11 +52,11 @@ static LIST_HEAD(slot_list);
 	do {							\
 		if (debug)					\
 			printk(KERN_DEBUG "%s: " format "\n",	\
-				MY_NAME , ## arg);		\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format "\n", MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format "\n", MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format "\n", MY_NAME, ## arg)
 
 /* local variables */
 static bool debug;
@@ -72,14 +72,14 @@ MODULE_LICENSE("GPL");
 module_param(debug, bool, 0644);
 MODULE_PARM_DESC(debug, "Debugging mode enabled or not");
 
-static int enable_slot		(struct hotplug_slot *slot);
-static int disable_slot		(struct hotplug_slot *slot);
-static int set_attention_status (struct hotplug_slot *slot, u8 value);
-static int hardware_test	(struct hotplug_slot *slot, u32 value);
-static int get_power_status	(struct hotplug_slot *slot, u8 *value);
-static int get_attention_status	(struct hotplug_slot *slot, u8 *value);
-static int get_latch_status	(struct hotplug_slot *slot, u8 *value);
-static int get_adapter_status	(struct hotplug_slot *slot, u8 *value);
+static int enable_slot(struct hotplug_slot *slot);
+static int disable_slot(struct hotplug_slot *slot);
+static int set_attention_status(struct hotplug_slot *slot, u8 value);
+static int hardware_test(struct hotplug_slot *slot, u32 value);
+static int get_power_status(struct hotplug_slot *slot, u8 *value);
+static int get_attention_status(struct hotplug_slot *slot, u8 *value);
+static int get_latch_status(struct hotplug_slot *slot, u8 *value);
+static int get_adapter_status(struct hotplug_slot *slot, u8 *value);
 
 static struct hotplug_slot_ops skel_hotplug_slot_ops = {
 	.enable_slot =		enable_slot,
@@ -321,17 +321,14 @@ error:
 
 static void __exit cleanup_slots(void)
 {
-	struct list_head *tmp;
-	struct list_head *next;
-	struct slot *slot;
+	struct slot *slot, *next;
 
 	/*
 	 * Unregister all of our slots with the pci_hotplug subsystem.
 	 * Memory will be freed in release_slot() callback after slot's
 	 * lifespan is finished.
 	 */
-	list_for_each_safe(tmp, next, &slot_list) {
-		slot = list_entry(tmp, struct slot, slot_list);
+	list_for_each_entry_safe(slot, next, &slot_list, slot_list) {
 		list_del(&slot->slot_list);
 		pci_hp_deregister(slot->hotplug_slot);
 	}
diff --git a/drivers/pci/hotplug/rpadlpar_core.c b/drivers/pci/hotplug/rpadlpar_core.c
index e12bafdc42e0..b46b57d870fc 100644
--- a/drivers/pci/hotplug/rpadlpar_core.c
+++ b/drivers/pci/hotplug/rpadlpar_core.c
@@ -114,11 +114,10 @@ static struct device_node *find_dlpar_node(char *drc_name, int *node_type)
  */
 static struct slot *find_php_slot(struct device_node *dn)
 {
-	struct list_head *tmp, *n;
-	struct slot *slot;
+	struct slot *slot, *next;
 
-	list_for_each_safe(tmp, n, &rpaphp_slot_head) {
-		slot = list_entry(tmp, struct slot, rpaphp_slot_list);
+	list_for_each_entry_safe(slot, next, &rpaphp_slot_head,
+				 rpaphp_slot_list) {
 		if (slot->dn == dn)
 			return slot;
 	}
diff --git a/drivers/pci/hotplug/rpaphp.h b/drivers/pci/hotplug/rpaphp.h
index b2593e876a09..7db024e68fe6 100644
--- a/drivers/pci/hotplug/rpaphp.h
+++ b/drivers/pci/hotplug/rpaphp.h
@@ -51,11 +51,11 @@ extern bool rpaphp_debug;
 	do {							\
 		if (rpaphp_debug)				\
 			printk(KERN_DEBUG "%s: " format,	\
-				MY_NAME , ## arg);		\
+				MY_NAME, ## arg);		\
 	} while (0)
-#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME , ## arg)
-#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME , ## arg)
-#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME , ## arg)
+#define err(format, arg...) printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
+#define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
+#define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 /* slot states */
 
diff --git a/drivers/pci/hotplug/rpaphp_core.c b/drivers/pci/hotplug/rpaphp_core.c
index f2945fa73d4f..611f6056221a 100644
--- a/drivers/pci/hotplug/rpaphp_core.c
+++ b/drivers/pci/hotplug/rpaphp_core.c
@@ -94,7 +94,7 @@ static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value)
 	int retval, level;
 	struct slot *slot = (struct slot *)hotplug_slot->private;
 
-	retval = rtas_get_power_level (slot->power_domain, &level);
+	retval = rtas_get_power_level(slot->power_domain, &level);
 	if (!retval)
 		*value = level;
 	return retval;
@@ -356,8 +356,7 @@ EXPORT_SYMBOL_GPL(rpaphp_add_slot);
 
 static void __exit cleanup_slots(void)
 {
-	struct list_head *tmp, *n;
-	struct slot *slot;
+	struct slot *slot, *next;
 
 	/*
 	 * Unregister all of our slots with the pci_hotplug subsystem,
@@ -365,8 +364,8 @@ static void __exit cleanup_slots(void)
 	 * memory will be freed in release_slot callback.
 	 */
 
-	list_for_each_safe(tmp, n, &rpaphp_slot_head) {
-		slot = list_entry(tmp, struct slot, rpaphp_slot_list);
+	list_for_each_entry_safe(slot, next, &rpaphp_slot_head,
+				 rpaphp_slot_list) {
 		list_del(&slot->rpaphp_slot_list);
 		pci_hp_deregister(slot->hotplug_slot);
 	}
diff --git a/drivers/pci/hotplug/rpaphp_pci.c b/drivers/pci/hotplug/rpaphp_pci.c
index 9243f3e7a1c9..7836d6913e67 100644
--- a/drivers/pci/hotplug/rpaphp_pci.c
+++ b/drivers/pci/hotplug/rpaphp_pci.c
@@ -126,7 +126,7 @@ int rpaphp_enable_slot(struct slot *slot)
 		if (rpaphp_debug) {
 			struct pci_dev *dev;
 			dbg("%s: pci_devs of slot[%s]\n", __func__, slot->dn->full_name);
-			list_for_each_entry (dev, &bus->devices, bus_list)
+			list_for_each_entry(dev, &bus->devices, bus_list)
 				dbg("\t%s\n", pci_name(dev));
 		}
 	}
diff --git a/drivers/pci/hotplug/rpaphp_slot.c b/drivers/pci/hotplug/rpaphp_slot.c
index a6082cc263f7..6937c725b00b 100644
--- a/drivers/pci/hotplug/rpaphp_slot.c
+++ b/drivers/pci/hotplug/rpaphp_slot.c
@@ -48,7 +48,7 @@ void dealloc_slot_struct(struct slot *slot)
 }
 
 struct slot *alloc_slot_struct(struct device_node *dn,
-                       int drc_index, char *drc_name, int power_domain)
+		int drc_index, char *drc_name, int power_domain)
 {
 	struct slot *slot;
 
diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c
index d77e46bca54c..eb5efaef06ea 100644
--- a/drivers/pci/hotplug/s390_pci_hpc.c
+++ b/drivers/pci/hotplug/s390_pci_hpc.c
@@ -201,11 +201,10 @@ error:
 
 void zpci_exit_slot(struct zpci_dev *zdev)
 {
-	struct list_head *tmp, *n;
-	struct slot *slot;
+	struct slot *slot, *next;
 
-	list_for_each_safe(tmp, n, &s390_hotplug_slot_list) {
-		slot = list_entry(tmp, struct slot, slot_list);
+	list_for_each_entry_safe(slot, next, &s390_hotplug_slot_list,
+				 slot_list) {
 		if (slot->zdev != zdev)
 			continue;
 		list_del(&slot->slot_list);
diff --git a/drivers/pci/hotplug/sgi_hotplug.c b/drivers/pci/hotplug/sgi_hotplug.c
index c32fb786d48e..339bce0403dd 100644
--- a/drivers/pci/hotplug/sgi_hotplug.c
+++ b/drivers/pci/hotplug/sgi_hotplug.c
@@ -99,7 +99,7 @@ static ssize_t path_show(struct pci_slot *pci_slot, char *buf)
 	if (!slot)
 		return retval;
 
-	retval = sprintf (buf, "%s\n", slot->physical_path);
+	retval = sprintf(buf, "%s\n", slot->physical_path);
 	return retval;
 }
 
@@ -313,7 +313,7 @@ static int sn_slot_disable(struct hotplug_slot *bss_hotplug_slot,
 	}
 
 	if ((action == PCI_REQ_SLOT_DISABLE) && rc) {
-		dev_dbg(&slot->pci_bus->self->dev,"remove failed rc = %d\n", rc);
+		dev_dbg(&slot->pci_bus->self->dev, "remove failed rc = %d\n", rc);
 	}
 
 	return rc;
@@ -488,7 +488,7 @@ static int disable_slot(struct hotplug_slot *bss_hotplug_slot)
 
 	/* free the ACPI resources for the slot */
 	if (SN_ACPI_BASE_SUPPORT() &&
-            PCI_CONTROLLER(slot->pci_bus)->companion) {
+		PCI_CONTROLLER(slot->pci_bus)->companion) {
 		unsigned long long adr;
 		struct acpi_device *device;
 		acpi_handle phandle;
diff --git a/drivers/pci/hotplug/shpchp.h b/drivers/pci/hotplug/shpchp.h
index 5897d516427b..4da8fc601467 100644
--- a/drivers/pci/hotplug/shpchp.h
+++ b/drivers/pci/hotplug/shpchp.h
@@ -50,14 +50,14 @@ extern bool shpchp_debug;
 #define dbg(format, arg...)						\
 do {									\
 	if (shpchp_debug)						\
-		printk(KERN_DEBUG "%s: " format, MY_NAME , ## arg);	\
+		printk(KERN_DEBUG "%s: " format, MY_NAME, ## arg);	\
 } while (0)
 #define err(format, arg...)						\
-	printk(KERN_ERR "%s: " format, MY_NAME , ## arg)
+	printk(KERN_ERR "%s: " format, MY_NAME, ## arg)
 #define info(format, arg...)						\
-	printk(KERN_INFO "%s: " format, MY_NAME , ## arg)
+	printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
 #define warn(format, arg...)						\
-	printk(KERN_WARNING "%s: " format, MY_NAME , ## arg)
+	printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
 #define ctrl_dbg(ctrl, format, arg...)					\
 	do {								\
@@ -84,7 +84,7 @@ struct slot {
 	u8 presence_save;
 	u8 pwr_save;
 	struct controller *ctrl;
-	struct hpc_ops *hpc_ops;
+	const struct hpc_ops *hpc_ops;
 	struct hotplug_slot *hotplug_slot;
 	struct list_head	slot_list;
 	struct delayed_work work;	/* work for button event */
@@ -106,7 +106,7 @@ struct controller {
 	int slot_num_inc;		/* 1 or -1 */
 	struct pci_dev *pci_dev;
 	struct list_head slot_list;
-	struct hpc_ops *hpc_ops;
+	const struct hpc_ops *hpc_ops;
 	wait_queue_head_t queue;	/* sleep & wake process */
 	u8 slot_device_offset;
 	u32 pcix_misc2_reg;	/* for amd pogo errata */
@@ -295,7 +295,7 @@ static inline void amd_pogo_errata_restore_misc_reg(struct slot *p_slot)
 		pci_write_config_dword(p_slot->ctrl->pci_dev, PCIX_MEM_BASE_LIMIT_OFFSET, rse_set);
 	}
 	/* restore MiscII register */
-	pci_read_config_dword(p_slot->ctrl->pci_dev, PCIX_MISCII_OFFSET, &pcix_misc2_temp );
+	pci_read_config_dword(p_slot->ctrl->pci_dev, PCIX_MISCII_OFFSET, &pcix_misc2_temp);
 
 	if (p_slot->ctrl->pcix_misc2_reg & SERRFATALENABLE_MASK)
 		pcix_misc2_temp |= SERRFATALENABLE_MASK;
diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c
index 294ef4b10cf1..3454dc7385f1 100644
--- a/drivers/pci/hotplug/shpchp_core.c
+++ b/drivers/pci/hotplug/shpchp_core.c
@@ -57,13 +57,13 @@ MODULE_PARM_DESC(shpchp_poll_time, "Polling mechanism frequency, in seconds");
 
 #define SHPC_MODULE_NAME "shpchp"
 
-static int set_attention_status (struct hotplug_slot *slot, u8 value);
-static int enable_slot		(struct hotplug_slot *slot);
-static int disable_slot		(struct hotplug_slot *slot);
-static int get_power_status	(struct hotplug_slot *slot, u8 *value);
-static int get_attention_status	(struct hotplug_slot *slot, u8 *value);
-static int get_latch_status	(struct hotplug_slot *slot, u8 *value);
-static int get_adapter_status	(struct hotplug_slot *slot, u8 *value);
+static int set_attention_status(struct hotplug_slot *slot, u8 value);
+static int enable_slot(struct hotplug_slot *slot);
+static int disable_slot(struct hotplug_slot *slot);
+static int get_power_status(struct hotplug_slot *slot, u8 *value);
+static int get_attention_status(struct hotplug_slot *slot, u8 *value);
+static int get_latch_status(struct hotplug_slot *slot, u8 *value);
+static int get_adapter_status(struct hotplug_slot *slot, u8 *value);
 
 static struct hotplug_slot_ops shpchp_hotplug_slot_ops = {
 	.set_attention_status =	set_attention_status,
@@ -178,12 +178,9 @@ error:
 
 void cleanup_slots(struct controller *ctrl)
 {
-	struct list_head *tmp;
-	struct list_head *next;
-	struct slot *slot;
+	struct slot *slot, *next;
 
-	list_for_each_safe(tmp, next, &ctrl->slot_list) {
-		slot = list_entry(tmp, struct slot, slot_list);
+	list_for_each_entry_safe(slot, next, &ctrl->slot_list, slot_list) {
 		list_del(&slot->slot_list);
 		cancel_delayed_work(&slot->work);
 		destroy_workqueue(slot->wq);
@@ -194,7 +191,7 @@ void cleanup_slots(struct controller *ctrl)
 /*
  * set_attention_status - Turns the Amber LED for a slot on, off or blink
  */
-static int set_attention_status (struct hotplug_slot *hotplug_slot, u8 status)
+static int set_attention_status(struct hotplug_slot *hotplug_slot, u8 status)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 
@@ -207,7 +204,7 @@ static int set_attention_status (struct hotplug_slot *hotplug_slot, u8 status)
 	return 0;
 }
 
-static int enable_slot (struct hotplug_slot *hotplug_slot)
+static int enable_slot(struct hotplug_slot *hotplug_slot)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 
@@ -217,7 +214,7 @@ static int enable_slot (struct hotplug_slot *hotplug_slot)
 	return shpchp_sysfs_enable_slot(slot);
 }
 
-static int disable_slot (struct hotplug_slot *hotplug_slot)
+static int disable_slot(struct hotplug_slot *hotplug_slot)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 
@@ -227,7 +224,7 @@ static int disable_slot (struct hotplug_slot *hotplug_slot)
 	return shpchp_sysfs_disable_slot(slot);
 }
 
-static int get_power_status (struct hotplug_slot *hotplug_slot, u8 *value)
+static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 	int retval;
@@ -242,7 +239,7 @@ static int get_power_status (struct hotplug_slot *hotplug_slot, u8 *value)
 	return 0;
 }
 
-static int get_attention_status (struct hotplug_slot *hotplug_slot, u8 *value)
+static int get_attention_status(struct hotplug_slot *hotplug_slot, u8 *value)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 	int retval;
@@ -257,7 +254,7 @@ static int get_attention_status (struct hotplug_slot *hotplug_slot, u8 *value)
 	return 0;
 }
 
-static int get_latch_status (struct hotplug_slot *hotplug_slot, u8 *value)
+static int get_latch_status(struct hotplug_slot *hotplug_slot, u8 *value)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 	int retval;
@@ -272,7 +269,7 @@ static int get_latch_status (struct hotplug_slot *hotplug_slot, u8 *value)
 	return 0;
 }
 
-static int get_adapter_status (struct hotplug_slot *hotplug_slot, u8 *value)
+static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value)
 {
 	struct slot *slot = get_slot(hotplug_slot);
 	int retval;
diff --git a/drivers/pci/hotplug/shpchp_hpc.c b/drivers/pci/hotplug/shpchp_hpc.c
index 7d223e9080ef..de0ea474fb73 100644
--- a/drivers/pci/hotplug/shpchp_hpc.c
+++ b/drivers/pci/hotplug/shpchp_hpc.c
@@ -542,7 +542,7 @@ static int hpc_set_attention_status(struct slot *slot, u8 value)
 	u8 slot_cmd = 0;
 
 	switch (value) {
-		case 0 :
+		case 0:
 			slot_cmd = SET_ATTN_OFF;	/* OFF */
 			break;
 		case 1:
@@ -910,7 +910,7 @@ static int shpc_get_max_bus_speed(struct controller *ctrl)
 	return retval;
 }
 
-static struct hpc_ops shpchp_hpc_ops = {
+static const struct hpc_ops shpchp_hpc_ops = {
 	.power_on_slot			= hpc_power_on_slot,
 	.slot_enable			= hpc_slot_enable,
 	.slot_disable			= hpc_slot_disable,
diff --git a/drivers/pci/hotplug/shpchp_sysfs.c b/drivers/pci/hotplug/shpchp_sysfs.c
index 52875b360463..7efb56a28c9f 100644
--- a/drivers/pci/hotplug/shpchp_sysfs.c
+++ b/drivers/pci/hotplug/shpchp_sysfs.c
@@ -35,7 +35,7 @@
 
 /* A few routines that create sysfs entries for the hot plug controller */
 
-static ssize_t show_ctrl (struct device *dev, struct device_attribute *attr, char *buf)
+static ssize_t show_ctrl(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct pci_dev *pdev;
 	char *out = buf;
@@ -43,7 +43,7 @@ static ssize_t show_ctrl (struct device *dev, struct device_attribute *attr, cha
 	struct resource *res;
 	struct pci_bus *bus;
 
-	pdev = container_of (dev, struct pci_dev, dev);
+	pdev = to_pci_dev(dev);
 	bus = pdev->subordinate;
 
 	out += sprintf(buf, "Free resources: memory\n");
@@ -83,11 +83,11 @@ static ssize_t show_ctrl (struct device *dev, struct device_attribute *attr, cha
 
 	return out - buf;
 }
-static DEVICE_ATTR (ctrl, S_IRUGO, show_ctrl, NULL);
+static DEVICE_ATTR(ctrl, S_IRUGO, show_ctrl, NULL);
 
-int shpchp_create_ctrl_files (struct controller *ctrl)
+int shpchp_create_ctrl_files(struct controller *ctrl)
 {
-	return device_create_file (&ctrl->pci_dev->dev, &dev_attr_ctrl);
+	return device_create_file(&ctrl->pci_dev->dev, &dev_attr_ctrl);
 }
 
 void shpchp_remove_ctrl_files(struct controller *ctrl)
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 7a0df3fdbfae..a080f4496fe2 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1026,10 +1026,6 @@ int pci_msi_enabled(void)
 }
 EXPORT_SYMBOL(pci_msi_enabled);
 
-void pci_msi_init_pci_dev(struct pci_dev *dev)
-{
-}
-
 /**
  * pci_enable_msi_range - configure device's MSI capability structure
  * @dev: device to configure
diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c
index 024b5c179348..0ae74d96ed85 100644
--- a/drivers/pci/pci-label.c
+++ b/drivers/pci/pci-label.c
@@ -77,7 +77,7 @@ static umode_t smbios_instance_string_exist(struct kobject *kobj,
 	struct device *dev;
 	struct pci_dev *pdev;
 
-	dev = container_of(kobj, struct device, kobj);
+	dev = kobj_to_dev(kobj);
 	pdev = to_pci_dev(dev);
 
 	return find_smbios_instance_string(pdev, NULL, SMBIOS_ATTR_NONE) ?
@@ -221,7 +221,7 @@ static umode_t acpi_index_string_exist(struct kobject *kobj,
 {
 	struct device *dev;
 
-	dev = container_of(kobj, struct device, kobj);
+	dev = kobj_to_dev(kobj);
 
 	if (device_has_dsm(dev))
 		return S_IRUGO;
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index eead54cd01b2..95d9e7bd933b 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -630,8 +630,7 @@ static ssize_t pci_read_config(struct file *filp, struct kobject *kobj,
 			       struct bin_attribute *bin_attr, char *buf,
 			       loff_t off, size_t count)
 {
-	struct pci_dev *dev = to_pci_dev(container_of(kobj, struct device,
-						      kobj));
+	struct pci_dev *dev = to_pci_dev(kobj_to_dev(kobj));
 	unsigned int size = 64;
 	loff_t init_off = off;
 	u8 *data = (u8 *) buf;
@@ -707,8 +706,7 @@ static ssize_t pci_write_config(struct file *filp, struct kobject *kobj,
 				struct bin_attribute *bin_attr, char *buf,
 				loff_t off, size_t count)
 {
-	struct pci_dev *dev = to_pci_dev(container_of(kobj, struct device,
-						      kobj));
+	struct pci_dev *dev = to_pci_dev(kobj_to_dev(kobj));
 	unsigned int size = count;
 	loff_t init_off = off;
 	u8 *data = (u8 *) buf;
@@ -769,8 +767,7 @@ static ssize_t read_vpd_attr(struct file *filp, struct kobject *kobj,
 			     struct bin_attribute *bin_attr, char *buf,
 			     loff_t off, size_t count)
 {
-	struct pci_dev *dev =
-		to_pci_dev(container_of(kobj, struct device, kobj));
+	struct pci_dev *dev = to_pci_dev(kobj_to_dev(kobj));
 
 	if (off > bin_attr->size)
 		count = 0;
@@ -784,8 +781,7 @@ static ssize_t write_vpd_attr(struct file *filp, struct kobject *kobj,
 			      struct bin_attribute *bin_attr, char *buf,
 			      loff_t off, size_t count)
 {
-	struct pci_dev *dev =
-		to_pci_dev(container_of(kobj, struct device, kobj));
+	struct pci_dev *dev = to_pci_dev(kobj_to_dev(kobj));
 
 	if (off > bin_attr->size)
 		count = 0;
@@ -812,8 +808,7 @@ static ssize_t pci_read_legacy_io(struct file *filp, struct kobject *kobj,
 				  struct bin_attribute *bin_attr, char *buf,
 				  loff_t off, size_t count)
 {
-	struct pci_bus *bus = to_pci_bus(container_of(kobj, struct device,
-						      kobj));
+	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
 
 	/* Only support 1, 2 or 4 byte accesses */
 	if (count != 1 && count != 2 && count != 4)
@@ -838,8 +833,7 @@ static ssize_t pci_write_legacy_io(struct file *filp, struct kobject *kobj,
 				   struct bin_attribute *bin_attr, char *buf,
 				   loff_t off, size_t count)
 {
-	struct pci_bus *bus = to_pci_bus(container_of(kobj, struct device,
-						      kobj));
+	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
 
 	/* Only support 1, 2 or 4 byte accesses */
 	if (count != 1 && count != 2 && count != 4)
@@ -863,8 +857,7 @@ static int pci_mmap_legacy_mem(struct file *filp, struct kobject *kobj,
 			       struct bin_attribute *attr,
 			       struct vm_area_struct *vma)
 {
-	struct pci_bus *bus = to_pci_bus(container_of(kobj, struct device,
-						      kobj));
+	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
 
 	return pci_mmap_legacy_page_range(bus, vma, pci_mmap_mem);
 }
@@ -884,8 +877,7 @@ static int pci_mmap_legacy_io(struct file *filp, struct kobject *kobj,
 			      struct bin_attribute *attr,
 			      struct vm_area_struct *vma)
 {
-	struct pci_bus *bus = to_pci_bus(container_of(kobj, struct device,
-						      kobj));
+	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
 
 	return pci_mmap_legacy_page_range(bus, vma, pci_mmap_io);
 }
@@ -1000,8 +992,7 @@ int pci_mmap_fits(struct pci_dev *pdev, int resno, struct vm_area_struct *vma,
 static int pci_mmap_resource(struct kobject *kobj, struct bin_attribute *attr,
 			     struct vm_area_struct *vma, int write_combine)
 {
-	struct pci_dev *pdev = to_pci_dev(container_of(kobj,
-						       struct device, kobj));
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 	struct resource *res = attr->private;
 	enum pci_mmap_state mmap_type;
 	resource_size_t start, end;
@@ -1054,8 +1045,7 @@ static ssize_t pci_resource_io(struct file *filp, struct kobject *kobj,
 			       struct bin_attribute *attr, char *buf,
 			       loff_t off, size_t count, bool write)
 {
-	struct pci_dev *pdev = to_pci_dev(container_of(kobj,
-						       struct device, kobj));
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 	struct resource *res = attr->private;
 	unsigned long port = off;
 	int i;
@@ -1225,7 +1215,7 @@ static ssize_t pci_write_rom(struct file *filp, struct kobject *kobj,
 			     struct bin_attribute *bin_attr, char *buf,
 			     loff_t off, size_t count)
 {
-	struct pci_dev *pdev = to_pci_dev(container_of(kobj, struct device, kobj));
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 
 	if ((off ==  0) && (*buf == '0') && (count == 2))
 		pdev->rom_attr_enabled = 0;
@@ -1251,7 +1241,7 @@ static ssize_t pci_read_rom(struct file *filp, struct kobject *kobj,
 			    struct bin_attribute *bin_attr, char *buf,
 			    loff_t off, size_t count)
 {
-	struct pci_dev *pdev = to_pci_dev(container_of(kobj, struct device, kobj));
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 	void __iomem *rom;
 	size_t size;
 
@@ -1372,10 +1362,10 @@ int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev)
 	if (!sysfs_initialized)
 		return -EACCES;
 
-	if (pdev->cfg_size < PCI_CFG_SPACE_EXP_SIZE)
-		retval = sysfs_create_bin_file(&pdev->dev.kobj, &pci_config_attr);
-	else
+	if (pdev->cfg_size > PCI_CFG_SPACE_SIZE)
 		retval = sysfs_create_bin_file(&pdev->dev.kobj, &pcie_config_attr);
+	else
+		retval = sysfs_create_bin_file(&pdev->dev.kobj, &pci_config_attr);
 	if (retval)
 		goto err;
 
@@ -1427,10 +1417,10 @@ err_rom_file:
 err_resource_files:
 	pci_remove_resource_files(pdev);
 err_config_file:
-	if (pdev->cfg_size < PCI_CFG_SPACE_EXP_SIZE)
-		sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
-	else
+	if (pdev->cfg_size > PCI_CFG_SPACE_SIZE)
 		sysfs_remove_bin_file(&pdev->dev.kobj, &pcie_config_attr);
+	else
+		sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
 err:
 	return retval;
 }
@@ -1464,10 +1454,10 @@ void pci_remove_sysfs_dev_files(struct pci_dev *pdev)
 
 	pci_remove_capabilities_sysfs(pdev);
 
-	if (pdev->cfg_size < PCI_CFG_SPACE_EXP_SIZE)
-		sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
-	else
+	if (pdev->cfg_size > PCI_CFG_SPACE_SIZE)
 		sysfs_remove_bin_file(&pdev->dev.kobj, &pcie_config_attr);
+	else
+		sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
 
 	pci_remove_resource_files(pdev);
 
@@ -1511,7 +1501,7 @@ static struct attribute *pci_dev_dev_attrs[] = {
 static umode_t pci_dev_attrs_are_visible(struct kobject *kobj,
 					 struct attribute *a, int n)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct pci_dev *pdev = to_pci_dev(dev);
 
 	if (a == &vga_attr.attr)
@@ -1530,7 +1520,7 @@ static struct attribute *pci_dev_hp_attrs[] = {
 static umode_t pci_dev_hp_attrs_are_visible(struct kobject *kobj,
 					    struct attribute *a, int n)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct pci_dev *pdev = to_pci_dev(dev);
 
 	if (pdev->is_virtfn)
@@ -1554,7 +1544,7 @@ static struct attribute *sriov_dev_attrs[] = {
 static umode_t sriov_attrs_are_visible(struct kobject *kobj,
 				       struct attribute *a, int n)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 
 	if (!dev_is_pf(dev))
 		return 0;
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index d1a7105b9276..602eb4223510 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1417,7 +1417,7 @@ struct pci_devres {
 
 static void pcim_release(struct device *gendev, void *res)
 {
-	struct pci_dev *dev = container_of(gendev, struct pci_dev, dev);
+	struct pci_dev *dev = to_pci_dev(gendev);
 	struct pci_devres *this = res;
 	int i;
 
@@ -1534,7 +1534,7 @@ void __weak pcibios_release_device(struct pci_dev *dev) {}
  * is the default implementation. Architecture implementations can
  * override this.
  */
-void __weak pcibios_disable_device (struct pci_dev *dev) {}
+void __weak pcibios_disable_device(struct pci_dev *dev) {}
 
 /**
  * pcibios_penalize_isa_irq - penalize an ISA IRQ
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index f6f151a42147..9a1660f592ef 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -144,10 +144,8 @@ extern unsigned int pci_pm_d3_delay;
 
 #ifdef CONFIG_PCI_MSI
 void pci_no_msi(void);
-void pci_msi_init_pci_dev(struct pci_dev *dev);
 #else
 static inline void pci_no_msi(void) { }
-static inline void pci_msi_init_pci_dev(struct pci_dev *dev) { }
 #endif
 
 static inline void pci_msi_set_enable(struct pci_dev *dev, int enable)
diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
index 182224acedbe..20db790465dd 100644
--- a/drivers/pci/pcie/aer/aer_inject.c
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -41,12 +41,12 @@ struct aer_error_inj {
 	u32 header_log1;
 	u32 header_log2;
 	u32 header_log3;
-	u16 domain;
+	u32 domain;
 };
 
 struct aer_error {
 	struct list_head list;
-	u16 domain;
+	u32 domain;
 	unsigned int bus;
 	unsigned int devfn;
 	int pos_cap_err;
@@ -74,7 +74,7 @@ static LIST_HEAD(pci_bus_ops_list);
 /* Protect einjected and pci_bus_ops_list */
 static DEFINE_SPINLOCK(inject_lock);
 
-static void aer_error_init(struct aer_error *err, u16 domain,
+static void aer_error_init(struct aer_error *err, u32 domain,
 			   unsigned int bus, unsigned int devfn,
 			   int pos_cap_err)
 {
@@ -86,7 +86,7 @@ static void aer_error_init(struct aer_error *err, u16 domain,
 }
 
 /* inject_lock must be held before calling */
-static struct aer_error *__find_aer_error(u16 domain, unsigned int bus,
+static struct aer_error *__find_aer_error(u32 domain, unsigned int bus,
 					  unsigned int devfn)
 {
 	struct aer_error *err;
@@ -106,7 +106,7 @@ static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
 	int domain = pci_domain_nr(dev->bus);
 	if (domain < 0)
 		return NULL;
-	return __find_aer_error((u16)domain, dev->bus->number, dev->devfn);
+	return __find_aer_error(domain, dev->bus->number, dev->devfn);
 }
 
 /* inject_lock must be held before calling */
@@ -196,7 +196,7 @@ static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where,
 	domain = pci_domain_nr(bus);
 	if (domain < 0)
 		goto out;
-	err = __find_aer_error((u16)domain, bus->number, devfn);
+	err = __find_aer_error(domain, bus->number, devfn);
 	if (!err)
 		goto out;
 
@@ -228,7 +228,7 @@ static int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where,
 	domain = pci_domain_nr(bus);
 	if (domain < 0)
 		goto out;
-	err = __find_aer_error((u16)domain, bus->number, devfn);
+	err = __find_aer_error(domain, bus->number, devfn);
 	if (!err)
 		goto out;
 
@@ -329,7 +329,7 @@ static int aer_inject(struct aer_error_inj *einj)
 	u32 sever, cor_mask, uncor_mask, cor_mask_orig = 0, uncor_mask_orig = 0;
 	int ret = 0;
 
-	dev = pci_get_domain_bus_and_slot((int)einj->domain, einj->bus, devfn);
+	dev = pci_get_domain_bus_and_slot(einj->domain, einj->bus, devfn);
 	if (!dev)
 		return -ENODEV;
 	rpdev = pcie_find_root_port(dev);
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index fba785e9df75..712392504ed9 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -246,7 +246,7 @@ static int report_error_detected(struct pci_dev *dev, void *data)
 		!dev->driver->err_handler ||
 		!dev->driver->err_handler->error_detected) {
 		if (result_data->state == pci_channel_io_frozen &&
-			!(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) {
+			dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) {
 			/*
 			 * In case of fatal recovery, if one of down-
 			 * stream device has no driver. We might be
@@ -269,7 +269,7 @@ static int report_error_detected(struct pci_dev *dev, void *data)
 		 * without recovery.
 		 */
 
-		if (!(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE))
+		if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
 			vote = PCI_ERS_RESULT_NO_AER_DRIVER;
 		else
 			vote = PCI_ERS_RESULT_NONE;
@@ -369,7 +369,7 @@ static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
 	else
 		result_data.result = PCI_ERS_RESULT_RECOVERED;
 
-	if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
+	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
 		/*
 		 * If the error is reported by a bridge, we think this error
 		 * is related to the downstream link of the bridge, so we
@@ -440,7 +440,7 @@ static pci_ers_result_t reset_link(struct pci_dev *dev)
 	pci_ers_result_t status;
 	struct pcie_port_service_driver *driver;
 
-	if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
+	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
 		/* Reset this port for all subordinates */
 		udev = dev;
 	} else {
@@ -660,7 +660,7 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 			&info->mask);
 		if (!(info->status & ~info->mask))
 			return 0;
-	} else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE ||
+	} else if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE ||
 		info->severity == AER_NONFATAL) {
 
 		/* Link is still healthy for IO reads */
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 317e3558a35e..2dfe7fdb77e7 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -834,21 +834,15 @@ static ssize_t link_state_store(struct device *dev,
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct pcie_link_state *link, *root = pdev->link_state->root;
-	u32 val, state = 0;
-
-	if (kstrtouint(buf, 10, &val))
-		return -EINVAL;
+	u32 state;
 
 	if (aspm_disabled)
 		return -EPERM;
-	if (n < 1 || val > 3)
-		return -EINVAL;
 
-	/* Convert requested state to ASPM state */
-	if (val & PCIE_LINK_STATE_L0S)
-		state |= ASPM_STATE_L0S;
-	if (val & PCIE_LINK_STATE_L1)
-		state |= ASPM_STATE_L1;
+	if (kstrtouint(buf, 10, &state))
+		return -EINVAL;
+	if ((state & ~ASPM_STATE_ALL) != 0)
+		return -EINVAL;
 
 	down_read(&pci_bus_sem);
 	mutex_lock(&aspm_lock);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 553a029e37f1..6d7ab9bb0d5a 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -1109,14 +1109,11 @@ static int pci_cfg_space_size_ext(struct pci_dev *dev)
 	int pos = PCI_CFG_SPACE_SIZE;
 
 	if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
-		goto fail;
+		return PCI_CFG_SPACE_SIZE;
 	if (status == 0xffffffff || pci_ext_cfg_is_aliased(dev))
-		goto fail;
+		return PCI_CFG_SPACE_SIZE;
 
 	return PCI_CFG_SPACE_EXP_SIZE;
-
- fail:
-	return PCI_CFG_SPACE_SIZE;
 }
 
 int pci_cfg_space_size(struct pci_dev *dev)
@@ -1129,25 +1126,23 @@ int pci_cfg_space_size(struct pci_dev *dev)
 	if (class == PCI_CLASS_BRIDGE_HOST)
 		return pci_cfg_space_size_ext(dev);
 
-	if (!pci_is_pcie(dev)) {
-		pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
-		if (!pos)
-			goto fail;
+	if (pci_is_pcie(dev))
+		return pci_cfg_space_size_ext(dev);
 
-		pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
-		if (!(status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ)))
-			goto fail;
-	}
+	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
+	if (!pos)
+		return PCI_CFG_SPACE_SIZE;
 
-	return pci_cfg_space_size_ext(dev);
+	pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
+	if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
+		return pci_cfg_space_size_ext(dev);
 
- fail:
 	return PCI_CFG_SPACE_SIZE;
 }
 
 #define LEGACY_IO_RESOURCE	(IORESOURCE_IO | IORESOURCE_PCI_FIXED)
 
-void pci_msi_setup_pci_dev(struct pci_dev *dev)
+static void pci_msi_setup_pci_dev(struct pci_dev *dev)
 {
 	/*
 	 * Disable the MSI hardware to avoid screaming interrupts
@@ -1214,8 +1209,6 @@ int pci_setup_device(struct pci_dev *dev)
 	/* "Unknown power state" */
 	dev->current_state = PCI_UNKNOWN;
 
-	pci_msi_setup_pci_dev(dev);
-
 	/* Early fixups, before probing the BARs */
 	pci_fixup_device(pci_fixup_early, dev);
 	/* device class may be changed after fixup */
@@ -1605,8 +1598,8 @@ static void pci_init_capabilities(struct pci_dev *dev)
 	/* Enhanced Allocation */
 	pci_ea_init(dev);
 
-	/* MSI/MSI-X list */
-	pci_msi_init_pci_dev(dev);
+	/* Setup MSI caps & disable MSI/MSI-X interrupts */
+	pci_msi_setup_pci_dev(dev);
 
 	/* Buffers for saving PCIe and PCI-X capabilities */
 	pci_allocate_cap_save_buffers(dev);
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 7e327309cf69..0575a1e026b4 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -287,6 +287,18 @@ static void quirk_citrine(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_IBM,	PCI_DEVICE_ID_IBM_CITRINE,	quirk_citrine);
 
+/*
+ * This chip can cause bus lockups if config addresses above 0x600
+ * are read or written.
+ */
+static void quirk_nfp6000(struct pci_dev *dev)
+{
+	dev->cfg_size = 0x600;
+}
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NETRONOME,	PCI_DEVICE_ID_NETRONOME_NFP4000,	quirk_nfp6000);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NETRONOME,	PCI_DEVICE_ID_NETRONOME_NFP6000,	quirk_nfp6000);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NETRONOME,	PCI_DEVICE_ID_NETRONOME_NFP6000_VF,	quirk_nfp6000);
+
 /*  On IBM Crocodile ipr SAS adapters, expand BAR to system page size */
 static void quirk_extend_bar_to_page(struct pci_dev *dev)
 {
@@ -3405,7 +3417,9 @@ static int reset_intel_82599_sfp_virtfn(struct pci_dev *dev, int probe)
 	return 0;
 }
 
-#include "../gpu/drm/i915/i915_reg.h"
+#define SOUTH_CHICKEN2		0xc2004
+#define PCH_PP_STATUS		0xc7200
+#define PCH_PP_CONTROL		0xc7204
 #define MSG_CTL			0x45010
 #define NSDE_PWR_STATE		0xd0100
 #define IGD_OPERATION_TIMEOUT	10000     /* set timeout 10 seconds */
@@ -3620,6 +3634,10 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TTI, 0x0642,
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JMICRON,
 			 PCI_DEVICE_ID_JMICRON_JMB388_ESD,
 			 quirk_dma_func1_alias);
+/* https://bugzilla.kernel.org/show_bug.cgi?id=42679#c117 */
+DECLARE_PCI_FIXUP_HEADER(0x1c28, /* Lite-On */
+			 0x0122, /* Plextor M6E (Marvell 88SS9183)*/
+			 quirk_dma_func1_alias);
 
 /*
  * Some devices DMA with the wrong devfn, not just the wrong function.
diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c
index eb0ad530dc43..9eaca39ef38d 100644
--- a/drivers/pci/rom.c
+++ b/drivers/pci/rom.c
@@ -77,25 +77,24 @@ size_t pci_get_rom_size(struct pci_dev *pdev, void __iomem *rom, size_t size)
 	do {
 		void __iomem *pds;
 		/* Standard PCI ROMs start out with these bytes 55 AA */
-		if (readb(image) != 0x55) {
-			dev_err(&pdev->dev, "Invalid ROM contents\n");
+		if (readw(image) != 0xAA55) {
+			dev_err(&pdev->dev, "Invalid PCI ROM header signature: expecting 0xaa55, got %#06x\n",
+				readw(image));
 			break;
 		}
-		if (readb(image + 1) != 0xAA)
-			break;
-		/* get the PCI data structure and check its signature */
+		/* get the PCI data structure and check its "PCIR" signature */
 		pds = image + readw(image + 24);
-		if (readb(pds) != 'P')
-			break;
-		if (readb(pds + 1) != 'C')
-			break;
-		if (readb(pds + 2) != 'I')
-			break;
-		if (readb(pds + 3) != 'R')
+		if (readl(pds) != 0x52494350) {
+			dev_err(&pdev->dev, "Invalid PCI ROM data signature: expecting 0x52494350, got %#010x\n",
+				readl(pds));
 			break;
+		}
 		last_image = readb(pds + 21) & 0x80;
 		length = readw(pds + 16);
 		image += length * 512;
+		/* Avoid iterating through memory outside the resource window */
+		if (image > rom + size)
+			break;
 	} while (length && !last_image);
 
 	/* never return a size larger than the PCI resource window */
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index 1723ac1b30e1..7796d0a5befa 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -442,7 +442,7 @@ static void __assign_resources_sorted(struct list_head *head,
 					break;
 				}
 			}
-               }
+		}
 
 	}
 
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index 75b0d8c8f058..0f5997ceb494 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -23,7 +23,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
@@ -337,14 +337,14 @@ static int bcm2835_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int bcm2835_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct bcm2835_pinctrl *pc = dev_get_drvdata(chip->dev);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 
 	return bcm2835_gpio_get_bit(pc, GPLEV0, offset);
 }
 
 static void bcm2835_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct bcm2835_pinctrl *pc = dev_get_drvdata(chip->dev);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 
 	bcm2835_gpio_set_bit(pc, value ? GPSET0 : GPCLR0, offset);
 }
@@ -358,7 +358,7 @@ static int bcm2835_gpio_direction_output(struct gpio_chip *chip,
 
 static int bcm2835_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct bcm2835_pinctrl *pc = dev_get_drvdata(chip->dev);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 
 	return irq_linear_revmap(pc->irq_domain, offset);
 }
@@ -964,7 +964,7 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 		return PTR_ERR(pc->base);
 
 	pc->gpio_chip = bcm2835_gpio_chip;
-	pc->gpio_chip.dev = dev;
+	pc->gpio_chip.parent = dev;
 	pc->gpio_chip.of_node = np;
 
 	pc->irq_domain = irq_domain_add_linear(np, BCM2835_NUM_GPIOS,
@@ -1021,7 +1021,7 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = gpiochip_add(&pc->gpio_chip);
+	err = gpiochip_add_data(&pc->gpio_chip, pc);
 	if (err) {
 		dev_err(dev, "could not add GPIO chip\n");
 		return err;
diff --git a/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c b/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c
index 314591a4609b..3b2ac8f771ed 100644
--- a/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c
+++ b/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c
@@ -694,7 +694,7 @@ static int iproc_gpio_probe(struct platform_device *pdev)
 	gc->ngpio = ngpios;
 	chip->num_banks = (ngpios + NGPIOS_PER_BANK - 1) / NGPIOS_PER_BANK;
 	gc->label = dev_name(dev);
-	gc->dev = dev;
+	gc->parent = dev;
 	gc->of_node = dev->of_node;
 	gc->request = iproc_gpio_request;
 	gc->free = iproc_gpio_free;
diff --git a/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c b/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c
index 725c36f917f9..ac900435dc39 100644
--- a/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c
+++ b/drivers/pinctrl/bcm/pinctrl-nsp-gpio.c
@@ -18,7 +18,7 @@
  * through the interaction with the NSP IOMUX controller.
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
@@ -81,11 +81,6 @@ enum base_type {
 	IO_CTRL
 };
 
-static inline struct nsp_gpio *to_nsp_gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct nsp_gpio, gc);
-}
-
 /*
  * Mapping from PINCONF pins to GPIO pins is 1-to-1
  */
@@ -297,7 +292,7 @@ static void nsp_gpio_free(struct gpio_chip *gc, unsigned offset)
 
 static int nsp_gpio_direction_input(struct gpio_chip *gc, unsigned gpio)
 {
-	struct nsp_gpio *chip = to_nsp_gpio(gc);
+	struct nsp_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -311,7 +306,7 @@ static int nsp_gpio_direction_input(struct gpio_chip *gc, unsigned gpio)
 static int nsp_gpio_direction_output(struct gpio_chip *gc, unsigned gpio,
 				     int val)
 {
-	struct nsp_gpio *chip = to_nsp_gpio(gc);
+	struct nsp_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -325,7 +320,7 @@ static int nsp_gpio_direction_output(struct gpio_chip *gc, unsigned gpio,
 
 static void nsp_gpio_set(struct gpio_chip *gc, unsigned gpio, int val)
 {
-	struct nsp_gpio *chip = to_nsp_gpio(gc);
+	struct nsp_gpio *chip = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->lock, flags);
@@ -337,14 +332,14 @@ static void nsp_gpio_set(struct gpio_chip *gc, unsigned gpio, int val)
 
 static int nsp_gpio_get(struct gpio_chip *gc, unsigned gpio)
 {
-	struct nsp_gpio *chip = to_nsp_gpio(gc);
+	struct nsp_gpio *chip = gpiochip_get_data(gc);
 
 	return !!(readl(chip->base + NSP_GPIO_DATA_IN) & BIT(gpio));
 }
 
 static int nsp_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct nsp_gpio *chip = to_nsp_gpio(gc);
+	struct nsp_gpio *chip = gpiochip_get_data(gc);
 
 	return irq_linear_revmap(chip->irq_domain, offset);
 }
@@ -669,7 +664,7 @@ static int nsp_gpio_probe(struct platform_device *pdev)
 	gc->can_sleep = false;
 	gc->ngpio = val;
 	gc->label = dev_name(dev);
-	gc->dev = dev;
+	gc->parent = dev;
 	gc->of_node = dev->of_node;
 	gc->request = nsp_gpio_request;
 	gc->free = nsp_gpio_free;
@@ -714,7 +709,7 @@ static int nsp_gpio_probe(struct platform_device *pdev)
 		writel(val, (chip->base + NSP_CHIP_A_INT_MASK));
 	}
 
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, chip);
 	if (ret < 0) {
 		dev_err(dev, "unable to add GPIO chip\n");
 		return ret;
diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c
index b59ce75b1947..21b79a446d5a 100644
--- a/drivers/pinctrl/intel/pinctrl-baytrail.c
+++ b/drivers/pinctrl/intel/pinctrl-baytrail.c
@@ -20,7 +20,7 @@
 #include <linux/types.h>
 #include <linux/bitops.h>
 #include <linux/interrupt.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/acpi.h>
 #include <linux/platform_device.h>
 #include <linux/seq_file.h>
@@ -147,12 +147,10 @@ struct byt_gpio {
 	struct byt_gpio_pin_context	*saved_context;
 };
 
-#define to_byt_gpio(c)	container_of(c, struct byt_gpio, chip)
-
 static void __iomem *byt_gpio_reg(struct gpio_chip *chip, unsigned offset,
 				 int reg)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	u32 reg_offset;
 
 	if (reg == BYT_INT_STAT_REG)
@@ -193,7 +191,7 @@ static u32 byt_get_gpio_mux(struct byt_gpio *vg, unsigned offset)
 
 static int byt_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	void __iomem *reg = byt_gpio_reg(chip, offset, BYT_CONF0_REG);
 	u32 value, gpio_mux;
 	unsigned long flags;
@@ -229,7 +227,7 @@ static int byt_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void byt_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 
 	byt_gpio_clear_triggering(vg, offset);
 	pm_runtime_put(&vg->pdev->dev);
@@ -237,7 +235,7 @@ static void byt_gpio_free(struct gpio_chip *chip, unsigned offset)
 
 static int byt_irq_type(struct irq_data *d, unsigned type)
 {
-	struct byt_gpio *vg = to_byt_gpio(irq_data_get_irq_chip_data(d));
+	struct byt_gpio *vg = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	u32 offset = irqd_to_hwirq(d);
 	u32 value;
 	unsigned long flags;
@@ -273,7 +271,7 @@ static int byt_irq_type(struct irq_data *d, unsigned type)
 static int byt_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	void __iomem *reg = byt_gpio_reg(chip, offset, BYT_VAL_REG);
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -281,12 +279,12 @@ static int byt_gpio_get(struct gpio_chip *chip, unsigned offset)
 	val = readl(reg);
 	raw_spin_unlock_irqrestore(&vg->lock, flags);
 
-	return val & BYT_LEVEL;
+	return !!(val & BYT_LEVEL);
 }
 
 static void byt_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	void __iomem *reg = byt_gpio_reg(chip, offset, BYT_VAL_REG);
 	unsigned long flags;
 	u32 old_val;
@@ -305,7 +303,7 @@ static void byt_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int byt_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	void __iomem *reg = byt_gpio_reg(chip, offset, BYT_VAL_REG);
 	unsigned long flags;
 	u32 value;
@@ -324,7 +322,7 @@ static int byt_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int byt_gpio_direction_output(struct gpio_chip *chip,
 				     unsigned gpio, int value)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	void __iomem *conf_reg = byt_gpio_reg(chip, gpio, BYT_CONF0_REG);
 	void __iomem *reg = byt_gpio_reg(chip, gpio, BYT_VAL_REG);
 	unsigned long flags;
@@ -356,7 +354,7 @@ static int byt_gpio_direction_output(struct gpio_chip *chip,
 
 static void byt_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct byt_gpio *vg = to_byt_gpio(chip);
+	struct byt_gpio *vg = gpiochip_get_data(chip);
 	int i;
 	u32 conf0, val, offs;
 
@@ -428,7 +426,7 @@ static void byt_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 static void byt_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct irq_data *data = irq_desc_get_irq_data(desc);
-	struct byt_gpio *vg = to_byt_gpio(irq_desc_get_handler_data(desc));
+	struct byt_gpio *vg = gpiochip_get_data(irq_desc_get_handler_data(desc));
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	u32 base, pin;
 	void __iomem *reg;
@@ -450,7 +448,7 @@ static void byt_gpio_irq_handler(struct irq_desc *desc)
 static void byt_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct byt_gpio *vg = to_byt_gpio(gc);
+	struct byt_gpio *vg = gpiochip_get_data(gc);
 	unsigned offset = irqd_to_hwirq(d);
 	void __iomem *reg;
 
@@ -463,7 +461,7 @@ static void byt_irq_ack(struct irq_data *d)
 static void byt_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct byt_gpio *vg = to_byt_gpio(gc);
+	struct byt_gpio *vg = gpiochip_get_data(gc);
 	unsigned offset = irqd_to_hwirq(d);
 	unsigned long flags;
 	void __iomem *reg;
@@ -498,7 +496,7 @@ static void byt_irq_unmask(struct irq_data *d)
 static void byt_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct byt_gpio *vg = to_byt_gpio(gc);
+	struct byt_gpio *vg = gpiochip_get_data(gc);
 
 	byt_gpio_clear_triggering(vg, irqd_to_hwirq(d));
 }
@@ -598,14 +596,14 @@ static int byt_gpio_probe(struct platform_device *pdev)
 	gc->dbg_show = byt_gpio_dbg_show;
 	gc->base = -1;
 	gc->can_sleep = false;
-	gc->dev = dev;
+	gc->parent = dev;
 
 #ifdef CONFIG_PM_SLEEP
 	vg->saved_context = devm_kcalloc(&pdev->dev, gc->ngpio,
 				       sizeof(*vg->saved_context), GFP_KERNEL);
 #endif
 
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, vg);
 	if (ret) {
 		dev_err(&pdev->dev, "failed adding byt-gpio chip\n");
 		return ret;
diff --git a/drivers/pinctrl/intel/pinctrl-cherryview.c b/drivers/pinctrl/intel/pinctrl-cherryview.c
index 84936bae6e5e..4251e0747a3a 100644
--- a/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -181,8 +181,6 @@ struct chv_pinctrl {
 	struct chv_pin_context *saved_pin_context;
 };
 
-#define gpiochip_to_pinctrl(c) container_of(c, struct chv_pinctrl, chip)
-
 #define ALTERNATE_FUNCTION(p, m, i)		\
 	{					\
 		.pin = (p),			\
@@ -1157,7 +1155,7 @@ static unsigned chv_gpio_offset_to_pin(struct chv_pinctrl *pctrl,
 
 static int chv_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(chip);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(chip);
 	int pin = chv_gpio_offset_to_pin(pctrl, offset);
 	unsigned long flags;
 	u32 ctrl0, cfg;
@@ -1176,7 +1174,7 @@ static int chv_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void chv_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(chip);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned pin = chv_gpio_offset_to_pin(pctrl, offset);
 	unsigned long flags;
 	void __iomem *reg;
@@ -1199,7 +1197,7 @@ static void chv_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int chv_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(chip);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned pin = chv_gpio_offset_to_pin(pctrl, offset);
 	u32 ctrl0, direction;
 	unsigned long flags;
@@ -1240,7 +1238,7 @@ static const struct gpio_chip chv_gpio_chip = {
 static void chv_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(gc);
 	int pin = chv_gpio_offset_to_pin(pctrl, irqd_to_hwirq(d));
 	u32 intr_line;
 
@@ -1257,7 +1255,7 @@ static void chv_gpio_irq_ack(struct irq_data *d)
 static void chv_gpio_irq_mask_unmask(struct irq_data *d, bool mask)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(gc);
 	int pin = chv_gpio_offset_to_pin(pctrl, irqd_to_hwirq(d));
 	u32 value, intr_line;
 	unsigned long flags;
@@ -1302,7 +1300,7 @@ static unsigned chv_gpio_irq_startup(struct irq_data *d)
 	 */
 	if (irqd_get_trigger_type(d) == IRQ_TYPE_NONE) {
 		struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-		struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+		struct chv_pinctrl *pctrl = gpiochip_get_data(gc);
 		unsigned offset = irqd_to_hwirq(d);
 		int pin = chv_gpio_offset_to_pin(pctrl, offset);
 		irq_flow_handler_t handler;
@@ -1334,7 +1332,7 @@ static unsigned chv_gpio_irq_startup(struct irq_data *d)
 static int chv_gpio_irq_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(gc);
 	unsigned offset = irqd_to_hwirq(d);
 	int pin = chv_gpio_offset_to_pin(pctrl, offset);
 	unsigned long flags;
@@ -1407,7 +1405,7 @@ static struct irq_chip chv_gpio_irqchip = {
 static void chv_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct chv_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct chv_pinctrl *pctrl = gpiochip_get_data(gc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	unsigned long pending;
 	u32 intr_line;
@@ -1436,10 +1434,10 @@ static int chv_gpio_probe(struct chv_pinctrl *pctrl, int irq)
 
 	chip->ngpio = pctrl->community->ngpios;
 	chip->label = dev_name(pctrl->dev);
-	chip->dev = pctrl->dev;
+	chip->parent = pctrl->dev;
 	chip->base = -1;
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, pctrl);
 	if (ret) {
 		dev_err(pctrl->dev, "Failed to register gpiochip\n");
 		return ret;
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 26f6b6ffea5b..c0f5586218c4 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -103,7 +103,6 @@ struct intel_pinctrl {
 	struct intel_pinctrl_context context;
 };
 
-#define gpiochip_to_pinctrl(c)	container_of(c, struct intel_pinctrl, chip)
 #define pin_to_padno(c, p)	((p) - (c)->pin_base)
 
 static struct intel_community *intel_get_community(struct intel_pinctrl *pctrl,
@@ -596,7 +595,7 @@ static const struct pinctrl_desc intel_pinctrl_desc = {
 
 static int intel_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(chip);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(chip);
 	void __iomem *reg;
 
 	reg = intel_get_padcfg(pctrl, offset, PADCFG0);
@@ -608,7 +607,7 @@ static int intel_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(chip);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(chip);
 	void __iomem *reg;
 
 	reg = intel_get_padcfg(pctrl, offset, PADCFG0);
@@ -652,7 +651,7 @@ static const struct gpio_chip intel_gpio_chip = {
 static void intel_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct intel_community *community;
 	unsigned pin = irqd_to_hwirq(d);
 
@@ -673,7 +672,7 @@ static void intel_gpio_irq_ack(struct irq_data *d)
 static void intel_gpio_irq_mask_unmask(struct irq_data *d, bool mask)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct intel_community *community;
 	unsigned pin = irqd_to_hwirq(d);
 	unsigned long flags;
@@ -713,7 +712,7 @@ static void intel_gpio_irq_unmask(struct irq_data *d)
 static int intel_gpio_irq_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
 	unsigned pin = irqd_to_hwirq(d);
 	unsigned long flags;
 	void __iomem *reg;
@@ -767,7 +766,7 @@ static int intel_gpio_irq_type(struct irq_data *d, unsigned type)
 static int intel_gpio_irq_wake(struct irq_data *d, unsigned int on)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct intel_pinctrl *pctrl = gpiochip_to_pinctrl(gc);
+	struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct intel_community *community;
 	unsigned pin = irqd_to_hwirq(d);
 	unsigned padno, gpp, gpp_offset;
@@ -872,10 +871,10 @@ static int intel_gpio_probe(struct intel_pinctrl *pctrl, int irq)
 
 	pctrl->chip.ngpio = pctrl->soc->npins;
 	pctrl->chip.label = dev_name(pctrl->dev);
-	pctrl->chip.dev = pctrl->dev;
+	pctrl->chip.parent = pctrl->dev;
 	pctrl->chip.base = -1;
 
-	ret = gpiochip_add(&pctrl->chip);
+	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
 		dev_err(pctrl->dev, "failed to register gpiochip\n");
 		return ret;
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
index e22cbaf9f9cf..16d48a4ed225 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -95,7 +95,7 @@ static void mtk_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
 	unsigned int reg_addr;
 	unsigned int bit;
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
 
 	reg_addr = mtk_get_port(pctl, offset) + pctl->devdata->dout_offset;
 	bit = BIT(offset & 0xf);
@@ -750,7 +750,7 @@ static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 	unsigned int bit;
 	unsigned int read_val = 0;
 
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
 
 	reg_addr =  mtk_get_port(pctl, offset) + pctl->devdata->dir_offset;
 	bit = BIT(offset & 0xf);
@@ -763,7 +763,7 @@ static int mtk_gpio_get(struct gpio_chip *chip, unsigned offset)
 	unsigned int reg_addr;
 	unsigned int bit;
 	unsigned int read_val = 0;
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
 
 	reg_addr = mtk_get_port(pctl, offset) +
 		pctl->devdata->din_offset;
@@ -776,7 +776,7 @@ static int mtk_gpio_get(struct gpio_chip *chip, unsigned offset)
 static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
 	const struct mtk_desc_pin *pin;
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
 	int irq;
 
 	pin = pctl->devdata->pins + offset;
@@ -944,7 +944,7 @@ static void mtk_eint_unmask(struct irq_data *d)
 static int mtk_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
 	unsigned debounce)
 {
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->parent);
 	int eint_num, virq, eint_offset;
 	unsigned int set_offset, bit, clr_bit, clr_offset, rst, i, unmask, dbnc;
 	static const unsigned int dbnc_arr[] = {0 , 1, 16, 32, 64, 128, 256};
@@ -1353,10 +1353,10 @@ int mtk_pctrl_init(struct platform_device *pdev,
 	*pctl->chip = mtk_gpio_chip;
 	pctl->chip->ngpio = pctl->devdata->npins;
 	pctl->chip->label = dev_name(&pdev->dev);
-	pctl->chip->dev = &pdev->dev;
+	pctl->chip->parent = &pdev->dev;
 	pctl->chip->base = -1;
 
-	ret = gpiochip_add(pctl->chip);
+	ret = gpiochip_add_data(pctl->chip, pctl);
 	if (ret) {
 		ret = -EINVAL;
 		goto pctrl_error;
diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c
index 84943e4cff09..50cab27c64d4 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.c
+++ b/drivers/pinctrl/meson/pinctrl-meson.c
@@ -448,11 +448,6 @@ static const struct pinconf_ops meson_pinconf_ops = {
 	.is_generic		= true,
 };
 
-static inline struct meson_domain *to_meson_domain(struct gpio_chip *chip)
-{
-	return container_of(chip, struct meson_domain, chip);
-}
-
 static int meson_gpio_request(struct gpio_chip *chip, unsigned gpio)
 {
 	return pinctrl_request_gpio(chip->base + gpio);
@@ -460,14 +455,14 @@ static int meson_gpio_request(struct gpio_chip *chip, unsigned gpio)
 
 static void meson_gpio_free(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = to_meson_domain(chip);
+	struct meson_domain *domain = gpiochip_get_data(chip);
 
 	pinctrl_free_gpio(domain->data->pin_base + gpio);
 }
 
 static int meson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = to_meson_domain(chip);
+	struct meson_domain *domain = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
@@ -485,7 +480,7 @@ static int meson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 static int meson_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 				       int value)
 {
-	struct meson_domain *domain = to_meson_domain(chip);
+	struct meson_domain *domain = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
@@ -507,7 +502,7 @@ static int meson_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 
 static void meson_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 {
-	struct meson_domain *domain = to_meson_domain(chip);
+	struct meson_domain *domain = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
@@ -524,7 +519,7 @@ static void meson_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 
 static int meson_gpio_get(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = to_meson_domain(chip);
+	struct meson_domain *domain = gpiochip_get_data(chip);
 	unsigned int reg, bit, val, pin;
 	struct meson_bank *bank;
 	int ret;
@@ -562,7 +557,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc)
 		domain = &pc->domains[i];
 
 		domain->chip.label = domain->data->name;
-		domain->chip.dev = pc->dev;
+		domain->chip.parent = pc->dev;
 		domain->chip.request = meson_gpio_request;
 		domain->chip.free = meson_gpio_free;
 		domain->chip.direction_input = meson_gpio_direction_input;
@@ -575,7 +570,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc)
 		domain->chip.of_node = domain->of_node;
 		domain->chip.of_gpio_n_cells = 2;
 
-		ret = gpiochip_add(&domain->chip);
+		ret = gpiochip_add_data(&domain->chip, domain);
 		if (ret) {
 			dev_err(pc->dev, "can't add gpio chip %s\n",
 				domain->data->name);
diff --git a/drivers/pinctrl/nomadik/pinctrl-abx500.c b/drivers/pinctrl/nomadik/pinctrl-abx500.c
index b59fbb4b1fb1..085e60106ec2 100644
--- a/drivers/pinctrl/nomadik/pinctrl-abx500.c
+++ b/drivers/pinctrl/nomadik/pinctrl-abx500.c
@@ -109,19 +109,10 @@ struct abx500_pinctrl {
 	int irq_cluster_size;
 };
 
-/**
- * to_abx500_pinctrl() - get the pointer to abx500_pinctrl
- * @chip:	Member of the structure abx500_pinctrl
- */
-static inline struct abx500_pinctrl *to_abx500_pinctrl(struct gpio_chip *chip)
-{
-	return container_of(chip, struct abx500_pinctrl, chip);
-}
-
 static int abx500_gpio_get_bit(struct gpio_chip *chip, u8 reg,
 			       unsigned offset, bool *bit)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	u8 pos = offset % 8;
 	u8 val;
 	int ret;
@@ -143,7 +134,7 @@ static int abx500_gpio_get_bit(struct gpio_chip *chip, u8 reg,
 static int abx500_gpio_set_bits(struct gpio_chip *chip, u8 reg,
 				unsigned offset, int val)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	u8 pos = offset % 8;
 	int ret;
 
@@ -164,7 +155,7 @@ static int abx500_gpio_set_bits(struct gpio_chip *chip, u8 reg,
  */
 static int abx500_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	bool bit;
 	bool is_out;
 	u8 gpio_offset = offset - 1;
@@ -192,7 +183,7 @@ out:
 
 static void abx500_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	int ret;
 
 	ret = abx500_gpio_set_bits(chip, AB8500_GPIO_OUT1_REG, offset, val);
@@ -272,7 +263,7 @@ out:
 
 static bool abx500_pullud_supported(struct gpio_chip *chip, unsigned gpio)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	struct pullud *pullud = pct->soc->pullud;
 
 	return (pullud &&
@@ -284,7 +275,7 @@ static int abx500_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset,
 					int val)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	unsigned gpio;
 	int ret;
 
@@ -332,7 +323,7 @@ static int abx500_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int abx500_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	/* The AB8500 GPIO numbers are off by one */
 	int gpio = offset + 1;
 	int hwirq;
@@ -634,7 +625,7 @@ static void abx500_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
 	unsigned i;
 	unsigned gpio = chip->base;
-	struct abx500_pinctrl *pct = to_abx500_pinctrl(chip);
+	struct abx500_pinctrl *pct = gpiochip_get_data(chip);
 	struct pinctrl_dev *pctldev = pct->pctldev;
 
 	for (i = 0; i < chip->ngpio; i++, gpio++) {
@@ -986,7 +977,7 @@ static int abx500_pin_config_set(struct pinctrl_dev *pctldev,
 		param = pinconf_to_config_param(configs[i]);
 		argument = pinconf_to_config_argument(configs[i]);
 
-		dev_dbg(chip->dev, "pin %d [%#lx]: %s %s\n",
+		dev_dbg(chip->parent, "pin %d [%#lx]: %s %s\n",
 			pin, configs[i],
 			(param == PIN_CONFIG_OUTPUT) ? "output " : "input",
 			(param == PIN_CONFIG_OUTPUT) ?
@@ -1077,7 +1068,8 @@ static int abx500_pin_config_set(struct pinctrl_dev *pctldev,
 			break;
 
 		default:
-			dev_err(chip->dev, "illegal configuration requested\n");
+			dev_err(chip->parent,
+				"illegal configuration requested\n");
 		}
 	} /* for each config */
 out:
@@ -1172,7 +1164,7 @@ static int abx500_gpio_probe(struct platform_device *pdev)
 	pct->dev = &pdev->dev;
 	pct->parent = dev_get_drvdata(pdev->dev.parent);
 	pct->chip = abx500gpio_chip;
-	pct->chip.dev = &pdev->dev;
+	pct->chip.parent = &pdev->dev;
 	pct->chip.base = -1; /* Dynamic allocation */
 
 	match = of_match_device(abx500_gpio_match, &pdev->dev);
@@ -1210,7 +1202,7 @@ static int abx500_gpio_probe(struct platform_device *pdev)
 	pct->irq_cluster = pct->soc->gpio_irq_cluster;
 	pct->irq_cluster_size = pct->soc->ngpio_irq_cluster;
 
-	ret = gpiochip_add(&pct->chip);
+	ret = gpiochip_add_data(&pct->chip, pct);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to add gpiochip: %d\n", ret);
 		return ret;
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
index eebfae0c9b7c..352406108fa0 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
@@ -438,7 +438,7 @@ nmk_gpio_disable_lazy_irq(struct nmk_gpio_chip *nmk_chip, unsigned offset)
 			       nmk_chip->addr + NMK_GPIO_FIMSC);
 	}
 
-	dev_dbg(nmk_chip->chip.dev, "%d: clearing interrupt mask\n", gpio);
+	dev_dbg(nmk_chip->chip.parent, "%d: clearing interrupt mask\n", gpio);
 }
 
 static void nmk_write_masked(void __iomem *reg, u32 mask, u32 value)
@@ -646,7 +646,7 @@ static inline int nmk_gpio_get_bitmask(int gpio)
 static void nmk_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
-	struct nmk_gpio_chip *nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 
 	clk_enable(nmk_chip->clk);
 	writel(nmk_gpio_get_bitmask(d->hwirq), nmk_chip->addr + NMK_GPIO_IC);
@@ -863,7 +863,7 @@ static void __nmk_gpio_irq_handler(struct irq_desc *desc, u32 status)
 static void nmk_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *chip = irq_desc_get_handler_data(desc);
-	struct nmk_gpio_chip *nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 	u32 status;
 
 	clk_enable(nmk_chip->clk);
@@ -876,7 +876,7 @@ static void nmk_gpio_irq_handler(struct irq_desc *desc)
 static void nmk_gpio_latent_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *chip = irq_desc_get_handler_data(desc);
-	struct nmk_gpio_chip *nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 	u32 status = nmk_chip->get_latent_status(nmk_chip->bank);
 
 	__nmk_gpio_irq_handler(desc, status);
@@ -886,8 +886,7 @@ static void nmk_gpio_latent_irq_handler(struct irq_desc *desc)
 
 static int nmk_gpio_make_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct nmk_gpio_chip *nmk_chip =
-		container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 
 	clk_enable(nmk_chip->clk);
 
@@ -900,8 +899,7 @@ static int nmk_gpio_make_input(struct gpio_chip *chip, unsigned offset)
 
 static int nmk_gpio_get_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct nmk_gpio_chip *nmk_chip =
-		container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 	u32 bit = 1 << offset;
 	int value;
 
@@ -917,8 +915,7 @@ static int nmk_gpio_get_input(struct gpio_chip *chip, unsigned offset)
 static void nmk_gpio_set_output(struct gpio_chip *chip, unsigned offset,
 				int val)
 {
-	struct nmk_gpio_chip *nmk_chip =
-		container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 
 	clk_enable(nmk_chip->clk);
 
@@ -930,8 +927,7 @@ static void nmk_gpio_set_output(struct gpio_chip *chip, unsigned offset,
 static int nmk_gpio_make_output(struct gpio_chip *chip, unsigned offset,
 				int val)
 {
-	struct nmk_gpio_chip *nmk_chip =
-		container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 
 	clk_enable(nmk_chip->clk);
 
@@ -951,8 +947,7 @@ static void nmk_gpio_dbg_show_one(struct seq_file *s,
 	unsigned offset, unsigned gpio)
 {
 	const char *label = gpiochip_is_requested(chip, offset);
-	struct nmk_gpio_chip *nmk_chip =
-		container_of(chip, struct nmk_gpio_chip, chip);
+	struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
 	int mode;
 	bool is_out;
 	bool data_out;
@@ -1188,7 +1183,7 @@ static struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np,
 	chip->base = id * NMK_GPIO_PER_CHIP;
 	chip->ngpio = NMK_GPIO_PER_CHIP;
 	chip->label = dev_name(&gpio_pdev->dev);
-	chip->dev = &gpio_pdev->dev;
+	chip->parent = &gpio_pdev->dev;
 
 	res = platform_get_resource(gpio_pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, res);
@@ -1278,7 +1273,7 @@ static int nmk_gpio_probe(struct platform_device *dev)
 	clk_disable(nmk_chip->clk);
 	chip->of_node = np;
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, nmk_chip);
 	if (ret)
 		return ret;
 
@@ -1789,7 +1784,7 @@ static int nmk_gpio_request_enable(struct pinctrl_dev *pctldev,
 		return -EINVAL;
 	}
 	chip = range->gc;
-	nmk_chip = container_of(chip, struct nmk_gpio_chip, chip);
+	nmk_chip = gpiochip_get_data(chip);
 
 	dev_dbg(npct->dev, "enable pin %u as GPIO\n", offset);
 
@@ -1890,7 +1885,7 @@ static int nmk_pin_config_set(struct pinctrl_dev *pctldev, unsigned pin,
 			if (slpm_val)
 				val = slpm_val - 1;
 
-			dev_dbg(nmk_chip->chip.dev,
+			dev_dbg(nmk_chip->chip.parent,
 				"pin %d: sleep pull %s, dir %s, val %s\n",
 				pin,
 				slpm_pull ? pullnames[pull] : "same",
@@ -1899,7 +1894,7 @@ static int nmk_pin_config_set(struct pinctrl_dev *pctldev, unsigned pin,
 				slpm_val ? (val ? "high" : "low") : "same");
 		}
 
-		dev_dbg(nmk_chip->chip.dev,
+		dev_dbg(nmk_chip->chip.parent,
 			"pin %d [%#lx]: pull %s, slpm %s (%s%s), lowemi %s\n",
 			pin, cfg, pullnames[pull], slpmnames[slpm],
 			output ? "output " : "input",
diff --git a/drivers/pinctrl/pinctrl-adi2.c b/drivers/pinctrl/pinctrl-adi2.c
index 8e9e8eab59ba..ecb57635a37e 100644
--- a/drivers/pinctrl/pinctrl-adi2.c
+++ b/drivers/pinctrl/pinctrl-adi2.c
@@ -636,7 +636,7 @@ static int adi_pinmux_set(struct pinctrl_dev *pctldev, unsigned func_id,
 		if (range == NULL) /* should not happen */
 			return -ENODEV;
 
-		port = container_of(range->gc, struct gpio_port, chip);
+		port = gpiochip_get_data(range->gc);
 
 		spin_lock_irqsave(&port->lock, flags);
 
@@ -684,7 +684,7 @@ static int adi_pinmux_request_gpio(struct pinctrl_dev *pctldev,
 	unsigned long flags;
 	u8 offset;
 
-	port = container_of(range->gc, struct gpio_port, chip);
+	port = gpiochip_get_data(range->gc);
 	offset = pin_to_offset(range, pin);
 
 	spin_lock_irqsave(&port->lock, flags);
@@ -718,7 +718,7 @@ static int adi_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 	struct gpio_port *port;
 	unsigned long flags;
 
-	port = container_of(chip, struct gpio_port, chip);
+	port = gpiochip_get_data(chip);
 
 	spin_lock_irqsave(&port->lock, flags);
 
@@ -733,7 +733,7 @@ static int adi_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static void adi_gpio_set_value(struct gpio_chip *chip, unsigned offset,
 	int value)
 {
-	struct gpio_port *port = container_of(chip, struct gpio_port, chip);
+	struct gpio_port *port = gpiochip_get_data(chip);
 	struct gpio_port_t *regs = port->regs;
 	unsigned long flags;
 
@@ -750,7 +750,7 @@ static void adi_gpio_set_value(struct gpio_chip *chip, unsigned offset,
 static int adi_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 	int value)
 {
-	struct gpio_port *port = container_of(chip, struct gpio_port, chip);
+	struct gpio_port *port = gpiochip_get_data(chip);
 	struct gpio_port_t *regs = port->regs;
 	unsigned long flags;
 
@@ -770,7 +770,7 @@ static int adi_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static int adi_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_port *port = container_of(chip, struct gpio_port, chip);
+	struct gpio_port *port = gpiochip_get_data(chip);
 	struct gpio_port_t *regs = port->regs;
 	unsigned long flags;
 	int ret;
@@ -786,7 +786,7 @@ static int adi_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 
 static int adi_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct gpio_port *port = container_of(chip, struct gpio_port, chip);
+	struct gpio_port *port = gpiochip_get_data(chip);
 
 	if (port->irq_base >= 0)
 		return irq_find_mapping(port->domain, offset);
@@ -994,7 +994,7 @@ static int adi_gpio_probe(struct platform_device *pdev)
 	port->chip.ngpio		= port->width;
 	gpio = port->chip.base + port->width;
 
-	ret = gpiochip_add(&port->chip);
+	ret = gpiochip_add_data(&port->chip, port);
 	if (ret) {
 		dev_err(&pdev->dev, "Fail to add GPIO chip.\n");
 		goto out_remove_domain;
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index 3318f1d6193c..657449431301 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -35,16 +35,11 @@
 #include "pinctrl-utils.h"
 #include "pinctrl-amd.h"
 
-static inline struct amd_gpio *to_amd_gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct amd_gpio, gc);
-}
-
 static int amd_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
 	unsigned long flags;
 	u32 pin_reg;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
@@ -71,7 +66,7 @@ static int amd_gpio_direction_output(struct gpio_chip *gc, unsigned offset,
 {
 	u32 pin_reg;
 	unsigned long flags;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
@@ -90,7 +85,7 @@ static int amd_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 {
 	u32 pin_reg;
 	unsigned long flags;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
@@ -103,7 +98,7 @@ static void amd_gpio_set_value(struct gpio_chip *gc, unsigned offset, int value)
 {
 	u32 pin_reg;
 	unsigned long flags;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
@@ -122,7 +117,7 @@ static int amd_gpio_set_debounce(struct gpio_chip *gc, unsigned offset,
 	u32 pin_reg;
 	int ret = 0;
 	unsigned long flags;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
@@ -186,7 +181,7 @@ static void amd_gpio_dbg_show(struct seq_file *s, struct gpio_chip *gc)
 	u32 pin_reg;
 	unsigned long flags;
 	unsigned int bank, i, pin_num;
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	char *level_trig;
 	char *active_level;
@@ -327,7 +322,7 @@ static void amd_gpio_irq_enable(struct irq_data *d)
 	u32 pin_reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
@@ -351,7 +346,7 @@ static void amd_gpio_irq_disable(struct irq_data *d)
 	u32 pin_reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
@@ -366,7 +361,7 @@ static void amd_gpio_irq_mask(struct irq_data *d)
 	u32 pin_reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
@@ -380,7 +375,7 @@ static void amd_gpio_irq_unmask(struct irq_data *d)
 	u32 pin_reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
@@ -394,7 +389,7 @@ static void amd_gpio_irq_eoi(struct irq_data *d)
 	u32 reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	reg = readl(gpio_dev->base + WAKE_INT_MASTER_REG);
@@ -409,7 +404,7 @@ static int amd_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	u32 pin_reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
@@ -504,7 +499,7 @@ static void amd_gpio_irq_handler(struct irq_desc *desc)
 	unsigned long flags;
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct amd_gpio *gpio_dev = to_amd_gpio(gc);
+	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	chained_irq_enter(chip, desc);
 	/*enable GPIO interrupt again*/
@@ -778,7 +773,7 @@ static int amd_gpio_probe(struct platform_device *pdev)
 	gpio_dev->gc.base			= 0;
 	gpio_dev->gc.label			= pdev->name;
 	gpio_dev->gc.owner			= THIS_MODULE;
-	gpio_dev->gc.dev			= &pdev->dev;
+	gpio_dev->gc.parent			= &pdev->dev;
 	gpio_dev->gc.ngpio			= TOTAL_NUMBER_OF_PINS;
 #if defined(CONFIG_OF_GPIO)
 	gpio_dev->gc.of_node			= pdev->dev.of_node;
@@ -795,7 +790,7 @@ static int amd_gpio_probe(struct platform_device *pdev)
 		return PTR_ERR(gpio_dev->pctrl);
 	}
 
-	ret = gpiochip_add(&gpio_dev->gc);
+	ret = gpiochip_add_data(&gpio_dev->gc, gpio_dev);
 	if (ret)
 		goto out1;
 
diff --git a/drivers/pinctrl/pinctrl-as3722.c b/drivers/pinctrl/pinctrl-as3722.c
index 56af28b95a44..e844fdc6d3a8 100644
--- a/drivers/pinctrl/pinctrl-as3722.c
+++ b/drivers/pinctrl/pinctrl-as3722.c
@@ -436,14 +436,9 @@ static struct pinctrl_desc as3722_pinctrl_desc = {
 	.owner = THIS_MODULE,
 };
 
-static inline struct as3722_pctrl_info *to_as_pci(struct gpio_chip *chip)
-{
-	return container_of(chip, struct as3722_pctrl_info, gpio_chip);
-}
-
 static int as3722_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct as3722_pctrl_info *as_pci = to_as_pci(chip);
+	struct as3722_pctrl_info *as_pci = gpiochip_get_data(chip);
 	struct as3722 *as3722 = as_pci->as3722;
 	int ret;
 	u32 reg;
@@ -491,7 +486,7 @@ static int as3722_gpio_get(struct gpio_chip *chip, unsigned offset)
 static void as3722_gpio_set(struct gpio_chip *chip, unsigned offset,
 		int value)
 {
-	struct as3722_pctrl_info *as_pci = to_as_pci(chip);
+	struct as3722_pctrl_info *as_pci = gpiochip_get_data(chip);
 	struct as3722 *as3722 = as_pci->as3722;
 	int en_invert;
 	u32 val;
@@ -531,7 +526,7 @@ static int as3722_gpio_direction_output(struct gpio_chip *chip,
 
 static int as3722_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct as3722_pctrl_info *as_pci = to_as_pci(chip);
+	struct as3722_pctrl_info *as_pci = gpiochip_get_data(chip);
 
 	return as3722_irq_get_virq(as_pci->as3722, offset);
 }
@@ -582,9 +577,9 @@ static int as3722_pinctrl_probe(struct platform_device *pdev)
 	}
 
 	as_pci->gpio_chip = as3722_gpio_chip;
-	as_pci->gpio_chip.dev = &pdev->dev;
+	as_pci->gpio_chip.parent = &pdev->dev;
 	as_pci->gpio_chip.of_node = pdev->dev.parent->of_node;
-	ret = gpiochip_add(&as_pci->gpio_chip);
+	ret = gpiochip_add_data(&as_pci->gpio_chip, as_pci);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Couldn't register gpiochip, %d\n", ret);
 		goto fail_chip_add;
diff --git a/drivers/pinctrl/pinctrl-at91-pio4.c b/drivers/pinctrl/pinctrl-at91-pio4.c
index d5bdcebc6aa6..ee69db6ae1c7 100644
--- a/drivers/pinctrl/pinctrl-at91-pio4.c
+++ b/drivers/pinctrl/pinctrl-at91-pio4.c
@@ -15,6 +15,8 @@
  */
 
 #include <linux/clk.h>
+#include <linux/gpio/driver.h>
+/* FIXME: needed for gpio_to_irq(), get rid of this */
 #include <linux/gpio.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -290,7 +292,7 @@ static void atmel_gpio_irq_handler(struct irq_desc *desc)
 
 static int atmel_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct atmel_pioctrl *atmel_pioctrl = dev_get_drvdata(chip->dev);
+	struct atmel_pioctrl *atmel_pioctrl = gpiochip_get_data(chip);
 	struct atmel_pin *pin = atmel_pioctrl->pins[offset];
 	unsigned reg;
 
@@ -305,7 +307,7 @@ static int atmel_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int atmel_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct atmel_pioctrl *atmel_pioctrl = dev_get_drvdata(chip->dev);
+	struct atmel_pioctrl *atmel_pioctrl = gpiochip_get_data(chip);
 	struct atmel_pin *pin = atmel_pioctrl->pins[offset];
 	unsigned reg;
 
@@ -317,7 +319,7 @@ static int atmel_gpio_get(struct gpio_chip *chip, unsigned offset)
 static int atmel_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 				       int value)
 {
-	struct atmel_pioctrl *atmel_pioctrl = dev_get_drvdata(chip->dev);
+	struct atmel_pioctrl *atmel_pioctrl = gpiochip_get_data(chip);
 	struct atmel_pin *pin = atmel_pioctrl->pins[offset];
 	unsigned reg;
 
@@ -336,7 +338,7 @@ static int atmel_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static void atmel_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 {
-	struct atmel_pioctrl *atmel_pioctrl = dev_get_drvdata(chip->dev);
+	struct atmel_pioctrl *atmel_pioctrl = gpiochip_get_data(chip);
 	struct atmel_pin *pin = atmel_pioctrl->pins[offset];
 
 	atmel_gpio_write(atmel_pioctrl, pin->bank,
@@ -346,7 +348,7 @@ static void atmel_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 
 static int atmel_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct atmel_pioctrl *atmel_pioctrl = dev_get_drvdata(chip->dev);
+	struct atmel_pioctrl *atmel_pioctrl = gpiochip_get_data(chip);
 
 	return irq_find_mapping(atmel_pioctrl->irq_domain, offset);
 }
@@ -972,7 +974,7 @@ static int atmel_pinctrl_probe(struct platform_device *pdev)
 	atmel_pioctrl->gpio_chip->of_node = dev->of_node;
 	atmel_pioctrl->gpio_chip->ngpio = atmel_pioctrl->npins;
 	atmel_pioctrl->gpio_chip->label = dev_name(dev);
-	atmel_pioctrl->gpio_chip->dev = dev;
+	atmel_pioctrl->gpio_chip->parent = dev;
 	atmel_pioctrl->gpio_chip->names = atmel_pioctrl->group_names;
 
 	atmel_pioctrl->pm_wakeup_sources = devm_kzalloc(dev,
@@ -1040,7 +1042,7 @@ static int atmel_pinctrl_probe(struct platform_device *pdev)
 		goto pinctrl_register_error;
 	}
 
-	ret = gpiochip_add(atmel_pioctrl->gpio_chip);
+	ret = gpiochip_add_data(atmel_pioctrl->gpio_chip, atmel_pioctrl);
 	if (ret) {
 		dev_err(dev, "failed to add gpiochip\n");
 		goto gpiochip_add_error;
diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c
index 47b625b1b789..523b6b794d1f 100644
--- a/drivers/pinctrl/pinctrl-at91.c
+++ b/drivers/pinctrl/pinctrl-at91.c
@@ -45,8 +45,6 @@ struct at91_gpio_chip {
 	struct at91_pinctrl_mux_ops *ops;	/* ops */
 };
 
-#define to_at91_gpio_chip(c) container_of(c, struct at91_gpio_chip, chip)
-
 static struct at91_gpio_chip *gpio_chips[MAX_GPIO_BANKS];
 
 static int gpio_banks;
@@ -811,7 +809,7 @@ static int at91_gpio_request_enable(struct pinctrl_dev *pctldev,
 		return -EINVAL;
 	}
 	chip = range->gc;
-	at91_chip = container_of(chip, struct at91_gpio_chip, chip);
+	at91_chip = gpiochip_get_data(chip);
 
 	dev_dbg(npct->dev, "enable pin %u as GPIO\n", offset);
 
@@ -1282,7 +1280,7 @@ static int at91_pinctrl_remove(struct platform_device *pdev)
 
 static int at91_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 	unsigned mask = 1 << offset;
 	u32 osr;
@@ -1293,7 +1291,7 @@ static int at91_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 
 static int at91_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 	unsigned mask = 1 << offset;
 
@@ -1303,7 +1301,7 @@ static int at91_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 
 static int at91_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 	unsigned mask = 1 << offset;
 	u32 pdsr;
@@ -1315,7 +1313,7 @@ static int at91_gpio_get(struct gpio_chip *chip, unsigned offset)
 static void at91_gpio_set(struct gpio_chip *chip, unsigned offset,
 				int val)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 	unsigned mask = 1 << offset;
 
@@ -1325,7 +1323,7 @@ static void at91_gpio_set(struct gpio_chip *chip, unsigned offset,
 static void at91_gpio_set_multiple(struct gpio_chip *chip,
 				      unsigned long *mask, unsigned long *bits)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 
 #define BITS_MASK(bits) (((bits) == 32) ? ~0U : (BIT(bits) - 1))
@@ -1340,7 +1338,7 @@ static void at91_gpio_set_multiple(struct gpio_chip *chip,
 static int at91_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 				int val)
 {
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 	unsigned mask = 1 << offset;
 
@@ -1355,7 +1353,7 @@ static void at91_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
 	enum at91_mux mode;
 	int i;
-	struct at91_gpio_chip *at91_gpio = to_at91_gpio_chip(chip);
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(chip);
 	void __iomem *pio = at91_gpio->regbase;
 
 	for (i = 0; i < chip->ngpio; i++) {
@@ -1570,9 +1568,7 @@ static void gpio_irq_handler(struct irq_desc *desc)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct gpio_chip *gpio_chip = irq_desc_get_handler_data(desc);
-	struct at91_gpio_chip *at91_gpio = container_of(gpio_chip,
-					   struct at91_gpio_chip, chip);
-
+	struct at91_gpio_chip *at91_gpio = gpiochip_get_data(gpio_chip);
 	void __iomem	*pio = at91_gpio->regbase;
 	unsigned long	isr;
 	int		n;
@@ -1648,7 +1644,7 @@ static int at91_gpio_of_irq_setup(struct platform_device *pdev,
 		return 0;
 	}
 
-	prev = container_of(gpiochip_prev, struct at91_gpio_chip, chip);
+	prev = gpiochip_get_data(gpiochip_prev);
 
 	/* we can only have 2 banks before */
 	for (i = 0; i < 2; i++) {
@@ -1750,7 +1746,7 @@ static int at91_gpio_probe(struct platform_device *pdev)
 	chip = &at91_chip->chip;
 	chip->of_node = np;
 	chip->label = dev_name(&pdev->dev);
-	chip->dev = &pdev->dev;
+	chip->parent = &pdev->dev;
 	chip->owner = THIS_MODULE;
 	chip->base = alias_idx * MAX_NB_GPIO_PER_BANK;
 
@@ -1783,7 +1779,7 @@ static int at91_gpio_probe(struct platform_device *pdev)
 	range->npins = chip->ngpio;
 	range->gc = chip;
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, at91_chip);
 	if (ret)
 		goto gpiochip_add_err;
 
diff --git a/drivers/pinctrl/pinctrl-coh901.c b/drivers/pinctrl/pinctrl-coh901.c
index 813eb7c771ec..cf7788df0f95 100644
--- a/drivers/pinctrl/pinctrl-coh901.c
+++ b/drivers/pinctrl/pinctrl-coh901.c
@@ -208,25 +208,16 @@ bs335_gpio_config[U300_GPIO_NUM_PORTS][U300_GPIO_PINS_PER_PORT] = {
 	}
 };
 
-/**
- * to_u300_gpio() - get the pointer to u300_gpio
- * @chip: the gpio chip member of the structure u300_gpio
- */
-static inline struct u300_gpio *to_u300_gpio(struct gpio_chip *chip)
-{
-	return container_of(chip, struct u300_gpio, chip);
-}
-
 static int u300_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 
-	return readl(U300_PIN_REG(offset, dir)) & U300_PIN_BIT(offset);
+	return !!(readl(U300_PIN_REG(offset, dir)) & U300_PIN_BIT(offset));
 }
 
 static void u300_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -243,7 +234,7 @@ static void u300_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 
 static int u300_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -259,7 +250,7 @@ static int u300_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int u300_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 				      int value)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 oldmode;
 	u32 val;
@@ -290,7 +281,7 @@ int u300_gpio_config_get(struct gpio_chip *chip,
 			 unsigned offset,
 			 unsigned long *config)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	enum pin_config_param param = (enum pin_config_param) *config;
 	bool biasmode;
 	u32 drmode;
@@ -348,7 +339,7 @@ int u300_gpio_config_get(struct gpio_chip *chip,
 int u300_gpio_config_set(struct gpio_chip *chip, unsigned offset,
 			 enum pin_config_param param)
 {
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -429,7 +420,7 @@ static void u300_toggle_trigger(struct u300_gpio *gpio, unsigned offset)
 static int u300_gpio_irq_type(struct irq_data *d, unsigned trigger)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	struct u300_gpio_port *port = &gpio->ports[d->hwirq >> 3];
 	int offset = d->hwirq;
 	u32 val;
@@ -466,7 +457,7 @@ static int u300_gpio_irq_type(struct irq_data *d, unsigned trigger)
 static void u300_gpio_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	struct u300_gpio_port *port = &gpio->ports[d->hwirq >> 3];
 	int offset = d->hwirq;
 	u32 val;
@@ -483,7 +474,7 @@ static void u300_gpio_irq_enable(struct irq_data *d)
 static void u300_gpio_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	int offset = d->hwirq;
 	u32 val;
 	unsigned long flags;
@@ -506,7 +497,7 @@ static void u300_gpio_irq_handler(struct irq_desc *desc)
 	unsigned int irq = irq_desc_get_irq(desc);
 	struct irq_chip *parent_chip = irq_desc_get_chip(desc);
 	struct gpio_chip *chip = irq_desc_get_handler_data(desc);
-	struct u300_gpio *gpio = to_u300_gpio(chip);
+	struct u300_gpio *gpio = gpiochip_get_data(chip);
 	struct u300_gpio_port *port = &gpio->ports[irq - chip->base];
 	int pinoffset = port->number << 3; /* get the right stride */
 	unsigned long val;
@@ -637,7 +628,7 @@ static int __init u300_gpio_probe(struct platform_device *pdev)
 
 	gpio->chip = u300_gpio_chip;
 	gpio->chip.ngpio = U300_GPIO_NUM_PORTS * U300_GPIO_PINS_PER_PORT;
-	gpio->chip.dev = &pdev->dev;
+	gpio->chip.parent = &pdev->dev;
 	gpio->chip.base = 0;
 	gpio->dev = &pdev->dev;
 
@@ -684,7 +675,7 @@ static int __init u300_gpio_probe(struct platform_device *pdev)
 #ifdef CONFIG_OF_GPIO
 	gpio->chip.of_node = pdev->dev.of_node;
 #endif
-	err = gpiochip_add(&gpio->chip);
+	err = gpiochip_add_data(&gpio->chip, gpio);
 	if (err) {
 		dev_err(gpio->dev, "unable to add gpiochip: %d\n", err);
 		goto err_no_chip;
diff --git a/drivers/pinctrl/pinctrl-digicolor.c b/drivers/pinctrl/pinctrl-digicolor.c
index 38a7799f8257..f1343d6ca823 100644
--- a/drivers/pinctrl/pinctrl-digicolor.c
+++ b/drivers/pinctrl/pinctrl-digicolor.c
@@ -171,7 +171,7 @@ static struct pinmux_ops dc_pmxops = {
 
 static int dc_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
-	struct dc_pinmap *pmap = container_of(chip, struct dc_pinmap, chip);
+	struct dc_pinmap *pmap = gpiochip_get_data(chip);
 	int reg_off = GP_DRIVE0(gpio/PINS_PER_COLLECTION);
 	int bit_off = gpio % PINS_PER_COLLECTION;
 	u8 drive;
@@ -191,7 +191,7 @@ static void dc_gpio_set(struct gpio_chip *chip, unsigned gpio, int value);
 static int dc_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 				    int value)
 {
-	struct dc_pinmap *pmap = container_of(chip, struct dc_pinmap, chip);
+	struct dc_pinmap *pmap = gpiochip_get_data(chip);
 	int reg_off = GP_DRIVE0(gpio/PINS_PER_COLLECTION);
 	int bit_off = gpio % PINS_PER_COLLECTION;
 	u8 drive;
@@ -210,7 +210,7 @@ static int dc_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 
 static int dc_gpio_get(struct gpio_chip *chip, unsigned gpio)
 {
-	struct dc_pinmap *pmap = container_of(chip, struct dc_pinmap, chip);
+	struct dc_pinmap *pmap = gpiochip_get_data(chip);
 	int reg_off = GP_INPUT(gpio/PINS_PER_COLLECTION);
 	int bit_off = gpio % PINS_PER_COLLECTION;
 	u8 input;
@@ -222,7 +222,7 @@ static int dc_gpio_get(struct gpio_chip *chip, unsigned gpio)
 
 static void dc_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 {
-	struct dc_pinmap *pmap = container_of(chip, struct dc_pinmap, chip);
+	struct dc_pinmap *pmap = gpiochip_get_data(chip);
 	int reg_off = GP_OUTPUT0(gpio/PINS_PER_COLLECTION);
 	int bit_off = gpio % PINS_PER_COLLECTION;
 	u8 output;
@@ -244,7 +244,7 @@ static int dc_gpiochip_add(struct dc_pinmap *pmap, struct device_node *np)
 	int ret;
 
 	chip->label		= DRIVER_NAME;
-	chip->dev		= pmap->dev;
+	chip->parent		= pmap->dev;
 	chip->request		= gpiochip_generic_request;
 	chip->free		= gpiochip_generic_free;
 	chip->direction_input	= dc_gpio_direction_input;
@@ -258,7 +258,7 @@ static int dc_gpiochip_add(struct dc_pinmap *pmap, struct device_node *np)
 
 	spin_lock_init(&pmap->lock);
 
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, pmap);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c
index 85c9046c690e..856f736cb1a6 100644
--- a/drivers/pinctrl/pinctrl-pistachio.c
+++ b/drivers/pinctrl/pinctrl-pistachio.c
@@ -842,14 +842,9 @@ static inline void pctl_writel(struct pistachio_pinctrl *pctl, u32 val, u32 reg)
 	writel(val, pctl->base + reg);
 }
 
-static inline struct pistachio_gpio_bank *gc_to_bank(struct gpio_chip *gc)
-{
-	return container_of(gc, struct pistachio_gpio_bank, gpio_chip);
-}
-
 static inline struct pistachio_gpio_bank *irqd_to_bank(struct irq_data *d)
 {
-	return gc_to_bank(irq_data_get_irq_chip_data(d));
+	return gpiochip_get_data(irq_data_get_irq_chip_data(d));
 }
 
 static inline u32 gpio_readl(struct pistachio_gpio_bank *bank, u32 reg)
@@ -992,7 +987,7 @@ static int pistachio_pinmux_enable(struct pinctrl_dev *pctldev,
 
 	range = pinctrl_find_gpio_range_from_pin(pctl->pctldev, pg->pin);
 	if (range)
-		gpio_disable(gc_to_bank(range->gc), pg->pin - range->pin_base);
+		gpio_disable(gpiochip_get_data(range->gc), pg->pin - range->pin_base);
 
 	return 0;
 }
@@ -1173,14 +1168,14 @@ static struct pinctrl_desc pistachio_pinctrl_desc = {
 
 static int pistachio_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct pistachio_gpio_bank *bank = gc_to_bank(chip);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(chip);
 
 	return !(gpio_readl(bank, GPIO_OUTPUT_EN) & BIT(offset));
 }
 
 static int pistachio_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct pistachio_gpio_bank *bank = gc_to_bank(chip);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(chip);
 	u32 reg;
 
 	if (gpio_readl(bank, GPIO_OUTPUT_EN) & BIT(offset))
@@ -1194,7 +1189,7 @@ static int pistachio_gpio_get(struct gpio_chip *chip, unsigned offset)
 static void pistachio_gpio_set(struct gpio_chip *chip, unsigned offset,
 			       int value)
 {
-	struct pistachio_gpio_bank *bank = gc_to_bank(chip);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(chip);
 
 	gpio_mask_writel(bank, GPIO_OUTPUT, offset, !!value);
 }
@@ -1202,7 +1197,7 @@ static void pistachio_gpio_set(struct gpio_chip *chip, unsigned offset,
 static int pistachio_gpio_direction_input(struct gpio_chip *chip,
 					  unsigned offset)
 {
-	struct pistachio_gpio_bank *bank = gc_to_bank(chip);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(chip);
 
 	gpio_mask_writel(bank, GPIO_OUTPUT_EN, offset, 0);
 	gpio_enable(bank, offset);
@@ -1213,7 +1208,7 @@ static int pistachio_gpio_direction_input(struct gpio_chip *chip,
 static int pistachio_gpio_direction_output(struct gpio_chip *chip,
 					   unsigned offset, int value)
 {
-	struct pistachio_gpio_bank *bank = gc_to_bank(chip);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(chip);
 
 	pistachio_gpio_set(chip, offset, value);
 	gpio_mask_writel(bank, GPIO_OUTPUT_EN, offset, 1);
@@ -1303,7 +1298,7 @@ static int pistachio_gpio_irq_set_type(struct irq_data *data, unsigned int type)
 static void pistachio_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct pistachio_gpio_bank *bank = gc_to_bank(gc);
+	struct pistachio_gpio_bank *bank = gpiochip_get_data(gc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	unsigned long pending;
 	unsigned int pin;
@@ -1388,9 +1383,9 @@ static int pistachio_gpio_register(struct pistachio_pinctrl *pctl)
 		bank->pctl = pctl;
 		bank->base = pctl->base + GPIO_BANK_BASE(i);
 
-		bank->gpio_chip.dev = pctl->dev;
+		bank->gpio_chip.parent = pctl->dev;
 		bank->gpio_chip.of_node = child;
-		ret = gpiochip_add(&bank->gpio_chip);
+		ret = gpiochip_add_data(&bank->gpio_chip, bank);
 		if (ret < 0) {
 			dev_err(pctl->dev, "Failed to add GPIO chip %u: %d\n",
 				i, ret);
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 91288265e856..183545a068ad 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -224,11 +224,6 @@ static struct regmap_config rockchip_regmap_config = {
 	.reg_stride = 4,
 };
 
-static inline struct rockchip_pin_bank *gc_to_pin_bank(struct gpio_chip *gc)
-{
-	return container_of(gc, struct rockchip_pin_bank, gpio_chip);
-}
-
 static const inline struct rockchip_pin_group *pinctrl_name_to_group(
 					const struct rockchip_pinctrl *info,
 					const char *name)
@@ -973,7 +968,7 @@ static int _rockchip_pmx_gpio_set_direction(struct gpio_chip *chip,
 	unsigned long flags;
 	u32 data;
 
-	bank = gc_to_pin_bank(chip);
+	bank = gpiochip_get_data(chip);
 
 	ret = rockchip_set_mux(bank, pin, RK_FUNC_GPIO);
 	if (ret < 0)
@@ -1413,7 +1408,7 @@ static int rockchip_pinctrl_register(struct platform_device *pdev,
 
 static void rockchip_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct rockchip_pin_bank *bank = gc_to_pin_bank(gc);
+	struct rockchip_pin_bank *bank = gpiochip_get_data(gc);
 	void __iomem *reg = bank->reg_base + GPIO_SWPORT_DR;
 	unsigned long flags;
 	u32 data;
@@ -1437,7 +1432,7 @@ static void rockchip_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
  */
 static int rockchip_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct rockchip_pin_bank *bank = gc_to_pin_bank(gc);
+	struct rockchip_pin_bank *bank = gpiochip_get_data(gc);
 	u32 data;
 
 	clk_enable(bank->clk);
@@ -1476,7 +1471,7 @@ static int rockchip_gpio_direction_output(struct gpio_chip *gc,
  */
 static int rockchip_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct rockchip_pin_bank *bank = gc_to_pin_bank(gc);
+	struct rockchip_pin_bank *bank = gpiochip_get_data(gc);
 	unsigned int virq;
 
 	if (!bank->domain)
@@ -1791,11 +1786,11 @@ static int rockchip_gpiolib_register(struct platform_device *pdev,
 		gc = &bank->gpio_chip;
 		gc->base = bank->pin_base;
 		gc->ngpio = bank->nr_pins;
-		gc->dev = &pdev->dev;
+		gc->parent = &pdev->dev;
 		gc->of_node = bank->of_node;
 		gc->label = bank->name;
 
-		ret = gpiochip_add(gc);
+		ret = gpiochip_add_data(gc, bank);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to register gpio_chip %s, error code: %d\n",
 							gc->label, ret);
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index b58d3f29148a..fac844a85cb4 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -203,9 +203,6 @@
 #define gpio_range_to_bank(chip) \
 		container_of(chip, struct st_gpio_bank, range)
 
-#define gpio_chip_to_bank(chip) \
-		container_of(chip, struct st_gpio_bank, gpio_chip)
-
 #define pc_to_bank(pc) \
 		container_of(pc, struct st_gpio_bank, pc)
 
@@ -744,14 +741,14 @@ static void st_gpio_direction(struct st_gpio_bank *bank,
 
 static int st_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct st_gpio_bank *bank = gpio_chip_to_bank(chip);
+	struct st_gpio_bank *bank = gpiochip_get_data(chip);
 
 	return !!(readl(bank->base + REG_PIO_PIN) & BIT(offset));
 }
 
 static void st_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct st_gpio_bank *bank = gpio_chip_to_bank(chip);
+	struct st_gpio_bank *bank = gpiochip_get_data(chip);
 	__st_gpio_set(bank, offset, value);
 }
 
@@ -765,7 +762,7 @@ static int st_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int st_gpio_direction_output(struct gpio_chip *chip,
 	unsigned offset, int value)
 {
-	struct st_gpio_bank *bank = gpio_chip_to_bank(chip);
+	struct st_gpio_bank *bank = gpiochip_get_data(chip);
 
 	__st_gpio_set(bank, offset, value);
 	pinctrl_gpio_direction_output(chip->base + offset);
@@ -775,7 +772,7 @@ static int st_gpio_direction_output(struct gpio_chip *chip,
 
 static int st_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct st_gpio_bank *bank = gpio_chip_to_bank(chip);
+	struct st_gpio_bank *bank = gpiochip_get_data(chip);
 	struct st_pio_control pc = bank->pc;
 	unsigned long config;
 	unsigned int direction = 0;
@@ -1325,7 +1322,7 @@ static int st_pctl_parse_functions(struct device_node *np,
 static void st_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct st_gpio_bank *bank = gpio_chip_to_bank(gc);
+	struct st_gpio_bank *bank = gpiochip_get_data(gc);
 
 	writel(BIT(d->hwirq), bank->base + REG_PIO_CLR_PMASK);
 }
@@ -1333,7 +1330,7 @@ static void st_gpio_irq_mask(struct irq_data *d)
 static void st_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct st_gpio_bank *bank = gpio_chip_to_bank(gc);
+	struct st_gpio_bank *bank = gpiochip_get_data(gc);
 
 	writel(BIT(d->hwirq), bank->base + REG_PIO_SET_PMASK);
 }
@@ -1341,7 +1338,7 @@ static void st_gpio_irq_unmask(struct irq_data *d)
 static int st_gpio_irq_set_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct st_gpio_bank *bank = gpio_chip_to_bank(gc);
+	struct st_gpio_bank *bank = gpiochip_get_data(gc);
 	unsigned long flags;
 	int comp, pin = d->hwirq;
 	u32 val;
@@ -1455,7 +1452,7 @@ static void st_gpio_irq_handler(struct irq_desc *desc)
 	/* interrupt dedicated per bank */
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct st_gpio_bank *bank = gpio_chip_to_bank(gc);
+	struct st_gpio_bank *bank = gpiochip_get_data(gc);
 
 	chained_irq_enter(chip, desc);
 	__gpio_irq_handler(bank);
@@ -1522,7 +1519,7 @@ static int st_gpiolib_register_bank(struct st_pinctrl *info,
 	bank->gpio_chip.base = bank_num * ST_GPIO_PINS_PER_BANK;
 	bank->gpio_chip.ngpio = ST_GPIO_PINS_PER_BANK;
 	bank->gpio_chip.of_node = np;
-	bank->gpio_chip.dev = dev;
+	bank->gpio_chip.parent = dev;
 	spin_lock_init(&bank->lock);
 
 	of_property_read_string(np, "st,bank-name", &range->name);
@@ -1532,7 +1529,7 @@ static int st_gpiolib_register_bank(struct st_pinctrl *info,
 	range->pin_base = range->base = range->id * ST_GPIO_PINS_PER_BANK;
 	range->npins = bank->gpio_chip.ngpio;
 	range->gc = &bank->gpio_chip;
-	err  = gpiochip_add(&bank->gpio_chip);
+	err  = gpiochip_add_data(&bank->gpio_chip, bank);
 	if (err) {
 		dev_err(dev, "Failed to add gpiochip(%d)!\n", bank_num);
 		return err;
diff --git a/drivers/pinctrl/pinctrl-xway.c b/drivers/pinctrl/pinctrl-xway.c
index 7db74699fda4..412c6b78140a 100644
--- a/drivers/pinctrl/pinctrl-xway.c
+++ b/drivers/pinctrl/pinctrl-xway.c
@@ -1535,7 +1535,7 @@ static struct ltq_pinmux_info xway_info = {
 /* ---------  gpio_chip related code --------- */
 static void xway_gpio_set(struct gpio_chip *chip, unsigned int pin, int val)
 {
-	struct ltq_pinmux_info *info = dev_get_drvdata(chip->dev);
+	struct ltq_pinmux_info *info = dev_get_drvdata(chip->parent);
 
 	if (val)
 		gpio_setbit(info->membase[0], GPIO_OUT(pin), PORT_PIN(pin));
@@ -1545,14 +1545,14 @@ static void xway_gpio_set(struct gpio_chip *chip, unsigned int pin, int val)
 
 static int xway_gpio_get(struct gpio_chip *chip, unsigned int pin)
 {
-	struct ltq_pinmux_info *info = dev_get_drvdata(chip->dev);
+	struct ltq_pinmux_info *info = dev_get_drvdata(chip->parent);
 
-	return gpio_getbit(info->membase[0], GPIO_IN(pin), PORT_PIN(pin));
+	return !!gpio_getbit(info->membase[0], GPIO_IN(pin), PORT_PIN(pin));
 }
 
 static int xway_gpio_dir_in(struct gpio_chip *chip, unsigned int pin)
 {
-	struct ltq_pinmux_info *info = dev_get_drvdata(chip->dev);
+	struct ltq_pinmux_info *info = dev_get_drvdata(chip->parent);
 
 	gpio_clearbit(info->membase[0], GPIO_DIR(pin), PORT_PIN(pin));
 
@@ -1561,7 +1561,7 @@ static int xway_gpio_dir_in(struct gpio_chip *chip, unsigned int pin)
 
 static int xway_gpio_dir_out(struct gpio_chip *chip, unsigned int pin, int val)
 {
-	struct ltq_pinmux_info *info = dev_get_drvdata(chip->dev);
+	struct ltq_pinmux_info *info = dev_get_drvdata(chip->parent);
 
 	if (PORT(pin) == PORT3)
 		gpio_setbit(info->membase[0], GPIO3_OD, PORT_PIN(pin));
@@ -1708,7 +1708,7 @@ static int pinmux_xway_probe(struct platform_device *pdev)
 	xway_pctrl_desc.pins = xway_info.pads;
 
 	/* load the gpio chip */
-	xway_chip.dev = &pdev->dev;
+	xway_chip.parent = &pdev->dev;
 	ret = gpiochip_add(&xway_chip);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register gpio chip\n");
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 146264a41ec8..8777cf083eef 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -69,11 +69,6 @@ struct msm_pinctrl {
 	void __iomem *regs;
 };
 
-static inline struct msm_pinctrl *to_msm_pinctrl(struct gpio_chip *gc)
-{
-	return container_of(gc, struct msm_pinctrl, chip);
-}
-
 static int msm_get_groups_count(struct pinctrl_dev *pctldev)
 {
 	struct msm_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
@@ -381,7 +376,7 @@ static struct pinctrl_desc msm_pinctrl_desc = {
 static int msm_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = container_of(chip, struct msm_pinctrl, chip);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -401,7 +396,7 @@ static int msm_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int msm_gpio_direction_output(struct gpio_chip *chip, unsigned offset, int value)
 {
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = container_of(chip, struct msm_pinctrl, chip);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -428,7 +423,7 @@ static int msm_gpio_direction_output(struct gpio_chip *chip, unsigned offset, in
 static int msm_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = container_of(chip, struct msm_pinctrl, chip);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
 	u32 val;
 
 	g = &pctrl->soc->groups[offset];
@@ -440,7 +435,7 @@ static int msm_gpio_get(struct gpio_chip *chip, unsigned offset)
 static void msm_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = container_of(chip, struct msm_pinctrl, chip);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned long flags;
 	u32 val;
 
@@ -468,7 +463,7 @@ static void msm_gpio_dbg_show_one(struct seq_file *s,
 				  unsigned gpio)
 {
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = container_of(chip, struct msm_pinctrl, chip);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(chip);
 	unsigned func;
 	int is_out;
 	int drive;
@@ -567,7 +562,7 @@ static void msm_gpio_update_dual_edge_pos(struct msm_pinctrl *pctrl,
 static void msm_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct msm_pingroup *g;
 	unsigned long flags;
 	u32 val;
@@ -588,7 +583,7 @@ static void msm_gpio_irq_mask(struct irq_data *d)
 static void msm_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct msm_pingroup *g;
 	unsigned long flags;
 	u32 val;
@@ -613,7 +608,7 @@ static void msm_gpio_irq_unmask(struct irq_data *d)
 static void msm_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct msm_pingroup *g;
 	unsigned long flags;
 	u32 val;
@@ -638,7 +633,7 @@ static void msm_gpio_irq_ack(struct irq_data *d)
 static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	const struct msm_pingroup *g;
 	unsigned long flags;
 	u32 val;
@@ -732,7 +727,7 @@ static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 static int msm_gpio_irq_set_wake(struct irq_data *d, unsigned int on)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&pctrl->lock, flags);
@@ -757,7 +752,7 @@ static void msm_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
 	const struct msm_pingroup *g;
-	struct msm_pinctrl *pctrl = to_msm_pinctrl(gc);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	int irq_pin;
 	int handled = 0;
@@ -800,11 +795,11 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl)
 	chip->base = 0;
 	chip->ngpio = ngpio;
 	chip->label = dev_name(pctrl->dev);
-	chip->dev = pctrl->dev;
+	chip->parent = pctrl->dev;
 	chip->owner = THIS_MODULE;
 	chip->of_node = pctrl->dev->of_node;
 
-	ret = gpiochip_add(&pctrl->chip);
+	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
 		dev_err(pctrl->dev, "Failed register gpiochip\n");
 		return ret;
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index 77f6a5cb1008..4e12ded3c773 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -159,11 +159,6 @@ static const char *const pmic_gpio_functions[] = {
 	PMIC_GPIO_FUNC_DTEST3, PMIC_GPIO_FUNC_DTEST4,
 };
 
-static inline struct pmic_gpio_state *to_gpio_state(struct gpio_chip *chip)
-{
-	return container_of(chip, struct pmic_gpio_state, chip);
-};
-
 static int pmic_gpio_read(struct pmic_gpio_state *state,
 			  struct pmic_gpio_pad *pad, unsigned int addr)
 {
@@ -496,7 +491,7 @@ static const struct pinconf_ops pmic_gpio_pinconf_ops = {
 
 static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_INPUT_ENABLE, 1);
@@ -507,7 +502,7 @@ static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 static int pmic_gpio_direction_output(struct gpio_chip *chip,
 				      unsigned pin, int val)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_OUTPUT, val);
@@ -517,7 +512,7 @@ static int pmic_gpio_direction_output(struct gpio_chip *chip,
 
 static int pmic_gpio_get(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	struct pmic_gpio_pad *pad;
 	int ret;
 
@@ -534,12 +529,12 @@ static int pmic_gpio_get(struct gpio_chip *chip, unsigned pin)
 		pad->out_value = ret & PMIC_MPP_REG_RT_STS_VAL_MASK;
 	}
 
-	return pad->out_value;
+	return !!pad->out_value;
 }
 
 static void pmic_gpio_set(struct gpio_chip *chip, unsigned pin, int value)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_OUTPUT, value);
@@ -562,7 +557,7 @@ static int pmic_gpio_of_xlate(struct gpio_chip *chip,
 
 static int pmic_gpio_to_irq(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	struct pmic_gpio_pad *pad;
 
 	pad = state->ctrl->desc->pins[pin].drv_data;
@@ -572,7 +567,7 @@ static int pmic_gpio_to_irq(struct gpio_chip *chip, unsigned pin)
 
 static void pmic_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct pmic_gpio_state *state = to_gpio_state(chip);
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
 	unsigned i;
 
 	for (i = 0; i < chip->ngpio; i++) {
@@ -762,7 +757,7 @@ static int pmic_gpio_probe(struct platform_device *pdev)
 	}
 
 	state->chip = pmic_gpio_gpio_template;
-	state->chip.dev = dev;
+	state->chip.parent = dev;
 	state->chip.base = -1;
 	state->chip.ngpio = npins;
 	state->chip.label = dev_name(dev);
@@ -773,7 +768,7 @@ static int pmic_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(state->ctrl))
 		return PTR_ERR(state->ctrl);
 
-	ret = gpiochip_add(&state->chip);
+	ret = gpiochip_add_data(&state->chip, state);
 	if (ret) {
 		dev_err(state->dev, "can't add gpio chip\n");
 		goto err_chip;
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c b/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
index 2df4f29175ae..2f18323571a6 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
@@ -179,11 +179,6 @@ static const char *const pmic_mpp_functions[] = {
 	"digital", "analog", "sink"
 };
 
-static inline struct pmic_mpp_state *to_mpp_state(struct gpio_chip *chip)
-{
-	return container_of(chip, struct pmic_mpp_state, chip);
-};
-
 static int pmic_mpp_read(struct pmic_mpp_state *state,
 			 struct pmic_mpp_pad *pad, unsigned int addr)
 {
@@ -557,7 +552,7 @@ static const struct pinconf_ops pmic_mpp_pinconf_ops = {
 
 static int pmic_mpp_direction_input(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_INPUT_ENABLE, 1);
@@ -568,7 +563,7 @@ static int pmic_mpp_direction_input(struct gpio_chip *chip, unsigned pin)
 static int pmic_mpp_direction_output(struct gpio_chip *chip,
 				     unsigned pin, int val)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_OUTPUT, val);
@@ -578,7 +573,7 @@ static int pmic_mpp_direction_output(struct gpio_chip *chip,
 
 static int pmic_mpp_get(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	struct pmic_mpp_pad *pad;
 	int ret;
 
@@ -592,12 +587,12 @@ static int pmic_mpp_get(struct gpio_chip *chip, unsigned pin)
 		pad->out_value = ret & PMIC_MPP_REG_RT_STS_VAL_MASK;
 	}
 
-	return pad->out_value;
+	return !!pad->out_value;
 }
 
 static void pmic_mpp_set(struct gpio_chip *chip, unsigned pin, int value)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	unsigned long config;
 
 	config = pinconf_to_config_packed(PIN_CONFIG_OUTPUT, value);
@@ -620,7 +615,7 @@ static int pmic_mpp_of_xlate(struct gpio_chip *chip,
 
 static int pmic_mpp_to_irq(struct gpio_chip *chip, unsigned pin)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	struct pmic_mpp_pad *pad;
 
 	pad = state->ctrl->desc->pins[pin].drv_data;
@@ -630,7 +625,7 @@ static int pmic_mpp_to_irq(struct gpio_chip *chip, unsigned pin)
 
 static void pmic_mpp_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
-	struct pmic_mpp_state *state = to_mpp_state(chip);
+	struct pmic_mpp_state *state = gpiochip_get_data(chip);
 	unsigned i;
 
 	for (i = 0; i < chip->ngpio; i++) {
@@ -865,7 +860,7 @@ static int pmic_mpp_probe(struct platform_device *pdev)
 	}
 
 	state->chip = pmic_mpp_gpio_template;
-	state->chip.dev = dev;
+	state->chip.parent = dev;
 	state->chip.base = -1;
 	state->chip.ngpio = npins;
 	state->chip.label = dev_name(dev);
@@ -876,7 +871,7 @@ static int pmic_mpp_probe(struct platform_device *pdev)
 	if (IS_ERR(state->ctrl))
 		return PTR_ERR(state->ctrl);
 
-	ret = gpiochip_add(&state->chip);
+	ret = gpiochip_add_data(&state->chip, state);
 	if (ret) {
 		dev_err(state->dev, "can't add gpio chip\n");
 		goto err_chip;
diff --git a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
index e51176ec83d2..cd8580d9741d 100644
--- a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
@@ -444,7 +444,7 @@ static struct pinctrl_desc pm8xxx_pinctrl_desc = {
 static int pm8xxx_gpio_direction_input(struct gpio_chip *chip,
 				       unsigned offset)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 	u8 val;
 
@@ -460,7 +460,7 @@ static int pm8xxx_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset,
 					int value)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 	u8 val;
 
@@ -478,7 +478,7 @@ static int pm8xxx_gpio_direction_output(struct gpio_chip *chip,
 
 static int pm8xxx_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 	bool state;
 	int ret;
@@ -496,7 +496,7 @@ static int pm8xxx_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static void pm8xxx_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 	u8 val;
 
@@ -525,7 +525,7 @@ static int pm8xxx_gpio_of_xlate(struct gpio_chip *chip,
 
 static int pm8xxx_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	return pin->irq;
@@ -540,7 +540,7 @@ static void pm8xxx_gpio_dbg_show_one(struct seq_file *s,
 				  unsigned offset,
 				  unsigned gpio)
 {
-	struct pm8xxx_gpio *pctrl = container_of(chip, struct pm8xxx_gpio, chip);
+	struct pm8xxx_gpio *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	static const char * const modes[] = {
@@ -737,12 +737,12 @@ static int pm8xxx_gpio_probe(struct platform_device *pdev)
 
 	pctrl->chip = pm8xxx_gpio_template;
 	pctrl->chip.base = -1;
-	pctrl->chip.dev = &pdev->dev;
+	pctrl->chip.parent = &pdev->dev;
 	pctrl->chip.of_node = pdev->dev.of_node;
 	pctrl->chip.of_gpio_n_cells = 2;
 	pctrl->chip.label = dev_name(pctrl->dev);
 	pctrl->chip.ngpio = pctrl->npins;
-	ret = gpiochip_add(&pctrl->chip);
+	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
 		dev_err(&pdev->dev, "failed register gpiochip\n");
 		goto unregister_pinctrl;
diff --git a/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c b/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
index e9f01de51e18..54a5402a9079 100644
--- a/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
+++ b/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
@@ -449,7 +449,7 @@ static struct pinctrl_desc pm8xxx_pinctrl_desc = {
 static int pm8xxx_mpp_direction_input(struct gpio_chip *chip,
 				       unsigned offset)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	switch (pin->mode) {
@@ -473,7 +473,7 @@ static int pm8xxx_mpp_direction_output(struct gpio_chip *chip,
 					unsigned offset,
 					int value)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	switch (pin->mode) {
@@ -497,13 +497,13 @@ static int pm8xxx_mpp_direction_output(struct gpio_chip *chip,
 
 static int pm8xxx_mpp_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 	bool state;
 	int ret;
 
 	if (!pin->input)
-		return pin->output_value;
+		return !!pin->output_value;
 
 	ret = irq_get_irqchip_state(pin->irq, IRQCHIP_STATE_LINE_LEVEL, &state);
 	if (!ret)
@@ -514,7 +514,7 @@ static int pm8xxx_mpp_get(struct gpio_chip *chip, unsigned offset)
 
 static void pm8xxx_mpp_set(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	pin->output_value = !!value;
@@ -538,7 +538,7 @@ static int pm8xxx_mpp_of_xlate(struct gpio_chip *chip,
 
 static int pm8xxx_mpp_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	return pin->irq;
@@ -553,7 +553,7 @@ static void pm8xxx_mpp_dbg_show_one(struct seq_file *s,
 				  unsigned offset,
 				  unsigned gpio)
 {
-	struct pm8xxx_mpp *pctrl = container_of(chip, struct pm8xxx_mpp, chip);
+	struct pm8xxx_mpp *pctrl = gpiochip_get_data(chip);
 	struct pm8xxx_pin_data *pin = pctrl->desc.pins[offset].drv_data;
 
 	static const char * const aout_lvls[] = {
@@ -828,12 +828,12 @@ static int pm8xxx_mpp_probe(struct platform_device *pdev)
 
 	pctrl->chip = pm8xxx_mpp_template;
 	pctrl->chip.base = -1;
-	pctrl->chip.dev = &pdev->dev;
+	pctrl->chip.parent = &pdev->dev;
 	pctrl->chip.of_node = pdev->dev.of_node;
 	pctrl->chip.of_gpio_n_cells = 2;
 	pctrl->chip.label = dev_name(pctrl->dev);
 	pctrl->chip.ngpio = pctrl->npins;
-	ret = gpiochip_add(&pctrl->chip);
+	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
 		dev_err(&pdev->dev, "failed register gpiochip\n");
 		goto unregister_pinctrl;
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c
index 16e2293cc2bc..051b5bf701a8 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -176,7 +176,8 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 
 	ret = gpiochip_lock_as_irq(&bank->gpio_chip, irqd->hwirq);
 	if (ret) {
-		dev_err(bank->gpio_chip.dev, "unable to lock pin %s-%lu IRQ\n",
+		dev_err(bank->gpio_chip.parent,
+			"unable to lock pin %s-%lu IRQ\n",
 			bank->name, irqd->hwirq);
 		return ret;
 	}
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos5440.c b/drivers/pinctrl/samsung/pinctrl-exynos5440.c
index 82dc109f7ed4..00ab63abf1d9 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos5440.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos5440.c
@@ -15,7 +15,7 @@
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/device.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
@@ -539,7 +539,7 @@ static const struct pinconf_ops exynos5440_pinconf_ops = {
 /* gpiolib gpio_set callback function */
 static void exynos5440_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct exynos5440_pinctrl_priv_data *priv = dev_get_drvdata(gc->dev);
+	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
 	void __iomem *base = priv->reg_base;
 	u32 data;
 
@@ -553,7 +553,7 @@ static void exynos5440_gpio_set(struct gpio_chip *gc, unsigned offset, int value
 /* gpiolib gpio_get callback function */
 static int exynos5440_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct exynos5440_pinctrl_priv_data *priv = dev_get_drvdata(gc->dev);
+	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
 	void __iomem *base = priv->reg_base;
 	u32 data;
 
@@ -566,7 +566,7 @@ static int exynos5440_gpio_get(struct gpio_chip *gc, unsigned offset)
 /* gpiolib gpio_direction_input callback function */
 static int exynos5440_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct exynos5440_pinctrl_priv_data *priv = dev_get_drvdata(gc->dev);
+	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
 	void __iomem *base = priv->reg_base;
 	u32 data;
 
@@ -586,7 +586,7 @@ static int exynos5440_gpio_direction_input(struct gpio_chip *gc, unsigned offset
 static int exynos5440_gpio_direction_output(struct gpio_chip *gc, unsigned offset,
 							int value)
 {
-	struct exynos5440_pinctrl_priv_data *priv = dev_get_drvdata(gc->dev);
+	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
 	void __iomem *base = priv->reg_base;
 	u32 data;
 
@@ -607,7 +607,7 @@ static int exynos5440_gpio_direction_output(struct gpio_chip *gc, unsigned offse
 /* gpiolib gpio_to_irq callback function */
 static int exynos5440_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct exynos5440_pinctrl_priv_data *priv = dev_get_drvdata(gc->dev);
+	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
 	unsigned int virq;
 
 	if (offset < 16 || offset > 23)
@@ -817,7 +817,7 @@ static int exynos5440_gpiolib_register(struct platform_device *pdev,
 	priv->gc = gc;
 	gc->base = 0;
 	gc->ngpio = EXYNOS5440_MAX_PINS;
-	gc->dev = &pdev->dev;
+	gc->parent = &pdev->dev;
 	gc->set = exynos5440_gpio_set;
 	gc->get = exynos5440_gpio_get;
 	gc->direction_input = exynos5440_gpio_direction_input;
@@ -825,7 +825,7 @@ static int exynos5440_gpiolib_register(struct platform_device *pdev,
 	gc->to_irq = exynos5440_gpio_to_irq;
 	gc->label = "gpiolib-exynos5440";
 	gc->owner = THIS_MODULE;
-	ret = gpiochip_add(gc);
+	ret = gpiochip_add_data(gc, priv);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register gpio_chip %s, error "
 					"code: %d\n", gc->label, ret);
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index 48294e7449a4..f67b1e958589 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -50,11 +50,6 @@ static LIST_HEAD(drvdata_list);
 
 static unsigned int pin_base;
 
-static inline struct samsung_pin_bank *gc_to_pin_bank(struct gpio_chip *gc)
-{
-	return container_of(gc, struct samsung_pin_bank, gpio_chip);
-}
-
 static int samsung_get_group_count(struct pinctrl_dev *pctldev)
 {
 	struct samsung_pinctrl_drv_data *pmx = pinctrl_dev_get_drvdata(pctldev);
@@ -522,7 +517,7 @@ static const struct pinconf_ops samsung_pinconf_ops = {
 /* gpiolib gpio_set callback function */
 static void samsung_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct samsung_pin_bank *bank = gc_to_pin_bank(gc);
+	struct samsung_pin_bank *bank = gpiochip_get_data(gc);
 	const struct samsung_pin_bank_type *type = bank->type;
 	unsigned long flags;
 	void __iomem *reg;
@@ -546,7 +541,7 @@ static int samsung_gpio_get(struct gpio_chip *gc, unsigned offset)
 {
 	void __iomem *reg;
 	u32 data;
-	struct samsung_pin_bank *bank = gc_to_pin_bank(gc);
+	struct samsung_pin_bank *bank = gpiochip_get_data(gc);
 	const struct samsung_pin_bank_type *type = bank->type;
 
 	reg = bank->drvdata->virt_base + bank->pctl_offset;
@@ -571,7 +566,7 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc,
 	u32 data, mask, shift;
 	unsigned long flags;
 
-	bank = gc_to_pin_bank(gc);
+	bank = gpiochip_get_data(gc);
 	type = bank->type;
 	drvdata = bank->drvdata;
 
@@ -619,7 +614,7 @@ static int samsung_gpio_direction_output(struct gpio_chip *gc, unsigned offset,
  */
 static int samsung_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
-	struct samsung_pin_bank *bank = gc_to_pin_bank(gc);
+	struct samsung_pin_bank *bank = gpiochip_get_data(gc);
 	unsigned int virq;
 
 	if (!bank->irq_domain)
@@ -914,11 +909,11 @@ static int samsung_gpiolib_register(struct platform_device *pdev,
 		gc = &bank->gpio_chip;
 		gc->base = drvdata->pin_base + bank->pin_base;
 		gc->ngpio = bank->nr_pins;
-		gc->dev = &pdev->dev;
+		gc->parent = &pdev->dev;
 		gc->of_node = bank->of_node;
 		gc->label = bank->name;
 
-		ret = gpiochip_add(gc);
+		ret = gpiochip_add_data(gc, bank);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to register gpio_chip %s, error code: %d\n",
 							gc->label, ret);
diff --git a/drivers/pinctrl/sh-pfc/gpio.c b/drivers/pinctrl/sh-pfc/gpio.c
index db3f09aa8993..a6681b8b17c3 100644
--- a/drivers/pinctrl/sh-pfc/gpio.c
+++ b/drivers/pinctrl/sh-pfc/gpio.c
@@ -38,14 +38,10 @@ struct sh_pfc_chip {
 	struct sh_pfc_gpio_pin		*pins;
 };
 
-static struct sh_pfc_chip *gpio_to_pfc_chip(struct gpio_chip *gc)
-{
-	return container_of(gc, struct sh_pfc_chip, gpio_chip);
-}
-
 static struct sh_pfc *gpio_to_pfc(struct gpio_chip *gc)
 {
-	return gpio_to_pfc_chip(gc)->pfc;
+	struct sh_pfc_chip *chip = gpiochip_get_data(gc);
+	return chip->pfc;
 }
 
 static void gpio_get_data_reg(struct sh_pfc_chip *chip, unsigned int offset,
@@ -178,14 +174,14 @@ static int gpio_pin_direction_input(struct gpio_chip *gc, unsigned offset)
 static int gpio_pin_direction_output(struct gpio_chip *gc, unsigned offset,
 				    int value)
 {
-	gpio_pin_set_value(gpio_to_pfc_chip(gc), offset, value);
+	gpio_pin_set_value(gpiochip_get_data(gc), offset, value);
 
 	return pinctrl_gpio_direction_output(offset);
 }
 
 static int gpio_pin_get(struct gpio_chip *gc, unsigned offset)
 {
-	struct sh_pfc_chip *chip = gpio_to_pfc_chip(gc);
+	struct sh_pfc_chip *chip = gpiochip_get_data(gc);
 	struct sh_pfc_gpio_data_reg *reg;
 	unsigned int bit;
 	unsigned int pos;
@@ -199,7 +195,7 @@ static int gpio_pin_get(struct gpio_chip *gc, unsigned offset)
 
 static void gpio_pin_set(struct gpio_chip *gc, unsigned offset, int value)
 {
-	gpio_pin_set_value(gpio_to_pfc_chip(gc), offset, value);
+	gpio_pin_set_value(gpiochip_get_data(gc), offset, value);
 }
 
 static int gpio_pin_to_irq(struct gpio_chip *gc, unsigned offset)
@@ -246,7 +242,7 @@ static int gpio_pin_setup(struct sh_pfc_chip *chip)
 	gc->to_irq = gpio_pin_to_irq;
 
 	gc->label = pfc->info->name;
-	gc->dev = pfc->dev;
+	gc->parent = pfc->dev;
 	gc->owner = THIS_MODULE;
 	gc->base = 0;
 	gc->ngpio = pfc->nr_gpio_pins;
@@ -322,7 +318,7 @@ sh_pfc_add_gpiochip(struct sh_pfc *pfc, int(*setup)(struct sh_pfc_chip *),
 	if (ret < 0)
 		return ERR_PTR(ret);
 
-	ret = gpiochip_add(&chip->gpio_chip);
+	ret = gpiochip_add_data(&chip->gpio_chip, chip);
 	if (unlikely(ret < 0))
 		return ERR_PTR(ret);
 
diff --git a/drivers/pinctrl/sirf/pinctrl-atlas7.c b/drivers/pinctrl/sirf/pinctrl-atlas7.c
index 053d98e33944..beb024c31a5d 100644
--- a/drivers/pinctrl/sirf/pinctrl-atlas7.c
+++ b/drivers/pinctrl/sirf/pinctrl-atlas7.c
@@ -358,11 +358,6 @@ struct atlas7_gpio_chip {
 	struct atlas7_gpio_bank banks[0];
 };
 
-static inline struct atlas7_gpio_chip *to_atlas7_gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct atlas7_gpio_chip, chip);
-}
-
 /**
  * @dev: a pointer back to containing device
  * @virtbase: the offset to the controller in virtual memory
@@ -5642,7 +5637,7 @@ static int __atlas7_gpio_to_pin(struct atlas7_gpio_chip *a7gc, u32 gpio)
 static void atlas7_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(gc);
 	struct atlas7_gpio_bank *bank;
 	void __iomem *ctrl_reg;
 	u32 val, pin_in_bank;
@@ -5680,7 +5675,7 @@ static void __atlas7_gpio_irq_mask(struct atlas7_gpio_chip *a7gc, int idx)
 static void atlas7_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	spin_lock_irqsave(&a7gc->lock, flags);
@@ -5693,7 +5688,7 @@ static void atlas7_gpio_irq_mask(struct irq_data *d)
 static void atlas7_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(gc);
 	struct atlas7_gpio_bank *bank;
 	void __iomem *ctrl_reg;
 	u32 val, pin_in_bank;
@@ -5717,7 +5712,7 @@ static int atlas7_gpio_irq_type(struct irq_data *d,
 				unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(gc);
 	struct atlas7_gpio_bank *bank;
 	void __iomem *ctrl_reg;
 	u32 val, pin_in_bank;
@@ -5786,7 +5781,7 @@ static struct irq_chip atlas7_gpio_irq_chip = {
 static void atlas7_gpio_handle_irq(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(gc);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(gc);
 	struct atlas7_gpio_bank *bank = NULL;
 	u32 status, ctrl;
 	int pin_in_bank = 0, idx;
@@ -5854,7 +5849,7 @@ static void __atlas7_gpio_set_input(struct atlas7_gpio_chip *a7gc,
 static int atlas7_gpio_request(struct gpio_chip *chip,
 				unsigned int gpio)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	int ret;
 	unsigned long flags;
 
@@ -5882,7 +5877,7 @@ static int atlas7_gpio_request(struct gpio_chip *chip,
 static void atlas7_gpio_free(struct gpio_chip *chip,
 				unsigned int gpio)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&a7gc->lock, flags);
@@ -5898,7 +5893,7 @@ static void atlas7_gpio_free(struct gpio_chip *chip,
 static int atlas7_gpio_direction_input(struct gpio_chip *chip,
 					unsigned int gpio)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&a7gc->lock, flags);
@@ -5935,7 +5930,7 @@ static void __atlas7_gpio_set_output(struct atlas7_gpio_chip *a7gc,
 static int atlas7_gpio_direction_output(struct gpio_chip *chip,
 				unsigned int gpio, int value)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	spin_lock_irqsave(&a7gc->lock, flags);
@@ -5950,7 +5945,7 @@ static int atlas7_gpio_direction_output(struct gpio_chip *chip,
 static int atlas7_gpio_get_value(struct gpio_chip *chip,
 					unsigned int gpio)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	struct atlas7_gpio_bank *bank;
 	u32 val, pin_in_bank;
 	unsigned long flags;
@@ -5970,7 +5965,7 @@ static int atlas7_gpio_get_value(struct gpio_chip *chip,
 static void atlas7_gpio_set_value(struct gpio_chip *chip,
 				unsigned int gpio, int value)
 {
-	struct atlas7_gpio_chip *a7gc = to_atlas7_gpio(chip);
+	struct atlas7_gpio_chip *a7gc = gpiochip_get_data(chip);
 	struct atlas7_gpio_bank *bank;
 	void __iomem *ctrl_reg;
 	u32 ctrl, pin_in_bank;
@@ -6054,10 +6049,10 @@ static int atlas7_gpio_probe(struct platform_device *pdev)
 	chip->label = kstrdup(np->name, GFP_KERNEL);
 	chip->of_node = np;
 	chip->of_gpio_n_cells = 2;
-	chip->dev = &pdev->dev;
+	chip->parent = &pdev->dev;
 
 	/* Add gpio chip to system */
-	ret = gpiochip_add(chip);
+	ret = gpiochip_add_data(chip, a7gc);
 	if (ret) {
 		dev_err(&pdev->dev,
 		"%s: error in probe function with status %d\n",
diff --git a/drivers/pinctrl/sirf/pinctrl-sirf.c b/drivers/pinctrl/sirf/pinctrl-sirf.c
index edf40df05ec0..762c0c9c1278 100644
--- a/drivers/pinctrl/sirf/pinctrl-sirf.c
+++ b/drivers/pinctrl/sirf/pinctrl-sirf.c
@@ -403,11 +403,6 @@ static int __init sirfsoc_pinmux_init(void)
 }
 arch_initcall(sirfsoc_pinmux_init);
 
-static inline struct sirfsoc_gpio_chip *to_sirfsoc_gpio(struct gpio_chip *gc)
-{
-	return container_of(gc, struct sirfsoc_gpio_chip, chip.gc);
-}
-
 static inline struct sirfsoc_gpio_bank *
 sirfsoc_gpio_to_bank(struct sirfsoc_gpio_chip *sgpio, unsigned int offset)
 {
@@ -422,7 +417,7 @@ static inline int sirfsoc_gpio_to_bankoff(unsigned int offset)
 static void sirfsoc_gpio_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(gc);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(gc);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, d->hwirq);
 	int idx = sirfsoc_gpio_to_bankoff(d->hwirq);
 	u32 val, offset;
@@ -461,7 +456,7 @@ static void __sirfsoc_gpio_irq_mask(struct sirfsoc_gpio_chip *sgpio,
 static void sirfsoc_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(gc);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(gc);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, d->hwirq);
 
 	__sirfsoc_gpio_irq_mask(sgpio, bank, d->hwirq % SIRFSOC_GPIO_BANK_SIZE);
@@ -470,7 +465,7 @@ static void sirfsoc_gpio_irq_mask(struct irq_data *d)
 static void sirfsoc_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(gc);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(gc);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, d->hwirq);
 	int idx = sirfsoc_gpio_to_bankoff(d->hwirq);
 	u32 val, offset;
@@ -491,7 +486,7 @@ static void sirfsoc_gpio_irq_unmask(struct irq_data *d)
 static int sirfsoc_gpio_irq_type(struct irq_data *d, unsigned type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(gc);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(gc);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, d->hwirq);
 	int idx = sirfsoc_gpio_to_bankoff(d->hwirq);
 	u32 val, offset;
@@ -553,7 +548,7 @@ static void sirfsoc_gpio_handle_irq(struct irq_desc *desc)
 {
 	unsigned int irq = irq_desc_get_irq(desc);
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(gc);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(gc);
 	struct sirfsoc_gpio_bank *bank;
 	u32 status, ctrl;
 	int idx = 0;
@@ -611,7 +606,7 @@ static inline void sirfsoc_gpio_set_input(struct sirfsoc_gpio_chip *sgpio,
 
 static int sirfsoc_gpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, offset);
 	unsigned long flags;
 
@@ -634,7 +629,7 @@ static int sirfsoc_gpio_request(struct gpio_chip *chip, unsigned offset)
 
 static void sirfsoc_gpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, offset);
 	unsigned long flags;
 
@@ -650,7 +645,7 @@ static void sirfsoc_gpio_free(struct gpio_chip *chip, unsigned offset)
 
 static int sirfsoc_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, gpio);
 	int idx = sirfsoc_gpio_to_bankoff(gpio);
 	unsigned long flags;
@@ -693,7 +688,7 @@ static inline void sirfsoc_gpio_set_output(struct sirfsoc_gpio_chip *sgpio,
 static int sirfsoc_gpio_direction_output(struct gpio_chip *chip,
 	unsigned gpio, int value)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, gpio);
 	int idx = sirfsoc_gpio_to_bankoff(gpio);
 	u32 offset;
@@ -712,7 +707,7 @@ static int sirfsoc_gpio_direction_output(struct gpio_chip *chip,
 
 static int sirfsoc_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, offset);
 	u32 val;
 	unsigned long flags;
@@ -729,7 +724,7 @@ static int sirfsoc_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 static void sirfsoc_gpio_set_value(struct gpio_chip *chip, unsigned offset,
 	int value)
 {
-	struct sirfsoc_gpio_chip *sgpio = to_sirfsoc_gpio(chip);
+	struct sirfsoc_gpio_chip *sgpio = gpiochip_get_data(chip);
 	struct sirfsoc_gpio_bank *bank = sirfsoc_gpio_to_bank(sgpio, offset);
 	u32 ctrl;
 	unsigned long flags;
@@ -815,10 +810,10 @@ static int sirfsoc_gpio_probe(struct device_node *np)
 	sgpio->chip.gc.of_node = np;
 	sgpio->chip.gc.of_xlate = sirfsoc_gpio_of_xlate;
 	sgpio->chip.gc.of_gpio_n_cells = 2;
-	sgpio->chip.gc.dev = &pdev->dev;
+	sgpio->chip.gc.parent = &pdev->dev;
 	sgpio->chip.regs = regs;
 
-	err = gpiochip_add(&sgpio->chip.gc);
+	err = gpiochip_add_data(&sgpio->chip.gc, sgpio);
 	if (err) {
 		dev_err(&pdev->dev, "%s: error in probe function with status %d\n",
 			np->full_name, err);
diff --git a/drivers/pinctrl/spear/pinctrl-plgpio.c b/drivers/pinctrl/spear/pinctrl-plgpio.c
index 1f0af250dbb5..4c9b863f8267 100644
--- a/drivers/pinctrl/spear/pinctrl-plgpio.c
+++ b/drivers/pinctrl/spear/pinctrl-plgpio.c
@@ -107,7 +107,7 @@ static inline void plgpio_reg_reset(void __iomem *base, u32 pin, u32 reg)
 /* gpio framework specific routines */
 static int plgpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 	unsigned long flags;
 
 	/* get correct offset for "offset" pin */
@@ -127,7 +127,7 @@ static int plgpio_direction_input(struct gpio_chip *chip, unsigned offset)
 static int plgpio_direction_output(struct gpio_chip *chip, unsigned offset,
 		int value)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 	unsigned long flags;
 	unsigned dir_offset = offset, wdata_offset = offset, tmp;
 
@@ -159,7 +159,7 @@ static int plgpio_direction_output(struct gpio_chip *chip, unsigned offset,
 
 static int plgpio_get_value(struct gpio_chip *chip, unsigned offset)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 
 	if (offset >= chip->ngpio)
 		return -EINVAL;
@@ -176,7 +176,7 @@ static int plgpio_get_value(struct gpio_chip *chip, unsigned offset)
 
 static void plgpio_set_value(struct gpio_chip *chip, unsigned offset, int value)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 
 	if (offset >= chip->ngpio)
 		return;
@@ -196,7 +196,7 @@ static void plgpio_set_value(struct gpio_chip *chip, unsigned offset, int value)
 
 static int plgpio_request(struct gpio_chip *chip, unsigned offset)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 	int gpio = chip->base + offset;
 	unsigned long flags;
 	int ret = 0;
@@ -248,7 +248,7 @@ err0:
 
 static void plgpio_free(struct gpio_chip *chip, unsigned offset)
 {
-	struct plgpio *plgpio = container_of(chip, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(chip);
 	int gpio = chip->base + offset;
 	unsigned long flags;
 
@@ -280,7 +280,7 @@ disable_clk:
 static void plgpio_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct plgpio *plgpio = container_of(gc, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	unsigned long flags;
 
@@ -299,7 +299,7 @@ static void plgpio_irq_disable(struct irq_data *d)
 static void plgpio_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct plgpio *plgpio = container_of(gc, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	unsigned long flags;
 
@@ -318,7 +318,7 @@ static void plgpio_irq_enable(struct irq_data *d)
 static int plgpio_irq_set_type(struct irq_data *d, unsigned trigger)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct plgpio *plgpio = container_of(gc, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	void __iomem *reg_off;
 	unsigned int supported_type = 0, val;
@@ -359,7 +359,7 @@ static struct irq_chip plgpio_irqchip = {
 static void plgpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct plgpio *plgpio = container_of(gc, struct plgpio, chip);
+	struct plgpio *plgpio = gpiochip_get_data(gc);
 	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 	int regs_count, count, pin, offset, i = 0;
 	unsigned long pending;
@@ -561,7 +561,7 @@ static int plgpio_probe(struct platform_device *pdev)
 	plgpio->chip.get = plgpio_get_value;
 	plgpio->chip.set = plgpio_set_value;
 	plgpio->chip.label = dev_name(&pdev->dev);
-	plgpio->chip.dev = &pdev->dev;
+	plgpio->chip.parent = &pdev->dev;
 	plgpio->chip.owner = THIS_MODULE;
 	plgpio->chip.of_node = pdev->dev.of_node;
 
@@ -573,7 +573,7 @@ static int plgpio_probe(struct platform_device *pdev)
 		}
 	}
 
-	ret = gpiochip_add(&plgpio->chip);
+	ret = gpiochip_add_data(&plgpio->chip, plgpio);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to add gpio chip\n");
 		goto unprepare_clk;
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index dead97daca35..7a2465f5e71e 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -12,7 +12,7 @@
 
 #include <linux/io.h>
 #include <linux/clk.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/module.h>
@@ -454,7 +454,7 @@ static int sunxi_pinctrl_gpio_direction_input(struct gpio_chip *chip,
 
 static int sunxi_pinctrl_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
-	struct sunxi_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct sunxi_pinctrl *pctl = gpiochip_get_data(chip);
 	u32 reg = sunxi_data_reg(offset);
 	u8 index = sunxi_data_offset(offset);
 	u32 set_mux = pctl->desc->irq_read_needs_mux &&
@@ -469,13 +469,13 @@ static int sunxi_pinctrl_gpio_get(struct gpio_chip *chip, unsigned offset)
 	if (set_mux)
 		sunxi_pmx_set(pctl->pctl_dev, offset, SUN4I_FUNC_IRQ);
 
-	return val;
+	return !!val;
 }
 
 static void sunxi_pinctrl_gpio_set(struct gpio_chip *chip,
 				unsigned offset, int value)
 {
-	struct sunxi_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct sunxi_pinctrl *pctl = gpiochip_get_data(chip);
 	u32 reg = sunxi_data_reg(offset);
 	u8 index = sunxi_data_offset(offset);
 	unsigned long flags;
@@ -522,7 +522,7 @@ static int sunxi_pinctrl_gpio_of_xlate(struct gpio_chip *gc,
 
 static int sunxi_pinctrl_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	struct sunxi_pinctrl *pctl = dev_get_drvdata(chip->dev);
+	struct sunxi_pinctrl *pctl = gpiochip_get_data(chip);
 	struct sunxi_desc_function *desc;
 	unsigned pinnum = pctl->desc->pin_base + offset;
 	unsigned irqnum;
@@ -536,7 +536,7 @@ static int sunxi_pinctrl_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 
 	irqnum = desc->irqbank * IRQ_PER_BANK + desc->irqnum;
 
-	dev_dbg(chip->dev, "%s: request IRQ for GPIO %d, return %d\n",
+	dev_dbg(chip->parent, "%s: request IRQ for GPIO %d, return %d\n",
 		chip->label, offset + chip->base, irqnum);
 
 	return irq_find_mapping(pctl->domain, irqnum);
@@ -959,10 +959,10 @@ int sunxi_pinctrl_init(struct platform_device *pdev,
 	pctl->chip->ngpio = round_up(last_pin, PINS_PER_BANK) -
 			    pctl->desc->pin_base;
 	pctl->chip->label = dev_name(&pdev->dev);
-	pctl->chip->dev = &pdev->dev;
+	pctl->chip->parent = &pdev->dev;
 	pctl->chip->base = pctl->desc->pin_base;
 
-	ret = gpiochip_add(pctl->chip);
+	ret = gpiochip_add_data(pctl->chip, pctl);
 	if (ret)
 		goto pinctrl_error;
 
diff --git a/drivers/pinctrl/vt8500/pinctrl-wmt.c b/drivers/pinctrl/vt8500/pinctrl-wmt.c
index fb22d3f62480..5c261bf5542f 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wmt.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wmt.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
@@ -488,7 +488,7 @@ static struct pinctrl_desc wmt_desc = {
 
 static int wmt_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
-	struct wmt_pinctrl_data *data = dev_get_drvdata(chip->dev);
+	struct wmt_pinctrl_data *data = gpiochip_get_data(chip);
 	u32 bank = WMT_BANK_FROM_PIN(offset);
 	u32 bit = WMT_BIT_FROM_PIN(offset);
 	u32 reg_dir = data->banks[bank].reg_dir;
@@ -503,7 +503,7 @@ static int wmt_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 
 static int wmt_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 {
-	struct wmt_pinctrl_data *data = dev_get_drvdata(chip->dev);
+	struct wmt_pinctrl_data *data = gpiochip_get_data(chip);
 	u32 bank = WMT_BANK_FROM_PIN(offset);
 	u32 bit = WMT_BIT_FROM_PIN(offset);
 	u32 reg_data_in = data->banks[bank].reg_data_in;
@@ -519,7 +519,7 @@ static int wmt_gpio_get_value(struct gpio_chip *chip, unsigned offset)
 static void wmt_gpio_set_value(struct gpio_chip *chip, unsigned offset,
 			       int val)
 {
-	struct wmt_pinctrl_data *data = dev_get_drvdata(chip->dev);
+	struct wmt_pinctrl_data *data = gpiochip_get_data(chip);
 	u32 bank = WMT_BANK_FROM_PIN(offset);
 	u32 bit = WMT_BIT_FROM_PIN(offset);
 	u32 reg_data_out = data->banks[bank].reg_data_out;
@@ -575,7 +575,7 @@ int wmt_pinctrl_probe(struct platform_device *pdev,
 	wmt_desc.npins = data->npins;
 
 	data->gpio_chip = wmt_gpio_chip;
-	data->gpio_chip.dev = &pdev->dev;
+	data->gpio_chip.parent = &pdev->dev;
 	data->gpio_chip.of_node = pdev->dev.of_node;
 	data->gpio_chip.ngpio = data->nbanks * 32;
 
@@ -589,7 +589,7 @@ int wmt_pinctrl_probe(struct platform_device *pdev,
 		return PTR_ERR(data->pctl_dev);
 	}
 
-	err = gpiochip_add(&data->gpio_chip);
+	err = gpiochip_add_data(&data->gpio_chip, data);
 	if (err) {
 		dev_err(&pdev->dev, "could not add GPIO chip\n");
 		goto fail_gpio;
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 1089eaa02b00..69f93a576e45 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -587,6 +587,20 @@ config EEEPC_WMI
 	  If you have an ACPI-WMI compatible Eee PC laptop (>= 1000), say Y or M
 	  here.
 
+config ASUS_WIRELESS
+	tristate "Asus Wireless Radio Control Driver"
+	depends on ACPI
+	depends on INPUT
+	---help---
+	  The Asus Wireless Radio Control handles the airplane mode hotkey
+	  present on some Asus laptops.
+
+	  Say Y or M here if you have an ASUS notebook with an airplane mode
+	  hotkey.
+
+	  If you choose to compile this driver as a module the module will be
+	  called asus-wireless.
+
 config ACPI_WMI
 	tristate "WMI"
 	depends on ACPI
@@ -641,6 +655,7 @@ config ACPI_TOSHIBA
 	depends on INPUT
 	depends on SERIO_I8042 || SERIO_I8042 = n
 	depends on ACPI_VIDEO || ACPI_VIDEO = n
+	depends on RFKILL || RFKILL = n
 	select INPUT_POLLDEV
 	select INPUT_SPARSEKMAP
 	---help---
@@ -731,6 +746,18 @@ config ACPI_CMPC
 	  keys as input device, backlight device, tablet and accelerometer
 	  devices.
 
+config INTEL_HID_EVENT
+	tristate "INTEL HID Event"
+	depends on ACPI
+	depends on INPUT
+	select INPUT_SPARSEKMAP
+	help
+	  This driver provides support for the Intel HID Event hotkey interface.
+	  Some laptops require this driver for hotkey support.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called intel_hid.
+
 config INTEL_SCU_IPC
 	bool "Intel SCU IPC Support"
 	depends on X86_INTEL_MID
@@ -940,8 +967,25 @@ config INTEL_PMC_IPC
 	with other entities in the CPU.
 
 config SURFACE_PRO3_BUTTON
-	tristate "Power/home/volume buttons driver for Microsoft Surface Pro 3 tablet"
+	tristate "Power/home/volume buttons driver for Microsoft Surface Pro 3/4 tablet"
 	depends on ACPI && INPUT
 	---help---
-	  This driver handles the power/home/volume buttons on the Microsoft Surface Pro 3 tablet.
+	  This driver handles the power/home/volume buttons on the Microsoft Surface Pro 3/4 tablet.
+
+config INTEL_PUNIT_IPC
+	tristate "Intel P-Unit IPC Driver"
+	---help---
+	  This driver provides support for Intel P-Unit Mailbox IPC mechanism,
+	  which is used to bridge the communications between kernel and P-Unit.
+
+config INTEL_TELEMETRY
+	tristate "Intel SoC Telemetry Driver"
+	default n
+	depends on INTEL_PMC_IPC && INTEL_PUNIT_IPC && X86_64
+	---help---
+	  This driver provides interfaces to configure and use
+	  telemetry for INTEL SoC from APL onwards. It is also
+	  used to get various SoC events and parameters
+	  directly via debugfs files. Various tools may use
+	  this interface for SoC state monitoring.
 endif # X86_PLATFORM_DEVICES
diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
index 3ca78a3eb6f8..40574e7390f3 100644
--- a/drivers/platform/x86/Makefile
+++ b/drivers/platform/x86/Makefile
@@ -5,6 +5,7 @@
 obj-$(CONFIG_ASUS_LAPTOP)	+= asus-laptop.o
 obj-$(CONFIG_ASUS_WMI)		+= asus-wmi.o
 obj-$(CONFIG_ASUS_NB_WMI)	+= asus-nb-wmi.o
+obj-$(CONFIG_ASUS_WIRELESS)	+= asus-wireless.o
 obj-$(CONFIG_EEEPC_LAPTOP)	+= eeepc-laptop.o
 obj-$(CONFIG_EEEPC_WMI)		+= eeepc-wmi.o
 obj-$(CONFIG_MSI_LAPTOP)	+= msi-laptop.o
@@ -41,6 +42,7 @@ obj-$(CONFIG_ACPI_TOSHIBA)	+= toshiba_acpi.o
 obj-$(CONFIG_TOSHIBA_BT_RFKILL)	+= toshiba_bluetooth.o
 obj-$(CONFIG_TOSHIBA_HAPS)	+= toshiba_haps.o
 obj-$(CONFIG_TOSHIBA_WMI)	+= toshiba-wmi.o
+obj-$(CONFIG_INTEL_HID_EVENT)	+= intel-hid.o
 obj-$(CONFIG_INTEL_SCU_IPC)	+= intel_scu_ipc.o
 obj-$(CONFIG_INTEL_SCU_IPC_UTIL) += intel_scu_ipcutil.o
 obj-$(CONFIG_INTEL_MFLD_THERMAL) += intel_mid_thermal.o
@@ -62,3 +64,7 @@ obj-$(CONFIG_PVPANIC)           += pvpanic.o
 obj-$(CONFIG_ALIENWARE_WMI)	+= alienware-wmi.o
 obj-$(CONFIG_INTEL_PMC_IPC)	+= intel_pmc_ipc.o
 obj-$(CONFIG_SURFACE_PRO3_BUTTON)	+= surfacepro3_button.o
+obj-$(CONFIG_INTEL_PUNIT_IPC)  += intel_punit_ipc.o
+obj-$(CONFIG_INTEL_TELEMETRY)	+= intel_telemetry_core.o \
+				   intel_telemetry_pltdrv.o \
+				   intel_telemetry_debugfs.o
diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
index 976efeb3f2ba..f236250ac106 100644
--- a/drivers/platform/x86/apple-gmux.c
+++ b/drivers/platform/x86/apple-gmux.c
@@ -3,6 +3,7 @@
  *
  *  Copyright (C) Canonical Ltd. <seth.forshee@canonical.com>
  *  Copyright (C) 2010-2012 Andreas Heider <andreas@meetr.de>
+ *  Copyright (C) 2015 Lukas Wunner <lukas@wunner.de>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License version 2 as
@@ -26,6 +27,24 @@
 #include <acpi/video.h>
 #include <asm/io.h>
 
+/**
+ * DOC: Overview
+ *
+ * :1:  http://www.latticesemi.com/en/Products/FPGAandCPLD/LatticeXP2.aspx
+ * :2:  http://www.renesas.com/products/mpumcu/h8s/h8s2100/h8s2113/index.jsp
+ *
+ * gmux is a microcontroller built into the MacBook Pro to support dual GPUs:
+ * A {1}[Lattice XP2] on pre-retinas, a {2}[Renesas R4F2113] on retinas.
+ *
+ * (The MacPro6,1 2013 also has a gmux, however it is unclear why since it has
+ * dual GPUs but no built-in display.)
+ *
+ * gmux is connected to the LPC bus of the southbridge. Its I/O ports are
+ * accessed differently depending on the microcontroller: Driver functions
+ * to access a pre-retina gmux are infixed `_pio_`, those for a retina gmux
+ * are infixed `_index_`.
+ */
+
 struct apple_gmux_data {
 	unsigned long iostart;
 	unsigned long iolen;
@@ -247,6 +266,20 @@ static bool gmux_is_indexed(struct apple_gmux_data *gmux_data)
 	return false;
 }
 
+/**
+ * DOC: Backlight control
+ *
+ * :3:  http://www.ti.com/lit/ds/symlink/lp8543.pdf
+ * :4:  http://www.ti.com/lit/ds/symlink/lp8545.pdf
+ *
+ * On single GPU MacBooks, the PWM signal for the backlight is generated by
+ * the GPU. On dual GPU MacBook Pros by contrast, either GPU may be suspended
+ * to conserve energy. Hence the PWM signal needs to be generated by a separate
+ * backlight driver which is controlled by gmux. The earliest generation
+ * MBP5 2008/09 uses a {3}[TI LP8543] backlight driver. All newer models
+ * use a {4}[TI LP8545].
+ */
+
 static int gmux_get_brightness(struct backlight_device *bd)
 {
 	struct apple_gmux_data *gmux_data = bl_get_data(bd);
@@ -273,6 +306,68 @@ static const struct backlight_ops gmux_bl_ops = {
 	.update_status = gmux_update_status,
 };
 
+/**
+ * DOC: Graphics mux
+ *
+ * :5:  http://pimg-fpiw.uspto.gov/fdd/07/870/086/0.pdf
+ * :6:  http://www.nxp.com/documents/data_sheet/CBTL06141.pdf
+ * :7:  http://www.ti.com/lit/ds/symlink/hd3ss212.pdf
+ * :8:  https://www.pericom.com/assets/Datasheets/PI3VDP12412.pdf
+ * :9:  http://www.ti.com/lit/ds/symlink/sn74lv4066a.pdf
+ * :10: http://pdf.datasheetarchive.com/indexerfiles/Datasheets-SW16/DSASW00308511.pdf
+ * :11: http://www.ti.com/lit/ds/symlink/ts3ds10224.pdf
+ *
+ * On pre-retinas, the LVDS outputs of both GPUs feed into gmux which muxes
+ * either of them to the panel. One of the tricks gmux has up its sleeve is
+ * to lengthen the blanking interval of its output during a switch to
+ * synchronize it with the GPU switched to. This allows for a flicker-free
+ * switch that is imperceptible by the user ({5}[US 8,687,007 B2]).
+ *
+ * On retinas, muxing is no longer done by gmux itself, but by a separate
+ * chip which is controlled by gmux. The chip is triple sourced, it is
+ * either an {6}[NXP CBTL06142], {7}[TI HD3SS212] or {8}[Pericom PI3VDP12412].
+ * The panel is driven with eDP instead of LVDS since the pixel clock
+ * required for retina resolution exceeds LVDS' limits.
+ *
+ * Pre-retinas are able to switch the panel's DDC pins separately.
+ * This is handled by a {9}[TI SN74LV4066A] which is controlled by gmux.
+ * The inactive GPU can thus probe the panel's EDID without switching over
+ * the entire panel. Retinas lack this functionality as the chips used for
+ * eDP muxing are incapable of switching the AUX channel separately (see
+ * the linked data sheets, Pericom would be capable but this is unused).
+ * However the retina panel has the NO_AUX_HANDSHAKE_LINK_TRAINING bit set
+ * in its DPCD, allowing the inactive GPU to skip the AUX handshake and
+ * set up the output with link parameters pre-calibrated by the active GPU.
+ *
+ * The external DP port is only fully switchable on the first two unibody
+ * MacBook Pro generations, MBP5 2008/09 and MBP6 2010. This is done by an
+ * {6}[NXP CBTL06141] which is controlled by gmux. It's the predecessor of the
+ * eDP mux on retinas, the difference being support for 2.7 versus 5.4 Gbit/s.
+ *
+ * The following MacBook Pro generations replaced the external DP port with a
+ * combined DP/Thunderbolt port and lost the ability to switch it between GPUs,
+ * connecting it either to the discrete GPU or the Thunderbolt controller.
+ * Oddly enough, while the full port is no longer switchable, AUX and HPD
+ * are still switchable by way of an {10}[NXP CBTL03062] (on pre-retinas
+ * MBP8 2011 and MBP9 2012) or two {11}[TI TS3DS10224] (on retinas) under the
+ * control of gmux. Since the integrated GPU is missing the main link,
+ * external displays appear to it as phantoms which fail to link-train.
+ *
+ * gmux receives the HPD signal of all display connectors and sends an
+ * interrupt on hotplug. On generations which cannot switch external ports,
+ * the discrete GPU can then be woken to drive the newly connected display.
+ * The ability to switch AUX on these generations could be used to improve
+ * reliability of hotplug detection by having the integrated GPU poll the
+ * ports while the discrete GPU is asleep, but currently we do not make use
+ * of this feature.
+ *
+ * gmux' initial switch state on bootup is user configurable via the EFI
+ * variable `gpu-power-prefs-fa4ce28d-b62f-4c99-9cc3-6815686e30f9` (5th byte,
+ * 1 = IGD, 0 = DIS). Based on this setting, the EFI firmware tells gmux to
+ * switch the panel and the external DP connector and allocates a framebuffer
+ * for the selected GPU.
+ */
+
 static int gmux_switchto(enum vga_switcheroo_client_id id)
 {
 	if (id == VGA_SWITCHEROO_IGD) {
@@ -288,6 +383,14 @@ static int gmux_switchto(enum vga_switcheroo_client_id id)
 	return 0;
 }
 
+/**
+ * DOC: Power control
+ *
+ * gmux is able to cut power to the discrete GPU. It automatically takes care
+ * of the correct sequence to tear down and bring up the power rails for
+ * core voltage, VRAM and PCIe.
+ */
+
 static int gmux_set_discrete_state(struct apple_gmux_data *gmux_data,
 				   enum vga_switcheroo_state state)
 {
@@ -352,6 +455,16 @@ static const struct vga_switcheroo_handler gmux_handler = {
 	.get_client_id = gmux_get_client_id,
 };
 
+/**
+ * DOC: Interrupt
+ *
+ * gmux is also connected to a GPIO pin of the southbridge and thereby is able
+ * to trigger an ACPI GPE. On the MBP5 2008/09 it's GPIO pin 22 of the Nvidia
+ * MCP79, on all following generations it's GPIO pin 6 of the Intel PCH.
+ * The GPE merely signals that an interrupt occurred, the actual type of event
+ * is identified by reading a gmux register.
+ */
+
 static inline void gmux_disable_interrupts(struct apple_gmux_data *gmux_data)
 {
 	gmux_write8(gmux_data, GMUX_PORT_INTERRUPT_ENABLE,
@@ -588,18 +701,20 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 		gmux_data->gpe = -1;
 	}
 
+	apple_gmux_data = gmux_data;
+	init_completion(&gmux_data->powerchange_done);
+	gmux_enable_interrupts(gmux_data);
+
 	if (vga_switcheroo_register_handler(&gmux_handler)) {
 		ret = -ENODEV;
 		goto err_register_handler;
 	}
 
-	init_completion(&gmux_data->powerchange_done);
-	apple_gmux_data = gmux_data;
-	gmux_enable_interrupts(gmux_data);
-
 	return 0;
 
 err_register_handler:
+	gmux_disable_interrupts(gmux_data);
+	apple_gmux_data = NULL;
 	if (gmux_data->gpe >= 0)
 		acpi_disable_gpe(NULL, gmux_data->gpe);
 err_enable_gpe:
diff --git a/drivers/platform/x86/asus-wireless.c b/drivers/platform/x86/asus-wireless.c
new file mode 100644
index 000000000000..9ec721e26532
--- /dev/null
+++ b/drivers/platform/x86/asus-wireless.c
@@ -0,0 +1,84 @@
+/*
+ * Asus Wireless Radio Control Driver
+ *
+ * Copyright (C) 2015-2016 Endless Mobile, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/acpi.h>
+#include <linux/input.h>
+#include <linux/pci_ids.h>
+
+struct asus_wireless_data {
+	struct input_dev *idev;
+};
+
+static void asus_wireless_notify(struct acpi_device *adev, u32 event)
+{
+	struct asus_wireless_data *data = acpi_driver_data(adev);
+
+	dev_dbg(&adev->dev, "event=%#x\n", event);
+	if (event != 0x88) {
+		dev_notice(&adev->dev, "Unknown ASHS event: %#x\n", event);
+		return;
+	}
+	input_report_key(data->idev, KEY_RFKILL, 1);
+	input_report_key(data->idev, KEY_RFKILL, 0);
+	input_sync(data->idev);
+}
+
+static int asus_wireless_add(struct acpi_device *adev)
+{
+	struct asus_wireless_data *data;
+
+	data = devm_kzalloc(&adev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	adev->driver_data = data;
+
+	data->idev = devm_input_allocate_device(&adev->dev);
+	if (!data->idev)
+		return -ENOMEM;
+	data->idev->name = "Asus Wireless Radio Control";
+	data->idev->phys = "asus-wireless/input0";
+	data->idev->id.bustype = BUS_HOST;
+	data->idev->id.vendor = PCI_VENDOR_ID_ASUSTEK;
+	set_bit(EV_KEY, data->idev->evbit);
+	set_bit(KEY_RFKILL, data->idev->keybit);
+	return input_register_device(data->idev);
+}
+
+static int asus_wireless_remove(struct acpi_device *adev)
+{
+	return 0;
+}
+
+static const struct acpi_device_id device_ids[] = {
+	{"ATK4001", 0},
+	{"ATK4002", 0},
+	{"", 0},
+};
+MODULE_DEVICE_TABLE(acpi, device_ids);
+
+static struct acpi_driver asus_wireless_driver = {
+	.name = "Asus Wireless Radio Control Driver",
+	.class = "hotkey",
+	.ids = device_ids,
+	.ops = {
+		.add = asus_wireless_add,
+		.remove = asus_wireless_remove,
+		.notify = asus_wireless_notify,
+	},
+};
+module_acpi_driver(asus_wireless_driver);
+
+MODULE_DESCRIPTION("Asus Wireless Radio Control Driver");
+MODULE_AUTHOR("João Paulo Rechi Vita <jprvita@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index f96f7b865267..a96630d52346 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -56,9 +56,6 @@ MODULE_AUTHOR("Corentin Chary <corentin.chary@gmail.com>, "
 MODULE_DESCRIPTION("Asus Generic WMI Driver");
 MODULE_LICENSE("GPL");
 
-#define to_platform_driver(drv)					\
-	(container_of((drv), struct platform_driver, driver))
-
 #define to_asus_wmi_driver(pdrv)					\
 	(container_of((pdrv), struct asus_wmi_driver, platform_driver))
 
diff --git a/drivers/platform/x86/dell-wmi.c b/drivers/platform/x86/dell-wmi.c
index cb8a9c2a3a1f..368e193c2741 100644
--- a/drivers/platform/x86/dell-wmi.c
+++ b/drivers/platform/x86/dell-wmi.c
@@ -2,6 +2,7 @@
  * Dell WMI hotkeys
  *
  * Copyright (C) 2008 Red Hat <mjg@redhat.com>
+ * Copyright (C) 2014-2015 Pali Rohár <pali.rohar@gmail.com>
  *
  * Portions based on wistron_btns.c:
  * Copyright (C) 2005 Miloslav Trmac <mitr@volny.cz>
@@ -38,12 +39,17 @@
 #include <acpi/video.h>
 
 MODULE_AUTHOR("Matthew Garrett <mjg@redhat.com>");
+MODULE_AUTHOR("Pali Rohár <pali.rohar@gmail.com>");
 MODULE_DESCRIPTION("Dell laptop WMI hotkeys driver");
 MODULE_LICENSE("GPL");
 
 #define DELL_EVENT_GUID "9DBB5994-A997-11DA-B012-B622A1EF5492"
+#define DELL_DESCRIPTOR_GUID "8D9DDCBC-A997-11DA-B012-B622A1EF5492"
+
+static u32 dell_wmi_interface_version;
 
 MODULE_ALIAS("wmi:"DELL_EVENT_GUID);
+MODULE_ALIAS("wmi:"DELL_DESCRIPTOR_GUID);
 
 /*
  * Certain keys are flagged as KE_IGNORE. All of these are either
@@ -116,28 +122,48 @@ struct dell_bios_hotkey_table {
 
 static const struct dell_bios_hotkey_table *dell_bios_hotkey_table;
 
+/* Uninitialized entries here are KEY_RESERVED == 0. */
 static const u16 bios_to_linux_keycode[256] __initconst = {
-
-	KEY_MEDIA,	KEY_NEXTSONG,	KEY_PLAYPAUSE, KEY_PREVIOUSSONG,
-	KEY_STOPCD,	KEY_UNKNOWN,	KEY_UNKNOWN,	KEY_UNKNOWN,
-	KEY_WWW,	KEY_UNKNOWN,	KEY_VOLUMEDOWN, KEY_MUTE,
-	KEY_VOLUMEUP,	KEY_UNKNOWN,	KEY_BATTERY,	KEY_EJECTCD,
-	KEY_UNKNOWN,	KEY_SLEEP,	KEY_PROG1, KEY_BRIGHTNESSDOWN,
-	KEY_BRIGHTNESSUP,	KEY_UNKNOWN,	KEY_KBDILLUMTOGGLE,
-	KEY_UNKNOWN,	KEY_SWITCHVIDEOMODE,	KEY_UNKNOWN, KEY_UNKNOWN,
-	KEY_SWITCHVIDEOMODE,	KEY_UNKNOWN,	KEY_UNKNOWN, KEY_PROG2,
-	KEY_UNKNOWN,	KEY_UNKNOWN,	KEY_UNKNOWN,	KEY_UNKNOWN,
-	KEY_UNKNOWN,	KEY_UNKNOWN,	KEY_UNKNOWN,	KEY_MICMUTE,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, KEY_PROG3
+	[0]	= KEY_MEDIA,
+	[1]	= KEY_NEXTSONG,
+	[2]	= KEY_PLAYPAUSE,
+	[3]	= KEY_PREVIOUSSONG,
+	[4]	= KEY_STOPCD,
+	[5]	= KEY_UNKNOWN,
+	[6]	= KEY_UNKNOWN,
+	[7]	= KEY_UNKNOWN,
+	[8]	= KEY_WWW,
+	[9]	= KEY_UNKNOWN,
+	[10]	= KEY_VOLUMEDOWN,
+	[11]	= KEY_MUTE,
+	[12]	= KEY_VOLUMEUP,
+	[13]	= KEY_UNKNOWN,
+	[14]	= KEY_BATTERY,
+	[15]	= KEY_EJECTCD,
+	[16]	= KEY_UNKNOWN,
+	[17]	= KEY_SLEEP,
+	[18]	= KEY_PROG1,
+	[19]	= KEY_BRIGHTNESSDOWN,
+	[20]	= KEY_BRIGHTNESSUP,
+	[21]	= KEY_UNKNOWN,
+	[22]	= KEY_KBDILLUMTOGGLE,
+	[23]	= KEY_UNKNOWN,
+	[24]	= KEY_SWITCHVIDEOMODE,
+	[25]	= KEY_UNKNOWN,
+	[26]	= KEY_UNKNOWN,
+	[27]	= KEY_SWITCHVIDEOMODE,
+	[28]	= KEY_UNKNOWN,
+	[29]	= KEY_UNKNOWN,
+	[30]	= KEY_PROG2,
+	[31]	= KEY_UNKNOWN,
+	[32]	= KEY_UNKNOWN,
+	[33]	= KEY_UNKNOWN,
+	[34]	= KEY_UNKNOWN,
+	[35]	= KEY_UNKNOWN,
+	[36]	= KEY_UNKNOWN,
+	[37]	= KEY_UNKNOWN,
+	[38]	= KEY_MICMUTE,
+	[255]	= KEY_PROG3,
 };
 
 static struct input_dev *dell_wmi_input_dev;
@@ -149,7 +175,8 @@ static void dell_wmi_process_key(int reported_key)
 	key = sparse_keymap_entry_from_scancode(dell_wmi_input_dev,
 						reported_key);
 	if (!key) {
-		pr_info("Unknown key %x pressed\n", reported_key);
+		pr_info("Unknown key with scancode 0x%x pressed\n",
+			reported_key);
 		return;
 	}
 
@@ -210,6 +237,22 @@ static void dell_wmi_notify(u32 value, void *context)
 
 	buffer_end = buffer_entry + buffer_size;
 
+	/*
+	 * BIOS/ACPI on devices with WMI interface version 0 does not clear
+	 * buffer before filling it. So next time when BIOS/ACPI send WMI event
+	 * which is smaller as previous then it contains garbage in buffer from
+	 * previous event.
+	 *
+	 * BIOS/ACPI on devices with WMI interface version 1 clears buffer and
+	 * sometimes send more events in buffer at one call.
+	 *
+	 * So to prevent reading garbage from buffer we will process only first
+	 * one event on devices with WMI interface version 0.
+	 */
+	if (dell_wmi_interface_version == 0 && buffer_entry < buffer_end)
+		if (buffer_end > buffer_entry + buffer_entry[0] + 1)
+			buffer_end = buffer_entry + buffer_entry[0] + 1;
+
 	while (buffer_entry < buffer_end) {
 
 		len = buffer_entry[0];
@@ -308,9 +351,23 @@ static const struct key_entry * __init dell_wmi_prepare_new_keymap(void)
 	for (i = 0; i < hotkey_num; i++) {
 		const struct dell_bios_keymap_entry *bios_entry =
 					&dell_bios_hotkey_table->keymap[i];
-		u16 keycode = bios_entry->keycode < 256 ?
-				    bios_to_linux_keycode[bios_entry->keycode] :
-				    KEY_RESERVED;
+
+		/* Uninitialized entries are 0 aka KEY_RESERVED. */
+		u16 keycode = (bios_entry->keycode <
+			       ARRAY_SIZE(bios_to_linux_keycode)) ?
+			bios_to_linux_keycode[bios_entry->keycode] :
+			KEY_RESERVED;
+
+		/*
+		 * Log if we find an entry in the DMI table that we don't
+		 * understand.  If this happens, we should figure out what
+		 * the entry means and add it to bios_to_linux_keycode.
+		 */
+		if (keycode == KEY_RESERVED) {
+			pr_info("firmware scancode 0x%x maps to unrecognized keycode 0x%x\n",
+				bios_entry->scancode, bios_entry->keycode);
+			continue;
+		}
 
 		if (keycode == KEY_KBDILLUMTOGGLE)
 			keymap[i].type = KE_IGNORE;
@@ -386,16 +443,87 @@ static void __init find_hk_type(const struct dmi_header *dm, void *dummy)
 	}
 }
 
+/*
+ * Descriptor buffer is 128 byte long and contains:
+ *
+ *       Name             Offset  Length  Value
+ * Vendor Signature          0       4    "DELL"
+ * Object Signature          4       4    " WMI"
+ * WMI Interface Version     8       4    <version>
+ * WMI buffer length        12       4    4096
+ */
+static int __init dell_wmi_check_descriptor_buffer(void)
+{
+	struct acpi_buffer out = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_status status;
+	u32 *buffer;
+
+	status = wmi_query_block(DELL_DESCRIPTOR_GUID, 0, &out);
+	if (ACPI_FAILURE(status)) {
+		pr_err("Cannot read Dell descriptor buffer - %d\n", status);
+		return status;
+	}
+
+	obj = (union acpi_object *)out.pointer;
+	if (!obj) {
+		pr_err("Dell descriptor buffer is empty\n");
+		return -EINVAL;
+	}
+
+	if (obj->type != ACPI_TYPE_BUFFER) {
+		pr_err("Cannot read Dell descriptor buffer\n");
+		kfree(obj);
+		return -EINVAL;
+	}
+
+	if (obj->buffer.length != 128) {
+		pr_err("Dell descriptor buffer has invalid length (%d)\n",
+			obj->buffer.length);
+		if (obj->buffer.length < 16) {
+			kfree(obj);
+			return -EINVAL;
+		}
+	}
+
+	buffer = (u32 *)obj->buffer.pointer;
+
+	if (buffer[0] != 0x4C4C4544 && buffer[1] != 0x494D5720)
+		pr_warn("Dell descriptor buffer has invalid signature (%*ph)\n",
+			8, buffer);
+
+	if (buffer[2] != 0 && buffer[2] != 1)
+		pr_warn("Dell descriptor buffer has unknown version (%d)\n",
+			buffer[2]);
+
+	if (buffer[3] != 4096)
+		pr_warn("Dell descriptor buffer has invalid buffer length (%d)\n",
+			buffer[3]);
+
+	dell_wmi_interface_version = buffer[2];
+
+	pr_info("Detected Dell WMI interface version %u\n",
+		dell_wmi_interface_version);
+
+	kfree(obj);
+	return 0;
+}
+
 static int __init dell_wmi_init(void)
 {
 	int err;
 	acpi_status status;
 
-	if (!wmi_has_guid(DELL_EVENT_GUID)) {
-		pr_warn("No known WMI GUID found\n");
+	if (!wmi_has_guid(DELL_EVENT_GUID) ||
+	    !wmi_has_guid(DELL_DESCRIPTOR_GUID)) {
+		pr_warn("Dell WMI GUID were not found\n");
 		return -ENODEV;
 	}
 
+	err = dell_wmi_check_descriptor_buffer();
+	if (err)
+		return err;
+
 	dmi_walk(find_hk_type, NULL);
 
 	err = dell_wmi_input_setup();
diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
index a313dfc0245f..d78ee151c9e4 100644
--- a/drivers/platform/x86/ideapad-laptop.c
+++ b/drivers/platform/x86/ideapad-laptop.c
@@ -865,6 +865,13 @@ static const struct dmi_system_id no_hw_rfkill_list[] = {
 		},
 	},
 	{
+		.ident = "Lenovo ideapad Y700-17ISK",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad Y700-17ISK"),
+		},
+	},
+	{
 		.ident = "Lenovo Yoga 2 11 / 13 / Pro",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
@@ -893,6 +900,13 @@ static const struct dmi_system_id no_hw_rfkill_list[] = {
 		},
 	},
 	{
+		.ident = "Lenovo Yoga 700",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo YOGA 700"),
+		},
+	},
+	{
 		.ident = "Lenovo Yoga 900",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
diff --git a/drivers/platform/x86/intel-hid.c b/drivers/platform/x86/intel-hid.c
new file mode 100644
index 000000000000..20f0ad9bb9f3
--- /dev/null
+++ b/drivers/platform/x86/intel-hid.c
@@ -0,0 +1,289 @@
+/*
+ *  Intel HID event driver for Windows 8
+ *
+ *  Copyright (C) 2015 Alex Hung <alex.hung@canonical.com>
+ *  Copyright (C) 2015 Andrew Lutomirski <luto@kernel.org>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/input.h>
+#include <linux/platform_device.h>
+#include <linux/input/sparse-keymap.h>
+#include <linux/acpi.h>
+#include <acpi/acpi_bus.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Alex Hung");
+
+static const struct acpi_device_id intel_hid_ids[] = {
+	{"INT33D5", 0},
+	{"", 0},
+};
+
+/* In theory, these are HID usages. */
+static const struct key_entry intel_hid_keymap[] = {
+	/* 1: LSuper (Page 0x07, usage 0xE3) -- unclear what to do */
+	/* 2: Toggle SW_ROTATE_LOCK -- easy to implement if seen in wild */
+	{ KE_KEY, 3, { KEY_NUMLOCK } },
+	{ KE_KEY, 4, { KEY_HOME } },
+	{ KE_KEY, 5, { KEY_END } },
+	{ KE_KEY, 6, { KEY_PAGEUP } },
+	{ KE_KEY, 4, { KEY_PAGEDOWN } },
+	{ KE_KEY, 4, { KEY_HOME } },
+	{ KE_KEY, 8, { KEY_RFKILL } },
+	{ KE_KEY, 9, { KEY_POWER } },
+	{ KE_KEY, 11, { KEY_SLEEP } },
+	/* 13 has two different meanings in the spec -- ignore it. */
+	{ KE_KEY, 14, { KEY_STOPCD } },
+	{ KE_KEY, 15, { KEY_PLAYPAUSE } },
+	{ KE_KEY, 16, { KEY_MUTE } },
+	{ KE_KEY, 17, { KEY_VOLUMEUP } },
+	{ KE_KEY, 18, { KEY_VOLUMEDOWN } },
+	{ KE_KEY, 19, { KEY_BRIGHTNESSUP } },
+	{ KE_KEY, 20, { KEY_BRIGHTNESSDOWN } },
+	/* 27: wake -- needs special handling */
+	{ KE_END },
+};
+
+struct intel_hid_priv {
+	struct input_dev *input_dev;
+};
+
+static int intel_hid_set_enable(struct device *device, int enable)
+{
+	union acpi_object arg0 = { ACPI_TYPE_INTEGER };
+	struct acpi_object_list args = { 1, &arg0 };
+	acpi_status status;
+
+	arg0.integer.value = enable;
+	status = acpi_evaluate_object(ACPI_HANDLE(device), "HDSM", &args, NULL);
+	if (!ACPI_SUCCESS(status)) {
+		dev_warn(device, "failed to %sable hotkeys\n",
+			 enable ? "en" : "dis");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int intel_hid_pl_suspend_handler(struct device *device)
+{
+	intel_hid_set_enable(device, 0);
+	return 0;
+}
+
+static int intel_hid_pl_resume_handler(struct device *device)
+{
+	intel_hid_set_enable(device, 1);
+	return 0;
+}
+
+static const struct dev_pm_ops intel_hid_pl_pm_ops = {
+	.suspend  = intel_hid_pl_suspend_handler,
+	.resume  = intel_hid_pl_resume_handler,
+};
+
+static int intel_hid_input_setup(struct platform_device *device)
+{
+	struct intel_hid_priv *priv = dev_get_drvdata(&device->dev);
+	int ret;
+
+	priv->input_dev = input_allocate_device();
+	if (!priv->input_dev)
+		return -ENOMEM;
+
+	ret = sparse_keymap_setup(priv->input_dev, intel_hid_keymap, NULL);
+	if (ret)
+		goto err_free_device;
+
+	priv->input_dev->dev.parent = &device->dev;
+	priv->input_dev->name = "Intel HID events";
+	priv->input_dev->id.bustype = BUS_HOST;
+	set_bit(KEY_RFKILL, priv->input_dev->keybit);
+
+	ret = input_register_device(priv->input_dev);
+	if (ret)
+		goto err_free_device;
+
+	return 0;
+
+err_free_device:
+		input_free_device(priv->input_dev);
+		return ret;
+}
+
+static void intel_hid_input_destroy(struct platform_device *device)
+{
+	struct intel_hid_priv *priv = dev_get_drvdata(&device->dev);
+
+	input_unregister_device(priv->input_dev);
+}
+
+static void notify_handler(acpi_handle handle, u32 event, void *context)
+{
+	struct platform_device *device = context;
+	struct intel_hid_priv *priv = dev_get_drvdata(&device->dev);
+	unsigned long long ev_index;
+	acpi_status status;
+
+	/* The platform spec only defines one event code: 0xC0. */
+	if (event != 0xc0) {
+		dev_warn(&device->dev, "received unknown event (0x%x)\n",
+			 event);
+		return;
+	}
+
+	status = acpi_evaluate_integer(handle, "HDEM", NULL, &ev_index);
+	if (!ACPI_SUCCESS(status)) {
+		dev_warn(&device->dev, "failed to get event index\n");
+		return;
+	}
+
+	if (!sparse_keymap_report_event(priv->input_dev, ev_index, 1, true))
+		dev_info(&device->dev, "unknown event index 0x%llx\n",
+			 ev_index);
+}
+
+static int intel_hid_probe(struct platform_device *device)
+{
+	acpi_handle handle = ACPI_HANDLE(&device->dev);
+	struct intel_hid_priv *priv;
+	unsigned long long mode;
+	acpi_status status;
+	int err;
+
+	status = acpi_evaluate_integer(handle, "HDMM", NULL, &mode);
+	if (!ACPI_SUCCESS(status)) {
+		dev_warn(&device->dev, "failed to read mode\n");
+		return -ENODEV;
+	}
+
+	if (mode != 0) {
+		/*
+		 * This driver only implements "simple" mode.  There appear
+		 * to be no other modes, but we should be paranoid and check
+		 * for compatibility.
+		 */
+		dev_info(&device->dev, "platform is not in simple mode\n");
+		return -ENODEV;
+	}
+
+	priv = devm_kzalloc(&device->dev,
+			    sizeof(struct intel_hid_priv *), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	dev_set_drvdata(&device->dev, priv);
+
+	err = intel_hid_input_setup(device);
+	if (err) {
+		pr_err("Failed to setup Intel HID hotkeys\n");
+		return err;
+	}
+
+	status = acpi_install_notify_handler(handle,
+					     ACPI_DEVICE_NOTIFY,
+					     notify_handler,
+					     device);
+	if (ACPI_FAILURE(status)) {
+		err = -EBUSY;
+		goto err_remove_input;
+	}
+
+	err = intel_hid_set_enable(&device->dev, 1);
+	if (err)
+		goto err_remove_notify;
+
+	return 0;
+
+err_remove_notify:
+	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
+
+err_remove_input:
+	intel_hid_input_destroy(device);
+
+	return err;
+}
+
+static int intel_hid_remove(struct platform_device *device)
+{
+	acpi_handle handle = ACPI_HANDLE(&device->dev);
+
+	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
+	intel_hid_input_destroy(device);
+	intel_hid_set_enable(&device->dev, 0);
+	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
+
+	/*
+	 * Even if we failed to shut off the event stream, we can still
+	 * safely detach from the device.
+	 */
+	return 0;
+}
+
+static struct platform_driver intel_hid_pl_driver = {
+	.driver = {
+		.name = "intel-hid",
+		.acpi_match_table = intel_hid_ids,
+		.pm = &intel_hid_pl_pm_ops,
+	},
+	.probe = intel_hid_probe,
+	.remove = intel_hid_remove,
+};
+MODULE_DEVICE_TABLE(acpi, intel_hid_ids);
+
+/*
+ * Unfortunately, some laptops provide a _HID="INT33D5" device with
+ * _CID="PNP0C02".  This causes the pnpacpi scan driver to claim the
+ * ACPI node, so no platform device will be created.  The pnpacpi
+ * driver rejects this device in subsequent processing, so no physical
+ * node is created at all.
+ *
+ * As a workaround until the ACPI core figures out how to handle
+ * this corner case, manually ask the ACPI platform device code to
+ * claim the ACPI node.
+ */
+static acpi_status __init
+check_acpi_dev(acpi_handle handle, u32 lvl, void *context, void **rv)
+{
+	const struct acpi_device_id *ids = context;
+	struct acpi_device *dev;
+
+	if (acpi_bus_get_device(handle, &dev) != 0)
+		return AE_OK;
+
+	if (acpi_match_device_ids(dev, ids) == 0)
+		if (acpi_create_platform_device(dev))
+			dev_info(&dev->dev,
+				 "intel-hid: created platform device\n");
+
+	return AE_OK;
+}
+
+static int __init intel_hid_init(void)
+{
+	acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+			    ACPI_UINT32_MAX, check_acpi_dev, NULL,
+			    (void *)intel_hid_ids, NULL);
+
+	return platform_driver_register(&intel_hid_pl_driver);
+}
+module_init(intel_hid_init);
+
+static void __exit intel_hid_exit(void)
+{
+	platform_driver_unregister(&intel_hid_pl_driver);
+}
+module_exit(intel_hid_exit);
diff --git a/drivers/platform/x86/intel_pmc_ipc.c b/drivers/platform/x86/intel_pmc_ipc.c
index 28b2a12bb26d..092519e37de6 100644
--- a/drivers/platform/x86/intel_pmc_ipc.c
+++ b/drivers/platform/x86/intel_pmc_ipc.c
@@ -68,8 +68,13 @@
 #define PLAT_RESOURCE_IPC_INDEX		0
 #define PLAT_RESOURCE_IPC_SIZE		0x1000
 #define PLAT_RESOURCE_GCR_SIZE		0x1000
-#define PLAT_RESOURCE_PUNIT_DATA_INDEX	1
-#define PLAT_RESOURCE_PUNIT_INTER_INDEX	2
+#define PLAT_RESOURCE_BIOS_DATA_INDEX	1
+#define PLAT_RESOURCE_BIOS_IFACE_INDEX	2
+#define PLAT_RESOURCE_TELEM_SSRAM_INDEX	3
+#define PLAT_RESOURCE_ISP_DATA_INDEX	4
+#define PLAT_RESOURCE_ISP_IFACE_INDEX	5
+#define PLAT_RESOURCE_GTD_DATA_INDEX	6
+#define PLAT_RESOURCE_GTD_IFACE_INDEX	7
 #define PLAT_RESOURCE_ACPI_IO_INDEX	0
 
 /*
@@ -84,6 +89,10 @@
 #define TCO_BASE_OFFSET			0x60
 #define TCO_REGS_SIZE			16
 #define PUNIT_DEVICE_NAME		"intel_punit_ipc"
+#define TELEMETRY_DEVICE_NAME		"intel_telemetry"
+#define TELEM_SSRAM_SIZE		240
+#define TELEM_PMC_SSRAM_OFFSET		0x1B00
+#define TELEM_PUNIT_SSRAM_OFFSET	0x1A00
 
 static const int iTCO_version = 3;
 
@@ -105,11 +114,15 @@ static struct intel_pmc_ipc_dev {
 	int gcr_size;
 
 	/* punit */
-	resource_size_t punit_base;
-	int punit_size;
-	resource_size_t punit_base2;
-	int punit_size2;
 	struct platform_device *punit_dev;
+
+	/* Telemetry */
+	resource_size_t telem_pmc_ssram_base;
+	resource_size_t telem_punit_ssram_base;
+	int telem_pmc_ssram_size;
+	int telem_punit_ssram_size;
+	u8 telem_res_inval;
+	struct platform_device *telemetry_dev;
 } ipcdev;
 
 static char *ipc_err_sources[] = {
@@ -444,9 +457,22 @@ static const struct attribute_group intel_ipc_group = {
 	.attrs = intel_ipc_attrs,
 };
 
-#define PUNIT_RESOURCE_INTER		1
-static struct resource punit_res[] = {
-	/* Punit */
+static struct resource punit_res_array[] = {
+	/* Punit BIOS */
+	{
+		.flags = IORESOURCE_MEM,
+	},
+	{
+		.flags = IORESOURCE_MEM,
+	},
+	/* Punit ISP */
+	{
+		.flags = IORESOURCE_MEM,
+	},
+	{
+		.flags = IORESOURCE_MEM,
+	},
+	/* Punit GTD */
 	{
 		.flags = IORESOURCE_MEM,
 	},
@@ -478,10 +504,21 @@ static struct itco_wdt_platform_data tco_info = {
 	.version = 3,
 };
 
+#define TELEMETRY_RESOURCE_PUNIT_SSRAM	0
+#define TELEMETRY_RESOURCE_PMC_SSRAM	1
+static struct resource telemetry_res[] = {
+	/*Telemetry*/
+	{
+		.flags = IORESOURCE_MEM,
+	},
+	{
+		.flags = IORESOURCE_MEM,
+	},
+};
+
 static int ipc_create_punit_device(void)
 {
 	struct platform_device *pdev;
-	struct resource *res;
 	int ret;
 
 	pdev = platform_device_alloc(PUNIT_DEVICE_NAME, -1);
@@ -491,17 +528,8 @@ static int ipc_create_punit_device(void)
 	}
 
 	pdev->dev.parent = ipcdev.dev;
-
-	res = punit_res;
-	res->start = ipcdev.punit_base;
-	res->end = res->start + ipcdev.punit_size - 1;
-
-	res = punit_res + PUNIT_RESOURCE_INTER;
-	res->start = ipcdev.punit_base2;
-	res->end = res->start + ipcdev.punit_size2 - 1;
-
-	ret = platform_device_add_resources(pdev, punit_res,
-					    ARRAY_SIZE(punit_res));
+	ret = platform_device_add_resources(pdev, punit_res_array,
+					    ARRAY_SIZE(punit_res_array));
 	if (ret) {
 		dev_err(ipcdev.dev, "Failed to add platform punit resources\n");
 		goto err;
@@ -571,6 +599,51 @@ err:
 	return ret;
 }
 
+static int ipc_create_telemetry_device(void)
+{
+	struct platform_device *pdev;
+	struct resource *res;
+	int ret;
+
+	pdev = platform_device_alloc(TELEMETRY_DEVICE_NAME, -1);
+	if (!pdev) {
+		dev_err(ipcdev.dev,
+			"Failed to allocate telemetry platform device\n");
+		return -ENOMEM;
+	}
+
+	pdev->dev.parent = ipcdev.dev;
+
+	res = telemetry_res + TELEMETRY_RESOURCE_PUNIT_SSRAM;
+	res->start = ipcdev.telem_punit_ssram_base;
+	res->end = res->start + ipcdev.telem_punit_ssram_size - 1;
+
+	res = telemetry_res + TELEMETRY_RESOURCE_PMC_SSRAM;
+	res->start = ipcdev.telem_pmc_ssram_base;
+	res->end = res->start + ipcdev.telem_pmc_ssram_size - 1;
+
+	ret = platform_device_add_resources(pdev, telemetry_res,
+					    ARRAY_SIZE(telemetry_res));
+	if (ret) {
+		dev_err(ipcdev.dev,
+			"Failed to add telemetry platform resources\n");
+		goto err;
+	}
+
+	ret = platform_device_add(pdev);
+	if (ret) {
+		dev_err(ipcdev.dev,
+			"Failed to add telemetry platform device\n");
+		goto err;
+	}
+	ipcdev.telemetry_dev = pdev;
+
+	return 0;
+err:
+	platform_device_put(pdev);
+	return ret;
+}
+
 static int ipc_create_pmc_devices(void)
 {
 	int ret;
@@ -585,12 +658,20 @@ static int ipc_create_pmc_devices(void)
 		dev_err(ipcdev.dev, "Failed to add punit platform device\n");
 		platform_device_unregister(ipcdev.tco_dev);
 	}
+
+	if (!ipcdev.telem_res_inval) {
+		ret = ipc_create_telemetry_device();
+		if (ret)
+			dev_warn(ipcdev.dev,
+				"Failed to add telemetry platform device\n");
+	}
+
 	return ret;
 }
 
 static int ipc_plat_get_res(struct platform_device *pdev)
 {
-	struct resource *res;
+	struct resource *res, *punit_res;
 	void __iomem *addr;
 	int size;
 
@@ -603,32 +684,68 @@ static int ipc_plat_get_res(struct platform_device *pdev)
 	size = resource_size(res);
 	ipcdev.acpi_io_base = res->start;
 	ipcdev.acpi_io_size = size;
-	dev_info(&pdev->dev, "io res: %llx %x\n",
-		 (long long)res->start, (int)resource_size(res));
+	dev_info(&pdev->dev, "io res: %pR\n", res);
 
+	/* This is index 0 to cover BIOS data register */
+	punit_res = punit_res_array;
 	res = platform_get_resource(pdev, IORESOURCE_MEM,
-				    PLAT_RESOURCE_PUNIT_DATA_INDEX);
+				    PLAT_RESOURCE_BIOS_DATA_INDEX);
 	if (!res) {
-		dev_err(&pdev->dev, "Failed to get punit resource\n");
+		dev_err(&pdev->dev, "Failed to get res of punit BIOS data\n");
 		return -ENXIO;
 	}
-	size = resource_size(res);
-	ipcdev.punit_base = res->start;
-	ipcdev.punit_size = size;
-	dev_info(&pdev->dev, "punit data res: %llx %x\n",
-		 (long long)res->start, (int)resource_size(res));
+	*punit_res = *res;
+	dev_info(&pdev->dev, "punit BIOS data res: %pR\n", res);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM,
-				    PLAT_RESOURCE_PUNIT_INTER_INDEX);
+				    PLAT_RESOURCE_BIOS_IFACE_INDEX);
 	if (!res) {
-		dev_err(&pdev->dev, "Failed to get punit inter resource\n");
+		dev_err(&pdev->dev, "Failed to get res of punit BIOS iface\n");
 		return -ENXIO;
 	}
-	size = resource_size(res);
-	ipcdev.punit_base2 = res->start;
-	ipcdev.punit_size2 = size;
-	dev_info(&pdev->dev, "punit interface res: %llx %x\n",
-		 (long long)res->start, (int)resource_size(res));
+	/* This is index 1 to cover BIOS interface register */
+	*++punit_res = *res;
+	dev_info(&pdev->dev, "punit BIOS interface res: %pR\n", res);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM,
+				    PLAT_RESOURCE_ISP_DATA_INDEX);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get res of punit ISP data\n");
+		return -ENXIO;
+	}
+	/* This is index 2 to cover ISP data register */
+	*++punit_res = *res;
+	dev_info(&pdev->dev, "punit ISP data res: %pR\n", res);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM,
+				    PLAT_RESOURCE_ISP_IFACE_INDEX);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get res of punit ISP iface\n");
+		return -ENXIO;
+	}
+	/* This is index 3 to cover ISP interface register */
+	*++punit_res = *res;
+	dev_info(&pdev->dev, "punit ISP interface res: %pR\n", res);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM,
+				    PLAT_RESOURCE_GTD_DATA_INDEX);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get res of punit GTD data\n");
+		return -ENXIO;
+	}
+	/* This is index 4 to cover GTD data register */
+	*++punit_res = *res;
+	dev_info(&pdev->dev, "punit GTD data res: %pR\n", res);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM,
+				    PLAT_RESOURCE_GTD_IFACE_INDEX);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get res of punit GTD iface\n");
+		return -ENXIO;
+	}
+	/* This is index 5 to cover GTD interface register */
+	*++punit_res = *res;
+	dev_info(&pdev->dev, "punit GTD interface res: %pR\n", res);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM,
 				    PLAT_RESOURCE_IPC_INDEX);
@@ -651,8 +768,23 @@ static int ipc_plat_get_res(struct platform_device *pdev)
 
 	ipcdev.gcr_base = res->start + size;
 	ipcdev.gcr_size = PLAT_RESOURCE_GCR_SIZE;
-	dev_info(&pdev->dev, "ipc res: %llx %x\n",
-		 (long long)res->start, (int)resource_size(res));
+	dev_info(&pdev->dev, "ipc res: %pR\n", res);
+
+	ipcdev.telem_res_inval = 0;
+	res = platform_get_resource(pdev, IORESOURCE_MEM,
+				    PLAT_RESOURCE_TELEM_SSRAM_INDEX);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get telemetry ssram resource\n");
+		ipcdev.telem_res_inval = 1;
+	} else {
+		ipcdev.telem_punit_ssram_base = res->start +
+						TELEM_PUNIT_SSRAM_OFFSET;
+		ipcdev.telem_punit_ssram_size = TELEM_SSRAM_SIZE;
+		ipcdev.telem_pmc_ssram_base = res->start +
+						TELEM_PMC_SSRAM_OFFSET;
+		ipcdev.telem_pmc_ssram_size = TELEM_SSRAM_SIZE;
+		dev_info(&pdev->dev, "telemetry ssram res: %pR\n", res);
+	}
 
 	return 0;
 }
@@ -711,6 +843,7 @@ err_sys:
 err_irq:
 	platform_device_unregister(ipcdev.tco_dev);
 	platform_device_unregister(ipcdev.punit_dev);
+	platform_device_unregister(ipcdev.telemetry_dev);
 err_device:
 	iounmap(ipcdev.ipc_base);
 	res = platform_get_resource(pdev, IORESOURCE_MEM,
@@ -728,6 +861,7 @@ static int ipc_plat_remove(struct platform_device *pdev)
 	free_irq(ipcdev.irq, &ipcdev);
 	platform_device_unregister(ipcdev.tco_dev);
 	platform_device_unregister(ipcdev.punit_dev);
+	platform_device_unregister(ipcdev.telemetry_dev);
 	iounmap(ipcdev.ipc_base);
 	res = platform_get_resource(pdev, IORESOURCE_MEM,
 				    PLAT_RESOURCE_IPC_INDEX);
diff --git a/drivers/platform/x86/intel_pmic_gpio.c b/drivers/platform/x86/intel_pmic_gpio.c
index 709f0afdafa8..0e73fd10ba72 100644
--- a/drivers/platform/x86/intel_pmic_gpio.c
+++ b/drivers/platform/x86/intel_pmic_gpio.c
@@ -274,11 +274,11 @@ static int platform_pmic_gpio_probe(struct platform_device *pdev)
 	pg->chip.base = pdata->gpio_base;
 	pg->chip.ngpio = NUM_GPIO;
 	pg->chip.can_sleep = 1;
-	pg->chip.dev = dev;
+	pg->chip.parent = dev;
 
 	mutex_init(&pg->buslock);
 
-	pg->chip.dev = dev;
+	pg->chip.parent = dev;
 	retval = gpiochip_add(&pg->chip);
 	if (retval) {
 		pr_err("Can not add pmic gpio chip\n");
diff --git a/drivers/platform/x86/intel_punit_ipc.c b/drivers/platform/x86/intel_punit_ipc.c
new file mode 100644
index 000000000000..bd875409a02d
--- /dev/null
+++ b/drivers/platform/x86/intel_punit_ipc.c
@@ -0,0 +1,342 @@
+/*
+ * Driver for the Intel P-Unit Mailbox IPC mechanism
+ *
+ * (C) Copyright 2015 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * The heart of the P-Unit is the Foxton microcontroller and its firmware,
+ * which provide mailbox interface for power management usage.
+ */
+
+#include <linux/module.h>
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <asm/intel_punit_ipc.h>
+
+/* IPC Mailbox registers */
+#define OFFSET_DATA_LOW		0x0
+#define OFFSET_DATA_HIGH	0x4
+/* bit field of interface register */
+#define	CMD_RUN			BIT(31)
+#define	CMD_ERRCODE_MASK	GENMASK(7, 0)
+#define	CMD_PARA1_SHIFT		8
+#define	CMD_PARA2_SHIFT		16
+
+#define CMD_TIMEOUT_SECONDS	1
+
+enum {
+	BASE_DATA = 0,
+	BASE_IFACE,
+	BASE_MAX,
+};
+
+typedef struct {
+	struct device *dev;
+	struct mutex lock;
+	int irq;
+	struct completion cmd_complete;
+	/* base of interface and data registers */
+	void __iomem *base[RESERVED_IPC][BASE_MAX];
+	IPC_TYPE type;
+} IPC_DEV;
+
+static IPC_DEV *punit_ipcdev;
+
+static inline u32 ipc_read_status(IPC_DEV *ipcdev, IPC_TYPE type)
+{
+	return readl(ipcdev->base[type][BASE_IFACE]);
+}
+
+static inline void ipc_write_cmd(IPC_DEV *ipcdev, IPC_TYPE type, u32 cmd)
+{
+	writel(cmd, ipcdev->base[type][BASE_IFACE]);
+}
+
+static inline u32 ipc_read_data_low(IPC_DEV *ipcdev, IPC_TYPE type)
+{
+	return readl(ipcdev->base[type][BASE_DATA] + OFFSET_DATA_LOW);
+}
+
+static inline u32 ipc_read_data_high(IPC_DEV *ipcdev, IPC_TYPE type)
+{
+	return readl(ipcdev->base[type][BASE_DATA] + OFFSET_DATA_HIGH);
+}
+
+static inline void ipc_write_data_low(IPC_DEV *ipcdev, IPC_TYPE type, u32 data)
+{
+	writel(data, ipcdev->base[type][BASE_DATA] + OFFSET_DATA_LOW);
+}
+
+static inline void ipc_write_data_high(IPC_DEV *ipcdev, IPC_TYPE type, u32 data)
+{
+	writel(data, ipcdev->base[type][BASE_DATA] + OFFSET_DATA_HIGH);
+}
+
+static const char *ipc_err_string(int error)
+{
+	if (error == IPC_PUNIT_ERR_SUCCESS)
+		return "no error";
+	else if (error == IPC_PUNIT_ERR_INVALID_CMD)
+		return "invalid command";
+	else if (error == IPC_PUNIT_ERR_INVALID_PARAMETER)
+		return "invalid parameter";
+	else if (error == IPC_PUNIT_ERR_CMD_TIMEOUT)
+		return "command timeout";
+	else if (error == IPC_PUNIT_ERR_CMD_LOCKED)
+		return "command locked";
+	else if (error == IPC_PUNIT_ERR_INVALID_VR_ID)
+		return "invalid vr id";
+	else if (error == IPC_PUNIT_ERR_VR_ERR)
+		return "vr error";
+	else
+		return "unknown error";
+}
+
+static int intel_punit_ipc_check_status(IPC_DEV *ipcdev, IPC_TYPE type)
+{
+	int loops = CMD_TIMEOUT_SECONDS * USEC_PER_SEC;
+	int errcode;
+	int status;
+
+	if (ipcdev->irq) {
+		if (!wait_for_completion_timeout(&ipcdev->cmd_complete,
+						 CMD_TIMEOUT_SECONDS * HZ)) {
+			dev_err(ipcdev->dev, "IPC timed out\n");
+			return -ETIMEDOUT;
+		}
+	} else {
+		while ((ipc_read_status(ipcdev, type) & CMD_RUN) && --loops)
+			udelay(1);
+		if (!loops) {
+			dev_err(ipcdev->dev, "IPC timed out\n");
+			return -ETIMEDOUT;
+		}
+	}
+
+	status = ipc_read_status(ipcdev, type);
+	errcode = status & CMD_ERRCODE_MASK;
+	if (errcode) {
+		dev_err(ipcdev->dev, "IPC failed: %s, IPC_STS=0x%x\n",
+			ipc_err_string(errcode), status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * intel_punit_ipc_simple_command() - Simple IPC command
+ * @cmd:	IPC command code.
+ * @para1:	First 8bit parameter, set 0 if not used.
+ * @para2:	Second 8bit parameter, set 0 if not used.
+ *
+ * Send a IPC command to P-Unit when there is no data transaction
+ *
+ * Return:	IPC error code or 0 on success.
+ */
+int intel_punit_ipc_simple_command(int cmd, int para1, int para2)
+{
+	IPC_DEV *ipcdev = punit_ipcdev;
+	IPC_TYPE type;
+	u32 val;
+	int ret;
+
+	mutex_lock(&ipcdev->lock);
+
+	reinit_completion(&ipcdev->cmd_complete);
+	type = (cmd & IPC_PUNIT_CMD_TYPE_MASK) >> IPC_TYPE_OFFSET;
+
+	val = cmd & ~IPC_PUNIT_CMD_TYPE_MASK;
+	val |= CMD_RUN | para2 << CMD_PARA2_SHIFT | para1 << CMD_PARA1_SHIFT;
+	ipc_write_cmd(ipcdev, type, val);
+	ret = intel_punit_ipc_check_status(ipcdev, type);
+
+	mutex_unlock(&ipcdev->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(intel_punit_ipc_simple_command);
+
+/**
+ * intel_punit_ipc_command() - IPC command with data and pointers
+ * @cmd:	IPC command code.
+ * @para1:	First 8bit parameter, set 0 if not used.
+ * @para2:	Second 8bit parameter, set 0 if not used.
+ * @in:		Input data, 32bit for BIOS cmd, two 32bit for GTD and ISPD.
+ * @out:	Output data.
+ *
+ * Send a IPC command to P-Unit with data transaction
+ *
+ * Return:	IPC error code or 0 on success.
+ */
+int intel_punit_ipc_command(u32 cmd, u32 para1, u32 para2, u32 *in, u32 *out)
+{
+	IPC_DEV *ipcdev = punit_ipcdev;
+	IPC_TYPE type;
+	u32 val;
+	int ret;
+
+	mutex_lock(&ipcdev->lock);
+
+	reinit_completion(&ipcdev->cmd_complete);
+	type = (cmd & IPC_PUNIT_CMD_TYPE_MASK) >> IPC_TYPE_OFFSET;
+
+	if (in) {
+		ipc_write_data_low(ipcdev, type, *in);
+		if (type == GTDRIVER_IPC || type == ISPDRIVER_IPC)
+			ipc_write_data_high(ipcdev, type, *++in);
+	}
+
+	val = cmd & ~IPC_PUNIT_CMD_TYPE_MASK;
+	val |= CMD_RUN | para2 << CMD_PARA2_SHIFT | para1 << CMD_PARA1_SHIFT;
+	ipc_write_cmd(ipcdev, type, val);
+
+	ret = intel_punit_ipc_check_status(ipcdev, type);
+	if (ret)
+		goto out;
+
+	if (out) {
+		*out = ipc_read_data_low(ipcdev, type);
+		if (type == GTDRIVER_IPC || type == ISPDRIVER_IPC)
+			*++out = ipc_read_data_high(ipcdev, type);
+	}
+
+out:
+	mutex_unlock(&ipcdev->lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(intel_punit_ipc_command);
+
+static irqreturn_t intel_punit_ioc(int irq, void *dev_id)
+{
+	IPC_DEV *ipcdev = dev_id;
+
+	complete(&ipcdev->cmd_complete);
+	return IRQ_HANDLED;
+}
+
+static int intel_punit_get_bars(struct platform_device *pdev)
+{
+	struct resource *res;
+	void __iomem *addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[BIOS_IPC][BASE_DATA] = addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[BIOS_IPC][BASE_IFACE] = addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[ISPDRIVER_IPC][BASE_DATA] = addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 3);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[ISPDRIVER_IPC][BASE_IFACE] = addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 4);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[GTDRIVER_IPC][BASE_DATA] = addr;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 5);
+	addr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+	punit_ipcdev->base[GTDRIVER_IPC][BASE_IFACE] = addr;
+
+	return 0;
+}
+
+static int intel_punit_ipc_probe(struct platform_device *pdev)
+{
+	int irq, ret;
+
+	punit_ipcdev = devm_kzalloc(&pdev->dev,
+				    sizeof(*punit_ipcdev), GFP_KERNEL);
+	if (!punit_ipcdev)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, punit_ipcdev);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		punit_ipcdev->irq = 0;
+		dev_warn(&pdev->dev, "Invalid IRQ, using polling mode\n");
+	} else {
+		ret = devm_request_irq(&pdev->dev, irq, intel_punit_ioc,
+				       IRQF_NO_SUSPEND, "intel_punit_ipc",
+				       &punit_ipcdev);
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to request irq: %d\n", irq);
+			return ret;
+		}
+		punit_ipcdev->irq = irq;
+	}
+
+	ret = intel_punit_get_bars(pdev);
+	if (ret)
+		goto out;
+
+	punit_ipcdev->dev = &pdev->dev;
+	mutex_init(&punit_ipcdev->lock);
+	init_completion(&punit_ipcdev->cmd_complete);
+
+out:
+	return ret;
+}
+
+static int intel_punit_ipc_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static const struct acpi_device_id punit_ipc_acpi_ids[] = {
+	{ "INT34D4", 0 },
+	{ }
+};
+
+static struct platform_driver intel_punit_ipc_driver = {
+	.probe = intel_punit_ipc_probe,
+	.remove = intel_punit_ipc_remove,
+	.driver = {
+		.name = "intel_punit_ipc",
+		.acpi_match_table = ACPI_PTR(punit_ipc_acpi_ids),
+	},
+};
+
+static int __init intel_punit_ipc_init(void)
+{
+	return platform_driver_register(&intel_punit_ipc_driver);
+}
+
+static void __exit intel_punit_ipc_exit(void)
+{
+	platform_driver_unregister(&intel_punit_ipc_driver);
+}
+
+MODULE_AUTHOR("Zha Qipeng <qipeng.zha@intel.com>");
+MODULE_DESCRIPTION("Intel P-Unit IPC driver");
+MODULE_LICENSE("GPL v2");
+
+/* Some modules are dependent on this, so init earlier */
+fs_initcall(intel_punit_ipc_init);
+module_exit(intel_punit_ipc_exit);
diff --git a/drivers/platform/x86/intel_telemetry_core.c b/drivers/platform/x86/intel_telemetry_core.c
new file mode 100644
index 000000000000..a695a436a1c3
--- /dev/null
+++ b/drivers/platform/x86/intel_telemetry_core.c
@@ -0,0 +1,464 @@
+/*
+ * Intel SoC Core Telemetry Driver
+ * Copyright (C) 2015, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * Telemetry Framework provides platform related PM and performance statistics.
+ * This file provides the core telemetry API implementation.
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+
+#include <asm/intel_telemetry.h>
+
+#define DRIVER_NAME "intel_telemetry_core"
+
+struct telemetry_core_config {
+	struct telemetry_plt_config *plt_config;
+	struct telemetry_core_ops *telem_ops;
+};
+
+static struct telemetry_core_config telm_core_conf;
+
+static int telemetry_def_update_events(struct telemetry_evtconfig pss_evtconfig,
+				      struct telemetry_evtconfig ioss_evtconfig)
+{
+	return 0;
+}
+
+static int telemetry_def_set_sampling_period(u8 pss_period, u8 ioss_period)
+{
+	return 0;
+}
+
+static int telemetry_def_get_sampling_period(u8 *pss_min_period,
+					     u8 *pss_max_period,
+					     u8 *ioss_min_period,
+					     u8 *ioss_max_period)
+{
+	return 0;
+}
+
+static int telemetry_def_get_eventconfig(
+			struct telemetry_evtconfig *pss_evtconfig,
+			struct telemetry_evtconfig *ioss_evtconfig,
+			int pss_len, int ioss_len)
+{
+	return 0;
+}
+
+static int telemetry_def_get_trace_verbosity(enum telemetry_unit telem_unit,
+					     u32 *verbosity)
+{
+	return 0;
+}
+
+
+static int telemetry_def_set_trace_verbosity(enum telemetry_unit telem_unit,
+					     u32 verbosity)
+{
+	return 0;
+}
+
+static int telemetry_def_raw_read_eventlog(enum telemetry_unit telem_unit,
+					   struct telemetry_evtlog *evtlog,
+					   int len, int log_all_evts)
+{
+	return 0;
+}
+
+static int telemetry_def_read_eventlog(enum telemetry_unit telem_unit,
+				       struct telemetry_evtlog *evtlog,
+				       int len, int log_all_evts)
+{
+	return 0;
+}
+
+static int telemetry_def_add_events(u8 num_pss_evts, u8 num_ioss_evts,
+				    u32 *pss_evtmap, u32 *ioss_evtmap)
+{
+	return 0;
+}
+
+static int telemetry_def_reset_events(void)
+{
+	return 0;
+}
+
+static struct telemetry_core_ops telm_defpltops = {
+	.set_sampling_period = telemetry_def_set_sampling_period,
+	.get_sampling_period = telemetry_def_get_sampling_period,
+	.get_trace_verbosity = telemetry_def_get_trace_verbosity,
+	.set_trace_verbosity = telemetry_def_set_trace_verbosity,
+	.raw_read_eventlog = telemetry_def_raw_read_eventlog,
+	.get_eventconfig = telemetry_def_get_eventconfig,
+	.read_eventlog = telemetry_def_read_eventlog,
+	.update_events = telemetry_def_update_events,
+	.reset_events = telemetry_def_reset_events,
+	.add_events = telemetry_def_add_events,
+};
+
+/**
+ * telemetry_update_events() - Update telemetry Configuration
+ * @pss_evtconfig: PSS related config. No change if num_evts = 0.
+ * @pss_evtconfig: IOSS related config. No change if num_evts = 0.
+ *
+ * This API updates the IOSS & PSS Telemetry configuration. Old config
+ * is overwritten. Call telemetry_reset_events when logging is over
+ * All sample period values should be in the form of:
+ * bits[6:3] -> value; bits [0:2]-> Exponent; Period = (Value *16^Exponent)
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_update_events(struct telemetry_evtconfig pss_evtconfig,
+			    struct telemetry_evtconfig ioss_evtconfig)
+{
+	return telm_core_conf.telem_ops->update_events(pss_evtconfig,
+						       ioss_evtconfig);
+}
+EXPORT_SYMBOL_GPL(telemetry_update_events);
+
+
+/**
+ * telemetry_set_sampling_period() - Sets the IOSS & PSS sampling period
+ * @pss_period:  placeholder for PSS Period to be set.
+ *		 Set to 0 if not required to be updated
+ * @ioss_period: placeholder for IOSS Period to be set
+ *		 Set to 0 if not required to be updated
+ *
+ * All values should be in the form of:
+ * bits[6:3] -> value; bits [0:2]-> Exponent; Period = (Value *16^Exponent)
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_set_sampling_period(u8 pss_period, u8 ioss_period)
+{
+	return telm_core_conf.telem_ops->set_sampling_period(pss_period,
+							     ioss_period);
+}
+EXPORT_SYMBOL_GPL(telemetry_set_sampling_period);
+
+/**
+ * telemetry_get_sampling_period() - Get IOSS & PSS min & max sampling period
+ * @pss_min_period:  placeholder for PSS Min Period supported
+ * @pss_max_period:  placeholder for PSS Max Period supported
+ * @ioss_min_period: placeholder for IOSS Min Period supported
+ * @ioss_max_period: placeholder for IOSS Max Period supported
+ *
+ * All values should be in the form of:
+ * bits[6:3] -> value; bits [0:2]-> Exponent; Period = (Value *16^Exponent)
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_get_sampling_period(u8 *pss_min_period, u8 *pss_max_period,
+				  u8 *ioss_min_period, u8 *ioss_max_period)
+{
+	return telm_core_conf.telem_ops->get_sampling_period(pss_min_period,
+							     pss_max_period,
+							     ioss_min_period,
+							     ioss_max_period);
+}
+EXPORT_SYMBOL_GPL(telemetry_get_sampling_period);
+
+
+/**
+ * telemetry_reset_events() - Restore the IOSS & PSS configuration to default
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_reset_events(void)
+{
+	return telm_core_conf.telem_ops->reset_events();
+}
+EXPORT_SYMBOL_GPL(telemetry_reset_events);
+
+/**
+ * telemetry_get_eventconfig() - Returns the pss and ioss events enabled
+ * @pss_evtconfig: Pointer to PSS related configuration.
+ * @pss_evtconfig: Pointer to IOSS related configuration.
+ * @pss_len:	   Number of u32 elements allocated for pss_evtconfig array
+ * @ioss_len:	   Number of u32 elements allocated for ioss_evtconfig array
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_get_eventconfig(struct telemetry_evtconfig *pss_evtconfig,
+			      struct telemetry_evtconfig *ioss_evtconfig,
+			      int pss_len, int ioss_len)
+{
+	return telm_core_conf.telem_ops->get_eventconfig(pss_evtconfig,
+							 ioss_evtconfig,
+							 pss_len, ioss_len);
+}
+EXPORT_SYMBOL_GPL(telemetry_get_eventconfig);
+
+/**
+ * telemetry_add_events() - Add IOSS & PSS configuration to existing settings.
+ * @num_pss_evts:  Number of PSS Events (<29) in pss_evtmap. Can be 0.
+ * @num_ioss_evts: Number of IOSS Events (<29) in ioss_evtmap. Can be 0.
+ * @pss_evtmap:    Array of PSS Event-IDs to Enable
+ * @ioss_evtmap:   Array of PSS Event-IDs to Enable
+ *
+ * Events are appended to Old Configuration. In case of total events > 28, it
+ * returns error. Call telemetry_reset_events to reset after eventlog done
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_add_events(u8 num_pss_evts, u8 num_ioss_evts,
+			 u32 *pss_evtmap, u32 *ioss_evtmap)
+{
+	return telm_core_conf.telem_ops->add_events(num_pss_evts,
+						    num_ioss_evts, pss_evtmap,
+						    ioss_evtmap);
+}
+EXPORT_SYMBOL_GPL(telemetry_add_events);
+
+/**
+ * telemetry_read_events() - Fetches samples as specified by evtlog.telem_evt_id
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @evtlog:     Array of telemetry_evtlog structs to fill data
+ *		evtlog.telem_evt_id specifies the ids to read
+ * @len:	Length of array of evtlog
+ *
+ * Return: number of eventlogs read for success, < 0 for failure
+ */
+int telemetry_read_events(enum telemetry_unit telem_unit,
+			  struct telemetry_evtlog *evtlog, int len)
+{
+	return telm_core_conf.telem_ops->read_eventlog(telem_unit, evtlog,
+						       len, 0);
+}
+EXPORT_SYMBOL_GPL(telemetry_read_events);
+
+/**
+ * telemetry_raw_read_events() - Fetch samples specified by evtlog.telem_evt_id
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @evtlog:	Array of telemetry_evtlog structs to fill data
+ *		evtlog.telem_evt_id specifies the ids to read
+ * @len:	Length of array of evtlog
+ *
+ * The caller must take care of locking in this case.
+ *
+ * Return: number of eventlogs read for success, < 0 for failure
+ */
+int telemetry_raw_read_events(enum telemetry_unit telem_unit,
+			      struct telemetry_evtlog *evtlog, int len)
+{
+	return telm_core_conf.telem_ops->raw_read_eventlog(telem_unit, evtlog,
+							   len, 0);
+}
+EXPORT_SYMBOL_GPL(telemetry_raw_read_events);
+
+/**
+ * telemetry_read_eventlog() - Fetch the Telemetry log from PSS or IOSS
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @evtlog:	Array of telemetry_evtlog structs to fill data
+ * @len:	Length of array of evtlog
+ *
+ * Return: number of eventlogs read for success, < 0 for failure
+ */
+int telemetry_read_eventlog(enum telemetry_unit telem_unit,
+			    struct telemetry_evtlog *evtlog, int len)
+{
+	return telm_core_conf.telem_ops->read_eventlog(telem_unit, evtlog,
+						       len, 1);
+}
+EXPORT_SYMBOL_GPL(telemetry_read_eventlog);
+
+/**
+ * telemetry_raw_read_eventlog() - Fetch the Telemetry log from PSS or IOSS
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @evtlog:	Array of telemetry_evtlog structs to fill data
+ * @len:	Length of array of evtlog
+ *
+ * The caller must take care of locking in this case.
+ *
+ * Return: number of eventlogs read for success, < 0 for failure
+ */
+int telemetry_raw_read_eventlog(enum telemetry_unit telem_unit,
+				struct telemetry_evtlog *evtlog, int len)
+{
+	return telm_core_conf.telem_ops->raw_read_eventlog(telem_unit, evtlog,
+							   len, 1);
+}
+EXPORT_SYMBOL_GPL(telemetry_raw_read_eventlog);
+
+
+/**
+ * telemetry_get_trace_verbosity() - Get the IOSS & PSS Trace verbosity
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @verbosity:	Pointer to return Verbosity
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_get_trace_verbosity(enum telemetry_unit telem_unit,
+				  u32 *verbosity)
+{
+	return telm_core_conf.telem_ops->get_trace_verbosity(telem_unit,
+							     verbosity);
+}
+EXPORT_SYMBOL_GPL(telemetry_get_trace_verbosity);
+
+
+/**
+ * telemetry_set_trace_verbosity() - Update the IOSS & PSS Trace verbosity
+ * @telem_unit: Specify whether IOSS or PSS Read
+ * @verbosity:	Verbosity to set
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_set_trace_verbosity(enum telemetry_unit telem_unit, u32 verbosity)
+{
+	return telm_core_conf.telem_ops->set_trace_verbosity(telem_unit,
+							     verbosity);
+}
+EXPORT_SYMBOL_GPL(telemetry_set_trace_verbosity);
+
+/**
+ * telemetry_set_pltdata() - Set the platform specific Data
+ * @ops:	Pointer to ops structure
+ * @pltconfig:	Platform config data
+ *
+ * Usage by other than telemetry pltdrv module is invalid
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_set_pltdata(struct telemetry_core_ops *ops,
+			  struct telemetry_plt_config *pltconfig)
+{
+	if (ops)
+		telm_core_conf.telem_ops = ops;
+
+	if (pltconfig)
+		telm_core_conf.plt_config = pltconfig;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(telemetry_set_pltdata);
+
+/**
+ * telemetry_clear_pltdata() - Clear the platform specific Data
+ *
+ * Usage by other than telemetry pltdrv module is invalid
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_clear_pltdata(void)
+{
+	telm_core_conf.telem_ops = &telm_defpltops;
+	telm_core_conf.plt_config = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(telemetry_clear_pltdata);
+
+/**
+ * telemetry_pltconfig_valid() - Checkif platform config is valid
+ *
+ * Usage by other than telemetry module is invalid
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_pltconfig_valid(void)
+{
+	if (telm_core_conf.plt_config)
+		return 0;
+
+	else
+		return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(telemetry_pltconfig_valid);
+
+static inline int telemetry_get_pssevtname(enum telemetry_unit telem_unit,
+					   const char **name, int len)
+{
+	struct telemetry_unit_config psscfg;
+	int i;
+
+	if (!telm_core_conf.plt_config)
+		return -EINVAL;
+
+	psscfg = telm_core_conf.plt_config->pss_config;
+
+	if (len > psscfg.ssram_evts_used)
+		len = psscfg.ssram_evts_used;
+
+	for (i = 0; i < len; i++)
+		name[i] = psscfg.telem_evts[i].name;
+
+	return 0;
+}
+
+static inline int telemetry_get_iossevtname(enum telemetry_unit telem_unit,
+					    const char **name, int len)
+{
+	struct telemetry_unit_config iosscfg;
+	int i;
+
+	if (!(telm_core_conf.plt_config))
+		return -EINVAL;
+
+	iosscfg = telm_core_conf.plt_config->ioss_config;
+
+	if (len > iosscfg.ssram_evts_used)
+		len = iosscfg.ssram_evts_used;
+
+	for (i = 0; i < len; i++)
+		name[i] = iosscfg.telem_evts[i].name;
+
+	return 0;
+
+}
+
+/**
+ * telemetry_get_evtname() - Checkif platform config is valid
+ * @telem_unit:	Telemetry Unit to check
+ * @name:	Array of character pointers to contain name
+ * @len:	length of array name provided by user
+ *
+ * Usage by other than telemetry debugfs module is invalid
+ *
+ * Return: 0 success, < 0 for failure
+ */
+int telemetry_get_evtname(enum telemetry_unit telem_unit,
+			  const char **name, int len)
+{
+	int ret = -EINVAL;
+
+	if (telem_unit == TELEM_PSS)
+		ret = telemetry_get_pssevtname(telem_unit, name, len);
+
+	else if (telem_unit == TELEM_IOSS)
+		ret = telemetry_get_iossevtname(telem_unit, name, len);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(telemetry_get_evtname);
+
+static int __init telemetry_module_init(void)
+{
+	pr_info(pr_fmt(DRIVER_NAME) " Init\n");
+
+	telm_core_conf.telem_ops = &telm_defpltops;
+	return 0;
+}
+
+static void __exit telemetry_module_exit(void)
+{
+}
+
+module_init(telemetry_module_init);
+module_exit(telemetry_module_exit);
+
+MODULE_AUTHOR("Souvik Kumar Chakravarty <souvik.k.chakravarty@intel.com>");
+MODULE_DESCRIPTION("Intel SoC Telemetry Interface");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/intel_telemetry_debugfs.c b/drivers/platform/x86/intel_telemetry_debugfs.c
new file mode 100644
index 000000000000..f5134acd6ff0
--- /dev/null
+++ b/drivers/platform/x86/intel_telemetry_debugfs.c
@@ -0,0 +1,1032 @@
+/*
+ * Intel SOC Telemetry debugfs Driver: Currently supports APL
+ * Copyright (c) 2015, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * This file provides the debugfs interfaces for telemetry.
+ * /sys/kernel/debug/telemetry/pss_info: Shows Primary Control Sub-Sys Counters
+ * /sys/kernel/debug/telemetry/ioss_info: Shows IO Sub-System Counters
+ * /sys/kernel/debug/telemetry/soc_states: Shows SoC State
+ * /sys/kernel/debug/telemetry/pss_trace_verbosity: Read and Change Tracing
+ *				Verbosity via firmware
+ * /sys/kernel/debug/telemetry/ioss_race_verbosity: Write and Change Tracing
+ *				Verbosity via firmware
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+#include <linux/pci.h>
+#include <linux/suspend.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/intel_pmc_ipc.h>
+#include <asm/intel_punit_ipc.h>
+#include <asm/intel_telemetry.h>
+
+#define DRIVER_NAME	"telemetry_soc_debugfs"
+#define DRIVER_VERSION	"1.0.0"
+
+/* ApolloLake SoC Event-IDs */
+#define TELEM_APL_PSS_PSTATES_ID	0x2802
+#define TELEM_APL_PSS_IDLE_ID		0x2806
+#define TELEM_APL_PCS_IDLE_BLOCKED_ID	0x2C00
+#define TELEM_APL_PCS_S0IX_BLOCKED_ID	0x2C01
+#define TELEM_APL_PSS_WAKEUP_ID		0x2C02
+#define TELEM_APL_PSS_LTR_BLOCKING_ID	0x2C03
+
+#define TELEM_APL_S0IX_TOTAL_OCC_ID	0x4000
+#define TELEM_APL_S0IX_SHLW_OCC_ID	0x4001
+#define TELEM_APL_S0IX_DEEP_OCC_ID	0x4002
+#define TELEM_APL_S0IX_TOTAL_RES_ID	0x4800
+#define TELEM_APL_S0IX_SHLW_RES_ID	0x4801
+#define TELEM_APL_S0IX_DEEP_RES_ID	0x4802
+#define TELEM_APL_D0IX_ID		0x581A
+#define TELEM_APL_D3_ID			0x5819
+#define TELEM_APL_PG_ID			0x5818
+
+#define TELEM_INFO_SRAMEVTS_MASK	0xFF00
+#define TELEM_INFO_SRAMEVTS_SHIFT	0x8
+#define TELEM_SSRAM_READ_TIMEOUT	10
+
+#define TELEM_MASK_BIT			1
+#define TELEM_MASK_BYTE			0xFF
+#define BYTES_PER_LONG			8
+#define TELEM_APL_MASK_PCS_STATE	0xF
+
+/* Max events in bitmap to check for */
+#define TELEM_PSS_IDLE_EVTS		25
+#define TELEM_PSS_IDLE_BLOCKED_EVTS	20
+#define TELEM_PSS_S0IX_BLOCKED_EVTS	20
+#define TELEM_PSS_S0IX_WAKEUP_EVTS	20
+#define TELEM_PSS_LTR_BLOCKING_EVTS	20
+#define TELEM_IOSS_DX_D0IX_EVTS		25
+#define TELEM_IOSS_PG_EVTS		30
+
+#define TELEM_EVT_LEN(x) (sizeof(x)/sizeof((x)[0]))
+
+#define TELEM_DEBUGFS_CPU(model, data) \
+	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&data}
+
+#define TELEM_CHECK_AND_PARSE_EVTS(EVTID, EVTNUM, BUF, EVTLOG, EVTDAT, MASK) { \
+	if (evtlog[index].telem_evtid == (EVTID)) { \
+		for (idx = 0; idx < (EVTNUM); idx++) \
+			(BUF)[idx] = ((EVTLOG) >> (EVTDAT)[idx].bit_pos) & \
+				     (MASK); \
+	continue; \
+	} \
+}
+
+#define TELEM_CHECK_AND_PARSE_CTRS(EVTID, CTR) { \
+	if (evtlog[index].telem_evtid == (EVTID)) { \
+		(CTR) = evtlog[index].telem_evtlog; \
+		continue; \
+	} \
+}
+
+#ifdef CONFIG_PM_SLEEP
+static u8 suspend_prep_ok;
+static u32 suspend_shlw_ctr_temp, suspend_deep_ctr_temp;
+static u64 suspend_shlw_res_temp, suspend_deep_res_temp;
+#endif
+
+struct telemetry_susp_stats {
+	u32 shlw_swake_ctr;
+	u32 deep_swake_ctr;
+	u64 shlw_swake_res;
+	u64 deep_swake_res;
+	u32 shlw_ctr;
+	u32 deep_ctr;
+	u64 shlw_res;
+	u64 deep_res;
+};
+
+/* Bitmap definitions for default counters in APL */
+struct telem_pss_idle_stateinfo {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_pss_idle_stateinfo telem_apl_pss_idle_data[] = {
+	{"IA_CORE0_C1E",		0},
+	{"IA_CORE1_C1E",		1},
+	{"IA_CORE2_C1E",		2},
+	{"IA_CORE3_C1E",		3},
+	{"IA_CORE0_C6",			16},
+	{"IA_CORE1_C6",			17},
+	{"IA_CORE2_C6",			18},
+	{"IA_CORE3_C6",			19},
+	{"IA_MODULE0_C7",		32},
+	{"IA_MODULE1_C7",		33},
+	{"GT_RC6",			40},
+	{"IUNIT_PROCESSING_IDLE",	41},
+	{"FAR_MEM_IDLE",		43},
+	{"DISPLAY_IDLE",		44},
+	{"IUNIT_INPUT_SYSTEM_IDLE",	45},
+	{"PCS_STATUS",			60},
+};
+
+struct telem_pcs_blkd_info {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_pcs_blkd_info telem_apl_pcs_idle_blkd_data[] = {
+	{"COMPUTE",			0},
+	{"MISC",			8},
+	{"MODULE_ACTIONS_PENDING",	16},
+	{"LTR",				24},
+	{"DISPLAY_WAKE",		32},
+	{"ISP_WAKE",			40},
+	{"PSF0_ACTIVE",			48},
+};
+
+static struct telem_pcs_blkd_info telem_apl_pcs_s0ix_blkd_data[] = {
+	{"LTR",				0},
+	{"IRTL",			8},
+	{"WAKE_DEADLINE_PENDING",	16},
+	{"DISPLAY",			24},
+	{"ISP",				32},
+	{"CORE",			40},
+	{"PMC",				48},
+	{"MISC",			56},
+};
+
+struct telem_pss_ltr_info {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_pss_ltr_info telem_apl_pss_ltr_data[] = {
+	{"CORE_ACTIVE",		0},
+	{"MEM_UP",		8},
+	{"DFX",			16},
+	{"DFX_FORCE_LTR",	24},
+	{"DISPLAY",		32},
+	{"ISP",			40},
+	{"SOUTH",		48},
+};
+
+struct telem_pss_wakeup_info {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_pss_wakeup_info telem_apl_pss_wakeup[] = {
+	{"IP_IDLE",			0},
+	{"DISPLAY_WAKE",		8},
+	{"VOLTAGE_REG_INT",		16},
+	{"DROWSY_TIMER (HOTPLUG)",	24},
+	{"CORE_WAKE",			32},
+	{"MISC_S0IX",			40},
+	{"MISC_ABORT",			56},
+};
+
+struct telem_ioss_d0ix_stateinfo {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_ioss_d0ix_stateinfo telem_apl_ioss_d0ix_data[] = {
+	{"CSE",		0},
+	{"SCC2",	1},
+	{"GMM",		2},
+	{"XDCI",	3},
+	{"XHCI",	4},
+	{"ISH",		5},
+	{"AVS",		6},
+	{"PCIE0P1",	7},
+	{"PECI0P0",	8},
+	{"LPSS",	9},
+	{"SCC",		10},
+	{"PWM",		11},
+	{"PCIE1_P3",    12},
+	{"PCIE1_P2",    13},
+	{"PCIE1_P1",    14},
+	{"PCIE1_P0",    15},
+	{"CNV",		16},
+	{"SATA",	17},
+	{"PRTC",	18},
+};
+
+struct telem_ioss_pg_info {
+	const char *name;
+	u32 bit_pos;
+};
+
+static struct telem_ioss_pg_info telem_apl_ioss_pg_data[] = {
+	{"LPSS",	0},
+	{"SCC",		1},
+	{"P2SB",	2},
+	{"SCC2",	3},
+	{"GMM",		4},
+	{"PCIE0",	5},
+	{"XDCI",	6},
+	{"xHCI",	7},
+	{"CSE",		8},
+	{"SPI",		9},
+	{"AVSPGD4",	10},
+	{"AVSPGD3",	11},
+	{"AVSPGD2",	12},
+	{"AVSPGD1",	13},
+	{"ISH",		14},
+	{"EXI",		15},
+	{"NPKVRC",	16},
+	{"NPKVNN",	17},
+	{"CUNIT",	18},
+	{"FUSE_CTRL",	19},
+	{"PCIE1",	20},
+	{"CNV",		21},
+	{"LPC",		22},
+	{"SATA",	23},
+	{"SMB",		24},
+	{"PRTC",	25},
+};
+
+
+struct telemetry_debugfs_conf {
+	struct telemetry_susp_stats suspend_stats;
+	struct dentry *telemetry_dbg_dir;
+
+	/* Bitmap Data */
+	struct telem_ioss_d0ix_stateinfo *ioss_d0ix_data;
+	struct telem_pss_idle_stateinfo *pss_idle_data;
+	struct telem_pcs_blkd_info *pcs_idle_blkd_data;
+	struct telem_pcs_blkd_info *pcs_s0ix_blkd_data;
+	struct telem_pss_wakeup_info *pss_wakeup;
+	struct telem_pss_ltr_info *pss_ltr_data;
+	struct telem_ioss_pg_info *ioss_pg_data;
+	u8 pcs_idle_blkd_evts;
+	u8 pcs_s0ix_blkd_evts;
+	u8 pss_wakeup_evts;
+	u8 pss_idle_evts;
+	u8 pss_ltr_evts;
+	u8 ioss_d0ix_evts;
+	u8 ioss_pg_evts;
+
+	/* IDs */
+	u16  pss_ltr_blocking_id;
+	u16  pcs_idle_blkd_id;
+	u16  pcs_s0ix_blkd_id;
+	u16  s0ix_total_occ_id;
+	u16  s0ix_shlw_occ_id;
+	u16  s0ix_deep_occ_id;
+	u16  s0ix_total_res_id;
+	u16  s0ix_shlw_res_id;
+	u16  s0ix_deep_res_id;
+	u16  pss_wakeup_id;
+	u16  ioss_d0ix_id;
+	u16  pstates_id;
+	u16  pss_idle_id;
+	u16  ioss_d3_id;
+	u16  ioss_pg_id;
+};
+
+static struct telemetry_debugfs_conf *debugfs_conf;
+
+static struct telemetry_debugfs_conf telem_apl_debugfs_conf = {
+	.pss_idle_data = telem_apl_pss_idle_data,
+	.pcs_idle_blkd_data = telem_apl_pcs_idle_blkd_data,
+	.pcs_s0ix_blkd_data = telem_apl_pcs_s0ix_blkd_data,
+	.pss_ltr_data = telem_apl_pss_ltr_data,
+	.pss_wakeup = telem_apl_pss_wakeup,
+	.ioss_d0ix_data = telem_apl_ioss_d0ix_data,
+	.ioss_pg_data = telem_apl_ioss_pg_data,
+
+	.pss_idle_evts = TELEM_EVT_LEN(telem_apl_pss_idle_data),
+	.pcs_idle_blkd_evts = TELEM_EVT_LEN(telem_apl_pcs_idle_blkd_data),
+	.pcs_s0ix_blkd_evts = TELEM_EVT_LEN(telem_apl_pcs_s0ix_blkd_data),
+	.pss_ltr_evts = TELEM_EVT_LEN(telem_apl_pss_ltr_data),
+	.pss_wakeup_evts = TELEM_EVT_LEN(telem_apl_pss_wakeup),
+	.ioss_d0ix_evts = TELEM_EVT_LEN(telem_apl_ioss_d0ix_data),
+	.ioss_pg_evts = TELEM_EVT_LEN(telem_apl_ioss_pg_data),
+
+	.pstates_id = TELEM_APL_PSS_PSTATES_ID,
+	.pss_idle_id = TELEM_APL_PSS_IDLE_ID,
+	.pcs_idle_blkd_id = TELEM_APL_PCS_IDLE_BLOCKED_ID,
+	.pcs_s0ix_blkd_id = TELEM_APL_PCS_S0IX_BLOCKED_ID,
+	.pss_wakeup_id = TELEM_APL_PSS_WAKEUP_ID,
+	.pss_ltr_blocking_id = TELEM_APL_PSS_LTR_BLOCKING_ID,
+	.s0ix_total_occ_id = TELEM_APL_S0IX_TOTAL_OCC_ID,
+	.s0ix_shlw_occ_id = TELEM_APL_S0IX_SHLW_OCC_ID,
+	.s0ix_deep_occ_id = TELEM_APL_S0IX_DEEP_OCC_ID,
+	.s0ix_total_res_id = TELEM_APL_S0IX_TOTAL_RES_ID,
+	.s0ix_shlw_res_id = TELEM_APL_S0IX_SHLW_RES_ID,
+	.s0ix_deep_res_id = TELEM_APL_S0IX_DEEP_RES_ID,
+	.ioss_d0ix_id = TELEM_APL_D0IX_ID,
+	.ioss_d3_id = TELEM_APL_D3_ID,
+	.ioss_pg_id = TELEM_APL_PG_ID,
+};
+
+static const struct x86_cpu_id telemetry_debugfs_cpu_ids[] = {
+	TELEM_DEBUGFS_CPU(0x5c, telem_apl_debugfs_conf),
+	{}
+};
+
+MODULE_DEVICE_TABLE(x86cpu, telemetry_debugfs_cpu_ids);
+
+static int telemetry_debugfs_check_evts(void)
+{
+	if ((debugfs_conf->pss_idle_evts > TELEM_PSS_IDLE_EVTS) ||
+	    (debugfs_conf->pcs_idle_blkd_evts > TELEM_PSS_IDLE_BLOCKED_EVTS) ||
+	    (debugfs_conf->pcs_s0ix_blkd_evts > TELEM_PSS_S0IX_BLOCKED_EVTS) ||
+	    (debugfs_conf->pss_ltr_evts > TELEM_PSS_LTR_BLOCKING_EVTS) ||
+	    (debugfs_conf->pss_wakeup_evts > TELEM_PSS_S0IX_WAKEUP_EVTS) ||
+	    (debugfs_conf->ioss_d0ix_evts > TELEM_IOSS_DX_D0IX_EVTS) ||
+	    (debugfs_conf->ioss_pg_evts > TELEM_IOSS_PG_EVTS))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int telem_pss_states_show(struct seq_file *s, void *unused)
+{
+	struct telemetry_evtlog evtlog[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	struct telemetry_debugfs_conf *conf = debugfs_conf;
+	const char *name[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	u32 pcs_idle_blkd[TELEM_PSS_IDLE_BLOCKED_EVTS],
+	    pcs_s0ix_blkd[TELEM_PSS_S0IX_BLOCKED_EVTS],
+	    pss_s0ix_wakeup[TELEM_PSS_S0IX_WAKEUP_EVTS],
+	    pss_ltr_blkd[TELEM_PSS_LTR_BLOCKING_EVTS],
+	    pss_idle[TELEM_PSS_IDLE_EVTS];
+	int index, idx, ret, err = 0;
+	u64 pstates = 0;
+
+	ret = telemetry_read_eventlog(TELEM_PSS, evtlog,
+				      TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (ret < 0)
+		return ret;
+
+	err = telemetry_get_evtname(TELEM_PSS, name,
+				    TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (err < 0)
+		return err;
+
+	seq_puts(s, "\n----------------------------------------------------\n");
+	seq_puts(s, "\tPSS TELEM EVENTLOG (Residency = field/19.2 us\n");
+	seq_puts(s, "----------------------------------------------------\n");
+	for (index = 0; index < ret; index++) {
+		seq_printf(s, "%-32s %llu\n",
+			   name[index], evtlog[index].telem_evtlog);
+
+		/* Fetch PSS IDLE State */
+		if (evtlog[index].telem_evtid == conf->pss_idle_id) {
+			pss_idle[conf->pss_idle_evts - 1] =
+			(evtlog[index].telem_evtlog >>
+			conf->pss_idle_data[conf->pss_idle_evts - 1].bit_pos) &
+			TELEM_APL_MASK_PCS_STATE;
+		}
+
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->pss_idle_id,
+					   conf->pss_idle_evts - 1,
+					   pss_idle, evtlog[index].telem_evtlog,
+					   conf->pss_idle_data, TELEM_MASK_BIT);
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->pcs_idle_blkd_id,
+					   conf->pcs_idle_blkd_evts,
+					   pcs_idle_blkd,
+					   evtlog[index].telem_evtlog,
+					   conf->pcs_idle_blkd_data,
+					   TELEM_MASK_BYTE);
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->pcs_s0ix_blkd_id,
+					   conf->pcs_s0ix_blkd_evts,
+					   pcs_s0ix_blkd,
+					   evtlog[index].telem_evtlog,
+					   conf->pcs_s0ix_blkd_data,
+					   TELEM_MASK_BYTE);
+
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->pss_wakeup_id,
+					   conf->pss_wakeup_evts,
+					   pss_s0ix_wakeup,
+					   evtlog[index].telem_evtlog,
+					   conf->pss_wakeup, TELEM_MASK_BYTE);
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->pss_ltr_blocking_id,
+					   conf->pss_ltr_evts, pss_ltr_blkd,
+					   evtlog[index].telem_evtlog,
+					   conf->pss_ltr_data, TELEM_MASK_BYTE);
+
+		if (evtlog[index].telem_evtid == debugfs_conf->pstates_id)
+			pstates = evtlog[index].telem_evtlog;
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "PStates\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Domain\t\t\t\tFreq(Mhz)\n");
+	seq_printf(s, " IA\t\t\t\t %llu\n GT\t\t\t\t %llu\n",
+		   (pstates & TELEM_MASK_BYTE)*100,
+		   ((pstates >> 8) & TELEM_MASK_BYTE)*50/3);
+
+	seq_printf(s, " IUNIT\t\t\t\t %llu\n SA\t\t\t\t %llu\n",
+		   ((pstates >> 16) & TELEM_MASK_BYTE)*25,
+		   ((pstates >> 24) & TELEM_MASK_BYTE)*50/3);
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "PSS IDLE Status\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Device\t\t\t\t\tIDLE\n");
+	for (index = 0; index < debugfs_conf->pss_idle_evts; index++) {
+		seq_printf(s, "%-32s\t%u\n",
+			   debugfs_conf->pss_idle_data[index].name,
+			   pss_idle[index]);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "PSS Idle blkd Status (~1ms saturating bucket)\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Blocker\t\t\t\t\tCount\n");
+	for (index = 0; index < debugfs_conf->pcs_idle_blkd_evts; index++) {
+		seq_printf(s, "%-32s\t%u\n",
+			   debugfs_conf->pcs_idle_blkd_data[index].name,
+			   pcs_idle_blkd[index]);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "PSS S0ix blkd Status (~1ms saturating bucket)\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Blocker\t\t\t\t\tCount\n");
+	for (index = 0; index < debugfs_conf->pcs_s0ix_blkd_evts; index++) {
+		seq_printf(s, "%-32s\t%u\n",
+			   debugfs_conf->pcs_s0ix_blkd_data[index].name,
+			   pcs_s0ix_blkd[index]);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "LTR Blocking Status (~1ms saturating bucket)\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Blocker\t\t\t\t\tCount\n");
+	for (index = 0; index < debugfs_conf->pss_ltr_evts; index++) {
+		seq_printf(s, "%-32s\t%u\n",
+			   debugfs_conf->pss_ltr_data[index].name,
+			   pss_s0ix_wakeup[index]);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "Wakes Status (~1ms saturating bucket)\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Wakes\t\t\t\t\tCount\n");
+	for (index = 0; index < debugfs_conf->pss_wakeup_evts; index++) {
+		seq_printf(s, "%-32s\t%u\n",
+			   debugfs_conf->pss_wakeup[index].name,
+			   pss_ltr_blkd[index]);
+	}
+
+	return 0;
+}
+
+static int telem_pss_state_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, telem_pss_states_show, inode->i_private);
+}
+
+static const struct file_operations telem_pss_ops = {
+	.open		= telem_pss_state_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+
+static int telem_ioss_states_show(struct seq_file *s, void *unused)
+{
+	struct telemetry_evtlog evtlog[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	const char *name[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	int index, ret, err;
+
+	ret = telemetry_read_eventlog(TELEM_IOSS, evtlog,
+				      TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (ret < 0)
+		return ret;
+
+	err = telemetry_get_evtname(TELEM_IOSS, name,
+				    TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (err < 0)
+		return err;
+
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "\tI0SS TELEMETRY EVENTLOG\n");
+	seq_puts(s, "--------------------------------------\n");
+	for (index = 0; index < ret; index++) {
+		seq_printf(s, "%-32s 0x%llx\n",
+			   name[index], evtlog[index].telem_evtlog);
+	}
+
+	return 0;
+}
+
+static int telem_ioss_state_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, telem_ioss_states_show, inode->i_private);
+}
+
+static const struct file_operations telem_ioss_ops = {
+	.open		= telem_ioss_state_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int telem_soc_states_show(struct seq_file *s, void *unused)
+{
+	u32 d3_sts[TELEM_IOSS_DX_D0IX_EVTS], d0ix_sts[TELEM_IOSS_DX_D0IX_EVTS];
+	u32 pg_sts[TELEM_IOSS_PG_EVTS], pss_idle[TELEM_PSS_IDLE_EVTS];
+	struct telemetry_evtlog evtlog[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	u32 s0ix_total_ctr = 0, s0ix_shlw_ctr = 0, s0ix_deep_ctr = 0;
+	u64 s0ix_total_res = 0, s0ix_shlw_res = 0, s0ix_deep_res = 0;
+	struct telemetry_debugfs_conf *conf = debugfs_conf;
+	struct pci_dev *dev = NULL;
+	int index, idx, ret;
+	u32 d3_state;
+	u16 pmcsr;
+
+	ret = telemetry_read_eventlog(TELEM_IOSS, evtlog,
+				      TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (ret < 0)
+		return ret;
+
+	for (index = 0; index < ret; index++) {
+		TELEM_CHECK_AND_PARSE_EVTS(conf->ioss_d3_id,
+					   conf->ioss_d0ix_evts,
+					   d3_sts, evtlog[index].telem_evtlog,
+					   conf->ioss_d0ix_data,
+					   TELEM_MASK_BIT);
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->ioss_pg_id, conf->ioss_pg_evts,
+					   pg_sts, evtlog[index].telem_evtlog,
+					   conf->ioss_pg_data, TELEM_MASK_BIT);
+
+		TELEM_CHECK_AND_PARSE_EVTS(conf->ioss_d0ix_id,
+					   conf->ioss_d0ix_evts,
+					   d0ix_sts, evtlog[index].telem_evtlog,
+					   conf->ioss_d0ix_data,
+					   TELEM_MASK_BIT);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_total_occ_id,
+					   s0ix_total_ctr);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_occ_id,
+					   s0ix_shlw_ctr);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_occ_id,
+					   s0ix_deep_ctr);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_total_res_id,
+					   s0ix_total_res);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_res_id,
+					   s0ix_shlw_res);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_res_id,
+					   s0ix_deep_res);
+	}
+
+	seq_puts(s, "\n---------------------------------------------------\n");
+	seq_puts(s, "S0IX Type\t\t\t Occurrence\t\t Residency(us)\n");
+	seq_puts(s, "---------------------------------------------------\n");
+
+	seq_printf(s, "S0IX Shallow\t\t\t %10u\t %10llu\n",
+		   s0ix_shlw_ctr -
+		   conf->suspend_stats.shlw_ctr -
+		   conf->suspend_stats.shlw_swake_ctr,
+		   (u64)((s0ix_shlw_res -
+		   conf->suspend_stats.shlw_res -
+		   conf->suspend_stats.shlw_swake_res)*10/192));
+
+	seq_printf(s, "S0IX Deep\t\t\t %10u\t %10llu\n",
+		   s0ix_deep_ctr -
+		   conf->suspend_stats.deep_ctr -
+		   conf->suspend_stats.deep_swake_ctr,
+		   (u64)((s0ix_deep_res -
+		   conf->suspend_stats.deep_res -
+		   conf->suspend_stats.deep_swake_res)*10/192));
+
+	seq_printf(s, "Suspend(With S0ixShallow)\t %10u\t %10llu\n",
+		   conf->suspend_stats.shlw_ctr,
+		   (u64)(conf->suspend_stats.shlw_res*10)/192);
+
+	seq_printf(s, "Suspend(With S0ixDeep)\t\t %10u\t %10llu\n",
+		   conf->suspend_stats.deep_ctr,
+		   (u64)(conf->suspend_stats.deep_res*10)/192);
+
+	seq_printf(s, "Suspend(With Shallow-Wakes)\t %10u\t %10llu\n",
+		   conf->suspend_stats.shlw_swake_ctr +
+		   conf->suspend_stats.deep_swake_ctr,
+		   (u64)((conf->suspend_stats.shlw_swake_res +
+		   conf->suspend_stats.deep_swake_res)*10/192));
+
+	seq_printf(s, "S0IX+Suspend Total\t\t %10u\t %10llu\n", s0ix_total_ctr,
+				(u64)(s0ix_total_res*10/192));
+	seq_puts(s, "\n-------------------------------------------------\n");
+	seq_puts(s, "\t\tDEVICE STATES\n");
+	seq_puts(s, "-------------------------------------------------\n");
+
+	for_each_pci_dev(dev) {
+		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
+		d3_state = ((pmcsr & PCI_PM_CTRL_STATE_MASK) ==
+			    (__force int)PCI_D3hot) ? 1 : 0;
+
+		seq_printf(s, "pci %04x %04X %s %20.20s: ",
+			   dev->vendor, dev->device, dev_name(&dev->dev),
+			   dev_driver_string(&dev->dev));
+		seq_printf(s, " d3:%x\n", d3_state);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "D3/D0i3 Status\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Block\t\t D3\t D0i3\n");
+	for (index = 0; index < conf->ioss_d0ix_evts; index++) {
+		seq_printf(s, "%-10s\t %u\t %u\n",
+			   conf->ioss_d0ix_data[index].name,
+			   d3_sts[index], d0ix_sts[index]);
+	}
+
+	seq_puts(s, "\n--------------------------------------\n");
+	seq_puts(s, "South Complex PowerGate Status\n");
+	seq_puts(s, "--------------------------------------\n");
+	seq_puts(s, "Device\t\t PG\n");
+	for (index = 0; index < conf->ioss_pg_evts; index++) {
+		seq_printf(s, "%-10s\t %u\n",
+			   conf->ioss_pg_data[index].name,
+			   pg_sts[index]);
+	}
+
+	evtlog->telem_evtid = conf->pss_idle_id;
+	ret = telemetry_read_events(TELEM_PSS, evtlog, 1);
+	if (ret < 0)
+		return ret;
+
+	seq_puts(s, "\n-----------------------------------------\n");
+	seq_puts(s, "North Idle Status\n");
+	seq_puts(s, "-----------------------------------------\n");
+	for (idx = 0; idx < conf->pss_idle_evts - 1; idx++) {
+		pss_idle[idx] =	(evtlog->telem_evtlog >>
+				conf->pss_idle_data[idx].bit_pos) &
+				TELEM_MASK_BIT;
+	}
+
+	pss_idle[idx] = (evtlog->telem_evtlog >>
+			conf->pss_idle_data[idx].bit_pos) &
+			TELEM_APL_MASK_PCS_STATE;
+
+	for (index = 0; index < conf->pss_idle_evts; index++) {
+		seq_printf(s, "%-30s %u\n",
+			   conf->pss_idle_data[index].name,
+			   pss_idle[index]);
+	}
+
+	seq_puts(s, "\nPCS_STATUS Code\n");
+	seq_puts(s, "0:C0 1:C1 2:C1_DN_WT_DEV 3:C2 4:C2_WT_DE_MEM_UP\n");
+	seq_puts(s, "5:C2_WT_DE_MEM_DOWN 6:C2_UP_WT_DEV 7:C2_DN 8:C2_VOA\n");
+	seq_puts(s, "9:C2_VOA_UP 10:S0IX_PRE 11:S0IX\n");
+
+	return 0;
+}
+
+static int telem_soc_state_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, telem_soc_states_show, inode->i_private);
+}
+
+static const struct file_operations telem_socstate_ops = {
+	.open		= telem_soc_state_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int telem_pss_trc_verb_show(struct seq_file *s, void *unused)
+{
+	u32 verbosity;
+	int err;
+
+	err = telemetry_get_trace_verbosity(TELEM_PSS, &verbosity);
+	if (err) {
+		pr_err("Get PSS Trace Verbosity Failed with Error %d\n", err);
+		return -EFAULT;
+	}
+
+	seq_printf(s, "PSS Trace Verbosity %u\n", verbosity);
+	return 0;
+}
+
+static ssize_t telem_pss_trc_verb_write(struct file *file,
+					const char __user *userbuf,
+					size_t count, loff_t *ppos)
+{
+	u32 verbosity;
+	int err;
+
+	if (kstrtou32_from_user(userbuf, count, 0, &verbosity))
+		return -EFAULT;
+
+	err = telemetry_set_trace_verbosity(TELEM_PSS, verbosity);
+	if (err) {
+		pr_err("Changing PSS Trace Verbosity Failed. Error %d\n", err);
+		count = err;
+	}
+
+	return count;
+}
+
+static int telem_pss_trc_verb_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, telem_pss_trc_verb_show, inode->i_private);
+}
+
+static const struct file_operations telem_pss_trc_verb_ops = {
+	.open		= telem_pss_trc_verb_open,
+	.read		= seq_read,
+	.write		= telem_pss_trc_verb_write,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+
+static int telem_ioss_trc_verb_show(struct seq_file *s, void *unused)
+{
+	u32 verbosity;
+	int err;
+
+	err = telemetry_get_trace_verbosity(TELEM_IOSS, &verbosity);
+	if (err) {
+		pr_err("Get IOSS Trace Verbosity Failed with Error %d\n", err);
+		return -EFAULT;
+	}
+
+	seq_printf(s, "IOSS Trace Verbosity %u\n", verbosity);
+	return 0;
+}
+
+static ssize_t telem_ioss_trc_verb_write(struct file *file,
+					 const char __user *userbuf,
+					 size_t count, loff_t *ppos)
+{
+	u32 verbosity;
+	int err;
+
+	if (kstrtou32_from_user(userbuf, count, 0, &verbosity))
+		return -EFAULT;
+
+	err = telemetry_set_trace_verbosity(TELEM_IOSS, verbosity);
+	if (err) {
+		pr_err("Changing IOSS Trace Verbosity Failed. Error %d\n", err);
+		count = err;
+	}
+
+	return count;
+}
+
+static int telem_ioss_trc_verb_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, telem_ioss_trc_verb_show, inode->i_private);
+}
+
+static const struct file_operations telem_ioss_trc_verb_ops = {
+	.open		= telem_ioss_trc_verb_open,
+	.read		= seq_read,
+	.write		= telem_ioss_trc_verb_write,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+#ifdef CONFIG_PM_SLEEP
+static int pm_suspend_prep_cb(void)
+{
+	struct telemetry_evtlog evtlog[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	struct telemetry_debugfs_conf *conf = debugfs_conf;
+	int ret, index;
+
+	ret = telemetry_raw_read_eventlog(TELEM_IOSS, evtlog,
+			TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (ret < 0) {
+		suspend_prep_ok = 0;
+		goto out;
+	}
+
+	for (index = 0; index < ret; index++) {
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_occ_id,
+					   suspend_shlw_ctr_temp);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_occ_id,
+					   suspend_deep_ctr_temp);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_res_id,
+					   suspend_shlw_res_temp);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_res_id,
+					   suspend_deep_res_temp);
+	}
+	suspend_prep_ok = 1;
+out:
+	return NOTIFY_OK;
+}
+
+static int pm_suspend_exit_cb(void)
+{
+	struct telemetry_evtlog evtlog[TELEM_MAX_OS_ALLOCATED_EVENTS];
+	static u32 suspend_shlw_ctr_exit, suspend_deep_ctr_exit;
+	static u64 suspend_shlw_res_exit, suspend_deep_res_exit;
+	struct telemetry_debugfs_conf *conf = debugfs_conf;
+	int ret, index;
+
+	if (!suspend_prep_ok)
+		goto out;
+
+	ret = telemetry_raw_read_eventlog(TELEM_IOSS, evtlog,
+					  TELEM_MAX_OS_ALLOCATED_EVENTS);
+	if (ret < 0)
+		goto out;
+
+	for (index = 0; index < ret; index++) {
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_occ_id,
+					   suspend_shlw_ctr_exit);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_occ_id,
+					   suspend_deep_ctr_exit);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_shlw_res_id,
+					   suspend_shlw_res_exit);
+
+		TELEM_CHECK_AND_PARSE_CTRS(conf->s0ix_deep_res_id,
+					   suspend_deep_res_exit);
+	}
+
+	if ((suspend_shlw_ctr_exit < suspend_shlw_ctr_temp) ||
+	    (suspend_deep_ctr_exit < suspend_deep_ctr_temp) ||
+	    (suspend_shlw_res_exit < suspend_shlw_res_temp) ||
+	    (suspend_deep_res_exit < suspend_deep_res_temp)) {
+		pr_err("Wrong s0ix counters detected\n");
+		goto out;
+	}
+
+	suspend_shlw_ctr_exit -= suspend_shlw_ctr_temp;
+	suspend_deep_ctr_exit -= suspend_deep_ctr_temp;
+	suspend_shlw_res_exit -= suspend_shlw_res_temp;
+	suspend_deep_res_exit -= suspend_deep_res_temp;
+
+	if (suspend_shlw_ctr_exit == 1) {
+		conf->suspend_stats.shlw_ctr +=
+		suspend_shlw_ctr_exit;
+
+		conf->suspend_stats.shlw_res +=
+		suspend_shlw_res_exit;
+	}
+	/* Shallow Wakes Case */
+	else if (suspend_shlw_ctr_exit > 1) {
+		conf->suspend_stats.shlw_swake_ctr +=
+		suspend_shlw_ctr_exit;
+
+		conf->suspend_stats.shlw_swake_res +=
+		suspend_shlw_res_exit;
+	}
+
+	if (suspend_deep_ctr_exit == 1) {
+		conf->suspend_stats.deep_ctr +=
+		suspend_deep_ctr_exit;
+
+		conf->suspend_stats.deep_res +=
+		suspend_deep_res_exit;
+	}
+
+	/* Shallow Wakes Case */
+	else if (suspend_deep_ctr_exit > 1) {
+		conf->suspend_stats.deep_swake_ctr +=
+		suspend_deep_ctr_exit;
+
+		conf->suspend_stats.deep_swake_res +=
+		suspend_deep_res_exit;
+	}
+
+out:
+	suspend_prep_ok = 0;
+	return NOTIFY_OK;
+}
+
+static int pm_notification(struct notifier_block *this,
+			   unsigned long event, void *ptr)
+{
+	switch (event) {
+	case PM_SUSPEND_PREPARE:
+		return pm_suspend_prep_cb();
+	case PM_POST_SUSPEND:
+		return pm_suspend_exit_cb();
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block pm_notifier = {
+	.notifier_call = pm_notification,
+};
+#endif /* CONFIG_PM_SLEEP */
+
+static int __init telemetry_debugfs_init(void)
+{
+	const struct x86_cpu_id *id;
+	int err = -ENOMEM;
+	struct dentry *f;
+
+	/* Only APL supported for now */
+	id = x86_match_cpu(telemetry_debugfs_cpu_ids);
+	if (!id)
+		return -ENODEV;
+
+	debugfs_conf = (struct telemetry_debugfs_conf *)id->driver_data;
+
+	err = telemetry_pltconfig_valid();
+	if (err < 0)
+		return -ENODEV;
+
+	err = telemetry_debugfs_check_evts();
+	if (err < 0)
+		return -EINVAL;
+
+
+#ifdef CONFIG_PM_SLEEP
+	register_pm_notifier(&pm_notifier);
+#endif /* CONFIG_PM_SLEEP */
+
+	debugfs_conf->telemetry_dbg_dir = debugfs_create_dir("telemetry", NULL);
+	if (!debugfs_conf->telemetry_dbg_dir)
+		return -ENOMEM;
+
+	f = debugfs_create_file("pss_info", S_IFREG | S_IRUGO,
+				debugfs_conf->telemetry_dbg_dir, NULL,
+				&telem_pss_ops);
+	if (!f) {
+		pr_err("pss_sample_info debugfs register failed\n");
+		goto out;
+	}
+
+	f = debugfs_create_file("ioss_info", S_IFREG | S_IRUGO,
+				debugfs_conf->telemetry_dbg_dir, NULL,
+				&telem_ioss_ops);
+	if (!f) {
+		pr_err("ioss_sample_info debugfs register failed\n");
+		goto out;
+	}
+
+	f = debugfs_create_file("soc_states", S_IFREG | S_IRUGO,
+				debugfs_conf->telemetry_dbg_dir,
+				NULL, &telem_socstate_ops);
+	if (!f) {
+		pr_err("ioss_sample_info debugfs register failed\n");
+		goto out;
+	}
+
+	f = debugfs_create_file("pss_trace_verbosity", S_IFREG | S_IRUGO,
+				debugfs_conf->telemetry_dbg_dir, NULL,
+				&telem_pss_trc_verb_ops);
+	if (!f) {
+		pr_err("pss_trace_verbosity debugfs register failed\n");
+		goto out;
+	}
+
+	f = debugfs_create_file("ioss_trace_verbosity", S_IFREG | S_IRUGO,
+				debugfs_conf->telemetry_dbg_dir, NULL,
+				&telem_ioss_trc_verb_ops);
+	if (!f) {
+		pr_err("ioss_trace_verbosity debugfs register failed\n");
+		goto out;
+	}
+
+	return 0;
+
+out:
+	debugfs_remove_recursive(debugfs_conf->telemetry_dbg_dir);
+	debugfs_conf->telemetry_dbg_dir = NULL;
+
+	return err;
+}
+
+static void __exit telemetry_debugfs_exit(void)
+{
+	debugfs_remove_recursive(debugfs_conf->telemetry_dbg_dir);
+	debugfs_conf->telemetry_dbg_dir = NULL;
+}
+
+late_initcall(telemetry_debugfs_init);
+module_exit(telemetry_debugfs_exit);
+
+MODULE_AUTHOR("Souvik Kumar Chakravarty <souvik.k.chakravarty@intel.com>");
+MODULE_DESCRIPTION("Intel SoC Telemetry debugfs Interface");
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/intel_telemetry_pltdrv.c b/drivers/platform/x86/intel_telemetry_pltdrv.c
new file mode 100644
index 000000000000..f97019b0106f
--- /dev/null
+++ b/drivers/platform/x86/intel_telemetry_pltdrv.c
@@ -0,0 +1,1206 @@
+/*
+ * Intel SOC Telemetry Platform Driver: Currently supports APL
+ * Copyright (c) 2015, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * This file provides the platform specific telemetry implementation for APL.
+ * It used the PUNIT and PMC IPC interfaces for configuring the counters.
+ * The accumulated results are fetched from SRAM.
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+#include <linux/pci.h>
+#include <linux/suspend.h>
+#include <linux/platform_device.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/intel_pmc_ipc.h>
+#include <asm/intel_punit_ipc.h>
+#include <asm/intel_telemetry.h>
+
+#define DRIVER_NAME	"intel_telemetry"
+#define DRIVER_VERSION	"1.0.0"
+
+#define TELEM_TRC_VERBOSITY_MASK	0x3
+
+#define TELEM_MIN_PERIOD(x)		((x) & 0x7F0000)
+#define TELEM_MAX_PERIOD(x)		((x) & 0x7F000000)
+#define TELEM_SAMPLE_PERIOD_INVALID(x)	((x) & (BIT(7)))
+#define TELEM_CLEAR_SAMPLE_PERIOD(x)	((x) &= ~0x7F)
+
+#define TELEM_SAMPLING_DEFAULT_PERIOD	0xD
+
+#define TELEM_MAX_EVENTS_SRAM		28
+#define TELEM_MAX_OS_ALLOCATED_EVENTS	20
+#define TELEM_SSRAM_STARTTIME_OFFSET	8
+#define TELEM_SSRAM_EVTLOG_OFFSET	16
+
+#define IOSS_TELEM_EVENT_READ		0x0
+#define IOSS_TELEM_EVENT_WRITE		0x1
+#define IOSS_TELEM_INFO_READ		0x2
+#define IOSS_TELEM_TRACE_CTL_READ	0x5
+#define IOSS_TELEM_TRACE_CTL_WRITE	0x6
+#define IOSS_TELEM_EVENT_CTL_READ	0x7
+#define IOSS_TELEM_EVENT_CTL_WRITE	0x8
+#define IOSS_TELEM_EVT_CTRL_WRITE_SIZE	0x4
+#define IOSS_TELEM_READ_WORD		0x1
+#define IOSS_TELEM_WRITE_FOURBYTES	0x4
+#define IOSS_TELEM_EVT_WRITE_SIZE	0x3
+
+#define TELEM_INFO_SRAMEVTS_MASK	0xFF00
+#define TELEM_INFO_SRAMEVTS_SHIFT	0x8
+#define TELEM_SSRAM_READ_TIMEOUT	10
+
+#define TELEM_INFO_NENABLES_MASK	0xFF
+#define TELEM_EVENT_ENABLE		0x8000
+
+#define TELEM_MASK_BIT			1
+#define TELEM_MASK_BYTE			0xFF
+#define BYTES_PER_LONG			8
+#define TELEM_MASK_PCS_STATE		0xF
+
+#define TELEM_DISABLE(x)		((x) &= ~(BIT(31)))
+#define TELEM_CLEAR_EVENTS(x)		((x) |= (BIT(30)))
+#define TELEM_ENABLE_SRAM_EVT_TRACE(x)	((x) &= ~(BIT(30) | BIT(24)))
+#define TELEM_ENABLE_PERIODIC(x)	((x) |= (BIT(23) | BIT(31) | BIT(7)))
+#define TELEM_EXTRACT_VERBOSITY(x, y)	((y) = (((x) >> 27) & 0x3))
+#define TELEM_CLEAR_VERBOSITY_BITS(x)	((x) &= ~(BIT(27) | BIT(28)))
+#define TELEM_SET_VERBOSITY_BITS(x, y)	((x) |= ((y) << 27))
+
+#define TELEM_CPU(model, data) \
+	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&data }
+
+enum telemetry_action {
+	TELEM_UPDATE = 0,
+	TELEM_ADD,
+	TELEM_RESET,
+	TELEM_ACTION_NONE
+};
+
+struct telem_ssram_region {
+	u64 timestamp;
+	u64 start_time;
+	u64 events[TELEM_MAX_EVENTS_SRAM];
+};
+
+static struct telemetry_plt_config *telm_conf;
+
+/*
+ * The following counters are programmed by default during setup.
+ * Only 20 allocated to kernel driver
+ */
+static struct telemetry_evtmap
+	telemetry_apl_ioss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
+	{"SOC_S0IX_TOTAL_RES",			0x4800},
+	{"SOC_S0IX_TOTAL_OCC",			0x4000},
+	{"SOC_S0IX_SHALLOW_RES",		0x4801},
+	{"SOC_S0IX_SHALLOW_OCC",		0x4001},
+	{"SOC_S0IX_DEEP_RES",			0x4802},
+	{"SOC_S0IX_DEEP_OCC",			0x4002},
+	{"PMC_POWER_GATE",			0x5818},
+	{"PMC_D3_STATES",			0x5819},
+	{"PMC_D0I3_STATES",			0x581A},
+	{"PMC_S0IX_WAKE_REASON_GPIO",		0x6000},
+	{"PMC_S0IX_WAKE_REASON_TIMER",		0x6001},
+	{"PMC_S0IX_WAKE_REASON_VNNREQ",         0x6002},
+	{"PMC_S0IX_WAKE_REASON_LOWPOWER",       0x6003},
+	{"PMC_S0IX_WAKE_REASON_EXTERNAL",       0x6004},
+	{"PMC_S0IX_WAKE_REASON_MISC",           0x6005},
+	{"PMC_S0IX_BLOCKING_IPS_D3_D0I3",       0x6006},
+	{"PMC_S0IX_BLOCKING_IPS_PG",            0x6007},
+	{"PMC_S0IX_BLOCKING_MISC_IPS_PG",       0x6008},
+	{"PMC_S0IX_BLOCK_IPS_VNN_REQ",          0x6009},
+	{"PMC_S0IX_BLOCK_IPS_CLOCKS",           0x600B},
+};
+
+
+static struct telemetry_evtmap
+	telemetry_apl_pss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
+	{"IA_CORE0_C6_RES",			0x0400},
+	{"IA_CORE0_C6_CTR",			0x0000},
+	{"IA_MODULE0_C7_RES",			0x0410},
+	{"IA_MODULE0_C7_CTR",			0x000E},
+	{"IA_C0_RES",				0x0805},
+	{"PCS_LTR",				0x2801},
+	{"PSTATES",				0x2802},
+	{"SOC_S0I3_RES",			0x0409},
+	{"SOC_S0I3_CTR",			0x000A},
+	{"PCS_S0I3_CTR",			0x0009},
+	{"PCS_C1E_RES",				0x041A},
+	{"PCS_IDLE_STATUS",			0x2806},
+	{"IA_PERF_LIMITS",			0x280B},
+	{"GT_PERF_LIMITS",			0x280C},
+	{"PCS_WAKEUP_S0IX_CTR",			0x0030},
+	{"PCS_IDLE_BLOCKED",			0x2C00},
+	{"PCS_S0IX_BLOCKED",			0x2C01},
+	{"PCS_S0IX_WAKE_REASONS",		0x2C02},
+	{"PCS_LTR_BLOCKING",			0x2C03},
+	{"PC2_AND_MEM_SHALLOW_IDLE_RES",	0x1D40},
+};
+
+/* APL specific Data */
+static struct telemetry_plt_config telem_apl_config = {
+	.pss_config = {
+		.telem_evts = telemetry_apl_pss_default_events,
+	},
+	.ioss_config = {
+		.telem_evts = telemetry_apl_ioss_default_events,
+	},
+};
+
+static const struct x86_cpu_id telemetry_cpu_ids[] = {
+	TELEM_CPU(0x5c, telem_apl_config),
+	{}
+};
+
+MODULE_DEVICE_TABLE(x86cpu, telemetry_cpu_ids);
+
+static inline int telem_get_unitconfig(enum telemetry_unit telem_unit,
+				     struct telemetry_unit_config **unit_config)
+{
+	if (telem_unit == TELEM_PSS)
+		*unit_config = &(telm_conf->pss_config);
+	else if (telem_unit == TELEM_IOSS)
+		*unit_config = &(telm_conf->ioss_config);
+	else
+		return -EINVAL;
+
+	return 0;
+
+}
+
+static int telemetry_check_evtid(enum telemetry_unit telem_unit,
+				 u32 *evtmap, u8 len,
+				 enum telemetry_action action)
+{
+	struct telemetry_unit_config *unit_config;
+	int ret;
+
+	ret = telem_get_unitconfig(telem_unit, &unit_config);
+	if (ret < 0)
+		return ret;
+
+	switch (action) {
+	case TELEM_RESET:
+		if (len > TELEM_MAX_EVENTS_SRAM)
+			return -EINVAL;
+
+		break;
+
+	case TELEM_UPDATE:
+		if (len > TELEM_MAX_EVENTS_SRAM)
+			return -EINVAL;
+
+		if ((len > 0) && (evtmap == NULL))
+			return -EINVAL;
+
+		break;
+
+	case TELEM_ADD:
+		if ((len + unit_config->ssram_evts_used) >
+		    TELEM_MAX_EVENTS_SRAM)
+			return -EINVAL;
+
+		if ((len > 0) && (evtmap == NULL))
+			return -EINVAL;
+
+		break;
+
+	default:
+		pr_err("Unknown Telemetry action Specified %d\n", action);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+static inline int telemetry_plt_config_ioss_event(u32 evt_id, int index)
+{
+	u32 write_buf;
+	int ret;
+
+	write_buf = evt_id | TELEM_EVENT_ENABLE;
+	write_buf <<= BITS_PER_BYTE;
+	write_buf |= index;
+
+	ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				    IOSS_TELEM_EVENT_WRITE, (u8 *)&write_buf,
+				    IOSS_TELEM_EVT_WRITE_SIZE, NULL, 0);
+
+	return ret;
+}
+
+static inline int telemetry_plt_config_pss_event(u32 evt_id, int index)
+{
+	u32 write_buf;
+	int ret;
+
+	write_buf = evt_id | TELEM_EVENT_ENABLE;
+	ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT,
+				      index, 0, &write_buf, NULL);
+
+	return ret;
+}
+
+static int telemetry_setup_iossevtconfig(struct telemetry_evtconfig evtconfig,
+					 enum telemetry_action action)
+{
+	u8 num_ioss_evts, ioss_period;
+	int ret, index, idx;
+	u32 *ioss_evtmap;
+	u32 telem_ctrl;
+
+	num_ioss_evts = evtconfig.num_evts;
+	ioss_period = evtconfig.period;
+	ioss_evtmap = evtconfig.evtmap;
+
+	/* Get telemetry EVENT CTL */
+	ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				    IOSS_TELEM_EVENT_CTL_READ, NULL, 0,
+				    &telem_ctrl, IOSS_TELEM_READ_WORD);
+	if (ret) {
+		pr_err("IOSS TELEM_CTRL Read Failed\n");
+		return ret;
+	}
+
+	/* Disable Telemetry */
+	TELEM_DISABLE(telem_ctrl);
+
+	ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				    IOSS_TELEM_EVENT_CTL_WRITE,
+				    (u8 *)&telem_ctrl,
+				    IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
+				    NULL, 0);
+	if (ret) {
+		pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
+		return ret;
+	}
+
+
+	/* Reset Everything */
+	if (action == TELEM_RESET) {
+		/* Clear All Events */
+		TELEM_CLEAR_EVENTS(telem_ctrl);
+
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+					    IOSS_TELEM_EVENT_CTL_WRITE,
+					    (u8 *)&telem_ctrl,
+					    IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
+					    NULL, 0);
+		if (ret) {
+			pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
+			return ret;
+		}
+		telm_conf->ioss_config.ssram_evts_used = 0;
+
+		/* Configure Events */
+		for (idx = 0; idx < num_ioss_evts; idx++) {
+			if (telemetry_plt_config_ioss_event(
+			    telm_conf->ioss_config.telem_evts[idx].evt_id,
+			    idx)) {
+				pr_err("IOSS TELEM_RESET Fail for data: %x\n",
+				telm_conf->ioss_config.telem_evts[idx].evt_id);
+				continue;
+			}
+			telm_conf->ioss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Re-Configure Everything */
+	if (action == TELEM_UPDATE) {
+		/* Clear All Events */
+		TELEM_CLEAR_EVENTS(telem_ctrl);
+
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+					    IOSS_TELEM_EVENT_CTL_WRITE,
+					    (u8 *)&telem_ctrl,
+					    IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
+					    NULL, 0);
+		if (ret) {
+			pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
+			return ret;
+		}
+		telm_conf->ioss_config.ssram_evts_used = 0;
+
+		/* Configure Events */
+		for (index = 0; index < num_ioss_evts; index++) {
+			telm_conf->ioss_config.telem_evts[index].evt_id =
+			ioss_evtmap[index];
+
+			if (telemetry_plt_config_ioss_event(
+			    telm_conf->ioss_config.telem_evts[index].evt_id,
+			    index)) {
+				pr_err("IOSS TELEM_UPDATE Fail for Evt%x\n",
+					ioss_evtmap[index]);
+				continue;
+			}
+			telm_conf->ioss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Add some Events */
+	if (action == TELEM_ADD) {
+		/* Configure Events */
+		for (index = telm_conf->ioss_config.ssram_evts_used, idx = 0;
+		     idx < num_ioss_evts; index++, idx++) {
+			telm_conf->ioss_config.telem_evts[index].evt_id =
+			ioss_evtmap[idx];
+
+			if (telemetry_plt_config_ioss_event(
+			    telm_conf->ioss_config.telem_evts[index].evt_id,
+			    index)) {
+				pr_err("IOSS TELEM_ADD Fail for Event %x\n",
+					ioss_evtmap[idx]);
+				continue;
+			}
+			telm_conf->ioss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Enable Periodic Telemetry Events and enable SRAM trace */
+	TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
+	TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
+	TELEM_ENABLE_PERIODIC(telem_ctrl);
+	telem_ctrl |= ioss_period;
+
+	ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				    IOSS_TELEM_EVENT_CTL_WRITE,
+				    (u8 *)&telem_ctrl,
+				    IOSS_TELEM_EVT_CTRL_WRITE_SIZE, NULL, 0);
+	if (ret) {
+		pr_err("IOSS TELEM_CTRL Event Enable Write Failed\n");
+		return ret;
+	}
+
+	telm_conf->ioss_config.curr_period = ioss_period;
+
+	return 0;
+}
+
+
+static int telemetry_setup_pssevtconfig(struct telemetry_evtconfig evtconfig,
+					enum telemetry_action action)
+{
+	u8 num_pss_evts, pss_period;
+	int ret, index, idx;
+	u32 *pss_evtmap;
+	u32 telem_ctrl;
+
+	num_pss_evts = evtconfig.num_evts;
+	pss_period = evtconfig.period;
+	pss_evtmap = evtconfig.evtmap;
+
+	/* PSS Config */
+	/* Get telemetry EVENT CTL */
+	ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_READ_TELE_EVENT_CTRL,
+				      0, 0, NULL, &telem_ctrl);
+	if (ret) {
+		pr_err("PSS TELEM_CTRL Read Failed\n");
+		return ret;
+	}
+
+	/* Disable Telemetry */
+	TELEM_DISABLE(telem_ctrl);
+	ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				      0, 0, &telem_ctrl, NULL);
+	if (ret) {
+		pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
+		return ret;
+	}
+
+	/* Reset Everything */
+	if (action == TELEM_RESET) {
+		/* Clear All Events */
+		TELEM_CLEAR_EVENTS(telem_ctrl);
+
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				0, 0, &telem_ctrl, NULL);
+		if (ret) {
+			pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
+			return ret;
+		}
+		telm_conf->pss_config.ssram_evts_used = 0;
+		/* Configure Events */
+		for (idx = 0; idx < num_pss_evts; idx++) {
+			if (telemetry_plt_config_pss_event(
+			    telm_conf->pss_config.telem_evts[idx].evt_id,
+			    idx)) {
+				pr_err("PSS TELEM_RESET Fail for Event %x\n",
+				telm_conf->pss_config.telem_evts[idx].evt_id);
+				continue;
+			}
+			telm_conf->pss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Re-Configure Everything */
+	if (action == TELEM_UPDATE) {
+		/* Clear All Events */
+		TELEM_CLEAR_EVENTS(telem_ctrl);
+
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				0, 0, &telem_ctrl, NULL);
+		if (ret) {
+			pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
+			return ret;
+		}
+		telm_conf->pss_config.ssram_evts_used = 0;
+
+		/* Configure Events */
+		for (index = 0; index < num_pss_evts; index++) {
+			telm_conf->pss_config.telem_evts[index].evt_id =
+			pss_evtmap[index];
+
+			if (telemetry_plt_config_pss_event(
+			    telm_conf->pss_config.telem_evts[index].evt_id,
+			    index)) {
+				pr_err("PSS TELEM_UPDATE Fail for Event %x\n",
+					pss_evtmap[index]);
+				continue;
+			}
+			telm_conf->pss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Add some Events */
+	if (action == TELEM_ADD) {
+		/* Configure Events */
+		for (index = telm_conf->pss_config.ssram_evts_used, idx = 0;
+		     idx < num_pss_evts; index++, idx++) {
+
+			telm_conf->pss_config.telem_evts[index].evt_id =
+			pss_evtmap[idx];
+
+			if (telemetry_plt_config_pss_event(
+			    telm_conf->pss_config.telem_evts[index].evt_id,
+			    index)) {
+				pr_err("PSS TELEM_ADD Fail for Event %x\n",
+					pss_evtmap[idx]);
+				continue;
+			}
+			telm_conf->pss_config.ssram_evts_used++;
+		}
+	}
+
+	/* Enable Periodic Telemetry Events and enable SRAM trace */
+	TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
+	TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
+	TELEM_ENABLE_PERIODIC(telem_ctrl);
+	telem_ctrl |= pss_period;
+
+	ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				      0, 0, &telem_ctrl, NULL);
+	if (ret) {
+		pr_err("PSS TELEM_CTRL Event Enable Write Failed\n");
+		return ret;
+	}
+
+	telm_conf->pss_config.curr_period = pss_period;
+
+	return 0;
+}
+
+static int telemetry_setup_evtconfig(struct telemetry_evtconfig pss_evtconfig,
+				     struct telemetry_evtconfig ioss_evtconfig,
+				     enum telemetry_action action)
+{
+	int ret;
+
+	mutex_lock(&(telm_conf->telem_lock));
+
+	if ((action == TELEM_UPDATE) && (telm_conf->telem_in_use)) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	ret = telemetry_check_evtid(TELEM_PSS, pss_evtconfig.evtmap,
+				    pss_evtconfig.num_evts, action);
+	if (ret)
+		goto out;
+
+	ret = telemetry_check_evtid(TELEM_IOSS, ioss_evtconfig.evtmap,
+				    ioss_evtconfig.num_evts, action);
+	if (ret)
+		goto out;
+
+	if (ioss_evtconfig.num_evts) {
+		ret = telemetry_setup_iossevtconfig(ioss_evtconfig, action);
+		if (ret)
+			goto out;
+	}
+
+	if (pss_evtconfig.num_evts) {
+		ret = telemetry_setup_pssevtconfig(pss_evtconfig, action);
+		if (ret)
+			goto out;
+	}
+
+	if ((action == TELEM_UPDATE) || (action == TELEM_ADD))
+		telm_conf->telem_in_use = true;
+	else
+		telm_conf->telem_in_use = false;
+
+out:
+	mutex_unlock(&(telm_conf->telem_lock));
+	return ret;
+}
+
+static int telemetry_setup(struct platform_device *pdev)
+{
+	struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
+	u32 read_buf, events, event_regs;
+	int ret;
+
+	ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY, IOSS_TELEM_INFO_READ,
+				    NULL, 0, &read_buf, IOSS_TELEM_READ_WORD);
+	if (ret) {
+		dev_err(&pdev->dev, "IOSS TELEM_INFO Read Failed\n");
+		return ret;
+	}
+
+	/* Get telemetry Info */
+	events = (read_buf & TELEM_INFO_SRAMEVTS_MASK) >>
+		  TELEM_INFO_SRAMEVTS_SHIFT;
+	event_regs = read_buf & TELEM_INFO_NENABLES_MASK;
+	if ((events < TELEM_MAX_EVENTS_SRAM) ||
+	    (event_regs < TELEM_MAX_EVENTS_SRAM)) {
+		dev_err(&pdev->dev, "IOSS:Insufficient Space for SRAM Trace\n");
+		dev_err(&pdev->dev, "SRAM Events %d; Event Regs %d\n",
+			events, event_regs);
+		return -ENOMEM;
+	}
+
+	telm_conf->ioss_config.min_period = TELEM_MIN_PERIOD(read_buf);
+	telm_conf->ioss_config.max_period = TELEM_MAX_PERIOD(read_buf);
+
+	/* PUNIT Mailbox Setup */
+	ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_READ_TELE_INFO, 0, 0,
+				      NULL, &read_buf);
+	if (ret) {
+		dev_err(&pdev->dev, "PSS TELEM_INFO Read Failed\n");
+		return ret;
+	}
+
+	/* Get telemetry Info */
+	events = (read_buf & TELEM_INFO_SRAMEVTS_MASK) >>
+		  TELEM_INFO_SRAMEVTS_SHIFT;
+	event_regs = read_buf & TELEM_INFO_SRAMEVTS_MASK;
+	if ((events < TELEM_MAX_EVENTS_SRAM) ||
+	    (event_regs < TELEM_MAX_EVENTS_SRAM)) {
+		dev_err(&pdev->dev, "PSS:Insufficient Space for SRAM Trace\n");
+		dev_err(&pdev->dev, "SRAM Events %d; Event Regs %d\n",
+			events, event_regs);
+		return -ENOMEM;
+	}
+
+	telm_conf->pss_config.min_period = TELEM_MIN_PERIOD(read_buf);
+	telm_conf->pss_config.max_period = TELEM_MAX_PERIOD(read_buf);
+
+	pss_evtconfig.evtmap = NULL;
+	pss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
+	pss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
+
+	ioss_evtconfig.evtmap = NULL;
+	ioss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
+	ioss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
+
+	ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
+					TELEM_RESET);
+	if (ret) {
+		dev_err(&pdev->dev, "TELEMTRY Setup Failed\n");
+		return ret;
+	}
+	return 0;
+}
+
+static int telemetry_plt_update_events(struct telemetry_evtconfig pss_evtconfig,
+				struct telemetry_evtconfig ioss_evtconfig)
+{
+	int ret;
+
+	if ((pss_evtconfig.num_evts > 0) &&
+	    (TELEM_SAMPLE_PERIOD_INVALID(pss_evtconfig.period))) {
+		pr_err("PSS Sampling Period Out of Range\n");
+		return -EINVAL;
+	}
+
+	if ((ioss_evtconfig.num_evts > 0) &&
+	    (TELEM_SAMPLE_PERIOD_INVALID(ioss_evtconfig.period))) {
+		pr_err("IOSS Sampling Period Out of Range\n");
+		return -EINVAL;
+	}
+
+	ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
+					TELEM_UPDATE);
+	if (ret)
+		pr_err("TELEMTRY Config Failed\n");
+
+	return ret;
+}
+
+
+static int telemetry_plt_set_sampling_period(u8 pss_period, u8 ioss_period)
+{
+	u32 telem_ctrl = 0;
+	int ret;
+
+	mutex_lock(&(telm_conf->telem_lock));
+	if (ioss_period) {
+		if (TELEM_SAMPLE_PERIOD_INVALID(ioss_period)) {
+			pr_err("IOSS Sampling Period Out of Range\n");
+			ret = -EINVAL;
+			goto out;
+		}
+
+		/* Get telemetry EVENT CTL */
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+					    IOSS_TELEM_EVENT_CTL_READ, NULL, 0,
+					    &telem_ctrl, IOSS_TELEM_READ_WORD);
+		if (ret) {
+			pr_err("IOSS TELEM_CTRL Read Failed\n");
+			goto out;
+		}
+
+		/* Disable Telemetry */
+		TELEM_DISABLE(telem_ctrl);
+
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+					    IOSS_TELEM_EVENT_CTL_WRITE,
+					    (u8 *)&telem_ctrl,
+					    IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
+					    NULL, 0);
+		if (ret) {
+			pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
+			goto out;
+		}
+
+		/* Enable Periodic Telemetry Events and enable SRAM trace */
+		TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
+		TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
+		TELEM_ENABLE_PERIODIC(telem_ctrl);
+		telem_ctrl |= ioss_period;
+
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+					    IOSS_TELEM_EVENT_CTL_WRITE,
+					    (u8 *)&telem_ctrl,
+					    IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
+					    NULL, 0);
+		if (ret) {
+			pr_err("IOSS TELEM_CTRL Event Enable Write Failed\n");
+			goto out;
+		}
+		telm_conf->ioss_config.curr_period = ioss_period;
+	}
+
+	if (pss_period) {
+		if (TELEM_SAMPLE_PERIOD_INVALID(pss_period)) {
+			pr_err("PSS Sampling Period Out of Range\n");
+			ret = -EINVAL;
+			goto out;
+		}
+
+		/* Get telemetry EVENT CTL */
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_READ_TELE_EVENT_CTRL,
+				0, 0, NULL, &telem_ctrl);
+		if (ret) {
+			pr_err("PSS TELEM_CTRL Read Failed\n");
+			goto out;
+		}
+
+		/* Disable Telemetry */
+		TELEM_DISABLE(telem_ctrl);
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				0, 0, &telem_ctrl, NULL);
+		if (ret) {
+			pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
+			goto out;
+		}
+
+		/* Enable Periodic Telemetry Events and enable SRAM trace */
+		TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
+		TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
+		TELEM_ENABLE_PERIODIC(telem_ctrl);
+		telem_ctrl |= pss_period;
+
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
+				0, 0, &telem_ctrl, NULL);
+		if (ret) {
+			pr_err("PSS TELEM_CTRL Event Enable Write Failed\n");
+			goto out;
+		}
+		telm_conf->pss_config.curr_period = pss_period;
+	}
+
+out:
+	mutex_unlock(&(telm_conf->telem_lock));
+	return ret;
+}
+
+
+static int telemetry_plt_get_sampling_period(u8 *pss_min_period,
+					     u8 *pss_max_period,
+					     u8 *ioss_min_period,
+					     u8 *ioss_max_period)
+{
+	*pss_min_period = telm_conf->pss_config.min_period;
+	*pss_max_period = telm_conf->pss_config.max_period;
+	*ioss_min_period = telm_conf->ioss_config.min_period;
+	*ioss_max_period = telm_conf->ioss_config.max_period;
+
+	return 0;
+}
+
+
+static int telemetry_plt_reset_events(void)
+{
+	struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
+	int ret;
+
+	pss_evtconfig.evtmap = NULL;
+	pss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
+	pss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
+
+	ioss_evtconfig.evtmap = NULL;
+	ioss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
+	ioss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
+
+	ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
+					TELEM_RESET);
+	if (ret)
+		pr_err("TELEMTRY Reset Failed\n");
+
+	return ret;
+}
+
+
+static int telemetry_plt_get_eventconfig(struct telemetry_evtconfig *pss_config,
+					struct telemetry_evtconfig *ioss_config,
+					int pss_len, int ioss_len)
+{
+	u32 *pss_evtmap, *ioss_evtmap;
+	u32 index;
+
+	pss_evtmap = pss_config->evtmap;
+	ioss_evtmap = ioss_config->evtmap;
+
+	mutex_lock(&(telm_conf->telem_lock));
+	pss_config->num_evts = telm_conf->pss_config.ssram_evts_used;
+	ioss_config->num_evts = telm_conf->ioss_config.ssram_evts_used;
+
+	pss_config->period = telm_conf->pss_config.curr_period;
+	ioss_config->period = telm_conf->ioss_config.curr_period;
+
+	if ((pss_len < telm_conf->pss_config.ssram_evts_used) ||
+	    (ioss_len < telm_conf->ioss_config.ssram_evts_used)) {
+		mutex_unlock(&(telm_conf->telem_lock));
+		return -EINVAL;
+	}
+
+	for (index = 0; index < telm_conf->pss_config.ssram_evts_used;
+	     index++) {
+		pss_evtmap[index] =
+		telm_conf->pss_config.telem_evts[index].evt_id;
+	}
+
+	for (index = 0; index < telm_conf->ioss_config.ssram_evts_used;
+	     index++) {
+		ioss_evtmap[index] =
+		telm_conf->ioss_config.telem_evts[index].evt_id;
+	}
+
+	mutex_unlock(&(telm_conf->telem_lock));
+	return 0;
+}
+
+
+static int telemetry_plt_add_events(u8 num_pss_evts, u8 num_ioss_evts,
+				    u32 *pss_evtmap, u32 *ioss_evtmap)
+{
+	struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
+	int ret;
+
+	pss_evtconfig.evtmap = pss_evtmap;
+	pss_evtconfig.num_evts = num_pss_evts;
+	pss_evtconfig.period = telm_conf->pss_config.curr_period;
+
+	ioss_evtconfig.evtmap = ioss_evtmap;
+	ioss_evtconfig.num_evts = num_ioss_evts;
+	ioss_evtconfig.period = telm_conf->ioss_config.curr_period;
+
+	ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
+					TELEM_ADD);
+	if (ret)
+		pr_err("TELEMTRY ADD Failed\n");
+
+	return ret;
+}
+
+static int telem_evtlog_read(enum telemetry_unit telem_unit,
+			     struct telem_ssram_region *ssram_region, u8 len)
+{
+	struct telemetry_unit_config *unit_config;
+	u64 timestamp_prev, timestamp_next;
+	int ret, index, timeout = 0;
+
+	ret = telem_get_unitconfig(telem_unit, &unit_config);
+	if (ret < 0)
+		return ret;
+
+	if (len > unit_config->ssram_evts_used)
+		len = unit_config->ssram_evts_used;
+
+	do {
+		timestamp_prev = readq(unit_config->regmap);
+		if (!timestamp_prev) {
+			pr_err("Ssram under update. Please Try Later\n");
+			return -EBUSY;
+		}
+
+		ssram_region->start_time = readq(unit_config->regmap +
+						 TELEM_SSRAM_STARTTIME_OFFSET);
+
+		for (index = 0; index < len; index++) {
+			ssram_region->events[index] =
+			readq(unit_config->regmap + TELEM_SSRAM_EVTLOG_OFFSET +
+			      BYTES_PER_LONG*index);
+		}
+
+		timestamp_next = readq(unit_config->regmap);
+		if (!timestamp_next) {
+			pr_err("Ssram under update. Please Try Later\n");
+			return -EBUSY;
+		}
+
+		if (timeout++ > TELEM_SSRAM_READ_TIMEOUT) {
+			pr_err("Timeout while reading Events\n");
+			return -EBUSY;
+		}
+
+	} while (timestamp_prev != timestamp_next);
+
+	ssram_region->timestamp = timestamp_next;
+
+	return len;
+}
+
+static int telemetry_plt_raw_read_eventlog(enum telemetry_unit telem_unit,
+					   struct telemetry_evtlog *evtlog,
+					   int len, int log_all_evts)
+{
+	int index, idx1, ret, readlen = len;
+	struct telem_ssram_region ssram_region;
+	struct telemetry_evtmap *evtmap;
+
+	switch (telem_unit)	{
+	case TELEM_PSS:
+		evtmap = telm_conf->pss_config.telem_evts;
+		break;
+
+	case TELEM_IOSS:
+		evtmap = telm_conf->ioss_config.telem_evts;
+		break;
+
+	default:
+		pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
+		return -EINVAL;
+	}
+
+	if (!log_all_evts)
+		readlen = TELEM_MAX_EVENTS_SRAM;
+
+	ret = telem_evtlog_read(telem_unit, &ssram_region, readlen);
+	if (ret < 0)
+		return ret;
+
+	/* Invalid evt-id array specified via length mismatch */
+	if ((!log_all_evts) && (len > ret))
+		return -EINVAL;
+
+	if (log_all_evts)
+		for (index = 0; index < ret; index++) {
+			evtlog[index].telem_evtlog = ssram_region.events[index];
+			evtlog[index].telem_evtid = evtmap[index].evt_id;
+		}
+	else
+		for (index = 0, readlen = 0; (index < ret) && (readlen < len);
+		     index++) {
+			for (idx1 = 0; idx1 < len; idx1++) {
+				/* Elements matched */
+				if (evtmap[index].evt_id ==
+				    evtlog[idx1].telem_evtid) {
+					evtlog[idx1].telem_evtlog =
+					ssram_region.events[index];
+					readlen++;
+
+					break;
+				}
+			}
+		}
+
+	return readlen;
+}
+
+static int telemetry_plt_read_eventlog(enum telemetry_unit telem_unit,
+		struct telemetry_evtlog *evtlog, int len, int log_all_evts)
+{
+	int ret;
+
+	mutex_lock(&(telm_conf->telem_lock));
+	ret = telemetry_plt_raw_read_eventlog(telem_unit, evtlog,
+					      len, log_all_evts);
+	mutex_unlock(&(telm_conf->telem_lock));
+
+	return ret;
+}
+
+static int telemetry_plt_get_trace_verbosity(enum telemetry_unit telem_unit,
+					     u32 *verbosity)
+{
+	u32 temp = 0;
+	int ret;
+
+	if (verbosity == NULL)
+		return -EINVAL;
+
+	mutex_lock(&(telm_conf->telem_trace_lock));
+	switch (telem_unit) {
+	case TELEM_PSS:
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_READ_TELE_TRACE_CTRL,
+				0, 0, NULL, &temp);
+		if (ret) {
+			pr_err("PSS TRACE_CTRL Read Failed\n");
+			goto out;
+		}
+
+		break;
+
+	case TELEM_IOSS:
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				IOSS_TELEM_TRACE_CTL_READ, NULL, 0, &temp,
+				IOSS_TELEM_READ_WORD);
+		if (ret) {
+			pr_err("IOSS TRACE_CTL Read Failed\n");
+			goto out;
+		}
+
+		break;
+
+	default:
+		pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
+		ret = -EINVAL;
+		break;
+	}
+	TELEM_EXTRACT_VERBOSITY(temp, *verbosity);
+
+out:
+	mutex_unlock(&(telm_conf->telem_trace_lock));
+	return ret;
+}
+
+static int telemetry_plt_set_trace_verbosity(enum telemetry_unit telem_unit,
+					     u32 verbosity)
+{
+	u32 temp = 0;
+	int ret;
+
+	verbosity &= TELEM_TRC_VERBOSITY_MASK;
+
+	mutex_lock(&(telm_conf->telem_trace_lock));
+	switch (telem_unit) {
+	case TELEM_PSS:
+		ret = intel_punit_ipc_command(
+				IPC_PUNIT_BIOS_WRITE_TELE_TRACE_CTRL,
+				0, 0, &verbosity, NULL);
+		if (ret) {
+			pr_err("PSS TRACE_CTRL Verbosity Set Failed\n");
+			goto out;
+		}
+		break;
+
+	case TELEM_IOSS:
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				IOSS_TELEM_TRACE_CTL_READ, NULL, 0, &temp,
+				IOSS_TELEM_READ_WORD);
+		if (ret) {
+			pr_err("IOSS TRACE_CTL Read Failed\n");
+			goto out;
+		}
+
+		TELEM_CLEAR_VERBOSITY_BITS(temp);
+		TELEM_SET_VERBOSITY_BITS(temp, verbosity);
+
+		ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
+				IOSS_TELEM_TRACE_CTL_WRITE, (u8 *)&temp,
+				IOSS_TELEM_WRITE_FOURBYTES, NULL, 0);
+		if (ret) {
+			pr_err("IOSS TRACE_CTL Verbosity Set Failed\n");
+			goto out;
+		}
+		break;
+
+	default:
+		pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
+		ret = -EINVAL;
+		break;
+	}
+
+out:
+	mutex_unlock(&(telm_conf->telem_trace_lock));
+	return ret;
+}
+
+static struct telemetry_core_ops telm_pltops = {
+	.get_trace_verbosity = telemetry_plt_get_trace_verbosity,
+	.set_trace_verbosity = telemetry_plt_set_trace_verbosity,
+	.set_sampling_period = telemetry_plt_set_sampling_period,
+	.get_sampling_period = telemetry_plt_get_sampling_period,
+	.raw_read_eventlog = telemetry_plt_raw_read_eventlog,
+	.get_eventconfig = telemetry_plt_get_eventconfig,
+	.update_events = telemetry_plt_update_events,
+	.read_eventlog = telemetry_plt_read_eventlog,
+	.reset_events = telemetry_plt_reset_events,
+	.add_events = telemetry_plt_add_events,
+};
+
+static int telemetry_pltdrv_probe(struct platform_device *pdev)
+{
+	struct resource *res0 = NULL, *res1 = NULL;
+	const struct x86_cpu_id *id;
+	int size, ret = -ENOMEM;
+
+	id = x86_match_cpu(telemetry_cpu_ids);
+	if (!id)
+		return -ENODEV;
+
+	telm_conf = (struct telemetry_plt_config *)id->driver_data;
+
+	res0 = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res0) {
+		ret = -EINVAL;
+		goto out;
+	}
+	size = resource_size(res0);
+	if (!devm_request_mem_region(&pdev->dev, res0->start, size,
+				     pdev->name)) {
+		ret = -EBUSY;
+		goto out;
+	}
+	telm_conf->pss_config.ssram_base_addr = res0->start;
+	telm_conf->pss_config.ssram_size = size;
+
+	res1 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!res1) {
+		ret = -EINVAL;
+		goto out;
+	}
+	size = resource_size(res1);
+	if (!devm_request_mem_region(&pdev->dev, res1->start, size,
+				     pdev->name)) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	telm_conf->ioss_config.ssram_base_addr = res1->start;
+	telm_conf->ioss_config.ssram_size = size;
+
+	telm_conf->pss_config.regmap = ioremap_nocache(
+					telm_conf->pss_config.ssram_base_addr,
+					telm_conf->pss_config.ssram_size);
+	if (!telm_conf->pss_config.regmap) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	telm_conf->ioss_config.regmap = ioremap_nocache(
+				telm_conf->ioss_config.ssram_base_addr,
+				telm_conf->ioss_config.ssram_size);
+	if (!telm_conf->ioss_config.regmap) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	mutex_init(&telm_conf->telem_lock);
+	mutex_init(&telm_conf->telem_trace_lock);
+
+	ret = telemetry_setup(pdev);
+	if (ret)
+		goto out;
+
+	ret = telemetry_set_pltdata(&telm_pltops, telm_conf);
+	if (ret) {
+		dev_err(&pdev->dev, "TELEMTRY Set Pltops Failed.\n");
+		goto out;
+	}
+
+	return 0;
+
+out:
+	if (res0)
+		release_mem_region(res0->start, resource_size(res0));
+	if (res1)
+		release_mem_region(res1->start, resource_size(res1));
+	if (telm_conf->pss_config.regmap)
+		iounmap(telm_conf->pss_config.regmap);
+	if (telm_conf->ioss_config.regmap)
+		iounmap(telm_conf->ioss_config.regmap);
+	dev_err(&pdev->dev, "TELEMTRY Setup Failed.\n");
+
+	return ret;
+}
+
+static int telemetry_pltdrv_remove(struct platform_device *pdev)
+{
+	telemetry_clear_pltdata();
+	iounmap(telm_conf->pss_config.regmap);
+	iounmap(telm_conf->ioss_config.regmap);
+
+	return 0;
+}
+
+static struct platform_driver telemetry_soc_driver = {
+	.probe		= telemetry_pltdrv_probe,
+	.remove		= telemetry_pltdrv_remove,
+	.driver		= {
+		.name	= DRIVER_NAME,
+	},
+};
+
+static int __init telemetry_module_init(void)
+{
+	pr_info(DRIVER_NAME ": version %s loaded\n", DRIVER_VERSION);
+	return platform_driver_register(&telemetry_soc_driver);
+}
+
+static void __exit telemetry_module_exit(void)
+{
+	platform_driver_unregister(&telemetry_soc_driver);
+}
+
+device_initcall(telemetry_module_init);
+module_exit(telemetry_module_exit);
+
+MODULE_AUTHOR("Souvik Kumar Chakravarty <souvik.k.chakravarty@intel.com>");
+MODULE_DESCRIPTION("Intel SoC Telemetry Platform Driver");
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index f73c29558cd3..e9caa347a9bf 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -1393,6 +1393,7 @@ static void sony_nc_function_setup(struct acpi_device *device,
 		case 0x0143:
 		case 0x014b:
 		case 0x014c:
+		case 0x0153:
 		case 0x0163:
 			result = sony_nc_kbd_backlight_setup(pf_device, handle);
 			if (result)
@@ -1490,6 +1491,7 @@ static void sony_nc_function_cleanup(struct platform_device *pd)
 		case 0x0143:
 		case 0x014b:
 		case 0x014c:
+		case 0x0153:
 		case 0x0163:
 			sony_nc_kbd_backlight_cleanup(pd, handle);
 			break;
@@ -1773,6 +1775,7 @@ struct kbd_backlight {
 	unsigned int base;
 	unsigned int mode;
 	unsigned int timeout;
+	unsigned int has_timeout;
 	struct device_attribute mode_attr;
 	struct device_attribute timeout_attr;
 };
@@ -1877,6 +1880,8 @@ static int sony_nc_kbd_backlight_setup(struct platform_device *pd,
 		unsigned int handle)
 {
 	int result;
+	int probe_base = 0;
+	int ctl_base = 0;
 	int ret = 0;
 
 	if (kbdbl_ctl) {
@@ -1885,11 +1890,25 @@ static int sony_nc_kbd_backlight_setup(struct platform_device *pd,
 		return -EBUSY;
 	}
 
-	/* verify the kbd backlight presence, these handles are not used for
-	 * keyboard backlight only
+	/* verify the kbd backlight presence, some of these handles are not used
+	 * for keyboard backlight only
 	 */
-	ret = sony_call_snc_handle(handle, handle == 0x0137 ? 0x0B00 : 0x0100,
-			&result);
+	switch (handle) {
+	case 0x0153:
+		probe_base = 0x0;
+		ctl_base = 0x0;
+		break;
+	case 0x0137:
+		probe_base = 0x0B00;
+		ctl_base = 0x0C00;
+		break;
+	default:
+		probe_base = 0x0100;
+		ctl_base = 0x4000;
+		break;
+	}
+
+	ret = sony_call_snc_handle(handle, probe_base, &result);
 	if (ret)
 		return ret;
 
@@ -1906,10 +1925,9 @@ static int sony_nc_kbd_backlight_setup(struct platform_device *pd,
 	kbdbl_ctl->mode = kbd_backlight;
 	kbdbl_ctl->timeout = kbd_backlight_timeout;
 	kbdbl_ctl->handle = handle;
-	if (handle == 0x0137)
-		kbdbl_ctl->base = 0x0C00;
-	else
-		kbdbl_ctl->base = 0x4000;
+	kbdbl_ctl->base = ctl_base;
+	/* Some models do not allow timeout control */
+	kbdbl_ctl->has_timeout = handle != 0x0153;
 
 	sysfs_attr_init(&kbdbl_ctl->mode_attr.attr);
 	kbdbl_ctl->mode_attr.attr.name = "kbd_backlight";
@@ -1917,22 +1935,28 @@ static int sony_nc_kbd_backlight_setup(struct platform_device *pd,
 	kbdbl_ctl->mode_attr.show = sony_nc_kbd_backlight_mode_show;
 	kbdbl_ctl->mode_attr.store = sony_nc_kbd_backlight_mode_store;
 
-	sysfs_attr_init(&kbdbl_ctl->timeout_attr.attr);
-	kbdbl_ctl->timeout_attr.attr.name = "kbd_backlight_timeout";
-	kbdbl_ctl->timeout_attr.attr.mode = S_IRUGO | S_IWUSR;
-	kbdbl_ctl->timeout_attr.show = sony_nc_kbd_backlight_timeout_show;
-	kbdbl_ctl->timeout_attr.store = sony_nc_kbd_backlight_timeout_store;
-
 	ret = device_create_file(&pd->dev, &kbdbl_ctl->mode_attr);
 	if (ret)
 		goto outkzalloc;
 
-	ret = device_create_file(&pd->dev, &kbdbl_ctl->timeout_attr);
-	if (ret)
-		goto outmode;
-
 	__sony_nc_kbd_backlight_mode_set(kbdbl_ctl->mode);
-	__sony_nc_kbd_backlight_timeout_set(kbdbl_ctl->timeout);
+
+	if (kbdbl_ctl->has_timeout) {
+		sysfs_attr_init(&kbdbl_ctl->timeout_attr.attr);
+		kbdbl_ctl->timeout_attr.attr.name = "kbd_backlight_timeout";
+		kbdbl_ctl->timeout_attr.attr.mode = S_IRUGO | S_IWUSR;
+		kbdbl_ctl->timeout_attr.show =
+			sony_nc_kbd_backlight_timeout_show;
+		kbdbl_ctl->timeout_attr.store =
+			sony_nc_kbd_backlight_timeout_store;
+
+		ret = device_create_file(&pd->dev, &kbdbl_ctl->timeout_attr);
+		if (ret)
+			goto outmode;
+
+		__sony_nc_kbd_backlight_timeout_set(kbdbl_ctl->timeout);
+	}
+
 
 	return 0;
 
@@ -1949,7 +1973,8 @@ static void sony_nc_kbd_backlight_cleanup(struct platform_device *pd,
 {
 	if (kbdbl_ctl && handle == kbdbl_ctl->handle) {
 		device_remove_file(&pd->dev, &kbdbl_ctl->mode_attr);
-		device_remove_file(&pd->dev, &kbdbl_ctl->timeout_attr);
+		if (kbdbl_ctl->has_timeout)
+			device_remove_file(&pd->dev, &kbdbl_ctl->timeout_attr);
 		kfree(kbdbl_ctl);
 		kbdbl_ctl = NULL;
 	}
diff --git a/drivers/platform/x86/surfacepro3_button.c b/drivers/platform/x86/surfacepro3_button.c
index f7dade3fd2ab..700e0fa0eec2 100644
--- a/drivers/platform/x86/surfacepro3_button.c
+++ b/drivers/platform/x86/surfacepro3_button.c
@@ -1,6 +1,6 @@
 /*
  * power/home/volume button support for
- * Microsoft Surface Pro 3 tablet.
+ * Microsoft Surface Pro 3/4 tablet.
  *
  * Copyright (c) 2015 Intel Corporation.
  * All rights reserved.
@@ -19,9 +19,10 @@
 #include <linux/acpi.h>
 #include <acpi/button.h>
 
-#define SURFACE_BUTTON_HID		"MSHW0028"
+#define SURFACE_PRO3_BUTTON_HID		"MSHW0028"
+#define SURFACE_PRO4_BUTTON_HID		"MSHW0040"
 #define SURFACE_BUTTON_OBJ_NAME		"VGBI"
-#define SURFACE_BUTTON_DEVICE_NAME	"Surface Pro 3 Buttons"
+#define SURFACE_BUTTON_DEVICE_NAME	"Surface Pro 3/4 Buttons"
 
 #define SURFACE_BUTTON_NOTIFY_PRESS_POWER	0xc6
 #define SURFACE_BUTTON_NOTIFY_RELEASE_POWER	0xc7
@@ -54,7 +55,8 @@ MODULE_LICENSE("GPL v2");
  * acpi_driver.
  */
 static const struct acpi_device_id surface_button_device_ids[] = {
-	{SURFACE_BUTTON_HID,    0},
+	{SURFACE_PRO3_BUTTON_HID,    0},
+	{SURFACE_PRO4_BUTTON_HID,    0},
 	{"", 0},
 };
 MODULE_DEVICE_TABLE(acpi, surface_button_device_ids);
@@ -109,7 +111,7 @@ static void surface_button_notify(struct acpi_device *device, u32 event)
 		break;
 	}
 	input = button->input;
-	if (KEY_RESERVED == key_code)
+	if (key_code == KEY_RESERVED)
 		return;
 	if (pressed)
 		pm_wakeup_event(&device->dev, 0);
diff --git a/drivers/platform/x86/tc1100-wmi.c b/drivers/platform/x86/tc1100-wmi.c
index 89aa976f0ab2..65b0a4845ddd 100644
--- a/drivers/platform/x86/tc1100-wmi.c
+++ b/drivers/platform/x86/tc1100-wmi.c
@@ -52,7 +52,9 @@ struct tc1100_data {
 	u32 jogdial;
 };
 
+#ifdef CONFIG_PM
 static struct tc1100_data suspend_data;
+#endif
 
 /* --------------------------------------------------------------------------
 				Device Management
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index f453d5dc085e..a268a7abf8ab 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -303,6 +303,7 @@ static struct {
 	u32 hotkey_mask:1;
 	u32 hotkey_wlsw:1;
 	u32 hotkey_tablet:1;
+	u32 kbdlight:1;
 	u32 light:1;
 	u32 light_status:1;
 	u32 bright_acpimode:1;
@@ -3488,7 +3489,7 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 	/* Do not issue duplicate brightness change events to
 	 * userspace. tpacpi_detect_brightness_capabilities() must have
 	 * been called before this point  */
-	if (acpi_video_handles_brightness_key_presses()) {
+	if (acpi_video_get_backlight_type() != acpi_backlight_vendor) {
 		pr_info("This ThinkPad has standard ACPI backlight "
 			"brightness control, supported by the ACPI "
 			"video driver\n");
@@ -4986,6 +4987,207 @@ static struct ibm_struct video_driver_data = {
 #endif /* CONFIG_THINKPAD_ACPI_VIDEO */
 
 /*************************************************************************
+ * Keyboard backlight subdriver
+ */
+
+static int kbdlight_set_level(int level)
+{
+	if (!hkey_handle)
+		return -ENXIO;
+
+	if (!acpi_evalf(hkey_handle, NULL, "MLCS", "dd", level))
+		return -EIO;
+
+	return 0;
+}
+
+static int kbdlight_get_level(void)
+{
+	int status = 0;
+
+	if (!hkey_handle)
+		return -ENXIO;
+
+	if (!acpi_evalf(hkey_handle, &status, "MLCG", "dd", 0))
+		return -EIO;
+
+	if (status < 0)
+		return status;
+
+	return status & 0x3;
+}
+
+static bool kbdlight_is_supported(void)
+{
+	int status = 0;
+
+	if (!hkey_handle)
+		return false;
+
+	if (!acpi_has_method(hkey_handle, "MLCG")) {
+		vdbg_printk(TPACPI_DBG_INIT, "kbdlight MLCG is unavailable\n");
+		return false;
+	}
+
+	if (!acpi_evalf(hkey_handle, &status, "MLCG", "qdd", 0)) {
+		vdbg_printk(TPACPI_DBG_INIT, "kbdlight MLCG failed\n");
+		return false;
+	}
+
+	if (status < 0) {
+		vdbg_printk(TPACPI_DBG_INIT, "kbdlight MLCG err: %d\n", status);
+		return false;
+	}
+
+	vdbg_printk(TPACPI_DBG_INIT, "kbdlight MLCG returned 0x%x\n", status);
+	/*
+	 * Guessed test for keyboard backlight:
+	 *
+	 * Machines with backlight keyboard return:
+	 *   b010100000010000000XX - ThinkPad X1 Carbon 3rd
+	 *   b110100010010000000XX - ThinkPad x230
+	 *   b010100000010000000XX - ThinkPad x240
+	 *   b010100000010000000XX - ThinkPad W541
+	 * (XX is current backlight level)
+	 *
+	 * Machines without backlight keyboard return:
+	 *   b10100001000000000000 - ThinkPad x230
+	 *   b10110001000000000000 - ThinkPad E430
+	 *   b00000000000000000000 - ThinkPad E450
+	 *
+	 * Candidate BITs for detection test (XOR):
+	 *   b01000000001000000000
+	 *              ^
+	 */
+	return status & BIT(9);
+}
+
+static void kbdlight_set_worker(struct work_struct *work)
+{
+	struct tpacpi_led_classdev *data =
+			container_of(work, struct tpacpi_led_classdev, work);
+
+	if (likely(tpacpi_lifecycle == TPACPI_LIFE_RUNNING))
+		kbdlight_set_level(data->new_state);
+}
+
+static void kbdlight_sysfs_set(struct led_classdev *led_cdev,
+			enum led_brightness brightness)
+{
+	struct tpacpi_led_classdev *data =
+			container_of(led_cdev,
+				     struct tpacpi_led_classdev,
+				     led_classdev);
+	data->new_state = brightness;
+	queue_work(tpacpi_wq, &data->work);
+}
+
+static enum led_brightness kbdlight_sysfs_get(struct led_classdev *led_cdev)
+{
+	int level;
+
+	level = kbdlight_get_level();
+	if (level < 0)
+		return 0;
+
+	return level;
+}
+
+static struct tpacpi_led_classdev tpacpi_led_kbdlight = {
+	.led_classdev = {
+		.name		= "tpacpi::kbd_backlight",
+		.max_brightness	= 2,
+		.brightness_set	= &kbdlight_sysfs_set,
+		.brightness_get	= &kbdlight_sysfs_get,
+		.flags		= LED_CORE_SUSPENDRESUME,
+	}
+};
+
+static int __init kbdlight_init(struct ibm_init_struct *iibm)
+{
+	int rc;
+
+	vdbg_printk(TPACPI_DBG_INIT, "initializing kbdlight subdriver\n");
+
+	TPACPI_ACPIHANDLE_INIT(hkey);
+	INIT_WORK(&tpacpi_led_kbdlight.work, kbdlight_set_worker);
+
+	if (!kbdlight_is_supported()) {
+		tp_features.kbdlight = 0;
+		vdbg_printk(TPACPI_DBG_INIT, "kbdlight is unsupported\n");
+		return 1;
+	}
+
+	tp_features.kbdlight = 1;
+
+	rc = led_classdev_register(&tpacpi_pdev->dev,
+				   &tpacpi_led_kbdlight.led_classdev);
+	if (rc < 0) {
+		tp_features.kbdlight = 0;
+		return rc;
+	}
+
+	return 0;
+}
+
+static void kbdlight_exit(void)
+{
+	if (tp_features.kbdlight)
+		led_classdev_unregister(&tpacpi_led_kbdlight.led_classdev);
+	flush_workqueue(tpacpi_wq);
+}
+
+static int kbdlight_read(struct seq_file *m)
+{
+	int level;
+
+	if (!tp_features.kbdlight) {
+		seq_printf(m, "status:\t\tnot supported\n");
+	} else {
+		level = kbdlight_get_level();
+		if (level < 0)
+			seq_printf(m, "status:\t\terror %d\n", level);
+		else
+			seq_printf(m, "status:\t\t%d\n", level);
+		seq_printf(m, "commands:\t0, 1, 2\n");
+	}
+
+	return 0;
+}
+
+static int kbdlight_write(char *buf)
+{
+	char *cmd;
+	int level = -1;
+
+	if (!tp_features.kbdlight)
+		return -ENODEV;
+
+	while ((cmd = next_cmd(&buf))) {
+		if (strlencmp(cmd, "0") == 0)
+			level = 0;
+		else if (strlencmp(cmd, "1") == 0)
+			level = 1;
+		else if (strlencmp(cmd, "2") == 0)
+			level = 2;
+		else
+			return -EINVAL;
+	}
+
+	if (level == -1)
+		return -EINVAL;
+
+	return kbdlight_set_level(level);
+}
+
+static struct ibm_struct kbdlight_driver_data = {
+	.name = "kbdlight",
+	.read = kbdlight_read,
+	.write = kbdlight_write,
+	.exit = kbdlight_exit,
+};
+
+/*************************************************************************
  * Light (thinklight) subdriver
  */
 
@@ -9207,6 +9409,10 @@ static struct ibm_init_struct ibms_init[] __initdata = {
 	},
 #endif
 	{
+		.init = kbdlight_init,
+		.data = &kbdlight_driver_data,
+	},
+	{
 		.init = light_init,
 		.data = &light_driver_data,
 	},
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index c01302989ee4..73833079bac8 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -51,6 +51,7 @@
 #include <linux/dmi.h>
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
+#include <linux/rfkill.h>
 #include <linux/toshiba.h>
 #include <acpi/video.h>
 
@@ -114,6 +115,7 @@ MODULE_LICENSE("GPL");
 #define HCI_VIDEO_OUT			0x001c
 #define HCI_HOTKEY_EVENT		0x001e
 #define HCI_LCD_BRIGHTNESS		0x002a
+#define HCI_WIRELESS			0x0056
 #define HCI_ACCELEROMETER		0x006d
 #define HCI_KBD_ILLUMINATION		0x0095
 #define HCI_ECO_MODE			0x0097
@@ -148,6 +150,10 @@ MODULE_LICENSE("GPL");
 #define SCI_KBD_MODE_ON			0x8
 #define SCI_KBD_MODE_OFF		0x10
 #define SCI_KBD_TIME_MAX		0x3c001a
+#define HCI_WIRELESS_STATUS		0x1
+#define HCI_WIRELESS_WWAN		0x3
+#define HCI_WIRELESS_WWAN_STATUS	0x2000
+#define HCI_WIRELESS_WWAN_POWER		0x4000
 #define SCI_USB_CHARGE_MODE_MASK	0xff
 #define SCI_USB_CHARGE_DISABLED		0x00
 #define SCI_USB_CHARGE_ALTERNATE	0x09
@@ -169,6 +175,7 @@ struct toshiba_acpi_dev {
 	struct led_classdev kbd_led;
 	struct led_classdev eco_led;
 	struct miscdevice miscdev;
+	struct rfkill *wwan_rfk;
 
 	int force_fan;
 	int last_key_event;
@@ -197,12 +204,15 @@ struct toshiba_acpi_dev {
 	unsigned int kbd_function_keys_supported:1;
 	unsigned int panel_power_on_supported:1;
 	unsigned int usb_three_supported:1;
+	unsigned int wwan_supported:1;
 	unsigned int sysfs_created:1;
 	unsigned int special_functions;
 
+	bool kbd_event_generated;
 	bool kbd_led_registered;
 	bool illumination_led_registered;
 	bool eco_led_registered;
+	bool killswitch;
 };
 
 static struct toshiba_acpi_dev *toshiba_acpi;
@@ -516,6 +526,7 @@ static void toshiba_kbd_illum_available(struct toshiba_acpi_dev *dev)
 
 	dev->kbd_illum_supported = 0;
 	dev->kbd_led_registered = false;
+	dev->kbd_event_generated = false;
 
 	if (!sci_open(dev))
 		return;
@@ -1085,6 +1096,104 @@ static int toshiba_hotkey_event_type_get(struct toshiba_acpi_dev *dev,
 	return -EIO;
 }
 
+/* Wireless status (RFKill, WLAN, BT, WWAN) */
+static int toshiba_wireless_status(struct toshiba_acpi_dev *dev)
+{
+	u32 in[TCI_WORDS] = { HCI_GET, HCI_WIRELESS, 0, 0, 0, 0 };
+	u32 out[TCI_WORDS];
+	acpi_status status;
+
+	in[3] = HCI_WIRELESS_STATUS;
+	status = tci_raw(dev, in, out);
+
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to get Wireless status failed\n");
+		return -EIO;
+	}
+
+	if (out[0] == TOS_NOT_SUPPORTED)
+		return -ENODEV;
+
+	if (out[0] != TOS_SUCCESS)
+		return -EIO;
+
+	dev->killswitch = !!(out[2] & HCI_WIRELESS_STATUS);
+
+	return 0;
+}
+
+/* WWAN */
+static void toshiba_wwan_available(struct toshiba_acpi_dev *dev)
+{
+	u32 in[TCI_WORDS] = { HCI_GET, HCI_WIRELESS, 0, 0, 0, 0 };
+	u32 out[TCI_WORDS];
+	acpi_status status;
+
+	dev->wwan_supported = 0;
+
+	/*
+	 * WWAN support can be queried by setting the in[3] value to
+	 * HCI_WIRELESS_WWAN (0x03).
+	 *
+	 * If supported, out[0] contains TOS_SUCCESS and out[2] contains
+	 * HCI_WIRELESS_WWAN_STATUS (0x2000).
+	 *
+	 * If not supported, out[0] contains TOS_INPUT_DATA_ERROR (0x8300)
+	 * or TOS_NOT_SUPPORTED (0x8000).
+	 */
+	in[3] = HCI_WIRELESS_WWAN;
+	status = tci_raw(dev, in, out);
+
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to get WWAN status failed\n");
+		return;
+	}
+
+	if (out[0] != TOS_SUCCESS)
+		return;
+
+	dev->wwan_supported = (out[2] == HCI_WIRELESS_WWAN_STATUS);
+}
+
+static int toshiba_wwan_set(struct toshiba_acpi_dev *dev, u32 state)
+{
+	u32 in[TCI_WORDS] = { HCI_SET, HCI_WIRELESS, state, 0, 0, 0 };
+	u32 out[TCI_WORDS];
+	acpi_status status;
+
+	in[3] = HCI_WIRELESS_WWAN_STATUS;
+	status = tci_raw(dev, in, out);
+
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to set WWAN status failed\n");
+		return -EIO;
+	}
+
+	if (out[0] == TOS_NOT_SUPPORTED)
+		return -ENODEV;
+
+	if (out[0] != TOS_SUCCESS)
+		return -EIO;
+
+	/*
+	 * Some devices only need to call HCI_WIRELESS_WWAN_STATUS to
+	 * (de)activate the device, but some others need the
+	 * HCI_WIRELESS_WWAN_POWER call as well.
+	 */
+	in[3] = HCI_WIRELESS_WWAN_POWER;
+	status = tci_raw(dev, in, out);
+
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to set WWAN power failed\n");
+		return -EIO;
+	}
+
+	if (out[0] == TOS_NOT_SUPPORTED)
+		return -ENODEV;
+
+	return out[0] == TOS_SUCCESS ? 0 : -EIO;
+}
+
 /* Transflective Backlight */
 static int get_tr_backlight_status(struct toshiba_acpi_dev *dev, u32 *status)
 {
@@ -1535,6 +1644,11 @@ static const struct backlight_ops toshiba_backlight_data = {
 	.update_status  = set_lcd_status,
 };
 
+/* Keyboard backlight work */
+static void toshiba_acpi_kbd_bl_work(struct work_struct *work);
+
+static DECLARE_WORK(kbd_bl_work, toshiba_acpi_kbd_bl_work);
+
 /*
  * Sysfs files
  */
@@ -1634,6 +1748,24 @@ static ssize_t kbd_backlight_mode_store(struct device *dev,
 			return ret;
 
 		toshiba->kbd_mode = mode;
+
+		/*
+		 * Some laptop models with the second generation backlit
+		 * keyboard (type 2) do not generate the keyboard backlight
+		 * changed event (0x92), and thus, the driver will never update
+		 * the sysfs entries.
+		 *
+		 * The event is generated right when changing the keyboard
+		 * backlight mode and the *notify function will set the
+		 * kbd_event_generated to true.
+		 *
+		 * In case the event is not generated, schedule the keyboard
+		 * backlight work to update the sysfs entries and emulate the
+		 * event via genetlink.
+		 */
+		if (toshiba->kbd_type == 2 &&
+		    !toshiba_acpi->kbd_event_generated)
+			schedule_work(&kbd_bl_work);
 	}
 
 	return count;
@@ -2166,6 +2298,21 @@ static struct attribute_group toshiba_attr_group = {
 	.attrs = toshiba_attributes,
 };
 
+static void toshiba_acpi_kbd_bl_work(struct work_struct *work)
+{
+	struct acpi_device *acpi_dev = toshiba_acpi->acpi_dev;
+
+	/* Update the sysfs entries */
+	if (sysfs_update_group(&acpi_dev->dev.kobj,
+			       &toshiba_attr_group))
+		pr_err("Unable to update sysfs entries\n");
+
+	/* Emulate the keyboard backlight event */
+	acpi_bus_generate_netlink_event(acpi_dev->pnp.device_class,
+					dev_name(&acpi_dev->dev),
+					0x92, 0);
+}
+
 /*
  * Misc device
  */
@@ -2242,6 +2389,67 @@ static const struct file_operations toshiba_acpi_fops = {
 };
 
 /*
+ * WWAN RFKill handlers
+ */
+static int toshiba_acpi_wwan_set_block(void *data, bool blocked)
+{
+	struct toshiba_acpi_dev *dev = data;
+	int ret;
+
+	ret = toshiba_wireless_status(dev);
+	if (ret)
+		return ret;
+
+	if (!dev->killswitch)
+		return 0;
+
+	return toshiba_wwan_set(dev, !blocked);
+}
+
+static void toshiba_acpi_wwan_poll(struct rfkill *rfkill, void *data)
+{
+	struct toshiba_acpi_dev *dev = data;
+
+	if (toshiba_wireless_status(dev))
+		return;
+
+	rfkill_set_hw_state(dev->wwan_rfk, !dev->killswitch);
+}
+
+static const struct rfkill_ops wwan_rfk_ops = {
+	.set_block = toshiba_acpi_wwan_set_block,
+	.poll = toshiba_acpi_wwan_poll,
+};
+
+static int toshiba_acpi_setup_wwan_rfkill(struct toshiba_acpi_dev *dev)
+{
+	int ret = toshiba_wireless_status(dev);
+
+	if (ret)
+		return ret;
+
+	dev->wwan_rfk = rfkill_alloc("Toshiba WWAN",
+				     &dev->acpi_dev->dev,
+				     RFKILL_TYPE_WWAN,
+				     &wwan_rfk_ops,
+				     dev);
+	if (!dev->wwan_rfk) {
+		pr_err("Unable to allocate WWAN rfkill device\n");
+		return -ENOMEM;
+	}
+
+	rfkill_set_hw_state(dev->wwan_rfk, !dev->killswitch);
+
+	ret = rfkill_register(dev->wwan_rfk);
+	if (ret) {
+		pr_err("Unable to register WWAN rfkill device\n");
+		rfkill_destroy(dev->wwan_rfk);
+	}
+
+	return ret;
+}
+
+/*
  * Hotkeys
  */
 static int toshiba_acpi_enable_hotkeys(struct toshiba_acpi_dev *dev)
@@ -2484,6 +2692,14 @@ static int toshiba_acpi_setup_backlight(struct toshiba_acpi_dev *dev)
 	brightness = __get_lcd_brightness(dev);
 	if (brightness < 0)
 		return 0;
+	/*
+	 * If transflective backlight is supported and the brightness is zero
+	 * (lowest brightness level), the set_lcd_brightness function will
+	 * activate the transflective backlight, making the LCD appear to be
+	 * turned off, simply increment the brightness level to avoid that.
+	 */
+	if (dev->tr_backlight_supported && brightness == 0)
+		brightness++;
 	ret = set_lcd_brightness(dev, brightness);
 	if (ret) {
 		pr_debug("Backlight method is read-only, disabling backlight support\n");
@@ -2561,6 +2777,8 @@ static void print_supported_features(struct toshiba_acpi_dev *dev)
 		pr_cont(" panel-power-on");
 	if (dev->usb_three_supported)
 		pr_cont(" usb3");
+	if (dev->wwan_supported)
+		pr_cont(" wwan");
 
 	pr_cont("\n");
 }
@@ -2598,6 +2816,11 @@ static int toshiba_acpi_remove(struct acpi_device *acpi_dev)
 	if (dev->eco_led_registered)
 		led_classdev_unregister(&dev->eco_led);
 
+	if (dev->wwan_rfk) {
+		rfkill_unregister(dev->wwan_rfk);
+		rfkill_destroy(dev->wwan_rfk);
+	}
+
 	if (toshiba_acpi)
 		toshiba_acpi = NULL;
 
@@ -2736,6 +2959,10 @@ static int toshiba_acpi_add(struct acpi_device *acpi_dev)
 	ret = get_fan_status(dev, &dummy);
 	dev->fan_supported = !ret;
 
+	toshiba_wwan_available(dev);
+	if (dev->wwan_supported)
+		toshiba_acpi_setup_wwan_rfkill(dev);
+
 	print_supported_features(dev);
 
 	ret = sysfs_create_group(&dev->acpi_dev->dev.kobj,
@@ -2760,7 +2987,6 @@ error:
 static void toshiba_acpi_notify(struct acpi_device *acpi_dev, u32 event)
 {
 	struct toshiba_acpi_dev *dev = acpi_driver_data(acpi_dev);
-	int ret;
 
 	switch (event) {
 	case 0x80: /* Hotkeys and some system events */
@@ -2790,10 +3016,10 @@ static void toshiba_acpi_notify(struct acpi_device *acpi_dev, u32 event)
 		pr_info("SATA power event received %x\n", event);
 		break;
 	case 0x92: /* Keyboard backlight mode changed */
+		toshiba_acpi->kbd_event_generated = true;
 		/* Update sysfs entries */
-		ret = sysfs_update_group(&acpi_dev->dev.kobj,
-					 &toshiba_attr_group);
-		if (ret)
+		if (sysfs_update_group(&acpi_dev->dev.kobj,
+				       &toshiba_attr_group))
 			pr_err("Unable to update sysfs entries\n");
 		break;
 	case 0x85: /* Unknown */
@@ -2808,7 +3034,8 @@ static void toshiba_acpi_notify(struct acpi_device *acpi_dev, u32 event)
 
 	acpi_bus_generate_netlink_event(acpi_dev->pnp.device_class,
 					dev_name(&acpi_dev->dev),
-					event, 0);
+					event, (event == 0x80) ?
+					dev->last_key_event : 0);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -2832,12 +3059,15 @@ static int toshiba_acpi_resume(struct device *device)
 	struct toshiba_acpi_dev *dev = acpi_driver_data(to_acpi_device(device));
 
 	if (dev->hotkey_dev) {
-		int error = toshiba_acpi_enable_hotkeys(dev);
-
-		if (error)
+		if (toshiba_acpi_enable_hotkeys(dev))
 			pr_info("Unable to re-enable hotkeys\n");
 	}
 
+	if (dev->wwan_rfk) {
+		if (!toshiba_wireless_status(dev))
+			rfkill_set_hw_state(dev->wwan_rfk, !dev->killswitch);
+	}
+
 	return 0;
 }
 #endif
diff --git a/drivers/platform/x86/toshiba_bluetooth.c b/drivers/platform/x86/toshiba_bluetooth.c
index c5e45089ac51..5db495dd018e 100644
--- a/drivers/platform/x86/toshiba_bluetooth.c
+++ b/drivers/platform/x86/toshiba_bluetooth.c
@@ -78,7 +78,7 @@ static int toshiba_bluetooth_present(acpi_handle handle)
 	 */
 	result = acpi_evaluate_integer(handle, "_STA", NULL, &bt_present);
 	if (ACPI_FAILURE(result)) {
-		pr_err("ACPI call to query Bluetooth presence failed");
+		pr_err("ACPI call to query Bluetooth presence failed\n");
 		return -ENXIO;
 	} else if (!bt_present) {
 		pr_info("Bluetooth device not present\n");
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index 2f4641a0e88b..8cf0dae78555 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -148,6 +148,7 @@ config PWM_EP93XX
 
 config PWM_FSL_FTM
 	tristate "Freescale FlexTimer Module (FTM) PWM support"
+	depends on HAS_IOMEM
 	depends on OF
 	select REGMAP_MMIO
 	help
@@ -222,18 +223,12 @@ config PWM_LPC32XX
 	  will be called pwm-lpc32xx.
 
 config PWM_LPSS
-	tristate "Intel LPSS PWM support"
-	depends on X86
-	help
-	  Generic PWM framework driver for Intel Low Power Subsystem PWM
-	  controller.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called pwm-lpss.
+	tristate
 
 config PWM_LPSS_PCI
 	tristate "Intel LPSS PWM PCI driver"
-	depends on PWM_LPSS && PCI
+	depends on X86 && PCI
+	select PWM_LPSS
 	help
 	  The PCI driver for Intel Low Power Subsystem PWM controller.
 
@@ -242,7 +237,8 @@ config PWM_LPSS_PCI
 
 config PWM_LPSS_PLATFORM
 	tristate "Intel LPSS PWM platform driver"
-	depends on PWM_LPSS && ACPI
+	depends on X86 && ACPI
+	select PWM_LPSS
 	help
 	  The platform driver for Intel Low Power Subsystem PWM controller.
 
@@ -270,6 +266,15 @@ config PWM_MXS
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-mxs.
 
+config PWM_OMAP_DMTIMER
+	tristate "OMAP Dual-Mode Timer PWM support"
+	depends on OF && ARCH_OMAP && OMAP_DM_TIMER
+	help
+	  Generic PWM framework driver for OMAP Dual-Mode Timer PWM output
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-omap-dmtimer
+
 config PWM_PCA9685
 	tristate "NXP PCA9685 PWM driver"
 	depends on I2C
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index 69b8275f3c08..dd35bc121a18 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_PWM_LPSS_PCI)	+= pwm-lpss-pci.o
 obj-$(CONFIG_PWM_LPSS_PLATFORM)	+= pwm-lpss-platform.o
 obj-$(CONFIG_PWM_MTK_DISP)	+= pwm-mtk-disp.o
 obj-$(CONFIG_PWM_MXS)		+= pwm-mxs.o
+obj-$(CONFIG_PWM_OMAP_DMTIMER)	+= pwm-omap-dmtimer.o
 obj-$(CONFIG_PWM_PCA9685)	+= pwm-pca9685.o
 obj-$(CONFIG_PWM_PUV3)		+= pwm-puv3.o
 obj-$(CONFIG_PWM_PXA)		+= pwm-pxa.o
diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c
index d24ca5f281b4..7831bc6b51dd 100644
--- a/drivers/pwm/core.c
+++ b/drivers/pwm/core.c
@@ -889,7 +889,7 @@ EXPORT_SYMBOL_GPL(devm_pwm_put);
   */
 bool pwm_can_sleep(struct pwm_device *pwm)
 {
-	return pwm->chip->can_sleep;
+	return true;
 }
 EXPORT_SYMBOL_GPL(pwm_can_sleep);
 
diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c
index b4c7f956b6fa..c5dbf16d810b 100644
--- a/drivers/pwm/pwm-bcm2835.c
+++ b/drivers/pwm/pwm-bcm2835.c
@@ -29,7 +29,6 @@
 struct bcm2835_pwm {
 	struct pwm_chip chip;
 	struct device *dev;
-	unsigned long scaler;
 	void __iomem *base;
 	struct clk *clk;
 };
@@ -66,6 +65,15 @@ static int bcm2835_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 			      int duty_ns, int period_ns)
 {
 	struct bcm2835_pwm *pc = to_bcm2835_pwm(chip);
+	unsigned long rate = clk_get_rate(pc->clk);
+	unsigned long scaler;
+
+	if (!rate) {
+		dev_err(pc->dev, "failed to get clock rate\n");
+		return -EINVAL;
+	}
+
+	scaler = NSEC_PER_SEC / rate;
 
 	if (period_ns <= MIN_PERIOD) {
 		dev_err(pc->dev, "period %d not supported, minimum %d\n",
@@ -73,8 +81,8 @@ static int bcm2835_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		return -EINVAL;
 	}
 
-	writel(duty_ns / pc->scaler, pc->base + DUTY(pwm->hwpwm));
-	writel(period_ns / pc->scaler, pc->base + PERIOD(pwm->hwpwm));
+	writel(duty_ns / scaler, pc->base + DUTY(pwm->hwpwm));
+	writel(period_ns / scaler, pc->base + PERIOD(pwm->hwpwm));
 
 	return 0;
 }
@@ -156,8 +164,6 @@ static int bcm2835_pwm_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	pc->scaler = NSEC_PER_SEC / clk_get_rate(pc->clk);
-
 	pc->chip.dev = &pdev->dev;
 	pc->chip.ops = &bcm2835_pwm_ops;
 	pc->chip.npwm = 2;
@@ -200,6 +206,6 @@ static struct platform_driver bcm2835_pwm_driver = {
 };
 module_platform_driver(bcm2835_pwm_driver);
 
-MODULE_AUTHOR("Bart Tanghe <bart.tanghe@thomasmore.be");
+MODULE_AUTHOR("Bart Tanghe <bart.tanghe@thomasmore.be>");
 MODULE_DESCRIPTION("Broadcom BCM2835 PWM driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c
index f9dfc8b6407a..7225ac6b3df5 100644
--- a/drivers/pwm/pwm-fsl-ftm.c
+++ b/drivers/pwm/pwm-fsl-ftm.c
@@ -80,7 +80,6 @@ struct fsl_pwm_chip {
 
 	struct mutex lock;
 
-	unsigned int use_count;
 	unsigned int cnt_select;
 	unsigned int clk_ps;
 
@@ -300,9 +299,6 @@ static int fsl_counter_clock_enable(struct fsl_pwm_chip *fpc)
 {
 	int ret;
 
-	if (fpc->use_count++ != 0)
-		return 0;
-
 	/* select counter clock source */
 	regmap_update_bits(fpc->regmap, FTM_SC, FTM_SC_CLK_MASK,
 			   FTM_SC_CLK(fpc->cnt_select));
@@ -334,25 +330,6 @@ static int fsl_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	return ret;
 }
 
-static void fsl_counter_clock_disable(struct fsl_pwm_chip *fpc)
-{
-	/*
-	 * already disabled, do nothing
-	 */
-	if (fpc->use_count == 0)
-		return;
-
-	/* there are still users, so can't disable yet */
-	if (--fpc->use_count > 0)
-		return;
-
-	/* no users left, disable PWM counter clock */
-	regmap_update_bits(fpc->regmap, FTM_SC, FTM_SC_CLK_MASK, 0);
-
-	clk_disable_unprepare(fpc->clk[FSL_PWM_CLK_CNTEN]);
-	clk_disable_unprepare(fpc->clk[fpc->cnt_select]);
-}
-
 static void fsl_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct fsl_pwm_chip *fpc = to_fsl_chip(chip);
@@ -362,7 +339,8 @@ static void fsl_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	regmap_update_bits(fpc->regmap, FTM_OUTMASK, BIT(pwm->hwpwm),
 			   BIT(pwm->hwpwm));
 
-	fsl_counter_clock_disable(fpc);
+	clk_disable_unprepare(fpc->clk[FSL_PWM_CLK_CNTEN]);
+	clk_disable_unprepare(fpc->clk[fpc->cnt_select]);
 
 	regmap_read(fpc->regmap, FTM_OUTMASK, &val);
 	if ((val & 0xFF) == 0xFF)
@@ -492,17 +470,24 @@ static int fsl_pwm_remove(struct platform_device *pdev)
 static int fsl_pwm_suspend(struct device *dev)
 {
 	struct fsl_pwm_chip *fpc = dev_get_drvdata(dev);
-	u32 val;
+	int i;
 
 	regcache_cache_only(fpc->regmap, true);
 	regcache_mark_dirty(fpc->regmap);
 
-	/* read from cache */
-	regmap_read(fpc->regmap, FTM_OUTMASK, &val);
-	if ((val & 0xFF) != 0xFF) {
+	for (i = 0; i < fpc->chip.npwm; i++) {
+		struct pwm_device *pwm = &fpc->chip.pwms[i];
+
+		if (!test_bit(PWMF_REQUESTED, &pwm->flags))
+			continue;
+
+		clk_disable_unprepare(fpc->clk[FSL_PWM_CLK_SYS]);
+
+		if (!pwm_is_enabled(pwm))
+			continue;
+
 		clk_disable_unprepare(fpc->clk[FSL_PWM_CLK_CNTEN]);
 		clk_disable_unprepare(fpc->clk[fpc->cnt_select]);
-		clk_disable_unprepare(fpc->clk[FSL_PWM_CLK_SYS]);
 	}
 
 	return 0;
@@ -511,12 +496,19 @@ static int fsl_pwm_suspend(struct device *dev)
 static int fsl_pwm_resume(struct device *dev)
 {
 	struct fsl_pwm_chip *fpc = dev_get_drvdata(dev);
-	u32 val;
+	int i;
+
+	for (i = 0; i < fpc->chip.npwm; i++) {
+		struct pwm_device *pwm = &fpc->chip.pwms[i];
+
+		if (!test_bit(PWMF_REQUESTED, &pwm->flags))
+			continue;
 
-	/* read from cache */
-	regmap_read(fpc->regmap, FTM_OUTMASK, &val);
-	if ((val & 0xFF) != 0xFF) {
 		clk_prepare_enable(fpc->clk[FSL_PWM_CLK_SYS]);
+
+		if (!pwm_is_enabled(pwm))
+			continue;
+
 		clk_prepare_enable(fpc->clk[fpc->cnt_select]);
 		clk_prepare_enable(fpc->clk[FSL_PWM_CLK_CNTEN]);
 	}
diff --git a/drivers/pwm/pwm-lpc32xx.c b/drivers/pwm/pwm-lpc32xx.c
index 9fde60ce8e7b..4d470c1a406a 100644
--- a/drivers/pwm/pwm-lpc32xx.c
+++ b/drivers/pwm/pwm-lpc32xx.c
@@ -24,9 +24,7 @@ struct lpc32xx_pwm_chip {
 	void __iomem *base;
 };
 
-#define PWM_ENABLE	(1 << 31)
-#define PWM_RELOADV(x)	(((x) & 0xFF) << 8)
-#define PWM_DUTY(x)	((x) & 0xFF)
+#define PWM_ENABLE	BIT(31)
 
 #define to_lpc32xx_pwm_chip(_chip) \
 	container_of(_chip, struct lpc32xx_pwm_chip, chip)
@@ -38,40 +36,27 @@ static int lpc32xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	unsigned long long c;
 	int period_cycles, duty_cycles;
 	u32 val;
-
-	c = clk_get_rate(lpc32xx->clk) / 256;
-	c = c * period_ns;
-	do_div(c, NSEC_PER_SEC);
-
-	/* Handle high and low extremes */
-	if (c == 0)
-		c = 1;
-	if (c > 255)
-		c = 0; /* 0 set division by 256 */
-	period_cycles = c;
-
-	/* The duty-cycle value is as follows:
-	 *
-	 *  DUTY-CYCLE     HIGH LEVEL
-	 *      1            99.9%
-	 *      25           90.0%
-	 *      128          50.0%
-	 *      220          10.0%
-	 *      255           0.1%
-	 *      0             0.0%
-	 *
-	 * In other words, the register value is duty-cycle % 256 with
-	 * duty-cycle in the range 1-256.
-	 */
-	c = 256 * duty_ns;
-	do_div(c, period_ns);
-	if (c > 255)
-		c = 255;
-	duty_cycles = 256 - c;
+	c = clk_get_rate(lpc32xx->clk);
+
+	/* The highest acceptable divisor is 256, which is represented by 0 */
+	period_cycles = div64_u64(c * period_ns,
+			       (unsigned long long)NSEC_PER_SEC * 256);
+	if (!period_cycles || period_cycles > 256)
+		return -ERANGE;
+	if (period_cycles == 256)
+		period_cycles = 0;
+
+	/* Compute 256 x #duty/period value and care for corner cases */
+	duty_cycles = div64_u64((unsigned long long)(period_ns - duty_ns) * 256,
+				period_ns);
+	if (!duty_cycles)
+		duty_cycles = 1;
+	if (duty_cycles > 255)
+		duty_cycles = 255;
 
 	val = readl(lpc32xx->base + (pwm->hwpwm << 2));
 	val &= ~0xFFFF;
-	val |= PWM_RELOADV(period_cycles) | PWM_DUTY(duty_cycles);
+	val |= (period_cycles << 8) | duty_cycles;
 	writel(val, lpc32xx->base + (pwm->hwpwm << 2));
 
 	return 0;
@@ -83,7 +68,7 @@ static int lpc32xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	u32 val;
 	int ret;
 
-	ret = clk_enable(lpc32xx->clk);
+	ret = clk_prepare_enable(lpc32xx->clk);
 	if (ret)
 		return ret;
 
@@ -103,7 +88,7 @@ static void lpc32xx_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	val &= ~PWM_ENABLE;
 	writel(val, lpc32xx->base + (pwm->hwpwm << 2));
 
-	clk_disable(lpc32xx->clk);
+	clk_disable_unprepare(lpc32xx->clk);
 }
 
 static const struct pwm_ops lpc32xx_pwm_ops = {
@@ -134,7 +119,7 @@ static int lpc32xx_pwm_probe(struct platform_device *pdev)
 
 	lpc32xx->chip.dev = &pdev->dev;
 	lpc32xx->chip.ops = &lpc32xx_pwm_ops;
-	lpc32xx->chip.npwm = 2;
+	lpc32xx->chip.npwm = 1;
 	lpc32xx->chip.base = -1;
 
 	ret = pwmchip_add(&lpc32xx->chip);
diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c
index 25044104003b..295b963dbddb 100644
--- a/drivers/pwm/pwm-lpss.c
+++ b/drivers/pwm/pwm-lpss.c
@@ -13,10 +13,12 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/time.h>
 
 #include "pwm-lpss.h"
 
@@ -24,11 +26,8 @@
 #define PWM_ENABLE			BIT(31)
 #define PWM_SW_UPDATE			BIT(30)
 #define PWM_BASE_UNIT_SHIFT		8
-#define PWM_BASE_UNIT_MASK		0x00ffff00
 #define PWM_ON_TIME_DIV_MASK		0x000000ff
 #define PWM_DIVISION_CORRECTION		0x2
-#define PWM_LIMIT			(0x8000 + PWM_DIVISION_CORRECTION)
-#define NSECS_PER_SEC			1000000000UL
 
 /* Size of each PWM register space if multiple */
 #define PWM_SIZE			0x400
@@ -36,13 +35,14 @@
 struct pwm_lpss_chip {
 	struct pwm_chip chip;
 	void __iomem *regs;
-	unsigned long clk_rate;
+	const struct pwm_lpss_boardinfo *info;
 };
 
 /* BayTrail */
 const struct pwm_lpss_boardinfo pwm_lpss_byt_info = {
 	.clk_rate = 25000000,
 	.npwm = 1,
+	.base_unit_bits = 16,
 };
 EXPORT_SYMBOL_GPL(pwm_lpss_byt_info);
 
@@ -50,6 +50,7 @@ EXPORT_SYMBOL_GPL(pwm_lpss_byt_info);
 const struct pwm_lpss_boardinfo pwm_lpss_bsw_info = {
 	.clk_rate = 19200000,
 	.npwm = 1,
+	.base_unit_bits = 16,
 };
 EXPORT_SYMBOL_GPL(pwm_lpss_bsw_info);
 
@@ -57,6 +58,7 @@ EXPORT_SYMBOL_GPL(pwm_lpss_bsw_info);
 const struct pwm_lpss_boardinfo pwm_lpss_bxt_info = {
 	.clk_rate = 19200000,
 	.npwm = 4,
+	.base_unit_bits = 22,
 };
 EXPORT_SYMBOL_GPL(pwm_lpss_bxt_info);
 
@@ -79,28 +81,37 @@ static inline void pwm_lpss_write(const struct pwm_device *pwm, u32 value)
 	writel(value, lpwm->regs + pwm->hwpwm * PWM_SIZE + PWM);
 }
 
+static void pwm_lpss_update(struct pwm_device *pwm)
+{
+	pwm_lpss_write(pwm, pwm_lpss_read(pwm) | PWM_SW_UPDATE);
+	/* Give it some time to propagate */
+	usleep_range(10, 50);
+}
+
 static int pwm_lpss_config(struct pwm_chip *chip, struct pwm_device *pwm,
 			   int duty_ns, int period_ns)
 {
 	struct pwm_lpss_chip *lpwm = to_lpwm(chip);
 	u8 on_time_div;
-	unsigned long c;
-	unsigned long long base_unit, freq = NSECS_PER_SEC;
+	unsigned long c, base_unit_range;
+	unsigned long long base_unit, freq = NSEC_PER_SEC;
 	u32 ctrl;
 
 	do_div(freq, period_ns);
 
-	/* The equation is: base_unit = ((freq / c) * 65536) + correction */
-	base_unit = freq * 65536;
+	/*
+	 * The equation is:
+	 * base_unit = ((freq / c) * base_unit_range) + correction
+	 */
+	base_unit_range = BIT(lpwm->info->base_unit_bits);
+	base_unit = freq * base_unit_range;
 
-	c = lpwm->clk_rate;
+	c = lpwm->info->clk_rate;
 	if (!c)
 		return -EINVAL;
 
 	do_div(base_unit, c);
 	base_unit += PWM_DIVISION_CORRECTION;
-	if (base_unit > PWM_LIMIT)
-		return -EINVAL;
 
 	if (duty_ns <= 0)
 		duty_ns = 1;
@@ -109,13 +120,20 @@ static int pwm_lpss_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	pm_runtime_get_sync(chip->dev);
 
 	ctrl = pwm_lpss_read(pwm);
-	ctrl &= ~(PWM_BASE_UNIT_MASK | PWM_ON_TIME_DIV_MASK);
-	ctrl |= (u16) base_unit << PWM_BASE_UNIT_SHIFT;
+	ctrl &= ~PWM_ON_TIME_DIV_MASK;
+	ctrl &= ~((base_unit_range - 1) << PWM_BASE_UNIT_SHIFT);
+	base_unit &= (base_unit_range - 1);
+	ctrl |= (u32) base_unit << PWM_BASE_UNIT_SHIFT;
 	ctrl |= on_time_div;
-	/* request PWM to update on next cycle */
-	ctrl |= PWM_SW_UPDATE;
 	pwm_lpss_write(pwm, ctrl);
 
+	/*
+	 * If the PWM is already enabled we need to notify the hardware
+	 * about the change by setting PWM_SW_UPDATE.
+	 */
+	if (pwm_is_enabled(pwm))
+		pwm_lpss_update(pwm);
+
 	pm_runtime_put(chip->dev);
 
 	return 0;
@@ -124,6 +142,12 @@ static int pwm_lpss_config(struct pwm_chip *chip, struct pwm_device *pwm,
 static int pwm_lpss_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	pm_runtime_get_sync(chip->dev);
+
+	/*
+	 * Hardware must first see PWM_SW_UPDATE before the PWM can be
+	 * enabled.
+	 */
+	pwm_lpss_update(pwm);
 	pwm_lpss_write(pwm, pwm_lpss_read(pwm) | PWM_ENABLE);
 	return 0;
 }
@@ -135,7 +159,6 @@ static void pwm_lpss_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 }
 
 static const struct pwm_ops pwm_lpss_ops = {
-	.free = pwm_lpss_disable,
 	.config = pwm_lpss_config,
 	.enable = pwm_lpss_enable,
 	.disable = pwm_lpss_disable,
@@ -156,7 +179,7 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r,
 	if (IS_ERR(lpwm->regs))
 		return ERR_CAST(lpwm->regs);
 
-	lpwm->clk_rate = info->clk_rate;
+	lpwm->info = info;
 	lpwm->chip.dev = dev;
 	lpwm->chip.ops = &pwm_lpss_ops;
 	lpwm->chip.base = -1;
diff --git a/drivers/pwm/pwm-lpss.h b/drivers/pwm/pwm-lpss.h
index e8cf337ae1d1..04766e0d41aa 100644
--- a/drivers/pwm/pwm-lpss.h
+++ b/drivers/pwm/pwm-lpss.h
@@ -21,6 +21,7 @@ struct pwm_lpss_chip;
 struct pwm_lpss_boardinfo {
 	unsigned long clk_rate;
 	unsigned int npwm;
+	unsigned long base_unit_bits;
 };
 
 extern const struct pwm_lpss_boardinfo pwm_lpss_byt_info;
diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c
new file mode 100644
index 000000000000..826634ec0d5c
--- /dev/null
+++ b/drivers/pwm/pwm-omap-dmtimer.c
@@ -0,0 +1,327 @@
+/*
+ * Copyright (c) 2015 Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (c) 2014 Joachim Eastwood <manabian@gmail.com>
+ * Copyright (c) 2012 NeilBrown <neilb@suse.de>
+ * Heavily based on earlier code which is:
+ * Copyright (c) 2010 Grant Erickson <marathon96@gmail.com>
+ *
+ * Also based on pwm-samsung.c
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * Description:
+ *   This file is the core OMAP support for the generic, Linux
+ *   PWM driver / controller, using the OMAP's dual-mode timers.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_data/pwm_omap_dmtimer.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/pwm.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+
+#define DM_TIMER_LOAD_MIN 0xfffffffe
+
+struct pwm_omap_dmtimer_chip {
+	struct pwm_chip chip;
+	struct mutex mutex;
+	pwm_omap_dmtimer *dm_timer;
+	struct pwm_omap_dmtimer_pdata *pdata;
+	struct platform_device *dm_timer_pdev;
+};
+
+static inline struct pwm_omap_dmtimer_chip *
+to_pwm_omap_dmtimer_chip(struct pwm_chip *chip)
+{
+	return container_of(chip, struct pwm_omap_dmtimer_chip, chip);
+}
+
+static int pwm_omap_dmtimer_calc_value(unsigned long clk_rate, int ns)
+{
+	u64 c = (u64)clk_rate * ns;
+
+	do_div(c, NSEC_PER_SEC);
+
+	return DM_TIMER_LOAD_MIN - c;
+}
+
+static void pwm_omap_dmtimer_start(struct pwm_omap_dmtimer_chip *omap)
+{
+	/*
+	 * According to OMAP 4 TRM section 22.2.4.10 the counter should be
+	 * started at 0xFFFFFFFE when overflow and match is used to ensure
+	 * that the PWM line is toggled on the first event.
+	 *
+	 * Note that omap_dm_timer_enable/disable is for register access and
+	 * not the timer counter itself.
+	 */
+	omap->pdata->enable(omap->dm_timer);
+	omap->pdata->write_counter(omap->dm_timer, DM_TIMER_LOAD_MIN);
+	omap->pdata->disable(omap->dm_timer);
+
+	omap->pdata->start(omap->dm_timer);
+}
+
+static int pwm_omap_dmtimer_enable(struct pwm_chip *chip,
+				   struct pwm_device *pwm)
+{
+	struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip);
+
+	mutex_lock(&omap->mutex);
+	pwm_omap_dmtimer_start(omap);
+	mutex_unlock(&omap->mutex);
+
+	return 0;
+}
+
+static void pwm_omap_dmtimer_disable(struct pwm_chip *chip,
+				     struct pwm_device *pwm)
+{
+	struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip);
+
+	mutex_lock(&omap->mutex);
+	omap->pdata->stop(omap->dm_timer);
+	mutex_unlock(&omap->mutex);
+}
+
+static int pwm_omap_dmtimer_config(struct pwm_chip *chip,
+				   struct pwm_device *pwm,
+				   int duty_ns, int period_ns)
+{
+	struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip);
+	int load_value, match_value;
+	struct clk *fclk;
+	unsigned long clk_rate;
+	bool timer_active;
+
+	dev_dbg(chip->dev, "duty cycle: %d, period %d\n", duty_ns, period_ns);
+
+	mutex_lock(&omap->mutex);
+	if (duty_ns == pwm_get_duty_cycle(pwm) &&
+	    period_ns == pwm_get_period(pwm)) {
+		/* No change - don't cause any transients. */
+		mutex_unlock(&omap->mutex);
+		return 0;
+	}
+
+	fclk = omap->pdata->get_fclk(omap->dm_timer);
+	if (!fclk) {
+		dev_err(chip->dev, "invalid pmtimer fclk\n");
+		mutex_unlock(&omap->mutex);
+		return -EINVAL;
+	}
+
+	clk_rate = clk_get_rate(fclk);
+	if (!clk_rate) {
+		dev_err(chip->dev, "invalid pmtimer fclk rate\n");
+		mutex_unlock(&omap->mutex);
+		return -EINVAL;
+	}
+
+	dev_dbg(chip->dev, "clk rate: %luHz\n", clk_rate);
+
+	/*
+	 * Calculate the appropriate load and match values based on the
+	 * specified period and duty cycle. The load value determines the
+	 * cycle time and the match value determines the duty cycle.
+	 */
+	load_value = pwm_omap_dmtimer_calc_value(clk_rate, period_ns);
+	match_value = pwm_omap_dmtimer_calc_value(clk_rate,
+						  period_ns - duty_ns);
+
+	/*
+	 * We MUST stop the associated dual-mode timer before attempting to
+	 * write its registers, but calls to omap_dm_timer_start/stop must
+	 * be balanced so check if timer is active before calling timer_stop.
+	 */
+	timer_active = pm_runtime_active(&omap->dm_timer_pdev->dev);
+	if (timer_active)
+		omap->pdata->stop(omap->dm_timer);
+
+	omap->pdata->set_load(omap->dm_timer, true, load_value);
+	omap->pdata->set_match(omap->dm_timer, true, match_value);
+
+	dev_dbg(chip->dev, "load value: %#08x (%d), match value: %#08x (%d)\n",
+		load_value, load_value,	match_value, match_value);
+
+	omap->pdata->set_pwm(omap->dm_timer,
+			      pwm->polarity == PWM_POLARITY_INVERSED,
+			      true,
+			      PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE);
+
+	/* If config was called while timer was running it must be reenabled. */
+	if (timer_active)
+		pwm_omap_dmtimer_start(omap);
+
+	mutex_unlock(&omap->mutex);
+
+	return 0;
+}
+
+static int pwm_omap_dmtimer_set_polarity(struct pwm_chip *chip,
+					 struct pwm_device *pwm,
+					 enum pwm_polarity polarity)
+{
+	struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip);
+
+	/*
+	 * PWM core will not call set_polarity while PWM is enabled so it's
+	 * safe to reconfigure the timer here without stopping it first.
+	 */
+	mutex_lock(&omap->mutex);
+	omap->pdata->set_pwm(omap->dm_timer,
+			      polarity == PWM_POLARITY_INVERSED,
+			      true,
+			      PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE);
+	mutex_unlock(&omap->mutex);
+
+	return 0;
+}
+
+static const struct pwm_ops pwm_omap_dmtimer_ops = {
+	.enable	= pwm_omap_dmtimer_enable,
+	.disable = pwm_omap_dmtimer_disable,
+	.config	= pwm_omap_dmtimer_config,
+	.set_polarity = pwm_omap_dmtimer_set_polarity,
+	.owner = THIS_MODULE,
+};
+
+static int pwm_omap_dmtimer_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *timer;
+	struct pwm_omap_dmtimer_chip *omap;
+	struct pwm_omap_dmtimer_pdata *pdata;
+	pwm_omap_dmtimer *dm_timer;
+	u32 prescaler;
+	int status;
+
+	pdata = dev_get_platdata(&pdev->dev);
+	if (!pdata) {
+		dev_err(&pdev->dev, "Missing dmtimer platform data\n");
+		return -EINVAL;
+	}
+
+	if (!pdata->request_by_node ||
+	    !pdata->free ||
+	    !pdata->enable ||
+	    !pdata->disable ||
+	    !pdata->get_fclk ||
+	    !pdata->start ||
+	    !pdata->stop ||
+	    !pdata->set_load ||
+	    !pdata->set_match ||
+	    !pdata->set_pwm ||
+	    !pdata->set_prescaler ||
+	    !pdata->write_counter) {
+		dev_err(&pdev->dev, "Incomplete dmtimer pdata structure\n");
+		return -EINVAL;
+	}
+
+	timer = of_parse_phandle(np, "ti,timers", 0);
+	if (!timer)
+		return -ENODEV;
+
+	if (!of_get_property(timer, "ti,timer-pwm", NULL)) {
+		dev_err(&pdev->dev, "Missing ti,timer-pwm capability\n");
+		return -ENODEV;
+	}
+
+	dm_timer = pdata->request_by_node(timer);
+	if (!dm_timer)
+		return -EPROBE_DEFER;
+
+	omap = devm_kzalloc(&pdev->dev, sizeof(*omap), GFP_KERNEL);
+	if (!omap) {
+		pdata->free(dm_timer);
+		return -ENOMEM;
+	}
+
+	omap->pdata = pdata;
+	omap->dm_timer = dm_timer;
+
+	omap->dm_timer_pdev = of_find_device_by_node(timer);
+	if (!omap->dm_timer_pdev) {
+		dev_err(&pdev->dev, "Unable to find timer pdev\n");
+		omap->pdata->free(dm_timer);
+		return -EINVAL;
+	}
+
+	/*
+	 * Ensure that the timer is stopped before we allow PWM core to call
+	 * pwm_enable.
+	 */
+	if (pm_runtime_active(&omap->dm_timer_pdev->dev))
+		omap->pdata->stop(omap->dm_timer);
+
+	/* setup dmtimer prescaler */
+	if (!of_property_read_u32(pdev->dev.of_node, "ti,prescaler",
+				&prescaler))
+		omap->pdata->set_prescaler(omap->dm_timer, prescaler);
+
+	omap->chip.dev = &pdev->dev;
+	omap->chip.ops = &pwm_omap_dmtimer_ops;
+	omap->chip.base = -1;
+	omap->chip.npwm = 1;
+	omap->chip.of_xlate = of_pwm_xlate_with_flags;
+	omap->chip.of_pwm_n_cells = 3;
+
+	mutex_init(&omap->mutex);
+
+	status = pwmchip_add(&omap->chip);
+	if (status < 0) {
+		dev_err(&pdev->dev, "failed to register PWM\n");
+		omap->pdata->free(omap->dm_timer);
+		return status;
+	}
+
+	platform_set_drvdata(pdev, omap);
+
+	return 0;
+}
+
+static int pwm_omap_dmtimer_remove(struct platform_device *pdev)
+{
+	struct pwm_omap_dmtimer_chip *omap = platform_get_drvdata(pdev);
+
+	if (pm_runtime_active(&omap->dm_timer_pdev->dev))
+		omap->pdata->stop(omap->dm_timer);
+
+	omap->pdata->free(omap->dm_timer);
+
+	mutex_destroy(&omap->mutex);
+
+	return pwmchip_remove(&omap->chip);
+}
+
+static const struct of_device_id pwm_omap_dmtimer_of_match[] = {
+	{.compatible = "ti,omap-dmtimer-pwm"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, pwm_omap_dmtimer_of_match);
+
+static struct platform_driver pwm_omap_dmtimer_driver = {
+	.driver = {
+		.name = "omap-dmtimer-pwm",
+		.of_match_table = of_match_ptr(pwm_omap_dmtimer_of_match),
+	},
+	.probe = pwm_omap_dmtimer_probe,
+	.remove	= pwm_omap_dmtimer_remove,
+};
+module_platform_driver(pwm_omap_dmtimer_driver);
+
+MODULE_AUTHOR("Grant Erickson <marathon96@gmail.com>");
+MODULE_AUTHOR("NeilBrown <neilb@suse.de>");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("OMAP PWM Driver using Dual-mode Timers");
diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c
index 6e99a63ffa29..7b8ac0678137 100644
--- a/drivers/pwm/pwm-rcar.c
+++ b/drivers/pwm/pwm-rcar.c
@@ -81,7 +81,7 @@ static int rcar_pwm_get_clock_division(struct rcar_pwm_chip *rp, int period_ns)
 		max = (unsigned long long)NSEC_PER_SEC * RCAR_PWM_MAX_CYCLE *
 			(1 << div);
 		do_div(max, clk_rate);
-		if (period_ns < max)
+		if (period_ns <= max)
 			break;
 	}
 
diff --git a/drivers/rapidio/rio-sysfs.c b/drivers/rapidio/rio-sysfs.c
index cdb005c0094d..eda41563d06d 100644
--- a/drivers/rapidio/rio-sysfs.c
+++ b/drivers/rapidio/rio-sysfs.c
@@ -125,8 +125,7 @@ rio_read_config(struct file *filp, struct kobject *kobj,
 		struct bin_attribute *bin_attr,
 		char *buf, loff_t off, size_t count)
 {
-	struct rio_dev *dev =
-	    to_rio_dev(container_of(kobj, struct device, kobj));
+	struct rio_dev *dev = to_rio_dev(kobj_to_dev(kobj));
 	unsigned int size = 0x100;
 	loff_t init_off = off;
 	u8 *data = (u8 *) buf;
@@ -197,8 +196,7 @@ rio_write_config(struct file *filp, struct kobject *kobj,
 		 struct bin_attribute *bin_attr,
 		 char *buf, loff_t off, size_t count)
 {
-	struct rio_dev *dev =
-	    to_rio_dev(container_of(kobj, struct device, kobj));
+	struct rio_dev *dev = to_rio_dev(kobj_to_dev(kobj));
 	unsigned int size = count;
 	loff_t init_off = off;
 	u8 *data = (u8 *) buf;
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 2805b014ae31..8155e80dd3f8 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -274,6 +274,15 @@ config REGULATOR_ISL6271A
 	help
 	  This driver supports ISL6271A voltage regulator chip.
 
+config REGULATOR_LM363X
+	tristate "TI LM363X voltage regulators"
+	depends on MFD_TI_LMU
+	help
+	  This driver supports LM3631 and LM3632 voltage regulators for
+	  the LCD bias.
+	  One boost output voltage is configurable and always on.
+	  Other LDOs are used for the display module.
+
 config REGULATOR_LP3971
 	tristate "National Semiconductors LP3971 PMIC regulator driver"
 	depends on I2C
@@ -446,6 +455,7 @@ config REGULATOR_MC13892
 config REGULATOR_MT6311
 	tristate "MediaTek MT6311 PMIC"
 	depends on I2C
+	select REGMAP_I2C
 	help
 	  Say y here to select this option to enable the power regulator of
 	  MediaTek MT6311 PMIC.
@@ -504,6 +514,22 @@ config REGULATOR_PFUZE100
 	  Say y here to support the regulators found on the Freescale
 	  PFUZE100/PFUZE200 PMIC.
 
+config REGULATOR_PV88060
+	tristate "Powerventure Semiconductor PV88060 regulator"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say y here to support the voltage regulators and convertors
+	  PV88060
+
+config REGULATOR_PV88090
+	tristate "Powerventure Semiconductor PV88090 regulator"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say y here to support the voltage regulators and convertors
+	  on PV88090
+
 config REGULATOR_PWM
 	tristate "PWM voltage regulator"
 	depends on PWM
@@ -680,6 +706,13 @@ config REGULATOR_TPS6507X
 	  three step-down converters and two general-purpose LDO voltage regulators.
 	  It supports TI's software based Class-2 SmartReflex implementation.
 
+config REGULATOR_TPS65086
+	tristate "TI TPS65086 Power regulators"
+	depends on MFD_TPS65086
+	help
+	  This driver provides support for the voltage regulators on
+	  TI TPS65086 PMICs.
+
 config REGULATOR_TPS65090
 	tristate "TI TPS65090 Power regulator"
 	depends on MFD_TPS65090
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 0f8174913c17..980b1943fa81 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -36,6 +36,7 @@ obj-$(CONFIG_REGULATOR_GPIO) += gpio-regulator.o
 obj-$(CONFIG_REGULATOR_HI6421) += hi6421-regulator.o
 obj-$(CONFIG_REGULATOR_ISL6271A) += isl6271a-regulator.o
 obj-$(CONFIG_REGULATOR_ISL9305) += isl9305.o
+obj-$(CONFIG_REGULATOR_LM363X) += lm363x-regulator.o
 obj-$(CONFIG_REGULATOR_LP3971) += lp3971.o
 obj-$(CONFIG_REGULATOR_LP3972) += lp3972.o
 obj-$(CONFIG_REGULATOR_LP872X) += lp872x.o
@@ -66,6 +67,8 @@ obj-$(CONFIG_REGULATOR_QCOM_SMD_RPM) += qcom_smd-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_SPMI) += qcom_spmi-regulator.o
 obj-$(CONFIG_REGULATOR_PALMAS) += palmas-regulator.o
 obj-$(CONFIG_REGULATOR_PFUZE100) += pfuze100-regulator.o
+obj-$(CONFIG_REGULATOR_PV88060) += pv88060-regulator.o
+obj-$(CONFIG_REGULATOR_PV88090) += pv88090-regulator.o
 obj-$(CONFIG_REGULATOR_PWM) += pwm-regulator.o
 obj-$(CONFIG_REGULATOR_TPS51632) += tps51632-regulator.o
 obj-$(CONFIG_REGULATOR_PBIAS) += pbias-regulator.o
@@ -85,6 +88,7 @@ obj-$(CONFIG_REGULATOR_TPS6105X) += tps6105x-regulator.o
 obj-$(CONFIG_REGULATOR_TPS62360) += tps62360-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65023) += tps65023-regulator.o
 obj-$(CONFIG_REGULATOR_TPS6507X) += tps6507x-regulator.o
+obj-$(CONFIG_REGULATOR_TPS65086) += tps65086-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65090) += tps65090-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65217) += tps65217-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65218) += tps65218-regulator.o
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index 35de22fdb7a0..f2e1a39ce0f3 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -27,8 +27,8 @@
 #define AXP20X_IO_ENABLED		0x03
 #define AXP20X_IO_DISABLED		0x07
 
-#define AXP22X_IO_ENABLED		0x04
-#define AXP22X_IO_DISABLED		0x03
+#define AXP22X_IO_ENABLED		0x03
+#define AXP22X_IO_DISABLED		0x04
 
 #define AXP20X_WORKMODE_DCDC2_MASK	BIT(2)
 #define AXP20X_WORKMODE_DCDC3_MASK	BIT(1)
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 73b7683355cd..744c9889f88d 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -132,24 +132,24 @@ static bool have_full_constraints(void)
 	return has_full_constraints || of_have_populated_dt();
 }
 
+static inline struct regulator_dev *rdev_get_supply(struct regulator_dev *rdev)
+{
+	if (rdev && rdev->supply)
+		return rdev->supply->rdev;
+
+	return NULL;
+}
+
 /**
  * regulator_lock_supply - lock a regulator and its supplies
  * @rdev:         regulator source
  */
 static void regulator_lock_supply(struct regulator_dev *rdev)
 {
-	struct regulator *supply;
-	int i = 0;
-
-	while (1) {
-		mutex_lock_nested(&rdev->mutex, i++);
-		supply = rdev->supply;
-
-		if (!rdev->supply)
-			return;
+	int i;
 
-		rdev = supply->rdev;
-	}
+	for (i = 0; rdev; rdev = rdev_get_supply(rdev), i++)
+		mutex_lock_nested(&rdev->mutex, i);
 }
 
 /**
@@ -2368,7 +2368,6 @@ static void regulator_disable_work(struct work_struct *work)
 int regulator_disable_deferred(struct regulator *regulator, int ms)
 {
 	struct regulator_dev *rdev = regulator->rdev;
-	int ret;
 
 	if (regulator->always_on)
 		return 0;
@@ -2380,13 +2379,9 @@ int regulator_disable_deferred(struct regulator *regulator, int ms)
 	rdev->deferred_disables++;
 	mutex_unlock(&rdev->mutex);
 
-	ret = queue_delayed_work(system_power_efficient_wq,
-				 &rdev->disable_work,
-				 msecs_to_jiffies(ms));
-	if (ret < 0)
-		return ret;
-	else
-		return 0;
+	queue_delayed_work(system_power_efficient_wq, &rdev->disable_work,
+			   msecs_to_jiffies(ms));
+	return 0;
 }
 EXPORT_SYMBOL_GPL(regulator_disable_deferred);
 
@@ -3451,8 +3446,10 @@ int regulator_bulk_get(struct device *dev, int num_consumers,
 		consumers[i].consumer = NULL;
 
 	for (i = 0; i < num_consumers; i++) {
-		consumers[i].consumer = regulator_get(dev,
-						      consumers[i].supply);
+		consumers[i].consumer = _regulator_get(dev,
+						       consumers[i].supply,
+						       false,
+						       !consumers[i].optional);
 		if (IS_ERR(consumers[i].consumer)) {
 			ret = PTR_ERR(consumers[i].consumer);
 			dev_err(dev, "Failed to get supply '%s': %d\n",
@@ -3708,7 +3705,7 @@ static umode_t regulator_attr_is_visible(struct kobject *kobj,
 					 struct attribute *attr, int idx)
 {
 	struct device *dev = kobj_to_dev(kobj);
-	struct regulator_dev *rdev = container_of(dev, struct regulator_dev, dev);
+	struct regulator_dev *rdev = dev_to_rdev(dev);
 	const struct regulator_ops *ops = rdev->desc->ops;
 	umode_t mode = attr->mode;
 
diff --git a/drivers/regulator/da903x.c b/drivers/regulator/da903x.c
index affa1b191314..33e8f3b8d2bd 100644
--- a/drivers/regulator/da903x.c
+++ b/drivers/regulator/da903x.c
@@ -257,7 +257,7 @@ static const struct regulator_linear_range da9034_ldo12_ranges[] = {
 	REGULATOR_LINEAR_RANGE(2700000, 8, 15, 50000),
 };
 
-static struct regulator_ops da903x_regulator_ldo_ops = {
+static const struct regulator_ops da903x_regulator_ldo_ops = {
 	.set_voltage_sel = da903x_set_voltage_sel,
 	.get_voltage_sel = da903x_get_voltage_sel,
 	.list_voltage	= regulator_list_voltage_linear,
@@ -268,7 +268,7 @@ static struct regulator_ops da903x_regulator_ldo_ops = {
 };
 
 /* NOTE: this is dedicated for the insane DA9030 LDO14 */
-static struct regulator_ops da9030_regulator_ldo14_ops = {
+static const struct regulator_ops da9030_regulator_ldo14_ops = {
 	.set_voltage_sel = da903x_set_voltage_sel,
 	.get_voltage_sel = da903x_get_voltage_sel,
 	.list_voltage	= da9030_list_ldo14_voltage,
@@ -279,7 +279,7 @@ static struct regulator_ops da9030_regulator_ldo14_ops = {
 };
 
 /* NOTE: this is dedicated for the DA9030 LDO1 and LDO15 that have locks  */
-static struct regulator_ops da9030_regulator_ldo1_15_ops = {
+static const struct regulator_ops da9030_regulator_ldo1_15_ops = {
 	.set_voltage_sel = da9030_set_ldo1_15_voltage_sel,
 	.get_voltage_sel = da903x_get_voltage_sel,
 	.list_voltage	= regulator_list_voltage_linear,
@@ -289,7 +289,7 @@ static struct regulator_ops da9030_regulator_ldo1_15_ops = {
 	.is_enabled	= da903x_is_enabled,
 };
 
-static struct regulator_ops da9034_regulator_dvc_ops = {
+static const struct regulator_ops da9034_regulator_dvc_ops = {
 	.set_voltage_sel = da9034_set_dvc_voltage_sel,
 	.get_voltage_sel = da903x_get_voltage_sel,
 	.list_voltage	= regulator_list_voltage_linear,
@@ -300,7 +300,7 @@ static struct regulator_ops da9034_regulator_dvc_ops = {
 };
 
 /* NOTE: this is dedicated for the insane LDO12 */
-static struct regulator_ops da9034_regulator_ldo12_ops = {
+static const struct regulator_ops da9034_regulator_ldo12_ops = {
 	.set_voltage_sel = da903x_set_voltage_sel,
 	.get_voltage_sel = da903x_get_voltage_sel,
 	.list_voltage	= regulator_list_voltage_linear_range,
diff --git a/drivers/regulator/da9052-regulator.c b/drivers/regulator/da9052-regulator.c
index 12a25b40e473..1050cb77561a 100644
--- a/drivers/regulator/da9052-regulator.c
+++ b/drivers/regulator/da9052-regulator.c
@@ -265,7 +265,7 @@ static int da9052_regulator_set_voltage_time_sel(struct regulator_dev *rdev,
 	return ret;
 }
 
-static struct regulator_ops da9052_dcdc_ops = {
+static const struct regulator_ops da9052_dcdc_ops = {
 	.get_current_limit = da9052_dcdc_get_current_limit,
 	.set_current_limit = da9052_dcdc_set_current_limit,
 
@@ -279,7 +279,7 @@ static struct regulator_ops da9052_dcdc_ops = {
 	.disable = regulator_disable_regmap,
 };
 
-static struct regulator_ops da9052_ldo_ops = {
+static const struct regulator_ops da9052_ldo_ops = {
 	.list_voltage = da9052_list_voltage,
 	.map_voltage = da9052_map_voltage,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
diff --git a/drivers/regulator/da9055-regulator.c b/drivers/regulator/da9055-regulator.c
index cafdafbffcaf..d029c941a1e1 100644
--- a/drivers/regulator/da9055-regulator.c
+++ b/drivers/regulator/da9055-regulator.c
@@ -324,7 +324,7 @@ static int da9055_suspend_disable(struct regulator_dev *rdev)
 		return 0;
 }
 
-static struct regulator_ops da9055_buck_ops = {
+static const struct regulator_ops da9055_buck_ops = {
 	.get_mode = da9055_buck_get_mode,
 	.set_mode = da9055_buck_set_mode,
 
@@ -345,7 +345,7 @@ static struct regulator_ops da9055_buck_ops = {
 	.set_suspend_mode = da9055_buck_set_mode,
 };
 
-static struct regulator_ops da9055_ldo_ops = {
+static const struct regulator_ops da9055_ldo_ops = {
 	.get_mode = da9055_ldo_get_mode,
 	.set_mode = da9055_ldo_set_mode,
 
diff --git a/drivers/regulator/da9062-regulator.c b/drivers/regulator/da9062-regulator.c
index 5638fe8d759d..0638c8b40521 100644
--- a/drivers/regulator/da9062-regulator.c
+++ b/drivers/regulator/da9062-regulator.c
@@ -371,7 +371,7 @@ static int da9062_ldo_set_suspend_mode(struct regulator_dev *rdev,
 	return regmap_field_write(regl->suspend_sleep, val);
 }
 
-static struct regulator_ops da9062_buck_ops = {
+static const struct regulator_ops da9062_buck_ops = {
 	.enable			= regulator_enable_regmap,
 	.disable		= regulator_disable_regmap,
 	.is_enabled		= regulator_is_enabled_regmap,
@@ -389,7 +389,7 @@ static struct regulator_ops da9062_buck_ops = {
 	.set_suspend_mode	= da9062_buck_set_suspend_mode,
 };
 
-static struct regulator_ops da9062_ldo_ops = {
+static const struct regulator_ops da9062_ldo_ops = {
 	.enable			= regulator_enable_regmap,
 	.disable		= regulator_disable_regmap,
 	.is_enabled		= regulator_is_enabled_regmap,
diff --git a/drivers/regulator/da9063-regulator.c b/drivers/regulator/da9063-regulator.c
index 536e931eb921..ed9e7e96f877 100644
--- a/drivers/regulator/da9063-regulator.c
+++ b/drivers/regulator/da9063-regulator.c
@@ -427,7 +427,7 @@ static int da9063_ldo_set_suspend_mode(struct regulator_dev *rdev, unsigned mode
 	return regmap_field_write(regl->suspend_sleep, val);
 }
 
-static struct regulator_ops da9063_buck_ops = {
+static const struct regulator_ops da9063_buck_ops = {
 	.enable			= regulator_enable_regmap,
 	.disable		= regulator_disable_regmap,
 	.is_enabled		= regulator_is_enabled_regmap,
@@ -445,7 +445,7 @@ static struct regulator_ops da9063_buck_ops = {
 	.set_suspend_mode	= da9063_buck_set_suspend_mode,
 };
 
-static struct regulator_ops da9063_ldo_ops = {
+static const struct regulator_ops da9063_ldo_ops = {
 	.enable			= regulator_enable_regmap,
 	.disable		= regulator_disable_regmap,
 	.is_enabled		= regulator_is_enabled_regmap,
diff --git a/drivers/regulator/da9210-regulator.c b/drivers/regulator/da9210-regulator.c
index b3517830edb6..8b3cc9f0cd64 100644
--- a/drivers/regulator/da9210-regulator.c
+++ b/drivers/regulator/da9210-regulator.c
@@ -46,7 +46,7 @@ static int da9210_set_current_limit(struct regulator_dev *rdev, int min_uA,
 				    int max_uA);
 static int da9210_get_current_limit(struct regulator_dev *rdev);
 
-static struct regulator_ops da9210_buck_ops = {
+static const struct regulator_ops da9210_buck_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
 	.is_enabled = regulator_is_enabled_regmap,
diff --git a/drivers/regulator/da9211-regulator.c b/drivers/regulator/da9211-regulator.c
index 04ef65b7eb3d..236abf473db5 100644
--- a/drivers/regulator/da9211-regulator.c
+++ b/drivers/regulator/da9211-regulator.c
@@ -219,7 +219,7 @@ static int da9211_get_current_limit(struct regulator_dev *rdev)
 	return current_limits[data];
 }
 
-static struct regulator_ops da9211_buck_ops = {
+static const struct regulator_ops da9211_buck_ops = {
 	.get_mode = da9211_buck_get_mode,
 	.set_mode = da9211_buck_set_mode,
 	.enable = regulator_enable_regmap,
diff --git a/drivers/regulator/devres.c b/drivers/regulator/devres.c
index 6ec1d400adae..6ad8ab4c578d 100644
--- a/drivers/regulator/devres.c
+++ b/drivers/regulator/devres.c
@@ -164,8 +164,11 @@ int devm_regulator_bulk_get(struct device *dev, int num_consumers,
 		consumers[i].consumer = NULL;
 
 	for (i = 0; i < num_consumers; i++) {
-		consumers[i].consumer = devm_regulator_get(dev,
-							   consumers[i].supply);
+		consumers[i].consumer = _devm_regulator_get(dev,
+							    consumers[i].supply,
+							    consumers[i].optional ?
+								OPTIONAL_GET :
+								NORMAL_GET);
 		if (IS_ERR(consumers[i].consumer)) {
 			ret = PTR_ERR(consumers[i].consumer);
 			dev_err(dev, "Failed to get supply '%s': %d\n",
diff --git a/drivers/regulator/lm363x-regulator.c b/drivers/regulator/lm363x-regulator.c
new file mode 100644
index 000000000000..f53e63301a20
--- /dev/null
+++ b/drivers/regulator/lm363x-regulator.c
@@ -0,0 +1,291 @@
+/*
+ * TI LM363X Regulator Driver
+ *
+ * Copyright 2015 Texas Instruments
+ *
+ * Author: Milo Kim <milo.kim@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/mfd/ti-lmu.h>
+#include <linux/mfd/ti-lmu-register.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/slab.h>
+
+/* LM3631 */
+#define LM3631_BOOST_VSEL_MAX		0x25
+#define LM3631_LDO_VSEL_MAX		0x28
+#define LM3631_CONT_VSEL_MAX		0x03
+#define LM3631_VBOOST_MIN		4500000
+#define LM3631_VCONT_MIN		1800000
+#define LM3631_VLDO_MIN			4000000
+#define ENABLE_TIME_USEC		1000
+
+/* LM3632 */
+#define LM3632_BOOST_VSEL_MAX		0x26
+#define LM3632_LDO_VSEL_MAX		0x29
+#define LM3632_VBOOST_MIN		4500000
+#define LM3632_VLDO_MIN			4000000
+
+/* Common */
+#define LM363X_STEP_50mV		50000
+#define LM363X_STEP_500mV		500000
+
+static const int ldo_cont_enable_time[] = {
+	0, 2000, 5000, 10000, 20000, 50000, 100000, 200000,
+};
+
+static int lm363x_regulator_enable_time(struct regulator_dev *rdev)
+{
+	enum lm363x_regulator_id id = rdev_get_id(rdev);
+	u8 val, addr, mask;
+
+	switch (id) {
+	case LM3631_LDO_CONT:
+		addr = LM3631_REG_ENTIME_VCONT;
+		mask = LM3631_ENTIME_CONT_MASK;
+		break;
+	case LM3631_LDO_OREF:
+		addr = LM3631_REG_ENTIME_VOREF;
+		mask = LM3631_ENTIME_MASK;
+		break;
+	case LM3631_LDO_POS:
+		addr = LM3631_REG_ENTIME_VPOS;
+		mask = LM3631_ENTIME_MASK;
+		break;
+	case LM3631_LDO_NEG:
+		addr = LM3631_REG_ENTIME_VNEG;
+		mask = LM3631_ENTIME_MASK;
+		break;
+	default:
+		return 0;
+	}
+
+	if (regmap_read(rdev->regmap, addr, (unsigned int *)&val))
+		return -EINVAL;
+
+	val = (val & mask) >> LM3631_ENTIME_SHIFT;
+
+	if (id == LM3631_LDO_CONT)
+		return ldo_cont_enable_time[val];
+	else
+		return ENABLE_TIME_USEC * val;
+}
+
+static struct regulator_ops lm363x_boost_voltage_table_ops = {
+	.list_voltage     = regulator_list_voltage_linear,
+	.set_voltage_sel  = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel  = regulator_get_voltage_sel_regmap,
+};
+
+static struct regulator_ops lm363x_regulator_voltage_table_ops = {
+	.list_voltage     = regulator_list_voltage_linear,
+	.set_voltage_sel  = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel  = regulator_get_voltage_sel_regmap,
+	.enable           = regulator_enable_regmap,
+	.disable          = regulator_disable_regmap,
+	.is_enabled       = regulator_is_enabled_regmap,
+	.enable_time      = lm363x_regulator_enable_time,
+};
+
+static const struct regulator_desc lm363x_regulator_desc[] = {
+	/* LM3631 */
+	{
+		.name           = "vboost",
+		.of_match	= "vboost",
+		.id             = LM3631_BOOST,
+		.ops            = &lm363x_boost_voltage_table_ops,
+		.n_voltages     = LM3631_BOOST_VSEL_MAX + 1,
+		.min_uV         = LM3631_VBOOST_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3631_REG_VOUT_BOOST,
+		.vsel_mask      = LM3631_VOUT_MASK,
+	},
+	{
+		.name           = "ldo_cont",
+		.of_match	= "vcont",
+		.id             = LM3631_LDO_CONT,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3631_CONT_VSEL_MAX + 1,
+		.min_uV         = LM3631_VCONT_MIN,
+		.uV_step        = LM363X_STEP_500mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3631_REG_VOUT_CONT,
+		.vsel_mask      = LM3631_VOUT_CONT_MASK,
+		.enable_reg     = LM3631_REG_LDO_CTRL2,
+		.enable_mask    = LM3631_EN_CONT_MASK,
+	},
+	{
+		.name           = "ldo_oref",
+		.of_match	= "voref",
+		.id             = LM3631_LDO_OREF,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3631_LDO_VSEL_MAX + 1,
+		.min_uV         = LM3631_VLDO_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3631_REG_VOUT_OREF,
+		.vsel_mask      = LM3631_VOUT_MASK,
+		.enable_reg     = LM3631_REG_LDO_CTRL1,
+		.enable_mask    = LM3631_EN_OREF_MASK,
+	},
+	{
+		.name           = "ldo_vpos",
+		.of_match	= "vpos",
+		.id             = LM3631_LDO_POS,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3631_LDO_VSEL_MAX + 1,
+		.min_uV         = LM3631_VLDO_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3631_REG_VOUT_POS,
+		.vsel_mask      = LM3631_VOUT_MASK,
+		.enable_reg     = LM3631_REG_LDO_CTRL1,
+		.enable_mask    = LM3631_EN_VPOS_MASK,
+	},
+	{
+		.name           = "ldo_vneg",
+		.of_match	= "vneg",
+		.id             = LM3631_LDO_NEG,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3631_LDO_VSEL_MAX + 1,
+		.min_uV         = LM3631_VLDO_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3631_REG_VOUT_NEG,
+		.vsel_mask      = LM3631_VOUT_MASK,
+		.enable_reg     = LM3631_REG_LDO_CTRL1,
+		.enable_mask    = LM3631_EN_VNEG_MASK,
+	},
+	/* LM3632 */
+	{
+		.name           = "vboost",
+		.of_match	= "vboost",
+		.id             = LM3632_BOOST,
+		.ops            = &lm363x_boost_voltage_table_ops,
+		.n_voltages     = LM3632_BOOST_VSEL_MAX + 1,
+		.min_uV         = LM3632_VBOOST_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3632_REG_VOUT_BOOST,
+		.vsel_mask      = LM3632_VOUT_MASK,
+	},
+	{
+		.name           = "ldo_vpos",
+		.of_match	= "vpos",
+		.id             = LM3632_LDO_POS,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3632_LDO_VSEL_MAX + 1,
+		.min_uV         = LM3632_VLDO_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3632_REG_VOUT_POS,
+		.vsel_mask      = LM3632_VOUT_MASK,
+		.enable_reg     = LM3632_REG_BIAS_CONFIG,
+		.enable_mask    = LM3632_EN_VPOS_MASK,
+	},
+	{
+		.name           = "ldo_vneg",
+		.of_match	= "vneg",
+		.id             = LM3632_LDO_NEG,
+		.ops            = &lm363x_regulator_voltage_table_ops,
+		.n_voltages     = LM3632_LDO_VSEL_MAX + 1,
+		.min_uV         = LM3632_VLDO_MIN,
+		.uV_step        = LM363X_STEP_50mV,
+		.type           = REGULATOR_VOLTAGE,
+		.owner          = THIS_MODULE,
+		.vsel_reg       = LM3632_REG_VOUT_NEG,
+		.vsel_mask      = LM3632_VOUT_MASK,
+		.enable_reg     = LM3632_REG_BIAS_CONFIG,
+		.enable_mask    = LM3632_EN_VNEG_MASK,
+	},
+};
+
+static int lm363x_regulator_of_get_enable_gpio(struct device_node *np, int id)
+{
+	/*
+	 * Check LCM_EN1/2_GPIO is configured.
+	 * Those pins are used for enabling VPOS/VNEG LDOs.
+	 */
+	switch (id) {
+	case LM3632_LDO_POS:
+		return of_get_named_gpio(np, "ti,lcm-en1-gpio", 0);
+	case LM3632_LDO_NEG:
+		return of_get_named_gpio(np, "ti,lcm-en2-gpio", 0);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int lm363x_regulator_probe(struct platform_device *pdev)
+{
+	struct ti_lmu *lmu = dev_get_drvdata(pdev->dev.parent);
+	struct regmap *regmap = lmu->regmap;
+	struct regulator_config cfg = { };
+	struct regulator_dev *rdev;
+	struct device *dev = &pdev->dev;
+	int id = pdev->id;
+	int ret, ena_gpio;
+
+	cfg.dev = dev;
+	cfg.regmap = regmap;
+
+	/*
+	 * LM3632 LDOs can be controlled by external pin.
+	 * Register update is required if the pin is used.
+	 */
+	ena_gpio = lm363x_regulator_of_get_enable_gpio(dev->of_node, id);
+	if (gpio_is_valid(ena_gpio)) {
+		cfg.ena_gpio = ena_gpio;
+		cfg.ena_gpio_flags = GPIOF_OUT_INIT_LOW;
+
+		ret = regmap_update_bits(regmap, LM3632_REG_BIAS_CONFIG,
+					 LM3632_EXT_EN_MASK,
+					 LM3632_EXT_EN_MASK);
+		if (ret) {
+			dev_err(dev, "External pin err: %d\n", ret);
+			return ret;
+		}
+	}
+
+	rdev = devm_regulator_register(dev, &lm363x_regulator_desc[id], &cfg);
+	if (IS_ERR(rdev)) {
+		ret = PTR_ERR(rdev);
+		dev_err(dev, "[%d] regulator register err: %d\n", id, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver lm363x_regulator_driver = {
+	.probe = lm363x_regulator_probe,
+	.driver = {
+		.name = "lm363x-regulator",
+	},
+};
+
+module_platform_driver(lm363x_regulator_driver);
+
+MODULE_DESCRIPTION("TI LM363X Regulator Driver");
+MODULE_AUTHOR("Milo Kim");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:lm363x-regulator");
diff --git a/drivers/regulator/lp872x.c b/drivers/regulator/lp872x.c
index e5af07208f9d..19d758486553 100644
--- a/drivers/regulator/lp872x.c
+++ b/drivers/regulator/lp872x.c
@@ -108,7 +108,6 @@ struct lp872x {
 	struct lp872x_platform_data *pdata;
 	int num_regulators;
 	enum lp872x_dvs_state dvs_pin;
-	int dvs_gpio;
 };
 
 /* LP8720/LP8725 shared voltage table for LDOs */
@@ -520,6 +519,7 @@ static struct regulator_ops lp8725_buck_ops = {
 static struct regulator_desc lp8720_regulator_desc[] = {
 	{
 		.name = "ldo1",
+		.of_match = of_match_ptr("ldo1"),
 		.id = LP8720_ID_LDO1,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -533,6 +533,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 	},
 	{
 		.name = "ldo2",
+		.of_match = of_match_ptr("ldo2"),
 		.id = LP8720_ID_LDO2,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -546,6 +547,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 	},
 	{
 		.name = "ldo3",
+		.of_match = of_match_ptr("ldo3"),
 		.id = LP8720_ID_LDO3,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -559,6 +561,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 	},
 	{
 		.name = "ldo4",
+		.of_match = of_match_ptr("ldo4"),
 		.id = LP8720_ID_LDO4,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp8720_ldo4_vtbl),
@@ -572,6 +575,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 	},
 	{
 		.name = "ldo5",
+		.of_match = of_match_ptr("ldo5"),
 		.id = LP8720_ID_LDO5,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -585,6 +589,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 	},
 	{
 		.name = "buck",
+		.of_match = of_match_ptr("buck"),
 		.id = LP8720_ID_BUCK,
 		.ops = &lp8720_buck_ops,
 		.n_voltages = ARRAY_SIZE(lp8720_buck_vtbl),
@@ -599,6 +604,7 @@ static struct regulator_desc lp8720_regulator_desc[] = {
 static struct regulator_desc lp8725_regulator_desc[] = {
 	{
 		.name = "ldo1",
+		.of_match = of_match_ptr("ldo1"),
 		.id = LP8725_ID_LDO1,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -612,6 +618,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "ldo2",
+		.of_match = of_match_ptr("ldo2"),
 		.id = LP8725_ID_LDO2,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -625,6 +632,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "ldo3",
+		.of_match = of_match_ptr("ldo3"),
 		.id = LP8725_ID_LDO3,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -638,6 +646,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "ldo4",
+		.of_match = of_match_ptr("ldo4"),
 		.id = LP8725_ID_LDO4,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -651,6 +660,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "ldo5",
+		.of_match = of_match_ptr("ldo5"),
 		.id = LP8725_ID_LDO5,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp872x_ldo_vtbl),
@@ -664,6 +674,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "lilo1",
+		.of_match = of_match_ptr("lilo1"),
 		.id = LP8725_ID_LILO1,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp8725_lilo_vtbl),
@@ -677,6 +688,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "lilo2",
+		.of_match = of_match_ptr("lilo2"),
 		.id = LP8725_ID_LILO2,
 		.ops = &lp872x_ldo_ops,
 		.n_voltages = ARRAY_SIZE(lp8725_lilo_vtbl),
@@ -690,6 +702,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "buck1",
+		.of_match = of_match_ptr("buck1"),
 		.id = LP8725_ID_BUCK1,
 		.ops = &lp8725_buck_ops,
 		.n_voltages = ARRAY_SIZE(lp8725_buck_vtbl),
@@ -701,6 +714,7 @@ static struct regulator_desc lp8725_regulator_desc[] = {
 	},
 	{
 		.name = "buck2",
+		.of_match = of_match_ptr("buck2"),
 		.id = LP8725_ID_BUCK2,
 		.ops = &lp8725_buck_ops,
 		.n_voltages = ARRAY_SIZE(lp8725_buck_vtbl),
@@ -737,7 +751,6 @@ static int lp872x_init_dvs(struct lp872x *lp)
 	}
 
 	lp->dvs_pin = pinstate;
-	lp->dvs_gpio = gpio;
 
 	return 0;
 
diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c
index a97bed90d39b..ec46290b647e 100644
--- a/drivers/regulator/lp8788-buck.c
+++ b/drivers/regulator/lp8788-buck.c
@@ -344,7 +344,7 @@ static unsigned int lp8788_buck_get_mode(struct regulator_dev *rdev)
 				REGULATOR_MODE_FAST : REGULATOR_MODE_NORMAL;
 }
 
-static struct regulator_ops lp8788_buck12_ops = {
+static const struct regulator_ops lp8788_buck12_ops = {
 	.list_voltage = regulator_list_voltage_table,
 	.map_voltage = regulator_map_voltage_ascend,
 	.set_voltage_sel = lp8788_buck12_set_voltage_sel,
@@ -357,7 +357,7 @@ static struct regulator_ops lp8788_buck12_ops = {
 	.get_mode = lp8788_buck_get_mode,
 };
 
-static struct regulator_ops lp8788_buck34_ops = {
+static const struct regulator_ops lp8788_buck34_ops = {
 	.list_voltage = regulator_list_voltage_table,
 	.map_voltage = regulator_map_voltage_ascend,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
diff --git a/drivers/regulator/lp8788-ldo.c b/drivers/regulator/lp8788-ldo.c
index 9f22d079c8cc..cbfd35873575 100644
--- a/drivers/regulator/lp8788-ldo.c
+++ b/drivers/regulator/lp8788-ldo.c
@@ -170,7 +170,7 @@ static int lp8788_ldo_enable_time(struct regulator_dev *rdev)
 	return ENABLE_TIME_USEC * val;
 }
 
-static struct regulator_ops lp8788_ldo_voltage_table_ops = {
+static const struct regulator_ops lp8788_ldo_voltage_table_ops = {
 	.list_voltage = regulator_list_voltage_table,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -180,7 +180,7 @@ static struct regulator_ops lp8788_ldo_voltage_table_ops = {
 	.enable_time = lp8788_ldo_enable_time,
 };
 
-static struct regulator_ops lp8788_ldo_voltage_fixed_ops = {
+static const struct regulator_ops lp8788_ldo_voltage_fixed_ops = {
 	.list_voltage = regulator_list_voltage_linear,
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
@@ -613,22 +613,20 @@ static struct platform_driver lp8788_aldo_driver = {
 	},
 };
 
+static struct platform_driver * const drivers[] = {
+	&lp8788_dldo_driver,
+	&lp8788_aldo_driver,
+};
+
 static int __init lp8788_ldo_init(void)
 {
-	int ret;
-
-	ret = platform_driver_register(&lp8788_dldo_driver);
-	if (ret)
-		return ret;
-
-	return platform_driver_register(&lp8788_aldo_driver);
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 subsys_initcall(lp8788_ldo_init);
 
 static void __exit lp8788_ldo_exit(void)
 {
-	platform_driver_unregister(&lp8788_aldo_driver);
-	platform_driver_unregister(&lp8788_dldo_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 module_exit(lp8788_ldo_exit);
 
diff --git a/drivers/regulator/mt6311-regulator.c b/drivers/regulator/mt6311-regulator.c
index 02c4e5feca8e..0495716fd35f 100644
--- a/drivers/regulator/mt6311-regulator.c
+++ b/drivers/regulator/mt6311-regulator.c
@@ -30,6 +30,7 @@ static const struct regmap_config mt6311_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 	.max_register = MT6311_FQMTR_CON4,
+	.cache_type = REGCACHE_RBTREE,
 };
 
 /* Default limits measured in millivolts and milliamps */
diff --git a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
index 8217613807d3..6efc7ee8aea3 100644
--- a/drivers/regulator/palmas-regulator.c
+++ b/drivers/regulator/palmas-regulator.c
@@ -612,6 +612,18 @@ static struct regulator_ops palmas_ops_ldo = {
 	.map_voltage		= regulator_map_voltage_linear,
 };
 
+static struct regulator_ops palmas_ops_ldo9 = {
+	.is_enabled		= palmas_is_enabled_ldo,
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear,
+	.map_voltage		= regulator_map_voltage_linear,
+	.set_bypass		= regulator_set_bypass_regmap,
+	.get_bypass		= regulator_get_bypass_regmap,
+};
+
 static struct regulator_ops palmas_ops_ext_control_ldo = {
 	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
 	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
@@ -639,6 +651,19 @@ static struct regulator_ops tps65917_ops_ldo = {
 	.set_voltage_time_sel	= regulator_set_voltage_time_sel,
 };
 
+static struct regulator_ops tps65917_ops_ldo_1_2 = {
+	.is_enabled		= palmas_is_enabled_ldo,
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear,
+	.map_voltage		= regulator_map_voltage_linear,
+	.set_voltage_time_sel	= regulator_set_voltage_time_sel,
+	.set_bypass		= regulator_set_bypass_regmap,
+	.get_bypass		= regulator_get_bypass_regmap,
+};
+
 static int palmas_regulator_config_external(struct palmas *palmas, int id,
 		struct palmas_reg_init *reg_init)
 {
@@ -915,6 +940,13 @@ static int palmas_ldo_registration(struct palmas_pmic *pmic,
 			if (pdata && pdata->ldo6_vibrator &&
 			    (id == PALMAS_REG_LDO6))
 				desc->enable_time = 2000;
+
+			if (id == PALMAS_REG_LDO9) {
+				desc->ops = &palmas_ops_ldo9;
+				desc->bypass_reg = desc->enable_reg;
+				desc->bypass_mask =
+						PALMAS_LDO9_CTRL_LDO_BYPASS_EN;
+			}
 		} else {
 			if (!ddata->has_regen3 && id == PALMAS_REG_REGEN3)
 				continue;
@@ -1019,6 +1051,13 @@ static int tps65917_ldo_registration(struct palmas_pmic *pmic,
 			 * It is of the order of ~60mV/uS.
 			 */
 			desc->ramp_delay = 2500;
+			if (id == TPS65917_REG_LDO1 ||
+			    id == TPS65917_REG_LDO2) {
+				desc->ops = &tps65917_ops_ldo_1_2;
+				desc->bypass_reg = desc->enable_reg;
+				desc->bypass_mask =
+						TPS65917_LDO1_CTRL_BYPASS_EN;
+			}
 		} else {
 			desc->n_voltages = 1;
 			if (reg_init && reg_init->roof_floor)
diff --git a/drivers/regulator/pv88060-regulator.c b/drivers/regulator/pv88060-regulator.c
new file mode 100644
index 000000000000..094376c8de4b
--- /dev/null
+++ b/drivers/regulator/pv88060-regulator.c
@@ -0,0 +1,437 @@
+/*
+ * pv88060-regulator.c - Regulator device driver for PV88060
+ * Copyright (C) 2015  Powerventure Semiconductor Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regmap.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/proc_fs.h>
+#include <linux/uaccess.h>
+#include "pv88060-regulator.h"
+
+#define PV88060_MAX_REGULATORS	14
+
+/* PV88060 REGULATOR IDs */
+enum {
+	/* BUCKs */
+	PV88060_ID_BUCK1,
+
+	/* LDOs */
+	PV88060_ID_LDO1,
+	PV88060_ID_LDO2,
+	PV88060_ID_LDO3,
+	PV88060_ID_LDO4,
+	PV88060_ID_LDO5,
+	PV88060_ID_LDO6,
+	PV88060_ID_LDO7,
+
+	/* SWTs */
+	PV88060_ID_SW1,
+	PV88060_ID_SW2,
+	PV88060_ID_SW3,
+	PV88060_ID_SW4,
+	PV88060_ID_SW5,
+	PV88060_ID_SW6,
+};
+
+struct pv88060_regulator {
+	struct regulator_desc desc;
+	/* Current limiting */
+	unsigned	n_current_limits;
+	const int	*current_limits;
+	unsigned int limit_mask;
+	unsigned int conf;		/* buck configuration register */
+};
+
+struct pv88060 {
+	struct device *dev;
+	struct regmap *regmap;
+	struct regulator_dev *rdev[PV88060_MAX_REGULATORS];
+};
+
+static const struct regmap_config pv88060_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+/* Current limits array (in uA) for BUCK1
+ * Entry indexes corresponds to register values.
+ */
+
+static const int pv88060_buck1_limits[] = {
+	1496000, 2393000, 3291000, 4189000
+};
+
+static unsigned int pv88060_buck_get_mode(struct regulator_dev *rdev)
+{
+	struct pv88060_regulator *info = rdev_get_drvdata(rdev);
+	unsigned int data;
+	int ret, mode = 0;
+
+	ret = regmap_read(rdev->regmap, info->conf, &data);
+	if (ret < 0)
+		return ret;
+
+	switch (data & PV88060_BUCK_MODE_MASK) {
+	case PV88060_BUCK_MODE_SYNC:
+		mode = REGULATOR_MODE_FAST;
+		break;
+	case PV88060_BUCK_MODE_AUTO:
+		mode = REGULATOR_MODE_NORMAL;
+		break;
+	case PV88060_BUCK_MODE_SLEEP:
+		mode = REGULATOR_MODE_STANDBY;
+		break;
+	}
+
+	return mode;
+}
+
+static int pv88060_buck_set_mode(struct regulator_dev *rdev,
+					unsigned int mode)
+{
+	struct pv88060_regulator *info = rdev_get_drvdata(rdev);
+	int val = 0;
+
+	switch (mode) {
+	case REGULATOR_MODE_FAST:
+		val = PV88060_BUCK_MODE_SYNC;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = PV88060_BUCK_MODE_AUTO;
+		break;
+	case REGULATOR_MODE_STANDBY:
+		val = PV88060_BUCK_MODE_SLEEP;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(rdev->regmap, info->conf,
+					PV88060_BUCK_MODE_MASK, val);
+}
+
+static int pv88060_set_current_limit(struct regulator_dev *rdev, int min,
+				    int max)
+{
+	struct pv88060_regulator *info = rdev_get_drvdata(rdev);
+	int i;
+
+	/* search for closest to maximum */
+	for (i = info->n_current_limits; i >= 0; i--) {
+		if (min <= info->current_limits[i]
+			&& max >= info->current_limits[i]) {
+			return regmap_update_bits(rdev->regmap,
+				info->conf,
+				info->limit_mask,
+				i << PV88060_BUCK_ILIM_SHIFT);
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int pv88060_get_current_limit(struct regulator_dev *rdev)
+{
+	struct pv88060_regulator *info = rdev_get_drvdata(rdev);
+	unsigned int data;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, info->conf, &data);
+	if (ret < 0)
+		return ret;
+
+	data = (data & info->limit_mask) >> PV88060_BUCK_ILIM_SHIFT;
+	return info->current_limits[data];
+}
+
+static struct regulator_ops pv88060_buck_ops = {
+	.get_mode = pv88060_buck_get_mode,
+	.set_mode = pv88060_buck_set_mode,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	.set_current_limit = pv88060_set_current_limit,
+	.get_current_limit = pv88060_get_current_limit,
+};
+
+static struct regulator_ops pv88060_ldo_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+};
+
+#define PV88060_BUCK(chip, regl_name, min, step, max, limits_array) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88060_buck_ops,\
+		.min_uV = min,\
+		.uV_step = step,\
+		.n_voltages = ((max) - (min))/(step) + 1,\
+		.enable_reg = PV88060_REG_##regl_name##_CONF0,\
+		.enable_mask = PV88060_BUCK_EN, \
+		.vsel_reg = PV88060_REG_##regl_name##_CONF0,\
+		.vsel_mask = PV88060_VBUCK_MASK,\
+	},\
+	.current_limits = limits_array,\
+	.n_current_limits = ARRAY_SIZE(limits_array),\
+	.limit_mask = PV88060_BUCK_ILIM_MASK, \
+	.conf = PV88060_REG_##regl_name##_CONF1,\
+}
+
+#define PV88060_LDO(chip, regl_name, min, step, max) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88060_ldo_ops,\
+		.min_uV = min, \
+		.uV_step = step, \
+		.n_voltages = (step) ? ((max - min) / step + 1) : 1, \
+		.enable_reg = PV88060_REG_##regl_name##_CONF, \
+		.enable_mask = PV88060_LDO_EN, \
+		.vsel_reg = PV88060_REG_##regl_name##_CONF, \
+		.vsel_mask = PV88060_VLDO_MASK, \
+	},\
+}
+
+#define PV88060_SW(chip, regl_name, max) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88060_ldo_ops,\
+		.min_uV = max,\
+		.uV_step = 0,\
+		.n_voltages = 1,\
+		.enable_reg = PV88060_REG_##regl_name##_CONF,\
+		.enable_mask = PV88060_SW_EN,\
+	},\
+}
+
+static const struct pv88060_regulator pv88060_regulator_info[] = {
+	PV88060_BUCK(PV88060, BUCK1, 2800000, 12500, 4387500,
+		pv88060_buck1_limits),
+	PV88060_LDO(PV88060, LDO1, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO2, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO3, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO4, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO5, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO6, 1200000, 50000, 3350000),
+	PV88060_LDO(PV88060, LDO7, 1200000, 50000, 3350000),
+	PV88060_SW(PV88060, SW1, 5000000),
+	PV88060_SW(PV88060, SW2, 5000000),
+	PV88060_SW(PV88060, SW3, 5000000),
+	PV88060_SW(PV88060, SW4, 5000000),
+	PV88060_SW(PV88060, SW5, 5000000),
+	PV88060_SW(PV88060, SW6, 5000000),
+};
+
+static irqreturn_t pv88060_irq_handler(int irq, void *data)
+{
+	struct pv88060 *chip = data;
+	int i, reg_val, err, ret = IRQ_NONE;
+
+	err = regmap_read(chip->regmap, PV88060_REG_EVENT_A, &reg_val);
+	if (err < 0)
+		goto error_i2c;
+
+	if (reg_val & PV88060_E_VDD_FLT) {
+		for (i = 0; i < PV88060_MAX_REGULATORS; i++) {
+			if (chip->rdev[i] != NULL) {
+				regulator_notifier_call_chain(chip->rdev[i],
+					REGULATOR_EVENT_UNDER_VOLTAGE,
+					NULL);
+			}
+		}
+
+		err = regmap_update_bits(chip->regmap, PV88060_REG_EVENT_A,
+			PV88060_E_VDD_FLT, PV88060_E_VDD_FLT);
+		if (err < 0)
+			goto error_i2c;
+
+		ret = IRQ_HANDLED;
+	}
+
+	if (reg_val & PV88060_E_OVER_TEMP) {
+		for (i = 0; i < PV88060_MAX_REGULATORS; i++) {
+			if (chip->rdev[i] != NULL) {
+				regulator_notifier_call_chain(chip->rdev[i],
+					REGULATOR_EVENT_OVER_TEMP,
+					NULL);
+			}
+		}
+
+		err = regmap_update_bits(chip->regmap, PV88060_REG_EVENT_A,
+			PV88060_E_OVER_TEMP, PV88060_E_OVER_TEMP);
+		if (err < 0)
+			goto error_i2c;
+
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+
+error_i2c:
+	dev_err(chip->dev, "I2C error : %d\n", err);
+	return IRQ_NONE;
+}
+
+/*
+ * I2C driver interface functions
+ */
+static int pv88060_i2c_probe(struct i2c_client *i2c,
+		const struct i2c_device_id *id)
+{
+	struct regulator_init_data *init_data = dev_get_platdata(&i2c->dev);
+	struct pv88060 *chip;
+	struct regulator_config config = { };
+	int error, i, ret = 0;
+
+	chip = devm_kzalloc(&i2c->dev, sizeof(struct pv88060), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	chip->dev = &i2c->dev;
+	chip->regmap = devm_regmap_init_i2c(i2c, &pv88060_regmap_config);
+	if (IS_ERR(chip->regmap)) {
+		error = PTR_ERR(chip->regmap);
+		dev_err(chip->dev, "Failed to allocate register map: %d\n",
+			error);
+		return error;
+	}
+
+	i2c_set_clientdata(i2c, chip);
+
+	if (i2c->irq != 0) {
+		ret = regmap_write(chip->regmap, PV88060_REG_MASK_A, 0xFF);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to mask A reg: %d\n", ret);
+			return ret;
+		}
+
+		ret = regmap_write(chip->regmap, PV88060_REG_MASK_B, 0xFF);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to mask B reg: %d\n", ret);
+			return ret;
+		}
+
+		ret = regmap_write(chip->regmap, PV88060_REG_MASK_C, 0xFF);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to mask C reg: %d\n", ret);
+			return ret;
+		}
+
+		ret = devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL,
+					pv88060_irq_handler,
+					IRQF_TRIGGER_LOW|IRQF_ONESHOT,
+					"pv88060", chip);
+		if (ret != 0) {
+			dev_err(chip->dev, "Failed to request IRQ: %d\n",
+				i2c->irq);
+			return ret;
+		}
+
+		ret = regmap_update_bits(chip->regmap, PV88060_REG_MASK_A,
+			PV88060_M_VDD_FLT | PV88060_M_OVER_TEMP, 0);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to update mask reg: %d\n", ret);
+			return ret;
+		}
+
+	} else {
+		dev_warn(chip->dev, "No IRQ configured\n");
+	}
+
+	config.dev = chip->dev;
+	config.regmap = chip->regmap;
+
+	for (i = 0; i < PV88060_MAX_REGULATORS; i++) {
+		if (init_data)
+			config.init_data = &init_data[i];
+
+		config.driver_data = (void *)&pv88060_regulator_info[i];
+		chip->rdev[i] = devm_regulator_register(chip->dev,
+			&pv88060_regulator_info[i].desc, &config);
+		if (IS_ERR(chip->rdev[i])) {
+			dev_err(chip->dev,
+				"Failed to register PV88060 regulator\n");
+			return PTR_ERR(chip->rdev[i]);
+		}
+	}
+
+	return 0;
+}
+
+static const struct i2c_device_id pv88060_i2c_id[] = {
+	{"pv88060", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, pv88060_i2c_id);
+
+#ifdef CONFIG_OF
+static const struct of_device_id pv88060_dt_ids[] = {
+	{ .compatible = "pvs,pv88060", .data = &pv88060_i2c_id[0] },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pv88060_dt_ids);
+#endif
+
+static struct i2c_driver pv88060_regulator_driver = {
+	.driver = {
+		.name = "pv88060",
+		.of_match_table = of_match_ptr(pv88060_dt_ids),
+	},
+	.probe = pv88060_i2c_probe,
+	.id_table = pv88060_i2c_id,
+};
+
+module_i2c_driver(pv88060_regulator_driver);
+
+MODULE_AUTHOR("James Ban <James.Ban.opensource@diasemi.com>");
+MODULE_DESCRIPTION("Regulator device driver for Powerventure PV88060");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/pv88060-regulator.h b/drivers/regulator/pv88060-regulator.h
new file mode 100644
index 000000000000..02ca9203a172
--- /dev/null
+++ b/drivers/regulator/pv88060-regulator.h
@@ -0,0 +1,69 @@
+/*
+ * pv88060-regulator.h - Regulator definitions for PV88060
+ * Copyright (C) 2015 Powerventure Semiconductor Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __PV88060_REGISTERS_H__
+#define __PV88060_REGISTERS_H__
+
+/* System Control and Event Registers */
+#define	PV88060_REG_EVENT_A			0x04
+#define	PV88060_REG_MASK_A			0x08
+#define	PV88060_REG_MASK_B			0x09
+#define	PV88060_REG_MASK_C			0x0A
+
+/* Regulator Registers */
+#define	PV88060_REG_BUCK1_CONF0			0x1B
+#define	PV88060_REG_BUCK1_CONF1			0x1C
+#define	PV88060_REG_LDO1_CONF			0x1D
+#define	PV88060_REG_LDO2_CONF			0x1E
+#define	PV88060_REG_LDO3_CONF			0x1F
+#define	PV88060_REG_LDO4_CONF			0x20
+#define	PV88060_REG_LDO5_CONF			0x21
+#define	PV88060_REG_LDO6_CONF			0x22
+#define	PV88060_REG_LDO7_CONF			0x23
+
+#define	PV88060_REG_SW1_CONF			0x3B
+#define	PV88060_REG_SW2_CONF			0x3C
+#define	PV88060_REG_SW3_CONF			0x3D
+#define	PV88060_REG_SW4_CONF			0x3E
+#define	PV88060_REG_SW5_CONF			0x3F
+#define	PV88060_REG_SW6_CONF			0x40
+
+/* PV88060_REG_EVENT_A (addr=0x04) */
+#define	PV88060_E_VDD_FLT			0x01
+#define	PV88060_E_OVER_TEMP			0x02
+
+/* PV88060_REG_MASK_A (addr=0x08) */
+#define	PV88060_M_VDD_FLT			0x01
+#define	PV88060_M_OVER_TEMP			0x02
+
+/* PV88060_REG_BUCK1_CONF0 (addr=0x1B) */
+#define	PV88060_BUCK_EN			0x80
+#define PV88060_VBUCK_MASK			0x7F
+/* PV88060_REG_LDO1/2/3/4/5/6/7_CONT */
+#define	PV88060_LDO_EN			0x40
+#define PV88060_VLDO_MASK			0x3F
+/* PV88060_REG_SW1/2/3/4/5_CONF */
+#define	PV88060_SW_EN			0x80
+
+/* PV88060_REG_BUCK1_CONF1 (addr=0x1C) */
+#define	PV88060_BUCK_ILIM_SHIFT			2
+#define	PV88060_BUCK_ILIM_MASK			0x0C
+#define	PV88060_BUCK_MODE_SHIFT			0
+#define	PV88060_BUCK_MODE_MASK			0x03
+#define	PV88060_BUCK_MODE_SLEEP			0x00
+#define	PV88060_BUCK_MODE_AUTO			0x01
+#define	PV88060_BUCK_MODE_SYNC			0x02
+
+#endif	/* __PV88060_REGISTERS_H__ */
diff --git a/drivers/regulator/pv88090-regulator.c b/drivers/regulator/pv88090-regulator.c
new file mode 100644
index 000000000000..ac15f31b5fe0
--- /dev/null
+++ b/drivers/regulator/pv88090-regulator.c
@@ -0,0 +1,458 @@
+/*
+ * pv88090-regulator.c - Regulator device driver for PV88090
+ * Copyright (C) 2015  Powerventure Semiconductor Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regmap.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/proc_fs.h>
+#include <linux/uaccess.h>
+#include "pv88090-regulator.h"
+
+#define PV88090_MAX_REGULATORS	5
+
+/* PV88090 REGULATOR IDs */
+enum {
+	/* BUCKs */
+	PV88090_ID_BUCK1,
+	PV88090_ID_BUCK2,
+	PV88090_ID_BUCK3,
+
+	/* LDOs */
+	PV88090_ID_LDO1,
+	PV88090_ID_LDO2,
+};
+
+struct pv88090_regulator {
+	struct regulator_desc desc;
+	/* Current limiting */
+	unsigned	n_current_limits;
+	const int	*current_limits;
+	unsigned int limit_mask;
+	unsigned int conf;
+	unsigned int conf2;
+};
+
+struct pv88090 {
+	struct device *dev;
+	struct regmap *regmap;
+	struct regulator_dev *rdev[PV88090_MAX_REGULATORS];
+};
+
+struct pv88090_buck_voltage {
+	int min_uV;
+	int max_uV;
+	int uV_step;
+};
+
+static const struct regmap_config pv88090_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+/* Current limits array (in uA) for BUCK1, BUCK2, BUCK3.
+ *  Entry indexes corresponds to register values.
+ */
+
+static const int pv88090_buck1_limits[] = {
+	 220000,  440000,  660000,  880000, 1100000, 1320000, 1540000, 1760000,
+	1980000, 2200000, 2420000, 2640000, 2860000, 3080000, 3300000, 3520000,
+	3740000, 3960000, 4180000, 4400000, 4620000, 4840000, 5060000, 5280000,
+	5500000, 5720000, 5940000, 6160000, 6380000, 6600000, 6820000, 7040000
+};
+
+static const int pv88090_buck23_limits[] = {
+	1496000, 2393000, 3291000, 4189000
+};
+
+static const struct pv88090_buck_voltage pv88090_buck_vol[3] = {
+	{
+		.min_uV = 600000,
+		.max_uV = 1393750,
+		.uV_step = 6250,
+	},
+
+	{
+		.min_uV = 1400000,
+		.max_uV = 2193750,
+		.uV_step = 6250,
+	},
+	{
+		.min_uV = 1250000,
+		.max_uV = 2837500,
+		.uV_step = 12500,
+	},
+};
+
+static unsigned int pv88090_buck_get_mode(struct regulator_dev *rdev)
+{
+	struct pv88090_regulator *info = rdev_get_drvdata(rdev);
+	unsigned int data;
+	int ret, mode = 0;
+
+	ret = regmap_read(rdev->regmap, info->conf, &data);
+	if (ret < 0)
+		return ret;
+
+	switch (data & PV88090_BUCK1_MODE_MASK) {
+	case PV88090_BUCK_MODE_SYNC:
+		mode = REGULATOR_MODE_FAST;
+		break;
+	case PV88090_BUCK_MODE_AUTO:
+		mode = REGULATOR_MODE_NORMAL;
+		break;
+	case PV88090_BUCK_MODE_SLEEP:
+		mode = REGULATOR_MODE_STANDBY;
+		break;
+	}
+
+	return mode;
+}
+
+static int pv88090_buck_set_mode(struct regulator_dev *rdev,
+					unsigned int mode)
+{
+	struct pv88090_regulator *info = rdev_get_drvdata(rdev);
+	int val = 0;
+
+	switch (mode) {
+	case REGULATOR_MODE_FAST:
+		val = PV88090_BUCK_MODE_SYNC;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = PV88090_BUCK_MODE_AUTO;
+		break;
+	case REGULATOR_MODE_STANDBY:
+		val = PV88090_BUCK_MODE_SLEEP;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(rdev->regmap, info->conf,
+					PV88090_BUCK1_MODE_MASK, val);
+}
+
+static int pv88090_set_current_limit(struct regulator_dev *rdev, int min,
+				    int max)
+{
+	struct pv88090_regulator *info = rdev_get_drvdata(rdev);
+	int i;
+
+	/* search for closest to maximum */
+	for (i = info->n_current_limits; i >= 0; i--) {
+		if (min <= info->current_limits[i]
+			&& max >= info->current_limits[i]) {
+			return regmap_update_bits(rdev->regmap,
+				info->conf,
+				info->limit_mask,
+				i << PV88090_BUCK1_ILIM_SHIFT);
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int pv88090_get_current_limit(struct regulator_dev *rdev)
+{
+	struct pv88090_regulator *info = rdev_get_drvdata(rdev);
+	unsigned int data;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, info->conf, &data);
+	if (ret < 0)
+		return ret;
+
+	data = (data & info->limit_mask) >> PV88090_BUCK1_ILIM_SHIFT;
+	return info->current_limits[data];
+}
+
+static struct regulator_ops pv88090_buck_ops = {
+	.get_mode = pv88090_buck_get_mode,
+	.set_mode = pv88090_buck_set_mode,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	.set_current_limit = pv88090_set_current_limit,
+	.get_current_limit = pv88090_get_current_limit,
+};
+
+static struct regulator_ops pv88090_ldo_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+};
+
+#define PV88090_BUCK(chip, regl_name, min, step, max, limits_array) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88090_buck_ops,\
+		.min_uV = min, \
+		.uV_step = step, \
+		.n_voltages = ((max) - (min))/(step) + 1, \
+		.enable_reg = PV88090_REG_##regl_name##_CONF0, \
+		.enable_mask = PV88090_##regl_name##_EN, \
+		.vsel_reg = PV88090_REG_##regl_name##_CONF0, \
+		.vsel_mask = PV88090_V##regl_name##_MASK, \
+	},\
+	.current_limits = limits_array, \
+	.n_current_limits = ARRAY_SIZE(limits_array), \
+	.limit_mask = PV88090_##regl_name##_ILIM_MASK, \
+	.conf = PV88090_REG_##regl_name##_CONF1, \
+	.conf2 = PV88090_REG_##regl_name##_CONF2, \
+}
+
+#define PV88090_LDO(chip, regl_name, min, step, max) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88090_ldo_ops,\
+		.min_uV = min, \
+		.uV_step = step, \
+		.n_voltages = ((max) - (min))/(step) + 1, \
+		.enable_reg = PV88090_REG_##regl_name##_CONT, \
+		.enable_mask = PV88090_##regl_name##_EN, \
+		.vsel_reg = PV88090_REG_##regl_name##_CONT, \
+		.vsel_mask = PV88090_V##regl_name##_MASK, \
+	},\
+}
+
+static struct pv88090_regulator pv88090_regulator_info[] = {
+	PV88090_BUCK(PV88090, BUCK1, 600000, 6250, 1393750,
+		pv88090_buck1_limits),
+	PV88090_BUCK(PV88090, BUCK2, 600000, 6250, 1393750,
+		pv88090_buck23_limits),
+	PV88090_BUCK(PV88090, BUCK3, 600000, 6250, 1393750,
+		pv88090_buck23_limits),
+	PV88090_LDO(PV88090, LDO1, 1200000, 50000, 4350000),
+	PV88090_LDO(PV88090, LDO2,  650000, 25000, 2225000),
+};
+
+static irqreturn_t pv88090_irq_handler(int irq, void *data)
+{
+	struct pv88090 *chip = data;
+	int i, reg_val, err, ret = IRQ_NONE;
+
+	err = regmap_read(chip->regmap, PV88090_REG_EVENT_A, &reg_val);
+	if (err < 0)
+		goto error_i2c;
+
+	if (reg_val & PV88090_E_VDD_FLT) {
+		for (i = 0; i < PV88090_MAX_REGULATORS; i++) {
+			if (chip->rdev[i] != NULL) {
+				regulator_notifier_call_chain(chip->rdev[i],
+					REGULATOR_EVENT_UNDER_VOLTAGE,
+					NULL);
+			}
+		}
+
+		err = regmap_update_bits(chip->regmap, PV88090_REG_EVENT_A,
+			PV88090_E_VDD_FLT, PV88090_E_VDD_FLT);
+		if (err < 0)
+			goto error_i2c;
+
+		ret = IRQ_HANDLED;
+	}
+
+	if (reg_val & PV88090_E_OVER_TEMP) {
+		for (i = 0; i < PV88090_MAX_REGULATORS; i++) {
+			if (chip->rdev[i] != NULL) {
+				regulator_notifier_call_chain(chip->rdev[i],
+					REGULATOR_EVENT_OVER_TEMP,
+					NULL);
+			}
+		}
+
+		err = regmap_update_bits(chip->regmap, PV88090_REG_EVENT_A,
+			PV88090_E_OVER_TEMP, PV88090_E_OVER_TEMP);
+		if (err < 0)
+			goto error_i2c;
+
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+
+error_i2c:
+	dev_err(chip->dev, "I2C error : %d\n", err);
+	return IRQ_NONE;
+}
+
+/*
+ * I2C driver interface functions
+ */
+static int pv88090_i2c_probe(struct i2c_client *i2c,
+		const struct i2c_device_id *id)
+{
+	struct regulator_init_data *init_data = dev_get_platdata(&i2c->dev);
+	struct pv88090 *chip;
+	struct regulator_config config = { };
+	int error, i, ret = 0;
+	unsigned int conf2, range, index;
+
+	chip = devm_kzalloc(&i2c->dev, sizeof(struct pv88090), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	chip->dev = &i2c->dev;
+	chip->regmap = devm_regmap_init_i2c(i2c, &pv88090_regmap_config);
+	if (IS_ERR(chip->regmap)) {
+		error = PTR_ERR(chip->regmap);
+		dev_err(chip->dev, "Failed to allocate register map: %d\n",
+			error);
+		return error;
+	}
+
+	i2c_set_clientdata(i2c, chip);
+
+	if (i2c->irq != 0) {
+		ret = regmap_write(chip->regmap, PV88090_REG_MASK_A, 0xFF);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to mask A reg: %d\n", ret);
+			return ret;
+		}
+
+		ret = regmap_write(chip->regmap, PV88090_REG_MASK_B, 0xFF);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to mask B reg: %d\n", ret);
+			return ret;
+		}
+
+		ret = devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL,
+					pv88090_irq_handler,
+					IRQF_TRIGGER_LOW|IRQF_ONESHOT,
+					"pv88090", chip);
+		if (ret != 0) {
+			dev_err(chip->dev, "Failed to request IRQ: %d\n",
+				i2c->irq);
+			return ret;
+		}
+
+		ret = regmap_update_bits(chip->regmap, PV88090_REG_MASK_A,
+			PV88090_M_VDD_FLT | PV88090_M_OVER_TEMP, 0);
+		if (ret < 0) {
+			dev_err(chip->dev,
+				"Failed to update mask reg: %d\n", ret);
+			return ret;
+		}
+
+	} else {
+		dev_warn(chip->dev, "No IRQ configured\n");
+	}
+
+	config.dev = chip->dev;
+	config.regmap = chip->regmap;
+
+	for (i = 0; i < PV88090_MAX_REGULATORS; i++) {
+		if (init_data)
+			config.init_data = &init_data[i];
+
+		if (i == PV88090_ID_BUCK2 || i == PV88090_ID_BUCK3) {
+			ret = regmap_read(chip->regmap,
+				pv88090_regulator_info[i].conf2, &conf2);
+			if (ret < 0)
+				return ret;
+
+			conf2 = (conf2 >> PV88090_BUCK_VDAC_RANGE_SHIFT) &
+				PV88090_BUCK_VDAC_RANGE_MASK;
+
+			ret = regmap_read(chip->regmap,
+				PV88090_REG_BUCK_FOLD_RANGE, &range);
+			if (ret < 0)
+				return ret;
+
+			range = (range >>
+				 (PV88080_BUCK_VRANGE_GAIN_SHIFT + i - 1)) &
+				PV88080_BUCK_VRANGE_GAIN_MASK;
+			index = ((range << 1) | conf2);
+
+			pv88090_regulator_info[i].desc.min_uV
+				= pv88090_buck_vol[index].min_uV;
+			pv88090_regulator_info[i].desc.uV_step
+				= pv88090_buck_vol[index].uV_step;
+			pv88090_regulator_info[i].desc.n_voltages
+				= ((pv88090_buck_vol[index].max_uV)
+				- (pv88090_buck_vol[index].min_uV))
+				/(pv88090_buck_vol[index].uV_step) + 1;
+		}
+
+		config.driver_data = (void *)&pv88090_regulator_info[i];
+		chip->rdev[i] = devm_regulator_register(chip->dev,
+			&pv88090_regulator_info[i].desc, &config);
+		if (IS_ERR(chip->rdev[i])) {
+			dev_err(chip->dev,
+				"Failed to register PV88090 regulator\n");
+			return PTR_ERR(chip->rdev[i]);
+		}
+	}
+
+	return 0;
+}
+
+static const struct i2c_device_id pv88090_i2c_id[] = {
+	{"pv88090", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, pv88090_i2c_id);
+
+#ifdef CONFIG_OF
+static const struct of_device_id pv88090_dt_ids[] = {
+	{ .compatible = "pvs,pv88090", .data = &pv88090_i2c_id[0] },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pv88090_dt_ids);
+#endif
+
+static struct i2c_driver pv88090_regulator_driver = {
+	.driver = {
+		.name = "pv88090",
+		.of_match_table = of_match_ptr(pv88090_dt_ids),
+	},
+	.probe = pv88090_i2c_probe,
+	.id_table = pv88090_i2c_id,
+};
+
+module_i2c_driver(pv88090_regulator_driver);
+
+MODULE_AUTHOR("James Ban <James.Ban.opensource@diasemi.com>");
+MODULE_DESCRIPTION("Regulator device driver for Powerventure PV88090");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/pv88090-regulator.h b/drivers/regulator/pv88090-regulator.h
new file mode 100644
index 000000000000..d7aca8d8266d
--- /dev/null
+++ b/drivers/regulator/pv88090-regulator.h
@@ -0,0 +1,98 @@
+/*
+ * pv88090-regulator.h - Regulator definitions for PV88090
+ * Copyright (C) 2015 Powerventure Semiconductor Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __PV88090_REGISTERS_H__
+#define __PV88090_REGISTERS_H__
+
+/* System Control and Event Registers */
+#define	PV88090_REG_EVENT_A			0x03
+#define	PV88090_REG_MASK_A			0x06
+#define	PV88090_REG_MASK_B			0x07
+
+/* Regulator Registers */
+#define	PV88090_REG_BUCK1_CONF0			0x18
+#define	PV88090_REG_BUCK1_CONF1			0x19
+#define	PV88090_REG_BUCK1_CONF2			0x1a
+#define	PV88090_REG_BUCK2_CONF0			0x1b
+#define	PV88090_REG_BUCK2_CONF1			0x1c
+#define	PV88090_REG_BUCK2_CONF2			0x58
+#define	PV88090_REG_BUCK3_CONF0			0x1d
+#define	PV88090_REG_BUCK3_CONF1			0x1e
+#define	PV88090_REG_BUCK3_CONF2			0x5c
+
+#define	PV88090_REG_LDO1_CONT			0x1f
+#define	PV88090_REG_LDO2_CONT			0x20
+#define	PV88090_REG_LDO3_CONT			0x21
+#define	PV88090_REG_BUCK_FOLD_RANGE			0x61
+
+/* PV88090_REG_EVENT_A (addr=0x03) */
+#define	PV88090_E_VDD_FLT				0x01
+#define	PV88090_E_OVER_TEMP			0x02
+
+/* PV88090_REG_MASK_A (addr=0x06) */
+#define	PV88090_M_VDD_FLT				0x01
+#define	PV88090_M_OVER_TEMP			0x02
+
+/* PV88090_REG_BUCK1_CONF0 (addr=0x18) */
+#define	PV88090_BUCK1_EN				0x80
+#define PV88090_VBUCK1_MASK			0x7F
+/* PV88090_REG_BUCK2_CONF0 (addr=0x1b) */
+#define	PV88090_BUCK2_EN				0x80
+#define PV88090_VBUCK2_MASK			0x7F
+/* PV88090_REG_BUCK3_CONF0 (addr=0x1d) */
+#define	PV88090_BUCK3_EN				0x80
+#define PV88090_VBUCK3_MASK			0x7F
+/* PV88090_REG_LDO1_CONT (addr=0x1f) */
+#define	PV88090_LDO1_EN				0x40
+#define PV88090_VLDO1_MASK			0x3F
+/* PV88090_REG_LDO2_CONT (addr=0x20) */
+#define	PV88090_LDO2_EN				0x40
+#define PV88090_VLDO2_MASK			0x3F
+
+/* PV88090_REG_BUCK1_CONF1 (addr=0x19) */
+#define PV88090_BUCK1_ILIM_SHIFT			2
+#define PV88090_BUCK1_ILIM_MASK			0x7C
+#define PV88090_BUCK1_MODE_MASK			0x03
+
+/* PV88090_REG_BUCK2_CONF1 (addr=0x1c) */
+#define PV88090_BUCK2_ILIM_SHIFT			2
+#define PV88090_BUCK2_ILIM_MASK			0x0C
+#define PV88090_BUCK2_MODE_MASK			0x03
+
+/* PV88090_REG_BUCK3_CONF1 (addr=0x1e) */
+#define PV88090_BUCK3_ILIM_SHIFT			2
+#define PV88090_BUCK3_ILIM_MASK			0x0C
+#define PV88090_BUCK3_MODE_MASK			0x03
+
+#define	PV88090_BUCK_MODE_SLEEP			0x00
+#define	PV88090_BUCK_MODE_AUTO			0x01
+#define	PV88090_BUCK_MODE_SYNC			0x02
+
+/* PV88090_REG_BUCK2_CONF2 (addr=0x58) */
+/* PV88090_REG_BUCK3_CONF2 (addr=0x5c) */
+#define PV88090_BUCK_VDAC_RANGE_SHIFT			7
+#define PV88090_BUCK_VDAC_RANGE_MASK			0x01
+
+#define PV88090_BUCK_VDAC_RANGE_1			0x00
+#define PV88090_BUCK_VDAC_RANGE_2			0x01
+
+/* PV88090_REG_BUCK_FOLD_RANGE (addr=0x61) */
+#define PV88080_BUCK_VRANGE_GAIN_SHIFT			3
+#define PV88080_BUCK_VRANGE_GAIN_MASK			0x01
+
+#define PV88080_BUCK_VRANGE_GAIN_1			0x00
+#define PV88080_BUCK_VRANGE_GAIN_2			0x01
+
+#endif	/* __PV88090_REGISTERS_H__ */
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index 6fa0c7d13290..56a17ec5b5ef 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -153,6 +153,49 @@ static const struct regulator_ops rpm_switch_ops = {
 	.is_enabled = rpm_reg_is_enabled,
 };
 
+static const struct regulator_desc pma8084_hfsmps = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(375000,  0,  95, 12500),
+		REGULATOR_LINEAR_RANGE(1550000, 96, 158, 25000),
+	},
+	.n_linear_ranges = 2,
+	.n_voltages = 159,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pma8084_ftsmps = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(350000,  0, 184, 5000),
+		REGULATOR_LINEAR_RANGE(700000, 185, 339, 10000),
+	},
+	.n_linear_ranges = 2,
+	.n_voltages = 340,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pma8084_pldo = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(750000,  0,  30, 25000),
+		REGULATOR_LINEAR_RANGE(1500000, 31, 99, 50000),
+	},
+	.n_linear_ranges = 2,
+	.n_voltages = 100,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pma8084_nldo = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(750000, 0, 63, 12500),
+	},
+	.n_linear_ranges = 1,
+	.n_voltages = 64,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pma8084_switch = {
+	.ops = &rpm_switch_ops,
+};
+
 static const struct regulator_desc pm8x41_hfsmps = {
 	.linear_ranges = (struct regulator_linear_range[]) {
 		REGULATOR_LINEAR_RANGE( 375000,  0,  95, 12500),
@@ -211,6 +254,43 @@ static const struct regulator_desc pm8941_switch = {
 	.ops = &rpm_switch_ops,
 };
 
+static const struct regulator_desc pm8916_pldo = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(750000, 0, 208, 12500),
+	},
+	.n_linear_ranges = 1,
+	.n_voltages = 209,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm8916_nldo = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(375000, 0, 93, 12500),
+	},
+	.n_linear_ranges = 1,
+	.n_voltages = 94,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm8916_buck_lvo_smps = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(375000, 0, 95, 12500),
+		REGULATOR_LINEAR_RANGE(750000, 96, 127, 25000),
+	},
+	.n_linear_ranges = 2,
+	.n_voltages = 128,
+	.ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm8916_buck_hvo_smps = {
+	.linear_ranges = (struct regulator_linear_range[]) {
+		REGULATOR_LINEAR_RANGE(1550000, 0, 31, 25000),
+	},
+	.n_linear_ranges = 1,
+	.n_voltages = 32,
+	.ops = &rpm_smps_ldo_ops,
+};
+
 struct rpm_regulator_data {
 	const char *name;
 	u32 type;
@@ -231,6 +311,32 @@ static const struct rpm_regulator_data rpm_pm8841_regulators[] = {
 	{}
 };
 
+static const struct rpm_regulator_data rpm_pm8916_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8916_buck_lvo_smps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8916_buck_lvo_smps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm8916_buck_lvo_smps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm8916_buck_hvo_smps, "vdd_s4" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm8916_nldo, "vdd_l1_l2_l3" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm8916_nldo, "vdd_l1_l2_l3" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm8916_nldo, "vdd_l1_l2_l3" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm8916_pldo, "vdd_l4_l5_l6" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm8916_pldo, "vdd_l4_l5_l6" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm8916_pldo, "vdd_l4_l5_l6" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm8916_pldo, "vdd_l7" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm8916_pldo, "vdd_l8_l9_l10_l11_l12_l13_l14_l15_l16_l17_l18"},
+	{}
+};
+
 static const struct rpm_regulator_data rpm_pm8941_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8x41_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8x41_hfsmps, "vdd_s2" },
@@ -272,9 +378,62 @@ static const struct rpm_regulator_data rpm_pm8941_regulators[] = {
 	{}
 };
 
+static const struct rpm_regulator_data rpm_pma8084_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pma8084_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pma8084_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pma8084_hfsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pma8084_hfsmps, "vdd_s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pma8084_hfsmps, "vdd_s5" },
+	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pma8084_ftsmps, "vdd_s6" },
+	{ "s7", QCOM_SMD_RPM_SMPA, 7, &pma8084_ftsmps, "vdd_s7" },
+	{ "s8", QCOM_SMD_RPM_SMPA, 8, &pma8084_ftsmps, "vdd_s8" },
+	{ "s9", QCOM_SMD_RPM_SMPA, 9, &pma8084_ftsmps, "vdd_s9" },
+	{ "s10", QCOM_SMD_RPM_SMPA, 10, &pma8084_ftsmps, "vdd_s10" },
+	{ "s11", QCOM_SMD_RPM_SMPA, 11, &pma8084_ftsmps, "vdd_s11" },
+	{ "s12", QCOM_SMD_RPM_SMPA, 12, &pma8084_ftsmps, "vdd_s12" },
+
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pma8084_nldo, "vdd_l1_l11" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pma8084_pldo, "vdd_l5_l7" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pma8084_pldo, "vdd_l5_l7" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pma8084_pldo, "vdd_l8" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pma8084_nldo, "vdd_l1_l11" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pma8084_pldo, "vdd_l16_l25" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pma8084_pldo, "vdd_l17" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pma8084_pldo, "vdd_l18" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pma8084_pldo, "vdd_l19" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pma8084_pldo, "vdd_l21" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pma8084_pldo, "vdd_l22" },
+	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l25", QCOM_SMD_RPM_LDOA, 25, &pma8084_pldo, "vdd_l16_l25" },
+	{ "l26", QCOM_SMD_RPM_LDOA, 26, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l27", QCOM_SMD_RPM_LDOA, 27, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+
+	{ "lvs1", QCOM_SMD_RPM_VSA, 1, &pma8084_switch },
+	{ "lvs2", QCOM_SMD_RPM_VSA, 2, &pma8084_switch },
+	{ "lvs3", QCOM_SMD_RPM_VSA, 3, &pma8084_switch },
+	{ "lvs4", QCOM_SMD_RPM_VSA, 4, &pma8084_switch },
+	{ "5vs1", QCOM_SMD_RPM_VSA, 5, &pma8084_switch },
+
+	{}
+};
+
 static const struct of_device_id rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-pm8841-regulators", .data = &rpm_pm8841_regulators },
+	{ .compatible = "qcom,rpm-pm8916-regulators", .data = &rpm_pm8916_regulators },
 	{ .compatible = "qcom,rpm-pm8941-regulators", .data = &rpm_pm8941_regulators },
+	{ .compatible = "qcom,rpm-pma8084-regulators", .data = &rpm_pma8084_regulators },
 	{}
 };
 MODULE_DEVICE_TABLE(of, rpm_of_match);
diff --git a/drivers/regulator/tps6105x-regulator.c b/drivers/regulator/tps6105x-regulator.c
index ddc4f10e268a..584ef3dedca6 100644
--- a/drivers/regulator/tps6105x-regulator.c
+++ b/drivers/regulator/tps6105x-regulator.c
@@ -27,90 +27,12 @@ static const unsigned int tps6105x_voltages[] = {
 	5000000, /* There is an additional 5V */
 };
 
-static int tps6105x_regulator_enable(struct regulator_dev *rdev)
-{
-	struct tps6105x *tps6105x = rdev_get_drvdata(rdev);
-	int ret;
-
-	/* Activate voltage mode */
-	ret = regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0,
-		TPS6105X_REG0_MODE_MASK,
-		TPS6105X_REG0_MODE_VOLTAGE << TPS6105X_REG0_MODE_SHIFT);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int tps6105x_regulator_disable(struct regulator_dev *rdev)
-{
-	struct tps6105x *tps6105x = rdev_get_drvdata(rdev);
-	int ret;
-
-	/* Set into shutdown mode */
-	ret = regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0,
-		TPS6105X_REG0_MODE_MASK,
-		TPS6105X_REG0_MODE_SHUTDOWN << TPS6105X_REG0_MODE_SHIFT);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int tps6105x_regulator_is_enabled(struct regulator_dev *rdev)
-{
-	struct tps6105x *tps6105x = rdev_get_drvdata(rdev);
-	unsigned int regval;
-	int ret;
-
-	ret = regmap_read(tps6105x->regmap, TPS6105X_REG_0, &regval);
-	if (ret)
-		return ret;
-	regval &= TPS6105X_REG0_MODE_MASK;
-	regval >>= TPS6105X_REG0_MODE_SHIFT;
-
-	if (regval == TPS6105X_REG0_MODE_VOLTAGE)
-		return 1;
-
-	return 0;
-}
-
-static int tps6105x_regulator_get_voltage_sel(struct regulator_dev *rdev)
-{
-	struct tps6105x *tps6105x = rdev_get_drvdata(rdev);
-	unsigned int regval;
-	int ret;
-
-	ret = regmap_read(tps6105x->regmap, TPS6105X_REG_0, &regval);
-	if (ret)
-		return ret;
-
-	regval &= TPS6105X_REG0_VOLTAGE_MASK;
-	regval >>= TPS6105X_REG0_VOLTAGE_SHIFT;
-	return (int) regval;
-}
-
-static int tps6105x_regulator_set_voltage_sel(struct regulator_dev *rdev,
-					      unsigned selector)
-{
-	struct tps6105x *tps6105x = rdev_get_drvdata(rdev);
-	int ret;
-
-	ret = regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0,
-				    TPS6105X_REG0_VOLTAGE_MASK,
-				    selector << TPS6105X_REG0_VOLTAGE_SHIFT);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 static struct regulator_ops tps6105x_regulator_ops = {
-	.enable		= tps6105x_regulator_enable,
-	.disable	= tps6105x_regulator_disable,
-	.is_enabled	= tps6105x_regulator_is_enabled,
-	.get_voltage_sel = tps6105x_regulator_get_voltage_sel,
-	.set_voltage_sel = tps6105x_regulator_set_voltage_sel,
+	.enable		= regulator_enable_regmap,
+	.disable	= regulator_disable_regmap,
+	.is_enabled	= regulator_is_enabled_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.list_voltage	= regulator_list_voltage_table,
 };
 
@@ -122,6 +44,12 @@ static const struct regulator_desc tps6105x_regulator_desc = {
 	.owner		= THIS_MODULE,
 	.n_voltages	= ARRAY_SIZE(tps6105x_voltages),
 	.volt_table	= tps6105x_voltages,
+	.vsel_reg	= TPS6105X_REG_0,
+	.vsel_mask	= TPS6105X_REG0_VOLTAGE_MASK,
+	.enable_reg	= TPS6105X_REG_0,
+	.enable_mask	= TPS6105X_REG0_MODE_MASK,
+	.enable_val	= TPS6105X_REG0_MODE_VOLTAGE <<
+			  TPS6105X_REG0_MODE_SHIFT,
 };
 
 /*
@@ -144,6 +72,7 @@ static int tps6105x_regulator_probe(struct platform_device *pdev)
 	config.dev = &tps6105x->client->dev;
 	config.init_data = pdata->regulator_data;
 	config.driver_data = tps6105x;
+	config.regmap = tps6105x->regmap;
 
 	/* Register regulator with framework */
 	tps6105x->regulator = devm_regulator_register(&pdev->dev,
diff --git a/drivers/regulator/tps65086-regulator.c b/drivers/regulator/tps65086-regulator.c
new file mode 100644
index 000000000000..33f389d583ef
--- /dev/null
+++ b/drivers/regulator/tps65086-regulator.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Author: Andrew F. Davis <afd@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether expressed or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ *
+ * Based on the TPS65912 driver
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+
+#include <linux/mfd/tps65086.h>
+
+enum tps65086_regulators { BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6, LDOA1,
+	LDOA2, LDOA3, SWA1, SWB1, SWB2, VTT };
+
+#define TPS65086_REGULATOR(_name, _of, _id, _nv, _vr, _vm, _er, _em, _lr, _dr, _dm)	\
+	[_id] = {							\
+		.desc = {						\
+			.name			= _name,		\
+			.of_match		= of_match_ptr(_of),	\
+			.regulators_node	= "regulators",		\
+			.of_parse_cb		= tps65086_of_parse_cb,	\
+			.id			= _id,			\
+			.ops			= &reg_ops,		\
+			.n_voltages		= _nv,			\
+			.type			= REGULATOR_VOLTAGE,	\
+			.owner			= THIS_MODULE,		\
+			.vsel_reg		= _vr,			\
+			.vsel_mask		= _vm,			\
+			.enable_reg		= _er,			\
+			.enable_mask		= _em,			\
+			.volt_table		= NULL,			\
+			.linear_ranges		= _lr,			\
+			.n_linear_ranges	= ARRAY_SIZE(_lr),	\
+		},							\
+		.decay_reg = _dr,					\
+		.decay_mask = _dm,					\
+	}
+
+#define TPS65086_SWITCH(_name, _of, _id, _er, _em)			\
+	[_id] = {							\
+		.desc = {						\
+			.name			= _name,		\
+			.of_match		= of_match_ptr(_of),	\
+			.regulators_node	= "regulators",		\
+			.of_parse_cb		= tps65086_of_parse_cb,	\
+			.id			= _id,			\
+			.ops			= &switch_ops,		\
+			.type			= REGULATOR_VOLTAGE,	\
+			.owner			= THIS_MODULE,		\
+			.enable_reg		= _er,			\
+			.enable_mask		= _em,			\
+		},							\
+	}
+
+struct tps65086_regulator {
+	struct regulator_desc desc;
+	unsigned int decay_reg;
+	unsigned int decay_mask;
+};
+
+static const struct regulator_linear_range tps65086_buck126_10mv_ranges[] = {
+	REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0),
+	REGULATOR_LINEAR_RANGE(410000, 0x1, 0x7F, 10000),
+};
+
+static const struct regulator_linear_range tps65086_buck126_25mv_ranges[] = {
+	REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0),
+	REGULATOR_LINEAR_RANGE(1000000, 0x1, 0x18, 0),
+	REGULATOR_LINEAR_RANGE(1025000, 0x19, 0x7F, 25000),
+};
+
+static const struct regulator_linear_range tps65086_buck345_ranges[] = {
+	REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0),
+	REGULATOR_LINEAR_RANGE(425000, 0x1, 0x7F, 25000),
+};
+
+static const struct regulator_linear_range tps65086_ldoa1_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1350000, 0x0, 0x0, 0),
+	REGULATOR_LINEAR_RANGE(1500000, 0x1, 0x7, 100000),
+	REGULATOR_LINEAR_RANGE(2300000, 0x8, 0xA, 100000),
+	REGULATOR_LINEAR_RANGE(2700000, 0xB, 0xD, 150000),
+	REGULATOR_LINEAR_RANGE(3300000, 0xE, 0xE, 0),
+};
+
+static const struct regulator_linear_range tps65086_ldoa23_ranges[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0x0, 0xD, 50000),
+	REGULATOR_LINEAR_RANGE(1400000, 0xE, 0xF, 100000),
+};
+
+/* Operations permitted on regulators */
+static struct regulator_ops reg_ops = {
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.is_enabled		= regulator_is_enabled_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.map_voltage		= regulator_map_voltage_linear_range,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear_range,
+};
+
+/* Operations permitted on load switches */
+static struct regulator_ops switch_ops = {
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.is_enabled		= regulator_is_enabled_regmap,
+};
+
+static int tps65086_of_parse_cb(struct device_node *dev,
+				const struct regulator_desc *desc,
+				struct regulator_config *config);
+
+static struct tps65086_regulator regulators[] = {
+	TPS65086_REGULATOR("BUCK1", "buck1", BUCK1, 0x80, TPS65086_BUCK1CTRL,
+			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(0),
+			   tps65086_buck126_10mv_ranges, TPS65086_BUCK1CTRL,
+			   BIT(0)),
+	TPS65086_REGULATOR("BUCK2", "buck2", BUCK2, 0x80, TPS65086_BUCK2CTRL,
+			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(1),
+			   tps65086_buck126_10mv_ranges, TPS65086_BUCK2CTRL,
+			   BIT(0)),
+	TPS65086_REGULATOR("BUCK3", "buck3", BUCK3, 0x80, TPS65086_BUCK3VID,
+			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(2),
+			   tps65086_buck345_ranges, TPS65086_BUCK3DECAY,
+			   BIT(0)),
+	TPS65086_REGULATOR("BUCK4", "buck4", BUCK4, 0x80, TPS65086_BUCK4VID,
+			   BUCK_VID_MASK, TPS65086_BUCK4CTRL, BIT(0),
+			   tps65086_buck345_ranges, TPS65086_BUCK4VID,
+			   BIT(0)),
+	TPS65086_REGULATOR("BUCK5", "buck5", BUCK5, 0x80, TPS65086_BUCK5VID,
+			   BUCK_VID_MASK, TPS65086_BUCK5CTRL, BIT(0),
+			   tps65086_buck345_ranges, TPS65086_BUCK5CTRL,
+			   BIT(0)),
+	TPS65086_REGULATOR("BUCK6", "buck6", BUCK6, 0x80, TPS65086_BUCK6VID,
+			   BUCK_VID_MASK, TPS65086_BUCK6CTRL, BIT(0),
+			   tps65086_buck126_10mv_ranges, TPS65086_BUCK6CTRL,
+			   BIT(0)),
+	TPS65086_REGULATOR("LDOA1", "ldoa1", LDOA1, 0xF, TPS65086_LDOA1CTRL,
+			   VDOA1_VID_MASK, TPS65086_LDOA1CTRL, BIT(0),
+			   tps65086_ldoa1_ranges, 0, 0),
+	TPS65086_REGULATOR("LDOA2", "ldoa2", LDOA2, 0x10, TPS65086_LDOA2VID,
+			   VDOA23_VID_MASK, TPS65086_LDOA2CTRL, BIT(0),
+			   tps65086_ldoa23_ranges, 0, 0),
+	TPS65086_REGULATOR("LDOA3", "ldoa3", LDOA3, 0x10, TPS65086_LDOA3VID,
+			   VDOA23_VID_MASK, TPS65086_LDOA3CTRL, BIT(0),
+			   tps65086_ldoa23_ranges, 0, 0),
+	TPS65086_SWITCH("SWA1", "swa1", SWA1, TPS65086_SWVTT_EN, BIT(5)),
+	TPS65086_SWITCH("SWB1", "swa2", SWB1, TPS65086_SWVTT_EN, BIT(6)),
+	TPS65086_SWITCH("SWB2", "swa3", SWB2, TPS65086_SWVTT_EN, BIT(7)),
+	TPS65086_SWITCH("VTT", "vtt", VTT, TPS65086_SWVTT_EN, BIT(4)),
+};
+
+static inline bool has_25mv_mode(int id)
+{
+	switch (id) {
+	case BUCK1:
+	case BUCK2:
+	case BUCK6:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int tps65086_of_parse_cb(struct device_node *dev,
+				const struct regulator_desc *desc,
+				struct regulator_config *config)
+{
+	int ret;
+
+	/* Check for 25mV step mode */
+	if (has_25mv_mode(desc->id) &&
+			of_property_read_bool(config->of_node, "ti,regulator-step-size-25mv")) {
+		regulators[desc->id].desc.linear_ranges =
+				tps65086_buck126_25mv_ranges;
+		regulators[desc->id].desc.n_linear_ranges =
+				ARRAY_SIZE(tps65086_buck126_25mv_ranges);
+	}
+
+	/* Check for decay mode */
+	if (desc->id <= BUCK6 && of_property_read_bool(config->of_node, "ti,regulator-decay")) {
+		ret = regmap_write_bits(config->regmap,
+					regulators[desc->id].decay_reg,
+					regulators[desc->id].decay_mask,
+					regulators[desc->id].decay_mask);
+		if (ret) {
+			dev_err(config->dev, "Error setting decay\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int tps65086_regulator_probe(struct platform_device *pdev)
+{
+	struct tps65086 *tps = dev_get_drvdata(pdev->dev.parent);
+	struct regulator_config config = { };
+	struct regulator_dev *rdev;
+	int i;
+
+	platform_set_drvdata(pdev, tps);
+
+	config.dev = &pdev->dev;
+	config.dev->of_node = tps->dev->of_node;
+	config.driver_data = tps;
+	config.regmap = tps->regmap;
+
+	for (i = 0; i < ARRAY_SIZE(regulators); i++) {
+		rdev = devm_regulator_register(&pdev->dev, &regulators[i].desc,
+					       &config);
+		if (IS_ERR(rdev)) {
+			dev_err(tps->dev, "failed to register %s regulator\n",
+				pdev->name);
+			return PTR_ERR(rdev);
+		}
+	}
+
+	return 0;
+}
+
+static const struct platform_device_id tps65086_regulator_id_table[] = {
+	{ "tps65086-regulator", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, tps65086_regulator_id_table);
+
+static struct platform_driver tps65086_regulator_driver = {
+	.driver = {
+		.name = "tps65086-regulator",
+	},
+	.probe = tps65086_regulator_probe,
+	.id_table = tps65086_regulator_id_table,
+};
+module_platform_driver(tps65086_regulator_driver);
+
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_DESCRIPTION("TPS65086 Regulator driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/regulator/tps65218-regulator.c b/drivers/regulator/tps65218-regulator.c
index a02c1b961039..a5e5634eeb9e 100644
--- a/drivers/regulator/tps65218-regulator.c
+++ b/drivers/regulator/tps65218-regulator.c
@@ -27,19 +27,22 @@
 #include <linux/regulator/machine.h>
 #include <linux/mfd/tps65218.h>
 
-enum tps65218_regulators { DCDC1, DCDC2, DCDC3, DCDC4, DCDC5, DCDC6, LDO1 };
+enum tps65218_regulators { DCDC1, DCDC2, DCDC3, DCDC4,
+			   DCDC5, DCDC6, LDO1, LS3 };
 
-#define TPS65218_REGULATOR(_name, _id, _ops, _n, _vr, _vm, _er, _em, \
-			    _lr, _nlr, _delay, _fuv)		\
+#define TPS65218_REGULATOR(_name, _id, _type, _ops, _n, _vr, _vm, _er, _em, \
+			    _cr, _cm, _lr, _nlr, _delay, _fuv)		\
 	{							\
 		.name			= _name,		\
 		.id			= _id,			\
 		.ops			= &_ops,		\
 		.n_voltages		= _n,			\
-		.type			= REGULATOR_VOLTAGE,	\
+		.type			= _type,	\
 		.owner			= THIS_MODULE,		\
 		.vsel_reg		= _vr,			\
 		.vsel_mask		= _vm,			\
+		.csel_reg		= _cr,			\
+		.csel_mask		= _cm,			\
 		.enable_reg		= _er,			\
 		.enable_mask		= _em,			\
 		.volt_table		= NULL,			\
@@ -80,6 +83,7 @@ static struct tps_info tps65218_pmic_regs[] = {
 	TPS65218_INFO(DCDC5, "DCDC5", 1000000, 1000000),
 	TPS65218_INFO(DCDC6, "DCDC6", 1800000, 1800000),
 	TPS65218_INFO(LDO1, "LDO1", 900000, 3400000),
+	TPS65218_INFO(LS3, "LS3", -1, -1),
 };
 
 #define TPS65218_OF_MATCH(comp, label) \
@@ -96,6 +100,7 @@ static const struct of_device_id tps65218_of_match[] = {
 	TPS65218_OF_MATCH("ti,tps65218-dcdc5", tps65218_pmic_regs[DCDC5]),
 	TPS65218_OF_MATCH("ti,tps65218-dcdc6", tps65218_pmic_regs[DCDC6]),
 	TPS65218_OF_MATCH("ti,tps65218-ldo1", tps65218_pmic_regs[LDO1]),
+	TPS65218_OF_MATCH("ti,tps65218-ls3", tps65218_pmic_regs[LS3]),
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tps65218_of_match);
@@ -175,6 +180,68 @@ static struct regulator_ops tps65218_ldo1_dcdc34_ops = {
 	.map_voltage		= regulator_map_voltage_linear_range,
 };
 
+static const int ls3_currents[] = { 100, 200, 500, 1000 };
+
+static int tps65218_pmic_set_input_current_lim(struct regulator_dev *dev,
+					       int lim_uA)
+{
+	unsigned int index = 0;
+	unsigned int num_currents = ARRAY_SIZE(ls3_currents);
+	struct tps65218 *tps = rdev_get_drvdata(dev);
+
+	while (index < num_currents && ls3_currents[index] != lim_uA)
+		index++;
+
+	if (index == num_currents)
+		return -EINVAL;
+
+	return tps65218_set_bits(tps, dev->desc->csel_reg, dev->desc->csel_mask,
+				 index << 2, TPS65218_PROTECT_L1);
+}
+
+static int tps65218_pmic_set_current_limit(struct regulator_dev *dev,
+					   int min_uA, int max_uA)
+{
+	int index = 0;
+	unsigned int num_currents = ARRAY_SIZE(ls3_currents);
+	struct tps65218 *tps = rdev_get_drvdata(dev);
+
+	while (index < num_currents && ls3_currents[index] < max_uA)
+		index++;
+
+	index--;
+
+	if (index < 0 || ls3_currents[index] < min_uA)
+		return -EINVAL;
+
+	return tps65218_set_bits(tps, dev->desc->csel_reg, dev->desc->csel_mask,
+				 index << 2, TPS65218_PROTECT_L1);
+}
+
+static int tps65218_pmic_get_current_limit(struct regulator_dev *dev)
+{
+	int retval;
+	unsigned int index;
+	struct tps65218 *tps = rdev_get_drvdata(dev);
+
+	retval = tps65218_reg_read(tps, dev->desc->csel_reg, &index);
+	if (retval < 0)
+		return retval;
+
+	index = (index & dev->desc->csel_mask) >> 2;
+
+	return ls3_currents[index];
+}
+
+static struct regulator_ops tps65218_ls3_ops = {
+	.is_enabled		= regulator_is_enabled_regmap,
+	.enable			= tps65218_pmic_enable,
+	.disable		= tps65218_pmic_disable,
+	.set_input_current_limit = tps65218_pmic_set_input_current_lim,
+	.set_current_limit	= tps65218_pmic_set_current_limit,
+	.get_current_limit	= tps65218_pmic_get_current_limit,
+};
+
 /* Operations permitted on DCDC5, DCDC6 */
 static struct regulator_ops tps65218_dcdc56_pmic_ops = {
 	.is_enabled		= regulator_is_enabled_regmap,
@@ -183,36 +250,46 @@ static struct regulator_ops tps65218_dcdc56_pmic_ops = {
 };
 
 static const struct regulator_desc regulators[] = {
-	TPS65218_REGULATOR("DCDC1", TPS65218_DCDC_1, tps65218_dcdc12_ops, 64,
-			   TPS65218_REG_CONTROL_DCDC1,
-			   TPS65218_CONTROL_DCDC1_MASK,
-			   TPS65218_REG_ENABLE1, TPS65218_ENABLE1_DC1_EN,
-			   dcdc1_dcdc2_ranges, 2, 4000, 0),
-	TPS65218_REGULATOR("DCDC2", TPS65218_DCDC_2, tps65218_dcdc12_ops, 64,
-			   TPS65218_REG_CONTROL_DCDC2,
-			   TPS65218_CONTROL_DCDC2_MASK,
-			   TPS65218_REG_ENABLE1, TPS65218_ENABLE1_DC2_EN,
-			   dcdc1_dcdc2_ranges, 2, 4000, 0),
-	TPS65218_REGULATOR("DCDC3", TPS65218_DCDC_3, tps65218_ldo1_dcdc34_ops,
-			   64, TPS65218_REG_CONTROL_DCDC3,
+	TPS65218_REGULATOR("DCDC1", TPS65218_DCDC_1, REGULATOR_VOLTAGE,
+			   tps65218_dcdc12_ops, 64, TPS65218_REG_CONTROL_DCDC1,
+			   TPS65218_CONTROL_DCDC1_MASK, TPS65218_REG_ENABLE1,
+			   TPS65218_ENABLE1_DC1_EN, 0, 0, dcdc1_dcdc2_ranges,
+			   2, 4000, 0),
+	TPS65218_REGULATOR("DCDC2", TPS65218_DCDC_2, REGULATOR_VOLTAGE,
+			   tps65218_dcdc12_ops, 64, TPS65218_REG_CONTROL_DCDC2,
+			   TPS65218_CONTROL_DCDC2_MASK, TPS65218_REG_ENABLE1,
+			   TPS65218_ENABLE1_DC2_EN, 0, 0, dcdc1_dcdc2_ranges,
+			   2, 4000, 0),
+	TPS65218_REGULATOR("DCDC3", TPS65218_DCDC_3, REGULATOR_VOLTAGE,
+			   tps65218_ldo1_dcdc34_ops, 64,
+			   TPS65218_REG_CONTROL_DCDC3,
 			   TPS65218_CONTROL_DCDC3_MASK, TPS65218_REG_ENABLE1,
-			   TPS65218_ENABLE1_DC3_EN, ldo1_dcdc3_ranges, 2, 0, 0),
-	TPS65218_REGULATOR("DCDC4", TPS65218_DCDC_4, tps65218_ldo1_dcdc34_ops,
-			   53, TPS65218_REG_CONTROL_DCDC4,
-			   TPS65218_CONTROL_DCDC4_MASK,
-			   TPS65218_REG_ENABLE1, TPS65218_ENABLE1_DC4_EN,
-			   dcdc4_ranges, 2, 0, 0),
-	TPS65218_REGULATOR("DCDC5", TPS65218_DCDC_5, tps65218_dcdc56_pmic_ops,
-			   1, -1, -1, TPS65218_REG_ENABLE1,
-			   TPS65218_ENABLE1_DC5_EN, NULL, 0, 0, 1000000),
-	TPS65218_REGULATOR("DCDC6", TPS65218_DCDC_6, tps65218_dcdc56_pmic_ops,
-			   1, -1, -1, TPS65218_REG_ENABLE1,
-			   TPS65218_ENABLE1_DC6_EN, NULL, 0, 0, 1800000),
-	TPS65218_REGULATOR("LDO1", TPS65218_LDO_1, tps65218_ldo1_dcdc34_ops, 64,
+			   TPS65218_ENABLE1_DC3_EN, 0, 0, ldo1_dcdc3_ranges, 2,
+			   0, 0),
+	TPS65218_REGULATOR("DCDC4", TPS65218_DCDC_4, REGULATOR_VOLTAGE,
+			   tps65218_ldo1_dcdc34_ops, 53,
+			   TPS65218_REG_CONTROL_DCDC4,
+			   TPS65218_CONTROL_DCDC4_MASK, TPS65218_REG_ENABLE1,
+			   TPS65218_ENABLE1_DC4_EN, 0, 0, dcdc4_ranges, 2,
+			   0, 0),
+	TPS65218_REGULATOR("DCDC5", TPS65218_DCDC_5, REGULATOR_VOLTAGE,
+			   tps65218_dcdc56_pmic_ops, 1, -1, -1,
+			   TPS65218_REG_ENABLE1, TPS65218_ENABLE1_DC5_EN, 0, 0,
+			   NULL, 0, 0, 1000000),
+	TPS65218_REGULATOR("DCDC6", TPS65218_DCDC_6, REGULATOR_VOLTAGE,
+			   tps65218_dcdc56_pmic_ops, 1, -1, -1,
+			   TPS65218_REG_ENABLE1, TPS65218_ENABLE1_DC6_EN, 0, 0,
+			   NULL, 0, 0, 1800000),
+	TPS65218_REGULATOR("LDO1", TPS65218_LDO_1, REGULATOR_VOLTAGE,
+			   tps65218_ldo1_dcdc34_ops, 64,
 			   TPS65218_REG_CONTROL_LDO1,
 			   TPS65218_CONTROL_LDO1_MASK, TPS65218_REG_ENABLE2,
-			   TPS65218_ENABLE2_LDO1_EN, ldo1_dcdc3_ranges,
+			   TPS65218_ENABLE2_LDO1_EN, 0, 0, ldo1_dcdc3_ranges,
 			   2, 0, 0),
+	TPS65218_REGULATOR("LS3", TPS65218_LS_3, REGULATOR_CURRENT,
+			   tps65218_ls3_ops, 0, 0, 0, TPS65218_REG_ENABLE2,
+			   TPS65218_ENABLE2_LS3_EN, TPS65218_REG_CONFIG2,
+			   TPS65218_CONFIG2_LS3ILIM_MASK, NULL, 0, 0, 0),
 };
 
 static int tps65218_regulator_probe(struct platform_device *pdev)
diff --git a/drivers/regulator/wm831x-dcdc.c b/drivers/regulator/wm831x-dcdc.c
index 8cbb82ceec40..5a5bc4bb08d2 100644
--- a/drivers/regulator/wm831x-dcdc.c
+++ b/drivers/regulator/wm831x-dcdc.c
@@ -365,7 +365,7 @@ static int wm831x_buckv_get_current_limit(struct regulator_dev *rdev)
 	return wm831x_dcdc_ilim[val];
 }
 
-static struct regulator_ops wm831x_buckv_ops = {
+static const struct regulator_ops wm831x_buckv_ops = {
 	.set_voltage_sel = wm831x_buckv_set_voltage_sel,
 	.get_voltage_sel = wm831x_buckv_get_voltage_sel,
 	.list_voltage = wm831x_buckv_list_voltage,
@@ -585,7 +585,7 @@ static int wm831x_buckp_set_suspend_voltage(struct regulator_dev *rdev, int uV)
 	return wm831x_set_bits(wm831x, reg, WM831X_DC3_ON_VSEL_MASK, sel);
 }
 
-static struct regulator_ops wm831x_buckp_ops = {
+static const struct regulator_ops wm831x_buckp_ops = {
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 	.list_voltage = regulator_list_voltage_linear,
@@ -725,7 +725,7 @@ static int wm831x_boostp_get_status(struct regulator_dev *rdev)
 		return REGULATOR_STATUS_OFF;
 }
 
-static struct regulator_ops wm831x_boostp_ops = {
+static const struct regulator_ops wm831x_boostp_ops = {
 	.get_status = wm831x_boostp_get_status,
 
 	.is_enabled = regulator_is_enabled_regmap,
@@ -818,7 +818,7 @@ static struct platform_driver wm831x_boostp_driver = {
 
 #define WM831X_EPE_BASE 6
 
-static struct regulator_ops wm831x_epe_ops = {
+static const struct regulator_ops wm831x_epe_ops = {
 	.is_enabled = regulator_is_enabled_regmap,
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
@@ -884,35 +884,22 @@ static struct platform_driver wm831x_epe_driver = {
 	},
 };
 
+static struct platform_driver * const drivers[] = {
+	&wm831x_buckv_driver,
+	&wm831x_buckp_driver,
+	&wm831x_boostp_driver,
+	&wm831x_epe_driver,
+};
+
 static int __init wm831x_dcdc_init(void)
 {
-	int ret;
-	ret = platform_driver_register(&wm831x_buckv_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x BUCKV driver: %d\n", ret);
-
-	ret = platform_driver_register(&wm831x_buckp_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x BUCKP driver: %d\n", ret);
-
-	ret = platform_driver_register(&wm831x_boostp_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x BOOST driver: %d\n", ret);
-
-	ret = platform_driver_register(&wm831x_epe_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x EPE driver: %d\n", ret);
-
-	return 0;
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 subsys_initcall(wm831x_dcdc_init);
 
 static void __exit wm831x_dcdc_exit(void)
 {
-	platform_driver_unregister(&wm831x_epe_driver);
-	platform_driver_unregister(&wm831x_boostp_driver);
-	platform_driver_unregister(&wm831x_buckp_driver);
-	platform_driver_unregister(&wm831x_buckv_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 module_exit(wm831x_dcdc_exit);
 
diff --git a/drivers/regulator/wm831x-isink.c b/drivers/regulator/wm831x-isink.c
index 1442828fcd9a..6dd891d7eee3 100644
--- a/drivers/regulator/wm831x-isink.c
+++ b/drivers/regulator/wm831x-isink.c
@@ -128,7 +128,7 @@ static int wm831x_isink_get_current(struct regulator_dev *rdev)
 	return wm831x_isinkv_values[ret];
 }
 
-static struct regulator_ops wm831x_isink_ops = {
+static const struct regulator_ops wm831x_isink_ops = {
 	.is_enabled = wm831x_isink_is_enabled,
 	.enable = wm831x_isink_enable,
 	.disable = wm831x_isink_disable,
diff --git a/drivers/regulator/wm831x-ldo.c b/drivers/regulator/wm831x-ldo.c
index 5a7b65e8a529..e4a6f888484e 100644
--- a/drivers/regulator/wm831x-ldo.c
+++ b/drivers/regulator/wm831x-ldo.c
@@ -198,7 +198,7 @@ static unsigned int wm831x_gp_ldo_get_optimum_mode(struct regulator_dev *rdev,
 }
 
 
-static struct regulator_ops wm831x_gp_ldo_ops = {
+static const struct regulator_ops wm831x_gp_ldo_ops = {
 	.list_voltage = regulator_list_voltage_linear_range,
 	.map_voltage = regulator_map_voltage_linear_range,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -409,7 +409,7 @@ static int wm831x_aldo_get_status(struct regulator_dev *rdev)
 		return regulator_mode_to_status(ret);
 }
 
-static struct regulator_ops wm831x_aldo_ops = {
+static const struct regulator_ops wm831x_aldo_ops = {
 	.list_voltage = regulator_list_voltage_linear_range,
 	.map_voltage = regulator_map_voltage_linear_range,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -557,7 +557,7 @@ static int wm831x_alive_ldo_get_status(struct regulator_dev *rdev)
 		return REGULATOR_STATUS_OFF;
 }
 
-static struct regulator_ops wm831x_alive_ldo_ops = {
+static const struct regulator_ops wm831x_alive_ldo_ops = {
 	.list_voltage = regulator_list_voltage_linear,
 	.map_voltage = regulator_map_voltage_linear,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -653,32 +653,21 @@ static struct platform_driver wm831x_alive_ldo_driver = {
 	},
 };
 
+static struct platform_driver * const drivers[] = {
+	&wm831x_gp_ldo_driver,
+	&wm831x_aldo_driver,
+	&wm831x_alive_ldo_driver,
+};
+
 static int __init wm831x_ldo_init(void)
 {
-	int ret;
-
-	ret = platform_driver_register(&wm831x_gp_ldo_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x GP LDO driver: %d\n", ret);
-
-	ret = platform_driver_register(&wm831x_aldo_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x ALDO driver: %d\n", ret);
-
-	ret = platform_driver_register(&wm831x_alive_ldo_driver);
-	if (ret != 0)
-		pr_err("Failed to register WM831x alive LDO driver: %d\n",
-		       ret);
-
-	return 0;
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 }
 subsys_initcall(wm831x_ldo_init);
 
 static void __exit wm831x_ldo_exit(void)
 {
-	platform_driver_unregister(&wm831x_alive_ldo_driver);
-	platform_driver_unregister(&wm831x_aldo_driver);
-	platform_driver_unregister(&wm831x_gp_ldo_driver);
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 module_exit(wm831x_ldo_exit);
 
diff --git a/drivers/regulator/wm8350-regulator.c b/drivers/regulator/wm8350-regulator.c
index 95f6b040186e..da9106bd2109 100644
--- a/drivers/regulator/wm8350-regulator.c
+++ b/drivers/regulator/wm8350-regulator.c
@@ -941,7 +941,7 @@ static unsigned int wm8350_dcdc_get_optimum_mode(struct regulator_dev *rdev,
 	return mode;
 }
 
-static struct regulator_ops wm8350_dcdc_ops = {
+static const struct regulator_ops wm8350_dcdc_ops = {
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 	.list_voltage = regulator_list_voltage_linear,
@@ -958,7 +958,7 @@ static struct regulator_ops wm8350_dcdc_ops = {
 	.set_suspend_mode = wm8350_dcdc_set_suspend_mode,
 };
 
-static struct regulator_ops wm8350_dcdc2_5_ops = {
+static const struct regulator_ops wm8350_dcdc2_5_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
 	.is_enabled = regulator_is_enabled_regmap,
@@ -966,7 +966,7 @@ static struct regulator_ops wm8350_dcdc2_5_ops = {
 	.set_suspend_disable = wm8350_dcdc25_set_suspend_disable,
 };
 
-static struct regulator_ops wm8350_ldo_ops = {
+static const struct regulator_ops wm8350_ldo_ops = {
 	.map_voltage = regulator_map_voltage_linear_range,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -980,7 +980,7 @@ static struct regulator_ops wm8350_ldo_ops = {
 	.set_suspend_disable = wm8350_ldo_set_suspend_disable,
 };
 
-static struct regulator_ops wm8350_isink_ops = {
+static const struct regulator_ops wm8350_isink_ops = {
 	.set_current_limit = wm8350_isink_set_current,
 	.get_current_limit = wm8350_isink_get_current,
 	.enable = wm8350_isink_enable,
diff --git a/drivers/regulator/wm8400-regulator.c b/drivers/regulator/wm8400-regulator.c
index 82d829000851..fb1837657b64 100644
--- a/drivers/regulator/wm8400-regulator.c
+++ b/drivers/regulator/wm8400-regulator.c
@@ -24,7 +24,7 @@ static const struct regulator_linear_range wm8400_ldo_ranges[] = {
 	REGULATOR_LINEAR_RANGE(1700000, 15, 31, 100000),
 };
 
-static struct regulator_ops wm8400_ldo_ops = {
+static const struct regulator_ops wm8400_ldo_ops = {
 	.is_enabled = regulator_is_enabled_regmap,
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
@@ -106,7 +106,7 @@ static unsigned int wm8400_dcdc_get_optimum_mode(struct regulator_dev *dev,
 	return REGULATOR_MODE_NORMAL;
 }
 
-static struct regulator_ops wm8400_dcdc_ops = {
+static const struct regulator_ops wm8400_dcdc_ops = {
 	.is_enabled = regulator_is_enabled_regmap,
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
diff --git a/drivers/regulator/wm8994-regulator.c b/drivers/regulator/wm8994-regulator.c
index 750e0bd61f9f..7a4ce6df4f22 100644
--- a/drivers/regulator/wm8994-regulator.c
+++ b/drivers/regulator/wm8994-regulator.c
@@ -36,7 +36,7 @@ struct wm8994_ldo {
 #define WM8994_LDO1_MAX_SELECTOR 0x7
 #define WM8994_LDO2_MAX_SELECTOR 0x3
 
-static struct regulator_ops wm8994_ldo1_ops = {
+static const struct regulator_ops wm8994_ldo1_ops = {
 	.list_voltage = regulator_list_voltage_linear,
 	.map_voltage = regulator_map_voltage_linear,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -69,7 +69,7 @@ static int wm8994_ldo2_list_voltage(struct regulator_dev *rdev,
 	}
 }
 
-static struct regulator_ops wm8994_ldo2_ops = {
+static const struct regulator_ops wm8994_ldo2_ops = {
 	.list_voltage = wm8994_ldo2_list_voltage,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index e1a10232a943..e44872fb9e5e 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -147,7 +147,7 @@ static void rproc_virtio_del_vqs(struct virtio_device *vdev)
 static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		       struct virtqueue *vqs[],
 		       vq_callback_t *callbacks[],
-		       const char *names[])
+		       const char * const names[])
 {
 	struct rproc *rproc = vdev_to_rproc(vdev);
 	int i, ret;
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 0615f50a14cd..df37212a5cbd 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -13,3 +13,4 @@ menuconfig RESET_CONTROLLER
 	  If unsure, say no.
 
 source "drivers/reset/sti/Kconfig"
+source "drivers/reset/hisilicon/Kconfig"
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index 85d5904e5480..4d7178e46afa 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -1,8 +1,9 @@
-obj-$(CONFIG_RESET_CONTROLLER) += core.o
+obj-y += core.o
 obj-$(CONFIG_ARCH_LPC18XX) += reset-lpc18xx.o
 obj-$(CONFIG_ARCH_SOCFPGA) += reset-socfpga.o
 obj-$(CONFIG_ARCH_BERLIN) += reset-berlin.o
 obj-$(CONFIG_ARCH_SUNXI) += reset-sunxi.o
 obj-$(CONFIG_ARCH_STI) += sti/
+obj-$(CONFIG_ARCH_HISI) += hisilicon/
 obj-$(CONFIG_ARCH_ZYNQ) += reset-zynq.o
 obj-$(CONFIG_ATH79) += reset-ath79.o
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index 7955e00d04d4..87376638948d 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -30,7 +30,6 @@ static LIST_HEAD(reset_controller_list);
  */
 struct reset_control {
 	struct reset_controller_dev *rcdev;
-	struct device *dev;
 	unsigned int id;
 };
 
@@ -95,7 +94,7 @@ int reset_control_reset(struct reset_control *rstc)
 	if (rstc->rcdev->ops->reset)
 		return rstc->rcdev->ops->reset(rstc->rcdev, rstc->id);
 
-	return -ENOSYS;
+	return -ENOTSUPP;
 }
 EXPORT_SYMBOL_GPL(reset_control_reset);
 
@@ -108,7 +107,7 @@ int reset_control_assert(struct reset_control *rstc)
 	if (rstc->rcdev->ops->assert)
 		return rstc->rcdev->ops->assert(rstc->rcdev, rstc->id);
 
-	return -ENOSYS;
+	return -ENOTSUPP;
 }
 EXPORT_SYMBOL_GPL(reset_control_assert);
 
@@ -121,7 +120,7 @@ int reset_control_deassert(struct reset_control *rstc)
 	if (rstc->rcdev->ops->deassert)
 		return rstc->rcdev->ops->deassert(rstc->rcdev, rstc->id);
 
-	return -ENOSYS;
+	return -ENOTSUPP;
 }
 EXPORT_SYMBOL_GPL(reset_control_deassert);
 
@@ -136,32 +135,29 @@ int reset_control_status(struct reset_control *rstc)
 	if (rstc->rcdev->ops->status)
 		return rstc->rcdev->ops->status(rstc->rcdev, rstc->id);
 
-	return -ENOSYS;
+	return -ENOTSUPP;
 }
 EXPORT_SYMBOL_GPL(reset_control_status);
 
 /**
- * of_reset_control_get - Lookup and obtain a reference to a reset controller.
+ * of_reset_control_get_by_index - Lookup and obtain a reference to a reset
+ * controller by index.
  * @node: device to be reset by the controller
- * @id: reset line name
+ * @index: index of the reset controller
  *
- * Returns a struct reset_control or IS_ERR() condition containing errno.
- *
- * Use of id names is optional.
+ * This is to be used to perform a list of resets for a device or power domain
+ * in whatever order. Returns a struct reset_control or IS_ERR() condition
+ * containing errno.
  */
-struct reset_control *of_reset_control_get(struct device_node *node,
-					   const char *id)
+struct reset_control *of_reset_control_get_by_index(struct device_node *node,
+					   int index)
 {
 	struct reset_control *rstc = ERR_PTR(-EPROBE_DEFER);
 	struct reset_controller_dev *r, *rcdev;
 	struct of_phandle_args args;
-	int index = 0;
 	int rstc_id;
 	int ret;
 
-	if (id)
-		index = of_property_match_string(node,
-						 "reset-names", id);
 	ret = of_parse_phandle_with_args(node, "resets", "#reset-cells",
 					 index, &args);
 	if (ret)
@@ -202,6 +198,30 @@ struct reset_control *of_reset_control_get(struct device_node *node,
 
 	return rstc;
 }
+EXPORT_SYMBOL_GPL(of_reset_control_get_by_index);
+
+/**
+ * of_reset_control_get - Lookup and obtain a reference to a reset controller.
+ * @node: device to be reset by the controller
+ * @id: reset line name
+ *
+ * Returns a struct reset_control or IS_ERR() condition containing errno.
+ *
+ * Use of id names is optional.
+ */
+struct reset_control *of_reset_control_get(struct device_node *node,
+					   const char *id)
+{
+	int index = 0;
+
+	if (id) {
+		index = of_property_match_string(node,
+						 "reset-names", id);
+		if (index < 0)
+			return ERR_PTR(-ENOENT);
+	}
+	return of_reset_control_get_by_index(node, index);
+}
 EXPORT_SYMBOL_GPL(of_reset_control_get);
 
 /**
@@ -215,16 +235,10 @@ EXPORT_SYMBOL_GPL(of_reset_control_get);
  */
 struct reset_control *reset_control_get(struct device *dev, const char *id)
 {
-	struct reset_control *rstc;
-
 	if (!dev)
 		return ERR_PTR(-EINVAL);
 
-	rstc = of_reset_control_get(dev->of_node, id);
-	if (!IS_ERR(rstc))
-		rstc->dev = dev;
-
-	return rstc;
+	return of_reset_control_get(dev->of_node, id);
 }
 EXPORT_SYMBOL_GPL(reset_control_get);
 
diff --git a/drivers/reset/hisilicon/Kconfig b/drivers/reset/hisilicon/Kconfig
new file mode 100644
index 000000000000..26bf95a83a8e
--- /dev/null
+++ b/drivers/reset/hisilicon/Kconfig
@@ -0,0 +1,5 @@
+config COMMON_RESET_HI6220
+	tristate "Hi6220 Reset Driver"
+	depends on (ARCH_HISI && RESET_CONTROLLER)
+	help
+	  Build the Hisilicon Hi6220 reset driver.
diff --git a/drivers/reset/hisilicon/Makefile b/drivers/reset/hisilicon/Makefile
new file mode 100644
index 000000000000..c932f86e2f10
--- /dev/null
+++ b/drivers/reset/hisilicon/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_COMMON_RESET_HI6220) += hi6220_reset.o
diff --git a/drivers/reset/hisilicon/hi6220_reset.c b/drivers/reset/hisilicon/hi6220_reset.c
new file mode 100644
index 000000000000..7787a9b1cc67
--- /dev/null
+++ b/drivers/reset/hisilicon/hi6220_reset.c
@@ -0,0 +1,109 @@
+/*
+ * Hisilicon Hi6220 reset controller driver
+ *
+ * Copyright (c) 2015 Hisilicon Limited.
+ *
+ * Author: Feng Chen <puck.chen@hisilicon.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/io.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/bitops.h>
+#include <linux/of.h>
+#include <linux/reset-controller.h>
+#include <linux/reset.h>
+#include <linux/platform_device.h>
+
+#define ASSERT_OFFSET            0x300
+#define DEASSERT_OFFSET          0x304
+#define MAX_INDEX                0x509
+
+#define to_reset_data(x) container_of(x, struct hi6220_reset_data, rc_dev)
+
+struct hi6220_reset_data {
+	void __iomem			*assert_base;
+	void __iomem			*deassert_base;
+	struct reset_controller_dev	rc_dev;
+};
+
+static int hi6220_reset_assert(struct reset_controller_dev *rc_dev,
+			       unsigned long idx)
+{
+	struct hi6220_reset_data *data = to_reset_data(rc_dev);
+
+	int bank = idx >> 8;
+	int offset = idx & 0xff;
+
+	writel(BIT(offset), data->assert_base + (bank * 0x10));
+
+	return 0;
+}
+
+static int hi6220_reset_deassert(struct reset_controller_dev *rc_dev,
+				 unsigned long idx)
+{
+	struct hi6220_reset_data *data = to_reset_data(rc_dev);
+
+	int bank = idx >> 8;
+	int offset = idx & 0xff;
+
+	writel(BIT(offset), data->deassert_base + (bank * 0x10));
+
+	return 0;
+}
+
+static struct reset_control_ops hi6220_reset_ops = {
+	.assert = hi6220_reset_assert,
+	.deassert = hi6220_reset_deassert,
+};
+
+static int hi6220_reset_probe(struct platform_device *pdev)
+{
+	struct hi6220_reset_data *data;
+	struct resource *res;
+	void __iomem *src_base;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	src_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(src_base))
+		return PTR_ERR(src_base);
+
+	data->assert_base = src_base + ASSERT_OFFSET;
+	data->deassert_base = src_base + DEASSERT_OFFSET;
+	data->rc_dev.nr_resets = MAX_INDEX;
+	data->rc_dev.ops = &hi6220_reset_ops;
+	data->rc_dev.of_node = pdev->dev.of_node;
+
+	reset_controller_register(&data->rc_dev);
+
+	return 0;
+}
+
+static const struct of_device_id hi6220_reset_match[] = {
+	{ .compatible = "hisilicon,hi6220-sysctrl" },
+	{ },
+};
+
+static struct platform_driver hi6220_reset_driver = {
+	.probe = hi6220_reset_probe,
+	.driver = {
+		.name = "reset-hi6220",
+		.of_match_table = hi6220_reset_match,
+	},
+};
+
+static int __init hi6220_reset_init(void)
+{
+	return platform_driver_register(&hi6220_reset_driver);
+}
+
+postcore_initcall(hi6220_reset_init);
diff --git a/drivers/reset/reset-ath79.c b/drivers/reset/reset-ath79.c
index 9aaf646ece55..692fc890e94b 100644
--- a/drivers/reset/reset-ath79.c
+++ b/drivers/reset/reset-ath79.c
@@ -15,13 +15,17 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/reset-controller.h>
+#include <linux/reboot.h>
 
 struct ath79_reset {
 	struct reset_controller_dev rcdev;
+	struct notifier_block restart_nb;
 	void __iomem *base;
 	spinlock_t lock;
 };
 
+#define FULL_CHIP_RESET 24
+
 static int ath79_reset_update(struct reset_controller_dev *rcdev,
 			unsigned long id, bool assert)
 {
@@ -72,10 +76,22 @@ static struct reset_control_ops ath79_reset_ops = {
 	.status = ath79_reset_status,
 };
 
+static int ath79_reset_restart_handler(struct notifier_block *nb,
+				unsigned long action, void *data)
+{
+	struct ath79_reset *ath79_reset =
+		container_of(nb, struct ath79_reset, restart_nb);
+
+	ath79_reset_assert(&ath79_reset->rcdev, FULL_CHIP_RESET);
+
+	return NOTIFY_DONE;
+}
+
 static int ath79_reset_probe(struct platform_device *pdev)
 {
 	struct ath79_reset *ath79_reset;
 	struct resource *res;
+	int err;
 
 	ath79_reset = devm_kzalloc(&pdev->dev,
 				sizeof(*ath79_reset), GFP_KERNEL);
@@ -96,13 +112,25 @@ static int ath79_reset_probe(struct platform_device *pdev)
 	ath79_reset->rcdev.of_reset_n_cells = 1;
 	ath79_reset->rcdev.nr_resets = 32;
 
-	return reset_controller_register(&ath79_reset->rcdev);
+	err = reset_controller_register(&ath79_reset->rcdev);
+	if (err)
+		return err;
+
+	ath79_reset->restart_nb.notifier_call = ath79_reset_restart_handler;
+	ath79_reset->restart_nb.priority = 128;
+
+	err = register_restart_handler(&ath79_reset->restart_nb);
+	if (err)
+		dev_warn(&pdev->dev, "Failed to register restart handler\n");
+
+	return 0;
 }
 
 static int ath79_reset_remove(struct platform_device *pdev)
 {
 	struct ath79_reset *ath79_reset = platform_get_drvdata(pdev);
 
+	unregister_restart_handler(&ath79_reset->restart_nb);
 	reset_controller_unregister(&ath79_reset->rcdev);
 
 	return 0;
diff --git a/drivers/reset/reset-berlin.c b/drivers/reset/reset-berlin.c
index 3c922d37255c..970b1ad60293 100644
--- a/drivers/reset/reset-berlin.c
+++ b/drivers/reset/reset-berlin.c
@@ -87,9 +87,7 @@ static int berlin2_reset_probe(struct platform_device *pdev)
 	priv->rcdev.of_reset_n_cells = 2;
 	priv->rcdev.of_xlate = berlin_reset_xlate;
 
-	reset_controller_register(&priv->rcdev);
-
-	return 0;
+	return reset_controller_register(&priv->rcdev);
 }
 
 static const struct of_device_id berlin_reset_dt_match[] = {
diff --git a/drivers/reset/reset-socfpga.c b/drivers/reset/reset-socfpga.c
index 1a6c5d66c83b..b7d773d9248c 100644
--- a/drivers/reset/reset-socfpga.c
+++ b/drivers/reset/reset-socfpga.c
@@ -133,9 +133,8 @@ static int socfpga_reset_probe(struct platform_device *pdev)
 	data->rcdev.nr_resets = NR_BANKS * BITS_PER_LONG;
 	data->rcdev.ops = &socfpga_reset_ops;
 	data->rcdev.of_node = pdev->dev.of_node;
-	reset_controller_register(&data->rcdev);
 
-	return 0;
+	return reset_controller_register(&data->rcdev);
 }
 
 static int socfpga_reset_remove(struct platform_device *pdev)
diff --git a/drivers/reset/reset-sunxi.c b/drivers/reset/reset-sunxi.c
index 3d95c87160b3..8d41a18da17f 100644
--- a/drivers/reset/reset-sunxi.c
+++ b/drivers/reset/reset-sunxi.c
@@ -108,9 +108,8 @@ static int sunxi_reset_init(struct device_node *np)
 	data->rcdev.nr_resets = size * 32;
 	data->rcdev.ops = &sunxi_reset_ops;
 	data->rcdev.of_node = np;
-	reset_controller_register(&data->rcdev);
 
-	return 0;
+	return reset_controller_register(&data->rcdev);
 
 err_alloc:
 	kfree(data);
@@ -122,7 +121,7 @@ err_alloc:
  * our system, before we can even think of using a regular device
  * driver for it.
  */
-static const struct of_device_id sunxi_early_reset_dt_ids[] __initdata = {
+static const struct of_device_id sunxi_early_reset_dt_ids[] __initconst = {
 	{ .compatible = "allwinner,sun6i-a31-ahb1-reset", },
 	{ /* sentinel */ },
 };
diff --git a/drivers/reset/reset-zynq.c b/drivers/reset/reset-zynq.c
index 89318a5d5bd7..c6b3cd8b40ad 100644
--- a/drivers/reset/reset-zynq.c
+++ b/drivers/reset/reset-zynq.c
@@ -121,9 +121,8 @@ static int zynq_reset_probe(struct platform_device *pdev)
 	priv->rcdev.nr_resets = resource_size(res) / 4 * BITS_PER_LONG;
 	priv->rcdev.ops = &zynq_reset_ops;
 	priv->rcdev.of_node = pdev->dev.of_node;
-	reset_controller_register(&priv->rcdev);
 
-	return 0;
+	return reset_controller_register(&priv->rcdev);
 }
 
 static int zynq_reset_remove(struct platform_device *pdev)
diff --git a/drivers/reset/sti/reset-stih407.c b/drivers/reset/sti/reset-stih407.c
index 827eb3dae47d..6fb22af990c0 100644
--- a/drivers/reset/sti/reset-stih407.c
+++ b/drivers/reset/sti/reset-stih407.c
@@ -52,6 +52,7 @@ static const struct syscfg_reset_channel_data stih407_powerdowns[] = {
 };
 
 /* Reset Generator control 0/1 */
+#define SYSCFG_5128	0x200
 #define SYSCFG_5131	0x20c
 #define SYSCFG_5132	0x210
 
@@ -96,6 +97,10 @@ static const struct syscfg_reset_channel_data stih407_softresets[] = {
 	[STIH407_ERAM_HVA_SOFTRESET] = STIH407_SRST_CORE(SYSCFG_5132, 1),
 	[STIH407_LPM_SOFTRESET] = STIH407_SRST_SBC(SYSCFG_4002, 2),
 	[STIH407_KEYSCAN_SOFTRESET] = STIH407_SRST_LPM(LPM_SYSCFG_1, 8),
+	[STIH407_ST231_AUD_SOFTRESET] = STIH407_SRST_CORE(SYSCFG_5131, 26),
+	[STIH407_ST231_DMU_SOFTRESET] = STIH407_SRST_CORE(SYSCFG_5131, 27),
+	[STIH407_ST231_GP0_SOFTRESET] = STIH407_SRST_CORE(SYSCFG_5131, 28),
+	[STIH407_ST231_GP1_SOFTRESET] = STIH407_SRST_CORE(SYSCFG_5128, 2),
 };
 
 /* PicoPHY reset/control */
diff --git a/drivers/reset/sti/reset-syscfg.c b/drivers/reset/sti/reset-syscfg.c
index a145cc066d4a..1600cc7557f5 100644
--- a/drivers/reset/sti/reset-syscfg.c
+++ b/drivers/reset/sti/reset-syscfg.c
@@ -103,17 +103,42 @@ static int syscfg_reset_deassert(struct reset_controller_dev *rcdev,
 static int syscfg_reset_dev(struct reset_controller_dev *rcdev,
 			    unsigned long idx)
 {
-	int err = syscfg_reset_assert(rcdev, idx);
+	int err;
+
+	err = syscfg_reset_assert(rcdev, idx);
 	if (err)
 		return err;
 
 	return syscfg_reset_deassert(rcdev, idx);
 }
 
+static int syscfg_reset_status(struct reset_controller_dev *rcdev,
+			       unsigned long idx)
+{
+	struct syscfg_reset_controller *rst = to_syscfg_reset_controller(rcdev);
+	const struct syscfg_reset_channel *ch;
+	u32 ret_val = 0;
+	int err;
+
+	if (idx >= rcdev->nr_resets)
+		return -EINVAL;
+
+	ch = &rst->channels[idx];
+	if (ch->ack)
+		err = regmap_field_read(ch->ack, &ret_val);
+	else
+		err = regmap_field_read(ch->reset, &ret_val);
+	if (err)
+		return err;
+
+	return rst->active_low ? !ret_val : !!ret_val;
+}
+
 static struct reset_control_ops syscfg_reset_ops = {
 	.reset    = syscfg_reset_dev,
 	.assert   = syscfg_reset_assert,
 	.deassert = syscfg_reset_deassert,
+	.status   = syscfg_reset_status,
 };
 
 static int syscfg_reset_controller_register(struct device *dev,
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index 73354ee27877..1fcd27c1f183 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -945,7 +945,7 @@ static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len,
 static int rpmsg_probe(struct virtio_device *vdev)
 {
 	vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
-	const char *names[] = { "input", "output" };
+	static const char * const names[] = { "input", "output" };
 	struct virtqueue *vqs[2];
 	struct virtproc_info *vrp;
 	void *bufs_va;
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index 2a524244afec..376322f71fd5 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -558,6 +558,16 @@ config RTC_DRV_FM3130
 	  This driver can also be built as a module. If so the module
 	  will be called rtc-fm3130.
 
+config RTC_DRV_RX8010
+	tristate "Epson RX8010SJ"
+	depends on I2C
+	help
+	  If you say yes here you get support for the Epson RX8010SJ RTC
+	  chip.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-rx8010.
+
 config RTC_DRV_RX8581
 	tristate "Epson RX-8581"
 	help
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index 231f76451615..62d61b26ca7e 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -128,6 +128,7 @@ obj-$(CONFIG_RTC_DRV_RS5C372)	+= rtc-rs5c372.o
 obj-$(CONFIG_RTC_DRV_RV3029C2)	+= rtc-rv3029c2.o
 obj-$(CONFIG_RTC_DRV_RV8803)	+= rtc-rv8803.o
 obj-$(CONFIG_RTC_DRV_RX4581)	+= rtc-rx4581.o
+obj-$(CONFIG_RTC_DRV_RX8010)	+= rtc-rx8010.o
 obj-$(CONFIG_RTC_DRV_RX8025)	+= rtc-rx8025.o
 obj-$(CONFIG_RTC_DRV_RX8581)	+= rtc-rx8581.o
 obj-$(CONFIG_RTC_DRV_S35390A)	+= rtc-s35390a.o
diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c
index afea84c7a155..d41bbcd653f6 100644
--- a/drivers/rtc/rtc-abx80x.c
+++ b/drivers/rtc/rtc-abx80x.c
@@ -27,10 +27,28 @@
 #define ABX8XX_REG_YR		0x06
 #define ABX8XX_REG_WD		0x07
 
+#define ABX8XX_REG_AHTH		0x08
+#define ABX8XX_REG_ASC		0x09
+#define ABX8XX_REG_AMN		0x0a
+#define ABX8XX_REG_AHR		0x0b
+#define ABX8XX_REG_ADA		0x0c
+#define ABX8XX_REG_AMO		0x0d
+#define ABX8XX_REG_AWD		0x0e
+
+#define ABX8XX_REG_STATUS	0x0f
+#define ABX8XX_STATUS_AF	BIT(2)
+
 #define ABX8XX_REG_CTRL1	0x10
 #define ABX8XX_CTRL_WRITE	BIT(0)
+#define ABX8XX_CTRL_ARST	BIT(2)
 #define ABX8XX_CTRL_12_24	BIT(6)
 
+#define ABX8XX_REG_IRQ		0x12
+#define ABX8XX_IRQ_AIE		BIT(2)
+#define ABX8XX_IRQ_IM_1_4	(0x3 << 5)
+
+#define ABX8XX_REG_CD_TIMER_CTL	0x18
+
 #define ABX8XX_REG_CFG_KEY	0x1f
 #define ABX8XX_CFG_KEY_MISC	0x9d
 
@@ -63,8 +81,6 @@ static struct abx80x_cap abx80x_caps[] = {
 	[ABX80X] = {.pn = 0}
 };
 
-static struct i2c_driver abx80x_driver;
-
 static int abx80x_enable_trickle_charger(struct i2c_client *client,
 					 u8 trickle_cfg)
 {
@@ -148,9 +164,111 @@ static int abx80x_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	return 0;
 }
 
+static irqreturn_t abx80x_handle_irq(int irq, void *dev_id)
+{
+	struct i2c_client *client = dev_id;
+	struct rtc_device *rtc = i2c_get_clientdata(client);
+	int status;
+
+	status = i2c_smbus_read_byte_data(client, ABX8XX_REG_STATUS);
+	if (status < 0)
+		return IRQ_NONE;
+
+	if (status & ABX8XX_STATUS_AF)
+		rtc_update_irq(rtc, 1, RTC_AF | RTC_IRQF);
+
+	i2c_smbus_write_byte_data(client, ABX8XX_REG_STATUS, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int abx80x_read_alarm(struct device *dev, struct rtc_wkalrm *t)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	unsigned char buf[7];
+
+	int irq_mask, err;
+
+	if (client->irq <= 0)
+		return -EINVAL;
+
+	err = i2c_smbus_read_i2c_block_data(client, ABX8XX_REG_ASC,
+					    sizeof(buf), buf);
+	if (err)
+		return err;
+
+	irq_mask = i2c_smbus_read_byte_data(client, ABX8XX_REG_IRQ);
+	if (irq_mask < 0)
+		return irq_mask;
+
+	t->time.tm_sec = bcd2bin(buf[0] & 0x7F);
+	t->time.tm_min = bcd2bin(buf[1] & 0x7F);
+	t->time.tm_hour = bcd2bin(buf[2] & 0x3F);
+	t->time.tm_mday = bcd2bin(buf[3] & 0x3F);
+	t->time.tm_mon = bcd2bin(buf[4] & 0x1F) - 1;
+	t->time.tm_wday = buf[5] & 0x7;
+
+	t->enabled = !!(irq_mask & ABX8XX_IRQ_AIE);
+	t->pending = (buf[6] & ABX8XX_STATUS_AF) && t->enabled;
+
+	return err;
+}
+
+static int abx80x_set_alarm(struct device *dev, struct rtc_wkalrm *t)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 alarm[6];
+	int err;
+
+	if (client->irq <= 0)
+		return -EINVAL;
+
+	alarm[0] = 0x0;
+	alarm[1] = bin2bcd(t->time.tm_sec);
+	alarm[2] = bin2bcd(t->time.tm_min);
+	alarm[3] = bin2bcd(t->time.tm_hour);
+	alarm[4] = bin2bcd(t->time.tm_mday);
+	alarm[5] = bin2bcd(t->time.tm_mon + 1);
+
+	err = i2c_smbus_write_i2c_block_data(client, ABX8XX_REG_AHTH,
+					     sizeof(alarm), alarm);
+	if (err < 0) {
+		dev_err(&client->dev, "Unable to write alarm registers\n");
+		return -EIO;
+	}
+
+	if (t->enabled) {
+		err = i2c_smbus_write_byte_data(client, ABX8XX_REG_IRQ,
+						(ABX8XX_IRQ_IM_1_4 |
+						 ABX8XX_IRQ_AIE));
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int abx80x_alarm_irq_enable(struct device *dev, unsigned int enabled)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	int err;
+
+	if (enabled)
+		err = i2c_smbus_write_byte_data(client, ABX8XX_REG_IRQ,
+						(ABX8XX_IRQ_IM_1_4 |
+						 ABX8XX_IRQ_AIE));
+	else
+		err = i2c_smbus_write_byte_data(client, ABX8XX_REG_IRQ,
+						ABX8XX_IRQ_IM_1_4);
+	return err;
+}
+
 static const struct rtc_class_ops abx80x_rtc_ops = {
 	.read_time	= abx80x_rtc_read_time,
 	.set_time	= abx80x_rtc_set_time,
+	.read_alarm	= abx80x_read_alarm,
+	.set_alarm	= abx80x_set_alarm,
+	.alarm_irq_enable = abx80x_alarm_irq_enable,
 };
 
 static int abx80x_dt_trickle_cfg(struct device_node *np)
@@ -225,7 +343,8 @@ static int abx80x_probe(struct i2c_client *client,
 	}
 
 	err = i2c_smbus_write_byte_data(client, ABX8XX_REG_CTRL1,
-					((data & ~ABX8XX_CTRL_12_24) |
+					((data & ~(ABX8XX_CTRL_12_24 |
+						   ABX8XX_CTRL_ARST)) |
 					 ABX8XX_CTRL_WRITE));
 	if (err < 0) {
 		dev_err(&client->dev, "Unable to write control register\n");
@@ -260,7 +379,12 @@ static int abx80x_probe(struct i2c_client *client,
 		abx80x_enable_trickle_charger(client, trickle_cfg);
 	}
 
-	rtc = devm_rtc_device_register(&client->dev, abx80x_driver.driver.name,
+	err = i2c_smbus_write_byte_data(client, ABX8XX_REG_CD_TIMER_CTL,
+					BIT(2));
+	if (err)
+		return err;
+
+	rtc = devm_rtc_device_register(&client->dev, "abx8xx",
 				       &abx80x_rtc_ops, THIS_MODULE);
 
 	if (IS_ERR(rtc))
@@ -268,6 +392,19 @@ static int abx80x_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, rtc);
 
+	if (client->irq > 0) {
+		dev_info(&client->dev, "IRQ %d supplied\n", client->irq);
+		err = devm_request_threaded_irq(&client->dev, client->irq, NULL,
+						abx80x_handle_irq,
+						IRQF_SHARED | IRQF_ONESHOT,
+						"abx8xx",
+						client);
+		if (err) {
+			dev_err(&client->dev, "unable to request IRQ, alarms disabled\n");
+			client->irq = 0;
+		}
+	}
+
 	return 0;
 }
 
@@ -286,6 +423,7 @@ static const struct i2c_device_id abx80x_id[] = {
 	{ "ab1803", AB1803 },
 	{ "ab1804", AB1804 },
 	{ "ab1805", AB1805 },
+	{ "rv1805", AB1805 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, abx80x_id);
diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c
index 8f7034ba7d9e..84fb541038be 100644
--- a/drivers/rtc/rtc-cmos.c
+++ b/drivers/rtc/rtc-cmos.c
@@ -725,7 +725,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq)
 			rtc_cmos_int_handler = cmos_interrupt;
 
 		retval = request_irq(rtc_irq, rtc_cmos_int_handler,
-				0, dev_name(&cmos_rtc.rtc->dev),
+				IRQF_SHARED, dev_name(&cmos_rtc.rtc->dev),
 				cmos_rtc.rtc);
 		if (retval < 0) {
 			dev_dbg(dev, "IRQ %d is already in use\n", rtc_irq);
diff --git a/drivers/rtc/rtc-da9063.c b/drivers/rtc/rtc-da9063.c
index d6c853bbfa9f..f85cae240f12 100644
--- a/drivers/rtc/rtc-da9063.c
+++ b/drivers/rtc/rtc-da9063.c
@@ -191,24 +191,13 @@ static void da9063_tm_to_data(struct rtc_time *tm, u8 *data,
 {
 	const struct da9063_compatible_rtc_regmap *config = rtc->config;
 
-	data[RTC_SEC] &= ~config->rtc_count_sec_mask;
-	data[RTC_SEC] |= tm->tm_sec & config->rtc_count_sec_mask;
-
-	data[RTC_MIN] &= ~config->rtc_count_min_mask;
-	data[RTC_MIN] |= tm->tm_min & config->rtc_count_min_mask;
-
-	data[RTC_HOUR] &= ~config->rtc_count_hour_mask;
-	data[RTC_HOUR] |= tm->tm_hour & config->rtc_count_hour_mask;
-
-	data[RTC_DAY] &= ~config->rtc_count_day_mask;
-	data[RTC_DAY] |= tm->tm_mday & config->rtc_count_day_mask;
-
-	data[RTC_MONTH] &= ~config->rtc_count_month_mask;
-	data[RTC_MONTH] |= MONTHS_TO_DA9063(tm->tm_mon) &
+	data[RTC_SEC]   = tm->tm_sec & config->rtc_count_sec_mask;
+	data[RTC_MIN]   = tm->tm_min & config->rtc_count_min_mask;
+	data[RTC_HOUR]  = tm->tm_hour & config->rtc_count_hour_mask;
+	data[RTC_DAY]   = tm->tm_mday & config->rtc_count_day_mask;
+	data[RTC_MONTH] = MONTHS_TO_DA9063(tm->tm_mon) &
 				config->rtc_count_month_mask;
-
-	data[RTC_YEAR] &= ~config->rtc_count_year_mask;
-	data[RTC_YEAR] |= YEARS_TO_DA9063(tm->tm_year) &
+	data[RTC_YEAR]  = YEARS_TO_DA9063(tm->tm_year) &
 				config->rtc_count_year_mask;
 }
 
diff --git a/drivers/rtc/rtc-ds1305.c b/drivers/rtc/rtc-ds1305.c
index 85706a9f82c9..f39691eea736 100644
--- a/drivers/rtc/rtc-ds1305.c
+++ b/drivers/rtc/rtc-ds1305.c
@@ -186,9 +186,7 @@ static int ds1305_get_time(struct device *dev, struct rtc_time *time)
 	if (status < 0)
 		return status;
 
-	dev_vdbg(dev, "%s: %02x %02x %02x, %02x %02x %02x %02x\n",
-		"read", buf[0], buf[1], buf[2], buf[3],
-		buf[4], buf[5], buf[6]);
+	dev_vdbg(dev, "%s: %3ph, %4ph\n", "read", &buf[0], &buf[3]);
 
 	/* Decode the registers */
 	time->tm_sec = bcd2bin(buf[DS1305_SEC]);
@@ -232,9 +230,7 @@ static int ds1305_set_time(struct device *dev, struct rtc_time *time)
 	*bp++ = bin2bcd(time->tm_mon + 1);
 	*bp++ = bin2bcd(time->tm_year - 100);
 
-	dev_dbg(dev, "%s: %02x %02x %02x, %02x %02x %02x %02x\n",
-		"write", buf[1], buf[2], buf[3],
-		buf[4], buf[5], buf[6], buf[7]);
+	dev_dbg(dev, "%s: %3ph, %4ph\n", "write", &buf[1], &buf[4]);
 
 	/* use write-then-read since dma from stack is nonportable */
 	return spi_write_then_read(ds1305->spi, buf, sizeof(buf),
diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c
index aa705bb4748c..cf685f67b391 100644
--- a/drivers/rtc/rtc-ds1307.c
+++ b/drivers/rtc/rtc-ds1307.c
@@ -460,13 +460,8 @@ static int ds1337_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 		return -EIO;
 	}
 
-	dev_dbg(dev, "%s: %02x %02x %02x %02x, %02x %02x %02x, %02x %02x\n",
-			"alarm read",
-			ds1307->regs[0], ds1307->regs[1],
-			ds1307->regs[2], ds1307->regs[3],
-			ds1307->regs[4], ds1307->regs[5],
-			ds1307->regs[6], ds1307->regs[7],
-			ds1307->regs[8]);
+	dev_dbg(dev, "%s: %4ph, %3ph, %2ph\n", "alarm read",
+		&ds1307->regs[0], &ds1307->regs[4], &ds1307->regs[7]);
 
 	/*
 	 * report alarm time (ALARM1); assume 24 hour and day-of-month modes,
@@ -522,12 +517,8 @@ static int ds1337_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 	control = ds1307->regs[7];
 	status = ds1307->regs[8];
 
-	dev_dbg(dev, "%s: %02x %02x %02x %02x, %02x %02x %02x, %02x %02x\n",
-			"alarm set (old status)",
-			ds1307->regs[0], ds1307->regs[1],
-			ds1307->regs[2], ds1307->regs[3],
-			ds1307->regs[4], ds1307->regs[5],
-			ds1307->regs[6], control, status);
+	dev_dbg(dev, "%s: %4ph, %3ph, %02x %02x\n", "alarm set (old status)",
+		&ds1307->regs[0], &ds1307->regs[4], control, status);
 
 	/* set ALARM1, using 24 hour and day-of-month modes */
 	buf[0] = bin2bcd(t->time.tm_sec);
diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c
index 05a51ef52703..535050fc5e9f 100644
--- a/drivers/rtc/rtc-ds1685.c
+++ b/drivers/rtc/rtc-ds1685.c
@@ -853,7 +853,7 @@ ds1685_rtc_proc(struct device *dev, struct seq_file *seq)
 	   "Periodic Rate\t: %s\n"
 	   "SQW Freq\t: %s\n"
 #ifdef CONFIG_RTC_DS1685_PROC_REGS
-	   "Serial #\t: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n"
+	   "Serial #\t: %8phC\n"
 	   "Register Status\t:\n"
 	   "   Ctrl A\t: UIP  DV2  DV1  DV0  RS3  RS2  RS1  RS0\n"
 	   "\t\t:  %s\n"
@@ -872,7 +872,7 @@ ds1685_rtc_proc(struct device *dev, struct seq_file *seq)
 	   "   Ctrl 4B\t: ABE  E32k  CS  RCE  PRS  RIE  WIE  KSE\n"
 	   "\t\t:  %s\n",
 #else
-	   "Serial #\t: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+	   "Serial #\t: %8phC\n",
 #endif
 	   model,
 	   ((ctrla & RTC_CTRL_A_DV1) ? "enabled" : "disabled"),
@@ -888,7 +888,7 @@ ds1685_rtc_proc(struct device *dev, struct seq_file *seq)
 	   (!((ctrl4b & RTC_CTRL_4B_E32K)) ?
 	    ds1685_rtc_sqw_freq[(ctrla & RTC_CTRL_A_RS_MASK)] : "32768Hz"),
 #ifdef CONFIG_RTC_DS1685_PROC_REGS
-	   ssn[0], ssn[1], ssn[2], ssn[3], ssn[4], ssn[5], ssn[6], ssn[7],
+	   ssn,
 	   ds1685_rtc_print_regs(ctrla, bits[0]),
 	   ds1685_rtc_print_regs(ctrlb, bits[1]),
 	   ds1685_rtc_print_regs(ctrlc, bits[2]),
@@ -896,7 +896,7 @@ ds1685_rtc_proc(struct device *dev, struct seq_file *seq)
 	   ds1685_rtc_print_regs(ctrl4a, bits[4]),
 	   ds1685_rtc_print_regs(ctrl4b, bits[5]));
 #else
-	   ssn[0], ssn[1], ssn[2], ssn[3], ssn[4], ssn[5], ssn[6], ssn[7]);
+	   ssn);
 #endif
 	return 0;
 }
@@ -1114,7 +1114,7 @@ ds1685_rtc_sysfs_battery_show(struct device *dev,
 
 	ctrld = rtc->read(rtc, RTC_CTRL_D);
 
-	return snprintf(buf, 13, "%s\n",
+	return sprintf(buf, "%s\n",
 			(ctrld & RTC_CTRL_D_VRT) ? "ok" : "not ok or N/A");
 }
 static DEVICE_ATTR(battery, S_IRUGO, ds1685_rtc_sysfs_battery_show, NULL);
@@ -1137,7 +1137,7 @@ ds1685_rtc_sysfs_auxbatt_show(struct device *dev,
 	ctrl4a = rtc->read(rtc, RTC_EXT_CTRL_4A);
 	ds1685_rtc_switch_to_bank0(rtc);
 
-	return snprintf(buf, 13, "%s\n",
+	return sprintf(buf, "%s\n",
 			(ctrl4a & RTC_CTRL_4A_VRT2) ? "ok" : "not ok or N/A");
 }
 static DEVICE_ATTR(auxbatt, S_IRUGO, ds1685_rtc_sysfs_auxbatt_show, NULL);
@@ -1160,11 +1160,7 @@ ds1685_rtc_sysfs_serial_show(struct device *dev,
 	ds1685_rtc_get_ssn(rtc, ssn);
 	ds1685_rtc_switch_to_bank0(rtc);
 
-	return snprintf(buf, 24, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
-			ssn[0], ssn[1], ssn[2], ssn[3], ssn[4], ssn[5],
-			ssn[6], ssn[7]);
-
-	return 0;
+	return sprintf(buf, "%8phC\n", ssn);
 }
 static DEVICE_ATTR(serial, S_IRUGO, ds1685_rtc_sysfs_serial_show, NULL);
 
@@ -1287,7 +1283,7 @@ ds1685_rtc_sysfs_ctrl_regs_show(struct device *dev,
 	tmp = rtc->read(rtc, reg_info->reg) & reg_info->bit;
 	ds1685_rtc_switch_to_bank0(rtc);
 
-	return snprintf(buf, 2, "%d\n", (tmp ? 1 : 0));
+	return sprintf(buf, "%d\n", (tmp ? 1 : 0));
 }
 
 /**
@@ -1623,7 +1619,7 @@ ds1685_rtc_sysfs_time_regs_show(struct device *dev,
 	tmp = ds1685_rtc_bcd2bin(rtc, tmp, bcd_reg_info->mask,
 				 bin_reg_info->mask);
 
-	return snprintf(buf, 4, "%d\n", tmp);
+	return sprintf(buf, "%d\n", tmp);
 }
 
 /**
diff --git a/drivers/rtc/rtc-ds2404.c b/drivers/rtc/rtc-ds2404.c
index 7885edd3d507..16310fe79d76 100644
--- a/drivers/rtc/rtc-ds2404.c
+++ b/drivers/rtc/rtc-ds2404.c
@@ -48,7 +48,7 @@ struct ds2404_gpio {
 
 struct ds2404 {
 	struct ds2404_gpio *gpio;
-	struct ds2404_chip_ops *ops;
+	const struct ds2404_chip_ops *ops;
 	struct rtc_device *rtc;
 };
 
@@ -95,7 +95,7 @@ static void ds2404_gpio_unmap(struct ds2404 *chip)
 		gpio_free(ds2404_gpio[i].gpio);
 }
 
-static struct ds2404_chip_ops ds2404_gpio_ops = {
+static const struct ds2404_chip_ops ds2404_gpio_ops = {
 	.map_io		= ds2404_gpio_map,
 	.unmap_io	= ds2404_gpio_unmap,
 };
diff --git a/drivers/rtc/rtc-efi.c b/drivers/rtc/rtc-efi.c
index 3806961b4348..96d38609d803 100644
--- a/drivers/rtc/rtc-efi.c
+++ b/drivers/rtc/rtc-efi.c
@@ -191,11 +191,69 @@ static int efi_set_time(struct device *dev, struct rtc_time *tm)
 	return status == EFI_SUCCESS ? 0 : -EINVAL;
 }
 
+static int efi_procfs(struct device *dev, struct seq_file *seq)
+{
+	efi_time_t      eft, alm;
+	efi_time_cap_t  cap;
+	efi_bool_t      enabled, pending;
+
+	memset(&eft, 0, sizeof(eft));
+	memset(&alm, 0, sizeof(alm));
+	memset(&cap, 0, sizeof(cap));
+
+	efi.get_time(&eft, &cap);
+	efi.get_wakeup_time(&enabled, &pending, &alm);
+
+	seq_printf(seq,
+		   "Time\t\t: %u:%u:%u.%09u\n"
+		   "Date\t\t: %u-%u-%u\n"
+		   "Daylight\t: %u\n",
+		   eft.hour, eft.minute, eft.second, eft.nanosecond,
+		   eft.year, eft.month, eft.day,
+		   eft.daylight);
+
+	if (eft.timezone == EFI_UNSPECIFIED_TIMEZONE)
+		seq_puts(seq, "Timezone\t: unspecified\n");
+	else
+		/* XXX fixme: convert to string? */
+		seq_printf(seq, "Timezone\t: %u\n", eft.timezone);
+
+	seq_printf(seq,
+		   "Alarm Time\t: %u:%u:%u.%09u\n"
+		   "Alarm Date\t: %u-%u-%u\n"
+		   "Alarm Daylight\t: %u\n"
+		   "Enabled\t\t: %s\n"
+		   "Pending\t\t: %s\n",
+		   alm.hour, alm.minute, alm.second, alm.nanosecond,
+		   alm.year, alm.month, alm.day,
+		   alm.daylight,
+		   enabled == 1 ? "yes" : "no",
+		   pending == 1 ? "yes" : "no");
+
+	if (eft.timezone == EFI_UNSPECIFIED_TIMEZONE)
+		seq_puts(seq, "Timezone\t: unspecified\n");
+	else
+		/* XXX fixme: convert to string? */
+		seq_printf(seq, "Timezone\t: %u\n", alm.timezone);
+
+	/*
+	 * now prints the capabilities
+	 */
+	seq_printf(seq,
+		   "Resolution\t: %u\n"
+		   "Accuracy\t: %u\n"
+		   "SetstoZero\t: %u\n",
+		   cap.resolution, cap.accuracy, cap.sets_to_zero);
+
+	return 0;
+}
+
 static const struct rtc_class_ops efi_rtc_ops = {
-	.read_time = efi_read_time,
-	.set_time = efi_set_time,
-	.read_alarm = efi_read_alarm,
-	.set_alarm = efi_set_alarm,
+	.read_time	= efi_read_time,
+	.set_time	= efi_set_time,
+	.read_alarm	= efi_read_alarm,
+	.set_alarm	= efi_set_alarm,
+	.proc		= efi_procfs,
 };
 
 static int __init efi_rtc_probe(struct platform_device *dev)
diff --git a/drivers/rtc/rtc-gemini.c b/drivers/rtc/rtc-gemini.c
index e84184647d15..f46b6d46a51b 100644
--- a/drivers/rtc/rtc-gemini.c
+++ b/drivers/rtc/rtc-gemini.c
@@ -156,7 +156,6 @@ static int gemini_rtc_remove(struct platform_device *pdev)
 	struct gemini_rtc *rtc = platform_get_drvdata(pdev);
 
 	rtc_device_unregister(rtc->rtc_dev);
-	platform_set_drvdata(pdev, NULL);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-imxdi.c b/drivers/rtc/rtc-imxdi.c
index 7bffd7f0e306..8d8049bdfaf6 100644
--- a/drivers/rtc/rtc-imxdi.c
+++ b/drivers/rtc/rtc-imxdi.c
@@ -303,7 +303,7 @@ static int di_handle_invalid_state(struct imxdi_dev *imxdi, u32 dsr)
 	sec = readl(imxdi->ioaddr + DTCMR);
 	if (sec != 0)
 		dev_warn(&imxdi->pdev->dev,
-			 "The security violation has happend at %u seconds\n",
+			 "The security violation has happened at %u seconds\n",
 			 sec);
 	/*
 	 * the timer cannot be set/modified if
diff --git a/drivers/rtc/rtc-lpc32xx.c b/drivers/rtc/rtc-lpc32xx.c
index f923f7324788..887871c3d526 100644
--- a/drivers/rtc/rtc-lpc32xx.c
+++ b/drivers/rtc/rtc-lpc32xx.c
@@ -205,7 +205,7 @@ static int lpc32xx_rtc_probe(struct platform_device *pdev)
 	u32 tmp;
 
 	rtcirq = platform_get_irq(pdev, 0);
-	if (rtcirq < 0 || rtcirq >= NR_IRQS) {
+	if (rtcirq < 0) {
 		dev_warn(&pdev->dev, "Can't get interrupt resource\n");
 		rtcirq = -1;
 	}
diff --git a/drivers/rtc/rtc-opal.c b/drivers/rtc/rtc-opal.c
index df39ce02a99d..9c18d6fd8107 100644
--- a/drivers/rtc/rtc-opal.c
+++ b/drivers/rtc/rtc-opal.c
@@ -40,7 +40,7 @@ static void opal_to_tm(u32 y_m_d, u64 h_m_s_ms, struct rtc_time *tm)
 	tm->tm_min  = bcd2bin((h_m_s_ms >> 48) & 0xff);
 	tm->tm_sec  = bcd2bin((h_m_s_ms >> 40) & 0xff);
 
-	GregorianDay(tm);
+	tm->tm_wday = -1;
 }
 
 static void tm_to_opal(struct rtc_time *tm, u32 *y_m_d, u64 *h_m_s_ms)
diff --git a/drivers/rtc/rtc-pcf8523.c b/drivers/rtc/rtc-pcf8523.c
index e7ebcc0b7e59..988566caaaa6 100644
--- a/drivers/rtc/rtc-pcf8523.c
+++ b/drivers/rtc/rtc-pcf8523.c
@@ -219,6 +219,17 @@ static int pcf8523_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	u8 regs[8];
 	int err;
 
+	/*
+	 * The hardware can only store values between 0 and 99 in it's YEAR
+	 * register (with 99 overflowing to 0 on increment).
+	 * After 2100-02-28 we could start interpreting the year to be in the
+	 * interval [2100, 2199], but there is no path to switch in a smooth way
+	 * because the chip handles YEAR=0x00 (and the out-of-spec
+	 * YEAR=0xa0) as a leap year, but 2100 isn't.
+	 */
+	if (tm->tm_year < 100 || tm->tm_year >= 200)
+		return -EINVAL;
+
 	err = pcf8523_stop_rtc(client);
 	if (err < 0)
 		return err;
diff --git a/drivers/rtc/rtc-proc.c b/drivers/rtc/rtc-proc.c
index ffa69e1c9245..31e7e23cc5be 100644
--- a/drivers/rtc/rtc-proc.c
+++ b/drivers/rtc/rtc-proc.c
@@ -112,19 +112,21 @@ static int rtc_proc_open(struct inode *inode, struct file *file)
 	int ret;
 	struct rtc_device *rtc = PDE_DATA(inode);
 
-	if (!try_module_get(THIS_MODULE))
+	if (!try_module_get(rtc->owner))
 		return -ENODEV;
 
 	ret = single_open(file, rtc_proc_show, rtc);
 	if (ret)
-		module_put(THIS_MODULE);
+		module_put(rtc->owner);
 	return ret;
 }
 
 static int rtc_proc_release(struct inode *inode, struct file *file)
 {
 	int res = single_release(inode, file);
-	module_put(THIS_MODULE);
+	struct rtc_device *rtc = PDE_DATA(inode);
+
+	module_put(rtc->owner);
 	return res;
 }
 
diff --git a/drivers/rtc/rtc-rv8803.c b/drivers/rtc/rtc-rv8803.c
index e7329e21bfe3..7155c0816aa6 100644
--- a/drivers/rtc/rtc-rv8803.c
+++ b/drivers/rtc/rtc-rv8803.c
@@ -61,7 +61,7 @@ static irqreturn_t rv8803_handle_irq(int irq, void *dev_id)
 	struct i2c_client *client = dev_id;
 	struct rv8803_data *rv8803 = i2c_get_clientdata(client);
 	unsigned long events = 0;
-	u8 flags;
+	int flags;
 
 	spin_lock(&rv8803->flags_lock);
 
@@ -502,6 +502,7 @@ static int rv8803_remove(struct i2c_client *client)
 
 static const struct i2c_device_id rv8803_id[] = {
 	{ "rv8803", 0 },
+	{ "rx8900", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, rv8803_id);
diff --git a/drivers/rtc/rtc-rx8010.c b/drivers/rtc/rtc-rx8010.c
new file mode 100644
index 000000000000..772d221ec2d9
--- /dev/null
+++ b/drivers/rtc/rtc-rx8010.c
@@ -0,0 +1,523 @@
+/*
+ * Driver for the Epson RTC module RX-8010 SJ
+ *
+ * Copyright(C) Timesys Corporation 2015
+ * Copyright(C) General Electric Company 2015
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/bcd.h>
+#include <linux/bitops.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/rtc.h>
+
+#define RX8010_SEC     0x10
+#define RX8010_MIN     0x11
+#define RX8010_HOUR    0x12
+#define RX8010_WDAY    0x13
+#define RX8010_MDAY    0x14
+#define RX8010_MONTH   0x15
+#define RX8010_YEAR    0x16
+#define RX8010_YEAR    0x16
+#define RX8010_RESV17  0x17
+#define RX8010_ALMIN   0x18
+#define RX8010_ALHOUR  0x19
+#define RX8010_ALWDAY  0x1A
+#define RX8010_TCOUNT0 0x1B
+#define RX8010_TCOUNT1 0x1C
+#define RX8010_EXT     0x1D
+#define RX8010_FLAG    0x1E
+#define RX8010_CTRL    0x1F
+/* 0x20 to 0x2F are user registers */
+#define RX8010_RESV30  0x30
+#define RX8010_RESV31  0x32
+#define RX8010_IRQ     0x32
+
+#define RX8010_EXT_WADA  BIT(3)
+
+#define RX8010_FLAG_VLF  BIT(1)
+#define RX8010_FLAG_AF   BIT(3)
+#define RX8010_FLAG_TF   BIT(4)
+#define RX8010_FLAG_UF   BIT(5)
+
+#define RX8010_CTRL_AIE  BIT(3)
+#define RX8010_CTRL_UIE  BIT(5)
+#define RX8010_CTRL_STOP BIT(6)
+#define RX8010_CTRL_TEST BIT(7)
+
+#define RX8010_ALARM_AE  BIT(7)
+
+static const struct i2c_device_id rx8010_id[] = {
+	{ "rx8010", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, rx8010_id);
+
+struct rx8010_data {
+	struct i2c_client *client;
+	struct rtc_device *rtc;
+	u8 ctrlreg;
+	spinlock_t flags_lock;
+};
+
+static irqreturn_t rx8010_irq_1_handler(int irq, void *dev_id)
+{
+	struct i2c_client *client = dev_id;
+	struct rx8010_data *rx8010 = i2c_get_clientdata(client);
+	int flagreg;
+
+	spin_lock(&rx8010->flags_lock);
+
+	flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG);
+
+	if (flagreg <= 0) {
+		spin_unlock(&rx8010->flags_lock);
+		return IRQ_NONE;
+	}
+
+	if (flagreg & RX8010_FLAG_VLF)
+		dev_warn(&client->dev, "Frequency stop detected\n");
+
+	if (flagreg & RX8010_FLAG_TF) {
+		flagreg &= ~RX8010_FLAG_TF;
+		rtc_update_irq(rx8010->rtc, 1, RTC_PF | RTC_IRQF);
+	}
+
+	if (flagreg & RX8010_FLAG_AF) {
+		flagreg &= ~RX8010_FLAG_AF;
+		rtc_update_irq(rx8010->rtc, 1, RTC_AF | RTC_IRQF);
+	}
+
+	if (flagreg & RX8010_FLAG_UF) {
+		flagreg &= ~RX8010_FLAG_UF;
+		rtc_update_irq(rx8010->rtc, 1, RTC_UF | RTC_IRQF);
+	}
+
+	i2c_smbus_write_byte_data(client, RX8010_FLAG, flagreg);
+
+	spin_unlock(&rx8010->flags_lock);
+	return IRQ_HANDLED;
+}
+
+static int rx8010_get_time(struct device *dev, struct rtc_time *dt)
+{
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	u8 date[7];
+	int flagreg;
+	int err;
+
+	flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG);
+	if (flagreg < 0)
+		return flagreg;
+
+	if (flagreg & RX8010_FLAG_VLF) {
+		dev_warn(dev, "Frequency stop detected\n");
+		return -EINVAL;
+	}
+
+	err = i2c_smbus_read_i2c_block_data(rx8010->client, RX8010_SEC,
+					    7, date);
+	if (err != 7)
+		return err < 0 ? err : -EIO;
+
+	dt->tm_sec = bcd2bin(date[RX8010_SEC - RX8010_SEC] & 0x7f);
+	dt->tm_min = bcd2bin(date[RX8010_MIN - RX8010_SEC] & 0x7f);
+	dt->tm_hour = bcd2bin(date[RX8010_HOUR - RX8010_SEC] & 0x3f);
+	dt->tm_mday = bcd2bin(date[RX8010_MDAY - RX8010_SEC] & 0x3f);
+	dt->tm_mon = bcd2bin(date[RX8010_MONTH - RX8010_SEC] & 0x1f) - 1;
+	dt->tm_year = bcd2bin(date[RX8010_YEAR - RX8010_SEC]) + 100;
+	dt->tm_wday = ffs(date[RX8010_WDAY - RX8010_SEC] & 0x7f);
+
+	return rtc_valid_tm(dt);
+}
+
+static int rx8010_set_time(struct device *dev, struct rtc_time *dt)
+{
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	u8 date[7];
+	int ctrl, flagreg;
+	int ret;
+	unsigned long irqflags;
+
+	if ((dt->tm_year < 100) || (dt->tm_year > 199))
+		return -EINVAL;
+
+	/* set STOP bit before changing clock/calendar */
+	ctrl = i2c_smbus_read_byte_data(rx8010->client, RX8010_CTRL);
+	if (ctrl < 0)
+		return ctrl;
+	rx8010->ctrlreg = ctrl | RX8010_CTRL_STOP;
+	ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL,
+					rx8010->ctrlreg);
+	if (ret < 0)
+		return ret;
+
+	date[RX8010_SEC - RX8010_SEC] = bin2bcd(dt->tm_sec);
+	date[RX8010_MIN - RX8010_SEC] = bin2bcd(dt->tm_min);
+	date[RX8010_HOUR - RX8010_SEC] = bin2bcd(dt->tm_hour);
+	date[RX8010_MDAY - RX8010_SEC] = bin2bcd(dt->tm_mday);
+	date[RX8010_MONTH - RX8010_SEC] = bin2bcd(dt->tm_mon + 1);
+	date[RX8010_YEAR - RX8010_SEC] = bin2bcd(dt->tm_year - 100);
+	date[RX8010_WDAY - RX8010_SEC] = bin2bcd(1 << dt->tm_wday);
+
+	ret = i2c_smbus_write_i2c_block_data(rx8010->client,
+					     RX8010_SEC, 7, date);
+	if (ret < 0)
+		return ret;
+
+	/* clear STOP bit after changing clock/calendar */
+	ctrl = i2c_smbus_read_byte_data(rx8010->client, RX8010_CTRL);
+	if (ctrl < 0)
+		return ctrl;
+	rx8010->ctrlreg = ctrl & ~RX8010_CTRL_STOP;
+	ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL,
+					rx8010->ctrlreg);
+	if (ret < 0)
+		return ret;
+
+	spin_lock_irqsave(&rx8010->flags_lock, irqflags);
+
+	flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG);
+	if (flagreg < 0) {
+		spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+		return flagreg;
+	}
+
+	if (flagreg & RX8010_FLAG_VLF)
+		ret = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG,
+						flagreg & ~RX8010_FLAG_VLF);
+
+	spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+
+	return 0;
+}
+
+static int rx8010_init_client(struct i2c_client *client)
+{
+	struct rx8010_data *rx8010 = i2c_get_clientdata(client);
+	u8 ctrl[2];
+	int need_clear = 0, err = 0;
+
+	/* Initialize reserved registers as specified in datasheet */
+	err = i2c_smbus_write_byte_data(client, RX8010_RESV17, 0xD8);
+	if (err < 0)
+		return err;
+
+	err = i2c_smbus_write_byte_data(client, RX8010_RESV30, 0x00);
+	if (err < 0)
+		return err;
+
+	err = i2c_smbus_write_byte_data(client, RX8010_RESV31, 0x08);
+	if (err < 0)
+		return err;
+
+	err = i2c_smbus_write_byte_data(client, RX8010_IRQ, 0x00);
+	if (err < 0)
+		return err;
+
+	err = i2c_smbus_read_i2c_block_data(rx8010->client, RX8010_FLAG,
+					    2, ctrl);
+	if (err != 2)
+		return err < 0 ? err : -EIO;
+
+	if (ctrl[0] & RX8010_FLAG_VLF)
+		dev_warn(&client->dev, "Frequency stop was detected\n");
+
+	if (ctrl[0] & RX8010_FLAG_AF) {
+		dev_warn(&client->dev, "Alarm was detected\n");
+		need_clear = 1;
+	}
+
+	if (ctrl[0] & RX8010_FLAG_TF)
+		need_clear = 1;
+
+	if (ctrl[0] & RX8010_FLAG_UF)
+		need_clear = 1;
+
+	if (need_clear) {
+		ctrl[0] &= ~(RX8010_FLAG_AF | RX8010_FLAG_TF | RX8010_FLAG_UF);
+		err = i2c_smbus_write_byte_data(client, RX8010_FLAG, ctrl[0]);
+		if (err < 0)
+			return err;
+	}
+
+	rx8010->ctrlreg = (ctrl[1] & ~RX8010_CTRL_TEST);
+
+	return err;
+}
+
+static int rx8010_read_alarm(struct device *dev, struct rtc_wkalrm *t)
+{
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	struct i2c_client *client = rx8010->client;
+	u8 alarmvals[3];
+	int flagreg;
+	int err;
+
+	err = i2c_smbus_read_i2c_block_data(client, RX8010_ALMIN, 3, alarmvals);
+	if (err != 3)
+		return err < 0 ? err : -EIO;
+
+	flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG);
+	if (flagreg < 0)
+		return flagreg;
+
+	t->time.tm_sec = 0;
+	t->time.tm_min = bcd2bin(alarmvals[0] & 0x7f);
+	t->time.tm_hour = bcd2bin(alarmvals[1] & 0x3f);
+
+	if (alarmvals[2] & RX8010_ALARM_AE)
+		t->time.tm_mday = -1;
+	else
+		t->time.tm_mday = bcd2bin(alarmvals[2] & 0x7f);
+
+	t->time.tm_wday = -1;
+	t->time.tm_mon = -1;
+	t->time.tm_year = -1;
+
+	t->enabled = !!(rx8010->ctrlreg & RX8010_CTRL_AIE);
+	t->pending = (flagreg & RX8010_FLAG_AF) && t->enabled;
+
+	return err;
+}
+
+static int rx8010_set_alarm(struct device *dev, struct rtc_wkalrm *t)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	u8 alarmvals[3];
+	int extreg, flagreg;
+	int err;
+	unsigned long irqflags;
+
+	spin_lock_irqsave(&rx8010->flags_lock, irqflags);
+	flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG);
+	if (flagreg < 0) {
+		spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+		return flagreg;
+	}
+
+	if (rx8010->ctrlreg & (RX8010_CTRL_AIE | RX8010_CTRL_UIE)) {
+		rx8010->ctrlreg &= ~(RX8010_CTRL_AIE | RX8010_CTRL_UIE);
+		err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL,
+						rx8010->ctrlreg);
+		if (err < 0) {
+			spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+			return err;
+		}
+	}
+
+	flagreg &= ~RX8010_FLAG_AF;
+	err = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG, flagreg);
+	spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+	if (err < 0)
+		return err;
+
+	alarmvals[0] = bin2bcd(t->time.tm_min);
+	alarmvals[1] = bin2bcd(t->time.tm_hour);
+	alarmvals[2] = bin2bcd(t->time.tm_mday);
+
+	err = i2c_smbus_write_i2c_block_data(rx8010->client, RX8010_ALMIN,
+					     2, alarmvals);
+	if (err < 0)
+		return err;
+
+	extreg = i2c_smbus_read_byte_data(client, RX8010_EXT);
+	if (extreg < 0)
+		return extreg;
+
+	extreg |= RX8010_EXT_WADA;
+	err = i2c_smbus_write_byte_data(rx8010->client, RX8010_EXT, extreg);
+	if (err < 0)
+		return err;
+
+	if (alarmvals[2] == 0)
+		alarmvals[2] |= RX8010_ALARM_AE;
+
+	err = i2c_smbus_write_byte_data(rx8010->client, RX8010_ALWDAY,
+					alarmvals[2]);
+	if (err < 0)
+		return err;
+
+	if (t->enabled) {
+		if (rx8010->rtc->uie_rtctimer.enabled)
+			rx8010->ctrlreg |= RX8010_CTRL_UIE;
+		if (rx8010->rtc->aie_timer.enabled)
+			rx8010->ctrlreg |=
+				(RX8010_CTRL_AIE | RX8010_CTRL_UIE);
+
+		err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL,
+						rx8010->ctrlreg);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+static int rx8010_alarm_irq_enable(struct device *dev,
+				   unsigned int enabled)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	int flagreg;
+	u8 ctrl;
+	int err;
+
+	ctrl = rx8010->ctrlreg;
+
+	if (enabled) {
+		if (rx8010->rtc->uie_rtctimer.enabled)
+			ctrl |= RX8010_CTRL_UIE;
+		if (rx8010->rtc->aie_timer.enabled)
+			ctrl |= (RX8010_CTRL_AIE | RX8010_CTRL_UIE);
+	} else {
+		if (!rx8010->rtc->uie_rtctimer.enabled)
+			ctrl &= ~RX8010_CTRL_UIE;
+		if (!rx8010->rtc->aie_timer.enabled)
+			ctrl &= ~RX8010_CTRL_AIE;
+	}
+
+	flagreg = i2c_smbus_read_byte_data(client, RX8010_FLAG);
+	if (flagreg < 0)
+		return flagreg;
+
+	flagreg &= ~RX8010_FLAG_AF;
+	err = i2c_smbus_write_byte_data(rx8010->client, RX8010_FLAG, flagreg);
+	if (err < 0)
+		return err;
+
+	if (ctrl != rx8010->ctrlreg) {
+		rx8010->ctrlreg = ctrl;
+		err = i2c_smbus_write_byte_data(rx8010->client, RX8010_CTRL,
+						rx8010->ctrlreg);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+static int rx8010_ioctl(struct device *dev, unsigned int cmd, unsigned long arg)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct rx8010_data *rx8010 = dev_get_drvdata(dev);
+	int ret, tmp;
+	int flagreg;
+	unsigned long irqflags;
+
+	switch (cmd) {
+	case RTC_VL_READ:
+		flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG);
+		if (flagreg < 0)
+			return flagreg;
+
+		tmp = !!(flagreg & RX8010_FLAG_VLF);
+		if (copy_to_user((void __user *)arg, &tmp, sizeof(int)))
+			return -EFAULT;
+
+		return 0;
+
+	case RTC_VL_CLR:
+		spin_lock_irqsave(&rx8010->flags_lock, irqflags);
+		flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG);
+		if (flagreg < 0) {
+			spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+			return flagreg;
+		}
+
+		flagreg &= ~RX8010_FLAG_VLF;
+		ret = i2c_smbus_write_byte_data(client, RX8010_FLAG, flagreg);
+		spin_unlock_irqrestore(&rx8010->flags_lock, irqflags);
+		if (ret < 0)
+			return ret;
+
+		return 0;
+
+	default:
+		return -ENOIOCTLCMD;
+	}
+}
+
+static struct rtc_class_ops rx8010_rtc_ops = {
+	.read_time = rx8010_get_time,
+	.set_time = rx8010_set_time,
+	.ioctl = rx8010_ioctl,
+};
+
+static int rx8010_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent);
+	struct rx8010_data *rx8010;
+	int err = 0;
+
+	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA
+		| I2C_FUNC_SMBUS_I2C_BLOCK)) {
+		dev_err(&adapter->dev, "doesn't support required functionality\n");
+		return -EIO;
+	}
+
+	rx8010 = devm_kzalloc(&client->dev, sizeof(struct rx8010_data),
+			      GFP_KERNEL);
+	if (!rx8010)
+		return -ENOMEM;
+
+	rx8010->client = client;
+	i2c_set_clientdata(client, rx8010);
+
+	spin_lock_init(&rx8010->flags_lock);
+
+	err = rx8010_init_client(client);
+	if (err)
+		return err;
+
+	if (client->irq > 0) {
+		dev_info(&client->dev, "IRQ %d supplied\n", client->irq);
+		err = devm_request_threaded_irq(&client->dev, client->irq, NULL,
+						rx8010_irq_1_handler,
+						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+						"rx8010", client);
+
+		if (err) {
+			dev_err(&client->dev, "unable to request IRQ\n");
+			client->irq = 0;
+		} else {
+			rx8010_rtc_ops.read_alarm = rx8010_read_alarm;
+			rx8010_rtc_ops.set_alarm = rx8010_set_alarm;
+			rx8010_rtc_ops.alarm_irq_enable = rx8010_alarm_irq_enable;
+		}
+	}
+
+	rx8010->rtc = devm_rtc_device_register(&client->dev, client->name,
+		&rx8010_rtc_ops, THIS_MODULE);
+
+	if (IS_ERR(rx8010->rtc)) {
+		dev_err(&client->dev, "unable to register the class device\n");
+		return PTR_ERR(rx8010->rtc);
+	}
+
+	rx8010->rtc->max_user_freq = 1;
+
+	return err;
+}
+
+static struct i2c_driver rx8010_driver = {
+	.driver = {
+		.name = "rtc-rx8010",
+	},
+	.probe		= rx8010_probe,
+	.id_table	= rx8010_id,
+};
+
+module_i2c_driver(rx8010_driver);
+
+MODULE_AUTHOR("Akshay Bhat <akshay.bhat@timesys.com>");
+MODULE_DESCRIPTION("Epson RX8010SJ RTC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rtc/rtc-s5m.c b/drivers/rtc/rtc-s5m.c
index 0d68a85dd429..7407d7394bb4 100644
--- a/drivers/rtc/rtc-s5m.c
+++ b/drivers/rtc/rtc-s5m.c
@@ -38,7 +38,22 @@
  */
 #define UDR_READ_RETRY_CNT	5
 
-/* Registers used by the driver which are different between chipsets. */
+/*
+ * Registers used by the driver which are different between chipsets.
+ *
+ * Operations like read time and write alarm/time require updating
+ * specific fields in UDR register. These fields usually are auto-cleared
+ * (with some exceptions).
+ *
+ * Table of operations per device:
+ *
+ * Device     | Write time | Read time | Write alarm
+ * =================================================
+ * S5M8767    | UDR + TIME |           | UDR
+ * S2MPS11/14 | WUDR       | RUDR      | WUDR + RUDR
+ * S2MPS13    | WUDR       | RUDR      | WUDR + AUDR
+ * S2MPS15    | WUDR       | RUDR      | AUDR
+ */
 struct s5m_rtc_reg_config {
 	/* Number of registers used for setting time/alarm0/alarm1 */
 	unsigned int regs_count;
@@ -55,9 +70,16 @@ struct s5m_rtc_reg_config {
 	 * will enable update of time or alarm register. Then it will be
 	 * auto-cleared after successful update.
 	 */
-	unsigned int rtc_udr_update;
-	/* Mask for UDR field in 'rtc_udr_update' register */
-	unsigned int rtc_udr_mask;
+	unsigned int udr_update;
+	/* Auto-cleared mask in UDR field for writing time and alarm */
+	unsigned int autoclear_udr_mask;
+	/*
+	 * Masks in UDR field for time and alarm operations.
+	 * The read time mask can be 0. Rest should not.
+	 */
+	unsigned int read_time_udr_mask;
+	unsigned int write_time_udr_mask;
+	unsigned int write_alarm_udr_mask;
 };
 
 /* Register map for S5M8763 and S5M8767 */
@@ -67,22 +89,56 @@ static const struct s5m_rtc_reg_config s5m_rtc_regs = {
 	.ctrl			= S5M_ALARM1_CONF,
 	.alarm0			= S5M_ALARM0_SEC,
 	.alarm1			= S5M_ALARM1_SEC,
-	.rtc_udr_update		= S5M_RTC_UDR_CON,
-	.rtc_udr_mask		= S5M_RTC_UDR_MASK,
+	.udr_update		= S5M_RTC_UDR_CON,
+	.autoclear_udr_mask	= S5M_RTC_UDR_MASK,
+	.read_time_udr_mask	= 0, /* Not needed */
+	.write_time_udr_mask	= S5M_RTC_UDR_MASK | S5M_RTC_TIME_EN_MASK,
+	.write_alarm_udr_mask	= S5M_RTC_UDR_MASK,
+};
+
+/* Register map for S2MPS13 */
+static const struct s5m_rtc_reg_config s2mps13_rtc_regs = {
+	.regs_count		= 7,
+	.time			= S2MPS_RTC_SEC,
+	.ctrl			= S2MPS_RTC_CTRL,
+	.alarm0			= S2MPS_ALARM0_SEC,
+	.alarm1			= S2MPS_ALARM1_SEC,
+	.udr_update		= S2MPS_RTC_UDR_CON,
+	.autoclear_udr_mask	= S2MPS_RTC_WUDR_MASK,
+	.read_time_udr_mask	= S2MPS_RTC_RUDR_MASK,
+	.write_time_udr_mask	= S2MPS_RTC_WUDR_MASK,
+	.write_alarm_udr_mask	= S2MPS_RTC_WUDR_MASK | S2MPS13_RTC_AUDR_MASK,
+};
+
+/* Register map for S2MPS11/14 */
+static const struct s5m_rtc_reg_config s2mps14_rtc_regs = {
+	.regs_count		= 7,
+	.time			= S2MPS_RTC_SEC,
+	.ctrl			= S2MPS_RTC_CTRL,
+	.alarm0			= S2MPS_ALARM0_SEC,
+	.alarm1			= S2MPS_ALARM1_SEC,
+	.udr_update		= S2MPS_RTC_UDR_CON,
+	.autoclear_udr_mask	= S2MPS_RTC_WUDR_MASK,
+	.read_time_udr_mask	= S2MPS_RTC_RUDR_MASK,
+	.write_time_udr_mask	= S2MPS_RTC_WUDR_MASK,
+	.write_alarm_udr_mask	= S2MPS_RTC_WUDR_MASK | S2MPS_RTC_RUDR_MASK,
 };
 
 /*
- * Register map for S2MPS14.
- * It may be also suitable for S2MPS11 but this was not tested.
+ * Register map for S2MPS15 - in comparison to S2MPS14 the WUDR and AUDR bits
+ * are swapped.
  */
-static const struct s5m_rtc_reg_config s2mps_rtc_regs = {
+static const struct s5m_rtc_reg_config s2mps15_rtc_regs = {
 	.regs_count		= 7,
 	.time			= S2MPS_RTC_SEC,
 	.ctrl			= S2MPS_RTC_CTRL,
 	.alarm0			= S2MPS_ALARM0_SEC,
 	.alarm1			= S2MPS_ALARM1_SEC,
-	.rtc_udr_update		= S2MPS_RTC_UDR_CON,
-	.rtc_udr_mask		= S2MPS_RTC_WUDR_MASK,
+	.udr_update		= S2MPS_RTC_UDR_CON,
+	.autoclear_udr_mask	= S2MPS_RTC_WUDR_MASK,
+	.read_time_udr_mask	= S2MPS_RTC_RUDR_MASK,
+	.write_time_udr_mask	= S2MPS15_RTC_WUDR_MASK,
+	.write_alarm_udr_mask	= S2MPS15_RTC_AUDR_MASK,
 };
 
 struct s5m_rtc_info {
@@ -166,9 +222,8 @@ static inline int s5m8767_wait_for_udr_update(struct s5m_rtc_info *info)
 	unsigned int data;
 
 	do {
-		ret = regmap_read(info->regmap, info->regs->rtc_udr_update,
-				&data);
-	} while (--retry && (data & info->regs->rtc_udr_mask) && !ret);
+		ret = regmap_read(info->regmap, info->regs->udr_update, &data);
+	} while (--retry && (data & info->regs->autoclear_udr_mask) && !ret);
 
 	if (!retry)
 		dev_err(info->dev, "waiting for UDR update, reached max number of retries\n");
@@ -214,30 +269,15 @@ static inline int s5m8767_rtc_set_time_reg(struct s5m_rtc_info *info)
 	int ret;
 	unsigned int data;
 
-	ret = regmap_read(info->regmap, info->regs->rtc_udr_update, &data);
+	ret = regmap_read(info->regmap, info->regs->udr_update, &data);
 	if (ret < 0) {
 		dev_err(info->dev, "failed to read update reg(%d)\n", ret);
 		return ret;
 	}
 
-	switch (info->device_type) {
-	case S5M8763X:
-	case S5M8767X:
-		data |= info->regs->rtc_udr_mask | S5M_RTC_TIME_EN_MASK;
-	case S2MPS15X:
-		/* As per UM, for write time register, set WUDR bit to high */
-		data |= S2MPS15_RTC_WUDR_MASK;
-		break;
-	case S2MPS14X:
-	case S2MPS13X:
-		data |= info->regs->rtc_udr_mask;
-		break;
-	default:
-		return -EINVAL;
-	}
+	data |= info->regs->write_time_udr_mask;
 
-
-	ret = regmap_write(info->regmap, info->regs->rtc_udr_update, data);
+	ret = regmap_write(info->regmap, info->regs->udr_update, data);
 	if (ret < 0) {
 		dev_err(info->dev, "failed to write update reg(%d)\n", ret);
 		return ret;
@@ -253,35 +293,29 @@ static inline int s5m8767_rtc_set_alarm_reg(struct s5m_rtc_info *info)
 	int ret;
 	unsigned int data;
 
-	ret = regmap_read(info->regmap, info->regs->rtc_udr_update, &data);
+	ret = regmap_read(info->regmap, info->regs->udr_update, &data);
 	if (ret < 0) {
 		dev_err(info->dev, "%s: fail to read update reg(%d)\n",
 			__func__, ret);
 		return ret;
 	}
 
-	data |= info->regs->rtc_udr_mask;
+	data |= info->regs->write_alarm_udr_mask;
 	switch (info->device_type) {
 	case S5M8763X:
 	case S5M8767X:
 		data &= ~S5M_RTC_TIME_EN_MASK;
 		break;
 	case S2MPS15X:
-		/* As per UM, for write alarm, set A_UDR(bit[4]) to high
-		 * rtc_udr_mask above sets bit[4]
-		 */
-		break;
 	case S2MPS14X:
-		data |= S2MPS_RTC_RUDR_MASK;
-		break;
 	case S2MPS13X:
-		data |= S2MPS13_RTC_AUDR_MASK;
+		/* No exceptions needed */
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	ret = regmap_write(info->regmap, info->regs->rtc_udr_update, data);
+	ret = regmap_write(info->regmap, info->regs->udr_update, data);
 	if (ret < 0) {
 		dev_err(info->dev, "%s: fail to write update reg(%d)\n",
 			__func__, ret);
@@ -292,7 +326,7 @@ static inline int s5m8767_rtc_set_alarm_reg(struct s5m_rtc_info *info)
 
 	/* On S2MPS13 the AUDR is not auto-cleared */
 	if (info->device_type == S2MPS13X)
-		regmap_update_bits(info->regmap, info->regs->rtc_udr_update,
+		regmap_update_bits(info->regmap, info->regs->udr_update,
 				   S2MPS13_RTC_AUDR_MASK, 0);
 
 	return ret;
@@ -336,11 +370,11 @@ static int s5m_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	u8 data[info->regs->regs_count];
 	int ret;
 
-	if (info->device_type == S2MPS15X || info->device_type == S2MPS14X ||
-			info->device_type == S2MPS13X) {
+	if (info->regs->read_time_udr_mask) {
 		ret = regmap_update_bits(info->regmap,
-				info->regs->rtc_udr_update,
-				S2MPS_RTC_RUDR_MASK, S2MPS_RTC_RUDR_MASK);
+				info->regs->udr_update,
+				info->regs->read_time_udr_mask,
+				info->regs->read_time_udr_mask);
 		if (ret) {
 			dev_err(dev,
 				"Failed to prepare registers for time reading: %d\n",
@@ -707,10 +741,18 @@ static int s5m_rtc_probe(struct platform_device *pdev)
 
 	switch (platform_get_device_id(pdev)->driver_data) {
 	case S2MPS15X:
+		regmap_cfg = &s2mps14_rtc_regmap_config;
+		info->regs = &s2mps15_rtc_regs;
+		alarm_irq = S2MPS14_IRQ_RTCA0;
+		break;
 	case S2MPS14X:
+		regmap_cfg = &s2mps14_rtc_regmap_config;
+		info->regs = &s2mps14_rtc_regs;
+		alarm_irq = S2MPS14_IRQ_RTCA0;
+		break;
 	case S2MPS13X:
 		regmap_cfg = &s2mps14_rtc_regmap_config;
-		info->regs = &s2mps_rtc_regs;
+		info->regs = &s2mps13_rtc_regs;
 		alarm_irq = S2MPS14_IRQ_RTCA0;
 		break;
 	case S5M8763X:
diff --git a/drivers/rtc/rtc-sunxi.c b/drivers/rtc/rtc-sunxi.c
index 52543ae37c98..abada609ddc7 100644
--- a/drivers/rtc/rtc-sunxi.c
+++ b/drivers/rtc/rtc-sunxi.c
@@ -133,7 +133,7 @@ struct sunxi_rtc_data_year {
 	unsigned char leap_shift;	/* bit shift to get the leap year */
 };
 
-static struct sunxi_rtc_data_year data_year_param[] = {
+static const struct sunxi_rtc_data_year data_year_param[] = {
 	[0] = {
 		.min		= 2010,
 		.max		= 2073,
@@ -151,7 +151,7 @@ static struct sunxi_rtc_data_year data_year_param[] = {
 struct sunxi_rtc_dev {
 	struct rtc_device *rtc;
 	struct device *dev;
-	struct sunxi_rtc_data_year *data_year;
+	const struct sunxi_rtc_data_year *data_year;
 	void __iomem *base;
 	int irq;
 };
@@ -175,7 +175,7 @@ static irqreturn_t sunxi_rtc_alarmirq(int irq, void *id)
 	return IRQ_NONE;
 }
 
-static void sunxi_rtc_setaie(int to, struct sunxi_rtc_dev *chip)
+static void sunxi_rtc_setaie(unsigned int to, struct sunxi_rtc_dev *chip)
 {
 	u32 alrm_val = 0;
 	u32 alrm_irq_val = 0;
@@ -343,7 +343,7 @@ static int sunxi_rtc_settime(struct device *dev, struct rtc_time *rtc_tm)
 	struct sunxi_rtc_dev *chip = dev_get_drvdata(dev);
 	u32 date = 0;
 	u32 time = 0;
-	int year;
+	unsigned int year;
 
 	/*
 	 * the input rtc_tm->tm_year is the offset relative to 1900. We use
@@ -353,8 +353,8 @@ static int sunxi_rtc_settime(struct device *dev, struct rtc_time *rtc_tm)
 
 	year = rtc_tm->tm_year + 1900;
 	if (year < chip->data_year->min || year > chip->data_year->max) {
-		dev_err(dev, "rtc only supports year in range %d - %d\n",
-				chip->data_year->min, chip->data_year->max);
+		dev_err(dev, "rtc only supports year in range %u - %u\n",
+			chip->data_year->min, chip->data_year->max);
 		return -EINVAL;
 	}
 
@@ -436,7 +436,6 @@ static int sunxi_rtc_probe(struct platform_device *pdev)
 {
 	struct sunxi_rtc_dev *chip;
 	struct resource *res;
-	const struct of_device_id *of_id;
 	int ret;
 
 	chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);
@@ -463,12 +462,11 @@ static int sunxi_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	of_id = of_match_device(sunxi_rtc_dt_ids, &pdev->dev);
-	if (!of_id) {
+	chip->data_year = of_device_get_match_data(&pdev->dev);
+	if (!chip->data_year) {
 		dev_err(&pdev->dev, "Unable to setup RTC data\n");
 		return -ENODEV;
 	}
-	chip->data_year = (struct sunxi_rtc_data_year *) of_id->data;
 
 	/* clear the alarm count value */
 	writel(0, chip->base + SUNXI_ALRM_DHMS);
diff --git a/drivers/rtc/rtc-sysfs.c b/drivers/rtc/rtc-sysfs.c
index 7273855ed02e..463e286064ab 100644
--- a/drivers/rtc/rtc-sysfs.c
+++ b/drivers/rtc/rtc-sysfs.c
@@ -91,7 +91,12 @@ max_user_freq_store(struct device *dev, struct device_attribute *attr,
 		const char *buf, size_t n)
 {
 	struct rtc_device *rtc = to_rtc_device(dev);
-	unsigned long val = simple_strtoul(buf, NULL, 0);
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 0, &val);
+	if (err)
+		return err;
 
 	if (val >= 4096 || val == 0)
 		return -EINVAL;
@@ -175,7 +180,9 @@ wakealarm_store(struct device *dev, struct device_attribute *attr,
 		} else
 			adjust = 1;
 	}
-	alarm = simple_strtoul(buf_ptr, NULL, 0);
+	retval = kstrtoul(buf_ptr, 0, &alarm);
+	if (retval)
+		return retval;
 	if (adjust) {
 		alarm += now;
 	}
diff --git a/drivers/rtc/rtc-v3020.c b/drivers/rtc/rtc-v3020.c
index f9f97098c254..7a0436329d6c 100644
--- a/drivers/rtc/rtc-v3020.c
+++ b/drivers/rtc/rtc-v3020.c
@@ -57,7 +57,7 @@ struct v3020 {
 	/* GPIO access */
 	struct gpio *gpio;
 
-	struct v3020_chip_ops *ops;
+	const struct v3020_chip_ops *ops;
 
 	struct rtc_device *rtc;
 };
@@ -95,7 +95,7 @@ static unsigned char v3020_mmio_read_bit(struct v3020 *chip)
 	return !!(readl(chip->ioaddress) & (1 << chip->leftshift));
 }
 
-static struct v3020_chip_ops v3020_mmio_ops = {
+static const struct v3020_chip_ops v3020_mmio_ops = {
 	.map_io		= v3020_mmio_map,
 	.unmap_io	= v3020_mmio_unmap,
 	.read_bit	= v3020_mmio_read_bit,
@@ -158,7 +158,7 @@ static unsigned char v3020_gpio_read_bit(struct v3020 *chip)
 	return bit;
 }
 
-static struct v3020_chip_ops v3020_gpio_ops = {
+static const struct v3020_chip_ops v3020_gpio_ops = {
 	.map_io		= v3020_gpio_map,
 	.unmap_io	= v3020_gpio_unmap,
 	.read_bit	= v3020_gpio_read_bit,
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 94a8f4ab57bc..ce7b70181740 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -17,6 +17,7 @@
 #include <linux/completion.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/pfn_t.h>
 #include <asm/extmem.h>
 #include <asm/io.h>
 
@@ -30,7 +31,7 @@ static void dcssblk_release(struct gendisk *disk, fmode_t mode);
 static blk_qc_t dcssblk_make_request(struct request_queue *q,
 						struct bio *bio);
 static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum,
-			 void __pmem **kaddr, unsigned long *pfn);
+			 void __pmem **kaddr, pfn_t *pfn);
 
 static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
 
@@ -883,20 +884,18 @@ fail:
 
 static long
 dcssblk_direct_access (struct block_device *bdev, sector_t secnum,
-			void __pmem **kaddr, unsigned long *pfn)
+			void __pmem **kaddr, pfn_t *pfn)
 {
 	struct dcssblk_dev_info *dev_info;
 	unsigned long offset, dev_sz;
-	void *addr;
 
 	dev_info = bdev->bd_disk->private_data;
 	if (!dev_info)
 		return -ENODEV;
 	dev_sz = dev_info->end - dev_info->start;
 	offset = secnum * 512;
-	addr = (void *) (dev_info->start + offset);
-	*pfn = virt_to_phys(addr) >> PAGE_SHIFT;
-	*kaddr = (void __pmem *) addr;
+	*kaddr = (void __pmem *) (dev_info->start + offset);
+	*pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset), PFN_DEV);
 
 	return dev_sz - offset;
 }
diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c
index c692dfebd0ba..50597f9522fe 100644
--- a/drivers/s390/cio/chp.c
+++ b/drivers/s390/cio/chp.c
@@ -139,11 +139,11 @@ static ssize_t chp_measurement_chars_read(struct file *filp,
 
 	device = container_of(kobj, struct device, kobj);
 	chp = to_channelpath(device);
-	if (!chp->cmg_chars)
+	if (chp->cmg == -1)
 		return 0;
 
-	return memory_read_from_buffer(buf, count, &off,
-				chp->cmg_chars, sizeof(struct cmg_chars));
+	return memory_read_from_buffer(buf, count, &off, &chp->cmg_chars,
+				       sizeof(chp->cmg_chars));
 }
 
 static struct bin_attribute chp_measurement_chars_attr = {
@@ -416,7 +416,8 @@ static void chp_release(struct device *dev)
  * chp_update_desc - update channel-path description
  * @chp - channel-path
  *
- * Update the channel-path description of the specified channel-path.
+ * Update the channel-path description of the specified channel-path
+ * including channel measurement related information.
  * Return zero on success, non-zero otherwise.
  */
 int chp_update_desc(struct channel_path *chp)
@@ -428,8 +429,10 @@ int chp_update_desc(struct channel_path *chp)
 		return rc;
 
 	rc = chsc_determine_fmt1_channel_path_desc(chp->chpid, &chp->desc_fmt1);
+	if (rc)
+		return rc;
 
-	return rc;
+	return chsc_get_channel_measurement_chars(chp);
 }
 
 /**
@@ -466,14 +469,6 @@ int chp_new(struct chp_id chpid)
 		ret = -ENODEV;
 		goto out_free;
 	}
-	/* Get channel-measurement characteristics. */
-	if (css_chsc_characteristics.scmc && css_chsc_characteristics.secm) {
-		ret = chsc_get_channel_measurement_chars(chp);
-		if (ret)
-			goto out_free;
-	} else {
-		chp->cmg = -1;
-	}
 	dev_set_name(&chp->dev, "chp%x.%02x", chpid.cssid, chpid.id);
 
 	/* make it known to the system */
diff --git a/drivers/s390/cio/chp.h b/drivers/s390/cio/chp.h
index 4efd5b867cc3..af0232290dc4 100644
--- a/drivers/s390/cio/chp.h
+++ b/drivers/s390/cio/chp.h
@@ -48,7 +48,7 @@ struct channel_path {
 	/* Channel-measurement related stuff: */
 	int cmg;
 	int shared;
-	void *cmg_chars;
+	struct cmg_chars cmg_chars;
 };
 
 /* Return channel_path struct for given chpid. */
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index a831d18596a5..c424c0c7367e 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -14,6 +14,7 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/device.h>
+#include <linux/mutex.h>
 #include <linux/pci.h>
 
 #include <asm/cio.h>
@@ -224,8 +225,9 @@ out_unreg:
 
 void chsc_chp_offline(struct chp_id chpid)
 {
-	char dbf_txt[15];
+	struct channel_path *chp = chpid_to_chp(chpid);
 	struct chp_link link;
+	char dbf_txt[15];
 
 	sprintf(dbf_txt, "chpr%x.%02x", chpid.cssid, chpid.id);
 	CIO_TRACE_EVENT(2, dbf_txt);
@@ -236,6 +238,11 @@ void chsc_chp_offline(struct chp_id chpid)
 	link.chpid = chpid;
 	/* Wait until previous actions have settled. */
 	css_wait_for_slow_path();
+
+	mutex_lock(&chp->lock);
+	chp_update_desc(chp);
+	mutex_unlock(&chp->lock);
+
 	for_each_subchannel_staged(s390_subchannel_remove_chpid, NULL, &link);
 }
 
@@ -690,8 +697,9 @@ static void chsc_process_crw(struct crw *crw0, struct crw *crw1, int overflow)
 
 void chsc_chp_online(struct chp_id chpid)
 {
-	char dbf_txt[15];
+	struct channel_path *chp = chpid_to_chp(chpid);
 	struct chp_link link;
+	char dbf_txt[15];
 
 	sprintf(dbf_txt, "cadd%x.%02x", chpid.cssid, chpid.id);
 	CIO_TRACE_EVENT(2, dbf_txt);
@@ -701,6 +709,11 @@ void chsc_chp_online(struct chp_id chpid)
 		link.chpid = chpid;
 		/* Wait until previous actions have settled. */
 		css_wait_for_slow_path();
+
+		mutex_lock(&chp->lock);
+		chp_update_desc(chp);
+		mutex_unlock(&chp->lock);
+
 		for_each_subchannel_staged(__s390_process_res_acc, NULL,
 					   &link);
 		css_schedule_reprobe();
@@ -967,22 +980,19 @@ static void
 chsc_initialize_cmg_chars(struct channel_path *chp, u8 cmcv,
 			  struct cmg_chars *chars)
 {
-	struct cmg_chars *cmg_chars;
 	int i, mask;
 
-	cmg_chars = chp->cmg_chars;
 	for (i = 0; i < NR_MEASUREMENT_CHARS; i++) {
 		mask = 0x80 >> (i + 3);
 		if (cmcv & mask)
-			cmg_chars->values[i] = chars->values[i];
+			chp->cmg_chars.values[i] = chars->values[i];
 		else
-			cmg_chars->values[i] = 0;
+			chp->cmg_chars.values[i] = 0;
 	}
 }
 
 int chsc_get_channel_measurement_chars(struct channel_path *chp)
 {
-	struct cmg_chars *cmg_chars;
 	int ccode, ret;
 
 	struct {
@@ -1006,10 +1016,11 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp)
 		u32 data[NR_MEASUREMENT_CHARS];
 	} __attribute__ ((packed)) *scmc_area;
 
-	chp->cmg_chars = NULL;
-	cmg_chars = kmalloc(sizeof(*cmg_chars), GFP_KERNEL);
-	if (!cmg_chars)
-		return -ENOMEM;
+	chp->shared = -1;
+	chp->cmg = -1;
+
+	if (!css_chsc_characteristics.scmc || !css_chsc_characteristics.secm)
+		return 0;
 
 	spin_lock_irq(&chsc_page_lock);
 	memset(chsc_page, 0, PAGE_SIZE);
@@ -1031,25 +1042,19 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp)
 			      scmc_area->response.code);
 		goto out;
 	}
-	if (scmc_area->not_valid) {
-		chp->cmg = -1;
-		chp->shared = -1;
+	if (scmc_area->not_valid)
 		goto out;
-	}
+
 	chp->cmg = scmc_area->cmg;
 	chp->shared = scmc_area->shared;
 	if (chp->cmg != 2 && chp->cmg != 3) {
 		/* No cmg-dependent data. */
 		goto out;
 	}
-	chp->cmg_chars = cmg_chars;
 	chsc_initialize_cmg_chars(chp, scmc_area->cmcv,
 				  (struct cmg_chars *) &scmc_area->data);
 out:
 	spin_unlock_irq(&chsc_page_lock);
-	if (!chp->cmg_chars)
-		kfree(cmg_chars);
-
 	return ret;
 }
 
diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h
index 7b23f43c7b08..de1b6c1d172c 100644
--- a/drivers/s390/crypto/zcrypt_error.h
+++ b/drivers/s390/crypto/zcrypt_error.h
@@ -112,9 +112,10 @@ static inline int convert_error(struct zcrypt_device *zdev,
 		atomic_set(&zcrypt_rescan_req, 1);
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			zdev->ap_dev->qid, zdev->online, ehdr->reply_code);
+			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
+			ehdr->reply_code);
 		return -EAGAIN;
 	case REP82_ERROR_TRANSPORT_FAIL:
 	case REP82_ERROR_MACHINE_FAILURE:
@@ -123,16 +124,18 @@ static inline int convert_error(struct zcrypt_device *zdev,
 		atomic_set(&zcrypt_rescan_req, 1);
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			zdev->ap_dev->qid, zdev->online, ehdr->reply_code);
+			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
+			ehdr->reply_code);
 		return -EAGAIN;
 	default:
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			zdev->ap_dev->qid, zdev->online, ehdr->reply_code);
+			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
+			ehdr->reply_code);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c
index 74edf2934e7c..eedfaa2cf715 100644
--- a/drivers/s390/crypto/zcrypt_msgtype50.c
+++ b/drivers/s390/crypto/zcrypt_msgtype50.c
@@ -336,9 +336,10 @@ static int convert_type80(struct zcrypt_device *zdev,
 		/* The result is too short, the CEX2A card may not do that.. */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			       zdev->ap_dev->qid, zdev->online, t80h->code);
+			       AP_QID_DEVICE(zdev->ap_dev->qid),
+			       zdev->online, t80h->code);
 
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
@@ -368,9 +369,9 @@ static int convert_response(struct zcrypt_device *zdev,
 	default: /* Unknown response type, this should NEVER EVER happen */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       zdev->ap_dev->qid, zdev->online);
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
index 9a2dd472c1cc..21959719daef 100644
--- a/drivers/s390/crypto/zcrypt_msgtype6.c
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -572,9 +572,9 @@ static int convert_type86_ica(struct zcrypt_device *zdev,
 			return -EINVAL;
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			       zdev->ap_dev->qid, zdev->online,
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
 			       msg->hdr.reply_code);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
@@ -715,9 +715,9 @@ static int convert_response_ica(struct zcrypt_device *zdev,
 	default: /* Unknown response type, this should NEVER EVER happen */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       zdev->ap_dev->qid, zdev->online);
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
@@ -747,9 +747,9 @@ static int convert_response_xcrb(struct zcrypt_device *zdev,
 		xcRB->status = 0x0008044DL; /* HDD_InvalidParm */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       zdev->ap_dev->qid, zdev->online);
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
@@ -773,9 +773,9 @@ static int convert_response_ep11_xcrb(struct zcrypt_device *zdev,
 	default: /* Unknown response type, this should NEVER EVER happen */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       zdev->ap_dev->qid, zdev->online);
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
 		return -EAGAIN; /* repeat the request on a different device. */
 	}
 }
@@ -800,9 +800,9 @@ static int convert_response_rng(struct zcrypt_device *zdev,
 	default: /* Unknown response type, this should NEVER EVER happen */
 		zdev->online = 0;
 		pr_err("Cryptographic device %x failed and was set offline\n",
-		       zdev->ap_dev->qid);
+		       AP_QID_DEVICE(zdev->ap_dev->qid));
 		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       zdev->ap_dev->qid, zdev->online);
+			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
index 53fb975c404b..1d060fd293a3 100644
--- a/drivers/s390/virtio/kvm_virtio.c
+++ b/drivers/s390/virtio/kvm_virtio.c
@@ -255,7 +255,7 @@ static void kvm_del_vqs(struct virtio_device *vdev)
 static int kvm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			struct virtqueue *vqs[],
 			vq_callback_t *callbacks[],
-			const char *names[])
+			const char * const names[])
 {
 	struct kvm_device *kdev = to_kvmdev(vdev);
 	int i;
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 1b831598df7c..bf2d1300a957 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -635,7 +635,7 @@ out:
 static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			       struct virtqueue *vqs[],
 			       vq_callback_t *callbacks[],
-			       const char *names[])
+			       const char * const names[])
 {
 	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
 	unsigned long *indicatorp = NULL;
diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c
index 2940bd769936..25aba1613e21 100644
--- a/drivers/scsi/3w-xxxx.c
+++ b/drivers/scsi/3w-xxxx.c
@@ -1045,6 +1045,9 @@ static int tw_chrdev_open(struct inode *inode, struct file *file)
 static const struct file_operations tw_fops = {
 	.owner		= THIS_MODULE,
 	.unlocked_ioctl	= tw_chrdev_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl   = tw_chrdev_ioctl,
+#endif
 	.open		= tw_chrdev_open,
 	.release	= NULL,
 	.llseek		= noop_llseek,
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index c1fe0d2f90ca..e2f31c93717d 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1106,6 +1106,7 @@ config SCSI_IPR
 	tristate "IBM Power Linux RAID adapter support"
 	depends on PCI && SCSI && ATA
 	select FW_LOADER
+	select IRQ_POLL
 	---help---
 	  This driver supports the IBM Power Linux family RAID adapters.
 	  This includes IBM pSeries 5712, 5703, 5709, and 570A, as well
@@ -1620,23 +1621,6 @@ config ATARI_SCSI
 	  ST-DMA, replacing ACSI).  It does NOT support other schemes, like
 	  in the Hades (without DMA).
 
-config ATARI_SCSI_TOSHIBA_DELAY
-	bool "Long delays for Toshiba CD-ROMs"
-	depends on ATARI_SCSI
-	help
-	  This option increases the delay after a SCSI arbitration to
-	  accommodate some flaky Toshiba CD-ROM drives. Say Y if you intend to
-	  use a Toshiba CD-ROM drive; otherwise, the option is not needed and
-	  would impact performance a bit, so say N.
-
-config ATARI_SCSI_RESET_BOOT
-	bool "Reset SCSI-devices at boottime"
-	depends on ATARI_SCSI
-	help
-	  Reset the devices on your Atari whenever it boots.  This makes the
-	  boot process fractionally longer but may assist recovery from errors
-	  that leave the devices with SCSI operations partway completed.
-
 config MAC_SCSI
 	tristate "Macintosh NCR5380 SCSI"
 	depends on MAC && SCSI=y
diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c
index a777e5c412df..d72867257346 100644
--- a/drivers/scsi/NCR5380.c
+++ b/drivers/scsi/NCR5380.c
@@ -1,17 +1,17 @@
-/* 
+/*
  * NCR 5380 generic driver routines.  These should make it *trivial*
- *      to implement 5380 SCSI drivers under Linux with a non-trantor
- *      architecture.
+ * to implement 5380 SCSI drivers under Linux with a non-trantor
+ * architecture.
  *
- *      Note that these routines also work with NR53c400 family chips.
+ * Note that these routines also work with NR53c400 family chips.
  *
  * Copyright 1993, Drew Eckhardt
- *      Visionary Computing 
- *      (Unix and Linux consulting and custom programming)
- *      drew@colorado.edu
- *      +1 (303) 666-5836
+ * Visionary Computing
+ * (Unix and Linux consulting and custom programming)
+ * drew@colorado.edu
+ * +1 (303) 666-5836
  *
- * For more information, please consult 
+ * For more information, please consult
  *
  * NCR 5380 Family
  * SCSI Protocol Controller
@@ -25,84 +25,28 @@
  */
 
 /*
- * Revision 1.10 1998/9/2	Alan Cox
- *				(alan@lxorguk.ukuu.org.uk)
- * Fixed up the timer lockups reported so far. Things still suck. Looking 
- * forward to 2.3 and per device request queues. Then it'll be possible to
- * SMP thread this beast and improve life no end.
- 
- * Revision 1.9  1997/7/27	Ronald van Cuijlenborg
- *				(ronald.van.cuijlenborg@tip.nl or nutty@dds.nl)
- * (hopefully) fixed and enhanced USLEEP
- * added support for DTC3181E card (for Mustek scanner)
- *
-
- * Revision 1.8			Ingmar Baumgart
- *				(ingmar@gonzo.schwaben.de)
- * added support for NCR53C400a card
- *
-
- * Revision 1.7  1996/3/2       Ray Van Tassle (rayvt@comm.mot.com)
- * added proc_info
- * added support needed for DTC 3180/3280
- * fixed a couple of bugs
- *
-
- * Revision 1.5  1994/01/19  09:14:57  drew
- * Fixed udelay() hack that was being used on DATAOUT phases
- * instead of a proper wait for the final handshake.
- *
- * Revision 1.4  1994/01/19  06:44:25  drew
- * *** empty log message ***
- *
- * Revision 1.3  1994/01/19  05:24:40  drew
- * Added support for TCR LAST_BYTE_SENT bit.
- *
- * Revision 1.2  1994/01/15  06:14:11  drew
- * REAL DMA support, bug fixes.
- *
- * Revision 1.1  1994/01/15  06:00:54  drew
- * Initial revision
- *
+ * With contributions from Ray Van Tassle, Ingmar Baumgart,
+ * Ronald van Cuijlenborg, Alan Cox and others.
  */
 
 /*
- * Further development / testing that should be done : 
+ * Further development / testing that should be done :
  * 1.  Cleanup the NCR5380_transfer_dma function and DMA operation complete
- *     code so that everything does the same thing that's done at the 
- *     end of a pseudo-DMA read operation.
+ * code so that everything does the same thing that's done at the
+ * end of a pseudo-DMA read operation.
  *
  * 2.  Fix REAL_DMA (interrupt driven, polled works fine) -
- *     basically, transfer size needs to be reduced by one 
- *     and the last byte read as is done with PSEUDO_DMA.
- * 
- * 4.  Test SCSI-II tagged queueing (I have no devices which support 
- *      tagged queueing)
- *
- * 5.  Test linked command handling code after Eric is ready with 
- *      the high level code.
+ * basically, transfer size needs to be reduced by one
+ * and the last byte read as is done with PSEUDO_DMA.
+ *
+ * 4.  Test SCSI-II tagged queueing (I have no devices which support
+ * tagged queueing)
  */
-#include <scsi/scsi_dbg.h>
-#include <scsi/scsi_transport_spi.h>
-
-#if (NDEBUG & NDEBUG_LISTS)
-#define LIST(x,y) {printk("LINE:%d   Adding %p to %p\n", __LINE__, (void*)(x), (void*)(y)); if ((x)==(y)) udelay(5); }
-#define REMOVE(w,x,y,z) {printk("LINE:%d   Removing: %p->%p  %p->%p \n", __LINE__, (void*)(w), (void*)(x), (void*)(y), (void*)(z)); if ((x)==(y)) udelay(5); }
-#else
-#define LIST(x,y)
-#define REMOVE(w,x,y,z)
-#endif
 
 #ifndef notyet
-#undef LINKED
 #undef REAL_DMA
 #endif
 
-#ifdef REAL_DMA_POLL
-#undef READ_OVERRUNS
-#define READ_OVERRUNS
-#endif
-
 #ifdef BOARD_REQUIRES_NO_DELAY
 #define io_recovery_delay(x)
 #else
@@ -112,44 +56,28 @@
 /*
  * Design
  *
- * This is a generic 5380 driver.  To use it on a different platform, 
+ * This is a generic 5380 driver.  To use it on a different platform,
  * one simply writes appropriate system specific macros (ie, data
- * transfer - some PC's will use the I/O bus, 68K's must use 
+ * transfer - some PC's will use the I/O bus, 68K's must use
  * memory mapped) and drops this file in their 'C' wrapper.
  *
- * (Note from hch:  unfortunately it was not enough for the different
- * m68k folks and instead of improving this driver they copied it
- * and hacked it up for their needs.  As a consequence they lost
- * most updates to this driver.  Maybe someone will fix all these
- * drivers to use a common core one day..)
- *
- * As far as command queueing, two queues are maintained for 
+ * As far as command queueing, two queues are maintained for
  * each 5380 in the system - commands that haven't been issued yet,
- * and commands that are currently executing.  This means that an 
- * unlimited number of commands may be queued, letting 
- * more commands propagate from the higher driver levels giving higher 
- * throughput.  Note that both I_T_L and I_T_L_Q nexuses are supported, 
- * allowing multiple commands to propagate all the way to a SCSI-II device 
+ * and commands that are currently executing.  This means that an
+ * unlimited number of commands may be queued, letting
+ * more commands propagate from the higher driver levels giving higher
+ * throughput.  Note that both I_T_L and I_T_L_Q nexuses are supported,
+ * allowing multiple commands to propagate all the way to a SCSI-II device
  * while a command is already executing.
  *
  *
- * Issues specific to the NCR5380 : 
+ * Issues specific to the NCR5380 :
  *
- * When used in a PIO or pseudo-dma mode, the NCR5380 is a braindead 
- * piece of hardware that requires you to sit in a loop polling for 
- * the REQ signal as long as you are connected.  Some devices are 
- * brain dead (ie, many TEXEL CD ROM drives) and won't disconnect 
- * while doing long seek operations.
- * 
- * The workaround for this is to keep track of devices that have
- * disconnected.  If the device hasn't disconnected, for commands that
- * should disconnect, we do something like 
- *
- * while (!REQ is asserted) { sleep for N usecs; poll for M usecs }
- * 
- * Some tweaking of N and M needs to be done.  An algorithm based 
- * on "time to data" would give the best results as long as short time
- * to datas (ie, on the same track) were considered, however these 
+ * When used in a PIO or pseudo-dma mode, the NCR5380 is a braindead
+ * piece of hardware that requires you to sit in a loop polling for
+ * the REQ signal as long as you are connected.  Some devices are
+ * brain dead (ie, many TEXEL CD ROM drives) and won't disconnect
+ * while doing long seek operations. [...] These
  * broken devices are the exception rather than the rule and I'd rather
  * spend my time optimizing for the normal case.
  *
@@ -159,23 +87,23 @@
  * which is started from a workqueue for each NCR5380 host in the
  * system.  It attempts to establish I_T_L or I_T_L_Q nexuses by
  * removing the commands from the issue queue and calling
- * NCR5380_select() if a nexus is not established. 
+ * NCR5380_select() if a nexus is not established.
  *
  * Once a nexus is established, the NCR5380_information_transfer()
  * phase goes through the various phases as instructed by the target.
  * if the target goes into MSG IN and sends a DISCONNECT message,
  * the command structure is placed into the per instance disconnected
- * queue, and NCR5380_main tries to find more work.  If the target is 
+ * queue, and NCR5380_main tries to find more work.  If the target is
  * idle for too long, the system will try to sleep.
  *
  * If a command has disconnected, eventually an interrupt will trigger,
  * calling NCR5380_intr()  which will in turn call NCR5380_reselect
  * to reestablish a nexus.  This will run main if necessary.
  *
- * On command termination, the done function will be called as 
+ * On command termination, the done function will be called as
  * appropriate.
  *
- * SCSI pointers are maintained in the SCp field of SCSI command 
+ * SCSI pointers are maintained in the SCp field of SCSI command
  * structures, being initialized after the command is connected
  * in NCR5380_select, and set as appropriate in NCR5380_information_transfer.
  * Note that in violation of the standard, an implicit SAVE POINTERS operation
@@ -185,73 +113,48 @@
 /*
  * Using this file :
  * This file a skeleton Linux SCSI driver for the NCR 5380 series
- * of chips.  To use it, you write an architecture specific functions 
+ * of chips.  To use it, you write an architecture specific functions
  * and macros and include this file in your driver.
  *
- * These macros control options : 
- * AUTOPROBE_IRQ - if defined, the NCR5380_probe_irq() function will be 
- *      defined.
- * 
+ * These macros control options :
+ * AUTOPROBE_IRQ - if defined, the NCR5380_probe_irq() function will be
+ * defined.
+ *
  * AUTOSENSE - if defined, REQUEST SENSE will be performed automatically
- *      for commands that return with a CHECK CONDITION status. 
+ * for commands that return with a CHECK CONDITION status.
  *
  * DIFFERENTIAL - if defined, NCR53c81 chips will use external differential
- *      transceivers. 
+ * transceivers.
  *
  * DONT_USE_INTR - if defined, never use interrupts, even if we probe or
- *      override-configure an IRQ.
- *
- * LIMIT_TRANSFERSIZE - if defined, limit the pseudo-dma transfers to 512
- *      bytes at a time.  Since interrupts are disabled by default during
- *      these transfers, we might need this to give reasonable interrupt
- *      service time if the transfer size gets too large.
- *
- * LINKED - if defined, linked commands are supported.
+ * override-configure an IRQ.
  *
  * PSEUDO_DMA - if defined, PSEUDO DMA is used during the data transfer phases.
  *
  * REAL_DMA - if defined, REAL DMA is used during the data transfer phases.
  *
  * REAL_DMA_POLL - if defined, REAL DMA is used but the driver doesn't
- *      rely on phase mismatch and EOP interrupts to determine end 
- *      of phase.
- *
- * UNSAFE - leave interrupts enabled during pseudo-DMA transfers.  You
- *          only really want to use this if you're having a problem with
- *          dropped characters during high speed communications, and even
- *          then, you're going to be better off twiddling with transfersize
- *          in the high level code.
- *
- * Defaults for these will be provided although the user may want to adjust 
- * these to allocate CPU resources to the SCSI driver or "real" code.
- * 
- * USLEEP_SLEEP - amount of time, in jiffies, to sleep
- *
- * USLEEP_POLL - amount of time, in jiffies, to poll
+ * rely on phase mismatch and EOP interrupts to determine end
+ * of phase.
  *
  * These macros MUST be defined :
- * NCR5380_local_declare() - declare any local variables needed for your
- *      transfer routines.
  *
- * NCR5380_setup(instance) - initialize any local variables needed from a given
- *      instance of the host adapter for NCR5380_{read,write,pread,pwrite}
- * 
  * NCR5380_read(register)  - read from the specified register
  *
- * NCR5380_write(register, value) - write to the specific register 
+ * NCR5380_write(register, value) - write to the specific register
  *
- * NCR5380_implementation_fields  - additional fields needed for this 
- *      specific implementation of the NCR5380
+ * NCR5380_implementation_fields  - additional fields needed for this
+ * specific implementation of the NCR5380
  *
  * Either real DMA *or* pseudo DMA may be implemented
- * REAL functions : 
+ * REAL functions :
  * NCR5380_REAL_DMA should be defined if real DMA is to be used.
- * Note that the DMA setup functions should return the number of bytes 
- *      that they were able to program the controller for.
+ * Note that the DMA setup functions should return the number of bytes
+ * that they were able to program the controller for.
  *
- * Also note that generic i386/PC versions of these macros are 
- *      available as NCR5380_i386_dma_write_setup,
- *      NCR5380_i386_dma_read_setup, and NCR5380_i386_dma_residual.
+ * Also note that generic i386/PC versions of these macros are
+ * available as NCR5380_i386_dma_write_setup,
+ * NCR5380_i386_dma_read_setup, and NCR5380_i386_dma_residual.
  *
  * NCR5380_dma_write_setup(instance, src, count) - initialize
  * NCR5380_dma_read_setup(instance, dst, count) - initialize
@@ -262,25 +165,25 @@
  * NCR5380_pread(instance, dst, count);
  *
  * The generic driver is initialized by calling NCR5380_init(instance),
- * after setting the appropriate host specific fields and ID.  If the 
+ * after setting the appropriate host specific fields and ID.  If the
  * driver wishes to autoprobe for an IRQ line, the NCR5380_probe_irq(instance,
  * possible) function may be used.
  */
 
-static int do_abort(struct Scsi_Host *host);
-static void do_reset(struct Scsi_Host *host);
+static int do_abort(struct Scsi_Host *);
+static void do_reset(struct Scsi_Host *);
 
-/*
- *	initialize_SCp		-	init the scsi pointer field
- *	@cmd: command block to set up
+/**
+ * initialize_SCp - init the scsi pointer field
+ * @cmd: command block to set up
  *
- *	Set up the internal fields in the SCSI command.
+ * Set up the internal fields in the SCSI command.
  */
 
 static inline void initialize_SCp(struct scsi_cmnd *cmd)
 {
-	/* 
-	 * Initialize the Scsi Pointer field so that all of the commands in the 
+	/*
+	 * Initialize the Scsi Pointer field so that all of the commands in the
 	 * various queues are valid.
 	 */
 
@@ -295,120 +198,123 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd)
 		cmd->SCp.ptr = NULL;
 		cmd->SCp.this_residual = 0;
 	}
+
+	cmd->SCp.Status = 0;
+	cmd->SCp.Message = 0;
 }
 
 /**
- *	NCR5380_poll_politely	-	wait for NCR5380 status bits
- *	@instance: controller to poll
- *	@reg: 5380 register to poll
- *	@bit: Bitmask to check
- *	@val: Value required to exit
- *
- *	Polls the NCR5380 in a reasonably efficient manner waiting for
- *	an event to occur, after a short quick poll we begin giving the
- *	CPU back in non IRQ contexts
- *
- *	Returns the value of the register or a negative error code.
+ * NCR5380_poll_politely2 - wait for two chip register values
+ * @instance: controller to poll
+ * @reg1: 5380 register to poll
+ * @bit1: Bitmask to check
+ * @val1: Expected value
+ * @reg2: Second 5380 register to poll
+ * @bit2: Second bitmask to check
+ * @val2: Second expected value
+ * @wait: Time-out in jiffies
+ *
+ * Polls the chip in a reasonably efficient manner waiting for an
+ * event to occur. After a short quick poll we begin to yield the CPU
+ * (if possible). In irq contexts the time-out is arbitrarily limited.
+ * Callers may hold locks as long as they are held in irq mode.
+ *
+ * Returns 0 if either or both event(s) occurred otherwise -ETIMEDOUT.
  */
- 
-static int NCR5380_poll_politely(struct Scsi_Host *instance, int reg, int bit, int val, int t)
+
+static int NCR5380_poll_politely2(struct Scsi_Host *instance,
+                                  int reg1, int bit1, int val1,
+                                  int reg2, int bit2, int val2, int wait)
 {
-	NCR5380_local_declare();
-	int n = 500;		/* At about 8uS a cycle for the cpu access */
-	unsigned long end = jiffies + t;
-	int r;
-	
-	NCR5380_setup(instance);
-
-	while( n-- > 0)
-	{
-		r = NCR5380_read(reg);
-		if((r & bit) == val)
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	unsigned long deadline = jiffies + wait;
+	unsigned long n;
+
+	/* Busy-wait for up to 10 ms */
+	n = min(10000U, jiffies_to_usecs(wait));
+	n *= hostdata->accesses_per_ms;
+	n /= 2000;
+	do {
+		if ((NCR5380_read(reg1) & bit1) == val1)
+			return 0;
+		if ((NCR5380_read(reg2) & bit2) == val2)
 			return 0;
 		cpu_relax();
-	}
-	
-	/* t time yet ? */
-	while(time_before(jiffies, end))
-	{
-		r = NCR5380_read(reg);
-		if((r & bit) == val)
+	} while (n--);
+
+	if (irqs_disabled() || in_interrupt())
+		return -ETIMEDOUT;
+
+	/* Repeatedly sleep for 1 ms until deadline */
+	while (time_is_after_jiffies(deadline)) {
+		schedule_timeout_uninterruptible(1);
+		if ((NCR5380_read(reg1) & bit1) == val1)
+			return 0;
+		if ((NCR5380_read(reg2) & bit2) == val2)
 			return 0;
-		if(!in_interrupt())
-			cond_resched();
-		else
-			cpu_relax();
 	}
+
 	return -ETIMEDOUT;
 }
 
-static struct {
-	unsigned char value;
-	const char *name;
-} phases[] __maybe_unused = {
-	{PHASE_DATAOUT, "DATAOUT"}, 
-	{PHASE_DATAIN, "DATAIN"}, 
-	{PHASE_CMDOUT, "CMDOUT"}, 
-	{PHASE_STATIN, "STATIN"}, 
-	{PHASE_MSGOUT, "MSGOUT"}, 
-	{PHASE_MSGIN, "MSGIN"}, 
-	{PHASE_UNKNOWN, "UNKNOWN"}
-};
+static inline int NCR5380_poll_politely(struct Scsi_Host *instance,
+                                        int reg, int bit, int val, int wait)
+{
+	return NCR5380_poll_politely2(instance, reg, bit, val,
+	                                        reg, bit, val, wait);
+}
 
 #if NDEBUG
 static struct {
 	unsigned char mask;
 	const char *name;
-} signals[] = { 
-	{SR_DBP, "PARITY"}, 
-	{SR_RST, "RST"}, 
-	{SR_BSY, "BSY"}, 
-	{SR_REQ, "REQ"}, 
-	{SR_MSG, "MSG"}, 
-	{SR_CD, "CD"}, 
-	{SR_IO, "IO"}, 
-	{SR_SEL, "SEL"}, 
+} signals[] = {
+	{SR_DBP, "PARITY"},
+	{SR_RST, "RST"},
+	{SR_BSY, "BSY"},
+	{SR_REQ, "REQ"},
+	{SR_MSG, "MSG"},
+	{SR_CD, "CD"},
+	{SR_IO, "IO"},
+	{SR_SEL, "SEL"},
 	{0, NULL}
-}, 
+},
 basrs[] = {
-	{BASR_ATN, "ATN"}, 
-	{BASR_ACK, "ACK"}, 
+	{BASR_ATN, "ATN"},
+	{BASR_ACK, "ACK"},
 	{0, NULL}
-}, 
-icrs[] = { 
-	{ICR_ASSERT_RST, "ASSERT RST"}, 
-	{ICR_ASSERT_ACK, "ASSERT ACK"}, 
-	{ICR_ASSERT_BSY, "ASSERT BSY"}, 
-	{ICR_ASSERT_SEL, "ASSERT SEL"}, 
-	{ICR_ASSERT_ATN, "ASSERT ATN"}, 
-	{ICR_ASSERT_DATA, "ASSERT DATA"}, 
+},
+icrs[] = {
+	{ICR_ASSERT_RST, "ASSERT RST"},
+	{ICR_ASSERT_ACK, "ASSERT ACK"},
+	{ICR_ASSERT_BSY, "ASSERT BSY"},
+	{ICR_ASSERT_SEL, "ASSERT SEL"},
+	{ICR_ASSERT_ATN, "ASSERT ATN"},
+	{ICR_ASSERT_DATA, "ASSERT DATA"},
 	{0, NULL}
-}, 
-mrs[] = { 
-	{MR_BLOCK_DMA_MODE, "MODE BLOCK DMA"}, 
-	{MR_TARGET, "MODE TARGET"}, 
-	{MR_ENABLE_PAR_CHECK, "MODE PARITY CHECK"}, 
-	{MR_ENABLE_PAR_INTR, "MODE PARITY INTR"}, 
-	{MR_MONITOR_BSY, "MODE MONITOR BSY"}, 
-	{MR_DMA_MODE, "MODE DMA"}, 
-	{MR_ARBITRATE, "MODE ARBITRATION"}, 
+},
+mrs[] = {
+	{MR_BLOCK_DMA_MODE, "MODE BLOCK DMA"},
+	{MR_TARGET, "MODE TARGET"},
+	{MR_ENABLE_PAR_CHECK, "MODE PARITY CHECK"},
+	{MR_ENABLE_PAR_INTR, "MODE PARITY INTR"},
+	{MR_ENABLE_EOP_INTR, "MODE EOP INTR"},
+	{MR_MONITOR_BSY, "MODE MONITOR BSY"},
+	{MR_DMA_MODE, "MODE DMA"},
+	{MR_ARBITRATE, "MODE ARBITRATION"},
 	{0, NULL}
 };
 
 /**
- *	NCR5380_print	-	print scsi bus signals
- *	@instance:	adapter state to dump
- *
- *	Print the SCSI bus signals for debugging purposes
+ * NCR5380_print - print scsi bus signals
+ * @instance: adapter state to dump
  *
- *	Locks: caller holds hostdata lock (not essential)
+ * Print the SCSI bus signals for debugging purposes
  */
 
 static void NCR5380_print(struct Scsi_Host *instance)
 {
-	NCR5380_local_declare();
 	unsigned char status, data, basr, mr, icr, i;
-	NCR5380_setup(instance);
 
 	data = NCR5380_read(CURRENT_SCSI_DATA_REG);
 	status = NCR5380_read(STATUS_REG);
@@ -435,117 +341,56 @@ static void NCR5380_print(struct Scsi_Host *instance)
 	printk("\n");
 }
 
+static struct {
+	unsigned char value;
+	const char *name;
+} phases[] = {
+	{PHASE_DATAOUT, "DATAOUT"},
+	{PHASE_DATAIN, "DATAIN"},
+	{PHASE_CMDOUT, "CMDOUT"},
+	{PHASE_STATIN, "STATIN"},
+	{PHASE_MSGOUT, "MSGOUT"},
+	{PHASE_MSGIN, "MSGIN"},
+	{PHASE_UNKNOWN, "UNKNOWN"}
+};
 
-/* 
- *	NCR5380_print_phase	-	show SCSI phase
- *	@instance: adapter to dump
- *
- * 	Print the current SCSI phase for debugging purposes
+/**
+ * NCR5380_print_phase - show SCSI phase
+ * @instance: adapter to dump
  *
- *	Locks: none
+ * Print the current SCSI phase for debugging purposes
  */
 
 static void NCR5380_print_phase(struct Scsi_Host *instance)
 {
-	NCR5380_local_declare();
 	unsigned char status;
 	int i;
-	NCR5380_setup(instance);
 
 	status = NCR5380_read(STATUS_REG);
 	if (!(status & SR_REQ))
-		printk("scsi%d : REQ not asserted, phase unknown.\n", instance->host_no);
+		shost_printk(KERN_DEBUG, instance, "REQ not asserted, phase unknown.\n");
 	else {
-		for (i = 0; (phases[i].value != PHASE_UNKNOWN) && (phases[i].value != (status & PHASE_MASK)); ++i);
-		printk("scsi%d : phase %s\n", instance->host_no, phases[i].name);
+		for (i = 0; (phases[i].value != PHASE_UNKNOWN) &&
+		     (phases[i].value != (status & PHASE_MASK)); ++i)
+			;
+		shost_printk(KERN_DEBUG, instance, "phase %s\n", phases[i].name);
 	}
 }
 #endif
 
-/*
- * These need tweaking, and would probably work best as per-device 
- * flags initialized differently for disk, tape, cd, etc devices.
- * People with broken devices are free to experiment as to what gives
- * the best results for them.
- *
- * USLEEP_SLEEP should be a minimum seek time.
- *
- * USLEEP_POLL should be a maximum rotational latency.
- */
-#ifndef USLEEP_SLEEP
-/* 20 ms (reasonable hard disk speed) */
-#define USLEEP_SLEEP msecs_to_jiffies(20)
-#endif
-/* 300 RPM (floppy speed) */
-#ifndef USLEEP_POLL
-#define USLEEP_POLL msecs_to_jiffies(200)
-#endif
-#ifndef USLEEP_WAITLONG
-/* RvC: (reasonable time to wait on select error) */
-#define USLEEP_WAITLONG USLEEP_SLEEP
-#endif
 
-/* 
- * Function : int should_disconnect (unsigned char cmd)
- *
- * Purpose : decide whether a command would normally disconnect or 
- *      not, since if it won't disconnect we should go to sleep.
- *
- * Input : cmd - opcode of SCSI command
- *
- * Returns : DISCONNECT_LONG if we should disconnect for a really long 
- *      time (ie always, sleep, look for REQ active, sleep), 
- *      DISCONNECT_TIME_TO_DATA if we would only disconnect for a normal
- *      time-to-data delay, DISCONNECT_NONE if this command would return
- *      immediately.
- *
- *      Future sleep algorithms based on time to data can exploit 
- *      something like this so they can differentiate between "normal" 
- *      (ie, read, write, seek) and unusual commands (ie, * format).
- *
- * Note : We don't deal with commands that handle an immediate disconnect,
- *        
- */
-
-static int should_disconnect(unsigned char cmd)
-{
-	switch (cmd) {
-	case READ_6:
-	case WRITE_6:
-	case SEEK_6:
-	case READ_10:
-	case WRITE_10:
-	case SEEK_10:
-		return DISCONNECT_TIME_TO_DATA;
-	case FORMAT_UNIT:
-	case SEARCH_HIGH:
-	case SEARCH_LOW:
-	case SEARCH_EQUAL:
-		return DISCONNECT_LONG;
-	default:
-		return DISCONNECT_NONE;
-	}
-}
-
-static void NCR5380_set_timer(struct NCR5380_hostdata *hostdata, unsigned long timeout)
-{
-	hostdata->time_expires = jiffies + timeout;
-	schedule_delayed_work(&hostdata->coroutine, timeout);
-}
-
-
-static int probe_irq __initdata = 0;
+static int probe_irq __initdata;
 
 /**
- *	probe_intr	-	helper for IRQ autoprobe
- *	@irq: interrupt number
- *	@dev_id: unused
- *	@regs: unused
+ * probe_intr	-	helper for IRQ autoprobe
+ * @irq: interrupt number
+ * @dev_id: unused
+ * @regs: unused
  *
- *	Set a flag to indicate the IRQ in question was received. This is
- *	used by the IRQ probe code.
+ * Set a flag to indicate the IRQ in question was received. This is
+ * used by the IRQ probe code.
  */
- 
+
 static irqreturn_t __init probe_intr(int irq, void *dev_id)
 {
 	probe_irq = irq;
@@ -553,24 +398,20 @@ static irqreturn_t __init probe_intr(int irq, void *dev_id)
 }
 
 /**
- *	NCR5380_probe_irq	-	find the IRQ of an NCR5380
- *	@instance: NCR5380 controller
- *	@possible: bitmask of ISA IRQ lines
+ * NCR5380_probe_irq	-	find the IRQ of an NCR5380
+ * @instance: NCR5380 controller
+ * @possible: bitmask of ISA IRQ lines
  *
- *	Autoprobe for the IRQ line used by the NCR5380 by triggering an IRQ
- *	and then looking to see what interrupt actually turned up.
- *
- *	Locks: none, irqs must be enabled on entry
+ * Autoprobe for the IRQ line used by the NCR5380 by triggering an IRQ
+ * and then looking to see what interrupt actually turned up.
  */
 
 static int __init __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance,
 						int possible)
 {
-	NCR5380_local_declare();
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned long timeout;
 	int trying_irqs, i, mask;
-	NCR5380_setup(instance);
 
 	for (trying_irqs = 0, i = 1, mask = 2; i < 16; ++i, mask <<= 1)
 		if ((mask & possible) && (request_irq(i, &probe_intr, 0, "NCR-probe", NULL) == 0))
@@ -581,7 +422,7 @@ static int __init __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance,
 
 	/*
 	 * A interrupt is triggered whenever BSY = false, SEL = true
-	 * and a bit set in the SELECT_ENABLE_REG is asserted on the 
+	 * and a bit set in the SELECT_ENABLE_REG is asserted on the
 	 * SCSI bus.
 	 *
 	 * Note that the bus is only driven when the phase control signals
@@ -596,7 +437,7 @@ static int __init __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance,
 
 	while (probe_irq == NO_IRQ && time_before(jiffies, timeout))
 		schedule_timeout_uninterruptible(1);
-	
+
 	NCR5380_write(SELECT_ENABLE_REG, 0);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 
@@ -608,12 +449,10 @@ static int __init __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance,
 }
 
 /**
- *	NCR58380_info - report driver and host information
- *	@instance: relevant scsi host instance
- *
- *	For use as the host template info() handler.
+ * NCR58380_info - report driver and host information
+ * @instance: relevant scsi host instance
  *
- *	Locks: none
+ * For use as the host template info() handler.
  */
 
 static const char *NCR5380_info(struct Scsi_Host *instance)
@@ -633,20 +472,14 @@ static void prepare_info(struct Scsi_Host *instance)
 	         "can_queue %d, cmd_per_lun %d, "
 	         "sg_tablesize %d, this_id %d, "
 	         "flags { %s%s%s}, "
-#if defined(USLEEP_POLL) && defined(USLEEP_WAITLONG)
-		 "USLEEP_POLL %lu, USLEEP_WAITLONG %lu, "
-#endif
 	         "options { %s} ",
 	         instance->hostt->name, instance->io_port, instance->n_io_port,
 	         instance->base, instance->irq,
 	         instance->can_queue, instance->cmd_per_lun,
 	         instance->sg_tablesize, instance->this_id,
-	         hostdata->flags & FLAG_NCR53C400     ? "NCR53C400 "     : "",
-	         hostdata->flags & FLAG_DTC3181E      ? "DTC3181E "      : "",
+	         hostdata->flags & FLAG_NO_DMA_FIXUP  ? "NO_DMA_FIXUP "  : "",
 	         hostdata->flags & FLAG_NO_PSEUDO_DMA ? "NO_PSEUDO_DMA " : "",
-#if defined(USLEEP_POLL) && defined(USLEEP_WAITLONG)
-	         USLEEP_POLL, USLEEP_WAITLONG,
-#endif
+	         hostdata->flags & FLAG_TOSHIBA_DELAY ? "TOSHIBA_DELAY "  : "",
 #ifdef AUTOPROBE_IRQ
 	         "AUTOPROBE_IRQ "
 #endif
@@ -665,46 +498,10 @@ static void prepare_info(struct Scsi_Host *instance)
 #ifdef PSEUDO_DMA
 	         "PSEUDO_DMA "
 #endif
-#ifdef UNSAFE
-	         "UNSAFE "
-#endif
-#ifdef NCR53C400
-	         "NCR53C400 "
-#endif
 	         "");
 }
 
-/**
- *	NCR5380_print_status 	-	dump controller info
- *	@instance: controller to dump
- *
- *	Print commands in the various queues, called from NCR5380_abort 
- *	and NCR5380_debug to aid debugging.
- *
- *	Locks: called functions disable irqs
- */
-
-static void NCR5380_print_status(struct Scsi_Host *instance)
-{
-	NCR5380_dprint(NDEBUG_ANY, instance);
-	NCR5380_dprint_phase(NDEBUG_ANY, instance);
-}
-
 #ifdef PSEUDO_DMA
-/******************************************/
-/*
- * /proc/scsi/[dtc pas16 t128 generic]/[0-ASC_NUM_BOARD_SUPPORTED]
- *
- * *buffer: I/O buffer
- * **start: if inout == FALSE pointer into buffer where user read should start
- * offset: current offset
- * length: length of buffer
- * hostno: Scsi_Host host_no
- * inout: TRUE - user is writing; FALSE - user is reading
- *
- * Return the number of bytes read from or written
- */
-
 static int __maybe_unused NCR5380_write_info(struct Scsi_Host *instance,
 	char *buffer, int length)
 {
@@ -714,104 +511,41 @@ static int __maybe_unused NCR5380_write_info(struct Scsi_Host *instance,
 	hostdata->spin_max_w = 0;
 	return 0;
 }
-#endif
-
-static
-void lprint_Scsi_Cmnd(struct scsi_cmnd *cmd, struct seq_file *m);
-static
-void lprint_command(unsigned char *cmd, struct seq_file *m);
-static
-void lprint_opcode(int opcode, struct seq_file *m);
 
 static int __maybe_unused NCR5380_show_info(struct seq_file *m,
-	struct Scsi_Host *instance)
+                                            struct Scsi_Host *instance)
 {
-	struct NCR5380_hostdata *hostdata;
-	struct scsi_cmnd *ptr;
-
-	hostdata = (struct NCR5380_hostdata *) instance->hostdata;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
-#ifdef PSEUDO_DMA
 	seq_printf(m, "Highwater I/O busy spin counts: write %d, read %d\n",
 	        hostdata->spin_max_w, hostdata->spin_max_r);
-#endif
-	spin_lock_irq(instance->host_lock);
-	if (!hostdata->connected)
-		seq_printf(m, "scsi%d: no currently connected command\n", instance->host_no);
-	else
-		lprint_Scsi_Cmnd((struct scsi_cmnd *) hostdata->connected, m);
-	seq_printf(m, "scsi%d: issue_queue\n", instance->host_no);
-	for (ptr = (struct scsi_cmnd *) hostdata->issue_queue; ptr; ptr = (struct scsi_cmnd *) ptr->host_scribble)
-		lprint_Scsi_Cmnd(ptr, m);
-
-	seq_printf(m, "scsi%d: disconnected_queue\n", instance->host_no);
-	for (ptr = (struct scsi_cmnd *) hostdata->disconnected_queue; ptr; ptr = (struct scsi_cmnd *) ptr->host_scribble)
-		lprint_Scsi_Cmnd(ptr, m);
-	spin_unlock_irq(instance->host_lock);
 	return 0;
 }
-
-static void lprint_Scsi_Cmnd(struct scsi_cmnd *cmd, struct seq_file *m)
-{
-	seq_printf(m, "scsi%d : destination target %d, lun %llu\n", cmd->device->host->host_no, cmd->device->id, cmd->device->lun);
-	seq_puts(m, "        command = ");
-	lprint_command(cmd->cmnd, m);
-}
-
-static void lprint_command(unsigned char *command, struct seq_file *m)
-{
-	int i, s;
-	lprint_opcode(command[0], m);
-	for (i = 1, s = COMMAND_SIZE(command[0]); i < s; ++i)
-		seq_printf(m, "%02x ", command[i]);
-	seq_putc(m, '\n');
-}
-
-static void lprint_opcode(int opcode, struct seq_file *m)
-{
-	seq_printf(m, "%2d (0x%02x)", opcode, opcode);
-}
-
+#endif
 
 /**
- *	NCR5380_init	-	initialise an NCR5380
- *	@instance: adapter to configure
- *	@flags: control flags
+ * NCR5380_init - initialise an NCR5380
+ * @instance: adapter to configure
+ * @flags: control flags
  *
- *	Initializes *instance and corresponding 5380 chip,
- *      with flags OR'd into the initial flags value.
+ * Initializes *instance and corresponding 5380 chip,
+ * with flags OR'd into the initial flags value.
  *
- *	Notes : I assume that the host, hostno, and id bits have been
- *      set correctly.  I don't care about the irq and other fields. 
+ * Notes : I assume that the host, hostno, and id bits have been
+ * set correctly. I don't care about the irq and other fields.
  *
- *	Returns 0 for success
- *
- *	Locks: interrupts must be enabled when we are called 
+ * Returns 0 for success
  */
 
 static int NCR5380_init(struct Scsi_Host *instance, int flags)
 {
-	NCR5380_local_declare();
-	int i, pass;
-	unsigned long timeout;
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-
-	if(in_interrupt())
-		printk(KERN_ERR "NCR5380_init called with interrupts off!\n");
-	/* 
-	 * On NCR53C400 boards, NCR5380 registers are mapped 8 past 
-	 * the base address.
-	 */
-
-#ifdef NCR53C400
-	if (flags & FLAG_NCR53C400)
-		instance->NCR5380_instance_name += NCR53C400_address_adjust;
-#endif
-
-	NCR5380_setup(instance);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int i;
+	unsigned long deadline;
 
-	hostdata->aborted = 0;
+	hostdata->host = instance;
 	hostdata->id_mask = 1 << instance->this_id;
+	hostdata->id_higher_mask = 0;
 	for (i = hostdata->id_mask; i <= 0x80; i <<= 1)
 		if (i > hostdata->id_mask)
 			hostdata->id_higher_mask |= i;
@@ -820,21 +554,21 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags)
 #ifdef REAL_DMA
 	hostdata->dmalen = 0;
 #endif
-	hostdata->targets_present = 0;
+	spin_lock_init(&hostdata->lock);
 	hostdata->connected = NULL;
-	hostdata->issue_queue = NULL;
-	hostdata->disconnected_queue = NULL;
-	
-	INIT_DELAYED_WORK(&hostdata->coroutine, NCR5380_main);
-	
-	/* The CHECK code seems to break the 53C400. Will check it later maybe */
-	if (flags & FLAG_NCR53C400)
-		hostdata->flags = FLAG_HAS_LAST_BYTE_SENT | flags;
-	else
-		hostdata->flags = FLAG_CHECK_LAST_BYTE_SENT | flags;
+	hostdata->sensing = NULL;
+	INIT_LIST_HEAD(&hostdata->autosense);
+	INIT_LIST_HEAD(&hostdata->unissued);
+	INIT_LIST_HEAD(&hostdata->disconnected);
 
-	hostdata->host = instance;
-	hostdata->time_expires = 0;
+	hostdata->flags = flags;
+
+	INIT_WORK(&hostdata->main_task, NCR5380_main);
+	hostdata->work_q = alloc_workqueue("ncr5380_%d",
+	                        WQ_UNBOUND | WQ_MEM_RECLAIM,
+	                        1, instance->host_no);
+	if (!hostdata->work_q)
+		return -ENOMEM;
 
 	prepare_info(instance);
 
@@ -843,43 +577,69 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags)
 	NCR5380_write(TARGET_COMMAND_REG, 0);
 	NCR5380_write(SELECT_ENABLE_REG, 0);
 
-#ifdef NCR53C400
-	if (hostdata->flags & FLAG_NCR53C400) {
-		NCR5380_write(C400_CONTROL_STATUS_REG, CSR_BASE);
-	}
-#endif
+	/* Calibrate register polling loop */
+	i = 0;
+	deadline = jiffies + 1;
+	do {
+		cpu_relax();
+	} while (time_is_after_jiffies(deadline));
+	deadline += msecs_to_jiffies(256);
+	do {
+		NCR5380_read(STATUS_REG);
+		++i;
+		cpu_relax();
+	} while (time_is_after_jiffies(deadline));
+	hostdata->accesses_per_ms = i / 256;
 
-	/* 
-	 * Detect and correct bus wedge problems.
-	 *
-	 * If the system crashed, it may have crashed in a state 
-	 * where a SCSI command was still executing, and the 
-	 * SCSI bus is not in a BUS FREE STATE.
-	 *
-	 * If this is the case, we'll try to abort the currently
-	 * established nexus which we know nothing about, and that
-	 * failing, do a hard reset of the SCSI bus 
-	 */
+	return 0;
+}
+
+/**
+ * NCR5380_maybe_reset_bus - Detect and correct bus wedge problems.
+ * @instance: adapter to check
+ *
+ * If the system crashed, it may have crashed with a connected target and
+ * the SCSI bus busy. Check for BUS FREE phase. If not, try to abort the
+ * currently established nexus, which we know nothing about. Failing that
+ * do a bus reset.
+ *
+ * Note that a bus reset will cause the chip to assert IRQ.
+ *
+ * Returns 0 if successful, otherwise -ENXIO.
+ */
+
+static int NCR5380_maybe_reset_bus(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int pass;
 
 	for (pass = 1; (NCR5380_read(STATUS_REG) & SR_BSY) && pass <= 6; ++pass) {
 		switch (pass) {
 		case 1:
 		case 3:
 		case 5:
-			printk(KERN_INFO "scsi%d: SCSI bus busy, waiting up to five seconds\n", instance->host_no);
-			timeout = jiffies + 5 * HZ;
-			NCR5380_poll_politely(instance, STATUS_REG, SR_BSY, 0, 5*HZ);
+			shost_printk(KERN_ERR, instance, "SCSI bus busy, waiting up to five seconds\n");
+			NCR5380_poll_politely(instance,
+			                      STATUS_REG, SR_BSY, 0, 5 * HZ);
 			break;
 		case 2:
-			printk(KERN_WARNING "scsi%d: bus busy, attempting abort\n", instance->host_no);
+			shost_printk(KERN_ERR, instance, "bus busy, attempting abort\n");
 			do_abort(instance);
 			break;
 		case 4:
-			printk(KERN_WARNING "scsi%d: bus busy, attempting reset\n", instance->host_no);
+			shost_printk(KERN_ERR, instance, "bus busy, attempting reset\n");
 			do_reset(instance);
+			/* Wait after a reset; the SCSI standard calls for
+			 * 250ms, we wait 500ms to be on the safe side.
+			 * But some Toshiba CD-ROMs need ten times that.
+			 */
+			if (hostdata->flags & FLAG_TOSHIBA_DELAY)
+				msleep(2500);
+			else
+				msleep(500);
 			break;
 		case 6:
-			printk(KERN_ERR "scsi%d: bus locked solid or invalid override\n", instance->host_no);
+			shost_printk(KERN_ERR, instance, "bus locked solid\n");
 			return -ENXIO;
 		}
 	}
@@ -887,450 +647,513 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags)
 }
 
 /**
- *	NCR5380_exit	-	remove an NCR5380
- *	@instance: adapter to remove
+ * NCR5380_exit - remove an NCR5380
+ * @instance: adapter to remove
+ *
+ * Assumes that no more work can be queued (e.g. by NCR5380_intr).
  */
 
 static void NCR5380_exit(struct Scsi_Host *instance)
 {
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
-	cancel_delayed_work_sync(&hostdata->coroutine);
+	cancel_work_sync(&hostdata->main_task);
+	destroy_workqueue(hostdata->work_q);
 }
 
 /**
- *	NCR5380_queue_command 		-	queue a command
- *	@cmd: SCSI command
- *	@done: completion handler
- *
- *      cmd is added to the per instance issue_queue, with minor 
- *      twiddling done to the host specific fields of cmd.  If the 
- *      main coroutine is not running, it is restarted.
+ * complete_cmd - finish processing a command and return it to the SCSI ML
+ * @instance: the host instance
+ * @cmd: command to complete
+ */
+
+static void complete_cmd(struct Scsi_Host *instance,
+                         struct scsi_cmnd *cmd)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+
+	dsprintk(NDEBUG_QUEUES, instance, "complete_cmd: cmd %p\n", cmd);
+
+	if (hostdata->sensing == cmd) {
+		/* Autosense processing ends here */
+		if ((cmd->result & 0xff) != SAM_STAT_GOOD) {
+			scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+			set_host_byte(cmd, DID_ERROR);
+		} else
+			scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+		hostdata->sensing = NULL;
+	}
+
+	hostdata->busy[scmd_id(cmd)] &= ~(1 << cmd->device->lun);
+
+	cmd->scsi_done(cmd);
+}
+
+/**
+ * NCR5380_queue_command - queue a command
+ * @instance: the relevant SCSI adapter
+ * @cmd: SCSI command
  *
- *	Locks: host lock taken by caller
+ * cmd is added to the per-instance issue queue, with minor
+ * twiddling done to the host specific fields of cmd.  If the
+ * main coroutine is not running, it is restarted.
  */
 
-static int NCR5380_queue_command_lck(struct scsi_cmnd *cmd, void (*done) (struct scsi_cmnd *))
+static int NCR5380_queue_command(struct Scsi_Host *instance,
+                                 struct scsi_cmnd *cmd)
 {
-	struct Scsi_Host *instance = cmd->device->host;
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-	struct scsi_cmnd *tmp;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
+	unsigned long flags;
 
 #if (NDEBUG & NDEBUG_NO_WRITE)
 	switch (cmd->cmnd[0]) {
 	case WRITE_6:
 	case WRITE_10:
-		printk("scsi%d : WRITE attempted with NO_WRITE debugging flag set\n", instance->host_no);
+		shost_printk(KERN_DEBUG, instance, "WRITE attempted with NDEBUG_NO_WRITE set\n");
 		cmd->result = (DID_ERROR << 16);
-		done(cmd);
+		cmd->scsi_done(cmd);
 		return 0;
 	}
-#endif				/* (NDEBUG & NDEBUG_NO_WRITE) */
-
-	/* 
-	 * We use the host_scribble field as a pointer to the next command  
-	 * in a queue 
-	 */
+#endif /* (NDEBUG & NDEBUG_NO_WRITE) */
 
-	cmd->host_scribble = NULL;
-	cmd->scsi_done = done;
 	cmd->result = 0;
 
-	/* 
-	 * Insert the cmd into the issue queue. Note that REQUEST SENSE 
+	spin_lock_irqsave(&hostdata->lock, flags);
+
+	/*
+	 * Insert the cmd into the issue queue. Note that REQUEST SENSE
 	 * commands are added to the head of the queue since any command will
-	 * clear the contingent allegiance condition that exists and the 
+	 * clear the contingent allegiance condition that exists and the
 	 * sense data is only guaranteed to be valid while the condition exists.
 	 */
 
-	if (!(hostdata->issue_queue) || (cmd->cmnd[0] == REQUEST_SENSE)) {
-		LIST(cmd, hostdata->issue_queue);
-		cmd->host_scribble = (unsigned char *) hostdata->issue_queue;
-		hostdata->issue_queue = cmd;
-	} else {
-		for (tmp = (struct scsi_cmnd *) hostdata->issue_queue; tmp->host_scribble; tmp = (struct scsi_cmnd *) tmp->host_scribble);
-		LIST(cmd, tmp);
-		tmp->host_scribble = (unsigned char *) cmd;
-	}
-	dprintk(NDEBUG_QUEUES, "scsi%d : command added to %s of queue\n", instance->host_no, (cmd->cmnd[0] == REQUEST_SENSE) ? "head" : "tail");
+	if (cmd->cmnd[0] == REQUEST_SENSE)
+		list_add(&ncmd->list, &hostdata->unissued);
+	else
+		list_add_tail(&ncmd->list, &hostdata->unissued);
+
+	spin_unlock_irqrestore(&hostdata->lock, flags);
+
+	dsprintk(NDEBUG_QUEUES, instance, "command %p added to %s of queue\n",
+	         cmd, (cmd->cmnd[0] == REQUEST_SENSE) ? "head" : "tail");
 
-	/* Run the coroutine if it isn't already running. */
 	/* Kick off command processing */
-	schedule_delayed_work(&hostdata->coroutine, 0);
+	queue_work(hostdata->work_q, &hostdata->main_task);
 	return 0;
 }
 
-static DEF_SCSI_QCMD(NCR5380_queue_command)
+/**
+ * dequeue_next_cmd - dequeue a command for processing
+ * @instance: the scsi host instance
+ *
+ * Priority is given to commands on the autosense queue. These commands
+ * need autosense because of a CHECK CONDITION result.
+ *
+ * Returns a command pointer if a command is found for a target that is
+ * not already busy. Otherwise returns NULL.
+ */
+
+static struct scsi_cmnd *dequeue_next_cmd(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	struct NCR5380_cmd *ncmd;
+	struct scsi_cmnd *cmd;
+
+	if (list_empty(&hostdata->autosense)) {
+		list_for_each_entry(ncmd, &hostdata->unissued, list) {
+			cmd = NCR5380_to_scmd(ncmd);
+			dsprintk(NDEBUG_QUEUES, instance, "dequeue: cmd=%p target=%d busy=0x%02x lun=%llu\n",
+			         cmd, scmd_id(cmd), hostdata->busy[scmd_id(cmd)], cmd->device->lun);
+
+			if (!(hostdata->busy[scmd_id(cmd)] & (1 << cmd->device->lun))) {
+				list_del(&ncmd->list);
+				dsprintk(NDEBUG_QUEUES, instance,
+				         "dequeue: removed %p from issue queue\n", cmd);
+				return cmd;
+			}
+		}
+	} else {
+		/* Autosense processing begins here */
+		ncmd = list_first_entry(&hostdata->autosense,
+		                        struct NCR5380_cmd, list);
+		list_del(&ncmd->list);
+		cmd = NCR5380_to_scmd(ncmd);
+		dsprintk(NDEBUG_QUEUES, instance,
+		         "dequeue: removed %p from autosense queue\n", cmd);
+		scsi_eh_prep_cmnd(cmd, &hostdata->ses, NULL, 0, ~0);
+		hostdata->sensing = cmd;
+		return cmd;
+	}
+	return NULL;
+}
+
+static void requeue_cmd(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
+
+	if (hostdata->sensing) {
+		scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+		list_add(&ncmd->list, &hostdata->autosense);
+		hostdata->sensing = NULL;
+	} else
+		list_add(&ncmd->list, &hostdata->unissued);
+}
 
 /**
- *	NCR5380_main	-	NCR state machines
- *
- *	NCR5380_main is a coroutine that runs as long as more work can 
- *      be done on the NCR5380 host adapters in a system.  Both 
- *      NCR5380_queue_command() and NCR5380_intr() will try to start it 
- *      in case it is not running.
- * 
- *	Locks: called as its own thread with no locks held. Takes the
- *	host lock and called routines may take the isa dma lock.
+ * NCR5380_main - NCR state machines
+ *
+ * NCR5380_main is a coroutine that runs as long as more work can
+ * be done on the NCR5380 host adapters in a system.  Both
+ * NCR5380_queue_command() and NCR5380_intr() will try to start it
+ * in case it is not running.
  */
 
 static void NCR5380_main(struct work_struct *work)
 {
 	struct NCR5380_hostdata *hostdata =
-		container_of(work, struct NCR5380_hostdata, coroutine.work);
+		container_of(work, struct NCR5380_hostdata, main_task);
 	struct Scsi_Host *instance = hostdata->host;
-	struct scsi_cmnd *tmp, *prev;
+	struct scsi_cmnd *cmd;
 	int done;
-	
-	spin_lock_irq(instance->host_lock);
+
 	do {
-		/* Lock held here */
 		done = 1;
-		if (!hostdata->connected && !hostdata->selecting) {
-			dprintk(NDEBUG_MAIN, "scsi%d : not connected\n", instance->host_no);
-			/*
-			 * Search through the issue_queue for a command destined
-			 * for a target that's not busy.
-			 */
-			for (tmp = (struct scsi_cmnd *) hostdata->issue_queue, prev = NULL; tmp; prev = tmp, tmp = (struct scsi_cmnd *) tmp->host_scribble)
-			{
-				if (prev != tmp)
-				    dprintk(NDEBUG_LISTS, "MAIN tmp=%p   target=%d   busy=%d lun=%llu\n", tmp, tmp->device->id, hostdata->busy[tmp->device->id], tmp->device->lun);
-				/*  When we find one, remove it from the issue queue. */
-				if (!(hostdata->busy[tmp->device->id] &
-				      (1 << (u8)(tmp->device->lun & 0xff)))) {
-					if (prev) {
-						REMOVE(prev, prev->host_scribble, tmp, tmp->host_scribble);
-						prev->host_scribble = tmp->host_scribble;
-					} else {
-						REMOVE(-1, hostdata->issue_queue, tmp, tmp->host_scribble);
-						hostdata->issue_queue = (struct scsi_cmnd *) tmp->host_scribble;
-					}
-					tmp->host_scribble = NULL;
 
-					/* 
-					 * Attempt to establish an I_T_L nexus here. 
-					 * On success, instance->hostdata->connected is set.
-					 * On failure, we must add the command back to the
-					 *   issue queue so we can keep trying. 
-					 */
-					dprintk(NDEBUG_MAIN|NDEBUG_QUEUES, "scsi%d : main() : command for target %d lun %llu removed from issue_queue\n", instance->host_no, tmp->device->id, tmp->device->lun);
-	
-					/*
-					 * A successful selection is defined as one that 
-					 * leaves us with the command connected and 
-					 * in hostdata->connected, OR has terminated the
-					 * command.
-					 *
-					 * With successful commands, we fall through
-					 * and see if we can do an information transfer,
-					 * with failures we will restart.
-					 */
-					hostdata->selecting = NULL;
-					/* RvC: have to preset this to indicate a new command is being performed */
+		spin_lock_irq(&hostdata->lock);
+		while (!hostdata->connected &&
+		       (cmd = dequeue_next_cmd(instance))) {
 
-					/*
-					 * REQUEST SENSE commands are issued without tagged
-					 * queueing, even on SCSI-II devices because the
-					 * contingent allegiance condition exists for the
-					 * entire unit.
-					 */
+			dsprintk(NDEBUG_MAIN, instance, "main: dequeued %p\n", cmd);
 
-					if (!NCR5380_select(instance, tmp)) {
-						break;
-					} else {
-						LIST(tmp, hostdata->issue_queue);
-						tmp->host_scribble = (unsigned char *) hostdata->issue_queue;
-						hostdata->issue_queue = tmp;
-						done = 0;
-						dprintk(NDEBUG_MAIN|NDEBUG_QUEUES, "scsi%d : main(): select() failed, returned to issue_queue\n", instance->host_no);
-					}
-					/* lock held here still */
-				}	/* if target/lun is not busy */
-			}	/* for */
-			/* exited locked */
-		}	/* if (!hostdata->connected) */
-		if (hostdata->selecting) {
-			tmp = (struct scsi_cmnd *) hostdata->selecting;
-			/* Selection will drop and retake the lock */
-			if (!NCR5380_select(instance, tmp)) {
-				/* Ok ?? */
+			/*
+			 * Attempt to establish an I_T_L nexus here.
+			 * On success, instance->hostdata->connected is set.
+			 * On failure, we must add the command back to the
+			 * issue queue so we can keep trying.
+			 */
+			/*
+			 * REQUEST SENSE commands are issued without tagged
+			 * queueing, even on SCSI-II devices because the
+			 * contingent allegiance condition exists for the
+			 * entire unit.
+			 */
+
+			cmd = NCR5380_select(instance, cmd);
+			if (!cmd) {
+				dsprintk(NDEBUG_MAIN, instance, "main: select complete\n");
 			} else {
-				/* RvC: device failed, so we wait a long time
-				   this is needed for Mustek scanners, that
-				   do not respond to commands immediately
-				   after a scan */
-				printk(KERN_DEBUG "scsi%d: device %d did not respond in time\n", instance->host_no, tmp->device->id);
-				LIST(tmp, hostdata->issue_queue);
-				tmp->host_scribble = (unsigned char *) hostdata->issue_queue;
-				hostdata->issue_queue = tmp;
-				NCR5380_set_timer(hostdata, USLEEP_WAITLONG);
+				dsprintk(NDEBUG_MAIN | NDEBUG_QUEUES, instance,
+				         "main: select failed, returning %p to queue\n", cmd);
+				requeue_cmd(instance, cmd);
 			}
-		}	/* if hostdata->selecting */
+		}
 		if (hostdata->connected
 #ifdef REAL_DMA
 		    && !hostdata->dmalen
 #endif
-		    && (!hostdata->time_expires || time_before_eq(hostdata->time_expires, jiffies))
 		    ) {
-			dprintk(NDEBUG_MAIN, "scsi%d : main() : performing information transfer\n", instance->host_no);
+			dsprintk(NDEBUG_MAIN, instance, "main: performing information transfer\n");
 			NCR5380_information_transfer(instance);
-			dprintk(NDEBUG_MAIN, "scsi%d : main() : done set false\n", instance->host_no);
 			done = 0;
-		} else
-			break;
+		}
+		spin_unlock_irq(&hostdata->lock);
+		if (!done)
+			cond_resched();
 	} while (!done);
-	
-	spin_unlock_irq(instance->host_lock);
 }
 
 #ifndef DONT_USE_INTR
 
 /**
- * 	NCR5380_intr	-	generic NCR5380 irq handler
- *	@irq: interrupt number
- *	@dev_id: device info
- *
- *	Handle interrupts, reestablishing I_T_L or I_T_L_Q nexuses
- *      from the disconnected queue, and restarting NCR5380_main() 
- *      as required.
- *
- *	Locks: takes the needed instance locks
+ * NCR5380_intr - generic NCR5380 irq handler
+ * @irq: interrupt number
+ * @dev_id: device info
+ *
+ * Handle interrupts, reestablishing I_T_L or I_T_L_Q nexuses
+ * from the disconnected queue, and restarting NCR5380_main()
+ * as required.
+ *
+ * The chip can assert IRQ in any of six different conditions. The IRQ flag
+ * is then cleared by reading the Reset Parity/Interrupt Register (RPIR).
+ * Three of these six conditions are latched in the Bus and Status Register:
+ * - End of DMA (cleared by ending DMA Mode)
+ * - Parity error (cleared by reading RPIR)
+ * - Loss of BSY (cleared by reading RPIR)
+ * Two conditions have flag bits that are not latched:
+ * - Bus phase mismatch (non-maskable in DMA Mode, cleared by ending DMA Mode)
+ * - Bus reset (non-maskable)
+ * The remaining condition has no flag bit at all:
+ * - Selection/reselection
+ *
+ * Hence, establishing the cause(s) of any interrupt is partly guesswork.
+ * In "The DP8490 and DP5380 Comparison Guide", National Semiconductor
+ * claimed that "the design of the [DP8490] interrupt logic ensures
+ * interrupts will not be lost (they can be on the DP5380)."
+ * The L5380/53C80 datasheet from LOGIC Devices has more details.
+ *
+ * Checking for bus reset by reading RST is futile because of interrupt
+ * latency, but a bus reset will reset chip logic. Checking for parity error
+ * is unnecessary because that interrupt is never enabled. A Loss of BSY
+ * condition will clear DMA Mode. We can tell when this occurs because the
+ * the Busy Monitor interrupt is enabled together with DMA Mode.
  */
 
-static irqreturn_t NCR5380_intr(int dummy, void *dev_id)
+static irqreturn_t NCR5380_intr(int irq, void *dev_id)
 {
-	NCR5380_local_declare();
 	struct Scsi_Host *instance = dev_id;
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-	int done;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int handled = 0;
 	unsigned char basr;
 	unsigned long flags;
 
-	dprintk(NDEBUG_INTR, "scsi : NCR5380 irq %d triggered\n",
-		instance->irq);
+	spin_lock_irqsave(&hostdata->lock, flags);
+
+	basr = NCR5380_read(BUS_AND_STATUS_REG);
+	if (basr & BASR_IRQ) {
+		unsigned char mr = NCR5380_read(MODE_REG);
+		unsigned char sr = NCR5380_read(STATUS_REG);
+
+		dsprintk(NDEBUG_INTR, instance, "IRQ %d, BASR 0x%02x, SR 0x%02x, MR 0x%02x\n",
+		         irq, basr, sr, mr);
 
-	do {
-		done = 1;
-		spin_lock_irqsave(instance->host_lock, flags);
-		/* Look for pending interrupts */
-		NCR5380_setup(instance);
-		basr = NCR5380_read(BUS_AND_STATUS_REG);
-		/* XXX dispatch to appropriate routine if found and done=0 */
-		if (basr & BASR_IRQ) {
-			NCR5380_dprint(NDEBUG_INTR, instance);
-			if ((NCR5380_read(STATUS_REG) & (SR_SEL | SR_IO)) == (SR_SEL | SR_IO)) {
-				done = 0;
-				dprintk(NDEBUG_INTR, "scsi%d : SEL interrupt\n", instance->host_no);
-				NCR5380_reselect(instance);
-				(void) NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-			} else if (basr & BASR_PARITY_ERROR) {
-				dprintk(NDEBUG_INTR, "scsi%d : PARITY interrupt\n", instance->host_no);
-				(void) NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-			} else if ((NCR5380_read(STATUS_REG) & SR_RST) == SR_RST) {
-				dprintk(NDEBUG_INTR, "scsi%d : RESET interrupt\n", instance->host_no);
-				(void) NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-			} else {
 #if defined(REAL_DMA)
-				/*
-				 * We should only get PHASE MISMATCH and EOP interrupts
-				 * if we have DMA enabled, so do a sanity check based on
-				 * the current setting of the MODE register.
-				 */
+		if ((mr & MR_DMA_MODE) || (mr & MR_MONITOR_BSY)) {
+			/* Probably End of DMA, Phase Mismatch or Loss of BSY.
+			 * We ack IRQ after clearing Mode Register. Workarounds
+			 * for End of DMA errata need to happen in DMA Mode.
+			 */
 
-				if ((NCR5380_read(MODE_REG) & MR_DMA) && ((basr & BASR_END_DMA_TRANSFER) || !(basr & BASR_PHASE_MATCH))) {
-					int transferred;
+			dsprintk(NDEBUG_INTR, instance, "interrupt in DMA mode\n");
 
-					if (!hostdata->connected)
-						panic("scsi%d : received end of DMA interrupt with no connected cmd\n", instance->hostno);
+			int transferred;
 
-					transferred = (hostdata->dmalen - NCR5380_dma_residual(instance));
-					hostdata->connected->SCp.this_residual -= transferred;
-					hostdata->connected->SCp.ptr += transferred;
-					hostdata->dmalen = 0;
+			if (!hostdata->connected)
+				panic("scsi%d : DMA interrupt with no connected cmd\n",
+				      instance->hostno);
 
-					(void) NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-							
-					/* FIXME: we need to poll briefly then defer a workqueue task ! */
-					NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG, BASR_ACK, 0, 2*HZ);
+			transferred = hostdata->dmalen - NCR5380_dma_residual(instance);
+			hostdata->connected->SCp.this_residual -= transferred;
+			hostdata->connected->SCp.ptr += transferred;
+			hostdata->dmalen = 0;
 
-					NCR5380_write(MODE_REG, MR_BASE);
-					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-				}
-#else
-				dprintk(NDEBUG_INTR, "scsi : unknown interrupt, BASR 0x%X, MR 0x%X, SR 0x%x\n", basr, NCR5380_read(MODE_REG), NCR5380_read(STATUS_REG));
-				(void) NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-#endif
+			/* FIXME: we need to poll briefly then defer a workqueue task ! */
+			NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG, BASR_ACK, 0, 2 * HZ);
+
+			NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+			NCR5380_write(MODE_REG, MR_BASE);
+			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+		} else
+#endif /* REAL_DMA */
+		if ((NCR5380_read(CURRENT_SCSI_DATA_REG) & hostdata->id_mask) &&
+		    (sr & (SR_SEL | SR_IO | SR_BSY | SR_RST)) == (SR_SEL | SR_IO)) {
+			/* Probably reselected */
+			NCR5380_write(SELECT_ENABLE_REG, 0);
+			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+
+			dsprintk(NDEBUG_INTR, instance, "interrupt with SEL and IO\n");
+
+			if (!hostdata->connected) {
+				NCR5380_reselect(instance);
+				queue_work(hostdata->work_q, &hostdata->main_task);
 			}
-		}	/* if BASR_IRQ */
-		spin_unlock_irqrestore(instance->host_lock, flags);
-		if(!done)
-			schedule_delayed_work(&hostdata->coroutine, 0);
-	} while (!done);
-	return IRQ_HANDLED;
+			if (!hostdata->connected)
+				NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+		} else {
+			/* Probably Bus Reset */
+			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+
+			dsprintk(NDEBUG_INTR, instance, "unknown interrupt\n");
+		}
+		handled = 1;
+	} else {
+		shost_printk(KERN_NOTICE, instance, "interrupt without IRQ bit\n");
+	}
+
+	spin_unlock_irqrestore(&hostdata->lock, flags);
+
+	return IRQ_RETVAL(handled);
 }
 
-#endif 
+#endif
 
-/* 
+/*
  * Function : int NCR5380_select(struct Scsi_Host *instance,
- *                               struct scsi_cmnd *cmd)
+ * struct scsi_cmnd *cmd)
  *
  * Purpose : establishes I_T_L or I_T_L_Q nexus for new or existing command,
- *      including ARBITRATION, SELECTION, and initial message out for 
- *      IDENTIFY and queue messages. 
- *
- * Inputs : instance - instantiation of the 5380 driver on which this 
- *      target lives, cmd - SCSI command to execute.
- * 
- * Returns : -1 if selection could not execute for some reason,
- *      0 if selection succeeded or failed because the target 
- *      did not respond.
- *
- * Side effects : 
- *      If bus busy, arbitration failed, etc, NCR5380_select() will exit 
- *              with registers as they should have been on entry - ie
- *              SELECT_ENABLE will be set appropriately, the NCR5380
- *              will cease to drive any SCSI bus signals.
- *
- *      If successful : I_T_L or I_T_L_Q nexus will be established, 
- *              instance->connected will be set to cmd.  
- *              SELECT interrupt will be disabled.
- *
- *      If failed (no target) : cmd->scsi_done() will be called, and the 
- *              cmd->result host byte set to DID_BAD_TARGET.
- *
- *	Locks: caller holds hostdata lock in IRQ mode
+ * including ARBITRATION, SELECTION, and initial message out for
+ * IDENTIFY and queue messages.
+ *
+ * Inputs : instance - instantiation of the 5380 driver on which this
+ * target lives, cmd - SCSI command to execute.
+ *
+ * Returns cmd if selection failed but should be retried,
+ * NULL if selection failed and should not be retried, or
+ * NULL if selection succeeded (hostdata->connected == cmd).
+ *
+ * Side effects :
+ * If bus busy, arbitration failed, etc, NCR5380_select() will exit
+ * with registers as they should have been on entry - ie
+ * SELECT_ENABLE will be set appropriately, the NCR5380
+ * will cease to drive any SCSI bus signals.
+ *
+ * If successful : I_T_L or I_T_L_Q nexus will be established,
+ * instance->connected will be set to cmd.
+ * SELECT interrupt will be disabled.
+ *
+ * If failed (no target) : cmd->scsi_done() will be called, and the
+ * cmd->result host byte set to DID_BAD_TARGET.
  */
- 
-static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
+
+static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance,
+                                        struct scsi_cmnd *cmd)
 {
-	NCR5380_local_declare();
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char tmp[3], phase;
 	unsigned char *data;
 	int len;
-	unsigned long timeout;
-	unsigned char value;
 	int err;
-	NCR5380_setup(instance);
-
-	if (hostdata->selecting)
-		goto part2;
-
-	hostdata->restart_select = 0;
 
 	NCR5380_dprint(NDEBUG_ARBITRATION, instance);
-	dprintk(NDEBUG_ARBITRATION, "scsi%d : starting arbitration, id = %d\n", instance->host_no, instance->this_id);
+	dsprintk(NDEBUG_ARBITRATION, instance, "starting arbitration, id = %d\n",
+	         instance->this_id);
+
+	/*
+	 * Arbitration and selection phases are slow and involve dropping the
+	 * lock, so we have to watch out for EH. An exception handler may
+	 * change 'selecting' to NULL. This function will then return NULL
+	 * so that the caller will forget about 'cmd'. (During information
+	 * transfer phases, EH may change 'connected' to NULL.)
+	 */
+	hostdata->selecting = cmd;
 
-	/* 
-	 * Set the phase bits to 0, otherwise the NCR5380 won't drive the 
+	/*
+	 * Set the phase bits to 0, otherwise the NCR5380 won't drive the
 	 * data bus during SELECTION.
 	 */
 
 	NCR5380_write(TARGET_COMMAND_REG, 0);
 
-	/* 
+	/*
 	 * Start arbitration.
 	 */
 
 	NCR5380_write(OUTPUT_DATA_REG, hostdata->id_mask);
 	NCR5380_write(MODE_REG, MR_ARBITRATE);
 
+	/* The chip now waits for BUS FREE phase. Then after the 800 ns
+	 * Bus Free Delay, arbitration will begin.
+	 */
 
-	/* We can be relaxed here, interrupts are on, we are
-	   in workqueue context, the birds are singing in the trees */
-	spin_unlock_irq(instance->host_lock);
-	err = NCR5380_poll_politely(instance, INITIATOR_COMMAND_REG, ICR_ARBITRATION_PROGRESS, ICR_ARBITRATION_PROGRESS, 5*HZ);
-	spin_lock_irq(instance->host_lock);
+	spin_unlock_irq(&hostdata->lock);
+	err = NCR5380_poll_politely2(instance, MODE_REG, MR_ARBITRATE, 0,
+	                INITIATOR_COMMAND_REG, ICR_ARBITRATION_PROGRESS,
+	                                       ICR_ARBITRATION_PROGRESS, HZ);
+	spin_lock_irq(&hostdata->lock);
+	if (!(NCR5380_read(MODE_REG) & MR_ARBITRATE)) {
+		/* Reselection interrupt */
+		goto out;
+	}
 	if (err < 0) {
-		printk(KERN_DEBUG "scsi: arbitration timeout at %d\n", __LINE__);
 		NCR5380_write(MODE_REG, MR_BASE);
-		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		goto failed;
+		shost_printk(KERN_ERR, instance,
+		             "select: arbitration timeout\n");
+		goto out;
 	}
+	spin_unlock_irq(&hostdata->lock);
 
-	dprintk(NDEBUG_ARBITRATION, "scsi%d : arbitration complete\n", instance->host_no);
-
-	/* 
-	 * The arbitration delay is 2.2us, but this is a minimum and there is 
-	 * no maximum so we can safely sleep for ceil(2.2) usecs to accommodate
-	 * the integral nature of udelay().
-	 *
-	 */
-
+	/* The SCSI-2 arbitration delay is 2.4 us */
 	udelay(3);
 
 	/* Check for lost arbitration */
-	if ((NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST) || (NCR5380_read(CURRENT_SCSI_DATA_REG) & hostdata->id_higher_mask) || (NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST)) {
-		NCR5380_write(MODE_REG, MR_BASE);
-		dprintk(NDEBUG_ARBITRATION, "scsi%d : lost arbitration, deasserting MR_ARBITRATE\n", instance->host_no);
-		goto failed;
-	}
-	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_SEL);
-
-	if (!(hostdata->flags & FLAG_DTC3181E) &&
-	    /* RvC: DTC3181E has some trouble with this
-	     *      so we simply removed it. Seems to work with
-	     *      only Mustek scanner attached
-	     */
+	if ((NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST) ||
+	    (NCR5380_read(CURRENT_SCSI_DATA_REG) & hostdata->id_higher_mask) ||
 	    (NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST)) {
 		NCR5380_write(MODE_REG, MR_BASE);
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		dprintk(NDEBUG_ARBITRATION, "scsi%d : lost arbitration, deasserting ICR_ASSERT_SEL\n", instance->host_no);
-		goto failed;
+		dsprintk(NDEBUG_ARBITRATION, instance, "lost arbitration, deasserting MR_ARBITRATE\n");
+		spin_lock_irq(&hostdata->lock);
+		goto out;
 	}
-	/* 
-	 * Again, bus clear + bus settle time is 1.2us, however, this is 
+
+	/* After/during arbitration, BSY should be asserted.
+	 * IBM DPES-31080 Version S31Q works now
+	 * Tnx to Thomas_Roesch@m2.maus.de for finding this! (Roman)
+	 */
+	NCR5380_write(INITIATOR_COMMAND_REG,
+		      ICR_BASE | ICR_ASSERT_SEL | ICR_ASSERT_BSY);
+
+	/*
+	 * Again, bus clear + bus settle time is 1.2us, however, this is
 	 * a minimum so we'll udelay ceil(1.2)
 	 */
 
-	udelay(2);
+	if (hostdata->flags & FLAG_TOSHIBA_DELAY)
+		udelay(15);
+	else
+		udelay(2);
+
+	spin_lock_irq(&hostdata->lock);
+
+	/* NCR5380_reselect() clears MODE_REG after a reselection interrupt */
+	if (!(NCR5380_read(MODE_REG) & MR_ARBITRATE))
+		goto out;
+
+	if (!hostdata->selecting) {
+		NCR5380_write(MODE_REG, MR_BASE);
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+		goto out;
+	}
 
-	dprintk(NDEBUG_ARBITRATION, "scsi%d : won arbitration\n", instance->host_no);
+	dsprintk(NDEBUG_ARBITRATION, instance, "won arbitration\n");
 
-	/* 
-	 * Now that we have won arbitration, start Selection process, asserting 
+	/*
+	 * Now that we have won arbitration, start Selection process, asserting
 	 * the host and target ID's on the SCSI bus.
 	 */
 
-	NCR5380_write(OUTPUT_DATA_REG, (hostdata->id_mask | (1 << scmd_id(cmd))));
+	NCR5380_write(OUTPUT_DATA_REG, hostdata->id_mask | (1 << scmd_id(cmd)));
 
-	/* 
+	/*
 	 * Raise ATN while SEL is true before BSY goes false from arbitration,
 	 * since this is the only way to guarantee that we'll get a MESSAGE OUT
 	 * phase immediately after selection.
 	 */
 
-	NCR5380_write(INITIATOR_COMMAND_REG, (ICR_BASE | ICR_ASSERT_BSY | ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_SEL));
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY |
+	              ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_SEL);
 	NCR5380_write(MODE_REG, MR_BASE);
 
-	/* 
+	/*
 	 * Reselect interrupts must be turned off prior to the dropping of BSY,
 	 * otherwise we will trigger an interrupt.
 	 */
 	NCR5380_write(SELECT_ENABLE_REG, 0);
 
+	spin_unlock_irq(&hostdata->lock);
+
 	/*
-	 * The initiator shall then wait at least two deskew delays and release 
+	 * The initiator shall then wait at least two deskew delays and release
 	 * the BSY signal.
 	 */
-	udelay(1);		/* wingel -- wait two bus deskew delay >2*45ns */
+	udelay(1);        /* wingel -- wait two bus deskew delay >2*45ns */
 
 	/* Reset BSY */
-	NCR5380_write(INITIATOR_COMMAND_REG, (ICR_BASE | ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_SEL));
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA |
+	              ICR_ASSERT_ATN | ICR_ASSERT_SEL);
 
-	/* 
+	/*
 	 * Something weird happens when we cease to drive BSY - looks
-	 * like the board/chip is letting us do another read before the 
+	 * like the board/chip is letting us do another read before the
 	 * appropriate propagation delay has expired, and we're confusing
 	 * a BSY signal from ourselves as the target's response to SELECTION.
 	 *
 	 * A small delay (the 'C++' frontend breaks the pipeline with an
 	 * unnecessary jump, making it work on my 386-33/Trantor T128, the
-	 * tighter 'C' code breaks and requires this) solves the problem - 
-	 * the 1 us delay is arbitrary, and only used because this delay will 
-	 * be the same on other platforms and since it works here, it should 
+	 * tighter 'C' code breaks and requires this) solves the problem -
+	 * the 1 us delay is arbitrary, and only used because this delay will
+	 * be the same on other platforms and since it works here, it should
 	 * work there.
 	 *
 	 * wingel suggests that this could be due to failing to wait
@@ -1339,50 +1162,43 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 
 	udelay(1);
 
-	dprintk(NDEBUG_SELECTION, "scsi%d : selecting target %d\n", instance->host_no, scmd_id(cmd));
+	dsprintk(NDEBUG_SELECTION, instance, "selecting target %d\n", scmd_id(cmd));
 
-	/* 
-	 * The SCSI specification calls for a 250 ms timeout for the actual 
+	/*
+	 * The SCSI specification calls for a 250 ms timeout for the actual
 	 * selection.
 	 */
 
-	timeout = jiffies + msecs_to_jiffies(250);
-
-	/* 
-	 * XXX very interesting - we're seeing a bounce where the BSY we 
-	 * asserted is being reflected / still asserted (propagation delay?)
-	 * and it's detecting as true.  Sigh.
-	 */
-
-	hostdata->select_time = 0;	/* we count the clock ticks at which we polled */
-	hostdata->selecting = cmd;
+	err = NCR5380_poll_politely(instance, STATUS_REG, SR_BSY, SR_BSY,
+	                            msecs_to_jiffies(250));
 
-part2:
-	/* RvC: here we enter after a sleeping period, or immediately after
-	   execution of part 1
-	   we poll only once ech clock tick */
-	value = NCR5380_read(STATUS_REG) & (SR_BSY | SR_IO);
-
-	if (!value && (hostdata->select_time < HZ/4)) {
-		/* RvC: we still must wait for a device response */
-		hostdata->select_time++;	/* after 25 ticks the device has failed */
-		NCR5380_set_timer(hostdata, 1);
-		return 0;	/* RvC: we return here with hostdata->selecting set,
-				   to go to sleep */
-	}
-
-	hostdata->selecting = NULL;/* clear this pointer, because we passed the
-					   waiting period */
 	if ((NCR5380_read(STATUS_REG) & (SR_SEL | SR_IO)) == (SR_SEL | SR_IO)) {
+		spin_lock_irq(&hostdata->lock);
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		NCR5380_reselect(instance);
-		printk("scsi%d : reselection after won arbitration?\n", instance->host_no);
+		if (!hostdata->connected)
+			NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+		shost_printk(KERN_ERR, instance, "reselection after won arbitration?\n");
+		goto out;
+	}
+
+	if (err < 0) {
+		spin_lock_irq(&hostdata->lock);
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		return -1;
+		/* Can't touch cmd if it has been reclaimed by the scsi ML */
+		if (hostdata->selecting) {
+			cmd->result = DID_BAD_TARGET << 16;
+			complete_cmd(instance, cmd);
+			dsprintk(NDEBUG_SELECTION, instance, "target did not respond within 250ms\n");
+			cmd = NULL;
+		}
+		goto out;
 	}
-	/* 
-	 * No less than two deskew delays after the initiator detects the 
-	 * BSY signal is true, it shall release the SEL signal and may 
+
+	/*
+	 * No less than two deskew delays after the initiator detects the
+	 * BSY signal is true, it shall release the SEL signal and may
 	 * change the DATA BUS.                                     -wingel
 	 */
 
@@ -1390,53 +1206,38 @@ part2:
 
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 
-	if (!(NCR5380_read(STATUS_REG) & SR_BSY)) {
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		if (hostdata->targets_present & (1 << scmd_id(cmd))) {
-			printk(KERN_DEBUG "scsi%d : weirdness\n", instance->host_no);
-			if (hostdata->restart_select)
-				printk(KERN_DEBUG "\trestart select\n");
-			NCR5380_dprint(NDEBUG_SELECTION, instance);
-			NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-			return -1;
-		}
-		cmd->result = DID_BAD_TARGET << 16;
-		cmd->scsi_done(cmd);
-		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		dprintk(NDEBUG_SELECTION, "scsi%d : target did not respond within 250ms\n", instance->host_no);
-		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		return 0;
-	}
-	hostdata->targets_present |= (1 << scmd_id(cmd));
-
 	/*
-	 * Since we followed the SCSI spec, and raised ATN while SEL 
+	 * Since we followed the SCSI spec, and raised ATN while SEL
 	 * was true but before BSY was false during selection, the information
 	 * transfer phase should be a MESSAGE OUT phase so that we can send the
 	 * IDENTIFY message.
-	 * 
+	 *
 	 * If SCSI-II tagged queuing is enabled, we also send a SIMPLE_QUEUE_TAG
 	 * message (2 bytes) with a tag ID that we increment with every command
 	 * until it wraps back to 0.
 	 *
 	 * XXX - it turns out that there are some broken SCSI-II devices,
-	 *       which claim to support tagged queuing but fail when more than
-	 *       some number of commands are issued at once.
+	 * which claim to support tagged queuing but fail when more than
+	 * some number of commands are issued at once.
 	 */
 
 	/* Wait for start of REQ/ACK handshake */
 
-	spin_unlock_irq(instance->host_lock);
 	err = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
-	spin_lock_irq(instance->host_lock);
-	
-	if(err) {
-		printk(KERN_ERR "scsi%d: timeout at NCR5380.c:%d\n", instance->host_no, __LINE__);
+	spin_lock_irq(&hostdata->lock);
+	if (err < 0) {
+		shost_printk(KERN_ERR, instance, "select: REQ timeout\n");
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		goto failed;
+		goto out;
+	}
+	if (!hostdata->selecting) {
+		do_abort(instance);
+		goto out;
 	}
 
-	dprintk(NDEBUG_SELECTION, "scsi%d : target %d selected, going into MESSAGE OUT phase.\n", instance->host_no, cmd->device->id);
+	dsprintk(NDEBUG_SELECTION, instance, "target %d selected, going into MESSAGE OUT phase.\n",
+	         scmd_id(cmd));
 	tmp[0] = IDENTIFY(((instance->irq == NO_IRQ) ? 0 : 1), cmd->device->lun);
 
 	len = 1;
@@ -1446,104 +1247,82 @@ part2:
 	data = tmp;
 	phase = PHASE_MSGOUT;
 	NCR5380_transfer_pio(instance, &phase, &len, &data);
-	dprintk(NDEBUG_SELECTION, "scsi%d : nexus established.\n", instance->host_no);
+	dsprintk(NDEBUG_SELECTION, instance, "nexus established.\n");
 	/* XXX need to handle errors here */
+
 	hostdata->connected = cmd;
-	hostdata->busy[cmd->device->id] |= (1 << (cmd->device->lun & 0xFF));
+	hostdata->busy[cmd->device->id] |= 1 << cmd->device->lun;
 
 	initialize_SCp(cmd);
 
-	return 0;
-
-	/* Selection failed */
-failed:
-	return -1;
+	cmd = NULL;
 
+out:
+	if (!hostdata->selecting)
+		return NULL;
+	hostdata->selecting = NULL;
+	return cmd;
 }
 
-/* 
- * Function : int NCR5380_transfer_pio (struct Scsi_Host *instance, 
- *      unsigned char *phase, int *count, unsigned char **data)
+/*
+ * Function : int NCR5380_transfer_pio (struct Scsi_Host *instance,
+ * unsigned char *phase, int *count, unsigned char **data)
  *
  * Purpose : transfers data in given phase using polled I/O
  *
- * Inputs : instance - instance of driver, *phase - pointer to 
- *      what phase is expected, *count - pointer to number of 
- *      bytes to transfer, **data - pointer to data pointer.
- * 
+ * Inputs : instance - instance of driver, *phase - pointer to
+ * what phase is expected, *count - pointer to number of
+ * bytes to transfer, **data - pointer to data pointer.
+ *
  * Returns : -1 when different phase is entered without transferring
- *      maximum number of bytes, 0 if all bytes or transferred or exit
- *      is in same phase.
+ * maximum number of bytes, 0 if all bytes are transferred or exit
+ * is in same phase.
  *
- *      Also, *phase, *count, *data are modified in place.
+ * Also, *phase, *count, *data are modified in place.
  *
  * XXX Note : handling for bus free may be useful.
  */
 
 /*
- * Note : this code is not as quick as it could be, however it 
+ * Note : this code is not as quick as it could be, however it
  * IS 100% reliable, and for the actual data transfer where speed
  * counts, we will always do a pseudo DMA or DMA transfer.
  */
 
-static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data) {
-	NCR5380_local_declare();
+static int NCR5380_transfer_pio(struct Scsi_Host *instance,
+				unsigned char *phase, int *count,
+				unsigned char **data)
+{
 	unsigned char p = *phase, tmp;
 	int c = *count;
 	unsigned char *d = *data;
-	/*
-	 *      RvC: some administrative data to process polling time
-	 */
-	int break_allowed = 0;
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-	NCR5380_setup(instance);
-
-	if (!(p & SR_IO))
-		dprintk(NDEBUG_PIO, "scsi%d : pio write %d bytes\n", instance->host_no, c);
-	else
-		dprintk(NDEBUG_PIO, "scsi%d : pio read %d bytes\n", instance->host_no, c);
 
-	/* 
-	 * The NCR5380 chip will only drive the SCSI bus when the 
+	/*
+	 * The NCR5380 chip will only drive the SCSI bus when the
 	 * phase specified in the appropriate bits of the TARGET COMMAND
 	 * REGISTER match the STATUS REGISTER
 	 */
 
-	 NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(p));
+	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(p));
 
-	/* RvC: don't know if this is necessary, but other SCSI I/O is short
-	 *      so breaks are not necessary there
-	 */
-	if ((p == PHASE_DATAIN) || (p == PHASE_DATAOUT)) {
-		break_allowed = 1;
-	}
 	do {
-		/* 
-		 * Wait for assertion of REQ, after which the phase bits will be 
-		 * valid 
-		 */
-
-		/* RvC: we simply poll once, after that we stop temporarily
-		 *      and let the device buffer fill up
-		 *      if breaking is not allowed, we keep polling as long as needed
+		/*
+		 * Wait for assertion of REQ, after which the phase bits will be
+		 * valid
 		 */
 
-		/* FIXME */
-		while (!((tmp = NCR5380_read(STATUS_REG)) & SR_REQ) && !break_allowed);
-		if (!(tmp & SR_REQ)) {
-			/* timeout condition */
-			NCR5380_set_timer(hostdata, USLEEP_SLEEP);
+		if (NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0)
 			break;
-		}
 
-		dprintk(NDEBUG_HANDSHAKE, "scsi%d : REQ detected\n", instance->host_no);
+		dsprintk(NDEBUG_HANDSHAKE, instance, "REQ asserted\n");
 
 		/* Check for phase mismatch */
-		if ((tmp & PHASE_MASK) != p) {
-			dprintk(NDEBUG_HANDSHAKE, "scsi%d : phase mismatch\n", instance->host_no);
-			NCR5380_dprint_phase(NDEBUG_HANDSHAKE, instance);
+		if ((NCR5380_read(STATUS_REG) & PHASE_MASK) != p) {
+			dsprintk(NDEBUG_PIO, instance, "phase mismatch\n");
+			NCR5380_dprint_phase(NDEBUG_PIO, instance);
 			break;
 		}
+
 		/* Do actual transfer from SCSI bus to / from memory */
 		if (!(p & SR_IO))
 			NCR5380_write(OUTPUT_DATA_REG, *d);
@@ -1552,7 +1331,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase
 
 		++d;
 
-		/* 
+		/*
 		 * The SCSI standard suggests that in MSGOUT phase, the initiator
 		 * should drop ATN on the last byte of the message phase
 		 * after REQ has been asserted for the handshake but before
@@ -1563,29 +1342,34 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase
 			if (!((p & SR_MSG) && c > 1)) {
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA);
 				NCR5380_dprint(NDEBUG_PIO, instance);
-				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA | ICR_ASSERT_ACK);
+				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
+				              ICR_ASSERT_DATA | ICR_ASSERT_ACK);
 			} else {
-				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA | ICR_ASSERT_ATN);
+				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
+				              ICR_ASSERT_DATA | ICR_ASSERT_ATN);
 				NCR5380_dprint(NDEBUG_PIO, instance);
-				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
+				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
+				              ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
 			}
 		} else {
 			NCR5380_dprint(NDEBUG_PIO, instance);
 			NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ACK);
 		}
 
-		/* FIXME - if this fails bus reset ?? */
-		NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, 0, 5*HZ);
-		dprintk(NDEBUG_HANDSHAKE, "scsi%d : req false, handshake complete\n", instance->host_no);
+		if (NCR5380_poll_politely(instance,
+		                          STATUS_REG, SR_REQ, 0, 5 * HZ) < 0)
+			break;
+
+		dsprintk(NDEBUG_HANDSHAKE, instance, "REQ negated, handshake complete\n");
 
 /*
- * We have several special cases to consider during REQ/ACK handshaking : 
- * 1.  We were in MSGOUT phase, and we are on the last byte of the 
- *      message.  ATN must be dropped as ACK is dropped.
+ * We have several special cases to consider during REQ/ACK handshaking :
+ * 1.  We were in MSGOUT phase, and we are on the last byte of the
+ * message.  ATN must be dropped as ACK is dropped.
  *
- * 2.  We are in a MSGIN phase, and we are on the last byte of the  
- *      message.  We must exit with ACK asserted, so that the calling
- *      code may raise ATN before dropping ACK to reject the message.
+ * 2.  We are in a MSGIN phase, and we are on the last byte of the
+ * message.  We must exit with ACK asserted, so that the calling
+ * code may raise ATN before dropping ACK to reject the message.
  *
  * 3.  ACK and ATN are clear and the target may proceed as normal.
  */
@@ -1597,12 +1381,16 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase
 		}
 	} while (--c);
 
-	dprintk(NDEBUG_PIO, "scsi%d : residual %d\n", instance->host_no, c);
+	dsprintk(NDEBUG_PIO, instance, "residual %d\n", c);
 
 	*count = c;
 	*data = d;
 	tmp = NCR5380_read(STATUS_REG);
-	if (tmp & SR_REQ)
+	/* The phase read from the bus is valid if either REQ is (already)
+	 * asserted or if ACK hasn't been released yet. The latter applies if
+	 * we're in MSG IN, DATA IN or STATUS and all bytes have been received.
+	 */
+	if ((tmp & SR_REQ) || ((tmp & SR_IO) && c == 0))
 		*phase = tmp & PHASE_MASK;
 	else
 		*phase = PHASE_UNKNOWN;
@@ -1614,79 +1402,80 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase
 }
 
 /**
- *	do_reset	-	issue a reset command
- *	@host: adapter to reset
+ * do_reset - issue a reset command
+ * @instance: adapter to reset
  *
- *	Issue a reset sequence to the NCR5380 and try and get the bus
- *	back into sane shape.
+ * Issue a reset sequence to the NCR5380 and try and get the bus
+ * back into sane shape.
  *
- *	Locks: caller holds queue lock
+ * This clears the reset interrupt flag because there may be no handler for
+ * it. When the driver is initialized, the NCR5380_intr() handler has not yet
+ * been installed. And when in EH we may have released the ST DMA interrupt.
  */
- 
-static void do_reset(struct Scsi_Host *host) {
-	NCR5380_local_declare();
-	NCR5380_setup(host);
 
-	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(NCR5380_read(STATUS_REG) & PHASE_MASK));
+static void do_reset(struct Scsi_Host *instance)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	NCR5380_write(TARGET_COMMAND_REG,
+	              PHASE_SR_TO_TCR(NCR5380_read(STATUS_REG) & PHASE_MASK));
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST);
-	udelay(25);
+	udelay(50);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+	(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+	local_irq_restore(flags);
 }
 
-/*
- * Function : do_abort (Scsi_Host *host)
- * 
- * Purpose : abort the currently established nexus.  Should only be 
- *      called from a routine which can drop into a 
- * 
- * Returns : 0 on success, -1 on failure.
- *
- * Locks: queue lock held by caller
- *	FIXME: sort this out and get new_eh running
+/**
+ * do_abort - abort the currently established nexus by going to
+ * MESSAGE OUT phase and sending an ABORT message.
+ * @instance: relevant scsi host instance
+ *
+ * Returns 0 on success, -1 on failure.
  */
 
-static int do_abort(struct Scsi_Host *host) {
-	NCR5380_local_declare();
+static int do_abort(struct Scsi_Host *instance)
+{
 	unsigned char *msgptr, phase, tmp;
 	int len;
 	int rc;
-	NCR5380_setup(host);
-
 
 	/* Request message out phase */
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 
-	/* 
-	 * Wait for the target to indicate a valid phase by asserting 
-	 * REQ.  Once this happens, we'll have either a MSGOUT phase 
-	 * and can immediately send the ABORT message, or we'll have some 
+	/*
+	 * Wait for the target to indicate a valid phase by asserting
+	 * REQ.  Once this happens, we'll have either a MSGOUT phase
+	 * and can immediately send the ABORT message, or we'll have some
 	 * other phase and will have to source/sink data.
-	 * 
+	 *
 	 * We really don't care what value was on the bus or what value
 	 * the target sees, so we just handshake.
 	 */
 
-	rc = NCR5380_poll_politely(host, STATUS_REG, SR_REQ, SR_REQ, 60 * HZ);
-	
-	if(rc < 0)
-		return -1;
+	rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ);
+	if (rc < 0)
+		goto timeout;
+
+	tmp = NCR5380_read(STATUS_REG) & PHASE_MASK;
 
-	tmp = (unsigned char)rc;
-	
 	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(tmp));
 
-	if ((tmp & PHASE_MASK) != PHASE_MSGOUT) {
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
-		rc = NCR5380_poll_politely(host, STATUS_REG, SR_REQ, 0, 3*HZ);
+	if (tmp != PHASE_MSGOUT) {
+		NCR5380_write(INITIATOR_COMMAND_REG,
+		              ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
+		rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, 0, 3 * HZ);
+		if (rc < 0)
+			goto timeout;
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
-		if(rc == -1)
-			return -1;
 	}
+
 	tmp = ABORT;
 	msgptr = &tmp;
 	len = 1;
 	phase = PHASE_MSGOUT;
-	NCR5380_transfer_pio(host, &phase, &len, &msgptr);
+	NCR5380_transfer_pio(instance, &phase, &len, &msgptr);
 
 	/*
 	 * If we got here, and the command completed successfully,
@@ -1694,32 +1483,37 @@ static int do_abort(struct Scsi_Host *host) {
 	 */
 
 	return len ? -1 : 0;
+
+timeout:
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+	return -1;
 }
 
 #if defined(REAL_DMA) || defined(PSEUDO_DMA) || defined (REAL_DMA_POLL)
-/* 
- * Function : int NCR5380_transfer_dma (struct Scsi_Host *instance, 
- *      unsigned char *phase, int *count, unsigned char **data)
+/*
+ * Function : int NCR5380_transfer_dma (struct Scsi_Host *instance,
+ * unsigned char *phase, int *count, unsigned char **data)
  *
  * Purpose : transfers data in given phase using either real
- *      or pseudo DMA.
+ * or pseudo DMA.
  *
- * Inputs : instance - instance of driver, *phase - pointer to 
- *      what phase is expected, *count - pointer to number of 
- *      bytes to transfer, **data - pointer to data pointer.
- * 
- * Returns : -1 when different phase is entered without transferring
- *      maximum number of bytes, 0 if all bytes or transferred or exit
- *      is in same phase.
+ * Inputs : instance - instance of driver, *phase - pointer to
+ * what phase is expected, *count - pointer to number of
+ * bytes to transfer, **data - pointer to data pointer.
  *
- *      Also, *phase, *count, *data are modified in place.
+ * Returns : -1 when different phase is entered without transferring
+ * maximum number of bytes, 0 if all bytes or transferred or exit
+ * is in same phase.
  *
- *	Locks: io_request lock held by caller
+ * Also, *phase, *count, *data are modified in place.
  */
 
 
-static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data) {
-	NCR5380_local_declare();
+static int NCR5380_transfer_dma(struct Scsi_Host *instance,
+				unsigned char *phase, int *count,
+				unsigned char **data)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	register int c = *count;
 	register unsigned char p = *phase;
 	register unsigned char *d = *data;
@@ -1730,54 +1524,47 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 	unsigned char saved_data = 0, overrun = 0, residue;
 #endif
 
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-
-	NCR5380_setup(instance);
-
 	if ((tmp = (NCR5380_read(STATUS_REG) & PHASE_MASK)) != p) {
 		*phase = tmp;
 		return -1;
 	}
 #if defined(REAL_DMA) || defined(REAL_DMA_POLL)
-#ifdef READ_OVERRUNS
 	if (p & SR_IO) {
-		c -= 2;
+		if (!(hostdata->flags & FLAG_NO_DMA_FIXUPS))
+			c -= 2;
 	}
-#endif
-	dprintk(NDEBUG_DMA, "scsi%d : initializing DMA channel %d for %s, %d bytes %s %0x\n", instance->host_no, instance->dma_channel, (p & SR_IO) ? "reading" : "writing", c, (p & SR_IO) ? "to" : "from", (unsigned) d);
 	hostdata->dma_len = (p & SR_IO) ? NCR5380_dma_read_setup(instance, d, c) : NCR5380_dma_write_setup(instance, d, c);
+
+	dsprintk(NDEBUG_DMA, instance, "initializing DMA %s: length %d, address %p\n",
+	         (p & SR_IO) ? "receive" : "send", c, *data);
 #endif
 
 	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(p));
 
 #ifdef REAL_DMA
-	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_ENABLE_EOP_INTR | MR_MONITOR_BSY);
+	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY |
+	                        MR_ENABLE_EOP_INTR);
 #elif defined(REAL_DMA_POLL)
-	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE);
+	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY);
 #else
 	/*
 	 * Note : on my sample board, watch-dog timeouts occurred when interrupts
-	 * were not disabled for the duration of a single DMA transfer, from 
+	 * were not disabled for the duration of a single DMA transfer, from
 	 * before the setting of DMA mode to after transfer of the last byte.
 	 */
 
-#if defined(PSEUDO_DMA) && defined(UNSAFE)
-	spin_unlock_irq(instance->host_lock);
-#endif
-	/* KLL May need eop and parity in 53c400 */
-	if (hostdata->flags & FLAG_NCR53C400)
-		NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE |
-				MR_ENABLE_PAR_CHECK | MR_ENABLE_PAR_INTR |
-				MR_ENABLE_EOP_INTR | MR_MONITOR_BSY);
+	if (hostdata->flags & FLAG_NO_DMA_FIXUP)
+		NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY |
+		                        MR_ENABLE_EOP_INTR);
 	else
-		NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE);
+		NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY);
 #endif				/* def REAL_DMA */
 
 	dprintk(NDEBUG_DMA, "scsi%d : mode reg = 0x%X\n", instance->host_no, NCR5380_read(MODE_REG));
 
-	/* 
-	 *	On the PAS16 at least I/O recovery delays are not needed here.
-	 *	Everyone else seems to want them.
+	/*
+	 * On the PAS16 at least I/O recovery delays are not needed here.
+	 * Everyone else seems to want them.
 	 */
 
 	if (p & SR_IO) {
@@ -1797,49 +1584,49 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 	} while ((tmp & BASR_PHASE_MATCH) && !(tmp & (BASR_BUSY_ERROR | BASR_END_DMA_TRANSFER)));
 
 /*
-   At this point, either we've completed DMA, or we have a phase mismatch,
-   or we've unexpectedly lost BUSY (which is a real error).
-
-   For write DMAs, we want to wait until the last byte has been
-   transferred out over the bus before we turn off DMA mode.  Alas, there
-   seems to be no terribly good way of doing this on a 5380 under all
-   conditions.  For non-scatter-gather operations, we can wait until REQ
-   and ACK both go false, or until a phase mismatch occurs.  Gather-writes
-   are nastier, since the device will be expecting more data than we
-   are prepared to send it, and REQ will remain asserted.  On a 53C8[01] we
-   could test LAST BIT SENT to assure transfer (I imagine this is precisely
-   why this signal was added to the newer chips) but on the older 538[01]
-   this signal does not exist.  The workaround for this lack is a watchdog;
-   we bail out of the wait-loop after a modest amount of wait-time if
-   the usual exit conditions are not met.  Not a terribly clean or
-   correct solution :-%
-
-   Reads are equally tricky due to a nasty characteristic of the NCR5380.
-   If the chip is in DMA mode for an READ, it will respond to a target's
-   REQ by latching the SCSI data into the INPUT DATA register and asserting
-   ACK, even if it has _already_ been notified by the DMA controller that
-   the current DMA transfer has completed!  If the NCR5380 is then taken
-   out of DMA mode, this already-acknowledged byte is lost.
-
-   This is not a problem for "one DMA transfer per command" reads, because
-   the situation will never arise... either all of the data is DMA'ed
-   properly, or the target switches to MESSAGE IN phase to signal a
-   disconnection (either operation bringing the DMA to a clean halt).
-   However, in order to handle scatter-reads, we must work around the
-   problem.  The chosen fix is to DMA N-2 bytes, then check for the
-   condition before taking the NCR5380 out of DMA mode.  One or two extra
-   bytes are transferred via PIO as necessary to fill out the original
-   request.
+ * At this point, either we've completed DMA, or we have a phase mismatch,
+ * or we've unexpectedly lost BUSY (which is a real error).
+ *
+ * For DMA sends, we want to wait until the last byte has been
+ * transferred out over the bus before we turn off DMA mode.  Alas, there
+ * seems to be no terribly good way of doing this on a 5380 under all
+ * conditions.  For non-scatter-gather operations, we can wait until REQ
+ * and ACK both go false, or until a phase mismatch occurs.  Gather-sends
+ * are nastier, since the device will be expecting more data than we
+ * are prepared to send it, and REQ will remain asserted.  On a 53C8[01] we
+ * could test Last Byte Sent to assure transfer (I imagine this is precisely
+ * why this signal was added to the newer chips) but on the older 538[01]
+ * this signal does not exist.  The workaround for this lack is a watchdog;
+ * we bail out of the wait-loop after a modest amount of wait-time if
+ * the usual exit conditions are not met.  Not a terribly clean or
+ * correct solution :-%
+ *
+ * DMA receive is equally tricky due to a nasty characteristic of the NCR5380.
+ * If the chip is in DMA receive mode, it will respond to a target's
+ * REQ by latching the SCSI data into the INPUT DATA register and asserting
+ * ACK, even if it has _already_ been notified by the DMA controller that
+ * the current DMA transfer has completed!  If the NCR5380 is then taken
+ * out of DMA mode, this already-acknowledged byte is lost. This is
+ * not a problem for "one DMA transfer per READ command", because
+ * the situation will never arise... either all of the data is DMA'ed
+ * properly, or the target switches to MESSAGE IN phase to signal a
+ * disconnection (either operation bringing the DMA to a clean halt).
+ * However, in order to handle scatter-receive, we must work around the
+ * problem.  The chosen fix is to DMA N-2 bytes, then check for the
+ * condition before taking the NCR5380 out of DMA mode.  One or two extra
+ * bytes are transferred via PIO as necessary to fill out the original
+ * request.
  */
 
 	if (p & SR_IO) {
-#ifdef READ_OVERRUNS
-		udelay(10);
-		if (((NCR5380_read(BUS_AND_STATUS_REG) & (BASR_PHASE_MATCH | BASR_ACK)) == (BASR_PHASE_MATCH | BASR_ACK))) {
-			saved_data = NCR5380_read(INPUT_DATA_REGISTER);
-			overrun = 1;
+		if (!(hostdata->flags & FLAG_NO_DMA_FIXUPS)) {
+			udelay(10);
+			if ((NCR5380_read(BUS_AND_STATUS_REG) & (BASR_PHASE_MATCH | BASR_ACK)) ==
+			    (BASR_PHASE_MATCH | BASR_ACK)) {
+				saved_data = NCR5380_read(INPUT_DATA_REGISTER);
+				overrun = 1;
+			}
 		}
-#endif
 	} else {
 		int limit = 100;
 		while (((tmp = NCR5380_read(BUS_AND_STATUS_REG)) & BASR_ACK) || (NCR5380_read(STATUS_REG) & SR_REQ)) {
@@ -1850,7 +1637,8 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 		}
 	}
 
-	dprintk(NDEBUG_DMA, "scsi%d : polled DMA transfer complete, basr 0x%X, sr 0x%X\n", instance->host_no, tmp, NCR5380_read(STATUS_REG));
+	dsprintk(NDEBUG_DMA, "polled DMA transfer complete, basr 0x%02x, sr 0x%02x\n",
+	         tmp, NCR5380_read(STATUS_REG));
 
 	NCR5380_write(MODE_REG, MR_BASE);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
@@ -1861,8 +1649,8 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 	*data += c;
 	*phase = NCR5380_read(STATUS_REG) & PHASE_MASK;
 
-#ifdef READ_OVERRUNS
-	if (*phase == p && (p & SR_IO) && residue == 0) {
+	if (!(hostdata->flags & FLAG_NO_DMA_FIXUPS) &&
+	    *phase == p && (p & SR_IO) && residue == 0) {
 		if (overrun) {
 			dprintk(NDEBUG_DMA, "Got an input overrun, using saved byte\n");
 			**data = saved_data;
@@ -1877,7 +1665,6 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 		NCR5380_transfer_pio(instance, phase, &cnt, data);
 		*count -= toPIO - cnt;
 	}
-#endif
 
 	dprintk(NDEBUG_DMA, "Return with data ptr = 0x%X, count %d, last 0x%X, next 0x%X\n", *data, *count, *(*data + *count - 1), *(*data + *count));
 	return 0;
@@ -1886,95 +1673,64 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 	return 0;
 #else				/* defined(REAL_DMA_POLL) */
 	if (p & SR_IO) {
-#ifdef DMA_WORKS_RIGHT
-		foo = NCR5380_pread(instance, d, c);
-#else
-		int diff = 1;
-		if (hostdata->flags & FLAG_NCR53C400) {
-			diff = 0;
-		}
-		if (!(foo = NCR5380_pread(instance, d, c - diff))) {
+		foo = NCR5380_pread(instance, d,
+			hostdata->flags & FLAG_NO_DMA_FIXUP ? c : c - 1);
+		if (!foo && !(hostdata->flags & FLAG_NO_DMA_FIXUP)) {
 			/*
-			 * We can't disable DMA mode after successfully transferring 
+			 * We can't disable DMA mode after successfully transferring
 			 * what we plan to be the last byte, since that would open up
-			 * a race condition where if the target asserted REQ before 
+			 * a race condition where if the target asserted REQ before
 			 * we got the DMA mode reset, the NCR5380 would have latched
 			 * an additional byte into the INPUT DATA register and we'd
 			 * have dropped it.
-			 * 
-			 * The workaround was to transfer one fewer bytes than we 
-			 * intended to with the pseudo-DMA read function, wait for 
+			 *
+			 * The workaround was to transfer one fewer bytes than we
+			 * intended to with the pseudo-DMA read function, wait for
 			 * the chip to latch the last byte, read it, and then disable
 			 * pseudo-DMA mode.
-			 * 
+			 *
 			 * After REQ is asserted, the NCR5380 asserts DRQ and ACK.
 			 * REQ is deasserted when ACK is asserted, and not reasserted
 			 * until ACK goes false.  Since the NCR5380 won't lower ACK
 			 * until DACK is asserted, which won't happen unless we twiddle
-			 * the DMA port or we take the NCR5380 out of DMA mode, we 
-			 * can guarantee that we won't handshake another extra 
+			 * the DMA port or we take the NCR5380 out of DMA mode, we
+			 * can guarantee that we won't handshake another extra
 			 * byte.
 			 */
 
-			if (!(hostdata->flags & FLAG_NCR53C400)) {
-				while (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_DRQ));
-				/* Wait for clean handshake */
-				while (NCR5380_read(STATUS_REG) & SR_REQ);
-				d[c - 1] = NCR5380_read(INPUT_DATA_REG);
+			if (NCR5380_poll_politely(instance, BUS_AND_STATUS_REG,
+			                          BASR_DRQ, BASR_DRQ, HZ) < 0) {
+				foo = -1;
+				shost_printk(KERN_ERR, instance, "PDMA read: DRQ timeout\n");
+			}
+			if (NCR5380_poll_politely(instance, STATUS_REG,
+			                          SR_REQ, 0, HZ) < 0) {
+				foo = -1;
+				shost_printk(KERN_ERR, instance, "PDMA read: !REQ timeout\n");
 			}
+			d[c - 1] = NCR5380_read(INPUT_DATA_REG);
 		}
-#endif
 	} else {
-#ifdef DMA_WORKS_RIGHT
 		foo = NCR5380_pwrite(instance, d, c);
-#else
-		int timeout;
-		dprintk(NDEBUG_C400_PWRITE, "About to pwrite %d bytes\n", c);
-		if (!(foo = NCR5380_pwrite(instance, d, c))) {
+		if (!foo && !(hostdata->flags & FLAG_NO_DMA_FIXUP)) {
 			/*
-			 * Wait for the last byte to be sent.  If REQ is being asserted for 
-			 * the byte we're interested, we'll ACK it and it will go false.  
+			 * Wait for the last byte to be sent.  If REQ is being asserted for
+			 * the byte we're interested, we'll ACK it and it will go false.
 			 */
-			if (!(hostdata->flags & FLAG_HAS_LAST_BYTE_SENT)) {
-				timeout = 20000;
-				while (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_DRQ) && (NCR5380_read(BUS_AND_STATUS_REG) & BASR_PHASE_MATCH));
-
-				if (!timeout)
-					dprintk(NDEBUG_LAST_BYTE_SENT, "scsi%d : timed out on last byte\n", instance->host_no);
-
-				if (hostdata->flags & FLAG_CHECK_LAST_BYTE_SENT) {
-					hostdata->flags &= ~FLAG_CHECK_LAST_BYTE_SENT;
-					if (NCR5380_read(TARGET_COMMAND_REG) & TCR_LAST_BYTE_SENT) {
-						hostdata->flags |= FLAG_HAS_LAST_BYTE_SENT;
-						dprintk(NDEBUG_LAST_BYTE_SENT, "scsi%d : last byte sent works\n", instance->host_no);
-					}
-				}
-			} else {
-				dprintk(NDEBUG_C400_PWRITE, "Waiting for LASTBYTE\n");
-				while (!(NCR5380_read(TARGET_COMMAND_REG) & TCR_LAST_BYTE_SENT));
-				dprintk(NDEBUG_C400_PWRITE, "Got LASTBYTE\n");
+			if (NCR5380_poll_politely2(instance,
+			     BUS_AND_STATUS_REG, BASR_DRQ, BASR_DRQ,
+			     BUS_AND_STATUS_REG, BASR_PHASE_MATCH, 0, HZ) < 0) {
+				foo = -1;
+				shost_printk(KERN_ERR, instance, "PDMA write: DRQ and phase timeout\n");
 			}
 		}
-#endif
 	}
 	NCR5380_write(MODE_REG, MR_BASE);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-
-	if ((!(p & SR_IO)) && (hostdata->flags & FLAG_NCR53C400)) {
-		dprintk(NDEBUG_C400_PWRITE, "53C400w: Checking for IRQ\n");
-		if (NCR5380_read(BUS_AND_STATUS_REG) & BASR_IRQ) {
-			dprintk(NDEBUG_C400_PWRITE, "53C400w:    got it, reading reset interrupt reg\n");
-			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-		} else {
-			printk("53C400w:    IRQ NOT THERE!\n");
-		}
-	}
+	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 	*data = d + c;
 	*count = 0;
 	*phase = NCR5380_read(STATUS_REG) & PHASE_MASK;
-#if defined(PSEUDO_DMA) && defined(UNSAFE)
-	spin_lock_irq(instance->host_lock);
-#endif				/* defined(REAL_DMA_POLL) */
 	return foo;
 #endif				/* def REAL_DMA */
 }
@@ -1983,25 +1739,23 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase
 /*
  * Function : NCR5380_information_transfer (struct Scsi_Host *instance)
  *
- * Purpose : run through the various SCSI phases and do as the target 
- *      directs us to.  Operates on the currently connected command, 
- *      instance->connected.
+ * Purpose : run through the various SCSI phases and do as the target
+ * directs us to.  Operates on the currently connected command,
+ * instance->connected.
  *
  * Inputs : instance, instance for which we are doing commands
  *
- * Side effects : SCSI things happen, the disconnected queue will be 
- *      modified if a command disconnects, *instance->connected will
- *      change.
- *
- * XXX Note : we need to watch for bus free or a reset condition here 
- *      to recover from an unexpected bus free condition.
+ * Side effects : SCSI things happen, the disconnected queue will be
+ * modified if a command disconnects, *instance->connected will
+ * change.
  *
- * Locks: io_request_lock held by caller in IRQ mode
+ * XXX Note : we need to watch for bus free or a reset condition here
+ * to recover from an unexpected bus free condition.
  */
 
-static void NCR5380_information_transfer(struct Scsi_Host *instance) {
-	NCR5380_local_declare();
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *)instance->hostdata;
+static void NCR5380_information_transfer(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char msgout = NOP;
 	int sink = 0;
 	int len;
@@ -2010,13 +1764,11 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 #endif
 	unsigned char *data;
 	unsigned char phase, tmp, extended_msg[10], old_phase = 0xff;
-	struct scsi_cmnd *cmd = (struct scsi_cmnd *) hostdata->connected;
-	/* RvC: we need to set the end of the polling time */
-	unsigned long poll_time = jiffies + USLEEP_POLL;
+	struct scsi_cmnd *cmd;
 
-	NCR5380_setup(instance);
+	while ((cmd = hostdata->connected)) {
+		struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
 
-	while (1) {
 		tmp = NCR5380_read(STATUS_REG);
 		/* We only have a valid SCSI phase when REQ is asserted */
 		if (tmp & SR_REQ) {
@@ -2028,24 +1780,28 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 			if (sink && (phase != PHASE_MSGOUT)) {
 				NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(tmp));
 
-				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
-				while (NCR5380_read(STATUS_REG) & SR_REQ);
-				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
+				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN |
+				              ICR_ASSERT_ACK);
+				while (NCR5380_read(STATUS_REG) & SR_REQ)
+					;
+				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
+				              ICR_ASSERT_ATN);
 				sink = 0;
 				continue;
 			}
+
 			switch (phase) {
-			case PHASE_DATAIN:
 			case PHASE_DATAOUT:
 #if (NDEBUG & NDEBUG_NO_DATAOUT)
-				printk("scsi%d : NDEBUG_NO_DATAOUT set, attempted DATAOUT aborted\n", instance->host_no);
+				shost_printk(KERN_DEBUG, instance, "NDEBUG_NO_DATAOUT set, attempted DATAOUT aborted\n");
 				sink = 1;
 				do_abort(instance);
 				cmd->result = DID_ERROR << 16;
-				cmd->scsi_done(cmd);
+				complete_cmd(instance, cmd);
 				return;
 #endif
-				/* 
+			case PHASE_DATAIN:
+				/*
 				 * If there is no room left in the current buffer in the
 				 * scatter-gather list, move onto the next one.
 				 */
@@ -2055,10 +1811,13 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 					--cmd->SCp.buffers_residual;
 					cmd->SCp.this_residual = cmd->SCp.buffer->length;
 					cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
-					dprintk(NDEBUG_INFORMATION, "scsi%d : %d bytes and %d buffers left\n", instance->host_no, cmd->SCp.this_residual, cmd->SCp.buffers_residual);
+					dsprintk(NDEBUG_INFORMATION, instance, "%d bytes and %d buffers left\n",
+					         cmd->SCp.this_residual,
+					         cmd->SCp.buffers_residual);
 				}
+
 				/*
-				 * The preferred transfer method is going to be 
+				 * The preferred transfer method is going to be
 				 * PSEUDO-DMA for systems that are strictly PIO,
 				 * since we can let the hardware do the handshaking.
 				 *
@@ -2068,50 +1827,39 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 				 */
 
 #if defined(PSEUDO_DMA) || defined(REAL_DMA_POLL)
-				/* KLL
-				 * PSEUDO_DMA is defined here. If this is the g_NCR5380
-				 * driver then it will always be defined, so the
-				 * FLAG_NO_PSEUDO_DMA is used to inhibit PDMA in the base
-				 * NCR5380 case.  I think this is a fairly clean solution.
-				 * We supplement these 2 if's with the flag.
-				 */
-#ifdef NCR5380_dma_xfer_len
-				if (!cmd->device->borken && !(hostdata->flags & FLAG_NO_PSEUDO_DMA) && (transfersize = NCR5380_dma_xfer_len(instance, cmd)) != 0) {
-#else
-				transfersize = cmd->transfersize;
-
-#ifdef LIMIT_TRANSFERSIZE	/* If we have problems with interrupt service */
-				if (transfersize > 512)
-					transfersize = 512;
-#endif				/* LIMIT_TRANSFERSIZE */
-
-				if (!cmd->device->borken && transfersize && !(hostdata->flags & FLAG_NO_PSEUDO_DMA) && cmd->SCp.this_residual && !(cmd->SCp.this_residual % transfersize)) {
-					/* Limit transfers to 32K, for xx400 & xx406
-					 * pseudoDMA that transfers in 128 bytes blocks. */
-					if (transfersize > 32 * 1024)
-						transfersize = 32 * 1024;
-#endif
+				transfersize = 0;
+				if (!cmd->device->borken &&
+				    !(hostdata->flags & FLAG_NO_PSEUDO_DMA))
+					transfersize = NCR5380_dma_xfer_len(instance, cmd, phase);
+
+				if (transfersize) {
 					len = transfersize;
-					if (NCR5380_transfer_dma(instance, &phase, &len, (unsigned char **) &cmd->SCp.ptr)) {
+					if (NCR5380_transfer_dma(instance, &phase,
+					    &len, (unsigned char **)&cmd->SCp.ptr)) {
 						/*
-						 * If the watchdog timer fires, all future accesses to this
-						 * device will use the polled-IO.
+						 * If the watchdog timer fires, all future
+						 * accesses to this device will use the
+						 * polled-IO.
 						 */
 						scmd_printk(KERN_INFO, cmd,
-							    "switching to slow handshake\n");
+							"switching to slow handshake\n");
 						cmd->device->borken = 1;
-						NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 						sink = 1;
 						do_abort(instance);
 						cmd->result = DID_ERROR << 16;
-						cmd->scsi_done(cmd);
+						complete_cmd(instance, cmd);
 						/* XXX - need to source or sink data here, as appropriate */
 					} else
 						cmd->SCp.this_residual -= transfersize - len;
 				} else
 #endif				/* defined(PSEUDO_DMA) || defined(REAL_DMA_POLL) */
-					NCR5380_transfer_pio(instance, &phase, (int *) &cmd->SCp.this_residual, (unsigned char **)
-							     &cmd->SCp.ptr);
+				{
+					spin_unlock_irq(&hostdata->lock);
+					NCR5380_transfer_pio(instance, &phase,
+					                     (int *)&cmd->SCp.this_residual,
+					                     (unsigned char **)&cmd->SCp.ptr);
+					spin_lock_irq(&hostdata->lock);
+				}
 				break;
 			case PHASE_MSGIN:
 				len = 1;
@@ -2120,101 +1868,42 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 				cmd->SCp.Message = tmp;
 
 				switch (tmp) {
-					/*
-					 * Linking lets us reduce the time required to get the 
-					 * next command out to the device, hopefully this will
-					 * mean we don't waste another revolution due to the delays
-					 * required by ARBITRATION and another SELECTION.
-					 *
-					 * In the current implementation proposal, low level drivers
-					 * merely have to start the next command, pointed to by 
-					 * next_link, done() is called as with unlinked commands.
-					 */
-#ifdef LINKED
-				case LINKED_CMD_COMPLETE:
-				case LINKED_FLG_CMD_COMPLETE:
-					/* Accept message by clearing ACK */
-					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					dprintk(NDEBUG_LINKED, "scsi%d : target %d lun %llu linked command complete.\n", instance->host_no, cmd->device->id, cmd->device->lun);
-					/* 
-					 * Sanity check : A linked command should only terminate with
-					 * one of these messages if there are more linked commands
-					 * available.
-					 */
-					if (!cmd->next_link) {
-					    printk("scsi%d : target %d lun %llu linked command complete, no next_link\n" instance->host_no, cmd->device->id, cmd->device->lun);
-						sink = 1;
-						do_abort(instance);
-						return;
-					}
-					initialize_SCp(cmd->next_link);
-					/* The next command is still part of this process */
-					cmd->next_link->tag = cmd->tag;
-					cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-					dprintk(NDEBUG_LINKED, "scsi%d : target %d lun %llu linked request done, calling scsi_done().\n", instance->host_no, cmd->device->id, cmd->device->lun);
-					cmd->scsi_done(cmd);
-					cmd = hostdata->connected;
-					break;
-#endif				/* def LINKED */
 				case ABORT:
 				case COMMAND_COMPLETE:
 					/* Accept message by clearing ACK */
 					sink = 1;
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					hostdata->connected = NULL;
-					dprintk(NDEBUG_QUEUES, "scsi%d : command for target %d, lun %llu completed\n", instance->host_no, cmd->device->id, cmd->device->lun);
-					hostdata->busy[cmd->device->id] &= ~(1 << (cmd->device->lun & 0xFF));
-
-					/* 
-					 * I'm not sure what the correct thing to do here is : 
-					 * 
-					 * If the command that just executed is NOT a request 
-					 * sense, the obvious thing to do is to set the result
-					 * code to the values of the stored parameters.
-					 * 
-					 * If it was a REQUEST SENSE command, we need some way 
-					 * to differentiate between the failure code of the original
-					 * and the failure code of the REQUEST sense - the obvious
-					 * case is success, where we fall through and leave the result
-					 * code unchanged.
-					 * 
-					 * The non-obvious place is where the REQUEST SENSE failed 
-					 */
-
-					if (cmd->cmnd[0] != REQUEST_SENSE)
-						cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-					else if (status_byte(cmd->SCp.Status) != GOOD)
-						cmd->result = (cmd->result & 0x00ffff) | (DID_ERROR << 16);
+					dsprintk(NDEBUG_QUEUES, instance,
+					         "COMMAND COMPLETE %p target %d lun %llu\n",
+					         cmd, scmd_id(cmd), cmd->device->lun);
 
-					if ((cmd->cmnd[0] == REQUEST_SENSE) &&
-						hostdata->ses.cmd_len) {
-						scsi_eh_restore_cmnd(cmd, &hostdata->ses);
-						hostdata->ses.cmd_len = 0 ;
-					}
-
-					if ((cmd->cmnd[0] != REQUEST_SENSE) && (status_byte(cmd->SCp.Status) == CHECK_CONDITION)) {
-						scsi_eh_prep_cmnd(cmd, &hostdata->ses, NULL, 0, ~0);
-
-						dprintk(NDEBUG_AUTOSENSE, "scsi%d : performing request sense\n", instance->host_no);
+					hostdata->connected = NULL;
 
-						LIST(cmd, hostdata->issue_queue);
-						cmd->host_scribble = (unsigned char *)
-						    hostdata->issue_queue;
-						hostdata->issue_queue = (struct scsi_cmnd *) cmd;
-						dprintk(NDEBUG_QUEUES, "scsi%d : REQUEST SENSE added to head of issue queue\n", instance->host_no);
-					} else {
-						cmd->scsi_done(cmd);
+					cmd->result &= ~0xffff;
+					cmd->result |= cmd->SCp.Status;
+					cmd->result |= cmd->SCp.Message << 8;
+
+					if (cmd->cmnd[0] == REQUEST_SENSE)
+						complete_cmd(instance, cmd);
+					else {
+						if (cmd->SCp.Status == SAM_STAT_CHECK_CONDITION ||
+						    cmd->SCp.Status == SAM_STAT_COMMAND_TERMINATED) {
+							dsprintk(NDEBUG_QUEUES, instance, "autosense: adding cmd %p to tail of autosense queue\n",
+							         cmd);
+							list_add_tail(&ncmd->list,
+							              &hostdata->autosense);
+						} else
+							complete_cmd(instance, cmd);
 					}
 
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-					/* 
-					 * Restore phase bits to 0 so an interrupted selection, 
+					/*
+					 * Restore phase bits to 0 so an interrupted selection,
 					 * arbitration can resume.
 					 */
 					NCR5380_write(TARGET_COMMAND_REG, 0);
 
-					while ((NCR5380_read(STATUS_REG) & SR_BSY) && !hostdata->connected)
-						barrier();
+					/* Enable reselect interrupts */
+					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					return;
 				case MESSAGE_REJECT:
 					/* Accept message by clearing ACK */
@@ -2229,38 +1918,33 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 					default:
 						break;
 					}
-				case DISCONNECT:{
-						/* Accept message by clearing ACK */
-						NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-						cmd->device->disconnect = 1;
-						LIST(cmd, hostdata->disconnected_queue);
-						cmd->host_scribble = (unsigned char *)
-						    hostdata->disconnected_queue;
-						hostdata->connected = NULL;
-						hostdata->disconnected_queue = cmd;
-						dprintk(NDEBUG_QUEUES, "scsi%d : command for target %d lun %llu was moved from connected to" "  the disconnected_queue\n", instance->host_no, cmd->device->id, cmd->device->lun);
-						/* 
-						 * Restore phase bits to 0 so an interrupted selection, 
-						 * arbitration can resume.
-						 */
-						NCR5380_write(TARGET_COMMAND_REG, 0);
-
-						/* Enable reselect interrupts */
-						NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-						/* Wait for bus free to avoid nasty timeouts - FIXME timeout !*/
-						/* NCR538_poll_politely(instance, STATUS_REG, SR_BSY, 0, 30 * HZ); */
-						while ((NCR5380_read(STATUS_REG) & SR_BSY) && !hostdata->connected)
-							barrier();
-						return;
-					}
-					/* 
+					break;
+				case DISCONNECT:
+					/* Accept message by clearing ACK */
+					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+					hostdata->connected = NULL;
+					list_add(&ncmd->list, &hostdata->disconnected);
+					dsprintk(NDEBUG_INFORMATION | NDEBUG_QUEUES,
+					         instance, "connected command %p for target %d lun %llu moved to disconnected queue\n",
+					         cmd, scmd_id(cmd), cmd->device->lun);
+
+					/*
+					 * Restore phase bits to 0 so an interrupted selection,
+					 * arbitration can resume.
+					 */
+					NCR5380_write(TARGET_COMMAND_REG, 0);
+
+					/* Enable reselect interrupts */
+					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+					return;
+					/*
 					 * The SCSI data pointer is *IMPLICITLY* saved on a disconnect
-					 * operation, in violation of the SCSI spec so we can safely 
+					 * operation, in violation of the SCSI spec so we can safely
 					 * ignore SAVE/RESTORE pointers calls.
 					 *
-					 * Unfortunately, some disks violate the SCSI spec and 
+					 * Unfortunately, some disks violate the SCSI spec and
 					 * don't issue the required SAVE_POINTERS message before
-					 * disconnecting, and we have to break spec to remain 
+					 * disconnecting, and we have to break spec to remain
 					 * compatible.
 					 */
 				case SAVE_POINTERS:
@@ -2269,31 +1953,28 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 					break;
 				case EXTENDED_MESSAGE:
-/* 
- * Extended messages are sent in the following format :
- * Byte         
- * 0            EXTENDED_MESSAGE == 1
- * 1            length (includes one byte for code, doesn't 
- *              include first two bytes)
- * 2            code
- * 3..length+1  arguments
- *
- * Start the extended message buffer with the EXTENDED_MESSAGE
- * byte, since spi_print_msg() wants the whole thing.  
- */
+					/*
+					 * Start the message buffer with the EXTENDED_MESSAGE
+					 * byte, since spi_print_msg() wants the whole thing.
+					 */
 					extended_msg[0] = EXTENDED_MESSAGE;
 					/* Accept first byte by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					dprintk(NDEBUG_EXTENDED, "scsi%d : receiving extended message\n", instance->host_no);
+
+					spin_unlock_irq(&hostdata->lock);
+
+					dsprintk(NDEBUG_EXTENDED, instance, "receiving extended message\n");
 
 					len = 2;
 					data = extended_msg + 1;
 					phase = PHASE_MSGIN;
 					NCR5380_transfer_pio(instance, &phase, &len, &data);
+					dsprintk(NDEBUG_EXTENDED, instance, "length %d, code 0x%02x\n",
+					         (int)extended_msg[1],
+					         (int)extended_msg[2]);
 
-					dprintk(NDEBUG_EXTENDED, "scsi%d : length=%d, code=0x%02x\n", instance->host_no, (int) extended_msg[1], (int) extended_msg[2]);
-
-					if (!len && extended_msg[1] <= (sizeof(extended_msg) - 1)) {
+					if (!len && extended_msg[1] > 0 &&
+					    extended_msg[1] <= sizeof(extended_msg) - 2) {
 						/* Accept third byte by clearing ACK */
 						NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 						len = extended_msg[1] - 1;
@@ -2301,7 +1982,8 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 						phase = PHASE_MSGIN;
 
 						NCR5380_transfer_pio(instance, &phase, &len, &data);
-						dprintk(NDEBUG_EXTENDED, "scsi%d : message received, residual %d\n", instance->host_no, len);
+						dsprintk(NDEBUG_EXTENDED, instance, "message received, residual %d\n",
+						         len);
 
 						switch (extended_msg[2]) {
 						case EXTENDED_SDTR:
@@ -2311,34 +1993,42 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 							tmp = 0;
 						}
 					} else if (len) {
-						printk("scsi%d: error receiving extended message\n", instance->host_no);
+						shost_printk(KERN_ERR, instance, "error receiving extended message\n");
 						tmp = 0;
 					} else {
-						printk("scsi%d: extended message code %02x length %d is too long\n", instance->host_no, extended_msg[2], extended_msg[1]);
+						shost_printk(KERN_NOTICE, instance, "extended message code %02x length %d is too long\n",
+						             extended_msg[2], extended_msg[1]);
 						tmp = 0;
 					}
+
+					spin_lock_irq(&hostdata->lock);
+					if (!hostdata->connected)
+						return;
+
 					/* Fall through to reject message */
 
-					/* 
-					 * If we get something weird that we aren't expecting, 
+					/*
+					 * If we get something weird that we aren't expecting,
 					 * reject it.
 					 */
 				default:
 					if (!tmp) {
-						printk("scsi%d: rejecting message ", instance->host_no);
+						shost_printk(KERN_ERR, instance, "rejecting message ");
 						spi_print_msg(extended_msg);
 						printk("\n");
 					} else if (tmp != EXTENDED_MESSAGE)
 						scmd_printk(KERN_INFO, cmd,
-							"rejecting unknown message %02x\n",tmp);
+						            "rejecting unknown message %02x\n",
+						            tmp);
 					else
 						scmd_printk(KERN_INFO, cmd,
-							"rejecting unknown extended message code %02x, length %d\n", extended_msg[1], extended_msg[0]);
+						            "rejecting unknown extended message code %02x, length %d\n",
+						            extended_msg[1], extended_msg[0]);
 
 					msgout = MESSAGE_REJECT;
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 					break;
-				}	/* switch (tmp) */
+				} /* switch (tmp) */
 				break;
 			case PHASE_MSGOUT:
 				len = 1;
@@ -2346,10 +2036,9 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 				hostdata->last_message = msgout;
 				NCR5380_transfer_pio(instance, &phase, &len, &data);
 				if (msgout == ABORT) {
-					hostdata->busy[cmd->device->id] &= ~(1 << (cmd->device->lun & 0xFF));
 					hostdata->connected = NULL;
 					cmd->result = DID_ERROR << 16;
-					cmd->scsi_done(cmd);
+					complete_cmd(instance, cmd);
 					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					return;
 				}
@@ -2358,17 +2047,12 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 			case PHASE_CMDOUT:
 				len = cmd->cmd_len;
 				data = cmd->cmnd;
-				/* 
-				 * XXX for performance reasons, on machines with a 
-				 * PSEUDO-DMA architecture we should probably 
-				 * use the dma transfer function.  
+				/*
+				 * XXX for performance reasons, on machines with a
+				 * PSEUDO-DMA architecture we should probably
+				 * use the dma transfer function.
 				 */
 				NCR5380_transfer_pio(instance, &phase, &len, &data);
-				if (!cmd->device->disconnect && should_disconnect(cmd->cmnd[0])) {
-					NCR5380_set_timer(hostdata, USLEEP_SLEEP);
-					dprintk(NDEBUG_USLEEP, "scsi%d : issued command, sleeping until %lu\n", instance->host_no, hostdata->time_expires);
-					return;
-				}
 				break;
 			case PHASE_STATIN:
 				len = 1;
@@ -2377,46 +2061,37 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance) {
 				cmd->SCp.Status = tmp;
 				break;
 			default:
-				printk("scsi%d : unknown phase\n", instance->host_no);
+				shost_printk(KERN_ERR, instance, "unknown phase\n");
 				NCR5380_dprint(NDEBUG_ANY, instance);
-			}	/* switch(phase) */
-		}		/* if (tmp * SR_REQ) */
-		else {
-			/* RvC: go to sleep if polling time expired
-			 */
-			if (!cmd->device->disconnect && time_after_eq(jiffies, poll_time)) {
-				NCR5380_set_timer(hostdata, USLEEP_SLEEP);
-				dprintk(NDEBUG_USLEEP, "scsi%d : poll timed out, sleeping until %lu\n", instance->host_no, hostdata->time_expires);
-				return;
-			}
+			} /* switch(phase) */
+		} else {
+			spin_unlock_irq(&hostdata->lock);
+			NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
+			spin_lock_irq(&hostdata->lock);
 		}
-	}			/* while (1) */
+	}
 }
 
 /*
  * Function : void NCR5380_reselect (struct Scsi_Host *instance)
  *
- * Purpose : does reselection, initializing the instance->connected 
- *      field to point to the scsi_cmnd for which the I_T_L or I_T_L_Q
- *      nexus has been reestablished,
- *      
- * Inputs : instance - this instance of the NCR5380.
+ * Purpose : does reselection, initializing the instance->connected
+ * field to point to the scsi_cmnd for which the I_T_L or I_T_L_Q
+ * nexus has been reestablished,
  *
- * Locks: io_request_lock held by caller if IRQ driven
+ * Inputs : instance - this instance of the NCR5380.
  */
 
-static void NCR5380_reselect(struct Scsi_Host *instance) {
-	NCR5380_local_declare();
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *)
-	 instance->hostdata;
+static void NCR5380_reselect(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char target_mask;
 	unsigned char lun, phase;
 	int len;
 	unsigned char msg[3];
 	unsigned char *data;
-	struct scsi_cmnd *tmp = NULL, *prev;
-	int abort = 0;
-	NCR5380_setup(instance);
+	struct NCR5380_cmd *ncmd;
+	struct scsi_cmnd *tmp;
 
 	/*
 	 * Disable arbitration, etc. since the host adapter obviously
@@ -2424,12 +2099,12 @@ static void NCR5380_reselect(struct Scsi_Host *instance) {
 	 */
 
 	NCR5380_write(MODE_REG, MR_BASE);
-	hostdata->restart_select = 1;
 
 	target_mask = NCR5380_read(CURRENT_SCSI_DATA_REG) & ~(hostdata->id_mask);
-	dprintk(NDEBUG_SELECTION, "scsi%d : reselect\n", instance->host_no);
 
-	/* 
+	dsprintk(NDEBUG_RESELECTION, instance, "reselect\n");
+
+	/*
 	 * At this point, we have detected that our SCSI ID is on the bus,
 	 * SEL is true and BSY was false for at least one bus settle delay
 	 * (400 ns).
@@ -2439,103 +2114,110 @@ static void NCR5380_reselect(struct Scsi_Host *instance) {
 	 */
 
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY);
-
-	/* FIXME: timeout too long, must fail to workqueue */	
-	if(NCR5380_poll_politely(instance, STATUS_REG, SR_SEL, 0, 2*HZ)<0)
-		abort = 1;
-		
+	if (NCR5380_poll_politely(instance,
+	                          STATUS_REG, SR_SEL, 0, 2 * HZ) < 0) {
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+		return;
+	}
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 
 	/*
 	 * Wait for target to go into MSGIN.
-	 * FIXME: timeout needed and fail to work queeu
 	 */
 
-	if(NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, 2*HZ))
-		abort = 1;
+	if (NCR5380_poll_politely(instance,
+	                          STATUS_REG, SR_REQ, SR_REQ, 2 * HZ) < 0) {
+		do_abort(instance);
+		return;
+	}
 
 	len = 1;
 	data = msg;
 	phase = PHASE_MSGIN;
 	NCR5380_transfer_pio(instance, &phase, &len, &data);
 
+	if (len) {
+		do_abort(instance);
+		return;
+	}
+
 	if (!(msg[0] & 0x80)) {
-		printk(KERN_ERR "scsi%d : expecting IDENTIFY message, got ", instance->host_no);
+		shost_printk(KERN_ERR, instance, "expecting IDENTIFY message, got ");
 		spi_print_msg(msg);
-		abort = 1;
-	} else {
-		/* Accept message by clearing ACK */
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		lun = (msg[0] & 0x07);
+		printk("\n");
+		do_abort(instance);
+		return;
+	}
+	lun = msg[0] & 0x07;
 
-		/* 
-		 * We need to add code for SCSI-II to track which devices have
-		 * I_T_L_Q nexuses established, and which have simple I_T_L
-		 * nexuses so we can chose to do additional data transfer.
-		 */
+	/*
+	 * We need to add code for SCSI-II to track which devices have
+	 * I_T_L_Q nexuses established, and which have simple I_T_L
+	 * nexuses so we can chose to do additional data transfer.
+	 */
 
-		/* 
-		 * Find the command corresponding to the I_T_L or I_T_L_Q  nexus we 
-		 * just reestablished, and remove it from the disconnected queue.
-		 */
+	/*
+	 * Find the command corresponding to the I_T_L or I_T_L_Q  nexus we
+	 * just reestablished, and remove it from the disconnected queue.
+	 */
 
+	tmp = NULL;
+	list_for_each_entry(ncmd, &hostdata->disconnected, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
 
-		for (tmp = (struct scsi_cmnd *) hostdata->disconnected_queue, prev = NULL; tmp; prev = tmp, tmp = (struct scsi_cmnd *) tmp->host_scribble)
-			if ((target_mask == (1 << tmp->device->id)) && (lun == (u8)tmp->device->lun)
-			    ) {
-				if (prev) {
-					REMOVE(prev, prev->host_scribble, tmp, tmp->host_scribble);
-					prev->host_scribble = tmp->host_scribble;
-				} else {
-					REMOVE(-1, hostdata->disconnected_queue, tmp, tmp->host_scribble);
-					hostdata->disconnected_queue = (struct scsi_cmnd *) tmp->host_scribble;
-				}
-				tmp->host_scribble = NULL;
-				break;
-			}
-		if (!tmp) {
-			printk(KERN_ERR "scsi%d : warning : target bitmask %02x lun %d not in disconnect_queue.\n", instance->host_no, target_mask, lun);
-			/* 
-			 * Since we have an established nexus that we can't do anything with,
-			 * we must abort it.  
-			 */
-			abort = 1;
+		if (target_mask == (1 << scmd_id(cmd)) &&
+		    lun == (u8)cmd->device->lun) {
+			list_del(&ncmd->list);
+			tmp = cmd;
+			break;
 		}
 	}
 
-	if (abort) {
-		do_abort(instance);
+	if (tmp) {
+		dsprintk(NDEBUG_RESELECTION | NDEBUG_QUEUES, instance,
+		         "reselect: removed %p from disconnected queue\n", tmp);
 	} else {
-		hostdata->connected = tmp;
-		dprintk(NDEBUG_RESELECTION, "scsi%d : nexus established, target = %d, lun = %llu, tag = %d\n", instance->host_no, tmp->device->id, tmp->device->lun, tmp->tag);
+		shost_printk(KERN_ERR, instance, "target bitmask 0x%02x lun %d not in disconnected queue.\n",
+		             target_mask, lun);
+		/*
+		 * Since we have an established nexus that we can't do anything
+		 * with, we must abort it.
+		 */
+		do_abort(instance);
+		return;
 	}
+
+	/* Accept message by clearing ACK */
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+
+	hostdata->connected = tmp;
+	dsprintk(NDEBUG_RESELECTION, instance, "nexus established, target %d, lun %llu, tag %d\n",
+	         scmd_id(tmp), tmp->device->lun, tmp->tag);
 }
 
 /*
  * Function : void NCR5380_dma_complete (struct Scsi_Host *instance)
  *
  * Purpose : called by interrupt handler when DMA finishes or a phase
- *      mismatch occurs (which would finish the DMA transfer).  
+ * mismatch occurs (which would finish the DMA transfer).
  *
  * Inputs : instance - this instance of the NCR5380.
  *
  * Returns : pointer to the scsi_cmnd structure for which the I_T_L
- *      nexus has been reestablished, on failure NULL is returned.
+ * nexus has been reestablished, on failure NULL is returned.
  */
 
 #ifdef REAL_DMA
 static void NCR5380_dma_complete(NCR5380_instance * instance) {
-	NCR5380_local_declare();
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int transferred;
-	NCR5380_setup(instance);
 
 	/*
 	 * XXX this might not be right.
 	 *
 	 * Wait for final byte to transfer, ie wait for ACK to go false.
 	 *
-	 * We should use the Last Byte Sent bit, unfortunately this is 
+	 * We should use the Last Byte Sent bit, unfortunately this is
 	 * not available on the 5380/5381 (only the various CMOS chips)
 	 *
 	 * FIXME: timeout, and need to handle long timeout/irq case
@@ -2543,7 +2225,6 @@ static void NCR5380_dma_complete(NCR5380_instance * instance) {
 
 	NCR5380_poll_politely(instance, BUS_AND_STATUS_REG, BASR_ACK, 0, 5*HZ);
 
-	NCR5380_write(MODE_REG, MR_BASE);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 
 	/*
@@ -2560,190 +2241,251 @@ static void NCR5380_dma_complete(NCR5380_instance * instance) {
 }
 #endif				/* def REAL_DMA */
 
-/*
- * Function : int NCR5380_abort (struct scsi_cmnd *cmd)
- *
- * Purpose : abort a command
- *
- * Inputs : cmd - the scsi_cmnd to abort, code - code to set the
- *      host byte of the result field to, if zero DID_ABORTED is
- *      used.
- *
- * Returns : SUCCESS - success, FAILED on failure.
- *
- *	XXX - there is no way to abort the command that is currently
- *	connected, you have to wait for it to complete.  If this is
- *	a problem, we could implement longjmp() / setjmp(), setjmp()
- *	called where the loop started in NCR5380_main().
- *
- * Locks: host lock taken by caller
+/**
+ * list_find_cmd - test for presence of a command in a linked list
+ * @haystack: list of commands
+ * @needle: command to search for
  */
 
-static int NCR5380_abort(struct scsi_cmnd *cmd)
+static bool list_find_cmd(struct list_head *haystack,
+                          struct scsi_cmnd *needle)
 {
-	NCR5380_local_declare();
-	struct Scsi_Host *instance = cmd->device->host;
-	struct NCR5380_hostdata *hostdata = (struct NCR5380_hostdata *) instance->hostdata;
-	struct scsi_cmnd *tmp, **prev;
+	struct NCR5380_cmd *ncmd;
 
-	scmd_printk(KERN_WARNING, cmd, "aborting command\n");
+	list_for_each_entry(ncmd, haystack, list)
+		if (NCR5380_to_scmd(ncmd) == needle)
+			return true;
+	return false;
+}
 
-	NCR5380_print_status(instance);
+/**
+ * list_remove_cmd - remove a command from linked list
+ * @haystack: list of commands
+ * @needle: command to remove
+ */
 
-	NCR5380_setup(instance);
+static bool list_del_cmd(struct list_head *haystack,
+                         struct scsi_cmnd *needle)
+{
+	if (list_find_cmd(haystack, needle)) {
+		struct NCR5380_cmd *ncmd = scsi_cmd_priv(needle);
 
-	dprintk(NDEBUG_ABORT, "scsi%d : abort called\n", instance->host_no);
-	dprintk(NDEBUG_ABORT, "        basr 0x%X, sr 0x%X\n", NCR5380_read(BUS_AND_STATUS_REG), NCR5380_read(STATUS_REG));
+		list_del(&ncmd->list);
+		return true;
+	}
+	return false;
+}
 
-#if 0
-/*
- * Case 1 : If the command is the currently executing command, 
- * we'll set the aborted flag and return control so that 
- * information transfer routine can exit cleanly.
+/**
+ * NCR5380_abort - scsi host eh_abort_handler() method
+ * @cmd: the command to be aborted
+ *
+ * Try to abort a given command by removing it from queues and/or sending
+ * the target an abort message. This may not succeed in causing a target
+ * to abort the command. Nonetheless, the low-level driver must forget about
+ * the command because the mid-layer reclaims it and it may be re-issued.
+ *
+ * The normal path taken by a command is as follows. For EH we trace this
+ * same path to locate and abort the command.
+ *
+ * unissued -> selecting -> [unissued -> selecting ->]... connected ->
+ * [disconnected -> connected ->]...
+ * [autosense -> connected ->] done
+ *
+ * If cmd is unissued then just remove it.
+ * If cmd is disconnected, try to select the target.
+ * If cmd is connected, try to send an abort message.
+ * If cmd is waiting for autosense, give it a chance to complete but check
+ * that it isn't left connected.
+ * If cmd was not found at all then presumably it has already been completed,
+ * in which case return SUCCESS to try to avoid further EH measures.
+ * If the command has not completed yet, we must not fail to find it.
  */
 
-	if (hostdata->connected == cmd) {
-		dprintk(NDEBUG_ABORT, "scsi%d : aborting connected command\n", instance->host_no);
-		hostdata->aborted = 1;
-/*
- * We should perform BSY checking, and make sure we haven't slipped
- * into BUS FREE.
- */
+static int NCR5380_abort(struct scsi_cmnd *cmd)
+{
+	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	unsigned long flags;
+	int result = SUCCESS;
 
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_ASSERT_ATN);
-/* 
- * Since we can't change phases until we've completed the current 
- * handshake, we have to source or sink a byte of data if the current
- * phase is not MSGOUT.
- */
+	spin_lock_irqsave(&hostdata->lock, flags);
 
-/* 
- * Return control to the executing NCR drive so we can clear the
- * aborted flag and get back into our main loop.
- */
+#if (NDEBUG & NDEBUG_ANY)
+	scmd_printk(KERN_INFO, cmd, __func__);
+#endif
+	NCR5380_dprint(NDEBUG_ANY, instance);
+	NCR5380_dprint_phase(NDEBUG_ANY, instance);
 
-		return SUCCESS;
+	if (list_del_cmd(&hostdata->unissued, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: removed %p from issue queue\n", cmd);
+		cmd->result = DID_ABORT << 16;
+		cmd->scsi_done(cmd); /* No tag or busy flag to worry about */
 	}
-#endif
 
-/* 
- * Case 2 : If the command hasn't been issued yet, we simply remove it 
- *          from the issue queue.
- */
- 
-	dprintk(NDEBUG_ABORT, "scsi%d : abort going into loop.\n", instance->host_no);
-	for (prev = (struct scsi_cmnd **) &(hostdata->issue_queue), tmp = (struct scsi_cmnd *) hostdata->issue_queue; tmp; prev = (struct scsi_cmnd **) &(tmp->host_scribble), tmp = (struct scsi_cmnd *) tmp->host_scribble)
-		if (cmd == tmp) {
-			REMOVE(5, *prev, tmp, tmp->host_scribble);
-			(*prev) = (struct scsi_cmnd *) tmp->host_scribble;
-			tmp->host_scribble = NULL;
-			tmp->result = DID_ABORT << 16;
-			dprintk(NDEBUG_ABORT, "scsi%d : abort removed command from issue queue.\n", instance->host_no);
-			tmp->scsi_done(tmp);
-			return SUCCESS;
+	if (hostdata->selecting == cmd) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: cmd %p == selecting\n", cmd);
+		hostdata->selecting = NULL;
+		cmd->result = DID_ABORT << 16;
+		complete_cmd(instance, cmd);
+		goto out;
+	}
+
+	if (list_del_cmd(&hostdata->disconnected, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: removed %p from disconnected list\n", cmd);
+		cmd->result = DID_ERROR << 16;
+		if (!hostdata->connected)
+			NCR5380_select(instance, cmd);
+		if (hostdata->connected != cmd) {
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
+		}
+	}
+
+	if (hostdata->connected == cmd) {
+		dsprintk(NDEBUG_ABORT, instance, "abort: cmd %p is connected\n", cmd);
+		hostdata->connected = NULL;
+		if (do_abort(instance)) {
+			set_host_byte(cmd, DID_ERROR);
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
 		}
-#if (NDEBUG  & NDEBUG_ABORT)
-	/* KLL */
-		else if (prev == tmp)
-			printk(KERN_ERR "scsi%d : LOOP\n", instance->host_no);
+		set_host_byte(cmd, DID_ABORT);
+#ifdef REAL_DMA
+		hostdata->dma_len = 0;
 #endif
+		if (cmd->cmnd[0] == REQUEST_SENSE)
+			complete_cmd(instance, cmd);
+		else {
+			struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
 
-/* 
- * Case 3 : If any commands are connected, we're going to fail the abort
- *          and let the high level SCSI driver retry at a later time or 
- *          issue a reset.
- *
- *          Timeouts, and therefore aborted commands, will be highly unlikely
- *          and handling them cleanly in this situation would make the common
- *          case of noresets less efficient, and would pollute our code.  So,
- *          we fail.
- */
+			/* Perform autosense for this command */
+			list_add(&ncmd->list, &hostdata->autosense);
+		}
+	}
 
-	if (hostdata->connected) {
-		dprintk(NDEBUG_ABORT, "scsi%d : abort failed, command connected.\n", instance->host_no);
-		return FAILED;
+	if (list_find_cmd(&hostdata->autosense, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: found %p on sense queue\n", cmd);
+		spin_unlock_irqrestore(&hostdata->lock, flags);
+		queue_work(hostdata->work_q, &hostdata->main_task);
+		msleep(1000);
+		spin_lock_irqsave(&hostdata->lock, flags);
+		if (list_del_cmd(&hostdata->autosense, cmd)) {
+			dsprintk(NDEBUG_ABORT, instance,
+			         "abort: removed %p from sense queue\n", cmd);
+			set_host_byte(cmd, DID_ABORT);
+			complete_cmd(instance, cmd);
+			goto out;
+		}
 	}
-/*
- * Case 4: If the command is currently disconnected from the bus, and 
- *      there are no connected commands, we reconnect the I_T_L or 
- *      I_T_L_Q nexus associated with it, go into message out, and send 
- *      an abort message.
- *
- * This case is especially ugly. In order to reestablish the nexus, we
- * need to call NCR5380_select().  The easiest way to implement this 
- * function was to abort if the bus was busy, and let the interrupt
- * handler triggered on the SEL for reselect take care of lost arbitrations
- * where necessary, meaning interrupts need to be enabled.
- *
- * When interrupts are enabled, the queues may change - so we 
- * can't remove it from the disconnected queue before selecting it
- * because that could cause a failure in hashing the nexus if that 
- * device reselected.
- * 
- * Since the queues may change, we can't use the pointers from when we
- * first locate it.
- *
- * So, we must first locate the command, and if NCR5380_select()
- * succeeds, then issue the abort, relocate the command and remove
- * it from the disconnected queue.
- */
 
-	for (tmp = (struct scsi_cmnd *) hostdata->disconnected_queue; tmp; tmp = (struct scsi_cmnd *) tmp->host_scribble)
-		if (cmd == tmp) {
-			dprintk(NDEBUG_ABORT, "scsi%d : aborting disconnected command.\n", instance->host_no);
+	if (hostdata->connected == cmd) {
+		dsprintk(NDEBUG_ABORT, instance, "abort: cmd %p is connected\n", cmd);
+		hostdata->connected = NULL;
+		if (do_abort(instance)) {
+			set_host_byte(cmd, DID_ERROR);
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
+		}
+		set_host_byte(cmd, DID_ABORT);
+#ifdef REAL_DMA
+		hostdata->dma_len = 0;
+#endif
+		complete_cmd(instance, cmd);
+	}
 
-			if (NCR5380_select(instance, cmd))
-				return FAILED;
-			dprintk(NDEBUG_ABORT, "scsi%d : nexus reestablished.\n", instance->host_no);
+out:
+	if (result == FAILED)
+		dsprintk(NDEBUG_ABORT, instance, "abort: failed to abort %p\n", cmd);
+	else
+		dsprintk(NDEBUG_ABORT, instance, "abort: successfully aborted %p\n", cmd);
 
-			do_abort(instance);
+	queue_work(hostdata->work_q, &hostdata->main_task);
+	spin_unlock_irqrestore(&hostdata->lock, flags);
 
-			for (prev = (struct scsi_cmnd **) &(hostdata->disconnected_queue), tmp = (struct scsi_cmnd *) hostdata->disconnected_queue; tmp; prev = (struct scsi_cmnd **) &(tmp->host_scribble), tmp = (struct scsi_cmnd *) tmp->host_scribble)
-				if (cmd == tmp) {
-					REMOVE(5, *prev, tmp, tmp->host_scribble);
-					*prev = (struct scsi_cmnd *) tmp->host_scribble;
-					tmp->host_scribble = NULL;
-					tmp->result = DID_ABORT << 16;
-					tmp->scsi_done(tmp);
-					return SUCCESS;
-				}
-		}
-/*
- * Case 5 : If we reached this point, the command was not found in any of 
- *          the queues.
- *
- * We probably reached this point because of an unlikely race condition
- * between the command completing successfully and the abortion code,
- * so we won't panic, but we will notify the user in case something really
- * broke.
- */
-	printk(KERN_WARNING "scsi%d : warning : SCSI command probably completed successfully\n"
-			"         before abortion\n", instance->host_no);
-	return FAILED;
+	return result;
 }
 
 
-/* 
- * Function : int NCR5380_bus_reset (struct scsi_cmnd *cmd)
- * 
- * Purpose : reset the SCSI bus.
- *
- * Returns : SUCCESS
+/**
+ * NCR5380_bus_reset - reset the SCSI bus
+ * @cmd: SCSI command undergoing EH
  *
- * Locks: host lock taken by caller
+ * Returns SUCCESS
  */
 
 static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
 {
 	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int i;
+	unsigned long flags;
+	struct NCR5380_cmd *ncmd;
 
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-	NCR5380_print_status(instance);
+	spin_lock_irqsave(&hostdata->lock, flags);
+
+#if (NDEBUG & NDEBUG_ANY)
+	scmd_printk(KERN_INFO, cmd, __func__);
+#endif
+	NCR5380_dprint(NDEBUG_ANY, instance);
+	NCR5380_dprint_phase(NDEBUG_ANY, instance);
 
-	spin_lock_irq(instance->host_lock);
 	do_reset(instance);
-	spin_unlock_irq(instance->host_lock);
+
+	/* reset NCR registers */
+	NCR5380_write(MODE_REG, MR_BASE);
+	NCR5380_write(TARGET_COMMAND_REG, 0);
+	NCR5380_write(SELECT_ENABLE_REG, 0);
+
+	/* After the reset, there are no more connected or disconnected commands
+	 * and no busy units; so clear the low-level status here to avoid
+	 * conflicts when the mid-level code tries to wake up the affected
+	 * commands!
+	 */
+
+	hostdata->selecting = NULL;
+
+	list_for_each_entry(ncmd, &hostdata->disconnected, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
+
+		set_host_byte(cmd, DID_RESET);
+		cmd->scsi_done(cmd);
+	}
+
+	list_for_each_entry(ncmd, &hostdata->autosense, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
+
+		set_host_byte(cmd, DID_RESET);
+		cmd->scsi_done(cmd);
+	}
+
+	if (hostdata->connected) {
+		set_host_byte(hostdata->connected, DID_RESET);
+		complete_cmd(instance, hostdata->connected);
+		hostdata->connected = NULL;
+	}
+
+	if (hostdata->sensing) {
+		set_host_byte(hostdata->connected, DID_RESET);
+		complete_cmd(instance, hostdata->sensing);
+		hostdata->sensing = NULL;
+	}
+
+	for (i = 0; i < 8; ++i)
+		hostdata->busy[i] = 0;
+#ifdef REAL_DMA
+	hostdata->dma_len = 0;
+#endif
+
+	queue_work(hostdata->work_q, &hostdata->main_task);
+	spin_unlock_irqrestore(&hostdata->lock, flags);
 
 	return SUCCESS;
 }
diff --git a/drivers/scsi/NCR5380.h b/drivers/scsi/NCR5380.h
index 162112dd1bf8..a79288682a74 100644
--- a/drivers/scsi/NCR5380.h
+++ b/drivers/scsi/NCR5380.h
@@ -22,8 +22,13 @@
 #ifndef NCR5380_H
 #define NCR5380_H
 
+#include <linux/delay.h>
 #include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/workqueue.h>
+#include <scsi/scsi_dbg.h>
 #include <scsi/scsi_eh.h>
+#include <scsi/scsi_transport_spi.h>
 
 #define NDEBUG_ARBITRATION	0x1
 #define NDEBUG_AUTOSENSE	0x2
@@ -158,8 +163,7 @@
 /* Write any value to this register to start an ini mode DMA receive */
 #define START_DMA_INITIATOR_RECEIVE_REG 7	/* wo */
 
-#define C400_CONTROL_STATUS_REG NCR53C400_register_offset-8	/* rw */
-
+/* NCR 53C400(A) Control Status Register bits: */
 #define CSR_RESET              0x80	/* wo  Resets 53c400 */
 #define CSR_53C80_REG          0x80	/* ro  5380 registers busy */
 #define CSR_TRANS_DIR          0x40	/* rw  Data transfer direction */
@@ -176,16 +180,6 @@
 #define CSR_BASE CSR_53C80_INTR
 #endif
 
-/* Number of 128-byte blocks to be transferred */
-#define C400_BLOCK_COUNTER_REG   NCR53C400_register_offset-7	/* rw */
-
-/* Resume transfer after disconnect */
-#define C400_RESUME_TRANSFER_REG NCR53C400_register_offset-6	/* wo */
-
-/* Access to host buffer stack */
-#define C400_HOST_BUFFER         NCR53C400_register_offset-4	/* rw */
-
-
 /* Note : PHASE_* macros are based on the values of the STATUS register */
 #define PHASE_MASK 	(SR_MSG | SR_CD | SR_IO)
 
@@ -205,16 +199,6 @@
 
 #define PHASE_SR_TO_TCR(phase) ((phase) >> 2)
 
-/*
- * The internal should_disconnect() function returns these based on the 
- * expected length of a disconnect if a device supports disconnect/
- * reconnect.
- */
-
-#define DISCONNECT_NONE		0
-#define DISCONNECT_TIME_TO_DATA	1
-#define DISCONNECT_LONG		2
-
 /* 
  * "Special" value for the (unsigned char) command tag, to indicate
  * I_T_L nexus instead of I_T_L_Q.
@@ -236,15 +220,11 @@
 #define NO_IRQ		0
 #endif
 
-#define FLAG_HAS_LAST_BYTE_SENT		1	/* NCR53c81 or better */
-#define FLAG_CHECK_LAST_BYTE_SENT	2	/* Only test once */
-#define FLAG_NCR53C400			4	/* NCR53c400 */
+#define FLAG_NO_DMA_FIXUP		1	/* No DMA errata workarounds */
 #define FLAG_NO_PSEUDO_DMA		8	/* Inhibit DMA */
-#define FLAG_DTC3181E			16	/* DTC3181E */
 #define FLAG_LATE_DMA_SETUP		32	/* Setup NCR before DMA H/W */
 #define FLAG_TAGGED_QUEUING		64	/* as X3T9.2 spelled it */
-
-#ifndef ASM
+#define FLAG_TOSHIBA_DELAY		128	/* Allow for borken CD-ROMs */
 
 #ifdef SUPPORT_TAGS
 struct tag_alloc {
@@ -258,33 +238,24 @@ struct NCR5380_hostdata {
 	NCR5380_implementation_fields;		/* implementation specific */
 	struct Scsi_Host *host;			/* Host backpointer */
 	unsigned char id_mask, id_higher_mask;	/* 1 << id, all bits greater */
-	unsigned char targets_present;		/* targets we have connected
-						   to, so we can call a select
-						   failure a retryable condition */
-	volatile unsigned char busy[8];		/* index = target, bit = lun */
+	unsigned char busy[8];			/* index = target, bit = lun */
 #if defined(REAL_DMA) || defined(REAL_DMA_POLL)
-	volatile int dma_len;			/* requested length of DMA */
+	int dma_len;				/* requested length of DMA */
 #endif
-	volatile unsigned char last_message;	/* last message OUT */
-	volatile struct scsi_cmnd *connected;	/* currently connected command */
-	volatile struct scsi_cmnd *issue_queue;	/* waiting to be issued */
-	volatile struct scsi_cmnd *disconnected_queue;	/* waiting for reconnect */
-	volatile int restart_select;		/* we have disconnected,
-						   used to restart 
-						   NCR5380_select() */
-	volatile unsigned aborted:1;		/* flag, says aborted */
+	unsigned char last_message;		/* last message OUT */
+	struct scsi_cmnd *connected;		/* currently connected cmnd */
+	struct scsi_cmnd *selecting;		/* cmnd to be connected */
+	struct list_head unissued;		/* waiting to be issued */
+	struct list_head autosense;		/* priority issue queue */
+	struct list_head disconnected;		/* waiting for reconnect */
+	spinlock_t lock;			/* protects this struct */
 	int flags;
-	unsigned long time_expires;		/* in jiffies, set prior to sleeping */
-	int select_time;			/* timer in select for target response */
-	volatile struct scsi_cmnd *selecting;
-	struct delayed_work coroutine;		/* our co-routine */
 	struct scsi_eh_save ses;
+	struct scsi_cmnd *sensing;
 	char info[256];
 	int read_overruns;                /* number of bytes to cut from a
 	                                   * transfer to handle chip overruns */
-	int retain_dma_intr;
 	struct work_struct main_task;
-	volatile int main_running;
 #ifdef SUPPORT_TAGS
 	struct tag_alloc TagAlloc[8][8];	/* 8 targets and 8 LUNs */
 #endif
@@ -292,10 +263,23 @@ struct NCR5380_hostdata {
 	unsigned spin_max_r;
 	unsigned spin_max_w;
 #endif
+	struct workqueue_struct *work_q;
+	unsigned long accesses_per_ms;	/* chip register accesses per ms */
 };
 
 #ifdef __KERNEL__
 
+struct NCR5380_cmd {
+	struct list_head list;
+};
+
+#define NCR5380_CMD_SIZE		(sizeof(struct NCR5380_cmd))
+
+static inline struct scsi_cmnd *NCR5380_to_scmd(struct NCR5380_cmd *ncmd_ptr)
+{
+	return ((struct scsi_cmnd *)ncmd_ptr) - 1;
+}
+
 #ifndef NDEBUG
 #define NDEBUG (0)
 #endif
@@ -304,6 +288,11 @@ struct NCR5380_hostdata {
 	do { if ((NDEBUG) & (flg)) \
 		printk(KERN_DEBUG fmt, ## __VA_ARGS__); } while (0)
 
+#define dsprintk(flg, host, fmt, ...) \
+	do { if ((NDEBUG) & (flg)) \
+		shost_printk(KERN_DEBUG, host, fmt, ## __VA_ARGS__); \
+	} while (0)
+
 #if NDEBUG
 #define NCR5380_dprint(flg, arg) \
 	do { if ((NDEBUG) & (flg)) NCR5380_print(arg); } while (0)
@@ -320,6 +309,7 @@ static void NCR5380_print(struct Scsi_Host *instance);
 static int NCR5380_probe_irq(struct Scsi_Host *instance, int possible);
 #endif
 static int NCR5380_init(struct Scsi_Host *instance, int flags);
+static int NCR5380_maybe_reset_bus(struct Scsi_Host *);
 static void NCR5380_exit(struct Scsi_Host *instance);
 static void NCR5380_information_transfer(struct Scsi_Host *instance);
 #ifndef DONT_USE_INTR
@@ -328,7 +318,7 @@ static irqreturn_t NCR5380_intr(int irq, void *dev_id);
 static void NCR5380_main(struct work_struct *work);
 static const char *NCR5380_info(struct Scsi_Host *instance);
 static void NCR5380_reselect(struct Scsi_Host *instance);
-static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd);
+static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *, struct scsi_cmnd *);
 #if defined(PSEUDO_DMA) || defined(REAL_DMA) || defined(REAL_DMA_POLL)
 static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data);
 #endif
@@ -443,5 +433,4 @@ static __inline__ int NCR5380_pc_dma_residual(struct Scsi_Host *instance)
 #endif				/* defined(i386) || defined(__alpha__) */
 #endif				/* defined(REAL_DMA)  */
 #endif				/* __KERNEL__ */
-#endif				/* ndef ASM */
 #endif				/* NCR5380_H */
diff --git a/drivers/scsi/aic7xxx/aic79xx.h b/drivers/scsi/aic7xxx/aic79xx.h
index df2e0e5367d2..d47b527b25dd 100644
--- a/drivers/scsi/aic7xxx/aic79xx.h
+++ b/drivers/scsi/aic7xxx/aic79xx.h
@@ -624,7 +624,7 @@ struct scb {
 };
 
 TAILQ_HEAD(scb_tailq, scb);
-LIST_HEAD(scb_list, scb);
+BSD_LIST_HEAD(scb_list, scb);
 
 struct scb_data {
 	/*
@@ -1069,7 +1069,7 @@ struct ahd_softc {
 	/*
 	 * SCBs that have been sent to the controller
 	 */
-	LIST_HEAD(, scb)	  pending_scbs;
+	BSD_LIST_HEAD(, scb)	  pending_scbs;
 
 	/*
 	 * Current register window mode information.
diff --git a/drivers/scsi/aic7xxx/aic79xx_osm.h b/drivers/scsi/aic7xxx/aic79xx_osm.h
index c58fa33c6592..728193a42e6e 100644
--- a/drivers/scsi/aic7xxx/aic79xx_osm.h
+++ b/drivers/scsi/aic7xxx/aic79xx_osm.h
@@ -65,11 +65,6 @@
 /* Core SCSI definitions */
 #define AIC_LIB_PREFIX ahd
 
-/* Name space conflict with BSD queue macros */
-#ifdef LIST_HEAD
-#undef LIST_HEAD
-#endif
-
 #include "cam.h"
 #include "queue.h"
 #include "scsi_message.h"
diff --git a/drivers/scsi/aic7xxx/aic7xxx.h b/drivers/scsi/aic7xxx/aic7xxx.h
index f695774645c1..4ce4e903a759 100644
--- a/drivers/scsi/aic7xxx/aic7xxx.h
+++ b/drivers/scsi/aic7xxx/aic7xxx.h
@@ -916,7 +916,7 @@ struct ahc_softc {
 	/*
 	 * SCBs that have been sent to the controller
 	 */
-	LIST_HEAD(, scb)	  pending_scbs;
+	BSD_LIST_HEAD(, scb)	  pending_scbs;
 
 	/*
 	 * Counting lock for deferring the release of additional
diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm.h b/drivers/scsi/aic7xxx/aic7xxx_osm.h
index bc4cca92ff04..54c702864103 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_osm.h
+++ b/drivers/scsi/aic7xxx/aic7xxx_osm.h
@@ -82,11 +82,6 @@
 /* Core SCSI definitions */
 #define AIC_LIB_PREFIX ahc
 
-/* Name space conflict with BSD queue macros */
-#ifdef LIST_HEAD
-#undef LIST_HEAD
-#endif
-
 #include "cam.h"
 #include "queue.h"
 #include "scsi_message.h"
diff --git a/drivers/scsi/aic7xxx/queue.h b/drivers/scsi/aic7xxx/queue.h
index 8adf8003a164..ba602981f193 100644
--- a/drivers/scsi/aic7xxx/queue.h
+++ b/drivers/scsi/aic7xxx/queue.h
@@ -246,7 +246,7 @@ struct {								\
 /*
  * List declarations.
  */
-#define	LIST_HEAD(name, type)						\
+#define	BSD_LIST_HEAD(name, type)					\
 struct name {								\
 	struct type *lh_first;	/* first element */			\
 }
diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c
index d28d6c0f18c0..221f18c5df93 100644
--- a/drivers/scsi/arm/cumana_1.c
+++ b/drivers/scsi/arm/cumana_1.c
@@ -4,9 +4,7 @@
  * Copyright 1995-2002, Russell King
  */
 #include <linux/module.h>
-#include <linux/signal.h>
 #include <linux/ioport.h>
-#include <linux/delay.h>
 #include <linux/blkdev.h>
 #include <linux/init.h>
 
@@ -15,15 +13,14 @@
 
 #include <scsi/scsi_host.h>
 
-#include <scsi/scsicam.h>
-
 #define PSEUDO_DMA
 
 #define priv(host)			((struct NCR5380_hostdata *)(host)->hostdata)
-#define NCR5380_local_declare()		struct Scsi_Host *_instance
-#define NCR5380_setup(instance)		_instance = instance
-#define NCR5380_read(reg)		cumanascsi_read(_instance, reg)
-#define NCR5380_write(reg, value)	cumanascsi_write(_instance, reg, value)
+#define NCR5380_read(reg)		cumanascsi_read(instance, reg)
+#define NCR5380_write(reg, value)	cumanascsi_write(instance, reg, value)
+
+#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
+
 #define NCR5380_intr			cumanascsi_intr
 #define NCR5380_queue_command		cumanascsi_queue_command
 #define NCR5380_info			cumanascsi_info
@@ -211,6 +208,8 @@ static struct scsi_host_template cumanascsi_template = {
 	.cmd_per_lun		= 2,
 	.use_clustering		= DISABLE_CLUSTERING,
 	.proc_name		= "CumanaSCSI-1",
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 
 static int cumanascsi1_probe(struct expansion_card *ec,
@@ -240,23 +239,21 @@ static int cumanascsi1_probe(struct expansion_card *ec,
 
 	host->irq = ec->irq;
 
-	NCR5380_init(host, 0);
+	ret = NCR5380_init(host, 0);
+	if (ret)
+		goto out_unmap;
+
+	NCR5380_maybe_reset_bus(host);
 
         priv(host)->ctrl = 0;
         writeb(0, priv(host)->base + CTRL);
 
-	host->n_io_port = 255;
-	if (!(request_region(host->io_port, host->n_io_port, "CumanaSCSI-1"))) {
-		ret = -EBUSY;
-		goto out_unmap;
-	}
-
 	ret = request_irq(host->irq, cumanascsi_intr, 0,
 			  "CumanaSCSI-1", host);
 	if (ret) {
 		printk("scsi%d: IRQ%d not free: %d\n",
 		    host->host_no, host->irq, ret);
-		goto out_unmap;
+		goto out_exit;
 	}
 
 	ret = scsi_add_host(host, &ec->dev);
@@ -268,6 +265,8 @@ static int cumanascsi1_probe(struct expansion_card *ec,
 
  out_free_irq:
 	free_irq(host->irq, host);
+ out_exit:
+	NCR5380_exit(host);
  out_unmap:
 	iounmap(priv(host)->base);
 	iounmap(priv(host)->dma);
diff --git a/drivers/scsi/arm/oak.c b/drivers/scsi/arm/oak.c
index 7c6fa1479c9c..1fab1d1896b1 100644
--- a/drivers/scsi/arm/oak.c
+++ b/drivers/scsi/arm/oak.c
@@ -5,9 +5,7 @@
  */
 
 #include <linux/module.h>
-#include <linux/signal.h>
 #include <linux/ioport.h>
-#include <linux/delay.h>
 #include <linux/blkdev.h>
 #include <linux/init.h>
 
@@ -20,14 +18,16 @@
 #define DONT_USE_INTR
 
 #define priv(host)			((struct NCR5380_hostdata *)(host)->hostdata)
-#define NCR5380_local_declare()		void __iomem *_base
-#define NCR5380_setup(host)		_base = priv(host)->base
 
-#define NCR5380_read(reg)		readb(_base + ((reg) << 2))
-#define NCR5380_write(reg, value)	writeb(value, _base + ((reg) << 2))
+#define NCR5380_read(reg) \
+	readb(priv(instance)->base + ((reg) << 2))
+#define NCR5380_write(reg, value) \
+	writeb(value, priv(instance)->base + ((reg) << 2))
+
+#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
+
 #define NCR5380_queue_command		oakscsi_queue_command
 #define NCR5380_info			oakscsi_info
-#define NCR5380_show_info		oakscsi_show_info
 
 #define NCR5380_implementation_fields	\
 	void __iomem *base
@@ -103,7 +103,6 @@ printk("reading %p len %d\n", addr, len);
 
 static struct scsi_host_template oakscsi_template = {
 	.module			= THIS_MODULE,
-	.show_info		= oakscsi_show_info,
 	.name			= "Oak 16-bit SCSI",
 	.info			= oakscsi_info,
 	.queuecommand		= oakscsi_queue_command,
@@ -115,6 +114,8 @@ static struct scsi_host_template oakscsi_template = {
 	.cmd_per_lun		= 2,
 	.use_clustering		= DISABLE_CLUSTERING,
 	.proc_name		= "oakscsi",
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 
 static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
@@ -142,15 +143,21 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
 	host->irq = NO_IRQ;
 	host->n_io_port = 255;
 
-	NCR5380_init(host, 0);
+	ret = NCR5380_init(host, 0);
+	if (ret)
+		goto out_unmap;
+
+	NCR5380_maybe_reset_bus(host);
 
 	ret = scsi_add_host(host, &ec->dev);
 	if (ret)
-		goto out_unmap;
+		goto out_exit;
 
 	scsi_scan_host(host);
 	goto out;
 
+ out_exit:
+	NCR5380_exit(host);
  out_unmap:
 	iounmap(priv(host)->base);
  unreg:
diff --git a/drivers/scsi/atari_NCR5380.c b/drivers/scsi/atari_NCR5380.c
index db87ece6edb2..e65478651ca9 100644
--- a/drivers/scsi/atari_NCR5380.c
+++ b/drivers/scsi/atari_NCR5380.c
@@ -1,15 +1,15 @@
 /*
  * NCR 5380 generic driver routines.  These should make it *trivial*
- *	to implement 5380 SCSI drivers under Linux with a non-trantor
- *	architecture.
+ * to implement 5380 SCSI drivers under Linux with a non-trantor
+ * architecture.
  *
- *	Note that these routines also work with NR53c400 family chips.
+ * Note that these routines also work with NR53c400 family chips.
  *
  * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *	+1 (303) 666-5836
+ * Visionary Computing
+ * (Unix and Linux consulting and custom programming)
+ * drew@colorado.edu
+ * +1 (303) 666-5836
  *
  * For more information, please consult
  *
@@ -24,84 +24,10 @@
  * 1+ (800) 334-5454
  */
 
-/*
- * ++roman: To port the 5380 driver to the Atari, I had to do some changes in
- * this file, too:
- *
- *  - Some of the debug statements were incorrect (undefined variables and the
- *    like). I fixed that.
- *
- *  - In information_transfer(), I think a #ifdef was wrong. Looking at the
- *    possible DMA transfer size should also happen for REAL_DMA. I added this
- *    in the #if statement.
- *
- *  - When using real DMA, information_transfer() should return in a DATAOUT
- *    phase after starting the DMA. It has nothing more to do.
- *
- *  - The interrupt service routine should run main after end of DMA, too (not
- *    only after RESELECTION interrupts). Additionally, it should _not_ test
- *    for more interrupts after running main, since a DMA process may have
- *    been started and interrupts are turned on now. The new int could happen
- *    inside the execution of NCR5380_intr(), leading to recursive
- *    calls.
- *
- *  - I've added a function merge_contiguous_buffers() that tries to
- *    merge scatter-gather buffers that are located at contiguous
- *    physical addresses and can be processed with the same DMA setup.
- *    Since most scatter-gather operations work on a page (4K) of
- *    4 buffers (1K), in more than 90% of all cases three interrupts and
- *    DMA setup actions are saved.
- *
- * - I've deleted all the stuff for AUTOPROBE_IRQ, REAL_DMA_POLL, PSEUDO_DMA
- *    and USLEEP, because these were messing up readability and will never be
- *    needed for Atari SCSI.
- *
- * - I've revised the NCR5380_main() calling scheme (relax the 'main_running'
- *   stuff), and 'main' is executed in a bottom half if awoken by an
- *   interrupt.
- *
- * - The code was quite cluttered up by "#if (NDEBUG & NDEBUG_*) printk..."
- *   constructs. In my eyes, this made the source rather unreadable, so I
- *   finally replaced that by the *_PRINTK() macros.
- *
- */
-
-/*
- * Further development / testing that should be done :
- * 1.  Test linked command handling code after Eric is ready with
- *     the high level code.
- */
+/* Ported to Atari by Roman Hodek and others. */
 
 /* Adapted for the sun3 by Sam Creasey. */
 
-#include <scsi/scsi_dbg.h>
-#include <scsi/scsi_transport_spi.h>
-
-#if (NDEBUG & NDEBUG_LISTS)
-#define LIST(x, y)						\
-	do {							\
-		printk("LINE:%d   Adding %p to %p\n",		\
-		       __LINE__, (void*)(x), (void*)(y));	\
-		if ((x) == (y))					\
-			udelay(5);				\
-	} while (0)
-#define REMOVE(w, x, y, z)					\
-	do {							\
-		printk("LINE:%d   Removing: %p->%p  %p->%p \n",	\
-		       __LINE__, (void*)(w), (void*)(x),	\
-		       (void*)(y), (void*)(z));			\
-		if ((x) == (y))					\
-			udelay(5);				\
-	} while (0)
-#else
-#define LIST(x,y)
-#define REMOVE(w,x,y,z)
-#endif
-
-#ifndef notyet
-#undef LINKED
-#endif
-
 /*
  * Design
  *
@@ -126,17 +52,7 @@
  * piece of hardware that requires you to sit in a loop polling for
  * the REQ signal as long as you are connected.  Some devices are
  * brain dead (ie, many TEXEL CD ROM drives) and won't disconnect
- * while doing long seek operations.
- *
- * The workaround for this is to keep track of devices that have
- * disconnected.  If the device hasn't disconnected, for commands that
- * should disconnect, we do something like
- *
- * while (!REQ is asserted) { sleep for N usecs; poll for M usecs }
- *
- * Some tweaking of N and M needs to be done.  An algorithm based
- * on "time to data" would give the best results as long as short time
- * to datas (ie, on the same track) were considered, however these
+ * while doing long seek operations. [...] These
  * broken devices are the exception rather than the rule and I'd rather
  * spend my time optimizing for the normal case.
  *
@@ -177,12 +93,10 @@
  *
  * These macros control options :
  * AUTOSENSE - if defined, REQUEST SENSE will be performed automatically
- *	for commands that return with a CHECK CONDITION status.
+ * for commands that return with a CHECK CONDITION status.
  *
  * DIFFERENTIAL - if defined, NCR53c81 chips will use external differential
- *	transceivers.
- *
- * LINKED - if defined, linked commands are supported.
+ * transceivers.
  *
  * REAL_DMA - if defined, REAL DMA is used during the data transfer phases.
  *
@@ -195,17 +109,17 @@
  * NCR5380_write(register, value) - write to the specific register
  *
  * NCR5380_implementation_fields  - additional fields needed for this
- *      specific implementation of the NCR5380
+ * specific implementation of the NCR5380
  *
  * Either real DMA *or* pseudo DMA may be implemented
  * REAL functions :
  * NCR5380_REAL_DMA should be defined if real DMA is to be used.
  * Note that the DMA setup functions should return the number of bytes
- *	that they were able to program the controller for.
+ * that they were able to program the controller for.
  *
  * Also note that generic i386/PC versions of these macros are
- *	available as NCR5380_i386_dma_write_setup,
- *	NCR5380_i386_dma_read_setup, and NCR5380_i386_dma_residual.
+ * available as NCR5380_i386_dma_write_setup,
+ * NCR5380_i386_dma_read_setup, and NCR5380_i386_dma_residual.
  *
  * NCR5380_dma_write_setup(instance, src, count) - initialize
  * NCR5380_dma_read_setup(instance, dst, count) - initialize
@@ -221,18 +135,8 @@
  * possible) function may be used.
  */
 
-/* Macros ease life... :-) */
-#define	SETUP_HOSTDATA(in)				\
-    struct NCR5380_hostdata *hostdata =			\
-	(struct NCR5380_hostdata *)(in)->hostdata
-#define	HOSTDATA(in) ((struct NCR5380_hostdata *)(in)->hostdata)
-
-#define	NEXT(cmd)		((struct scsi_cmnd *)(cmd)->host_scribble)
-#define	SET_NEXT(cmd,next)	((cmd)->host_scribble = (void *)(next))
-#define	NEXTADDR(cmd)		((struct scsi_cmnd **)&(cmd)->host_scribble)
-
-#define	HOSTNO		instance->host_no
-#define	H_NO(cmd)	(cmd)->device->host->host_no
+static int do_abort(struct Scsi_Host *);
+static void do_reset(struct Scsi_Host *);
 
 #ifdef SUPPORT_TAGS
 
@@ -251,9 +155,7 @@
  * cannot know it in advance :-( We just see a QUEUE_FULL status being
  * returned. So, in this case, the driver internal queue size assumption is
  * reduced to the number of active tags if QUEUE_FULL is returned by the
- * target. The command is returned to the mid-level, but with status changed
- * to BUSY, since --as I've seen-- the mid-level can't handle QUEUE_FULL
- * correctly.
+ * target.
  *
  * We're also not allowed running tagged commands as long as an untagged
  * command is active. And REQUEST SENSE commands after a contingent allegiance
@@ -304,7 +206,8 @@ static void __init init_tags(struct NCR5380_hostdata *hostdata)
 static int is_lun_busy(struct scsi_cmnd *cmd, int should_be_tagged)
 {
 	u8 lun = cmd->device->lun;
-	SETUP_HOSTDATA(cmd->device->host);
+	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	if (hostdata->busy[cmd->device->id] & (1 << lun))
 		return 1;
@@ -314,8 +217,8 @@ static int is_lun_busy(struct scsi_cmnd *cmd, int should_be_tagged)
 		return 0;
 	if (hostdata->TagAlloc[scmd_id(cmd)][lun].nr_allocated >=
 	    hostdata->TagAlloc[scmd_id(cmd)][lun].queue_size) {
-		dprintk(NDEBUG_TAGS, "scsi%d: target %d lun %d: no free tags\n",
-			   H_NO(cmd), cmd->device->id, lun);
+		dsprintk(NDEBUG_TAGS, instance, "target %d lun %d: no free tags\n",
+		         scmd_id(cmd), lun);
 		return 1;
 	}
 	return 0;
@@ -330,7 +233,8 @@ static int is_lun_busy(struct scsi_cmnd *cmd, int should_be_tagged)
 static void cmd_get_tag(struct scsi_cmnd *cmd, int should_be_tagged)
 {
 	u8 lun = cmd->device->lun;
-	SETUP_HOSTDATA(cmd->device->host);
+	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	/* If we or the target don't support tagged queuing, allocate the LUN for
 	 * an untagged command.
@@ -340,18 +244,16 @@ static void cmd_get_tag(struct scsi_cmnd *cmd, int should_be_tagged)
 	    !cmd->device->tagged_supported) {
 		cmd->tag = TAG_NONE;
 		hostdata->busy[cmd->device->id] |= (1 << lun);
-		dprintk(NDEBUG_TAGS, "scsi%d: target %d lun %d now allocated by untagged "
-			   "command\n", H_NO(cmd), cmd->device->id, lun);
+		dsprintk(NDEBUG_TAGS, instance, "target %d lun %d now allocated by untagged command\n",
+		         scmd_id(cmd), lun);
 	} else {
 		struct tag_alloc *ta = &hostdata->TagAlloc[scmd_id(cmd)][lun];
 
 		cmd->tag = find_first_zero_bit(ta->allocated, MAX_TAGS);
 		set_bit(cmd->tag, ta->allocated);
 		ta->nr_allocated++;
-		dprintk(NDEBUG_TAGS, "scsi%d: using tag %d for target %d lun %d "
-			   "(now %d tags in use)\n",
-			   H_NO(cmd), cmd->tag, cmd->device->id,
-			   lun, ta->nr_allocated);
+		dsprintk(NDEBUG_TAGS, instance, "using tag %d for target %d lun %d (%d tags allocated)\n",
+		         cmd->tag, scmd_id(cmd), lun, ta->nr_allocated);
 	}
 }
 
@@ -363,21 +265,22 @@ static void cmd_get_tag(struct scsi_cmnd *cmd, int should_be_tagged)
 static void cmd_free_tag(struct scsi_cmnd *cmd)
 {
 	u8 lun = cmd->device->lun;
-	SETUP_HOSTDATA(cmd->device->host);
+	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	if (cmd->tag == TAG_NONE) {
 		hostdata->busy[cmd->device->id] &= ~(1 << lun);
-		dprintk(NDEBUG_TAGS, "scsi%d: target %d lun %d untagged cmd finished\n",
-			   H_NO(cmd), cmd->device->id, lun);
+		dsprintk(NDEBUG_TAGS, instance, "target %d lun %d untagged cmd freed\n",
+		         scmd_id(cmd), lun);
 	} else if (cmd->tag >= MAX_TAGS) {
-		printk(KERN_NOTICE "scsi%d: trying to free bad tag %d!\n",
-		       H_NO(cmd), cmd->tag);
+		shost_printk(KERN_NOTICE, instance,
+		             "trying to free bad tag %d!\n", cmd->tag);
 	} else {
 		struct tag_alloc *ta = &hostdata->TagAlloc[scmd_id(cmd)][lun];
 		clear_bit(cmd->tag, ta->allocated);
 		ta->nr_allocated--;
-		dprintk(NDEBUG_TAGS, "scsi%d: freed tag %d for target %d lun %d\n",
-			   H_NO(cmd), cmd->tag, cmd->device->id, lun);
+		dsprintk(NDEBUG_TAGS, instance, "freed tag %d for target %d lun %d\n",
+		         cmd->tag, scmd_id(cmd), lun);
 	}
 }
 
@@ -401,17 +304,15 @@ static void free_all_tags(struct NCR5380_hostdata *hostdata)
 
 #endif /* SUPPORT_TAGS */
 
-
-/*
- * Function: void merge_contiguous_buffers( struct scsi_cmnd *cmd )
- *
- * Purpose: Try to merge several scatter-gather requests into one DMA
- *    transfer. This is possible if the scatter buffers lie on
- *    physical contiguous addresses.
- *
- * Parameters: struct scsi_cmnd *cmd
- *    The command to work on. The first scatter buffer's data are
- *    assumed to be already transferred into ptr/this_residual.
+/**
+ * merge_contiguous_buffers - coalesce scatter-gather list entries
+ * @cmd: command requesting IO
+ *
+ * Try to merge several scatter-gather buffers into one DMA transfer.
+ * This is possible if the scatter buffers lie on physically
+ * contiguous addresses. The first scatter-gather buffer's data are
+ * assumed to be already transferred into cmd->SCp.this_residual.
+ * Every buffer merged avoids an interrupt and a DMA setup operation.
  */
 
 static void merge_contiguous_buffers(struct scsi_cmnd *cmd)
@@ -463,9 +364,7 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd)
 		cmd->SCp.buffers_residual = scsi_sg_count(cmd) - 1;
 		cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
 		cmd->SCp.this_residual = cmd->SCp.buffer->length;
-		/* ++roman: Try to merge some scatter-buffers if they are at
-		 * contiguous physical addresses.
-		 */
+
 		merge_contiguous_buffers(cmd);
 	} else {
 		cmd->SCp.buffer = NULL;
@@ -473,31 +372,110 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd)
 		cmd->SCp.ptr = NULL;
 		cmd->SCp.this_residual = 0;
 	}
+
+	cmd->SCp.Status = 0;
+	cmd->SCp.Message = 0;
+}
+
+/**
+ * NCR5380_poll_politely2 - wait for two chip register values
+ * @instance: controller to poll
+ * @reg1: 5380 register to poll
+ * @bit1: Bitmask to check
+ * @val1: Expected value
+ * @reg2: Second 5380 register to poll
+ * @bit2: Second bitmask to check
+ * @val2: Second expected value
+ * @wait: Time-out in jiffies
+ *
+ * Polls the chip in a reasonably efficient manner waiting for an
+ * event to occur. After a short quick poll we begin to yield the CPU
+ * (if possible). In irq contexts the time-out is arbitrarily limited.
+ * Callers may hold locks as long as they are held in irq mode.
+ *
+ * Returns 0 if either or both event(s) occurred otherwise -ETIMEDOUT.
+ */
+
+static int NCR5380_poll_politely2(struct Scsi_Host *instance,
+                                  int reg1, int bit1, int val1,
+                                  int reg2, int bit2, int val2, int wait)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	unsigned long deadline = jiffies + wait;
+	unsigned long n;
+
+	/* Busy-wait for up to 10 ms */
+	n = min(10000U, jiffies_to_usecs(wait));
+	n *= hostdata->accesses_per_ms;
+	n /= 2000;
+	do {
+		if ((NCR5380_read(reg1) & bit1) == val1)
+			return 0;
+		if ((NCR5380_read(reg2) & bit2) == val2)
+			return 0;
+		cpu_relax();
+	} while (n--);
+
+	if (irqs_disabled() || in_interrupt())
+		return -ETIMEDOUT;
+
+	/* Repeatedly sleep for 1 ms until deadline */
+	while (time_is_after_jiffies(deadline)) {
+		schedule_timeout_uninterruptible(1);
+		if ((NCR5380_read(reg1) & bit1) == val1)
+			return 0;
+		if ((NCR5380_read(reg2) & bit2) == val2)
+			return 0;
+	}
+
+	return -ETIMEDOUT;
 }
 
-#include <linux/delay.h>
+static inline int NCR5380_poll_politely(struct Scsi_Host *instance,
+                                        int reg, int bit, int val, int wait)
+{
+	return NCR5380_poll_politely2(instance, reg, bit, val,
+	                                        reg, bit, val, wait);
+}
 
 #if NDEBUG
 static struct {
 	unsigned char mask;
 	const char *name;
 } signals[] = {
-	{ SR_DBP, "PARITY"}, { SR_RST, "RST" }, { SR_BSY, "BSY" },
-	{ SR_REQ, "REQ" }, { SR_MSG, "MSG" }, { SR_CD,  "CD" }, { SR_IO, "IO" },
-	{ SR_SEL, "SEL" }, {0, NULL}
-}, basrs[] = {
-	{BASR_ATN, "ATN"}, {BASR_ACK, "ACK"}, {0, NULL}
-}, icrs[] = {
-	{ICR_ASSERT_RST, "ASSERT RST"},{ICR_ASSERT_ACK, "ASSERT ACK"},
-	{ICR_ASSERT_BSY, "ASSERT BSY"}, {ICR_ASSERT_SEL, "ASSERT SEL"},
-	{ICR_ASSERT_ATN, "ASSERT ATN"}, {ICR_ASSERT_DATA, "ASSERT DATA"},
+	{SR_DBP, "PARITY"},
+	{SR_RST, "RST"},
+	{SR_BSY, "BSY"},
+	{SR_REQ, "REQ"},
+	{SR_MSG, "MSG"},
+	{SR_CD, "CD"},
+	{SR_IO, "IO"},
+	{SR_SEL, "SEL"},
+	{0, NULL}
+},
+basrs[] = {
+	{BASR_ATN, "ATN"},
+	{BASR_ACK, "ACK"},
 	{0, NULL}
-}, mrs[] = {
-	{MR_BLOCK_DMA_MODE, "MODE BLOCK DMA"}, {MR_TARGET, "MODE TARGET"},
-	{MR_ENABLE_PAR_CHECK, "MODE PARITY CHECK"}, {MR_ENABLE_PAR_INTR,
-	"MODE PARITY INTR"}, {MR_ENABLE_EOP_INTR,"MODE EOP INTR"},
+},
+icrs[] = {
+	{ICR_ASSERT_RST, "ASSERT RST"},
+	{ICR_ASSERT_ACK, "ASSERT ACK"},
+	{ICR_ASSERT_BSY, "ASSERT BSY"},
+	{ICR_ASSERT_SEL, "ASSERT SEL"},
+	{ICR_ASSERT_ATN, "ASSERT ATN"},
+	{ICR_ASSERT_DATA, "ASSERT DATA"},
+	{0, NULL}
+},
+mrs[] = {
+	{MR_BLOCK_DMA_MODE, "MODE BLOCK DMA"},
+	{MR_TARGET, "MODE TARGET"},
+	{MR_ENABLE_PAR_CHECK, "MODE PARITY CHECK"},
+	{MR_ENABLE_PAR_INTR, "MODE PARITY INTR"},
+	{MR_ENABLE_EOP_INTR, "MODE EOP INTR"},
 	{MR_MONITOR_BSY, "MODE MONITOR BSY"},
-	{MR_DMA_MODE, "MODE DMA"}, {MR_ARBITRATE, "MODE ARBITRATION"},
+	{MR_DMA_MODE, "MODE DMA"},
+	{MR_ARBITRATE, "MODE ARBITRATION"},
 	{0, NULL}
 };
 
@@ -511,15 +489,13 @@ static struct {
 static void NCR5380_print(struct Scsi_Host *instance)
 {
 	unsigned char status, data, basr, mr, icr, i;
-	unsigned long flags;
 
-	local_irq_save(flags);
 	data = NCR5380_read(CURRENT_SCSI_DATA_REG);
 	status = NCR5380_read(STATUS_REG);
 	mr = NCR5380_read(MODE_REG);
 	icr = NCR5380_read(INITIATOR_COMMAND_REG);
 	basr = NCR5380_read(BUS_AND_STATUS_REG);
-	local_irq_restore(flags);
+
 	printk("STATUS_REG: %02x ", status);
 	for (i = 0; signals[i].mask; ++i)
 		if (status & signals[i].mask)
@@ -543,8 +519,12 @@ static struct {
 	unsigned char value;
 	const char *name;
 } phases[] = {
-	{PHASE_DATAOUT, "DATAOUT"}, {PHASE_DATAIN, "DATAIN"}, {PHASE_CMDOUT, "CMDOUT"},
-	{PHASE_STATIN, "STATIN"}, {PHASE_MSGOUT, "MSGOUT"}, {PHASE_MSGIN, "MSGIN"},
+	{PHASE_DATAOUT, "DATAOUT"},
+	{PHASE_DATAIN, "DATAIN"},
+	{PHASE_CMDOUT, "CMDOUT"},
+	{PHASE_STATIN, "STATIN"},
+	{PHASE_MSGOUT, "MSGOUT"},
+	{PHASE_MSGIN, "MSGIN"},
 	{PHASE_UNKNOWN, "UNKNOWN"}
 };
 
@@ -553,8 +533,6 @@ static struct {
  * @instance: adapter to dump
  *
  * Print the current SCSI phase for debugging purposes
- *
- * Locks: none
  */
 
 static void NCR5380_print_phase(struct Scsi_Host *instance)
@@ -564,54 +542,21 @@ static void NCR5380_print_phase(struct Scsi_Host *instance)
 
 	status = NCR5380_read(STATUS_REG);
 	if (!(status & SR_REQ))
-		printk(KERN_DEBUG "scsi%d: REQ not asserted, phase unknown.\n", HOSTNO);
+		shost_printk(KERN_DEBUG, instance, "REQ not asserted, phase unknown.\n");
 	else {
 		for (i = 0; (phases[i].value != PHASE_UNKNOWN) &&
 		     (phases[i].value != (status & PHASE_MASK)); ++i)
 			;
-		printk(KERN_DEBUG "scsi%d: phase %s\n", HOSTNO, phases[i].name);
+		shost_printk(KERN_DEBUG, instance, "phase %s\n", phases[i].name);
 	}
 }
-
 #endif
 
-/*
- * ++roman: New scheme of calling NCR5380_main()
- *
- * If we're not in an interrupt, we can call our main directly, it cannot be
- * already running. Else, we queue it on a task queue, if not 'main_running'
- * tells us that a lower level is already executing it. This way,
- * 'main_running' needs not be protected in a special way.
- *
- * queue_main() is a utility function for putting our main onto the task
- * queue, if main_running is false. It should be called only from a
- * interrupt or bottom half.
- */
-
-#include <linux/gfp.h>
-#include <linux/workqueue.h>
-#include <linux/interrupt.h>
-
-static inline void queue_main(struct NCR5380_hostdata *hostdata)
-{
-	if (!hostdata->main_running) {
-		/* If in interrupt and NCR5380_main() not already running,
-		   queue it on the 'immediate' task queue, to be processed
-		   immediately after the current interrupt processing has
-		   finished. */
-		schedule_work(&hostdata->main_task);
-	}
-	/* else: nothing to do: the running NCR5380_main() will pick up
-	   any newly queued command. */
-}
-
 /**
  * NCR58380_info - report driver and host information
  * @instance: relevant scsi host instance
  *
  * For use as the host template info() handler.
- *
- * Locks: none
  */
 
 static const char *NCR5380_info(struct Scsi_Host *instance)
@@ -630,13 +575,14 @@ static void prepare_info(struct Scsi_Host *instance)
 	         "base 0x%lx, irq %d, "
 	         "can_queue %d, cmd_per_lun %d, "
 	         "sg_tablesize %d, this_id %d, "
-	         "flags { %s}, "
+	         "flags { %s%s}, "
 	         "options { %s} ",
 	         instance->hostt->name, instance->io_port, instance->n_io_port,
 	         instance->base, instance->irq,
 	         instance->can_queue, instance->cmd_per_lun,
 	         instance->sg_tablesize, instance->this_id,
 	         hostdata->flags & FLAG_TAGGED_QUEUING ? "TAGGED_QUEUING " : "",
+	         hostdata->flags & FLAG_TOSHIBA_DELAY  ? "TOSHIBA_DELAY "  : "",
 #ifdef DIFFERENTIAL
 	         "DIFFERENTIAL "
 #endif
@@ -653,102 +599,6 @@ static void prepare_info(struct Scsi_Host *instance)
 }
 
 /**
- * NCR5380_print_status - dump controller info
- * @instance: controller to dump
- *
- * Print commands in the various queues, called from NCR5380_abort
- * to aid debugging.
- */
-
-static void lprint_Scsi_Cmnd(struct scsi_cmnd *cmd)
-{
-	int i, s;
-	unsigned char *command;
-	printk("scsi%d: destination target %d, lun %llu\n",
-		H_NO(cmd), cmd->device->id, cmd->device->lun);
-	printk(KERN_CONT "        command = ");
-	command = cmd->cmnd;
-	printk(KERN_CONT "%2d (0x%02x)", command[0], command[0]);
-	for (i = 1, s = COMMAND_SIZE(command[0]); i < s; ++i)
-		printk(KERN_CONT " %02x", command[i]);
-	printk("\n");
-}
-
-static void NCR5380_print_status(struct Scsi_Host *instance)
-{
-	struct NCR5380_hostdata *hostdata;
-	struct scsi_cmnd *ptr;
-	unsigned long flags;
-
-	NCR5380_dprint(NDEBUG_ANY, instance);
-	NCR5380_dprint_phase(NDEBUG_ANY, instance);
-
-	hostdata = (struct NCR5380_hostdata *)instance->hostdata;
-
-	local_irq_save(flags);
-	printk("NCR5380: coroutine is%s running.\n",
-		hostdata->main_running ? "" : "n't");
-	if (!hostdata->connected)
-		printk("scsi%d: no currently connected command\n", HOSTNO);
-	else
-		lprint_Scsi_Cmnd((struct scsi_cmnd *) hostdata->connected);
-	printk("scsi%d: issue_queue\n", HOSTNO);
-	for (ptr = (struct scsi_cmnd *)hostdata->issue_queue; ptr; ptr = NEXT(ptr))
-		lprint_Scsi_Cmnd(ptr);
-
-	printk("scsi%d: disconnected_queue\n", HOSTNO);
-	for (ptr = (struct scsi_cmnd *) hostdata->disconnected_queue; ptr;
-	     ptr = NEXT(ptr))
-		lprint_Scsi_Cmnd(ptr);
-
-	local_irq_restore(flags);
-	printk("\n");
-}
-
-static void show_Scsi_Cmnd(struct scsi_cmnd *cmd, struct seq_file *m)
-{
-	int i, s;
-	unsigned char *command;
-	seq_printf(m, "scsi%d: destination target %d, lun %llu\n",
-		H_NO(cmd), cmd->device->id, cmd->device->lun);
-	seq_puts(m, "        command = ");
-	command = cmd->cmnd;
-	seq_printf(m, "%2d (0x%02x)", command[0], command[0]);
-	for (i = 1, s = COMMAND_SIZE(command[0]); i < s; ++i)
-		seq_printf(m, " %02x", command[i]);
-	seq_putc(m, '\n');
-}
-
-static int __maybe_unused NCR5380_show_info(struct seq_file *m,
-                                            struct Scsi_Host *instance)
-{
-	struct NCR5380_hostdata *hostdata;
-	struct scsi_cmnd *ptr;
-	unsigned long flags;
-
-	hostdata = (struct NCR5380_hostdata *)instance->hostdata;
-
-	local_irq_save(flags);
-	seq_printf(m, "NCR5380: coroutine is%s running.\n",
-		hostdata->main_running ? "" : "n't");
-	if (!hostdata->connected)
-		seq_printf(m, "scsi%d: no currently connected command\n", HOSTNO);
-	else
-		show_Scsi_Cmnd((struct scsi_cmnd *) hostdata->connected, m);
-	seq_printf(m, "scsi%d: issue_queue\n", HOSTNO);
-	for (ptr = (struct scsi_cmnd *)hostdata->issue_queue; ptr; ptr = NEXT(ptr))
-		show_Scsi_Cmnd(ptr, m);
-
-	seq_printf(m, "scsi%d: disconnected_queue\n", HOSTNO);
-	for (ptr = (struct scsi_cmnd *) hostdata->disconnected_queue; ptr;
-	     ptr = NEXT(ptr))
-		show_Scsi_Cmnd(ptr, m);
-
-	local_irq_restore(flags);
-	return 0;
-}
-
-/**
  * NCR5380_init - initialise an NCR5380
  * @instance: adapter to configure
  * @flags: control flags
@@ -764,11 +614,11 @@ static int __maybe_unused NCR5380_show_info(struct seq_file *m,
 
 static int __init NCR5380_init(struct Scsi_Host *instance, int flags)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int i;
-	SETUP_HOSTDATA(instance);
+	unsigned long deadline;
 
 	hostdata->host = instance;
-	hostdata->aborted = 0;
 	hostdata->id_mask = 1 << instance->this_id;
 	hostdata->id_higher_mask = 0;
 	for (i = hostdata->id_mask; i <= 0x80; i <<= 1)
@@ -782,13 +632,21 @@ static int __init NCR5380_init(struct Scsi_Host *instance, int flags)
 #if defined (REAL_DMA)
 	hostdata->dma_len = 0;
 #endif
-	hostdata->targets_present = 0;
+	spin_lock_init(&hostdata->lock);
 	hostdata->connected = NULL;
-	hostdata->issue_queue = NULL;
-	hostdata->disconnected_queue = NULL;
+	hostdata->sensing = NULL;
+	INIT_LIST_HEAD(&hostdata->autosense);
+	INIT_LIST_HEAD(&hostdata->unissued);
+	INIT_LIST_HEAD(&hostdata->disconnected);
+
 	hostdata->flags = flags;
 
 	INIT_WORK(&hostdata->main_task, NCR5380_main);
+	hostdata->work_q = alloc_workqueue("ncr5380_%d",
+	                        WQ_UNBOUND | WQ_MEM_RECLAIM,
+	                        1, instance->host_no);
+	if (!hostdata->work_q)
+		return -ENOMEM;
 
 	prepare_info(instance);
 
@@ -797,6 +655,72 @@ static int __init NCR5380_init(struct Scsi_Host *instance, int flags)
 	NCR5380_write(TARGET_COMMAND_REG, 0);
 	NCR5380_write(SELECT_ENABLE_REG, 0);
 
+	/* Calibrate register polling loop */
+	i = 0;
+	deadline = jiffies + 1;
+	do {
+		cpu_relax();
+	} while (time_is_after_jiffies(deadline));
+	deadline += msecs_to_jiffies(256);
+	do {
+		NCR5380_read(STATUS_REG);
+		++i;
+		cpu_relax();
+	} while (time_is_after_jiffies(deadline));
+	hostdata->accesses_per_ms = i / 256;
+
+	return 0;
+}
+
+/**
+ * NCR5380_maybe_reset_bus - Detect and correct bus wedge problems.
+ * @instance: adapter to check
+ *
+ * If the system crashed, it may have crashed with a connected target and
+ * the SCSI bus busy. Check for BUS FREE phase. If not, try to abort the
+ * currently established nexus, which we know nothing about. Failing that
+ * do a bus reset.
+ *
+ * Note that a bus reset will cause the chip to assert IRQ.
+ *
+ * Returns 0 if successful, otherwise -ENXIO.
+ */
+
+static int NCR5380_maybe_reset_bus(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int pass;
+
+	for (pass = 1; (NCR5380_read(STATUS_REG) & SR_BSY) && pass <= 6; ++pass) {
+		switch (pass) {
+		case 1:
+		case 3:
+		case 5:
+			shost_printk(KERN_ERR, instance, "SCSI bus busy, waiting up to five seconds\n");
+			NCR5380_poll_politely(instance,
+			                      STATUS_REG, SR_BSY, 0, 5 * HZ);
+			break;
+		case 2:
+			shost_printk(KERN_ERR, instance, "bus busy, attempting abort\n");
+			do_abort(instance);
+			break;
+		case 4:
+			shost_printk(KERN_ERR, instance, "bus busy, attempting reset\n");
+			do_reset(instance);
+			/* Wait after a reset; the SCSI standard calls for
+			 * 250ms, we wait 500ms to be on the safe side.
+			 * But some Toshiba CD-ROMs need ten times that.
+			 */
+			if (hostdata->flags & FLAG_TOSHIBA_DELAY)
+				msleep(2500);
+			else
+				msleep(500);
+			break;
+		case 6:
+			shost_printk(KERN_ERR, instance, "bus locked solid\n");
+			return -ENXIO;
+		}
+	}
 	return 0;
 }
 
@@ -812,6 +736,38 @@ static void NCR5380_exit(struct Scsi_Host *instance)
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	cancel_work_sync(&hostdata->main_task);
+	destroy_workqueue(hostdata->work_q);
+}
+
+/**
+ * complete_cmd - finish processing a command and return it to the SCSI ML
+ * @instance: the host instance
+ * @cmd: command to complete
+ */
+
+static void complete_cmd(struct Scsi_Host *instance,
+                         struct scsi_cmnd *cmd)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+
+	dsprintk(NDEBUG_QUEUES, instance, "complete_cmd: cmd %p\n", cmd);
+
+	if (hostdata->sensing == cmd) {
+		/* Autosense processing ends here */
+		if ((cmd->result & 0xff) != SAM_STAT_GOOD) {
+			scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+			set_host_byte(cmd, DID_ERROR);
+		} else
+			scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+		hostdata->sensing = NULL;
+	}
+
+#ifdef SUPPORT_TAGS
+	cmd_free_tag(cmd);
+#else
+	hostdata->busy[scmd_id(cmd)] &= ~(1 << cmd->device->lun);
+#endif
+	cmd->scsi_done(cmd);
 }
 
 /**
@@ -819,7 +775,7 @@ static void NCR5380_exit(struct Scsi_Host *instance)
  * @instance: the relevant SCSI adapter
  * @cmd: SCSI command
  *
- * cmd is added to the per instance issue_queue, with minor
+ * cmd is added to the per-instance issue queue, with minor
  * twiddling done to the host specific fields of cmd.  If the
  * main coroutine is not running, it is restarted.
  */
@@ -828,44 +784,23 @@ static int NCR5380_queue_command(struct Scsi_Host *instance,
                                  struct scsi_cmnd *cmd)
 {
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	struct scsi_cmnd *tmp;
+	struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
 	unsigned long flags;
 
 #if (NDEBUG & NDEBUG_NO_WRITE)
 	switch (cmd->cmnd[0]) {
 	case WRITE_6:
 	case WRITE_10:
-		printk(KERN_NOTICE "scsi%d: WRITE attempted with NO_WRITE debugging flag set\n",
-		       H_NO(cmd));
+		shost_printk(KERN_DEBUG, instance, "WRITE attempted with NDEBUG_NO_WRITE set\n");
 		cmd->result = (DID_ERROR << 16);
 		cmd->scsi_done(cmd);
 		return 0;
 	}
 #endif /* (NDEBUG & NDEBUG_NO_WRITE) */
 
-	/*
-	 * We use the host_scribble field as a pointer to the next command
-	 * in a queue
-	 */
-
-	SET_NEXT(cmd, NULL);
 	cmd->result = 0;
 
 	/*
-	 * Insert the cmd into the issue queue. Note that REQUEST SENSE
-	 * commands are added to the head of the queue since any command will
-	 * clear the contingent allegiance condition that exists and the
-	 * sense data is only guaranteed to be valid while the condition exists.
-	 */
-
-	/* ++guenther: now that the issue queue is being set up, we can lock ST-DMA.
-	 * Otherwise a running NCR5380_main may steal the lock.
-	 * Lock before actually inserting due to fairness reasons explained in
-	 * atari_scsi.c. If we insert first, then it's impossible for this driver
-	 * to release the lock.
-	 * Stop timer for this command while waiting for the lock, or timeouts
-	 * may happen (and they really do), and it's no good if the command doesn't
-	 * appear in any of the queues.
 	 * ++roman: Just disabling the NCR interrupt isn't sufficient here,
 	 * because also a timer int can trigger an abort or reset, which would
 	 * alter queues and touch the lock.
@@ -873,7 +808,7 @@ static int NCR5380_queue_command(struct Scsi_Host *instance,
 	if (!NCR5380_acquire_dma_irq(instance))
 		return SCSI_MLQUEUE_HOST_BUSY;
 
-	local_irq_save(flags);
+	spin_lock_irqsave(&hostdata->lock, flags);
 
 	/*
 	 * Insert the cmd into the issue queue. Note that REQUEST SENSE
@@ -882,33 +817,18 @@ static int NCR5380_queue_command(struct Scsi_Host *instance,
 	 * sense data is only guaranteed to be valid while the condition exists.
 	 */
 
-	if (!(hostdata->issue_queue) || (cmd->cmnd[0] == REQUEST_SENSE)) {
-		LIST(cmd, hostdata->issue_queue);
-		SET_NEXT(cmd, hostdata->issue_queue);
-		hostdata->issue_queue = cmd;
-	} else {
-		for (tmp = (struct scsi_cmnd *)hostdata->issue_queue;
-		     NEXT(tmp); tmp = NEXT(tmp))
-			;
-		LIST(cmd, tmp);
-		SET_NEXT(tmp, cmd);
-	}
-	local_irq_restore(flags);
+	if (cmd->cmnd[0] == REQUEST_SENSE)
+		list_add(&ncmd->list, &hostdata->unissued);
+	else
+		list_add_tail(&ncmd->list, &hostdata->unissued);
 
-	dprintk(NDEBUG_QUEUES, "scsi%d: command added to %s of queue\n", H_NO(cmd),
-		  (cmd->cmnd[0] == REQUEST_SENSE) ? "head" : "tail");
+	spin_unlock_irqrestore(&hostdata->lock, flags);
 
-	/* If queue_command() is called from an interrupt (real one or bottom
-	 * half), we let queue_main() do the job of taking care about main. If it
-	 * is already running, this is a no-op, else main will be queued.
-	 *
-	 * If we're not in an interrupt, we can call NCR5380_main()
-	 * unconditionally, because it cannot be already running.
-	 */
-	if (in_interrupt() || irqs_disabled())
-		queue_main(hostdata);
-	else
-		NCR5380_main(&hostdata->main_task);
+	dsprintk(NDEBUG_QUEUES, instance, "command %p added to %s of queue\n",
+	         cmd, (cmd->cmnd[0] == REQUEST_SENSE) ? "head" : "tail");
+
+	/* Kick off command processing */
+	queue_work(hostdata->work_q, &hostdata->main_task);
 	return 0;
 }
 
@@ -917,22 +837,85 @@ static inline void maybe_release_dma_irq(struct Scsi_Host *instance)
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	/* Caller does the locking needed to set & test these data atomically */
-	if (!hostdata->disconnected_queue &&
-	    !hostdata->issue_queue &&
+	if (list_empty(&hostdata->disconnected) &&
+	    list_empty(&hostdata->unissued) &&
+	    list_empty(&hostdata->autosense) &&
 	    !hostdata->connected &&
-	    !hostdata->retain_dma_intr)
+	    !hostdata->selecting)
 		NCR5380_release_dma_irq(instance);
 }
 
 /**
+ * dequeue_next_cmd - dequeue a command for processing
+ * @instance: the scsi host instance
+ *
+ * Priority is given to commands on the autosense queue. These commands
+ * need autosense because of a CHECK CONDITION result.
+ *
+ * Returns a command pointer if a command is found for a target that is
+ * not already busy. Otherwise returns NULL.
+ */
+
+static struct scsi_cmnd *dequeue_next_cmd(struct Scsi_Host *instance)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	struct NCR5380_cmd *ncmd;
+	struct scsi_cmnd *cmd;
+
+	if (list_empty(&hostdata->autosense)) {
+		list_for_each_entry(ncmd, &hostdata->unissued, list) {
+			cmd = NCR5380_to_scmd(ncmd);
+			dsprintk(NDEBUG_QUEUES, instance, "dequeue: cmd=%p target=%d busy=0x%02x lun=%llu\n",
+			         cmd, scmd_id(cmd), hostdata->busy[scmd_id(cmd)], cmd->device->lun);
+
+			if (
+#ifdef SUPPORT_TAGS
+			    !is_lun_busy(cmd, 1)
+#else
+			    !(hostdata->busy[scmd_id(cmd)] & (1 << cmd->device->lun))
+#endif
+			) {
+				list_del(&ncmd->list);
+				dsprintk(NDEBUG_QUEUES, instance,
+				         "dequeue: removed %p from issue queue\n", cmd);
+				return cmd;
+			}
+		}
+	} else {
+		/* Autosense processing begins here */
+		ncmd = list_first_entry(&hostdata->autosense,
+		                        struct NCR5380_cmd, list);
+		list_del(&ncmd->list);
+		cmd = NCR5380_to_scmd(ncmd);
+		dsprintk(NDEBUG_QUEUES, instance,
+		         "dequeue: removed %p from autosense queue\n", cmd);
+		scsi_eh_prep_cmnd(cmd, &hostdata->ses, NULL, 0, ~0);
+		hostdata->sensing = cmd;
+		return cmd;
+	}
+	return NULL;
+}
+
+static void requeue_cmd(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
+
+	if (hostdata->sensing) {
+		scsi_eh_restore_cmnd(cmd, &hostdata->ses);
+		list_add(&ncmd->list, &hostdata->autosense);
+		hostdata->sensing = NULL;
+	} else
+		list_add(&ncmd->list, &hostdata->unissued);
+}
+
+/**
  * NCR5380_main - NCR state machines
  *
  * NCR5380_main is a coroutine that runs as long as more work can
  * be done on the NCR5380 host adapters in a system.  Both
  * NCR5380_queue_command() and NCR5380_intr() will try to start it
  * in case it is not running.
- *
- * Locks: called as its own thread with no locks held.
  */
 
 static void NCR5380_main(struct work_struct *work)
@@ -940,154 +923,69 @@ static void NCR5380_main(struct work_struct *work)
 	struct NCR5380_hostdata *hostdata =
 		container_of(work, struct NCR5380_hostdata, main_task);
 	struct Scsi_Host *instance = hostdata->host;
-	struct scsi_cmnd *tmp, *prev;
+	struct scsi_cmnd *cmd;
 	int done;
-	unsigned long flags;
 
 	/*
-	 * We run (with interrupts disabled) until we're sure that none of
-	 * the host adapters have anything that can be done, at which point
-	 * we set main_running to 0 and exit.
-	 *
-	 * Interrupts are enabled before doing various other internal
-	 * instructions, after we've decided that we need to run through
-	 * the loop again.
-	 *
-	 * this should prevent any race conditions.
-	 *
 	 * ++roman: Just disabling the NCR interrupt isn't sufficient here,
 	 * because also a timer int can trigger an abort or reset, which can
 	 * alter queues and touch the Falcon lock.
 	 */
 
-	/* Tell int handlers main() is now already executing.  Note that
-	   no races are possible here. If an int comes in before
-	   'main_running' is set here, and queues/executes main via the
-	   task queue, it doesn't do any harm, just this instance of main
-	   won't find any work left to do. */
-	if (hostdata->main_running)
-		return;
-	hostdata->main_running = 1;
-
-	local_save_flags(flags);
 	do {
-		local_irq_disable();	/* Freeze request queues */
 		done = 1;
 
-		if (!hostdata->connected) {
-			dprintk(NDEBUG_MAIN, "scsi%d: not connected\n", HOSTNO);
-			/*
-			 * Search through the issue_queue for a command destined
-			 * for a target that's not busy.
-			 */
-#if (NDEBUG & NDEBUG_LISTS)
-			for (tmp = (struct scsi_cmnd *) hostdata->issue_queue, prev = NULL;
-			     tmp && (tmp != prev); prev = tmp, tmp = NEXT(tmp))
-				;
-			/*printk("%p  ", tmp);*/
-			if ((tmp == prev) && tmp)
-				printk(" LOOP\n");
-			/* else printk("\n"); */
-#endif
-			for (tmp = (struct scsi_cmnd *) hostdata->issue_queue,
-			     prev = NULL; tmp; prev = tmp, tmp = NEXT(tmp)) {
-				u8 lun = tmp->device->lun;
-
-				dprintk(NDEBUG_LISTS,
-				        "MAIN tmp=%p target=%d busy=%d lun=%d\n",
-				        tmp, scmd_id(tmp), hostdata->busy[scmd_id(tmp)],
-				        lun);
-				/*  When we find one, remove it from the issue queue. */
-				/* ++guenther: possible race with Falcon locking */
-				if (
-#ifdef SUPPORT_TAGS
-				    !is_lun_busy( tmp, tmp->cmnd[0] != REQUEST_SENSE)
-#else
-				    !(hostdata->busy[tmp->device->id] & (1 << lun))
-#endif
-				    ) {
-					/* ++guenther: just to be sure, this must be atomic */
-					local_irq_disable();
-					if (prev) {
-						REMOVE(prev, NEXT(prev), tmp, NEXT(tmp));
-						SET_NEXT(prev, NEXT(tmp));
-					} else {
-						REMOVE(-1, hostdata->issue_queue, tmp, NEXT(tmp));
-						hostdata->issue_queue = NEXT(tmp);
-					}
-					SET_NEXT(tmp, NULL);
-					hostdata->retain_dma_intr++;
+		spin_lock_irq(&hostdata->lock);
+		while (!hostdata->connected &&
+		       (cmd = dequeue_next_cmd(instance))) {
 
-					/* reenable interrupts after finding one */
-					local_irq_restore(flags);
+			dsprintk(NDEBUG_MAIN, instance, "main: dequeued %p\n", cmd);
 
-					/*
-					 * Attempt to establish an I_T_L nexus here.
-					 * On success, instance->hostdata->connected is set.
-					 * On failure, we must add the command back to the
-					 *   issue queue so we can keep trying.
-					 */
-					dprintk(NDEBUG_MAIN, "scsi%d: main(): command for target %d "
-						    "lun %d removed from issue_queue\n",
-						    HOSTNO, tmp->device->id, lun);
-					/*
-					 * REQUEST SENSE commands are issued without tagged
-					 * queueing, even on SCSI-II devices because the
-					 * contingent allegiance condition exists for the
-					 * entire unit.
-					 */
-					/* ++roman: ...and the standard also requires that
-					 * REQUEST SENSE command are untagged.
-					 */
+			/*
+			 * Attempt to establish an I_T_L nexus here.
+			 * On success, instance->hostdata->connected is set.
+			 * On failure, we must add the command back to the
+			 * issue queue so we can keep trying.
+			 */
+			/*
+			 * REQUEST SENSE commands are issued without tagged
+			 * queueing, even on SCSI-II devices because the
+			 * contingent allegiance condition exists for the
+			 * entire unit.
+			 */
+			/* ++roman: ...and the standard also requires that
+			 * REQUEST SENSE command are untagged.
+			 */
 
 #ifdef SUPPORT_TAGS
-					cmd_get_tag(tmp, tmp->cmnd[0] != REQUEST_SENSE);
+			cmd_get_tag(cmd, cmd->cmnd[0] != REQUEST_SENSE);
 #endif
-					if (!NCR5380_select(instance, tmp)) {
-						local_irq_disable();
-						hostdata->retain_dma_intr--;
-						/* release if target did not response! */
-						maybe_release_dma_irq(instance);
-						local_irq_restore(flags);
-						break;
-					} else {
-						local_irq_disable();
-						LIST(tmp, hostdata->issue_queue);
-						SET_NEXT(tmp, hostdata->issue_queue);
-						hostdata->issue_queue = tmp;
+			cmd = NCR5380_select(instance, cmd);
+			if (!cmd) {
+				dsprintk(NDEBUG_MAIN, instance, "main: select complete\n");
+				maybe_release_dma_irq(instance);
+			} else {
+				dsprintk(NDEBUG_MAIN | NDEBUG_QUEUES, instance,
+				         "main: select failed, returning %p to queue\n", cmd);
+				requeue_cmd(instance, cmd);
 #ifdef SUPPORT_TAGS
-						cmd_free_tag(tmp);
+				cmd_free_tag(cmd);
 #endif
-						hostdata->retain_dma_intr--;
-						local_irq_restore(flags);
-						dprintk(NDEBUG_MAIN, "scsi%d: main(): select() failed, "
-							    "returned to issue_queue\n", HOSTNO);
-						if (hostdata->connected)
-							break;
-					}
-				} /* if target/lun/target queue is not busy */
-			} /* for issue_queue */
-		} /* if (!hostdata->connected) */
-
+			}
+		}
 		if (hostdata->connected
 #ifdef REAL_DMA
 		    && !hostdata->dma_len
 #endif
 		    ) {
-			local_irq_restore(flags);
-			dprintk(NDEBUG_MAIN, "scsi%d: main: performing information transfer\n",
-				    HOSTNO);
+			dsprintk(NDEBUG_MAIN, instance, "main: performing information transfer\n");
 			NCR5380_information_transfer(instance);
-			dprintk(NDEBUG_MAIN, "scsi%d: main: done set false\n", HOSTNO);
 			done = 0;
 		}
+		spin_unlock_irq(&hostdata->lock);
+		if (!done)
+			cond_resched();
 	} while (!done);
-
-	/* Better allow ints _after_ 'main_running' has been cleared, else
-	   an interrupt could believe we'll pick up the work it left for
-	   us, but we won't see it anymore here... */
-	hostdata->main_running = 0;
-	local_irq_restore(flags);
 }
 
 
@@ -1096,27 +994,20 @@ static void NCR5380_main(struct work_struct *work)
  * Function : void NCR5380_dma_complete (struct Scsi_Host *instance)
  *
  * Purpose : Called by interrupt handler when DMA finishes or a phase
- *	mismatch occurs (which would finish the DMA transfer).
+ * mismatch occurs (which would finish the DMA transfer).
  *
  * Inputs : instance - this instance of the NCR5380.
- *
  */
 
 static void NCR5380_dma_complete(struct Scsi_Host *instance)
 {
-	SETUP_HOSTDATA(instance);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int transferred;
 	unsigned char **data;
-	volatile int *count;
+	int *count;
 	int saved_data = 0, overrun = 0;
 	unsigned char p;
 
-	if (!hostdata->connected) {
-		printk(KERN_WARNING "scsi%d: received end of DMA interrupt with "
-		       "no connected cmd\n", HOSTNO);
-		return;
-	}
-
 	if (hostdata->read_overruns) {
 		p = hostdata->connected->SCp.phase;
 		if (p & SR_IO) {
@@ -1126,15 +1017,11 @@ static void NCR5380_dma_complete(struct Scsi_Host *instance)
 			    (BASR_PHASE_MATCH|BASR_ACK)) {
 				saved_data = NCR5380_read(INPUT_DATA_REG);
 				overrun = 1;
-				dprintk(NDEBUG_DMA, "scsi%d: read overrun handled\n", HOSTNO);
+				dsprintk(NDEBUG_DMA, instance, "read overrun handled\n");
 			}
 		}
 	}
 
-	dprintk(NDEBUG_DMA, "scsi%d: real DMA transfer complete, basr 0x%X, sr 0x%X\n",
-		   HOSTNO, NCR5380_read(BUS_AND_STATUS_REG),
-		   NCR5380_read(STATUS_REG));
-
 #if defined(CONFIG_SUN3)
 	if ((sun3scsi_dma_finish(rq_data_dir(hostdata->connected->request)))) {
 		pr_err("scsi%d: overrun in UDC counter -- not prepared to deal with this!\n",
@@ -1153,9 +1040,9 @@ static void NCR5380_dma_complete(struct Scsi_Host *instance)
 	}
 #endif
 
-	(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 	NCR5380_write(MODE_REG, MR_BASE);
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 
 	transferred = hostdata->dma_len - NCR5380_dma_residual(instance);
 	hostdata->dma_len = 0;
@@ -1194,140 +1081,160 @@ static void NCR5380_dma_complete(struct Scsi_Host *instance)
  * Handle interrupts, reestablishing I_T_L or I_T_L_Q nexuses
  * from the disconnected queue, and restarting NCR5380_main()
  * as required.
+ *
+ * The chip can assert IRQ in any of six different conditions. The IRQ flag
+ * is then cleared by reading the Reset Parity/Interrupt Register (RPIR).
+ * Three of these six conditions are latched in the Bus and Status Register:
+ * - End of DMA (cleared by ending DMA Mode)
+ * - Parity error (cleared by reading RPIR)
+ * - Loss of BSY (cleared by reading RPIR)
+ * Two conditions have flag bits that are not latched:
+ * - Bus phase mismatch (non-maskable in DMA Mode, cleared by ending DMA Mode)
+ * - Bus reset (non-maskable)
+ * The remaining condition has no flag bit at all:
+ * - Selection/reselection
+ *
+ * Hence, establishing the cause(s) of any interrupt is partly guesswork.
+ * In "The DP8490 and DP5380 Comparison Guide", National Semiconductor
+ * claimed that "the design of the [DP8490] interrupt logic ensures
+ * interrupts will not be lost (they can be on the DP5380)."
+ * The L5380/53C80 datasheet from LOGIC Devices has more details.
+ *
+ * Checking for bus reset by reading RST is futile because of interrupt
+ * latency, but a bus reset will reset chip logic. Checking for parity error
+ * is unnecessary because that interrupt is never enabled. A Loss of BSY
+ * condition will clear DMA Mode. We can tell when this occurs because the
+ * the Busy Monitor interrupt is enabled together with DMA Mode.
  */
 
 static irqreturn_t NCR5380_intr(int irq, void *dev_id)
 {
 	struct Scsi_Host *instance = dev_id;
-	int done = 1, handled = 0;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	int handled = 0;
 	unsigned char basr;
+	unsigned long flags;
 
-	dprintk(NDEBUG_INTR, "scsi%d: NCR5380 irq triggered\n", HOSTNO);
+	spin_lock_irqsave(&hostdata->lock, flags);
 
-	/* Look for pending interrupts */
 	basr = NCR5380_read(BUS_AND_STATUS_REG);
-	dprintk(NDEBUG_INTR, "scsi%d: BASR=%02x\n", HOSTNO, basr);
-	/* dispatch to appropriate routine if found and done=0 */
 	if (basr & BASR_IRQ) {
-		NCR5380_dprint(NDEBUG_INTR, instance);
-		if ((NCR5380_read(STATUS_REG) & (SR_SEL|SR_IO)) == (SR_SEL|SR_IO)) {
-			done = 0;
-			dprintk(NDEBUG_INTR, "scsi%d: SEL interrupt\n", HOSTNO);
-			NCR5380_reselect(instance);
-			(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-		} else if (basr & BASR_PARITY_ERROR) {
-			dprintk(NDEBUG_INTR, "scsi%d: PARITY interrupt\n", HOSTNO);
-			(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-		} else if ((NCR5380_read(STATUS_REG) & SR_RST) == SR_RST) {
-			dprintk(NDEBUG_INTR, "scsi%d: RESET interrupt\n", HOSTNO);
-			(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-		} else {
-			/*
-			 * The rest of the interrupt conditions can occur only during a
-			 * DMA transfer
-			 */
+		unsigned char mr = NCR5380_read(MODE_REG);
+		unsigned char sr = NCR5380_read(STATUS_REG);
+
+		dsprintk(NDEBUG_INTR, instance, "IRQ %d, BASR 0x%02x, SR 0x%02x, MR 0x%02x\n",
+		         irq, basr, sr, mr);
 
 #if defined(REAL_DMA)
-			/*
-			 * We should only get PHASE MISMATCH and EOP interrupts if we have
-			 * DMA enabled, so do a sanity check based on the current setting
-			 * of the MODE register.
+		if ((mr & MR_DMA_MODE) || (mr & MR_MONITOR_BSY)) {
+			/* Probably End of DMA, Phase Mismatch or Loss of BSY.
+			 * We ack IRQ after clearing Mode Register. Workarounds
+			 * for End of DMA errata need to happen in DMA Mode.
 			 */
 
-			if ((NCR5380_read(MODE_REG) & MR_DMA_MODE) &&
-			    ((basr & BASR_END_DMA_TRANSFER) ||
-			     !(basr & BASR_PHASE_MATCH))) {
+			dsprintk(NDEBUG_INTR, instance, "interrupt in DMA mode\n");
 
-				dprintk(NDEBUG_INTR, "scsi%d: PHASE MISM or EOP interrupt\n", HOSTNO);
-				NCR5380_dma_complete( instance );
-				done = 0;
-			} else
+			if (hostdata->connected) {
+				NCR5380_dma_complete(instance);
+				queue_work(hostdata->work_q, &hostdata->main_task);
+			} else {
+				NCR5380_write(MODE_REG, MR_BASE);
+				NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+			}
+		} else
 #endif /* REAL_DMA */
-			{
-/* MS: Ignore unknown phase mismatch interrupts (caused by EOP interrupt) */
-				if (basr & BASR_PHASE_MATCH)
-					dprintk(NDEBUG_INTR, "scsi%d: unknown interrupt, "
-					       "BASR 0x%x, MR 0x%x, SR 0x%x\n",
-					       HOSTNO, basr, NCR5380_read(MODE_REG),
-					       NCR5380_read(STATUS_REG));
-				(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+		if ((NCR5380_read(CURRENT_SCSI_DATA_REG) & hostdata->id_mask) &&
+		    (sr & (SR_SEL | SR_IO | SR_BSY | SR_RST)) == (SR_SEL | SR_IO)) {
+			/* Probably reselected */
+			NCR5380_write(SELECT_ENABLE_REG, 0);
+			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+
+			dsprintk(NDEBUG_INTR, instance, "interrupt with SEL and IO\n");
+
+			if (!hostdata->connected) {
+				NCR5380_reselect(instance);
+				queue_work(hostdata->work_q, &hostdata->main_task);
+			}
+			if (!hostdata->connected)
+				NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+		} else {
+			/* Probably Bus Reset */
+			NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+
+			dsprintk(NDEBUG_INTR, instance, "unknown interrupt\n");
 #ifdef SUN3_SCSI_VME
-				dregs->csr |= CSR_DMA_ENABLE;
+			dregs->csr |= CSR_DMA_ENABLE;
 #endif
-			}
-		} /* if !(SELECTION || PARITY) */
+		}
 		handled = 1;
-	} /* BASR & IRQ */ else {
-		printk(KERN_NOTICE "scsi%d: interrupt without IRQ bit set in BASR, "
-		       "BASR 0x%X, MR 0x%X, SR 0x%x\n", HOSTNO, basr,
-		       NCR5380_read(MODE_REG), NCR5380_read(STATUS_REG));
-		(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+	} else {
+		shost_printk(KERN_NOTICE, instance, "interrupt without IRQ bit\n");
 #ifdef SUN3_SCSI_VME
 		dregs->csr |= CSR_DMA_ENABLE;
 #endif
 	}
 
-	if (!done) {
-		dprintk(NDEBUG_INTR, "scsi%d: in int routine, calling main\n", HOSTNO);
-		/* Put a call to NCR5380_main() on the queue... */
-		queue_main(shost_priv(instance));
-	}
+	spin_unlock_irqrestore(&hostdata->lock, flags);
+
 	return IRQ_RETVAL(handled);
 }
 
 /*
  * Function : int NCR5380_select(struct Scsi_Host *instance,
- *                               struct scsi_cmnd *cmd)
+ * struct scsi_cmnd *cmd)
  *
  * Purpose : establishes I_T_L or I_T_L_Q nexus for new or existing command,
- *	including ARBITRATION, SELECTION, and initial message out for
- *	IDENTIFY and queue messages.
+ * including ARBITRATION, SELECTION, and initial message out for
+ * IDENTIFY and queue messages.
  *
  * Inputs : instance - instantiation of the 5380 driver on which this
- *	target lives, cmd - SCSI command to execute.
+ * target lives, cmd - SCSI command to execute.
  *
- * Returns : -1 if selection could not execute for some reason,
- *	0 if selection succeeded or failed because the target
- *	did not respond.
+ * Returns cmd if selection failed but should be retried,
+ * NULL if selection failed and should not be retried, or
+ * NULL if selection succeeded (hostdata->connected == cmd).
  *
  * Side effects :
- *	If bus busy, arbitration failed, etc, NCR5380_select() will exit
- *		with registers as they should have been on entry - ie
- *		SELECT_ENABLE will be set appropriately, the NCR5380
- *		will cease to drive any SCSI bus signals.
+ * If bus busy, arbitration failed, etc, NCR5380_select() will exit
+ * with registers as they should have been on entry - ie
+ * SELECT_ENABLE will be set appropriately, the NCR5380
+ * will cease to drive any SCSI bus signals.
  *
- *	If successful : I_T_L or I_T_L_Q nexus will be established,
- *		instance->connected will be set to cmd.
- *		SELECT interrupt will be disabled.
+ * If successful : I_T_L or I_T_L_Q nexus will be established,
+ * instance->connected will be set to cmd.
+ * SELECT interrupt will be disabled.
  *
- *	If failed (no target) : cmd->scsi_done() will be called, and the
- *		cmd->result host byte set to DID_BAD_TARGET.
+ * If failed (no target) : cmd->scsi_done() will be called, and the
+ * cmd->result host byte set to DID_BAD_TARGET.
  */
 
-static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
+static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance,
+                                        struct scsi_cmnd *cmd)
 {
-	SETUP_HOSTDATA(instance);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char tmp[3], phase;
 	unsigned char *data;
 	int len;
-	unsigned long timeout;
-	unsigned long flags;
+	int err;
 
-	hostdata->restart_select = 0;
 	NCR5380_dprint(NDEBUG_ARBITRATION, instance);
-	dprintk(NDEBUG_ARBITRATION, "scsi%d: starting arbitration, id = %d\n", HOSTNO,
-		   instance->this_id);
+	dsprintk(NDEBUG_ARBITRATION, instance, "starting arbitration, id = %d\n",
+	         instance->this_id);
+
+	/*
+	 * Arbitration and selection phases are slow and involve dropping the
+	 * lock, so we have to watch out for EH. An exception handler may
+	 * change 'selecting' to NULL. This function will then return NULL
+	 * so that the caller will forget about 'cmd'. (During information
+	 * transfer phases, EH may change 'connected' to NULL.)
+	 */
+	hostdata->selecting = cmd;
 
 	/*
 	 * Set the phase bits to 0, otherwise the NCR5380 won't drive the
 	 * data bus during SELECTION.
 	 */
 
-	local_irq_save(flags);
-	if (hostdata->connected) {
-		local_irq_restore(flags);
-		return -1;
-	}
 	NCR5380_write(TARGET_COMMAND_REG, 0);
 
 	/*
@@ -1337,96 +1244,77 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 	NCR5380_write(OUTPUT_DATA_REG, hostdata->id_mask);
 	NCR5380_write(MODE_REG, MR_ARBITRATE);
 
-	local_irq_restore(flags);
-
-	/* Wait for arbitration logic to complete */
-#if defined(NCR_TIMEOUT)
-	{
-		unsigned long timeout = jiffies + 2*NCR_TIMEOUT;
+	/* The chip now waits for BUS FREE phase. Then after the 800 ns
+	 * Bus Free Delay, arbitration will begin.
+	 */
 
-		while (!(NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_PROGRESS) &&
-		       time_before(jiffies, timeout) && !hostdata->connected)
-			;
-		if (time_after_eq(jiffies, timeout)) {
-			printk("scsi : arbitration timeout at %d\n", __LINE__);
-			NCR5380_write(MODE_REG, MR_BASE);
-			NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-			return -1;
-		}
+	spin_unlock_irq(&hostdata->lock);
+	err = NCR5380_poll_politely2(instance, MODE_REG, MR_ARBITRATE, 0,
+	                INITIATOR_COMMAND_REG, ICR_ARBITRATION_PROGRESS,
+	                                       ICR_ARBITRATION_PROGRESS, HZ);
+	spin_lock_irq(&hostdata->lock);
+	if (!(NCR5380_read(MODE_REG) & MR_ARBITRATE)) {
+		/* Reselection interrupt */
+		goto out;
 	}
-#else /* NCR_TIMEOUT */
-	while (!(NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_PROGRESS) &&
-	       !hostdata->connected)
-		;
-#endif
-
-	dprintk(NDEBUG_ARBITRATION, "scsi%d: arbitration complete\n", HOSTNO);
-
-	if (hostdata->connected) {
+	if (err < 0) {
 		NCR5380_write(MODE_REG, MR_BASE);
-		return -1;
+		shost_printk(KERN_ERR, instance,
+		             "select: arbitration timeout\n");
+		goto out;
 	}
-	/*
-	 * The arbitration delay is 2.2us, but this is a minimum and there is
-	 * no maximum so we can safely sleep for ceil(2.2) usecs to accommodate
-	 * the integral nature of udelay().
-	 *
-	 */
+	spin_unlock_irq(&hostdata->lock);
 
+	/* The SCSI-2 arbitration delay is 2.4 us */
 	udelay(3);
 
 	/* Check for lost arbitration */
 	if ((NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST) ||
 	    (NCR5380_read(CURRENT_SCSI_DATA_REG) & hostdata->id_higher_mask) ||
-	    (NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST) ||
-	    hostdata->connected) {
+	    (NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST)) {
 		NCR5380_write(MODE_REG, MR_BASE);
-		dprintk(NDEBUG_ARBITRATION, "scsi%d: lost arbitration, deasserting MR_ARBITRATE\n",
-			   HOSTNO);
-		return -1;
+		dsprintk(NDEBUG_ARBITRATION, instance, "lost arbitration, deasserting MR_ARBITRATE\n");
+		spin_lock_irq(&hostdata->lock);
+		goto out;
 	}
 
-	/* after/during arbitration, BSY should be asserted.
-	   IBM DPES-31080 Version S31Q works now */
-	/* Tnx to Thomas_Roesch@m2.maus.de for finding this! (Roman) */
+	/* After/during arbitration, BSY should be asserted.
+	 * IBM DPES-31080 Version S31Q works now
+	 * Tnx to Thomas_Roesch@m2.maus.de for finding this! (Roman)
+	 */
 	NCR5380_write(INITIATOR_COMMAND_REG,
 		      ICR_BASE | ICR_ASSERT_SEL | ICR_ASSERT_BSY);
 
-	if ((NCR5380_read(INITIATOR_COMMAND_REG) & ICR_ARBITRATION_LOST) ||
-	    hostdata->connected) {
-		NCR5380_write(MODE_REG, MR_BASE);
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		dprintk(NDEBUG_ARBITRATION, "scsi%d: lost arbitration, deasserting ICR_ASSERT_SEL\n",
-			   HOSTNO);
-		return -1;
-	}
-
 	/*
 	 * Again, bus clear + bus settle time is 1.2us, however, this is
 	 * a minimum so we'll udelay ceil(1.2)
 	 */
 
-#ifdef CONFIG_ATARI_SCSI_TOSHIBA_DELAY
-	/* ++roman: But some targets (see above :-) seem to need a bit more... */
-	udelay(15);
-#else
-	udelay(2);
-#endif
+	if (hostdata->flags & FLAG_TOSHIBA_DELAY)
+		udelay(15);
+	else
+		udelay(2);
 
-	if (hostdata->connected) {
+	spin_lock_irq(&hostdata->lock);
+
+	/* NCR5380_reselect() clears MODE_REG after a reselection interrupt */
+	if (!(NCR5380_read(MODE_REG) & MR_ARBITRATE))
+		goto out;
+
+	if (!hostdata->selecting) {
 		NCR5380_write(MODE_REG, MR_BASE);
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		return -1;
+		goto out;
 	}
 
-	dprintk(NDEBUG_ARBITRATION, "scsi%d: won arbitration\n", HOSTNO);
+	dsprintk(NDEBUG_ARBITRATION, instance, "won arbitration\n");
 
 	/*
 	 * Now that we have won arbitration, start Selection process, asserting
 	 * the host and target ID's on the SCSI bus.
 	 */
 
-	NCR5380_write(OUTPUT_DATA_REG, (hostdata->id_mask | (1 << cmd->device->id)));
+	NCR5380_write(OUTPUT_DATA_REG, hostdata->id_mask | (1 << scmd_id(cmd)));
 
 	/*
 	 * Raise ATN while SEL is true before BSY goes false from arbitration,
@@ -1434,22 +1322,18 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 	 * phase immediately after selection.
 	 */
 
-	NCR5380_write(INITIATOR_COMMAND_REG, (ICR_BASE | ICR_ASSERT_BSY |
-		      ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_SEL ));
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY |
+	              ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_SEL);
 	NCR5380_write(MODE_REG, MR_BASE);
 
 	/*
 	 * Reselect interrupts must be turned off prior to the dropping of BSY,
 	 * otherwise we will trigger an interrupt.
 	 */
-
-	if (hostdata->connected) {
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		return -1;
-	}
-
 	NCR5380_write(SELECT_ENABLE_REG, 0);
 
+	spin_unlock_irq(&hostdata->lock);
+
 	/*
 	 * The initiator shall then wait at least two deskew delays and release
 	 * the BSY signal.
@@ -1457,8 +1341,8 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 	udelay(1);        /* wingel -- wait two bus deskew delay >2*45ns */
 
 	/* Reset BSY */
-	NCR5380_write(INITIATOR_COMMAND_REG, (ICR_BASE | ICR_ASSERT_DATA |
-		      ICR_ASSERT_ATN | ICR_ASSERT_SEL));
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA |
+	              ICR_ASSERT_ATN | ICR_ASSERT_SEL);
 
 	/*
 	 * Something weird happens when we cease to drive BSY - looks
@@ -1479,45 +1363,39 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 
 	udelay(1);
 
-	dprintk(NDEBUG_SELECTION, "scsi%d: selecting target %d\n", HOSTNO, cmd->device->id);
+	dsprintk(NDEBUG_SELECTION, instance, "selecting target %d\n", scmd_id(cmd));
 
 	/*
 	 * The SCSI specification calls for a 250 ms timeout for the actual
 	 * selection.
 	 */
 
-	timeout = jiffies + msecs_to_jiffies(250);
-
-	/*
-	 * XXX very interesting - we're seeing a bounce where the BSY we
-	 * asserted is being reflected / still asserted (propagation delay?)
-	 * and it's detecting as true.  Sigh.
-	 */
-
-#if 0
-	/* ++roman: If a target conformed to the SCSI standard, it wouldn't assert
-	 * IO while SEL is true. But again, there are some disks out the in the
-	 * world that do that nevertheless. (Somebody claimed that this announces
-	 * reselection capability of the target.) So we better skip that test and
-	 * only wait for BSY... (Famous german words: Der Klügere gibt nach :-)
-	 */
-
-	while (time_before(jiffies, timeout) &&
-	       !(NCR5380_read(STATUS_REG) & (SR_BSY | SR_IO)))
-		;
+	err = NCR5380_poll_politely(instance, STATUS_REG, SR_BSY, SR_BSY,
+	                            msecs_to_jiffies(250));
 
 	if ((NCR5380_read(STATUS_REG) & (SR_SEL | SR_IO)) == (SR_SEL | SR_IO)) {
+		spin_lock_irq(&hostdata->lock);
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		NCR5380_reselect(instance);
-		printk(KERN_ERR "scsi%d: reselection after won arbitration?\n",
-		       HOSTNO);
+		if (!hostdata->connected)
+			NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+		shost_printk(KERN_ERR, instance, "reselection after won arbitration?\n");
+		goto out;
+	}
+
+	if (err < 0) {
+		spin_lock_irq(&hostdata->lock);
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		return -1;
+		/* Can't touch cmd if it has been reclaimed by the scsi ML */
+		if (hostdata->selecting) {
+			cmd->result = DID_BAD_TARGET << 16;
+			complete_cmd(instance, cmd);
+			dsprintk(NDEBUG_SELECTION, instance, "target did not respond within 250ms\n");
+			cmd = NULL;
+		}
+		goto out;
 	}
-#else
-	while (time_before(jiffies, timeout) && !(NCR5380_read(STATUS_REG) & SR_BSY))
-		;
-#endif
 
 	/*
 	 * No less than two deskew delays after the initiator detects the
@@ -1529,29 +1407,6 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 
-	if (!(NCR5380_read(STATUS_REG) & SR_BSY)) {
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-		if (hostdata->targets_present & (1 << cmd->device->id)) {
-			printk(KERN_ERR "scsi%d: weirdness\n", HOSTNO);
-			if (hostdata->restart_select)
-				printk(KERN_NOTICE "\trestart select\n");
-			NCR5380_dprint(NDEBUG_ANY, instance);
-			NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-			return -1;
-		}
-		cmd->result = DID_BAD_TARGET << 16;
-#ifdef SUPPORT_TAGS
-		cmd_free_tag(cmd);
-#endif
-		cmd->scsi_done(cmd);
-		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		dprintk(NDEBUG_SELECTION, "scsi%d: target did not respond within 250ms\n", HOSTNO);
-		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-		return 0;
-	}
-
-	hostdata->targets_present |= (1 << cmd->device->id);
-
 	/*
 	 * Since we followed the SCSI spec, and raised ATN while SEL
 	 * was true but before BSY was false during selection, the information
@@ -1563,16 +1418,27 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 	 * until it wraps back to 0.
 	 *
 	 * XXX - it turns out that there are some broken SCSI-II devices,
-	 *	     which claim to support tagged queuing but fail when more than
-	 *	     some number of commands are issued at once.
+	 * which claim to support tagged queuing but fail when more than
+	 * some number of commands are issued at once.
 	 */
 
 	/* Wait for start of REQ/ACK handshake */
-	while (!(NCR5380_read(STATUS_REG) & SR_REQ))
-		;
 
-	dprintk(NDEBUG_SELECTION, "scsi%d: target %d selected, going into MESSAGE OUT phase.\n",
-		   HOSTNO, cmd->device->id);
+	err = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
+	spin_lock_irq(&hostdata->lock);
+	if (err < 0) {
+		shost_printk(KERN_ERR, instance, "select: REQ timeout\n");
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+		NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+		goto out;
+	}
+	if (!hostdata->selecting) {
+		do_abort(instance);
+		goto out;
+	}
+
+	dsprintk(NDEBUG_SELECTION, instance, "target %d selected, going into MESSAGE OUT phase.\n",
+	         scmd_id(cmd));
 	tmp[0] = IDENTIFY(1, cmd->device->lun);
 
 #ifdef SUPPORT_TAGS
@@ -1591,11 +1457,12 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 	data = tmp;
 	phase = PHASE_MSGOUT;
 	NCR5380_transfer_pio(instance, &phase, &len, &data);
-	dprintk(NDEBUG_SELECTION, "scsi%d: nexus established.\n", HOSTNO);
+	dsprintk(NDEBUG_SELECTION, instance, "nexus established.\n");
 	/* XXX need to handle errors here */
+
 	hostdata->connected = cmd;
 #ifndef SUPPORT_TAGS
-	hostdata->busy[cmd->device->id] |= (1 << cmd->device->lun);
+	hostdata->busy[cmd->device->id] |= 1 << cmd->device->lun;
 #endif
 #ifdef SUN3_SCSI_VME
 	dregs->csr |= CSR_INTR;
@@ -1603,24 +1470,30 @@ static int NCR5380_select(struct Scsi_Host *instance, struct scsi_cmnd *cmd)
 
 	initialize_SCp(cmd);
 
-	return 0;
+	cmd = NULL;
+
+out:
+	if (!hostdata->selecting)
+		return NULL;
+	hostdata->selecting = NULL;
+	return cmd;
 }
 
 /*
  * Function : int NCR5380_transfer_pio (struct Scsi_Host *instance,
- *      unsigned char *phase, int *count, unsigned char **data)
+ * unsigned char *phase, int *count, unsigned char **data)
  *
  * Purpose : transfers data in given phase using polled I/O
  *
  * Inputs : instance - instance of driver, *phase - pointer to
- *	what phase is expected, *count - pointer to number of
- *	bytes to transfer, **data - pointer to data pointer.
+ * what phase is expected, *count - pointer to number of
+ * bytes to transfer, **data - pointer to data pointer.
  *
  * Returns : -1 when different phase is entered without transferring
- *	maximum number of bytes, 0 if all bytes are transferred or exit
- *	is in same phase.
+ * maximum number of bytes, 0 if all bytes are transferred or exit
+ * is in same phase.
  *
- *	Also, *phase, *count, *data are modified in place.
+ * Also, *phase, *count, *data are modified in place.
  *
  * XXX Note : handling for bus free may be useful.
  */
@@ -1635,9 +1508,9 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 				unsigned char *phase, int *count,
 				unsigned char **data)
 {
-	register unsigned char p = *phase, tmp;
-	register int c = *count;
-	register unsigned char *d = *data;
+	unsigned char p = *phase, tmp;
+	int c = *count;
+	unsigned char *d = *data;
 
 	/*
 	 * The NCR5380 chip will only drive the SCSI bus when the
@@ -1652,14 +1525,15 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 		 * Wait for assertion of REQ, after which the phase bits will be
 		 * valid
 		 */
-		while (!((tmp = NCR5380_read(STATUS_REG)) & SR_REQ))
-			;
 
-		dprintk(NDEBUG_HANDSHAKE, "scsi%d: REQ detected\n", HOSTNO);
+		if (NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0)
+			break;
+
+		dsprintk(NDEBUG_HANDSHAKE, instance, "REQ asserted\n");
 
 		/* Check for phase mismatch */
-		if ((tmp & PHASE_MASK) != p) {
-			dprintk(NDEBUG_PIO, "scsi%d: phase mismatch\n", HOSTNO);
+		if ((NCR5380_read(STATUS_REG) & PHASE_MASK) != p) {
+			dsprintk(NDEBUG_PIO, instance, "phase mismatch\n");
 			NCR5380_dprint_phase(NDEBUG_PIO, instance);
 			break;
 		}
@@ -1684,35 +1558,36 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_DATA);
 				NCR5380_dprint(NDEBUG_PIO, instance);
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
-					      ICR_ASSERT_DATA | ICR_ASSERT_ACK);
+				              ICR_ASSERT_DATA | ICR_ASSERT_ACK);
 			} else {
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
-					      ICR_ASSERT_DATA | ICR_ASSERT_ATN);
+				              ICR_ASSERT_DATA | ICR_ASSERT_ATN);
 				NCR5380_dprint(NDEBUG_PIO, instance);
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
-					      ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
+				              ICR_ASSERT_DATA | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
 			}
 		} else {
 			NCR5380_dprint(NDEBUG_PIO, instance);
 			NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ACK);
 		}
 
-		while (NCR5380_read(STATUS_REG) & SR_REQ)
-			;
+		if (NCR5380_poll_politely(instance,
+		                          STATUS_REG, SR_REQ, 0, 5 * HZ) < 0)
+			break;
 
-		dprintk(NDEBUG_HANDSHAKE, "scsi%d: req false, handshake complete\n", HOSTNO);
+		dsprintk(NDEBUG_HANDSHAKE, instance, "REQ negated, handshake complete\n");
 
-		/*
-		 * We have several special cases to consider during REQ/ACK handshaking :
-		 * 1.  We were in MSGOUT phase, and we are on the last byte of the
-		 *	message.  ATN must be dropped as ACK is dropped.
-		 *
-		 * 2.  We are in a MSGIN phase, and we are on the last byte of the
-		 *	message.  We must exit with ACK asserted, so that the calling
-		 *	code may raise ATN before dropping ACK to reject the message.
-		 *
-		 * 3.  ACK and ATN are clear and the target may proceed as normal.
-		 */
+/*
+ * We have several special cases to consider during REQ/ACK handshaking :
+ * 1.  We were in MSGOUT phase, and we are on the last byte of the
+ * message.  ATN must be dropped as ACK is dropped.
+ *
+ * 2.  We are in a MSGIN phase, and we are on the last byte of the
+ * message.  We must exit with ACK asserted, so that the calling
+ * code may raise ATN before dropping ACK to reject the message.
+ *
+ * 3.  ACK and ATN are clear and the target may proceed as normal.
+ */
 		if (!(p == PHASE_MSGIN && c == 1)) {
 			if (p == PHASE_MSGOUT && c > 1)
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
@@ -1721,16 +1596,16 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 		}
 	} while (--c);
 
-	dprintk(NDEBUG_PIO, "scsi%d: residual %d\n", HOSTNO, c);
+	dsprintk(NDEBUG_PIO, instance, "residual %d\n", c);
 
 	*count = c;
 	*data = d;
 	tmp = NCR5380_read(STATUS_REG);
 	/* The phase read from the bus is valid if either REQ is (already)
-	 * asserted or if ACK hasn't been released yet. The latter is the case if
-	 * we're in MSGIN and all wanted bytes have been received.
+	 * asserted or if ACK hasn't been released yet. The latter applies if
+	 * we're in MSG IN, DATA IN or STATUS and all bytes have been received.
 	 */
-	if ((tmp & SR_REQ) || (p == PHASE_MSGIN && c == 0))
+	if ((tmp & SR_REQ) || ((tmp & SR_IO) && c == 0))
 		*phase = tmp & PHASE_MASK;
 	else
 		*phase = PHASE_UNKNOWN;
@@ -1741,19 +1616,45 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 		return -1;
 }
 
-/*
- * Function : do_abort (Scsi_Host *host)
+/**
+ * do_reset - issue a reset command
+ * @instance: adapter to reset
  *
- * Purpose : abort the currently established nexus.  Should only be
- *	called from a routine which can drop into a
+ * Issue a reset sequence to the NCR5380 and try and get the bus
+ * back into sane shape.
  *
- * Returns : 0 on success, -1 on failure.
+ * This clears the reset interrupt flag because there may be no handler for
+ * it. When the driver is initialized, the NCR5380_intr() handler has not yet
+ * been installed. And when in EH we may have released the ST DMA interrupt.
+ */
+
+static void do_reset(struct Scsi_Host *instance)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	NCR5380_write(TARGET_COMMAND_REG,
+	              PHASE_SR_TO_TCR(NCR5380_read(STATUS_REG) & PHASE_MASK));
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST);
+	udelay(50);
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+	(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
+	local_irq_restore(flags);
+}
+
+/**
+ * do_abort - abort the currently established nexus by going to
+ * MESSAGE OUT phase and sending an ABORT message.
+ * @instance: relevant scsi host instance
+ *
+ * Returns 0 on success, -1 on failure.
  */
 
 static int do_abort(struct Scsi_Host *instance)
 {
-	unsigned char tmp, *msgptr, phase;
+	unsigned char *msgptr, phase, tmp;
 	int len;
+	int rc;
 
 	/* Request message out phase */
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
@@ -1768,16 +1669,20 @@ static int do_abort(struct Scsi_Host *instance)
 	 * the target sees, so we just handshake.
 	 */
 
-	while (!((tmp = NCR5380_read(STATUS_REG)) & SR_REQ))
-		;
+	rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ);
+	if (rc < 0)
+		goto timeout;
+
+	tmp = NCR5380_read(STATUS_REG) & PHASE_MASK;
 
 	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(tmp));
 
-	if ((tmp & PHASE_MASK) != PHASE_MSGOUT) {
-		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN |
-			      ICR_ASSERT_ACK);
-		while (NCR5380_read(STATUS_REG) & SR_REQ)
-			;
+	if (tmp != PHASE_MSGOUT) {
+		NCR5380_write(INITIATOR_COMMAND_REG,
+		              ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
+		rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, 0, 3 * HZ);
+		if (rc < 0)
+			goto timeout;
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
 	}
 
@@ -1793,26 +1698,29 @@ static int do_abort(struct Scsi_Host *instance)
 	 */
 
 	return len ? -1 : 0;
+
+timeout:
+	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+	return -1;
 }
 
 #if defined(REAL_DMA)
 /*
  * Function : int NCR5380_transfer_dma (struct Scsi_Host *instance,
- *      unsigned char *phase, int *count, unsigned char **data)
+ * unsigned char *phase, int *count, unsigned char **data)
  *
  * Purpose : transfers data in given phase using either real
- *	or pseudo DMA.
+ * or pseudo DMA.
  *
  * Inputs : instance - instance of driver, *phase - pointer to
- *	what phase is expected, *count - pointer to number of
- *	bytes to transfer, **data - pointer to data pointer.
+ * what phase is expected, *count - pointer to number of
+ * bytes to transfer, **data - pointer to data pointer.
  *
  * Returns : -1 when different phase is entered without transferring
- *	maximum number of bytes, 0 if all bytes or transferred or exit
- *	is in same phase.
- *
- *	Also, *phase, *count, *data are modified in place.
+ * maximum number of bytes, 0 if all bytes or transferred or exit
+ * is in same phase.
  *
+ * Also, *phase, *count, *data are modified in place.
  */
 
 
@@ -1820,10 +1728,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 				unsigned char *phase, int *count,
 				unsigned char **data)
 {
-	SETUP_HOSTDATA(instance);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	register int c = *count;
 	register unsigned char p = *phase;
-	unsigned long flags;
 
 #if defined(CONFIG_SUN3)
 	/* sanity check */
@@ -1834,29 +1741,22 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 	}
 	hostdata->dma_len = c;
 
-	dprintk(NDEBUG_DMA, "scsi%d: initializing DMA for %s, %d bytes %s %p\n",
-		instance->host_no, (p & SR_IO) ? "reading" : "writing",
-		c, (p & SR_IO) ? "to" : "from", *data);
+	dsprintk(NDEBUG_DMA, instance, "initializing DMA %s: length %d, address %p\n",
+	         (p & SR_IO) ? "receive" : "send", c, *data);
 
 	/* netbsd turns off ints here, why not be safe and do it too */
-	local_irq_save(flags);
 
 	/* send start chain */
 	sun3scsi_dma_start(c, *data);
 
+	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(p));
+	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY |
+	                        MR_ENABLE_EOP_INTR);
 	if (p & SR_IO) {
-		NCR5380_write(TARGET_COMMAND_REG, 1);
-		NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 		NCR5380_write(INITIATOR_COMMAND_REG, 0);
-		NCR5380_write(MODE_REG,
-			      (NCR5380_read(MODE_REG) | MR_DMA_MODE | MR_ENABLE_EOP_INTR));
 		NCR5380_write(START_DMA_INITIATOR_RECEIVE_REG, 0);
 	} else {
-		NCR5380_write(TARGET_COMMAND_REG, 0);
-		NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_ASSERT_DATA);
-		NCR5380_write(MODE_REG,
-			      (NCR5380_read(MODE_REG) | MR_DMA_MODE | MR_ENABLE_EOP_INTR));
 		NCR5380_write(START_DMA_SEND_REG, 0);
 	}
 
@@ -1864,8 +1764,6 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 	dregs->csr |= CSR_DMA_ENABLE;
 #endif
 
-	local_irq_restore(flags);
-
 	sun3_dma_active = 1;
 
 #else /* !defined(CONFIG_SUN3) */
@@ -1880,25 +1778,20 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 	if (hostdata->read_overruns && (p & SR_IO))
 		c -= hostdata->read_overruns;
 
-	dprintk(NDEBUG_DMA, "scsi%d: initializing DMA for %s, %d bytes %s %p\n",
-		   HOSTNO, (p & SR_IO) ? "reading" : "writing",
-		   c, (p & SR_IO) ? "to" : "from", d);
+	dsprintk(NDEBUG_DMA, instance, "initializing DMA %s: length %d, address %p\n",
+	         (p & SR_IO) ? "receive" : "send", c, d);
 
 	NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(p));
-
-#ifdef REAL_DMA
-	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_ENABLE_EOP_INTR | MR_MONITOR_BSY);
-#endif /* def REAL_DMA  */
+	NCR5380_write(MODE_REG, MR_BASE | MR_DMA_MODE | MR_MONITOR_BSY |
+	                        MR_ENABLE_EOP_INTR);
 
 	if (!(hostdata->flags & FLAG_LATE_DMA_SETUP)) {
 		/* On the Medusa, it is a must to initialize the DMA before
 		 * starting the NCR. This is also the cleaner way for the TT.
 		 */
-		local_irq_save(flags);
 		hostdata->dma_len = (p & SR_IO) ?
 			NCR5380_dma_read_setup(instance, d, c) :
 			NCR5380_dma_write_setup(instance, d, c);
-		local_irq_restore(flags);
 	}
 
 	if (p & SR_IO)
@@ -1912,11 +1805,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 		/* On the Falcon, the DMA setup must be done after the last */
 		/* NCR access, else the DMA setup gets trashed!
 		 */
-		local_irq_save(flags);
 		hostdata->dma_len = (p & SR_IO) ?
 			NCR5380_dma_read_setup(instance, d, c) :
 			NCR5380_dma_write_setup(instance, d, c);
-		local_irq_restore(flags);
 	}
 #endif /* !defined(CONFIG_SUN3) */
 
@@ -1928,23 +1819,22 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
  * Function : NCR5380_information_transfer (struct Scsi_Host *instance)
  *
  * Purpose : run through the various SCSI phases and do as the target
- *	directs us to.  Operates on the currently connected command,
- *	instance->connected.
+ * directs us to.  Operates on the currently connected command,
+ * instance->connected.
  *
  * Inputs : instance, instance for which we are doing commands
  *
  * Side effects : SCSI things happen, the disconnected queue will be
- *	modified if a command disconnects, *instance->connected will
- *	change.
+ * modified if a command disconnects, *instance->connected will
+ * change.
  *
  * XXX Note : we need to watch for bus free or a reset condition here
- *	to recover from an unexpected bus free condition.
+ * to recover from an unexpected bus free condition.
  */
 
 static void NCR5380_information_transfer(struct Scsi_Host *instance)
 {
-	SETUP_HOSTDATA(instance);
-	unsigned long flags;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char msgout = NOP;
 	int sink = 0;
 	int len;
@@ -1953,13 +1843,15 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 #endif
 	unsigned char *data;
 	unsigned char phase, tmp, extended_msg[10], old_phase = 0xff;
-	struct scsi_cmnd *cmd = (struct scsi_cmnd *) hostdata->connected;
+	struct scsi_cmnd *cmd;
 
 #ifdef SUN3_SCSI_VME
 	dregs->csr |= CSR_INTR;
 #endif
 
-	while (1) {
+	while ((cmd = hostdata->connected)) {
+		struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
+
 		tmp = NCR5380_read(STATUS_REG);
 		/* We only have a valid SCSI phase when REQ is asserted */
 		if (tmp & SR_REQ) {
@@ -1984,7 +1876,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				/* this command setup for dma yet? */
 				if ((count >= DMA_MIN_SIZE) && (sun3_dma_setup_done != cmd)) {
 					if (cmd->request->cmd_type == REQ_TYPE_FS) {
-						sun3scsi_dma_setup(d, count,
+						sun3scsi_dma_setup(instance, d, count,
 						                   rq_data_dir(cmd->request));
 						sun3_dma_setup_done = cmd;
 					}
@@ -2000,11 +1892,11 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				NCR5380_write(TARGET_COMMAND_REG, PHASE_SR_TO_TCR(tmp));
 
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN |
-					      ICR_ASSERT_ACK);
+				              ICR_ASSERT_ACK);
 				while (NCR5380_read(STATUS_REG) & SR_REQ)
 					;
 				NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
-					      ICR_ASSERT_ATN);
+				              ICR_ASSERT_ATN);
 				sink = 0;
 				continue;
 			}
@@ -2012,12 +1904,11 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 			switch (phase) {
 			case PHASE_DATAOUT:
 #if (NDEBUG & NDEBUG_NO_DATAOUT)
-				printk("scsi%d: NDEBUG_NO_DATAOUT set, attempted DATAOUT "
-				       "aborted\n", HOSTNO);
+				shost_printk(KERN_DEBUG, instance, "NDEBUG_NO_DATAOUT set, attempted DATAOUT aborted\n");
 				sink = 1;
 				do_abort(instance);
 				cmd->result = DID_ERROR << 16;
-				cmd->scsi_done(cmd);
+				complete_cmd(instance, cmd);
 				return;
 #endif
 			case PHASE_DATAIN:
@@ -2031,13 +1922,10 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 					--cmd->SCp.buffers_residual;
 					cmd->SCp.this_residual = cmd->SCp.buffer->length;
 					cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
-					/* ++roman: Try to merge some scatter-buffers if
-					 * they are at contiguous physical addresses.
-					 */
 					merge_contiguous_buffers(cmd);
-					dprintk(NDEBUG_INFORMATION, "scsi%d: %d bytes and %d buffers left\n",
-						   HOSTNO, cmd->SCp.this_residual,
-						   cmd->SCp.buffers_residual);
+					dsprintk(NDEBUG_INFORMATION, instance, "%d bytes and %d buffers left\n",
+					         cmd->SCp.this_residual,
+					         cmd->SCp.buffers_residual);
 				}
 
 				/*
@@ -2051,16 +1939,18 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				 */
 
 				/* ++roman: I suggest, this should be
-				 *   #if def(REAL_DMA)
+				 * #if def(REAL_DMA)
 				 * instead of leaving REAL_DMA out.
 				 */
 
 #if defined(REAL_DMA)
-				if (
 #if !defined(CONFIG_SUN3)
-				    !cmd->device->borken &&
+				transfersize = 0;
+				if (!cmd->device->borken)
 #endif
-				    (transfersize = NCR5380_dma_xfer_len(instance, cmd, phase)) >= DMA_MIN_SIZE) {
+					transfersize = NCR5380_dma_xfer_len(instance, cmd, phase);
+
+				if (transfersize >= DMA_MIN_SIZE) {
 					len = transfersize;
 					cmd->SCp.phase = phase;
 					if (NCR5380_transfer_dma(instance, &phase,
@@ -2068,16 +1958,15 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 						/*
 						 * If the watchdog timer fires, all future
 						 * accesses to this device will use the
-						 * polled-IO. */
+						 * polled-IO.
+						 */
 						scmd_printk(KERN_INFO, cmd,
 							"switching to slow handshake\n");
 						cmd->device->borken = 1;
-						NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE |
-							ICR_ASSERT_ATN);
 						sink = 1;
 						do_abort(instance);
 						cmd->result = DID_ERROR << 16;
-						cmd->scsi_done(cmd);
+						complete_cmd(instance, cmd);
 						/* XXX - need to source or sink data here, as appropriate */
 					} else {
 #ifdef REAL_DMA
@@ -2093,9 +1982,13 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 					}
 				} else
 #endif /* defined(REAL_DMA) */
+				{
+					spin_unlock_irq(&hostdata->lock);
 					NCR5380_transfer_pio(instance, &phase,
-							     (int *)&cmd->SCp.this_residual,
-							     (unsigned char **)&cmd->SCp.ptr);
+					                     (int *)&cmd->SCp.this_residual,
+					                     (unsigned char **)&cmd->SCp.ptr);
+					spin_lock_irq(&hostdata->lock);
+				}
 #if defined(CONFIG_SUN3) && defined(REAL_DMA)
 				/* if we had intended to dma that command clear it */
 				if (sun3_dma_setup_done == cmd)
@@ -2105,162 +1998,64 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 			case PHASE_MSGIN:
 				len = 1;
 				data = &tmp;
-				NCR5380_write(SELECT_ENABLE_REG, 0);	/* disable reselects */
 				NCR5380_transfer_pio(instance, &phase, &len, &data);
 				cmd->SCp.Message = tmp;
 
 				switch (tmp) {
-				/*
-				 * Linking lets us reduce the time required to get the
-				 * next command out to the device, hopefully this will
-				 * mean we don't waste another revolution due to the delays
-				 * required by ARBITRATION and another SELECTION.
-				 *
-				 * In the current implementation proposal, low level drivers
-				 * merely have to start the next command, pointed to by
-				 * next_link, done() is called as with unlinked commands.
-				 */
-#ifdef LINKED
-				case LINKED_CMD_COMPLETE:
-				case LINKED_FLG_CMD_COMPLETE:
-					/* Accept message by clearing ACK */
-					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-
-					dprintk(NDEBUG_LINKED, "scsi%d: target %d lun %llu linked command "
-						   "complete.\n", HOSTNO, cmd->device->id, cmd->device->lun);
-
-					/* Enable reselect interrupts */
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-					/*
-					 * Sanity check : A linked command should only terminate
-					 * with one of these messages if there are more linked
-					 * commands available.
-					 */
-
-					if (!cmd->next_link) {
-						 printk(KERN_NOTICE "scsi%d: target %d lun %llu "
-							"linked command complete, no next_link\n",
-							HOSTNO, cmd->device->id, cmd->device->lun);
-						sink = 1;
-						do_abort(instance);
-						return;
-					}
-
-					initialize_SCp(cmd->next_link);
-					/* The next command is still part of this process; copy it
-					 * and don't free it! */
-					cmd->next_link->tag = cmd->tag;
-					cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-					dprintk(NDEBUG_LINKED, "scsi%d: target %d lun %llu linked request "
-						   "done, calling scsi_done().\n",
-						   HOSTNO, cmd->device->id, cmd->device->lun);
-					cmd->scsi_done(cmd);
-					cmd = hostdata->connected;
-					break;
-#endif /* def LINKED */
 				case ABORT:
 				case COMMAND_COMPLETE:
 					/* Accept message by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					dprintk(NDEBUG_QUEUES, "scsi%d: command for target %d, lun %llu "
-						  "completed\n", HOSTNO, cmd->device->id, cmd->device->lun);
+					dsprintk(NDEBUG_QUEUES, instance,
+					         "COMMAND COMPLETE %p target %d lun %llu\n",
+					         cmd, scmd_id(cmd), cmd->device->lun);
 
-					local_irq_save(flags);
-					hostdata->retain_dma_intr++;
 					hostdata->connected = NULL;
 #ifdef SUPPORT_TAGS
 					cmd_free_tag(cmd);
 					if (status_byte(cmd->SCp.Status) == QUEUE_FULL) {
-						/* Turn a QUEUE FULL status into BUSY, I think the
-						 * mid level cannot handle QUEUE FULL :-( (The
-						 * command is retried after BUSY). Also update our
-						 * queue size to the number of currently issued
-						 * commands now.
-						 */
-						/* ++Andreas: the mid level code knows about
-						   QUEUE_FULL now. */
-						struct tag_alloc *ta = &hostdata->TagAlloc[scmd_id(cmd)][cmd->device->lun];
-						dprintk(NDEBUG_TAGS, "scsi%d: target %d lun %llu returned "
-							   "QUEUE_FULL after %d commands\n",
-							   HOSTNO, cmd->device->id, cmd->device->lun,
-							   ta->nr_allocated);
+						u8 lun = cmd->device->lun;
+						struct tag_alloc *ta = &hostdata->TagAlloc[scmd_id(cmd)][lun];
+
+						dsprintk(NDEBUG_TAGS, instance,
+						         "QUEUE_FULL %p target %d lun %d nr_allocated %d\n",
+						         cmd, scmd_id(cmd), lun, ta->nr_allocated);
 						if (ta->queue_size > ta->nr_allocated)
-							ta->nr_allocated = ta->queue_size;
+							ta->queue_size = ta->nr_allocated;
 					}
-#else
-					hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
 #endif
-					/* Enable reselect interrupts */
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-
-					/*
-					 * I'm not sure what the correct thing to do here is :
-					 *
-					 * If the command that just executed is NOT a request
-					 * sense, the obvious thing to do is to set the result
-					 * code to the values of the stored parameters.
-					 *
-					 * If it was a REQUEST SENSE command, we need some way to
-					 * differentiate between the failure code of the original
-					 * and the failure code of the REQUEST sense - the obvious
-					 * case is success, where we fall through and leave the
-					 * result code unchanged.
-					 *
-					 * The non-obvious place is where the REQUEST SENSE failed
-					 */
-
-					if (cmd->cmnd[0] != REQUEST_SENSE)
-						cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-					else if (status_byte(cmd->SCp.Status) != GOOD)
-						cmd->result = (cmd->result & 0x00ffff) | (DID_ERROR << 16);
-
-					if ((cmd->cmnd[0] == REQUEST_SENSE) &&
-						hostdata->ses.cmd_len) {
-						scsi_eh_restore_cmnd(cmd, &hostdata->ses);
-						hostdata->ses.cmd_len = 0 ;
-					}
-
-					if ((cmd->cmnd[0] != REQUEST_SENSE) &&
-					    (status_byte(cmd->SCp.Status) == CHECK_CONDITION)) {
-						scsi_eh_prep_cmnd(cmd, &hostdata->ses, NULL, 0, ~0);
-
-						dprintk(NDEBUG_AUTOSENSE, "scsi%d: performing request sense\n", HOSTNO);
 
-						LIST(cmd,hostdata->issue_queue);
-						SET_NEXT(cmd, hostdata->issue_queue);
-						hostdata->issue_queue = (struct scsi_cmnd *) cmd;
-						dprintk(NDEBUG_QUEUES, "scsi%d: REQUEST SENSE added to head of "
-							  "issue queue\n", H_NO(cmd));
-					} else {
-						cmd->scsi_done(cmd);
+					cmd->result &= ~0xffff;
+					cmd->result |= cmd->SCp.Status;
+					cmd->result |= cmd->SCp.Message << 8;
+
+					if (cmd->cmnd[0] == REQUEST_SENSE)
+						complete_cmd(instance, cmd);
+					else {
+						if (cmd->SCp.Status == SAM_STAT_CHECK_CONDITION ||
+						    cmd->SCp.Status == SAM_STAT_COMMAND_TERMINATED) {
+							dsprintk(NDEBUG_QUEUES, instance, "autosense: adding cmd %p to tail of autosense queue\n",
+							         cmd);
+							list_add_tail(&ncmd->list,
+							              &hostdata->autosense);
+						} else
+							complete_cmd(instance, cmd);
 					}
 
-					local_irq_restore(flags);
-
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					/*
 					 * Restore phase bits to 0 so an interrupted selection,
 					 * arbitration can resume.
 					 */
 					NCR5380_write(TARGET_COMMAND_REG, 0);
 
-					while ((NCR5380_read(STATUS_REG) & SR_BSY) && !hostdata->connected)
-						barrier();
+					/* Enable reselect interrupts */
+					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 
-					local_irq_save(flags);
-					hostdata->retain_dma_intr--;
-					/* ++roman: For Falcon SCSI, release the lock on the
-					 * ST-DMA here if no other commands are waiting on the
-					 * disconnected queue.
-					 */
 					maybe_release_dma_irq(instance);
-					local_irq_restore(flags);
 					return;
 				case MESSAGE_REJECT:
 					/* Accept message by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					/* Enable reselect interrupts */
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					switch (hostdata->last_message) {
 					case HEAD_OF_QUEUE_TAG:
 					case ORDERED_QUEUE_TAG:
@@ -2274,27 +2069,20 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 						cmd->device->tagged_supported = 0;
 						hostdata->busy[cmd->device->id] |= (1 << cmd->device->lun);
 						cmd->tag = TAG_NONE;
-						dprintk(NDEBUG_TAGS, "scsi%d: target %d lun %llu rejected "
-							   "QUEUE_TAG message; tagged queuing "
-							   "disabled\n",
-							   HOSTNO, cmd->device->id, cmd->device->lun);
+						dsprintk(NDEBUG_TAGS, instance, "target %d lun %llu rejected QUEUE_TAG message; tagged queuing disabled\n",
+						         scmd_id(cmd), cmd->device->lun);
 						break;
 					}
 					break;
 				case DISCONNECT:
 					/* Accept message by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					local_irq_save(flags);
-					cmd->device->disconnect = 1;
-					LIST(cmd,hostdata->disconnected_queue);
-					SET_NEXT(cmd, hostdata->disconnected_queue);
 					hostdata->connected = NULL;
-					hostdata->disconnected_queue = cmd;
-					local_irq_restore(flags);
-					dprintk(NDEBUG_QUEUES, "scsi%d: command for target %d lun %llu was "
-						  "moved from connected to the "
-						  "disconnected_queue\n", HOSTNO,
-						  cmd->device->id, cmd->device->lun);
+					list_add(&ncmd->list, &hostdata->disconnected);
+					dsprintk(NDEBUG_INFORMATION | NDEBUG_QUEUES,
+					         instance, "connected command %p for target %d lun %llu moved to disconnected queue\n",
+					         cmd, scmd_id(cmd), cmd->device->lun);
+
 					/*
 					 * Restore phase bits to 0 so an interrupted selection,
 					 * arbitration can resume.
@@ -2303,9 +2091,6 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 
 					/* Enable reselect interrupts */
 					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
-					/* Wait for bus free to avoid nasty timeouts */
-					while ((NCR5380_read(STATUS_REG) & SR_BSY) && !hostdata->connected)
-						barrier();
 #ifdef SUN3_SCSI_VME
 					dregs->csr |= CSR_DMA_ENABLE;
 #endif
@@ -2324,37 +2109,30 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				case RESTORE_POINTERS:
 					/* Accept message by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-					/* Enable reselect interrupts */
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					break;
 				case EXTENDED_MESSAGE:
 					/*
-					 * Extended messages are sent in the following format :
-					 * Byte
-					 * 0		EXTENDED_MESSAGE == 1
-					 * 1		length (includes one byte for code, doesn't
-					 *		include first two bytes)
-					 * 2		code
-					 * 3..length+1	arguments
-					 *
-					 * Start the extended message buffer with the EXTENDED_MESSAGE
+					 * Start the message buffer with the EXTENDED_MESSAGE
 					 * byte, since spi_print_msg() wants the whole thing.
 					 */
 					extended_msg[0] = EXTENDED_MESSAGE;
 					/* Accept first byte by clearing ACK */
 					NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 
-					dprintk(NDEBUG_EXTENDED, "scsi%d: receiving extended message\n", HOSTNO);
+					spin_unlock_irq(&hostdata->lock);
+
+					dsprintk(NDEBUG_EXTENDED, instance, "receiving extended message\n");
 
 					len = 2;
 					data = extended_msg + 1;
 					phase = PHASE_MSGIN;
 					NCR5380_transfer_pio(instance, &phase, &len, &data);
-					dprintk(NDEBUG_EXTENDED, "scsi%d: length=%d, code=0x%02x\n", HOSTNO,
-						   (int)extended_msg[1], (int)extended_msg[2]);
+					dsprintk(NDEBUG_EXTENDED, instance, "length %d, code 0x%02x\n",
+					         (int)extended_msg[1],
+					         (int)extended_msg[2]);
 
-					if (!len && extended_msg[1] <=
-					    (sizeof(extended_msg) - 1)) {
+					if (!len && extended_msg[1] > 0 &&
+					    extended_msg[1] <= sizeof(extended_msg) - 2) {
 						/* Accept third byte by clearing ACK */
 						NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 						len = extended_msg[1] - 1;
@@ -2362,8 +2140,8 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 						phase = PHASE_MSGIN;
 
 						NCR5380_transfer_pio(instance, &phase, &len, &data);
-						dprintk(NDEBUG_EXTENDED, "scsi%d: message received, residual %d\n",
-							   HOSTNO, len);
+						dsprintk(NDEBUG_EXTENDED, instance, "message received, residual %d\n",
+						         len);
 
 						switch (extended_msg[2]) {
 						case EXTENDED_SDTR:
@@ -2373,15 +2151,18 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 							tmp = 0;
 						}
 					} else if (len) {
-						printk(KERN_NOTICE "scsi%d: error receiving "
-						       "extended message\n", HOSTNO);
+						shost_printk(KERN_ERR, instance, "error receiving extended message\n");
 						tmp = 0;
 					} else {
-						printk(KERN_NOTICE "scsi%d: extended message "
-							   "code %02x length %d is too long\n",
-							   HOSTNO, extended_msg[2], extended_msg[1]);
+						shost_printk(KERN_NOTICE, instance, "extended message code %02x length %d is too long\n",
+						             extended_msg[2], extended_msg[1]);
 						tmp = 0;
 					}
+
+					spin_lock_irq(&hostdata->lock);
+					if (!hostdata->connected)
+						return;
+
 					/* Fall through to reject message */
 
 					/*
@@ -2390,8 +2171,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 					 */
 				default:
 					if (!tmp) {
-						printk(KERN_INFO "scsi%d: rejecting message ",
-						       instance->host_no);
+						shost_printk(KERN_ERR, instance, "rejecting message ");
 						spi_print_msg(extended_msg);
 						printk("\n");
 					} else if (tmp != EXTENDED_MESSAGE)
@@ -2414,18 +2194,11 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				hostdata->last_message = msgout;
 				NCR5380_transfer_pio(instance, &phase, &len, &data);
 				if (msgout == ABORT) {
-					local_irq_save(flags);
-#ifdef SUPPORT_TAGS
-					cmd_free_tag(cmd);
-#else
-					hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-#endif
 					hostdata->connected = NULL;
 					cmd->result = DID_ERROR << 16;
-					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
+					complete_cmd(instance, cmd);
 					maybe_release_dma_irq(instance);
-					local_irq_restore(flags);
-					cmd->scsi_done(cmd);
+					NCR5380_write(SELECT_ENABLE_REG, hostdata->id_mask);
 					return;
 				}
 				msgout = NOP;
@@ -2447,22 +2220,25 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				cmd->SCp.Status = tmp;
 				break;
 			default:
-				printk("scsi%d: unknown phase\n", HOSTNO);
+				shost_printk(KERN_ERR, instance, "unknown phase\n");
 				NCR5380_dprint(NDEBUG_ANY, instance);
 			} /* switch(phase) */
-		} /* if (tmp * SR_REQ) */
-	} /* while (1) */
+		} else {
+			spin_unlock_irq(&hostdata->lock);
+			NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
+			spin_lock_irq(&hostdata->lock);
+		}
+	}
 }
 
 /*
  * Function : void NCR5380_reselect (struct Scsi_Host *instance)
  *
  * Purpose : does reselection, initializing the instance->connected
- *	field to point to the scsi_cmnd for which the I_T_L or I_T_L_Q
- *	nexus has been reestablished,
+ * field to point to the scsi_cmnd for which the I_T_L or I_T_L_Q
+ * nexus has been reestablished,
  *
  * Inputs : instance - this instance of the NCR5380.
- *
  */
 
 
@@ -2471,7 +2247,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 
 static void NCR5380_reselect(struct Scsi_Host *instance)
 {
-	SETUP_HOSTDATA(instance);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char target_mask;
 	unsigned char lun;
 #ifdef SUPPORT_TAGS
@@ -2480,7 +2256,8 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	unsigned char msg[3];
 	int __maybe_unused len;
 	unsigned char __maybe_unused *data, __maybe_unused phase;
-	struct scsi_cmnd *tmp = NULL, *prev;
+	struct NCR5380_cmd *ncmd;
+	struct scsi_cmnd *tmp;
 
 	/*
 	 * Disable arbitration, etc. since the host adapter obviously
@@ -2488,11 +2265,10 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	 */
 
 	NCR5380_write(MODE_REG, MR_BASE);
-	hostdata->restart_select = 1;
 
 	target_mask = NCR5380_read(CURRENT_SCSI_DATA_REG) & ~(hostdata->id_mask);
 
-	dprintk(NDEBUG_RESELECTION, "scsi%d: reselect\n", HOSTNO);
+	dsprintk(NDEBUG_RESELECTION, instance, "reselect\n");
 
 	/*
 	 * At this point, we have detected that our SCSI ID is on the bus,
@@ -2504,17 +2280,22 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	 */
 
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY);
-
-	while (NCR5380_read(STATUS_REG) & SR_SEL)
-		;
+	if (NCR5380_poll_politely(instance,
+	                          STATUS_REG, SR_SEL, 0, 2 * HZ) < 0) {
+		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
+		return;
+	}
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 
 	/*
 	 * Wait for target to go into MSGIN.
 	 */
 
-	while (!(NCR5380_read(STATUS_REG) & SR_REQ))
-		;
+	if (NCR5380_poll_politely(instance,
+	                          STATUS_REG, SR_REQ, SR_REQ, 2 * HZ) < 0) {
+		do_abort(instance);
+		return;
+	}
 
 #if defined(CONFIG_SUN3) && defined(REAL_DMA)
 	/* acknowledge toggle to MSGIN */
@@ -2527,15 +2308,21 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	data = msg;
 	phase = PHASE_MSGIN;
 	NCR5380_transfer_pio(instance, &phase, &len, &data);
+
+	if (len) {
+		do_abort(instance);
+		return;
+	}
 #endif
 
 	if (!(msg[0] & 0x80)) {
-		printk(KERN_DEBUG "scsi%d: expecting IDENTIFY message, got ", HOSTNO);
+		shost_printk(KERN_ERR, instance, "expecting IDENTIFY message, got ");
 		spi_print_msg(msg);
+		printk("\n");
 		do_abort(instance);
 		return;
 	}
-	lun = (msg[0] & 0x07);
+	lun = msg[0] & 0x07;
 
 #if defined(SUPPORT_TAGS) && !defined(CONFIG_SUN3)
 	/* If the phase is still MSGIN, the target wants to send some more
@@ -2551,8 +2338,8 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 		if (!NCR5380_transfer_pio(instance, &phase, &len, &data) &&
 		    msg[1] == SIMPLE_QUEUE_TAG)
 			tag = msg[2];
-		dprintk(NDEBUG_TAGS, "scsi%d: target mask %02x, lun %d sent tag %d at "
-			   "reselection\n", HOSTNO, target_mask, lun, tag);
+		dsprintk(NDEBUG_TAGS, instance, "reselect: target mask %02x, lun %d sent tag %d\n",
+		         target_mask, lun, tag);
 	}
 #endif
 
@@ -2561,36 +2348,34 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	 * just reestablished, and remove it from the disconnected queue.
 	 */
 
-	for (tmp = (struct scsi_cmnd *) hostdata->disconnected_queue, prev = NULL;
-	     tmp; prev = tmp, tmp = NEXT(tmp)) {
-		if ((target_mask == (1 << tmp->device->id)) && (lun == tmp->device->lun)
+	tmp = NULL;
+	list_for_each_entry(ncmd, &hostdata->disconnected, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
+
+		if (target_mask == (1 << scmd_id(cmd)) &&
+		    lun == (u8)cmd->device->lun
 #ifdef SUPPORT_TAGS
-		    && (tag == tmp->tag)
+		    && (tag == cmd->tag)
 #endif
 		    ) {
-			if (prev) {
-				REMOVE(prev, NEXT(prev), tmp, NEXT(tmp));
-				SET_NEXT(prev, NEXT(tmp));
-			} else {
-				REMOVE(-1, hostdata->disconnected_queue, tmp, NEXT(tmp));
-				hostdata->disconnected_queue = NEXT(tmp);
-			}
-			SET_NEXT(tmp, NULL);
+			list_del(&ncmd->list);
+			tmp = cmd;
 			break;
 		}
 	}
 
-	if (!tmp) {
-		printk(KERN_WARNING "scsi%d: warning: target bitmask %02x lun %d "
-#ifdef SUPPORT_TAGS
-		       "tag %d "
-#endif
-		       "not in disconnected_queue.\n",
-		       HOSTNO, target_mask, lun
+	if (tmp) {
+		dsprintk(NDEBUG_RESELECTION | NDEBUG_QUEUES, instance,
+		         "reselect: removed %p from disconnected queue\n", tmp);
+	} else {
+
 #ifdef SUPPORT_TAGS
-		       , tag
+		shost_printk(KERN_ERR, instance, "target bitmask 0x%02x lun %d tag %d not in disconnected queue.\n",
+		             target_mask, lun, tag);
+#else
+		shost_printk(KERN_ERR, instance, "target bitmask 0x%02x lun %d not in disconnected queue.\n",
+		             target_mask, lun);
 #endif
-			);
 		/*
 		 * Since we have an established nexus that we can't do anything
 		 * with, we must abort it.
@@ -2614,7 +2399,8 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 		}
 		/* setup this command for dma if not already */
 		if ((count >= DMA_MIN_SIZE) && (sun3_dma_setup_done != tmp)) {
-			sun3scsi_dma_setup(d, count, rq_data_dir(tmp->request));
+			sun3scsi_dma_setup(instance, d, count,
+			                   rq_data_dir(tmp->request));
 			sun3_dma_setup_done = tmp;
 		}
 	}
@@ -2639,235 +2425,196 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 		if (!NCR5380_transfer_pio(instance, &phase, &len, &data) &&
 		    msg[1] == SIMPLE_QUEUE_TAG)
 			tag = msg[2];
-		dprintk(NDEBUG_TAGS, "scsi%d: target mask %02x, lun %d sent tag %d at reselection\n"
-			HOSTNO, target_mask, lun, tag);
+		dsprintk(NDEBUG_TAGS, instance, "reselect: target mask %02x, lun %d sent tag %d\n"
+		         target_mask, lun, tag);
 	}
 #endif
 
 	hostdata->connected = tmp;
-	dprintk(NDEBUG_RESELECTION, "scsi%d: nexus established, target = %d, lun = %llu, tag = %d\n",
-		   HOSTNO, tmp->device->id, tmp->device->lun, tmp->tag);
+	dsprintk(NDEBUG_RESELECTION, instance, "nexus established, target %d, lun %llu, tag %d\n",
+	         scmd_id(tmp), tmp->device->lun, tmp->tag);
 }
 
 
-/*
- * Function : int NCR5380_abort (struct scsi_cmnd *cmd)
- *
- * Purpose : abort a command
- *
- * Inputs : cmd - the scsi_cmnd to abort, code - code to set the
- *	host byte of the result field to, if zero DID_ABORTED is
- *	used.
- *
- * Returns : SUCCESS - success, FAILED on failure.
- *
- * XXX - there is no way to abort the command that is currently
- *	 connected, you have to wait for it to complete.  If this is
- *	 a problem, we could implement longjmp() / setjmp(), setjmp()
- *	 called where the loop started in NCR5380_main().
+/**
+ * list_find_cmd - test for presence of a command in a linked list
+ * @haystack: list of commands
+ * @needle: command to search for
  */
 
-static
-int NCR5380_abort(struct scsi_cmnd *cmd)
+static bool list_find_cmd(struct list_head *haystack,
+                          struct scsi_cmnd *needle)
 {
-	struct Scsi_Host *instance = cmd->device->host;
-	SETUP_HOSTDATA(instance);
-	struct scsi_cmnd *tmp, **prev;
-	unsigned long flags;
+	struct NCR5380_cmd *ncmd;
 
-	scmd_printk(KERN_NOTICE, cmd, "aborting command\n");
+	list_for_each_entry(ncmd, haystack, list)
+		if (NCR5380_to_scmd(ncmd) == needle)
+			return true;
+	return false;
+}
 
-	NCR5380_print_status(instance);
+/**
+ * list_remove_cmd - remove a command from linked list
+ * @haystack: list of commands
+ * @needle: command to remove
+ */
 
-	local_irq_save(flags);
+static bool list_del_cmd(struct list_head *haystack,
+                         struct scsi_cmnd *needle)
+{
+	if (list_find_cmd(haystack, needle)) {
+		struct NCR5380_cmd *ncmd = scsi_cmd_priv(needle);
 
-	dprintk(NDEBUG_ABORT, "scsi%d: abort called basr 0x%02x, sr 0x%02x\n", HOSTNO,
-		    NCR5380_read(BUS_AND_STATUS_REG),
-		    NCR5380_read(STATUS_REG));
+		list_del(&ncmd->list);
+		return true;
+	}
+	return false;
+}
 
-#if 1
-	/*
-	 * Case 1 : If the command is the currently executing command,
-	 * we'll set the aborted flag and return control so that
-	 * information transfer routine can exit cleanly.
-	 */
+/**
+ * NCR5380_abort - scsi host eh_abort_handler() method
+ * @cmd: the command to be aborted
+ *
+ * Try to abort a given command by removing it from queues and/or sending
+ * the target an abort message. This may not succeed in causing a target
+ * to abort the command. Nonetheless, the low-level driver must forget about
+ * the command because the mid-layer reclaims it and it may be re-issued.
+ *
+ * The normal path taken by a command is as follows. For EH we trace this
+ * same path to locate and abort the command.
+ *
+ * unissued -> selecting -> [unissued -> selecting ->]... connected ->
+ * [disconnected -> connected ->]...
+ * [autosense -> connected ->] done
+ *
+ * If cmd is unissued then just remove it.
+ * If cmd is disconnected, try to select the target.
+ * If cmd is connected, try to send an abort message.
+ * If cmd is waiting for autosense, give it a chance to complete but check
+ * that it isn't left connected.
+ * If cmd was not found at all then presumably it has already been completed,
+ * in which case return SUCCESS to try to avoid further EH measures.
+ * If the command has not completed yet, we must not fail to find it.
+ */
 
-	if (hostdata->connected == cmd) {
+static int NCR5380_abort(struct scsi_cmnd *cmd)
+{
+	struct Scsi_Host *instance = cmd->device->host;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	unsigned long flags;
+	int result = SUCCESS;
 
-		dprintk(NDEBUG_ABORT, "scsi%d: aborting connected command\n", HOSTNO);
-		/*
-		 * We should perform BSY checking, and make sure we haven't slipped
-		 * into BUS FREE.
-		 */
+	spin_lock_irqsave(&hostdata->lock, flags);
 
-		/*	NCR5380_write(INITIATOR_COMMAND_REG, ICR_ASSERT_ATN); */
-		/*
-		 * Since we can't change phases until we've completed the current
-		 * handshake, we have to source or sink a byte of data if the current
-		 * phase is not MSGOUT.
-		 */
+#if (NDEBUG & NDEBUG_ANY)
+	scmd_printk(KERN_INFO, cmd, __func__);
+#endif
+	NCR5380_dprint(NDEBUG_ANY, instance);
+	NCR5380_dprint_phase(NDEBUG_ANY, instance);
 
-		/*
-		 * Return control to the executing NCR drive so we can clear the
-		 * aborted flag and get back into our main loop.
-		 */
+	if (list_del_cmd(&hostdata->unissued, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: removed %p from issue queue\n", cmd);
+		cmd->result = DID_ABORT << 16;
+		cmd->scsi_done(cmd); /* No tag or busy flag to worry about */
+	}
 
-		if (do_abort(instance) == 0) {
-			hostdata->aborted = 1;
-			hostdata->connected = NULL;
-			cmd->result = DID_ABORT << 16;
-#ifdef SUPPORT_TAGS
-			cmd_free_tag(cmd);
-#else
-			hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-#endif
-			maybe_release_dma_irq(instance);
-			local_irq_restore(flags);
-			cmd->scsi_done(cmd);
-			return SUCCESS;
-		} else {
-			local_irq_restore(flags);
-			printk("scsi%d: abort of connected command failed!\n", HOSTNO);
-			return FAILED;
-		}
+	if (hostdata->selecting == cmd) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: cmd %p == selecting\n", cmd);
+		hostdata->selecting = NULL;
+		cmd->result = DID_ABORT << 16;
+		complete_cmd(instance, cmd);
+		goto out;
 	}
-#endif
 
-	/*
-	 * Case 2 : If the command hasn't been issued yet, we simply remove it
-	 *	    from the issue queue.
-	 */
-	for (prev = (struct scsi_cmnd **)&(hostdata->issue_queue),
-	     tmp = (struct scsi_cmnd *)hostdata->issue_queue;
-	     tmp; prev = NEXTADDR(tmp), tmp = NEXT(tmp)) {
-		if (cmd == tmp) {
-			REMOVE(5, *prev, tmp, NEXT(tmp));
-			(*prev) = NEXT(tmp);
-			SET_NEXT(tmp, NULL);
-			tmp->result = DID_ABORT << 16;
-			maybe_release_dma_irq(instance);
-			local_irq_restore(flags);
-			dprintk(NDEBUG_ABORT, "scsi%d: abort removed command from issue queue.\n",
-				    HOSTNO);
-			/* Tagged queuing note: no tag to free here, hasn't been assigned
-			 * yet... */
-			tmp->scsi_done(tmp);
-			return SUCCESS;
+	if (list_del_cmd(&hostdata->disconnected, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: removed %p from disconnected list\n", cmd);
+		cmd->result = DID_ERROR << 16;
+		if (!hostdata->connected)
+			NCR5380_select(instance, cmd);
+		if (hostdata->connected != cmd) {
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
 		}
 	}
 
-	/*
-	 * Case 3 : If any commands are connected, we're going to fail the abort
-	 *	    and let the high level SCSI driver retry at a later time or
-	 *	    issue a reset.
-	 *
-	 *	    Timeouts, and therefore aborted commands, will be highly unlikely
-	 *          and handling them cleanly in this situation would make the common
-	 *	    case of noresets less efficient, and would pollute our code.  So,
-	 *	    we fail.
-	 */
+	if (hostdata->connected == cmd) {
+		dsprintk(NDEBUG_ABORT, instance, "abort: cmd %p is connected\n", cmd);
+		hostdata->connected = NULL;
+		if (do_abort(instance)) {
+			set_host_byte(cmd, DID_ERROR);
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
+		}
+		set_host_byte(cmd, DID_ABORT);
+#ifdef REAL_DMA
+		hostdata->dma_len = 0;
+#endif
+		if (cmd->cmnd[0] == REQUEST_SENSE)
+			complete_cmd(instance, cmd);
+		else {
+			struct NCR5380_cmd *ncmd = scsi_cmd_priv(cmd);
 
-	if (hostdata->connected) {
-		local_irq_restore(flags);
-		dprintk(NDEBUG_ABORT, "scsi%d: abort failed, command connected.\n", HOSTNO);
-		return FAILED;
+			/* Perform autosense for this command */
+			list_add(&ncmd->list, &hostdata->autosense);
+		}
 	}
 
-	/*
-	 * Case 4: If the command is currently disconnected from the bus, and
-	 *	there are no connected commands, we reconnect the I_T_L or
-	 *	I_T_L_Q nexus associated with it, go into message out, and send
-	 *      an abort message.
-	 *
-	 * This case is especially ugly. In order to reestablish the nexus, we
-	 * need to call NCR5380_select().  The easiest way to implement this
-	 * function was to abort if the bus was busy, and let the interrupt
-	 * handler triggered on the SEL for reselect take care of lost arbitrations
-	 * where necessary, meaning interrupts need to be enabled.
-	 *
-	 * When interrupts are enabled, the queues may change - so we
-	 * can't remove it from the disconnected queue before selecting it
-	 * because that could cause a failure in hashing the nexus if that
-	 * device reselected.
-	 *
-	 * Since the queues may change, we can't use the pointers from when we
-	 * first locate it.
-	 *
-	 * So, we must first locate the command, and if NCR5380_select()
-	 * succeeds, then issue the abort, relocate the command and remove
-	 * it from the disconnected queue.
-	 */
-
-	for (tmp = (struct scsi_cmnd *) hostdata->disconnected_queue; tmp;
-	     tmp = NEXT(tmp)) {
-		if (cmd == tmp) {
-			local_irq_restore(flags);
-			dprintk(NDEBUG_ABORT, "scsi%d: aborting disconnected command.\n", HOSTNO);
-
-			if (NCR5380_select(instance, cmd))
-				return FAILED;
-
-			dprintk(NDEBUG_ABORT, "scsi%d: nexus reestablished.\n", HOSTNO);
-
-			do_abort(instance);
-
-			local_irq_save(flags);
-			for (prev = (struct scsi_cmnd **)&(hostdata->disconnected_queue),
-			     tmp = (struct scsi_cmnd *)hostdata->disconnected_queue;
-			     tmp; prev = NEXTADDR(tmp), tmp = NEXT(tmp)) {
-				if (cmd == tmp) {
-					REMOVE(5, *prev, tmp, NEXT(tmp));
-					*prev = NEXT(tmp);
-					SET_NEXT(tmp, NULL);
-					tmp->result = DID_ABORT << 16;
-					/* We must unlock the tag/LUN immediately here, since the
-					 * target goes to BUS FREE and doesn't send us another
-					 * message (COMMAND_COMPLETE or the like)
-					 */
-#ifdef SUPPORT_TAGS
-					cmd_free_tag(tmp);
-#else
-					hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-#endif
-					maybe_release_dma_irq(instance);
-					local_irq_restore(flags);
-					tmp->scsi_done(tmp);
-					return SUCCESS;
-				}
-			}
+	if (list_find_cmd(&hostdata->autosense, cmd)) {
+		dsprintk(NDEBUG_ABORT, instance,
+		         "abort: found %p on sense queue\n", cmd);
+		spin_unlock_irqrestore(&hostdata->lock, flags);
+		queue_work(hostdata->work_q, &hostdata->main_task);
+		msleep(1000);
+		spin_lock_irqsave(&hostdata->lock, flags);
+		if (list_del_cmd(&hostdata->autosense, cmd)) {
+			dsprintk(NDEBUG_ABORT, instance,
+			         "abort: removed %p from sense queue\n", cmd);
+			set_host_byte(cmd, DID_ABORT);
+			complete_cmd(instance, cmd);
+			goto out;
 		}
 	}
 
-	/* Maybe it is sufficient just to release the ST-DMA lock... (if
-	 * possible at all) At least, we should check if the lock could be
-	 * released after the abort, in case it is kept due to some bug.
-	 */
-	maybe_release_dma_irq(instance);
-	local_irq_restore(flags);
+	if (hostdata->connected == cmd) {
+		dsprintk(NDEBUG_ABORT, instance, "abort: cmd %p is connected\n", cmd);
+		hostdata->connected = NULL;
+		if (do_abort(instance)) {
+			set_host_byte(cmd, DID_ERROR);
+			complete_cmd(instance, cmd);
+			result = FAILED;
+			goto out;
+		}
+		set_host_byte(cmd, DID_ABORT);
+#ifdef REAL_DMA
+		hostdata->dma_len = 0;
+#endif
+		complete_cmd(instance, cmd);
+	}
 
-	/*
-	 * Case 5 : If we reached this point, the command was not found in any of
-	 *	    the queues.
-	 *
-	 * We probably reached this point because of an unlikely race condition
-	 * between the command completing successfully and the abortion code,
-	 * so we won't panic, but we will notify the user in case something really
-	 * broke.
-	 */
+out:
+	if (result == FAILED)
+		dsprintk(NDEBUG_ABORT, instance, "abort: failed to abort %p\n", cmd);
+	else
+		dsprintk(NDEBUG_ABORT, instance, "abort: successfully aborted %p\n", cmd);
 
-	printk(KERN_INFO "scsi%d: warning : SCSI command probably completed successfully before abortion\n", HOSTNO);
+	queue_work(hostdata->work_q, &hostdata->main_task);
+	maybe_release_dma_irq(instance);
+	spin_unlock_irqrestore(&hostdata->lock, flags);
 
-	return FAILED;
+	return result;
 }
 
 
-/*
- * Function : int NCR5380_reset (struct scsi_cmnd *cmd)
- *
- * Purpose : reset the SCSI bus.
- *
- * Returns : SUCCESS or FAILURE
+/**
+ * NCR5380_bus_reset - reset the SCSI bus
+ * @cmd: SCSI command undergoing EH
  *
+ * Returns SUCCESS
  */
 
 static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
@@ -2876,23 +2623,22 @@ static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int i;
 	unsigned long flags;
+	struct NCR5380_cmd *ncmd;
 
-	NCR5380_print_status(instance);
+	spin_lock_irqsave(&hostdata->lock, flags);
+
+#if (NDEBUG & NDEBUG_ANY)
+	scmd_printk(KERN_INFO, cmd, __func__);
+#endif
+	NCR5380_dprint(NDEBUG_ANY, instance);
+	NCR5380_dprint_phase(NDEBUG_ANY, instance);
+
+	do_reset(instance);
 
-	/* get in phase */
-	NCR5380_write(TARGET_COMMAND_REG,
-		      PHASE_SR_TO_TCR(NCR5380_read(STATUS_REG)));
-	/* assert RST */
-	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST);
-	udelay(40);
 	/* reset NCR registers */
-	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 	NCR5380_write(MODE_REG, MR_BASE);
 	NCR5380_write(TARGET_COMMAND_REG, 0);
 	NCR5380_write(SELECT_ENABLE_REG, 0);
-	/* ++roman: reset interrupt condition! otherwise no interrupts don't get
-	 * through anymore ... */
-	(void)NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 
 	/* After the reset, there are no more connected or disconnected commands
 	 * and no busy units; so clear the low-level status here to avoid
@@ -2900,17 +2646,34 @@ static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
 	 * commands!
 	 */
 
-	if (hostdata->issue_queue)
-		dprintk(NDEBUG_ABORT, "scsi%d: reset aborted issued command(s)\n", H_NO(cmd));
-	if (hostdata->connected)
-		dprintk(NDEBUG_ABORT, "scsi%d: reset aborted a connected command\n", H_NO(cmd));
-	if (hostdata->disconnected_queue)
-		dprintk(NDEBUG_ABORT, "scsi%d: reset aborted disconnected command(s)\n", H_NO(cmd));
+	hostdata->selecting = NULL;
+
+	list_for_each_entry(ncmd, &hostdata->disconnected, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
+
+		set_host_byte(cmd, DID_RESET);
+		cmd->scsi_done(cmd);
+	}
+
+	list_for_each_entry(ncmd, &hostdata->autosense, list) {
+		struct scsi_cmnd *cmd = NCR5380_to_scmd(ncmd);
+
+		set_host_byte(cmd, DID_RESET);
+		cmd->scsi_done(cmd);
+	}
+
+	if (hostdata->connected) {
+		set_host_byte(hostdata->connected, DID_RESET);
+		complete_cmd(instance, hostdata->connected);
+		hostdata->connected = NULL;
+	}
+
+	if (hostdata->sensing) {
+		set_host_byte(hostdata->connected, DID_RESET);
+		complete_cmd(instance, hostdata->sensing);
+		hostdata->sensing = NULL;
+	}
 
-	local_irq_save(flags);
-	hostdata->issue_queue = NULL;
-	hostdata->connected = NULL;
-	hostdata->disconnected_queue = NULL;
 #ifdef SUPPORT_TAGS
 	free_all_tags(hostdata);
 #endif
@@ -2920,8 +2683,9 @@ static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
 	hostdata->dma_len = 0;
 #endif
 
+	queue_work(hostdata->work_q, &hostdata->main_task);
 	maybe_release_dma_irq(instance);
-	local_irq_restore(flags);
+	spin_unlock_irqrestore(&hostdata->lock, flags);
 
 	return SUCCESS;
 }
diff --git a/drivers/scsi/atari_scsi.c b/drivers/scsi/atari_scsi.c
index 5ede3daa93dc..78d1b2963f2c 100644
--- a/drivers/scsi/atari_scsi.c
+++ b/drivers/scsi/atari_scsi.c
@@ -66,7 +66,6 @@
 
 #include <linux/module.h>
 #include <linux/types.h>
-#include <linux/delay.h>
 #include <linux/blkdev.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
@@ -98,7 +97,6 @@
 
 #define NCR5380_queue_command           atari_scsi_queue_command
 #define NCR5380_abort                   atari_scsi_abort
-#define NCR5380_show_info               atari_scsi_show_info
 #define NCR5380_info                    atari_scsi_info
 
 #define NCR5380_dma_read_setup(instance, data, count) \
@@ -161,23 +159,10 @@ static inline unsigned long SCSI_DMA_GETADR(void)
 	return adr;
 }
 
-#define HOSTDATA_DMALEN		(((struct NCR5380_hostdata *) \
-				(atari_scsi_host->hostdata))->dma_len)
-
-/* Time (in jiffies) to wait after a reset; the SCSI standard calls for 250ms,
- * we usually do 0.5s to be on the safe side. But Toshiba CD-ROMs once more
- * need ten times the standard value... */
-#ifndef CONFIG_ATARI_SCSI_TOSHIBA_DELAY
-#define	AFTER_RESET_DELAY	(HZ/2)
-#else
-#define	AFTER_RESET_DELAY	(5*HZ/2)
-#endif
-
 #ifdef REAL_DMA
 static void atari_scsi_fetch_restbytes(void);
 #endif
 
-static struct Scsi_Host *atari_scsi_host;
 static unsigned char (*atari_scsi_reg_read)(unsigned char reg);
 static void (*atari_scsi_reg_write)(unsigned char reg, unsigned char value);
 
@@ -208,12 +193,12 @@ static int setup_cmd_per_lun = -1;
 module_param(setup_cmd_per_lun, int, 0);
 static int setup_sg_tablesize = -1;
 module_param(setup_sg_tablesize, int, 0);
-#ifdef SUPPORT_TAGS
 static int setup_use_tagged_queuing = -1;
 module_param(setup_use_tagged_queuing, int, 0);
-#endif
 static int setup_hostid = -1;
 module_param(setup_hostid, int, 0);
+static int setup_toshiba_delay = -1;
+module_param(setup_toshiba_delay, int, 0);
 
 
 #if defined(REAL_DMA)
@@ -273,15 +258,17 @@ static void scsi_dma_buserr(int irq, void *dummy)
 #endif
 
 
-static irqreturn_t scsi_tt_intr(int irq, void *dummy)
+static irqreturn_t scsi_tt_intr(int irq, void *dev)
 {
 #ifdef REAL_DMA
+	struct Scsi_Host *instance = dev;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int dma_stat;
 
 	dma_stat = tt_scsi_dma.dma_ctrl;
 
-	dprintk(NDEBUG_INTR, "scsi%d: NCR5380 interrupt, DMA status = %02x\n",
-		   atari_scsi_host->host_no, dma_stat & 0xff);
+	dsprintk(NDEBUG_INTR, instance, "NCR5380 interrupt, DMA status = %02x\n",
+	         dma_stat & 0xff);
 
 	/* Look if it was the DMA that has interrupted: First possibility
 	 * is that a bus error occurred...
@@ -304,7 +291,8 @@ static irqreturn_t scsi_tt_intr(int irq, void *dummy)
 	 * data reg!
 	 */
 	if ((dma_stat & 0x02) && !(dma_stat & 0x40)) {
-		atari_dma_residual = HOSTDATA_DMALEN - (SCSI_DMA_READ_P(dma_addr) - atari_dma_startaddr);
+		atari_dma_residual = hostdata->dma_len -
+			(SCSI_DMA_READ_P(dma_addr) - atari_dma_startaddr);
 
 		dprintk(NDEBUG_DMA, "SCSI DMA: There are %ld residual bytes.\n",
 			   atari_dma_residual);
@@ -356,15 +344,17 @@ static irqreturn_t scsi_tt_intr(int irq, void *dummy)
 
 #endif /* REAL_DMA */
 
-	NCR5380_intr(irq, dummy);
+	NCR5380_intr(irq, dev);
 
 	return IRQ_HANDLED;
 }
 
 
-static irqreturn_t scsi_falcon_intr(int irq, void *dummy)
+static irqreturn_t scsi_falcon_intr(int irq, void *dev)
 {
 #ifdef REAL_DMA
+	struct Scsi_Host *instance = dev;
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int dma_stat;
 
 	/* Turn off DMA and select sector counter register before
@@ -399,7 +389,7 @@ static irqreturn_t scsi_falcon_intr(int irq, void *dummy)
 			printk(KERN_ERR "SCSI DMA error: %ld bytes lost in "
 			       "ST-DMA fifo\n", transferred & 15);
 
-		atari_dma_residual = HOSTDATA_DMALEN - transferred;
+		atari_dma_residual = hostdata->dma_len - transferred;
 		dprintk(NDEBUG_DMA, "SCSI DMA: There are %ld residual bytes.\n",
 			   atari_dma_residual);
 	} else
@@ -411,13 +401,14 @@ static irqreturn_t scsi_falcon_intr(int irq, void *dummy)
 		 * data to the original destination address.
 		 */
 		memcpy(atari_dma_orig_addr, phys_to_virt(atari_dma_startaddr),
-		       HOSTDATA_DMALEN - atari_dma_residual);
+		       hostdata->dma_len - atari_dma_residual);
 		atari_dma_orig_addr = NULL;
 	}
 
 #endif /* REAL_DMA */
 
-	NCR5380_intr(irq, dummy);
+	NCR5380_intr(irq, dev);
+
 	return IRQ_HANDLED;
 }
 
@@ -488,7 +479,7 @@ static int __init atari_scsi_setup(char *str)
 	 * Defaults depend on TT or Falcon, determined at run time.
 	 * Negative values mean don't change.
 	 */
-	int ints[6];
+	int ints[8];
 
 	get_options(str, ARRAY_SIZE(ints), ints);
 
@@ -504,10 +495,11 @@ static int __init atari_scsi_setup(char *str)
 		setup_sg_tablesize = ints[3];
 	if (ints[0] >= 4)
 		setup_hostid = ints[4];
-#ifdef SUPPORT_TAGS
 	if (ints[0] >= 5)
 		setup_use_tagged_queuing = ints[5];
-#endif
+	/* ints[6] (use_pdma) is ignored */
+	if (ints[0] >= 7)
+		setup_toshiba_delay = ints[7];
 
 	return 1;
 }
@@ -516,38 +508,6 @@ __setup("atascsi=", atari_scsi_setup);
 #endif /* !MODULE */
 
 
-#ifdef CONFIG_ATARI_SCSI_RESET_BOOT
-static void __init atari_scsi_reset_boot(void)
-{
-	unsigned long end;
-
-	/*
-	 * Do a SCSI reset to clean up the bus during initialization. No messing
-	 * with the queues, interrupts, or locks necessary here.
-	 */
-
-	printk("Atari SCSI: resetting the SCSI bus...");
-
-	/* get in phase */
-	NCR5380_write(TARGET_COMMAND_REG,
-		      PHASE_SR_TO_TCR(NCR5380_read(STATUS_REG)));
-
-	/* assert RST */
-	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST);
-	/* The min. reset hold time is 25us, so 40us should be enough */
-	udelay(50);
-	/* reset RST and interrupt */
-	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
-	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-
-	end = jiffies + AFTER_RESET_DELAY;
-	while (time_before(jiffies, end))
-		barrier();
-
-	printk(" done\n");
-}
-#endif
-
 #if defined(REAL_DMA)
 
 static unsigned long atari_scsi_dma_setup(struct Scsi_Host *instance,
@@ -815,14 +775,14 @@ static int atari_scsi_bus_reset(struct scsi_cmnd *cmd)
 static struct scsi_host_template atari_scsi_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= DRV_MODULE_NAME,
-	.show_info		= atari_scsi_show_info,
 	.name			= "Atari native SCSI",
 	.info			= atari_scsi_info,
 	.queuecommand		= atari_scsi_queue_command,
 	.eh_abort_handler	= atari_scsi_abort,
 	.eh_bus_reset_handler	= atari_scsi_bus_reset,
 	.this_id		= 7,
-	.use_clustering		= DISABLE_CLUSTERING
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
 };
 
 static int __init atari_scsi_probe(struct platform_device *pdev)
@@ -880,7 +840,7 @@ static int __init atari_scsi_probe(struct platform_device *pdev)
 	} else {
 		/* Test if a host id is set in the NVRam */
 		if (ATARIHW_PRESENT(TT_CLK) && nvram_check_checksum()) {
-			unsigned char b = nvram_read_byte(14);
+			unsigned char b = nvram_read_byte(16);
 
 			/* Arbitration enabled? (for TOS)
 			 * If yes, use configured host ID
@@ -915,21 +875,18 @@ static int __init atari_scsi_probe(struct platform_device *pdev)
 		error = -ENOMEM;
 		goto fail_alloc;
 	}
-	atari_scsi_host = instance;
-
-#ifdef CONFIG_ATARI_SCSI_RESET_BOOT
-	atari_scsi_reset_boot();
-#endif
 
 	instance->irq = irq->start;
 
 	host_flags |= IS_A_TT() ? 0 : FLAG_LATE_DMA_SETUP;
-
 #ifdef SUPPORT_TAGS
 	host_flags |= setup_use_tagged_queuing > 0 ? FLAG_TAGGED_QUEUING : 0;
 #endif
+	host_flags |= setup_toshiba_delay > 0 ? FLAG_TOSHIBA_DELAY : 0;
 
-	NCR5380_init(instance, host_flags);
+	error = NCR5380_init(instance, host_flags);
+	if (error)
+		goto fail_init;
 
 	if (IS_A_TT()) {
 		error = request_irq(instance->irq, scsi_tt_intr, 0,
@@ -975,6 +932,8 @@ static int __init atari_scsi_probe(struct platform_device *pdev)
 #endif
 	}
 
+	NCR5380_maybe_reset_bus(instance);
+
 	error = scsi_add_host(instance, NULL);
 	if (error)
 		goto fail_host;
@@ -989,6 +948,7 @@ fail_host:
 		free_irq(instance->irq, instance);
 fail_irq:
 	NCR5380_exit(instance);
+fail_init:
 	scsi_host_put(instance);
 fail_alloc:
 	if (atari_dma_buffer)
diff --git a/drivers/scsi/be2iscsi/Kconfig b/drivers/scsi/be2iscsi/Kconfig
index 4e7cad272469..bad5f32e1f67 100644
--- a/drivers/scsi/be2iscsi/Kconfig
+++ b/drivers/scsi/be2iscsi/Kconfig
@@ -3,6 +3,7 @@ config BE2ISCSI
 	depends on PCI && SCSI && NET
 	select SCSI_ISCSI_ATTRS
 	select ISCSI_BOOT_SYSFS
+	select IRQ_POLL
 
 	help
 	This driver implements the iSCSI functionality for Emulex
diff --git a/drivers/scsi/be2iscsi/be.h b/drivers/scsi/be2iscsi/be.h
index 77f992e74726..a41c6432f444 100644
--- a/drivers/scsi/be2iscsi/be.h
+++ b/drivers/scsi/be2iscsi/be.h
@@ -20,7 +20,7 @@
 
 #include <linux/pci.h>
 #include <linux/if_vlan.h>
-#include <linux/blk-iopoll.h>
+#include <linux/irq_poll.h>
 #define FW_VER_LEN	32
 #define MCC_Q_LEN	128
 #define MCC_CQ_LEN	256
@@ -101,7 +101,7 @@ struct be_eq_obj {
 	struct beiscsi_hba *phba;
 	struct be_queue_info *cq;
 	struct work_struct work_cqs; /* Work Item */
-	struct blk_iopoll	iopoll;
+	struct irq_poll	iopoll;
 };
 
 struct be_mcc_obj {
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index b7087ba69d8d..022e87b62e40 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -1292,9 +1292,9 @@ static void beiscsi_flush_cq(struct beiscsi_hba *phba)
 
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		blk_iopoll_disable(&pbe_eq->iopoll);
+		irq_poll_disable(&pbe_eq->iopoll);
 		beiscsi_process_cq(pbe_eq);
-		blk_iopoll_enable(&pbe_eq->iopoll);
+		irq_poll_enable(&pbe_eq->iopoll);
 	}
 }
 
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index fe0c5143f8e6..cb9072a841be 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -910,8 +910,7 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
 	num_eq_processed = 0;
 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 				& EQE_VALID_MASK) {
-		if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
-			blk_iopoll_sched(&pbe_eq->iopoll);
+		irq_poll_sched(&pbe_eq->iopoll);
 
 		AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 		queue_tail_inc(eq);
@@ -972,8 +971,7 @@ static irqreturn_t be_isr(int irq, void *dev_id)
 			spin_unlock_irqrestore(&phba->isr_lock, flags);
 			num_mcceq_processed++;
 		} else {
-			if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
-				blk_iopoll_sched(&pbe_eq->iopoll);
+			irq_poll_sched(&pbe_eq->iopoll);
 			num_ioeq_processed++;
 		}
 		AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
@@ -2295,7 +2293,7 @@ void beiscsi_process_all_cqs(struct work_struct *work)
 	hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
 }
 
-static int be_iopoll(struct blk_iopoll *iop, int budget)
+static int be_iopoll(struct irq_poll *iop, int budget)
 {
 	unsigned int ret;
 	struct beiscsi_hba *phba;
@@ -2306,7 +2304,7 @@ static int be_iopoll(struct blk_iopoll *iop, int budget)
 	pbe_eq->cq_count += ret;
 	if (ret < budget) {
 		phba = pbe_eq->phba;
-		blk_iopoll_complete(iop);
+		irq_poll_complete(iop);
 		beiscsi_log(phba, KERN_INFO,
 			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
 			    "BM_%d : rearm pbe_eq->q.id =%d\n",
@@ -5293,7 +5291,7 @@ static void beiscsi_quiesce(struct beiscsi_hba *phba,
 
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		blk_iopoll_disable(&pbe_eq->iopoll);
+		irq_poll_disable(&pbe_eq->iopoll);
 	}
 
 	if (unload_state == BEISCSI_CLEAN_UNLOAD) {
@@ -5579,9 +5577,8 @@ static void beiscsi_eeh_resume(struct pci_dev *pdev)
 
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
+		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
 				be_iopoll);
-		blk_iopoll_enable(&pbe_eq->iopoll);
 	}
 
 	i = (phba->msix_enabled) ? i : 0;
@@ -5752,9 +5749,8 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
+		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
 				be_iopoll);
-		blk_iopoll_enable(&pbe_eq->iopoll);
 	}
 
 	i = (phba->msix_enabled) ? i : 0;
@@ -5795,7 +5791,7 @@ free_blkenbld:
 	destroy_workqueue(phba->wq);
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		blk_iopoll_disable(&pbe_eq->iopoll);
+		irq_poll_disable(&pbe_eq->iopoll);
 	}
 free_twq:
 	beiscsi_clean_port(phba);
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
index 0e2bee937fe8..e22a268fd311 100644
--- a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -57,7 +57,7 @@ MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
 
 static int cxgb3i_rx_credit_thres = 10 * 1024;
 module_param(cxgb3i_rx_credit_thres, int, 0644);
-MODULE_PARM_DESC(rx_credit_thres,
+MODULE_PARM_DESC(cxgb3i_rx_credit_thres,
 		 "RX credits return threshold in bytes (default=10KB)");
 
 static unsigned int cxgb3i_max_connect = 8 * 1024;
diff --git a/drivers/scsi/dmx3191d.c b/drivers/scsi/dmx3191d.c
index 3e088125a8be..6c14e68b9e1a 100644
--- a/drivers/scsi/dmx3191d.c
+++ b/drivers/scsi/dmx3191d.c
@@ -36,17 +36,10 @@
 
 #define DONT_USE_INTR
 
-#define NCR5380_read(reg)		inb(port + reg)
-#define NCR5380_write(reg, value)	outb(value, port + reg)
+#define NCR5380_read(reg)		inb(instance->io_port + reg)
+#define NCR5380_write(reg, value)	outb(value, instance->io_port + reg)
 
 #define NCR5380_implementation_fields	/* none */
-#define NCR5380_local_declare()		unsigned int port
-#define NCR5380_setup(instance)		port = instance->io_port
-
-/*
- * Includes needed for NCR5380.[ch] (XXX: Move them to NCR5380.h)
- */
-#include <linux/delay.h>
 
 #include "NCR5380.h"
 #include "NCR5380.c"
@@ -56,6 +49,7 @@
 
 
 static struct scsi_host_template dmx3191d_driver_template = {
+	.module			= THIS_MODULE,
 	.proc_name		= DMX3191D_DRIVER_NAME,
 	.name			= "Domex DMX3191D",
 	.info			= NCR5380_info,
@@ -67,6 +61,8 @@ static struct scsi_host_template dmx3191d_driver_template = {
 	.sg_tablesize		= SG_ALL,
 	.cmd_per_lun		= 2,
 	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 
 static int dmx3191d_probe_one(struct pci_dev *pdev,
@@ -97,17 +93,25 @@ static int dmx3191d_probe_one(struct pci_dev *pdev,
 	 */
 	shost->irq = NO_IRQ;
 
-	NCR5380_init(shost, FLAG_NO_PSEUDO_DMA | FLAG_DTC3181E);
+	error = NCR5380_init(shost, FLAG_NO_PSEUDO_DMA);
+	if (error)
+		goto out_host_put;
+
+	NCR5380_maybe_reset_bus(shost);
 
 	pci_set_drvdata(pdev, shost);
 
 	error = scsi_add_host(shost, &pdev->dev);
 	if (error)
-		goto out_release_region;
+		goto out_exit;
 
 	scsi_scan_host(shost);
 	return 0;
 
+out_exit:
+	NCR5380_exit(shost);
+out_host_put:
+	scsi_host_put(shost);
  out_release_region:
 	release_region(io, DMX3191D_REGION_LEN);
  out_disable_device:
@@ -119,15 +123,14 @@ static int dmx3191d_probe_one(struct pci_dev *pdev,
 static void dmx3191d_remove_one(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	unsigned long io = shost->io_port;
 
 	scsi_remove_host(shost);
 
 	NCR5380_exit(shost);
-
-	release_region(shost->io_port, DMX3191D_REGION_LEN);
-	pci_disable_device(pdev);
-
 	scsi_host_put(shost);
+	release_region(io, DMX3191D_REGION_LEN);
+	pci_disable_device(pdev);
 }
 
 static struct pci_device_id dmx3191d_pci_tbl[] = {
diff --git a/drivers/scsi/dtc.c b/drivers/scsi/dtc.c
index 4c74c7ba2dff..6c736b071cf4 100644
--- a/drivers/scsi/dtc.c
+++ b/drivers/scsi/dtc.c
@@ -1,9 +1,5 @@
-
 #define PSEUDO_DMA
 #define DONT_USE_INTR
-#define UNSAFE			/* Leave interrupts enabled during pseudo-dma I/O */
-#define DMA_WORKS_RIGHT
-
 
 /*
  * DTC 3180/3280 driver, by
@@ -50,15 +46,13 @@
 
 
 #include <linux/module.h>
-#include <linux/signal.h>
 #include <linux/blkdev.h>
-#include <linux/delay.h>
-#include <linux/stat.h>
 #include <linux/string.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <scsi/scsi_host.h>
+
 #include "dtc.h"
 #define AUTOPROBE_IRQ
 #include "NCR5380.h"
@@ -150,7 +144,7 @@ static const struct signature {
 
 static int __init dtc_setup(char *str)
 {
-	static int commandline_current = 0;
+	static int commandline_current;
 	int i;
 	int ints[10];
 
@@ -188,7 +182,7 @@ __setup("dtc=", dtc_setup);
 
 static int __init dtc_detect(struct scsi_host_template * tpnt)
 {
-	static int current_override = 0, current_base = 0;
+	static int current_override, current_base;
 	struct Scsi_Host *instance;
 	unsigned int addr;
 	void __iomem *base;
@@ -205,9 +199,8 @@ static int __init dtc_detect(struct scsi_host_template * tpnt)
 				addr = 0;
 		} else
 			for (; !addr && (current_base < NO_BASES); ++current_base) {
-#if (DTCDEBUG & DTCDEBUG_INIT)
-				printk(KERN_DEBUG "scsi-dtc : probing address %08x\n", bases[current_base].address);
-#endif
+				dprintk(NDEBUG_INIT, "dtc: probing address 0x%08x\n",
+				        (unsigned int)bases[current_base].address);
 				if (bases[current_base].noauto)
 					continue;
 				base = ioremap(bases[current_base].address, 0x2000);
@@ -216,18 +209,14 @@ static int __init dtc_detect(struct scsi_host_template * tpnt)
 				for (sig = 0; sig < NO_SIGNATURES; ++sig) {
 					if (check_signature(base + signatures[sig].offset, signatures[sig].string, strlen(signatures[sig].string))) {
 						addr = bases[current_base].address;
-#if (DTCDEBUG & DTCDEBUG_INIT)
-						printk(KERN_DEBUG "scsi-dtc : detected board.\n");
-#endif
+						dprintk(NDEBUG_INIT, "dtc: detected board\n");
 						goto found;
 					}
 				}
 				iounmap(base);
 			}
 
-#if defined(DTCDEBUG) && (DTCDEBUG & DTCDEBUG_INIT)
-		printk(KERN_DEBUG "scsi-dtc : base = %08x\n", addr);
-#endif
+		dprintk(NDEBUG_INIT, "dtc: addr = 0x%08x\n", addr);
 
 		if (!addr)
 			break;
@@ -235,12 +224,15 @@ static int __init dtc_detect(struct scsi_host_template * tpnt)
 found:
 		instance = scsi_register(tpnt, sizeof(struct NCR5380_hostdata));
 		if (instance == NULL)
-			break;
+			goto out_unmap;
 
 		instance->base = addr;
 		((struct NCR5380_hostdata *)(instance)->hostdata)->base = base;
 
-		NCR5380_init(instance, 0);
+		if (NCR5380_init(instance, FLAG_NO_DMA_FIXUP))
+			goto out_unregister;
+
+		NCR5380_maybe_reset_bus(instance);
 
 		NCR5380_write(DTC_CONTROL_REG, CSR_5380_INTR);	/* Enable int's */
 		if (overrides[current_override].irq != IRQ_AUTO)
@@ -271,14 +263,19 @@ found:
 			printk(KERN_WARNING "scsi%d : interrupts not used. Might as well not jumper it.\n", instance->host_no);
 		instance->irq = NO_IRQ;
 #endif
-#if defined(DTCDEBUG) && (DTCDEBUG & DTCDEBUG_INIT)
-		printk("scsi%d : irq = %d\n", instance->host_no, instance->irq);
-#endif
+		dprintk(NDEBUG_INIT, "scsi%d : irq = %d\n",
+		        instance->host_no, instance->irq);
 
 		++current_override;
 		++count;
 	}
 	return count;
+
+out_unregister:
+	scsi_unregister(instance);
+out_unmap:
+	iounmap(base);
+	return count;
 }
 
 /*
@@ -331,12 +328,8 @@ static inline int NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst,
 	unsigned char *d = dst;
 	int i;			/* For counting time spent in the poll-loop */
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
 
 	i = 0;
-	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-	NCR5380_write(MODE_REG, MR_ENABLE_EOP_INTR | MR_DMA_MODE);
 	if (instance->irq == NO_IRQ)
 		NCR5380_write(DTC_CONTROL_REG, CSR_DIR_READ);
 	else
@@ -348,7 +341,7 @@ static inline int NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst,
 		while (NCR5380_read(DTC_CONTROL_REG) & CSR_HOST_BUF_NOT_RDY)
 			++i;
 		rtrc(3);
-		memcpy_fromio(d, base + DTC_DATA_BUF, 128);
+		memcpy_fromio(d, hostdata->base + DTC_DATA_BUF, 128);
 		d += 128;
 		len -= 128;
 		rtrc(7);
@@ -358,9 +351,7 @@ static inline int NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst,
 	rtrc(4);
 	while (!(NCR5380_read(DTC_CONTROL_REG) & D_CR_ACCESS))
 		++i;
-	NCR5380_write(MODE_REG, 0);	/* Clear the operating mode */
 	rtrc(0);
-	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 	if (i > hostdata->spin_max_r)
 		hostdata->spin_max_r = i;
 	return (0);
@@ -383,12 +374,7 @@ static inline int NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src,
 {
 	int i;
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
 
-	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
-	NCR5380_write(MODE_REG, MR_ENABLE_EOP_INTR | MR_DMA_MODE);
-	/* set direction (write) */
 	if (instance->irq == NO_IRQ)
 		NCR5380_write(DTC_CONTROL_REG, 0);
 	else
@@ -400,7 +386,7 @@ static inline int NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src,
 		while (NCR5380_read(DTC_CONTROL_REG) & CSR_HOST_BUF_NOT_RDY)
 			++i;
 		rtrc(3);
-		memcpy_toio(base + DTC_DATA_BUF, src, 128);
+		memcpy_toio(hostdata->base + DTC_DATA_BUF, src, 128);
 		src += 128;
 		len -= 128;
 	}
@@ -413,47 +399,60 @@ static inline int NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src,
 		++i;
 	rtrc(7);
 	/* Check for parity error here. fixme. */
-	NCR5380_write(MODE_REG, 0);	/* Clear the operating mode */
 	rtrc(0);
 	if (i > hostdata->spin_max_w)
 		hostdata->spin_max_w = i;
 	return (0);
 }
 
+static int dtc_dma_xfer_len(struct scsi_cmnd *cmd)
+{
+	int transfersize = cmd->transfersize;
+
+	/* Limit transfers to 32K, for xx400 & xx406
+	 * pseudoDMA that transfers in 128 bytes blocks.
+	 */
+	if (transfersize > 32 * 1024 && cmd->SCp.this_residual &&
+	    !(cmd->SCp.this_residual % transfersize))
+		transfersize = 32 * 1024;
+
+	return transfersize;
+}
+
 MODULE_LICENSE("GPL");
 
 #include "NCR5380.c"
 
 static int dtc_release(struct Scsi_Host *shost)
 {
-	NCR5380_local_declare();
-	NCR5380_setup(shost);
+	struct NCR5380_hostdata *hostdata = shost_priv(shost);
+
 	if (shost->irq != NO_IRQ)
 		free_irq(shost->irq, shost);
 	NCR5380_exit(shost);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
 	scsi_unregister(shost);
-	iounmap(base);
+	iounmap(hostdata->base);
 	return 0;
 }
 
 static struct scsi_host_template driver_template = {
-	.name				= "DTC 3180/3280 ",
-	.detect				= dtc_detect,
-	.release			= dtc_release,
-	.proc_name			= "dtc3x80",
-	.show_info			= dtc_show_info,
-	.write_info			= dtc_write_info,
-	.info				= dtc_info,
-	.queuecommand			= dtc_queue_command,
-	.eh_abort_handler		= dtc_abort,
-	.eh_bus_reset_handler		= dtc_bus_reset,
-	.bios_param     		= dtc_biosparam,
-	.can_queue      		= CAN_QUEUE,
-	.this_id        		= 7,
-	.sg_tablesize   		= SG_ALL,
-	.cmd_per_lun    		= CMD_PER_LUN,
-	.use_clustering 		= DISABLE_CLUSTERING,
+	.name			= "DTC 3180/3280",
+	.detect			= dtc_detect,
+	.release		= dtc_release,
+	.proc_name		= "dtc3x80",
+	.show_info		= dtc_show_info,
+	.write_info		= dtc_write_info,
+	.info			= dtc_info,
+	.queuecommand		= dtc_queue_command,
+	.eh_abort_handler	= dtc_abort,
+	.eh_bus_reset_handler	= dtc_bus_reset,
+	.bios_param		= dtc_biosparam,
+	.can_queue		= 32,
+	.this_id		= 7,
+	.sg_tablesize		= SG_ALL,
+	.cmd_per_lun		= 2,
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 #include "scsi_module.c"
diff --git a/drivers/scsi/dtc.h b/drivers/scsi/dtc.h
index 78a2332e9064..56732cba8aba 100644
--- a/drivers/scsi/dtc.h
+++ b/drivers/scsi/dtc.h
@@ -10,54 +10,17 @@
 #ifndef DTC3280_H
 #define DTC3280_H
 
-#define DTCDEBUG 0
-#define DTCDEBUG_INIT	0x1
-#define DTCDEBUG_TRANSFER 0x2
-
-#ifndef CMD_PER_LUN
-#define CMD_PER_LUN 2
-#endif
-
-#ifndef CAN_QUEUE
-#define CAN_QUEUE 32 
-#endif
-
 #define NCR5380_implementation_fields \
     void __iomem *base
 
-#define NCR5380_local_declare() \
-    void __iomem *base
-
-#define NCR5380_setup(instance) \
-    base = ((struct NCR5380_hostdata *)(instance)->hostdata)->base
+#define DTC_address(reg) \
+	(((struct NCR5380_hostdata *)shost_priv(instance))->base + DTC_5380_OFFSET + reg)
 
-#define DTC_address(reg) (base + DTC_5380_OFFSET + reg)
-
-#define dbNCR5380_read(reg)                                              \
-    (rval=readb(DTC_address(reg)), \
-     (((unsigned char) printk("DTC : read register %d at addr %p is: %02x\n"\
-    , (reg), DTC_address(reg), rval)), rval ) )
-
-#define dbNCR5380_write(reg, value) do {                                  \
-    printk("DTC : write %02x to register %d at address %p\n",         \
-            (value), (reg), DTC_address(reg));     \
-    writeb(value, DTC_address(reg));} while(0)
-
-
-#if !(DTCDEBUG & DTCDEBUG_TRANSFER) 
 #define NCR5380_read(reg) (readb(DTC_address(reg)))
 #define NCR5380_write(reg, value) (writeb(value, DTC_address(reg)))
-#else
-#define NCR5380_read(reg) (readb(DTC_address(reg)))
-#define xNCR5380_read(reg)						\
-    (((unsigned char) printk("DTC : read register %d at address %p\n"\
-    , (reg), DTC_address(reg))), readb(DTC_address(reg)))
 
-#define NCR5380_write(reg, value) do {					\
-    printk("DTC : write %02x to register %d at address %p\n", 	\
-	    (value), (reg), DTC_address(reg));	\
-    writeb(value, DTC_address(reg));} while(0)
-#endif
+#define NCR5380_dma_xfer_len(instance, cmd, phase) \
+        dtc_dma_xfer_len(cmd)
 
 #define NCR5380_intr			dtc_intr
 #define NCR5380_queue_command		dtc_queue_command
diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c
index f8d2478b11cc..90091e693020 100644
--- a/drivers/scsi/g_NCR5380.c
+++ b/drivers/scsi/g_NCR5380.c
@@ -56,40 +56,31 @@
  *     
  */
 
-/* settings for DTC3181E card with only Mustek scanner attached */
-#define USLEEP_POLL	msecs_to_jiffies(10)
-#define USLEEP_SLEEP	msecs_to_jiffies(200)
-#define USLEEP_WAITLONG	msecs_to_jiffies(5000)
-
 #define AUTOPROBE_IRQ
 
 #ifdef CONFIG_SCSI_GENERIC_NCR53C400
-#define NCR53C400_PSEUDO_DMA 1
 #define PSEUDO_DMA
-#define NCR53C400
 #endif
 
 #include <asm/io.h>
-#include <linux/signal.h>
 #include <linux/blkdev.h>
+#include <linux/module.h>
 #include <scsi/scsi_host.h>
 #include "g_NCR5380.h"
 #include "NCR5380.h"
-#include <linux/stat.h>
 #include <linux/init.h>
 #include <linux/ioport.h>
 #include <linux/isapnp.h>
-#include <linux/delay.h>
 #include <linux/interrupt.h>
 
-#define NCR_NOT_SET 0
-static int ncr_irq = NCR_NOT_SET;
-static int ncr_dma = NCR_NOT_SET;
-static int ncr_addr = NCR_NOT_SET;
-static int ncr_5380 = NCR_NOT_SET;
-static int ncr_53c400 = NCR_NOT_SET;
-static int ncr_53c400a = NCR_NOT_SET;
-static int dtc_3181e = NCR_NOT_SET;
+static int ncr_irq;
+static int ncr_dma;
+static int ncr_addr;
+static int ncr_5380;
+static int ncr_53c400;
+static int ncr_53c400a;
+static int dtc_3181e;
+static int hp_c2502;
 
 static struct override {
 	NCR5380_map_type NCR5380_map_name;
@@ -121,7 +112,7 @@ static struct override {
 
 static void __init internal_setup(int board, char *str, int *ints)
 {
-	static int commandline_current = 0;
+	static int commandline_current;
 	switch (board) {
 	case BOARD_NCR5380:
 		if (ints[0] != 2 && ints[0] != 3) {
@@ -235,6 +226,30 @@ static int __init do_DTC3181E_setup(char *str)
 
 #endif
 
+#ifndef SCSI_G_NCR5380_MEM
+/*
+ * Configure I/O address of 53C400A or DTC436 by writing magic numbers
+ * to ports 0x779 and 0x379.
+ */
+static void magic_configure(int idx, u8 irq, u8 magic[])
+{
+	u8 cfg = 0;
+
+	outb(magic[0], 0x779);
+	outb(magic[1], 0x379);
+	outb(magic[2], 0x379);
+	outb(magic[3], 0x379);
+	outb(magic[4], 0x379);
+
+	/* allowed IRQs for HP C2502 */
+	if (irq != 2 && irq != 3 && irq != 4 && irq != 5 && irq != 7)
+		irq = 0;
+	if (idx >= 0 && idx <= 7)
+		cfg = 0x80 | idx | (irq << 4);
+	outb(cfg, 0x379);
+}
+#endif
+
 /**
  * 	generic_NCR5380_detect	-	look for NCR5380 controllers
  *	@tpnt: the scsi template
@@ -243,19 +258,18 @@ static int __init do_DTC3181E_setup(char *str)
  *	and DTC436(ISAPnP) controllers. If overrides have been set we use
  *	them.
  *
- *	The caller supplied NCR5380_init function is invoked from here, before
- *	the interrupt line is taken.
- *
  *	Locks: none
  */
 
 static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 {
-	static int current_override = 0;
+	static int current_override;
 	int count;
 	unsigned int *ports;
+	u8 *magic = NULL;
 #ifndef SCSI_G_NCR5380_MEM
 	int i;
+	int port_idx = -1;
 	unsigned long region_size = 16;
 #endif
 	static unsigned int __initdata ncr_53c400a_ports[] = {
@@ -264,27 +278,36 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 	static unsigned int __initdata dtc_3181e_ports[] = {
 		0x220, 0x240, 0x280, 0x2a0, 0x2c0, 0x300, 0x320, 0x340, 0
 	};
-	int flags = 0;
+	static u8 ncr_53c400a_magic[] __initdata = {	/* 53C400A & DTC436 */
+		0x59, 0xb9, 0xc5, 0xae, 0xa6
+	};
+	static u8 hp_c2502_magic[] __initdata = {	/* HP C2502 */
+		0x0f, 0x22, 0xf0, 0x20, 0x80
+	};
+	int flags;
 	struct Scsi_Host *instance;
+	struct NCR5380_hostdata *hostdata;
 #ifdef SCSI_G_NCR5380_MEM
 	unsigned long base;
 	void __iomem *iomem;
 #endif
 
-	if (ncr_irq != NCR_NOT_SET)
+	if (ncr_irq)
 		overrides[0].irq = ncr_irq;
-	if (ncr_dma != NCR_NOT_SET)
+	if (ncr_dma)
 		overrides[0].dma = ncr_dma;
-	if (ncr_addr != NCR_NOT_SET)
+	if (ncr_addr)
 		overrides[0].NCR5380_map_name = (NCR5380_map_type) ncr_addr;
-	if (ncr_5380 != NCR_NOT_SET)
+	if (ncr_5380)
 		overrides[0].board = BOARD_NCR5380;
-	else if (ncr_53c400 != NCR_NOT_SET)
+	else if (ncr_53c400)
 		overrides[0].board = BOARD_NCR53C400;
-	else if (ncr_53c400a != NCR_NOT_SET)
+	else if (ncr_53c400a)
 		overrides[0].board = BOARD_NCR53C400A;
-	else if (dtc_3181e != NCR_NOT_SET)
+	else if (dtc_3181e)
 		overrides[0].board = BOARD_DTC3181E;
+	else if (hp_c2502)
+		overrides[0].board = BOARD_HP_C2502;
 #ifndef SCSI_G_NCR5380_MEM
 	if (!current_override && isapnp_present()) {
 		struct pnp_dev *dev = NULL;
@@ -318,41 +341,45 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 		}
 	}
 #endif
-	tpnt->proc_name = "g_NCR5380";
 
 	for (count = 0; current_override < NO_OVERRIDES; ++current_override) {
 		if (!(overrides[current_override].NCR5380_map_name))
 			continue;
 
 		ports = NULL;
+		flags = 0;
 		switch (overrides[current_override].board) {
 		case BOARD_NCR5380:
 			flags = FLAG_NO_PSEUDO_DMA;
 			break;
 		case BOARD_NCR53C400:
-			flags = FLAG_NCR53C400;
+#ifdef PSEUDO_DMA
+			flags = FLAG_NO_DMA_FIXUP;
+#endif
 			break;
 		case BOARD_NCR53C400A:
-			flags = FLAG_NO_PSEUDO_DMA;
+			flags = FLAG_NO_DMA_FIXUP;
+			ports = ncr_53c400a_ports;
+			magic = ncr_53c400a_magic;
+			break;
+		case BOARD_HP_C2502:
+			flags = FLAG_NO_DMA_FIXUP;
 			ports = ncr_53c400a_ports;
+			magic = hp_c2502_magic;
 			break;
 		case BOARD_DTC3181E:
-			flags = FLAG_NO_PSEUDO_DMA | FLAG_DTC3181E;
+			flags = FLAG_NO_DMA_FIXUP;
 			ports = dtc_3181e_ports;
+			magic = ncr_53c400a_magic;
 			break;
 		}
 
 #ifndef SCSI_G_NCR5380_MEM
-		if (ports) {
+		if (ports && magic) {
 			/* wakeup sequence for the NCR53C400A and DTC3181E */
 
 			/* Disable the adapter and look for a free io port */
-			outb(0x59, 0x779);
-			outb(0xb9, 0x379);
-			outb(0xc5, 0x379);
-			outb(0xae, 0x379);
-			outb(0xa6, 0x379);
-			outb(0x00, 0x379);
+			magic_configure(-1, 0, magic);
 
 			if (overrides[current_override].NCR5380_map_name != PORT_AUTO)
 				for (i = 0; ports[i]; i++) {
@@ -371,17 +398,12 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 				}
 			if (ports[i]) {
 				/* At this point we have our region reserved */
-				outb(0x59, 0x779);
-				outb(0xb9, 0x379);
-				outb(0xc5, 0x379);
-				outb(0xae, 0x379);
-				outb(0xa6, 0x379);
-				outb(0x80 | i, 0x379);	/* set io port to be used */
+				magic_configure(i, 0, magic); /* no IRQ yet */
 				outb(0xc0, ports[i] + 9);
 				if (inb(ports[i] + 9) != 0x80)
 					continue;
-				else
-					overrides[current_override].NCR5380_map_name = ports[i];
+				overrides[current_override].NCR5380_map_name = ports[i];
+				port_idx = i;
 			} else
 				continue;
 		}
@@ -403,24 +425,65 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 		}
 #endif
 		instance = scsi_register(tpnt, sizeof(struct NCR5380_hostdata));
-		if (instance == NULL) {
-#ifndef SCSI_G_NCR5380_MEM
-			release_region(overrides[current_override].NCR5380_map_name, region_size);
-#else
-			iounmap(iomem);
-			release_mem_region(base, NCR5380_region_size);
-#endif
-			continue;
-		}
+		if (instance == NULL)
+			goto out_release;
+		hostdata = shost_priv(instance);
 
-		instance->NCR5380_instance_name = overrides[current_override].NCR5380_map_name;
 #ifndef SCSI_G_NCR5380_MEM
+		instance->io_port = overrides[current_override].NCR5380_map_name;
 		instance->n_io_port = region_size;
+		hostdata->io_width = 1; /* 8-bit PDMA by default */
+
+		/*
+		 * On NCR53C400 boards, NCR5380 registers are mapped 8 past
+		 * the base address.
+		 */
+		switch (overrides[current_override].board) {
+		case BOARD_NCR53C400:
+			instance->io_port += 8;
+			hostdata->c400_ctl_status = 0;
+			hostdata->c400_blk_cnt = 1;
+			hostdata->c400_host_buf = 4;
+			break;
+		case BOARD_DTC3181E:
+			hostdata->io_width = 2;	/* 16-bit PDMA */
+			/* fall through */
+		case BOARD_NCR53C400A:
+		case BOARD_HP_C2502:
+			hostdata->c400_ctl_status = 9;
+			hostdata->c400_blk_cnt = 10;
+			hostdata->c400_host_buf = 8;
+			break;
+		}
 #else
-		((struct NCR5380_hostdata *)instance->hostdata)->iomem = iomem;
+		instance->base = overrides[current_override].NCR5380_map_name;
+		hostdata->iomem = iomem;
+		switch (overrides[current_override].board) {
+		case BOARD_NCR53C400:
+			hostdata->c400_ctl_status = 0x100;
+			hostdata->c400_blk_cnt = 0x101;
+			hostdata->c400_host_buf = 0x104;
+			break;
+		case BOARD_DTC3181E:
+		case BOARD_NCR53C400A:
+		case BOARD_HP_C2502:
+			pr_err(DRV_MODULE_NAME ": unknown register offsets\n");
+			goto out_unregister;
+		}
 #endif
 
-		NCR5380_init(instance, flags);
+		if (NCR5380_init(instance, flags))
+			goto out_unregister;
+
+		switch (overrides[current_override].board) {
+		case BOARD_NCR53C400:
+		case BOARD_DTC3181E:
+		case BOARD_NCR53C400A:
+		case BOARD_HP_C2502:
+			NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
+		}
+
+		NCR5380_maybe_reset_bus(instance);
 
 		if (overrides[current_override].irq != IRQ_AUTO)
 			instance->irq = overrides[current_override].irq;
@@ -431,12 +494,18 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 		if (instance->irq == 255)
 			instance->irq = NO_IRQ;
 
-		if (instance->irq != NO_IRQ)
+		if (instance->irq != NO_IRQ) {
+#ifndef SCSI_G_NCR5380_MEM
+			/* set IRQ for HP C2502 */
+			if (overrides[current_override].board == BOARD_HP_C2502)
+				magic_configure(port_idx, instance->irq, magic);
+#endif
 			if (request_irq(instance->irq, generic_NCR5380_intr,
 					0, "NCR5380", instance)) {
 				printk(KERN_WARNING "scsi%d : IRQ%d not free, interrupts disabled\n", instance->host_no, instance->irq);
 				instance->irq = NO_IRQ;
 			}
+		}
 
 		if (instance->irq == NO_IRQ) {
 			printk(KERN_INFO "scsi%d : interrupts not enabled. for better interactive performance,\n", instance->host_no);
@@ -447,6 +516,17 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
 		++count;
 	}
 	return count;
+
+out_unregister:
+	scsi_unregister(instance);
+out_release:
+#ifndef SCSI_G_NCR5380_MEM
+	release_region(overrides[current_override].NCR5380_map_name, region_size);
+#else
+	iounmap(iomem);
+	release_mem_region(base, NCR5380_region_size);
+#endif
+	return count;
 }
 
 /**
@@ -460,21 +540,15 @@ static int __init generic_NCR5380_detect(struct scsi_host_template *tpnt)
  
 static int generic_NCR5380_release_resources(struct Scsi_Host *instance)
 {
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-	
 	if (instance->irq != NO_IRQ)
 		free_irq(instance->irq, instance);
 	NCR5380_exit(instance);
-
 #ifndef SCSI_G_NCR5380_MEM
-	release_region(instance->NCR5380_instance_name, instance->n_io_port);
+	release_region(instance->io_port, instance->n_io_port);
 #else
 	iounmap(((struct NCR5380_hostdata *)instance->hostdata)->iomem);
-	release_mem_region(instance->NCR5380_instance_name, NCR5380_region_size);
+	release_mem_region(instance->base, NCR5380_region_size);
 #endif
-
-
 	return 0;
 }
 
@@ -507,7 +581,7 @@ generic_NCR5380_biosparam(struct scsi_device *sdev, struct block_device *bdev,
 }
 #endif
 
-#ifdef NCR53C400_PSEUDO_DMA
+#ifdef PSEUDO_DMA
 
 /**
  *	NCR5380_pread		-	pseudo DMA read
@@ -521,75 +595,68 @@ generic_NCR5380_biosparam(struct scsi_device *sdev, struct block_device *bdev,
  
 static inline int NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst, int len)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int blocks = len / 128;
 	int start = 0;
-	int bl;
-
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
 
-	NCR5380_write(C400_CONTROL_STATUS_REG, CSR_BASE | CSR_TRANS_DIR);
-	NCR5380_write(C400_BLOCK_COUNTER_REG, blocks);
+	NCR5380_write(hostdata->c400_ctl_status, CSR_BASE | CSR_TRANS_DIR);
+	NCR5380_write(hostdata->c400_blk_cnt, blocks);
 	while (1) {
-		if ((bl = NCR5380_read(C400_BLOCK_COUNTER_REG)) == 0) {
+		if (NCR5380_read(hostdata->c400_blk_cnt) == 0)
 			break;
-		}
-		if (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_GATED_53C80_IRQ) {
+		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ) {
 			printk(KERN_ERR "53C400r: Got 53C80_IRQ start=%d, blocks=%d\n", start, blocks);
 			return -1;
 		}
-		while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_HOST_BUF_NOT_RDY);
+		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
+			; /* FIXME - no timeout */
 
 #ifndef SCSI_G_NCR5380_MEM
-		{
-			int i;
-			for (i = 0; i < 128; i++)
-				dst[start + i] = NCR5380_read(C400_HOST_BUFFER);
-		}
+		if (hostdata->io_width == 2)
+			insw(instance->io_port + hostdata->c400_host_buf,
+							dst + start, 64);
+		else
+			insb(instance->io_port + hostdata->c400_host_buf,
+							dst + start, 128);
 #else
 		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_fromio(dst + start, iomem + NCR53C400_host_buffer, 128);
+		memcpy_fromio(dst + start,
+		              hostdata->iomem + NCR53C400_host_buffer, 128);
 #endif
 		start += 128;
 		blocks--;
 	}
 
 	if (blocks) {
-		while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_HOST_BUF_NOT_RDY)
-		{
-			// FIXME - no timeout
-		}
+		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
+			; /* FIXME - no timeout */
 
 #ifndef SCSI_G_NCR5380_MEM
-		{
-			int i;	
-			for (i = 0; i < 128; i++)
-				dst[start + i] = NCR5380_read(C400_HOST_BUFFER);
-		}
+		if (hostdata->io_width == 2)
+			insw(instance->io_port + hostdata->c400_host_buf,
+							dst + start, 64);
+		else
+			insb(instance->io_port + hostdata->c400_host_buf,
+							dst + start, 128);
 #else
 		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_fromio(dst + start, iomem + NCR53C400_host_buffer, 128);
+		memcpy_fromio(dst + start,
+		              hostdata->iomem + NCR53C400_host_buffer, 128);
 #endif
 		start += 128;
 		blocks--;
 	}
 
-	if (!(NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_GATED_53C80_IRQ))
+	if (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ))
 		printk("53C400r: no 53C80 gated irq after transfer");
 
-#if 0
-	/*
-	 *	DON'T DO THIS - THEY NEVER ARRIVE!
-	 */
-	printk("53C400r: Waiting for 53C80 registers\n");
-	while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_53C80_REG)
+	/* wait for 53C80 registers to be available */
+	while (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_53C80_REG))
 		;
-#endif
+
 	if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_END_DMA_TRANSFER))
 		printk(KERN_ERR "53C400r: no end dma signal\n");
 		
-	NCR5380_write(MODE_REG, MR_BASE);
-	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 	return 0;
 }
 
@@ -605,89 +672,91 @@ static inline int NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst,
 
 static inline int NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src, int len)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int blocks = len / 128;
 	int start = 0;
-	int bl;
-	int i;
 
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-
-	NCR5380_write(C400_CONTROL_STATUS_REG, CSR_BASE);
-	NCR5380_write(C400_BLOCK_COUNTER_REG, blocks);
+	NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
+	NCR5380_write(hostdata->c400_blk_cnt, blocks);
 	while (1) {
-		if (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_GATED_53C80_IRQ) {
+		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ) {
 			printk(KERN_ERR "53C400w: Got 53C80_IRQ start=%d, blocks=%d\n", start, blocks);
 			return -1;
 		}
 
-		if ((bl = NCR5380_read(C400_BLOCK_COUNTER_REG)) == 0) {
+		if (NCR5380_read(hostdata->c400_blk_cnt) == 0)
 			break;
-		}
-		while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_HOST_BUF_NOT_RDY)
+		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; // FIXME - timeout
 #ifndef SCSI_G_NCR5380_MEM
-		{
-			for (i = 0; i < 128; i++)
-				NCR5380_write(C400_HOST_BUFFER, src[start + i]);
-		}
+		if (hostdata->io_width == 2)
+			outsw(instance->io_port + hostdata->c400_host_buf,
+							src + start, 64);
+		else
+			outsb(instance->io_port + hostdata->c400_host_buf,
+							src + start, 128);
 #else
 		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_toio(iomem + NCR53C400_host_buffer, src + start, 128);
+		memcpy_toio(hostdata->iomem + NCR53C400_host_buffer,
+		            src + start, 128);
 #endif
 		start += 128;
 		blocks--;
 	}
 	if (blocks) {
-		while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_HOST_BUF_NOT_RDY)
+		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; // FIXME - no timeout
 
 #ifndef SCSI_G_NCR5380_MEM
-		{
-			for (i = 0; i < 128; i++)
-				NCR5380_write(C400_HOST_BUFFER, src[start + i]);
-		}
+		if (hostdata->io_width == 2)
+			outsw(instance->io_port + hostdata->c400_host_buf,
+							src + start, 64);
+		else
+			outsb(instance->io_port + hostdata->c400_host_buf,
+							src + start, 128);
 #else
 		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_toio(iomem + NCR53C400_host_buffer, src + start, 128);
+		memcpy_toio(hostdata->iomem + NCR53C400_host_buffer,
+		            src + start, 128);
 #endif
 		start += 128;
 		blocks--;
 	}
 
-#if 0
-	printk("53C400w: waiting for registers to be available\n");
-	THEY NEVER DO ! while (NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_53C80_REG);
-	printk("53C400w: Got em\n");
-#endif
-
-	/* Let's wait for this instead - could be ugly */
-	/* All documentation says to check for this. Maybe my hardware is too
-	 * fast. Waiting for it seems to work fine! KLL
-	 */
-	while (!(i = NCR5380_read(C400_CONTROL_STATUS_REG) & CSR_GATED_53C80_IRQ))
-		;	// FIXME - no timeout
-
-	/*
-	 * I know. i is certainly != 0 here but the loop is new. See previous
-	 * comment.
-	 */
-	if (i) {
-		if (!((i = NCR5380_read(BUS_AND_STATUS_REG)) & BASR_END_DMA_TRANSFER))
-			printk(KERN_ERR "53C400w: No END OF DMA bit - WHOOPS! BASR=%0x\n", i);
-	} else
-		printk(KERN_ERR "53C400w: no 53C80 gated irq after transfer (last block)\n");
+	/* wait for 53C80 registers to be available */
+	while (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_53C80_REG)) {
+		udelay(4); /* DTC436 chip hangs without this */
+		/* FIXME - no timeout */
+	}
 
-#if 0
 	if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_END_DMA_TRANSFER)) {
 		printk(KERN_ERR "53C400w: no end dma signal\n");
 	}
-#endif
+
 	while (!(NCR5380_read(TARGET_COMMAND_REG) & TCR_LAST_BYTE_SENT))
 		; 	// TIMEOUT
 	return 0;
 }
-#endif				/* PSEUDO_DMA */
+
+static int generic_NCR5380_dma_xfer_len(struct scsi_cmnd *cmd)
+{
+	int transfersize = cmd->transfersize;
+
+	/* Limit transfers to 32K, for xx400 & xx406
+	 * pseudoDMA that transfers in 128 bytes blocks.
+	 */
+	if (transfersize > 32 * 1024 && cmd->SCp.this_residual &&
+	    !(cmd->SCp.this_residual % transfersize))
+		transfersize = 32 * 1024;
+
+	/* 53C400 datasheet: non-modulo-128-byte transfers should use PIO */
+	if (transfersize % 128)
+		transfersize = 0;
+
+	return transfersize;
+}
+
+#endif /* PSEUDO_DMA */
 
 /*
  *	Include the NCR5380 core code that we build our driver around	
@@ -696,22 +765,24 @@ static inline int NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src,
 #include "NCR5380.c"
 
 static struct scsi_host_template driver_template = {
-	.show_info      	= generic_NCR5380_show_info,
-	.name           	= "Generic NCR5380/NCR53C400 SCSI",
-	.detect         	= generic_NCR5380_detect,
-	.release        	= generic_NCR5380_release_resources,
-	.info           	= generic_NCR5380_info,
-	.queuecommand   	= generic_NCR5380_queue_command,
+	.proc_name		= DRV_MODULE_NAME,
+	.name			= "Generic NCR5380/NCR53C400 SCSI",
+	.detect			= generic_NCR5380_detect,
+	.release		= generic_NCR5380_release_resources,
+	.info			= generic_NCR5380_info,
+	.queuecommand		= generic_NCR5380_queue_command,
 	.eh_abort_handler	= generic_NCR5380_abort,
 	.eh_bus_reset_handler	= generic_NCR5380_bus_reset,
-	.bios_param     	= NCR5380_BIOSPARAM,
-	.can_queue      	= CAN_QUEUE,
-        .this_id        	= 7,
-        .sg_tablesize   	= SG_ALL,
-	.cmd_per_lun    	= CMD_PER_LUN,
-        .use_clustering		= DISABLE_CLUSTERING,
+	.bios_param		= NCR5380_BIOSPARAM,
+	.can_queue		= 16,
+	.this_id		= 7,
+	.sg_tablesize		= SG_ALL,
+	.cmd_per_lun		= 2,
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
-#include <linux/module.h>
+
 #include "scsi_module.c"
 
 module_param(ncr_irq, int, 0);
@@ -721,6 +792,7 @@ module_param(ncr_5380, int, 0);
 module_param(ncr_53c400, int, 0);
 module_param(ncr_53c400a, int, 0);
 module_param(dtc_3181e, int, 0);
+module_param(hp_c2502, int, 0);
 MODULE_LICENSE("GPL");
 
 #if !defined(SCSI_G_NCR5380_MEM) && defined(MODULE)
diff --git a/drivers/scsi/g_NCR5380.h b/drivers/scsi/g_NCR5380.h
index bea1a3b9b862..6f3d2ac4f185 100644
--- a/drivers/scsi/g_NCR5380.h
+++ b/drivers/scsi/g_NCR5380.h
@@ -14,81 +14,67 @@
 #ifndef GENERIC_NCR5380_H
 #define GENERIC_NCR5380_H
 
-#ifdef NCR53C400
+#ifdef CONFIG_SCSI_GENERIC_NCR53C400
 #define BIOSPARAM
 #define NCR5380_BIOSPARAM generic_NCR5380_biosparam
 #else
 #define NCR5380_BIOSPARAM NULL
 #endif
 
-#ifndef ASM
-
-#ifndef CMD_PER_LUN
-#define CMD_PER_LUN 2
-#endif
-
-#ifndef CAN_QUEUE
-#define CAN_QUEUE 16
-#endif
-
 #define __STRVAL(x) #x
 #define STRVAL(x) __STRVAL(x)
 
 #ifndef SCSI_G_NCR5380_MEM
+#define DRV_MODULE_NAME "g_NCR5380"
 
-#define NCR5380_map_config port
 #define NCR5380_map_type int
 #define NCR5380_map_name port
-#define NCR5380_instance_name io_port
-#define NCR53C400_register_offset 0
-#define NCR53C400_address_adjust 8
 
-#ifdef NCR53C400
+#ifdef CONFIG_SCSI_GENERIC_NCR53C400
 #define NCR5380_region_size 16
 #else
 #define NCR5380_region_size 8
 #endif
 
-#define NCR5380_read(reg) (inb(NCR5380_map_name + (reg)))
-#define NCR5380_write(reg, value) (outb((value), (NCR5380_map_name + (reg))))
+#define NCR5380_read(reg) \
+	inb(instance->io_port + (reg))
+#define NCR5380_write(reg, value) \
+	outb(value, instance->io_port + (reg))
 
 #define NCR5380_implementation_fields \
-    NCR5380_map_type NCR5380_map_name
-
-#define NCR5380_local_declare() \
-    register NCR5380_implementation_fields
-
-#define NCR5380_setup(instance) \
-    NCR5380_map_name = (NCR5380_map_type)((instance)->NCR5380_instance_name)
+	int c400_ctl_status; \
+	int c400_blk_cnt; \
+	int c400_host_buf; \
+	int io_width;
 
 #else 
 /* therefore SCSI_G_NCR5380_MEM */
+#define DRV_MODULE_NAME "g_NCR5380_mmio"
 
-#define NCR5380_map_config memory
 #define NCR5380_map_type unsigned long
 #define NCR5380_map_name base
-#define NCR5380_instance_name base
-#define NCR53C400_register_offset 0x108
-#define NCR53C400_address_adjust 0
 #define NCR53C400_mem_base 0x3880
 #define NCR53C400_host_buffer 0x3900
 #define NCR5380_region_size 0x3a00
 
-#define NCR5380_read(reg) readb(iomem + NCR53C400_mem_base + (reg))
-#define NCR5380_write(reg, value) writeb(value, iomem + NCR53C400_mem_base + (reg))
+#define NCR5380_read(reg) \
+	readb(((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \
+	      NCR53C400_mem_base + (reg))
+#define NCR5380_write(reg, value) \
+	writeb(value, ((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \
+	       NCR53C400_mem_base + (reg))
 
 #define NCR5380_implementation_fields \
-    NCR5380_map_type NCR5380_map_name; \
-    void __iomem *iomem;
-
-#define NCR5380_local_declare() \
-    register void __iomem *iomem
-
-#define NCR5380_setup(instance) \
-    iomem = (((struct NCR5380_hostdata *)(instance)->hostdata)->iomem)
+	void __iomem *iomem; \
+	int c400_ctl_status; \
+	int c400_blk_cnt; \
+	int c400_host_buf;
 
 #endif
 
+#define NCR5380_dma_xfer_len(instance, cmd, phase) \
+        generic_NCR5380_dma_xfer_len(cmd)
+
 #define NCR5380_intr generic_NCR5380_intr
 #define NCR5380_queue_command generic_NCR5380_queue_command
 #define NCR5380_abort generic_NCR5380_abort
@@ -102,7 +88,7 @@
 #define BOARD_NCR53C400	1
 #define BOARD_NCR53C400A 2
 #define BOARD_DTC3181E	3
+#define BOARD_HP_C2502	4
 
-#endif /* ndef ASM */
 #endif /* GENERIC_NCR5380_H */
 
diff --git a/drivers/scsi/hisi_sas/Kconfig b/drivers/scsi/hisi_sas/Kconfig
index 37a0c7156087..b67661836c9f 100644
--- a/drivers/scsi/hisi_sas/Kconfig
+++ b/drivers/scsi/hisi_sas/Kconfig
@@ -1,5 +1,7 @@
 config SCSI_HISI_SAS
 	tristate "HiSilicon SAS"
+	depends on HAS_DMA
+	depends on ARM64 || COMPILE_TEST
 	select SCSI_SAS_LIBSAS
 	select BLK_DEV_INTEGRITY
 	help
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index d54381149c0d..057fdeb720ac 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -247,41 +247,36 @@
 /* ITCT header */
 /* qw0 */
 #define ITCT_HDR_DEV_TYPE_OFF		0
-#define ITCT_HDR_DEV_TYPE_MSK		(0x3 << ITCT_HDR_DEV_TYPE_OFF)
+#define ITCT_HDR_DEV_TYPE_MSK		(0x3ULL << ITCT_HDR_DEV_TYPE_OFF)
 #define ITCT_HDR_VALID_OFF		2
-#define ITCT_HDR_VALID_MSK		(0x1 << ITCT_HDR_VALID_OFF)
-#define ITCT_HDR_BREAK_REPLY_ENA_OFF	3
-#define ITCT_HDR_BREAK_REPLY_ENA_MSK	(0x1 << ITCT_HDR_BREAK_REPLY_ENA_OFF)
+#define ITCT_HDR_VALID_MSK		(0x1ULL << ITCT_HDR_VALID_OFF)
 #define ITCT_HDR_AWT_CONTROL_OFF	4
-#define ITCT_HDR_AWT_CONTROL_MSK	(0x1 << ITCT_HDR_AWT_CONTROL_OFF)
+#define ITCT_HDR_AWT_CONTROL_MSK	(0x1ULL << ITCT_HDR_AWT_CONTROL_OFF)
 #define ITCT_HDR_MAX_CONN_RATE_OFF	5
-#define ITCT_HDR_MAX_CONN_RATE_MSK	(0xf << ITCT_HDR_MAX_CONN_RATE_OFF)
+#define ITCT_HDR_MAX_CONN_RATE_MSK	(0xfULL << ITCT_HDR_MAX_CONN_RATE_OFF)
 #define ITCT_HDR_VALID_LINK_NUM_OFF	9
-#define ITCT_HDR_VALID_LINK_NUM_MSK	(0xf << ITCT_HDR_VALID_LINK_NUM_OFF)
+#define ITCT_HDR_VALID_LINK_NUM_MSK	(0xfULL << ITCT_HDR_VALID_LINK_NUM_OFF)
 #define ITCT_HDR_PORT_ID_OFF		13
-#define ITCT_HDR_PORT_ID_MSK		(0x7 << ITCT_HDR_PORT_ID_OFF)
+#define ITCT_HDR_PORT_ID_MSK		(0x7ULL << ITCT_HDR_PORT_ID_OFF)
 #define ITCT_HDR_SMP_TIMEOUT_OFF	16
-#define ITCT_HDR_SMP_TIMEOUT_MSK	(0xffff << ITCT_HDR_SMP_TIMEOUT_OFF)
-#define ITCT_HDR_MAX_BURST_BYTES_OFF	16
-#define ITCT_HDR_MAX_BURST_BYTES_MSK	(0xffffffff << \
-					ITCT_MAX_BURST_BYTES_OFF)
+#define ITCT_HDR_SMP_TIMEOUT_MSK	(0xffffULL << ITCT_HDR_SMP_TIMEOUT_OFF)
 /* qw1 */
 #define ITCT_HDR_MAX_SAS_ADDR_OFF	0
 #define ITCT_HDR_MAX_SAS_ADDR_MSK	(0xffffffffffffffff << \
 					ITCT_HDR_MAX_SAS_ADDR_OFF)
 /* qw2 */
 #define ITCT_HDR_IT_NEXUS_LOSS_TL_OFF	0
-#define ITCT_HDR_IT_NEXUS_LOSS_TL_MSK	(0xffff << \
+#define ITCT_HDR_IT_NEXUS_LOSS_TL_MSK	(0xffffULL << \
 					ITCT_HDR_IT_NEXUS_LOSS_TL_OFF)
 #define ITCT_HDR_BUS_INACTIVE_TL_OFF	16
-#define ITCT_HDR_BUS_INACTIVE_TL_MSK	(0xffff << \
+#define ITCT_HDR_BUS_INACTIVE_TL_MSK	(0xffffULL << \
 					ITCT_HDR_BUS_INACTIVE_TL_OFF)
 #define ITCT_HDR_MAX_CONN_TL_OFF	32
-#define ITCT_HDR_MAX_CONN_TL_MSK	(0xffff << \
+#define ITCT_HDR_MAX_CONN_TL_MSK	(0xffffULL << \
 					ITCT_HDR_MAX_CONN_TL_OFF)
 #define ITCT_HDR_REJ_OPEN_TL_OFF	48
-#define ITCT_HDR_REJ_OPEN_TL_MSK	(0xffff << \
-					ITCT_REJ_OPEN_TL_OFF)
+#define ITCT_HDR_REJ_OPEN_TL_MSK	(0xffffULL << \
+					ITCT_HDR_REJ_OPEN_TL_OFF)
 
 /* Err record header */
 #define ERR_HDR_DMA_TX_ERR_TYPE_OFF	0
@@ -533,10 +528,10 @@ static void setup_itct_v1_hw(struct hisi_hba *hisi_hba,
 	itct->sas_addr = __swab64(itct->sas_addr);
 
 	/* qw2 */
-	itct->qw2 = cpu_to_le64((500 < ITCT_HDR_IT_NEXUS_LOSS_TL_OFF) |
-				(0xff00 < ITCT_HDR_BUS_INACTIVE_TL_OFF) |
-				(0xff00 < ITCT_HDR_MAX_CONN_TL_OFF) |
-				(0xff00 < ITCT_HDR_REJ_OPEN_TL_OFF));
+	itct->qw2 = cpu_to_le64((500ULL << ITCT_HDR_IT_NEXUS_LOSS_TL_OFF) |
+				(0xff00ULL << ITCT_HDR_BUS_INACTIVE_TL_OFF) |
+				(0xff00ULL << ITCT_HDR_MAX_CONN_TL_OFF) |
+				(0xff00ULL << ITCT_HDR_REJ_OPEN_TL_OFF));
 }
 
 static void free_device_v1_hw(struct hisi_hba *hisi_hba,
@@ -544,7 +539,8 @@ static void free_device_v1_hw(struct hisi_hba *hisi_hba,
 {
 	u64 dev_id = sas_dev->device_id;
 	struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
-	u32 qw0, reg_val = hisi_sas_read32(hisi_hba, CFG_AGING_TIME);
+	u64 qw0;
+	u32 reg_val = hisi_sas_read32(hisi_hba, CFG_AGING_TIME);
 
 	reg_val |= CFG_AGING_TIME_ITCT_REL_MSK;
 	hisi_sas_write32(hisi_hba, CFG_AGING_TIME, reg_val);
diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c
index 4e1a632ccf16..f8b88fa78e62 100644
--- a/drivers/scsi/imm.c
+++ b/drivers/scsi/imm.c
@@ -43,6 +43,7 @@ typedef struct {
 	unsigned dp:1;		/* Data phase present           */
 	unsigned rd:1;		/* Read data in data phase      */
 	unsigned wanted:1;	/* Parport sharing busy flag    */
+	unsigned int dev_no;	/* Device number		*/
 	wait_queue_head_t *waiting;
 	struct Scsi_Host *host;
 	struct list_head list;
@@ -1120,15 +1121,40 @@ static struct scsi_host_template imm_template = {
 
 static LIST_HEAD(imm_hosts);
 
+/*
+ * Finds the first available device number that can be alloted to the
+ * new imm device and returns the address of the previous node so that
+ * we can add to the tail and have a list in the ascending order.
+ */
+
+static inline imm_struct *find_parent(void)
+{
+	imm_struct *dev, *par = NULL;
+	unsigned int cnt = 0;
+
+	if (list_empty(&imm_hosts))
+		return NULL;
+
+	list_for_each_entry(dev, &imm_hosts, list) {
+		if (dev->dev_no != cnt)
+			return par;
+		cnt++;
+		par = dev;
+	}
+
+	return par;
+}
+
 static int __imm_attach(struct parport *pb)
 {
 	struct Scsi_Host *host;
-	imm_struct *dev;
+	imm_struct *dev, *temp;
 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waiting);
 	DEFINE_WAIT(wait);
 	int ports;
 	int modes, ppb;
 	int err = -ENOMEM;
+	struct pardev_cb imm_cb;
 
 	init_waitqueue_head(&waiting);
 
@@ -1141,9 +1167,15 @@ static int __imm_attach(struct parport *pb)
 	dev->mode = IMM_AUTODETECT;
 	INIT_LIST_HEAD(&dev->list);
 
-	dev->dev = parport_register_device(pb, "imm", NULL, imm_wakeup,
-						NULL, 0, dev);
+	temp = find_parent();
+	if (temp)
+		dev->dev_no = temp->dev_no + 1;
+
+	memset(&imm_cb, 0, sizeof(imm_cb));
+	imm_cb.private = dev;
+	imm_cb.wakeup = imm_wakeup;
 
+	dev->dev = parport_register_dev_model(pb, "imm", &imm_cb, dev->dev_no);
 	if (!dev->dev)
 		goto out;
 
@@ -1207,7 +1239,10 @@ static int __imm_attach(struct parport *pb)
 	host->unique_id = pb->number;
 	*(imm_struct **)&host->hostdata = dev;
 	dev->host = host;
-	list_add_tail(&dev->list, &imm_hosts);
+	if (!temp)
+		list_add_tail(&dev->list, &imm_hosts);
+	else
+		list_add_tail(&dev->list, &temp->list);
 	err = scsi_add_host(host, NULL);
 	if (err)
 		goto out2;
@@ -1245,9 +1280,10 @@ static void imm_detach(struct parport *pb)
 }
 
 static struct parport_driver imm_driver = {
-	.name	= "imm",
-	.attach	= imm_attach,
-	.detach	= imm_detach,
+	.name		= "imm",
+	.match_port	= imm_attach,
+	.detach		= imm_detach,
+	.devmodel	= true,
 };
 
 static int __init imm_driver_init(void)
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 536cd5a80422..3b3e0998fa6e 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -3638,7 +3638,7 @@ static struct device_attribute ipr_ioa_reset_attr = {
 	.store = ipr_store_reset_adapter
 };
 
-static int ipr_iopoll(struct blk_iopoll *iop, int budget);
+static int ipr_iopoll(struct irq_poll *iop, int budget);
  /**
  * ipr_show_iopoll_weight - Show ipr polling mode
  * @dev:	class device struct
@@ -3681,34 +3681,33 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
 	int i;
 
 	if (!ioa_cfg->sis64) {
-		dev_info(&ioa_cfg->pdev->dev, "blk-iopoll not supported on this adapter\n");
+		dev_info(&ioa_cfg->pdev->dev, "irq_poll not supported on this adapter\n");
 		return -EINVAL;
 	}
 	if (kstrtoul(buf, 10, &user_iopoll_weight))
 		return -EINVAL;
 
 	if (user_iopoll_weight > 256) {
-		dev_info(&ioa_cfg->pdev->dev, "Invalid blk-iopoll weight. It must be less than 256\n");
+		dev_info(&ioa_cfg->pdev->dev, "Invalid irq_poll weight. It must be less than 256\n");
 		return -EINVAL;
 	}
 
 	if (user_iopoll_weight == ioa_cfg->iopoll_weight) {
-		dev_info(&ioa_cfg->pdev->dev, "Current blk-iopoll weight has the same weight\n");
+		dev_info(&ioa_cfg->pdev->dev, "Current irq_poll weight has the same weight\n");
 		return strlen(buf);
 	}
 
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		for (i = 1; i < ioa_cfg->hrrq_num; i++)
-			blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
+			irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
 	}
 
 	spin_lock_irqsave(shost->host_lock, lock_flags);
 	ioa_cfg->iopoll_weight = user_iopoll_weight;
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		for (i = 1; i < ioa_cfg->hrrq_num; i++) {
-			blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
+			irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
 					ioa_cfg->iopoll_weight, ipr_iopoll);
-			blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll);
 		}
 	}
 	spin_unlock_irqrestore(shost->host_lock, lock_flags);
@@ -4003,13 +4002,12 @@ static ssize_t ipr_store_update_fw(struct device *dev,
 	struct ipr_sglist *sglist;
 	char fname[100];
 	char *src;
-	int len, result, dnld_size;
+	int result, dnld_size;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	len = snprintf(fname, 99, "%s", buf);
-	fname[len-1] = '\0';
+	snprintf(fname, sizeof(fname), "%s", buf);
 
 	if (request_firmware(&fw_entry, fname, &ioa_cfg->pdev->dev)) {
 		dev_err(&ioa_cfg->pdev->dev, "Firmware file %s not found\n", fname);
@@ -5569,7 +5567,7 @@ static int ipr_process_hrrq(struct ipr_hrr_queue *hrr_queue, int budget,
 	return num_hrrq;
 }
 
-static int ipr_iopoll(struct blk_iopoll *iop, int budget)
+static int ipr_iopoll(struct irq_poll *iop, int budget)
 {
 	struct ipr_ioa_cfg *ioa_cfg;
 	struct ipr_hrr_queue *hrrq;
@@ -5585,7 +5583,7 @@ static int ipr_iopoll(struct blk_iopoll *iop, int budget)
 	completed_ops = ipr_process_hrrq(hrrq, budget, &doneq);
 
 	if (completed_ops < budget)
-		blk_iopoll_complete(iop);
+		irq_poll_complete(iop);
 	spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
 
 	list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) {
@@ -5693,8 +5691,7 @@ static irqreturn_t ipr_isr_mhrrq(int irq, void *devp)
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) ==
 		       hrrq->toggle_bit) {
-			if (!blk_iopoll_sched_prep(&hrrq->iopoll))
-				blk_iopoll_sched(&hrrq->iopoll);
+			irq_poll_sched(&hrrq->iopoll);
 			spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
 			return IRQ_HANDLED;
 		}
@@ -10405,9 +10402,8 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
 
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		for (i = 1; i < ioa_cfg->hrrq_num; i++) {
-			blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
+			irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
 					ioa_cfg->iopoll_weight, ipr_iopoll);
-			blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll);
 		}
 	}
 
@@ -10436,7 +10432,7 @@ static void ipr_shutdown(struct pci_dev *pdev)
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		ioa_cfg->iopoll_weight = 0;
 		for (i = 1; i < ioa_cfg->hrrq_num; i++)
-			blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
+			irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
 	}
 
 	while (ioa_cfg->in_reset_reload) {
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index a34c7a5a995e..56c57068300a 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -32,7 +32,7 @@
 #include <linux/libata.h>
 #include <linux/list.h>
 #include <linux/kref.h>
-#include <linux/blk-iopoll.h>
+#include <linux/irq_poll.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
 
@@ -517,7 +517,7 @@ struct ipr_hrr_queue {
 	u8 allow_cmds:1;
 	u8 removing_ioa:1;
 
-	struct blk_iopoll iopoll;
+	struct irq_poll iopoll;
 };
 
 /* Command packet structure */
diff --git a/drivers/scsi/mac_scsi.c b/drivers/scsi/mac_scsi.c
index d64a769b8155..bb2381314a2b 100644
--- a/drivers/scsi/mac_scsi.c
+++ b/drivers/scsi/mac_scsi.c
@@ -12,7 +12,6 @@
  */
 
 #include <linux/types.h>
-#include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/ioport.h>
 #include <linux/init.h>
@@ -32,14 +31,13 @@
 #define PSEUDO_DMA
 
 #define NCR5380_implementation_fields   unsigned char *pdma_base
-#define NCR5380_local_declare()         struct Scsi_Host *_instance
-#define NCR5380_setup(instance)         _instance = instance
 
-#define NCR5380_read(reg)               macscsi_read(_instance, reg)
-#define NCR5380_write(reg, value)       macscsi_write(_instance, reg, value)
+#define NCR5380_read(reg)               macscsi_read(instance, reg)
+#define NCR5380_write(reg, value)       macscsi_write(instance, reg, value)
 
 #define NCR5380_pread                   macscsi_pread
 #define NCR5380_pwrite                  macscsi_pwrite
+#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
 
 #define NCR5380_intr                    macscsi_intr
 #define NCR5380_queue_command           macscsi_queue_command
@@ -51,8 +49,6 @@
 
 #include "NCR5380.h"
 
-#define RESET_BOOT
-
 static int setup_can_queue = -1;
 module_param(setup_can_queue, int, 0);
 static int setup_cmd_per_lun = -1;
@@ -65,17 +61,8 @@ static int setup_use_tagged_queuing = -1;
 module_param(setup_use_tagged_queuing, int, 0);
 static int setup_hostid = -1;
 module_param(setup_hostid, int, 0);
-
-/* Time (in jiffies) to wait after a reset; the SCSI standard calls for 250ms,
- * we usually do 0.5s to be on the safe side. But Toshiba CD-ROMs once more
- * need ten times the standard value... */
-#define TOSHIBA_DELAY
-
-#ifdef TOSHIBA_DELAY
-#define	AFTER_RESET_DELAY	(5*HZ/2)
-#else
-#define	AFTER_RESET_DELAY	(HZ/2)
-#endif
+static int setup_toshiba_delay = -1;
+module_param(setup_toshiba_delay, int, 0);
 
 /*
  * NCR 5380 register access functions
@@ -94,12 +81,12 @@ static inline void macscsi_write(struct Scsi_Host *instance, int reg, int value)
 #ifndef MODULE
 static int __init mac_scsi_setup(char *str)
 {
-	int ints[7];
+	int ints[8];
 
 	(void)get_options(str, ARRAY_SIZE(ints), ints);
 
-	if (ints[0] < 1 || ints[0] > 6) {
-		pr_err("Usage: mac5380=<can_queue>[,<cmd_per_lun>[,<sg_tablesize>[,<hostid>[,<use_tags>[,<use_pdma>]]]]]\n");
+	if (ints[0] < 1) {
+		pr_err("Usage: mac5380=<can_queue>[,<cmd_per_lun>[,<sg_tablesize>[,<hostid>[,<use_tags>[,<use_pdma>[,<toshiba_delay>]]]]]]\n");
 		return 0;
 	}
 	if (ints[0] >= 1)
@@ -114,50 +101,14 @@ static int __init mac_scsi_setup(char *str)
 		setup_use_tagged_queuing = ints[5];
 	if (ints[0] >= 6)
 		setup_use_pdma = ints[6];
+	if (ints[0] >= 7)
+		setup_toshiba_delay = ints[7];
 	return 1;
 }
 
 __setup("mac5380=", mac_scsi_setup);
 #endif /* !MODULE */
 
-#ifdef RESET_BOOT
-/*
- * Our 'bus reset on boot' function
- */
-
-static void mac_scsi_reset_boot(struct Scsi_Host *instance)
-{
-	unsigned long end;
-
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-	
-	/*
-	 * Do a SCSI reset to clean up the bus during initialization. No messing
-	 * with the queues, interrupts, or locks necessary here.
-	 */
-
-	printk(KERN_INFO "Macintosh SCSI: resetting the SCSI bus..." );
-
-	/* get in phase */
-	NCR5380_write( TARGET_COMMAND_REG,
-		      PHASE_SR_TO_TCR( NCR5380_read(STATUS_REG) ));
-
-	/* assert RST */
-	NCR5380_write( INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST );
-	/* The min. reset hold time is 25us, so 40us should be enough */
-	udelay( 50 );
-	/* reset RST and interrupt */
-	NCR5380_write( INITIATOR_COMMAND_REG, ICR_BASE );
-	NCR5380_read( RESET_PARITY_INTERRUPT_REG );
-
-	for( end = jiffies + AFTER_RESET_DELAY; time_before(jiffies, end); )
-		barrier();
-
-	printk(KERN_INFO " done\n" );
-}
-#endif
-
 #ifdef PSEUDO_DMA
 /* 
    Pseudo-DMA: (Ove Edlund)
@@ -235,9 +186,6 @@ static int macscsi_pread(struct Scsi_Host *instance,
 	unsigned char *d;
 	unsigned char *s;
 
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-
 	s = hostdata->pdma_base + (INPUT_DATA_REG << 4);
 	d = dst;
 
@@ -329,9 +277,6 @@ static int macscsi_pwrite(struct Scsi_Host *instance,
 	unsigned char *s;
 	unsigned char *d;
 
-	NCR5380_local_declare();
-	NCR5380_setup(instance);
-
 	s = src;
 	d = hostdata->pdma_base + (OUTPUT_DATA_REG << 4);
 
@@ -364,20 +309,22 @@ static int macscsi_pwrite(struct Scsi_Host *instance,
 #define PFX                     DRV_MODULE_NAME ": "
 
 static struct scsi_host_template mac_scsi_template = {
-	.module				= THIS_MODULE,
-	.proc_name			= DRV_MODULE_NAME,
-	.show_info			= macscsi_show_info,
-	.write_info			= macscsi_write_info,
-	.name				= "Macintosh NCR5380 SCSI",
-	.info				= macscsi_info,
-	.queuecommand			= macscsi_queue_command,
-	.eh_abort_handler		= macscsi_abort,
-	.eh_bus_reset_handler		= macscsi_bus_reset,
-	.can_queue			= 16,
-	.this_id			= 7,
-	.sg_tablesize			= SG_ALL,
-	.cmd_per_lun			= 2,
-	.use_clustering			= DISABLE_CLUSTERING
+	.module			= THIS_MODULE,
+	.proc_name		= DRV_MODULE_NAME,
+	.show_info		= macscsi_show_info,
+	.write_info		= macscsi_write_info,
+	.name			= "Macintosh NCR5380 SCSI",
+	.info			= macscsi_info,
+	.queuecommand		= macscsi_queue_command,
+	.eh_abort_handler	= macscsi_abort,
+	.eh_bus_reset_handler	= macscsi_bus_reset,
+	.can_queue		= 16,
+	.this_id		= 7,
+	.sg_tablesize		= SG_ALL,
+	.cmd_per_lun		= 2,
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 
 static int __init mac_scsi_probe(struct platform_device *pdev)
@@ -432,15 +379,14 @@ static int __init mac_scsi_probe(struct platform_device *pdev)
 	} else
 		host_flags |= FLAG_NO_PSEUDO_DMA;
 
-#ifdef RESET_BOOT
-	mac_scsi_reset_boot(instance);
-#endif
-
 #ifdef SUPPORT_TAGS
 	host_flags |= setup_use_tagged_queuing > 0 ? FLAG_TAGGED_QUEUING : 0;
 #endif
+	host_flags |= setup_toshiba_delay > 0 ? FLAG_TOSHIBA_DELAY : 0;
 
-	NCR5380_init(instance, host_flags);
+	error = NCR5380_init(instance, host_flags);
+	if (error)
+		goto fail_init;
 
 	if (instance->irq != NO_IRQ) {
 		error = request_irq(instance->irq, macscsi_intr, IRQF_SHARED,
@@ -449,6 +395,8 @@ static int __init mac_scsi_probe(struct platform_device *pdev)
 			goto fail_irq;
 	}
 
+	NCR5380_maybe_reset_bus(instance);
+
 	error = scsi_add_host(instance, NULL);
 	if (error)
 		goto fail_host;
@@ -463,6 +411,7 @@ fail_host:
 		free_irq(instance->irq, instance);
 fail_irq:
 	NCR5380_exit(instance);
+fail_init:
 	scsi_host_put(instance);
 	return error;
 }
diff --git a/drivers/scsi/megaraid/megaraid_mm.c b/drivers/scsi/megaraid/megaraid_mm.c
index a70692779a16..4cf9ed96414f 100644
--- a/drivers/scsi/megaraid/megaraid_mm.c
+++ b/drivers/scsi/megaraid/megaraid_mm.c
@@ -179,8 +179,12 @@ mraid_mm_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 
 	/*
 	 * The following call will block till a kioc is available
+	 * or return NULL if the list head is empty for the pointer
+	 * of type mraid_mmapt passed to mraid_mm_alloc_kioc
 	 */
 	kioc = mraid_mm_alloc_kioc(adp);
+	if (!kioc)
+		return -ENXIO;
 
 	/*
 	 * User sent the old mimd_t ioctl packet. Convert it to uioc_t.
diff --git a/drivers/scsi/pas16.c b/drivers/scsi/pas16.c
index e81eadd08afc..512037e27783 100644
--- a/drivers/scsi/pas16.c
+++ b/drivers/scsi/pas16.c
@@ -1,6 +1,4 @@
 #define PSEUDO_DMA
-#define UNSAFE  /* Not unsafe for PAS16 -- use it */
-#define PDEBUG 0
 
 /*
  * This driver adapted from Drew Eckhardt's Trantor T128 driver
@@ -71,14 +69,10 @@
  
 #include <linux/module.h>
 
-#include <linux/signal.h>
-#include <linux/proc_fs.h>
 #include <asm/io.h>
 #include <asm/dma.h>
 #include <linux/blkdev.h>
-#include <linux/delay.h>
 #include <linux/interrupt.h>
-#include <linux/stat.h>
 #include <linux/init.h>
 
 #include <scsi/scsi_host.h>
@@ -87,8 +81,8 @@
 #include "NCR5380.h"
 
 
-static unsigned short pas16_addr = 0;
-static int pas16_irq = 0;
+static unsigned short pas16_addr;
+static int pas16_irq;
  
 
 static const int scsi_irq_translate[] =
@@ -146,22 +140,6 @@ static const unsigned short  pas16_offset[ 8 ] =
 		    * START_DMA_INITIATOR_RECEIVE_REG wo
 		    */
     };
-/*----------------------------------------------------------------*/
-/* the following will set the monitor border color (useful to find
- where something crashed or gets stuck at */
-/* 1 = blue
- 2 = green
- 3 = cyan
- 4 = red
- 5 = magenta
- 6 = yellow
- 7 = white
-*/
-#if 1
-#define rtrc(i) {inb(0x3da); outb(0x31, 0x3c0); outb((i), 0x3c0);}
-#else
-#define rtrc(i) {}
-#endif
 
 
 /*
@@ -205,7 +183,7 @@ static void __init
 	outb( 0x01, io_port + P_TIMEOUT_STATUS_REG_OFFSET );   /* Reset TC */
 	outb( 0x01, io_port + WAIT_STATE );   /* 1 Wait state */
 
-	NCR5380_read( RESET_PARITY_INTERRUPT_REG );
+	inb(io_port + pas16_offset[RESET_PARITY_INTERRUPT_REG]);
 
 	/* Set the SCSI interrupt pointer without mucking up the sound
 	 * interrupt pointer in the same byte.
@@ -280,13 +258,13 @@ static int __init
      * put in an additional test to try to weed them out.
      */
 
-    outb( 0x01, io_port + WAIT_STATE ); 	/* 1 Wait state */
-    NCR5380_write( MODE_REG, 0x20 );		/* Is it really SCSI? */
-    if( NCR5380_read( MODE_REG ) != 0x20 )	/* Write to a reg.    */
-	return 0;				/* and try to read    */
-    NCR5380_write( MODE_REG, 0x00 );		/* it back.	      */
-    if( NCR5380_read( MODE_REG ) != 0x00 )
-	return 0;
+	outb(0x01, io_port + WAIT_STATE);             /* 1 Wait state */
+	outb(0x20, io_port + pas16_offset[MODE_REG]); /* Is it really SCSI? */
+	if (inb(io_port + pas16_offset[MODE_REG]) != 0x20) /* Write to a reg. */
+		return 0;                                  /* and try to read */
+	outb(0x00, io_port + pas16_offset[MODE_REG]);      /* it back. */
+	if (inb(io_port + pas16_offset[MODE_REG]) != 0x00)
+		return 0;
 
     return 1;
 }
@@ -305,7 +283,7 @@ static int __init
 
 static int __init pas16_setup(char *str)
 {
-    static int commandline_current = 0;
+	static int commandline_current;
     int i;
     int ints[10];
 
@@ -344,8 +322,8 @@ __setup("pas16=", pas16_setup);
 
 static int __init pas16_detect(struct scsi_host_template *tpnt)
 {
-    static int current_override = 0;
-    static unsigned short current_base = 0;
+	static int current_override;
+	static unsigned short current_base;
     struct Scsi_Host *instance;
     unsigned short io_port;
     int  count;
@@ -377,34 +355,32 @@ static int __init pas16_detect(struct scsi_host_template *tpnt)
 	}
 	else
 	    for (; !io_port && (current_base < NO_BASES); ++current_base) {
-#if (PDEBUG & PDEBUG_INIT)
-    printk("scsi-pas16 : probing io_port %04x\n", (unsigned int) bases[current_base].io_port);
-#endif
+		dprintk(NDEBUG_INIT, "pas16: probing io_port 0x%04x\n",
+		        (unsigned int)bases[current_base].io_port);
 		if ( !bases[current_base].noauto &&
 		     pas16_hw_detect( current_base ) ){
 			io_port = bases[current_base].io_port;
 			init_board( io_port, default_irqs[ current_base ], 0 ); 
-#if (PDEBUG & PDEBUG_INIT)
-			printk("scsi-pas16 : detected board.\n");
-#endif
+			dprintk(NDEBUG_INIT, "pas16: detected board\n");
 		}
     }
 
-
-#if defined(PDEBUG) && (PDEBUG & PDEBUG_INIT)
-	printk("scsi-pas16 : io_port = %04x\n", (unsigned int) io_port);
-#endif
+	dprintk(NDEBUG_INIT, "pas16: io_port = 0x%04x\n",
+	        (unsigned int)io_port);
 
 	if (!io_port)
 	    break;
 
 	instance = scsi_register (tpnt, sizeof(struct NCR5380_hostdata));
 	if(instance == NULL)
-		break;
+		goto out;
 		
 	instance->io_port = io_port;
 
-	NCR5380_init(instance, 0);
+	if (NCR5380_init(instance, 0))
+		goto out_unregister;
+
+	NCR5380_maybe_reset_bus(instance);
 
 	if (overrides[current_override].irq != IRQ_AUTO)
 	    instance->irq = overrides[current_override].irq;
@@ -431,14 +407,18 @@ static int __init pas16_detect(struct scsi_host_template *tpnt)
 	    outb( (inb(io_port + IO_CONFIG_3) & 0x0f), io_port + IO_CONFIG_3 );
 	}
 
-#if defined(PDEBUG) && (PDEBUG & PDEBUG_INIT)
-	printk("scsi%d : irq = %d\n", instance->host_no, instance->irq);
-#endif
+	dprintk(NDEBUG_INIT, "scsi%d : irq = %d\n",
+	        instance->host_no, instance->irq);
 
 	++current_override;
 	++count;
     }
     return count;
+
+out_unregister:
+	scsi_unregister(instance);
+out:
+	return count;
 }
 
 /*
@@ -561,29 +541,29 @@ static int pas16_release(struct Scsi_Host *shost)
 	if (shost->irq != NO_IRQ)
 		free_irq(shost->irq, shost);
 	NCR5380_exit(shost);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
 	scsi_unregister(shost);
 	return 0;
 }
 
 static struct scsi_host_template driver_template = {
-	.name           = "Pro Audio Spectrum-16 SCSI",
-	.detect         = pas16_detect,
-	.release        = pas16_release,
-	.proc_name      = "pas16",
-	.show_info      = pas16_show_info,
-	.write_info     = pas16_write_info,
-	.info           = pas16_info,
-	.queuecommand   = pas16_queue_command,
-	.eh_abort_handler = pas16_abort,
-	.eh_bus_reset_handler = pas16_bus_reset,
-	.bios_param     = pas16_biosparam, 
-	.can_queue      = CAN_QUEUE,
-	.this_id        = 7,
-	.sg_tablesize   = SG_ALL,
-	.cmd_per_lun    = CMD_PER_LUN,
-	.use_clustering = DISABLE_CLUSTERING,
+	.name			= "Pro Audio Spectrum-16 SCSI",
+	.detect			= pas16_detect,
+	.release		= pas16_release,
+	.proc_name		= "pas16",
+	.show_info		= pas16_show_info,
+	.write_info		= pas16_write_info,
+	.info			= pas16_info,
+	.queuecommand		= pas16_queue_command,
+	.eh_abort_handler	= pas16_abort,
+	.eh_bus_reset_handler	= pas16_bus_reset,
+	.bios_param		= pas16_biosparam,
+	.can_queue		= 32,
+	.this_id		= 7,
+	.sg_tablesize		= SG_ALL,
+	.cmd_per_lun		= 2,
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 #include "scsi_module.c"
 
diff --git a/drivers/scsi/pas16.h b/drivers/scsi/pas16.h
index c6109c80050b..d37527717225 100644
--- a/drivers/scsi/pas16.h
+++ b/drivers/scsi/pas16.h
@@ -24,9 +24,6 @@
 #ifndef PAS16_H
 #define PAS16_H
 
-#define PDEBUG_INIT	0x1
-#define PDEBUG_TRANSFER 0x2
-
 #define PAS16_DEFAULT_BASE_1  0x388
 #define PAS16_DEFAULT_BASE_2  0x384
 #define PAS16_DEFAULT_BASE_3  0x38c
@@ -98,46 +95,16 @@
 #define OPERATION_MODE_1 0xec03
 #define IO_CONFIG_3 0xf002
 
+#define NCR5380_implementation_fields /* none */
 
-#ifndef ASM
-
-#ifndef CMD_PER_LUN
-#define CMD_PER_LUN 2
-#endif
-
-#ifndef CAN_QUEUE
-#define CAN_QUEUE 32 
-#endif
-
-#define NCR5380_implementation_fields \
-    volatile unsigned short io_port
-
-#define NCR5380_local_declare() \
-    volatile unsigned short io_port
+#define PAS16_io_port(reg) (instance->io_port + pas16_offset[(reg)])
 
-#define NCR5380_setup(instance) \
-    io_port = (instance)->io_port
-
-#define PAS16_io_port(reg) ( io_port + pas16_offset[(reg)] )
-
-#if !(PDEBUG & PDEBUG_TRANSFER) 
 #define NCR5380_read(reg) ( inb(PAS16_io_port(reg)) )
 #define NCR5380_write(reg, value) ( outb((value),PAS16_io_port(reg)) )
-#else
-#define NCR5380_read(reg)						\
-    (((unsigned char) printk("scsi%d : read register %d at io_port %04x\n"\
-    , instance->hostno, (reg), PAS16_io_port(reg))), inb( PAS16_io_port(reg)) )
-
-#define NCR5380_write(reg, value) 					\
-    (printk("scsi%d : write %02x to register %d at io_port %04x\n", 	\
-	    instance->hostno, (value), (reg), PAS16_io_port(reg)),	\
-    outb( (value),PAS16_io_port(reg) ) )
-
-#endif
 
+#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
 
 #define NCR5380_intr pas16_intr
-#define do_NCR5380_intr do_pas16_intr
 #define NCR5380_queue_command pas16_queue_command
 #define NCR5380_abort pas16_abort
 #define NCR5380_bus_reset pas16_bus_reset
@@ -150,5 +117,4 @@
    
 #define PAS16_IRQS 0xd4a8 
 
-#endif /* ndef ASM */
 #endif /* PAS16_H */
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index 6b942d9e5b74..6992ebc50c87 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -824,6 +824,41 @@ static struct bin_attribute sysfs_reset_attr = {
 };
 
 static ssize_t
+qla2x00_issue_logo(struct file *filp, struct kobject *kobj,
+			struct bin_attribute *bin_attr,
+			char *buf, loff_t off, size_t count)
+{
+	struct scsi_qla_host *vha = shost_priv(dev_to_shost(container_of(kobj,
+	    struct device, kobj)));
+	int type;
+	int rval = 0;
+	port_id_t did;
+
+	type = simple_strtol(buf, NULL, 10);
+
+	did.b.domain = (type & 0x00ff0000) >> 16;
+	did.b.area = (type & 0x0000ff00) >> 8;
+	did.b.al_pa = (type & 0x000000ff);
+
+	ql_log(ql_log_info, vha, 0x70e3, "portid=%02x%02x%02x done\n",
+	    did.b.domain, did.b.area, did.b.al_pa);
+
+	ql_log(ql_log_info, vha, 0x70e4, "%s: %d\n", __func__, type);
+
+	rval = qla24xx_els_dcmd_iocb(vha, ELS_DCMD_LOGO, did);
+	return count;
+}
+
+static struct bin_attribute sysfs_issue_logo_attr = {
+	.attr = {
+		.name = "issue_logo",
+		.mode = S_IWUSR,
+	},
+	.size = 0,
+	.write = qla2x00_issue_logo,
+};
+
+static ssize_t
 qla2x00_sysfs_read_xgmac_stats(struct file *filp, struct kobject *kobj,
 		       struct bin_attribute *bin_attr,
 		       char *buf, loff_t off, size_t count)
@@ -937,6 +972,7 @@ static struct sysfs_entry {
 	{ "vpd", &sysfs_vpd_attr, 1 },
 	{ "sfp", &sysfs_sfp_attr, 1 },
 	{ "reset", &sysfs_reset_attr, },
+	{ "issue_logo", &sysfs_issue_logo_attr, },
 	{ "xgmac_stats", &sysfs_xgmac_stats_attr, 3 },
 	{ "dcbx_tlv", &sysfs_dcbx_tlv_attr, 3 },
 	{ NULL },
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index 34dc9a35670b..cd0d94ea7f74 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -14,25 +14,24 @@
  * | Module Init and Probe        |       0x017f       | 0x0146         |
  * |                              |                    | 0x015b-0x0160	|
  * |                              |                    | 0x016e-0x0170  |
- * | Mailbox commands             |       0x118d       | 0x1115-0x1116	|
- * |                              |                    | 0x111a-0x111b  |
+ * | Mailbox commands             |       0x1192       |		|
+ * |                              |                    |		|
  * | Device Discovery             |       0x2016       | 0x2020-0x2022, |
  * |                              |                    | 0x2011-0x2012, |
  * |                              |                    | 0x2099-0x20a4  |
- * | Queue Command and IO tracing |       0x3075       | 0x300b         |
+ * | Queue Command and IO tracing |       0x3074       | 0x300b         |
  * |                              |                    | 0x3027-0x3028  |
  * |                              |                    | 0x303d-0x3041  |
  * |                              |                    | 0x302d,0x3033  |
  * |                              |                    | 0x3036,0x3038  |
  * |                              |                    | 0x303a		|
  * | DPC Thread                   |       0x4023       | 0x4002,0x4013  |
- * | Async Events                 |       0x508a       | 0x502b-0x502f  |
- * |                              |                    | 0x5047		|
+ * | Async Events                 |       0x5089       | 0x502b-0x502f  |
  * |                              |                    | 0x5084,0x5075	|
  * |                              |                    | 0x503d,0x5044  |
  * |                              |                    | 0x507b,0x505f	|
  * | Timer Routines               |       0x6012       |                |
- * | User Space Interactions      |       0x70e2       | 0x7018,0x702e  |
+ * | User Space Interactions      |       0x70e65      | 0x7018,0x702e  |
  * |				  |		       | 0x7020,0x7024  |
  * |                              |                    | 0x7039,0x7045  |
  * |                              |                    | 0x7073-0x7075  |
@@ -60,15 +59,11 @@
  * |                              |                    | 0xb13c-0xb140  |
  * |                              |                    | 0xb149		|
  * | MultiQ                       |       0xc00c       |		|
- * | Misc                         |       0xd300       | 0xd016-0xd017	|
- * |                              |                    | 0xd021,0xd024	|
- * |                              |                    | 0xd025,0xd029	|
- * |                              |                    | 0xd02a,0xd02e	|
- * |                              |                    | 0xd031-0xd0ff	|
+ * | Misc                         |       0xd301       | 0xd031-0xd0ff	|
  * |                              |                    | 0xd101-0xd1fe	|
  * |                              |                    | 0xd214-0xd2fe	|
  * | Target Mode		  |	  0xe080       |		|
- * | Target Mode Management	  |	  0xf096       | 0xf002		|
+ * | Target Mode Management	  |	  0xf09b       | 0xf002		|
  * |                              |                    | 0xf046-0xf049  |
  * | Target Mode Task Management  |	  0x1000d      |		|
  * ----------------------------------------------------------------------
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 388d79088b59..9872f3429e53 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -259,7 +259,7 @@
 #define LOOP_DOWN_TIME			255	/* 240 */
 #define	LOOP_DOWN_RESET			(LOOP_DOWN_TIME - 30)
 
-#define DEFAULT_OUTSTANDING_COMMANDS	1024
+#define DEFAULT_OUTSTANDING_COMMANDS	4096
 #define MIN_OUTSTANDING_COMMANDS	128
 
 /* ISP request and response entry counts (37-65535) */
@@ -267,11 +267,13 @@
 #define REQUEST_ENTRY_CNT_2200		2048	/* Number of request entries. */
 #define REQUEST_ENTRY_CNT_24XX		2048	/* Number of request entries. */
 #define REQUEST_ENTRY_CNT_83XX		8192	/* Number of request entries. */
+#define RESPONSE_ENTRY_CNT_83XX		4096	/* Number of response entries.*/
 #define RESPONSE_ENTRY_CNT_2100		64	/* Number of response entries.*/
 #define RESPONSE_ENTRY_CNT_2300		512	/* Number of response entries.*/
 #define RESPONSE_ENTRY_CNT_MQ		128	/* Number of response entries.*/
 #define ATIO_ENTRY_CNT_24XX		4096	/* Number of ATIO entries. */
 #define RESPONSE_ENTRY_CNT_FX00		256     /* Number of response entries.*/
+#define EXTENDED_EXCH_ENTRY_CNT		32768   /* Entries for offload case */
 
 struct req_que;
 struct qla_tgt_sess;
@@ -309,6 +311,14 @@ struct srb_cmd {
 /* To identify if a srb is of T10-CRC type. @sp => srb_t pointer */
 #define IS_PROT_IO(sp)	(sp->flags & SRB_CRC_CTX_DSD_VALID)
 
+struct els_logo_payload {
+	uint8_t opcode;
+	uint8_t rsvd[3];
+	uint8_t s_id[3];
+	uint8_t rsvd1[1];
+	uint8_t wwpn[WWN_SIZE];
+};
+
 /*
  * SRB extensions.
  */
@@ -322,6 +332,15 @@ struct srb_iocb {
 			uint16_t data[2];
 		} logio;
 		struct {
+#define ELS_DCMD_TIMEOUT 20
+#define ELS_DCMD_LOGO 0x5
+			uint32_t flags;
+			uint32_t els_cmd;
+			struct completion comp;
+			struct els_logo_payload *els_logo_pyld;
+			dma_addr_t els_logo_pyld_dma;
+		} els_logo;
+		struct {
 			/*
 			 * Values for flags field below are as
 			 * defined in tsk_mgmt_entry struct
@@ -382,7 +401,7 @@ struct srb_iocb {
 #define SRB_FXIOCB_DCMD	10
 #define SRB_FXIOCB_BCMD	11
 #define SRB_ABT_CMD	12
-
+#define SRB_ELS_DCMD	13
 
 typedef struct srb {
 	atomic_t ref_count;
@@ -891,6 +910,7 @@ struct mbx_cmd_32 {
 #define MBC_DISABLE_VI			0x24	/* Disable VI operation. */
 #define MBC_ENABLE_VI			0x25	/* Enable VI operation. */
 #define MBC_GET_FIRMWARE_OPTION		0x28	/* Get Firmware Options. */
+#define MBC_GET_MEM_OFFLOAD_CNTRL_STAT	0x34	/* Memory Offload ctrl/Stat*/
 #define MBC_SET_FIRMWARE_OPTION		0x38	/* Set Firmware Options. */
 #define MBC_LOOP_PORT_BYPASS		0x40	/* Loop Port Bypass. */
 #define MBC_LOOP_PORT_ENABLE		0x41	/* Loop Port Enable. */
@@ -2695,11 +2715,16 @@ struct isp_operations {
 
 struct scsi_qla_host;
 
+
+#define QLA83XX_RSPQ_MSIX_ENTRY_NUMBER 1 /* refer to qla83xx_msix_entries */
+
 struct qla_msix_entry {
 	int have_irq;
 	uint32_t vector;
 	uint16_t entry;
 	struct rsp_que *rsp;
+	struct irq_affinity_notify irq_notify;
+	int cpuid;
 };
 
 #define	WATCH_INTERVAL		1       /* number of seconds */
@@ -2910,12 +2935,15 @@ struct qlt_hw_data {
 	uint32_t num_qfull_cmds_dropped;
 	spinlock_t q_full_lock;
 	uint32_t leak_exchg_thresh_hold;
+	spinlock_t sess_lock;
+	int rspq_vector_cpuid;
+	spinlock_t atio_lock ____cacheline_aligned;
 };
 
 #define MAX_QFULL_CMDS_ALLOC	8192
 #define Q_FULL_THRESH_HOLD_PERCENT 90
 #define Q_FULL_THRESH_HOLD(ha) \
-	((ha->fw_xcb_count/100) * Q_FULL_THRESH_HOLD_PERCENT)
+	((ha->cur_fw_xcb_count/100) * Q_FULL_THRESH_HOLD_PERCENT)
 
 #define LEAK_EXCHG_THRESH_HOLD_PERCENT 75	/* 75 percent */
 
@@ -2962,10 +2990,12 @@ struct qla_hw_data {
 		uint32_t	isp82xx_no_md_cap:1;
 		uint32_t	host_shutting_down:1;
 		uint32_t	idc_compl_status:1;
-
 		uint32_t        mr_reset_hdlr_active:1;
 		uint32_t        mr_intr_valid:1;
+
 		uint32_t	fawwpn_enabled:1;
+		uint32_t	exlogins_enabled:1;
+		uint32_t	exchoffld_enabled:1;
 		/* 35 bits */
 	} flags;
 
@@ -3237,6 +3267,21 @@ struct qla_hw_data {
 	void		*async_pd;
 	dma_addr_t	async_pd_dma;
 
+#define ENABLE_EXTENDED_LOGIN	BIT_7
+
+	/* Extended Logins  */
+	void		*exlogin_buf;
+	dma_addr_t	exlogin_buf_dma;
+	int		exlogin_size;
+
+#define ENABLE_EXCHANGE_OFFLD	BIT_2
+
+	/* Exchange Offload */
+	void		*exchoffld_buf;
+	dma_addr_t	exchoffld_buf_dma;
+	int		exchoffld_size;
+	int 		exchoffld_count;
+
 	void		*swl;
 
 	/* These are used by mailbox operations. */
@@ -3279,8 +3324,14 @@ struct qla_hw_data {
 #define RISC_START_ADDRESS_2100 0x1000
 #define RISC_START_ADDRESS_2300 0x800
 #define RISC_START_ADDRESS_2400 0x100000
-	uint16_t	fw_xcb_count;
-	uint16_t	fw_iocb_count;
+
+	uint16_t	orig_fw_tgt_xcb_count;
+	uint16_t	cur_fw_tgt_xcb_count;
+	uint16_t	orig_fw_xcb_count;
+	uint16_t	cur_fw_xcb_count;
+	uint16_t	orig_fw_iocb_count;
+	uint16_t	cur_fw_iocb_count;
+	uint16_t	fw_max_fcf_count;
 
 	uint32_t	fw_shared_ram_start;
 	uint32_t	fw_shared_ram_end;
@@ -3323,6 +3374,9 @@ struct qla_hw_data {
 	uint32_t	chain_offset;
 	struct dentry *dfs_dir;
 	struct dentry *dfs_fce;
+	struct dentry *dfs_tgt_counters;
+	struct dentry *dfs_fw_resource_cnt;
+
 	dma_addr_t	fce_dma;
 	void		*fce;
 	uint32_t	fce_bufs;
@@ -3480,6 +3534,18 @@ struct qla_hw_data {
 	int	allow_cna_fw_dump;
 };
 
+struct qla_tgt_counters {
+	uint64_t qla_core_sbt_cmd;
+	uint64_t core_qla_que_buf;
+	uint64_t qla_core_ret_ctio;
+	uint64_t core_qla_snd_status;
+	uint64_t qla_core_ret_sta_ctio;
+	uint64_t core_qla_free_cmd;
+	uint64_t num_q_full_sent;
+	uint64_t num_alloc_iocb_failed;
+	uint64_t num_term_xchg_sent;
+};
+
 /*
  * Qlogic scsi host structure
  */
@@ -3595,6 +3661,10 @@ typedef struct scsi_qla_host {
 	atomic_t		generation_tick;
 	/* Time when global fcport update has been scheduled */
 	int			total_fcport_update_gen;
+	/* List of pending LOGOs, protected by tgt_mutex */
+	struct list_head	logo_list;
+	/* List of pending PLOGI acks, protected by hw lock */
+	struct list_head	plogi_ack_list;
 
 	uint32_t	vp_abort_cnt;
 
@@ -3632,6 +3702,7 @@ typedef struct scsi_qla_host {
 
 	atomic_t	vref_count;
 	struct qla8044_reset_template reset_tmplt;
+	struct qla_tgt_counters tgt_counters;
 } scsi_qla_host_t;
 
 #define SET_VP_IDX	1
diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c
index 15cf074ffa3c..cd8b96a4b0dd 100644
--- a/drivers/scsi/qla2xxx/qla_dfs.c
+++ b/drivers/scsi/qla2xxx/qla_dfs.c
@@ -13,6 +13,85 @@ static struct dentry *qla2x00_dfs_root;
 static atomic_t qla2x00_dfs_root_count;
 
 static int
+qla_dfs_fw_resource_cnt_show(struct seq_file *s, void *unused)
+{
+	struct scsi_qla_host *vha = s->private;
+	struct qla_hw_data *ha = vha->hw;
+
+	seq_puts(s, "FW Resource count\n\n");
+	seq_printf(s, "Original TGT exchg count[%d]\n",
+	    ha->orig_fw_tgt_xcb_count);
+	seq_printf(s, "current TGT exchg count[%d]\n",
+	    ha->cur_fw_tgt_xcb_count);
+	seq_printf(s, "original Initiator Exchange count[%d]\n",
+	    ha->orig_fw_xcb_count);
+	seq_printf(s, "Current Initiator Exchange count[%d]\n",
+	    ha->cur_fw_xcb_count);
+	seq_printf(s, "Original IOCB count[%d]\n", ha->orig_fw_iocb_count);
+	seq_printf(s, "Current IOCB count[%d]\n", ha->cur_fw_iocb_count);
+	seq_printf(s, "MAX VP count[%d]\n", ha->max_npiv_vports);
+	seq_printf(s, "MAX FCF count[%d]\n", ha->fw_max_fcf_count);
+
+	return 0;
+}
+
+static int
+qla_dfs_fw_resource_cnt_open(struct inode *inode, struct file *file)
+{
+	struct scsi_qla_host *vha = inode->i_private;
+	return single_open(file, qla_dfs_fw_resource_cnt_show, vha);
+}
+
+static const struct file_operations dfs_fw_resource_cnt_ops = {
+	.open           = qla_dfs_fw_resource_cnt_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static int
+qla_dfs_tgt_counters_show(struct seq_file *s, void *unused)
+{
+	struct scsi_qla_host *vha = s->private;
+
+	seq_puts(s, "Target Counters\n");
+	seq_printf(s, "qla_core_sbt_cmd = %lld\n",
+		vha->tgt_counters.qla_core_sbt_cmd);
+	seq_printf(s, "qla_core_ret_sta_ctio = %lld\n",
+		vha->tgt_counters.qla_core_ret_sta_ctio);
+	seq_printf(s, "qla_core_ret_ctio = %lld\n",
+		vha->tgt_counters.qla_core_ret_ctio);
+	seq_printf(s, "core_qla_que_buf = %lld\n",
+		vha->tgt_counters.core_qla_que_buf);
+	seq_printf(s, "core_qla_snd_status = %lld\n",
+		vha->tgt_counters.core_qla_snd_status);
+	seq_printf(s, "core_qla_free_cmd = %lld\n",
+		vha->tgt_counters.core_qla_free_cmd);
+	seq_printf(s, "num alloc iocb failed = %lld\n",
+		vha->tgt_counters.num_alloc_iocb_failed);
+	seq_printf(s, "num term exchange sent = %lld\n",
+		vha->tgt_counters.num_term_xchg_sent);
+	seq_printf(s, "num Q full sent = %lld\n",
+		vha->tgt_counters.num_q_full_sent);
+
+	return 0;
+}
+
+static int
+qla_dfs_tgt_counters_open(struct inode *inode, struct file *file)
+{
+	struct scsi_qla_host *vha = inode->i_private;
+	return single_open(file, qla_dfs_tgt_counters_show, vha);
+}
+
+static const struct file_operations dfs_tgt_counters_ops = {
+	.open           = qla_dfs_tgt_counters_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static int
 qla2x00_dfs_fce_show(struct seq_file *s, void *unused)
 {
 	scsi_qla_host_t *vha = s->private;
@@ -146,6 +225,22 @@ create_dir:
 	atomic_inc(&qla2x00_dfs_root_count);
 
 create_nodes:
+	ha->dfs_fw_resource_cnt = debugfs_create_file("fw_resource_count",
+	    S_IRUSR, ha->dfs_dir, vha, &dfs_fw_resource_cnt_ops);
+	if (!ha->dfs_fw_resource_cnt) {
+		ql_log(ql_log_warn, vha, 0x00fd,
+		    "Unable to create debugFS fw_resource_count node.\n");
+		goto out;
+	}
+
+	ha->dfs_tgt_counters = debugfs_create_file("tgt_counters", S_IRUSR,
+	    ha->dfs_dir, vha, &dfs_tgt_counters_ops);
+	if (!ha->dfs_tgt_counters) {
+		ql_log(ql_log_warn, vha, 0xd301,
+		    "Unable to create debugFS tgt_counters node.\n");
+		goto out;
+	}
+
 	ha->dfs_fce = debugfs_create_file("fce", S_IRUSR, ha->dfs_dir, vha,
 	    &dfs_fce_ops);
 	if (!ha->dfs_fce) {
@@ -161,6 +256,17 @@ int
 qla2x00_dfs_remove(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
+
+	if (ha->dfs_fw_resource_cnt) {
+		debugfs_remove(ha->dfs_fw_resource_cnt);
+		ha->dfs_fw_resource_cnt = NULL;
+	}
+
+	if (ha->dfs_tgt_counters) {
+		debugfs_remove(ha->dfs_tgt_counters);
+		ha->dfs_tgt_counters = NULL;
+	}
+
 	if (ha->dfs_fce) {
 		debugfs_remove(ha->dfs_fce);
 		ha->dfs_fce = NULL;
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 7686bfe9a4a9..0103e468e357 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -44,6 +44,8 @@ extern int qla2x00_find_new_loop_id(scsi_qla_host_t *, fc_port_t *);
 extern int qla2x00_fabric_login(scsi_qla_host_t *, fc_port_t *, uint16_t *);
 extern int qla2x00_local_device_login(scsi_qla_host_t *, fc_port_t *);
 
+extern int qla24xx_els_dcmd_iocb(scsi_qla_host_t *, int, port_id_t);
+
 extern void qla2x00_update_fcports(scsi_qla_host_t *);
 
 extern int qla2x00_abort_isp(scsi_qla_host_t *);
@@ -117,6 +119,8 @@ extern int ql2xdontresethba;
 extern uint64_t ql2xmaxlun;
 extern int ql2xmdcapmask;
 extern int ql2xmdenable;
+extern int ql2xexlogins;
+extern int ql2xexchoffld;
 
 extern int qla2x00_loop_reset(scsi_qla_host_t *);
 extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);
@@ -135,6 +139,10 @@ extern int qla2x00_post_async_adisc_work(struct scsi_qla_host *, fc_port_t *,
     uint16_t *);
 extern int qla2x00_post_async_adisc_done_work(struct scsi_qla_host *,
     fc_port_t *, uint16_t *);
+extern int qla2x00_set_exlogins_buffer(struct scsi_qla_host *);
+extern void qla2x00_free_exlogin_buffer(struct qla_hw_data *);
+extern int qla2x00_set_exchoffld_buffer(struct scsi_qla_host *);
+extern void qla2x00_free_exchoffld_buffer(struct qla_hw_data *);
 
 extern int qla81xx_restart_mpi_firmware(scsi_qla_host_t *);
 
@@ -323,8 +331,7 @@ extern int
 qla2x00_get_id_list(scsi_qla_host_t *, void *, dma_addr_t, uint16_t *);
 
 extern int
-qla2x00_get_resource_cnts(scsi_qla_host_t *, uint16_t *, uint16_t *,
-    uint16_t *, uint16_t *, uint16_t *, uint16_t *);
+qla2x00_get_resource_cnts(scsi_qla_host_t *);
 
 extern int
 qla2x00_get_fcal_position_map(scsi_qla_host_t *ha, char *pos_map);
@@ -766,4 +773,11 @@ extern int qla8044_abort_isp(scsi_qla_host_t *);
 extern int qla8044_check_fw_alive(struct scsi_qla_host *);
 
 extern void qlt_host_reset_handler(struct qla_hw_data *ha);
+extern int qla_get_exlogin_status(scsi_qla_host_t *, uint16_t *,
+	uint16_t *);
+extern int qla_set_exlogin_mem_cfg(scsi_qla_host_t *vha, dma_addr_t phys_addr);
+extern int qla_get_exchoffld_status(scsi_qla_host_t *, uint16_t *, uint16_t *);
+extern int qla_set_exchoffld_mem_cfg(scsi_qla_host_t *, dma_addr_t);
+extern void qlt_handle_abts_recv(struct scsi_qla_host *, response_t *);
+
 #endif /* _QLA_GBL_H */
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 16a1935cc9c1..52a87657c7dd 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1766,10 +1766,10 @@ qla2x00_alloc_outstanding_cmds(struct qla_hw_data *ha, struct req_que *req)
 	    (ql2xmultique_tag || ql2xmaxqueues > 1)))
 		req->num_outstanding_cmds = DEFAULT_OUTSTANDING_COMMANDS;
 	else {
-		if (ha->fw_xcb_count <= ha->fw_iocb_count)
-			req->num_outstanding_cmds = ha->fw_xcb_count;
+		if (ha->cur_fw_xcb_count <= ha->cur_fw_iocb_count)
+			req->num_outstanding_cmds = ha->cur_fw_xcb_count;
 		else
-			req->num_outstanding_cmds = ha->fw_iocb_count;
+			req->num_outstanding_cmds = ha->cur_fw_iocb_count;
 	}
 
 	req->outstanding_cmds = kzalloc(sizeof(srb_t *) *
@@ -1843,9 +1843,23 @@ qla2x00_setup_chip(scsi_qla_host_t *vha)
 			ql_dbg(ql_dbg_init, vha, 0x00ca,
 			    "Starting firmware.\n");
 
+			if (ql2xexlogins)
+				ha->flags.exlogins_enabled = 1;
+
+			if (ql2xexchoffld)
+				ha->flags.exchoffld_enabled = 1;
+
 			rval = qla2x00_execute_fw(vha, srisc_address);
 			/* Retrieve firmware information. */
 			if (rval == QLA_SUCCESS) {
+				rval = qla2x00_set_exlogins_buffer(vha);
+				if (rval != QLA_SUCCESS)
+					goto failed;
+
+				rval = qla2x00_set_exchoffld_buffer(vha);
+				if (rval != QLA_SUCCESS)
+					goto failed;
+
 enable_82xx_npiv:
 				fw_major_version = ha->fw_major_version;
 				if (IS_P3P_TYPE(ha))
@@ -1864,9 +1878,7 @@ enable_82xx_npiv:
 						ha->max_npiv_vports =
 						    MIN_MULTI_ID_FABRIC - 1;
 				}
-				qla2x00_get_resource_cnts(vha, NULL,
-				    &ha->fw_xcb_count, NULL, &ha->fw_iocb_count,
-				    &ha->max_npiv_vports, NULL);
+				qla2x00_get_resource_cnts(vha);
 
 				/*
 				 * Allocate the array of outstanding commands
@@ -2248,7 +2260,7 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
 	if (IS_FWI2_CAPABLE(ha)) {
 		mid_init_cb->options = cpu_to_le16(BIT_1);
 		mid_init_cb->init_cb.execution_throttle =
-		    cpu_to_le16(ha->fw_xcb_count);
+		    cpu_to_le16(ha->cur_fw_xcb_count);
 		/* D-Port Status */
 		if (IS_DPORT_CAPABLE(ha))
 			mid_init_cb->init_cb.firmware_options_1 |=
@@ -3053,6 +3065,26 @@ qla2x00_configure_loop(scsi_qla_host_t *vha)
 			atomic_set(&vha->loop_state, LOOP_READY);
 			ql_dbg(ql_dbg_disc, vha, 0x2069,
 			    "LOOP READY.\n");
+
+			/*
+			 * Process any ATIO queue entries that came in
+			 * while we weren't online.
+			 */
+			if (qla_tgt_mode_enabled(vha)) {
+				if (IS_QLA27XX(ha) || IS_QLA83XX(ha)) {
+					spin_lock_irqsave(&ha->tgt.atio_lock,
+					    flags);
+					qlt_24xx_process_atio_queue(vha, 0);
+					spin_unlock_irqrestore(
+					    &ha->tgt.atio_lock, flags);
+				} else {
+					spin_lock_irqsave(&ha->hardware_lock,
+					    flags);
+					qlt_24xx_process_atio_queue(vha, 1);
+					spin_unlock_irqrestore(
+					    &ha->hardware_lock, flags);
+				}
+			}
 		}
 	}
 
@@ -4907,7 +4939,6 @@ qla2x00_restart_isp(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 	struct rsp_que *rsp = ha->rsp_q_map[0];
-	unsigned long flags;
 
 	/* If firmware needs to be loaded */
 	if (qla2x00_isp_firmware(vha)) {
@@ -4929,17 +4960,6 @@ qla2x00_restart_isp(scsi_qla_host_t *vha)
 			/* Issue a marker after FW becomes ready. */
 			qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL);
 
-			vha->flags.online = 1;
-
-			/*
-			 * Process any ATIO queue entries that came in
-			 * while we weren't online.
-			 */
-			spin_lock_irqsave(&ha->hardware_lock, flags);
-			if (qla_tgt_mode_enabled(vha))
-				qlt_24xx_process_atio_queue(vha);
-			spin_unlock_irqrestore(&ha->hardware_lock, flags);
-
 			set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
 		}
 
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index fee9eb7c8a60..a6b7f1588aa4 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -258,6 +258,8 @@ qla2x00_init_timer(srb_t *sp, unsigned long tmo)
 	if ((IS_QLAFX00(sp->fcport->vha->hw)) &&
 	    (sp->type == SRB_FXIOCB_DCMD))
 		init_completion(&sp->u.iocb_cmd.u.fxiocb.fxiocb_comp);
+	if (sp->type == SRB_ELS_DCMD)
+		init_completion(&sp->u.iocb_cmd.u.els_logo.comp);
 }
 
 static inline int
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index c49df34e9b35..b41265a75ed5 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -1868,6 +1868,7 @@ skip_cmd_array:
 	}
 
 queuing_error:
+	vha->tgt_counters.num_alloc_iocb_failed++;
 	return pkt;
 }
 
@@ -2010,6 +2011,190 @@ qla24xx_tm_iocb(srb_t *sp, struct tsk_mgmt_entry *tsk)
 }
 
 static void
+qla2x00_els_dcmd_sp_free(void *ptr, void *data)
+{
+	struct scsi_qla_host *vha = (scsi_qla_host_t *)ptr;
+	struct qla_hw_data *ha = vha->hw;
+	srb_t *sp = (srb_t *)data;
+	struct srb_iocb *elsio = &sp->u.iocb_cmd;
+
+	kfree(sp->fcport);
+
+	if (elsio->u.els_logo.els_logo_pyld)
+		dma_free_coherent(&ha->pdev->dev, DMA_POOL_SIZE,
+		    elsio->u.els_logo.els_logo_pyld,
+		    elsio->u.els_logo.els_logo_pyld_dma);
+
+	del_timer(&elsio->timer);
+	qla2x00_rel_sp(vha, sp);
+}
+
+static void
+qla2x00_els_dcmd_iocb_timeout(void *data)
+{
+	srb_t *sp = (srb_t *)data;
+	struct srb_iocb *lio = &sp->u.iocb_cmd;
+	fc_port_t *fcport = sp->fcport;
+	struct scsi_qla_host *vha = fcport->vha;
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long flags = 0;
+
+	ql_dbg(ql_dbg_io, vha, 0x3069,
+	    "%s Timeout, hdl=%x, portid=%02x%02x%02x\n",
+	    sp->name, sp->handle, fcport->d_id.b.domain, fcport->d_id.b.area,
+	    fcport->d_id.b.al_pa);
+
+	/* Abort the exchange */
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+	if (ha->isp_ops->abort_command(sp)) {
+		ql_dbg(ql_dbg_io, vha, 0x3070,
+		    "mbx abort_command failed.\n");
+	} else {
+		ql_dbg(ql_dbg_io, vha, 0x3071,
+		    "mbx abort_command success.\n");
+	}
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
+	complete(&lio->u.els_logo.comp);
+}
+
+static void
+qla2x00_els_dcmd_sp_done(void *data, void *ptr, int res)
+{
+	srb_t *sp = (srb_t *)ptr;
+	fc_port_t *fcport = sp->fcport;
+	struct srb_iocb *lio = &sp->u.iocb_cmd;
+	struct scsi_qla_host *vha = fcport->vha;
+
+	ql_dbg(ql_dbg_io, vha, 0x3072,
+	    "%s hdl=%x, portid=%02x%02x%02x done\n",
+	    sp->name, sp->handle, fcport->d_id.b.domain,
+	    fcport->d_id.b.area, fcport->d_id.b.al_pa);
+
+	complete(&lio->u.els_logo.comp);
+}
+
+int
+qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode,
+    port_id_t remote_did)
+{
+	srb_t *sp;
+	fc_port_t *fcport = NULL;
+	struct srb_iocb *elsio = NULL;
+	struct qla_hw_data *ha = vha->hw;
+	struct els_logo_payload logo_pyld;
+	int rval = QLA_SUCCESS;
+
+	fcport = qla2x00_alloc_fcport(vha, GFP_KERNEL);
+	if (!fcport) {
+	       ql_log(ql_log_info, vha, 0x70e5, "fcport allocation failed\n");
+	       return -ENOMEM;
+	}
+
+	/* Alloc SRB structure */
+	sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+	if (!sp) {
+		kfree(fcport);
+		ql_log(ql_log_info, vha, 0x70e6,
+		 "SRB allocation failed\n");
+		return -ENOMEM;
+	}
+
+	elsio = &sp->u.iocb_cmd;
+	fcport->loop_id = 0xFFFF;
+	fcport->d_id.b.domain = remote_did.b.domain;
+	fcport->d_id.b.area = remote_did.b.area;
+	fcport->d_id.b.al_pa = remote_did.b.al_pa;
+
+	ql_dbg(ql_dbg_io, vha, 0x3073, "portid=%02x%02x%02x done\n",
+	    fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa);
+
+	sp->type = SRB_ELS_DCMD;
+	sp->name = "ELS_DCMD";
+	sp->fcport = fcport;
+	qla2x00_init_timer(sp, ELS_DCMD_TIMEOUT);
+	elsio->timeout = qla2x00_els_dcmd_iocb_timeout;
+	sp->done = qla2x00_els_dcmd_sp_done;
+	sp->free = qla2x00_els_dcmd_sp_free;
+
+	elsio->u.els_logo.els_logo_pyld = dma_alloc_coherent(&ha->pdev->dev,
+			    DMA_POOL_SIZE, &elsio->u.els_logo.els_logo_pyld_dma,
+			    GFP_KERNEL);
+
+	if (!elsio->u.els_logo.els_logo_pyld) {
+		sp->free(vha, sp);
+		return QLA_FUNCTION_FAILED;
+	}
+
+	memset(&logo_pyld, 0, sizeof(struct els_logo_payload));
+
+	elsio->u.els_logo.els_cmd = els_opcode;
+	logo_pyld.opcode = els_opcode;
+	logo_pyld.s_id[0] = vha->d_id.b.al_pa;
+	logo_pyld.s_id[1] = vha->d_id.b.area;
+	logo_pyld.s_id[2] = vha->d_id.b.domain;
+	host_to_fcp_swap(logo_pyld.s_id, sizeof(uint32_t));
+	memcpy(&logo_pyld.wwpn, vha->port_name, WWN_SIZE);
+
+	memcpy(elsio->u.els_logo.els_logo_pyld, &logo_pyld,
+	    sizeof(struct els_logo_payload));
+
+	rval = qla2x00_start_sp(sp);
+	if (rval != QLA_SUCCESS) {
+		sp->free(vha, sp);
+		return QLA_FUNCTION_FAILED;
+	}
+
+	ql_dbg(ql_dbg_io, vha, 0x3074,
+	    "%s LOGO sent, hdl=%x, loopid=%x, portid=%02x%02x%02x.\n",
+	    sp->name, sp->handle, fcport->loop_id, fcport->d_id.b.domain,
+	    fcport->d_id.b.area, fcport->d_id.b.al_pa);
+
+	wait_for_completion(&elsio->u.els_logo.comp);
+
+	sp->free(vha, sp);
+	return rval;
+}
+
+static void
+qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
+{
+	scsi_qla_host_t *vha = sp->fcport->vha;
+	struct srb_iocb *elsio = &sp->u.iocb_cmd;
+
+	els_iocb->entry_type = ELS_IOCB_TYPE;
+	els_iocb->entry_count = 1;
+	els_iocb->sys_define = 0;
+	els_iocb->entry_status = 0;
+	els_iocb->handle = sp->handle;
+	els_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
+	els_iocb->tx_dsd_count = 1;
+	els_iocb->vp_index = vha->vp_idx;
+	els_iocb->sof_type = EST_SOFI3;
+	els_iocb->rx_dsd_count = 0;
+	els_iocb->opcode = elsio->u.els_logo.els_cmd;
+
+	els_iocb->port_id[0] = sp->fcport->d_id.b.al_pa;
+	els_iocb->port_id[1] = sp->fcport->d_id.b.area;
+	els_iocb->port_id[2] = sp->fcport->d_id.b.domain;
+	els_iocb->control_flags = 0;
+
+	els_iocb->tx_byte_count = sizeof(struct els_logo_payload);
+	els_iocb->tx_address[0] =
+	    cpu_to_le32(LSD(elsio->u.els_logo.els_logo_pyld_dma));
+	els_iocb->tx_address[1] =
+	    cpu_to_le32(MSD(elsio->u.els_logo.els_logo_pyld_dma));
+	els_iocb->tx_len = cpu_to_le32(sizeof(struct els_logo_payload));
+
+	els_iocb->rx_byte_count = 0;
+	els_iocb->rx_address[0] = 0;
+	els_iocb->rx_address[1] = 0;
+	els_iocb->rx_len = 0;
+
+	sp->fcport->vha->qla_stats.control_requests++;
+}
+
+static void
 qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 {
 	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
@@ -2623,6 +2808,9 @@ qla2x00_start_sp(srb_t *sp)
 			qlafx00_abort_iocb(sp, pkt) :
 			qla24xx_abort_iocb(sp, pkt);
 		break;
+	case SRB_ELS_DCMD:
+		qla24xx_els_logo_iocb(sp, pkt);
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index ccf6a7f99024..d4d65eb0e9b4 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -18,6 +18,10 @@ static void qla2x00_status_entry(scsi_qla_host_t *, struct rsp_que *, void *);
 static void qla2x00_status_cont_entry(struct rsp_que *, sts_cont_entry_t *);
 static void qla2x00_error_entry(scsi_qla_host_t *, struct rsp_que *,
 	sts_entry_t *);
+static void qla_irq_affinity_notify(struct irq_affinity_notify *,
+    const cpumask_t *);
+static void qla_irq_affinity_release(struct kref *);
+
 
 /**
  * qla2100_intr_handler() - Process interrupts for the ISP2100 and ISP2200.
@@ -1418,6 +1422,12 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	case SRB_CT_CMD:
 		type = "ct pass-through";
 		break;
+	case SRB_ELS_DCMD:
+		type = "Driver ELS logo";
+		ql_dbg(ql_dbg_user, vha, 0x5047,
+		    "Completing %s: (%p) type=%d.\n", type, sp, sp->type);
+		sp->done(vha, sp, 0);
+		return;
 	default:
 		ql_dbg(ql_dbg_user, vha, 0x503e,
 		    "Unrecognized SRB: (%p) type=%d.\n", sp, sp->type);
@@ -2542,6 +2552,14 @@ void qla24xx_process_response_queue(struct scsi_qla_host *vha,
 	if (!vha->flags.online)
 		return;
 
+	if (rsp->msix->cpuid != smp_processor_id()) {
+		/* if kernel does not notify qla of IRQ's CPU change,
+		 * then set it here.
+		 */
+		rsp->msix->cpuid = smp_processor_id();
+		ha->tgt.rspq_vector_cpuid = rsp->msix->cpuid;
+	}
+
 	while (rsp->ring_ptr->signature != RESPONSE_PROCESSED) {
 		pkt = (struct sts_entry_24xx *)rsp->ring_ptr;
 
@@ -2587,8 +2605,14 @@ process_err:
 			qla24xx_els_ct_entry(vha, rsp->req, pkt, ELS_IOCB_TYPE);
 			break;
 		case ABTS_RECV_24XX:
-			/* ensure that the ATIO queue is empty */
-			qlt_24xx_process_atio_queue(vha);
+			if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+				/* ensure that the ATIO queue is empty */
+				qlt_handle_abts_recv(vha, (response_t *)pkt);
+				break;
+			} else {
+				/* drop through */
+				qlt_24xx_process_atio_queue(vha, 1);
+			}
 		case ABTS_RESP_24XX:
 		case CTIO_TYPE7:
 		case NOTIFY_ACK_TYPE:
@@ -2755,13 +2779,22 @@ qla24xx_intr_handler(int irq, void *dev_id)
 		case INTR_RSP_QUE_UPDATE_83XX:
 			qla24xx_process_response_queue(vha, rsp);
 			break;
-		case INTR_ATIO_QUE_UPDATE:
-			qlt_24xx_process_atio_queue(vha);
+		case INTR_ATIO_QUE_UPDATE:{
+			unsigned long flags2;
+			spin_lock_irqsave(&ha->tgt.atio_lock, flags2);
+			qlt_24xx_process_atio_queue(vha, 1);
+			spin_unlock_irqrestore(&ha->tgt.atio_lock, flags2);
 			break;
-		case INTR_ATIO_RSP_QUE_UPDATE:
-			qlt_24xx_process_atio_queue(vha);
+		}
+		case INTR_ATIO_RSP_QUE_UPDATE: {
+			unsigned long flags2;
+			spin_lock_irqsave(&ha->tgt.atio_lock, flags2);
+			qlt_24xx_process_atio_queue(vha, 1);
+			spin_unlock_irqrestore(&ha->tgt.atio_lock, flags2);
+
 			qla24xx_process_response_queue(vha, rsp);
 			break;
+		}
 		default:
 			ql_dbg(ql_dbg_async, vha, 0x504f,
 			    "Unrecognized interrupt type (%d).\n", stat * 0xff);
@@ -2920,13 +2953,22 @@ qla24xx_msix_default(int irq, void *dev_id)
 		case INTR_RSP_QUE_UPDATE_83XX:
 			qla24xx_process_response_queue(vha, rsp);
 			break;
-		case INTR_ATIO_QUE_UPDATE:
-			qlt_24xx_process_atio_queue(vha);
+		case INTR_ATIO_QUE_UPDATE:{
+			unsigned long flags2;
+			spin_lock_irqsave(&ha->tgt.atio_lock, flags2);
+			qlt_24xx_process_atio_queue(vha, 1);
+			spin_unlock_irqrestore(&ha->tgt.atio_lock, flags2);
 			break;
-		case INTR_ATIO_RSP_QUE_UPDATE:
-			qlt_24xx_process_atio_queue(vha);
+		}
+		case INTR_ATIO_RSP_QUE_UPDATE: {
+			unsigned long flags2;
+			spin_lock_irqsave(&ha->tgt.atio_lock, flags2);
+			qlt_24xx_process_atio_queue(vha, 1);
+			spin_unlock_irqrestore(&ha->tgt.atio_lock, flags2);
+
 			qla24xx_process_response_queue(vha, rsp);
 			break;
+		}
 		default:
 			ql_dbg(ql_dbg_async, vha, 0x5051,
 			    "Unrecognized interrupt type (%d).\n", stat & 0xff);
@@ -2973,8 +3015,11 @@ qla24xx_disable_msix(struct qla_hw_data *ha)
 
 	for (i = 0; i < ha->msix_count; i++) {
 		qentry = &ha->msix_entries[i];
-		if (qentry->have_irq)
+		if (qentry->have_irq) {
+			/* un-register irq cpu affinity notification */
+			irq_set_affinity_notifier(qentry->vector, NULL);
 			free_irq(qentry->vector, qentry->rsp);
+		}
 	}
 	pci_disable_msix(ha->pdev);
 	kfree(ha->msix_entries);
@@ -3037,6 +3082,9 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp)
 		qentry->entry = entries[i].entry;
 		qentry->have_irq = 0;
 		qentry->rsp = NULL;
+		qentry->irq_notify.notify  = qla_irq_affinity_notify;
+		qentry->irq_notify.release = qla_irq_affinity_release;
+		qentry->cpuid = -1;
 	}
 
 	/* Enable MSI-X vectors for the base queue */
@@ -3055,6 +3103,18 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp)
 		qentry->have_irq = 1;
 		qentry->rsp = rsp;
 		rsp->msix = qentry;
+
+		/* Register for CPU affinity notification. */
+		irq_set_affinity_notifier(qentry->vector, &qentry->irq_notify);
+
+		/* Schedule work (ie. trigger a notification) to read cpu
+		 * mask for this specific irq.
+		 * kref_get is required because
+		* irq_affinity_notify() will do
+		* kref_put().
+		*/
+		kref_get(&qentry->irq_notify.kref);
+		schedule_work(&qentry->irq_notify.work);
 	}
 
 	/*
@@ -3234,3 +3294,47 @@ int qla25xx_request_irq(struct rsp_que *rsp)
 	msix->rsp = rsp;
 	return ret;
 }
+
+
+/* irq_set_affinity/irqbalance will trigger notification of cpu mask update */
+static void qla_irq_affinity_notify(struct irq_affinity_notify *notify,
+	const cpumask_t *mask)
+{
+	struct qla_msix_entry *e =
+		container_of(notify, struct qla_msix_entry, irq_notify);
+	struct qla_hw_data *ha;
+	struct scsi_qla_host *base_vha;
+
+	/* user is recommended to set mask to just 1 cpu */
+	e->cpuid = cpumask_first(mask);
+
+	ha = e->rsp->hw;
+	base_vha = pci_get_drvdata(ha->pdev);
+
+	ql_dbg(ql_dbg_init, base_vha, 0xffff,
+	    "%s: host %ld : vector %d cpu %d \n", __func__,
+	    base_vha->host_no, e->vector, e->cpuid);
+
+	if (e->have_irq) {
+		if ((IS_QLA83XX(ha) || IS_QLA27XX(ha)) &&
+		    (e->entry == QLA83XX_RSPQ_MSIX_ENTRY_NUMBER)) {
+			ha->tgt.rspq_vector_cpuid = e->cpuid;
+			ql_dbg(ql_dbg_init, base_vha, 0xffff,
+			    "%s: host%ld: rspq vector %d cpu %d  runtime change\n",
+			    __func__, base_vha->host_no, e->vector, e->cpuid);
+		}
+	}
+}
+
+static void qla_irq_affinity_release(struct kref *ref)
+{
+	struct irq_affinity_notify *notify =
+		container_of(ref, struct irq_affinity_notify, kref);
+	struct qla_msix_entry *e =
+		container_of(notify, struct qla_msix_entry, irq_notify);
+	struct scsi_qla_host *base_vha = pci_get_drvdata(e->rsp->hw->pdev);
+
+	ql_dbg(ql_dbg_init, base_vha, 0xffff,
+	    "%s: host%ld: vector %d cpu %d \n", __func__,
+	    base_vha->host_no, e->vector, e->cpuid);
+}
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index cb11e04be568..87e6758302f6 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -489,6 +489,13 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 			    EXTENDED_BB_CREDITS);
 		} else
 			mcp->mb[4] = 0;
+
+		if (ha->flags.exlogins_enabled)
+			mcp->mb[4] |= ENABLE_EXTENDED_LOGIN;
+
+		if (ha->flags.exchoffld_enabled)
+			mcp->mb[4] |= ENABLE_EXCHANGE_OFFLD;
+
 		mcp->out_mb |= MBX_4|MBX_3|MBX_2|MBX_1;
 		mcp->in_mb |= MBX_1;
 	} else {
@@ -521,6 +528,226 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 }
 
 /*
+ * qla_get_exlogin_status
+ *	Get extended login status
+ *	uses the memory offload control/status Mailbox
+ *
+ * Input:
+ *	ha:		adapter state pointer.
+ *	fwopt:		firmware options
+ *
+ * Returns:
+ *	qla2x00 local function status
+ *
+ * Context:
+ *	Kernel context.
+ */
+#define	FETCH_XLOGINS_STAT	0x8
+int
+qla_get_exlogin_status(scsi_qla_host_t *vha, uint16_t *buf_sz,
+	uint16_t *ex_logins_cnt)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x118f,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_MEM_OFFLOAD_CNTRL_STAT;
+	mcp->mb[1] = FETCH_XLOGINS_STAT;
+	mcp->out_mb = MBX_1|MBX_0;
+	mcp->in_mb = MBX_10|MBX_4|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1115, "Failed=%x.\n", rval);
+	} else {
+		*buf_sz = mcp->mb[4];
+		*ex_logins_cnt = mcp->mb[10];
+
+		ql_log(ql_log_info, vha, 0x1190,
+		    "buffer size 0x%x, exchange login count=%d\n",
+		    mcp->mb[4], mcp->mb[10]);
+
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1116,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+/*
+ * qla_set_exlogin_mem_cfg
+ *	set extended login memory configuration
+ *	Mbx needs to be issues before init_cb is set
+ *
+ * Input:
+ *	ha:		adapter state pointer.
+ *	buffer:		buffer pointer
+ *	phys_addr:	physical address of buffer
+ *	size:		size of buffer
+ *	TARGET_QUEUE_LOCK must be released
+ *	ADAPTER_STATE_LOCK must be release
+ *
+ * Returns:
+ *	qla2x00 local funxtion status code.
+ *
+ * Context:
+ *	Kernel context.
+ */
+#define CONFIG_XLOGINS_MEM	0x3
+int
+qla_set_exlogin_mem_cfg(scsi_qla_host_t *vha, dma_addr_t phys_addr)
+{
+	int		rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
+	int configured_count;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x111a,
+	    "Entered %s.\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_MEM_OFFLOAD_CNTRL_STAT;
+	mcp->mb[1] = CONFIG_XLOGINS_MEM;
+	mcp->mb[2] = MSW(phys_addr);
+	mcp->mb[3] = LSW(phys_addr);
+	mcp->mb[6] = MSW(MSD(phys_addr));
+	mcp->mb[7] = LSW(MSD(phys_addr));
+	mcp->mb[8] = MSW(ha->exlogin_size);
+	mcp->mb[9] = LSW(ha->exlogin_size);
+	mcp->out_mb = MBX_9|MBX_8|MBX_7|MBX_6|MBX_3|MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_11|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		/*EMPTY*/
+		ql_dbg(ql_dbg_mbx, vha, 0x111b, "Failed=%x.\n", rval);
+	} else {
+		configured_count = mcp->mb[11];
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x118c,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+/*
+ * qla_get_exchoffld_status
+ *	Get exchange offload status
+ *	uses the memory offload control/status Mailbox
+ *
+ * Input:
+ *	ha:		adapter state pointer.
+ *	fwopt:		firmware options
+ *
+ * Returns:
+ *	qla2x00 local function status
+ *
+ * Context:
+ *	Kernel context.
+ */
+#define	FETCH_XCHOFFLD_STAT	0x2
+int
+qla_get_exchoffld_status(scsi_qla_host_t *vha, uint16_t *buf_sz,
+	uint16_t *ex_logins_cnt)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1019,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_MEM_OFFLOAD_CNTRL_STAT;
+	mcp->mb[1] = FETCH_XCHOFFLD_STAT;
+	mcp->out_mb = MBX_1|MBX_0;
+	mcp->in_mb = MBX_10|MBX_4|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1155, "Failed=%x.\n", rval);
+	} else {
+		*buf_sz = mcp->mb[4];
+		*ex_logins_cnt = mcp->mb[10];
+
+		ql_log(ql_log_info, vha, 0x118e,
+		    "buffer size 0x%x, exchange offload count=%d\n",
+		    mcp->mb[4], mcp->mb[10]);
+
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1156,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+/*
+ * qla_set_exchoffld_mem_cfg
+ *	Set exchange offload memory configuration
+ *	Mbx needs to be issues before init_cb is set
+ *
+ * Input:
+ *	ha:		adapter state pointer.
+ *	buffer:		buffer pointer
+ *	phys_addr:	physical address of buffer
+ *	size:		size of buffer
+ *	TARGET_QUEUE_LOCK must be released
+ *	ADAPTER_STATE_LOCK must be release
+ *
+ * Returns:
+ *	qla2x00 local funxtion status code.
+ *
+ * Context:
+ *	Kernel context.
+ */
+#define CONFIG_XCHOFFLD_MEM	0x3
+int
+qla_set_exchoffld_mem_cfg(scsi_qla_host_t *vha, dma_addr_t phys_addr)
+{
+	int		rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1157,
+	    "Entered %s.\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_MEM_OFFLOAD_CNTRL_STAT;
+	mcp->mb[1] = CONFIG_XCHOFFLD_MEM;
+	mcp->mb[2] = MSW(phys_addr);
+	mcp->mb[3] = LSW(phys_addr);
+	mcp->mb[6] = MSW(MSD(phys_addr));
+	mcp->mb[7] = LSW(MSD(phys_addr));
+	mcp->mb[8] = MSW(ha->exlogin_size);
+	mcp->mb[9] = LSW(ha->exlogin_size);
+	mcp->out_mb = MBX_9|MBX_8|MBX_7|MBX_6|MBX_3|MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_11|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		/*EMPTY*/
+		ql_dbg(ql_dbg_mbx, vha, 0x1158, "Failed=%x.\n", rval);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1192,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+/*
  * qla2x00_get_fw_version
  *	Get firmware version.
  *
@@ -594,6 +821,16 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x112f,
 		    "%s: Ext_FwAttributes Upper: 0x%x, Lower: 0x%x.\n",
 		    __func__, mcp->mb[17], mcp->mb[16]);
+
+		if (ha->fw_attributes_h & 0x4)
+			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x118d,
+			    "%s: Firmware supports Extended Login 0x%x\n",
+			    __func__, ha->fw_attributes_h);
+
+		if (ha->fw_attributes_h & 0x8)
+			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1191,
+			    "%s: Firmware supports Exchange Offload 0x%x\n",
+			    __func__, ha->fw_attributes_h);
 	}
 
 	if (IS_QLA27XX(ha)) {
@@ -2383,10 +2620,9 @@ qla2x00_get_id_list(scsi_qla_host_t *vha, void *id_list, dma_addr_t id_list_dma,
  *	Kernel context.
  */
 int
-qla2x00_get_resource_cnts(scsi_qla_host_t *vha, uint16_t *cur_xchg_cnt,
-    uint16_t *orig_xchg_cnt, uint16_t *cur_iocb_cnt,
-    uint16_t *orig_iocb_cnt, uint16_t *max_npiv_vports, uint16_t *max_fcfs)
+qla2x00_get_resource_cnts(scsi_qla_host_t *vha)
 {
+	struct qla_hw_data *ha = vha->hw;
 	int rval;
 	mbx_cmd_t mc;
 	mbx_cmd_t *mcp = &mc;
@@ -2414,19 +2650,16 @@ qla2x00_get_resource_cnts(scsi_qla_host_t *vha, uint16_t *cur_xchg_cnt,
 		    mcp->mb[3], mcp->mb[6], mcp->mb[7], mcp->mb[10],
 		    mcp->mb[11], mcp->mb[12]);
 
-		if (cur_xchg_cnt)
-			*cur_xchg_cnt = mcp->mb[3];
-		if (orig_xchg_cnt)
-			*orig_xchg_cnt = mcp->mb[6];
-		if (cur_iocb_cnt)
-			*cur_iocb_cnt = mcp->mb[7];
-		if (orig_iocb_cnt)
-			*orig_iocb_cnt = mcp->mb[10];
-		if (vha->hw->flags.npiv_supported && max_npiv_vports)
-			*max_npiv_vports = mcp->mb[11];
-		if ((IS_QLA81XX(vha->hw) || IS_QLA83XX(vha->hw) ||
-		    IS_QLA27XX(vha->hw)) && max_fcfs)
-			*max_fcfs = mcp->mb[12];
+		ha->orig_fw_tgt_xcb_count =  mcp->mb[1];
+		ha->cur_fw_tgt_xcb_count = mcp->mb[2];
+		ha->cur_fw_xcb_count = mcp->mb[3];
+		ha->orig_fw_xcb_count = mcp->mb[6];
+		ha->cur_fw_iocb_count = mcp->mb[7];
+		ha->orig_fw_iocb_count = mcp->mb[10];
+		if (ha->flags.npiv_supported)
+			ha->max_npiv_vports = mcp->mb[11];
+		if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+			ha->fw_max_fcf_count = mcp->mb[12];
 	}
 
 	return (rval);
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 6be32fdab365..f1788db43195 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -221,6 +221,18 @@ MODULE_PARM_DESC(ql2xmdenable,
 		"0 - MiniDump disabled. "
 		"1 (Default) - MiniDump enabled.");
 
+int ql2xexlogins = 0;
+module_param(ql2xexlogins, uint, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(ql2xexlogins,
+		 "Number of extended Logins. "
+		 "0 (Default)- Disabled.");
+
+int ql2xexchoffld = 0;
+module_param(ql2xexchoffld, uint, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(ql2xexchoffld,
+		 "Number of exchanges to offload. "
+		 "0 (Default)- Disabled.");
+
 /*
  * SCSI host template entry points
  */
@@ -2324,6 +2336,9 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	ha->tgt.enable_class_2 = ql2xenableclass2;
 	INIT_LIST_HEAD(&ha->tgt.q_full_list);
 	spin_lock_init(&ha->tgt.q_full_lock);
+	spin_lock_init(&ha->tgt.sess_lock);
+	spin_lock_init(&ha->tgt.atio_lock);
+
 
 	/* Clear our data area */
 	ha->bars = bars;
@@ -2468,7 +2483,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		ha->max_fibre_devices = MAX_FIBRE_DEVICES_2400;
 		ha->mbx_count = MAILBOX_REGISTER_COUNT;
 		req_length = REQUEST_ENTRY_CNT_83XX;
-		rsp_length = RESPONSE_ENTRY_CNT_2300;
+		rsp_length = RESPONSE_ENTRY_CNT_83XX;
 		ha->tgt.atio_q_length = ATIO_ENTRY_CNT_24XX;
 		ha->max_loop_id = SNS_LAST_LOOP_ID_2300;
 		ha->init_cb_size = sizeof(struct mid_init_cb_81xx);
@@ -2498,8 +2513,8 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		ha->portnum = PCI_FUNC(ha->pdev->devfn);
 		ha->max_fibre_devices = MAX_FIBRE_DEVICES_2400;
 		ha->mbx_count = MAILBOX_REGISTER_COUNT;
-		req_length = REQUEST_ENTRY_CNT_24XX;
-		rsp_length = RESPONSE_ENTRY_CNT_2300;
+		req_length = REQUEST_ENTRY_CNT_83XX;
+		rsp_length = RESPONSE_ENTRY_CNT_83XX;
 		ha->tgt.atio_q_length = ATIO_ENTRY_CNT_24XX;
 		ha->max_loop_id = SNS_LAST_LOOP_ID_2300;
 		ha->init_cb_size = sizeof(struct mid_init_cb_81xx);
@@ -3128,6 +3143,14 @@ qla2x00_remove_one(struct pci_dev *pdev)
 
 	base_vha->flags.online = 0;
 
+	/* free DMA memory */
+	if (ha->exlogin_buf)
+		qla2x00_free_exlogin_buffer(ha);
+
+	/* free DMA memory */
+	if (ha->exchoffld_buf)
+		qla2x00_free_exchoffld_buffer(ha);
+
 	qla2x00_destroy_deferred_work(ha);
 
 	qlt_remove_target(ha, base_vha);
@@ -3587,6 +3610,140 @@ fail:
 	return -ENOMEM;
 }
 
+int
+qla2x00_set_exlogins_buffer(scsi_qla_host_t *vha)
+{
+	int rval;
+	uint16_t	size, max_cnt, temp;
+	struct qla_hw_data *ha = vha->hw;
+
+	/* Return if we don't need to alloacate any extended logins */
+	if (!ql2xexlogins)
+		return QLA_SUCCESS;
+
+	ql_log(ql_log_info, vha, 0xd021, "EXLOGIN count: %d.\n", ql2xexlogins);
+	max_cnt = 0;
+	rval = qla_get_exlogin_status(vha, &size, &max_cnt);
+	if (rval != QLA_SUCCESS) {
+		ql_log_pci(ql_log_fatal, ha->pdev, 0xd029,
+		    "Failed to get exlogin status.\n");
+		return rval;
+	}
+
+	temp = (ql2xexlogins > max_cnt) ? max_cnt : ql2xexlogins;
+	ha->exlogin_size = (size * temp);
+	ql_log(ql_log_info, vha, 0xd024,
+		"EXLOGIN: max_logins=%d, portdb=0x%x, total=%d.\n",
+		max_cnt, size, temp);
+
+	ql_log(ql_log_info, vha, 0xd025, "EXLOGIN: requested size=0x%x\n",
+		ha->exlogin_size);
+
+	/* Get consistent memory for extended logins */
+	ha->exlogin_buf = dma_alloc_coherent(&ha->pdev->dev,
+	    ha->exlogin_size, &ha->exlogin_buf_dma, GFP_KERNEL);
+	if (!ha->exlogin_buf) {
+		ql_log_pci(ql_log_fatal, ha->pdev, 0xd02a,
+		    "Failed to allocate memory for exlogin_buf_dma.\n");
+		return -ENOMEM;
+	}
+
+	/* Now configure the dma buffer */
+	rval = qla_set_exlogin_mem_cfg(vha, ha->exlogin_buf_dma);
+	if (rval) {
+		ql_log(ql_log_fatal, vha, 0x00cf,
+		    "Setup extended login buffer  ****FAILED****.\n");
+		qla2x00_free_exlogin_buffer(ha);
+	}
+
+	return rval;
+}
+
+/*
+* qla2x00_free_exlogin_buffer
+*
+* Input:
+*	ha = adapter block pointer
+*/
+void
+qla2x00_free_exlogin_buffer(struct qla_hw_data *ha)
+{
+	if (ha->exlogin_buf) {
+		dma_free_coherent(&ha->pdev->dev, ha->exlogin_size,
+		    ha->exlogin_buf, ha->exlogin_buf_dma);
+		ha->exlogin_buf = NULL;
+		ha->exlogin_size = 0;
+	}
+}
+
+int
+qla2x00_set_exchoffld_buffer(scsi_qla_host_t *vha)
+{
+	int rval;
+	uint16_t	size, max_cnt, temp;
+	struct qla_hw_data *ha = vha->hw;
+
+	/* Return if we don't need to alloacate any extended logins */
+	if (!ql2xexchoffld)
+		return QLA_SUCCESS;
+
+	ql_log(ql_log_info, vha, 0xd014,
+	    "Exchange offload count: %d.\n", ql2xexlogins);
+
+	max_cnt = 0;
+	rval = qla_get_exchoffld_status(vha, &size, &max_cnt);
+	if (rval != QLA_SUCCESS) {
+		ql_log_pci(ql_log_fatal, ha->pdev, 0xd012,
+		    "Failed to get exlogin status.\n");
+		return rval;
+	}
+
+	temp = (ql2xexchoffld > max_cnt) ? max_cnt : ql2xexchoffld;
+	ha->exchoffld_size = (size * temp);
+	ql_log(ql_log_info, vha, 0xd016,
+		"Exchange offload: max_count=%d, buffers=0x%x, total=%d.\n",
+		max_cnt, size, temp);
+
+	ql_log(ql_log_info, vha, 0xd017,
+	    "Exchange Buffers requested size = 0x%x\n", ha->exchoffld_size);
+
+	/* Get consistent memory for extended logins */
+	ha->exchoffld_buf = dma_alloc_coherent(&ha->pdev->dev,
+	    ha->exchoffld_size, &ha->exchoffld_buf_dma, GFP_KERNEL);
+	if (!ha->exchoffld_buf) {
+		ql_log_pci(ql_log_fatal, ha->pdev, 0xd013,
+		    "Failed to allocate memory for exchoffld_buf_dma.\n");
+		return -ENOMEM;
+	}
+
+	/* Now configure the dma buffer */
+	rval = qla_set_exchoffld_mem_cfg(vha, ha->exchoffld_buf_dma);
+	if (rval) {
+		ql_log(ql_log_fatal, vha, 0xd02e,
+		    "Setup exchange offload buffer ****FAILED****.\n");
+		qla2x00_free_exchoffld_buffer(ha);
+	}
+
+	return rval;
+}
+
+/*
+* qla2x00_free_exchoffld_buffer
+*
+* Input:
+*	ha = adapter block pointer
+*/
+void
+qla2x00_free_exchoffld_buffer(struct qla_hw_data *ha)
+{
+	if (ha->exchoffld_buf) {
+		dma_free_coherent(&ha->pdev->dev, ha->exchoffld_size,
+		    ha->exchoffld_buf, ha->exchoffld_buf_dma);
+		ha->exchoffld_buf = NULL;
+		ha->exchoffld_size = 0;
+	}
+}
+
 /*
 * qla2x00_free_fw_dump
 *	Frees fw dump stuff.
@@ -3766,6 +3923,8 @@ struct scsi_qla_host *qla2x00_create_host(struct scsi_host_template *sht,
 	INIT_LIST_HEAD(&vha->list);
 	INIT_LIST_HEAD(&vha->qla_cmd_list);
 	INIT_LIST_HEAD(&vha->qla_sess_op_cmd_list);
+	INIT_LIST_HEAD(&vha->logo_list);
+	INIT_LIST_HEAD(&vha->plogi_ack_list);
 
 	spin_lock_init(&vha->work_lock);
 	spin_lock_init(&vha->cmd_list_lock);
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 75514a15bea0..8075a4cdb45c 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -100,7 +100,7 @@ enum fcp_resp_rsp_codes {
  */
 /* Predefs for callbacks handed to qla2xxx LLD */
 static void qlt_24xx_atio_pkt(struct scsi_qla_host *ha,
-	struct atio_from_isp *pkt);
+	struct atio_from_isp *pkt, uint8_t);
 static void qlt_response_pkt(struct scsi_qla_host *ha, response_t *pkt);
 static int qlt_issue_task_mgmt(struct qla_tgt_sess *sess, uint32_t lun,
 	int fn, void *iocb, int flags);
@@ -118,10 +118,13 @@ static void qlt_send_notify_ack(struct scsi_qla_host *vha,
 	struct imm_ntfy_from_isp *ntfy,
 	uint32_t add_flags, uint16_t resp_code, int resp_code_valid,
 	uint16_t srr_flags, uint16_t srr_reject_code, uint8_t srr_explan);
+static void qlt_send_term_imm_notif(struct scsi_qla_host *vha,
+	struct imm_ntfy_from_isp *imm, int ha_locked);
 /*
  * Global Variables
  */
 static struct kmem_cache *qla_tgt_mgmt_cmd_cachep;
+static struct kmem_cache *qla_tgt_plogi_cachep;
 static mempool_t *qla_tgt_mgmt_cmd_mempool;
 static struct workqueue_struct *qla_tgt_wq;
 static DEFINE_MUTEX(qla_tgt_mutex);
@@ -226,8 +229,8 @@ static inline void qlt_decr_num_pend_cmds(struct scsi_qla_host *vha)
 	spin_unlock_irqrestore(&vha->hw->tgt.q_full_lock, flags);
 }
 
-static void qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
-	struct atio_from_isp *atio)
+static bool qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
+	struct atio_from_isp *atio, uint8_t ha_locked)
 {
 	ql_dbg(ql_dbg_tgt, vha, 0xe072,
 		"%s: qla_target(%d): type %x ox_id %04x\n",
@@ -248,7 +251,7 @@ static void qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
 			    atio->u.isp24.fcp_hdr.d_id[2]);
 			break;
 		}
-		qlt_24xx_atio_pkt(host, atio);
+		qlt_24xx_atio_pkt(host, atio, ha_locked);
 		break;
 	}
 
@@ -271,7 +274,7 @@ static void qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
 				break;
 			}
 		}
-		qlt_24xx_atio_pkt(host, atio);
+		qlt_24xx_atio_pkt(host, atio, ha_locked);
 		break;
 	}
 
@@ -282,7 +285,7 @@ static void qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
 		break;
 	}
 
-	return;
+	return false;
 }
 
 void qlt_response_pkt_all_vps(struct scsi_qla_host *vha, response_t *pkt)
@@ -389,6 +392,131 @@ void qlt_response_pkt_all_vps(struct scsi_qla_host *vha, response_t *pkt)
 
 }
 
+/*
+ * All qlt_plogi_ack_t operations are protected by hardware_lock
+ */
+
+/*
+ * This is a zero-base ref-counting solution, since hardware_lock
+ * guarantees that ref_count is not modified concurrently.
+ * Upon successful return content of iocb is undefined
+ */
+static qlt_plogi_ack_t *
+qlt_plogi_ack_find_add(struct scsi_qla_host *vha, port_id_t *id,
+		       struct imm_ntfy_from_isp *iocb)
+{
+	qlt_plogi_ack_t *pla;
+
+	list_for_each_entry(pla, &vha->plogi_ack_list, list) {
+		if (pla->id.b24 == id->b24) {
+			qlt_send_term_imm_notif(vha, &pla->iocb, 1);
+			pla->iocb = *iocb;
+			return pla;
+		}
+	}
+
+	pla = kmem_cache_zalloc(qla_tgt_plogi_cachep, GFP_ATOMIC);
+	if (!pla) {
+		ql_dbg(ql_dbg_async, vha, 0x5088,
+		       "qla_target(%d): Allocation of plogi_ack failed\n",
+		       vha->vp_idx);
+		return NULL;
+	}
+
+	pla->iocb = *iocb;
+	pla->id = *id;
+	list_add_tail(&pla->list, &vha->plogi_ack_list);
+
+	return pla;
+}
+
+static void qlt_plogi_ack_unref(struct scsi_qla_host *vha, qlt_plogi_ack_t *pla)
+{
+	BUG_ON(!pla->ref_count);
+	pla->ref_count--;
+
+	if (pla->ref_count)
+		return;
+
+	ql_dbg(ql_dbg_async, vha, 0x5089,
+	    "Sending PLOGI ACK to wwn %8phC s_id %02x:%02x:%02x loop_id %#04x"
+	    " exch %#x ox_id %#x\n", pla->iocb.u.isp24.port_name,
+	    pla->iocb.u.isp24.port_id[2], pla->iocb.u.isp24.port_id[1],
+	    pla->iocb.u.isp24.port_id[0],
+	    le16_to_cpu(pla->iocb.u.isp24.nport_handle),
+	    pla->iocb.u.isp24.exchange_address, pla->iocb.ox_id);
+	qlt_send_notify_ack(vha, &pla->iocb, 0, 0, 0, 0, 0, 0);
+
+	list_del(&pla->list);
+	kmem_cache_free(qla_tgt_plogi_cachep, pla);
+}
+
+static void
+qlt_plogi_ack_link(struct scsi_qla_host *vha, qlt_plogi_ack_t *pla,
+    struct qla_tgt_sess *sess, qlt_plogi_link_t link)
+{
+	/* Inc ref_count first because link might already be pointing at pla */
+	pla->ref_count++;
+
+	if (sess->plogi_link[link])
+		qlt_plogi_ack_unref(vha, sess->plogi_link[link]);
+
+	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf097,
+	    "Linking sess %p [%d] wwn %8phC with PLOGI ACK to wwn %8phC"
+	    " s_id %02x:%02x:%02x, ref=%d\n", sess, link, sess->port_name,
+	    pla->iocb.u.isp24.port_name, pla->iocb.u.isp24.port_id[2],
+	    pla->iocb.u.isp24.port_id[1], pla->iocb.u.isp24.port_id[0],
+	    pla->ref_count);
+
+	sess->plogi_link[link] = pla;
+}
+
+typedef struct {
+	/* These fields must be initialized by the caller */
+	port_id_t id;
+	/*
+	 * number of cmds dropped while we were waiting for
+	 * initiator to ack LOGO initialize to 1 if LOGO is
+	 * triggered by a command, otherwise, to 0
+	 */
+	int cmd_count;
+
+	/* These fields are used by callee */
+	struct list_head list;
+} qlt_port_logo_t;
+
+static void
+qlt_send_first_logo(struct scsi_qla_host *vha, qlt_port_logo_t *logo)
+{
+	qlt_port_logo_t *tmp;
+	int res;
+
+	mutex_lock(&vha->vha_tgt.tgt_mutex);
+
+	list_for_each_entry(tmp, &vha->logo_list, list) {
+		if (tmp->id.b24 == logo->id.b24) {
+			tmp->cmd_count += logo->cmd_count;
+			mutex_unlock(&vha->vha_tgt.tgt_mutex);
+			return;
+		}
+	}
+
+	list_add_tail(&logo->list, &vha->logo_list);
+
+	mutex_unlock(&vha->vha_tgt.tgt_mutex);
+
+	res = qla24xx_els_dcmd_iocb(vha, ELS_DCMD_LOGO, logo->id);
+
+	mutex_lock(&vha->vha_tgt.tgt_mutex);
+	list_del(&logo->list);
+	mutex_unlock(&vha->vha_tgt.tgt_mutex);
+
+	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf098,
+	    "Finished LOGO to %02x:%02x:%02x, dropped %d cmds, res = %#x\n",
+	    logo->id.b.domain, logo->id.b.area, logo->id.b.al_pa,
+	    logo->cmd_count, res);
+}
+
 static void qlt_free_session_done(struct work_struct *work)
 {
 	struct qla_tgt_sess *sess = container_of(work, struct qla_tgt_sess,
@@ -402,14 +530,21 @@ static void qlt_free_session_done(struct work_struct *work)
 
 	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf084,
 		"%s: se_sess %p / sess %p from port %8phC loop_id %#04x"
-		" s_id %02x:%02x:%02x logout %d keep %d plogi %d\n",
+		" s_id %02x:%02x:%02x logout %d keep %d els_logo %d\n",
 		__func__, sess->se_sess, sess, sess->port_name, sess->loop_id,
 		sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa,
 		sess->logout_on_delete, sess->keep_nport_handle,
-		sess->plogi_ack_needed);
+		sess->send_els_logo);
 
 	BUG_ON(!tgt);
 
+	if (sess->send_els_logo) {
+		qlt_port_logo_t logo;
+		logo.id = sess->s_id;
+		logo.cmd_count = 0;
+		qlt_send_first_logo(vha, &logo);
+	}
+
 	if (sess->logout_on_delete) {
 		int rc;
 
@@ -455,9 +590,34 @@ static void qlt_free_session_done(struct work_struct *work)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
-	if (sess->plogi_ack_needed)
-		qlt_send_notify_ack(vha, &sess->tm_iocb,
-				    0, 0, 0, 0, 0, 0);
+	{
+		qlt_plogi_ack_t *own =
+		    sess->plogi_link[QLT_PLOGI_LINK_SAME_WWN];
+		qlt_plogi_ack_t *con =
+		    sess->plogi_link[QLT_PLOGI_LINK_CONFLICT];
+
+		if (con) {
+			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf099,
+			    "se_sess %p / sess %p port %8phC is gone,"
+			    " %s (ref=%d), releasing PLOGI for %8phC (ref=%d)\n",
+			    sess->se_sess, sess, sess->port_name,
+			    own ? "releasing own PLOGI" :
+			    "no own PLOGI pending",
+			    own ? own->ref_count : -1,
+			    con->iocb.u.isp24.port_name, con->ref_count);
+			qlt_plogi_ack_unref(vha, con);
+		} else {
+			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf09a,
+			    "se_sess %p / sess %p port %8phC is gone, %s (ref=%d)\n",
+			    sess->se_sess, sess, sess->port_name,
+			    own ? "releasing own PLOGI" :
+			    "no own PLOGI pending",
+			    own ? own->ref_count : -1);
+		}
+
+		if (own)
+			qlt_plogi_ack_unref(vha, own);
+	}
 
 	list_del(&sess->sess_list_entry);
 
@@ -476,7 +636,7 @@ static void qlt_free_session_done(struct work_struct *work)
 		wake_up_all(&tgt->waitQ);
 }
 
-/* ha->hardware_lock supposed to be held on entry */
+/* ha->tgt.sess_lock supposed to be held on entry */
 void qlt_unreg_sess(struct qla_tgt_sess *sess)
 {
 	struct scsi_qla_host *vha = sess->vha;
@@ -492,7 +652,7 @@ void qlt_unreg_sess(struct qla_tgt_sess *sess)
 }
 EXPORT_SYMBOL(qlt_unreg_sess);
 
-/* ha->hardware_lock supposed to be held on entry */
+
 static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd)
 {
 	struct qla_hw_data *ha = vha->hw;
@@ -502,12 +662,15 @@ static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd)
 	int res = 0;
 	struct imm_ntfy_from_isp *n = (struct imm_ntfy_from_isp *)iocb;
 	struct atio_from_isp *a = (struct atio_from_isp *)iocb;
+	unsigned long flags;
 
 	loop_id = le16_to_cpu(n->u.isp24.nport_handle);
 	if (loop_id == 0xFFFF) {
 		/* Global event */
 		atomic_inc(&vha->vha_tgt.qla_tgt->tgt_global_resets_count);
+		spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 		qlt_clear_tgt_db(vha->vha_tgt.qla_tgt);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 #if 0 /* FIXME: do we need to choose a session here? */
 		if (!list_empty(&ha->tgt.qla_tgt->sess_list)) {
 			sess = list_entry(ha->tgt.qla_tgt->sess_list.next,
@@ -534,7 +697,9 @@ static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd)
 			sess = NULL;
 #endif
 	} else {
+		spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 		sess = ha->tgt.tgt_ops->find_sess_by_loop_id(vha, loop_id);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	}
 
 	ql_dbg(ql_dbg_tgt, vha, 0xe000,
@@ -556,7 +721,7 @@ static int qlt_reset(struct scsi_qla_host *vha, void *iocb, int mcmd)
 	    iocb, QLA24XX_MGMT_SEND_NACK);
 }
 
-/* ha->hardware_lock supposed to be held on entry */
+/* ha->tgt.sess_lock supposed to be held on entry */
 static void qlt_schedule_sess_for_deletion(struct qla_tgt_sess *sess,
 	bool immediate)
 {
@@ -600,7 +765,7 @@ static void qlt_schedule_sess_for_deletion(struct qla_tgt_sess *sess,
 		    sess->expires - jiffies);
 }
 
-/* ha->hardware_lock supposed to be held on entry */
+/* ha->tgt.sess_lock supposed to be held on entry */
 static void qlt_clear_tgt_db(struct qla_tgt *tgt)
 {
 	struct qla_tgt_sess *sess;
@@ -636,12 +801,12 @@ static int qla24xx_get_loop_id(struct scsi_qla_host *vha, const uint8_t *s_id,
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf045,
 		    "qla_target(%d): get_id_list() failed: %x\n",
 		    vha->vp_idx, rc);
-		res = -1;
+		res = -EBUSY;
 		goto out_free_id_list;
 	}
 
 	id_iter = (char *)gid_list;
-	res = -1;
+	res = -ENOENT;
 	for (i = 0; i < entries; i++) {
 		struct gid_list_info *gid = (struct gid_list_info *)id_iter;
 		if ((gid->al_pa == s_id[2]) &&
@@ -660,7 +825,7 @@ out_free_id_list:
 	return res;
 }
 
-/* ha->hardware_lock supposed to be held on entry */
+/* ha->tgt.sess_lock supposed to be held on entry */
 static void qlt_undelete_sess(struct qla_tgt_sess *sess)
 {
 	BUG_ON(sess->deleted != QLA_SESS_DELETION_PENDING);
@@ -678,7 +843,7 @@ static void qlt_del_sess_work_fn(struct delayed_work *work)
 	struct qla_tgt_sess *sess;
 	unsigned long flags, elapsed;
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	while (!list_empty(&tgt->del_sess_list)) {
 		sess = list_entry(tgt->del_sess_list.next, typeof(*sess),
 		    del_list_entry);
@@ -699,7 +864,7 @@ static void qlt_del_sess_work_fn(struct delayed_work *work)
 			break;
 		}
 	}
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 /*
@@ -717,7 +882,7 @@ static struct qla_tgt_sess *qlt_create_sess(
 	unsigned char be_sid[3];
 
 	/* Check to avoid double sessions */
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	list_for_each_entry(sess, &vha->vha_tgt.qla_tgt->sess_list,
 				sess_list_entry) {
 		if (!memcmp(sess->port_name, fcport->port_name, WWN_SIZE)) {
@@ -732,7 +897,7 @@ static struct qla_tgt_sess *qlt_create_sess(
 
 			/* Cannot undelete at this point */
 			if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
-				spin_unlock_irqrestore(&ha->hardware_lock,
+				spin_unlock_irqrestore(&ha->tgt.sess_lock,
 				    flags);
 				return NULL;
 			}
@@ -749,12 +914,12 @@ static struct qla_tgt_sess *qlt_create_sess(
 
 			qlt_do_generation_tick(vha, &sess->generation);
 
-			spin_unlock_irqrestore(&ha->hardware_lock, flags);
+			spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
 			return sess;
 		}
 	}
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
 	sess = kzalloc(sizeof(*sess), GFP_KERNEL);
 	if (!sess) {
@@ -799,7 +964,7 @@ static struct qla_tgt_sess *qlt_create_sess(
 	}
 	/*
 	 * Take an extra reference to ->sess_kref here to handle qla_tgt_sess
-	 * access across ->hardware_lock reaquire.
+	 * access across ->tgt.sess_lock reaquire.
 	 */
 	kref_get(&sess->se_sess->sess_kref);
 
@@ -807,11 +972,11 @@ static struct qla_tgt_sess *qlt_create_sess(
 	BUILD_BUG_ON(sizeof(sess->port_name) != sizeof(fcport->port_name));
 	memcpy(sess->port_name, fcport->port_name, sizeof(sess->port_name));
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	list_add_tail(&sess->sess_list_entry, &vha->vha_tgt.qla_tgt->sess_list);
 	vha->vha_tgt.qla_tgt->sess_count++;
 	qlt_do_generation_tick(vha, &sess->generation);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
 	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf04b,
 	    "qla_target(%d): %ssession for wwn %8phC (loop_id %d, "
@@ -842,23 +1007,23 @@ void qlt_fc_port_added(struct scsi_qla_host *vha, fc_port_t *fcport)
 	if (qla_ini_mode_enabled(vha))
 		return;
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	if (tgt->tgt_stop) {
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 		return;
 	}
 	sess = qlt_find_sess_by_port_name(tgt, fcport->port_name);
 	if (!sess) {
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
 		mutex_lock(&vha->vha_tgt.tgt_mutex);
 		sess = qlt_create_sess(vha, fcport, false);
 		mutex_unlock(&vha->vha_tgt.tgt_mutex);
 
-		spin_lock_irqsave(&ha->hardware_lock, flags);
+		spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	} else if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
 		/* Point of no return */
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 		return;
 	} else {
 		kref_get(&sess->se_sess->sess_kref);
@@ -887,7 +1052,7 @@ void qlt_fc_port_added(struct scsi_qla_host *vha, fc_port_t *fcport)
 		sess->local = 0;
 	}
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 /*
@@ -899,6 +1064,7 @@ qlt_fc_port_deleted(struct scsi_qla_host *vha, fc_port_t *fcport, int max_gen)
 {
 	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
 	struct qla_tgt_sess *sess;
+	unsigned long flags;
 
 	if (!vha->hw->tgt.tgt_ops)
 		return;
@@ -906,15 +1072,19 @@ qlt_fc_port_deleted(struct scsi_qla_host *vha, fc_port_t *fcport, int max_gen)
 	if (!tgt)
 		return;
 
+	spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 	if (tgt->tgt_stop) {
+		spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 		return;
 	}
 	sess = qlt_find_sess_by_port_name(tgt, fcport->port_name);
 	if (!sess) {
+		spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 		return;
 	}
 
 	if (max_gen - sess->generation < 0) {
+		spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf092,
 		    "Ignoring stale deletion request for se_sess %p / sess %p"
 		    " for port %8phC, req_gen %d, sess_gen %d\n",
@@ -927,6 +1097,7 @@ qlt_fc_port_deleted(struct scsi_qla_host *vha, fc_port_t *fcport, int max_gen)
 
 	sess->local = 1;
 	qlt_schedule_sess_for_deletion(sess, false);
+	spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 }
 
 static inline int test_tgt_sess_count(struct qla_tgt *tgt)
@@ -984,10 +1155,10 @@ int qlt_stop_phase1(struct qla_tgt *tgt)
 	 * Lock is needed, because we still can get an incoming packet.
 	 */
 	mutex_lock(&vha->vha_tgt.tgt_mutex);
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	tgt->tgt_stop = 1;
 	qlt_clear_tgt_db(tgt);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	mutex_unlock(&vha->vha_tgt.tgt_mutex);
 	mutex_unlock(&qla_tgt_mutex);
 
@@ -1040,7 +1211,7 @@ void qlt_stop_phase2(struct qla_tgt *tgt)
 
 	mutex_lock(&vha->vha_tgt.tgt_mutex);
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	while (tgt->irq_cmd_count != 0) {
+	while ((tgt->irq_cmd_count != 0) || (tgt->atio_irq_cmd_count != 0)) {
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 		udelay(2);
 		spin_lock_irqsave(&ha->hardware_lock, flags);
@@ -1309,7 +1480,7 @@ static int abort_cmd_for_tag(struct scsi_qla_host *vha, uint32_t tag)
 
 	list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
 		if (tag == cmd->atio.u.isp24.exchange_addr) {
-			cmd->state = QLA_TGT_STATE_ABORTED;
+			cmd->aborted = 1;
 			spin_unlock(&vha->cmd_list_lock);
 			return 1;
 		}
@@ -1351,7 +1522,7 @@ static void abort_cmds_for_lun(struct scsi_qla_host *vha,
 		cmd_lun = scsilun_to_int(
 			(struct scsi_lun *)&cmd->atio.u.isp24.fcp_cmnd.lun);
 		if (cmd_key == key && cmd_lun == lun)
-			cmd->state = QLA_TGT_STATE_ABORTED;
+			cmd->aborted = 1;
 	}
 	spin_unlock(&vha->cmd_list_lock);
 }
@@ -1435,6 +1606,7 @@ static void qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 	uint32_t tag = abts->exchange_addr_to_abort;
 	uint8_t s_id[3];
 	int rc;
+	unsigned long flags;
 
 	if (le32_to_cpu(abts->fcp_hdr_le.parameter) & ABTS_PARAM_ABORT_SEQ) {
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf053,
@@ -1462,6 +1634,7 @@ static void qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 	s_id[1] = abts->fcp_hdr_le.s_id[1];
 	s_id[2] = abts->fcp_hdr_le.s_id[0];
 
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha, s_id);
 	if (!sess) {
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf012,
@@ -1469,12 +1642,17 @@ static void qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 		    vha->vp_idx);
 		rc = qlt_sched_sess_work(vha->vha_tgt.qla_tgt,
 		    QLA_TGT_SESS_WORK_ABORT, abts, sizeof(*abts));
+
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
 		if (rc != 0) {
 			qlt_24xx_send_abts_resp(vha, abts, FCP_TMF_REJECTED,
 			    false);
 		}
 		return;
 	}
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
 
 	if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
 		qlt_24xx_send_abts_resp(vha, abts, FCP_TMF_REJECTED, false);
@@ -1560,15 +1738,15 @@ void qlt_xmit_tm_rsp(struct qla_tgt_mgmt_cmd *mcmd)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
-	if (qla2x00_reset_active(vha) || mcmd->reset_count != ha->chip_reset) {
+	if (!vha->flags.online || mcmd->reset_count != ha->chip_reset) {
 		/*
-		 * Either a chip reset is active or this request was from
+		 * Either the port is not online or this request was from
 		 * previous life, just abort the processing.
 		 */
 		ql_dbg(ql_dbg_async, vha, 0xe100,
-			"RESET-TMR active/old-count/new-count = %d/%d/%d.\n",
-			qla2x00_reset_active(vha), mcmd->reset_count,
-			ha->chip_reset);
+			"RESET-TMR online/active/old-count/new-count = %d/%d/%d/%d.\n",
+			vha->flags.online, qla2x00_reset_active(vha),
+			mcmd->reset_count, ha->chip_reset);
 		ha->tgt.tgt_ops->free_mcmd(mcmd);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 		return;
@@ -2510,17 +2688,22 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type,
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
-	if (qla2x00_reset_active(vha) || cmd->reset_count != ha->chip_reset) {
+	if (xmit_type == QLA_TGT_XMIT_STATUS)
+		vha->tgt_counters.core_qla_snd_status++;
+	else
+		vha->tgt_counters.core_qla_que_buf++;
+
+	if (!vha->flags.online || cmd->reset_count != ha->chip_reset) {
 		/*
-		 * Either a chip reset is active or this request was from
+		 * Either the port is not online or this request was from
 		 * previous life, just abort the processing.
 		 */
 		cmd->state = QLA_TGT_STATE_PROCESSED;
 		qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
 		ql_dbg(ql_dbg_async, vha, 0xe101,
-			"RESET-RSP active/old-count/new-count = %d/%d/%d.\n",
-			qla2x00_reset_active(vha), cmd->reset_count,
-			ha->chip_reset);
+			"RESET-RSP online/active/old-count/new-count = %d/%d/%d/%d.\n",
+			vha->flags.online, qla2x00_reset_active(vha),
+			cmd->reset_count, ha->chip_reset);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 		return 0;
 	}
@@ -2651,18 +2834,18 @@ int qlt_rdy_to_xfer(struct qla_tgt_cmd *cmd)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
-	if (qla2x00_reset_active(vha) || (cmd->reset_count != ha->chip_reset) ||
+	if (!vha->flags.online || (cmd->reset_count != ha->chip_reset) ||
 	    (cmd->sess && cmd->sess->deleted == QLA_SESS_DELETION_IN_PROGRESS)) {
 		/*
-		 * Either a chip reset is active or this request was from
+		 * Either the port is not online or this request was from
 		 * previous life, just abort the processing.
 		 */
 		cmd->state = QLA_TGT_STATE_NEED_DATA;
 		qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
 		ql_dbg(ql_dbg_async, vha, 0xe102,
-			"RESET-XFR active/old-count/new-count = %d/%d/%d.\n",
-			qla2x00_reset_active(vha), cmd->reset_count,
-			ha->chip_reset);
+			"RESET-XFR online/active/old-count/new-count = %d/%d/%d/%d.\n",
+			vha->flags.online, qla2x00_reset_active(vha),
+			cmd->reset_count, ha->chip_reset);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 		return 0;
 	}
@@ -2957,12 +3140,13 @@ static int __qlt_send_term_exchange(struct scsi_qla_host *vha,
 			ret = 1;
 	}
 
+	vha->tgt_counters.num_term_xchg_sent++;
 	pkt->entry_count = 1;
 	pkt->handle = QLA_TGT_SKIP_HANDLE | CTIO_COMPLETION_HANDLE_MARK;
 
 	ctio24 = (struct ctio7_to_24xx *)pkt;
 	ctio24->entry_type = CTIO_TYPE7;
-	ctio24->nport_handle = cmd ? cmd->loop_id : CTIO7_NHANDLE_UNRECOGNIZED;
+	ctio24->nport_handle = CTIO7_NHANDLE_UNRECOGNIZED;
 	ctio24->timeout = cpu_to_le16(QLA_TGT_TIMEOUT);
 	ctio24->vp_index = vha->vp_idx;
 	ctio24->initiator_id[0] = atio->u.isp24.fcp_hdr.s_id[2];
@@ -3009,7 +3193,7 @@ static void qlt_send_term_exchange(struct scsi_qla_host *vha,
 		qlt_alloc_qfull_cmd(vha, atio, 0, 0);
 
 done:
-	if (cmd && ((cmd->state != QLA_TGT_STATE_ABORTED) ||
+	if (cmd && (!cmd->aborted ||
 	    !cmd->cmd_sent_to_fw)) {
 		if (cmd->sg_mapped)
 			qlt_unmap_sg(vha, cmd);
@@ -3028,7 +3212,7 @@ static void qlt_init_term_exchange(struct scsi_qla_host *vha)
 	struct qla_tgt_cmd *cmd, *tcmd;
 
 	vha->hw->tgt.leak_exchg_thresh_hold =
-	    (vha->hw->fw_xcb_count/100) * LEAK_EXCHG_THRESH_HOLD_PERCENT;
+	    (vha->hw->cur_fw_xcb_count/100) * LEAK_EXCHG_THRESH_HOLD_PERCENT;
 
 	cmd = tcmd = NULL;
 	if (!list_empty(&vha->hw->tgt.q_full_list)) {
@@ -3058,7 +3242,7 @@ static void qlt_chk_exch_leak_thresh_hold(struct scsi_qla_host *vha)
 
 		ql_dbg(ql_dbg_tgt, vha, 0xe079,
 		    "Chip reset due to exchange starvation: %d/%d.\n",
-		    total_leaked, vha->hw->fw_xcb_count);
+		    total_leaked, vha->hw->cur_fw_xcb_count);
 
 		if (IS_P3P_TYPE(vha->hw))
 			set_bit(FCOE_CTX_RESET_NEEDED, &vha->dpc_flags);
@@ -3080,7 +3264,7 @@ void qlt_abort_cmd(struct qla_tgt_cmd *cmd)
 	    "(se_cmd=%p, tag=%llu)", vha->vp_idx, cmd, &cmd->se_cmd,
 	    se_cmd->tag);
 
-	cmd->state = QLA_TGT_STATE_ABORTED;
+	cmd->aborted = 1;
 	cmd->cmd_flags |= BIT_6;
 
 	qlt_send_term_exchange(vha, cmd, &cmd->atio, 0);
@@ -3300,9 +3484,6 @@ qlt_abort_cmd_on_host_reset(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd)
 
 		ha->tgt.tgt_ops->handle_data(cmd);
 		return;
-	} else if (cmd->state == QLA_TGT_STATE_ABORTED) {
-		ql_dbg(ql_dbg_io, vha, 0xff02,
-		    "HOST-ABORT: handle=%d, state=ABORTED.\n", handle);
 	} else {
 		ql_dbg(ql_dbg_io, vha, 0xff03,
 		    "HOST-ABORT: handle=%d, state=BAD(%d).\n", handle,
@@ -3398,13 +3579,26 @@ static void qlt_do_ctio_completion(struct scsi_qla_host *vha, uint32_t handle,
 
 		case CTIO_PORT_LOGGED_OUT:
 		case CTIO_PORT_UNAVAILABLE:
+		{
+			int logged_out = (status & 0xFFFF);
 			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf059,
-			    "qla_target(%d): CTIO with PORT LOGGED "
-			    "OUT (29) or PORT UNAVAILABLE (28) status %x "
+			    "qla_target(%d): CTIO with %s status %x "
 			    "received (state %x, se_cmd %p)\n", vha->vp_idx,
+			    (logged_out == CTIO_PORT_LOGGED_OUT) ?
+			    "PORT LOGGED OUT" : "PORT UNAVAILABLE",
 			    status, cmd->state, se_cmd);
-			break;
 
+			if (logged_out && cmd->sess) {
+				/*
+				 * Session is already logged out, but we need
+				 * to notify initiator, who's not aware of this
+				 */
+				cmd->sess->logout_on_delete = 0;
+				cmd->sess->send_els_logo = 1;
+				qlt_schedule_sess_for_deletion(cmd->sess, true);
+			}
+			break;
+		}
 		case CTIO_SRR_RECEIVED:
 			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf05a,
 			    "qla_target(%d): CTIO with SRR_RECEIVED"
@@ -3454,14 +3648,14 @@ static void qlt_do_ctio_completion(struct scsi_qla_host *vha, uint32_t handle,
 		}
 
 
-		/* "cmd->state == QLA_TGT_STATE_ABORTED" means
+		/* "cmd->aborted" means
 		 * cmd is already aborted/terminated, we don't
 		 * need to terminate again.  The exchange is already
 		 * cleaned up/freed at FW level.  Just cleanup at driver
 		 * level.
 		 */
 		if ((cmd->state != QLA_TGT_STATE_NEED_DATA) &&
-		    (cmd->state != QLA_TGT_STATE_ABORTED)) {
+		    (!cmd->aborted)) {
 			cmd->cmd_flags |= BIT_13;
 			if (qlt_term_ctio_exchange(vha, ctio, cmd, status))
 				return;
@@ -3479,7 +3673,7 @@ skip_term:
 
 		ha->tgt.tgt_ops->handle_data(cmd);
 		return;
-	} else if (cmd->state == QLA_TGT_STATE_ABORTED) {
+	} else if (cmd->aborted) {
 		cmd->cmd_flags |= BIT_18;
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01e,
 		  "Aborted command %p (tag %lld) finished\n", cmd, se_cmd->tag);
@@ -3491,7 +3685,7 @@ skip_term:
 	}
 
 	if (unlikely(status != CTIO_SUCCESS) &&
-		(cmd->state != QLA_TGT_STATE_ABORTED)) {
+		!cmd->aborted) {
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf01f, "Finishing failed CTIO\n");
 		dump_stack();
 	}
@@ -3553,7 +3747,7 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 	if (tgt->tgt_stop)
 		goto out_term;
 
-	if (cmd->state == QLA_TGT_STATE_ABORTED) {
+	if (cmd->aborted) {
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf082,
 		    "cmd with tag %u is aborted\n",
 		    cmd->atio.u.isp24.exchange_addr);
@@ -3589,9 +3783,9 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 	/*
 	 * Drop extra session reference from qla_tgt_handle_cmd_for_atio*(
 	 */
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	return;
 
 out_term:
@@ -3606,8 +3800,11 @@ out_term:
 
 	qlt_decr_num_pend_cmds(vha);
 	percpu_ida_free(&sess->se_sess->sess_tag_pool, cmd->se_cmd.map_tag);
-	ha->tgt.tgt_ops->put_sess(sess);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
+	ha->tgt.tgt_ops->put_sess(sess);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 static void qlt_do_work(struct work_struct *work)
@@ -3692,10 +3889,8 @@ static void qlt_create_sess_from_atio(struct work_struct *work)
 		goto out_term;
 	}
 
-	mutex_lock(&vha->vha_tgt.tgt_mutex);
 	sess = qlt_make_local_sess(vha, s_id);
 	/* sess has an extra creation ref. */
-	mutex_unlock(&vha->vha_tgt.tgt_mutex);
 
 	if (!sess)
 		goto out_term;
@@ -3787,13 +3982,24 @@ static int qlt_handle_cmd_for_atio(struct scsi_qla_host *vha,
 
 	cmd->cmd_in_wq = 1;
 	cmd->cmd_flags |= BIT_0;
+	cmd->se_cmd.cpuid = -1;
 
 	spin_lock(&vha->cmd_list_lock);
 	list_add_tail(&cmd->cmd_list, &vha->qla_cmd_list);
 	spin_unlock(&vha->cmd_list_lock);
 
 	INIT_WORK(&cmd->work, qlt_do_work);
-	queue_work(qla_tgt_wq, &cmd->work);
+	if (ha->msix_count) {
+		cmd->se_cmd.cpuid = ha->tgt.rspq_vector_cpuid;
+		if (cmd->atio.u.isp24.fcp_cmnd.rddata)
+			queue_work_on(smp_processor_id(), qla_tgt_wq,
+			    &cmd->work);
+		else
+			queue_work_on(cmd->se_cmd.cpuid, qla_tgt_wq,
+			    &cmd->work);
+	} else {
+		queue_work(qla_tgt_wq, &cmd->work);
+	}
 	return 0;
 
 }
@@ -3917,13 +4123,18 @@ static int qlt_handle_task_mgmt(struct scsi_qla_host *vha, void *iocb)
 	struct qla_tgt_sess *sess;
 	uint32_t lun, unpacked_lun;
 	int fn;
+	unsigned long flags;
 
 	tgt = vha->vha_tgt.qla_tgt;
 
 	lun = a->u.isp24.fcp_cmnd.lun;
 	fn = a->u.isp24.fcp_cmnd.task_mgmt_flags;
+
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha,
 	    a->u.isp24.fcp_hdr.s_id);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
 	unpacked_lun = scsilun_to_int((struct scsi_lun *)&lun);
 
 	if (!sess) {
@@ -3987,10 +4198,14 @@ static int qlt_abort_task(struct scsi_qla_host *vha,
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_tgt_sess *sess;
 	int loop_id;
+	unsigned long flags;
 
 	loop_id = GET_TARGET_ID(ha, (struct atio_from_isp *)iocb);
 
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	sess = ha->tgt.tgt_ops->find_sess_by_loop_id(vha, loop_id);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
 	if (sess == NULL) {
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf025,
 		    "qla_target(%d): task abort for unexisting "
@@ -4022,15 +4237,6 @@ void qlt_logo_completion_handler(fc_port_t *fcport, int rc)
 	}
 }
 
-static void qlt_swap_imm_ntfy_iocb(struct imm_ntfy_from_isp *a,
-    struct imm_ntfy_from_isp *b)
-{
-	struct imm_ntfy_from_isp tmp;
-	memcpy(&tmp, a, sizeof(struct imm_ntfy_from_isp));
-	memcpy(a, b, sizeof(struct imm_ntfy_from_isp));
-	memcpy(b, &tmp, sizeof(struct imm_ntfy_from_isp));
-}
-
 /*
 * ha->hardware_lock supposed to be held on entry (to protect tgt->sess_list)
 *
@@ -4040,11 +4246,13 @@ static void qlt_swap_imm_ntfy_iocb(struct imm_ntfy_from_isp *a,
 */
 static struct qla_tgt_sess *
 qlt_find_sess_invalidate_other(struct qla_tgt *tgt, uint64_t wwn,
-    port_id_t port_id, uint16_t loop_id)
+    port_id_t port_id, uint16_t loop_id, struct qla_tgt_sess **conflict_sess)
 {
 	struct qla_tgt_sess *sess = NULL, *other_sess;
 	uint64_t other_wwn;
 
+	*conflict_sess = NULL;
+
 	list_for_each_entry(other_sess, &tgt->sess_list, sess_list_entry) {
 
 		other_wwn = wwn_to_u64(other_sess->port_name);
@@ -4072,9 +4280,10 @@ qlt_find_sess_invalidate_other(struct qla_tgt *tgt, uint64_t wwn,
 			} else {
 				/*
 				 * Another wwn used to have our s_id/loop_id
-				 * combo - kill the session, but don't log out
+				 * kill the session, but don't free the loop_id
 				 */
-				sess->logout_on_delete = 0;
+				other_sess->keep_nport_handle = 1;
+				*conflict_sess = other_sess;
 				qlt_schedule_sess_for_deletion(other_sess,
 				    true);
 			}
@@ -4119,7 +4328,7 @@ static int abort_cmds_for_s_id(struct scsi_qla_host *vha, port_id_t *s_id)
 	list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
 		uint32_t cmd_key = sid_to_key(cmd->atio.u.isp24.fcp_hdr.s_id);
 		if (cmd_key == key) {
-			cmd->state = QLA_TGT_STATE_ABORTED;
+			cmd->aborted = 1;
 			count++;
 		}
 	}
@@ -4136,12 +4345,14 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 {
 	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
 	struct qla_hw_data *ha = vha->hw;
-	struct qla_tgt_sess *sess = NULL;
+	struct qla_tgt_sess *sess = NULL, *conflict_sess = NULL;
 	uint64_t wwn;
 	port_id_t port_id;
 	uint16_t loop_id;
 	uint16_t wd3_lo;
 	int res = 0;
+	qlt_plogi_ack_t *pla;
+	unsigned long flags;
 
 	wwn = wwn_to_u64(iocb->u.isp24.port_name);
 
@@ -4165,27 +4376,20 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 		/* Mark all stale commands in qla_tgt_wq for deletion */
 		abort_cmds_for_s_id(vha, &port_id);
 
-		if (wwn)
+		if (wwn) {
+			spin_lock_irqsave(&tgt->ha->tgt.sess_lock, flags);
 			sess = qlt_find_sess_invalidate_other(tgt, wwn,
-			    port_id, loop_id);
+			    port_id, loop_id, &conflict_sess);
+			spin_unlock_irqrestore(&tgt->ha->tgt.sess_lock, flags);
+		}
 
-		if (!sess || IS_SW_RESV_ADDR(sess->s_id)) {
+		if (IS_SW_RESV_ADDR(port_id) || (!sess && !conflict_sess)) {
 			res = 1;
 			break;
 		}
 
-		if (sess->plogi_ack_needed) {
-			/*
-			 * Initiator sent another PLOGI before last PLOGI could
-			 * finish. Swap plogi iocbs and terminate old one
-			 * without acking, new one will get acked when session
-			 * deletion completes.
-			 */
-			ql_log(ql_log_warn, sess->vha, 0xf094,
-			    "sess %p received double plogi.\n", sess);
-
-			qlt_swap_imm_ntfy_iocb(iocb, &sess->tm_iocb);
-
+		pla = qlt_plogi_ack_find_add(vha, &port_id, iocb);
+		if (!pla) {
 			qlt_send_term_imm_notif(vha, iocb, 1);
 
 			res = 0;
@@ -4194,13 +4398,14 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 
 		res = 0;
 
-		/*
-		 * Save immediate Notif IOCB for Ack when sess is done
-		 * and being deleted.
-		 */
-		memcpy(&sess->tm_iocb, iocb, sizeof(sess->tm_iocb));
-		sess->plogi_ack_needed  = 1;
+		if (conflict_sess)
+			qlt_plogi_ack_link(vha, pla, conflict_sess,
+			    QLT_PLOGI_LINK_CONFLICT);
+
+		if (!sess)
+			break;
 
+		qlt_plogi_ack_link(vha, pla, sess, QLT_PLOGI_LINK_SAME_WWN);
 		 /*
 		  * Under normal circumstances we want to release nport handle
 		  * during LOGO process to avoid nport handle leaks inside FW.
@@ -4227,9 +4432,21 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 	case ELS_PRLI:
 		wd3_lo = le16_to_cpu(iocb->u.isp24.u.prli.wd3_lo);
 
-		if (wwn)
+		if (wwn) {
+			spin_lock_irqsave(&tgt->ha->tgt.sess_lock, flags);
 			sess = qlt_find_sess_invalidate_other(tgt, wwn, port_id,
-			    loop_id);
+			    loop_id, &conflict_sess);
+			spin_unlock_irqrestore(&tgt->ha->tgt.sess_lock, flags);
+		}
+
+		if (conflict_sess) {
+			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf09b,
+			    "PRLI with conflicting sess %p port %8phC\n",
+			    conflict_sess, conflict_sess->port_name);
+			qlt_send_term_imm_notif(vha, iocb, 1);
+			res = 0;
+			break;
+		}
 
 		if (sess != NULL) {
 			if (sess->deleted) {
@@ -4899,9 +5116,12 @@ static int __qlt_send_busy(struct scsi_qla_host *vha,
 	struct qla_hw_data *ha = vha->hw;
 	request_t *pkt;
 	struct qla_tgt_sess *sess = NULL;
+	unsigned long flags;
 
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha,
 	    atio->u.isp24.fcp_hdr.s_id);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	if (!sess) {
 		qlt_send_term_exchange(vha, NULL, atio, 1);
 		return 0;
@@ -4916,6 +5136,7 @@ static int __qlt_send_busy(struct scsi_qla_host *vha,
 		return -ENOMEM;
 	}
 
+	vha->tgt_counters.num_q_full_sent++;
 	pkt->entry_count = 1;
 	pkt->handle = QLA_TGT_SKIP_HANDLE | CTIO_COMPLETION_HANDLE_MARK;
 
@@ -5129,11 +5350,12 @@ qlt_chk_qfull_thresh_hold(struct scsi_qla_host *vha,
 /* ha->hardware_lock supposed to be held on entry */
 /* called via callback from qla2xxx */
 static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
-	struct atio_from_isp *atio)
+	struct atio_from_isp *atio, uint8_t ha_locked)
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
 	int rc;
+	unsigned long flags;
 
 	if (unlikely(tgt == NULL)) {
 		ql_dbg(ql_dbg_io, vha, 0x3064,
@@ -5145,7 +5367,7 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 	 * Otherwise, some commands can stuck.
 	 */
 
-	tgt->irq_cmd_count++;
+	tgt->atio_irq_cmd_count++;
 
 	switch (atio->u.raw.entry_type) {
 	case ATIO_TYPE7:
@@ -5155,7 +5377,11 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 			    "qla_target(%d): ATIO_TYPE7 "
 			    "received with UNKNOWN exchange address, "
 			    "sending QUEUE_FULL\n", vha->vp_idx);
+			if (!ha_locked)
+				spin_lock_irqsave(&ha->hardware_lock, flags);
 			qlt_send_busy(vha, atio, SAM_STAT_TASK_SET_FULL);
+			if (!ha_locked)
+				spin_unlock_irqrestore(&ha->hardware_lock, flags);
 			break;
 		}
 
@@ -5164,7 +5390,7 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 		if (likely(atio->u.isp24.fcp_cmnd.task_mgmt_flags == 0)) {
 			rc = qlt_chk_qfull_thresh_hold(vha, atio);
 			if (rc != 0) {
-				tgt->irq_cmd_count--;
+				tgt->atio_irq_cmd_count--;
 				return;
 			}
 			rc = qlt_handle_cmd_for_atio(vha, atio);
@@ -5173,11 +5399,20 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 		}
 		if (unlikely(rc != 0)) {
 			if (rc == -ESRCH) {
+				if (!ha_locked)
+					spin_lock_irqsave
+						(&ha->hardware_lock, flags);
+
 #if 1 /* With TERM EXCHANGE some FC cards refuse to boot */
 				qlt_send_busy(vha, atio, SAM_STAT_BUSY);
 #else
 				qlt_send_term_exchange(vha, NULL, atio, 1);
 #endif
+
+				if (!ha_locked)
+					spin_unlock_irqrestore
+						(&ha->hardware_lock, flags);
+
 			} else {
 				if (tgt->tgt_stop) {
 					ql_dbg(ql_dbg_tgt, vha, 0xe059,
@@ -5189,7 +5424,13 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 					    "qla_target(%d): Unable to send "
 					    "command to target, sending BUSY "
 					    "status.\n", vha->vp_idx);
+					if (!ha_locked)
+						spin_lock_irqsave(
+						    &ha->hardware_lock, flags);
 					qlt_send_busy(vha, atio, SAM_STAT_BUSY);
+					if (!ha_locked)
+						spin_unlock_irqrestore(
+						    &ha->hardware_lock, flags);
 				}
 			}
 		}
@@ -5206,7 +5447,12 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 			break;
 		}
 		ql_dbg(ql_dbg_tgt, vha, 0xe02e, "%s", "IMMED_NOTIFY ATIO");
+
+		if (!ha_locked)
+			spin_lock_irqsave(&ha->hardware_lock, flags);
 		qlt_handle_imm_notify(vha, (struct imm_ntfy_from_isp *)atio);
+		if (!ha_locked)
+			spin_unlock_irqrestore(&ha->hardware_lock, flags);
 		break;
 	}
 
@@ -5217,7 +5463,7 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 		break;
 	}
 
-	tgt->irq_cmd_count--;
+	tgt->atio_irq_cmd_count--;
 }
 
 /* ha->hardware_lock supposed to be held on entry */
@@ -5534,12 +5780,16 @@ static struct qla_tgt_sess *qlt_make_local_sess(struct scsi_qla_host *vha,
 	int rc, global_resets;
 	uint16_t loop_id = 0;
 
+	mutex_lock(&vha->vha_tgt.tgt_mutex);
+
 retry:
 	global_resets =
 	    atomic_read(&vha->vha_tgt.qla_tgt->tgt_global_resets_count);
 
 	rc = qla24xx_get_loop_id(vha, s_id, &loop_id);
 	if (rc != 0) {
+		mutex_unlock(&vha->vha_tgt.tgt_mutex);
+
 		if ((s_id[0] == 0xFF) &&
 		    (s_id[1] == 0xFC)) {
 			/*
@@ -5550,17 +5800,27 @@ retry:
 			    "Unable to find initiator with S_ID %x:%x:%x",
 			    s_id[0], s_id[1], s_id[2]);
 		} else
-			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf071,
+			ql_log(ql_log_info, vha, 0xf071,
 			    "qla_target(%d): Unable to find "
 			    "initiator with S_ID %x:%x:%x",
 			    vha->vp_idx, s_id[0], s_id[1],
 			    s_id[2]);
+
+		if (rc == -ENOENT) {
+			qlt_port_logo_t logo;
+			sid_to_portid(s_id, &logo.id);
+			logo.cmd_count = 1;
+			qlt_send_first_logo(vha, &logo);
+		}
+
 		return NULL;
 	}
 
 	fcport = qlt_get_port_database(vha, loop_id);
-	if (!fcport)
+	if (!fcport) {
+		mutex_unlock(&vha->vha_tgt.tgt_mutex);
 		return NULL;
+	}
 
 	if (global_resets !=
 	    atomic_read(&vha->vha_tgt.qla_tgt->tgt_global_resets_count)) {
@@ -5575,6 +5835,8 @@ retry:
 
 	sess = qlt_create_sess(vha, fcport, true);
 
+	mutex_unlock(&vha->vha_tgt.tgt_mutex);
+
 	kfree(fcport);
 	return sess;
 }
@@ -5585,15 +5847,15 @@ static void qlt_abort_work(struct qla_tgt *tgt,
 	struct scsi_qla_host *vha = tgt->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_tgt_sess *sess = NULL;
-	unsigned long flags;
+	unsigned long flags = 0, flags2 = 0;
 	uint32_t be_s_id;
 	uint8_t s_id[3];
 	int rc;
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags2);
 
 	if (tgt->tgt_stop)
-		goto out_term;
+		goto out_term2;
 
 	s_id[0] = prm->abts.fcp_hdr_le.s_id[2];
 	s_id[1] = prm->abts.fcp_hdr_le.s_id[1];
@@ -5602,41 +5864,47 @@ static void qlt_abort_work(struct qla_tgt *tgt,
 	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha,
 	    (unsigned char *)&be_s_id);
 	if (!sess) {
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
 
-		mutex_lock(&vha->vha_tgt.tgt_mutex);
 		sess = qlt_make_local_sess(vha, s_id);
 		/* sess has got an extra creation ref */
-		mutex_unlock(&vha->vha_tgt.tgt_mutex);
 
-		spin_lock_irqsave(&ha->hardware_lock, flags);
+		spin_lock_irqsave(&ha->tgt.sess_lock, flags2);
 		if (!sess)
-			goto out_term;
+			goto out_term2;
 	} else {
 		if (sess->deleted == QLA_SESS_DELETION_IN_PROGRESS) {
 			sess = NULL;
-			goto out_term;
+			goto out_term2;
 		}
 
 		kref_get(&sess->se_sess->sess_kref);
 	}
 
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+
 	if (tgt->tgt_stop)
 		goto out_term;
 
 	rc = __qlt_24xx_handle_abts(vha, &prm->abts, sess);
 	if (rc != 0)
 		goto out_term;
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
 	return;
 
+out_term2:
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+
 out_term:
 	qlt_24xx_send_abts_resp(vha, &prm->abts, FCP_TMF_REJECTED, false);
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
 	if (sess)
 		ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
 }
 
 static void qlt_tmr_work(struct qla_tgt *tgt,
@@ -5653,7 +5921,7 @@ static void qlt_tmr_work(struct qla_tgt *tgt,
 	int fn;
 	void *iocb;
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 
 	if (tgt->tgt_stop)
 		goto out_term;
@@ -5661,14 +5929,12 @@ static void qlt_tmr_work(struct qla_tgt *tgt,
 	s_id = prm->tm_iocb2.u.isp24.fcp_hdr.s_id;
 	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha, s_id);
 	if (!sess) {
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
+		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
-		mutex_lock(&vha->vha_tgt.tgt_mutex);
 		sess = qlt_make_local_sess(vha, s_id);
 		/* sess has got an extra creation ref */
-		mutex_unlock(&vha->vha_tgt.tgt_mutex);
 
-		spin_lock_irqsave(&ha->hardware_lock, flags);
+		spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 		if (!sess)
 			goto out_term;
 	} else {
@@ -5690,14 +5956,14 @@ static void qlt_tmr_work(struct qla_tgt *tgt,
 		goto out_term;
 
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	return;
 
 out_term:
-	qlt_send_term_exchange(vha, NULL, &prm->tm_iocb2, 1);
+	qlt_send_term_exchange(vha, NULL, &prm->tm_iocb2, 0);
 	if (sess)
 		ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 static void qlt_sess_work_fn(struct work_struct *work)
@@ -6002,6 +6268,7 @@ qlt_enable_vha(struct scsi_qla_host *vha)
 	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
 	unsigned long flags;
 	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
+	int rspq_ent = QLA83XX_RSPQ_MSIX_ENTRY_NUMBER;
 
 	if (!tgt) {
 		ql_dbg(ql_dbg_tgt, vha, 0xe069,
@@ -6020,6 +6287,17 @@ qlt_enable_vha(struct scsi_qla_host *vha)
 		qla24xx_disable_vp(vha);
 		qla24xx_enable_vp(vha);
 	} else {
+		if (ha->msix_entries) {
+			ql_dbg(ql_dbg_tgt, vha, 0xffff,
+			    "%s: host%ld : vector %d cpu %d\n",
+			    __func__, vha->host_no,
+			    ha->msix_entries[rspq_ent].vector,
+			    ha->msix_entries[rspq_ent].cpuid);
+
+			ha->tgt.rspq_vector_cpuid =
+			    ha->msix_entries[rspq_ent].cpuid;
+		}
+
 		set_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags);
 		qla2xxx_wake_dpc(base_vha);
 		qla2x00_wait_for_hba_online(base_vha);
@@ -6131,7 +6409,7 @@ qlt_init_atio_q_entries(struct scsi_qla_host *vha)
  * @ha: SCSI driver HA context
  */
 void
-qlt_24xx_process_atio_queue(struct scsi_qla_host *vha)
+qlt_24xx_process_atio_queue(struct scsi_qla_host *vha, uint8_t ha_locked)
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct atio_from_isp *pkt;
@@ -6144,7 +6422,8 @@ qlt_24xx_process_atio_queue(struct scsi_qla_host *vha)
 		pkt = (struct atio_from_isp *)ha->tgt.atio_ring_ptr;
 		cnt = pkt->u.raw.entry_count;
 
-		qlt_24xx_atio_pkt_all_vps(vha, (struct atio_from_isp *)pkt);
+		qlt_24xx_atio_pkt_all_vps(vha, (struct atio_from_isp *)pkt,
+		    ha_locked);
 
 		for (i = 0; i < cnt; i++) {
 			ha->tgt.atio_ring_index++;
@@ -6265,10 +6544,21 @@ qlt_24xx_config_nvram_stage2(struct scsi_qla_host *vha,
 {
 	struct qla_hw_data *ha = vha->hw;
 
+	if (!QLA_TGT_MODE_ENABLED())
+		return;
+
 	if (ha->tgt.node_name_set) {
 		memcpy(icb->node_name, ha->tgt.tgt_node_name, WWN_SIZE);
 		icb->firmware_options_1 |= cpu_to_le32(BIT_14);
 	}
+
+	/* disable ZIO at start time. */
+	if (!vha->flags.init_done) {
+		uint32_t tmp;
+		tmp = le32_to_cpu(icb->firmware_options_2);
+		tmp &= ~(BIT_3 | BIT_2 | BIT_1 | BIT_0);
+		icb->firmware_options_2 = cpu_to_le32(tmp);
+	}
 }
 
 void
@@ -6359,6 +6649,15 @@ qlt_81xx_config_nvram_stage2(struct scsi_qla_host *vha,
 		memcpy(icb->node_name, ha->tgt.tgt_node_name, WWN_SIZE);
 		icb->firmware_options_1 |= cpu_to_le32(BIT_14);
 	}
+
+	/* disable ZIO at start time. */
+	if (!vha->flags.init_done) {
+		uint32_t tmp;
+		tmp = le32_to_cpu(icb->firmware_options_2);
+		tmp &= ~(BIT_3 | BIT_2 | BIT_1 | BIT_0);
+		icb->firmware_options_2 = cpu_to_le32(tmp);
+	}
+
 }
 
 void
@@ -6428,16 +6727,59 @@ qla83xx_msix_atio_q(int irq, void *dev_id)
 	ha = rsp->hw;
 	vha = pci_get_drvdata(ha->pdev);
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.atio_lock, flags);
 
-	qlt_24xx_process_atio_queue(vha);
-	qla24xx_process_response_queue(vha, rsp);
+	qlt_24xx_process_atio_queue(vha, 0);
 
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.atio_lock, flags);
 
 	return IRQ_HANDLED;
 }
 
+static void
+qlt_handle_abts_recv_work(struct work_struct *work)
+{
+	struct qla_tgt_sess_op *op = container_of(work,
+		struct qla_tgt_sess_op, work);
+	scsi_qla_host_t *vha = op->vha;
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long flags;
+
+	if (qla2x00_reset_active(vha) || (op->chip_reset != ha->chip_reset))
+		return;
+
+	spin_lock_irqsave(&ha->tgt.atio_lock, flags);
+	qlt_24xx_process_atio_queue(vha, 0);
+	spin_unlock_irqrestore(&ha->tgt.atio_lock, flags);
+
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+	qlt_response_pkt_all_vps(vha, (response_t *)&op->atio);
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+}
+
+void
+qlt_handle_abts_recv(struct scsi_qla_host *vha, response_t *pkt)
+{
+	struct qla_tgt_sess_op *op;
+
+	op = kzalloc(sizeof(*op), GFP_ATOMIC);
+
+	if (!op) {
+		/* do not reach for ATIO queue here.  This is best effort err
+		 * recovery at this point.
+		 */
+		qlt_response_pkt_all_vps(vha, pkt);
+		return;
+	}
+
+	memcpy(&op->atio, pkt, sizeof(*pkt));
+	op->vha = vha;
+	op->chip_reset = vha->hw->chip_reset;
+	INIT_WORK(&op->work, qlt_handle_abts_recv_work);
+	queue_work(qla_tgt_wq, &op->work);
+	return;
+}
+
 int
 qlt_mem_alloc(struct qla_hw_data *ha)
 {
@@ -6532,13 +6874,25 @@ int __init qlt_init(void)
 		return -ENOMEM;
 	}
 
+	qla_tgt_plogi_cachep = kmem_cache_create("qla_tgt_plogi_cachep",
+						 sizeof(qlt_plogi_ack_t),
+						 __alignof__(qlt_plogi_ack_t),
+						 0, NULL);
+
+	if (!qla_tgt_plogi_cachep) {
+		ql_log(ql_log_fatal, NULL, 0xe06d,
+		    "kmem_cache_create for qla_tgt_plogi_cachep failed\n");
+		ret = -ENOMEM;
+		goto out_mgmt_cmd_cachep;
+	}
+
 	qla_tgt_mgmt_cmd_mempool = mempool_create(25, mempool_alloc_slab,
 	    mempool_free_slab, qla_tgt_mgmt_cmd_cachep);
 	if (!qla_tgt_mgmt_cmd_mempool) {
 		ql_log(ql_log_fatal, NULL, 0xe06e,
 		    "mempool_create for qla_tgt_mgmt_cmd_mempool failed\n");
 		ret = -ENOMEM;
-		goto out_mgmt_cmd_cachep;
+		goto out_plogi_cachep;
 	}
 
 	qla_tgt_wq = alloc_workqueue("qla_tgt_wq", 0, 0);
@@ -6555,6 +6909,8 @@ int __init qlt_init(void)
 
 out_cmd_mempool:
 	mempool_destroy(qla_tgt_mgmt_cmd_mempool);
+out_plogi_cachep:
+	kmem_cache_destroy(qla_tgt_plogi_cachep);
 out_mgmt_cmd_cachep:
 	kmem_cache_destroy(qla_tgt_mgmt_cmd_cachep);
 	return ret;
@@ -6567,5 +6923,6 @@ void qlt_exit(void)
 
 	destroy_workqueue(qla_tgt_wq);
 	mempool_destroy(qla_tgt_mgmt_cmd_mempool);
+	kmem_cache_destroy(qla_tgt_plogi_cachep);
 	kmem_cache_destroy(qla_tgt_mgmt_cmd_cachep);
 }
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index bca584ae45b7..71b2865ba3c8 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -787,7 +787,7 @@ int qla2x00_wait_for_hba_online(struct scsi_qla_host *);
 #define QLA_TGT_STATE_NEED_DATA		1 /* target needs data to continue */
 #define QLA_TGT_STATE_DATA_IN		2 /* Data arrived + target processing */
 #define QLA_TGT_STATE_PROCESSED		3 /* target done processing */
-#define QLA_TGT_STATE_ABORTED		4 /* Command aborted */
+
 
 /* Special handles */
 #define QLA_TGT_NULL_HANDLE	0
@@ -835,6 +835,7 @@ struct qla_tgt {
 	 * HW lock.
 	 */
 	int irq_cmd_count;
+	int atio_irq_cmd_count;
 
 	int datasegs_per_cmd, datasegs_per_cont, sg_tablesize;
 
@@ -883,6 +884,7 @@ struct qla_tgt {
 
 struct qla_tgt_sess_op {
 	struct scsi_qla_host *vha;
+	uint32_t chip_reset;
 	struct atio_from_isp atio;
 	struct work_struct work;
 	struct list_head cmd_list;
@@ -896,6 +898,19 @@ enum qla_sess_deletion {
 	QLA_SESS_DELETION_IN_PROGRESS	= 2,
 };
 
+typedef enum {
+	QLT_PLOGI_LINK_SAME_WWN,
+	QLT_PLOGI_LINK_CONFLICT,
+	QLT_PLOGI_LINK_MAX
+} qlt_plogi_link_t;
+
+typedef struct {
+	struct list_head		list;
+	struct imm_ntfy_from_isp	iocb;
+	port_id_t			id;
+	int				ref_count;
+} qlt_plogi_ack_t;
+
 /*
  * Equivilant to IT Nexus (Initiator-Target)
  */
@@ -907,8 +922,8 @@ struct qla_tgt_sess {
 	unsigned int deleted:2;
 	unsigned int local:1;
 	unsigned int logout_on_delete:1;
-	unsigned int plogi_ack_needed:1;
 	unsigned int keep_nport_handle:1;
+	unsigned int send_els_logo:1;
 
 	unsigned char logout_completed;
 
@@ -925,9 +940,7 @@ struct qla_tgt_sess {
 	uint8_t port_name[WWN_SIZE];
 	struct work_struct free_work;
 
-	union {
-		struct imm_ntfy_from_isp tm_iocb;
-	};
+	qlt_plogi_ack_t *plogi_link[QLT_PLOGI_LINK_MAX];
 };
 
 struct qla_tgt_cmd {
@@ -949,6 +962,7 @@ struct qla_tgt_cmd {
 	unsigned int term_exchg:1;
 	unsigned int cmd_sent_to_fw:1;
 	unsigned int cmd_in_wq:1;
+	unsigned int aborted:1;
 
 	struct scatterlist *sg;	/* cmd data buffer SG vector */
 	int sg_cnt;		/* SG segments count */
@@ -1120,6 +1134,14 @@ static inline uint32_t sid_to_key(const uint8_t *s_id)
 	return key;
 }
 
+static inline void sid_to_portid(const uint8_t *s_id, port_id_t *p)
+{
+	memset(p, 0, sizeof(*p));
+	p->b.domain = s_id[0];
+	p->b.area = s_id[1];
+	p->b.al_pa = s_id[2];
+}
+
 /*
  * Exported symbols from qla_target.c LLD logic used by qla2xxx code..
  */
@@ -1135,7 +1157,7 @@ extern void qlt_enable_vha(struct scsi_qla_host *);
 extern void qlt_vport_create(struct scsi_qla_host *, struct qla_hw_data *);
 extern void qlt_rff_id(struct scsi_qla_host *, struct ct_sns_req *);
 extern void qlt_init_atio_q_entries(struct scsi_qla_host *);
-extern void qlt_24xx_process_atio_queue(struct scsi_qla_host *);
+extern void qlt_24xx_process_atio_queue(struct scsi_qla_host *, uint8_t);
 extern void qlt_24xx_config_rings(struct scsi_qla_host *);
 extern void qlt_24xx_config_nvram_stage1(struct scsi_qla_host *,
 	struct nvram_24xx *);
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 81af294f15a7..faf0a126627f 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -284,6 +284,7 @@ static void tcm_qla2xxx_complete_free(struct work_struct *work)
 
 	WARN_ON(cmd->cmd_flags &  BIT_16);
 
+	cmd->vha->tgt_counters.qla_core_ret_sta_ctio++;
 	cmd->cmd_flags |= BIT_16;
 	transport_generic_free_cmd(&cmd->se_cmd, 0);
 }
@@ -295,9 +296,10 @@ static void tcm_qla2xxx_complete_free(struct work_struct *work)
  */
 static void tcm_qla2xxx_free_cmd(struct qla_tgt_cmd *cmd)
 {
+	cmd->vha->tgt_counters.core_qla_free_cmd++;
 	cmd->cmd_in_wq = 1;
 	INIT_WORK(&cmd->work, tcm_qla2xxx_complete_free);
-	queue_work(tcm_qla2xxx_free_wq, &cmd->work);
+	queue_work_on(smp_processor_id(), tcm_qla2xxx_free_wq, &cmd->work);
 }
 
 /*
@@ -342,9 +344,9 @@ static int tcm_qla2xxx_shutdown_session(struct se_session *se_sess)
 	BUG_ON(!sess);
 	vha = sess->vha;
 
-	spin_lock_irqsave(&vha->hw->hardware_lock, flags);
+	spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 	target_sess_cmd_list_set_waiting(se_sess);
-	spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
+	spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 
 	return 1;
 }
@@ -358,9 +360,9 @@ static void tcm_qla2xxx_close_session(struct se_session *se_sess)
 	BUG_ON(!sess);
 	vha = sess->vha;
 
-	spin_lock_irqsave(&vha->hw->hardware_lock, flags);
+	spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 	qlt_unreg_sess(sess);
-	spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
+	spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 }
 
 static u32 tcm_qla2xxx_sess_get_index(struct se_session *se_sess)
@@ -454,6 +456,7 @@ static int tcm_qla2xxx_handle_cmd(scsi_qla_host_t *vha, struct qla_tgt_cmd *cmd,
 		return -EINVAL;
 	}
 
+	cmd->vha->tgt_counters.qla_core_sbt_cmd++;
 	return target_submit_cmd(se_cmd, se_sess, cdb, &cmd->sense_buffer[0],
 				cmd->unpacked_lun, data_length, fcp_task_attr,
 				data_dir, flags);
@@ -469,6 +472,7 @@ static void tcm_qla2xxx_handle_data_work(struct work_struct *work)
 	 */
 	cmd->cmd_in_wq = 0;
 	cmd->cmd_flags |= BIT_11;
+	cmd->vha->tgt_counters.qla_core_ret_ctio++;
 	if (!cmd->write_data_transferred) {
 		/*
 		 * Check if se_cmd has already been aborted via LUN_RESET, and
@@ -500,7 +504,7 @@ static void tcm_qla2xxx_handle_data(struct qla_tgt_cmd *cmd)
 	cmd->cmd_flags |= BIT_10;
 	cmd->cmd_in_wq = 1;
 	INIT_WORK(&cmd->work, tcm_qla2xxx_handle_data_work);
-	queue_work(tcm_qla2xxx_free_wq, &cmd->work);
+	queue_work_on(smp_processor_id(), tcm_qla2xxx_free_wq, &cmd->work);
 }
 
 static void tcm_qla2xxx_handle_dif_work(struct work_struct *work)
@@ -643,7 +647,7 @@ static void tcm_qla2xxx_aborted_task(struct se_cmd *se_cmd)
 static void tcm_qla2xxx_clear_sess_lookup(struct tcm_qla2xxx_lport *,
 			struct tcm_qla2xxx_nacl *, struct qla_tgt_sess *);
 /*
- * Expected to be called with struct qla_hw_data->hardware_lock held
+ * Expected to be called with struct qla_hw_data->tgt.sess_lock held
  */
 static void tcm_qla2xxx_clear_nacl_from_fcport_map(struct qla_tgt_sess *sess)
 {
@@ -697,13 +701,13 @@ static void tcm_qla2xxx_put_sess(struct qla_tgt_sess *sess)
 	if (!sess)
 		return;
 
-	assert_spin_locked(&sess->vha->hw->hardware_lock);
+	assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
 	kref_put(&sess->se_sess->sess_kref, tcm_qla2xxx_release_session);
 }
 
 static void tcm_qla2xxx_shutdown_sess(struct qla_tgt_sess *sess)
 {
-	assert_spin_locked(&sess->vha->hw->hardware_lock);
+	assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
 	target_sess_cmd_list_set_waiting(sess->se_sess);
 }
 
@@ -1077,7 +1081,7 @@ static struct se_portal_group *tcm_qla2xxx_npiv_make_tpg(
 }
 
 /*
- * Expected to be called with struct qla_hw_data->hardware_lock held
+ * Expected to be called with struct qla_hw_data->tgt.sess_lock held
  */
 static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_s_id(
 	scsi_qla_host_t *vha,
@@ -1116,7 +1120,7 @@ static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_s_id(
 }
 
 /*
- * Expected to be called with struct qla_hw_data->hardware_lock held
+ * Expected to be called with struct qla_hw_data->tgt.sess_lock held
  */
 static void tcm_qla2xxx_set_sess_by_s_id(
 	struct tcm_qla2xxx_lport *lport,
@@ -1182,7 +1186,7 @@ static void tcm_qla2xxx_set_sess_by_s_id(
 }
 
 /*
- * Expected to be called with struct qla_hw_data->hardware_lock held
+ * Expected to be called with struct qla_hw_data->tgt.sess_lock held
  */
 static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_loop_id(
 	scsi_qla_host_t *vha,
@@ -1221,7 +1225,7 @@ static struct qla_tgt_sess *tcm_qla2xxx_find_sess_by_loop_id(
 }
 
 /*
- * Expected to be called with struct qla_hw_data->hardware_lock held
+ * Expected to be called with struct qla_hw_data->tgt.sess_lock held
  */
 static void tcm_qla2xxx_set_sess_by_loop_id(
 	struct tcm_qla2xxx_lport *lport,
@@ -1285,7 +1289,7 @@ static void tcm_qla2xxx_set_sess_by_loop_id(
 }
 
 /*
- * Should always be called with qla_hw_data->hardware_lock held.
+ * Should always be called with qla_hw_data->tgt.sess_lock held.
  */
 static void tcm_qla2xxx_clear_sess_lookup(struct tcm_qla2xxx_lport *lport,
 		struct tcm_qla2xxx_nacl *nacl, struct qla_tgt_sess *sess)
@@ -1353,7 +1357,7 @@ static int tcm_qla2xxx_check_initiator_node_acl(
 	struct qla_tgt_sess *sess = qla_tgt_sess;
 	unsigned char port_name[36];
 	unsigned long flags;
-	int num_tags = (ha->fw_xcb_count) ? ha->fw_xcb_count :
+	int num_tags = (ha->cur_fw_xcb_count) ? ha->cur_fw_xcb_count :
 		       TCM_QLA2XXX_DEFAULT_TAGS;
 
 	lport = vha->vha_tgt.target_lport_ptr;
@@ -1401,12 +1405,12 @@ static int tcm_qla2xxx_check_initiator_node_acl(
 	 * And now setup the new se_nacl and session pointers into our HW lport
 	 * mappings for fabric S_ID and LOOP_ID.
 	 */
-	spin_lock_irqsave(&ha->hardware_lock, flags);
+	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	tcm_qla2xxx_set_sess_by_s_id(lport, se_nacl, nacl, se_sess,
 			qla_tgt_sess, s_id);
 	tcm_qla2xxx_set_sess_by_loop_id(lport, se_nacl, nacl, se_sess,
 			qla_tgt_sess, loop_id);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	/*
 	 * Finally register the new FC Nexus with TCM
 	 */
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index 2c1160c7ec92..47b9d13f97b8 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -227,6 +227,7 @@ static struct {
 	{"Promise", "VTrak E610f", NULL, BLIST_SPARSELUN | BLIST_NO_RSOC},
 	{"Promise", "", NULL, BLIST_SPARSELUN},
 	{"QNAP", "iSCSI Storage", NULL, BLIST_MAX_1024},
+	{"SYNOLOGY", "iSCSI Storage", NULL, BLIST_MAX_1024},
 	{"QUANTUM", "XP34301", "1071", BLIST_NOTQ},
 	{"REGAL", "CDC-4X", NULL, BLIST_MAX5LUN | BLIST_SINGLELUN},
 	{"SanDisk", "ImageMate CF-SD1", NULL, BLIST_FORCELUN},
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 4e08d1cd704d..bb669d32ccd0 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -2893,7 +2893,7 @@ static int sd_revalidate_disk(struct gendisk *disk)
 	    sdkp->opt_xfer_blocks <= SD_DEF_XFER_BLOCKS &&
 	    sdkp->opt_xfer_blocks * sdp->sector_size >= PAGE_CACHE_SIZE)
 		rw_max = q->limits.io_opt =
-			logical_to_sectors(sdp, sdkp->opt_xfer_blocks);
+			sdkp->opt_xfer_blocks * sdp->sector_size;
 	else
 		rw_max = BLK_DEF_MAX_SECTORS;
 
@@ -3268,8 +3268,8 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors)
 	struct scsi_disk *sdkp = dev_get_drvdata(dev);
 	int ret = 0;
 
-	if (!sdkp)
-		return 0;	/* this can happen */
+	if (!sdkp)	/* E.g.: runtime suspend following sd_remove() */
+		return 0;
 
 	if (sdkp->WCE && sdkp->media_present) {
 		sd_printk(KERN_NOTICE, sdkp, "Synchronizing SCSI cache\n");
@@ -3308,6 +3308,9 @@ static int sd_resume(struct device *dev)
 {
 	struct scsi_disk *sdkp = dev_get_drvdata(dev);
 
+	if (!sdkp)	/* E.g.: runtime resume at the start of sd_probe() */
+		return 0;
+
 	if (!sdkp->device->manage_start_stop)
 		return 0;
 
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index 8bd54a64efd6..64c867405ad4 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -144,6 +144,9 @@ static int sr_runtime_suspend(struct device *dev)
 {
 	struct scsi_cd *cd = dev_get_drvdata(dev);
 
+	if (!cd)	/* E.g.: runtime suspend following sr_remove() */
+		return 0;
+
 	if (cd->media_present)
 		return -EBUSY;
 	else
@@ -985,6 +988,7 @@ static int sr_remove(struct device *dev)
 	scsi_autopm_get_device(cd->device);
 
 	del_gendisk(cd->disk);
+	dev_set_drvdata(dev, NULL);
 
 	mutex_lock(&sr_ref_mutex);
 	kref_put(&cd->kref, sr_kref_release);
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 41c115c230d9..55627d097873 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -390,7 +390,7 @@ module_param(storvsc_ringbuffer_size, int, S_IRUGO);
 MODULE_PARM_DESC(storvsc_ringbuffer_size, "Ring buffer size (bytes)");
 
 module_param(storvsc_vcpus_per_sub_channel, int, S_IRUGO);
-MODULE_PARM_DESC(vcpus_per_sub_channel, "Ratio of VCPUs to subchannels");
+MODULE_PARM_DESC(storvsc_vcpus_per_sub_channel, "Ratio of VCPUs to subchannels");
 /*
  * Timeout in seconds for all devices managed by this driver.
  */
diff --git a/drivers/scsi/sun3_scsi.c b/drivers/scsi/sun3_scsi.c
index 22a42836d193..b9de487bbd31 100644
--- a/drivers/scsi/sun3_scsi.c
+++ b/drivers/scsi/sun3_scsi.c
@@ -53,13 +53,12 @@
 #define NCR5380_queue_command           sun3scsi_queue_command
 #define NCR5380_bus_reset               sun3scsi_bus_reset
 #define NCR5380_abort                   sun3scsi_abort
-#define NCR5380_show_info               sun3scsi_show_info
 #define NCR5380_info                    sun3scsi_info
 
 #define NCR5380_dma_read_setup(instance, data, count) \
-        sun3scsi_dma_setup(data, count, 0)
+        sun3scsi_dma_setup(instance, data, count, 0)
 #define NCR5380_dma_write_setup(instance, data, count) \
-        sun3scsi_dma_setup(data, count, 1)
+        sun3scsi_dma_setup(instance, data, count, 1)
 #define NCR5380_dma_residual(instance) \
         sun3scsi_dma_residual(instance)
 #define NCR5380_dma_xfer_len(instance, cmd, phase) \
@@ -86,10 +85,6 @@ module_param(setup_use_tagged_queuing, int, 0);
 static int setup_hostid = -1;
 module_param(setup_hostid, int, 0);
 
-/* #define RESET_BOOT */
-
-#define	AFTER_RESET_DELAY	(HZ/2)
-
 /* ms to wait after hitting dma regs */
 #define SUN3_DMA_DELAY 10
 
@@ -100,11 +95,10 @@ static struct scsi_cmnd *sun3_dma_setup_done;
 static unsigned char *sun3_scsi_regp;
 static volatile struct sun3_dma_regs *dregs;
 static struct sun3_udc_regs *udc_regs;
-static unsigned char *sun3_dma_orig_addr = NULL;
-static unsigned long sun3_dma_orig_count = 0;
-static int sun3_dma_active = 0;
-static unsigned long last_residual = 0;
-static struct Scsi_Host *default_instance;
+static unsigned char *sun3_dma_orig_addr;
+static unsigned long sun3_dma_orig_count;
+static int sun3_dma_active;
+static unsigned long last_residual;
 
 /*
  * NCR 5380 register access functions
@@ -144,50 +138,12 @@ static inline void sun3_udc_write(unsigned short val, unsigned char reg)
 }
 #endif
 
-#ifdef RESET_BOOT
-static void sun3_scsi_reset_boot(struct Scsi_Host *instance)
-{
-	unsigned long end;
-	
-	/*
-	 * Do a SCSI reset to clean up the bus during initialization. No
-	 * messing with the queues, interrupts, or locks necessary here.
-	 */
-
-	printk( "Sun3 SCSI: resetting the SCSI bus..." );
-
-	/* switch off SCSI IRQ - catch an interrupt without IRQ bit set else */
-//       	sun3_disable_irq( IRQ_SUN3_SCSI );
-
-	/* get in phase */
-	NCR5380_write( TARGET_COMMAND_REG,
-		      PHASE_SR_TO_TCR( NCR5380_read(STATUS_REG) ));
-
-	/* assert RST */
-	NCR5380_write( INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_RST );
-
-	/* The min. reset hold time is 25us, so 40us should be enough */
-	udelay( 50 );
-
-	/* reset RST and interrupt */
-	NCR5380_write( INITIATOR_COMMAND_REG, ICR_BASE );
-	NCR5380_read( RESET_PARITY_INTERRUPT_REG );
-
-	for( end = jiffies + AFTER_RESET_DELAY; time_before(jiffies, end); )
-		barrier();
-
-	/* switch on SCSI IRQ again */
-//       	sun3_enable_irq( IRQ_SUN3_SCSI );
-
-	printk( " done\n" );
-}
-#endif
-
 // safe bits for the CSR
 #define CSR_GOOD 0x060f
 
-static irqreturn_t scsi_sun3_intr(int irq, void *dummy)
+static irqreturn_t scsi_sun3_intr(int irq, void *dev)
 {
+	struct Scsi_Host *instance = dev;
 	unsigned short csr = dregs->csr;
 	int handled = 0;
 
@@ -196,46 +152,24 @@ static irqreturn_t scsi_sun3_intr(int irq, void *dummy)
 #endif
 
 	if(csr & ~CSR_GOOD) {
-		if(csr & CSR_DMA_BUSERR) {
-			printk("scsi%d: bus error in dma\n", default_instance->host_no);
-		}
-
-		if(csr & CSR_DMA_CONFLICT) {
-			printk("scsi%d: dma conflict\n", default_instance->host_no);
-		}
+		if (csr & CSR_DMA_BUSERR)
+			shost_printk(KERN_ERR, instance, "bus error in DMA\n");
+		if (csr & CSR_DMA_CONFLICT)
+			shost_printk(KERN_ERR, instance, "DMA conflict\n");
 		handled = 1;
 	}
 
 	if(csr & (CSR_SDB_INT | CSR_DMA_INT)) {
-		NCR5380_intr(irq, dummy);
+		NCR5380_intr(irq, dev);
 		handled = 1;
 	}
 
 	return IRQ_RETVAL(handled);
 }
 
-/*
- * Debug stuff - to be called on NMI, or sysrq key. Use at your own risk; 
- * reentering NCR5380_print_status seems to have ugly side effects
- */
-
-/* this doesn't seem to get used at all -- sam */
-#if 0
-void sun3_sun3_debug (void)
-{
-	unsigned long flags;
-
-	if (default_instance) {
-			local_irq_save(flags);
-			NCR5380_print_status(default_instance);
-			local_irq_restore(flags);
-	}
-}
-#endif
-
-
 /* sun3scsi_dma_setup() -- initialize the dma controller for a read/write */
-static unsigned long sun3scsi_dma_setup(void *data, unsigned long count, int write_flag)
+static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance,
+                                void *data, unsigned long count, int write_flag)
 {
 	void *addr;
 
@@ -287,10 +221,9 @@ static unsigned long sun3scsi_dma_setup(void *data, unsigned long count, int wri
 	dregs->csr |= CSR_FIFO;
 	
 	if(dregs->fifo_count != count) { 
-		printk("scsi%d: fifo_mismatch %04x not %04x\n",
-		       default_instance->host_no, dregs->fifo_count,
-		       (unsigned int) count);
-		NCR5380_dprint(NDEBUG_DMA, default_instance);
+		shost_printk(KERN_ERR, instance, "FIFO mismatch %04x not %04x\n",
+		             dregs->fifo_count, (unsigned int) count);
+		NCR5380_dprint(NDEBUG_DMA, instance);
 	}
 
 	/* setup udc */
@@ -325,21 +258,6 @@ static unsigned long sun3scsi_dma_setup(void *data, unsigned long count, int wri
 
 }
 
-#ifndef SUN3_SCSI_VME
-static inline unsigned long sun3scsi_dma_count(struct Scsi_Host *instance)
-{
-	unsigned short resid;
-
-	dregs->udc_addr = 0x32; 
-	udelay(SUN3_DMA_DELAY);
-	resid = dregs->udc_data;
-	udelay(SUN3_DMA_DELAY);
-	resid *= 2;
-
-	return (unsigned long) resid;
-}
-#endif
-
 static inline unsigned long sun3scsi_dma_residual(struct Scsi_Host *instance)
 {
 	return last_residual;
@@ -437,7 +355,10 @@ static int sun3scsi_dma_finish(int write_flag)
 		}
 	}
 
-	count = sun3scsi_dma_count(default_instance);
+	dregs->udc_addr = 0x32;
+	udelay(SUN3_DMA_DELAY);
+	count = 2 * dregs->udc_data;
+	udelay(SUN3_DMA_DELAY);
 
 	fifo = dregs->fifo_count;
 	last_residual = fifo;
@@ -502,17 +423,17 @@ static int sun3scsi_dma_finish(int write_flag)
 static struct scsi_host_template sun3_scsi_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= DRV_MODULE_NAME,
-	.show_info		= sun3scsi_show_info,
 	.name			= SUN3_SCSI_NAME,
 	.info			= sun3scsi_info,
 	.queuecommand		= sun3scsi_queue_command,
-	.eh_abort_handler      	= sun3scsi_abort,
-	.eh_bus_reset_handler  	= sun3scsi_bus_reset,
+	.eh_abort_handler	= sun3scsi_abort,
+	.eh_bus_reset_handler	= sun3scsi_bus_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_NONE,
 	.cmd_per_lun		= 2,
-	.use_clustering		= DISABLE_CLUSTERING
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
 };
 
 static int __init sun3_scsi_probe(struct platform_device *pdev)
@@ -591,7 +512,6 @@ static int __init sun3_scsi_probe(struct platform_device *pdev)
 		error = -ENOMEM;
 		goto fail_alloc;
 	}
-	default_instance = instance;
 
 	instance->io_port = (unsigned long)ioaddr;
 	instance->irq = irq->start;
@@ -600,7 +520,9 @@ static int __init sun3_scsi_probe(struct platform_device *pdev)
 	host_flags |= setup_use_tagged_queuing > 0 ? FLAG_TAGGED_QUEUING : 0;
 #endif
 
-	NCR5380_init(instance, host_flags);
+	error = NCR5380_init(instance, host_flags);
+	if (error)
+		goto fail_init;
 
 	error = request_irq(instance->irq, scsi_sun3_intr, 0,
 	                    "NCR5380", instance);
@@ -631,9 +553,7 @@ static int __init sun3_scsi_probe(struct platform_device *pdev)
 	dregs->ivect = VME_DATA24 | (instance->irq & 0xff);
 #endif
 
-#ifdef RESET_BOOT
-	sun3_scsi_reset_boot(instance);
-#endif
+	NCR5380_maybe_reset_bus(instance);
 
 	error = scsi_add_host(instance, NULL);
 	if (error)
@@ -649,6 +569,7 @@ fail_host:
 		free_irq(instance->irq, instance);
 fail_irq:
 	NCR5380_exit(instance);
+fail_init:
 	scsi_host_put(instance);
 fail_alloc:
 	if (udc_regs)
diff --git a/drivers/scsi/t128.c b/drivers/scsi/t128.c
index 87828acbf7c6..4615fda60dbd 100644
--- a/drivers/scsi/t128.c
+++ b/drivers/scsi/t128.c
@@ -68,14 +68,11 @@
  * 15 9-11
  */
  
-#include <linux/signal.h>
 #include <linux/io.h>
 #include <linux/blkdev.h>
 #include <linux/interrupt.h>
-#include <linux/stat.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/delay.h>
 
 #include <scsi/scsi_host.h>
 #include "t128.h"
@@ -126,7 +123,7 @@ static struct signature {
 
 static int __init t128_setup(char *str)
 {
-    static int commandline_current = 0;
+	static int commandline_current;
     int i;
     int ints[10];
 
@@ -165,7 +162,7 @@ __setup("t128=", t128_setup);
 
 static int __init t128_detect(struct scsi_host_template *tpnt)
 {
-    static int current_override = 0, current_base = 0;
+	static int current_override, current_base;
     struct Scsi_Host *instance;
     unsigned long base;
     void __iomem *p;
@@ -182,9 +179,8 @@ static int __init t128_detect(struct scsi_host_template *tpnt)
 		base = 0;
 	} else 
 	    for (; !base && (current_base < NO_BASES); ++current_base) {
-#if (TDEBUG & TDEBUG_INIT)
-    printk("scsi-t128 : probing address %08x\n", bases[current_base].address);
-#endif
+		dprintk(NDEBUG_INIT, "t128: probing address 0x%08x\n",
+		        bases[current_base].address);
 		if (bases[current_base].noauto)
 			continue;
 		p = ioremap(bases[current_base].address, 0x2000);
@@ -195,17 +191,13 @@ static int __init t128_detect(struct scsi_host_template *tpnt)
 					signatures[sig].string,
 					strlen(signatures[sig].string))) {
 			base = bases[current_base].address;
-#if (TDEBUG & TDEBUG_INIT)
-			printk("scsi-t128 : detected board.\n");
-#endif
+			dprintk(NDEBUG_INIT, "t128: detected board\n");
 			goto found;
 		    }
 		iounmap(p);
 	    }
 
-#if defined(TDEBUG) && (TDEBUG & TDEBUG_INIT)
-	printk("scsi-t128 : base = %08x\n", (unsigned int) base);
-#endif
+	dprintk(NDEBUG_INIT, "t128: base = 0x%08x\n", (unsigned int)base);
 
 	if (!base)
 	    break;
@@ -213,12 +205,15 @@ static int __init t128_detect(struct scsi_host_template *tpnt)
 found:
 	instance = scsi_register (tpnt, sizeof(struct NCR5380_hostdata));
 	if(instance == NULL)
-		break;
-		
+		goto out_unmap;
+
 	instance->base = base;
 	((struct NCR5380_hostdata *)instance->hostdata)->base = p;
 
-	NCR5380_init(instance, 0);
+	if (NCR5380_init(instance, 0))
+		goto out_unregister;
+
+	NCR5380_maybe_reset_bus(instance);
 
 	if (overrides[current_override].irq != IRQ_AUTO)
 	    instance->irq = overrides[current_override].irq;
@@ -242,27 +237,30 @@ found:
 	    printk("scsi%d : please jumper the board for a free IRQ.\n", instance->host_no);
 	}
 
-#if defined(TDEBUG) && (TDEBUG & TDEBUG_INIT)
-	printk("scsi%d : irq = %d\n", instance->host_no, instance->irq);
-#endif
+	dprintk(NDEBUG_INIT, "scsi%d: irq = %d\n",
+	        instance->host_no, instance->irq);
 
 	++current_override;
 	++count;
     }
     return count;
+
+out_unregister:
+	scsi_unregister(instance);
+out_unmap:
+	iounmap(p);
+	return count;
 }
 
 static int t128_release(struct Scsi_Host *shost)
 {
-	NCR5380_local_declare();
-	NCR5380_setup(shost);
+	struct NCR5380_hostdata *hostdata = shost_priv(shost);
+
 	if (shost->irq != NO_IRQ)
 		free_irq(shost->irq, shost);
 	NCR5380_exit(shost);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
 	scsi_unregister(shost);
-	iounmap(base);
+	iounmap(hostdata->base);
 	return 0;
 }
 
@@ -308,14 +306,14 @@ static int t128_biosparam(struct scsi_device *sdev, struct block_device *bdev,
  * 	timeout.
  */
 
-static inline int NCR5380_pread (struct Scsi_Host *instance, unsigned char *dst,
-    int len) {
-    NCR5380_local_declare();
-    void __iomem *reg;
+static inline int
+NCR5380_pread(struct Scsi_Host *instance, unsigned char *dst, int len)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	void __iomem *reg, *base = hostdata->base;
     unsigned char *d = dst;
     register int i = len;
 
-    NCR5380_setup(instance);
     reg = base + T_DATA_REG_OFFSET;
 
 #if 0
@@ -354,14 +352,14 @@ static inline int NCR5380_pread (struct Scsi_Host *instance, unsigned char *dst,
  * 	timeout.
  */
 
-static inline int NCR5380_pwrite (struct Scsi_Host *instance, unsigned char *src,
-    int len) {
-    NCR5380_local_declare();
-    void __iomem *reg;
+static inline int
+NCR5380_pwrite(struct Scsi_Host *instance, unsigned char *src, int len)
+{
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	void __iomem *reg, *base = hostdata->base;
     unsigned char *s = src;
     register int i = len;
 
-    NCR5380_setup(instance);
     reg = base + T_DATA_REG_OFFSET;
 
 #if 0
@@ -392,21 +390,23 @@ MODULE_LICENSE("GPL");
 #include "NCR5380.c"
 
 static struct scsi_host_template driver_template = {
-	.name           = "Trantor T128/T128F/T228",
-	.detect         = t128_detect,
-	.release        = t128_release,
-	.proc_name      = "t128",
-	.show_info      = t128_show_info,
-	.write_info     = t128_write_info,
-	.info           = t128_info,
-	.queuecommand   = t128_queue_command,
-	.eh_abort_handler = t128_abort,
-	.eh_bus_reset_handler    = t128_bus_reset,
-	.bios_param     = t128_biosparam,
-	.can_queue      = CAN_QUEUE,
-        .this_id        = 7,
-	.sg_tablesize   = SG_ALL,
-	.cmd_per_lun    = CMD_PER_LUN,
-	.use_clustering = DISABLE_CLUSTERING,
+	.name			= "Trantor T128/T128F/T228",
+	.detect			= t128_detect,
+	.release		= t128_release,
+	.proc_name		= "t128",
+	.show_info		= t128_show_info,
+	.write_info		= t128_write_info,
+	.info			= t128_info,
+	.queuecommand		= t128_queue_command,
+	.eh_abort_handler	= t128_abort,
+	.eh_bus_reset_handler	= t128_bus_reset,
+	.bios_param		= t128_biosparam,
+	.can_queue		= 32,
+	.this_id		= 7,
+	.sg_tablesize		= SG_ALL,
+	.cmd_per_lun		= 2,
+	.use_clustering		= DISABLE_CLUSTERING,
+	.cmd_size		= NCR5380_CMD_SIZE,
+	.max_sectors		= 128,
 };
 #include "scsi_module.c"
diff --git a/drivers/scsi/t128.h b/drivers/scsi/t128.h
index 2c7371454dfd..dd16d85497e1 100644
--- a/drivers/scsi/t128.h
+++ b/drivers/scsi/t128.h
@@ -23,10 +23,6 @@
 #ifndef T128_H
 #define T128_H
 
-#define TDEBUG		0
-#define TDEBUG_INIT	0x1
-#define TDEBUG_TRANSFER 0x2
-
 /*
  * The trantor boards are memory mapped. They use an NCR5380 or
  * equivalent (my sample board had part second sourced from ZILOG).
@@ -71,44 +67,18 @@
 
 #define T_DATA_REG_OFFSET	0x1e00	/* rw 512 bytes long */
 
-#ifndef ASM
-
-#ifndef CMD_PER_LUN
-#define CMD_PER_LUN 2
-#endif
-
-#ifndef CAN_QUEUE
-#define CAN_QUEUE 32
-#endif
-
 #define NCR5380_implementation_fields \
     void __iomem *base
 
-#define NCR5380_local_declare() \
-    void __iomem *base
-
-#define NCR5380_setup(instance) \
-    base = ((struct NCR5380_hostdata *)(instance->hostdata))->base
+#define T128_address(reg) \
+	(((struct NCR5380_hostdata *)shost_priv(instance))->base + T_5380_OFFSET + ((reg) * 0x20))
 
-#define T128_address(reg) (base + T_5380_OFFSET + ((reg) * 0x20))
-
-#if !(TDEBUG & TDEBUG_TRANSFER)
 #define NCR5380_read(reg) readb(T128_address(reg))
 #define NCR5380_write(reg, value) writeb((value),(T128_address(reg)))
-#else
-#define NCR5380_read(reg)						\
-    (((unsigned char) printk("scsi%d : read register %d at address %08x\n"\
-    , instance->hostno, (reg), T128_address(reg))), readb(T128_address(reg)))
-
-#define NCR5380_write(reg, value) {					\
-    printk("scsi%d : write %02x to register %d at address %08x\n",	\
-	    instance->hostno, (value), (reg), T128_address(reg));	\
-    writeb((value), (T128_address(reg)));				\
-}
-#endif
+
+#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
 
 #define NCR5380_intr t128_intr
-#define do_NCR5380_intr do_t128_intr
 #define NCR5380_queue_command t128_queue_command
 #define NCR5380_abort t128_abort
 #define NCR5380_bus_reset t128_bus_reset
@@ -121,5 +91,4 @@
 
 #define T128_IRQS 0xc4a8
 
-#endif /* ndef ASM */
 #endif /* T128_H */
diff --git a/drivers/sh/clk/core.c b/drivers/sh/clk/core.c
index be56b22ca941..92863e3818e5 100644
--- a/drivers/sh/clk/core.c
+++ b/drivers/sh/clk/core.c
@@ -469,6 +469,9 @@ void clk_enable_init_clocks(void)
 
 unsigned long clk_get_rate(struct clk *clk)
 {
+	if (!clk)
+		return 0;
+
 	return clk->rate;
 }
 EXPORT_SYMBOL_GPL(clk_get_rate);
@@ -478,6 +481,9 @@ int clk_set_rate(struct clk *clk, unsigned long rate)
 	int ret = -EOPNOTSUPP;
 	unsigned long flags;
 
+	if (!clk)
+		return 0;
+
 	spin_lock_irqsave(&clock_lock, flags);
 
 	if (likely(clk->ops && clk->ops->set_rate)) {
@@ -535,12 +541,18 @@ EXPORT_SYMBOL_GPL(clk_set_parent);
 
 struct clk *clk_get_parent(struct clk *clk)
 {
+	if (!clk)
+		return NULL;
+
 	return clk->parent;
 }
 EXPORT_SYMBOL_GPL(clk_get_parent);
 
 long clk_round_rate(struct clk *clk, unsigned long rate)
 {
+	if (!clk)
+		return 0;
+
 	if (likely(clk->ops && clk->ops->round_rate)) {
 		unsigned long flags, rounded;
 
@@ -555,94 +567,6 @@ long clk_round_rate(struct clk *clk, unsigned long rate)
 }
 EXPORT_SYMBOL_GPL(clk_round_rate);
 
-long clk_round_parent(struct clk *clk, unsigned long target,
-		      unsigned long *best_freq, unsigned long *parent_freq,
-		      unsigned int div_min, unsigned int div_max)
-{
-	struct cpufreq_frequency_table *freq, *best = NULL;
-	unsigned long error = ULONG_MAX, freq_high, freq_low, div;
-	struct clk *parent = clk_get_parent(clk);
-
-	if (!parent) {
-		*parent_freq = 0;
-		*best_freq = clk_round_rate(clk, target);
-		return abs(target - *best_freq);
-	}
-
-	cpufreq_for_each_valid_entry(freq, parent->freq_table) {
-		if (unlikely(freq->frequency / target <= div_min - 1)) {
-			unsigned long freq_max;
-
-			freq_max = (freq->frequency + div_min / 2) / div_min;
-			if (error > target - freq_max) {
-				error = target - freq_max;
-				best = freq;
-				if (best_freq)
-					*best_freq = freq_max;
-			}
-
-			pr_debug("too low freq %u, error %lu\n", freq->frequency,
-				 target - freq_max);
-
-			if (!error)
-				break;
-
-			continue;
-		}
-
-		if (unlikely(freq->frequency / target >= div_max)) {
-			unsigned long freq_min;
-
-			freq_min = (freq->frequency + div_max / 2) / div_max;
-			if (error > freq_min - target) {
-				error = freq_min - target;
-				best = freq;
-				if (best_freq)
-					*best_freq = freq_min;
-			}
-
-			pr_debug("too high freq %u, error %lu\n", freq->frequency,
-				 freq_min - target);
-
-			if (!error)
-				break;
-
-			continue;
-		}
-
-		div = freq->frequency / target;
-		freq_high = freq->frequency / div;
-		freq_low = freq->frequency / (div + 1);
-
-		if (freq_high - target < error) {
-			error = freq_high - target;
-			best = freq;
-			if (best_freq)
-				*best_freq = freq_high;
-		}
-
-		if (target - freq_low < error) {
-			error = target - freq_low;
-			best = freq;
-			if (best_freq)
-				*best_freq = freq_low;
-		}
-
-		pr_debug("%u / %lu = %lu, / %lu = %lu, best %lu, parent %u\n",
-			 freq->frequency, div, freq_high, div + 1, freq_low,
-			 *best_freq, best->frequency);
-
-		if (!error)
-			break;
-	}
-
-	if (parent_freq)
-		*parent_freq = best->frequency;
-
-	return error;
-}
-EXPORT_SYMBOL_GPL(clk_round_parent);
-
 #ifdef CONFIG_PM
 static void clks_core_resume(void)
 {
diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig
index 4e853ed2c82b..88260205a261 100644
--- a/drivers/soc/Kconfig
+++ b/drivers/soc/Kconfig
@@ -1,10 +1,13 @@
 menu "SOC (System On Chip) specific Drivers"
 
+source "drivers/soc/bcm/Kconfig"
 source "drivers/soc/brcmstb/Kconfig"
+source "drivers/soc/fsl/qe/Kconfig"
 source "drivers/soc/mediatek/Kconfig"
 source "drivers/soc/qcom/Kconfig"
 source "drivers/soc/rockchip/Kconfig"
 source "drivers/soc/sunxi/Kconfig"
+source "drivers/soc/tegra/Kconfig"
 source "drivers/soc/ti/Kconfig"
 source "drivers/soc/versatile/Kconfig"
 
diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile
index f2ba2e932ae1..2afdc74f7491 100644
--- a/drivers/soc/Makefile
+++ b/drivers/soc/Makefile
@@ -2,8 +2,11 @@
 # Makefile for the Linux Kernel SOC specific device drivers.
 #
 
+obj-y				+= bcm/
 obj-$(CONFIG_SOC_BRCMSTB)	+= brcmstb/
+obj-$(CONFIG_ARCH_DOVE)		+= dove/
 obj-$(CONFIG_MACH_DOVE)		+= dove/
+obj-y				+= fsl/
 obj-$(CONFIG_ARCH_MEDIATEK)	+= mediatek/
 obj-$(CONFIG_ARCH_QCOM)		+= qcom/
 obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
diff --git a/drivers/soc/bcm/Kconfig b/drivers/soc/bcm/Kconfig
new file mode 100644
index 000000000000..3066edea184d
--- /dev/null
+++ b/drivers/soc/bcm/Kconfig
@@ -0,0 +1,9 @@
+config RASPBERRYPI_POWER
+	bool "Raspberry Pi power domain driver"
+	depends on ARCH_BCM2835 || COMPILE_TEST
+	depends on RASPBERRYPI_FIRMWARE=y
+	select PM_GENERIC_DOMAINS if PM
+	select PM_GENERIC_DOMAINS_OF if PM
+	help
+	  This enables support for the RPi power domains which can be enabled
+	  or disabled via the RPi firmware.
diff --git a/drivers/soc/bcm/Makefile b/drivers/soc/bcm/Makefile
new file mode 100644
index 000000000000..63aa3eb23087
--- /dev/null
+++ b/drivers/soc/bcm/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_RASPBERRYPI_POWER)	+= raspberrypi-power.o
diff --git a/drivers/soc/bcm/raspberrypi-power.c b/drivers/soc/bcm/raspberrypi-power.c
new file mode 100644
index 000000000000..fe96a8b956fb
--- /dev/null
+++ b/drivers/soc/bcm/raspberrypi-power.c
@@ -0,0 +1,247 @@
+/* (C) 2015 Pengutronix, Alexander Aring <aar@pengutronix.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Authors:
+ * Alexander Aring <aar@pengutronix.de>
+ * Eric Anholt <eric@anholt.net>
+ */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <dt-bindings/power/raspberrypi-power.h>
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
+/*
+ * Firmware indices for the old power domains interface.  Only a few
+ * of them were actually implemented.
+ */
+#define RPI_OLD_POWER_DOMAIN_USB		3
+#define RPI_OLD_POWER_DOMAIN_V3D		10
+
+struct rpi_power_domain {
+	u32 domain;
+	bool enabled;
+	bool old_interface;
+	struct generic_pm_domain base;
+	struct rpi_firmware *fw;
+};
+
+struct rpi_power_domains {
+	bool has_new_interface;
+	struct genpd_onecell_data xlate;
+	struct rpi_firmware *fw;
+	struct rpi_power_domain domains[RPI_POWER_DOMAIN_COUNT];
+};
+
+/*
+ * Packet definition used by RPI_FIRMWARE_SET_POWER_STATE and
+ * RPI_FIRMWARE_SET_DOMAIN_STATE
+ */
+struct rpi_power_domain_packet {
+	u32 domain;
+	u32 on;
+} __packet;
+
+/*
+ * Asks the firmware to enable or disable power on a specific power
+ * domain.
+ */
+static int rpi_firmware_set_power(struct rpi_power_domain *rpi_domain, bool on)
+{
+	struct rpi_power_domain_packet packet;
+
+	packet.domain = rpi_domain->domain;
+	packet.on = on;
+	return rpi_firmware_property(rpi_domain->fw,
+				     rpi_domain->old_interface ?
+				     RPI_FIRMWARE_SET_POWER_STATE :
+				     RPI_FIRMWARE_SET_DOMAIN_STATE,
+				     &packet, sizeof(packet));
+}
+
+static int rpi_domain_off(struct generic_pm_domain *domain)
+{
+	struct rpi_power_domain *rpi_domain =
+		container_of(domain, struct rpi_power_domain, base);
+
+	return rpi_firmware_set_power(rpi_domain, false);
+}
+
+static int rpi_domain_on(struct generic_pm_domain *domain)
+{
+	struct rpi_power_domain *rpi_domain =
+		container_of(domain, struct rpi_power_domain, base);
+
+	return rpi_firmware_set_power(rpi_domain, true);
+}
+
+static void rpi_common_init_power_domain(struct rpi_power_domains *rpi_domains,
+					 int xlate_index, const char *name)
+{
+	struct rpi_power_domain *dom = &rpi_domains->domains[xlate_index];
+
+	dom->fw = rpi_domains->fw;
+
+	dom->base.name = name;
+	dom->base.power_on = rpi_domain_on;
+	dom->base.power_off = rpi_domain_off;
+
+	/*
+	 * Treat all power domains as off at boot.
+	 *
+	 * The firmware itself may be keeping some domains on, but
+	 * from Linux's perspective all we control is the refcounts
+	 * that we give to the firmware, and we can't ask the firmware
+	 * to turn off something that we haven't ourselves turned on.
+	 */
+	pm_genpd_init(&dom->base, NULL, true);
+
+	rpi_domains->xlate.domains[xlate_index] = &dom->base;
+}
+
+static void rpi_init_power_domain(struct rpi_power_domains *rpi_domains,
+				  int xlate_index, const char *name)
+{
+	struct rpi_power_domain *dom = &rpi_domains->domains[xlate_index];
+
+	if (!rpi_domains->has_new_interface)
+		return;
+
+	/* The DT binding index is the firmware's domain index minus one. */
+	dom->domain = xlate_index + 1;
+
+	rpi_common_init_power_domain(rpi_domains, xlate_index, name);
+}
+
+static void rpi_init_old_power_domain(struct rpi_power_domains *rpi_domains,
+				      int xlate_index, int domain,
+				      const char *name)
+{
+	struct rpi_power_domain *dom = &rpi_domains->domains[xlate_index];
+
+	dom->old_interface = true;
+	dom->domain = domain;
+
+	rpi_common_init_power_domain(rpi_domains, xlate_index, name);
+}
+
+/*
+ * Detects whether the firmware supports the new power domains interface.
+ *
+ * The firmware doesn't actually return an error on an unknown tag,
+ * and just skips over it, so we do the detection by putting an
+ * unexpected value in the return field and checking if it was
+ * unchanged.
+ */
+static bool
+rpi_has_new_domain_support(struct rpi_power_domains *rpi_domains)
+{
+	struct rpi_power_domain_packet packet;
+	int ret;
+
+	packet.domain = RPI_POWER_DOMAIN_ARM;
+	packet.on = ~0;
+
+	ret = rpi_firmware_property(rpi_domains->fw,
+				    RPI_FIRMWARE_GET_DOMAIN_STATE,
+				    &packet, sizeof(packet));
+
+	return ret == 0 && packet.on != ~0;
+}
+
+static int rpi_power_probe(struct platform_device *pdev)
+{
+	struct device_node *fw_np;
+	struct device *dev = &pdev->dev;
+	struct rpi_power_domains *rpi_domains;
+
+	rpi_domains = devm_kzalloc(dev, sizeof(*rpi_domains), GFP_KERNEL);
+	if (!rpi_domains)
+		return -ENOMEM;
+
+	rpi_domains->xlate.domains =
+		devm_kzalloc(dev, sizeof(*rpi_domains->xlate.domains) *
+			     RPI_POWER_DOMAIN_COUNT, GFP_KERNEL);
+	if (!rpi_domains->xlate.domains)
+		return -ENOMEM;
+
+	rpi_domains->xlate.num_domains = RPI_POWER_DOMAIN_COUNT;
+
+	fw_np = of_parse_phandle(pdev->dev.of_node, "firmware", 0);
+	if (!fw_np) {
+		dev_err(&pdev->dev, "no firmware node\n");
+		return -ENODEV;
+	}
+
+	rpi_domains->fw = rpi_firmware_get(fw_np);
+	of_node_put(fw_np);
+	if (!rpi_domains->fw)
+		return -EPROBE_DEFER;
+
+	rpi_domains->has_new_interface =
+		rpi_has_new_domain_support(rpi_domains);
+
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_I2C0, "I2C0");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_I2C1, "I2C1");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_I2C2, "I2C2");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_VIDEO_SCALER,
+			      "VIDEO_SCALER");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_VPU1, "VPU1");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_HDMI, "HDMI");
+
+	/*
+	 * Use the old firmware interface for USB power, so that we
+	 * can turn it on even if the firmware hasn't been updated.
+	 */
+	rpi_init_old_power_domain(rpi_domains, RPI_POWER_DOMAIN_USB,
+				  RPI_OLD_POWER_DOMAIN_USB, "USB");
+
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_VEC, "VEC");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_JPEG, "JPEG");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_H264, "H264");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_V3D, "V3D");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_ISP, "ISP");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_UNICAM0, "UNICAM0");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_UNICAM1, "UNICAM1");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_CCP2RX, "CCP2RX");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_CSI2, "CSI2");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_CPI, "CPI");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_DSI0, "DSI0");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_DSI1, "DSI1");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_TRANSPOSER,
+			      "TRANSPOSER");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_CCP2TX, "CCP2TX");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_CDP, "CDP");
+	rpi_init_power_domain(rpi_domains, RPI_POWER_DOMAIN_ARM, "ARM");
+
+	of_genpd_add_provider_onecell(dev->of_node, &rpi_domains->xlate);
+
+	platform_set_drvdata(pdev, rpi_domains);
+
+	return 0;
+}
+
+static const struct of_device_id rpi_power_of_match[] = {
+	{ .compatible = "raspberrypi,bcm2835-power", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rpi_power_of_match);
+
+static struct platform_driver rpi_power_driver = {
+	.driver = {
+		.name = "raspberrypi-power",
+		.of_match_table = rpi_power_of_match,
+	},
+	.probe		= rpi_power_probe,
+};
+builtin_platform_driver(rpi_power_driver);
+
+MODULE_AUTHOR("Alexander Aring <aar@pengutronix.de>");
+MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
+MODULE_DESCRIPTION("Raspberry Pi power domain driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/dove/pmu.c b/drivers/soc/dove/pmu.c
index abd087917f80..039374e9fdc0 100644
--- a/drivers/soc/dove/pmu.c
+++ b/drivers/soc/dove/pmu.c
@@ -305,6 +305,49 @@ static int __init dove_init_pmu_irq(struct pmu_data *pmu, int irq)
 	return 0;
 }
 
+int __init dove_init_pmu_legacy(const struct dove_pmu_initdata *initdata)
+{
+	const struct dove_pmu_domain_initdata *domain_initdata;
+	struct pmu_data *pmu;
+	int ret;
+
+	pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
+	if (!pmu)
+		return -ENOMEM;
+
+	spin_lock_init(&pmu->lock);
+	pmu->pmc_base = initdata->pmc_base;
+	pmu->pmu_base = initdata->pmu_base;
+
+	pmu_reset_init(pmu);
+	for (domain_initdata = initdata->domains; domain_initdata->name;
+	     domain_initdata++) {
+		struct pmu_domain *domain;
+
+		domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+		if (domain) {
+			domain->pmu = pmu;
+			domain->pwr_mask = domain_initdata->pwr_mask;
+			domain->rst_mask = domain_initdata->rst_mask;
+			domain->iso_mask = domain_initdata->iso_mask;
+			domain->base.name = domain_initdata->name;
+
+			__pmu_domain_register(domain, NULL);
+		}
+	}
+
+	ret = dove_init_pmu_irq(pmu, initdata->irq);
+	if (ret)
+		pr_err("dove_init_pmu_irq() failed: %d\n", ret);
+
+	if (pmu->irq_domain)
+		irq_domain_associate_many(pmu->irq_domain,
+					  initdata->irq_domain_start,
+					  0, NR_PMU_IRQS);
+
+	return 0;
+}
+
 /*
  * pmu: power-manager@d0000 {
  *	compatible = "marvell,dove-pmu";
diff --git a/drivers/soc/fsl/Makefile b/drivers/soc/fsl/Makefile
new file mode 100644
index 000000000000..203307fd92c1
--- /dev/null
+++ b/drivers/soc/fsl/Makefile
@@ -0,0 +1,6 @@
+#
+# Makefile for the Linux Kernel SOC fsl specific device drivers
+#
+
+obj-$(CONFIG_QUICC_ENGINE)		+= qe/
+obj-$(CONFIG_CPM)			+= qe/
diff --git a/drivers/soc/fsl/qe/Kconfig b/drivers/soc/fsl/qe/Kconfig
new file mode 100644
index 000000000000..20978f2058a6
--- /dev/null
+++ b/drivers/soc/fsl/qe/Kconfig
@@ -0,0 +1,38 @@
+#
+# QE Communication options
+#
+
+config QUICC_ENGINE
+	bool "Freescale QUICC Engine (QE) Support"
+	depends on FSL_SOC && PPC32
+	select GENERIC_ALLOCATOR
+	select CRC32
+	help
+	  The QUICC Engine (QE) is a new generation of communications
+	  coprocessors on Freescale embedded CPUs (akin to CPM in older chips).
+	  Selecting this option means that you wish to build a kernel
+	  for a machine with a QE coprocessor.
+
+config UCC_SLOW
+	bool
+	default y if SERIAL_QE
+	help
+	  This option provides qe_lib support to UCC slow
+	  protocols: UART, BISYNC, QMC
+
+config UCC_FAST
+	bool
+	default y if UCC_GETH
+	help
+	  This option provides qe_lib support to UCC fast
+	  protocols: HDLC, Ethernet, ATM, transparent
+
+config UCC
+	bool
+	default y if UCC_FAST || UCC_SLOW
+
+config QE_USB
+	bool
+	default y if USB_FSL_QE
+	help
+	  QE USB Controller support
diff --git a/drivers/soc/fsl/qe/Makefile b/drivers/soc/fsl/qe/Makefile
new file mode 100644
index 000000000000..ffac5410c5c7
--- /dev/null
+++ b/drivers/soc/fsl/qe/Makefile
@@ -0,0 +1,10 @@
+#
+# Makefile for the linux ppc-specific parts of QE
+#
+obj-$(CONFIG_QUICC_ENGINE)+= qe.o qe_common.o qe_ic.o qe_io.o
+obj-$(CONFIG_CPM)	+= qe_common.o
+obj-$(CONFIG_UCC)	+= ucc.o
+obj-$(CONFIG_UCC_SLOW)	+= ucc_slow.o
+obj-$(CONFIG_UCC_FAST)	+= ucc_fast.o
+obj-$(CONFIG_QE_USB)	+= usb.o
+obj-$(CONFIG_QE_GPIO)	+= gpio.o
diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c
new file mode 100644
index 000000000000..aa5c11acf212
--- /dev/null
+++ b/drivers/soc/fsl/qe/gpio.c
@@ -0,0 +1,317 @@
+/*
+ * QUICC Engine GPIOs
+ *
+ * Copyright (c) MontaVista Software, Inc. 2008.
+ *
+ * Author: Anton Vorontsov <avorontsov@ru.mvista.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/spinlock.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/gpio.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+#include <soc/fsl/qe/qe.h>
+
+struct qe_gpio_chip {
+	struct of_mm_gpio_chip mm_gc;
+	spinlock_t lock;
+
+	unsigned long pin_flags[QE_PIO_PINS];
+#define QE_PIN_REQUESTED 0
+
+	/* shadowed data register to clear/set bits safely */
+	u32 cpdata;
+
+	/* saved_regs used to restore dedicated functions */
+	struct qe_pio_regs saved_regs;
+};
+
+static inline struct qe_gpio_chip *
+to_qe_gpio_chip(struct of_mm_gpio_chip *mm_gc)
+{
+	return container_of(mm_gc, struct qe_gpio_chip, mm_gc);
+}
+
+static void qe_gpio_save_regs(struct of_mm_gpio_chip *mm_gc)
+{
+	struct qe_gpio_chip *qe_gc = to_qe_gpio_chip(mm_gc);
+	struct qe_pio_regs __iomem *regs = mm_gc->regs;
+
+	qe_gc->cpdata = in_be32(&regs->cpdata);
+	qe_gc->saved_regs.cpdata = qe_gc->cpdata;
+	qe_gc->saved_regs.cpdir1 = in_be32(&regs->cpdir1);
+	qe_gc->saved_regs.cpdir2 = in_be32(&regs->cpdir2);
+	qe_gc->saved_regs.cppar1 = in_be32(&regs->cppar1);
+	qe_gc->saved_regs.cppar2 = in_be32(&regs->cppar2);
+	qe_gc->saved_regs.cpodr = in_be32(&regs->cpodr);
+}
+
+static int qe_gpio_get(struct gpio_chip *gc, unsigned int gpio)
+{
+	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
+	struct qe_pio_regs __iomem *regs = mm_gc->regs;
+	u32 pin_mask = 1 << (QE_PIO_PINS - 1 - gpio);
+
+	return in_be32(&regs->cpdata) & pin_mask;
+}
+
+static void qe_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
+{
+	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
+	struct qe_gpio_chip *qe_gc = to_qe_gpio_chip(mm_gc);
+	struct qe_pio_regs __iomem *regs = mm_gc->regs;
+	unsigned long flags;
+	u32 pin_mask = 1 << (QE_PIO_PINS - 1 - gpio);
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	if (val)
+		qe_gc->cpdata |= pin_mask;
+	else
+		qe_gc->cpdata &= ~pin_mask;
+
+	out_be32(&regs->cpdata, qe_gc->cpdata);
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+}
+
+static int qe_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
+{
+	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
+	struct qe_gpio_chip *qe_gc = to_qe_gpio_chip(mm_gc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	__par_io_config_pin(mm_gc->regs, gpio, QE_PIO_DIR_IN, 0, 0, 0);
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+
+	return 0;
+}
+
+static int qe_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
+{
+	struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
+	struct qe_gpio_chip *qe_gc = to_qe_gpio_chip(mm_gc);
+	unsigned long flags;
+
+	qe_gpio_set(gc, gpio, val);
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	__par_io_config_pin(mm_gc->regs, gpio, QE_PIO_DIR_OUT, 0, 0, 0);
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+
+	return 0;
+}
+
+struct qe_pin {
+	/*
+	 * The qe_gpio_chip name is unfortunate, we should change that to
+	 * something like qe_pio_controller. Someday.
+	 */
+	struct qe_gpio_chip *controller;
+	int num;
+};
+
+/**
+ * qe_pin_request - Request a QE pin
+ * @np:		device node to get a pin from
+ * @index:	index of a pin in the device tree
+ * Context:	non-atomic
+ *
+ * This function return qe_pin so that you could use it with the rest of
+ * the QE Pin Multiplexing API.
+ */
+struct qe_pin *qe_pin_request(struct device_node *np, int index)
+{
+	struct qe_pin *qe_pin;
+	struct gpio_chip *gc;
+	struct of_mm_gpio_chip *mm_gc;
+	struct qe_gpio_chip *qe_gc;
+	int err;
+	unsigned long flags;
+
+	qe_pin = kzalloc(sizeof(*qe_pin), GFP_KERNEL);
+	if (!qe_pin) {
+		pr_debug("%s: can't allocate memory\n", __func__);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	err = of_get_gpio(np, index);
+	if (err < 0)
+		goto err0;
+	gc = gpio_to_chip(err);
+	if (WARN_ON(!gc))
+		goto err0;
+
+	if (!of_device_is_compatible(gc->of_node, "fsl,mpc8323-qe-pario-bank")) {
+		pr_debug("%s: tried to get a non-qe pin\n", __func__);
+		err = -EINVAL;
+		goto err0;
+	}
+
+	mm_gc = to_of_mm_gpio_chip(gc);
+	qe_gc = to_qe_gpio_chip(mm_gc);
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	err -= gc->base;
+	if (test_and_set_bit(QE_PIN_REQUESTED, &qe_gc->pin_flags[err]) == 0) {
+		qe_pin->controller = qe_gc;
+		qe_pin->num = err;
+		err = 0;
+	} else {
+		err = -EBUSY;
+	}
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+
+	if (!err)
+		return qe_pin;
+err0:
+	kfree(qe_pin);
+	pr_debug("%s failed with status %d\n", __func__, err);
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL(qe_pin_request);
+
+/**
+ * qe_pin_free - Free a pin
+ * @qe_pin:	pointer to the qe_pin structure
+ * Context:	any
+ *
+ * This function frees the qe_pin structure and makes a pin available
+ * for further qe_pin_request() calls.
+ */
+void qe_pin_free(struct qe_pin *qe_pin)
+{
+	struct qe_gpio_chip *qe_gc = qe_pin->controller;
+	unsigned long flags;
+	const int pin = qe_pin->num;
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+	test_and_clear_bit(QE_PIN_REQUESTED, &qe_gc->pin_flags[pin]);
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+
+	kfree(qe_pin);
+}
+EXPORT_SYMBOL(qe_pin_free);
+
+/**
+ * qe_pin_set_dedicated - Revert a pin to a dedicated peripheral function mode
+ * @qe_pin:	pointer to the qe_pin structure
+ * Context:	any
+ *
+ * This function resets a pin to a dedicated peripheral function that
+ * has been set up by the firmware.
+ */
+void qe_pin_set_dedicated(struct qe_pin *qe_pin)
+{
+	struct qe_gpio_chip *qe_gc = qe_pin->controller;
+	struct qe_pio_regs __iomem *regs = qe_gc->mm_gc.regs;
+	struct qe_pio_regs *sregs = &qe_gc->saved_regs;
+	int pin = qe_pin->num;
+	u32 mask1 = 1 << (QE_PIO_PINS - (pin + 1));
+	u32 mask2 = 0x3 << (QE_PIO_PINS - (pin % (QE_PIO_PINS / 2) + 1) * 2);
+	bool second_reg = pin > (QE_PIO_PINS / 2) - 1;
+	unsigned long flags;
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	if (second_reg) {
+		clrsetbits_be32(&regs->cpdir2, mask2, sregs->cpdir2 & mask2);
+		clrsetbits_be32(&regs->cppar2, mask2, sregs->cppar2 & mask2);
+	} else {
+		clrsetbits_be32(&regs->cpdir1, mask2, sregs->cpdir1 & mask2);
+		clrsetbits_be32(&regs->cppar1, mask2, sregs->cppar1 & mask2);
+	}
+
+	if (sregs->cpdata & mask1)
+		qe_gc->cpdata |= mask1;
+	else
+		qe_gc->cpdata &= ~mask1;
+
+	out_be32(&regs->cpdata, qe_gc->cpdata);
+	clrsetbits_be32(&regs->cpodr, mask1, sregs->cpodr & mask1);
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+}
+EXPORT_SYMBOL(qe_pin_set_dedicated);
+
+/**
+ * qe_pin_set_gpio - Set a pin to the GPIO mode
+ * @qe_pin:	pointer to the qe_pin structure
+ * Context:	any
+ *
+ * This function sets a pin to the GPIO mode.
+ */
+void qe_pin_set_gpio(struct qe_pin *qe_pin)
+{
+	struct qe_gpio_chip *qe_gc = qe_pin->controller;
+	struct qe_pio_regs __iomem *regs = qe_gc->mm_gc.regs;
+	unsigned long flags;
+
+	spin_lock_irqsave(&qe_gc->lock, flags);
+
+	/* Let's make it input by default, GPIO API is able to change that. */
+	__par_io_config_pin(regs, qe_pin->num, QE_PIO_DIR_IN, 0, 0, 0);
+
+	spin_unlock_irqrestore(&qe_gc->lock, flags);
+}
+EXPORT_SYMBOL(qe_pin_set_gpio);
+
+static int __init qe_add_gpiochips(void)
+{
+	struct device_node *np;
+
+	for_each_compatible_node(np, NULL, "fsl,mpc8323-qe-pario-bank") {
+		int ret;
+		struct qe_gpio_chip *qe_gc;
+		struct of_mm_gpio_chip *mm_gc;
+		struct gpio_chip *gc;
+
+		qe_gc = kzalloc(sizeof(*qe_gc), GFP_KERNEL);
+		if (!qe_gc) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		spin_lock_init(&qe_gc->lock);
+
+		mm_gc = &qe_gc->mm_gc;
+		gc = &mm_gc->gc;
+
+		mm_gc->save_regs = qe_gpio_save_regs;
+		gc->ngpio = QE_PIO_PINS;
+		gc->direction_input = qe_gpio_dir_in;
+		gc->direction_output = qe_gpio_dir_out;
+		gc->get = qe_gpio_get;
+		gc->set = qe_gpio_set;
+
+		ret = of_mm_gpiochip_add(np, mm_gc);
+		if (ret)
+			goto err;
+		continue;
+err:
+		pr_err("%s: registration failed with status %d\n",
+		       np->full_name, ret);
+		kfree(qe_gc);
+		/* try others anyway */
+	}
+	return 0;
+}
+arch_initcall(qe_add_gpiochips);
diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c
new file mode 100644
index 000000000000..709fc63809e5
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe.c
@@ -0,0 +1,719 @@
+/*
+ * Copyright (C) 2006-2010 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors: 	Shlomi Gridish <gridish@freescale.com>
+ * 		Li Yang <leoli@freescale.com>
+ * Based on cpm2_common.c from Dan Malek (dmalek@jlc.net)
+ *
+ * Description:
+ * General Purpose functions for the global management of the
+ * QUICC Engine (QE).
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/param.h>
+#include <linux/string.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/ioport.h>
+#include <linux/crc32.h>
+#include <linux/mod_devicetable.h>
+#include <linux/of_platform.h>
+#include <asm/irq.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+#include <asm/prom.h>
+#include <asm/rheap.h>
+
+static void qe_snums_init(void);
+static int qe_sdma_init(void);
+
+static DEFINE_SPINLOCK(qe_lock);
+DEFINE_SPINLOCK(cmxgcr_lock);
+EXPORT_SYMBOL(cmxgcr_lock);
+
+/* QE snum state */
+enum qe_snum_state {
+	QE_SNUM_STATE_USED,
+	QE_SNUM_STATE_FREE
+};
+
+/* QE snum */
+struct qe_snum {
+	u8 num;
+	enum qe_snum_state state;
+};
+
+/* We allocate this here because it is used almost exclusively for
+ * the communication processor devices.
+ */
+struct qe_immap __iomem *qe_immr;
+EXPORT_SYMBOL(qe_immr);
+
+static struct qe_snum snums[QE_NUM_OF_SNUM];	/* Dynamically allocated SNUMs */
+static unsigned int qe_num_of_snum;
+
+static phys_addr_t qebase = -1;
+
+phys_addr_t get_qe_base(void)
+{
+	struct device_node *qe;
+	int size;
+	const u32 *prop;
+
+	if (qebase != -1)
+		return qebase;
+
+	qe = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!qe) {
+		qe = of_find_node_by_type(NULL, "qe");
+		if (!qe)
+			return qebase;
+	}
+
+	prop = of_get_property(qe, "reg", &size);
+	if (prop && size >= sizeof(*prop))
+		qebase = of_translate_address(qe, prop);
+	of_node_put(qe);
+
+	return qebase;
+}
+
+EXPORT_SYMBOL(get_qe_base);
+
+void qe_reset(void)
+{
+	if (qe_immr == NULL)
+		qe_immr = ioremap(get_qe_base(), QE_IMMAP_SIZE);
+
+	qe_snums_init();
+
+	qe_issue_cmd(QE_RESET, QE_CR_SUBBLOCK_INVALID,
+		     QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+	/* Reclaim the MURAM memory for our use. */
+	qe_muram_init();
+
+	if (qe_sdma_init())
+		panic("sdma init failed!");
+}
+
+int qe_issue_cmd(u32 cmd, u32 device, u8 mcn_protocol, u32 cmd_input)
+{
+	unsigned long flags;
+	u8 mcn_shift = 0, dev_shift = 0;
+	u32 ret;
+
+	spin_lock_irqsave(&qe_lock, flags);
+	if (cmd == QE_RESET) {
+		out_be32(&qe_immr->cp.cecr, (u32) (cmd | QE_CR_FLG));
+	} else {
+		if (cmd == QE_ASSIGN_PAGE) {
+			/* Here device is the SNUM, not sub-block */
+			dev_shift = QE_CR_SNUM_SHIFT;
+		} else if (cmd == QE_ASSIGN_RISC) {
+			/* Here device is the SNUM, and mcnProtocol is
+			 * e_QeCmdRiscAssignment value */
+			dev_shift = QE_CR_SNUM_SHIFT;
+			mcn_shift = QE_CR_MCN_RISC_ASSIGN_SHIFT;
+		} else {
+			if (device == QE_CR_SUBBLOCK_USB)
+				mcn_shift = QE_CR_MCN_USB_SHIFT;
+			else
+				mcn_shift = QE_CR_MCN_NORMAL_SHIFT;
+		}
+
+		out_be32(&qe_immr->cp.cecdr, cmd_input);
+		out_be32(&qe_immr->cp.cecr,
+			 (cmd | QE_CR_FLG | ((u32) device << dev_shift) | (u32)
+			  mcn_protocol << mcn_shift));
+	}
+
+	/* wait for the QE_CR_FLG to clear */
+	ret = spin_event_timeout((in_be32(&qe_immr->cp.cecr) & QE_CR_FLG) == 0,
+			   100, 0);
+	/* On timeout (e.g. failure), the expression will be false (ret == 0),
+	   otherwise it will be true (ret == 1). */
+	spin_unlock_irqrestore(&qe_lock, flags);
+
+	return ret == 1;
+}
+EXPORT_SYMBOL(qe_issue_cmd);
+
+/* Set a baud rate generator. This needs lots of work. There are
+ * 16 BRGs, which can be connected to the QE channels or output
+ * as clocks. The BRGs are in two different block of internal
+ * memory mapped space.
+ * The BRG clock is the QE clock divided by 2.
+ * It was set up long ago during the initial boot phase and is
+ * is given to us.
+ * Baud rate clocks are zero-based in the driver code (as that maps
+ * to port numbers). Documentation uses 1-based numbering.
+ */
+static unsigned int brg_clk = 0;
+
+unsigned int qe_get_brg_clk(void)
+{
+	struct device_node *qe;
+	int size;
+	const u32 *prop;
+
+	if (brg_clk)
+		return brg_clk;
+
+	qe = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!qe) {
+		qe = of_find_node_by_type(NULL, "qe");
+		if (!qe)
+			return brg_clk;
+	}
+
+	prop = of_get_property(qe, "brg-frequency", &size);
+	if (prop && size == sizeof(*prop))
+		brg_clk = *prop;
+
+	of_node_put(qe);
+
+	return brg_clk;
+}
+EXPORT_SYMBOL(qe_get_brg_clk);
+
+/* Program the BRG to the given sampling rate and multiplier
+ *
+ * @brg: the BRG, QE_BRG1 - QE_BRG16
+ * @rate: the desired sampling rate
+ * @multiplier: corresponds to the value programmed in GUMR_L[RDCR] or
+ * GUMR_L[TDCR].  E.g., if this BRG is the RX clock, and GUMR_L[RDCR]=01,
+ * then 'multiplier' should be 8.
+ */
+int qe_setbrg(enum qe_clock brg, unsigned int rate, unsigned int multiplier)
+{
+	u32 divisor, tempval;
+	u32 div16 = 0;
+
+	if ((brg < QE_BRG1) || (brg > QE_BRG16))
+		return -EINVAL;
+
+	divisor = qe_get_brg_clk() / (rate * multiplier);
+
+	if (divisor > QE_BRGC_DIVISOR_MAX + 1) {
+		div16 = QE_BRGC_DIV16;
+		divisor /= 16;
+	}
+
+	/* Errata QE_General4, which affects some MPC832x and MPC836x SOCs, says
+	   that the BRG divisor must be even if you're not using divide-by-16
+	   mode. */
+	if (!div16 && (divisor & 1) && (divisor > 3))
+		divisor++;
+
+	tempval = ((divisor - 1) << QE_BRGC_DIVISOR_SHIFT) |
+		QE_BRGC_ENABLE | div16;
+
+	out_be32(&qe_immr->brg.brgc[brg - QE_BRG1], tempval);
+
+	return 0;
+}
+EXPORT_SYMBOL(qe_setbrg);
+
+/* Convert a string to a QE clock source enum
+ *
+ * This function takes a string, typically from a property in the device
+ * tree, and returns the corresponding "enum qe_clock" value.
+*/
+enum qe_clock qe_clock_source(const char *source)
+{
+	unsigned int i;
+
+	if (strcasecmp(source, "none") == 0)
+		return QE_CLK_NONE;
+
+	if (strncasecmp(source, "brg", 3) == 0) {
+		i = simple_strtoul(source + 3, NULL, 10);
+		if ((i >= 1) && (i <= 16))
+			return (QE_BRG1 - 1) + i;
+		else
+			return QE_CLK_DUMMY;
+	}
+
+	if (strncasecmp(source, "clk", 3) == 0) {
+		i = simple_strtoul(source + 3, NULL, 10);
+		if ((i >= 1) && (i <= 24))
+			return (QE_CLK1 - 1) + i;
+		else
+			return QE_CLK_DUMMY;
+	}
+
+	return QE_CLK_DUMMY;
+}
+EXPORT_SYMBOL(qe_clock_source);
+
+/* Initialize SNUMs (thread serial numbers) according to
+ * QE Module Control chapter, SNUM table
+ */
+static void qe_snums_init(void)
+{
+	int i;
+	static const u8 snum_init_76[] = {
+		0x04, 0x05, 0x0C, 0x0D, 0x14, 0x15, 0x1C, 0x1D,
+		0x24, 0x25, 0x2C, 0x2D, 0x34, 0x35, 0x88, 0x89,
+		0x98, 0x99, 0xA8, 0xA9, 0xB8, 0xB9, 0xC8, 0xC9,
+		0xD8, 0xD9, 0xE8, 0xE9, 0x44, 0x45, 0x4C, 0x4D,
+		0x54, 0x55, 0x5C, 0x5D, 0x64, 0x65, 0x6C, 0x6D,
+		0x74, 0x75, 0x7C, 0x7D, 0x84, 0x85, 0x8C, 0x8D,
+		0x94, 0x95, 0x9C, 0x9D, 0xA4, 0xA5, 0xAC, 0xAD,
+		0xB4, 0xB5, 0xBC, 0xBD, 0xC4, 0xC5, 0xCC, 0xCD,
+		0xD4, 0xD5, 0xDC, 0xDD, 0xE4, 0xE5, 0xEC, 0xED,
+		0xF4, 0xF5, 0xFC, 0xFD,
+	};
+	static const u8 snum_init_46[] = {
+		0x04, 0x05, 0x0C, 0x0D, 0x14, 0x15, 0x1C, 0x1D,
+		0x24, 0x25, 0x2C, 0x2D, 0x34, 0x35, 0x88, 0x89,
+		0x98, 0x99, 0xA8, 0xA9, 0xB8, 0xB9, 0xC8, 0xC9,
+		0xD8, 0xD9, 0xE8, 0xE9, 0x08, 0x09, 0x18, 0x19,
+		0x28, 0x29, 0x38, 0x39, 0x48, 0x49, 0x58, 0x59,
+		0x68, 0x69, 0x78, 0x79, 0x80, 0x81,
+	};
+	static const u8 *snum_init;
+
+	qe_num_of_snum = qe_get_num_of_snums();
+
+	if (qe_num_of_snum == 76)
+		snum_init = snum_init_76;
+	else
+		snum_init = snum_init_46;
+
+	for (i = 0; i < qe_num_of_snum; i++) {
+		snums[i].num = snum_init[i];
+		snums[i].state = QE_SNUM_STATE_FREE;
+	}
+}
+
+int qe_get_snum(void)
+{
+	unsigned long flags;
+	int snum = -EBUSY;
+	int i;
+
+	spin_lock_irqsave(&qe_lock, flags);
+	for (i = 0; i < qe_num_of_snum; i++) {
+		if (snums[i].state == QE_SNUM_STATE_FREE) {
+			snums[i].state = QE_SNUM_STATE_USED;
+			snum = snums[i].num;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&qe_lock, flags);
+
+	return snum;
+}
+EXPORT_SYMBOL(qe_get_snum);
+
+void qe_put_snum(u8 snum)
+{
+	int i;
+
+	for (i = 0; i < qe_num_of_snum; i++) {
+		if (snums[i].num == snum) {
+			snums[i].state = QE_SNUM_STATE_FREE;
+			break;
+		}
+	}
+}
+EXPORT_SYMBOL(qe_put_snum);
+
+static int qe_sdma_init(void)
+{
+	struct sdma __iomem *sdma = &qe_immr->sdma;
+	static unsigned long sdma_buf_offset = (unsigned long)-ENOMEM;
+
+	if (!sdma)
+		return -ENODEV;
+
+	/* allocate 2 internal temporary buffers (512 bytes size each) for
+	 * the SDMA */
+	if (IS_ERR_VALUE(sdma_buf_offset)) {
+		sdma_buf_offset = qe_muram_alloc(512 * 2, 4096);
+		if (IS_ERR_VALUE(sdma_buf_offset))
+			return -ENOMEM;
+	}
+
+	out_be32(&sdma->sdebcr, (u32) sdma_buf_offset & QE_SDEBCR_BA_MASK);
+ 	out_be32(&sdma->sdmr, (QE_SDMR_GLB_1_MSK |
+ 					(0x1 << QE_SDMR_CEN_SHIFT)));
+
+	return 0;
+}
+
+/* The maximum number of RISCs we support */
+#define MAX_QE_RISC     4
+
+/* Firmware information stored here for qe_get_firmware_info() */
+static struct qe_firmware_info qe_firmware_info;
+
+/*
+ * Set to 1 if QE firmware has been uploaded, and therefore
+ * qe_firmware_info contains valid data.
+ */
+static int qe_firmware_uploaded;
+
+/*
+ * Upload a QE microcode
+ *
+ * This function is a worker function for qe_upload_firmware().  It does
+ * the actual uploading of the microcode.
+ */
+static void qe_upload_microcode(const void *base,
+	const struct qe_microcode *ucode)
+{
+	const __be32 *code = base + be32_to_cpu(ucode->code_offset);
+	unsigned int i;
+
+	if (ucode->major || ucode->minor || ucode->revision)
+		printk(KERN_INFO "qe-firmware: "
+			"uploading microcode '%s' version %u.%u.%u\n",
+			ucode->id, ucode->major, ucode->minor, ucode->revision);
+	else
+		printk(KERN_INFO "qe-firmware: "
+			"uploading microcode '%s'\n", ucode->id);
+
+	/* Use auto-increment */
+	out_be32(&qe_immr->iram.iadd, be32_to_cpu(ucode->iram_offset) |
+		QE_IRAM_IADD_AIE | QE_IRAM_IADD_BADDR);
+
+	for (i = 0; i < be32_to_cpu(ucode->count); i++)
+		out_be32(&qe_immr->iram.idata, be32_to_cpu(code[i]));
+	
+	/* Set I-RAM Ready Register */
+	out_be32(&qe_immr->iram.iready, be32_to_cpu(QE_IRAM_READY));
+}
+
+/*
+ * Upload a microcode to the I-RAM at a specific address.
+ *
+ * See Documentation/powerpc/qe_firmware.txt for information on QE microcode
+ * uploading.
+ *
+ * Currently, only version 1 is supported, so the 'version' field must be
+ * set to 1.
+ *
+ * The SOC model and revision are not validated, they are only displayed for
+ * informational purposes.
+ *
+ * 'calc_size' is the calculated size, in bytes, of the firmware structure and
+ * all of the microcode structures, minus the CRC.
+ *
+ * 'length' is the size that the structure says it is, including the CRC.
+ */
+int qe_upload_firmware(const struct qe_firmware *firmware)
+{
+	unsigned int i;
+	unsigned int j;
+	u32 crc;
+	size_t calc_size = sizeof(struct qe_firmware);
+	size_t length;
+	const struct qe_header *hdr;
+
+	if (!firmware) {
+		printk(KERN_ERR "qe-firmware: invalid pointer\n");
+		return -EINVAL;
+	}
+
+	hdr = &firmware->header;
+	length = be32_to_cpu(hdr->length);
+
+	/* Check the magic */
+	if ((hdr->magic[0] != 'Q') || (hdr->magic[1] != 'E') ||
+	    (hdr->magic[2] != 'F')) {
+		printk(KERN_ERR "qe-firmware: not a microcode\n");
+		return -EPERM;
+	}
+
+	/* Check the version */
+	if (hdr->version != 1) {
+		printk(KERN_ERR "qe-firmware: unsupported version\n");
+		return -EPERM;
+	}
+
+	/* Validate some of the fields */
+	if ((firmware->count < 1) || (firmware->count > MAX_QE_RISC)) {
+		printk(KERN_ERR "qe-firmware: invalid data\n");
+		return -EINVAL;
+	}
+
+	/* Validate the length and check if there's a CRC */
+	calc_size += (firmware->count - 1) * sizeof(struct qe_microcode);
+
+	for (i = 0; i < firmware->count; i++)
+		/*
+		 * For situations where the second RISC uses the same microcode
+		 * as the first, the 'code_offset' and 'count' fields will be
+		 * zero, so it's okay to add those.
+		 */
+		calc_size += sizeof(__be32) *
+			be32_to_cpu(firmware->microcode[i].count);
+
+	/* Validate the length */
+	if (length != calc_size + sizeof(__be32)) {
+		printk(KERN_ERR "qe-firmware: invalid length\n");
+		return -EPERM;
+	}
+
+	/* Validate the CRC */
+	crc = be32_to_cpu(*(__be32 *)((void *)firmware + calc_size));
+	if (crc != crc32(0, firmware, calc_size)) {
+		printk(KERN_ERR "qe-firmware: firmware CRC is invalid\n");
+		return -EIO;
+	}
+
+	/*
+	 * If the microcode calls for it, split the I-RAM.
+	 */
+	if (!firmware->split)
+		setbits16(&qe_immr->cp.cercr, QE_CP_CERCR_CIR);
+
+	if (firmware->soc.model)
+		printk(KERN_INFO
+			"qe-firmware: firmware '%s' for %u V%u.%u\n",
+			firmware->id, be16_to_cpu(firmware->soc.model),
+			firmware->soc.major, firmware->soc.minor);
+	else
+		printk(KERN_INFO "qe-firmware: firmware '%s'\n",
+			firmware->id);
+
+	/*
+	 * The QE only supports one microcode per RISC, so clear out all the
+	 * saved microcode information and put in the new.
+	 */
+	memset(&qe_firmware_info, 0, sizeof(qe_firmware_info));
+	strlcpy(qe_firmware_info.id, firmware->id, sizeof(qe_firmware_info.id));
+	qe_firmware_info.extended_modes = firmware->extended_modes;
+	memcpy(qe_firmware_info.vtraps, firmware->vtraps,
+		sizeof(firmware->vtraps));
+
+	/* Loop through each microcode. */
+	for (i = 0; i < firmware->count; i++) {
+		const struct qe_microcode *ucode = &firmware->microcode[i];
+
+		/* Upload a microcode if it's present */
+		if (ucode->code_offset)
+			qe_upload_microcode(firmware, ucode);
+
+		/* Program the traps for this processor */
+		for (j = 0; j < 16; j++) {
+			u32 trap = be32_to_cpu(ucode->traps[j]);
+
+			if (trap)
+				out_be32(&qe_immr->rsp[i].tibcr[j], trap);
+		}
+
+		/* Enable traps */
+		out_be32(&qe_immr->rsp[i].eccr, be32_to_cpu(ucode->eccr));
+	}
+
+	qe_firmware_uploaded = 1;
+
+	return 0;
+}
+EXPORT_SYMBOL(qe_upload_firmware);
+
+/*
+ * Get info on the currently-loaded firmware
+ *
+ * This function also checks the device tree to see if the boot loader has
+ * uploaded a firmware already.
+ */
+struct qe_firmware_info *qe_get_firmware_info(void)
+{
+	static int initialized;
+	struct property *prop;
+	struct device_node *qe;
+	struct device_node *fw = NULL;
+	const char *sprop;
+	unsigned int i;
+
+	/*
+	 * If we haven't checked yet, and a driver hasn't uploaded a firmware
+	 * yet, then check the device tree for information.
+	 */
+	if (qe_firmware_uploaded)
+		return &qe_firmware_info;
+
+	if (initialized)
+		return NULL;
+
+	initialized = 1;
+
+	/*
+	 * Newer device trees have an "fsl,qe" compatible property for the QE
+	 * node, but we still need to support older device trees.
+	*/
+	qe = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!qe) {
+		qe = of_find_node_by_type(NULL, "qe");
+		if (!qe)
+			return NULL;
+	}
+
+	/* Find the 'firmware' child node */
+	for_each_child_of_node(qe, fw) {
+		if (strcmp(fw->name, "firmware") == 0)
+			break;
+	}
+
+	of_node_put(qe);
+
+	/* Did we find the 'firmware' node? */
+	if (!fw)
+		return NULL;
+
+	qe_firmware_uploaded = 1;
+
+	/* Copy the data into qe_firmware_info*/
+	sprop = of_get_property(fw, "id", NULL);
+	if (sprop)
+		strlcpy(qe_firmware_info.id, sprop,
+			sizeof(qe_firmware_info.id));
+
+	prop = of_find_property(fw, "extended-modes", NULL);
+	if (prop && (prop->length == sizeof(u64))) {
+		const u64 *iprop = prop->value;
+
+		qe_firmware_info.extended_modes = *iprop;
+	}
+
+	prop = of_find_property(fw, "virtual-traps", NULL);
+	if (prop && (prop->length == 32)) {
+		const u32 *iprop = prop->value;
+
+		for (i = 0; i < ARRAY_SIZE(qe_firmware_info.vtraps); i++)
+			qe_firmware_info.vtraps[i] = iprop[i];
+	}
+
+	of_node_put(fw);
+
+	return &qe_firmware_info;
+}
+EXPORT_SYMBOL(qe_get_firmware_info);
+
+unsigned int qe_get_num_of_risc(void)
+{
+	struct device_node *qe;
+	int size;
+	unsigned int num_of_risc = 0;
+	const u32 *prop;
+
+	qe = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!qe) {
+		/* Older devices trees did not have an "fsl,qe"
+		 * compatible property, so we need to look for
+		 * the QE node by name.
+		 */
+		qe = of_find_node_by_type(NULL, "qe");
+		if (!qe)
+			return num_of_risc;
+	}
+
+	prop = of_get_property(qe, "fsl,qe-num-riscs", &size);
+	if (prop && size == sizeof(*prop))
+		num_of_risc = *prop;
+
+	of_node_put(qe);
+
+	return num_of_risc;
+}
+EXPORT_SYMBOL(qe_get_num_of_risc);
+
+unsigned int qe_get_num_of_snums(void)
+{
+	struct device_node *qe;
+	int size;
+	unsigned int num_of_snums;
+	const u32 *prop;
+
+	num_of_snums = 28; /* The default number of snum for threads is 28 */
+	qe = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!qe) {
+		/* Older devices trees did not have an "fsl,qe"
+		 * compatible property, so we need to look for
+		 * the QE node by name.
+		 */
+		qe = of_find_node_by_type(NULL, "qe");
+		if (!qe)
+			return num_of_snums;
+	}
+
+	prop = of_get_property(qe, "fsl,qe-num-snums", &size);
+	if (prop && size == sizeof(*prop)) {
+		num_of_snums = *prop;
+		if ((num_of_snums < 28) || (num_of_snums > QE_NUM_OF_SNUM)) {
+			/* No QE ever has fewer than 28 SNUMs */
+			pr_err("QE: number of snum is invalid\n");
+			of_node_put(qe);
+			return -EINVAL;
+		}
+	}
+
+	of_node_put(qe);
+
+	return num_of_snums;
+}
+EXPORT_SYMBOL(qe_get_num_of_snums);
+
+static int __init qe_init(void)
+{
+	struct device_node *np;
+
+	np = of_find_compatible_node(NULL, NULL, "fsl,qe");
+	if (!np)
+		return -ENODEV;
+	qe_reset();
+	of_node_put(np);
+	return 0;
+}
+subsys_initcall(qe_init);
+
+#if defined(CONFIG_SUSPEND) && defined(CONFIG_PPC_85xx)
+static int qe_resume(struct platform_device *ofdev)
+{
+	if (!qe_alive_during_sleep())
+		qe_reset();
+	return 0;
+}
+
+static int qe_probe(struct platform_device *ofdev)
+{
+	return 0;
+}
+
+static const struct of_device_id qe_ids[] = {
+	{ .compatible = "fsl,qe", },
+	{ },
+};
+
+static struct platform_driver qe_driver = {
+	.driver = {
+		.name = "fsl-qe",
+		.of_match_table = qe_ids,
+	},
+	.probe = qe_probe,
+	.resume = qe_resume,
+};
+
+static int __init qe_drv_init(void)
+{
+	return platform_driver_register(&qe_driver);
+}
+device_initcall(qe_drv_init);
+#endif /* defined(CONFIG_SUSPEND) && defined(CONFIG_PPC_85xx) */
diff --git a/drivers/soc/fsl/qe/qe_common.c b/drivers/soc/fsl/qe/qe_common.c
new file mode 100644
index 000000000000..419fa5b7be4d
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe_common.c
@@ -0,0 +1,235 @@
+/*
+ * Common CPM code
+ *
+ * Author: Scott Wood <scottwood@freescale.com>
+ *
+ * Copyright 2007-2008,2010 Freescale Semiconductor, Inc.
+ *
+ * Some parts derived from commproc.c/cpm2_common.c, which is:
+ * Copyright (c) 1997 Dan error_act (dmalek@jlc.net)
+ * Copyright (c) 1999-2001 Dan Malek <dan@embeddedalley.com>
+ * Copyright (c) 2000 MontaVista Software, Inc (source@mvista.com)
+ * 2006 (c) MontaVista Software, Inc.
+ * Vitaly Bordug <vbordug@ru.mvista.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ */
+#include <linux/genalloc.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/of_device.h>
+#include <linux/spinlock.h>
+#include <linux/export.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <soc/fsl/qe/qe.h>
+
+static struct gen_pool *muram_pool;
+static spinlock_t cpm_muram_lock;
+static u8 __iomem *muram_vbase;
+static phys_addr_t muram_pbase;
+
+struct muram_block {
+	struct list_head head;
+	unsigned long start;
+	int size;
+};
+
+static LIST_HEAD(muram_block_list);
+
+/* max address size we deal with */
+#define OF_MAX_ADDR_CELLS	4
+#define GENPOOL_OFFSET		(4096 * 8)
+
+int cpm_muram_init(void)
+{
+	struct device_node *np;
+	struct resource r;
+	u32 zero[OF_MAX_ADDR_CELLS] = {};
+	resource_size_t max = 0;
+	int i = 0;
+	int ret = 0;
+
+	if (muram_pbase)
+		return 0;
+
+	spin_lock_init(&cpm_muram_lock);
+	np = of_find_compatible_node(NULL, NULL, "fsl,cpm-muram-data");
+	if (!np) {
+		/* try legacy bindings */
+		np = of_find_node_by_name(NULL, "data-only");
+		if (!np) {
+			pr_err("Cannot find CPM muram data node");
+			ret = -ENODEV;
+			goto out_muram;
+		}
+	}
+
+	muram_pool = gen_pool_create(0, -1);
+	muram_pbase = of_translate_address(np, zero);
+	if (muram_pbase == (phys_addr_t)OF_BAD_ADDR) {
+		pr_err("Cannot translate zero through CPM muram node");
+		ret = -ENODEV;
+		goto out_pool;
+	}
+
+	while (of_address_to_resource(np, i++, &r) == 0) {
+		if (r.end > max)
+			max = r.end;
+		ret = gen_pool_add(muram_pool, r.start - muram_pbase +
+				   GENPOOL_OFFSET, resource_size(&r), -1);
+		if (ret) {
+			pr_err("QE: couldn't add muram to pool!\n");
+			goto out_pool;
+		}
+	}
+
+	muram_vbase = ioremap(muram_pbase, max - muram_pbase + 1);
+	if (!muram_vbase) {
+		pr_err("Cannot map QE muram");
+		ret = -ENOMEM;
+		goto out_pool;
+	}
+	goto out_muram;
+out_pool:
+	gen_pool_destroy(muram_pool);
+out_muram:
+	of_node_put(np);
+	return ret;
+}
+
+/*
+ * cpm_muram_alloc - allocate the requested size worth of multi-user ram
+ * @size: number of bytes to allocate
+ * @align: requested alignment, in bytes
+ *
+ * This function returns an offset into the muram area.
+ * Use cpm_dpram_addr() to get the virtual address of the area.
+ * Use cpm_muram_free() to free the allocation.
+ */
+unsigned long cpm_muram_alloc(unsigned long size, unsigned long align)
+{
+	unsigned long start;
+	unsigned long flags;
+	struct genpool_data_align muram_pool_data;
+
+	spin_lock_irqsave(&cpm_muram_lock, flags);
+	muram_pool_data.align = align;
+	start = cpm_muram_alloc_common(size, gen_pool_first_fit_align,
+				       &muram_pool_data);
+	spin_unlock_irqrestore(&cpm_muram_lock, flags);
+	return start;
+}
+EXPORT_SYMBOL(cpm_muram_alloc);
+
+/**
+ * cpm_muram_free - free a chunk of multi-user ram
+ * @offset: The beginning of the chunk as returned by cpm_muram_alloc().
+ */
+int cpm_muram_free(unsigned long offset)
+{
+	unsigned long flags;
+	int size;
+	struct muram_block *tmp;
+
+	size = 0;
+	spin_lock_irqsave(&cpm_muram_lock, flags);
+	list_for_each_entry(tmp, &muram_block_list, head) {
+		if (tmp->start == offset) {
+			size = tmp->size;
+			list_del(&tmp->head);
+			kfree(tmp);
+			break;
+		}
+	}
+	gen_pool_free(muram_pool, offset + GENPOOL_OFFSET, size);
+	spin_unlock_irqrestore(&cpm_muram_lock, flags);
+	return size;
+}
+EXPORT_SYMBOL(cpm_muram_free);
+
+/*
+ * cpm_muram_alloc_fixed - reserve a specific region of multi-user ram
+ * @offset: offset of allocation start address
+ * @size: number of bytes to allocate
+ * This function returns an offset into the muram area
+ * Use cpm_dpram_addr() to get the virtual address of the area.
+ * Use cpm_muram_free() to free the allocation.
+ */
+unsigned long cpm_muram_alloc_fixed(unsigned long offset, unsigned long size)
+{
+	unsigned long start;
+	unsigned long flags;
+	struct genpool_data_fixed muram_pool_data_fixed;
+
+	spin_lock_irqsave(&cpm_muram_lock, flags);
+	muram_pool_data_fixed.offset = offset + GENPOOL_OFFSET;
+	start = cpm_muram_alloc_common(size, gen_pool_fixed_alloc,
+				       &muram_pool_data_fixed);
+	spin_unlock_irqrestore(&cpm_muram_lock, flags);
+	return start;
+}
+EXPORT_SYMBOL(cpm_muram_alloc_fixed);
+
+/*
+ * cpm_muram_alloc_common - cpm_muram_alloc common code
+ * @size: number of bytes to allocate
+ * @algo: algorithm for alloc.
+ * @data: data for genalloc's algorithm.
+ *
+ * This function returns an offset into the muram area.
+ */
+unsigned long cpm_muram_alloc_common(unsigned long size, genpool_algo_t algo,
+				     void *data)
+{
+	struct muram_block *entry;
+	unsigned long start;
+
+	start = gen_pool_alloc_algo(muram_pool, size, algo, data);
+	if (!start)
+		goto out2;
+	start = start - GENPOOL_OFFSET;
+	memset_io(cpm_muram_addr(start), 0, size);
+	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry)
+		goto out1;
+	entry->start = start;
+	entry->size = size;
+	list_add(&entry->head, &muram_block_list);
+
+	return start;
+out1:
+	gen_pool_free(muram_pool, start, size);
+out2:
+	return (unsigned long)-ENOMEM;
+}
+
+/**
+ * cpm_muram_addr - turn a muram offset into a virtual address
+ * @offset: muram offset to convert
+ */
+void __iomem *cpm_muram_addr(unsigned long offset)
+{
+	return muram_vbase + offset;
+}
+EXPORT_SYMBOL(cpm_muram_addr);
+
+unsigned long cpm_muram_offset(void __iomem *addr)
+{
+	return addr - (void __iomem *)muram_vbase;
+}
+EXPORT_SYMBOL(cpm_muram_offset);
+
+/**
+ * cpm_muram_dma - turn a muram virtual address into a DMA address
+ * @offset: virtual address from cpm_muram_addr() to convert
+ */
+dma_addr_t cpm_muram_dma(void __iomem *addr)
+{
+	return muram_pbase + ((u8 __iomem *)addr - muram_vbase);
+}
+EXPORT_SYMBOL(cpm_muram_dma);
diff --git a/drivers/soc/fsl/qe/qe_ic.c b/drivers/soc/fsl/qe/qe_ic.c
new file mode 100644
index 000000000000..b77d01ff8330
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe_ic.c
@@ -0,0 +1,503 @@
+/*
+ * arch/powerpc/sysdev/qe_lib/qe_ic.c
+ *
+ * Copyright (C) 2006 Freescale Semiconductor, Inc.  All rights reserved.
+ *
+ * Author: Li Yang <leoli@freescale.com>
+ * Based on code from Shlomi Gridish <gridish@freescale.com>
+ *
+ * QUICC ENGINE Interrupt Controller
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/reboot.h>
+#include <linux/slab.h>
+#include <linux/stddef.h>
+#include <linux/sched.h>
+#include <linux/signal.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <asm/irq.h>
+#include <asm/io.h>
+#include <soc/fsl/qe/qe_ic.h>
+
+#include "qe_ic.h"
+
+static DEFINE_RAW_SPINLOCK(qe_ic_lock);
+
+static struct qe_ic_info qe_ic_info[] = {
+	[1] = {
+	       .mask = 0x00008000,
+	       .mask_reg = QEIC_CIMR,
+	       .pri_code = 0,
+	       .pri_reg = QEIC_CIPWCC,
+	       },
+	[2] = {
+	       .mask = 0x00004000,
+	       .mask_reg = QEIC_CIMR,
+	       .pri_code = 1,
+	       .pri_reg = QEIC_CIPWCC,
+	       },
+	[3] = {
+	       .mask = 0x00002000,
+	       .mask_reg = QEIC_CIMR,
+	       .pri_code = 2,
+	       .pri_reg = QEIC_CIPWCC,
+	       },
+	[10] = {
+		.mask = 0x00000040,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 1,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[11] = {
+		.mask = 0x00000020,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 2,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[12] = {
+		.mask = 0x00000010,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 3,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[13] = {
+		.mask = 0x00000008,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 4,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[14] = {
+		.mask = 0x00000004,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 5,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[15] = {
+		.mask = 0x00000002,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 6,
+		.pri_reg = QEIC_CIPZCC,
+		},
+	[20] = {
+		.mask = 0x10000000,
+		.mask_reg = QEIC_CRIMR,
+		.pri_code = 3,
+		.pri_reg = QEIC_CIPRTA,
+		},
+	[25] = {
+		.mask = 0x00800000,
+		.mask_reg = QEIC_CRIMR,
+		.pri_code = 0,
+		.pri_reg = QEIC_CIPRTB,
+		},
+	[26] = {
+		.mask = 0x00400000,
+		.mask_reg = QEIC_CRIMR,
+		.pri_code = 1,
+		.pri_reg = QEIC_CIPRTB,
+		},
+	[27] = {
+		.mask = 0x00200000,
+		.mask_reg = QEIC_CRIMR,
+		.pri_code = 2,
+		.pri_reg = QEIC_CIPRTB,
+		},
+	[28] = {
+		.mask = 0x00100000,
+		.mask_reg = QEIC_CRIMR,
+		.pri_code = 3,
+		.pri_reg = QEIC_CIPRTB,
+		},
+	[32] = {
+		.mask = 0x80000000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 0,
+		.pri_reg = QEIC_CIPXCC,
+		},
+	[33] = {
+		.mask = 0x40000000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 1,
+		.pri_reg = QEIC_CIPXCC,
+		},
+	[34] = {
+		.mask = 0x20000000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 2,
+		.pri_reg = QEIC_CIPXCC,
+		},
+	[35] = {
+		.mask = 0x10000000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 3,
+		.pri_reg = QEIC_CIPXCC,
+		},
+	[36] = {
+		.mask = 0x08000000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 4,
+		.pri_reg = QEIC_CIPXCC,
+		},
+	[40] = {
+		.mask = 0x00800000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 0,
+		.pri_reg = QEIC_CIPYCC,
+		},
+	[41] = {
+		.mask = 0x00400000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 1,
+		.pri_reg = QEIC_CIPYCC,
+		},
+	[42] = {
+		.mask = 0x00200000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 2,
+		.pri_reg = QEIC_CIPYCC,
+		},
+	[43] = {
+		.mask = 0x00100000,
+		.mask_reg = QEIC_CIMR,
+		.pri_code = 3,
+		.pri_reg = QEIC_CIPYCC,
+		},
+};
+
+static inline u32 qe_ic_read(volatile __be32  __iomem * base, unsigned int reg)
+{
+	return in_be32(base + (reg >> 2));
+}
+
+static inline void qe_ic_write(volatile __be32  __iomem * base, unsigned int reg,
+			       u32 value)
+{
+	out_be32(base + (reg >> 2), value);
+}
+
+static inline struct qe_ic *qe_ic_from_irq(unsigned int virq)
+{
+	return irq_get_chip_data(virq);
+}
+
+static inline struct qe_ic *qe_ic_from_irq_data(struct irq_data *d)
+{
+	return irq_data_get_irq_chip_data(d);
+}
+
+static void qe_ic_unmask_irq(struct irq_data *d)
+{
+	struct qe_ic *qe_ic = qe_ic_from_irq_data(d);
+	unsigned int src = irqd_to_hwirq(d);
+	unsigned long flags;
+	u32 temp;
+
+	raw_spin_lock_irqsave(&qe_ic_lock, flags);
+
+	temp = qe_ic_read(qe_ic->regs, qe_ic_info[src].mask_reg);
+	qe_ic_write(qe_ic->regs, qe_ic_info[src].mask_reg,
+		    temp | qe_ic_info[src].mask);
+
+	raw_spin_unlock_irqrestore(&qe_ic_lock, flags);
+}
+
+static void qe_ic_mask_irq(struct irq_data *d)
+{
+	struct qe_ic *qe_ic = qe_ic_from_irq_data(d);
+	unsigned int src = irqd_to_hwirq(d);
+	unsigned long flags;
+	u32 temp;
+
+	raw_spin_lock_irqsave(&qe_ic_lock, flags);
+
+	temp = qe_ic_read(qe_ic->regs, qe_ic_info[src].mask_reg);
+	qe_ic_write(qe_ic->regs, qe_ic_info[src].mask_reg,
+		    temp & ~qe_ic_info[src].mask);
+
+	/* Flush the above write before enabling interrupts; otherwise,
+	 * spurious interrupts will sometimes happen.  To be 100% sure
+	 * that the write has reached the device before interrupts are
+	 * enabled, the mask register would have to be read back; however,
+	 * this is not required for correctness, only to avoid wasting
+	 * time on a large number of spurious interrupts.  In testing,
+	 * a sync reduced the observed spurious interrupts to zero.
+	 */
+	mb();
+
+	raw_spin_unlock_irqrestore(&qe_ic_lock, flags);
+}
+
+static struct irq_chip qe_ic_irq_chip = {
+	.name = "QEIC",
+	.irq_unmask = qe_ic_unmask_irq,
+	.irq_mask = qe_ic_mask_irq,
+	.irq_mask_ack = qe_ic_mask_irq,
+};
+
+static int qe_ic_host_match(struct irq_domain *h, struct device_node *node,
+			    enum irq_domain_bus_token bus_token)
+{
+	/* Exact match, unless qe_ic node is NULL */
+	struct device_node *of_node = irq_domain_get_of_node(h);
+	return of_node == NULL || of_node == node;
+}
+
+static int qe_ic_host_map(struct irq_domain *h, unsigned int virq,
+			  irq_hw_number_t hw)
+{
+	struct qe_ic *qe_ic = h->host_data;
+	struct irq_chip *chip;
+
+	if (qe_ic_info[hw].mask == 0) {
+		printk(KERN_ERR "Can't map reserved IRQ\n");
+		return -EINVAL;
+	}
+	/* Default chip */
+	chip = &qe_ic->hc_irq;
+
+	irq_set_chip_data(virq, qe_ic);
+	irq_set_status_flags(virq, IRQ_LEVEL);
+
+	irq_set_chip_and_handler(virq, chip, handle_level_irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops qe_ic_host_ops = {
+	.match = qe_ic_host_match,
+	.map = qe_ic_host_map,
+	.xlate = irq_domain_xlate_onetwocell,
+};
+
+/* Return an interrupt vector or NO_IRQ if no interrupt is pending. */
+unsigned int qe_ic_get_low_irq(struct qe_ic *qe_ic)
+{
+	int irq;
+
+	BUG_ON(qe_ic == NULL);
+
+	/* get the interrupt source vector. */
+	irq = qe_ic_read(qe_ic->regs, QEIC_CIVEC) >> 26;
+
+	if (irq == 0)
+		return NO_IRQ;
+
+	return irq_linear_revmap(qe_ic->irqhost, irq);
+}
+
+/* Return an interrupt vector or NO_IRQ if no interrupt is pending. */
+unsigned int qe_ic_get_high_irq(struct qe_ic *qe_ic)
+{
+	int irq;
+
+	BUG_ON(qe_ic == NULL);
+
+	/* get the interrupt source vector. */
+	irq = qe_ic_read(qe_ic->regs, QEIC_CHIVEC) >> 26;
+
+	if (irq == 0)
+		return NO_IRQ;
+
+	return irq_linear_revmap(qe_ic->irqhost, irq);
+}
+
+void __init qe_ic_init(struct device_node *node, unsigned int flags,
+		       void (*low_handler)(struct irq_desc *desc),
+		       void (*high_handler)(struct irq_desc *desc))
+{
+	struct qe_ic *qe_ic;
+	struct resource res;
+	u32 temp = 0, ret, high_active = 0;
+
+	ret = of_address_to_resource(node, 0, &res);
+	if (ret)
+		return;
+
+	qe_ic = kzalloc(sizeof(*qe_ic), GFP_KERNEL);
+	if (qe_ic == NULL)
+		return;
+
+	qe_ic->irqhost = irq_domain_add_linear(node, NR_QE_IC_INTS,
+					       &qe_ic_host_ops, qe_ic);
+	if (qe_ic->irqhost == NULL) {
+		kfree(qe_ic);
+		return;
+	}
+
+	qe_ic->regs = ioremap(res.start, resource_size(&res));
+
+	qe_ic->hc_irq = qe_ic_irq_chip;
+
+	qe_ic->virq_high = irq_of_parse_and_map(node, 0);
+	qe_ic->virq_low = irq_of_parse_and_map(node, 1);
+
+	if (qe_ic->virq_low == NO_IRQ) {
+		printk(KERN_ERR "Failed to map QE_IC low IRQ\n");
+		kfree(qe_ic);
+		return;
+	}
+
+	/* default priority scheme is grouped. If spread mode is    */
+	/* required, configure cicr accordingly.                    */
+	if (flags & QE_IC_SPREADMODE_GRP_W)
+		temp |= CICR_GWCC;
+	if (flags & QE_IC_SPREADMODE_GRP_X)
+		temp |= CICR_GXCC;
+	if (flags & QE_IC_SPREADMODE_GRP_Y)
+		temp |= CICR_GYCC;
+	if (flags & QE_IC_SPREADMODE_GRP_Z)
+		temp |= CICR_GZCC;
+	if (flags & QE_IC_SPREADMODE_GRP_RISCA)
+		temp |= CICR_GRTA;
+	if (flags & QE_IC_SPREADMODE_GRP_RISCB)
+		temp |= CICR_GRTB;
+
+	/* choose destination signal for highest priority interrupt */
+	if (flags & QE_IC_HIGH_SIGNAL) {
+		temp |= (SIGNAL_HIGH << CICR_HPIT_SHIFT);
+		high_active = 1;
+	}
+
+	qe_ic_write(qe_ic->regs, QEIC_CICR, temp);
+
+	irq_set_handler_data(qe_ic->virq_low, qe_ic);
+	irq_set_chained_handler(qe_ic->virq_low, low_handler);
+
+	if (qe_ic->virq_high != NO_IRQ &&
+			qe_ic->virq_high != qe_ic->virq_low) {
+		irq_set_handler_data(qe_ic->virq_high, qe_ic);
+		irq_set_chained_handler(qe_ic->virq_high, high_handler);
+	}
+}
+
+void qe_ic_set_highest_priority(unsigned int virq, int high)
+{
+	struct qe_ic *qe_ic = qe_ic_from_irq(virq);
+	unsigned int src = virq_to_hw(virq);
+	u32 temp = 0;
+
+	temp = qe_ic_read(qe_ic->regs, QEIC_CICR);
+
+	temp &= ~CICR_HP_MASK;
+	temp |= src << CICR_HP_SHIFT;
+
+	temp &= ~CICR_HPIT_MASK;
+	temp |= (high ? SIGNAL_HIGH : SIGNAL_LOW) << CICR_HPIT_SHIFT;
+
+	qe_ic_write(qe_ic->regs, QEIC_CICR, temp);
+}
+
+/* Set Priority level within its group, from 1 to 8 */
+int qe_ic_set_priority(unsigned int virq, unsigned int priority)
+{
+	struct qe_ic *qe_ic = qe_ic_from_irq(virq);
+	unsigned int src = virq_to_hw(virq);
+	u32 temp;
+
+	if (priority > 8 || priority == 0)
+		return -EINVAL;
+	if (src > 127)
+		return -EINVAL;
+	if (qe_ic_info[src].pri_reg == 0)
+		return -EINVAL;
+
+	temp = qe_ic_read(qe_ic->regs, qe_ic_info[src].pri_reg);
+
+	if (priority < 4) {
+		temp &= ~(0x7 << (32 - priority * 3));
+		temp |= qe_ic_info[src].pri_code << (32 - priority * 3);
+	} else {
+		temp &= ~(0x7 << (24 - priority * 3));
+		temp |= qe_ic_info[src].pri_code << (24 - priority * 3);
+	}
+
+	qe_ic_write(qe_ic->regs, qe_ic_info[src].pri_reg, temp);
+
+	return 0;
+}
+
+/* Set a QE priority to use high irq, only priority 1~2 can use high irq */
+int qe_ic_set_high_priority(unsigned int virq, unsigned int priority, int high)
+{
+	struct qe_ic *qe_ic = qe_ic_from_irq(virq);
+	unsigned int src = virq_to_hw(virq);
+	u32 temp, control_reg = QEIC_CICNR, shift = 0;
+
+	if (priority > 2 || priority == 0)
+		return -EINVAL;
+
+	switch (qe_ic_info[src].pri_reg) {
+	case QEIC_CIPZCC:
+		shift = CICNR_ZCC1T_SHIFT;
+		break;
+	case QEIC_CIPWCC:
+		shift = CICNR_WCC1T_SHIFT;
+		break;
+	case QEIC_CIPYCC:
+		shift = CICNR_YCC1T_SHIFT;
+		break;
+	case QEIC_CIPXCC:
+		shift = CICNR_XCC1T_SHIFT;
+		break;
+	case QEIC_CIPRTA:
+		shift = CRICR_RTA1T_SHIFT;
+		control_reg = QEIC_CRICR;
+		break;
+	case QEIC_CIPRTB:
+		shift = CRICR_RTB1T_SHIFT;
+		control_reg = QEIC_CRICR;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	shift += (2 - priority) * 2;
+	temp = qe_ic_read(qe_ic->regs, control_reg);
+	temp &= ~(SIGNAL_MASK << shift);
+	temp |= (high ? SIGNAL_HIGH : SIGNAL_LOW) << shift;
+	qe_ic_write(qe_ic->regs, control_reg, temp);
+
+	return 0;
+}
+
+static struct bus_type qe_ic_subsys = {
+	.name = "qe_ic",
+	.dev_name = "qe_ic",
+};
+
+static struct device device_qe_ic = {
+	.id = 0,
+	.bus = &qe_ic_subsys,
+};
+
+static int __init init_qe_ic_sysfs(void)
+{
+	int rc;
+
+	printk(KERN_DEBUG "Registering qe_ic with sysfs...\n");
+
+	rc = subsys_system_register(&qe_ic_subsys, NULL);
+	if (rc) {
+		printk(KERN_ERR "Failed registering qe_ic sys class\n");
+		return -ENODEV;
+	}
+	rc = device_register(&device_qe_ic);
+	if (rc) {
+		printk(KERN_ERR "Failed registering qe_ic sys device\n");
+		return -ENODEV;
+	}
+	return 0;
+}
+
+subsys_initcall(init_qe_ic_sysfs);
diff --git a/drivers/soc/fsl/qe/qe_ic.h b/drivers/soc/fsl/qe/qe_ic.h
new file mode 100644
index 000000000000..926a2ed42319
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe_ic.h
@@ -0,0 +1,103 @@
+/*
+ * drivers/soc/fsl/qe/qe_ic.h
+ *
+ * QUICC ENGINE Interrupt Controller Header
+ *
+ * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Author: Li Yang <leoli@freescale.com>
+ * Based on code from Shlomi Gridish <gridish@freescale.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#ifndef _POWERPC_SYSDEV_QE_IC_H
+#define _POWERPC_SYSDEV_QE_IC_H
+
+#include <soc/fsl/qe/qe_ic.h>
+
+#define NR_QE_IC_INTS		64
+
+/* QE IC registers offset */
+#define QEIC_CICR		0x00
+#define QEIC_CIVEC		0x04
+#define QEIC_CRIPNR		0x08
+#define QEIC_CIPNR		0x0c
+#define QEIC_CIPXCC		0x10
+#define QEIC_CIPYCC		0x14
+#define QEIC_CIPWCC		0x18
+#define QEIC_CIPZCC		0x1c
+#define QEIC_CIMR		0x20
+#define QEIC_CRIMR		0x24
+#define QEIC_CICNR		0x28
+#define QEIC_CIPRTA		0x30
+#define QEIC_CIPRTB		0x34
+#define QEIC_CRICR		0x3c
+#define QEIC_CHIVEC		0x60
+
+/* Interrupt priority registers */
+#define CIPCC_SHIFT_PRI0	29
+#define CIPCC_SHIFT_PRI1	26
+#define CIPCC_SHIFT_PRI2	23
+#define CIPCC_SHIFT_PRI3	20
+#define CIPCC_SHIFT_PRI4	13
+#define CIPCC_SHIFT_PRI5	10
+#define CIPCC_SHIFT_PRI6	7
+#define CIPCC_SHIFT_PRI7	4
+
+/* CICR priority modes */
+#define CICR_GWCC		0x00040000
+#define CICR_GXCC		0x00020000
+#define CICR_GYCC		0x00010000
+#define CICR_GZCC		0x00080000
+#define CICR_GRTA		0x00200000
+#define CICR_GRTB		0x00400000
+#define CICR_HPIT_SHIFT		8
+#define CICR_HPIT_MASK		0x00000300
+#define CICR_HP_SHIFT		24
+#define CICR_HP_MASK		0x3f000000
+
+/* CICNR */
+#define CICNR_WCC1T_SHIFT	20
+#define CICNR_ZCC1T_SHIFT	28
+#define CICNR_YCC1T_SHIFT	12
+#define CICNR_XCC1T_SHIFT	4
+
+/* CRICR */
+#define CRICR_RTA1T_SHIFT	20
+#define CRICR_RTB1T_SHIFT	28
+
+/* Signal indicator */
+#define SIGNAL_MASK		3
+#define SIGNAL_HIGH		2
+#define SIGNAL_LOW		0
+
+struct qe_ic {
+	/* Control registers offset */
+	volatile u32 __iomem *regs;
+
+	/* The remapper for this QEIC */
+	struct irq_domain *irqhost;
+
+	/* The "linux" controller struct */
+	struct irq_chip hc_irq;
+
+	/* VIRQ numbers of QE high/low irqs */
+	unsigned int virq_high;
+	unsigned int virq_low;
+};
+
+/*
+ * QE interrupt controller internal structure
+ */
+struct qe_ic_info {
+	u32	mask;	  /* location of this source at the QIMR register. */
+	u32	mask_reg; /* Mask register offset */
+	u8	pri_code; /* for grouped interrupts sources - the interrupt
+			     code as appears at the group priority register */
+	u32	pri_reg;  /* Group priority register offset */
+};
+
+#endif /* _POWERPC_SYSDEV_QE_IC_H */
diff --git a/drivers/soc/fsl/qe/qe_io.c b/drivers/soc/fsl/qe/qe_io.c
new file mode 100644
index 000000000000..7ae59abc7863
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe_io.c
@@ -0,0 +1,192 @@
+/*
+ * arch/powerpc/sysdev/qe_lib/qe_io.c
+ *
+ * QE Parallel I/O ports configuration routines
+ *
+ * Copyright 2006 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Author: Li Yang <LeoLi@freescale.com>
+ * Based on code from Shlomi Gridish <gridish@freescale.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/stddef.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/ioport.h>
+
+#include <asm/io.h>
+#include <soc/fsl/qe/qe.h>
+#include <asm/prom.h>
+#include <sysdev/fsl_soc.h>
+
+#undef DEBUG
+
+static struct qe_pio_regs __iomem *par_io;
+static int num_par_io_ports = 0;
+
+int par_io_init(struct device_node *np)
+{
+	struct resource res;
+	int ret;
+	const u32 *num_ports;
+
+	/* Map Parallel I/O ports registers */
+	ret = of_address_to_resource(np, 0, &res);
+	if (ret)
+		return ret;
+	par_io = ioremap(res.start, resource_size(&res));
+
+	num_ports = of_get_property(np, "num-ports", NULL);
+	if (num_ports)
+		num_par_io_ports = *num_ports;
+
+	return 0;
+}
+
+void __par_io_config_pin(struct qe_pio_regs __iomem *par_io, u8 pin, int dir,
+			 int open_drain, int assignment, int has_irq)
+{
+	u32 pin_mask1bit;
+	u32 pin_mask2bits;
+	u32 new_mask2bits;
+	u32 tmp_val;
+
+	/* calculate pin location for single and 2 bits information */
+	pin_mask1bit = (u32) (1 << (QE_PIO_PINS - (pin + 1)));
+
+	/* Set open drain, if required */
+	tmp_val = in_be32(&par_io->cpodr);
+	if (open_drain)
+		out_be32(&par_io->cpodr, pin_mask1bit | tmp_val);
+	else
+		out_be32(&par_io->cpodr, ~pin_mask1bit & tmp_val);
+
+	/* define direction */
+	tmp_val = (pin > (QE_PIO_PINS / 2) - 1) ?
+		in_be32(&par_io->cpdir2) :
+		in_be32(&par_io->cpdir1);
+
+	/* get all bits mask for 2 bit per port */
+	pin_mask2bits = (u32) (0x3 << (QE_PIO_PINS -
+				(pin % (QE_PIO_PINS / 2) + 1) * 2));
+
+	/* Get the final mask we need for the right definition */
+	new_mask2bits = (u32) (dir << (QE_PIO_PINS -
+				(pin % (QE_PIO_PINS / 2) + 1) * 2));
+
+	/* clear and set 2 bits mask */
+	if (pin > (QE_PIO_PINS / 2) - 1) {
+		out_be32(&par_io->cpdir2,
+			 ~pin_mask2bits & tmp_val);
+		tmp_val &= ~pin_mask2bits;
+		out_be32(&par_io->cpdir2, new_mask2bits | tmp_val);
+	} else {
+		out_be32(&par_io->cpdir1,
+			 ~pin_mask2bits & tmp_val);
+		tmp_val &= ~pin_mask2bits;
+		out_be32(&par_io->cpdir1, new_mask2bits | tmp_val);
+	}
+	/* define pin assignment */
+	tmp_val = (pin > (QE_PIO_PINS / 2) - 1) ?
+		in_be32(&par_io->cppar2) :
+		in_be32(&par_io->cppar1);
+
+	new_mask2bits = (u32) (assignment << (QE_PIO_PINS -
+			(pin % (QE_PIO_PINS / 2) + 1) * 2));
+	/* clear and set 2 bits mask */
+	if (pin > (QE_PIO_PINS / 2) - 1) {
+		out_be32(&par_io->cppar2,
+			 ~pin_mask2bits & tmp_val);
+		tmp_val &= ~pin_mask2bits;
+		out_be32(&par_io->cppar2, new_mask2bits | tmp_val);
+	} else {
+		out_be32(&par_io->cppar1,
+			 ~pin_mask2bits & tmp_val);
+		tmp_val &= ~pin_mask2bits;
+		out_be32(&par_io->cppar1, new_mask2bits | tmp_val);
+	}
+}
+EXPORT_SYMBOL(__par_io_config_pin);
+
+int par_io_config_pin(u8 port, u8 pin, int dir, int open_drain,
+		      int assignment, int has_irq)
+{
+	if (!par_io || port >= num_par_io_ports)
+		return -EINVAL;
+
+	__par_io_config_pin(&par_io[port], pin, dir, open_drain, assignment,
+			    has_irq);
+	return 0;
+}
+EXPORT_SYMBOL(par_io_config_pin);
+
+int par_io_data_set(u8 port, u8 pin, u8 val)
+{
+	u32 pin_mask, tmp_val;
+
+	if (port >= num_par_io_ports)
+		return -EINVAL;
+	if (pin >= QE_PIO_PINS)
+		return -EINVAL;
+	/* calculate pin location */
+	pin_mask = (u32) (1 << (QE_PIO_PINS - 1 - pin));
+
+	tmp_val = in_be32(&par_io[port].cpdata);
+
+	if (val == 0)		/* clear */
+		out_be32(&par_io[port].cpdata, ~pin_mask & tmp_val);
+	else			/* set */
+		out_be32(&par_io[port].cpdata, pin_mask | tmp_val);
+
+	return 0;
+}
+EXPORT_SYMBOL(par_io_data_set);
+
+int par_io_of_config(struct device_node *np)
+{
+	struct device_node *pio;
+	const phandle *ph;
+	int pio_map_len;
+	const unsigned int *pio_map;
+
+	if (par_io == NULL) {
+		printk(KERN_ERR "par_io not initialized\n");
+		return -1;
+	}
+
+	ph = of_get_property(np, "pio-handle", NULL);
+	if (ph == NULL) {
+		printk(KERN_ERR "pio-handle not available\n");
+		return -1;
+	}
+
+	pio = of_find_node_by_phandle(*ph);
+
+	pio_map = of_get_property(pio, "pio-map", &pio_map_len);
+	if (pio_map == NULL) {
+		printk(KERN_ERR "pio-map is not set!\n");
+		return -1;
+	}
+	pio_map_len /= sizeof(unsigned int);
+	if ((pio_map_len % 6) != 0) {
+		printk(KERN_ERR "pio-map format wrong!\n");
+		return -1;
+	}
+
+	while (pio_map_len > 0) {
+		par_io_config_pin((u8) pio_map[0], (u8) pio_map[1],
+				(int) pio_map[2], (int) pio_map[3],
+				(int) pio_map[4], (int) pio_map[5]);
+		pio_map += 6;
+		pio_map_len -= 6;
+	}
+	of_node_put(pio);
+	return 0;
+}
+EXPORT_SYMBOL(par_io_of_config);
diff --git a/drivers/soc/fsl/qe/ucc.c b/drivers/soc/fsl/qe/ucc.c
new file mode 100644
index 000000000000..b59d3358f9bd
--- /dev/null
+++ b/drivers/soc/fsl/qe/ucc.c
@@ -0,0 +1,212 @@
+/*
+ * arch/powerpc/sysdev/qe_lib/ucc.c
+ *
+ * QE UCC API Set - UCC specific routines implementations.
+ *
+ * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors: 	Shlomi Gridish <gridish@freescale.com>
+ * 		Li Yang <leoli@freescale.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/stddef.h>
+#include <linux/spinlock.h>
+#include <linux/export.h>
+
+#include <asm/irq.h>
+#include <asm/io.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+#include <soc/fsl/qe/ucc.h>
+
+int ucc_set_qe_mux_mii_mng(unsigned int ucc_num)
+{
+	unsigned long flags;
+
+	if (ucc_num > UCC_MAX_NUM - 1)
+		return -EINVAL;
+
+	spin_lock_irqsave(&cmxgcr_lock, flags);
+	clrsetbits_be32(&qe_immr->qmx.cmxgcr, QE_CMXGCR_MII_ENET_MNG,
+		ucc_num << QE_CMXGCR_MII_ENET_MNG_SHIFT);
+	spin_unlock_irqrestore(&cmxgcr_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(ucc_set_qe_mux_mii_mng);
+
+/* Configure the UCC to either Slow or Fast.
+ *
+ * A given UCC can be figured to support either "slow" devices (e.g. UART)
+ * or "fast" devices (e.g. Ethernet).
+ *
+ * 'ucc_num' is the UCC number, from 0 - 7.
+ *
+ * This function also sets the UCC_GUEMR_SET_RESERVED3 bit because that bit
+ * must always be set to 1.
+ */
+int ucc_set_type(unsigned int ucc_num, enum ucc_speed_type speed)
+{
+	u8 __iomem *guemr;
+
+	/* The GUEMR register is at the same location for both slow and fast
+	   devices, so we just use uccX.slow.guemr. */
+	switch (ucc_num) {
+	case 0: guemr = &qe_immr->ucc1.slow.guemr;
+		break;
+	case 1: guemr = &qe_immr->ucc2.slow.guemr;
+		break;
+	case 2: guemr = &qe_immr->ucc3.slow.guemr;
+		break;
+	case 3: guemr = &qe_immr->ucc4.slow.guemr;
+		break;
+	case 4: guemr = &qe_immr->ucc5.slow.guemr;
+		break;
+	case 5: guemr = &qe_immr->ucc6.slow.guemr;
+		break;
+	case 6: guemr = &qe_immr->ucc7.slow.guemr;
+		break;
+	case 7: guemr = &qe_immr->ucc8.slow.guemr;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	clrsetbits_8(guemr, UCC_GUEMR_MODE_MASK,
+		UCC_GUEMR_SET_RESERVED3 | speed);
+
+	return 0;
+}
+
+static void get_cmxucr_reg(unsigned int ucc_num, __be32 __iomem **cmxucr,
+	unsigned int *reg_num, unsigned int *shift)
+{
+	unsigned int cmx = ((ucc_num & 1) << 1) + (ucc_num > 3);
+
+	*reg_num = cmx + 1;
+	*cmxucr = &qe_immr->qmx.cmxucr[cmx];
+	*shift = 16 - 8 * (ucc_num & 2);
+}
+
+int ucc_mux_set_grant_tsa_bkpt(unsigned int ucc_num, int set, u32 mask)
+{
+	__be32 __iomem *cmxucr;
+	unsigned int reg_num;
+	unsigned int shift;
+
+	/* check if the UCC number is in range. */
+	if (ucc_num > UCC_MAX_NUM - 1)
+		return -EINVAL;
+
+	get_cmxucr_reg(ucc_num, &cmxucr, &reg_num, &shift);
+
+	if (set)
+		setbits32(cmxucr, mask << shift);
+	else
+		clrbits32(cmxucr, mask << shift);
+
+	return 0;
+}
+
+int ucc_set_qe_mux_rxtx(unsigned int ucc_num, enum qe_clock clock,
+	enum comm_dir mode)
+{
+	__be32 __iomem *cmxucr;
+	unsigned int reg_num;
+	unsigned int shift;
+	u32 clock_bits = 0;
+
+	/* check if the UCC number is in range. */
+	if (ucc_num > UCC_MAX_NUM - 1)
+		return -EINVAL;
+
+	/* The communications direction must be RX or TX */
+	if (!((mode == COMM_DIR_RX) || (mode == COMM_DIR_TX)))
+		return -EINVAL;
+
+	get_cmxucr_reg(ucc_num, &cmxucr, &reg_num, &shift);
+
+	switch (reg_num) {
+	case 1:
+		switch (clock) {
+		case QE_BRG1:	clock_bits = 1; break;
+		case QE_BRG2:	clock_bits = 2; break;
+		case QE_BRG7:	clock_bits = 3; break;
+		case QE_BRG8:	clock_bits = 4; break;
+		case QE_CLK9:	clock_bits = 5; break;
+		case QE_CLK10:	clock_bits = 6; break;
+		case QE_CLK11:	clock_bits = 7; break;
+		case QE_CLK12:	clock_bits = 8; break;
+		case QE_CLK15:	clock_bits = 9; break;
+		case QE_CLK16:	clock_bits = 10; break;
+		default: break;
+		}
+		break;
+	case 2:
+		switch (clock) {
+		case QE_BRG5:	clock_bits = 1; break;
+		case QE_BRG6:	clock_bits = 2; break;
+		case QE_BRG7:	clock_bits = 3; break;
+		case QE_BRG8:	clock_bits = 4; break;
+		case QE_CLK13:	clock_bits = 5; break;
+		case QE_CLK14:	clock_bits = 6; break;
+		case QE_CLK19:	clock_bits = 7; break;
+		case QE_CLK20:	clock_bits = 8; break;
+		case QE_CLK15:	clock_bits = 9; break;
+		case QE_CLK16:	clock_bits = 10; break;
+		default: break;
+		}
+		break;
+	case 3:
+		switch (clock) {
+		case QE_BRG9:	clock_bits = 1; break;
+		case QE_BRG10:	clock_bits = 2; break;
+		case QE_BRG15:	clock_bits = 3; break;
+		case QE_BRG16:	clock_bits = 4; break;
+		case QE_CLK3:	clock_bits = 5; break;
+		case QE_CLK4:	clock_bits = 6; break;
+		case QE_CLK17:	clock_bits = 7; break;
+		case QE_CLK18:	clock_bits = 8; break;
+		case QE_CLK7:	clock_bits = 9; break;
+		case QE_CLK8:	clock_bits = 10; break;
+		case QE_CLK16:	clock_bits = 11; break;
+		default: break;
+		}
+		break;
+	case 4:
+		switch (clock) {
+		case QE_BRG13:	clock_bits = 1; break;
+		case QE_BRG14:	clock_bits = 2; break;
+		case QE_BRG15:	clock_bits = 3; break;
+		case QE_BRG16:	clock_bits = 4; break;
+		case QE_CLK5:	clock_bits = 5; break;
+		case QE_CLK6:	clock_bits = 6; break;
+		case QE_CLK21:	clock_bits = 7; break;
+		case QE_CLK22:	clock_bits = 8; break;
+		case QE_CLK7:	clock_bits = 9; break;
+		case QE_CLK8:	clock_bits = 10; break;
+		case QE_CLK16:	clock_bits = 11; break;
+		default: break;
+		}
+		break;
+	default: break;
+	}
+
+	/* Check for invalid combination of clock and UCC number */
+	if (!clock_bits)
+		return -ENOENT;
+
+	if (mode == COMM_DIR_RX)
+		shift += 4;
+
+	clrsetbits_be32(cmxucr, QE_CMXUCR_TX_CLK_SRC_MASK << shift,
+		clock_bits << shift);
+
+	return 0;
+}
diff --git a/drivers/soc/fsl/qe/ucc_fast.c b/drivers/soc/fsl/qe/ucc_fast.c
new file mode 100644
index 000000000000..a7689310fe40
--- /dev/null
+++ b/drivers/soc/fsl/qe/ucc_fast.c
@@ -0,0 +1,363 @@
+/*
+ * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors: 	Shlomi Gridish <gridish@freescale.com>
+ * 		Li Yang <leoli@freescale.com>
+ *
+ * Description:
+ * QE UCC Fast API Set - UCC Fast specific routines implementations.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/stddef.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/export.h>
+
+#include <asm/io.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_fast.h>
+
+void ucc_fast_dump_regs(struct ucc_fast_private * uccf)
+{
+	printk(KERN_INFO "UCC%u Fast registers:\n", uccf->uf_info->ucc_num);
+	printk(KERN_INFO "Base address: 0x%p\n", uccf->uf_regs);
+
+	printk(KERN_INFO "gumr  : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->gumr, in_be32(&uccf->uf_regs->gumr));
+	printk(KERN_INFO "upsmr : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->upsmr, in_be32(&uccf->uf_regs->upsmr));
+	printk(KERN_INFO "utodr : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->utodr, in_be16(&uccf->uf_regs->utodr));
+	printk(KERN_INFO "udsr  : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->udsr, in_be16(&uccf->uf_regs->udsr));
+	printk(KERN_INFO "ucce  : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->ucce, in_be32(&uccf->uf_regs->ucce));
+	printk(KERN_INFO "uccm  : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->uccm, in_be32(&uccf->uf_regs->uccm));
+	printk(KERN_INFO "uccs  : addr=0x%p, val=0x%02x\n",
+		  &uccf->uf_regs->uccs, in_8(&uccf->uf_regs->uccs));
+	printk(KERN_INFO "urfb  : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->urfb, in_be32(&uccf->uf_regs->urfb));
+	printk(KERN_INFO "urfs  : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->urfs, in_be16(&uccf->uf_regs->urfs));
+	printk(KERN_INFO "urfet : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->urfet, in_be16(&uccf->uf_regs->urfet));
+	printk(KERN_INFO "urfset: addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->urfset, in_be16(&uccf->uf_regs->urfset));
+	printk(KERN_INFO "utfb  : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->utfb, in_be32(&uccf->uf_regs->utfb));
+	printk(KERN_INFO "utfs  : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->utfs, in_be16(&uccf->uf_regs->utfs));
+	printk(KERN_INFO "utfet : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->utfet, in_be16(&uccf->uf_regs->utfet));
+	printk(KERN_INFO "utftt : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->utftt, in_be16(&uccf->uf_regs->utftt));
+	printk(KERN_INFO "utpt  : addr=0x%p, val=0x%04x\n",
+		  &uccf->uf_regs->utpt, in_be16(&uccf->uf_regs->utpt));
+	printk(KERN_INFO "urtry : addr=0x%p, val=0x%08x\n",
+		  &uccf->uf_regs->urtry, in_be32(&uccf->uf_regs->urtry));
+	printk(KERN_INFO "guemr : addr=0x%p, val=0x%02x\n",
+		  &uccf->uf_regs->guemr, in_8(&uccf->uf_regs->guemr));
+}
+EXPORT_SYMBOL(ucc_fast_dump_regs);
+
+u32 ucc_fast_get_qe_cr_subblock(int uccf_num)
+{
+	switch (uccf_num) {
+	case 0: return QE_CR_SUBBLOCK_UCCFAST1;
+	case 1: return QE_CR_SUBBLOCK_UCCFAST2;
+	case 2: return QE_CR_SUBBLOCK_UCCFAST3;
+	case 3: return QE_CR_SUBBLOCK_UCCFAST4;
+	case 4: return QE_CR_SUBBLOCK_UCCFAST5;
+	case 5: return QE_CR_SUBBLOCK_UCCFAST6;
+	case 6: return QE_CR_SUBBLOCK_UCCFAST7;
+	case 7: return QE_CR_SUBBLOCK_UCCFAST8;
+	default: return QE_CR_SUBBLOCK_INVALID;
+	}
+}
+EXPORT_SYMBOL(ucc_fast_get_qe_cr_subblock);
+
+void ucc_fast_transmit_on_demand(struct ucc_fast_private * uccf)
+{
+	out_be16(&uccf->uf_regs->utodr, UCC_FAST_TOD);
+}
+EXPORT_SYMBOL(ucc_fast_transmit_on_demand);
+
+void ucc_fast_enable(struct ucc_fast_private * uccf, enum comm_dir mode)
+{
+	struct ucc_fast __iomem *uf_regs;
+	u32 gumr;
+
+	uf_regs = uccf->uf_regs;
+
+	/* Enable reception and/or transmission on this UCC. */
+	gumr = in_be32(&uf_regs->gumr);
+	if (mode & COMM_DIR_TX) {
+		gumr |= UCC_FAST_GUMR_ENT;
+		uccf->enabled_tx = 1;
+	}
+	if (mode & COMM_DIR_RX) {
+		gumr |= UCC_FAST_GUMR_ENR;
+		uccf->enabled_rx = 1;
+	}
+	out_be32(&uf_regs->gumr, gumr);
+}
+EXPORT_SYMBOL(ucc_fast_enable);
+
+void ucc_fast_disable(struct ucc_fast_private * uccf, enum comm_dir mode)
+{
+	struct ucc_fast __iomem *uf_regs;
+	u32 gumr;
+
+	uf_regs = uccf->uf_regs;
+
+	/* Disable reception and/or transmission on this UCC. */
+	gumr = in_be32(&uf_regs->gumr);
+	if (mode & COMM_DIR_TX) {
+		gumr &= ~UCC_FAST_GUMR_ENT;
+		uccf->enabled_tx = 0;
+	}
+	if (mode & COMM_DIR_RX) {
+		gumr &= ~UCC_FAST_GUMR_ENR;
+		uccf->enabled_rx = 0;
+	}
+	out_be32(&uf_regs->gumr, gumr);
+}
+EXPORT_SYMBOL(ucc_fast_disable);
+
+int ucc_fast_init(struct ucc_fast_info * uf_info, struct ucc_fast_private ** uccf_ret)
+{
+	struct ucc_fast_private *uccf;
+	struct ucc_fast __iomem *uf_regs;
+	u32 gumr;
+	int ret;
+
+	if (!uf_info)
+		return -EINVAL;
+
+	/* check if the UCC port number is in range. */
+	if ((uf_info->ucc_num < 0) || (uf_info->ucc_num > UCC_MAX_NUM - 1)) {
+		printk(KERN_ERR "%s: illegal UCC number\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Check that 'max_rx_buf_length' is properly aligned (4). */
+	if (uf_info->max_rx_buf_length & (UCC_FAST_MRBLR_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: max_rx_buf_length not aligned\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	/* Validate Virtual Fifo register values */
+	if (uf_info->urfs < UCC_FAST_URFS_MIN_VAL) {
+		printk(KERN_ERR "%s: urfs is too small\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->urfs & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: urfs is not aligned\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->urfet & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: urfet is not aligned.\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->urfset & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: urfset is not aligned\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->utfs & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: utfs is not aligned\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->utfet & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: utfet is not aligned\n", __func__);
+		return -EINVAL;
+	}
+
+	if (uf_info->utftt & (UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT - 1)) {
+		printk(KERN_ERR "%s: utftt is not aligned\n", __func__);
+		return -EINVAL;
+	}
+
+	uccf = kzalloc(sizeof(struct ucc_fast_private), GFP_KERNEL);
+	if (!uccf) {
+		printk(KERN_ERR "%s: Cannot allocate private data\n",
+			__func__);
+		return -ENOMEM;
+	}
+
+	/* Fill fast UCC structure */
+	uccf->uf_info = uf_info;
+	/* Set the PHY base address */
+	uccf->uf_regs = ioremap(uf_info->regs, sizeof(struct ucc_fast));
+	if (uccf->uf_regs == NULL) {
+		printk(KERN_ERR "%s: Cannot map UCC registers\n", __func__);
+		kfree(uccf);
+		return -ENOMEM;
+	}
+
+	uccf->enabled_tx = 0;
+	uccf->enabled_rx = 0;
+	uccf->stopped_tx = 0;
+	uccf->stopped_rx = 0;
+	uf_regs = uccf->uf_regs;
+	uccf->p_ucce = &uf_regs->ucce;
+	uccf->p_uccm = &uf_regs->uccm;
+#ifdef CONFIG_UGETH_TX_ON_DEMAND
+	uccf->p_utodr = &uf_regs->utodr;
+#endif
+#ifdef STATISTICS
+	uccf->tx_frames = 0;
+	uccf->rx_frames = 0;
+	uccf->rx_discarded = 0;
+#endif				/* STATISTICS */
+
+	/* Set UCC to fast type */
+	ret = ucc_set_type(uf_info->ucc_num, UCC_SPEED_TYPE_FAST);
+	if (ret) {
+		printk(KERN_ERR "%s: cannot set UCC type\n", __func__);
+		ucc_fast_free(uccf);
+		return ret;
+	}
+
+	uccf->mrblr = uf_info->max_rx_buf_length;
+
+	/* Set GUMR */
+	/* For more details see the hardware spec. */
+	gumr = uf_info->ttx_trx;
+	if (uf_info->tci)
+		gumr |= UCC_FAST_GUMR_TCI;
+	if (uf_info->cdp)
+		gumr |= UCC_FAST_GUMR_CDP;
+	if (uf_info->ctsp)
+		gumr |= UCC_FAST_GUMR_CTSP;
+	if (uf_info->cds)
+		gumr |= UCC_FAST_GUMR_CDS;
+	if (uf_info->ctss)
+		gumr |= UCC_FAST_GUMR_CTSS;
+	if (uf_info->txsy)
+		gumr |= UCC_FAST_GUMR_TXSY;
+	if (uf_info->rsyn)
+		gumr |= UCC_FAST_GUMR_RSYN;
+	gumr |= uf_info->synl;
+	if (uf_info->rtsm)
+		gumr |= UCC_FAST_GUMR_RTSM;
+	gumr |= uf_info->renc;
+	if (uf_info->revd)
+		gumr |= UCC_FAST_GUMR_REVD;
+	gumr |= uf_info->tenc;
+	gumr |= uf_info->tcrc;
+	gumr |= uf_info->mode;
+	out_be32(&uf_regs->gumr, gumr);
+
+	/* Allocate memory for Tx Virtual Fifo */
+	uccf->ucc_fast_tx_virtual_fifo_base_offset =
+	    qe_muram_alloc(uf_info->utfs, UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT);
+	if (IS_ERR_VALUE(uccf->ucc_fast_tx_virtual_fifo_base_offset)) {
+		printk(KERN_ERR "%s: cannot allocate MURAM for TX FIFO\n",
+			__func__);
+		uccf->ucc_fast_tx_virtual_fifo_base_offset = 0;
+		ucc_fast_free(uccf);
+		return -ENOMEM;
+	}
+
+	/* Allocate memory for Rx Virtual Fifo */
+	uccf->ucc_fast_rx_virtual_fifo_base_offset =
+		qe_muram_alloc(uf_info->urfs +
+			   UCC_FAST_RECEIVE_VIRTUAL_FIFO_SIZE_FUDGE_FACTOR,
+			   UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT);
+	if (IS_ERR_VALUE(uccf->ucc_fast_rx_virtual_fifo_base_offset)) {
+		printk(KERN_ERR "%s: cannot allocate MURAM for RX FIFO\n",
+			__func__);
+		uccf->ucc_fast_rx_virtual_fifo_base_offset = 0;
+		ucc_fast_free(uccf);
+		return -ENOMEM;
+	}
+
+	/* Set Virtual Fifo registers */
+	out_be16(&uf_regs->urfs, uf_info->urfs);
+	out_be16(&uf_regs->urfet, uf_info->urfet);
+	out_be16(&uf_regs->urfset, uf_info->urfset);
+	out_be16(&uf_regs->utfs, uf_info->utfs);
+	out_be16(&uf_regs->utfet, uf_info->utfet);
+	out_be16(&uf_regs->utftt, uf_info->utftt);
+	/* utfb, urfb are offsets from MURAM base */
+	out_be32(&uf_regs->utfb, uccf->ucc_fast_tx_virtual_fifo_base_offset);
+	out_be32(&uf_regs->urfb, uccf->ucc_fast_rx_virtual_fifo_base_offset);
+
+	/* Mux clocking */
+	/* Grant Support */
+	ucc_set_qe_mux_grant(uf_info->ucc_num, uf_info->grant_support);
+	/* Breakpoint Support */
+	ucc_set_qe_mux_bkpt(uf_info->ucc_num, uf_info->brkpt_support);
+	/* Set Tsa or NMSI mode. */
+	ucc_set_qe_mux_tsa(uf_info->ucc_num, uf_info->tsa);
+	/* If NMSI (not Tsa), set Tx and Rx clock. */
+	if (!uf_info->tsa) {
+		/* Rx clock routing */
+		if ((uf_info->rx_clock != QE_CLK_NONE) &&
+		    ucc_set_qe_mux_rxtx(uf_info->ucc_num, uf_info->rx_clock,
+					COMM_DIR_RX)) {
+			printk(KERN_ERR "%s: illegal value for RX clock\n",
+			       __func__);
+			ucc_fast_free(uccf);
+			return -EINVAL;
+		}
+		/* Tx clock routing */
+		if ((uf_info->tx_clock != QE_CLK_NONE) &&
+		    ucc_set_qe_mux_rxtx(uf_info->ucc_num, uf_info->tx_clock,
+					COMM_DIR_TX)) {
+			printk(KERN_ERR "%s: illegal value for TX clock\n",
+			       __func__);
+			ucc_fast_free(uccf);
+			return -EINVAL;
+		}
+	}
+
+	/* Set interrupt mask register at UCC level. */
+	out_be32(&uf_regs->uccm, uf_info->uccm_mask);
+
+	/* First, clear anything pending at UCC level,
+	 * otherwise, old garbage may come through
+	 * as soon as the dam is opened. */
+
+	/* Writing '1' clears */
+	out_be32(&uf_regs->ucce, 0xffffffff);
+
+	*uccf_ret = uccf;
+	return 0;
+}
+EXPORT_SYMBOL(ucc_fast_init);
+
+void ucc_fast_free(struct ucc_fast_private * uccf)
+{
+	if (!uccf)
+		return;
+
+	if (uccf->ucc_fast_tx_virtual_fifo_base_offset)
+		qe_muram_free(uccf->ucc_fast_tx_virtual_fifo_base_offset);
+
+	if (uccf->ucc_fast_rx_virtual_fifo_base_offset)
+		qe_muram_free(uccf->ucc_fast_rx_virtual_fifo_base_offset);
+
+	if (uccf->uf_regs)
+		iounmap(uccf->uf_regs);
+
+	kfree(uccf);
+}
+EXPORT_SYMBOL(ucc_fast_free);
diff --git a/drivers/soc/fsl/qe/ucc_slow.c b/drivers/soc/fsl/qe/ucc_slow.c
new file mode 100644
index 000000000000..9334bdbd9b30
--- /dev/null
+++ b/drivers/soc/fsl/qe/ucc_slow.c
@@ -0,0 +1,374 @@
+/*
+ * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors: 	Shlomi Gridish <gridish@freescale.com>
+ * 		Li Yang <leoli@freescale.com>
+ *
+ * Description:
+ * QE UCC Slow API Set - UCC Slow specific routines implementations.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/stddef.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/export.h>
+
+#include <asm/io.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_slow.h>
+
+u32 ucc_slow_get_qe_cr_subblock(int uccs_num)
+{
+	switch (uccs_num) {
+	case 0: return QE_CR_SUBBLOCK_UCCSLOW1;
+	case 1: return QE_CR_SUBBLOCK_UCCSLOW2;
+	case 2: return QE_CR_SUBBLOCK_UCCSLOW3;
+	case 3: return QE_CR_SUBBLOCK_UCCSLOW4;
+	case 4: return QE_CR_SUBBLOCK_UCCSLOW5;
+	case 5: return QE_CR_SUBBLOCK_UCCSLOW6;
+	case 6: return QE_CR_SUBBLOCK_UCCSLOW7;
+	case 7: return QE_CR_SUBBLOCK_UCCSLOW8;
+	default: return QE_CR_SUBBLOCK_INVALID;
+	}
+}
+EXPORT_SYMBOL(ucc_slow_get_qe_cr_subblock);
+
+void ucc_slow_graceful_stop_tx(struct ucc_slow_private * uccs)
+{
+	struct ucc_slow_info *us_info = uccs->us_info;
+	u32 id;
+
+	id = ucc_slow_get_qe_cr_subblock(us_info->ucc_num);
+	qe_issue_cmd(QE_GRACEFUL_STOP_TX, id,
+			 QE_CR_PROTOCOL_UNSPECIFIED, 0);
+}
+EXPORT_SYMBOL(ucc_slow_graceful_stop_tx);
+
+void ucc_slow_stop_tx(struct ucc_slow_private * uccs)
+{
+	struct ucc_slow_info *us_info = uccs->us_info;
+	u32 id;
+
+	id = ucc_slow_get_qe_cr_subblock(us_info->ucc_num);
+	qe_issue_cmd(QE_STOP_TX, id, QE_CR_PROTOCOL_UNSPECIFIED, 0);
+}
+EXPORT_SYMBOL(ucc_slow_stop_tx);
+
+void ucc_slow_restart_tx(struct ucc_slow_private * uccs)
+{
+	struct ucc_slow_info *us_info = uccs->us_info;
+	u32 id;
+
+	id = ucc_slow_get_qe_cr_subblock(us_info->ucc_num);
+	qe_issue_cmd(QE_RESTART_TX, id, QE_CR_PROTOCOL_UNSPECIFIED, 0);
+}
+EXPORT_SYMBOL(ucc_slow_restart_tx);
+
+void ucc_slow_enable(struct ucc_slow_private * uccs, enum comm_dir mode)
+{
+	struct ucc_slow *us_regs;
+	u32 gumr_l;
+
+	us_regs = uccs->us_regs;
+
+	/* Enable reception and/or transmission on this UCC. */
+	gumr_l = in_be32(&us_regs->gumr_l);
+	if (mode & COMM_DIR_TX) {
+		gumr_l |= UCC_SLOW_GUMR_L_ENT;
+		uccs->enabled_tx = 1;
+	}
+	if (mode & COMM_DIR_RX) {
+		gumr_l |= UCC_SLOW_GUMR_L_ENR;
+		uccs->enabled_rx = 1;
+	}
+	out_be32(&us_regs->gumr_l, gumr_l);
+}
+EXPORT_SYMBOL(ucc_slow_enable);
+
+void ucc_slow_disable(struct ucc_slow_private * uccs, enum comm_dir mode)
+{
+	struct ucc_slow *us_regs;
+	u32 gumr_l;
+
+	us_regs = uccs->us_regs;
+
+	/* Disable reception and/or transmission on this UCC. */
+	gumr_l = in_be32(&us_regs->gumr_l);
+	if (mode & COMM_DIR_TX) {
+		gumr_l &= ~UCC_SLOW_GUMR_L_ENT;
+		uccs->enabled_tx = 0;
+	}
+	if (mode & COMM_DIR_RX) {
+		gumr_l &= ~UCC_SLOW_GUMR_L_ENR;
+		uccs->enabled_rx = 0;
+	}
+	out_be32(&us_regs->gumr_l, gumr_l);
+}
+EXPORT_SYMBOL(ucc_slow_disable);
+
+/* Initialize the UCC for Slow operations
+ *
+ * The caller should initialize the following us_info
+ */
+int ucc_slow_init(struct ucc_slow_info * us_info, struct ucc_slow_private ** uccs_ret)
+{
+	struct ucc_slow_private *uccs;
+	u32 i;
+	struct ucc_slow __iomem *us_regs;
+	u32 gumr;
+	struct qe_bd *bd;
+	u32 id;
+	u32 command;
+	int ret = 0;
+
+	if (!us_info)
+		return -EINVAL;
+
+	/* check if the UCC port number is in range. */
+	if ((us_info->ucc_num < 0) || (us_info->ucc_num > UCC_MAX_NUM - 1)) {
+		printk(KERN_ERR "%s: illegal UCC number\n", __func__);
+		return -EINVAL;
+	}
+
+	/*
+	 * Set mrblr
+	 * Check that 'max_rx_buf_length' is properly aligned (4), unless
+	 * rfw is 1, meaning that QE accepts one byte at a time, unlike normal
+	 * case when QE accepts 32 bits at a time.
+	 */
+	if ((!us_info->rfw) &&
+		(us_info->max_rx_buf_length & (UCC_SLOW_MRBLR_ALIGNMENT - 1))) {
+		printk(KERN_ERR "max_rx_buf_length not aligned.\n");
+		return -EINVAL;
+	}
+
+	uccs = kzalloc(sizeof(struct ucc_slow_private), GFP_KERNEL);
+	if (!uccs) {
+		printk(KERN_ERR "%s: Cannot allocate private data\n",
+			__func__);
+		return -ENOMEM;
+	}
+
+	/* Fill slow UCC structure */
+	uccs->us_info = us_info;
+	/* Set the PHY base address */
+	uccs->us_regs = ioremap(us_info->regs, sizeof(struct ucc_slow));
+	if (uccs->us_regs == NULL) {
+		printk(KERN_ERR "%s: Cannot map UCC registers\n", __func__);
+		kfree(uccs);
+		return -ENOMEM;
+	}
+
+	uccs->saved_uccm = 0;
+	uccs->p_rx_frame = 0;
+	us_regs = uccs->us_regs;
+	uccs->p_ucce = (u16 *) & (us_regs->ucce);
+	uccs->p_uccm = (u16 *) & (us_regs->uccm);
+#ifdef STATISTICS
+	uccs->rx_frames = 0;
+	uccs->tx_frames = 0;
+	uccs->rx_discarded = 0;
+#endif				/* STATISTICS */
+
+	/* Get PRAM base */
+	uccs->us_pram_offset =
+		qe_muram_alloc(UCC_SLOW_PRAM_SIZE, ALIGNMENT_OF_UCC_SLOW_PRAM);
+	if (IS_ERR_VALUE(uccs->us_pram_offset)) {
+		printk(KERN_ERR "%s: cannot allocate MURAM for PRAM", __func__);
+		ucc_slow_free(uccs);
+		return -ENOMEM;
+	}
+	id = ucc_slow_get_qe_cr_subblock(us_info->ucc_num);
+	qe_issue_cmd(QE_ASSIGN_PAGE_TO_DEVICE, id, us_info->protocol,
+		     uccs->us_pram_offset);
+
+	uccs->us_pram = qe_muram_addr(uccs->us_pram_offset);
+
+	/* Set UCC to slow type */
+	ret = ucc_set_type(us_info->ucc_num, UCC_SPEED_TYPE_SLOW);
+	if (ret) {
+		printk(KERN_ERR "%s: cannot set UCC type", __func__);
+		ucc_slow_free(uccs);
+		return ret;
+	}
+
+	out_be16(&uccs->us_pram->mrblr, us_info->max_rx_buf_length);
+
+	INIT_LIST_HEAD(&uccs->confQ);
+
+	/* Allocate BDs. */
+	uccs->rx_base_offset =
+		qe_muram_alloc(us_info->rx_bd_ring_len * sizeof(struct qe_bd),
+				QE_ALIGNMENT_OF_BD);
+	if (IS_ERR_VALUE(uccs->rx_base_offset)) {
+		printk(KERN_ERR "%s: cannot allocate %u RX BDs\n", __func__,
+			us_info->rx_bd_ring_len);
+		uccs->rx_base_offset = 0;
+		ucc_slow_free(uccs);
+		return -ENOMEM;
+	}
+
+	uccs->tx_base_offset =
+		qe_muram_alloc(us_info->tx_bd_ring_len * sizeof(struct qe_bd),
+			QE_ALIGNMENT_OF_BD);
+	if (IS_ERR_VALUE(uccs->tx_base_offset)) {
+		printk(KERN_ERR "%s: cannot allocate TX BDs", __func__);
+		uccs->tx_base_offset = 0;
+		ucc_slow_free(uccs);
+		return -ENOMEM;
+	}
+
+	/* Init Tx bds */
+	bd = uccs->confBd = uccs->tx_bd = qe_muram_addr(uccs->tx_base_offset);
+	for (i = 0; i < us_info->tx_bd_ring_len - 1; i++) {
+		/* clear bd buffer */
+		out_be32(&bd->buf, 0);
+		/* set bd status and length */
+		out_be32((u32 *) bd, 0);
+		bd++;
+	}
+	/* for last BD set Wrap bit */
+	out_be32(&bd->buf, 0);
+	out_be32((u32 *) bd, cpu_to_be32(T_W));
+
+	/* Init Rx bds */
+	bd = uccs->rx_bd = qe_muram_addr(uccs->rx_base_offset);
+	for (i = 0; i < us_info->rx_bd_ring_len - 1; i++) {
+		/* set bd status and length */
+		out_be32((u32*)bd, 0);
+		/* clear bd buffer */
+		out_be32(&bd->buf, 0);
+		bd++;
+	}
+	/* for last BD set Wrap bit */
+	out_be32((u32*)bd, cpu_to_be32(R_W));
+	out_be32(&bd->buf, 0);
+
+	/* Set GUMR (For more details see the hardware spec.). */
+	/* gumr_h */
+	gumr = us_info->tcrc;
+	if (us_info->cdp)
+		gumr |= UCC_SLOW_GUMR_H_CDP;
+	if (us_info->ctsp)
+		gumr |= UCC_SLOW_GUMR_H_CTSP;
+	if (us_info->cds)
+		gumr |= UCC_SLOW_GUMR_H_CDS;
+	if (us_info->ctss)
+		gumr |= UCC_SLOW_GUMR_H_CTSS;
+	if (us_info->tfl)
+		gumr |= UCC_SLOW_GUMR_H_TFL;
+	if (us_info->rfw)
+		gumr |= UCC_SLOW_GUMR_H_RFW;
+	if (us_info->txsy)
+		gumr |= UCC_SLOW_GUMR_H_TXSY;
+	if (us_info->rtsm)
+		gumr |= UCC_SLOW_GUMR_H_RTSM;
+	out_be32(&us_regs->gumr_h, gumr);
+
+	/* gumr_l */
+	gumr = us_info->tdcr | us_info->rdcr | us_info->tenc | us_info->renc |
+		us_info->diag | us_info->mode;
+	if (us_info->tci)
+		gumr |= UCC_SLOW_GUMR_L_TCI;
+	if (us_info->rinv)
+		gumr |= UCC_SLOW_GUMR_L_RINV;
+	if (us_info->tinv)
+		gumr |= UCC_SLOW_GUMR_L_TINV;
+	if (us_info->tend)
+		gumr |= UCC_SLOW_GUMR_L_TEND;
+	out_be32(&us_regs->gumr_l, gumr);
+
+	/* Function code registers */
+
+	/* if the data is in cachable memory, the 'global' */
+	/* in the function code should be set. */
+	uccs->us_pram->tbmr = UCC_BMR_BO_BE;
+	uccs->us_pram->rbmr = UCC_BMR_BO_BE;
+
+	/* rbase, tbase are offsets from MURAM base */
+	out_be16(&uccs->us_pram->rbase, uccs->rx_base_offset);
+	out_be16(&uccs->us_pram->tbase, uccs->tx_base_offset);
+
+	/* Mux clocking */
+	/* Grant Support */
+	ucc_set_qe_mux_grant(us_info->ucc_num, us_info->grant_support);
+	/* Breakpoint Support */
+	ucc_set_qe_mux_bkpt(us_info->ucc_num, us_info->brkpt_support);
+	/* Set Tsa or NMSI mode. */
+	ucc_set_qe_mux_tsa(us_info->ucc_num, us_info->tsa);
+	/* If NMSI (not Tsa), set Tx and Rx clock. */
+	if (!us_info->tsa) {
+		/* Rx clock routing */
+		if (ucc_set_qe_mux_rxtx(us_info->ucc_num, us_info->rx_clock,
+					COMM_DIR_RX)) {
+			printk(KERN_ERR "%s: illegal value for RX clock\n",
+			       __func__);
+			ucc_slow_free(uccs);
+			return -EINVAL;
+		}
+		/* Tx clock routing */
+		if (ucc_set_qe_mux_rxtx(us_info->ucc_num, us_info->tx_clock,
+					COMM_DIR_TX)) {
+			printk(KERN_ERR "%s: illegal value for TX clock\n",
+			       __func__);
+			ucc_slow_free(uccs);
+			return -EINVAL;
+		}
+	}
+
+	/* Set interrupt mask register at UCC level. */
+	out_be16(&us_regs->uccm, us_info->uccm_mask);
+
+	/* First, clear anything pending at UCC level,
+	 * otherwise, old garbage may come through
+	 * as soon as the dam is opened. */
+
+	/* Writing '1' clears */
+	out_be16(&us_regs->ucce, 0xffff);
+
+	/* Issue QE Init command */
+	if (us_info->init_tx && us_info->init_rx)
+		command = QE_INIT_TX_RX;
+	else if (us_info->init_tx)
+		command = QE_INIT_TX;
+	else
+		command = QE_INIT_RX;	/* We know at least one is TRUE */
+
+	qe_issue_cmd(command, id, us_info->protocol, 0);
+
+	*uccs_ret = uccs;
+	return 0;
+}
+EXPORT_SYMBOL(ucc_slow_init);
+
+void ucc_slow_free(struct ucc_slow_private * uccs)
+{
+	if (!uccs)
+		return;
+
+	if (uccs->rx_base_offset)
+		qe_muram_free(uccs->rx_base_offset);
+
+	if (uccs->tx_base_offset)
+		qe_muram_free(uccs->tx_base_offset);
+
+	if (uccs->us_pram)
+		qe_muram_free(uccs->us_pram_offset);
+
+	if (uccs->us_regs)
+		iounmap(uccs->us_regs);
+
+	kfree(uccs);
+}
+EXPORT_SYMBOL(ucc_slow_free);
+
diff --git a/drivers/soc/fsl/qe/usb.c b/drivers/soc/fsl/qe/usb.c
new file mode 100644
index 000000000000..111f7ab80f04
--- /dev/null
+++ b/drivers/soc/fsl/qe/usb.c
@@ -0,0 +1,56 @@
+/*
+ * QE USB routines
+ *
+ * Copyright 2006 Freescale Semiconductor, Inc.
+ *               Shlomi Gridish <gridish@freescale.com>
+ *               Jerry Huang <Chang-Ming.Huang@freescale.com>
+ * Copyright (c) MontaVista Software, Inc. 2008.
+ *               Anton Vorontsov <avorontsov@ru.mvista.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/io.h>
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+
+int qe_usb_clock_set(enum qe_clock clk, int rate)
+{
+	struct qe_mux __iomem *mux = &qe_immr->qmx;
+	unsigned long flags;
+	u32 val;
+
+	switch (clk) {
+	case QE_CLK3:  val = QE_CMXGCR_USBCS_CLK3;  break;
+	case QE_CLK5:  val = QE_CMXGCR_USBCS_CLK5;  break;
+	case QE_CLK7:  val = QE_CMXGCR_USBCS_CLK7;  break;
+	case QE_CLK9:  val = QE_CMXGCR_USBCS_CLK9;  break;
+	case QE_CLK13: val = QE_CMXGCR_USBCS_CLK13; break;
+	case QE_CLK17: val = QE_CMXGCR_USBCS_CLK17; break;
+	case QE_CLK19: val = QE_CMXGCR_USBCS_CLK19; break;
+	case QE_CLK21: val = QE_CMXGCR_USBCS_CLK21; break;
+	case QE_BRG9:  val = QE_CMXGCR_USBCS_BRG9;  break;
+	case QE_BRG10: val = QE_CMXGCR_USBCS_BRG10; break;
+	default:
+		pr_err("%s: requested unknown clock %d\n", __func__, clk);
+		return -EINVAL;
+	}
+
+	if (qe_clock_is_brg(clk))
+		qe_setbrg(clk, rate, 1);
+
+	spin_lock_irqsave(&cmxgcr_lock, flags);
+
+	clrsetbits_be32(&mux->cmxgcr, QE_CMXGCR_USBCS, val);
+
+	spin_unlock_irqrestore(&cmxgcr_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(qe_usb_clock_set);
diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c
index 4d4203c896c4..0221387e5e27 100644
--- a/drivers/soc/mediatek/mtk-scpsys.c
+++ b/drivers/soc/mediatek/mtk-scpsys.c
@@ -15,12 +15,13 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/regmap.h>
 #include <linux/soc/mediatek/infracfg.h>
+#include <linux/regulator/consumer.h>
 #include <dt-bindings/power/mt8173-power.h>
 
 #define SPM_VDE_PWR_CON			0x0210
@@ -179,6 +180,7 @@ struct scp_domain {
 	u32 sram_pdn_ack_bits;
 	u32 bus_prot_mask;
 	bool active_wakeup;
+	struct regulator *supply;
 };
 
 struct scp {
@@ -221,6 +223,12 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	int ret;
 	int i;
 
+	if (scpd->supply) {
+		ret = regulator_enable(scpd->supply);
+		if (ret)
+			return ret;
+	}
+
 	for (i = 0; i < MAX_CLKS && scpd->clk[i]; i++) {
 		ret = clk_prepare_enable(scpd->clk[i]);
 		if (ret) {
@@ -299,6 +307,9 @@ err_pwr_ack:
 			clk_disable_unprepare(scpd->clk[i]);
 	}
 err_clk:
+	if (scpd->supply)
+		regulator_disable(scpd->supply);
+
 	dev_err(scp->dev, "Failed to power on domain %s\n", genpd->name);
 
 	return ret;
@@ -379,6 +390,9 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 	for (i = 0; i < MAX_CLKS && scpd->clk[i]; i++)
 		clk_disable_unprepare(scpd->clk[i]);
 
+	if (scpd->supply)
+		regulator_disable(scpd->supply);
+
 	return 0;
 
 out:
@@ -448,6 +462,19 @@ static int __init scpsys_probe(struct platform_device *pdev)
 		return PTR_ERR(scp->infracfg);
 	}
 
+	for (i = 0; i < NUM_DOMAINS; i++) {
+		struct scp_domain *scpd = &scp->domains[i];
+		const struct scp_domain_data *data = &scp_domain_data[i];
+
+		scpd->supply = devm_regulator_get_optional(&pdev->dev, data->name);
+		if (IS_ERR(scpd->supply)) {
+			if (PTR_ERR(scpd->supply) == -ENODEV)
+				scpd->supply = NULL;
+			else
+				return PTR_ERR(scpd->supply);
+		}
+	}
+
 	pd_data->num_domains = NUM_DOMAINS;
 
 	for (i = 0; i < NUM_DOMAINS; i++) {
@@ -521,5 +548,4 @@ static struct platform_driver scpsys_drv = {
 		.of_match_table = of_match_ptr(of_scpsys_match_tbl),
 	},
 };
-
-module_platform_driver_probe(scpsys_drv, scpsys_probe);
+builtin_platform_driver_probe(scpsys_drv, scpsys_probe);
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index eec76141d9b9..461b387d03cc 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -13,6 +13,7 @@ config QCOM_GSBI
 config QCOM_PM
 	bool "Qualcomm Power Management"
 	depends on ARCH_QCOM && !ARM64
+	select ARM_CPU_SUSPEND
 	select QCOM_SCM
 	help
 	  QCOM Platform specific power driver to manage cores and L2 low power
@@ -49,3 +50,29 @@ config QCOM_SMD_RPM
 
 	  Say M here if you want to include support for the Qualcomm RPM as a
 	  module. This will build a module called "qcom-smd-rpm".
+
+config QCOM_SMEM_STATE
+	bool
+
+config QCOM_SMP2P
+	tristate "Qualcomm Shared Memory Point to Point support"
+	depends on QCOM_SMEM
+	select QCOM_SMEM_STATE
+	help
+	  Say yes here to support the Qualcomm Shared Memory Point to Point
+	  protocol.
+
+config QCOM_SMSM
+	tristate "Qualcomm Shared Memory State Machine"
+	depends on QCOM_SMEM
+	select QCOM_SMEM_STATE
+	help
+	  Say yes here to support the Qualcomm Shared Memory State Machine.
+	  The state machine is represented by bits in shared memory.
+
+config QCOM_WCNSS_CTRL
+	tristate "Qualcomm WCNSS control driver"
+	depends on QCOM_SMD
+	help
+	  Client driver for the WCNSS_CTRL SMD channel, used to download nv
+	  firmware to a newly booted WCNSS chip.
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index 10a93d168e0e..fdd664edf0bd 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -3,3 +3,7 @@ obj-$(CONFIG_QCOM_PM)	+=	spm.o
 obj-$(CONFIG_QCOM_SMD) +=	smd.o
 obj-$(CONFIG_QCOM_SMD_RPM)	+= smd-rpm.o
 obj-$(CONFIG_QCOM_SMEM) +=	smem.o
+obj-$(CONFIG_QCOM_SMEM_STATE) += smem_state.o
+obj-$(CONFIG_QCOM_SMP2P)	+= smp2p.o
+obj-$(CONFIG_QCOM_SMSM)	+= smsm.o
+obj-$(CONFIG_QCOM_WCNSS_CTRL) += wcnss_ctrl.o
diff --git a/drivers/soc/qcom/smd-rpm.c b/drivers/soc/qcom/smd-rpm.c
index 2969321e1b09..731fa066f712 100644
--- a/drivers/soc/qcom/smd-rpm.c
+++ b/drivers/soc/qcom/smd-rpm.c
@@ -219,6 +219,8 @@ static void qcom_smd_rpm_remove(struct qcom_smd_device *sdev)
 }
 
 static const struct of_device_id qcom_smd_rpm_of_match[] = {
+	{ .compatible = "qcom,rpm-apq8084" },
+	{ .compatible = "qcom,rpm-msm8916" },
 	{ .compatible = "qcom,rpm-msm8974" },
 	{}
 };
diff --git a/drivers/soc/qcom/smd.c b/drivers/soc/qcom/smd.c
index 86b598cff91a..498fd0581a45 100644
--- a/drivers/soc/qcom/smd.c
+++ b/drivers/soc/qcom/smd.c
@@ -434,20 +434,15 @@ static void smd_copy_to_fifo(void __iomem *dst,
 /*
  * Copy count bytes of data using 32bit accesses, if that is required.
  */
-static void smd_copy_from_fifo(void *_dst,
-			       const void __iomem *_src,
+static void smd_copy_from_fifo(void *dst,
+			       const void __iomem *src,
 			       size_t count,
 			       bool word_aligned)
 {
-	u32 *dst = (u32 *)_dst;
-	u32 *src = (u32 *)_src;
-
 	if (word_aligned) {
-		count /= sizeof(u32);
-		while (count--)
-			*dst++ = __raw_readl(src++);
+		__ioread32_copy(dst, src, count / sizeof(u32));
 	} else {
-		memcpy_fromio(_dst, _src, count);
+		memcpy_fromio(dst, src, count);
 	}
 }
 
diff --git a/drivers/soc/qcom/smem_state.c b/drivers/soc/qcom/smem_state.c
new file mode 100644
index 000000000000..54261decb369
--- /dev/null
+++ b/drivers/soc/qcom/smem_state.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2015, Sony Mobile Communications Inc.
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smem_state.h>
+
+static LIST_HEAD(smem_states);
+static DEFINE_MUTEX(list_lock);
+
+/**
+ * struct qcom_smem_state - state context
+ * @refcount:	refcount for the state
+ * @orphan:	boolean indicator that this state has been unregistered
+ * @list:	entry in smem_states list
+ * @of_node:	of_node to use for matching the state in DT
+ * @priv:	implementation private data
+ * @ops:	ops for the state
+ */
+struct qcom_smem_state {
+	struct kref refcount;
+	bool orphan;
+
+	struct list_head list;
+	struct device_node *of_node;
+
+	void *priv;
+
+	struct qcom_smem_state_ops ops;
+};
+
+/**
+ * qcom_smem_state_update_bits() - update the masked bits in state with value
+ * @state:	state handle acquired by calling qcom_smem_state_get()
+ * @mask:	bit mask for the change
+ * @value:	new value for the masked bits
+ *
+ * Returns 0 on success, otherwise negative errno.
+ */
+int qcom_smem_state_update_bits(struct qcom_smem_state *state,
+				u32 mask,
+				u32 value)
+{
+	if (state->orphan)
+		return -ENXIO;
+
+	if (!state->ops.update_bits)
+		return -ENOTSUPP;
+
+	return state->ops.update_bits(state->priv, mask, value);
+}
+EXPORT_SYMBOL_GPL(qcom_smem_state_update_bits);
+
+static struct qcom_smem_state *of_node_to_state(struct device_node *np)
+{
+	struct qcom_smem_state *state;
+
+	mutex_lock(&list_lock);
+
+	list_for_each_entry(state, &smem_states, list) {
+		if (state->of_node == np) {
+			kref_get(&state->refcount);
+			goto unlock;
+		}
+	}
+	state = ERR_PTR(-EPROBE_DEFER);
+
+unlock:
+	mutex_unlock(&list_lock);
+
+	return state;
+}
+
+/**
+ * qcom_smem_state_get() - acquire handle to a state
+ * @dev:	client device pointer
+ * @con_id:	name of the state to lookup
+ * @bit:	flags from the state reference, indicating which bit's affected
+ *
+ * Returns handle to the state, or ERR_PTR(). qcom_smem_state_put() must be
+ * called to release the returned state handle.
+ */
+struct qcom_smem_state *qcom_smem_state_get(struct device *dev,
+					    const char *con_id,
+					    unsigned *bit)
+{
+	struct qcom_smem_state *state;
+	struct of_phandle_args args;
+	int index = 0;
+	int ret;
+
+	if (con_id) {
+		index = of_property_match_string(dev->of_node,
+						 "qcom,state-names",
+						 con_id);
+		if (index < 0) {
+			dev_err(dev, "missing qcom,state-names\n");
+			return ERR_PTR(index);
+		}
+	}
+
+	ret = of_parse_phandle_with_args(dev->of_node,
+					 "qcom,state",
+					 "#qcom,state-cells",
+					 index,
+					 &args);
+	if (ret) {
+		dev_err(dev, "failed to parse qcom,state property\n");
+		return ERR_PTR(ret);
+	}
+
+	if (args.args_count != 1) {
+		dev_err(dev, "invalid #qcom,state-cells\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	state = of_node_to_state(args.np);
+	if (IS_ERR(state))
+		goto put;
+
+	*bit = args.args[0];
+
+put:
+	of_node_put(args.np);
+	return state;
+}
+EXPORT_SYMBOL_GPL(qcom_smem_state_get);
+
+static void qcom_smem_state_release(struct kref *ref)
+{
+	struct qcom_smem_state *state = container_of(ref, struct qcom_smem_state, refcount);
+
+	list_del(&state->list);
+	kfree(state);
+}
+
+/**
+ * qcom_smem_state_put() - release state handle
+ * @state:	state handle to be released
+ */
+void qcom_smem_state_put(struct qcom_smem_state *state)
+{
+	mutex_lock(&list_lock);
+	kref_put(&state->refcount, qcom_smem_state_release);
+	mutex_unlock(&list_lock);
+}
+EXPORT_SYMBOL_GPL(qcom_smem_state_put);
+
+/**
+ * qcom_smem_state_register() - register a new state
+ * @of_node:	of_node used for matching client lookups
+ * @ops:	implementation ops
+ * @priv:	implementation specific private data
+ */
+struct qcom_smem_state *qcom_smem_state_register(struct device_node *of_node,
+						 const struct qcom_smem_state_ops *ops,
+						 void *priv)
+{
+	struct qcom_smem_state *state;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&state->refcount);
+
+	state->of_node = of_node;
+	state->ops = *ops;
+	state->priv = priv;
+
+	mutex_lock(&list_lock);
+	list_add(&state->list, &smem_states);
+	mutex_unlock(&list_lock);
+
+	return state;
+}
+EXPORT_SYMBOL_GPL(qcom_smem_state_register);
+
+/**
+ * qcom_smem_state_unregister() - unregister a registered state
+ * @state:	state handle to be unregistered
+ */
+void qcom_smem_state_unregister(struct qcom_smem_state *state)
+{
+	state->orphan = true;
+	qcom_smem_state_put(state);
+}
+EXPORT_SYMBOL_GPL(qcom_smem_state_unregister);
diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c
new file mode 100644
index 000000000000..f1eed7f9dd67
--- /dev/null
+++ b/drivers/soc/qcom/smp2p.c
@@ -0,0 +1,578 @@
+/*
+ * Copyright (c) 2015, Sony Mobile Communications AB.
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/soc/qcom/smem_state.h>
+#include <linux/spinlock.h>
+
+/*
+ * The Shared Memory Point to Point (SMP2P) protocol facilitates communication
+ * of a single 32-bit value between two processors.  Each value has a single
+ * writer (the local side) and a single reader (the remote side). Values are
+ * uniquely identified in the system by the directed edge (local processor ID
+ * to remote processor ID) and a string identifier.
+ *
+ * Each processor is responsible for creating the outgoing SMEM items and each
+ * item is writable by the local processor and readable by the remote
+ * processor.  By using two separate SMEM items that are single-reader and
+ * single-writer, SMP2P does not require any remote locking mechanisms.
+ *
+ * The driver uses the Linux GPIO and interrupt framework to expose a virtual
+ * GPIO for each outbound entry and a virtual interrupt controller for each
+ * inbound entry.
+ */
+
+#define SMP2P_MAX_ENTRY 16
+#define SMP2P_MAX_ENTRY_NAME 16
+
+#define SMP2P_FEATURE_SSR_ACK 0x1
+
+#define SMP2P_MAGIC 0x504d5324
+
+/**
+ * struct smp2p_smem_item - in memory communication structure
+ * @magic:		magic number
+ * @version:		version - must be 1
+ * @features:		features flag - currently unused
+ * @local_pid:		processor id of sending end
+ * @remote_pid:		processor id of receiving end
+ * @total_entries:	number of entries - always SMP2P_MAX_ENTRY
+ * @valid_entries:	number of allocated entries
+ * @flags:
+ * @entries:		individual communication entries
+ *     @name:		name of the entry
+ *     @value:		content of the entry
+ */
+struct smp2p_smem_item {
+	u32 magic;
+	u8 version;
+	unsigned features:24;
+	u16 local_pid;
+	u16 remote_pid;
+	u16 total_entries;
+	u16 valid_entries;
+	u32 flags;
+
+	struct {
+		u8 name[SMP2P_MAX_ENTRY_NAME];
+		u32 value;
+	} entries[SMP2P_MAX_ENTRY];
+} __packed;
+
+/**
+ * struct smp2p_entry - driver context matching one entry
+ * @node:	list entry to keep track of allocated entries
+ * @smp2p:	reference to the device driver context
+ * @name:	name of the entry, to match against smp2p_smem_item
+ * @value:	pointer to smp2p_smem_item entry value
+ * @last_value:	last handled value
+ * @domain:	irq_domain for inbound entries
+ * @irq_enabled:bitmap to track enabled irq bits
+ * @irq_rising:	bitmap to mark irq bits for rising detection
+ * @irq_falling:bitmap to mark irq bits for falling detection
+ * @state:	smem state handle
+ * @lock:	spinlock to protect read-modify-write of the value
+ */
+struct smp2p_entry {
+	struct list_head node;
+	struct qcom_smp2p *smp2p;
+
+	const char *name;
+	u32 *value;
+	u32 last_value;
+
+	struct irq_domain *domain;
+	DECLARE_BITMAP(irq_enabled, 32);
+	DECLARE_BITMAP(irq_rising, 32);
+	DECLARE_BITMAP(irq_falling, 32);
+
+	struct qcom_smem_state *state;
+
+	spinlock_t lock;
+};
+
+#define SMP2P_INBOUND	0
+#define SMP2P_OUTBOUND	1
+
+/**
+ * struct qcom_smp2p - device driver context
+ * @dev:	device driver handle
+ * @in:		pointer to the inbound smem item
+ * @smem_items:	ids of the two smem items
+ * @valid_entries: already scanned inbound entries
+ * @local_pid:	processor id of the inbound edge
+ * @remote_pid:	processor id of the outbound edge
+ * @ipc_regmap:	regmap for the outbound ipc
+ * @ipc_offset:	offset within the regmap
+ * @ipc_bit:	bit in regmap@offset to kick to signal remote processor
+ * @inbound:	list of inbound entries
+ * @outbound:	list of outbound entries
+ */
+struct qcom_smp2p {
+	struct device *dev;
+
+	struct smp2p_smem_item *in;
+	struct smp2p_smem_item *out;
+
+	unsigned smem_items[SMP2P_OUTBOUND + 1];
+
+	unsigned valid_entries;
+
+	unsigned local_pid;
+	unsigned remote_pid;
+
+	struct regmap *ipc_regmap;
+	int ipc_offset;
+	int ipc_bit;
+
+	struct list_head inbound;
+	struct list_head outbound;
+};
+
+static void qcom_smp2p_kick(struct qcom_smp2p *smp2p)
+{
+	/* Make sure any updated data is written before the kick */
+	wmb();
+	regmap_write(smp2p->ipc_regmap, smp2p->ipc_offset, BIT(smp2p->ipc_bit));
+}
+
+/**
+ * qcom_smp2p_intr() - interrupt handler for incoming notifications
+ * @irq:	unused
+ * @data:	smp2p driver context
+ *
+ * Handle notifications from the remote side to handle newly allocated entries
+ * or any changes to the state bits of existing entries.
+ */
+static irqreturn_t qcom_smp2p_intr(int irq, void *data)
+{
+	struct smp2p_smem_item *in;
+	struct smp2p_entry *entry;
+	struct qcom_smp2p *smp2p = data;
+	unsigned smem_id = smp2p->smem_items[SMP2P_INBOUND];
+	unsigned pid = smp2p->remote_pid;
+	size_t size;
+	int irq_pin;
+	u32 status;
+	char buf[SMP2P_MAX_ENTRY_NAME];
+	u32 val;
+	int i;
+
+	in = smp2p->in;
+
+	/* Acquire smem item, if not already found */
+	if (!in) {
+		in = qcom_smem_get(pid, smem_id, &size);
+		if (IS_ERR(in)) {
+			dev_err(smp2p->dev,
+				"Unable to acquire remote smp2p item\n");
+			return IRQ_HANDLED;
+		}
+
+		smp2p->in = in;
+	}
+
+	/* Match newly created entries */
+	for (i = smp2p->valid_entries; i < in->valid_entries; i++) {
+		list_for_each_entry(entry, &smp2p->inbound, node) {
+			memcpy_fromio(buf, in->entries[i].name, sizeof(buf));
+			if (!strcmp(buf, entry->name)) {
+				entry->value = &in->entries[i].value;
+				break;
+			}
+		}
+	}
+	smp2p->valid_entries = i;
+
+	/* Fire interrupts based on any value changes */
+	list_for_each_entry(entry, &smp2p->inbound, node) {
+		/* Ignore entries not yet allocated by the remote side */
+		if (!entry->value)
+			continue;
+
+		val = readl(entry->value);
+
+		status = val ^ entry->last_value;
+		entry->last_value = val;
+
+		/* No changes of this entry? */
+		if (!status)
+			continue;
+
+		for_each_set_bit(i, entry->irq_enabled, 32) {
+			if (!(status & BIT(i)))
+				continue;
+
+			if ((val & BIT(i) && test_bit(i, entry->irq_rising)) ||
+			    (!(val & BIT(i)) && test_bit(i, entry->irq_falling))) {
+				irq_pin = irq_find_mapping(entry->domain, i);
+				handle_nested_irq(irq_pin);
+			}
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void smp2p_mask_irq(struct irq_data *irqd)
+{
+	struct smp2p_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+
+	clear_bit(irq, entry->irq_enabled);
+}
+
+static void smp2p_unmask_irq(struct irq_data *irqd)
+{
+	struct smp2p_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+
+	set_bit(irq, entry->irq_enabled);
+}
+
+static int smp2p_set_irq_type(struct irq_data *irqd, unsigned int type)
+{
+	struct smp2p_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+
+	if (!(type & IRQ_TYPE_EDGE_BOTH))
+		return -EINVAL;
+
+	if (type & IRQ_TYPE_EDGE_RISING)
+		set_bit(irq, entry->irq_rising);
+	else
+		clear_bit(irq, entry->irq_rising);
+
+	if (type & IRQ_TYPE_EDGE_FALLING)
+		set_bit(irq, entry->irq_falling);
+	else
+		clear_bit(irq, entry->irq_falling);
+
+	return 0;
+}
+
+static struct irq_chip smp2p_irq_chip = {
+	.name           = "smp2p",
+	.irq_mask       = smp2p_mask_irq,
+	.irq_unmask     = smp2p_unmask_irq,
+	.irq_set_type	= smp2p_set_irq_type,
+};
+
+static int smp2p_irq_map(struct irq_domain *d,
+			 unsigned int irq,
+			 irq_hw_number_t hw)
+{
+	struct smp2p_entry *entry = d->host_data;
+
+	irq_set_chip_and_handler(irq, &smp2p_irq_chip, handle_level_irq);
+	irq_set_chip_data(irq, entry);
+	irq_set_nested_thread(irq, 1);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops smp2p_irq_ops = {
+	.map = smp2p_irq_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
+static int qcom_smp2p_inbound_entry(struct qcom_smp2p *smp2p,
+				    struct smp2p_entry *entry,
+				    struct device_node *node)
+{
+	entry->domain = irq_domain_add_linear(node, 32, &smp2p_irq_ops, entry);
+	if (!entry->domain) {
+		dev_err(smp2p->dev, "failed to add irq_domain\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int smp2p_update_bits(void *data, u32 mask, u32 value)
+{
+	struct smp2p_entry *entry = data;
+	u32 orig;
+	u32 val;
+
+	spin_lock(&entry->lock);
+	val = orig = readl(entry->value);
+	val &= ~mask;
+	val |= value;
+	writel(val, entry->value);
+	spin_unlock(&entry->lock);
+
+	if (val != orig)
+		qcom_smp2p_kick(entry->smp2p);
+
+	return 0;
+}
+
+static const struct qcom_smem_state_ops smp2p_state_ops = {
+	.update_bits = smp2p_update_bits,
+};
+
+static int qcom_smp2p_outbound_entry(struct qcom_smp2p *smp2p,
+				     struct smp2p_entry *entry,
+				     struct device_node *node)
+{
+	struct smp2p_smem_item *out = smp2p->out;
+	char buf[SMP2P_MAX_ENTRY_NAME] = {};
+
+	/* Allocate an entry from the smem item */
+	strlcpy(buf, entry->name, SMP2P_MAX_ENTRY_NAME);
+	memcpy_toio(out->entries[out->valid_entries].name, buf, SMP2P_MAX_ENTRY_NAME);
+	out->valid_entries++;
+
+	/* Make the logical entry reference the physical value */
+	entry->value = &out->entries[out->valid_entries].value;
+
+	entry->state = qcom_smem_state_register(node, &smp2p_state_ops, entry);
+	if (IS_ERR(entry->state)) {
+		dev_err(smp2p->dev, "failed to register qcom_smem_state\n");
+		return PTR_ERR(entry->state);
+	}
+
+	return 0;
+}
+
+static int qcom_smp2p_alloc_outbound_item(struct qcom_smp2p *smp2p)
+{
+	struct smp2p_smem_item *out;
+	unsigned smem_id = smp2p->smem_items[SMP2P_OUTBOUND];
+	unsigned pid = smp2p->remote_pid;
+	int ret;
+
+	ret = qcom_smem_alloc(pid, smem_id, sizeof(*out));
+	if (ret < 0 && ret != -EEXIST) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(smp2p->dev,
+				"unable to allocate local smp2p item\n");
+		return ret;
+	}
+
+	out = qcom_smem_get(pid, smem_id, NULL);
+	if (IS_ERR(out)) {
+		dev_err(smp2p->dev, "Unable to acquire local smp2p item\n");
+		return PTR_ERR(out);
+	}
+
+	memset(out, 0, sizeof(*out));
+	out->magic = SMP2P_MAGIC;
+	out->local_pid = smp2p->local_pid;
+	out->remote_pid = smp2p->remote_pid;
+	out->total_entries = SMP2P_MAX_ENTRY;
+	out->valid_entries = 0;
+
+	/*
+	 * Make sure the rest of the header is written before we validate the
+	 * item by writing a valid version number.
+	 */
+	wmb();
+	out->version = 1;
+
+	qcom_smp2p_kick(smp2p);
+
+	smp2p->out = out;
+
+	return 0;
+}
+
+static int smp2p_parse_ipc(struct qcom_smp2p *smp2p)
+{
+	struct device_node *syscon;
+	struct device *dev = smp2p->dev;
+	const char *key;
+	int ret;
+
+	syscon = of_parse_phandle(dev->of_node, "qcom,ipc", 0);
+	if (!syscon) {
+		dev_err(dev, "no qcom,ipc node\n");
+		return -ENODEV;
+	}
+
+	smp2p->ipc_regmap = syscon_node_to_regmap(syscon);
+	if (IS_ERR(smp2p->ipc_regmap))
+		return PTR_ERR(smp2p->ipc_regmap);
+
+	key = "qcom,ipc";
+	ret = of_property_read_u32_index(dev->of_node, key, 1, &smp2p->ipc_offset);
+	if (ret < 0) {
+		dev_err(dev, "no offset in %s\n", key);
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32_index(dev->of_node, key, 2, &smp2p->ipc_bit);
+	if (ret < 0) {
+		dev_err(dev, "no bit in %s\n", key);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int qcom_smp2p_probe(struct platform_device *pdev)
+{
+	struct smp2p_entry *entry;
+	struct device_node *node;
+	struct qcom_smp2p *smp2p;
+	const char *key;
+	int irq;
+	int ret;
+
+	smp2p = devm_kzalloc(&pdev->dev, sizeof(*smp2p), GFP_KERNEL);
+	if (!smp2p)
+		return -ENOMEM;
+
+	smp2p->dev = &pdev->dev;
+	INIT_LIST_HEAD(&smp2p->inbound);
+	INIT_LIST_HEAD(&smp2p->outbound);
+
+	platform_set_drvdata(pdev, smp2p);
+
+	ret = smp2p_parse_ipc(smp2p);
+	if (ret)
+		return ret;
+
+	key = "qcom,smem";
+	ret = of_property_read_u32_array(pdev->dev.of_node, key,
+					 smp2p->smem_items, 2);
+	if (ret)
+		return ret;
+
+	key = "qcom,local-pid";
+	ret = of_property_read_u32(pdev->dev.of_node, key, &smp2p->local_pid);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to read %s\n", key);
+		return -EINVAL;
+	}
+
+	key = "qcom,remote-pid";
+	ret = of_property_read_u32(pdev->dev.of_node, key, &smp2p->remote_pid);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to read %s\n", key);
+		return -EINVAL;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "unable to acquire smp2p interrupt\n");
+		return irq;
+	}
+
+	ret = qcom_smp2p_alloc_outbound_item(smp2p);
+	if (ret < 0)
+		return ret;
+
+	for_each_available_child_of_node(pdev->dev.of_node, node) {
+		entry = devm_kzalloc(&pdev->dev, sizeof(*entry), GFP_KERNEL);
+		if (!entry) {
+			ret = -ENOMEM;
+			goto unwind_interfaces;
+		}
+
+		entry->smp2p = smp2p;
+		spin_lock_init(&entry->lock);
+
+		ret = of_property_read_string(node, "qcom,entry-name", &entry->name);
+		if (ret < 0)
+			goto unwind_interfaces;
+
+		if (of_property_read_bool(node, "interrupt-controller")) {
+			ret = qcom_smp2p_inbound_entry(smp2p, entry, node);
+			if (ret < 0)
+				goto unwind_interfaces;
+
+			list_add(&entry->node, &smp2p->inbound);
+		} else  {
+			ret = qcom_smp2p_outbound_entry(smp2p, entry, node);
+			if (ret < 0)
+				goto unwind_interfaces;
+
+			list_add(&entry->node, &smp2p->outbound);
+		}
+	}
+
+	/* Kick the outgoing edge after allocating entries */
+	qcom_smp2p_kick(smp2p);
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq,
+					NULL, qcom_smp2p_intr,
+					IRQF_ONESHOT,
+					"smp2p", (void *)smp2p);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request interrupt\n");
+		goto unwind_interfaces;
+	}
+
+
+	return 0;
+
+unwind_interfaces:
+	list_for_each_entry(entry, &smp2p->inbound, node)
+		irq_domain_remove(entry->domain);
+
+	list_for_each_entry(entry, &smp2p->outbound, node)
+		qcom_smem_state_unregister(entry->state);
+
+	smp2p->out->valid_entries = 0;
+
+	return ret;
+}
+
+static int qcom_smp2p_remove(struct platform_device *pdev)
+{
+	struct qcom_smp2p *smp2p = platform_get_drvdata(pdev);
+	struct smp2p_entry *entry;
+
+	list_for_each_entry(entry, &smp2p->inbound, node)
+		irq_domain_remove(entry->domain);
+
+	list_for_each_entry(entry, &smp2p->outbound, node)
+		qcom_smem_state_unregister(entry->state);
+
+	smp2p->out->valid_entries = 0;
+
+	return 0;
+}
+
+static const struct of_device_id qcom_smp2p_of_match[] = {
+	{ .compatible = "qcom,smp2p" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_smp2p_of_match);
+
+static struct platform_driver qcom_smp2p_driver = {
+	.probe = qcom_smp2p_probe,
+	.remove = qcom_smp2p_remove,
+	.driver  = {
+		.name  = "qcom_smp2p",
+		.of_match_table = qcom_smp2p_of_match,
+	},
+};
+module_platform_driver(qcom_smp2p_driver);
+
+MODULE_DESCRIPTION("Qualcomm Shared Memory Point to Point driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c
new file mode 100644
index 000000000000..6b777af1bc19
--- /dev/null
+++ b/drivers/soc/qcom/smsm.c
@@ -0,0 +1,625 @@
+/*
+ * Copyright (c) 2015, Sony Mobile Communications Inc.
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/soc/qcom/smem_state.h>
+
+/*
+ * This driver implements the Qualcomm Shared Memory State Machine, a mechanism
+ * for communicating single bit state information to remote processors.
+ *
+ * The implementation is based on two sections of shared memory; the first
+ * holding the state bits and the second holding a matrix of subscription bits.
+ *
+ * The state bits are structured in entries of 32 bits, each belonging to one
+ * system in the SoC. The entry belonging to the local system is considered
+ * read-write, while the rest should be considered read-only.
+ *
+ * The subscription matrix consists of N bitmaps per entry, denoting interest
+ * in updates of the entry for each of the N hosts. Upon updating a state bit
+ * each host's subscription bitmap should be queried and the remote system
+ * should be interrupted if they request so.
+ *
+ * The subscription matrix is laid out in entry-major order:
+ * entry0: [host0 ... hostN]
+ *	.
+ *	.
+ * entryM: [host0 ... hostN]
+ *
+ * A third, optional, shared memory region might contain information regarding
+ * the number of entries in the state bitmap as well as number of columns in
+ * the subscription matrix.
+ */
+
+/*
+ * Shared memory identifiers, used to acquire handles to respective memory
+ * region.
+ */
+#define SMEM_SMSM_SHARED_STATE		85
+#define SMEM_SMSM_CPU_INTR_MASK		333
+#define SMEM_SMSM_SIZE_INFO		419
+
+/*
+ * Default sizes, in case SMEM_SMSM_SIZE_INFO is not found.
+ */
+#define SMSM_DEFAULT_NUM_ENTRIES	8
+#define SMSM_DEFAULT_NUM_HOSTS		3
+
+struct smsm_entry;
+struct smsm_host;
+
+/**
+ * struct qcom_smsm - smsm driver context
+ * @dev:	smsm device pointer
+ * @local_host:	column in the subscription matrix representing this system
+ * @num_hosts:	number of columns in the subscription matrix
+ * @num_entries: number of entries in the state map and rows in the subscription
+ *		matrix
+ * @local_state: pointer to the local processor's state bits
+ * @subscription: pointer to local processor's row in subscription matrix
+ * @state:	smem state handle
+ * @lock:	spinlock for read-modify-write of the outgoing state
+ * @entries:	context for each of the entries
+ * @hosts:	context for each of the hosts
+ */
+struct qcom_smsm {
+	struct device *dev;
+
+	u32 local_host;
+
+	u32 num_hosts;
+	u32 num_entries;
+
+	u32 *local_state;
+	u32 *subscription;
+	struct qcom_smem_state *state;
+
+	spinlock_t lock;
+
+	struct smsm_entry *entries;
+	struct smsm_host *hosts;
+};
+
+/**
+ * struct smsm_entry - per remote processor entry context
+ * @smsm:	back-reference to driver context
+ * @domain:	IRQ domain for this entry, if representing a remote system
+ * @irq_enabled: bitmap of which state bits IRQs are enabled
+ * @irq_rising:	bitmap tracking if rising bits should be propagated
+ * @irq_falling: bitmap tracking if falling bits should be propagated
+ * @last_value:	snapshot of state bits last time the interrupts where propagated
+ * @remote_state: pointer to this entry's state bits
+ * @subscription: pointer to a row in the subscription matrix representing this
+ *		entry
+ */
+struct smsm_entry {
+	struct qcom_smsm *smsm;
+
+	struct irq_domain *domain;
+	DECLARE_BITMAP(irq_enabled, 32);
+	DECLARE_BITMAP(irq_rising, 32);
+	DECLARE_BITMAP(irq_falling, 32);
+	u32 last_value;
+
+	u32 *remote_state;
+	u32 *subscription;
+};
+
+/**
+ * struct smsm_host - representation of a remote host
+ * @ipc_regmap:	regmap for outgoing interrupt
+ * @ipc_offset:	offset in @ipc_regmap for outgoing interrupt
+ * @ipc_bit:	bit in @ipc_regmap + @ipc_offset for outgoing interrupt
+ */
+struct smsm_host {
+	struct regmap *ipc_regmap;
+	int ipc_offset;
+	int ipc_bit;
+};
+
+/**
+ * smsm_update_bits() - change bit in outgoing entry and inform subscribers
+ * @data:	smsm context pointer
+ * @offset:	bit in the entry
+ * @value:	new value
+ *
+ * Used to set and clear the bits in the outgoing/local entry and inform
+ * subscribers about the change.
+ */
+static int smsm_update_bits(void *data, u32 mask, u32 value)
+{
+	struct qcom_smsm *smsm = data;
+	struct smsm_host *hostp;
+	unsigned long flags;
+	u32 changes;
+	u32 host;
+	u32 orig;
+	u32 val;
+
+	spin_lock_irqsave(&smsm->lock, flags);
+
+	/* Update the entry */
+	val = orig = readl(smsm->local_state);
+	val &= ~mask;
+	val |= value;
+
+	/* Don't signal if we didn't change the value */
+	changes = val ^ orig;
+	if (!changes) {
+		spin_unlock_irqrestore(&smsm->lock, flags);
+		goto done;
+	}
+
+	/* Write out the new value */
+	writel(val, smsm->local_state);
+	spin_unlock_irqrestore(&smsm->lock, flags);
+
+	/* Make sure the value update is ordered before any kicks */
+	wmb();
+
+	/* Iterate over all hosts to check whom wants a kick */
+	for (host = 0; host < smsm->num_hosts; host++) {
+		hostp = &smsm->hosts[host];
+
+		val = readl(smsm->subscription + host);
+		if (val & changes && hostp->ipc_regmap) {
+			regmap_write(hostp->ipc_regmap,
+				     hostp->ipc_offset,
+				     BIT(hostp->ipc_bit));
+		}
+	}
+
+done:
+	return 0;
+}
+
+static const struct qcom_smem_state_ops smsm_state_ops = {
+	.update_bits = smsm_update_bits,
+};
+
+/**
+ * smsm_intr() - cascading IRQ handler for SMSM
+ * @irq:	unused
+ * @data:	entry related to this IRQ
+ *
+ * This function cascades an incoming interrupt from a remote system, based on
+ * the state bits and configuration.
+ */
+static irqreturn_t smsm_intr(int irq, void *data)
+{
+	struct smsm_entry *entry = data;
+	unsigned i;
+	int irq_pin;
+	u32 changed;
+	u32 val;
+
+	val = readl(entry->remote_state);
+	changed = val ^ entry->last_value;
+	entry->last_value = val;
+
+	for_each_set_bit(i, entry->irq_enabled, 32) {
+		if (!(changed & BIT(i)))
+			continue;
+
+		if (val & BIT(i)) {
+			if (test_bit(i, entry->irq_rising)) {
+				irq_pin = irq_find_mapping(entry->domain, i);
+				handle_nested_irq(irq_pin);
+			}
+		} else {
+			if (test_bit(i, entry->irq_falling)) {
+				irq_pin = irq_find_mapping(entry->domain, i);
+				handle_nested_irq(irq_pin);
+			}
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * smsm_mask_irq() - un-subscribe from cascades of IRQs of a certain staus bit
+ * @irqd:	IRQ handle to be masked
+ *
+ * This un-subscribes the local CPU from interrupts upon changes to the defines
+ * status bit. The bit is also cleared from cascading.
+ */
+static void smsm_mask_irq(struct irq_data *irqd)
+{
+	struct smsm_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+	struct qcom_smsm *smsm = entry->smsm;
+	u32 val;
+
+	if (entry->subscription) {
+		val = readl(entry->subscription + smsm->local_host);
+		val &= ~BIT(irq);
+		writel(val, entry->subscription + smsm->local_host);
+	}
+
+	clear_bit(irq, entry->irq_enabled);
+}
+
+/**
+ * smsm_unmask_irq() - subscribe to cascades of IRQs of a certain status bit
+ * @irqd:	IRQ handle to be unmasked
+ *
+
+ * This subscribes the local CPU to interrupts upon changes to the defined
+ * status bit. The bit is also marked for cascading.
+
+ */
+static void smsm_unmask_irq(struct irq_data *irqd)
+{
+	struct smsm_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+	struct qcom_smsm *smsm = entry->smsm;
+	u32 val;
+
+	set_bit(irq, entry->irq_enabled);
+
+	if (entry->subscription) {
+		val = readl(entry->subscription + smsm->local_host);
+		val |= BIT(irq);
+		writel(val, entry->subscription + smsm->local_host);
+	}
+}
+
+/**
+ * smsm_set_irq_type() - updates the requested IRQ type for the cascading
+ * @irqd:	consumer interrupt handle
+ * @type:	requested flags
+ */
+static int smsm_set_irq_type(struct irq_data *irqd, unsigned int type)
+{
+	struct smsm_entry *entry = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t irq = irqd_to_hwirq(irqd);
+
+	if (!(type & IRQ_TYPE_EDGE_BOTH))
+		return -EINVAL;
+
+	if (type & IRQ_TYPE_EDGE_RISING)
+		set_bit(irq, entry->irq_rising);
+	else
+		clear_bit(irq, entry->irq_rising);
+
+	if (type & IRQ_TYPE_EDGE_FALLING)
+		set_bit(irq, entry->irq_falling);
+	else
+		clear_bit(irq, entry->irq_falling);
+
+	return 0;
+}
+
+static struct irq_chip smsm_irq_chip = {
+	.name           = "smsm",
+	.irq_mask       = smsm_mask_irq,
+	.irq_unmask     = smsm_unmask_irq,
+	.irq_set_type	= smsm_set_irq_type,
+};
+
+/**
+ * smsm_irq_map() - sets up a mapping for a cascaded IRQ
+ * @d:		IRQ domain representing an entry
+ * @irq:	IRQ to set up
+ * @hw:		unused
+ */
+static int smsm_irq_map(struct irq_domain *d,
+			unsigned int irq,
+			irq_hw_number_t hw)
+{
+	struct smsm_entry *entry = d->host_data;
+
+	irq_set_chip_and_handler(irq, &smsm_irq_chip, handle_level_irq);
+	irq_set_chip_data(irq, entry);
+	irq_set_nested_thread(irq, 1);
+
+	return 0;
+}
+
+static const struct irq_domain_ops smsm_irq_ops = {
+	.map = smsm_irq_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
+/**
+ * smsm_parse_ipc() - parses a qcom,ipc-%d device tree property
+ * @smsm:	smsm driver context
+ * @host_id:	index of the remote host to be resolved
+ *
+ * Parses device tree to acquire the information needed for sending the
+ * outgoing interrupts to a remote host - identified by @host_id.
+ */
+static int smsm_parse_ipc(struct qcom_smsm *smsm, unsigned host_id)
+{
+	struct device_node *syscon;
+	struct device_node *node = smsm->dev->of_node;
+	struct smsm_host *host = &smsm->hosts[host_id];
+	char key[16];
+	int ret;
+
+	snprintf(key, sizeof(key), "qcom,ipc-%d", host_id);
+	syscon = of_parse_phandle(node, key, 0);
+	if (!syscon)
+		return 0;
+
+	host->ipc_regmap = syscon_node_to_regmap(syscon);
+	if (IS_ERR(host->ipc_regmap))
+		return PTR_ERR(host->ipc_regmap);
+
+	ret = of_property_read_u32_index(node, key, 1, &host->ipc_offset);
+	if (ret < 0) {
+		dev_err(smsm->dev, "no offset in %s\n", key);
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32_index(node, key, 2, &host->ipc_bit);
+	if (ret < 0) {
+		dev_err(smsm->dev, "no bit in %s\n", key);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * smsm_inbound_entry() - parse DT and set up an entry representing a remote system
+ * @smsm:	smsm driver context
+ * @entry:	entry context to be set up
+ * @node:	dt node containing the entry's properties
+ */
+static int smsm_inbound_entry(struct qcom_smsm *smsm,
+			      struct smsm_entry *entry,
+			      struct device_node *node)
+{
+	int ret;
+	int irq;
+
+	irq = irq_of_parse_and_map(node, 0);
+	if (!irq) {
+		dev_err(smsm->dev, "failed to parse smsm interrupt\n");
+		return -EINVAL;
+	}
+
+	ret = devm_request_threaded_irq(smsm->dev, irq,
+					NULL, smsm_intr,
+					IRQF_ONESHOT,
+					"smsm", (void *)entry);
+	if (ret) {
+		dev_err(smsm->dev, "failed to request interrupt\n");
+		return ret;
+	}
+
+	entry->domain = irq_domain_add_linear(node, 32, &smsm_irq_ops, entry);
+	if (!entry->domain) {
+		dev_err(smsm->dev, "failed to add irq_domain\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * smsm_get_size_info() - parse the optional memory segment for sizes
+ * @smsm:	smsm driver context
+ *
+ * Attempt to acquire the number of hosts and entries from the optional shared
+ * memory location. Not being able to find this segment should indicate that
+ * we're on a older system where these values was hard coded to
+ * SMSM_DEFAULT_NUM_ENTRIES and SMSM_DEFAULT_NUM_HOSTS.
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+static int smsm_get_size_info(struct qcom_smsm *smsm)
+{
+	size_t size;
+	struct {
+		u32 num_hosts;
+		u32 num_entries;
+		u32 reserved0;
+		u32 reserved1;
+	} *info;
+
+	info = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SIZE_INFO, &size);
+	if (PTR_ERR(info) == -ENOENT || size != sizeof(*info)) {
+		dev_warn(smsm->dev, "no smsm size info, using defaults\n");
+		smsm->num_entries = SMSM_DEFAULT_NUM_ENTRIES;
+		smsm->num_hosts = SMSM_DEFAULT_NUM_HOSTS;
+		return 0;
+	} else if (IS_ERR(info)) {
+		dev_err(smsm->dev, "unable to retrieve smsm size info\n");
+		return PTR_ERR(info);
+	}
+
+	smsm->num_entries = info->num_entries;
+	smsm->num_hosts = info->num_hosts;
+
+	dev_dbg(smsm->dev,
+		"found custom size of smsm: %d entries %d hosts\n",
+		smsm->num_entries, smsm->num_hosts);
+
+	return 0;
+}
+
+static int qcom_smsm_probe(struct platform_device *pdev)
+{
+	struct device_node *local_node;
+	struct device_node *node;
+	struct smsm_entry *entry;
+	struct qcom_smsm *smsm;
+	u32 *intr_mask;
+	size_t size;
+	u32 *states;
+	u32 id;
+	int ret;
+
+	smsm = devm_kzalloc(&pdev->dev, sizeof(*smsm), GFP_KERNEL);
+	if (!smsm)
+		return -ENOMEM;
+	smsm->dev = &pdev->dev;
+	spin_lock_init(&smsm->lock);
+
+	ret = smsm_get_size_info(smsm);
+	if (ret)
+		return ret;
+
+	smsm->entries = devm_kcalloc(&pdev->dev,
+				     smsm->num_entries,
+				     sizeof(struct smsm_entry),
+				     GFP_KERNEL);
+	if (!smsm->entries)
+		return -ENOMEM;
+
+	smsm->hosts = devm_kcalloc(&pdev->dev,
+				   smsm->num_hosts,
+				   sizeof(struct smsm_host),
+				   GFP_KERNEL);
+	if (!smsm->hosts)
+		return -ENOMEM;
+
+	local_node = of_find_node_with_property(pdev->dev.of_node, "#qcom,state-cells");
+	if (!local_node) {
+		dev_err(&pdev->dev, "no state entry\n");
+		return -EINVAL;
+	}
+
+	of_property_read_u32(pdev->dev.of_node,
+			     "qcom,local-host",
+			     &smsm->local_host);
+
+	/* Parse the host properties */
+	for (id = 0; id < smsm->num_hosts; id++) {
+		ret = smsm_parse_ipc(smsm, id);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* Acquire the main SMSM state vector */
+	ret = qcom_smem_alloc(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE,
+			      smsm->num_entries * sizeof(u32));
+	if (ret < 0 && ret != -EEXIST) {
+		dev_err(&pdev->dev, "unable to allocate shared state entry\n");
+		return ret;
+	}
+
+	states = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE, NULL);
+	if (IS_ERR(states)) {
+		dev_err(&pdev->dev, "Unable to acquire shared state entry\n");
+		return PTR_ERR(states);
+	}
+
+	/* Acquire the list of interrupt mask vectors */
+	size = smsm->num_entries * smsm->num_hosts * sizeof(u32);
+	ret = qcom_smem_alloc(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, size);
+	if (ret < 0 && ret != -EEXIST) {
+		dev_err(&pdev->dev, "unable to allocate smsm interrupt mask\n");
+		return ret;
+	}
+
+	intr_mask = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, NULL);
+	if (IS_ERR(intr_mask)) {
+		dev_err(&pdev->dev, "unable to acquire shared memory interrupt mask\n");
+		return PTR_ERR(intr_mask);
+	}
+
+	/* Setup the reference to the local state bits */
+	smsm->local_state = states + smsm->local_host;
+	smsm->subscription = intr_mask + smsm->local_host * smsm->num_hosts;
+
+	/* Register the outgoing state */
+	smsm->state = qcom_smem_state_register(local_node, &smsm_state_ops, smsm);
+	if (IS_ERR(smsm->state)) {
+		dev_err(smsm->dev, "failed to register qcom_smem_state\n");
+		return PTR_ERR(smsm->state);
+	}
+
+	/* Register handlers for remote processor entries of interest. */
+	for_each_available_child_of_node(pdev->dev.of_node, node) {
+		if (!of_property_read_bool(node, "interrupt-controller"))
+			continue;
+
+		ret = of_property_read_u32(node, "reg", &id);
+		if (ret || id >= smsm->num_entries) {
+			dev_err(&pdev->dev, "invalid reg of entry\n");
+			if (!ret)
+				ret = -EINVAL;
+			goto unwind_interfaces;
+		}
+		entry = &smsm->entries[id];
+
+		entry->smsm = smsm;
+		entry->remote_state = states + id;
+
+		/* Setup subscription pointers and unsubscribe to any kicks */
+		entry->subscription = intr_mask + id * smsm->num_hosts;
+		writel(0, entry->subscription + smsm->local_host);
+
+		ret = smsm_inbound_entry(smsm, entry, node);
+		if (ret < 0)
+			goto unwind_interfaces;
+	}
+
+	platform_set_drvdata(pdev, smsm);
+
+	return 0;
+
+unwind_interfaces:
+	for (id = 0; id < smsm->num_entries; id++)
+		if (smsm->entries[id].domain)
+			irq_domain_remove(smsm->entries[id].domain);
+
+	qcom_smem_state_unregister(smsm->state);
+
+	return ret;
+}
+
+static int qcom_smsm_remove(struct platform_device *pdev)
+{
+	struct qcom_smsm *smsm = platform_get_drvdata(pdev);
+	unsigned id;
+
+	for (id = 0; id < smsm->num_entries; id++)
+		if (smsm->entries[id].domain)
+			irq_domain_remove(smsm->entries[id].domain);
+
+	qcom_smem_state_unregister(smsm->state);
+
+	return 0;
+}
+
+static const struct of_device_id qcom_smsm_of_match[] = {
+	{ .compatible = "qcom,smsm" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_smsm_of_match);
+
+static struct platform_driver qcom_smsm_driver = {
+	.probe = qcom_smsm_probe,
+	.remove = qcom_smsm_remove,
+	.driver  = {
+		.name  = "qcom-smsm",
+		.of_match_table = qcom_smsm_of_match,
+	},
+};
+module_platform_driver(qcom_smsm_driver);
+
+MODULE_DESCRIPTION("Qualcomm Shared Memory State Machine driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/spm.c b/drivers/soc/qcom/spm.c
index 0ad66fa9bb1a..5548a31e1a39 100644
--- a/drivers/soc/qcom/spm.c
+++ b/drivers/soc/qcom/spm.c
@@ -288,7 +288,7 @@ static struct spm_driver_data *spm_get_drv(struct platform_device *pdev,
 	struct spm_driver_data *drv = NULL;
 	struct device_node *cpu_node, *saw_node;
 	int cpu;
-	bool found;
+	bool found = 0;
 
 	for_each_possible_cpu(cpu) {
 		cpu_node = of_cpu_device_node_get(cpu);
diff --git a/drivers/soc/qcom/wcnss_ctrl.c b/drivers/soc/qcom/wcnss_ctrl.c
new file mode 100644
index 000000000000..7a986f881d5c
--- /dev/null
+++ b/drivers/soc/qcom/wcnss_ctrl.c
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2015, Sony Mobile Communications Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smd.h>
+
+#define WCNSS_REQUEST_TIMEOUT	(5 * HZ)
+
+#define NV_FRAGMENT_SIZE	3072
+#define NVBIN_FILE		"wlan/prima/WCNSS_qcom_wlan_nv.bin"
+
+/**
+ * struct wcnss_ctrl - driver context
+ * @dev:	device handle
+ * @channel:	SMD channel handle
+ * @ack:	completion for outstanding requests
+ * @ack_status:	status of the outstanding request
+ * @download_nv_work: worker for uploading nv binary
+ */
+struct wcnss_ctrl {
+	struct device *dev;
+	struct qcom_smd_channel *channel;
+
+	struct completion ack;
+	int ack_status;
+
+	struct work_struct download_nv_work;
+};
+
+/* message types */
+enum {
+	WCNSS_VERSION_REQ = 0x01000000,
+	WCNSS_VERSION_RESP,
+	WCNSS_DOWNLOAD_NV_REQ,
+	WCNSS_DOWNLOAD_NV_RESP,
+	WCNSS_UPLOAD_CAL_REQ,
+	WCNSS_UPLOAD_CAL_RESP,
+	WCNSS_DOWNLOAD_CAL_REQ,
+	WCNSS_DOWNLOAD_CAL_RESP,
+};
+
+/**
+ * struct wcnss_msg_hdr - common packet header for requests and responses
+ * @type:	packet message type
+ * @len:	total length of the packet, including this header
+ */
+struct wcnss_msg_hdr {
+	u32 type;
+	u32 len;
+} __packed;
+
+/**
+ * struct wcnss_version_resp - version request response
+ * @hdr:	common packet wcnss_msg_hdr header
+ */
+struct wcnss_version_resp {
+	struct wcnss_msg_hdr hdr;
+	u8 major;
+	u8 minor;
+	u8 version;
+	u8 revision;
+} __packed;
+
+/**
+ * struct wcnss_download_nv_req - firmware fragment request
+ * @hdr:	common packet wcnss_msg_hdr header
+ * @seq:	sequence number of this fragment
+ * @last:	boolean indicator of this being the last fragment of the binary
+ * @frag_size:	length of this fragment
+ * @fragment:	fragment data
+ */
+struct wcnss_download_nv_req {
+	struct wcnss_msg_hdr hdr;
+	u16 seq;
+	u16 last;
+	u32 frag_size;
+	u8 fragment[];
+} __packed;
+
+/**
+ * struct wcnss_download_nv_resp - firmware download response
+ * @hdr:	common packet wcnss_msg_hdr header
+ * @status:	boolean to indicate success of the download
+ */
+struct wcnss_download_nv_resp {
+	struct wcnss_msg_hdr hdr;
+	u8 status;
+} __packed;
+
+/**
+ * wcnss_ctrl_smd_callback() - handler from SMD responses
+ * @qsdev:	smd device handle
+ * @data:	pointer to the incoming data packet
+ * @count:	size of the incoming data packet
+ *
+ * Handles any incoming packets from the remote WCNSS_CTRL service.
+ */
+static int wcnss_ctrl_smd_callback(struct qcom_smd_device *qsdev,
+				   const void *data,
+				   size_t count)
+{
+	struct wcnss_ctrl *wcnss = dev_get_drvdata(&qsdev->dev);
+	const struct wcnss_download_nv_resp *nvresp;
+	const struct wcnss_version_resp *version;
+	const struct wcnss_msg_hdr *hdr = data;
+
+	switch (hdr->type) {
+	case WCNSS_VERSION_RESP:
+		if (count != sizeof(*version)) {
+			dev_err(wcnss->dev,
+				"invalid size of version response\n");
+			break;
+		}
+
+		version = data;
+		dev_info(wcnss->dev, "WCNSS Version %d.%d %d.%d\n",
+			 version->major, version->minor,
+			 version->version, version->revision);
+
+		schedule_work(&wcnss->download_nv_work);
+		break;
+	case WCNSS_DOWNLOAD_NV_RESP:
+		if (count != sizeof(*nvresp)) {
+			dev_err(wcnss->dev,
+				"invalid size of download response\n");
+			break;
+		}
+
+		nvresp = data;
+		wcnss->ack_status = nvresp->status;
+		complete(&wcnss->ack);
+		break;
+	default:
+		dev_info(wcnss->dev, "unknown message type %d\n", hdr->type);
+		break;
+	}
+
+	return 0;
+}
+
+/**
+ * wcnss_request_version() - send a version request to WCNSS
+ * @wcnss:	wcnss ctrl driver context
+ */
+static int wcnss_request_version(struct wcnss_ctrl *wcnss)
+{
+	struct wcnss_msg_hdr msg;
+
+	msg.type = WCNSS_VERSION_REQ;
+	msg.len = sizeof(msg);
+
+	return qcom_smd_send(wcnss->channel, &msg, sizeof(msg));
+}
+
+/**
+ * wcnss_download_nv() - send nv binary to WCNSS
+ * @work:	work struct to acquire wcnss context
+ */
+static void wcnss_download_nv(struct work_struct *work)
+{
+	struct wcnss_ctrl *wcnss = container_of(work, struct wcnss_ctrl, download_nv_work);
+	struct wcnss_download_nv_req *req;
+	const struct firmware *fw;
+	const void *data;
+	ssize_t left;
+	int ret;
+
+	req = kzalloc(sizeof(*req) + NV_FRAGMENT_SIZE, GFP_KERNEL);
+	if (!req)
+		return;
+
+	ret = request_firmware(&fw, NVBIN_FILE, wcnss->dev);
+	if (ret) {
+		dev_err(wcnss->dev, "Failed to load nv file %s: %d\n",
+			NVBIN_FILE, ret);
+		goto free_req;
+	}
+
+	data = fw->data;
+	left = fw->size;
+
+	req->hdr.type = WCNSS_DOWNLOAD_NV_REQ;
+	req->hdr.len = sizeof(*req) + NV_FRAGMENT_SIZE;
+
+	req->last = 0;
+	req->frag_size = NV_FRAGMENT_SIZE;
+
+	req->seq = 0;
+	do {
+		if (left <= NV_FRAGMENT_SIZE) {
+			req->last = 1;
+			req->frag_size = left;
+			req->hdr.len = sizeof(*req) + left;
+		}
+
+		memcpy(req->fragment, data, req->frag_size);
+
+		ret = qcom_smd_send(wcnss->channel, req, req->hdr.len);
+		if (ret) {
+			dev_err(wcnss->dev, "failed to send smd packet\n");
+			goto release_fw;
+		}
+
+		/* Increment for next fragment */
+		req->seq++;
+
+		data += req->hdr.len;
+		left -= NV_FRAGMENT_SIZE;
+	} while (left > 0);
+
+	ret = wait_for_completion_timeout(&wcnss->ack, WCNSS_REQUEST_TIMEOUT);
+	if (!ret)
+		dev_err(wcnss->dev, "timeout waiting for nv upload ack\n");
+	else if (wcnss->ack_status != 1)
+		dev_err(wcnss->dev, "nv upload response failed err: %d\n",
+			wcnss->ack_status);
+
+release_fw:
+	release_firmware(fw);
+free_req:
+	kfree(req);
+}
+
+static int wcnss_ctrl_probe(struct qcom_smd_device *sdev)
+{
+	struct wcnss_ctrl *wcnss;
+
+	wcnss = devm_kzalloc(&sdev->dev, sizeof(*wcnss), GFP_KERNEL);
+	if (!wcnss)
+		return -ENOMEM;
+
+	wcnss->dev = &sdev->dev;
+	wcnss->channel = sdev->channel;
+
+	init_completion(&wcnss->ack);
+	INIT_WORK(&wcnss->download_nv_work, wcnss_download_nv);
+
+	dev_set_drvdata(&sdev->dev, wcnss);
+
+	return wcnss_request_version(wcnss);
+}
+
+static const struct qcom_smd_id wcnss_ctrl_smd_match[] = {
+	{ .name = "WCNSS_CTRL" },
+	{}
+};
+
+static struct qcom_smd_driver wcnss_ctrl_driver = {
+	.probe = wcnss_ctrl_probe,
+	.callback = wcnss_ctrl_smd_callback,
+	.smd_match_table = wcnss_ctrl_smd_match,
+	.driver  = {
+		.name  = "qcom_wcnss_ctrl",
+		.owner = THIS_MODULE,
+	},
+};
+
+module_qcom_smd_driver(wcnss_ctrl_driver);
+
+MODULE_DESCRIPTION("Qualcomm WCNSS control driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
new file mode 100644
index 000000000000..d0c3c3e085e3
--- /dev/null
+++ b/drivers/soc/tegra/Kconfig
@@ -0,0 +1,83 @@
+if ARCH_TEGRA
+
+# 32-bit ARM SoCs
+if ARM
+
+config ARCH_TEGRA_2x_SOC
+	bool "Enable support for Tegra20 family"
+	select ARCH_NEEDS_CPU_IDLE_COUPLED if SMP
+	select ARM_ERRATA_720789
+	select ARM_ERRATA_754327 if SMP
+	select ARM_ERRATA_764369 if SMP
+	select PINCTRL_TEGRA20
+	select PL310_ERRATA_727915 if CACHE_L2X0
+	select PL310_ERRATA_769419 if CACHE_L2X0
+	select TEGRA_TIMER
+	help
+	  Support for NVIDIA Tegra AP20 and T20 processors, based on the
+	  ARM CortexA9MP CPU and the ARM PL310 L2 cache controller
+
+config ARCH_TEGRA_3x_SOC
+	bool "Enable support for Tegra30 family"
+	select ARM_ERRATA_754322
+	select ARM_ERRATA_764369 if SMP
+	select PINCTRL_TEGRA30
+	select PL310_ERRATA_769419 if CACHE_L2X0
+	select TEGRA_TIMER
+	help
+	  Support for NVIDIA Tegra T30 processor family, based on the
+	  ARM CortexA9MP CPU and the ARM PL310 L2 cache controller
+
+config ARCH_TEGRA_114_SOC
+	bool "Enable support for Tegra114 family"
+	select ARM_ERRATA_798181 if SMP
+	select ARM_L1_CACHE_SHIFT_6
+	select HAVE_ARM_ARCH_TIMER
+	select PINCTRL_TEGRA114
+	select TEGRA_TIMER
+	help
+	  Support for NVIDIA Tegra T114 processor family, based on the
+	  ARM CortexA15MP CPU
+
+config ARCH_TEGRA_124_SOC
+	bool "Enable support for Tegra124 family"
+	select ARM_L1_CACHE_SHIFT_6
+	select HAVE_ARM_ARCH_TIMER
+	select PINCTRL_TEGRA124
+	select TEGRA_TIMER
+	help
+	  Support for NVIDIA Tegra T124 processor family, based on the
+	  ARM CortexA15MP CPU
+
+endif
+
+# 64-bit ARM SoCs
+if ARM64
+
+config ARCH_TEGRA_132_SOC
+	bool "NVIDIA Tegra132 SoC"
+	select PINCTRL_TEGRA124
+	help
+	  Enable support for NVIDIA Tegra132 SoC, based on the Denver
+	  ARMv8 CPU.  The Tegra132 SoC is similar to the Tegra124 SoC,
+	  but contains an NVIDIA Denver CPU complex in place of
+	  Tegra124's "4+1" Cortex-A15 CPU complex.
+
+config ARCH_TEGRA_210_SOC
+	bool "NVIDIA Tegra210 SoC"
+	select PINCTRL_TEGRA210
+	help
+	  Enable support for the NVIDIA Tegra210 SoC. Also known as Tegra X1,
+	  the Tegra210 has four Cortex-A57 cores paired with four Cortex-A53
+	  cores in a switched configuration. It features a GPU of the Maxwell
+	  architecture with support for DX11, SM4, OpenGL 4.5, OpenGL ES 3.1
+	  and providing 256 CUDA cores. It supports hardware-accelerated en-
+	  and decoding of various video standards including H.265, H.264 and
+	  VP8 at 4K resolution and up to 60 fps.
+
+	  Besides the multimedia features it also comes with a variety of I/O
+	  controllers, such as GPIO, I2C, SPI, SDHCI, PCIe, SATA and XHCI, to
+	  name only a few.
+
+endif
+endif
diff --git a/drivers/soc/ti/Kconfig b/drivers/soc/ti/Kconfig
index 7266b2165183..3557c5e32a93 100644
--- a/drivers/soc/ti/Kconfig
+++ b/drivers/soc/ti/Kconfig
@@ -28,4 +28,14 @@ config KEYSTONE_NAVIGATOR_DMA
 
 	  If unsure, say N.
 
+config WKUP_M3_IPC
+	tristate "TI AMx3 Wkup-M3 IPC Driver"
+	depends on WKUP_M3_RPROC
+	depends on OMAP2PLUS_MBOX
+	help
+	  TI AM33XX and AM43XX have a Cortex M3, the Wakeup M3, to handle
+	  low power transitions. This IPC driver provides the necessary API
+	  to communicate and use the Wakeup M3 for PM features like suspend
+	  resume and boots it using wkup_m3_rproc driver.
+
 endif # SOC_TI
diff --git a/drivers/soc/ti/Makefile b/drivers/soc/ti/Makefile
index 135bdad7a6de..48ff3a79634f 100644
--- a/drivers/soc/ti/Makefile
+++ b/drivers/soc/ti/Makefile
@@ -4,3 +4,4 @@
 obj-$(CONFIG_KEYSTONE_NAVIGATOR_QMSS)	+= knav_qmss.o
 knav_qmss-y := knav_qmss_queue.o knav_qmss_acc.o
 obj-$(CONFIG_KEYSTONE_NAVIGATOR_DMA)	+= knav_dma.o
+obj-$(CONFIG_WKUP_M3_IPC)		+= wkup_m3_ipc.o
diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c
index bc1b80ec6afe..1a7b5caa127b 100644
--- a/drivers/soc/ti/knav_dma.c
+++ b/drivers/soc/ti/knav_dma.c
@@ -389,7 +389,7 @@ static int of_channel_match_helper(struct device_node *np, const char *name,
 	*dma_instance = dma_node->name;
 	index = of_property_match_string(np, "ti,navigator-dma-names", name);
 	if (index < 0) {
-		dev_err(kdev->dev, "No 'ti,navigator-dma-names' propery\n");
+		dev_err(kdev->dev, "No 'ti,navigator-dma-names' property\n");
 		return -ENODEV;
 	}
 
diff --git a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c
new file mode 100644
index 000000000000..8823cc81ae45
--- /dev/null
+++ b/drivers/soc/ti/wkup_m3_ipc.c
@@ -0,0 +1,508 @@
+/*
+ * AMx3 Wkup M3 IPC driver
+ *
+ * Copyright (C) 2015 Texas Instruments, Inc.
+ *
+ * Dave Gerlach <d-gerlach@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/omap-mailbox.h>
+#include <linux/platform_device.h>
+#include <linux/remoteproc.h>
+#include <linux/suspend.h>
+#include <linux/wkup_m3_ipc.h>
+
+#define AM33XX_CTRL_IPC_REG_COUNT	0x8
+#define AM33XX_CTRL_IPC_REG_OFFSET(m)	(0x4 + 4 * (m))
+
+/* AM33XX M3_TXEV_EOI register */
+#define AM33XX_CONTROL_M3_TXEV_EOI	0x00
+
+#define AM33XX_M3_TXEV_ACK		(0x1 << 0)
+#define AM33XX_M3_TXEV_ENABLE		(0x0 << 0)
+
+#define IPC_CMD_DS0			0x4
+#define IPC_CMD_STANDBY			0xc
+#define IPC_CMD_IDLE			0x10
+#define IPC_CMD_RESET			0xe
+#define DS_IPC_DEFAULT			0xffffffff
+#define M3_VERSION_UNKNOWN		0x0000ffff
+#define M3_BASELINE_VERSION		0x191
+#define M3_STATUS_RESP_MASK		(0xffff << 16)
+#define M3_FW_VERSION_MASK		0xffff
+
+#define M3_STATE_UNKNOWN		0
+#define M3_STATE_RESET			1
+#define M3_STATE_INITED			2
+#define M3_STATE_MSG_FOR_LP		3
+#define M3_STATE_MSG_FOR_RESET		4
+
+static struct wkup_m3_ipc *m3_ipc_state;
+
+static void am33xx_txev_eoi(struct wkup_m3_ipc *m3_ipc)
+{
+	writel(AM33XX_M3_TXEV_ACK,
+	       m3_ipc->ipc_mem_base + AM33XX_CONTROL_M3_TXEV_EOI);
+}
+
+static void am33xx_txev_enable(struct wkup_m3_ipc *m3_ipc)
+{
+	writel(AM33XX_M3_TXEV_ENABLE,
+	       m3_ipc->ipc_mem_base + AM33XX_CONTROL_M3_TXEV_EOI);
+}
+
+static void wkup_m3_ctrl_ipc_write(struct wkup_m3_ipc *m3_ipc,
+				   u32 val, int ipc_reg_num)
+{
+	if (WARN(ipc_reg_num < 0 || ipc_reg_num > AM33XX_CTRL_IPC_REG_COUNT,
+		 "ipc register operation out of range"))
+		return;
+
+	writel(val, m3_ipc->ipc_mem_base +
+	       AM33XX_CTRL_IPC_REG_OFFSET(ipc_reg_num));
+}
+
+static unsigned int wkup_m3_ctrl_ipc_read(struct wkup_m3_ipc *m3_ipc,
+					  int ipc_reg_num)
+{
+	if (WARN(ipc_reg_num < 0 || ipc_reg_num > AM33XX_CTRL_IPC_REG_COUNT,
+		 "ipc register operation out of range"))
+		return 0;
+
+	return readl(m3_ipc->ipc_mem_base +
+		     AM33XX_CTRL_IPC_REG_OFFSET(ipc_reg_num));
+}
+
+static int wkup_m3_fw_version_read(struct wkup_m3_ipc *m3_ipc)
+{
+	int val;
+
+	val = wkup_m3_ctrl_ipc_read(m3_ipc, 2);
+
+	return val & M3_FW_VERSION_MASK;
+}
+
+static irqreturn_t wkup_m3_txev_handler(int irq, void *ipc_data)
+{
+	struct wkup_m3_ipc *m3_ipc = ipc_data;
+	struct device *dev = m3_ipc->dev;
+	int ver = 0;
+
+	am33xx_txev_eoi(m3_ipc);
+
+	switch (m3_ipc->state) {
+	case M3_STATE_RESET:
+		ver = wkup_m3_fw_version_read(m3_ipc);
+
+		if (ver == M3_VERSION_UNKNOWN ||
+		    ver < M3_BASELINE_VERSION) {
+			dev_warn(dev, "CM3 Firmware Version %x not supported\n",
+				 ver);
+		} else {
+			dev_info(dev, "CM3 Firmware Version = 0x%x\n", ver);
+		}
+
+		m3_ipc->state = M3_STATE_INITED;
+		complete(&m3_ipc->sync_complete);
+		break;
+	case M3_STATE_MSG_FOR_RESET:
+		m3_ipc->state = M3_STATE_INITED;
+		complete(&m3_ipc->sync_complete);
+		break;
+	case M3_STATE_MSG_FOR_LP:
+		complete(&m3_ipc->sync_complete);
+		break;
+	case M3_STATE_UNKNOWN:
+		dev_warn(dev, "Unknown CM3 State\n");
+	}
+
+	am33xx_txev_enable(m3_ipc);
+
+	return IRQ_HANDLED;
+}
+
+static int wkup_m3_ping(struct wkup_m3_ipc *m3_ipc)
+{
+	struct device *dev = m3_ipc->dev;
+	mbox_msg_t dummy_msg = 0;
+	int ret;
+
+	if (!m3_ipc->mbox) {
+		dev_err(dev,
+			"No IPC channel to communicate with wkup_m3!\n");
+		return -EIO;
+	}
+
+	/*
+	 * Write a dummy message to the mailbox in order to trigger the RX
+	 * interrupt to alert the M3 that data is available in the IPC
+	 * registers. We must enable the IRQ here and disable it after in
+	 * the RX callback to avoid multiple interrupts being received
+	 * by the CM3.
+	 */
+	ret = mbox_send_message(m3_ipc->mbox, &dummy_msg);
+	if (ret < 0) {
+		dev_err(dev, "%s: mbox_send_message() failed: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	ret = wait_for_completion_timeout(&m3_ipc->sync_complete,
+					  msecs_to_jiffies(500));
+	if (!ret) {
+		dev_err(dev, "MPU<->CM3 sync failure\n");
+		m3_ipc->state = M3_STATE_UNKNOWN;
+		return -EIO;
+	}
+
+	mbox_client_txdone(m3_ipc->mbox, 0);
+	return 0;
+}
+
+static int wkup_m3_ping_noirq(struct wkup_m3_ipc *m3_ipc)
+{
+	struct device *dev = m3_ipc->dev;
+	mbox_msg_t dummy_msg = 0;
+	int ret;
+
+	if (!m3_ipc->mbox) {
+		dev_err(dev,
+			"No IPC channel to communicate with wkup_m3!\n");
+		return -EIO;
+	}
+
+	ret = mbox_send_message(m3_ipc->mbox, &dummy_msg);
+	if (ret < 0) {
+		dev_err(dev, "%s: mbox_send_message() failed: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	mbox_client_txdone(m3_ipc->mbox, 0);
+	return 0;
+}
+
+static int wkup_m3_is_available(struct wkup_m3_ipc *m3_ipc)
+{
+	return ((m3_ipc->state != M3_STATE_RESET) &&
+		(m3_ipc->state != M3_STATE_UNKNOWN));
+}
+
+/* Public functions */
+/**
+ * wkup_m3_set_mem_type - Pass wkup_m3 which type of memory is in use
+ * @mem_type: memory type value read directly from emif
+ *
+ * wkup_m3 must know what memory type is in use to properly suspend
+ * and resume.
+ */
+static void wkup_m3_set_mem_type(struct wkup_m3_ipc *m3_ipc, int mem_type)
+{
+	m3_ipc->mem_type = mem_type;
+}
+
+/**
+ * wkup_m3_set_resume_address - Pass wkup_m3 resume address
+ * @addr: Physical address from which resume code should execute
+ */
+static void wkup_m3_set_resume_address(struct wkup_m3_ipc *m3_ipc, void *addr)
+{
+	m3_ipc->resume_addr = (unsigned long)addr;
+}
+
+/**
+ * wkup_m3_request_pm_status - Retrieve wkup_m3 status code after suspend
+ *
+ * Returns code representing the status of a low power mode transition.
+ *	0 - Successful transition
+ *	1 - Failure to transition to low power state
+ */
+static int wkup_m3_request_pm_status(struct wkup_m3_ipc *m3_ipc)
+{
+	unsigned int i;
+	int val;
+
+	val = wkup_m3_ctrl_ipc_read(m3_ipc, 1);
+
+	i = M3_STATUS_RESP_MASK & val;
+	i >>= __ffs(M3_STATUS_RESP_MASK);
+
+	return i;
+}
+
+/**
+ * wkup_m3_prepare_low_power - Request preparation for transition to
+ *			       low power state
+ * @state: A kernel suspend state to enter, either MEM or STANDBY
+ *
+ * Returns 0 if preparation was successful, otherwise returns error code
+ */
+static int wkup_m3_prepare_low_power(struct wkup_m3_ipc *m3_ipc, int state)
+{
+	struct device *dev = m3_ipc->dev;
+	int m3_power_state;
+	int ret = 0;
+
+	if (!wkup_m3_is_available(m3_ipc))
+		return -ENODEV;
+
+	switch (state) {
+	case WKUP_M3_DEEPSLEEP:
+		m3_power_state = IPC_CMD_DS0;
+		break;
+	case WKUP_M3_STANDBY:
+		m3_power_state = IPC_CMD_STANDBY;
+		break;
+	case WKUP_M3_IDLE:
+		m3_power_state = IPC_CMD_IDLE;
+		break;
+	default:
+		return 1;
+	}
+
+	/* Program each required IPC register then write defaults to others */
+	wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->resume_addr, 0);
+	wkup_m3_ctrl_ipc_write(m3_ipc, m3_power_state, 1);
+	wkup_m3_ctrl_ipc_write(m3_ipc, m3_ipc->mem_type, 4);
+
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 2);
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 3);
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 5);
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 6);
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 7);
+
+	m3_ipc->state = M3_STATE_MSG_FOR_LP;
+
+	if (state == WKUP_M3_IDLE)
+		ret = wkup_m3_ping_noirq(m3_ipc);
+	else
+		ret = wkup_m3_ping(m3_ipc);
+
+	if (ret) {
+		dev_err(dev, "Unable to ping CM3\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * wkup_m3_finish_low_power - Return m3 to reset state
+ *
+ * Returns 0 if reset was successful, otherwise returns error code
+ */
+static int wkup_m3_finish_low_power(struct wkup_m3_ipc *m3_ipc)
+{
+	struct device *dev = m3_ipc->dev;
+	int ret = 0;
+
+	if (!wkup_m3_is_available(m3_ipc))
+		return -ENODEV;
+
+	wkup_m3_ctrl_ipc_write(m3_ipc, IPC_CMD_RESET, 1);
+	wkup_m3_ctrl_ipc_write(m3_ipc, DS_IPC_DEFAULT, 2);
+
+	m3_ipc->state = M3_STATE_MSG_FOR_RESET;
+
+	ret = wkup_m3_ping(m3_ipc);
+	if (ret) {
+		dev_err(dev, "Unable to ping CM3\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct wkup_m3_ipc_ops ipc_ops = {
+	.set_mem_type = wkup_m3_set_mem_type,
+	.set_resume_address = wkup_m3_set_resume_address,
+	.prepare_low_power = wkup_m3_prepare_low_power,
+	.finish_low_power = wkup_m3_finish_low_power,
+	.request_pm_status = wkup_m3_request_pm_status,
+};
+
+/**
+ * wkup_m3_ipc_get - Return handle to wkup_m3_ipc
+ *
+ * Returns NULL if the wkup_m3 is not yet available, otherwise returns
+ * pointer to wkup_m3_ipc struct.
+ */
+struct wkup_m3_ipc *wkup_m3_ipc_get(void)
+{
+	if (m3_ipc_state)
+		get_device(m3_ipc_state->dev);
+	else
+		return NULL;
+
+	return m3_ipc_state;
+}
+EXPORT_SYMBOL_GPL(wkup_m3_ipc_get);
+
+/**
+ * wkup_m3_ipc_put - Free handle to wkup_m3_ipc returned from wkup_m3_ipc_get
+ * @m3_ipc: A pointer to wkup_m3_ipc struct returned by wkup_m3_ipc_get
+ */
+void wkup_m3_ipc_put(struct wkup_m3_ipc *m3_ipc)
+{
+	if (m3_ipc_state)
+		put_device(m3_ipc_state->dev);
+}
+EXPORT_SYMBOL_GPL(wkup_m3_ipc_put);
+
+static void wkup_m3_rproc_boot_thread(struct wkup_m3_ipc *m3_ipc)
+{
+	struct device *dev = m3_ipc->dev;
+	int ret;
+
+	wait_for_completion(&m3_ipc->rproc->firmware_loading_complete);
+
+	init_completion(&m3_ipc->sync_complete);
+
+	ret = rproc_boot(m3_ipc->rproc);
+	if (ret)
+		dev_err(dev, "rproc_boot failed\n");
+
+	do_exit(0);
+}
+
+static int wkup_m3_ipc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int irq, ret;
+	phandle rproc_phandle;
+	struct rproc *m3_rproc;
+	struct resource *res;
+	struct task_struct *task;
+	struct wkup_m3_ipc *m3_ipc;
+
+	m3_ipc = devm_kzalloc(dev, sizeof(*m3_ipc), GFP_KERNEL);
+	if (!m3_ipc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	m3_ipc->ipc_mem_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(m3_ipc->ipc_mem_base)) {
+		dev_err(dev, "could not ioremap ipc_mem\n");
+		return PTR_ERR(m3_ipc->ipc_mem_base);
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (!irq) {
+		dev_err(&pdev->dev, "no irq resource\n");
+		return -ENXIO;
+	}
+
+	ret = devm_request_irq(dev, irq, wkup_m3_txev_handler,
+			       0, "wkup_m3_txev", m3_ipc);
+	if (ret) {
+		dev_err(dev, "request_irq failed\n");
+		return ret;
+	}
+
+	m3_ipc->mbox_client.dev = dev;
+	m3_ipc->mbox_client.tx_done = NULL;
+	m3_ipc->mbox_client.tx_prepare = NULL;
+	m3_ipc->mbox_client.rx_callback = NULL;
+	m3_ipc->mbox_client.tx_block = false;
+	m3_ipc->mbox_client.knows_txdone = false;
+
+	m3_ipc->mbox = mbox_request_channel(&m3_ipc->mbox_client, 0);
+
+	if (IS_ERR(m3_ipc->mbox)) {
+		dev_err(dev, "IPC Request for A8->M3 Channel failed! %ld\n",
+			PTR_ERR(m3_ipc->mbox));
+		return PTR_ERR(m3_ipc->mbox);
+	}
+
+	if (of_property_read_u32(dev->of_node, "ti,rproc", &rproc_phandle)) {
+		dev_err(&pdev->dev, "could not get rproc phandle\n");
+		ret = -ENODEV;
+		goto err_free_mbox;
+	}
+
+	m3_rproc = rproc_get_by_phandle(rproc_phandle);
+	if (!m3_rproc) {
+		dev_err(&pdev->dev, "could not get rproc handle\n");
+		ret = -EPROBE_DEFER;
+		goto err_free_mbox;
+	}
+
+	m3_ipc->rproc = m3_rproc;
+	m3_ipc->dev = dev;
+	m3_ipc->state = M3_STATE_RESET;
+
+	m3_ipc->ops = &ipc_ops;
+
+	/*
+	 * Wait for firmware loading completion in a thread so we
+	 * can boot the wkup_m3 as soon as it's ready without holding
+	 * up kernel boot
+	 */
+	task = kthread_run((void *)wkup_m3_rproc_boot_thread, m3_ipc,
+			   "wkup_m3_rproc_loader");
+
+	if (IS_ERR(task)) {
+		dev_err(dev, "can't create rproc_boot thread\n");
+		goto err_put_rproc;
+	}
+
+	m3_ipc_state = m3_ipc;
+
+	return 0;
+
+err_put_rproc:
+	rproc_put(m3_rproc);
+err_free_mbox:
+	mbox_free_channel(m3_ipc->mbox);
+	return ret;
+}
+
+static int wkup_m3_ipc_remove(struct platform_device *pdev)
+{
+	mbox_free_channel(m3_ipc_state->mbox);
+
+	rproc_shutdown(m3_ipc_state->rproc);
+	rproc_put(m3_ipc_state->rproc);
+
+	m3_ipc_state = NULL;
+
+	return 0;
+}
+
+static const struct of_device_id wkup_m3_ipc_of_match[] = {
+	{ .compatible = "ti,am3352-wkup-m3-ipc", },
+	{ .compatible = "ti,am4372-wkup-m3-ipc", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, wkup_m3_ipc_of_match);
+
+static struct platform_driver wkup_m3_ipc_driver = {
+	.probe = wkup_m3_ipc_probe,
+	.remove = wkup_m3_ipc_remove,
+	.driver = {
+		.name = "wkup_m3_ipc",
+		.of_match_table = wkup_m3_ipc_of_match,
+	},
+};
+
+module_platform_driver(wkup_m3_ipc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("wkup m3 remote processor ipc driver");
+MODULE_AUTHOR("Dave Gerlach <d-gerlach@ti.com>");
diff --git a/drivers/soc/versatile/soc-realview.c b/drivers/soc/versatile/soc-realview.c
index e642c4540dda..c337764de867 100644
--- a/drivers/soc/versatile/soc-realview.c
+++ b/drivers/soc/versatile/soc-realview.c
@@ -36,6 +36,8 @@ static const char *realview_board_str(u32 id)
 	switch ((id >> 16) & 0xfff) {
 	case 0x0147:
 		return "HBI-0147";
+	case 0x0159:
+		return "HBI-0159";
 	default:
 		return "Unknown";
 	}
@@ -44,6 +46,8 @@ static const char *realview_board_str(u32 id)
 static const char *realview_arch_str(u32 id)
 {
 	switch ((id >> 8) & 0xf) {
+	case 0x04:
+		return "AHB";
 	case 0x05:
 		return "Multi-layer AXI";
 	default:
diff --git a/drivers/spi/spi-fsl-cpm.c b/drivers/spi/spi-fsl-cpm.c
index 896add8cfd3b..8f7b26ec181e 100644
--- a/drivers/spi/spi-fsl-cpm.c
+++ b/drivers/spi/spi-fsl-cpm.c
@@ -16,7 +16,7 @@
  * option) any later version.
  */
 #include <asm/cpm.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/qe.h>
 #include <linux/dma-mapping.h>
 #include <linux/fsl_devices.h>
 #include <linux/kernel.h>
diff --git a/drivers/staging/board/armadillo800eva.c b/drivers/staging/board/armadillo800eva.c
index 9c41652ee908..912c96b0536d 100644
--- a/drivers/staging/board/armadillo800eva.c
+++ b/drivers/staging/board/armadillo800eva.c
@@ -97,7 +97,7 @@ static const struct board_staging_dev armadillo800eva_devices[] __initconst = {
 
 static void __init armadillo800eva_init(void)
 {
-	board_staging_gic_setup_xlate("arm,cortex-a9-gic", 32);
+	board_staging_gic_setup_xlate("arm,pl390", 32);
 	board_staging_register_devices(armadillo800eva_devices,
 				       ARRAY_SIZE(armadillo800eva_devices));
 }
diff --git a/drivers/staging/board/kzm9d.c b/drivers/staging/board/kzm9d.c
index 8d1eb09bc66e..05a6d434d307 100644
--- a/drivers/staging/board/kzm9d.c
+++ b/drivers/staging/board/kzm9d.c
@@ -11,7 +11,7 @@ static struct resource usbs1_res[] __initdata = {
 
 static void __init kzm9d_init(void)
 {
-	board_staging_gic_setup_xlate("arm,cortex-a9-gic", 32);
+	board_staging_gic_setup_xlate("arm,pl390", 32);
 
 	if (!board_staging_dt_node_available(usbs1_res,
 					     ARRAY_SIZE(usbs1_res))) {
diff --git a/drivers/staging/iio/adc/Kconfig b/drivers/staging/iio/adc/Kconfig
index 94ae4232ee77..58d4517e1836 100644
--- a/drivers/staging/iio/adc/Kconfig
+++ b/drivers/staging/iio/adc/Kconfig
@@ -23,7 +23,7 @@ config AD7606_IFACE_PARALLEL
 	  ADC driver.
 
 	  To compile this driver as a module, choose M here: the
-	  module will be called ad7606_iface_parallel.
+	  module will be called ad7606_parallel.
 
 config AD7606_IFACE_SPI
 	tristate "spi interface support"
@@ -34,7 +34,7 @@ config AD7606_IFACE_SPI
 	  ADC driver.
 
 	  To compile this driver as a module, choose M here: the
-	  module will be called ad7606_iface_spi.
+	  module will be called ad7606_spi.
 
 config AD7780
 	tristate "Analog Devices AD7780 and similar ADCs driver"
diff --git a/drivers/staging/iio/adc/Makefile b/drivers/staging/iio/adc/Makefile
index 1c4277dbd318..0c87ce3530f8 100644
--- a/drivers/staging/iio/adc/Makefile
+++ b/drivers/staging/iio/adc/Makefile
@@ -2,10 +2,9 @@
 # Makefile for industrial I/O ADC drivers
 #
 
-ad7606-y := ad7606_core.o
-ad7606-$(CONFIG_IIO_BUFFER) += ad7606_ring.o
-ad7606-$(CONFIG_AD7606_IFACE_PARALLEL) += ad7606_par.o
-ad7606-$(CONFIG_AD7606_IFACE_SPI) += ad7606_spi.o
+ad7606-y := ad7606_core.o ad7606_ring.o
+obj-$(CONFIG_AD7606_IFACE_PARALLEL) += ad7606_par.o
+obj-$(CONFIG_AD7606_IFACE_SPI) += ad7606_spi.o
 obj-$(CONFIG_AD7606) += ad7606.o
 
 obj-$(CONFIG_AD7780) += ad7780.o
diff --git a/drivers/staging/iio/adc/ad7606_core.c b/drivers/staging/iio/adc/ad7606_core.c
index 5796ed2409d0..2c9d8b7de9f5 100644
--- a/drivers/staging/iio/adc/ad7606_core.c
+++ b/drivers/staging/iio/adc/ad7606_core.c
@@ -559,6 +559,7 @@ error_disable_reg:
 		regulator_disable(st->reg);
 	return ERR_PTR(ret);
 }
+EXPORT_SYMBOL_GPL(ad7606_probe);
 
 int ad7606_remove(struct iio_dev *indio_dev, int irq)
 {
@@ -575,6 +576,7 @@ int ad7606_remove(struct iio_dev *indio_dev, int irq)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(ad7606_remove);
 
 void ad7606_suspend(struct iio_dev *indio_dev)
 {
@@ -586,6 +588,7 @@ void ad7606_suspend(struct iio_dev *indio_dev)
 		gpio_set_value(st->pdata->gpio_stby, 0);
 	}
 }
+EXPORT_SYMBOL_GPL(ad7606_suspend);
 
 void ad7606_resume(struct iio_dev *indio_dev)
 {
@@ -600,6 +603,7 @@ void ad7606_resume(struct iio_dev *indio_dev)
 		ad7606_reset(st);
 	}
 }
+EXPORT_SYMBOL_GPL(ad7606_resume);
 
 MODULE_AUTHOR("Michael Hennerich <hennerich@blackfin.uclinux.org>");
 MODULE_DESCRIPTION("Analog Devices AD7606 ADC");
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
index d6273e143324..a80d993b882e 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
@@ -151,16 +151,12 @@ do {									    \
 
 #define LIBCFS_FREE(ptr, size)					  \
 do {								    \
-	int s = (size);						 \
 	if (unlikely((ptr) == NULL)) {				  \
 		CERROR("LIBCFS: free NULL '" #ptr "' (%d bytes) at "    \
-		       "%s:%d\n", s, __FILE__, __LINE__);	       \
+		       "%s:%d\n", (int)(size), __FILE__, __LINE__);	\
 		break;						  \
 	}							       \
-	if (unlikely(s > LIBCFS_VMALLOC_SIZE))			  \
-		vfree(ptr);				    \
-	else							    \
-		kfree(ptr);					  \
+	kvfree(ptr);					  \
 } while (0)
 
 /******************************************************************************/
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 72af486b65df..cb74ae731b95 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2070,32 +2070,13 @@ static int kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 
 static int kiblnd_hdev_get_attr(kib_hca_dev_t *hdev)
 {
-	struct ib_device_attr *attr;
-	int rc;
-
 	/* It's safe to assume a HCA can handle a page size
 	 * matching that of the native system */
 	hdev->ibh_page_shift = PAGE_SHIFT;
 	hdev->ibh_page_size  = 1 << PAGE_SHIFT;
 	hdev->ibh_page_mask  = ~((__u64)hdev->ibh_page_size - 1);
 
-	LIBCFS_ALLOC(attr, sizeof(*attr));
-	if (attr == NULL) {
-		CERROR("Out of memory\n");
-		return -ENOMEM;
-	}
-
-	rc = ib_query_device(hdev->ibh_ibdev, attr);
-	if (rc == 0)
-		hdev->ibh_mr_size = attr->max_mr_size;
-
-	LIBCFS_FREE(attr, sizeof(*attr));
-
-	if (rc != 0) {
-		CERROR("Failed to query IB device: %d\n", rc);
-		return rc;
-	}
-
+	hdev->ibh_mr_size = hdev->ibh_ibdev->attrs.max_mr_size;
 	if (hdev->ibh_mr_size == ~0ULL) {
 		hdev->ibh_mr_shift = 64;
 		return 0;
diff --git a/drivers/staging/lustre/lustre/llite/dir.c b/drivers/staging/lustre/lustre/llite/dir.c
index 7b355319079c..8982f7d1b374 100644
--- a/drivers/staging/lustre/lustre/llite/dir.c
+++ b/drivers/staging/lustre/lustre/llite/dir.c
@@ -1858,7 +1858,7 @@ static loff_t ll_dir_seek(struct file *file, loff_t offset, int origin)
 	int api32 = ll_need_32bit_api(sbi);
 	loff_t ret = -EINVAL;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	switch (origin) {
 	case SEEK_SET:
 		break;
@@ -1896,7 +1896,7 @@ static loff_t ll_dir_seek(struct file *file, loff_t offset, int origin)
 	goto out;
 
 out:
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 	return ret;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/file.c b/drivers/staging/lustre/lustre/llite/file.c
index c92d58b770ec..39e2ffd5f97f 100644
--- a/drivers/staging/lustre/lustre/llite/file.c
+++ b/drivers/staging/lustre/lustre/llite/file.c
@@ -2082,17 +2082,17 @@ putgl:
 	/* update time if requested */
 	rc = 0;
 	if (llss->ia2.ia_valid != 0) {
-		mutex_lock(&llss->inode1->i_mutex);
+		inode_lock(llss->inode1);
 		rc = ll_setattr(file1->f_path.dentry, &llss->ia2);
-		mutex_unlock(&llss->inode1->i_mutex);
+		inode_unlock(llss->inode1);
 	}
 
 	if (llss->ia1.ia_valid != 0) {
 		int rc1;
 
-		mutex_lock(&llss->inode2->i_mutex);
+		inode_lock(llss->inode2);
 		rc1 = ll_setattr(file2->f_path.dentry, &llss->ia1);
-		mutex_unlock(&llss->inode2->i_mutex);
+		inode_unlock(llss->inode2);
 		if (rc == 0)
 			rc = rc1;
 	}
@@ -2179,13 +2179,13 @@ static int ll_hsm_import(struct inode *inode, struct file *file,
 			 ATTR_MTIME | ATTR_MTIME_SET |
 			 ATTR_ATIME | ATTR_ATIME_SET;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 
 	rc = ll_setattr_raw(file->f_path.dentry, attr, true);
 	if (rc == -ENODATA)
 		rc = 0;
 
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 
 	kfree(attr);
 free_hss:
@@ -2609,7 +2609,7 @@ int ll_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_FSYNC, 1);
 
 	rc = filemap_write_and_wait_range(inode->i_mapping, start, end);
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 
 	/* catch async errors that were recorded back when async writeback
 	 * failed for pages in this mapping. */
@@ -2641,7 +2641,7 @@ int ll_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 			fd->fd_write_failed = false;
 	}
 
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 	return rc;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_internal.h b/drivers/staging/lustre/lustre/llite/llite_internal.h
index ee8a1d67d191..845e992ca5fc 100644
--- a/drivers/staging/lustre/lustre/llite/llite_internal.h
+++ b/drivers/staging/lustre/lustre/llite/llite_internal.h
@@ -631,8 +631,6 @@ struct ll_file_data {
 
 struct lov_stripe_md;
 
-extern spinlock_t inode_lock;
-
 extern struct dentry *llite_root;
 extern struct kset *llite_kset;
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_lib.c b/drivers/staging/lustre/lustre/llite/llite_lib.c
index 1db93af62bad..b2fc5b3786ee 100644
--- a/drivers/staging/lustre/lustre/llite/llite_lib.c
+++ b/drivers/staging/lustre/lustre/llite/llite_lib.c
@@ -1277,7 +1277,7 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 		return -ENOMEM;
 
 	if (!S_ISDIR(inode->i_mode))
-		mutex_unlock(&inode->i_mutex);
+		inode_unlock(inode);
 
 	memcpy(&op_data->op_attr, attr, sizeof(*attr));
 
@@ -1358,7 +1358,7 @@ out:
 	ll_finish_md_op_data(op_data);
 
 	if (!S_ISDIR(inode->i_mode)) {
-		mutex_lock(&inode->i_mutex);
+		inode_lock(inode);
 		if ((attr->ia_valid & ATTR_SIZE) && !hsm_import)
 			inode_dio_wait(inode);
 	}
diff --git a/drivers/staging/lustre/lustre/llite/llite_nfs.c b/drivers/staging/lustre/lustre/llite/llite_nfs.c
index e578a1130ad1..18aab25f9cd9 100644
--- a/drivers/staging/lustre/lustre/llite/llite_nfs.c
+++ b/drivers/staging/lustre/lustre/llite/llite_nfs.c
@@ -245,9 +245,9 @@ static int ll_get_name(struct dentry *dentry, char *name,
 		goto out;
 	}
 
-	mutex_lock(&dir->i_mutex);
+	inode_lock(dir);
 	rc = ll_dir_read(dir, &lgd.ctx);
-	mutex_unlock(&dir->i_mutex);
+	inode_unlock(dir);
 	if (!rc && !lgd.lgd_found)
 		rc = -ENOENT;
 out:
diff --git a/drivers/staging/lustre/lustre/llite/lloop.c b/drivers/staging/lustre/lustre/llite/lloop.c
index 420d39123877..871924b3f2e7 100644
--- a/drivers/staging/lustre/lustre/llite/lloop.c
+++ b/drivers/staging/lustre/lustre/llite/lloop.c
@@ -257,9 +257,9 @@ static int do_bio_lustrebacked(struct lloop_device *lo, struct bio *head)
 	 *    be asked to write less pages once, this purely depends on
 	 *    implementation. Anyway, we should be careful to avoid deadlocking.
 	 */
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	bytes = ll_direct_rw_pages(env, io, rw, inode, pvec);
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 	cl_io_fini(env, io);
 	return (bytes == pvec->ldp_size) ? 0 : (int)bytes;
 }
diff --git a/drivers/staging/lustre/lustre/llite/rw.c b/drivers/staging/lustre/lustre/llite/rw.c
index 95cdb0c58b04..f355474967d6 100644
--- a/drivers/staging/lustre/lustre/llite/rw.c
+++ b/drivers/staging/lustre/lustre/llite/rw.c
@@ -115,8 +115,8 @@ static struct ll_cl_context *ll_cl_init(struct file *file,
 		struct inode *inode = vmpage->mapping->host;
 		loff_t pos;
 
-		if (mutex_trylock(&inode->i_mutex)) {
-			mutex_unlock(&(inode)->i_mutex);
+		if (inode_trylock(inode)) {
+			inode_unlock((inode));
 
 			/* this is too bad. Someone is trying to write the
 			 * page w/o holding inode mutex. This means we can
diff --git a/drivers/staging/lustre/lustre/llite/rw26.c b/drivers/staging/lustre/lustre/llite/rw26.c
index 39fa13b74cbd..711fda93a58d 100644
--- a/drivers/staging/lustre/lustre/llite/rw26.c
+++ b/drivers/staging/lustre/lustre/llite/rw26.c
@@ -403,7 +403,7 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter,
 	 * 1. Need inode mutex to operate transient pages.
 	 */
 	if (iov_iter_rw(iter) == READ)
-		mutex_lock(&inode->i_mutex);
+		inode_lock(inode);
 
 	LASSERT(obj->cob_transient_pages == 0);
 	while (iov_iter_count(iter)) {
@@ -454,7 +454,7 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter,
 out:
 	LASSERT(obj->cob_transient_pages == 0);
 	if (iov_iter_rw(iter) == READ)
-		mutex_unlock(&inode->i_mutex);
+		inode_unlock(inode);
 
 	if (tot_bytes > 0) {
 		if (iov_iter_rw(iter) == WRITE) {
diff --git a/drivers/staging/lustre/lustre/llite/super25.c b/drivers/staging/lustre/lustre/llite/super25.c
index 7a9fafc67693..86c371ef71ea 100644
--- a/drivers/staging/lustre/lustre/llite/super25.c
+++ b/drivers/staging/lustre/lustre/llite/super25.c
@@ -106,7 +106,8 @@ static int __init init_lustre_lite(void)
 	rc = -ENOMEM;
 	ll_inode_cachep = kmem_cache_create("lustre_inode_cache",
 					    sizeof(struct ll_inode_info),
-					    0, SLAB_HWCACHE_ALIGN, NULL);
+					    0, SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT,
+					    NULL);
 	if (ll_inode_cachep == NULL)
 		goto out_cache;
 
diff --git a/drivers/staging/lustre/lustre/llite/vvp_io.c b/drivers/staging/lustre/lustre/llite/vvp_io.c
index f68e972886ca..0920ac6b3003 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_io.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_io.c
@@ -439,7 +439,7 @@ static int vvp_io_setattr_start(const struct lu_env *env,
 	struct inode	*inode = ccc_object_inode(io->ci_obj);
 	int result = 0;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	if (cl_io_is_trunc(io))
 		result = vvp_io_setattr_trunc(env, ios, inode,
 					io->u.ci_setattr.sa_attr.lvb_size);
@@ -459,7 +459,7 @@ static void vvp_io_setattr_end(const struct lu_env *env,
 		 * because osc has already notified to destroy osc_extents. */
 		vvp_do_vmtruncate(inode, io->u.ci_setattr.sa_attr.lvb_size);
 
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 }
 
 static void vvp_io_setattr_fini(const struct lu_env *env,
diff --git a/drivers/staging/lustre/lustre/llite/vvp_page.c b/drivers/staging/lustre/lustre/llite/vvp_page.c
index 99c0d7aee921..a133475a7c74 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_page.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_page.c
@@ -428,7 +428,7 @@ static void vvp_transient_page_verify(const struct cl_page *page)
 {
 	struct inode *inode = ccc_object_inode(page->cp_obj);
 
-	LASSERT(!mutex_trylock(&inode->i_mutex));
+	LASSERT(!inode_trylock(inode));
 }
 
 static int vvp_transient_page_own(const struct lu_env *env,
@@ -480,9 +480,9 @@ static int vvp_transient_page_is_vmlocked(const struct lu_env *env,
 	struct inode    *inode = ccc_object_inode(slice->cpl_obj);
 	int	locked;
 
-	locked = !mutex_trylock(&inode->i_mutex);
+	locked = !inode_trylock(inode);
 	if (!locked)
-		mutex_unlock(&inode->i_mutex);
+		inode_unlock(inode);
 	return locked ? -EBUSY : -ENODATA;
 }
 
@@ -502,7 +502,7 @@ static void vvp_transient_page_fini(const struct lu_env *env,
 	struct ccc_object *clobj = cl2ccc(clp->cp_obj);
 
 	vvp_page_fini_common(cp);
-	LASSERT(!mutex_trylock(&clobj->cob_inode->i_mutex));
+	LASSERT(!inode_trylock(clobj->cob_inode));
 	clobj->cob_transient_pages--;
 }
 
@@ -548,7 +548,7 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
 	} else {
 		struct ccc_object *clobj = cl2ccc(obj);
 
-		LASSERT(!mutex_trylock(&clobj->cob_inode->i_mutex));
+		LASSERT(!inode_trylock(clobj->cob_inode));
 		cl_page_slice_add(page, &cpg->cpg_cl, obj,
 				&vvp_transient_page_ops);
 		clobj->cob_transient_pages++;
diff --git a/drivers/staging/rdma/amso1100/c2_cq.c b/drivers/staging/rdma/amso1100/c2_cq.c
index 3ef881f2da0f..7ad0c082485a 100644
--- a/drivers/staging/rdma/amso1100/c2_cq.c
+++ b/drivers/staging/rdma/amso1100/c2_cq.c
@@ -173,9 +173,6 @@ static inline int c2_poll_one(struct c2_dev *c2dev,
 	case C2_WR_TYPE_RDMA_READ:
 		entry->opcode = IB_WC_RDMA_READ;
 		break;
-	case C2_WR_TYPE_BIND_MW:
-		entry->opcode = IB_WC_BIND_MW;
-		break;
 	case C2_WR_TYPE_RECV:
 		entry->byte_len = be32_to_cpu(ce->bytes_rcvd);
 		entry->opcode = IB_WC_RECV;
diff --git a/drivers/staging/rdma/amso1100/c2_provider.c b/drivers/staging/rdma/amso1100/c2_provider.c
index a092ac743c72..de8d10e1bde3 100644
--- a/drivers/staging/rdma/amso1100/c2_provider.c
+++ b/drivers/staging/rdma/amso1100/c2_provider.c
@@ -337,43 +337,21 @@ static inline u32 c2_convert_access(int acc)
 	    C2_ACF_LOCAL_READ | C2_ACF_WINDOW_BIND;
 }
 
-static struct ib_mr *c2_reg_phys_mr(struct ib_pd *ib_pd,
-				    struct ib_phys_buf *buffer_list,
-				    int num_phys_buf, int acc, u64 * iova_start)
+static struct ib_mr *c2_get_dma_mr(struct ib_pd *pd, int acc)
 {
 	struct c2_mr *mr;
 	u64 *page_list;
-	u32 total_len;
-	int err, i, j, k, page_shift, pbl_depth;
+	const u32 total_len = 0xffffffff;	/* AMSO1100 limit */
+	int err, page_shift, pbl_depth, i;
+	u64 kva = 0;
 
-	pbl_depth = 0;
-	total_len = 0;
+	pr_debug("%s:%u\n", __func__, __LINE__);
 
-	page_shift = PAGE_SHIFT;
 	/*
-	 * If there is only 1 buffer we assume this could
-	 * be a map of all phy mem...use a 32k page_shift.
+	 * This is a map of all phy mem...use a 32k page_shift.
 	 */
-	if (num_phys_buf == 1)
-		page_shift += 3;
-
-	for (i = 0; i < num_phys_buf; i++) {
-
-		if (offset_in_page(buffer_list[i].addr)) {
-			pr_debug("Unaligned Memory Buffer: 0x%x\n",
-				(unsigned int) buffer_list[i].addr);
-			return ERR_PTR(-EINVAL);
-		}
-
-		if (!buffer_list[i].size) {
-			pr_debug("Invalid Buffer Size\n");
-			return ERR_PTR(-EINVAL);
-		}
-
-		total_len += buffer_list[i].size;
-		pbl_depth += ALIGN(buffer_list[i].size,
-				   BIT(page_shift)) >> page_shift;
-	}
+	page_shift = PAGE_SHIFT + 3;
+	pbl_depth = ALIGN(total_len, BIT(page_shift)) >> page_shift;
 
 	page_list = vmalloc(sizeof(u64) * pbl_depth);
 	if (!page_list) {
@@ -382,16 +360,8 @@ static struct ib_mr *c2_reg_phys_mr(struct ib_pd *ib_pd,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	for (i = 0, j = 0; i < num_phys_buf; i++) {
-
-		int naddrs;
-
- 		naddrs = ALIGN(buffer_list[i].size,
-			       BIT(page_shift)) >> page_shift;
-		for (k = 0; k < naddrs; k++)
-			page_list[j++] = (buffer_list[i].addr +
-						     (k << page_shift));
-	}
+	for (i = 0; i < pbl_depth; i++)
+		page_list[i] = (i << page_shift);
 
 	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
 	if (!mr) {
@@ -399,17 +369,17 @@ static struct ib_mr *c2_reg_phys_mr(struct ib_pd *ib_pd,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	mr->pd = to_c2pd(ib_pd);
+	mr->pd = to_c2pd(pd);
 	mr->umem = NULL;
 	pr_debug("%s - page shift %d, pbl_depth %d, total_len %u, "
 		"*iova_start %llx, first pa %llx, last pa %llx\n",
 		__func__, page_shift, pbl_depth, total_len,
-		(unsigned long long) *iova_start,
+		(unsigned long long) kva,
 	       	(unsigned long long) page_list[0],
 	       	(unsigned long long) page_list[pbl_depth-1]);
-  	err = c2_nsmr_register_phys_kern(to_c2dev(ib_pd->device), page_list,
+	err = c2_nsmr_register_phys_kern(to_c2dev(pd->device), page_list,
 					 BIT(page_shift), pbl_depth,
-					 total_len, 0, iova_start,
+					 total_len, 0, &kva,
 					 c2_convert_access(acc), mr);
 	vfree(page_list);
 	if (err) {
@@ -420,19 +390,6 @@ static struct ib_mr *c2_reg_phys_mr(struct ib_pd *ib_pd,
 	return &mr->ibmr;
 }
 
-static struct ib_mr *c2_get_dma_mr(struct ib_pd *pd, int acc)
-{
-	struct ib_phys_buf bl;
-	u64 kva = 0;
-
-	pr_debug("%s:%u\n", __func__, __LINE__);
-
-	/* AMSO1100 limit */
-	bl.size = 0xffffffff;
-	bl.addr = 0;
-	return c2_reg_phys_mr(pd, &bl, 1, acc, &kva);
-}
-
 static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				    u64 virt, int acc, struct ib_udata *udata)
 {
@@ -840,7 +797,6 @@ int c2_register_device(struct c2_dev *dev)
 	dev->ibdev.destroy_cq = c2_destroy_cq;
 	dev->ibdev.poll_cq = c2_poll_cq;
 	dev->ibdev.get_dma_mr = c2_get_dma_mr;
-	dev->ibdev.reg_phys_mr = c2_reg_phys_mr;
 	dev->ibdev.reg_user_mr = c2_reg_user_mr;
 	dev->ibdev.dereg_mr = c2_dereg_mr;
 	dev->ibdev.get_port_immutable = c2_port_immutable;
diff --git a/drivers/staging/rdma/ehca/ehca_classes.h b/drivers/staging/rdma/ehca/ehca_classes.h
index bd45e0f3923f..e8c3387d7aaa 100644
--- a/drivers/staging/rdma/ehca/ehca_classes.h
+++ b/drivers/staging/rdma/ehca/ehca_classes.h
@@ -316,9 +316,8 @@ struct ehca_mr_pginfo {
 
 	union {
 		struct { /* type EHCA_MR_PGI_PHYS section */
-			int num_phys_buf;
-			struct ib_phys_buf *phys_buf_array;
-			u64 next_buf;
+			u64 addr;
+			u16 size;
 		} phy;
 		struct { /* type EHCA_MR_PGI_USER section */
 			struct ib_umem *region;
diff --git a/drivers/staging/rdma/ehca/ehca_iverbs.h b/drivers/staging/rdma/ehca/ehca_iverbs.h
index 80e6a3d5df3e..cca5933fcda6 100644
--- a/drivers/staging/rdma/ehca/ehca_iverbs.h
+++ b/drivers/staging/rdma/ehca/ehca_iverbs.h
@@ -80,30 +80,14 @@ int ehca_destroy_ah(struct ib_ah *ah);
 
 struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
 
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
-			       struct ib_phys_buf *phys_buf_array,
-			       int num_phys_buf,
-			       int mr_access_flags, u64 *iova_start);
-
 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			       u64 virt, int mr_access_flags,
 			       struct ib_udata *udata);
 
-int ehca_rereg_phys_mr(struct ib_mr *mr,
-		       int mr_rereg_mask,
-		       struct ib_pd *pd,
-		       struct ib_phys_buf *phys_buf_array,
-		       int num_phys_buf, int mr_access_flags, u64 *iova_start);
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
-
 int ehca_dereg_mr(struct ib_mr *mr);
 
 struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
 
-int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
-		 struct ib_mw_bind *mw_bind);
-
 int ehca_dealloc_mw(struct ib_mw *mw);
 
 struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
diff --git a/drivers/staging/rdma/ehca/ehca_main.c b/drivers/staging/rdma/ehca/ehca_main.c
index 860b974e9faa..832f22f40862 100644
--- a/drivers/staging/rdma/ehca/ehca_main.c
+++ b/drivers/staging/rdma/ehca/ehca_main.c
@@ -511,13 +511,9 @@ static int ehca_init_device(struct ehca_shca *shca)
 	shca->ib_device.req_notify_cq	    = ehca_req_notify_cq;
 	/* shca->ib_device.req_ncomp_notif  = ehca_req_ncomp_notif; */
 	shca->ib_device.get_dma_mr	    = ehca_get_dma_mr;
-	shca->ib_device.reg_phys_mr	    = ehca_reg_phys_mr;
 	shca->ib_device.reg_user_mr	    = ehca_reg_user_mr;
-	shca->ib_device.query_mr	    = ehca_query_mr;
 	shca->ib_device.dereg_mr	    = ehca_dereg_mr;
-	shca->ib_device.rereg_phys_mr	    = ehca_rereg_phys_mr;
 	shca->ib_device.alloc_mw	    = ehca_alloc_mw;
-	shca->ib_device.bind_mw		    = ehca_bind_mw;
 	shca->ib_device.dealloc_mw	    = ehca_dealloc_mw;
 	shca->ib_device.alloc_fmr	    = ehca_alloc_fmr;
 	shca->ib_device.map_phys_fmr	    = ehca_map_phys_fmr;
diff --git a/drivers/staging/rdma/ehca/ehca_mrmw.c b/drivers/staging/rdma/ehca/ehca_mrmw.c
index 553e883a5718..3367205e3160 100644
--- a/drivers/staging/rdma/ehca/ehca_mrmw.c
+++ b/drivers/staging/rdma/ehca/ehca_mrmw.c
@@ -196,120 +196,6 @@ get_dma_mr_exit0:
 
 /*----------------------------------------------------------------------*/
 
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
-			       struct ib_phys_buf *phys_buf_array,
-			       int num_phys_buf,
-			       int mr_access_flags,
-			       u64 *iova_start)
-{
-	struct ib_mr *ib_mr;
-	int ret;
-	struct ehca_mr *e_mr;
-	struct ehca_shca *shca =
-		container_of(pd->device, struct ehca_shca, ib_device);
-	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
-
-	u64 size;
-
-	if ((num_phys_buf <= 0) || !phys_buf_array) {
-		ehca_err(pd->device, "bad input values: num_phys_buf=%x "
-			 "phys_buf_array=%p", num_phys_buf, phys_buf_array);
-		ib_mr = ERR_PTR(-EINVAL);
-		goto reg_phys_mr_exit0;
-	}
-	if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
-	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
-	    ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
-	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
-		/*
-		 * Remote Write Access requires Local Write Access
-		 * Remote Atomic Access requires Local Write Access
-		 */
-		ehca_err(pd->device, "bad input values: mr_access_flags=%x",
-			 mr_access_flags);
-		ib_mr = ERR_PTR(-EINVAL);
-		goto reg_phys_mr_exit0;
-	}
-
-	/* check physical buffer list and calculate size */
-	ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
-					    iova_start, &size);
-	if (ret) {
-		ib_mr = ERR_PTR(ret);
-		goto reg_phys_mr_exit0;
-	}
-	if ((size == 0) ||
-	    (((u64)iova_start + size) < (u64)iova_start)) {
-		ehca_err(pd->device, "bad input values: size=%llx iova_start=%p",
-			 size, iova_start);
-		ib_mr = ERR_PTR(-EINVAL);
-		goto reg_phys_mr_exit0;
-	}
-
-	e_mr = ehca_mr_new();
-	if (!e_mr) {
-		ehca_err(pd->device, "out of memory");
-		ib_mr = ERR_PTR(-ENOMEM);
-		goto reg_phys_mr_exit0;
-	}
-
-	/* register MR on HCA */
-	if (ehca_mr_is_maxmr(size, iova_start)) {
-		e_mr->flags |= EHCA_MR_FLAG_MAXMR;
-		ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
-				     e_pd, &e_mr->ib.ib_mr.lkey,
-				     &e_mr->ib.ib_mr.rkey);
-		if (ret) {
-			ib_mr = ERR_PTR(ret);
-			goto reg_phys_mr_exit1;
-		}
-	} else {
-		struct ehca_mr_pginfo pginfo;
-		u32 num_kpages;
-		u32 num_hwpages;
-		u64 hw_pgsize;
-
-		num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
-					PAGE_SIZE);
-		/* for kernel space we try most possible pgsize */
-		hw_pgsize = ehca_get_max_hwpage_size(shca);
-		num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
-					 hw_pgsize);
-		memset(&pginfo, 0, sizeof(pginfo));
-		pginfo.type = EHCA_MR_PGI_PHYS;
-		pginfo.num_kpages = num_kpages;
-		pginfo.hwpage_size = hw_pgsize;
-		pginfo.num_hwpages = num_hwpages;
-		pginfo.u.phy.num_phys_buf = num_phys_buf;
-		pginfo.u.phy.phys_buf_array = phys_buf_array;
-		pginfo.next_hwpage =
-			((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
-
-		ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
-				  e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
-				  &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
-		if (ret) {
-			ib_mr = ERR_PTR(ret);
-			goto reg_phys_mr_exit1;
-		}
-	}
-
-	/* successful registration of all pages */
-	return &e_mr->ib.ib_mr;
-
-reg_phys_mr_exit1:
-	ehca_mr_delete(e_mr);
-reg_phys_mr_exit0:
-	if (IS_ERR(ib_mr))
-		ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p "
-			 "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
-			 PTR_ERR(ib_mr), pd, phys_buf_array,
-			 num_phys_buf, mr_access_flags, iova_start);
-	return ib_mr;
-} /* end ehca_reg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			       u64 virt, int mr_access_flags,
 			       struct ib_udata *udata)
@@ -437,207 +323,6 @@ reg_user_mr_exit0:
 
 /*----------------------------------------------------------------------*/
 
-int ehca_rereg_phys_mr(struct ib_mr *mr,
-		       int mr_rereg_mask,
-		       struct ib_pd *pd,
-		       struct ib_phys_buf *phys_buf_array,
-		       int num_phys_buf,
-		       int mr_access_flags,
-		       u64 *iova_start)
-{
-	int ret;
-
-	struct ehca_shca *shca =
-		container_of(mr->device, struct ehca_shca, ib_device);
-	struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
-	u64 new_size;
-	u64 *new_start;
-	u32 new_acl;
-	struct ehca_pd *new_pd;
-	u32 tmp_lkey, tmp_rkey;
-	unsigned long sl_flags;
-	u32 num_kpages = 0;
-	u32 num_hwpages = 0;
-	struct ehca_mr_pginfo pginfo;
-
-	if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
-		/* TODO not supported, because PHYP rereg hCall needs pages */
-		ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
-			 "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
-		ret = -EINVAL;
-		goto rereg_phys_mr_exit0;
-	}
-
-	if (mr_rereg_mask & IB_MR_REREG_PD) {
-		if (!pd) {
-			ehca_err(mr->device, "rereg with bad pd, pd=%p "
-				 "mr_rereg_mask=%x", pd, mr_rereg_mask);
-			ret = -EINVAL;
-			goto rereg_phys_mr_exit0;
-		}
-	}
-
-	if ((mr_rereg_mask &
-	     ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
-	    (mr_rereg_mask == 0)) {
-		ret = -EINVAL;
-		goto rereg_phys_mr_exit0;
-	}
-
-	/* check other parameters */
-	if (e_mr == shca->maxmr) {
-		/* should be impossible, however reject to be sure */
-		ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
-			 "shca->maxmr=%p mr->lkey=%x",
-			 mr, shca->maxmr, mr->lkey);
-		ret = -EINVAL;
-		goto rereg_phys_mr_exit0;
-	}
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
-		if (e_mr->flags & EHCA_MR_FLAG_FMR) {
-			ehca_err(mr->device, "not supported for FMR, mr=%p "
-				 "flags=%x", mr, e_mr->flags);
-			ret = -EINVAL;
-			goto rereg_phys_mr_exit0;
-		}
-		if (!phys_buf_array || num_phys_buf <= 0) {
-			ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
-				 " phys_buf_array=%p num_phys_buf=%x",
-				 mr_rereg_mask, phys_buf_array, num_phys_buf);
-			ret = -EINVAL;
-			goto rereg_phys_mr_exit0;
-		}
-	}
-	if ((mr_rereg_mask & IB_MR_REREG_ACCESS) &&	/* change ACL */
-	    (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
-	      !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
-	     ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
-	      !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
-		/*
-		 * Remote Write Access requires Local Write Access
-		 * Remote Atomic Access requires Local Write Access
-		 */
-		ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
-			 "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
-		ret = -EINVAL;
-		goto rereg_phys_mr_exit0;
-	}
-
-	/* set requested values dependent on rereg request */
-	spin_lock_irqsave(&e_mr->mrlock, sl_flags);
-	new_start = e_mr->start;
-	new_size = e_mr->size;
-	new_acl = e_mr->acl;
-	new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
-
-	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
-		u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
-
-		new_start = iova_start;	/* change address */
-		/* check physical buffer list and calculate size */
-		ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
-						    num_phys_buf, iova_start,
-						    &new_size);
-		if (ret)
-			goto rereg_phys_mr_exit1;
-		if ((new_size == 0) ||
-		    (((u64)iova_start + new_size) < (u64)iova_start)) {
-			ehca_err(mr->device, "bad input values: new_size=%llx "
-				 "iova_start=%p", new_size, iova_start);
-			ret = -EINVAL;
-			goto rereg_phys_mr_exit1;
-		}
-		num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
-					new_size, PAGE_SIZE);
-		num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
-					 new_size, hw_pgsize);
-		memset(&pginfo, 0, sizeof(pginfo));
-		pginfo.type = EHCA_MR_PGI_PHYS;
-		pginfo.num_kpages = num_kpages;
-		pginfo.hwpage_size = hw_pgsize;
-		pginfo.num_hwpages = num_hwpages;
-		pginfo.u.phy.num_phys_buf = num_phys_buf;
-		pginfo.u.phy.phys_buf_array = phys_buf_array;
-		pginfo.next_hwpage =
-			((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
-	}
-	if (mr_rereg_mask & IB_MR_REREG_ACCESS)
-		new_acl = mr_access_flags;
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		new_pd = container_of(pd, struct ehca_pd, ib_pd);
-
-	ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
-			    new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
-	if (ret)
-		goto rereg_phys_mr_exit1;
-
-	/* successful reregistration */
-	if (mr_rereg_mask & IB_MR_REREG_PD)
-		mr->pd = pd;
-	mr->lkey = tmp_lkey;
-	mr->rkey = tmp_rkey;
-
-rereg_phys_mr_exit1:
-	spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-rereg_phys_mr_exit0:
-	if (ret)
-		ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p "
-			 "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
-			 "iova_start=%p",
-			 ret, mr, mr_rereg_mask, pd, phys_buf_array,
-			 num_phys_buf, mr_access_flags, iova_start);
-	return ret;
-} /* end ehca_rereg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
-{
-	int ret = 0;
-	u64 h_ret;
-	struct ehca_shca *shca =
-		container_of(mr->device, struct ehca_shca, ib_device);
-	struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
-	unsigned long sl_flags;
-	struct ehca_mr_hipzout_parms hipzout;
-
-	if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
-		ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
-			 "e_mr->flags=%x", mr, e_mr, e_mr->flags);
-		ret = -EINVAL;
-		goto query_mr_exit0;
-	}
-
-	memset(mr_attr, 0, sizeof(struct ib_mr_attr));
-	spin_lock_irqsave(&e_mr->mrlock, sl_flags);
-
-	h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
-	if (h_ret != H_SUCCESS) {
-		ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p "
-			 "hca_hndl=%llx mr_hndl=%llx lkey=%x",
-			 h_ret, mr, shca->ipz_hca_handle.handle,
-			 e_mr->ipz_mr_handle.handle, mr->lkey);
-		ret = ehca2ib_return_code(h_ret);
-		goto query_mr_exit1;
-	}
-	mr_attr->pd = mr->pd;
-	mr_attr->device_virt_addr = hipzout.vaddr;
-	mr_attr->size = hipzout.len;
-	mr_attr->lkey = hipzout.lkey;
-	mr_attr->rkey = hipzout.rkey;
-	ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
-
-query_mr_exit1:
-	spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-query_mr_exit0:
-	if (ret)
-		ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p",
-			 ret, mr, mr_attr);
-	return ret;
-} /* end ehca_query_mr() */
-
-/*----------------------------------------------------------------------*/
-
 int ehca_dereg_mr(struct ib_mr *mr)
 {
 	int ret = 0;
@@ -728,18 +413,6 @@ alloc_mw_exit0:
 
 /*----------------------------------------------------------------------*/
 
-int ehca_bind_mw(struct ib_qp *qp,
-		 struct ib_mw *mw,
-		 struct ib_mw_bind *mw_bind)
-{
-	/* TODO: not supported up to now */
-	ehca_gen_err("bind MW currently not supported by HCAD");
-
-	return -EPERM;
-} /* end ehca_bind_mw() */
-
-/*----------------------------------------------------------------------*/
-
 int ehca_dealloc_mw(struct ib_mw *mw)
 {
 	u64 h_ret;
@@ -1616,7 +1289,6 @@ int ehca_reg_internal_maxmr(
 	u64 *iova_start;
 	u64 size_maxmr;
 	struct ehca_mr_pginfo pginfo;
-	struct ib_phys_buf ib_pbuf;
 	u32 num_kpages;
 	u32 num_hwpages;
 	u64 hw_pgsize;
@@ -1637,8 +1309,6 @@ int ehca_reg_internal_maxmr(
 	/* register internal max-MR on HCA */
 	size_maxmr = ehca_mr_len;
 	iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START));
-	ib_pbuf.addr = 0;
-	ib_pbuf.size = size_maxmr;
 	num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
 				PAGE_SIZE);
 	hw_pgsize = ehca_get_max_hwpage_size(shca);
@@ -1650,8 +1320,8 @@ int ehca_reg_internal_maxmr(
 	pginfo.num_kpages = num_kpages;
 	pginfo.num_hwpages = num_hwpages;
 	pginfo.hwpage_size = hw_pgsize;
-	pginfo.u.phy.num_phys_buf = 1;
-	pginfo.u.phy.phys_buf_array = &ib_pbuf;
+	pginfo.u.phy.addr = 0;
+	pginfo.u.phy.size = size_maxmr;
 
 	ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
 			  &pginfo, &e_mr->ib.ib_mr.lkey,
@@ -1669,7 +1339,6 @@ int ehca_reg_internal_maxmr(
 	e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
 	e_mr->ib.ib_mr.uobject = NULL;
 	atomic_inc(&(e_pd->ib_pd.usecnt));
-	atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
 	*e_maxmr = e_mr;
 	return 0;
 
@@ -1762,61 +1431,6 @@ ehca_dereg_internal_maxmr_exit0:
 
 /*----------------------------------------------------------------------*/
 
-/*
- * check physical buffer array of MR verbs for validness and
- * calculates MR size
- */
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
-				  int num_phys_buf,
-				  u64 *iova_start,
-				  u64 *size)
-{
-	struct ib_phys_buf *pbuf = phys_buf_array;
-	u64 size_count = 0;
-	u32 i;
-
-	if (num_phys_buf == 0) {
-		ehca_gen_err("bad phys buf array len, num_phys_buf=0");
-		return -EINVAL;
-	}
-	/* check first buffer */
-	if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
-		ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
-			     "pbuf->addr=%llx pbuf->size=%llx",
-			     iova_start, pbuf->addr, pbuf->size);
-		return -EINVAL;
-	}
-	if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
-	    (num_phys_buf > 1)) {
-		ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx "
-			     "pbuf->size=%llx", pbuf->addr, pbuf->size);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < num_phys_buf; i++) {
-		if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
-			ehca_gen_err("bad address, i=%x pbuf->addr=%llx "
-				     "pbuf->size=%llx",
-				     i, pbuf->addr, pbuf->size);
-			return -EINVAL;
-		}
-		if (((i > 0) &&	/* not 1st */
-		     (i < (num_phys_buf - 1)) &&	/* not last */
-		     (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
-			ehca_gen_err("bad size, i=%x pbuf->size=%llx",
-				     i, pbuf->size);
-			return -EINVAL;
-		}
-		size_count += pbuf->size;
-		pbuf++;
-	}
-
-	*size = size_count;
-	return 0;
-} /* end ehca_mr_chk_buf_and_calc_size() */
-
-/*----------------------------------------------------------------------*/
-
 /* check page list of map FMR verb for validness */
 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
 			     u64 *page_list,
@@ -2002,57 +1616,54 @@ static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
 				 u32 number, u64 *kpage)
 {
 	int ret = 0;
-	struct ib_phys_buf *pbuf;
+	u64 addr = pginfo->u.phy.addr;
+	u64 size = pginfo->u.phy.size;
 	u64 num_hw, offs_hw;
 	u32 i = 0;
 
-	/* loop over desired phys_buf_array entries */
-	while (i < number) {
-		pbuf   = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
-		num_hw  = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
-				     pbuf->size, pginfo->hwpage_size);
-		offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
-			pginfo->hwpage_size;
-		while (pginfo->next_hwpage < offs_hw + num_hw) {
-			/* sanity check */
-			if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
-			    (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
-				ehca_gen_err("kpage_cnt >= num_kpages, "
-					     "kpage_cnt=%llx num_kpages=%llx "
-					     "hwpage_cnt=%llx "
-					     "num_hwpages=%llx i=%x",
-					     pginfo->kpage_cnt,
-					     pginfo->num_kpages,
-					     pginfo->hwpage_cnt,
-					     pginfo->num_hwpages, i);
-				return -EFAULT;
-			}
-			*kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
-				 (pginfo->next_hwpage * pginfo->hwpage_size);
-			if ( !(*kpage) && pbuf->addr ) {
-				ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx "
-					     "next_hwpage=%llx", pbuf->addr,
-					     pbuf->size, pginfo->next_hwpage);
-				return -EFAULT;
-			}
-			(pginfo->hwpage_cnt)++;
-			(pginfo->next_hwpage)++;
-			if (PAGE_SIZE >= pginfo->hwpage_size) {
-				if (pginfo->next_hwpage %
-				    (PAGE_SIZE / pginfo->hwpage_size) == 0)
-					(pginfo->kpage_cnt)++;
-			} else
-				pginfo->kpage_cnt += pginfo->hwpage_size /
-					PAGE_SIZE;
-			kpage++;
-			i++;
-			if (i >= number) break;
+	num_hw  = NUM_CHUNKS((addr % pginfo->hwpage_size) + size,
+				pginfo->hwpage_size);
+	offs_hw = (addr & ~(pginfo->hwpage_size - 1)) / pginfo->hwpage_size;
+
+	while (pginfo->next_hwpage < offs_hw + num_hw) {
+		/* sanity check */
+		if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
+		    (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
+			ehca_gen_err("kpage_cnt >= num_kpages, "
+				     "kpage_cnt=%llx num_kpages=%llx "
+				     "hwpage_cnt=%llx "
+				     "num_hwpages=%llx i=%x",
+				     pginfo->kpage_cnt,
+				     pginfo->num_kpages,
+				     pginfo->hwpage_cnt,
+				     pginfo->num_hwpages, i);
+			return -EFAULT;
 		}
-		if (pginfo->next_hwpage >= offs_hw + num_hw) {
-			(pginfo->u.phy.next_buf)++;
-			pginfo->next_hwpage = 0;
+		*kpage = (addr & ~(pginfo->hwpage_size - 1)) +
+			 (pginfo->next_hwpage * pginfo->hwpage_size);
+		if ( !(*kpage) && addr ) {
+			ehca_gen_err("addr=%llx size=%llx "
+				     "next_hwpage=%llx", addr,
+				     size, pginfo->next_hwpage);
+			return -EFAULT;
 		}
+		(pginfo->hwpage_cnt)++;
+		(pginfo->next_hwpage)++;
+		if (PAGE_SIZE >= pginfo->hwpage_size) {
+			if (pginfo->next_hwpage %
+			    (PAGE_SIZE / pginfo->hwpage_size) == 0)
+				(pginfo->kpage_cnt)++;
+		} else
+			pginfo->kpage_cnt += pginfo->hwpage_size /
+				PAGE_SIZE;
+		kpage++;
+		i++;
+		if (i >= number) break;
 	}
+	if (pginfo->next_hwpage >= offs_hw + num_hw) {
+		pginfo->next_hwpage = 0;
+	}
+
 	return ret;
 }
 
diff --git a/drivers/staging/rdma/ehca/ehca_mrmw.h b/drivers/staging/rdma/ehca/ehca_mrmw.h
index 50d8b51306dd..52bfa95697f7 100644
--- a/drivers/staging/rdma/ehca/ehca_mrmw.h
+++ b/drivers/staging/rdma/ehca/ehca_mrmw.h
@@ -98,11 +98,6 @@ int ehca_reg_maxmr(struct ehca_shca *shca,
 
 int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
 
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
-				  int num_phys_buf,
-				  u64 *iova_start,
-				  u64 *size);
-
 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
 			     u64 *page_list,
 			     int list_len);
diff --git a/drivers/staging/rdma/ehca/ehca_reqs.c b/drivers/staging/rdma/ehca/ehca_reqs.c
index 10e2074384f5..11813b880e16 100644
--- a/drivers/staging/rdma/ehca/ehca_reqs.c
+++ b/drivers/staging/rdma/ehca/ehca_reqs.c
@@ -614,7 +614,6 @@ int ehca_post_srq_recv(struct ib_srq *srq,
 static const u8 ib_wc_opcode[255] = {
 	[0x01] = IB_WC_RECV+1,
 	[0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
-	[0x04] = IB_WC_BIND_MW+1,
 	[0x08] = IB_WC_FETCH_ADD+1,
 	[0x10] = IB_WC_COMP_SWAP+1,
 	[0x20] = IB_WC_RDMA_WRITE+1,
diff --git a/drivers/staging/rdma/hfi1/mr.c b/drivers/staging/rdma/hfi1/mr.c
index 568f185a022d..a3f8b884fdd6 100644
--- a/drivers/staging/rdma/hfi1/mr.c
+++ b/drivers/staging/rdma/hfi1/mr.c
@@ -167,10 +167,7 @@ static struct hfi1_mr *alloc_mr(int count, struct ib_pd *pd)
 	rval = init_mregion(&mr->mr, pd, count);
 	if (rval)
 		goto bail;
-	/*
-	 * ib_reg_phys_mr() will initialize mr->ibmr except for
-	 * lkey and rkey.
-	 */
+
 	rval = hfi1_alloc_lkey(&mr->mr, 0);
 	if (rval)
 		goto bail_mregion;
@@ -188,52 +185,6 @@ bail:
 }
 
 /**
- * hfi1_reg_phys_mr - register a physical memory region
- * @pd: protection domain for this memory region
- * @buffer_list: pointer to the list of physical buffers to register
- * @num_phys_buf: the number of physical buffers to register
- * @iova_start: the starting address passed over IB which maps to this MR
- *
- * Returns the memory region on success, otherwise returns an errno.
- */
-struct ib_mr *hfi1_reg_phys_mr(struct ib_pd *pd,
-			       struct ib_phys_buf *buffer_list,
-			       int num_phys_buf, int acc, u64 *iova_start)
-{
-	struct hfi1_mr *mr;
-	int n, m, i;
-	struct ib_mr *ret;
-
-	mr = alloc_mr(num_phys_buf, pd);
-	if (IS_ERR(mr)) {
-		ret = (struct ib_mr *)mr;
-		goto bail;
-	}
-
-	mr->mr.user_base = *iova_start;
-	mr->mr.iova = *iova_start;
-	mr->mr.access_flags = acc;
-
-	m = 0;
-	n = 0;
-	for (i = 0; i < num_phys_buf; i++) {
-		mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr;
-		mr->mr.map[m]->segs[n].length = buffer_list[i].size;
-		mr->mr.length += buffer_list[i].size;
-		n++;
-		if (n == HFI1_SEGSZ) {
-			m++;
-			n = 0;
-		}
-	}
-
-	ret = &mr->ibmr;
-
-bail:
-	return ret;
-}
-
-/**
  * hfi1_reg_user_mr - register a userspace memory region
  * @pd: protection domain for this memory region
  * @start: starting userspace address
diff --git a/drivers/staging/rdma/hfi1/verbs.c b/drivers/staging/rdma/hfi1/verbs.c
index ef0feaa684a4..09b8d412ee90 100644
--- a/drivers/staging/rdma/hfi1/verbs.c
+++ b/drivers/staging/rdma/hfi1/verbs.c
@@ -2052,7 +2052,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	ibdev->poll_cq = hfi1_poll_cq;
 	ibdev->req_notify_cq = hfi1_req_notify_cq;
 	ibdev->get_dma_mr = hfi1_get_dma_mr;
-	ibdev->reg_phys_mr = hfi1_reg_phys_mr;
 	ibdev->reg_user_mr = hfi1_reg_user_mr;
 	ibdev->dereg_mr = hfi1_dereg_mr;
 	ibdev->alloc_mr = hfi1_alloc_mr;
diff --git a/drivers/staging/rdma/hfi1/verbs.h b/drivers/staging/rdma/hfi1/verbs.h
index 72106e5362b9..286e468b0479 100644
--- a/drivers/staging/rdma/hfi1/verbs.h
+++ b/drivers/staging/rdma/hfi1/verbs.h
@@ -1024,10 +1024,6 @@ int hfi1_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 
 struct ib_mr *hfi1_get_dma_mr(struct ib_pd *pd, int acc);
 
-struct ib_mr *hfi1_reg_phys_mr(struct ib_pd *pd,
-			       struct ib_phys_buf *buffer_list,
-			       int num_phys_buf, int acc, u64 *iova_start);
-
 struct ib_mr *hfi1_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			       u64 virt_addr, int mr_access_flags,
 			       struct ib_udata *udata);
diff --git a/drivers/staging/rdma/ipath/ipath_fs.c b/drivers/staging/rdma/ipath/ipath_fs.c
index 796af6867007..476fcdf05acb 100644
--- a/drivers/staging/rdma/ipath/ipath_fs.c
+++ b/drivers/staging/rdma/ipath/ipath_fs.c
@@ -82,14 +82,14 @@ static int create_file(const char *name, umode_t mode,
 {
 	int error;
 
-	mutex_lock(&d_inode(parent)->i_mutex);
+	inode_lock(d_inode(parent));
 	*dentry = lookup_one_len(name, parent, strlen(name));
 	if (!IS_ERR(*dentry))
 		error = ipathfs_mknod(d_inode(parent), *dentry,
 				      mode, fops, data);
 	else
 		error = PTR_ERR(*dentry);
-	mutex_unlock(&d_inode(parent)->i_mutex);
+	inode_unlock(d_inode(parent));
 
 	return error;
 }
@@ -295,7 +295,7 @@ static int remove_device_files(struct super_block *sb,
 	int ret;
 
 	root = dget(sb->s_root);
-	mutex_lock(&d_inode(root)->i_mutex);
+	inode_lock(d_inode(root));
 	snprintf(unit, sizeof unit, "%02d", dd->ipath_unit);
 	dir = lookup_one_len(unit, root, strlen(unit));
 
@@ -311,7 +311,7 @@ static int remove_device_files(struct super_block *sb,
 	ret = simple_rmdir(d_inode(root), dir);
 
 bail:
-	mutex_unlock(&d_inode(root)->i_mutex);
+	inode_unlock(d_inode(root));
 	dput(root);
 	return ret;
 }
diff --git a/drivers/staging/rdma/ipath/ipath_mr.c b/drivers/staging/rdma/ipath/ipath_mr.c
index c7278f6a8217..b76b0ce66709 100644
--- a/drivers/staging/rdma/ipath/ipath_mr.c
+++ b/drivers/staging/rdma/ipath/ipath_mr.c
@@ -98,10 +98,6 @@ static struct ipath_mr *alloc_mr(int count,
 	}
 	mr->mr.mapsz = m;
 
-	/*
-	 * ib_reg_phys_mr() will initialize mr->ibmr except for
-	 * lkey and rkey.
-	 */
 	if (!ipath_alloc_lkey(lk_table, &mr->mr))
 		goto bail;
 	mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey;
@@ -121,57 +117,6 @@ done:
 }
 
 /**
- * ipath_reg_phys_mr - register a physical memory region
- * @pd: protection domain for this memory region
- * @buffer_list: pointer to the list of physical buffers to register
- * @num_phys_buf: the number of physical buffers to register
- * @iova_start: the starting address passed over IB which maps to this MR
- *
- * Returns the memory region on success, otherwise returns an errno.
- */
-struct ib_mr *ipath_reg_phys_mr(struct ib_pd *pd,
-				struct ib_phys_buf *buffer_list,
-				int num_phys_buf, int acc, u64 *iova_start)
-{
-	struct ipath_mr *mr;
-	int n, m, i;
-	struct ib_mr *ret;
-
-	mr = alloc_mr(num_phys_buf, &to_idev(pd->device)->lk_table);
-	if (mr == NULL) {
-		ret = ERR_PTR(-ENOMEM);
-		goto bail;
-	}
-
-	mr->mr.pd = pd;
-	mr->mr.user_base = *iova_start;
-	mr->mr.iova = *iova_start;
-	mr->mr.length = 0;
-	mr->mr.offset = 0;
-	mr->mr.access_flags = acc;
-	mr->mr.max_segs = num_phys_buf;
-	mr->umem = NULL;
-
-	m = 0;
-	n = 0;
-	for (i = 0; i < num_phys_buf; i++) {
-		mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr;
-		mr->mr.map[m]->segs[n].length = buffer_list[i].size;
-		mr->mr.length += buffer_list[i].size;
-		n++;
-		if (n == IPATH_SEGSZ) {
-			m++;
-			n = 0;
-		}
-	}
-
-	ret = &mr->ibmr;
-
-bail:
-	return ret;
-}
-
-/**
  * ipath_reg_user_mr - register a userspace memory region
  * @pd: protection domain for this memory region
  * @start: starting userspace address
diff --git a/drivers/staging/rdma/ipath/ipath_verbs.c b/drivers/staging/rdma/ipath/ipath_verbs.c
index 1778dee13f99..53f9dcab180d 100644
--- a/drivers/staging/rdma/ipath/ipath_verbs.c
+++ b/drivers/staging/rdma/ipath/ipath_verbs.c
@@ -2201,7 +2201,6 @@ int ipath_register_ib_device(struct ipath_devdata *dd)
 	dev->poll_cq = ipath_poll_cq;
 	dev->req_notify_cq = ipath_req_notify_cq;
 	dev->get_dma_mr = ipath_get_dma_mr;
-	dev->reg_phys_mr = ipath_reg_phys_mr;
 	dev->reg_user_mr = ipath_reg_user_mr;
 	dev->dereg_mr = ipath_dereg_mr;
 	dev->alloc_fmr = ipath_alloc_fmr;
diff --git a/drivers/staging/rdma/ipath/ipath_verbs.h b/drivers/staging/rdma/ipath/ipath_verbs.h
index 0a90a56870ab..6c70a89667a9 100644
--- a/drivers/staging/rdma/ipath/ipath_verbs.h
+++ b/drivers/staging/rdma/ipath/ipath_verbs.h
@@ -828,10 +828,6 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 
 struct ib_mr *ipath_get_dma_mr(struct ib_pd *pd, int acc);
 
-struct ib_mr *ipath_reg_phys_mr(struct ib_pd *pd,
-				struct ib_phys_buf *buffer_list,
-				int num_phys_buf, int acc, u64 *iova_start);
-
 struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				u64 virt_addr, int mr_access_flags,
 				struct ib_udata *udata);
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index 72204fbf2bb1..576a7a43470c 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -1333,7 +1333,7 @@ iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf,
 			/*
 			 * Check if a delayed TASK_ABORTED status needs to
 			 * be sent now if the ISCSI_FLAG_CMD_FINAL has been
-			 * received with the unsolicitied data out.
+			 * received with the unsolicited data out.
 			 */
 			if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
 				iscsit_stop_dataout_timer(cmd);
@@ -3435,7 +3435,7 @@ iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
 
 			if ((tpg->tpg_attrib.generate_node_acls == 0) &&
 			    (tpg->tpg_attrib.demo_mode_discovery == 0) &&
-			    (!core_tpg_get_initiator_node_acl(&tpg->tpg_se_tpg,
+			    (!target_tpg_has_node_acl(&tpg->tpg_se_tpg,
 				cmd->conn->sess->sess_ops->InitiatorName))) {
 				continue;
 			}
@@ -4459,9 +4459,6 @@ int iscsit_close_connection(
 
 		return 0;
 	}
-	spin_unlock_bh(&sess->conn_lock);
-
-	return 0;
 }
 
 int iscsit_close_session(struct iscsi_session *sess)
diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
index 255204cc43e6..2f821de63049 100644
--- a/drivers/target/iscsi/iscsi_target_configfs.c
+++ b/drivers/target/iscsi/iscsi_target_configfs.c
@@ -725,11 +725,8 @@ static ssize_t lio_target_nacl_cmdsn_depth_store(struct config_item *item,
 
 	if (iscsit_get_tpg(tpg) < 0)
 		return -EINVAL;
-	/*
-	 * iscsit_tpg_set_initiator_node_queue_depth() assumes force=1
-	 */
-	ret = iscsit_tpg_set_initiator_node_queue_depth(tpg,
-				config_item_name(acl_ci), cmdsn_depth, 1);
+
+	ret = core_tpg_set_initiator_node_queue_depth(se_nacl, cmdsn_depth);
 
 	pr_debug("LIO_Target_ConfigFS: %s/%s Set CmdSN Window: %u for"
 		"InitiatorName: %s\n", config_item_name(wwn_ci),
@@ -1593,28 +1590,30 @@ static int lio_tpg_check_prot_fabric_only(
 }
 
 /*
- * Called with spin_lock_bh(struct se_portal_group->session_lock) held..
- *
- * Also, this function calls iscsit_inc_session_usage_count() on the
+ * This function calls iscsit_inc_session_usage_count() on the
  * struct iscsi_session in question.
  */
 static int lio_tpg_shutdown_session(struct se_session *se_sess)
 {
 	struct iscsi_session *sess = se_sess->fabric_sess_ptr;
+	struct se_portal_group *se_tpg = &sess->tpg->tpg_se_tpg;
 
+	spin_lock_bh(&se_tpg->session_lock);
 	spin_lock(&sess->conn_lock);
 	if (atomic_read(&sess->session_fall_back_to_erl0) ||
 	    atomic_read(&sess->session_logout) ||
 	    (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
 		spin_unlock(&sess->conn_lock);
+		spin_unlock_bh(&se_tpg->session_lock);
 		return 0;
 	}
 	atomic_set(&sess->session_reinstatement, 1);
 	spin_unlock(&sess->conn_lock);
 
 	iscsit_stop_time2retain_timer(sess);
-	iscsit_stop_session(sess, 1, 1);
+	spin_unlock_bh(&se_tpg->session_lock);
 
+	iscsit_stop_session(sess, 1, 1);
 	return 1;
 }
 
diff --git a/drivers/target/iscsi/iscsi_target_erl1.c b/drivers/target/iscsi/iscsi_target_erl1.c
index 2e561deb30a2..9214c9dafa2b 100644
--- a/drivers/target/iscsi/iscsi_target_erl1.c
+++ b/drivers/target/iscsi/iscsi_target_erl1.c
@@ -160,8 +160,7 @@ static int iscsit_handle_r2t_snack(
 			" protocol error.\n", cmd->init_task_tag, begrun,
 			(begrun + runlength), cmd->acked_data_sn);
 
-			return iscsit_reject_cmd(cmd,
-					ISCSI_REASON_PROTOCOL_ERROR, buf);
+		return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
 	}
 
 	if (runlength) {
@@ -628,8 +627,8 @@ int iscsit_dataout_datapduinorder_no_fbit(
 			if (cmd->pdu_list[i].seq_no == pdu->seq_no) {
 				if (!first_pdu)
 					first_pdu = &cmd->pdu_list[i];
-				 xfer_len += cmd->pdu_list[i].length;
-				 pdu_count++;
+				xfer_len += cmd->pdu_list[i].length;
+				pdu_count++;
 			} else if (pdu_count)
 				break;
 		}
diff --git a/drivers/target/iscsi/iscsi_target_parameters.c b/drivers/target/iscsi/iscsi_target_parameters.c
index 2cbea2af7cd0..3a1f9a7e6bb6 100644
--- a/drivers/target/iscsi/iscsi_target_parameters.c
+++ b/drivers/target/iscsi/iscsi_target_parameters.c
@@ -1668,7 +1668,7 @@ void iscsi_set_session_parameters(
 				param->value);
 		} else if (!strcmp(param->name, INITIALR2T)) {
 			ops->InitialR2T = !strcmp(param->value, YES);
-			 pr_debug("InitialR2T:                   %s\n",
+			pr_debug("InitialR2T:                   %s\n",
 				param->value);
 		} else if (!strcmp(param->name, IMMEDIATEDATA)) {
 			ops->ImmediateData = !strcmp(param->value, YES);
diff --git a/drivers/target/iscsi/iscsi_target_tmr.c b/drivers/target/iscsi/iscsi_target_tmr.c
index 11320df939f7..3d637055c36f 100644
--- a/drivers/target/iscsi/iscsi_target_tmr.c
+++ b/drivers/target/iscsi/iscsi_target_tmr.c
@@ -82,7 +82,7 @@ int iscsit_tmr_task_warm_reset(
 		pr_err("TMR Opcode TARGET_WARM_RESET authorization"
 			" failed for Initiator Node: %s\n",
 			sess->se_sess->se_node_acl->initiatorname);
-		 return -1;
+		return -1;
 	}
 	/*
 	 * Do the real work in transport_generic_do_tmr().
diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
index 23c95cd14167..0814e5894a96 100644
--- a/drivers/target/iscsi/iscsi_target_tpg.c
+++ b/drivers/target/iscsi/iscsi_target_tpg.c
@@ -590,16 +590,6 @@ int iscsit_tpg_del_network_portal(
 	return iscsit_tpg_release_np(tpg_np, tpg, np);
 }
 
-int iscsit_tpg_set_initiator_node_queue_depth(
-	struct iscsi_portal_group *tpg,
-	unsigned char *initiatorname,
-	u32 queue_depth,
-	int force)
-{
-	return core_tpg_set_initiator_node_queue_depth(&tpg->tpg_se_tpg,
-		initiatorname, queue_depth, force);
-}
-
 int iscsit_ta_authentication(struct iscsi_portal_group *tpg, u32 authentication)
 {
 	unsigned char buf1[256], buf2[256], *none = NULL;
diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
index 9db32bd24cd4..2da211920c18 100644
--- a/drivers/target/iscsi/iscsi_target_tpg.h
+++ b/drivers/target/iscsi/iscsi_target_tpg.h
@@ -26,8 +26,6 @@ extern struct iscsi_tpg_np *iscsit_tpg_add_network_portal(struct iscsi_portal_gr
 			int);
 extern int iscsit_tpg_del_network_portal(struct iscsi_portal_group *,
 			struct iscsi_tpg_np *);
-extern int iscsit_tpg_set_initiator_node_queue_depth(struct iscsi_portal_group *,
-			unsigned char *, u32, int);
 extern int iscsit_ta_authentication(struct iscsi_portal_group *, u32);
 extern int iscsit_ta_login_timeout(struct iscsi_portal_group *, u32);
 extern int iscsit_ta_netif_timeout(struct iscsi_portal_group *, u32);
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index 4fb0eca86857..d41a5c300e31 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -1036,12 +1036,26 @@ static ssize_t tcm_loop_tpg_transport_status_store(struct config_item *item,
 	return -EINVAL;
 }
 
+static ssize_t tcm_loop_tpg_address_show(struct config_item *item,
+					 char *page)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	struct tcm_loop_tpg *tl_tpg = container_of(se_tpg,
+			struct tcm_loop_tpg, tl_se_tpg);
+	struct tcm_loop_hba *tl_hba = tl_tpg->tl_hba;
+
+	return snprintf(page, PAGE_SIZE, "%d:0:%d\n",
+			tl_hba->sh->host_no, tl_tpg->tl_tpgt);
+}
+
 CONFIGFS_ATTR(tcm_loop_tpg_, nexus);
 CONFIGFS_ATTR(tcm_loop_tpg_, transport_status);
+CONFIGFS_ATTR_RO(tcm_loop_tpg_, address);
 
 static struct configfs_attribute *tcm_loop_tpg_attrs[] = {
 	&tcm_loop_tpg_attr_nexus,
 	&tcm_loop_tpg_attr_transport_status,
+	&tcm_loop_tpg_attr_address,
 	NULL,
 };
 
diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c
index 35f7d31b29d2..3072f1aca8ec 100644
--- a/drivers/target/sbp/sbp_target.c
+++ b/drivers/target/sbp/sbp_target.c
@@ -39,8 +39,6 @@
 
 #include "sbp_target.h"
 
-static const struct target_core_fabric_ops sbp_ops;
-
 /* FireWire address region for management and command block address handlers */
 static const struct fw_address_region sbp_register_region = {
 	.start	= CSR_REGISTER_BASE + 0x10000,
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index b9b9ffde4c7a..3327c49674d3 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -278,7 +278,7 @@ EXPORT_SYMBOL(target_depend_item);
 
 void target_undepend_item(struct config_item *item)
 {
-	return configfs_undepend_item(&target_core_fabrics, item);
+	return configfs_undepend_item(item);
 }
 EXPORT_SYMBOL(target_undepend_item);
 
@@ -499,6 +499,7 @@ DEF_CONFIGFS_ATTRIB_SHOW(max_unmap_lba_count);
 DEF_CONFIGFS_ATTRIB_SHOW(max_unmap_block_desc_count);
 DEF_CONFIGFS_ATTRIB_SHOW(unmap_granularity);
 DEF_CONFIGFS_ATTRIB_SHOW(unmap_granularity_alignment);
+DEF_CONFIGFS_ATTRIB_SHOW(unmap_zeroes_data);
 DEF_CONFIGFS_ATTRIB_SHOW(max_write_same_len);
 
 #define DEF_CONFIGFS_ATTRIB_STORE_U32(_name)				\
@@ -548,7 +549,8 @@ static ssize_t _name##_store(struct config_item *item, const char *page,\
 		size_t count)						\
 {									\
 	printk_once(KERN_WARNING					\
-		"ignoring deprecated ##_name## attribute\n");	\
+		"ignoring deprecated %s attribute\n",			\
+		__stringify(_name));					\
 	return count;							\
 }
 
@@ -866,6 +868,39 @@ static ssize_t emulate_rest_reord_store(struct config_item *item,
 	return count;
 }
 
+static ssize_t unmap_zeroes_data_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_dev_attrib *da = to_attrib(item);
+	bool flag;
+	int ret;
+
+	ret = strtobool(page, &flag);
+	if (ret < 0)
+		return ret;
+
+	if (da->da_dev->export_count) {
+		pr_err("dev[%p]: Unable to change SE Device"
+		       " unmap_zeroes_data while export_count is %d\n",
+		       da->da_dev, da->da_dev->export_count);
+		return -EINVAL;
+	}
+	/*
+	 * We expect this value to be non-zero when generic Block Layer
+	 * Discard supported is detected iblock_configure_device().
+	 */
+	if (flag && !da->max_unmap_block_desc_count) {
+		pr_err("dev[%p]: Thin Provisioning LBPRZ will not be set"
+		       " because max_unmap_block_desc_count is zero\n",
+		       da->da_dev);
+		return -ENOSYS;
+	}
+	da->unmap_zeroes_data = flag;
+	pr_debug("dev[%p]: SE Device Thin Provisioning LBPRZ bit: %d\n",
+		 da->da_dev, flag);
+	return 0;
+}
+
 /*
  * Note, this can only be called on unexported SE Device Object.
  */
@@ -998,6 +1033,7 @@ CONFIGFS_ATTR(, max_unmap_lba_count);
 CONFIGFS_ATTR(, max_unmap_block_desc_count);
 CONFIGFS_ATTR(, unmap_granularity);
 CONFIGFS_ATTR(, unmap_granularity_alignment);
+CONFIGFS_ATTR(, unmap_zeroes_data);
 CONFIGFS_ATTR(, max_write_same_len);
 
 /*
@@ -1034,6 +1070,7 @@ struct configfs_attribute *sbc_attrib_attrs[] = {
 	&attr_max_unmap_block_desc_count,
 	&attr_unmap_granularity,
 	&attr_unmap_granularity_alignment,
+	&attr_unmap_zeroes_data,
 	&attr_max_write_same_len,
 	NULL,
 };
@@ -1980,14 +2017,14 @@ static ssize_t target_dev_lba_map_store(struct config_item *item,
 	struct se_device *dev = to_device(item);
 	struct t10_alua_lba_map *lba_map = NULL;
 	struct list_head lba_list;
-	char *map_entries, *ptr;
+	char *map_entries, *orig, *ptr;
 	char state;
 	int pg_num = -1, pg;
 	int ret = 0, num = 0, pg_id, alua_state;
 	unsigned long start_lba = -1, end_lba = -1;
 	unsigned long segment_size = -1, segment_mult = -1;
 
-	map_entries = kstrdup(page, GFP_KERNEL);
+	orig = map_entries = kstrdup(page, GFP_KERNEL);
 	if (!map_entries)
 		return -ENOMEM;
 
@@ -2085,7 +2122,7 @@ out:
 	} else
 		core_alua_set_lba_map(dev, &lba_list,
 				      segment_size, segment_mult);
-	kfree(map_entries);
+	kfree(orig);
 	return count;
 }
 
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 88ea4e4f124b..cacd97a8cbd0 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -813,6 +813,8 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 	dev->dev_attrib.unmap_granularity = DA_UNMAP_GRANULARITY_DEFAULT;
 	dev->dev_attrib.unmap_granularity_alignment =
 				DA_UNMAP_GRANULARITY_ALIGNMENT_DEFAULT;
+	dev->dev_attrib.unmap_zeroes_data =
+				DA_UNMAP_ZEROES_DATA_DEFAULT;
 	dev->dev_attrib.max_write_same_len = DA_MAX_WRITE_SAME_LEN;
 
 	xcopy_lun = &dev->xcopy_lun;
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index f29c69120054..5a2899f9f50b 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -138,6 +138,8 @@ static int iblock_configure_device(struct se_device *dev)
 				q->limits.discard_granularity >> 9;
 		dev->dev_attrib.unmap_granularity_alignment =
 				q->limits.discard_alignment;
+		dev->dev_attrib.unmap_zeroes_data =
+				q->limits.discard_zeroes_data;
 
 		pr_debug("IBLOCK: BLOCK Discard support available,"
 				" disabled by default\n");
@@ -613,9 +615,9 @@ iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio)
 	}
 
 	bip = bio_integrity_alloc(bio, GFP_NOIO, cmd->t_prot_nents);
-	if (!bip) {
+	if (IS_ERR(bip)) {
 		pr_err("Unable to allocate bio_integrity_payload\n");
-		return -ENOMEM;
+		return PTR_ERR(bip);
 	}
 
 	bip->bip_iter.bi_size = (cmd->data_length / dev->dev_attrib.block_size) *
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index e7933115087a..b1795735eafc 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -1457,8 +1457,7 @@ static void core_scsi3_nodeacl_undepend_item(struct se_node_acl *nacl)
 static int core_scsi3_lunacl_depend_item(struct se_dev_entry *se_deve)
 {
 	struct se_lun_acl *lun_acl;
-	struct se_node_acl *nacl;
-	struct se_portal_group *tpg;
+
 	/*
 	 * For nacl->dynamic_node_acl=1
 	 */
@@ -1467,17 +1466,13 @@ static int core_scsi3_lunacl_depend_item(struct se_dev_entry *se_deve)
 	if (!lun_acl)
 		return 0;
 
-	nacl = lun_acl->se_lun_nacl;
-	tpg = nacl->se_tpg;
-
 	return target_depend_item(&lun_acl->se_lun_group.cg_item);
 }
 
 static void core_scsi3_lunacl_undepend_item(struct se_dev_entry *se_deve)
 {
 	struct se_lun_acl *lun_acl;
-	struct se_node_acl *nacl;
-	struct se_portal_group *tpg;
+
 	/*
 	 * For nacl->dynamic_node_acl=1
 	 */
@@ -1487,8 +1482,6 @@ static void core_scsi3_lunacl_undepend_item(struct se_dev_entry *se_deve)
 		kref_put(&se_deve->pr_kref, target_pr_kref_release);
 		return;
 	}
-	nacl = lun_acl->se_lun_nacl;
-	tpg = nacl->se_tpg;
 
 	target_undepend_item(&lun_acl->se_lun_group.cg_item);
 	kref_put(&se_deve->pr_kref, target_pr_kref_release);
diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c
index 98698d875742..a9057aa07176 100644
--- a/drivers/target/target_core_sbc.c
+++ b/drivers/target/target_core_sbc.c
@@ -141,9 +141,17 @@ sbc_emulate_readcapacity_16(struct se_cmd *cmd)
 	 * Set Thin Provisioning Enable bit following sbc3r22 in section
 	 * READ CAPACITY (16) byte 14 if emulate_tpu or emulate_tpws is enabled.
 	 */
-	if (dev->dev_attrib.emulate_tpu || dev->dev_attrib.emulate_tpws)
+	if (dev->dev_attrib.emulate_tpu || dev->dev_attrib.emulate_tpws) {
 		buf[14] |= 0x80;
 
+		/*
+		 * LBPRZ signifies that zeroes will be read back from an LBA after
+		 * an UNMAP or WRITE SAME w/ unmap bit (sbc3r36 5.16.2)
+		 */
+		if (dev->dev_attrib.unmap_zeroes_data)
+			buf[14] |= 0x40;
+	}
+
 	rbuf = transport_kmap_data_sg(cmd);
 	if (rbuf) {
 		memcpy(rbuf, buf, min_t(u32, sizeof(buf), cmd->data_length));
diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c
index 9413e1a949e5..0aa47babd16c 100644
--- a/drivers/target/target_core_spc.c
+++ b/drivers/target/target_core_spc.c
@@ -635,6 +635,18 @@ spc_emulate_evpd_b2(struct se_cmd *cmd, unsigned char *buf)
 	if (dev->dev_attrib.emulate_tpws != 0)
 		buf[5] |= 0x40 | 0x20;
 
+	/*
+	 * The unmap_zeroes_data set means that the underlying device supports
+	 * REQ_DISCARD and has the discard_zeroes_data bit set. This satisfies
+	 * the SBC requirements for LBPRZ, meaning that a subsequent read
+	 * will return zeroes after an UNMAP or WRITE SAME (16) to an LBA
+	 * See sbc4r36 6.6.4.
+	 */
+	if (((dev->dev_attrib.emulate_tpu != 0) ||
+	     (dev->dev_attrib.emulate_tpws != 0)) &&
+	     (dev->dev_attrib.unmap_zeroes_data != 0))
+		buf[5] |= 0x04;
+
 	return 0;
 }
 
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index 28fb3016370f..fcdcb117c60d 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -201,7 +201,7 @@ static void core_tmr_drain_tmr_list(
 		/*
 		 * If this function was called with a valid pr_res_key
 		 * parameter (eg: for PROUT PREEMPT_AND_ABORT service action
-		 * skip non regisration key matching TMRs.
+		 * skip non registration key matching TMRs.
 		 */
 		if (target_check_cdb_and_preempt(preempt_and_abort_list, cmd))
 			continue;
diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
index 5fb9dd7f08bb..3608b1b5ecf7 100644
--- a/drivers/target/target_core_tpg.c
+++ b/drivers/target/target_core_tpg.c
@@ -75,9 +75,21 @@ struct se_node_acl *core_tpg_get_initiator_node_acl(
 	unsigned char *initiatorname)
 {
 	struct se_node_acl *acl;
-
+	/*
+	 * Obtain se_node_acl->acl_kref using fabric driver provided
+	 * initiatorname[] during node acl endpoint lookup driven by
+	 * new se_session login.
+	 *
+	 * The reference is held until se_session shutdown -> release
+	 * occurs via fabric driver invoked transport_deregister_session()
+	 * or transport_free_session() code.
+	 */
 	mutex_lock(&tpg->acl_node_mutex);
 	acl = __core_tpg_get_initiator_node_acl(tpg, initiatorname);
+	if (acl) {
+		if (!kref_get_unless_zero(&acl->acl_kref))
+			acl = NULL;
+	}
 	mutex_unlock(&tpg->acl_node_mutex);
 
 	return acl;
@@ -157,28 +169,25 @@ void core_tpg_add_node_to_devs(
 	mutex_unlock(&tpg->tpg_lun_mutex);
 }
 
-/*      core_set_queue_depth_for_node():
- *
- *
- */
-static int core_set_queue_depth_for_node(
-	struct se_portal_group *tpg,
-	struct se_node_acl *acl)
+static void
+target_set_nacl_queue_depth(struct se_portal_group *tpg,
+			    struct se_node_acl *acl, u32 queue_depth)
 {
+	acl->queue_depth = queue_depth;
+
 	if (!acl->queue_depth) {
-		pr_err("Queue depth for %s Initiator Node: %s is 0,"
+		pr_warn("Queue depth for %s Initiator Node: %s is 0,"
 			"defaulting to 1.\n", tpg->se_tpg_tfo->get_fabric_name(),
 			acl->initiatorname);
 		acl->queue_depth = 1;
 	}
-
-	return 0;
 }
 
 static struct se_node_acl *target_alloc_node_acl(struct se_portal_group *tpg,
 		const unsigned char *initiatorname)
 {
 	struct se_node_acl *acl;
+	u32 queue_depth;
 
 	acl = kzalloc(max(sizeof(*acl), tpg->se_tpg_tfo->node_acl_size),
 			GFP_KERNEL);
@@ -193,24 +202,20 @@ static struct se_node_acl *target_alloc_node_acl(struct se_portal_group *tpg,
 	spin_lock_init(&acl->nacl_sess_lock);
 	mutex_init(&acl->lun_entry_mutex);
 	atomic_set(&acl->acl_pr_ref_count, 0);
+
 	if (tpg->se_tpg_tfo->tpg_get_default_depth)
-		acl->queue_depth = tpg->se_tpg_tfo->tpg_get_default_depth(tpg);
+		queue_depth = tpg->se_tpg_tfo->tpg_get_default_depth(tpg);
 	else
-		acl->queue_depth = 1;
+		queue_depth = 1;
+	target_set_nacl_queue_depth(tpg, acl, queue_depth);
+
 	snprintf(acl->initiatorname, TRANSPORT_IQN_LEN, "%s", initiatorname);
 	acl->se_tpg = tpg;
 	acl->acl_index = scsi_get_new_index(SCSI_AUTH_INTR_INDEX);
 
 	tpg->se_tpg_tfo->set_default_node_attributes(acl);
 
-	if (core_set_queue_depth_for_node(tpg, acl) < 0)
-		goto out_free_acl;
-
 	return acl;
-
-out_free_acl:
-	kfree(acl);
-	return NULL;
 }
 
 static void target_add_node_acl(struct se_node_acl *acl)
@@ -219,7 +224,6 @@ static void target_add_node_acl(struct se_node_acl *acl)
 
 	mutex_lock(&tpg->acl_node_mutex);
 	list_add_tail(&acl->acl_list, &tpg->acl_node_list);
-	tpg->num_node_acls++;
 	mutex_unlock(&tpg->acl_node_mutex);
 
 	pr_debug("%s_TPG[%hu] - Added %s ACL with TCQ Depth: %d for %s"
@@ -232,6 +236,25 @@ static void target_add_node_acl(struct se_node_acl *acl)
 		acl->initiatorname);
 }
 
+bool target_tpg_has_node_acl(struct se_portal_group *tpg,
+			     const char *initiatorname)
+{
+	struct se_node_acl *acl;
+	bool found = false;
+
+	mutex_lock(&tpg->acl_node_mutex);
+	list_for_each_entry(acl, &tpg->acl_node_list, acl_list) {
+		if (!strcmp(acl->initiatorname, initiatorname)) {
+			found = true;
+			break;
+		}
+	}
+	mutex_unlock(&tpg->acl_node_mutex);
+
+	return found;
+}
+EXPORT_SYMBOL(target_tpg_has_node_acl);
+
 struct se_node_acl *core_tpg_check_initiator_node_acl(
 	struct se_portal_group *tpg,
 	unsigned char *initiatorname)
@@ -248,6 +271,15 @@ struct se_node_acl *core_tpg_check_initiator_node_acl(
 	acl = target_alloc_node_acl(tpg, initiatorname);
 	if (!acl)
 		return NULL;
+	/*
+	 * When allocating a dynamically generated node_acl, go ahead
+	 * and take the extra kref now before returning to the fabric
+	 * driver caller.
+	 *
+	 * Note this reference will be released at session shutdown
+	 * time within transport_free_session() code.
+	 */
+	kref_get(&acl->acl_kref);
 	acl->dynamic_node_acl = 1;
 
 	/*
@@ -318,7 +350,6 @@ void core_tpg_del_initiator_node_acl(struct se_node_acl *acl)
 		acl->dynamic_node_acl = 0;
 	}
 	list_del(&acl->acl_list);
-	tpg->num_node_acls--;
 	mutex_unlock(&tpg->acl_node_mutex);
 
 	spin_lock_irqsave(&acl->nacl_sess_lock, flags);
@@ -329,7 +360,8 @@ void core_tpg_del_initiator_node_acl(struct se_node_acl *acl)
 		if (sess->sess_tearing_down != 0)
 			continue;
 
-		target_get_session(sess);
+		if (!target_get_session(sess))
+			continue;
 		list_move(&sess->sess_acl_list, &sess_list);
 	}
 	spin_unlock_irqrestore(&acl->nacl_sess_lock, flags);
@@ -366,108 +398,52 @@ void core_tpg_del_initiator_node_acl(struct se_node_acl *acl)
  *
  */
 int core_tpg_set_initiator_node_queue_depth(
-	struct se_portal_group *tpg,
-	unsigned char *initiatorname,
-	u32 queue_depth,
-	int force)
+	struct se_node_acl *acl,
+	u32 queue_depth)
 {
-	struct se_session *sess, *init_sess = NULL;
-	struct se_node_acl *acl;
+	LIST_HEAD(sess_list);
+	struct se_portal_group *tpg = acl->se_tpg;
+	struct se_session *sess, *sess_tmp;
 	unsigned long flags;
-	int dynamic_acl = 0;
-
-	mutex_lock(&tpg->acl_node_mutex);
-	acl = __core_tpg_get_initiator_node_acl(tpg, initiatorname);
-	if (!acl) {
-		pr_err("Access Control List entry for %s Initiator"
-			" Node %s does not exists for TPG %hu, ignoring"
-			" request.\n", tpg->se_tpg_tfo->get_fabric_name(),
-			initiatorname, tpg->se_tpg_tfo->tpg_get_tag(tpg));
-		mutex_unlock(&tpg->acl_node_mutex);
-		return -ENODEV;
-	}
-	if (acl->dynamic_node_acl) {
-		acl->dynamic_node_acl = 0;
-		dynamic_acl = 1;
-	}
-	mutex_unlock(&tpg->acl_node_mutex);
-
-	spin_lock_irqsave(&tpg->session_lock, flags);
-	list_for_each_entry(sess, &tpg->tpg_sess_list, sess_list) {
-		if (sess->se_node_acl != acl)
-			continue;
-
-		if (!force) {
-			pr_err("Unable to change queue depth for %s"
-				" Initiator Node: %s while session is"
-				" operational.  To forcefully change the queue"
-				" depth and force session reinstatement"
-				" use the \"force=1\" parameter.\n",
-				tpg->se_tpg_tfo->get_fabric_name(), initiatorname);
-			spin_unlock_irqrestore(&tpg->session_lock, flags);
-
-			mutex_lock(&tpg->acl_node_mutex);
-			if (dynamic_acl)
-				acl->dynamic_node_acl = 1;
-			mutex_unlock(&tpg->acl_node_mutex);
-			return -EEXIST;
-		}
-		/*
-		 * Determine if the session needs to be closed by our context.
-		 */
-		if (!tpg->se_tpg_tfo->shutdown_session(sess))
-			continue;
-
-		init_sess = sess;
-		break;
-	}
+	int rc;
 
 	/*
 	 * User has requested to change the queue depth for a Initiator Node.
 	 * Change the value in the Node's struct se_node_acl, and call
-	 * core_set_queue_depth_for_node() to add the requested queue depth.
-	 *
-	 * Finally call  tpg->se_tpg_tfo->close_session() to force session
-	 * reinstatement to occur if there is an active session for the
-	 * $FABRIC_MOD Initiator Node in question.
+	 * target_set_nacl_queue_depth() to set the new queue depth.
 	 */
-	acl->queue_depth = queue_depth;
+	target_set_nacl_queue_depth(tpg, acl, queue_depth);
+
+	spin_lock_irqsave(&acl->nacl_sess_lock, flags);
+	list_for_each_entry_safe(sess, sess_tmp, &acl->acl_sess_list,
+				 sess_acl_list) {
+		if (sess->sess_tearing_down != 0)
+			continue;
+		if (!target_get_session(sess))
+			continue;
+		spin_unlock_irqrestore(&acl->nacl_sess_lock, flags);
 
-	if (core_set_queue_depth_for_node(tpg, acl) < 0) {
-		spin_unlock_irqrestore(&tpg->session_lock, flags);
 		/*
-		 * Force session reinstatement if
-		 * core_set_queue_depth_for_node() failed, because we assume
-		 * the $FABRIC_MOD has already the set session reinstatement
-		 * bit from tpg->se_tpg_tfo->shutdown_session() called above.
+		 * Finally call tpg->se_tpg_tfo->close_session() to force session
+		 * reinstatement to occur if there is an active session for the
+		 * $FABRIC_MOD Initiator Node in question.
 		 */
-		if (init_sess)
-			tpg->se_tpg_tfo->close_session(init_sess);
-
-		mutex_lock(&tpg->acl_node_mutex);
-		if (dynamic_acl)
-			acl->dynamic_node_acl = 1;
-		mutex_unlock(&tpg->acl_node_mutex);
-		return -EINVAL;
+		rc = tpg->se_tpg_tfo->shutdown_session(sess);
+		target_put_session(sess);
+		if (!rc) {
+			spin_lock_irqsave(&acl->nacl_sess_lock, flags);
+			continue;
+		}
+		target_put_session(sess);
+		spin_lock_irqsave(&acl->nacl_sess_lock, flags);
 	}
-	spin_unlock_irqrestore(&tpg->session_lock, flags);
-	/*
-	 * If the $FABRIC_MOD session for the Initiator Node ACL exists,
-	 * forcefully shutdown the $FABRIC_MOD session/nexus.
-	 */
-	if (init_sess)
-		tpg->se_tpg_tfo->close_session(init_sess);
+	spin_unlock_irqrestore(&acl->nacl_sess_lock, flags);
 
 	pr_debug("Successfully changed queue depth to: %d for Initiator"
-		" Node: %s on %s Target Portal Group: %u\n", queue_depth,
-		initiatorname, tpg->se_tpg_tfo->get_fabric_name(),
+		" Node: %s on %s Target Portal Group: %u\n", acl->queue_depth,
+		acl->initiatorname, tpg->se_tpg_tfo->get_fabric_name(),
 		tpg->se_tpg_tfo->tpg_get_tag(tpg));
 
-	mutex_lock(&tpg->acl_node_mutex);
-	if (dynamic_acl)
-		acl->dynamic_node_acl = 1;
-	mutex_unlock(&tpg->acl_node_mutex);
-
 	return 0;
 }
 EXPORT_SYMBOL(core_tpg_set_initiator_node_queue_depth);
@@ -595,7 +571,6 @@ int core_tpg_deregister(struct se_portal_group *se_tpg)
 	 */
 	list_for_each_entry_safe(nacl, nacl_tmp, &node_list, acl_list) {
 		list_del(&nacl->acl_list);
-		se_tpg->num_node_acls--;
 
 		core_tpg_wait_for_nacl_pr_ref(nacl);
 		core_free_device_list_for_node(nacl, se_tpg);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 4fdcee2006d1..9f3608e10f25 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -341,7 +341,6 @@ void __transport_register_session(
 					&buf[0], PR_REG_ISID_LEN);
 			se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
 		}
-		kref_get(&se_nacl->acl_kref);
 
 		spin_lock_irq(&se_nacl->nacl_sess_lock);
 		/*
@@ -384,9 +383,9 @@ static void target_release_session(struct kref *kref)
 	se_tpg->se_tpg_tfo->close_session(se_sess);
 }
 
-void target_get_session(struct se_session *se_sess)
+int target_get_session(struct se_session *se_sess)
 {
-	kref_get(&se_sess->sess_kref);
+	return kref_get_unless_zero(&se_sess->sess_kref);
 }
 EXPORT_SYMBOL(target_get_session);
 
@@ -432,6 +431,7 @@ void target_put_nacl(struct se_node_acl *nacl)
 {
 	kref_put(&nacl->acl_kref, target_complete_nacl);
 }
+EXPORT_SYMBOL(target_put_nacl);
 
 void transport_deregister_session_configfs(struct se_session *se_sess)
 {
@@ -464,6 +464,15 @@ EXPORT_SYMBOL(transport_deregister_session_configfs);
 
 void transport_free_session(struct se_session *se_sess)
 {
+	struct se_node_acl *se_nacl = se_sess->se_node_acl;
+	/*
+	 * Drop the se_node_acl->nacl_kref obtained from within
+	 * core_tpg_get_initiator_node_acl().
+	 */
+	if (se_nacl) {
+		se_sess->se_node_acl = NULL;
+		target_put_nacl(se_nacl);
+	}
 	if (se_sess->sess_cmd_map) {
 		percpu_ida_destroy(&se_sess->sess_tag_pool);
 		kvfree(se_sess->sess_cmd_map);
@@ -478,7 +487,7 @@ void transport_deregister_session(struct se_session *se_sess)
 	const struct target_core_fabric_ops *se_tfo;
 	struct se_node_acl *se_nacl;
 	unsigned long flags;
-	bool comp_nacl = true, drop_nacl = false;
+	bool drop_nacl = false;
 
 	if (!se_tpg) {
 		transport_free_session(se_sess);
@@ -502,7 +511,6 @@ void transport_deregister_session(struct se_session *se_sess)
 	if (se_nacl && se_nacl->dynamic_node_acl) {
 		if (!se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
 			list_del(&se_nacl->acl_list);
-			se_tpg->num_node_acls--;
 			drop_nacl = true;
 		}
 	}
@@ -511,18 +519,16 @@ void transport_deregister_session(struct se_session *se_sess)
 	if (drop_nacl) {
 		core_tpg_wait_for_nacl_pr_ref(se_nacl);
 		core_free_device_list_for_node(se_nacl, se_tpg);
+		se_sess->se_node_acl = NULL;
 		kfree(se_nacl);
-		comp_nacl = false;
 	}
 	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
 		se_tpg->se_tpg_tfo->get_fabric_name());
 	/*
 	 * If last kref is dropping now for an explicit NodeACL, awake sleeping
 	 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
-	 * removal context.
+	 * removal context from within transport_free_session() code.
 	 */
-	if (se_nacl && comp_nacl)
-		target_put_nacl(se_nacl);
 
 	transport_free_session(se_sess);
 }
@@ -715,7 +721,10 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
 	cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
 	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 
-	queue_work(target_completion_wq, &cmd->work);
+	if (cmd->cpuid == -1)
+		queue_work(target_completion_wq, &cmd->work);
+	else
+		queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
 }
 EXPORT_SYMBOL(target_complete_cmd);
 
@@ -1309,7 +1318,7 @@ EXPORT_SYMBOL(target_setup_cmd_from_cdb);
 
 /*
  * Used by fabric module frontends to queue tasks directly.
- * Many only be used from process context only
+ * May only be used from process context.
  */
 int transport_handle_cdb_direct(
 	struct se_cmd *cmd)
@@ -1582,7 +1591,7 @@ static void target_complete_tmr_failure(struct work_struct *work)
 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
 		unsigned char *sense, u64 unpacked_lun,
 		void *fabric_tmr_ptr, unsigned char tm_type,
-		gfp_t gfp, unsigned int tag, int flags)
+		gfp_t gfp, u64 tag, int flags)
 {
 	struct se_portal_group *se_tpg;
 	int ret;
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index 5e6d6cb348fc..dd600e5ead71 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -152,6 +152,7 @@ static struct genl_family tcmu_genl_family = {
 	.maxattr = TCMU_ATTR_MAX,
 	.mcgrps = tcmu_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
+	.netnsok = true,
 };
 
 static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
@@ -194,7 +195,7 @@ static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
 
 static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
 {
-	unsigned long offset = (unsigned long) vaddr & ~PAGE_MASK;
+	unsigned long offset = offset_in_page(vaddr);
 
 	size = round_up(size+offset, PAGE_SIZE);
 	vaddr -= offset;
@@ -840,7 +841,7 @@ static int tcmu_netlink_event(enum tcmu_genl_cmd cmd, const char *name, int mino
 
 	genlmsg_end(skb, msg_header);
 
-	ret = genlmsg_multicast(&tcmu_genl_family, skb, 0,
+	ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
 				TCMU_MCGRP_CONFIG, GFP_KERNEL);
 
 	/* We don't care if no one is listening */
@@ -917,8 +918,10 @@ static int tcmu_configure_device(struct se_device *dev)
 	if (ret)
 		goto err_register;
 
+	/* User can set hw_block_size before enable the device */
+	if (dev->dev_attrib.hw_block_size == 0)
+		dev->dev_attrib.hw_block_size = 512;
 	/* Other attributes can be configured in userspace */
-	dev->dev_attrib.hw_block_size = 512;
 	dev->dev_attrib.hw_max_sectors = 128;
 	dev->dev_attrib.hw_queue_depth = 128;
 
diff --git a/drivers/target/tcm_fc/tcm_fc.h b/drivers/target/tcm_fc/tcm_fc.h
index 39909dadef3e..c30003bd4ff0 100644
--- a/drivers/target/tcm_fc/tcm_fc.h
+++ b/drivers/target/tcm_fc/tcm_fc.h
@@ -166,7 +166,6 @@ void ft_aborted_task(struct se_cmd *);
  */
 void ft_recv_req(struct ft_sess *, struct fc_frame *);
 struct ft_tpg *ft_lport_find_tpg(struct fc_lport *);
-struct ft_node_acl *ft_acl_get(struct ft_tpg *, struct fc_rport_priv *);
 
 void ft_recv_write_data(struct ft_cmd *, struct fc_frame *);
 void ft_dump_cmd(struct ft_cmd *, const char *caller);
diff --git a/drivers/target/tcm_fc/tfc_conf.c b/drivers/target/tcm_fc/tfc_conf.c
index 85aeaa0ad303..4d375e95841b 100644
--- a/drivers/target/tcm_fc/tfc_conf.c
+++ b/drivers/target/tcm_fc/tfc_conf.c
@@ -171,9 +171,31 @@ static ssize_t ft_nacl_node_name_store(struct config_item *item,
 CONFIGFS_ATTR(ft_nacl_, node_name);
 CONFIGFS_ATTR(ft_nacl_, port_name);
 
+static ssize_t ft_nacl_tag_show(struct config_item *item,
+		char *page)
+{
+	return snprintf(page, PAGE_SIZE, "%s", acl_to_nacl(item)->acl_tag);
+}
+
+static ssize_t ft_nacl_tag_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_node_acl *se_nacl = acl_to_nacl(item);
+	int ret;
+
+	ret = core_tpg_set_initiator_node_tag(se_nacl->se_tpg, se_nacl, page);
+
+	if (ret < 0)
+		return ret;
+	return count;
+}
+
+CONFIGFS_ATTR(ft_nacl_, tag);
+
 static struct configfs_attribute *ft_nacl_base_attrs[] = {
 	&ft_nacl_attr_port_name,
 	&ft_nacl_attr_node_name,
+	&ft_nacl_attr_tag,
 	NULL,
 };
 
@@ -198,31 +220,6 @@ static int ft_init_nodeacl(struct se_node_acl *nacl, const char *name)
 	return 0;
 }
 
-struct ft_node_acl *ft_acl_get(struct ft_tpg *tpg, struct fc_rport_priv *rdata)
-{
-	struct ft_node_acl *found = NULL;
-	struct ft_node_acl *acl;
-	struct se_portal_group *se_tpg = &tpg->se_tpg;
-	struct se_node_acl *se_acl;
-
-	mutex_lock(&se_tpg->acl_node_mutex);
-	list_for_each_entry(se_acl, &se_tpg->acl_node_list, acl_list) {
-		acl = container_of(se_acl, struct ft_node_acl, se_node_acl);
-		pr_debug("acl %p port_name %llx\n",
-			acl, (unsigned long long)acl->node_auth.port_name);
-		if (acl->node_auth.port_name == rdata->ids.port_name ||
-		    acl->node_auth.node_name == rdata->ids.node_name) {
-			pr_debug("acl %p port_name %llx matched\n", acl,
-				    (unsigned long long)rdata->ids.port_name);
-			found = acl;
-			/* XXX need to hold onto ACL */
-			break;
-		}
-	}
-	mutex_unlock(&se_tpg->acl_node_mutex);
-	return found;
-}
-
 /*
  * local_port port_group (tpg) ops.
  */
diff --git a/drivers/target/tcm_fc/tfc_io.c b/drivers/target/tcm_fc/tfc_io.c
index 847c1aa6fbf4..6f7c65abfe2a 100644
--- a/drivers/target/tcm_fc/tfc_io.c
+++ b/drivers/target/tcm_fc/tfc_io.c
@@ -154,9 +154,9 @@ int ft_queue_data_in(struct se_cmd *se_cmd)
 			BUG_ON(!page);
 			from = kmap_atomic(page + (mem_off >> PAGE_SHIFT));
 			page_addr = from;
-			from += mem_off & ~PAGE_MASK;
+			from += offset_in_page(mem_off);
 			tlen = min(tlen, (size_t)(PAGE_SIZE -
-						(mem_off & ~PAGE_MASK)));
+						offset_in_page(mem_off)));
 			memcpy(to, from, tlen);
 			kunmap_atomic(page_addr);
 			to += tlen;
@@ -314,9 +314,9 @@ void ft_recv_write_data(struct ft_cmd *cmd, struct fc_frame *fp)
 
 		to = kmap_atomic(page + (mem_off >> PAGE_SHIFT));
 		page_addr = to;
-		to += mem_off & ~PAGE_MASK;
+		to += offset_in_page(mem_off);
 		tlen = min(tlen, (size_t)(PAGE_SIZE -
-					  (mem_off & ~PAGE_MASK)));
+					  offset_in_page(mem_off)));
 		memcpy(to, from, tlen);
 		kunmap_atomic(page_addr);
 
diff --git a/drivers/target/tcm_fc/tfc_sess.c b/drivers/target/tcm_fc/tfc_sess.c
index 7b934eac995d..e19f4c58c6fa 100644
--- a/drivers/target/tcm_fc/tfc_sess.c
+++ b/drivers/target/tcm_fc/tfc_sess.c
@@ -191,10 +191,15 @@ out:
  * Caller holds ft_lport_lock.
  */
 static struct ft_sess *ft_sess_create(struct ft_tport *tport, u32 port_id,
-				      struct ft_node_acl *acl)
+				      struct fc_rport_priv *rdata)
 {
+	struct se_portal_group *se_tpg = &tport->tpg->se_tpg;
+	struct se_node_acl *se_acl;
 	struct ft_sess *sess;
 	struct hlist_head *head;
+	unsigned char initiatorname[TRANSPORT_IQN_LEN];
+
+	ft_format_wwn(&initiatorname[0], TRANSPORT_IQN_LEN, rdata->ids.port_name);
 
 	head = &tport->hash[ft_sess_hash(port_id)];
 	hlist_for_each_entry_rcu(sess, head, hash)
@@ -212,7 +217,14 @@ static struct ft_sess *ft_sess_create(struct ft_tport *tport, u32 port_id,
 		kfree(sess);
 		return NULL;
 	}
-	sess->se_sess->se_node_acl = &acl->se_node_acl;
+
+	se_acl = core_tpg_get_initiator_node_acl(se_tpg, &initiatorname[0]);
+	if (!se_acl) {
+		transport_free_session(sess->se_sess);
+		kfree(sess);
+		return NULL;
+	}
+	sess->se_sess->se_node_acl = se_acl;
 	sess->tport = tport;
 	sess->port_id = port_id;
 	kref_init(&sess->kref);	/* ref for table entry */
@@ -221,7 +233,7 @@ static struct ft_sess *ft_sess_create(struct ft_tport *tport, u32 port_id,
 
 	pr_debug("port_id %x sess %p\n", port_id, sess);
 
-	transport_register_session(&tport->tpg->se_tpg, &acl->se_node_acl,
+	transport_register_session(&tport->tpg->se_tpg, se_acl,
 				   sess->se_sess, sess);
 	return sess;
 }
@@ -260,6 +272,14 @@ static struct ft_sess *ft_sess_delete(struct ft_tport *tport, u32 port_id)
 	return NULL;
 }
 
+static void ft_close_sess(struct ft_sess *sess)
+{
+	transport_deregister_session_configfs(sess->se_sess);
+	target_sess_cmd_list_set_waiting(sess->se_sess);
+	target_wait_for_sess_cmds(sess->se_sess);
+	ft_sess_put(sess);
+}
+
 /*
  * Delete all sessions from tport.
  * Caller holds ft_lport_lock.
@@ -273,8 +293,7 @@ static void ft_sess_delete_all(struct ft_tport *tport)
 	     head < &tport->hash[FT_SESS_HASH_SIZE]; head++) {
 		hlist_for_each_entry_rcu(sess, head, hash) {
 			ft_sess_unhash(sess);
-			transport_deregister_session_configfs(sess->se_sess);
-			ft_sess_put(sess);	/* release from table */
+			ft_close_sess(sess);	/* release from table */
 		}
 	}
 }
@@ -313,8 +332,7 @@ void ft_sess_close(struct se_session *se_sess)
 	pr_debug("port_id %x\n", port_id);
 	ft_sess_unhash(sess);
 	mutex_unlock(&ft_lport_lock);
-	transport_deregister_session_configfs(se_sess);
-	ft_sess_put(sess);
+	ft_close_sess(sess);
 	/* XXX Send LOGO or PRLO */
 	synchronize_rcu();		/* let transport deregister happen */
 }
@@ -343,17 +361,12 @@ static int ft_prli_locked(struct fc_rport_priv *rdata, u32 spp_len,
 {
 	struct ft_tport *tport;
 	struct ft_sess *sess;
-	struct ft_node_acl *acl;
 	u32 fcp_parm;
 
 	tport = ft_tport_get(rdata->local_port);
 	if (!tport)
 		goto not_target;	/* not a target for this local port */
 
-	acl = ft_acl_get(tport->tpg, rdata);
-	if (!acl)
-		goto not_target;	/* no target for this remote */
-
 	if (!rspp)
 		goto fill;
 
@@ -375,7 +388,7 @@ static int ft_prli_locked(struct fc_rport_priv *rdata, u32 spp_len,
 		spp->spp_flags |= FC_SPP_EST_IMG_PAIR;
 		if (!(fcp_parm & FCP_SPPF_INIT_FCN))
 			return FC_SPP_RESP_CONF;
-		sess = ft_sess_create(tport, rdata->ids.port_id, acl);
+		sess = ft_sess_create(tport, rdata->ids.port_id, rdata);
 		if (!sess)
 			return FC_SPP_RESP_RES;
 		if (!sess->params)
@@ -460,8 +473,7 @@ static void ft_prlo(struct fc_rport_priv *rdata)
 		return;
 	}
 	mutex_unlock(&ft_lport_lock);
-	transport_deregister_session_configfs(sess->se_sess);
-	ft_sess_put(sess);		/* release from table */
+	ft_close_sess(sess);		/* release from table */
 	rdata->prli_count--;
 	/* XXX TBD - clearing actions.  unit attn, see 4.10 */
 }
diff --git a/drivers/thermal/int340x_thermal/processor_thermal_device.c b/drivers/thermal/int340x_thermal/processor_thermal_device.c
index ccc0ad02d066..36fa724a36c8 100644
--- a/drivers/thermal/int340x_thermal/processor_thermal_device.c
+++ b/drivers/thermal/int340x_thermal/processor_thermal_device.c
@@ -33,6 +33,12 @@
 /* Braswell thermal reporting device */
 #define PCI_DEVICE_ID_PROC_BSW_THERMAL	0x22DC
 
+/* Broxton thermal reporting device */
+#define PCI_DEVICE_ID_PROC_BXT0_THERMAL  0x0A8C
+#define PCI_DEVICE_ID_PROC_BXT1_THERMAL  0x1A8C
+#define PCI_DEVICE_ID_PROC_BXTX_THERMAL  0x4A8C
+#define PCI_DEVICE_ID_PROC_BXTP_THERMAL  0x5A8C
+
 struct power_config {
 	u32	index;
 	u32	min_uw;
@@ -404,6 +410,10 @@ static const struct pci_device_id proc_thermal_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_HSB_THERMAL)},
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_SKL_THERMAL)},
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BSW_THERMAL)},
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXT0_THERMAL)},
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXT1_THERMAL)},
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXTX_THERMAL)},
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXTP_THERMAL)},
 	{ 0, },
 };
 
diff --git a/drivers/thermal/intel_pch_thermal.c b/drivers/thermal/intel_pch_thermal.c
index 50c7da79be83..00d81af648b8 100644
--- a/drivers/thermal/intel_pch_thermal.c
+++ b/drivers/thermal/intel_pch_thermal.c
@@ -136,7 +136,7 @@ struct pch_dev_ops {
 
 
 /* dev ops for Wildcat Point */
-static struct pch_dev_ops pch_dev_ops_wpt = {
+static const struct pch_dev_ops pch_dev_ops_wpt = {
 	.hw_init = pch_wpt_init,
 	.get_temp = pch_wpt_get_temp,
 };
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index 13d01edc7a04..44b9c485157d 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -75,11 +75,11 @@ struct rcar_thermal_priv {
 #define rcar_has_irq_support(priv)	((priv)->common->base)
 #define rcar_id_to_shift(priv)		((priv)->id * 8)
 
-#ifdef DEBUG
-# define rcar_force_update_temp(priv)	1
-#else
-# define rcar_force_update_temp(priv)	0
-#endif
+static const struct of_device_id rcar_thermal_dt_ids[] = {
+	{ .compatible = "renesas,rcar-thermal", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rcar_thermal_dt_ids);
 
 /*
  *		basic functions
@@ -203,14 +203,26 @@ err_out_unlock:
 static int rcar_thermal_get_temp(struct thermal_zone_device *zone, int *temp)
 {
 	struct rcar_thermal_priv *priv = rcar_zone_to_priv(zone);
+	int tmp;
+	int ret;
 
-	if (!rcar_has_irq_support(priv) || rcar_force_update_temp(priv))
-		rcar_thermal_update_temp(priv);
+	ret = rcar_thermal_update_temp(priv);
+	if (ret < 0)
+		return ret;
 
 	mutex_lock(&priv->lock);
-	*temp =  MCELSIUS((priv->ctemp * 5) - 65);
+	tmp =  MCELSIUS((priv->ctemp * 5) - 65);
 	mutex_unlock(&priv->lock);
 
+	if ((tmp < MCELSIUS(-45)) || (tmp > MCELSIUS(125))) {
+		struct device *dev = rcar_priv_to_dev(priv);
+
+		dev_err(dev, "it couldn't measure temperature correctly\n");
+		return -EIO;
+	}
+
+	*temp = tmp;
+
 	return 0;
 }
 
@@ -288,6 +300,9 @@ static void _rcar_thermal_irq_ctrl(struct rcar_thermal_priv *priv, int enable)
 	unsigned long flags;
 	u32 mask = 0x3 << rcar_id_to_shift(priv); /* enable Rising/Falling */
 
+	if (!rcar_has_irq_support(priv))
+		return;
+
 	spin_lock_irqsave(&common->lock, flags);
 
 	rcar_thermal_common_bset(common, INTMSK, mask, enable ? 0 : mask);
@@ -299,11 +314,15 @@ static void rcar_thermal_work(struct work_struct *work)
 {
 	struct rcar_thermal_priv *priv;
 	int cctemp, nctemp;
+	int ret;
 
 	priv = container_of(work, struct rcar_thermal_priv, work.work);
 
 	rcar_thermal_get_temp(priv->zone, &cctemp);
-	rcar_thermal_update_temp(priv);
+	ret = rcar_thermal_update_temp(priv);
+	if (ret < 0)
+		return;
+
 	rcar_thermal_irq_enable(priv);
 
 	rcar_thermal_get_temp(priv->zone, &nctemp);
@@ -368,8 +387,7 @@ static int rcar_thermal_remove(struct platform_device *pdev)
 	struct rcar_thermal_priv *priv;
 
 	rcar_thermal_for_each_priv(priv, common) {
-		if (rcar_has_irq_support(priv))
-			rcar_thermal_irq_disable(priv);
+		rcar_thermal_irq_disable(priv);
 		thermal_zone_device_unregister(priv->zone);
 	}
 
@@ -441,7 +459,9 @@ static int rcar_thermal_probe(struct platform_device *pdev)
 		mutex_init(&priv->lock);
 		INIT_LIST_HEAD(&priv->list);
 		INIT_DELAYED_WORK(&priv->work, rcar_thermal_work);
-		rcar_thermal_update_temp(priv);
+		ret = rcar_thermal_update_temp(priv);
+		if (ret < 0)
+			goto error_unregister;
 
 		priv->zone = thermal_zone_device_register("rcar_thermal",
 						1, 0, priv,
@@ -453,8 +473,7 @@ static int rcar_thermal_probe(struct platform_device *pdev)
 			goto error_unregister;
 		}
 
-		if (rcar_has_irq_support(priv))
-			rcar_thermal_irq_enable(priv);
+		rcar_thermal_irq_enable(priv);
 
 		list_move_tail(&priv->list, &common->head);
 
@@ -484,12 +503,6 @@ error_unregister:
 	return ret;
 }
 
-static const struct of_device_id rcar_thermal_dt_ids[] = {
-	{ .compatible = "renesas,rcar-thermal", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, rcar_thermal_dt_ids);
-
 static struct platform_driver rcar_thermal_driver = {
 	.driver	= {
 		.name	= "rcar_thermal",
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index e845841ab036..b58e3fb9b311 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -38,7 +38,7 @@ enum tshut_mode {
 };
 
 /**
- * the system Temperature Sensors tshut(tshut) polarity
+ * The system Temperature Sensors tshut(tshut) polarity
  * the bit 8 is tshut polarity.
  * 0: low active, 1: high active
  */
@@ -57,10 +57,10 @@ enum sensor_id {
 };
 
 /**
-* The conversion table has the adc value and temperature.
-* ADC_DECREMENT is the adc value decremnet.(e.g. v2_code_table)
-* ADC_INCREMNET is the adc value incremnet.(e.g. v3_code_table)
-*/
+ * The conversion table has the adc value and temperature.
+ * ADC_DECREMENT: the adc value is of diminishing.(e.g. v2_code_table)
+ * ADC_INCREMENT: the adc value is incremental.(e.g. v3_code_table)
+ */
 enum adc_sort_mode {
 	ADC_DECREMENT = 0,
 	ADC_INCREMENT,
@@ -72,16 +72,17 @@ enum adc_sort_mode {
  */
 #define SOC_MAX_SENSORS	2
 
+/**
+ * struct chip_tsadc_table: hold information about chip-specific differences
+ * @id: conversion table
+ * @length: size of conversion table
+ * @data_mask: mask to apply on data inputs
+ * @mode: sort mode of this adc variant (incrementing or decrementing)
+ */
 struct chip_tsadc_table {
 	const struct tsadc_table *id;
-
-	/* the array table size*/
 	unsigned int length;
-
-	/* that analogic mask data */
 	u32 data_mask;
-
-	/* the sort mode is adc value that increment or decrement in table */
 	enum adc_sort_mode mode;
 };
 
@@ -153,6 +154,7 @@ struct rockchip_thermal_data {
 #define TSADCV2_SHUT_2GPIO_SRC_EN(chn)		BIT(4 + (chn))
 #define TSADCV2_SHUT_2CRU_SRC_EN(chn)		BIT(8 + (chn))
 
+#define TSADCV1_INT_PD_CLEAR_MASK		~BIT(16)
 #define TSADCV2_INT_PD_CLEAR_MASK		~BIT(8)
 
 #define TSADCV2_DATA_MASK			0xfff
@@ -168,6 +170,51 @@ struct tsadc_table {
 	int temp;
 };
 
+/**
+ * Note:
+ * Code to Temperature mapping of the Temperature sensor is a piece wise linear
+ * curve.Any temperature, code faling between to 2 give temperatures can be
+ * linearly interpolated.
+ * Code to Temperature mapping should be updated based on sillcon results.
+ */
+static const struct tsadc_table v1_code_table[] = {
+	{TSADCV3_DATA_MASK, -40000},
+	{436, -40000},
+	{431, -35000},
+	{426, -30000},
+	{421, -25000},
+	{416, -20000},
+	{411, -15000},
+	{406, -10000},
+	{401, -5000},
+	{395, 0},
+	{390, 5000},
+	{385, 10000},
+	{380, 15000},
+	{375, 20000},
+	{370, 25000},
+	{364, 30000},
+	{359, 35000},
+	{354, 40000},
+	{349, 45000},
+	{343, 50000},
+	{338, 55000},
+	{333, 60000},
+	{328, 65000},
+	{322, 70000},
+	{317, 75000},
+	{312, 80000},
+	{307, 85000},
+	{301, 90000},
+	{296, 95000},
+	{291, 100000},
+	{286, 105000},
+	{280, 110000},
+	{275, 115000},
+	{270, 120000},
+	{264, 125000},
+};
+
 static const struct tsadc_table v2_code_table[] = {
 	{TSADCV2_DATA_MASK, -40000},
 	{3800, -40000},
@@ -245,6 +292,44 @@ static const struct tsadc_table v3_code_table[] = {
 	{TSADCV3_DATA_MASK, 125000},
 };
 
+static const struct tsadc_table v4_code_table[] = {
+	{TSADCV3_DATA_MASK, -40000},
+	{431, -40000},
+	{426, -35000},
+	{421, -30000},
+	{415, -25000},
+	{410, -20000},
+	{405, -15000},
+	{399, -10000},
+	{394, -5000},
+	{389, 0},
+	{383, 5000},
+	{378, 10000},
+	{373, 15000},
+	{367, 20000},
+	{362, 25000},
+	{357, 30000},
+	{351, 35000},
+	{346, 40000},
+	{340, 45000},
+	{335, 50000},
+	{330, 55000},
+	{324, 60000},
+	{319, 65000},
+	{313, 70000},
+	{308, 75000},
+	{302, 80000},
+	{297, 85000},
+	{291, 90000},
+	{286, 95000},
+	{281, 100000},
+	{275, 105000},
+	{270, 110000},
+	{264, 115000},
+	{259, 120000},
+	{253, 125000},
+};
+
 static u32 rk_tsadcv2_temp_to_code(struct chip_tsadc_table table,
 				   int temp)
 {
@@ -368,6 +453,14 @@ static void rk_tsadcv2_initialize(void __iomem *regs,
 		       regs + TSADCV2_HIGHT_TSHUT_DEBOUNCE);
 }
 
+static void rk_tsadcv1_irq_ack(void __iomem *regs)
+{
+	u32 val;
+
+	val = readl_relaxed(regs + TSADCV2_INT_PD);
+	writel_relaxed(val & TSADCV1_INT_PD_CLEAR_MASK, regs + TSADCV2_INT_PD);
+}
+
 static void rk_tsadcv2_irq_ack(void __iomem *regs)
 {
 	u32 val;
@@ -429,6 +522,29 @@ static void rk_tsadcv2_tshut_mode(int chn, void __iomem *regs,
 	writel_relaxed(val, regs + TSADCV2_INT_EN);
 }
 
+static const struct rockchip_tsadc_chip rk3228_tsadc_data = {
+	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	.chn_num = 1, /* one channel for tsadc */
+
+	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
+	.tshut_polarity = TSHUT_LOW_ACTIVE, /* default TSHUT LOW ACTIVE */
+	.tshut_temp = 95000,
+
+	.initialize = rk_tsadcv2_initialize,
+	.irq_ack = rk_tsadcv1_irq_ack,
+	.control = rk_tsadcv2_control,
+	.get_temp = rk_tsadcv2_get_temp,
+	.set_tshut_temp = rk_tsadcv2_tshut_temp,
+	.set_tshut_mode = rk_tsadcv2_tshut_mode,
+
+	.table = {
+		.id = v1_code_table,
+		.length = ARRAY_SIZE(v1_code_table),
+		.data_mask = TSADCV3_DATA_MASK,
+		.mode = ADC_DECREMENT,
+	},
+};
+
 static const struct rockchip_tsadc_chip rk3288_tsadc_data = {
 	.chn_id[SENSOR_CPU] = 1, /* cpu sensor is channel 1 */
 	.chn_id[SENSOR_GPU] = 2, /* gpu sensor is channel 2 */
@@ -477,8 +593,36 @@ static const struct rockchip_tsadc_chip rk3368_tsadc_data = {
 	},
 };
 
+static const struct rockchip_tsadc_chip rk3399_tsadc_data = {
+	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	.chn_num = 2, /* two channels for tsadc */
+
+	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
+	.tshut_polarity = TSHUT_LOW_ACTIVE, /* default TSHUT LOW ACTIVE */
+	.tshut_temp = 95000,
+
+	.initialize = rk_tsadcv2_initialize,
+	.irq_ack = rk_tsadcv1_irq_ack,
+	.control = rk_tsadcv2_control,
+	.get_temp = rk_tsadcv2_get_temp,
+	.set_tshut_temp = rk_tsadcv2_tshut_temp,
+	.set_tshut_mode = rk_tsadcv2_tshut_mode,
+
+	.table = {
+		.id = v4_code_table,
+		.length = ARRAY_SIZE(v4_code_table),
+		.data_mask = TSADCV3_DATA_MASK,
+		.mode = ADC_DECREMENT,
+	},
+};
+
 static const struct of_device_id of_rockchip_thermal_match[] = {
 	{
+		.compatible = "rockchip,rk3228-tsadc",
+		.data = (void *)&rk3228_tsadc_data,
+	},
+	{
 		.compatible = "rockchip,rk3288-tsadc",
 		.data = (void *)&rk3288_tsadc_data,
 	},
@@ -486,6 +630,10 @@ static const struct of_device_id of_rockchip_thermal_match[] = {
 		.compatible = "rockchip,rk3368-tsadc",
 		.data = (void *)&rk3368_tsadc_data,
 	},
+	{
+		.compatible = "rockchip,rk3399-tsadc",
+		.data = (void *)&rk3399_tsadc_data,
+	},
 	{ /* end */ },
 };
 MODULE_DEVICE_TABLE(of, of_rockchip_thermal_match);
@@ -617,7 +765,7 @@ rockchip_thermal_register_sensor(struct platform_device *pdev,
 	return 0;
 }
 
-/*
+/**
  * Reset TSADC Controller, reset all tsadc registers.
  */
 static void rockchip_thermal_reset_controller(struct reset_control *reset)
diff --git a/drivers/thermal/step_wise.c b/drivers/thermal/step_wise.c
index 2f9f7086ac3d..ea9366ad3e6b 100644
--- a/drivers/thermal/step_wise.c
+++ b/drivers/thermal/step_wise.c
@@ -63,6 +63,19 @@ static unsigned long get_target_state(struct thermal_instance *instance,
 	next_target = instance->target;
 	dev_dbg(&cdev->device, "cur_state=%ld\n", cur_state);
 
+	if (!instance->initialized) {
+		if (throttle) {
+			next_target = (cur_state + 1) >= instance->upper ?
+					instance->upper :
+					((cur_state + 1) < instance->lower ?
+					instance->lower : (cur_state + 1));
+		} else {
+			next_target = THERMAL_NO_TARGET;
+		}
+
+		return next_target;
+	}
+
 	switch (trend) {
 	case THERMAL_TREND_RAISING:
 		if (throttle) {
@@ -149,7 +162,7 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 		dev_dbg(&instance->cdev->device, "old_target=%d, target=%d\n",
 					old_target, (int)instance->target);
 
-		if (old_target == instance->target)
+		if (instance->initialized && old_target == instance->target)
 			continue;
 
 		/* Activate a passive thermal instance */
@@ -161,7 +174,7 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 			instance->target == THERMAL_NO_TARGET)
 			update_passive_instance(tz, trip_type, -1);
 
-
+		instance->initialized = true;
 		instance->cdev->updated = false; /* cdev needs update */
 	}
 
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index d9e525cc9c1c..a0a8fd1235e2 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -37,6 +37,7 @@
 #include <linux/of.h>
 #include <net/netlink.h>
 #include <net/genetlink.h>
+#include <linux/suspend.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/thermal.h>
@@ -59,6 +60,8 @@ static LIST_HEAD(thermal_governor_list);
 static DEFINE_MUTEX(thermal_list_lock);
 static DEFINE_MUTEX(thermal_governor_lock);
 
+static atomic_t in_suspend;
+
 static struct thermal_governor *def_governor;
 
 static struct thermal_governor *__find_governor(const char *name)
@@ -532,14 +535,31 @@ static void update_temperature(struct thermal_zone_device *tz)
 	mutex_unlock(&tz->lock);
 
 	trace_thermal_temperature(tz);
-	dev_dbg(&tz->device, "last_temperature=%d, current_temperature=%d\n",
-				tz->last_temperature, tz->temperature);
+	if (tz->last_temperature == THERMAL_TEMP_INVALID)
+		dev_dbg(&tz->device, "last_temperature N/A, current_temperature=%d\n",
+			tz->temperature);
+	else
+		dev_dbg(&tz->device, "last_temperature=%d, current_temperature=%d\n",
+			tz->last_temperature, tz->temperature);
+}
+
+static void thermal_zone_device_reset(struct thermal_zone_device *tz)
+{
+	struct thermal_instance *pos;
+
+	tz->temperature = THERMAL_TEMP_INVALID;
+	tz->passive = 0;
+	list_for_each_entry(pos, &tz->thermal_instances, tz_node)
+		pos->initialized = false;
 }
 
 void thermal_zone_device_update(struct thermal_zone_device *tz)
 {
 	int count;
 
+	if (atomic_read(&in_suspend))
+		return;
+
 	if (!tz->ops->get_temp)
 		return;
 
@@ -676,8 +696,12 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr,
 		return -EINVAL;
 
 	ret = tz->ops->set_trip_temp(tz, trip, temperature);
+	if (ret)
+		return ret;
 
-	return ret ? ret : count;
+	thermal_zone_device_update(tz);
+
+	return count;
 }
 
 static ssize_t
@@ -1321,6 +1345,7 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz,
 	if (!result) {
 		list_add_tail(&dev->tz_node, &tz->thermal_instances);
 		list_add_tail(&dev->cdev_node, &cdev->thermal_instances);
+		atomic_set(&tz->need_update, 1);
 	}
 	mutex_unlock(&cdev->lock);
 	mutex_unlock(&tz->lock);
@@ -1430,6 +1455,7 @@ __thermal_cooling_device_register(struct device_node *np,
 				  const struct thermal_cooling_device_ops *ops)
 {
 	struct thermal_cooling_device *cdev;
+	struct thermal_zone_device *pos = NULL;
 	int result;
 
 	if (type && strlen(type) >= THERMAL_NAME_LENGTH)
@@ -1474,6 +1500,12 @@ __thermal_cooling_device_register(struct device_node *np,
 	/* Update binding information for 'this' new cdev */
 	bind_cdev(cdev);
 
+	mutex_lock(&thermal_list_lock);
+	list_for_each_entry(pos, &thermal_tz_list, node)
+		if (atomic_cmpxchg(&pos->need_update, 1, 0))
+			thermal_zone_device_update(pos);
+	mutex_unlock(&thermal_list_lock);
+
 	return cdev;
 }
 
@@ -1806,6 +1838,8 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 	tz->trips = trips;
 	tz->passive_delay = passive_delay;
 	tz->polling_delay = polling_delay;
+	/* A new thermal zone needs to be updated anyway. */
+	atomic_set(&tz->need_update, 1);
 
 	dev_set_name(&tz->device, "thermal_zone%d", tz->id);
 	result = device_register(&tz->device);
@@ -1900,7 +1934,10 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 
 	INIT_DELAYED_WORK(&(tz->poll_queue), thermal_zone_device_check);
 
-	thermal_zone_device_update(tz);
+	thermal_zone_device_reset(tz);
+	/* Update the new thermal zone and mark it as already updated. */
+	if (atomic_cmpxchg(&tz->need_update, 1, 0))
+		thermal_zone_device_update(tz);
 
 	return tz;
 
@@ -2140,6 +2177,36 @@ static void thermal_unregister_governors(void)
 	thermal_gov_power_allocator_unregister();
 }
 
+static int thermal_pm_notify(struct notifier_block *nb,
+				unsigned long mode, void *_unused)
+{
+	struct thermal_zone_device *tz;
+
+	switch (mode) {
+	case PM_HIBERNATION_PREPARE:
+	case PM_RESTORE_PREPARE:
+	case PM_SUSPEND_PREPARE:
+		atomic_set(&in_suspend, 1);
+		break;
+	case PM_POST_HIBERNATION:
+	case PM_POST_RESTORE:
+	case PM_POST_SUSPEND:
+		atomic_set(&in_suspend, 0);
+		list_for_each_entry(tz, &thermal_tz_list, node) {
+			thermal_zone_device_reset(tz);
+			thermal_zone_device_update(tz);
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static struct notifier_block thermal_pm_nb = {
+	.notifier_call = thermal_pm_notify,
+};
+
 static int __init thermal_init(void)
 {
 	int result;
@@ -2160,6 +2227,11 @@ static int __init thermal_init(void)
 	if (result)
 		goto exit_netlink;
 
+	result = register_pm_notifier(&thermal_pm_nb);
+	if (result)
+		pr_warn("Thermal: Can not register suspend notifier, return %d\n",
+			result);
+
 	return 0;
 
 exit_netlink:
@@ -2179,6 +2251,7 @@ error:
 
 static void __exit thermal_exit(void)
 {
+	unregister_pm_notifier(&thermal_pm_nb);
 	of_thermal_destroy_zones();
 	genetlink_exit();
 	class_unregister(&thermal_class);
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index d7ac1fccd659..749d41abfbab 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -41,6 +41,7 @@ struct thermal_instance {
 	struct thermal_zone_device *tz;
 	struct thermal_cooling_device *cdev;
 	int trip;
+	bool initialized;
 	unsigned long upper;	/* Highest cooling state for this trip point */
 	unsigned long lower;	/* Lowest cooling state for this trip point */
 	unsigned long target;	/* expected cooling state */
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index d27a0c62a75f..39721ec4f415 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1047,7 +1047,7 @@ config SERIAL_SGI_IOC3
 	  say Y or M.  Otherwise, say N.
 
 config SERIAL_MSM
-	bool "MSM on-chip serial port support"
+	tristate "MSM on-chip serial port support"
 	depends on ARCH_QCOM
 	select SERIAL_CORE
 
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index d45133056f51..3f98165b479c 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -1174,7 +1174,7 @@ static int max310x_probe(struct device *dev, struct max310x_devtype *devtype,
 #ifdef CONFIG_GPIOLIB
 	/* Setup GPIO cotroller */
 	s->gpio.owner		= THIS_MODULE;
-	s->gpio.dev		= dev;
+	s->gpio.parent		= dev;
 	s->gpio.label		= dev_name(dev);
 	s->gpio.direction_input	= max310x_gpio_direction_input;
 	s->gpio.get		= max310x_gpio_get;
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index 5815bcbc55b2..13f8d5f70272 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -1193,7 +1193,7 @@ static int sc16is7xx_probe(struct device *dev,
 	if (devtype->nr_gpio) {
 		/* Setup GPIO cotroller */
 		s->gpio.owner		 = THIS_MODULE;
-		s->gpio.dev		 = dev;
+		s->gpio.parent		 = dev;
 		s->gpio.label		 = dev_name(dev);
 		s->gpio.direction_input	 = sc16is7xx_gpio_direction_input;
 		s->gpio.get		 = sc16is7xx_gpio_get;
diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c
index 73190f5d2832..1a7dc3c590b1 100644
--- a/drivers/tty/serial/ucc_uart.c
+++ b/drivers/tty/serial/ucc_uart.c
@@ -31,7 +31,7 @@
 #include <linux/dma-mapping.h>
 
 #include <linux/fs_uart_pd.h>
-#include <asm/ucc_slow.h>
+#include <soc/fsl/qe/ucc_slow.h>
 
 #include <linux/firmware.h>
 #include <asm/reg.h>
diff --git a/drivers/usb/chipidea/otg_fsm.h b/drivers/usb/chipidea/otg_fsm.h
index 2689375ae5da..262d6ef8df7c 100644
--- a/drivers/usb/chipidea/otg_fsm.h
+++ b/drivers/usb/chipidea/otg_fsm.h
@@ -62,7 +62,7 @@
 /* SSEND time before SRP */
 #define TB_SSEND_SRP         (1500)	/* minimum 1.5 sec, section:5.1.2 */
 
-#ifdef CONFIG_USB_OTG_FSM
+#if IS_ENABLED(CONFIG_USB_OTG_FSM)
 
 int ci_hdrc_otg_fsm_init(struct ci_hdrc *ci);
 int ci_otg_fsm_work(struct ci_hdrc *ci);
diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c
index 460c855be0d0..14718a9ffcfb 100644
--- a/drivers/usb/core/port.c
+++ b/drivers/usb/core/port.c
@@ -249,12 +249,18 @@ static int usb_port_runtime_suspend(struct device *dev)
 
 	return retval;
 }
+
+static int usb_port_prepare(struct device *dev)
+{
+	return 1;
+}
 #endif
 
 static const struct dev_pm_ops usb_port_pm_ops = {
 #ifdef CONFIG_PM
 	.runtime_suspend =	usb_port_runtime_suspend,
 	.runtime_resume =	usb_port_runtime_resume,
+	.prepare =		usb_port_prepare,
 #endif
 };
 
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 77e4c9bc0ab1..ebb29caa3fe4 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -311,7 +311,13 @@ static int usb_dev_uevent(struct device *dev, struct kobj_uevent_env *env)
 
 static int usb_dev_prepare(struct device *dev)
 {
-	return 0;		/* Implement eventually? */
+	struct usb_device *udev = to_usb_device(dev);
+
+	/* Return 0 if the current wakeup setting is wrong, otherwise 1 */
+	if (udev->do_remote_wakeup != device_may_wakeup(dev))
+		return 0;
+
+	return 1;
 }
 
 static void usb_dev_complete(struct device *dev)
diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig
index be5aab9c13f2..af5d922a8f5d 100644
--- a/drivers/usb/gadget/Kconfig
+++ b/drivers/usb/gadget/Kconfig
@@ -205,6 +205,9 @@ config USB_F_HID
 config USB_F_PRINTER
 	tristate
 
+config USB_F_TCM
+	tristate
+
 choice
 	tristate "USB Gadget Drivers"
 	default USB_ETH
@@ -457,6 +460,20 @@ config USB_CONFIGFS_F_PRINTER
 	  For more information, see Documentation/usb/gadget_printer.txt
 	  which includes sample code for accessing the device file.
 
+config USB_CONFIGFS_F_TCM
+	bool "USB Gadget Target Fabric"
+	depends on TARGET_CORE
+	depends on USB_CONFIGFS
+	select USB_LIBCOMPOSITE
+	select USB_F_TCM
+	help
+	  This fabric is a USB gadget component. Two USB protocols are
+	  supported that is BBB or BOT (Bulk Only Transport) and UAS
+	  (USB Attached SCSI). BOT is advertised on alternative
+	  interface 0 (primary) and UAS is on alternative interface 1.
+	  Both protocols can work on USB2.0 and USB3.0.
+	  UAS utilizes the USB 3.0 feature called streams support.
+
 source "drivers/usb/gadget/legacy/Kconfig"
 
 endchoice
diff --git a/drivers/usb/gadget/function/Makefile b/drivers/usb/gadget/function/Makefile
index bd7def576955..cb8c225e8549 100644
--- a/drivers/usb/gadget/function/Makefile
+++ b/drivers/usb/gadget/function/Makefile
@@ -44,3 +44,5 @@ usb_f_hid-y			:= f_hid.o
 obj-$(CONFIG_USB_F_HID)		+= usb_f_hid.o
 usb_f_printer-y			:= f_printer.o
 obj-$(CONFIG_USB_F_PRINTER)	+= usb_f_printer.o
+usb_f_tcm-y			:= f_tcm.o
+obj-$(CONFIG_USB_F_TCM)		+= usb_f_tcm.o
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 0fbfb2b2aa08..26ccad5d8680 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -673,7 +673,7 @@ printer_fsync(struct file *fd, loff_t start, loff_t end, int datasync)
 	unsigned long		flags;
 	int			tx_list_empty;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	spin_lock_irqsave(&dev->lock, flags);
 	tx_list_empty = (likely(list_empty(&dev->tx_reqs)));
 	spin_unlock_irqrestore(&dev->lock, flags);
@@ -683,7 +683,7 @@ printer_fsync(struct file *fd, loff_t start, loff_t end, int datasync)
 		wait_event_interruptible(dev->tx_flush_wait,
 				(likely(list_empty(&dev->tx_reqs_active))));
 	}
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 
 	return 0;
 }
diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c
new file mode 100644
index 000000000000..bad007b5a190
--- /dev/null
+++ b/drivers/usb/gadget/function/f_tcm.c
@@ -0,0 +1,2381 @@
+/* Target based USB-Gadget
+ *
+ * UAS protocol handling, target callbacks, configfs handling,
+ * BBB (USB Mass Storage Class Bulk-Only (BBB) and Transport protocol handling.
+ *
+ * Author: Sebastian Andrzej Siewior <bigeasy at linutronix dot de>
+ * License: GPLv2 as published by FSF.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/configfs.h>
+#include <linux/ctype.h>
+#include <linux/usb/ch9.h>
+#include <linux/usb/composite.h>
+#include <linux/usb/gadget.h>
+#include <linux/usb/storage.h>
+#include <scsi/scsi_tcq.h>
+#include <target/target_core_base.h>
+#include <target/target_core_fabric.h>
+#include <asm/unaligned.h>
+
+#include "tcm.h"
+#include "u_tcm.h"
+#include "configfs.h"
+
+#define TPG_INSTANCES		1
+
+struct tpg_instance {
+	struct usb_function_instance	*func_inst;
+	struct usbg_tpg			*tpg;
+};
+
+static struct tpg_instance tpg_instances[TPG_INSTANCES];
+
+static DEFINE_MUTEX(tpg_instances_lock);
+
+static inline struct f_uas *to_f_uas(struct usb_function *f)
+{
+	return container_of(f, struct f_uas, function);
+}
+
+static void usbg_cmd_release(struct kref *);
+
+static inline void usbg_cleanup_cmd(struct usbg_cmd *cmd)
+{
+	kref_put(&cmd->ref, usbg_cmd_release);
+}
+
+/* Start bot.c code */
+
+static int bot_enqueue_cmd_cbw(struct f_uas *fu)
+{
+	int ret;
+
+	if (fu->flags & USBG_BOT_CMD_PEND)
+		return 0;
+
+	ret = usb_ep_queue(fu->ep_out, fu->cmd.req, GFP_ATOMIC);
+	if (!ret)
+		fu->flags |= USBG_BOT_CMD_PEND;
+	return ret;
+}
+
+static void bot_status_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct usbg_cmd *cmd = req->context;
+	struct f_uas *fu = cmd->fu;
+
+	usbg_cleanup_cmd(cmd);
+	if (req->status < 0) {
+		pr_err("ERR %s(%d)\n", __func__, __LINE__);
+		return;
+	}
+
+	/* CSW completed, wait for next CBW */
+	bot_enqueue_cmd_cbw(fu);
+}
+
+static void bot_enqueue_sense_code(struct f_uas *fu, struct usbg_cmd *cmd)
+{
+	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
+	int ret;
+	unsigned int csw_stat;
+
+	csw_stat = cmd->csw_code;
+	csw->Tag = cmd->bot_tag;
+	csw->Status = csw_stat;
+	fu->bot_status.req->context = cmd;
+	ret = usb_ep_queue(fu->ep_in, fu->bot_status.req, GFP_ATOMIC);
+	if (ret)
+		pr_err("%s(%d) ERR: %d\n", __func__, __LINE__, ret);
+}
+
+static void bot_err_compl(struct usb_ep *ep, struct usb_request *req)
+{
+	struct usbg_cmd *cmd = req->context;
+	struct f_uas *fu = cmd->fu;
+
+	if (req->status < 0)
+		pr_err("ERR %s(%d)\n", __func__, __LINE__);
+
+	if (cmd->data_len) {
+		if (cmd->data_len > ep->maxpacket) {
+			req->length = ep->maxpacket;
+			cmd->data_len -= ep->maxpacket;
+		} else {
+			req->length = cmd->data_len;
+			cmd->data_len = 0;
+		}
+
+		usb_ep_queue(ep, req, GFP_ATOMIC);
+		return;
+	}
+	bot_enqueue_sense_code(fu, cmd);
+}
+
+static void bot_send_bad_status(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
+	struct usb_request *req;
+	struct usb_ep *ep;
+
+	csw->Residue = cpu_to_le32(cmd->data_len);
+
+	if (cmd->data_len) {
+		if (cmd->is_read) {
+			ep = fu->ep_in;
+			req = fu->bot_req_in;
+		} else {
+			ep = fu->ep_out;
+			req = fu->bot_req_out;
+		}
+
+		if (cmd->data_len > fu->ep_in->maxpacket) {
+			req->length = ep->maxpacket;
+			cmd->data_len -= ep->maxpacket;
+		} else {
+			req->length = cmd->data_len;
+			cmd->data_len = 0;
+		}
+		req->complete = bot_err_compl;
+		req->context = cmd;
+		req->buf = fu->cmd.buf;
+		usb_ep_queue(ep, req, GFP_KERNEL);
+	} else {
+		bot_enqueue_sense_code(fu, cmd);
+	}
+}
+
+static int bot_send_status(struct usbg_cmd *cmd, bool moved_data)
+{
+	struct f_uas *fu = cmd->fu;
+	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
+	int ret;
+
+	if (cmd->se_cmd.scsi_status == SAM_STAT_GOOD) {
+		if (!moved_data && cmd->data_len) {
+			/*
+			 * the host wants to move data, we don't. Fill / empty
+			 * the pipe and then send the csw with reside set.
+			 */
+			cmd->csw_code = US_BULK_STAT_OK;
+			bot_send_bad_status(cmd);
+			return 0;
+		}
+
+		csw->Tag = cmd->bot_tag;
+		csw->Residue = cpu_to_le32(0);
+		csw->Status = US_BULK_STAT_OK;
+		fu->bot_status.req->context = cmd;
+
+		ret = usb_ep_queue(fu->ep_in, fu->bot_status.req, GFP_KERNEL);
+		if (ret)
+			pr_err("%s(%d) ERR: %d\n", __func__, __LINE__, ret);
+	} else {
+		cmd->csw_code = US_BULK_STAT_FAIL;
+		bot_send_bad_status(cmd);
+	}
+	return 0;
+}
+
+/*
+ * Called after command (no data transfer) or after the write (to device)
+ * operation is completed
+ */
+static int bot_send_status_response(struct usbg_cmd *cmd)
+{
+	bool moved_data = false;
+
+	if (!cmd->is_read)
+		moved_data = true;
+	return bot_send_status(cmd, moved_data);
+}
+
+/* Read request completed, now we have to send the CSW */
+static void bot_read_compl(struct usb_ep *ep, struct usb_request *req)
+{
+	struct usbg_cmd *cmd = req->context;
+
+	if (req->status < 0)
+		pr_err("ERR %s(%d)\n", __func__, __LINE__);
+
+	bot_send_status(cmd, true);
+}
+
+static int bot_send_read_response(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct usb_gadget *gadget = fuas_to_gadget(fu);
+	int ret;
+
+	if (!cmd->data_len) {
+		cmd->csw_code = US_BULK_STAT_PHASE;
+		bot_send_bad_status(cmd);
+		return 0;
+	}
+
+	if (!gadget->sg_supported) {
+		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
+		if (!cmd->data_buf)
+			return -ENOMEM;
+
+		sg_copy_to_buffer(se_cmd->t_data_sg,
+				se_cmd->t_data_nents,
+				cmd->data_buf,
+				se_cmd->data_length);
+
+		fu->bot_req_in->buf = cmd->data_buf;
+	} else {
+		fu->bot_req_in->buf = NULL;
+		fu->bot_req_in->num_sgs = se_cmd->t_data_nents;
+		fu->bot_req_in->sg = se_cmd->t_data_sg;
+	}
+
+	fu->bot_req_in->complete = bot_read_compl;
+	fu->bot_req_in->length = se_cmd->data_length;
+	fu->bot_req_in->context = cmd;
+	ret = usb_ep_queue(fu->ep_in, fu->bot_req_in, GFP_ATOMIC);
+	if (ret)
+		pr_err("%s(%d)\n", __func__, __LINE__);
+	return 0;
+}
+
+static void usbg_data_write_cmpl(struct usb_ep *, struct usb_request *);
+static int usbg_prepare_w_request(struct usbg_cmd *, struct usb_request *);
+
+static int bot_send_write_request(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct usb_gadget *gadget = fuas_to_gadget(fu);
+	int ret;
+
+	init_completion(&cmd->write_complete);
+	cmd->fu = fu;
+
+	if (!cmd->data_len) {
+		cmd->csw_code = US_BULK_STAT_PHASE;
+		return -EINVAL;
+	}
+
+	if (!gadget->sg_supported) {
+		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_KERNEL);
+		if (!cmd->data_buf)
+			return -ENOMEM;
+
+		fu->bot_req_out->buf = cmd->data_buf;
+	} else {
+		fu->bot_req_out->buf = NULL;
+		fu->bot_req_out->num_sgs = se_cmd->t_data_nents;
+		fu->bot_req_out->sg = se_cmd->t_data_sg;
+	}
+
+	fu->bot_req_out->complete = usbg_data_write_cmpl;
+	fu->bot_req_out->length = se_cmd->data_length;
+	fu->bot_req_out->context = cmd;
+
+	ret = usbg_prepare_w_request(cmd, fu->bot_req_out);
+	if (ret)
+		goto cleanup;
+	ret = usb_ep_queue(fu->ep_out, fu->bot_req_out, GFP_KERNEL);
+	if (ret)
+		pr_err("%s(%d)\n", __func__, __LINE__);
+
+	wait_for_completion(&cmd->write_complete);
+	target_execute_cmd(se_cmd);
+cleanup:
+	return ret;
+}
+
+static int bot_submit_command(struct f_uas *, void *, unsigned int);
+
+static void bot_cmd_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct f_uas *fu = req->context;
+	int ret;
+
+	fu->flags &= ~USBG_BOT_CMD_PEND;
+
+	if (req->status < 0)
+		return;
+
+	ret = bot_submit_command(fu, req->buf, req->actual);
+	if (ret)
+		pr_err("%s(%d): %d\n", __func__, __LINE__, ret);
+}
+
+static int bot_prepare_reqs(struct f_uas *fu)
+{
+	int ret;
+
+	fu->bot_req_in = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
+	if (!fu->bot_req_in)
+		goto err;
+
+	fu->bot_req_out = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
+	if (!fu->bot_req_out)
+		goto err_out;
+
+	fu->cmd.req = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
+	if (!fu->cmd.req)
+		goto err_cmd;
+
+	fu->bot_status.req = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
+	if (!fu->bot_status.req)
+		goto err_sts;
+
+	fu->bot_status.req->buf = &fu->bot_status.csw;
+	fu->bot_status.req->length = US_BULK_CS_WRAP_LEN;
+	fu->bot_status.req->complete = bot_status_complete;
+	fu->bot_status.csw.Signature = cpu_to_le32(US_BULK_CS_SIGN);
+
+	fu->cmd.buf = kmalloc(fu->ep_out->maxpacket, GFP_KERNEL);
+	if (!fu->cmd.buf)
+		goto err_buf;
+
+	fu->cmd.req->complete = bot_cmd_complete;
+	fu->cmd.req->buf = fu->cmd.buf;
+	fu->cmd.req->length = fu->ep_out->maxpacket;
+	fu->cmd.req->context = fu;
+
+	ret = bot_enqueue_cmd_cbw(fu);
+	if (ret)
+		goto err_queue;
+	return 0;
+err_queue:
+	kfree(fu->cmd.buf);
+	fu->cmd.buf = NULL;
+err_buf:
+	usb_ep_free_request(fu->ep_in, fu->bot_status.req);
+err_sts:
+	usb_ep_free_request(fu->ep_out, fu->cmd.req);
+	fu->cmd.req = NULL;
+err_cmd:
+	usb_ep_free_request(fu->ep_out, fu->bot_req_out);
+	fu->bot_req_out = NULL;
+err_out:
+	usb_ep_free_request(fu->ep_in, fu->bot_req_in);
+	fu->bot_req_in = NULL;
+err:
+	pr_err("BOT: endpoint setup failed\n");
+	return -ENOMEM;
+}
+
+static void bot_cleanup_old_alt(struct f_uas *fu)
+{
+	if (!(fu->flags & USBG_ENABLED))
+		return;
+
+	usb_ep_disable(fu->ep_in);
+	usb_ep_disable(fu->ep_out);
+
+	if (!fu->bot_req_in)
+		return;
+
+	usb_ep_free_request(fu->ep_in, fu->bot_req_in);
+	usb_ep_free_request(fu->ep_out, fu->bot_req_out);
+	usb_ep_free_request(fu->ep_out, fu->cmd.req);
+	usb_ep_free_request(fu->ep_out, fu->bot_status.req);
+
+	kfree(fu->cmd.buf);
+
+	fu->bot_req_in = NULL;
+	fu->bot_req_out = NULL;
+	fu->cmd.req = NULL;
+	fu->bot_status.req = NULL;
+	fu->cmd.buf = NULL;
+}
+
+static void bot_set_alt(struct f_uas *fu)
+{
+	struct usb_function *f = &fu->function;
+	struct usb_gadget *gadget = f->config->cdev->gadget;
+	int ret;
+
+	fu->flags = USBG_IS_BOT;
+
+	config_ep_by_speed(gadget, f, fu->ep_in);
+	ret = usb_ep_enable(fu->ep_in);
+	if (ret)
+		goto err_b_in;
+
+	config_ep_by_speed(gadget, f, fu->ep_out);
+	ret = usb_ep_enable(fu->ep_out);
+	if (ret)
+		goto err_b_out;
+
+	ret = bot_prepare_reqs(fu);
+	if (ret)
+		goto err_wq;
+	fu->flags |= USBG_ENABLED;
+	pr_info("Using the BOT protocol\n");
+	return;
+err_wq:
+	usb_ep_disable(fu->ep_out);
+err_b_out:
+	usb_ep_disable(fu->ep_in);
+err_b_in:
+	fu->flags = USBG_IS_BOT;
+}
+
+static int usbg_bot_setup(struct usb_function *f,
+		const struct usb_ctrlrequest *ctrl)
+{
+	struct f_uas *fu = to_f_uas(f);
+	struct usb_composite_dev *cdev = f->config->cdev;
+	u16 w_value = le16_to_cpu(ctrl->wValue);
+	u16 w_length = le16_to_cpu(ctrl->wLength);
+	int luns;
+	u8 *ret_lun;
+
+	switch (ctrl->bRequest) {
+	case US_BULK_GET_MAX_LUN:
+		if (ctrl->bRequestType != (USB_DIR_IN | USB_TYPE_CLASS |
+					USB_RECIP_INTERFACE))
+			return -ENOTSUPP;
+
+		if (w_length < 1)
+			return -EINVAL;
+		if (w_value != 0)
+			return -EINVAL;
+		luns = atomic_read(&fu->tpg->tpg_port_count);
+		if (!luns) {
+			pr_err("No LUNs configured?\n");
+			return -EINVAL;
+		}
+		/*
+		 * If 4 LUNs are present we return 3 i.e. LUN 0..3 can be
+		 * accessed. The upper limit is 0xf
+		 */
+		luns--;
+		if (luns > 0xf) {
+			pr_info_once("Limiting the number of luns to 16\n");
+			luns = 0xf;
+		}
+		ret_lun = cdev->req->buf;
+		*ret_lun = luns;
+		cdev->req->length = 1;
+		return usb_ep_queue(cdev->gadget->ep0, cdev->req, GFP_ATOMIC);
+
+	case US_BULK_RESET_REQUEST:
+		/* XXX maybe we should remove previous requests for IN + OUT */
+		bot_enqueue_cmd_cbw(fu);
+		return 0;
+	}
+	return -ENOTSUPP;
+}
+
+/* Start uas.c code */
+
+static void uasp_cleanup_one_stream(struct f_uas *fu, struct uas_stream *stream)
+{
+	/* We have either all three allocated or none */
+	if (!stream->req_in)
+		return;
+
+	usb_ep_free_request(fu->ep_in, stream->req_in);
+	usb_ep_free_request(fu->ep_out, stream->req_out);
+	usb_ep_free_request(fu->ep_status, stream->req_status);
+
+	stream->req_in = NULL;
+	stream->req_out = NULL;
+	stream->req_status = NULL;
+}
+
+static void uasp_free_cmdreq(struct f_uas *fu)
+{
+	usb_ep_free_request(fu->ep_cmd, fu->cmd.req);
+	kfree(fu->cmd.buf);
+	fu->cmd.req = NULL;
+	fu->cmd.buf = NULL;
+}
+
+static void uasp_cleanup_old_alt(struct f_uas *fu)
+{
+	int i;
+
+	if (!(fu->flags & USBG_ENABLED))
+		return;
+
+	usb_ep_disable(fu->ep_in);
+	usb_ep_disable(fu->ep_out);
+	usb_ep_disable(fu->ep_status);
+	usb_ep_disable(fu->ep_cmd);
+
+	for (i = 0; i < UASP_SS_EP_COMP_NUM_STREAMS; i++)
+		uasp_cleanup_one_stream(fu, &fu->stream[i]);
+	uasp_free_cmdreq(fu);
+}
+
+static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req);
+
+static int uasp_prepare_r_request(struct usbg_cmd *cmd)
+{
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct f_uas *fu = cmd->fu;
+	struct usb_gadget *gadget = fuas_to_gadget(fu);
+	struct uas_stream *stream = cmd->stream;
+
+	if (!gadget->sg_supported) {
+		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
+		if (!cmd->data_buf)
+			return -ENOMEM;
+
+		sg_copy_to_buffer(se_cmd->t_data_sg,
+				se_cmd->t_data_nents,
+				cmd->data_buf,
+				se_cmd->data_length);
+
+		stream->req_in->buf = cmd->data_buf;
+	} else {
+		stream->req_in->buf = NULL;
+		stream->req_in->num_sgs = se_cmd->t_data_nents;
+		stream->req_in->sg = se_cmd->t_data_sg;
+	}
+
+	stream->req_in->complete = uasp_status_data_cmpl;
+	stream->req_in->length = se_cmd->data_length;
+	stream->req_in->context = cmd;
+
+	cmd->state = UASP_SEND_STATUS;
+	return 0;
+}
+
+static void uasp_prepare_status(struct usbg_cmd *cmd)
+{
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct sense_iu *iu = &cmd->sense_iu;
+	struct uas_stream *stream = cmd->stream;
+
+	cmd->state = UASP_QUEUE_COMMAND;
+	iu->iu_id = IU_ID_STATUS;
+	iu->tag = cpu_to_be16(cmd->tag);
+
+	/*
+	 * iu->status_qual = cpu_to_be16(STATUS QUALIFIER SAM-4. Where R U?);
+	 */
+	iu->len = cpu_to_be16(se_cmd->scsi_sense_length);
+	iu->status = se_cmd->scsi_status;
+	stream->req_status->context = cmd;
+	stream->req_status->length = se_cmd->scsi_sense_length + 16;
+	stream->req_status->buf = iu;
+	stream->req_status->complete = uasp_status_data_cmpl;
+}
+
+static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req)
+{
+	struct usbg_cmd *cmd = req->context;
+	struct uas_stream *stream = cmd->stream;
+	struct f_uas *fu = cmd->fu;
+	int ret;
+
+	if (req->status < 0)
+		goto cleanup;
+
+	switch (cmd->state) {
+	case UASP_SEND_DATA:
+		ret = uasp_prepare_r_request(cmd);
+		if (ret)
+			goto cleanup;
+		ret = usb_ep_queue(fu->ep_in, stream->req_in, GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
+		break;
+
+	case UASP_RECEIVE_DATA:
+		ret = usbg_prepare_w_request(cmd, stream->req_out);
+		if (ret)
+			goto cleanup;
+		ret = usb_ep_queue(fu->ep_out, stream->req_out, GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
+		break;
+
+	case UASP_SEND_STATUS:
+		uasp_prepare_status(cmd);
+		ret = usb_ep_queue(fu->ep_status, stream->req_status,
+				GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
+		break;
+
+	case UASP_QUEUE_COMMAND:
+		usbg_cleanup_cmd(cmd);
+		usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
+		break;
+
+	default:
+		BUG();
+	}
+	return;
+
+cleanup:
+	usbg_cleanup_cmd(cmd);
+}
+
+static int uasp_send_status_response(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct uas_stream *stream = cmd->stream;
+	struct sense_iu *iu = &cmd->sense_iu;
+
+	iu->tag = cpu_to_be16(cmd->tag);
+	stream->req_status->complete = uasp_status_data_cmpl;
+	stream->req_status->context = cmd;
+	cmd->fu = fu;
+	uasp_prepare_status(cmd);
+	return usb_ep_queue(fu->ep_status, stream->req_status, GFP_ATOMIC);
+}
+
+static int uasp_send_read_response(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct uas_stream *stream = cmd->stream;
+	struct sense_iu *iu = &cmd->sense_iu;
+	int ret;
+
+	cmd->fu = fu;
+
+	iu->tag = cpu_to_be16(cmd->tag);
+	if (fu->flags & USBG_USE_STREAMS) {
+
+		ret = uasp_prepare_r_request(cmd);
+		if (ret)
+			goto out;
+		ret = usb_ep_queue(fu->ep_in, stream->req_in, GFP_ATOMIC);
+		if (ret) {
+			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
+			kfree(cmd->data_buf);
+			cmd->data_buf = NULL;
+		}
+
+	} else {
+
+		iu->iu_id = IU_ID_READ_READY;
+		iu->tag = cpu_to_be16(cmd->tag);
+
+		stream->req_status->complete = uasp_status_data_cmpl;
+		stream->req_status->context = cmd;
+
+		cmd->state = UASP_SEND_DATA;
+		stream->req_status->buf = iu;
+		stream->req_status->length = sizeof(struct iu);
+
+		ret = usb_ep_queue(fu->ep_status, stream->req_status,
+				GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
+	}
+out:
+	return ret;
+}
+
+static int uasp_send_write_request(struct usbg_cmd *cmd)
+{
+	struct f_uas *fu = cmd->fu;
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct uas_stream *stream = cmd->stream;
+	struct sense_iu *iu = &cmd->sense_iu;
+	int ret;
+
+	init_completion(&cmd->write_complete);
+	cmd->fu = fu;
+
+	iu->tag = cpu_to_be16(cmd->tag);
+
+	if (fu->flags & USBG_USE_STREAMS) {
+
+		ret = usbg_prepare_w_request(cmd, stream->req_out);
+		if (ret)
+			goto cleanup;
+		ret = usb_ep_queue(fu->ep_out, stream->req_out, GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d)\n", __func__, __LINE__);
+
+	} else {
+
+		iu->iu_id = IU_ID_WRITE_READY;
+		iu->tag = cpu_to_be16(cmd->tag);
+
+		stream->req_status->complete = uasp_status_data_cmpl;
+		stream->req_status->context = cmd;
+
+		cmd->state = UASP_RECEIVE_DATA;
+		stream->req_status->buf = iu;
+		stream->req_status->length = sizeof(struct iu);
+
+		ret = usb_ep_queue(fu->ep_status, stream->req_status,
+				GFP_ATOMIC);
+		if (ret)
+			pr_err("%s(%d)\n", __func__, __LINE__);
+	}
+
+	wait_for_completion(&cmd->write_complete);
+	target_execute_cmd(se_cmd);
+cleanup:
+	return ret;
+}
+
+static int usbg_submit_command(struct f_uas *, void *, unsigned int);
+
+static void uasp_cmd_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct f_uas *fu = req->context;
+	int ret;
+
+	if (req->status < 0)
+		return;
+
+	ret = usbg_submit_command(fu, req->buf, req->actual);
+	/*
+	 * Once we tune for performance enqueue the command req here again so
+	 * we can receive a second command while we processing this one. Pay
+	 * attention to properly sync STAUS endpoint with DATA IN + OUT so you
+	 * don't break HS.
+	 */
+	if (!ret)
+		return;
+	usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
+}
+
+static int uasp_alloc_stream_res(struct f_uas *fu, struct uas_stream *stream)
+{
+	stream->req_in = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
+	if (!stream->req_in)
+		goto out;
+
+	stream->req_out = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
+	if (!stream->req_out)
+		goto err_out;
+
+	stream->req_status = usb_ep_alloc_request(fu->ep_status, GFP_KERNEL);
+	if (!stream->req_status)
+		goto err_sts;
+
+	return 0;
+err_sts:
+	usb_ep_free_request(fu->ep_status, stream->req_status);
+	stream->req_status = NULL;
+err_out:
+	usb_ep_free_request(fu->ep_out, stream->req_out);
+	stream->req_out = NULL;
+out:
+	return -ENOMEM;
+}
+
+static int uasp_alloc_cmd(struct f_uas *fu)
+{
+	fu->cmd.req = usb_ep_alloc_request(fu->ep_cmd, GFP_KERNEL);
+	if (!fu->cmd.req)
+		goto err;
+
+	fu->cmd.buf = kmalloc(fu->ep_cmd->maxpacket, GFP_KERNEL);
+	if (!fu->cmd.buf)
+		goto err_buf;
+
+	fu->cmd.req->complete = uasp_cmd_complete;
+	fu->cmd.req->buf = fu->cmd.buf;
+	fu->cmd.req->length = fu->ep_cmd->maxpacket;
+	fu->cmd.req->context = fu;
+	return 0;
+
+err_buf:
+	usb_ep_free_request(fu->ep_cmd, fu->cmd.req);
+err:
+	return -ENOMEM;
+}
+
+static void uasp_setup_stream_res(struct f_uas *fu, int max_streams)
+{
+	int i;
+
+	for (i = 0; i < max_streams; i++) {
+		struct uas_stream *s = &fu->stream[i];
+
+		s->req_in->stream_id = i + 1;
+		s->req_out->stream_id = i + 1;
+		s->req_status->stream_id = i + 1;
+	}
+}
+
+static int uasp_prepare_reqs(struct f_uas *fu)
+{
+	int ret;
+	int i;
+	int max_streams;
+
+	if (fu->flags & USBG_USE_STREAMS)
+		max_streams = UASP_SS_EP_COMP_NUM_STREAMS;
+	else
+		max_streams = 1;
+
+	for (i = 0; i < max_streams; i++) {
+		ret = uasp_alloc_stream_res(fu, &fu->stream[i]);
+		if (ret)
+			goto err_cleanup;
+	}
+
+	ret = uasp_alloc_cmd(fu);
+	if (ret)
+		goto err_free_stream;
+	uasp_setup_stream_res(fu, max_streams);
+
+	ret = usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
+	if (ret)
+		goto err_free_stream;
+
+	return 0;
+
+err_free_stream:
+	uasp_free_cmdreq(fu);
+
+err_cleanup:
+	if (i) {
+		do {
+			uasp_cleanup_one_stream(fu, &fu->stream[i - 1]);
+			i--;
+		} while (i);
+	}
+	pr_err("UASP: endpoint setup failed\n");
+	return ret;
+}
+
+static void uasp_set_alt(struct f_uas *fu)
+{
+	struct usb_function *f = &fu->function;
+	struct usb_gadget *gadget = f->config->cdev->gadget;
+	int ret;
+
+	fu->flags = USBG_IS_UAS;
+
+	if (gadget->speed == USB_SPEED_SUPER)
+		fu->flags |= USBG_USE_STREAMS;
+
+	config_ep_by_speed(gadget, f, fu->ep_in);
+	ret = usb_ep_enable(fu->ep_in);
+	if (ret)
+		goto err_b_in;
+
+	config_ep_by_speed(gadget, f, fu->ep_out);
+	ret = usb_ep_enable(fu->ep_out);
+	if (ret)
+		goto err_b_out;
+
+	config_ep_by_speed(gadget, f, fu->ep_cmd);
+	ret = usb_ep_enable(fu->ep_cmd);
+	if (ret)
+		goto err_cmd;
+	config_ep_by_speed(gadget, f, fu->ep_status);
+	ret = usb_ep_enable(fu->ep_status);
+	if (ret)
+		goto err_status;
+
+	ret = uasp_prepare_reqs(fu);
+	if (ret)
+		goto err_wq;
+	fu->flags |= USBG_ENABLED;
+
+	pr_info("Using the UAS protocol\n");
+	return;
+err_wq:
+	usb_ep_disable(fu->ep_status);
+err_status:
+	usb_ep_disable(fu->ep_cmd);
+err_cmd:
+	usb_ep_disable(fu->ep_out);
+err_b_out:
+	usb_ep_disable(fu->ep_in);
+err_b_in:
+	fu->flags = 0;
+}
+
+static int get_cmd_dir(const unsigned char *cdb)
+{
+	int ret;
+
+	switch (cdb[0]) {
+	case READ_6:
+	case READ_10:
+	case READ_12:
+	case READ_16:
+	case INQUIRY:
+	case MODE_SENSE:
+	case MODE_SENSE_10:
+	case SERVICE_ACTION_IN_16:
+	case MAINTENANCE_IN:
+	case PERSISTENT_RESERVE_IN:
+	case SECURITY_PROTOCOL_IN:
+	case ACCESS_CONTROL_IN:
+	case REPORT_LUNS:
+	case READ_BLOCK_LIMITS:
+	case READ_POSITION:
+	case READ_CAPACITY:
+	case READ_TOC:
+	case READ_FORMAT_CAPACITIES:
+	case REQUEST_SENSE:
+		ret = DMA_FROM_DEVICE;
+		break;
+
+	case WRITE_6:
+	case WRITE_10:
+	case WRITE_12:
+	case WRITE_16:
+	case MODE_SELECT:
+	case MODE_SELECT_10:
+	case WRITE_VERIFY:
+	case WRITE_VERIFY_12:
+	case PERSISTENT_RESERVE_OUT:
+	case MAINTENANCE_OUT:
+	case SECURITY_PROTOCOL_OUT:
+	case ACCESS_CONTROL_OUT:
+		ret = DMA_TO_DEVICE;
+		break;
+	case ALLOW_MEDIUM_REMOVAL:
+	case TEST_UNIT_READY:
+	case SYNCHRONIZE_CACHE:
+	case START_STOP:
+	case ERASE:
+	case REZERO_UNIT:
+	case SEEK_10:
+	case SPACE:
+	case VERIFY:
+	case WRITE_FILEMARKS:
+		ret = DMA_NONE;
+		break;
+	default:
+#define CMD_DIR_MSG "target: Unknown data direction for SCSI Opcode 0x%02x\n"
+		pr_warn(CMD_DIR_MSG, cdb[0]);
+#undef CMD_DIR_MSG
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+static void usbg_data_write_cmpl(struct usb_ep *ep, struct usb_request *req)
+{
+	struct usbg_cmd *cmd = req->context;
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+
+	if (req->status < 0) {
+		pr_err("%s() state %d transfer failed\n", __func__, cmd->state);
+		goto cleanup;
+	}
+
+	if (req->num_sgs == 0) {
+		sg_copy_from_buffer(se_cmd->t_data_sg,
+				se_cmd->t_data_nents,
+				cmd->data_buf,
+				se_cmd->data_length);
+	}
+
+	complete(&cmd->write_complete);
+	return;
+
+cleanup:
+	usbg_cleanup_cmd(cmd);
+}
+
+static int usbg_prepare_w_request(struct usbg_cmd *cmd, struct usb_request *req)
+{
+	struct se_cmd *se_cmd = &cmd->se_cmd;
+	struct f_uas *fu = cmd->fu;
+	struct usb_gadget *gadget = fuas_to_gadget(fu);
+
+	if (!gadget->sg_supported) {
+		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
+		if (!cmd->data_buf)
+			return -ENOMEM;
+
+		req->buf = cmd->data_buf;
+	} else {
+		req->buf = NULL;
+		req->num_sgs = se_cmd->t_data_nents;
+		req->sg = se_cmd->t_data_sg;
+	}
+
+	req->complete = usbg_data_write_cmpl;
+	req->length = se_cmd->data_length;
+	req->context = cmd;
+	return 0;
+}
+
+static int usbg_send_status_response(struct se_cmd *se_cmd)
+{
+	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
+			se_cmd);
+	struct f_uas *fu = cmd->fu;
+
+	if (fu->flags & USBG_IS_BOT)
+		return bot_send_status_response(cmd);
+	else
+		return uasp_send_status_response(cmd);
+}
+
+static int usbg_send_write_request(struct se_cmd *se_cmd)
+{
+	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
+			se_cmd);
+	struct f_uas *fu = cmd->fu;
+
+	if (fu->flags & USBG_IS_BOT)
+		return bot_send_write_request(cmd);
+	else
+		return uasp_send_write_request(cmd);
+}
+
+static int usbg_send_read_response(struct se_cmd *se_cmd)
+{
+	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
+			se_cmd);
+	struct f_uas *fu = cmd->fu;
+
+	if (fu->flags & USBG_IS_BOT)
+		return bot_send_read_response(cmd);
+	else
+		return uasp_send_read_response(cmd);
+}
+
+static void usbg_cmd_work(struct work_struct *work)
+{
+	struct usbg_cmd *cmd = container_of(work, struct usbg_cmd, work);
+	struct se_cmd *se_cmd;
+	struct tcm_usbg_nexus *tv_nexus;
+	struct usbg_tpg *tpg;
+	int dir;
+
+	se_cmd = &cmd->se_cmd;
+	tpg = cmd->fu->tpg;
+	tv_nexus = tpg->tpg_nexus;
+	dir = get_cmd_dir(cmd->cmd_buf);
+	if (dir < 0) {
+		transport_init_se_cmd(se_cmd,
+				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
+				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
+				cmd->prio_attr, cmd->sense_iu.sense);
+		goto out;
+	}
+
+	if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess,
+			cmd->cmd_buf, cmd->sense_iu.sense, cmd->unpacked_lun,
+			0, cmd->prio_attr, dir, TARGET_SCF_UNKNOWN_SIZE) < 0)
+		goto out;
+
+	return;
+
+out:
+	transport_send_check_condition_and_sense(se_cmd,
+			TCM_UNSUPPORTED_SCSI_OPCODE, 1);
+	usbg_cleanup_cmd(cmd);
+}
+
+static int usbg_submit_command(struct f_uas *fu,
+		void *cmdbuf, unsigned int len)
+{
+	struct command_iu *cmd_iu = cmdbuf;
+	struct usbg_cmd *cmd;
+	struct usbg_tpg *tpg;
+	struct tcm_usbg_nexus *tv_nexus;
+	u32 cmd_len;
+
+	if (cmd_iu->iu_id != IU_ID_COMMAND) {
+		pr_err("Unsupported type %d\n", cmd_iu->iu_id);
+		return -EINVAL;
+	}
+
+	cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
+	if (!cmd)
+		return -ENOMEM;
+
+	cmd->fu = fu;
+
+	/* XXX until I figure out why I can't free in on complete */
+	kref_init(&cmd->ref);
+	kref_get(&cmd->ref);
+
+	tpg = fu->tpg;
+	cmd_len = (cmd_iu->len & ~0x3) + 16;
+	if (cmd_len > USBG_MAX_CMD)
+		goto err;
+
+	memcpy(cmd->cmd_buf, cmd_iu->cdb, cmd_len);
+
+	cmd->tag = be16_to_cpup(&cmd_iu->tag);
+	cmd->se_cmd.tag = cmd->tag;
+	if (fu->flags & USBG_USE_STREAMS) {
+		if (cmd->tag > UASP_SS_EP_COMP_NUM_STREAMS)
+			goto err;
+		if (!cmd->tag)
+			cmd->stream = &fu->stream[0];
+		else
+			cmd->stream = &fu->stream[cmd->tag - 1];
+	} else {
+		cmd->stream = &fu->stream[0];
+	}
+
+	tv_nexus = tpg->tpg_nexus;
+	if (!tv_nexus) {
+		pr_err("Missing nexus, ignoring command\n");
+		goto err;
+	}
+
+	switch (cmd_iu->prio_attr & 0x7) {
+	case UAS_HEAD_TAG:
+		cmd->prio_attr = TCM_HEAD_TAG;
+		break;
+	case UAS_ORDERED_TAG:
+		cmd->prio_attr = TCM_ORDERED_TAG;
+		break;
+	case UAS_ACA:
+		cmd->prio_attr = TCM_ACA_TAG;
+		break;
+	default:
+		pr_debug_once("Unsupported prio_attr: %02x.\n",
+				cmd_iu->prio_attr);
+	case UAS_SIMPLE_TAG:
+		cmd->prio_attr = TCM_SIMPLE_TAG;
+		break;
+	}
+
+	cmd->unpacked_lun = scsilun_to_int(&cmd_iu->lun);
+
+	INIT_WORK(&cmd->work, usbg_cmd_work);
+	queue_work(tpg->workqueue, &cmd->work);
+
+	return 0;
+err:
+	kfree(cmd);
+	return -EINVAL;
+}
+
+static void bot_cmd_work(struct work_struct *work)
+{
+	struct usbg_cmd *cmd = container_of(work, struct usbg_cmd, work);
+	struct se_cmd *se_cmd;
+	struct tcm_usbg_nexus *tv_nexus;
+	struct usbg_tpg *tpg;
+	int dir;
+
+	se_cmd = &cmd->se_cmd;
+	tpg = cmd->fu->tpg;
+	tv_nexus = tpg->tpg_nexus;
+	dir = get_cmd_dir(cmd->cmd_buf);
+	if (dir < 0) {
+		transport_init_se_cmd(se_cmd,
+				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
+				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
+				cmd->prio_attr, cmd->sense_iu.sense);
+		goto out;
+	}
+
+	if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess,
+			cmd->cmd_buf, cmd->sense_iu.sense, cmd->unpacked_lun,
+			cmd->data_len, cmd->prio_attr, dir, 0) < 0)
+		goto out;
+
+	return;
+
+out:
+	transport_send_check_condition_and_sense(se_cmd,
+				TCM_UNSUPPORTED_SCSI_OPCODE, 1);
+	usbg_cleanup_cmd(cmd);
+}
+
+static int bot_submit_command(struct f_uas *fu,
+		void *cmdbuf, unsigned int len)
+{
+	struct bulk_cb_wrap *cbw = cmdbuf;
+	struct usbg_cmd *cmd;
+	struct usbg_tpg *tpg;
+	struct tcm_usbg_nexus *tv_nexus;
+	u32 cmd_len;
+
+	if (cbw->Signature != cpu_to_le32(US_BULK_CB_SIGN)) {
+		pr_err("Wrong signature on CBW\n");
+		return -EINVAL;
+	}
+	if (len != 31) {
+		pr_err("Wrong length for CBW\n");
+		return -EINVAL;
+	}
+
+	cmd_len = cbw->Length;
+	if (cmd_len < 1 || cmd_len > 16)
+		return -EINVAL;
+
+	cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
+	if (!cmd)
+		return -ENOMEM;
+
+	cmd->fu = fu;
+
+	/* XXX until I figure out why I can't free in on complete */
+	kref_init(&cmd->ref);
+	kref_get(&cmd->ref);
+
+	tpg = fu->tpg;
+
+	memcpy(cmd->cmd_buf, cbw->CDB, cmd_len);
+
+	cmd->bot_tag = cbw->Tag;
+
+	tv_nexus = tpg->tpg_nexus;
+	if (!tv_nexus) {
+		pr_err("Missing nexus, ignoring command\n");
+		goto err;
+	}
+
+	cmd->prio_attr = TCM_SIMPLE_TAG;
+	cmd->unpacked_lun = cbw->Lun;
+	cmd->is_read = cbw->Flags & US_BULK_FLAG_IN ? 1 : 0;
+	cmd->data_len = le32_to_cpu(cbw->DataTransferLength);
+	cmd->se_cmd.tag = le32_to_cpu(cmd->bot_tag);
+
+	INIT_WORK(&cmd->work, bot_cmd_work);
+	queue_work(tpg->workqueue, &cmd->work);
+
+	return 0;
+err:
+	kfree(cmd);
+	return -EINVAL;
+}
+
+/* Start fabric.c code */
+
+static int usbg_check_true(struct se_portal_group *se_tpg)
+{
+	return 1;
+}
+
+static int usbg_check_false(struct se_portal_group *se_tpg)
+{
+	return 0;
+}
+
+static char *usbg_get_fabric_name(void)
+{
+	return "usb_gadget";
+}
+
+static char *usbg_get_fabric_wwn(struct se_portal_group *se_tpg)
+{
+	struct usbg_tpg *tpg = container_of(se_tpg,
+				struct usbg_tpg, se_tpg);
+	struct usbg_tport *tport = tpg->tport;
+
+	return &tport->tport_name[0];
+}
+
+static u16 usbg_get_tag(struct se_portal_group *se_tpg)
+{
+	struct usbg_tpg *tpg = container_of(se_tpg,
+				struct usbg_tpg, se_tpg);
+	return tpg->tport_tpgt;
+}
+
+static u32 usbg_tpg_get_inst_index(struct se_portal_group *se_tpg)
+{
+	return 1;
+}
+
+static void usbg_cmd_release(struct kref *ref)
+{
+	struct usbg_cmd *cmd = container_of(ref, struct usbg_cmd,
+			ref);
+
+	transport_generic_free_cmd(&cmd->se_cmd, 0);
+}
+
+static void usbg_release_cmd(struct se_cmd *se_cmd)
+{
+	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
+			se_cmd);
+	kfree(cmd->data_buf);
+	kfree(cmd);
+}
+
+static int usbg_shutdown_session(struct se_session *se_sess)
+{
+	return 0;
+}
+
+static void usbg_close_session(struct se_session *se_sess)
+{
+}
+
+static u32 usbg_sess_get_index(struct se_session *se_sess)
+{
+	return 0;
+}
+
+/*
+ * XXX Error recovery: return != 0 if we expect writes. Dunno when that could be
+ */
+static int usbg_write_pending_status(struct se_cmd *se_cmd)
+{
+	return 0;
+}
+
+static void usbg_set_default_node_attrs(struct se_node_acl *nacl)
+{
+}
+
+static int usbg_get_cmd_state(struct se_cmd *se_cmd)
+{
+	return 0;
+}
+
+static void usbg_queue_tm_rsp(struct se_cmd *se_cmd)
+{
+}
+
+static void usbg_aborted_task(struct se_cmd *se_cmd)
+{
+}
+
+static const char *usbg_check_wwn(const char *name)
+{
+	const char *n;
+	unsigned int len;
+
+	n = strstr(name, "naa.");
+	if (!n)
+		return NULL;
+	n += 4;
+	len = strlen(n);
+	if (len == 0 || len > USBG_NAMELEN - 1)
+		return NULL;
+	return n;
+}
+
+static int usbg_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
+{
+	if (!usbg_check_wwn(name))
+		return -EINVAL;
+	return 0;
+}
+
+static struct se_portal_group *usbg_make_tpg(
+	struct se_wwn *wwn,
+	struct config_group *group,
+	const char *name)
+{
+	struct usbg_tport *tport = container_of(wwn, struct usbg_tport,
+			tport_wwn);
+	struct usbg_tpg *tpg;
+	unsigned long tpgt;
+	int ret;
+	struct f_tcm_opts *opts;
+	unsigned i;
+
+	if (strstr(name, "tpgt_") != name)
+		return ERR_PTR(-EINVAL);
+	if (kstrtoul(name + 5, 0, &tpgt) || tpgt > UINT_MAX)
+		return ERR_PTR(-EINVAL);
+	ret = -ENODEV;
+	mutex_lock(&tpg_instances_lock);
+	for (i = 0; i < TPG_INSTANCES; ++i)
+		if (tpg_instances[i].func_inst && !tpg_instances[i].tpg)
+			break;
+	if (i == TPG_INSTANCES)
+		goto unlock_inst;
+
+	opts = container_of(tpg_instances[i].func_inst, struct f_tcm_opts,
+		func_inst);
+	mutex_lock(&opts->dep_lock);
+	if (!opts->ready)
+		goto unlock_dep;
+
+	if (opts->has_dep) {
+		if (!try_module_get(opts->dependent))
+			goto unlock_dep;
+	} else {
+		ret = configfs_depend_item_unlocked(
+			group->cg_subsys,
+			&opts->func_inst.group.cg_item);
+		if (ret)
+			goto unlock_dep;
+	}
+
+	tpg = kzalloc(sizeof(struct usbg_tpg), GFP_KERNEL);
+	ret = -ENOMEM;
+	if (!tpg)
+		goto unref_dep;
+	mutex_init(&tpg->tpg_mutex);
+	atomic_set(&tpg->tpg_port_count, 0);
+	tpg->workqueue = alloc_workqueue("tcm_usb_gadget", 0, 1);
+	if (!tpg->workqueue)
+		goto free_tpg;
+
+	tpg->tport = tport;
+	tpg->tport_tpgt = tpgt;
+
+	/*
+	 * SPC doesn't assign a protocol identifier for USB-SCSI, so we
+	 * pretend to be SAS..
+	 */
+	ret = core_tpg_register(wwn, &tpg->se_tpg, SCSI_PROTOCOL_SAS);
+	if (ret < 0)
+		goto free_workqueue;
+
+	tpg_instances[i].tpg = tpg;
+	tpg->fi = tpg_instances[i].func_inst;
+	mutex_unlock(&opts->dep_lock);
+	mutex_unlock(&tpg_instances_lock);
+	return &tpg->se_tpg;
+
+free_workqueue:
+	destroy_workqueue(tpg->workqueue);
+free_tpg:
+	kfree(tpg);
+unref_dep:
+	if (opts->has_dep)
+		module_put(opts->dependent);
+	else
+		configfs_undepend_item_unlocked(&opts->func_inst.group.cg_item);
+unlock_dep:
+	mutex_unlock(&opts->dep_lock);
+unlock_inst:
+	mutex_unlock(&tpg_instances_lock);
+
+	return ERR_PTR(ret);
+}
+
+static int tcm_usbg_drop_nexus(struct usbg_tpg *);
+
+static void usbg_drop_tpg(struct se_portal_group *se_tpg)
+{
+	struct usbg_tpg *tpg = container_of(se_tpg,
+				struct usbg_tpg, se_tpg);
+	unsigned i;
+	struct f_tcm_opts *opts;
+
+	tcm_usbg_drop_nexus(tpg);
+	core_tpg_deregister(se_tpg);
+	destroy_workqueue(tpg->workqueue);
+
+	mutex_lock(&tpg_instances_lock);
+	for (i = 0; i < TPG_INSTANCES; ++i)
+		if (tpg_instances[i].tpg == tpg)
+			break;
+	if (i < TPG_INSTANCES)
+		tpg_instances[i].tpg = NULL;
+	opts = container_of(tpg_instances[i].func_inst,
+		struct f_tcm_opts, func_inst);
+	mutex_lock(&opts->dep_lock);
+	if (opts->has_dep)
+		module_put(opts->dependent);
+	else
+		configfs_undepend_item_unlocked(&opts->func_inst.group.cg_item);
+	mutex_unlock(&opts->dep_lock);
+	mutex_unlock(&tpg_instances_lock);
+
+	kfree(tpg);
+}
+
+static struct se_wwn *usbg_make_tport(
+	struct target_fabric_configfs *tf,
+	struct config_group *group,
+	const char *name)
+{
+	struct usbg_tport *tport;
+	const char *wnn_name;
+	u64 wwpn = 0;
+
+	wnn_name = usbg_check_wwn(name);
+	if (!wnn_name)
+		return ERR_PTR(-EINVAL);
+
+	tport = kzalloc(sizeof(struct usbg_tport), GFP_KERNEL);
+	if (!(tport))
+		return ERR_PTR(-ENOMEM);
+
+	tport->tport_wwpn = wwpn;
+	snprintf(tport->tport_name, sizeof(tport->tport_name), "%s", wnn_name);
+	return &tport->tport_wwn;
+}
+
+static void usbg_drop_tport(struct se_wwn *wwn)
+{
+	struct usbg_tport *tport = container_of(wwn,
+				struct usbg_tport, tport_wwn);
+	kfree(tport);
+}
+
+/*
+ * If somebody feels like dropping the version property, go ahead.
+ */
+static ssize_t usbg_wwn_version_show(struct config_item *item,  char *page)
+{
+	return sprintf(page, "usb-gadget fabric module\n");
+}
+
+CONFIGFS_ATTR_RO(usbg_wwn_, version);
+
+static struct configfs_attribute *usbg_wwn_attrs[] = {
+	&usbg_wwn_attr_version,
+	NULL,
+};
+
+static ssize_t tcm_usbg_tpg_enable_show(struct config_item *item, char *page)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	struct usbg_tpg  *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+
+	return snprintf(page, PAGE_SIZE, "%u\n", tpg->gadget_connect);
+}
+
+static int usbg_attach(struct usbg_tpg *);
+static void usbg_detach(struct usbg_tpg *);
+
+static ssize_t tcm_usbg_tpg_enable_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	struct usbg_tpg  *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+	bool op;
+	ssize_t ret;
+
+	ret = strtobool(page, &op);
+	if (ret)
+		return ret;
+
+	if ((op && tpg->gadget_connect) || (!op && !tpg->gadget_connect))
+		return -EINVAL;
+
+	if (op)
+		ret = usbg_attach(tpg);
+	else
+		usbg_detach(tpg);
+	if (ret)
+		return ret;
+
+	tpg->gadget_connect = op;
+
+	return count;
+}
+
+static ssize_t tcm_usbg_tpg_nexus_show(struct config_item *item, char *page)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+	struct tcm_usbg_nexus *tv_nexus;
+	ssize_t ret;
+
+	mutex_lock(&tpg->tpg_mutex);
+	tv_nexus = tpg->tpg_nexus;
+	if (!tv_nexus) {
+		ret = -ENODEV;
+		goto out;
+	}
+	ret = snprintf(page, PAGE_SIZE, "%s\n",
+			tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
+out:
+	mutex_unlock(&tpg->tpg_mutex);
+	return ret;
+}
+
+static int tcm_usbg_make_nexus(struct usbg_tpg *tpg, char *name)
+{
+	struct se_portal_group *se_tpg;
+	struct tcm_usbg_nexus *tv_nexus;
+	int ret;
+
+	mutex_lock(&tpg->tpg_mutex);
+	if (tpg->tpg_nexus) {
+		ret = -EEXIST;
+		pr_debug("tpg->tpg_nexus already exists\n");
+		goto err_unlock;
+	}
+	se_tpg = &tpg->se_tpg;
+
+	ret = -ENOMEM;
+	tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
+	if (!tv_nexus)
+		goto err_unlock;
+	tv_nexus->tvn_se_sess = transport_init_session(TARGET_PROT_NORMAL);
+	if (IS_ERR(tv_nexus->tvn_se_sess))
+		goto err_free;
+
+	/*
+	 * Since we are running in 'demo mode' this call with generate a
+	 * struct se_node_acl for the tcm_vhost struct se_portal_group with
+	 * the SCSI Initiator port name of the passed configfs group 'name'.
+	 */
+	tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl(
+			se_tpg, name);
+	if (!tv_nexus->tvn_se_sess->se_node_acl) {
+#define MAKE_NEXUS_MSG "core_tpg_check_initiator_node_acl() failed for %s\n"
+		pr_debug(MAKE_NEXUS_MSG, name);
+#undef MAKE_NEXUS_MSG
+		goto err_session;
+	}
+	/*
+	 * Now register the TCM vHost virtual I_T Nexus as active.
+	 */
+	transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl,
+			tv_nexus->tvn_se_sess, tv_nexus);
+	tpg->tpg_nexus = tv_nexus;
+	mutex_unlock(&tpg->tpg_mutex);
+	return 0;
+
+err_session:
+	transport_free_session(tv_nexus->tvn_se_sess);
+err_free:
+	kfree(tv_nexus);
+err_unlock:
+	mutex_unlock(&tpg->tpg_mutex);
+	return ret;
+}
+
+static int tcm_usbg_drop_nexus(struct usbg_tpg *tpg)
+{
+	struct se_session *se_sess;
+	struct tcm_usbg_nexus *tv_nexus;
+	int ret = -ENODEV;
+
+	mutex_lock(&tpg->tpg_mutex);
+	tv_nexus = tpg->tpg_nexus;
+	if (!tv_nexus)
+		goto out;
+
+	se_sess = tv_nexus->tvn_se_sess;
+	if (!se_sess)
+		goto out;
+
+	if (atomic_read(&tpg->tpg_port_count)) {
+		ret = -EPERM;
+#define MSG "Unable to remove Host I_T Nexus with active TPG port count: %d\n"
+		pr_err(MSG, atomic_read(&tpg->tpg_port_count));
+#undef MSG
+		goto out;
+	}
+
+	pr_debug("Removing I_T Nexus to Initiator Port: %s\n",
+			tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
+	/*
+	 * Release the SCSI I_T Nexus to the emulated vHost Target Port
+	 */
+	transport_deregister_session(tv_nexus->tvn_se_sess);
+	tpg->tpg_nexus = NULL;
+
+	kfree(tv_nexus);
+	ret = 0;
+out:
+	mutex_unlock(&tpg->tpg_mutex);
+	return ret;
+}
+
+static ssize_t tcm_usbg_tpg_nexus_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+	unsigned char i_port[USBG_NAMELEN], *ptr;
+	int ret;
+
+	if (!strncmp(page, "NULL", 4)) {
+		ret = tcm_usbg_drop_nexus(tpg);
+		return (!ret) ? count : ret;
+	}
+	if (strlen(page) >= USBG_NAMELEN) {
+
+#define NEXUS_STORE_MSG "Emulated NAA Sas Address: %s, exceeds max: %d\n"
+		pr_err(NEXUS_STORE_MSG, page, USBG_NAMELEN);
+#undef NEXUS_STORE_MSG
+		return -EINVAL;
+	}
+	snprintf(i_port, USBG_NAMELEN, "%s", page);
+
+	ptr = strstr(i_port, "naa.");
+	if (!ptr) {
+		pr_err("Missing 'naa.' prefix\n");
+		return -EINVAL;
+	}
+
+	if (i_port[strlen(i_port) - 1] == '\n')
+		i_port[strlen(i_port) - 1] = '\0';
+
+	ret = tcm_usbg_make_nexus(tpg, &i_port[0]);
+	if (ret < 0)
+		return ret;
+	return count;
+}
+
+CONFIGFS_ATTR(tcm_usbg_tpg_, enable);
+CONFIGFS_ATTR(tcm_usbg_tpg_, nexus);
+
+static struct configfs_attribute *usbg_base_attrs[] = {
+	&tcm_usbg_tpg_attr_enable,
+	&tcm_usbg_tpg_attr_nexus,
+	NULL,
+};
+
+static int usbg_port_link(struct se_portal_group *se_tpg, struct se_lun *lun)
+{
+	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+
+	atomic_inc(&tpg->tpg_port_count);
+	smp_mb__after_atomic();
+	return 0;
+}
+
+static void usbg_port_unlink(struct se_portal_group *se_tpg,
+		struct se_lun *se_lun)
+{
+	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
+
+	atomic_dec(&tpg->tpg_port_count);
+	smp_mb__after_atomic();
+}
+
+static int usbg_check_stop_free(struct se_cmd *se_cmd)
+{
+	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
+			se_cmd);
+
+	kref_put(&cmd->ref, usbg_cmd_release);
+	return 1;
+}
+
+static const struct target_core_fabric_ops usbg_ops = {
+	.module				= THIS_MODULE,
+	.name				= "usb_gadget",
+	.get_fabric_name		= usbg_get_fabric_name,
+	.tpg_get_wwn			= usbg_get_fabric_wwn,
+	.tpg_get_tag			= usbg_get_tag,
+	.tpg_check_demo_mode		= usbg_check_true,
+	.tpg_check_demo_mode_cache	= usbg_check_false,
+	.tpg_check_demo_mode_write_protect = usbg_check_false,
+	.tpg_check_prod_mode_write_protect = usbg_check_false,
+	.tpg_get_inst_index		= usbg_tpg_get_inst_index,
+	.release_cmd			= usbg_release_cmd,
+	.shutdown_session		= usbg_shutdown_session,
+	.close_session			= usbg_close_session,
+	.sess_get_index			= usbg_sess_get_index,
+	.sess_get_initiator_sid		= NULL,
+	.write_pending			= usbg_send_write_request,
+	.write_pending_status		= usbg_write_pending_status,
+	.set_default_node_attributes	= usbg_set_default_node_attrs,
+	.get_cmd_state			= usbg_get_cmd_state,
+	.queue_data_in			= usbg_send_read_response,
+	.queue_status			= usbg_send_status_response,
+	.queue_tm_rsp			= usbg_queue_tm_rsp,
+	.aborted_task			= usbg_aborted_task,
+	.check_stop_free		= usbg_check_stop_free,
+
+	.fabric_make_wwn		= usbg_make_tport,
+	.fabric_drop_wwn		= usbg_drop_tport,
+	.fabric_make_tpg		= usbg_make_tpg,
+	.fabric_drop_tpg		= usbg_drop_tpg,
+	.fabric_post_link		= usbg_port_link,
+	.fabric_pre_unlink		= usbg_port_unlink,
+	.fabric_init_nodeacl		= usbg_init_nodeacl,
+
+	.tfc_wwn_attrs			= usbg_wwn_attrs,
+	.tfc_tpg_base_attrs		= usbg_base_attrs,
+};
+
+/* Start gadget.c code */
+
+static struct usb_interface_descriptor bot_intf_desc = {
+	.bLength =              sizeof(bot_intf_desc),
+	.bDescriptorType =      USB_DT_INTERFACE,
+	.bNumEndpoints =        2,
+	.bAlternateSetting =	USB_G_ALT_INT_BBB,
+	.bInterfaceClass =      USB_CLASS_MASS_STORAGE,
+	.bInterfaceSubClass =   USB_SC_SCSI,
+	.bInterfaceProtocol =   USB_PR_BULK,
+};
+
+static struct usb_interface_descriptor uasp_intf_desc = {
+	.bLength =		sizeof(uasp_intf_desc),
+	.bDescriptorType =	USB_DT_INTERFACE,
+	.bNumEndpoints =	4,
+	.bAlternateSetting =	USB_G_ALT_INT_UAS,
+	.bInterfaceClass =	USB_CLASS_MASS_STORAGE,
+	.bInterfaceSubClass =	USB_SC_SCSI,
+	.bInterfaceProtocol =	USB_PR_UAS,
+};
+
+static struct usb_endpoint_descriptor uasp_bi_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor uasp_fs_bi_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_pipe_usage_descriptor uasp_bi_pipe_desc = {
+	.bLength =		sizeof(uasp_bi_pipe_desc),
+	.bDescriptorType =	USB_DT_PIPE_USAGE,
+	.bPipeID =		DATA_IN_PIPE_ID,
+};
+
+static struct usb_endpoint_descriptor uasp_ss_bi_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor uasp_bi_ep_comp_desc = {
+	.bLength =		sizeof(uasp_bi_ep_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+	.bMaxBurst =		0,
+	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
+	.wBytesPerInterval =	0,
+};
+
+static struct usb_ss_ep_comp_descriptor bot_bi_ep_comp_desc = {
+	.bLength =		sizeof(bot_bi_ep_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+	.bMaxBurst =		0,
+};
+
+static struct usb_endpoint_descriptor uasp_bo_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor uasp_fs_bo_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_pipe_usage_descriptor uasp_bo_pipe_desc = {
+	.bLength =		sizeof(uasp_bo_pipe_desc),
+	.bDescriptorType =	USB_DT_PIPE_USAGE,
+	.bPipeID =		DATA_OUT_PIPE_ID,
+};
+
+static struct usb_endpoint_descriptor uasp_ss_bo_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(0x400),
+};
+
+static struct usb_ss_ep_comp_descriptor uasp_bo_ep_comp_desc = {
+	.bLength =		sizeof(uasp_bo_ep_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
+};
+
+static struct usb_ss_ep_comp_descriptor bot_bo_ep_comp_desc = {
+	.bLength =		sizeof(bot_bo_ep_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+};
+
+static struct usb_endpoint_descriptor uasp_status_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor uasp_fs_status_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_pipe_usage_descriptor uasp_status_pipe_desc = {
+	.bLength =		sizeof(uasp_status_pipe_desc),
+	.bDescriptorType =	USB_DT_PIPE_USAGE,
+	.bPipeID =		STATUS_PIPE_ID,
+};
+
+static struct usb_endpoint_descriptor uasp_ss_status_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor uasp_status_in_ep_comp_desc = {
+	.bLength =		sizeof(uasp_status_in_ep_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
+};
+
+static struct usb_endpoint_descriptor uasp_cmd_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor uasp_fs_cmd_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_pipe_usage_descriptor uasp_cmd_pipe_desc = {
+	.bLength =		sizeof(uasp_cmd_pipe_desc),
+	.bDescriptorType =	USB_DT_PIPE_USAGE,
+	.bPipeID =		CMD_PIPE_ID,
+};
+
+static struct usb_endpoint_descriptor uasp_ss_cmd_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor uasp_cmd_comp_desc = {
+	.bLength =		sizeof(uasp_cmd_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+};
+
+static struct usb_descriptor_header *uasp_fs_function_desc[] = {
+	(struct usb_descriptor_header *) &bot_intf_desc,
+	(struct usb_descriptor_header *) &uasp_fs_bi_desc,
+	(struct usb_descriptor_header *) &uasp_fs_bo_desc,
+
+	(struct usb_descriptor_header *) &uasp_intf_desc,
+	(struct usb_descriptor_header *) &uasp_fs_bi_desc,
+	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_fs_bo_desc,
+	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_fs_status_desc,
+	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_fs_cmd_desc,
+	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
+	NULL,
+};
+
+static struct usb_descriptor_header *uasp_hs_function_desc[] = {
+	(struct usb_descriptor_header *) &bot_intf_desc,
+	(struct usb_descriptor_header *) &uasp_bi_desc,
+	(struct usb_descriptor_header *) &uasp_bo_desc,
+
+	(struct usb_descriptor_header *) &uasp_intf_desc,
+	(struct usb_descriptor_header *) &uasp_bi_desc,
+	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_bo_desc,
+	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_status_desc,
+	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_cmd_desc,
+	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
+	NULL,
+};
+
+static struct usb_descriptor_header *uasp_ss_function_desc[] = {
+	(struct usb_descriptor_header *) &bot_intf_desc,
+	(struct usb_descriptor_header *) &uasp_ss_bi_desc,
+	(struct usb_descriptor_header *) &bot_bi_ep_comp_desc,
+	(struct usb_descriptor_header *) &uasp_ss_bo_desc,
+	(struct usb_descriptor_header *) &bot_bo_ep_comp_desc,
+
+	(struct usb_descriptor_header *) &uasp_intf_desc,
+	(struct usb_descriptor_header *) &uasp_ss_bi_desc,
+	(struct usb_descriptor_header *) &uasp_bi_ep_comp_desc,
+	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_ss_bo_desc,
+	(struct usb_descriptor_header *) &uasp_bo_ep_comp_desc,
+	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_ss_status_desc,
+	(struct usb_descriptor_header *) &uasp_status_in_ep_comp_desc,
+	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
+	(struct usb_descriptor_header *) &uasp_ss_cmd_desc,
+	(struct usb_descriptor_header *) &uasp_cmd_comp_desc,
+	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
+	NULL,
+};
+
+static struct usb_string	tcm_us_strings[] = {
+	[USB_G_STR_INT_UAS].s		= "USB Attached SCSI",
+	[USB_G_STR_INT_BBB].s		= "Bulk Only Transport",
+	{ },
+};
+
+static struct usb_gadget_strings tcm_stringtab = {
+	.language = 0x0409,
+	.strings = tcm_us_strings,
+};
+
+static struct usb_gadget_strings *tcm_strings[] = {
+	&tcm_stringtab,
+	NULL,
+};
+
+static int tcm_bind(struct usb_configuration *c, struct usb_function *f)
+{
+	struct f_uas		*fu = to_f_uas(f);
+	struct usb_string	*us;
+	struct usb_gadget	*gadget = c->cdev->gadget;
+	struct usb_ep		*ep;
+	struct f_tcm_opts	*opts;
+	int			iface;
+	int			ret;
+
+	opts = container_of(f->fi, struct f_tcm_opts, func_inst);
+
+	mutex_lock(&opts->dep_lock);
+	if (!opts->can_attach) {
+		mutex_unlock(&opts->dep_lock);
+		return -ENODEV;
+	}
+	mutex_unlock(&opts->dep_lock);
+	us = usb_gstrings_attach(c->cdev, tcm_strings,
+		ARRAY_SIZE(tcm_us_strings));
+	if (IS_ERR(us))
+		return PTR_ERR(us);
+	bot_intf_desc.iInterface = us[USB_G_STR_INT_BBB].id;
+	uasp_intf_desc.iInterface = us[USB_G_STR_INT_UAS].id;
+
+	iface = usb_interface_id(c, f);
+	if (iface < 0)
+		return iface;
+
+	bot_intf_desc.bInterfaceNumber = iface;
+	uasp_intf_desc.bInterfaceNumber = iface;
+	fu->iface = iface;
+	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_bi_desc,
+			&uasp_bi_ep_comp_desc);
+	if (!ep)
+		goto ep_fail;
+
+	fu->ep_in = ep;
+
+	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_bo_desc,
+			&uasp_bo_ep_comp_desc);
+	if (!ep)
+		goto ep_fail;
+	fu->ep_out = ep;
+
+	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_status_desc,
+			&uasp_status_in_ep_comp_desc);
+	if (!ep)
+		goto ep_fail;
+	fu->ep_status = ep;
+
+	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_cmd_desc,
+			&uasp_cmd_comp_desc);
+	if (!ep)
+		goto ep_fail;
+	fu->ep_cmd = ep;
+
+	/* Assume endpoint addresses are the same for both speeds */
+	uasp_bi_desc.bEndpointAddress =	uasp_ss_bi_desc.bEndpointAddress;
+	uasp_bo_desc.bEndpointAddress = uasp_ss_bo_desc.bEndpointAddress;
+	uasp_status_desc.bEndpointAddress =
+		uasp_ss_status_desc.bEndpointAddress;
+	uasp_cmd_desc.bEndpointAddress = uasp_ss_cmd_desc.bEndpointAddress;
+
+	uasp_fs_bi_desc.bEndpointAddress = uasp_ss_bi_desc.bEndpointAddress;
+	uasp_fs_bo_desc.bEndpointAddress = uasp_ss_bo_desc.bEndpointAddress;
+	uasp_fs_status_desc.bEndpointAddress =
+		uasp_ss_status_desc.bEndpointAddress;
+	uasp_fs_cmd_desc.bEndpointAddress = uasp_ss_cmd_desc.bEndpointAddress;
+
+	ret = usb_assign_descriptors(f, uasp_fs_function_desc,
+			uasp_hs_function_desc, uasp_ss_function_desc);
+	if (ret)
+		goto ep_fail;
+
+	return 0;
+ep_fail:
+	pr_err("Can't claim all required eps\n");
+
+	return -ENOTSUPP;
+}
+
+struct guas_setup_wq {
+	struct work_struct work;
+	struct f_uas *fu;
+	unsigned int alt;
+};
+
+static void tcm_delayed_set_alt(struct work_struct *wq)
+{
+	struct guas_setup_wq *work = container_of(wq, struct guas_setup_wq,
+			work);
+	struct f_uas *fu = work->fu;
+	int alt = work->alt;
+
+	kfree(work);
+
+	if (fu->flags & USBG_IS_BOT)
+		bot_cleanup_old_alt(fu);
+	if (fu->flags & USBG_IS_UAS)
+		uasp_cleanup_old_alt(fu);
+
+	if (alt == USB_G_ALT_INT_BBB)
+		bot_set_alt(fu);
+	else if (alt == USB_G_ALT_INT_UAS)
+		uasp_set_alt(fu);
+	usb_composite_setup_continue(fu->function.config->cdev);
+}
+
+static int tcm_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
+{
+	struct f_uas *fu = to_f_uas(f);
+
+	if ((alt == USB_G_ALT_INT_BBB) || (alt == USB_G_ALT_INT_UAS)) {
+		struct guas_setup_wq *work;
+
+		work = kmalloc(sizeof(*work), GFP_ATOMIC);
+		if (!work)
+			return -ENOMEM;
+		INIT_WORK(&work->work, tcm_delayed_set_alt);
+		work->fu = fu;
+		work->alt = alt;
+		schedule_work(&work->work);
+		return USB_GADGET_DELAYED_STATUS;
+	}
+	return -EOPNOTSUPP;
+}
+
+static void tcm_disable(struct usb_function *f)
+{
+	struct f_uas *fu = to_f_uas(f);
+
+	if (fu->flags & USBG_IS_UAS)
+		uasp_cleanup_old_alt(fu);
+	else if (fu->flags & USBG_IS_BOT)
+		bot_cleanup_old_alt(fu);
+	fu->flags = 0;
+}
+
+static int tcm_setup(struct usb_function *f,
+		const struct usb_ctrlrequest *ctrl)
+{
+	struct f_uas *fu = to_f_uas(f);
+
+	if (!(fu->flags & USBG_IS_BOT))
+		return -EOPNOTSUPP;
+
+	return usbg_bot_setup(f, ctrl);
+}
+
+static inline struct f_tcm_opts *to_f_tcm_opts(struct config_item *item)
+{
+	return container_of(to_config_group(item), struct f_tcm_opts,
+		func_inst.group);
+}
+
+static void tcm_attr_release(struct config_item *item)
+{
+	struct f_tcm_opts *opts = to_f_tcm_opts(item);
+
+	usb_put_function_instance(&opts->func_inst);
+}
+
+static struct configfs_item_operations tcm_item_ops = {
+	.release		= tcm_attr_release,
+};
+
+static struct config_item_type tcm_func_type = {
+	.ct_item_ops	= &tcm_item_ops,
+	.ct_owner	= THIS_MODULE,
+};
+
+static void tcm_free_inst(struct usb_function_instance *f)
+{
+	struct f_tcm_opts *opts;
+	unsigned i;
+
+	opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	mutex_lock(&tpg_instances_lock);
+	for (i = 0; i < TPG_INSTANCES; ++i)
+		if (tpg_instances[i].func_inst == f)
+			break;
+	if (i < TPG_INSTANCES)
+		tpg_instances[i].func_inst = NULL;
+	mutex_unlock(&tpg_instances_lock);
+
+	kfree(opts);
+}
+
+static int tcm_register_callback(struct usb_function_instance *f)
+{
+	struct f_tcm_opts *opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	mutex_lock(&opts->dep_lock);
+	opts->can_attach = true;
+	mutex_unlock(&opts->dep_lock);
+
+	return 0;
+}
+
+static void tcm_unregister_callback(struct usb_function_instance *f)
+{
+	struct f_tcm_opts *opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	mutex_lock(&opts->dep_lock);
+	unregister_gadget_item(opts->
+		func_inst.group.cg_item.ci_parent->ci_parent);
+	opts->can_attach = false;
+	mutex_unlock(&opts->dep_lock);
+}
+
+static int usbg_attach(struct usbg_tpg *tpg)
+{
+	struct usb_function_instance *f = tpg->fi;
+	struct f_tcm_opts *opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	if (opts->tcm_register_callback)
+		return opts->tcm_register_callback(f);
+
+	return 0;
+}
+
+static void usbg_detach(struct usbg_tpg *tpg)
+{
+	struct usb_function_instance *f = tpg->fi;
+	struct f_tcm_opts *opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	if (opts->tcm_unregister_callback)
+		opts->tcm_unregister_callback(f);
+}
+
+static int tcm_set_name(struct usb_function_instance *f, const char *name)
+{
+	struct f_tcm_opts *opts = container_of(f, struct f_tcm_opts, func_inst);
+
+	pr_debug("tcm: Activating %s\n", name);
+
+	mutex_lock(&opts->dep_lock);
+	opts->ready = true;
+	mutex_unlock(&opts->dep_lock);
+
+	return 0;
+}
+
+static struct usb_function_instance *tcm_alloc_inst(void)
+{
+	struct f_tcm_opts *opts;
+	int i;
+
+
+	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
+	if (!opts)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&tpg_instances_lock);
+	for (i = 0; i < TPG_INSTANCES; ++i)
+		if (!tpg_instances[i].func_inst)
+			break;
+
+	if (i == TPG_INSTANCES) {
+		mutex_unlock(&tpg_instances_lock);
+		kfree(opts);
+		return ERR_PTR(-EBUSY);
+	}
+	tpg_instances[i].func_inst = &opts->func_inst;
+	mutex_unlock(&tpg_instances_lock);
+
+	mutex_init(&opts->dep_lock);
+	opts->func_inst.set_inst_name = tcm_set_name;
+	opts->func_inst.free_func_inst = tcm_free_inst;
+	opts->tcm_register_callback = tcm_register_callback;
+	opts->tcm_unregister_callback = tcm_unregister_callback;
+
+	config_group_init_type_name(&opts->func_inst.group, "",
+			&tcm_func_type);
+
+	return &opts->func_inst;
+}
+
+static void tcm_free(struct usb_function *f)
+{
+	struct f_uas *tcm = to_f_uas(f);
+
+	kfree(tcm);
+}
+
+static void tcm_unbind(struct usb_configuration *c, struct usb_function *f)
+{
+	usb_free_all_descriptors(f);
+}
+
+static struct usb_function *tcm_alloc(struct usb_function_instance *fi)
+{
+	struct f_uas *fu;
+	unsigned i;
+
+	mutex_lock(&tpg_instances_lock);
+	for (i = 0; i < TPG_INSTANCES; ++i)
+		if (tpg_instances[i].func_inst == fi)
+			break;
+	if (i == TPG_INSTANCES) {
+		mutex_unlock(&tpg_instances_lock);
+		return ERR_PTR(-ENODEV);
+	}
+
+	fu = kzalloc(sizeof(*fu), GFP_KERNEL);
+	if (!fu) {
+		mutex_unlock(&tpg_instances_lock);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	fu->function.name = "Target Function";
+	fu->function.bind = tcm_bind;
+	fu->function.unbind = tcm_unbind;
+	fu->function.set_alt = tcm_set_alt;
+	fu->function.setup = tcm_setup;
+	fu->function.disable = tcm_disable;
+	fu->function.free_func = tcm_free;
+	fu->tpg = tpg_instances[i].tpg;
+	mutex_unlock(&tpg_instances_lock);
+
+	return &fu->function;
+}
+
+DECLARE_USB_FUNCTION(tcm, tcm_alloc_inst, tcm_alloc);
+
+static int tcm_init(void)
+{
+	int ret;
+
+	ret = usb_function_register(&tcmusb_func);
+	if (ret)
+		return ret;
+
+	ret = target_register_template(&usbg_ops);
+	if (ret)
+		usb_function_unregister(&tcmusb_func);
+
+	return ret;
+}
+module_init(tcm_init);
+
+static void tcm_exit(void)
+{
+	target_unregister_template(&usbg_ops);
+	usb_function_unregister(&tcmusb_func);
+}
+module_exit(tcm_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Sebastian Andrzej Siewior");
diff --git a/drivers/usb/gadget/legacy/tcm_usb_gadget.h b/drivers/usb/gadget/function/tcm.h
index 0b749e1aa2f1..b75c6f3e1980 100644
--- a/drivers/usb/gadget/legacy/tcm_usb_gadget.h
+++ b/drivers/usb/gadget/function/tcm.h
@@ -16,8 +16,7 @@
 #define UASP_SS_EP_COMP_NUM_STREAMS (1 << UASP_SS_EP_COMP_LOG_STREAMS)
 
 enum {
-	USB_G_STR_CONFIG = USB_GADGET_FIRST_AVAIL_IDX,
-	USB_G_STR_INT_UAS,
+	USB_G_STR_INT_UAS = 0,
 	USB_G_STR_INT_BBB,
 };
 
@@ -40,6 +39,8 @@ struct usbg_tpg {
 	u32 gadget_connect;
 	struct tcm_usbg_nexus *tpg_nexus;
 	atomic_t tpg_port_count;
+
+	struct usb_function_instance *fi;
 };
 
 struct usbg_tport {
@@ -128,6 +129,4 @@ struct f_uas {
 	struct usb_request	*bot_req_out;
 };
 
-extern struct usbg_tpg *the_only_tpg_I_currently_have;
-
-#endif
+#endif /* __TARGET_USB_GADGET_H__ */
diff --git a/drivers/usb/gadget/function/u_tcm.h b/drivers/usb/gadget/function/u_tcm.h
new file mode 100644
index 000000000000..0bd751e0483f
--- /dev/null
+++ b/drivers/usb/gadget/function/u_tcm.h
@@ -0,0 +1,50 @@
+/*
+ * u_tcm.h
+ *
+ * Utility definitions for the tcm function
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ *		http://www.samsung.com
+ *
+ * Author: Andrzej Pietrasiewicz <andrzej.p@xxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef U_TCM_H
+#define U_TCM_H
+
+#include <linux/usb/composite.h>
+
+/**
+ * @dependent: optional dependent module. Meant for legacy gadget.
+ * If non-null its refcount will be increased when a tpg is created and
+ * decreased when tpg is dropped.
+ * @dep_lock: lock for dependent module operations.
+ * @ready: true if the dependent module information is set.
+ * @can_attach: true a function can be bound to gadget
+ * @has_dep: true if there is a dependent module
+ *
+ */
+struct f_tcm_opts {
+	struct usb_function_instance	func_inst;
+	struct module			*dependent;
+	struct mutex			dep_lock;
+	bool				ready;
+	bool				can_attach;
+	bool				has_dep;
+
+	/*
+	 * Callbacks to be removed when legacy tcm gadget disappears.
+	 *
+	 * If you use the new function registration interface
+	 * programmatically, you MUST set these callbacks to
+	 * something sensible (e.g. probe/remove the composite).
+	 */
+	int (*tcm_register_callback)(struct usb_function_instance *);
+	void (*tcm_unregister_callback)(struct usb_function_instance *);
+};
+
+#endif /* U_TCM_H */
diff --git a/drivers/usb/gadget/legacy/Kconfig b/drivers/usb/gadget/legacy/Kconfig
index 4d682ad7bf23..a23d1b90454c 100644
--- a/drivers/usb/gadget/legacy/Kconfig
+++ b/drivers/usb/gadget/legacy/Kconfig
@@ -250,6 +250,7 @@ config USB_GADGET_TARGET
 	tristate "USB Gadget Target Fabric Module"
 	depends on TARGET_CORE
 	select USB_LIBCOMPOSITE
+	select USB_F_TCM
 	help
 	  This fabric is an USB gadget. Two USB protocols are supported that is
 	  BBB or BOT (Bulk Only Transport) and UAS (USB Attached SCSI). BOT is
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index 365afd7e14f8..7e179f81d05c 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -1521,10 +1521,10 @@ static void destroy_ep_files (struct dev_data *dev)
 		spin_unlock_irq (&dev->lock);
 
 		/* break link to dcache */
-		mutex_lock (&parent->i_mutex);
+		inode_lock(parent);
 		d_delete (dentry);
 		dput (dentry);
-		mutex_unlock (&parent->i_mutex);
+		inode_unlock(parent);
 
 		spin_lock_irq (&dev->lock);
 	}
diff --git a/drivers/usb/gadget/legacy/tcm_usb_gadget.c b/drivers/usb/gadget/legacy/tcm_usb_gadget.c
index 7857fa411636..0b0bb98319cd 100644
--- a/drivers/usb/gadget/legacy/tcm_usb_gadget.c
+++ b/drivers/usb/gadget/legacy/tcm_usb_gadget.c
@@ -21,1953 +21,10 @@
 #include <target/target_core_fabric.h>
 #include <asm/unaligned.h>
 
-#include "tcm_usb_gadget.h"
+#include "u_tcm.h"
 
 USB_GADGET_COMPOSITE_OPTIONS();
 
-static inline struct f_uas *to_f_uas(struct usb_function *f)
-{
-	return container_of(f, struct f_uas, function);
-}
-
-static void usbg_cmd_release(struct kref *);
-
-static inline void usbg_cleanup_cmd(struct usbg_cmd *cmd)
-{
-	kref_put(&cmd->ref, usbg_cmd_release);
-}
-
-/* Start bot.c code */
-
-static int bot_enqueue_cmd_cbw(struct f_uas *fu)
-{
-	int ret;
-
-	if (fu->flags & USBG_BOT_CMD_PEND)
-		return 0;
-
-	ret = usb_ep_queue(fu->ep_out, fu->cmd.req, GFP_ATOMIC);
-	if (!ret)
-		fu->flags |= USBG_BOT_CMD_PEND;
-	return ret;
-}
-
-static void bot_status_complete(struct usb_ep *ep, struct usb_request *req)
-{
-	struct usbg_cmd *cmd = req->context;
-	struct f_uas *fu = cmd->fu;
-
-	usbg_cleanup_cmd(cmd);
-	if (req->status < 0) {
-		pr_err("ERR %s(%d)\n", __func__, __LINE__);
-		return;
-	}
-
-	/* CSW completed, wait for next CBW */
-	bot_enqueue_cmd_cbw(fu);
-}
-
-static void bot_enqueue_sense_code(struct f_uas *fu, struct usbg_cmd *cmd)
-{
-	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
-	int ret;
-	u8 *sense;
-	unsigned int csw_stat;
-
-	csw_stat = cmd->csw_code;
-
-	/*
-	 * We can't send SENSE as a response. So we take ASC & ASCQ from our
-	 * sense buffer and queue it and hope the host sends a REQUEST_SENSE
-	 * command where it learns why we failed.
-	 */
-	sense = cmd->sense_iu.sense;
-
-	csw->Tag = cmd->bot_tag;
-	csw->Status = csw_stat;
-	fu->bot_status.req->context = cmd;
-	ret = usb_ep_queue(fu->ep_in, fu->bot_status.req, GFP_ATOMIC);
-	if (ret)
-		pr_err("%s(%d) ERR: %d\n", __func__, __LINE__, ret);
-}
-
-static void bot_err_compl(struct usb_ep *ep, struct usb_request *req)
-{
-	struct usbg_cmd *cmd = req->context;
-	struct f_uas *fu = cmd->fu;
-
-	if (req->status < 0)
-		pr_err("ERR %s(%d)\n", __func__, __LINE__);
-
-	if (cmd->data_len) {
-		if (cmd->data_len > ep->maxpacket) {
-			req->length = ep->maxpacket;
-			cmd->data_len -= ep->maxpacket;
-		} else {
-			req->length = cmd->data_len;
-			cmd->data_len = 0;
-		}
-
-		usb_ep_queue(ep, req, GFP_ATOMIC);
-		return ;
-	}
-	bot_enqueue_sense_code(fu, cmd);
-}
-
-static void bot_send_bad_status(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
-	struct usb_request *req;
-	struct usb_ep *ep;
-
-	csw->Residue = cpu_to_le32(cmd->data_len);
-
-	if (cmd->data_len) {
-		if (cmd->is_read) {
-			ep = fu->ep_in;
-			req = fu->bot_req_in;
-		} else {
-			ep = fu->ep_out;
-			req = fu->bot_req_out;
-		}
-
-		if (cmd->data_len > fu->ep_in->maxpacket) {
-			req->length = ep->maxpacket;
-			cmd->data_len -= ep->maxpacket;
-		} else {
-			req->length = cmd->data_len;
-			cmd->data_len = 0;
-		}
-		req->complete = bot_err_compl;
-		req->context = cmd;
-		req->buf = fu->cmd.buf;
-		usb_ep_queue(ep, req, GFP_KERNEL);
-	} else {
-		bot_enqueue_sense_code(fu, cmd);
-	}
-}
-
-static int bot_send_status(struct usbg_cmd *cmd, bool moved_data)
-{
-	struct f_uas *fu = cmd->fu;
-	struct bulk_cs_wrap *csw = &fu->bot_status.csw;
-	int ret;
-
-	if (cmd->se_cmd.scsi_status == SAM_STAT_GOOD) {
-		if (!moved_data && cmd->data_len) {
-			/*
-			 * the host wants to move data, we don't. Fill / empty
-			 * the pipe and then send the csw with reside set.
-			 */
-			cmd->csw_code = US_BULK_STAT_OK;
-			bot_send_bad_status(cmd);
-			return 0;
-		}
-
-		csw->Tag = cmd->bot_tag;
-		csw->Residue = cpu_to_le32(0);
-		csw->Status = US_BULK_STAT_OK;
-		fu->bot_status.req->context = cmd;
-
-		ret = usb_ep_queue(fu->ep_in, fu->bot_status.req, GFP_KERNEL);
-		if (ret)
-			pr_err("%s(%d) ERR: %d\n", __func__, __LINE__, ret);
-	} else {
-		cmd->csw_code = US_BULK_STAT_FAIL;
-		bot_send_bad_status(cmd);
-	}
-	return 0;
-}
-
-/*
- * Called after command (no data transfer) or after the write (to device)
- * operation is completed
- */
-static int bot_send_status_response(struct usbg_cmd *cmd)
-{
-	bool moved_data = false;
-
-	if (!cmd->is_read)
-		moved_data = true;
-	return bot_send_status(cmd, moved_data);
-}
-
-/* Read request completed, now we have to send the CSW */
-static void bot_read_compl(struct usb_ep *ep, struct usb_request *req)
-{
-	struct usbg_cmd *cmd = req->context;
-
-	if (req->status < 0)
-		pr_err("ERR %s(%d)\n", __func__, __LINE__);
-
-	bot_send_status(cmd, true);
-}
-
-static int bot_send_read_response(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct usb_gadget *gadget = fuas_to_gadget(fu);
-	int ret;
-
-	if (!cmd->data_len) {
-		cmd->csw_code = US_BULK_STAT_PHASE;
-		bot_send_bad_status(cmd);
-		return 0;
-	}
-
-	if (!gadget->sg_supported) {
-		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
-		if (!cmd->data_buf)
-			return -ENOMEM;
-
-		sg_copy_to_buffer(se_cmd->t_data_sg,
-				se_cmd->t_data_nents,
-				cmd->data_buf,
-				se_cmd->data_length);
-
-		fu->bot_req_in->buf = cmd->data_buf;
-	} else {
-		fu->bot_req_in->buf = NULL;
-		fu->bot_req_in->num_sgs = se_cmd->t_data_nents;
-		fu->bot_req_in->sg = se_cmd->t_data_sg;
-	}
-
-	fu->bot_req_in->complete = bot_read_compl;
-	fu->bot_req_in->length = se_cmd->data_length;
-	fu->bot_req_in->context = cmd;
-	ret = usb_ep_queue(fu->ep_in, fu->bot_req_in, GFP_ATOMIC);
-	if (ret)
-		pr_err("%s(%d)\n", __func__, __LINE__);
-	return 0;
-}
-
-static void usbg_data_write_cmpl(struct usb_ep *, struct usb_request *);
-static int usbg_prepare_w_request(struct usbg_cmd *, struct usb_request *);
-
-static int bot_send_write_request(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct usb_gadget *gadget = fuas_to_gadget(fu);
-	int ret;
-
-	init_completion(&cmd->write_complete);
-	cmd->fu = fu;
-
-	if (!cmd->data_len) {
-		cmd->csw_code = US_BULK_STAT_PHASE;
-		return -EINVAL;
-	}
-
-	if (!gadget->sg_supported) {
-		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_KERNEL);
-		if (!cmd->data_buf)
-			return -ENOMEM;
-
-		fu->bot_req_out->buf = cmd->data_buf;
-	} else {
-		fu->bot_req_out->buf = NULL;
-		fu->bot_req_out->num_sgs = se_cmd->t_data_nents;
-		fu->bot_req_out->sg = se_cmd->t_data_sg;
-	}
-
-	fu->bot_req_out->complete = usbg_data_write_cmpl;
-	fu->bot_req_out->length = se_cmd->data_length;
-	fu->bot_req_out->context = cmd;
-
-	ret = usbg_prepare_w_request(cmd, fu->bot_req_out);
-	if (ret)
-		goto cleanup;
-	ret = usb_ep_queue(fu->ep_out, fu->bot_req_out, GFP_KERNEL);
-	if (ret)
-		pr_err("%s(%d)\n", __func__, __LINE__);
-
-	wait_for_completion(&cmd->write_complete);
-	target_execute_cmd(se_cmd);
-cleanup:
-	return ret;
-}
-
-static int bot_submit_command(struct f_uas *, void *, unsigned int);
-
-static void bot_cmd_complete(struct usb_ep *ep, struct usb_request *req)
-{
-	struct f_uas *fu = req->context;
-	int ret;
-
-	fu->flags &= ~USBG_BOT_CMD_PEND;
-
-	if (req->status < 0)
-		return;
-
-	ret = bot_submit_command(fu, req->buf, req->actual);
-	if (ret)
-		pr_err("%s(%d): %d\n", __func__, __LINE__, ret);
-}
-
-static int bot_prepare_reqs(struct f_uas *fu)
-{
-	int ret;
-
-	fu->bot_req_in = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
-	if (!fu->bot_req_in)
-		goto err;
-
-	fu->bot_req_out = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
-	if (!fu->bot_req_out)
-		goto err_out;
-
-	fu->cmd.req = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
-	if (!fu->cmd.req)
-		goto err_cmd;
-
-	fu->bot_status.req = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
-	if (!fu->bot_status.req)
-		goto err_sts;
-
-	fu->bot_status.req->buf = &fu->bot_status.csw;
-	fu->bot_status.req->length = US_BULK_CS_WRAP_LEN;
-	fu->bot_status.req->complete = bot_status_complete;
-	fu->bot_status.csw.Signature = cpu_to_le32(US_BULK_CS_SIGN);
-
-	fu->cmd.buf = kmalloc(fu->ep_out->maxpacket, GFP_KERNEL);
-	if (!fu->cmd.buf)
-		goto err_buf;
-
-	fu->cmd.req->complete = bot_cmd_complete;
-	fu->cmd.req->buf = fu->cmd.buf;
-	fu->cmd.req->length = fu->ep_out->maxpacket;
-	fu->cmd.req->context = fu;
-
-	ret = bot_enqueue_cmd_cbw(fu);
-	if (ret)
-		goto err_queue;
-	return 0;
-err_queue:
-	kfree(fu->cmd.buf);
-	fu->cmd.buf = NULL;
-err_buf:
-	usb_ep_free_request(fu->ep_in, fu->bot_status.req);
-err_sts:
-	usb_ep_free_request(fu->ep_out, fu->cmd.req);
-	fu->cmd.req = NULL;
-err_cmd:
-	usb_ep_free_request(fu->ep_out, fu->bot_req_out);
-	fu->bot_req_out = NULL;
-err_out:
-	usb_ep_free_request(fu->ep_in, fu->bot_req_in);
-	fu->bot_req_in = NULL;
-err:
-	pr_err("BOT: endpoint setup failed\n");
-	return -ENOMEM;
-}
-
-static void bot_cleanup_old_alt(struct f_uas *fu)
-{
-	if (!(fu->flags & USBG_ENABLED))
-		return;
-
-	usb_ep_disable(fu->ep_in);
-	usb_ep_disable(fu->ep_out);
-
-	if (!fu->bot_req_in)
-		return;
-
-	usb_ep_free_request(fu->ep_in, fu->bot_req_in);
-	usb_ep_free_request(fu->ep_out, fu->bot_req_out);
-	usb_ep_free_request(fu->ep_out, fu->cmd.req);
-	usb_ep_free_request(fu->ep_out, fu->bot_status.req);
-
-	kfree(fu->cmd.buf);
-
-	fu->bot_req_in = NULL;
-	fu->bot_req_out = NULL;
-	fu->cmd.req = NULL;
-	fu->bot_status.req = NULL;
-	fu->cmd.buf = NULL;
-}
-
-static void bot_set_alt(struct f_uas *fu)
-{
-	struct usb_function *f = &fu->function;
-	struct usb_gadget *gadget = f->config->cdev->gadget;
-	int ret;
-
-	fu->flags = USBG_IS_BOT;
-
-	config_ep_by_speed(gadget, f, fu->ep_in);
-	ret = usb_ep_enable(fu->ep_in);
-	if (ret)
-		goto err_b_in;
-
-	config_ep_by_speed(gadget, f, fu->ep_out);
-	ret = usb_ep_enable(fu->ep_out);
-	if (ret)
-		goto err_b_out;
-
-	ret = bot_prepare_reqs(fu);
-	if (ret)
-		goto err_wq;
-	fu->flags |= USBG_ENABLED;
-	pr_info("Using the BOT protocol\n");
-	return;
-err_wq:
-	usb_ep_disable(fu->ep_out);
-err_b_out:
-	usb_ep_disable(fu->ep_in);
-err_b_in:
-	fu->flags = USBG_IS_BOT;
-}
-
-static int usbg_bot_setup(struct usb_function *f,
-		const struct usb_ctrlrequest *ctrl)
-{
-	struct f_uas *fu = to_f_uas(f);
-	struct usb_composite_dev *cdev = f->config->cdev;
-	u16 w_value = le16_to_cpu(ctrl->wValue);
-	u16 w_length = le16_to_cpu(ctrl->wLength);
-	int luns;
-	u8 *ret_lun;
-
-	switch (ctrl->bRequest) {
-	case US_BULK_GET_MAX_LUN:
-		if (ctrl->bRequestType != (USB_DIR_IN | USB_TYPE_CLASS |
-					USB_RECIP_INTERFACE))
-			return -ENOTSUPP;
-
-		if (w_length < 1)
-			return -EINVAL;
-		if (w_value != 0)
-			return -EINVAL;
-		luns = atomic_read(&fu->tpg->tpg_port_count);
-		if (!luns) {
-			pr_err("No LUNs configured?\n");
-			return -EINVAL;
-		}
-		/*
-		 * If 4 LUNs are present we return 3 i.e. LUN 0..3 can be
-		 * accessed. The upper limit is 0xf
-		 */
-		luns--;
-		if (luns > 0xf) {
-			pr_info_once("Limiting the number of luns to 16\n");
-			luns = 0xf;
-		}
-		ret_lun = cdev->req->buf;
-		*ret_lun = luns;
-		cdev->req->length = 1;
-		return usb_ep_queue(cdev->gadget->ep0, cdev->req, GFP_ATOMIC);
-		break;
-
-	case US_BULK_RESET_REQUEST:
-		/* XXX maybe we should remove previous requests for IN + OUT */
-		bot_enqueue_cmd_cbw(fu);
-		return 0;
-		break;
-	}
-	return -ENOTSUPP;
-}
-
-/* Start uas.c code */
-
-static void uasp_cleanup_one_stream(struct f_uas *fu, struct uas_stream *stream)
-{
-	/* We have either all three allocated or none */
-	if (!stream->req_in)
-		return;
-
-	usb_ep_free_request(fu->ep_in, stream->req_in);
-	usb_ep_free_request(fu->ep_out, stream->req_out);
-	usb_ep_free_request(fu->ep_status, stream->req_status);
-
-	stream->req_in = NULL;
-	stream->req_out = NULL;
-	stream->req_status = NULL;
-}
-
-static void uasp_free_cmdreq(struct f_uas *fu)
-{
-	usb_ep_free_request(fu->ep_cmd, fu->cmd.req);
-	kfree(fu->cmd.buf);
-	fu->cmd.req = NULL;
-	fu->cmd.buf = NULL;
-}
-
-static void uasp_cleanup_old_alt(struct f_uas *fu)
-{
-	int i;
-
-	if (!(fu->flags & USBG_ENABLED))
-		return;
-
-	usb_ep_disable(fu->ep_in);
-	usb_ep_disable(fu->ep_out);
-	usb_ep_disable(fu->ep_status);
-	usb_ep_disable(fu->ep_cmd);
-
-	for (i = 0; i < UASP_SS_EP_COMP_NUM_STREAMS; i++)
-		uasp_cleanup_one_stream(fu, &fu->stream[i]);
-	uasp_free_cmdreq(fu);
-}
-
-static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req);
-
-static int uasp_prepare_r_request(struct usbg_cmd *cmd)
-{
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct f_uas *fu = cmd->fu;
-	struct usb_gadget *gadget = fuas_to_gadget(fu);
-	struct uas_stream *stream = cmd->stream;
-
-	if (!gadget->sg_supported) {
-		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
-		if (!cmd->data_buf)
-			return -ENOMEM;
-
-		sg_copy_to_buffer(se_cmd->t_data_sg,
-				se_cmd->t_data_nents,
-				cmd->data_buf,
-				se_cmd->data_length);
-
-		stream->req_in->buf = cmd->data_buf;
-	} else {
-		stream->req_in->buf = NULL;
-		stream->req_in->num_sgs = se_cmd->t_data_nents;
-		stream->req_in->sg = se_cmd->t_data_sg;
-	}
-
-	stream->req_in->complete = uasp_status_data_cmpl;
-	stream->req_in->length = se_cmd->data_length;
-	stream->req_in->context = cmd;
-
-	cmd->state = UASP_SEND_STATUS;
-	return 0;
-}
-
-static void uasp_prepare_status(struct usbg_cmd *cmd)
-{
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct sense_iu *iu = &cmd->sense_iu;
-	struct uas_stream *stream = cmd->stream;
-
-	cmd->state = UASP_QUEUE_COMMAND;
-	iu->iu_id = IU_ID_STATUS;
-	iu->tag = cpu_to_be16(cmd->tag);
-
-	/*
-	 * iu->status_qual = cpu_to_be16(STATUS QUALIFIER SAM-4. Where R U?);
-	 */
-	iu->len = cpu_to_be16(se_cmd->scsi_sense_length);
-	iu->status = se_cmd->scsi_status;
-	stream->req_status->context = cmd;
-	stream->req_status->length = se_cmd->scsi_sense_length + 16;
-	stream->req_status->buf = iu;
-	stream->req_status->complete = uasp_status_data_cmpl;
-}
-
-static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req)
-{
-	struct usbg_cmd *cmd = req->context;
-	struct uas_stream *stream = cmd->stream;
-	struct f_uas *fu = cmd->fu;
-	int ret;
-
-	if (req->status < 0)
-		goto cleanup;
-
-	switch (cmd->state) {
-	case UASP_SEND_DATA:
-		ret = uasp_prepare_r_request(cmd);
-		if (ret)
-			goto cleanup;
-		ret = usb_ep_queue(fu->ep_in, stream->req_in, GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
-		break;
-
-	case UASP_RECEIVE_DATA:
-		ret = usbg_prepare_w_request(cmd, stream->req_out);
-		if (ret)
-			goto cleanup;
-		ret = usb_ep_queue(fu->ep_out, stream->req_out, GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
-		break;
-
-	case UASP_SEND_STATUS:
-		uasp_prepare_status(cmd);
-		ret = usb_ep_queue(fu->ep_status, stream->req_status,
-				GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
-		break;
-
-	case UASP_QUEUE_COMMAND:
-		usbg_cleanup_cmd(cmd);
-		usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
-		break;
-
-	default:
-		BUG();
-	}
-	return;
-
-cleanup:
-	usbg_cleanup_cmd(cmd);
-}
-
-static int uasp_send_status_response(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct uas_stream *stream = cmd->stream;
-	struct sense_iu *iu = &cmd->sense_iu;
-
-	iu->tag = cpu_to_be16(cmd->tag);
-	stream->req_status->complete = uasp_status_data_cmpl;
-	stream->req_status->context = cmd;
-	cmd->fu = fu;
-	uasp_prepare_status(cmd);
-	return usb_ep_queue(fu->ep_status, stream->req_status, GFP_ATOMIC);
-}
-
-static int uasp_send_read_response(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct uas_stream *stream = cmd->stream;
-	struct sense_iu *iu = &cmd->sense_iu;
-	int ret;
-
-	cmd->fu = fu;
-
-	iu->tag = cpu_to_be16(cmd->tag);
-	if (fu->flags & USBG_USE_STREAMS) {
-
-		ret = uasp_prepare_r_request(cmd);
-		if (ret)
-			goto out;
-		ret = usb_ep_queue(fu->ep_in, stream->req_in, GFP_ATOMIC);
-		if (ret) {
-			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
-			kfree(cmd->data_buf);
-			cmd->data_buf = NULL;
-		}
-
-	} else {
-
-		iu->iu_id = IU_ID_READ_READY;
-		iu->tag = cpu_to_be16(cmd->tag);
-
-		stream->req_status->complete = uasp_status_data_cmpl;
-		stream->req_status->context = cmd;
-
-		cmd->state = UASP_SEND_DATA;
-		stream->req_status->buf = iu;
-		stream->req_status->length = sizeof(struct iu);
-
-		ret = usb_ep_queue(fu->ep_status, stream->req_status,
-				GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d) => %d\n", __func__, __LINE__, ret);
-	}
-out:
-	return ret;
-}
-
-static int uasp_send_write_request(struct usbg_cmd *cmd)
-{
-	struct f_uas *fu = cmd->fu;
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct uas_stream *stream = cmd->stream;
-	struct sense_iu *iu = &cmd->sense_iu;
-	int ret;
-
-	init_completion(&cmd->write_complete);
-	cmd->fu = fu;
-
-	iu->tag = cpu_to_be16(cmd->tag);
-
-	if (fu->flags & USBG_USE_STREAMS) {
-
-		ret = usbg_prepare_w_request(cmd, stream->req_out);
-		if (ret)
-			goto cleanup;
-		ret = usb_ep_queue(fu->ep_out, stream->req_out, GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d)\n", __func__, __LINE__);
-
-	} else {
-
-		iu->iu_id = IU_ID_WRITE_READY;
-		iu->tag = cpu_to_be16(cmd->tag);
-
-		stream->req_status->complete = uasp_status_data_cmpl;
-		stream->req_status->context = cmd;
-
-		cmd->state = UASP_RECEIVE_DATA;
-		stream->req_status->buf = iu;
-		stream->req_status->length = sizeof(struct iu);
-
-		ret = usb_ep_queue(fu->ep_status, stream->req_status,
-				GFP_ATOMIC);
-		if (ret)
-			pr_err("%s(%d)\n", __func__, __LINE__);
-	}
-
-	wait_for_completion(&cmd->write_complete);
-	target_execute_cmd(se_cmd);
-cleanup:
-	return ret;
-}
-
-static int usbg_submit_command(struct f_uas *, void *, unsigned int);
-
-static void uasp_cmd_complete(struct usb_ep *ep, struct usb_request *req)
-{
-	struct f_uas *fu = req->context;
-	int ret;
-
-	if (req->status < 0)
-		return;
-
-	ret = usbg_submit_command(fu, req->buf, req->actual);
-	/*
-	 * Once we tune for performance enqueue the command req here again so
-	 * we can receive a second command while we processing this one. Pay
-	 * attention to properly sync STAUS endpoint with DATA IN + OUT so you
-	 * don't break HS.
-	 */
-	if (!ret)
-		return;
-	usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
-}
-
-static int uasp_alloc_stream_res(struct f_uas *fu, struct uas_stream *stream)
-{
-	stream->req_in = usb_ep_alloc_request(fu->ep_in, GFP_KERNEL);
-	if (!stream->req_in)
-		goto out;
-
-	stream->req_out = usb_ep_alloc_request(fu->ep_out, GFP_KERNEL);
-	if (!stream->req_out)
-		goto err_out;
-
-	stream->req_status = usb_ep_alloc_request(fu->ep_status, GFP_KERNEL);
-	if (!stream->req_status)
-		goto err_sts;
-
-	return 0;
-err_sts:
-	usb_ep_free_request(fu->ep_status, stream->req_status);
-	stream->req_status = NULL;
-err_out:
-	usb_ep_free_request(fu->ep_out, stream->req_out);
-	stream->req_out = NULL;
-out:
-	return -ENOMEM;
-}
-
-static int uasp_alloc_cmd(struct f_uas *fu)
-{
-	fu->cmd.req = usb_ep_alloc_request(fu->ep_cmd, GFP_KERNEL);
-	if (!fu->cmd.req)
-		goto err;
-
-	fu->cmd.buf = kmalloc(fu->ep_cmd->maxpacket, GFP_KERNEL);
-	if (!fu->cmd.buf)
-		goto err_buf;
-
-	fu->cmd.req->complete = uasp_cmd_complete;
-	fu->cmd.req->buf = fu->cmd.buf;
-	fu->cmd.req->length = fu->ep_cmd->maxpacket;
-	fu->cmd.req->context = fu;
-	return 0;
-
-err_buf:
-	usb_ep_free_request(fu->ep_cmd, fu->cmd.req);
-err:
-	return -ENOMEM;
-}
-
-static void uasp_setup_stream_res(struct f_uas *fu, int max_streams)
-{
-	int i;
-
-	for (i = 0; i < max_streams; i++) {
-		struct uas_stream *s = &fu->stream[i];
-
-		s->req_in->stream_id = i + 1;
-		s->req_out->stream_id = i + 1;
-		s->req_status->stream_id = i + 1;
-	}
-}
-
-static int uasp_prepare_reqs(struct f_uas *fu)
-{
-	int ret;
-	int i;
-	int max_streams;
-
-	if (fu->flags & USBG_USE_STREAMS)
-		max_streams = UASP_SS_EP_COMP_NUM_STREAMS;
-	else
-		max_streams = 1;
-
-	for (i = 0; i < max_streams; i++) {
-		ret = uasp_alloc_stream_res(fu, &fu->stream[i]);
-		if (ret)
-			goto err_cleanup;
-	}
-
-	ret = uasp_alloc_cmd(fu);
-	if (ret)
-		goto err_free_stream;
-	uasp_setup_stream_res(fu, max_streams);
-
-	ret = usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC);
-	if (ret)
-		goto err_free_stream;
-
-	return 0;
-
-err_free_stream:
-	uasp_free_cmdreq(fu);
-
-err_cleanup:
-	if (i) {
-		do {
-			uasp_cleanup_one_stream(fu, &fu->stream[i - 1]);
-			i--;
-		} while (i);
-	}
-	pr_err("UASP: endpoint setup failed\n");
-	return ret;
-}
-
-static void uasp_set_alt(struct f_uas *fu)
-{
-	struct usb_function *f = &fu->function;
-	struct usb_gadget *gadget = f->config->cdev->gadget;
-	int ret;
-
-	fu->flags = USBG_IS_UAS;
-
-	if (gadget->speed == USB_SPEED_SUPER)
-		fu->flags |= USBG_USE_STREAMS;
-
-	config_ep_by_speed(gadget, f, fu->ep_in);
-	ret = usb_ep_enable(fu->ep_in);
-	if (ret)
-		goto err_b_in;
-
-	config_ep_by_speed(gadget, f, fu->ep_out);
-	ret = usb_ep_enable(fu->ep_out);
-	if (ret)
-		goto err_b_out;
-
-	config_ep_by_speed(gadget, f, fu->ep_cmd);
-	ret = usb_ep_enable(fu->ep_cmd);
-	if (ret)
-		goto err_cmd;
-	config_ep_by_speed(gadget, f, fu->ep_status);
-	ret = usb_ep_enable(fu->ep_status);
-	if (ret)
-		goto err_status;
-
-	ret = uasp_prepare_reqs(fu);
-	if (ret)
-		goto err_wq;
-	fu->flags |= USBG_ENABLED;
-
-	pr_info("Using the UAS protocol\n");
-	return;
-err_wq:
-	usb_ep_disable(fu->ep_status);
-err_status:
-	usb_ep_disable(fu->ep_cmd);
-err_cmd:
-	usb_ep_disable(fu->ep_out);
-err_b_out:
-	usb_ep_disable(fu->ep_in);
-err_b_in:
-	fu->flags = 0;
-}
-
-static int get_cmd_dir(const unsigned char *cdb)
-{
-	int ret;
-
-	switch (cdb[0]) {
-	case READ_6:
-	case READ_10:
-	case READ_12:
-	case READ_16:
-	case INQUIRY:
-	case MODE_SENSE:
-	case MODE_SENSE_10:
-	case SERVICE_ACTION_IN_16:
-	case MAINTENANCE_IN:
-	case PERSISTENT_RESERVE_IN:
-	case SECURITY_PROTOCOL_IN:
-	case ACCESS_CONTROL_IN:
-	case REPORT_LUNS:
-	case READ_BLOCK_LIMITS:
-	case READ_POSITION:
-	case READ_CAPACITY:
-	case READ_TOC:
-	case READ_FORMAT_CAPACITIES:
-	case REQUEST_SENSE:
-		ret = DMA_FROM_DEVICE;
-		break;
-
-	case WRITE_6:
-	case WRITE_10:
-	case WRITE_12:
-	case WRITE_16:
-	case MODE_SELECT:
-	case MODE_SELECT_10:
-	case WRITE_VERIFY:
-	case WRITE_VERIFY_12:
-	case PERSISTENT_RESERVE_OUT:
-	case MAINTENANCE_OUT:
-	case SECURITY_PROTOCOL_OUT:
-	case ACCESS_CONTROL_OUT:
-		ret = DMA_TO_DEVICE;
-		break;
-	case ALLOW_MEDIUM_REMOVAL:
-	case TEST_UNIT_READY:
-	case SYNCHRONIZE_CACHE:
-	case START_STOP:
-	case ERASE:
-	case REZERO_UNIT:
-	case SEEK_10:
-	case SPACE:
-	case VERIFY:
-	case WRITE_FILEMARKS:
-		ret = DMA_NONE;
-		break;
-	default:
-		pr_warn("target: Unknown data direction for SCSI Opcode "
-				"0x%02x\n", cdb[0]);
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static void usbg_data_write_cmpl(struct usb_ep *ep, struct usb_request *req)
-{
-	struct usbg_cmd *cmd = req->context;
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-
-	if (req->status < 0) {
-		pr_err("%s() state %d transfer failed\n", __func__, cmd->state);
-		goto cleanup;
-	}
-
-	if (req->num_sgs == 0) {
-		sg_copy_from_buffer(se_cmd->t_data_sg,
-				se_cmd->t_data_nents,
-				cmd->data_buf,
-				se_cmd->data_length);
-	}
-
-	complete(&cmd->write_complete);
-	return;
-
-cleanup:
-	usbg_cleanup_cmd(cmd);
-}
-
-static int usbg_prepare_w_request(struct usbg_cmd *cmd, struct usb_request *req)
-{
-	struct se_cmd *se_cmd = &cmd->se_cmd;
-	struct f_uas *fu = cmd->fu;
-	struct usb_gadget *gadget = fuas_to_gadget(fu);
-
-	if (!gadget->sg_supported) {
-		cmd->data_buf = kmalloc(se_cmd->data_length, GFP_ATOMIC);
-		if (!cmd->data_buf)
-			return -ENOMEM;
-
-		req->buf = cmd->data_buf;
-	} else {
-		req->buf = NULL;
-		req->num_sgs = se_cmd->t_data_nents;
-		req->sg = se_cmd->t_data_sg;
-	}
-
-	req->complete = usbg_data_write_cmpl;
-	req->length = se_cmd->data_length;
-	req->context = cmd;
-	return 0;
-}
-
-static int usbg_send_status_response(struct se_cmd *se_cmd)
-{
-	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
-			se_cmd);
-	struct f_uas *fu = cmd->fu;
-
-	if (fu->flags & USBG_IS_BOT)
-		return bot_send_status_response(cmd);
-	else
-		return uasp_send_status_response(cmd);
-}
-
-static int usbg_send_write_request(struct se_cmd *se_cmd)
-{
-	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
-			se_cmd);
-	struct f_uas *fu = cmd->fu;
-
-	if (fu->flags & USBG_IS_BOT)
-		return bot_send_write_request(cmd);
-	else
-		return uasp_send_write_request(cmd);
-}
-
-static int usbg_send_read_response(struct se_cmd *se_cmd)
-{
-	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
-			se_cmd);
-	struct f_uas *fu = cmd->fu;
-
-	if (fu->flags & USBG_IS_BOT)
-		return bot_send_read_response(cmd);
-	else
-		return uasp_send_read_response(cmd);
-}
-
-static void usbg_cmd_work(struct work_struct *work)
-{
-	struct usbg_cmd *cmd = container_of(work, struct usbg_cmd, work);
-	struct se_cmd *se_cmd;
-	struct tcm_usbg_nexus *tv_nexus;
-	struct usbg_tpg *tpg;
-	int dir;
-
-	se_cmd = &cmd->se_cmd;
-	tpg = cmd->fu->tpg;
-	tv_nexus = tpg->tpg_nexus;
-	dir = get_cmd_dir(cmd->cmd_buf);
-	if (dir < 0) {
-		transport_init_se_cmd(se_cmd,
-				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
-				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
-				cmd->prio_attr, cmd->sense_iu.sense);
-		goto out;
-	}
-
-	if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess,
-			cmd->cmd_buf, cmd->sense_iu.sense, cmd->unpacked_lun,
-			0, cmd->prio_attr, dir, TARGET_SCF_UNKNOWN_SIZE) < 0)
-		goto out;
-
-	return;
-
-out:
-	transport_send_check_condition_and_sense(se_cmd,
-			TCM_UNSUPPORTED_SCSI_OPCODE, 1);
-	usbg_cleanup_cmd(cmd);
-}
-
-static int usbg_submit_command(struct f_uas *fu,
-		void *cmdbuf, unsigned int len)
-{
-	struct command_iu *cmd_iu = cmdbuf;
-	struct usbg_cmd *cmd;
-	struct usbg_tpg *tpg;
-	struct se_cmd *se_cmd;
-	struct tcm_usbg_nexus *tv_nexus;
-	u32 cmd_len;
-	int ret;
-
-	if (cmd_iu->iu_id != IU_ID_COMMAND) {
-		pr_err("Unsupported type %d\n", cmd_iu->iu_id);
-		return -EINVAL;
-	}
-
-	cmd = kzalloc(sizeof *cmd, GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
-
-	cmd->fu = fu;
-
-	/* XXX until I figure out why I can't free in on complete */
-	kref_init(&cmd->ref);
-	kref_get(&cmd->ref);
-
-	tpg = fu->tpg;
-	cmd_len = (cmd_iu->len & ~0x3) + 16;
-	if (cmd_len > USBG_MAX_CMD)
-		goto err;
-
-	memcpy(cmd->cmd_buf, cmd_iu->cdb, cmd_len);
-
-	cmd->tag = be16_to_cpup(&cmd_iu->tag);
-	cmd->se_cmd.tag = cmd->tag;
-	if (fu->flags & USBG_USE_STREAMS) {
-		if (cmd->tag > UASP_SS_EP_COMP_NUM_STREAMS)
-			goto err;
-		if (!cmd->tag)
-			cmd->stream = &fu->stream[0];
-		else
-			cmd->stream = &fu->stream[cmd->tag - 1];
-	} else {
-		cmd->stream = &fu->stream[0];
-	}
-
-	tv_nexus = tpg->tpg_nexus;
-	if (!tv_nexus) {
-		pr_err("Missing nexus, ignoring command\n");
-		goto err;
-	}
-
-	switch (cmd_iu->prio_attr & 0x7) {
-	case UAS_HEAD_TAG:
-		cmd->prio_attr = TCM_HEAD_TAG;
-		break;
-	case UAS_ORDERED_TAG:
-		cmd->prio_attr = TCM_ORDERED_TAG;
-		break;
-	case UAS_ACA:
-		cmd->prio_attr = TCM_ACA_TAG;
-		break;
-	default:
-		pr_debug_once("Unsupported prio_attr: %02x.\n",
-				cmd_iu->prio_attr);
-	case UAS_SIMPLE_TAG:
-		cmd->prio_attr = TCM_SIMPLE_TAG;
-		break;
-	}
-
-	se_cmd = &cmd->se_cmd;
-	cmd->unpacked_lun = scsilun_to_int(&cmd_iu->lun);
-
-	INIT_WORK(&cmd->work, usbg_cmd_work);
-	ret = queue_work(tpg->workqueue, &cmd->work);
-	if (ret < 0)
-		goto err;
-
-	return 0;
-err:
-	kfree(cmd);
-	return -EINVAL;
-}
-
-static void bot_cmd_work(struct work_struct *work)
-{
-	struct usbg_cmd *cmd = container_of(work, struct usbg_cmd, work);
-	struct se_cmd *se_cmd;
-	struct tcm_usbg_nexus *tv_nexus;
-	struct usbg_tpg *tpg;
-	int dir;
-
-	se_cmd = &cmd->se_cmd;
-	tpg = cmd->fu->tpg;
-	tv_nexus = tpg->tpg_nexus;
-	dir = get_cmd_dir(cmd->cmd_buf);
-	if (dir < 0) {
-		transport_init_se_cmd(se_cmd,
-				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
-				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
-				cmd->prio_attr, cmd->sense_iu.sense);
-		goto out;
-	}
-
-	if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess,
-			cmd->cmd_buf, cmd->sense_iu.sense, cmd->unpacked_lun,
-			cmd->data_len, cmd->prio_attr, dir, 0) < 0)
-		goto out;
-
-	return;
-
-out:
-	transport_send_check_condition_and_sense(se_cmd,
-				TCM_UNSUPPORTED_SCSI_OPCODE, 1);
-	usbg_cleanup_cmd(cmd);
-}
-
-static int bot_submit_command(struct f_uas *fu,
-		void *cmdbuf, unsigned int len)
-{
-	struct bulk_cb_wrap *cbw = cmdbuf;
-	struct usbg_cmd *cmd;
-	struct usbg_tpg *tpg;
-	struct se_cmd *se_cmd;
-	struct tcm_usbg_nexus *tv_nexus;
-	u32 cmd_len;
-	int ret;
-
-	if (cbw->Signature != cpu_to_le32(US_BULK_CB_SIGN)) {
-		pr_err("Wrong signature on CBW\n");
-		return -EINVAL;
-	}
-	if (len != 31) {
-		pr_err("Wrong length for CBW\n");
-		return -EINVAL;
-	}
-
-	cmd_len = cbw->Length;
-	if (cmd_len < 1 || cmd_len > 16)
-		return -EINVAL;
-
-	cmd = kzalloc(sizeof *cmd, GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
-
-	cmd->fu = fu;
-
-	/* XXX until I figure out why I can't free in on complete */
-	kref_init(&cmd->ref);
-	kref_get(&cmd->ref);
-
-	tpg = fu->tpg;
-
-	memcpy(cmd->cmd_buf, cbw->CDB, cmd_len);
-
-	cmd->bot_tag = cbw->Tag;
-
-	tv_nexus = tpg->tpg_nexus;
-	if (!tv_nexus) {
-		pr_err("Missing nexus, ignoring command\n");
-		goto err;
-	}
-
-	cmd->prio_attr = TCM_SIMPLE_TAG;
-	se_cmd = &cmd->se_cmd;
-	cmd->unpacked_lun = cbw->Lun;
-	cmd->is_read = cbw->Flags & US_BULK_FLAG_IN ? 1 : 0;
-	cmd->data_len = le32_to_cpu(cbw->DataTransferLength);
-	cmd->se_cmd.tag = le32_to_cpu(cmd->bot_tag);
-
-	INIT_WORK(&cmd->work, bot_cmd_work);
-	ret = queue_work(tpg->workqueue, &cmd->work);
-	if (ret < 0)
-		goto err;
-
-	return 0;
-err:
-	kfree(cmd);
-	return -EINVAL;
-}
-
-/* Start fabric.c code */
-
-static int usbg_check_true(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
-static int usbg_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
-static char *usbg_get_fabric_name(void)
-{
-	return "usb_gadget";
-}
-
-static char *usbg_get_fabric_wwn(struct se_portal_group *se_tpg)
-{
-	struct usbg_tpg *tpg = container_of(se_tpg,
-				struct usbg_tpg, se_tpg);
-	struct usbg_tport *tport = tpg->tport;
-
-	return &tport->tport_name[0];
-}
-
-static u16 usbg_get_tag(struct se_portal_group *se_tpg)
-{
-	struct usbg_tpg *tpg = container_of(se_tpg,
-				struct usbg_tpg, se_tpg);
-	return tpg->tport_tpgt;
-}
-
-static u32 usbg_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
-static void usbg_cmd_release(struct kref *ref)
-{
-	struct usbg_cmd *cmd = container_of(ref, struct usbg_cmd,
-			ref);
-
-	transport_generic_free_cmd(&cmd->se_cmd, 0);
-}
-
-static void usbg_release_cmd(struct se_cmd *se_cmd)
-{
-	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
-			se_cmd);
-	kfree(cmd->data_buf);
-	kfree(cmd);
-	return;
-}
-
-static int usbg_shutdown_session(struct se_session *se_sess)
-{
-	return 0;
-}
-
-static void usbg_close_session(struct se_session *se_sess)
-{
-	return;
-}
-
-static u32 usbg_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
-/*
- * XXX Error recovery: return != 0 if we expect writes. Dunno when that could be
- */
-static int usbg_write_pending_status(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
-static void usbg_set_default_node_attrs(struct se_node_acl *nacl)
-{
-	return;
-}
-
-static int usbg_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
-static void usbg_queue_tm_rsp(struct se_cmd *se_cmd)
-{
-}
-
-static void usbg_aborted_task(struct se_cmd *se_cmd)
-{
-	return;
-}
-
-static const char *usbg_check_wwn(const char *name)
-{
-	const char *n;
-	unsigned int len;
-
-	n = strstr(name, "naa.");
-	if (!n)
-		return NULL;
-	n += 4;
-	len = strlen(n);
-	if (len == 0 || len > USBG_NAMELEN - 1)
-		return NULL;
-	return n;
-}
-
-static int usbg_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
-{
-	if (!usbg_check_wwn(name))
-		return -EINVAL;
-	return 0;
-}
-
-struct usbg_tpg *the_only_tpg_I_currently_have;
-
-static struct se_portal_group *usbg_make_tpg(
-	struct se_wwn *wwn,
-	struct config_group *group,
-	const char *name)
-{
-	struct usbg_tport *tport = container_of(wwn, struct usbg_tport,
-			tport_wwn);
-	struct usbg_tpg *tpg;
-	unsigned long tpgt;
-	int ret;
-
-	if (strstr(name, "tpgt_") != name)
-		return ERR_PTR(-EINVAL);
-	if (kstrtoul(name + 5, 0, &tpgt) || tpgt > UINT_MAX)
-		return ERR_PTR(-EINVAL);
-	if (the_only_tpg_I_currently_have) {
-		pr_err("Until the gadget framework can't handle multiple\n");
-		pr_err("gadgets, you can't do this here.\n");
-		return ERR_PTR(-EBUSY);
-	}
-
-	tpg = kzalloc(sizeof(struct usbg_tpg), GFP_KERNEL);
-	if (!tpg)
-		return ERR_PTR(-ENOMEM);
-	mutex_init(&tpg->tpg_mutex);
-	atomic_set(&tpg->tpg_port_count, 0);
-	tpg->workqueue = alloc_workqueue("tcm_usb_gadget", 0, 1);
-	if (!tpg->workqueue) {
-		kfree(tpg);
-		return NULL;
-	}
-
-	tpg->tport = tport;
-	tpg->tport_tpgt = tpgt;
-
-	/*
-	 * SPC doesn't assign a protocol identifier for USB-SCSI, so we
-	 * pretend to be SAS..
-	 */
-	ret = core_tpg_register(wwn, &tpg->se_tpg, SCSI_PROTOCOL_SAS);
-	if (ret < 0) {
-		destroy_workqueue(tpg->workqueue);
-		kfree(tpg);
-		return NULL;
-	}
-	the_only_tpg_I_currently_have = tpg;
-	return &tpg->se_tpg;
-}
-
-static void usbg_drop_tpg(struct se_portal_group *se_tpg)
-{
-	struct usbg_tpg *tpg = container_of(se_tpg,
-				struct usbg_tpg, se_tpg);
-
-	core_tpg_deregister(se_tpg);
-	destroy_workqueue(tpg->workqueue);
-	kfree(tpg);
-	the_only_tpg_I_currently_have = NULL;
-}
-
-static struct se_wwn *usbg_make_tport(
-	struct target_fabric_configfs *tf,
-	struct config_group *group,
-	const char *name)
-{
-	struct usbg_tport *tport;
-	const char *wnn_name;
-	u64 wwpn = 0;
-
-	wnn_name = usbg_check_wwn(name);
-	if (!wnn_name)
-		return ERR_PTR(-EINVAL);
-
-	tport = kzalloc(sizeof(struct usbg_tport), GFP_KERNEL);
-	if (!(tport))
-		return ERR_PTR(-ENOMEM);
-	tport->tport_wwpn = wwpn;
-	snprintf(tport->tport_name, sizeof(tport->tport_name), "%s", wnn_name);
-	return &tport->tport_wwn;
-}
-
-static void usbg_drop_tport(struct se_wwn *wwn)
-{
-	struct usbg_tport *tport = container_of(wwn,
-				struct usbg_tport, tport_wwn);
-	kfree(tport);
-}
-
-/*
- * If somebody feels like dropping the version property, go ahead.
- */
-static ssize_t usbg_wwn_version_show(struct config_item *item, char *page)
-{
-	return sprintf(page, "usb-gadget fabric module\n");
-}
-
-CONFIGFS_ATTR_RO(usbg_wwn_, version);
-
-static struct configfs_attribute *usbg_wwn_attrs[] = {
-	&usbg_wwn_attr_version,
-	NULL,
-};
-
-static ssize_t tcm_usbg_tpg_enable_show(struct config_item *item, char *page)
-{
-	struct se_portal_group *se_tpg = to_tpg(item);
-	struct usbg_tpg  *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-
-	return snprintf(page, PAGE_SIZE, "%u\n", tpg->gadget_connect);
-}
-
-static int usbg_attach(struct usbg_tpg *);
-static void usbg_detach(struct usbg_tpg *);
-
-static ssize_t tcm_usbg_tpg_enable_store(struct config_item *item,
-		const char *page, size_t count)
-{
-	struct se_portal_group *se_tpg = to_tpg(item);
-	struct usbg_tpg  *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-	unsigned long op;
-	ssize_t ret;
-
-	ret = kstrtoul(page, 0, &op);
-	if (ret < 0)
-		return -EINVAL;
-	if (op > 1)
-		return -EINVAL;
-
-	if (op && tpg->gadget_connect)
-		goto out;
-	if (!op && !tpg->gadget_connect)
-		goto out;
-
-	if (op) {
-		ret = usbg_attach(tpg);
-		if (ret)
-			goto out;
-	} else {
-		usbg_detach(tpg);
-	}
-	tpg->gadget_connect = op;
-out:
-	return count;
-}
-
-static ssize_t tcm_usbg_tpg_nexus_show(struct config_item *item, char *page)
-{
-	struct se_portal_group *se_tpg = to_tpg(item);
-	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-	struct tcm_usbg_nexus *tv_nexus;
-	ssize_t ret;
-
-	mutex_lock(&tpg->tpg_mutex);
-	tv_nexus = tpg->tpg_nexus;
-	if (!tv_nexus) {
-		ret = -ENODEV;
-		goto out;
-	}
-	ret = snprintf(page, PAGE_SIZE, "%s\n",
-			tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
-out:
-	mutex_unlock(&tpg->tpg_mutex);
-	return ret;
-}
-
-static int tcm_usbg_make_nexus(struct usbg_tpg *tpg, char *name)
-{
-	struct se_portal_group *se_tpg;
-	struct tcm_usbg_nexus *tv_nexus;
-	int ret;
-
-	mutex_lock(&tpg->tpg_mutex);
-	if (tpg->tpg_nexus) {
-		ret = -EEXIST;
-		pr_debug("tpg->tpg_nexus already exists\n");
-		goto err_unlock;
-	}
-	se_tpg = &tpg->se_tpg;
-
-	ret = -ENOMEM;
-	tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
-	if (!tv_nexus)
-		goto err_unlock;
-	tv_nexus->tvn_se_sess = transport_init_session(TARGET_PROT_NORMAL);
-	if (IS_ERR(tv_nexus->tvn_se_sess))
-		goto err_free;
-
-	/*
-	 * Since we are running in 'demo mode' this call with generate a
-	 * struct se_node_acl for the tcm_vhost struct se_portal_group with
-	 * the SCSI Initiator port name of the passed configfs group 'name'.
-	 */
-	tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl(
-			se_tpg, name);
-	if (!tv_nexus->tvn_se_sess->se_node_acl) {
-		pr_debug("core_tpg_check_initiator_node_acl() failed"
-				" for %s\n", name);
-		goto err_session;
-	}
-	/*
-	 * Now register the TCM vHost virtual I_T Nexus as active.
-	 */
-	transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl,
-			tv_nexus->tvn_se_sess, tv_nexus);
-	tpg->tpg_nexus = tv_nexus;
-	mutex_unlock(&tpg->tpg_mutex);
-	return 0;
-
-err_session:
-	transport_free_session(tv_nexus->tvn_se_sess);
-err_free:
-	kfree(tv_nexus);
-err_unlock:
-	mutex_unlock(&tpg->tpg_mutex);
-	return ret;
-}
-
-static int tcm_usbg_drop_nexus(struct usbg_tpg *tpg)
-{
-	struct se_session *se_sess;
-	struct tcm_usbg_nexus *tv_nexus;
-	int ret = -ENODEV;
-
-	mutex_lock(&tpg->tpg_mutex);
-	tv_nexus = tpg->tpg_nexus;
-	if (!tv_nexus)
-		goto out;
-
-	se_sess = tv_nexus->tvn_se_sess;
-	if (!se_sess)
-		goto out;
-
-	if (atomic_read(&tpg->tpg_port_count)) {
-		ret = -EPERM;
-		pr_err("Unable to remove Host I_T Nexus with"
-				" active TPG port count: %d\n",
-				atomic_read(&tpg->tpg_port_count));
-		goto out;
-	}
-
-	pr_debug("Removing I_T Nexus to Initiator Port: %s\n",
-			tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
-	/*
-	 * Release the SCSI I_T Nexus to the emulated vHost Target Port
-	 */
-	transport_deregister_session(tv_nexus->tvn_se_sess);
-	tpg->tpg_nexus = NULL;
-
-	kfree(tv_nexus);
-	ret = 0;
-out:
-	mutex_unlock(&tpg->tpg_mutex);
-	return ret;
-}
-
-static ssize_t tcm_usbg_tpg_nexus_store(struct config_item *item,
-		const char *page, size_t count)
-{
-	struct se_portal_group *se_tpg = to_tpg(item);
-	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-	unsigned char i_port[USBG_NAMELEN], *ptr;
-	int ret;
-
-	if (!strncmp(page, "NULL", 4)) {
-		ret = tcm_usbg_drop_nexus(tpg);
-		return (!ret) ? count : ret;
-	}
-	if (strlen(page) >= USBG_NAMELEN) {
-		pr_err("Emulated NAA Sas Address: %s, exceeds"
-				" max: %d\n", page, USBG_NAMELEN);
-		return -EINVAL;
-	}
-	snprintf(i_port, USBG_NAMELEN, "%s", page);
-
-	ptr = strstr(i_port, "naa.");
-	if (!ptr) {
-		pr_err("Missing 'naa.' prefix\n");
-		return -EINVAL;
-	}
-
-	if (i_port[strlen(i_port) - 1] == '\n')
-		i_port[strlen(i_port) - 1] = '\0';
-
-	ret = tcm_usbg_make_nexus(tpg, &i_port[4]);
-	if (ret < 0)
-		return ret;
-	return count;
-}
-
-CONFIGFS_ATTR(tcm_usbg_tpg_, enable);
-CONFIGFS_ATTR(tcm_usbg_tpg_, nexus);
-
-static struct configfs_attribute *usbg_base_attrs[] = {
-	&tcm_usbg_tpg_attr_enable,
-	&tcm_usbg_tpg_attr_nexus,
-	NULL,
-};
-
-static int usbg_port_link(struct se_portal_group *se_tpg, struct se_lun *lun)
-{
-	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-
-	atomic_inc(&tpg->tpg_port_count);
-	smp_mb__after_atomic();
-	return 0;
-}
-
-static void usbg_port_unlink(struct se_portal_group *se_tpg,
-		struct se_lun *se_lun)
-{
-	struct usbg_tpg *tpg = container_of(se_tpg, struct usbg_tpg, se_tpg);
-
-	atomic_dec(&tpg->tpg_port_count);
-	smp_mb__after_atomic();
-}
-
-static int usbg_check_stop_free(struct se_cmd *se_cmd)
-{
-	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
-			se_cmd);
-
-	kref_put(&cmd->ref, usbg_cmd_release);
-	return 1;
-}
-
-static const struct target_core_fabric_ops usbg_ops = {
-	.module				= THIS_MODULE,
-	.name				= "usb_gadget",
-	.get_fabric_name		= usbg_get_fabric_name,
-	.tpg_get_wwn			= usbg_get_fabric_wwn,
-	.tpg_get_tag			= usbg_get_tag,
-	.tpg_check_demo_mode		= usbg_check_true,
-	.tpg_check_demo_mode_cache	= usbg_check_false,
-	.tpg_check_demo_mode_write_protect = usbg_check_false,
-	.tpg_check_prod_mode_write_protect = usbg_check_false,
-	.tpg_get_inst_index		= usbg_tpg_get_inst_index,
-	.release_cmd			= usbg_release_cmd,
-	.shutdown_session		= usbg_shutdown_session,
-	.close_session			= usbg_close_session,
-	.sess_get_index			= usbg_sess_get_index,
-	.sess_get_initiator_sid		= NULL,
-	.write_pending			= usbg_send_write_request,
-	.write_pending_status		= usbg_write_pending_status,
-	.set_default_node_attributes	= usbg_set_default_node_attrs,
-	.get_cmd_state			= usbg_get_cmd_state,
-	.queue_data_in			= usbg_send_read_response,
-	.queue_status			= usbg_send_status_response,
-	.queue_tm_rsp			= usbg_queue_tm_rsp,
-	.aborted_task			= usbg_aborted_task,
-	.check_stop_free		= usbg_check_stop_free,
-
-	.fabric_make_wwn		= usbg_make_tport,
-	.fabric_drop_wwn		= usbg_drop_tport,
-	.fabric_make_tpg		= usbg_make_tpg,
-	.fabric_drop_tpg		= usbg_drop_tpg,
-	.fabric_post_link		= usbg_port_link,
-	.fabric_pre_unlink		= usbg_port_unlink,
-	.fabric_init_nodeacl		= usbg_init_nodeacl,
-
-	.tfc_wwn_attrs			= usbg_wwn_attrs,
-	.tfc_tpg_base_attrs		= usbg_base_attrs,
-};
-
-/* Start gadget.c code */
-
-static struct usb_interface_descriptor bot_intf_desc = {
-	.bLength =              sizeof(bot_intf_desc),
-	.bDescriptorType =      USB_DT_INTERFACE,
-	.bNumEndpoints =        2,
-	.bAlternateSetting =	USB_G_ALT_INT_BBB,
-	.bInterfaceClass =      USB_CLASS_MASS_STORAGE,
-	.bInterfaceSubClass =   USB_SC_SCSI,
-	.bInterfaceProtocol =   USB_PR_BULK,
-};
-
-static struct usb_interface_descriptor uasp_intf_desc = {
-	.bLength =		sizeof(uasp_intf_desc),
-	.bDescriptorType =	USB_DT_INTERFACE,
-	.bNumEndpoints =	4,
-	.bAlternateSetting =	USB_G_ALT_INT_UAS,
-	.bInterfaceClass =	USB_CLASS_MASS_STORAGE,
-	.bInterfaceSubClass =	USB_SC_SCSI,
-	.bInterfaceProtocol =	USB_PR_UAS,
-};
-
-static struct usb_endpoint_descriptor uasp_bi_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(512),
-};
-
-static struct usb_endpoint_descriptor uasp_fs_bi_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-};
-
-static struct usb_pipe_usage_descriptor uasp_bi_pipe_desc = {
-	.bLength =		sizeof(uasp_bi_pipe_desc),
-	.bDescriptorType =	USB_DT_PIPE_USAGE,
-	.bPipeID =		DATA_IN_PIPE_ID,
-};
-
-static struct usb_endpoint_descriptor uasp_ss_bi_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(1024),
-};
-
-static struct usb_ss_ep_comp_descriptor uasp_bi_ep_comp_desc = {
-	.bLength =		sizeof(uasp_bi_ep_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-	.bMaxBurst =		0,
-	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
-	.wBytesPerInterval =	0,
-};
-
-static struct usb_ss_ep_comp_descriptor bot_bi_ep_comp_desc = {
-	.bLength =		sizeof(bot_bi_ep_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-	.bMaxBurst =		0,
-};
-
-static struct usb_endpoint_descriptor uasp_bo_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(512),
-};
-
-static struct usb_endpoint_descriptor uasp_fs_bo_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-};
-
-static struct usb_pipe_usage_descriptor uasp_bo_pipe_desc = {
-	.bLength =		sizeof(uasp_bo_pipe_desc),
-	.bDescriptorType =	USB_DT_PIPE_USAGE,
-	.bPipeID =		DATA_OUT_PIPE_ID,
-};
-
-static struct usb_endpoint_descriptor uasp_ss_bo_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(0x400),
-};
-
-static struct usb_ss_ep_comp_descriptor uasp_bo_ep_comp_desc = {
-	.bLength =		sizeof(uasp_bo_ep_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
-};
-
-static struct usb_ss_ep_comp_descriptor bot_bo_ep_comp_desc = {
-	.bLength =		sizeof(bot_bo_ep_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-};
-
-static struct usb_endpoint_descriptor uasp_status_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(512),
-};
-
-static struct usb_endpoint_descriptor uasp_fs_status_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-};
-
-static struct usb_pipe_usage_descriptor uasp_status_pipe_desc = {
-	.bLength =		sizeof(uasp_status_pipe_desc),
-	.bDescriptorType =	USB_DT_PIPE_USAGE,
-	.bPipeID =		STATUS_PIPE_ID,
-};
-
-static struct usb_endpoint_descriptor uasp_ss_status_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_IN,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(1024),
-};
-
-static struct usb_ss_ep_comp_descriptor uasp_status_in_ep_comp_desc = {
-	.bLength =		sizeof(uasp_status_in_ep_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-	.bmAttributes =		UASP_SS_EP_COMP_LOG_STREAMS,
-};
-
-static struct usb_endpoint_descriptor uasp_cmd_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(512),
-};
-
-static struct usb_endpoint_descriptor uasp_fs_cmd_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-};
-
-static struct usb_pipe_usage_descriptor uasp_cmd_pipe_desc = {
-	.bLength =		sizeof(uasp_cmd_pipe_desc),
-	.bDescriptorType =	USB_DT_PIPE_USAGE,
-	.bPipeID =		CMD_PIPE_ID,
-};
-
-static struct usb_endpoint_descriptor uasp_ss_cmd_desc = {
-	.bLength =		USB_DT_ENDPOINT_SIZE,
-	.bDescriptorType =	USB_DT_ENDPOINT,
-	.bEndpointAddress =	USB_DIR_OUT,
-	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
-	.wMaxPacketSize =	cpu_to_le16(1024),
-};
-
-static struct usb_ss_ep_comp_descriptor uasp_cmd_comp_desc = {
-	.bLength =		sizeof(uasp_cmd_comp_desc),
-	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
-};
-
-static struct usb_descriptor_header *uasp_fs_function_desc[] = {
-	(struct usb_descriptor_header *) &bot_intf_desc,
-	(struct usb_descriptor_header *) &uasp_fs_bi_desc,
-	(struct usb_descriptor_header *) &uasp_fs_bo_desc,
-
-	(struct usb_descriptor_header *) &uasp_intf_desc,
-	(struct usb_descriptor_header *) &uasp_fs_bi_desc,
-	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_fs_bo_desc,
-	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_fs_status_desc,
-	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_fs_cmd_desc,
-	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
-	NULL,
-};
-
-static struct usb_descriptor_header *uasp_hs_function_desc[] = {
-	(struct usb_descriptor_header *) &bot_intf_desc,
-	(struct usb_descriptor_header *) &uasp_bi_desc,
-	(struct usb_descriptor_header *) &uasp_bo_desc,
-
-	(struct usb_descriptor_header *) &uasp_intf_desc,
-	(struct usb_descriptor_header *) &uasp_bi_desc,
-	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_bo_desc,
-	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_status_desc,
-	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_cmd_desc,
-	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
-	NULL,
-};
-
-static struct usb_descriptor_header *uasp_ss_function_desc[] = {
-	(struct usb_descriptor_header *) &bot_intf_desc,
-	(struct usb_descriptor_header *) &uasp_ss_bi_desc,
-	(struct usb_descriptor_header *) &bot_bi_ep_comp_desc,
-	(struct usb_descriptor_header *) &uasp_ss_bo_desc,
-	(struct usb_descriptor_header *) &bot_bo_ep_comp_desc,
-
-	(struct usb_descriptor_header *) &uasp_intf_desc,
-	(struct usb_descriptor_header *) &uasp_ss_bi_desc,
-	(struct usb_descriptor_header *) &uasp_bi_ep_comp_desc,
-	(struct usb_descriptor_header *) &uasp_bi_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_ss_bo_desc,
-	(struct usb_descriptor_header *) &uasp_bo_ep_comp_desc,
-	(struct usb_descriptor_header *) &uasp_bo_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_ss_status_desc,
-	(struct usb_descriptor_header *) &uasp_status_in_ep_comp_desc,
-	(struct usb_descriptor_header *) &uasp_status_pipe_desc,
-	(struct usb_descriptor_header *) &uasp_ss_cmd_desc,
-	(struct usb_descriptor_header *) &uasp_cmd_comp_desc,
-	(struct usb_descriptor_header *) &uasp_cmd_pipe_desc,
-	NULL,
-};
-
 #define UAS_VENDOR_ID	0x0525	/* NetChip */
 #define UAS_PRODUCT_ID	0xa4a5	/* Linux-USB File-backed Storage Gadget */
 
@@ -1981,13 +38,13 @@ static struct usb_device_descriptor usbg_device_desc = {
 	.bNumConfigurations =   1,
 };
 
+#define USB_G_STR_CONFIG USB_GADGET_FIRST_AVAIL_IDX
+
 static struct usb_string	usbg_us_strings[] = {
 	[USB_GADGET_MANUFACTURER_IDX].s	= "Target Manufactor",
 	[USB_GADGET_PRODUCT_IDX].s	= "Target Product",
 	[USB_GADGET_SERIAL_IDX].s	= "000000000001",
 	[USB_G_STR_CONFIG].s		= "default config",
-	[USB_G_STR_INT_UAS].s		= "USB Attached SCSI",
-	[USB_G_STR_INT_BBB].s		= "Bulk Only Transport",
 	{ },
 };
 
@@ -2001,184 +58,42 @@ static struct usb_gadget_strings *usbg_strings[] = {
 	NULL,
 };
 
-static int guas_unbind(struct usb_composite_dev *cdev)
-{
-	return 0;
-}
-
-static struct usb_configuration usbg_config_driver = {
-	.label                  = "Linux Target",
-	.bConfigurationValue    = 1,
-	.bmAttributes           = USB_CONFIG_ATT_SELFPOWER,
-};
+static struct usb_function_instance *fi_tcm;
+static struct usb_function *f_tcm;
 
-static int usbg_bind(struct usb_configuration *c, struct usb_function *f)
+static int guas_unbind(struct usb_composite_dev *cdev)
 {
-	struct f_uas		*fu = to_f_uas(f);
-	struct usb_gadget	*gadget = c->cdev->gadget;
-	struct usb_ep		*ep;
-	int			iface;
-	int			ret;
-
-	iface = usb_interface_id(c, f);
-	if (iface < 0)
-		return iface;
-
-	bot_intf_desc.bInterfaceNumber = iface;
-	uasp_intf_desc.bInterfaceNumber = iface;
-	fu->iface = iface;
-	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_bi_desc,
-			&uasp_bi_ep_comp_desc);
-	if (!ep)
-		goto ep_fail;
-	fu->ep_in = ep;
-
-	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_bo_desc,
-			&uasp_bo_ep_comp_desc);
-	if (!ep)
-		goto ep_fail;
-	fu->ep_out = ep;
-
-	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_status_desc,
-			&uasp_status_in_ep_comp_desc);
-	if (!ep)
-		goto ep_fail;
-	fu->ep_status = ep;
-
-	ep = usb_ep_autoconfig_ss(gadget, &uasp_ss_cmd_desc,
-			&uasp_cmd_comp_desc);
-	if (!ep)
-		goto ep_fail;
-	fu->ep_cmd = ep;
-
-	/* Assume endpoint addresses are the same for both speeds */
-	uasp_bi_desc.bEndpointAddress =	uasp_ss_bi_desc.bEndpointAddress;
-	uasp_bo_desc.bEndpointAddress = uasp_ss_bo_desc.bEndpointAddress;
-	uasp_status_desc.bEndpointAddress =
-		uasp_ss_status_desc.bEndpointAddress;
-	uasp_cmd_desc.bEndpointAddress = uasp_ss_cmd_desc.bEndpointAddress;
-
-	uasp_fs_bi_desc.bEndpointAddress = uasp_ss_bi_desc.bEndpointAddress;
-	uasp_fs_bo_desc.bEndpointAddress = uasp_ss_bo_desc.bEndpointAddress;
-	uasp_fs_status_desc.bEndpointAddress =
-		uasp_ss_status_desc.bEndpointAddress;
-	uasp_fs_cmd_desc.bEndpointAddress = uasp_ss_cmd_desc.bEndpointAddress;
-
-	ret = usb_assign_descriptors(f, uasp_fs_function_desc,
-			uasp_hs_function_desc, uasp_ss_function_desc);
-	if (ret)
-		goto ep_fail;
+	if (!IS_ERR_OR_NULL(f_tcm))
+		usb_put_function(f_tcm);
 
 	return 0;
-ep_fail:
-	pr_err("Can't claim all required eps\n");
-	return -ENOTSUPP;
 }
 
-static void usbg_unbind(struct usb_configuration *c, struct usb_function *f)
+static int tcm_do_config(struct usb_configuration *c)
 {
-	struct f_uas *fu = to_f_uas(f);
+	int status;
 
-	usb_free_all_descriptors(f);
-	kfree(fu);
-}
-
-struct guas_setup_wq {
-	struct work_struct work;
-	struct f_uas *fu;
-	unsigned int alt;
-};
-
-static void usbg_delayed_set_alt(struct work_struct *wq)
-{
-	struct guas_setup_wq *work = container_of(wq, struct guas_setup_wq,
-			work);
-	struct f_uas *fu = work->fu;
-	int alt = work->alt;
-
-	kfree(work);
-
-	if (fu->flags & USBG_IS_BOT)
-		bot_cleanup_old_alt(fu);
-	if (fu->flags & USBG_IS_UAS)
-		uasp_cleanup_old_alt(fu);
-
-	if (alt == USB_G_ALT_INT_BBB)
-		bot_set_alt(fu);
-	else if (alt == USB_G_ALT_INT_UAS)
-		uasp_set_alt(fu);
-	usb_composite_setup_continue(fu->function.config->cdev);
-}
-
-static int usbg_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
-{
-	struct f_uas *fu = to_f_uas(f);
-
-	if ((alt == USB_G_ALT_INT_BBB) || (alt == USB_G_ALT_INT_UAS)) {
-		struct guas_setup_wq *work;
+	f_tcm = usb_get_function(fi_tcm);
+	if (IS_ERR(f_tcm))
+		return PTR_ERR(f_tcm);
 
-		work = kmalloc(sizeof(*work), GFP_ATOMIC);
-		if (!work)
-			return -ENOMEM;
-		INIT_WORK(&work->work, usbg_delayed_set_alt);
-		work->fu = fu;
-		work->alt = alt;
-		schedule_work(&work->work);
-		return USB_GADGET_DELAYED_STATUS;
+	status = usb_add_function(c, f_tcm);
+	if (status < 0) {
+		usb_put_function(f_tcm);
+		return status;
 	}
-	return -EOPNOTSUPP;
-}
-
-static void usbg_disable(struct usb_function *f)
-{
-	struct f_uas *fu = to_f_uas(f);
-
-	if (fu->flags & USBG_IS_UAS)
-		uasp_cleanup_old_alt(fu);
-	else if (fu->flags & USBG_IS_BOT)
-		bot_cleanup_old_alt(fu);
-	fu->flags = 0;
-}
-
-static int usbg_setup(struct usb_function *f,
-		const struct usb_ctrlrequest *ctrl)
-{
-	struct f_uas *fu = to_f_uas(f);
-
-	if (!(fu->flags & USBG_IS_BOT))
-		return -EOPNOTSUPP;
 
-	return usbg_bot_setup(f, ctrl);
+	return 0;
 }
 
-static int usbg_cfg_bind(struct usb_configuration *c)
-{
-	struct f_uas *fu;
-	int ret;
-
-	fu = kzalloc(sizeof(*fu), GFP_KERNEL);
-	if (!fu)
-		return -ENOMEM;
-	fu->function.name = "Target Function";
-	fu->function.bind = usbg_bind;
-	fu->function.unbind = usbg_unbind;
-	fu->function.set_alt = usbg_set_alt;
-	fu->function.setup = usbg_setup;
-	fu->function.disable = usbg_disable;
-	fu->tpg = the_only_tpg_I_currently_have;
-
-	bot_intf_desc.iInterface = usbg_us_strings[USB_G_STR_INT_BBB].id;
-	uasp_intf_desc.iInterface = usbg_us_strings[USB_G_STR_INT_UAS].id;
-
-	ret = usb_add_function(c, &fu->function);
-	if (ret)
-		goto err;
+static struct usb_configuration usbg_config_driver = {
+	.label                  = "Linux Target",
+	.bConfigurationValue    = 1,
+	.bmAttributes           = USB_CONFIG_ATT_SELFPOWER,
+};
 
-	return 0;
-err:
-	kfree(fu);
-	return ret;
-}
+static int usbg_attach(struct usb_function_instance *f);
+static void usbg_detach(struct usb_function_instance *f);
 
 static int usb_target_bind(struct usb_composite_dev *cdev)
 {
@@ -2196,8 +111,7 @@ static int usb_target_bind(struct usb_composite_dev *cdev)
 	usbg_config_driver.iConfiguration =
 		usbg_us_strings[USB_G_STR_CONFIG].id;
 
-	ret = usb_add_config(cdev, &usbg_config_driver,
-			usbg_cfg_bind);
+	ret = usb_add_config(cdev, &usbg_config_driver, tcm_do_config);
 	if (ret)
 		return ret;
 	usb_composite_overwrite_options(cdev, &coverwrite);
@@ -2213,25 +127,44 @@ static struct usb_composite_driver usbg_driver = {
 	.unbind         = guas_unbind,
 };
 
-static int usbg_attach(struct usbg_tpg *tpg)
+static int usbg_attach(struct usb_function_instance *f)
 {
 	return usb_composite_probe(&usbg_driver);
 }
 
-static void usbg_detach(struct usbg_tpg *tpg)
+static void usbg_detach(struct usb_function_instance *f)
 {
 	usb_composite_unregister(&usbg_driver);
 }
 
 static int __init usb_target_gadget_init(void)
 {
-	return target_register_template(&usbg_ops);
+	struct f_tcm_opts *tcm_opts;
+
+	fi_tcm = usb_get_function_instance("tcm");
+	if (IS_ERR(fi_tcm))
+		return PTR_ERR(fi_tcm);
+
+	tcm_opts = container_of(fi_tcm, struct f_tcm_opts, func_inst);
+	mutex_lock(&tcm_opts->dep_lock);
+	tcm_opts->tcm_register_callback = usbg_attach;
+	tcm_opts->tcm_unregister_callback = usbg_detach;
+	tcm_opts->dependent = THIS_MODULE;
+	tcm_opts->can_attach = true;
+	tcm_opts->has_dep = true;
+	mutex_unlock(&tcm_opts->dep_lock);
+
+	fi_tcm->set_inst_name(fi_tcm, "tcm-legacy");
+
+	return 0;
 }
 module_init(usb_target_gadget_init);
 
 static void __exit usb_target_gadget_exit(void)
 {
-	target_unregister_template(&usbg_ops);
+	if (!IS_ERR_OR_NULL(fi_tcm))
+		usb_put_function_instance(fi_tcm);
+
 }
 module_exit(usb_target_gadget_exit);
 
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c
index f92f5aff0dd5..8755b2c2aada 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.c
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.c
@@ -91,7 +91,7 @@ static ssize_t queue_dbg_read(struct file *file, char __user *buf,
 	if (!access_ok(VERIFY_WRITE, buf, nbytes))
 		return -EFAULT;
 
-	mutex_lock(&file_inode(file)->i_mutex);
+	inode_lock(file_inode(file));
 	list_for_each_entry_safe(req, tmp_req, queue, queue) {
 		len = snprintf(tmpbuf, sizeof(tmpbuf),
 				"%8p %08x %c%c%c %5d %c%c%c\n",
@@ -118,7 +118,7 @@ static ssize_t queue_dbg_read(struct file *file, char __user *buf,
 		nbytes -= len;
 		buf += len;
 	}
-	mutex_unlock(&file_inode(file)->i_mutex);
+	inode_unlock(file_inode(file));
 
 	return actual;
 }
@@ -143,7 +143,7 @@ static int regs_dbg_open(struct inode *inode, struct file *file)
 	u32 *data;
 	int ret = -ENOMEM;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	udc = inode->i_private;
 	data = kmalloc(inode->i_size, GFP_KERNEL);
 	if (!data)
@@ -158,7 +158,7 @@ static int regs_dbg_open(struct inode *inode, struct file *file)
 	ret = 0;
 
 out:
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 
 	return ret;
 }
@@ -169,11 +169,11 @@ static ssize_t regs_dbg_read(struct file *file, char __user *buf,
 	struct inode *inode = file_inode(file);
 	int ret;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	ret = simple_read_from_buffer(buf, nbytes, ppos,
 			file->private_data,
 			file_inode(file)->i_size);
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 
 	return ret;
 }
diff --git a/drivers/usb/gadget/udc/fsl_qe_udc.c b/drivers/usb/gadget/udc/fsl_qe_udc.c
index 5fb6f8b4f0b4..53c0692f1b09 100644
--- a/drivers/usb/gadget/udc/fsl_qe_udc.c
+++ b/drivers/usb/gadget/udc/fsl_qe_udc.c
@@ -38,7 +38,7 @@
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
 #include <linux/usb/otg.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/qe.h>
 #include <asm/cpm.h>
 #include <asm/dma.h>
 #include <asm/reg.h>
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index daa563ff1fa0..1f117c360ebb 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -229,6 +229,8 @@ config USB_EHCI_TEGRA
        depends on ARCH_TEGRA
        select USB_EHCI_ROOT_HUB_TT
        select USB_PHY
+	select USB_ULPI
+	select USB_ULPI_VIEWPORT
        help
          This driver enables support for the internal USB Host Controllers
          found in NVIDIA Tegra SoCs. The controllers are EHCI compliant.
diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c
index c6cebb96fd21..0960f41f945a 100644
--- a/drivers/usb/host/fhci-hcd.c
+++ b/drivers/usb/host/fhci-hcd.c
@@ -31,7 +31,7 @@
 #include <linux/of_platform.h>
 #include <linux/of_gpio.h>
 #include <linux/slab.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/qe.h>
 #include <asm/fsl_gtm.h>
 #include "fhci.h"
 
diff --git a/drivers/usb/host/fhci-hub.c b/drivers/usb/host/fhci-hub.c
index 3bacdd7befe9..60d55eb3de0d 100644
--- a/drivers/usb/host/fhci-hub.c
+++ b/drivers/usb/host/fhci-hub.c
@@ -24,7 +24,7 @@
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
 #include <linux/gpio.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/qe.h>
 #include "fhci.h"
 
 /* virtual root hub specific descriptor */
diff --git a/drivers/usb/host/fhci-sched.c b/drivers/usb/host/fhci-sched.c
index 95ca5986e672..a9609a336efe 100644
--- a/drivers/usb/host/fhci-sched.c
+++ b/drivers/usb/host/fhci-sched.c
@@ -25,7 +25,7 @@
 #include <linux/io.h>
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
-#include <asm/qe.h>
+#include <soc/fsl/qe/qe.h>
 #include <asm/fsl_gtm.h>
 #include "fhci.h"
 
diff --git a/drivers/usb/host/fhci.h b/drivers/usb/host/fhci.h
index 154e6a007727..3fc82c1c3c73 100644
--- a/drivers/usb/host/fhci.h
+++ b/drivers/usb/host/fhci.h
@@ -27,8 +27,8 @@
 #include <linux/io.h>
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
-#include <asm/qe.h>
-#include <asm/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+#include <soc/fsl/qe/immap_qe.h>
 
 #define USB_CLOCK	48000000
 
diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
index 850d86ca685b..da6e2ce77495 100644
--- a/drivers/vfio/Kconfig
+++ b/drivers/vfio/Kconfig
@@ -31,6 +31,21 @@ menuconfig VFIO
 
 	  If you don't know what to do here, say N.
 
+menuconfig VFIO_NOIOMMU
+	bool "VFIO No-IOMMU support"
+	depends on VFIO
+	help
+	  VFIO is built on the ability to isolate devices using the IOMMU.
+	  Only with an IOMMU can userspace access to DMA capable devices be
+	  considered secure.  VFIO No-IOMMU mode enables IOMMU groups for
+	  devices without IOMMU backing for the purpose of re-using the VFIO
+	  infrastructure in a non-secure mode.  Use of this mode will result
+	  in an unsupportable kernel and will therefore taint the kernel.
+	  Device assignment to virtual machines is also not possible with
+	  this mode since there is no IOMMU to provide DMA translation.
+
+	  If you don't know what to do here, say N.
+
 source "drivers/vfio/pci/Kconfig"
 source "drivers/vfio/platform/Kconfig"
 source "virt/lib/Kconfig"
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 56bf6dbb93db..2760a7ba3f30 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -940,13 +940,13 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 		return -EINVAL;
 
-	group = iommu_group_get(&pdev->dev);
+	group = vfio_iommu_group_get(&pdev->dev);
 	if (!group)
 		return -EINVAL;
 
 	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 	if (!vdev) {
-		iommu_group_put(group);
+		vfio_iommu_group_put(group, &pdev->dev);
 		return -ENOMEM;
 	}
 
@@ -957,7 +957,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
 	if (ret) {
-		iommu_group_put(group);
+		vfio_iommu_group_put(group, &pdev->dev);
 		kfree(vdev);
 		return ret;
 	}
@@ -993,7 +993,7 @@ static void vfio_pci_remove(struct pci_dev *pdev)
 	if (!vdev)
 		return;
 
-	iommu_group_put(pdev->dev.iommu_group);
+	vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
 	kfree(vdev);
 
 	if (vfio_pci_is_vga(pdev)) {
diff --git a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
index da5356f48d0b..d4030d0c38e9 100644
--- a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
+++ b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
@@ -110,7 +110,7 @@ int vfio_platform_amdxgbe_reset(struct vfio_platform_device *vdev)
 	usleep_range(10, 15);
 
 	count = 2000;
-	while (count-- && (ioread32(xgmac_regs->ioaddr + DMA_MR) & 1))
+	while (--count && (ioread32(xgmac_regs->ioaddr + DMA_MR) & 1))
 		usleep_range(500, 600);
 
 	if (!count)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 6070b793cbcb..ecca316386f5 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -62,6 +62,7 @@ struct vfio_container {
 	struct rw_semaphore		group_lock;
 	struct vfio_iommu_driver	*iommu_driver;
 	void				*iommu_data;
+	bool				noiommu;
 };
 
 struct vfio_unbound_dev {
@@ -84,6 +85,7 @@ struct vfio_group {
 	struct list_head		unbound_list;
 	struct mutex			unbound_lock;
 	atomic_t			opened;
+	bool				noiommu;
 };
 
 struct vfio_device {
@@ -95,6 +97,123 @@ struct vfio_device {
 	void				*device_data;
 };
 
+#ifdef CONFIG_VFIO_NOIOMMU
+static bool noiommu __read_mostly;
+module_param_named(enable_unsafe_noiommu_mode,
+		   noiommu, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
+#endif
+
+/*
+ * vfio_iommu_group_{get,put} are only intended for VFIO bus driver probe
+ * and remove functions, any use cases other than acquiring the first
+ * reference for the purpose of calling vfio_add_group_dev() or removing
+ * that symmetric reference after vfio_del_group_dev() should use the raw
+ * iommu_group_{get,put} functions.  In particular, vfio_iommu_group_put()
+ * removes the device from the dummy group and cannot be nested.
+ */
+struct iommu_group *vfio_iommu_group_get(struct device *dev)
+{
+	struct iommu_group *group;
+	int __maybe_unused ret;
+
+	group = iommu_group_get(dev);
+
+#ifdef CONFIG_VFIO_NOIOMMU
+	/*
+	 * With noiommu enabled, an IOMMU group will be created for a device
+	 * that doesn't already have one and doesn't have an iommu_ops on their
+	 * bus.  We set iommudata simply to be able to identify these groups
+	 * as special use and for reclamation later.
+	 */
+	if (group || !noiommu || iommu_present(dev->bus))
+		return group;
+
+	group = iommu_group_alloc();
+	if (IS_ERR(group))
+		return NULL;
+
+	iommu_group_set_name(group, "vfio-noiommu");
+	iommu_group_set_iommudata(group, &noiommu, NULL);
+	ret = iommu_group_add_device(group, dev);
+	iommu_group_put(group);
+	if (ret)
+		return NULL;
+
+	/*
+	 * Where to taint?  At this point we've added an IOMMU group for a
+	 * device that is not backed by iommu_ops, therefore any iommu_
+	 * callback using iommu_ops can legitimately Oops.  So, while we may
+	 * be about to give a DMA capable device to a user without IOMMU
+	 * protection, which is clearly taint-worthy, let's go ahead and do
+	 * it here.
+	 */
+	add_taint(TAINT_USER, LOCKDEP_STILL_OK);
+	dev_warn(dev, "Adding kernel taint for vfio-noiommu group on device\n");
+#endif
+
+	return group;
+}
+EXPORT_SYMBOL_GPL(vfio_iommu_group_get);
+
+void vfio_iommu_group_put(struct iommu_group *group, struct device *dev)
+{
+#ifdef CONFIG_VFIO_NOIOMMU
+	if (iommu_group_get_iommudata(group) == &noiommu)
+		iommu_group_remove_device(dev);
+#endif
+
+	iommu_group_put(group);
+}
+EXPORT_SYMBOL_GPL(vfio_iommu_group_put);
+
+#ifdef CONFIG_VFIO_NOIOMMU
+static void *vfio_noiommu_open(unsigned long arg)
+{
+	if (arg != VFIO_NOIOMMU_IOMMU)
+		return ERR_PTR(-EINVAL);
+	if (!capable(CAP_SYS_RAWIO))
+		return ERR_PTR(-EPERM);
+
+	return NULL;
+}
+
+static void vfio_noiommu_release(void *iommu_data)
+{
+}
+
+static long vfio_noiommu_ioctl(void *iommu_data,
+			       unsigned int cmd, unsigned long arg)
+{
+	if (cmd == VFIO_CHECK_EXTENSION)
+		return noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;
+
+	return -ENOTTY;
+}
+
+static int vfio_noiommu_attach_group(void *iommu_data,
+				     struct iommu_group *iommu_group)
+{
+	return iommu_group_get_iommudata(iommu_group) == &noiommu ? 0 : -EINVAL;
+}
+
+static void vfio_noiommu_detach_group(void *iommu_data,
+				      struct iommu_group *iommu_group)
+{
+}
+
+static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
+	.name = "vfio-noiommu",
+	.owner = THIS_MODULE,
+	.open = vfio_noiommu_open,
+	.release = vfio_noiommu_release,
+	.ioctl = vfio_noiommu_ioctl,
+	.attach_group = vfio_noiommu_attach_group,
+	.detach_group = vfio_noiommu_detach_group,
+};
+#endif
+
+
 /**
  * IOMMU driver registration
  */
@@ -217,6 +336,9 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
 	atomic_set(&group->container_users, 0);
 	atomic_set(&group->opened, 0);
 	group->iommu_group = iommu_group;
+#ifdef CONFIG_VFIO_NOIOMMU
+	group->noiommu = (iommu_group_get_iommudata(iommu_group) == &noiommu);
+#endif
 
 	group->nb.notifier_call = vfio_iommu_group_notifier;
 
@@ -252,7 +374,8 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
 
 	dev = device_create(vfio.class, NULL,
 			    MKDEV(MAJOR(vfio.group_devt), minor),
-			    group, "%d", iommu_group_id(iommu_group));
+			    group, "%s%d", group->noiommu ? "noiommu-" : "",
+			    iommu_group_id(iommu_group));
 	if (IS_ERR(dev)) {
 		vfio_free_group_minor(minor);
 		vfio_group_unlock_and_free(group);
@@ -854,6 +977,14 @@ static long vfio_ioctl_check_extension(struct vfio_container *container,
 			mutex_lock(&vfio.iommu_drivers_lock);
 			list_for_each_entry(driver, &vfio.iommu_drivers_list,
 					    vfio_next) {
+
+#ifdef CONFIG_VFIO_NOIOMMU
+				if (!list_empty(&container->group_list) &&
+				    (container->noiommu !=
+				     (driver->ops == &vfio_noiommu_ops)))
+					continue;
+#endif
+
 				if (!try_module_get(driver->ops->owner))
 					continue;
 
@@ -925,6 +1056,15 @@ static long vfio_ioctl_set_iommu(struct vfio_container *container,
 	list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
 		void *data;
 
+#ifdef CONFIG_VFIO_NOIOMMU
+		/*
+		 * Only noiommu containers can use vfio-noiommu and noiommu
+		 * containers can only use vfio-noiommu.
+		 */
+		if (container->noiommu != (driver->ops == &vfio_noiommu_ops))
+			continue;
+#endif
+
 		if (!try_module_get(driver->ops->owner))
 			continue;
 
@@ -1187,6 +1327,9 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 	if (atomic_read(&group->container_users))
 		return -EINVAL;
 
+	if (group->noiommu && !capable(CAP_SYS_RAWIO))
+		return -EPERM;
+
 	f = fdget(container_fd);
 	if (!f.file)
 		return -EBADF;
@@ -1202,6 +1345,13 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 
 	down_write(&container->group_lock);
 
+	/* Real groups and fake groups cannot mix */
+	if (!list_empty(&container->group_list) &&
+	    container->noiommu != group->noiommu) {
+		ret = -EPERM;
+		goto unlock_out;
+	}
+
 	driver = container->iommu_driver;
 	if (driver) {
 		ret = driver->ops->attach_group(container->iommu_data,
@@ -1211,6 +1361,7 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 	}
 
 	group->container = container;
+	container->noiommu = group->noiommu;
 	list_add(&group->container_next, &container->group_list);
 
 	/* Get a reference on the container and mark a user within the group */
@@ -1241,6 +1392,9 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
 	    !group->container->iommu_driver || !vfio_group_viable(group))
 		return -EINVAL;
 
+	if (group->noiommu && !capable(CAP_SYS_RAWIO))
+		return -EPERM;
+
 	device = vfio_device_get_from_name(group, buf);
 	if (!device)
 		return -ENODEV;
@@ -1283,6 +1437,10 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
 
 	fd_install(ret, filep);
 
+	if (group->noiommu)
+		dev_warn(device->dev, "vfio-noiommu device opened by user "
+			 "(%s:%d)\n", current->comm, task_pid_nr(current));
+
 	return ret;
 }
 
@@ -1371,6 +1529,11 @@ static int vfio_group_fops_open(struct inode *inode, struct file *filep)
 	if (!group)
 		return -ENODEV;
 
+	if (group->noiommu && !capable(CAP_SYS_RAWIO)) {
+		vfio_group_put(group);
+		return -EPERM;
+	}
+
 	/* Do we need multiple instances of the group open?  Seems not. */
 	opened = atomic_cmpxchg(&group->opened, 0, 1);
 	if (opened) {
@@ -1533,6 +1696,11 @@ struct vfio_group *vfio_group_get_external_user(struct file *filep)
 	if (!atomic_inc_not_zero(&group->container_users))
 		return ERR_PTR(-EINVAL);
 
+	if (group->noiommu) {
+		atomic_dec(&group->container_users);
+		return ERR_PTR(-EPERM);
+	}
+
 	if (!group->container->iommu_driver ||
 			!vfio_group_viable(group)) {
 		atomic_dec(&group->container_users);
@@ -1625,6 +1793,9 @@ static int __init vfio_init(void)
 	request_module_nowait("vfio_iommu_type1");
 	request_module_nowait("vfio_iommu_spapr_tce");
 
+#ifdef CONFIG_VFIO_NOIOMMU
+	vfio_register_iommu_driver(&vfio_noiommu_ops);
+#endif
 	return 0;
 
 err_cdev_add:
@@ -1641,6 +1812,9 @@ static void __exit vfio_cleanup(void)
 {
 	WARN_ON(!list_empty(&vfio.group_list));
 
+#ifdef CONFIG_VFIO_NOIOMMU
+	vfio_unregister_iommu_driver(&vfio_noiommu_ops);
+#endif
 	idr_destroy(&vfio.group_idr);
 	cdev_del(&vfio.group_cdev);
 	unregister_chrdev_region(vfio.group_devt, MINORMASK);
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 59d47cb638d5..6f1ea3dddbad 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -995,7 +995,7 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
 		if (info.argsz < minsz)
 			return -EINVAL;
 
-		info.flags = 0;
+		info.flags = VFIO_IOMMU_INFO_PGSIZES;
 
 		info.iova_pgsizes = vfio_pgsize_bitmap(iommu);
 
diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig
index e6d16d65e4e6..8ea45a5cd806 100644
--- a/drivers/video/fbdev/Kconfig
+++ b/drivers/video/fbdev/Kconfig
@@ -5,6 +5,7 @@
 menuconfig FB
 	tristate "Support for frame buffer devices"
 	select FB_CMDLINE
+	select FB_NOTIFY
 	---help---
 	  The frame buffer device provides an abstraction for the graphics
 	  hardware. It represents the frame buffer of some video hardware and
@@ -56,6 +57,9 @@ config FIRMWARE_EDID
 config FB_CMDLINE
 	bool
 
+config FB_NOTIFY
+	bool
+
 config FB_DDC
        tristate
        depends on FB
@@ -1506,6 +1510,7 @@ config FB_SIS
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
 	select FB_BOOT_VESA_SUPPORT if FB_SIS = y
+	select FB_SIS_300 if !FB_SIS_315
 	help
 	  This is the frame buffer device driver for the SiS 300, 315, 330
 	  and 340 series as well as XGI V3XT, V5, V8, Z7 graphics chipsets.
@@ -1880,6 +1885,8 @@ config FB_PXA
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
+	select VIDEOMODE_HELPERS if OF
+	select FB_MODE_HELPERS if OF
 	---help---
 	  Frame buffer driver for the built-in LCD controller in the Intel
 	  PXA2x0 processor.
@@ -1990,16 +1997,6 @@ config FB_SH_MOBILE_LCDC
 	---help---
 	  Frame buffer driver for the on-chip SH-Mobile LCD controller.
 
-config FB_SH_MOBILE_HDMI
-	tristate "SuperH Mobile HDMI controller support"
-	depends on FB_SH_MOBILE_LCDC
-	select FB_MODE_HELPERS
-	select SOUND
-	select SND
-	select SND_SOC
-	---help---
-	  Driver for the on-chip SH-Mobile HDMI controller.
-
 config FB_TMIO
 	tristate "Toshiba Mobile IO FrameBuffer support"
 	depends on FB && (MFD_TMIO || COMPILE_TEST)
diff --git a/drivers/video/fbdev/Makefile b/drivers/video/fbdev/Makefile
index 50ed1b4fc2bf..65fb15075c8f 100644
--- a/drivers/video/fbdev/Makefile
+++ b/drivers/video/fbdev/Makefile
@@ -118,7 +118,6 @@ obj-$(CONFIG_FB_UDL)		  += udlfb.o
 obj-$(CONFIG_FB_SMSCUFX)	  += smscufx.o
 obj-$(CONFIG_FB_XILINX)           += xilinxfb.o
 obj-$(CONFIG_SH_MIPI_DSI)	  += sh_mipi_dsi.o
-obj-$(CONFIG_FB_SH_MOBILE_HDMI)	  += sh_mobile_hdmi.o
 obj-$(CONFIG_FB_SH_MOBILE_MERAM)  += sh_mobile_meram.o
 obj-$(CONFIG_FB_SH_MOBILE_LCDC)	  += sh_mobile_lcdcfb.o
 obj-$(CONFIG_FB_OMAP)             += omap/
diff --git a/drivers/video/fbdev/auo_k190x.c b/drivers/video/fbdev/auo_k190x.c
index 8d2499d1cafb..9580374667ba 100644
--- a/drivers/video/fbdev/auo_k190x.c
+++ b/drivers/video/fbdev/auo_k190x.c
@@ -773,9 +773,7 @@ static void auok190x_recover(struct auok190xfb_par *par)
 /*
  * Power-management
  */
-
-#ifdef CONFIG_PM
-static int auok190x_runtime_suspend(struct device *dev)
+static int __maybe_unused auok190x_runtime_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct fb_info *info = platform_get_drvdata(pdev);
@@ -822,7 +820,7 @@ finish:
 	return 0;
 }
 
-static int auok190x_runtime_resume(struct device *dev)
+static int __maybe_unused auok190x_runtime_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct fb_info *info = platform_get_drvdata(pdev);
@@ -856,7 +854,7 @@ static int auok190x_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static int auok190x_suspend(struct device *dev)
+static int __maybe_unused auok190x_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct fb_info *info = platform_get_drvdata(pdev);
@@ -896,7 +894,7 @@ static int auok190x_suspend(struct device *dev)
 	return 0;
 }
 
-static int auok190x_resume(struct device *dev)
+static int __maybe_unused auok190x_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct fb_info *info = platform_get_drvdata(pdev);
@@ -933,7 +931,6 @@ static int auok190x_resume(struct device *dev)
 
 	return 0;
 }
-#endif
 
 const struct dev_pm_ops auok190x_pm = {
 	SET_RUNTIME_PM_OPS(auok190x_runtime_suspend, auok190x_runtime_resume,
diff --git a/drivers/video/fbdev/core/Makefile b/drivers/video/fbdev/core/Makefile
index 23d86a8b7d7b..9e3ddf225393 100644
--- a/drivers/video/fbdev/core/Makefile
+++ b/drivers/video/fbdev/core/Makefile
@@ -1,5 +1,5 @@
-obj-y                             += fb_notify.o
 obj-$(CONFIG_FB_CMDLINE)          += fb_cmdline.o
+obj-$(CONFIG_FB_NOTIFY)           += fb_notify.o
 obj-$(CONFIG_FB)                  += fb.o
 fb-y                              := fbmem.o fbmon.o fbcmap.o fbsysfs.o \
                                      modedb.o fbcvt.o
diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c
index 3fc63c208d08..57721c73177f 100644
--- a/drivers/video/fbdev/core/fb_defio.c
+++ b/drivers/video/fbdev/core/fb_defio.c
@@ -78,13 +78,13 @@ int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasy
 	if (!info->fbdefio)
 		return 0;
 
-	mutex_lock(&inode->i_mutex);
+	inode_lock(inode);
 	/* Kill off the delayed work */
 	cancel_delayed_work_sync(&info->deferred_work);
 
 	/* Run it immediately */
 	schedule_delayed_work(&info->deferred_work, 0);
-	mutex_unlock(&inode->i_mutex);
+	inode_unlock(inode);
 
 	return 0;
 }
diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index 0705d8883ede..4e73b6f6b1c0 100644
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -1608,6 +1608,11 @@ static int do_remove_conflicting_framebuffers(struct apertures_struct *a,
 	return 0;
 }
 
+static bool lockless_register_fb;
+module_param_named_unsafe(lockless_register_fb, lockless_register_fb, bool, 0400);
+MODULE_PARM_DESC(lockless_register_fb,
+	"Lockless framebuffer registration for debugging [default=off]");
+
 static int do_register_framebuffer(struct fb_info *fb_info)
 {
 	int i, ret;
@@ -1675,15 +1680,18 @@ static int do_register_framebuffer(struct fb_info *fb_info)
 	registered_fb[i] = fb_info;
 
 	event.info = fb_info;
-	console_lock();
+	if (!lockless_register_fb)
+		console_lock();
 	if (!lock_fb_info(fb_info)) {
-		console_unlock();
+		if (!lockless_register_fb)
+			console_unlock();
 		return -ENODEV;
 	}
 
 	fb_notifier_call_chain(FB_EVENT_FB_REGISTERED, &event);
 	unlock_fb_info(fb_info);
-	console_unlock();
+	if (!lockless_register_fb)
+		console_unlock();
 	return 0;
 }
 
diff --git a/drivers/video/fbdev/geode/display_gx1.c b/drivers/video/fbdev/geode/display_gx1.c
index 926d53eeb549..b383eb9882bf 100644
--- a/drivers/video/fbdev/geode/display_gx1.c
+++ b/drivers/video/fbdev/geode/display_gx1.c
@@ -208,7 +208,7 @@ static void gx1_set_hw_palette_reg(struct fb_info *info, unsigned regno,
 	writel(val, par->dc_regs + DC_PAL_DATA);
 }
 
-struct geode_dc_ops gx1_dc_ops = {
+const struct geode_dc_ops gx1_dc_ops = {
 	.set_mode	 = gx1_set_mode,
 	.set_palette_reg = gx1_set_hw_palette_reg,
 };
diff --git a/drivers/video/fbdev/geode/display_gx1.h b/drivers/video/fbdev/geode/display_gx1.h
index 671c05558c79..e1cc41b343ca 100644
--- a/drivers/video/fbdev/geode/display_gx1.h
+++ b/drivers/video/fbdev/geode/display_gx1.h
@@ -18,7 +18,7 @@
 unsigned gx1_gx_base(void);
 int gx1_frame_buffer_size(void);
 
-extern struct geode_dc_ops gx1_dc_ops;
+extern const struct geode_dc_ops gx1_dc_ops;
 
 /* GX1 configuration I/O registers */
 
diff --git a/drivers/video/fbdev/geode/geodefb.h b/drivers/video/fbdev/geode/geodefb.h
index ae04820e0c57..e2e07934868f 100644
--- a/drivers/video/fbdev/geode/geodefb.h
+++ b/drivers/video/fbdev/geode/geodefb.h
@@ -31,8 +31,8 @@ struct geodefb_par {
 	int panel_y;
 	void __iomem *dc_regs;
 	void __iomem *vid_regs;
-	struct geode_dc_ops  *dc_ops;
-	struct geode_vid_ops *vid_ops;
+	const struct geode_dc_ops  *dc_ops;
+	const struct geode_vid_ops *vid_ops;
 };
 
 #endif /* !__GEODEFB_H__ */
diff --git a/drivers/video/fbdev/geode/video_cs5530.c b/drivers/video/fbdev/geode/video_cs5530.c
index 649c3943d431..880613200ecf 100644
--- a/drivers/video/fbdev/geode/video_cs5530.c
+++ b/drivers/video/fbdev/geode/video_cs5530.c
@@ -186,7 +186,7 @@ static int cs5530_blank_display(struct fb_info *info, int blank_mode)
 	return 0;
 }
 
-struct geode_vid_ops cs5530_vid_ops = {
+const struct geode_vid_ops cs5530_vid_ops = {
 	.set_dclk          = cs5530_set_dclk_frequency,
 	.configure_display = cs5530_configure_display,
 	.blank_display     = cs5530_blank_display,
diff --git a/drivers/video/fbdev/geode/video_cs5530.h b/drivers/video/fbdev/geode/video_cs5530.h
index 56cecca7f1ce..c843348bfa20 100644
--- a/drivers/video/fbdev/geode/video_cs5530.h
+++ b/drivers/video/fbdev/geode/video_cs5530.h
@@ -15,7 +15,7 @@
 #ifndef __VIDEO_CS5530_H__
 #define __VIDEO_CS5530_H__
 
-extern struct geode_vid_ops cs5530_vid_ops;
+extern const struct geode_vid_ops cs5530_vid_ops;
 
 /* CS5530 Video device registers */
 
diff --git a/drivers/video/fbdev/i740fb.c b/drivers/video/fbdev/i740fb.c
index 452e1163ad02..cf5ccd0f2252 100644
--- a/drivers/video/fbdev/i740fb.c
+++ b/drivers/video/fbdev/i740fb.c
@@ -346,11 +346,10 @@ static void i740_calc_vclk(u32 freq, struct i740fb_par *par)
 	const u32 err_target = freq / (1000 * I740_RFREQ / I740_FFIX);
 	u32 err_best = 512 * I740_FFIX;
 	u32 f_err, f_vco;
-	int m_best = 0, n_best = 0, p_best = 0, d_best = 0;
+	int m_best = 0, n_best = 0, p_best = 0;
 	int m, n;
 
 	p_best = min(15, ilog2(I740_MAX_VCO_FREQ / (freq / I740_RFREQ_FIX)));
-	d_best = 0;
 	f_vco = (freq * (1 << p_best)) / I740_RFREQ_FIX;
 	freq = freq / I740_RFREQ_FIX;
 
@@ -363,7 +362,7 @@ static void i740_calc_vclk(u32 freq, struct i740fb_par *par)
 			m = 3;
 
 		{
-			u32 f_out = (((m * I740_REF_FREQ * (4 << 2 * d_best))
+			u32 f_out = (((m * I740_REF_FREQ * 4)
 				 / n) + ((1 << p_best) / 2)) / (1 << p_best);
 
 			f_err = (freq - f_out);
@@ -386,8 +385,7 @@ static void i740_calc_vclk(u32 freq, struct i740fb_par *par)
 	par->video_clk2_n = (n_best - 2) & 0xFF;
 	par->video_clk2_mn_msbs = ((((n_best - 2) >> 4) & VCO_N_MSBS)
 				 | (((m_best - 2) >> 8) & VCO_M_MSBS));
-	par->video_clk2_div_sel =
-		((p_best << 4) | (d_best ? 4 : 0) | REF_DIV_1);
+	par->video_clk2_div_sel = ((p_best << 4) | REF_DIV_1);
 }
 
 static int i740fb_decode_var(const struct fb_var_screeninfo *var,
diff --git a/drivers/video/fbdev/omap2/Kconfig b/drivers/video/fbdev/omap2/Kconfig
index c22955d2de9a..0921c4de8407 100644
--- a/drivers/video/fbdev/omap2/Kconfig
+++ b/drivers/video/fbdev/omap2/Kconfig
@@ -1,10 +1,5 @@
-config OMAP2_VRFB
-	bool
-
 if ARCH_OMAP2PLUS
 
-source "drivers/video/fbdev/omap2/dss/Kconfig"
 source "drivers/video/fbdev/omap2/omapfb/Kconfig"
-source "drivers/video/fbdev/omap2/displays-new/Kconfig"
 
 endif
diff --git a/drivers/video/fbdev/omap2/Makefile b/drivers/video/fbdev/omap2/Makefile
index f8745ec369cc..71ab5ac91106 100644
--- a/drivers/video/fbdev/omap2/Makefile
+++ b/drivers/video/fbdev/omap2/Makefile
@@ -1,5 +1 @@
-obj-$(CONFIG_OMAP2_VRFB) += vrfb.o
-
-obj-y += dss/
-obj-y += displays-new/
-obj-$(CONFIG_FB_OMAP2) += omapfb/
+obj-y += omapfb/
diff --git a/drivers/video/fbdev/omap2/omapfb/Kconfig b/drivers/video/fbdev/omap2/omapfb/Kconfig
index 4cb12ce68855..e6226aeed17e 100644
--- a/drivers/video/fbdev/omap2/omapfb/Kconfig
+++ b/drivers/video/fbdev/omap2/omapfb/Kconfig
@@ -1,7 +1,12 @@
+config OMAP2_VRFB
+	bool
+
 menuconfig FB_OMAP2
         tristate "OMAP2+ frame buffer support"
-        depends on FB && OMAP2_DSS && !DRM_OMAP
+        depends on FB
+        depends on DRM_OMAP = n
 
+        select FB_OMAP2_DSS
 	select OMAP2_VRFB if ARCH_OMAP2 || ARCH_OMAP3
         select FB_CFB_FILLRECT
         select FB_CFB_COPYAREA
@@ -9,6 +14,8 @@ menuconfig FB_OMAP2
         help
 	  Frame buffer driver for OMAP2+ based boards.
 
+if FB_OMAP2
+
 config FB_OMAP2_DEBUG_SUPPORT
         bool "Debug support for OMAP2+ FB"
 	default y
@@ -25,3 +32,8 @@ config FB_OMAP2_NUM_FBS
 	help
 	  Select the number of framebuffers created. OMAP2/3 has 3 overlays
 	  so normally this would be 3.
+
+source "drivers/video/fbdev/omap2/omapfb/dss/Kconfig"
+source "drivers/video/fbdev/omap2/omapfb/displays/Kconfig"
+
+endif
diff --git a/drivers/video/fbdev/omap2/omapfb/Makefile b/drivers/video/fbdev/omap2/omapfb/Makefile
index 51c2e00d9bf8..ad68ecf141af 100644
--- a/drivers/video/fbdev/omap2/omapfb/Makefile
+++ b/drivers/video/fbdev/omap2/omapfb/Makefile
@@ -1,2 +1,5 @@
+obj-$(CONFIG_OMAP2_VRFB) += vrfb.o
+obj-y += dss/
+obj-y += displays/
 obj-$(CONFIG_FB_OMAP2) += omapfb.o
 omapfb-y := omapfb-main.o omapfb-sysfs.o omapfb-ioctl.o
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/Kconfig b/drivers/video/fbdev/omap2/omapfb/displays/Kconfig
new file mode 100644
index 000000000000..08f12039dd02
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/Kconfig
@@ -0,0 +1,86 @@
+menu "OMAPFB Panel and Encoder Drivers"
+        depends on FB_OMAP2_DSS
+
+config FB_OMAP2_ENCODER_OPA362
+	tristate "OPA362 external analog amplifier"
+	help
+	  Driver for OPA362 external analog TV amplifier controlled
+	  through a GPIO.
+
+config FB_OMAP2_ENCODER_TFP410
+        tristate "TFP410 DPI to DVI Encoder"
+	help
+	  Driver for TFP410 DPI to DVI encoder.
+
+config FB_OMAP2_ENCODER_TPD12S015
+        tristate "TPD12S015 HDMI ESD protection and level shifter"
+	help
+	  Driver for TPD12S015, which offers HDMI ESD protection and level
+	  shifting.
+
+config FB_OMAP2_CONNECTOR_DVI
+        tristate "DVI Connector"
+	depends on I2C
+	help
+	  Driver for a generic DVI connector.
+
+config FB_OMAP2_CONNECTOR_HDMI
+        tristate "HDMI Connector"
+	help
+	  Driver for a generic HDMI connector.
+
+config FB_OMAP2_CONNECTOR_ANALOG_TV
+        tristate "Analog TV Connector"
+	help
+	  Driver for a generic analog TV connector.
+
+config FB_OMAP2_PANEL_DPI
+	tristate "Generic DPI panel"
+	help
+	  Driver for generic DPI panels.
+
+config FB_OMAP2_PANEL_DSI_CM
+	tristate "Generic DSI Command Mode Panel"
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Driver for generic DSI command mode panels.
+
+config FB_OMAP2_PANEL_SONY_ACX565AKM
+	tristate "ACX565AKM Panel"
+	depends on SPI && BACKLIGHT_CLASS_DEVICE
+	help
+	  This is the LCD panel used on Nokia N900
+
+config FB_OMAP2_PANEL_LGPHILIPS_LB035Q02
+	tristate "LG.Philips LB035Q02 LCD Panel"
+	depends on SPI
+	help
+	  LCD Panel used on the Gumstix Overo Palo35
+
+config FB_OMAP2_PANEL_SHARP_LS037V7DW01
+        tristate "Sharp LS037V7DW01 LCD Panel"
+        depends on BACKLIGHT_CLASS_DEVICE
+        help
+          LCD Panel used in TI's SDP3430 and EVM boards
+
+config FB_OMAP2_PANEL_TPO_TD028TTEC1
+        tristate "TPO TD028TTEC1 LCD Panel"
+        depends on SPI
+        help
+          LCD panel used in Openmoko.
+
+config FB_OMAP2_PANEL_TPO_TD043MTEA1
+        tristate "TPO TD043MTEA1 LCD Panel"
+        depends on SPI
+        help
+          LCD Panel used in OMAP3 Pandora
+
+config FB_OMAP2_PANEL_NEC_NL8048HL11
+	tristate "NEC NL8048HL11 Panel"
+	depends on SPI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+		This NEC NL8048HL11 panel is TFT LCD used in the
+		Zoom2/3/3630 sdp boards.
+
+endmenu
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/Makefile b/drivers/video/fbdev/omap2/omapfb/displays/Makefile
new file mode 100644
index 000000000000..4f7459272256
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/Makefile
@@ -0,0 +1,14 @@
+obj-$(CONFIG_FB_OMAP2_ENCODER_OPA362) += encoder-opa362.o
+obj-$(CONFIG_FB_OMAP2_ENCODER_TFP410) += encoder-tfp410.o
+obj-$(CONFIG_FB_OMAP2_ENCODER_TPD12S015) += encoder-tpd12s015.o
+obj-$(CONFIG_FB_OMAP2_CONNECTOR_DVI) += connector-dvi.o
+obj-$(CONFIG_FB_OMAP2_CONNECTOR_HDMI) += connector-hdmi.o
+obj-$(CONFIG_FB_OMAP2_CONNECTOR_ANALOG_TV) += connector-analog-tv.o
+obj-$(CONFIG_FB_OMAP2_PANEL_DPI) += panel-dpi.o
+obj-$(CONFIG_FB_OMAP2_PANEL_DSI_CM) += panel-dsi-cm.o
+obj-$(CONFIG_FB_OMAP2_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
+obj-$(CONFIG_FB_OMAP2_PANEL_LGPHILIPS_LB035Q02) += panel-lgphilips-lb035q02.o
+obj-$(CONFIG_FB_OMAP2_PANEL_SHARP_LS037V7DW01) += panel-sharp-ls037v7dw01.o
+obj-$(CONFIG_FB_OMAP2_PANEL_TPO_TD028TTEC1) += panel-tpo-td028ttec1.o
+obj-$(CONFIG_FB_OMAP2_PANEL_TPO_TD043MTEA1) += panel-tpo-td043mtea1.o
+obj-$(CONFIG_FB_OMAP2_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c
new file mode 100644
index 000000000000..8511c648a15c
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c
@@ -0,0 +1,320 @@
+/*
+ * Analog TV Connector driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct device *dev;
+
+	struct omap_video_timings timings;
+
+	enum omap_dss_venc_type connector_type;
+	bool invert_polarity;
+};
+
+static const struct omap_video_timings tvc_pal_timings = {
+	.x_res		= 720,
+	.y_res		= 574,
+	.pixelclock	= 13500000,
+	.hsw		= 64,
+	.hfp		= 12,
+	.hbp		= 68,
+	.vsw		= 5,
+	.vfp		= 5,
+	.vbp		= 41,
+
+	.interlace	= true,
+};
+
+static const struct of_device_id tvc_of_match[];
+
+struct tvc_of_data {
+	enum omap_dss_venc_type connector_type;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int tvc_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(ddata->dev, "connect\n");
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.atv->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void tvc_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(ddata->dev, "disconnect\n");
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.atv->disconnect(in, dssdev);
+}
+
+static int tvc_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(ddata->dev, "enable\n");
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	in->ops.atv->set_timings(in, &ddata->timings);
+
+	if (!ddata->dev->of_node) {
+		in->ops.atv->set_type(in, ddata->connector_type);
+
+		in->ops.atv->invert_vid_out_polarity(in,
+			ddata->invert_polarity);
+	}
+
+	r = in->ops.atv->enable(in);
+	if (r)
+		return r;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return r;
+}
+
+static void tvc_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(ddata->dev, "disable\n");
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	in->ops.atv->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void tvc_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.atv->set_timings(in, timings);
+}
+
+static void tvc_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->timings;
+}
+
+static int tvc_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.atv->check_timings(in, timings);
+}
+
+static u32 tvc_get_wss(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.atv->get_wss(in);
+}
+
+static int tvc_set_wss(struct omap_dss_device *dssdev, u32 wss)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.atv->set_wss(in, wss);
+}
+
+static struct omap_dss_driver tvc_driver = {
+	.connect		= tvc_connect,
+	.disconnect		= tvc_disconnect,
+
+	.enable			= tvc_enable,
+	.disable		= tvc_disable,
+
+	.set_timings		= tvc_set_timings,
+	.get_timings		= tvc_get_timings,
+	.check_timings		= tvc_check_timings,
+
+	.get_resolution		= omapdss_default_get_resolution,
+
+	.get_wss		= tvc_get_wss,
+	.set_wss		= tvc_set_wss,
+};
+
+static int tvc_probe_pdata(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct connector_atv_platform_data *pdata;
+	struct omap_dss_device *in, *dssdev;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "Failed to find video source\n");
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	ddata->connector_type = pdata->connector_type;
+	ddata->invert_polarity = pdata->invert_polarity;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int tvc_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int tvc_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+	ddata->dev = &pdev->dev;
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = tvc_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = tvc_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->timings = tvc_pal_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->driver = &tvc_driver;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = tvc_pal_timings;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+err_reg:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit tvc_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(&ddata->dssdev);
+
+	tvc_disable(dssdev);
+	tvc_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id tvc_of_match[] = {
+	{ .compatible = "omapdss,svideo-connector", },
+	{ .compatible = "omapdss,composite-video-connector", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, tvc_of_match);
+
+static struct platform_driver tvc_connector_driver = {
+	.probe	= tvc_probe,
+	.remove	= __exit_p(tvc_remove),
+	.driver	= {
+		.name	= "connector-analog-tv",
+		.of_match_table = tvc_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(tvc_connector_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("Analog TV Connector driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
new file mode 100644
index 000000000000..d811e6dcaef7
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
@@ -0,0 +1,398 @@
+/*
+ * Generic DVI Connector driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include <drm/drm_edid.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+static const struct omap_video_timings dvic_default_timings = {
+	.x_res		= 640,
+	.y_res		= 480,
+
+	.pixelclock	= 23500000,
+
+	.hfp		= 48,
+	.hsw		= 32,
+	.hbp		= 80,
+
+	.vfp		= 3,
+	.vsw		= 4,
+	.vbp		= 7,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+};
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct omap_video_timings timings;
+
+	struct i2c_adapter *i2c_adapter;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int dvic_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dvi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void dvic_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dvi->disconnect(in, dssdev);
+}
+
+static int dvic_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	in->ops.dvi->set_timings(in, &ddata->timings);
+
+	r = in->ops.dvi->enable(in);
+	if (r)
+		return r;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void dvic_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	in->ops.dvi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void dvic_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dvi->set_timings(in, timings);
+}
+
+static void dvic_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->timings;
+}
+
+static int dvic_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dvi->check_timings(in, timings);
+}
+
+static int dvic_ddc_read(struct i2c_adapter *adapter,
+		unsigned char *buf, u16 count, u8 offset)
+{
+	int r, retries;
+
+	for (retries = 3; retries > 0; retries--) {
+		struct i2c_msg msgs[] = {
+			{
+				.addr   = DDC_ADDR,
+				.flags  = 0,
+				.len    = 1,
+				.buf    = &offset,
+			}, {
+				.addr   = DDC_ADDR,
+				.flags  = I2C_M_RD,
+				.len    = count,
+				.buf    = buf,
+			}
+		};
+
+		r = i2c_transfer(adapter, msgs, 2);
+		if (r == 2)
+			return 0;
+
+		if (r != -EAGAIN)
+			break;
+	}
+
+	return r < 0 ? r : -EIO;
+}
+
+static int dvic_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	int r, l, bytes_read;
+
+	if (!ddata->i2c_adapter)
+		return -ENODEV;
+
+	l = min(EDID_LENGTH, len);
+	r = dvic_ddc_read(ddata->i2c_adapter, edid, l, 0);
+	if (r)
+		return r;
+
+	bytes_read = l;
+
+	/* if there are extensions, read second block */
+	if (len > EDID_LENGTH && edid[0x7e] > 0) {
+		l = min(EDID_LENGTH, len - EDID_LENGTH);
+
+		r = dvic_ddc_read(ddata->i2c_adapter, edid + EDID_LENGTH,
+				l, EDID_LENGTH);
+		if (r)
+			return r;
+
+		bytes_read += l;
+	}
+
+	return bytes_read;
+}
+
+static bool dvic_detect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	unsigned char out;
+	int r;
+
+	if (!ddata->i2c_adapter)
+		return true;
+
+	r = dvic_ddc_read(ddata->i2c_adapter, &out, 1, 0);
+
+	return r == 0;
+}
+
+static struct omap_dss_driver dvic_driver = {
+	.connect	= dvic_connect,
+	.disconnect	= dvic_disconnect,
+
+	.enable		= dvic_enable,
+	.disable	= dvic_disable,
+
+	.set_timings	= dvic_set_timings,
+	.get_timings	= dvic_get_timings,
+	.check_timings	= dvic_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+
+	.read_edid	= dvic_read_edid,
+	.detect		= dvic_detect,
+};
+
+static int dvic_probe_pdata(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct connector_dvi_platform_data *pdata;
+	struct omap_dss_device *in, *dssdev;
+	int i2c_bus_num;
+
+	pdata = dev_get_platdata(&pdev->dev);
+	i2c_bus_num = pdata->i2c_bus_num;
+
+	if (i2c_bus_num != -1) {
+		struct i2c_adapter *adapter;
+
+		adapter = i2c_get_adapter(i2c_bus_num);
+		if (!adapter) {
+			dev_err(&pdev->dev,
+					"Failed to get I2C adapter, bus %d\n",
+					i2c_bus_num);
+			return -EPROBE_DEFER;
+		}
+
+		ddata->i2c_adapter = adapter;
+	}
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		i2c_put_adapter(ddata->i2c_adapter);
+
+		dev_err(&pdev->dev, "Failed to find video source\n");
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int dvic_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+	struct device_node *adapter_node;
+	struct i2c_adapter *adapter;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
+	if (adapter_node) {
+		adapter = of_get_i2c_adapter_by_node(adapter_node);
+		if (adapter == NULL) {
+			dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
+			omap_dss_put_device(ddata->in);
+			return -EPROBE_DEFER;
+		}
+
+		ddata->i2c_adapter = adapter;
+	}
+
+	return 0;
+}
+
+static int dvic_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = dvic_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = dvic_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->timings = dvic_default_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->driver = &dvic_driver;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_DVI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = dvic_default_timings;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	omap_dss_put_device(ddata->in);
+
+	i2c_put_adapter(ddata->i2c_adapter);
+
+	return r;
+}
+
+static int __exit dvic_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(&ddata->dssdev);
+
+	dvic_disable(dssdev);
+	dvic_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	i2c_put_adapter(ddata->i2c_adapter);
+
+	return 0;
+}
+
+static const struct of_device_id dvic_of_match[] = {
+	{ .compatible = "omapdss,dvi-connector", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dvic_of_match);
+
+static struct platform_driver dvi_connector_driver = {
+	.probe	= dvic_probe,
+	.remove	= __exit_p(dvic_remove),
+	.driver	= {
+		.name	= "connector-dvi",
+		.of_match_table = dvic_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(dvi_connector_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("Generic DVI Connector driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c
new file mode 100644
index 000000000000..6ee4129bc0c0
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c
@@ -0,0 +1,348 @@
+/*
+ * HDMI Connector driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+
+#include <drm/drm_edid.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+static const struct omap_video_timings hdmic_default_timings = {
+	.x_res		= 640,
+	.y_res		= 480,
+	.pixelclock	= 25175000,
+	.hsw		= 96,
+	.hfp		= 16,
+	.hbp		= 48,
+	.vsw		= 2,
+	.vfp		= 11,
+	.vbp		= 31,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+
+	.interlace	= false,
+};
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct device *dev;
+
+	struct omap_video_timings timings;
+
+	int hpd_gpio;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int hdmic_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(ddata->dev, "connect\n");
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.hdmi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void hdmic_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(ddata->dev, "disconnect\n");
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.hdmi->disconnect(in, dssdev);
+}
+
+static int hdmic_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(ddata->dev, "enable\n");
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	in->ops.hdmi->set_timings(in, &ddata->timings);
+
+	r = in->ops.hdmi->enable(in);
+	if (r)
+		return r;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return r;
+}
+
+static void hdmic_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(ddata->dev, "disable\n");
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	in->ops.hdmi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void hdmic_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.hdmi->set_timings(in, timings);
+}
+
+static void hdmic_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->timings;
+}
+
+static int hdmic_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->check_timings(in, timings);
+}
+
+static int hdmic_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->read_edid(in, edid, len);
+}
+
+static bool hdmic_detect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (gpio_is_valid(ddata->hpd_gpio))
+		return gpio_get_value_cansleep(ddata->hpd_gpio);
+	else
+		return in->ops.hdmi->detect(in);
+}
+
+static int hdmic_set_hdmi_mode(struct omap_dss_device *dssdev, bool hdmi_mode)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->set_hdmi_mode(in, hdmi_mode);
+}
+
+static int hdmic_set_infoframe(struct omap_dss_device *dssdev,
+		const struct hdmi_avi_infoframe *avi)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->set_infoframe(in, avi);
+}
+
+static struct omap_dss_driver hdmic_driver = {
+	.connect		= hdmic_connect,
+	.disconnect		= hdmic_disconnect,
+
+	.enable			= hdmic_enable,
+	.disable		= hdmic_disable,
+
+	.set_timings		= hdmic_set_timings,
+	.get_timings		= hdmic_get_timings,
+	.check_timings		= hdmic_check_timings,
+
+	.get_resolution		= omapdss_default_get_resolution,
+
+	.read_edid		= hdmic_read_edid,
+	.detect			= hdmic_detect,
+	.set_hdmi_mode		= hdmic_set_hdmi_mode,
+	.set_hdmi_infoframe	= hdmic_set_infoframe,
+};
+
+static int hdmic_probe_pdata(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct connector_hdmi_platform_data *pdata;
+	struct omap_dss_device *in, *dssdev;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	ddata->hpd_gpio = -ENODEV;
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "Failed to find video source\n");
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int hdmic_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+	int gpio;
+
+	/* HPD GPIO */
+	gpio = of_get_named_gpio(node, "hpd-gpios", 0);
+	if (gpio_is_valid(gpio))
+		ddata->hpd_gpio = gpio;
+	else
+		ddata->hpd_gpio = -ENODEV;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int hdmic_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+	ddata->dev = &pdev->dev;
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = hdmic_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = hdmic_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->hpd_gpio)) {
+		r = devm_gpio_request_one(&pdev->dev, ddata->hpd_gpio,
+				GPIOF_DIR_IN, "hdmi_hpd");
+		if (r)
+			goto err_reg;
+	}
+
+	ddata->timings = hdmic_default_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->driver = &hdmic_driver;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = hdmic_default_timings;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+err_reg:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit hdmic_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(&ddata->dssdev);
+
+	hdmic_disable(dssdev);
+	hdmic_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id hdmic_of_match[] = {
+	{ .compatible = "omapdss,hdmi-connector", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, hdmic_of_match);
+
+static struct platform_driver hdmi_connector_driver = {
+	.probe	= hdmic_probe,
+	.remove	= __exit_p(hdmic_remove),
+	.driver	= {
+		.name	= "connector-hdmi",
+		.of_match_table = hdmic_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(hdmi_connector_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("HDMI Connector driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c
new file mode 100644
index 000000000000..8c246c213e06
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c
@@ -0,0 +1,278 @@
+/*
+ * OPA362 analog video amplifier with output/power control
+ *
+ * Copyright (C) 2014 Golden Delicious Computers
+ * Author: H. Nikolaus Schaller <hns@goldelico.com>
+ *
+ * based on encoder-tfp410
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct gpio_desc *enable_gpio;
+
+	struct omap_video_timings timings;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int opa362_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(dssdev->dev, "connect\n");
+
+	if (omapdss_device_is_connected(dssdev))
+		return -EBUSY;
+
+	r = in->ops.atv->connect(in, dssdev);
+	if (r)
+		return r;
+
+	dst->src = dssdev;
+	dssdev->dst = dst;
+
+	return 0;
+}
+
+static void opa362_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(dssdev->dev, "disconnect\n");
+
+	WARN_ON(!omapdss_device_is_connected(dssdev));
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	WARN_ON(dst != dssdev->dst);
+	if (dst != dssdev->dst)
+		return;
+
+	dst->src = NULL;
+	dssdev->dst = NULL;
+
+	in->ops.atv->disconnect(in, &ddata->dssdev);
+}
+
+static int opa362_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(dssdev->dev, "enable\n");
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	in->ops.atv->set_timings(in, &ddata->timings);
+
+	r = in->ops.atv->enable(in);
+	if (r)
+		return r;
+
+	if (ddata->enable_gpio)
+		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void opa362_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(dssdev->dev, "disable\n");
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (ddata->enable_gpio)
+		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
+
+	in->ops.atv->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void opa362_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(dssdev->dev, "set_timings\n");
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.atv->set_timings(in, timings);
+}
+
+static void opa362_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	dev_dbg(dssdev->dev, "get_timings\n");
+
+	*timings = ddata->timings;
+}
+
+static int opa362_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(dssdev->dev, "check_timings\n");
+
+	return in->ops.atv->check_timings(in, timings);
+}
+
+static void opa362_set_type(struct omap_dss_device *dssdev,
+		enum omap_dss_venc_type type)
+{
+	/* we can only drive a COMPOSITE output */
+	WARN_ON(type != OMAP_DSS_VENC_TYPE_COMPOSITE);
+
+}
+
+static const struct omapdss_atv_ops opa362_atv_ops = {
+	.connect	= opa362_connect,
+	.disconnect	= opa362_disconnect,
+
+	.enable		= opa362_enable,
+	.disable	= opa362_disable,
+
+	.check_timings	= opa362_check_timings,
+	.set_timings	= opa362_set_timings,
+	.get_timings	= opa362_get_timings,
+
+	.set_type	= opa362_set_type,
+};
+
+static int opa362_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev, *in;
+	struct gpio_desc *gpio;
+	int r;
+
+	dev_dbg(&pdev->dev, "probe\n");
+
+	if (node == NULL) {
+		dev_err(&pdev->dev, "Unable to find device tree\n");
+		return -EINVAL;
+	}
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ddata->enable_gpio = gpio;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	dssdev = &ddata->dssdev;
+	dssdev->ops.atv = &opa362_atv_ops;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
+	dssdev->output_type = OMAP_DISPLAY_TYPE_VENC;
+	dssdev->owner = THIS_MODULE;
+
+	r = omapdss_register_output(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register output\n");
+		goto err_reg;
+	}
+
+	return 0;
+err_reg:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit opa362_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_output(&ddata->dssdev);
+
+	WARN_ON(omapdss_device_is_enabled(dssdev));
+	if (omapdss_device_is_enabled(dssdev))
+		opa362_disable(dssdev);
+
+	WARN_ON(omapdss_device_is_connected(dssdev));
+	if (omapdss_device_is_connected(dssdev))
+		opa362_disconnect(dssdev, dssdev->dst);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id opa362_of_match[] = {
+	{ .compatible = "omapdss,ti,opa362", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, opa362_of_match);
+
+static struct platform_driver opa362_driver = {
+	.probe	= opa362_probe,
+	.remove	= __exit_p(opa362_remove),
+	.driver	= {
+		.name	= "amplifier-opa362",
+		.of_match_table = opa362_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(opa362_driver);
+
+MODULE_AUTHOR("H. Nikolaus Schaller <hns@goldelico.com>");
+MODULE_DESCRIPTION("OPA362 analog video amplifier with output/power control");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c
new file mode 100644
index 000000000000..d9048b3df495
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c
@@ -0,0 +1,320 @@
+/*
+ * TFP410 DPI-to-DVI encoder driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	int pd_gpio;
+	int data_lines;
+
+	struct omap_video_timings timings;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int tfp410_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return -EBUSY;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	dst->src = dssdev;
+	dssdev->dst = dst;
+
+	return 0;
+}
+
+static void tfp410_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	WARN_ON(!omapdss_device_is_connected(dssdev));
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	WARN_ON(dst != dssdev->dst);
+	if (dst != dssdev->dst)
+		return;
+
+	dst->src = NULL;
+	dssdev->dst = NULL;
+
+	in->ops.dpi->disconnect(in, &ddata->dssdev);
+}
+
+static int tfp410_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	in->ops.dpi->set_timings(in, &ddata->timings);
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	if (gpio_is_valid(ddata->pd_gpio))
+		gpio_set_value_cansleep(ddata->pd_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void tfp410_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (gpio_is_valid(ddata->pd_gpio))
+		gpio_set_value_cansleep(ddata->pd_gpio, 0);
+
+	in->ops.dpi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void tfp410_fix_timings(struct omap_video_timings *timings)
+{
+	timings->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	timings->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	timings->de_level = OMAPDSS_SIG_ACTIVE_HIGH;
+}
+
+static void tfp410_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	tfp410_fix_timings(timings);
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void tfp410_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->timings;
+}
+
+static int tfp410_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	tfp410_fix_timings(timings);
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static const struct omapdss_dvi_ops tfp410_dvi_ops = {
+	.connect	= tfp410_connect,
+	.disconnect	= tfp410_disconnect,
+
+	.enable		= tfp410_enable,
+	.disable	= tfp410_disable,
+
+	.check_timings	= tfp410_check_timings,
+	.set_timings	= tfp410_set_timings,
+	.get_timings	= tfp410_get_timings,
+};
+
+static int tfp410_probe_pdata(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct encoder_tfp410_platform_data *pdata;
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	ddata->pd_gpio = pdata->power_down_gpio;
+
+	ddata->data_lines = pdata->data_lines;
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "Failed to find video source\n");
+		return -ENODEV;
+	}
+
+	ddata->in = in;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int tfp410_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+	int gpio;
+
+	gpio = of_get_named_gpio(node, "powerdown-gpios", 0);
+
+	if (gpio_is_valid(gpio) || gpio == -ENOENT) {
+		ddata->pd_gpio = gpio;
+	} else {
+		dev_err(&pdev->dev, "failed to parse PD gpio\n");
+		return gpio;
+	}
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int tfp410_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = tfp410_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = tfp410_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->pd_gpio)) {
+		r = devm_gpio_request_one(&pdev->dev, ddata->pd_gpio,
+				GPIOF_OUT_INIT_LOW, "tfp410 PD");
+		if (r) {
+			dev_err(&pdev->dev, "Failed to request PD GPIO %d\n",
+					ddata->pd_gpio);
+			goto err_gpio;
+		}
+	}
+
+	dssdev = &ddata->dssdev;
+	dssdev->ops.dvi = &tfp410_dvi_ops;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->output_type = OMAP_DISPLAY_TYPE_DVI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->phy.dpi.data_lines = ddata->data_lines;
+	dssdev->port_num = 1;
+
+	r = omapdss_register_output(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register output\n");
+		goto err_reg;
+	}
+
+	return 0;
+err_reg:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit tfp410_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_output(&ddata->dssdev);
+
+	WARN_ON(omapdss_device_is_enabled(dssdev));
+	if (omapdss_device_is_enabled(dssdev))
+		tfp410_disable(dssdev);
+
+	WARN_ON(omapdss_device_is_connected(dssdev));
+	if (omapdss_device_is_connected(dssdev))
+		tfp410_disconnect(dssdev, dssdev->dst);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id tfp410_of_match[] = {
+	{ .compatible = "omapdss,ti,tfp410", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, tfp410_of_match);
+
+static struct platform_driver tfp410_driver = {
+	.probe	= tfp410_probe,
+	.remove	= __exit_p(tfp410_remove),
+	.driver	= {
+		.name	= "tfp410",
+		.of_match_table = tfp410_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(tfp410_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("TFP410 DPI to DVI encoder driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c
new file mode 100644
index 000000000000..677e2545fcbe
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c
@@ -0,0 +1,328 @@
+/*
+ * TPD12S015 HDMI ESD protection & level shifter chip driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/gpio/consumer.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct gpio_desc *ct_cp_hpd_gpio;
+	struct gpio_desc *ls_oe_gpio;
+	struct gpio_desc *hpd_gpio;
+
+	struct omap_video_timings timings;
+};
+
+#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
+
+static int tpd_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	r = in->ops.hdmi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	dst->src = dssdev;
+	dssdev->dst = dst;
+
+	if (ddata->ct_cp_hpd_gpio) {
+		gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 1);
+		/* DC-DC converter needs at max 300us to get to 90% of 5V */
+		udelay(300);
+	}
+
+	return 0;
+}
+
+static void tpd_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	gpiod_set_value_cansleep(ddata->ct_cp_hpd_gpio, 0);
+
+	dst->src = NULL;
+	dssdev->dst = NULL;
+
+	in->ops.hdmi->disconnect(in, &ddata->dssdev);
+}
+
+static int tpd_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
+		return 0;
+
+	in->ops.hdmi->set_timings(in, &ddata->timings);
+
+	r = in->ops.hdmi->enable(in);
+	if (r)
+		return r;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return r;
+}
+
+static void tpd_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (dssdev->state != OMAP_DSS_DISPLAY_ACTIVE)
+		return;
+
+	in->ops.hdmi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void tpd_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->timings = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.hdmi->set_timings(in, timings);
+}
+
+static void tpd_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->timings;
+}
+
+static int tpd_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	r = in->ops.hdmi->check_timings(in, timings);
+
+	return r;
+}
+
+static int tpd_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!gpiod_get_value_cansleep(ddata->hpd_gpio))
+		return -ENODEV;
+
+	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 1);
+
+	r = in->ops.hdmi->read_edid(in, edid, len);
+
+	gpiod_set_value_cansleep(ddata->ls_oe_gpio, 0);
+
+	return r;
+}
+
+static bool tpd_detect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	return gpiod_get_value_cansleep(ddata->hpd_gpio);
+}
+
+static int tpd_set_infoframe(struct omap_dss_device *dssdev,
+		const struct hdmi_avi_infoframe *avi)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->set_infoframe(in, avi);
+}
+
+static int tpd_set_hdmi_mode(struct omap_dss_device *dssdev,
+		bool hdmi_mode)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.hdmi->set_hdmi_mode(in, hdmi_mode);
+}
+
+static const struct omapdss_hdmi_ops tpd_hdmi_ops = {
+	.connect		= tpd_connect,
+	.disconnect		= tpd_disconnect,
+
+	.enable			= tpd_enable,
+	.disable		= tpd_disable,
+
+	.check_timings		= tpd_check_timings,
+	.set_timings		= tpd_set_timings,
+	.get_timings		= tpd_get_timings,
+
+	.read_edid		= tpd_read_edid,
+	.detect			= tpd_detect,
+	.set_infoframe		= tpd_set_infoframe,
+	.set_hdmi_mode		= tpd_set_hdmi_mode,
+};
+
+static int tpd_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int tpd_probe(struct platform_device *pdev)
+{
+	struct omap_dss_device *in, *dssdev;
+	struct panel_drv_data *ddata;
+	int r;
+	struct gpio_desc *gpio;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	if (pdev->dev.of_node) {
+		r = tpd_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+
+	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 0,
+		GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		goto err_gpio;
+
+	ddata->ct_cp_hpd_gpio = gpio;
+
+	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 1,
+		GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		goto err_gpio;
+
+	ddata->ls_oe_gpio = gpio;
+
+	gpio = devm_gpiod_get_index(&pdev->dev, NULL, 2,
+		GPIOD_IN);
+	if (IS_ERR(gpio))
+		goto err_gpio;
+
+	ddata->hpd_gpio = gpio;
+
+	dssdev = &ddata->dssdev;
+	dssdev->ops.hdmi = &tpd_hdmi_ops;
+	dssdev->dev = &pdev->dev;
+	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->output_type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->port_num = 1;
+
+	in = ddata->in;
+
+	r = omapdss_register_output(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register output\n");
+		goto err_reg;
+	}
+
+	return 0;
+err_reg:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit tpd_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_output(&ddata->dssdev);
+
+	WARN_ON(omapdss_device_is_enabled(dssdev));
+	if (omapdss_device_is_enabled(dssdev))
+		tpd_disable(dssdev);
+
+	WARN_ON(omapdss_device_is_connected(dssdev));
+	if (omapdss_device_is_connected(dssdev))
+		tpd_disconnect(dssdev, dssdev->dst);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id tpd_of_match[] = {
+	{ .compatible = "omapdss,ti,tpd12s015", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, tpd_of_match);
+
+static struct platform_driver tpd_driver = {
+	.probe	= tpd_probe,
+	.remove	= __exit_p(tpd_remove),
+	.driver	= {
+		.name	= "tpd12s015",
+		.of_match_table = tpd_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(tpd_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("TPD12S015 driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c
new file mode 100644
index 000000000000..e780fd4f8b46
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c
@@ -0,0 +1,328 @@
+/*
+ * Generic MIPI DPI Panel Driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+#include <video/of_display_timing.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	int data_lines;
+
+	struct omap_video_timings videomode;
+
+	/* used for non-DT boot, to be removed */
+	int backlight_gpio;
+
+	struct gpio_desc *enable_gpio;
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int panel_dpi_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void panel_dpi_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int panel_dpi_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	gpiod_set_value_cansleep(ddata->enable_gpio, 1);
+
+	if (gpio_is_valid(ddata->backlight_gpio))
+		gpio_set_value_cansleep(ddata->backlight_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void panel_dpi_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (gpio_is_valid(ddata->backlight_gpio))
+		gpio_set_value_cansleep(ddata->backlight_gpio, 0);
+
+	gpiod_set_value_cansleep(ddata->enable_gpio, 0);
+
+	in->ops.dpi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void panel_dpi_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void panel_dpi_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int panel_dpi_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver panel_dpi_ops = {
+	.connect	= panel_dpi_connect,
+	.disconnect	= panel_dpi_disconnect,
+
+	.enable		= panel_dpi_enable,
+	.disable	= panel_dpi_disable,
+
+	.set_timings	= panel_dpi_set_timings,
+	.get_timings	= panel_dpi_get_timings,
+	.check_timings	= panel_dpi_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+static int panel_dpi_probe_pdata(struct platform_device *pdev)
+{
+	const struct panel_dpi_platform_data *pdata;
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev, *in;
+	struct videomode vm;
+	int r;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	videomode_from_timing(pdata->display_timing, &vm);
+	videomode_to_omap_video_timings(&vm, &ddata->videomode);
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	r = devm_gpio_request_one(&pdev->dev, pdata->enable_gpio,
+					GPIOF_OUT_INIT_LOW, "panel enable");
+	if (r)
+		goto err_gpio;
+
+	ddata->enable_gpio = gpio_to_desc(pdata->enable_gpio);
+
+	ddata->backlight_gpio = pdata->backlight_gpio;
+
+	return 0;
+
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int panel_dpi_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+	int r;
+	struct display_timing timing;
+	struct videomode vm;
+	struct gpio_desc *gpio;
+
+	gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+
+	ddata->enable_gpio = gpio;
+
+	ddata->backlight_gpio = -ENOENT;
+
+	r = of_get_display_timing(node, "panel-timing", &timing);
+	if (r) {
+		dev_err(&pdev->dev, "failed to get video timing\n");
+		return r;
+	}
+
+	videomode_from_timing(&timing, &vm);
+	videomode_to_omap_video_timings(&vm, &ddata->videomode);
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int panel_dpi_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = panel_dpi_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = panel_dpi_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->backlight_gpio)) {
+		r = devm_gpio_request_one(&pdev->dev, ddata->backlight_gpio,
+				GPIOF_OUT_INIT_LOW, "panel backlight");
+		if (r)
+			goto err_gpio;
+	}
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &pdev->dev;
+	dssdev->driver = &panel_dpi_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+	dssdev->phy.dpi.data_lines = ddata->data_lines;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit panel_dpi_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(dssdev);
+
+	panel_dpi_disable(dssdev);
+	panel_dpi_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id panel_dpi_of_match[] = {
+	{ .compatible = "omapdss,panel-dpi", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, panel_dpi_of_match);
+
+static struct platform_driver panel_dpi_driver = {
+	.probe = panel_dpi_probe,
+	.remove = __exit_p(panel_dpi_remove),
+	.driver = {
+		.name = "panel-dpi",
+		.of_match_table = panel_dpi_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(panel_dpi_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("Generic MIPI DPI Panel Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
new file mode 100644
index 000000000000..3414c2609320
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
@@ -0,0 +1,1388 @@
+/*
+ * Generic DSI Command Mode panel driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+/* #define DEBUG */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/fb.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+#include <video/mipi_display.h>
+
+/* DSI Virtual channel. Hardcoded for now. */
+#define TCH 0
+
+#define DCS_READ_NUM_ERRORS	0x05
+#define DCS_BRIGHTNESS		0x51
+#define DCS_CTRL_DISPLAY	0x53
+#define DCS_GET_ID1		0xda
+#define DCS_GET_ID2		0xdb
+#define DCS_GET_ID3		0xdc
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct omap_video_timings timings;
+
+	struct platform_device *pdev;
+
+	struct mutex lock;
+
+	struct backlight_device *bldev;
+
+	unsigned long	hw_guard_end;	/* next value of jiffies when we can
+					 * issue the next sleep in/out command
+					 */
+	unsigned long	hw_guard_wait;	/* max guard time in jiffies */
+
+	/* panel HW configuration from DT or platform data */
+	int reset_gpio;
+	int ext_te_gpio;
+
+	bool use_dsi_backlight;
+
+	struct omap_dsi_pin_config pin_config;
+
+	/* runtime variables */
+	bool enabled;
+
+	bool te_enabled;
+
+	atomic_t do_update;
+	int channel;
+
+	struct delayed_work te_timeout_work;
+
+	bool intro_printed;
+
+	struct workqueue_struct *workqueue;
+
+	bool ulps_enabled;
+	unsigned ulps_timeout;
+	struct delayed_work ulps_work;
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static irqreturn_t dsicm_te_isr(int irq, void *data);
+static void dsicm_te_timeout_work_callback(struct work_struct *work);
+static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable);
+
+static int dsicm_panel_reset(struct panel_drv_data *ddata);
+
+static void dsicm_ulps_work(struct work_struct *work);
+
+static void hw_guard_start(struct panel_drv_data *ddata, int guard_msec)
+{
+	ddata->hw_guard_wait = msecs_to_jiffies(guard_msec);
+	ddata->hw_guard_end = jiffies + ddata->hw_guard_wait;
+}
+
+static void hw_guard_wait(struct panel_drv_data *ddata)
+{
+	unsigned long wait = ddata->hw_guard_end - jiffies;
+
+	if ((long)wait > 0 && wait <= ddata->hw_guard_wait) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_timeout(wait);
+	}
+}
+
+static int dsicm_dcs_read_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 *data)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+	u8 buf[1];
+
+	r = in->ops.dsi->dcs_read(in, ddata->channel, dcs_cmd, buf, 1);
+
+	if (r < 0)
+		return r;
+
+	*data = buf[0];
+
+	return 0;
+}
+
+static int dsicm_dcs_write_0(struct panel_drv_data *ddata, u8 dcs_cmd)
+{
+	struct omap_dss_device *in = ddata->in;
+	return in->ops.dsi->dcs_write(in, ddata->channel, &dcs_cmd, 1);
+}
+
+static int dsicm_dcs_write_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 param)
+{
+	struct omap_dss_device *in = ddata->in;
+	u8 buf[2] = { dcs_cmd, param };
+
+	return in->ops.dsi->dcs_write(in, ddata->channel, buf, 2);
+}
+
+static int dsicm_sleep_in(struct panel_drv_data *ddata)
+
+{
+	struct omap_dss_device *in = ddata->in;
+	u8 cmd;
+	int r;
+
+	hw_guard_wait(ddata);
+
+	cmd = MIPI_DCS_ENTER_SLEEP_MODE;
+	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, &cmd, 1);
+	if (r)
+		return r;
+
+	hw_guard_start(ddata, 120);
+
+	usleep_range(5000, 10000);
+
+	return 0;
+}
+
+static int dsicm_sleep_out(struct panel_drv_data *ddata)
+{
+	int r;
+
+	hw_guard_wait(ddata);
+
+	r = dsicm_dcs_write_0(ddata, MIPI_DCS_EXIT_SLEEP_MODE);
+	if (r)
+		return r;
+
+	hw_guard_start(ddata, 120);
+
+	usleep_range(5000, 10000);
+
+	return 0;
+}
+
+static int dsicm_get_id(struct panel_drv_data *ddata, u8 *id1, u8 *id2, u8 *id3)
+{
+	int r;
+
+	r = dsicm_dcs_read_1(ddata, DCS_GET_ID1, id1);
+	if (r)
+		return r;
+	r = dsicm_dcs_read_1(ddata, DCS_GET_ID2, id2);
+	if (r)
+		return r;
+	r = dsicm_dcs_read_1(ddata, DCS_GET_ID3, id3);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static int dsicm_set_update_window(struct panel_drv_data *ddata,
+		u16 x, u16 y, u16 w, u16 h)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+	u16 x1 = x;
+	u16 x2 = x + w - 1;
+	u16 y1 = y;
+	u16 y2 = y + h - 1;
+
+	u8 buf[5];
+	buf[0] = MIPI_DCS_SET_COLUMN_ADDRESS;
+	buf[1] = (x1 >> 8) & 0xff;
+	buf[2] = (x1 >> 0) & 0xff;
+	buf[3] = (x2 >> 8) & 0xff;
+	buf[4] = (x2 >> 0) & 0xff;
+
+	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, buf, sizeof(buf));
+	if (r)
+		return r;
+
+	buf[0] = MIPI_DCS_SET_PAGE_ADDRESS;
+	buf[1] = (y1 >> 8) & 0xff;
+	buf[2] = (y1 >> 0) & 0xff;
+	buf[3] = (y2 >> 8) & 0xff;
+	buf[4] = (y2 >> 0) & 0xff;
+
+	r = in->ops.dsi->dcs_write_nosync(in, ddata->channel, buf, sizeof(buf));
+	if (r)
+		return r;
+
+	in->ops.dsi->bta_sync(in, ddata->channel);
+
+	return r;
+}
+
+static void dsicm_queue_ulps_work(struct panel_drv_data *ddata)
+{
+	if (ddata->ulps_timeout > 0)
+		queue_delayed_work(ddata->workqueue, &ddata->ulps_work,
+				msecs_to_jiffies(ddata->ulps_timeout));
+}
+
+static void dsicm_cancel_ulps_work(struct panel_drv_data *ddata)
+{
+	cancel_delayed_work(&ddata->ulps_work);
+}
+
+static int dsicm_enter_ulps(struct panel_drv_data *ddata)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (ddata->ulps_enabled)
+		return 0;
+
+	dsicm_cancel_ulps_work(ddata);
+
+	r = _dsicm_enable_te(ddata, false);
+	if (r)
+		goto err;
+
+	if (gpio_is_valid(ddata->ext_te_gpio))
+		disable_irq(gpio_to_irq(ddata->ext_te_gpio));
+
+	in->ops.dsi->disable(in, false, true);
+
+	ddata->ulps_enabled = true;
+
+	return 0;
+
+err:
+	dev_err(&ddata->pdev->dev, "enter ULPS failed");
+	dsicm_panel_reset(ddata);
+
+	ddata->ulps_enabled = false;
+
+	dsicm_queue_ulps_work(ddata);
+
+	return r;
+}
+
+static int dsicm_exit_ulps(struct panel_drv_data *ddata)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!ddata->ulps_enabled)
+		return 0;
+
+	r = in->ops.dsi->enable(in);
+	if (r) {
+		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
+		goto err1;
+	}
+
+	in->ops.dsi->enable_hs(in, ddata->channel, true);
+
+	r = _dsicm_enable_te(ddata, true);
+	if (r) {
+		dev_err(&ddata->pdev->dev, "failed to re-enable TE");
+		goto err2;
+	}
+
+	if (gpio_is_valid(ddata->ext_te_gpio))
+		enable_irq(gpio_to_irq(ddata->ext_te_gpio));
+
+	dsicm_queue_ulps_work(ddata);
+
+	ddata->ulps_enabled = false;
+
+	return 0;
+
+err2:
+	dev_err(&ddata->pdev->dev, "failed to exit ULPS");
+
+	r = dsicm_panel_reset(ddata);
+	if (!r) {
+		if (gpio_is_valid(ddata->ext_te_gpio))
+			enable_irq(gpio_to_irq(ddata->ext_te_gpio));
+		ddata->ulps_enabled = false;
+	}
+err1:
+	dsicm_queue_ulps_work(ddata);
+
+	return r;
+}
+
+static int dsicm_wake_up(struct panel_drv_data *ddata)
+{
+	if (ddata->ulps_enabled)
+		return dsicm_exit_ulps(ddata);
+
+	dsicm_cancel_ulps_work(ddata);
+	dsicm_queue_ulps_work(ddata);
+	return 0;
+}
+
+static int dsicm_bl_update_status(struct backlight_device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+	int level;
+
+	if (dev->props.fb_blank == FB_BLANK_UNBLANK &&
+			dev->props.power == FB_BLANK_UNBLANK)
+		level = dev->props.brightness;
+	else
+		level = 0;
+
+	dev_dbg(&ddata->pdev->dev, "update brightness to %d\n", level);
+
+	mutex_lock(&ddata->lock);
+
+	if (ddata->enabled) {
+		in->ops.dsi->bus_lock(in);
+
+		r = dsicm_wake_up(ddata);
+		if (!r)
+			r = dsicm_dcs_write_1(ddata, DCS_BRIGHTNESS, level);
+
+		in->ops.dsi->bus_unlock(in);
+	} else {
+		r = 0;
+	}
+
+	mutex_unlock(&ddata->lock);
+
+	return r;
+}
+
+static int dsicm_bl_get_intensity(struct backlight_device *dev)
+{
+	if (dev->props.fb_blank == FB_BLANK_UNBLANK &&
+			dev->props.power == FB_BLANK_UNBLANK)
+		return dev->props.brightness;
+
+	return 0;
+}
+
+static const struct backlight_ops dsicm_bl_ops = {
+	.get_brightness = dsicm_bl_get_intensity,
+	.update_status  = dsicm_bl_update_status,
+};
+
+static void dsicm_get_resolution(struct omap_dss_device *dssdev,
+		u16 *xres, u16 *yres)
+{
+	*xres = dssdev->panel.timings.x_res;
+	*yres = dssdev->panel.timings.y_res;
+}
+
+static ssize_t dsicm_num_errors_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *in = ddata->in;
+	u8 errors = 0;
+	int r;
+
+	mutex_lock(&ddata->lock);
+
+	if (ddata->enabled) {
+		in->ops.dsi->bus_lock(in);
+
+		r = dsicm_wake_up(ddata);
+		if (!r)
+			r = dsicm_dcs_read_1(ddata, DCS_READ_NUM_ERRORS,
+					&errors);
+
+		in->ops.dsi->bus_unlock(in);
+	} else {
+		r = -ENODEV;
+	}
+
+	mutex_unlock(&ddata->lock);
+
+	if (r)
+		return r;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", errors);
+}
+
+static ssize_t dsicm_hw_revision_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *in = ddata->in;
+	u8 id1, id2, id3;
+	int r;
+
+	mutex_lock(&ddata->lock);
+
+	if (ddata->enabled) {
+		in->ops.dsi->bus_lock(in);
+
+		r = dsicm_wake_up(ddata);
+		if (!r)
+			r = dsicm_get_id(ddata, &id1, &id2, &id3);
+
+		in->ops.dsi->bus_unlock(in);
+	} else {
+		r = -ENODEV;
+	}
+
+	mutex_unlock(&ddata->lock);
+
+	if (r)
+		return r;
+
+	return snprintf(buf, PAGE_SIZE, "%02x.%02x.%02x\n", id1, id2, id3);
+}
+
+static ssize_t dsicm_store_ulps(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *in = ddata->in;
+	unsigned long t;
+	int r;
+
+	r = kstrtoul(buf, 0, &t);
+	if (r)
+		return r;
+
+	mutex_lock(&ddata->lock);
+
+	if (ddata->enabled) {
+		in->ops.dsi->bus_lock(in);
+
+		if (t)
+			r = dsicm_enter_ulps(ddata);
+		else
+			r = dsicm_wake_up(ddata);
+
+		in->ops.dsi->bus_unlock(in);
+	}
+
+	mutex_unlock(&ddata->lock);
+
+	if (r)
+		return r;
+
+	return count;
+}
+
+static ssize_t dsicm_show_ulps(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	unsigned t;
+
+	mutex_lock(&ddata->lock);
+	t = ddata->ulps_enabled;
+	mutex_unlock(&ddata->lock);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", t);
+}
+
+static ssize_t dsicm_store_ulps_timeout(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *in = ddata->in;
+	unsigned long t;
+	int r;
+
+	r = kstrtoul(buf, 0, &t);
+	if (r)
+		return r;
+
+	mutex_lock(&ddata->lock);
+	ddata->ulps_timeout = t;
+
+	if (ddata->enabled) {
+		/* dsicm_wake_up will restart the timer */
+		in->ops.dsi->bus_lock(in);
+		r = dsicm_wake_up(ddata);
+		in->ops.dsi->bus_unlock(in);
+	}
+
+	mutex_unlock(&ddata->lock);
+
+	if (r)
+		return r;
+
+	return count;
+}
+
+static ssize_t dsicm_show_ulps_timeout(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	unsigned t;
+
+	mutex_lock(&ddata->lock);
+	t = ddata->ulps_timeout;
+	mutex_unlock(&ddata->lock);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", t);
+}
+
+static DEVICE_ATTR(num_dsi_errors, S_IRUGO, dsicm_num_errors_show, NULL);
+static DEVICE_ATTR(hw_revision, S_IRUGO, dsicm_hw_revision_show, NULL);
+static DEVICE_ATTR(ulps, S_IRUGO | S_IWUSR,
+		dsicm_show_ulps, dsicm_store_ulps);
+static DEVICE_ATTR(ulps_timeout, S_IRUGO | S_IWUSR,
+		dsicm_show_ulps_timeout, dsicm_store_ulps_timeout);
+
+static struct attribute *dsicm_attrs[] = {
+	&dev_attr_num_dsi_errors.attr,
+	&dev_attr_hw_revision.attr,
+	&dev_attr_ulps.attr,
+	&dev_attr_ulps_timeout.attr,
+	NULL,
+};
+
+static struct attribute_group dsicm_attr_group = {
+	.attrs = dsicm_attrs,
+};
+
+static void dsicm_hw_reset(struct panel_drv_data *ddata)
+{
+	if (!gpio_is_valid(ddata->reset_gpio))
+		return;
+
+	gpio_set_value(ddata->reset_gpio, 1);
+	udelay(10);
+	/* reset the panel */
+	gpio_set_value(ddata->reset_gpio, 0);
+	/* assert reset */
+	udelay(10);
+	gpio_set_value(ddata->reset_gpio, 1);
+	/* wait after releasing reset */
+	usleep_range(5000, 10000);
+}
+
+static int dsicm_power_on(struct panel_drv_data *ddata)
+{
+	struct omap_dss_device *in = ddata->in;
+	u8 id1, id2, id3;
+	int r;
+	struct omap_dss_dsi_config dsi_config = {
+		.mode = OMAP_DSS_DSI_CMD_MODE,
+		.pixel_format = OMAP_DSS_DSI_FMT_RGB888,
+		.timings = &ddata->timings,
+		.hs_clk_min = 150000000,
+		.hs_clk_max = 300000000,
+		.lp_clk_min = 7000000,
+		.lp_clk_max = 10000000,
+	};
+
+	if (ddata->pin_config.num_pins > 0) {
+		r = in->ops.dsi->configure_pins(in, &ddata->pin_config);
+		if (r) {
+			dev_err(&ddata->pdev->dev,
+				"failed to configure DSI pins\n");
+			goto err0;
+		}
+	}
+
+	r = in->ops.dsi->set_config(in, &dsi_config);
+	if (r) {
+		dev_err(&ddata->pdev->dev, "failed to configure DSI\n");
+		goto err0;
+	}
+
+	r = in->ops.dsi->enable(in);
+	if (r) {
+		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
+		goto err0;
+	}
+
+	dsicm_hw_reset(ddata);
+
+	in->ops.dsi->enable_hs(in, ddata->channel, false);
+
+	r = dsicm_sleep_out(ddata);
+	if (r)
+		goto err;
+
+	r = dsicm_get_id(ddata, &id1, &id2, &id3);
+	if (r)
+		goto err;
+
+	r = dsicm_dcs_write_1(ddata, DCS_BRIGHTNESS, 0xff);
+	if (r)
+		goto err;
+
+	r = dsicm_dcs_write_1(ddata, DCS_CTRL_DISPLAY,
+			(1<<2) | (1<<5));	/* BL | BCTRL */
+	if (r)
+		goto err;
+
+	r = dsicm_dcs_write_1(ddata, MIPI_DCS_SET_PIXEL_FORMAT,
+		MIPI_DCS_PIXEL_FMT_24BIT);
+	if (r)
+		goto err;
+
+	r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_DISPLAY_ON);
+	if (r)
+		goto err;
+
+	r = _dsicm_enable_te(ddata, ddata->te_enabled);
+	if (r)
+		goto err;
+
+	r = in->ops.dsi->enable_video_output(in, ddata->channel);
+	if (r)
+		goto err;
+
+	ddata->enabled = 1;
+
+	if (!ddata->intro_printed) {
+		dev_info(&ddata->pdev->dev, "panel revision %02x.%02x.%02x\n",
+			id1, id2, id3);
+		ddata->intro_printed = true;
+	}
+
+	in->ops.dsi->enable_hs(in, ddata->channel, true);
+
+	return 0;
+err:
+	dev_err(&ddata->pdev->dev, "error while enabling panel, issuing HW reset\n");
+
+	dsicm_hw_reset(ddata);
+
+	in->ops.dsi->disable(in, true, false);
+err0:
+	return r;
+}
+
+static void dsicm_power_off(struct panel_drv_data *ddata)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	in->ops.dsi->disable_video_output(in, ddata->channel);
+
+	r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_DISPLAY_OFF);
+	if (!r)
+		r = dsicm_sleep_in(ddata);
+
+	if (r) {
+		dev_err(&ddata->pdev->dev,
+				"error disabling panel, issuing HW reset\n");
+		dsicm_hw_reset(ddata);
+	}
+
+	in->ops.dsi->disable(in, true, false);
+
+	ddata->enabled = 0;
+}
+
+static int dsicm_panel_reset(struct panel_drv_data *ddata)
+{
+	dev_err(&ddata->pdev->dev, "performing LCD reset\n");
+
+	dsicm_power_off(ddata);
+	dsicm_hw_reset(ddata);
+	return dsicm_power_on(ddata);
+}
+
+static int dsicm_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	struct device *dev = &ddata->pdev->dev;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dsi->connect(in, dssdev);
+	if (r) {
+		dev_err(dev, "Failed to connect to video source\n");
+		return r;
+	}
+
+	r = in->ops.dsi->request_vc(ddata->in, &ddata->channel);
+	if (r) {
+		dev_err(dev, "failed to get virtual channel\n");
+		goto err_req_vc;
+	}
+
+	r = in->ops.dsi->set_vc_id(ddata->in, ddata->channel, TCH);
+	if (r) {
+		dev_err(dev, "failed to set VC_ID\n");
+		goto err_vc_id;
+	}
+
+	return 0;
+
+err_vc_id:
+	in->ops.dsi->release_vc(ddata->in, ddata->channel);
+err_req_vc:
+	in->ops.dsi->disconnect(in, dssdev);
+	return r;
+}
+
+static void dsicm_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dsi->release_vc(in, ddata->channel);
+	in->ops.dsi->disconnect(in, dssdev);
+}
+
+static int dsicm_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(&ddata->pdev->dev, "enable\n");
+
+	mutex_lock(&ddata->lock);
+
+	if (!omapdss_device_is_connected(dssdev)) {
+		r = -ENODEV;
+		goto err;
+	}
+
+	if (omapdss_device_is_enabled(dssdev)) {
+		r = 0;
+		goto err;
+	}
+
+	in->ops.dsi->bus_lock(in);
+
+	r = dsicm_power_on(ddata);
+
+	in->ops.dsi->bus_unlock(in);
+
+	if (r)
+		goto err;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	mutex_unlock(&ddata->lock);
+
+	return 0;
+err:
+	dev_dbg(&ddata->pdev->dev, "enable failed\n");
+	mutex_unlock(&ddata->lock);
+	return r;
+}
+
+static void dsicm_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(&ddata->pdev->dev, "disable\n");
+
+	mutex_lock(&ddata->lock);
+
+	dsicm_cancel_ulps_work(ddata);
+
+	in->ops.dsi->bus_lock(in);
+
+	if (omapdss_device_is_enabled(dssdev)) {
+		r = dsicm_wake_up(ddata);
+		if (!r)
+			dsicm_power_off(ddata);
+	}
+
+	in->ops.dsi->bus_unlock(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+
+	mutex_unlock(&ddata->lock);
+}
+
+static void dsicm_framedone_cb(int err, void *data)
+{
+	struct panel_drv_data *ddata = data;
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->pdev->dev, "framedone, err %d\n", err);
+	in->ops.dsi->bus_unlock(ddata->in);
+}
+
+static irqreturn_t dsicm_te_isr(int irq, void *data)
+{
+	struct panel_drv_data *ddata = data;
+	struct omap_dss_device *in = ddata->in;
+	int old;
+	int r;
+
+	old = atomic_cmpxchg(&ddata->do_update, 1, 0);
+
+	if (old) {
+		cancel_delayed_work(&ddata->te_timeout_work);
+
+		r = in->ops.dsi->update(in, ddata->channel, dsicm_framedone_cb,
+				ddata);
+		if (r)
+			goto err;
+	}
+
+	return IRQ_HANDLED;
+err:
+	dev_err(&ddata->pdev->dev, "start update failed\n");
+	in->ops.dsi->bus_unlock(in);
+	return IRQ_HANDLED;
+}
+
+static void dsicm_te_timeout_work_callback(struct work_struct *work)
+{
+	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
+					te_timeout_work.work);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_err(&ddata->pdev->dev, "TE not received for 250ms!\n");
+
+	atomic_set(&ddata->do_update, 0);
+	in->ops.dsi->bus_unlock(in);
+}
+
+static int dsicm_update(struct omap_dss_device *dssdev,
+				    u16 x, u16 y, u16 w, u16 h)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(&ddata->pdev->dev, "update %d, %d, %d x %d\n", x, y, w, h);
+
+	mutex_lock(&ddata->lock);
+	in->ops.dsi->bus_lock(in);
+
+	r = dsicm_wake_up(ddata);
+	if (r)
+		goto err;
+
+	if (!ddata->enabled) {
+		r = 0;
+		goto err;
+	}
+
+	/* XXX no need to send this every frame, but dsi break if not done */
+	r = dsicm_set_update_window(ddata, 0, 0,
+			dssdev->panel.timings.x_res,
+			dssdev->panel.timings.y_res);
+	if (r)
+		goto err;
+
+	if (ddata->te_enabled && gpio_is_valid(ddata->ext_te_gpio)) {
+		schedule_delayed_work(&ddata->te_timeout_work,
+				msecs_to_jiffies(250));
+		atomic_set(&ddata->do_update, 1);
+	} else {
+		r = in->ops.dsi->update(in, ddata->channel, dsicm_framedone_cb,
+				ddata);
+		if (r)
+			goto err;
+	}
+
+	/* note: no bus_unlock here. unlock is in framedone_cb */
+	mutex_unlock(&ddata->lock);
+	return 0;
+err:
+	in->ops.dsi->bus_unlock(in);
+	mutex_unlock(&ddata->lock);
+	return r;
+}
+
+static int dsicm_sync(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->pdev->dev, "sync\n");
+
+	mutex_lock(&ddata->lock);
+	in->ops.dsi->bus_lock(in);
+	in->ops.dsi->bus_unlock(in);
+	mutex_unlock(&ddata->lock);
+
+	dev_dbg(&ddata->pdev->dev, "sync done\n");
+
+	return 0;
+}
+
+static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable)
+{
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (enable)
+		r = dsicm_dcs_write_1(ddata, MIPI_DCS_SET_TEAR_ON, 0);
+	else
+		r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_TEAR_OFF);
+
+	if (!gpio_is_valid(ddata->ext_te_gpio))
+		in->ops.dsi->enable_te(in, enable);
+
+	/* possible panel bug */
+	msleep(100);
+
+	return r;
+}
+
+static int dsicm_enable_te(struct omap_dss_device *dssdev, bool enable)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	mutex_lock(&ddata->lock);
+
+	if (ddata->te_enabled == enable)
+		goto end;
+
+	in->ops.dsi->bus_lock(in);
+
+	if (ddata->enabled) {
+		r = dsicm_wake_up(ddata);
+		if (r)
+			goto err;
+
+		r = _dsicm_enable_te(ddata, enable);
+		if (r)
+			goto err;
+	}
+
+	ddata->te_enabled = enable;
+
+	in->ops.dsi->bus_unlock(in);
+end:
+	mutex_unlock(&ddata->lock);
+
+	return 0;
+err:
+	in->ops.dsi->bus_unlock(in);
+	mutex_unlock(&ddata->lock);
+
+	return r;
+}
+
+static int dsicm_get_te(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	int r;
+
+	mutex_lock(&ddata->lock);
+	r = ddata->te_enabled;
+	mutex_unlock(&ddata->lock);
+
+	return r;
+}
+
+static int dsicm_memory_read(struct omap_dss_device *dssdev,
+		void *buf, size_t size,
+		u16 x, u16 y, u16 w, u16 h)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+	int first = 1;
+	int plen;
+	unsigned buf_used = 0;
+
+	if (size < w * h * 3)
+		return -ENOMEM;
+
+	mutex_lock(&ddata->lock);
+
+	if (!ddata->enabled) {
+		r = -ENODEV;
+		goto err1;
+	}
+
+	size = min(w * h * 3,
+			dssdev->panel.timings.x_res *
+			dssdev->panel.timings.y_res * 3);
+
+	in->ops.dsi->bus_lock(in);
+
+	r = dsicm_wake_up(ddata);
+	if (r)
+		goto err2;
+
+	/* plen 1 or 2 goes into short packet. until checksum error is fixed,
+	 * use short packets. plen 32 works, but bigger packets seem to cause
+	 * an error. */
+	if (size % 2)
+		plen = 1;
+	else
+		plen = 2;
+
+	dsicm_set_update_window(ddata, x, y, w, h);
+
+	r = in->ops.dsi->set_max_rx_packet_size(in, ddata->channel, plen);
+	if (r)
+		goto err2;
+
+	while (buf_used < size) {
+		u8 dcs_cmd = first ? 0x2e : 0x3e;
+		first = 0;
+
+		r = in->ops.dsi->dcs_read(in, ddata->channel, dcs_cmd,
+				buf + buf_used, size - buf_used);
+
+		if (r < 0) {
+			dev_err(dssdev->dev, "read error\n");
+			goto err3;
+		}
+
+		buf_used += r;
+
+		if (r < plen) {
+			dev_err(&ddata->pdev->dev, "short read\n");
+			break;
+		}
+
+		if (signal_pending(current)) {
+			dev_err(&ddata->pdev->dev, "signal pending, "
+					"aborting memory read\n");
+			r = -ERESTARTSYS;
+			goto err3;
+		}
+	}
+
+	r = buf_used;
+
+err3:
+	in->ops.dsi->set_max_rx_packet_size(in, ddata->channel, 1);
+err2:
+	in->ops.dsi->bus_unlock(in);
+err1:
+	mutex_unlock(&ddata->lock);
+	return r;
+}
+
+static void dsicm_ulps_work(struct work_struct *work)
+{
+	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
+			ulps_work.work);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	mutex_lock(&ddata->lock);
+
+	if (dssdev->state != OMAP_DSS_DISPLAY_ACTIVE || !ddata->enabled) {
+		mutex_unlock(&ddata->lock);
+		return;
+	}
+
+	in->ops.dsi->bus_lock(in);
+
+	dsicm_enter_ulps(ddata);
+
+	in->ops.dsi->bus_unlock(in);
+	mutex_unlock(&ddata->lock);
+}
+
+static struct omap_dss_driver dsicm_ops = {
+	.connect	= dsicm_connect,
+	.disconnect	= dsicm_disconnect,
+
+	.enable		= dsicm_enable,
+	.disable	= dsicm_disable,
+
+	.update		= dsicm_update,
+	.sync		= dsicm_sync,
+
+	.get_resolution	= dsicm_get_resolution,
+	.get_recommended_bpp = omapdss_default_get_recommended_bpp,
+
+	.enable_te	= dsicm_enable_te,
+	.get_te		= dsicm_get_te,
+
+	.memory_read	= dsicm_memory_read,
+};
+
+static int dsicm_probe_pdata(struct platform_device *pdev)
+{
+	const struct panel_dsicm_platform_data *pdata;
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return -EPROBE_DEFER;
+	}
+	ddata->in = in;
+
+	ddata->reset_gpio = pdata->reset_gpio;
+
+	if (pdata->use_ext_te)
+		ddata->ext_te_gpio = pdata->ext_te_gpio;
+	else
+		ddata->ext_te_gpio = -1;
+
+	ddata->ulps_timeout = pdata->ulps_timeout;
+
+	ddata->use_dsi_backlight = pdata->use_dsi_backlight;
+
+	ddata->pin_config = pdata->pin_config;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int dsicm_probe_of(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *in;
+	int gpio;
+
+	gpio = of_get_named_gpio(node, "reset-gpios", 0);
+	if (!gpio_is_valid(gpio)) {
+		dev_err(&pdev->dev, "failed to parse reset gpio\n");
+		return gpio;
+	}
+	ddata->reset_gpio = gpio;
+
+	gpio = of_get_named_gpio(node, "te-gpios", 0);
+	if (gpio_is_valid(gpio) || gpio == -ENOENT) {
+		ddata->ext_te_gpio = gpio;
+	} else {
+		dev_err(&pdev->dev, "failed to parse TE gpio\n");
+		return gpio;
+	}
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	/* TODO: ulps, backlight */
+
+	return 0;
+}
+
+static int dsicm_probe(struct platform_device *pdev)
+{
+	struct backlight_properties props;
+	struct panel_drv_data *ddata;
+	struct backlight_device *bldev = NULL;
+	struct device *dev = &pdev->dev;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	dev_dbg(dev, "probe\n");
+
+	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+	ddata->pdev = pdev;
+
+	if (dev_get_platdata(dev)) {
+		r = dsicm_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = dsicm_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->timings.x_res = 864;
+	ddata->timings.y_res = 480;
+	ddata->timings.pixelclock = 864 * 480 * 60;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = dev;
+	dssdev->driver = &dsicm_ops;
+	dssdev->panel.timings = ddata->timings;
+	dssdev->type = OMAP_DISPLAY_TYPE_DSI;
+	dssdev->owner = THIS_MODULE;
+
+	dssdev->panel.dsi_pix_fmt = OMAP_DSS_DSI_FMT_RGB888;
+	dssdev->caps = OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE |
+		OMAP_DSS_DISPLAY_CAP_TEAR_ELIM;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	mutex_init(&ddata->lock);
+
+	atomic_set(&ddata->do_update, 0);
+
+	if (gpio_is_valid(ddata->reset_gpio)) {
+		r = devm_gpio_request_one(dev, ddata->reset_gpio,
+				GPIOF_OUT_INIT_LOW, "taal rst");
+		if (r) {
+			dev_err(dev, "failed to request reset gpio\n");
+			return r;
+		}
+	}
+
+	if (gpio_is_valid(ddata->ext_te_gpio)) {
+		r = devm_gpio_request_one(dev, ddata->ext_te_gpio,
+				GPIOF_IN, "taal irq");
+		if (r) {
+			dev_err(dev, "GPIO request failed\n");
+			return r;
+		}
+
+		r = devm_request_irq(dev, gpio_to_irq(ddata->ext_te_gpio),
+				dsicm_te_isr,
+				IRQF_TRIGGER_RISING,
+				"taal vsync", ddata);
+
+		if (r) {
+			dev_err(dev, "IRQ request failed\n");
+			return r;
+		}
+
+		INIT_DEFERRABLE_WORK(&ddata->te_timeout_work,
+					dsicm_te_timeout_work_callback);
+
+		dev_dbg(dev, "Using GPIO TE\n");
+	}
+
+	ddata->workqueue = create_singlethread_workqueue("dsicm_wq");
+	if (ddata->workqueue == NULL) {
+		dev_err(dev, "can't create workqueue\n");
+		return -ENOMEM;
+	}
+	INIT_DELAYED_WORK(&ddata->ulps_work, dsicm_ulps_work);
+
+	dsicm_hw_reset(ddata);
+
+	if (ddata->use_dsi_backlight) {
+		memset(&props, 0, sizeof(struct backlight_properties));
+		props.max_brightness = 255;
+
+		props.type = BACKLIGHT_RAW;
+		bldev = backlight_device_register(dev_name(dev),
+				dev, ddata, &dsicm_bl_ops, &props);
+		if (IS_ERR(bldev)) {
+			r = PTR_ERR(bldev);
+			goto err_bl;
+		}
+
+		ddata->bldev = bldev;
+
+		bldev->props.fb_blank = FB_BLANK_UNBLANK;
+		bldev->props.power = FB_BLANK_UNBLANK;
+		bldev->props.brightness = 255;
+
+		dsicm_bl_update_status(bldev);
+	}
+
+	r = sysfs_create_group(&dev->kobj, &dsicm_attr_group);
+	if (r) {
+		dev_err(dev, "failed to create sysfs files\n");
+		goto err_sysfs_create;
+	}
+
+	return 0;
+
+err_sysfs_create:
+	if (bldev != NULL)
+		backlight_device_unregister(bldev);
+err_bl:
+	destroy_workqueue(ddata->workqueue);
+err_reg:
+	return r;
+}
+
+static int __exit dsicm_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct backlight_device *bldev;
+
+	dev_dbg(&pdev->dev, "remove\n");
+
+	omapdss_unregister_display(dssdev);
+
+	dsicm_disable(dssdev);
+	dsicm_disconnect(dssdev);
+
+	sysfs_remove_group(&pdev->dev.kobj, &dsicm_attr_group);
+
+	bldev = ddata->bldev;
+	if (bldev != NULL) {
+		bldev->props.power = FB_BLANK_POWERDOWN;
+		dsicm_bl_update_status(bldev);
+		backlight_device_unregister(bldev);
+	}
+
+	omap_dss_put_device(ddata->in);
+
+	dsicm_cancel_ulps_work(ddata);
+	destroy_workqueue(ddata->workqueue);
+
+	/* reset, to be sure that the panel is in a valid state */
+	dsicm_hw_reset(ddata);
+
+	return 0;
+}
+
+static const struct of_device_id dsicm_of_match[] = {
+	{ .compatible = "omapdss,panel-dsi-cm", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dsicm_of_match);
+
+static struct platform_driver dsicm_driver = {
+	.probe = dsicm_probe,
+	.remove = __exit_p(dsicm_remove),
+	.driver = {
+		.name = "panel-dsi-cm",
+		.of_match_table = dsicm_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(dsicm_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("Generic DSI Command Mode Panel Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
new file mode 100644
index 000000000000..18eb60e9c9ec
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
@@ -0,0 +1,404 @@
+/*
+ * LG.Philips LB035Q02 LCD Panel driver
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ * Based on a driver by: Steve Sakoman <steve@sakoman.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/spi/spi.h>
+#include <linux/mutex.h>
+#include <linux/gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+static struct omap_video_timings lb035q02_timings = {
+	.x_res = 320,
+	.y_res = 240,
+
+	.pixelclock	= 6500000,
+
+	.hsw		= 2,
+	.hfp		= 20,
+	.hbp		= 68,
+
+	.vsw		= 2,
+	.vfp		= 4,
+	.vbp		= 18,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+};
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	struct spi_device *spi;
+
+	int data_lines;
+
+	struct omap_video_timings videomode;
+
+	/* used for non-DT boot, to be removed */
+	int backlight_gpio;
+
+	struct gpio_desc *enable_gpio;
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int lb035q02_write_reg(struct spi_device *spi, u8 reg, u16 val)
+{
+	struct spi_message msg;
+	struct spi_transfer index_xfer = {
+		.len		= 3,
+		.cs_change	= 1,
+	};
+	struct spi_transfer value_xfer = {
+		.len		= 3,
+	};
+	u8	buffer[16];
+
+	spi_message_init(&msg);
+
+	/* register index */
+	buffer[0] = 0x70;
+	buffer[1] = 0x00;
+	buffer[2] = reg & 0x7f;
+	index_xfer.tx_buf = buffer;
+	spi_message_add_tail(&index_xfer, &msg);
+
+	/* register value */
+	buffer[4] = 0x72;
+	buffer[5] = val >> 8;
+	buffer[6] = val;
+	value_xfer.tx_buf = buffer + 4;
+	spi_message_add_tail(&value_xfer, &msg);
+
+	return spi_sync(spi, &msg);
+}
+
+static void init_lb035q02_panel(struct spi_device *spi)
+{
+	/* Init sequence from page 28 of the lb035q02 spec */
+	lb035q02_write_reg(spi, 0x01, 0x6300);
+	lb035q02_write_reg(spi, 0x02, 0x0200);
+	lb035q02_write_reg(spi, 0x03, 0x0177);
+	lb035q02_write_reg(spi, 0x04, 0x04c7);
+	lb035q02_write_reg(spi, 0x05, 0xffc0);
+	lb035q02_write_reg(spi, 0x06, 0xe806);
+	lb035q02_write_reg(spi, 0x0a, 0x4008);
+	lb035q02_write_reg(spi, 0x0b, 0x0000);
+	lb035q02_write_reg(spi, 0x0d, 0x0030);
+	lb035q02_write_reg(spi, 0x0e, 0x2800);
+	lb035q02_write_reg(spi, 0x0f, 0x0000);
+	lb035q02_write_reg(spi, 0x16, 0x9f80);
+	lb035q02_write_reg(spi, 0x17, 0x0a0f);
+	lb035q02_write_reg(spi, 0x1e, 0x00c1);
+	lb035q02_write_reg(spi, 0x30, 0x0300);
+	lb035q02_write_reg(spi, 0x31, 0x0007);
+	lb035q02_write_reg(spi, 0x32, 0x0000);
+	lb035q02_write_reg(spi, 0x33, 0x0000);
+	lb035q02_write_reg(spi, 0x34, 0x0707);
+	lb035q02_write_reg(spi, 0x35, 0x0004);
+	lb035q02_write_reg(spi, 0x36, 0x0302);
+	lb035q02_write_reg(spi, 0x37, 0x0202);
+	lb035q02_write_reg(spi, 0x3a, 0x0a0d);
+	lb035q02_write_reg(spi, 0x3b, 0x0806);
+}
+
+static int lb035q02_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	init_lb035q02_panel(ddata->spi);
+
+	return 0;
+}
+
+static void lb035q02_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int lb035q02_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	if (ddata->enable_gpio)
+		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
+
+	if (gpio_is_valid(ddata->backlight_gpio))
+		gpio_set_value_cansleep(ddata->backlight_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void lb035q02_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (ddata->enable_gpio)
+		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
+
+	if (gpio_is_valid(ddata->backlight_gpio))
+		gpio_set_value_cansleep(ddata->backlight_gpio, 0);
+
+	in->ops.dpi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void lb035q02_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void lb035q02_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int lb035q02_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver lb035q02_ops = {
+	.connect	= lb035q02_connect,
+	.disconnect	= lb035q02_disconnect,
+
+	.enable		= lb035q02_enable,
+	.disable	= lb035q02_disable,
+
+	.set_timings	= lb035q02_set_timings,
+	.get_timings	= lb035q02_get_timings,
+	.check_timings	= lb035q02_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+static int lb035q02_probe_pdata(struct spi_device *spi)
+{
+	const struct panel_lb035q02_platform_data *pdata;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev, *in;
+	int r;
+
+	pdata = dev_get_platdata(&spi->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&spi->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	r = devm_gpio_request_one(&spi->dev, pdata->enable_gpio,
+					GPIOF_OUT_INIT_LOW, "panel enable");
+	if (r)
+		goto err_gpio;
+
+	ddata->enable_gpio = gpio_to_desc(pdata->enable_gpio);
+
+	ddata->backlight_gpio = pdata->backlight_gpio;
+
+	return 0;
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int lb035q02_probe_of(struct spi_device *spi)
+{
+	struct device_node *node = spi->dev.of_node;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *in;
+	struct gpio_desc *gpio;
+
+	gpio = devm_gpiod_get(&spi->dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio)) {
+		dev_err(&spi->dev, "failed to parse enable gpio\n");
+		return PTR_ERR(gpio);
+	}
+
+	ddata->enable_gpio = gpio;
+
+	ddata->backlight_gpio = -ENOENT;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&spi->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int lb035q02_panel_spi_probe(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	dev_set_drvdata(&spi->dev, ddata);
+
+	ddata->spi = spi;
+
+	if (dev_get_platdata(&spi->dev)) {
+		r = lb035q02_probe_pdata(spi);
+		if (r)
+			return r;
+	} else if (spi->dev.of_node) {
+		r = lb035q02_probe_of(spi);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->backlight_gpio)) {
+		r = devm_gpio_request_one(&spi->dev, ddata->backlight_gpio,
+				GPIOF_OUT_INIT_LOW, "panel backlight");
+		if (r)
+			goto err_gpio;
+	}
+
+	ddata->videomode = lb035q02_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &spi->dev;
+	dssdev->driver = &lb035q02_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+	dssdev->phy.dpi.data_lines = ddata->data_lines;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&spi->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int lb035q02_panel_spi_remove(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(dssdev);
+
+	lb035q02_disable(dssdev);
+	lb035q02_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id lb035q02_of_match[] = {
+	{ .compatible = "omapdss,lgphilips,lb035q02", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, lb035q02_of_match);
+
+static struct spi_driver lb035q02_spi_driver = {
+	.probe		= lb035q02_panel_spi_probe,
+	.remove		= lb035q02_panel_spi_remove,
+	.driver		= {
+		.name	= "panel_lgphilips_lb035q02",
+		.of_match_table = lb035q02_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_spi_driver(lb035q02_spi_driver);
+
+MODULE_ALIAS("spi:lgphilips,lb035q02");
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("LG.Philips LB035Q02 LCD Panel driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c
new file mode 100644
index 000000000000..8a928c9a2fc9
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c
@@ -0,0 +1,437 @@
+/*
+ * NEC NL8048HL11 Panel driver
+ *
+ * Copyright (C) 2010 Texas Instruments Inc.
+ * Author: Erik Gilling <konkers@android.com>
+ * Converted to new DSS device model: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/spi/spi.h>
+#include <linux/fb.h>
+#include <linux/gpio.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device	dssdev;
+	struct omap_dss_device *in;
+
+	struct omap_video_timings videomode;
+
+	int data_lines;
+
+	int res_gpio;
+	int qvga_gpio;
+
+	struct spi_device *spi;
+};
+
+#define LCD_XRES		800
+#define LCD_YRES		480
+/*
+ * NEC PIX Clock Ratings
+ * MIN:21.8MHz TYP:23.8MHz MAX:25.7MHz
+ */
+#define LCD_PIXEL_CLOCK		23800000
+
+static const struct {
+	unsigned char addr;
+	unsigned char dat;
+} nec_8048_init_seq[] = {
+	{ 3, 0x01 }, { 0, 0x00 }, { 1, 0x01 }, { 4, 0x00 }, { 5, 0x14 },
+	{ 6, 0x24 }, { 16, 0xD7 }, { 17, 0x00 }, { 18, 0x00 }, { 19, 0x55 },
+	{ 20, 0x01 }, { 21, 0x70 }, { 22, 0x1E }, { 23, 0x25 },	{ 24, 0x25 },
+	{ 25, 0x02 }, { 26, 0x02 }, { 27, 0xA0 }, { 32, 0x2F }, { 33, 0x0F },
+	{ 34, 0x0F }, { 35, 0x0F }, { 36, 0x0F }, { 37, 0x0F },	{ 38, 0x0F },
+	{ 39, 0x00 }, { 40, 0x02 }, { 41, 0x02 }, { 42, 0x02 },	{ 43, 0x0F },
+	{ 44, 0x0F }, { 45, 0x0F }, { 46, 0x0F }, { 47, 0x0F },	{ 48, 0x0F },
+	{ 49, 0x0F }, { 50, 0x00 }, { 51, 0x02 }, { 52, 0x02 }, { 53, 0x02 },
+	{ 80, 0x0C }, { 83, 0x42 }, { 84, 0x42 }, { 85, 0x41 },	{ 86, 0x14 },
+	{ 89, 0x88 }, { 90, 0x01 }, { 91, 0x00 }, { 92, 0x02 },	{ 93, 0x0C },
+	{ 94, 0x1C }, { 95, 0x27 }, { 98, 0x49 }, { 99, 0x27 }, { 102, 0x76 },
+	{ 103, 0x27 }, { 112, 0x01 }, { 113, 0x0E }, { 114, 0x02 },
+	{ 115, 0x0C }, { 118, 0x0C }, { 121, 0x30 }, { 130, 0x00 },
+	{ 131, 0x00 }, { 132, 0xFC }, { 134, 0x00 }, { 136, 0x00 },
+	{ 138, 0x00 }, { 139, 0x00 }, { 140, 0x00 }, { 141, 0xFC },
+	{ 143, 0x00 }, { 145, 0x00 }, { 147, 0x00 }, { 148, 0x00 },
+	{ 149, 0x00 }, { 150, 0xFC }, { 152, 0x00 }, { 154, 0x00 },
+	{ 156, 0x00 }, { 157, 0x00 }, { 2, 0x00 },
+};
+
+static const struct omap_video_timings nec_8048_panel_timings = {
+	.x_res		= LCD_XRES,
+	.y_res		= LCD_YRES,
+	.pixelclock	= LCD_PIXEL_CLOCK,
+	.hfp		= 6,
+	.hsw		= 1,
+	.hbp		= 4,
+	.vfp		= 3,
+	.vsw		= 1,
+	.vbp		= 4,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int nec_8048_spi_send(struct spi_device *spi, unsigned char reg_addr,
+			unsigned char reg_data)
+{
+	int ret = 0;
+	unsigned int cmd = 0, data = 0;
+
+	cmd = 0x0000 | reg_addr; /* register address write */
+	data = 0x0100 | reg_data; /* register data write */
+	data = (cmd << 16) | data;
+
+	ret = spi_write(spi, (unsigned char *)&data, 4);
+	if (ret)
+		pr_err("error in spi_write %x\n", data);
+
+	return ret;
+}
+
+static int init_nec_8048_wvga_lcd(struct spi_device *spi)
+{
+	unsigned int i;
+	/* Initialization Sequence */
+	/* nec_8048_spi_send(spi, REG, VAL) */
+	for (i = 0; i < (ARRAY_SIZE(nec_8048_init_seq) - 1); i++)
+		nec_8048_spi_send(spi, nec_8048_init_seq[i].addr,
+				nec_8048_init_seq[i].dat);
+	udelay(20);
+	nec_8048_spi_send(spi, nec_8048_init_seq[i].addr,
+				nec_8048_init_seq[i].dat);
+	return 0;
+}
+
+static int nec_8048_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void nec_8048_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int nec_8048_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	if (gpio_is_valid(ddata->res_gpio))
+		gpio_set_value_cansleep(ddata->res_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void nec_8048_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (gpio_is_valid(ddata->res_gpio))
+		gpio_set_value_cansleep(ddata->res_gpio, 0);
+
+	in->ops.dpi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void nec_8048_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void nec_8048_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int nec_8048_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver nec_8048_ops = {
+	.connect	= nec_8048_connect,
+	.disconnect	= nec_8048_disconnect,
+
+	.enable		= nec_8048_enable,
+	.disable	= nec_8048_disable,
+
+	.set_timings	= nec_8048_set_timings,
+	.get_timings	= nec_8048_get_timings,
+	.check_timings	= nec_8048_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+
+static int nec_8048_probe_pdata(struct spi_device *spi)
+{
+	const struct panel_nec_nl8048hl11_platform_data *pdata;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&spi->dev);
+
+	ddata->qvga_gpio = pdata->qvga_gpio;
+	ddata->res_gpio = pdata->res_gpio;
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&spi->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int nec_8048_probe_of(struct spi_device *spi)
+{
+	struct device_node *node = spi->dev.of_node;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *in;
+	int gpio;
+
+	gpio = of_get_named_gpio(node, "reset-gpios", 0);
+	if (!gpio_is_valid(gpio)) {
+		dev_err(&spi->dev, "failed to parse enable gpio\n");
+		return gpio;
+	}
+	ddata->res_gpio = gpio;
+
+	/* XXX the panel spec doesn't mention any QVGA pin?? */
+	ddata->qvga_gpio = -ENOENT;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&spi->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int nec_8048_probe(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	dev_dbg(&spi->dev, "%s\n", __func__);
+
+	spi->mode = SPI_MODE_0;
+	spi->bits_per_word = 32;
+
+	r = spi_setup(spi);
+	if (r < 0) {
+		dev_err(&spi->dev, "spi_setup failed: %d\n", r);
+		return r;
+	}
+
+	init_nec_8048_wvga_lcd(spi);
+
+	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	dev_set_drvdata(&spi->dev, ddata);
+
+	ddata->spi = spi;
+
+	if (dev_get_platdata(&spi->dev)) {
+		r = nec_8048_probe_pdata(spi);
+		if (r)
+			return r;
+	} else if (spi->dev.of_node) {
+		r = nec_8048_probe_of(spi);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->qvga_gpio)) {
+		r = devm_gpio_request_one(&spi->dev, ddata->qvga_gpio,
+				GPIOF_OUT_INIT_HIGH, "lcd QVGA");
+		if (r)
+			goto err_gpio;
+	}
+
+	if (gpio_is_valid(ddata->res_gpio)) {
+		r = devm_gpio_request_one(&spi->dev, ddata->res_gpio,
+				GPIOF_OUT_INIT_LOW, "lcd RES");
+		if (r)
+			goto err_gpio;
+	}
+
+	ddata->videomode = nec_8048_panel_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &spi->dev;
+	dssdev->driver = &nec_8048_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&spi->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int nec_8048_remove(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
+
+	omapdss_unregister_display(dssdev);
+
+	nec_8048_disable(dssdev);
+	nec_8048_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int nec_8048_suspend(struct device *dev)
+{
+	struct spi_device *spi = to_spi_device(dev);
+
+	nec_8048_spi_send(spi, 2, 0x01);
+	mdelay(40);
+
+	return 0;
+}
+
+static int nec_8048_resume(struct device *dev)
+{
+	struct spi_device *spi = to_spi_device(dev);
+
+	/* reinitialize the panel */
+	spi_setup(spi);
+	nec_8048_spi_send(spi, 2, 0x00);
+	init_nec_8048_wvga_lcd(spi);
+
+	return 0;
+}
+static SIMPLE_DEV_PM_OPS(nec_8048_pm_ops, nec_8048_suspend,
+		nec_8048_resume);
+#define NEC_8048_PM_OPS (&nec_8048_pm_ops)
+#else
+#define NEC_8048_PM_OPS NULL
+#endif
+
+static const struct of_device_id nec_8048_of_match[] = {
+	{ .compatible = "omapdss,nec,nl8048hl11", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, nec_8048_of_match);
+
+static struct spi_driver nec_8048_driver = {
+	.driver = {
+		.name	= "panel-nec-nl8048hl11",
+		.pm	= NEC_8048_PM_OPS,
+		.of_match_table = nec_8048_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe	= nec_8048_probe,
+	.remove	= nec_8048_remove,
+};
+
+module_spi_driver(nec_8048_driver);
+
+MODULE_ALIAS("spi:nec,nl8048hl11");
+MODULE_AUTHOR("Erik Gilling <konkers@android.com>");
+MODULE_DESCRIPTION("NEC-NL8048HL11 Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c
new file mode 100644
index 000000000000..abfd1f6e3327
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c
@@ -0,0 +1,415 @@
+/*
+ * LCD panel driver for Sharp LS037V7DW01
+ *
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/regulator/consumer.h>
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+	struct regulator *vcc;
+
+	int data_lines;
+
+	struct omap_video_timings videomode;
+
+	struct gpio_desc *resb_gpio;	/* low = reset active min 20 us */
+	struct gpio_desc *ini_gpio;	/* high = power on */
+	struct gpio_desc *mo_gpio;	/* low = 480x640, high = 240x320 */
+	struct gpio_desc *lr_gpio;	/* high = conventional horizontal scanning */
+	struct gpio_desc *ud_gpio;	/* high = conventional vertical scanning */
+};
+
+static const struct omap_video_timings sharp_ls_timings = {
+	.x_res = 480,
+	.y_res = 640,
+
+	.pixelclock	= 19200000,
+
+	.hsw		= 2,
+	.hfp		= 1,
+	.hbp		= 28,
+
+	.vsw		= 1,
+	.vfp		= 1,
+	.vbp		= 1,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int sharp_ls_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void sharp_ls_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int sharp_ls_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	if (ddata->vcc) {
+		r = regulator_enable(ddata->vcc);
+		if (r != 0)
+			return r;
+	}
+
+	r = in->ops.dpi->enable(in);
+	if (r) {
+		regulator_disable(ddata->vcc);
+		return r;
+	}
+
+	/* wait couple of vsyncs until enabling the LCD */
+	msleep(50);
+
+	if (ddata->resb_gpio)
+		gpiod_set_value_cansleep(ddata->resb_gpio, 1);
+
+	if (ddata->ini_gpio)
+		gpiod_set_value_cansleep(ddata->ini_gpio, 1);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void sharp_ls_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	if (ddata->ini_gpio)
+		gpiod_set_value_cansleep(ddata->ini_gpio, 0);
+
+	if (ddata->resb_gpio)
+		gpiod_set_value_cansleep(ddata->resb_gpio, 0);
+
+	/* wait at least 5 vsyncs after disabling the LCD */
+
+	msleep(100);
+
+	in->ops.dpi->disable(in);
+
+	if (ddata->vcc)
+		regulator_disable(ddata->vcc);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void sharp_ls_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void sharp_ls_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int sharp_ls_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver sharp_ls_ops = {
+	.connect	= sharp_ls_connect,
+	.disconnect	= sharp_ls_disconnect,
+
+	.enable		= sharp_ls_enable,
+	.disable	= sharp_ls_disable,
+
+	.set_timings	= sharp_ls_set_timings,
+	.get_timings	= sharp_ls_get_timings,
+	.check_timings	= sharp_ls_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+static int sharp_ls_get_gpio(struct device *dev, int gpio, unsigned long flags,
+		  char *desc, struct gpio_desc **gpiod)
+{
+	struct gpio_desc *gd;
+	int r;
+
+	*gpiod = NULL;
+
+	r = devm_gpio_request_one(dev, gpio, flags, desc);
+	if (r)
+		return r == -ENOENT ? 0 : r;
+
+	gd = gpio_to_desc(gpio);
+	if (IS_ERR(gd))
+		return PTR_ERR(gd) == -ENOENT ? 0 : PTR_ERR(gd);
+
+	*gpiod = gd;
+	return 0;
+}
+
+static int sharp_ls_probe_pdata(struct platform_device *pdev)
+{
+	const struct panel_sharp_ls037v7dw01_platform_data *pdata;
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev, *in;
+	int r;
+
+	pdata = dev_get_platdata(&pdev->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&pdev->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	r = sharp_ls_get_gpio(&pdev->dev, pdata->mo_gpio, GPIOF_OUT_INIT_LOW,
+		"lcd MO", &ddata->mo_gpio);
+	if (r)
+		return r;
+	r = sharp_ls_get_gpio(&pdev->dev, pdata->lr_gpio, GPIOF_OUT_INIT_HIGH,
+		"lcd LR", &ddata->lr_gpio);
+	if (r)
+		return r;
+	r = sharp_ls_get_gpio(&pdev->dev, pdata->ud_gpio, GPIOF_OUT_INIT_HIGH,
+		"lcd UD", &ddata->ud_gpio);
+	if (r)
+		return r;
+	r = sharp_ls_get_gpio(&pdev->dev, pdata->resb_gpio, GPIOF_OUT_INIT_LOW,
+		"lcd RESB", &ddata->resb_gpio);
+	if (r)
+		return r;
+	r = sharp_ls_get_gpio(&pdev->dev, pdata->ini_gpio, GPIOF_OUT_INIT_LOW,
+		"lcd INI", &ddata->ini_gpio);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static  int sharp_ls_get_gpio_of(struct device *dev, int index, int val,
+	const char *desc, struct gpio_desc **gpiod)
+{
+	struct gpio_desc *gd;
+
+	*gpiod = NULL;
+
+	gd = devm_gpiod_get_index(dev, desc, index, GPIOD_OUT_LOW);
+	if (IS_ERR(gd))
+		return PTR_ERR(gd);
+
+	*gpiod = gd;
+	return 0;
+}
+
+static int sharp_ls_probe_of(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct device_node *node = pdev->dev.of_node;
+	struct omap_dss_device *in;
+	int r;
+
+	ddata->vcc = devm_regulator_get(&pdev->dev, "envdd");
+	if (IS_ERR(ddata->vcc)) {
+		dev_err(&pdev->dev, "failed to get regulator\n");
+		return PTR_ERR(ddata->vcc);
+	}
+
+	/* lcd INI */
+	r = sharp_ls_get_gpio_of(&pdev->dev, 0, 0, "enable", &ddata->ini_gpio);
+	if (r)
+		return r;
+
+	/* lcd RESB */
+	r = sharp_ls_get_gpio_of(&pdev->dev, 0, 0, "reset", &ddata->resb_gpio);
+	if (r)
+		return r;
+
+	/* lcd MO */
+	r = sharp_ls_get_gpio_of(&pdev->dev, 0, 0, "mode", &ddata->mo_gpio);
+	if (r)
+		return r;
+
+	/* lcd LR */
+	r = sharp_ls_get_gpio_of(&pdev->dev, 1, 1, "mode", &ddata->lr_gpio);
+	if (r)
+		return r;
+
+	/* lcd UD */
+	r = sharp_ls_get_gpio_of(&pdev->dev, 2, 1, "mode", &ddata->ud_gpio);
+	if (r)
+		return r;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&pdev->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int sharp_ls_probe(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	if (dev_get_platdata(&pdev->dev)) {
+		r = sharp_ls_probe_pdata(pdev);
+		if (r)
+			return r;
+	} else if (pdev->dev.of_node) {
+		r = sharp_ls_probe_of(pdev);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->videomode = sharp_ls_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &pdev->dev;
+	dssdev->driver = &sharp_ls_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+	dssdev->phy.dpi.data_lines = ddata->data_lines;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&pdev->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int __exit sharp_ls_remove(struct platform_device *pdev)
+{
+	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	omapdss_unregister_display(dssdev);
+
+	sharp_ls_disable(dssdev);
+	sharp_ls_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id sharp_ls_of_match[] = {
+	{ .compatible = "omapdss,sharp,ls037v7dw01", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, sharp_ls_of_match);
+
+static struct platform_driver sharp_ls_driver = {
+	.probe = sharp_ls_probe,
+	.remove = __exit_p(sharp_ls_remove),
+	.driver = {
+		.name = "panel-sharp-ls037v7dw01",
+		.of_match_table = sharp_ls_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_platform_driver(sharp_ls_driver);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
+MODULE_DESCRIPTION("Sharp LS037V7DW01 Panel Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
new file mode 100644
index 000000000000..31efcca801bd
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
@@ -0,0 +1,917 @@
+/*
+ * Sony ACX565AKM LCD Panel driver
+ *
+ * Copyright (C) 2010 Nokia Corporation
+ *
+ * Original Driver Author: Imre Deak <imre.deak@nokia.com>
+ * Based on panel-generic.c by Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ * Adapted to new DSS2 framework: Roger Quadros <roger.quadros@nokia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/spi/spi.h>
+#include <linux/jiffies.h>
+#include <linux/sched.h>
+#include <linux/backlight.h>
+#include <linux/fb.h>
+#include <linux/gpio.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+#define MIPID_CMD_READ_DISP_ID		0x04
+#define MIPID_CMD_READ_RED		0x06
+#define MIPID_CMD_READ_GREEN		0x07
+#define MIPID_CMD_READ_BLUE		0x08
+#define MIPID_CMD_READ_DISP_STATUS	0x09
+#define MIPID_CMD_RDDSDR		0x0F
+#define MIPID_CMD_SLEEP_IN		0x10
+#define MIPID_CMD_SLEEP_OUT		0x11
+#define MIPID_CMD_DISP_OFF		0x28
+#define MIPID_CMD_DISP_ON		0x29
+#define MIPID_CMD_WRITE_DISP_BRIGHTNESS	0x51
+#define MIPID_CMD_READ_DISP_BRIGHTNESS	0x52
+#define MIPID_CMD_WRITE_CTRL_DISP	0x53
+
+#define CTRL_DISP_BRIGHTNESS_CTRL_ON	(1 << 5)
+#define CTRL_DISP_AMBIENT_LIGHT_CTRL_ON	(1 << 4)
+#define CTRL_DISP_BACKLIGHT_ON		(1 << 2)
+#define CTRL_DISP_AUTO_BRIGHTNESS_ON	(1 << 1)
+
+#define MIPID_CMD_READ_CTRL_DISP	0x54
+#define MIPID_CMD_WRITE_CABC		0x55
+#define MIPID_CMD_READ_CABC		0x56
+
+#define MIPID_VER_LPH8923		3
+#define MIPID_VER_LS041Y3		4
+#define MIPID_VER_L4F00311		8
+#define MIPID_VER_ACX565AKM		9
+
+struct panel_drv_data {
+	struct omap_dss_device	dssdev;
+	struct omap_dss_device *in;
+
+	int reset_gpio;
+	int datapairs;
+
+	struct omap_video_timings videomode;
+
+	char		*name;
+	int		enabled;
+	int		model;
+	int		revision;
+	u8		display_id[3];
+	unsigned	has_bc:1;
+	unsigned	has_cabc:1;
+	unsigned	cabc_mode;
+	unsigned long	hw_guard_end;		/* next value of jiffies
+						   when we can issue the
+						   next sleep in/out command */
+	unsigned long	hw_guard_wait;		/* max guard time in jiffies */
+
+	struct spi_device	*spi;
+	struct mutex		mutex;
+
+	struct backlight_device *bl_dev;
+};
+
+static const struct omap_video_timings acx565akm_panel_timings = {
+	.x_res		= 800,
+	.y_res		= 480,
+	.pixelclock	= 24000000,
+	.hfp		= 28,
+	.hsw		= 4,
+	.hbp		= 24,
+	.vfp		= 3,
+	.vsw		= 3,
+	.vbp		= 4,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static void acx565akm_transfer(struct panel_drv_data *ddata, int cmd,
+			      const u8 *wbuf, int wlen, u8 *rbuf, int rlen)
+{
+	struct spi_message	m;
+	struct spi_transfer	*x, xfer[5];
+	int			r;
+
+	BUG_ON(ddata->spi == NULL);
+
+	spi_message_init(&m);
+
+	memset(xfer, 0, sizeof(xfer));
+	x = &xfer[0];
+
+	cmd &=  0xff;
+	x->tx_buf = &cmd;
+	x->bits_per_word = 9;
+	x->len = 2;
+
+	if (rlen > 1 && wlen == 0) {
+		/*
+		 * Between the command and the response data there is a
+		 * dummy clock cycle. Add an extra bit after the command
+		 * word to account for this.
+		 */
+		x->bits_per_word = 10;
+		cmd <<= 1;
+	}
+	spi_message_add_tail(x, &m);
+
+	if (wlen) {
+		x++;
+		x->tx_buf = wbuf;
+		x->len = wlen;
+		x->bits_per_word = 9;
+		spi_message_add_tail(x, &m);
+	}
+
+	if (rlen) {
+		x++;
+		x->rx_buf	= rbuf;
+		x->len		= rlen;
+		spi_message_add_tail(x, &m);
+	}
+
+	r = spi_sync(ddata->spi, &m);
+	if (r < 0)
+		dev_dbg(&ddata->spi->dev, "spi_sync %d\n", r);
+}
+
+static inline void acx565akm_cmd(struct panel_drv_data *ddata, int cmd)
+{
+	acx565akm_transfer(ddata, cmd, NULL, 0, NULL, 0);
+}
+
+static inline void acx565akm_write(struct panel_drv_data *ddata,
+			       int reg, const u8 *buf, int len)
+{
+	acx565akm_transfer(ddata, reg, buf, len, NULL, 0);
+}
+
+static inline void acx565akm_read(struct panel_drv_data *ddata,
+			      int reg, u8 *buf, int len)
+{
+	acx565akm_transfer(ddata, reg, NULL, 0, buf, len);
+}
+
+static void hw_guard_start(struct panel_drv_data *ddata, int guard_msec)
+{
+	ddata->hw_guard_wait = msecs_to_jiffies(guard_msec);
+	ddata->hw_guard_end = jiffies + ddata->hw_guard_wait;
+}
+
+static void hw_guard_wait(struct panel_drv_data *ddata)
+{
+	unsigned long wait = ddata->hw_guard_end - jiffies;
+
+	if ((long)wait > 0 && wait <= ddata->hw_guard_wait) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_timeout(wait);
+	}
+}
+
+static void set_sleep_mode(struct panel_drv_data *ddata, int on)
+{
+	int cmd;
+
+	if (on)
+		cmd = MIPID_CMD_SLEEP_IN;
+	else
+		cmd = MIPID_CMD_SLEEP_OUT;
+	/*
+	 * We have to keep 120msec between sleep in/out commands.
+	 * (8.2.15, 8.2.16).
+	 */
+	hw_guard_wait(ddata);
+	acx565akm_cmd(ddata, cmd);
+	hw_guard_start(ddata, 120);
+}
+
+static void set_display_state(struct panel_drv_data *ddata, int enabled)
+{
+	int cmd = enabled ? MIPID_CMD_DISP_ON : MIPID_CMD_DISP_OFF;
+
+	acx565akm_cmd(ddata, cmd);
+}
+
+static int panel_enabled(struct panel_drv_data *ddata)
+{
+	u32 disp_status;
+	int enabled;
+
+	acx565akm_read(ddata, MIPID_CMD_READ_DISP_STATUS,
+			(u8 *)&disp_status, 4);
+	disp_status = __be32_to_cpu(disp_status);
+	enabled = (disp_status & (1 << 17)) && (disp_status & (1 << 10));
+	dev_dbg(&ddata->spi->dev,
+		"LCD panel %senabled by bootloader (status 0x%04x)\n",
+		enabled ? "" : "not ", disp_status);
+	return enabled;
+}
+
+static int panel_detect(struct panel_drv_data *ddata)
+{
+	acx565akm_read(ddata, MIPID_CMD_READ_DISP_ID, ddata->display_id, 3);
+	dev_dbg(&ddata->spi->dev, "MIPI display ID: %02x%02x%02x\n",
+		ddata->display_id[0],
+		ddata->display_id[1],
+		ddata->display_id[2]);
+
+	switch (ddata->display_id[0]) {
+	case 0x10:
+		ddata->model = MIPID_VER_ACX565AKM;
+		ddata->name = "acx565akm";
+		ddata->has_bc = 1;
+		ddata->has_cabc = 1;
+		break;
+	case 0x29:
+		ddata->model = MIPID_VER_L4F00311;
+		ddata->name = "l4f00311";
+		break;
+	case 0x45:
+		ddata->model = MIPID_VER_LPH8923;
+		ddata->name = "lph8923";
+		break;
+	case 0x83:
+		ddata->model = MIPID_VER_LS041Y3;
+		ddata->name = "ls041y3";
+		break;
+	default:
+		ddata->name = "unknown";
+		dev_err(&ddata->spi->dev, "invalid display ID\n");
+		return -ENODEV;
+	}
+
+	ddata->revision = ddata->display_id[1];
+
+	dev_info(&ddata->spi->dev, "omapfb: %s rev %02x LCD detected\n",
+			ddata->name, ddata->revision);
+
+	return 0;
+}
+
+/*----------------------Backlight Control-------------------------*/
+
+static void enable_backlight_ctrl(struct panel_drv_data *ddata, int enable)
+{
+	u16 ctrl;
+
+	acx565akm_read(ddata, MIPID_CMD_READ_CTRL_DISP, (u8 *)&ctrl, 1);
+	if (enable) {
+		ctrl |= CTRL_DISP_BRIGHTNESS_CTRL_ON |
+			CTRL_DISP_BACKLIGHT_ON;
+	} else {
+		ctrl &= ~(CTRL_DISP_BRIGHTNESS_CTRL_ON |
+			  CTRL_DISP_BACKLIGHT_ON);
+	}
+
+	ctrl |= 1 << 8;
+	acx565akm_write(ddata, MIPID_CMD_WRITE_CTRL_DISP, (u8 *)&ctrl, 2);
+}
+
+static void set_cabc_mode(struct panel_drv_data *ddata, unsigned mode)
+{
+	u16 cabc_ctrl;
+
+	ddata->cabc_mode = mode;
+	if (!ddata->enabled)
+		return;
+	cabc_ctrl = 0;
+	acx565akm_read(ddata, MIPID_CMD_READ_CABC, (u8 *)&cabc_ctrl, 1);
+	cabc_ctrl &= ~3;
+	cabc_ctrl |= (1 << 8) | (mode & 3);
+	acx565akm_write(ddata, MIPID_CMD_WRITE_CABC, (u8 *)&cabc_ctrl, 2);
+}
+
+static unsigned get_cabc_mode(struct panel_drv_data *ddata)
+{
+	return ddata->cabc_mode;
+}
+
+static unsigned get_hw_cabc_mode(struct panel_drv_data *ddata)
+{
+	u8 cabc_ctrl;
+
+	acx565akm_read(ddata, MIPID_CMD_READ_CABC, &cabc_ctrl, 1);
+	return cabc_ctrl & 3;
+}
+
+static void acx565akm_set_brightness(struct panel_drv_data *ddata, int level)
+{
+	int bv;
+
+	bv = level | (1 << 8);
+	acx565akm_write(ddata, MIPID_CMD_WRITE_DISP_BRIGHTNESS, (u8 *)&bv, 2);
+
+	if (level)
+		enable_backlight_ctrl(ddata, 1);
+	else
+		enable_backlight_ctrl(ddata, 0);
+}
+
+static int acx565akm_get_actual_brightness(struct panel_drv_data *ddata)
+{
+	u8 bv;
+
+	acx565akm_read(ddata, MIPID_CMD_READ_DISP_BRIGHTNESS, &bv, 1);
+
+	return bv;
+}
+
+
+static int acx565akm_bl_update_status(struct backlight_device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
+	int level;
+
+	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
+
+	if (dev->props.fb_blank == FB_BLANK_UNBLANK &&
+			dev->props.power == FB_BLANK_UNBLANK)
+		level = dev->props.brightness;
+	else
+		level = 0;
+
+	if (ddata->has_bc)
+		acx565akm_set_brightness(ddata, level);
+	else
+		return -ENODEV;
+
+	return 0;
+}
+
+static int acx565akm_bl_get_intensity(struct backlight_device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
+
+	dev_dbg(&dev->dev, "%s\n", __func__);
+
+	if (!ddata->has_bc)
+		return -ENODEV;
+
+	if (dev->props.fb_blank == FB_BLANK_UNBLANK &&
+			dev->props.power == FB_BLANK_UNBLANK) {
+		if (ddata->has_bc)
+			return acx565akm_get_actual_brightness(ddata);
+		else
+			return dev->props.brightness;
+	}
+
+	return 0;
+}
+
+static int acx565akm_bl_update_status_locked(struct backlight_device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
+	int r;
+
+	mutex_lock(&ddata->mutex);
+	r = acx565akm_bl_update_status(dev);
+	mutex_unlock(&ddata->mutex);
+
+	return r;
+}
+
+static int acx565akm_bl_get_intensity_locked(struct backlight_device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
+	int r;
+
+	mutex_lock(&ddata->mutex);
+	r = acx565akm_bl_get_intensity(dev);
+	mutex_unlock(&ddata->mutex);
+
+	return r;
+}
+
+static const struct backlight_ops acx565akm_bl_ops = {
+	.get_brightness = acx565akm_bl_get_intensity_locked,
+	.update_status  = acx565akm_bl_update_status_locked,
+};
+
+/*--------------------Auto Brightness control via Sysfs---------------------*/
+
+static const char * const cabc_modes[] = {
+	"off",		/* always used when CABC is not supported */
+	"ui",
+	"still-image",
+	"moving-image",
+};
+
+static ssize_t show_cabc_mode(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	const char *mode_str;
+	int mode;
+	int len;
+
+	if (!ddata->has_cabc)
+		mode = 0;
+	else
+		mode = get_cabc_mode(ddata);
+	mode_str = "unknown";
+	if (mode >= 0 && mode < ARRAY_SIZE(cabc_modes))
+		mode_str = cabc_modes[mode];
+	len = snprintf(buf, PAGE_SIZE, "%s\n", mode_str);
+
+	return len < PAGE_SIZE - 1 ? len : PAGE_SIZE - 1;
+}
+
+static ssize_t store_cabc_mode(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(cabc_modes); i++) {
+		const char *mode_str = cabc_modes[i];
+		int cmp_len = strlen(mode_str);
+
+		if (count > 0 && buf[count - 1] == '\n')
+			count--;
+		if (count != cmp_len)
+			continue;
+
+		if (strncmp(buf, mode_str, cmp_len) == 0)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(cabc_modes))
+		return -EINVAL;
+
+	if (!ddata->has_cabc && i != 0)
+		return -EINVAL;
+
+	mutex_lock(&ddata->mutex);
+	set_cabc_mode(ddata, i);
+	mutex_unlock(&ddata->mutex);
+
+	return count;
+}
+
+static ssize_t show_cabc_available_modes(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	int len;
+	int i;
+
+	if (!ddata->has_cabc)
+		return snprintf(buf, PAGE_SIZE, "%s\n", cabc_modes[0]);
+
+	for (i = 0, len = 0;
+	     len < PAGE_SIZE && i < ARRAY_SIZE(cabc_modes); i++)
+		len += snprintf(&buf[len], PAGE_SIZE - len, "%s%s%s",
+			i ? " " : "", cabc_modes[i],
+			i == ARRAY_SIZE(cabc_modes) - 1 ? "\n" : "");
+
+	return len < PAGE_SIZE ? len : PAGE_SIZE - 1;
+}
+
+static DEVICE_ATTR(cabc_mode, S_IRUGO | S_IWUSR,
+		show_cabc_mode, store_cabc_mode);
+static DEVICE_ATTR(cabc_available_modes, S_IRUGO,
+		show_cabc_available_modes, NULL);
+
+static struct attribute *bldev_attrs[] = {
+	&dev_attr_cabc_mode.attr,
+	&dev_attr_cabc_available_modes.attr,
+	NULL,
+};
+
+static struct attribute_group bldev_attr_group = {
+	.attrs = bldev_attrs,
+};
+
+static int acx565akm_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.sdi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void acx565akm_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.sdi->disconnect(in, dssdev);
+}
+
+static int acx565akm_panel_power_on(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
+
+	in->ops.sdi->set_timings(in, &ddata->videomode);
+
+	if (ddata->datapairs > 0)
+		in->ops.sdi->set_datapairs(in, ddata->datapairs);
+
+	r = in->ops.sdi->enable(in);
+	if (r) {
+		pr_err("%s sdi enable failed\n", __func__);
+		return r;
+	}
+
+	/*FIXME tweak me */
+	msleep(50);
+
+	if (gpio_is_valid(ddata->reset_gpio))
+		gpio_set_value(ddata->reset_gpio, 1);
+
+	if (ddata->enabled) {
+		dev_dbg(&ddata->spi->dev, "panel already enabled\n");
+		return 0;
+	}
+
+	/*
+	 * We have to meet all the following delay requirements:
+	 * 1. tRW: reset pulse width 10usec (7.12.1)
+	 * 2. tRT: reset cancel time 5msec (7.12.1)
+	 * 3. Providing PCLK,HS,VS signals for 2 frames = ~50msec worst
+	 *    case (7.6.2)
+	 * 4. 120msec before the sleep out command (7.12.1)
+	 */
+	msleep(120);
+
+	set_sleep_mode(ddata, 0);
+	ddata->enabled = 1;
+
+	/* 5msec between sleep out and the next command. (8.2.16) */
+	usleep_range(5000, 10000);
+	set_display_state(ddata, 1);
+	set_cabc_mode(ddata, ddata->cabc_mode);
+
+	return acx565akm_bl_update_status(ddata->bl_dev);
+}
+
+static void acx565akm_panel_power_off(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(dssdev->dev, "%s\n", __func__);
+
+	if (!ddata->enabled)
+		return;
+
+	set_display_state(ddata, 0);
+	set_sleep_mode(ddata, 1);
+	ddata->enabled = 0;
+	/*
+	 * We have to provide PCLK,HS,VS signals for 2 frames (worst case
+	 * ~50msec) after sending the sleep in command and asserting the
+	 * reset signal. We probably could assert the reset w/o the delay
+	 * but we still delay to avoid possible artifacts. (7.6.1)
+	 */
+	msleep(50);
+
+	if (gpio_is_valid(ddata->reset_gpio))
+		gpio_set_value(ddata->reset_gpio, 0);
+
+	/* FIXME need to tweak this delay */
+	msleep(100);
+
+	in->ops.sdi->disable(in);
+}
+
+static int acx565akm_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	int r;
+
+	dev_dbg(dssdev->dev, "%s\n", __func__);
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	mutex_lock(&ddata->mutex);
+	r = acx565akm_panel_power_on(dssdev);
+	mutex_unlock(&ddata->mutex);
+	if (r)
+		return r;
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void acx565akm_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	dev_dbg(dssdev->dev, "%s\n", __func__);
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	mutex_lock(&ddata->mutex);
+	acx565akm_panel_power_off(dssdev);
+	mutex_unlock(&ddata->mutex);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void acx565akm_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.sdi->set_timings(in, timings);
+}
+
+static void acx565akm_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int acx565akm_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.sdi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver acx565akm_ops = {
+	.connect	= acx565akm_connect,
+	.disconnect	= acx565akm_disconnect,
+
+	.enable		= acx565akm_enable,
+	.disable	= acx565akm_disable,
+
+	.set_timings	= acx565akm_set_timings,
+	.get_timings	= acx565akm_get_timings,
+	.check_timings	= acx565akm_check_timings,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+static int acx565akm_probe_pdata(struct spi_device *spi)
+{
+	const struct panel_acx565akm_platform_data *pdata;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&spi->dev);
+
+	ddata->reset_gpio = pdata->reset_gpio;
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&spi->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+	ddata->in = in;
+
+	ddata->datapairs = pdata->datapairs;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int acx565akm_probe_of(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct device_node *np = spi->dev.of_node;
+
+	ddata->reset_gpio = of_get_named_gpio(np, "reset-gpios", 0);
+
+	ddata->in = omapdss_of_find_source_for_first_ep(np);
+	if (IS_ERR(ddata->in)) {
+		dev_err(&spi->dev, "failed to find video source\n");
+		return PTR_ERR(ddata->in);
+	}
+
+	return 0;
+}
+
+static int acx565akm_probe(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	struct backlight_device *bldev;
+	int max_brightness, brightness;
+	struct backlight_properties props;
+	int r;
+
+	dev_dbg(&spi->dev, "%s\n", __func__);
+
+	spi->mode = SPI_MODE_3;
+
+	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	dev_set_drvdata(&spi->dev, ddata);
+
+	ddata->spi = spi;
+
+	mutex_init(&ddata->mutex);
+
+	if (dev_get_platdata(&spi->dev)) {
+		r = acx565akm_probe_pdata(spi);
+		if (r)
+			return r;
+	} else if (spi->dev.of_node) {
+		r = acx565akm_probe_of(spi);
+		if (r)
+			return r;
+	} else {
+		dev_err(&spi->dev, "platform data missing!\n");
+		return -ENODEV;
+	}
+
+	if (gpio_is_valid(ddata->reset_gpio)) {
+		r = devm_gpio_request_one(&spi->dev, ddata->reset_gpio,
+				GPIOF_OUT_INIT_LOW, "lcd reset");
+		if (r)
+			goto err_gpio;
+	}
+
+	if (gpio_is_valid(ddata->reset_gpio))
+		gpio_set_value(ddata->reset_gpio, 1);
+
+	/*
+	 * After reset we have to wait 5 msec before the first
+	 * command can be sent.
+	 */
+	usleep_range(5000, 10000);
+
+	ddata->enabled = panel_enabled(ddata);
+
+	r = panel_detect(ddata);
+
+	if (!ddata->enabled && gpio_is_valid(ddata->reset_gpio))
+		gpio_set_value(ddata->reset_gpio, 0);
+
+	if (r) {
+		dev_err(&spi->dev, "%s panel detect error\n", __func__);
+		goto err_detect;
+	}
+
+	memset(&props, 0, sizeof(props));
+	props.fb_blank = FB_BLANK_UNBLANK;
+	props.power = FB_BLANK_UNBLANK;
+	props.type = BACKLIGHT_RAW;
+
+	bldev = backlight_device_register("acx565akm", &ddata->spi->dev,
+			ddata, &acx565akm_bl_ops, &props);
+	if (IS_ERR(bldev)) {
+		r = PTR_ERR(bldev);
+		goto err_reg_bl;
+	}
+	ddata->bl_dev = bldev;
+	if (ddata->has_cabc) {
+		r = sysfs_create_group(&bldev->dev.kobj, &bldev_attr_group);
+		if (r) {
+			dev_err(&bldev->dev,
+				"%s failed to create sysfs files\n", __func__);
+			goto err_sysfs;
+		}
+		ddata->cabc_mode = get_hw_cabc_mode(ddata);
+	}
+
+	max_brightness = 255;
+
+	if (ddata->has_bc)
+		brightness = acx565akm_get_actual_brightness(ddata);
+	else
+		brightness = 0;
+
+	bldev->props.max_brightness = max_brightness;
+	bldev->props.brightness = brightness;
+
+	acx565akm_bl_update_status(bldev);
+
+
+	ddata->videomode = acx565akm_panel_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &spi->dev;
+	dssdev->driver = &acx565akm_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_SDI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&spi->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	sysfs_remove_group(&bldev->dev.kobj, &bldev_attr_group);
+err_sysfs:
+	backlight_device_unregister(bldev);
+err_reg_bl:
+err_detect:
+err_gpio:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int acx565akm_remove(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
+
+	sysfs_remove_group(&ddata->bl_dev->dev.kobj, &bldev_attr_group);
+	backlight_device_unregister(ddata->bl_dev);
+
+	omapdss_unregister_display(dssdev);
+
+	acx565akm_disable(dssdev);
+	acx565akm_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id acx565akm_of_match[] = {
+	{ .compatible = "omapdss,sony,acx565akm", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, acx565akm_of_match);
+
+static struct spi_driver acx565akm_driver = {
+	.driver = {
+		.name	= "acx565akm",
+		.of_match_table = acx565akm_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe	= acx565akm_probe,
+	.remove	= acx565akm_remove,
+};
+
+module_spi_driver(acx565akm_driver);
+
+MODULE_AUTHOR("Nokia Corporation");
+MODULE_DESCRIPTION("acx565akm LCD Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
new file mode 100644
index 000000000000..4d657f3ab679
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
@@ -0,0 +1,511 @@
+/*
+ * Toppoly TD028TTEC1 panel support
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Neo 1973 code (jbt6k74.c):
+ * Copyright (C) 2006-2007 by OpenMoko, Inc.
+ * Author: Harald Welte <laforge@openmoko.org>
+ *
+ * Ported and adapted from Neo 1973 U-Boot by:
+ * H. Nikolaus Schaller <hns@goldelico.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/spi/spi.h>
+#include <linux/gpio.h>
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+struct panel_drv_data {
+	struct omap_dss_device dssdev;
+	struct omap_dss_device *in;
+
+	int data_lines;
+
+	struct omap_video_timings videomode;
+
+	struct spi_device *spi_dev;
+};
+
+static struct omap_video_timings td028ttec1_panel_timings = {
+	.x_res		= 480,
+	.y_res		= 640,
+	.pixelclock	= 22153000,
+	.hfp		= 24,
+	.hsw		= 8,
+	.hbp		= 8,
+	.vfp		= 4,
+	.vsw		= 2,
+	.vbp		= 2,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+};
+
+#define JBT_COMMAND	0x000
+#define JBT_DATA	0x100
+
+static int jbt_ret_write_0(struct panel_drv_data *ddata, u8 reg)
+{
+	int rc;
+	u16 tx_buf = JBT_COMMAND | reg;
+
+	rc = spi_write(ddata->spi_dev, (u8 *)&tx_buf,
+			1*sizeof(u16));
+	if (rc != 0)
+		dev_err(&ddata->spi_dev->dev,
+			"jbt_ret_write_0 spi_write ret %d\n", rc);
+
+	return rc;
+}
+
+static int jbt_reg_write_1(struct panel_drv_data *ddata, u8 reg, u8 data)
+{
+	int rc;
+	u16 tx_buf[2];
+
+	tx_buf[0] = JBT_COMMAND | reg;
+	tx_buf[1] = JBT_DATA | data;
+	rc = spi_write(ddata->spi_dev, (u8 *)tx_buf,
+			2*sizeof(u16));
+	if (rc != 0)
+		dev_err(&ddata->spi_dev->dev,
+			"jbt_reg_write_1 spi_write ret %d\n", rc);
+
+	return rc;
+}
+
+static int jbt_reg_write_2(struct panel_drv_data *ddata, u8 reg, u16 data)
+{
+	int rc;
+	u16 tx_buf[3];
+
+	tx_buf[0] = JBT_COMMAND | reg;
+	tx_buf[1] = JBT_DATA | (data >> 8);
+	tx_buf[2] = JBT_DATA | (data & 0xff);
+
+	rc = spi_write(ddata->spi_dev, (u8 *)tx_buf,
+			3*sizeof(u16));
+
+	if (rc != 0)
+		dev_err(&ddata->spi_dev->dev,
+			"jbt_reg_write_2 spi_write ret %d\n", rc);
+
+	return rc;
+}
+
+enum jbt_register {
+	JBT_REG_SLEEP_IN		= 0x10,
+	JBT_REG_SLEEP_OUT		= 0x11,
+
+	JBT_REG_DISPLAY_OFF		= 0x28,
+	JBT_REG_DISPLAY_ON		= 0x29,
+
+	JBT_REG_RGB_FORMAT		= 0x3a,
+	JBT_REG_QUAD_RATE		= 0x3b,
+
+	JBT_REG_POWER_ON_OFF		= 0xb0,
+	JBT_REG_BOOSTER_OP		= 0xb1,
+	JBT_REG_BOOSTER_MODE		= 0xb2,
+	JBT_REG_BOOSTER_FREQ		= 0xb3,
+	JBT_REG_OPAMP_SYSCLK		= 0xb4,
+	JBT_REG_VSC_VOLTAGE		= 0xb5,
+	JBT_REG_VCOM_VOLTAGE		= 0xb6,
+	JBT_REG_EXT_DISPL		= 0xb7,
+	JBT_REG_OUTPUT_CONTROL		= 0xb8,
+	JBT_REG_DCCLK_DCEV		= 0xb9,
+	JBT_REG_DISPLAY_MODE1		= 0xba,
+	JBT_REG_DISPLAY_MODE2		= 0xbb,
+	JBT_REG_DISPLAY_MODE		= 0xbc,
+	JBT_REG_ASW_SLEW		= 0xbd,
+	JBT_REG_DUMMY_DISPLAY		= 0xbe,
+	JBT_REG_DRIVE_SYSTEM		= 0xbf,
+
+	JBT_REG_SLEEP_OUT_FR_A		= 0xc0,
+	JBT_REG_SLEEP_OUT_FR_B		= 0xc1,
+	JBT_REG_SLEEP_OUT_FR_C		= 0xc2,
+	JBT_REG_SLEEP_IN_LCCNT_D	= 0xc3,
+	JBT_REG_SLEEP_IN_LCCNT_E	= 0xc4,
+	JBT_REG_SLEEP_IN_LCCNT_F	= 0xc5,
+	JBT_REG_SLEEP_IN_LCCNT_G	= 0xc6,
+
+	JBT_REG_GAMMA1_FINE_1		= 0xc7,
+	JBT_REG_GAMMA1_FINE_2		= 0xc8,
+	JBT_REG_GAMMA1_INCLINATION	= 0xc9,
+	JBT_REG_GAMMA1_BLUE_OFFSET	= 0xca,
+
+	JBT_REG_BLANK_CONTROL		= 0xcf,
+	JBT_REG_BLANK_TH_TV		= 0xd0,
+	JBT_REG_CKV_ON_OFF		= 0xd1,
+	JBT_REG_CKV_1_2			= 0xd2,
+	JBT_REG_OEV_TIMING		= 0xd3,
+	JBT_REG_ASW_TIMING_1		= 0xd4,
+	JBT_REG_ASW_TIMING_2		= 0xd5,
+
+	JBT_REG_HCLOCK_VGA		= 0xec,
+	JBT_REG_HCLOCK_QVGA		= 0xed,
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int td028ttec1_panel_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void td028ttec1_panel_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	dev_dbg(dssdev->dev, "td028ttec1_panel_enable() - state %d\n",
+		dssdev->state);
+
+	/* three times command zero */
+	r |= jbt_ret_write_0(ddata, 0x00);
+	usleep_range(1000, 2000);
+	r |= jbt_ret_write_0(ddata, 0x00);
+	usleep_range(1000, 2000);
+	r |= jbt_ret_write_0(ddata, 0x00);
+	usleep_range(1000, 2000);
+
+	if (r) {
+		dev_warn(dssdev->dev, "transfer error\n");
+		goto transfer_err;
+	}
+
+	/* deep standby out */
+	r |= jbt_reg_write_1(ddata, JBT_REG_POWER_ON_OFF, 0x17);
+
+	/* RGB I/F on, RAM write off, QVGA through, SIGCON enable */
+	r |= jbt_reg_write_1(ddata, JBT_REG_DISPLAY_MODE, 0x80);
+
+	/* Quad mode off */
+	r |= jbt_reg_write_1(ddata, JBT_REG_QUAD_RATE, 0x00);
+
+	/* AVDD on, XVDD on */
+	r |= jbt_reg_write_1(ddata, JBT_REG_POWER_ON_OFF, 0x16);
+
+	/* Output control */
+	r |= jbt_reg_write_2(ddata, JBT_REG_OUTPUT_CONTROL, 0xfff9);
+
+	/* Sleep mode off */
+	r |= jbt_ret_write_0(ddata, JBT_REG_SLEEP_OUT);
+
+	/* at this point we have like 50% grey */
+
+	/* initialize register set */
+	r |= jbt_reg_write_1(ddata, JBT_REG_DISPLAY_MODE1, 0x01);
+	r |= jbt_reg_write_1(ddata, JBT_REG_DISPLAY_MODE2, 0x00);
+	r |= jbt_reg_write_1(ddata, JBT_REG_RGB_FORMAT, 0x60);
+	r |= jbt_reg_write_1(ddata, JBT_REG_DRIVE_SYSTEM, 0x10);
+	r |= jbt_reg_write_1(ddata, JBT_REG_BOOSTER_OP, 0x56);
+	r |= jbt_reg_write_1(ddata, JBT_REG_BOOSTER_MODE, 0x33);
+	r |= jbt_reg_write_1(ddata, JBT_REG_BOOSTER_FREQ, 0x11);
+	r |= jbt_reg_write_1(ddata, JBT_REG_BOOSTER_FREQ, 0x11);
+	r |= jbt_reg_write_1(ddata, JBT_REG_OPAMP_SYSCLK, 0x02);
+	r |= jbt_reg_write_1(ddata, JBT_REG_VSC_VOLTAGE, 0x2b);
+	r |= jbt_reg_write_1(ddata, JBT_REG_VCOM_VOLTAGE, 0x40);
+	r |= jbt_reg_write_1(ddata, JBT_REG_EXT_DISPL, 0x03);
+	r |= jbt_reg_write_1(ddata, JBT_REG_DCCLK_DCEV, 0x04);
+	/*
+	 * default of 0x02 in JBT_REG_ASW_SLEW responsible for 72Hz requirement
+	 * to avoid red / blue flicker
+	 */
+	r |= jbt_reg_write_1(ddata, JBT_REG_ASW_SLEW, 0x04);
+	r |= jbt_reg_write_1(ddata, JBT_REG_DUMMY_DISPLAY, 0x00);
+
+	r |= jbt_reg_write_1(ddata, JBT_REG_SLEEP_OUT_FR_A, 0x11);
+	r |= jbt_reg_write_1(ddata, JBT_REG_SLEEP_OUT_FR_B, 0x11);
+	r |= jbt_reg_write_1(ddata, JBT_REG_SLEEP_OUT_FR_C, 0x11);
+	r |= jbt_reg_write_2(ddata, JBT_REG_SLEEP_IN_LCCNT_D, 0x2040);
+	r |= jbt_reg_write_2(ddata, JBT_REG_SLEEP_IN_LCCNT_E, 0x60c0);
+	r |= jbt_reg_write_2(ddata, JBT_REG_SLEEP_IN_LCCNT_F, 0x1020);
+	r |= jbt_reg_write_2(ddata, JBT_REG_SLEEP_IN_LCCNT_G, 0x60c0);
+
+	r |= jbt_reg_write_2(ddata, JBT_REG_GAMMA1_FINE_1, 0x5533);
+	r |= jbt_reg_write_1(ddata, JBT_REG_GAMMA1_FINE_2, 0x00);
+	r |= jbt_reg_write_1(ddata, JBT_REG_GAMMA1_INCLINATION, 0x00);
+	r |= jbt_reg_write_1(ddata, JBT_REG_GAMMA1_BLUE_OFFSET, 0x00);
+
+	r |= jbt_reg_write_2(ddata, JBT_REG_HCLOCK_VGA, 0x1f0);
+	r |= jbt_reg_write_1(ddata, JBT_REG_BLANK_CONTROL, 0x02);
+	r |= jbt_reg_write_2(ddata, JBT_REG_BLANK_TH_TV, 0x0804);
+
+	r |= jbt_reg_write_1(ddata, JBT_REG_CKV_ON_OFF, 0x01);
+	r |= jbt_reg_write_2(ddata, JBT_REG_CKV_1_2, 0x0000);
+
+	r |= jbt_reg_write_2(ddata, JBT_REG_OEV_TIMING, 0x0d0e);
+	r |= jbt_reg_write_2(ddata, JBT_REG_ASW_TIMING_1, 0x11a4);
+	r |= jbt_reg_write_1(ddata, JBT_REG_ASW_TIMING_2, 0x0e);
+
+	r |= jbt_ret_write_0(ddata, JBT_REG_DISPLAY_ON);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+transfer_err:
+
+	return r ? -EIO : 0;
+}
+
+static void td028ttec1_panel_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	dev_dbg(dssdev->dev, "td028ttec1_panel_disable()\n");
+
+	jbt_ret_write_0(ddata, JBT_REG_DISPLAY_OFF);
+	jbt_reg_write_2(ddata, JBT_REG_OUTPUT_CONTROL, 0x8002);
+	jbt_ret_write_0(ddata, JBT_REG_SLEEP_IN);
+	jbt_reg_write_1(ddata, JBT_REG_POWER_ON_OFF, 0x00);
+
+	in->ops.dpi->disable(in);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void td028ttec1_panel_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void td028ttec1_panel_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int td028ttec1_panel_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver td028ttec1_ops = {
+	.connect	= td028ttec1_panel_connect,
+	.disconnect	= td028ttec1_panel_disconnect,
+
+	.enable		= td028ttec1_panel_enable,
+	.disable	= td028ttec1_panel_disable,
+
+	.set_timings	= td028ttec1_panel_set_timings,
+	.get_timings	= td028ttec1_panel_get_timings,
+	.check_timings	= td028ttec1_panel_check_timings,
+};
+
+static int td028ttec1_panel_probe_pdata(struct spi_device *spi)
+{
+	const struct panel_tpo_td028ttec1_platform_data *pdata;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&spi->dev);
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&spi->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int td028ttec1_probe_of(struct spi_device *spi)
+{
+	struct device_node *node = spi->dev.of_node;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *in;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&spi->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int td028ttec1_panel_probe(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	dev_dbg(&spi->dev, "%s\n", __func__);
+
+	spi->bits_per_word = 9;
+	spi->mode = SPI_MODE_3;
+
+	r = spi_setup(spi);
+	if (r < 0) {
+		dev_err(&spi->dev, "spi_setup failed: %d\n", r);
+		return r;
+	}
+
+	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	dev_set_drvdata(&spi->dev, ddata);
+
+	ddata->spi_dev = spi;
+
+	if (dev_get_platdata(&spi->dev)) {
+		r = td028ttec1_panel_probe_pdata(spi);
+		if (r)
+			return r;
+	} else if (spi->dev.of_node) {
+		r = td028ttec1_probe_of(spi);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->videomode = td028ttec1_panel_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &spi->dev;
+	dssdev->driver = &td028ttec1_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+	dssdev->phy.dpi.data_lines = ddata->data_lines;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&spi->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int td028ttec1_panel_remove(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->spi_dev->dev, "%s\n", __func__);
+
+	omapdss_unregister_display(dssdev);
+
+	td028ttec1_panel_disable(dssdev);
+	td028ttec1_panel_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	return 0;
+}
+
+static const struct of_device_id td028ttec1_of_match[] = {
+	{ .compatible = "omapdss,toppoly,td028ttec1", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, td028ttec1_of_match);
+
+static struct spi_driver td028ttec1_spi_driver = {
+	.probe		= td028ttec1_panel_probe,
+	.remove		= td028ttec1_panel_remove,
+
+	.driver         = {
+		.name   = "panel-tpo-td028ttec1",
+		.of_match_table = td028ttec1_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+module_spi_driver(td028ttec1_spi_driver);
+
+MODULE_ALIAS("spi:toppoly,td028ttec1");
+MODULE_AUTHOR("H. Nikolaus Schaller <hns@goldelico.com>");
+MODULE_DESCRIPTION("Toppoly TD028TTEC1 panel driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
new file mode 100644
index 000000000000..68e3b68a2920
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
@@ -0,0 +1,686 @@
+/*
+ * TPO TD043MTEA1 Panel driver
+ *
+ * Author: Gražvydas Ignotas <notasas@gmail.com>
+ * Converted to new DSS device model: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/spi/spi.h>
+#include <linux/regulator/consumer.h>
+#include <linux/gpio.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/of_gpio.h>
+
+#include <video/omapdss.h>
+#include <video/omap-panel-data.h>
+
+#define TPO_R02_MODE(x)		((x) & 7)
+#define TPO_R02_MODE_800x480	7
+#define TPO_R02_NCLK_RISING	BIT(3)
+#define TPO_R02_HSYNC_HIGH	BIT(4)
+#define TPO_R02_VSYNC_HIGH	BIT(5)
+
+#define TPO_R03_NSTANDBY	BIT(0)
+#define TPO_R03_EN_CP_CLK	BIT(1)
+#define TPO_R03_EN_VGL_PUMP	BIT(2)
+#define TPO_R03_EN_PWM		BIT(3)
+#define TPO_R03_DRIVING_CAP_100	BIT(4)
+#define TPO_R03_EN_PRE_CHARGE	BIT(6)
+#define TPO_R03_SOFTWARE_CTL	BIT(7)
+
+#define TPO_R04_NFLIP_H		BIT(0)
+#define TPO_R04_NFLIP_V		BIT(1)
+#define TPO_R04_CP_CLK_FREQ_1H	BIT(2)
+#define TPO_R04_VGL_FREQ_1H	BIT(4)
+
+#define TPO_R03_VAL_NORMAL (TPO_R03_NSTANDBY | TPO_R03_EN_CP_CLK | \
+			TPO_R03_EN_VGL_PUMP |  TPO_R03_EN_PWM | \
+			TPO_R03_DRIVING_CAP_100 | TPO_R03_EN_PRE_CHARGE | \
+			TPO_R03_SOFTWARE_CTL)
+
+#define TPO_R03_VAL_STANDBY (TPO_R03_DRIVING_CAP_100 | \
+			TPO_R03_EN_PRE_CHARGE | TPO_R03_SOFTWARE_CTL)
+
+static const u16 tpo_td043_def_gamma[12] = {
+	105, 315, 381, 431, 490, 537, 579, 686, 780, 837, 880, 1023
+};
+
+struct panel_drv_data {
+	struct omap_dss_device	dssdev;
+	struct omap_dss_device *in;
+
+	struct omap_video_timings videomode;
+
+	int data_lines;
+
+	struct spi_device *spi;
+	struct regulator *vcc_reg;
+	int nreset_gpio;
+	u16 gamma[12];
+	u32 mode;
+	u32 hmirror:1;
+	u32 vmirror:1;
+	u32 powered_on:1;
+	u32 spi_suspended:1;
+	u32 power_on_resume:1;
+};
+
+static const struct omap_video_timings tpo_td043_timings = {
+	.x_res		= 800,
+	.y_res		= 480,
+
+	.pixelclock	= 36000000,
+
+	.hsw		= 1,
+	.hfp		= 68,
+	.hbp		= 214,
+
+	.vsw		= 1,
+	.vfp		= 39,
+	.vbp		= 34,
+
+	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
+	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
+	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+};
+
+#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
+
+static int tpo_td043_write(struct spi_device *spi, u8 addr, u8 data)
+{
+	struct spi_message	m;
+	struct spi_transfer	xfer;
+	u16			w;
+	int			r;
+
+	spi_message_init(&m);
+
+	memset(&xfer, 0, sizeof(xfer));
+
+	w = ((u16)addr << 10) | (1 << 8) | data;
+	xfer.tx_buf = &w;
+	xfer.bits_per_word = 16;
+	xfer.len = 2;
+	spi_message_add_tail(&xfer, &m);
+
+	r = spi_sync(spi, &m);
+	if (r < 0)
+		dev_warn(&spi->dev, "failed to write to LCD reg (%d)\n", r);
+	return r;
+}
+
+static void tpo_td043_write_gamma(struct spi_device *spi, u16 gamma[12])
+{
+	u8 i, val;
+
+	/* gamma bits [9:8] */
+	for (val = i = 0; i < 4; i++)
+		val |= (gamma[i] & 0x300) >> ((i + 1) * 2);
+	tpo_td043_write(spi, 0x11, val);
+
+	for (val = i = 0; i < 4; i++)
+		val |= (gamma[i+4] & 0x300) >> ((i + 1) * 2);
+	tpo_td043_write(spi, 0x12, val);
+
+	for (val = i = 0; i < 4; i++)
+		val |= (gamma[i+8] & 0x300) >> ((i + 1) * 2);
+	tpo_td043_write(spi, 0x13, val);
+
+	/* gamma bits [7:0] */
+	for (val = i = 0; i < 12; i++)
+		tpo_td043_write(spi, 0x14 + i, gamma[i] & 0xff);
+}
+
+static int tpo_td043_write_mirror(struct spi_device *spi, bool h, bool v)
+{
+	u8 reg4 = TPO_R04_NFLIP_H | TPO_R04_NFLIP_V |
+		TPO_R04_CP_CLK_FREQ_1H | TPO_R04_VGL_FREQ_1H;
+	if (h)
+		reg4 &= ~TPO_R04_NFLIP_H;
+	if (v)
+		reg4 &= ~TPO_R04_NFLIP_V;
+
+	return tpo_td043_write(spi, 4, reg4);
+}
+
+static int tpo_td043_set_hmirror(struct omap_dss_device *dssdev, bool enable)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dssdev->dev);
+
+	ddata->hmirror = enable;
+	return tpo_td043_write_mirror(ddata->spi, ddata->hmirror,
+			ddata->vmirror);
+}
+
+static bool tpo_td043_get_hmirror(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dssdev->dev);
+
+	return ddata->hmirror;
+}
+
+static ssize_t tpo_td043_vmirror_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ddata->vmirror);
+}
+
+static ssize_t tpo_td043_vmirror_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	int val;
+	int ret;
+
+	ret = kstrtoint(buf, 0, &val);
+	if (ret < 0)
+		return ret;
+
+	val = !!val;
+
+	ret = tpo_td043_write_mirror(ddata->spi, ddata->hmirror, val);
+	if (ret < 0)
+		return ret;
+
+	ddata->vmirror = val;
+
+	return count;
+}
+
+static ssize_t tpo_td043_mode_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ddata->mode);
+}
+
+static ssize_t tpo_td043_mode_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	long val;
+	int ret;
+
+	ret = kstrtol(buf, 0, &val);
+	if (ret != 0 || val & ~7)
+		return -EINVAL;
+
+	ddata->mode = val;
+
+	val |= TPO_R02_NCLK_RISING;
+	tpo_td043_write(ddata->spi, 2, val);
+
+	return count;
+}
+
+static ssize_t tpo_td043_gamma_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	ssize_t len = 0;
+	int ret;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ddata->gamma); i++) {
+		ret = snprintf(buf + len, PAGE_SIZE - len, "%u ",
+				ddata->gamma[i]);
+		if (ret < 0)
+			return ret;
+		len += ret;
+	}
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static ssize_t tpo_td043_gamma_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	unsigned int g[12];
+	int ret;
+	int i;
+
+	ret = sscanf(buf, "%u %u %u %u %u %u %u %u %u %u %u %u",
+			&g[0], &g[1], &g[2], &g[3], &g[4], &g[5],
+			&g[6], &g[7], &g[8], &g[9], &g[10], &g[11]);
+
+	if (ret != 12)
+		return -EINVAL;
+
+	for (i = 0; i < 12; i++)
+		ddata->gamma[i] = g[i];
+
+	tpo_td043_write_gamma(ddata->spi, ddata->gamma);
+
+	return count;
+}
+
+static DEVICE_ATTR(vmirror, S_IRUGO | S_IWUSR,
+		tpo_td043_vmirror_show, tpo_td043_vmirror_store);
+static DEVICE_ATTR(mode, S_IRUGO | S_IWUSR,
+		tpo_td043_mode_show, tpo_td043_mode_store);
+static DEVICE_ATTR(gamma, S_IRUGO | S_IWUSR,
+		tpo_td043_gamma_show, tpo_td043_gamma_store);
+
+static struct attribute *tpo_td043_attrs[] = {
+	&dev_attr_vmirror.attr,
+	&dev_attr_mode.attr,
+	&dev_attr_gamma.attr,
+	NULL,
+};
+
+static struct attribute_group tpo_td043_attr_group = {
+	.attrs = tpo_td043_attrs,
+};
+
+static int tpo_td043_power_on(struct panel_drv_data *ddata)
+{
+	int r;
+
+	if (ddata->powered_on)
+		return 0;
+
+	r = regulator_enable(ddata->vcc_reg);
+	if (r != 0)
+		return r;
+
+	/* wait for panel to stabilize */
+	msleep(160);
+
+	if (gpio_is_valid(ddata->nreset_gpio))
+		gpio_set_value(ddata->nreset_gpio, 1);
+
+	tpo_td043_write(ddata->spi, 2,
+			TPO_R02_MODE(ddata->mode) | TPO_R02_NCLK_RISING);
+	tpo_td043_write(ddata->spi, 3, TPO_R03_VAL_NORMAL);
+	tpo_td043_write(ddata->spi, 0x20, 0xf0);
+	tpo_td043_write(ddata->spi, 0x21, 0xf0);
+	tpo_td043_write_mirror(ddata->spi, ddata->hmirror,
+			ddata->vmirror);
+	tpo_td043_write_gamma(ddata->spi, ddata->gamma);
+
+	ddata->powered_on = 1;
+	return 0;
+}
+
+static void tpo_td043_power_off(struct panel_drv_data *ddata)
+{
+	if (!ddata->powered_on)
+		return;
+
+	tpo_td043_write(ddata->spi, 3,
+			TPO_R03_VAL_STANDBY | TPO_R03_EN_PWM);
+
+	if (gpio_is_valid(ddata->nreset_gpio))
+		gpio_set_value(ddata->nreset_gpio, 0);
+
+	/* wait for at least 2 vsyncs before cutting off power */
+	msleep(50);
+
+	tpo_td043_write(ddata->spi, 3, TPO_R03_VAL_STANDBY);
+
+	regulator_disable(ddata->vcc_reg);
+
+	ddata->powered_on = 0;
+}
+
+static int tpo_td043_connect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (omapdss_device_is_connected(dssdev))
+		return 0;
+
+	r = in->ops.dpi->connect(in, dssdev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void tpo_td043_disconnect(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return;
+
+	in->ops.dpi->disconnect(in, dssdev);
+}
+
+static int tpo_td043_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+	int r;
+
+	if (!omapdss_device_is_connected(dssdev))
+		return -ENODEV;
+
+	if (omapdss_device_is_enabled(dssdev))
+		return 0;
+
+	if (ddata->data_lines)
+		in->ops.dpi->set_data_lines(in, ddata->data_lines);
+	in->ops.dpi->set_timings(in, &ddata->videomode);
+
+	r = in->ops.dpi->enable(in);
+	if (r)
+		return r;
+
+	/*
+	 * If we are resuming from system suspend, SPI clocks might not be
+	 * enabled yet, so we'll program the LCD from SPI PM resume callback.
+	 */
+	if (!ddata->spi_suspended) {
+		r = tpo_td043_power_on(ddata);
+		if (r) {
+			in->ops.dpi->disable(in);
+			return r;
+		}
+	}
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+
+	return 0;
+}
+
+static void tpo_td043_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (!omapdss_device_is_enabled(dssdev))
+		return;
+
+	in->ops.dpi->disable(in);
+
+	if (!ddata->spi_suspended)
+		tpo_td043_power_off(ddata);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+
+static void tpo_td043_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	ddata->videomode = *timings;
+	dssdev->panel.timings = *timings;
+
+	in->ops.dpi->set_timings(in, timings);
+}
+
+static void tpo_td043_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*timings = ddata->videomode;
+}
+
+static int tpo_td043_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	return in->ops.dpi->check_timings(in, timings);
+}
+
+static struct omap_dss_driver tpo_td043_ops = {
+	.connect	= tpo_td043_connect,
+	.disconnect	= tpo_td043_disconnect,
+
+	.enable		= tpo_td043_enable,
+	.disable	= tpo_td043_disable,
+
+	.set_timings	= tpo_td043_set_timings,
+	.get_timings	= tpo_td043_get_timings,
+	.check_timings	= tpo_td043_check_timings,
+
+	.set_mirror	= tpo_td043_set_hmirror,
+	.get_mirror	= tpo_td043_get_hmirror,
+
+	.get_resolution	= omapdss_default_get_resolution,
+};
+
+
+static int tpo_td043_probe_pdata(struct spi_device *spi)
+{
+	const struct panel_tpo_td043mtea1_platform_data *pdata;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev, *in;
+
+	pdata = dev_get_platdata(&spi->dev);
+
+	ddata->nreset_gpio = pdata->nreset_gpio;
+
+	in = omap_dss_find_output(pdata->source);
+	if (in == NULL) {
+		dev_err(&spi->dev, "failed to find video source '%s'\n",
+				pdata->source);
+		return -EPROBE_DEFER;
+	}
+	ddata->in = in;
+
+	ddata->data_lines = pdata->data_lines;
+
+	dssdev = &ddata->dssdev;
+	dssdev->name = pdata->name;
+
+	return 0;
+}
+
+static int tpo_td043_probe_of(struct spi_device *spi)
+{
+	struct device_node *node = spi->dev.of_node;
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *in;
+	int gpio;
+
+	gpio = of_get_named_gpio(node, "reset-gpios", 0);
+	if (!gpio_is_valid(gpio)) {
+		dev_err(&spi->dev, "failed to parse enable gpio\n");
+		return gpio;
+	}
+	ddata->nreset_gpio = gpio;
+
+	in = omapdss_of_find_source_for_first_ep(node);
+	if (IS_ERR(in)) {
+		dev_err(&spi->dev, "failed to find video source\n");
+		return PTR_ERR(in);
+	}
+
+	ddata->in = in;
+
+	return 0;
+}
+
+static int tpo_td043_probe(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata;
+	struct omap_dss_device *dssdev;
+	int r;
+
+	dev_dbg(&spi->dev, "%s\n", __func__);
+
+	spi->bits_per_word = 16;
+	spi->mode = SPI_MODE_0;
+
+	r = spi_setup(spi);
+	if (r < 0) {
+		dev_err(&spi->dev, "spi_setup failed: %d\n", r);
+		return r;
+	}
+
+	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
+	if (ddata == NULL)
+		return -ENOMEM;
+
+	dev_set_drvdata(&spi->dev, ddata);
+
+	ddata->spi = spi;
+
+	if (dev_get_platdata(&spi->dev)) {
+		r = tpo_td043_probe_pdata(spi);
+		if (r)
+			return r;
+	} else if (spi->dev.of_node) {
+		r = tpo_td043_probe_of(spi);
+		if (r)
+			return r;
+	} else {
+		return -ENODEV;
+	}
+
+	ddata->mode = TPO_R02_MODE_800x480;
+	memcpy(ddata->gamma, tpo_td043_def_gamma, sizeof(ddata->gamma));
+
+	ddata->vcc_reg = devm_regulator_get(&spi->dev, "vcc");
+	if (IS_ERR(ddata->vcc_reg)) {
+		dev_err(&spi->dev, "failed to get LCD VCC regulator\n");
+		r = PTR_ERR(ddata->vcc_reg);
+		goto err_regulator;
+	}
+
+	if (gpio_is_valid(ddata->nreset_gpio)) {
+		r = devm_gpio_request_one(&spi->dev,
+				ddata->nreset_gpio, GPIOF_OUT_INIT_LOW,
+				"lcd reset");
+		if (r < 0) {
+			dev_err(&spi->dev, "couldn't request reset GPIO\n");
+			goto err_gpio_req;
+		}
+	}
+
+	r = sysfs_create_group(&spi->dev.kobj, &tpo_td043_attr_group);
+	if (r) {
+		dev_err(&spi->dev, "failed to create sysfs files\n");
+		goto err_sysfs;
+	}
+
+	ddata->videomode = tpo_td043_timings;
+
+	dssdev = &ddata->dssdev;
+	dssdev->dev = &spi->dev;
+	dssdev->driver = &tpo_td043_ops;
+	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->owner = THIS_MODULE;
+	dssdev->panel.timings = ddata->videomode;
+
+	r = omapdss_register_display(dssdev);
+	if (r) {
+		dev_err(&spi->dev, "Failed to register panel\n");
+		goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	sysfs_remove_group(&spi->dev.kobj, &tpo_td043_attr_group);
+err_sysfs:
+err_gpio_req:
+err_regulator:
+	omap_dss_put_device(ddata->in);
+	return r;
+}
+
+static int tpo_td043_remove(struct spi_device *spi)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
+	struct omap_dss_device *dssdev = &ddata->dssdev;
+	struct omap_dss_device *in = ddata->in;
+
+	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
+
+	omapdss_unregister_display(dssdev);
+
+	tpo_td043_disable(dssdev);
+	tpo_td043_disconnect(dssdev);
+
+	omap_dss_put_device(in);
+
+	sysfs_remove_group(&spi->dev.kobj, &tpo_td043_attr_group);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int tpo_td043_spi_suspend(struct device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+
+	dev_dbg(dev, "tpo_td043_spi_suspend, tpo %p\n", ddata);
+
+	ddata->power_on_resume = ddata->powered_on;
+	tpo_td043_power_off(ddata);
+	ddata->spi_suspended = 1;
+
+	return 0;
+}
+
+static int tpo_td043_spi_resume(struct device *dev)
+{
+	struct panel_drv_data *ddata = dev_get_drvdata(dev);
+	int ret;
+
+	dev_dbg(dev, "tpo_td043_spi_resume\n");
+
+	if (ddata->power_on_resume) {
+		ret = tpo_td043_power_on(ddata);
+		if (ret)
+			return ret;
+	}
+	ddata->spi_suspended = 0;
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(tpo_td043_spi_pm,
+	tpo_td043_spi_suspend, tpo_td043_spi_resume);
+
+static const struct of_device_id tpo_td043_of_match[] = {
+	{ .compatible = "omapdss,tpo,td043mtea1", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, tpo_td043_of_match);
+
+static struct spi_driver tpo_td043_spi_driver = {
+	.driver = {
+		.name	= "panel-tpo-td043mtea1",
+		.pm	= &tpo_td043_spi_pm,
+		.of_match_table = tpo_td043_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe	= tpo_td043_probe,
+	.remove	= tpo_td043_remove,
+};
+
+module_spi_driver(tpo_td043_spi_driver);
+
+MODULE_ALIAS("spi:tpo,td043mtea1");
+MODULE_AUTHOR("Gražvydas Ignotas <notasas@gmail.com>");
+MODULE_DESCRIPTION("TPO TD043MTEA1 LCD Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/Kconfig b/drivers/video/fbdev/omap2/omapfb/dss/Kconfig
new file mode 100644
index 000000000000..27d220212870
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/Kconfig
@@ -0,0 +1,129 @@
+config FB_OMAP2_DSS_INIT
+	bool
+
+config FB_OMAP2_DSS
+        tristate
+	select VIDEOMODE_HELPERS
+	select FB_OMAP2_DSS_INIT
+	select HDMI
+
+config FB_OMAP2_DSS_DEBUG
+	bool "Debug support"
+	default n
+	help
+	  This enables printing of debug messages. Alternatively, debug messages
+	  can also be enabled by setting CONFIG_DYNAMIC_DEBUG and then setting
+	  appropriate flags in <debugfs>/dynamic_debug/control.
+
+config FB_OMAP2_DSS_DEBUGFS
+	bool "Debugfs filesystem support"
+	depends on DEBUG_FS
+	default n
+	help
+	  This enables debugfs for OMAPDSS at <debugfs>/omapdss. This enables
+	  querying about clock configuration and register configuration of dss,
+	  dispc, dsi, hdmi and rfbi.
+
+config FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	bool "Collect DSS IRQ statistics"
+	depends on FB_OMAP2_DSS_DEBUGFS
+	default n
+	help
+	  Collect DSS IRQ statistics, printable via debugfs.
+
+	  The statistics can be found from
+	  <debugfs>/omapdss/dispc_irq for DISPC interrupts, and
+	  <debugfs>/omapdss/dsi_irq for DSI interrupts.
+
+config FB_OMAP2_DSS_DPI
+	bool "DPI support"
+	default y
+	help
+	  DPI Interface. This is the Parallel Display Interface.
+
+config FB_OMAP2_DSS_RFBI
+	bool "RFBI support"
+	depends on BROKEN
+        default n
+	help
+	  MIPI DBI support (RFBI, Remote Framebuffer Interface, in Texas
+	  Instrument's terminology).
+
+	  DBI is a bus between the host processor and a peripheral,
+	  such as a display or a framebuffer chip.
+
+	  See http://www.mipi.org/ for DBI specifications.
+
+config FB_OMAP2_DSS_VENC
+	bool "VENC support"
+        default y
+	help
+	  OMAP Video Encoder support for S-Video and composite TV-out.
+
+config FB_OMAP2_DSS_HDMI_COMMON
+	bool
+
+config FB_OMAP4_DSS_HDMI
+	bool "HDMI support for OMAP4"
+        default y
+	select FB_OMAP2_DSS_HDMI_COMMON
+	help
+	  HDMI support for OMAP4 based SoCs.
+
+config FB_OMAP5_DSS_HDMI
+	bool "HDMI support for OMAP5"
+	default n
+	select FB_OMAP2_DSS_HDMI_COMMON
+	help
+	  HDMI Interface for OMAP5 and similar cores. This adds the High
+	  Definition Multimedia Interface. See http://www.hdmi.org/ for HDMI
+	  specification.
+
+config FB_OMAP2_DSS_SDI
+	bool "SDI support"
+        default n
+	help
+	  SDI (Serial Display Interface) support.
+
+	  SDI is a high speed one-way display serial bus between the host
+	  processor and a display.
+
+config FB_OMAP2_DSS_DSI
+	bool "DSI support"
+        default n
+	help
+	  MIPI DSI (Display Serial Interface) support.
+
+	  DSI is a high speed half-duplex serial interface between the host
+	  processor and a peripheral, such as a display or a framebuffer chip.
+
+	  See http://www.mipi.org/ for DSI specifications.
+
+config FB_OMAP2_DSS_MIN_FCK_PER_PCK
+	int "Minimum FCK/PCK ratio (for scaling)"
+	range 0 32
+	default 0
+	help
+	  This can be used to adjust the minimum FCK/PCK ratio.
+
+	  With this you can make sure that DISPC FCK is at least
+	  n x PCK. Video plane scaling requires higher FCK than
+	  normally.
+
+	  If this is set to 0, there's no extra constraint on the
+	  DISPC FCK. However, the FCK will at minimum be
+	  2xPCK (if active matrix) or 3xPCK (if passive matrix).
+
+	  Max FCK is 173MHz, so this doesn't work if your PCK
+	  is very high.
+
+config FB_OMAP2_DSS_SLEEP_AFTER_VENC_RESET
+	bool "Sleep 20ms after VENC reset"
+	default y
+	help
+	  There is a 20ms sleep after VENC reset which seemed to fix the
+	  reset. The reason for the bug is unclear, and it's also unclear
+	  on what platforms this happens.
+
+	  This option enables the sleep, and is enabled by default. You can
+	  disable the sleep if it doesn't cause problems on your platform.
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/Makefile b/drivers/video/fbdev/omap2/omapfb/dss/Makefile
new file mode 100644
index 000000000000..02308e24f3ef
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/Makefile
@@ -0,0 +1,18 @@
+obj-$(CONFIG_FB_OMAP2_DSS_INIT) += omapdss-boot-init.o
+obj-$(CONFIG_FB_OMAP2_DSS) += omapdss.o
+# Core DSS files
+omapdss-y := core.o dss.o dss_features.o dispc.o dispc_coefs.o display.o \
+	output.o dss-of.o pll.o video-pll.o
+# DSS compat layer files
+omapdss-y += manager.o manager-sysfs.o overlay.o overlay-sysfs.o apply.o \
+	dispc-compat.o display-sysfs.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_DPI) += dpi.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_RFBI) += rfbi.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_VENC) += venc.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_SDI) += sdi.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_DSI) += dsi.o
+omapdss-$(CONFIG_FB_OMAP2_DSS_HDMI_COMMON) += hdmi_common.o hdmi_wp.o hdmi_pll.o \
+	hdmi_phy.o
+omapdss-$(CONFIG_FB_OMAP4_DSS_HDMI) += hdmi4.o hdmi4_core.o
+omapdss-$(CONFIG_FB_OMAP5_DSS_HDMI) += hdmi5.o hdmi5_core.o
+ccflags-$(CONFIG_FB_OMAP2_DSS_DEBUG) += -DDEBUG
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/apply.c b/drivers/video/fbdev/omap2/omapfb/dss/apply.c
new file mode 100644
index 000000000000..663ccc3bf4e5
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/apply.c
@@ -0,0 +1,1702 @@
+/*
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "APPLY"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/jiffies.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+#include "dispc-compat.h"
+
+/*
+ * We have 4 levels of cache for the dispc settings. First two are in SW and
+ * the latter two in HW.
+ *
+ *       set_info()
+ *          v
+ * +--------------------+
+ * |     user_info      |
+ * +--------------------+
+ *          v
+ *        apply()
+ *          v
+ * +--------------------+
+ * |       info         |
+ * +--------------------+
+ *          v
+ *      write_regs()
+ *          v
+ * +--------------------+
+ * |  shadow registers  |
+ * +--------------------+
+ *          v
+ * VFP or lcd/digit_enable
+ *          v
+ * +--------------------+
+ * |      registers     |
+ * +--------------------+
+ */
+
+struct ovl_priv_data {
+
+	bool user_info_dirty;
+	struct omap_overlay_info user_info;
+
+	bool info_dirty;
+	struct omap_overlay_info info;
+
+	bool shadow_info_dirty;
+
+	bool extra_info_dirty;
+	bool shadow_extra_info_dirty;
+
+	bool enabled;
+	u32 fifo_low, fifo_high;
+
+	/*
+	 * True if overlay is to be enabled. Used to check and calculate configs
+	 * for the overlay before it is enabled in the HW.
+	 */
+	bool enabling;
+};
+
+struct mgr_priv_data {
+
+	bool user_info_dirty;
+	struct omap_overlay_manager_info user_info;
+
+	bool info_dirty;
+	struct omap_overlay_manager_info info;
+
+	bool shadow_info_dirty;
+
+	/* If true, GO bit is up and shadow registers cannot be written.
+	 * Never true for manual update displays */
+	bool busy;
+
+	/* If true, dispc output is enabled */
+	bool updating;
+
+	/* If true, a display is enabled using this manager */
+	bool enabled;
+
+	bool extra_info_dirty;
+	bool shadow_extra_info_dirty;
+
+	struct omap_video_timings timings;
+	struct dss_lcd_mgr_config lcd_config;
+
+	void (*framedone_handler)(void *);
+	void *framedone_handler_data;
+};
+
+static struct {
+	struct ovl_priv_data ovl_priv_data_array[MAX_DSS_OVERLAYS];
+	struct mgr_priv_data mgr_priv_data_array[MAX_DSS_MANAGERS];
+
+	bool irq_enabled;
+} dss_data;
+
+/* protects dss_data */
+static spinlock_t data_lock;
+/* lock for blocking functions */
+static DEFINE_MUTEX(apply_lock);
+static DECLARE_COMPLETION(extra_updated_completion);
+
+static void dss_register_vsync_isr(void);
+
+static struct ovl_priv_data *get_ovl_priv(struct omap_overlay *ovl)
+{
+	return &dss_data.ovl_priv_data_array[ovl->id];
+}
+
+static struct mgr_priv_data *get_mgr_priv(struct omap_overlay_manager *mgr)
+{
+	return &dss_data.mgr_priv_data_array[mgr->id];
+}
+
+static void apply_init_priv(void)
+{
+	const int num_ovls = dss_feat_get_num_ovls();
+	struct mgr_priv_data *mp;
+	int i;
+
+	spin_lock_init(&data_lock);
+
+	for (i = 0; i < num_ovls; ++i) {
+		struct ovl_priv_data *op;
+
+		op = &dss_data.ovl_priv_data_array[i];
+
+		op->info.color_mode = OMAP_DSS_COLOR_RGB16;
+		op->info.rotation_type = OMAP_DSS_ROT_DMA;
+
+		op->info.global_alpha = 255;
+
+		switch (i) {
+		case 0:
+			op->info.zorder = 0;
+			break;
+		case 1:
+			op->info.zorder =
+				dss_has_feature(FEAT_ALPHA_FREE_ZORDER) ? 3 : 0;
+			break;
+		case 2:
+			op->info.zorder =
+				dss_has_feature(FEAT_ALPHA_FREE_ZORDER) ? 2 : 0;
+			break;
+		case 3:
+			op->info.zorder =
+				dss_has_feature(FEAT_ALPHA_FREE_ZORDER) ? 1 : 0;
+			break;
+		}
+
+		op->user_info = op->info;
+	}
+
+	/*
+	 * Initialize some of the lcd_config fields for TV manager, this lets
+	 * us prevent checking if the manager is LCD or TV at some places
+	 */
+	mp = &dss_data.mgr_priv_data_array[OMAP_DSS_CHANNEL_DIGIT];
+
+	mp->lcd_config.video_port_width = 24;
+	mp->lcd_config.clock_info.lck_div = 1;
+	mp->lcd_config.clock_info.pck_div = 1;
+}
+
+/*
+ * A LCD manager's stallmode decides whether it is in manual or auto update. TV
+ * manager is always auto update, stallmode field for TV manager is false by
+ * default
+ */
+static bool ovl_manual_update(struct omap_overlay *ovl)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(ovl->manager);
+
+	return mp->lcd_config.stallmode;
+}
+
+static bool mgr_manual_update(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	return mp->lcd_config.stallmode;
+}
+
+static int dss_check_settings_low(struct omap_overlay_manager *mgr,
+		bool applying)
+{
+	struct omap_overlay_info *oi;
+	struct omap_overlay_manager_info *mi;
+	struct omap_overlay *ovl;
+	struct omap_overlay_info *ois[MAX_DSS_OVERLAYS];
+	struct ovl_priv_data *op;
+	struct mgr_priv_data *mp;
+
+	mp = get_mgr_priv(mgr);
+
+	if (!mp->enabled)
+		return 0;
+
+	if (applying && mp->user_info_dirty)
+		mi = &mp->user_info;
+	else
+		mi = &mp->info;
+
+	/* collect the infos to be tested into the array */
+	list_for_each_entry(ovl, &mgr->overlays, list) {
+		op = get_ovl_priv(ovl);
+
+		if (!op->enabled && !op->enabling)
+			oi = NULL;
+		else if (applying && op->user_info_dirty)
+			oi = &op->user_info;
+		else
+			oi = &op->info;
+
+		ois[ovl->id] = oi;
+	}
+
+	return dss_mgr_check(mgr, mi, &mp->timings, &mp->lcd_config, ois);
+}
+
+/*
+ * check manager and overlay settings using overlay_info from data->info
+ */
+static int dss_check_settings(struct omap_overlay_manager *mgr)
+{
+	return dss_check_settings_low(mgr, false);
+}
+
+/*
+ * check manager and overlay settings using overlay_info from ovl->info if
+ * dirty and from data->info otherwise
+ */
+static int dss_check_settings_apply(struct omap_overlay_manager *mgr)
+{
+	return dss_check_settings_low(mgr, true);
+}
+
+static bool need_isr(void)
+{
+	const int num_mgrs = dss_feat_get_num_mgrs();
+	int i;
+
+	for (i = 0; i < num_mgrs; ++i) {
+		struct omap_overlay_manager *mgr;
+		struct mgr_priv_data *mp;
+		struct omap_overlay *ovl;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mp->enabled)
+			continue;
+
+		if (mgr_manual_update(mgr)) {
+			/* to catch FRAMEDONE */
+			if (mp->updating)
+				return true;
+		} else {
+			/* to catch GO bit going down */
+			if (mp->busy)
+				return true;
+
+			/* to write new values to registers */
+			if (mp->info_dirty)
+				return true;
+
+			/* to set GO bit */
+			if (mp->shadow_info_dirty)
+				return true;
+
+			/*
+			 * NOTE: we don't check extra_info flags for disabled
+			 * managers, once the manager is enabled, the extra_info
+			 * related manager changes will be taken in by HW.
+			 */
+
+			/* to write new values to registers */
+			if (mp->extra_info_dirty)
+				return true;
+
+			/* to set GO bit */
+			if (mp->shadow_extra_info_dirty)
+				return true;
+
+			list_for_each_entry(ovl, &mgr->overlays, list) {
+				struct ovl_priv_data *op;
+
+				op = get_ovl_priv(ovl);
+
+				/*
+				 * NOTE: we check extra_info flags even for
+				 * disabled overlays, as extra_infos need to be
+				 * always written.
+				 */
+
+				/* to write new values to registers */
+				if (op->extra_info_dirty)
+					return true;
+
+				/* to set GO bit */
+				if (op->shadow_extra_info_dirty)
+					return true;
+
+				if (!op->enabled)
+					continue;
+
+				/* to write new values to registers */
+				if (op->info_dirty)
+					return true;
+
+				/* to set GO bit */
+				if (op->shadow_info_dirty)
+					return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+static bool need_go(struct omap_overlay_manager *mgr)
+{
+	struct omap_overlay *ovl;
+	struct mgr_priv_data *mp;
+	struct ovl_priv_data *op;
+
+	mp = get_mgr_priv(mgr);
+
+	if (mp->shadow_info_dirty || mp->shadow_extra_info_dirty)
+		return true;
+
+	list_for_each_entry(ovl, &mgr->overlays, list) {
+		op = get_ovl_priv(ovl);
+		if (op->shadow_info_dirty || op->shadow_extra_info_dirty)
+			return true;
+	}
+
+	return false;
+}
+
+/* returns true if an extra_info field is currently being updated */
+static bool extra_info_update_ongoing(void)
+{
+	const int num_mgrs = dss_feat_get_num_mgrs();
+	int i;
+
+	for (i = 0; i < num_mgrs; ++i) {
+		struct omap_overlay_manager *mgr;
+		struct omap_overlay *ovl;
+		struct mgr_priv_data *mp;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mp->enabled)
+			continue;
+
+		if (!mp->updating)
+			continue;
+
+		if (mp->extra_info_dirty || mp->shadow_extra_info_dirty)
+			return true;
+
+		list_for_each_entry(ovl, &mgr->overlays, list) {
+			struct ovl_priv_data *op = get_ovl_priv(ovl);
+
+			if (op->extra_info_dirty || op->shadow_extra_info_dirty)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+/* wait until no extra_info updates are pending */
+static void wait_pending_extra_info_updates(void)
+{
+	bool updating;
+	unsigned long flags;
+	unsigned long t;
+	int r;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	updating = extra_info_update_ongoing();
+
+	if (!updating) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		return;
+	}
+
+	init_completion(&extra_updated_completion);
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	t = msecs_to_jiffies(500);
+	r = wait_for_completion_timeout(&extra_updated_completion, t);
+	if (r == 0)
+		DSSWARN("timeout in wait_pending_extra_info_updates\n");
+}
+
+static struct omap_dss_device *dss_mgr_get_device(struct omap_overlay_manager *mgr)
+{
+	struct omap_dss_device *dssdev;
+
+	dssdev = mgr->output;
+	if (dssdev == NULL)
+		return NULL;
+
+	while (dssdev->dst)
+		dssdev = dssdev->dst;
+
+	if (dssdev->driver)
+		return dssdev;
+	else
+		return NULL;
+}
+
+static struct omap_dss_device *dss_ovl_get_device(struct omap_overlay *ovl)
+{
+	return ovl->manager ? dss_mgr_get_device(ovl->manager) : NULL;
+}
+
+static int dss_mgr_wait_for_vsync(struct omap_overlay_manager *mgr)
+{
+	unsigned long timeout = msecs_to_jiffies(500);
+	u32 irq;
+	int r;
+
+	if (mgr->output == NULL)
+		return -ENODEV;
+
+	r = dispc_runtime_get();
+	if (r)
+		return r;
+
+	switch (mgr->output->id) {
+	case OMAP_DSS_OUTPUT_VENC:
+		irq = DISPC_IRQ_EVSYNC_ODD;
+		break;
+	case OMAP_DSS_OUTPUT_HDMI:
+		irq = DISPC_IRQ_EVSYNC_EVEN;
+		break;
+	default:
+		irq = dispc_mgr_get_vsync_irq(mgr->id);
+		break;
+	}
+
+	r = omap_dispc_wait_for_irq_interruptible_timeout(irq, timeout);
+
+	dispc_runtime_put();
+
+	return r;
+}
+
+static int dss_mgr_wait_for_go(struct omap_overlay_manager *mgr)
+{
+	unsigned long timeout = msecs_to_jiffies(500);
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	u32 irq;
+	unsigned long flags;
+	int r;
+	int i;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (mgr_manual_update(mgr)) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		return 0;
+	}
+
+	if (!mp->enabled) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		return 0;
+	}
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	r = dispc_runtime_get();
+	if (r)
+		return r;
+
+	irq = dispc_mgr_get_vsync_irq(mgr->id);
+
+	i = 0;
+	while (1) {
+		bool shadow_dirty, dirty;
+
+		spin_lock_irqsave(&data_lock, flags);
+		dirty = mp->info_dirty;
+		shadow_dirty = mp->shadow_info_dirty;
+		spin_unlock_irqrestore(&data_lock, flags);
+
+		if (!dirty && !shadow_dirty) {
+			r = 0;
+			break;
+		}
+
+		/* 4 iterations is the worst case:
+		 * 1 - initial iteration, dirty = true (between VFP and VSYNC)
+		 * 2 - first VSYNC, dirty = true
+		 * 3 - dirty = false, shadow_dirty = true
+		 * 4 - shadow_dirty = false */
+		if (i++ == 3) {
+			DSSERR("mgr(%d)->wait_for_go() not finishing\n",
+					mgr->id);
+			r = 0;
+			break;
+		}
+
+		r = omap_dispc_wait_for_irq_interruptible_timeout(irq, timeout);
+		if (r == -ERESTARTSYS)
+			break;
+
+		if (r) {
+			DSSERR("mgr(%d)->wait_for_go() timeout\n", mgr->id);
+			break;
+		}
+	}
+
+	dispc_runtime_put();
+
+	return r;
+}
+
+static int dss_mgr_wait_for_go_ovl(struct omap_overlay *ovl)
+{
+	unsigned long timeout = msecs_to_jiffies(500);
+	struct ovl_priv_data *op;
+	struct mgr_priv_data *mp;
+	u32 irq;
+	unsigned long flags;
+	int r;
+	int i;
+
+	if (!ovl->manager)
+		return 0;
+
+	mp = get_mgr_priv(ovl->manager);
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (ovl_manual_update(ovl)) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		return 0;
+	}
+
+	if (!mp->enabled) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		return 0;
+	}
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	r = dispc_runtime_get();
+	if (r)
+		return r;
+
+	irq = dispc_mgr_get_vsync_irq(ovl->manager->id);
+
+	op = get_ovl_priv(ovl);
+	i = 0;
+	while (1) {
+		bool shadow_dirty, dirty;
+
+		spin_lock_irqsave(&data_lock, flags);
+		dirty = op->info_dirty;
+		shadow_dirty = op->shadow_info_dirty;
+		spin_unlock_irqrestore(&data_lock, flags);
+
+		if (!dirty && !shadow_dirty) {
+			r = 0;
+			break;
+		}
+
+		/* 4 iterations is the worst case:
+		 * 1 - initial iteration, dirty = true (between VFP and VSYNC)
+		 * 2 - first VSYNC, dirty = true
+		 * 3 - dirty = false, shadow_dirty = true
+		 * 4 - shadow_dirty = false */
+		if (i++ == 3) {
+			DSSERR("ovl(%d)->wait_for_go() not finishing\n",
+					ovl->id);
+			r = 0;
+			break;
+		}
+
+		r = omap_dispc_wait_for_irq_interruptible_timeout(irq, timeout);
+		if (r == -ERESTARTSYS)
+			break;
+
+		if (r) {
+			DSSERR("ovl(%d)->wait_for_go() timeout\n", ovl->id);
+			break;
+		}
+	}
+
+	dispc_runtime_put();
+
+	return r;
+}
+
+static void dss_ovl_write_regs(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	struct omap_overlay_info *oi;
+	bool replication;
+	struct mgr_priv_data *mp;
+	int r;
+
+	DSSDBG("writing ovl %d regs\n", ovl->id);
+
+	if (!op->enabled || !op->info_dirty)
+		return;
+
+	oi = &op->info;
+
+	mp = get_mgr_priv(ovl->manager);
+
+	replication = dss_ovl_use_replication(mp->lcd_config, oi->color_mode);
+
+	r = dispc_ovl_setup(ovl->id, oi, replication, &mp->timings, false);
+	if (r) {
+		/*
+		 * We can't do much here, as this function can be called from
+		 * vsync interrupt.
+		 */
+		DSSERR("dispc_ovl_setup failed for ovl %d\n", ovl->id);
+
+		/* This will leave fifo configurations in a nonoptimal state */
+		op->enabled = false;
+		dispc_ovl_enable(ovl->id, false);
+		return;
+	}
+
+	op->info_dirty = false;
+	if (mp->updating)
+		op->shadow_info_dirty = true;
+}
+
+static void dss_ovl_write_regs_extra(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	struct mgr_priv_data *mp;
+
+	DSSDBG("writing ovl %d regs extra\n", ovl->id);
+
+	if (!op->extra_info_dirty)
+		return;
+
+	/* note: write also when op->enabled == false, so that the ovl gets
+	 * disabled */
+
+	dispc_ovl_enable(ovl->id, op->enabled);
+	dispc_ovl_set_fifo_threshold(ovl->id, op->fifo_low, op->fifo_high);
+
+	mp = get_mgr_priv(ovl->manager);
+
+	op->extra_info_dirty = false;
+	if (mp->updating)
+		op->shadow_extra_info_dirty = true;
+}
+
+static void dss_mgr_write_regs(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	struct omap_overlay *ovl;
+
+	DSSDBG("writing mgr %d regs\n", mgr->id);
+
+	if (!mp->enabled)
+		return;
+
+	WARN_ON(mp->busy);
+
+	/* Commit overlay settings */
+	list_for_each_entry(ovl, &mgr->overlays, list) {
+		dss_ovl_write_regs(ovl);
+		dss_ovl_write_regs_extra(ovl);
+	}
+
+	if (mp->info_dirty) {
+		dispc_mgr_setup(mgr->id, &mp->info);
+
+		mp->info_dirty = false;
+		if (mp->updating)
+			mp->shadow_info_dirty = true;
+	}
+}
+
+static void dss_mgr_write_regs_extra(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	DSSDBG("writing mgr %d regs extra\n", mgr->id);
+
+	if (!mp->extra_info_dirty)
+		return;
+
+	dispc_mgr_set_timings(mgr->id, &mp->timings);
+
+	/* lcd_config parameters */
+	if (dss_mgr_is_lcd(mgr->id))
+		dispc_mgr_set_lcd_config(mgr->id, &mp->lcd_config);
+
+	mp->extra_info_dirty = false;
+	if (mp->updating)
+		mp->shadow_extra_info_dirty = true;
+}
+
+static void dss_write_regs(void)
+{
+	const int num_mgrs = omap_dss_get_num_overlay_managers();
+	int i;
+
+	for (i = 0; i < num_mgrs; ++i) {
+		struct omap_overlay_manager *mgr;
+		struct mgr_priv_data *mp;
+		int r;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mp->enabled || mgr_manual_update(mgr) || mp->busy)
+			continue;
+
+		r = dss_check_settings(mgr);
+		if (r) {
+			DSSERR("cannot write registers for manager %s: "
+					"illegal configuration\n", mgr->name);
+			continue;
+		}
+
+		dss_mgr_write_regs(mgr);
+		dss_mgr_write_regs_extra(mgr);
+	}
+}
+
+static void dss_set_go_bits(void)
+{
+	const int num_mgrs = omap_dss_get_num_overlay_managers();
+	int i;
+
+	for (i = 0; i < num_mgrs; ++i) {
+		struct omap_overlay_manager *mgr;
+		struct mgr_priv_data *mp;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mp->enabled || mgr_manual_update(mgr) || mp->busy)
+			continue;
+
+		if (!need_go(mgr))
+			continue;
+
+		mp->busy = true;
+
+		if (!dss_data.irq_enabled && need_isr())
+			dss_register_vsync_isr();
+
+		dispc_mgr_go(mgr->id);
+	}
+
+}
+
+static void mgr_clear_shadow_dirty(struct omap_overlay_manager *mgr)
+{
+	struct omap_overlay *ovl;
+	struct mgr_priv_data *mp;
+	struct ovl_priv_data *op;
+
+	mp = get_mgr_priv(mgr);
+	mp->shadow_info_dirty = false;
+	mp->shadow_extra_info_dirty = false;
+
+	list_for_each_entry(ovl, &mgr->overlays, list) {
+		op = get_ovl_priv(ovl);
+		op->shadow_info_dirty = false;
+		op->shadow_extra_info_dirty = false;
+	}
+}
+
+static int dss_mgr_connect_compat(struct omap_overlay_manager *mgr,
+		struct omap_dss_device *dst)
+{
+	return mgr->set_output(mgr, dst);
+}
+
+static void dss_mgr_disconnect_compat(struct omap_overlay_manager *mgr,
+		struct omap_dss_device *dst)
+{
+	mgr->unset_output(mgr);
+}
+
+static void dss_mgr_start_update_compat(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	WARN_ON(mp->updating);
+
+	r = dss_check_settings(mgr);
+	if (r) {
+		DSSERR("cannot start manual update: illegal configuration\n");
+		spin_unlock_irqrestore(&data_lock, flags);
+		return;
+	}
+
+	dss_mgr_write_regs(mgr);
+	dss_mgr_write_regs_extra(mgr);
+
+	mp->updating = true;
+
+	if (!dss_data.irq_enabled && need_isr())
+		dss_register_vsync_isr();
+
+	dispc_mgr_enable_sync(mgr->id);
+
+	spin_unlock_irqrestore(&data_lock, flags);
+}
+
+static void dss_apply_irq_handler(void *data, u32 mask);
+
+static void dss_register_vsync_isr(void)
+{
+	const int num_mgrs = dss_feat_get_num_mgrs();
+	u32 mask;
+	int r, i;
+
+	mask = 0;
+	for (i = 0; i < num_mgrs; ++i)
+		mask |= dispc_mgr_get_vsync_irq(i);
+
+	for (i = 0; i < num_mgrs; ++i)
+		mask |= dispc_mgr_get_framedone_irq(i);
+
+	r = omap_dispc_register_isr(dss_apply_irq_handler, NULL, mask);
+	WARN_ON(r);
+
+	dss_data.irq_enabled = true;
+}
+
+static void dss_unregister_vsync_isr(void)
+{
+	const int num_mgrs = dss_feat_get_num_mgrs();
+	u32 mask;
+	int r, i;
+
+	mask = 0;
+	for (i = 0; i < num_mgrs; ++i)
+		mask |= dispc_mgr_get_vsync_irq(i);
+
+	for (i = 0; i < num_mgrs; ++i)
+		mask |= dispc_mgr_get_framedone_irq(i);
+
+	r = omap_dispc_unregister_isr(dss_apply_irq_handler, NULL, mask);
+	WARN_ON(r);
+
+	dss_data.irq_enabled = false;
+}
+
+static void dss_apply_irq_handler(void *data, u32 mask)
+{
+	const int num_mgrs = dss_feat_get_num_mgrs();
+	int i;
+	bool extra_updating;
+
+	spin_lock(&data_lock);
+
+	/* clear busy, updating flags, shadow_dirty flags */
+	for (i = 0; i < num_mgrs; i++) {
+		struct omap_overlay_manager *mgr;
+		struct mgr_priv_data *mp;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mp->enabled)
+			continue;
+
+		mp->updating = dispc_mgr_is_enabled(i);
+
+		if (!mgr_manual_update(mgr)) {
+			bool was_busy = mp->busy;
+			mp->busy = dispc_mgr_go_busy(i);
+
+			if (was_busy && !mp->busy)
+				mgr_clear_shadow_dirty(mgr);
+		}
+	}
+
+	dss_write_regs();
+	dss_set_go_bits();
+
+	extra_updating = extra_info_update_ongoing();
+	if (!extra_updating)
+		complete_all(&extra_updated_completion);
+
+	/* call framedone handlers for manual update displays */
+	for (i = 0; i < num_mgrs; i++) {
+		struct omap_overlay_manager *mgr;
+		struct mgr_priv_data *mp;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		mp = get_mgr_priv(mgr);
+
+		if (!mgr_manual_update(mgr) || !mp->framedone_handler)
+			continue;
+
+		if (mask & dispc_mgr_get_framedone_irq(i))
+			mp->framedone_handler(mp->framedone_handler_data);
+	}
+
+	if (!need_isr())
+		dss_unregister_vsync_isr();
+
+	spin_unlock(&data_lock);
+}
+
+static void omap_dss_mgr_apply_ovl(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op;
+
+	op = get_ovl_priv(ovl);
+
+	if (!op->user_info_dirty)
+		return;
+
+	op->user_info_dirty = false;
+	op->info_dirty = true;
+	op->info = op->user_info;
+}
+
+static void omap_dss_mgr_apply_mgr(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp;
+
+	mp = get_mgr_priv(mgr);
+
+	if (!mp->user_info_dirty)
+		return;
+
+	mp->user_info_dirty = false;
+	mp->info_dirty = true;
+	mp->info = mp->user_info;
+}
+
+static int omap_dss_mgr_apply(struct omap_overlay_manager *mgr)
+{
+	unsigned long flags;
+	struct omap_overlay *ovl;
+	int r;
+
+	DSSDBG("omap_dss_mgr_apply(%s)\n", mgr->name);
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	r = dss_check_settings_apply(mgr);
+	if (r) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		DSSERR("failed to apply settings: illegal configuration.\n");
+		return r;
+	}
+
+	/* Configure overlays */
+	list_for_each_entry(ovl, &mgr->overlays, list)
+		omap_dss_mgr_apply_ovl(ovl);
+
+	/* Configure manager */
+	omap_dss_mgr_apply_mgr(mgr);
+
+	dss_write_regs();
+	dss_set_go_bits();
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	return 0;
+}
+
+static void dss_apply_ovl_enable(struct omap_overlay *ovl, bool enable)
+{
+	struct ovl_priv_data *op;
+
+	op = get_ovl_priv(ovl);
+
+	if (op->enabled == enable)
+		return;
+
+	op->enabled = enable;
+	op->extra_info_dirty = true;
+}
+
+static void dss_apply_ovl_fifo_thresholds(struct omap_overlay *ovl,
+		u32 fifo_low, u32 fifo_high)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+
+	if (op->fifo_low == fifo_low && op->fifo_high == fifo_high)
+		return;
+
+	op->fifo_low = fifo_low;
+	op->fifo_high = fifo_high;
+	op->extra_info_dirty = true;
+}
+
+static void dss_ovl_setup_fifo(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	u32 fifo_low, fifo_high;
+	bool use_fifo_merge = false;
+
+	if (!op->enabled && !op->enabling)
+		return;
+
+	dispc_ovl_compute_fifo_thresholds(ovl->id, &fifo_low, &fifo_high,
+			use_fifo_merge, ovl_manual_update(ovl));
+
+	dss_apply_ovl_fifo_thresholds(ovl, fifo_low, fifo_high);
+}
+
+static void dss_mgr_setup_fifos(struct omap_overlay_manager *mgr)
+{
+	struct omap_overlay *ovl;
+	struct mgr_priv_data *mp;
+
+	mp = get_mgr_priv(mgr);
+
+	if (!mp->enabled)
+		return;
+
+	list_for_each_entry(ovl, &mgr->overlays, list)
+		dss_ovl_setup_fifo(ovl);
+}
+
+static void dss_setup_fifos(void)
+{
+	const int num_mgrs = omap_dss_get_num_overlay_managers();
+	struct omap_overlay_manager *mgr;
+	int i;
+
+	for (i = 0; i < num_mgrs; ++i) {
+		mgr = omap_dss_get_overlay_manager(i);
+		dss_mgr_setup_fifos(mgr);
+	}
+}
+
+static int dss_mgr_enable_compat(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+	int r;
+
+	mutex_lock(&apply_lock);
+
+	if (mp->enabled)
+		goto out;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	mp->enabled = true;
+
+	r = dss_check_settings(mgr);
+	if (r) {
+		DSSERR("failed to enable manager %d: check_settings failed\n",
+				mgr->id);
+		goto err;
+	}
+
+	dss_setup_fifos();
+
+	dss_write_regs();
+	dss_set_go_bits();
+
+	if (!mgr_manual_update(mgr))
+		mp->updating = true;
+
+	if (!dss_data.irq_enabled && need_isr())
+		dss_register_vsync_isr();
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	if (!mgr_manual_update(mgr))
+		dispc_mgr_enable_sync(mgr->id);
+
+out:
+	mutex_unlock(&apply_lock);
+
+	return 0;
+
+err:
+	mp->enabled = false;
+	spin_unlock_irqrestore(&data_lock, flags);
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static void dss_mgr_disable_compat(struct omap_overlay_manager *mgr)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+
+	mutex_lock(&apply_lock);
+
+	if (!mp->enabled)
+		goto out;
+
+	wait_pending_extra_info_updates();
+
+	if (!mgr_manual_update(mgr))
+		dispc_mgr_disable_sync(mgr->id);
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	mp->updating = false;
+	mp->enabled = false;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+out:
+	mutex_unlock(&apply_lock);
+}
+
+static int dss_mgr_set_info(struct omap_overlay_manager *mgr,
+		struct omap_overlay_manager_info *info)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+	int r;
+
+	r = dss_mgr_simple_check(mgr, info);
+	if (r)
+		return r;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	mp->user_info = *info;
+	mp->user_info_dirty = true;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	return 0;
+}
+
+static void dss_mgr_get_info(struct omap_overlay_manager *mgr,
+		struct omap_overlay_manager_info *info)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	*info = mp->user_info;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+}
+
+static int dss_mgr_set_output(struct omap_overlay_manager *mgr,
+		struct omap_dss_device *output)
+{
+	int r;
+
+	mutex_lock(&apply_lock);
+
+	if (mgr->output) {
+		DSSERR("manager %s is already connected to an output\n",
+			mgr->name);
+		r = -EINVAL;
+		goto err;
+	}
+
+	if ((mgr->supported_outputs & output->id) == 0) {
+		DSSERR("output does not support manager %s\n",
+			mgr->name);
+		r = -EINVAL;
+		goto err;
+	}
+
+	output->manager = mgr;
+	mgr->output = output;
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+err:
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static int dss_mgr_unset_output(struct omap_overlay_manager *mgr)
+{
+	int r;
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+	unsigned long flags;
+
+	mutex_lock(&apply_lock);
+
+	if (!mgr->output) {
+		DSSERR("failed to unset output, output not set\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (mp->enabled) {
+		DSSERR("output can't be unset when manager is enabled\n");
+		r = -EINVAL;
+		goto err1;
+	}
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	mgr->output->manager = NULL;
+	mgr->output = NULL;
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+err1:
+	spin_unlock_irqrestore(&data_lock, flags);
+err:
+	mutex_unlock(&apply_lock);
+
+	return r;
+}
+
+static void dss_apply_mgr_timings(struct omap_overlay_manager *mgr,
+		const struct omap_video_timings *timings)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	mp->timings = *timings;
+	mp->extra_info_dirty = true;
+}
+
+static void dss_mgr_set_timings_compat(struct omap_overlay_manager *mgr,
+		const struct omap_video_timings *timings)
+{
+	unsigned long flags;
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (mp->updating) {
+		DSSERR("cannot set timings for %s: manager needs to be disabled\n",
+			mgr->name);
+		goto out;
+	}
+
+	dss_apply_mgr_timings(mgr, timings);
+out:
+	spin_unlock_irqrestore(&data_lock, flags);
+}
+
+static void dss_apply_mgr_lcd_config(struct omap_overlay_manager *mgr,
+		const struct dss_lcd_mgr_config *config)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	mp->lcd_config = *config;
+	mp->extra_info_dirty = true;
+}
+
+static void dss_mgr_set_lcd_config_compat(struct omap_overlay_manager *mgr,
+		const struct dss_lcd_mgr_config *config)
+{
+	unsigned long flags;
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (mp->enabled) {
+		DSSERR("cannot apply lcd config for %s: manager needs to be disabled\n",
+			mgr->name);
+		goto out;
+	}
+
+	dss_apply_mgr_lcd_config(mgr, config);
+out:
+	spin_unlock_irqrestore(&data_lock, flags);
+}
+
+static int dss_ovl_set_info(struct omap_overlay *ovl,
+		struct omap_overlay_info *info)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	int r;
+
+	r = dss_ovl_simple_check(ovl, info);
+	if (r)
+		return r;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	op->user_info = *info;
+	op->user_info_dirty = true;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	return 0;
+}
+
+static void dss_ovl_get_info(struct omap_overlay *ovl,
+		struct omap_overlay_info *info)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	*info = op->user_info;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+}
+
+static int dss_ovl_set_manager(struct omap_overlay *ovl,
+		struct omap_overlay_manager *mgr)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	int r;
+
+	if (!mgr)
+		return -EINVAL;
+
+	mutex_lock(&apply_lock);
+
+	if (ovl->manager) {
+		DSSERR("overlay '%s' already has a manager '%s'\n",
+				ovl->name, ovl->manager->name);
+		r = -EINVAL;
+		goto err;
+	}
+
+	r = dispc_runtime_get();
+	if (r)
+		goto err;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (op->enabled) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		DSSERR("overlay has to be disabled to change the manager\n");
+		r = -EINVAL;
+		goto err1;
+	}
+
+	dispc_ovl_set_channel_out(ovl->id, mgr->id);
+
+	ovl->manager = mgr;
+	list_add_tail(&ovl->list, &mgr->overlays);
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	dispc_runtime_put();
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+
+err1:
+	dispc_runtime_put();
+err:
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static int dss_ovl_unset_manager(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	int r;
+
+	mutex_lock(&apply_lock);
+
+	if (!ovl->manager) {
+		DSSERR("failed to detach overlay: manager not set\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (op->enabled) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		DSSERR("overlay has to be disabled to unset the manager\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	/* wait for pending extra_info updates to ensure the ovl is disabled */
+	wait_pending_extra_info_updates();
+
+	/*
+	 * For a manual update display, there is no guarantee that the overlay
+	 * is really disabled in HW, we may need an extra update from this
+	 * manager before the configurations can go in. Return an error if the
+	 * overlay needed an update from the manager.
+	 *
+	 * TODO: Instead of returning an error, try to do a dummy manager update
+	 * here to disable the overlay in hardware. Use the *GATED fields in
+	 * the DISPC_CONFIG registers to do a dummy update.
+	 */
+	spin_lock_irqsave(&data_lock, flags);
+
+	if (ovl_manual_update(ovl) && op->extra_info_dirty) {
+		spin_unlock_irqrestore(&data_lock, flags);
+		DSSERR("need an update to change the manager\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	ovl->manager = NULL;
+	list_del(&ovl->list);
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+err:
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static bool dss_ovl_is_enabled(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	bool e;
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	e = op->enabled;
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	return e;
+}
+
+static int dss_ovl_enable(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	int r;
+
+	mutex_lock(&apply_lock);
+
+	if (op->enabled) {
+		r = 0;
+		goto err1;
+	}
+
+	if (ovl->manager == NULL || ovl->manager->output == NULL) {
+		r = -EINVAL;
+		goto err1;
+	}
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	op->enabling = true;
+
+	r = dss_check_settings(ovl->manager);
+	if (r) {
+		DSSERR("failed to enable overlay %d: check_settings failed\n",
+				ovl->id);
+		goto err2;
+	}
+
+	dss_setup_fifos();
+
+	op->enabling = false;
+	dss_apply_ovl_enable(ovl, true);
+
+	dss_write_regs();
+	dss_set_go_bits();
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+err2:
+	op->enabling = false;
+	spin_unlock_irqrestore(&data_lock, flags);
+err1:
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static int dss_ovl_disable(struct omap_overlay *ovl)
+{
+	struct ovl_priv_data *op = get_ovl_priv(ovl);
+	unsigned long flags;
+	int r;
+
+	mutex_lock(&apply_lock);
+
+	if (!op->enabled) {
+		r = 0;
+		goto err;
+	}
+
+	if (ovl->manager == NULL || ovl->manager->output == NULL) {
+		r = -EINVAL;
+		goto err;
+	}
+
+	spin_lock_irqsave(&data_lock, flags);
+
+	dss_apply_ovl_enable(ovl, false);
+	dss_write_regs();
+	dss_set_go_bits();
+
+	spin_unlock_irqrestore(&data_lock, flags);
+
+	mutex_unlock(&apply_lock);
+
+	return 0;
+
+err:
+	mutex_unlock(&apply_lock);
+	return r;
+}
+
+static int dss_mgr_register_framedone_handler_compat(struct omap_overlay_manager *mgr,
+		void (*handler)(void *), void *data)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	if (mp->framedone_handler)
+		return -EBUSY;
+
+	mp->framedone_handler = handler;
+	mp->framedone_handler_data = data;
+
+	return 0;
+}
+
+static void dss_mgr_unregister_framedone_handler_compat(struct omap_overlay_manager *mgr,
+		void (*handler)(void *), void *data)
+{
+	struct mgr_priv_data *mp = get_mgr_priv(mgr);
+
+	WARN_ON(mp->framedone_handler != handler ||
+			mp->framedone_handler_data != data);
+
+	mp->framedone_handler = NULL;
+	mp->framedone_handler_data = NULL;
+}
+
+static const struct dss_mgr_ops apply_mgr_ops = {
+	.connect = dss_mgr_connect_compat,
+	.disconnect = dss_mgr_disconnect_compat,
+	.start_update = dss_mgr_start_update_compat,
+	.enable = dss_mgr_enable_compat,
+	.disable = dss_mgr_disable_compat,
+	.set_timings = dss_mgr_set_timings_compat,
+	.set_lcd_config = dss_mgr_set_lcd_config_compat,
+	.register_framedone_handler = dss_mgr_register_framedone_handler_compat,
+	.unregister_framedone_handler = dss_mgr_unregister_framedone_handler_compat,
+};
+
+static int compat_refcnt;
+static DEFINE_MUTEX(compat_init_lock);
+
+int omapdss_compat_init(void)
+{
+	struct platform_device *pdev = dss_get_core_pdev();
+	int i, r;
+
+	mutex_lock(&compat_init_lock);
+
+	if (compat_refcnt++ > 0)
+		goto out;
+
+	apply_init_priv();
+
+	dss_init_overlay_managers_sysfs(pdev);
+	dss_init_overlays(pdev);
+
+	for (i = 0; i < omap_dss_get_num_overlay_managers(); i++) {
+		struct omap_overlay_manager *mgr;
+
+		mgr = omap_dss_get_overlay_manager(i);
+
+		mgr->set_output = &dss_mgr_set_output;
+		mgr->unset_output = &dss_mgr_unset_output;
+		mgr->apply = &omap_dss_mgr_apply;
+		mgr->set_manager_info = &dss_mgr_set_info;
+		mgr->get_manager_info = &dss_mgr_get_info;
+		mgr->wait_for_go = &dss_mgr_wait_for_go;
+		mgr->wait_for_vsync = &dss_mgr_wait_for_vsync;
+		mgr->get_device = &dss_mgr_get_device;
+	}
+
+	for (i = 0; i < omap_dss_get_num_overlays(); i++) {
+		struct omap_overlay *ovl = omap_dss_get_overlay(i);
+
+		ovl->is_enabled = &dss_ovl_is_enabled;
+		ovl->enable = &dss_ovl_enable;
+		ovl->disable = &dss_ovl_disable;
+		ovl->set_manager = &dss_ovl_set_manager;
+		ovl->unset_manager = &dss_ovl_unset_manager;
+		ovl->set_overlay_info = &dss_ovl_set_info;
+		ovl->get_overlay_info = &dss_ovl_get_info;
+		ovl->wait_for_go = &dss_mgr_wait_for_go_ovl;
+		ovl->get_device = &dss_ovl_get_device;
+	}
+
+	r = dss_install_mgr_ops(&apply_mgr_ops);
+	if (r)
+		goto err_mgr_ops;
+
+	r = display_init_sysfs(pdev);
+	if (r)
+		goto err_disp_sysfs;
+
+	dispc_runtime_get();
+
+	r = dss_dispc_initialize_irq();
+	if (r)
+		goto err_init_irq;
+
+	dispc_runtime_put();
+
+out:
+	mutex_unlock(&compat_init_lock);
+
+	return 0;
+
+err_init_irq:
+	dispc_runtime_put();
+	display_uninit_sysfs(pdev);
+
+err_disp_sysfs:
+	dss_uninstall_mgr_ops();
+
+err_mgr_ops:
+	dss_uninit_overlay_managers_sysfs(pdev);
+	dss_uninit_overlays(pdev);
+
+	compat_refcnt--;
+
+	mutex_unlock(&compat_init_lock);
+
+	return r;
+}
+EXPORT_SYMBOL(omapdss_compat_init);
+
+void omapdss_compat_uninit(void)
+{
+	struct platform_device *pdev = dss_get_core_pdev();
+
+	mutex_lock(&compat_init_lock);
+
+	if (--compat_refcnt > 0)
+		goto out;
+
+	dss_dispc_uninitialize_irq();
+
+	display_uninit_sysfs(pdev);
+
+	dss_uninstall_mgr_ops();
+
+	dss_uninit_overlay_managers_sysfs(pdev);
+	dss_uninit_overlays(pdev);
+out:
+	mutex_unlock(&compat_init_lock);
+}
+EXPORT_SYMBOL(omapdss_compat_uninit);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/core.c b/drivers/video/fbdev/omap2/omapfb/dss/core.c
new file mode 100644
index 000000000000..5a87179b7312
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/core.c
@@ -0,0 +1,343 @@
+/*
+ * linux/drivers/video/omap2/dss/core.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "CORE"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include <linux/io.h>
+#include <linux/device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/suspend.h>
+#include <linux/slab.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+static struct {
+	struct platform_device *pdev;
+
+	const char *default_display_name;
+} core;
+
+static char *def_disp_name;
+module_param_named(def_disp, def_disp_name, charp, 0);
+MODULE_PARM_DESC(def_disp, "default display name");
+
+const char *omapdss_get_default_display_name(void)
+{
+	return core.default_display_name;
+}
+EXPORT_SYMBOL(omapdss_get_default_display_name);
+
+enum omapdss_version omapdss_get_version(void)
+{
+	struct omap_dss_board_info *pdata = core.pdev->dev.platform_data;
+	return pdata->version;
+}
+EXPORT_SYMBOL(omapdss_get_version);
+
+struct platform_device *dss_get_core_pdev(void)
+{
+	return core.pdev;
+}
+
+int dss_dsi_enable_pads(int dsi_id, unsigned lane_mask)
+{
+	struct omap_dss_board_info *board_data = core.pdev->dev.platform_data;
+
+	if (!board_data->dsi_enable_pads)
+		return -ENOENT;
+
+	return board_data->dsi_enable_pads(dsi_id, lane_mask);
+}
+
+void dss_dsi_disable_pads(int dsi_id, unsigned lane_mask)
+{
+	struct omap_dss_board_info *board_data = core.pdev->dev.platform_data;
+
+	if (!board_data->dsi_disable_pads)
+		return;
+
+	return board_data->dsi_disable_pads(dsi_id, lane_mask);
+}
+
+int dss_set_min_bus_tput(struct device *dev, unsigned long tput)
+{
+	struct omap_dss_board_info *pdata = core.pdev->dev.platform_data;
+
+	if (pdata->set_min_bus_tput)
+		return pdata->set_min_bus_tput(dev, tput);
+	else
+		return 0;
+}
+
+#if defined(CONFIG_FB_OMAP2_DSS_DEBUGFS)
+static int dss_debug_show(struct seq_file *s, void *unused)
+{
+	void (*func)(struct seq_file *) = s->private;
+	func(s);
+	return 0;
+}
+
+static int dss_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dss_debug_show, inode->i_private);
+}
+
+static const struct file_operations dss_debug_fops = {
+	.open           = dss_debug_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static struct dentry *dss_debugfs_dir;
+
+static int dss_initialize_debugfs(void)
+{
+	dss_debugfs_dir = debugfs_create_dir("omapdss", NULL);
+	if (IS_ERR(dss_debugfs_dir)) {
+		int err = PTR_ERR(dss_debugfs_dir);
+		dss_debugfs_dir = NULL;
+		return err;
+	}
+
+	debugfs_create_file("clk", S_IRUGO, dss_debugfs_dir,
+			&dss_debug_dump_clocks, &dss_debug_fops);
+
+	return 0;
+}
+
+static void dss_uninitialize_debugfs(void)
+{
+	if (dss_debugfs_dir)
+		debugfs_remove_recursive(dss_debugfs_dir);
+}
+
+int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *))
+{
+	struct dentry *d;
+
+	d = debugfs_create_file(name, S_IRUGO, dss_debugfs_dir,
+			write, &dss_debug_fops);
+
+	return PTR_ERR_OR_ZERO(d);
+}
+#else /* CONFIG_FB_OMAP2_DSS_DEBUGFS */
+static inline int dss_initialize_debugfs(void)
+{
+	return 0;
+}
+static inline void dss_uninitialize_debugfs(void)
+{
+}
+int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *))
+{
+	return 0;
+}
+#endif /* CONFIG_FB_OMAP2_DSS_DEBUGFS */
+
+/* PLATFORM DEVICE */
+static int omap_dss_pm_notif(struct notifier_block *b, unsigned long v, void *d)
+{
+	DSSDBG("pm notif %lu\n", v);
+
+	switch (v) {
+	case PM_SUSPEND_PREPARE:
+	case PM_HIBERNATION_PREPARE:
+	case PM_RESTORE_PREPARE:
+		DSSDBG("suspending displays\n");
+		return dss_suspend_all_devices();
+
+	case PM_POST_SUSPEND:
+	case PM_POST_HIBERNATION:
+	case PM_POST_RESTORE:
+		DSSDBG("resuming displays\n");
+		return dss_resume_all_devices();
+
+	default:
+		return 0;
+	}
+}
+
+static struct notifier_block omap_dss_pm_notif_block = {
+	.notifier_call = omap_dss_pm_notif,
+};
+
+static int __init omap_dss_probe(struct platform_device *pdev)
+{
+	struct omap_dss_board_info *pdata = pdev->dev.platform_data;
+	int r;
+
+	core.pdev = pdev;
+
+	dss_features_init(omapdss_get_version());
+
+	r = dss_initialize_debugfs();
+	if (r)
+		goto err_debugfs;
+
+	if (def_disp_name)
+		core.default_display_name = def_disp_name;
+	else if (pdata->default_display_name)
+		core.default_display_name = pdata->default_display_name;
+	else if (pdata->default_device)
+		core.default_display_name = pdata->default_device->name;
+
+	register_pm_notifier(&omap_dss_pm_notif_block);
+
+	return 0;
+
+err_debugfs:
+
+	return r;
+}
+
+static int omap_dss_remove(struct platform_device *pdev)
+{
+	unregister_pm_notifier(&omap_dss_pm_notif_block);
+
+	dss_uninitialize_debugfs();
+
+	return 0;
+}
+
+static void omap_dss_shutdown(struct platform_device *pdev)
+{
+	DSSDBG("shutdown\n");
+	dss_disable_all_devices();
+}
+
+static struct platform_driver omap_dss_driver = {
+	.remove         = omap_dss_remove,
+	.shutdown	= omap_dss_shutdown,
+	.driver         = {
+		.name   = "omapdss",
+	},
+};
+
+/* INIT */
+static int (*dss_output_drv_reg_funcs[])(void) __initdata = {
+	dss_init_platform_driver,
+	dispc_init_platform_driver,
+#ifdef CONFIG_FB_OMAP2_DSS_DSI
+	dsi_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_DPI
+	dpi_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_SDI
+	sdi_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_RFBI
+	rfbi_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_VENC
+	venc_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP4_DSS_HDMI
+	hdmi4_init_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP5_DSS_HDMI
+	hdmi5_init_platform_driver,
+#endif
+};
+
+static void (*dss_output_drv_unreg_funcs[])(void) = {
+#ifdef CONFIG_FB_OMAP5_DSS_HDMI
+	hdmi5_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP4_DSS_HDMI
+	hdmi4_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_VENC
+	venc_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_RFBI
+	rfbi_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_SDI
+	sdi_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_DPI
+	dpi_uninit_platform_driver,
+#endif
+#ifdef CONFIG_FB_OMAP2_DSS_DSI
+	dsi_uninit_platform_driver,
+#endif
+	dispc_uninit_platform_driver,
+	dss_uninit_platform_driver,
+};
+
+static int __init omap_dss_init(void)
+{
+	int r;
+	int i;
+
+	r = platform_driver_probe(&omap_dss_driver, omap_dss_probe);
+	if (r)
+		return r;
+
+	for (i = 0; i < ARRAY_SIZE(dss_output_drv_reg_funcs); ++i) {
+		r = dss_output_drv_reg_funcs[i]();
+		if (r)
+			goto err_reg;
+	}
+
+	return 0;
+
+err_reg:
+	for (i = ARRAY_SIZE(dss_output_drv_reg_funcs) - i;
+			i < ARRAY_SIZE(dss_output_drv_reg_funcs);
+			++i)
+		dss_output_drv_unreg_funcs[i]();
+
+	platform_driver_unregister(&omap_dss_driver);
+
+	return r;
+}
+
+static void __exit omap_dss_exit(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dss_output_drv_unreg_funcs); ++i)
+		dss_output_drv_unreg_funcs[i]();
+
+	platform_driver_unregister(&omap_dss_driver);
+}
+
+module_init(omap_dss_init);
+module_exit(omap_dss_exit);
+
+MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@nokia.com>");
+MODULE_DESCRIPTION("OMAP2/3 Display Subsystem");
+MODULE_LICENSE("GPL v2");
+
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c
new file mode 100644
index 000000000000..6607db37a5e4
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c
@@ -0,0 +1,667 @@
+/*
+ * Copyright (C) 2012 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "APPLY"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/jiffies.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/seq_file.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+#include "dispc-compat.h"
+
+#define DISPC_IRQ_MASK_ERROR            (DISPC_IRQ_GFX_FIFO_UNDERFLOW | \
+					 DISPC_IRQ_OCP_ERR | \
+					 DISPC_IRQ_VID1_FIFO_UNDERFLOW | \
+					 DISPC_IRQ_VID2_FIFO_UNDERFLOW | \
+					 DISPC_IRQ_SYNC_LOST | \
+					 DISPC_IRQ_SYNC_LOST_DIGIT)
+
+#define DISPC_MAX_NR_ISRS		8
+
+struct omap_dispc_isr_data {
+	omap_dispc_isr_t	isr;
+	void			*arg;
+	u32			mask;
+};
+
+struct dispc_irq_stats {
+	unsigned long last_reset;
+	unsigned irq_count;
+	unsigned irqs[32];
+};
+
+static struct {
+	spinlock_t irq_lock;
+	u32 irq_error_mask;
+	struct omap_dispc_isr_data registered_isr[DISPC_MAX_NR_ISRS];
+	u32 error_irqs;
+	struct work_struct error_work;
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	spinlock_t irq_stats_lock;
+	struct dispc_irq_stats irq_stats;
+#endif
+} dispc_compat;
+
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+static void dispc_dump_irqs(struct seq_file *s)
+{
+	unsigned long flags;
+	struct dispc_irq_stats stats;
+
+	spin_lock_irqsave(&dispc_compat.irq_stats_lock, flags);
+
+	stats = dispc_compat.irq_stats;
+	memset(&dispc_compat.irq_stats, 0, sizeof(dispc_compat.irq_stats));
+	dispc_compat.irq_stats.last_reset = jiffies;
+
+	spin_unlock_irqrestore(&dispc_compat.irq_stats_lock, flags);
+
+	seq_printf(s, "period %u ms\n",
+			jiffies_to_msecs(jiffies - stats.last_reset));
+
+	seq_printf(s, "irqs %d\n", stats.irq_count);
+#define PIS(x) \
+	seq_printf(s, "%-20s %10d\n", #x, stats.irqs[ffs(DISPC_IRQ_##x)-1]);
+
+	PIS(FRAMEDONE);
+	PIS(VSYNC);
+	PIS(EVSYNC_EVEN);
+	PIS(EVSYNC_ODD);
+	PIS(ACBIAS_COUNT_STAT);
+	PIS(PROG_LINE_NUM);
+	PIS(GFX_FIFO_UNDERFLOW);
+	PIS(GFX_END_WIN);
+	PIS(PAL_GAMMA_MASK);
+	PIS(OCP_ERR);
+	PIS(VID1_FIFO_UNDERFLOW);
+	PIS(VID1_END_WIN);
+	PIS(VID2_FIFO_UNDERFLOW);
+	PIS(VID2_END_WIN);
+	if (dss_feat_get_num_ovls() > 3) {
+		PIS(VID3_FIFO_UNDERFLOW);
+		PIS(VID3_END_WIN);
+	}
+	PIS(SYNC_LOST);
+	PIS(SYNC_LOST_DIGIT);
+	PIS(WAKEUP);
+	if (dss_has_feature(FEAT_MGR_LCD2)) {
+		PIS(FRAMEDONE2);
+		PIS(VSYNC2);
+		PIS(ACBIAS_COUNT_STAT2);
+		PIS(SYNC_LOST2);
+	}
+	if (dss_has_feature(FEAT_MGR_LCD3)) {
+		PIS(FRAMEDONE3);
+		PIS(VSYNC3);
+		PIS(ACBIAS_COUNT_STAT3);
+		PIS(SYNC_LOST3);
+	}
+#undef PIS
+}
+#endif
+
+/* dispc.irq_lock has to be locked by the caller */
+static void _omap_dispc_set_irqs(void)
+{
+	u32 mask;
+	int i;
+	struct omap_dispc_isr_data *isr_data;
+
+	mask = dispc_compat.irq_error_mask;
+
+	for (i = 0; i < DISPC_MAX_NR_ISRS; i++) {
+		isr_data = &dispc_compat.registered_isr[i];
+
+		if (isr_data->isr == NULL)
+			continue;
+
+		mask |= isr_data->mask;
+	}
+
+	dispc_write_irqenable(mask);
+}
+
+int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask)
+{
+	int i;
+	int ret;
+	unsigned long flags;
+	struct omap_dispc_isr_data *isr_data;
+
+	if (isr == NULL)
+		return -EINVAL;
+
+	spin_lock_irqsave(&dispc_compat.irq_lock, flags);
+
+	/* check for duplicate entry */
+	for (i = 0; i < DISPC_MAX_NR_ISRS; i++) {
+		isr_data = &dispc_compat.registered_isr[i];
+		if (isr_data->isr == isr && isr_data->arg == arg &&
+				isr_data->mask == mask) {
+			ret = -EINVAL;
+			goto err;
+		}
+	}
+
+	isr_data = NULL;
+	ret = -EBUSY;
+
+	for (i = 0; i < DISPC_MAX_NR_ISRS; i++) {
+		isr_data = &dispc_compat.registered_isr[i];
+
+		if (isr_data->isr != NULL)
+			continue;
+
+		isr_data->isr = isr;
+		isr_data->arg = arg;
+		isr_data->mask = mask;
+		ret = 0;
+
+		break;
+	}
+
+	if (ret)
+		goto err;
+
+	_omap_dispc_set_irqs();
+
+	spin_unlock_irqrestore(&dispc_compat.irq_lock, flags);
+
+	return 0;
+err:
+	spin_unlock_irqrestore(&dispc_compat.irq_lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(omap_dispc_register_isr);
+
+int omap_dispc_unregister_isr(omap_dispc_isr_t isr, void *arg, u32 mask)
+{
+	int i;
+	unsigned long flags;
+	int ret = -EINVAL;
+	struct omap_dispc_isr_data *isr_data;
+
+	spin_lock_irqsave(&dispc_compat.irq_lock, flags);
+
+	for (i = 0; i < DISPC_MAX_NR_ISRS; i++) {
+		isr_data = &dispc_compat.registered_isr[i];
+		if (isr_data->isr != isr || isr_data->arg != arg ||
+				isr_data->mask != mask)
+			continue;
+
+		/* found the correct isr */
+
+		isr_data->isr = NULL;
+		isr_data->arg = NULL;
+		isr_data->mask = 0;
+
+		ret = 0;
+		break;
+	}
+
+	if (ret == 0)
+		_omap_dispc_set_irqs();
+
+	spin_unlock_irqrestore(&dispc_compat.irq_lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(omap_dispc_unregister_isr);
+
+static void print_irq_status(u32 status)
+{
+	if ((status & dispc_compat.irq_error_mask) == 0)
+		return;
+
+#define PIS(x) (status & DISPC_IRQ_##x) ? (#x " ") : ""
+
+	pr_debug("DISPC IRQ: 0x%x: %s%s%s%s%s%s%s%s%s\n",
+		status,
+		PIS(OCP_ERR),
+		PIS(GFX_FIFO_UNDERFLOW),
+		PIS(VID1_FIFO_UNDERFLOW),
+		PIS(VID2_FIFO_UNDERFLOW),
+		dss_feat_get_num_ovls() > 3 ? PIS(VID3_FIFO_UNDERFLOW) : "",
+		PIS(SYNC_LOST),
+		PIS(SYNC_LOST_DIGIT),
+		dss_has_feature(FEAT_MGR_LCD2) ? PIS(SYNC_LOST2) : "",
+		dss_has_feature(FEAT_MGR_LCD3) ? PIS(SYNC_LOST3) : "");
+#undef PIS
+}
+
+/* Called from dss.c. Note that we don't touch clocks here,
+ * but we presume they are on because we got an IRQ. However,
+ * an irq handler may turn the clocks off, so we may not have
+ * clock later in the function. */
+static irqreturn_t omap_dispc_irq_handler(int irq, void *arg)
+{
+	int i;
+	u32 irqstatus, irqenable;
+	u32 handledirqs = 0;
+	u32 unhandled_errors;
+	struct omap_dispc_isr_data *isr_data;
+	struct omap_dispc_isr_data registered_isr[DISPC_MAX_NR_ISRS];
+
+	spin_lock(&dispc_compat.irq_lock);
+
+	irqstatus = dispc_read_irqstatus();
+	irqenable = dispc_read_irqenable();
+
+	/* IRQ is not for us */
+	if (!(irqstatus & irqenable)) {
+		spin_unlock(&dispc_compat.irq_lock);
+		return IRQ_NONE;
+	}
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	spin_lock(&dispc_compat.irq_stats_lock);
+	dispc_compat.irq_stats.irq_count++;
+	dss_collect_irq_stats(irqstatus, dispc_compat.irq_stats.irqs);
+	spin_unlock(&dispc_compat.irq_stats_lock);
+#endif
+
+	print_irq_status(irqstatus);
+
+	/* Ack the interrupt. Do it here before clocks are possibly turned
+	 * off */
+	dispc_clear_irqstatus(irqstatus);
+	/* flush posted write */
+	dispc_read_irqstatus();
+
+	/* make a copy and unlock, so that isrs can unregister
+	 * themselves */
+	memcpy(registered_isr, dispc_compat.registered_isr,
+			sizeof(registered_isr));
+
+	spin_unlock(&dispc_compat.irq_lock);
+
+	for (i = 0; i < DISPC_MAX_NR_ISRS; i++) {
+		isr_data = &registered_isr[i];
+
+		if (!isr_data->isr)
+			continue;
+
+		if (isr_data->mask & irqstatus) {
+			isr_data->isr(isr_data->arg, irqstatus);
+			handledirqs |= isr_data->mask;
+		}
+	}
+
+	spin_lock(&dispc_compat.irq_lock);
+
+	unhandled_errors = irqstatus & ~handledirqs & dispc_compat.irq_error_mask;
+
+	if (unhandled_errors) {
+		dispc_compat.error_irqs |= unhandled_errors;
+
+		dispc_compat.irq_error_mask &= ~unhandled_errors;
+		_omap_dispc_set_irqs();
+
+		schedule_work(&dispc_compat.error_work);
+	}
+
+	spin_unlock(&dispc_compat.irq_lock);
+
+	return IRQ_HANDLED;
+}
+
+static void dispc_error_worker(struct work_struct *work)
+{
+	int i;
+	u32 errors;
+	unsigned long flags;
+	static const unsigned fifo_underflow_bits[] = {
+		DISPC_IRQ_GFX_FIFO_UNDERFLOW,
+		DISPC_IRQ_VID1_FIFO_UNDERFLOW,
+		DISPC_IRQ_VID2_FIFO_UNDERFLOW,
+		DISPC_IRQ_VID3_FIFO_UNDERFLOW,
+	};
+
+	spin_lock_irqsave(&dispc_compat.irq_lock, flags);
+	errors = dispc_compat.error_irqs;
+	dispc_compat.error_irqs = 0;
+	spin_unlock_irqrestore(&dispc_compat.irq_lock, flags);
+
+	dispc_runtime_get();
+
+	for (i = 0; i < omap_dss_get_num_overlays(); ++i) {
+		struct omap_overlay *ovl;
+		unsigned bit;
+
+		ovl = omap_dss_get_overlay(i);
+		bit = fifo_underflow_bits[i];
+
+		if (bit & errors) {
+			DSSERR("FIFO UNDERFLOW on %s, disabling the overlay\n",
+					ovl->name);
+			ovl->disable(ovl);
+			msleep(50);
+		}
+	}
+
+	for (i = 0; i < omap_dss_get_num_overlay_managers(); ++i) {
+		struct omap_overlay_manager *mgr;
+		unsigned bit;
+
+		mgr = omap_dss_get_overlay_manager(i);
+		bit = dispc_mgr_get_sync_lost_irq(i);
+
+		if (bit & errors) {
+			int j;
+
+			DSSERR("SYNC_LOST on channel %s, restarting the output "
+					"with video overlays disabled\n",
+					mgr->name);
+
+			dss_mgr_disable(mgr);
+
+			for (j = 0; j < omap_dss_get_num_overlays(); ++j) {
+				struct omap_overlay *ovl;
+				ovl = omap_dss_get_overlay(j);
+
+				if (ovl->id != OMAP_DSS_GFX &&
+						ovl->manager == mgr)
+					ovl->disable(ovl);
+			}
+
+			dss_mgr_enable(mgr);
+		}
+	}
+
+	if (errors & DISPC_IRQ_OCP_ERR) {
+		DSSERR("OCP_ERR\n");
+		for (i = 0; i < omap_dss_get_num_overlay_managers(); ++i) {
+			struct omap_overlay_manager *mgr;
+
+			mgr = omap_dss_get_overlay_manager(i);
+			dss_mgr_disable(mgr);
+		}
+	}
+
+	spin_lock_irqsave(&dispc_compat.irq_lock, flags);
+	dispc_compat.irq_error_mask |= errors;
+	_omap_dispc_set_irqs();
+	spin_unlock_irqrestore(&dispc_compat.irq_lock, flags);
+
+	dispc_runtime_put();
+}
+
+int dss_dispc_initialize_irq(void)
+{
+	int r;
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	spin_lock_init(&dispc_compat.irq_stats_lock);
+	dispc_compat.irq_stats.last_reset = jiffies;
+	dss_debugfs_create_file("dispc_irq", dispc_dump_irqs);
+#endif
+
+	spin_lock_init(&dispc_compat.irq_lock);
+
+	memset(dispc_compat.registered_isr, 0,
+			sizeof(dispc_compat.registered_isr));
+
+	dispc_compat.irq_error_mask = DISPC_IRQ_MASK_ERROR;
+	if (dss_has_feature(FEAT_MGR_LCD2))
+		dispc_compat.irq_error_mask |= DISPC_IRQ_SYNC_LOST2;
+	if (dss_has_feature(FEAT_MGR_LCD3))
+		dispc_compat.irq_error_mask |= DISPC_IRQ_SYNC_LOST3;
+	if (dss_feat_get_num_ovls() > 3)
+		dispc_compat.irq_error_mask |= DISPC_IRQ_VID3_FIFO_UNDERFLOW;
+
+	/*
+	 * there's SYNC_LOST_DIGIT waiting after enabling the DSS,
+	 * so clear it
+	 */
+	dispc_clear_irqstatus(dispc_read_irqstatus());
+
+	INIT_WORK(&dispc_compat.error_work, dispc_error_worker);
+
+	_omap_dispc_set_irqs();
+
+	r = dispc_request_irq(omap_dispc_irq_handler, &dispc_compat);
+	if (r) {
+		DSSERR("dispc_request_irq failed\n");
+		return r;
+	}
+
+	return 0;
+}
+
+void dss_dispc_uninitialize_irq(void)
+{
+	dispc_free_irq(&dispc_compat);
+}
+
+static void dispc_mgr_disable_isr(void *data, u32 mask)
+{
+	struct completion *compl = data;
+	complete(compl);
+}
+
+static void dispc_mgr_enable_lcd_out(enum omap_channel channel)
+{
+	dispc_mgr_enable(channel, true);
+}
+
+static void dispc_mgr_disable_lcd_out(enum omap_channel channel)
+{
+	DECLARE_COMPLETION_ONSTACK(framedone_compl);
+	int r;
+	u32 irq;
+
+	if (!dispc_mgr_is_enabled(channel))
+		return;
+
+	/*
+	 * When we disable LCD output, we need to wait for FRAMEDONE to know
+	 * that DISPC has finished with the LCD output.
+	 */
+
+	irq = dispc_mgr_get_framedone_irq(channel);
+
+	r = omap_dispc_register_isr(dispc_mgr_disable_isr, &framedone_compl,
+			irq);
+	if (r)
+		DSSERR("failed to register FRAMEDONE isr\n");
+
+	dispc_mgr_enable(channel, false);
+
+	/* if we couldn't register for framedone, just sleep and exit */
+	if (r) {
+		msleep(100);
+		return;
+	}
+
+	if (!wait_for_completion_timeout(&framedone_compl,
+				msecs_to_jiffies(100)))
+		DSSERR("timeout waiting for FRAME DONE\n");
+
+	r = omap_dispc_unregister_isr(dispc_mgr_disable_isr, &framedone_compl,
+			irq);
+	if (r)
+		DSSERR("failed to unregister FRAMEDONE isr\n");
+}
+
+static void dispc_digit_out_enable_isr(void *data, u32 mask)
+{
+	struct completion *compl = data;
+
+	/* ignore any sync lost interrupts */
+	if (mask & (DISPC_IRQ_EVSYNC_EVEN | DISPC_IRQ_EVSYNC_ODD))
+		complete(compl);
+}
+
+static void dispc_mgr_enable_digit_out(void)
+{
+	DECLARE_COMPLETION_ONSTACK(vsync_compl);
+	int r;
+	u32 irq_mask;
+
+	if (dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT))
+		return;
+
+	/*
+	 * Digit output produces some sync lost interrupts during the first
+	 * frame when enabling. Those need to be ignored, so we register for the
+	 * sync lost irq to prevent the error handler from triggering.
+	 */
+
+	irq_mask = dispc_mgr_get_vsync_irq(OMAP_DSS_CHANNEL_DIGIT) |
+		dispc_mgr_get_sync_lost_irq(OMAP_DSS_CHANNEL_DIGIT);
+
+	r = omap_dispc_register_isr(dispc_digit_out_enable_isr, &vsync_compl,
+			irq_mask);
+	if (r) {
+		DSSERR("failed to register %x isr\n", irq_mask);
+		return;
+	}
+
+	dispc_mgr_enable(OMAP_DSS_CHANNEL_DIGIT, true);
+
+	/* wait for the first evsync */
+	if (!wait_for_completion_timeout(&vsync_compl, msecs_to_jiffies(100)))
+		DSSERR("timeout waiting for digit out to start\n");
+
+	r = omap_dispc_unregister_isr(dispc_digit_out_enable_isr, &vsync_compl,
+			irq_mask);
+	if (r)
+		DSSERR("failed to unregister %x isr\n", irq_mask);
+}
+
+static void dispc_mgr_disable_digit_out(void)
+{
+	DECLARE_COMPLETION_ONSTACK(framedone_compl);
+	int r, i;
+	u32 irq_mask;
+	int num_irqs;
+
+	if (!dispc_mgr_is_enabled(OMAP_DSS_CHANNEL_DIGIT))
+		return;
+
+	/*
+	 * When we disable the digit output, we need to wait for FRAMEDONE to
+	 * know that DISPC has finished with the output.
+	 */
+
+	irq_mask = dispc_mgr_get_framedone_irq(OMAP_DSS_CHANNEL_DIGIT);
+	num_irqs = 1;
+
+	if (!irq_mask) {
+		/*
+		 * omap 2/3 don't have framedone irq for TV, so we need to use
+		 * vsyncs for this.
+		 */
+
+		irq_mask = dispc_mgr_get_vsync_irq(OMAP_DSS_CHANNEL_DIGIT);
+		/*
+		 * We need to wait for both even and odd vsyncs. Note that this
+		 * is not totally reliable, as we could get a vsync interrupt
+		 * before we disable the output, which leads to timeout in the
+		 * wait_for_completion.
+		 */
+		num_irqs = 2;
+	}
+
+	r = omap_dispc_register_isr(dispc_mgr_disable_isr, &framedone_compl,
+			irq_mask);
+	if (r)
+		DSSERR("failed to register %x isr\n", irq_mask);
+
+	dispc_mgr_enable(OMAP_DSS_CHANNEL_DIGIT, false);
+
+	/* if we couldn't register the irq, just sleep and exit */
+	if (r) {
+		msleep(100);
+		return;
+	}
+
+	for (i = 0; i < num_irqs; ++i) {
+		if (!wait_for_completion_timeout(&framedone_compl,
+					msecs_to_jiffies(100)))
+			DSSERR("timeout waiting for digit out to stop\n");
+	}
+
+	r = omap_dispc_unregister_isr(dispc_mgr_disable_isr, &framedone_compl,
+			irq_mask);
+	if (r)
+		DSSERR("failed to unregister %x isr\n", irq_mask);
+}
+
+void dispc_mgr_enable_sync(enum omap_channel channel)
+{
+	if (dss_mgr_is_lcd(channel))
+		dispc_mgr_enable_lcd_out(channel);
+	else if (channel == OMAP_DSS_CHANNEL_DIGIT)
+		dispc_mgr_enable_digit_out();
+	else
+		WARN_ON(1);
+}
+
+void dispc_mgr_disable_sync(enum omap_channel channel)
+{
+	if (dss_mgr_is_lcd(channel))
+		dispc_mgr_disable_lcd_out(channel);
+	else if (channel == OMAP_DSS_CHANNEL_DIGIT)
+		dispc_mgr_disable_digit_out();
+	else
+		WARN_ON(1);
+}
+
+static inline void dispc_irq_wait_handler(void *data, u32 mask)
+{
+	complete((struct completion *)data);
+}
+
+int omap_dispc_wait_for_irq_interruptible_timeout(u32 irqmask,
+		unsigned long timeout)
+{
+
+	int r;
+	DECLARE_COMPLETION_ONSTACK(completion);
+
+	r = omap_dispc_register_isr(dispc_irq_wait_handler, &completion,
+			irqmask);
+
+	if (r)
+		return r;
+
+	timeout = wait_for_completion_interruptible_timeout(&completion,
+			timeout);
+
+	omap_dispc_unregister_isr(dispc_irq_wait_handler, &completion, irqmask);
+
+	if (timeout == 0)
+		return -ETIMEDOUT;
+
+	if (timeout == -ERESTARTSYS)
+		return -ERESTARTSYS;
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.h b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.h
new file mode 100644
index 000000000000..14a69b3d4fb0
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2012 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAP2_DSS_DISPC_COMPAT_H
+#define __OMAP2_DSS_DISPC_COMPAT_H
+
+void dispc_mgr_enable_sync(enum omap_channel channel);
+void dispc_mgr_disable_sync(enum omap_channel channel);
+
+int omap_dispc_wait_for_irq_interruptible_timeout(u32 irqmask,
+		unsigned long timeout);
+
+int dss_dispc_initialize_irq(void);
+void dss_dispc_uninitialize_irq(void);
+
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
new file mode 100644
index 000000000000..5491e304f4fe
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
@@ -0,0 +1,4234 @@
+/*
+ * linux/drivers/video/omap2/dss/dispc.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DISPC"
+
+#include <linux/kernel.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+#include <linux/export.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/jiffies.h>
+#include <linux/seq_file.h>
+#include <linux/delay.h>
+#include <linux/workqueue.h>
+#include <linux/hardirq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sizes.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+#include "dispc.h"
+
+/* DISPC */
+#define DISPC_SZ_REGS			SZ_4K
+
+enum omap_burst_size {
+	BURST_SIZE_X2 = 0,
+	BURST_SIZE_X4 = 1,
+	BURST_SIZE_X8 = 2,
+};
+
+#define REG_GET(idx, start, end) \
+	FLD_GET(dispc_read_reg(idx), start, end)
+
+#define REG_FLD_MOD(idx, val, start, end)				\
+	dispc_write_reg(idx, FLD_MOD(dispc_read_reg(idx), val, start, end))
+
+struct dispc_features {
+	u8 sw_start;
+	u8 fp_start;
+	u8 bp_start;
+	u16 sw_max;
+	u16 vp_max;
+	u16 hp_max;
+	u8 mgr_width_start;
+	u8 mgr_height_start;
+	u16 mgr_width_max;
+	u16 mgr_height_max;
+	unsigned long max_lcd_pclk;
+	unsigned long max_tv_pclk;
+	int (*calc_scaling) (unsigned long pclk, unsigned long lclk,
+		const struct omap_video_timings *mgr_timings,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode, bool *five_taps,
+		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
+		u16 pos_x, unsigned long *core_clk, bool mem_to_mem);
+	unsigned long (*calc_core_clk) (unsigned long pclk,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		bool mem_to_mem);
+	u8 num_fifos;
+
+	/* swap GFX & WB fifos */
+	bool gfx_fifo_workaround:1;
+
+	/* no DISPC_IRQ_FRAMEDONETV on this SoC */
+	bool no_framedone_tv:1;
+
+	/* revert to the OMAP4 mechanism of DISPC Smart Standby operation */
+	bool mstandby_workaround:1;
+
+	bool set_max_preload:1;
+
+	/* PIXEL_INC is not added to the last pixel of a line */
+	bool last_pixel_inc_missing:1;
+
+	/* POL_FREQ has ALIGN bit */
+	bool supports_sync_align:1;
+
+	bool has_writeback:1;
+};
+
+#define DISPC_MAX_NR_FIFOS 5
+
+static struct {
+	struct platform_device *pdev;
+	void __iomem    *base;
+
+	int irq;
+	irq_handler_t user_handler;
+	void *user_data;
+
+	unsigned long core_clk_rate;
+	unsigned long tv_pclk_rate;
+
+	u32 fifo_size[DISPC_MAX_NR_FIFOS];
+	/* maps which plane is using a fifo. fifo-id -> plane-id */
+	int fifo_assignment[DISPC_MAX_NR_FIFOS];
+
+	bool		ctx_valid;
+	u32		ctx[DISPC_SZ_REGS / sizeof(u32)];
+
+	const struct dispc_features *feat;
+
+	bool is_enabled;
+
+	struct regmap *syscon_pol;
+	u32 syscon_pol_offset;
+
+	/* DISPC_CONTROL & DISPC_CONFIG lock*/
+	spinlock_t control_lock;
+} dispc;
+
+enum omap_color_component {
+	/* used for all color formats for OMAP3 and earlier
+	 * and for RGB and Y color component on OMAP4
+	 */
+	DISPC_COLOR_COMPONENT_RGB_Y		= 1 << 0,
+	/* used for UV component for
+	 * OMAP_DSS_COLOR_YUV2, OMAP_DSS_COLOR_UYVY, OMAP_DSS_COLOR_NV12
+	 * color formats on OMAP4
+	 */
+	DISPC_COLOR_COMPONENT_UV		= 1 << 1,
+};
+
+enum mgr_reg_fields {
+	DISPC_MGR_FLD_ENABLE,
+	DISPC_MGR_FLD_STNTFT,
+	DISPC_MGR_FLD_GO,
+	DISPC_MGR_FLD_TFTDATALINES,
+	DISPC_MGR_FLD_STALLMODE,
+	DISPC_MGR_FLD_TCKENABLE,
+	DISPC_MGR_FLD_TCKSELECTION,
+	DISPC_MGR_FLD_CPR,
+	DISPC_MGR_FLD_FIFOHANDCHECK,
+	/* used to maintain a count of the above fields */
+	DISPC_MGR_FLD_NUM,
+};
+
+struct dispc_reg_field {
+	u16 reg;
+	u8 high;
+	u8 low;
+};
+
+static const struct {
+	const char *name;
+	u32 vsync_irq;
+	u32 framedone_irq;
+	u32 sync_lost_irq;
+	struct dispc_reg_field reg_desc[DISPC_MGR_FLD_NUM];
+} mgr_desc[] = {
+	[OMAP_DSS_CHANNEL_LCD] = {
+		.name		= "LCD",
+		.vsync_irq	= DISPC_IRQ_VSYNC,
+		.framedone_irq	= DISPC_IRQ_FRAMEDONE,
+		.sync_lost_irq	= DISPC_IRQ_SYNC_LOST,
+		.reg_desc	= {
+			[DISPC_MGR_FLD_ENABLE]		= { DISPC_CONTROL,  0,  0 },
+			[DISPC_MGR_FLD_STNTFT]		= { DISPC_CONTROL,  3,  3 },
+			[DISPC_MGR_FLD_GO]		= { DISPC_CONTROL,  5,  5 },
+			[DISPC_MGR_FLD_TFTDATALINES]	= { DISPC_CONTROL,  9,  8 },
+			[DISPC_MGR_FLD_STALLMODE]	= { DISPC_CONTROL, 11, 11 },
+			[DISPC_MGR_FLD_TCKENABLE]	= { DISPC_CONFIG,  10, 10 },
+			[DISPC_MGR_FLD_TCKSELECTION]	= { DISPC_CONFIG,  11, 11 },
+			[DISPC_MGR_FLD_CPR]		= { DISPC_CONFIG,  15, 15 },
+			[DISPC_MGR_FLD_FIFOHANDCHECK]	= { DISPC_CONFIG,  16, 16 },
+		},
+	},
+	[OMAP_DSS_CHANNEL_DIGIT] = {
+		.name		= "DIGIT",
+		.vsync_irq	= DISPC_IRQ_EVSYNC_ODD | DISPC_IRQ_EVSYNC_EVEN,
+		.framedone_irq	= DISPC_IRQ_FRAMEDONETV,
+		.sync_lost_irq	= DISPC_IRQ_SYNC_LOST_DIGIT,
+		.reg_desc	= {
+			[DISPC_MGR_FLD_ENABLE]		= { DISPC_CONTROL,  1,  1 },
+			[DISPC_MGR_FLD_STNTFT]		= { },
+			[DISPC_MGR_FLD_GO]		= { DISPC_CONTROL,  6,  6 },
+			[DISPC_MGR_FLD_TFTDATALINES]	= { },
+			[DISPC_MGR_FLD_STALLMODE]	= { },
+			[DISPC_MGR_FLD_TCKENABLE]	= { DISPC_CONFIG,  12, 12 },
+			[DISPC_MGR_FLD_TCKSELECTION]	= { DISPC_CONFIG,  13, 13 },
+			[DISPC_MGR_FLD_CPR]		= { },
+			[DISPC_MGR_FLD_FIFOHANDCHECK]	= { DISPC_CONFIG,  16, 16 },
+		},
+	},
+	[OMAP_DSS_CHANNEL_LCD2] = {
+		.name		= "LCD2",
+		.vsync_irq	= DISPC_IRQ_VSYNC2,
+		.framedone_irq	= DISPC_IRQ_FRAMEDONE2,
+		.sync_lost_irq	= DISPC_IRQ_SYNC_LOST2,
+		.reg_desc	= {
+			[DISPC_MGR_FLD_ENABLE]		= { DISPC_CONTROL2,  0,  0 },
+			[DISPC_MGR_FLD_STNTFT]		= { DISPC_CONTROL2,  3,  3 },
+			[DISPC_MGR_FLD_GO]		= { DISPC_CONTROL2,  5,  5 },
+			[DISPC_MGR_FLD_TFTDATALINES]	= { DISPC_CONTROL2,  9,  8 },
+			[DISPC_MGR_FLD_STALLMODE]	= { DISPC_CONTROL2, 11, 11 },
+			[DISPC_MGR_FLD_TCKENABLE]	= { DISPC_CONFIG2,  10, 10 },
+			[DISPC_MGR_FLD_TCKSELECTION]	= { DISPC_CONFIG2,  11, 11 },
+			[DISPC_MGR_FLD_CPR]		= { DISPC_CONFIG2,  15, 15 },
+			[DISPC_MGR_FLD_FIFOHANDCHECK]	= { DISPC_CONFIG2,  16, 16 },
+		},
+	},
+	[OMAP_DSS_CHANNEL_LCD3] = {
+		.name		= "LCD3",
+		.vsync_irq	= DISPC_IRQ_VSYNC3,
+		.framedone_irq	= DISPC_IRQ_FRAMEDONE3,
+		.sync_lost_irq	= DISPC_IRQ_SYNC_LOST3,
+		.reg_desc	= {
+			[DISPC_MGR_FLD_ENABLE]		= { DISPC_CONTROL3,  0,  0 },
+			[DISPC_MGR_FLD_STNTFT]		= { DISPC_CONTROL3,  3,  3 },
+			[DISPC_MGR_FLD_GO]		= { DISPC_CONTROL3,  5,  5 },
+			[DISPC_MGR_FLD_TFTDATALINES]	= { DISPC_CONTROL3,  9,  8 },
+			[DISPC_MGR_FLD_STALLMODE]	= { DISPC_CONTROL3, 11, 11 },
+			[DISPC_MGR_FLD_TCKENABLE]	= { DISPC_CONFIG3,  10, 10 },
+			[DISPC_MGR_FLD_TCKSELECTION]	= { DISPC_CONFIG3,  11, 11 },
+			[DISPC_MGR_FLD_CPR]		= { DISPC_CONFIG3,  15, 15 },
+			[DISPC_MGR_FLD_FIFOHANDCHECK]	= { DISPC_CONFIG3,  16, 16 },
+		},
+	},
+};
+
+struct color_conv_coef {
+	int ry, rcr, rcb, gy, gcr, gcb, by, bcr, bcb;
+	int full_range;
+};
+
+static unsigned long dispc_fclk_rate(void);
+static unsigned long dispc_core_clk_rate(void);
+static unsigned long dispc_mgr_lclk_rate(enum omap_channel channel);
+static unsigned long dispc_mgr_pclk_rate(enum omap_channel channel);
+
+static unsigned long dispc_plane_pclk_rate(enum omap_plane plane);
+static unsigned long dispc_plane_lclk_rate(enum omap_plane plane);
+
+static inline void dispc_write_reg(const u16 idx, u32 val)
+{
+	__raw_writel(val, dispc.base + idx);
+}
+
+static inline u32 dispc_read_reg(const u16 idx)
+{
+	return __raw_readl(dispc.base + idx);
+}
+
+static u32 mgr_fld_read(enum omap_channel channel, enum mgr_reg_fields regfld)
+{
+	const struct dispc_reg_field rfld = mgr_desc[channel].reg_desc[regfld];
+	return REG_GET(rfld.reg, rfld.high, rfld.low);
+}
+
+static void mgr_fld_write(enum omap_channel channel,
+					enum mgr_reg_fields regfld, int val) {
+	const struct dispc_reg_field rfld = mgr_desc[channel].reg_desc[regfld];
+	const bool need_lock = rfld.reg == DISPC_CONTROL || rfld.reg == DISPC_CONFIG;
+	unsigned long flags;
+
+	if (need_lock)
+		spin_lock_irqsave(&dispc.control_lock, flags);
+
+	REG_FLD_MOD(rfld.reg, val, rfld.high, rfld.low);
+
+	if (need_lock)
+		spin_unlock_irqrestore(&dispc.control_lock, flags);
+}
+
+#define SR(reg) \
+	dispc.ctx[DISPC_##reg / sizeof(u32)] = dispc_read_reg(DISPC_##reg)
+#define RR(reg) \
+	dispc_write_reg(DISPC_##reg, dispc.ctx[DISPC_##reg / sizeof(u32)])
+
+static void dispc_save_context(void)
+{
+	int i, j;
+
+	DSSDBG("dispc_save_context\n");
+
+	SR(IRQENABLE);
+	SR(CONTROL);
+	SR(CONFIG);
+	SR(LINE_NUMBER);
+	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+		SR(GLOBAL_ALPHA);
+	if (dss_has_feature(FEAT_MGR_LCD2)) {
+		SR(CONTROL2);
+		SR(CONFIG2);
+	}
+	if (dss_has_feature(FEAT_MGR_LCD3)) {
+		SR(CONTROL3);
+		SR(CONFIG3);
+	}
+
+	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+		SR(DEFAULT_COLOR(i));
+		SR(TRANS_COLOR(i));
+		SR(SIZE_MGR(i));
+		if (i == OMAP_DSS_CHANNEL_DIGIT)
+			continue;
+		SR(TIMING_H(i));
+		SR(TIMING_V(i));
+		SR(POL_FREQ(i));
+		SR(DIVISORo(i));
+
+		SR(DATA_CYCLE1(i));
+		SR(DATA_CYCLE2(i));
+		SR(DATA_CYCLE3(i));
+
+		if (dss_has_feature(FEAT_CPR)) {
+			SR(CPR_COEF_R(i));
+			SR(CPR_COEF_G(i));
+			SR(CPR_COEF_B(i));
+		}
+	}
+
+	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+		SR(OVL_BA0(i));
+		SR(OVL_BA1(i));
+		SR(OVL_POSITION(i));
+		SR(OVL_SIZE(i));
+		SR(OVL_ATTRIBUTES(i));
+		SR(OVL_FIFO_THRESHOLD(i));
+		SR(OVL_ROW_INC(i));
+		SR(OVL_PIXEL_INC(i));
+		if (dss_has_feature(FEAT_PRELOAD))
+			SR(OVL_PRELOAD(i));
+		if (i == OMAP_DSS_GFX) {
+			SR(OVL_WINDOW_SKIP(i));
+			SR(OVL_TABLE_BA(i));
+			continue;
+		}
+		SR(OVL_FIR(i));
+		SR(OVL_PICTURE_SIZE(i));
+		SR(OVL_ACCU0(i));
+		SR(OVL_ACCU1(i));
+
+		for (j = 0; j < 8; j++)
+			SR(OVL_FIR_COEF_H(i, j));
+
+		for (j = 0; j < 8; j++)
+			SR(OVL_FIR_COEF_HV(i, j));
+
+		for (j = 0; j < 5; j++)
+			SR(OVL_CONV_COEF(i, j));
+
+		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+			for (j = 0; j < 8; j++)
+				SR(OVL_FIR_COEF_V(i, j));
+		}
+
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			SR(OVL_BA0_UV(i));
+			SR(OVL_BA1_UV(i));
+			SR(OVL_FIR2(i));
+			SR(OVL_ACCU2_0(i));
+			SR(OVL_ACCU2_1(i));
+
+			for (j = 0; j < 8; j++)
+				SR(OVL_FIR_COEF_H2(i, j));
+
+			for (j = 0; j < 8; j++)
+				SR(OVL_FIR_COEF_HV2(i, j));
+
+			for (j = 0; j < 8; j++)
+				SR(OVL_FIR_COEF_V2(i, j));
+		}
+		if (dss_has_feature(FEAT_ATTR2))
+			SR(OVL_ATTRIBUTES2(i));
+	}
+
+	if (dss_has_feature(FEAT_CORE_CLK_DIV))
+		SR(DIVISOR);
+
+	dispc.ctx_valid = true;
+
+	DSSDBG("context saved\n");
+}
+
+static void dispc_restore_context(void)
+{
+	int i, j;
+
+	DSSDBG("dispc_restore_context\n");
+
+	if (!dispc.ctx_valid)
+		return;
+
+	/*RR(IRQENABLE);*/
+	/*RR(CONTROL);*/
+	RR(CONFIG);
+	RR(LINE_NUMBER);
+	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+		RR(GLOBAL_ALPHA);
+	if (dss_has_feature(FEAT_MGR_LCD2))
+		RR(CONFIG2);
+	if (dss_has_feature(FEAT_MGR_LCD3))
+		RR(CONFIG3);
+
+	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+		RR(DEFAULT_COLOR(i));
+		RR(TRANS_COLOR(i));
+		RR(SIZE_MGR(i));
+		if (i == OMAP_DSS_CHANNEL_DIGIT)
+			continue;
+		RR(TIMING_H(i));
+		RR(TIMING_V(i));
+		RR(POL_FREQ(i));
+		RR(DIVISORo(i));
+
+		RR(DATA_CYCLE1(i));
+		RR(DATA_CYCLE2(i));
+		RR(DATA_CYCLE3(i));
+
+		if (dss_has_feature(FEAT_CPR)) {
+			RR(CPR_COEF_R(i));
+			RR(CPR_COEF_G(i));
+			RR(CPR_COEF_B(i));
+		}
+	}
+
+	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+		RR(OVL_BA0(i));
+		RR(OVL_BA1(i));
+		RR(OVL_POSITION(i));
+		RR(OVL_SIZE(i));
+		RR(OVL_ATTRIBUTES(i));
+		RR(OVL_FIFO_THRESHOLD(i));
+		RR(OVL_ROW_INC(i));
+		RR(OVL_PIXEL_INC(i));
+		if (dss_has_feature(FEAT_PRELOAD))
+			RR(OVL_PRELOAD(i));
+		if (i == OMAP_DSS_GFX) {
+			RR(OVL_WINDOW_SKIP(i));
+			RR(OVL_TABLE_BA(i));
+			continue;
+		}
+		RR(OVL_FIR(i));
+		RR(OVL_PICTURE_SIZE(i));
+		RR(OVL_ACCU0(i));
+		RR(OVL_ACCU1(i));
+
+		for (j = 0; j < 8; j++)
+			RR(OVL_FIR_COEF_H(i, j));
+
+		for (j = 0; j < 8; j++)
+			RR(OVL_FIR_COEF_HV(i, j));
+
+		for (j = 0; j < 5; j++)
+			RR(OVL_CONV_COEF(i, j));
+
+		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+			for (j = 0; j < 8; j++)
+				RR(OVL_FIR_COEF_V(i, j));
+		}
+
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			RR(OVL_BA0_UV(i));
+			RR(OVL_BA1_UV(i));
+			RR(OVL_FIR2(i));
+			RR(OVL_ACCU2_0(i));
+			RR(OVL_ACCU2_1(i));
+
+			for (j = 0; j < 8; j++)
+				RR(OVL_FIR_COEF_H2(i, j));
+
+			for (j = 0; j < 8; j++)
+				RR(OVL_FIR_COEF_HV2(i, j));
+
+			for (j = 0; j < 8; j++)
+				RR(OVL_FIR_COEF_V2(i, j));
+		}
+		if (dss_has_feature(FEAT_ATTR2))
+			RR(OVL_ATTRIBUTES2(i));
+	}
+
+	if (dss_has_feature(FEAT_CORE_CLK_DIV))
+		RR(DIVISOR);
+
+	/* enable last, because LCD & DIGIT enable are here */
+	RR(CONTROL);
+	if (dss_has_feature(FEAT_MGR_LCD2))
+		RR(CONTROL2);
+	if (dss_has_feature(FEAT_MGR_LCD3))
+		RR(CONTROL3);
+	/* clear spurious SYNC_LOST_DIGIT interrupts */
+	dispc_clear_irqstatus(DISPC_IRQ_SYNC_LOST_DIGIT);
+
+	/*
+	 * enable last so IRQs won't trigger before
+	 * the context is fully restored
+	 */
+	RR(IRQENABLE);
+
+	DSSDBG("context restored\n");
+}
+
+#undef SR
+#undef RR
+
+int dispc_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("dispc_runtime_get\n");
+
+	r = pm_runtime_get_sync(&dispc.pdev->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+EXPORT_SYMBOL(dispc_runtime_get);
+
+void dispc_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("dispc_runtime_put\n");
+
+	r = pm_runtime_put_sync(&dispc.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+EXPORT_SYMBOL(dispc_runtime_put);
+
+u32 dispc_mgr_get_vsync_irq(enum omap_channel channel)
+{
+	return mgr_desc[channel].vsync_irq;
+}
+EXPORT_SYMBOL(dispc_mgr_get_vsync_irq);
+
+u32 dispc_mgr_get_framedone_irq(enum omap_channel channel)
+{
+	if (channel == OMAP_DSS_CHANNEL_DIGIT && dispc.feat->no_framedone_tv)
+		return 0;
+
+	return mgr_desc[channel].framedone_irq;
+}
+EXPORT_SYMBOL(dispc_mgr_get_framedone_irq);
+
+u32 dispc_mgr_get_sync_lost_irq(enum omap_channel channel)
+{
+	return mgr_desc[channel].sync_lost_irq;
+}
+EXPORT_SYMBOL(dispc_mgr_get_sync_lost_irq);
+
+u32 dispc_wb_get_framedone_irq(void)
+{
+	return DISPC_IRQ_FRAMEDONEWB;
+}
+
+bool dispc_mgr_go_busy(enum omap_channel channel)
+{
+	return mgr_fld_read(channel, DISPC_MGR_FLD_GO) == 1;
+}
+EXPORT_SYMBOL(dispc_mgr_go_busy);
+
+void dispc_mgr_go(enum omap_channel channel)
+{
+	WARN_ON(!dispc_mgr_is_enabled(channel));
+	WARN_ON(dispc_mgr_go_busy(channel));
+
+	DSSDBG("GO %s\n", mgr_desc[channel].name);
+
+	mgr_fld_write(channel, DISPC_MGR_FLD_GO, 1);
+}
+EXPORT_SYMBOL(dispc_mgr_go);
+
+bool dispc_wb_go_busy(void)
+{
+	return REG_GET(DISPC_CONTROL2, 6, 6) == 1;
+}
+
+void dispc_wb_go(void)
+{
+	enum omap_plane plane = OMAP_DSS_WB;
+	bool enable, go;
+
+	enable = REG_GET(DISPC_OVL_ATTRIBUTES(plane), 0, 0) == 1;
+
+	if (!enable)
+		return;
+
+	go = REG_GET(DISPC_CONTROL2, 6, 6) == 1;
+	if (go) {
+		DSSERR("GO bit not down for WB\n");
+		return;
+	}
+
+	REG_FLD_MOD(DISPC_CONTROL2, 1, 6, 6);
+}
+
+static void dispc_ovl_write_firh_reg(enum omap_plane plane, int reg, u32 value)
+{
+	dispc_write_reg(DISPC_OVL_FIR_COEF_H(plane, reg), value);
+}
+
+static void dispc_ovl_write_firhv_reg(enum omap_plane plane, int reg, u32 value)
+{
+	dispc_write_reg(DISPC_OVL_FIR_COEF_HV(plane, reg), value);
+}
+
+static void dispc_ovl_write_firv_reg(enum omap_plane plane, int reg, u32 value)
+{
+	dispc_write_reg(DISPC_OVL_FIR_COEF_V(plane, reg), value);
+}
+
+static void dispc_ovl_write_firh2_reg(enum omap_plane plane, int reg, u32 value)
+{
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	dispc_write_reg(DISPC_OVL_FIR_COEF_H2(plane, reg), value);
+}
+
+static void dispc_ovl_write_firhv2_reg(enum omap_plane plane, int reg,
+		u32 value)
+{
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	dispc_write_reg(DISPC_OVL_FIR_COEF_HV2(plane, reg), value);
+}
+
+static void dispc_ovl_write_firv2_reg(enum omap_plane plane, int reg, u32 value)
+{
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	dispc_write_reg(DISPC_OVL_FIR_COEF_V2(plane, reg), value);
+}
+
+static void dispc_ovl_set_scale_coef(enum omap_plane plane, int fir_hinc,
+				int fir_vinc, int five_taps,
+				enum omap_color_component color_comp)
+{
+	const struct dispc_coef *h_coef, *v_coef;
+	int i;
+
+	h_coef = dispc_ovl_get_scale_coef(fir_hinc, true);
+	v_coef = dispc_ovl_get_scale_coef(fir_vinc, five_taps);
+
+	for (i = 0; i < 8; i++) {
+		u32 h, hv;
+
+		h = FLD_VAL(h_coef[i].hc0_vc00, 7, 0)
+			| FLD_VAL(h_coef[i].hc1_vc0, 15, 8)
+			| FLD_VAL(h_coef[i].hc2_vc1, 23, 16)
+			| FLD_VAL(h_coef[i].hc3_vc2, 31, 24);
+		hv = FLD_VAL(h_coef[i].hc4_vc22, 7, 0)
+			| FLD_VAL(v_coef[i].hc1_vc0, 15, 8)
+			| FLD_VAL(v_coef[i].hc2_vc1, 23, 16)
+			| FLD_VAL(v_coef[i].hc3_vc2, 31, 24);
+
+		if (color_comp == DISPC_COLOR_COMPONENT_RGB_Y) {
+			dispc_ovl_write_firh_reg(plane, i, h);
+			dispc_ovl_write_firhv_reg(plane, i, hv);
+		} else {
+			dispc_ovl_write_firh2_reg(plane, i, h);
+			dispc_ovl_write_firhv2_reg(plane, i, hv);
+		}
+
+	}
+
+	if (five_taps) {
+		for (i = 0; i < 8; i++) {
+			u32 v;
+			v = FLD_VAL(v_coef[i].hc0_vc00, 7, 0)
+				| FLD_VAL(v_coef[i].hc4_vc22, 15, 8);
+			if (color_comp == DISPC_COLOR_COMPONENT_RGB_Y)
+				dispc_ovl_write_firv_reg(plane, i, v);
+			else
+				dispc_ovl_write_firv2_reg(plane, i, v);
+		}
+	}
+}
+
+
+static void dispc_ovl_write_color_conv_coef(enum omap_plane plane,
+		const struct color_conv_coef *ct)
+{
+#define CVAL(x, y) (FLD_VAL(x, 26, 16) | FLD_VAL(y, 10, 0))
+
+	dispc_write_reg(DISPC_OVL_CONV_COEF(plane, 0), CVAL(ct->rcr, ct->ry));
+	dispc_write_reg(DISPC_OVL_CONV_COEF(plane, 1), CVAL(ct->gy,  ct->rcb));
+	dispc_write_reg(DISPC_OVL_CONV_COEF(plane, 2), CVAL(ct->gcb, ct->gcr));
+	dispc_write_reg(DISPC_OVL_CONV_COEF(plane, 3), CVAL(ct->bcr, ct->by));
+	dispc_write_reg(DISPC_OVL_CONV_COEF(plane, 4), CVAL(0, ct->bcb));
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), ct->full_range, 11, 11);
+
+#undef CVAL
+}
+
+static void dispc_setup_color_conv_coef(void)
+{
+	int i;
+	int num_ovl = dss_feat_get_num_ovls();
+	const struct color_conv_coef ctbl_bt601_5_ovl = {
+		/* YUV -> RGB */
+		298, 409, 0, 298, -208, -100, 298, 0, 517, 0,
+	};
+	const struct color_conv_coef ctbl_bt601_5_wb = {
+		/* RGB -> YUV */
+		66, 129, 25, 112, -94, -18, -38, -74, 112, 0,
+	};
+
+	for (i = 1; i < num_ovl; i++)
+		dispc_ovl_write_color_conv_coef(i, &ctbl_bt601_5_ovl);
+
+	if (dispc.feat->has_writeback)
+		dispc_ovl_write_color_conv_coef(OMAP_DSS_WB, &ctbl_bt601_5_wb);
+}
+
+static void dispc_ovl_set_ba0(enum omap_plane plane, u32 paddr)
+{
+	dispc_write_reg(DISPC_OVL_BA0(plane), paddr);
+}
+
+static void dispc_ovl_set_ba1(enum omap_plane plane, u32 paddr)
+{
+	dispc_write_reg(DISPC_OVL_BA1(plane), paddr);
+}
+
+static void dispc_ovl_set_ba0_uv(enum omap_plane plane, u32 paddr)
+{
+	dispc_write_reg(DISPC_OVL_BA0_UV(plane), paddr);
+}
+
+static void dispc_ovl_set_ba1_uv(enum omap_plane plane, u32 paddr)
+{
+	dispc_write_reg(DISPC_OVL_BA1_UV(plane), paddr);
+}
+
+static void dispc_ovl_set_pos(enum omap_plane plane,
+		enum omap_overlay_caps caps, int x, int y)
+{
+	u32 val;
+
+	if ((caps & OMAP_DSS_OVL_CAP_POS) == 0)
+		return;
+
+	val = FLD_VAL(y, 26, 16) | FLD_VAL(x, 10, 0);
+
+	dispc_write_reg(DISPC_OVL_POSITION(plane), val);
+}
+
+static void dispc_ovl_set_input_size(enum omap_plane plane, int width,
+		int height)
+{
+	u32 val = FLD_VAL(height - 1, 26, 16) | FLD_VAL(width - 1, 10, 0);
+
+	if (plane == OMAP_DSS_GFX || plane == OMAP_DSS_WB)
+		dispc_write_reg(DISPC_OVL_SIZE(plane), val);
+	else
+		dispc_write_reg(DISPC_OVL_PICTURE_SIZE(plane), val);
+}
+
+static void dispc_ovl_set_output_size(enum omap_plane plane, int width,
+		int height)
+{
+	u32 val;
+
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	val = FLD_VAL(height - 1, 26, 16) | FLD_VAL(width - 1, 10, 0);
+
+	if (plane == OMAP_DSS_WB)
+		dispc_write_reg(DISPC_OVL_PICTURE_SIZE(plane), val);
+	else
+		dispc_write_reg(DISPC_OVL_SIZE(plane), val);
+}
+
+static void dispc_ovl_set_zorder(enum omap_plane plane,
+		enum omap_overlay_caps caps, u8 zorder)
+{
+	if ((caps & OMAP_DSS_OVL_CAP_ZORDER) == 0)
+		return;
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), zorder, 27, 26);
+}
+
+static void dispc_ovl_enable_zorder_planes(void)
+{
+	int i;
+
+	if (!dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+		return;
+
+	for (i = 0; i < dss_feat_get_num_ovls(); i++)
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(i), 1, 25, 25);
+}
+
+static void dispc_ovl_set_pre_mult_alpha(enum omap_plane plane,
+		enum omap_overlay_caps caps, bool enable)
+{
+	if ((caps & OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA) == 0)
+		return;
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), enable ? 1 : 0, 28, 28);
+}
+
+static void dispc_ovl_setup_global_alpha(enum omap_plane plane,
+		enum omap_overlay_caps caps, u8 global_alpha)
+{
+	static const unsigned shifts[] = { 0, 8, 16, 24, };
+	int shift;
+
+	if ((caps & OMAP_DSS_OVL_CAP_GLOBAL_ALPHA) == 0)
+		return;
+
+	shift = shifts[plane];
+	REG_FLD_MOD(DISPC_GLOBAL_ALPHA, global_alpha, shift + 7, shift);
+}
+
+static void dispc_ovl_set_pix_inc(enum omap_plane plane, s32 inc)
+{
+	dispc_write_reg(DISPC_OVL_PIXEL_INC(plane), inc);
+}
+
+static void dispc_ovl_set_row_inc(enum omap_plane plane, s32 inc)
+{
+	dispc_write_reg(DISPC_OVL_ROW_INC(plane), inc);
+}
+
+static void dispc_ovl_set_color_mode(enum omap_plane plane,
+		enum omap_color_mode color_mode)
+{
+	u32 m = 0;
+	if (plane != OMAP_DSS_GFX) {
+		switch (color_mode) {
+		case OMAP_DSS_COLOR_NV12:
+			m = 0x0; break;
+		case OMAP_DSS_COLOR_RGBX16:
+			m = 0x1; break;
+		case OMAP_DSS_COLOR_RGBA16:
+			m = 0x2; break;
+		case OMAP_DSS_COLOR_RGB12U:
+			m = 0x4; break;
+		case OMAP_DSS_COLOR_ARGB16:
+			m = 0x5; break;
+		case OMAP_DSS_COLOR_RGB16:
+			m = 0x6; break;
+		case OMAP_DSS_COLOR_ARGB16_1555:
+			m = 0x7; break;
+		case OMAP_DSS_COLOR_RGB24U:
+			m = 0x8; break;
+		case OMAP_DSS_COLOR_RGB24P:
+			m = 0x9; break;
+		case OMAP_DSS_COLOR_YUV2:
+			m = 0xa; break;
+		case OMAP_DSS_COLOR_UYVY:
+			m = 0xb; break;
+		case OMAP_DSS_COLOR_ARGB32:
+			m = 0xc; break;
+		case OMAP_DSS_COLOR_RGBA32:
+			m = 0xd; break;
+		case OMAP_DSS_COLOR_RGBX32:
+			m = 0xe; break;
+		case OMAP_DSS_COLOR_XRGB16_1555:
+			m = 0xf; break;
+		default:
+			BUG(); return;
+		}
+	} else {
+		switch (color_mode) {
+		case OMAP_DSS_COLOR_CLUT1:
+			m = 0x0; break;
+		case OMAP_DSS_COLOR_CLUT2:
+			m = 0x1; break;
+		case OMAP_DSS_COLOR_CLUT4:
+			m = 0x2; break;
+		case OMAP_DSS_COLOR_CLUT8:
+			m = 0x3; break;
+		case OMAP_DSS_COLOR_RGB12U:
+			m = 0x4; break;
+		case OMAP_DSS_COLOR_ARGB16:
+			m = 0x5; break;
+		case OMAP_DSS_COLOR_RGB16:
+			m = 0x6; break;
+		case OMAP_DSS_COLOR_ARGB16_1555:
+			m = 0x7; break;
+		case OMAP_DSS_COLOR_RGB24U:
+			m = 0x8; break;
+		case OMAP_DSS_COLOR_RGB24P:
+			m = 0x9; break;
+		case OMAP_DSS_COLOR_RGBX16:
+			m = 0xa; break;
+		case OMAP_DSS_COLOR_RGBA16:
+			m = 0xb; break;
+		case OMAP_DSS_COLOR_ARGB32:
+			m = 0xc; break;
+		case OMAP_DSS_COLOR_RGBA32:
+			m = 0xd; break;
+		case OMAP_DSS_COLOR_RGBX32:
+			m = 0xe; break;
+		case OMAP_DSS_COLOR_XRGB16_1555:
+			m = 0xf; break;
+		default:
+			BUG(); return;
+		}
+	}
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), m, 4, 1);
+}
+
+static void dispc_ovl_configure_burst_type(enum omap_plane plane,
+		enum omap_dss_rotation_type rotation_type)
+{
+	if (dss_has_feature(FEAT_BURST_2D) == 0)
+		return;
+
+	if (rotation_type == OMAP_DSS_ROT_TILER)
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), 1, 29, 29);
+	else
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), 0, 29, 29);
+}
+
+void dispc_ovl_set_channel_out(enum omap_plane plane, enum omap_channel channel)
+{
+	int shift;
+	u32 val;
+	int chan = 0, chan2 = 0;
+
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		shift = 8;
+		break;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+	case OMAP_DSS_VIDEO3:
+		shift = 16;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	val = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
+	if (dss_has_feature(FEAT_MGR_LCD2)) {
+		switch (channel) {
+		case OMAP_DSS_CHANNEL_LCD:
+			chan = 0;
+			chan2 = 0;
+			break;
+		case OMAP_DSS_CHANNEL_DIGIT:
+			chan = 1;
+			chan2 = 0;
+			break;
+		case OMAP_DSS_CHANNEL_LCD2:
+			chan = 0;
+			chan2 = 1;
+			break;
+		case OMAP_DSS_CHANNEL_LCD3:
+			if (dss_has_feature(FEAT_MGR_LCD3)) {
+				chan = 0;
+				chan2 = 2;
+			} else {
+				BUG();
+				return;
+			}
+			break;
+		case OMAP_DSS_CHANNEL_WB:
+			chan = 0;
+			chan2 = 3;
+			break;
+		default:
+			BUG();
+			return;
+		}
+
+		val = FLD_MOD(val, chan, shift, shift);
+		val = FLD_MOD(val, chan2, 31, 30);
+	} else {
+		val = FLD_MOD(val, channel, shift, shift);
+	}
+	dispc_write_reg(DISPC_OVL_ATTRIBUTES(plane), val);
+}
+EXPORT_SYMBOL(dispc_ovl_set_channel_out);
+
+static enum omap_channel dispc_ovl_get_channel_out(enum omap_plane plane)
+{
+	int shift;
+	u32 val;
+
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		shift = 8;
+		break;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+	case OMAP_DSS_VIDEO3:
+		shift = 16;
+		break;
+	default:
+		BUG();
+		return 0;
+	}
+
+	val = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
+
+	if (FLD_GET(val, shift, shift) == 1)
+		return OMAP_DSS_CHANNEL_DIGIT;
+
+	if (!dss_has_feature(FEAT_MGR_LCD2))
+		return OMAP_DSS_CHANNEL_LCD;
+
+	switch (FLD_GET(val, 31, 30)) {
+	case 0:
+	default:
+		return OMAP_DSS_CHANNEL_LCD;
+	case 1:
+		return OMAP_DSS_CHANNEL_LCD2;
+	case 2:
+		return OMAP_DSS_CHANNEL_LCD3;
+	case 3:
+		return OMAP_DSS_CHANNEL_WB;
+	}
+}
+
+void dispc_wb_set_channel_in(enum dss_writeback_channel channel)
+{
+	enum omap_plane plane = OMAP_DSS_WB;
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), channel, 18, 16);
+}
+
+static void dispc_ovl_set_burst_size(enum omap_plane plane,
+		enum omap_burst_size burst_size)
+{
+	static const unsigned shifts[] = { 6, 14, 14, 14, 14, };
+	int shift;
+
+	shift = shifts[plane];
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), burst_size, shift + 1, shift);
+}
+
+static void dispc_configure_burst_sizes(void)
+{
+	int i;
+	const int burst_size = BURST_SIZE_X8;
+
+	/* Configure burst size always to maximum size */
+	for (i = 0; i < dss_feat_get_num_ovls(); ++i)
+		dispc_ovl_set_burst_size(i, burst_size);
+	if (dispc.feat->has_writeback)
+		dispc_ovl_set_burst_size(OMAP_DSS_WB, burst_size);
+}
+
+static u32 dispc_ovl_get_burst_size(enum omap_plane plane)
+{
+	unsigned unit = dss_feat_get_burst_size_unit();
+	/* burst multiplier is always x8 (see dispc_configure_burst_sizes()) */
+	return unit * 8;
+}
+
+void dispc_enable_gamma_table(bool enable)
+{
+	/*
+	 * This is partially implemented to support only disabling of
+	 * the gamma table.
+	 */
+	if (enable) {
+		DSSWARN("Gamma table enabling for TV not yet supported");
+		return;
+	}
+
+	REG_FLD_MOD(DISPC_CONFIG, enable, 9, 9);
+}
+
+static void dispc_mgr_enable_cpr(enum omap_channel channel, bool enable)
+{
+	if (channel == OMAP_DSS_CHANNEL_DIGIT)
+		return;
+
+	mgr_fld_write(channel, DISPC_MGR_FLD_CPR, enable);
+}
+
+static void dispc_mgr_set_cpr_coef(enum omap_channel channel,
+		const struct omap_dss_cpr_coefs *coefs)
+{
+	u32 coef_r, coef_g, coef_b;
+
+	if (!dss_mgr_is_lcd(channel))
+		return;
+
+	coef_r = FLD_VAL(coefs->rr, 31, 22) | FLD_VAL(coefs->rg, 20, 11) |
+		FLD_VAL(coefs->rb, 9, 0);
+	coef_g = FLD_VAL(coefs->gr, 31, 22) | FLD_VAL(coefs->gg, 20, 11) |
+		FLD_VAL(coefs->gb, 9, 0);
+	coef_b = FLD_VAL(coefs->br, 31, 22) | FLD_VAL(coefs->bg, 20, 11) |
+		FLD_VAL(coefs->bb, 9, 0);
+
+	dispc_write_reg(DISPC_CPR_COEF_R(channel), coef_r);
+	dispc_write_reg(DISPC_CPR_COEF_G(channel), coef_g);
+	dispc_write_reg(DISPC_CPR_COEF_B(channel), coef_b);
+}
+
+static void dispc_ovl_set_vid_color_conv(enum omap_plane plane, bool enable)
+{
+	u32 val;
+
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	val = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
+	val = FLD_MOD(val, enable, 9, 9);
+	dispc_write_reg(DISPC_OVL_ATTRIBUTES(plane), val);
+}
+
+static void dispc_ovl_enable_replication(enum omap_plane plane,
+		enum omap_overlay_caps caps, bool enable)
+{
+	static const unsigned shifts[] = { 5, 10, 10, 10 };
+	int shift;
+
+	if ((caps & OMAP_DSS_OVL_CAP_REPLICATION) == 0)
+		return;
+
+	shift = shifts[plane];
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), enable, shift, shift);
+}
+
+static void dispc_mgr_set_size(enum omap_channel channel, u16 width,
+		u16 height)
+{
+	u32 val;
+
+	val = FLD_VAL(height - 1, dispc.feat->mgr_height_start, 16) |
+		FLD_VAL(width - 1, dispc.feat->mgr_width_start, 0);
+
+	dispc_write_reg(DISPC_SIZE_MGR(channel), val);
+}
+
+static void dispc_init_fifos(void)
+{
+	u32 size;
+	int fifo;
+	u8 start, end;
+	u32 unit;
+	int i;
+
+	unit = dss_feat_get_buffer_size_unit();
+
+	dss_feat_get_reg_field(FEAT_REG_FIFOSIZE, &start, &end);
+
+	for (fifo = 0; fifo < dispc.feat->num_fifos; ++fifo) {
+		size = REG_GET(DISPC_OVL_FIFO_SIZE_STATUS(fifo), start, end);
+		size *= unit;
+		dispc.fifo_size[fifo] = size;
+
+		/*
+		 * By default fifos are mapped directly to overlays, fifo 0 to
+		 * ovl 0, fifo 1 to ovl 1, etc.
+		 */
+		dispc.fifo_assignment[fifo] = fifo;
+	}
+
+	/*
+	 * The GFX fifo on OMAP4 is smaller than the other fifos. The small fifo
+	 * causes problems with certain use cases, like using the tiler in 2D
+	 * mode. The below hack swaps the fifos of GFX and WB planes, thus
+	 * giving GFX plane a larger fifo. WB but should work fine with a
+	 * smaller fifo.
+	 */
+	if (dispc.feat->gfx_fifo_workaround) {
+		u32 v;
+
+		v = dispc_read_reg(DISPC_GLOBAL_BUFFER);
+
+		v = FLD_MOD(v, 4, 2, 0); /* GFX BUF top to WB */
+		v = FLD_MOD(v, 4, 5, 3); /* GFX BUF bottom to WB */
+		v = FLD_MOD(v, 0, 26, 24); /* WB BUF top to GFX */
+		v = FLD_MOD(v, 0, 29, 27); /* WB BUF bottom to GFX */
+
+		dispc_write_reg(DISPC_GLOBAL_BUFFER, v);
+
+		dispc.fifo_assignment[OMAP_DSS_GFX] = OMAP_DSS_WB;
+		dispc.fifo_assignment[OMAP_DSS_WB] = OMAP_DSS_GFX;
+	}
+
+	/*
+	 * Setup default fifo thresholds.
+	 */
+	for (i = 0; i < dss_feat_get_num_ovls(); ++i) {
+		u32 low, high;
+		const bool use_fifomerge = false;
+		const bool manual_update = false;
+
+		dispc_ovl_compute_fifo_thresholds(i, &low, &high,
+			use_fifomerge, manual_update);
+
+		dispc_ovl_set_fifo_threshold(i, low, high);
+	}
+
+	if (dispc.feat->has_writeback) {
+		u32 low, high;
+		const bool use_fifomerge = false;
+		const bool manual_update = false;
+
+		dispc_ovl_compute_fifo_thresholds(OMAP_DSS_WB, &low, &high,
+			use_fifomerge, manual_update);
+
+		dispc_ovl_set_fifo_threshold(OMAP_DSS_WB, low, high);
+	}
+}
+
+static u32 dispc_ovl_get_fifo_size(enum omap_plane plane)
+{
+	int fifo;
+	u32 size = 0;
+
+	for (fifo = 0; fifo < dispc.feat->num_fifos; ++fifo) {
+		if (dispc.fifo_assignment[fifo] == plane)
+			size += dispc.fifo_size[fifo];
+	}
+
+	return size;
+}
+
+void dispc_ovl_set_fifo_threshold(enum omap_plane plane, u32 low, u32 high)
+{
+	u8 hi_start, hi_end, lo_start, lo_end;
+	u32 unit;
+
+	unit = dss_feat_get_buffer_size_unit();
+
+	WARN_ON(low % unit != 0);
+	WARN_ON(high % unit != 0);
+
+	low /= unit;
+	high /= unit;
+
+	dss_feat_get_reg_field(FEAT_REG_FIFOHIGHTHRESHOLD, &hi_start, &hi_end);
+	dss_feat_get_reg_field(FEAT_REG_FIFOLOWTHRESHOLD, &lo_start, &lo_end);
+
+	DSSDBG("fifo(%d) threshold (bytes), old %u/%u, new %u/%u\n",
+			plane,
+			REG_GET(DISPC_OVL_FIFO_THRESHOLD(plane),
+				lo_start, lo_end) * unit,
+			REG_GET(DISPC_OVL_FIFO_THRESHOLD(plane),
+				hi_start, hi_end) * unit,
+			low * unit, high * unit);
+
+	dispc_write_reg(DISPC_OVL_FIFO_THRESHOLD(plane),
+			FLD_VAL(high, hi_start, hi_end) |
+			FLD_VAL(low, lo_start, lo_end));
+
+	/*
+	 * configure the preload to the pipeline's high threhold, if HT it's too
+	 * large for the preload field, set the threshold to the maximum value
+	 * that can be held by the preload register
+	 */
+	if (dss_has_feature(FEAT_PRELOAD) && dispc.feat->set_max_preload &&
+			plane != OMAP_DSS_WB)
+		dispc_write_reg(DISPC_OVL_PRELOAD(plane), min(high, 0xfffu));
+}
+
+void dispc_enable_fifomerge(bool enable)
+{
+	if (!dss_has_feature(FEAT_FIFO_MERGE)) {
+		WARN_ON(enable);
+		return;
+	}
+
+	DSSDBG("FIFO merge %s\n", enable ? "enabled" : "disabled");
+	REG_FLD_MOD(DISPC_CONFIG, enable ? 1 : 0, 14, 14);
+}
+
+void dispc_ovl_compute_fifo_thresholds(enum omap_plane plane,
+		u32 *fifo_low, u32 *fifo_high, bool use_fifomerge,
+		bool manual_update)
+{
+	/*
+	 * All sizes are in bytes. Both the buffer and burst are made of
+	 * buffer_units, and the fifo thresholds must be buffer_unit aligned.
+	 */
+
+	unsigned buf_unit = dss_feat_get_buffer_size_unit();
+	unsigned ovl_fifo_size, total_fifo_size, burst_size;
+	int i;
+
+	burst_size = dispc_ovl_get_burst_size(plane);
+	ovl_fifo_size = dispc_ovl_get_fifo_size(plane);
+
+	if (use_fifomerge) {
+		total_fifo_size = 0;
+		for (i = 0; i < dss_feat_get_num_ovls(); ++i)
+			total_fifo_size += dispc_ovl_get_fifo_size(i);
+	} else {
+		total_fifo_size = ovl_fifo_size;
+	}
+
+	/*
+	 * We use the same low threshold for both fifomerge and non-fifomerge
+	 * cases, but for fifomerge we calculate the high threshold using the
+	 * combined fifo size
+	 */
+
+	if (manual_update && dss_has_feature(FEAT_OMAP3_DSI_FIFO_BUG)) {
+		*fifo_low = ovl_fifo_size - burst_size * 2;
+		*fifo_high = total_fifo_size - burst_size;
+	} else if (plane == OMAP_DSS_WB) {
+		/*
+		 * Most optimal configuration for writeback is to push out data
+		 * to the interconnect the moment writeback pushes enough pixels
+		 * in the FIFO to form a burst
+		 */
+		*fifo_low = 0;
+		*fifo_high = burst_size;
+	} else {
+		*fifo_low = ovl_fifo_size - burst_size;
+		*fifo_high = total_fifo_size - buf_unit;
+	}
+}
+
+static void dispc_ovl_set_mflag(enum omap_plane plane, bool enable)
+{
+	int bit;
+
+	if (plane == OMAP_DSS_GFX)
+		bit = 14;
+	else
+		bit = 23;
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), enable, bit, bit);
+}
+
+static void dispc_ovl_set_mflag_threshold(enum omap_plane plane,
+	int low, int high)
+{
+	dispc_write_reg(DISPC_OVL_MFLAG_THRESHOLD(plane),
+		FLD_VAL(high, 31, 16) |	FLD_VAL(low, 15, 0));
+}
+
+static void dispc_init_mflag(void)
+{
+	int i;
+
+	/*
+	 * HACK: NV12 color format and MFLAG seem to have problems working
+	 * together: using two displays, and having an NV12 overlay on one of
+	 * the displays will cause underflows/synclosts when MFLAG_CTRL=2.
+	 * Changing MFLAG thresholds and PRELOAD to certain values seem to
+	 * remove the errors, but there doesn't seem to be a clear logic on
+	 * which values work and which not.
+	 *
+	 * As a work-around, set force MFLAG to always on.
+	 */
+	dispc_write_reg(DISPC_GLOBAL_MFLAG_ATTRIBUTE,
+		(1 << 0) |	/* MFLAG_CTRL = force always on */
+		(0 << 2));	/* MFLAG_START = disable */
+
+	for (i = 0; i < dss_feat_get_num_ovls(); ++i) {
+		u32 size = dispc_ovl_get_fifo_size(i);
+		u32 unit = dss_feat_get_buffer_size_unit();
+		u32 low, high;
+
+		dispc_ovl_set_mflag(i, true);
+
+		/*
+		 * Simulation team suggests below thesholds:
+		 * HT = fifosize * 5 / 8;
+		 * LT = fifosize * 4 / 8;
+		 */
+
+		low = size * 4 / 8 / unit;
+		high = size * 5 / 8 / unit;
+
+		dispc_ovl_set_mflag_threshold(i, low, high);
+	}
+
+	if (dispc.feat->has_writeback) {
+		u32 size = dispc_ovl_get_fifo_size(OMAP_DSS_WB);
+		u32 unit = dss_feat_get_buffer_size_unit();
+		u32 low, high;
+
+		dispc_ovl_set_mflag(OMAP_DSS_WB, true);
+
+		/*
+		 * Simulation team suggests below thesholds:
+		 * HT = fifosize * 5 / 8;
+		 * LT = fifosize * 4 / 8;
+		 */
+
+		low = size * 4 / 8 / unit;
+		high = size * 5 / 8 / unit;
+
+		dispc_ovl_set_mflag_threshold(OMAP_DSS_WB, low, high);
+	}
+}
+
+static void dispc_ovl_set_fir(enum omap_plane plane,
+				int hinc, int vinc,
+				enum omap_color_component color_comp)
+{
+	u32 val;
+
+	if (color_comp == DISPC_COLOR_COMPONENT_RGB_Y) {
+		u8 hinc_start, hinc_end, vinc_start, vinc_end;
+
+		dss_feat_get_reg_field(FEAT_REG_FIRHINC,
+					&hinc_start, &hinc_end);
+		dss_feat_get_reg_field(FEAT_REG_FIRVINC,
+					&vinc_start, &vinc_end);
+		val = FLD_VAL(vinc, vinc_start, vinc_end) |
+				FLD_VAL(hinc, hinc_start, hinc_end);
+
+		dispc_write_reg(DISPC_OVL_FIR(plane), val);
+	} else {
+		val = FLD_VAL(vinc, 28, 16) | FLD_VAL(hinc, 12, 0);
+		dispc_write_reg(DISPC_OVL_FIR2(plane), val);
+	}
+}
+
+static void dispc_ovl_set_vid_accu0(enum omap_plane plane, int haccu, int vaccu)
+{
+	u32 val;
+	u8 hor_start, hor_end, vert_start, vert_end;
+
+	dss_feat_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
+	dss_feat_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
+
+	val = FLD_VAL(vaccu, vert_start, vert_end) |
+			FLD_VAL(haccu, hor_start, hor_end);
+
+	dispc_write_reg(DISPC_OVL_ACCU0(plane), val);
+}
+
+static void dispc_ovl_set_vid_accu1(enum omap_plane plane, int haccu, int vaccu)
+{
+	u32 val;
+	u8 hor_start, hor_end, vert_start, vert_end;
+
+	dss_feat_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
+	dss_feat_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
+
+	val = FLD_VAL(vaccu, vert_start, vert_end) |
+			FLD_VAL(haccu, hor_start, hor_end);
+
+	dispc_write_reg(DISPC_OVL_ACCU1(plane), val);
+}
+
+static void dispc_ovl_set_vid_accu2_0(enum omap_plane plane, int haccu,
+		int vaccu)
+{
+	u32 val;
+
+	val = FLD_VAL(vaccu, 26, 16) | FLD_VAL(haccu, 10, 0);
+	dispc_write_reg(DISPC_OVL_ACCU2_0(plane), val);
+}
+
+static void dispc_ovl_set_vid_accu2_1(enum omap_plane plane, int haccu,
+		int vaccu)
+{
+	u32 val;
+
+	val = FLD_VAL(vaccu, 26, 16) | FLD_VAL(haccu, 10, 0);
+	dispc_write_reg(DISPC_OVL_ACCU2_1(plane), val);
+}
+
+static void dispc_ovl_set_scale_param(enum omap_plane plane,
+		u16 orig_width, u16 orig_height,
+		u16 out_width, u16 out_height,
+		bool five_taps, u8 rotation,
+		enum omap_color_component color_comp)
+{
+	int fir_hinc, fir_vinc;
+
+	fir_hinc = 1024 * orig_width / out_width;
+	fir_vinc = 1024 * orig_height / out_height;
+
+	dispc_ovl_set_scale_coef(plane, fir_hinc, fir_vinc, five_taps,
+				color_comp);
+	dispc_ovl_set_fir(plane, fir_hinc, fir_vinc, color_comp);
+}
+
+static void dispc_ovl_set_accu_uv(enum omap_plane plane,
+		u16 orig_width,	u16 orig_height, u16 out_width, u16 out_height,
+		bool ilace, enum omap_color_mode color_mode, u8 rotation)
+{
+	int h_accu2_0, h_accu2_1;
+	int v_accu2_0, v_accu2_1;
+	int chroma_hinc, chroma_vinc;
+	int idx;
+
+	struct accu {
+		s8 h0_m, h0_n;
+		s8 h1_m, h1_n;
+		s8 v0_m, v0_n;
+		s8 v1_m, v1_n;
+	};
+
+	const struct accu *accu_table;
+	const struct accu *accu_val;
+
+	static const struct accu accu_nv12[4] = {
+		{  0, 1,  0, 1 , -1, 2, 0, 1 },
+		{  1, 2, -3, 4 ,  0, 1, 0, 1 },
+		{ -1, 1,  0, 1 , -1, 2, 0, 1 },
+		{ -1, 2, -1, 2 , -1, 1, 0, 1 },
+	};
+
+	static const struct accu accu_nv12_ilace[4] = {
+		{  0, 1,  0, 1 , -3, 4, -1, 4 },
+		{ -1, 4, -3, 4 ,  0, 1,  0, 1 },
+		{ -1, 1,  0, 1 , -1, 4, -3, 4 },
+		{ -3, 4, -3, 4 , -1, 1,  0, 1 },
+	};
+
+	static const struct accu accu_yuv[4] = {
+		{  0, 1, 0, 1,  0, 1, 0, 1 },
+		{  0, 1, 0, 1,  0, 1, 0, 1 },
+		{ -1, 1, 0, 1,  0, 1, 0, 1 },
+		{  0, 1, 0, 1, -1, 1, 0, 1 },
+	};
+
+	switch (rotation) {
+	case OMAP_DSS_ROT_0:
+		idx = 0;
+		break;
+	case OMAP_DSS_ROT_90:
+		idx = 1;
+		break;
+	case OMAP_DSS_ROT_180:
+		idx = 2;
+		break;
+	case OMAP_DSS_ROT_270:
+		idx = 3;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_NV12:
+		if (ilace)
+			accu_table = accu_nv12_ilace;
+		else
+			accu_table = accu_nv12;
+		break;
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+		accu_table = accu_yuv;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	accu_val = &accu_table[idx];
+
+	chroma_hinc = 1024 * orig_width / out_width;
+	chroma_vinc = 1024 * orig_height / out_height;
+
+	h_accu2_0 = (accu_val->h0_m * chroma_hinc / accu_val->h0_n) % 1024;
+	h_accu2_1 = (accu_val->h1_m * chroma_hinc / accu_val->h1_n) % 1024;
+	v_accu2_0 = (accu_val->v0_m * chroma_vinc / accu_val->v0_n) % 1024;
+	v_accu2_1 = (accu_val->v1_m * chroma_vinc / accu_val->v1_n) % 1024;
+
+	dispc_ovl_set_vid_accu2_0(plane, h_accu2_0, v_accu2_0);
+	dispc_ovl_set_vid_accu2_1(plane, h_accu2_1, v_accu2_1);
+}
+
+static void dispc_ovl_set_scaling_common(enum omap_plane plane,
+		u16 orig_width, u16 orig_height,
+		u16 out_width, u16 out_height,
+		bool ilace, bool five_taps,
+		bool fieldmode, enum omap_color_mode color_mode,
+		u8 rotation)
+{
+	int accu0 = 0;
+	int accu1 = 0;
+	u32 l;
+
+	dispc_ovl_set_scale_param(plane, orig_width, orig_height,
+				out_width, out_height, five_taps,
+				rotation, DISPC_COLOR_COMPONENT_RGB_Y);
+	l = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
+
+	/* RESIZEENABLE and VERTICALTAPS */
+	l &= ~((0x3 << 5) | (0x1 << 21));
+	l |= (orig_width != out_width) ? (1 << 5) : 0;
+	l |= (orig_height != out_height) ? (1 << 6) : 0;
+	l |= five_taps ? (1 << 21) : 0;
+
+	/* VRESIZECONF and HRESIZECONF */
+	if (dss_has_feature(FEAT_RESIZECONF)) {
+		l &= ~(0x3 << 7);
+		l |= (orig_width <= out_width) ? 0 : (1 << 7);
+		l |= (orig_height <= out_height) ? 0 : (1 << 8);
+	}
+
+	/* LINEBUFFERSPLIT */
+	if (dss_has_feature(FEAT_LINEBUFFERSPLIT)) {
+		l &= ~(0x1 << 22);
+		l |= five_taps ? (1 << 22) : 0;
+	}
+
+	dispc_write_reg(DISPC_OVL_ATTRIBUTES(plane), l);
+
+	/*
+	 * field 0 = even field = bottom field
+	 * field 1 = odd field = top field
+	 */
+	if (ilace && !fieldmode) {
+		accu1 = 0;
+		accu0 = ((1024 * orig_height / out_height) / 2) & 0x3ff;
+		if (accu0 >= 1024/2) {
+			accu1 = 1024/2;
+			accu0 -= accu1;
+		}
+	}
+
+	dispc_ovl_set_vid_accu0(plane, 0, accu0);
+	dispc_ovl_set_vid_accu1(plane, 0, accu1);
+}
+
+static void dispc_ovl_set_scaling_uv(enum omap_plane plane,
+		u16 orig_width, u16 orig_height,
+		u16 out_width, u16 out_height,
+		bool ilace, bool five_taps,
+		bool fieldmode, enum omap_color_mode color_mode,
+		u8 rotation)
+{
+	int scale_x = out_width != orig_width;
+	int scale_y = out_height != orig_height;
+	bool chroma_upscale = plane != OMAP_DSS_WB ? true : false;
+
+	if (!dss_has_feature(FEAT_HANDLE_UV_SEPARATE))
+		return;
+	if ((color_mode != OMAP_DSS_COLOR_YUV2 &&
+			color_mode != OMAP_DSS_COLOR_UYVY &&
+			color_mode != OMAP_DSS_COLOR_NV12)) {
+		/* reset chroma resampling for RGB formats  */
+		if (plane != OMAP_DSS_WB)
+			REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), 0, 8, 8);
+		return;
+	}
+
+	dispc_ovl_set_accu_uv(plane, orig_width, orig_height, out_width,
+			out_height, ilace, color_mode, rotation);
+
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_NV12:
+		if (chroma_upscale) {
+			/* UV is subsampled by 2 horizontally and vertically */
+			orig_height >>= 1;
+			orig_width >>= 1;
+		} else {
+			/* UV is downsampled by 2 horizontally and vertically */
+			orig_height <<= 1;
+			orig_width <<= 1;
+		}
+
+		break;
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+		/* For YUV422 with 90/270 rotation, we don't upsample chroma */
+		if (rotation == OMAP_DSS_ROT_0 ||
+				rotation == OMAP_DSS_ROT_180) {
+			if (chroma_upscale)
+				/* UV is subsampled by 2 horizontally */
+				orig_width >>= 1;
+			else
+				/* UV is downsampled by 2 horizontally */
+				orig_width <<= 1;
+		}
+
+		/* must use FIR for YUV422 if rotated */
+		if (rotation != OMAP_DSS_ROT_0)
+			scale_x = scale_y = true;
+
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	if (out_width != orig_width)
+		scale_x = true;
+	if (out_height != orig_height)
+		scale_y = true;
+
+	dispc_ovl_set_scale_param(plane, orig_width, orig_height,
+			out_width, out_height, five_taps,
+				rotation, DISPC_COLOR_COMPONENT_UV);
+
+	if (plane != OMAP_DSS_WB)
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane),
+			(scale_x || scale_y) ? 1 : 0, 8, 8);
+
+	/* set H scaling */
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), scale_x ? 1 : 0, 5, 5);
+	/* set V scaling */
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), scale_y ? 1 : 0, 6, 6);
+}
+
+static void dispc_ovl_set_scaling(enum omap_plane plane,
+		u16 orig_width, u16 orig_height,
+		u16 out_width, u16 out_height,
+		bool ilace, bool five_taps,
+		bool fieldmode, enum omap_color_mode color_mode,
+		u8 rotation)
+{
+	BUG_ON(plane == OMAP_DSS_GFX);
+
+	dispc_ovl_set_scaling_common(plane,
+			orig_width, orig_height,
+			out_width, out_height,
+			ilace, five_taps,
+			fieldmode, color_mode,
+			rotation);
+
+	dispc_ovl_set_scaling_uv(plane,
+		orig_width, orig_height,
+		out_width, out_height,
+		ilace, five_taps,
+		fieldmode, color_mode,
+		rotation);
+}
+
+static void dispc_ovl_set_rotation_attrs(enum omap_plane plane, u8 rotation,
+		enum omap_dss_rotation_type rotation_type,
+		bool mirroring, enum omap_color_mode color_mode)
+{
+	bool row_repeat = false;
+	int vidrot = 0;
+
+	if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY) {
+
+		if (mirroring) {
+			switch (rotation) {
+			case OMAP_DSS_ROT_0:
+				vidrot = 2;
+				break;
+			case OMAP_DSS_ROT_90:
+				vidrot = 1;
+				break;
+			case OMAP_DSS_ROT_180:
+				vidrot = 0;
+				break;
+			case OMAP_DSS_ROT_270:
+				vidrot = 3;
+				break;
+			}
+		} else {
+			switch (rotation) {
+			case OMAP_DSS_ROT_0:
+				vidrot = 0;
+				break;
+			case OMAP_DSS_ROT_90:
+				vidrot = 1;
+				break;
+			case OMAP_DSS_ROT_180:
+				vidrot = 2;
+				break;
+			case OMAP_DSS_ROT_270:
+				vidrot = 3;
+				break;
+			}
+		}
+
+		if (rotation == OMAP_DSS_ROT_90 || rotation == OMAP_DSS_ROT_270)
+			row_repeat = true;
+		else
+			row_repeat = false;
+	}
+
+	/*
+	 * OMAP4/5 Errata i631:
+	 * NV12 in 1D mode must use ROTATION=1. Otherwise DSS will fetch extra
+	 * rows beyond the framebuffer, which may cause OCP error.
+	 */
+	if (color_mode == OMAP_DSS_COLOR_NV12 &&
+			rotation_type != OMAP_DSS_ROT_TILER)
+		vidrot = 1;
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), vidrot, 13, 12);
+	if (dss_has_feature(FEAT_ROWREPEATENABLE))
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane),
+			row_repeat ? 1 : 0, 18, 18);
+
+	if (color_mode == OMAP_DSS_COLOR_NV12) {
+		bool doublestride = (rotation_type == OMAP_DSS_ROT_TILER) &&
+					(rotation == OMAP_DSS_ROT_0 ||
+					rotation == OMAP_DSS_ROT_180);
+		/* DOUBLESTRIDE */
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), doublestride, 22, 22);
+	}
+
+}
+
+static int color_mode_to_bpp(enum omap_color_mode color_mode)
+{
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_CLUT1:
+		return 1;
+	case OMAP_DSS_COLOR_CLUT2:
+		return 2;
+	case OMAP_DSS_COLOR_CLUT4:
+		return 4;
+	case OMAP_DSS_COLOR_CLUT8:
+	case OMAP_DSS_COLOR_NV12:
+		return 8;
+	case OMAP_DSS_COLOR_RGB12U:
+	case OMAP_DSS_COLOR_RGB16:
+	case OMAP_DSS_COLOR_ARGB16:
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+	case OMAP_DSS_COLOR_RGBA16:
+	case OMAP_DSS_COLOR_RGBX16:
+	case OMAP_DSS_COLOR_ARGB16_1555:
+	case OMAP_DSS_COLOR_XRGB16_1555:
+		return 16;
+	case OMAP_DSS_COLOR_RGB24P:
+		return 24;
+	case OMAP_DSS_COLOR_RGB24U:
+	case OMAP_DSS_COLOR_ARGB32:
+	case OMAP_DSS_COLOR_RGBA32:
+	case OMAP_DSS_COLOR_RGBX32:
+		return 32;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static s32 pixinc(int pixels, u8 ps)
+{
+	if (pixels == 1)
+		return 1;
+	else if (pixels > 1)
+		return 1 + (pixels - 1) * ps;
+	else if (pixels < 0)
+		return 1 - (-pixels + 1) * ps;
+	else
+		BUG();
+		return 0;
+}
+
+static void calc_vrfb_rotation_offset(u8 rotation, bool mirror,
+		u16 screen_width,
+		u16 width, u16 height,
+		enum omap_color_mode color_mode, bool fieldmode,
+		unsigned int field_offset,
+		unsigned *offset0, unsigned *offset1,
+		s32 *row_inc, s32 *pix_inc, int x_predecim, int y_predecim)
+{
+	u8 ps;
+
+	/* FIXME CLUT formats */
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_CLUT1:
+	case OMAP_DSS_COLOR_CLUT2:
+	case OMAP_DSS_COLOR_CLUT4:
+	case OMAP_DSS_COLOR_CLUT8:
+		BUG();
+		return;
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+		ps = 4;
+		break;
+	default:
+		ps = color_mode_to_bpp(color_mode) / 8;
+		break;
+	}
+
+	DSSDBG("calc_rot(%d): scrw %d, %dx%d\n", rotation, screen_width,
+			width, height);
+
+	/*
+	 * field 0 = even field = bottom field
+	 * field 1 = odd field = top field
+	 */
+	switch (rotation + mirror * 4) {
+	case OMAP_DSS_ROT_0:
+	case OMAP_DSS_ROT_180:
+		/*
+		 * If the pixel format is YUV or UYVY divide the width
+		 * of the image by 2 for 0 and 180 degree rotation.
+		 */
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			width = width >> 1;
+	case OMAP_DSS_ROT_90:
+	case OMAP_DSS_ROT_270:
+		*offset1 = 0;
+		if (field_offset)
+			*offset0 = field_offset * screen_width * ps;
+		else
+			*offset0 = 0;
+
+		*row_inc = pixinc(1 +
+			(y_predecim * screen_width - x_predecim * width) +
+			(fieldmode ? screen_width : 0), ps);
+		*pix_inc = pixinc(x_predecim, ps);
+		break;
+
+	case OMAP_DSS_ROT_0 + 4:
+	case OMAP_DSS_ROT_180 + 4:
+		/* If the pixel format is YUV or UYVY divide the width
+		 * of the image by 2  for 0 degree and 180 degree
+		 */
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			width = width >> 1;
+	case OMAP_DSS_ROT_90 + 4:
+	case OMAP_DSS_ROT_270 + 4:
+		*offset1 = 0;
+		if (field_offset)
+			*offset0 = field_offset * screen_width * ps;
+		else
+			*offset0 = 0;
+		*row_inc = pixinc(1 -
+			(y_predecim * screen_width + x_predecim * width) -
+			(fieldmode ? screen_width : 0), ps);
+		*pix_inc = pixinc(x_predecim, ps);
+		break;
+
+	default:
+		BUG();
+		return;
+	}
+}
+
+static void calc_dma_rotation_offset(u8 rotation, bool mirror,
+		u16 screen_width,
+		u16 width, u16 height,
+		enum omap_color_mode color_mode, bool fieldmode,
+		unsigned int field_offset,
+		unsigned *offset0, unsigned *offset1,
+		s32 *row_inc, s32 *pix_inc, int x_predecim, int y_predecim)
+{
+	u8 ps;
+	u16 fbw, fbh;
+
+	/* FIXME CLUT formats */
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_CLUT1:
+	case OMAP_DSS_COLOR_CLUT2:
+	case OMAP_DSS_COLOR_CLUT4:
+	case OMAP_DSS_COLOR_CLUT8:
+		BUG();
+		return;
+	default:
+		ps = color_mode_to_bpp(color_mode) / 8;
+		break;
+	}
+
+	DSSDBG("calc_rot(%d): scrw %d, %dx%d\n", rotation, screen_width,
+			width, height);
+
+	/* width & height are overlay sizes, convert to fb sizes */
+
+	if (rotation == OMAP_DSS_ROT_0 || rotation == OMAP_DSS_ROT_180) {
+		fbw = width;
+		fbh = height;
+	} else {
+		fbw = height;
+		fbh = width;
+	}
+
+	/*
+	 * field 0 = even field = bottom field
+	 * field 1 = odd field = top field
+	 */
+	switch (rotation + mirror * 4) {
+	case OMAP_DSS_ROT_0:
+		*offset1 = 0;
+		if (field_offset)
+			*offset0 = *offset1 + field_offset * screen_width * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(1 +
+			(y_predecim * screen_width - fbw * x_predecim) +
+			(fieldmode ? screen_width : 0),	ps);
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			*pix_inc = pixinc(x_predecim, 2 * ps);
+		else
+			*pix_inc = pixinc(x_predecim, ps);
+		break;
+	case OMAP_DSS_ROT_90:
+		*offset1 = screen_width * (fbh - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 + field_offset * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(screen_width * (fbh * x_predecim - 1) +
+				y_predecim + (fieldmode ? 1 : 0), ps);
+		*pix_inc = pixinc(-x_predecim * screen_width, ps);
+		break;
+	case OMAP_DSS_ROT_180:
+		*offset1 = (screen_width * (fbh - 1) + fbw - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 - field_offset * screen_width * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(-1 -
+			(y_predecim * screen_width - fbw * x_predecim) -
+			(fieldmode ? screen_width : 0),	ps);
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			*pix_inc = pixinc(-x_predecim, 2 * ps);
+		else
+			*pix_inc = pixinc(-x_predecim, ps);
+		break;
+	case OMAP_DSS_ROT_270:
+		*offset1 = (fbw - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 - field_offset * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(-screen_width * (fbh * x_predecim - 1) -
+				y_predecim - (fieldmode ? 1 : 0), ps);
+		*pix_inc = pixinc(x_predecim * screen_width, ps);
+		break;
+
+	/* mirroring */
+	case OMAP_DSS_ROT_0 + 4:
+		*offset1 = (fbw - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 + field_offset * screen_width * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(y_predecim * screen_width * 2 - 1 +
+				(fieldmode ? screen_width : 0),
+				ps);
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			*pix_inc = pixinc(-x_predecim, 2 * ps);
+		else
+			*pix_inc = pixinc(-x_predecim, ps);
+		break;
+
+	case OMAP_DSS_ROT_90 + 4:
+		*offset1 = 0;
+		if (field_offset)
+			*offset0 = *offset1 + field_offset * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(-screen_width * (fbh * x_predecim - 1) +
+				y_predecim + (fieldmode ? 1 : 0),
+				ps);
+		*pix_inc = pixinc(x_predecim * screen_width, ps);
+		break;
+
+	case OMAP_DSS_ROT_180 + 4:
+		*offset1 = screen_width * (fbh - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 - field_offset * screen_width * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(1 - y_predecim * screen_width * 2 -
+				(fieldmode ? screen_width : 0),
+				ps);
+		if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY)
+			*pix_inc = pixinc(x_predecim, 2 * ps);
+		else
+			*pix_inc = pixinc(x_predecim, ps);
+		break;
+
+	case OMAP_DSS_ROT_270 + 4:
+		*offset1 = (screen_width * (fbh - 1) + fbw - 1) * ps;
+		if (field_offset)
+			*offset0 = *offset1 - field_offset * ps;
+		else
+			*offset0 = *offset1;
+		*row_inc = pixinc(screen_width * (fbh * x_predecim - 1) -
+				y_predecim - (fieldmode ? 1 : 0),
+				ps);
+		*pix_inc = pixinc(-x_predecim * screen_width, ps);
+		break;
+
+	default:
+		BUG();
+		return;
+	}
+}
+
+static void calc_tiler_rotation_offset(u16 screen_width, u16 width,
+		enum omap_color_mode color_mode, bool fieldmode,
+		unsigned int field_offset, unsigned *offset0, unsigned *offset1,
+		s32 *row_inc, s32 *pix_inc, int x_predecim, int y_predecim)
+{
+	u8 ps;
+
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_CLUT1:
+	case OMAP_DSS_COLOR_CLUT2:
+	case OMAP_DSS_COLOR_CLUT4:
+	case OMAP_DSS_COLOR_CLUT8:
+		BUG();
+		return;
+	default:
+		ps = color_mode_to_bpp(color_mode) / 8;
+		break;
+	}
+
+	DSSDBG("scrw %d, width %d\n", screen_width, width);
+
+	/*
+	 * field 0 = even field = bottom field
+	 * field 1 = odd field = top field
+	 */
+	*offset1 = 0;
+	if (field_offset)
+		*offset0 = *offset1 + field_offset * screen_width * ps;
+	else
+		*offset0 = *offset1;
+	*row_inc = pixinc(1 + (y_predecim * screen_width - width * x_predecim) +
+			(fieldmode ? screen_width : 0), ps);
+	if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+		color_mode == OMAP_DSS_COLOR_UYVY)
+		*pix_inc = pixinc(x_predecim, 2 * ps);
+	else
+		*pix_inc = pixinc(x_predecim, ps);
+}
+
+/*
+ * This function is used to avoid synclosts in OMAP3, because of some
+ * undocumented horizontal position and timing related limitations.
+ */
+static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
+		const struct omap_video_timings *t, u16 pos_x,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		bool five_taps)
+{
+	const int ds = DIV_ROUND_UP(height, out_height);
+	unsigned long nonactive;
+	static const u8 limits[3] = { 8, 10, 20 };
+	u64 val, blank;
+	int i;
+
+	nonactive = t->x_res + t->hfp + t->hsw + t->hbp - out_width;
+
+	i = 0;
+	if (out_height < height)
+		i++;
+	if (out_width < width)
+		i++;
+	blank = div_u64((u64)(t->hbp + t->hsw + t->hfp) * lclk, pclk);
+	DSSDBG("blanking period + ppl = %llu (limit = %u)\n", blank, limits[i]);
+	if (blank <= limits[i])
+		return -EINVAL;
+
+	/* FIXME add checks for 3-tap filter once the limitations are known */
+	if (!five_taps)
+		return 0;
+
+	/*
+	 * Pixel data should be prepared before visible display point starts.
+	 * So, atleast DS-2 lines must have already been fetched by DISPC
+	 * during nonactive - pos_x period.
+	 */
+	val = div_u64((u64)(nonactive - pos_x) * lclk, pclk);
+	DSSDBG("(nonactive - pos_x) * pcd = %llu max(0, DS - 2) * width = %d\n",
+		val, max(0, ds - 2) * width);
+	if (val < max(0, ds - 2) * width)
+		return -EINVAL;
+
+	/*
+	 * All lines need to be refilled during the nonactive period of which
+	 * only one line can be loaded during the active period. So, atleast
+	 * DS - 1 lines should be loaded during nonactive period.
+	 */
+	val =  div_u64((u64)nonactive * lclk, pclk);
+	DSSDBG("nonactive * pcd  = %llu, max(0, DS - 1) * width = %d\n",
+		val, max(0, ds - 1) * width);
+	if (val < max(0, ds - 1) * width)
+		return -EINVAL;
+
+	return 0;
+}
+
+static unsigned long calc_core_clk_five_taps(unsigned long pclk,
+		const struct omap_video_timings *mgr_timings, u16 width,
+		u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode)
+{
+	u32 core_clk = 0;
+	u64 tmp;
+
+	if (height <= out_height && width <= out_width)
+		return (unsigned long) pclk;
+
+	if (height > out_height) {
+		unsigned int ppl = mgr_timings->x_res;
+
+		tmp = (u64)pclk * height * out_width;
+		do_div(tmp, 2 * out_height * ppl);
+		core_clk = tmp;
+
+		if (height > 2 * out_height) {
+			if (ppl == out_width)
+				return 0;
+
+			tmp = (u64)pclk * (height - 2 * out_height) * out_width;
+			do_div(tmp, 2 * out_height * (ppl - out_width));
+			core_clk = max_t(u32, core_clk, tmp);
+		}
+	}
+
+	if (width > out_width) {
+		tmp = (u64)pclk * width;
+		do_div(tmp, out_width);
+		core_clk = max_t(u32, core_clk, tmp);
+
+		if (color_mode == OMAP_DSS_COLOR_RGB24U)
+			core_clk <<= 1;
+	}
+
+	return core_clk;
+}
+
+static unsigned long calc_core_clk_24xx(unsigned long pclk, u16 width,
+		u16 height, u16 out_width, u16 out_height, bool mem_to_mem)
+{
+	if (height > out_height && width > out_width)
+		return pclk * 4;
+	else
+		return pclk * 2;
+}
+
+static unsigned long calc_core_clk_34xx(unsigned long pclk, u16 width,
+		u16 height, u16 out_width, u16 out_height, bool mem_to_mem)
+{
+	unsigned int hf, vf;
+
+	/*
+	 * FIXME how to determine the 'A' factor
+	 * for the no downscaling case ?
+	 */
+
+	if (width > 3 * out_width)
+		hf = 4;
+	else if (width > 2 * out_width)
+		hf = 3;
+	else if (width > out_width)
+		hf = 2;
+	else
+		hf = 1;
+	if (height > out_height)
+		vf = 2;
+	else
+		vf = 1;
+
+	return pclk * vf * hf;
+}
+
+static unsigned long calc_core_clk_44xx(unsigned long pclk, u16 width,
+		u16 height, u16 out_width, u16 out_height, bool mem_to_mem)
+{
+	/*
+	 * If the overlay/writeback is in mem to mem mode, there are no
+	 * downscaling limitations with respect to pixel clock, return 1 as
+	 * required core clock to represent that we have sufficient enough
+	 * core clock to do maximum downscaling
+	 */
+	if (mem_to_mem)
+		return 1;
+
+	if (width > out_width)
+		return DIV_ROUND_UP(pclk, out_width) * width;
+	else
+		return pclk;
+}
+
+static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk,
+		const struct omap_video_timings *mgr_timings,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode, bool *five_taps,
+		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
+		u16 pos_x, unsigned long *core_clk, bool mem_to_mem)
+{
+	int error;
+	u16 in_width, in_height;
+	int min_factor = min(*decim_x, *decim_y);
+	const int maxsinglelinewidth =
+			dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
+
+	*five_taps = false;
+
+	do {
+		in_height = height / *decim_y;
+		in_width = width / *decim_x;
+		*core_clk = dispc.feat->calc_core_clk(pclk, in_width,
+				in_height, out_width, out_height, mem_to_mem);
+		error = (in_width > maxsinglelinewidth || !*core_clk ||
+			*core_clk > dispc_core_clk_rate());
+		if (error) {
+			if (*decim_x == *decim_y) {
+				*decim_x = min_factor;
+				++*decim_y;
+			} else {
+				swap(*decim_x, *decim_y);
+				if (*decim_x < *decim_y)
+					++*decim_x;
+			}
+		}
+	} while (*decim_x <= *x_predecim && *decim_y <= *y_predecim && error);
+
+	if (error) {
+		DSSERR("failed to find scaling settings\n");
+		return -EINVAL;
+	}
+
+	if (in_width > maxsinglelinewidth) {
+		DSSERR("Cannot scale max input width exceeded");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk,
+		const struct omap_video_timings *mgr_timings,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode, bool *five_taps,
+		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
+		u16 pos_x, unsigned long *core_clk, bool mem_to_mem)
+{
+	int error;
+	u16 in_width, in_height;
+	const int maxsinglelinewidth =
+			dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
+
+	do {
+		in_height = height / *decim_y;
+		in_width = width / *decim_x;
+		*five_taps = in_height > out_height;
+
+		if (in_width > maxsinglelinewidth)
+			if (in_height > out_height &&
+						in_height < out_height * 2)
+				*five_taps = false;
+again:
+		if (*five_taps)
+			*core_clk = calc_core_clk_five_taps(pclk, mgr_timings,
+						in_width, in_height, out_width,
+						out_height, color_mode);
+		else
+			*core_clk = dispc.feat->calc_core_clk(pclk, in_width,
+					in_height, out_width, out_height,
+					mem_to_mem);
+
+		error = check_horiz_timing_omap3(pclk, lclk, mgr_timings,
+				pos_x, in_width, in_height, out_width,
+				out_height, *five_taps);
+		if (error && *five_taps) {
+			*five_taps = false;
+			goto again;
+		}
+
+		error = (error || in_width > maxsinglelinewidth * 2 ||
+			(in_width > maxsinglelinewidth && *five_taps) ||
+			!*core_clk || *core_clk > dispc_core_clk_rate());
+
+		if (!error) {
+			/* verify that we're inside the limits of scaler */
+			if (in_width / 4 > out_width)
+					error = 1;
+
+			if (*five_taps) {
+				if (in_height / 4 > out_height)
+					error = 1;
+			} else {
+				if (in_height / 2 > out_height)
+					error = 1;
+			}
+		}
+
+		if (error)
+			++*decim_y;
+	} while (*decim_x <= *x_predecim && *decim_y <= *y_predecim && error);
+
+	if (error) {
+		DSSERR("failed to find scaling settings\n");
+		return -EINVAL;
+	}
+
+	if (check_horiz_timing_omap3(pclk, lclk, mgr_timings, pos_x, in_width,
+				in_height, out_width, out_height, *five_taps)) {
+			DSSERR("horizontal timing too tight\n");
+			return -EINVAL;
+	}
+
+	if (in_width > (maxsinglelinewidth * 2)) {
+		DSSERR("Cannot setup scaling");
+		DSSERR("width exceeds maximum width possible");
+		return -EINVAL;
+	}
+
+	if (in_width > maxsinglelinewidth && *five_taps) {
+		DSSERR("cannot setup scaling with five taps");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk,
+		const struct omap_video_timings *mgr_timings,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode, bool *five_taps,
+		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
+		u16 pos_x, unsigned long *core_clk, bool mem_to_mem)
+{
+	u16 in_width, in_width_max;
+	int decim_x_min = *decim_x;
+	u16 in_height = height / *decim_y;
+	const int maxsinglelinewidth =
+				dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
+	const int maxdownscale = dss_feat_get_param_max(FEAT_PARAM_DOWNSCALE);
+
+	if (mem_to_mem) {
+		in_width_max = out_width * maxdownscale;
+	} else {
+		in_width_max = dispc_core_clk_rate() /
+					DIV_ROUND_UP(pclk, out_width);
+	}
+
+	*decim_x = DIV_ROUND_UP(width, in_width_max);
+
+	*decim_x = *decim_x > decim_x_min ? *decim_x : decim_x_min;
+	if (*decim_x > *x_predecim)
+		return -EINVAL;
+
+	do {
+		in_width = width / *decim_x;
+	} while (*decim_x <= *x_predecim &&
+			in_width > maxsinglelinewidth && ++*decim_x);
+
+	if (in_width > maxsinglelinewidth) {
+		DSSERR("Cannot scale width exceeds max line width");
+		return -EINVAL;
+	}
+
+	*core_clk = dispc.feat->calc_core_clk(pclk, in_width, in_height,
+				out_width, out_height, mem_to_mem);
+	return 0;
+}
+
+#define DIV_FRAC(dividend, divisor) \
+	((dividend) * 100 / (divisor) - ((dividend) / (divisor) * 100))
+
+static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
+		enum omap_overlay_caps caps,
+		const struct omap_video_timings *mgr_timings,
+		u16 width, u16 height, u16 out_width, u16 out_height,
+		enum omap_color_mode color_mode, bool *five_taps,
+		int *x_predecim, int *y_predecim, u16 pos_x,
+		enum omap_dss_rotation_type rotation_type, bool mem_to_mem)
+{
+	const int maxdownscale = dss_feat_get_param_max(FEAT_PARAM_DOWNSCALE);
+	const int max_decim_limit = 16;
+	unsigned long core_clk = 0;
+	int decim_x, decim_y, ret;
+
+	if (width == out_width && height == out_height)
+		return 0;
+
+	if (!mem_to_mem && (pclk == 0 || mgr_timings->pixelclock == 0)) {
+		DSSERR("cannot calculate scaling settings: pclk is zero\n");
+		return -EINVAL;
+	}
+
+	if ((caps & OMAP_DSS_OVL_CAP_SCALE) == 0)
+		return -EINVAL;
+
+	if (mem_to_mem) {
+		*x_predecim = *y_predecim = 1;
+	} else {
+		*x_predecim = max_decim_limit;
+		*y_predecim = (rotation_type == OMAP_DSS_ROT_TILER &&
+				dss_has_feature(FEAT_BURST_2D)) ?
+				2 : max_decim_limit;
+	}
+
+	if (color_mode == OMAP_DSS_COLOR_CLUT1 ||
+	    color_mode == OMAP_DSS_COLOR_CLUT2 ||
+	    color_mode == OMAP_DSS_COLOR_CLUT4 ||
+	    color_mode == OMAP_DSS_COLOR_CLUT8) {
+		*x_predecim = 1;
+		*y_predecim = 1;
+		*five_taps = false;
+		return 0;
+	}
+
+	decim_x = DIV_ROUND_UP(DIV_ROUND_UP(width, out_width), maxdownscale);
+	decim_y = DIV_ROUND_UP(DIV_ROUND_UP(height, out_height), maxdownscale);
+
+	if (decim_x > *x_predecim || out_width > width * 8)
+		return -EINVAL;
+
+	if (decim_y > *y_predecim || out_height > height * 8)
+		return -EINVAL;
+
+	ret = dispc.feat->calc_scaling(pclk, lclk, mgr_timings, width, height,
+		out_width, out_height, color_mode, five_taps,
+		x_predecim, y_predecim, &decim_x, &decim_y, pos_x, &core_clk,
+		mem_to_mem);
+	if (ret)
+		return ret;
+
+	DSSDBG("%dx%d -> %dx%d (%d.%02d x %d.%02d), decim %dx%d %dx%d (%d.%02d x %d.%02d), taps %d, req clk %lu, cur clk %lu\n",
+		width, height,
+		out_width, out_height,
+		out_width / width, DIV_FRAC(out_width, width),
+		out_height / height, DIV_FRAC(out_height, height),
+
+		decim_x, decim_y,
+		width / decim_x, height / decim_y,
+		out_width / (width / decim_x), DIV_FRAC(out_width, width / decim_x),
+		out_height / (height / decim_y), DIV_FRAC(out_height, height / decim_y),
+
+		*five_taps ? 5 : 3,
+		core_clk, dispc_core_clk_rate());
+
+	if (!core_clk || core_clk > dispc_core_clk_rate()) {
+		DSSERR("failed to set up scaling, "
+			"required core clk rate = %lu Hz, "
+			"current core clk rate = %lu Hz\n",
+			core_clk, dispc_core_clk_rate());
+		return -EINVAL;
+	}
+
+	*x_predecim = decim_x;
+	*y_predecim = decim_y;
+	return 0;
+}
+
+int dispc_ovl_check(enum omap_plane plane, enum omap_channel channel,
+		const struct omap_overlay_info *oi,
+		const struct omap_video_timings *timings,
+		int *x_predecim, int *y_predecim)
+{
+	enum omap_overlay_caps caps = dss_feat_get_overlay_caps(plane);
+	bool five_taps = true;
+	bool fieldmode = false;
+	u16 in_height = oi->height;
+	u16 in_width = oi->width;
+	bool ilace = timings->interlace;
+	u16 out_width, out_height;
+	int pos_x = oi->pos_x;
+	unsigned long pclk = dispc_mgr_pclk_rate(channel);
+	unsigned long lclk = dispc_mgr_lclk_rate(channel);
+
+	out_width = oi->out_width == 0 ? oi->width : oi->out_width;
+	out_height = oi->out_height == 0 ? oi->height : oi->out_height;
+
+	if (ilace && oi->height == out_height)
+		fieldmode = true;
+
+	if (ilace) {
+		if (fieldmode)
+			in_height /= 2;
+		out_height /= 2;
+
+		DSSDBG("adjusting for ilace: height %d, out_height %d\n",
+				in_height, out_height);
+	}
+
+	if (!dss_feat_color_mode_supported(plane, oi->color_mode))
+		return -EINVAL;
+
+	return dispc_ovl_calc_scaling(pclk, lclk, caps, timings, in_width,
+			in_height, out_width, out_height, oi->color_mode,
+			&five_taps, x_predecim, y_predecim, pos_x,
+			oi->rotation_type, false);
+}
+EXPORT_SYMBOL(dispc_ovl_check);
+
+static int dispc_ovl_setup_common(enum omap_plane plane,
+		enum omap_overlay_caps caps, u32 paddr, u32 p_uv_addr,
+		u16 screen_width, int pos_x, int pos_y, u16 width, u16 height,
+		u16 out_width, u16 out_height, enum omap_color_mode color_mode,
+		u8 rotation, bool mirror, u8 zorder, u8 pre_mult_alpha,
+		u8 global_alpha, enum omap_dss_rotation_type rotation_type,
+		bool replication, const struct omap_video_timings *mgr_timings,
+		bool mem_to_mem)
+{
+	bool five_taps = true;
+	bool fieldmode = false;
+	int r, cconv = 0;
+	unsigned offset0, offset1;
+	s32 row_inc;
+	s32 pix_inc;
+	u16 frame_width, frame_height;
+	unsigned int field_offset = 0;
+	u16 in_height = height;
+	u16 in_width = width;
+	int x_predecim = 1, y_predecim = 1;
+	bool ilace = mgr_timings->interlace;
+	unsigned long pclk = dispc_plane_pclk_rate(plane);
+	unsigned long lclk = dispc_plane_lclk_rate(plane);
+
+	if (paddr == 0 && rotation_type != OMAP_DSS_ROT_TILER)
+		return -EINVAL;
+
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+	case OMAP_DSS_COLOR_NV12:
+		if (in_width & 1) {
+			DSSERR("input width %d is not even for YUV format\n",
+				in_width);
+			return -EINVAL;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	out_width = out_width == 0 ? width : out_width;
+	out_height = out_height == 0 ? height : out_height;
+
+	if (ilace && height == out_height)
+		fieldmode = true;
+
+	if (ilace) {
+		if (fieldmode)
+			in_height /= 2;
+		pos_y /= 2;
+		out_height /= 2;
+
+		DSSDBG("adjusting for ilace: height %d, pos_y %d, "
+			"out_height %d\n", in_height, pos_y,
+			out_height);
+	}
+
+	if (!dss_feat_color_mode_supported(plane, color_mode))
+		return -EINVAL;
+
+	r = dispc_ovl_calc_scaling(pclk, lclk, caps, mgr_timings, in_width,
+			in_height, out_width, out_height, color_mode,
+			&five_taps, &x_predecim, &y_predecim, pos_x,
+			rotation_type, mem_to_mem);
+	if (r)
+		return r;
+
+	in_width = in_width / x_predecim;
+	in_height = in_height / y_predecim;
+
+	if (x_predecim > 1 || y_predecim > 1)
+		DSSDBG("predecimation %d x %x, new input size %d x %d\n",
+			x_predecim, y_predecim, in_width, in_height);
+
+	switch (color_mode) {
+	case OMAP_DSS_COLOR_YUV2:
+	case OMAP_DSS_COLOR_UYVY:
+	case OMAP_DSS_COLOR_NV12:
+		if (in_width & 1) {
+			DSSDBG("predecimated input width is not even for YUV format\n");
+			DSSDBG("adjusting input width %d -> %d\n",
+				in_width, in_width & ~1);
+
+			in_width &= ~1;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	if (color_mode == OMAP_DSS_COLOR_YUV2 ||
+			color_mode == OMAP_DSS_COLOR_UYVY ||
+			color_mode == OMAP_DSS_COLOR_NV12)
+		cconv = 1;
+
+	if (ilace && !fieldmode) {
+		/*
+		 * when downscaling the bottom field may have to start several
+		 * source lines below the top field. Unfortunately ACCUI
+		 * registers will only hold the fractional part of the offset
+		 * so the integer part must be added to the base address of the
+		 * bottom field.
+		 */
+		if (!in_height || in_height == out_height)
+			field_offset = 0;
+		else
+			field_offset = in_height / out_height / 2;
+	}
+
+	/* Fields are independent but interleaved in memory. */
+	if (fieldmode)
+		field_offset = 1;
+
+	offset0 = 0;
+	offset1 = 0;
+	row_inc = 0;
+	pix_inc = 0;
+
+	if (plane == OMAP_DSS_WB) {
+		frame_width = out_width;
+		frame_height = out_height;
+	} else {
+		frame_width = in_width;
+		frame_height = height;
+	}
+
+	if (rotation_type == OMAP_DSS_ROT_TILER)
+		calc_tiler_rotation_offset(screen_width, frame_width,
+				color_mode, fieldmode, field_offset,
+				&offset0, &offset1, &row_inc, &pix_inc,
+				x_predecim, y_predecim);
+	else if (rotation_type == OMAP_DSS_ROT_DMA)
+		calc_dma_rotation_offset(rotation, mirror, screen_width,
+				frame_width, frame_height,
+				color_mode, fieldmode, field_offset,
+				&offset0, &offset1, &row_inc, &pix_inc,
+				x_predecim, y_predecim);
+	else
+		calc_vrfb_rotation_offset(rotation, mirror,
+				screen_width, frame_width, frame_height,
+				color_mode, fieldmode, field_offset,
+				&offset0, &offset1, &row_inc, &pix_inc,
+				x_predecim, y_predecim);
+
+	DSSDBG("offset0 %u, offset1 %u, row_inc %d, pix_inc %d\n",
+			offset0, offset1, row_inc, pix_inc);
+
+	dispc_ovl_set_color_mode(plane, color_mode);
+
+	dispc_ovl_configure_burst_type(plane, rotation_type);
+
+	dispc_ovl_set_ba0(plane, paddr + offset0);
+	dispc_ovl_set_ba1(plane, paddr + offset1);
+
+	if (OMAP_DSS_COLOR_NV12 == color_mode) {
+		dispc_ovl_set_ba0_uv(plane, p_uv_addr + offset0);
+		dispc_ovl_set_ba1_uv(plane, p_uv_addr + offset1);
+	}
+
+	if (dispc.feat->last_pixel_inc_missing)
+		row_inc += pix_inc - 1;
+
+	dispc_ovl_set_row_inc(plane, row_inc);
+	dispc_ovl_set_pix_inc(plane, pix_inc);
+
+	DSSDBG("%d,%d %dx%d -> %dx%d\n", pos_x, pos_y, in_width,
+			in_height, out_width, out_height);
+
+	dispc_ovl_set_pos(plane, caps, pos_x, pos_y);
+
+	dispc_ovl_set_input_size(plane, in_width, in_height);
+
+	if (caps & OMAP_DSS_OVL_CAP_SCALE) {
+		dispc_ovl_set_scaling(plane, in_width, in_height, out_width,
+				   out_height, ilace, five_taps, fieldmode,
+				   color_mode, rotation);
+		dispc_ovl_set_output_size(plane, out_width, out_height);
+		dispc_ovl_set_vid_color_conv(plane, cconv);
+	}
+
+	dispc_ovl_set_rotation_attrs(plane, rotation, rotation_type, mirror,
+			color_mode);
+
+	dispc_ovl_set_zorder(plane, caps, zorder);
+	dispc_ovl_set_pre_mult_alpha(plane, caps, pre_mult_alpha);
+	dispc_ovl_setup_global_alpha(plane, caps, global_alpha);
+
+	dispc_ovl_enable_replication(plane, caps, replication);
+
+	return 0;
+}
+
+int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi,
+		bool replication, const struct omap_video_timings *mgr_timings,
+		bool mem_to_mem)
+{
+	int r;
+	enum omap_overlay_caps caps = dss_feat_get_overlay_caps(plane);
+	enum omap_channel channel;
+
+	channel = dispc_ovl_get_channel_out(plane);
+
+	DSSDBG("dispc_ovl_setup %d, pa %pad, pa_uv %pad, sw %d, %d,%d, %dx%d ->"
+		" %dx%d, cmode %x, rot %d, mir %d, chan %d repl %d\n",
+		plane, &oi->paddr, &oi->p_uv_addr, oi->screen_width, oi->pos_x,
+		oi->pos_y, oi->width, oi->height, oi->out_width, oi->out_height,
+		oi->color_mode, oi->rotation, oi->mirror, channel, replication);
+
+	r = dispc_ovl_setup_common(plane, caps, oi->paddr, oi->p_uv_addr,
+		oi->screen_width, oi->pos_x, oi->pos_y, oi->width, oi->height,
+		oi->out_width, oi->out_height, oi->color_mode, oi->rotation,
+		oi->mirror, oi->zorder, oi->pre_mult_alpha, oi->global_alpha,
+		oi->rotation_type, replication, mgr_timings, mem_to_mem);
+
+	return r;
+}
+EXPORT_SYMBOL(dispc_ovl_setup);
+
+int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
+		bool mem_to_mem, const struct omap_video_timings *mgr_timings)
+{
+	int r;
+	u32 l;
+	enum omap_plane plane = OMAP_DSS_WB;
+	const int pos_x = 0, pos_y = 0;
+	const u8 zorder = 0, global_alpha = 0;
+	const bool replication = false;
+	bool truncation;
+	int in_width = mgr_timings->x_res;
+	int in_height = mgr_timings->y_res;
+	enum omap_overlay_caps caps =
+		OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA;
+
+	DSSDBG("dispc_wb_setup, pa %x, pa_uv %x, %d,%d -> %dx%d, cmode %x, "
+		"rot %d, mir %d\n", wi->paddr, wi->p_uv_addr, in_width,
+		in_height, wi->width, wi->height, wi->color_mode, wi->rotation,
+		wi->mirror);
+
+	r = dispc_ovl_setup_common(plane, caps, wi->paddr, wi->p_uv_addr,
+		wi->buf_width, pos_x, pos_y, in_width, in_height, wi->width,
+		wi->height, wi->color_mode, wi->rotation, wi->mirror, zorder,
+		wi->pre_mult_alpha, global_alpha, wi->rotation_type,
+		replication, mgr_timings, mem_to_mem);
+
+	switch (wi->color_mode) {
+	case OMAP_DSS_COLOR_RGB16:
+	case OMAP_DSS_COLOR_RGB24P:
+	case OMAP_DSS_COLOR_ARGB16:
+	case OMAP_DSS_COLOR_RGBA16:
+	case OMAP_DSS_COLOR_RGB12U:
+	case OMAP_DSS_COLOR_ARGB16_1555:
+	case OMAP_DSS_COLOR_XRGB16_1555:
+	case OMAP_DSS_COLOR_RGBX16:
+		truncation = true;
+		break;
+	default:
+		truncation = false;
+		break;
+	}
+
+	/* setup extra DISPC_WB_ATTRIBUTES */
+	l = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
+	l = FLD_MOD(l, truncation, 10, 10);	/* TRUNCATIONENABLE */
+	l = FLD_MOD(l, mem_to_mem, 19, 19);	/* WRITEBACKMODE */
+	if (mem_to_mem)
+		l = FLD_MOD(l, 1, 26, 24);	/* CAPTUREMODE */
+	else
+		l = FLD_MOD(l, 0, 26, 24);	/* CAPTUREMODE */
+	dispc_write_reg(DISPC_OVL_ATTRIBUTES(plane), l);
+
+	if (mem_to_mem) {
+		/* WBDELAYCOUNT */
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), 0, 7, 0);
+	} else {
+		int wbdelay;
+
+		wbdelay = min(mgr_timings->vfp + mgr_timings->vsw +
+			mgr_timings->vbp, 255);
+
+		/* WBDELAYCOUNT */
+		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), wbdelay, 7, 0);
+	}
+
+	return r;
+}
+
+int dispc_ovl_enable(enum omap_plane plane, bool enable)
+{
+	DSSDBG("dispc_enable_plane %d, %d\n", plane, enable);
+
+	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), enable ? 1 : 0, 0, 0);
+
+	return 0;
+}
+EXPORT_SYMBOL(dispc_ovl_enable);
+
+bool dispc_ovl_enabled(enum omap_plane plane)
+{
+	return REG_GET(DISPC_OVL_ATTRIBUTES(plane), 0, 0);
+}
+EXPORT_SYMBOL(dispc_ovl_enabled);
+
+void dispc_mgr_enable(enum omap_channel channel, bool enable)
+{
+	mgr_fld_write(channel, DISPC_MGR_FLD_ENABLE, enable);
+	/* flush posted write */
+	mgr_fld_read(channel, DISPC_MGR_FLD_ENABLE);
+}
+EXPORT_SYMBOL(dispc_mgr_enable);
+
+bool dispc_mgr_is_enabled(enum omap_channel channel)
+{
+	return !!mgr_fld_read(channel, DISPC_MGR_FLD_ENABLE);
+}
+EXPORT_SYMBOL(dispc_mgr_is_enabled);
+
+void dispc_wb_enable(bool enable)
+{
+	dispc_ovl_enable(OMAP_DSS_WB, enable);
+}
+
+bool dispc_wb_is_enabled(void)
+{
+	return dispc_ovl_enabled(OMAP_DSS_WB);
+}
+
+static void dispc_lcd_enable_signal_polarity(bool act_high)
+{
+	if (!dss_has_feature(FEAT_LCDENABLEPOL))
+		return;
+
+	REG_FLD_MOD(DISPC_CONTROL, act_high ? 1 : 0, 29, 29);
+}
+
+void dispc_lcd_enable_signal(bool enable)
+{
+	if (!dss_has_feature(FEAT_LCDENABLESIGNAL))
+		return;
+
+	REG_FLD_MOD(DISPC_CONTROL, enable ? 1 : 0, 28, 28);
+}
+
+void dispc_pck_free_enable(bool enable)
+{
+	if (!dss_has_feature(FEAT_PCKFREEENABLE))
+		return;
+
+	REG_FLD_MOD(DISPC_CONTROL, enable ? 1 : 0, 27, 27);
+}
+
+static void dispc_mgr_enable_fifohandcheck(enum omap_channel channel, bool enable)
+{
+	mgr_fld_write(channel, DISPC_MGR_FLD_FIFOHANDCHECK, enable);
+}
+
+
+static void dispc_mgr_set_lcd_type_tft(enum omap_channel channel)
+{
+	mgr_fld_write(channel, DISPC_MGR_FLD_STNTFT, 1);
+}
+
+static void dispc_set_loadmode(enum omap_dss_load_mode mode)
+{
+	REG_FLD_MOD(DISPC_CONFIG, mode, 2, 1);
+}
+
+
+static void dispc_mgr_set_default_color(enum omap_channel channel, u32 color)
+{
+	dispc_write_reg(DISPC_DEFAULT_COLOR(channel), color);
+}
+
+static void dispc_mgr_set_trans_key(enum omap_channel ch,
+		enum omap_dss_trans_key_type type,
+		u32 trans_key)
+{
+	mgr_fld_write(ch, DISPC_MGR_FLD_TCKSELECTION, type);
+
+	dispc_write_reg(DISPC_TRANS_COLOR(ch), trans_key);
+}
+
+static void dispc_mgr_enable_trans_key(enum omap_channel ch, bool enable)
+{
+	mgr_fld_write(ch, DISPC_MGR_FLD_TCKENABLE, enable);
+}
+
+static void dispc_mgr_enable_alpha_fixed_zorder(enum omap_channel ch,
+		bool enable)
+{
+	if (!dss_has_feature(FEAT_ALPHA_FIXED_ZORDER))
+		return;
+
+	if (ch == OMAP_DSS_CHANNEL_LCD)
+		REG_FLD_MOD(DISPC_CONFIG, enable, 18, 18);
+	else if (ch == OMAP_DSS_CHANNEL_DIGIT)
+		REG_FLD_MOD(DISPC_CONFIG, enable, 19, 19);
+}
+
+void dispc_mgr_setup(enum omap_channel channel,
+		const struct omap_overlay_manager_info *info)
+{
+	dispc_mgr_set_default_color(channel, info->default_color);
+	dispc_mgr_set_trans_key(channel, info->trans_key_type, info->trans_key);
+	dispc_mgr_enable_trans_key(channel, info->trans_enabled);
+	dispc_mgr_enable_alpha_fixed_zorder(channel,
+			info->partial_alpha_enabled);
+	if (dss_has_feature(FEAT_CPR)) {
+		dispc_mgr_enable_cpr(channel, info->cpr_enable);
+		dispc_mgr_set_cpr_coef(channel, &info->cpr_coefs);
+	}
+}
+EXPORT_SYMBOL(dispc_mgr_setup);
+
+static void dispc_mgr_set_tft_data_lines(enum omap_channel channel, u8 data_lines)
+{
+	int code;
+
+	switch (data_lines) {
+	case 12:
+		code = 0;
+		break;
+	case 16:
+		code = 1;
+		break;
+	case 18:
+		code = 2;
+		break;
+	case 24:
+		code = 3;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	mgr_fld_write(channel, DISPC_MGR_FLD_TFTDATALINES, code);
+}
+
+static void dispc_mgr_set_io_pad_mode(enum dss_io_pad_mode mode)
+{
+	u32 l;
+	int gpout0, gpout1;
+
+	switch (mode) {
+	case DSS_IO_PAD_MODE_RESET:
+		gpout0 = 0;
+		gpout1 = 0;
+		break;
+	case DSS_IO_PAD_MODE_RFBI:
+		gpout0 = 1;
+		gpout1 = 0;
+		break;
+	case DSS_IO_PAD_MODE_BYPASS:
+		gpout0 = 1;
+		gpout1 = 1;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	l = dispc_read_reg(DISPC_CONTROL);
+	l = FLD_MOD(l, gpout0, 15, 15);
+	l = FLD_MOD(l, gpout1, 16, 16);
+	dispc_write_reg(DISPC_CONTROL, l);
+}
+
+static void dispc_mgr_enable_stallmode(enum omap_channel channel, bool enable)
+{
+	mgr_fld_write(channel, DISPC_MGR_FLD_STALLMODE, enable);
+}
+
+void dispc_mgr_set_lcd_config(enum omap_channel channel,
+		const struct dss_lcd_mgr_config *config)
+{
+	dispc_mgr_set_io_pad_mode(config->io_pad_mode);
+
+	dispc_mgr_enable_stallmode(channel, config->stallmode);
+	dispc_mgr_enable_fifohandcheck(channel, config->fifohandcheck);
+
+	dispc_mgr_set_clock_div(channel, &config->clock_info);
+
+	dispc_mgr_set_tft_data_lines(channel, config->video_port_width);
+
+	dispc_lcd_enable_signal_polarity(config->lcden_sig_polarity);
+
+	dispc_mgr_set_lcd_type_tft(channel);
+}
+EXPORT_SYMBOL(dispc_mgr_set_lcd_config);
+
+static bool _dispc_mgr_size_ok(u16 width, u16 height)
+{
+	return width <= dispc.feat->mgr_width_max &&
+		height <= dispc.feat->mgr_height_max;
+}
+
+static bool _dispc_lcd_timings_ok(int hsw, int hfp, int hbp,
+		int vsw, int vfp, int vbp)
+{
+	if (hsw < 1 || hsw > dispc.feat->sw_max ||
+			hfp < 1 || hfp > dispc.feat->hp_max ||
+			hbp < 1 || hbp > dispc.feat->hp_max ||
+			vsw < 1 || vsw > dispc.feat->sw_max ||
+			vfp < 0 || vfp > dispc.feat->vp_max ||
+			vbp < 0 || vbp > dispc.feat->vp_max)
+		return false;
+	return true;
+}
+
+static bool _dispc_mgr_pclk_ok(enum omap_channel channel,
+		unsigned long pclk)
+{
+	if (dss_mgr_is_lcd(channel))
+		return pclk <= dispc.feat->max_lcd_pclk ? true : false;
+	else
+		return pclk <= dispc.feat->max_tv_pclk ? true : false;
+}
+
+bool dispc_mgr_timings_ok(enum omap_channel channel,
+		const struct omap_video_timings *timings)
+{
+	if (!_dispc_mgr_size_ok(timings->x_res, timings->y_res))
+		return false;
+
+	if (!_dispc_mgr_pclk_ok(channel, timings->pixelclock))
+		return false;
+
+	if (dss_mgr_is_lcd(channel)) {
+		/* TODO: OMAP4+ supports interlace for LCD outputs */
+		if (timings->interlace)
+			return false;
+
+		if (!_dispc_lcd_timings_ok(timings->hsw, timings->hfp,
+				timings->hbp, timings->vsw, timings->vfp,
+				timings->vbp))
+			return false;
+	}
+
+	return true;
+}
+
+static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw,
+		int hfp, int hbp, int vsw, int vfp, int vbp,
+		enum omap_dss_signal_level vsync_level,
+		enum omap_dss_signal_level hsync_level,
+		enum omap_dss_signal_edge data_pclk_edge,
+		enum omap_dss_signal_level de_level,
+		enum omap_dss_signal_edge sync_pclk_edge)
+
+{
+	u32 timing_h, timing_v, l;
+	bool onoff, rf, ipc, vs, hs, de;
+
+	timing_h = FLD_VAL(hsw-1, dispc.feat->sw_start, 0) |
+			FLD_VAL(hfp-1, dispc.feat->fp_start, 8) |
+			FLD_VAL(hbp-1, dispc.feat->bp_start, 20);
+	timing_v = FLD_VAL(vsw-1, dispc.feat->sw_start, 0) |
+			FLD_VAL(vfp, dispc.feat->fp_start, 8) |
+			FLD_VAL(vbp, dispc.feat->bp_start, 20);
+
+	dispc_write_reg(DISPC_TIMING_H(channel), timing_h);
+	dispc_write_reg(DISPC_TIMING_V(channel), timing_v);
+
+	switch (vsync_level) {
+	case OMAPDSS_SIG_ACTIVE_LOW:
+		vs = true;
+		break;
+	case OMAPDSS_SIG_ACTIVE_HIGH:
+		vs = false;
+		break;
+	default:
+		BUG();
+	}
+
+	switch (hsync_level) {
+	case OMAPDSS_SIG_ACTIVE_LOW:
+		hs = true;
+		break;
+	case OMAPDSS_SIG_ACTIVE_HIGH:
+		hs = false;
+		break;
+	default:
+		BUG();
+	}
+
+	switch (de_level) {
+	case OMAPDSS_SIG_ACTIVE_LOW:
+		de = true;
+		break;
+	case OMAPDSS_SIG_ACTIVE_HIGH:
+		de = false;
+		break;
+	default:
+		BUG();
+	}
+
+	switch (data_pclk_edge) {
+	case OMAPDSS_DRIVE_SIG_RISING_EDGE:
+		ipc = false;
+		break;
+	case OMAPDSS_DRIVE_SIG_FALLING_EDGE:
+		ipc = true;
+		break;
+	default:
+		BUG();
+	}
+
+	/* always use the 'rf' setting */
+	onoff = true;
+
+	switch (sync_pclk_edge) {
+	case OMAPDSS_DRIVE_SIG_FALLING_EDGE:
+		rf = false;
+		break;
+	case OMAPDSS_DRIVE_SIG_RISING_EDGE:
+		rf = true;
+		break;
+	default:
+		BUG();
+	}
+
+	l = FLD_VAL(onoff, 17, 17) |
+		FLD_VAL(rf, 16, 16) |
+		FLD_VAL(de, 15, 15) |
+		FLD_VAL(ipc, 14, 14) |
+		FLD_VAL(hs, 13, 13) |
+		FLD_VAL(vs, 12, 12);
+
+	/* always set ALIGN bit when available */
+	if (dispc.feat->supports_sync_align)
+		l |= (1 << 18);
+
+	dispc_write_reg(DISPC_POL_FREQ(channel), l);
+
+	if (dispc.syscon_pol) {
+		const int shifts[] = {
+			[OMAP_DSS_CHANNEL_LCD] = 0,
+			[OMAP_DSS_CHANNEL_LCD2] = 1,
+			[OMAP_DSS_CHANNEL_LCD3] = 2,
+		};
+
+		u32 mask, val;
+
+		mask = (1 << 0) | (1 << 3) | (1 << 6);
+		val = (rf << 0) | (ipc << 3) | (onoff << 6);
+
+		mask <<= 16 + shifts[channel];
+		val <<= 16 + shifts[channel];
+
+		regmap_update_bits(dispc.syscon_pol, dispc.syscon_pol_offset,
+			mask, val);
+	}
+}
+
+/* change name to mode? */
+void dispc_mgr_set_timings(enum omap_channel channel,
+		const struct omap_video_timings *timings)
+{
+	unsigned xtot, ytot;
+	unsigned long ht, vt;
+	struct omap_video_timings t = *timings;
+
+	DSSDBG("channel %d xres %u yres %u\n", channel, t.x_res, t.y_res);
+
+	if (!dispc_mgr_timings_ok(channel, &t)) {
+		BUG();
+		return;
+	}
+
+	if (dss_mgr_is_lcd(channel)) {
+		_dispc_mgr_set_lcd_timings(channel, t.hsw, t.hfp, t.hbp, t.vsw,
+				t.vfp, t.vbp, t.vsync_level, t.hsync_level,
+				t.data_pclk_edge, t.de_level, t.sync_pclk_edge);
+
+		xtot = t.x_res + t.hfp + t.hsw + t.hbp;
+		ytot = t.y_res + t.vfp + t.vsw + t.vbp;
+
+		ht = timings->pixelclock / xtot;
+		vt = timings->pixelclock / xtot / ytot;
+
+		DSSDBG("pck %u\n", timings->pixelclock);
+		DSSDBG("hsw %d hfp %d hbp %d vsw %d vfp %d vbp %d\n",
+			t.hsw, t.hfp, t.hbp, t.vsw, t.vfp, t.vbp);
+		DSSDBG("vsync_level %d hsync_level %d data_pclk_edge %d de_level %d sync_pclk_edge %d\n",
+			t.vsync_level, t.hsync_level, t.data_pclk_edge,
+			t.de_level, t.sync_pclk_edge);
+
+		DSSDBG("hsync %luHz, vsync %luHz\n", ht, vt);
+	} else {
+		if (t.interlace)
+			t.y_res /= 2;
+	}
+
+	dispc_mgr_set_size(channel, t.x_res, t.y_res);
+}
+EXPORT_SYMBOL(dispc_mgr_set_timings);
+
+static void dispc_mgr_set_lcd_divisor(enum omap_channel channel, u16 lck_div,
+		u16 pck_div)
+{
+	BUG_ON(lck_div < 1);
+	BUG_ON(pck_div < 1);
+
+	dispc_write_reg(DISPC_DIVISORo(channel),
+			FLD_VAL(lck_div, 23, 16) | FLD_VAL(pck_div, 7, 0));
+
+	if (!dss_has_feature(FEAT_CORE_CLK_DIV) &&
+			channel == OMAP_DSS_CHANNEL_LCD)
+		dispc.core_clk_rate = dispc_fclk_rate() / lck_div;
+}
+
+static void dispc_mgr_get_lcd_divisor(enum omap_channel channel, int *lck_div,
+		int *pck_div)
+{
+	u32 l;
+	l = dispc_read_reg(DISPC_DIVISORo(channel));
+	*lck_div = FLD_GET(l, 23, 16);
+	*pck_div = FLD_GET(l, 7, 0);
+}
+
+static unsigned long dispc_fclk_rate(void)
+{
+	struct dss_pll *pll;
+	unsigned long r = 0;
+
+	switch (dss_get_dispc_clk_source()) {
+	case OMAP_DSS_CLK_SRC_FCK:
+		r = dss_get_dispc_clk_rate();
+		break;
+	case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
+		pll = dss_pll_find("dsi0");
+		if (!pll)
+			pll = dss_pll_find("video0");
+
+		r = pll->cinfo.clkout[0];
+		break;
+	case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
+		pll = dss_pll_find("dsi1");
+		if (!pll)
+			pll = dss_pll_find("video1");
+
+		r = pll->cinfo.clkout[0];
+		break;
+	default:
+		BUG();
+		return 0;
+	}
+
+	return r;
+}
+
+static unsigned long dispc_mgr_lclk_rate(enum omap_channel channel)
+{
+	struct dss_pll *pll;
+	int lcd;
+	unsigned long r;
+	u32 l;
+
+	if (dss_mgr_is_lcd(channel)) {
+		l = dispc_read_reg(DISPC_DIVISORo(channel));
+
+		lcd = FLD_GET(l, 23, 16);
+
+		switch (dss_get_lcd_clk_source(channel)) {
+		case OMAP_DSS_CLK_SRC_FCK:
+			r = dss_get_dispc_clk_rate();
+			break;
+		case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
+			pll = dss_pll_find("dsi0");
+			if (!pll)
+				pll = dss_pll_find("video0");
+
+			r = pll->cinfo.clkout[0];
+			break;
+		case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
+			pll = dss_pll_find("dsi1");
+			if (!pll)
+				pll = dss_pll_find("video1");
+
+			r = pll->cinfo.clkout[0];
+			break;
+		default:
+			BUG();
+			return 0;
+		}
+
+		return r / lcd;
+	} else {
+		return dispc_fclk_rate();
+	}
+}
+
+static unsigned long dispc_mgr_pclk_rate(enum omap_channel channel)
+{
+	unsigned long r;
+
+	if (dss_mgr_is_lcd(channel)) {
+		int pcd;
+		u32 l;
+
+		l = dispc_read_reg(DISPC_DIVISORo(channel));
+
+		pcd = FLD_GET(l, 7, 0);
+
+		r = dispc_mgr_lclk_rate(channel);
+
+		return r / pcd;
+	} else {
+		return dispc.tv_pclk_rate;
+	}
+}
+
+void dispc_set_tv_pclk(unsigned long pclk)
+{
+	dispc.tv_pclk_rate = pclk;
+}
+
+static unsigned long dispc_core_clk_rate(void)
+{
+	return dispc.core_clk_rate;
+}
+
+static unsigned long dispc_plane_pclk_rate(enum omap_plane plane)
+{
+	enum omap_channel channel;
+
+	if (plane == OMAP_DSS_WB)
+		return 0;
+
+	channel = dispc_ovl_get_channel_out(plane);
+
+	return dispc_mgr_pclk_rate(channel);
+}
+
+static unsigned long dispc_plane_lclk_rate(enum omap_plane plane)
+{
+	enum omap_channel channel;
+
+	if (plane == OMAP_DSS_WB)
+		return 0;
+
+	channel	= dispc_ovl_get_channel_out(plane);
+
+	return dispc_mgr_lclk_rate(channel);
+}
+
+static void dispc_dump_clocks_channel(struct seq_file *s, enum omap_channel channel)
+{
+	int lcd, pcd;
+	enum omap_dss_clk_source lcd_clk_src;
+
+	seq_printf(s, "- %s -\n", mgr_desc[channel].name);
+
+	lcd_clk_src = dss_get_lcd_clk_source(channel);
+
+	seq_printf(s, "%s clk source = %s (%s)\n", mgr_desc[channel].name,
+		dss_get_generic_clk_source_name(lcd_clk_src),
+		dss_feat_get_clk_source_name(lcd_clk_src));
+
+	dispc_mgr_get_lcd_divisor(channel, &lcd, &pcd);
+
+	seq_printf(s, "lck\t\t%-16lulck div\t%u\n",
+		dispc_mgr_lclk_rate(channel), lcd);
+	seq_printf(s, "pck\t\t%-16lupck div\t%u\n",
+		dispc_mgr_pclk_rate(channel), pcd);
+}
+
+void dispc_dump_clocks(struct seq_file *s)
+{
+	int lcd;
+	u32 l;
+	enum omap_dss_clk_source dispc_clk_src = dss_get_dispc_clk_source();
+
+	if (dispc_runtime_get())
+		return;
+
+	seq_printf(s, "- DISPC -\n");
+
+	seq_printf(s, "dispc fclk source = %s (%s)\n",
+			dss_get_generic_clk_source_name(dispc_clk_src),
+			dss_feat_get_clk_source_name(dispc_clk_src));
+
+	seq_printf(s, "fck\t\t%-16lu\n", dispc_fclk_rate());
+
+	if (dss_has_feature(FEAT_CORE_CLK_DIV)) {
+		seq_printf(s, "- DISPC-CORE-CLK -\n");
+		l = dispc_read_reg(DISPC_DIVISOR);
+		lcd = FLD_GET(l, 23, 16);
+
+		seq_printf(s, "lck\t\t%-16lulck div\t%u\n",
+				(dispc_fclk_rate()/lcd), lcd);
+	}
+
+	dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD);
+
+	if (dss_has_feature(FEAT_MGR_LCD2))
+		dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD2);
+	if (dss_has_feature(FEAT_MGR_LCD3))
+		dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD3);
+
+	dispc_runtime_put();
+}
+
+static void dispc_dump_regs(struct seq_file *s)
+{
+	int i, j;
+	const char *mgr_names[] = {
+		[OMAP_DSS_CHANNEL_LCD]		= "LCD",
+		[OMAP_DSS_CHANNEL_DIGIT]	= "TV",
+		[OMAP_DSS_CHANNEL_LCD2]		= "LCD2",
+		[OMAP_DSS_CHANNEL_LCD3]		= "LCD3",
+	};
+	const char *ovl_names[] = {
+		[OMAP_DSS_GFX]		= "GFX",
+		[OMAP_DSS_VIDEO1]	= "VID1",
+		[OMAP_DSS_VIDEO2]	= "VID2",
+		[OMAP_DSS_VIDEO3]	= "VID3",
+		[OMAP_DSS_WB]		= "WB",
+	};
+	const char **p_names;
+
+#define DUMPREG(r) seq_printf(s, "%-50s %08x\n", #r, dispc_read_reg(r))
+
+	if (dispc_runtime_get())
+		return;
+
+	/* DISPC common registers */
+	DUMPREG(DISPC_REVISION);
+	DUMPREG(DISPC_SYSCONFIG);
+	DUMPREG(DISPC_SYSSTATUS);
+	DUMPREG(DISPC_IRQSTATUS);
+	DUMPREG(DISPC_IRQENABLE);
+	DUMPREG(DISPC_CONTROL);
+	DUMPREG(DISPC_CONFIG);
+	DUMPREG(DISPC_CAPABLE);
+	DUMPREG(DISPC_LINE_STATUS);
+	DUMPREG(DISPC_LINE_NUMBER);
+	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+		DUMPREG(DISPC_GLOBAL_ALPHA);
+	if (dss_has_feature(FEAT_MGR_LCD2)) {
+		DUMPREG(DISPC_CONTROL2);
+		DUMPREG(DISPC_CONFIG2);
+	}
+	if (dss_has_feature(FEAT_MGR_LCD3)) {
+		DUMPREG(DISPC_CONTROL3);
+		DUMPREG(DISPC_CONFIG3);
+	}
+	if (dss_has_feature(FEAT_MFLAG))
+		DUMPREG(DISPC_GLOBAL_MFLAG_ATTRIBUTE);
+
+#undef DUMPREG
+
+#define DISPC_REG(i, name) name(i)
+#define DUMPREG(i, r) seq_printf(s, "%s(%s)%*s %08x\n", #r, p_names[i], \
+	(int)(48 - strlen(#r) - strlen(p_names[i])), " ", \
+	dispc_read_reg(DISPC_REG(i, r)))
+
+	p_names = mgr_names;
+
+	/* DISPC channel specific registers */
+	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+		DUMPREG(i, DISPC_DEFAULT_COLOR);
+		DUMPREG(i, DISPC_TRANS_COLOR);
+		DUMPREG(i, DISPC_SIZE_MGR);
+
+		if (i == OMAP_DSS_CHANNEL_DIGIT)
+			continue;
+
+		DUMPREG(i, DISPC_TIMING_H);
+		DUMPREG(i, DISPC_TIMING_V);
+		DUMPREG(i, DISPC_POL_FREQ);
+		DUMPREG(i, DISPC_DIVISORo);
+
+		DUMPREG(i, DISPC_DATA_CYCLE1);
+		DUMPREG(i, DISPC_DATA_CYCLE2);
+		DUMPREG(i, DISPC_DATA_CYCLE3);
+
+		if (dss_has_feature(FEAT_CPR)) {
+			DUMPREG(i, DISPC_CPR_COEF_R);
+			DUMPREG(i, DISPC_CPR_COEF_G);
+			DUMPREG(i, DISPC_CPR_COEF_B);
+		}
+	}
+
+	p_names = ovl_names;
+
+	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+		DUMPREG(i, DISPC_OVL_BA0);
+		DUMPREG(i, DISPC_OVL_BA1);
+		DUMPREG(i, DISPC_OVL_POSITION);
+		DUMPREG(i, DISPC_OVL_SIZE);
+		DUMPREG(i, DISPC_OVL_ATTRIBUTES);
+		DUMPREG(i, DISPC_OVL_FIFO_THRESHOLD);
+		DUMPREG(i, DISPC_OVL_FIFO_SIZE_STATUS);
+		DUMPREG(i, DISPC_OVL_ROW_INC);
+		DUMPREG(i, DISPC_OVL_PIXEL_INC);
+
+		if (dss_has_feature(FEAT_PRELOAD))
+			DUMPREG(i, DISPC_OVL_PRELOAD);
+		if (dss_has_feature(FEAT_MFLAG))
+			DUMPREG(i, DISPC_OVL_MFLAG_THRESHOLD);
+
+		if (i == OMAP_DSS_GFX) {
+			DUMPREG(i, DISPC_OVL_WINDOW_SKIP);
+			DUMPREG(i, DISPC_OVL_TABLE_BA);
+			continue;
+		}
+
+		DUMPREG(i, DISPC_OVL_FIR);
+		DUMPREG(i, DISPC_OVL_PICTURE_SIZE);
+		DUMPREG(i, DISPC_OVL_ACCU0);
+		DUMPREG(i, DISPC_OVL_ACCU1);
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			DUMPREG(i, DISPC_OVL_BA0_UV);
+			DUMPREG(i, DISPC_OVL_BA1_UV);
+			DUMPREG(i, DISPC_OVL_FIR2);
+			DUMPREG(i, DISPC_OVL_ACCU2_0);
+			DUMPREG(i, DISPC_OVL_ACCU2_1);
+		}
+		if (dss_has_feature(FEAT_ATTR2))
+			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
+	}
+
+	if (dispc.feat->has_writeback) {
+		i = OMAP_DSS_WB;
+		DUMPREG(i, DISPC_OVL_BA0);
+		DUMPREG(i, DISPC_OVL_BA1);
+		DUMPREG(i, DISPC_OVL_SIZE);
+		DUMPREG(i, DISPC_OVL_ATTRIBUTES);
+		DUMPREG(i, DISPC_OVL_FIFO_THRESHOLD);
+		DUMPREG(i, DISPC_OVL_FIFO_SIZE_STATUS);
+		DUMPREG(i, DISPC_OVL_ROW_INC);
+		DUMPREG(i, DISPC_OVL_PIXEL_INC);
+
+		if (dss_has_feature(FEAT_MFLAG))
+			DUMPREG(i, DISPC_OVL_MFLAG_THRESHOLD);
+
+		DUMPREG(i, DISPC_OVL_FIR);
+		DUMPREG(i, DISPC_OVL_PICTURE_SIZE);
+		DUMPREG(i, DISPC_OVL_ACCU0);
+		DUMPREG(i, DISPC_OVL_ACCU1);
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			DUMPREG(i, DISPC_OVL_BA0_UV);
+			DUMPREG(i, DISPC_OVL_BA1_UV);
+			DUMPREG(i, DISPC_OVL_FIR2);
+			DUMPREG(i, DISPC_OVL_ACCU2_0);
+			DUMPREG(i, DISPC_OVL_ACCU2_1);
+		}
+		if (dss_has_feature(FEAT_ATTR2))
+			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
+	}
+
+#undef DISPC_REG
+#undef DUMPREG
+
+#define DISPC_REG(plane, name, i) name(plane, i)
+#define DUMPREG(plane, name, i) \
+	seq_printf(s, "%s_%d(%s)%*s %08x\n", #name, i, p_names[plane], \
+	(int)(46 - strlen(#name) - strlen(p_names[plane])), " ", \
+	dispc_read_reg(DISPC_REG(plane, name, i)))
+
+	/* Video pipeline coefficient registers */
+
+	/* start from OMAP_DSS_VIDEO1 */
+	for (i = 1; i < dss_feat_get_num_ovls(); i++) {
+		for (j = 0; j < 8; j++)
+			DUMPREG(i, DISPC_OVL_FIR_COEF_H, j);
+
+		for (j = 0; j < 8; j++)
+			DUMPREG(i, DISPC_OVL_FIR_COEF_HV, j);
+
+		for (j = 0; j < 5; j++)
+			DUMPREG(i, DISPC_OVL_CONV_COEF, j);
+
+		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+			for (j = 0; j < 8; j++)
+				DUMPREG(i, DISPC_OVL_FIR_COEF_V, j);
+		}
+
+		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+			for (j = 0; j < 8; j++)
+				DUMPREG(i, DISPC_OVL_FIR_COEF_H2, j);
+
+			for (j = 0; j < 8; j++)
+				DUMPREG(i, DISPC_OVL_FIR_COEF_HV2, j);
+
+			for (j = 0; j < 8; j++)
+				DUMPREG(i, DISPC_OVL_FIR_COEF_V2, j);
+		}
+	}
+
+	dispc_runtime_put();
+
+#undef DISPC_REG
+#undef DUMPREG
+}
+
+/* calculate clock rates using dividers in cinfo */
+int dispc_calc_clock_rates(unsigned long dispc_fclk_rate,
+		struct dispc_clock_info *cinfo)
+{
+	if (cinfo->lck_div > 255 || cinfo->lck_div == 0)
+		return -EINVAL;
+	if (cinfo->pck_div < 1 || cinfo->pck_div > 255)
+		return -EINVAL;
+
+	cinfo->lck = dispc_fclk_rate / cinfo->lck_div;
+	cinfo->pck = cinfo->lck / cinfo->pck_div;
+
+	return 0;
+}
+
+bool dispc_div_calc(unsigned long dispc,
+		unsigned long pck_min, unsigned long pck_max,
+		dispc_div_calc_func func, void *data)
+{
+	int lckd, lckd_start, lckd_stop;
+	int pckd, pckd_start, pckd_stop;
+	unsigned long pck, lck;
+	unsigned long lck_max;
+	unsigned long pckd_hw_min, pckd_hw_max;
+	unsigned min_fck_per_pck;
+	unsigned long fck;
+
+#ifdef CONFIG_FB_OMAP2_DSS_MIN_FCK_PER_PCK
+	min_fck_per_pck = CONFIG_FB_OMAP2_DSS_MIN_FCK_PER_PCK;
+#else
+	min_fck_per_pck = 0;
+#endif
+
+	pckd_hw_min = dss_feat_get_param_min(FEAT_PARAM_DSS_PCD);
+	pckd_hw_max = dss_feat_get_param_max(FEAT_PARAM_DSS_PCD);
+
+	lck_max = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+
+	pck_min = pck_min ? pck_min : 1;
+	pck_max = pck_max ? pck_max : ULONG_MAX;
+
+	lckd_start = max(DIV_ROUND_UP(dispc, lck_max), 1ul);
+	lckd_stop = min(dispc / pck_min, 255ul);
+
+	for (lckd = lckd_start; lckd <= lckd_stop; ++lckd) {
+		lck = dispc / lckd;
+
+		pckd_start = max(DIV_ROUND_UP(lck, pck_max), pckd_hw_min);
+		pckd_stop = min(lck / pck_min, pckd_hw_max);
+
+		for (pckd = pckd_start; pckd <= pckd_stop; ++pckd) {
+			pck = lck / pckd;
+
+			/*
+			 * For OMAP2/3 the DISPC fclk is the same as LCD's logic
+			 * clock, which means we're configuring DISPC fclk here
+			 * also. Thus we need to use the calculated lck. For
+			 * OMAP4+ the DISPC fclk is a separate clock.
+			 */
+			if (dss_has_feature(FEAT_CORE_CLK_DIV))
+				fck = dispc_core_clk_rate();
+			else
+				fck = lck;
+
+			if (fck < pck * min_fck_per_pck)
+				continue;
+
+			if (func(lckd, pckd, lck, pck, data))
+				return true;
+		}
+	}
+
+	return false;
+}
+
+void dispc_mgr_set_clock_div(enum omap_channel channel,
+		const struct dispc_clock_info *cinfo)
+{
+	DSSDBG("lck = %lu (%u)\n", cinfo->lck, cinfo->lck_div);
+	DSSDBG("pck = %lu (%u)\n", cinfo->pck, cinfo->pck_div);
+
+	dispc_mgr_set_lcd_divisor(channel, cinfo->lck_div, cinfo->pck_div);
+}
+
+int dispc_mgr_get_clock_div(enum omap_channel channel,
+		struct dispc_clock_info *cinfo)
+{
+	unsigned long fck;
+
+	fck = dispc_fclk_rate();
+
+	cinfo->lck_div = REG_GET(DISPC_DIVISORo(channel), 23, 16);
+	cinfo->pck_div = REG_GET(DISPC_DIVISORo(channel), 7, 0);
+
+	cinfo->lck = fck / cinfo->lck_div;
+	cinfo->pck = cinfo->lck / cinfo->pck_div;
+
+	return 0;
+}
+
+u32 dispc_read_irqstatus(void)
+{
+	return dispc_read_reg(DISPC_IRQSTATUS);
+}
+EXPORT_SYMBOL(dispc_read_irqstatus);
+
+void dispc_clear_irqstatus(u32 mask)
+{
+	dispc_write_reg(DISPC_IRQSTATUS, mask);
+}
+EXPORT_SYMBOL(dispc_clear_irqstatus);
+
+u32 dispc_read_irqenable(void)
+{
+	return dispc_read_reg(DISPC_IRQENABLE);
+}
+EXPORT_SYMBOL(dispc_read_irqenable);
+
+void dispc_write_irqenable(u32 mask)
+{
+	u32 old_mask = dispc_read_reg(DISPC_IRQENABLE);
+
+	/* clear the irqstatus for newly enabled irqs */
+	dispc_clear_irqstatus((mask ^ old_mask) & mask);
+
+	dispc_write_reg(DISPC_IRQENABLE, mask);
+}
+EXPORT_SYMBOL(dispc_write_irqenable);
+
+void dispc_enable_sidle(void)
+{
+	REG_FLD_MOD(DISPC_SYSCONFIG, 2, 4, 3);	/* SIDLEMODE: smart idle */
+}
+
+void dispc_disable_sidle(void)
+{
+	REG_FLD_MOD(DISPC_SYSCONFIG, 1, 4, 3);	/* SIDLEMODE: no idle */
+}
+
+static void _omap_dispc_initial_config(void)
+{
+	u32 l;
+
+	/* Exclusively enable DISPC_CORE_CLK and set divider to 1 */
+	if (dss_has_feature(FEAT_CORE_CLK_DIV)) {
+		l = dispc_read_reg(DISPC_DIVISOR);
+		/* Use DISPC_DIVISOR.LCD, instead of DISPC_DIVISOR1.LCD */
+		l = FLD_MOD(l, 1, 0, 0);
+		l = FLD_MOD(l, 1, 23, 16);
+		dispc_write_reg(DISPC_DIVISOR, l);
+
+		dispc.core_clk_rate = dispc_fclk_rate();
+	}
+
+	/* FUNCGATED */
+	if (dss_has_feature(FEAT_FUNCGATED))
+		REG_FLD_MOD(DISPC_CONFIG, 1, 9, 9);
+
+	dispc_setup_color_conv_coef();
+
+	dispc_set_loadmode(OMAP_DSS_LOAD_FRAME_ONLY);
+
+	dispc_init_fifos();
+
+	dispc_configure_burst_sizes();
+
+	dispc_ovl_enable_zorder_planes();
+
+	if (dispc.feat->mstandby_workaround)
+		REG_FLD_MOD(DISPC_MSTANDBY_CTRL, 1, 0, 0);
+
+	if (dss_has_feature(FEAT_MFLAG))
+		dispc_init_mflag();
+}
+
+static const struct dispc_features omap24xx_dispc_feats = {
+	.sw_start		=	5,
+	.fp_start		=	15,
+	.bp_start		=	27,
+	.sw_max			=	64,
+	.vp_max			=	255,
+	.hp_max			=	256,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	66500000,
+	.calc_scaling		=	dispc_ovl_calc_scaling_24xx,
+	.calc_core_clk		=	calc_core_clk_24xx,
+	.num_fifos		=	3,
+	.no_framedone_tv	=	true,
+	.set_max_preload	=	false,
+	.last_pixel_inc_missing	=	true,
+};
+
+static const struct dispc_features omap34xx_rev1_0_dispc_feats = {
+	.sw_start		=	5,
+	.fp_start		=	15,
+	.bp_start		=	27,
+	.sw_max			=	64,
+	.vp_max			=	255,
+	.hp_max			=	256,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	173000000,
+	.max_tv_pclk		=	59000000,
+	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
+	.calc_core_clk		=	calc_core_clk_34xx,
+	.num_fifos		=	3,
+	.no_framedone_tv	=	true,
+	.set_max_preload	=	false,
+	.last_pixel_inc_missing	=	true,
+};
+
+static const struct dispc_features omap34xx_rev3_0_dispc_feats = {
+	.sw_start		=	7,
+	.fp_start		=	19,
+	.bp_start		=	31,
+	.sw_max			=	256,
+	.vp_max			=	4095,
+	.hp_max			=	4096,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	173000000,
+	.max_tv_pclk		=	59000000,
+	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
+	.calc_core_clk		=	calc_core_clk_34xx,
+	.num_fifos		=	3,
+	.no_framedone_tv	=	true,
+	.set_max_preload	=	false,
+	.last_pixel_inc_missing	=	true,
+};
+
+static const struct dispc_features omap44xx_dispc_feats = {
+	.sw_start		=	7,
+	.fp_start		=	19,
+	.bp_start		=	31,
+	.sw_max			=	256,
+	.vp_max			=	4095,
+	.hp_max			=	4096,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	170000000,
+	.max_tv_pclk		=	185625000,
+	.calc_scaling		=	dispc_ovl_calc_scaling_44xx,
+	.calc_core_clk		=	calc_core_clk_44xx,
+	.num_fifos		=	5,
+	.gfx_fifo_workaround	=	true,
+	.set_max_preload	=	true,
+	.supports_sync_align	=	true,
+	.has_writeback		=	true,
+};
+
+static const struct dispc_features omap54xx_dispc_feats = {
+	.sw_start		=	7,
+	.fp_start		=	19,
+	.bp_start		=	31,
+	.sw_max			=	256,
+	.vp_max			=	4095,
+	.hp_max			=	4096,
+	.mgr_width_start	=	11,
+	.mgr_height_start	=	27,
+	.mgr_width_max		=	4096,
+	.mgr_height_max		=	4096,
+	.max_lcd_pclk		=	170000000,
+	.max_tv_pclk		=	186000000,
+	.calc_scaling		=	dispc_ovl_calc_scaling_44xx,
+	.calc_core_clk		=	calc_core_clk_44xx,
+	.num_fifos		=	5,
+	.gfx_fifo_workaround	=	true,
+	.mstandby_workaround	=	true,
+	.set_max_preload	=	true,
+	.supports_sync_align	=	true,
+	.has_writeback		=	true,
+};
+
+static int dispc_init_features(struct platform_device *pdev)
+{
+	const struct dispc_features *src;
+	struct dispc_features *dst;
+
+	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
+	if (!dst) {
+		dev_err(&pdev->dev, "Failed to allocate DISPC Features\n");
+		return -ENOMEM;
+	}
+
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP24xx:
+		src = &omap24xx_dispc_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP34xx_ES1:
+		src = &omap34xx_rev1_0_dispc_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_OMAP3630:
+	case OMAPDSS_VER_AM35xx:
+	case OMAPDSS_VER_AM43xx:
+		src = &omap34xx_rev3_0_dispc_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		src = &omap44xx_dispc_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+	case OMAPDSS_VER_DRA7xx:
+		src = &omap54xx_dispc_feats;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	memcpy(dst, src, sizeof(*dst));
+	dispc.feat = dst;
+
+	return 0;
+}
+
+static irqreturn_t dispc_irq_handler(int irq, void *arg)
+{
+	if (!dispc.is_enabled)
+		return IRQ_NONE;
+
+	return dispc.user_handler(irq, dispc.user_data);
+}
+
+int dispc_request_irq(irq_handler_t handler, void *dev_id)
+{
+	int r;
+
+	if (dispc.user_handler != NULL)
+		return -EBUSY;
+
+	dispc.user_handler = handler;
+	dispc.user_data = dev_id;
+
+	/* ensure the dispc_irq_handler sees the values above */
+	smp_wmb();
+
+	r = devm_request_irq(&dispc.pdev->dev, dispc.irq, dispc_irq_handler,
+			     IRQF_SHARED, "OMAP DISPC", &dispc);
+	if (r) {
+		dispc.user_handler = NULL;
+		dispc.user_data = NULL;
+	}
+
+	return r;
+}
+EXPORT_SYMBOL(dispc_request_irq);
+
+void dispc_free_irq(void *dev_id)
+{
+	devm_free_irq(&dispc.pdev->dev, dispc.irq, &dispc);
+
+	dispc.user_handler = NULL;
+	dispc.user_data = NULL;
+}
+EXPORT_SYMBOL(dispc_free_irq);
+
+/* DISPC HW IP initialisation */
+static int dispc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	u32 rev;
+	int r = 0;
+	struct resource *dispc_mem;
+	struct device_node *np = pdev->dev.of_node;
+
+	dispc.pdev = pdev;
+
+	spin_lock_init(&dispc.control_lock);
+
+	r = dispc_init_features(dispc.pdev);
+	if (r)
+		return r;
+
+	dispc_mem = platform_get_resource(dispc.pdev, IORESOURCE_MEM, 0);
+	if (!dispc_mem) {
+		DSSERR("can't get IORESOURCE_MEM DISPC\n");
+		return -EINVAL;
+	}
+
+	dispc.base = devm_ioremap(&pdev->dev, dispc_mem->start,
+				  resource_size(dispc_mem));
+	if (!dispc.base) {
+		DSSERR("can't ioremap DISPC\n");
+		return -ENOMEM;
+	}
+
+	dispc.irq = platform_get_irq(dispc.pdev, 0);
+	if (dispc.irq < 0) {
+		DSSERR("platform_get_irq failed\n");
+		return -ENODEV;
+	}
+
+	if (np && of_property_read_bool(np, "syscon-pol")) {
+		dispc.syscon_pol = syscon_regmap_lookup_by_phandle(np, "syscon-pol");
+		if (IS_ERR(dispc.syscon_pol)) {
+			dev_err(&pdev->dev, "failed to get syscon-pol regmap\n");
+			return PTR_ERR(dispc.syscon_pol);
+		}
+
+		if (of_property_read_u32_index(np, "syscon-pol", 1,
+				&dispc.syscon_pol_offset)) {
+			dev_err(&pdev->dev, "failed to get syscon-pol offset\n");
+			return -EINVAL;
+		}
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	r = dispc_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	_omap_dispc_initial_config();
+
+	rev = dispc_read_reg(DISPC_REVISION);
+	dev_dbg(&pdev->dev, "OMAP DISPC rev %d.%d\n",
+	       FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
+
+	dispc_runtime_put();
+
+	dss_init_overlay_managers();
+
+	dss_debugfs_create_file("dispc", dispc_dump_regs);
+
+	return 0;
+
+err_runtime_get:
+	pm_runtime_disable(&pdev->dev);
+	return r;
+}
+
+static void dispc_unbind(struct device *dev, struct device *master,
+			       void *data)
+{
+	pm_runtime_disable(dev);
+
+	dss_uninit_overlay_managers();
+}
+
+static const struct component_ops dispc_component_ops = {
+	.bind	= dispc_bind,
+	.unbind	= dispc_unbind,
+};
+
+static int dispc_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dispc_component_ops);
+}
+
+static int dispc_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dispc_component_ops);
+	return 0;
+}
+
+static int dispc_runtime_suspend(struct device *dev)
+{
+	dispc.is_enabled = false;
+	/* ensure the dispc_irq_handler sees the is_enabled value */
+	smp_wmb();
+	/* wait for current handler to finish before turning the DISPC off */
+	synchronize_irq(dispc.irq);
+
+	dispc_save_context();
+
+	return 0;
+}
+
+static int dispc_runtime_resume(struct device *dev)
+{
+	/*
+	 * The reset value for load mode is 0 (OMAP_DSS_LOAD_CLUT_AND_FRAME)
+	 * but we always initialize it to 2 (OMAP_DSS_LOAD_FRAME_ONLY) in
+	 * _omap_dispc_initial_config(). We can thus use it to detect if
+	 * we have lost register context.
+	 */
+	if (REG_GET(DISPC_CONFIG, 2, 1) != OMAP_DSS_LOAD_FRAME_ONLY) {
+		_omap_dispc_initial_config();
+
+		dispc_restore_context();
+	}
+
+	dispc.is_enabled = true;
+	/* ensure the dispc_irq_handler sees the is_enabled value */
+	smp_wmb();
+
+	return 0;
+}
+
+static const struct dev_pm_ops dispc_pm_ops = {
+	.runtime_suspend = dispc_runtime_suspend,
+	.runtime_resume = dispc_runtime_resume,
+};
+
+static const struct of_device_id dispc_of_match[] = {
+	{ .compatible = "ti,omap2-dispc", },
+	{ .compatible = "ti,omap3-dispc", },
+	{ .compatible = "ti,omap4-dispc", },
+	{ .compatible = "ti,omap5-dispc", },
+	{ .compatible = "ti,dra7-dispc", },
+	{},
+};
+
+static struct platform_driver omap_dispchw_driver = {
+	.probe		= dispc_probe,
+	.remove         = dispc_remove,
+	.driver         = {
+		.name   = "omapdss_dispc",
+		.pm	= &dispc_pm_ops,
+		.of_match_table = dispc_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init dispc_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_dispchw_driver);
+}
+
+void dispc_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_dispchw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.h b/drivers/video/fbdev/omap2/omapfb/dss/dispc.h
new file mode 100644
index 000000000000..483744223dd1
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.h
@@ -0,0 +1,918 @@
+/*
+ * linux/drivers/video/omap2/dss/dispc.h
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Archit Taneja <archit@ti.com>
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAP2_DISPC_REG_H
+#define __OMAP2_DISPC_REG_H
+
+/* DISPC common registers */
+#define DISPC_REVISION			0x0000
+#define DISPC_SYSCONFIG			0x0010
+#define DISPC_SYSSTATUS			0x0014
+#define DISPC_IRQSTATUS			0x0018
+#define DISPC_IRQENABLE			0x001C
+#define DISPC_CONTROL			0x0040
+#define DISPC_CONFIG			0x0044
+#define DISPC_CAPABLE			0x0048
+#define DISPC_LINE_STATUS		0x005C
+#define DISPC_LINE_NUMBER		0x0060
+#define DISPC_GLOBAL_ALPHA		0x0074
+#define DISPC_CONTROL2			0x0238
+#define DISPC_CONFIG2			0x0620
+#define DISPC_DIVISOR			0x0804
+#define DISPC_GLOBAL_BUFFER		0x0800
+#define DISPC_CONTROL3                  0x0848
+#define DISPC_CONFIG3                   0x084C
+#define DISPC_MSTANDBY_CTRL		0x0858
+#define DISPC_GLOBAL_MFLAG_ATTRIBUTE	0x085C
+
+/* DISPC overlay registers */
+#define DISPC_OVL_BA0(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_BA0_OFFSET(n))
+#define DISPC_OVL_BA1(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_BA1_OFFSET(n))
+#define DISPC_OVL_BA0_UV(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_BA0_UV_OFFSET(n))
+#define DISPC_OVL_BA1_UV(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_BA1_UV_OFFSET(n))
+#define DISPC_OVL_POSITION(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_POS_OFFSET(n))
+#define DISPC_OVL_SIZE(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_SIZE_OFFSET(n))
+#define DISPC_OVL_ATTRIBUTES(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ATTR_OFFSET(n))
+#define DISPC_OVL_ATTRIBUTES2(n)	(DISPC_OVL_BASE(n) + \
+					DISPC_ATTR2_OFFSET(n))
+#define DISPC_OVL_FIFO_THRESHOLD(n)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIFO_THRESH_OFFSET(n))
+#define DISPC_OVL_FIFO_SIZE_STATUS(n)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIFO_SIZE_STATUS_OFFSET(n))
+#define DISPC_OVL_ROW_INC(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ROW_INC_OFFSET(n))
+#define DISPC_OVL_PIXEL_INC(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_PIX_INC_OFFSET(n))
+#define DISPC_OVL_WINDOW_SKIP(n)	(DISPC_OVL_BASE(n) + \
+					DISPC_WINDOW_SKIP_OFFSET(n))
+#define DISPC_OVL_TABLE_BA(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_TABLE_BA_OFFSET(n))
+#define DISPC_OVL_FIR(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_OFFSET(n))
+#define DISPC_OVL_FIR2(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_FIR2_OFFSET(n))
+#define DISPC_OVL_PICTURE_SIZE(n)	(DISPC_OVL_BASE(n) + \
+					DISPC_PIC_SIZE_OFFSET(n))
+#define DISPC_OVL_ACCU0(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ACCU0_OFFSET(n))
+#define DISPC_OVL_ACCU1(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ACCU1_OFFSET(n))
+#define DISPC_OVL_ACCU2_0(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ACCU2_0_OFFSET(n))
+#define DISPC_OVL_ACCU2_1(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_ACCU2_1_OFFSET(n))
+#define DISPC_OVL_FIR_COEF_H(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_H_OFFSET(n, i))
+#define DISPC_OVL_FIR_COEF_HV(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_HV_OFFSET(n, i))
+#define DISPC_OVL_FIR_COEF_H2(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_H2_OFFSET(n, i))
+#define DISPC_OVL_FIR_COEF_HV2(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_HV2_OFFSET(n, i))
+#define DISPC_OVL_CONV_COEF(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_CONV_COEF_OFFSET(n, i))
+#define DISPC_OVL_FIR_COEF_V(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_V_OFFSET(n, i))
+#define DISPC_OVL_FIR_COEF_V2(n, i)	(DISPC_OVL_BASE(n) + \
+					DISPC_FIR_COEF_V2_OFFSET(n, i))
+#define DISPC_OVL_PRELOAD(n)		(DISPC_OVL_BASE(n) + \
+					DISPC_PRELOAD_OFFSET(n))
+#define DISPC_OVL_MFLAG_THRESHOLD(n)	DISPC_MFLAG_THRESHOLD_OFFSET(n)
+
+/* DISPC up/downsampling FIR filter coefficient structure */
+struct dispc_coef {
+	s8 hc4_vc22;
+	s8 hc3_vc2;
+	u8 hc2_vc1;
+	s8 hc1_vc0;
+	s8 hc0_vc00;
+};
+
+const struct dispc_coef *dispc_ovl_get_scale_coef(int inc, int five_taps);
+
+/* DISPC manager/channel specific registers */
+static inline u16 DISPC_DEFAULT_COLOR(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x004C;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		return 0x0050;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03AC;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0814;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_TRANS_COLOR(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0054;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		return 0x0058;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03B0;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0818;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_TIMING_H(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0064;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x0400;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0840;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_TIMING_V(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0068;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x0404;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0844;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_POL_FREQ(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x006C;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x0408;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x083C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_DIVISORo(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0070;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x040C;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0838;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* Named as DISPC_SIZE_LCD, DISPC_SIZE_DIGIT and DISPC_SIZE_LCD2 in TRM */
+static inline u16 DISPC_SIZE_MGR(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x007C;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		return 0x0078;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03CC;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0834;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_DATA_CYCLE1(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x01D4;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03C0;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0828;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_DATA_CYCLE2(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x01D8;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03C4;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x082C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_DATA_CYCLE3(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x01DC;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03C8;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0830;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_CPR_COEF_R(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0220;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03BC;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0824;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_CPR_COEF_G(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0224;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03B8;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x0820;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_CPR_COEF_B(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return 0x0228;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		BUG();
+		return 0;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return 0x03B4;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return 0x081C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* DISPC overlay register base addresses */
+static inline u16 DISPC_OVL_BASE(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0080;
+	case OMAP_DSS_VIDEO1:
+		return 0x00BC;
+	case OMAP_DSS_VIDEO2:
+		return 0x014C;
+	case OMAP_DSS_VIDEO3:
+		return 0x0300;
+	case OMAP_DSS_WB:
+		return 0x0500;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* DISPC overlay register offsets */
+static inline u16 DISPC_BA0_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0000;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0008;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_BA1_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0004;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x000C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_BA0_UV_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0544;
+	case OMAP_DSS_VIDEO2:
+		return 0x04BC;
+	case OMAP_DSS_VIDEO3:
+		return 0x0310;
+	case OMAP_DSS_WB:
+		return 0x0118;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_BA1_UV_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0548;
+	case OMAP_DSS_VIDEO2:
+		return 0x04C0;
+	case OMAP_DSS_VIDEO3:
+		return 0x0314;
+	case OMAP_DSS_WB:
+		return 0x011C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_POS_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0008;
+	case OMAP_DSS_VIDEO3:
+		return 0x009C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_SIZE_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x000C;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x00A8;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ATTR_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0020;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0010;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0070;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ATTR2_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0568;
+	case OMAP_DSS_VIDEO2:
+		return 0x04DC;
+	case OMAP_DSS_VIDEO3:
+		return 0x032C;
+	case OMAP_DSS_WB:
+		return 0x0310;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_FIFO_THRESH_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0024;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0014;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x008C;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_FIFO_SIZE_STATUS_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0028;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0018;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0088;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ROW_INC_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x002C;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x001C;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x00A4;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_PIX_INC_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0030;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0020;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0098;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_WINDOW_SKIP_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0034;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+	case OMAP_DSS_VIDEO3:
+		BUG();
+		return 0;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_TABLE_BA_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0038;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+	case OMAP_DSS_VIDEO3:
+		BUG();
+		return 0;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_FIR_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0024;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0090;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_FIR2_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0580;
+	case OMAP_DSS_VIDEO2:
+		return 0x055C;
+	case OMAP_DSS_VIDEO3:
+		return 0x0424;
+	case OMAP_DSS_WB:
+		return 0x290;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_PIC_SIZE_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0028;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0094;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+
+static inline u16 DISPC_ACCU0_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x002C;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0000;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ACCU2_0_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0584;
+	case OMAP_DSS_VIDEO2:
+		return 0x0560;
+	case OMAP_DSS_VIDEO3:
+		return 0x0428;
+	case OMAP_DSS_WB:
+		return 0x0294;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ACCU1_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0030;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0004;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_ACCU2_1_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0588;
+	case OMAP_DSS_VIDEO2:
+		return 0x0564;
+	case OMAP_DSS_VIDEO3:
+		return 0x042C;
+	case OMAP_DSS_WB:
+		return 0x0298;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_H_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0034 + i * 0x8;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0010 + i * 0x8;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_H2_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x058C + i * 0x8;
+	case OMAP_DSS_VIDEO2:
+		return 0x0568 + i * 0x8;
+	case OMAP_DSS_VIDEO3:
+		return 0x0430 + i * 0x8;
+	case OMAP_DSS_WB:
+		return 0x02A0 + i * 0x8;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_HV_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+		return 0x0038 + i * 0x8;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0014 + i * 0x8;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_HV2_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0590 + i * 8;
+	case OMAP_DSS_VIDEO2:
+		return 0x056C + i * 0x8;
+	case OMAP_DSS_VIDEO3:
+		return 0x0434 + i * 0x8;
+	case OMAP_DSS_WB:
+		return 0x02A4 + i * 0x8;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4,} */
+static inline u16 DISPC_CONV_COEF_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+	case OMAP_DSS_VIDEO2:
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0074 + i * 0x4;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_V_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x0124 + i * 0x4;
+	case OMAP_DSS_VIDEO2:
+		return 0x00B4 + i * 0x4;
+	case OMAP_DSS_VIDEO3:
+	case OMAP_DSS_WB:
+		return 0x0050 + i * 0x4;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+/* coef index i = {0, 1, 2, 3, 4, 5, 6, 7} */
+static inline u16 DISPC_FIR_COEF_V2_OFFSET(enum omap_plane plane, u16 i)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		BUG();
+		return 0;
+	case OMAP_DSS_VIDEO1:
+		return 0x05CC + i * 0x4;
+	case OMAP_DSS_VIDEO2:
+		return 0x05A8 + i * 0x4;
+	case OMAP_DSS_VIDEO3:
+		return 0x0470 + i * 0x4;
+	case OMAP_DSS_WB:
+		return 0x02E0 + i * 0x4;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_PRELOAD_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x01AC;
+	case OMAP_DSS_VIDEO1:
+		return 0x0174;
+	case OMAP_DSS_VIDEO2:
+		return 0x00E8;
+	case OMAP_DSS_VIDEO3:
+		return 0x00A0;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static inline u16 DISPC_MFLAG_THRESHOLD_OFFSET(enum omap_plane plane)
+{
+	switch (plane) {
+	case OMAP_DSS_GFX:
+		return 0x0860;
+	case OMAP_DSS_VIDEO1:
+		return 0x0864;
+	case OMAP_DSS_VIDEO2:
+		return 0x0868;
+	case OMAP_DSS_VIDEO3:
+		return 0x086c;
+	case OMAP_DSS_WB:
+		return 0x0870;
+	default:
+		BUG();
+		return 0;
+	}
+}
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c
new file mode 100644
index 000000000000..038c15b04215
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c
@@ -0,0 +1,325 @@
+/*
+ * linux/drivers/video/omap2/dss/dispc_coefs.c
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Chandrabhanu Mahapatra <cmahapatra@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <video/omapdss.h>
+
+#include "dispc.h"
+
+static const struct dispc_coef coef3_M8[8] = {
+	{ 0,  0, 128,  0, 0 },
+	{ 0, -4, 123,  9, 0 },
+	{ 0, -4, 108, 24, 0 },
+	{ 0, -2,  87, 43, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 43,  87, -2, 0 },
+	{ 0, 24, 108, -4, 0 },
+	{ 0,  9, 123, -4, 0 },
+};
+
+static const struct dispc_coef coef3_M9[8] = {
+	{ 0,  6, 116,  6, 0 },
+	{ 0,  0, 112, 16, 0 },
+	{ 0, -2, 100, 30, 0 },
+	{ 0, -2,  83, 47, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 47,  83, -2, 0 },
+	{ 0, 30, 100, -2, 0 },
+	{ 0, 16, 112,  0, 0 },
+};
+
+static const struct dispc_coef coef3_M10[8] = {
+	{ 0, 10, 108, 10, 0 },
+	{ 0,  3, 104, 21, 0 },
+	{ 0,  0,  94, 34, 0 },
+	{ 0, -1,  80, 49, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 49,  80, -1, 0 },
+	{ 0, 34,  94,  0, 0 },
+	{ 0, 21, 104,  3, 0 },
+};
+
+static const struct dispc_coef coef3_M11[8] = {
+	{ 0, 14, 100, 14, 0 },
+	{ 0,  6,  98, 24, 0 },
+	{ 0,  2,  90, 36, 0 },
+	{ 0,  0,  78, 50, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 50,  78,  0, 0 },
+	{ 0, 36,  90,  2, 0 },
+	{ 0, 24,  98,  6, 0 },
+};
+
+static const struct dispc_coef coef3_M12[8] = {
+	{ 0, 16,  96, 16, 0 },
+	{ 0,  9,  93, 26, 0 },
+	{ 0,  4,  86, 38, 0 },
+	{ 0,  1,  76, 51, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 51,  76,  1, 0 },
+	{ 0, 38,  86,  4, 0 },
+	{ 0, 26,  93,  9, 0 },
+};
+
+static const struct dispc_coef coef3_M13[8] = {
+	{ 0, 18,  92, 18, 0 },
+	{ 0, 10,  90, 28, 0 },
+	{ 0,  5,  83, 40, 0 },
+	{ 0,  1,  75, 52, 0 },
+	{ 0, 64,  64,  0, 0 },
+	{ 0, 52,  75,  1, 0 },
+	{ 0, 40,  83,  5, 0 },
+	{ 0, 28,  90, 10, 0 },
+};
+
+static const struct dispc_coef coef3_M14[8] = {
+	{ 0, 20, 88, 20, 0 },
+	{ 0, 12, 86, 30, 0 },
+	{ 0,  6, 81, 41, 0 },
+	{ 0,  2, 74, 52, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 52, 74,  2, 0 },
+	{ 0, 41, 81,  6, 0 },
+	{ 0, 30, 86, 12, 0 },
+};
+
+static const struct dispc_coef coef3_M16[8] = {
+	{ 0, 22, 84, 22, 0 },
+	{ 0, 14, 82, 32, 0 },
+	{ 0,  8, 78, 42, 0 },
+	{ 0,  3, 72, 53, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 53, 72,  3, 0 },
+	{ 0, 42, 78,  8, 0 },
+	{ 0, 32, 82, 14, 0 },
+};
+
+static const struct dispc_coef coef3_M19[8] = {
+	{ 0, 24, 80, 24, 0 },
+	{ 0, 16, 79, 33, 0 },
+	{ 0,  9, 76, 43, 0 },
+	{ 0,  4, 70, 54, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 54, 70,  4, 0 },
+	{ 0, 43, 76,  9, 0 },
+	{ 0, 33, 79, 16, 0 },
+};
+
+static const struct dispc_coef coef3_M22[8] = {
+	{ 0, 25, 78, 25, 0 },
+	{ 0, 17, 77, 34, 0 },
+	{ 0, 10, 74, 44, 0 },
+	{ 0,  5, 69, 54, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 54, 69,  5, 0 },
+	{ 0, 44, 74, 10, 0 },
+	{ 0, 34, 77, 17, 0 },
+};
+
+static const struct dispc_coef coef3_M26[8] = {
+	{ 0, 26, 76, 26, 0 },
+	{ 0, 19, 74, 35, 0 },
+	{ 0, 11, 72, 45, 0 },
+	{ 0,  5, 69, 54, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 54, 69,  5, 0 },
+	{ 0, 45, 72, 11, 0 },
+	{ 0, 35, 74, 19, 0 },
+};
+
+static const struct dispc_coef coef3_M32[8] = {
+	{ 0, 27, 74, 27, 0 },
+	{ 0, 19, 73, 36, 0 },
+	{ 0, 12, 71, 45, 0 },
+	{ 0,  6, 68, 54, 0 },
+	{ 0, 64, 64,  0, 0 },
+	{ 0, 54, 68,  6, 0 },
+	{ 0, 45, 71, 12, 0 },
+	{ 0, 36, 73, 19, 0 },
+};
+
+static const struct dispc_coef coef5_M8[8] = {
+	{   0,   0, 128,   0,   0 },
+	{  -2,  14, 125, -10,   1 },
+	{  -6,  33, 114, -15,   2 },
+	{ -10,  55,  98, -16,   1 },
+	{   0, -14,  78,  78, -14 },
+	{   1, -16,  98,  55, -10 },
+	{   2, -15, 114,  33,  -6 },
+	{   1, -10, 125,  14,  -2 },
+};
+
+static const struct dispc_coef coef5_M9[8] = {
+	{  -3,  10, 114,  10,  -3 },
+	{  -6,  24, 111,   0,  -1 },
+	{  -8,  40, 103,  -7,   0 },
+	{ -11,  58,  91, -11,   1 },
+	{   0, -12,  76,  76, -12 },
+	{   1, -11,  91,  58, -11 },
+	{   0,  -7, 103,  40,  -8 },
+	{  -1,   0, 111,  24,  -6 },
+};
+
+static const struct dispc_coef coef5_M10[8] = {
+	{  -4,  18, 100,  18,  -4 },
+	{  -6,  30,  99,   8,  -3 },
+	{  -8,  44,  93,   0,  -1 },
+	{  -9,  58,  84,  -5,   0 },
+	{   0,  -8,  72,  72,  -8 },
+	{   0,  -5,  84,  58,  -9 },
+	{  -1,   0,  93,  44,  -8 },
+	{  -3,   8,  99,  30,  -6 },
+};
+
+static const struct dispc_coef coef5_M11[8] = {
+	{  -5,  23,  92,  23,  -5 },
+	{  -6,  34,  90,  13,  -3 },
+	{  -6,  45,  85,   6,  -2 },
+	{  -6,  57,  78,   0,  -1 },
+	{   0,  -4,  68,  68,  -4 },
+	{  -1,   0,  78,  57,  -6 },
+	{  -2,   6,  85,  45,  -6 },
+	{  -3,  13,  90,  34,  -6 },
+};
+
+static const struct dispc_coef coef5_M12[8] = {
+	{  -4,  26,  84,  26,  -4 },
+	{  -5,  36,  82,  18,  -3 },
+	{  -4,  46,  78,  10,  -2 },
+	{  -3,  55,  72,   5,  -1 },
+	{   0,   0,  64,  64,   0 },
+	{  -1,   5,  72,  55,  -3 },
+	{  -2,  10,  78,  46,  -4 },
+	{  -3,  18,  82,  36,  -5 },
+};
+
+static const struct dispc_coef coef5_M13[8] = {
+	{  -3,  28,  78,  28,  -3 },
+	{  -3,  37,  76,  21,  -3 },
+	{  -2,  45,  73,  14,  -2 },
+	{   0,  53,  68,   8,  -1 },
+	{   0,   3,  61,  61,   3 },
+	{  -1,   8,  68,  53,   0 },
+	{  -2,  14,  73,  45,  -2 },
+	{  -3,  21,  76,  37,  -3 },
+};
+
+static const struct dispc_coef coef5_M14[8] = {
+	{  -2,  30,  72,  30,  -2 },
+	{  -1,  37,  71,  23,  -2 },
+	{   0,  45,  69,  16,  -2 },
+	{   3,  52,  64,  10,  -1 },
+	{   0,   6,  58,  58,   6 },
+	{  -1,  10,  64,  52,   3 },
+	{  -2,  16,  69,  45,   0 },
+	{  -2,  23,  71,  37,  -1 },
+};
+
+static const struct dispc_coef coef5_M16[8] = {
+	{   0,  31,  66,  31,   0 },
+	{   1,  38,  65,  25,  -1 },
+	{   3,  44,  62,  20,  -1 },
+	{   6,  49,  59,  14,   0 },
+	{   0,  10,  54,  54,  10 },
+	{   0,  14,  59,  49,   6 },
+	{  -1,  20,  62,  44,   3 },
+	{  -1,  25,  65,  38,   1 },
+};
+
+static const struct dispc_coef coef5_M19[8] = {
+	{   3,  32,  58,  32,   3 },
+	{   4,  38,  58,  27,   1 },
+	{   7,  42,  55,  23,   1 },
+	{  10,  46,  54,  18,   0 },
+	{   0,  14,  50,  50,  14 },
+	{   0,  18,  54,  46,  10 },
+	{   1,  23,  55,  42,   7 },
+	{   1,  27,  58,  38,   4 },
+};
+
+static const struct dispc_coef coef5_M22[8] = {
+	{   4,  33,  54,  33,   4 },
+	{   6,  37,  54,  28,   3 },
+	{   9,  41,  53,  24,   1 },
+	{  12,  45,  51,  20,   0 },
+	{   0,  16,  48,  48,  16 },
+	{   0,  20,  51,  45,  12 },
+	{   1,  24,  53,  41,   9 },
+	{   3,  28,  54,  37,   6 },
+};
+
+static const struct dispc_coef coef5_M26[8] = {
+	{   6,  33,  50,  33,   6 },
+	{   8,  36,  51,  29,   4 },
+	{  11,  40,  50,  25,   2 },
+	{  14,  43,  48,  22,   1 },
+	{   0,  18,  46,  46,  18 },
+	{   1,  22,  48,  43,  14 },
+	{   2,  25,  50,  40,  11 },
+	{   4,  29,  51,  36,   8 },
+};
+
+static const struct dispc_coef coef5_M32[8] = {
+	{   7,  33,  48,  33,   7 },
+	{  10,  36,  48,  29,   5 },
+	{  13,  39,  47,  26,   3 },
+	{  16,  42,  46,  23,   1 },
+	{   0,  19,  45,  45,  19 },
+	{   1,  23,  46,  42,  16 },
+	{   3,  26,  47,  39,  13 },
+	{   5,  29,  48,  36,  10 },
+};
+
+const struct dispc_coef *dispc_ovl_get_scale_coef(int inc, int five_taps)
+{
+	int i;
+	static const struct {
+		int Mmin;
+		int Mmax;
+		const struct dispc_coef *coef_3;
+		const struct dispc_coef *coef_5;
+	} coefs[] = {
+		{ 27, 32, coef3_M32, coef5_M32 },
+		{ 23, 26, coef3_M26, coef5_M26 },
+		{ 20, 22, coef3_M22, coef5_M22 },
+		{ 17, 19, coef3_M19, coef5_M19 },
+		{ 15, 16, coef3_M16, coef5_M16 },
+		{ 14, 14, coef3_M14, coef5_M14 },
+		{ 13, 13, coef3_M13, coef5_M13 },
+		{ 12, 12, coef3_M12, coef5_M12 },
+		{ 11, 11, coef3_M11, coef5_M11 },
+		{ 10, 10, coef3_M10, coef5_M10 },
+		{  9,  9,  coef3_M9,  coef5_M9 },
+		{  4,  8,  coef3_M8,  coef5_M8 },
+		/*
+		 * When upscaling more than two times, blockiness and outlines
+		 * around the image are observed when M8 tables are used. M11,
+		 * M16 and M19 tables are used to prevent this.
+		 */
+		{  3,  3, coef3_M11, coef5_M11 },
+		{  2,  2, coef3_M16, coef5_M16 },
+		{  0,  1, coef3_M19, coef5_M19 },
+	};
+
+	inc /= 128;
+	for (i = 0; i < ARRAY_SIZE(coefs); ++i)
+		if (inc >= coefs[i].Mmin && inc <= coefs[i].Mmax)
+			return five_taps ? coefs[i].coef_5 : coefs[i].coef_3;
+	return NULL;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c
new file mode 100644
index 000000000000..75b5286029ee
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c
@@ -0,0 +1,356 @@
+/*
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DISPLAY"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sysfs.h>
+
+#include <video/omapdss.h>
+#include "dss.h"
+
+static ssize_t display_name_show(struct omap_dss_device *dssdev, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s\n",
+			dssdev->name ?
+			dssdev->name : "");
+}
+
+static ssize_t display_enabled_show(struct omap_dss_device *dssdev, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			omapdss_device_is_enabled(dssdev));
+}
+
+static ssize_t display_enabled_store(struct omap_dss_device *dssdev,
+		const char *buf, size_t size)
+{
+	int r;
+	bool enable;
+
+	r = strtobool(buf, &enable);
+	if (r)
+		return r;
+
+	if (enable == omapdss_device_is_enabled(dssdev))
+		return size;
+
+	if (omapdss_device_is_connected(dssdev) == false)
+		return -ENODEV;
+
+	if (enable) {
+		r = dssdev->driver->enable(dssdev);
+		if (r)
+			return r;
+	} else {
+		dssdev->driver->disable(dssdev);
+	}
+
+	return size;
+}
+
+static ssize_t display_tear_show(struct omap_dss_device *dssdev, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			dssdev->driver->get_te ?
+			dssdev->driver->get_te(dssdev) : 0);
+}
+
+static ssize_t display_tear_store(struct omap_dss_device *dssdev,
+	const char *buf, size_t size)
+{
+	int r;
+	bool te;
+
+	if (!dssdev->driver->enable_te || !dssdev->driver->get_te)
+		return -ENOENT;
+
+	r = strtobool(buf, &te);
+	if (r)
+		return r;
+
+	r = dssdev->driver->enable_te(dssdev, te);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t display_timings_show(struct omap_dss_device *dssdev, char *buf)
+{
+	struct omap_video_timings t;
+
+	if (!dssdev->driver->get_timings)
+		return -ENOENT;
+
+	dssdev->driver->get_timings(dssdev, &t);
+
+	return snprintf(buf, PAGE_SIZE, "%u,%u/%u/%u/%u,%u/%u/%u/%u\n",
+			t.pixelclock,
+			t.x_res, t.hfp, t.hbp, t.hsw,
+			t.y_res, t.vfp, t.vbp, t.vsw);
+}
+
+static ssize_t display_timings_store(struct omap_dss_device *dssdev,
+	const char *buf, size_t size)
+{
+	struct omap_video_timings t = dssdev->panel.timings;
+	int r, found;
+
+	if (!dssdev->driver->set_timings || !dssdev->driver->check_timings)
+		return -ENOENT;
+
+	found = 0;
+#ifdef CONFIG_FB_OMAP2_DSS_VENC
+	if (strncmp("pal", buf, 3) == 0) {
+		t = omap_dss_pal_timings;
+		found = 1;
+	} else if (strncmp("ntsc", buf, 4) == 0) {
+		t = omap_dss_ntsc_timings;
+		found = 1;
+	}
+#endif
+	if (!found && sscanf(buf, "%u,%hu/%hu/%hu/%hu,%hu/%hu/%hu/%hu",
+				&t.pixelclock,
+				&t.x_res, &t.hfp, &t.hbp, &t.hsw,
+				&t.y_res, &t.vfp, &t.vbp, &t.vsw) != 9)
+		return -EINVAL;
+
+	r = dssdev->driver->check_timings(dssdev, &t);
+	if (r)
+		return r;
+
+	dssdev->driver->disable(dssdev);
+	dssdev->driver->set_timings(dssdev, &t);
+	r = dssdev->driver->enable(dssdev);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t display_rotate_show(struct omap_dss_device *dssdev, char *buf)
+{
+	int rotate;
+	if (!dssdev->driver->get_rotate)
+		return -ENOENT;
+	rotate = dssdev->driver->get_rotate(dssdev);
+	return snprintf(buf, PAGE_SIZE, "%u\n", rotate);
+}
+
+static ssize_t display_rotate_store(struct omap_dss_device *dssdev,
+	const char *buf, size_t size)
+{
+	int rot, r;
+
+	if (!dssdev->driver->set_rotate || !dssdev->driver->get_rotate)
+		return -ENOENT;
+
+	r = kstrtoint(buf, 0, &rot);
+	if (r)
+		return r;
+
+	r = dssdev->driver->set_rotate(dssdev, rot);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t display_mirror_show(struct omap_dss_device *dssdev, char *buf)
+{
+	int mirror;
+	if (!dssdev->driver->get_mirror)
+		return -ENOENT;
+	mirror = dssdev->driver->get_mirror(dssdev);
+	return snprintf(buf, PAGE_SIZE, "%u\n", mirror);
+}
+
+static ssize_t display_mirror_store(struct omap_dss_device *dssdev,
+	const char *buf, size_t size)
+{
+	int r;
+	bool mirror;
+
+	if (!dssdev->driver->set_mirror || !dssdev->driver->get_mirror)
+		return -ENOENT;
+
+	r = strtobool(buf, &mirror);
+	if (r)
+		return r;
+
+	r = dssdev->driver->set_mirror(dssdev, mirror);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t display_wss_show(struct omap_dss_device *dssdev, char *buf)
+{
+	unsigned int wss;
+
+	if (!dssdev->driver->get_wss)
+		return -ENOENT;
+
+	wss = dssdev->driver->get_wss(dssdev);
+
+	return snprintf(buf, PAGE_SIZE, "0x%05x\n", wss);
+}
+
+static ssize_t display_wss_store(struct omap_dss_device *dssdev,
+	const char *buf, size_t size)
+{
+	u32 wss;
+	int r;
+
+	if (!dssdev->driver->get_wss || !dssdev->driver->set_wss)
+		return -ENOENT;
+
+	r = kstrtou32(buf, 0, &wss);
+	if (r)
+		return r;
+
+	if (wss > 0xfffff)
+		return -EINVAL;
+
+	r = dssdev->driver->set_wss(dssdev, wss);
+	if (r)
+		return r;
+
+	return size;
+}
+
+struct display_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct omap_dss_device *, char *);
+	ssize_t	(*store)(struct omap_dss_device *, const char *, size_t);
+};
+
+#define DISPLAY_ATTR(_name, _mode, _show, _store) \
+	struct display_attribute display_attr_##_name = \
+	__ATTR(_name, _mode, _show, _store)
+
+static DISPLAY_ATTR(name, S_IRUGO, display_name_show, NULL);
+static DISPLAY_ATTR(display_name, S_IRUGO, display_name_show, NULL);
+static DISPLAY_ATTR(enabled, S_IRUGO|S_IWUSR,
+		display_enabled_show, display_enabled_store);
+static DISPLAY_ATTR(tear_elim, S_IRUGO|S_IWUSR,
+		display_tear_show, display_tear_store);
+static DISPLAY_ATTR(timings, S_IRUGO|S_IWUSR,
+		display_timings_show, display_timings_store);
+static DISPLAY_ATTR(rotate, S_IRUGO|S_IWUSR,
+		display_rotate_show, display_rotate_store);
+static DISPLAY_ATTR(mirror, S_IRUGO|S_IWUSR,
+		display_mirror_show, display_mirror_store);
+static DISPLAY_ATTR(wss, S_IRUGO|S_IWUSR,
+		display_wss_show, display_wss_store);
+
+static struct attribute *display_sysfs_attrs[] = {
+	&display_attr_name.attr,
+	&display_attr_display_name.attr,
+	&display_attr_enabled.attr,
+	&display_attr_tear_elim.attr,
+	&display_attr_timings.attr,
+	&display_attr_rotate.attr,
+	&display_attr_mirror.attr,
+	&display_attr_wss.attr,
+	NULL
+};
+
+static ssize_t display_attr_show(struct kobject *kobj, struct attribute *attr,
+		char *buf)
+{
+	struct omap_dss_device *dssdev;
+	struct display_attribute *display_attr;
+
+	dssdev = container_of(kobj, struct omap_dss_device, kobj);
+	display_attr = container_of(attr, struct display_attribute, attr);
+
+	if (!display_attr->show)
+		return -ENOENT;
+
+	return display_attr->show(dssdev, buf);
+}
+
+static ssize_t display_attr_store(struct kobject *kobj, struct attribute *attr,
+		const char *buf, size_t size)
+{
+	struct omap_dss_device *dssdev;
+	struct display_attribute *display_attr;
+
+	dssdev = container_of(kobj, struct omap_dss_device, kobj);
+	display_attr = container_of(attr, struct display_attribute, attr);
+
+	if (!display_attr->store)
+		return -ENOENT;
+
+	return display_attr->store(dssdev, buf, size);
+}
+
+static const struct sysfs_ops display_sysfs_ops = {
+	.show = display_attr_show,
+	.store = display_attr_store,
+};
+
+static struct kobj_type display_ktype = {
+	.sysfs_ops = &display_sysfs_ops,
+	.default_attrs = display_sysfs_attrs,
+};
+
+int display_init_sysfs(struct platform_device *pdev)
+{
+	struct omap_dss_device *dssdev = NULL;
+	int r;
+
+	for_each_dss_dev(dssdev) {
+		r = kobject_init_and_add(&dssdev->kobj, &display_ktype,
+			&pdev->dev.kobj, "%s", dssdev->alias);
+		if (r) {
+			DSSERR("failed to create sysfs files\n");
+			omap_dss_put_device(dssdev);
+			goto err;
+		}
+	}
+
+	return 0;
+
+err:
+	display_uninit_sysfs(pdev);
+
+	return r;
+}
+
+void display_uninit_sysfs(struct platform_device *pdev)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (kobject_name(&dssdev->kobj) == NULL)
+			continue;
+
+		kobject_del(&dssdev->kobj);
+		kobject_put(&dssdev->kobj);
+
+		memset(&dssdev->kobj, 0, sizeof(dssdev->kobj));
+	}
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/display.c b/drivers/video/fbdev/omap2/omapfb/dss/display.c
new file mode 100644
index 000000000000..ef5b9027985d
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/display.c
@@ -0,0 +1,338 @@
+/*
+ * linux/drivers/video/omap2/dss/display.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DISPLAY"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include <video/omapdss.h>
+#include "dss.h"
+#include "dss_features.h"
+
+void omapdss_default_get_resolution(struct omap_dss_device *dssdev,
+			u16 *xres, u16 *yres)
+{
+	*xres = dssdev->panel.timings.x_res;
+	*yres = dssdev->panel.timings.y_res;
+}
+EXPORT_SYMBOL(omapdss_default_get_resolution);
+
+int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev)
+{
+	switch (dssdev->type) {
+	case OMAP_DISPLAY_TYPE_DPI:
+		if (dssdev->phy.dpi.data_lines == 24)
+			return 24;
+		else
+			return 16;
+
+	case OMAP_DISPLAY_TYPE_DBI:
+		if (dssdev->ctrl.pixel_size == 24)
+			return 24;
+		else
+			return 16;
+	case OMAP_DISPLAY_TYPE_DSI:
+		if (dsi_get_pixel_size(dssdev->panel.dsi_pix_fmt) > 16)
+			return 24;
+		else
+			return 16;
+	case OMAP_DISPLAY_TYPE_VENC:
+	case OMAP_DISPLAY_TYPE_SDI:
+	case OMAP_DISPLAY_TYPE_HDMI:
+	case OMAP_DISPLAY_TYPE_DVI:
+		return 24;
+	default:
+		BUG();
+		return 0;
+	}
+}
+EXPORT_SYMBOL(omapdss_default_get_recommended_bpp);
+
+void omapdss_default_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	*timings = dssdev->panel.timings;
+}
+EXPORT_SYMBOL(omapdss_default_get_timings);
+
+int dss_suspend_all_devices(void)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (!dssdev->driver)
+			continue;
+
+		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE) {
+			dssdev->driver->disable(dssdev);
+			dssdev->activate_after_resume = true;
+		} else {
+			dssdev->activate_after_resume = false;
+		}
+	}
+
+	return 0;
+}
+
+int dss_resume_all_devices(void)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (!dssdev->driver)
+			continue;
+
+		if (dssdev->activate_after_resume) {
+			dssdev->driver->enable(dssdev);
+			dssdev->activate_after_resume = false;
+		}
+	}
+
+	return 0;
+}
+
+void dss_disable_all_devices(void)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (!dssdev->driver)
+			continue;
+
+		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
+			dssdev->driver->disable(dssdev);
+	}
+}
+
+static LIST_HEAD(panel_list);
+static DEFINE_MUTEX(panel_list_mutex);
+static int disp_num_counter;
+
+int omapdss_register_display(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_driver *drv = dssdev->driver;
+	int id;
+
+	/*
+	 * Note: this presumes all the displays are either using DT or non-DT,
+	 * which normally should be the case. This also presumes that all
+	 * displays either have an DT alias, or none has.
+	 */
+
+	if (dssdev->dev->of_node) {
+		id = of_alias_get_id(dssdev->dev->of_node, "display");
+
+		if (id < 0)
+			id = disp_num_counter++;
+	} else {
+		id = disp_num_counter++;
+	}
+
+	snprintf(dssdev->alias, sizeof(dssdev->alias), "display%d", id);
+
+	/* Use 'label' property for name, if it exists */
+	if (dssdev->dev->of_node)
+		of_property_read_string(dssdev->dev->of_node, "label",
+			&dssdev->name);
+
+	if (dssdev->name == NULL)
+		dssdev->name = dssdev->alias;
+
+	if (drv && drv->get_resolution == NULL)
+		drv->get_resolution = omapdss_default_get_resolution;
+	if (drv && drv->get_recommended_bpp == NULL)
+		drv->get_recommended_bpp = omapdss_default_get_recommended_bpp;
+	if (drv && drv->get_timings == NULL)
+		drv->get_timings = omapdss_default_get_timings;
+
+	mutex_lock(&panel_list_mutex);
+	list_add_tail(&dssdev->panel_list, &panel_list);
+	mutex_unlock(&panel_list_mutex);
+	return 0;
+}
+EXPORT_SYMBOL(omapdss_register_display);
+
+void omapdss_unregister_display(struct omap_dss_device *dssdev)
+{
+	mutex_lock(&panel_list_mutex);
+	list_del(&dssdev->panel_list);
+	mutex_unlock(&panel_list_mutex);
+}
+EXPORT_SYMBOL(omapdss_unregister_display);
+
+struct omap_dss_device *omap_dss_get_device(struct omap_dss_device *dssdev)
+{
+	if (!try_module_get(dssdev->owner))
+		return NULL;
+
+	if (get_device(dssdev->dev) == NULL) {
+		module_put(dssdev->owner);
+		return NULL;
+	}
+
+	return dssdev;
+}
+EXPORT_SYMBOL(omap_dss_get_device);
+
+void omap_dss_put_device(struct omap_dss_device *dssdev)
+{
+	put_device(dssdev->dev);
+	module_put(dssdev->owner);
+}
+EXPORT_SYMBOL(omap_dss_put_device);
+
+/*
+ * ref count of the found device is incremented.
+ * ref count of from-device is decremented.
+ */
+struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from)
+{
+	struct list_head *l;
+	struct omap_dss_device *dssdev;
+
+	mutex_lock(&panel_list_mutex);
+
+	if (list_empty(&panel_list)) {
+		dssdev = NULL;
+		goto out;
+	}
+
+	if (from == NULL) {
+		dssdev = list_first_entry(&panel_list, struct omap_dss_device,
+				panel_list);
+		omap_dss_get_device(dssdev);
+		goto out;
+	}
+
+	omap_dss_put_device(from);
+
+	list_for_each(l, &panel_list) {
+		dssdev = list_entry(l, struct omap_dss_device, panel_list);
+		if (dssdev == from) {
+			if (list_is_last(l, &panel_list)) {
+				dssdev = NULL;
+				goto out;
+			}
+
+			dssdev = list_entry(l->next, struct omap_dss_device,
+					panel_list);
+			omap_dss_get_device(dssdev);
+			goto out;
+		}
+	}
+
+	WARN(1, "'from' dssdev not found\n");
+
+	dssdev = NULL;
+out:
+	mutex_unlock(&panel_list_mutex);
+	return dssdev;
+}
+EXPORT_SYMBOL(omap_dss_get_next_device);
+
+struct omap_dss_device *omap_dss_find_device(void *data,
+		int (*match)(struct omap_dss_device *dssdev, void *data))
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	while ((dssdev = omap_dss_get_next_device(dssdev)) != NULL) {
+		if (match(dssdev, data))
+			return dssdev;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(omap_dss_find_device);
+
+void videomode_to_omap_video_timings(const struct videomode *vm,
+		struct omap_video_timings *ovt)
+{
+	memset(ovt, 0, sizeof(*ovt));
+
+	ovt->pixelclock = vm->pixelclock;
+	ovt->x_res = vm->hactive;
+	ovt->hbp = vm->hback_porch;
+	ovt->hfp = vm->hfront_porch;
+	ovt->hsw = vm->hsync_len;
+	ovt->y_res = vm->vactive;
+	ovt->vbp = vm->vback_porch;
+	ovt->vfp = vm->vfront_porch;
+	ovt->vsw = vm->vsync_len;
+
+	ovt->vsync_level = vm->flags & DISPLAY_FLAGS_VSYNC_HIGH ?
+		OMAPDSS_SIG_ACTIVE_HIGH :
+		OMAPDSS_SIG_ACTIVE_LOW;
+	ovt->hsync_level = vm->flags & DISPLAY_FLAGS_HSYNC_HIGH ?
+		OMAPDSS_SIG_ACTIVE_HIGH :
+		OMAPDSS_SIG_ACTIVE_LOW;
+	ovt->de_level = vm->flags & DISPLAY_FLAGS_DE_HIGH ?
+		OMAPDSS_SIG_ACTIVE_HIGH :
+		OMAPDSS_SIG_ACTIVE_LOW;
+	ovt->data_pclk_edge = vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE ?
+		OMAPDSS_DRIVE_SIG_RISING_EDGE :
+		OMAPDSS_DRIVE_SIG_FALLING_EDGE;
+
+	ovt->sync_pclk_edge = ovt->data_pclk_edge;
+}
+EXPORT_SYMBOL(videomode_to_omap_video_timings);
+
+void omap_video_timings_to_videomode(const struct omap_video_timings *ovt,
+		struct videomode *vm)
+{
+	memset(vm, 0, sizeof(*vm));
+
+	vm->pixelclock = ovt->pixelclock;
+
+	vm->hactive = ovt->x_res;
+	vm->hback_porch = ovt->hbp;
+	vm->hfront_porch = ovt->hfp;
+	vm->hsync_len = ovt->hsw;
+	vm->vactive = ovt->y_res;
+	vm->vback_porch = ovt->vbp;
+	vm->vfront_porch = ovt->vfp;
+	vm->vsync_len = ovt->vsw;
+
+	if (ovt->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
+		vm->flags |= DISPLAY_FLAGS_HSYNC_HIGH;
+	else
+		vm->flags |= DISPLAY_FLAGS_HSYNC_LOW;
+
+	if (ovt->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
+		vm->flags |= DISPLAY_FLAGS_VSYNC_HIGH;
+	else
+		vm->flags |= DISPLAY_FLAGS_VSYNC_LOW;
+
+	if (ovt->de_level == OMAPDSS_SIG_ACTIVE_HIGH)
+		vm->flags |= DISPLAY_FLAGS_DE_HIGH;
+	else
+		vm->flags |= DISPLAY_FLAGS_DE_LOW;
+
+	if (ovt->data_pclk_edge == OMAPDSS_DRIVE_SIG_RISING_EDGE)
+		vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+	else
+		vm->flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+}
+EXPORT_SYMBOL(omap_video_timings_to_videomode);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dpi.c b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
new file mode 100644
index 000000000000..7953e6a52346
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
@@ -0,0 +1,899 @@
+/*
+ * linux/drivers/video/omap2/dss/dpi.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DPI"
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/string.h>
+#include <linux/of.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+#define HSDIV_DISPC	0
+
+struct dpi_data {
+	struct platform_device *pdev;
+
+	struct regulator *vdds_dsi_reg;
+	struct dss_pll *pll;
+
+	struct mutex lock;
+
+	struct omap_video_timings timings;
+	struct dss_lcd_mgr_config mgr_config;
+	int data_lines;
+
+	struct omap_dss_device output;
+
+	bool port_initialized;
+};
+
+static struct dpi_data *dpi_get_data_from_dssdev(struct omap_dss_device *dssdev)
+{
+	return container_of(dssdev, struct dpi_data, output);
+}
+
+/* only used in non-DT mode */
+static struct dpi_data *dpi_get_data_from_pdev(struct platform_device *pdev)
+{
+	return dev_get_drvdata(&pdev->dev);
+}
+
+static struct dss_pll *dpi_get_pll(enum omap_channel channel)
+{
+	/*
+	 * XXX we can't currently use DSI PLL for DPI with OMAP3, as the DSI PLL
+	 * would also be used for DISPC fclk. Meaning, when the DPI output is
+	 * disabled, DISPC clock will be disabled, and TV out will stop.
+	 */
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP24xx:
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_OMAP3630:
+	case OMAPDSS_VER_AM35xx:
+	case OMAPDSS_VER_AM43xx:
+		return NULL;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		switch (channel) {
+		case OMAP_DSS_CHANNEL_LCD:
+			return dss_pll_find("dsi0");
+		case OMAP_DSS_CHANNEL_LCD2:
+			return dss_pll_find("dsi1");
+		default:
+			return NULL;
+		}
+
+	case OMAPDSS_VER_OMAP5:
+		switch (channel) {
+		case OMAP_DSS_CHANNEL_LCD:
+			return dss_pll_find("dsi0");
+		case OMAP_DSS_CHANNEL_LCD3:
+			return dss_pll_find("dsi1");
+		default:
+			return NULL;
+		}
+
+	case OMAPDSS_VER_DRA7xx:
+		switch (channel) {
+		case OMAP_DSS_CHANNEL_LCD:
+		case OMAP_DSS_CHANNEL_LCD2:
+			return dss_pll_find("video0");
+		case OMAP_DSS_CHANNEL_LCD3:
+			return dss_pll_find("video1");
+		default:
+			return NULL;
+		}
+
+	default:
+		return NULL;
+	}
+}
+
+static enum omap_dss_clk_source dpi_get_alt_clk_src(enum omap_channel channel)
+{
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		return OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC;
+	case OMAP_DSS_CHANNEL_LCD2:
+		return OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC;
+	case OMAP_DSS_CHANNEL_LCD3:
+		return OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC;
+	default:
+		/* this shouldn't happen */
+		WARN_ON(1);
+		return OMAP_DSS_CLK_SRC_FCK;
+	}
+}
+
+struct dpi_clk_calc_ctx {
+	struct dss_pll *pll;
+
+	/* inputs */
+
+	unsigned long pck_min, pck_max;
+
+	/* outputs */
+
+	struct dss_pll_clock_info dsi_cinfo;
+	unsigned long fck;
+	struct dispc_clock_info dispc_cinfo;
+};
+
+static bool dpi_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
+		unsigned long pck, void *data)
+{
+	struct dpi_clk_calc_ctx *ctx = data;
+
+	/*
+	 * Odd dividers give us uneven duty cycle, causing problem when level
+	 * shifted. So skip all odd dividers when the pixel clock is on the
+	 * higher side.
+	 */
+	if (ctx->pck_min >= 100000000) {
+		if (lckd > 1 && lckd % 2 != 0)
+			return false;
+
+		if (pckd > 1 && pckd % 2 != 0)
+			return false;
+	}
+
+	ctx->dispc_cinfo.lck_div = lckd;
+	ctx->dispc_cinfo.pck_div = pckd;
+	ctx->dispc_cinfo.lck = lck;
+	ctx->dispc_cinfo.pck = pck;
+
+	return true;
+}
+
+
+static bool dpi_calc_hsdiv_cb(int m_dispc, unsigned long dispc,
+		void *data)
+{
+	struct dpi_clk_calc_ctx *ctx = data;
+
+	/*
+	 * Odd dividers give us uneven duty cycle, causing problem when level
+	 * shifted. So skip all odd dividers when the pixel clock is on the
+	 * higher side.
+	 */
+	if (m_dispc > 1 && m_dispc % 2 != 0 && ctx->pck_min >= 100000000)
+		return false;
+
+	ctx->dsi_cinfo.mX[HSDIV_DISPC] = m_dispc;
+	ctx->dsi_cinfo.clkout[HSDIV_DISPC] = dispc;
+
+	return dispc_div_calc(dispc, ctx->pck_min, ctx->pck_max,
+			dpi_calc_dispc_cb, ctx);
+}
+
+
+static bool dpi_calc_pll_cb(int n, int m, unsigned long fint,
+		unsigned long clkdco,
+		void *data)
+{
+	struct dpi_clk_calc_ctx *ctx = data;
+
+	ctx->dsi_cinfo.n = n;
+	ctx->dsi_cinfo.m = m;
+	ctx->dsi_cinfo.fint = fint;
+	ctx->dsi_cinfo.clkdco = clkdco;
+
+	return dss_pll_hsdiv_calc(ctx->pll, clkdco,
+		ctx->pck_min, dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+		dpi_calc_hsdiv_cb, ctx);
+}
+
+static bool dpi_calc_dss_cb(unsigned long fck, void *data)
+{
+	struct dpi_clk_calc_ctx *ctx = data;
+
+	ctx->fck = fck;
+
+	return dispc_div_calc(fck, ctx->pck_min, ctx->pck_max,
+			dpi_calc_dispc_cb, ctx);
+}
+
+static bool dpi_dsi_clk_calc(struct dpi_data *dpi, unsigned long pck,
+		struct dpi_clk_calc_ctx *ctx)
+{
+	unsigned long clkin;
+	unsigned long pll_min, pll_max;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->pll = dpi->pll;
+	ctx->pck_min = pck - 1000;
+	ctx->pck_max = pck + 1000;
+
+	pll_min = 0;
+	pll_max = 0;
+
+	clkin = clk_get_rate(ctx->pll->clkin);
+
+	return dss_pll_calc(ctx->pll, clkin,
+			pll_min, pll_max,
+			dpi_calc_pll_cb, ctx);
+}
+
+static bool dpi_dss_clk_calc(unsigned long pck, struct dpi_clk_calc_ctx *ctx)
+{
+	int i;
+
+	/*
+	 * DSS fck gives us very few possibilities, so finding a good pixel
+	 * clock may not be possible. We try multiple times to find the clock,
+	 * each time widening the pixel clock range we look for, up to
+	 * +/- ~15MHz.
+	 */
+
+	for (i = 0; i < 25; ++i) {
+		bool ok;
+
+		memset(ctx, 0, sizeof(*ctx));
+		if (pck > 1000 * i * i * i)
+			ctx->pck_min = max(pck - 1000 * i * i * i, 0lu);
+		else
+			ctx->pck_min = 0;
+		ctx->pck_max = pck + 1000 * i * i * i;
+
+		ok = dss_div_calc(pck, ctx->pck_min, dpi_calc_dss_cb, ctx);
+		if (ok)
+			return ok;
+	}
+
+	return false;
+}
+
+
+
+static int dpi_set_dsi_clk(struct dpi_data *dpi, enum omap_channel channel,
+		unsigned long pck_req, unsigned long *fck, int *lck_div,
+		int *pck_div)
+{
+	struct dpi_clk_calc_ctx ctx;
+	int r;
+	bool ok;
+
+	ok = dpi_dsi_clk_calc(dpi, pck_req, &ctx);
+	if (!ok)
+		return -EINVAL;
+
+	r = dss_pll_set_config(dpi->pll, &ctx.dsi_cinfo);
+	if (r)
+		return r;
+
+	dss_select_lcd_clk_source(channel,
+			dpi_get_alt_clk_src(channel));
+
+	dpi->mgr_config.clock_info = ctx.dispc_cinfo;
+
+	*fck = ctx.dsi_cinfo.clkout[HSDIV_DISPC];
+	*lck_div = ctx.dispc_cinfo.lck_div;
+	*pck_div = ctx.dispc_cinfo.pck_div;
+
+	return 0;
+}
+
+static int dpi_set_dispc_clk(struct dpi_data *dpi, unsigned long pck_req,
+		unsigned long *fck, int *lck_div, int *pck_div)
+{
+	struct dpi_clk_calc_ctx ctx;
+	int r;
+	bool ok;
+
+	ok = dpi_dss_clk_calc(pck_req, &ctx);
+	if (!ok)
+		return -EINVAL;
+
+	r = dss_set_fck_rate(ctx.fck);
+	if (r)
+		return r;
+
+	dpi->mgr_config.clock_info = ctx.dispc_cinfo;
+
+	*fck = ctx.fck;
+	*lck_div = ctx.dispc_cinfo.lck_div;
+	*pck_div = ctx.dispc_cinfo.pck_div;
+
+	return 0;
+}
+
+static int dpi_set_mode(struct dpi_data *dpi)
+{
+	struct omap_dss_device *out = &dpi->output;
+	struct omap_overlay_manager *mgr = out->manager;
+	struct omap_video_timings *t = &dpi->timings;
+	int lck_div = 0, pck_div = 0;
+	unsigned long fck = 0;
+	unsigned long pck;
+	int r = 0;
+
+	if (dpi->pll)
+		r = dpi_set_dsi_clk(dpi, mgr->id, t->pixelclock, &fck,
+				&lck_div, &pck_div);
+	else
+		r = dpi_set_dispc_clk(dpi, t->pixelclock, &fck,
+				&lck_div, &pck_div);
+	if (r)
+		return r;
+
+	pck = fck / lck_div / pck_div;
+
+	if (pck != t->pixelclock) {
+		DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n",
+			t->pixelclock, pck);
+
+		t->pixelclock = pck;
+	}
+
+	dss_mgr_set_timings(mgr, t);
+
+	return 0;
+}
+
+static void dpi_config_lcd_manager(struct dpi_data *dpi)
+{
+	struct omap_dss_device *out = &dpi->output;
+	struct omap_overlay_manager *mgr = out->manager;
+
+	dpi->mgr_config.io_pad_mode = DSS_IO_PAD_MODE_BYPASS;
+
+	dpi->mgr_config.stallmode = false;
+	dpi->mgr_config.fifohandcheck = false;
+
+	dpi->mgr_config.video_port_width = dpi->data_lines;
+
+	dpi->mgr_config.lcden_sig_polarity = 0;
+
+	dss_mgr_set_lcd_config(mgr, &dpi->mgr_config);
+}
+
+static int dpi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+	struct omap_dss_device *out = &dpi->output;
+	int r;
+
+	mutex_lock(&dpi->lock);
+
+	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI) && !dpi->vdds_dsi_reg) {
+		DSSERR("no VDSS_DSI regulator\n");
+		r = -ENODEV;
+		goto err_no_reg;
+	}
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		r = -ENODEV;
+		goto err_no_out_mgr;
+	}
+
+	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI)) {
+		r = regulator_enable(dpi->vdds_dsi_reg);
+		if (r)
+			goto err_reg_enable;
+	}
+
+	r = dispc_runtime_get();
+	if (r)
+		goto err_get_dispc;
+
+	r = dss_dpi_select_source(out->port_num, out->manager->id);
+	if (r)
+		goto err_src_sel;
+
+	if (dpi->pll) {
+		r = dss_pll_enable(dpi->pll);
+		if (r)
+			goto err_dsi_pll_init;
+	}
+
+	r = dpi_set_mode(dpi);
+	if (r)
+		goto err_set_mode;
+
+	dpi_config_lcd_manager(dpi);
+
+	mdelay(2);
+
+	r = dss_mgr_enable(out->manager);
+	if (r)
+		goto err_mgr_enable;
+
+	mutex_unlock(&dpi->lock);
+
+	return 0;
+
+err_mgr_enable:
+err_set_mode:
+	if (dpi->pll)
+		dss_pll_disable(dpi->pll);
+err_dsi_pll_init:
+err_src_sel:
+	dispc_runtime_put();
+err_get_dispc:
+	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+		regulator_disable(dpi->vdds_dsi_reg);
+err_reg_enable:
+err_no_out_mgr:
+err_no_reg:
+	mutex_unlock(&dpi->lock);
+	return r;
+}
+
+static void dpi_display_disable(struct omap_dss_device *dssdev)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+	struct omap_overlay_manager *mgr = dpi->output.manager;
+
+	mutex_lock(&dpi->lock);
+
+	dss_mgr_disable(mgr);
+
+	if (dpi->pll) {
+		dss_select_lcd_clk_source(mgr->id, OMAP_DSS_CLK_SRC_FCK);
+		dss_pll_disable(dpi->pll);
+	}
+
+	dispc_runtime_put();
+
+	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+		regulator_disable(dpi->vdds_dsi_reg);
+
+	mutex_unlock(&dpi->lock);
+}
+
+static void dpi_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+
+	DSSDBG("dpi_set_timings\n");
+
+	mutex_lock(&dpi->lock);
+
+	dpi->timings = *timings;
+
+	mutex_unlock(&dpi->lock);
+}
+
+static void dpi_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+
+	mutex_lock(&dpi->lock);
+
+	*timings = dpi->timings;
+
+	mutex_unlock(&dpi->lock);
+}
+
+static int dpi_check_timings(struct omap_dss_device *dssdev,
+			struct omap_video_timings *timings)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+	struct omap_overlay_manager *mgr = dpi->output.manager;
+	int lck_div, pck_div;
+	unsigned long fck;
+	unsigned long pck;
+	struct dpi_clk_calc_ctx ctx;
+	bool ok;
+
+	if (mgr && !dispc_mgr_timings_ok(mgr->id, timings))
+		return -EINVAL;
+
+	if (timings->pixelclock == 0)
+		return -EINVAL;
+
+	if (dpi->pll) {
+		ok = dpi_dsi_clk_calc(dpi, timings->pixelclock, &ctx);
+		if (!ok)
+			return -EINVAL;
+
+		fck = ctx.dsi_cinfo.clkout[HSDIV_DISPC];
+	} else {
+		ok = dpi_dss_clk_calc(timings->pixelclock, &ctx);
+		if (!ok)
+			return -EINVAL;
+
+		fck = ctx.fck;
+	}
+
+	lck_div = ctx.dispc_cinfo.lck_div;
+	pck_div = ctx.dispc_cinfo.pck_div;
+
+	pck = fck / lck_div / pck_div;
+
+	timings->pixelclock = pck;
+
+	return 0;
+}
+
+static void dpi_set_data_lines(struct omap_dss_device *dssdev, int data_lines)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+
+	mutex_lock(&dpi->lock);
+
+	dpi->data_lines = data_lines;
+
+	mutex_unlock(&dpi->lock);
+}
+
+static int dpi_verify_dsi_pll(struct dss_pll *pll)
+{
+	int r;
+
+	/* do initial setup with the PLL to see if it is operational */
+
+	r = dss_pll_enable(pll);
+	if (r)
+		return r;
+
+	dss_pll_disable(pll);
+
+	return 0;
+}
+
+static int dpi_init_regulator(struct dpi_data *dpi)
+{
+	struct regulator *vdds_dsi;
+
+	if (!dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+		return 0;
+
+	if (dpi->vdds_dsi_reg)
+		return 0;
+
+	vdds_dsi = devm_regulator_get(&dpi->pdev->dev, "vdds_dsi");
+	if (IS_ERR(vdds_dsi)) {
+		if (PTR_ERR(vdds_dsi) != -EPROBE_DEFER)
+			DSSERR("can't get VDDS_DSI regulator\n");
+		return PTR_ERR(vdds_dsi);
+	}
+
+	dpi->vdds_dsi_reg = vdds_dsi;
+
+	return 0;
+}
+
+static void dpi_init_pll(struct dpi_data *dpi)
+{
+	struct dss_pll *pll;
+
+	if (dpi->pll)
+		return;
+
+	pll = dpi_get_pll(dpi->output.dispc_channel);
+	if (!pll)
+		return;
+
+	/* On DRA7 we need to set a mux to use the PLL */
+	if (omapdss_get_version() == OMAPDSS_VER_DRA7xx)
+		dss_ctrl_pll_set_control_mux(pll->id, dpi->output.dispc_channel);
+
+	if (dpi_verify_dsi_pll(pll)) {
+		DSSWARN("DSI PLL not operational\n");
+		return;
+	}
+
+	dpi->pll = pll;
+}
+
+/*
+ * Return a hardcoded channel for the DPI output. This should work for
+ * current use cases, but this can be later expanded to either resolve
+ * the channel in some more dynamic manner, or get the channel as a user
+ * parameter.
+ */
+static enum omap_channel dpi_get_channel(int port_num)
+{
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP24xx:
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_OMAP3630:
+	case OMAPDSS_VER_AM35xx:
+	case OMAPDSS_VER_AM43xx:
+		return OMAP_DSS_CHANNEL_LCD;
+
+	case OMAPDSS_VER_DRA7xx:
+		switch (port_num) {
+		case 2:
+			return OMAP_DSS_CHANNEL_LCD3;
+		case 1:
+			return OMAP_DSS_CHANNEL_LCD2;
+		case 0:
+		default:
+			return OMAP_DSS_CHANNEL_LCD;
+		}
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		return OMAP_DSS_CHANNEL_LCD2;
+
+	case OMAPDSS_VER_OMAP5:
+		return OMAP_DSS_CHANNEL_LCD3;
+
+	default:
+		DSSWARN("unsupported DSS version\n");
+		return OMAP_DSS_CHANNEL_LCD;
+	}
+}
+
+static int dpi_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = dpi_init_regulator(dpi);
+	if (r)
+		return r;
+
+	dpi_init_pll(dpi);
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dst->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void dpi_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static const struct omapdss_dpi_ops dpi_ops = {
+	.connect = dpi_connect,
+	.disconnect = dpi_disconnect,
+
+	.enable = dpi_display_enable,
+	.disable = dpi_display_disable,
+
+	.check_timings = dpi_check_timings,
+	.set_timings = dpi_set_timings,
+	.get_timings = dpi_get_timings,
+
+	.set_data_lines = dpi_set_data_lines,
+};
+
+static void dpi_init_output(struct platform_device *pdev)
+{
+	struct dpi_data *dpi = dpi_get_data_from_pdev(pdev);
+	struct omap_dss_device *out = &dpi->output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_DPI;
+	out->output_type = OMAP_DISPLAY_TYPE_DPI;
+	out->name = "dpi.0";
+	out->dispc_channel = dpi_get_channel(0);
+	out->ops.dpi = &dpi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void dpi_uninit_output(struct platform_device *pdev)
+{
+	struct dpi_data *dpi = dpi_get_data_from_pdev(pdev);
+	struct omap_dss_device *out = &dpi->output;
+
+	omapdss_unregister_output(out);
+}
+
+static void dpi_init_output_port(struct platform_device *pdev,
+	struct device_node *port)
+{
+	struct dpi_data *dpi = port->data;
+	struct omap_dss_device *out = &dpi->output;
+	int r;
+	u32 port_num;
+
+	r = of_property_read_u32(port, "reg", &port_num);
+	if (r)
+		port_num = 0;
+
+	switch (port_num) {
+	case 2:
+		out->name = "dpi.2";
+		break;
+	case 1:
+		out->name = "dpi.1";
+		break;
+	case 0:
+	default:
+		out->name = "dpi.0";
+		break;
+	}
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_DPI;
+	out->output_type = OMAP_DISPLAY_TYPE_DPI;
+	out->dispc_channel = dpi_get_channel(port_num);
+	out->port_num = port_num;
+	out->ops.dpi = &dpi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void dpi_uninit_output_port(struct device_node *port)
+{
+	struct dpi_data *dpi = port->data;
+	struct omap_dss_device *out = &dpi->output;
+
+	omapdss_unregister_output(out);
+}
+
+static int dpi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dpi_data *dpi;
+
+	dpi = devm_kzalloc(&pdev->dev, sizeof(*dpi), GFP_KERNEL);
+	if (!dpi)
+		return -ENOMEM;
+
+	dpi->pdev = pdev;
+
+	dev_set_drvdata(&pdev->dev, dpi);
+
+	mutex_init(&dpi->lock);
+
+	dpi_init_output(pdev);
+
+	return 0;
+}
+
+static void dpi_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	dpi_uninit_output(pdev);
+}
+
+static const struct component_ops dpi_component_ops = {
+	.bind	= dpi_bind,
+	.unbind	= dpi_unbind,
+};
+
+static int dpi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dpi_component_ops);
+}
+
+static int dpi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dpi_component_ops);
+	return 0;
+}
+
+static struct platform_driver omap_dpi_driver = {
+	.probe		= dpi_probe,
+	.remove		= dpi_remove,
+	.driver         = {
+		.name   = "omapdss_dpi",
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init dpi_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_dpi_driver);
+}
+
+void dpi_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_dpi_driver);
+}
+
+int dpi_init_port(struct platform_device *pdev, struct device_node *port)
+{
+	struct dpi_data *dpi;
+	struct device_node *ep;
+	u32 datalines;
+	int r;
+
+	dpi = devm_kzalloc(&pdev->dev, sizeof(*dpi), GFP_KERNEL);
+	if (!dpi)
+		return -ENOMEM;
+
+	ep = omapdss_of_get_next_endpoint(port, NULL);
+	if (!ep)
+		return 0;
+
+	r = of_property_read_u32(ep, "data-lines", &datalines);
+	if (r) {
+		DSSERR("failed to parse datalines\n");
+		goto err_datalines;
+	}
+
+	dpi->data_lines = datalines;
+
+	of_node_put(ep);
+
+	dpi->pdev = pdev;
+	port->data = dpi;
+
+	mutex_init(&dpi->lock);
+
+	dpi_init_output_port(pdev, port);
+
+	dpi->port_initialized = true;
+
+	return 0;
+
+err_datalines:
+	of_node_put(ep);
+
+	return r;
+}
+
+void dpi_uninit_port(struct device_node *port)
+{
+	struct dpi_data *dpi = port->data;
+
+	if (!dpi->port_initialized)
+		return;
+
+	dpi_uninit_output_port(port);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
new file mode 100644
index 000000000000..0eec073b3919
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
@@ -0,0 +1,5607 @@
+/*
+ * linux/drivers/video/omap2/dss/dsi.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DSI"
+
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/mutex.h>
+#include <linux/module.h>
+#include <linux/semaphore.h>
+#include <linux/seq_file.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/debugfs.h>
+#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+#include <video/mipi_display.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+#define DSI_CATCH_MISSING_TE
+
+struct dsi_reg { u16 module; u16 idx; };
+
+#define DSI_REG(mod, idx)		((const struct dsi_reg) { mod, idx })
+
+/* DSI Protocol Engine */
+
+#define DSI_PROTO			0
+#define DSI_PROTO_SZ			0x200
+
+#define DSI_REVISION			DSI_REG(DSI_PROTO, 0x0000)
+#define DSI_SYSCONFIG			DSI_REG(DSI_PROTO, 0x0010)
+#define DSI_SYSSTATUS			DSI_REG(DSI_PROTO, 0x0014)
+#define DSI_IRQSTATUS			DSI_REG(DSI_PROTO, 0x0018)
+#define DSI_IRQENABLE			DSI_REG(DSI_PROTO, 0x001C)
+#define DSI_CTRL			DSI_REG(DSI_PROTO, 0x0040)
+#define DSI_GNQ				DSI_REG(DSI_PROTO, 0x0044)
+#define DSI_COMPLEXIO_CFG1		DSI_REG(DSI_PROTO, 0x0048)
+#define DSI_COMPLEXIO_IRQ_STATUS	DSI_REG(DSI_PROTO, 0x004C)
+#define DSI_COMPLEXIO_IRQ_ENABLE	DSI_REG(DSI_PROTO, 0x0050)
+#define DSI_CLK_CTRL			DSI_REG(DSI_PROTO, 0x0054)
+#define DSI_TIMING1			DSI_REG(DSI_PROTO, 0x0058)
+#define DSI_TIMING2			DSI_REG(DSI_PROTO, 0x005C)
+#define DSI_VM_TIMING1			DSI_REG(DSI_PROTO, 0x0060)
+#define DSI_VM_TIMING2			DSI_REG(DSI_PROTO, 0x0064)
+#define DSI_VM_TIMING3			DSI_REG(DSI_PROTO, 0x0068)
+#define DSI_CLK_TIMING			DSI_REG(DSI_PROTO, 0x006C)
+#define DSI_TX_FIFO_VC_SIZE		DSI_REG(DSI_PROTO, 0x0070)
+#define DSI_RX_FIFO_VC_SIZE		DSI_REG(DSI_PROTO, 0x0074)
+#define DSI_COMPLEXIO_CFG2		DSI_REG(DSI_PROTO, 0x0078)
+#define DSI_RX_FIFO_VC_FULLNESS		DSI_REG(DSI_PROTO, 0x007C)
+#define DSI_VM_TIMING4			DSI_REG(DSI_PROTO, 0x0080)
+#define DSI_TX_FIFO_VC_EMPTINESS	DSI_REG(DSI_PROTO, 0x0084)
+#define DSI_VM_TIMING5			DSI_REG(DSI_PROTO, 0x0088)
+#define DSI_VM_TIMING6			DSI_REG(DSI_PROTO, 0x008C)
+#define DSI_VM_TIMING7			DSI_REG(DSI_PROTO, 0x0090)
+#define DSI_STOPCLK_TIMING		DSI_REG(DSI_PROTO, 0x0094)
+#define DSI_VC_CTRL(n)			DSI_REG(DSI_PROTO, 0x0100 + (n * 0x20))
+#define DSI_VC_TE(n)			DSI_REG(DSI_PROTO, 0x0104 + (n * 0x20))
+#define DSI_VC_LONG_PACKET_HEADER(n)	DSI_REG(DSI_PROTO, 0x0108 + (n * 0x20))
+#define DSI_VC_LONG_PACKET_PAYLOAD(n)	DSI_REG(DSI_PROTO, 0x010C + (n * 0x20))
+#define DSI_VC_SHORT_PACKET_HEADER(n)	DSI_REG(DSI_PROTO, 0x0110 + (n * 0x20))
+#define DSI_VC_IRQSTATUS(n)		DSI_REG(DSI_PROTO, 0x0118 + (n * 0x20))
+#define DSI_VC_IRQENABLE(n)		DSI_REG(DSI_PROTO, 0x011C + (n * 0x20))
+
+/* DSIPHY_SCP */
+
+#define DSI_PHY				1
+#define DSI_PHY_OFFSET			0x200
+#define DSI_PHY_SZ			0x40
+
+#define DSI_DSIPHY_CFG0			DSI_REG(DSI_PHY, 0x0000)
+#define DSI_DSIPHY_CFG1			DSI_REG(DSI_PHY, 0x0004)
+#define DSI_DSIPHY_CFG2			DSI_REG(DSI_PHY, 0x0008)
+#define DSI_DSIPHY_CFG5			DSI_REG(DSI_PHY, 0x0014)
+#define DSI_DSIPHY_CFG10		DSI_REG(DSI_PHY, 0x0028)
+
+/* DSI_PLL_CTRL_SCP */
+
+#define DSI_PLL				2
+#define DSI_PLL_OFFSET			0x300
+#define DSI_PLL_SZ			0x20
+
+#define DSI_PLL_CONTROL			DSI_REG(DSI_PLL, 0x0000)
+#define DSI_PLL_STATUS			DSI_REG(DSI_PLL, 0x0004)
+#define DSI_PLL_GO			DSI_REG(DSI_PLL, 0x0008)
+#define DSI_PLL_CONFIGURATION1		DSI_REG(DSI_PLL, 0x000C)
+#define DSI_PLL_CONFIGURATION2		DSI_REG(DSI_PLL, 0x0010)
+
+#define REG_GET(dsidev, idx, start, end) \
+	FLD_GET(dsi_read_reg(dsidev, idx), start, end)
+
+#define REG_FLD_MOD(dsidev, idx, val, start, end) \
+	dsi_write_reg(dsidev, idx, FLD_MOD(dsi_read_reg(dsidev, idx), val, start, end))
+
+/* Global interrupts */
+#define DSI_IRQ_VC0		(1 << 0)
+#define DSI_IRQ_VC1		(1 << 1)
+#define DSI_IRQ_VC2		(1 << 2)
+#define DSI_IRQ_VC3		(1 << 3)
+#define DSI_IRQ_WAKEUP		(1 << 4)
+#define DSI_IRQ_RESYNC		(1 << 5)
+#define DSI_IRQ_PLL_LOCK	(1 << 7)
+#define DSI_IRQ_PLL_UNLOCK	(1 << 8)
+#define DSI_IRQ_PLL_RECALL	(1 << 9)
+#define DSI_IRQ_COMPLEXIO_ERR	(1 << 10)
+#define DSI_IRQ_HS_TX_TIMEOUT	(1 << 14)
+#define DSI_IRQ_LP_RX_TIMEOUT	(1 << 15)
+#define DSI_IRQ_TE_TRIGGER	(1 << 16)
+#define DSI_IRQ_ACK_TRIGGER	(1 << 17)
+#define DSI_IRQ_SYNC_LOST	(1 << 18)
+#define DSI_IRQ_LDO_POWER_GOOD	(1 << 19)
+#define DSI_IRQ_TA_TIMEOUT	(1 << 20)
+#define DSI_IRQ_ERROR_MASK \
+	(DSI_IRQ_HS_TX_TIMEOUT | DSI_IRQ_LP_RX_TIMEOUT | DSI_IRQ_SYNC_LOST | \
+	DSI_IRQ_TA_TIMEOUT)
+#define DSI_IRQ_CHANNEL_MASK	0xf
+
+/* Virtual channel interrupts */
+#define DSI_VC_IRQ_CS		(1 << 0)
+#define DSI_VC_IRQ_ECC_CORR	(1 << 1)
+#define DSI_VC_IRQ_PACKET_SENT	(1 << 2)
+#define DSI_VC_IRQ_FIFO_TX_OVF	(1 << 3)
+#define DSI_VC_IRQ_FIFO_RX_OVF	(1 << 4)
+#define DSI_VC_IRQ_BTA		(1 << 5)
+#define DSI_VC_IRQ_ECC_NO_CORR	(1 << 6)
+#define DSI_VC_IRQ_FIFO_TX_UDF	(1 << 7)
+#define DSI_VC_IRQ_PP_BUSY_CHANGE (1 << 8)
+#define DSI_VC_IRQ_ERROR_MASK \
+	(DSI_VC_IRQ_CS | DSI_VC_IRQ_ECC_CORR | DSI_VC_IRQ_FIFO_TX_OVF | \
+	DSI_VC_IRQ_FIFO_RX_OVF | DSI_VC_IRQ_ECC_NO_CORR | \
+	DSI_VC_IRQ_FIFO_TX_UDF)
+
+/* ComplexIO interrupts */
+#define DSI_CIO_IRQ_ERRSYNCESC1		(1 << 0)
+#define DSI_CIO_IRQ_ERRSYNCESC2		(1 << 1)
+#define DSI_CIO_IRQ_ERRSYNCESC3		(1 << 2)
+#define DSI_CIO_IRQ_ERRSYNCESC4		(1 << 3)
+#define DSI_CIO_IRQ_ERRSYNCESC5		(1 << 4)
+#define DSI_CIO_IRQ_ERRESC1		(1 << 5)
+#define DSI_CIO_IRQ_ERRESC2		(1 << 6)
+#define DSI_CIO_IRQ_ERRESC3		(1 << 7)
+#define DSI_CIO_IRQ_ERRESC4		(1 << 8)
+#define DSI_CIO_IRQ_ERRESC5		(1 << 9)
+#define DSI_CIO_IRQ_ERRCONTROL1		(1 << 10)
+#define DSI_CIO_IRQ_ERRCONTROL2		(1 << 11)
+#define DSI_CIO_IRQ_ERRCONTROL3		(1 << 12)
+#define DSI_CIO_IRQ_ERRCONTROL4		(1 << 13)
+#define DSI_CIO_IRQ_ERRCONTROL5		(1 << 14)
+#define DSI_CIO_IRQ_STATEULPS1		(1 << 15)
+#define DSI_CIO_IRQ_STATEULPS2		(1 << 16)
+#define DSI_CIO_IRQ_STATEULPS3		(1 << 17)
+#define DSI_CIO_IRQ_STATEULPS4		(1 << 18)
+#define DSI_CIO_IRQ_STATEULPS5		(1 << 19)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP0_1	(1 << 20)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP1_1	(1 << 21)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP0_2	(1 << 22)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP1_2	(1 << 23)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP0_3	(1 << 24)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP1_3	(1 << 25)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP0_4	(1 << 26)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP1_4	(1 << 27)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP0_5	(1 << 28)
+#define DSI_CIO_IRQ_ERRCONTENTIONLP1_5	(1 << 29)
+#define DSI_CIO_IRQ_ULPSACTIVENOT_ALL0	(1 << 30)
+#define DSI_CIO_IRQ_ULPSACTIVENOT_ALL1	(1 << 31)
+#define DSI_CIO_IRQ_ERROR_MASK \
+	(DSI_CIO_IRQ_ERRSYNCESC1 | DSI_CIO_IRQ_ERRSYNCESC2 | \
+	 DSI_CIO_IRQ_ERRSYNCESC3 | DSI_CIO_IRQ_ERRSYNCESC4 | \
+	 DSI_CIO_IRQ_ERRSYNCESC5 | \
+	 DSI_CIO_IRQ_ERRESC1 | DSI_CIO_IRQ_ERRESC2 | \
+	 DSI_CIO_IRQ_ERRESC3 | DSI_CIO_IRQ_ERRESC4 | \
+	 DSI_CIO_IRQ_ERRESC5 | \
+	 DSI_CIO_IRQ_ERRCONTROL1 | DSI_CIO_IRQ_ERRCONTROL2 | \
+	 DSI_CIO_IRQ_ERRCONTROL3 | DSI_CIO_IRQ_ERRCONTROL4 | \
+	 DSI_CIO_IRQ_ERRCONTROL5 | \
+	 DSI_CIO_IRQ_ERRCONTENTIONLP0_1 | DSI_CIO_IRQ_ERRCONTENTIONLP1_1 | \
+	 DSI_CIO_IRQ_ERRCONTENTIONLP0_2 | DSI_CIO_IRQ_ERRCONTENTIONLP1_2 | \
+	 DSI_CIO_IRQ_ERRCONTENTIONLP0_3 | DSI_CIO_IRQ_ERRCONTENTIONLP1_3 | \
+	 DSI_CIO_IRQ_ERRCONTENTIONLP0_4 | DSI_CIO_IRQ_ERRCONTENTIONLP1_4 | \
+	 DSI_CIO_IRQ_ERRCONTENTIONLP0_5 | DSI_CIO_IRQ_ERRCONTENTIONLP1_5)
+
+typedef void (*omap_dsi_isr_t) (void *arg, u32 mask);
+
+static int dsi_display_init_dispc(struct platform_device *dsidev,
+	struct omap_overlay_manager *mgr);
+static void dsi_display_uninit_dispc(struct platform_device *dsidev,
+	struct omap_overlay_manager *mgr);
+
+static int dsi_vc_send_null(struct omap_dss_device *dssdev, int channel);
+
+/* DSI PLL HSDIV indices */
+#define HSDIV_DISPC	0
+#define HSDIV_DSI	1
+
+#define DSI_MAX_NR_ISRS                2
+#define DSI_MAX_NR_LANES	5
+
+enum dsi_lane_function {
+	DSI_LANE_UNUSED	= 0,
+	DSI_LANE_CLK,
+	DSI_LANE_DATA1,
+	DSI_LANE_DATA2,
+	DSI_LANE_DATA3,
+	DSI_LANE_DATA4,
+};
+
+struct dsi_lane_config {
+	enum dsi_lane_function function;
+	u8 polarity;
+};
+
+struct dsi_isr_data {
+	omap_dsi_isr_t	isr;
+	void		*arg;
+	u32		mask;
+};
+
+enum fifo_size {
+	DSI_FIFO_SIZE_0		= 0,
+	DSI_FIFO_SIZE_32	= 1,
+	DSI_FIFO_SIZE_64	= 2,
+	DSI_FIFO_SIZE_96	= 3,
+	DSI_FIFO_SIZE_128	= 4,
+};
+
+enum dsi_vc_source {
+	DSI_VC_SOURCE_L4 = 0,
+	DSI_VC_SOURCE_VP,
+};
+
+struct dsi_irq_stats {
+	unsigned long last_reset;
+	unsigned irq_count;
+	unsigned dsi_irqs[32];
+	unsigned vc_irqs[4][32];
+	unsigned cio_irqs[32];
+};
+
+struct dsi_isr_tables {
+	struct dsi_isr_data isr_table[DSI_MAX_NR_ISRS];
+	struct dsi_isr_data isr_table_vc[4][DSI_MAX_NR_ISRS];
+	struct dsi_isr_data isr_table_cio[DSI_MAX_NR_ISRS];
+};
+
+struct dsi_clk_calc_ctx {
+	struct platform_device *dsidev;
+	struct dss_pll *pll;
+
+	/* inputs */
+
+	const struct omap_dss_dsi_config *config;
+
+	unsigned long req_pck_min, req_pck_nom, req_pck_max;
+
+	/* outputs */
+
+	struct dss_pll_clock_info dsi_cinfo;
+	struct dispc_clock_info dispc_cinfo;
+
+	struct omap_video_timings dispc_vm;
+	struct omap_dss_dsi_videomode_timings dsi_vm;
+};
+
+struct dsi_lp_clock_info {
+	unsigned long lp_clk;
+	u16 lp_clk_div;
+};
+
+struct dsi_data {
+	struct platform_device *pdev;
+	void __iomem *proto_base;
+	void __iomem *phy_base;
+	void __iomem *pll_base;
+
+	int module_id;
+
+	int irq;
+
+	bool is_enabled;
+
+	struct clk *dss_clk;
+
+	struct dispc_clock_info user_dispc_cinfo;
+	struct dss_pll_clock_info user_dsi_cinfo;
+
+	struct dsi_lp_clock_info user_lp_cinfo;
+	struct dsi_lp_clock_info current_lp_cinfo;
+
+	struct dss_pll pll;
+
+	bool vdds_dsi_enabled;
+	struct regulator *vdds_dsi_reg;
+
+	struct {
+		enum dsi_vc_source source;
+		struct omap_dss_device *dssdev;
+		enum fifo_size tx_fifo_size;
+		enum fifo_size rx_fifo_size;
+		int vc_id;
+	} vc[4];
+
+	struct mutex lock;
+	struct semaphore bus_lock;
+
+	spinlock_t irq_lock;
+	struct dsi_isr_tables isr_tables;
+	/* space for a copy used by the interrupt handler */
+	struct dsi_isr_tables isr_tables_copy;
+
+	int update_channel;
+#ifdef DSI_PERF_MEASURE
+	unsigned update_bytes;
+#endif
+
+	bool te_enabled;
+	bool ulps_enabled;
+
+	void (*framedone_callback)(int, void *);
+	void *framedone_data;
+
+	struct delayed_work framedone_timeout_work;
+
+#ifdef DSI_CATCH_MISSING_TE
+	struct timer_list te_timer;
+#endif
+
+	unsigned long cache_req_pck;
+	unsigned long cache_clk_freq;
+	struct dss_pll_clock_info cache_cinfo;
+
+	u32		errors;
+	spinlock_t	errors_lock;
+#ifdef DSI_PERF_MEASURE
+	ktime_t perf_setup_time;
+	ktime_t perf_start_time;
+#endif
+	int debug_read;
+	int debug_write;
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	spinlock_t irq_stats_lock;
+	struct dsi_irq_stats irq_stats;
+#endif
+
+	unsigned num_lanes_supported;
+	unsigned line_buffer_size;
+
+	struct dsi_lane_config lanes[DSI_MAX_NR_LANES];
+	unsigned num_lanes_used;
+
+	unsigned scp_clk_refcount;
+
+	struct dss_lcd_mgr_config mgr_config;
+	struct omap_video_timings timings;
+	enum omap_dss_dsi_pixel_format pix_fmt;
+	enum omap_dss_dsi_mode mode;
+	struct omap_dss_dsi_videomode_timings vm_timings;
+
+	struct omap_dss_device output;
+};
+
+struct dsi_packet_sent_handler_data {
+	struct platform_device *dsidev;
+	struct completion *completion;
+};
+
+struct dsi_module_id_data {
+	u32 address;
+	int id;
+};
+
+static const struct of_device_id dsi_of_match[];
+
+#ifdef DSI_PERF_MEASURE
+static bool dsi_perf;
+module_param(dsi_perf, bool, 0644);
+#endif
+
+static inline struct dsi_data *dsi_get_dsidrv_data(struct platform_device *dsidev)
+{
+	return dev_get_drvdata(&dsidev->dev);
+}
+
+static inline struct platform_device *dsi_get_dsidev_from_dssdev(struct omap_dss_device *dssdev)
+{
+	return to_platform_device(dssdev->dev);
+}
+
+static struct platform_device *dsi_get_dsidev_from_id(int module)
+{
+	struct omap_dss_device *out;
+	enum omap_dss_output_id	id;
+
+	switch (module) {
+	case 0:
+		id = OMAP_DSS_OUTPUT_DSI1;
+		break;
+	case 1:
+		id = OMAP_DSS_OUTPUT_DSI2;
+		break;
+	default:
+		return NULL;
+	}
+
+	out = omap_dss_get_output(id);
+
+	return out ? to_platform_device(out->dev) : NULL;
+}
+
+static inline void dsi_write_reg(struct platform_device *dsidev,
+		const struct dsi_reg idx, u32 val)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	void __iomem *base;
+
+	switch(idx.module) {
+		case DSI_PROTO: base = dsi->proto_base; break;
+		case DSI_PHY: base = dsi->phy_base; break;
+		case DSI_PLL: base = dsi->pll_base; break;
+		default: return;
+	}
+
+	__raw_writel(val, base + idx.idx);
+}
+
+static inline u32 dsi_read_reg(struct platform_device *dsidev,
+		const struct dsi_reg idx)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	void __iomem *base;
+
+	switch(idx.module) {
+		case DSI_PROTO: base = dsi->proto_base; break;
+		case DSI_PHY: base = dsi->phy_base; break;
+		case DSI_PLL: base = dsi->pll_base; break;
+		default: return 0;
+	}
+
+	return __raw_readl(base + idx.idx);
+}
+
+static void dsi_bus_lock(struct omap_dss_device *dssdev)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	down(&dsi->bus_lock);
+}
+
+static void dsi_bus_unlock(struct omap_dss_device *dssdev)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	up(&dsi->bus_lock);
+}
+
+static bool dsi_bus_is_locked(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	return dsi->bus_lock.count == 0;
+}
+
+static void dsi_completion_handler(void *data, u32 mask)
+{
+	complete((struct completion *)data);
+}
+
+static inline int wait_for_bit_change(struct platform_device *dsidev,
+		const struct dsi_reg idx, int bitnum, int value)
+{
+	unsigned long timeout;
+	ktime_t wait;
+	int t;
+
+	/* first busyloop to see if the bit changes right away */
+	t = 100;
+	while (t-- > 0) {
+		if (REG_GET(dsidev, idx, bitnum, bitnum) == value)
+			return value;
+	}
+
+	/* then loop for 500ms, sleeping for 1ms in between */
+	timeout = jiffies + msecs_to_jiffies(500);
+	while (time_before(jiffies, timeout)) {
+		if (REG_GET(dsidev, idx, bitnum, bitnum) == value)
+			return value;
+
+		wait = ns_to_ktime(1000 * 1000);
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_hrtimeout(&wait, HRTIMER_MODE_REL);
+	}
+
+	return !value;
+}
+
+u8 dsi_get_pixel_size(enum omap_dss_dsi_pixel_format fmt)
+{
+	switch (fmt) {
+	case OMAP_DSS_DSI_FMT_RGB888:
+	case OMAP_DSS_DSI_FMT_RGB666:
+		return 24;
+	case OMAP_DSS_DSI_FMT_RGB666_PACKED:
+		return 18;
+	case OMAP_DSS_DSI_FMT_RGB565:
+		return 16;
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+#ifdef DSI_PERF_MEASURE
+static void dsi_perf_mark_setup(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	dsi->perf_setup_time = ktime_get();
+}
+
+static void dsi_perf_mark_start(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	dsi->perf_start_time = ktime_get();
+}
+
+static void dsi_perf_show(struct platform_device *dsidev, const char *name)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	ktime_t t, setup_time, trans_time;
+	u32 total_bytes;
+	u32 setup_us, trans_us, total_us;
+
+	if (!dsi_perf)
+		return;
+
+	t = ktime_get();
+
+	setup_time = ktime_sub(dsi->perf_start_time, dsi->perf_setup_time);
+	setup_us = (u32)ktime_to_us(setup_time);
+	if (setup_us == 0)
+		setup_us = 1;
+
+	trans_time = ktime_sub(t, dsi->perf_start_time);
+	trans_us = (u32)ktime_to_us(trans_time);
+	if (trans_us == 0)
+		trans_us = 1;
+
+	total_us = setup_us + trans_us;
+
+	total_bytes = dsi->update_bytes;
+
+	printk(KERN_INFO "DSI(%s): %u us + %u us = %u us (%uHz), "
+			"%u bytes, %u kbytes/sec\n",
+			name,
+			setup_us,
+			trans_us,
+			total_us,
+			1000*1000 / total_us,
+			total_bytes,
+			total_bytes * 1000 / total_us);
+}
+#else
+static inline void dsi_perf_mark_setup(struct platform_device *dsidev)
+{
+}
+
+static inline void dsi_perf_mark_start(struct platform_device *dsidev)
+{
+}
+
+static inline void dsi_perf_show(struct platform_device *dsidev,
+		const char *name)
+{
+}
+#endif
+
+static int verbose_irq;
+
+static void print_irq_status(u32 status)
+{
+	if (status == 0)
+		return;
+
+	if (!verbose_irq && (status & ~DSI_IRQ_CHANNEL_MASK) == 0)
+		return;
+
+#define PIS(x) (status & DSI_IRQ_##x) ? (#x " ") : ""
+
+	pr_debug("DSI IRQ: 0x%x: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+		status,
+		verbose_irq ? PIS(VC0) : "",
+		verbose_irq ? PIS(VC1) : "",
+		verbose_irq ? PIS(VC2) : "",
+		verbose_irq ? PIS(VC3) : "",
+		PIS(WAKEUP),
+		PIS(RESYNC),
+		PIS(PLL_LOCK),
+		PIS(PLL_UNLOCK),
+		PIS(PLL_RECALL),
+		PIS(COMPLEXIO_ERR),
+		PIS(HS_TX_TIMEOUT),
+		PIS(LP_RX_TIMEOUT),
+		PIS(TE_TRIGGER),
+		PIS(ACK_TRIGGER),
+		PIS(SYNC_LOST),
+		PIS(LDO_POWER_GOOD),
+		PIS(TA_TIMEOUT));
+#undef PIS
+}
+
+static void print_irq_status_vc(int channel, u32 status)
+{
+	if (status == 0)
+		return;
+
+	if (!verbose_irq && (status & ~DSI_VC_IRQ_PACKET_SENT) == 0)
+		return;
+
+#define PIS(x) (status & DSI_VC_IRQ_##x) ? (#x " ") : ""
+
+	pr_debug("DSI VC(%d) IRQ 0x%x: %s%s%s%s%s%s%s%s%s\n",
+		channel,
+		status,
+		PIS(CS),
+		PIS(ECC_CORR),
+		PIS(ECC_NO_CORR),
+		verbose_irq ? PIS(PACKET_SENT) : "",
+		PIS(BTA),
+		PIS(FIFO_TX_OVF),
+		PIS(FIFO_RX_OVF),
+		PIS(FIFO_TX_UDF),
+		PIS(PP_BUSY_CHANGE));
+#undef PIS
+}
+
+static void print_irq_status_cio(u32 status)
+{
+	if (status == 0)
+		return;
+
+#define PIS(x) (status & DSI_CIO_IRQ_##x) ? (#x " ") : ""
+
+	pr_debug("DSI CIO IRQ 0x%x: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+		status,
+		PIS(ERRSYNCESC1),
+		PIS(ERRSYNCESC2),
+		PIS(ERRSYNCESC3),
+		PIS(ERRESC1),
+		PIS(ERRESC2),
+		PIS(ERRESC3),
+		PIS(ERRCONTROL1),
+		PIS(ERRCONTROL2),
+		PIS(ERRCONTROL3),
+		PIS(STATEULPS1),
+		PIS(STATEULPS2),
+		PIS(STATEULPS3),
+		PIS(ERRCONTENTIONLP0_1),
+		PIS(ERRCONTENTIONLP1_1),
+		PIS(ERRCONTENTIONLP0_2),
+		PIS(ERRCONTENTIONLP1_2),
+		PIS(ERRCONTENTIONLP0_3),
+		PIS(ERRCONTENTIONLP1_3),
+		PIS(ULPSACTIVENOT_ALL0),
+		PIS(ULPSACTIVENOT_ALL1));
+#undef PIS
+}
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+static void dsi_collect_irq_stats(struct platform_device *dsidev, u32 irqstatus,
+		u32 *vcstatus, u32 ciostatus)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int i;
+
+	spin_lock(&dsi->irq_stats_lock);
+
+	dsi->irq_stats.irq_count++;
+	dss_collect_irq_stats(irqstatus, dsi->irq_stats.dsi_irqs);
+
+	for (i = 0; i < 4; ++i)
+		dss_collect_irq_stats(vcstatus[i], dsi->irq_stats.vc_irqs[i]);
+
+	dss_collect_irq_stats(ciostatus, dsi->irq_stats.cio_irqs);
+
+	spin_unlock(&dsi->irq_stats_lock);
+}
+#else
+#define dsi_collect_irq_stats(dsidev, irqstatus, vcstatus, ciostatus)
+#endif
+
+static int debug_irq;
+
+static void dsi_handle_irq_errors(struct platform_device *dsidev, u32 irqstatus,
+		u32 *vcstatus, u32 ciostatus)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int i;
+
+	if (irqstatus & DSI_IRQ_ERROR_MASK) {
+		DSSERR("DSI error, irqstatus %x\n", irqstatus);
+		print_irq_status(irqstatus);
+		spin_lock(&dsi->errors_lock);
+		dsi->errors |= irqstatus & DSI_IRQ_ERROR_MASK;
+		spin_unlock(&dsi->errors_lock);
+	} else if (debug_irq) {
+		print_irq_status(irqstatus);
+	}
+
+	for (i = 0; i < 4; ++i) {
+		if (vcstatus[i] & DSI_VC_IRQ_ERROR_MASK) {
+			DSSERR("DSI VC(%d) error, vc irqstatus %x\n",
+				       i, vcstatus[i]);
+			print_irq_status_vc(i, vcstatus[i]);
+		} else if (debug_irq) {
+			print_irq_status_vc(i, vcstatus[i]);
+		}
+	}
+
+	if (ciostatus & DSI_CIO_IRQ_ERROR_MASK) {
+		DSSERR("DSI CIO error, cio irqstatus %x\n", ciostatus);
+		print_irq_status_cio(ciostatus);
+	} else if (debug_irq) {
+		print_irq_status_cio(ciostatus);
+	}
+}
+
+static void dsi_call_isrs(struct dsi_isr_data *isr_array,
+		unsigned isr_array_size, u32 irqstatus)
+{
+	struct dsi_isr_data *isr_data;
+	int i;
+
+	for (i = 0; i < isr_array_size; i++) {
+		isr_data = &isr_array[i];
+		if (isr_data->isr && isr_data->mask & irqstatus)
+			isr_data->isr(isr_data->arg, irqstatus);
+	}
+}
+
+static void dsi_handle_isrs(struct dsi_isr_tables *isr_tables,
+		u32 irqstatus, u32 *vcstatus, u32 ciostatus)
+{
+	int i;
+
+	dsi_call_isrs(isr_tables->isr_table,
+			ARRAY_SIZE(isr_tables->isr_table),
+			irqstatus);
+
+	for (i = 0; i < 4; ++i) {
+		if (vcstatus[i] == 0)
+			continue;
+		dsi_call_isrs(isr_tables->isr_table_vc[i],
+				ARRAY_SIZE(isr_tables->isr_table_vc[i]),
+				vcstatus[i]);
+	}
+
+	if (ciostatus != 0)
+		dsi_call_isrs(isr_tables->isr_table_cio,
+				ARRAY_SIZE(isr_tables->isr_table_cio),
+				ciostatus);
+}
+
+static irqreturn_t omap_dsi_irq_handler(int irq, void *arg)
+{
+	struct platform_device *dsidev;
+	struct dsi_data *dsi;
+	u32 irqstatus, vcstatus[4], ciostatus;
+	int i;
+
+	dsidev = (struct platform_device *) arg;
+	dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (!dsi->is_enabled)
+		return IRQ_NONE;
+
+	spin_lock(&dsi->irq_lock);
+
+	irqstatus = dsi_read_reg(dsidev, DSI_IRQSTATUS);
+
+	/* IRQ is not for us */
+	if (!irqstatus) {
+		spin_unlock(&dsi->irq_lock);
+		return IRQ_NONE;
+	}
+
+	dsi_write_reg(dsidev, DSI_IRQSTATUS, irqstatus & ~DSI_IRQ_CHANNEL_MASK);
+	/* flush posted write */
+	dsi_read_reg(dsidev, DSI_IRQSTATUS);
+
+	for (i = 0; i < 4; ++i) {
+		if ((irqstatus & (1 << i)) == 0) {
+			vcstatus[i] = 0;
+			continue;
+		}
+
+		vcstatus[i] = dsi_read_reg(dsidev, DSI_VC_IRQSTATUS(i));
+
+		dsi_write_reg(dsidev, DSI_VC_IRQSTATUS(i), vcstatus[i]);
+		/* flush posted write */
+		dsi_read_reg(dsidev, DSI_VC_IRQSTATUS(i));
+	}
+
+	if (irqstatus & DSI_IRQ_COMPLEXIO_ERR) {
+		ciostatus = dsi_read_reg(dsidev, DSI_COMPLEXIO_IRQ_STATUS);
+
+		dsi_write_reg(dsidev, DSI_COMPLEXIO_IRQ_STATUS, ciostatus);
+		/* flush posted write */
+		dsi_read_reg(dsidev, DSI_COMPLEXIO_IRQ_STATUS);
+	} else {
+		ciostatus = 0;
+	}
+
+#ifdef DSI_CATCH_MISSING_TE
+	if (irqstatus & DSI_IRQ_TE_TRIGGER)
+		del_timer(&dsi->te_timer);
+#endif
+
+	/* make a copy and unlock, so that isrs can unregister
+	 * themselves */
+	memcpy(&dsi->isr_tables_copy, &dsi->isr_tables,
+		sizeof(dsi->isr_tables));
+
+	spin_unlock(&dsi->irq_lock);
+
+	dsi_handle_isrs(&dsi->isr_tables_copy, irqstatus, vcstatus, ciostatus);
+
+	dsi_handle_irq_errors(dsidev, irqstatus, vcstatus, ciostatus);
+
+	dsi_collect_irq_stats(dsidev, irqstatus, vcstatus, ciostatus);
+
+	return IRQ_HANDLED;
+}
+
+/* dsi->irq_lock has to be locked by the caller */
+static void _omap_dsi_configure_irqs(struct platform_device *dsidev,
+		struct dsi_isr_data *isr_array,
+		unsigned isr_array_size, u32 default_mask,
+		const struct dsi_reg enable_reg,
+		const struct dsi_reg status_reg)
+{
+	struct dsi_isr_data *isr_data;
+	u32 mask;
+	u32 old_mask;
+	int i;
+
+	mask = default_mask;
+
+	for (i = 0; i < isr_array_size; i++) {
+		isr_data = &isr_array[i];
+
+		if (isr_data->isr == NULL)
+			continue;
+
+		mask |= isr_data->mask;
+	}
+
+	old_mask = dsi_read_reg(dsidev, enable_reg);
+	/* clear the irqstatus for newly enabled irqs */
+	dsi_write_reg(dsidev, status_reg, (mask ^ old_mask) & mask);
+	dsi_write_reg(dsidev, enable_reg, mask);
+
+	/* flush posted writes */
+	dsi_read_reg(dsidev, enable_reg);
+	dsi_read_reg(dsidev, status_reg);
+}
+
+/* dsi->irq_lock has to be locked by the caller */
+static void _omap_dsi_set_irqs(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 mask = DSI_IRQ_ERROR_MASK;
+#ifdef DSI_CATCH_MISSING_TE
+	mask |= DSI_IRQ_TE_TRIGGER;
+#endif
+	_omap_dsi_configure_irqs(dsidev, dsi->isr_tables.isr_table,
+			ARRAY_SIZE(dsi->isr_tables.isr_table), mask,
+			DSI_IRQENABLE, DSI_IRQSTATUS);
+}
+
+/* dsi->irq_lock has to be locked by the caller */
+static void _omap_dsi_set_irqs_vc(struct platform_device *dsidev, int vc)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	_omap_dsi_configure_irqs(dsidev, dsi->isr_tables.isr_table_vc[vc],
+			ARRAY_SIZE(dsi->isr_tables.isr_table_vc[vc]),
+			DSI_VC_IRQ_ERROR_MASK,
+			DSI_VC_IRQENABLE(vc), DSI_VC_IRQSTATUS(vc));
+}
+
+/* dsi->irq_lock has to be locked by the caller */
+static void _omap_dsi_set_irqs_cio(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	_omap_dsi_configure_irqs(dsidev, dsi->isr_tables.isr_table_cio,
+			ARRAY_SIZE(dsi->isr_tables.isr_table_cio),
+			DSI_CIO_IRQ_ERROR_MASK,
+			DSI_COMPLEXIO_IRQ_ENABLE, DSI_COMPLEXIO_IRQ_STATUS);
+}
+
+static void _dsi_initialize_irq(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int vc;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	memset(&dsi->isr_tables, 0, sizeof(dsi->isr_tables));
+
+	_omap_dsi_set_irqs(dsidev);
+	for (vc = 0; vc < 4; ++vc)
+		_omap_dsi_set_irqs_vc(dsidev, vc);
+	_omap_dsi_set_irqs_cio(dsidev);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+}
+
+static int _dsi_register_isr(omap_dsi_isr_t isr, void *arg, u32 mask,
+		struct dsi_isr_data *isr_array, unsigned isr_array_size)
+{
+	struct dsi_isr_data *isr_data;
+	int free_idx;
+	int i;
+
+	BUG_ON(isr == NULL);
+
+	/* check for duplicate entry and find a free slot */
+	free_idx = -1;
+	for (i = 0; i < isr_array_size; i++) {
+		isr_data = &isr_array[i];
+
+		if (isr_data->isr == isr && isr_data->arg == arg &&
+				isr_data->mask == mask) {
+			return -EINVAL;
+		}
+
+		if (isr_data->isr == NULL && free_idx == -1)
+			free_idx = i;
+	}
+
+	if (free_idx == -1)
+		return -EBUSY;
+
+	isr_data = &isr_array[free_idx];
+	isr_data->isr = isr;
+	isr_data->arg = arg;
+	isr_data->mask = mask;
+
+	return 0;
+}
+
+static int _dsi_unregister_isr(omap_dsi_isr_t isr, void *arg, u32 mask,
+		struct dsi_isr_data *isr_array, unsigned isr_array_size)
+{
+	struct dsi_isr_data *isr_data;
+	int i;
+
+	for (i = 0; i < isr_array_size; i++) {
+		isr_data = &isr_array[i];
+		if (isr_data->isr != isr || isr_data->arg != arg ||
+				isr_data->mask != mask)
+			continue;
+
+		isr_data->isr = NULL;
+		isr_data->arg = NULL;
+		isr_data->mask = 0;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int dsi_register_isr(struct platform_device *dsidev, omap_dsi_isr_t isr,
+		void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_register_isr(isr, arg, mask, dsi->isr_tables.isr_table,
+			ARRAY_SIZE(dsi->isr_tables.isr_table));
+
+	if (r == 0)
+		_omap_dsi_set_irqs(dsidev);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static int dsi_unregister_isr(struct platform_device *dsidev,
+		omap_dsi_isr_t isr, void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_unregister_isr(isr, arg, mask, dsi->isr_tables.isr_table,
+			ARRAY_SIZE(dsi->isr_tables.isr_table));
+
+	if (r == 0)
+		_omap_dsi_set_irqs(dsidev);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static int dsi_register_isr_vc(struct platform_device *dsidev, int channel,
+		omap_dsi_isr_t isr, void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_register_isr(isr, arg, mask,
+			dsi->isr_tables.isr_table_vc[channel],
+			ARRAY_SIZE(dsi->isr_tables.isr_table_vc[channel]));
+
+	if (r == 0)
+		_omap_dsi_set_irqs_vc(dsidev, channel);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static int dsi_unregister_isr_vc(struct platform_device *dsidev, int channel,
+		omap_dsi_isr_t isr, void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_unregister_isr(isr, arg, mask,
+			dsi->isr_tables.isr_table_vc[channel],
+			ARRAY_SIZE(dsi->isr_tables.isr_table_vc[channel]));
+
+	if (r == 0)
+		_omap_dsi_set_irqs_vc(dsidev, channel);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static int dsi_register_isr_cio(struct platform_device *dsidev,
+		omap_dsi_isr_t isr, void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_register_isr(isr, arg, mask, dsi->isr_tables.isr_table_cio,
+			ARRAY_SIZE(dsi->isr_tables.isr_table_cio));
+
+	if (r == 0)
+		_omap_dsi_set_irqs_cio(dsidev);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static int dsi_unregister_isr_cio(struct platform_device *dsidev,
+		omap_dsi_isr_t isr, void *arg, u32 mask)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	int r;
+
+	spin_lock_irqsave(&dsi->irq_lock, flags);
+
+	r = _dsi_unregister_isr(isr, arg, mask, dsi->isr_tables.isr_table_cio,
+			ARRAY_SIZE(dsi->isr_tables.isr_table_cio));
+
+	if (r == 0)
+		_omap_dsi_set_irqs_cio(dsidev);
+
+	spin_unlock_irqrestore(&dsi->irq_lock, flags);
+
+	return r;
+}
+
+static u32 dsi_get_errors(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	u32 e;
+	spin_lock_irqsave(&dsi->errors_lock, flags);
+	e = dsi->errors;
+	dsi->errors = 0;
+	spin_unlock_irqrestore(&dsi->errors_lock, flags);
+	return e;
+}
+
+static int dsi_runtime_get(struct platform_device *dsidev)
+{
+	int r;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	DSSDBG("dsi_runtime_get\n");
+
+	r = pm_runtime_get_sync(&dsi->pdev->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+static void dsi_runtime_put(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r;
+
+	DSSDBG("dsi_runtime_put\n");
+
+	r = pm_runtime_put_sync(&dsi->pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+
+static int dsi_regulator_init(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct regulator *vdds_dsi;
+	int r;
+
+	if (dsi->vdds_dsi_reg != NULL)
+		return 0;
+
+	vdds_dsi = devm_regulator_get(&dsi->pdev->dev, "vdd");
+
+	if (IS_ERR(vdds_dsi)) {
+		if (PTR_ERR(vdds_dsi) != -EPROBE_DEFER)
+			DSSERR("can't get DSI VDD regulator\n");
+		return PTR_ERR(vdds_dsi);
+	}
+
+	if (regulator_can_change_voltage(vdds_dsi)) {
+		r = regulator_set_voltage(vdds_dsi, 1800000, 1800000);
+		if (r) {
+			devm_regulator_put(vdds_dsi);
+			DSSERR("can't set the DSI regulator voltage\n");
+			return r;
+		}
+	}
+
+	dsi->vdds_dsi_reg = vdds_dsi;
+
+	return 0;
+}
+
+static void _dsi_print_reset_status(struct platform_device *dsidev)
+{
+	u32 l;
+	int b0, b1, b2;
+
+	/* A dummy read using the SCP interface to any DSIPHY register is
+	 * required after DSIPHY reset to complete the reset of the DSI complex
+	 * I/O. */
+	l = dsi_read_reg(dsidev, DSI_DSIPHY_CFG5);
+
+	if (dss_has_feature(FEAT_DSI_REVERSE_TXCLKESC)) {
+		b0 = 28;
+		b1 = 27;
+		b2 = 26;
+	} else {
+		b0 = 24;
+		b1 = 25;
+		b2 = 26;
+	}
+
+#define DSI_FLD_GET(fld, start, end)\
+	FLD_GET(dsi_read_reg(dsidev, DSI_##fld), start, end)
+
+	pr_debug("DSI resets: PLL (%d) CIO (%d) PHY (%x%x%x, %d, %d, %d)\n",
+		DSI_FLD_GET(PLL_STATUS, 0, 0),
+		DSI_FLD_GET(COMPLEXIO_CFG1, 29, 29),
+		DSI_FLD_GET(DSIPHY_CFG5, b0, b0),
+		DSI_FLD_GET(DSIPHY_CFG5, b1, b1),
+		DSI_FLD_GET(DSIPHY_CFG5, b2, b2),
+		DSI_FLD_GET(DSIPHY_CFG5, 29, 29),
+		DSI_FLD_GET(DSIPHY_CFG5, 30, 30),
+		DSI_FLD_GET(DSIPHY_CFG5, 31, 31));
+
+#undef DSI_FLD_GET
+}
+
+static inline int dsi_if_enable(struct platform_device *dsidev, bool enable)
+{
+	DSSDBG("dsi_if_enable(%d)\n", enable);
+
+	enable = enable ? 1 : 0;
+	REG_FLD_MOD(dsidev, DSI_CTRL, enable, 0, 0); /* IF_EN */
+
+	if (wait_for_bit_change(dsidev, DSI_CTRL, 0, enable) != enable) {
+			DSSERR("Failed to set dsi_if_enable to %d\n", enable);
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static unsigned long dsi_get_pll_hsdiv_dispc_rate(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	return dsi->pll.cinfo.clkout[HSDIV_DISPC];
+}
+
+static unsigned long dsi_get_pll_hsdiv_dsi_rate(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	return dsi->pll.cinfo.clkout[HSDIV_DSI];
+}
+
+static unsigned long dsi_get_txbyteclkhs(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	return dsi->pll.cinfo.clkdco / 16;
+}
+
+static unsigned long dsi_fclk_rate(struct platform_device *dsidev)
+{
+	unsigned long r;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (dss_get_dsi_clk_source(dsi->module_id) == OMAP_DSS_CLK_SRC_FCK) {
+		/* DSI FCLK source is DSS_CLK_FCK */
+		r = clk_get_rate(dsi->dss_clk);
+	} else {
+		/* DSI FCLK source is dsi_pll_hsdiv_dsi_clk */
+		r = dsi_get_pll_hsdiv_dsi_rate(dsidev);
+	}
+
+	return r;
+}
+
+static int dsi_lp_clock_calc(unsigned long dsi_fclk,
+		unsigned long lp_clk_min, unsigned long lp_clk_max,
+		struct dsi_lp_clock_info *lp_cinfo)
+{
+	unsigned lp_clk_div;
+	unsigned long lp_clk;
+
+	lp_clk_div = DIV_ROUND_UP(dsi_fclk, lp_clk_max * 2);
+	lp_clk = dsi_fclk / 2 / lp_clk_div;
+
+	if (lp_clk < lp_clk_min || lp_clk > lp_clk_max)
+		return -EINVAL;
+
+	lp_cinfo->lp_clk_div = lp_clk_div;
+	lp_cinfo->lp_clk = lp_clk;
+
+	return 0;
+}
+
+static int dsi_set_lp_clk_divisor(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long dsi_fclk;
+	unsigned lp_clk_div;
+	unsigned long lp_clk;
+	unsigned lpdiv_max = dss_feat_get_param_max(FEAT_PARAM_DSIPLL_LPDIV);
+
+
+	lp_clk_div = dsi->user_lp_cinfo.lp_clk_div;
+
+	if (lp_clk_div == 0 || lp_clk_div > lpdiv_max)
+		return -EINVAL;
+
+	dsi_fclk = dsi_fclk_rate(dsidev);
+
+	lp_clk = dsi_fclk / 2 / lp_clk_div;
+
+	DSSDBG("LP_CLK_DIV %u, LP_CLK %lu\n", lp_clk_div, lp_clk);
+	dsi->current_lp_cinfo.lp_clk = lp_clk;
+	dsi->current_lp_cinfo.lp_clk_div = lp_clk_div;
+
+	/* LP_CLK_DIVISOR */
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, lp_clk_div, 12, 0);
+
+	/* LP_RX_SYNCHRO_ENABLE */
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, dsi_fclk > 30000000 ? 1 : 0, 21, 21);
+
+	return 0;
+}
+
+static void dsi_enable_scp_clk(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (dsi->scp_clk_refcount++ == 0)
+		REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 1, 14, 14); /* CIO_CLK_ICG */
+}
+
+static void dsi_disable_scp_clk(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	WARN_ON(dsi->scp_clk_refcount == 0);
+	if (--dsi->scp_clk_refcount == 0)
+		REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 0, 14, 14); /* CIO_CLK_ICG */
+}
+
+enum dsi_pll_power_state {
+	DSI_PLL_POWER_OFF	= 0x0,
+	DSI_PLL_POWER_ON_HSCLK	= 0x1,
+	DSI_PLL_POWER_ON_ALL	= 0x2,
+	DSI_PLL_POWER_ON_DIV	= 0x3,
+};
+
+static int dsi_pll_power(struct platform_device *dsidev,
+		enum dsi_pll_power_state state)
+{
+	int t = 0;
+
+	/* DSI-PLL power command 0x3 is not working */
+	if (dss_has_feature(FEAT_DSI_PLL_PWR_BUG) &&
+			state == DSI_PLL_POWER_ON_DIV)
+		state = DSI_PLL_POWER_ON_ALL;
+
+	/* PLL_PWR_CMD */
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, state, 31, 30);
+
+	/* PLL_PWR_STATUS */
+	while (FLD_GET(dsi_read_reg(dsidev, DSI_CLK_CTRL), 29, 28) != state) {
+		if (++t > 1000) {
+			DSSERR("Failed to set DSI PLL power mode to %d\n",
+					state);
+			return -ENODEV;
+		}
+		udelay(1);
+	}
+
+	return 0;
+}
+
+
+static void dsi_pll_calc_dsi_fck(struct dss_pll_clock_info *cinfo)
+{
+	unsigned long max_dsi_fck;
+
+	max_dsi_fck = dss_feat_get_param_max(FEAT_PARAM_DSI_FCK);
+
+	cinfo->mX[HSDIV_DSI] = DIV_ROUND_UP(cinfo->clkdco, max_dsi_fck);
+	cinfo->clkout[HSDIV_DSI] = cinfo->clkdco / cinfo->mX[HSDIV_DSI];
+}
+
+static int dsi_pll_enable(struct dss_pll *pll)
+{
+	struct dsi_data *dsi = container_of(pll, struct dsi_data, pll);
+	struct platform_device *dsidev = dsi->pdev;
+	int r = 0;
+
+	DSSDBG("PLL init\n");
+
+	r = dsi_regulator_init(dsidev);
+	if (r)
+		return r;
+
+	r = dsi_runtime_get(dsidev);
+	if (r)
+		return r;
+
+	/*
+	 * Note: SCP CLK is not required on OMAP3, but it is required on OMAP4.
+	 */
+	dsi_enable_scp_clk(dsidev);
+
+	if (!dsi->vdds_dsi_enabled) {
+		r = regulator_enable(dsi->vdds_dsi_reg);
+		if (r)
+			goto err0;
+		dsi->vdds_dsi_enabled = true;
+	}
+
+	/* XXX PLL does not come out of reset without this... */
+	dispc_pck_free_enable(1);
+
+	if (wait_for_bit_change(dsidev, DSI_PLL_STATUS, 0, 1) != 1) {
+		DSSERR("PLL not coming out of reset.\n");
+		r = -ENODEV;
+		dispc_pck_free_enable(0);
+		goto err1;
+	}
+
+	/* XXX ... but if left on, we get problems when planes do not
+	 * fill the whole display. No idea about this */
+	dispc_pck_free_enable(0);
+
+	r = dsi_pll_power(dsidev, DSI_PLL_POWER_ON_ALL);
+
+	if (r)
+		goto err1;
+
+	DSSDBG("PLL init done\n");
+
+	return 0;
+err1:
+	if (dsi->vdds_dsi_enabled) {
+		regulator_disable(dsi->vdds_dsi_reg);
+		dsi->vdds_dsi_enabled = false;
+	}
+err0:
+	dsi_disable_scp_clk(dsidev);
+	dsi_runtime_put(dsidev);
+	return r;
+}
+
+static void dsi_pll_uninit(struct platform_device *dsidev, bool disconnect_lanes)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	dsi_pll_power(dsidev, DSI_PLL_POWER_OFF);
+	if (disconnect_lanes) {
+		WARN_ON(!dsi->vdds_dsi_enabled);
+		regulator_disable(dsi->vdds_dsi_reg);
+		dsi->vdds_dsi_enabled = false;
+	}
+
+	dsi_disable_scp_clk(dsidev);
+	dsi_runtime_put(dsidev);
+
+	DSSDBG("PLL uninit done\n");
+}
+
+static void dsi_pll_disable(struct dss_pll *pll)
+{
+	struct dsi_data *dsi = container_of(pll, struct dsi_data, pll);
+	struct platform_device *dsidev = dsi->pdev;
+
+	dsi_pll_uninit(dsidev, true);
+}
+
+static void dsi_dump_dsidev_clocks(struct platform_device *dsidev,
+		struct seq_file *s)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct dss_pll_clock_info *cinfo = &dsi->pll.cinfo;
+	enum omap_dss_clk_source dispc_clk_src, dsi_clk_src;
+	int dsi_module = dsi->module_id;
+	struct dss_pll *pll = &dsi->pll;
+
+	dispc_clk_src = dss_get_dispc_clk_source();
+	dsi_clk_src = dss_get_dsi_clk_source(dsi_module);
+
+	if (dsi_runtime_get(dsidev))
+		return;
+
+	seq_printf(s,	"- DSI%d PLL -\n", dsi_module + 1);
+
+	seq_printf(s,	"dsi pll clkin\t%lu\n", clk_get_rate(pll->clkin));
+
+	seq_printf(s,	"Fint\t\t%-16lun %u\n", cinfo->fint, cinfo->n);
+
+	seq_printf(s,	"CLKIN4DDR\t%-16lum %u\n",
+			cinfo->clkdco, cinfo->m);
+
+	seq_printf(s,	"DSI_PLL_HSDIV_DISPC (%s)\t%-16lum_dispc %u\t(%s)\n",
+			dss_feat_get_clk_source_name(dsi_module == 0 ?
+				OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC :
+				OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC),
+			cinfo->clkout[HSDIV_DISPC],
+			cinfo->mX[HSDIV_DISPC],
+			dispc_clk_src == OMAP_DSS_CLK_SRC_FCK ?
+			"off" : "on");
+
+	seq_printf(s,	"DSI_PLL_HSDIV_DSI (%s)\t%-16lum_dsi %u\t(%s)\n",
+			dss_feat_get_clk_source_name(dsi_module == 0 ?
+				OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI :
+				OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI),
+			cinfo->clkout[HSDIV_DSI],
+			cinfo->mX[HSDIV_DSI],
+			dsi_clk_src == OMAP_DSS_CLK_SRC_FCK ?
+			"off" : "on");
+
+	seq_printf(s,	"- DSI%d -\n", dsi_module + 1);
+
+	seq_printf(s,	"dsi fclk source = %s (%s)\n",
+			dss_get_generic_clk_source_name(dsi_clk_src),
+			dss_feat_get_clk_source_name(dsi_clk_src));
+
+	seq_printf(s,	"DSI_FCLK\t%lu\n", dsi_fclk_rate(dsidev));
+
+	seq_printf(s,	"DDR_CLK\t\t%lu\n",
+			cinfo->clkdco / 4);
+
+	seq_printf(s,	"TxByteClkHS\t%lu\n", dsi_get_txbyteclkhs(dsidev));
+
+	seq_printf(s,	"LP_CLK\t\t%lu\n", dsi->current_lp_cinfo.lp_clk);
+
+	dsi_runtime_put(dsidev);
+}
+
+void dsi_dump_clocks(struct seq_file *s)
+{
+	struct platform_device *dsidev;
+	int i;
+
+	for  (i = 0; i < MAX_NUM_DSI; i++) {
+		dsidev = dsi_get_dsidev_from_id(i);
+		if (dsidev)
+			dsi_dump_dsidev_clocks(dsidev, s);
+	}
+}
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+static void dsi_dump_dsidev_irqs(struct platform_device *dsidev,
+		struct seq_file *s)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned long flags;
+	struct dsi_irq_stats stats;
+
+	spin_lock_irqsave(&dsi->irq_stats_lock, flags);
+
+	stats = dsi->irq_stats;
+	memset(&dsi->irq_stats, 0, sizeof(dsi->irq_stats));
+	dsi->irq_stats.last_reset = jiffies;
+
+	spin_unlock_irqrestore(&dsi->irq_stats_lock, flags);
+
+	seq_printf(s, "period %u ms\n",
+			jiffies_to_msecs(jiffies - stats.last_reset));
+
+	seq_printf(s, "irqs %d\n", stats.irq_count);
+#define PIS(x) \
+	seq_printf(s, "%-20s %10d\n", #x, stats.dsi_irqs[ffs(DSI_IRQ_##x)-1]);
+
+	seq_printf(s, "-- DSI%d interrupts --\n", dsi->module_id + 1);
+	PIS(VC0);
+	PIS(VC1);
+	PIS(VC2);
+	PIS(VC3);
+	PIS(WAKEUP);
+	PIS(RESYNC);
+	PIS(PLL_LOCK);
+	PIS(PLL_UNLOCK);
+	PIS(PLL_RECALL);
+	PIS(COMPLEXIO_ERR);
+	PIS(HS_TX_TIMEOUT);
+	PIS(LP_RX_TIMEOUT);
+	PIS(TE_TRIGGER);
+	PIS(ACK_TRIGGER);
+	PIS(SYNC_LOST);
+	PIS(LDO_POWER_GOOD);
+	PIS(TA_TIMEOUT);
+#undef PIS
+
+#define PIS(x) \
+	seq_printf(s, "%-20s %10d %10d %10d %10d\n", #x, \
+			stats.vc_irqs[0][ffs(DSI_VC_IRQ_##x)-1], \
+			stats.vc_irqs[1][ffs(DSI_VC_IRQ_##x)-1], \
+			stats.vc_irqs[2][ffs(DSI_VC_IRQ_##x)-1], \
+			stats.vc_irqs[3][ffs(DSI_VC_IRQ_##x)-1]);
+
+	seq_printf(s, "-- VC interrupts --\n");
+	PIS(CS);
+	PIS(ECC_CORR);
+	PIS(PACKET_SENT);
+	PIS(FIFO_TX_OVF);
+	PIS(FIFO_RX_OVF);
+	PIS(BTA);
+	PIS(ECC_NO_CORR);
+	PIS(FIFO_TX_UDF);
+	PIS(PP_BUSY_CHANGE);
+#undef PIS
+
+#define PIS(x) \
+	seq_printf(s, "%-20s %10d\n", #x, \
+			stats.cio_irqs[ffs(DSI_CIO_IRQ_##x)-1]);
+
+	seq_printf(s, "-- CIO interrupts --\n");
+	PIS(ERRSYNCESC1);
+	PIS(ERRSYNCESC2);
+	PIS(ERRSYNCESC3);
+	PIS(ERRESC1);
+	PIS(ERRESC2);
+	PIS(ERRESC3);
+	PIS(ERRCONTROL1);
+	PIS(ERRCONTROL2);
+	PIS(ERRCONTROL3);
+	PIS(STATEULPS1);
+	PIS(STATEULPS2);
+	PIS(STATEULPS3);
+	PIS(ERRCONTENTIONLP0_1);
+	PIS(ERRCONTENTIONLP1_1);
+	PIS(ERRCONTENTIONLP0_2);
+	PIS(ERRCONTENTIONLP1_2);
+	PIS(ERRCONTENTIONLP0_3);
+	PIS(ERRCONTENTIONLP1_3);
+	PIS(ULPSACTIVENOT_ALL0);
+	PIS(ULPSACTIVENOT_ALL1);
+#undef PIS
+}
+
+static void dsi1_dump_irqs(struct seq_file *s)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_id(0);
+
+	dsi_dump_dsidev_irqs(dsidev, s);
+}
+
+static void dsi2_dump_irqs(struct seq_file *s)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_id(1);
+
+	dsi_dump_dsidev_irqs(dsidev, s);
+}
+#endif
+
+static void dsi_dump_dsidev_regs(struct platform_device *dsidev,
+		struct seq_file *s)
+{
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, dsi_read_reg(dsidev, r))
+
+	if (dsi_runtime_get(dsidev))
+		return;
+	dsi_enable_scp_clk(dsidev);
+
+	DUMPREG(DSI_REVISION);
+	DUMPREG(DSI_SYSCONFIG);
+	DUMPREG(DSI_SYSSTATUS);
+	DUMPREG(DSI_IRQSTATUS);
+	DUMPREG(DSI_IRQENABLE);
+	DUMPREG(DSI_CTRL);
+	DUMPREG(DSI_COMPLEXIO_CFG1);
+	DUMPREG(DSI_COMPLEXIO_IRQ_STATUS);
+	DUMPREG(DSI_COMPLEXIO_IRQ_ENABLE);
+	DUMPREG(DSI_CLK_CTRL);
+	DUMPREG(DSI_TIMING1);
+	DUMPREG(DSI_TIMING2);
+	DUMPREG(DSI_VM_TIMING1);
+	DUMPREG(DSI_VM_TIMING2);
+	DUMPREG(DSI_VM_TIMING3);
+	DUMPREG(DSI_CLK_TIMING);
+	DUMPREG(DSI_TX_FIFO_VC_SIZE);
+	DUMPREG(DSI_RX_FIFO_VC_SIZE);
+	DUMPREG(DSI_COMPLEXIO_CFG2);
+	DUMPREG(DSI_RX_FIFO_VC_FULLNESS);
+	DUMPREG(DSI_VM_TIMING4);
+	DUMPREG(DSI_TX_FIFO_VC_EMPTINESS);
+	DUMPREG(DSI_VM_TIMING5);
+	DUMPREG(DSI_VM_TIMING6);
+	DUMPREG(DSI_VM_TIMING7);
+	DUMPREG(DSI_STOPCLK_TIMING);
+
+	DUMPREG(DSI_VC_CTRL(0));
+	DUMPREG(DSI_VC_TE(0));
+	DUMPREG(DSI_VC_LONG_PACKET_HEADER(0));
+	DUMPREG(DSI_VC_LONG_PACKET_PAYLOAD(0));
+	DUMPREG(DSI_VC_SHORT_PACKET_HEADER(0));
+	DUMPREG(DSI_VC_IRQSTATUS(0));
+	DUMPREG(DSI_VC_IRQENABLE(0));
+
+	DUMPREG(DSI_VC_CTRL(1));
+	DUMPREG(DSI_VC_TE(1));
+	DUMPREG(DSI_VC_LONG_PACKET_HEADER(1));
+	DUMPREG(DSI_VC_LONG_PACKET_PAYLOAD(1));
+	DUMPREG(DSI_VC_SHORT_PACKET_HEADER(1));
+	DUMPREG(DSI_VC_IRQSTATUS(1));
+	DUMPREG(DSI_VC_IRQENABLE(1));
+
+	DUMPREG(DSI_VC_CTRL(2));
+	DUMPREG(DSI_VC_TE(2));
+	DUMPREG(DSI_VC_LONG_PACKET_HEADER(2));
+	DUMPREG(DSI_VC_LONG_PACKET_PAYLOAD(2));
+	DUMPREG(DSI_VC_SHORT_PACKET_HEADER(2));
+	DUMPREG(DSI_VC_IRQSTATUS(2));
+	DUMPREG(DSI_VC_IRQENABLE(2));
+
+	DUMPREG(DSI_VC_CTRL(3));
+	DUMPREG(DSI_VC_TE(3));
+	DUMPREG(DSI_VC_LONG_PACKET_HEADER(3));
+	DUMPREG(DSI_VC_LONG_PACKET_PAYLOAD(3));
+	DUMPREG(DSI_VC_SHORT_PACKET_HEADER(3));
+	DUMPREG(DSI_VC_IRQSTATUS(3));
+	DUMPREG(DSI_VC_IRQENABLE(3));
+
+	DUMPREG(DSI_DSIPHY_CFG0);
+	DUMPREG(DSI_DSIPHY_CFG1);
+	DUMPREG(DSI_DSIPHY_CFG2);
+	DUMPREG(DSI_DSIPHY_CFG5);
+
+	DUMPREG(DSI_PLL_CONTROL);
+	DUMPREG(DSI_PLL_STATUS);
+	DUMPREG(DSI_PLL_GO);
+	DUMPREG(DSI_PLL_CONFIGURATION1);
+	DUMPREG(DSI_PLL_CONFIGURATION2);
+
+	dsi_disable_scp_clk(dsidev);
+	dsi_runtime_put(dsidev);
+#undef DUMPREG
+}
+
+static void dsi1_dump_regs(struct seq_file *s)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_id(0);
+
+	dsi_dump_dsidev_regs(dsidev, s);
+}
+
+static void dsi2_dump_regs(struct seq_file *s)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_id(1);
+
+	dsi_dump_dsidev_regs(dsidev, s);
+}
+
+enum dsi_cio_power_state {
+	DSI_COMPLEXIO_POWER_OFF		= 0x0,
+	DSI_COMPLEXIO_POWER_ON		= 0x1,
+	DSI_COMPLEXIO_POWER_ULPS	= 0x2,
+};
+
+static int dsi_cio_power(struct platform_device *dsidev,
+		enum dsi_cio_power_state state)
+{
+	int t = 0;
+
+	/* PWR_CMD */
+	REG_FLD_MOD(dsidev, DSI_COMPLEXIO_CFG1, state, 28, 27);
+
+	/* PWR_STATUS */
+	while (FLD_GET(dsi_read_reg(dsidev, DSI_COMPLEXIO_CFG1),
+			26, 25) != state) {
+		if (++t > 1000) {
+			DSSERR("failed to set complexio power state to "
+					"%d\n", state);
+			return -ENODEV;
+		}
+		udelay(1);
+	}
+
+	return 0;
+}
+
+static unsigned dsi_get_line_buf_size(struct platform_device *dsidev)
+{
+	int val;
+
+	/* line buffer on OMAP3 is 1024 x 24bits */
+	/* XXX: for some reason using full buffer size causes
+	 * considerable TX slowdown with update sizes that fill the
+	 * whole buffer */
+	if (!dss_has_feature(FEAT_DSI_GNQ))
+		return 1023 * 3;
+
+	val = REG_GET(dsidev, DSI_GNQ, 14, 12); /* VP1_LINE_BUFFER_SIZE */
+
+	switch (val) {
+	case 1:
+		return 512 * 3;		/* 512x24 bits */
+	case 2:
+		return 682 * 3;		/* 682x24 bits */
+	case 3:
+		return 853 * 3;		/* 853x24 bits */
+	case 4:
+		return 1024 * 3;	/* 1024x24 bits */
+	case 5:
+		return 1194 * 3;	/* 1194x24 bits */
+	case 6:
+		return 1365 * 3;	/* 1365x24 bits */
+	case 7:
+		return 1920 * 3;	/* 1920x24 bits */
+	default:
+		BUG();
+		return 0;
+	}
+}
+
+static int dsi_set_lane_config(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	static const u8 offsets[] = { 0, 4, 8, 12, 16 };
+	static const enum dsi_lane_function functions[] = {
+		DSI_LANE_CLK,
+		DSI_LANE_DATA1,
+		DSI_LANE_DATA2,
+		DSI_LANE_DATA3,
+		DSI_LANE_DATA4,
+	};
+	u32 r;
+	int i;
+
+	r = dsi_read_reg(dsidev, DSI_COMPLEXIO_CFG1);
+
+	for (i = 0; i < dsi->num_lanes_used; ++i) {
+		unsigned offset = offsets[i];
+		unsigned polarity, lane_number;
+		unsigned t;
+
+		for (t = 0; t < dsi->num_lanes_supported; ++t)
+			if (dsi->lanes[t].function == functions[i])
+				break;
+
+		if (t == dsi->num_lanes_supported)
+			return -EINVAL;
+
+		lane_number = t;
+		polarity = dsi->lanes[t].polarity;
+
+		r = FLD_MOD(r, lane_number + 1, offset + 2, offset);
+		r = FLD_MOD(r, polarity, offset + 3, offset + 3);
+	}
+
+	/* clear the unused lanes */
+	for (; i < dsi->num_lanes_supported; ++i) {
+		unsigned offset = offsets[i];
+
+		r = FLD_MOD(r, 0, offset + 2, offset);
+		r = FLD_MOD(r, 0, offset + 3, offset + 3);
+	}
+
+	dsi_write_reg(dsidev, DSI_COMPLEXIO_CFG1, r);
+
+	return 0;
+}
+
+static inline unsigned ns2ddr(struct platform_device *dsidev, unsigned ns)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	/* convert time in ns to ddr ticks, rounding up */
+	unsigned long ddr_clk = dsi->pll.cinfo.clkdco / 4;
+	return (ns * (ddr_clk / 1000 / 1000) + 999) / 1000;
+}
+
+static inline unsigned ddr2ns(struct platform_device *dsidev, unsigned ddr)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	unsigned long ddr_clk = dsi->pll.cinfo.clkdco / 4;
+	return ddr * 1000 * 1000 / (ddr_clk / 1000);
+}
+
+static void dsi_cio_timings(struct platform_device *dsidev)
+{
+	u32 r;
+	u32 ths_prepare, ths_prepare_ths_zero, ths_trail, ths_exit;
+	u32 tlpx_half, tclk_trail, tclk_zero;
+	u32 tclk_prepare;
+
+	/* calculate timings */
+
+	/* 1 * DDR_CLK = 2 * UI */
+
+	/* min 40ns + 4*UI	max 85ns + 6*UI */
+	ths_prepare = ns2ddr(dsidev, 70) + 2;
+
+	/* min 145ns + 10*UI */
+	ths_prepare_ths_zero = ns2ddr(dsidev, 175) + 2;
+
+	/* min max(8*UI, 60ns+4*UI) */
+	ths_trail = ns2ddr(dsidev, 60) + 5;
+
+	/* min 100ns */
+	ths_exit = ns2ddr(dsidev, 145);
+
+	/* tlpx min 50n */
+	tlpx_half = ns2ddr(dsidev, 25);
+
+	/* min 60ns */
+	tclk_trail = ns2ddr(dsidev, 60) + 2;
+
+	/* min 38ns, max 95ns */
+	tclk_prepare = ns2ddr(dsidev, 65);
+
+	/* min tclk-prepare + tclk-zero = 300ns */
+	tclk_zero = ns2ddr(dsidev, 260);
+
+	DSSDBG("ths_prepare %u (%uns), ths_prepare_ths_zero %u (%uns)\n",
+		ths_prepare, ddr2ns(dsidev, ths_prepare),
+		ths_prepare_ths_zero, ddr2ns(dsidev, ths_prepare_ths_zero));
+	DSSDBG("ths_trail %u (%uns), ths_exit %u (%uns)\n",
+			ths_trail, ddr2ns(dsidev, ths_trail),
+			ths_exit, ddr2ns(dsidev, ths_exit));
+
+	DSSDBG("tlpx_half %u (%uns), tclk_trail %u (%uns), "
+			"tclk_zero %u (%uns)\n",
+			tlpx_half, ddr2ns(dsidev, tlpx_half),
+			tclk_trail, ddr2ns(dsidev, tclk_trail),
+			tclk_zero, ddr2ns(dsidev, tclk_zero));
+	DSSDBG("tclk_prepare %u (%uns)\n",
+			tclk_prepare, ddr2ns(dsidev, tclk_prepare));
+
+	/* program timings */
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG0);
+	r = FLD_MOD(r, ths_prepare, 31, 24);
+	r = FLD_MOD(r, ths_prepare_ths_zero, 23, 16);
+	r = FLD_MOD(r, ths_trail, 15, 8);
+	r = FLD_MOD(r, ths_exit, 7, 0);
+	dsi_write_reg(dsidev, DSI_DSIPHY_CFG0, r);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG1);
+	r = FLD_MOD(r, tlpx_half, 20, 16);
+	r = FLD_MOD(r, tclk_trail, 15, 8);
+	r = FLD_MOD(r, tclk_zero, 7, 0);
+
+	if (dss_has_feature(FEAT_DSI_PHY_DCC)) {
+		r = FLD_MOD(r, 0, 21, 21);	/* DCCEN = disable */
+		r = FLD_MOD(r, 1, 22, 22);	/* CLKINP_DIVBY2EN = enable */
+		r = FLD_MOD(r, 1, 23, 23);	/* CLKINP_SEL = enable */
+	}
+
+	dsi_write_reg(dsidev, DSI_DSIPHY_CFG1, r);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG2);
+	r = FLD_MOD(r, tclk_prepare, 7, 0);
+	dsi_write_reg(dsidev, DSI_DSIPHY_CFG2, r);
+}
+
+/* lane masks have lane 0 at lsb. mask_p for positive lines, n for negative */
+static void dsi_cio_enable_lane_override(struct platform_device *dsidev,
+		unsigned mask_p, unsigned mask_n)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int i;
+	u32 l;
+	u8 lptxscp_start = dsi->num_lanes_supported == 3 ? 22 : 26;
+
+	l = 0;
+
+	for (i = 0; i < dsi->num_lanes_supported; ++i) {
+		unsigned p = dsi->lanes[i].polarity;
+
+		if (mask_p & (1 << i))
+			l |= 1 << (i * 2 + (p ? 0 : 1));
+
+		if (mask_n & (1 << i))
+			l |= 1 << (i * 2 + (p ? 1 : 0));
+	}
+
+	/*
+	 * Bits in REGLPTXSCPDAT4TO0DXDY:
+	 * 17: DY0 18: DX0
+	 * 19: DY1 20: DX1
+	 * 21: DY2 22: DX2
+	 * 23: DY3 24: DX3
+	 * 25: DY4 26: DX4
+	 */
+
+	/* Set the lane override configuration */
+
+	/* REGLPTXSCPDAT4TO0DXDY */
+	REG_FLD_MOD(dsidev, DSI_DSIPHY_CFG10, l, lptxscp_start, 17);
+
+	/* Enable lane override */
+
+	/* ENLPTXSCPDAT */
+	REG_FLD_MOD(dsidev, DSI_DSIPHY_CFG10, 1, 27, 27);
+}
+
+static void dsi_cio_disable_lane_override(struct platform_device *dsidev)
+{
+	/* Disable lane override */
+	REG_FLD_MOD(dsidev, DSI_DSIPHY_CFG10, 0, 27, 27); /* ENLPTXSCPDAT */
+	/* Reset the lane override configuration */
+	/* REGLPTXSCPDAT4TO0DXDY */
+	REG_FLD_MOD(dsidev, DSI_DSIPHY_CFG10, 0, 22, 17);
+}
+
+static int dsi_cio_wait_tx_clk_esc_reset(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int t, i;
+	bool in_use[DSI_MAX_NR_LANES];
+	static const u8 offsets_old[] = { 28, 27, 26 };
+	static const u8 offsets_new[] = { 24, 25, 26, 27, 28 };
+	const u8 *offsets;
+
+	if (dss_has_feature(FEAT_DSI_REVERSE_TXCLKESC))
+		offsets = offsets_old;
+	else
+		offsets = offsets_new;
+
+	for (i = 0; i < dsi->num_lanes_supported; ++i)
+		in_use[i] = dsi->lanes[i].function != DSI_LANE_UNUSED;
+
+	t = 100000;
+	while (true) {
+		u32 l;
+		int ok;
+
+		l = dsi_read_reg(dsidev, DSI_DSIPHY_CFG5);
+
+		ok = 0;
+		for (i = 0; i < dsi->num_lanes_supported; ++i) {
+			if (!in_use[i] || (l & (1 << offsets[i])))
+				ok++;
+		}
+
+		if (ok == dsi->num_lanes_supported)
+			break;
+
+		if (--t == 0) {
+			for (i = 0; i < dsi->num_lanes_supported; ++i) {
+				if (!in_use[i] || (l & (1 << offsets[i])))
+					continue;
+
+				DSSERR("CIO TXCLKESC%d domain not coming " \
+						"out of reset\n", i);
+			}
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+/* return bitmask of enabled lanes, lane0 being the lsb */
+static unsigned dsi_get_lane_mask(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned mask = 0;
+	int i;
+
+	for (i = 0; i < dsi->num_lanes_supported; ++i) {
+		if (dsi->lanes[i].function != DSI_LANE_UNUSED)
+			mask |= 1 << i;
+	}
+
+	return mask;
+}
+
+static int dsi_cio_init(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r;
+	u32 l;
+
+	DSSDBG("DSI CIO init starts");
+
+	r = dss_dsi_enable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+	if (r)
+		return r;
+
+	dsi_enable_scp_clk(dsidev);
+
+	/* A dummy read using the SCP interface to any DSIPHY register is
+	 * required after DSIPHY reset to complete the reset of the DSI complex
+	 * I/O. */
+	dsi_read_reg(dsidev, DSI_DSIPHY_CFG5);
+
+	if (wait_for_bit_change(dsidev, DSI_DSIPHY_CFG5, 30, 1) != 1) {
+		DSSERR("CIO SCP Clock domain not coming out of reset.\n");
+		r = -EIO;
+		goto err_scp_clk_dom;
+	}
+
+	r = dsi_set_lane_config(dsidev);
+	if (r)
+		goto err_scp_clk_dom;
+
+	/* set TX STOP MODE timer to maximum for this operation */
+	l = dsi_read_reg(dsidev, DSI_TIMING1);
+	l = FLD_MOD(l, 1, 15, 15);	/* FORCE_TX_STOP_MODE_IO */
+	l = FLD_MOD(l, 1, 14, 14);	/* STOP_STATE_X16_IO */
+	l = FLD_MOD(l, 1, 13, 13);	/* STOP_STATE_X4_IO */
+	l = FLD_MOD(l, 0x1fff, 12, 0);	/* STOP_STATE_COUNTER_IO */
+	dsi_write_reg(dsidev, DSI_TIMING1, l);
+
+	if (dsi->ulps_enabled) {
+		unsigned mask_p;
+		int i;
+
+		DSSDBG("manual ulps exit\n");
+
+		/* ULPS is exited by Mark-1 state for 1ms, followed by
+		 * stop state. DSS HW cannot do this via the normal
+		 * ULPS exit sequence, as after reset the DSS HW thinks
+		 * that we are not in ULPS mode, and refuses to send the
+		 * sequence. So we need to send the ULPS exit sequence
+		 * manually by setting positive lines high and negative lines
+		 * low for 1ms.
+		 */
+
+		mask_p = 0;
+
+		for (i = 0; i < dsi->num_lanes_supported; ++i) {
+			if (dsi->lanes[i].function == DSI_LANE_UNUSED)
+				continue;
+			mask_p |= 1 << i;
+		}
+
+		dsi_cio_enable_lane_override(dsidev, mask_p, 0);
+	}
+
+	r = dsi_cio_power(dsidev, DSI_COMPLEXIO_POWER_ON);
+	if (r)
+		goto err_cio_pwr;
+
+	if (wait_for_bit_change(dsidev, DSI_COMPLEXIO_CFG1, 29, 1) != 1) {
+		DSSERR("CIO PWR clock domain not coming out of reset.\n");
+		r = -ENODEV;
+		goto err_cio_pwr_dom;
+	}
+
+	dsi_if_enable(dsidev, true);
+	dsi_if_enable(dsidev, false);
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 1, 20, 20); /* LP_CLK_ENABLE */
+
+	r = dsi_cio_wait_tx_clk_esc_reset(dsidev);
+	if (r)
+		goto err_tx_clk_esc_rst;
+
+	if (dsi->ulps_enabled) {
+		/* Keep Mark-1 state for 1ms (as per DSI spec) */
+		ktime_t wait = ns_to_ktime(1000 * 1000);
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_hrtimeout(&wait, HRTIMER_MODE_REL);
+
+		/* Disable the override. The lanes should be set to Mark-11
+		 * state by the HW */
+		dsi_cio_disable_lane_override(dsidev);
+	}
+
+	/* FORCE_TX_STOP_MODE_IO */
+	REG_FLD_MOD(dsidev, DSI_TIMING1, 0, 15, 15);
+
+	dsi_cio_timings(dsidev);
+
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		/* DDR_CLK_ALWAYS_ON */
+		REG_FLD_MOD(dsidev, DSI_CLK_CTRL,
+			dsi->vm_timings.ddr_clk_always_on, 13, 13);
+	}
+
+	dsi->ulps_enabled = false;
+
+	DSSDBG("CIO init done\n");
+
+	return 0;
+
+err_tx_clk_esc_rst:
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 0, 20, 20); /* LP_CLK_ENABLE */
+err_cio_pwr_dom:
+	dsi_cio_power(dsidev, DSI_COMPLEXIO_POWER_OFF);
+err_cio_pwr:
+	if (dsi->ulps_enabled)
+		dsi_cio_disable_lane_override(dsidev);
+err_scp_clk_dom:
+	dsi_disable_scp_clk(dsidev);
+	dss_dsi_disable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+	return r;
+}
+
+static void dsi_cio_uninit(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	/* DDR_CLK_ALWAYS_ON */
+	REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 0, 13, 13);
+
+	dsi_cio_power(dsidev, DSI_COMPLEXIO_POWER_OFF);
+	dsi_disable_scp_clk(dsidev);
+	dss_dsi_disable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+}
+
+static void dsi_config_tx_fifo(struct platform_device *dsidev,
+		enum fifo_size size1, enum fifo_size size2,
+		enum fifo_size size3, enum fifo_size size4)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 r = 0;
+	int add = 0;
+	int i;
+
+	dsi->vc[0].tx_fifo_size = size1;
+	dsi->vc[1].tx_fifo_size = size2;
+	dsi->vc[2].tx_fifo_size = size3;
+	dsi->vc[3].tx_fifo_size = size4;
+
+	for (i = 0; i < 4; i++) {
+		u8 v;
+		int size = dsi->vc[i].tx_fifo_size;
+
+		if (add + size > 4) {
+			DSSERR("Illegal FIFO configuration\n");
+			BUG();
+			return;
+		}
+
+		v = FLD_VAL(add, 2, 0) | FLD_VAL(size, 7, 4);
+		r |= v << (8 * i);
+		/*DSSDBG("TX FIFO vc %d: size %d, add %d\n", i, size, add); */
+		add += size;
+	}
+
+	dsi_write_reg(dsidev, DSI_TX_FIFO_VC_SIZE, r);
+}
+
+static void dsi_config_rx_fifo(struct platform_device *dsidev,
+		enum fifo_size size1, enum fifo_size size2,
+		enum fifo_size size3, enum fifo_size size4)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 r = 0;
+	int add = 0;
+	int i;
+
+	dsi->vc[0].rx_fifo_size = size1;
+	dsi->vc[1].rx_fifo_size = size2;
+	dsi->vc[2].rx_fifo_size = size3;
+	dsi->vc[3].rx_fifo_size = size4;
+
+	for (i = 0; i < 4; i++) {
+		u8 v;
+		int size = dsi->vc[i].rx_fifo_size;
+
+		if (add + size > 4) {
+			DSSERR("Illegal FIFO configuration\n");
+			BUG();
+			return;
+		}
+
+		v = FLD_VAL(add, 2, 0) | FLD_VAL(size, 7, 4);
+		r |= v << (8 * i);
+		/*DSSDBG("RX FIFO vc %d: size %d, add %d\n", i, size, add); */
+		add += size;
+	}
+
+	dsi_write_reg(dsidev, DSI_RX_FIFO_VC_SIZE, r);
+}
+
+static int dsi_force_tx_stop_mode_io(struct platform_device *dsidev)
+{
+	u32 r;
+
+	r = dsi_read_reg(dsidev, DSI_TIMING1);
+	r = FLD_MOD(r, 1, 15, 15);	/* FORCE_TX_STOP_MODE_IO */
+	dsi_write_reg(dsidev, DSI_TIMING1, r);
+
+	if (wait_for_bit_change(dsidev, DSI_TIMING1, 15, 0) != 0) {
+		DSSERR("TX_STOP bit not going down\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static bool dsi_vc_is_enabled(struct platform_device *dsidev, int channel)
+{
+	return REG_GET(dsidev, DSI_VC_CTRL(channel), 0, 0);
+}
+
+static void dsi_packet_sent_handler_vp(void *data, u32 mask)
+{
+	struct dsi_packet_sent_handler_data *vp_data =
+		(struct dsi_packet_sent_handler_data *) data;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(vp_data->dsidev);
+	const int channel = dsi->update_channel;
+	u8 bit = dsi->te_enabled ? 30 : 31;
+
+	if (REG_GET(vp_data->dsidev, DSI_VC_TE(channel), bit, bit) == 0)
+		complete(vp_data->completion);
+}
+
+static int dsi_sync_vc_vp(struct platform_device *dsidev, int channel)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	DECLARE_COMPLETION_ONSTACK(completion);
+	struct dsi_packet_sent_handler_data vp_data = {
+		.dsidev = dsidev,
+		.completion = &completion
+	};
+	int r = 0;
+	u8 bit;
+
+	bit = dsi->te_enabled ? 30 : 31;
+
+	r = dsi_register_isr_vc(dsidev, channel, dsi_packet_sent_handler_vp,
+		&vp_data, DSI_VC_IRQ_PACKET_SENT);
+	if (r)
+		goto err0;
+
+	/* Wait for completion only if TE_EN/TE_START is still set */
+	if (REG_GET(dsidev, DSI_VC_TE(channel), bit, bit)) {
+		if (wait_for_completion_timeout(&completion,
+				msecs_to_jiffies(10)) == 0) {
+			DSSERR("Failed to complete previous frame transfer\n");
+			r = -EIO;
+			goto err1;
+		}
+	}
+
+	dsi_unregister_isr_vc(dsidev, channel, dsi_packet_sent_handler_vp,
+		&vp_data, DSI_VC_IRQ_PACKET_SENT);
+
+	return 0;
+err1:
+	dsi_unregister_isr_vc(dsidev, channel, dsi_packet_sent_handler_vp,
+		&vp_data, DSI_VC_IRQ_PACKET_SENT);
+err0:
+	return r;
+}
+
+static void dsi_packet_sent_handler_l4(void *data, u32 mask)
+{
+	struct dsi_packet_sent_handler_data *l4_data =
+		(struct dsi_packet_sent_handler_data *) data;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(l4_data->dsidev);
+	const int channel = dsi->update_channel;
+
+	if (REG_GET(l4_data->dsidev, DSI_VC_CTRL(channel), 5, 5) == 0)
+		complete(l4_data->completion);
+}
+
+static int dsi_sync_vc_l4(struct platform_device *dsidev, int channel)
+{
+	DECLARE_COMPLETION_ONSTACK(completion);
+	struct dsi_packet_sent_handler_data l4_data = {
+		.dsidev = dsidev,
+		.completion = &completion
+	};
+	int r = 0;
+
+	r = dsi_register_isr_vc(dsidev, channel, dsi_packet_sent_handler_l4,
+		&l4_data, DSI_VC_IRQ_PACKET_SENT);
+	if (r)
+		goto err0;
+
+	/* Wait for completion only if TX_FIFO_NOT_EMPTY is still set */
+	if (REG_GET(dsidev, DSI_VC_CTRL(channel), 5, 5)) {
+		if (wait_for_completion_timeout(&completion,
+				msecs_to_jiffies(10)) == 0) {
+			DSSERR("Failed to complete previous l4 transfer\n");
+			r = -EIO;
+			goto err1;
+		}
+	}
+
+	dsi_unregister_isr_vc(dsidev, channel, dsi_packet_sent_handler_l4,
+		&l4_data, DSI_VC_IRQ_PACKET_SENT);
+
+	return 0;
+err1:
+	dsi_unregister_isr_vc(dsidev, channel, dsi_packet_sent_handler_l4,
+		&l4_data, DSI_VC_IRQ_PACKET_SENT);
+err0:
+	return r;
+}
+
+static int dsi_sync_vc(struct platform_device *dsidev, int channel)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	WARN_ON(in_interrupt());
+
+	if (!dsi_vc_is_enabled(dsidev, channel))
+		return 0;
+
+	switch (dsi->vc[channel].source) {
+	case DSI_VC_SOURCE_VP:
+		return dsi_sync_vc_vp(dsidev, channel);
+	case DSI_VC_SOURCE_L4:
+		return dsi_sync_vc_l4(dsidev, channel);
+	default:
+		BUG();
+		return -EINVAL;
+	}
+}
+
+static int dsi_vc_enable(struct platform_device *dsidev, int channel,
+		bool enable)
+{
+	DSSDBG("dsi_vc_enable channel %d, enable %d\n",
+			channel, enable);
+
+	enable = enable ? 1 : 0;
+
+	REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), enable, 0, 0);
+
+	if (wait_for_bit_change(dsidev, DSI_VC_CTRL(channel),
+		0, enable) != enable) {
+			DSSERR("Failed to set dsi_vc_enable to %d\n", enable);
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static void dsi_vc_initial_config(struct platform_device *dsidev, int channel)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 r;
+
+	DSSDBG("Initial config of virtual channel %d", channel);
+
+	r = dsi_read_reg(dsidev, DSI_VC_CTRL(channel));
+
+	if (FLD_GET(r, 15, 15)) /* VC_BUSY */
+		DSSERR("VC(%d) busy when trying to configure it!\n",
+				channel);
+
+	r = FLD_MOD(r, 0, 1, 1); /* SOURCE, 0 = L4 */
+	r = FLD_MOD(r, 0, 2, 2); /* BTA_SHORT_EN  */
+	r = FLD_MOD(r, 0, 3, 3); /* BTA_LONG_EN */
+	r = FLD_MOD(r, 0, 4, 4); /* MODE, 0 = command */
+	r = FLD_MOD(r, 1, 7, 7); /* CS_TX_EN */
+	r = FLD_MOD(r, 1, 8, 8); /* ECC_TX_EN */
+	r = FLD_MOD(r, 0, 9, 9); /* MODE_SPEED, high speed on/off */
+	if (dss_has_feature(FEAT_DSI_VC_OCP_WIDTH))
+		r = FLD_MOD(r, 3, 11, 10);	/* OCP_WIDTH = 32 bit */
+
+	r = FLD_MOD(r, 4, 29, 27); /* DMA_RX_REQ_NB = no dma */
+	r = FLD_MOD(r, 4, 23, 21); /* DMA_TX_REQ_NB = no dma */
+
+	dsi_write_reg(dsidev, DSI_VC_CTRL(channel), r);
+
+	dsi->vc[channel].source = DSI_VC_SOURCE_L4;
+}
+
+static int dsi_vc_config_source(struct platform_device *dsidev, int channel,
+		enum dsi_vc_source source)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (dsi->vc[channel].source == source)
+		return 0;
+
+	DSSDBG("Source config of virtual channel %d", channel);
+
+	dsi_sync_vc(dsidev, channel);
+
+	dsi_vc_enable(dsidev, channel, 0);
+
+	/* VC_BUSY */
+	if (wait_for_bit_change(dsidev, DSI_VC_CTRL(channel), 15, 0) != 0) {
+		DSSERR("vc(%d) busy when trying to config for VP\n", channel);
+		return -EIO;
+	}
+
+	/* SOURCE, 0 = L4, 1 = video port */
+	REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), source, 1, 1);
+
+	/* DCS_CMD_ENABLE */
+	if (dss_has_feature(FEAT_DSI_DCS_CMD_CONFIG_VC)) {
+		bool enable = source == DSI_VC_SOURCE_VP;
+		REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), enable, 30, 30);
+	}
+
+	dsi_vc_enable(dsidev, channel, 1);
+
+	dsi->vc[channel].source = source;
+
+	return 0;
+}
+
+static void dsi_vc_enable_hs(struct omap_dss_device *dssdev, int channel,
+		bool enable)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	DSSDBG("dsi_vc_enable_hs(%d, %d)\n", channel, enable);
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	dsi_vc_enable(dsidev, channel, 0);
+	dsi_if_enable(dsidev, 0);
+
+	REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), enable, 9, 9);
+
+	dsi_vc_enable(dsidev, channel, 1);
+	dsi_if_enable(dsidev, 1);
+
+	dsi_force_tx_stop_mode_io(dsidev);
+
+	/* start the DDR clock by sending a NULL packet */
+	if (dsi->vm_timings.ddr_clk_always_on && enable)
+		dsi_vc_send_null(dssdev, channel);
+}
+
+static void dsi_vc_flush_long_data(struct platform_device *dsidev, int channel)
+{
+	while (REG_GET(dsidev, DSI_VC_CTRL(channel), 20, 20)) {
+		u32 val;
+		val = dsi_read_reg(dsidev, DSI_VC_SHORT_PACKET_HEADER(channel));
+		DSSDBG("\t\tb1 %#02x b2 %#02x b3 %#02x b4 %#02x\n",
+				(val >> 0) & 0xff,
+				(val >> 8) & 0xff,
+				(val >> 16) & 0xff,
+				(val >> 24) & 0xff);
+	}
+}
+
+static void dsi_show_rx_ack_with_err(u16 err)
+{
+	DSSERR("\tACK with ERROR (%#x):\n", err);
+	if (err & (1 << 0))
+		DSSERR("\t\tSoT Error\n");
+	if (err & (1 << 1))
+		DSSERR("\t\tSoT Sync Error\n");
+	if (err & (1 << 2))
+		DSSERR("\t\tEoT Sync Error\n");
+	if (err & (1 << 3))
+		DSSERR("\t\tEscape Mode Entry Command Error\n");
+	if (err & (1 << 4))
+		DSSERR("\t\tLP Transmit Sync Error\n");
+	if (err & (1 << 5))
+		DSSERR("\t\tHS Receive Timeout Error\n");
+	if (err & (1 << 6))
+		DSSERR("\t\tFalse Control Error\n");
+	if (err & (1 << 7))
+		DSSERR("\t\t(reserved7)\n");
+	if (err & (1 << 8))
+		DSSERR("\t\tECC Error, single-bit (corrected)\n");
+	if (err & (1 << 9))
+		DSSERR("\t\tECC Error, multi-bit (not corrected)\n");
+	if (err & (1 << 10))
+		DSSERR("\t\tChecksum Error\n");
+	if (err & (1 << 11))
+		DSSERR("\t\tData type not recognized\n");
+	if (err & (1 << 12))
+		DSSERR("\t\tInvalid VC ID\n");
+	if (err & (1 << 13))
+		DSSERR("\t\tInvalid Transmission Length\n");
+	if (err & (1 << 14))
+		DSSERR("\t\t(reserved14)\n");
+	if (err & (1 << 15))
+		DSSERR("\t\tDSI Protocol Violation\n");
+}
+
+static u16 dsi_vc_flush_receive_data(struct platform_device *dsidev,
+		int channel)
+{
+	/* RX_FIFO_NOT_EMPTY */
+	while (REG_GET(dsidev, DSI_VC_CTRL(channel), 20, 20)) {
+		u32 val;
+		u8 dt;
+		val = dsi_read_reg(dsidev, DSI_VC_SHORT_PACKET_HEADER(channel));
+		DSSERR("\trawval %#08x\n", val);
+		dt = FLD_GET(val, 5, 0);
+		if (dt == MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT) {
+			u16 err = FLD_GET(val, 23, 8);
+			dsi_show_rx_ack_with_err(err);
+		} else if (dt == MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE) {
+			DSSERR("\tDCS short response, 1 byte: %#x\n",
+					FLD_GET(val, 23, 8));
+		} else if (dt == MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE) {
+			DSSERR("\tDCS short response, 2 byte: %#x\n",
+					FLD_GET(val, 23, 8));
+		} else if (dt == MIPI_DSI_RX_DCS_LONG_READ_RESPONSE) {
+			DSSERR("\tDCS long response, len %d\n",
+					FLD_GET(val, 23, 8));
+			dsi_vc_flush_long_data(dsidev, channel);
+		} else {
+			DSSERR("\tunknown datatype 0x%02x\n", dt);
+		}
+	}
+	return 0;
+}
+
+static int dsi_vc_send_bta(struct platform_device *dsidev, int channel)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (dsi->debug_write || dsi->debug_read)
+		DSSDBG("dsi_vc_send_bta %d\n", channel);
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	/* RX_FIFO_NOT_EMPTY */
+	if (REG_GET(dsidev, DSI_VC_CTRL(channel), 20, 20)) {
+		DSSERR("rx fifo not empty when sending BTA, dumping data:\n");
+		dsi_vc_flush_receive_data(dsidev, channel);
+	}
+
+	REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), 1, 6, 6); /* BTA_EN */
+
+	/* flush posted write */
+	dsi_read_reg(dsidev, DSI_VC_CTRL(channel));
+
+	return 0;
+}
+
+static int dsi_vc_send_bta_sync(struct omap_dss_device *dssdev, int channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	DECLARE_COMPLETION_ONSTACK(completion);
+	int r = 0;
+	u32 err;
+
+	r = dsi_register_isr_vc(dsidev, channel, dsi_completion_handler,
+			&completion, DSI_VC_IRQ_BTA);
+	if (r)
+		goto err0;
+
+	r = dsi_register_isr(dsidev, dsi_completion_handler, &completion,
+			DSI_IRQ_ERROR_MASK);
+	if (r)
+		goto err1;
+
+	r = dsi_vc_send_bta(dsidev, channel);
+	if (r)
+		goto err2;
+
+	if (wait_for_completion_timeout(&completion,
+				msecs_to_jiffies(500)) == 0) {
+		DSSERR("Failed to receive BTA\n");
+		r = -EIO;
+		goto err2;
+	}
+
+	err = dsi_get_errors(dsidev);
+	if (err) {
+		DSSERR("Error while sending BTA: %x\n", err);
+		r = -EIO;
+		goto err2;
+	}
+err2:
+	dsi_unregister_isr(dsidev, dsi_completion_handler, &completion,
+			DSI_IRQ_ERROR_MASK);
+err1:
+	dsi_unregister_isr_vc(dsidev, channel, dsi_completion_handler,
+			&completion, DSI_VC_IRQ_BTA);
+err0:
+	return r;
+}
+
+static inline void dsi_vc_write_long_header(struct platform_device *dsidev,
+		int channel, u8 data_type, u16 len, u8 ecc)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 val;
+	u8 data_id;
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	data_id = data_type | dsi->vc[channel].vc_id << 6;
+
+	val = FLD_VAL(data_id, 7, 0) | FLD_VAL(len, 23, 8) |
+		FLD_VAL(ecc, 31, 24);
+
+	dsi_write_reg(dsidev, DSI_VC_LONG_PACKET_HEADER(channel), val);
+}
+
+static inline void dsi_vc_write_long_payload(struct platform_device *dsidev,
+		int channel, u8 b1, u8 b2, u8 b3, u8 b4)
+{
+	u32 val;
+
+	val = b4 << 24 | b3 << 16 | b2 << 8  | b1 << 0;
+
+/*	DSSDBG("\twriting %02x, %02x, %02x, %02x (%#010x)\n",
+			b1, b2, b3, b4, val); */
+
+	dsi_write_reg(dsidev, DSI_VC_LONG_PACKET_PAYLOAD(channel), val);
+}
+
+static int dsi_vc_send_long(struct platform_device *dsidev, int channel,
+		u8 data_type, u8 *data, u16 len, u8 ecc)
+{
+	/*u32 val; */
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int i;
+	u8 *p;
+	int r = 0;
+	u8 b1, b2, b3, b4;
+
+	if (dsi->debug_write)
+		DSSDBG("dsi_vc_send_long, %d bytes\n", len);
+
+	/* len + header */
+	if (dsi->vc[channel].tx_fifo_size * 32 * 4 < len + 4) {
+		DSSERR("unable to send long packet: packet too long.\n");
+		return -EINVAL;
+	}
+
+	dsi_vc_config_source(dsidev, channel, DSI_VC_SOURCE_L4);
+
+	dsi_vc_write_long_header(dsidev, channel, data_type, len, ecc);
+
+	p = data;
+	for (i = 0; i < len >> 2; i++) {
+		if (dsi->debug_write)
+			DSSDBG("\tsending full packet %d\n", i);
+
+		b1 = *p++;
+		b2 = *p++;
+		b3 = *p++;
+		b4 = *p++;
+
+		dsi_vc_write_long_payload(dsidev, channel, b1, b2, b3, b4);
+	}
+
+	i = len % 4;
+	if (i) {
+		b1 = 0; b2 = 0; b3 = 0;
+
+		if (dsi->debug_write)
+			DSSDBG("\tsending remainder bytes %d\n", i);
+
+		switch (i) {
+		case 3:
+			b1 = *p++;
+			b2 = *p++;
+			b3 = *p++;
+			break;
+		case 2:
+			b1 = *p++;
+			b2 = *p++;
+			break;
+		case 1:
+			b1 = *p++;
+			break;
+		}
+
+		dsi_vc_write_long_payload(dsidev, channel, b1, b2, b3, 0);
+	}
+
+	return r;
+}
+
+static int dsi_vc_send_short(struct platform_device *dsidev, int channel,
+		u8 data_type, u16 data, u8 ecc)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 r;
+	u8 data_id;
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	if (dsi->debug_write)
+		DSSDBG("dsi_vc_send_short(ch%d, dt %#x, b1 %#x, b2 %#x)\n",
+				channel,
+				data_type, data & 0xff, (data >> 8) & 0xff);
+
+	dsi_vc_config_source(dsidev, channel, DSI_VC_SOURCE_L4);
+
+	if (FLD_GET(dsi_read_reg(dsidev, DSI_VC_CTRL(channel)), 16, 16)) {
+		DSSERR("ERROR FIFO FULL, aborting transfer\n");
+		return -EINVAL;
+	}
+
+	data_id = data_type | dsi->vc[channel].vc_id << 6;
+
+	r = (data_id << 0) | (data << 8) | (ecc << 24);
+
+	dsi_write_reg(dsidev, DSI_VC_SHORT_PACKET_HEADER(channel), r);
+
+	return 0;
+}
+
+static int dsi_vc_send_null(struct omap_dss_device *dssdev, int channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+
+	return dsi_vc_send_long(dsidev, channel, MIPI_DSI_NULL_PACKET, NULL,
+		0, 0);
+}
+
+static int dsi_vc_write_nosync_common(struct platform_device *dsidev,
+		int channel, u8 *data, int len, enum dss_dsi_content_type type)
+{
+	int r;
+
+	if (len == 0) {
+		BUG_ON(type == DSS_DSI_CONTENT_DCS);
+		r = dsi_vc_send_short(dsidev, channel,
+				MIPI_DSI_GENERIC_SHORT_WRITE_0_PARAM, 0, 0);
+	} else if (len == 1) {
+		r = dsi_vc_send_short(dsidev, channel,
+				type == DSS_DSI_CONTENT_GENERIC ?
+				MIPI_DSI_GENERIC_SHORT_WRITE_1_PARAM :
+				MIPI_DSI_DCS_SHORT_WRITE, data[0], 0);
+	} else if (len == 2) {
+		r = dsi_vc_send_short(dsidev, channel,
+				type == DSS_DSI_CONTENT_GENERIC ?
+				MIPI_DSI_GENERIC_SHORT_WRITE_2_PARAM :
+				MIPI_DSI_DCS_SHORT_WRITE_PARAM,
+				data[0] | (data[1] << 8), 0);
+	} else {
+		r = dsi_vc_send_long(dsidev, channel,
+				type == DSS_DSI_CONTENT_GENERIC ?
+				MIPI_DSI_GENERIC_LONG_WRITE :
+				MIPI_DSI_DCS_LONG_WRITE, data, len, 0);
+	}
+
+	return r;
+}
+
+static int dsi_vc_dcs_write_nosync(struct omap_dss_device *dssdev, int channel,
+		u8 *data, int len)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+
+	return dsi_vc_write_nosync_common(dsidev, channel, data, len,
+			DSS_DSI_CONTENT_DCS);
+}
+
+static int dsi_vc_generic_write_nosync(struct omap_dss_device *dssdev, int channel,
+		u8 *data, int len)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+
+	return dsi_vc_write_nosync_common(dsidev, channel, data, len,
+			DSS_DSI_CONTENT_GENERIC);
+}
+
+static int dsi_vc_write_common(struct omap_dss_device *dssdev, int channel,
+		u8 *data, int len, enum dss_dsi_content_type type)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	int r;
+
+	r = dsi_vc_write_nosync_common(dsidev, channel, data, len, type);
+	if (r)
+		goto err;
+
+	r = dsi_vc_send_bta_sync(dssdev, channel);
+	if (r)
+		goto err;
+
+	/* RX_FIFO_NOT_EMPTY */
+	if (REG_GET(dsidev, DSI_VC_CTRL(channel), 20, 20)) {
+		DSSERR("rx fifo not empty after write, dumping data:\n");
+		dsi_vc_flush_receive_data(dsidev, channel);
+		r = -EIO;
+		goto err;
+	}
+
+	return 0;
+err:
+	DSSERR("dsi_vc_write_common(ch %d, cmd 0x%02x, len %d) failed\n",
+			channel, data[0], len);
+	return r;
+}
+
+static int dsi_vc_dcs_write(struct omap_dss_device *dssdev, int channel, u8 *data,
+		int len)
+{
+	return dsi_vc_write_common(dssdev, channel, data, len,
+			DSS_DSI_CONTENT_DCS);
+}
+
+static int dsi_vc_generic_write(struct omap_dss_device *dssdev, int channel, u8 *data,
+		int len)
+{
+	return dsi_vc_write_common(dssdev, channel, data, len,
+			DSS_DSI_CONTENT_GENERIC);
+}
+
+static int dsi_vc_dcs_send_read_request(struct platform_device *dsidev,
+		int channel, u8 dcs_cmd)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r;
+
+	if (dsi->debug_read)
+		DSSDBG("dsi_vc_dcs_send_read_request(ch%d, dcs_cmd %x)\n",
+			channel, dcs_cmd);
+
+	r = dsi_vc_send_short(dsidev, channel, MIPI_DSI_DCS_READ, dcs_cmd, 0);
+	if (r) {
+		DSSERR("dsi_vc_dcs_send_read_request(ch %d, cmd 0x%02x)"
+			" failed\n", channel, dcs_cmd);
+		return r;
+	}
+
+	return 0;
+}
+
+static int dsi_vc_generic_send_read_request(struct platform_device *dsidev,
+		int channel, u8 *reqdata, int reqlen)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u16 data;
+	u8 data_type;
+	int r;
+
+	if (dsi->debug_read)
+		DSSDBG("dsi_vc_generic_send_read_request(ch %d, reqlen %d)\n",
+			channel, reqlen);
+
+	if (reqlen == 0) {
+		data_type = MIPI_DSI_GENERIC_READ_REQUEST_0_PARAM;
+		data = 0;
+	} else if (reqlen == 1) {
+		data_type = MIPI_DSI_GENERIC_READ_REQUEST_1_PARAM;
+		data = reqdata[0];
+	} else if (reqlen == 2) {
+		data_type = MIPI_DSI_GENERIC_READ_REQUEST_2_PARAM;
+		data = reqdata[0] | (reqdata[1] << 8);
+	} else {
+		BUG();
+		return -EINVAL;
+	}
+
+	r = dsi_vc_send_short(dsidev, channel, data_type, data, 0);
+	if (r) {
+		DSSERR("dsi_vc_generic_send_read_request(ch %d, reqlen %d)"
+			" failed\n", channel, reqlen);
+		return r;
+	}
+
+	return 0;
+}
+
+static int dsi_vc_read_rx_fifo(struct platform_device *dsidev, int channel,
+		u8 *buf, int buflen, enum dss_dsi_content_type type)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 val;
+	u8 dt;
+	int r;
+
+	/* RX_FIFO_NOT_EMPTY */
+	if (REG_GET(dsidev, DSI_VC_CTRL(channel), 20, 20) == 0) {
+		DSSERR("RX fifo empty when trying to read.\n");
+		r = -EIO;
+		goto err;
+	}
+
+	val = dsi_read_reg(dsidev, DSI_VC_SHORT_PACKET_HEADER(channel));
+	if (dsi->debug_read)
+		DSSDBG("\theader: %08x\n", val);
+	dt = FLD_GET(val, 5, 0);
+	if (dt == MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT) {
+		u16 err = FLD_GET(val, 23, 8);
+		dsi_show_rx_ack_with_err(err);
+		r = -EIO;
+		goto err;
+
+	} else if (dt == (type == DSS_DSI_CONTENT_GENERIC ?
+			MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE :
+			MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE)) {
+		u8 data = FLD_GET(val, 15, 8);
+		if (dsi->debug_read)
+			DSSDBG("\t%s short response, 1 byte: %02x\n",
+				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
+				"DCS", data);
+
+		if (buflen < 1) {
+			r = -EIO;
+			goto err;
+		}
+
+		buf[0] = data;
+
+		return 1;
+	} else if (dt == (type == DSS_DSI_CONTENT_GENERIC ?
+			MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_2BYTE :
+			MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE)) {
+		u16 data = FLD_GET(val, 23, 8);
+		if (dsi->debug_read)
+			DSSDBG("\t%s short response, 2 byte: %04x\n",
+				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
+				"DCS", data);
+
+		if (buflen < 2) {
+			r = -EIO;
+			goto err;
+		}
+
+		buf[0] = data & 0xff;
+		buf[1] = (data >> 8) & 0xff;
+
+		return 2;
+	} else if (dt == (type == DSS_DSI_CONTENT_GENERIC ?
+			MIPI_DSI_RX_GENERIC_LONG_READ_RESPONSE :
+			MIPI_DSI_RX_DCS_LONG_READ_RESPONSE)) {
+		int w;
+		int len = FLD_GET(val, 23, 8);
+		if (dsi->debug_read)
+			DSSDBG("\t%s long response, len %d\n",
+				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
+				"DCS", len);
+
+		if (len > buflen) {
+			r = -EIO;
+			goto err;
+		}
+
+		/* two byte checksum ends the packet, not included in len */
+		for (w = 0; w < len + 2;) {
+			int b;
+			val = dsi_read_reg(dsidev,
+				DSI_VC_SHORT_PACKET_HEADER(channel));
+			if (dsi->debug_read)
+				DSSDBG("\t\t%02x %02x %02x %02x\n",
+						(val >> 0) & 0xff,
+						(val >> 8) & 0xff,
+						(val >> 16) & 0xff,
+						(val >> 24) & 0xff);
+
+			for (b = 0; b < 4; ++b) {
+				if (w < len)
+					buf[w] = (val >> (b * 8)) & 0xff;
+				/* we discard the 2 byte checksum */
+				++w;
+			}
+		}
+
+		return len;
+	} else {
+		DSSERR("\tunknown datatype 0x%02x\n", dt);
+		r = -EIO;
+		goto err;
+	}
+
+err:
+	DSSERR("dsi_vc_read_rx_fifo(ch %d type %s) failed\n", channel,
+		type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" : "DCS");
+
+	return r;
+}
+
+static int dsi_vc_dcs_read(struct omap_dss_device *dssdev, int channel, u8 dcs_cmd,
+		u8 *buf, int buflen)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	int r;
+
+	r = dsi_vc_dcs_send_read_request(dsidev, channel, dcs_cmd);
+	if (r)
+		goto err;
+
+	r = dsi_vc_send_bta_sync(dssdev, channel);
+	if (r)
+		goto err;
+
+	r = dsi_vc_read_rx_fifo(dsidev, channel, buf, buflen,
+		DSS_DSI_CONTENT_DCS);
+	if (r < 0)
+		goto err;
+
+	if (r != buflen) {
+		r = -EIO;
+		goto err;
+	}
+
+	return 0;
+err:
+	DSSERR("dsi_vc_dcs_read(ch %d, cmd 0x%02x) failed\n", channel, dcs_cmd);
+	return r;
+}
+
+static int dsi_vc_generic_read(struct omap_dss_device *dssdev, int channel,
+		u8 *reqdata, int reqlen, u8 *buf, int buflen)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	int r;
+
+	r = dsi_vc_generic_send_read_request(dsidev, channel, reqdata, reqlen);
+	if (r)
+		return r;
+
+	r = dsi_vc_send_bta_sync(dssdev, channel);
+	if (r)
+		return r;
+
+	r = dsi_vc_read_rx_fifo(dsidev, channel, buf, buflen,
+		DSS_DSI_CONTENT_GENERIC);
+	if (r < 0)
+		return r;
+
+	if (r != buflen) {
+		r = -EIO;
+		return r;
+	}
+
+	return 0;
+}
+
+static int dsi_vc_set_max_rx_packet_size(struct omap_dss_device *dssdev, int channel,
+		u16 len)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+
+	return dsi_vc_send_short(dsidev, channel,
+			MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE, len, 0);
+}
+
+static int dsi_enter_ulps(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	DECLARE_COMPLETION_ONSTACK(completion);
+	int r, i;
+	unsigned mask;
+
+	DSSDBG("Entering ULPS");
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	WARN_ON(dsi->ulps_enabled);
+
+	if (dsi->ulps_enabled)
+		return 0;
+
+	/* DDR_CLK_ALWAYS_ON */
+	if (REG_GET(dsidev, DSI_CLK_CTRL, 13, 13)) {
+		dsi_if_enable(dsidev, 0);
+		REG_FLD_MOD(dsidev, DSI_CLK_CTRL, 0, 13, 13);
+		dsi_if_enable(dsidev, 1);
+	}
+
+	dsi_sync_vc(dsidev, 0);
+	dsi_sync_vc(dsidev, 1);
+	dsi_sync_vc(dsidev, 2);
+	dsi_sync_vc(dsidev, 3);
+
+	dsi_force_tx_stop_mode_io(dsidev);
+
+	dsi_vc_enable(dsidev, 0, false);
+	dsi_vc_enable(dsidev, 1, false);
+	dsi_vc_enable(dsidev, 2, false);
+	dsi_vc_enable(dsidev, 3, false);
+
+	if (REG_GET(dsidev, DSI_COMPLEXIO_CFG2, 16, 16)) {	/* HS_BUSY */
+		DSSERR("HS busy when enabling ULPS\n");
+		return -EIO;
+	}
+
+	if (REG_GET(dsidev, DSI_COMPLEXIO_CFG2, 17, 17)) {	/* LP_BUSY */
+		DSSERR("LP busy when enabling ULPS\n");
+		return -EIO;
+	}
+
+	r = dsi_register_isr_cio(dsidev, dsi_completion_handler, &completion,
+			DSI_CIO_IRQ_ULPSACTIVENOT_ALL0);
+	if (r)
+		return r;
+
+	mask = 0;
+
+	for (i = 0; i < dsi->num_lanes_supported; ++i) {
+		if (dsi->lanes[i].function == DSI_LANE_UNUSED)
+			continue;
+		mask |= 1 << i;
+	}
+	/* Assert TxRequestEsc for data lanes and TxUlpsClk for clk lane */
+	/* LANEx_ULPS_SIG2 */
+	REG_FLD_MOD(dsidev, DSI_COMPLEXIO_CFG2, mask, 9, 5);
+
+	/* flush posted write and wait for SCP interface to finish the write */
+	dsi_read_reg(dsidev, DSI_COMPLEXIO_CFG2);
+
+	if (wait_for_completion_timeout(&completion,
+				msecs_to_jiffies(1000)) == 0) {
+		DSSERR("ULPS enable timeout\n");
+		r = -EIO;
+		goto err;
+	}
+
+	dsi_unregister_isr_cio(dsidev, dsi_completion_handler, &completion,
+			DSI_CIO_IRQ_ULPSACTIVENOT_ALL0);
+
+	/* Reset LANEx_ULPS_SIG2 */
+	REG_FLD_MOD(dsidev, DSI_COMPLEXIO_CFG2, 0, 9, 5);
+
+	/* flush posted write and wait for SCP interface to finish the write */
+	dsi_read_reg(dsidev, DSI_COMPLEXIO_CFG2);
+
+	dsi_cio_power(dsidev, DSI_COMPLEXIO_POWER_ULPS);
+
+	dsi_if_enable(dsidev, false);
+
+	dsi->ulps_enabled = true;
+
+	return 0;
+
+err:
+	dsi_unregister_isr_cio(dsidev, dsi_completion_handler, &completion,
+			DSI_CIO_IRQ_ULPSACTIVENOT_ALL0);
+	return r;
+}
+
+static void dsi_set_lp_rx_timeout(struct platform_device *dsidev,
+		unsigned ticks, bool x4, bool x16)
+{
+	unsigned long fck;
+	unsigned long total_ticks;
+	u32 r;
+
+	BUG_ON(ticks > 0x1fff);
+
+	/* ticks in DSI_FCK */
+	fck = dsi_fclk_rate(dsidev);
+
+	r = dsi_read_reg(dsidev, DSI_TIMING2);
+	r = FLD_MOD(r, 1, 15, 15);	/* LP_RX_TO */
+	r = FLD_MOD(r, x16 ? 1 : 0, 14, 14);	/* LP_RX_TO_X16 */
+	r = FLD_MOD(r, x4 ? 1 : 0, 13, 13);	/* LP_RX_TO_X4 */
+	r = FLD_MOD(r, ticks, 12, 0);	/* LP_RX_COUNTER */
+	dsi_write_reg(dsidev, DSI_TIMING2, r);
+
+	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
+
+	DSSDBG("LP_RX_TO %lu ticks (%#x%s%s) = %lu ns\n",
+			total_ticks,
+			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+			(total_ticks * 1000) / (fck / 1000 / 1000));
+}
+
+static void dsi_set_ta_timeout(struct platform_device *dsidev, unsigned ticks,
+		bool x8, bool x16)
+{
+	unsigned long fck;
+	unsigned long total_ticks;
+	u32 r;
+
+	BUG_ON(ticks > 0x1fff);
+
+	/* ticks in DSI_FCK */
+	fck = dsi_fclk_rate(dsidev);
+
+	r = dsi_read_reg(dsidev, DSI_TIMING1);
+	r = FLD_MOD(r, 1, 31, 31);	/* TA_TO */
+	r = FLD_MOD(r, x16 ? 1 : 0, 30, 30);	/* TA_TO_X16 */
+	r = FLD_MOD(r, x8 ? 1 : 0, 29, 29);	/* TA_TO_X8 */
+	r = FLD_MOD(r, ticks, 28, 16);	/* TA_TO_COUNTER */
+	dsi_write_reg(dsidev, DSI_TIMING1, r);
+
+	total_ticks = ticks * (x16 ? 16 : 1) * (x8 ? 8 : 1);
+
+	DSSDBG("TA_TO %lu ticks (%#x%s%s) = %lu ns\n",
+			total_ticks,
+			ticks, x8 ? " x8" : "", x16 ? " x16" : "",
+			(total_ticks * 1000) / (fck / 1000 / 1000));
+}
+
+static void dsi_set_stop_state_counter(struct platform_device *dsidev,
+		unsigned ticks, bool x4, bool x16)
+{
+	unsigned long fck;
+	unsigned long total_ticks;
+	u32 r;
+
+	BUG_ON(ticks > 0x1fff);
+
+	/* ticks in DSI_FCK */
+	fck = dsi_fclk_rate(dsidev);
+
+	r = dsi_read_reg(dsidev, DSI_TIMING1);
+	r = FLD_MOD(r, 1, 15, 15);	/* FORCE_TX_STOP_MODE_IO */
+	r = FLD_MOD(r, x16 ? 1 : 0, 14, 14);	/* STOP_STATE_X16_IO */
+	r = FLD_MOD(r, x4 ? 1 : 0, 13, 13);	/* STOP_STATE_X4_IO */
+	r = FLD_MOD(r, ticks, 12, 0);	/* STOP_STATE_COUNTER_IO */
+	dsi_write_reg(dsidev, DSI_TIMING1, r);
+
+	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
+
+	DSSDBG("STOP_STATE_COUNTER %lu ticks (%#x%s%s) = %lu ns\n",
+			total_ticks,
+			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+			(total_ticks * 1000) / (fck / 1000 / 1000));
+}
+
+static void dsi_set_hs_tx_timeout(struct platform_device *dsidev,
+		unsigned ticks, bool x4, bool x16)
+{
+	unsigned long fck;
+	unsigned long total_ticks;
+	u32 r;
+
+	BUG_ON(ticks > 0x1fff);
+
+	/* ticks in TxByteClkHS */
+	fck = dsi_get_txbyteclkhs(dsidev);
+
+	r = dsi_read_reg(dsidev, DSI_TIMING2);
+	r = FLD_MOD(r, 1, 31, 31);	/* HS_TX_TO */
+	r = FLD_MOD(r, x16 ? 1 : 0, 30, 30);	/* HS_TX_TO_X16 */
+	r = FLD_MOD(r, x4 ? 1 : 0, 29, 29);	/* HS_TX_TO_X8 (4 really) */
+	r = FLD_MOD(r, ticks, 28, 16);	/* HS_TX_TO_COUNTER */
+	dsi_write_reg(dsidev, DSI_TIMING2, r);
+
+	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
+
+	DSSDBG("HS_TX_TO %lu ticks (%#x%s%s) = %lu ns\n",
+			total_ticks,
+			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+			(total_ticks * 1000) / (fck / 1000 / 1000));
+}
+
+static void dsi_config_vp_num_line_buffers(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int num_line_buffers;
+
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
+		struct omap_video_timings *timings = &dsi->timings;
+		/*
+		 * Don't use line buffers if width is greater than the video
+		 * port's line buffer size
+		 */
+		if (dsi->line_buffer_size <= timings->x_res * bpp / 8)
+			num_line_buffers = 0;
+		else
+			num_line_buffers = 2;
+	} else {
+		/* Use maximum number of line buffers in command mode */
+		num_line_buffers = 2;
+	}
+
+	/* LINE_BUFFER */
+	REG_FLD_MOD(dsidev, DSI_CTRL, num_line_buffers, 13, 12);
+}
+
+static void dsi_config_vp_sync_events(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	bool sync_end;
+	u32 r;
+
+	if (dsi->vm_timings.trans_mode == OMAP_DSS_DSI_PULSE_MODE)
+		sync_end = true;
+	else
+		sync_end = false;
+
+	r = dsi_read_reg(dsidev, DSI_CTRL);
+	r = FLD_MOD(r, 1, 9, 9);		/* VP_DE_POL */
+	r = FLD_MOD(r, 1, 10, 10);		/* VP_HSYNC_POL */
+	r = FLD_MOD(r, 1, 11, 11);		/* VP_VSYNC_POL */
+	r = FLD_MOD(r, 1, 15, 15);		/* VP_VSYNC_START */
+	r = FLD_MOD(r, sync_end, 16, 16);	/* VP_VSYNC_END */
+	r = FLD_MOD(r, 1, 17, 17);		/* VP_HSYNC_START */
+	r = FLD_MOD(r, sync_end, 18, 18);	/* VP_HSYNC_END */
+	dsi_write_reg(dsidev, DSI_CTRL, r);
+}
+
+static void dsi_config_blanking_modes(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int blanking_mode = dsi->vm_timings.blanking_mode;
+	int hfp_blanking_mode = dsi->vm_timings.hfp_blanking_mode;
+	int hbp_blanking_mode = dsi->vm_timings.hbp_blanking_mode;
+	int hsa_blanking_mode = dsi->vm_timings.hsa_blanking_mode;
+	u32 r;
+
+	/*
+	 * 0 = TX FIFO packets sent or LPS in corresponding blanking periods
+	 * 1 = Long blanking packets are sent in corresponding blanking periods
+	 */
+	r = dsi_read_reg(dsidev, DSI_CTRL);
+	r = FLD_MOD(r, blanking_mode, 20, 20);		/* BLANKING_MODE */
+	r = FLD_MOD(r, hfp_blanking_mode, 21, 21);	/* HFP_BLANKING */
+	r = FLD_MOD(r, hbp_blanking_mode, 22, 22);	/* HBP_BLANKING */
+	r = FLD_MOD(r, hsa_blanking_mode, 23, 23);	/* HSA_BLANKING */
+	dsi_write_reg(dsidev, DSI_CTRL, r);
+}
+
+/*
+ * According to section 'HS Command Mode Interleaving' in OMAP TRM, Scenario 3
+ * results in maximum transition time for data and clock lanes to enter and
+ * exit HS mode. Hence, this is the scenario where the least amount of command
+ * mode data can be interleaved. We program the minimum amount of TXBYTECLKHS
+ * clock cycles that can be used to interleave command mode data in HS so that
+ * all scenarios are satisfied.
+ */
+static int dsi_compute_interleave_hs(int blank, bool ddr_alwon, int enter_hs,
+		int exit_hs, int exiths_clk, int ddr_pre, int ddr_post)
+{
+	int transition;
+
+	/*
+	 * If DDR_CLK_ALWAYS_ON is set, we need to consider HS mode transition
+	 * time of data lanes only, if it isn't set, we need to consider HS
+	 * transition time of both data and clock lanes. HS transition time
+	 * of Scenario 3 is considered.
+	 */
+	if (ddr_alwon) {
+		transition = enter_hs + exit_hs + max(enter_hs, 2) + 1;
+	} else {
+		int trans1, trans2;
+		trans1 = ddr_pre + enter_hs + exit_hs + max(enter_hs, 2) + 1;
+		trans2 = ddr_pre + enter_hs + exiths_clk + ddr_post + ddr_pre +
+				enter_hs + 1;
+		transition = max(trans1, trans2);
+	}
+
+	return blank > transition ? blank - transition : 0;
+}
+
+/*
+ * According to section 'LP Command Mode Interleaving' in OMAP TRM, Scenario 1
+ * results in maximum transition time for data lanes to enter and exit LP mode.
+ * Hence, this is the scenario where the least amount of command mode data can
+ * be interleaved. We program the minimum amount of bytes that can be
+ * interleaved in LP so that all scenarios are satisfied.
+ */
+static int dsi_compute_interleave_lp(int blank, int enter_hs, int exit_hs,
+		int lp_clk_div, int tdsi_fclk)
+{
+	int trans_lp;	/* time required for a LP transition, in TXBYTECLKHS */
+	int tlp_avail;	/* time left for interleaving commands, in CLKIN4DDR */
+	int ttxclkesc;	/* period of LP transmit escape clock, in CLKIN4DDR */
+	int thsbyte_clk = 16;	/* Period of TXBYTECLKHS clock, in CLKIN4DDR */
+	int lp_inter;	/* cmd mode data that can be interleaved, in bytes */
+
+	/* maximum LP transition time according to Scenario 1 */
+	trans_lp = exit_hs + max(enter_hs, 2) + 1;
+
+	/* CLKIN4DDR = 16 * TXBYTECLKHS */
+	tlp_avail = thsbyte_clk * (blank - trans_lp);
+
+	ttxclkesc = tdsi_fclk * lp_clk_div;
+
+	lp_inter = ((tlp_avail - 8 * thsbyte_clk - 5 * tdsi_fclk) / ttxclkesc -
+			26) / 16;
+
+	return max(lp_inter, 0);
+}
+
+static void dsi_config_cmd_mode_interleaving(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int blanking_mode;
+	int hfp_blanking_mode, hbp_blanking_mode, hsa_blanking_mode;
+	int hsa, hfp, hbp, width_bytes, bllp, lp_clk_div;
+	int ddr_clk_pre, ddr_clk_post, enter_hs_mode_lat, exit_hs_mode_lat;
+	int tclk_trail, ths_exit, exiths_clk;
+	bool ddr_alwon;
+	struct omap_video_timings *timings = &dsi->timings;
+	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
+	int ndl = dsi->num_lanes_used - 1;
+	int dsi_fclk_hsdiv = dsi->user_dsi_cinfo.mX[HSDIV_DSI] + 1;
+	int hsa_interleave_hs = 0, hsa_interleave_lp = 0;
+	int hfp_interleave_hs = 0, hfp_interleave_lp = 0;
+	int hbp_interleave_hs = 0, hbp_interleave_lp = 0;
+	int bl_interleave_hs = 0, bl_interleave_lp = 0;
+	u32 r;
+
+	r = dsi_read_reg(dsidev, DSI_CTRL);
+	blanking_mode = FLD_GET(r, 20, 20);
+	hfp_blanking_mode = FLD_GET(r, 21, 21);
+	hbp_blanking_mode = FLD_GET(r, 22, 22);
+	hsa_blanking_mode = FLD_GET(r, 23, 23);
+
+	r = dsi_read_reg(dsidev, DSI_VM_TIMING1);
+	hbp = FLD_GET(r, 11, 0);
+	hfp = FLD_GET(r, 23, 12);
+	hsa = FLD_GET(r, 31, 24);
+
+	r = dsi_read_reg(dsidev, DSI_CLK_TIMING);
+	ddr_clk_post = FLD_GET(r, 7, 0);
+	ddr_clk_pre = FLD_GET(r, 15, 8);
+
+	r = dsi_read_reg(dsidev, DSI_VM_TIMING7);
+	exit_hs_mode_lat = FLD_GET(r, 15, 0);
+	enter_hs_mode_lat = FLD_GET(r, 31, 16);
+
+	r = dsi_read_reg(dsidev, DSI_CLK_CTRL);
+	lp_clk_div = FLD_GET(r, 12, 0);
+	ddr_alwon = FLD_GET(r, 13, 13);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG0);
+	ths_exit = FLD_GET(r, 7, 0);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG1);
+	tclk_trail = FLD_GET(r, 15, 8);
+
+	exiths_clk = ths_exit + tclk_trail;
+
+	width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8);
+	bllp = hbp + hfp + hsa + DIV_ROUND_UP(width_bytes + 6, ndl);
+
+	if (!hsa_blanking_mode) {
+		hsa_interleave_hs = dsi_compute_interleave_hs(hsa, ddr_alwon,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					exiths_clk, ddr_clk_pre, ddr_clk_post);
+		hsa_interleave_lp = dsi_compute_interleave_lp(hsa,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					lp_clk_div, dsi_fclk_hsdiv);
+	}
+
+	if (!hfp_blanking_mode) {
+		hfp_interleave_hs = dsi_compute_interleave_hs(hfp, ddr_alwon,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					exiths_clk, ddr_clk_pre, ddr_clk_post);
+		hfp_interleave_lp = dsi_compute_interleave_lp(hfp,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					lp_clk_div, dsi_fclk_hsdiv);
+	}
+
+	if (!hbp_blanking_mode) {
+		hbp_interleave_hs = dsi_compute_interleave_hs(hbp, ddr_alwon,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					exiths_clk, ddr_clk_pre, ddr_clk_post);
+
+		hbp_interleave_lp = dsi_compute_interleave_lp(hbp,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					lp_clk_div, dsi_fclk_hsdiv);
+	}
+
+	if (!blanking_mode) {
+		bl_interleave_hs = dsi_compute_interleave_hs(bllp, ddr_alwon,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					exiths_clk, ddr_clk_pre, ddr_clk_post);
+
+		bl_interleave_lp = dsi_compute_interleave_lp(bllp,
+					enter_hs_mode_lat, exit_hs_mode_lat,
+					lp_clk_div, dsi_fclk_hsdiv);
+	}
+
+	DSSDBG("DSI HS interleaving(TXBYTECLKHS) HSA %d, HFP %d, HBP %d, BLLP %d\n",
+		hsa_interleave_hs, hfp_interleave_hs, hbp_interleave_hs,
+		bl_interleave_hs);
+
+	DSSDBG("DSI LP interleaving(bytes) HSA %d, HFP %d, HBP %d, BLLP %d\n",
+		hsa_interleave_lp, hfp_interleave_lp, hbp_interleave_lp,
+		bl_interleave_lp);
+
+	r = dsi_read_reg(dsidev, DSI_VM_TIMING4);
+	r = FLD_MOD(r, hsa_interleave_hs, 23, 16);
+	r = FLD_MOD(r, hfp_interleave_hs, 15, 8);
+	r = FLD_MOD(r, hbp_interleave_hs, 7, 0);
+	dsi_write_reg(dsidev, DSI_VM_TIMING4, r);
+
+	r = dsi_read_reg(dsidev, DSI_VM_TIMING5);
+	r = FLD_MOD(r, hsa_interleave_lp, 23, 16);
+	r = FLD_MOD(r, hfp_interleave_lp, 15, 8);
+	r = FLD_MOD(r, hbp_interleave_lp, 7, 0);
+	dsi_write_reg(dsidev, DSI_VM_TIMING5, r);
+
+	r = dsi_read_reg(dsidev, DSI_VM_TIMING6);
+	r = FLD_MOD(r, bl_interleave_hs, 31, 15);
+	r = FLD_MOD(r, bl_interleave_lp, 16, 0);
+	dsi_write_reg(dsidev, DSI_VM_TIMING6, r);
+}
+
+static int dsi_proto_config(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u32 r;
+	int buswidth = 0;
+
+	dsi_config_tx_fifo(dsidev, DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32);
+
+	dsi_config_rx_fifo(dsidev, DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32,
+			DSI_FIFO_SIZE_32);
+
+	/* XXX what values for the timeouts? */
+	dsi_set_stop_state_counter(dsidev, 0x1000, false, false);
+	dsi_set_ta_timeout(dsidev, 0x1fff, true, true);
+	dsi_set_lp_rx_timeout(dsidev, 0x1fff, true, true);
+	dsi_set_hs_tx_timeout(dsidev, 0x1fff, true, true);
+
+	switch (dsi_get_pixel_size(dsi->pix_fmt)) {
+	case 16:
+		buswidth = 0;
+		break;
+	case 18:
+		buswidth = 1;
+		break;
+	case 24:
+		buswidth = 2;
+		break;
+	default:
+		BUG();
+		return -EINVAL;
+	}
+
+	r = dsi_read_reg(dsidev, DSI_CTRL);
+	r = FLD_MOD(r, 1, 1, 1);	/* CS_RX_EN */
+	r = FLD_MOD(r, 1, 2, 2);	/* ECC_RX_EN */
+	r = FLD_MOD(r, 1, 3, 3);	/* TX_FIFO_ARBITRATION */
+	r = FLD_MOD(r, 1, 4, 4);	/* VP_CLK_RATIO, always 1, see errata*/
+	r = FLD_MOD(r, buswidth, 7, 6); /* VP_DATA_BUS_WIDTH */
+	r = FLD_MOD(r, 0, 8, 8);	/* VP_CLK_POL */
+	r = FLD_MOD(r, 1, 14, 14);	/* TRIGGER_RESET_MODE */
+	r = FLD_MOD(r, 1, 19, 19);	/* EOT_ENABLE */
+	if (!dss_has_feature(FEAT_DSI_DCS_CMD_CONFIG_VC)) {
+		r = FLD_MOD(r, 1, 24, 24);	/* DCS_CMD_ENABLE */
+		/* DCS_CMD_CODE, 1=start, 0=continue */
+		r = FLD_MOD(r, 0, 25, 25);
+	}
+
+	dsi_write_reg(dsidev, DSI_CTRL, r);
+
+	dsi_config_vp_num_line_buffers(dsidev);
+
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		dsi_config_vp_sync_events(dsidev);
+		dsi_config_blanking_modes(dsidev);
+		dsi_config_cmd_mode_interleaving(dsidev);
+	}
+
+	dsi_vc_initial_config(dsidev, 0);
+	dsi_vc_initial_config(dsidev, 1);
+	dsi_vc_initial_config(dsidev, 2);
+	dsi_vc_initial_config(dsidev, 3);
+
+	return 0;
+}
+
+static void dsi_proto_timings(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	unsigned tlpx, tclk_zero, tclk_prepare, tclk_trail;
+	unsigned tclk_pre, tclk_post;
+	unsigned ths_prepare, ths_prepare_ths_zero, ths_zero;
+	unsigned ths_trail, ths_exit;
+	unsigned ddr_clk_pre, ddr_clk_post;
+	unsigned enter_hs_mode_lat, exit_hs_mode_lat;
+	unsigned ths_eot;
+	int ndl = dsi->num_lanes_used - 1;
+	u32 r;
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG0);
+	ths_prepare = FLD_GET(r, 31, 24);
+	ths_prepare_ths_zero = FLD_GET(r, 23, 16);
+	ths_zero = ths_prepare_ths_zero - ths_prepare;
+	ths_trail = FLD_GET(r, 15, 8);
+	ths_exit = FLD_GET(r, 7, 0);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG1);
+	tlpx = FLD_GET(r, 20, 16) * 2;
+	tclk_trail = FLD_GET(r, 15, 8);
+	tclk_zero = FLD_GET(r, 7, 0);
+
+	r = dsi_read_reg(dsidev, DSI_DSIPHY_CFG2);
+	tclk_prepare = FLD_GET(r, 7, 0);
+
+	/* min 8*UI */
+	tclk_pre = 20;
+	/* min 60ns + 52*UI */
+	tclk_post = ns2ddr(dsidev, 60) + 26;
+
+	ths_eot = DIV_ROUND_UP(4, ndl);
+
+	ddr_clk_pre = DIV_ROUND_UP(tclk_pre + tlpx + tclk_zero + tclk_prepare,
+			4);
+	ddr_clk_post = DIV_ROUND_UP(tclk_post + ths_trail, 4) + ths_eot;
+
+	BUG_ON(ddr_clk_pre == 0 || ddr_clk_pre > 255);
+	BUG_ON(ddr_clk_post == 0 || ddr_clk_post > 255);
+
+	r = dsi_read_reg(dsidev, DSI_CLK_TIMING);
+	r = FLD_MOD(r, ddr_clk_pre, 15, 8);
+	r = FLD_MOD(r, ddr_clk_post, 7, 0);
+	dsi_write_reg(dsidev, DSI_CLK_TIMING, r);
+
+	DSSDBG("ddr_clk_pre %u, ddr_clk_post %u\n",
+			ddr_clk_pre,
+			ddr_clk_post);
+
+	enter_hs_mode_lat = 1 + DIV_ROUND_UP(tlpx, 4) +
+		DIV_ROUND_UP(ths_prepare, 4) +
+		DIV_ROUND_UP(ths_zero + 3, 4);
+
+	exit_hs_mode_lat = DIV_ROUND_UP(ths_trail + ths_exit, 4) + 1 + ths_eot;
+
+	r = FLD_VAL(enter_hs_mode_lat, 31, 16) |
+		FLD_VAL(exit_hs_mode_lat, 15, 0);
+	dsi_write_reg(dsidev, DSI_VM_TIMING7, r);
+
+	DSSDBG("enter_hs_mode_lat %u, exit_hs_mode_lat %u\n",
+			enter_hs_mode_lat, exit_hs_mode_lat);
+
+	 if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		/* TODO: Implement a video mode check_timings function */
+		int hsa = dsi->vm_timings.hsa;
+		int hfp = dsi->vm_timings.hfp;
+		int hbp = dsi->vm_timings.hbp;
+		int vsa = dsi->vm_timings.vsa;
+		int vfp = dsi->vm_timings.vfp;
+		int vbp = dsi->vm_timings.vbp;
+		int window_sync = dsi->vm_timings.window_sync;
+		bool hsync_end;
+		struct omap_video_timings *timings = &dsi->timings;
+		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
+		int tl, t_he, width_bytes;
+
+		hsync_end = dsi->vm_timings.trans_mode == OMAP_DSS_DSI_PULSE_MODE;
+		t_he = hsync_end ?
+			((hsa == 0 && ndl == 3) ? 1 : DIV_ROUND_UP(4, ndl)) : 0;
+
+		width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8);
+
+		/* TL = t_HS + HSA + t_HE + HFP + ceil((WC + 6) / NDL) + HBP */
+		tl = DIV_ROUND_UP(4, ndl) + (hsync_end ? hsa : 0) + t_he + hfp +
+			DIV_ROUND_UP(width_bytes + 6, ndl) + hbp;
+
+		DSSDBG("HBP: %d, HFP: %d, HSA: %d, TL: %d TXBYTECLKHS\n", hbp,
+			hfp, hsync_end ? hsa : 0, tl);
+		DSSDBG("VBP: %d, VFP: %d, VSA: %d, VACT: %d lines\n", vbp, vfp,
+			vsa, timings->y_res);
+
+		r = dsi_read_reg(dsidev, DSI_VM_TIMING1);
+		r = FLD_MOD(r, hbp, 11, 0);	/* HBP */
+		r = FLD_MOD(r, hfp, 23, 12);	/* HFP */
+		r = FLD_MOD(r, hsync_end ? hsa : 0, 31, 24);	/* HSA */
+		dsi_write_reg(dsidev, DSI_VM_TIMING1, r);
+
+		r = dsi_read_reg(dsidev, DSI_VM_TIMING2);
+		r = FLD_MOD(r, vbp, 7, 0);	/* VBP */
+		r = FLD_MOD(r, vfp, 15, 8);	/* VFP */
+		r = FLD_MOD(r, vsa, 23, 16);	/* VSA */
+		r = FLD_MOD(r, window_sync, 27, 24);	/* WINDOW_SYNC */
+		dsi_write_reg(dsidev, DSI_VM_TIMING2, r);
+
+		r = dsi_read_reg(dsidev, DSI_VM_TIMING3);
+		r = FLD_MOD(r, timings->y_res, 14, 0);	/* VACT */
+		r = FLD_MOD(r, tl, 31, 16);		/* TL */
+		dsi_write_reg(dsidev, DSI_VM_TIMING3, r);
+	}
+}
+
+static int dsi_configure_pins(struct omap_dss_device *dssdev,
+		const struct omap_dsi_pin_config *pin_cfg)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int num_pins;
+	const int *pins;
+	struct dsi_lane_config lanes[DSI_MAX_NR_LANES];
+	int num_lanes;
+	int i;
+
+	static const enum dsi_lane_function functions[] = {
+		DSI_LANE_CLK,
+		DSI_LANE_DATA1,
+		DSI_LANE_DATA2,
+		DSI_LANE_DATA3,
+		DSI_LANE_DATA4,
+	};
+
+	num_pins = pin_cfg->num_pins;
+	pins = pin_cfg->pins;
+
+	if (num_pins < 4 || num_pins > dsi->num_lanes_supported * 2
+			|| num_pins % 2 != 0)
+		return -EINVAL;
+
+	for (i = 0; i < DSI_MAX_NR_LANES; ++i)
+		lanes[i].function = DSI_LANE_UNUSED;
+
+	num_lanes = 0;
+
+	for (i = 0; i < num_pins; i += 2) {
+		u8 lane, pol;
+		int dx, dy;
+
+		dx = pins[i];
+		dy = pins[i + 1];
+
+		if (dx < 0 || dx >= dsi->num_lanes_supported * 2)
+			return -EINVAL;
+
+		if (dy < 0 || dy >= dsi->num_lanes_supported * 2)
+			return -EINVAL;
+
+		if (dx & 1) {
+			if (dy != dx - 1)
+				return -EINVAL;
+			pol = 1;
+		} else {
+			if (dy != dx + 1)
+				return -EINVAL;
+			pol = 0;
+		}
+
+		lane = dx / 2;
+
+		lanes[lane].function = functions[i / 2];
+		lanes[lane].polarity = pol;
+		num_lanes++;
+	}
+
+	memcpy(dsi->lanes, lanes, sizeof(dsi->lanes));
+	dsi->num_lanes_used = num_lanes;
+
+	return 0;
+}
+
+static int dsi_enable_video_output(struct omap_dss_device *dssdev, int channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct omap_overlay_manager *mgr = dsi->output.manager;
+	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
+	struct omap_dss_device *out = &dsi->output;
+	u8 data_type;
+	u16 word_count;
+	int r;
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		return -ENODEV;
+	}
+
+	r = dsi_display_init_dispc(dsidev, mgr);
+	if (r)
+		goto err_init_dispc;
+
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		switch (dsi->pix_fmt) {
+		case OMAP_DSS_DSI_FMT_RGB888:
+			data_type = MIPI_DSI_PACKED_PIXEL_STREAM_24;
+			break;
+		case OMAP_DSS_DSI_FMT_RGB666:
+			data_type = MIPI_DSI_PIXEL_STREAM_3BYTE_18;
+			break;
+		case OMAP_DSS_DSI_FMT_RGB666_PACKED:
+			data_type = MIPI_DSI_PACKED_PIXEL_STREAM_18;
+			break;
+		case OMAP_DSS_DSI_FMT_RGB565:
+			data_type = MIPI_DSI_PACKED_PIXEL_STREAM_16;
+			break;
+		default:
+			r = -EINVAL;
+			goto err_pix_fmt;
+		}
+
+		dsi_if_enable(dsidev, false);
+		dsi_vc_enable(dsidev, channel, false);
+
+		/* MODE, 1 = video mode */
+		REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), 1, 4, 4);
+
+		word_count = DIV_ROUND_UP(dsi->timings.x_res * bpp, 8);
+
+		dsi_vc_write_long_header(dsidev, channel, data_type,
+				word_count, 0);
+
+		dsi_vc_enable(dsidev, channel, true);
+		dsi_if_enable(dsidev, true);
+	}
+
+	r = dss_mgr_enable(mgr);
+	if (r)
+		goto err_mgr_enable;
+
+	return 0;
+
+err_mgr_enable:
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		dsi_if_enable(dsidev, false);
+		dsi_vc_enable(dsidev, channel, false);
+	}
+err_pix_fmt:
+	dsi_display_uninit_dispc(dsidev, mgr);
+err_init_dispc:
+	return r;
+}
+
+static void dsi_disable_video_output(struct omap_dss_device *dssdev, int channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct omap_overlay_manager *mgr = dsi->output.manager;
+
+	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
+		dsi_if_enable(dsidev, false);
+		dsi_vc_enable(dsidev, channel, false);
+
+		/* MODE, 0 = command mode */
+		REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), 0, 4, 4);
+
+		dsi_vc_enable(dsidev, channel, true);
+		dsi_if_enable(dsidev, true);
+	}
+
+	dss_mgr_disable(mgr);
+
+	dsi_display_uninit_dispc(dsidev, mgr);
+}
+
+static void dsi_update_screen_dispc(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct omap_overlay_manager *mgr = dsi->output.manager;
+	unsigned bytespp;
+	unsigned bytespl;
+	unsigned bytespf;
+	unsigned total_len;
+	unsigned packet_payload;
+	unsigned packet_len;
+	u32 l;
+	int r;
+	const unsigned channel = dsi->update_channel;
+	const unsigned line_buf_size = dsi->line_buffer_size;
+	u16 w = dsi->timings.x_res;
+	u16 h = dsi->timings.y_res;
+
+	DSSDBG("dsi_update_screen_dispc(%dx%d)\n", w, h);
+
+	dsi_vc_config_source(dsidev, channel, DSI_VC_SOURCE_VP);
+
+	bytespp	= dsi_get_pixel_size(dsi->pix_fmt) / 8;
+	bytespl = w * bytespp;
+	bytespf = bytespl * h;
+
+	/* NOTE: packet_payload has to be equal to N * bytespl, where N is
+	 * number of lines in a packet.  See errata about VP_CLK_RATIO */
+
+	if (bytespf < line_buf_size)
+		packet_payload = bytespf;
+	else
+		packet_payload = (line_buf_size) / bytespl * bytespl;
+
+	packet_len = packet_payload + 1;	/* 1 byte for DCS cmd */
+	total_len = (bytespf / packet_payload) * packet_len;
+
+	if (bytespf % packet_payload)
+		total_len += (bytespf % packet_payload) + 1;
+
+	l = FLD_VAL(total_len, 23, 0); /* TE_SIZE */
+	dsi_write_reg(dsidev, DSI_VC_TE(channel), l);
+
+	dsi_vc_write_long_header(dsidev, channel, MIPI_DSI_DCS_LONG_WRITE,
+		packet_len, 0);
+
+	if (dsi->te_enabled)
+		l = FLD_MOD(l, 1, 30, 30); /* TE_EN */
+	else
+		l = FLD_MOD(l, 1, 31, 31); /* TE_START */
+	dsi_write_reg(dsidev, DSI_VC_TE(channel), l);
+
+	/* We put SIDLEMODE to no-idle for the duration of the transfer,
+	 * because DSS interrupts are not capable of waking up the CPU and the
+	 * framedone interrupt could be delayed for quite a long time. I think
+	 * the same goes for any DSS interrupts, but for some reason I have not
+	 * seen the problem anywhere else than here.
+	 */
+	dispc_disable_sidle();
+
+	dsi_perf_mark_start(dsidev);
+
+	r = schedule_delayed_work(&dsi->framedone_timeout_work,
+		msecs_to_jiffies(250));
+	BUG_ON(r == 0);
+
+	dss_mgr_set_timings(mgr, &dsi->timings);
+
+	dss_mgr_start_update(mgr);
+
+	if (dsi->te_enabled) {
+		/* disable LP_RX_TO, so that we can receive TE.  Time to wait
+		 * for TE is longer than the timer allows */
+		REG_FLD_MOD(dsidev, DSI_TIMING2, 0, 15, 15); /* LP_RX_TO */
+
+		dsi_vc_send_bta(dsidev, channel);
+
+#ifdef DSI_CATCH_MISSING_TE
+		mod_timer(&dsi->te_timer, jiffies + msecs_to_jiffies(250));
+#endif
+	}
+}
+
+#ifdef DSI_CATCH_MISSING_TE
+static void dsi_te_timeout(unsigned long arg)
+{
+	DSSERR("TE not received for 250ms!\n");
+}
+#endif
+
+static void dsi_handle_framedone(struct platform_device *dsidev, int error)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	/* SIDLEMODE back to smart-idle */
+	dispc_enable_sidle();
+
+	if (dsi->te_enabled) {
+		/* enable LP_RX_TO again after the TE */
+		REG_FLD_MOD(dsidev, DSI_TIMING2, 1, 15, 15); /* LP_RX_TO */
+	}
+
+	dsi->framedone_callback(error, dsi->framedone_data);
+
+	if (!error)
+		dsi_perf_show(dsidev, "DISPC");
+}
+
+static void dsi_framedone_timeout_work_callback(struct work_struct *work)
+{
+	struct dsi_data *dsi = container_of(work, struct dsi_data,
+			framedone_timeout_work.work);
+	/* XXX While extremely unlikely, we could get FRAMEDONE interrupt after
+	 * 250ms which would conflict with this timeout work. What should be
+	 * done is first cancel the transfer on the HW, and then cancel the
+	 * possibly scheduled framedone work. However, cancelling the transfer
+	 * on the HW is buggy, and would probably require resetting the whole
+	 * DSI */
+
+	DSSERR("Framedone not received for 250ms!\n");
+
+	dsi_handle_framedone(dsi->pdev, -ETIMEDOUT);
+}
+
+static void dsi_framedone_irq_callback(void *data)
+{
+	struct platform_device *dsidev = (struct platform_device *) data;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	/* Note: We get FRAMEDONE when DISPC has finished sending pixels and
+	 * turns itself off. However, DSI still has the pixels in its buffers,
+	 * and is sending the data.
+	 */
+
+	cancel_delayed_work(&dsi->framedone_timeout_work);
+
+	dsi_handle_framedone(dsidev, 0);
+}
+
+static int dsi_update(struct omap_dss_device *dssdev, int channel,
+		void (*callback)(int, void *), void *data)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	u16 dw, dh;
+
+	dsi_perf_mark_setup(dsidev);
+
+	dsi->update_channel = channel;
+
+	dsi->framedone_callback = callback;
+	dsi->framedone_data = data;
+
+	dw = dsi->timings.x_res;
+	dh = dsi->timings.y_res;
+
+#ifdef DSI_PERF_MEASURE
+	dsi->update_bytes = dw * dh *
+		dsi_get_pixel_size(dsi->pix_fmt) / 8;
+#endif
+	dsi_update_screen_dispc(dsidev);
+
+	return 0;
+}
+
+/* Display funcs */
+
+static int dsi_configure_dispc_clocks(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct dispc_clock_info dispc_cinfo;
+	int r;
+	unsigned long fck;
+
+	fck = dsi_get_pll_hsdiv_dispc_rate(dsidev);
+
+	dispc_cinfo.lck_div = dsi->user_dispc_cinfo.lck_div;
+	dispc_cinfo.pck_div = dsi->user_dispc_cinfo.pck_div;
+
+	r = dispc_calc_clock_rates(fck, &dispc_cinfo);
+	if (r) {
+		DSSERR("Failed to calc dispc clocks\n");
+		return r;
+	}
+
+	dsi->mgr_config.clock_info = dispc_cinfo;
+
+	return 0;
+}
+
+static int dsi_display_init_dispc(struct platform_device *dsidev,
+		struct omap_overlay_manager *mgr)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r;
+
+	dss_select_lcd_clk_source(mgr->id, dsi->module_id == 0 ?
+			OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC :
+			OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC);
+
+	if (dsi->mode == OMAP_DSS_DSI_CMD_MODE) {
+		r = dss_mgr_register_framedone_handler(mgr,
+				dsi_framedone_irq_callback, dsidev);
+		if (r) {
+			DSSERR("can't register FRAMEDONE handler\n");
+			goto err;
+		}
+
+		dsi->mgr_config.stallmode = true;
+		dsi->mgr_config.fifohandcheck = true;
+	} else {
+		dsi->mgr_config.stallmode = false;
+		dsi->mgr_config.fifohandcheck = false;
+	}
+
+	/*
+	 * override interlace, logic level and edge related parameters in
+	 * omap_video_timings with default values
+	 */
+	dsi->timings.interlace = false;
+	dsi->timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	dsi->timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	dsi->timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	dsi->timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	dsi->timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE;
+
+	dss_mgr_set_timings(mgr, &dsi->timings);
+
+	r = dsi_configure_dispc_clocks(dsidev);
+	if (r)
+		goto err1;
+
+	dsi->mgr_config.io_pad_mode = DSS_IO_PAD_MODE_BYPASS;
+	dsi->mgr_config.video_port_width =
+			dsi_get_pixel_size(dsi->pix_fmt);
+	dsi->mgr_config.lcden_sig_polarity = 0;
+
+	dss_mgr_set_lcd_config(mgr, &dsi->mgr_config);
+
+	return 0;
+err1:
+	if (dsi->mode == OMAP_DSS_DSI_CMD_MODE)
+		dss_mgr_unregister_framedone_handler(mgr,
+				dsi_framedone_irq_callback, dsidev);
+err:
+	dss_select_lcd_clk_source(mgr->id, OMAP_DSS_CLK_SRC_FCK);
+	return r;
+}
+
+static void dsi_display_uninit_dispc(struct platform_device *dsidev,
+		struct omap_overlay_manager *mgr)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (dsi->mode == OMAP_DSS_DSI_CMD_MODE)
+		dss_mgr_unregister_framedone_handler(mgr,
+				dsi_framedone_irq_callback, dsidev);
+
+	dss_select_lcd_clk_source(mgr->id, OMAP_DSS_CLK_SRC_FCK);
+}
+
+static int dsi_configure_dsi_clocks(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct dss_pll_clock_info cinfo;
+	int r;
+
+	cinfo = dsi->user_dsi_cinfo;
+
+	r = dss_pll_set_config(&dsi->pll, &cinfo);
+	if (r) {
+		DSSERR("Failed to set dsi clocks\n");
+		return r;
+	}
+
+	return 0;
+}
+
+static int dsi_display_init_dsi(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r;
+
+	r = dss_pll_enable(&dsi->pll);
+	if (r)
+		goto err0;
+
+	r = dsi_configure_dsi_clocks(dsidev);
+	if (r)
+		goto err1;
+
+	dss_select_dsi_clk_source(dsi->module_id, dsi->module_id == 0 ?
+			OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI :
+			OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI);
+
+	DSSDBG("PLL OK\n");
+
+	r = dsi_cio_init(dsidev);
+	if (r)
+		goto err2;
+
+	_dsi_print_reset_status(dsidev);
+
+	dsi_proto_timings(dsidev);
+	dsi_set_lp_clk_divisor(dsidev);
+
+	if (1)
+		_dsi_print_reset_status(dsidev);
+
+	r = dsi_proto_config(dsidev);
+	if (r)
+		goto err3;
+
+	/* enable interface */
+	dsi_vc_enable(dsidev, 0, 1);
+	dsi_vc_enable(dsidev, 1, 1);
+	dsi_vc_enable(dsidev, 2, 1);
+	dsi_vc_enable(dsidev, 3, 1);
+	dsi_if_enable(dsidev, 1);
+	dsi_force_tx_stop_mode_io(dsidev);
+
+	return 0;
+err3:
+	dsi_cio_uninit(dsidev);
+err2:
+	dss_select_dsi_clk_source(dsi->module_id, OMAP_DSS_CLK_SRC_FCK);
+err1:
+	dss_pll_disable(&dsi->pll);
+err0:
+	return r;
+}
+
+static void dsi_display_uninit_dsi(struct platform_device *dsidev,
+		bool disconnect_lanes, bool enter_ulps)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (enter_ulps && !dsi->ulps_enabled)
+		dsi_enter_ulps(dsidev);
+
+	/* disable interface */
+	dsi_if_enable(dsidev, 0);
+	dsi_vc_enable(dsidev, 0, 0);
+	dsi_vc_enable(dsidev, 1, 0);
+	dsi_vc_enable(dsidev, 2, 0);
+	dsi_vc_enable(dsidev, 3, 0);
+
+	dss_select_dsi_clk_source(dsi->module_id, OMAP_DSS_CLK_SRC_FCK);
+	dsi_cio_uninit(dsidev);
+	dsi_pll_uninit(dsidev, disconnect_lanes);
+}
+
+static int dsi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int r = 0;
+
+	DSSDBG("dsi_display_enable\n");
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	mutex_lock(&dsi->lock);
+
+	r = dsi_runtime_get(dsidev);
+	if (r)
+		goto err_get_dsi;
+
+	_dsi_initialize_irq(dsidev);
+
+	r = dsi_display_init_dsi(dsidev);
+	if (r)
+		goto err_init_dsi;
+
+	mutex_unlock(&dsi->lock);
+
+	return 0;
+
+err_init_dsi:
+	dsi_runtime_put(dsidev);
+err_get_dsi:
+	mutex_unlock(&dsi->lock);
+	DSSDBG("dsi_display_enable FAILED\n");
+	return r;
+}
+
+static void dsi_display_disable(struct omap_dss_device *dssdev,
+		bool disconnect_lanes, bool enter_ulps)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	DSSDBG("dsi_display_disable\n");
+
+	WARN_ON(!dsi_bus_is_locked(dsidev));
+
+	mutex_lock(&dsi->lock);
+
+	dsi_sync_vc(dsidev, 0);
+	dsi_sync_vc(dsidev, 1);
+	dsi_sync_vc(dsidev, 2);
+	dsi_sync_vc(dsidev, 3);
+
+	dsi_display_uninit_dsi(dsidev, disconnect_lanes, enter_ulps);
+
+	dsi_runtime_put(dsidev);
+
+	mutex_unlock(&dsi->lock);
+}
+
+static int dsi_enable_te(struct omap_dss_device *dssdev, bool enable)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	dsi->te_enabled = enable;
+	return 0;
+}
+
+#ifdef PRINT_VERBOSE_VM_TIMINGS
+static void print_dsi_vm(const char *str,
+		const struct omap_dss_dsi_videomode_timings *t)
+{
+	unsigned long byteclk = t->hsclk / 4;
+	int bl, wc, pps, tot;
+
+	wc = DIV_ROUND_UP(t->hact * t->bitspp, 8);
+	pps = DIV_ROUND_UP(wc + 6, t->ndl); /* pixel packet size */
+	bl = t->hss + t->hsa + t->hse + t->hbp + t->hfp;
+	tot = bl + pps;
+
+#define TO_DSI_T(x) ((u32)div64_u64((u64)x * 1000000000llu, byteclk))
+
+	pr_debug("%s bck %lu, %u/%u/%u/%u/%u/%u = %u+%u = %u, "
+			"%u/%u/%u/%u/%u/%u = %u + %u = %u\n",
+			str,
+			byteclk,
+			t->hss, t->hsa, t->hse, t->hbp, pps, t->hfp,
+			bl, pps, tot,
+			TO_DSI_T(t->hss),
+			TO_DSI_T(t->hsa),
+			TO_DSI_T(t->hse),
+			TO_DSI_T(t->hbp),
+			TO_DSI_T(pps),
+			TO_DSI_T(t->hfp),
+
+			TO_DSI_T(bl),
+			TO_DSI_T(pps),
+
+			TO_DSI_T(tot));
+#undef TO_DSI_T
+}
+
+static void print_dispc_vm(const char *str, const struct omap_video_timings *t)
+{
+	unsigned long pck = t->pixelclock;
+	int hact, bl, tot;
+
+	hact = t->x_res;
+	bl = t->hsw + t->hbp + t->hfp;
+	tot = hact + bl;
+
+#define TO_DISPC_T(x) ((u32)div64_u64((u64)x * 1000000000llu, pck))
+
+	pr_debug("%s pck %lu, %u/%u/%u/%u = %u+%u = %u, "
+			"%u/%u/%u/%u = %u + %u = %u\n",
+			str,
+			pck,
+			t->hsw, t->hbp, hact, t->hfp,
+			bl, hact, tot,
+			TO_DISPC_T(t->hsw),
+			TO_DISPC_T(t->hbp),
+			TO_DISPC_T(hact),
+			TO_DISPC_T(t->hfp),
+			TO_DISPC_T(bl),
+			TO_DISPC_T(hact),
+			TO_DISPC_T(tot));
+#undef TO_DISPC_T
+}
+
+/* note: this is not quite accurate */
+static void print_dsi_dispc_vm(const char *str,
+		const struct omap_dss_dsi_videomode_timings *t)
+{
+	struct omap_video_timings vm = { 0 };
+	unsigned long byteclk = t->hsclk / 4;
+	unsigned long pck;
+	u64 dsi_tput;
+	int dsi_hact, dsi_htot;
+
+	dsi_tput = (u64)byteclk * t->ndl * 8;
+	pck = (u32)div64_u64(dsi_tput, t->bitspp);
+	dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(t->hact * t->bitspp, 8) + 6, t->ndl);
+	dsi_htot = t->hss + t->hsa + t->hse + t->hbp + dsi_hact + t->hfp;
+
+	vm.pixelclock = pck;
+	vm.hsw = div64_u64((u64)(t->hsa + t->hse) * pck, byteclk);
+	vm.hbp = div64_u64((u64)t->hbp * pck, byteclk);
+	vm.hfp = div64_u64((u64)t->hfp * pck, byteclk);
+	vm.x_res = t->hact;
+
+	print_dispc_vm(str, &vm);
+}
+#endif /* PRINT_VERBOSE_VM_TIMINGS */
+
+static bool dsi_cm_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
+		unsigned long pck, void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+	struct omap_video_timings *t = &ctx->dispc_vm;
+
+	ctx->dispc_cinfo.lck_div = lckd;
+	ctx->dispc_cinfo.pck_div = pckd;
+	ctx->dispc_cinfo.lck = lck;
+	ctx->dispc_cinfo.pck = pck;
+
+	*t = *ctx->config->timings;
+	t->pixelclock = pck;
+	t->x_res = ctx->config->timings->x_res;
+	t->y_res = ctx->config->timings->y_res;
+	t->hsw = t->hfp = t->hbp = t->vsw = 1;
+	t->vfp = t->vbp = 0;
+
+	return true;
+}
+
+static bool dsi_cm_calc_hsdiv_cb(int m_dispc, unsigned long dispc,
+		void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+
+	ctx->dsi_cinfo.mX[HSDIV_DISPC] = m_dispc;
+	ctx->dsi_cinfo.clkout[HSDIV_DISPC] = dispc;
+
+	return dispc_div_calc(dispc, ctx->req_pck_min, ctx->req_pck_max,
+			dsi_cm_calc_dispc_cb, ctx);
+}
+
+static bool dsi_cm_calc_pll_cb(int n, int m, unsigned long fint,
+		unsigned long clkdco, void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+
+	ctx->dsi_cinfo.n = n;
+	ctx->dsi_cinfo.m = m;
+	ctx->dsi_cinfo.fint = fint;
+	ctx->dsi_cinfo.clkdco = clkdco;
+
+	return dss_pll_hsdiv_calc(ctx->pll, clkdco, ctx->req_pck_min,
+			dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+			dsi_cm_calc_hsdiv_cb, ctx);
+}
+
+static bool dsi_cm_calc(struct dsi_data *dsi,
+		const struct omap_dss_dsi_config *cfg,
+		struct dsi_clk_calc_ctx *ctx)
+{
+	unsigned long clkin;
+	int bitspp, ndl;
+	unsigned long pll_min, pll_max;
+	unsigned long pck, txbyteclk;
+
+	clkin = clk_get_rate(dsi->pll.clkin);
+	bitspp = dsi_get_pixel_size(cfg->pixel_format);
+	ndl = dsi->num_lanes_used - 1;
+
+	/*
+	 * Here we should calculate minimum txbyteclk to be able to send the
+	 * frame in time, and also to handle TE. That's not very simple, though,
+	 * especially as we go to LP between each pixel packet due to HW
+	 * "feature". So let's just estimate very roughly and multiply by 1.5.
+	 */
+	pck = cfg->timings->pixelclock;
+	pck = pck * 3 / 2;
+	txbyteclk = pck * bitspp / 8 / ndl;
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->dsidev = dsi->pdev;
+	ctx->pll = &dsi->pll;
+	ctx->config = cfg;
+	ctx->req_pck_min = pck;
+	ctx->req_pck_nom = pck;
+	ctx->req_pck_max = pck * 3 / 2;
+
+	pll_min = max(cfg->hs_clk_min * 4, txbyteclk * 4 * 4);
+	pll_max = cfg->hs_clk_max * 4;
+
+	return dss_pll_calc(ctx->pll, clkin,
+			pll_min, pll_max,
+			dsi_cm_calc_pll_cb, ctx);
+}
+
+static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(ctx->dsidev);
+	const struct omap_dss_dsi_config *cfg = ctx->config;
+	int bitspp = dsi_get_pixel_size(cfg->pixel_format);
+	int ndl = dsi->num_lanes_used - 1;
+	unsigned long hsclk = ctx->dsi_cinfo.clkdco / 4;
+	unsigned long byteclk = hsclk / 4;
+
+	unsigned long dispc_pck, req_pck_min, req_pck_nom, req_pck_max;
+	int xres;
+	int panel_htot, panel_hbl; /* pixels */
+	int dispc_htot, dispc_hbl; /* pixels */
+	int dsi_htot, dsi_hact, dsi_hbl, hss, hse; /* byteclks */
+	int hfp, hsa, hbp;
+	const struct omap_video_timings *req_vm;
+	struct omap_video_timings *dispc_vm;
+	struct omap_dss_dsi_videomode_timings *dsi_vm;
+	u64 dsi_tput, dispc_tput;
+
+	dsi_tput = (u64)byteclk * ndl * 8;
+
+	req_vm = cfg->timings;
+	req_pck_min = ctx->req_pck_min;
+	req_pck_max = ctx->req_pck_max;
+	req_pck_nom = ctx->req_pck_nom;
+
+	dispc_pck = ctx->dispc_cinfo.pck;
+	dispc_tput = (u64)dispc_pck * bitspp;
+
+	xres = req_vm->x_res;
+
+	panel_hbl = req_vm->hfp + req_vm->hbp + req_vm->hsw;
+	panel_htot = xres + panel_hbl;
+
+	dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(xres * bitspp, 8) + 6, ndl);
+
+	/*
+	 * When there are no line buffers, DISPC and DSI must have the
+	 * same tput. Otherwise DISPC tput needs to be higher than DSI's.
+	 */
+	if (dsi->line_buffer_size < xres * bitspp / 8) {
+		if (dispc_tput != dsi_tput)
+			return false;
+	} else {
+		if (dispc_tput < dsi_tput)
+			return false;
+	}
+
+	/* DSI tput must be over the min requirement */
+	if (dsi_tput < (u64)bitspp * req_pck_min)
+		return false;
+
+	/* When non-burst mode, DSI tput must be below max requirement. */
+	if (cfg->trans_mode != OMAP_DSS_DSI_BURST_MODE) {
+		if (dsi_tput > (u64)bitspp * req_pck_max)
+			return false;
+	}
+
+	hss = DIV_ROUND_UP(4, ndl);
+
+	if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) {
+		if (ndl == 3 && req_vm->hsw == 0)
+			hse = 1;
+		else
+			hse = DIV_ROUND_UP(4, ndl);
+	} else {
+		hse = 0;
+	}
+
+	/* DSI htot to match the panel's nominal pck */
+	dsi_htot = div64_u64((u64)panel_htot * byteclk, req_pck_nom);
+
+	/* fail if there would be no time for blanking */
+	if (dsi_htot < hss + hse + dsi_hact)
+		return false;
+
+	/* total DSI blanking needed to achieve panel's TL */
+	dsi_hbl = dsi_htot - dsi_hact;
+
+	/* DISPC htot to match the DSI TL */
+	dispc_htot = div64_u64((u64)dsi_htot * dispc_pck, byteclk);
+
+	/* verify that the DSI and DISPC TLs are the same */
+	if ((u64)dsi_htot * dispc_pck != (u64)dispc_htot * byteclk)
+		return false;
+
+	dispc_hbl = dispc_htot - xres;
+
+	/* setup DSI videomode */
+
+	dsi_vm = &ctx->dsi_vm;
+	memset(dsi_vm, 0, sizeof(*dsi_vm));
+
+	dsi_vm->hsclk = hsclk;
+
+	dsi_vm->ndl = ndl;
+	dsi_vm->bitspp = bitspp;
+
+	if (cfg->trans_mode != OMAP_DSS_DSI_PULSE_MODE) {
+		hsa = 0;
+	} else if (ndl == 3 && req_vm->hsw == 0) {
+		hsa = 0;
+	} else {
+		hsa = div64_u64((u64)req_vm->hsw * byteclk, req_pck_nom);
+		hsa = max(hsa - hse, 1);
+	}
+
+	hbp = div64_u64((u64)req_vm->hbp * byteclk, req_pck_nom);
+	hbp = max(hbp, 1);
+
+	hfp = dsi_hbl - (hss + hsa + hse + hbp);
+	if (hfp < 1) {
+		int t;
+		/* we need to take cycles from hbp */
+
+		t = 1 - hfp;
+		hbp = max(hbp - t, 1);
+		hfp = dsi_hbl - (hss + hsa + hse + hbp);
+
+		if (hfp < 1 && hsa > 0) {
+			/* we need to take cycles from hsa */
+			t = 1 - hfp;
+			hsa = max(hsa - t, 1);
+			hfp = dsi_hbl - (hss + hsa + hse + hbp);
+		}
+	}
+
+	if (hfp < 1)
+		return false;
+
+	dsi_vm->hss = hss;
+	dsi_vm->hsa = hsa;
+	dsi_vm->hse = hse;
+	dsi_vm->hbp = hbp;
+	dsi_vm->hact = xres;
+	dsi_vm->hfp = hfp;
+
+	dsi_vm->vsa = req_vm->vsw;
+	dsi_vm->vbp = req_vm->vbp;
+	dsi_vm->vact = req_vm->y_res;
+	dsi_vm->vfp = req_vm->vfp;
+
+	dsi_vm->trans_mode = cfg->trans_mode;
+
+	dsi_vm->blanking_mode = 0;
+	dsi_vm->hsa_blanking_mode = 1;
+	dsi_vm->hfp_blanking_mode = 1;
+	dsi_vm->hbp_blanking_mode = 1;
+
+	dsi_vm->ddr_clk_always_on = cfg->ddr_clk_always_on;
+	dsi_vm->window_sync = 4;
+
+	/* setup DISPC videomode */
+
+	dispc_vm = &ctx->dispc_vm;
+	*dispc_vm = *req_vm;
+	dispc_vm->pixelclock = dispc_pck;
+
+	if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) {
+		hsa = div64_u64((u64)req_vm->hsw * dispc_pck,
+				req_pck_nom);
+		hsa = max(hsa, 1);
+	} else {
+		hsa = 1;
+	}
+
+	hbp = div64_u64((u64)req_vm->hbp * dispc_pck, req_pck_nom);
+	hbp = max(hbp, 1);
+
+	hfp = dispc_hbl - hsa - hbp;
+	if (hfp < 1) {
+		int t;
+		/* we need to take cycles from hbp */
+
+		t = 1 - hfp;
+		hbp = max(hbp - t, 1);
+		hfp = dispc_hbl - hsa - hbp;
+
+		if (hfp < 1) {
+			/* we need to take cycles from hsa */
+			t = 1 - hfp;
+			hsa = max(hsa - t, 1);
+			hfp = dispc_hbl - hsa - hbp;
+		}
+	}
+
+	if (hfp < 1)
+		return false;
+
+	dispc_vm->hfp = hfp;
+	dispc_vm->hsw = hsa;
+	dispc_vm->hbp = hbp;
+
+	return true;
+}
+
+
+static bool dsi_vm_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
+		unsigned long pck, void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+
+	ctx->dispc_cinfo.lck_div = lckd;
+	ctx->dispc_cinfo.pck_div = pckd;
+	ctx->dispc_cinfo.lck = lck;
+	ctx->dispc_cinfo.pck = pck;
+
+	if (dsi_vm_calc_blanking(ctx) == false)
+		return false;
+
+#ifdef PRINT_VERBOSE_VM_TIMINGS
+	print_dispc_vm("dispc", &ctx->dispc_vm);
+	print_dsi_vm("dsi  ", &ctx->dsi_vm);
+	print_dispc_vm("req  ", ctx->config->timings);
+	print_dsi_dispc_vm("act  ", &ctx->dsi_vm);
+#endif
+
+	return true;
+}
+
+static bool dsi_vm_calc_hsdiv_cb(int m_dispc, unsigned long dispc,
+		void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+	unsigned long pck_max;
+
+	ctx->dsi_cinfo.mX[HSDIV_DISPC] = m_dispc;
+	ctx->dsi_cinfo.clkout[HSDIV_DISPC] = dispc;
+
+	/*
+	 * In burst mode we can let the dispc pck be arbitrarily high, but it
+	 * limits our scaling abilities. So for now, don't aim too high.
+	 */
+
+	if (ctx->config->trans_mode == OMAP_DSS_DSI_BURST_MODE)
+		pck_max = ctx->req_pck_max + 10000000;
+	else
+		pck_max = ctx->req_pck_max;
+
+	return dispc_div_calc(dispc, ctx->req_pck_min, pck_max,
+			dsi_vm_calc_dispc_cb, ctx);
+}
+
+static bool dsi_vm_calc_pll_cb(int n, int m, unsigned long fint,
+		unsigned long clkdco, void *data)
+{
+	struct dsi_clk_calc_ctx *ctx = data;
+
+	ctx->dsi_cinfo.n = n;
+	ctx->dsi_cinfo.m = m;
+	ctx->dsi_cinfo.fint = fint;
+	ctx->dsi_cinfo.clkdco = clkdco;
+
+	return dss_pll_hsdiv_calc(ctx->pll, clkdco, ctx->req_pck_min,
+			dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+			dsi_vm_calc_hsdiv_cb, ctx);
+}
+
+static bool dsi_vm_calc(struct dsi_data *dsi,
+		const struct omap_dss_dsi_config *cfg,
+		struct dsi_clk_calc_ctx *ctx)
+{
+	const struct omap_video_timings *t = cfg->timings;
+	unsigned long clkin;
+	unsigned long pll_min;
+	unsigned long pll_max;
+	int ndl = dsi->num_lanes_used - 1;
+	int bitspp = dsi_get_pixel_size(cfg->pixel_format);
+	unsigned long byteclk_min;
+
+	clkin = clk_get_rate(dsi->pll.clkin);
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->dsidev = dsi->pdev;
+	ctx->pll = &dsi->pll;
+	ctx->config = cfg;
+
+	/* these limits should come from the panel driver */
+	ctx->req_pck_min = t->pixelclock - 1000;
+	ctx->req_pck_nom = t->pixelclock;
+	ctx->req_pck_max = t->pixelclock + 1000;
+
+	byteclk_min = div64_u64((u64)ctx->req_pck_min * bitspp, ndl * 8);
+	pll_min = max(cfg->hs_clk_min * 4, byteclk_min * 4 * 4);
+
+	if (cfg->trans_mode == OMAP_DSS_DSI_BURST_MODE) {
+		pll_max = cfg->hs_clk_max * 4;
+	} else {
+		unsigned long byteclk_max;
+		byteclk_max = div64_u64((u64)ctx->req_pck_max * bitspp,
+				ndl * 8);
+
+		pll_max = byteclk_max * 4 * 4;
+	}
+
+	return dss_pll_calc(ctx->pll, clkin,
+			pll_min, pll_max,
+			dsi_vm_calc_pll_cb, ctx);
+}
+
+static int dsi_set_config(struct omap_dss_device *dssdev,
+		const struct omap_dss_dsi_config *config)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct dsi_clk_calc_ctx ctx;
+	bool ok;
+	int r;
+
+	mutex_lock(&dsi->lock);
+
+	dsi->pix_fmt = config->pixel_format;
+	dsi->mode = config->mode;
+
+	if (config->mode == OMAP_DSS_DSI_VIDEO_MODE)
+		ok = dsi_vm_calc(dsi, config, &ctx);
+	else
+		ok = dsi_cm_calc(dsi, config, &ctx);
+
+	if (!ok) {
+		DSSERR("failed to find suitable DSI clock settings\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	dsi_pll_calc_dsi_fck(&ctx.dsi_cinfo);
+
+	r = dsi_lp_clock_calc(ctx.dsi_cinfo.clkout[HSDIV_DSI],
+		config->lp_clk_min, config->lp_clk_max, &dsi->user_lp_cinfo);
+	if (r) {
+		DSSERR("failed to find suitable DSI LP clock settings\n");
+		goto err;
+	}
+
+	dsi->user_dsi_cinfo = ctx.dsi_cinfo;
+	dsi->user_dispc_cinfo = ctx.dispc_cinfo;
+
+	dsi->timings = ctx.dispc_vm;
+	dsi->vm_timings = ctx.dsi_vm;
+
+	mutex_unlock(&dsi->lock);
+
+	return 0;
+err:
+	mutex_unlock(&dsi->lock);
+
+	return r;
+}
+
+/*
+ * Return a hardcoded channel for the DSI output. This should work for
+ * current use cases, but this can be later expanded to either resolve
+ * the channel in some more dynamic manner, or get the channel as a user
+ * parameter.
+ */
+static enum omap_channel dsi_get_channel(int module_id)
+{
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP24xx:
+	case OMAPDSS_VER_AM43xx:
+		DSSWARN("DSI not supported\n");
+		return OMAP_DSS_CHANNEL_LCD;
+
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_OMAP3630:
+	case OMAPDSS_VER_AM35xx:
+		return OMAP_DSS_CHANNEL_LCD;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		switch (module_id) {
+		case 0:
+			return OMAP_DSS_CHANNEL_LCD;
+		case 1:
+			return OMAP_DSS_CHANNEL_LCD2;
+		default:
+			DSSWARN("unsupported module id\n");
+			return OMAP_DSS_CHANNEL_LCD;
+		}
+
+	case OMAPDSS_VER_OMAP5:
+		switch (module_id) {
+		case 0:
+			return OMAP_DSS_CHANNEL_LCD;
+		case 1:
+			return OMAP_DSS_CHANNEL_LCD3;
+		default:
+			DSSWARN("unsupported module id\n");
+			return OMAP_DSS_CHANNEL_LCD;
+		}
+
+	default:
+		DSSWARN("unsupported DSS version\n");
+		return OMAP_DSS_CHANNEL_LCD;
+	}
+}
+
+static int dsi_request_vc(struct omap_dss_device *dssdev, int *channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dsi->vc); i++) {
+		if (!dsi->vc[i].dssdev) {
+			dsi->vc[i].dssdev = dssdev;
+			*channel = i;
+			return 0;
+		}
+	}
+
+	DSSERR("cannot get VC for display %s", dssdev->name);
+	return -ENOSPC;
+}
+
+static int dsi_set_vc_id(struct omap_dss_device *dssdev, int channel, int vc_id)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if (vc_id < 0 || vc_id > 3) {
+		DSSERR("VC ID out of range\n");
+		return -EINVAL;
+	}
+
+	if (channel < 0 || channel > 3) {
+		DSSERR("Virtual Channel out of range\n");
+		return -EINVAL;
+	}
+
+	if (dsi->vc[channel].dssdev != dssdev) {
+		DSSERR("Virtual Channel not allocated to display %s\n",
+			dssdev->name);
+		return -EINVAL;
+	}
+
+	dsi->vc[channel].vc_id = vc_id;
+
+	return 0;
+}
+
+static void dsi_release_vc(struct omap_dss_device *dssdev, int channel)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	if ((channel >= 0 && channel <= 3) &&
+		dsi->vc[channel].dssdev == dssdev) {
+		dsi->vc[channel].dssdev = NULL;
+		dsi->vc[channel].vc_id = 0;
+	}
+}
+
+
+static int dsi_get_clocks(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct clk *clk;
+
+	clk = devm_clk_get(&dsidev->dev, "fck");
+	if (IS_ERR(clk)) {
+		DSSERR("can't get fck\n");
+		return PTR_ERR(clk);
+	}
+
+	dsi->dss_clk = clk;
+
+	return 0;
+}
+
+static int dsi_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct platform_device *dsidev = dsi_get_dsidev_from_dssdev(dssdev);
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = dsi_regulator_init(dsidev);
+	if (r)
+		return r;
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dssdev->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void dsi_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static const struct omapdss_dsi_ops dsi_ops = {
+	.connect = dsi_connect,
+	.disconnect = dsi_disconnect,
+
+	.bus_lock = dsi_bus_lock,
+	.bus_unlock = dsi_bus_unlock,
+
+	.enable = dsi_display_enable,
+	.disable = dsi_display_disable,
+
+	.enable_hs = dsi_vc_enable_hs,
+
+	.configure_pins = dsi_configure_pins,
+	.set_config = dsi_set_config,
+
+	.enable_video_output = dsi_enable_video_output,
+	.disable_video_output = dsi_disable_video_output,
+
+	.update = dsi_update,
+
+	.enable_te = dsi_enable_te,
+
+	.request_vc = dsi_request_vc,
+	.set_vc_id = dsi_set_vc_id,
+	.release_vc = dsi_release_vc,
+
+	.dcs_write = dsi_vc_dcs_write,
+	.dcs_write_nosync = dsi_vc_dcs_write_nosync,
+	.dcs_read = dsi_vc_dcs_read,
+
+	.gen_write = dsi_vc_generic_write,
+	.gen_write_nosync = dsi_vc_generic_write_nosync,
+	.gen_read = dsi_vc_generic_read,
+
+	.bta_sync = dsi_vc_send_bta_sync,
+
+	.set_max_rx_packet_size = dsi_vc_set_max_rx_packet_size,
+};
+
+static void dsi_init_output(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct omap_dss_device *out = &dsi->output;
+
+	out->dev = &dsidev->dev;
+	out->id = dsi->module_id == 0 ?
+			OMAP_DSS_OUTPUT_DSI1 : OMAP_DSS_OUTPUT_DSI2;
+
+	out->output_type = OMAP_DISPLAY_TYPE_DSI;
+	out->name = dsi->module_id == 0 ? "dsi.0" : "dsi.1";
+	out->dispc_channel = dsi_get_channel(dsi->module_id);
+	out->ops.dsi = &dsi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void dsi_uninit_output(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct omap_dss_device *out = &dsi->output;
+
+	omapdss_unregister_output(out);
+}
+
+static int dsi_probe_of(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(pdev);
+	struct property *prop;
+	u32 lane_arr[10];
+	int len, num_pins;
+	int r, i;
+	struct device_node *ep;
+	struct omap_dsi_pin_config pin_cfg;
+
+	ep = omapdss_of_get_first_endpoint(node);
+	if (!ep)
+		return 0;
+
+	prop = of_find_property(ep, "lanes", &len);
+	if (prop == NULL) {
+		dev_err(&pdev->dev, "failed to find lane data\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	num_pins = len / sizeof(u32);
+
+	if (num_pins < 4 || num_pins % 2 != 0 ||
+		num_pins > dsi->num_lanes_supported * 2) {
+		dev_err(&pdev->dev, "bad number of lanes\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	r = of_property_read_u32_array(ep, "lanes", lane_arr, num_pins);
+	if (r) {
+		dev_err(&pdev->dev, "failed to read lane data\n");
+		goto err;
+	}
+
+	pin_cfg.num_pins = num_pins;
+	for (i = 0; i < num_pins; ++i)
+		pin_cfg.pins[i] = (int)lane_arr[i];
+
+	r = dsi_configure_pins(&dsi->output, &pin_cfg);
+	if (r) {
+		dev_err(&pdev->dev, "failed to configure pins");
+		goto err;
+	}
+
+	of_node_put(ep);
+
+	return 0;
+
+err:
+	of_node_put(ep);
+	return r;
+}
+
+static const struct dss_pll_ops dsi_pll_ops = {
+	.enable = dsi_pll_enable,
+	.disable = dsi_pll_disable,
+	.set_config = dss_pll_write_config_type_a,
+};
+
+static const struct dss_pll_hw dss_omap3_dsi_pll_hw = {
+	.n_max = (1 << 7) - 1,
+	.m_max = (1 << 11) - 1,
+	.mX_max = (1 << 4) - 1,
+	.fint_min = 750000,
+	.fint_max = 2100000,
+	.clkdco_low = 1000000000,
+	.clkdco_max = 1800000000,
+
+	.n_msb = 7,
+	.n_lsb = 1,
+	.m_msb = 18,
+	.m_lsb = 8,
+
+	.mX_msb[0] = 22,
+	.mX_lsb[0] = 19,
+	.mX_msb[1] = 26,
+	.mX_lsb[1] = 23,
+
+	.has_stopmode = true,
+	.has_freqsel = true,
+	.has_selfreqdco = false,
+	.has_refsel = false,
+};
+
+static const struct dss_pll_hw dss_omap4_dsi_pll_hw = {
+	.n_max = (1 << 8) - 1,
+	.m_max = (1 << 12) - 1,
+	.mX_max = (1 << 5) - 1,
+	.fint_min = 500000,
+	.fint_max = 2500000,
+	.clkdco_low = 1000000000,
+	.clkdco_max = 1800000000,
+
+	.n_msb = 8,
+	.n_lsb = 1,
+	.m_msb = 20,
+	.m_lsb = 9,
+
+	.mX_msb[0] = 25,
+	.mX_lsb[0] = 21,
+	.mX_msb[1] = 30,
+	.mX_lsb[1] = 26,
+
+	.has_stopmode = true,
+	.has_freqsel = false,
+	.has_selfreqdco = false,
+	.has_refsel = false,
+};
+
+static const struct dss_pll_hw dss_omap5_dsi_pll_hw = {
+	.n_max = (1 << 8) - 1,
+	.m_max = (1 << 12) - 1,
+	.mX_max = (1 << 5) - 1,
+	.fint_min = 150000,
+	.fint_max = 52000000,
+	.clkdco_low = 1000000000,
+	.clkdco_max = 1800000000,
+
+	.n_msb = 8,
+	.n_lsb = 1,
+	.m_msb = 20,
+	.m_lsb = 9,
+
+	.mX_msb[0] = 25,
+	.mX_lsb[0] = 21,
+	.mX_msb[1] = 30,
+	.mX_lsb[1] = 26,
+
+	.has_stopmode = true,
+	.has_freqsel = false,
+	.has_selfreqdco = true,
+	.has_refsel = true,
+};
+
+static int dsi_init_pll_data(struct platform_device *dsidev)
+{
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+	struct dss_pll *pll = &dsi->pll;
+	struct clk *clk;
+	int r;
+
+	clk = devm_clk_get(&dsidev->dev, "sys_clk");
+	if (IS_ERR(clk)) {
+		DSSERR("can't get sys_clk\n");
+		return PTR_ERR(clk);
+	}
+
+	pll->name = dsi->module_id == 0 ? "dsi0" : "dsi1";
+	pll->id = dsi->module_id == 0 ? DSS_PLL_DSI1 : DSS_PLL_DSI2;
+	pll->clkin = clk;
+	pll->base = dsi->pll_base;
+
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_OMAP3630:
+	case OMAPDSS_VER_AM35xx:
+		pll->hw = &dss_omap3_dsi_pll_hw;
+		break;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		pll->hw = &dss_omap4_dsi_pll_hw;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+		pll->hw = &dss_omap5_dsi_pll_hw;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	pll->ops = &dsi_pll_ops;
+
+	r = dss_pll_register(pll);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+/* DSI1 HW IP initialisation */
+static int dsi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *dsidev = to_platform_device(dev);
+	u32 rev;
+	int r, i;
+	struct dsi_data *dsi;
+	struct resource *dsi_mem;
+	struct resource *res;
+	struct resource temp_res;
+
+	dsi = devm_kzalloc(&dsidev->dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+
+	dsi->pdev = dsidev;
+	dev_set_drvdata(&dsidev->dev, dsi);
+
+	spin_lock_init(&dsi->irq_lock);
+	spin_lock_init(&dsi->errors_lock);
+	dsi->errors = 0;
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	spin_lock_init(&dsi->irq_stats_lock);
+	dsi->irq_stats.last_reset = jiffies;
+#endif
+
+	mutex_init(&dsi->lock);
+	sema_init(&dsi->bus_lock, 1);
+
+	INIT_DEFERRABLE_WORK(&dsi->framedone_timeout_work,
+			     dsi_framedone_timeout_work_callback);
+
+#ifdef DSI_CATCH_MISSING_TE
+	init_timer(&dsi->te_timer);
+	dsi->te_timer.function = dsi_te_timeout;
+	dsi->te_timer.data = 0;
+#endif
+
+	res = platform_get_resource_byname(dsidev, IORESOURCE_MEM, "proto");
+	if (!res) {
+		res = platform_get_resource(dsidev, IORESOURCE_MEM, 0);
+		if (!res) {
+			DSSERR("can't get IORESOURCE_MEM DSI\n");
+			return -EINVAL;
+		}
+
+		temp_res.start = res->start;
+		temp_res.end = temp_res.start + DSI_PROTO_SZ - 1;
+		res = &temp_res;
+	}
+
+	dsi_mem = res;
+
+	dsi->proto_base = devm_ioremap(&dsidev->dev, res->start,
+		resource_size(res));
+	if (!dsi->proto_base) {
+		DSSERR("can't ioremap DSI protocol engine\n");
+		return -ENOMEM;
+	}
+
+	res = platform_get_resource_byname(dsidev, IORESOURCE_MEM, "phy");
+	if (!res) {
+		res = platform_get_resource(dsidev, IORESOURCE_MEM, 0);
+		if (!res) {
+			DSSERR("can't get IORESOURCE_MEM DSI\n");
+			return -EINVAL;
+		}
+
+		temp_res.start = res->start + DSI_PHY_OFFSET;
+		temp_res.end = temp_res.start + DSI_PHY_SZ - 1;
+		res = &temp_res;
+	}
+
+	dsi->phy_base = devm_ioremap(&dsidev->dev, res->start,
+		resource_size(res));
+	if (!dsi->proto_base) {
+		DSSERR("can't ioremap DSI PHY\n");
+		return -ENOMEM;
+	}
+
+	res = platform_get_resource_byname(dsidev, IORESOURCE_MEM, "pll");
+	if (!res) {
+		res = platform_get_resource(dsidev, IORESOURCE_MEM, 0);
+		if (!res) {
+			DSSERR("can't get IORESOURCE_MEM DSI\n");
+			return -EINVAL;
+		}
+
+		temp_res.start = res->start + DSI_PLL_OFFSET;
+		temp_res.end = temp_res.start + DSI_PLL_SZ - 1;
+		res = &temp_res;
+	}
+
+	dsi->pll_base = devm_ioremap(&dsidev->dev, res->start,
+		resource_size(res));
+	if (!dsi->proto_base) {
+		DSSERR("can't ioremap DSI PLL\n");
+		return -ENOMEM;
+	}
+
+	dsi->irq = platform_get_irq(dsi->pdev, 0);
+	if (dsi->irq < 0) {
+		DSSERR("platform_get_irq failed\n");
+		return -ENODEV;
+	}
+
+	r = devm_request_irq(&dsidev->dev, dsi->irq, omap_dsi_irq_handler,
+			     IRQF_SHARED, dev_name(&dsidev->dev), dsi->pdev);
+	if (r < 0) {
+		DSSERR("request_irq failed\n");
+		return r;
+	}
+
+	if (dsidev->dev.of_node) {
+		const struct of_device_id *match;
+		const struct dsi_module_id_data *d;
+
+		match = of_match_node(dsi_of_match, dsidev->dev.of_node);
+		if (!match) {
+			DSSERR("unsupported DSI module\n");
+			return -ENODEV;
+		}
+
+		d = match->data;
+
+		while (d->address != 0 && d->address != dsi_mem->start)
+			d++;
+
+		if (d->address == 0) {
+			DSSERR("unsupported DSI module\n");
+			return -ENODEV;
+		}
+
+		dsi->module_id = d->id;
+	} else {
+		dsi->module_id = dsidev->id;
+	}
+
+	/* DSI VCs initialization */
+	for (i = 0; i < ARRAY_SIZE(dsi->vc); i++) {
+		dsi->vc[i].source = DSI_VC_SOURCE_L4;
+		dsi->vc[i].dssdev = NULL;
+		dsi->vc[i].vc_id = 0;
+	}
+
+	r = dsi_get_clocks(dsidev);
+	if (r)
+		return r;
+
+	dsi_init_pll_data(dsidev);
+
+	pm_runtime_enable(&dsidev->dev);
+
+	r = dsi_runtime_get(dsidev);
+	if (r)
+		goto err_runtime_get;
+
+	rev = dsi_read_reg(dsidev, DSI_REVISION);
+	dev_dbg(&dsidev->dev, "OMAP DSI rev %d.%d\n",
+	       FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
+
+	/* DSI on OMAP3 doesn't have register DSI_GNQ, set number
+	 * of data to 3 by default */
+	if (dss_has_feature(FEAT_DSI_GNQ))
+		/* NB_DATA_LANES */
+		dsi->num_lanes_supported = 1 + REG_GET(dsidev, DSI_GNQ, 11, 9);
+	else
+		dsi->num_lanes_supported = 3;
+
+	dsi->line_buffer_size = dsi_get_line_buf_size(dsidev);
+
+	dsi_init_output(dsidev);
+
+	if (dsidev->dev.of_node) {
+		r = dsi_probe_of(dsidev);
+		if (r) {
+			DSSERR("Invalid DSI DT data\n");
+			goto err_probe_of;
+		}
+
+		r = of_platform_populate(dsidev->dev.of_node, NULL, NULL,
+			&dsidev->dev);
+		if (r)
+			DSSERR("Failed to populate DSI child devices: %d\n", r);
+	}
+
+	dsi_runtime_put(dsidev);
+
+	if (dsi->module_id == 0)
+		dss_debugfs_create_file("dsi1_regs", dsi1_dump_regs);
+	else if (dsi->module_id == 1)
+		dss_debugfs_create_file("dsi2_regs", dsi2_dump_regs);
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+	if (dsi->module_id == 0)
+		dss_debugfs_create_file("dsi1_irqs", dsi1_dump_irqs);
+	else if (dsi->module_id == 1)
+		dss_debugfs_create_file("dsi2_irqs", dsi2_dump_irqs);
+#endif
+
+	return 0;
+
+err_probe_of:
+	dsi_uninit_output(dsidev);
+	dsi_runtime_put(dsidev);
+
+err_runtime_get:
+	pm_runtime_disable(&dsidev->dev);
+	return r;
+}
+
+static void dsi_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *dsidev = to_platform_device(dev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
+
+	of_platform_depopulate(&dsidev->dev);
+
+	WARN_ON(dsi->scp_clk_refcount > 0);
+
+	dss_pll_unregister(&dsi->pll);
+
+	dsi_uninit_output(dsidev);
+
+	pm_runtime_disable(&dsidev->dev);
+
+	if (dsi->vdds_dsi_reg != NULL && dsi->vdds_dsi_enabled) {
+		regulator_disable(dsi->vdds_dsi_reg);
+		dsi->vdds_dsi_enabled = false;
+	}
+}
+
+static const struct component_ops dsi_component_ops = {
+	.bind	= dsi_bind,
+	.unbind	= dsi_unbind,
+};
+
+static int dsi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dsi_component_ops);
+}
+
+static int dsi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dsi_component_ops);
+	return 0;
+}
+
+static int dsi_runtime_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(pdev);
+
+	dsi->is_enabled = false;
+	/* ensure the irq handler sees the is_enabled value */
+	smp_wmb();
+	/* wait for current handler to finish before turning the DSI off */
+	synchronize_irq(dsi->irq);
+
+	dispc_runtime_put();
+
+	return 0;
+}
+
+static int dsi_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dsi_data *dsi = dsi_get_dsidrv_data(pdev);
+	int r;
+
+	r = dispc_runtime_get();
+	if (r)
+		return r;
+
+	dsi->is_enabled = true;
+	/* ensure the irq handler sees the is_enabled value */
+	smp_wmb();
+
+	return 0;
+}
+
+static const struct dev_pm_ops dsi_pm_ops = {
+	.runtime_suspend = dsi_runtime_suspend,
+	.runtime_resume = dsi_runtime_resume,
+};
+
+static const struct dsi_module_id_data dsi_of_data_omap3[] = {
+	{ .address = 0x4804fc00, .id = 0, },
+	{ },
+};
+
+static const struct dsi_module_id_data dsi_of_data_omap4[] = {
+	{ .address = 0x58004000, .id = 0, },
+	{ .address = 0x58005000, .id = 1, },
+	{ },
+};
+
+static const struct dsi_module_id_data dsi_of_data_omap5[] = {
+	{ .address = 0x58004000, .id = 0, },
+	{ .address = 0x58009000, .id = 1, },
+	{ },
+};
+
+static const struct of_device_id dsi_of_match[] = {
+	{ .compatible = "ti,omap3-dsi", .data = dsi_of_data_omap3, },
+	{ .compatible = "ti,omap4-dsi", .data = dsi_of_data_omap4, },
+	{ .compatible = "ti,omap5-dsi", .data = dsi_of_data_omap5, },
+	{},
+};
+
+static struct platform_driver omap_dsihw_driver = {
+	.probe		= dsi_probe,
+	.remove		= dsi_remove,
+	.driver         = {
+		.name   = "omapdss_dsi",
+		.pm	= &dsi_pm_ops,
+		.of_match_table = dsi_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init dsi_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_dsihw_driver);
+}
+
+void dsi_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_dsihw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
new file mode 100644
index 000000000000..bf407b6ba15c
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2013 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/seq_file.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+
+struct device_node *
+omapdss_of_get_next_port(const struct device_node *parent,
+			 struct device_node *prev)
+{
+	struct device_node *port = NULL;
+
+	if (!parent)
+		return NULL;
+
+	if (!prev) {
+		struct device_node *ports;
+		/*
+		 * It's the first call, we have to find a port subnode
+		 * within this node or within an optional 'ports' node.
+		 */
+		ports = of_get_child_by_name(parent, "ports");
+		if (ports)
+			parent = ports;
+
+		port = of_get_child_by_name(parent, "port");
+
+		/* release the 'ports' node */
+		of_node_put(ports);
+	} else {
+		struct device_node *ports;
+
+		ports = of_get_parent(prev);
+		if (!ports)
+			return NULL;
+
+		do {
+			port = of_get_next_child(ports, prev);
+			if (!port) {
+				of_node_put(ports);
+				return NULL;
+			}
+			prev = port;
+		} while (of_node_cmp(port->name, "port") != 0);
+
+		of_node_put(ports);
+	}
+
+	return port;
+}
+EXPORT_SYMBOL_GPL(omapdss_of_get_next_port);
+
+struct device_node *
+omapdss_of_get_next_endpoint(const struct device_node *parent,
+			     struct device_node *prev)
+{
+	struct device_node *ep = NULL;
+
+	if (!parent)
+		return NULL;
+
+	do {
+		ep = of_get_next_child(parent, prev);
+		if (!ep)
+			return NULL;
+		prev = ep;
+	} while (of_node_cmp(ep->name, "endpoint") != 0);
+
+	return ep;
+}
+EXPORT_SYMBOL_GPL(omapdss_of_get_next_endpoint);
+
+struct device_node *dss_of_port_get_parent_device(struct device_node *port)
+{
+	struct device_node *np;
+	int i;
+
+	if (!port)
+		return NULL;
+
+	np = of_get_parent(port);
+
+	for (i = 0; i < 2 && np; ++i) {
+		struct property *prop;
+
+		prop = of_find_property(np, "compatible", NULL);
+
+		if (prop)
+			return np;
+
+		np = of_get_next_parent(np);
+	}
+
+	return NULL;
+}
+
+u32 dss_of_port_get_port_number(struct device_node *port)
+{
+	int r;
+	u32 reg;
+
+	r = of_property_read_u32(port, "reg", &reg);
+	if (r)
+		reg = 0;
+
+	return reg;
+}
+
+static struct device_node *omapdss_of_get_remote_port(const struct device_node *node)
+{
+	struct device_node *np;
+
+	np = of_parse_phandle(node, "remote-endpoint", 0);
+	if (!np)
+		return NULL;
+
+	np = of_get_next_parent(np);
+
+	return np;
+}
+
+struct device_node *
+omapdss_of_get_first_endpoint(const struct device_node *parent)
+{
+	struct device_node *port, *ep;
+
+	port = omapdss_of_get_next_port(parent, NULL);
+
+	if (!port)
+		return NULL;
+
+	ep = omapdss_of_get_next_endpoint(port, NULL);
+
+	of_node_put(port);
+
+	return ep;
+}
+EXPORT_SYMBOL_GPL(omapdss_of_get_first_endpoint);
+
+struct omap_dss_device *
+omapdss_of_find_source_for_first_ep(struct device_node *node)
+{
+	struct device_node *ep;
+	struct device_node *src_port;
+	struct omap_dss_device *src;
+
+	ep = omapdss_of_get_first_endpoint(node);
+	if (!ep)
+		return ERR_PTR(-EINVAL);
+
+	src_port = omapdss_of_get_remote_port(ep);
+	if (!src_port) {
+		of_node_put(ep);
+		return ERR_PTR(-EINVAL);
+	}
+
+	of_node_put(ep);
+
+	src = omap_dss_find_output_by_port_node(src_port);
+
+	of_node_put(src_port);
+
+	return src ? src : ERR_PTR(-EPROBE_DEFER);
+}
+EXPORT_SYMBOL_GPL(omapdss_of_find_source_for_first_ep);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.c b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
new file mode 100644
index 000000000000..0078c4d1fc31
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
@@ -0,0 +1,1329 @@
+/*
+ * linux/drivers/video/omap2/dss/dss.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "DSS"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/export.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/seq_file.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/gfp.h>
+#include <linux/sizes.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+#include <linux/suspend.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+#define DSS_SZ_REGS			SZ_512
+
+struct dss_reg {
+	u16 idx;
+};
+
+#define DSS_REG(idx)			((const struct dss_reg) { idx })
+
+#define DSS_REVISION			DSS_REG(0x0000)
+#define DSS_SYSCONFIG			DSS_REG(0x0010)
+#define DSS_SYSSTATUS			DSS_REG(0x0014)
+#define DSS_CONTROL			DSS_REG(0x0040)
+#define DSS_SDI_CONTROL			DSS_REG(0x0044)
+#define DSS_PLL_CONTROL			DSS_REG(0x0048)
+#define DSS_SDI_STATUS			DSS_REG(0x005C)
+
+#define REG_GET(idx, start, end) \
+	FLD_GET(dss_read_reg(idx), start, end)
+
+#define REG_FLD_MOD(idx, val, start, end) \
+	dss_write_reg(idx, FLD_MOD(dss_read_reg(idx), val, start, end))
+
+struct dss_features {
+	u8 fck_div_max;
+	u8 dss_fck_multiplier;
+	const char *parent_clk_name;
+	const enum omap_display_type *ports;
+	int num_ports;
+	int (*dpi_select_source)(int port, enum omap_channel channel);
+};
+
+static struct {
+	struct platform_device *pdev;
+	void __iomem    *base;
+	struct regmap	*syscon_pll_ctrl;
+	u32		syscon_pll_ctrl_offset;
+
+	struct clk	*parent_clk;
+	struct clk	*dss_clk;
+	unsigned long	dss_clk_rate;
+
+	unsigned long	cache_req_pck;
+	unsigned long	cache_prate;
+	struct dispc_clock_info cache_dispc_cinfo;
+
+	enum omap_dss_clk_source dsi_clk_source[MAX_NUM_DSI];
+	enum omap_dss_clk_source dispc_clk_source;
+	enum omap_dss_clk_source lcd_clk_source[MAX_DSS_LCD_MANAGERS];
+
+	bool		ctx_valid;
+	u32		ctx[DSS_SZ_REGS / sizeof(u32)];
+
+	const struct dss_features *feat;
+
+	struct dss_pll	*video1_pll;
+	struct dss_pll	*video2_pll;
+} dss;
+
+static const char * const dss_generic_clk_source_names[] = {
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]	= "DSI_PLL_HSDIV_DISPC",
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]	= "DSI_PLL_HSDIV_DSI",
+	[OMAP_DSS_CLK_SRC_FCK]			= "DSS_FCK",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC]	= "DSI_PLL2_HSDIV_DISPC",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]	= "DSI_PLL2_HSDIV_DSI",
+};
+
+static bool dss_initialized;
+
+bool omapdss_is_initialized(void)
+{
+	return dss_initialized;
+}
+EXPORT_SYMBOL(omapdss_is_initialized);
+
+static inline void dss_write_reg(const struct dss_reg idx, u32 val)
+{
+	__raw_writel(val, dss.base + idx.idx);
+}
+
+static inline u32 dss_read_reg(const struct dss_reg idx)
+{
+	return __raw_readl(dss.base + idx.idx);
+}
+
+#define SR(reg) \
+	dss.ctx[(DSS_##reg).idx / sizeof(u32)] = dss_read_reg(DSS_##reg)
+#define RR(reg) \
+	dss_write_reg(DSS_##reg, dss.ctx[(DSS_##reg).idx / sizeof(u32)])
+
+static void dss_save_context(void)
+{
+	DSSDBG("dss_save_context\n");
+
+	SR(CONTROL);
+
+	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
+			OMAP_DISPLAY_TYPE_SDI) {
+		SR(SDI_CONTROL);
+		SR(PLL_CONTROL);
+	}
+
+	dss.ctx_valid = true;
+
+	DSSDBG("context saved\n");
+}
+
+static void dss_restore_context(void)
+{
+	DSSDBG("dss_restore_context\n");
+
+	if (!dss.ctx_valid)
+		return;
+
+	RR(CONTROL);
+
+	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
+			OMAP_DISPLAY_TYPE_SDI) {
+		RR(SDI_CONTROL);
+		RR(PLL_CONTROL);
+	}
+
+	DSSDBG("context restored\n");
+}
+
+#undef SR
+#undef RR
+
+void dss_ctrl_pll_enable(enum dss_pll_id pll_id, bool enable)
+{
+	unsigned shift;
+	unsigned val;
+
+	if (!dss.syscon_pll_ctrl)
+		return;
+
+	val = !enable;
+
+	switch (pll_id) {
+	case DSS_PLL_VIDEO1:
+		shift = 0;
+		break;
+	case DSS_PLL_VIDEO2:
+		shift = 1;
+		break;
+	case DSS_PLL_HDMI:
+		shift = 2;
+		break;
+	default:
+		DSSERR("illegal DSS PLL ID %d\n", pll_id);
+		return;
+	}
+
+	regmap_update_bits(dss.syscon_pll_ctrl, dss.syscon_pll_ctrl_offset,
+		1 << shift, val << shift);
+}
+
+void dss_ctrl_pll_set_control_mux(enum dss_pll_id pll_id,
+	enum omap_channel channel)
+{
+	unsigned shift, val;
+
+	if (!dss.syscon_pll_ctrl)
+		return;
+
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		shift = 3;
+
+		switch (pll_id) {
+		case DSS_PLL_VIDEO1:
+			val = 0; break;
+		case DSS_PLL_HDMI:
+			val = 1; break;
+		default:
+			DSSERR("error in PLL mux config for LCD\n");
+			return;
+		}
+
+		break;
+	case OMAP_DSS_CHANNEL_LCD2:
+		shift = 5;
+
+		switch (pll_id) {
+		case DSS_PLL_VIDEO1:
+			val = 0; break;
+		case DSS_PLL_VIDEO2:
+			val = 1; break;
+		case DSS_PLL_HDMI:
+			val = 2; break;
+		default:
+			DSSERR("error in PLL mux config for LCD2\n");
+			return;
+		}
+
+		break;
+	case OMAP_DSS_CHANNEL_LCD3:
+		shift = 7;
+
+		switch (pll_id) {
+		case DSS_PLL_VIDEO1:
+			val = 1; break;
+		case DSS_PLL_VIDEO2:
+			val = 0; break;
+		case DSS_PLL_HDMI:
+			val = 2; break;
+		default:
+			DSSERR("error in PLL mux config for LCD3\n");
+			return;
+		}
+
+		break;
+	default:
+		DSSERR("error in PLL mux config\n");
+		return;
+	}
+
+	regmap_update_bits(dss.syscon_pll_ctrl, dss.syscon_pll_ctrl_offset,
+		0x3 << shift, val << shift);
+}
+
+void dss_sdi_init(int datapairs)
+{
+	u32 l;
+
+	BUG_ON(datapairs > 3 || datapairs < 1);
+
+	l = dss_read_reg(DSS_SDI_CONTROL);
+	l = FLD_MOD(l, 0xf, 19, 15);		/* SDI_PDIV */
+	l = FLD_MOD(l, datapairs-1, 3, 2);	/* SDI_PRSEL */
+	l = FLD_MOD(l, 2, 1, 0);		/* SDI_BWSEL */
+	dss_write_reg(DSS_SDI_CONTROL, l);
+
+	l = dss_read_reg(DSS_PLL_CONTROL);
+	l = FLD_MOD(l, 0x7, 25, 22);	/* SDI_PLL_FREQSEL */
+	l = FLD_MOD(l, 0xb, 16, 11);	/* SDI_PLL_REGN */
+	l = FLD_MOD(l, 0xb4, 10, 1);	/* SDI_PLL_REGM */
+	dss_write_reg(DSS_PLL_CONTROL, l);
+}
+
+int dss_sdi_enable(void)
+{
+	unsigned long timeout;
+
+	dispc_pck_free_enable(1);
+
+	/* Reset SDI PLL */
+	REG_FLD_MOD(DSS_PLL_CONTROL, 1, 18, 18); /* SDI_PLL_SYSRESET */
+	udelay(1);	/* wait 2x PCLK */
+
+	/* Lock SDI PLL */
+	REG_FLD_MOD(DSS_PLL_CONTROL, 1, 28, 28); /* SDI_PLL_GOBIT */
+
+	/* Waiting for PLL lock request to complete */
+	timeout = jiffies + msecs_to_jiffies(500);
+	while (dss_read_reg(DSS_SDI_STATUS) & (1 << 6)) {
+		if (time_after_eq(jiffies, timeout)) {
+			DSSERR("PLL lock request timed out\n");
+			goto err1;
+		}
+	}
+
+	/* Clearing PLL_GO bit */
+	REG_FLD_MOD(DSS_PLL_CONTROL, 0, 28, 28);
+
+	/* Waiting for PLL to lock */
+	timeout = jiffies + msecs_to_jiffies(500);
+	while (!(dss_read_reg(DSS_SDI_STATUS) & (1 << 5))) {
+		if (time_after_eq(jiffies, timeout)) {
+			DSSERR("PLL lock timed out\n");
+			goto err1;
+		}
+	}
+
+	dispc_lcd_enable_signal(1);
+
+	/* Waiting for SDI reset to complete */
+	timeout = jiffies + msecs_to_jiffies(500);
+	while (!(dss_read_reg(DSS_SDI_STATUS) & (1 << 2))) {
+		if (time_after_eq(jiffies, timeout)) {
+			DSSERR("SDI reset timed out\n");
+			goto err2;
+		}
+	}
+
+	return 0;
+
+ err2:
+	dispc_lcd_enable_signal(0);
+ err1:
+	/* Reset SDI PLL */
+	REG_FLD_MOD(DSS_PLL_CONTROL, 0, 18, 18); /* SDI_PLL_SYSRESET */
+
+	dispc_pck_free_enable(0);
+
+	return -ETIMEDOUT;
+}
+
+void dss_sdi_disable(void)
+{
+	dispc_lcd_enable_signal(0);
+
+	dispc_pck_free_enable(0);
+
+	/* Reset SDI PLL */
+	REG_FLD_MOD(DSS_PLL_CONTROL, 0, 18, 18); /* SDI_PLL_SYSRESET */
+}
+
+const char *dss_get_generic_clk_source_name(enum omap_dss_clk_source clk_src)
+{
+	return dss_generic_clk_source_names[clk_src];
+}
+
+void dss_dump_clocks(struct seq_file *s)
+{
+	const char *fclk_name, *fclk_real_name;
+	unsigned long fclk_rate;
+
+	if (dss_runtime_get())
+		return;
+
+	seq_printf(s, "- DSS -\n");
+
+	fclk_name = dss_get_generic_clk_source_name(OMAP_DSS_CLK_SRC_FCK);
+	fclk_real_name = dss_feat_get_clk_source_name(OMAP_DSS_CLK_SRC_FCK);
+	fclk_rate = clk_get_rate(dss.dss_clk);
+
+	seq_printf(s, "%s (%s) = %lu\n",
+			fclk_name, fclk_real_name,
+			fclk_rate);
+
+	dss_runtime_put();
+}
+
+static void dss_dump_regs(struct seq_file *s)
+{
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, dss_read_reg(r))
+
+	if (dss_runtime_get())
+		return;
+
+	DUMPREG(DSS_REVISION);
+	DUMPREG(DSS_SYSCONFIG);
+	DUMPREG(DSS_SYSSTATUS);
+	DUMPREG(DSS_CONTROL);
+
+	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
+			OMAP_DISPLAY_TYPE_SDI) {
+		DUMPREG(DSS_SDI_CONTROL);
+		DUMPREG(DSS_PLL_CONTROL);
+		DUMPREG(DSS_SDI_STATUS);
+	}
+
+	dss_runtime_put();
+#undef DUMPREG
+}
+
+static void dss_select_dispc_clk_source(enum omap_dss_clk_source clk_src)
+{
+	int b;
+	u8 start, end;
+
+	switch (clk_src) {
+	case OMAP_DSS_CLK_SRC_FCK:
+		b = 0;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
+		b = 1;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
+		b = 2;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	dss_feat_get_reg_field(FEAT_REG_DISPC_CLK_SWITCH, &start, &end);
+
+	REG_FLD_MOD(DSS_CONTROL, b, start, end);	/* DISPC_CLK_SWITCH */
+
+	dss.dispc_clk_source = clk_src;
+}
+
+void dss_select_dsi_clk_source(int dsi_module,
+		enum omap_dss_clk_source clk_src)
+{
+	int b, pos;
+
+	switch (clk_src) {
+	case OMAP_DSS_CLK_SRC_FCK:
+		b = 0;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI:
+		BUG_ON(dsi_module != 0);
+		b = 1;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI:
+		BUG_ON(dsi_module != 1);
+		b = 1;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	pos = dsi_module == 0 ? 1 : 10;
+	REG_FLD_MOD(DSS_CONTROL, b, pos, pos);	/* DSIx_CLK_SWITCH */
+
+	dss.dsi_clk_source[dsi_module] = clk_src;
+}
+
+void dss_select_lcd_clk_source(enum omap_channel channel,
+		enum omap_dss_clk_source clk_src)
+{
+	int b, ix, pos;
+
+	if (!dss_has_feature(FEAT_LCD_CLK_SRC)) {
+		dss_select_dispc_clk_source(clk_src);
+		return;
+	}
+
+	switch (clk_src) {
+	case OMAP_DSS_CLK_SRC_FCK:
+		b = 0;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
+		BUG_ON(channel != OMAP_DSS_CHANNEL_LCD);
+		b = 1;
+		break;
+	case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
+		BUG_ON(channel != OMAP_DSS_CHANNEL_LCD2 &&
+		       channel != OMAP_DSS_CHANNEL_LCD3);
+		b = 1;
+		break;
+	default:
+		BUG();
+		return;
+	}
+
+	pos = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
+	     (channel == OMAP_DSS_CHANNEL_LCD2 ? 12 : 19);
+	REG_FLD_MOD(DSS_CONTROL, b, pos, pos);	/* LCDx_CLK_SWITCH */
+
+	ix = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
+	    (channel == OMAP_DSS_CHANNEL_LCD2 ? 1 : 2);
+	dss.lcd_clk_source[ix] = clk_src;
+}
+
+enum omap_dss_clk_source dss_get_dispc_clk_source(void)
+{
+	return dss.dispc_clk_source;
+}
+
+enum omap_dss_clk_source dss_get_dsi_clk_source(int dsi_module)
+{
+	return dss.dsi_clk_source[dsi_module];
+}
+
+enum omap_dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel)
+{
+	if (dss_has_feature(FEAT_LCD_CLK_SRC)) {
+		int ix = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
+			(channel == OMAP_DSS_CHANNEL_LCD2 ? 1 : 2);
+		return dss.lcd_clk_source[ix];
+	} else {
+		/* LCD_CLK source is the same as DISPC_FCLK source for
+		 * OMAP2 and OMAP3 */
+		return dss.dispc_clk_source;
+	}
+}
+
+bool dss_div_calc(unsigned long pck, unsigned long fck_min,
+		dss_div_calc_func func, void *data)
+{
+	int fckd, fckd_start, fckd_stop;
+	unsigned long fck;
+	unsigned long fck_hw_max;
+	unsigned long fckd_hw_max;
+	unsigned long prate;
+	unsigned m;
+
+	fck_hw_max = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+
+	if (dss.parent_clk == NULL) {
+		unsigned pckd;
+
+		pckd = fck_hw_max / pck;
+
+		fck = pck * pckd;
+
+		fck = clk_round_rate(dss.dss_clk, fck);
+
+		return func(fck, data);
+	}
+
+	fckd_hw_max = dss.feat->fck_div_max;
+
+	m = dss.feat->dss_fck_multiplier;
+	prate = clk_get_rate(dss.parent_clk);
+
+	fck_min = fck_min ? fck_min : 1;
+
+	fckd_start = min(prate * m / fck_min, fckd_hw_max);
+	fckd_stop = max(DIV_ROUND_UP(prate * m, fck_hw_max), 1ul);
+
+	for (fckd = fckd_start; fckd >= fckd_stop; --fckd) {
+		fck = DIV_ROUND_UP(prate, fckd) * m;
+
+		if (func(fck, data))
+			return true;
+	}
+
+	return false;
+}
+
+int dss_set_fck_rate(unsigned long rate)
+{
+	int r;
+
+	DSSDBG("set fck to %lu\n", rate);
+
+	r = clk_set_rate(dss.dss_clk, rate);
+	if (r)
+		return r;
+
+	dss.dss_clk_rate = clk_get_rate(dss.dss_clk);
+
+	WARN_ONCE(dss.dss_clk_rate != rate,
+			"clk rate mismatch: %lu != %lu", dss.dss_clk_rate,
+			rate);
+
+	return 0;
+}
+
+unsigned long dss_get_dispc_clk_rate(void)
+{
+	return dss.dss_clk_rate;
+}
+
+static int dss_setup_default_clock(void)
+{
+	unsigned long max_dss_fck, prate;
+	unsigned long fck;
+	unsigned fck_div;
+	int r;
+
+	max_dss_fck = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+
+	if (dss.parent_clk == NULL) {
+		fck = clk_round_rate(dss.dss_clk, max_dss_fck);
+	} else {
+		prate = clk_get_rate(dss.parent_clk);
+
+		fck_div = DIV_ROUND_UP(prate * dss.feat->dss_fck_multiplier,
+				max_dss_fck);
+		fck = DIV_ROUND_UP(prate, fck_div) * dss.feat->dss_fck_multiplier;
+	}
+
+	r = dss_set_fck_rate(fck);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+void dss_set_venc_output(enum omap_dss_venc_type type)
+{
+	int l = 0;
+
+	if (type == OMAP_DSS_VENC_TYPE_COMPOSITE)
+		l = 0;
+	else if (type == OMAP_DSS_VENC_TYPE_SVIDEO)
+		l = 1;
+	else
+		BUG();
+
+	/* venc out selection. 0 = comp, 1 = svideo */
+	REG_FLD_MOD(DSS_CONTROL, l, 6, 6);
+}
+
+void dss_set_dac_pwrdn_bgz(bool enable)
+{
+	REG_FLD_MOD(DSS_CONTROL, enable, 5, 5);	/* DAC Power-Down Control */
+}
+
+void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select src)
+{
+	enum omap_display_type dp;
+	dp = dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_DIGIT);
+
+	/* Complain about invalid selections */
+	WARN_ON((src == DSS_VENC_TV_CLK) && !(dp & OMAP_DISPLAY_TYPE_VENC));
+	WARN_ON((src == DSS_HDMI_M_PCLK) && !(dp & OMAP_DISPLAY_TYPE_HDMI));
+
+	/* Select only if we have options */
+	if ((dp & OMAP_DISPLAY_TYPE_VENC) && (dp & OMAP_DISPLAY_TYPE_HDMI))
+		REG_FLD_MOD(DSS_CONTROL, src, 15, 15);	/* VENC_HDMI_SWITCH */
+}
+
+enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void)
+{
+	enum omap_display_type displays;
+
+	displays = dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_DIGIT);
+	if ((displays & OMAP_DISPLAY_TYPE_HDMI) == 0)
+		return DSS_VENC_TV_CLK;
+
+	if ((displays & OMAP_DISPLAY_TYPE_VENC) == 0)
+		return DSS_HDMI_M_PCLK;
+
+	return REG_GET(DSS_CONTROL, 15, 15);
+}
+
+static int dss_dpi_select_source_omap2_omap3(int port, enum omap_channel channel)
+{
+	if (channel != OMAP_DSS_CHANNEL_LCD)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int dss_dpi_select_source_omap4(int port, enum omap_channel channel)
+{
+	int val;
+
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD2:
+		val = 0;
+		break;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		val = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	REG_FLD_MOD(DSS_CONTROL, val, 17, 17);
+
+	return 0;
+}
+
+static int dss_dpi_select_source_omap5(int port, enum omap_channel channel)
+{
+	int val;
+
+	switch (channel) {
+	case OMAP_DSS_CHANNEL_LCD:
+		val = 1;
+		break;
+	case OMAP_DSS_CHANNEL_LCD2:
+		val = 2;
+		break;
+	case OMAP_DSS_CHANNEL_LCD3:
+		val = 3;
+		break;
+	case OMAP_DSS_CHANNEL_DIGIT:
+		val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	REG_FLD_MOD(DSS_CONTROL, val, 17, 16);
+
+	return 0;
+}
+
+static int dss_dpi_select_source_dra7xx(int port, enum omap_channel channel)
+{
+	switch (port) {
+	case 0:
+		return dss_dpi_select_source_omap5(port, channel);
+	case 1:
+		if (channel != OMAP_DSS_CHANNEL_LCD2)
+			return -EINVAL;
+		break;
+	case 2:
+		if (channel != OMAP_DSS_CHANNEL_LCD3)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int dss_dpi_select_source(int port, enum omap_channel channel)
+{
+	return dss.feat->dpi_select_source(port, channel);
+}
+
+static int dss_get_clocks(void)
+{
+	struct clk *clk;
+
+	clk = devm_clk_get(&dss.pdev->dev, "fck");
+	if (IS_ERR(clk)) {
+		DSSERR("can't get clock fck\n");
+		return PTR_ERR(clk);
+	}
+
+	dss.dss_clk = clk;
+
+	if (dss.feat->parent_clk_name) {
+		clk = clk_get(NULL, dss.feat->parent_clk_name);
+		if (IS_ERR(clk)) {
+			DSSERR("Failed to get %s\n", dss.feat->parent_clk_name);
+			return PTR_ERR(clk);
+		}
+	} else {
+		clk = NULL;
+	}
+
+	dss.parent_clk = clk;
+
+	return 0;
+}
+
+static void dss_put_clocks(void)
+{
+	if (dss.parent_clk)
+		clk_put(dss.parent_clk);
+}
+
+int dss_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("dss_runtime_get\n");
+
+	r = pm_runtime_get_sync(&dss.pdev->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+void dss_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("dss_runtime_put\n");
+
+	r = pm_runtime_put_sync(&dss.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS && r != -EBUSY);
+}
+
+/* DEBUGFS */
+#if defined(CONFIG_FB_OMAP2_DSS_DEBUGFS)
+void dss_debug_dump_clocks(struct seq_file *s)
+{
+	dss_dump_clocks(s);
+	dispc_dump_clocks(s);
+#ifdef CONFIG_FB_OMAP2_DSS_DSI
+	dsi_dump_clocks(s);
+#endif
+}
+#endif
+
+
+static const enum omap_display_type omap2plus_ports[] = {
+	OMAP_DISPLAY_TYPE_DPI,
+};
+
+static const enum omap_display_type omap34xx_ports[] = {
+	OMAP_DISPLAY_TYPE_DPI,
+	OMAP_DISPLAY_TYPE_SDI,
+};
+
+static const enum omap_display_type dra7xx_ports[] = {
+	OMAP_DISPLAY_TYPE_DPI,
+	OMAP_DISPLAY_TYPE_DPI,
+	OMAP_DISPLAY_TYPE_DPI,
+};
+
+static const struct dss_features omap24xx_dss_feats = {
+	/*
+	 * fck div max is really 16, but the divider range has gaps. The range
+	 * from 1 to 6 has no gaps, so let's use that as a max.
+	 */
+	.fck_div_max		=	6,
+	.dss_fck_multiplier	=	2,
+	.parent_clk_name	=	"core_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
+	.ports			=	omap2plus_ports,
+	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+};
+
+static const struct dss_features omap34xx_dss_feats = {
+	.fck_div_max		=	16,
+	.dss_fck_multiplier	=	2,
+	.parent_clk_name	=	"dpll4_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
+	.ports			=	omap34xx_ports,
+	.num_ports		=	ARRAY_SIZE(omap34xx_ports),
+};
+
+static const struct dss_features omap3630_dss_feats = {
+	.fck_div_max		=	32,
+	.dss_fck_multiplier	=	1,
+	.parent_clk_name	=	"dpll4_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
+	.ports			=	omap2plus_ports,
+	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+};
+
+static const struct dss_features omap44xx_dss_feats = {
+	.fck_div_max		=	32,
+	.dss_fck_multiplier	=	1,
+	.parent_clk_name	=	"dpll_per_x2_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_omap4,
+	.ports			=	omap2plus_ports,
+	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+};
+
+static const struct dss_features omap54xx_dss_feats = {
+	.fck_div_max		=	64,
+	.dss_fck_multiplier	=	1,
+	.parent_clk_name	=	"dpll_per_x2_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_omap5,
+	.ports			=	omap2plus_ports,
+	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+};
+
+static const struct dss_features am43xx_dss_feats = {
+	.fck_div_max		=	0,
+	.dss_fck_multiplier	=	0,
+	.parent_clk_name	=	NULL,
+	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
+	.ports			=	omap2plus_ports,
+	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+};
+
+static const struct dss_features dra7xx_dss_feats = {
+	.fck_div_max		=	64,
+	.dss_fck_multiplier	=	1,
+	.parent_clk_name	=	"dpll_per_x2_ck",
+	.dpi_select_source	=	&dss_dpi_select_source_dra7xx,
+	.ports			=	dra7xx_ports,
+	.num_ports		=	ARRAY_SIZE(dra7xx_ports),
+};
+
+static int dss_init_features(struct platform_device *pdev)
+{
+	const struct dss_features *src;
+	struct dss_features *dst;
+
+	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
+	if (!dst) {
+		dev_err(&pdev->dev, "Failed to allocate local DSS Features\n");
+		return -ENOMEM;
+	}
+
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP24xx:
+		src = &omap24xx_dss_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+	case OMAPDSS_VER_AM35xx:
+		src = &omap34xx_dss_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP3630:
+		src = &omap3630_dss_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		src = &omap44xx_dss_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+		src = &omap54xx_dss_feats;
+		break;
+
+	case OMAPDSS_VER_AM43xx:
+		src = &am43xx_dss_feats;
+		break;
+
+	case OMAPDSS_VER_DRA7xx:
+		src = &dra7xx_dss_feats;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	memcpy(dst, src, sizeof(*dst));
+	dss.feat = dst;
+
+	return 0;
+}
+
+static int dss_init_ports(struct platform_device *pdev)
+{
+	struct device_node *parent = pdev->dev.of_node;
+	struct device_node *port;
+	int r;
+
+	if (parent == NULL)
+		return 0;
+
+	port = omapdss_of_get_next_port(parent, NULL);
+	if (!port)
+		return 0;
+
+	if (dss.feat->num_ports == 0)
+		return 0;
+
+	do {
+		enum omap_display_type port_type;
+		u32 reg;
+
+		r = of_property_read_u32(port, "reg", &reg);
+		if (r)
+			reg = 0;
+
+		if (reg >= dss.feat->num_ports)
+			continue;
+
+		port_type = dss.feat->ports[reg];
+
+		switch (port_type) {
+		case OMAP_DISPLAY_TYPE_DPI:
+			dpi_init_port(pdev, port);
+			break;
+		case OMAP_DISPLAY_TYPE_SDI:
+			sdi_init_port(pdev, port);
+			break;
+		default:
+			break;
+		}
+	} while ((port = omapdss_of_get_next_port(parent, port)) != NULL);
+
+	return 0;
+}
+
+static void dss_uninit_ports(struct platform_device *pdev)
+{
+	struct device_node *parent = pdev->dev.of_node;
+	struct device_node *port;
+
+	if (parent == NULL)
+		return;
+
+	port = omapdss_of_get_next_port(parent, NULL);
+	if (!port)
+		return;
+
+	if (dss.feat->num_ports == 0)
+		return;
+
+	do {
+		enum omap_display_type port_type;
+		u32 reg;
+		int r;
+
+		r = of_property_read_u32(port, "reg", &reg);
+		if (r)
+			reg = 0;
+
+		if (reg >= dss.feat->num_ports)
+			continue;
+
+		port_type = dss.feat->ports[reg];
+
+		switch (port_type) {
+		case OMAP_DISPLAY_TYPE_DPI:
+			dpi_uninit_port(port);
+			break;
+		case OMAP_DISPLAY_TYPE_SDI:
+			sdi_uninit_port(port);
+			break;
+		default:
+			break;
+		}
+	} while ((port = omapdss_of_get_next_port(parent, port)) != NULL);
+}
+
+static int dss_video_pll_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct regulator *pll_regulator;
+	int r;
+
+	if (!np)
+		return 0;
+
+	if (of_property_read_bool(np, "syscon-pll-ctrl")) {
+		dss.syscon_pll_ctrl = syscon_regmap_lookup_by_phandle(np,
+			"syscon-pll-ctrl");
+		if (IS_ERR(dss.syscon_pll_ctrl)) {
+			dev_err(&pdev->dev,
+				"failed to get syscon-pll-ctrl regmap\n");
+			return PTR_ERR(dss.syscon_pll_ctrl);
+		}
+
+		if (of_property_read_u32_index(np, "syscon-pll-ctrl", 1,
+				&dss.syscon_pll_ctrl_offset)) {
+			dev_err(&pdev->dev,
+				"failed to get syscon-pll-ctrl offset\n");
+			return -EINVAL;
+		}
+	}
+
+	pll_regulator = devm_regulator_get(&pdev->dev, "vdda_video");
+	if (IS_ERR(pll_regulator)) {
+		r = PTR_ERR(pll_regulator);
+
+		switch (r) {
+		case -ENOENT:
+			pll_regulator = NULL;
+			break;
+
+		case -EPROBE_DEFER:
+			return -EPROBE_DEFER;
+
+		default:
+			DSSERR("can't get DPLL VDDA regulator\n");
+			return r;
+		}
+	}
+
+	if (of_property_match_string(np, "reg-names", "pll1") >= 0) {
+		dss.video1_pll = dss_video_pll_init(pdev, 0, pll_regulator);
+		if (IS_ERR(dss.video1_pll))
+			return PTR_ERR(dss.video1_pll);
+	}
+
+	if (of_property_match_string(np, "reg-names", "pll2") >= 0) {
+		dss.video2_pll = dss_video_pll_init(pdev, 1, pll_regulator);
+		if (IS_ERR(dss.video2_pll)) {
+			dss_video_pll_uninit(dss.video1_pll);
+			return PTR_ERR(dss.video2_pll);
+		}
+	}
+
+	return 0;
+}
+
+/* DSS HW IP initialisation */
+static int dss_bind(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *dss_mem;
+	u32 rev;
+	int r;
+
+	dss.pdev = pdev;
+
+	r = dss_init_features(dss.pdev);
+	if (r)
+		return r;
+
+	dss_mem = platform_get_resource(dss.pdev, IORESOURCE_MEM, 0);
+	if (!dss_mem) {
+		DSSERR("can't get IORESOURCE_MEM DSS\n");
+		return -EINVAL;
+	}
+
+	dss.base = devm_ioremap(&pdev->dev, dss_mem->start,
+				resource_size(dss_mem));
+	if (!dss.base) {
+		DSSERR("can't ioremap DSS\n");
+		return -ENOMEM;
+	}
+
+	r = dss_get_clocks();
+	if (r)
+		return r;
+
+	r = dss_setup_default_clock();
+	if (r)
+		goto err_setup_clocks;
+
+	r = dss_video_pll_probe(pdev);
+	if (r)
+		goto err_pll_init;
+
+	r = dss_init_ports(pdev);
+	if (r)
+		goto err_init_ports;
+
+	pm_runtime_enable(&pdev->dev);
+
+	r = dss_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	dss.dss_clk_rate = clk_get_rate(dss.dss_clk);
+
+	/* Select DPLL */
+	REG_FLD_MOD(DSS_CONTROL, 0, 0, 0);
+
+	dss_select_dispc_clk_source(OMAP_DSS_CLK_SRC_FCK);
+
+#ifdef CONFIG_FB_OMAP2_DSS_VENC
+	REG_FLD_MOD(DSS_CONTROL, 1, 4, 4);	/* venc dac demen */
+	REG_FLD_MOD(DSS_CONTROL, 1, 3, 3);	/* venc clock 4x enable */
+	REG_FLD_MOD(DSS_CONTROL, 0, 2, 2);	/* venc clock mode = normal */
+#endif
+	dss.dsi_clk_source[0] = OMAP_DSS_CLK_SRC_FCK;
+	dss.dsi_clk_source[1] = OMAP_DSS_CLK_SRC_FCK;
+	dss.dispc_clk_source = OMAP_DSS_CLK_SRC_FCK;
+	dss.lcd_clk_source[0] = OMAP_DSS_CLK_SRC_FCK;
+	dss.lcd_clk_source[1] = OMAP_DSS_CLK_SRC_FCK;
+
+	rev = dss_read_reg(DSS_REVISION);
+	printk(KERN_INFO "OMAP DSS rev %d.%d\n",
+			FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
+
+	dss_runtime_put();
+
+	r = component_bind_all(&pdev->dev, NULL);
+	if (r)
+		goto err_component;
+
+	dss_debugfs_create_file("dss", dss_dump_regs);
+
+	pm_set_vt_switch(0);
+
+	dss_initialized = true;
+
+	return 0;
+
+err_component:
+err_runtime_get:
+	pm_runtime_disable(&pdev->dev);
+	dss_uninit_ports(pdev);
+err_init_ports:
+	if (dss.video1_pll)
+		dss_video_pll_uninit(dss.video1_pll);
+
+	if (dss.video2_pll)
+		dss_video_pll_uninit(dss.video2_pll);
+err_pll_init:
+err_setup_clocks:
+	dss_put_clocks();
+	return r;
+}
+
+static void dss_unbind(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	dss_initialized = false;
+
+	component_unbind_all(&pdev->dev, NULL);
+
+	if (dss.video1_pll)
+		dss_video_pll_uninit(dss.video1_pll);
+
+	if (dss.video2_pll)
+		dss_video_pll_uninit(dss.video2_pll);
+
+	dss_uninit_ports(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	dss_put_clocks();
+}
+
+static const struct component_master_ops dss_component_ops = {
+	.bind = dss_bind,
+	.unbind = dss_unbind,
+};
+
+static int dss_component_compare(struct device *dev, void *data)
+{
+	struct device *child = data;
+	return dev == child;
+}
+
+static int dss_add_child_component(struct device *dev, void *data)
+{
+	struct component_match **match = data;
+
+	/*
+	 * HACK
+	 * We don't have a working driver for rfbi, so skip it here always.
+	 * Otherwise dss will never get probed successfully, as it will wait
+	 * for rfbi to get probed.
+	 */
+	if (strstr(dev_name(dev), "rfbi"))
+		return 0;
+
+	component_match_add(dev->parent, match, dss_component_compare, dev);
+
+	return 0;
+}
+
+static int dss_probe(struct platform_device *pdev)
+{
+	struct component_match *match = NULL;
+	int r;
+
+	/* add all the child devices as components */
+	device_for_each_child(&pdev->dev, &match, dss_add_child_component);
+
+	r = component_master_add_with_match(&pdev->dev, &dss_component_ops, match);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static int dss_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &dss_component_ops);
+	return 0;
+}
+
+static int dss_runtime_suspend(struct device *dev)
+{
+	dss_save_context();
+	dss_set_min_bus_tput(dev, 0);
+
+	pinctrl_pm_select_sleep_state(dev);
+
+	return 0;
+}
+
+static int dss_runtime_resume(struct device *dev)
+{
+	int r;
+
+	pinctrl_pm_select_default_state(dev);
+
+	/*
+	 * Set an arbitrarily high tput request to ensure OPP100.
+	 * What we should really do is to make a request to stay in OPP100,
+	 * without any tput requirements, but that is not currently possible
+	 * via the PM layer.
+	 */
+
+	r = dss_set_min_bus_tput(dev, 1000000000);
+	if (r)
+		return r;
+
+	dss_restore_context();
+	return 0;
+}
+
+static const struct dev_pm_ops dss_pm_ops = {
+	.runtime_suspend = dss_runtime_suspend,
+	.runtime_resume = dss_runtime_resume,
+};
+
+static const struct of_device_id dss_of_match[] = {
+	{ .compatible = "ti,omap2-dss", },
+	{ .compatible = "ti,omap3-dss", },
+	{ .compatible = "ti,omap4-dss", },
+	{ .compatible = "ti,omap5-dss", },
+	{ .compatible = "ti,dra7-dss", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, dss_of_match);
+
+static struct platform_driver omap_dsshw_driver = {
+	.probe		= dss_probe,
+	.remove		= dss_remove,
+	.driver         = {
+		.name   = "omapdss_dss",
+		.pm	= &dss_pm_ops,
+		.of_match_table = dss_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init dss_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_dsshw_driver);
+}
+
+void dss_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_dsshw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.h b/drivers/video/fbdev/omap2/omapfb/dss/dss.h
new file mode 100644
index 000000000000..b9066afee301
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.h
@@ -0,0 +1,468 @@
+/*
+ * linux/drivers/video/omap2/dss/dss.h
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAP2_DSS_H
+#define __OMAP2_DSS_H
+
+#include <linux/interrupt.h>
+
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+
+#ifdef DSS_SUBSYS_NAME
+#define pr_fmt(fmt) DSS_SUBSYS_NAME ": " fmt
+#else
+#define pr_fmt(fmt) fmt
+#endif
+
+#define DSSDBG(format, ...) \
+	pr_debug(format, ## __VA_ARGS__)
+
+#ifdef DSS_SUBSYS_NAME
+#define DSSERR(format, ...) \
+	printk(KERN_ERR "omapdss " DSS_SUBSYS_NAME " error: " format, \
+	## __VA_ARGS__)
+#else
+#define DSSERR(format, ...) \
+	printk(KERN_ERR "omapdss error: " format, ## __VA_ARGS__)
+#endif
+
+#ifdef DSS_SUBSYS_NAME
+#define DSSINFO(format, ...) \
+	printk(KERN_INFO "omapdss " DSS_SUBSYS_NAME ": " format, \
+	## __VA_ARGS__)
+#else
+#define DSSINFO(format, ...) \
+	printk(KERN_INFO "omapdss: " format, ## __VA_ARGS__)
+#endif
+
+#ifdef DSS_SUBSYS_NAME
+#define DSSWARN(format, ...) \
+	printk(KERN_WARNING "omapdss " DSS_SUBSYS_NAME ": " format, \
+	## __VA_ARGS__)
+#else
+#define DSSWARN(format, ...) \
+	printk(KERN_WARNING "omapdss: " format, ## __VA_ARGS__)
+#endif
+
+/* OMAP TRM gives bitfields as start:end, where start is the higher bit
+   number. For example 7:0 */
+#define FLD_MASK(start, end)	(((1 << ((start) - (end) + 1)) - 1) << (end))
+#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
+#define FLD_GET(val, start, end) (((val) & FLD_MASK(start, end)) >> (end))
+#define FLD_MOD(orig, val, start, end) \
+	(((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end))
+
+enum dss_io_pad_mode {
+	DSS_IO_PAD_MODE_RESET,
+	DSS_IO_PAD_MODE_RFBI,
+	DSS_IO_PAD_MODE_BYPASS,
+};
+
+enum dss_hdmi_venc_clk_source_select {
+	DSS_VENC_TV_CLK = 0,
+	DSS_HDMI_M_PCLK = 1,
+};
+
+enum dss_dsi_content_type {
+	DSS_DSI_CONTENT_DCS,
+	DSS_DSI_CONTENT_GENERIC,
+};
+
+enum dss_writeback_channel {
+	DSS_WB_LCD1_MGR =	0,
+	DSS_WB_LCD2_MGR =	1,
+	DSS_WB_TV_MGR =		2,
+	DSS_WB_OVL0 =		3,
+	DSS_WB_OVL1 =		4,
+	DSS_WB_OVL2 =		5,
+	DSS_WB_OVL3 =		6,
+	DSS_WB_LCD3_MGR =	7,
+};
+
+enum dss_pll_id {
+	DSS_PLL_DSI1,
+	DSS_PLL_DSI2,
+	DSS_PLL_HDMI,
+	DSS_PLL_VIDEO1,
+	DSS_PLL_VIDEO2,
+};
+
+struct dss_pll;
+
+#define DSS_PLL_MAX_HSDIVS 4
+
+/*
+ * Type-A PLLs: clkout[]/mX[] refer to hsdiv outputs m4, m5, m6, m7.
+ * Type-B PLLs: clkout[0] refers to m2.
+ */
+struct dss_pll_clock_info {
+	/* rates that we get with dividers below */
+	unsigned long fint;
+	unsigned long clkdco;
+	unsigned long clkout[DSS_PLL_MAX_HSDIVS];
+
+	/* dividers */
+	u16 n;
+	u16 m;
+	u32 mf;
+	u16 mX[DSS_PLL_MAX_HSDIVS];
+	u16 sd;
+};
+
+struct dss_pll_ops {
+	int (*enable)(struct dss_pll *pll);
+	void (*disable)(struct dss_pll *pll);
+	int (*set_config)(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo);
+};
+
+struct dss_pll_hw {
+	unsigned n_max;
+	unsigned m_min;
+	unsigned m_max;
+	unsigned mX_max;
+
+	unsigned long fint_min, fint_max;
+	unsigned long clkdco_min, clkdco_low, clkdco_max;
+
+	u8 n_msb, n_lsb;
+	u8 m_msb, m_lsb;
+	u8 mX_msb[DSS_PLL_MAX_HSDIVS], mX_lsb[DSS_PLL_MAX_HSDIVS];
+
+	bool has_stopmode;
+	bool has_freqsel;
+	bool has_selfreqdco;
+	bool has_refsel;
+};
+
+struct dss_pll {
+	const char *name;
+	enum dss_pll_id id;
+
+	struct clk *clkin;
+	struct regulator *regulator;
+
+	void __iomem *base;
+
+	const struct dss_pll_hw *hw;
+
+	const struct dss_pll_ops *ops;
+
+	struct dss_pll_clock_info cinfo;
+};
+
+struct dispc_clock_info {
+	/* rates that we get with dividers below */
+	unsigned long lck;
+	unsigned long pck;
+
+	/* dividers */
+	u16 lck_div;
+	u16 pck_div;
+};
+
+struct dss_lcd_mgr_config {
+	enum dss_io_pad_mode io_pad_mode;
+
+	bool stallmode;
+	bool fifohandcheck;
+
+	struct dispc_clock_info clock_info;
+
+	int video_port_width;
+
+	int lcden_sig_polarity;
+};
+
+struct seq_file;
+struct platform_device;
+
+/* core */
+struct platform_device *dss_get_core_pdev(void);
+int dss_dsi_enable_pads(int dsi_id, unsigned lane_mask);
+void dss_dsi_disable_pads(int dsi_id, unsigned lane_mask);
+int dss_set_min_bus_tput(struct device *dev, unsigned long tput);
+int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *));
+
+/* display */
+int dss_suspend_all_devices(void);
+int dss_resume_all_devices(void);
+void dss_disable_all_devices(void);
+
+int display_init_sysfs(struct platform_device *pdev);
+void display_uninit_sysfs(struct platform_device *pdev);
+
+/* manager */
+int dss_init_overlay_managers(void);
+void dss_uninit_overlay_managers(void);
+int dss_init_overlay_managers_sysfs(struct platform_device *pdev);
+void dss_uninit_overlay_managers_sysfs(struct platform_device *pdev);
+int dss_mgr_simple_check(struct omap_overlay_manager *mgr,
+		const struct omap_overlay_manager_info *info);
+int dss_mgr_check_timings(struct omap_overlay_manager *mgr,
+		const struct omap_video_timings *timings);
+int dss_mgr_check(struct omap_overlay_manager *mgr,
+		struct omap_overlay_manager_info *info,
+		const struct omap_video_timings *mgr_timings,
+		const struct dss_lcd_mgr_config *config,
+		struct omap_overlay_info **overlay_infos);
+
+static inline bool dss_mgr_is_lcd(enum omap_channel id)
+{
+	if (id == OMAP_DSS_CHANNEL_LCD || id == OMAP_DSS_CHANNEL_LCD2 ||
+			id == OMAP_DSS_CHANNEL_LCD3)
+		return true;
+	else
+		return false;
+}
+
+int dss_manager_kobj_init(struct omap_overlay_manager *mgr,
+		struct platform_device *pdev);
+void dss_manager_kobj_uninit(struct omap_overlay_manager *mgr);
+
+/* overlay */
+void dss_init_overlays(struct platform_device *pdev);
+void dss_uninit_overlays(struct platform_device *pdev);
+void dss_overlay_setup_dispc_manager(struct omap_overlay_manager *mgr);
+int dss_ovl_simple_check(struct omap_overlay *ovl,
+		const struct omap_overlay_info *info);
+int dss_ovl_check(struct omap_overlay *ovl, struct omap_overlay_info *info,
+		const struct omap_video_timings *mgr_timings);
+bool dss_ovl_use_replication(struct dss_lcd_mgr_config config,
+		enum omap_color_mode mode);
+int dss_overlay_kobj_init(struct omap_overlay *ovl,
+		struct platform_device *pdev);
+void dss_overlay_kobj_uninit(struct omap_overlay *ovl);
+
+/* DSS */
+int dss_init_platform_driver(void) __init;
+void dss_uninit_platform_driver(void);
+
+int dss_runtime_get(void);
+void dss_runtime_put(void);
+
+unsigned long dss_get_dispc_clk_rate(void);
+int dss_dpi_select_source(int port, enum omap_channel channel);
+void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select);
+enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void);
+const char *dss_get_generic_clk_source_name(enum omap_dss_clk_source clk_src);
+void dss_dump_clocks(struct seq_file *s);
+
+/* DSS VIDEO PLL */
+struct dss_pll *dss_video_pll_init(struct platform_device *pdev, int id,
+	struct regulator *regulator);
+void dss_video_pll_uninit(struct dss_pll *pll);
+
+/* dss-of */
+struct device_node *dss_of_port_get_parent_device(struct device_node *port);
+u32 dss_of_port_get_port_number(struct device_node *port);
+
+#if defined(CONFIG_FB_OMAP2_DSS_DEBUGFS)
+void dss_debug_dump_clocks(struct seq_file *s);
+#endif
+
+void dss_ctrl_pll_enable(enum dss_pll_id pll_id, bool enable);
+void dss_ctrl_pll_set_control_mux(enum dss_pll_id pll_id,
+	enum omap_channel channel);
+
+void dss_sdi_init(int datapairs);
+int dss_sdi_enable(void);
+void dss_sdi_disable(void);
+
+void dss_select_dsi_clk_source(int dsi_module,
+		enum omap_dss_clk_source clk_src);
+void dss_select_lcd_clk_source(enum omap_channel channel,
+		enum omap_dss_clk_source clk_src);
+enum omap_dss_clk_source dss_get_dispc_clk_source(void);
+enum omap_dss_clk_source dss_get_dsi_clk_source(int dsi_module);
+enum omap_dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel);
+
+void dss_set_venc_output(enum omap_dss_venc_type type);
+void dss_set_dac_pwrdn_bgz(bool enable);
+
+int dss_set_fck_rate(unsigned long rate);
+
+typedef bool (*dss_div_calc_func)(unsigned long fck, void *data);
+bool dss_div_calc(unsigned long pck, unsigned long fck_min,
+		dss_div_calc_func func, void *data);
+
+/* SDI */
+int sdi_init_platform_driver(void) __init;
+void sdi_uninit_platform_driver(void);
+
+#ifdef CONFIG_FB_OMAP2_DSS_SDI
+int sdi_init_port(struct platform_device *pdev, struct device_node *port);
+void sdi_uninit_port(struct device_node *port);
+#else
+static inline int sdi_init_port(struct platform_device *pdev,
+		struct device_node *port)
+{
+	return 0;
+}
+static inline void sdi_uninit_port(struct device_node *port)
+{
+}
+#endif
+
+/* DSI */
+
+#ifdef CONFIG_FB_OMAP2_DSS_DSI
+
+struct dentry;
+struct file_operations;
+
+int dsi_init_platform_driver(void) __init;
+void dsi_uninit_platform_driver(void);
+
+void dsi_dump_clocks(struct seq_file *s);
+
+void dsi_irq_handler(void);
+u8 dsi_get_pixel_size(enum omap_dss_dsi_pixel_format fmt);
+
+#else
+static inline u8 dsi_get_pixel_size(enum omap_dss_dsi_pixel_format fmt)
+{
+	WARN(1, "%s: DSI not compiled in, returning pixel_size as 0\n",
+	     __func__);
+	return 0;
+}
+#endif
+
+/* DPI */
+int dpi_init_platform_driver(void) __init;
+void dpi_uninit_platform_driver(void);
+
+#ifdef CONFIG_FB_OMAP2_DSS_DPI
+int dpi_init_port(struct platform_device *pdev, struct device_node *port);
+void dpi_uninit_port(struct device_node *port);
+#else
+static inline int dpi_init_port(struct platform_device *pdev,
+		struct device_node *port)
+{
+	return 0;
+}
+static inline void dpi_uninit_port(struct device_node *port)
+{
+}
+#endif
+
+/* DISPC */
+int dispc_init_platform_driver(void) __init;
+void dispc_uninit_platform_driver(void);
+void dispc_dump_clocks(struct seq_file *s);
+
+void dispc_enable_sidle(void);
+void dispc_disable_sidle(void);
+
+void dispc_lcd_enable_signal(bool enable);
+void dispc_pck_free_enable(bool enable);
+void dispc_enable_fifomerge(bool enable);
+void dispc_enable_gamma_table(bool enable);
+
+typedef bool (*dispc_div_calc_func)(int lckd, int pckd, unsigned long lck,
+		unsigned long pck, void *data);
+bool dispc_div_calc(unsigned long dispc,
+		unsigned long pck_min, unsigned long pck_max,
+		dispc_div_calc_func func, void *data);
+
+bool dispc_mgr_timings_ok(enum omap_channel channel,
+		const struct omap_video_timings *timings);
+int dispc_calc_clock_rates(unsigned long dispc_fclk_rate,
+		struct dispc_clock_info *cinfo);
+
+
+void dispc_ovl_set_fifo_threshold(enum omap_plane plane, u32 low, u32 high);
+void dispc_ovl_compute_fifo_thresholds(enum omap_plane plane,
+		u32 *fifo_low, u32 *fifo_high, bool use_fifomerge,
+		bool manual_update);
+
+void dispc_mgr_set_clock_div(enum omap_channel channel,
+		const struct dispc_clock_info *cinfo);
+int dispc_mgr_get_clock_div(enum omap_channel channel,
+		struct dispc_clock_info *cinfo);
+void dispc_set_tv_pclk(unsigned long pclk);
+
+u32 dispc_wb_get_framedone_irq(void);
+bool dispc_wb_go_busy(void);
+void dispc_wb_go(void);
+void dispc_wb_enable(bool enable);
+bool dispc_wb_is_enabled(void);
+void dispc_wb_set_channel_in(enum dss_writeback_channel channel);
+int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
+		bool mem_to_mem, const struct omap_video_timings *timings);
+
+/* VENC */
+int venc_init_platform_driver(void) __init;
+void venc_uninit_platform_driver(void);
+
+/* HDMI */
+int hdmi4_init_platform_driver(void) __init;
+void hdmi4_uninit_platform_driver(void);
+
+int hdmi5_init_platform_driver(void) __init;
+void hdmi5_uninit_platform_driver(void);
+
+/* RFBI */
+int rfbi_init_platform_driver(void) __init;
+void rfbi_uninit_platform_driver(void);
+
+
+#ifdef CONFIG_FB_OMAP2_DSS_COLLECT_IRQ_STATS
+static inline void dss_collect_irq_stats(u32 irqstatus, unsigned *irq_arr)
+{
+	int b;
+	for (b = 0; b < 32; ++b) {
+		if (irqstatus & (1 << b))
+			irq_arr[b]++;
+	}
+}
+#endif
+
+/* PLL */
+typedef bool (*dss_pll_calc_func)(int n, int m, unsigned long fint,
+		unsigned long clkdco, void *data);
+typedef bool (*dss_hsdiv_calc_func)(int m_dispc, unsigned long dispc,
+		void *data);
+
+int dss_pll_register(struct dss_pll *pll);
+void dss_pll_unregister(struct dss_pll *pll);
+struct dss_pll *dss_pll_find(const char *name);
+int dss_pll_enable(struct dss_pll *pll);
+void dss_pll_disable(struct dss_pll *pll);
+int dss_pll_set_config(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo);
+
+bool dss_pll_hsdiv_calc(const struct dss_pll *pll, unsigned long clkdco,
+		unsigned long out_min, unsigned long out_max,
+		dss_hsdiv_calc_func func, void *data);
+bool dss_pll_calc(const struct dss_pll *pll, unsigned long clkin,
+		unsigned long pll_min, unsigned long pll_max,
+		dss_pll_calc_func func, void *data);
+int dss_pll_write_config_type_a(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo);
+int dss_pll_write_config_type_b(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo);
+int dss_pll_wait_reset_done(struct dss_pll *pll);
+
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c
new file mode 100644
index 000000000000..c886a2927f73
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c
@@ -0,0 +1,951 @@
+/*
+ * linux/drivers/video/omap2/dss/dss_features.c
+ *
+ * Copyright (C) 2010 Texas Instruments
+ * Author: Archit Taneja <archit@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+/* Defines a generic omap register field */
+struct dss_reg_field {
+	u8 start, end;
+};
+
+struct dss_param_range {
+	int min, max;
+};
+
+struct omap_dss_features {
+	const struct dss_reg_field *reg_fields;
+	const int num_reg_fields;
+
+	const enum dss_feat_id *features;
+	const int num_features;
+
+	const int num_mgrs;
+	const int num_ovls;
+	const enum omap_display_type *supported_displays;
+	const enum omap_dss_output_id *supported_outputs;
+	const enum omap_color_mode *supported_color_modes;
+	const enum omap_overlay_caps *overlay_caps;
+	const char * const *clksrc_names;
+	const struct dss_param_range *dss_params;
+
+	const enum omap_dss_rotation_type supported_rotation_types;
+
+	const u32 buffer_size_unit;
+	const u32 burst_size_unit;
+};
+
+/* This struct is assigned to one of the below during initialization */
+static const struct omap_dss_features *omap_current_dss_features;
+
+static const struct dss_reg_field omap2_dss_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 11, 0 },
+	[FEAT_REG_FIRVINC]			= { 27, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 8, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 24, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 8, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
+	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
+};
+
+static const struct dss_reg_field omap3_dss_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 11, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 27, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 10, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
+	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
+};
+
+static const struct dss_reg_field am43xx_dss_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]	= { 11, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 27, 16 },
+	[FEAT_REG_FIFOSIZE]		= { 10, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
+	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
+};
+
+static const struct dss_reg_field omap4_dss_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 15, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 31, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 15, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 10, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 26, 16 },
+	[FEAT_REG_DISPC_CLK_SWITCH]		= { 9, 8 },
+};
+
+static const struct dss_reg_field omap5_dss_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 15, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 31, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 15, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 10, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 26, 16 },
+	[FEAT_REG_DISPC_CLK_SWITCH]		= { 9, 7 },
+};
+
+static const enum omap_display_type omap2_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DISPLAY_TYPE_VENC,
+};
+
+static const enum omap_display_type omap3430_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
+	OMAP_DISPLAY_TYPE_SDI | OMAP_DISPLAY_TYPE_DSI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DISPLAY_TYPE_VENC,
+};
+
+static const enum omap_display_type omap3630_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
+	OMAP_DISPLAY_TYPE_DSI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DISPLAY_TYPE_VENC,
+};
+
+static const enum omap_display_type am43xx_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI,
+};
+
+static const enum omap_display_type omap4_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DBI | OMAP_DISPLAY_TYPE_DSI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DISPLAY_TYPE_VENC | OMAP_DISPLAY_TYPE_HDMI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
+	OMAP_DISPLAY_TYPE_DSI,
+};
+
+static const enum omap_display_type omap5_dss_supported_displays[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
+	OMAP_DISPLAY_TYPE_DSI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DISPLAY_TYPE_HDMI | OMAP_DISPLAY_TYPE_DPI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
+	OMAP_DISPLAY_TYPE_DSI,
+};
+
+static const enum omap_dss_output_id omap2_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id omap3430_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_SDI | OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id omap3630_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id am43xx_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
+};
+
+static const enum omap_dss_output_id omap4_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DBI | OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC | OMAP_DSS_OUTPUT_HDMI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI2,
+};
+
+static const enum omap_dss_output_id omap5_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1 | OMAP_DSS_OUTPUT_DSI2,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_HDMI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_LCD3 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI2,
+};
+
+static const enum omap_color_mode omap2_dss_supported_color_modes[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_COLOR_CLUT1 | OMAP_DSS_COLOR_CLUT2 |
+	OMAP_DSS_COLOR_CLUT4 | OMAP_DSS_COLOR_CLUT8 |
+	OMAP_DSS_COLOR_RGB12U | OMAP_DSS_COLOR_RGB16 |
+	OMAP_DSS_COLOR_RGB24U | OMAP_DSS_COLOR_RGB24P,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_YUV2 |
+	OMAP_DSS_COLOR_UYVY,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_YUV2 |
+	OMAP_DSS_COLOR_UYVY,
+};
+
+static const enum omap_color_mode omap3_dss_supported_color_modes[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_COLOR_CLUT1 | OMAP_DSS_COLOR_CLUT2 |
+	OMAP_DSS_COLOR_CLUT4 | OMAP_DSS_COLOR_CLUT8 |
+	OMAP_DSS_COLOR_RGB12U | OMAP_DSS_COLOR_ARGB16 |
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_ARGB32 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_RGBX32,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_COLOR_RGB24U | OMAP_DSS_COLOR_RGB24P |
+	OMAP_DSS_COLOR_RGB12U | OMAP_DSS_COLOR_RGB16 |
+	OMAP_DSS_COLOR_YUV2 | OMAP_DSS_COLOR_UYVY,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_COLOR_RGB12U | OMAP_DSS_COLOR_ARGB16 |
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_YUV2 |
+	OMAP_DSS_COLOR_UYVY | OMAP_DSS_COLOR_ARGB32 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_RGBX32,
+};
+
+static const enum omap_color_mode omap4_dss_supported_color_modes[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_COLOR_CLUT1 | OMAP_DSS_COLOR_CLUT2 |
+	OMAP_DSS_COLOR_CLUT4 | OMAP_DSS_COLOR_CLUT8 |
+	OMAP_DSS_COLOR_RGB12U | OMAP_DSS_COLOR_ARGB16 |
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_ARGB32 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_RGBX32 |
+	OMAP_DSS_COLOR_ARGB16_1555 | OMAP_DSS_COLOR_RGBX16 |
+	OMAP_DSS_COLOR_RGBA16 | OMAP_DSS_COLOR_XRGB16_1555,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB12U |
+	OMAP_DSS_COLOR_YUV2 | OMAP_DSS_COLOR_ARGB16_1555 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_NV12 |
+	OMAP_DSS_COLOR_RGBA16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_UYVY |
+	OMAP_DSS_COLOR_ARGB16 | OMAP_DSS_COLOR_XRGB16_1555 |
+	OMAP_DSS_COLOR_ARGB32 | OMAP_DSS_COLOR_RGBX16 |
+	OMAP_DSS_COLOR_RGBX32,
+
+       /* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB12U |
+	OMAP_DSS_COLOR_YUV2 | OMAP_DSS_COLOR_ARGB16_1555 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_NV12 |
+	OMAP_DSS_COLOR_RGBA16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_UYVY |
+	OMAP_DSS_COLOR_ARGB16 | OMAP_DSS_COLOR_XRGB16_1555 |
+	OMAP_DSS_COLOR_ARGB32 | OMAP_DSS_COLOR_RGBX16 |
+	OMAP_DSS_COLOR_RGBX32,
+
+	/* OMAP_DSS_VIDEO3 */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB12U |
+	OMAP_DSS_COLOR_YUV2 | OMAP_DSS_COLOR_ARGB16_1555 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_NV12 |
+	OMAP_DSS_COLOR_RGBA16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_UYVY |
+	OMAP_DSS_COLOR_ARGB16 | OMAP_DSS_COLOR_XRGB16_1555 |
+	OMAP_DSS_COLOR_ARGB32 | OMAP_DSS_COLOR_RGBX16 |
+	OMAP_DSS_COLOR_RGBX32,
+
+	/* OMAP_DSS_WB */
+	OMAP_DSS_COLOR_RGB16 | OMAP_DSS_COLOR_RGB12U |
+	OMAP_DSS_COLOR_YUV2 | OMAP_DSS_COLOR_ARGB16_1555 |
+	OMAP_DSS_COLOR_RGBA32 | OMAP_DSS_COLOR_NV12 |
+	OMAP_DSS_COLOR_RGBA16 | OMAP_DSS_COLOR_RGB24U |
+	OMAP_DSS_COLOR_RGB24P | OMAP_DSS_COLOR_UYVY |
+	OMAP_DSS_COLOR_ARGB16 | OMAP_DSS_COLOR_XRGB16_1555 |
+	OMAP_DSS_COLOR_ARGB32 | OMAP_DSS_COLOR_RGBX16 |
+	OMAP_DSS_COLOR_RGBX32,
+};
+
+static const enum omap_overlay_caps omap2_dss_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap3430_dss_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap3630_dss_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap4_dss_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
+		OMAP_DSS_OVL_CAP_ZORDER | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO3 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const char * const omap2_dss_clk_source_names[] = {
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]	= "N/A",
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]	= "N/A",
+	[OMAP_DSS_CLK_SRC_FCK]			= "DSS_FCLK1",
+};
+
+static const char * const omap3_dss_clk_source_names[] = {
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]	= "DSI1_PLL_FCLK",
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]	= "DSI2_PLL_FCLK",
+	[OMAP_DSS_CLK_SRC_FCK]			= "DSS1_ALWON_FCLK",
+};
+
+static const char * const omap4_dss_clk_source_names[] = {
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]	= "PLL1_CLK1",
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]	= "PLL1_CLK2",
+	[OMAP_DSS_CLK_SRC_FCK]			= "DSS_FCLK",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC]	= "PLL2_CLK1",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]	= "PLL2_CLK2",
+};
+
+static const char * const omap5_dss_clk_source_names[] = {
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]	= "DPLL_DSI1_A_CLK1",
+	[OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]	= "DPLL_DSI1_A_CLK2",
+	[OMAP_DSS_CLK_SRC_FCK]			= "DSS_CLK",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC]	= "DPLL_DSI1_C_CLK1",
+	[OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]	= "DPLL_DSI1_C_CLK2",
+};
+
+static const struct dss_param_range omap2_dss_param_range[] = {
+	[FEAT_PARAM_DSS_FCK]			= { 0, 133000000 },
+	[FEAT_PARAM_DSS_PCD]			= { 2, 255 },
+	[FEAT_PARAM_DOWNSCALE]			= { 1, 2 },
+	/*
+	 * Assuming the line width buffer to be 768 pixels as OMAP2 DISPC
+	 * scaler cannot scale a image with width more than 768.
+	 */
+	[FEAT_PARAM_LINEWIDTH]			= { 1, 768 },
+};
+
+static const struct dss_param_range omap3_dss_param_range[] = {
+	[FEAT_PARAM_DSS_FCK]			= { 0, 173000000 },
+	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
+	[FEAT_PARAM_DSIPLL_LPDIV]		= { 1, (1 << 13) - 1},
+	[FEAT_PARAM_DSI_FCK]			= { 0, 173000000 },
+	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
+	[FEAT_PARAM_LINEWIDTH]			= { 1, 1024 },
+};
+
+static const struct dss_param_range am43xx_dss_param_range[] = {
+	[FEAT_PARAM_DSS_FCK]			= { 0, 200000000 },
+	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
+	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
+	[FEAT_PARAM_LINEWIDTH]			= { 1, 1024 },
+};
+
+static const struct dss_param_range omap4_dss_param_range[] = {
+	[FEAT_PARAM_DSS_FCK]			= { 0, 186000000 },
+	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
+	[FEAT_PARAM_DSIPLL_LPDIV]		= { 0, (1 << 13) - 1 },
+	[FEAT_PARAM_DSI_FCK]			= { 0, 170000000 },
+	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
+	[FEAT_PARAM_LINEWIDTH]			= { 1, 2048 },
+};
+
+static const struct dss_param_range omap5_dss_param_range[] = {
+	[FEAT_PARAM_DSS_FCK]			= { 0, 209250000 },
+	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
+	[FEAT_PARAM_DSIPLL_LPDIV]		= { 0, (1 << 13) - 1 },
+	[FEAT_PARAM_DSI_FCK]			= { 0, 209250000 },
+	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
+	[FEAT_PARAM_LINEWIDTH]			= { 1, 2048 },
+};
+
+static const enum dss_feat_id omap2_dss_feat_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+};
+
+static const enum dss_feat_id omap3430_dss_feat_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_DSI_REVERSE_TXCLKESC,
+	FEAT_VENC_REQUIRES_TV_DAC_CLK,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_OMAP3_DSI_FIFO_BUG,
+	FEAT_DPI_USES_VDDS_DSI,
+};
+
+static const enum dss_feat_id am35xx_dss_feat_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_DSI_REVERSE_TXCLKESC,
+	FEAT_VENC_REQUIRES_TV_DAC_CLK,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_OMAP3_DSI_FIFO_BUG,
+};
+
+static const enum dss_feat_id am43xx_dss_feat_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+};
+
+static const enum dss_feat_id omap3630_dss_feat_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_DSI_PLL_PWR_BUG,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_OMAP3_DSI_FIFO_BUG,
+	FEAT_DPI_USES_VDDS_DSI,
+};
+
+static const enum dss_feat_id omap4430_es1_0_dss_feat_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_CORE_CLK_DIV,
+	FEAT_LCD_CLK_SRC,
+	FEAT_DSI_DCS_CMD_CONFIG_VC,
+	FEAT_DSI_VC_OCP_WIDTH,
+	FEAT_DSI_GNQ,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+};
+
+static const enum dss_feat_id omap4430_es2_0_1_2_dss_feat_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_CORE_CLK_DIV,
+	FEAT_LCD_CLK_SRC,
+	FEAT_DSI_DCS_CMD_CONFIG_VC,
+	FEAT_DSI_VC_OCP_WIDTH,
+	FEAT_DSI_GNQ,
+	FEAT_HDMI_CTS_SWMODE,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+};
+
+static const enum dss_feat_id omap4_dss_feat_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_CORE_CLK_DIV,
+	FEAT_LCD_CLK_SRC,
+	FEAT_DSI_DCS_CMD_CONFIG_VC,
+	FEAT_DSI_VC_OCP_WIDTH,
+	FEAT_DSI_GNQ,
+	FEAT_HDMI_CTS_SWMODE,
+	FEAT_HDMI_AUDIO_USE_MCLK,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+};
+
+static const enum dss_feat_id omap5_dss_feat_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_MGR_LCD3,
+	FEAT_CORE_CLK_DIV,
+	FEAT_LCD_CLK_SRC,
+	FEAT_DSI_DCS_CMD_CONFIG_VC,
+	FEAT_DSI_VC_OCP_WIDTH,
+	FEAT_DSI_GNQ,
+	FEAT_HDMI_CTS_SWMODE,
+	FEAT_HDMI_AUDIO_USE_MCLK,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+	FEAT_DSI_PHY_DCC,
+	FEAT_MFLAG,
+};
+
+/* OMAP2 DSS Features */
+static const struct omap_dss_features omap2_dss_features = {
+	.reg_fields = omap2_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap2_dss_reg_fields),
+
+	.features = omap2_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap2_dss_feat_list),
+
+	.num_mgrs = 2,
+	.num_ovls = 3,
+	.supported_displays = omap2_dss_supported_displays,
+	.supported_outputs = omap2_dss_supported_outputs,
+	.supported_color_modes = omap2_dss_supported_color_modes,
+	.overlay_caps = omap2_dss_overlay_caps,
+	.clksrc_names = omap2_dss_clk_source_names,
+	.dss_params = omap2_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
+	.buffer_size_unit = 1,
+	.burst_size_unit = 8,
+};
+
+/* OMAP3 DSS Features */
+static const struct omap_dss_features omap3430_dss_features = {
+	.reg_fields = omap3_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
+
+	.features = omap3430_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap3430_dss_feat_list),
+
+	.num_mgrs = 2,
+	.num_ovls = 3,
+	.supported_displays = omap3430_dss_supported_displays,
+	.supported_outputs = omap3430_dss_supported_outputs,
+	.supported_color_modes = omap3_dss_supported_color_modes,
+	.overlay_caps = omap3430_dss_overlay_caps,
+	.clksrc_names = omap3_dss_clk_source_names,
+	.dss_params = omap3_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
+	.buffer_size_unit = 1,
+	.burst_size_unit = 8,
+};
+
+/*
+ * AM35xx DSS Features. This is basically OMAP3 DSS Features without the
+ * vdds_dsi regulator.
+ */
+static const struct omap_dss_features am35xx_dss_features = {
+	.reg_fields = omap3_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
+
+	.features = am35xx_dss_feat_list,
+	.num_features = ARRAY_SIZE(am35xx_dss_feat_list),
+
+	.num_mgrs = 2,
+	.num_ovls = 3,
+	.supported_displays = omap3430_dss_supported_displays,
+	.supported_outputs = omap3430_dss_supported_outputs,
+	.supported_color_modes = omap3_dss_supported_color_modes,
+	.overlay_caps = omap3430_dss_overlay_caps,
+	.clksrc_names = omap3_dss_clk_source_names,
+	.dss_params = omap3_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
+	.buffer_size_unit = 1,
+	.burst_size_unit = 8,
+};
+
+static const struct omap_dss_features am43xx_dss_features = {
+	.reg_fields = am43xx_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(am43xx_dss_reg_fields),
+
+	.features = am43xx_dss_feat_list,
+	.num_features = ARRAY_SIZE(am43xx_dss_feat_list),
+
+	.num_mgrs = 1,
+	.num_ovls = 3,
+	.supported_displays = am43xx_dss_supported_displays,
+	.supported_outputs = am43xx_dss_supported_outputs,
+	.supported_color_modes = omap3_dss_supported_color_modes,
+	.overlay_caps = omap3430_dss_overlay_caps,
+	.clksrc_names = omap2_dss_clk_source_names,
+	.dss_params = am43xx_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA,
+	.buffer_size_unit = 1,
+	.burst_size_unit = 8,
+};
+
+static const struct omap_dss_features omap3630_dss_features = {
+	.reg_fields = omap3_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
+
+	.features = omap3630_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap3630_dss_feat_list),
+
+	.num_mgrs = 2,
+	.num_ovls = 3,
+	.supported_displays = omap3630_dss_supported_displays,
+	.supported_outputs = omap3630_dss_supported_outputs,
+	.supported_color_modes = omap3_dss_supported_color_modes,
+	.overlay_caps = omap3630_dss_overlay_caps,
+	.clksrc_names = omap3_dss_clk_source_names,
+	.dss_params = omap3_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
+	.buffer_size_unit = 1,
+	.burst_size_unit = 8,
+};
+
+/* OMAP4 DSS Features */
+/* For OMAP4430 ES 1.0 revision */
+static const struct omap_dss_features omap4430_es1_0_dss_features  = {
+	.reg_fields = omap4_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
+
+	.features = omap4430_es1_0_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap4430_es1_0_dss_feat_list),
+
+	.num_mgrs = 3,
+	.num_ovls = 4,
+	.supported_displays = omap4_dss_supported_displays,
+	.supported_outputs = omap4_dss_supported_outputs,
+	.supported_color_modes = omap4_dss_supported_color_modes,
+	.overlay_caps = omap4_dss_overlay_caps,
+	.clksrc_names = omap4_dss_clk_source_names,
+	.dss_params = omap4_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
+	.buffer_size_unit = 16,
+	.burst_size_unit = 16,
+};
+
+/* For OMAP4430 ES 2.0, 2.1 and 2.2 revisions */
+static const struct omap_dss_features omap4430_es2_0_1_2_dss_features = {
+	.reg_fields = omap4_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
+
+	.features = omap4430_es2_0_1_2_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap4430_es2_0_1_2_dss_feat_list),
+
+	.num_mgrs = 3,
+	.num_ovls = 4,
+	.supported_displays = omap4_dss_supported_displays,
+	.supported_outputs = omap4_dss_supported_outputs,
+	.supported_color_modes = omap4_dss_supported_color_modes,
+	.overlay_caps = omap4_dss_overlay_caps,
+	.clksrc_names = omap4_dss_clk_source_names,
+	.dss_params = omap4_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
+	.buffer_size_unit = 16,
+	.burst_size_unit = 16,
+};
+
+/* For all the other OMAP4 versions */
+static const struct omap_dss_features omap4_dss_features = {
+	.reg_fields = omap4_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
+
+	.features = omap4_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap4_dss_feat_list),
+
+	.num_mgrs = 3,
+	.num_ovls = 4,
+	.supported_displays = omap4_dss_supported_displays,
+	.supported_outputs = omap4_dss_supported_outputs,
+	.supported_color_modes = omap4_dss_supported_color_modes,
+	.overlay_caps = omap4_dss_overlay_caps,
+	.clksrc_names = omap4_dss_clk_source_names,
+	.dss_params = omap4_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
+	.buffer_size_unit = 16,
+	.burst_size_unit = 16,
+};
+
+/* OMAP5 DSS Features */
+static const struct omap_dss_features omap5_dss_features = {
+	.reg_fields = omap5_dss_reg_fields,
+	.num_reg_fields = ARRAY_SIZE(omap5_dss_reg_fields),
+
+	.features = omap5_dss_feat_list,
+	.num_features = ARRAY_SIZE(omap5_dss_feat_list),
+
+	.num_mgrs = 4,
+	.num_ovls = 4,
+	.supported_displays = omap5_dss_supported_displays,
+	.supported_outputs = omap5_dss_supported_outputs,
+	.supported_color_modes = omap4_dss_supported_color_modes,
+	.overlay_caps = omap4_dss_overlay_caps,
+	.clksrc_names = omap5_dss_clk_source_names,
+	.dss_params = omap5_dss_param_range,
+	.supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
+	.buffer_size_unit = 16,
+	.burst_size_unit = 16,
+};
+
+/* Functions returning values related to a DSS feature */
+int dss_feat_get_num_mgrs(void)
+{
+	return omap_current_dss_features->num_mgrs;
+}
+EXPORT_SYMBOL(dss_feat_get_num_mgrs);
+
+int dss_feat_get_num_ovls(void)
+{
+	return omap_current_dss_features->num_ovls;
+}
+EXPORT_SYMBOL(dss_feat_get_num_ovls);
+
+unsigned long dss_feat_get_param_min(enum dss_range_param param)
+{
+	return omap_current_dss_features->dss_params[param].min;
+}
+
+unsigned long dss_feat_get_param_max(enum dss_range_param param)
+{
+	return omap_current_dss_features->dss_params[param].max;
+}
+
+enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel)
+{
+	return omap_current_dss_features->supported_displays[channel];
+}
+
+enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel)
+{
+	return omap_current_dss_features->supported_outputs[channel];
+}
+
+enum omap_color_mode dss_feat_get_supported_color_modes(enum omap_plane plane)
+{
+	return omap_current_dss_features->supported_color_modes[plane];
+}
+EXPORT_SYMBOL(dss_feat_get_supported_color_modes);
+
+enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane plane)
+{
+	return omap_current_dss_features->overlay_caps[plane];
+}
+
+bool dss_feat_color_mode_supported(enum omap_plane plane,
+		enum omap_color_mode color_mode)
+{
+	return omap_current_dss_features->supported_color_modes[plane] &
+			color_mode;
+}
+
+const char *dss_feat_get_clk_source_name(enum omap_dss_clk_source id)
+{
+	return omap_current_dss_features->clksrc_names[id];
+}
+
+u32 dss_feat_get_buffer_size_unit(void)
+{
+	return omap_current_dss_features->buffer_size_unit;
+}
+
+u32 dss_feat_get_burst_size_unit(void)
+{
+	return omap_current_dss_features->burst_size_unit;
+}
+
+/* DSS has_feature check */
+bool dss_has_feature(enum dss_feat_id id)
+{
+	int i;
+	const enum dss_feat_id *features = omap_current_dss_features->features;
+	const int num_features = omap_current_dss_features->num_features;
+
+	for (i = 0; i < num_features; i++) {
+		if (features[i] == id)
+			return true;
+	}
+
+	return false;
+}
+
+void dss_feat_get_reg_field(enum dss_feat_reg_field id, u8 *start, u8 *end)
+{
+	if (id >= omap_current_dss_features->num_reg_fields)
+		BUG();
+
+	*start = omap_current_dss_features->reg_fields[id].start;
+	*end = omap_current_dss_features->reg_fields[id].end;
+}
+
+bool dss_feat_rotation_type_supported(enum omap_dss_rotation_type rot_type)
+{
+	return omap_current_dss_features->supported_rotation_types & rot_type;
+}
+
+void dss_features_init(enum omapdss_version version)
+{
+	switch (version) {
+	case OMAPDSS_VER_OMAP24xx:
+		omap_current_dss_features = &omap2_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP34xx_ES1:
+	case OMAPDSS_VER_OMAP34xx_ES3:
+		omap_current_dss_features = &omap3430_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP3630:
+		omap_current_dss_features = &omap3630_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP4430_ES1:
+		omap_current_dss_features = &omap4430_es1_0_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP4430_ES2:
+		omap_current_dss_features = &omap4430_es2_0_1_2_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP4:
+		omap_current_dss_features = &omap4_dss_features;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+	case OMAPDSS_VER_DRA7xx:
+		omap_current_dss_features = &omap5_dss_features;
+		break;
+
+	case OMAPDSS_VER_AM35xx:
+		omap_current_dss_features = &am35xx_dss_features;
+		break;
+
+	case OMAPDSS_VER_AM43xx:
+		omap_current_dss_features = &am43xx_dss_features;
+		break;
+
+	default:
+		DSSWARN("Unsupported OMAP version");
+		break;
+	}
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss_features.h b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.h
new file mode 100644
index 000000000000..3d67d39f192f
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.h
@@ -0,0 +1,108 @@
+/*
+ * linux/drivers/video/omap2/dss/dss_features.h
+ *
+ * Copyright (C) 2010 Texas Instruments
+ * Author: Archit Taneja <archit@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAP2_DSS_FEATURES_H
+#define __OMAP2_DSS_FEATURES_H
+
+#define MAX_DSS_MANAGERS	4
+#define MAX_DSS_OVERLAYS	4
+#define MAX_DSS_LCD_MANAGERS	3
+#define MAX_NUM_DSI		2
+
+/* DSS has feature id */
+enum dss_feat_id {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_MGR_LCD2,
+	FEAT_MGR_LCD3,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	/* Independent core clk divider */
+	FEAT_CORE_CLK_DIV,
+	FEAT_LCD_CLK_SRC,
+	/* DSI-PLL power command 0x3 is not working */
+	FEAT_DSI_PLL_PWR_BUG,
+	FEAT_DSI_DCS_CMD_CONFIG_VC,
+	FEAT_DSI_VC_OCP_WIDTH,
+	FEAT_DSI_REVERSE_TXCLKESC,
+	FEAT_DSI_GNQ,
+	FEAT_DPI_USES_VDDS_DSI,
+	FEAT_HDMI_CTS_SWMODE,
+	FEAT_HDMI_AUDIO_USE_MCLK,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_VENC_REQUIRES_TV_DAC_CLK,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	/* An unknown HW bug causing the normal FIFO thresholds not to work */
+	FEAT_OMAP3_DSI_FIFO_BUG,
+	FEAT_BURST_2D,
+	FEAT_DSI_PHY_DCC,
+	FEAT_MFLAG,
+};
+
+/* DSS register field id */
+enum dss_feat_reg_field {
+	FEAT_REG_FIRHINC,
+	FEAT_REG_FIRVINC,
+	FEAT_REG_FIFOHIGHTHRESHOLD,
+	FEAT_REG_FIFOLOWTHRESHOLD,
+	FEAT_REG_FIFOSIZE,
+	FEAT_REG_HORIZONTALACCU,
+	FEAT_REG_VERTICALACCU,
+	FEAT_REG_DISPC_CLK_SWITCH,
+};
+
+enum dss_range_param {
+	FEAT_PARAM_DSS_FCK,
+	FEAT_PARAM_DSS_PCD,
+	FEAT_PARAM_DSIPLL_LPDIV,
+	FEAT_PARAM_DSI_FCK,
+	FEAT_PARAM_DOWNSCALE,
+	FEAT_PARAM_LINEWIDTH,
+};
+
+/* DSS Feature Functions */
+unsigned long dss_feat_get_param_min(enum dss_range_param param);
+unsigned long dss_feat_get_param_max(enum dss_range_param param);
+enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane plane);
+bool dss_feat_color_mode_supported(enum omap_plane plane,
+		enum omap_color_mode color_mode);
+const char *dss_feat_get_clk_source_name(enum omap_dss_clk_source id);
+
+u32 dss_feat_get_buffer_size_unit(void);	/* in bytes */
+u32 dss_feat_get_burst_size_unit(void);		/* in bytes */
+
+bool dss_feat_rotation_type_supported(enum omap_dss_rotation_type rot_type);
+
+bool dss_has_feature(enum dss_feat_id id);
+void dss_feat_get_reg_field(enum dss_feat_reg_field id, u8 *start, u8 *end);
+void dss_features_init(enum omapdss_version version);
+
+enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel);
+enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel);
+
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h b/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h
new file mode 100644
index 000000000000..53616b02b613
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h
@@ -0,0 +1,370 @@
+/*
+ * HDMI driver definition for TI OMAP4 Processor.
+ *
+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _HDMI_H
+#define _HDMI_H
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/hdmi.h>
+#include <video/omapdss.h>
+
+#include "dss.h"
+
+/* HDMI Wrapper */
+
+#define HDMI_WP_REVISION			0x0
+#define HDMI_WP_SYSCONFIG			0x10
+#define HDMI_WP_IRQSTATUS_RAW			0x24
+#define HDMI_WP_IRQSTATUS			0x28
+#define HDMI_WP_IRQENABLE_SET			0x2C
+#define HDMI_WP_IRQENABLE_CLR			0x30
+#define HDMI_WP_IRQWAKEEN			0x34
+#define HDMI_WP_PWR_CTRL			0x40
+#define HDMI_WP_DEBOUNCE			0x44
+#define HDMI_WP_VIDEO_CFG			0x50
+#define HDMI_WP_VIDEO_SIZE			0x60
+#define HDMI_WP_VIDEO_TIMING_H			0x68
+#define HDMI_WP_VIDEO_TIMING_V			0x6C
+#define HDMI_WP_CLK				0x70
+#define HDMI_WP_AUDIO_CFG			0x80
+#define HDMI_WP_AUDIO_CFG2			0x84
+#define HDMI_WP_AUDIO_CTRL			0x88
+#define HDMI_WP_AUDIO_DATA			0x8C
+
+/* HDMI WP IRQ flags */
+#define HDMI_IRQ_CORE				(1 << 0)
+#define HDMI_IRQ_OCP_TIMEOUT			(1 << 4)
+#define HDMI_IRQ_AUDIO_FIFO_UNDERFLOW		(1 << 8)
+#define HDMI_IRQ_AUDIO_FIFO_OVERFLOW		(1 << 9)
+#define HDMI_IRQ_AUDIO_FIFO_SAMPLE_REQ		(1 << 10)
+#define HDMI_IRQ_VIDEO_VSYNC			(1 << 16)
+#define HDMI_IRQ_VIDEO_FRAME_DONE		(1 << 17)
+#define HDMI_IRQ_PHY_LINE5V_ASSERT		(1 << 24)
+#define HDMI_IRQ_LINK_CONNECT			(1 << 25)
+#define HDMI_IRQ_LINK_DISCONNECT		(1 << 26)
+#define HDMI_IRQ_PLL_LOCK			(1 << 29)
+#define HDMI_IRQ_PLL_UNLOCK			(1 << 30)
+#define HDMI_IRQ_PLL_RECAL			(1 << 31)
+
+/* HDMI PLL */
+
+#define PLLCTRL_PLL_CONTROL			0x0
+#define PLLCTRL_PLL_STATUS			0x4
+#define PLLCTRL_PLL_GO				0x8
+#define PLLCTRL_CFG1				0xC
+#define PLLCTRL_CFG2				0x10
+#define PLLCTRL_CFG3				0x14
+#define PLLCTRL_SSC_CFG1			0x18
+#define PLLCTRL_SSC_CFG2			0x1C
+#define PLLCTRL_CFG4				0x20
+
+/* HDMI PHY */
+
+#define HDMI_TXPHY_TX_CTRL			0x0
+#define HDMI_TXPHY_DIGITAL_CTRL			0x4
+#define HDMI_TXPHY_POWER_CTRL			0x8
+#define HDMI_TXPHY_PAD_CFG_CTRL			0xC
+#define HDMI_TXPHY_BIST_CONTROL			0x1C
+
+enum hdmi_pll_pwr {
+	HDMI_PLLPWRCMD_ALLOFF = 0,
+	HDMI_PLLPWRCMD_PLLONLY = 1,
+	HDMI_PLLPWRCMD_BOTHON_ALLCLKS = 2,
+	HDMI_PLLPWRCMD_BOTHON_NOPHYCLK = 3
+};
+
+enum hdmi_phy_pwr {
+	HDMI_PHYPWRCMD_OFF = 0,
+	HDMI_PHYPWRCMD_LDOON = 1,
+	HDMI_PHYPWRCMD_TXON = 2
+};
+
+enum hdmi_core_hdmi_dvi {
+	HDMI_DVI = 0,
+	HDMI_HDMI = 1
+};
+
+enum hdmi_packing_mode {
+	HDMI_PACK_10b_RGB_YUV444 = 0,
+	HDMI_PACK_24b_RGB_YUV444_YUV422 = 1,
+	HDMI_PACK_20b_YUV422 = 2,
+	HDMI_PACK_ALREADYPACKED = 7
+};
+
+enum hdmi_stereo_channels {
+	HDMI_AUDIO_STEREO_NOCHANNELS = 0,
+	HDMI_AUDIO_STEREO_ONECHANNEL = 1,
+	HDMI_AUDIO_STEREO_TWOCHANNELS = 2,
+	HDMI_AUDIO_STEREO_THREECHANNELS = 3,
+	HDMI_AUDIO_STEREO_FOURCHANNELS = 4
+};
+
+enum hdmi_audio_type {
+	HDMI_AUDIO_TYPE_LPCM = 0,
+	HDMI_AUDIO_TYPE_IEC = 1
+};
+
+enum hdmi_audio_justify {
+	HDMI_AUDIO_JUSTIFY_LEFT = 0,
+	HDMI_AUDIO_JUSTIFY_RIGHT = 1
+};
+
+enum hdmi_audio_sample_order {
+	HDMI_AUDIO_SAMPLE_RIGHT_FIRST = 0,
+	HDMI_AUDIO_SAMPLE_LEFT_FIRST = 1
+};
+
+enum hdmi_audio_samples_perword {
+	HDMI_AUDIO_ONEWORD_ONESAMPLE = 0,
+	HDMI_AUDIO_ONEWORD_TWOSAMPLES = 1
+};
+
+enum hdmi_audio_sample_size_omap {
+	HDMI_AUDIO_SAMPLE_16BITS = 0,
+	HDMI_AUDIO_SAMPLE_24BITS = 1
+};
+
+enum hdmi_audio_transf_mode {
+	HDMI_AUDIO_TRANSF_DMA = 0,
+	HDMI_AUDIO_TRANSF_IRQ = 1
+};
+
+enum hdmi_audio_blk_strt_end_sig {
+	HDMI_AUDIO_BLOCK_SIG_STARTEND_ON = 0,
+	HDMI_AUDIO_BLOCK_SIG_STARTEND_OFF = 1
+};
+
+enum hdmi_core_audio_layout {
+	HDMI_AUDIO_LAYOUT_2CH = 0,
+	HDMI_AUDIO_LAYOUT_8CH = 1,
+	HDMI_AUDIO_LAYOUT_6CH = 2
+};
+
+enum hdmi_core_cts_mode {
+	HDMI_AUDIO_CTS_MODE_HW = 0,
+	HDMI_AUDIO_CTS_MODE_SW = 1
+};
+
+enum hdmi_audio_mclk_mode {
+	HDMI_AUDIO_MCLK_128FS = 0,
+	HDMI_AUDIO_MCLK_256FS = 1,
+	HDMI_AUDIO_MCLK_384FS = 2,
+	HDMI_AUDIO_MCLK_512FS = 3,
+	HDMI_AUDIO_MCLK_768FS = 4,
+	HDMI_AUDIO_MCLK_1024FS = 5,
+	HDMI_AUDIO_MCLK_1152FS = 6,
+	HDMI_AUDIO_MCLK_192FS = 7
+};
+
+struct hdmi_video_format {
+	enum hdmi_packing_mode	packing_mode;
+	u32			y_res;	/* Line per panel */
+	u32			x_res;	/* pixel per line */
+};
+
+struct hdmi_config {
+	struct omap_video_timings timings;
+	struct hdmi_avi_infoframe infoframe;
+	enum hdmi_core_hdmi_dvi hdmi_dvi_mode;
+};
+
+struct hdmi_audio_format {
+	enum hdmi_stereo_channels		stereo_channels;
+	u8					active_chnnls_msk;
+	enum hdmi_audio_type			type;
+	enum hdmi_audio_justify			justification;
+	enum hdmi_audio_sample_order		sample_order;
+	enum hdmi_audio_samples_perword		samples_per_word;
+	enum hdmi_audio_sample_size_omap	sample_size;
+	enum hdmi_audio_blk_strt_end_sig	en_sig_blk_strt_end;
+};
+
+struct hdmi_audio_dma {
+	u8				transfer_size;
+	u8				block_size;
+	enum hdmi_audio_transf_mode	mode;
+	u16				fifo_threshold;
+};
+
+struct hdmi_core_audio_i2s_config {
+	u8 in_length_bits;
+	u8 justification;
+	u8 sck_edge_mode;
+	u8 vbit;
+	u8 direction;
+	u8 shift;
+	u8 active_sds;
+};
+
+struct hdmi_core_audio_config {
+	struct hdmi_core_audio_i2s_config	i2s_cfg;
+	struct snd_aes_iec958			*iec60958_cfg;
+	bool					fs_override;
+	u32					n;
+	u32					cts;
+	u32					aud_par_busclk;
+	enum hdmi_core_audio_layout		layout;
+	enum hdmi_core_cts_mode			cts_mode;
+	bool					use_mclk;
+	enum hdmi_audio_mclk_mode		mclk_mode;
+	bool					en_acr_pkt;
+	bool					en_dsd_audio;
+	bool					en_parallel_aud_input;
+	bool					en_spdif;
+};
+
+struct hdmi_wp_data {
+	void __iomem *base;
+	phys_addr_t phys_base;
+};
+
+struct hdmi_pll_data {
+	struct dss_pll pll;
+
+	void __iomem *base;
+
+	struct hdmi_wp_data *wp;
+};
+
+struct hdmi_phy_data {
+	void __iomem *base;
+
+	u8 lane_function[4];
+	u8 lane_polarity[4];
+};
+
+struct hdmi_core_data {
+	void __iomem *base;
+};
+
+static inline void hdmi_write_reg(void __iomem *base_addr, const u32 idx,
+		u32 val)
+{
+	__raw_writel(val, base_addr + idx);
+}
+
+static inline u32 hdmi_read_reg(void __iomem *base_addr, const u32 idx)
+{
+	return __raw_readl(base_addr + idx);
+}
+
+#define REG_FLD_MOD(base, idx, val, start, end) \
+	hdmi_write_reg(base, idx, FLD_MOD(hdmi_read_reg(base, idx),\
+							val, start, end))
+#define REG_GET(base, idx, start, end) \
+	FLD_GET(hdmi_read_reg(base, idx), start, end)
+
+static inline int hdmi_wait_for_bit_change(void __iomem *base_addr,
+		const u32 idx, int b2, int b1, u32 val)
+{
+	u32 t = 0, v;
+	while (val != (v = REG_GET(base_addr, idx, b2, b1))) {
+		if (t++ > 10000)
+			return v;
+		udelay(1);
+	}
+	return v;
+}
+
+/* HDMI wrapper funcs */
+int hdmi_wp_video_start(struct hdmi_wp_data *wp);
+void hdmi_wp_video_stop(struct hdmi_wp_data *wp);
+void hdmi_wp_dump(struct hdmi_wp_data *wp, struct seq_file *s);
+u32 hdmi_wp_get_irqstatus(struct hdmi_wp_data *wp);
+void hdmi_wp_set_irqstatus(struct hdmi_wp_data *wp, u32 irqstatus);
+void hdmi_wp_set_irqenable(struct hdmi_wp_data *wp, u32 mask);
+void hdmi_wp_clear_irqenable(struct hdmi_wp_data *wp, u32 mask);
+int hdmi_wp_set_phy_pwr(struct hdmi_wp_data *wp, enum hdmi_phy_pwr val);
+int hdmi_wp_set_pll_pwr(struct hdmi_wp_data *wp, enum hdmi_pll_pwr val);
+void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
+		struct hdmi_video_format *video_fmt);
+void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
+		struct omap_video_timings *timings);
+void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
+		struct omap_video_timings *timings);
+void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
+		struct omap_video_timings *timings, struct hdmi_config *param);
+int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp);
+phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp);
+
+/* HDMI PLL funcs */
+void hdmi_pll_dump(struct hdmi_pll_data *pll, struct seq_file *s);
+void hdmi_pll_compute(struct hdmi_pll_data *pll,
+	unsigned long target_tmds, struct dss_pll_clock_info *pi);
+int hdmi_pll_init(struct platform_device *pdev, struct hdmi_pll_data *pll,
+	struct hdmi_wp_data *wp);
+void hdmi_pll_uninit(struct hdmi_pll_data *hpll);
+
+/* HDMI PHY funcs */
+int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
+	unsigned long lfbitclk);
+void hdmi_phy_dump(struct hdmi_phy_data *phy, struct seq_file *s);
+int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy);
+int hdmi_phy_parse_lanes(struct hdmi_phy_data *phy, const u32 *lanes);
+
+/* HDMI common funcs */
+int hdmi_parse_lanes_of(struct platform_device *pdev, struct device_node *ep,
+	struct hdmi_phy_data *phy);
+
+/* Audio funcs */
+int hdmi_compute_acr(u32 pclk, u32 sample_freq, u32 *n, u32 *cts);
+int hdmi_wp_audio_enable(struct hdmi_wp_data *wp, bool enable);
+int hdmi_wp_audio_core_req_enable(struct hdmi_wp_data *wp, bool enable);
+void hdmi_wp_audio_config_format(struct hdmi_wp_data *wp,
+		struct hdmi_audio_format *aud_fmt);
+void hdmi_wp_audio_config_dma(struct hdmi_wp_data *wp,
+		struct hdmi_audio_dma *aud_dma);
+static inline bool hdmi_mode_has_audio(struct hdmi_config *cfg)
+{
+	return cfg->hdmi_dvi_mode == HDMI_HDMI ? true : false;
+}
+
+/* HDMI DRV data */
+struct omap_hdmi {
+	struct mutex lock;
+	struct platform_device *pdev;
+
+	struct hdmi_wp_data	wp;
+	struct hdmi_pll_data	pll;
+	struct hdmi_phy_data	phy;
+	struct hdmi_core_data	core;
+
+	struct hdmi_config cfg;
+
+	struct regulator *vdda_reg;
+
+	bool core_enabled;
+
+	struct omap_dss_device output;
+
+	struct platform_device *audio_pdev;
+	void (*audio_abort_cb)(struct device *dev);
+	int wp_idlemode;
+
+	bool audio_configured;
+	struct omap_dss_audio audio_config;
+
+	/* This lock should be taken when booleans bellow are touched. */
+	spinlock_t audio_playing_lock;
+	bool audio_playing;
+	bool display_enabled;
+};
+
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
new file mode 100644
index 000000000000..7103c659a534
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
@@ -0,0 +1,839 @@
+/*
+ * HDMI interface DSS driver for TI's OMAP4 family of SoCs.
+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
+ * Authors: Yong Zhi
+ *	Mythri pk <mythripk@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "HDMI"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/clk.h>
+#include <linux/gpio.h>
+#include <linux/regulator/consumer.h>
+#include <linux/component.h>
+#include <video/omapdss.h>
+#include <sound/omap-hdmi-audio.h>
+
+#include "hdmi4_core.h"
+#include "dss.h"
+#include "dss_features.h"
+#include "hdmi.h"
+
+static struct omap_hdmi hdmi;
+
+static int hdmi_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("hdmi_runtime_get\n");
+
+	r = pm_runtime_get_sync(&hdmi.pdev->dev);
+	WARN_ON(r < 0);
+	if (r < 0)
+		return r;
+
+	return 0;
+}
+
+static void hdmi_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("hdmi_runtime_put\n");
+
+	r = pm_runtime_put_sync(&hdmi.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+
+static irqreturn_t hdmi_irq_handler(int irq, void *data)
+{
+	struct hdmi_wp_data *wp = data;
+	u32 irqstatus;
+
+	irqstatus = hdmi_wp_get_irqstatus(wp);
+	hdmi_wp_set_irqstatus(wp, irqstatus);
+
+	if ((irqstatus & HDMI_IRQ_LINK_CONNECT) &&
+			irqstatus & HDMI_IRQ_LINK_DISCONNECT) {
+		/*
+		 * If we get both connect and disconnect interrupts at the same
+		 * time, turn off the PHY, clear interrupts, and restart, which
+		 * raises connect interrupt if a cable is connected, or nothing
+		 * if cable is not connected.
+		 */
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_OFF);
+
+		hdmi_wp_set_irqstatus(wp, HDMI_IRQ_LINK_CONNECT |
+				HDMI_IRQ_LINK_DISCONNECT);
+
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_LDOON);
+	} else if (irqstatus & HDMI_IRQ_LINK_CONNECT) {
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_TXON);
+	} else if (irqstatus & HDMI_IRQ_LINK_DISCONNECT) {
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_LDOON);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int hdmi_init_regulator(void)
+{
+	int r;
+	struct regulator *reg;
+
+	if (hdmi.vdda_reg != NULL)
+		return 0;
+
+	reg = devm_regulator_get(&hdmi.pdev->dev, "vdda");
+
+	if (IS_ERR(reg)) {
+		if (PTR_ERR(reg) != -EPROBE_DEFER)
+			DSSERR("can't get VDDA regulator\n");
+		return PTR_ERR(reg);
+	}
+
+	if (regulator_can_change_voltage(reg)) {
+		r = regulator_set_voltage(reg, 1800000, 1800000);
+		if (r) {
+			devm_regulator_put(reg);
+			DSSWARN("can't set the regulator voltage\n");
+			return r;
+		}
+	}
+
+	hdmi.vdda_reg = reg;
+
+	return 0;
+}
+
+static int hdmi_power_on_core(struct omap_dss_device *dssdev)
+{
+	int r;
+
+	r = regulator_enable(hdmi.vdda_reg);
+	if (r)
+		return r;
+
+	r = hdmi_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	/* Make selection of HDMI in DSS */
+	dss_select_hdmi_venc_clk_source(DSS_HDMI_M_PCLK);
+
+	hdmi.core_enabled = true;
+
+	return 0;
+
+err_runtime_get:
+	regulator_disable(hdmi.vdda_reg);
+
+	return r;
+}
+
+static void hdmi_power_off_core(struct omap_dss_device *dssdev)
+{
+	hdmi.core_enabled = false;
+
+	hdmi_runtime_put();
+	regulator_disable(hdmi.vdda_reg);
+}
+
+static int hdmi_power_on_full(struct omap_dss_device *dssdev)
+{
+	int r;
+	struct omap_video_timings *p;
+	struct omap_overlay_manager *mgr = hdmi.output.manager;
+	struct hdmi_wp_data *wp = &hdmi.wp;
+	struct dss_pll_clock_info hdmi_cinfo = { 0 };
+
+	r = hdmi_power_on_core(dssdev);
+	if (r)
+		return r;
+
+	/* disable and clear irqs */
+	hdmi_wp_clear_irqenable(wp, 0xffffffff);
+	hdmi_wp_set_irqstatus(wp, 0xffffffff);
+
+	p = &hdmi.cfg.timings;
+
+	DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res);
+
+	hdmi_pll_compute(&hdmi.pll, p->pixelclock, &hdmi_cinfo);
+
+	r = dss_pll_enable(&hdmi.pll.pll);
+	if (r) {
+		DSSERR("Failed to enable PLL\n");
+		goto err_pll_enable;
+	}
+
+	r = dss_pll_set_config(&hdmi.pll.pll, &hdmi_cinfo);
+	if (r) {
+		DSSERR("Failed to configure PLL\n");
+		goto err_pll_cfg;
+	}
+
+	r = hdmi_phy_configure(&hdmi.phy, hdmi_cinfo.clkdco,
+		hdmi_cinfo.clkout[0]);
+	if (r) {
+		DSSDBG("Failed to configure PHY\n");
+		goto err_phy_cfg;
+	}
+
+	r = hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_LDOON);
+	if (r)
+		goto err_phy_pwr;
+
+	hdmi4_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
+
+	/* bypass TV gamma table */
+	dispc_enable_gamma_table(0);
+
+	/* tv size */
+	dss_mgr_set_timings(mgr, p);
+
+	r = hdmi_wp_video_start(&hdmi.wp);
+	if (r)
+		goto err_vid_enable;
+
+	r = dss_mgr_enable(mgr);
+	if (r)
+		goto err_mgr_enable;
+
+	hdmi_wp_set_irqenable(wp,
+		HDMI_IRQ_LINK_CONNECT | HDMI_IRQ_LINK_DISCONNECT);
+
+	return 0;
+
+err_mgr_enable:
+	hdmi_wp_video_stop(&hdmi.wp);
+err_vid_enable:
+	hdmi_wp_set_phy_pwr(&hdmi.wp, HDMI_PHYPWRCMD_OFF);
+err_phy_pwr:
+err_phy_cfg:
+err_pll_cfg:
+	dss_pll_disable(&hdmi.pll.pll);
+err_pll_enable:
+	hdmi_power_off_core(dssdev);
+	return -EIO;
+}
+
+static void hdmi_power_off_full(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = hdmi.output.manager;
+
+	hdmi_wp_clear_irqenable(&hdmi.wp, 0xffffffff);
+
+	dss_mgr_disable(mgr);
+
+	hdmi_wp_video_stop(&hdmi.wp);
+
+	hdmi_wp_set_phy_pwr(&hdmi.wp, HDMI_PHYPWRCMD_OFF);
+
+	dss_pll_disable(&hdmi.pll.pll);
+
+	hdmi_power_off_core(dssdev);
+}
+
+static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
+					struct omap_video_timings *timings)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	if (!dispc_mgr_timings_ok(out->dispc_channel, timings))
+		return -EINVAL;
+
+	return 0;
+}
+
+static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	mutex_lock(&hdmi.lock);
+
+	hdmi.cfg.timings = *timings;
+
+	dispc_set_tv_pclk(timings->pixelclock);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	*timings = hdmi.cfg.timings;
+}
+
+static void hdmi_dump_regs(struct seq_file *s)
+{
+	mutex_lock(&hdmi.lock);
+
+	if (hdmi_runtime_get()) {
+		mutex_unlock(&hdmi.lock);
+		return;
+	}
+
+	hdmi_wp_dump(&hdmi.wp, s);
+	hdmi_pll_dump(&hdmi.pll, s);
+	hdmi_phy_dump(&hdmi.phy, s);
+	hdmi4_core_dump(&hdmi.core, s);
+
+	hdmi_runtime_put();
+	mutex_unlock(&hdmi.lock);
+}
+
+static int read_edid(u8 *buf, int len)
+{
+	int r;
+
+	mutex_lock(&hdmi.lock);
+
+	r = hdmi_runtime_get();
+	BUG_ON(r);
+
+	r = hdmi4_read_edid(&hdmi.core,  buf, len);
+
+	hdmi_runtime_put();
+	mutex_unlock(&hdmi.lock);
+
+	return r;
+}
+
+static void hdmi_start_audio_stream(struct omap_hdmi *hd)
+{
+	hdmi_wp_audio_enable(&hd->wp, true);
+	hdmi4_audio_start(&hd->core, &hd->wp);
+}
+
+static void hdmi_stop_audio_stream(struct omap_hdmi *hd)
+{
+	hdmi4_audio_stop(&hd->core, &hd->wp);
+	hdmi_wp_audio_enable(&hd->wp, false);
+}
+
+static int hdmi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+	unsigned long flags;
+	int r = 0;
+
+	DSSDBG("ENTER hdmi_display_enable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		r = -ENODEV;
+		goto err0;
+	}
+
+	r = hdmi_power_on_full(dssdev);
+	if (r) {
+		DSSERR("failed to power on device\n");
+		goto err0;
+	}
+
+	if (hdmi.audio_configured) {
+		r = hdmi4_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config,
+				       hdmi.cfg.timings.pixelclock);
+		if (r) {
+			DSSERR("Error restoring audio configuration: %d", r);
+			hdmi.audio_abort_cb(&hdmi.pdev->dev);
+			hdmi.audio_configured = false;
+		}
+	}
+
+	spin_lock_irqsave(&hdmi.audio_playing_lock, flags);
+	if (hdmi.audio_configured && hdmi.audio_playing)
+		hdmi_start_audio_stream(&hdmi);
+	hdmi.display_enabled = true;
+	spin_unlock_irqrestore(&hdmi.audio_playing_lock, flags);
+
+	mutex_unlock(&hdmi.lock);
+	return 0;
+
+err0:
+	mutex_unlock(&hdmi.lock);
+	return r;
+}
+
+static void hdmi_display_disable(struct omap_dss_device *dssdev)
+{
+	unsigned long flags;
+
+	DSSDBG("Enter hdmi_display_disable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	spin_lock_irqsave(&hdmi.audio_playing_lock, flags);
+	hdmi_stop_audio_stream(&hdmi);
+	hdmi.display_enabled = false;
+	spin_unlock_irqrestore(&hdmi.audio_playing_lock, flags);
+
+	hdmi_power_off_full(dssdev);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static int hdmi_core_enable(struct omap_dss_device *dssdev)
+{
+	int r = 0;
+
+	DSSDBG("ENTER omapdss_hdmi_core_enable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	r = hdmi_power_on_core(dssdev);
+	if (r) {
+		DSSERR("failed to power on device\n");
+		goto err0;
+	}
+
+	mutex_unlock(&hdmi.lock);
+	return 0;
+
+err0:
+	mutex_unlock(&hdmi.lock);
+	return r;
+}
+
+static void hdmi_core_disable(struct omap_dss_device *dssdev)
+{
+	DSSDBG("Enter omapdss_hdmi_core_disable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	hdmi_power_off_core(dssdev);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static int hdmi_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = hdmi_init_regulator();
+	if (r)
+		return r;
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dst->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void hdmi_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static int hdmi_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	bool need_enable;
+	int r;
+
+	need_enable = hdmi.core_enabled == false;
+
+	if (need_enable) {
+		r = hdmi_core_enable(dssdev);
+		if (r)
+			return r;
+	}
+
+	r = read_edid(edid, len);
+
+	if (need_enable)
+		hdmi_core_disable(dssdev);
+
+	return r;
+}
+
+static int hdmi_set_infoframe(struct omap_dss_device *dssdev,
+		const struct hdmi_avi_infoframe *avi)
+{
+	hdmi.cfg.infoframe = *avi;
+	return 0;
+}
+
+static int hdmi_set_hdmi_mode(struct omap_dss_device *dssdev,
+		bool hdmi_mode)
+{
+	hdmi.cfg.hdmi_dvi_mode = hdmi_mode ? HDMI_HDMI : HDMI_DVI;
+	return 0;
+}
+
+static const struct omapdss_hdmi_ops hdmi_ops = {
+	.connect		= hdmi_connect,
+	.disconnect		= hdmi_disconnect,
+
+	.enable			= hdmi_display_enable,
+	.disable		= hdmi_display_disable,
+
+	.check_timings		= hdmi_display_check_timing,
+	.set_timings		= hdmi_display_set_timing,
+	.get_timings		= hdmi_display_get_timings,
+
+	.read_edid		= hdmi_read_edid,
+	.set_infoframe		= hdmi_set_infoframe,
+	.set_hdmi_mode		= hdmi_set_hdmi_mode,
+};
+
+static void hdmi_init_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_HDMI;
+	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
+	out->name = "hdmi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
+	out->ops.hdmi = &hdmi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void hdmi_uninit_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	omapdss_unregister_output(out);
+}
+
+static int hdmi_probe_of(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *ep;
+	int r;
+
+	ep = omapdss_of_get_first_endpoint(node);
+	if (!ep)
+		return 0;
+
+	r = hdmi_parse_lanes_of(pdev, ep, &hdmi.phy);
+	if (r)
+		goto err;
+
+	of_node_put(ep);
+	return 0;
+
+err:
+	of_node_put(ep);
+	return r;
+}
+
+/* Audio callbacks */
+static int hdmi_audio_startup(struct device *dev,
+			      void (*abort_cb)(struct device *dev))
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	int ret = 0;
+
+	mutex_lock(&hd->lock);
+
+	if (!hdmi_mode_has_audio(&hd->cfg) || !hd->display_enabled) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	hd->audio_abort_cb = abort_cb;
+
+out:
+	mutex_unlock(&hd->lock);
+
+	return ret;
+}
+
+static int hdmi_audio_shutdown(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+
+	mutex_lock(&hd->lock);
+	hd->audio_abort_cb = NULL;
+	hd->audio_configured = false;
+	hd->audio_playing = false;
+	mutex_unlock(&hd->lock);
+
+	return 0;
+}
+
+static int hdmi_audio_start(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	WARN_ON(!hdmi_mode_has_audio(&hd->cfg));
+
+	spin_lock_irqsave(&hd->audio_playing_lock, flags);
+
+	if (hd->display_enabled)
+		hdmi_start_audio_stream(hd);
+	hd->audio_playing = true;
+
+	spin_unlock_irqrestore(&hd->audio_playing_lock, flags);
+	return 0;
+}
+
+static void hdmi_audio_stop(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	WARN_ON(!hdmi_mode_has_audio(&hd->cfg));
+
+	spin_lock_irqsave(&hd->audio_playing_lock, flags);
+
+	if (hd->display_enabled)
+		hdmi_stop_audio_stream(hd);
+	hd->audio_playing = false;
+
+	spin_unlock_irqrestore(&hd->audio_playing_lock, flags);
+}
+
+static int hdmi_audio_config(struct device *dev,
+			     struct omap_dss_audio *dss_audio)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	int ret;
+
+	mutex_lock(&hd->lock);
+
+	if (!hdmi_mode_has_audio(&hd->cfg) || !hd->display_enabled) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	ret = hdmi4_audio_config(&hd->core, &hd->wp, dss_audio,
+				 hd->cfg.timings.pixelclock);
+	if (!ret) {
+		hd->audio_configured = true;
+		hd->audio_config = *dss_audio;
+	}
+out:
+	mutex_unlock(&hd->lock);
+
+	return ret;
+}
+
+static const struct omap_hdmi_audio_ops hdmi_audio_ops = {
+	.audio_startup = hdmi_audio_startup,
+	.audio_shutdown = hdmi_audio_shutdown,
+	.audio_start = hdmi_audio_start,
+	.audio_stop = hdmi_audio_stop,
+	.audio_config = hdmi_audio_config,
+};
+
+static int hdmi_audio_register(struct device *dev)
+{
+	struct omap_hdmi_audio_pdata pdata = {
+		.dev = dev,
+		.dss_version = omapdss_get_version(),
+		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
+		.ops = &hdmi_audio_ops,
+	};
+
+	hdmi.audio_pdev = platform_device_register_data(
+		dev, "omap-hdmi-audio", PLATFORM_DEVID_AUTO,
+		&pdata, sizeof(pdata));
+
+	if (IS_ERR(hdmi.audio_pdev))
+		return PTR_ERR(hdmi.audio_pdev);
+
+	return 0;
+}
+
+/* HDMI HW IP initialisation */
+static int hdmi4_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	int r;
+	int irq;
+
+	hdmi.pdev = pdev;
+	dev_set_drvdata(&pdev->dev, &hdmi);
+
+	mutex_init(&hdmi.lock);
+	spin_lock_init(&hdmi.audio_playing_lock);
+
+	if (pdev->dev.of_node) {
+		r = hdmi_probe_of(pdev);
+		if (r)
+			return r;
+	}
+
+	r = hdmi_wp_init(pdev, &hdmi.wp);
+	if (r)
+		return r;
+
+	r = hdmi_pll_init(pdev, &hdmi.pll, &hdmi.wp);
+	if (r)
+		return r;
+
+	r = hdmi_phy_init(pdev, &hdmi.phy);
+	if (r)
+		goto err;
+
+	r = hdmi4_core_init(pdev, &hdmi.core);
+	if (r)
+		goto err;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		DSSERR("platform_get_irq failed\n");
+		r = -ENODEV;
+		goto err;
+	}
+
+	r = devm_request_threaded_irq(&pdev->dev, irq,
+			NULL, hdmi_irq_handler,
+			IRQF_ONESHOT, "OMAP HDMI", &hdmi.wp);
+	if (r) {
+		DSSERR("HDMI IRQ request failed\n");
+		goto err;
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	hdmi_init_output(pdev);
+
+	r = hdmi_audio_register(&pdev->dev);
+	if (r) {
+		DSSERR("Registering HDMI audio failed\n");
+		hdmi_uninit_output(pdev);
+		pm_runtime_disable(&pdev->dev);
+		return r;
+	}
+
+	dss_debugfs_create_file("hdmi", hdmi_dump_regs);
+
+	return 0;
+err:
+	hdmi_pll_uninit(&hdmi.pll);
+	return r;
+}
+
+static void hdmi4_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	if (hdmi.audio_pdev)
+		platform_device_unregister(hdmi.audio_pdev);
+
+	hdmi_uninit_output(pdev);
+
+	hdmi_pll_uninit(&hdmi.pll);
+
+	pm_runtime_disable(&pdev->dev);
+}
+
+static const struct component_ops hdmi4_component_ops = {
+	.bind	= hdmi4_bind,
+	.unbind	= hdmi4_unbind,
+};
+
+static int hdmi4_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &hdmi4_component_ops);
+}
+
+static int hdmi4_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &hdmi4_component_ops);
+	return 0;
+}
+
+static int hdmi_runtime_suspend(struct device *dev)
+{
+	dispc_runtime_put();
+
+	return 0;
+}
+
+static int hdmi_runtime_resume(struct device *dev)
+{
+	int r;
+
+	r = dispc_runtime_get();
+	if (r < 0)
+		return r;
+
+	return 0;
+}
+
+static const struct dev_pm_ops hdmi_pm_ops = {
+	.runtime_suspend = hdmi_runtime_suspend,
+	.runtime_resume = hdmi_runtime_resume,
+};
+
+static const struct of_device_id hdmi_of_match[] = {
+	{ .compatible = "ti,omap4-hdmi", },
+	{},
+};
+
+static struct platform_driver omapdss_hdmihw_driver = {
+	.probe		= hdmi4_probe,
+	.remove		= hdmi4_remove,
+	.driver         = {
+		.name   = "omapdss_hdmi",
+		.pm	= &hdmi_pm_ops,
+		.of_match_table = hdmi_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init hdmi4_init_platform_driver(void)
+{
+	return platform_driver_register(&omapdss_hdmihw_driver);
+}
+
+void hdmi4_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omapdss_hdmihw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c
new file mode 100644
index 000000000000..fa72e735dad2
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c
@@ -0,0 +1,904 @@
+/*
+ * ti_hdmi_4xxx_ip.c
+ *
+ * HDMI TI81xx, TI38xx, TI OMAP4 etc IP driver Library
+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
+ * Authors: Yong Zhi
+ *	Mythri pk <mythripk@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "HDMICORE"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/string.h>
+#include <linux/seq_file.h>
+#include <sound/asound.h>
+#include <sound/asoundef.h>
+
+#include "hdmi4_core.h"
+#include "dss_features.h"
+
+#define HDMI_CORE_AV		0x500
+
+static inline void __iomem *hdmi_av_base(struct hdmi_core_data *core)
+{
+	return core->base + HDMI_CORE_AV;
+}
+
+static int hdmi_core_ddc_init(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+
+	/* Turn on CLK for DDC */
+	REG_FLD_MOD(base, HDMI_CORE_AV_DPD, 0x7, 2, 0);
+
+	/* IN_PROG */
+	if (REG_GET(base, HDMI_CORE_DDC_STATUS, 4, 4) == 1) {
+		/* Abort transaction */
+		REG_FLD_MOD(base, HDMI_CORE_DDC_CMD, 0xf, 3, 0);
+		/* IN_PROG */
+		if (hdmi_wait_for_bit_change(base, HDMI_CORE_DDC_STATUS,
+					4, 4, 0) != 0) {
+			DSSERR("Timeout aborting DDC transaction\n");
+			return -ETIMEDOUT;
+		}
+	}
+
+	/* Clk SCL Devices */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_CMD, 0xA, 3, 0);
+
+	/* HDMI_CORE_DDC_STATUS_IN_PROG */
+	if (hdmi_wait_for_bit_change(base, HDMI_CORE_DDC_STATUS,
+				4, 4, 0) != 0) {
+		DSSERR("Timeout starting SCL clock\n");
+		return -ETIMEDOUT;
+	}
+
+	/* Clear FIFO */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_CMD, 0x9, 3, 0);
+
+	/* HDMI_CORE_DDC_STATUS_IN_PROG */
+	if (hdmi_wait_for_bit_change(base, HDMI_CORE_DDC_STATUS,
+				4, 4, 0) != 0) {
+		DSSERR("Timeout clearing DDC fifo\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int hdmi_core_ddc_edid(struct hdmi_core_data *core,
+		u8 *pedid, int ext)
+{
+	void __iomem *base = core->base;
+	u32 i;
+	char checksum;
+	u32 offset = 0;
+
+	/* HDMI_CORE_DDC_STATUS_IN_PROG */
+	if (hdmi_wait_for_bit_change(base, HDMI_CORE_DDC_STATUS,
+				4, 4, 0) != 0) {
+		DSSERR("Timeout waiting DDC to be ready\n");
+		return -ETIMEDOUT;
+	}
+
+	if (ext % 2 != 0)
+		offset = 0x80;
+
+	/* Load Segment Address Register */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_SEGM, ext / 2, 7, 0);
+
+	/* Load Slave Address Register */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_ADDR, 0xA0 >> 1, 7, 1);
+
+	/* Load Offset Address Register */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_OFFSET, offset, 7, 0);
+
+	/* Load Byte Count */
+	REG_FLD_MOD(base, HDMI_CORE_DDC_COUNT1, 0x80, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_DDC_COUNT2, 0x0, 1, 0);
+
+	/* Set DDC_CMD */
+	if (ext)
+		REG_FLD_MOD(base, HDMI_CORE_DDC_CMD, 0x4, 3, 0);
+	else
+		REG_FLD_MOD(base, HDMI_CORE_DDC_CMD, 0x2, 3, 0);
+
+	/* HDMI_CORE_DDC_STATUS_BUS_LOW */
+	if (REG_GET(base, HDMI_CORE_DDC_STATUS, 6, 6) == 1) {
+		DSSERR("I2C Bus Low?\n");
+		return -EIO;
+	}
+	/* HDMI_CORE_DDC_STATUS_NO_ACK */
+	if (REG_GET(base, HDMI_CORE_DDC_STATUS, 5, 5) == 1) {
+		DSSERR("I2C No Ack\n");
+		return -EIO;
+	}
+
+	for (i = 0; i < 0x80; ++i) {
+		int t;
+
+		/* IN_PROG */
+		if (REG_GET(base, HDMI_CORE_DDC_STATUS, 4, 4) == 0) {
+			DSSERR("operation stopped when reading edid\n");
+			return -EIO;
+		}
+
+		t = 0;
+		/* FIFO_EMPTY */
+		while (REG_GET(base, HDMI_CORE_DDC_STATUS, 2, 2) == 1) {
+			if (t++ > 10000) {
+				DSSERR("timeout reading edid\n");
+				return -ETIMEDOUT;
+			}
+			udelay(1);
+		}
+
+		pedid[i] = REG_GET(base, HDMI_CORE_DDC_DATA, 7, 0);
+	}
+
+	checksum = 0;
+	for (i = 0; i < 0x80; ++i)
+		checksum += pedid[i];
+
+	if (checksum != 0) {
+		DSSERR("E-EDID checksum failed!!\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int hdmi4_read_edid(struct hdmi_core_data *core, u8 *edid, int len)
+{
+	int r, l;
+
+	if (len < 128)
+		return -EINVAL;
+
+	r = hdmi_core_ddc_init(core);
+	if (r)
+		return r;
+
+	r = hdmi_core_ddc_edid(core, edid, 0);
+	if (r)
+		return r;
+
+	l = 128;
+
+	if (len >= 128 * 2 && edid[0x7e] > 0) {
+		r = hdmi_core_ddc_edid(core, edid + 0x80, 1);
+		if (r)
+			return r;
+		l += 128;
+	}
+
+	return l;
+}
+
+static void hdmi_core_init(struct hdmi_core_video_config *video_cfg)
+{
+	DSSDBG("Enter hdmi_core_init\n");
+
+	/* video core */
+	video_cfg->ip_bus_width = HDMI_INPUT_8BIT;
+	video_cfg->op_dither_truc = HDMI_OUTPUTTRUNCATION_8BIT;
+	video_cfg->deep_color_pkt = HDMI_DEEPCOLORPACKECTDISABLE;
+	video_cfg->pkt_mode = HDMI_PACKETMODERESERVEDVALUE;
+	video_cfg->hdmi_dvi = HDMI_DVI;
+	video_cfg->tclk_sel_clkmult = HDMI_FPLL10IDCK;
+}
+
+static void hdmi_core_powerdown_disable(struct hdmi_core_data *core)
+{
+	DSSDBG("Enter hdmi_core_powerdown_disable\n");
+	REG_FLD_MOD(core->base, HDMI_CORE_SYS_SYS_CTRL1, 0x0, 0, 0);
+}
+
+static void hdmi_core_swreset_release(struct hdmi_core_data *core)
+{
+	DSSDBG("Enter hdmi_core_swreset_release\n");
+	REG_FLD_MOD(core->base, HDMI_CORE_SYS_SRST, 0x0, 0, 0);
+}
+
+static void hdmi_core_swreset_assert(struct hdmi_core_data *core)
+{
+	DSSDBG("Enter hdmi_core_swreset_assert\n");
+	REG_FLD_MOD(core->base, HDMI_CORE_SYS_SRST, 0x1, 0, 0);
+}
+
+/* HDMI_CORE_VIDEO_CONFIG */
+static void hdmi_core_video_config(struct hdmi_core_data *core,
+				struct hdmi_core_video_config *cfg)
+{
+	u32 r = 0;
+	void __iomem *core_sys_base = core->base;
+	void __iomem *core_av_base = hdmi_av_base(core);
+
+	/* sys_ctrl1 default configuration not tunable */
+	r = hdmi_read_reg(core_sys_base, HDMI_CORE_SYS_SYS_CTRL1);
+	r = FLD_MOD(r, HDMI_CORE_SYS_SYS_CTRL1_VEN_FOLLOWVSYNC, 5, 5);
+	r = FLD_MOD(r, HDMI_CORE_SYS_SYS_CTRL1_HEN_FOLLOWHSYNC, 4, 4);
+	r = FLD_MOD(r, HDMI_CORE_SYS_SYS_CTRL1_BSEL_24BITBUS, 2, 2);
+	r = FLD_MOD(r, HDMI_CORE_SYS_SYS_CTRL1_EDGE_RISINGEDGE, 1, 1);
+	hdmi_write_reg(core_sys_base, HDMI_CORE_SYS_SYS_CTRL1, r);
+
+	REG_FLD_MOD(core_sys_base,
+			HDMI_CORE_SYS_VID_ACEN, cfg->ip_bus_width, 7, 6);
+
+	/* Vid_Mode */
+	r = hdmi_read_reg(core_sys_base, HDMI_CORE_SYS_VID_MODE);
+
+	/* dither truncation configuration */
+	if (cfg->op_dither_truc > HDMI_OUTPUTTRUNCATION_12BIT) {
+		r = FLD_MOD(r, cfg->op_dither_truc - 3, 7, 6);
+		r = FLD_MOD(r, 1, 5, 5);
+	} else {
+		r = FLD_MOD(r, cfg->op_dither_truc, 7, 6);
+		r = FLD_MOD(r, 0, 5, 5);
+	}
+	hdmi_write_reg(core_sys_base, HDMI_CORE_SYS_VID_MODE, r);
+
+	/* HDMI_Ctrl */
+	r = hdmi_read_reg(core_av_base, HDMI_CORE_AV_HDMI_CTRL);
+	r = FLD_MOD(r, cfg->deep_color_pkt, 6, 6);
+	r = FLD_MOD(r, cfg->pkt_mode, 5, 3);
+	r = FLD_MOD(r, cfg->hdmi_dvi, 0, 0);
+	hdmi_write_reg(core_av_base, HDMI_CORE_AV_HDMI_CTRL, r);
+
+	/* TMDS_CTRL */
+	REG_FLD_MOD(core_sys_base,
+			HDMI_CORE_SYS_TMDS_CTRL, cfg->tclk_sel_clkmult, 6, 5);
+}
+
+static void hdmi_core_write_avi_infoframe(struct hdmi_core_data *core,
+	struct hdmi_avi_infoframe *frame)
+{
+	void __iomem *av_base = hdmi_av_base(core);
+	u8 data[HDMI_INFOFRAME_SIZE(AVI)];
+	int i;
+
+	hdmi_avi_infoframe_pack(frame, data, sizeof(data));
+
+	print_hex_dump_debug("AVI: ", DUMP_PREFIX_NONE, 16, 1, data,
+		HDMI_INFOFRAME_SIZE(AVI), false);
+
+	for (i = 0; i < sizeof(data); ++i) {
+		hdmi_write_reg(av_base, HDMI_CORE_AV_AVI_BASE + i * 4,
+			data[i]);
+	}
+}
+
+static void hdmi_core_av_packet_config(struct hdmi_core_data *core,
+		struct hdmi_core_packet_enable_repeat repeat_cfg)
+{
+	/* enable/repeat the infoframe */
+	hdmi_write_reg(hdmi_av_base(core), HDMI_CORE_AV_PB_CTRL1,
+		(repeat_cfg.audio_pkt << 5) |
+		(repeat_cfg.audio_pkt_repeat << 4) |
+		(repeat_cfg.avi_infoframe << 1) |
+		(repeat_cfg.avi_infoframe_repeat));
+
+	/* enable/repeat the packet */
+	hdmi_write_reg(hdmi_av_base(core), HDMI_CORE_AV_PB_CTRL2,
+		(repeat_cfg.gen_cntrl_pkt << 3) |
+		(repeat_cfg.gen_cntrl_pkt_repeat << 2) |
+		(repeat_cfg.generic_pkt << 1) |
+		(repeat_cfg.generic_pkt_repeat));
+}
+
+void hdmi4_configure(struct hdmi_core_data *core,
+	struct hdmi_wp_data *wp, struct hdmi_config *cfg)
+{
+	/* HDMI */
+	struct omap_video_timings video_timing;
+	struct hdmi_video_format video_format;
+	/* HDMI core */
+	struct hdmi_core_video_config v_core_cfg;
+	struct hdmi_core_packet_enable_repeat repeat_cfg = { 0 };
+
+	hdmi_core_init(&v_core_cfg);
+
+	hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg);
+
+	hdmi_wp_video_config_timing(wp, &video_timing);
+
+	/* video config */
+	video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422;
+
+	hdmi_wp_video_config_format(wp, &video_format);
+
+	hdmi_wp_video_config_interface(wp, &video_timing);
+
+	/*
+	 * configure core video part
+	 * set software reset in the core
+	 */
+	hdmi_core_swreset_assert(core);
+
+	/* power down off */
+	hdmi_core_powerdown_disable(core);
+
+	v_core_cfg.pkt_mode = HDMI_PACKETMODE24BITPERPIXEL;
+	v_core_cfg.hdmi_dvi = cfg->hdmi_dvi_mode;
+
+	hdmi_core_video_config(core, &v_core_cfg);
+
+	/* release software reset in the core */
+	hdmi_core_swreset_release(core);
+
+	if (cfg->hdmi_dvi_mode == HDMI_HDMI) {
+		hdmi_core_write_avi_infoframe(core, &cfg->infoframe);
+
+		/* enable/repeat the infoframe */
+		repeat_cfg.avi_infoframe = HDMI_PACKETENABLE;
+		repeat_cfg.avi_infoframe_repeat = HDMI_PACKETREPEATON;
+		/* wakeup */
+		repeat_cfg.audio_pkt = HDMI_PACKETENABLE;
+		repeat_cfg.audio_pkt_repeat = HDMI_PACKETREPEATON;
+	}
+
+	hdmi_core_av_packet_config(core, repeat_cfg);
+}
+
+void hdmi4_core_dump(struct hdmi_core_data *core, struct seq_file *s)
+{
+	int i;
+
+#define CORE_REG(i, name) name(i)
+#define DUMPCORE(r) seq_printf(s, "%-35s %08x\n", #r,\
+		hdmi_read_reg(core->base, r))
+#define DUMPCOREAV(r) seq_printf(s, "%-35s %08x\n", #r,\
+		hdmi_read_reg(hdmi_av_base(core), r))
+#define DUMPCOREAV2(i, r) seq_printf(s, "%s[%d]%*s %08x\n", #r, i, \
+		(i < 10) ? 32 - (int)strlen(#r) : 31 - (int)strlen(#r), " ", \
+		hdmi_read_reg(hdmi_av_base(core), CORE_REG(i, r)))
+
+	DUMPCORE(HDMI_CORE_SYS_VND_IDL);
+	DUMPCORE(HDMI_CORE_SYS_DEV_IDL);
+	DUMPCORE(HDMI_CORE_SYS_DEV_IDH);
+	DUMPCORE(HDMI_CORE_SYS_DEV_REV);
+	DUMPCORE(HDMI_CORE_SYS_SRST);
+	DUMPCORE(HDMI_CORE_SYS_SYS_CTRL1);
+	DUMPCORE(HDMI_CORE_SYS_SYS_STAT);
+	DUMPCORE(HDMI_CORE_SYS_SYS_CTRL3);
+	DUMPCORE(HDMI_CORE_SYS_DE_DLY);
+	DUMPCORE(HDMI_CORE_SYS_DE_CTRL);
+	DUMPCORE(HDMI_CORE_SYS_DE_TOP);
+	DUMPCORE(HDMI_CORE_SYS_DE_CNTL);
+	DUMPCORE(HDMI_CORE_SYS_DE_CNTH);
+	DUMPCORE(HDMI_CORE_SYS_DE_LINL);
+	DUMPCORE(HDMI_CORE_SYS_DE_LINH_1);
+	DUMPCORE(HDMI_CORE_SYS_HRES_L);
+	DUMPCORE(HDMI_CORE_SYS_HRES_H);
+	DUMPCORE(HDMI_CORE_SYS_VRES_L);
+	DUMPCORE(HDMI_CORE_SYS_VRES_H);
+	DUMPCORE(HDMI_CORE_SYS_IADJUST);
+	DUMPCORE(HDMI_CORE_SYS_POLDETECT);
+	DUMPCORE(HDMI_CORE_SYS_HWIDTH1);
+	DUMPCORE(HDMI_CORE_SYS_HWIDTH2);
+	DUMPCORE(HDMI_CORE_SYS_VWIDTH);
+	DUMPCORE(HDMI_CORE_SYS_VID_CTRL);
+	DUMPCORE(HDMI_CORE_SYS_VID_ACEN);
+	DUMPCORE(HDMI_CORE_SYS_VID_MODE);
+	DUMPCORE(HDMI_CORE_SYS_VID_BLANK1);
+	DUMPCORE(HDMI_CORE_SYS_VID_BLANK3);
+	DUMPCORE(HDMI_CORE_SYS_VID_BLANK1);
+	DUMPCORE(HDMI_CORE_SYS_DC_HEADER);
+	DUMPCORE(HDMI_CORE_SYS_VID_DITHER);
+	DUMPCORE(HDMI_CORE_SYS_RGB2XVYCC_CT);
+	DUMPCORE(HDMI_CORE_SYS_R2Y_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_R2Y_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_G2Y_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_G2Y_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_B2Y_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_B2Y_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_R2CB_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_R2CB_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_G2CB_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_G2CB_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_B2CB_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_B2CB_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_R2CR_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_R2CR_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_G2CR_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_G2CR_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_B2CR_COEFF_LOW);
+	DUMPCORE(HDMI_CORE_SYS_B2CR_COEFF_UP);
+	DUMPCORE(HDMI_CORE_SYS_RGB_OFFSET_LOW);
+	DUMPCORE(HDMI_CORE_SYS_RGB_OFFSET_UP);
+	DUMPCORE(HDMI_CORE_SYS_Y_OFFSET_LOW);
+	DUMPCORE(HDMI_CORE_SYS_Y_OFFSET_UP);
+	DUMPCORE(HDMI_CORE_SYS_CBCR_OFFSET_LOW);
+	DUMPCORE(HDMI_CORE_SYS_CBCR_OFFSET_UP);
+	DUMPCORE(HDMI_CORE_SYS_INTR_STATE);
+	DUMPCORE(HDMI_CORE_SYS_INTR1);
+	DUMPCORE(HDMI_CORE_SYS_INTR2);
+	DUMPCORE(HDMI_CORE_SYS_INTR3);
+	DUMPCORE(HDMI_CORE_SYS_INTR4);
+	DUMPCORE(HDMI_CORE_SYS_INTR_UNMASK1);
+	DUMPCORE(HDMI_CORE_SYS_INTR_UNMASK2);
+	DUMPCORE(HDMI_CORE_SYS_INTR_UNMASK3);
+	DUMPCORE(HDMI_CORE_SYS_INTR_UNMASK4);
+	DUMPCORE(HDMI_CORE_SYS_INTR_CTRL);
+	DUMPCORE(HDMI_CORE_SYS_TMDS_CTRL);
+
+	DUMPCORE(HDMI_CORE_DDC_ADDR);
+	DUMPCORE(HDMI_CORE_DDC_SEGM);
+	DUMPCORE(HDMI_CORE_DDC_OFFSET);
+	DUMPCORE(HDMI_CORE_DDC_COUNT1);
+	DUMPCORE(HDMI_CORE_DDC_COUNT2);
+	DUMPCORE(HDMI_CORE_DDC_STATUS);
+	DUMPCORE(HDMI_CORE_DDC_CMD);
+	DUMPCORE(HDMI_CORE_DDC_DATA);
+
+	DUMPCOREAV(HDMI_CORE_AV_ACR_CTRL);
+	DUMPCOREAV(HDMI_CORE_AV_FREQ_SVAL);
+	DUMPCOREAV(HDMI_CORE_AV_N_SVAL1);
+	DUMPCOREAV(HDMI_CORE_AV_N_SVAL2);
+	DUMPCOREAV(HDMI_CORE_AV_N_SVAL3);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_SVAL1);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_SVAL2);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_SVAL3);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_HVAL1);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_HVAL2);
+	DUMPCOREAV(HDMI_CORE_AV_CTS_HVAL3);
+	DUMPCOREAV(HDMI_CORE_AV_AUD_MODE);
+	DUMPCOREAV(HDMI_CORE_AV_SPDIF_CTRL);
+	DUMPCOREAV(HDMI_CORE_AV_HW_SPDIF_FS);
+	DUMPCOREAV(HDMI_CORE_AV_SWAP_I2S);
+	DUMPCOREAV(HDMI_CORE_AV_SPDIF_ERTH);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_IN_MAP);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_IN_CTRL);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_CHST0);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_CHST1);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_CHST2);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_CHST4);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_CHST5);
+	DUMPCOREAV(HDMI_CORE_AV_ASRC);
+	DUMPCOREAV(HDMI_CORE_AV_I2S_IN_LEN);
+	DUMPCOREAV(HDMI_CORE_AV_HDMI_CTRL);
+	DUMPCOREAV(HDMI_CORE_AV_AUDO_TXSTAT);
+	DUMPCOREAV(HDMI_CORE_AV_AUD_PAR_BUSCLK_1);
+	DUMPCOREAV(HDMI_CORE_AV_AUD_PAR_BUSCLK_2);
+	DUMPCOREAV(HDMI_CORE_AV_AUD_PAR_BUSCLK_3);
+	DUMPCOREAV(HDMI_CORE_AV_TEST_TXCTRL);
+	DUMPCOREAV(HDMI_CORE_AV_DPD);
+	DUMPCOREAV(HDMI_CORE_AV_PB_CTRL1);
+	DUMPCOREAV(HDMI_CORE_AV_PB_CTRL2);
+	DUMPCOREAV(HDMI_CORE_AV_AVI_TYPE);
+	DUMPCOREAV(HDMI_CORE_AV_AVI_VERS);
+	DUMPCOREAV(HDMI_CORE_AV_AVI_LEN);
+	DUMPCOREAV(HDMI_CORE_AV_AVI_CHSUM);
+
+	for (i = 0; i < HDMI_CORE_AV_AVI_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_AVI_DBYTE);
+
+	DUMPCOREAV(HDMI_CORE_AV_SPD_TYPE);
+	DUMPCOREAV(HDMI_CORE_AV_SPD_VERS);
+	DUMPCOREAV(HDMI_CORE_AV_SPD_LEN);
+	DUMPCOREAV(HDMI_CORE_AV_SPD_CHSUM);
+
+	for (i = 0; i < HDMI_CORE_AV_SPD_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_SPD_DBYTE);
+
+	DUMPCOREAV(HDMI_CORE_AV_AUDIO_TYPE);
+	DUMPCOREAV(HDMI_CORE_AV_AUDIO_VERS);
+	DUMPCOREAV(HDMI_CORE_AV_AUDIO_LEN);
+	DUMPCOREAV(HDMI_CORE_AV_AUDIO_CHSUM);
+
+	for (i = 0; i < HDMI_CORE_AV_AUD_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_AUD_DBYTE);
+
+	DUMPCOREAV(HDMI_CORE_AV_MPEG_TYPE);
+	DUMPCOREAV(HDMI_CORE_AV_MPEG_VERS);
+	DUMPCOREAV(HDMI_CORE_AV_MPEG_LEN);
+	DUMPCOREAV(HDMI_CORE_AV_MPEG_CHSUM);
+
+	for (i = 0; i < HDMI_CORE_AV_MPEG_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_MPEG_DBYTE);
+
+	for (i = 0; i < HDMI_CORE_AV_GEN_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_GEN_DBYTE);
+
+	DUMPCOREAV(HDMI_CORE_AV_CP_BYTE1);
+
+	for (i = 0; i < HDMI_CORE_AV_GEN2_DBYTE_NELEMS; i++)
+		DUMPCOREAV2(i, HDMI_CORE_AV_GEN2_DBYTE);
+
+	DUMPCOREAV(HDMI_CORE_AV_CEC_ADDR_ID);
+}
+
+static void hdmi_core_audio_config(struct hdmi_core_data *core,
+					struct hdmi_core_audio_config *cfg)
+{
+	u32 r;
+	void __iomem *av_base = hdmi_av_base(core);
+
+	/*
+	 * Parameters for generation of Audio Clock Recovery packets
+	 */
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_N_SVAL1, cfg->n, 7, 0);
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_N_SVAL2, cfg->n >> 8, 7, 0);
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_N_SVAL3, cfg->n >> 16, 7, 0);
+
+	if (cfg->cts_mode == HDMI_AUDIO_CTS_MODE_SW) {
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_CTS_SVAL1, cfg->cts, 7, 0);
+		REG_FLD_MOD(av_base,
+				HDMI_CORE_AV_CTS_SVAL2, cfg->cts >> 8, 7, 0);
+		REG_FLD_MOD(av_base,
+				HDMI_CORE_AV_CTS_SVAL3, cfg->cts >> 16, 7, 0);
+	} else {
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_AUD_PAR_BUSCLK_1,
+				cfg->aud_par_busclk, 7, 0);
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_AUD_PAR_BUSCLK_2,
+				(cfg->aud_par_busclk >> 8), 7, 0);
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_AUD_PAR_BUSCLK_3,
+				(cfg->aud_par_busclk >> 16), 7, 0);
+	}
+
+	/* Set ACR clock divisor */
+	REG_FLD_MOD(av_base,
+			HDMI_CORE_AV_FREQ_SVAL, cfg->mclk_mode, 2, 0);
+
+	r = hdmi_read_reg(av_base, HDMI_CORE_AV_ACR_CTRL);
+	/*
+	 * Use TMDS clock for ACR packets. For devices that use
+	 * the MCLK, this is the first part of the MCLK initialization.
+	 */
+	r = FLD_MOD(r, 0, 2, 2);
+
+	r = FLD_MOD(r, cfg->en_acr_pkt, 1, 1);
+	r = FLD_MOD(r, cfg->cts_mode, 0, 0);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_ACR_CTRL, r);
+
+	/* For devices using MCLK, this completes its initialization. */
+	if (cfg->use_mclk)
+		REG_FLD_MOD(av_base, HDMI_CORE_AV_ACR_CTRL, 1, 2, 2);
+
+	/* Override of SPDIF sample frequency with value in I2S_CHST4 */
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_SPDIF_CTRL,
+						cfg->fs_override, 1, 1);
+
+	/*
+	 * Set IEC-60958-3 channel status word. It is passed to the IP
+	 * just as it is received. The user of the driver is responsible
+	 * for its contents.
+	 */
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_CHST0,
+		       cfg->iec60958_cfg->status[0]);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_CHST1,
+		       cfg->iec60958_cfg->status[1]);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_CHST2,
+		       cfg->iec60958_cfg->status[2]);
+	/* yes, this is correct: status[3] goes to CHST4 register */
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_CHST4,
+		       cfg->iec60958_cfg->status[3]);
+	/* yes, this is correct: status[4] goes to CHST5 register */
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_CHST5,
+		       cfg->iec60958_cfg->status[4]);
+
+	/* set I2S parameters */
+	r = hdmi_read_reg(av_base, HDMI_CORE_AV_I2S_IN_CTRL);
+	r = FLD_MOD(r, cfg->i2s_cfg.sck_edge_mode, 6, 6);
+	r = FLD_MOD(r, cfg->i2s_cfg.vbit, 4, 4);
+	r = FLD_MOD(r, cfg->i2s_cfg.justification, 2, 2);
+	r = FLD_MOD(r, cfg->i2s_cfg.direction, 1, 1);
+	r = FLD_MOD(r, cfg->i2s_cfg.shift, 0, 0);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_IN_CTRL, r);
+
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_I2S_IN_LEN,
+			cfg->i2s_cfg.in_length_bits, 3, 0);
+
+	/* Audio channels and mode parameters */
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_HDMI_CTRL, cfg->layout, 2, 1);
+	r = hdmi_read_reg(av_base, HDMI_CORE_AV_AUD_MODE);
+	r = FLD_MOD(r, cfg->i2s_cfg.active_sds, 7, 4);
+	r = FLD_MOD(r, cfg->en_dsd_audio, 3, 3);
+	r = FLD_MOD(r, cfg->en_parallel_aud_input, 2, 2);
+	r = FLD_MOD(r, cfg->en_spdif, 1, 1);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_MODE, r);
+
+	/* Audio channel mappings */
+	/* TODO: Make channel mapping dynamic. For now, map channels
+	 * in the ALSA order: FL/FR/RL/RR/C/LFE/SL/SR. Remapping is needed as
+	 * HDMI speaker order is different. See CEA-861 Section 6.6.2.
+	 */
+	hdmi_write_reg(av_base, HDMI_CORE_AV_I2S_IN_MAP, 0x78);
+	REG_FLD_MOD(av_base, HDMI_CORE_AV_SWAP_I2S, 1, 5, 5);
+}
+
+static void hdmi_core_audio_infoframe_cfg(struct hdmi_core_data *core,
+		struct snd_cea_861_aud_if *info_aud)
+{
+	u8 sum = 0, checksum = 0;
+	void __iomem *av_base = hdmi_av_base(core);
+
+	/*
+	 * Set audio info frame type, version and length as
+	 * described in HDMI 1.4a Section 8.2.2 specification.
+	 * Checksum calculation is defined in Section 5.3.5.
+	 */
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUDIO_TYPE, 0x84);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUDIO_VERS, 0x01);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUDIO_LEN, 0x0a);
+	sum += 0x84 + 0x001 + 0x00a;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(0),
+		       info_aud->db1_ct_cc);
+	sum += info_aud->db1_ct_cc;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(1),
+		       info_aud->db2_sf_ss);
+	sum += info_aud->db2_sf_ss;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(2), info_aud->db3);
+	sum += info_aud->db3;
+
+	/*
+	 * The OMAP HDMI IP requires to use the 8-channel channel code when
+	 * transmitting more than two channels.
+	 */
+	if (info_aud->db4_ca != 0x00)
+		info_aud->db4_ca = 0x13;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(3), info_aud->db4_ca);
+	sum += info_aud->db4_ca;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(4),
+		       info_aud->db5_dminh_lsv);
+	sum += info_aud->db5_dminh_lsv;
+
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(5), 0x00);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(6), 0x00);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(7), 0x00);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(8), 0x00);
+	hdmi_write_reg(av_base, HDMI_CORE_AV_AUD_DBYTE(9), 0x00);
+
+	checksum = 0x100 - sum;
+	hdmi_write_reg(av_base,
+					HDMI_CORE_AV_AUDIO_CHSUM, checksum);
+
+	/*
+	 * TODO: Add MPEG and SPD enable and repeat cfg when EDID parsing
+	 * is available.
+	 */
+}
+
+int hdmi4_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+		struct omap_dss_audio *audio, u32 pclk)
+{
+	struct hdmi_audio_format audio_format;
+	struct hdmi_audio_dma audio_dma;
+	struct hdmi_core_audio_config acore;
+	int err, n, cts, channel_count;
+	unsigned int fs_nr;
+	bool word_length_16b = false;
+
+	if (!audio || !audio->iec || !audio->cea || !core)
+		return -EINVAL;
+
+	acore.iec60958_cfg = audio->iec;
+	/*
+	 * In the IEC-60958 status word, check if the audio sample word length
+	 * is 16-bit as several optimizations can be performed in such case.
+	 */
+	if (!(audio->iec->status[4] & IEC958_AES4_CON_MAX_WORDLEN_24))
+		if (audio->iec->status[4] & IEC958_AES4_CON_WORDLEN_20_16)
+			word_length_16b = true;
+
+	/* I2S configuration. See Phillips' specification */
+	if (word_length_16b)
+		acore.i2s_cfg.justification = HDMI_AUDIO_JUSTIFY_LEFT;
+	else
+		acore.i2s_cfg.justification = HDMI_AUDIO_JUSTIFY_RIGHT;
+	/*
+	 * The I2S input word length is twice the lenght given in the IEC-60958
+	 * status word. If the word size is greater than
+	 * 20 bits, increment by one.
+	 */
+	acore.i2s_cfg.in_length_bits = audio->iec->status[4]
+		& IEC958_AES4_CON_WORDLEN;
+	if (audio->iec->status[4] & IEC958_AES4_CON_MAX_WORDLEN_24)
+		acore.i2s_cfg.in_length_bits++;
+	acore.i2s_cfg.sck_edge_mode = HDMI_AUDIO_I2S_SCK_EDGE_RISING;
+	acore.i2s_cfg.vbit = HDMI_AUDIO_I2S_VBIT_FOR_PCM;
+	acore.i2s_cfg.direction = HDMI_AUDIO_I2S_MSB_SHIFTED_FIRST;
+	acore.i2s_cfg.shift = HDMI_AUDIO_I2S_FIRST_BIT_SHIFT;
+
+	/* convert sample frequency to a number */
+	switch (audio->iec->status[3] & IEC958_AES3_CON_FS) {
+	case IEC958_AES3_CON_FS_32000:
+		fs_nr = 32000;
+		break;
+	case IEC958_AES3_CON_FS_44100:
+		fs_nr = 44100;
+		break;
+	case IEC958_AES3_CON_FS_48000:
+		fs_nr = 48000;
+		break;
+	case IEC958_AES3_CON_FS_88200:
+		fs_nr = 88200;
+		break;
+	case IEC958_AES3_CON_FS_96000:
+		fs_nr = 96000;
+		break;
+	case IEC958_AES3_CON_FS_176400:
+		fs_nr = 176400;
+		break;
+	case IEC958_AES3_CON_FS_192000:
+		fs_nr = 192000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	err = hdmi_compute_acr(pclk, fs_nr, &n, &cts);
+
+	/* Audio clock regeneration settings */
+	acore.n = n;
+	acore.cts = cts;
+	if (dss_has_feature(FEAT_HDMI_CTS_SWMODE)) {
+		acore.aud_par_busclk = 0;
+		acore.cts_mode = HDMI_AUDIO_CTS_MODE_SW;
+		acore.use_mclk = dss_has_feature(FEAT_HDMI_AUDIO_USE_MCLK);
+	} else {
+		acore.aud_par_busclk = (((128 * 31) - 1) << 8);
+		acore.cts_mode = HDMI_AUDIO_CTS_MODE_HW;
+		acore.use_mclk = true;
+	}
+
+	if (acore.use_mclk)
+		acore.mclk_mode = HDMI_AUDIO_MCLK_128FS;
+
+	/* Audio channels settings */
+	channel_count = (audio->cea->db1_ct_cc &
+			 CEA861_AUDIO_INFOFRAME_DB1CC) + 1;
+
+	switch (channel_count) {
+	case 2:
+		audio_format.active_chnnls_msk = 0x03;
+		break;
+	case 3:
+		audio_format.active_chnnls_msk = 0x07;
+		break;
+	case 4:
+		audio_format.active_chnnls_msk = 0x0f;
+		break;
+	case 5:
+		audio_format.active_chnnls_msk = 0x1f;
+		break;
+	case 6:
+		audio_format.active_chnnls_msk = 0x3f;
+		break;
+	case 7:
+		audio_format.active_chnnls_msk = 0x7f;
+		break;
+	case 8:
+		audio_format.active_chnnls_msk = 0xff;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/*
+	 * the HDMI IP needs to enable four stereo channels when transmitting
+	 * more than 2 audio channels.  Similarly, the channel count in the
+	 * Audio InfoFrame has to match the sample_present bits (some channels
+	 * are padded with zeroes)
+	 */
+	if (channel_count == 2) {
+		audio_format.stereo_channels = HDMI_AUDIO_STEREO_ONECHANNEL;
+		acore.i2s_cfg.active_sds = HDMI_AUDIO_I2S_SD0_EN;
+		acore.layout = HDMI_AUDIO_LAYOUT_2CH;
+	} else {
+		audio_format.stereo_channels = HDMI_AUDIO_STEREO_FOURCHANNELS;
+		acore.i2s_cfg.active_sds = HDMI_AUDIO_I2S_SD0_EN |
+				HDMI_AUDIO_I2S_SD1_EN | HDMI_AUDIO_I2S_SD2_EN |
+				HDMI_AUDIO_I2S_SD3_EN;
+		acore.layout = HDMI_AUDIO_LAYOUT_8CH;
+		audio->cea->db1_ct_cc = 7;
+	}
+
+	acore.en_spdif = false;
+	/* use sample frequency from channel status word */
+	acore.fs_override = true;
+	/* enable ACR packets */
+	acore.en_acr_pkt = true;
+	/* disable direct streaming digital audio */
+	acore.en_dsd_audio = false;
+	/* use parallel audio interface */
+	acore.en_parallel_aud_input = true;
+
+	/* DMA settings */
+	if (word_length_16b)
+		audio_dma.transfer_size = 0x10;
+	else
+		audio_dma.transfer_size = 0x20;
+	audio_dma.block_size = 0xC0;
+	audio_dma.mode = HDMI_AUDIO_TRANSF_DMA;
+	audio_dma.fifo_threshold = 0x20; /* in number of samples */
+
+	/* audio FIFO format settings */
+	if (word_length_16b) {
+		audio_format.samples_per_word = HDMI_AUDIO_ONEWORD_TWOSAMPLES;
+		audio_format.sample_size = HDMI_AUDIO_SAMPLE_16BITS;
+		audio_format.justification = HDMI_AUDIO_JUSTIFY_LEFT;
+	} else {
+		audio_format.samples_per_word = HDMI_AUDIO_ONEWORD_ONESAMPLE;
+		audio_format.sample_size = HDMI_AUDIO_SAMPLE_24BITS;
+		audio_format.justification = HDMI_AUDIO_JUSTIFY_RIGHT;
+	}
+	audio_format.type = HDMI_AUDIO_TYPE_LPCM;
+	audio_format.sample_order = HDMI_AUDIO_SAMPLE_LEFT_FIRST;
+	/* disable start/stop signals of IEC 60958 blocks */
+	audio_format.en_sig_blk_strt_end = HDMI_AUDIO_BLOCK_SIG_STARTEND_ON;
+
+	/* configure DMA and audio FIFO format*/
+	hdmi_wp_audio_config_dma(wp, &audio_dma);
+	hdmi_wp_audio_config_format(wp, &audio_format);
+
+	/* configure the core*/
+	hdmi_core_audio_config(core, &acore);
+
+	/* configure CEA 861 audio infoframe*/
+	hdmi_core_audio_infoframe_cfg(core, audio->cea);
+
+	return 0;
+}
+
+int hdmi4_audio_start(struct hdmi_core_data *core, struct hdmi_wp_data *wp)
+{
+	REG_FLD_MOD(hdmi_av_base(core),
+		    HDMI_CORE_AV_AUD_MODE, true, 0, 0);
+
+	hdmi_wp_audio_core_req_enable(wp, true);
+
+	return 0;
+}
+
+void hdmi4_audio_stop(struct hdmi_core_data *core, struct hdmi_wp_data *wp)
+{
+	REG_FLD_MOD(hdmi_av_base(core),
+		    HDMI_CORE_AV_AUD_MODE, false, 0, 0);
+
+	hdmi_wp_audio_core_req_enable(wp, false);
+}
+
+int hdmi4_core_init(struct platform_device *pdev, struct hdmi_core_data *core)
+{
+	struct resource *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core");
+	if (!res) {
+		DSSERR("can't get CORE mem resource\n");
+		return -EINVAL;
+	}
+
+	core->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(core->base)) {
+		DSSERR("can't ioremap CORE\n");
+		return PTR_ERR(core->base);
+	}
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.h b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.h
new file mode 100644
index 000000000000..a069f96ec6f6
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.h
@@ -0,0 +1,273 @@
+/*
+ * HDMI header definition for OMAP4 HDMI core IP
+ *
+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _HDMI4_CORE_H_
+#define _HDMI4_CORE_H_
+
+#include "hdmi.h"
+
+/* OMAP4 HDMI IP Core System */
+
+#define HDMI_CORE_SYS_VND_IDL			0x0
+#define HDMI_CORE_SYS_DEV_IDL			0x8
+#define HDMI_CORE_SYS_DEV_IDH			0xC
+#define HDMI_CORE_SYS_DEV_REV			0x10
+#define HDMI_CORE_SYS_SRST			0x14
+#define HDMI_CORE_SYS_SYS_CTRL1			0x20
+#define HDMI_CORE_SYS_SYS_STAT			0x24
+#define HDMI_CORE_SYS_SYS_CTRL3			0x28
+#define HDMI_CORE_SYS_DCTL			0x34
+#define HDMI_CORE_SYS_DE_DLY			0xC8
+#define HDMI_CORE_SYS_DE_CTRL			0xCC
+#define HDMI_CORE_SYS_DE_TOP			0xD0
+#define HDMI_CORE_SYS_DE_CNTL			0xD8
+#define HDMI_CORE_SYS_DE_CNTH			0xDC
+#define HDMI_CORE_SYS_DE_LINL			0xE0
+#define HDMI_CORE_SYS_DE_LINH_1			0xE4
+#define HDMI_CORE_SYS_HRES_L			0xE8
+#define HDMI_CORE_SYS_HRES_H			0xEC
+#define HDMI_CORE_SYS_VRES_L			0xF0
+#define HDMI_CORE_SYS_VRES_H			0xF4
+#define HDMI_CORE_SYS_IADJUST			0xF8
+#define HDMI_CORE_SYS_POLDETECT			0xFC
+#define HDMI_CORE_SYS_HWIDTH1			0x110
+#define HDMI_CORE_SYS_HWIDTH2			0x114
+#define HDMI_CORE_SYS_VWIDTH			0x11C
+#define HDMI_CORE_SYS_VID_CTRL			0x120
+#define HDMI_CORE_SYS_VID_ACEN			0x124
+#define HDMI_CORE_SYS_VID_MODE			0x128
+#define HDMI_CORE_SYS_VID_BLANK1		0x12C
+#define HDMI_CORE_SYS_VID_BLANK2		0x130
+#define HDMI_CORE_SYS_VID_BLANK3		0x134
+#define HDMI_CORE_SYS_DC_HEADER			0x138
+#define HDMI_CORE_SYS_VID_DITHER		0x13C
+#define HDMI_CORE_SYS_RGB2XVYCC_CT		0x140
+#define HDMI_CORE_SYS_R2Y_COEFF_LOW		0x144
+#define HDMI_CORE_SYS_R2Y_COEFF_UP		0x148
+#define HDMI_CORE_SYS_G2Y_COEFF_LOW		0x14C
+#define HDMI_CORE_SYS_G2Y_COEFF_UP		0x150
+#define HDMI_CORE_SYS_B2Y_COEFF_LOW		0x154
+#define HDMI_CORE_SYS_B2Y_COEFF_UP		0x158
+#define HDMI_CORE_SYS_R2CB_COEFF_LOW		0x15C
+#define HDMI_CORE_SYS_R2CB_COEFF_UP		0x160
+#define HDMI_CORE_SYS_G2CB_COEFF_LOW		0x164
+#define HDMI_CORE_SYS_G2CB_COEFF_UP		0x168
+#define HDMI_CORE_SYS_B2CB_COEFF_LOW		0x16C
+#define HDMI_CORE_SYS_B2CB_COEFF_UP		0x170
+#define HDMI_CORE_SYS_R2CR_COEFF_LOW		0x174
+#define HDMI_CORE_SYS_R2CR_COEFF_UP		0x178
+#define HDMI_CORE_SYS_G2CR_COEFF_LOW		0x17C
+#define HDMI_CORE_SYS_G2CR_COEFF_UP		0x180
+#define HDMI_CORE_SYS_B2CR_COEFF_LOW		0x184
+#define HDMI_CORE_SYS_B2CR_COEFF_UP		0x188
+#define HDMI_CORE_SYS_RGB_OFFSET_LOW		0x18C
+#define HDMI_CORE_SYS_RGB_OFFSET_UP		0x190
+#define HDMI_CORE_SYS_Y_OFFSET_LOW		0x194
+#define HDMI_CORE_SYS_Y_OFFSET_UP		0x198
+#define HDMI_CORE_SYS_CBCR_OFFSET_LOW		0x19C
+#define HDMI_CORE_SYS_CBCR_OFFSET_UP		0x1A0
+#define HDMI_CORE_SYS_INTR_STATE		0x1C0
+#define HDMI_CORE_SYS_INTR1			0x1C4
+#define HDMI_CORE_SYS_INTR2			0x1C8
+#define HDMI_CORE_SYS_INTR3			0x1CC
+#define HDMI_CORE_SYS_INTR4			0x1D0
+#define HDMI_CORE_SYS_INTR_UNMASK1		0x1D4
+#define HDMI_CORE_SYS_INTR_UNMASK2		0x1D8
+#define HDMI_CORE_SYS_INTR_UNMASK3		0x1DC
+#define HDMI_CORE_SYS_INTR_UNMASK4		0x1E0
+#define HDMI_CORE_SYS_INTR_CTRL			0x1E4
+#define HDMI_CORE_SYS_TMDS_CTRL			0x208
+
+/* value definitions for HDMI_CORE_SYS_SYS_CTRL1 fields */
+#define HDMI_CORE_SYS_SYS_CTRL1_VEN_FOLLOWVSYNC	0x1
+#define HDMI_CORE_SYS_SYS_CTRL1_HEN_FOLLOWHSYNC	0x1
+#define HDMI_CORE_SYS_SYS_CTRL1_BSEL_24BITBUS	0x1
+#define HDMI_CORE_SYS_SYS_CTRL1_EDGE_RISINGEDGE	0x1
+
+/* HDMI DDC E-DID */
+#define HDMI_CORE_DDC_ADDR			0x3B4
+#define HDMI_CORE_DDC_SEGM			0x3B8
+#define HDMI_CORE_DDC_OFFSET			0x3BC
+#define HDMI_CORE_DDC_COUNT1			0x3C0
+#define HDMI_CORE_DDC_COUNT2			0x3C4
+#define HDMI_CORE_DDC_STATUS			0x3C8
+#define HDMI_CORE_DDC_CMD			0x3CC
+#define HDMI_CORE_DDC_DATA			0x3D0
+
+/* HDMI IP Core Audio Video */
+
+#define HDMI_CORE_AV_ACR_CTRL			0x4
+#define HDMI_CORE_AV_FREQ_SVAL			0x8
+#define HDMI_CORE_AV_N_SVAL1			0xC
+#define HDMI_CORE_AV_N_SVAL2			0x10
+#define HDMI_CORE_AV_N_SVAL3			0x14
+#define HDMI_CORE_AV_CTS_SVAL1			0x18
+#define HDMI_CORE_AV_CTS_SVAL2			0x1C
+#define HDMI_CORE_AV_CTS_SVAL3			0x20
+#define HDMI_CORE_AV_CTS_HVAL1			0x24
+#define HDMI_CORE_AV_CTS_HVAL2			0x28
+#define HDMI_CORE_AV_CTS_HVAL3			0x2C
+#define HDMI_CORE_AV_AUD_MODE			0x50
+#define HDMI_CORE_AV_SPDIF_CTRL			0x54
+#define HDMI_CORE_AV_HW_SPDIF_FS		0x60
+#define HDMI_CORE_AV_SWAP_I2S			0x64
+#define HDMI_CORE_AV_SPDIF_ERTH			0x6C
+#define HDMI_CORE_AV_I2S_IN_MAP			0x70
+#define HDMI_CORE_AV_I2S_IN_CTRL		0x74
+#define HDMI_CORE_AV_I2S_CHST0			0x78
+#define HDMI_CORE_AV_I2S_CHST1			0x7C
+#define HDMI_CORE_AV_I2S_CHST2			0x80
+#define HDMI_CORE_AV_I2S_CHST4			0x84
+#define HDMI_CORE_AV_I2S_CHST5			0x88
+#define HDMI_CORE_AV_ASRC			0x8C
+#define HDMI_CORE_AV_I2S_IN_LEN			0x90
+#define HDMI_CORE_AV_HDMI_CTRL			0xBC
+#define HDMI_CORE_AV_AUDO_TXSTAT		0xC0
+#define HDMI_CORE_AV_AUD_PAR_BUSCLK_1		0xCC
+#define HDMI_CORE_AV_AUD_PAR_BUSCLK_2		0xD0
+#define HDMI_CORE_AV_AUD_PAR_BUSCLK_3		0xD4
+#define HDMI_CORE_AV_TEST_TXCTRL		0xF0
+#define HDMI_CORE_AV_DPD			0xF4
+#define HDMI_CORE_AV_PB_CTRL1			0xF8
+#define HDMI_CORE_AV_PB_CTRL2			0xFC
+#define HDMI_CORE_AV_AVI_BASE			0x100
+#define HDMI_CORE_AV_AVI_TYPE			0x100
+#define HDMI_CORE_AV_AVI_VERS			0x104
+#define HDMI_CORE_AV_AVI_LEN			0x108
+#define HDMI_CORE_AV_AVI_CHSUM			0x10C
+#define HDMI_CORE_AV_AVI_DBYTE(n)		(n * 4 + 0x110)
+#define HDMI_CORE_AV_SPD_TYPE			0x180
+#define HDMI_CORE_AV_SPD_VERS			0x184
+#define HDMI_CORE_AV_SPD_LEN			0x188
+#define HDMI_CORE_AV_SPD_CHSUM			0x18C
+#define HDMI_CORE_AV_SPD_DBYTE(n)		(n * 4 + 0x190)
+#define HDMI_CORE_AV_AUDIO_TYPE			0x200
+#define HDMI_CORE_AV_AUDIO_VERS			0x204
+#define HDMI_CORE_AV_AUDIO_LEN			0x208
+#define HDMI_CORE_AV_AUDIO_CHSUM		0x20C
+#define HDMI_CORE_AV_AUD_DBYTE(n)		(n * 4 + 0x210)
+#define HDMI_CORE_AV_MPEG_TYPE			0x280
+#define HDMI_CORE_AV_MPEG_VERS			0x284
+#define HDMI_CORE_AV_MPEG_LEN			0x288
+#define HDMI_CORE_AV_MPEG_CHSUM			0x28C
+#define HDMI_CORE_AV_MPEG_DBYTE(n)		(n * 4 + 0x290)
+#define HDMI_CORE_AV_GEN_DBYTE(n)		(n * 4 + 0x300)
+#define HDMI_CORE_AV_CP_BYTE1			0x37C
+#define HDMI_CORE_AV_GEN2_DBYTE(n)		(n * 4 + 0x380)
+#define HDMI_CORE_AV_CEC_ADDR_ID		0x3FC
+
+#define HDMI_CORE_AV_SPD_DBYTE_ELSIZE		0x4
+#define HDMI_CORE_AV_GEN2_DBYTE_ELSIZE		0x4
+#define HDMI_CORE_AV_MPEG_DBYTE_ELSIZE		0x4
+#define HDMI_CORE_AV_GEN_DBYTE_ELSIZE		0x4
+
+#define HDMI_CORE_AV_AVI_DBYTE_NELEMS		15
+#define HDMI_CORE_AV_SPD_DBYTE_NELEMS		27
+#define HDMI_CORE_AV_AUD_DBYTE_NELEMS		10
+#define HDMI_CORE_AV_MPEG_DBYTE_NELEMS		27
+#define HDMI_CORE_AV_GEN_DBYTE_NELEMS		31
+#define HDMI_CORE_AV_GEN2_DBYTE_NELEMS		31
+
+enum hdmi_core_inputbus_width {
+	HDMI_INPUT_8BIT = 0,
+	HDMI_INPUT_10BIT = 1,
+	HDMI_INPUT_12BIT = 2
+};
+
+enum hdmi_core_dither_trunc {
+	HDMI_OUTPUTTRUNCATION_8BIT = 0,
+	HDMI_OUTPUTTRUNCATION_10BIT = 1,
+	HDMI_OUTPUTTRUNCATION_12BIT = 2,
+	HDMI_OUTPUTDITHER_8BIT = 3,
+	HDMI_OUTPUTDITHER_10BIT = 4,
+	HDMI_OUTPUTDITHER_12BIT = 5
+};
+
+enum hdmi_core_deepcolor_ed {
+	HDMI_DEEPCOLORPACKECTDISABLE = 0,
+	HDMI_DEEPCOLORPACKECTENABLE = 1
+};
+
+enum hdmi_core_packet_mode {
+	HDMI_PACKETMODERESERVEDVALUE = 0,
+	HDMI_PACKETMODE24BITPERPIXEL = 4,
+	HDMI_PACKETMODE30BITPERPIXEL = 5,
+	HDMI_PACKETMODE36BITPERPIXEL = 6,
+	HDMI_PACKETMODE48BITPERPIXEL = 7
+};
+
+enum hdmi_core_tclkselclkmult {
+	HDMI_FPLL05IDCK = 0,
+	HDMI_FPLL10IDCK = 1,
+	HDMI_FPLL20IDCK = 2,
+	HDMI_FPLL40IDCK = 3
+};
+
+enum hdmi_core_packet_ctrl {
+	HDMI_PACKETENABLE = 1,
+	HDMI_PACKETDISABLE = 0,
+	HDMI_PACKETREPEATON = 1,
+	HDMI_PACKETREPEATOFF = 0
+};
+
+enum hdmi_audio_i2s_config {
+	HDMI_AUDIO_I2S_MSB_SHIFTED_FIRST = 0,
+	HDMI_AUDIO_I2S_LSB_SHIFTED_FIRST = 1,
+	HDMI_AUDIO_I2S_SCK_EDGE_FALLING = 0,
+	HDMI_AUDIO_I2S_SCK_EDGE_RISING = 1,
+	HDMI_AUDIO_I2S_VBIT_FOR_PCM = 0,
+	HDMI_AUDIO_I2S_VBIT_FOR_COMPRESSED = 1,
+	HDMI_AUDIO_I2S_FIRST_BIT_SHIFT = 0,
+	HDMI_AUDIO_I2S_FIRST_BIT_NO_SHIFT = 1,
+	HDMI_AUDIO_I2S_SD0_EN = 1,
+	HDMI_AUDIO_I2S_SD1_EN = 1 << 1,
+	HDMI_AUDIO_I2S_SD2_EN = 1 << 2,
+	HDMI_AUDIO_I2S_SD3_EN = 1 << 3,
+};
+
+struct hdmi_core_video_config {
+	enum hdmi_core_inputbus_width	ip_bus_width;
+	enum hdmi_core_dither_trunc	op_dither_truc;
+	enum hdmi_core_deepcolor_ed	deep_color_pkt;
+	enum hdmi_core_packet_mode	pkt_mode;
+	enum hdmi_core_hdmi_dvi		hdmi_dvi;
+	enum hdmi_core_tclkselclkmult	tclk_sel_clkmult;
+};
+
+struct hdmi_core_packet_enable_repeat {
+	u32	audio_pkt;
+	u32	audio_pkt_repeat;
+	u32	avi_infoframe;
+	u32	avi_infoframe_repeat;
+	u32	gen_cntrl_pkt;
+	u32	gen_cntrl_pkt_repeat;
+	u32	generic_pkt;
+	u32	generic_pkt_repeat;
+};
+
+int hdmi4_read_edid(struct hdmi_core_data *core, u8 *edid, int len);
+void hdmi4_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+		struct hdmi_config *cfg);
+void hdmi4_core_dump(struct hdmi_core_data *core, struct seq_file *s);
+int hdmi4_core_init(struct platform_device *pdev, struct hdmi_core_data *core);
+
+int hdmi4_audio_start(struct hdmi_core_data *core, struct hdmi_wp_data *wp);
+void hdmi4_audio_stop(struct hdmi_core_data *core, struct hdmi_wp_data *wp);
+int hdmi4_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+		struct omap_dss_audio *audio, u32 pclk);
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
new file mode 100644
index 000000000000..a955a2c4c061
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
@@ -0,0 +1,876 @@
+/*
+ * HDMI driver for OMAP5
+ *
+ * Copyright (C) 2014 Texas Instruments Incorporated
+ *
+ * Authors:
+ *	Yong Zhi
+ *	Mythri pk
+ *	Archit Taneja <archit@ti.com>
+ *	Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "HDMI"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/clk.h>
+#include <linux/gpio.h>
+#include <linux/regulator/consumer.h>
+#include <linux/component.h>
+#include <video/omapdss.h>
+#include <sound/omap-hdmi-audio.h>
+
+#include "hdmi5_core.h"
+#include "dss.h"
+#include "dss_features.h"
+
+static struct omap_hdmi hdmi;
+
+static int hdmi_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("hdmi_runtime_get\n");
+
+	r = pm_runtime_get_sync(&hdmi.pdev->dev);
+	WARN_ON(r < 0);
+	if (r < 0)
+		return r;
+
+	return 0;
+}
+
+static void hdmi_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("hdmi_runtime_put\n");
+
+	r = pm_runtime_put_sync(&hdmi.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+
+static irqreturn_t hdmi_irq_handler(int irq, void *data)
+{
+	struct hdmi_wp_data *wp = data;
+	u32 irqstatus;
+
+	irqstatus = hdmi_wp_get_irqstatus(wp);
+	hdmi_wp_set_irqstatus(wp, irqstatus);
+
+	if ((irqstatus & HDMI_IRQ_LINK_CONNECT) &&
+			irqstatus & HDMI_IRQ_LINK_DISCONNECT) {
+		u32 v;
+		/*
+		 * If we get both connect and disconnect interrupts at the same
+		 * time, turn off the PHY, clear interrupts, and restart, which
+		 * raises connect interrupt if a cable is connected, or nothing
+		 * if cable is not connected.
+		 */
+
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_OFF);
+
+		/*
+		 * We always get bogus CONNECT & DISCONNECT interrupts when
+		 * setting the PHY to LDOON. To ignore those, we force the RXDET
+		 * line to 0 until the PHY power state has been changed.
+		 */
+		v = hdmi_read_reg(hdmi.phy.base, HDMI_TXPHY_PAD_CFG_CTRL);
+		v = FLD_MOD(v, 1, 15, 15); /* FORCE_RXDET_HIGH */
+		v = FLD_MOD(v, 0, 14, 7); /* RXDET_LINE */
+		hdmi_write_reg(hdmi.phy.base, HDMI_TXPHY_PAD_CFG_CTRL, v);
+
+		hdmi_wp_set_irqstatus(wp, HDMI_IRQ_LINK_CONNECT |
+				HDMI_IRQ_LINK_DISCONNECT);
+
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_LDOON);
+
+		REG_FLD_MOD(hdmi.phy.base, HDMI_TXPHY_PAD_CFG_CTRL, 0, 15, 15);
+
+	} else if (irqstatus & HDMI_IRQ_LINK_CONNECT) {
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_TXON);
+	} else if (irqstatus & HDMI_IRQ_LINK_DISCONNECT) {
+		hdmi_wp_set_phy_pwr(wp, HDMI_PHYPWRCMD_LDOON);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int hdmi_init_regulator(void)
+{
+	int r;
+	struct regulator *reg;
+
+	if (hdmi.vdda_reg != NULL)
+		return 0;
+
+	reg = devm_regulator_get(&hdmi.pdev->dev, "vdda");
+	if (IS_ERR(reg)) {
+		DSSERR("can't get VDDA regulator\n");
+		return PTR_ERR(reg);
+	}
+
+	if (regulator_can_change_voltage(reg)) {
+		r = regulator_set_voltage(reg, 1800000, 1800000);
+		if (r) {
+			devm_regulator_put(reg);
+			DSSWARN("can't set the regulator voltage\n");
+			return r;
+		}
+	}
+
+	hdmi.vdda_reg = reg;
+
+	return 0;
+}
+
+static int hdmi_power_on_core(struct omap_dss_device *dssdev)
+{
+	int r;
+
+	r = regulator_enable(hdmi.vdda_reg);
+	if (r)
+		return r;
+
+	r = hdmi_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	/* Make selection of HDMI in DSS */
+	dss_select_hdmi_venc_clk_source(DSS_HDMI_M_PCLK);
+
+	hdmi.core_enabled = true;
+
+	return 0;
+
+err_runtime_get:
+	regulator_disable(hdmi.vdda_reg);
+
+	return r;
+}
+
+static void hdmi_power_off_core(struct omap_dss_device *dssdev)
+{
+	hdmi.core_enabled = false;
+
+	hdmi_runtime_put();
+	regulator_disable(hdmi.vdda_reg);
+}
+
+static int hdmi_power_on_full(struct omap_dss_device *dssdev)
+{
+	int r;
+	struct omap_video_timings *p;
+	struct omap_overlay_manager *mgr = hdmi.output.manager;
+	struct dss_pll_clock_info hdmi_cinfo = { 0 };
+
+	r = hdmi_power_on_core(dssdev);
+	if (r)
+		return r;
+
+	p = &hdmi.cfg.timings;
+
+	DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res);
+
+	hdmi_pll_compute(&hdmi.pll, p->pixelclock, &hdmi_cinfo);
+
+	/* disable and clear irqs */
+	hdmi_wp_clear_irqenable(&hdmi.wp, 0xffffffff);
+	hdmi_wp_set_irqstatus(&hdmi.wp,
+			hdmi_wp_get_irqstatus(&hdmi.wp));
+
+	r = dss_pll_enable(&hdmi.pll.pll);
+	if (r) {
+		DSSERR("Failed to enable PLL\n");
+		goto err_pll_enable;
+	}
+
+	r = dss_pll_set_config(&hdmi.pll.pll, &hdmi_cinfo);
+	if (r) {
+		DSSERR("Failed to configure PLL\n");
+		goto err_pll_cfg;
+	}
+
+	r = hdmi_phy_configure(&hdmi.phy, hdmi_cinfo.clkdco,
+		hdmi_cinfo.clkout[0]);
+	if (r) {
+		DSSDBG("Failed to start PHY\n");
+		goto err_phy_cfg;
+	}
+
+	r = hdmi_wp_set_phy_pwr(&hdmi.wp, HDMI_PHYPWRCMD_LDOON);
+	if (r)
+		goto err_phy_pwr;
+
+	hdmi5_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
+
+	/* bypass TV gamma table */
+	dispc_enable_gamma_table(0);
+
+	/* tv size */
+	dss_mgr_set_timings(mgr, p);
+
+	r = hdmi_wp_video_start(&hdmi.wp);
+	if (r)
+		goto err_vid_enable;
+
+	r = dss_mgr_enable(mgr);
+	if (r)
+		goto err_mgr_enable;
+
+	hdmi_wp_set_irqenable(&hdmi.wp,
+			HDMI_IRQ_LINK_CONNECT | HDMI_IRQ_LINK_DISCONNECT);
+
+	return 0;
+
+err_mgr_enable:
+	hdmi_wp_video_stop(&hdmi.wp);
+err_vid_enable:
+	hdmi_wp_set_phy_pwr(&hdmi.wp, HDMI_PHYPWRCMD_OFF);
+err_phy_pwr:
+err_phy_cfg:
+err_pll_cfg:
+	dss_pll_disable(&hdmi.pll.pll);
+err_pll_enable:
+	hdmi_power_off_core(dssdev);
+	return -EIO;
+}
+
+static void hdmi_power_off_full(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = hdmi.output.manager;
+
+	hdmi_wp_clear_irqenable(&hdmi.wp, 0xffffffff);
+
+	dss_mgr_disable(mgr);
+
+	hdmi_wp_video_stop(&hdmi.wp);
+
+	hdmi_wp_set_phy_pwr(&hdmi.wp, HDMI_PHYPWRCMD_OFF);
+
+	dss_pll_disable(&hdmi.pll.pll);
+
+	hdmi_power_off_core(dssdev);
+}
+
+static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
+					struct omap_video_timings *timings)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	/* TODO: proper interlace support */
+	if (timings->interlace)
+		return -EINVAL;
+
+	if (!dispc_mgr_timings_ok(out->dispc_channel, timings))
+		return -EINVAL;
+
+	return 0;
+}
+
+static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	mutex_lock(&hdmi.lock);
+
+	hdmi.cfg.timings = *timings;
+
+	dispc_set_tv_pclk(timings->pixelclock);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	*timings = hdmi.cfg.timings;
+}
+
+static void hdmi_dump_regs(struct seq_file *s)
+{
+	mutex_lock(&hdmi.lock);
+
+	if (hdmi_runtime_get()) {
+		mutex_unlock(&hdmi.lock);
+		return;
+	}
+
+	hdmi_wp_dump(&hdmi.wp, s);
+	hdmi_pll_dump(&hdmi.pll, s);
+	hdmi_phy_dump(&hdmi.phy, s);
+	hdmi5_core_dump(&hdmi.core, s);
+
+	hdmi_runtime_put();
+	mutex_unlock(&hdmi.lock);
+}
+
+static int read_edid(u8 *buf, int len)
+{
+	int r;
+	int idlemode;
+
+	mutex_lock(&hdmi.lock);
+
+	r = hdmi_runtime_get();
+	BUG_ON(r);
+
+	idlemode = REG_GET(hdmi.wp.base, HDMI_WP_SYSCONFIG, 3, 2);
+	/* No-idle mode */
+	REG_FLD_MOD(hdmi.wp.base, HDMI_WP_SYSCONFIG, 1, 3, 2);
+
+	r = hdmi5_read_edid(&hdmi.core,  buf, len);
+
+	REG_FLD_MOD(hdmi.wp.base, HDMI_WP_SYSCONFIG, idlemode, 3, 2);
+
+	hdmi_runtime_put();
+	mutex_unlock(&hdmi.lock);
+
+	return r;
+}
+
+static void hdmi_start_audio_stream(struct omap_hdmi *hd)
+{
+	REG_FLD_MOD(hdmi.wp.base, HDMI_WP_SYSCONFIG, 1, 3, 2);
+	hdmi_wp_audio_enable(&hd->wp, true);
+	hdmi_wp_audio_core_req_enable(&hd->wp, true);
+}
+
+static void hdmi_stop_audio_stream(struct omap_hdmi *hd)
+{
+	hdmi_wp_audio_core_req_enable(&hd->wp, false);
+	hdmi_wp_audio_enable(&hd->wp, false);
+	REG_FLD_MOD(hd->wp.base, HDMI_WP_SYSCONFIG, hd->wp_idlemode, 3, 2);
+}
+
+static int hdmi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+	unsigned long flags;
+	int r = 0;
+
+	DSSDBG("ENTER hdmi_display_enable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		r = -ENODEV;
+		goto err0;
+	}
+
+	r = hdmi_power_on_full(dssdev);
+	if (r) {
+		DSSERR("failed to power on device\n");
+		goto err0;
+	}
+
+	if (hdmi.audio_configured) {
+		r = hdmi5_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config,
+				       hdmi.cfg.timings.pixelclock);
+		if (r) {
+			DSSERR("Error restoring audio configuration: %d", r);
+			hdmi.audio_abort_cb(&hdmi.pdev->dev);
+			hdmi.audio_configured = false;
+		}
+	}
+
+	spin_lock_irqsave(&hdmi.audio_playing_lock, flags);
+	if (hdmi.audio_configured && hdmi.audio_playing)
+		hdmi_start_audio_stream(&hdmi);
+	hdmi.display_enabled = true;
+	spin_unlock_irqrestore(&hdmi.audio_playing_lock, flags);
+
+	mutex_unlock(&hdmi.lock);
+	return 0;
+
+err0:
+	mutex_unlock(&hdmi.lock);
+	return r;
+}
+
+static void hdmi_display_disable(struct omap_dss_device *dssdev)
+{
+	unsigned long flags;
+
+	DSSDBG("Enter hdmi_display_disable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	spin_lock_irqsave(&hdmi.audio_playing_lock, flags);
+	hdmi_stop_audio_stream(&hdmi);
+	hdmi.display_enabled = false;
+	spin_unlock_irqrestore(&hdmi.audio_playing_lock, flags);
+
+	hdmi_power_off_full(dssdev);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static int hdmi_core_enable(struct omap_dss_device *dssdev)
+{
+	int r = 0;
+
+	DSSDBG("ENTER omapdss_hdmi_core_enable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	r = hdmi_power_on_core(dssdev);
+	if (r) {
+		DSSERR("failed to power on device\n");
+		goto err0;
+	}
+
+	mutex_unlock(&hdmi.lock);
+	return 0;
+
+err0:
+	mutex_unlock(&hdmi.lock);
+	return r;
+}
+
+static void hdmi_core_disable(struct omap_dss_device *dssdev)
+{
+	DSSDBG("Enter omapdss_hdmi_core_disable\n");
+
+	mutex_lock(&hdmi.lock);
+
+	hdmi_power_off_core(dssdev);
+
+	mutex_unlock(&hdmi.lock);
+}
+
+static int hdmi_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = hdmi_init_regulator();
+	if (r)
+		return r;
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dst->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void hdmi_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static int hdmi_read_edid(struct omap_dss_device *dssdev,
+		u8 *edid, int len)
+{
+	bool need_enable;
+	int r;
+
+	need_enable = hdmi.core_enabled == false;
+
+	if (need_enable) {
+		r = hdmi_core_enable(dssdev);
+		if (r)
+			return r;
+	}
+
+	r = read_edid(edid, len);
+
+	if (need_enable)
+		hdmi_core_disable(dssdev);
+
+	return r;
+}
+
+static int hdmi_set_infoframe(struct omap_dss_device *dssdev,
+		const struct hdmi_avi_infoframe *avi)
+{
+	hdmi.cfg.infoframe = *avi;
+	return 0;
+}
+
+static int hdmi_set_hdmi_mode(struct omap_dss_device *dssdev,
+		bool hdmi_mode)
+{
+	hdmi.cfg.hdmi_dvi_mode = hdmi_mode ? HDMI_HDMI : HDMI_DVI;
+	return 0;
+}
+
+static const struct omapdss_hdmi_ops hdmi_ops = {
+	.connect		= hdmi_connect,
+	.disconnect		= hdmi_disconnect,
+
+	.enable			= hdmi_display_enable,
+	.disable		= hdmi_display_disable,
+
+	.check_timings		= hdmi_display_check_timing,
+	.set_timings		= hdmi_display_set_timing,
+	.get_timings		= hdmi_display_get_timings,
+
+	.read_edid		= hdmi_read_edid,
+	.set_infoframe		= hdmi_set_infoframe,
+	.set_hdmi_mode		= hdmi_set_hdmi_mode,
+};
+
+static void hdmi_init_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_HDMI;
+	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
+	out->name = "hdmi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
+	out->ops.hdmi = &hdmi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void hdmi_uninit_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &hdmi.output;
+
+	omapdss_unregister_output(out);
+}
+
+static int hdmi_probe_of(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *ep;
+	int r;
+
+	ep = omapdss_of_get_first_endpoint(node);
+	if (!ep)
+		return 0;
+
+	r = hdmi_parse_lanes_of(pdev, ep, &hdmi.phy);
+	if (r)
+		goto err;
+
+	of_node_put(ep);
+	return 0;
+
+err:
+	of_node_put(ep);
+	return r;
+}
+
+/* Audio callbacks */
+static int hdmi_audio_startup(struct device *dev,
+			      void (*abort_cb)(struct device *dev))
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	int ret = 0;
+
+	mutex_lock(&hd->lock);
+
+	if (!hdmi_mode_has_audio(&hd->cfg) || !hd->display_enabled) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	hd->audio_abort_cb = abort_cb;
+
+out:
+	mutex_unlock(&hd->lock);
+
+	return ret;
+}
+
+static int hdmi_audio_shutdown(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+
+	mutex_lock(&hd->lock);
+	hd->audio_abort_cb = NULL;
+	hd->audio_configured = false;
+	hd->audio_playing = false;
+	mutex_unlock(&hd->lock);
+
+	return 0;
+}
+
+static int hdmi_audio_start(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	WARN_ON(!hdmi_mode_has_audio(&hd->cfg));
+
+	spin_lock_irqsave(&hd->audio_playing_lock, flags);
+
+	if (hd->display_enabled)
+		hdmi_start_audio_stream(hd);
+	hd->audio_playing = true;
+
+	spin_unlock_irqrestore(&hd->audio_playing_lock, flags);
+	return 0;
+}
+
+static void hdmi_audio_stop(struct device *dev)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	WARN_ON(!hdmi_mode_has_audio(&hd->cfg));
+
+	spin_lock_irqsave(&hd->audio_playing_lock, flags);
+
+	if (hd->display_enabled)
+		hdmi_stop_audio_stream(hd);
+	hd->audio_playing = false;
+
+	spin_unlock_irqrestore(&hd->audio_playing_lock, flags);
+}
+
+static int hdmi_audio_config(struct device *dev,
+			     struct omap_dss_audio *dss_audio)
+{
+	struct omap_hdmi *hd = dev_get_drvdata(dev);
+	int ret;
+
+	mutex_lock(&hd->lock);
+
+	if (!hdmi_mode_has_audio(&hd->cfg) || !hd->display_enabled) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	ret = hdmi5_audio_config(&hd->core, &hd->wp, dss_audio,
+				 hd->cfg.timings.pixelclock);
+
+	if (!ret) {
+		hd->audio_configured = true;
+		hd->audio_config = *dss_audio;
+	}
+out:
+	mutex_unlock(&hd->lock);
+
+	return ret;
+}
+
+static const struct omap_hdmi_audio_ops hdmi_audio_ops = {
+	.audio_startup = hdmi_audio_startup,
+	.audio_shutdown = hdmi_audio_shutdown,
+	.audio_start = hdmi_audio_start,
+	.audio_stop = hdmi_audio_stop,
+	.audio_config = hdmi_audio_config,
+};
+
+static int hdmi_audio_register(struct device *dev)
+{
+	struct omap_hdmi_audio_pdata pdata = {
+		.dev = dev,
+		.dss_version = omapdss_get_version(),
+		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
+		.ops = &hdmi_audio_ops,
+	};
+
+	hdmi.audio_pdev = platform_device_register_data(
+		dev, "omap-hdmi-audio", PLATFORM_DEVID_AUTO,
+		&pdata, sizeof(pdata));
+
+	if (IS_ERR(hdmi.audio_pdev))
+		return PTR_ERR(hdmi.audio_pdev);
+
+	hdmi_runtime_get();
+	hdmi.wp_idlemode =
+		REG_GET(hdmi.wp.base, HDMI_WP_SYSCONFIG, 3, 2);
+	hdmi_runtime_put();
+
+	return 0;
+}
+
+/* HDMI HW IP initialisation */
+static int hdmi5_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	int r;
+	int irq;
+
+	hdmi.pdev = pdev;
+	dev_set_drvdata(&pdev->dev, &hdmi);
+
+	mutex_init(&hdmi.lock);
+	spin_lock_init(&hdmi.audio_playing_lock);
+
+	if (pdev->dev.of_node) {
+		r = hdmi_probe_of(pdev);
+		if (r)
+			return r;
+	}
+
+	r = hdmi_wp_init(pdev, &hdmi.wp);
+	if (r)
+		return r;
+
+	r = hdmi_pll_init(pdev, &hdmi.pll, &hdmi.wp);
+	if (r)
+		return r;
+
+	r = hdmi_phy_init(pdev, &hdmi.phy);
+	if (r)
+		goto err;
+
+	r = hdmi5_core_init(pdev, &hdmi.core);
+	if (r)
+		goto err;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		DSSERR("platform_get_irq failed\n");
+		r = -ENODEV;
+		goto err;
+	}
+
+	r = devm_request_threaded_irq(&pdev->dev, irq,
+			NULL, hdmi_irq_handler,
+			IRQF_ONESHOT, "OMAP HDMI", &hdmi.wp);
+	if (r) {
+		DSSERR("HDMI IRQ request failed\n");
+		goto err;
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	hdmi_init_output(pdev);
+
+	r = hdmi_audio_register(&pdev->dev);
+	if (r) {
+		DSSERR("Registering HDMI audio failed %d\n", r);
+		hdmi_uninit_output(pdev);
+		pm_runtime_disable(&pdev->dev);
+		return r;
+	}
+
+	dss_debugfs_create_file("hdmi", hdmi_dump_regs);
+
+	return 0;
+err:
+	hdmi_pll_uninit(&hdmi.pll);
+	return r;
+}
+
+static void hdmi5_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	if (hdmi.audio_pdev)
+		platform_device_unregister(hdmi.audio_pdev);
+
+	hdmi_uninit_output(pdev);
+
+	hdmi_pll_uninit(&hdmi.pll);
+
+	pm_runtime_disable(&pdev->dev);
+}
+
+static const struct component_ops hdmi5_component_ops = {
+	.bind	= hdmi5_bind,
+	.unbind	= hdmi5_unbind,
+};
+
+static int hdmi5_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &hdmi5_component_ops);
+}
+
+static int hdmi5_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &hdmi5_component_ops);
+	return 0;
+}
+
+static int hdmi_runtime_suspend(struct device *dev)
+{
+	dispc_runtime_put();
+
+	return 0;
+}
+
+static int hdmi_runtime_resume(struct device *dev)
+{
+	int r;
+
+	r = dispc_runtime_get();
+	if (r < 0)
+		return r;
+
+	return 0;
+}
+
+static const struct dev_pm_ops hdmi_pm_ops = {
+	.runtime_suspend = hdmi_runtime_suspend,
+	.runtime_resume = hdmi_runtime_resume,
+};
+
+static const struct of_device_id hdmi_of_match[] = {
+	{ .compatible = "ti,omap5-hdmi", },
+	{ .compatible = "ti,dra7-hdmi", },
+	{},
+};
+
+static struct platform_driver omapdss_hdmihw_driver = {
+	.probe		= hdmi5_probe,
+	.remove		= hdmi5_remove,
+	.driver         = {
+		.name   = "omapdss_hdmi5",
+		.pm	= &hdmi_pm_ops,
+		.of_match_table = hdmi_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init hdmi5_init_platform_driver(void)
+{
+	return platform_driver_register(&omapdss_hdmihw_driver);
+}
+
+void hdmi5_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omapdss_hdmihw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.c
new file mode 100644
index 000000000000..8ea531d2652c
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.c
@@ -0,0 +1,916 @@
+/*
+ * OMAP5 HDMI CORE IP driver library
+ *
+ * Copyright (C) 2014 Texas Instruments Incorporated
+ *
+ * Authors:
+ *	Yong Zhi
+ *	Mythri pk
+ *	Archit Taneja <archit@ti.com>
+ *	Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/seq_file.h>
+#include <drm/drm_edid.h>
+#include <sound/asound.h>
+#include <sound/asoundef.h>
+
+#include "hdmi5_core.h"
+
+/* only 24 bit color depth used for now */
+static const struct csc_table csc_table_deepcolor[] = {
+	/* HDMI_DEEP_COLOR_24BIT */
+	[0] = { 7036, 0, 0, 32, 0, 7036, 0, 32, 0, 0, 7036, 32, },
+	/* HDMI_DEEP_COLOR_30BIT */
+	[1] = { 7015, 0, 0, 128, 0, 7015, 0, 128, 0, 0, 7015, 128, },
+	/* HDMI_DEEP_COLOR_36BIT */
+	[2] = { 7010, 0, 0, 512, 0, 7010, 0, 512, 0, 0, 7010, 512, },
+	/* FULL RANGE */
+	[3] = { 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, },
+};
+
+static void hdmi_core_ddc_init(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+	const unsigned long long iclk = 266000000;	/* DSS L3 ICLK */
+	const unsigned ss_scl_high = 4000;		/* ns */
+	const unsigned ss_scl_low = 4700;		/* ns */
+	const unsigned fs_scl_high = 600;		/* ns */
+	const unsigned fs_scl_low = 1300;		/* ns */
+	const unsigned sda_hold = 1000;			/* ns */
+	const unsigned sfr_div = 10;
+	unsigned long long sfr;
+	unsigned v;
+
+	sfr = iclk / sfr_div;	/* SFR_DIV */
+	sfr /= 1000;		/* SFR clock in kHz */
+
+	/* Reset */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SOFTRSTZ, 0, 0, 0);
+	if (hdmi_wait_for_bit_change(base, HDMI_CORE_I2CM_SOFTRSTZ,
+				0, 0, 1) != 1)
+		DSSERR("HDMI I2CM reset failed\n");
+
+	/* Standard (0) or Fast (1) Mode */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_DIV, 0, 3, 3);
+
+	/* Standard Mode SCL High counter */
+	v = DIV_ROUND_UP_ULL(ss_scl_high * sfr, 1000000);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SS_SCL_HCNT_1_ADDR,
+			(v >> 8) & 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SS_SCL_HCNT_0_ADDR,
+			v & 0xff, 7, 0);
+
+	/* Standard Mode SCL Low counter */
+	v = DIV_ROUND_UP_ULL(ss_scl_low * sfr, 1000000);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SS_SCL_LCNT_1_ADDR,
+			(v >> 8) & 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SS_SCL_LCNT_0_ADDR,
+			v & 0xff, 7, 0);
+
+	/* Fast Mode SCL High Counter */
+	v = DIV_ROUND_UP_ULL(fs_scl_high * sfr, 1000000);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_FS_SCL_HCNT_1_ADDR,
+			(v >> 8) & 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_FS_SCL_HCNT_0_ADDR,
+			v & 0xff, 7, 0);
+
+	/* Fast Mode SCL Low Counter */
+	v = DIV_ROUND_UP_ULL(fs_scl_low * sfr, 1000000);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_FS_SCL_LCNT_1_ADDR,
+			(v >> 8) & 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_FS_SCL_LCNT_0_ADDR,
+			v & 0xff, 7, 0);
+
+	/* SDA Hold Time */
+	v = DIV_ROUND_UP_ULL(sda_hold * sfr, 1000000);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SDA_HOLD_ADDR, v & 0xff, 7, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SLAVE, 0x50, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SEGADDR, 0x30, 6, 0);
+
+	/* NACK_POL to high */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 7, 7);
+
+	/* NACK_MASK to unmasked */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x0, 6, 6);
+
+	/* ARBITRATION_POL to high */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 3, 3);
+
+	/* ARBITRATION_MASK to unmasked */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x0, 2, 2);
+
+	/* DONE_POL to high */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_INT, 0x1, 3, 3);
+
+	/* DONE_MASK to unmasked */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_INT, 0x0, 2, 2);
+}
+
+static void hdmi_core_ddc_uninit(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+
+	/* Mask I2C interrupts */
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 6, 6);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 2, 2);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_INT, 0x1, 2, 2);
+}
+
+static int hdmi_core_ddc_edid(struct hdmi_core_data *core, u8 *pedid, u8 ext)
+{
+	void __iomem *base = core->base;
+	u8 cur_addr;
+	char checksum = 0;
+	const int retries = 1000;
+	u8 seg_ptr = ext / 2;
+	u8 edidbase = ((ext % 2) * 0x80);
+
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_SEGPTR, seg_ptr, 7, 0);
+
+	/*
+	 * TODO: We use polling here, although we probably should use proper
+	 * interrupts.
+	 */
+	for (cur_addr = 0; cur_addr < 128; ++cur_addr) {
+		int i;
+
+		/* clear ERROR and DONE */
+		REG_FLD_MOD(base, HDMI_CORE_IH_I2CM_STAT0, 0x3, 1, 0);
+
+		REG_FLD_MOD(base, HDMI_CORE_I2CM_ADDRESS,
+				edidbase + cur_addr, 7, 0);
+
+		if (seg_ptr)
+			REG_FLD_MOD(base, HDMI_CORE_I2CM_OPERATION, 1, 1, 1);
+		else
+			REG_FLD_MOD(base, HDMI_CORE_I2CM_OPERATION, 1, 0, 0);
+
+		for (i = 0; i < retries; ++i) {
+			u32 stat;
+
+			stat = REG_GET(base, HDMI_CORE_IH_I2CM_STAT0, 1, 0);
+
+			/* I2CM_ERROR */
+			if (stat & 1) {
+				DSSERR("HDMI I2C Master Error\n");
+				return -EIO;
+			}
+
+			/* I2CM_DONE */
+			if (stat & (1 << 1))
+				break;
+
+			usleep_range(250, 1000);
+		}
+
+		if (i == retries) {
+			DSSERR("HDMI I2C timeout reading EDID\n");
+			return -EIO;
+		}
+
+		pedid[cur_addr] = REG_GET(base, HDMI_CORE_I2CM_DATAI, 7, 0);
+		checksum += pedid[cur_addr];
+	}
+
+	return 0;
+
+}
+
+int hdmi5_read_edid(struct hdmi_core_data *core, u8 *edid, int len)
+{
+	int r, n, i;
+	int max_ext_blocks = (len / 128) - 1;
+
+	if (len < 128)
+		return -EINVAL;
+
+	hdmi_core_ddc_init(core);
+
+	r = hdmi_core_ddc_edid(core, edid, 0);
+	if (r)
+		goto out;
+
+	n = edid[0x7e];
+
+	if (n > max_ext_blocks)
+		n = max_ext_blocks;
+
+	for (i = 1; i <= n; i++) {
+		r = hdmi_core_ddc_edid(core, edid + i * EDID_LENGTH, i);
+		if (r)
+			goto out;
+	}
+
+out:
+	hdmi_core_ddc_uninit(core);
+
+	return r ? r : len;
+}
+
+void hdmi5_core_dump(struct hdmi_core_data *core, struct seq_file *s)
+{
+
+#define DUMPCORE(r) seq_printf(s, "%-35s %08x\n", #r,\
+		hdmi_read_reg(core->base, r))
+
+	DUMPCORE(HDMI_CORE_FC_INVIDCONF);
+	DUMPCORE(HDMI_CORE_FC_INHACTIV0);
+	DUMPCORE(HDMI_CORE_FC_INHACTIV1);
+	DUMPCORE(HDMI_CORE_FC_INHBLANK0);
+	DUMPCORE(HDMI_CORE_FC_INHBLANK1);
+	DUMPCORE(HDMI_CORE_FC_INVACTIV0);
+	DUMPCORE(HDMI_CORE_FC_INVACTIV1);
+	DUMPCORE(HDMI_CORE_FC_INVBLANK);
+	DUMPCORE(HDMI_CORE_FC_HSYNCINDELAY0);
+	DUMPCORE(HDMI_CORE_FC_HSYNCINDELAY1);
+	DUMPCORE(HDMI_CORE_FC_HSYNCINWIDTH0);
+	DUMPCORE(HDMI_CORE_FC_HSYNCINWIDTH1);
+	DUMPCORE(HDMI_CORE_FC_VSYNCINDELAY);
+	DUMPCORE(HDMI_CORE_FC_VSYNCINWIDTH);
+	DUMPCORE(HDMI_CORE_FC_CTRLDUR);
+	DUMPCORE(HDMI_CORE_FC_EXCTRLDUR);
+	DUMPCORE(HDMI_CORE_FC_EXCTRLSPAC);
+	DUMPCORE(HDMI_CORE_FC_CH0PREAM);
+	DUMPCORE(HDMI_CORE_FC_CH1PREAM);
+	DUMPCORE(HDMI_CORE_FC_CH2PREAM);
+	DUMPCORE(HDMI_CORE_FC_AVICONF0);
+	DUMPCORE(HDMI_CORE_FC_AVICONF1);
+	DUMPCORE(HDMI_CORE_FC_AVICONF2);
+	DUMPCORE(HDMI_CORE_FC_AVIVID);
+	DUMPCORE(HDMI_CORE_FC_PRCONF);
+
+	DUMPCORE(HDMI_CORE_MC_CLKDIS);
+	DUMPCORE(HDMI_CORE_MC_SWRSTZREQ);
+	DUMPCORE(HDMI_CORE_MC_FLOWCTRL);
+	DUMPCORE(HDMI_CORE_MC_PHYRSTZ);
+	DUMPCORE(HDMI_CORE_MC_LOCKONCLOCK);
+
+	DUMPCORE(HDMI_CORE_I2CM_SLAVE);
+	DUMPCORE(HDMI_CORE_I2CM_ADDRESS);
+	DUMPCORE(HDMI_CORE_I2CM_DATAO);
+	DUMPCORE(HDMI_CORE_I2CM_DATAI);
+	DUMPCORE(HDMI_CORE_I2CM_OPERATION);
+	DUMPCORE(HDMI_CORE_I2CM_INT);
+	DUMPCORE(HDMI_CORE_I2CM_CTLINT);
+	DUMPCORE(HDMI_CORE_I2CM_DIV);
+	DUMPCORE(HDMI_CORE_I2CM_SEGADDR);
+	DUMPCORE(HDMI_CORE_I2CM_SOFTRSTZ);
+	DUMPCORE(HDMI_CORE_I2CM_SEGPTR);
+	DUMPCORE(HDMI_CORE_I2CM_SS_SCL_HCNT_1_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_SS_SCL_HCNT_0_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_SS_SCL_LCNT_1_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_SS_SCL_LCNT_0_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_FS_SCL_HCNT_1_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_FS_SCL_HCNT_0_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_FS_SCL_LCNT_1_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_FS_SCL_LCNT_0_ADDR);
+	DUMPCORE(HDMI_CORE_I2CM_SDA_HOLD_ADDR);
+}
+
+static void hdmi_core_init(struct hdmi_core_vid_config *video_cfg,
+			struct hdmi_config *cfg)
+{
+	DSSDBG("hdmi_core_init\n");
+
+	/* video core */
+	video_cfg->data_enable_pol = 1; /* It is always 1*/
+	video_cfg->v_fc_config.timings.hsync_level = cfg->timings.hsync_level;
+	video_cfg->v_fc_config.timings.x_res = cfg->timings.x_res;
+	video_cfg->v_fc_config.timings.hsw = cfg->timings.hsw - 1;
+	video_cfg->v_fc_config.timings.hbp = cfg->timings.hbp;
+	video_cfg->v_fc_config.timings.hfp = cfg->timings.hfp;
+	video_cfg->hblank = cfg->timings.hfp +
+				cfg->timings.hbp + cfg->timings.hsw - 1;
+	video_cfg->v_fc_config.timings.vsync_level = cfg->timings.vsync_level;
+	video_cfg->v_fc_config.timings.y_res = cfg->timings.y_res;
+	video_cfg->v_fc_config.timings.vsw = cfg->timings.vsw;
+	video_cfg->v_fc_config.timings.vfp = cfg->timings.vfp;
+	video_cfg->v_fc_config.timings.vbp = cfg->timings.vbp;
+	video_cfg->vblank_osc = 0; /* Always 0 - need to confirm */
+	video_cfg->vblank = cfg->timings.vsw +
+				cfg->timings.vfp + cfg->timings.vbp;
+	video_cfg->v_fc_config.hdmi_dvi_mode = cfg->hdmi_dvi_mode;
+	video_cfg->v_fc_config.timings.interlace = cfg->timings.interlace;
+}
+
+/* DSS_HDMI_CORE_VIDEO_CONFIG */
+static void hdmi_core_video_config(struct hdmi_core_data *core,
+			struct hdmi_core_vid_config *cfg)
+{
+	void __iomem *base = core->base;
+	unsigned char r = 0;
+	bool vsync_pol, hsync_pol;
+
+	vsync_pol =
+		cfg->v_fc_config.timings.vsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+	hsync_pol =
+		cfg->v_fc_config.timings.hsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+
+	/* Set hsync, vsync and data-enable polarity  */
+	r = hdmi_read_reg(base, HDMI_CORE_FC_INVIDCONF);
+	r = FLD_MOD(r, vsync_pol, 6, 6);
+	r = FLD_MOD(r, hsync_pol, 5, 5);
+	r = FLD_MOD(r, cfg->data_enable_pol, 4, 4);
+	r = FLD_MOD(r, cfg->vblank_osc, 1, 1);
+	r = FLD_MOD(r, cfg->v_fc_config.timings.interlace, 0, 0);
+	hdmi_write_reg(base, HDMI_CORE_FC_INVIDCONF, r);
+
+	/* set x resolution */
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV1,
+			cfg->v_fc_config.timings.x_res >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV0,
+			cfg->v_fc_config.timings.x_res & 0xFF, 7, 0);
+
+	/* set y resolution */
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV1,
+			cfg->v_fc_config.timings.y_res >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV0,
+			cfg->v_fc_config.timings.y_res & 0xFF, 7, 0);
+
+	/* set horizontal blanking pixels */
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHBLANK1, cfg->hblank >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHBLANK0, cfg->hblank & 0xFF, 7, 0);
+
+	/* set vertial blanking pixels */
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVBLANK, cfg->vblank, 7, 0);
+
+	/* set horizontal sync offset */
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY1,
+			cfg->v_fc_config.timings.hfp >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY0,
+			cfg->v_fc_config.timings.hfp & 0xFF, 7, 0);
+
+	/* set vertical sync offset */
+	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINDELAY,
+			cfg->v_fc_config.timings.vfp, 7, 0);
+
+	/* set horizontal sync pulse width */
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH1,
+			(cfg->v_fc_config.timings.hsw >> 8), 1, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH0,
+			cfg->v_fc_config.timings.hsw & 0xFF, 7, 0);
+
+	/*  set vertical sync pulse width */
+	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINWIDTH,
+			cfg->v_fc_config.timings.vsw, 5, 0);
+
+	/* select DVI mode */
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVIDCONF,
+			cfg->v_fc_config.hdmi_dvi_mode, 3, 3);
+}
+
+static void hdmi_core_config_video_packetizer(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+	int clr_depth = 0;	/* 24 bit color depth */
+
+	/* COLOR_DEPTH */
+	REG_FLD_MOD(base, HDMI_CORE_VP_PR_CD, clr_depth, 7, 4);
+	/* BYPASS_EN */
+	REG_FLD_MOD(base, HDMI_CORE_VP_CONF, clr_depth ? 0 : 1, 6, 6);
+	/* PP_EN */
+	REG_FLD_MOD(base, HDMI_CORE_VP_CONF, clr_depth ? 1 : 0, 5, 5);
+	/* YCC422_EN */
+	REG_FLD_MOD(base, HDMI_CORE_VP_CONF, 0, 3, 3);
+	/* PP_STUFFING */
+	REG_FLD_MOD(base, HDMI_CORE_VP_STUFF, clr_depth ? 1 : 0, 1, 1);
+	/* YCC422_STUFFING */
+	REG_FLD_MOD(base, HDMI_CORE_VP_STUFF, 1, 2, 2);
+	/* OUTPUT_SELECTOR */
+	REG_FLD_MOD(base, HDMI_CORE_VP_CONF, clr_depth ? 0 : 2, 1, 0);
+}
+
+static void hdmi_core_config_csc(struct hdmi_core_data *core)
+{
+	int clr_depth = 0;	/* 24 bit color depth */
+
+	/* CSC_COLORDEPTH */
+	REG_FLD_MOD(core->base, HDMI_CORE_CSC_SCALE, clr_depth, 7, 4);
+}
+
+static void hdmi_core_config_video_sampler(struct hdmi_core_data *core)
+{
+	int video_mapping = 1;	/* for 24 bit color depth */
+
+	/* VIDEO_MAPPING */
+	REG_FLD_MOD(core->base, HDMI_CORE_TX_INVID0, video_mapping, 4, 0);
+}
+
+static void hdmi_core_write_avi_infoframe(struct hdmi_core_data *core,
+	struct hdmi_avi_infoframe *frame)
+{
+	void __iomem *base = core->base;
+	u8 data[HDMI_INFOFRAME_SIZE(AVI)];
+	u8 *ptr;
+	unsigned y, a, b, s;
+	unsigned c, m, r;
+	unsigned itc, ec, q, sc;
+	unsigned vic;
+	unsigned yq, cn, pr;
+
+	hdmi_avi_infoframe_pack(frame, data, sizeof(data));
+
+	print_hex_dump_debug("AVI: ", DUMP_PREFIX_NONE, 16, 1, data,
+		HDMI_INFOFRAME_SIZE(AVI), false);
+
+	ptr = data + HDMI_INFOFRAME_HEADER_SIZE;
+
+	y = (ptr[0] >> 5) & 0x3;
+	a = (ptr[0] >> 4) & 0x1;
+	b = (ptr[0] >> 2) & 0x3;
+	s = (ptr[0] >> 0) & 0x3;
+
+	c = (ptr[1] >> 6) & 0x3;
+	m = (ptr[1] >> 4) & 0x3;
+	r = (ptr[1] >> 0) & 0x3;
+
+	itc = (ptr[2] >> 7) & 0x1;
+	ec = (ptr[2] >> 4) & 0x7;
+	q = (ptr[2] >> 2) & 0x3;
+	sc = (ptr[2] >> 0) & 0x3;
+
+	vic = ptr[3];
+
+	yq = (ptr[4] >> 6) & 0x3;
+	cn = (ptr[4] >> 4) & 0x3;
+	pr = (ptr[4] >> 0) & 0xf;
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AVICONF0,
+		(a << 6) | (s << 4) | (b << 2) | (y << 0));
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AVICONF1,
+		(c << 6) | (m << 4) | (r << 0));
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AVICONF2,
+		(itc << 7) | (ec << 4) | (q << 2) | (sc << 0));
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AVIVID, vic);
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AVICONF3,
+		(yq << 2) | (cn << 0));
+
+	REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, pr, 3, 0);
+}
+
+static void hdmi_core_csc_config(struct hdmi_core_data *core,
+		struct csc_table csc_coeff)
+{
+	void __iomem *base = core->base;
+
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_MSB, csc_coeff.a1 >> 8 , 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A1_LSB, csc_coeff.a1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_MSB, csc_coeff.a2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A2_LSB, csc_coeff.a2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_MSB, csc_coeff.a3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A3_LSB, csc_coeff.a3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_MSB, csc_coeff.a4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_A4_LSB, csc_coeff.a4, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_MSB, csc_coeff.b1 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B1_LSB, csc_coeff.b1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_MSB, csc_coeff.b2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B2_LSB, csc_coeff.b2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_MSB, csc_coeff.b3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B3_LSB, csc_coeff.b3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_MSB, csc_coeff.b4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_B4_LSB, csc_coeff.b4, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_MSB, csc_coeff.c1 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C1_LSB, csc_coeff.c1, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_MSB, csc_coeff.c2 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C2_LSB, csc_coeff.c2, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_MSB, csc_coeff.c3 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C3_LSB, csc_coeff.c3, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_MSB, csc_coeff.c4 >> 8, 6, 0);
+	REG_FLD_MOD(base, HDMI_CORE_CSC_COEF_C4_LSB, csc_coeff.c4, 7, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_MC_FLOWCTRL, 0x1, 0, 0);
+}
+
+static void hdmi_core_configure_range(struct hdmi_core_data *core)
+{
+	struct csc_table csc_coeff = { 0 };
+
+	/* support limited range with 24 bit color depth for now */
+	csc_coeff = csc_table_deepcolor[0];
+
+	hdmi_core_csc_config(core, csc_coeff);
+}
+
+static void hdmi_core_enable_video_path(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+
+	DSSDBG("hdmi_core_enable_video_path\n");
+
+	REG_FLD_MOD(base, HDMI_CORE_FC_CTRLDUR, 0x0C, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_EXCTRLDUR, 0x20, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_EXCTRLSPAC, 0x01, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_CH0PREAM, 0x0B, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_CH1PREAM, 0x16, 5, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_CH2PREAM, 0x21, 5, 0);
+	REG_FLD_MOD(base, HDMI_CORE_MC_CLKDIS, 0x00, 0, 0);
+	REG_FLD_MOD(base, HDMI_CORE_MC_CLKDIS, 0x00, 1, 1);
+}
+
+static void hdmi_core_mask_interrupts(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+
+	/* Master IRQ mask */
+	REG_FLD_MOD(base, HDMI_CORE_IH_MUTE, 0x3, 1, 0);
+
+	/* Mask all the interrupts in HDMI core */
+
+	REG_FLD_MOD(base, HDMI_CORE_VP_MASK, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_MASK0, 0xe7, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_MASK1, 0xfb, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_MASK2, 0x3, 1, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_AUD_INT, 0x3, 3, 2);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_GP_MASK, 0x3, 1, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_CEC_MASK, 0x7f, 6, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 6, 6);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_CTLINT, 0x1, 2, 2);
+	REG_FLD_MOD(base, HDMI_CORE_I2CM_INT, 0x1, 2, 2);
+
+	REG_FLD_MOD(base, HDMI_CORE_PHY_MASK0, 0xf3, 7, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_PHY_STAT0, 0xff, 7, 0);
+
+	/* Clear all the current interrupt bits */
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_VP_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT0, 0xe7, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT1, 0xfb, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT2, 0x3, 1, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_AS_STAT0, 0x7, 2, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_CEC_STAT0, 0x7f, 6, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_I2CM_STAT0, 0x3, 1, 0);
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_PHY_STAT0, 0xff, 7, 0);
+}
+
+static void hdmi_core_enable_interrupts(struct hdmi_core_data *core)
+{
+	/* Unmute interrupts */
+	REG_FLD_MOD(core->base, HDMI_CORE_IH_MUTE, 0x0, 1, 0);
+}
+
+int hdmi5_core_handle_irqs(struct hdmi_core_data *core)
+{
+	void __iomem *base = core->base;
+
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT1, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_FC_STAT2, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_AS_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_PHY_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_I2CM_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_CEC_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_VP_STAT0, 0xff, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_IH_I2CMPHY_STAT0, 0xff, 7, 0);
+
+	return 0;
+}
+
+void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+		struct hdmi_config *cfg)
+{
+	struct omap_video_timings video_timing;
+	struct hdmi_video_format video_format;
+	struct hdmi_core_vid_config v_core_cfg;
+
+	hdmi_core_mask_interrupts(core);
+
+	hdmi_core_init(&v_core_cfg, cfg);
+
+	hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg);
+
+	hdmi_wp_video_config_timing(wp, &video_timing);
+
+	/* video config */
+	video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422;
+
+	hdmi_wp_video_config_format(wp, &video_format);
+
+	hdmi_wp_video_config_interface(wp, &video_timing);
+
+	/* support limited range with 24 bit color depth for now */
+	hdmi_core_configure_range(core);
+	cfg->infoframe.quantization_range = HDMI_QUANTIZATION_RANGE_LIMITED;
+
+	/*
+	 * configure core video part, set software reset in the core
+	 */
+	v_core_cfg.packet_mode = HDMI_PACKETMODE24BITPERPIXEL;
+
+	hdmi_core_video_config(core, &v_core_cfg);
+
+	hdmi_core_config_video_packetizer(core);
+	hdmi_core_config_csc(core);
+	hdmi_core_config_video_sampler(core);
+
+	if (cfg->hdmi_dvi_mode == HDMI_HDMI)
+		hdmi_core_write_avi_infoframe(core, &cfg->infoframe);
+
+	hdmi_core_enable_video_path(core);
+
+	hdmi_core_enable_interrupts(core);
+}
+
+static void hdmi5_core_audio_config(struct hdmi_core_data *core,
+			struct hdmi_core_audio_config *cfg)
+{
+	void __iomem *base = core->base;
+	u8 val;
+
+	/* Mute audio before configuring */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCONF, 0xf, 7, 4);
+
+	/* Set the N parameter */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_N1, cfg->n, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_N2, cfg->n >> 8, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_N3, cfg->n >> 16, 3, 0);
+
+	/*
+	 * CTS manual mode. Automatic mode is not supported when using audio
+	 * parallel interface.
+	 */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_CTS3, 1, 4, 4);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_CTS1, cfg->cts, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_CTS2, cfg->cts >> 8, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_AUD_CTS3, cfg->cts >> 16, 3, 0);
+
+	/* Layout of Audio Sample Packets: 2-channel or multichannels */
+	if (cfg->layout == HDMI_AUDIO_LAYOUT_2CH)
+		REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCONF, 0, 0, 0);
+	else
+		REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCONF, 1, 0, 0);
+
+	/* Configure IEC-609580 Validity bits */
+	/* Channel 0 is valid */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, 0, 0, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, 0, 4, 4);
+
+	if (cfg->layout == HDMI_AUDIO_LAYOUT_2CH)
+		val = 1;
+	else
+		val = 0;
+
+	/* Channels 1, 2 setting */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 1, 1);
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 5, 5);
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 2, 2);
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 6, 6);
+	/* Channel 3 setting */
+	if (cfg->layout == HDMI_AUDIO_LAYOUT_6CH)
+		val = 1;
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 3, 3);
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSV, val, 7, 7);
+
+	/* Configure IEC-60958 User bits */
+	/* TODO: should be set by user. */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSU, 0, 7, 0);
+
+	/* Configure IEC-60958 Channel Status word */
+	/* CGMSA */
+	val = cfg->iec60958_cfg->status[5] & IEC958_AES5_CON_CGMSA;
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(0), val, 5, 4);
+
+	/* Copyright */
+	val = (cfg->iec60958_cfg->status[0] &
+			IEC958_AES0_CON_NOT_COPYRIGHT) >> 2;
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(0), val, 0, 0);
+
+	/* Category */
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDSCHNLS(1),
+		cfg->iec60958_cfg->status[1]);
+
+	/* PCM audio mode */
+	val = (cfg->iec60958_cfg->status[0] & IEC958_AES0_CON_MODE) >> 6;
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(2), val, 6, 4);
+
+	/* Source number */
+	val = cfg->iec60958_cfg->status[2] & IEC958_AES2_CON_SOURCE;
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(2), val, 3, 0);
+
+	/* Channel number right 0  */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(3), 2, 3, 0);
+	/* Channel number right 1*/
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(3), 4, 7, 4);
+	/* Channel number right 2  */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(4), 6, 3, 0);
+	/* Channel number right 3*/
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(4), 8, 7, 4);
+	/* Channel number left 0  */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(5), 1, 3, 0);
+	/* Channel number left 1*/
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(5), 3, 7, 4);
+	/* Channel number left 2  */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(6), 5, 3, 0);
+	/* Channel number left 3*/
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCHNLS(6), 7, 7, 4);
+
+	/* Clock accuracy and sample rate */
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDSCHNLS(7),
+		cfg->iec60958_cfg->status[3]);
+
+	/* Original sample rate and word length */
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDSCHNLS(8),
+		cfg->iec60958_cfg->status[4]);
+
+	/* Enable FIFO empty and full interrupts */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_INT, 3, 3, 2);
+
+	/* Configure GPA */
+	/* select HBR/SPDIF interfaces */
+	if (cfg->layout == HDMI_AUDIO_LAYOUT_2CH) {
+		/* select HBR/SPDIF interfaces */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_CONF0, 0, 5, 5);
+		/* enable two channels in GPA */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_GP_CONF1, 3, 7, 0);
+	} else if (cfg->layout == HDMI_AUDIO_LAYOUT_6CH) {
+		/* select HBR/SPDIF interfaces */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_CONF0, 0, 5, 5);
+		/* enable six channels in GPA */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_GP_CONF1, 0x3F, 7, 0);
+	} else {
+		/* select HBR/SPDIF interfaces */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_CONF0, 0, 5, 5);
+		/* enable eight channels in GPA */
+		REG_FLD_MOD(base, HDMI_CORE_AUD_GP_CONF1, 0xFF, 7, 0);
+	}
+
+	/* disable HBR */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_GP_CONF2, 0, 0, 0);
+	/* enable PCUV */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_GP_CONF2, 1, 1, 1);
+	/* enable GPA FIFO full and empty mask */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_GP_MASK, 3, 1, 0);
+	/* set polarity of GPA FIFO empty interrupts */
+	REG_FLD_MOD(base, HDMI_CORE_AUD_GP_POL, 1, 0, 0);
+
+	/* unmute audio */
+	REG_FLD_MOD(base, HDMI_CORE_FC_AUDSCONF, 0, 7, 4);
+}
+
+static void hdmi5_core_audio_infoframe_cfg(struct hdmi_core_data *core,
+	 struct snd_cea_861_aud_if *info_aud)
+{
+	void __iomem *base = core->base;
+
+	/* channel count and coding type fields in AUDICONF0 are swapped */
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDICONF0,
+		(info_aud->db1_ct_cc & CEA861_AUDIO_INFOFRAME_DB1CC) << 4 |
+		(info_aud->db1_ct_cc & CEA861_AUDIO_INFOFRAME_DB1CT) >> 4);
+
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDICONF1, info_aud->db2_sf_ss);
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDICONF2, info_aud->db4_ca);
+	hdmi_write_reg(base, HDMI_CORE_FC_AUDICONF3,
+	  (info_aud->db5_dminh_lsv & CEA861_AUDIO_INFOFRAME_DB5_DM_INH) >> 3 |
+	  (info_aud->db5_dminh_lsv & CEA861_AUDIO_INFOFRAME_DB5_LSV));
+}
+
+int hdmi5_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+			struct omap_dss_audio *audio, u32 pclk)
+{
+	struct hdmi_audio_format audio_format;
+	struct hdmi_audio_dma audio_dma;
+	struct hdmi_core_audio_config core_cfg;
+	int err, n, cts, channel_count;
+	unsigned int fs_nr;
+	bool word_length_16b = false;
+
+	if (!audio || !audio->iec || !audio->cea || !core)
+		return -EINVAL;
+
+	core_cfg.iec60958_cfg = audio->iec;
+
+	if (!(audio->iec->status[4] & IEC958_AES4_CON_MAX_WORDLEN_24) &&
+		(audio->iec->status[4] & IEC958_AES4_CON_WORDLEN_20_16))
+			word_length_16b = true;
+
+	/* only 16-bit word length supported atm */
+	if (!word_length_16b)
+		return -EINVAL;
+
+	switch (audio->iec->status[3] & IEC958_AES3_CON_FS) {
+	case IEC958_AES3_CON_FS_32000:
+		fs_nr = 32000;
+		break;
+	case IEC958_AES3_CON_FS_44100:
+		fs_nr = 44100;
+		break;
+	case IEC958_AES3_CON_FS_48000:
+		fs_nr = 48000;
+		break;
+	case IEC958_AES3_CON_FS_88200:
+		fs_nr = 88200;
+		break;
+	case IEC958_AES3_CON_FS_96000:
+		fs_nr = 96000;
+		break;
+	case IEC958_AES3_CON_FS_176400:
+		fs_nr = 176400;
+		break;
+	case IEC958_AES3_CON_FS_192000:
+		fs_nr = 192000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	err = hdmi_compute_acr(pclk, fs_nr, &n, &cts);
+	core_cfg.n = n;
+	core_cfg.cts = cts;
+
+	/* Audio channels settings */
+	channel_count = (audio->cea->db1_ct_cc & CEA861_AUDIO_INFOFRAME_DB1CC)
+				+ 1;
+
+	if (channel_count == 2)
+		core_cfg.layout = HDMI_AUDIO_LAYOUT_2CH;
+	else if (channel_count == 6)
+		core_cfg.layout = HDMI_AUDIO_LAYOUT_6CH;
+	else
+		core_cfg.layout = HDMI_AUDIO_LAYOUT_8CH;
+
+	/* DMA settings */
+	if (word_length_16b)
+		audio_dma.transfer_size = 0x10;
+	else
+		audio_dma.transfer_size = 0x20;
+	audio_dma.block_size = 0xC0;
+	audio_dma.mode = HDMI_AUDIO_TRANSF_DMA;
+	audio_dma.fifo_threshold = 0x20; /* in number of samples */
+
+	/* audio FIFO format settings for 16-bit samples*/
+	audio_format.samples_per_word = HDMI_AUDIO_ONEWORD_TWOSAMPLES;
+	audio_format.sample_size = HDMI_AUDIO_SAMPLE_16BITS;
+	audio_format.justification = HDMI_AUDIO_JUSTIFY_LEFT;
+	audio_format.sample_order = HDMI_AUDIO_SAMPLE_LEFT_FIRST;
+
+	/* only LPCM atm */
+	audio_format.type = HDMI_AUDIO_TYPE_LPCM;
+
+	/* only allowed option */
+	audio_format.sample_order = HDMI_AUDIO_SAMPLE_LEFT_FIRST;
+
+	/* disable start/stop signals of IEC 60958 blocks */
+	audio_format.en_sig_blk_strt_end = HDMI_AUDIO_BLOCK_SIG_STARTEND_ON;
+
+	/* configure DMA and audio FIFO format*/
+	hdmi_wp_audio_config_dma(wp, &audio_dma);
+	hdmi_wp_audio_config_format(wp, &audio_format);
+
+	/* configure the core */
+	hdmi5_core_audio_config(core, &core_cfg);
+
+	/* configure CEA 861 audio infoframe */
+	hdmi5_core_audio_infoframe_cfg(core, audio->cea);
+
+	return 0;
+}
+
+int hdmi5_core_init(struct platform_device *pdev, struct hdmi_core_data *core)
+{
+	struct resource *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core");
+	if (!res) {
+		DSSERR("can't get CORE IORESOURCE_MEM HDMI\n");
+		return -EINVAL;
+	}
+
+	core->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(core->base)) {
+		DSSERR("can't ioremap HDMI core\n");
+		return PTR_ERR(core->base);
+	}
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.h b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.h
new file mode 100644
index 000000000000..f2f1022c5516
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5_core.h
@@ -0,0 +1,304 @@
+/*
+ * HDMI driver definition for TI OMAP5 processors.
+ *
+ * Copyright (C) 2011-2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _HDMI5_CORE_H_
+#define _HDMI5_CORE_H_
+
+#include "hdmi.h"
+
+/* HDMI IP Core System */
+
+/* HDMI Identification */
+#define HDMI_CORE_DESIGN_ID			0x00000
+#define HDMI_CORE_REVISION_ID			0x00004
+#define HDMI_CORE_PRODUCT_ID0			0x00008
+#define HDMI_CORE_PRODUCT_ID1			0x0000C
+#define HDMI_CORE_CONFIG0_ID			0x00010
+#define HDMI_CORE_CONFIG1_ID			0x00014
+#define HDMI_CORE_CONFIG2_ID			0x00018
+#define HDMI_CORE_CONFIG3_ID			0x0001C
+
+/* HDMI Interrupt */
+#define HDMI_CORE_IH_FC_STAT0			0x00400
+#define HDMI_CORE_IH_FC_STAT1			0x00404
+#define HDMI_CORE_IH_FC_STAT2			0x00408
+#define HDMI_CORE_IH_AS_STAT0			0x0040C
+#define HDMI_CORE_IH_PHY_STAT0			0x00410
+#define HDMI_CORE_IH_I2CM_STAT0			0x00414
+#define HDMI_CORE_IH_CEC_STAT0			0x00418
+#define HDMI_CORE_IH_VP_STAT0			0x0041C
+#define HDMI_CORE_IH_I2CMPHY_STAT0		0x00420
+#define HDMI_CORE_IH_MUTE			0x007FC
+
+/* HDMI Video Sampler */
+#define HDMI_CORE_TX_INVID0			0x00800
+#define HDMI_CORE_TX_INSTUFFING			0x00804
+#define HDMI_CORE_TX_RGYDATA0			0x00808
+#define HDMI_CORE_TX_RGYDATA1			0x0080C
+#define HDMI_CORE_TX_RCRDATA0			0x00810
+#define HDMI_CORE_TX_RCRDATA1			0x00814
+#define HDMI_CORE_TX_BCBDATA0			0x00818
+#define HDMI_CORE_TX_BCBDATA1			0x0081C
+
+/* HDMI Video Packetizer */
+#define HDMI_CORE_VP_STATUS			0x02000
+#define HDMI_CORE_VP_PR_CD			0x02004
+#define HDMI_CORE_VP_STUFF			0x02008
+#define HDMI_CORE_VP_REMAP			0x0200C
+#define HDMI_CORE_VP_CONF			0x02010
+#define HDMI_CORE_VP_STAT			0x02014
+#define HDMI_CORE_VP_INT			0x02018
+#define HDMI_CORE_VP_MASK			0x0201C
+#define HDMI_CORE_VP_POL			0x02020
+
+/* Frame Composer */
+#define HDMI_CORE_FC_INVIDCONF			0x04000
+#define HDMI_CORE_FC_INHACTIV0			0x04004
+#define HDMI_CORE_FC_INHACTIV1			0x04008
+#define HDMI_CORE_FC_INHBLANK0			0x0400C
+#define HDMI_CORE_FC_INHBLANK1			0x04010
+#define HDMI_CORE_FC_INVACTIV0			0x04014
+#define HDMI_CORE_FC_INVACTIV1			0x04018
+#define HDMI_CORE_FC_INVBLANK			0x0401C
+#define HDMI_CORE_FC_HSYNCINDELAY0		0x04020
+#define HDMI_CORE_FC_HSYNCINDELAY1		0x04024
+#define HDMI_CORE_FC_HSYNCINWIDTH0		0x04028
+#define HDMI_CORE_FC_HSYNCINWIDTH1		0x0402C
+#define HDMI_CORE_FC_VSYNCINDELAY		0x04030
+#define HDMI_CORE_FC_VSYNCINWIDTH		0x04034
+#define HDMI_CORE_FC_INFREQ0			0x04038
+#define HDMI_CORE_FC_INFREQ1			0x0403C
+#define HDMI_CORE_FC_INFREQ2			0x04040
+#define HDMI_CORE_FC_CTRLDUR			0x04044
+#define HDMI_CORE_FC_EXCTRLDUR			0x04048
+#define HDMI_CORE_FC_EXCTRLSPAC			0x0404C
+#define HDMI_CORE_FC_CH0PREAM			0x04050
+#define HDMI_CORE_FC_CH1PREAM			0x04054
+#define HDMI_CORE_FC_CH2PREAM			0x04058
+#define HDMI_CORE_FC_AVICONF3			0x0405C
+#define HDMI_CORE_FC_GCP			0x04060
+#define HDMI_CORE_FC_AVICONF0			0x04064
+#define HDMI_CORE_FC_AVICONF1			0x04068
+#define HDMI_CORE_FC_AVICONF2			0x0406C
+#define HDMI_CORE_FC_AVIVID			0x04070
+#define HDMI_CORE_FC_AVIETB0			0x04074
+#define HDMI_CORE_FC_AVIETB1			0x04078
+#define HDMI_CORE_FC_AVISBB0			0x0407C
+#define HDMI_CORE_FC_AVISBB1			0x04080
+#define HDMI_CORE_FC_AVIELB0			0x04084
+#define HDMI_CORE_FC_AVIELB1			0x04088
+#define HDMI_CORE_FC_AVISRB0			0x0408C
+#define HDMI_CORE_FC_AVISRB1			0x04090
+#define HDMI_CORE_FC_AUDICONF0			0x04094
+#define HDMI_CORE_FC_AUDICONF1			0x04098
+#define HDMI_CORE_FC_AUDICONF2			0x0409C
+#define HDMI_CORE_FC_AUDICONF3			0x040A0
+#define HDMI_CORE_FC_VSDIEEEID0			0x040A4
+#define HDMI_CORE_FC_VSDSIZE			0x040A8
+#define HDMI_CORE_FC_VSDIEEEID1			0x040C0
+#define HDMI_CORE_FC_VSDIEEEID2			0x040C4
+#define HDMI_CORE_FC_VSDPAYLOAD(n)		(n * 4 + 0x040C8)
+#define HDMI_CORE_FC_SPDVENDORNAME(n)		(n * 4 + 0x04128)
+#define HDMI_CORE_FC_SPDPRODUCTNAME(n)		(n * 4 + 0x04148)
+#define HDMI_CORE_FC_SPDDEVICEINF		0x04188
+#define HDMI_CORE_FC_AUDSCONF			0x0418C
+#define HDMI_CORE_FC_AUDSSTAT			0x04190
+#define HDMI_CORE_FC_AUDSV			0x04194
+#define HDMI_CORE_FC_AUDSU			0x04198
+#define HDMI_CORE_FC_AUDSCHNLS(n)		(n * 4 + 0x0419C)
+#define HDMI_CORE_FC_CTRLQHIGH			0x041CC
+#define HDMI_CORE_FC_CTRLQLOW			0x041D0
+#define HDMI_CORE_FC_ACP0			0x041D4
+#define HDMI_CORE_FC_ACP(n)			((16-n) * 4 + 0x04208)
+#define HDMI_CORE_FC_ISCR1_0			0x04248
+#define HDMI_CORE_FC_ISCR1(n)			((16-n) * 4 + 0x0424C)
+#define HDMI_CORE_FC_ISCR2(n)			((15-n) * 4 + 0x0428C)
+#define HDMI_CORE_FC_DATAUTO0			0x042CC
+#define HDMI_CORE_FC_DATAUTO1			0x042D0
+#define HDMI_CORE_FC_DATAUTO2			0x042D4
+#define HDMI_CORE_FC_DATMAN			0x042D8
+#define HDMI_CORE_FC_DATAUTO3			0x042DC
+#define HDMI_CORE_FC_RDRB(n)			(n * 4 + 0x042E0)
+#define HDMI_CORE_FC_STAT0			0x04340
+#define HDMI_CORE_FC_INT0			0x04344
+#define HDMI_CORE_FC_MASK0			0x04348
+#define HDMI_CORE_FC_POL0			0x0434C
+#define HDMI_CORE_FC_STAT1			0x04350
+#define HDMI_CORE_FC_INT1			0x04354
+#define HDMI_CORE_FC_MASK1			0x04358
+#define HDMI_CORE_FC_POL1			0x0435C
+#define HDMI_CORE_FC_STAT2			0x04360
+#define HDMI_CORE_FC_INT2			0x04364
+#define HDMI_CORE_FC_MASK2			0x04368
+#define HDMI_CORE_FC_POL2			0x0436C
+#define HDMI_CORE_FC_PRCONF			0x04380
+#define HDMI_CORE_FC_GMD_STAT			0x04400
+#define HDMI_CORE_FC_GMD_EN			0x04404
+#define HDMI_CORE_FC_GMD_UP			0x04408
+#define HDMI_CORE_FC_GMD_CONF			0x0440C
+#define HDMI_CORE_FC_GMD_HB			0x04410
+#define HDMI_CORE_FC_GMD_PB(n)			(n * 4 + 0x04414)
+#define HDMI_CORE_FC_DBGFORCE			0x04800
+#define HDMI_CORE_FC_DBGAUD0CH0			0x04804
+#define HDMI_CORE_FC_DBGAUD1CH0			0x04808
+#define HDMI_CORE_FC_DBGAUD2CH0			0x0480C
+#define HDMI_CORE_FC_DBGAUD0CH1			0x04810
+#define HDMI_CORE_FC_DBGAUD1CH1			0x04814
+#define HDMI_CORE_FC_DBGAUD2CH1			0x04818
+#define HDMI_CORE_FC_DBGAUD0CH2			0x0481C
+#define HDMI_CORE_FC_DBGAUD1CH2			0x04820
+#define HDMI_CORE_FC_DBGAUD2CH2			0x04824
+#define HDMI_CORE_FC_DBGAUD0CH3			0x04828
+#define HDMI_CORE_FC_DBGAUD1CH3			0x0482C
+#define HDMI_CORE_FC_DBGAUD2CH3			0x04830
+#define HDMI_CORE_FC_DBGAUD0CH4			0x04834
+#define HDMI_CORE_FC_DBGAUD1CH4			0x04838
+#define HDMI_CORE_FC_DBGAUD2CH4			0x0483C
+#define HDMI_CORE_FC_DBGAUD0CH5			0x04840
+#define HDMI_CORE_FC_DBGAUD1CH5			0x04844
+#define HDMI_CORE_FC_DBGAUD2CH5			0x04848
+#define HDMI_CORE_FC_DBGAUD0CH6			0x0484C
+#define HDMI_CORE_FC_DBGAUD1CH6			0x04850
+#define HDMI_CORE_FC_DBGAUD2CH6			0x04854
+#define HDMI_CORE_FC_DBGAUD0CH7			0x04858
+#define HDMI_CORE_FC_DBGAUD1CH7			0x0485C
+#define HDMI_CORE_FC_DBGAUD2CH7			0x04860
+#define HDMI_CORE_FC_DBGTMDS0			0x04864
+#define HDMI_CORE_FC_DBGTMDS1			0x04868
+#define HDMI_CORE_FC_DBGTMDS2			0x0486C
+#define HDMI_CORE_PHY_MASK0			0x0C018
+#define HDMI_CORE_PHY_I2CM_INT_ADDR		0x0C09C
+#define HDMI_CORE_PHY_I2CM_CTLINT_ADDR		0x0C0A0
+
+/* HDMI Audio */
+#define HDMI_CORE_AUD_CONF0			0x0C400
+#define HDMI_CORE_AUD_CONF1			0x0C404
+#define HDMI_CORE_AUD_INT			0x0C408
+#define HDMI_CORE_AUD_N1			0x0C800
+#define HDMI_CORE_AUD_N2			0x0C804
+#define HDMI_CORE_AUD_N3			0x0C808
+#define HDMI_CORE_AUD_CTS1			0x0C80C
+#define HDMI_CORE_AUD_CTS2			0x0C810
+#define HDMI_CORE_AUD_CTS3			0x0C814
+#define HDMI_CORE_AUD_INCLKFS			0x0C818
+#define HDMI_CORE_AUD_CC08			0x0CC08
+#define HDMI_CORE_AUD_GP_CONF0			0x0D400
+#define HDMI_CORE_AUD_GP_CONF1			0x0D404
+#define HDMI_CORE_AUD_GP_CONF2			0x0D408
+#define HDMI_CORE_AUD_D010			0x0D010
+#define HDMI_CORE_AUD_GP_STAT			0x0D40C
+#define HDMI_CORE_AUD_GP_INT			0x0D410
+#define HDMI_CORE_AUD_GP_POL			0x0D414
+#define HDMI_CORE_AUD_GP_MASK			0x0D418
+
+/* HDMI Main Controller */
+#define HDMI_CORE_MC_CLKDIS			0x10004
+#define HDMI_CORE_MC_SWRSTZREQ			0x10008
+#define HDMI_CORE_MC_FLOWCTRL			0x10010
+#define HDMI_CORE_MC_PHYRSTZ			0x10014
+#define HDMI_CORE_MC_LOCKONCLOCK		0x10018
+
+/* HDMI COLOR SPACE CONVERTER */
+#define HDMI_CORE_CSC_CFG			0x10400
+#define HDMI_CORE_CSC_SCALE			0x10404
+#define HDMI_CORE_CSC_COEF_A1_MSB		0x10408
+#define HDMI_CORE_CSC_COEF_A1_LSB		0x1040C
+#define HDMI_CORE_CSC_COEF_A2_MSB		0x10410
+#define HDMI_CORE_CSC_COEF_A2_LSB		0x10414
+#define HDMI_CORE_CSC_COEF_A3_MSB		0x10418
+#define HDMI_CORE_CSC_COEF_A3_LSB		0x1041C
+#define HDMI_CORE_CSC_COEF_A4_MSB		0x10420
+#define HDMI_CORE_CSC_COEF_A4_LSB		0x10424
+#define HDMI_CORE_CSC_COEF_B1_MSB		0x10428
+#define HDMI_CORE_CSC_COEF_B1_LSB		0x1042C
+#define HDMI_CORE_CSC_COEF_B2_MSB		0x10430
+#define HDMI_CORE_CSC_COEF_B2_LSB		0x10434
+#define HDMI_CORE_CSC_COEF_B3_MSB		0x10438
+#define HDMI_CORE_CSC_COEF_B3_LSB		0x1043C
+#define HDMI_CORE_CSC_COEF_B4_MSB		0x10440
+#define HDMI_CORE_CSC_COEF_B4_LSB		0x10444
+#define HDMI_CORE_CSC_COEF_C1_MSB		0x10448
+#define HDMI_CORE_CSC_COEF_C1_LSB		0x1044C
+#define HDMI_CORE_CSC_COEF_C2_MSB		0x10450
+#define HDMI_CORE_CSC_COEF_C2_LSB		0x10454
+#define HDMI_CORE_CSC_COEF_C3_MSB		0x10458
+#define HDMI_CORE_CSC_COEF_C3_LSB		0x1045C
+#define HDMI_CORE_CSC_COEF_C4_MSB		0x10460
+#define HDMI_CORE_CSC_COEF_C4_LSB		0x10464
+
+/* HDMI HDCP */
+#define HDMI_CORE_HDCP_MASK			0x14020
+
+/* HDMI CEC */
+#define HDMI_CORE_CEC_MASK			0x17408
+
+/* HDMI I2C Master */
+#define HDMI_CORE_I2CM_SLAVE			0x157C8
+#define HDMI_CORE_I2CM_ADDRESS			0x157CC
+#define HDMI_CORE_I2CM_DATAO			0x157D0
+#define HDMI_CORE_I2CM_DATAI			0X157D4
+#define HDMI_CORE_I2CM_OPERATION		0x157D8
+#define HDMI_CORE_I2CM_INT			0x157DC
+#define HDMI_CORE_I2CM_CTLINT			0x157E0
+#define HDMI_CORE_I2CM_DIV			0x157E4
+#define HDMI_CORE_I2CM_SEGADDR			0x157E8
+#define HDMI_CORE_I2CM_SOFTRSTZ			0x157EC
+#define HDMI_CORE_I2CM_SEGPTR			0x157F0
+#define HDMI_CORE_I2CM_SS_SCL_HCNT_1_ADDR	0x157F4
+#define HDMI_CORE_I2CM_SS_SCL_HCNT_0_ADDR	0x157F8
+#define HDMI_CORE_I2CM_SS_SCL_LCNT_1_ADDR	0x157FC
+#define HDMI_CORE_I2CM_SS_SCL_LCNT_0_ADDR	0x15800
+#define HDMI_CORE_I2CM_FS_SCL_HCNT_1_ADDR	0x15804
+#define HDMI_CORE_I2CM_FS_SCL_HCNT_0_ADDR	0x15808
+#define HDMI_CORE_I2CM_FS_SCL_LCNT_1_ADDR	0x1580C
+#define HDMI_CORE_I2CM_FS_SCL_LCNT_0_ADDR	0x15810
+#define HDMI_CORE_I2CM_SDA_HOLD_ADDR		0x15814
+
+enum hdmi_core_packet_mode {
+	HDMI_PACKETMODERESERVEDVALUE = 0,
+	HDMI_PACKETMODE24BITPERPIXEL = 4,
+	HDMI_PACKETMODE30BITPERPIXEL = 5,
+	HDMI_PACKETMODE36BITPERPIXEL = 6,
+	HDMI_PACKETMODE48BITPERPIXEL = 7,
+};
+
+struct hdmi_core_vid_config {
+	struct hdmi_config v_fc_config;
+	enum hdmi_core_packet_mode packet_mode;
+	int data_enable_pol;
+	int vblank_osc;
+	int hblank;
+	int vblank;
+};
+
+struct csc_table {
+	u16 a1, a2, a3, a4;
+	u16 b1, b2, b3, b4;
+	u16 c1, c2, c3, c4;
+};
+
+int hdmi5_read_edid(struct hdmi_core_data *core, u8 *edid, int len);
+void hdmi5_core_dump(struct hdmi_core_data *core, struct seq_file *s);
+int hdmi5_core_handle_irqs(struct hdmi_core_data *core);
+void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+			struct hdmi_config *cfg);
+int hdmi5_core_init(struct platform_device *pdev, struct hdmi_core_data *core);
+
+int hdmi5_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
+			struct omap_dss_audio *audio, u32 pclk);
+#endif
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c
new file mode 100644
index 000000000000..1b8fcc6c4ba1
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c
@@ -0,0 +1,148 @@
+
+#define DSS_SUBSYS_NAME "HDMI"
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <video/omapdss.h>
+
+#include "hdmi.h"
+
+int hdmi_parse_lanes_of(struct platform_device *pdev, struct device_node *ep,
+	struct hdmi_phy_data *phy)
+{
+	struct property *prop;
+	int r, len;
+
+	prop = of_find_property(ep, "lanes", &len);
+	if (prop) {
+		u32 lanes[8];
+
+		if (len / sizeof(u32) != ARRAY_SIZE(lanes)) {
+			dev_err(&pdev->dev, "bad number of lanes\n");
+			return -EINVAL;
+		}
+
+		r = of_property_read_u32_array(ep, "lanes", lanes,
+			ARRAY_SIZE(lanes));
+		if (r) {
+			dev_err(&pdev->dev, "failed to read lane data\n");
+			return r;
+		}
+
+		r = hdmi_phy_parse_lanes(phy, lanes);
+		if (r) {
+			dev_err(&pdev->dev, "failed to parse lane data\n");
+			return r;
+		}
+	} else {
+		static const u32 default_lanes[] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+
+		r = hdmi_phy_parse_lanes(phy, default_lanes);
+		if (WARN_ON(r)) {
+			dev_err(&pdev->dev, "failed to parse lane data\n");
+			return r;
+		}
+	}
+
+	return 0;
+}
+
+int hdmi_compute_acr(u32 pclk, u32 sample_freq, u32 *n, u32 *cts)
+{
+	u32 deep_color;
+	bool deep_color_correct = false;
+
+	if (n == NULL || cts == NULL)
+		return -EINVAL;
+
+	/* TODO: When implemented, query deep color mode here. */
+	deep_color = 100;
+
+	/*
+	 * When using deep color, the default N value (as in the HDMI
+	 * specification) yields to an non-integer CTS. Hence, we
+	 * modify it while keeping the restrictions described in
+	 * section 7.2.1 of the HDMI 1.4a specification.
+	 */
+	switch (sample_freq) {
+	case 32000:
+	case 48000:
+	case 96000:
+	case 192000:
+		if (deep_color == 125)
+			if (pclk == 27027000 || pclk == 74250000)
+				deep_color_correct = true;
+		if (deep_color == 150)
+			if (pclk == 27027000)
+				deep_color_correct = true;
+		break;
+	case 44100:
+	case 88200:
+	case 176400:
+		if (deep_color == 125)
+			if (pclk == 27027000)
+				deep_color_correct = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (deep_color_correct) {
+		switch (sample_freq) {
+		case 32000:
+			*n = 8192;
+			break;
+		case 44100:
+			*n = 12544;
+			break;
+		case 48000:
+			*n = 8192;
+			break;
+		case 88200:
+			*n = 25088;
+			break;
+		case 96000:
+			*n = 16384;
+			break;
+		case 176400:
+			*n = 50176;
+			break;
+		case 192000:
+			*n = 32768;
+			break;
+		default:
+			return -EINVAL;
+		}
+	} else {
+		switch (sample_freq) {
+		case 32000:
+			*n = 4096;
+			break;
+		case 44100:
+			*n = 6272;
+			break;
+		case 48000:
+			*n = 6144;
+			break;
+		case 88200:
+			*n = 12544;
+			break;
+		case 96000:
+			*n = 12288;
+			break;
+		case 176400:
+			*n = 25088;
+			break;
+		case 192000:
+			*n = 24576;
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+	/* Calculate CTS. See HDMI 1.3a or 1.4a specifications */
+	*cts = (pclk/1000) * (*n / 128) * deep_color / (sample_freq / 10);
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c
new file mode 100644
index 000000000000..1f5d19c119ce
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c
@@ -0,0 +1,247 @@
+/*
+ * HDMI PHY
+ *
+ * Copyright (C) 2013 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "hdmi.h"
+
+struct hdmi_phy_features {
+	bool bist_ctrl;
+	bool ldo_voltage;
+	unsigned long max_phy;
+};
+
+static const struct hdmi_phy_features *phy_feat;
+
+void hdmi_phy_dump(struct hdmi_phy_data *phy, struct seq_file *s)
+{
+#define DUMPPHY(r) seq_printf(s, "%-35s %08x\n", #r,\
+		hdmi_read_reg(phy->base, r))
+
+	DUMPPHY(HDMI_TXPHY_TX_CTRL);
+	DUMPPHY(HDMI_TXPHY_DIGITAL_CTRL);
+	DUMPPHY(HDMI_TXPHY_POWER_CTRL);
+	DUMPPHY(HDMI_TXPHY_PAD_CFG_CTRL);
+	if (phy_feat->bist_ctrl)
+		DUMPPHY(HDMI_TXPHY_BIST_CONTROL);
+}
+
+int hdmi_phy_parse_lanes(struct hdmi_phy_data *phy, const u32 *lanes)
+{
+	int i;
+
+	for (i = 0; i < 8; i += 2) {
+		u8 lane, pol;
+		int dx, dy;
+
+		dx = lanes[i];
+		dy = lanes[i + 1];
+
+		if (dx < 0 || dx >= 8)
+			return -EINVAL;
+
+		if (dy < 0 || dy >= 8)
+			return -EINVAL;
+
+		if (dx & 1) {
+			if (dy != dx - 1)
+				return -EINVAL;
+			pol = 1;
+		} else {
+			if (dy != dx + 1)
+				return -EINVAL;
+			pol = 0;
+		}
+
+		lane = dx / 2;
+
+		phy->lane_function[lane] = i / 2;
+		phy->lane_polarity[lane] = pol;
+	}
+
+	return 0;
+}
+
+static void hdmi_phy_configure_lanes(struct hdmi_phy_data *phy)
+{
+	static const u16 pad_cfg_list[] = {
+		0x0123,
+		0x0132,
+		0x0312,
+		0x0321,
+		0x0231,
+		0x0213,
+		0x1023,
+		0x1032,
+		0x3012,
+		0x3021,
+		0x2031,
+		0x2013,
+		0x1203,
+		0x1302,
+		0x3102,
+		0x3201,
+		0x2301,
+		0x2103,
+		0x1230,
+		0x1320,
+		0x3120,
+		0x3210,
+		0x2310,
+		0x2130,
+	};
+
+	u16 lane_cfg = 0;
+	int i;
+	unsigned lane_cfg_val;
+	u16 pol_val = 0;
+
+	for (i = 0; i < 4; ++i)
+		lane_cfg |= phy->lane_function[i] << ((3 - i) * 4);
+
+	pol_val |= phy->lane_polarity[0] << 0;
+	pol_val |= phy->lane_polarity[1] << 3;
+	pol_val |= phy->lane_polarity[2] << 2;
+	pol_val |= phy->lane_polarity[3] << 1;
+
+	for (i = 0; i < ARRAY_SIZE(pad_cfg_list); ++i)
+		if (pad_cfg_list[i] == lane_cfg)
+			break;
+
+	if (WARN_ON(i == ARRAY_SIZE(pad_cfg_list)))
+		i = 0;
+
+	lane_cfg_val = i;
+
+	REG_FLD_MOD(phy->base, HDMI_TXPHY_PAD_CFG_CTRL, lane_cfg_val, 26, 22);
+	REG_FLD_MOD(phy->base, HDMI_TXPHY_PAD_CFG_CTRL, pol_val, 30, 27);
+}
+
+int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
+	unsigned long lfbitclk)
+{
+	u8 freqout;
+
+	/*
+	 * Read address 0 in order to get the SCP reset done completed
+	 * Dummy access performed to make sure reset is done
+	 */
+	hdmi_read_reg(phy->base, HDMI_TXPHY_TX_CTRL);
+
+	/*
+	 * In OMAP5+, the HFBITCLK must be divided by 2 before issuing the
+	 * HDMI_PHYPWRCMD_LDOON command.
+	*/
+	if (phy_feat->bist_ctrl)
+		REG_FLD_MOD(phy->base, HDMI_TXPHY_BIST_CONTROL, 1, 11, 11);
+
+	/*
+	 * If the hfbitclk != lfbitclk, it means the lfbitclk was configured
+	 * to be used for TMDS.
+	 */
+	if (hfbitclk != lfbitclk)
+		freqout = 0;
+	else if (hfbitclk / 10 < phy_feat->max_phy)
+		freqout = 1;
+	else
+		freqout = 2;
+
+	/*
+	 * Write to phy address 0 to configure the clock
+	 * use HFBITCLK write HDMI_TXPHY_TX_CONTROL_FREQOUT field
+	 */
+	REG_FLD_MOD(phy->base, HDMI_TXPHY_TX_CTRL, freqout, 31, 30);
+
+	/* Write to phy address 1 to start HDMI line (TXVALID and TMDSCLKEN) */
+	hdmi_write_reg(phy->base, HDMI_TXPHY_DIGITAL_CTRL, 0xF0000000);
+
+	/* Setup max LDO voltage */
+	if (phy_feat->ldo_voltage)
+		REG_FLD_MOD(phy->base, HDMI_TXPHY_POWER_CTRL, 0xB, 3, 0);
+
+	hdmi_phy_configure_lanes(phy);
+
+	return 0;
+}
+
+static const struct hdmi_phy_features omap44xx_phy_feats = {
+	.bist_ctrl	=	false,
+	.ldo_voltage	=	true,
+	.max_phy	=	185675000,
+};
+
+static const struct hdmi_phy_features omap54xx_phy_feats = {
+	.bist_ctrl	=	true,
+	.ldo_voltage	=	false,
+	.max_phy	=	186000000,
+};
+
+static int hdmi_phy_init_features(struct platform_device *pdev)
+{
+	struct hdmi_phy_features *dst;
+	const struct hdmi_phy_features *src;
+
+	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
+	if (!dst) {
+		dev_err(&pdev->dev, "Failed to allocate HDMI PHY Features\n");
+		return -ENOMEM;
+	}
+
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		src = &omap44xx_phy_feats;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+	case OMAPDSS_VER_DRA7xx:
+		src = &omap54xx_phy_feats;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	memcpy(dst, src, sizeof(*dst));
+	phy_feat = dst;
+
+	return 0;
+}
+
+int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy)
+{
+	int r;
+	struct resource *res;
+
+	r = hdmi_phy_init_features(pdev);
+	if (r)
+		return r;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "phy");
+	if (!res) {
+		DSSERR("can't get PHY mem resource\n");
+		return -EINVAL;
+	}
+
+	phy->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(phy->base)) {
+		DSSERR("can't ioremap TX PHY\n");
+		return PTR_ERR(phy->base);
+	}
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c
new file mode 100644
index 000000000000..06e23a7c432c
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c
@@ -0,0 +1,255 @@
+/*
+ * HDMI PLL
+ *
+ * Copyright (C) 2013 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#define DSS_SUBSYS_NAME "HDMIPLL"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "hdmi.h"
+
+void hdmi_pll_dump(struct hdmi_pll_data *pll, struct seq_file *s)
+{
+#define DUMPPLL(r) seq_printf(s, "%-35s %08x\n", #r,\
+		hdmi_read_reg(pll->base, r))
+
+	DUMPPLL(PLLCTRL_PLL_CONTROL);
+	DUMPPLL(PLLCTRL_PLL_STATUS);
+	DUMPPLL(PLLCTRL_PLL_GO);
+	DUMPPLL(PLLCTRL_CFG1);
+	DUMPPLL(PLLCTRL_CFG2);
+	DUMPPLL(PLLCTRL_CFG3);
+	DUMPPLL(PLLCTRL_SSC_CFG1);
+	DUMPPLL(PLLCTRL_SSC_CFG2);
+	DUMPPLL(PLLCTRL_CFG4);
+}
+
+void hdmi_pll_compute(struct hdmi_pll_data *pll,
+	unsigned long target_tmds, struct dss_pll_clock_info *pi)
+{
+	unsigned long fint, clkdco, clkout;
+	unsigned long target_bitclk, target_clkdco;
+	unsigned long min_dco;
+	unsigned n, m, mf, m2, sd;
+	unsigned long clkin;
+	const struct dss_pll_hw *hw = pll->pll.hw;
+
+	clkin = clk_get_rate(pll->pll.clkin);
+
+	DSSDBG("clkin %lu, target tmds %lu\n", clkin, target_tmds);
+
+	target_bitclk = target_tmds * 10;
+
+	/* Fint */
+	n = DIV_ROUND_UP(clkin, hw->fint_max);
+	fint = clkin / n;
+
+	/* adjust m2 so that the clkdco will be high enough */
+	min_dco = roundup(hw->clkdco_min, fint);
+	m2 = DIV_ROUND_UP(min_dco, target_bitclk);
+	if (m2 == 0)
+		m2 = 1;
+
+	target_clkdco = target_bitclk * m2;
+	m = target_clkdco / fint;
+
+	clkdco = fint * m;
+
+	/* adjust clkdco with fractional mf */
+	if (WARN_ON(target_clkdco - clkdco > fint))
+		mf = 0;
+	else
+		mf = (u32)div_u64(262144ull * (target_clkdco - clkdco), fint);
+
+	if (mf > 0)
+		clkdco += (u32)div_u64((u64)mf * fint, 262144);
+
+	clkout = clkdco / m2;
+
+	/* sigma-delta */
+	sd = DIV_ROUND_UP(fint * m, 250000000);
+
+	DSSDBG("N = %u, M = %u, M.f = %u, M2 = %u, SD = %u\n",
+		n, m, mf, m2, sd);
+	DSSDBG("Fint %lu, clkdco %lu, clkout %lu\n", fint, clkdco, clkout);
+
+	pi->n = n;
+	pi->m = m;
+	pi->mf = mf;
+	pi->mX[0] = m2;
+	pi->sd = sd;
+
+	pi->fint = fint;
+	pi->clkdco = clkdco;
+	pi->clkout[0] = clkout;
+}
+
+static int hdmi_pll_enable(struct dss_pll *dsspll)
+{
+	struct hdmi_pll_data *pll = container_of(dsspll, struct hdmi_pll_data, pll);
+	struct hdmi_wp_data *wp = pll->wp;
+	u16 r = 0;
+
+	dss_ctrl_pll_enable(DSS_PLL_HDMI, true);
+
+	r = hdmi_wp_set_pll_pwr(wp, HDMI_PLLPWRCMD_BOTHON_ALLCLKS);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void hdmi_pll_disable(struct dss_pll *dsspll)
+{
+	struct hdmi_pll_data *pll = container_of(dsspll, struct hdmi_pll_data, pll);
+	struct hdmi_wp_data *wp = pll->wp;
+
+	hdmi_wp_set_pll_pwr(wp, HDMI_PLLPWRCMD_ALLOFF);
+
+	dss_ctrl_pll_enable(DSS_PLL_HDMI, false);
+}
+
+static const struct dss_pll_ops dsi_pll_ops = {
+	.enable = hdmi_pll_enable,
+	.disable = hdmi_pll_disable,
+	.set_config = dss_pll_write_config_type_b,
+};
+
+static const struct dss_pll_hw dss_omap4_hdmi_pll_hw = {
+	.n_max = 255,
+	.m_min = 20,
+	.m_max = 4095,
+	.mX_max = 127,
+	.fint_min = 500000,
+	.fint_max = 2500000,
+
+	.clkdco_min = 500000000,
+	.clkdco_low = 1000000000,
+	.clkdco_max = 2000000000,
+
+	.n_msb = 8,
+	.n_lsb = 1,
+	.m_msb = 20,
+	.m_lsb = 9,
+
+	.mX_msb[0] = 24,
+	.mX_lsb[0] = 18,
+
+	.has_selfreqdco = true,
+};
+
+static const struct dss_pll_hw dss_omap5_hdmi_pll_hw = {
+	.n_max = 255,
+	.m_min = 20,
+	.m_max = 2045,
+	.mX_max = 127,
+	.fint_min = 620000,
+	.fint_max = 2500000,
+
+	.clkdco_min = 750000000,
+	.clkdco_low = 1500000000,
+	.clkdco_max = 2500000000UL,
+
+	.n_msb = 8,
+	.n_lsb = 1,
+	.m_msb = 20,
+	.m_lsb = 9,
+
+	.mX_msb[0] = 24,
+	.mX_lsb[0] = 18,
+
+	.has_selfreqdco = true,
+	.has_refsel = true,
+};
+
+static int dsi_init_pll_data(struct platform_device *pdev, struct hdmi_pll_data *hpll)
+{
+	struct dss_pll *pll = &hpll->pll;
+	struct clk *clk;
+	int r;
+
+	clk = devm_clk_get(&pdev->dev, "sys_clk");
+	if (IS_ERR(clk)) {
+		DSSERR("can't get sys_clk\n");
+		return PTR_ERR(clk);
+	}
+
+	pll->name = "hdmi";
+	pll->id = DSS_PLL_HDMI;
+	pll->base = hpll->base;
+	pll->clkin = clk;
+
+	switch (omapdss_get_version()) {
+	case OMAPDSS_VER_OMAP4430_ES1:
+	case OMAPDSS_VER_OMAP4430_ES2:
+	case OMAPDSS_VER_OMAP4:
+		pll->hw = &dss_omap4_hdmi_pll_hw;
+		break;
+
+	case OMAPDSS_VER_OMAP5:
+	case OMAPDSS_VER_DRA7xx:
+		pll->hw = &dss_omap5_hdmi_pll_hw;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	pll->ops = &dsi_pll_ops;
+
+	r = dss_pll_register(pll);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+int hdmi_pll_init(struct platform_device *pdev, struct hdmi_pll_data *pll,
+	struct hdmi_wp_data *wp)
+{
+	int r;
+	struct resource *res;
+
+	pll->wp = wp;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pll");
+	if (!res) {
+		DSSERR("can't get PLL mem resource\n");
+		return -EINVAL;
+	}
+
+	pll->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pll->base)) {
+		DSSERR("can't ioremap PLLCTRL\n");
+		return PTR_ERR(pll->base);
+	}
+
+	r = dsi_init_pll_data(pdev, pll);
+	if (r) {
+		DSSERR("failed to init HDMI PLL\n");
+		return r;
+	}
+
+	return 0;
+}
+
+void hdmi_pll_uninit(struct hdmi_pll_data *hpll)
+{
+	struct dss_pll *pll = &hpll->pll;
+
+	dss_pll_unregister(pll);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c
new file mode 100644
index 000000000000..7c544bc56fb5
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c
@@ -0,0 +1,282 @@
+/*
+ * HDMI wrapper
+ *
+ * Copyright (C) 2013 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#define DSS_SUBSYS_NAME "HDMIWP"
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "hdmi.h"
+
+void hdmi_wp_dump(struct hdmi_wp_data *wp, struct seq_file *s)
+{
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, hdmi_read_reg(wp->base, r))
+
+	DUMPREG(HDMI_WP_REVISION);
+	DUMPREG(HDMI_WP_SYSCONFIG);
+	DUMPREG(HDMI_WP_IRQSTATUS_RAW);
+	DUMPREG(HDMI_WP_IRQSTATUS);
+	DUMPREG(HDMI_WP_IRQENABLE_SET);
+	DUMPREG(HDMI_WP_IRQENABLE_CLR);
+	DUMPREG(HDMI_WP_IRQWAKEEN);
+	DUMPREG(HDMI_WP_PWR_CTRL);
+	DUMPREG(HDMI_WP_DEBOUNCE);
+	DUMPREG(HDMI_WP_VIDEO_CFG);
+	DUMPREG(HDMI_WP_VIDEO_SIZE);
+	DUMPREG(HDMI_WP_VIDEO_TIMING_H);
+	DUMPREG(HDMI_WP_VIDEO_TIMING_V);
+	DUMPREG(HDMI_WP_CLK);
+	DUMPREG(HDMI_WP_AUDIO_CFG);
+	DUMPREG(HDMI_WP_AUDIO_CFG2);
+	DUMPREG(HDMI_WP_AUDIO_CTRL);
+	DUMPREG(HDMI_WP_AUDIO_DATA);
+}
+
+u32 hdmi_wp_get_irqstatus(struct hdmi_wp_data *wp)
+{
+	return hdmi_read_reg(wp->base, HDMI_WP_IRQSTATUS);
+}
+
+void hdmi_wp_set_irqstatus(struct hdmi_wp_data *wp, u32 irqstatus)
+{
+	hdmi_write_reg(wp->base, HDMI_WP_IRQSTATUS, irqstatus);
+	/* flush posted write */
+	hdmi_read_reg(wp->base, HDMI_WP_IRQSTATUS);
+}
+
+void hdmi_wp_set_irqenable(struct hdmi_wp_data *wp, u32 mask)
+{
+	hdmi_write_reg(wp->base, HDMI_WP_IRQENABLE_SET, mask);
+}
+
+void hdmi_wp_clear_irqenable(struct hdmi_wp_data *wp, u32 mask)
+{
+	hdmi_write_reg(wp->base, HDMI_WP_IRQENABLE_CLR, mask);
+}
+
+/* PHY_PWR_CMD */
+int hdmi_wp_set_phy_pwr(struct hdmi_wp_data *wp, enum hdmi_phy_pwr val)
+{
+	/* Return if already the state */
+	if (REG_GET(wp->base, HDMI_WP_PWR_CTRL, 5, 4) == val)
+		return 0;
+
+	/* Command for power control of HDMI PHY */
+	REG_FLD_MOD(wp->base, HDMI_WP_PWR_CTRL, val, 7, 6);
+
+	/* Status of the power control of HDMI PHY */
+	if (hdmi_wait_for_bit_change(wp->base, HDMI_WP_PWR_CTRL, 5, 4, val)
+			!= val) {
+		DSSERR("Failed to set PHY power mode to %d\n", val);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+/* PLL_PWR_CMD */
+int hdmi_wp_set_pll_pwr(struct hdmi_wp_data *wp, enum hdmi_pll_pwr val)
+{
+	/* Command for power control of HDMI PLL */
+	REG_FLD_MOD(wp->base, HDMI_WP_PWR_CTRL, val, 3, 2);
+
+	/* wait till PHY_PWR_STATUS is set */
+	if (hdmi_wait_for_bit_change(wp->base, HDMI_WP_PWR_CTRL, 1, 0, val)
+			!= val) {
+		DSSERR("Failed to set PLL_PWR_STATUS\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+int hdmi_wp_video_start(struct hdmi_wp_data *wp)
+{
+	REG_FLD_MOD(wp->base, HDMI_WP_VIDEO_CFG, true, 31, 31);
+
+	return 0;
+}
+
+void hdmi_wp_video_stop(struct hdmi_wp_data *wp)
+{
+	int i;
+
+	hdmi_write_reg(wp->base, HDMI_WP_IRQSTATUS, HDMI_IRQ_VIDEO_FRAME_DONE);
+
+	REG_FLD_MOD(wp->base, HDMI_WP_VIDEO_CFG, false, 31, 31);
+
+	for (i = 0; i < 50; ++i) {
+		u32 v;
+
+		msleep(20);
+
+		v = hdmi_read_reg(wp->base, HDMI_WP_IRQSTATUS_RAW);
+		if (v & HDMI_IRQ_VIDEO_FRAME_DONE)
+			return;
+	}
+
+	DSSERR("no HDMI FRAMEDONE when disabling output\n");
+}
+
+void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
+		struct hdmi_video_format *video_fmt)
+{
+	u32 l = 0;
+
+	REG_FLD_MOD(wp->base, HDMI_WP_VIDEO_CFG, video_fmt->packing_mode,
+		10, 8);
+
+	l |= FLD_VAL(video_fmt->y_res, 31, 16);
+	l |= FLD_VAL(video_fmt->x_res, 15, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_SIZE, l);
+}
+
+void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
+		struct omap_video_timings *timings)
+{
+	u32 r;
+	bool vsync_pol, hsync_pol;
+	DSSDBG("Enter hdmi_wp_video_config_interface\n");
+
+	vsync_pol = timings->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+	hsync_pol = timings->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+
+	r = hdmi_read_reg(wp->base, HDMI_WP_VIDEO_CFG);
+	r = FLD_MOD(r, vsync_pol, 7, 7);
+	r = FLD_MOD(r, hsync_pol, 6, 6);
+	r = FLD_MOD(r, timings->interlace, 3, 3);
+	r = FLD_MOD(r, 1, 1, 0); /* HDMI_TIMING_MASTER_24BIT */
+	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_CFG, r);
+}
+
+void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
+		struct omap_video_timings *timings)
+{
+	u32 timing_h = 0;
+	u32 timing_v = 0;
+
+	DSSDBG("Enter hdmi_wp_video_config_timing\n");
+
+	timing_h |= FLD_VAL(timings->hbp, 31, 20);
+	timing_h |= FLD_VAL(timings->hfp, 19, 8);
+	timing_h |= FLD_VAL(timings->hsw, 7, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_H, timing_h);
+
+	timing_v |= FLD_VAL(timings->vbp, 31, 20);
+	timing_v |= FLD_VAL(timings->vfp, 19, 8);
+	timing_v |= FLD_VAL(timings->vsw, 7, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_V, timing_v);
+}
+
+void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
+		struct omap_video_timings *timings, struct hdmi_config *param)
+{
+	DSSDBG("Enter hdmi_wp_video_init_format\n");
+
+	video_fmt->packing_mode = HDMI_PACK_10b_RGB_YUV444;
+	video_fmt->y_res = param->timings.y_res;
+	video_fmt->x_res = param->timings.x_res;
+	if (param->timings.interlace)
+		video_fmt->y_res /= 2;
+
+	timings->hbp = param->timings.hbp;
+	timings->hfp = param->timings.hfp;
+	timings->hsw = param->timings.hsw;
+	timings->vbp = param->timings.vbp;
+	timings->vfp = param->timings.vfp;
+	timings->vsw = param->timings.vsw;
+	timings->vsync_level = param->timings.vsync_level;
+	timings->hsync_level = param->timings.hsync_level;
+	timings->interlace = param->timings.interlace;
+}
+
+void hdmi_wp_audio_config_format(struct hdmi_wp_data *wp,
+		struct hdmi_audio_format *aud_fmt)
+{
+	u32 r;
+
+	DSSDBG("Enter hdmi_wp_audio_config_format\n");
+
+	r = hdmi_read_reg(wp->base, HDMI_WP_AUDIO_CFG);
+	if (omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES1 ||
+	    omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES2 ||
+	    omapdss_get_version() == OMAPDSS_VER_OMAP4) {
+		r = FLD_MOD(r, aud_fmt->stereo_channels, 26, 24);
+		r = FLD_MOD(r, aud_fmt->active_chnnls_msk, 23, 16);
+	}
+	r = FLD_MOD(r, aud_fmt->en_sig_blk_strt_end, 5, 5);
+	r = FLD_MOD(r, aud_fmt->type, 4, 4);
+	r = FLD_MOD(r, aud_fmt->justification, 3, 3);
+	r = FLD_MOD(r, aud_fmt->sample_order, 2, 2);
+	r = FLD_MOD(r, aud_fmt->samples_per_word, 1, 1);
+	r = FLD_MOD(r, aud_fmt->sample_size, 0, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_AUDIO_CFG, r);
+}
+
+void hdmi_wp_audio_config_dma(struct hdmi_wp_data *wp,
+		struct hdmi_audio_dma *aud_dma)
+{
+	u32 r;
+
+	DSSDBG("Enter hdmi_wp_audio_config_dma\n");
+
+	r = hdmi_read_reg(wp->base, HDMI_WP_AUDIO_CFG2);
+	r = FLD_MOD(r, aud_dma->transfer_size, 15, 8);
+	r = FLD_MOD(r, aud_dma->block_size, 7, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_AUDIO_CFG2, r);
+
+	r = hdmi_read_reg(wp->base, HDMI_WP_AUDIO_CTRL);
+	r = FLD_MOD(r, aud_dma->mode, 9, 9);
+	r = FLD_MOD(r, aud_dma->fifo_threshold, 8, 0);
+	hdmi_write_reg(wp->base, HDMI_WP_AUDIO_CTRL, r);
+}
+
+int hdmi_wp_audio_enable(struct hdmi_wp_data *wp, bool enable)
+{
+	REG_FLD_MOD(wp->base, HDMI_WP_AUDIO_CTRL, enable, 31, 31);
+
+	return 0;
+}
+
+int hdmi_wp_audio_core_req_enable(struct hdmi_wp_data *wp, bool enable)
+{
+	REG_FLD_MOD(wp->base, HDMI_WP_AUDIO_CTRL, enable, 30, 30);
+
+	return 0;
+}
+
+int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp)
+{
+	struct resource *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wp");
+	if (!res) {
+		DSSERR("can't get WP mem resource\n");
+		return -EINVAL;
+	}
+	wp->phys_base = res->start;
+
+	wp->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(wp->base)) {
+		DSSERR("can't ioremap HDMI WP\n");
+		return PTR_ERR(wp->base);
+	}
+
+	return 0;
+}
+
+phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp)
+{
+	return wp->phys_base + HDMI_WP_AUDIO_DATA;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c
new file mode 100644
index 000000000000..a7414fb12830
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c
@@ -0,0 +1,531 @@
+/*
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "MANAGER"
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/jiffies.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+static ssize_t manager_name_show(struct omap_overlay_manager *mgr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s\n", mgr->name);
+}
+
+static ssize_t manager_display_show(struct omap_overlay_manager *mgr, char *buf)
+{
+	struct omap_dss_device *dssdev = mgr->get_device(mgr);
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", dssdev ?
+			dssdev->name : "<none>");
+}
+
+static int manager_display_match(struct omap_dss_device *dssdev, void *data)
+{
+	const char *str = data;
+
+	return sysfs_streq(dssdev->name, str);
+}
+
+static ssize_t manager_display_store(struct omap_overlay_manager *mgr,
+		const char *buf, size_t size)
+{
+	int r = 0;
+	size_t len = size;
+	struct omap_dss_device *dssdev = NULL;
+	struct omap_dss_device *old_dssdev;
+
+	if (buf[size-1] == '\n')
+		--len;
+
+	if (len > 0)
+		dssdev = omap_dss_find_device((void *)buf,
+			manager_display_match);
+
+	if (len > 0 && dssdev == NULL)
+		return -EINVAL;
+
+	if (dssdev) {
+		DSSDBG("display %s found\n", dssdev->name);
+
+		if (omapdss_device_is_connected(dssdev)) {
+			DSSERR("new display is already connected\n");
+			r = -EINVAL;
+			goto put_device;
+		}
+
+		if (omapdss_device_is_enabled(dssdev)) {
+			DSSERR("new display is not disabled\n");
+			r = -EINVAL;
+			goto put_device;
+		}
+	}
+
+	old_dssdev = mgr->get_device(mgr);
+	if (old_dssdev) {
+		if (omapdss_device_is_enabled(old_dssdev)) {
+			DSSERR("old display is not disabled\n");
+			r = -EINVAL;
+			goto put_device;
+		}
+
+		old_dssdev->driver->disconnect(old_dssdev);
+	}
+
+	if (dssdev) {
+		r = dssdev->driver->connect(dssdev);
+		if (r) {
+			DSSERR("failed to connect new device\n");
+			goto put_device;
+		}
+
+		old_dssdev = mgr->get_device(mgr);
+		if (old_dssdev != dssdev) {
+			DSSERR("failed to connect device to this manager\n");
+			dssdev->driver->disconnect(dssdev);
+			goto put_device;
+		}
+
+		r = mgr->apply(mgr);
+		if (r) {
+			DSSERR("failed to apply dispc config\n");
+			goto put_device;
+		}
+	}
+
+put_device:
+	if (dssdev)
+		omap_dss_put_device(dssdev);
+
+	return r ? r : size;
+}
+
+static ssize_t manager_default_color_show(struct omap_overlay_manager *mgr,
+					  char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%#x\n", info.default_color);
+}
+
+static ssize_t manager_default_color_store(struct omap_overlay_manager *mgr,
+					   const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	u32 color;
+	int r;
+
+	r = kstrtouint(buf, 0, &color);
+	if (r)
+		return r;
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.default_color = color;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static const char *trans_key_type_str[] = {
+	"gfx-destination",
+	"video-source",
+};
+
+static ssize_t manager_trans_key_type_show(struct omap_overlay_manager *mgr,
+					   char *buf)
+{
+	enum omap_dss_trans_key_type key_type;
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	key_type = info.trans_key_type;
+	BUG_ON(key_type >= ARRAY_SIZE(trans_key_type_str));
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", trans_key_type_str[key_type]);
+}
+
+static ssize_t manager_trans_key_type_store(struct omap_overlay_manager *mgr,
+					    const char *buf, size_t size)
+{
+	enum omap_dss_trans_key_type key_type;
+	struct omap_overlay_manager_info info;
+	int r;
+
+	for (key_type = OMAP_DSS_COLOR_KEY_GFX_DST;
+			key_type < ARRAY_SIZE(trans_key_type_str); key_type++) {
+		if (sysfs_streq(buf, trans_key_type_str[key_type]))
+			break;
+	}
+
+	if (key_type == ARRAY_SIZE(trans_key_type_str))
+		return -EINVAL;
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.trans_key_type = key_type;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t manager_trans_key_value_show(struct omap_overlay_manager *mgr,
+					    char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%#x\n", info.trans_key);
+}
+
+static ssize_t manager_trans_key_value_store(struct omap_overlay_manager *mgr,
+					     const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	u32 key_value;
+	int r;
+
+	r = kstrtouint(buf, 0, &key_value);
+	if (r)
+		return r;
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.trans_key = key_value;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t manager_trans_key_enabled_show(struct omap_overlay_manager *mgr,
+					      char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", info.trans_enabled);
+}
+
+static ssize_t manager_trans_key_enabled_store(struct omap_overlay_manager *mgr,
+					       const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	bool enable;
+	int r;
+
+	r = strtobool(buf, &enable);
+	if (r)
+		return r;
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.trans_enabled = enable;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t manager_alpha_blending_enabled_show(
+		struct omap_overlay_manager *mgr, char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	if(!dss_has_feature(FEAT_ALPHA_FIXED_ZORDER))
+		return -ENODEV;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+		info.partial_alpha_enabled);
+}
+
+static ssize_t manager_alpha_blending_enabled_store(
+		struct omap_overlay_manager *mgr,
+		const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	bool enable;
+	int r;
+
+	if(!dss_has_feature(FEAT_ALPHA_FIXED_ZORDER))
+		return -ENODEV;
+
+	r = strtobool(buf, &enable);
+	if (r)
+		return r;
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.partial_alpha_enabled = enable;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t manager_cpr_enable_show(struct omap_overlay_manager *mgr,
+		char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", info.cpr_enable);
+}
+
+static ssize_t manager_cpr_enable_store(struct omap_overlay_manager *mgr,
+		const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	int r;
+	bool enable;
+
+	if (!dss_has_feature(FEAT_CPR))
+		return -ENODEV;
+
+	r = strtobool(buf, &enable);
+	if (r)
+		return r;
+
+	mgr->get_manager_info(mgr, &info);
+
+	if (info.cpr_enable == enable)
+		return size;
+
+	info.cpr_enable = enable;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t manager_cpr_coef_show(struct omap_overlay_manager *mgr,
+		char *buf)
+{
+	struct omap_overlay_manager_info info;
+
+	mgr->get_manager_info(mgr, &info);
+
+	return snprintf(buf, PAGE_SIZE,
+			"%d %d %d %d %d %d %d %d %d\n",
+			info.cpr_coefs.rr,
+			info.cpr_coefs.rg,
+			info.cpr_coefs.rb,
+			info.cpr_coefs.gr,
+			info.cpr_coefs.gg,
+			info.cpr_coefs.gb,
+			info.cpr_coefs.br,
+			info.cpr_coefs.bg,
+			info.cpr_coefs.bb);
+}
+
+static ssize_t manager_cpr_coef_store(struct omap_overlay_manager *mgr,
+		const char *buf, size_t size)
+{
+	struct omap_overlay_manager_info info;
+	struct omap_dss_cpr_coefs coefs;
+	int r, i;
+	s16 *arr;
+
+	if (!dss_has_feature(FEAT_CPR))
+		return -ENODEV;
+
+	if (sscanf(buf, "%hd %hd %hd %hd %hd %hd %hd %hd %hd",
+				&coefs.rr, &coefs.rg, &coefs.rb,
+				&coefs.gr, &coefs.gg, &coefs.gb,
+				&coefs.br, &coefs.bg, &coefs.bb) != 9)
+		return -EINVAL;
+
+	arr = (s16[]){ coefs.rr, coefs.rg, coefs.rb,
+		coefs.gr, coefs.gg, coefs.gb,
+		coefs.br, coefs.bg, coefs.bb };
+
+	for (i = 0; i < 9; ++i) {
+		if (arr[i] < -512 || arr[i] > 511)
+			return -EINVAL;
+	}
+
+	mgr->get_manager_info(mgr, &info);
+
+	info.cpr_coefs = coefs;
+
+	r = mgr->set_manager_info(mgr, &info);
+	if (r)
+		return r;
+
+	r = mgr->apply(mgr);
+	if (r)
+		return r;
+
+	return size;
+}
+
+struct manager_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct omap_overlay_manager *, char *);
+	ssize_t	(*store)(struct omap_overlay_manager *, const char *, size_t);
+};
+
+#define MANAGER_ATTR(_name, _mode, _show, _store) \
+	struct manager_attribute manager_attr_##_name = \
+	__ATTR(_name, _mode, _show, _store)
+
+static MANAGER_ATTR(name, S_IRUGO, manager_name_show, NULL);
+static MANAGER_ATTR(display, S_IRUGO|S_IWUSR,
+		manager_display_show, manager_display_store);
+static MANAGER_ATTR(default_color, S_IRUGO|S_IWUSR,
+		manager_default_color_show, manager_default_color_store);
+static MANAGER_ATTR(trans_key_type, S_IRUGO|S_IWUSR,
+		manager_trans_key_type_show, manager_trans_key_type_store);
+static MANAGER_ATTR(trans_key_value, S_IRUGO|S_IWUSR,
+		manager_trans_key_value_show, manager_trans_key_value_store);
+static MANAGER_ATTR(trans_key_enabled, S_IRUGO|S_IWUSR,
+		manager_trans_key_enabled_show,
+		manager_trans_key_enabled_store);
+static MANAGER_ATTR(alpha_blending_enabled, S_IRUGO|S_IWUSR,
+		manager_alpha_blending_enabled_show,
+		manager_alpha_blending_enabled_store);
+static MANAGER_ATTR(cpr_enable, S_IRUGO|S_IWUSR,
+		manager_cpr_enable_show,
+		manager_cpr_enable_store);
+static MANAGER_ATTR(cpr_coef, S_IRUGO|S_IWUSR,
+		manager_cpr_coef_show,
+		manager_cpr_coef_store);
+
+
+static struct attribute *manager_sysfs_attrs[] = {
+	&manager_attr_name.attr,
+	&manager_attr_display.attr,
+	&manager_attr_default_color.attr,
+	&manager_attr_trans_key_type.attr,
+	&manager_attr_trans_key_value.attr,
+	&manager_attr_trans_key_enabled.attr,
+	&manager_attr_alpha_blending_enabled.attr,
+	&manager_attr_cpr_enable.attr,
+	&manager_attr_cpr_coef.attr,
+	NULL
+};
+
+static ssize_t manager_attr_show(struct kobject *kobj, struct attribute *attr,
+		char *buf)
+{
+	struct omap_overlay_manager *manager;
+	struct manager_attribute *manager_attr;
+
+	manager = container_of(kobj, struct omap_overlay_manager, kobj);
+	manager_attr = container_of(attr, struct manager_attribute, attr);
+
+	if (!manager_attr->show)
+		return -ENOENT;
+
+	return manager_attr->show(manager, buf);
+}
+
+static ssize_t manager_attr_store(struct kobject *kobj, struct attribute *attr,
+		const char *buf, size_t size)
+{
+	struct omap_overlay_manager *manager;
+	struct manager_attribute *manager_attr;
+
+	manager = container_of(kobj, struct omap_overlay_manager, kobj);
+	manager_attr = container_of(attr, struct manager_attribute, attr);
+
+	if (!manager_attr->store)
+		return -ENOENT;
+
+	return manager_attr->store(manager, buf, size);
+}
+
+static const struct sysfs_ops manager_sysfs_ops = {
+	.show = manager_attr_show,
+	.store = manager_attr_store,
+};
+
+static struct kobj_type manager_ktype = {
+	.sysfs_ops = &manager_sysfs_ops,
+	.default_attrs = manager_sysfs_attrs,
+};
+
+int dss_manager_kobj_init(struct omap_overlay_manager *mgr,
+		struct platform_device *pdev)
+{
+	return kobject_init_and_add(&mgr->kobj, &manager_ktype,
+			&pdev->dev.kobj, "manager%d", mgr->id);
+}
+
+void dss_manager_kobj_uninit(struct omap_overlay_manager *mgr)
+{
+	kobject_del(&mgr->kobj);
+	kobject_put(&mgr->kobj);
+
+	memset(&mgr->kobj, 0, sizeof(mgr->kobj));
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/manager.c b/drivers/video/fbdev/omap2/omapfb/dss/manager.c
new file mode 100644
index 000000000000..08a67f4f6a20
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/manager.c
@@ -0,0 +1,263 @@
+/*
+ * linux/drivers/video/omap2/dss/manager.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "MANAGER"
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/jiffies.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+static int num_managers;
+static struct omap_overlay_manager *managers;
+
+int dss_init_overlay_managers(void)
+{
+	int i;
+
+	num_managers = dss_feat_get_num_mgrs();
+
+	managers = kzalloc(sizeof(struct omap_overlay_manager) * num_managers,
+			GFP_KERNEL);
+
+	BUG_ON(managers == NULL);
+
+	for (i = 0; i < num_managers; ++i) {
+		struct omap_overlay_manager *mgr = &managers[i];
+
+		switch (i) {
+		case 0:
+			mgr->name = "lcd";
+			mgr->id = OMAP_DSS_CHANNEL_LCD;
+			break;
+		case 1:
+			mgr->name = "tv";
+			mgr->id = OMAP_DSS_CHANNEL_DIGIT;
+			break;
+		case 2:
+			mgr->name = "lcd2";
+			mgr->id = OMAP_DSS_CHANNEL_LCD2;
+			break;
+		case 3:
+			mgr->name = "lcd3";
+			mgr->id = OMAP_DSS_CHANNEL_LCD3;
+			break;
+		}
+
+		mgr->caps = 0;
+		mgr->supported_displays =
+			dss_feat_get_supported_displays(mgr->id);
+		mgr->supported_outputs =
+			dss_feat_get_supported_outputs(mgr->id);
+
+		INIT_LIST_HEAD(&mgr->overlays);
+	}
+
+	return 0;
+}
+
+int dss_init_overlay_managers_sysfs(struct platform_device *pdev)
+{
+	int i, r;
+
+	for (i = 0; i < num_managers; ++i) {
+		struct omap_overlay_manager *mgr = &managers[i];
+
+		r = dss_manager_kobj_init(mgr, pdev);
+		if (r)
+			DSSERR("failed to create sysfs file\n");
+	}
+
+	return 0;
+}
+
+void dss_uninit_overlay_managers(void)
+{
+	kfree(managers);
+	managers = NULL;
+	num_managers = 0;
+}
+
+void dss_uninit_overlay_managers_sysfs(struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < num_managers; ++i) {
+		struct omap_overlay_manager *mgr = &managers[i];
+
+		dss_manager_kobj_uninit(mgr);
+	}
+}
+
+int omap_dss_get_num_overlay_managers(void)
+{
+	return num_managers;
+}
+EXPORT_SYMBOL(omap_dss_get_num_overlay_managers);
+
+struct omap_overlay_manager *omap_dss_get_overlay_manager(int num)
+{
+	if (num >= num_managers)
+		return NULL;
+
+	return &managers[num];
+}
+EXPORT_SYMBOL(omap_dss_get_overlay_manager);
+
+int dss_mgr_simple_check(struct omap_overlay_manager *mgr,
+		const struct omap_overlay_manager_info *info)
+{
+	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER)) {
+		/*
+		 * OMAP3 supports only graphics source transparency color key
+		 * and alpha blending simultaneously. See TRM 15.4.2.4.2.2
+		 * Alpha Mode.
+		 */
+		if (info->partial_alpha_enabled && info->trans_enabled
+			&& info->trans_key_type != OMAP_DSS_COLOR_KEY_GFX_DST) {
+			DSSERR("check_manager: illegal transparency key\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int dss_mgr_check_zorder(struct omap_overlay_manager *mgr,
+		struct omap_overlay_info **overlay_infos)
+{
+	struct omap_overlay *ovl1, *ovl2;
+	struct omap_overlay_info *info1, *info2;
+
+	list_for_each_entry(ovl1, &mgr->overlays, list) {
+		info1 = overlay_infos[ovl1->id];
+
+		if (info1 == NULL)
+			continue;
+
+		list_for_each_entry(ovl2, &mgr->overlays, list) {
+			if (ovl1 == ovl2)
+				continue;
+
+			info2 = overlay_infos[ovl2->id];
+
+			if (info2 == NULL)
+				continue;
+
+			if (info1->zorder == info2->zorder) {
+				DSSERR("overlays %d and %d have the same "
+						"zorder %d\n",
+					ovl1->id, ovl2->id, info1->zorder);
+				return -EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+int dss_mgr_check_timings(struct omap_overlay_manager *mgr,
+		const struct omap_video_timings *timings)
+{
+	if (!dispc_mgr_timings_ok(mgr->id, timings)) {
+		DSSERR("check_manager: invalid timings\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dss_mgr_check_lcd_config(struct omap_overlay_manager *mgr,
+		const struct dss_lcd_mgr_config *config)
+{
+	struct dispc_clock_info cinfo = config->clock_info;
+	int dl = config->video_port_width;
+	bool stallmode = config->stallmode;
+	bool fifohandcheck = config->fifohandcheck;
+
+	if (cinfo.lck_div < 1 || cinfo.lck_div > 255)
+		return -EINVAL;
+
+	if (cinfo.pck_div < 1 || cinfo.pck_div > 255)
+		return -EINVAL;
+
+	if (dl != 12 && dl != 16 && dl != 18 && dl != 24)
+		return -EINVAL;
+
+	/* fifohandcheck should be used only with stallmode */
+	if (!stallmode && fifohandcheck)
+		return -EINVAL;
+
+	/*
+	 * io pad mode can be only checked by using dssdev connected to the
+	 * manager. Ignore checking these for now, add checks when manager
+	 * is capable of holding information related to the connected interface
+	 */
+
+	return 0;
+}
+
+int dss_mgr_check(struct omap_overlay_manager *mgr,
+		struct omap_overlay_manager_info *info,
+		const struct omap_video_timings *mgr_timings,
+		const struct dss_lcd_mgr_config *lcd_config,
+		struct omap_overlay_info **overlay_infos)
+{
+	struct omap_overlay *ovl;
+	int r;
+
+	if (dss_has_feature(FEAT_ALPHA_FREE_ZORDER)) {
+		r = dss_mgr_check_zorder(mgr, overlay_infos);
+		if (r)
+			return r;
+	}
+
+	r = dss_mgr_check_timings(mgr, mgr_timings);
+	if (r)
+		return r;
+
+	r = dss_mgr_check_lcd_config(mgr, lcd_config);
+	if (r)
+		return r;
+
+	list_for_each_entry(ovl, &mgr->overlays, list) {
+		struct omap_overlay_info *oi;
+		int r;
+
+		oi = overlay_infos[ovl->id];
+
+		if (oi == NULL)
+			continue;
+
+		r = dss_ovl_check(ovl, oi, mgr_timings);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/omapdss-boot-init.c b/drivers/video/fbdev/omap2/omapfb/dss/omapdss-boot-init.c
new file mode 100644
index 000000000000..136d30484d02
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/omapdss-boot-init.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2014 Texas Instruments
+ * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * As omapdss panel drivers are omapdss specific, but we want to define the
+ * DT-data in generic manner, we convert the compatible strings of the panel and
+ * encoder nodes from "panel-foo" to "omapdss,panel-foo". This way we can have
+ * both correct DT data and omapdss specific drivers.
+ *
+ * When we get generic panel drivers to the kernel, this file will be removed.
+ */
+
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+
+static struct list_head dss_conv_list __initdata;
+
+static const char prefix[] __initconst = "omapdss,";
+
+struct dss_conv_node {
+	struct list_head list;
+	struct device_node *node;
+	bool root;
+};
+
+static int __init omapdss_count_strings(const struct property *prop)
+{
+	const char *p = prop->value;
+	int l = 0, total = 0;
+	int i;
+
+	for (i = 0; total < prop->length; total += l, p += l, i++)
+		l = strlen(p) + 1;
+
+	return i;
+}
+
+static void __init omapdss_update_prop(struct device_node *node, char *compat,
+	int len)
+{
+	struct property *prop;
+
+	prop = kzalloc(sizeof(*prop), GFP_KERNEL);
+	if (!prop)
+		return;
+
+	prop->name = "compatible";
+	prop->value = compat;
+	prop->length = len;
+
+	of_update_property(node, prop);
+}
+
+static void __init omapdss_prefix_strcpy(char *dst, int dst_len,
+	const char *src, int src_len)
+{
+	size_t total = 0;
+
+	while (total < src_len) {
+		size_t l = strlen(src) + 1;
+
+		strcpy(dst, prefix);
+		dst += strlen(prefix);
+
+		strcpy(dst, src);
+		dst += l;
+
+		src += l;
+		total += l;
+	}
+}
+
+/* prepend compatible property strings with "omapdss," */
+static void __init omapdss_omapify_node(struct device_node *node)
+{
+	struct property *prop;
+	char *new_compat;
+	int num_strs;
+	int new_len;
+
+	prop = of_find_property(node, "compatible", NULL);
+
+	if (!prop || !prop->value)
+		return;
+
+	if (strnlen(prop->value, prop->length) >= prop->length)
+		return;
+
+	/* is it already prefixed? */
+	if (strncmp(prefix, prop->value, strlen(prefix)) == 0)
+		return;
+
+	num_strs = omapdss_count_strings(prop);
+
+	new_len = prop->length + strlen(prefix) * num_strs;
+	new_compat = kmalloc(new_len, GFP_KERNEL);
+
+	omapdss_prefix_strcpy(new_compat, new_len, prop->value, prop->length);
+
+	omapdss_update_prop(node, new_compat, new_len);
+}
+
+static void __init omapdss_add_to_list(struct device_node *node, bool root)
+{
+	struct dss_conv_node *n = kmalloc(sizeof(struct dss_conv_node),
+		GFP_KERNEL);
+	if (n) {
+		n->node = node;
+		n->root = root;
+		list_add(&n->list, &dss_conv_list);
+	}
+}
+
+static bool __init omapdss_list_contains(const struct device_node *node)
+{
+	struct dss_conv_node *n;
+
+	list_for_each_entry(n, &dss_conv_list, list) {
+		if (n->node == node)
+			return true;
+	}
+
+	return false;
+}
+
+static void __init omapdss_walk_device(struct device_node *node, bool root)
+{
+	struct device_node *n;
+
+	omapdss_add_to_list(node, root);
+
+	/*
+	 * of_graph_get_remote_port_parent() prints an error if there is no
+	 * port/ports node. To avoid that, check first that there's the node.
+	 */
+	n = of_get_child_by_name(node, "ports");
+	if (!n)
+		n = of_get_child_by_name(node, "port");
+	if (!n)
+		return;
+
+	of_node_put(n);
+
+	n = NULL;
+	while ((n = of_graph_get_next_endpoint(node, n)) != NULL) {
+		struct device_node *pn;
+
+		pn = of_graph_get_remote_port_parent(n);
+
+		if (!pn)
+			continue;
+
+		if (!of_device_is_available(pn) || omapdss_list_contains(pn)) {
+			of_node_put(pn);
+			continue;
+		}
+
+		omapdss_walk_device(pn, false);
+	}
+}
+
+static const struct of_device_id omapdss_of_match[] __initconst = {
+	{ .compatible = "ti,omap2-dss", },
+	{ .compatible = "ti,omap3-dss", },
+	{ .compatible = "ti,omap4-dss", },
+	{ .compatible = "ti,omap5-dss", },
+	{ .compatible = "ti,dra7-dss", },
+	{},
+};
+
+static int __init omapdss_boot_init(void)
+{
+	struct device_node *dss, *child;
+
+	INIT_LIST_HEAD(&dss_conv_list);
+
+	dss = of_find_matching_node(NULL, omapdss_of_match);
+
+	if (dss == NULL || !of_device_is_available(dss))
+		return 0;
+
+	omapdss_walk_device(dss, true);
+
+	for_each_available_child_of_node(dss, child) {
+		if (!of_find_property(child, "compatible", NULL))
+			continue;
+
+		omapdss_walk_device(child, true);
+	}
+
+	while (!list_empty(&dss_conv_list)) {
+		struct dss_conv_node *n;
+
+		n = list_first_entry(&dss_conv_list, struct dss_conv_node,
+			list);
+
+		if (!n->root)
+			omapdss_omapify_node(n->node);
+
+		list_del(&n->list);
+		of_node_put(n->node);
+		kfree(n);
+	}
+
+	return 0;
+}
+
+subsys_initcall(omapdss_boot_init);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/output.c b/drivers/video/fbdev/omap2/omapfb/dss/output.c
new file mode 100644
index 000000000000..16072159bd24
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/output.c
@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2012 Texas Instruments Ltd
+ * Author: Archit Taneja <archit@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+
+static LIST_HEAD(output_list);
+static DEFINE_MUTEX(output_lock);
+
+int omapdss_output_set_device(struct omap_dss_device *out,
+		struct omap_dss_device *dssdev)
+{
+	int r;
+
+	mutex_lock(&output_lock);
+
+	if (out->dst) {
+		DSSERR("output already has device %s connected to it\n",
+			out->dst->name);
+		r = -EINVAL;
+		goto err;
+	}
+
+	if (out->output_type != dssdev->type) {
+		DSSERR("output type and display type don't match\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	out->dst = dssdev;
+	dssdev->src = out;
+
+	mutex_unlock(&output_lock);
+
+	return 0;
+err:
+	mutex_unlock(&output_lock);
+
+	return r;
+}
+EXPORT_SYMBOL(omapdss_output_set_device);
+
+int omapdss_output_unset_device(struct omap_dss_device *out)
+{
+	int r;
+
+	mutex_lock(&output_lock);
+
+	if (!out->dst) {
+		DSSERR("output doesn't have a device connected to it\n");
+		r = -EINVAL;
+		goto err;
+	}
+
+	if (out->dst->state != OMAP_DSS_DISPLAY_DISABLED) {
+		DSSERR("device %s is not disabled, cannot unset device\n",
+				out->dst->name);
+		r = -EINVAL;
+		goto err;
+	}
+
+	out->dst->src = NULL;
+	out->dst = NULL;
+
+	mutex_unlock(&output_lock);
+
+	return 0;
+err:
+	mutex_unlock(&output_lock);
+
+	return r;
+}
+EXPORT_SYMBOL(omapdss_output_unset_device);
+
+int omapdss_register_output(struct omap_dss_device *out)
+{
+	list_add_tail(&out->list, &output_list);
+	return 0;
+}
+EXPORT_SYMBOL(omapdss_register_output);
+
+void omapdss_unregister_output(struct omap_dss_device *out)
+{
+	list_del(&out->list);
+}
+EXPORT_SYMBOL(omapdss_unregister_output);
+
+struct omap_dss_device *omap_dss_get_output(enum omap_dss_output_id id)
+{
+	struct omap_dss_device *out;
+
+	list_for_each_entry(out, &output_list, list) {
+		if (out->id == id)
+			return out;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(omap_dss_get_output);
+
+struct omap_dss_device *omap_dss_find_output(const char *name)
+{
+	struct omap_dss_device *out;
+
+	list_for_each_entry(out, &output_list, list) {
+		if (strcmp(out->name, name) == 0)
+			return omap_dss_get_device(out);
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(omap_dss_find_output);
+
+struct omap_dss_device *omap_dss_find_output_by_port_node(struct device_node *port)
+{
+	struct device_node *src_node;
+	struct omap_dss_device *out;
+	u32 reg;
+
+	src_node = dss_of_port_get_parent_device(port);
+	if (!src_node)
+		return NULL;
+
+	reg = dss_of_port_get_port_number(port);
+
+	list_for_each_entry(out, &output_list, list) {
+		if (out->dev->of_node == src_node && out->port_num == reg) {
+			of_node_put(src_node);
+			return omap_dss_get_device(out);
+		}
+	}
+
+	of_node_put(src_node);
+
+	return NULL;
+}
+EXPORT_SYMBOL(omap_dss_find_output_by_port_node);
+
+struct omap_dss_device *omapdss_find_output_from_display(struct omap_dss_device *dssdev)
+{
+	while (dssdev->src)
+		dssdev = dssdev->src;
+
+	if (dssdev->id != 0)
+		return omap_dss_get_device(dssdev);
+
+	return NULL;
+}
+EXPORT_SYMBOL(omapdss_find_output_from_display);
+
+struct omap_overlay_manager *omapdss_find_mgr_from_display(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out;
+	struct omap_overlay_manager *mgr;
+
+	out = omapdss_find_output_from_display(dssdev);
+
+	if (out == NULL)
+		return NULL;
+
+	mgr = out->manager;
+
+	omap_dss_put_device(out);
+
+	return mgr;
+}
+EXPORT_SYMBOL(omapdss_find_mgr_from_display);
+
+static const struct dss_mgr_ops *dss_mgr_ops;
+
+int dss_install_mgr_ops(const struct dss_mgr_ops *mgr_ops)
+{
+	if (dss_mgr_ops)
+		return -EBUSY;
+
+	dss_mgr_ops = mgr_ops;
+
+	return 0;
+}
+EXPORT_SYMBOL(dss_install_mgr_ops);
+
+void dss_uninstall_mgr_ops(void)
+{
+	dss_mgr_ops = NULL;
+}
+EXPORT_SYMBOL(dss_uninstall_mgr_ops);
+
+int dss_mgr_connect(struct omap_overlay_manager *mgr,
+		struct omap_dss_device *dst)
+{
+	return dss_mgr_ops->connect(mgr, dst);
+}
+EXPORT_SYMBOL(dss_mgr_connect);
+
+void dss_mgr_disconnect(struct omap_overlay_manager *mgr,
+		struct omap_dss_device *dst)
+{
+	dss_mgr_ops->disconnect(mgr, dst);
+}
+EXPORT_SYMBOL(dss_mgr_disconnect);
+
+void dss_mgr_set_timings(struct omap_overlay_manager *mgr,
+		const struct omap_video_timings *timings)
+{
+	dss_mgr_ops->set_timings(mgr, timings);
+}
+EXPORT_SYMBOL(dss_mgr_set_timings);
+
+void dss_mgr_set_lcd_config(struct omap_overlay_manager *mgr,
+		const struct dss_lcd_mgr_config *config)
+{
+	dss_mgr_ops->set_lcd_config(mgr, config);
+}
+EXPORT_SYMBOL(dss_mgr_set_lcd_config);
+
+int dss_mgr_enable(struct omap_overlay_manager *mgr)
+{
+	return dss_mgr_ops->enable(mgr);
+}
+EXPORT_SYMBOL(dss_mgr_enable);
+
+void dss_mgr_disable(struct omap_overlay_manager *mgr)
+{
+	dss_mgr_ops->disable(mgr);
+}
+EXPORT_SYMBOL(dss_mgr_disable);
+
+void dss_mgr_start_update(struct omap_overlay_manager *mgr)
+{
+	dss_mgr_ops->start_update(mgr);
+}
+EXPORT_SYMBOL(dss_mgr_start_update);
+
+int dss_mgr_register_framedone_handler(struct omap_overlay_manager *mgr,
+		void (*handler)(void *), void *data)
+{
+	return dss_mgr_ops->register_framedone_handler(mgr, handler, data);
+}
+EXPORT_SYMBOL(dss_mgr_register_framedone_handler);
+
+void dss_mgr_unregister_framedone_handler(struct omap_overlay_manager *mgr,
+		void (*handler)(void *), void *data)
+{
+	dss_mgr_ops->unregister_framedone_handler(mgr, handler, data);
+}
+EXPORT_SYMBOL(dss_mgr_unregister_framedone_handler);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c
new file mode 100644
index 000000000000..4cc5ddebfb34
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c
@@ -0,0 +1,456 @@
+/*
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "OVERLAY"
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/sysfs.h>
+#include <linux/kobject.h>
+#include <linux/platform_device.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+static ssize_t overlay_name_show(struct omap_overlay *ovl, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s\n", ovl->name);
+}
+
+static ssize_t overlay_manager_show(struct omap_overlay *ovl, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s\n",
+			ovl->manager ? ovl->manager->name : "<none>");
+}
+
+static ssize_t overlay_manager_store(struct omap_overlay *ovl, const char *buf,
+		size_t size)
+{
+	int i, r;
+	struct omap_overlay_manager *mgr = NULL;
+	struct omap_overlay_manager *old_mgr;
+	int len = size;
+
+	if (buf[size-1] == '\n')
+		--len;
+
+	if (len > 0) {
+		for (i = 0; i < omap_dss_get_num_overlay_managers(); ++i) {
+			mgr = omap_dss_get_overlay_manager(i);
+
+			if (sysfs_streq(buf, mgr->name))
+				break;
+
+			mgr = NULL;
+		}
+	}
+
+	if (len > 0 && mgr == NULL)
+		return -EINVAL;
+
+	if (mgr)
+		DSSDBG("manager %s found\n", mgr->name);
+
+	if (mgr == ovl->manager)
+		return size;
+
+	old_mgr = ovl->manager;
+
+	r = dispc_runtime_get();
+	if (r)
+		return r;
+
+	/* detach old manager */
+	if (old_mgr) {
+		r = ovl->unset_manager(ovl);
+		if (r) {
+			DSSERR("detach failed\n");
+			goto err;
+		}
+
+		r = old_mgr->apply(old_mgr);
+		if (r)
+			goto err;
+	}
+
+	if (mgr) {
+		r = ovl->set_manager(ovl, mgr);
+		if (r) {
+			DSSERR("Failed to attach overlay\n");
+			goto err;
+		}
+
+		r = mgr->apply(mgr);
+		if (r)
+			goto err;
+	}
+
+	dispc_runtime_put();
+
+	return size;
+
+err:
+	dispc_runtime_put();
+	return r;
+}
+
+static ssize_t overlay_input_size_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d,%d\n",
+			info.width, info.height);
+}
+
+static ssize_t overlay_screen_width_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", info.screen_width);
+}
+
+static ssize_t overlay_position_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d,%d\n",
+			info.pos_x, info.pos_y);
+}
+
+static ssize_t overlay_position_store(struct omap_overlay *ovl,
+		const char *buf, size_t size)
+{
+	int r;
+	char *last;
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	info.pos_x = simple_strtoul(buf, &last, 10);
+	++last;
+	if (last - buf >= size)
+		return -EINVAL;
+
+	info.pos_y = simple_strtoul(last, &last, 10);
+
+	r = ovl->set_overlay_info(ovl, &info);
+	if (r)
+		return r;
+
+	if (ovl->manager) {
+		r = ovl->manager->apply(ovl->manager);
+		if (r)
+			return r;
+	}
+
+	return size;
+}
+
+static ssize_t overlay_output_size_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d,%d\n",
+			info.out_width, info.out_height);
+}
+
+static ssize_t overlay_output_size_store(struct omap_overlay *ovl,
+		const char *buf, size_t size)
+{
+	int r;
+	char *last;
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	info.out_width = simple_strtoul(buf, &last, 10);
+	++last;
+	if (last - buf >= size)
+		return -EINVAL;
+
+	info.out_height = simple_strtoul(last, &last, 10);
+
+	r = ovl->set_overlay_info(ovl, &info);
+	if (r)
+		return r;
+
+	if (ovl->manager) {
+		r = ovl->manager->apply(ovl->manager);
+		if (r)
+			return r;
+	}
+
+	return size;
+}
+
+static ssize_t overlay_enabled_show(struct omap_overlay *ovl, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", ovl->is_enabled(ovl));
+}
+
+static ssize_t overlay_enabled_store(struct omap_overlay *ovl, const char *buf,
+		size_t size)
+{
+	int r;
+	bool enable;
+
+	r = strtobool(buf, &enable);
+	if (r)
+		return r;
+
+	if (enable)
+		r = ovl->enable(ovl);
+	else
+		r = ovl->disable(ovl);
+
+	if (r)
+		return r;
+
+	return size;
+}
+
+static ssize_t overlay_global_alpha_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			info.global_alpha);
+}
+
+static ssize_t overlay_global_alpha_store(struct omap_overlay *ovl,
+		const char *buf, size_t size)
+{
+	int r;
+	u8 alpha;
+	struct omap_overlay_info info;
+
+	if ((ovl->caps & OMAP_DSS_OVL_CAP_GLOBAL_ALPHA) == 0)
+		return -ENODEV;
+
+	r = kstrtou8(buf, 0, &alpha);
+	if (r)
+		return r;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	info.global_alpha = alpha;
+
+	r = ovl->set_overlay_info(ovl, &info);
+	if (r)
+		return r;
+
+	if (ovl->manager) {
+		r = ovl->manager->apply(ovl->manager);
+		if (r)
+			return r;
+	}
+
+	return size;
+}
+
+static ssize_t overlay_pre_mult_alpha_show(struct omap_overlay *ovl,
+		char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			info.pre_mult_alpha);
+}
+
+static ssize_t overlay_pre_mult_alpha_store(struct omap_overlay *ovl,
+		const char *buf, size_t size)
+{
+	int r;
+	u8 alpha;
+	struct omap_overlay_info info;
+
+	if ((ovl->caps & OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA) == 0)
+		return -ENODEV;
+
+	r = kstrtou8(buf, 0, &alpha);
+	if (r)
+		return r;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	info.pre_mult_alpha = alpha;
+
+	r = ovl->set_overlay_info(ovl, &info);
+	if (r)
+		return r;
+
+	if (ovl->manager) {
+		r = ovl->manager->apply(ovl->manager);
+		if (r)
+			return r;
+	}
+
+	return size;
+}
+
+static ssize_t overlay_zorder_show(struct omap_overlay *ovl, char *buf)
+{
+	struct omap_overlay_info info;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", info.zorder);
+}
+
+static ssize_t overlay_zorder_store(struct omap_overlay *ovl,
+		const char *buf, size_t size)
+{
+	int r;
+	u8 zorder;
+	struct omap_overlay_info info;
+
+	if ((ovl->caps & OMAP_DSS_OVL_CAP_ZORDER) == 0)
+		return -ENODEV;
+
+	r = kstrtou8(buf, 0, &zorder);
+	if (r)
+		return r;
+
+	ovl->get_overlay_info(ovl, &info);
+
+	info.zorder = zorder;
+
+	r = ovl->set_overlay_info(ovl, &info);
+	if (r)
+		return r;
+
+	if (ovl->manager) {
+		r = ovl->manager->apply(ovl->manager);
+		if (r)
+			return r;
+	}
+
+	return size;
+}
+
+struct overlay_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct omap_overlay *, char *);
+	ssize_t	(*store)(struct omap_overlay *, const char *, size_t);
+};
+
+#define OVERLAY_ATTR(_name, _mode, _show, _store) \
+	struct overlay_attribute overlay_attr_##_name = \
+	__ATTR(_name, _mode, _show, _store)
+
+static OVERLAY_ATTR(name, S_IRUGO, overlay_name_show, NULL);
+static OVERLAY_ATTR(manager, S_IRUGO|S_IWUSR,
+		overlay_manager_show, overlay_manager_store);
+static OVERLAY_ATTR(input_size, S_IRUGO, overlay_input_size_show, NULL);
+static OVERLAY_ATTR(screen_width, S_IRUGO, overlay_screen_width_show, NULL);
+static OVERLAY_ATTR(position, S_IRUGO|S_IWUSR,
+		overlay_position_show, overlay_position_store);
+static OVERLAY_ATTR(output_size, S_IRUGO|S_IWUSR,
+		overlay_output_size_show, overlay_output_size_store);
+static OVERLAY_ATTR(enabled, S_IRUGO|S_IWUSR,
+		overlay_enabled_show, overlay_enabled_store);
+static OVERLAY_ATTR(global_alpha, S_IRUGO|S_IWUSR,
+		overlay_global_alpha_show, overlay_global_alpha_store);
+static OVERLAY_ATTR(pre_mult_alpha, S_IRUGO|S_IWUSR,
+		overlay_pre_mult_alpha_show,
+		overlay_pre_mult_alpha_store);
+static OVERLAY_ATTR(zorder, S_IRUGO|S_IWUSR,
+		overlay_zorder_show, overlay_zorder_store);
+
+static struct attribute *overlay_sysfs_attrs[] = {
+	&overlay_attr_name.attr,
+	&overlay_attr_manager.attr,
+	&overlay_attr_input_size.attr,
+	&overlay_attr_screen_width.attr,
+	&overlay_attr_position.attr,
+	&overlay_attr_output_size.attr,
+	&overlay_attr_enabled.attr,
+	&overlay_attr_global_alpha.attr,
+	&overlay_attr_pre_mult_alpha.attr,
+	&overlay_attr_zorder.attr,
+	NULL
+};
+
+static ssize_t overlay_attr_show(struct kobject *kobj, struct attribute *attr,
+		char *buf)
+{
+	struct omap_overlay *overlay;
+	struct overlay_attribute *overlay_attr;
+
+	overlay = container_of(kobj, struct omap_overlay, kobj);
+	overlay_attr = container_of(attr, struct overlay_attribute, attr);
+
+	if (!overlay_attr->show)
+		return -ENOENT;
+
+	return overlay_attr->show(overlay, buf);
+}
+
+static ssize_t overlay_attr_store(struct kobject *kobj, struct attribute *attr,
+		const char *buf, size_t size)
+{
+	struct omap_overlay *overlay;
+	struct overlay_attribute *overlay_attr;
+
+	overlay = container_of(kobj, struct omap_overlay, kobj);
+	overlay_attr = container_of(attr, struct overlay_attribute, attr);
+
+	if (!overlay_attr->store)
+		return -ENOENT;
+
+	return overlay_attr->store(overlay, buf, size);
+}
+
+static const struct sysfs_ops overlay_sysfs_ops = {
+	.show = overlay_attr_show,
+	.store = overlay_attr_store,
+};
+
+static struct kobj_type overlay_ktype = {
+	.sysfs_ops = &overlay_sysfs_ops,
+	.default_attrs = overlay_sysfs_attrs,
+};
+
+int dss_overlay_kobj_init(struct omap_overlay *ovl,
+		struct platform_device *pdev)
+{
+	return kobject_init_and_add(&ovl->kobj, &overlay_ktype,
+			&pdev->dev.kobj, "overlay%d", ovl->id);
+}
+
+void dss_overlay_kobj_uninit(struct omap_overlay *ovl)
+{
+	kobject_del(&ovl->kobj);
+	kobject_put(&ovl->kobj);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/overlay.c b/drivers/video/fbdev/omap2/omapfb/dss/overlay.c
new file mode 100644
index 000000000000..2f7cee985cdd
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/overlay.c
@@ -0,0 +1,202 @@
+/*
+ * linux/drivers/video/omap2/dss/overlay.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "OVERLAY"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/sysfs.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+static int num_overlays;
+static struct omap_overlay *overlays;
+
+int omap_dss_get_num_overlays(void)
+{
+	return num_overlays;
+}
+EXPORT_SYMBOL(omap_dss_get_num_overlays);
+
+struct omap_overlay *omap_dss_get_overlay(int num)
+{
+	if (num >= num_overlays)
+		return NULL;
+
+	return &overlays[num];
+}
+EXPORT_SYMBOL(omap_dss_get_overlay);
+
+void dss_init_overlays(struct platform_device *pdev)
+{
+	int i, r;
+
+	num_overlays = dss_feat_get_num_ovls();
+
+	overlays = kzalloc(sizeof(struct omap_overlay) * num_overlays,
+			GFP_KERNEL);
+
+	BUG_ON(overlays == NULL);
+
+	for (i = 0; i < num_overlays; ++i) {
+		struct omap_overlay *ovl = &overlays[i];
+
+		switch (i) {
+		case 0:
+			ovl->name = "gfx";
+			ovl->id = OMAP_DSS_GFX;
+			break;
+		case 1:
+			ovl->name = "vid1";
+			ovl->id = OMAP_DSS_VIDEO1;
+			break;
+		case 2:
+			ovl->name = "vid2";
+			ovl->id = OMAP_DSS_VIDEO2;
+			break;
+		case 3:
+			ovl->name = "vid3";
+			ovl->id = OMAP_DSS_VIDEO3;
+			break;
+		}
+
+		ovl->caps = dss_feat_get_overlay_caps(ovl->id);
+		ovl->supported_modes =
+			dss_feat_get_supported_color_modes(ovl->id);
+
+		r = dss_overlay_kobj_init(ovl, pdev);
+		if (r)
+			DSSERR("failed to create sysfs file\n");
+	}
+}
+
+void dss_uninit_overlays(struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < num_overlays; ++i) {
+		struct omap_overlay *ovl = &overlays[i];
+		dss_overlay_kobj_uninit(ovl);
+	}
+
+	kfree(overlays);
+	overlays = NULL;
+	num_overlays = 0;
+}
+
+int dss_ovl_simple_check(struct omap_overlay *ovl,
+		const struct omap_overlay_info *info)
+{
+	if ((ovl->caps & OMAP_DSS_OVL_CAP_SCALE) == 0) {
+		if (info->out_width != 0 && info->width != info->out_width) {
+			DSSERR("check_overlay: overlay %d doesn't support "
+					"scaling\n", ovl->id);
+			return -EINVAL;
+		}
+
+		if (info->out_height != 0 && info->height != info->out_height) {
+			DSSERR("check_overlay: overlay %d doesn't support "
+					"scaling\n", ovl->id);
+			return -EINVAL;
+		}
+	}
+
+	if ((ovl->supported_modes & info->color_mode) == 0) {
+		DSSERR("check_overlay: overlay %d doesn't support mode %d\n",
+				ovl->id, info->color_mode);
+		return -EINVAL;
+	}
+
+	if (info->zorder >= omap_dss_get_num_overlays()) {
+		DSSERR("check_overlay: zorder %d too high\n", info->zorder);
+		return -EINVAL;
+	}
+
+	if (dss_feat_rotation_type_supported(info->rotation_type) == 0) {
+		DSSERR("check_overlay: rotation type %d not supported\n",
+				info->rotation_type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int dss_ovl_check(struct omap_overlay *ovl, struct omap_overlay_info *info,
+		const struct omap_video_timings *mgr_timings)
+{
+	u16 outw, outh;
+	u16 dw, dh;
+
+	dw = mgr_timings->x_res;
+	dh = mgr_timings->y_res;
+
+	if ((ovl->caps & OMAP_DSS_OVL_CAP_SCALE) == 0) {
+		outw = info->width;
+		outh = info->height;
+	} else {
+		if (info->out_width == 0)
+			outw = info->width;
+		else
+			outw = info->out_width;
+
+		if (info->out_height == 0)
+			outh = info->height;
+		else
+			outh = info->out_height;
+	}
+
+	if (dw < info->pos_x + outw) {
+		DSSERR("overlay %d horizontally not inside the display area "
+				"(%d + %d >= %d)\n",
+				ovl->id, info->pos_x, outw, dw);
+		return -EINVAL;
+	}
+
+	if (dh < info->pos_y + outh) {
+		DSSERR("overlay %d vertically not inside the display area "
+				"(%d + %d >= %d)\n",
+				ovl->id, info->pos_y, outh, dh);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * Checks if replication logic should be used. Only use when overlay is in
+ * RGB12U or RGB16 mode, and video port width interface is 18bpp or 24bpp
+ */
+bool dss_ovl_use_replication(struct dss_lcd_mgr_config config,
+		enum omap_color_mode mode)
+{
+	if (mode != OMAP_DSS_COLOR_RGB12U && mode != OMAP_DSS_COLOR_RGB16)
+		return false;
+
+	return config.video_port_width > 16;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/pll.c b/drivers/video/fbdev/omap2/omapfb/dss/pll.c
new file mode 100644
index 000000000000..f974ddcd3b6e
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/pll.c
@@ -0,0 +1,389 @@
+/*
+ * Copyright (C) 2014 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "PLL"
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/regulator/consumer.h>
+#include <linux/sched.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+
+#define PLL_CONTROL			0x0000
+#define PLL_STATUS			0x0004
+#define PLL_GO				0x0008
+#define PLL_CONFIGURATION1		0x000C
+#define PLL_CONFIGURATION2		0x0010
+#define PLL_CONFIGURATION3		0x0014
+#define PLL_SSC_CONFIGURATION1		0x0018
+#define PLL_SSC_CONFIGURATION2		0x001C
+#define PLL_CONFIGURATION4		0x0020
+
+static struct dss_pll *dss_plls[4];
+
+int dss_pll_register(struct dss_pll *pll)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dss_plls); ++i) {
+		if (!dss_plls[i]) {
+			dss_plls[i] = pll;
+			return 0;
+		}
+	}
+
+	return -EBUSY;
+}
+
+void dss_pll_unregister(struct dss_pll *pll)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dss_plls); ++i) {
+		if (dss_plls[i] == pll) {
+			dss_plls[i] = NULL;
+			return;
+		}
+	}
+}
+
+struct dss_pll *dss_pll_find(const char *name)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dss_plls); ++i) {
+		if (dss_plls[i] && strcmp(dss_plls[i]->name, name) == 0)
+			return dss_plls[i];
+	}
+
+	return NULL;
+}
+
+int dss_pll_enable(struct dss_pll *pll)
+{
+	int r;
+
+	r = clk_prepare_enable(pll->clkin);
+	if (r)
+		return r;
+
+	if (pll->regulator) {
+		r = regulator_enable(pll->regulator);
+		if (r)
+			goto err_reg;
+	}
+
+	r = pll->ops->enable(pll);
+	if (r)
+		goto err_enable;
+
+	return 0;
+
+err_enable:
+	if (pll->regulator)
+		regulator_disable(pll->regulator);
+err_reg:
+	clk_disable_unprepare(pll->clkin);
+	return r;
+}
+
+void dss_pll_disable(struct dss_pll *pll)
+{
+	pll->ops->disable(pll);
+
+	if (pll->regulator)
+		regulator_disable(pll->regulator);
+
+	clk_disable_unprepare(pll->clkin);
+
+	memset(&pll->cinfo, 0, sizeof(pll->cinfo));
+}
+
+int dss_pll_set_config(struct dss_pll *pll, const struct dss_pll_clock_info *cinfo)
+{
+	int r;
+
+	r = pll->ops->set_config(pll, cinfo);
+	if (r)
+		return r;
+
+	pll->cinfo = *cinfo;
+
+	return 0;
+}
+
+bool dss_pll_hsdiv_calc(const struct dss_pll *pll, unsigned long clkdco,
+		unsigned long out_min, unsigned long out_max,
+		dss_hsdiv_calc_func func, void *data)
+{
+	const struct dss_pll_hw *hw = pll->hw;
+	int m, m_start, m_stop;
+	unsigned long out;
+
+	out_min = out_min ? out_min : 1;
+	out_max = out_max ? out_max : ULONG_MAX;
+
+	m_start = max(DIV_ROUND_UP(clkdco, out_max), 1ul);
+
+	m_stop = min((unsigned)(clkdco / out_min), hw->mX_max);
+
+	for (m = m_start; m <= m_stop; ++m) {
+		out = clkdco / m;
+
+		if (func(m, out, data))
+			return true;
+	}
+
+	return false;
+}
+
+bool dss_pll_calc(const struct dss_pll *pll, unsigned long clkin,
+		unsigned long pll_min, unsigned long pll_max,
+		dss_pll_calc_func func, void *data)
+{
+	const struct dss_pll_hw *hw = pll->hw;
+	int n, n_start, n_stop;
+	int m, m_start, m_stop;
+	unsigned long fint, clkdco;
+	unsigned long pll_hw_max;
+	unsigned long fint_hw_min, fint_hw_max;
+
+	pll_hw_max = hw->clkdco_max;
+
+	fint_hw_min = hw->fint_min;
+	fint_hw_max = hw->fint_max;
+
+	n_start = max(DIV_ROUND_UP(clkin, fint_hw_max), 1ul);
+	n_stop = min((unsigned)(clkin / fint_hw_min), hw->n_max);
+
+	pll_max = pll_max ? pll_max : ULONG_MAX;
+
+	for (n = n_start; n <= n_stop; ++n) {
+		fint = clkin / n;
+
+		m_start = max(DIV_ROUND_UP(DIV_ROUND_UP(pll_min, fint), 2),
+				1ul);
+		m_stop = min3((unsigned)(pll_max / fint / 2),
+				(unsigned)(pll_hw_max / fint / 2),
+				hw->m_max);
+
+		for (m = m_start; m <= m_stop; ++m) {
+			clkdco = 2 * m * fint;
+
+			if (func(n, m, fint, clkdco, data))
+				return true;
+		}
+	}
+
+	return false;
+}
+
+static int wait_for_bit_change(void __iomem *reg, int bitnum, int value)
+{
+	unsigned long timeout;
+	ktime_t wait;
+	int t;
+
+	/* first busyloop to see if the bit changes right away */
+	t = 100;
+	while (t-- > 0) {
+		if (FLD_GET(readl_relaxed(reg), bitnum, bitnum) == value)
+			return value;
+	}
+
+	/* then loop for 500ms, sleeping for 1ms in between */
+	timeout = jiffies + msecs_to_jiffies(500);
+	while (time_before(jiffies, timeout)) {
+		if (FLD_GET(readl_relaxed(reg), bitnum, bitnum) == value)
+			return value;
+
+		wait = ns_to_ktime(1000 * 1000);
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_hrtimeout(&wait, HRTIMER_MODE_REL);
+	}
+
+	return !value;
+}
+
+int dss_pll_wait_reset_done(struct dss_pll *pll)
+{
+	void __iomem *base = pll->base;
+
+	if (wait_for_bit_change(base + PLL_STATUS, 0, 1) != 1)
+		return -ETIMEDOUT;
+	else
+		return 0;
+}
+
+static int dss_wait_hsdiv_ack(struct dss_pll *pll, u32 hsdiv_ack_mask)
+{
+	int t = 100;
+
+	while (t-- > 0) {
+		u32 v = readl_relaxed(pll->base + PLL_STATUS);
+		v &= hsdiv_ack_mask;
+		if (v == hsdiv_ack_mask)
+			return 0;
+	}
+
+	return -ETIMEDOUT;
+}
+
+int dss_pll_write_config_type_a(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo)
+{
+	const struct dss_pll_hw *hw = pll->hw;
+	void __iomem *base = pll->base;
+	int r = 0;
+	u32 l;
+
+	l = 0;
+	if (hw->has_stopmode)
+		l = FLD_MOD(l, 1, 0, 0);		/* PLL_STOPMODE */
+	l = FLD_MOD(l, cinfo->n - 1, hw->n_msb, hw->n_lsb);	/* PLL_REGN */
+	l = FLD_MOD(l, cinfo->m, hw->m_msb, hw->m_lsb);		/* PLL_REGM */
+	/* M4 */
+	l = FLD_MOD(l, cinfo->mX[0] ? cinfo->mX[0] - 1 : 0,
+			hw->mX_msb[0], hw->mX_lsb[0]);
+	/* M5 */
+	l = FLD_MOD(l, cinfo->mX[1] ? cinfo->mX[1] - 1 : 0,
+			hw->mX_msb[1], hw->mX_lsb[1]);
+	writel_relaxed(l, base + PLL_CONFIGURATION1);
+
+	l = 0;
+	/* M6 */
+	l = FLD_MOD(l, cinfo->mX[2] ? cinfo->mX[2] - 1 : 0,
+			hw->mX_msb[2], hw->mX_lsb[2]);
+	/* M7 */
+	l = FLD_MOD(l, cinfo->mX[3] ? cinfo->mX[3] - 1 : 0,
+			hw->mX_msb[3], hw->mX_lsb[3]);
+	writel_relaxed(l, base + PLL_CONFIGURATION3);
+
+	l = readl_relaxed(base + PLL_CONFIGURATION2);
+	if (hw->has_freqsel) {
+		u32 f = cinfo->fint < 1000000 ? 0x3 :
+			cinfo->fint < 1250000 ? 0x4 :
+			cinfo->fint < 1500000 ? 0x5 :
+			cinfo->fint < 1750000 ? 0x6 :
+			0x7;
+
+		l = FLD_MOD(l, f, 4, 1);	/* PLL_FREQSEL */
+	} else if (hw->has_selfreqdco) {
+		u32 f = cinfo->clkdco < hw->clkdco_low ? 0x2 : 0x4;
+
+		l = FLD_MOD(l, f, 3, 1);	/* PLL_SELFREQDCO */
+	}
+	l = FLD_MOD(l, 1, 13, 13);		/* PLL_REFEN */
+	l = FLD_MOD(l, 0, 14, 14);		/* PHY_CLKINEN */
+	l = FLD_MOD(l, 0, 16, 16);		/* M4_CLOCK_EN */
+	l = FLD_MOD(l, 0, 18, 18);		/* M5_CLOCK_EN */
+	l = FLD_MOD(l, 1, 20, 20);		/* HSDIVBYPASS */
+	if (hw->has_refsel)
+		l = FLD_MOD(l, 3, 22, 21);	/* REFSEL = sysclk */
+	l = FLD_MOD(l, 0, 23, 23);		/* M6_CLOCK_EN */
+	l = FLD_MOD(l, 0, 25, 25);		/* M7_CLOCK_EN */
+	writel_relaxed(l, base + PLL_CONFIGURATION2);
+
+	writel_relaxed(1, base + PLL_GO);	/* PLL_GO */
+
+	if (wait_for_bit_change(base + PLL_GO, 0, 0) != 0) {
+		DSSERR("DSS DPLL GO bit not going down.\n");
+		r = -EIO;
+		goto err;
+	}
+
+	if (wait_for_bit_change(base + PLL_STATUS, 1, 1) != 1) {
+		DSSERR("cannot lock DSS DPLL\n");
+		r = -EIO;
+		goto err;
+	}
+
+	l = readl_relaxed(base + PLL_CONFIGURATION2);
+	l = FLD_MOD(l, 1, 14, 14);			/* PHY_CLKINEN */
+	l = FLD_MOD(l, cinfo->mX[0] ? 1 : 0, 16, 16);	/* M4_CLOCK_EN */
+	l = FLD_MOD(l, cinfo->mX[1] ? 1 : 0, 18, 18);	/* M5_CLOCK_EN */
+	l = FLD_MOD(l, 0, 20, 20);			/* HSDIVBYPASS */
+	l = FLD_MOD(l, cinfo->mX[2] ? 1 : 0, 23, 23);	/* M6_CLOCK_EN */
+	l = FLD_MOD(l, cinfo->mX[3] ? 1 : 0, 25, 25);	/* M7_CLOCK_EN */
+	writel_relaxed(l, base + PLL_CONFIGURATION2);
+
+	r = dss_wait_hsdiv_ack(pll,
+		(cinfo->mX[0] ? BIT(7) : 0) |
+		(cinfo->mX[1] ? BIT(8) : 0) |
+		(cinfo->mX[2] ? BIT(10) : 0) |
+		(cinfo->mX[3] ? BIT(11) : 0));
+	if (r) {
+		DSSERR("failed to enable HSDIV clocks\n");
+		goto err;
+	}
+
+err:
+	return r;
+}
+
+int dss_pll_write_config_type_b(struct dss_pll *pll,
+		const struct dss_pll_clock_info *cinfo)
+{
+	const struct dss_pll_hw *hw = pll->hw;
+	void __iomem *base = pll->base;
+	u32 l;
+
+	l = 0;
+	l = FLD_MOD(l, cinfo->m, 20, 9);	/* PLL_REGM */
+	l = FLD_MOD(l, cinfo->n - 1, 8, 1);	/* PLL_REGN */
+	writel_relaxed(l, base + PLL_CONFIGURATION1);
+
+	l = readl_relaxed(base + PLL_CONFIGURATION2);
+	l = FLD_MOD(l, 0x0, 12, 12);	/* PLL_HIGHFREQ divide by 2 */
+	l = FLD_MOD(l, 0x1, 13, 13);	/* PLL_REFEN */
+	l = FLD_MOD(l, 0x0, 14, 14);	/* PHY_CLKINEN */
+	if (hw->has_refsel)
+		l = FLD_MOD(l, 0x3, 22, 21);	/* REFSEL = SYSCLK */
+
+	/* PLL_SELFREQDCO */
+	if (cinfo->clkdco > hw->clkdco_low)
+		l = FLD_MOD(l, 0x4, 3, 1);
+	else
+		l = FLD_MOD(l, 0x2, 3, 1);
+	writel_relaxed(l, base + PLL_CONFIGURATION2);
+
+	l = readl_relaxed(base + PLL_CONFIGURATION3);
+	l = FLD_MOD(l, cinfo->sd, 17, 10);	/* PLL_REGSD */
+	writel_relaxed(l, base + PLL_CONFIGURATION3);
+
+	l = readl_relaxed(base + PLL_CONFIGURATION4);
+	l = FLD_MOD(l, cinfo->mX[0], 24, 18);	/* PLL_REGM2 */
+	l = FLD_MOD(l, cinfo->mf, 17, 0);	/* PLL_REGM_F */
+	writel_relaxed(l, base + PLL_CONFIGURATION4);
+
+	writel_relaxed(1, base + PLL_GO);	/* PLL_GO */
+
+	if (wait_for_bit_change(base + PLL_GO, 0, 0) != 0) {
+		DSSERR("DSS DPLL GO bit not going down.\n");
+		return -EIO;
+	}
+
+	if (wait_for_bit_change(base + PLL_STATUS, 1, 1) != 1) {
+		DSSERR("cannot lock DSS DPLL\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c b/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c
new file mode 100644
index 000000000000..aea6a1d0fb20
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c
@@ -0,0 +1,1078 @@
+/*
+ * linux/drivers/video/omap2/dss/rfbi.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * Some code and ideas taken from drivers/video/omap/ driver
+ * by Imre Deak.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "RFBI"
+
+#include <linux/kernel.h>
+#include <linux/dma-mapping.h>
+#include <linux/export.h>
+#include <linux/vmalloc.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/kfifo.h>
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+#include <linux/seq_file.h>
+#include <linux/semaphore.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+#include "dss.h"
+
+struct rfbi_reg { u16 idx; };
+
+#define RFBI_REG(idx)		((const struct rfbi_reg) { idx })
+
+#define RFBI_REVISION		RFBI_REG(0x0000)
+#define RFBI_SYSCONFIG		RFBI_REG(0x0010)
+#define RFBI_SYSSTATUS		RFBI_REG(0x0014)
+#define RFBI_CONTROL		RFBI_REG(0x0040)
+#define RFBI_PIXEL_CNT		RFBI_REG(0x0044)
+#define RFBI_LINE_NUMBER	RFBI_REG(0x0048)
+#define RFBI_CMD		RFBI_REG(0x004c)
+#define RFBI_PARAM		RFBI_REG(0x0050)
+#define RFBI_DATA		RFBI_REG(0x0054)
+#define RFBI_READ		RFBI_REG(0x0058)
+#define RFBI_STATUS		RFBI_REG(0x005c)
+
+#define RFBI_CONFIG(n)		RFBI_REG(0x0060 + (n)*0x18)
+#define RFBI_ONOFF_TIME(n)	RFBI_REG(0x0064 + (n)*0x18)
+#define RFBI_CYCLE_TIME(n)	RFBI_REG(0x0068 + (n)*0x18)
+#define RFBI_DATA_CYCLE1(n)	RFBI_REG(0x006c + (n)*0x18)
+#define RFBI_DATA_CYCLE2(n)	RFBI_REG(0x0070 + (n)*0x18)
+#define RFBI_DATA_CYCLE3(n)	RFBI_REG(0x0074 + (n)*0x18)
+
+#define RFBI_VSYNC_WIDTH	RFBI_REG(0x0090)
+#define RFBI_HSYNC_WIDTH	RFBI_REG(0x0094)
+
+#define REG_FLD_MOD(idx, val, start, end) \
+	rfbi_write_reg(idx, FLD_MOD(rfbi_read_reg(idx), val, start, end))
+
+enum omap_rfbi_cycleformat {
+	OMAP_DSS_RFBI_CYCLEFORMAT_1_1 = 0,
+	OMAP_DSS_RFBI_CYCLEFORMAT_2_1 = 1,
+	OMAP_DSS_RFBI_CYCLEFORMAT_3_1 = 2,
+	OMAP_DSS_RFBI_CYCLEFORMAT_3_2 = 3,
+};
+
+enum omap_rfbi_datatype {
+	OMAP_DSS_RFBI_DATATYPE_12 = 0,
+	OMAP_DSS_RFBI_DATATYPE_16 = 1,
+	OMAP_DSS_RFBI_DATATYPE_18 = 2,
+	OMAP_DSS_RFBI_DATATYPE_24 = 3,
+};
+
+enum omap_rfbi_parallelmode {
+	OMAP_DSS_RFBI_PARALLELMODE_8 = 0,
+	OMAP_DSS_RFBI_PARALLELMODE_9 = 1,
+	OMAP_DSS_RFBI_PARALLELMODE_12 = 2,
+	OMAP_DSS_RFBI_PARALLELMODE_16 = 3,
+};
+
+static int rfbi_convert_timings(struct rfbi_timings *t);
+static void rfbi_get_clk_info(u32 *clk_period, u32 *max_clk_div);
+
+static struct {
+	struct platform_device *pdev;
+	void __iomem	*base;
+
+	unsigned long	l4_khz;
+
+	enum omap_rfbi_datatype datatype;
+	enum omap_rfbi_parallelmode parallelmode;
+
+	enum omap_rfbi_te_mode te_mode;
+	int te_enabled;
+
+	void (*framedone_callback)(void *data);
+	void *framedone_callback_data;
+
+	struct omap_dss_device *dssdev[2];
+
+	struct semaphore bus_lock;
+
+	struct omap_video_timings timings;
+	int pixel_size;
+	int data_lines;
+	struct rfbi_timings intf_timings;
+
+	struct omap_dss_device output;
+} rfbi;
+
+static inline void rfbi_write_reg(const struct rfbi_reg idx, u32 val)
+{
+	__raw_writel(val, rfbi.base + idx.idx);
+}
+
+static inline u32 rfbi_read_reg(const struct rfbi_reg idx)
+{
+	return __raw_readl(rfbi.base + idx.idx);
+}
+
+static int rfbi_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("rfbi_runtime_get\n");
+
+	r = pm_runtime_get_sync(&rfbi.pdev->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+static void rfbi_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("rfbi_runtime_put\n");
+
+	r = pm_runtime_put_sync(&rfbi.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+
+static void rfbi_bus_lock(void)
+{
+	down(&rfbi.bus_lock);
+}
+
+static void rfbi_bus_unlock(void)
+{
+	up(&rfbi.bus_lock);
+}
+
+static void rfbi_write_command(const void *buf, u32 len)
+{
+	switch (rfbi.parallelmode) {
+	case OMAP_DSS_RFBI_PARALLELMODE_8:
+	{
+		const u8 *b = buf;
+		for (; len; len--)
+			rfbi_write_reg(RFBI_CMD, *b++);
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_16:
+	{
+		const u16 *w = buf;
+		BUG_ON(len & 1);
+		for (; len; len -= 2)
+			rfbi_write_reg(RFBI_CMD, *w++);
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_9:
+	case OMAP_DSS_RFBI_PARALLELMODE_12:
+	default:
+		BUG();
+	}
+}
+
+static void rfbi_read_data(void *buf, u32 len)
+{
+	switch (rfbi.parallelmode) {
+	case OMAP_DSS_RFBI_PARALLELMODE_8:
+	{
+		u8 *b = buf;
+		for (; len; len--) {
+			rfbi_write_reg(RFBI_READ, 0);
+			*b++ = rfbi_read_reg(RFBI_READ);
+		}
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_16:
+	{
+		u16 *w = buf;
+		BUG_ON(len & ~1);
+		for (; len; len -= 2) {
+			rfbi_write_reg(RFBI_READ, 0);
+			*w++ = rfbi_read_reg(RFBI_READ);
+		}
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_9:
+	case OMAP_DSS_RFBI_PARALLELMODE_12:
+	default:
+		BUG();
+	}
+}
+
+static void rfbi_write_data(const void *buf, u32 len)
+{
+	switch (rfbi.parallelmode) {
+	case OMAP_DSS_RFBI_PARALLELMODE_8:
+	{
+		const u8 *b = buf;
+		for (; len; len--)
+			rfbi_write_reg(RFBI_PARAM, *b++);
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_16:
+	{
+		const u16 *w = buf;
+		BUG_ON(len & 1);
+		for (; len; len -= 2)
+			rfbi_write_reg(RFBI_PARAM, *w++);
+		break;
+	}
+
+	case OMAP_DSS_RFBI_PARALLELMODE_9:
+	case OMAP_DSS_RFBI_PARALLELMODE_12:
+	default:
+		BUG();
+
+	}
+}
+
+static void rfbi_write_pixels(const void __iomem *buf, int scr_width,
+		u16 x, u16 y,
+		u16 w, u16 h)
+{
+	int start_offset = scr_width * y + x;
+	int horiz_offset = scr_width - w;
+	int i;
+
+	if (rfbi.datatype == OMAP_DSS_RFBI_DATATYPE_16 &&
+	   rfbi.parallelmode == OMAP_DSS_RFBI_PARALLELMODE_8) {
+		const u16 __iomem *pd = buf;
+		pd += start_offset;
+
+		for (; h; --h) {
+			for (i = 0; i < w; ++i) {
+				const u8 __iomem *b = (const u8 __iomem *)pd;
+				rfbi_write_reg(RFBI_PARAM, __raw_readb(b+1));
+				rfbi_write_reg(RFBI_PARAM, __raw_readb(b+0));
+				++pd;
+			}
+			pd += horiz_offset;
+		}
+	} else if (rfbi.datatype == OMAP_DSS_RFBI_DATATYPE_24 &&
+	   rfbi.parallelmode == OMAP_DSS_RFBI_PARALLELMODE_8) {
+		const u32 __iomem *pd = buf;
+		pd += start_offset;
+
+		for (; h; --h) {
+			for (i = 0; i < w; ++i) {
+				const u8 __iomem *b = (const u8 __iomem *)pd;
+				rfbi_write_reg(RFBI_PARAM, __raw_readb(b+2));
+				rfbi_write_reg(RFBI_PARAM, __raw_readb(b+1));
+				rfbi_write_reg(RFBI_PARAM, __raw_readb(b+0));
+				++pd;
+			}
+			pd += horiz_offset;
+		}
+	} else if (rfbi.datatype == OMAP_DSS_RFBI_DATATYPE_16 &&
+	   rfbi.parallelmode == OMAP_DSS_RFBI_PARALLELMODE_16) {
+		const u16 __iomem *pd = buf;
+		pd += start_offset;
+
+		for (; h; --h) {
+			for (i = 0; i < w; ++i) {
+				rfbi_write_reg(RFBI_PARAM, __raw_readw(pd));
+				++pd;
+			}
+			pd += horiz_offset;
+		}
+	} else {
+		BUG();
+	}
+}
+
+static int rfbi_transfer_area(struct omap_dss_device *dssdev,
+		void (*callback)(void *data), void *data)
+{
+	u32 l;
+	int r;
+	struct omap_overlay_manager *mgr = rfbi.output.manager;
+	u16 width = rfbi.timings.x_res;
+	u16 height = rfbi.timings.y_res;
+
+	/*BUG_ON(callback == 0);*/
+	BUG_ON(rfbi.framedone_callback != NULL);
+
+	DSSDBG("rfbi_transfer_area %dx%d\n", width, height);
+
+	dss_mgr_set_timings(mgr, &rfbi.timings);
+
+	r = dss_mgr_enable(mgr);
+	if (r)
+		return r;
+
+	rfbi.framedone_callback = callback;
+	rfbi.framedone_callback_data = data;
+
+	rfbi_write_reg(RFBI_PIXEL_CNT, width * height);
+
+	l = rfbi_read_reg(RFBI_CONTROL);
+	l = FLD_MOD(l, 1, 0, 0); /* enable */
+	if (!rfbi.te_enabled)
+		l = FLD_MOD(l, 1, 4, 4); /* ITE */
+
+	rfbi_write_reg(RFBI_CONTROL, l);
+
+	return 0;
+}
+
+static void framedone_callback(void *data)
+{
+	void (*callback)(void *data);
+
+	DSSDBG("FRAMEDONE\n");
+
+	REG_FLD_MOD(RFBI_CONTROL, 0, 0, 0);
+
+	callback = rfbi.framedone_callback;
+	rfbi.framedone_callback = NULL;
+
+	if (callback != NULL)
+		callback(rfbi.framedone_callback_data);
+}
+
+#if 1 /* VERBOSE */
+static void rfbi_print_timings(void)
+{
+	u32 l;
+	u32 time;
+
+	l = rfbi_read_reg(RFBI_CONFIG(0));
+	time = 1000000000 / rfbi.l4_khz;
+	if (l & (1 << 4))
+		time *= 2;
+
+	DSSDBG("Tick time %u ps\n", time);
+	l = rfbi_read_reg(RFBI_ONOFF_TIME(0));
+	DSSDBG("CSONTIME %d, CSOFFTIME %d, WEONTIME %d, WEOFFTIME %d, "
+		"REONTIME %d, REOFFTIME %d\n",
+		l & 0x0f, (l >> 4) & 0x3f, (l >> 10) & 0x0f, (l >> 14) & 0x3f,
+		(l >> 20) & 0x0f, (l >> 24) & 0x3f);
+
+	l = rfbi_read_reg(RFBI_CYCLE_TIME(0));
+	DSSDBG("WECYCLETIME %d, RECYCLETIME %d, CSPULSEWIDTH %d, "
+		"ACCESSTIME %d\n",
+		(l & 0x3f), (l >> 6) & 0x3f, (l >> 12) & 0x3f,
+		(l >> 22) & 0x3f);
+}
+#else
+static void rfbi_print_timings(void) {}
+#endif
+
+
+
+
+static u32 extif_clk_period;
+
+static inline unsigned long round_to_extif_ticks(unsigned long ps, int div)
+{
+	int bus_tick = extif_clk_period * div;
+	return (ps + bus_tick - 1) / bus_tick * bus_tick;
+}
+
+static int calc_reg_timing(struct rfbi_timings *t, int div)
+{
+	t->clk_div = div;
+
+	t->cs_on_time = round_to_extif_ticks(t->cs_on_time, div);
+
+	t->we_on_time = round_to_extif_ticks(t->we_on_time, div);
+	t->we_off_time = round_to_extif_ticks(t->we_off_time, div);
+	t->we_cycle_time = round_to_extif_ticks(t->we_cycle_time, div);
+
+	t->re_on_time = round_to_extif_ticks(t->re_on_time, div);
+	t->re_off_time = round_to_extif_ticks(t->re_off_time, div);
+	t->re_cycle_time = round_to_extif_ticks(t->re_cycle_time, div);
+
+	t->access_time = round_to_extif_ticks(t->access_time, div);
+	t->cs_off_time = round_to_extif_ticks(t->cs_off_time, div);
+	t->cs_pulse_width = round_to_extif_ticks(t->cs_pulse_width, div);
+
+	DSSDBG("[reg]cson %d csoff %d reon %d reoff %d\n",
+	       t->cs_on_time, t->cs_off_time, t->re_on_time, t->re_off_time);
+	DSSDBG("[reg]weon %d weoff %d recyc %d wecyc %d\n",
+	       t->we_on_time, t->we_off_time, t->re_cycle_time,
+	       t->we_cycle_time);
+	DSSDBG("[reg]rdaccess %d cspulse %d\n",
+	       t->access_time, t->cs_pulse_width);
+
+	return rfbi_convert_timings(t);
+}
+
+static int calc_extif_timings(struct rfbi_timings *t)
+{
+	u32 max_clk_div;
+	int div;
+
+	rfbi_get_clk_info(&extif_clk_period, &max_clk_div);
+	for (div = 1; div <= max_clk_div; div++) {
+		if (calc_reg_timing(t, div) == 0)
+			break;
+	}
+
+	if (div <= max_clk_div)
+		return 0;
+
+	DSSERR("can't setup timings\n");
+	return -1;
+}
+
+
+static void rfbi_set_timings(int rfbi_module, struct rfbi_timings *t)
+{
+	int r;
+
+	if (!t->converted) {
+		r = calc_extif_timings(t);
+		if (r < 0)
+			DSSERR("Failed to calc timings\n");
+	}
+
+	BUG_ON(!t->converted);
+
+	rfbi_write_reg(RFBI_ONOFF_TIME(rfbi_module), t->tim[0]);
+	rfbi_write_reg(RFBI_CYCLE_TIME(rfbi_module), t->tim[1]);
+
+	/* TIMEGRANULARITY */
+	REG_FLD_MOD(RFBI_CONFIG(rfbi_module),
+		    (t->tim[2] ? 1 : 0), 4, 4);
+
+	rfbi_print_timings();
+}
+
+static int ps_to_rfbi_ticks(int time, int div)
+{
+	unsigned long tick_ps;
+	int ret;
+
+	/* Calculate in picosecs to yield more exact results */
+	tick_ps = 1000000000 / (rfbi.l4_khz) * div;
+
+	ret = (time + tick_ps - 1) / tick_ps;
+
+	return ret;
+}
+
+static void rfbi_get_clk_info(u32 *clk_period, u32 *max_clk_div)
+{
+	*clk_period = 1000000000 / rfbi.l4_khz;
+	*max_clk_div = 2;
+}
+
+static int rfbi_convert_timings(struct rfbi_timings *t)
+{
+	u32 l;
+	int reon, reoff, weon, weoff, cson, csoff, cs_pulse;
+	int actim, recyc, wecyc;
+	int div = t->clk_div;
+
+	if (div <= 0 || div > 2)
+		return -1;
+
+	/* Make sure that after conversion it still holds that:
+	 * weoff > weon, reoff > reon, recyc >= reoff, wecyc >= weoff,
+	 * csoff > cson, csoff >= max(weoff, reoff), actim > reon
+	 */
+	weon = ps_to_rfbi_ticks(t->we_on_time, div);
+	weoff = ps_to_rfbi_ticks(t->we_off_time, div);
+	if (weoff <= weon)
+		weoff = weon + 1;
+	if (weon > 0x0f)
+		return -1;
+	if (weoff > 0x3f)
+		return -1;
+
+	reon = ps_to_rfbi_ticks(t->re_on_time, div);
+	reoff = ps_to_rfbi_ticks(t->re_off_time, div);
+	if (reoff <= reon)
+		reoff = reon + 1;
+	if (reon > 0x0f)
+		return -1;
+	if (reoff > 0x3f)
+		return -1;
+
+	cson = ps_to_rfbi_ticks(t->cs_on_time, div);
+	csoff = ps_to_rfbi_ticks(t->cs_off_time, div);
+	if (csoff <= cson)
+		csoff = cson + 1;
+	if (csoff < max(weoff, reoff))
+		csoff = max(weoff, reoff);
+	if (cson > 0x0f)
+		return -1;
+	if (csoff > 0x3f)
+		return -1;
+
+	l =  cson;
+	l |= csoff << 4;
+	l |= weon  << 10;
+	l |= weoff << 14;
+	l |= reon  << 20;
+	l |= reoff << 24;
+
+	t->tim[0] = l;
+
+	actim = ps_to_rfbi_ticks(t->access_time, div);
+	if (actim <= reon)
+		actim = reon + 1;
+	if (actim > 0x3f)
+		return -1;
+
+	wecyc = ps_to_rfbi_ticks(t->we_cycle_time, div);
+	if (wecyc < weoff)
+		wecyc = weoff;
+	if (wecyc > 0x3f)
+		return -1;
+
+	recyc = ps_to_rfbi_ticks(t->re_cycle_time, div);
+	if (recyc < reoff)
+		recyc = reoff;
+	if (recyc > 0x3f)
+		return -1;
+
+	cs_pulse = ps_to_rfbi_ticks(t->cs_pulse_width, div);
+	if (cs_pulse > 0x3f)
+		return -1;
+
+	l =  wecyc;
+	l |= recyc    << 6;
+	l |= cs_pulse << 12;
+	l |= actim    << 22;
+
+	t->tim[1] = l;
+
+	t->tim[2] = div - 1;
+
+	t->converted = 1;
+
+	return 0;
+}
+
+/* xxx FIX module selection missing */
+static int rfbi_setup_te(enum omap_rfbi_te_mode mode,
+			     unsigned hs_pulse_time, unsigned vs_pulse_time,
+			     int hs_pol_inv, int vs_pol_inv, int extif_div)
+{
+	int hs, vs;
+	int min;
+	u32 l;
+
+	hs = ps_to_rfbi_ticks(hs_pulse_time, 1);
+	vs = ps_to_rfbi_ticks(vs_pulse_time, 1);
+	if (hs < 2)
+		return -EDOM;
+	if (mode == OMAP_DSS_RFBI_TE_MODE_2)
+		min = 2;
+	else /* OMAP_DSS_RFBI_TE_MODE_1 */
+		min = 4;
+	if (vs < min)
+		return -EDOM;
+	if (vs == hs)
+		return -EINVAL;
+	rfbi.te_mode = mode;
+	DSSDBG("setup_te: mode %d hs %d vs %d hs_inv %d vs_inv %d\n",
+		mode, hs, vs, hs_pol_inv, vs_pol_inv);
+
+	rfbi_write_reg(RFBI_HSYNC_WIDTH, hs);
+	rfbi_write_reg(RFBI_VSYNC_WIDTH, vs);
+
+	l = rfbi_read_reg(RFBI_CONFIG(0));
+	if (hs_pol_inv)
+		l &= ~(1 << 21);
+	else
+		l |= 1 << 21;
+	if (vs_pol_inv)
+		l &= ~(1 << 20);
+	else
+		l |= 1 << 20;
+
+	return 0;
+}
+
+/* xxx FIX module selection missing */
+static int rfbi_enable_te(bool enable, unsigned line)
+{
+	u32 l;
+
+	DSSDBG("te %d line %d mode %d\n", enable, line, rfbi.te_mode);
+	if (line > (1 << 11) - 1)
+		return -EINVAL;
+
+	l = rfbi_read_reg(RFBI_CONFIG(0));
+	l &= ~(0x3 << 2);
+	if (enable) {
+		rfbi.te_enabled = 1;
+		l |= rfbi.te_mode << 2;
+	} else
+		rfbi.te_enabled = 0;
+	rfbi_write_reg(RFBI_CONFIG(0), l);
+	rfbi_write_reg(RFBI_LINE_NUMBER, line);
+
+	return 0;
+}
+
+static int rfbi_configure_bus(int rfbi_module, int bpp, int lines)
+{
+	u32 l;
+	int cycle1 = 0, cycle2 = 0, cycle3 = 0;
+	enum omap_rfbi_cycleformat cycleformat;
+	enum omap_rfbi_datatype datatype;
+	enum omap_rfbi_parallelmode parallelmode;
+
+	switch (bpp) {
+	case 12:
+		datatype = OMAP_DSS_RFBI_DATATYPE_12;
+		break;
+	case 16:
+		datatype = OMAP_DSS_RFBI_DATATYPE_16;
+		break;
+	case 18:
+		datatype = OMAP_DSS_RFBI_DATATYPE_18;
+		break;
+	case 24:
+		datatype = OMAP_DSS_RFBI_DATATYPE_24;
+		break;
+	default:
+		BUG();
+		return 1;
+	}
+	rfbi.datatype = datatype;
+
+	switch (lines) {
+	case 8:
+		parallelmode = OMAP_DSS_RFBI_PARALLELMODE_8;
+		break;
+	case 9:
+		parallelmode = OMAP_DSS_RFBI_PARALLELMODE_9;
+		break;
+	case 12:
+		parallelmode = OMAP_DSS_RFBI_PARALLELMODE_12;
+		break;
+	case 16:
+		parallelmode = OMAP_DSS_RFBI_PARALLELMODE_16;
+		break;
+	default:
+		BUG();
+		return 1;
+	}
+	rfbi.parallelmode = parallelmode;
+
+	if ((bpp % lines) == 0) {
+		switch (bpp / lines) {
+		case 1:
+			cycleformat = OMAP_DSS_RFBI_CYCLEFORMAT_1_1;
+			break;
+		case 2:
+			cycleformat = OMAP_DSS_RFBI_CYCLEFORMAT_2_1;
+			break;
+		case 3:
+			cycleformat = OMAP_DSS_RFBI_CYCLEFORMAT_3_1;
+			break;
+		default:
+			BUG();
+			return 1;
+		}
+	} else if ((2 * bpp % lines) == 0) {
+		if ((2 * bpp / lines) == 3)
+			cycleformat = OMAP_DSS_RFBI_CYCLEFORMAT_3_2;
+		else {
+			BUG();
+			return 1;
+		}
+	} else {
+		BUG();
+		return 1;
+	}
+
+	switch (cycleformat) {
+	case OMAP_DSS_RFBI_CYCLEFORMAT_1_1:
+		cycle1 = lines;
+		break;
+
+	case OMAP_DSS_RFBI_CYCLEFORMAT_2_1:
+		cycle1 = lines;
+		cycle2 = lines;
+		break;
+
+	case OMAP_DSS_RFBI_CYCLEFORMAT_3_1:
+		cycle1 = lines;
+		cycle2 = lines;
+		cycle3 = lines;
+		break;
+
+	case OMAP_DSS_RFBI_CYCLEFORMAT_3_2:
+		cycle1 = lines;
+		cycle2 = (lines / 2) | ((lines / 2) << 16);
+		cycle3 = (lines << 16);
+		break;
+	}
+
+	REG_FLD_MOD(RFBI_CONTROL, 0, 3, 2); /* clear CS */
+
+	l = 0;
+	l |= FLD_VAL(parallelmode, 1, 0);
+	l |= FLD_VAL(0, 3, 2);		/* TRIGGERMODE: ITE */
+	l |= FLD_VAL(0, 4, 4);		/* TIMEGRANULARITY */
+	l |= FLD_VAL(datatype, 6, 5);
+	/* l |= FLD_VAL(2, 8, 7); */	/* L4FORMAT, 2pix/L4 */
+	l |= FLD_VAL(0, 8, 7);	/* L4FORMAT, 1pix/L4 */
+	l |= FLD_VAL(cycleformat, 10, 9);
+	l |= FLD_VAL(0, 12, 11);	/* UNUSEDBITS */
+	l |= FLD_VAL(0, 16, 16);	/* A0POLARITY */
+	l |= FLD_VAL(0, 17, 17);	/* REPOLARITY */
+	l |= FLD_VAL(0, 18, 18);	/* WEPOLARITY */
+	l |= FLD_VAL(0, 19, 19);	/* CSPOLARITY */
+	l |= FLD_VAL(1, 20, 20);	/* TE_VSYNC_POLARITY */
+	l |= FLD_VAL(1, 21, 21);	/* HSYNCPOLARITY */
+	rfbi_write_reg(RFBI_CONFIG(rfbi_module), l);
+
+	rfbi_write_reg(RFBI_DATA_CYCLE1(rfbi_module), cycle1);
+	rfbi_write_reg(RFBI_DATA_CYCLE2(rfbi_module), cycle2);
+	rfbi_write_reg(RFBI_DATA_CYCLE3(rfbi_module), cycle3);
+
+
+	l = rfbi_read_reg(RFBI_CONTROL);
+	l = FLD_MOD(l, rfbi_module+1, 3, 2); /* Select CSx */
+	l = FLD_MOD(l, 0, 1, 1); /* clear bypass */
+	rfbi_write_reg(RFBI_CONTROL, l);
+
+
+	DSSDBG("RFBI config: bpp %d, lines %d, cycles: 0x%x 0x%x 0x%x\n",
+	       bpp, lines, cycle1, cycle2, cycle3);
+
+	return 0;
+}
+
+static int rfbi_configure(struct omap_dss_device *dssdev)
+{
+	return rfbi_configure_bus(dssdev->phy.rfbi.channel, rfbi.pixel_size,
+			rfbi.data_lines);
+}
+
+static int rfbi_update(struct omap_dss_device *dssdev, void (*callback)(void *),
+		void *data)
+{
+	return rfbi_transfer_area(dssdev, callback, data);
+}
+
+static void rfbi_set_size(struct omap_dss_device *dssdev, u16 w, u16 h)
+{
+	rfbi.timings.x_res = w;
+	rfbi.timings.y_res = h;
+}
+
+static void rfbi_set_pixel_size(struct omap_dss_device *dssdev, int pixel_size)
+{
+	rfbi.pixel_size = pixel_size;
+}
+
+static void rfbi_set_data_lines(struct omap_dss_device *dssdev, int data_lines)
+{
+	rfbi.data_lines = data_lines;
+}
+
+static void rfbi_set_interface_timings(struct omap_dss_device *dssdev,
+		struct rfbi_timings *timings)
+{
+	rfbi.intf_timings = *timings;
+}
+
+static void rfbi_dump_regs(struct seq_file *s)
+{
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, rfbi_read_reg(r))
+
+	if (rfbi_runtime_get())
+		return;
+
+	DUMPREG(RFBI_REVISION);
+	DUMPREG(RFBI_SYSCONFIG);
+	DUMPREG(RFBI_SYSSTATUS);
+	DUMPREG(RFBI_CONTROL);
+	DUMPREG(RFBI_PIXEL_CNT);
+	DUMPREG(RFBI_LINE_NUMBER);
+	DUMPREG(RFBI_CMD);
+	DUMPREG(RFBI_PARAM);
+	DUMPREG(RFBI_DATA);
+	DUMPREG(RFBI_READ);
+	DUMPREG(RFBI_STATUS);
+
+	DUMPREG(RFBI_CONFIG(0));
+	DUMPREG(RFBI_ONOFF_TIME(0));
+	DUMPREG(RFBI_CYCLE_TIME(0));
+	DUMPREG(RFBI_DATA_CYCLE1(0));
+	DUMPREG(RFBI_DATA_CYCLE2(0));
+	DUMPREG(RFBI_DATA_CYCLE3(0));
+
+	DUMPREG(RFBI_CONFIG(1));
+	DUMPREG(RFBI_ONOFF_TIME(1));
+	DUMPREG(RFBI_CYCLE_TIME(1));
+	DUMPREG(RFBI_DATA_CYCLE1(1));
+	DUMPREG(RFBI_DATA_CYCLE2(1));
+	DUMPREG(RFBI_DATA_CYCLE3(1));
+
+	DUMPREG(RFBI_VSYNC_WIDTH);
+	DUMPREG(RFBI_HSYNC_WIDTH);
+
+	rfbi_runtime_put();
+#undef DUMPREG
+}
+
+static void rfbi_config_lcd_manager(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = rfbi.output.manager;
+	struct dss_lcd_mgr_config mgr_config;
+
+	mgr_config.io_pad_mode = DSS_IO_PAD_MODE_RFBI;
+
+	mgr_config.stallmode = true;
+	/* Do we need fifohandcheck for RFBI? */
+	mgr_config.fifohandcheck = false;
+
+	mgr_config.video_port_width = rfbi.pixel_size;
+	mgr_config.lcden_sig_polarity = 0;
+
+	dss_mgr_set_lcd_config(mgr, &mgr_config);
+
+	/*
+	 * Set rfbi.timings with default values, the x_res and y_res fields
+	 * are expected to be already configured by the panel driver via
+	 * omapdss_rfbi_set_size()
+	 */
+	rfbi.timings.hsw = 1;
+	rfbi.timings.hfp = 1;
+	rfbi.timings.hbp = 1;
+	rfbi.timings.vsw = 1;
+	rfbi.timings.vfp = 0;
+	rfbi.timings.vbp = 0;
+
+	rfbi.timings.interlace = false;
+	rfbi.timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	rfbi.timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	rfbi.timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	rfbi.timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	rfbi.timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE;
+
+	dss_mgr_set_timings(mgr, &rfbi.timings);
+}
+
+static int rfbi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &rfbi.output;
+	int r;
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		return -ENODEV;
+	}
+
+	r = rfbi_runtime_get();
+	if (r)
+		return r;
+
+	r = dss_mgr_register_framedone_handler(out->manager,
+			framedone_callback, NULL);
+	if (r) {
+		DSSERR("can't get FRAMEDONE irq\n");
+		goto err1;
+	}
+
+	rfbi_config_lcd_manager(dssdev);
+
+	rfbi_configure_bus(dssdev->phy.rfbi.channel, rfbi.pixel_size,
+			rfbi.data_lines);
+
+	rfbi_set_timings(dssdev->phy.rfbi.channel, &rfbi.intf_timings);
+
+	return 0;
+err1:
+	rfbi_runtime_put();
+	return r;
+}
+
+static void rfbi_display_disable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &rfbi.output;
+
+	dss_mgr_unregister_framedone_handler(out->manager,
+			framedone_callback, NULL);
+
+	rfbi_runtime_put();
+}
+
+static int rfbi_init_display(struct omap_dss_device *dssdev)
+{
+	rfbi.dssdev[dssdev->phy.rfbi.channel] = dssdev;
+	return 0;
+}
+
+static void rfbi_init_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &rfbi.output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_DBI;
+	out->output_type = OMAP_DISPLAY_TYPE_DBI;
+	out->name = "rfbi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_LCD;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void rfbi_uninit_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &rfbi.output;
+
+	omapdss_unregister_output(out);
+}
+
+/* RFBI HW IP initialisation */
+static int rfbi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	u32 rev;
+	struct resource *rfbi_mem;
+	struct clk *clk;
+	int r;
+
+	rfbi.pdev = pdev;
+
+	sema_init(&rfbi.bus_lock, 1);
+
+	rfbi_mem = platform_get_resource(rfbi.pdev, IORESOURCE_MEM, 0);
+	if (!rfbi_mem) {
+		DSSERR("can't get IORESOURCE_MEM RFBI\n");
+		return -EINVAL;
+	}
+
+	rfbi.base = devm_ioremap(&pdev->dev, rfbi_mem->start,
+				 resource_size(rfbi_mem));
+	if (!rfbi.base) {
+		DSSERR("can't ioremap RFBI\n");
+		return -ENOMEM;
+	}
+
+	clk = clk_get(&pdev->dev, "ick");
+	if (IS_ERR(clk)) {
+		DSSERR("can't get ick\n");
+		return PTR_ERR(clk);
+	}
+
+	rfbi.l4_khz = clk_get_rate(clk) / 1000;
+
+	clk_put(clk);
+
+	pm_runtime_enable(&pdev->dev);
+
+	r = rfbi_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	msleep(10);
+
+	rev = rfbi_read_reg(RFBI_REVISION);
+	dev_dbg(&pdev->dev, "OMAP RFBI rev %d.%d\n",
+	       FLD_GET(rev, 7, 4), FLD_GET(rev, 3, 0));
+
+	rfbi_runtime_put();
+
+	dss_debugfs_create_file("rfbi", rfbi_dump_regs);
+
+	rfbi_init_output(pdev);
+
+	return 0;
+
+err_runtime_get:
+	pm_runtime_disable(&pdev->dev);
+	return r;
+}
+
+static void rfbi_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	rfbi_uninit_output(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct component_ops rfbi_component_ops = {
+	.bind	= rfbi_bind,
+	.unbind	= rfbi_unbind,
+};
+
+static int rfbi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &rfbi_component_ops);
+}
+
+static int rfbi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &rfbi_component_ops);
+	return 0;
+}
+
+static int rfbi_runtime_suspend(struct device *dev)
+{
+	dispc_runtime_put();
+
+	return 0;
+}
+
+static int rfbi_runtime_resume(struct device *dev)
+{
+	int r;
+
+	r = dispc_runtime_get();
+	if (r < 0)
+		return r;
+
+	return 0;
+}
+
+static const struct dev_pm_ops rfbi_pm_ops = {
+	.runtime_suspend = rfbi_runtime_suspend,
+	.runtime_resume = rfbi_runtime_resume,
+};
+
+static struct platform_driver omap_rfbihw_driver = {
+	.probe		= rfbi_probe,
+	.remove         = rfbi_remove,
+	.driver         = {
+		.name   = "omapdss_rfbi",
+		.pm	= &rfbi_pm_ops,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init rfbi_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_rfbihw_driver);
+}
+
+void rfbi_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_rfbihw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/sdi.c b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
new file mode 100644
index 000000000000..d747cc6b59e1
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
@@ -0,0 +1,454 @@
+/*
+ * linux/drivers/video/omap2/dss/sdi.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "SDI"
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/regulator/consumer.h>
+#include <linux/export.h>
+#include <linux/platform_device.h>
+#include <linux/string.h>
+#include <linux/of.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+#include "dss.h"
+
+static struct {
+	struct platform_device *pdev;
+
+	bool update_enabled;
+	struct regulator *vdds_sdi_reg;
+
+	struct dss_lcd_mgr_config mgr_config;
+	struct omap_video_timings timings;
+	int datapairs;
+
+	struct omap_dss_device output;
+
+	bool port_initialized;
+} sdi;
+
+struct sdi_clk_calc_ctx {
+	unsigned long pck_min, pck_max;
+
+	unsigned long fck;
+	struct dispc_clock_info dispc_cinfo;
+};
+
+static bool dpi_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
+		unsigned long pck, void *data)
+{
+	struct sdi_clk_calc_ctx *ctx = data;
+
+	ctx->dispc_cinfo.lck_div = lckd;
+	ctx->dispc_cinfo.pck_div = pckd;
+	ctx->dispc_cinfo.lck = lck;
+	ctx->dispc_cinfo.pck = pck;
+
+	return true;
+}
+
+static bool dpi_calc_dss_cb(unsigned long fck, void *data)
+{
+	struct sdi_clk_calc_ctx *ctx = data;
+
+	ctx->fck = fck;
+
+	return dispc_div_calc(fck, ctx->pck_min, ctx->pck_max,
+			dpi_calc_dispc_cb, ctx);
+}
+
+static int sdi_calc_clock_div(unsigned long pclk,
+		unsigned long *fck,
+		struct dispc_clock_info *dispc_cinfo)
+{
+	int i;
+	struct sdi_clk_calc_ctx ctx;
+
+	/*
+	 * DSS fclk gives us very few possibilities, so finding a good pixel
+	 * clock may not be possible. We try multiple times to find the clock,
+	 * each time widening the pixel clock range we look for, up to
+	 * +/- 1MHz.
+	 */
+
+	for (i = 0; i < 10; ++i) {
+		bool ok;
+
+		memset(&ctx, 0, sizeof(ctx));
+		if (pclk > 1000 * i * i * i)
+			ctx.pck_min = max(pclk - 1000 * i * i * i, 0lu);
+		else
+			ctx.pck_min = 0;
+		ctx.pck_max = pclk + 1000 * i * i * i;
+
+		ok = dss_div_calc(pclk, ctx.pck_min, dpi_calc_dss_cb, &ctx);
+		if (ok) {
+			*fck = ctx.fck;
+			*dispc_cinfo = ctx.dispc_cinfo;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static void sdi_config_lcd_manager(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = sdi.output.manager;
+
+	sdi.mgr_config.io_pad_mode = DSS_IO_PAD_MODE_BYPASS;
+
+	sdi.mgr_config.stallmode = false;
+	sdi.mgr_config.fifohandcheck = false;
+
+	sdi.mgr_config.video_port_width = 24;
+	sdi.mgr_config.lcden_sig_polarity = 1;
+
+	dss_mgr_set_lcd_config(mgr, &sdi.mgr_config);
+}
+
+static int sdi_display_enable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &sdi.output;
+	struct omap_video_timings *t = &sdi.timings;
+	unsigned long fck;
+	struct dispc_clock_info dispc_cinfo;
+	unsigned long pck;
+	int r;
+
+	if (out->manager == NULL) {
+		DSSERR("failed to enable display: no output/manager\n");
+		return -ENODEV;
+	}
+
+	r = regulator_enable(sdi.vdds_sdi_reg);
+	if (r)
+		goto err_reg_enable;
+
+	r = dispc_runtime_get();
+	if (r)
+		goto err_get_dispc;
+
+	/* 15.5.9.1.2 */
+	t->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	t->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+
+	r = sdi_calc_clock_div(t->pixelclock, &fck, &dispc_cinfo);
+	if (r)
+		goto err_calc_clock_div;
+
+	sdi.mgr_config.clock_info = dispc_cinfo;
+
+	pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div;
+
+	if (pck != t->pixelclock) {
+		DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n",
+			t->pixelclock, pck);
+
+		t->pixelclock = pck;
+	}
+
+
+	dss_mgr_set_timings(out->manager, t);
+
+	r = dss_set_fck_rate(fck);
+	if (r)
+		goto err_set_dss_clock_div;
+
+	sdi_config_lcd_manager(dssdev);
+
+	/*
+	 * LCLK and PCLK divisors are located in shadow registers, and we
+	 * normally write them to DISPC registers when enabling the output.
+	 * However, SDI uses pck-free as source clock for its PLL, and pck-free
+	 * is affected by the divisors. And as we need the PLL before enabling
+	 * the output, we need to write the divisors early.
+	 *
+	 * It seems just writing to the DISPC register is enough, and we don't
+	 * need to care about the shadow register mechanism for pck-free. The
+	 * exact reason for this is unknown.
+	 */
+	dispc_mgr_set_clock_div(out->manager->id, &sdi.mgr_config.clock_info);
+
+	dss_sdi_init(sdi.datapairs);
+	r = dss_sdi_enable();
+	if (r)
+		goto err_sdi_enable;
+	mdelay(2);
+
+	r = dss_mgr_enable(out->manager);
+	if (r)
+		goto err_mgr_enable;
+
+	return 0;
+
+err_mgr_enable:
+	dss_sdi_disable();
+err_sdi_enable:
+err_set_dss_clock_div:
+err_calc_clock_div:
+	dispc_runtime_put();
+err_get_dispc:
+	regulator_disable(sdi.vdds_sdi_reg);
+err_reg_enable:
+	return r;
+}
+
+static void sdi_display_disable(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = sdi.output.manager;
+
+	dss_mgr_disable(mgr);
+
+	dss_sdi_disable();
+
+	dispc_runtime_put();
+
+	regulator_disable(sdi.vdds_sdi_reg);
+}
+
+static void sdi_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	sdi.timings = *timings;
+}
+
+static void sdi_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	*timings = sdi.timings;
+}
+
+static int sdi_check_timings(struct omap_dss_device *dssdev,
+			struct omap_video_timings *timings)
+{
+	struct omap_overlay_manager *mgr = sdi.output.manager;
+
+	if (mgr && !dispc_mgr_timings_ok(mgr->id, timings))
+		return -EINVAL;
+
+	if (timings->pixelclock == 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void sdi_set_datapairs(struct omap_dss_device *dssdev, int datapairs)
+{
+	sdi.datapairs = datapairs;
+}
+
+static int sdi_init_regulator(void)
+{
+	struct regulator *vdds_sdi;
+
+	if (sdi.vdds_sdi_reg)
+		return 0;
+
+	vdds_sdi = devm_regulator_get(&sdi.pdev->dev, "vdds_sdi");
+	if (IS_ERR(vdds_sdi)) {
+		if (PTR_ERR(vdds_sdi) != -EPROBE_DEFER)
+			DSSERR("can't get VDDS_SDI regulator\n");
+		return PTR_ERR(vdds_sdi);
+	}
+
+	sdi.vdds_sdi_reg = vdds_sdi;
+
+	return 0;
+}
+
+static int sdi_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = sdi_init_regulator();
+	if (r)
+		return r;
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dst->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void sdi_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static const struct omapdss_sdi_ops sdi_ops = {
+	.connect = sdi_connect,
+	.disconnect = sdi_disconnect,
+
+	.enable = sdi_display_enable,
+	.disable = sdi_display_disable,
+
+	.check_timings = sdi_check_timings,
+	.set_timings = sdi_set_timings,
+	.get_timings = sdi_get_timings,
+
+	.set_datapairs = sdi_set_datapairs,
+};
+
+static void sdi_init_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &sdi.output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_SDI;
+	out->output_type = OMAP_DISPLAY_TYPE_SDI;
+	out->name = "sdi.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_LCD;
+	/* We have SDI only on OMAP3, where it's on port 1 */
+	out->port_num = 1;
+	out->ops.sdi = &sdi_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void sdi_uninit_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &sdi.output;
+
+	omapdss_unregister_output(out);
+}
+
+static int sdi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	sdi.pdev = pdev;
+
+	sdi_init_output(pdev);
+
+	return 0;
+}
+
+static void sdi_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	sdi_uninit_output(pdev);
+}
+
+static const struct component_ops sdi_component_ops = {
+	.bind	= sdi_bind,
+	.unbind	= sdi_unbind,
+};
+
+static int sdi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &sdi_component_ops);
+}
+
+static int sdi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &sdi_component_ops);
+	return 0;
+}
+
+static struct platform_driver omap_sdi_driver = {
+	.probe		= sdi_probe,
+	.remove         = sdi_remove,
+	.driver         = {
+		.name   = "omapdss_sdi",
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init sdi_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_sdi_driver);
+}
+
+void sdi_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_sdi_driver);
+}
+
+int sdi_init_port(struct platform_device *pdev, struct device_node *port)
+{
+	struct device_node *ep;
+	u32 datapairs;
+	int r;
+
+	ep = omapdss_of_get_next_endpoint(port, NULL);
+	if (!ep)
+		return 0;
+
+	r = of_property_read_u32(ep, "datapairs", &datapairs);
+	if (r) {
+		DSSERR("failed to parse datapairs\n");
+		goto err_datapairs;
+	}
+
+	sdi.datapairs = datapairs;
+
+	of_node_put(ep);
+
+	sdi.pdev = pdev;
+
+	sdi_init_output(pdev);
+
+	sdi.port_initialized = true;
+
+	return 0;
+
+err_datapairs:
+	of_node_put(ep);
+
+	return r;
+}
+
+void sdi_uninit_port(struct device_node *port)
+{
+	if (!sdi.port_initialized)
+		return;
+
+	sdi_uninit_output(sdi.pdev);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/venc.c b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
new file mode 100644
index 000000000000..26e0ee30adf8
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
@@ -0,0 +1,997 @@
+/*
+ * linux/drivers/video/omap2/dss/venc.c
+ *
+ * Copyright (C) 2009 Nokia Corporation
+ * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
+ *
+ * VENC settings from TI's DSS driver
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define DSS_SUBSYS_NAME "VENC"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/mutex.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/seq_file.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/component.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+/* Venc registers */
+#define VENC_REV_ID				0x00
+#define VENC_STATUS				0x04
+#define VENC_F_CONTROL				0x08
+#define VENC_VIDOUT_CTRL			0x10
+#define VENC_SYNC_CTRL				0x14
+#define VENC_LLEN				0x1C
+#define VENC_FLENS				0x20
+#define VENC_HFLTR_CTRL				0x24
+#define VENC_CC_CARR_WSS_CARR			0x28
+#define VENC_C_PHASE				0x2C
+#define VENC_GAIN_U				0x30
+#define VENC_GAIN_V				0x34
+#define VENC_GAIN_Y				0x38
+#define VENC_BLACK_LEVEL			0x3C
+#define VENC_BLANK_LEVEL			0x40
+#define VENC_X_COLOR				0x44
+#define VENC_M_CONTROL				0x48
+#define VENC_BSTAMP_WSS_DATA			0x4C
+#define VENC_S_CARR				0x50
+#define VENC_LINE21				0x54
+#define VENC_LN_SEL				0x58
+#define VENC_L21__WC_CTL			0x5C
+#define VENC_HTRIGGER_VTRIGGER			0x60
+#define VENC_SAVID__EAVID			0x64
+#define VENC_FLEN__FAL				0x68
+#define VENC_LAL__PHASE_RESET			0x6C
+#define VENC_HS_INT_START_STOP_X		0x70
+#define VENC_HS_EXT_START_STOP_X		0x74
+#define VENC_VS_INT_START_X			0x78
+#define VENC_VS_INT_STOP_X__VS_INT_START_Y	0x7C
+#define VENC_VS_INT_STOP_Y__VS_EXT_START_X	0x80
+#define VENC_VS_EXT_STOP_X__VS_EXT_START_Y	0x84
+#define VENC_VS_EXT_STOP_Y			0x88
+#define VENC_AVID_START_STOP_X			0x90
+#define VENC_AVID_START_STOP_Y			0x94
+#define VENC_FID_INT_START_X__FID_INT_START_Y	0xA0
+#define VENC_FID_INT_OFFSET_Y__FID_EXT_START_X	0xA4
+#define VENC_FID_EXT_START_Y__FID_EXT_OFFSET_Y	0xA8
+#define VENC_TVDETGP_INT_START_STOP_X		0xB0
+#define VENC_TVDETGP_INT_START_STOP_Y		0xB4
+#define VENC_GEN_CTRL				0xB8
+#define VENC_OUTPUT_CONTROL			0xC4
+#define VENC_OUTPUT_TEST			0xC8
+#define VENC_DAC_B__DAC_C			0xC8
+
+struct venc_config {
+	u32 f_control;
+	u32 vidout_ctrl;
+	u32 sync_ctrl;
+	u32 llen;
+	u32 flens;
+	u32 hfltr_ctrl;
+	u32 cc_carr_wss_carr;
+	u32 c_phase;
+	u32 gain_u;
+	u32 gain_v;
+	u32 gain_y;
+	u32 black_level;
+	u32 blank_level;
+	u32 x_color;
+	u32 m_control;
+	u32 bstamp_wss_data;
+	u32 s_carr;
+	u32 line21;
+	u32 ln_sel;
+	u32 l21__wc_ctl;
+	u32 htrigger_vtrigger;
+	u32 savid__eavid;
+	u32 flen__fal;
+	u32 lal__phase_reset;
+	u32 hs_int_start_stop_x;
+	u32 hs_ext_start_stop_x;
+	u32 vs_int_start_x;
+	u32 vs_int_stop_x__vs_int_start_y;
+	u32 vs_int_stop_y__vs_ext_start_x;
+	u32 vs_ext_stop_x__vs_ext_start_y;
+	u32 vs_ext_stop_y;
+	u32 avid_start_stop_x;
+	u32 avid_start_stop_y;
+	u32 fid_int_start_x__fid_int_start_y;
+	u32 fid_int_offset_y__fid_ext_start_x;
+	u32 fid_ext_start_y__fid_ext_offset_y;
+	u32 tvdetgp_int_start_stop_x;
+	u32 tvdetgp_int_start_stop_y;
+	u32 gen_ctrl;
+};
+
+/* from TRM */
+static const struct venc_config venc_config_pal_trm = {
+	.f_control				= 0,
+	.vidout_ctrl				= 1,
+	.sync_ctrl				= 0x40,
+	.llen					= 0x35F, /* 863 */
+	.flens					= 0x270, /* 624 */
+	.hfltr_ctrl				= 0,
+	.cc_carr_wss_carr			= 0x2F7225ED,
+	.c_phase				= 0,
+	.gain_u					= 0x111,
+	.gain_v					= 0x181,
+	.gain_y					= 0x140,
+	.black_level				= 0x3B,
+	.blank_level				= 0x3B,
+	.x_color				= 0x7,
+	.m_control				= 0x2,
+	.bstamp_wss_data			= 0x3F,
+	.s_carr					= 0x2A098ACB,
+	.line21					= 0,
+	.ln_sel					= 0x01290015,
+	.l21__wc_ctl				= 0x0000F603,
+	.htrigger_vtrigger			= 0,
+
+	.savid__eavid				= 0x06A70108,
+	.flen__fal				= 0x00180270,
+	.lal__phase_reset			= 0x00040135,
+	.hs_int_start_stop_x			= 0x00880358,
+	.hs_ext_start_stop_x			= 0x000F035F,
+	.vs_int_start_x				= 0x01A70000,
+	.vs_int_stop_x__vs_int_start_y		= 0x000001A7,
+	.vs_int_stop_y__vs_ext_start_x		= 0x01AF0000,
+	.vs_ext_stop_x__vs_ext_start_y		= 0x000101AF,
+	.vs_ext_stop_y				= 0x00000025,
+	.avid_start_stop_x			= 0x03530083,
+	.avid_start_stop_y			= 0x026C002E,
+	.fid_int_start_x__fid_int_start_y	= 0x0001008A,
+	.fid_int_offset_y__fid_ext_start_x	= 0x002E0138,
+	.fid_ext_start_y__fid_ext_offset_y	= 0x01380001,
+
+	.tvdetgp_int_start_stop_x		= 0x00140001,
+	.tvdetgp_int_start_stop_y		= 0x00010001,
+	.gen_ctrl				= 0x00FF0000,
+};
+
+/* from TRM */
+static const struct venc_config venc_config_ntsc_trm = {
+	.f_control				= 0,
+	.vidout_ctrl				= 1,
+	.sync_ctrl				= 0x8040,
+	.llen					= 0x359,
+	.flens					= 0x20C,
+	.hfltr_ctrl				= 0,
+	.cc_carr_wss_carr			= 0x043F2631,
+	.c_phase				= 0,
+	.gain_u					= 0x102,
+	.gain_v					= 0x16C,
+	.gain_y					= 0x12F,
+	.black_level				= 0x43,
+	.blank_level				= 0x38,
+	.x_color				= 0x7,
+	.m_control				= 0x1,
+	.bstamp_wss_data			= 0x38,
+	.s_carr					= 0x21F07C1F,
+	.line21					= 0,
+	.ln_sel					= 0x01310011,
+	.l21__wc_ctl				= 0x0000F003,
+	.htrigger_vtrigger			= 0,
+
+	.savid__eavid				= 0x069300F4,
+	.flen__fal				= 0x0016020C,
+	.lal__phase_reset			= 0x00060107,
+	.hs_int_start_stop_x			= 0x008E0350,
+	.hs_ext_start_stop_x			= 0x000F0359,
+	.vs_int_start_x				= 0x01A00000,
+	.vs_int_stop_x__vs_int_start_y		= 0x020701A0,
+	.vs_int_stop_y__vs_ext_start_x		= 0x01AC0024,
+	.vs_ext_stop_x__vs_ext_start_y		= 0x020D01AC,
+	.vs_ext_stop_y				= 0x00000006,
+	.avid_start_stop_x			= 0x03480078,
+	.avid_start_stop_y			= 0x02060024,
+	.fid_int_start_x__fid_int_start_y	= 0x0001008A,
+	.fid_int_offset_y__fid_ext_start_x	= 0x01AC0106,
+	.fid_ext_start_y__fid_ext_offset_y	= 0x01060006,
+
+	.tvdetgp_int_start_stop_x		= 0x00140001,
+	.tvdetgp_int_start_stop_y		= 0x00010001,
+	.gen_ctrl				= 0x00F90000,
+};
+
+static const struct venc_config venc_config_pal_bdghi = {
+	.f_control				= 0,
+	.vidout_ctrl				= 0,
+	.sync_ctrl				= 0,
+	.hfltr_ctrl				= 0,
+	.x_color				= 0,
+	.line21					= 0,
+	.ln_sel					= 21,
+	.htrigger_vtrigger			= 0,
+	.tvdetgp_int_start_stop_x		= 0x00140001,
+	.tvdetgp_int_start_stop_y		= 0x00010001,
+	.gen_ctrl				= 0x00FB0000,
+
+	.llen					= 864-1,
+	.flens					= 625-1,
+	.cc_carr_wss_carr			= 0x2F7625ED,
+	.c_phase				= 0xDF,
+	.gain_u					= 0x111,
+	.gain_v					= 0x181,
+	.gain_y					= 0x140,
+	.black_level				= 0x3e,
+	.blank_level				= 0x3e,
+	.m_control				= 0<<2 | 1<<1,
+	.bstamp_wss_data			= 0x42,
+	.s_carr					= 0x2a098acb,
+	.l21__wc_ctl				= 0<<13 | 0x16<<8 | 0<<0,
+	.savid__eavid				= 0x06A70108,
+	.flen__fal				= 23<<16 | 624<<0,
+	.lal__phase_reset			= 2<<17 | 310<<0,
+	.hs_int_start_stop_x			= 0x00920358,
+	.hs_ext_start_stop_x			= 0x000F035F,
+	.vs_int_start_x				= 0x1a7<<16,
+	.vs_int_stop_x__vs_int_start_y		= 0x000601A7,
+	.vs_int_stop_y__vs_ext_start_x		= 0x01AF0036,
+	.vs_ext_stop_x__vs_ext_start_y		= 0x27101af,
+	.vs_ext_stop_y				= 0x05,
+	.avid_start_stop_x			= 0x03530082,
+	.avid_start_stop_y			= 0x0270002E,
+	.fid_int_start_x__fid_int_start_y	= 0x0005008A,
+	.fid_int_offset_y__fid_ext_start_x	= 0x002E0138,
+	.fid_ext_start_y__fid_ext_offset_y	= 0x01380005,
+};
+
+const struct omap_video_timings omap_dss_pal_timings = {
+	.x_res		= 720,
+	.y_res		= 574,
+	.pixelclock	= 13500000,
+	.hsw		= 64,
+	.hfp		= 12,
+	.hbp		= 68,
+	.vsw		= 5,
+	.vfp		= 5,
+	.vbp		= 41,
+
+	.interlace	= true,
+};
+EXPORT_SYMBOL(omap_dss_pal_timings);
+
+const struct omap_video_timings omap_dss_ntsc_timings = {
+	.x_res		= 720,
+	.y_res		= 482,
+	.pixelclock	= 13500000,
+	.hsw		= 64,
+	.hfp		= 16,
+	.hbp		= 58,
+	.vsw		= 6,
+	.vfp		= 6,
+	.vbp		= 31,
+
+	.interlace	= true,
+};
+EXPORT_SYMBOL(omap_dss_ntsc_timings);
+
+static struct {
+	struct platform_device *pdev;
+	void __iomem *base;
+	struct mutex venc_lock;
+	u32 wss_data;
+	struct regulator *vdda_dac_reg;
+
+	struct clk	*tv_dac_clk;
+
+	struct omap_video_timings timings;
+	enum omap_dss_venc_type type;
+	bool invert_polarity;
+
+	struct omap_dss_device output;
+} venc;
+
+static inline void venc_write_reg(int idx, u32 val)
+{
+	__raw_writel(val, venc.base + idx);
+}
+
+static inline u32 venc_read_reg(int idx)
+{
+	u32 l = __raw_readl(venc.base + idx);
+	return l;
+}
+
+static void venc_write_config(const struct venc_config *config)
+{
+	DSSDBG("write venc conf\n");
+
+	venc_write_reg(VENC_LLEN, config->llen);
+	venc_write_reg(VENC_FLENS, config->flens);
+	venc_write_reg(VENC_CC_CARR_WSS_CARR, config->cc_carr_wss_carr);
+	venc_write_reg(VENC_C_PHASE, config->c_phase);
+	venc_write_reg(VENC_GAIN_U, config->gain_u);
+	venc_write_reg(VENC_GAIN_V, config->gain_v);
+	venc_write_reg(VENC_GAIN_Y, config->gain_y);
+	venc_write_reg(VENC_BLACK_LEVEL, config->black_level);
+	venc_write_reg(VENC_BLANK_LEVEL, config->blank_level);
+	venc_write_reg(VENC_M_CONTROL, config->m_control);
+	venc_write_reg(VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data |
+			venc.wss_data);
+	venc_write_reg(VENC_S_CARR, config->s_carr);
+	venc_write_reg(VENC_L21__WC_CTL, config->l21__wc_ctl);
+	venc_write_reg(VENC_SAVID__EAVID, config->savid__eavid);
+	venc_write_reg(VENC_FLEN__FAL, config->flen__fal);
+	venc_write_reg(VENC_LAL__PHASE_RESET, config->lal__phase_reset);
+	venc_write_reg(VENC_HS_INT_START_STOP_X, config->hs_int_start_stop_x);
+	venc_write_reg(VENC_HS_EXT_START_STOP_X, config->hs_ext_start_stop_x);
+	venc_write_reg(VENC_VS_INT_START_X, config->vs_int_start_x);
+	venc_write_reg(VENC_VS_INT_STOP_X__VS_INT_START_Y,
+		       config->vs_int_stop_x__vs_int_start_y);
+	venc_write_reg(VENC_VS_INT_STOP_Y__VS_EXT_START_X,
+		       config->vs_int_stop_y__vs_ext_start_x);
+	venc_write_reg(VENC_VS_EXT_STOP_X__VS_EXT_START_Y,
+		       config->vs_ext_stop_x__vs_ext_start_y);
+	venc_write_reg(VENC_VS_EXT_STOP_Y, config->vs_ext_stop_y);
+	venc_write_reg(VENC_AVID_START_STOP_X, config->avid_start_stop_x);
+	venc_write_reg(VENC_AVID_START_STOP_Y, config->avid_start_stop_y);
+	venc_write_reg(VENC_FID_INT_START_X__FID_INT_START_Y,
+		       config->fid_int_start_x__fid_int_start_y);
+	venc_write_reg(VENC_FID_INT_OFFSET_Y__FID_EXT_START_X,
+		       config->fid_int_offset_y__fid_ext_start_x);
+	venc_write_reg(VENC_FID_EXT_START_Y__FID_EXT_OFFSET_Y,
+		       config->fid_ext_start_y__fid_ext_offset_y);
+
+	venc_write_reg(VENC_DAC_B__DAC_C,  venc_read_reg(VENC_DAC_B__DAC_C));
+	venc_write_reg(VENC_VIDOUT_CTRL, config->vidout_ctrl);
+	venc_write_reg(VENC_HFLTR_CTRL, config->hfltr_ctrl);
+	venc_write_reg(VENC_X_COLOR, config->x_color);
+	venc_write_reg(VENC_LINE21, config->line21);
+	venc_write_reg(VENC_LN_SEL, config->ln_sel);
+	venc_write_reg(VENC_HTRIGGER_VTRIGGER, config->htrigger_vtrigger);
+	venc_write_reg(VENC_TVDETGP_INT_START_STOP_X,
+		       config->tvdetgp_int_start_stop_x);
+	venc_write_reg(VENC_TVDETGP_INT_START_STOP_Y,
+		       config->tvdetgp_int_start_stop_y);
+	venc_write_reg(VENC_GEN_CTRL, config->gen_ctrl);
+	venc_write_reg(VENC_F_CONTROL, config->f_control);
+	venc_write_reg(VENC_SYNC_CTRL, config->sync_ctrl);
+}
+
+static void venc_reset(void)
+{
+	int t = 1000;
+
+	venc_write_reg(VENC_F_CONTROL, 1<<8);
+	while (venc_read_reg(VENC_F_CONTROL) & (1<<8)) {
+		if (--t == 0) {
+			DSSERR("Failed to reset venc\n");
+			return;
+		}
+	}
+
+#ifdef CONFIG_FB_OMAP2_DSS_SLEEP_AFTER_VENC_RESET
+	/* the magical sleep that makes things work */
+	/* XXX more info? What bug this circumvents? */
+	msleep(20);
+#endif
+}
+
+static int venc_runtime_get(void)
+{
+	int r;
+
+	DSSDBG("venc_runtime_get\n");
+
+	r = pm_runtime_get_sync(&venc.pdev->dev);
+	WARN_ON(r < 0);
+	return r < 0 ? r : 0;
+}
+
+static void venc_runtime_put(void)
+{
+	int r;
+
+	DSSDBG("venc_runtime_put\n");
+
+	r = pm_runtime_put_sync(&venc.pdev->dev);
+	WARN_ON(r < 0 && r != -ENOSYS);
+}
+
+static const struct venc_config *venc_timings_to_config(
+		struct omap_video_timings *timings)
+{
+	if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0)
+		return &venc_config_pal_trm;
+
+	if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0)
+		return &venc_config_ntsc_trm;
+
+	BUG();
+	return NULL;
+}
+
+static int venc_power_on(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = venc.output.manager;
+	u32 l;
+	int r;
+
+	r = venc_runtime_get();
+	if (r)
+		goto err0;
+
+	venc_reset();
+	venc_write_config(venc_timings_to_config(&venc.timings));
+
+	dss_set_venc_output(venc.type);
+	dss_set_dac_pwrdn_bgz(1);
+
+	l = 0;
+
+	if (venc.type == OMAP_DSS_VENC_TYPE_COMPOSITE)
+		l |= 1 << 1;
+	else /* S-Video */
+		l |= (1 << 0) | (1 << 2);
+
+	if (venc.invert_polarity == false)
+		l |= 1 << 3;
+
+	venc_write_reg(VENC_OUTPUT_CONTROL, l);
+
+	dss_mgr_set_timings(mgr, &venc.timings);
+
+	r = regulator_enable(venc.vdda_dac_reg);
+	if (r)
+		goto err1;
+
+	r = dss_mgr_enable(mgr);
+	if (r)
+		goto err2;
+
+	return 0;
+
+err2:
+	regulator_disable(venc.vdda_dac_reg);
+err1:
+	venc_write_reg(VENC_OUTPUT_CONTROL, 0);
+	dss_set_dac_pwrdn_bgz(0);
+
+	venc_runtime_put();
+err0:
+	return r;
+}
+
+static void venc_power_off(struct omap_dss_device *dssdev)
+{
+	struct omap_overlay_manager *mgr = venc.output.manager;
+
+	venc_write_reg(VENC_OUTPUT_CONTROL, 0);
+	dss_set_dac_pwrdn_bgz(0);
+
+	dss_mgr_disable(mgr);
+
+	regulator_disable(venc.vdda_dac_reg);
+
+	venc_runtime_put();
+}
+
+static int venc_display_enable(struct omap_dss_device *dssdev)
+{
+	struct omap_dss_device *out = &venc.output;
+	int r;
+
+	DSSDBG("venc_display_enable\n");
+
+	mutex_lock(&venc.venc_lock);
+
+	if (out->manager == NULL) {
+		DSSERR("Failed to enable display: no output/manager\n");
+		r = -ENODEV;
+		goto err0;
+	}
+
+	r = venc_power_on(dssdev);
+	if (r)
+		goto err0;
+
+	venc.wss_data = 0;
+
+	mutex_unlock(&venc.venc_lock);
+
+	return 0;
+err0:
+	mutex_unlock(&venc.venc_lock);
+	return r;
+}
+
+static void venc_display_disable(struct omap_dss_device *dssdev)
+{
+	DSSDBG("venc_display_disable\n");
+
+	mutex_lock(&venc.venc_lock);
+
+	venc_power_off(dssdev);
+
+	mutex_unlock(&venc.venc_lock);
+}
+
+static void venc_set_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	DSSDBG("venc_set_timings\n");
+
+	mutex_lock(&venc.venc_lock);
+
+	/* Reset WSS data when the TV standard changes. */
+	if (memcmp(&venc.timings, timings, sizeof(*timings)))
+		venc.wss_data = 0;
+
+	venc.timings = *timings;
+
+	dispc_set_tv_pclk(13500000);
+
+	mutex_unlock(&venc.venc_lock);
+}
+
+static int venc_check_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	DSSDBG("venc_check_timings\n");
+
+	if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0)
+		return 0;
+
+	if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0)
+		return 0;
+
+	return -EINVAL;
+}
+
+static void venc_get_timings(struct omap_dss_device *dssdev,
+		struct omap_video_timings *timings)
+{
+	mutex_lock(&venc.venc_lock);
+
+	*timings = venc.timings;
+
+	mutex_unlock(&venc.venc_lock);
+}
+
+static u32 venc_get_wss(struct omap_dss_device *dssdev)
+{
+	/* Invert due to VENC_L21_WC_CTL:INV=1 */
+	return (venc.wss_data >> 8) ^ 0xfffff;
+}
+
+static int venc_set_wss(struct omap_dss_device *dssdev, u32 wss)
+{
+	const struct venc_config *config;
+	int r;
+
+	DSSDBG("venc_set_wss\n");
+
+	mutex_lock(&venc.venc_lock);
+
+	config = venc_timings_to_config(&venc.timings);
+
+	/* Invert due to VENC_L21_WC_CTL:INV=1 */
+	venc.wss_data = (wss ^ 0xfffff) << 8;
+
+	r = venc_runtime_get();
+	if (r)
+		goto err;
+
+	venc_write_reg(VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data |
+			venc.wss_data);
+
+	venc_runtime_put();
+
+err:
+	mutex_unlock(&venc.venc_lock);
+
+	return r;
+}
+
+static void venc_set_type(struct omap_dss_device *dssdev,
+		enum omap_dss_venc_type type)
+{
+	mutex_lock(&venc.venc_lock);
+
+	venc.type = type;
+
+	mutex_unlock(&venc.venc_lock);
+}
+
+static void venc_invert_vid_out_polarity(struct omap_dss_device *dssdev,
+		bool invert_polarity)
+{
+	mutex_lock(&venc.venc_lock);
+
+	venc.invert_polarity = invert_polarity;
+
+	mutex_unlock(&venc.venc_lock);
+}
+
+static int venc_init_regulator(void)
+{
+	struct regulator *vdda_dac;
+
+	if (venc.vdda_dac_reg != NULL)
+		return 0;
+
+	if (venc.pdev->dev.of_node)
+		vdda_dac = devm_regulator_get(&venc.pdev->dev, "vdda");
+	else
+		vdda_dac = devm_regulator_get(&venc.pdev->dev, "vdda_dac");
+
+	if (IS_ERR(vdda_dac)) {
+		if (PTR_ERR(vdda_dac) != -EPROBE_DEFER)
+			DSSERR("can't get VDDA_DAC regulator\n");
+		return PTR_ERR(vdda_dac);
+	}
+
+	venc.vdda_dac_reg = vdda_dac;
+
+	return 0;
+}
+
+static void venc_dump_regs(struct seq_file *s)
+{
+#define DUMPREG(r) seq_printf(s, "%-35s %08x\n", #r, venc_read_reg(r))
+
+	if (venc_runtime_get())
+		return;
+
+	DUMPREG(VENC_F_CONTROL);
+	DUMPREG(VENC_VIDOUT_CTRL);
+	DUMPREG(VENC_SYNC_CTRL);
+	DUMPREG(VENC_LLEN);
+	DUMPREG(VENC_FLENS);
+	DUMPREG(VENC_HFLTR_CTRL);
+	DUMPREG(VENC_CC_CARR_WSS_CARR);
+	DUMPREG(VENC_C_PHASE);
+	DUMPREG(VENC_GAIN_U);
+	DUMPREG(VENC_GAIN_V);
+	DUMPREG(VENC_GAIN_Y);
+	DUMPREG(VENC_BLACK_LEVEL);
+	DUMPREG(VENC_BLANK_LEVEL);
+	DUMPREG(VENC_X_COLOR);
+	DUMPREG(VENC_M_CONTROL);
+	DUMPREG(VENC_BSTAMP_WSS_DATA);
+	DUMPREG(VENC_S_CARR);
+	DUMPREG(VENC_LINE21);
+	DUMPREG(VENC_LN_SEL);
+	DUMPREG(VENC_L21__WC_CTL);
+	DUMPREG(VENC_HTRIGGER_VTRIGGER);
+	DUMPREG(VENC_SAVID__EAVID);
+	DUMPREG(VENC_FLEN__FAL);
+	DUMPREG(VENC_LAL__PHASE_RESET);
+	DUMPREG(VENC_HS_INT_START_STOP_X);
+	DUMPREG(VENC_HS_EXT_START_STOP_X);
+	DUMPREG(VENC_VS_INT_START_X);
+	DUMPREG(VENC_VS_INT_STOP_X__VS_INT_START_Y);
+	DUMPREG(VENC_VS_INT_STOP_Y__VS_EXT_START_X);
+	DUMPREG(VENC_VS_EXT_STOP_X__VS_EXT_START_Y);
+	DUMPREG(VENC_VS_EXT_STOP_Y);
+	DUMPREG(VENC_AVID_START_STOP_X);
+	DUMPREG(VENC_AVID_START_STOP_Y);
+	DUMPREG(VENC_FID_INT_START_X__FID_INT_START_Y);
+	DUMPREG(VENC_FID_INT_OFFSET_Y__FID_EXT_START_X);
+	DUMPREG(VENC_FID_EXT_START_Y__FID_EXT_OFFSET_Y);
+	DUMPREG(VENC_TVDETGP_INT_START_STOP_X);
+	DUMPREG(VENC_TVDETGP_INT_START_STOP_Y);
+	DUMPREG(VENC_GEN_CTRL);
+	DUMPREG(VENC_OUTPUT_CONTROL);
+	DUMPREG(VENC_OUTPUT_TEST);
+
+	venc_runtime_put();
+
+#undef DUMPREG
+}
+
+static int venc_get_clocks(struct platform_device *pdev)
+{
+	struct clk *clk;
+
+	if (dss_has_feature(FEAT_VENC_REQUIRES_TV_DAC_CLK)) {
+		clk = devm_clk_get(&pdev->dev, "tv_dac_clk");
+		if (IS_ERR(clk)) {
+			DSSERR("can't get tv_dac_clk\n");
+			return PTR_ERR(clk);
+		}
+	} else {
+		clk = NULL;
+	}
+
+	venc.tv_dac_clk = clk;
+
+	return 0;
+}
+
+static int venc_connect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	struct omap_overlay_manager *mgr;
+	int r;
+
+	r = venc_init_regulator();
+	if (r)
+		return r;
+
+	mgr = omap_dss_get_overlay_manager(dssdev->dispc_channel);
+	if (!mgr)
+		return -ENODEV;
+
+	r = dss_mgr_connect(mgr, dssdev);
+	if (r)
+		return r;
+
+	r = omapdss_output_set_device(dssdev, dst);
+	if (r) {
+		DSSERR("failed to connect output to new device: %s\n",
+				dst->name);
+		dss_mgr_disconnect(mgr, dssdev);
+		return r;
+	}
+
+	return 0;
+}
+
+static void venc_disconnect(struct omap_dss_device *dssdev,
+		struct omap_dss_device *dst)
+{
+	WARN_ON(dst != dssdev->dst);
+
+	if (dst != dssdev->dst)
+		return;
+
+	omapdss_output_unset_device(dssdev);
+
+	if (dssdev->manager)
+		dss_mgr_disconnect(dssdev->manager, dssdev);
+}
+
+static const struct omapdss_atv_ops venc_ops = {
+	.connect = venc_connect,
+	.disconnect = venc_disconnect,
+
+	.enable = venc_display_enable,
+	.disable = venc_display_disable,
+
+	.check_timings = venc_check_timings,
+	.set_timings = venc_set_timings,
+	.get_timings = venc_get_timings,
+
+	.set_type = venc_set_type,
+	.invert_vid_out_polarity = venc_invert_vid_out_polarity,
+
+	.set_wss = venc_set_wss,
+	.get_wss = venc_get_wss,
+};
+
+static void venc_init_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &venc.output;
+
+	out->dev = &pdev->dev;
+	out->id = OMAP_DSS_OUTPUT_VENC;
+	out->output_type = OMAP_DISPLAY_TYPE_VENC;
+	out->name = "venc.0";
+	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
+	out->ops.atv = &venc_ops;
+	out->owner = THIS_MODULE;
+
+	omapdss_register_output(out);
+}
+
+static void venc_uninit_output(struct platform_device *pdev)
+{
+	struct omap_dss_device *out = &venc.output;
+
+	omapdss_unregister_output(out);
+}
+
+static int venc_probe_of(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *ep;
+	u32 channels;
+	int r;
+
+	ep = omapdss_of_get_first_endpoint(node);
+	if (!ep)
+		return 0;
+
+	venc.invert_polarity = of_property_read_bool(ep, "ti,invert-polarity");
+
+	r = of_property_read_u32(ep, "ti,channels", &channels);
+	if (r) {
+		dev_err(&pdev->dev,
+			"failed to read property 'ti,channels': %d\n", r);
+		goto err;
+	}
+
+	switch (channels) {
+	case 1:
+		venc.type = OMAP_DSS_VENC_TYPE_COMPOSITE;
+		break;
+	case 2:
+		venc.type = OMAP_DSS_VENC_TYPE_SVIDEO;
+		break;
+	default:
+		dev_err(&pdev->dev, "bad channel propert '%d'\n", channels);
+		r = -EINVAL;
+		goto err;
+	}
+
+	of_node_put(ep);
+
+	return 0;
+err:
+	of_node_put(ep);
+
+	return 0;
+}
+
+/* VENC HW IP initialisation */
+static int venc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	u8 rev_id;
+	struct resource *venc_mem;
+	int r;
+
+	venc.pdev = pdev;
+
+	mutex_init(&venc.venc_lock);
+
+	venc.wss_data = 0;
+
+	venc_mem = platform_get_resource(venc.pdev, IORESOURCE_MEM, 0);
+	if (!venc_mem) {
+		DSSERR("can't get IORESOURCE_MEM VENC\n");
+		return -EINVAL;
+	}
+
+	venc.base = devm_ioremap(&pdev->dev, venc_mem->start,
+				 resource_size(venc_mem));
+	if (!venc.base) {
+		DSSERR("can't ioremap VENC\n");
+		return -ENOMEM;
+	}
+
+	r = venc_get_clocks(pdev);
+	if (r)
+		return r;
+
+	pm_runtime_enable(&pdev->dev);
+
+	r = venc_runtime_get();
+	if (r)
+		goto err_runtime_get;
+
+	rev_id = (u8)(venc_read_reg(VENC_REV_ID) & 0xff);
+	dev_dbg(&pdev->dev, "OMAP VENC rev %d\n", rev_id);
+
+	venc_runtime_put();
+
+	if (pdev->dev.of_node) {
+		r = venc_probe_of(pdev);
+		if (r) {
+			DSSERR("Invalid DT data\n");
+			goto err_probe_of;
+		}
+	}
+
+	dss_debugfs_create_file("venc", venc_dump_regs);
+
+	venc_init_output(pdev);
+
+	return 0;
+
+err_probe_of:
+err_runtime_get:
+	pm_runtime_disable(&pdev->dev);
+	return r;
+}
+
+static void venc_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	venc_uninit_output(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+}
+
+static const struct component_ops venc_component_ops = {
+	.bind	= venc_bind,
+	.unbind	= venc_unbind,
+};
+
+static int venc_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &venc_component_ops);
+}
+
+static int venc_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &venc_component_ops);
+	return 0;
+}
+
+static int venc_runtime_suspend(struct device *dev)
+{
+	if (venc.tv_dac_clk)
+		clk_disable_unprepare(venc.tv_dac_clk);
+
+	dispc_runtime_put();
+
+	return 0;
+}
+
+static int venc_runtime_resume(struct device *dev)
+{
+	int r;
+
+	r = dispc_runtime_get();
+	if (r < 0)
+		return r;
+
+	if (venc.tv_dac_clk)
+		clk_prepare_enable(venc.tv_dac_clk);
+
+	return 0;
+}
+
+static const struct dev_pm_ops venc_pm_ops = {
+	.runtime_suspend = venc_runtime_suspend,
+	.runtime_resume = venc_runtime_resume,
+};
+
+static const struct of_device_id venc_of_match[] = {
+	{ .compatible = "ti,omap2-venc", },
+	{ .compatible = "ti,omap3-venc", },
+	{ .compatible = "ti,omap4-venc", },
+	{},
+};
+
+static struct platform_driver omap_venchw_driver = {
+	.probe		= venc_probe,
+	.remove		= venc_remove,
+	.driver         = {
+		.name   = "omapdss_venc",
+		.pm	= &venc_pm_ops,
+		.of_match_table = venc_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+int __init venc_init_platform_driver(void)
+{
+	return platform_driver_register(&omap_venchw_driver);
+}
+
+void venc_uninit_platform_driver(void)
+{
+	platform_driver_unregister(&omap_venchw_driver);
+}
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c b/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c
new file mode 100644
index 000000000000..b1ec59e42940
--- /dev/null
+++ b/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c
@@ -0,0 +1,211 @@
+/*
+* Copyright (C) 2014 Texas Instruments Ltd
+*
+* This program is free software; you can redistribute it and/or modify it
+* under the terms of the GNU General Public License version 2 as published by
+* the Free Software Foundation.
+*
+* You should have received a copy of the GNU General Public License along with
+* this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+
+#include <video/omapdss.h>
+
+#include "dss.h"
+#include "dss_features.h"
+
+struct dss_video_pll {
+	struct dss_pll pll;
+
+	struct device *dev;
+
+	void __iomem *clkctrl_base;
+};
+
+#define REG_MOD(reg, val, start, end) \
+	writel_relaxed(FLD_MOD(readl_relaxed(reg), val, start, end), reg)
+
+static void dss_dpll_enable_scp_clk(struct dss_video_pll *vpll)
+{
+	REG_MOD(vpll->clkctrl_base, 1, 14, 14); /* CIO_CLK_ICG */
+}
+
+static void dss_dpll_disable_scp_clk(struct dss_video_pll *vpll)
+{
+	REG_MOD(vpll->clkctrl_base, 0, 14, 14); /* CIO_CLK_ICG */
+}
+
+static void dss_dpll_power_enable(struct dss_video_pll *vpll)
+{
+	REG_MOD(vpll->clkctrl_base, 2, 31, 30); /* PLL_POWER_ON_ALL */
+
+	/*
+	 * DRA7x PLL CTRL's PLL_PWR_STATUS seems to always return 0,
+	 * so we have to use fixed delay here.
+	 */
+	msleep(1);
+}
+
+static void dss_dpll_power_disable(struct dss_video_pll *vpll)
+{
+	REG_MOD(vpll->clkctrl_base, 0, 31, 30);	/* PLL_POWER_OFF */
+}
+
+static int dss_video_pll_enable(struct dss_pll *pll)
+{
+	struct dss_video_pll *vpll = container_of(pll, struct dss_video_pll, pll);
+	int r;
+
+	r = dss_runtime_get();
+	if (r)
+		return r;
+
+	dss_ctrl_pll_enable(pll->id, true);
+
+	dss_dpll_enable_scp_clk(vpll);
+
+	r = dss_pll_wait_reset_done(pll);
+	if (r)
+		goto err_reset;
+
+	dss_dpll_power_enable(vpll);
+
+	return 0;
+
+err_reset:
+	dss_dpll_disable_scp_clk(vpll);
+	dss_ctrl_pll_enable(pll->id, false);
+	dss_runtime_put();
+
+	return r;
+}
+
+static void dss_video_pll_disable(struct dss_pll *pll)
+{
+	struct dss_video_pll *vpll = container_of(pll, struct dss_video_pll, pll);
+
+	dss_dpll_power_disable(vpll);
+
+	dss_dpll_disable_scp_clk(vpll);
+
+	dss_ctrl_pll_enable(pll->id, false);
+
+	dss_runtime_put();
+}
+
+static const struct dss_pll_ops dss_pll_ops = {
+	.enable = dss_video_pll_enable,
+	.disable = dss_video_pll_disable,
+	.set_config = dss_pll_write_config_type_a,
+};
+
+static const struct dss_pll_hw dss_dra7_video_pll_hw = {
+	.n_max = (1 << 8) - 1,
+	.m_max = (1 << 12) - 1,
+	.mX_max = (1 << 5) - 1,
+	.fint_min = 500000,
+	.fint_max = 2500000,
+	.clkdco_max = 1800000000,
+
+	.n_msb = 8,
+	.n_lsb = 1,
+	.m_msb = 20,
+	.m_lsb = 9,
+
+	.mX_msb[0] = 25,
+	.mX_lsb[0] = 21,
+	.mX_msb[1] = 30,
+	.mX_lsb[1] = 26,
+
+	.has_refsel = true,
+};
+
+struct dss_pll *dss_video_pll_init(struct platform_device *pdev, int id,
+	struct regulator *regulator)
+{
+	const char * const reg_name[] = { "pll1", "pll2" };
+	const char * const clkctrl_name[] = { "pll1_clkctrl", "pll2_clkctrl" };
+	const char * const clkin_name[] = { "video1_clk", "video2_clk" };
+
+	struct resource *res;
+	struct dss_video_pll *vpll;
+	void __iomem *pll_base, *clkctrl_base;
+	struct clk *clk;
+	struct dss_pll *pll;
+	int r;
+
+	/* PLL CONTROL */
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, reg_name[id]);
+	if (!res) {
+		dev_err(&pdev->dev,
+			"missing platform resource data for pll%d\n", id);
+		return ERR_PTR(-ENODEV);
+	}
+
+	pll_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pll_base)) {
+		dev_err(&pdev->dev, "failed to ioremap pll%d reg_name\n", id);
+		return ERR_CAST(pll_base);
+	}
+
+	/* CLOCK CONTROL */
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+		clkctrl_name[id]);
+	if (!res) {
+		dev_err(&pdev->dev,
+			"missing platform resource data for pll%d\n", id);
+		return ERR_PTR(-ENODEV);
+	}
+
+	clkctrl_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(clkctrl_base)) {
+		dev_err(&pdev->dev, "failed to ioremap pll%d clkctrl\n", id);
+		return ERR_CAST(clkctrl_base);
+	}
+
+	/* CLKIN */
+
+	clk = devm_clk_get(&pdev->dev, clkin_name[id]);
+	if (IS_ERR(clk)) {
+		DSSERR("can't get video pll clkin\n");
+		return ERR_CAST(clk);
+	}
+
+	vpll = devm_kzalloc(&pdev->dev, sizeof(*vpll), GFP_KERNEL);
+	if (!vpll)
+		return ERR_PTR(-ENOMEM);
+
+	vpll->dev = &pdev->dev;
+	vpll->clkctrl_base = clkctrl_base;
+
+	pll = &vpll->pll;
+
+	pll->name = id == 0 ? "video0" : "video1";
+	pll->id = id == 0 ? DSS_PLL_VIDEO1 : DSS_PLL_VIDEO2;
+	pll->clkin = clk;
+	pll->regulator = regulator;
+	pll->base = pll_base;
+	pll->hw = &dss_dra7_video_pll_hw;
+	pll->ops = &dss_pll_ops;
+
+	r = dss_pll_register(pll);
+	if (r)
+		return ERR_PTR(r);
+
+	return pll;
+}
+
+void dss_video_pll_uninit(struct dss_pll *pll)
+{
+	dss_pll_unregister(pll);
+}
diff --git a/drivers/video/fbdev/omap2/vrfb.c b/drivers/video/fbdev/omap2/omapfb/vrfb.c
index f346b02eee1d..f346b02eee1d 100644
--- a/drivers/video/fbdev/omap2/vrfb.c
+++ b/drivers/video/fbdev/omap2/omapfb/vrfb.c
diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c
index 94813af97f09..33b2bb315a2a 100644
--- a/drivers/video/fbdev/pxafb.c
+++ b/drivers/video/fbdev/pxafb.c
@@ -55,6 +55,9 @@
 #include <linux/kthread.h>
 #include <linux/freezer.h>
 #include <linux/console.h>
+#include <linux/of_graph.h>
+#include <video/of_display_timing.h>
+#include <video/videomode.h>
 
 #include <mach/hardware.h>
 #include <asm/io.h>
@@ -457,7 +460,7 @@ static int pxafb_adjust_timing(struct pxafb_info *fbi,
 static int pxafb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 {
 	struct pxafb_info *fbi = container_of(info, struct pxafb_info, fb);
-	struct pxafb_mach_info *inf = dev_get_platdata(fbi->dev);
+	struct pxafb_mach_info *inf = fbi->inf;
 	int err;
 
 	if (inf->fixed_modes) {
@@ -1230,7 +1233,7 @@ static unsigned int __smart_timing(unsigned time_ns, unsigned long lcd_clk)
 static void setup_smart_timing(struct pxafb_info *fbi,
 				struct fb_var_screeninfo *var)
 {
-	struct pxafb_mach_info *inf = dev_get_platdata(fbi->dev);
+	struct pxafb_mach_info *inf = fbi->inf;
 	struct pxafb_mode_info *mode = &inf->modes[0];
 	unsigned long lclk = clk_get_rate(fbi->clk);
 	unsigned t1, t2, t3, t4;
@@ -1258,14 +1261,13 @@ static void setup_smart_timing(struct pxafb_info *fbi,
 static int pxafb_smart_thread(void *arg)
 {
 	struct pxafb_info *fbi = arg;
-	struct pxafb_mach_info *inf = dev_get_platdata(fbi->dev);
+	struct pxafb_mach_info *inf = fbi->inf;
 
 	if (!inf->smart_update) {
 		pr_err("%s: not properly initialized, thread terminated\n",
 				__func__);
 		return -EINVAL;
 	}
-	inf = dev_get_platdata(fbi->dev);
 
 	pr_debug("%s(): task starting\n", __func__);
 
@@ -1788,11 +1790,11 @@ decode_mode:
 		fbi->video_mem_size = video_mem_size;
 }
 
-static struct pxafb_info *pxafb_init_fbinfo(struct device *dev)
+static struct pxafb_info *pxafb_init_fbinfo(struct device *dev,
+					    struct pxafb_mach_info *inf)
 {
 	struct pxafb_info *fbi;
 	void *addr;
-	struct pxafb_mach_info *inf = dev_get_platdata(dev);
 
 	/* Alloc the pxafb_info and pseudo_palette in one step */
 	fbi = kmalloc(sizeof(struct pxafb_info) + sizeof(u32) * 16, GFP_KERNEL);
@@ -1801,6 +1803,7 @@ static struct pxafb_info *pxafb_init_fbinfo(struct device *dev)
 
 	memset(fbi, 0, sizeof(struct pxafb_info));
 	fbi->dev = dev;
+	fbi->inf = inf;
 
 	fbi->clk = clk_get(dev, NULL);
 	if (IS_ERR(fbi->clk)) {
@@ -1852,10 +1855,9 @@ static struct pxafb_info *pxafb_init_fbinfo(struct device *dev)
 }
 
 #ifdef CONFIG_FB_PXA_PARAMETERS
-static int parse_opt_mode(struct device *dev, const char *this_opt)
+static int parse_opt_mode(struct device *dev, const char *this_opt,
+			  struct pxafb_mach_info *inf)
 {
-	struct pxafb_mach_info *inf = dev_get_platdata(dev);
-
 	const char *name = this_opt+5;
 	unsigned int namelen = strlen(name);
 	int res_specified = 0, bpp_specified = 0;
@@ -1911,9 +1913,9 @@ done:
 	return 0;
 }
 
-static int parse_opt(struct device *dev, char *this_opt)
+static int parse_opt(struct device *dev, char *this_opt,
+		     struct pxafb_mach_info *inf)
 {
-	struct pxafb_mach_info *inf = dev_get_platdata(dev);
 	struct pxafb_mode_info *mode = &inf->modes[0];
 	char s[64];
 
@@ -1922,7 +1924,7 @@ static int parse_opt(struct device *dev, char *this_opt)
 	if (!strncmp(this_opt, "vmem:", 5)) {
 		video_mem_size = memparse(this_opt + 5, NULL);
 	} else if (!strncmp(this_opt, "mode:", 5)) {
-		return parse_opt_mode(dev, this_opt);
+		return parse_opt_mode(dev, this_opt, inf);
 	} else if (!strncmp(this_opt, "pixclock:", 9)) {
 		mode->pixclock = simple_strtoul(this_opt+9, NULL, 0);
 		sprintf(s, "pixclock: %ld\n", mode->pixclock);
@@ -2011,7 +2013,8 @@ static int parse_opt(struct device *dev, char *this_opt)
 	return 0;
 }
 
-static int pxafb_parse_options(struct device *dev, char *options)
+static int pxafb_parse_options(struct device *dev, char *options,
+			       struct pxafb_mach_info *inf)
 {
 	char *this_opt;
 	int ret;
@@ -2023,7 +2026,7 @@ static int pxafb_parse_options(struct device *dev, char *options)
 
 	/* could be made table driven or similar?... */
 	while ((this_opt = strsep(&options, ",")) != NULL) {
-		ret = parse_opt(dev, this_opt);
+		ret = parse_opt(dev, this_opt, inf);
 		if (ret)
 			return ret;
 	}
@@ -2092,22 +2095,180 @@ static void pxafb_check_options(struct device *dev, struct pxafb_mach_info *inf)
 #define pxafb_check_options(...)	do {} while (0)
 #endif
 
+#if defined(CONFIG_OF)
+static const char * const lcd_types[] = {
+	"unknown", "mono-stn", "mono-dstn", "color-stn", "color-dstn",
+	"color-tft", "smart-panel", NULL
+};
+
+static int of_get_pxafb_display(struct device *dev, struct device_node *disp,
+				struct pxafb_mach_info *info, u32 bus_width)
+{
+	struct display_timings *timings;
+	struct videomode vm;
+	int i, ret = -EINVAL;
+	const char *s;
+
+	ret = of_property_read_string(disp, "lcd-type", &s);
+	if (ret)
+		s = "color-tft";
+
+	for (i = 0; lcd_types[i]; i++)
+		if (!strcmp(s, lcd_types[i]))
+			break;
+	if (!i || !lcd_types[i]) {
+		dev_err(dev, "lcd-type %s is unknown\n", s);
+		return -EINVAL;
+	}
+	info->lcd_conn |= LCD_CONN_TYPE(i);
+	info->lcd_conn |= LCD_CONN_WIDTH(bus_width);
+
+	timings = of_get_display_timings(disp);
+	if (!timings)
+		goto out;
+
+	ret = -ENOMEM;
+	info->modes = kmalloc_array(timings->num_timings,
+				    sizeof(info->modes[0]), GFP_KERNEL);
+	if (!info->modes)
+		goto out;
+	info->num_modes = timings->num_timings;
+
+	for (i = 0; i < timings->num_timings; i++) {
+		ret = videomode_from_timings(timings, &vm, i);
+		if (ret) {
+			dev_err(dev, "videomode_from_timings %d failed: %d\n",
+				i, ret);
+			goto out;
+		}
+		if (vm.flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
+			info->lcd_conn |= LCD_PCLK_EDGE_RISE;
+		if (vm.flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
+			info->lcd_conn |= LCD_PCLK_EDGE_FALL;
+		if (vm.flags & DISPLAY_FLAGS_DE_HIGH)
+			info->lcd_conn |= LCD_BIAS_ACTIVE_HIGH;
+		if (vm.flags & DISPLAY_FLAGS_DE_LOW)
+			info->lcd_conn |= LCD_BIAS_ACTIVE_LOW;
+		if (vm.flags & DISPLAY_FLAGS_HSYNC_HIGH)
+			info->modes[i].sync |= FB_SYNC_HOR_HIGH_ACT;
+		if (vm.flags & DISPLAY_FLAGS_VSYNC_HIGH)
+			info->modes[i].sync |= FB_SYNC_VERT_HIGH_ACT;
+
+		info->modes[i].pixclock = 1000000000UL / (vm.pixelclock / 1000);
+		info->modes[i].xres = vm.hactive;
+		info->modes[i].yres = vm.vactive;
+		info->modes[i].hsync_len = vm.hsync_len;
+		info->modes[i].left_margin = vm.hback_porch;
+		info->modes[i].right_margin = vm.hfront_porch;
+		info->modes[i].vsync_len = vm.vsync_len;
+		info->modes[i].upper_margin = vm.vback_porch;
+		info->modes[i].lower_margin = vm.vfront_porch;
+	}
+	ret = 0;
+
+out:
+	display_timings_release(timings);
+	return ret;
+}
+
+static int of_get_pxafb_mode_info(struct device *dev,
+				  struct pxafb_mach_info *info)
+{
+	struct device_node *display, *np;
+	u32 bus_width;
+	int ret, i;
+
+	np = of_graph_get_next_endpoint(dev->of_node, NULL);
+	if (!np) {
+		dev_err(dev, "could not find endpoint\n");
+		return -EINVAL;
+	}
+	ret = of_property_read_u32(np, "bus-width", &bus_width);
+	if (ret) {
+		dev_err(dev, "no bus-width specified: %d\n", ret);
+		return ret;
+	}
+
+	display = of_graph_get_remote_port_parent(np);
+	of_node_put(np);
+	if (!display) {
+		dev_err(dev, "no display defined\n");
+		return -EINVAL;
+	}
+
+	ret = of_get_pxafb_display(dev, display, info, bus_width);
+	of_node_put(display);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < info->num_modes; i++)
+		info->modes[i].bpp = bus_width;
+
+	return 0;
+}
+
+static struct pxafb_mach_info *of_pxafb_of_mach_info(struct device *dev)
+{
+	int ret;
+	struct pxafb_mach_info *info;
+
+	if (!dev->of_node)
+		return NULL;
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return ERR_PTR(-ENOMEM);
+	ret = of_get_pxafb_mode_info(dev, info);
+	if (ret) {
+		kfree(info->modes);
+		return ERR_PTR(ret);
+	}
+
+	/*
+	 * On purpose, neither lccrX registers nor video memory size can be
+	 * specified through device-tree, they are considered more a debug hack
+	 * available through command line.
+	 */
+	return info;
+}
+#else
+static struct pxafb_mach_info *of_pxafb_of_mach_info(struct device *dev)
+{
+	return NULL;
+}
+#endif
+
 static int pxafb_probe(struct platform_device *dev)
 {
 	struct pxafb_info *fbi;
-	struct pxafb_mach_info *inf;
+	struct pxafb_mach_info *inf, *pdata;
 	struct resource *r;
-	int irq, ret;
+	int i, irq, ret;
 
 	dev_dbg(&dev->dev, "pxafb_probe\n");
 
-	inf = dev_get_platdata(&dev->dev);
 	ret = -ENOMEM;
-	fbi = NULL;
+	pdata = dev_get_platdata(&dev->dev);
+	inf = devm_kmalloc(&dev->dev, sizeof(*inf), GFP_KERNEL);
 	if (!inf)
 		goto failed;
 
-	ret = pxafb_parse_options(&dev->dev, g_options);
+	if (pdata) {
+		*inf = *pdata;
+		inf->modes =
+			devm_kmalloc_array(&dev->dev, pdata->num_modes,
+					   sizeof(inf->modes[0]), GFP_KERNEL);
+		if (!inf->modes)
+			goto failed;
+		for (i = 0; i < inf->num_modes; i++)
+			inf->modes[i] = pdata->modes[i];
+	}
+
+	if (!pdata)
+		inf = of_pxafb_of_mach_info(&dev->dev);
+	if (IS_ERR_OR_NULL(inf))
+		goto failed;
+
+	ret = pxafb_parse_options(&dev->dev, g_options, inf);
 	if (ret < 0)
 		goto failed;
 
@@ -2125,7 +2286,7 @@ static int pxafb_probe(struct platform_device *dev)
 		goto failed;
 	}
 
-	fbi = pxafb_init_fbinfo(&dev->dev);
+	fbi = pxafb_init_fbinfo(&dev->dev, inf);
 	if (!fbi) {
 		/* only reason for pxafb_init_fbinfo to fail is kmalloc */
 		dev_err(&dev->dev, "Failed to initialize framebuffer device\n");
@@ -2299,11 +2460,20 @@ static int pxafb_remove(struct platform_device *dev)
 	return 0;
 }
 
+static const struct of_device_id pxafb_of_dev_id[] = {
+	{ .compatible = "marvell,pxa270-lcdc", },
+	{ .compatible = "marvell,pxa300-lcdc", },
+	{ .compatible = "marvell,pxa2xx-lcdc", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, pxafb_of_dev_id);
+
 static struct platform_driver pxafb_driver = {
 	.probe		= pxafb_probe,
 	.remove 	= pxafb_remove,
 	.driver		= {
 		.name	= "pxa2xx-fb",
+		.of_match_table = pxafb_of_dev_id,
 #ifdef CONFIG_PM
 		.pm	= &pxafb_pm_ops,
 #endif
diff --git a/drivers/video/fbdev/pxafb.h b/drivers/video/fbdev/pxafb.h
index 26ba9fa3f737..5dc414e26fc8 100644
--- a/drivers/video/fbdev/pxafb.h
+++ b/drivers/video/fbdev/pxafb.h
@@ -167,6 +167,8 @@ struct pxafb_info {
 
 	void (*lcd_power)(int, struct fb_var_screeninfo *);
 	void (*backlight_power)(int);
+
+	struct pxafb_mach_info	*inf;
 };
 
 #define TO_INF(ptr,member) container_of(ptr,struct pxafb_info,member)
diff --git a/drivers/video/fbdev/riva/fbdev.c b/drivers/video/fbdev/riva/fbdev.c
index f1ad2747064b..2ef26ad99341 100644
--- a/drivers/video/fbdev/riva/fbdev.c
+++ b/drivers/video/fbdev/riva/fbdev.c
@@ -1765,6 +1765,7 @@ static int riva_get_EDID_i2c(struct fb_info *info)
 	int i;
 
 	NVTRACE_ENTER();
+	par->riva.LockUnlock(&par->riva, 0);
 	riva_create_i2c_busses(par);
 	for (i = 0; i < 3; i++) {
 		if (!par->chan[i].par)
diff --git a/drivers/video/fbdev/sh_mobile_hdmi.c b/drivers/video/fbdev/sh_mobile_hdmi.c
deleted file mode 100644
index 7c72a3f02056..000000000000
--- a/drivers/video/fbdev/sh_mobile_hdmi.c
+++ /dev/null
@@ -1,1489 +0,0 @@
-/*
- * SH-Mobile High-Definition Multimedia Interface (HDMI) driver
- * for SLISHDMI13T and SLIPHDMIT IP cores
- *
- * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/clk.h>
-#include <linux/console.h>
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/workqueue.h>
-#include <sound/soc.h>
-#include <sound/soc-dapm.h>
-#include <sound/initval.h>
-
-#include <video/sh_mobile_hdmi.h>
-#include <video/sh_mobile_lcdc.h>
-
-#include "sh_mobile_lcdcfb.h"
-
-/* HDMI Core Control Register (HTOP0) */
-#define HDMI_SYSTEM_CTRL			0x00 /* System control */
-#define HDMI_L_R_DATA_SWAP_CTRL_RPKT		0x01 /* L/R data swap control,
-							bits 19..16 of 20-bit N for Audio Clock Regeneration packet */
-#define HDMI_20_BIT_N_FOR_AUDIO_RPKT_15_8	0x02 /* bits 15..8 of 20-bit N for Audio Clock Regeneration packet */
-#define HDMI_20_BIT_N_FOR_AUDIO_RPKT_7_0	0x03 /* bits 7..0 of 20-bit N for Audio Clock Regeneration packet */
-#define HDMI_SPDIF_AUDIO_SAMP_FREQ_CTS		0x04 /* SPDIF audio sampling frequency,
-							bits 19..16 of Internal CTS */
-#define HDMI_INTERNAL_CTS_15_8			0x05 /* bits 15..8 of Internal CTS */
-#define HDMI_INTERNAL_CTS_7_0			0x06 /* bits 7..0 of Internal CTS */
-#define HDMI_EXTERNAL_CTS_19_16			0x07 /* External CTS */
-#define HDMI_EXTERNAL_CTS_15_8			0x08 /* External CTS */
-#define HDMI_EXTERNAL_CTS_7_0			0x09 /* External CTS */
-#define HDMI_AUDIO_SETTING_1			0x0A /* Audio setting.1 */
-#define HDMI_AUDIO_SETTING_2			0x0B /* Audio setting.2 */
-#define HDMI_I2S_AUDIO_SET			0x0C /* I2S audio setting */
-#define HDMI_DSD_AUDIO_SET			0x0D /* DSD audio setting */
-#define HDMI_DEBUG_MONITOR_1			0x0E /* Debug monitor.1 */
-#define HDMI_DEBUG_MONITOR_2			0x0F /* Debug monitor.2 */
-#define HDMI_I2S_INPUT_PIN_SWAP			0x10 /* I2S input pin swap */
-#define HDMI_AUDIO_STATUS_BITS_SETTING_1	0x11 /* Audio status bits setting.1 */
-#define HDMI_AUDIO_STATUS_BITS_SETTING_2	0x12 /* Audio status bits setting.2 */
-#define HDMI_CATEGORY_CODE			0x13 /* Category code */
-#define HDMI_SOURCE_NUM_AUDIO_WORD_LEN		0x14 /* Source number/Audio word length */
-#define HDMI_AUDIO_VIDEO_SETTING_1		0x15 /* Audio/Video setting.1 */
-#define HDMI_VIDEO_SETTING_1			0x16 /* Video setting.1 */
-#define HDMI_DEEP_COLOR_MODES			0x17 /* Deep Color Modes */
-
-/* 12 16- and 10-bit Color space conversion parameters: 0x18..0x2f */
-#define HDMI_COLOR_SPACE_CONVERSION_PARAMETERS	0x18
-
-#define HDMI_EXTERNAL_VIDEO_PARAM_SETTINGS	0x30 /* External video parameter settings */
-#define HDMI_EXTERNAL_H_TOTAL_7_0		0x31 /* External horizontal total (LSB) */
-#define HDMI_EXTERNAL_H_TOTAL_11_8		0x32 /* External horizontal total (MSB) */
-#define HDMI_EXTERNAL_H_BLANK_7_0		0x33 /* External horizontal blank (LSB) */
-#define HDMI_EXTERNAL_H_BLANK_9_8		0x34 /* External horizontal blank (MSB) */
-#define HDMI_EXTERNAL_H_DELAY_7_0		0x35 /* External horizontal delay (LSB) */
-#define HDMI_EXTERNAL_H_DELAY_9_8		0x36 /* External horizontal delay (MSB) */
-#define HDMI_EXTERNAL_H_DURATION_7_0		0x37 /* External horizontal duration (LSB) */
-#define HDMI_EXTERNAL_H_DURATION_9_8		0x38 /* External horizontal duration (MSB) */
-#define HDMI_EXTERNAL_V_TOTAL_7_0		0x39 /* External vertical total (LSB) */
-#define HDMI_EXTERNAL_V_TOTAL_9_8		0x3A /* External vertical total (MSB) */
-#define HDMI_AUDIO_VIDEO_SETTING_2		0x3B /* Audio/Video setting.2 */
-#define HDMI_EXTERNAL_V_BLANK			0x3D /* External vertical blank */
-#define HDMI_EXTERNAL_V_DELAY			0x3E /* External vertical delay */
-#define HDMI_EXTERNAL_V_DURATION		0x3F /* External vertical duration */
-#define HDMI_CTRL_PKT_MANUAL_SEND_CONTROL	0x40 /* Control packet manual send control */
-#define HDMI_CTRL_PKT_AUTO_SEND			0x41 /* Control packet auto send with VSYNC control */
-#define HDMI_AUTO_CHECKSUM_OPTION		0x42 /* Auto checksum option */
-#define HDMI_VIDEO_SETTING_2			0x45 /* Video setting.2 */
-#define HDMI_OUTPUT_OPTION			0x46 /* Output option */
-#define HDMI_SLIPHDMIT_PARAM_OPTION		0x51 /* SLIPHDMIT parameter option */
-#define HDMI_HSYNC_PMENT_AT_EMB_7_0		0x52 /* HSYNC placement at embedded sync (LSB) */
-#define HDMI_HSYNC_PMENT_AT_EMB_15_8		0x53 /* HSYNC placement at embedded sync (MSB) */
-#define HDMI_VSYNC_PMENT_AT_EMB_7_0		0x54 /* VSYNC placement at embedded sync (LSB) */
-#define HDMI_VSYNC_PMENT_AT_EMB_14_8		0x55 /* VSYNC placement at embedded sync (MSB) */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_1		0x56 /* SLIPHDMIT parameter settings.1 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_2		0x57 /* SLIPHDMIT parameter settings.2 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_3		0x58 /* SLIPHDMIT parameter settings.3 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_5		0x59 /* SLIPHDMIT parameter settings.5 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_6		0x5A /* SLIPHDMIT parameter settings.6 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_7		0x5B /* SLIPHDMIT parameter settings.7 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_8		0x5C /* SLIPHDMIT parameter settings.8 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_9		0x5D /* SLIPHDMIT parameter settings.9 */
-#define HDMI_SLIPHDMIT_PARAM_SETTINGS_10	0x5E /* SLIPHDMIT parameter settings.10 */
-#define HDMI_CTRL_PKT_BUF_INDEX			0x5F /* Control packet buffer index */
-#define HDMI_CTRL_PKT_BUF_ACCESS_HB0		0x60 /* Control packet data buffer access window - HB0 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_HB1		0x61 /* Control packet data buffer access window - HB1 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_HB2		0x62 /* Control packet data buffer access window - HB2 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB0		0x63 /* Control packet data buffer access window - PB0 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB1		0x64 /* Control packet data buffer access window - PB1 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB2		0x65 /* Control packet data buffer access window - PB2 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB3		0x66 /* Control packet data buffer access window - PB3 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB4		0x67 /* Control packet data buffer access window - PB4 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB5		0x68 /* Control packet data buffer access window - PB5 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB6		0x69 /* Control packet data buffer access window - PB6 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB7		0x6A /* Control packet data buffer access window - PB7 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB8		0x6B /* Control packet data buffer access window - PB8 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB9		0x6C /* Control packet data buffer access window - PB9 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB10		0x6D /* Control packet data buffer access window - PB10 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB11		0x6E /* Control packet data buffer access window - PB11 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB12		0x6F /* Control packet data buffer access window - PB12 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB13		0x70 /* Control packet data buffer access window - PB13 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB14		0x71 /* Control packet data buffer access window - PB14 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB15		0x72 /* Control packet data buffer access window - PB15 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB16		0x73 /* Control packet data buffer access window - PB16 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB17		0x74 /* Control packet data buffer access window - PB17 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB18		0x75 /* Control packet data buffer access window - PB18 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB19		0x76 /* Control packet data buffer access window - PB19 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB20		0x77 /* Control packet data buffer access window - PB20 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB21		0x78 /* Control packet data buffer access window - PB21 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB22		0x79 /* Control packet data buffer access window - PB22 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB23		0x7A /* Control packet data buffer access window - PB23 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB24		0x7B /* Control packet data buffer access window - PB24 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB25		0x7C /* Control packet data buffer access window - PB25 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB26		0x7D /* Control packet data buffer access window - PB26 */
-#define HDMI_CTRL_PKT_BUF_ACCESS_PB27		0x7E /* Control packet data buffer access window - PB27 */
-#define HDMI_EDID_KSV_FIFO_ACCESS_WINDOW	0x80 /* EDID/KSV FIFO access window */
-#define HDMI_DDC_BUS_ACCESS_FREQ_CTRL_7_0	0x81 /* DDC bus access frequency control (LSB) */
-#define HDMI_DDC_BUS_ACCESS_FREQ_CTRL_15_8	0x82 /* DDC bus access frequency control (MSB) */
-#define HDMI_INTERRUPT_MASK_1			0x92 /* Interrupt mask.1 */
-#define HDMI_INTERRUPT_MASK_2			0x93 /* Interrupt mask.2 */
-#define HDMI_INTERRUPT_STATUS_1			0x94 /* Interrupt status.1 */
-#define HDMI_INTERRUPT_STATUS_2			0x95 /* Interrupt status.2 */
-#define HDMI_INTERRUPT_MASK_3			0x96 /* Interrupt mask.3 */
-#define HDMI_INTERRUPT_MASK_4			0x97 /* Interrupt mask.4 */
-#define HDMI_INTERRUPT_STATUS_3			0x98 /* Interrupt status.3 */
-#define HDMI_INTERRUPT_STATUS_4			0x99 /* Interrupt status.4 */
-#define HDMI_SOFTWARE_HDCP_CONTROL_1		0x9A /* Software HDCP control.1 */
-#define HDMI_FRAME_COUNTER			0x9C /* Frame counter */
-#define HDMI_FRAME_COUNTER_FOR_RI_CHECK		0x9D /* Frame counter for Ri check */
-#define HDMI_HDCP_CONTROL			0xAF /* HDCP control */
-#define HDMI_RI_FRAME_COUNT_REGISTER		0xB2 /* Ri frame count register */
-#define HDMI_DDC_BUS_CONTROL			0xB7 /* DDC bus control */
-#define HDMI_HDCP_STATUS			0xB8 /* HDCP status */
-#define HDMI_SHA0				0xB9 /* sha0 */
-#define HDMI_SHA1				0xBA /* sha1 */
-#define HDMI_SHA2				0xBB /* sha2 */
-#define HDMI_SHA3				0xBC /* sha3 */
-#define HDMI_SHA4				0xBD /* sha4 */
-#define HDMI_BCAPS_READ				0xBE /* BCAPS read / debug */
-#define HDMI_AKSV_BKSV_7_0_MONITOR		0xBF /* AKSV/BKSV[7:0] monitor */
-#define HDMI_AKSV_BKSV_15_8_MONITOR		0xC0 /* AKSV/BKSV[15:8] monitor */
-#define HDMI_AKSV_BKSV_23_16_MONITOR		0xC1 /* AKSV/BKSV[23:16] monitor */
-#define HDMI_AKSV_BKSV_31_24_MONITOR		0xC2 /* AKSV/BKSV[31:24] monitor */
-#define HDMI_AKSV_BKSV_39_32_MONITOR		0xC3 /* AKSV/BKSV[39:32] monitor */
-#define HDMI_EDID_SEGMENT_POINTER		0xC4 /* EDID segment pointer */
-#define HDMI_EDID_WORD_ADDRESS			0xC5 /* EDID word address */
-#define HDMI_EDID_DATA_FIFO_ADDRESS		0xC6 /* EDID data FIFO address */
-#define HDMI_NUM_OF_HDMI_DEVICES		0xC7 /* Number of HDMI devices */
-#define HDMI_HDCP_ERROR_CODE			0xC8 /* HDCP error code */
-#define HDMI_100MS_TIMER_SET			0xC9 /* 100ms timer setting */
-#define HDMI_5SEC_TIMER_SET			0xCA /* 5sec timer setting */
-#define HDMI_RI_READ_COUNT			0xCB /* Ri read count */
-#define HDMI_AN_SEED				0xCC /* An seed */
-#define HDMI_MAX_NUM_OF_RCIVRS_ALLOWED		0xCD /* Maximum number of receivers allowed */
-#define HDMI_HDCP_MEMORY_ACCESS_CONTROL_1	0xCE /* HDCP memory access control.1 */
-#define HDMI_HDCP_MEMORY_ACCESS_CONTROL_2	0xCF /* HDCP memory access control.2 */
-#define HDMI_HDCP_CONTROL_2			0xD0 /* HDCP Control 2 */
-#define HDMI_HDCP_KEY_MEMORY_CONTROL		0xD2 /* HDCP Key Memory Control */
-#define HDMI_COLOR_SPACE_CONV_CONFIG_1		0xD3 /* Color space conversion configuration.1 */
-#define HDMI_VIDEO_SETTING_3			0xD4 /* Video setting.3 */
-#define HDMI_RI_7_0				0xD5 /* Ri[7:0] */
-#define HDMI_RI_15_8				0xD6 /* Ri[15:8] */
-#define HDMI_PJ					0xD7 /* Pj */
-#define HDMI_SHA_RD				0xD8 /* sha_rd */
-#define HDMI_RI_7_0_SAVED			0xD9 /* Ri[7:0] saved */
-#define HDMI_RI_15_8_SAVED			0xDA /* Ri[15:8] saved */
-#define HDMI_PJ_SAVED				0xDB /* Pj saved */
-#define HDMI_NUM_OF_DEVICES			0xDC /* Number of devices */
-#define HDMI_HOT_PLUG_MSENS_STATUS		0xDF /* Hot plug/MSENS status */
-#define HDMI_BCAPS_WRITE			0xE0 /* bcaps */
-#define HDMI_BSTAT_7_0				0xE1 /* bstat[7:0] */
-#define HDMI_BSTAT_15_8				0xE2 /* bstat[15:8] */
-#define HDMI_BKSV_7_0				0xE3 /* bksv[7:0] */
-#define HDMI_BKSV_15_8				0xE4 /* bksv[15:8] */
-#define HDMI_BKSV_23_16				0xE5 /* bksv[23:16] */
-#define HDMI_BKSV_31_24				0xE6 /* bksv[31:24] */
-#define HDMI_BKSV_39_32				0xE7 /* bksv[39:32] */
-#define HDMI_AN_7_0				0xE8 /* An[7:0] */
-#define HDMI_AN_15_8				0xE9 /* An [15:8] */
-#define HDMI_AN_23_16				0xEA /* An [23:16] */
-#define HDMI_AN_31_24				0xEB /* An [31:24] */
-#define HDMI_AN_39_32				0xEC /* An [39:32] */
-#define HDMI_AN_47_40				0xED /* An [47:40] */
-#define HDMI_AN_55_48				0xEE /* An [55:48] */
-#define HDMI_AN_63_56				0xEF /* An [63:56] */
-#define HDMI_PRODUCT_ID				0xF0 /* Product ID */
-#define HDMI_REVISION_ID			0xF1 /* Revision ID */
-#define HDMI_TEST_MODE				0xFE /* Test mode */
-
-/* HDMI Control Register (HTOP1) */
-#define HDMI_HTOP1_TEST_MODE			0x0000 /* Test mode */
-#define HDMI_HTOP1_VIDEO_INPUT			0x0008 /* VideoInput */
-#define HDMI_HTOP1_CORE_RSTN			0x000C /* CoreResetn */
-#define HDMI_HTOP1_PLLBW			0x0018 /* PLLBW */
-#define HDMI_HTOP1_CLK_TO_PHY			0x001C /* Clk to Phy */
-#define HDMI_HTOP1_VIDEO_INPUT2			0x0020 /* VideoInput2 */
-#define HDMI_HTOP1_TISEMP0_1			0x0024 /* tisemp0-1 */
-#define HDMI_HTOP1_TISEMP2_C			0x0028 /* tisemp2-c */
-#define HDMI_HTOP1_TISIDRV			0x002C /* tisidrv */
-#define HDMI_HTOP1_TISEN			0x0034 /* tisen */
-#define HDMI_HTOP1_TISDREN			0x0038 /* tisdren  */
-#define HDMI_HTOP1_CISRANGE			0x003C /* cisrange  */
-#define HDMI_HTOP1_ENABLE_SELECTOR		0x0040 /* Enable Selector */
-#define HDMI_HTOP1_MACRO_RESET			0x0044 /* Macro reset */
-#define HDMI_HTOP1_PLL_CALIBRATION		0x0048 /* PLL calibration */
-#define HDMI_HTOP1_RE_CALIBRATION		0x004C /* Re-calibration */
-#define HDMI_HTOP1_CURRENT			0x0050 /* Current */
-#define HDMI_HTOP1_PLL_LOCK_DETECT		0x0054 /* PLL lock detect */
-#define HDMI_HTOP1_PHY_TEST_MODE		0x0058 /* PHY Test Mode */
-#define HDMI_HTOP1_CLK_SET			0x0080 /* Clock Set */
-#define HDMI_HTOP1_DDC_FAIL_SAFE		0x0084 /* DDC fail safe */
-#define HDMI_HTOP1_PRBS				0x0088 /* PRBS */
-#define HDMI_HTOP1_EDID_AINC_CONTROL		0x008C /* EDID ainc Control */
-#define HDMI_HTOP1_HTOP_DCL_MODE		0x00FC /* Deep Coloer Mode */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF0		0x0100 /* Deep Color:FRC COEF0 */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF1		0x0104 /* Deep Color:FRC COEF1 */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF2		0x0108 /* Deep Color:FRC COEF2 */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF3		0x010C /* Deep Color:FRC COEF3 */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF0_C		0x0110 /* Deep Color:FRC COEF0C */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF1_C		0x0114 /* Deep Color:FRC COEF1C */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF2_C		0x0118 /* Deep Color:FRC COEF2C */
-#define HDMI_HTOP1_HTOP_DCL_FRC_COEF3_C		0x011C /* Deep Color:FRC COEF3C */
-#define HDMI_HTOP1_HTOP_DCL_FRC_MODE		0x0120 /* Deep Color:FRC Mode */
-#define HDMI_HTOP1_HTOP_DCL_RECT_START1		0x0124 /* Deep Color:Rect Start1 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_SIZE1		0x0128 /* Deep Color:Rect Size1 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_START2		0x012C /* Deep Color:Rect Start2 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_SIZE2		0x0130 /* Deep Color:Rect Size2 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_START3		0x0134 /* Deep Color:Rect Start3 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_SIZE3		0x0138 /* Deep Color:Rect Size3 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_START4		0x013C /* Deep Color:Rect Start4 */
-#define HDMI_HTOP1_HTOP_DCL_RECT_SIZE4		0x0140 /* Deep Color:Rect Size4 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y1_1	0x0144 /* Deep Color:Fil Para Y1_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y1_2	0x0148 /* Deep Color:Fil Para Y1_2 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB1_1	0x014C /* Deep Color:Fil Para CB1_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB1_2	0x0150 /* Deep Color:Fil Para CB1_2 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR1_1	0x0154 /* Deep Color:Fil Para CR1_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR1_2	0x0158 /* Deep Color:Fil Para CR1_2 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y2_1	0x015C /* Deep Color:Fil Para Y2_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y2_2	0x0160 /* Deep Color:Fil Para Y2_2 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB2_1	0x0164 /* Deep Color:Fil Para CB2_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB2_2	0x0168 /* Deep Color:Fil Para CB2_2 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR2_1	0x016C /* Deep Color:Fil Para CR2_1 */
-#define HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR2_2	0x0170 /* Deep Color:Fil Para CR2_2 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_Y1		0x0174 /* Deep Color:Cor Para Y1 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_CB1	0x0178 /* Deep Color:Cor Para CB1 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_CR1	0x017C /* Deep Color:Cor Para CR1 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_Y2		0x0180 /* Deep Color:Cor Para Y2 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_CB2	0x0184 /* Deep Color:Cor Para CB2 */
-#define HDMI_HTOP1_HTOP_DCL_COR_PARA_CR2	0x0188 /* Deep Color:Cor Para CR2 */
-#define HDMI_HTOP1_EDID_DATA_READ		0x0200 /* EDID Data Read 128Byte:0x03FC */
-
-enum hotplug_state {
-	HDMI_HOTPLUG_DISCONNECTED,
-	HDMI_HOTPLUG_CONNECTED,
-	HDMI_HOTPLUG_EDID_DONE,
-};
-
-struct sh_hdmi {
-	struct sh_mobile_lcdc_entity entity;
-
-	void __iomem *base;
-	void __iomem *htop1;
-	enum hotplug_state hp_state;	/* hot-plug status */
-	u8 preprogrammed_vic;		/* use a pre-programmed VIC or
-					   the external mode */
-	u8 edid_block_addr;
-	u8 edid_segment_nr;
-	u8 edid_blocks;
-	int irq;
-	struct clk *hdmi_clk;
-	struct device *dev;
-	struct delayed_work edid_work;
-	struct fb_videomode mode;
-	struct fb_monspecs monspec;
-
-	/* register access functions */
-	void (*write)(struct sh_hdmi *hdmi, u8 data, u8 reg);
-	u8 (*read)(struct sh_hdmi *hdmi, u8 reg);
-};
-
-#define entity_to_sh_hdmi(e)	container_of(e, struct sh_hdmi, entity)
-
-static void __hdmi_write8(struct sh_hdmi *hdmi, u8 data, u8 reg)
-{
-	iowrite8(data, hdmi->base + reg);
-}
-
-static u8 __hdmi_read8(struct sh_hdmi *hdmi, u8 reg)
-{
-	return ioread8(hdmi->base + reg);
-}
-
-static void __hdmi_write32(struct sh_hdmi *hdmi, u8 data, u8 reg)
-{
-	iowrite32((u32)data, hdmi->base + (reg * 4));
-	udelay(100);
-}
-
-static u8 __hdmi_read32(struct sh_hdmi *hdmi, u8 reg)
-{
-	return (u8)ioread32(hdmi->base + (reg * 4));
-}
-
-static void hdmi_write(struct sh_hdmi *hdmi, u8 data, u8 reg)
-{
-	hdmi->write(hdmi, data, reg);
-}
-
-static u8 hdmi_read(struct sh_hdmi *hdmi, u8 reg)
-{
-	return hdmi->read(hdmi, reg);
-}
-
-static void hdmi_bit_set(struct sh_hdmi *hdmi, u8 mask, u8 data, u8 reg)
-{
-	u8 val = hdmi_read(hdmi, reg);
-
-	val &= ~mask;
-	val |= (data & mask);
-
-	hdmi_write(hdmi, val, reg);
-}
-
-static void hdmi_htop1_write(struct sh_hdmi *hdmi, u32 data, u32 reg)
-{
-	iowrite32(data, hdmi->htop1 + reg);
-	udelay(100);
-}
-
-static u32 hdmi_htop1_read(struct sh_hdmi *hdmi, u32 reg)
-{
-	return ioread32(hdmi->htop1 + reg);
-}
-
-/*
- *	HDMI sound
- */
-static unsigned int sh_hdmi_snd_read(struct snd_soc_codec *codec,
-				     unsigned int reg)
-{
-	struct sh_hdmi *hdmi = snd_soc_codec_get_drvdata(codec);
-
-	return hdmi_read(hdmi, reg);
-}
-
-static int sh_hdmi_snd_write(struct snd_soc_codec *codec,
-			     unsigned int reg,
-			     unsigned int value)
-{
-	struct sh_hdmi *hdmi = snd_soc_codec_get_drvdata(codec);
-
-	hdmi_write(hdmi, value, reg);
-	return 0;
-}
-
-static struct snd_soc_dai_driver sh_hdmi_dai = {
-	.name = "sh_mobile_hdmi-hifi",
-	.playback = {
-		.stream_name = "Playback",
-		.channels_min = 2,
-		.channels_max = 8,
-		.rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100  |
-			 SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200  |
-			 SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_176400 |
-			 SNDRV_PCM_RATE_192000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE,
-	},
-};
-
-static int sh_hdmi_snd_probe(struct snd_soc_codec *codec)
-{
-	dev_info(codec->dev, "SH Mobile HDMI Audio Codec");
-
-	return 0;
-}
-
-static struct snd_soc_codec_driver soc_codec_dev_sh_hdmi = {
-	.probe		= sh_hdmi_snd_probe,
-	.read		= sh_hdmi_snd_read,
-	.write		= sh_hdmi_snd_write,
-};
-
-/*
- *	HDMI video
- */
-
-/* External video parameter settings */
-static void sh_hdmi_external_video_param(struct sh_hdmi *hdmi)
-{
-	struct fb_videomode *mode = &hdmi->mode;
-	u16 htotal, hblank, hdelay, vtotal, vblank, vdelay, voffset;
-	u8 sync = 0;
-
-	htotal = mode->xres + mode->right_margin + mode->left_margin
-	       + mode->hsync_len;
-	hdelay = mode->hsync_len + mode->left_margin;
-	hblank = mode->right_margin + hdelay;
-
-	/*
-	 * Vertical timing looks a bit different in Figure 18,
-	 * but let's try the same first by setting offset = 0
-	 */
-	vtotal = mode->yres + mode->upper_margin + mode->lower_margin
-	       + mode->vsync_len;
-	vdelay = mode->vsync_len + mode->upper_margin;
-	vblank = mode->lower_margin + vdelay;
-	voffset = min(mode->upper_margin / 2, 6U);
-
-	/*
-	 * [3]: VSYNC polarity: Positive
-	 * [2]: HSYNC polarity: Positive
-	 * [1]: Interlace/Progressive: Progressive
-	 * [0]: External video settings enable: used.
-	 */
-	if (mode->sync & FB_SYNC_HOR_HIGH_ACT)
-		sync |= 4;
-	if (mode->sync & FB_SYNC_VERT_HIGH_ACT)
-		sync |= 8;
-
-	dev_dbg(hdmi->dev, "H: %u, %u, %u, %u; V: %u, %u, %u, %u; sync 0x%x\n",
-		htotal, hblank, hdelay, mode->hsync_len,
-		vtotal, vblank, vdelay, mode->vsync_len, sync);
-
-	hdmi_write(hdmi, sync | (voffset << 4), HDMI_EXTERNAL_VIDEO_PARAM_SETTINGS);
-
-	hdmi_write(hdmi, htotal, HDMI_EXTERNAL_H_TOTAL_7_0);
-	hdmi_write(hdmi, htotal >> 8, HDMI_EXTERNAL_H_TOTAL_11_8);
-
-	hdmi_write(hdmi, hblank, HDMI_EXTERNAL_H_BLANK_7_0);
-	hdmi_write(hdmi, hblank >> 8, HDMI_EXTERNAL_H_BLANK_9_8);
-
-	hdmi_write(hdmi, hdelay, HDMI_EXTERNAL_H_DELAY_7_0);
-	hdmi_write(hdmi, hdelay >> 8, HDMI_EXTERNAL_H_DELAY_9_8);
-
-	hdmi_write(hdmi, mode->hsync_len, HDMI_EXTERNAL_H_DURATION_7_0);
-	hdmi_write(hdmi, mode->hsync_len >> 8, HDMI_EXTERNAL_H_DURATION_9_8);
-
-	hdmi_write(hdmi, vtotal, HDMI_EXTERNAL_V_TOTAL_7_0);
-	hdmi_write(hdmi, vtotal >> 8, HDMI_EXTERNAL_V_TOTAL_9_8);
-
-	hdmi_write(hdmi, vblank, HDMI_EXTERNAL_V_BLANK);
-
-	hdmi_write(hdmi, vdelay, HDMI_EXTERNAL_V_DELAY);
-
-	hdmi_write(hdmi, mode->vsync_len, HDMI_EXTERNAL_V_DURATION);
-
-	/* Set bit 0 of HDMI_EXTERNAL_VIDEO_PARAM_SETTINGS here for external mode */
-	if (!hdmi->preprogrammed_vic)
-		hdmi_write(hdmi, sync | 1 | (voffset << 4),
-			   HDMI_EXTERNAL_VIDEO_PARAM_SETTINGS);
-}
-
-/**
- * sh_hdmi_video_config()
- */
-static void sh_hdmi_video_config(struct sh_hdmi *hdmi)
-{
-	/*
-	 * [7:4]: Audio sampling frequency: 48kHz
-	 * [3:1]: Input video format: RGB and YCbCr 4:4:4 (Y on Green)
-	 * [0]: Internal/External DE select: internal
-	 */
-	hdmi_write(hdmi, 0x20, HDMI_AUDIO_VIDEO_SETTING_1);
-
-	/*
-	 * [7:6]: Video output format: RGB 4:4:4
-	 * [5:4]: Input video data width: 8 bit
-	 * [3:1]: EAV/SAV location: channel 1
-	 * [0]: Video input color space: RGB
-	 */
-	hdmi_write(hdmi, 0x34, HDMI_VIDEO_SETTING_1);
-
-	/*
-	 * [7:6]: Together with bit [6] of HDMI_AUDIO_VIDEO_SETTING_2, which is
-	 * left at 0 by default, this configures 24bpp and sets the Color Depth
-	 * (CD) field in the General Control Packet
-	 */
-	hdmi_write(hdmi, 0x20, HDMI_DEEP_COLOR_MODES);
-}
-
-/**
- * sh_hdmi_audio_config()
- */
-static void sh_hdmi_audio_config(struct sh_hdmi *hdmi)
-{
-	u8 data;
-	struct sh_mobile_hdmi_info *pdata = dev_get_platdata(hdmi->dev);
-
-	/*
-	 * [7:4] L/R data swap control
-	 * [3:0] appropriate N[19:16]
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_L_R_DATA_SWAP_CTRL_RPKT);
-	/* appropriate N[15:8] */
-	hdmi_write(hdmi, 0x18, HDMI_20_BIT_N_FOR_AUDIO_RPKT_15_8);
-	/* appropriate N[7:0] */
-	hdmi_write(hdmi, 0x00, HDMI_20_BIT_N_FOR_AUDIO_RPKT_7_0);
-
-	/* [7:4] 48 kHz	SPDIF not used */
-	hdmi_write(hdmi, 0x20, HDMI_SPDIF_AUDIO_SAMP_FREQ_CTS);
-
-	/*
-	 * [6:5] set required down sampling rate if required
-	 * [4:3] set required audio source
-	 */
-	switch (pdata->flags & HDMI_SND_SRC_MASK) {
-	default:
-		/* fall through */
-	case HDMI_SND_SRC_I2S:
-		data = 0x0 << 3;
-		break;
-	case HDMI_SND_SRC_SPDIF:
-		data = 0x1 << 3;
-		break;
-	case HDMI_SND_SRC_DSD:
-		data = 0x2 << 3;
-		break;
-	case HDMI_SND_SRC_HBR:
-		data = 0x3 << 3;
-		break;
-	}
-	hdmi_write(hdmi, data, HDMI_AUDIO_SETTING_1);
-
-	/* [3:0] set sending channel number for channel status */
-	hdmi_write(hdmi, 0x40, HDMI_AUDIO_SETTING_2);
-
-	/*
-	 * [5:2] set valid I2S source input pin
-	 * [1:0] set input I2S source mode
-	 */
-	hdmi_write(hdmi, 0x04, HDMI_I2S_AUDIO_SET);
-
-	/* [7:4] set valid DSD source input pin */
-	hdmi_write(hdmi, 0x00, HDMI_DSD_AUDIO_SET);
-
-	/* [7:0] set appropriate I2S input pin swap settings if required */
-	hdmi_write(hdmi, 0x00, HDMI_I2S_INPUT_PIN_SWAP);
-
-	/*
-	 * [7] set validity bit for channel status
-	 * [3:0] set original sample frequency for channel status
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_AUDIO_STATUS_BITS_SETTING_1);
-
-	/*
-	 * [7] set value for channel status
-	 * [6] set value for channel status
-	 * [5] set copyright bit for channel status
-	 * [4:2] set additional information for channel status
-	 * [1:0] set clock accuracy for channel status
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_AUDIO_STATUS_BITS_SETTING_2);
-
-	/* [7:0] set category code for channel status */
-	hdmi_write(hdmi, 0x00, HDMI_CATEGORY_CODE);
-
-	/*
-	 * [7:4] set source number for channel status
-	 * [3:0] set word length for channel status
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_SOURCE_NUM_AUDIO_WORD_LEN);
-
-	/* [7:4] set sample frequency for channel status */
-	hdmi_write(hdmi, 0x20, HDMI_AUDIO_VIDEO_SETTING_1);
-}
-
-/**
- * sh_hdmi_phy_config() - configure the HDMI PHY for the used video mode
- */
-static void sh_hdmi_phy_config(struct sh_hdmi *hdmi)
-{
-	if (hdmi->mode.pixclock < 10000) {
-		/* for 1080p8bit 148MHz */
-		hdmi_write(hdmi, 0x1d, HDMI_SLIPHDMIT_PARAM_SETTINGS_1);
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_2);
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_3);
-		hdmi_write(hdmi, 0x4c, HDMI_SLIPHDMIT_PARAM_SETTINGS_5);
-		hdmi_write(hdmi, 0x1e, HDMI_SLIPHDMIT_PARAM_SETTINGS_6);
-		hdmi_write(hdmi, 0x48, HDMI_SLIPHDMIT_PARAM_SETTINGS_7);
-		hdmi_write(hdmi, 0x0e, HDMI_SLIPHDMIT_PARAM_SETTINGS_8);
-		hdmi_write(hdmi, 0x25, HDMI_SLIPHDMIT_PARAM_SETTINGS_9);
-		hdmi_write(hdmi, 0x04, HDMI_SLIPHDMIT_PARAM_SETTINGS_10);
-	} else if (hdmi->mode.pixclock < 30000) {
-		/* 720p, 8bit, 74.25MHz. Might need to be adjusted for other formats */
-		/*
-		 * [1:0]	Speed_A
-		 * [3:2]	Speed_B
-		 * [4]		PLLA_Bypass
-		 * [6]		DRV_TEST_EN
-		 * [7]		DRV_TEST_IN
-		 */
-		hdmi_write(hdmi, 0x0f, HDMI_SLIPHDMIT_PARAM_SETTINGS_1);
-		/* PLLB_CONFIG[17], PLLA_CONFIG[17] - not in PHY datasheet */
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_2);
-		/*
-		 * [2:0]	BGR_I_OFFSET
-		 * [6:4]	BGR_V_OFFSET
-		 */
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_3);
-		/* PLLA_CONFIG[7:0]: VCO gain, VCO offset, LPF resistance[0] */
-		hdmi_write(hdmi, 0x44, HDMI_SLIPHDMIT_PARAM_SETTINGS_5);
-		/*
-		 * PLLA_CONFIG[15:8]: regulator voltage[0], CP current,
-		 * LPF capacitance, LPF resistance[1]
-		 */
-		hdmi_write(hdmi, 0x32, HDMI_SLIPHDMIT_PARAM_SETTINGS_6);
-		/* PLLB_CONFIG[7:0]: LPF resistance[0], VCO offset, VCO gain */
-		hdmi_write(hdmi, 0x4A, HDMI_SLIPHDMIT_PARAM_SETTINGS_7);
-		/*
-		 * PLLB_CONFIG[15:8]: regulator voltage[0], CP current,
-		 * LPF capacitance, LPF resistance[1]
-		 */
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_8);
-		/* DRV_CONFIG, PE_CONFIG */
-		hdmi_write(hdmi, 0x25, HDMI_SLIPHDMIT_PARAM_SETTINGS_9);
-		/*
-		 * [2:0]	AMON_SEL (4 == LPF voltage)
-		 * [4]		PLLA_CONFIG[16]
-		 * [5]		PLLB_CONFIG[16]
-		 */
-		hdmi_write(hdmi, 0x04, HDMI_SLIPHDMIT_PARAM_SETTINGS_10);
-	} else {
-		/* for 480p8bit 27MHz */
-		hdmi_write(hdmi, 0x19, HDMI_SLIPHDMIT_PARAM_SETTINGS_1);
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_2);
-		hdmi_write(hdmi, 0x00, HDMI_SLIPHDMIT_PARAM_SETTINGS_3);
-		hdmi_write(hdmi, 0x44, HDMI_SLIPHDMIT_PARAM_SETTINGS_5);
-		hdmi_write(hdmi, 0x32, HDMI_SLIPHDMIT_PARAM_SETTINGS_6);
-		hdmi_write(hdmi, 0x48, HDMI_SLIPHDMIT_PARAM_SETTINGS_7);
-		hdmi_write(hdmi, 0x0F, HDMI_SLIPHDMIT_PARAM_SETTINGS_8);
-		hdmi_write(hdmi, 0x20, HDMI_SLIPHDMIT_PARAM_SETTINGS_9);
-		hdmi_write(hdmi, 0x04, HDMI_SLIPHDMIT_PARAM_SETTINGS_10);
-	}
-}
-
-/**
- * sh_hdmi_avi_infoframe_setup() - Auxiliary Video Information InfoFrame CONTROL PACKET
- */
-static void sh_hdmi_avi_infoframe_setup(struct sh_hdmi *hdmi)
-{
-	u8 vic;
-
-	/* AVI InfoFrame */
-	hdmi_write(hdmi, 0x06, HDMI_CTRL_PKT_BUF_INDEX);
-
-	/* Packet Type = 0x82 */
-	hdmi_write(hdmi, 0x82, HDMI_CTRL_PKT_BUF_ACCESS_HB0);
-
-	/* Version = 0x02 */
-	hdmi_write(hdmi, 0x02, HDMI_CTRL_PKT_BUF_ACCESS_HB1);
-
-	/* Length = 13 (0x0D) */
-	hdmi_write(hdmi, 0x0D, HDMI_CTRL_PKT_BUF_ACCESS_HB2);
-
-	/* N. A. Checksum */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB0);
-
-	/*
-	 * Y = RGB
-	 * A0 = No Data
-	 * B = Bar Data not valid
-	 * S = No Data
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB1);
-
-	/*
-	 * [7:6] C = Colorimetry: no data
-	 * [5:4] M = 2: 16:9, 1: 4:3 Picture Aspect Ratio
-	 * [3:0] R = 8: Active Frame Aspect Ratio: same as picture aspect ratio
-	 */
-	hdmi_write(hdmi, 0x28, HDMI_CTRL_PKT_BUF_ACCESS_PB2);
-
-	/*
-	 * ITC = No Data
-	 * EC = xvYCC601
-	 * Q = Default (depends on video format)
-	 * SC = No Known non_uniform Scaling
-	 */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB3);
-
-	/*
-	 * VIC should be ignored if external config is used, so, we could just use 0,
-	 * but play safe and use a valid value in any case just in case
-	 */
-	if (hdmi->preprogrammed_vic)
-		vic = hdmi->preprogrammed_vic;
-	else
-		vic = 4;
-	hdmi_write(hdmi, vic, HDMI_CTRL_PKT_BUF_ACCESS_PB4);
-
-	/* PR = No Repetition */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB5);
-
-	/* Line Number of End of Top Bar (lower 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB6);
-
-	/* Line Number of End of Top Bar (upper 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB7);
-
-	/* Line Number of Start of Bottom Bar (lower 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB8);
-
-	/* Line Number of Start of Bottom Bar (upper 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB9);
-
-	/* Pixel Number of End of Left Bar (lower 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB10);
-
-	/* Pixel Number of End of Left Bar (upper 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB11);
-
-	/* Pixel Number of Start of Right Bar (lower 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB12);
-
-	/* Pixel Number of Start of Right Bar (upper 8 bits) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB13);
-}
-
-/**
- * sh_hdmi_audio_infoframe_setup() - Audio InfoFrame of CONTROL PACKET
- */
-static void sh_hdmi_audio_infoframe_setup(struct sh_hdmi *hdmi)
-{
-	/* Audio InfoFrame */
-	hdmi_write(hdmi, 0x08, HDMI_CTRL_PKT_BUF_INDEX);
-
-	/* Packet Type = 0x84 */
-	hdmi_write(hdmi, 0x84, HDMI_CTRL_PKT_BUF_ACCESS_HB0);
-
-	/* Version Number = 0x01 */
-	hdmi_write(hdmi, 0x01, HDMI_CTRL_PKT_BUF_ACCESS_HB1);
-
-	/* 0 Length = 10 (0x0A) */
-	hdmi_write(hdmi, 0x0A, HDMI_CTRL_PKT_BUF_ACCESS_HB2);
-
-	/* n. a. Checksum */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB0);
-
-	/* Audio Channel Count = Refer to Stream Header */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB1);
-
-	/* Refer to Stream Header */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB2);
-
-	/* Format depends on coding type (i.e. CT0...CT3) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB3);
-
-	/* Speaker Channel Allocation = Front Right + Front Left */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB4);
-
-	/* Level Shift Value = 0 dB, Down - mix is permitted or no information */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB5);
-
-	/* Reserved (0) */
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB6);
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB7);
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB8);
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB9);
-	hdmi_write(hdmi, 0x00, HDMI_CTRL_PKT_BUF_ACCESS_PB10);
-}
-
-/**
- * sh_hdmi_configure() - Initialise HDMI for output
- */
-static void sh_hdmi_configure(struct sh_hdmi *hdmi)
-{
-	/* Configure video format */
-	sh_hdmi_video_config(hdmi);
-
-	/* Configure audio format */
-	sh_hdmi_audio_config(hdmi);
-
-	/* Configure PHY */
-	sh_hdmi_phy_config(hdmi);
-
-	/* Auxiliary Video Information (AVI) InfoFrame */
-	sh_hdmi_avi_infoframe_setup(hdmi);
-
-	/* Audio InfoFrame */
-	sh_hdmi_audio_infoframe_setup(hdmi);
-
-	/*
-	 * Control packet auto send with VSYNC control: auto send
-	 * General control, Gamut metadata, ISRC, and ACP packets
-	 */
-	hdmi_write(hdmi, 0x8E, HDMI_CTRL_PKT_AUTO_SEND);
-
-	/* FIXME */
-	msleep(10);
-
-	/* PS mode b->d, reset PLLA and PLLB */
-	hdmi_bit_set(hdmi, 0xFC, 0x4C, HDMI_SYSTEM_CTRL);
-
-	udelay(10);
-
-	hdmi_bit_set(hdmi, 0xFC, 0x40, HDMI_SYSTEM_CTRL);
-}
-
-static unsigned long sh_hdmi_rate_error(struct sh_hdmi *hdmi,
-		const struct fb_videomode *mode,
-		unsigned long *hdmi_rate, unsigned long *parent_rate)
-{
-	unsigned long target = PICOS2KHZ(mode->pixclock) * 1000, rate_error;
-	struct sh_mobile_hdmi_info *pdata = dev_get_platdata(hdmi->dev);
-
-	*hdmi_rate = clk_round_rate(hdmi->hdmi_clk, target);
-	if ((long)*hdmi_rate < 0)
-		*hdmi_rate = clk_get_rate(hdmi->hdmi_clk);
-
-	rate_error = (long)*hdmi_rate > 0 ? abs(*hdmi_rate - target) : ULONG_MAX;
-	if (rate_error && pdata->clk_optimize_parent)
-		rate_error = pdata->clk_optimize_parent(target, hdmi_rate, parent_rate);
-	else if (clk_get_parent(hdmi->hdmi_clk))
-		*parent_rate = clk_get_rate(clk_get_parent(hdmi->hdmi_clk));
-
-	dev_dbg(hdmi->dev, "%u-%u-%u-%u x %u-%u-%u-%u\n",
-		mode->left_margin, mode->xres,
-		mode->right_margin, mode->hsync_len,
-		mode->upper_margin, mode->yres,
-		mode->lower_margin, mode->vsync_len);
-
-	dev_dbg(hdmi->dev, "\t@%lu(+/-%lu)Hz, e=%lu / 1000, r=%uHz, p=%luHz\n", target,
-		rate_error, rate_error ? 10000 / (10 * target / rate_error) : 0,
-		mode->refresh, *parent_rate);
-
-	return rate_error;
-}
-
-static int sh_hdmi_read_edid(struct sh_hdmi *hdmi, unsigned long *hdmi_rate,
-			     unsigned long *parent_rate)
-{
-	struct sh_mobile_lcdc_chan *ch = hdmi->entity.lcdc;
-	const struct fb_videomode *mode, *found = NULL;
-	unsigned int f_width = 0, f_height = 0, f_refresh = 0;
-	unsigned long found_rate_error = ULONG_MAX; /* silly compiler... */
-	bool scanning = false, preferred_bad = false;
-	bool use_edid_mode = false;
-	u8 edid[128];
-	char *forced;
-	int i;
-
-	/* Read EDID */
-	dev_dbg(hdmi->dev, "Read back EDID code:");
-	for (i = 0; i < 128; i++) {
-		edid[i] = (hdmi->htop1) ?
-			(u8)hdmi_htop1_read(hdmi, HDMI_HTOP1_EDID_DATA_READ + (i * 4)) :
-			hdmi_read(hdmi, HDMI_EDID_KSV_FIFO_ACCESS_WINDOW);
-#ifdef DEBUG
-		if ((i % 16) == 0) {
-			printk(KERN_CONT "\n");
-			printk(KERN_DEBUG "%02X | %02X", i, edid[i]);
-		} else {
-			printk(KERN_CONT " %02X", edid[i]);
-		}
-#endif
-	}
-#ifdef DEBUG
-	printk(KERN_CONT "\n");
-#endif
-
-	if (!hdmi->edid_blocks) {
-		fb_edid_to_monspecs(edid, &hdmi->monspec);
-		hdmi->edid_blocks = edid[126] + 1;
-
-		dev_dbg(hdmi->dev, "%d main modes, %d extension blocks\n",
-			hdmi->monspec.modedb_len, hdmi->edid_blocks - 1);
-	} else {
-		dev_dbg(hdmi->dev, "Extension %u detected, DTD start %u\n",
-			edid[0], edid[2]);
-		fb_edid_add_monspecs(edid, &hdmi->monspec);
-	}
-
-	if (hdmi->edid_blocks > hdmi->edid_segment_nr * 2 +
-	    (hdmi->edid_block_addr >> 7) + 1) {
-		/* More blocks to read */
-		if (hdmi->edid_block_addr) {
-			hdmi->edid_block_addr = 0;
-			hdmi->edid_segment_nr++;
-		} else {
-			hdmi->edid_block_addr = 0x80;
-		}
-		/* Set EDID word address  */
-		hdmi_write(hdmi, hdmi->edid_block_addr, HDMI_EDID_WORD_ADDRESS);
-		/* Enable EDID interrupt */
-		hdmi_write(hdmi, 0xC6, HDMI_INTERRUPT_MASK_1);
-		/* Set EDID segment pointer - starts reading EDID */
-		hdmi_write(hdmi, hdmi->edid_segment_nr, HDMI_EDID_SEGMENT_POINTER);
-		return -EAGAIN;
-	}
-
-	/* All E-EDID blocks ready */
-	dev_dbg(hdmi->dev, "%d main and extended modes\n", hdmi->monspec.modedb_len);
-
-	fb_get_options("sh_mobile_lcdc", &forced);
-	if (forced && *forced) {
-		/* Only primitive parsing so far */
-		i = sscanf(forced, "%ux%u@%u",
-			   &f_width, &f_height, &f_refresh);
-		if (i < 2) {
-			f_width = 0;
-			f_height = 0;
-		} else {
-			/* The user wants us to use the EDID data */
-			scanning = true;
-		}
-		dev_dbg(hdmi->dev, "Forced mode %ux%u@%uHz\n",
-			f_width, f_height, f_refresh);
-	}
-
-	/* Walk monitor modes to find the best or the exact match */
-	for (i = 0, mode = hdmi->monspec.modedb;
-	     i < hdmi->monspec.modedb_len && scanning;
-	     i++, mode++) {
-		unsigned long rate_error;
-
-		if (!f_width && !f_height) {
-			/*
-			 * A parameter string "video=sh_mobile_lcdc:0x0" means
-			 * use the preferred EDID mode. If it is rejected by
-			 * .fb_check_var(), keep looking, until an acceptable
-			 * one is found.
-			 */
-			if ((mode->flag & FB_MODE_IS_FIRST) || preferred_bad)
-				scanning = false;
-			else
-				continue;
-		} else if (f_width != mode->xres || f_height != mode->yres) {
-			/* No interest in unmatching modes */
-			continue;
-		}
-
-		rate_error = sh_hdmi_rate_error(hdmi, mode, hdmi_rate, parent_rate);
-
-		if (scanning) {
-			if (f_refresh == mode->refresh || (!f_refresh && !rate_error))
-				/*
-				 * Exact match if either the refresh rate
-				 * matches or it hasn't been specified and we've
-				 * found a mode, for which we can configure the
-				 * clock precisely
-				 */
-				scanning = false;
-			else if (found && found_rate_error <= rate_error)
-				/*
-				 * We otherwise search for the closest matching
-				 * clock rate - either if no refresh rate has
-				 * been specified or we cannot find an exactly
-				 * matching one
-				 */
-				continue;
-		}
-
-		/* Check if supported: sufficient fb memory, supported clock-rate */
-		if (ch && ch->notify &&
-		    ch->notify(ch, SH_MOBILE_LCDC_EVENT_DISPLAY_MODE, mode,
-			       NULL)) {
-			scanning = true;
-			preferred_bad = true;
-			continue;
-		}
-
-		found = mode;
-		found_rate_error = rate_error;
-		use_edid_mode = true;
-	}
-
-	/*
-	 * TODO 1: if no default mode is present, postpone running the config
-	 * until after the LCDC channel is initialized.
-	 * TODO 2: consider registering the HDMI platform device from the LCDC
-	 * driver.
-	 */
-	if (!found && hdmi->entity.def_mode.xres != 0) {
-		found = &hdmi->entity.def_mode;
-		found_rate_error = sh_hdmi_rate_error(hdmi, found, hdmi_rate,
-						      parent_rate);
-	}
-
-	/* No cookie today */
-	if (!found)
-		return -ENXIO;
-
-	if (found->xres == 640 && found->yres == 480 && found->refresh == 60)
-		hdmi->preprogrammed_vic = 1;
-	else if (found->xres == 720 && found->yres == 480 && found->refresh == 60)
-		hdmi->preprogrammed_vic = 2;
-	else if (found->xres == 720 && found->yres == 576 && found->refresh == 50)
-		hdmi->preprogrammed_vic = 17;
-	else if (found->xres == 1280 && found->yres == 720 && found->refresh == 60)
-		hdmi->preprogrammed_vic = 4;
-	else if (found->xres == 1920 && found->yres == 1080 && found->refresh == 24)
-		hdmi->preprogrammed_vic = 32;
-	else if (found->xres == 1920 && found->yres == 1080 && found->refresh == 50)
-		hdmi->preprogrammed_vic = 31;
-	else if (found->xres == 1920 && found->yres == 1080 && found->refresh == 60)
-		hdmi->preprogrammed_vic = 16;
-	else
-		hdmi->preprogrammed_vic = 0;
-
-	dev_dbg(hdmi->dev, "Using %s %s mode %ux%u@%uHz (%luHz), "
-		"clock error %luHz\n", use_edid_mode ? "EDID" : "default",
-		hdmi->preprogrammed_vic ? "VIC" : "external", found->xres,
-		found->yres, found->refresh, PICOS2KHZ(found->pixclock) * 1000,
-		found_rate_error);
-
-	hdmi->mode = *found;
-	sh_hdmi_external_video_param(hdmi);
-
-	return 0;
-}
-
-static irqreturn_t sh_hdmi_hotplug(int irq, void *dev_id)
-{
-	struct sh_hdmi *hdmi = dev_id;
-	u8 status1, status2, mask1, mask2;
-
-	/* mode_b and PLLA and PLLB reset */
-	hdmi_bit_set(hdmi, 0xFC, 0x2C, HDMI_SYSTEM_CTRL);
-
-	/* How long shall reset be held? */
-	udelay(10);
-
-	/* mode_b and PLLA and PLLB reset release */
-	hdmi_bit_set(hdmi, 0xFC, 0x20, HDMI_SYSTEM_CTRL);
-
-	status1 = hdmi_read(hdmi, HDMI_INTERRUPT_STATUS_1);
-	status2 = hdmi_read(hdmi, HDMI_INTERRUPT_STATUS_2);
-
-	mask1 = hdmi_read(hdmi, HDMI_INTERRUPT_MASK_1);
-	mask2 = hdmi_read(hdmi, HDMI_INTERRUPT_MASK_2);
-
-	/* Correct would be to ack only set bits, but the datasheet requires 0xff */
-	hdmi_write(hdmi, 0xFF, HDMI_INTERRUPT_STATUS_1);
-	hdmi_write(hdmi, 0xFF, HDMI_INTERRUPT_STATUS_2);
-
-	if (printk_ratelimit())
-		dev_dbg(hdmi->dev, "IRQ #%d: Status #1: 0x%x & 0x%x, #2: 0x%x & 0x%x\n",
-			irq, status1, mask1, status2, mask2);
-
-	if (!((status1 & mask1) | (status2 & mask2))) {
-		return IRQ_NONE;
-	} else if (status1 & 0xc0) {
-		u8 msens;
-
-		/* Datasheet specifies 10ms... */
-		udelay(500);
-
-		msens = hdmi_read(hdmi, HDMI_HOT_PLUG_MSENS_STATUS);
-		dev_dbg(hdmi->dev, "MSENS 0x%x\n", msens);
-		/* Check, if hot plug & MSENS pin status are both high */
-		if ((msens & 0xC0) == 0xC0) {
-			/* Display plug in */
-			hdmi->edid_segment_nr = 0;
-			hdmi->edid_block_addr = 0;
-			hdmi->edid_blocks = 0;
-			hdmi->hp_state = HDMI_HOTPLUG_CONNECTED;
-
-			/* Set EDID word address  */
-			hdmi_write(hdmi, 0x00, HDMI_EDID_WORD_ADDRESS);
-			/* Enable EDID interrupt */
-			hdmi_write(hdmi, 0xC6, HDMI_INTERRUPT_MASK_1);
-			/* Set EDID segment pointer - starts reading EDID */
-			hdmi_write(hdmi, 0x00, HDMI_EDID_SEGMENT_POINTER);
-		} else if (!(status1 & 0x80)) {
-			/* Display unplug, beware multiple interrupts */
-			if (hdmi->hp_state != HDMI_HOTPLUG_DISCONNECTED) {
-				hdmi->hp_state = HDMI_HOTPLUG_DISCONNECTED;
-				schedule_delayed_work(&hdmi->edid_work, 0);
-			}
-			/* display_off will switch back to mode_a */
-		}
-	} else if (status1 & 2) {
-		/* EDID error interrupt: retry */
-		/* Set EDID word address  */
-		hdmi_write(hdmi, hdmi->edid_block_addr, HDMI_EDID_WORD_ADDRESS);
-		/* Set EDID segment pointer */
-		hdmi_write(hdmi, hdmi->edid_segment_nr, HDMI_EDID_SEGMENT_POINTER);
-	} else if (status1 & 4) {
-		/* Disable EDID interrupt */
-		hdmi_write(hdmi, 0xC0, HDMI_INTERRUPT_MASK_1);
-		schedule_delayed_work(&hdmi->edid_work, msecs_to_jiffies(10));
-	}
-
-	return IRQ_HANDLED;
-}
-
-static int sh_hdmi_display_on(struct sh_mobile_lcdc_entity *entity)
-{
-	struct sh_hdmi *hdmi = entity_to_sh_hdmi(entity);
-
-	dev_dbg(hdmi->dev, "%s(%p): state %x\n", __func__, hdmi,
-		hdmi->hp_state);
-
-	/*
-	 * hp_state can be set to
-	 * HDMI_HOTPLUG_DISCONNECTED:	on monitor unplug
-	 * HDMI_HOTPLUG_CONNECTED:	on monitor plug-in
-	 * HDMI_HOTPLUG_EDID_DONE:	on EDID read completion
-	 */
-	if (hdmi->hp_state == HDMI_HOTPLUG_EDID_DONE) {
-		/* PS mode d->e. All functions are active */
-		hdmi_bit_set(hdmi, 0xFC, 0x80, HDMI_SYSTEM_CTRL);
-		dev_dbg(hdmi->dev, "HDMI running\n");
-	}
-
-	return hdmi->hp_state == HDMI_HOTPLUG_DISCONNECTED
-		? SH_MOBILE_LCDC_DISPLAY_DISCONNECTED
-		: SH_MOBILE_LCDC_DISPLAY_CONNECTED;
-}
-
-static void sh_hdmi_display_off(struct sh_mobile_lcdc_entity *entity)
-{
-	struct sh_hdmi *hdmi = entity_to_sh_hdmi(entity);
-
-	dev_dbg(hdmi->dev, "%s(%p)\n", __func__, hdmi);
-	/* PS mode e->a */
-	hdmi_bit_set(hdmi, 0xFC, 0x10, HDMI_SYSTEM_CTRL);
-}
-
-static const struct sh_mobile_lcdc_entity_ops sh_hdmi_ops = {
-	.display_on = sh_hdmi_display_on,
-	.display_off = sh_hdmi_display_off,
-};
-
-/**
- * sh_hdmi_clk_configure() - set HDMI clock frequency and enable the clock
- * @hdmi:		driver context
- * @hdmi_rate:		HDMI clock frequency in Hz
- * @parent_rate:	if != 0 - set parent clock rate for optimal precision
- * return:		configured positive rate if successful
- *			0 if couldn't set the rate, but managed to enable the
- *			clock, negative error, if couldn't enable the clock
- */
-static long sh_hdmi_clk_configure(struct sh_hdmi *hdmi, unsigned long hdmi_rate,
-				  unsigned long parent_rate)
-{
-	int ret;
-
-	if (parent_rate && clk_get_parent(hdmi->hdmi_clk)) {
-		ret = clk_set_rate(clk_get_parent(hdmi->hdmi_clk), parent_rate);
-		if (ret < 0) {
-			dev_warn(hdmi->dev, "Cannot set parent rate %ld: %d\n", parent_rate, ret);
-			hdmi_rate = clk_round_rate(hdmi->hdmi_clk, hdmi_rate);
-		} else {
-			dev_dbg(hdmi->dev, "HDMI set parent frequency %lu\n", parent_rate);
-		}
-	}
-
-	ret = clk_set_rate(hdmi->hdmi_clk, hdmi_rate);
-	if (ret < 0) {
-		dev_warn(hdmi->dev, "Cannot set rate %ld: %d\n", hdmi_rate, ret);
-		hdmi_rate = 0;
-	} else {
-		dev_dbg(hdmi->dev, "HDMI set frequency %lu\n", hdmi_rate);
-	}
-
-	return hdmi_rate;
-}
-
-/* Hotplug interrupt occurred, read EDID */
-static void sh_hdmi_edid_work_fn(struct work_struct *work)
-{
-	struct sh_hdmi *hdmi = container_of(work, struct sh_hdmi, edid_work.work);
-	struct sh_mobile_lcdc_chan *ch = hdmi->entity.lcdc;
-	int ret;
-
-	dev_dbg(hdmi->dev, "%s(%p): begin, hotplug status %d\n", __func__, hdmi,
-		hdmi->hp_state);
-
-	if (hdmi->hp_state == HDMI_HOTPLUG_CONNECTED) {
-		unsigned long parent_rate = 0, hdmi_rate;
-
-		ret = sh_hdmi_read_edid(hdmi, &hdmi_rate, &parent_rate);
-		if (ret < 0)
-			goto out;
-
-		hdmi->hp_state = HDMI_HOTPLUG_EDID_DONE;
-
-		/* Reconfigure the clock */
-		ret = sh_hdmi_clk_configure(hdmi, hdmi_rate, parent_rate);
-		if (ret < 0)
-			goto out;
-
-		msleep(10);
-		sh_hdmi_configure(hdmi);
-		/* Switched to another (d) power-save mode */
-		msleep(10);
-
-		if (ch && ch->notify)
-			ch->notify(ch, SH_MOBILE_LCDC_EVENT_DISPLAY_CONNECT,
-				   &hdmi->mode, &hdmi->monspec);
-	} else {
-		hdmi->monspec.modedb_len = 0;
-		fb_destroy_modedb(hdmi->monspec.modedb);
-		hdmi->monspec.modedb = NULL;
-
-		if (ch && ch->notify)
-			ch->notify(ch, SH_MOBILE_LCDC_EVENT_DISPLAY_DISCONNECT,
-				   NULL, NULL);
-
-		ret = 0;
-	}
-
-out:
-	if (ret < 0 && ret != -EAGAIN)
-		hdmi->hp_state = HDMI_HOTPLUG_DISCONNECTED;
-
-	dev_dbg(hdmi->dev, "%s(%p): end\n", __func__, hdmi);
-}
-
-static void sh_hdmi_htop1_init(struct sh_hdmi *hdmi)
-{
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_MODE);
-	hdmi_htop1_write(hdmi, 0x0000000b, 0x0010);
-	hdmi_htop1_write(hdmi, 0x00006710, HDMI_HTOP1_HTOP_DCL_FRC_MODE);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y1_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y1_2);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB1_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB1_2);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR1_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR1_2);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y2_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_Y2_2);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB2_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CB2_2);
-	hdmi_htop1_write(hdmi, 0x01020406, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR2_1);
-	hdmi_htop1_write(hdmi, 0x07080806, HDMI_HTOP1_HTOP_DCL_FIL_PARA_CR2_2);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_Y1);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_CB1);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_CR1);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_Y2);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_CB2);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_HTOP_DCL_COR_PARA_CR2);
-	hdmi_htop1_write(hdmi, 0x00000008, HDMI_HTOP1_CURRENT);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_TISEMP0_1);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_TISEMP2_C);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_PHY_TEST_MODE);
-	hdmi_htop1_write(hdmi, 0x00000081, HDMI_HTOP1_TISIDRV);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_PLLBW);
-	hdmi_htop1_write(hdmi, 0x0000000f, HDMI_HTOP1_TISEN);
-	hdmi_htop1_write(hdmi, 0x0000000f, HDMI_HTOP1_TISDREN);
-	hdmi_htop1_write(hdmi, 0x00000003, HDMI_HTOP1_ENABLE_SELECTOR);
-	hdmi_htop1_write(hdmi, 0x00000001, HDMI_HTOP1_MACRO_RESET);
-	hdmi_htop1_write(hdmi, 0x00000016, HDMI_HTOP1_CISRANGE);
-	msleep(100);
-	hdmi_htop1_write(hdmi, 0x00000001, HDMI_HTOP1_ENABLE_SELECTOR);
-	msleep(100);
-	hdmi_htop1_write(hdmi, 0x00000003, HDMI_HTOP1_ENABLE_SELECTOR);
-	hdmi_htop1_write(hdmi, 0x00000001, HDMI_HTOP1_MACRO_RESET);
-	hdmi_htop1_write(hdmi, 0x0000000f, HDMI_HTOP1_TISEN);
-	hdmi_htop1_write(hdmi, 0x0000000f, HDMI_HTOP1_TISDREN);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_VIDEO_INPUT);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_CLK_TO_PHY);
-	hdmi_htop1_write(hdmi, 0x00000000, HDMI_HTOP1_VIDEO_INPUT2);
-	hdmi_htop1_write(hdmi, 0x0000000a, HDMI_HTOP1_CLK_SET);
-}
-
-static int __init sh_hdmi_probe(struct platform_device *pdev)
-{
-	struct sh_mobile_hdmi_info *pdata = dev_get_platdata(&pdev->dev);
-	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	struct resource *htop1_res;
-	int irq = platform_get_irq(pdev, 0), ret;
-	struct sh_hdmi *hdmi;
-	long rate;
-
-	if (!res || !pdata || irq < 0)
-		return -ENODEV;
-
-	htop1_res = NULL;
-	if (pdata->flags & HDMI_HAS_HTOP1) {
-		htop1_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-		if (!htop1_res) {
-			dev_err(&pdev->dev, "htop1 needs register base\n");
-			return -EINVAL;
-		}
-	}
-
-	hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
-	if (!hdmi) {
-		dev_err(&pdev->dev, "Cannot allocate device data\n");
-		return -ENOMEM;
-	}
-
-	hdmi->dev = &pdev->dev;
-	hdmi->entity.owner = THIS_MODULE;
-	hdmi->entity.ops = &sh_hdmi_ops;
-	hdmi->irq = irq;
-
-	hdmi->hdmi_clk = clk_get(&pdev->dev, "ick");
-	if (IS_ERR(hdmi->hdmi_clk)) {
-		ret = PTR_ERR(hdmi->hdmi_clk);
-		dev_err(&pdev->dev, "Unable to get clock: %d\n", ret);
-		return ret;
-	}
-
-	/* select register access functions */
-	if (pdata->flags & HDMI_32BIT_REG) {
-		hdmi->write	= __hdmi_write32;
-		hdmi->read	= __hdmi_read32;
-	} else {
-		hdmi->write	= __hdmi_write8;
-		hdmi->read	= __hdmi_read8;
-	}
-
-	/* An arbitrary relaxed pixclock just to get things started: from standard 480p */
-	rate = clk_round_rate(hdmi->hdmi_clk, PICOS2KHZ(37037));
-	if (rate > 0)
-		rate = sh_hdmi_clk_configure(hdmi, rate, 0);
-
-	if (rate < 0) {
-		ret = rate;
-		goto erate;
-	}
-
-	ret = clk_prepare_enable(hdmi->hdmi_clk);
-	if (ret < 0) {
-		dev_err(hdmi->dev, "Cannot enable clock: %d\n", ret);
-		goto erate;
-	}
-
-	dev_dbg(&pdev->dev, "Enabled HDMI clock at %luHz\n", rate);
-
-	if (!request_mem_region(res->start, resource_size(res), dev_name(&pdev->dev))) {
-		dev_err(&pdev->dev, "HDMI register region already claimed\n");
-		ret = -EBUSY;
-		goto ereqreg;
-	}
-
-	hdmi->base = ioremap(res->start, resource_size(res));
-	if (!hdmi->base) {
-		dev_err(&pdev->dev, "HDMI register region already claimed\n");
-		ret = -ENOMEM;
-		goto emap;
-	}
-
-	platform_set_drvdata(pdev, &hdmi->entity);
-
-	INIT_DELAYED_WORK(&hdmi->edid_work, sh_hdmi_edid_work_fn);
-
-	pm_runtime_enable(&pdev->dev);
-	pm_runtime_get_sync(&pdev->dev);
-
-	/* init interrupt polarity */
-	if (pdata->flags & HDMI_OUTPUT_PUSH_PULL)
-		hdmi_bit_set(hdmi, 0x02, 0x02, HDMI_SYSTEM_CTRL);
-
-	if (pdata->flags & HDMI_OUTPUT_POLARITY_HI)
-		hdmi_bit_set(hdmi, 0x01, 0x01, HDMI_SYSTEM_CTRL);
-
-	/* enable htop1 register if needed */
-	if (htop1_res) {
-		hdmi->htop1 = ioremap(htop1_res->start, resource_size(htop1_res));
-		if (!hdmi->htop1) {
-			dev_err(&pdev->dev, "control register region already claimed\n");
-			ret = -ENOMEM;
-			goto emap_htop1;
-		}
-		sh_hdmi_htop1_init(hdmi);
-	}
-
-	/* Product and revision IDs are 0 in sh-mobile version */
-	dev_info(&pdev->dev, "Detected HDMI controller 0x%x:0x%x\n",
-		 hdmi_read(hdmi, HDMI_PRODUCT_ID), hdmi_read(hdmi, HDMI_REVISION_ID));
-
-	ret = request_irq(irq, sh_hdmi_hotplug, 0,
-			  dev_name(&pdev->dev), hdmi);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Unable to request irq: %d\n", ret);
-		goto ereqirq;
-	}
-
-	ret = snd_soc_register_codec(&pdev->dev,
-			&soc_codec_dev_sh_hdmi, &sh_hdmi_dai, 1);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "codec registration failed\n");
-		goto ecodec;
-	}
-
-	return 0;
-
-ecodec:
-	free_irq(irq, hdmi);
-ereqirq:
-	if (hdmi->htop1)
-		iounmap(hdmi->htop1);
-emap_htop1:
-	pm_runtime_put(&pdev->dev);
-	pm_runtime_disable(&pdev->dev);
-	iounmap(hdmi->base);
-emap:
-	release_mem_region(res->start, resource_size(res));
-ereqreg:
-	clk_disable_unprepare(hdmi->hdmi_clk);
-erate:
-	clk_put(hdmi->hdmi_clk);
-
-	return ret;
-}
-
-static int __exit sh_hdmi_remove(struct platform_device *pdev)
-{
-	struct sh_hdmi *hdmi = entity_to_sh_hdmi(platform_get_drvdata(pdev));
-	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	snd_soc_unregister_codec(&pdev->dev);
-
-	/* No new work will be scheduled, wait for running ISR */
-	free_irq(hdmi->irq, hdmi);
-	/* Wait for already scheduled work */
-	cancel_delayed_work_sync(&hdmi->edid_work);
-	pm_runtime_put(&pdev->dev);
-	pm_runtime_disable(&pdev->dev);
-	clk_disable_unprepare(hdmi->hdmi_clk);
-	clk_put(hdmi->hdmi_clk);
-	if (hdmi->htop1)
-		iounmap(hdmi->htop1);
-	iounmap(hdmi->base);
-	release_mem_region(res->start, resource_size(res));
-
-	return 0;
-}
-
-static int sh_hdmi_suspend(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_hdmi *hdmi = entity_to_sh_hdmi(platform_get_drvdata(pdev));
-
-	disable_irq(hdmi->irq);
-	/* Wait for already scheduled work */
-	cancel_delayed_work_sync(&hdmi->edid_work);
-	return 0;
-}
-
-static int sh_hdmi_resume(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_mobile_hdmi_info *pdata = dev_get_platdata(dev);
-	struct sh_hdmi *hdmi = entity_to_sh_hdmi(platform_get_drvdata(pdev));
-
-	/* Re-init interrupt polarity */
-	if (pdata->flags & HDMI_OUTPUT_PUSH_PULL)
-		hdmi_bit_set(hdmi, 0x02, 0x02, HDMI_SYSTEM_CTRL);
-
-	if (pdata->flags & HDMI_OUTPUT_POLARITY_HI)
-		hdmi_bit_set(hdmi, 0x01, 0x01, HDMI_SYSTEM_CTRL);
-
-	/* Re-init htop1 */
-	if (hdmi->htop1)
-		sh_hdmi_htop1_init(hdmi);
-
-	/* Now it's safe to enable interrupts again */
-	enable_irq(hdmi->irq);
-	return 0;
-}
-
-static const struct dev_pm_ops sh_hdmi_pm_ops = {
-	.suspend	= sh_hdmi_suspend,
-	.resume		= sh_hdmi_resume,
-};
-
-static struct platform_driver sh_hdmi_driver = {
-	.remove		= __exit_p(sh_hdmi_remove),
-	.driver = {
-		.name	= "sh-mobile-hdmi",
-		.pm	= &sh_hdmi_pm_ops,
-	},
-};
-
-module_platform_driver_probe(sh_hdmi_driver, sh_hdmi_probe);
-
-MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
-MODULE_DESCRIPTION("SuperH / ARM-shmobile HDMI driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/video/fbdev/simplefb.c b/drivers/video/fbdev/simplefb.c
index 52c5c7e63b52..48ccf6db62a2 100644
--- a/drivers/video/fbdev/simplefb.c
+++ b/drivers/video/fbdev/simplefb.c
@@ -28,7 +28,10 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/parser.h>
+#include <linux/regulator/consumer.h>
 
 static struct fb_fix_screeninfo simplefb_fix = {
 	.id		= "simple",
@@ -174,6 +177,10 @@ struct simplefb_par {
 	int clk_count;
 	struct clk **clks;
 #endif
+#if defined CONFIG_OF && defined CONFIG_REGULATOR
+	u32 regulator_count;
+	struct regulator **regulators;
+#endif
 };
 
 #if defined CONFIG_OF && defined CONFIG_COMMON_CLK
@@ -269,6 +276,110 @@ static int simplefb_clocks_init(struct simplefb_par *par,
 static void simplefb_clocks_destroy(struct simplefb_par *par) { }
 #endif
 
+#if defined CONFIG_OF && defined CONFIG_REGULATOR
+
+#define SUPPLY_SUFFIX "-supply"
+
+/*
+ * Regulator handling code.
+ *
+ * Here we handle the num-supplies and vin*-supply properties of our
+ * "simple-framebuffer" dt node. This is necessary so that we can make sure
+ * that any regulators needed by the display hardware that the bootloader
+ * set up for us (and for which it provided a simplefb dt node), stay up,
+ * for the life of the simplefb driver.
+ *
+ * When the driver unloads, we cleanly disable, and then release the
+ * regulators.
+ *
+ * We only complain about errors here, no action is taken as the most likely
+ * error can only happen due to a mismatch between the bootloader which set
+ * up simplefb, and the regulator definitions in the device tree. Chances are
+ * that there are no adverse effects, and if there are, a clean teardown of
+ * the fb probe will not help us much either. So just complain and carry on,
+ * and hope that the user actually gets a working fb at the end of things.
+ */
+static int simplefb_regulators_init(struct simplefb_par *par,
+	struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct property *prop;
+	struct regulator *regulator;
+	const char *p;
+	int count = 0, i = 0, ret;
+
+	if (dev_get_platdata(&pdev->dev) || !np)
+		return 0;
+
+	/* Count the number of regulator supplies */
+	for_each_property_of_node(np, prop) {
+		p = strstr(prop->name, SUPPLY_SUFFIX);
+		if (p && p != prop->name)
+			count++;
+	}
+
+	if (!count)
+		return 0;
+
+	par->regulators = devm_kcalloc(&pdev->dev, count,
+				       sizeof(struct regulator *), GFP_KERNEL);
+	if (!par->regulators)
+		return -ENOMEM;
+
+	/* Get all the regulators */
+	for_each_property_of_node(np, prop) {
+		char name[32]; /* 32 is max size of property name */
+
+		p = strstr(prop->name, SUPPLY_SUFFIX);
+		if (!p || p == prop->name)
+			continue;
+
+		strlcpy(name, prop->name,
+			strlen(prop->name) - strlen(SUPPLY_SUFFIX) + 1);
+		regulator = devm_regulator_get_optional(&pdev->dev, name);
+		if (IS_ERR(regulator)) {
+			if (PTR_ERR(regulator) == -EPROBE_DEFER)
+				return -EPROBE_DEFER;
+			dev_err(&pdev->dev, "regulator %s not found: %ld\n",
+				name, PTR_ERR(regulator));
+			continue;
+		}
+		par->regulators[i++] = regulator;
+	}
+	par->regulator_count = i;
+
+	/* Enable all the regulators */
+	for (i = 0; i < par->regulator_count; i++) {
+		ret = regulator_enable(par->regulators[i]);
+		if (ret) {
+			dev_err(&pdev->dev,
+				"failed to enable regulator %d: %d\n",
+				i, ret);
+			devm_regulator_put(par->regulators[i]);
+			par->regulators[i] = NULL;
+		}
+	}
+
+	return 0;
+}
+
+static void simplefb_regulators_destroy(struct simplefb_par *par)
+{
+	int i;
+
+	if (!par->regulators)
+		return;
+
+	for (i = 0; i < par->regulator_count; i++)
+		if (par->regulators[i])
+			regulator_disable(par->regulators[i]);
+}
+#else
+static int simplefb_regulators_init(struct simplefb_par *par,
+	struct platform_device *pdev) { return 0; }
+static void simplefb_regulators_destroy(struct simplefb_par *par) { }
+#endif
+
 static int simplefb_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -340,6 +451,10 @@ static int simplefb_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto error_unmap;
 
+	ret = simplefb_regulators_init(par, pdev);
+	if (ret < 0)
+		goto error_clocks;
+
 	dev_info(&pdev->dev, "framebuffer at 0x%lx, 0x%x bytes, mapped to 0x%p\n",
 			     info->fix.smem_start, info->fix.smem_len,
 			     info->screen_base);
@@ -351,13 +466,15 @@ static int simplefb_probe(struct platform_device *pdev)
 	ret = register_framebuffer(info);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Unable to register simplefb: %d\n", ret);
-		goto error_clocks;
+		goto error_regulators;
 	}
 
 	dev_info(&pdev->dev, "fb%d: simplefb registered!\n", info->node);
 
 	return 0;
 
+error_regulators:
+	simplefb_regulators_destroy(par);
 error_clocks:
 	simplefb_clocks_destroy(par);
 error_unmap:
@@ -373,6 +490,7 @@ static int simplefb_remove(struct platform_device *pdev)
 	struct simplefb_par *par = info->par;
 
 	unregister_framebuffer(info);
+	simplefb_regulators_destroy(par);
 	simplefb_clocks_destroy(par);
 	framebuffer_release(info);
 
diff --git a/drivers/video/fbdev/sm712fb.c b/drivers/video/fbdev/sm712fb.c
index 629bfa2d2f51..86ae1d4556fc 100644
--- a/drivers/video/fbdev/sm712fb.c
+++ b/drivers/video/fbdev/sm712fb.c
@@ -28,9 +28,7 @@
 #include <linux/console.h>
 #include <linux/screen_info.h>
 
-#ifdef CONFIG_PM
 #include <linux/pm.h>
-#endif
 
 #include "sm712.h"
 
@@ -1545,8 +1543,7 @@ static void smtcfb_pci_remove(struct pci_dev *pdev)
 	pci_disable_device(pdev);
 }
 
-#ifdef CONFIG_PM
-static int smtcfb_pci_suspend(struct device *device)
+static int __maybe_unused smtcfb_pci_suspend(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
 	struct smtcfb_info *sfb;
@@ -1569,7 +1566,7 @@ static int smtcfb_pci_suspend(struct device *device)
 	return 0;
 }
 
-static int smtcfb_pci_resume(struct device *device)
+static int __maybe_unused smtcfb_pci_resume(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
 	struct smtcfb_info *sfb;
@@ -1610,20 +1607,13 @@ static int smtcfb_pci_resume(struct device *device)
 }
 
 static SIMPLE_DEV_PM_OPS(sm7xx_pm_ops, smtcfb_pci_suspend, smtcfb_pci_resume);
-#define SM7XX_PM_OPS (&sm7xx_pm_ops)
-
-#else  /* !CONFIG_PM */
-
-#define SM7XX_PM_OPS NULL
-
-#endif /* !CONFIG_PM */
 
 static struct pci_driver smtcfb_driver = {
 	.name = "smtcfb",
 	.id_table = smtcfb_pci_table,
 	.probe = smtcfb_pci_probe,
 	.remove = smtcfb_pci_remove,
-	.driver.pm  = SM7XX_PM_OPS,
+	.driver.pm  = &sm7xx_pm_ops,
 };
 
 static int __init sm712fb_init(void)
diff --git a/drivers/video/fbdev/via/via-gpio.c b/drivers/video/fbdev/via/via-gpio.c
index 6f433b8cee12..1e89c3434071 100644
--- a/drivers/video/fbdev/via/via-gpio.c
+++ b/drivers/video/fbdev/via/via-gpio.c
@@ -6,7 +6,7 @@
  */
 
 #include <linux/spinlock.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 #include <linux/via-core.h>
 #include <linux/via-gpio.h>
@@ -83,9 +83,7 @@ struct viafb_gpio_cfg {
 static void via_gpio_set(struct gpio_chip *chip, unsigned int nr,
 			 int value)
 {
-	struct viafb_gpio_cfg *cfg = container_of(chip,
-						  struct viafb_gpio_cfg,
-						  gpio_chip);
+	struct viafb_gpio_cfg *cfg = gpiochip_get_data(chip);
 	u8 reg;
 	struct viafb_gpio *gpio;
 	unsigned long flags;
@@ -115,9 +113,7 @@ static int via_gpio_dir_out(struct gpio_chip *chip, unsigned int nr,
  */
 static int via_gpio_dir_input(struct gpio_chip *chip, unsigned int nr)
 {
-	struct viafb_gpio_cfg *cfg = container_of(chip,
-						  struct viafb_gpio_cfg,
-						  gpio_chip);
+	struct viafb_gpio_cfg *cfg = gpiochip_get_data(chip);
 	struct viafb_gpio *gpio;
 	unsigned long flags;
 
@@ -131,9 +127,7 @@ static int via_gpio_dir_input(struct gpio_chip *chip, unsigned int nr)
 
 static int via_gpio_get(struct gpio_chip *chip, unsigned int nr)
 {
-	struct viafb_gpio_cfg *cfg = container_of(chip,
-						  struct viafb_gpio_cfg,
-						  gpio_chip);
+	struct viafb_gpio_cfg *cfg = gpiochip_get_data(chip);
 	u8 reg;
 	struct viafb_gpio *gpio;
 	unsigned long flags;
@@ -142,7 +136,7 @@ static int via_gpio_get(struct gpio_chip *chip, unsigned int nr)
 	gpio = cfg->active_gpios[nr];
 	reg = via_read_reg(VIASR, gpio->vg_port_index);
 	spin_unlock_irqrestore(&cfg->vdev->reg_lock, flags);
-	return reg & (0x04 << gpio->vg_mask_shift);
+	return !!(reg & (0x04 << gpio->vg_mask_shift));
 }
 
 
@@ -255,7 +249,8 @@ static int viafb_gpio_probe(struct platform_device *platdev)
 	 * Get registered.
 	 */
 	viafb_gpio_config.gpio_chip.base = -1;  /* Dynamic */
-	ret = gpiochip_add(&viafb_gpio_config.gpio_chip);
+	ret = gpiochip_add_data(&viafb_gpio_config.gpio_chip,
+				&viafb_gpio_config);
 	if (ret) {
 		printk(KERN_ERR "viafb: failed to add gpios (%d)\n", ret);
 		viafb_gpio_config.gpio_chip.ngpio = 0;
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 7efc32945810..0c3691f46575 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -209,8 +209,8 @@ static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
 	 */
 	if (vb->num_pfns != 0)
 		tell_host(vb, vb->deflate_vq);
-	mutex_unlock(&vb->balloon_lock);
 	release_pages_balloon(vb);
+	mutex_unlock(&vb->balloon_lock);
 	return num_freed_pages;
 }
 
@@ -388,7 +388,7 @@ static int init_vqs(struct virtio_balloon *vb)
 {
 	struct virtqueue *vqs[3];
 	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
-	const char *names[] = { "inflate", "deflate", "stats" };
+	static const char * const names[] = { "inflate", "deflate", "stats" };
 	int err, nvqs;
 
 	/*
diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c
index c96944b59856..350a2a5a49db 100644
--- a/drivers/virtio/virtio_input.c
+++ b/drivers/virtio/virtio_input.c
@@ -170,7 +170,7 @@ static int virtinput_init_vqs(struct virtio_input *vi)
 	struct virtqueue *vqs[2];
 	vq_callback_t *cbs[] = { virtinput_recv_events,
 				 virtinput_recv_status };
-	static const char *names[] = { "events", "status" };
+	static const char * const names[] = { "events", "status" };
 	int err;
 
 	err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names);
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index f499d9da7237..745c6ee1bb3e 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -482,7 +482,7 @@ error_available:
 static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		       struct virtqueue *vqs[],
 		       vq_callback_t *callbacks[],
-		       const char *names[])
+		       const char * const names[])
 {
 	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
 	unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index 78f804af6c20..f6bed86c17f9 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -296,7 +296,7 @@ void vp_del_vqs(struct virtio_device *vdev)
 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			      struct virtqueue *vqs[],
 			      vq_callback_t *callbacks[],
-			      const char *names[],
+			      const char * const names[],
 			      bool use_msix,
 			      bool per_vq_vectors)
 {
@@ -376,7 +376,7 @@ error_find:
 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		struct virtqueue *vqs[],
 		vq_callback_t *callbacks[],
-		const char *names[])
+		const char * const names[])
 {
 	int err;
 
@@ -545,6 +545,7 @@ err_enable_device:
 static void virtio_pci_remove(struct pci_dev *pci_dev)
 {
 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
+	struct device *dev = get_device(&vp_dev->vdev.dev);
 
 	unregister_virtio_device(&vp_dev->vdev);
 
@@ -554,6 +555,7 @@ static void virtio_pci_remove(struct pci_dev *pci_dev)
 		virtio_pci_modern_remove(vp_dev);
 
 	pci_disable_device(pci_dev);
+	put_device(dev);
 }
 
 static struct pci_driver virtio_pci_driver = {
diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
index b976d968e793..2cc252270b2d 100644
--- a/drivers/virtio/virtio_pci_common.h
+++ b/drivers/virtio/virtio_pci_common.h
@@ -139,7 +139,7 @@ void vp_del_vqs(struct virtio_device *vdev);
 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		       struct virtqueue *vqs[],
 		       vq_callback_t *callbacks[],
-		       const char *names[]);
+		       const char * const names[]);
 const char *vp_bus_name(struct virtio_device *vdev);
 
 /* Setup the affinity for a virtqueue:
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index 8e5cf194cc0b..c0c11fad4611 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -418,7 +418,7 @@ err_new_queue:
 static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			      struct virtqueue *vqs[],
 			      vq_callback_t *callbacks[],
-			      const char *names[])
+			      const char * const names[])
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	struct virtqueue *vq;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index ee663c458b20..e12e385f7ac3 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -517,10 +517,10 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
 	/* If we expect an interrupt for the next entry, tell host
 	 * by writing event index and flush out the write before
 	 * the read in the next get_buf call. */
-	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
-		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);
-		virtio_mb(vq->weak_barriers);
-	}
+	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
+		virtio_store_mb(vq->weak_barriers,
+				&vring_used_event(&vq->vring),
+				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
 
 #ifdef DEBUG
 	vq->last_add_time_valid = false;
@@ -653,8 +653,11 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	}
 	/* TODO: tune this threshold */
 	bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
-	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);
-	virtio_mb(vq->weak_barriers);
+
+	virtio_store_mb(vq->weak_barriers,
+			&vring_used_event(&vq->vring),
+			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
+
 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index 1c427beffadd..4f0e7be0da34 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -46,6 +46,13 @@ config WATCHDOG_NOWAYOUT
 	  get killed. If you say Y here, the watchdog cannot be stopped once
 	  it has been started.
 
+config WATCHDOG_SYSFS
+	bool "Read different watchdog information through sysfs"
+	default n
+	help
+	  Say Y here if you want to enable watchdog device status read through
+	  sysfs attributes.
+
 #
 # General Watchdog drivers
 #
@@ -135,6 +142,16 @@ config MENF21BMC_WATCHDOG
 	  This driver can also be built as a module. If so the module
 	  will be called menf21bmc_wdt.
 
+config TANGOX_WATCHDOG
+	tristate "Sigma Designs SMP86xx/SMP87xx watchdog"
+	select WATCHDOG_CORE
+	depends on ARCH_TANGOX || COMPILE_TEST
+	help
+	  Support for the watchdog in Sigma Designs SMP86xx (tango3)
+	  and SMP87xx (tango4) family chips.
+
+	  This driver can be built as a module. The module name is tangox_wdt.
+
 config WM831X_WATCHDOG
 	tristate "WM831x watchdog"
 	depends on MFD_WM831X
@@ -161,6 +178,17 @@ config XILINX_WATCHDOG
 	  To compile this driver as a module, choose M here: the
 	  module will be called of_xilinx_wdt.
 
+config ZIIRAVE_WATCHDOG
+	tristate "Zodiac RAVE Watchdog Timer"
+	depends on I2C
+	select WATCHDOG_CORE
+	help
+	  Watchdog driver for the Zodiac Aerospace RAVE Switch Watchdog
+	  Processor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ziirave_wdt.
+
 # ALPHA Architecture
 
 # ARM Architecture
@@ -173,6 +201,16 @@ config ARM_SP805_WATCHDOG
 	  ARM Primecell SP805 Watchdog timer. This will reboot your system when
 	  the timeout is reached.
 
+config ASM9260_WATCHDOG
+	tristate "Alphascale ASM9260 watchdog"
+	depends on MACH_ASM9260
+	depends on OF
+	select WATCHDOG_CORE
+	select RESET_CONTROLLER
+	help
+	  Watchdog timer embedded into Alphascale asm9260 chips. This will reboot your
+	  system when the timeout is reached.
+
 config AT91RM9200_WATCHDOG
 	tristate "AT91RM9200 watchdog"
 	depends on SOC_AT91RM9200 && MFD_SYSCON
@@ -426,6 +464,16 @@ config NUC900_WATCHDOG
 	  To compile this driver as a module, choose M here: the
 	  module will be called nuc900_wdt.
 
+config TS4800_WATCHDOG
+	tristate "TS-4800 Watchdog"
+	depends on HAS_IOMEM && OF
+	select WATCHDOG_CORE
+	select MFD_SYSCON
+	help
+	  Technologic Systems TS-4800 has watchdog timer implemented in
+	  an external FPGA. Say Y here if you want to support for the
+	  watchdog timer on TS-4800 board.
+
 config TS72XX_WATCHDOG
 	tristate "TS-72XX SBC Watchdog"
 	depends on MACH_TS72XX
@@ -578,6 +626,16 @@ config LPC18XX_WATCHDOG
 	  To compile this driver as a module, choose M here: the
 	  module will be called lpc18xx_wdt.
 
+config ATLAS7_WATCHDOG
+	tristate "CSRatlas7 watchdog"
+	depends on ARCH_ATLAS7
+	help
+	  Say Y here to include Watchdog timer support for the watchdog
+	  existing on the CSRatlas7 series platforms.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called atlas7_wdt.
+
 # AVR32 Architecture
 
 config AT32AP700X_WDT
@@ -1345,6 +1403,13 @@ config RALINK_WDT
 	help
 	  Hardware driver for the Ralink SoC Watchdog Timer.
 
+config MT7621_WDT
+	tristate "Mediatek SoC watchdog"
+	select WATCHDOG_CORE
+	depends on SOC_MT7620 || SOC_MT7621
+	help
+	  Hardware driver for the Mediatek/Ralink MT7621/8 SoC Watchdog Timer.
+
 # PARISC Architecture
 
 # POWERPC Architecture
diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
index 53d4827ddfe1..f566753256ab 100644
--- a/drivers/watchdog/Makefile
+++ b/drivers/watchdog/Makefile
@@ -30,6 +30,7 @@ obj-$(CONFIG_USBPCWATCHDOG) += pcwd_usb.o
 
 # ARM Architecture
 obj-$(CONFIG_ARM_SP805_WATCHDOG) += sp805_wdt.o
+obj-$(CONFIG_ASM9260_WATCHDOG) += asm9260_wdt.o
 obj-$(CONFIG_AT91RM9200_WATCHDOG) += at91rm9200_wdt.o
 obj-$(CONFIG_AT91SAM9X_WATCHDOG) += at91sam9_wdt.o
 obj-$(CONFIG_CADENCE_WATCHDOG) += cadence_wdt.o
@@ -53,6 +54,7 @@ obj-$(CONFIG_RN5T618_WATCHDOG) += rn5t618_wdt.o
 obj-$(CONFIG_COH901327_WATCHDOG) += coh901327_wdt.o
 obj-$(CONFIG_STMP3XXX_RTC_WATCHDOG) += stmp3xxx_rtc_wdt.o
 obj-$(CONFIG_NUC900_WATCHDOG) += nuc900_wdt.o
+obj-$(CONFIG_TS4800_WATCHDOG) += ts4800_wdt.o
 obj-$(CONFIG_TS72XX_WATCHDOG) += ts72xx_wdt.o
 obj-$(CONFIG_IMX2_WDT) += imx2_wdt.o
 obj-$(CONFIG_UX500_WATCHDOG) += ux500_wdt.o
@@ -69,6 +71,7 @@ obj-$(CONFIG_MEDIATEK_WATCHDOG) += mtk_wdt.o
 obj-$(CONFIG_DIGICOLOR_WATCHDOG) += digicolor_wdt.o
 obj-$(CONFIG_LPC18XX_WATCHDOG) += lpc18xx_wdt.o
 obj-$(CONFIG_BCM7038_WDT) += bcm7038_wdt.o
+obj-$(CONFIG_ATLAS7_WATCHDOG) += atlas7_wdt.o
 
 # AVR32 Architecture
 obj-$(CONFIG_AT32AP700X_WDT) += at32ap700x_wdt.o
@@ -149,6 +152,7 @@ octeon-wdt-y := octeon-wdt-main.o octeon-wdt-nmi.o
 obj-$(CONFIG_LANTIQ_WDT) += lantiq_wdt.o
 obj-$(CONFIG_RALINK_WDT) += rt2880_wdt.o
 obj-$(CONFIG_IMGPDC_WDT) += imgpdc_wdt.o
+obj-$(CONFIG_MT7621_WDT) += mt7621_wdt.o
 
 # PARISC Architecture
 
@@ -187,8 +191,10 @@ obj-$(CONFIG_DA9055_WATCHDOG) += da9055_wdt.o
 obj-$(CONFIG_DA9062_WATCHDOG) += da9062_wdt.o
 obj-$(CONFIG_DA9063_WATCHDOG) += da9063_wdt.o
 obj-$(CONFIG_GPIO_WATCHDOG)	+= gpio_wdt.o
+obj-$(CONFIG_TANGOX_WATCHDOG) += tangox_wdt.o
 obj-$(CONFIG_WM831X_WATCHDOG) += wm831x_wdt.o
 obj-$(CONFIG_WM8350_WATCHDOG) += wm8350_wdt.o
 obj-$(CONFIG_MAX63XX_WATCHDOG) += max63xx_wdt.o
+obj-$(CONFIG_ZIIRAVE_WATCHDOG) += ziirave_wdt.o
 obj-$(CONFIG_SOFT_WATCHDOG) += softdog.o
 obj-$(CONFIG_MENF21BMC_WATCHDOG) += menf21bmc_wdt.o
diff --git a/drivers/watchdog/asm9260_wdt.c b/drivers/watchdog/asm9260_wdt.c
new file mode 100644
index 000000000000..c9686b2fdafd
--- /dev/null
+++ b/drivers/watchdog/asm9260_wdt.c
@@ -0,0 +1,403 @@
+/*
+ * Watchdog driver for Alphascale ASM9260.
+ *
+ * Copyright (c) 2014 Oleksij Rempel <linux@rempel-privat.de>
+ *
+ * Licensed under GPLv2 or later.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reboot.h>
+#include <linux/reset.h>
+#include <linux/watchdog.h>
+
+#define CLOCK_FREQ	1000000
+
+/* Watchdog Mode register */
+#define HW_WDMOD			0x00
+/* Wake interrupt. Set by HW, can't be cleared. */
+#define BM_MOD_WDINT			BIT(3)
+/* This bit set if timeout reached. Cleared by SW. */
+#define BM_MOD_WDTOF			BIT(2)
+/* HW Reset on timeout */
+#define BM_MOD_WDRESET			BIT(1)
+/* WD enable */
+#define BM_MOD_WDEN			BIT(0)
+
+/*
+ * Watchdog Timer Constant register
+ * Minimal value is 0xff, the meaning of this value
+ * depends on used clock: T = WDCLK * (0xff + 1) * 4
+ */
+#define HW_WDTC				0x04
+#define BM_WDTC_MAX(freq)		(0x7fffffff / (freq))
+
+/* Watchdog Feed register */
+#define HW_WDFEED			0x08
+
+/* Watchdog Timer Value register */
+#define HW_WDTV				0x0c
+
+#define ASM9260_WDT_DEFAULT_TIMEOUT	30
+
+enum asm9260_wdt_mode {
+	HW_RESET,
+	SW_RESET,
+	DEBUG,
+};
+
+struct asm9260_wdt_priv {
+	struct device		*dev;
+	struct watchdog_device	wdd;
+	struct clk		*clk;
+	struct clk		*clk_ahb;
+	struct reset_control	*rst;
+	struct notifier_block	restart_handler;
+
+	void __iomem		*iobase;
+	int			irq;
+	unsigned long		wdt_freq;
+	enum asm9260_wdt_mode	mode;
+};
+
+static int asm9260_wdt_feed(struct watchdog_device *wdd)
+{
+	struct asm9260_wdt_priv *priv = watchdog_get_drvdata(wdd);
+
+	iowrite32(0xaa, priv->iobase + HW_WDFEED);
+	iowrite32(0x55, priv->iobase + HW_WDFEED);
+
+	return 0;
+}
+
+static unsigned int asm9260_wdt_gettimeleft(struct watchdog_device *wdd)
+{
+	struct asm9260_wdt_priv *priv = watchdog_get_drvdata(wdd);
+	u32 counter;
+
+	counter = ioread32(priv->iobase + HW_WDTV);
+
+	return DIV_ROUND_CLOSEST(counter, priv->wdt_freq);
+}
+
+static int asm9260_wdt_updatetimeout(struct watchdog_device *wdd)
+{
+	struct asm9260_wdt_priv *priv = watchdog_get_drvdata(wdd);
+	u32 counter;
+
+	counter = wdd->timeout * priv->wdt_freq;
+
+	iowrite32(counter, priv->iobase + HW_WDTC);
+
+	return 0;
+}
+
+static int asm9260_wdt_enable(struct watchdog_device *wdd)
+{
+	struct asm9260_wdt_priv *priv = watchdog_get_drvdata(wdd);
+	u32 mode = 0;
+
+	if (priv->mode == HW_RESET)
+		mode = BM_MOD_WDRESET;
+
+	iowrite32(BM_MOD_WDEN | mode, priv->iobase + HW_WDMOD);
+
+	asm9260_wdt_updatetimeout(wdd);
+
+	asm9260_wdt_feed(wdd);
+
+	return 0;
+}
+
+static int asm9260_wdt_disable(struct watchdog_device *wdd)
+{
+	struct asm9260_wdt_priv *priv = watchdog_get_drvdata(wdd);
+
+	/* The only way to disable WD is to reset it. */
+	reset_control_assert(priv->rst);
+	reset_control_deassert(priv->rst);
+
+	return 0;
+}
+
+static int asm9260_wdt_settimeout(struct watchdog_device *wdd, unsigned int to)
+{
+	wdd->timeout = to;
+	asm9260_wdt_updatetimeout(wdd);
+
+	return 0;
+}
+
+static void asm9260_wdt_sys_reset(struct asm9260_wdt_priv *priv)
+{
+	/* init WD if it was not started */
+
+	iowrite32(BM_MOD_WDEN | BM_MOD_WDRESET, priv->iobase + HW_WDMOD);
+
+	iowrite32(0xff, priv->iobase + HW_WDTC);
+	/* first pass correct sequence */
+	asm9260_wdt_feed(&priv->wdd);
+	/*
+	 * Then write wrong pattern to the feed to trigger reset
+	 * ASAP.
+	 */
+	iowrite32(0xff, priv->iobase + HW_WDFEED);
+
+	mdelay(1000);
+}
+
+static irqreturn_t asm9260_wdt_irq(int irq, void *devid)
+{
+	struct asm9260_wdt_priv *priv = devid;
+	u32 stat;
+
+	stat = ioread32(priv->iobase + HW_WDMOD);
+	if (!(stat & BM_MOD_WDINT))
+		return IRQ_NONE;
+
+	if (priv->mode == DEBUG) {
+		dev_info(priv->dev, "Watchdog Timeout. Do nothing.\n");
+	} else {
+		dev_info(priv->dev, "Watchdog Timeout. Doing SW Reset.\n");
+		asm9260_wdt_sys_reset(priv);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int asm9260_restart_handler(struct notifier_block *this,
+				   unsigned long mode, void *cmd)
+{
+	struct asm9260_wdt_priv *priv =
+		container_of(this, struct asm9260_wdt_priv, restart_handler);
+
+	asm9260_wdt_sys_reset(priv);
+
+	return NOTIFY_DONE;
+}
+
+static const struct watchdog_info asm9260_wdt_ident = {
+	.options          =     WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING
+				| WDIOF_MAGICCLOSE,
+	.identity         =	"Alphascale asm9260 Watchdog",
+};
+
+static struct watchdog_ops asm9260_wdt_ops = {
+	.owner		= THIS_MODULE,
+	.start		= asm9260_wdt_enable,
+	.stop		= asm9260_wdt_disable,
+	.get_timeleft	= asm9260_wdt_gettimeleft,
+	.ping		= asm9260_wdt_feed,
+	.set_timeout	= asm9260_wdt_settimeout,
+};
+
+static int asm9260_wdt_get_dt_clks(struct asm9260_wdt_priv *priv)
+{
+	int err;
+	unsigned long clk;
+
+	priv->clk = devm_clk_get(priv->dev, "mod");
+	if (IS_ERR(priv->clk)) {
+		dev_err(priv->dev, "Failed to get \"mod\" clk\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	/* configure AHB clock */
+	priv->clk_ahb = devm_clk_get(priv->dev, "ahb");
+	if (IS_ERR(priv->clk_ahb)) {
+		dev_err(priv->dev, "Failed to get \"ahb\" clk\n");
+		return PTR_ERR(priv->clk_ahb);
+	}
+
+	err = clk_prepare_enable(priv->clk_ahb);
+	if (err) {
+		dev_err(priv->dev, "Failed to enable ahb_clk!\n");
+		return err;
+	}
+
+	err = clk_set_rate(priv->clk, CLOCK_FREQ);
+	if (err) {
+		clk_disable_unprepare(priv->clk_ahb);
+		dev_err(priv->dev, "Failed to set rate!\n");
+		return err;
+	}
+
+	err = clk_prepare_enable(priv->clk);
+	if (err) {
+		clk_disable_unprepare(priv->clk_ahb);
+		dev_err(priv->dev, "Failed to enable clk!\n");
+		return err;
+	}
+
+	/* wdt has internal divider */
+	clk = clk_get_rate(priv->clk);
+	if (!clk) {
+		clk_disable_unprepare(priv->clk);
+		clk_disable_unprepare(priv->clk_ahb);
+		dev_err(priv->dev, "Failed, clk is 0!\n");
+		return -EINVAL;
+	}
+
+	priv->wdt_freq = clk / 2;
+
+	return 0;
+}
+
+static void asm9260_wdt_get_dt_mode(struct asm9260_wdt_priv *priv)
+{
+	const char *tmp;
+	int ret;
+
+	/* default mode */
+	priv->mode = HW_RESET;
+
+	ret = of_property_read_string(priv->dev->of_node,
+				      "alphascale,mode", &tmp);
+	if (ret < 0)
+		return;
+
+	if (!strcmp(tmp, "hw"))
+		priv->mode = HW_RESET;
+	else if (!strcmp(tmp, "sw"))
+		priv->mode = SW_RESET;
+	else if (!strcmp(tmp, "debug"))
+		priv->mode = DEBUG;
+	else
+		dev_warn(priv->dev, "unknown reset-type: %s. Using default \"hw\" mode.",
+			 tmp);
+}
+
+static int asm9260_wdt_probe(struct platform_device *pdev)
+{
+	struct asm9260_wdt_priv *priv;
+	struct watchdog_device *wdd;
+	struct resource *res;
+	int ret;
+	const char * const mode_name[] = { "hw", "sw", "debug", };
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(struct asm9260_wdt_priv),
+			    GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = &pdev->dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->iobase = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->iobase))
+		return PTR_ERR(priv->iobase);
+
+	ret = asm9260_wdt_get_dt_clks(priv);
+	if (ret)
+		return ret;
+
+	priv->rst = devm_reset_control_get(&pdev->dev, "wdt_rst");
+	if (IS_ERR(priv->rst))
+		return PTR_ERR(priv->rst);
+
+	wdd = &priv->wdd;
+	wdd->info = &asm9260_wdt_ident;
+	wdd->ops = &asm9260_wdt_ops;
+	wdd->min_timeout = 1;
+	wdd->max_timeout = BM_WDTC_MAX(priv->wdt_freq);
+	wdd->parent = &pdev->dev;
+
+	watchdog_set_drvdata(wdd, priv);
+
+	/*
+	 * If 'timeout-sec' unspecified in devicetree, assume a 30 second
+	 * default, unless the max timeout is less than 30 seconds, then use
+	 * the max instead.
+	 */
+	wdd->timeout = ASM9260_WDT_DEFAULT_TIMEOUT;
+	watchdog_init_timeout(wdd, 0, &pdev->dev);
+
+	asm9260_wdt_get_dt_mode(priv);
+
+	if (priv->mode != HW_RESET)
+		priv->irq = platform_get_irq(pdev, 0);
+
+	if (priv->irq > 0) {
+		/*
+		 * Not all supported platforms specify an interrupt for the
+		 * watchdog, so let's make it optional.
+		 */
+		ret = devm_request_irq(&pdev->dev, priv->irq,
+				       asm9260_wdt_irq, 0, pdev->name, priv);
+		if (ret < 0)
+			dev_warn(&pdev->dev, "failed to request IRQ\n");
+	}
+
+	ret = watchdog_register_device(wdd);
+	if (ret)
+		goto clk_off;
+
+	platform_set_drvdata(pdev, priv);
+
+	priv->restart_handler.notifier_call = asm9260_restart_handler;
+	priv->restart_handler.priority = 128;
+	ret = register_restart_handler(&priv->restart_handler);
+	if (ret)
+		dev_warn(&pdev->dev, "cannot register restart handler\n");
+
+	dev_info(&pdev->dev, "Watchdog enabled (timeout: %d sec, mode: %s)\n",
+		 wdd->timeout, mode_name[priv->mode]);
+	return 0;
+
+clk_off:
+	clk_disable_unprepare(priv->clk);
+	clk_disable_unprepare(priv->clk_ahb);
+	return ret;
+}
+
+static void asm9260_wdt_shutdown(struct platform_device *pdev)
+{
+	struct asm9260_wdt_priv *priv = platform_get_drvdata(pdev);
+
+	asm9260_wdt_disable(&priv->wdd);
+}
+
+static int asm9260_wdt_remove(struct platform_device *pdev)
+{
+	struct asm9260_wdt_priv *priv = platform_get_drvdata(pdev);
+
+	asm9260_wdt_disable(&priv->wdd);
+
+	unregister_restart_handler(&priv->restart_handler);
+
+	watchdog_unregister_device(&priv->wdd);
+
+	clk_disable_unprepare(priv->clk);
+	clk_disable_unprepare(priv->clk_ahb);
+
+	return 0;
+}
+
+static const struct of_device_id asm9260_wdt_of_match[] = {
+	{ .compatible = "alphascale,asm9260-wdt"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, asm9260_wdt_of_match);
+
+static struct platform_driver asm9260_wdt_driver = {
+	.driver = {
+		.name = "asm9260-wdt",
+		.owner = THIS_MODULE,
+		.of_match_table	= asm9260_wdt_of_match,
+	},
+	.probe = asm9260_wdt_probe,
+	.remove = asm9260_wdt_remove,
+	.shutdown = asm9260_wdt_shutdown,
+};
+module_platform_driver(asm9260_wdt_driver);
+
+MODULE_DESCRIPTION("asm9260 WatchDog Timer Driver");
+MODULE_AUTHOR("Oleksij Rempel <linux@rempel-privat.de>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/atlas7_wdt.c b/drivers/watchdog/atlas7_wdt.c
new file mode 100644
index 000000000000..df6d9242a319
--- /dev/null
+++ b/drivers/watchdog/atlas7_wdt.c
@@ -0,0 +1,242 @@
+/*
+ * Watchdog driver for CSR Atlas7
+ *
+ * Copyright (c) 2015 Cambridge Silicon Radio Limited, a CSR plc group company.
+ *
+ * Licensed under GPLv2.
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/watchdog.h>
+
+#define ATLAS7_TIMER_WDT_INDEX		5
+#define ATLAS7_WDT_DEFAULT_TIMEOUT	20
+
+#define ATLAS7_WDT_CNT_CTRL	(0 + 4 * ATLAS7_TIMER_WDT_INDEX)
+#define ATLAS7_WDT_CNT_MATCH	(0x18 + 4 * ATLAS7_TIMER_WDT_INDEX)
+#define ATLAS7_WDT_CNT		(0x48 +  4 * ATLAS7_TIMER_WDT_INDEX)
+#define ATLAS7_WDT_CNT_EN	(BIT(0) | BIT(1))
+#define ATLAS7_WDT_EN		0x64
+
+static unsigned int timeout = ATLAS7_WDT_DEFAULT_TIMEOUT;
+static bool nowayout = WATCHDOG_NOWAYOUT;
+
+module_param(timeout, uint, 0);
+module_param(nowayout, bool, 0);
+
+MODULE_PARM_DESC(timeout, "Default watchdog timeout (in seconds)");
+MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
+			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+struct atlas7_wdog {
+	struct device *dev;
+	void __iomem *base;
+	unsigned long tick_rate;
+	struct clk *clk;
+};
+
+static unsigned int atlas7_wdt_gettimeleft(struct watchdog_device *wdd)
+{
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+	u32 counter, match, delta;
+
+	counter = readl(wdt->base + ATLAS7_WDT_CNT);
+	match = readl(wdt->base + ATLAS7_WDT_CNT_MATCH);
+	delta = match - counter;
+
+	return  delta / wdt->tick_rate;
+}
+
+static int atlas7_wdt_ping(struct watchdog_device *wdd)
+{
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+	u32 counter, match, delta;
+
+	counter = readl(wdt->base + ATLAS7_WDT_CNT);
+	delta = wdd->timeout * wdt->tick_rate;
+	match = counter + delta;
+
+	writel(match, wdt->base + ATLAS7_WDT_CNT_MATCH);
+
+	return 0;
+}
+
+static int atlas7_wdt_enable(struct watchdog_device *wdd)
+{
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+
+	atlas7_wdt_ping(wdd);
+
+	writel(readl(wdt->base + ATLAS7_WDT_CNT_CTRL) | ATLAS7_WDT_CNT_EN,
+	      wdt->base + ATLAS7_WDT_CNT_CTRL);
+	writel(1, wdt->base + ATLAS7_WDT_EN);
+
+	return 0;
+}
+
+static int atlas7_wdt_disable(struct watchdog_device *wdd)
+{
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+
+	writel(0, wdt->base + ATLAS7_WDT_EN);
+	writel(readl(wdt->base + ATLAS7_WDT_CNT_CTRL) & ~ATLAS7_WDT_CNT_EN,
+	      wdt->base + ATLAS7_WDT_CNT_CTRL);
+
+	return 0;
+}
+
+static int atlas7_wdt_settimeout(struct watchdog_device *wdd, unsigned int to)
+{
+	wdd->timeout = to;
+
+	return 0;
+}
+
+#define OPTIONS (WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE)
+
+static const struct watchdog_info atlas7_wdt_ident = {
+	.options = OPTIONS,
+	.firmware_version = 0,
+	.identity = "atlas7 Watchdog",
+};
+
+static struct watchdog_ops atlas7_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = atlas7_wdt_enable,
+	.stop = atlas7_wdt_disable,
+	.get_timeleft = atlas7_wdt_gettimeleft,
+	.ping = atlas7_wdt_ping,
+	.set_timeout = atlas7_wdt_settimeout,
+};
+
+static struct watchdog_device atlas7_wdd = {
+	.info = &atlas7_wdt_ident,
+	.ops = &atlas7_wdt_ops,
+	.timeout = ATLAS7_WDT_DEFAULT_TIMEOUT,
+};
+
+static const struct of_device_id atlas7_wdt_ids[] = {
+	{ .compatible = "sirf,atlas7-tick"},
+	{}
+};
+
+static int atlas7_wdt_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct atlas7_wdog *wdt;
+	struct resource *res;
+	struct clk *clk;
+	int ret;
+
+	wdt = devm_kzalloc(&pdev->dev, sizeof(*wdt), GFP_KERNEL);
+	if (!wdt)
+		return -ENOMEM;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	wdt->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(wdt->base))
+		return PTR_ERR(wdt->base);
+
+	clk = of_clk_get(np, 0);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(&pdev->dev, "clk enable failed\n");
+		goto err;
+	}
+
+	/* disable watchdog hardware */
+	writel(0, wdt->base + ATLAS7_WDT_CNT_CTRL);
+
+	wdt->tick_rate = clk_get_rate(clk);
+	wdt->clk = clk;
+	atlas7_wdd.min_timeout = 1;
+	atlas7_wdd.max_timeout = UINT_MAX / wdt->tick_rate;
+
+	watchdog_init_timeout(&atlas7_wdd, 0, &pdev->dev);
+	watchdog_set_nowayout(&atlas7_wdd, nowayout);
+
+	watchdog_set_drvdata(&atlas7_wdd, wdt);
+	platform_set_drvdata(pdev, &atlas7_wdd);
+
+	ret = watchdog_register_device(&atlas7_wdd);
+	if (ret)
+		goto err1;
+
+	return 0;
+
+err1:
+	clk_disable_unprepare(clk);
+err:
+	clk_put(clk);
+	return ret;
+}
+
+static void atlas7_wdt_shutdown(struct platform_device *pdev)
+{
+	struct watchdog_device *wdd = platform_get_drvdata(pdev);
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+
+	atlas7_wdt_disable(wdd);
+	clk_disable_unprepare(wdt->clk);
+}
+
+static int atlas7_wdt_remove(struct platform_device *pdev)
+{
+	struct watchdog_device *wdd = platform_get_drvdata(pdev);
+	struct atlas7_wdog *wdt = watchdog_get_drvdata(wdd);
+
+	atlas7_wdt_shutdown(pdev);
+	clk_put(wdt->clk);
+	return 0;
+}
+
+static int __maybe_unused atlas7_wdt_suspend(struct device *dev)
+{
+	/*
+	 * NOTE:timer controller registers settings are saved
+	 * and restored back by the timer-atlas7.c
+	 */
+	return 0;
+}
+
+static int __maybe_unused atlas7_wdt_resume(struct device *dev)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	/*
+	 * NOTE: Since timer controller registers settings are saved
+	 * and restored back by the timer-atlas7.c, so we need not
+	 * update WD settings except refreshing timeout.
+	 */
+	atlas7_wdt_ping(wdd);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(atlas7_wdt_pm_ops,
+		atlas7_wdt_suspend, atlas7_wdt_resume);
+
+MODULE_DEVICE_TABLE(of, atlas7_wdt_ids);
+
+static struct platform_driver atlas7_wdt_driver = {
+	.driver = {
+		.name = "atlas7-wdt",
+		.pm = &atlas7_wdt_pm_ops,
+		.of_match_table	= atlas7_wdt_ids,
+	},
+	.probe = atlas7_wdt_probe,
+	.remove = atlas7_wdt_remove,
+	.shutdown = atlas7_wdt_shutdown,
+};
+module_platform_driver(atlas7_wdt_driver);
+
+MODULE_DESCRIPTION("CSRatlas7 watchdog driver");
+MODULE_AUTHOR("Guo Zeng <Guo.Zeng@csr.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:atlas7-wdt");
diff --git a/drivers/watchdog/bcm2835_wdt.c b/drivers/watchdog/bcm2835_wdt.c
index 8a5ce5b5a0b6..2e6164c4abc0 100644
--- a/drivers/watchdog/bcm2835_wdt.c
+++ b/drivers/watchdog/bcm2835_wdt.c
@@ -79,7 +79,6 @@ static int bcm2835_wdt_stop(struct watchdog_device *wdog)
 	struct bcm2835_wdt *wdt = watchdog_get_drvdata(wdog);
 
 	writel_relaxed(PM_PASSWORD | PM_RSTC_RESET, wdt->base + PM_RSTC);
-	dev_info(wdog->dev, "Watchdog timer stopped");
 	return 0;
 }
 
diff --git a/drivers/watchdog/bcm47xx_wdt.c b/drivers/watchdog/bcm47xx_wdt.c
index 4064a43f1360..df1c2a4b0165 100644
--- a/drivers/watchdog/bcm47xx_wdt.c
+++ b/drivers/watchdog/bcm47xx_wdt.c
@@ -20,7 +20,6 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <linux/timer.h>
@@ -88,12 +87,22 @@ static int bcm47xx_wdt_hard_set_timeout(struct watchdog_device *wdd,
 	return 0;
 }
 
+static int bcm47xx_wdt_restart(struct watchdog_device *wdd)
+{
+	struct bcm47xx_wdt *wdt = bcm47xx_wdt_get(wdd);
+
+	wdt->timer_set(wdt, 1);
+
+	return 0;
+}
+
 static struct watchdog_ops bcm47xx_wdt_hard_ops = {
 	.owner		= THIS_MODULE,
 	.start		= bcm47xx_wdt_hard_start,
 	.stop		= bcm47xx_wdt_hard_stop,
 	.ping		= bcm47xx_wdt_hard_keepalive,
 	.set_timeout	= bcm47xx_wdt_hard_set_timeout,
+	.restart        = bcm47xx_wdt_restart,
 };
 
 static void bcm47xx_wdt_soft_timer_tick(unsigned long data)
@@ -158,34 +167,13 @@ static const struct watchdog_info bcm47xx_wdt_info = {
 				WDIOF_MAGICCLOSE,
 };
 
-static int bcm47xx_wdt_notify_sys(struct notifier_block *this,
-				  unsigned long code, void *unused)
-{
-	struct bcm47xx_wdt *wdt;
-
-	wdt = container_of(this, struct bcm47xx_wdt, notifier);
-	if (code == SYS_DOWN || code == SYS_HALT)
-		wdt->wdd.ops->stop(&wdt->wdd);
-	return NOTIFY_DONE;
-}
-
-static int bcm47xx_wdt_restart(struct notifier_block *this, unsigned long mode,
-			       void *cmd)
-{
-	struct bcm47xx_wdt *wdt;
-
-	wdt = container_of(this, struct bcm47xx_wdt, restart_handler);
-	wdt->timer_set(wdt, 1);
-
-	return NOTIFY_DONE;
-}
-
 static struct watchdog_ops bcm47xx_wdt_soft_ops = {
 	.owner		= THIS_MODULE,
 	.start		= bcm47xx_wdt_soft_start,
 	.stop		= bcm47xx_wdt_soft_stop,
 	.ping		= bcm47xx_wdt_soft_keepalive,
 	.set_timeout	= bcm47xx_wdt_soft_set_timeout,
+	.restart        = bcm47xx_wdt_restart,
 };
 
 static int bcm47xx_wdt_probe(struct platform_device *pdev)
@@ -214,32 +202,18 @@ static int bcm47xx_wdt_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_timer;
 	watchdog_set_nowayout(&wdt->wdd, nowayout);
-
-	wdt->notifier.notifier_call = &bcm47xx_wdt_notify_sys;
-
-	ret = register_reboot_notifier(&wdt->notifier);
-	if (ret)
-		goto err_timer;
-
-	wdt->restart_handler.notifier_call = &bcm47xx_wdt_restart;
-	wdt->restart_handler.priority = 64;
-	ret = register_restart_handler(&wdt->restart_handler);
-	if (ret)
-		goto err_notifier;
+	watchdog_set_restart_priority(&wdt->wdd, 64);
+	watchdog_stop_on_reboot(&wdt->wdd);
 
 	ret = watchdog_register_device(&wdt->wdd);
 	if (ret)
-		goto err_handler;
+		goto err_timer;
 
 	dev_info(&pdev->dev, "BCM47xx Watchdog Timer enabled (%d seconds%s%s)\n",
 		timeout, nowayout ? ", nowayout" : "",
 		soft ? ", Software Timer" : "");
 	return 0;
 
-err_handler:
-	unregister_restart_handler(&wdt->restart_handler);
-err_notifier:
-	unregister_reboot_notifier(&wdt->notifier);
 err_timer:
 	if (soft)
 		del_timer_sync(&wdt->soft_timer);
@@ -255,7 +229,6 @@ static int bcm47xx_wdt_remove(struct platform_device *pdev)
 		return -ENXIO;
 
 	watchdog_unregister_device(&wdt->wdd);
-	unregister_reboot_notifier(&wdt->notifier);
 
 	return 0;
 }
diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c
index bcfd2a22208f..4dda9024e229 100644
--- a/drivers/watchdog/cadence_wdt.c
+++ b/drivers/watchdog/cadence_wdt.c
@@ -18,7 +18,6 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/watchdog.h>
 
 #define CDNS_WDT_DEFAULT_TIMEOUT	10
@@ -72,7 +71,6 @@ MODULE_PARM_DESC(nowayout,
  * @ctrl_clksel: counter clock prescaler selection
  * @io_lock: spinlock for IO register access
  * @cdns_wdt_device: watchdog device structure
- * @cdns_wdt_notifier: notifier structure
  *
  * Structure containing parameters specific to cadence watchdog.
  */
@@ -84,7 +82,6 @@ struct cdns_wdt {
 	u32			ctrl_clksel;
 	spinlock_t		io_lock;
 	struct watchdog_device	cdns_wdt_device;
-	struct notifier_block	cdns_wdt_notifier;
 };
 
 /* Write access to Registers */
@@ -280,29 +277,6 @@ static struct watchdog_ops cdns_wdt_ops = {
 	.set_timeout = cdns_wdt_settimeout,
 };
 
-/**
- * cdns_wdt_notify_sys - Notifier for reboot or shutdown.
- *
- * @this: handle to notifier block
- * @code: turn off indicator
- * @unused: unused
- * Return: NOTIFY_DONE
- *
- * This notifier is invoked whenever the system reboot or shutdown occur
- * because we need to disable the WDT before system goes down as WDT might
- * reset on the next boot.
- */
-static int cdns_wdt_notify_sys(struct notifier_block *this, unsigned long code,
-			       void *unused)
-{
-	struct cdns_wdt *wdt = container_of(this, struct cdns_wdt,
-					    cdns_wdt_notifier);
-	if (code == SYS_DOWN || code == SYS_HALT)
-		cdns_wdt_stop(&wdt->cdns_wdt_device);
-
-	return NOTIFY_DONE;
-}
-
 /************************Platform Operations*****************************/
 /**
  * cdns_wdt_probe - Probe call for the device.
@@ -360,6 +334,7 @@ static int cdns_wdt_probe(struct platform_device *pdev)
 	}
 
 	watchdog_set_nowayout(cdns_wdt_device, nowayout);
+	watchdog_stop_on_reboot(cdns_wdt_device);
 	watchdog_set_drvdata(cdns_wdt_device, wdt);
 
 	wdt->clk = devm_clk_get(&pdev->dev, NULL);
@@ -386,14 +361,6 @@ static int cdns_wdt_probe(struct platform_device *pdev)
 
 	spin_lock_init(&wdt->io_lock);
 
-	wdt->cdns_wdt_notifier.notifier_call = &cdns_wdt_notify_sys;
-	ret = register_reboot_notifier(&wdt->cdns_wdt_notifier);
-	if (ret != 0) {
-		dev_err(&pdev->dev, "cannot register reboot notifier err=%d)\n",
-			ret);
-		goto err_clk_disable;
-	}
-
 	ret = watchdog_register_device(cdns_wdt_device);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register wdt device\n");
@@ -427,7 +394,6 @@ static int cdns_wdt_remove(struct platform_device *pdev)
 
 	cdns_wdt_stop(&wdt->cdns_wdt_device);
 	watchdog_unregister_device(&wdt->cdns_wdt_device);
-	unregister_reboot_notifier(&wdt->cdns_wdt_notifier);
 	clk_disable_unprepare(wdt->clk);
 
 	return 0;
@@ -455,8 +421,7 @@ static void cdns_wdt_shutdown(struct platform_device *pdev)
  */
 static int __maybe_unused cdns_wdt_suspend(struct device *dev)
 {
-	struct platform_device *pdev = container_of(dev,
-			struct platform_device, dev);
+	struct platform_device *pdev = to_platform_device(dev);
 	struct cdns_wdt *wdt = platform_get_drvdata(pdev);
 
 	cdns_wdt_stop(&wdt->cdns_wdt_device);
@@ -474,8 +439,7 @@ static int __maybe_unused cdns_wdt_suspend(struct device *dev)
 static int __maybe_unused cdns_wdt_resume(struct device *dev)
 {
 	int ret;
-	struct platform_device *pdev = container_of(dev,
-			struct platform_device, dev);
+	struct platform_device *pdev = to_platform_device(dev);
 	struct cdns_wdt *wdt = platform_get_drvdata(pdev);
 
 	ret = clk_prepare_enable(wdt->clk);
diff --git a/drivers/watchdog/da9052_wdt.c b/drivers/watchdog/da9052_wdt.c
index 67e67977bd29..2fc19a32a320 100644
--- a/drivers/watchdog/da9052_wdt.c
+++ b/drivers/watchdog/da9052_wdt.c
@@ -31,7 +31,6 @@
 struct da9052_wdt_data {
 	struct watchdog_device wdt;
 	struct da9052 *da9052;
-	struct kref kref;
 	unsigned long jpast;
 };
 
@@ -51,10 +50,6 @@ static const struct {
 };
 
 
-static void da9052_wdt_release_resources(struct kref *r)
-{
-}
-
 static int da9052_wdt_set_timeout(struct watchdog_device *wdt_dev,
 				  unsigned int timeout)
 {
@@ -104,20 +99,6 @@ static int da9052_wdt_set_timeout(struct watchdog_device *wdt_dev,
 	return 0;
 }
 
-static void da9052_wdt_ref(struct watchdog_device *wdt_dev)
-{
-	struct da9052_wdt_data *driver_data = watchdog_get_drvdata(wdt_dev);
-
-	kref_get(&driver_data->kref);
-}
-
-static void da9052_wdt_unref(struct watchdog_device *wdt_dev)
-{
-	struct da9052_wdt_data *driver_data = watchdog_get_drvdata(wdt_dev);
-
-	kref_put(&driver_data->kref, da9052_wdt_release_resources);
-}
-
 static int da9052_wdt_start(struct watchdog_device *wdt_dev)
 {
 	return da9052_wdt_set_timeout(wdt_dev, wdt_dev->timeout);
@@ -170,8 +151,6 @@ static const struct watchdog_ops da9052_wdt_ops = {
 	.stop = da9052_wdt_stop,
 	.ping = da9052_wdt_ping,
 	.set_timeout = da9052_wdt_set_timeout,
-	.ref = da9052_wdt_ref,
-	.unref = da9052_wdt_unref,
 };
 
 
@@ -198,8 +177,6 @@ static int da9052_wdt_probe(struct platform_device *pdev)
 	da9052_wdt->parent = &pdev->dev;
 	watchdog_set_drvdata(da9052_wdt, driver_data);
 
-	kref_init(&driver_data->kref);
-
 	ret = da9052_reg_update(da9052, DA9052_CONTROL_D_REG,
 				DA9052_CONTROLD_TWDSCALE, 0);
 	if (ret < 0) {
@@ -225,7 +202,6 @@ static int da9052_wdt_remove(struct platform_device *pdev)
 	struct da9052_wdt_data *driver_data = platform_get_drvdata(pdev);
 
 	watchdog_unregister_device(&driver_data->wdt);
-	kref_put(&driver_data->kref, da9052_wdt_release_resources);
 
 	return 0;
 }
diff --git a/drivers/watchdog/da9055_wdt.c b/drivers/watchdog/da9055_wdt.c
index 04d1430d93d2..8377c43f3f20 100644
--- a/drivers/watchdog/da9055_wdt.c
+++ b/drivers/watchdog/da9055_wdt.c
@@ -35,7 +35,6 @@ MODULE_PARM_DESC(nowayout,
 struct da9055_wdt_data {
 	struct watchdog_device wdt;
 	struct da9055 *da9055;
-	struct kref kref;
 };
 
 static const struct {
@@ -99,24 +98,6 @@ static int da9055_wdt_ping(struct watchdog_device *wdt_dev)
 				 DA9055_WATCHDOG_MASK, 1);
 }
 
-static void da9055_wdt_release_resources(struct kref *r)
-{
-}
-
-static void da9055_wdt_ref(struct watchdog_device *wdt_dev)
-{
-	struct da9055_wdt_data *driver_data = watchdog_get_drvdata(wdt_dev);
-
-	kref_get(&driver_data->kref);
-}
-
-static void da9055_wdt_unref(struct watchdog_device *wdt_dev)
-{
-	struct da9055_wdt_data *driver_data = watchdog_get_drvdata(wdt_dev);
-
-	kref_put(&driver_data->kref, da9055_wdt_release_resources);
-}
-
 static int da9055_wdt_start(struct watchdog_device *wdt_dev)
 {
 	return da9055_wdt_set_timeout(wdt_dev, wdt_dev->timeout);
@@ -138,8 +119,6 @@ static const struct watchdog_ops da9055_wdt_ops = {
 	.stop = da9055_wdt_stop,
 	.ping = da9055_wdt_ping,
 	.set_timeout = da9055_wdt_set_timeout,
-	.ref = da9055_wdt_ref,
-	.unref = da9055_wdt_unref,
 };
 
 static int da9055_wdt_probe(struct platform_device *pdev)
@@ -165,8 +144,6 @@ static int da9055_wdt_probe(struct platform_device *pdev)
 	watchdog_set_nowayout(da9055_wdt, nowayout);
 	watchdog_set_drvdata(da9055_wdt, driver_data);
 
-	kref_init(&driver_data->kref);
-
 	ret = da9055_wdt_stop(da9055_wdt);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to stop watchdog, %d\n", ret);
@@ -189,7 +166,6 @@ static int da9055_wdt_remove(struct platform_device *pdev)
 	struct da9055_wdt_data *driver_data = platform_get_drvdata(pdev);
 
 	watchdog_unregister_device(&driver_data->wdt);
-	kref_put(&driver_data->kref, da9055_wdt_release_resources);
 
 	return 0;
 }
diff --git a/drivers/watchdog/da9063_wdt.c b/drivers/watchdog/da9063_wdt.c
index 6bf130bd863d..11e887572649 100644
--- a/drivers/watchdog/da9063_wdt.c
+++ b/drivers/watchdog/da9063_wdt.c
@@ -20,7 +20,6 @@
 #include <linux/delay.h>
 #include <linux/mfd/da9063/registers.h>
 #include <linux/mfd/da9063/core.h>
-#include <linux/reboot.h>
 #include <linux/regmap.h>
 
 /*
@@ -39,7 +38,6 @@ static const unsigned int wdt_timeout[] = { 0, 2, 4, 8, 16, 32, 65, 131 };
 struct da9063_watchdog {
 	struct da9063 *da9063;
 	struct watchdog_device wdtdev;
-	struct notifier_block restart_handler;
 };
 
 static unsigned int da9063_wdt_timeout_to_sel(unsigned int secs)
@@ -121,12 +119,9 @@ static int da9063_wdt_set_timeout(struct watchdog_device *wdd,
 	return ret;
 }
 
-static int da9063_wdt_restart_handler(struct notifier_block *this,
-				      unsigned long mode, void *cmd)
+static int da9063_wdt_restart(struct watchdog_device *wdd)
 {
-	struct da9063_watchdog *wdt = container_of(this,
-						   struct da9063_watchdog,
-						   restart_handler);
+	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
 	int ret;
 
 	ret = regmap_write(wdt->da9063->regmap, DA9063_REG_CONTROL_F,
@@ -135,7 +130,7 @@ static int da9063_wdt_restart_handler(struct notifier_block *this,
 		dev_alert(wdt->da9063->dev, "Failed to shutdown (err = %d)\n",
 			  ret);
 
-	return NOTIFY_DONE;
+	return ret;
 }
 
 static const struct watchdog_info da9063_watchdog_info = {
@@ -149,6 +144,7 @@ static const struct watchdog_ops da9063_watchdog_ops = {
 	.stop = da9063_wdt_stop,
 	.ping = da9063_wdt_ping,
 	.set_timeout = da9063_wdt_set_timeout,
+	.restart = da9063_wdt_restart,
 };
 
 static int da9063_wdt_probe(struct platform_device *pdev)
@@ -179,6 +175,8 @@ static int da9063_wdt_probe(struct platform_device *pdev)
 
 	wdt->wdtdev.status = WATCHDOG_NOWAYOUT_INIT_STATUS;
 
+	watchdog_set_restart_priority(&wdt->wdtdev, 128);
+
 	watchdog_set_drvdata(&wdt->wdtdev, wdt);
 	dev_set_drvdata(&pdev->dev, wdt);
 
@@ -186,13 +184,6 @@ static int da9063_wdt_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	wdt->restart_handler.notifier_call = da9063_wdt_restart_handler;
-	wdt->restart_handler.priority = 128;
-	ret = register_restart_handler(&wdt->restart_handler);
-	if (ret)
-		dev_err(wdt->da9063->dev,
-			"Failed to register restart handler (err = %d)\n", ret);
-
 	return 0;
 }
 
@@ -200,8 +191,6 @@ static int da9063_wdt_remove(struct platform_device *pdev)
 {
 	struct da9063_watchdog *wdt = dev_get_drvdata(&pdev->dev);
 
-	unregister_restart_handler(&wdt->restart_handler);
-
 	watchdog_unregister_device(&wdt->wdtdev);
 
 	return 0;
diff --git a/drivers/watchdog/diag288_wdt.c b/drivers/watchdog/diag288_wdt.c
index 3db9d0e0673d..861d3d3133f8 100644
--- a/drivers/watchdog/diag288_wdt.c
+++ b/drivers/watchdog/diag288_wdt.c
@@ -106,6 +106,10 @@ static int __diag288_lpar(unsigned int func, unsigned int timeout,
 	return __diag288(func, timeout, action, 0);
 }
 
+static unsigned long wdt_status;
+
+#define DIAG_WDOG_BUSY	0
+
 static int wdt_start(struct watchdog_device *dev)
 {
 	char *ebc_cmd;
@@ -113,12 +117,17 @@ static int wdt_start(struct watchdog_device *dev)
 	int ret;
 	unsigned int func;
 
+	if (test_and_set_bit(DIAG_WDOG_BUSY, &wdt_status))
+		return -EBUSY;
+
 	ret = -ENODEV;
 
 	if (MACHINE_IS_VM) {
 		ebc_cmd = kmalloc(MAX_CMDLEN, GFP_KERNEL);
-		if (!ebc_cmd)
+		if (!ebc_cmd) {
+			clear_bit(DIAG_WDOG_BUSY, &wdt_status);
 			return -ENOMEM;
+		}
 		len = strlcpy(ebc_cmd, wdt_cmd, MAX_CMDLEN);
 		ASCEBC(ebc_cmd, MAX_CMDLEN);
 		EBC_TOUPPER(ebc_cmd, MAX_CMDLEN);
@@ -135,6 +144,7 @@ static int wdt_start(struct watchdog_device *dev)
 
 	if (ret) {
 		pr_err("The watchdog cannot be activated\n");
+		clear_bit(DIAG_WDOG_BUSY, &wdt_status);
 		return ret;
 	}
 	return 0;
@@ -146,6 +156,9 @@ static int wdt_stop(struct watchdog_device *dev)
 
 	diag_stat_inc(DIAG_STAT_X288);
 	ret = __diag288(WDT_FUNC_CANCEL, 0, 0, 0);
+
+	clear_bit(DIAG_WDOG_BUSY, &wdt_status);
+
 	return ret;
 }
 
@@ -220,17 +233,10 @@ static struct watchdog_device wdt_dev = {
  * It makes no sense to go into suspend while the watchdog is running.
  * Depending on the memory size, the watchdog might trigger, while we
  * are still saving the memory.
- * We reuse the open flag to ensure that suspend and watchdog open are
- * exclusive operations
  */
 static int wdt_suspend(void)
 {
-	if (test_and_set_bit(WDOG_DEV_OPEN, &wdt_dev.status)) {
-		pr_err("Linux cannot be suspended while the watchdog is in use\n");
-		return notifier_from_errno(-EBUSY);
-	}
-	if (test_bit(WDOG_ACTIVE, &wdt_dev.status)) {
-		clear_bit(WDOG_DEV_OPEN, &wdt_dev.status);
+	if (test_and_set_bit(DIAG_WDOG_BUSY, &wdt_status)) {
 		pr_err("Linux cannot be suspended while the watchdog is in use\n");
 		return notifier_from_errno(-EBUSY);
 	}
@@ -239,7 +245,7 @@ static int wdt_suspend(void)
 
 static int wdt_resume(void)
 {
-	clear_bit(WDOG_DEV_OPEN, &wdt_dev.status);
+	clear_bit(DIAG_WDOG_BUSY, &wdt_status);
 	return NOTIFY_DONE;
 }
 
diff --git a/drivers/watchdog/digicolor_wdt.c b/drivers/watchdog/digicolor_wdt.c
index 50abe1bf62a5..1ccb0b239348 100644
--- a/drivers/watchdog/digicolor_wdt.c
+++ b/drivers/watchdog/digicolor_wdt.c
@@ -15,7 +15,6 @@
 #include <linux/delay.h>
 #include <linux/clk.h>
 #include <linux/watchdog.h>
-#include <linux/reboot.h>
 #include <linux/platform_device.h>
 #include <linux/of_address.h>
 
@@ -28,7 +27,6 @@
 struct dc_wdt {
 	void __iomem		*base;
 	struct clk		*clk;
-	struct notifier_block	restart_handler;
 	spinlock_t		lock;
 };
 
@@ -50,16 +48,15 @@ static void dc_wdt_set(struct dc_wdt *wdt, u32 ticks)
 	spin_unlock_irqrestore(&wdt->lock, flags);
 }
 
-static int dc_restart_handler(struct notifier_block *this, unsigned long mode,
-			      void *cmd)
+static int dc_wdt_restart(struct watchdog_device *wdog)
 {
-	struct dc_wdt *wdt = container_of(this, struct dc_wdt, restart_handler);
+	struct dc_wdt *wdt = watchdog_get_drvdata(wdog);
 
 	dc_wdt_set(wdt, 1);
 	/* wait for reset to assert... */
 	mdelay(500);
 
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static int dc_wdt_start(struct watchdog_device *wdog)
@@ -104,6 +101,7 @@ static struct watchdog_ops dc_wdt_ops = {
 	.stop		= dc_wdt_stop,
 	.set_timeout	= dc_wdt_set_timeout,
 	.get_timeleft	= dc_wdt_get_timeleft,
+	.restart        = dc_wdt_restart,
 };
 
 static struct watchdog_info dc_wdt_info = {
@@ -148,6 +146,7 @@ static int dc_wdt_probe(struct platform_device *pdev)
 	spin_lock_init(&wdt->lock);
 
 	watchdog_set_drvdata(&dc_wdt_wdd, wdt);
+	watchdog_set_restart_priority(&dc_wdt_wdd, 128);
 	watchdog_init_timeout(&dc_wdt_wdd, timeout, dev);
 	ret = watchdog_register_device(&dc_wdt_wdd);
 	if (ret) {
@@ -155,12 +154,6 @@ static int dc_wdt_probe(struct platform_device *pdev)
 		goto err_iounmap;
 	}
 
-	wdt->restart_handler.notifier_call = dc_restart_handler;
-	wdt->restart_handler.priority = 128;
-	ret = register_restart_handler(&wdt->restart_handler);
-	if (ret)
-		dev_warn(&pdev->dev, "cannot register restart handler\n");
-
 	return 0;
 
 err_iounmap:
@@ -172,7 +165,6 @@ static int dc_wdt_remove(struct platform_device *pdev)
 {
 	struct dc_wdt *wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&wdt->restart_handler);
 	watchdog_unregister_device(&dc_wdt_wdd);
 	iounmap(wdt->base);
 
diff --git a/drivers/watchdog/dw_wdt.c b/drivers/watchdog/dw_wdt.c
index 6ea0634345e9..8fefa4ad46d4 100644
--- a/drivers/watchdog/dw_wdt.c
+++ b/drivers/watchdog/dw_wdt.c
@@ -81,7 +81,7 @@ static inline int dw_wdt_top_in_seconds(unsigned top)
 	 * There are 16 possible timeout values in 0..15 where the number of
 	 * cycles is 2 ^ (16 + i) and the watchdog counts down.
 	 */
-	return (1 << (16 + top)) / clk_get_rate(dw_wdt.clk);
+	return (1U << (16 + top)) / clk_get_rate(dw_wdt.clk);
 }
 
 static int dw_wdt_get_top(void)
diff --git a/drivers/watchdog/gpio_wdt.c b/drivers/watchdog/gpio_wdt.c
index 90d59d3f38a3..ba066e4a707b 100644
--- a/drivers/watchdog/gpio_wdt.c
+++ b/drivers/watchdog/gpio_wdt.c
@@ -12,10 +12,8 @@
 #include <linux/err.h>
 #include <linux/delay.h>
 #include <linux/module.h>
-#include <linux/notifier.h>
 #include <linux/of_gpio.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/watchdog.h>
 
 #define SOFT_TIMEOUT_MIN	1
@@ -36,7 +34,6 @@ struct gpio_wdt_priv {
 	unsigned int		hw_algo;
 	unsigned int		hw_margin;
 	unsigned long		last_jiffies;
-	struct notifier_block	notifier;
 	struct timer_list	timer;
 	struct watchdog_device	wdd;
 };
@@ -57,7 +54,8 @@ static void gpio_wdt_hwping(unsigned long data)
 
 	if (priv->armed && time_after(jiffies, priv->last_jiffies +
 				      msecs_to_jiffies(wdd->timeout * 1000))) {
-		dev_crit(wdd->dev, "Timer expired. System will reboot soon!\n");
+		dev_crit(wdd->parent,
+			 "Timer expired. System will reboot soon!\n");
 		return;
 	}
 
@@ -126,26 +124,6 @@ static int gpio_wdt_set_timeout(struct watchdog_device *wdd, unsigned int t)
 	return gpio_wdt_ping(wdd);
 }
 
-static int gpio_wdt_notify_sys(struct notifier_block *nb, unsigned long code,
-			       void *unused)
-{
-	struct gpio_wdt_priv *priv = container_of(nb, struct gpio_wdt_priv,
-						  notifier);
-
-	mod_timer(&priv->timer, 0);
-
-	switch (code) {
-	case SYS_HALT:
-	case SYS_POWER_OFF:
-		gpio_wdt_disable(priv);
-		break;
-	default:
-		break;
-	}
-
-	return NOTIFY_DONE;
-}
-
 static const struct watchdog_info gpio_wdt_ident = {
 	.options	= WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING |
 			  WDIOF_SETTIMEOUT,
@@ -224,23 +202,16 @@ static int gpio_wdt_probe(struct platform_device *pdev)
 
 	setup_timer(&priv->timer, gpio_wdt_hwping, (unsigned long)&priv->wdd);
 
+	watchdog_stop_on_reboot(&priv->wdd);
+
 	ret = watchdog_register_device(&priv->wdd);
 	if (ret)
 		return ret;
 
-	priv->notifier.notifier_call = gpio_wdt_notify_sys;
-	ret = register_reboot_notifier(&priv->notifier);
-	if (ret)
-		goto error_unregister;
-
 	if (priv->always_running)
 		gpio_wdt_start_impl(priv);
 
 	return 0;
-
-error_unregister:
-	watchdog_unregister_device(&priv->wdd);
-	return ret;
 }
 
 static int gpio_wdt_remove(struct platform_device *pdev)
@@ -248,7 +219,6 @@ static int gpio_wdt_remove(struct platform_device *pdev)
 	struct gpio_wdt_priv *priv = platform_get_drvdata(pdev);
 
 	del_timer_sync(&priv->timer);
-	unregister_reboot_notifier(&priv->notifier);
 	watchdog_unregister_device(&priv->wdd);
 
 	return 0;
diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c
index 286369d4f0f5..92443c319e59 100644
--- a/drivers/watchdog/hpwdt.c
+++ b/drivers/watchdog/hpwdt.c
@@ -1,11 +1,11 @@
 /*
- *	HP WatchDog Driver
+ *	HPE WatchDog Driver
  *	based on
  *
  *	SoftDog	0.05:	A Software Watchdog Device
  *
- *	(c) Copyright 2007 Hewlett-Packard Development Company, L.P.
- *	Thomas Mingarelli <thomas.mingarelli@hp.com>
+ *	(c) Copyright 2015 Hewlett Packard Enterprise Development LP
+ *	Thomas Mingarelli <thomas.mingarelli@hpe.com>
  *
  *	This program is free software; you can redistribute it and/or
  *	modify it under the terms of the GNU General Public License
@@ -580,7 +580,7 @@ static const struct watchdog_info ident = {
 	.options = WDIOF_SETTIMEOUT |
 		   WDIOF_KEEPALIVEPING |
 		   WDIOF_MAGICCLOSE,
-	.identity = "HP iLO2+ HW Watchdog Timer",
+	.identity = "HPE iLO2+ HW Watchdog Timer",
 };
 
 static long hpwdt_ioctl(struct file *file, unsigned int cmd,
@@ -758,7 +758,7 @@ static int hpwdt_init_nmi_decoding(struct pci_dev *dev)
 		goto error2;
 
 	dev_info(&dev->dev,
-			"HP Watchdog Timer Driver: NMI decoding initialized"
+			"HPE Watchdog Timer Driver: NMI decoding initialized"
 			", allow kernel dump: %s (default = 1/ON)\n",
 			(allow_kdump == 0) ? "OFF" : "ON");
 	return 0;
@@ -863,7 +863,7 @@ static int hpwdt_init_one(struct pci_dev *dev,
 		goto error_misc_register;
 	}
 
-	dev_info(&dev->dev, "HP Watchdog Timer Driver: %s"
+	dev_info(&dev->dev, "HPE Watchdog Timer Driver: %s"
 			", timer margin: %d seconds (nowayout=%d).\n",
 			HPWDT_VERSION, soft_margin, nowayout);
 	return 0;
diff --git a/drivers/watchdog/imgpdc_wdt.c b/drivers/watchdog/imgpdc_wdt.c
index 15ab07230960..3679f2e1922f 100644
--- a/drivers/watchdog/imgpdc_wdt.c
+++ b/drivers/watchdog/imgpdc_wdt.c
@@ -45,7 +45,6 @@
 #include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/slab.h>
 #include <linux/watchdog.h>
 
@@ -87,7 +86,6 @@ struct pdc_wdt_dev {
 	struct clk *wdt_clk;
 	struct clk *sys_clk;
 	void __iomem *base;
-	struct notifier_block restart_handler;
 };
 
 static int pdc_wdt_keepalive(struct watchdog_device *wdt_dev)
@@ -152,6 +150,16 @@ static int pdc_wdt_start(struct watchdog_device *wdt_dev)
 	return 0;
 }
 
+static int pdc_wdt_restart(struct watchdog_device *wdt_dev)
+{
+	struct pdc_wdt_dev *wdt = watchdog_get_drvdata(wdt_dev);
+
+	/* Assert SOFT_RESET */
+	writel(0x1, wdt->base + PDC_WDT_SOFT_RESET);
+
+	return 0;
+}
+
 static struct watchdog_info pdc_wdt_info = {
 	.identity	= "IMG PDC Watchdog",
 	.options	= WDIOF_SETTIMEOUT |
@@ -165,20 +173,9 @@ static const struct watchdog_ops pdc_wdt_ops = {
 	.stop		= pdc_wdt_stop,
 	.ping		= pdc_wdt_keepalive,
 	.set_timeout	= pdc_wdt_set_timeout,
+	.restart        = pdc_wdt_restart,
 };
 
-static int pdc_wdt_restart(struct notifier_block *this, unsigned long mode,
-			   void *cmd)
-{
-	struct pdc_wdt_dev *wdt = container_of(this, struct pdc_wdt_dev,
-					       restart_handler);
-
-	/* Assert SOFT_RESET */
-	writel(0x1, wdt->base + PDC_WDT_SOFT_RESET);
-
-	return NOTIFY_OK;
-}
-
 static int pdc_wdt_probe(struct platform_device *pdev)
 {
 	u64 div;
@@ -282,6 +279,7 @@ static int pdc_wdt_probe(struct platform_device *pdev)
 	}
 
 	watchdog_set_nowayout(&pdc_wdt->wdt_dev, nowayout);
+	watchdog_set_restart_priority(&pdc_wdt->wdt_dev, 128);
 
 	platform_set_drvdata(pdev, pdc_wdt);
 
@@ -289,13 +287,6 @@ static int pdc_wdt_probe(struct platform_device *pdev)
 	if (ret)
 		goto disable_wdt_clk;
 
-	pdc_wdt->restart_handler.notifier_call = pdc_wdt_restart;
-	pdc_wdt->restart_handler.priority = 128;
-	ret = register_restart_handler(&pdc_wdt->restart_handler);
-	if (ret)
-		dev_warn(&pdev->dev, "failed to register restart handler: %d\n",
-			 ret);
-
 	return 0;
 
 disable_wdt_clk:
@@ -316,7 +307,6 @@ static int pdc_wdt_remove(struct platform_device *pdev)
 {
 	struct pdc_wdt_dev *pdc_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&pdc_wdt->restart_handler);
 	pdc_wdt_stop(&pdc_wdt->wdt_dev);
 	watchdog_unregister_device(&pdc_wdt->wdt_dev);
 	clk_disable_unprepare(pdc_wdt->wdt_clk);
diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c
index 29ef719a6a3c..e47966aa2db0 100644
--- a/drivers/watchdog/imx2_wdt.c
+++ b/drivers/watchdog/imx2_wdt.c
@@ -29,10 +29,8 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <linux/notifier.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/regmap.h>
 #include <linux/timer.h>
 #include <linux/watchdog.h>
@@ -64,7 +62,6 @@ struct imx2_wdt_device {
 	struct regmap *regmap;
 	struct timer_list timer;	/* Pings the watchdog when closed */
 	struct watchdog_device wdog;
-	struct notifier_block restart_handler;
 };
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
@@ -83,13 +80,11 @@ static const struct watchdog_info imx2_wdt_info = {
 	.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE,
 };
 
-static int imx2_restart_handler(struct notifier_block *this, unsigned long mode,
-				void *cmd)
+static int imx2_wdt_restart(struct watchdog_device *wdog)
 {
+	struct imx2_wdt_device *wdev = watchdog_get_drvdata(wdog);
 	unsigned int wcr_enable = IMX2_WDT_WCR_WDE;
-	struct imx2_wdt_device *wdev = container_of(this,
-						    struct imx2_wdt_device,
-						    restart_handler);
+
 	/* Assert SRS signal */
 	regmap_write(wdev->regmap, IMX2_WDT_WCR, wcr_enable);
 	/*
@@ -105,7 +100,7 @@ static int imx2_restart_handler(struct notifier_block *this, unsigned long mode,
 	/* wait for reset to assert... */
 	mdelay(500);
 
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static inline void imx2_wdt_setup(struct watchdog_device *wdog)
@@ -213,6 +208,7 @@ static const struct watchdog_ops imx2_wdt_ops = {
 	.stop = imx2_wdt_stop,
 	.ping = imx2_wdt_ping,
 	.set_timeout = imx2_wdt_set_timeout,
+	.restart = imx2_wdt_restart,
 };
 
 static const struct regmap_config imx2_wdt_regmap_config = {
@@ -275,6 +271,7 @@ static int __init imx2_wdt_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, wdog);
 	watchdog_set_drvdata(wdog, wdev);
 	watchdog_set_nowayout(wdog, nowayout);
+	watchdog_set_restart_priority(wdog, 128);
 	watchdog_init_timeout(wdog, timeout, &pdev->dev);
 
 	setup_timer(&wdev->timer, imx2_wdt_timer_ping, (unsigned long)wdog);
@@ -294,12 +291,6 @@ static int __init imx2_wdt_probe(struct platform_device *pdev)
 		goto disable_clk;
 	}
 
-	wdev->restart_handler.notifier_call = imx2_restart_handler;
-	wdev->restart_handler.priority = 128;
-	ret = register_restart_handler(&wdev->restart_handler);
-	if (ret)
-		dev_err(&pdev->dev, "cannot register restart handler\n");
-
 	dev_info(&pdev->dev, "timeout %d sec (nowayout=%d)\n",
 		 wdog->timeout, nowayout);
 
@@ -315,8 +306,6 @@ static int __exit imx2_wdt_remove(struct platform_device *pdev)
 	struct watchdog_device *wdog = platform_get_drvdata(pdev);
 	struct imx2_wdt_device *wdev = watchdog_get_drvdata(wdog);
 
-	unregister_restart_handler(&wdev->restart_handler);
-
 	watchdog_unregister_device(wdog);
 
 	if (imx2_wdt_is_running(wdev)) {
diff --git a/drivers/watchdog/lpc18xx_wdt.c b/drivers/watchdog/lpc18xx_wdt.c
index ab7b8b185d99..6914c83aa6d9 100644
--- a/drivers/watchdog/lpc18xx_wdt.c
+++ b/drivers/watchdog/lpc18xx_wdt.c
@@ -18,7 +18,6 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/watchdog.h>
 
 /* Registers */
@@ -59,7 +58,6 @@ struct lpc18xx_wdt_dev {
 	unsigned long		clk_rate;
 	void __iomem		*base;
 	struct timer_list	timer;
-	struct notifier_block	restart_handler;
 	spinlock_t		lock;
 };
 
@@ -155,27 +153,9 @@ static int lpc18xx_wdt_start(struct watchdog_device *wdt_dev)
 	return 0;
 }
 
-static struct watchdog_info lpc18xx_wdt_info = {
-	.identity	= "NXP LPC18xx Watchdog",
-	.options	= WDIOF_SETTIMEOUT |
-			  WDIOF_KEEPALIVEPING |
-			  WDIOF_MAGICCLOSE,
-};
-
-static const struct watchdog_ops lpc18xx_wdt_ops = {
-	.owner		= THIS_MODULE,
-	.start		= lpc18xx_wdt_start,
-	.stop		= lpc18xx_wdt_stop,
-	.ping		= lpc18xx_wdt_feed,
-	.set_timeout	= lpc18xx_wdt_set_timeout,
-	.get_timeleft	= lpc18xx_wdt_get_timeleft,
-};
-
-static int lpc18xx_wdt_restart(struct notifier_block *this, unsigned long mode,
-			       void *cmd)
+static int lpc18xx_wdt_restart(struct watchdog_device *wdt_dev)
 {
-	struct lpc18xx_wdt_dev *lpc18xx_wdt = container_of(this,
-				struct lpc18xx_wdt_dev, restart_handler);
+	struct lpc18xx_wdt_dev *lpc18xx_wdt = watchdog_get_drvdata(wdt_dev);
 	unsigned long flags;
 	int val;
 
@@ -197,9 +177,26 @@ static int lpc18xx_wdt_restart(struct notifier_block *this, unsigned long mode,
 
 	spin_unlock_irqrestore(&lpc18xx_wdt->lock, flags);
 
-	return NOTIFY_OK;
+	return 0;
 }
 
+static struct watchdog_info lpc18xx_wdt_info = {
+	.identity	= "NXP LPC18xx Watchdog",
+	.options	= WDIOF_SETTIMEOUT |
+			  WDIOF_KEEPALIVEPING |
+			  WDIOF_MAGICCLOSE,
+};
+
+static const struct watchdog_ops lpc18xx_wdt_ops = {
+	.owner		= THIS_MODULE,
+	.start		= lpc18xx_wdt_start,
+	.stop		= lpc18xx_wdt_stop,
+	.ping		= lpc18xx_wdt_feed,
+	.set_timeout	= lpc18xx_wdt_set_timeout,
+	.get_timeleft	= lpc18xx_wdt_get_timeleft,
+	.restart        = lpc18xx_wdt_restart,
+};
+
 static int lpc18xx_wdt_probe(struct platform_device *pdev)
 {
 	struct lpc18xx_wdt_dev *lpc18xx_wdt;
@@ -273,6 +270,7 @@ static int lpc18xx_wdt_probe(struct platform_device *pdev)
 		    (unsigned long)&lpc18xx_wdt->wdt_dev);
 
 	watchdog_set_nowayout(&lpc18xx_wdt->wdt_dev, nowayout);
+	watchdog_set_restart_priority(&lpc18xx_wdt->wdt_dev, 128);
 
 	platform_set_drvdata(pdev, lpc18xx_wdt);
 
@@ -280,12 +278,6 @@ static int lpc18xx_wdt_probe(struct platform_device *pdev)
 	if (ret)
 		goto disable_wdt_clk;
 
-	lpc18xx_wdt->restart_handler.notifier_call = lpc18xx_wdt_restart;
-	lpc18xx_wdt->restart_handler.priority = 128;
-	ret = register_restart_handler(&lpc18xx_wdt->restart_handler);
-	if (ret)
-		dev_warn(dev, "failed to register restart handler: %d\n", ret);
-
 	return 0;
 
 disable_wdt_clk:
@@ -306,8 +298,6 @@ static int lpc18xx_wdt_remove(struct platform_device *pdev)
 {
 	struct lpc18xx_wdt_dev *lpc18xx_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&lpc18xx_wdt->restart_handler);
-
 	dev_warn(&pdev->dev, "I quit now, hardware will probably reboot!\n");
 	del_timer(&lpc18xx_wdt->timer);
 
diff --git a/drivers/watchdog/mena21_wdt.c b/drivers/watchdog/mena21_wdt.c
index 098fa9c34d6d..af6a7c489f08 100644
--- a/drivers/watchdog/mena21_wdt.c
+++ b/drivers/watchdog/mena21_wdt.c
@@ -100,12 +100,12 @@ static int a21_wdt_set_timeout(struct watchdog_device *wdt,
 	struct a21_wdt_drv *drv = watchdog_get_drvdata(wdt);
 
 	if (timeout != 1 && timeout != 30) {
-		dev_err(wdt->dev, "Only 1 and 30 allowed as timeout\n");
+		dev_err(wdt->parent, "Only 1 and 30 allowed as timeout\n");
 		return -EINVAL;
 	}
 
 	if (timeout == 30 && wdt->timeout == 1) {
-		dev_err(wdt->dev,
+		dev_err(wdt->parent,
 			"Transition from fast to slow mode not allowed\n");
 		return -EINVAL;
 	}
diff --git a/drivers/watchdog/meson_wdt.c b/drivers/watchdog/meson_wdt.c
index 1f4155ee3404..aea5d2f44ad7 100644
--- a/drivers/watchdog/meson_wdt.c
+++ b/drivers/watchdog/meson_wdt.c
@@ -17,51 +17,64 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <linux/notifier.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
 #define DRV_NAME		"meson_wdt"
 
 #define MESON_WDT_TC		0x00
-#define MESON_WDT_TC_EN		BIT(22)
-#define MESON_WDT_TC_TM_MASK	0x3fffff
 #define MESON_WDT_DC_RESET	(3 << 24)
 
 #define MESON_WDT_RESET		0x04
 
 #define MESON_WDT_TIMEOUT	30
 #define MESON_WDT_MIN_TIMEOUT	1
-#define MESON_WDT_MAX_TIMEOUT	(MESON_WDT_TC_TM_MASK / 100000)
 
-#define MESON_SEC_TO_TC(s)	((s) * 100000)
+#define MESON_SEC_TO_TC(s, c)	((s) * (c))
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 static unsigned int timeout = MESON_WDT_TIMEOUT;
 
+struct meson_wdt_data {
+	unsigned int enable;
+	unsigned int terminal_count_mask;
+	unsigned int count_unit;
+};
+
+static struct meson_wdt_data meson6_wdt_data = {
+	.enable			= BIT(22),
+	.terminal_count_mask	= 0x3fffff,
+	.count_unit		= 100000, /* 10 us */
+};
+
+static struct meson_wdt_data meson8b_wdt_data = {
+	.enable			= BIT(19),
+	.terminal_count_mask	= 0xffff,
+	.count_unit		= 7812, /* 128 us */
+};
+
 struct meson_wdt_dev {
 	struct watchdog_device wdt_dev;
 	void __iomem *wdt_base;
-	struct notifier_block restart_handler;
+	const struct meson_wdt_data *data;
 };
 
-static int meson_restart_handle(struct notifier_block *this, unsigned long mode,
-				void *cmd)
+static int meson_wdt_restart(struct watchdog_device *wdt_dev)
 {
-	u32 tc_reboot = MESON_WDT_DC_RESET | MESON_WDT_TC_EN;
-	struct meson_wdt_dev *meson_wdt = container_of(this,
-						       struct meson_wdt_dev,
-						       restart_handler);
+	struct meson_wdt_dev *meson_wdt = watchdog_get_drvdata(wdt_dev);
+	u32 tc_reboot = MESON_WDT_DC_RESET;
+
+	tc_reboot |= meson_wdt->data->enable;
 
 	while (1) {
 		writel(tc_reboot, meson_wdt->wdt_base + MESON_WDT_TC);
 		mdelay(5);
 	}
 
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static int meson_wdt_ping(struct watchdog_device *wdt_dev)
@@ -80,8 +93,8 @@ static void meson_wdt_change_timeout(struct watchdog_device *wdt_dev,
 	u32 reg;
 
 	reg = readl(meson_wdt->wdt_base + MESON_WDT_TC);
-	reg &= ~MESON_WDT_TC_TM_MASK;
-	reg |= MESON_SEC_TO_TC(timeout);
+	reg &= ~meson_wdt->data->terminal_count_mask;
+	reg |= MESON_SEC_TO_TC(timeout, meson_wdt->data->count_unit);
 	writel(reg, meson_wdt->wdt_base + MESON_WDT_TC);
 }
 
@@ -102,7 +115,7 @@ static int meson_wdt_stop(struct watchdog_device *wdt_dev)
 	u32 reg;
 
 	reg = readl(meson_wdt->wdt_base + MESON_WDT_TC);
-	reg &= ~MESON_WDT_TC_EN;
+	reg &= ~meson_wdt->data->enable;
 	writel(reg, meson_wdt->wdt_base + MESON_WDT_TC);
 
 	return 0;
@@ -117,7 +130,7 @@ static int meson_wdt_start(struct watchdog_device *wdt_dev)
 	meson_wdt_ping(wdt_dev);
 
 	reg = readl(meson_wdt->wdt_base + MESON_WDT_TC);
-	reg |= MESON_WDT_TC_EN;
+	reg |= meson_wdt->data->enable;
 	writel(reg, meson_wdt->wdt_base + MESON_WDT_TC);
 
 	return 0;
@@ -136,12 +149,21 @@ static const struct watchdog_ops meson_wdt_ops = {
 	.stop		= meson_wdt_stop,
 	.ping		= meson_wdt_ping,
 	.set_timeout	= meson_wdt_set_timeout,
+	.restart        = meson_wdt_restart,
+};
+
+static const struct of_device_id meson_wdt_dt_ids[] = {
+	{ .compatible = "amlogic,meson6-wdt", .data = &meson6_wdt_data },
+	{ .compatible = "amlogic,meson8b-wdt", .data = &meson8b_wdt_data },
+	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, meson_wdt_dt_ids);
 
 static int meson_wdt_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct meson_wdt_dev *meson_wdt;
+	const struct of_device_id *of_id;
 	int err;
 
 	meson_wdt = devm_kzalloc(&pdev->dev, sizeof(*meson_wdt), GFP_KERNEL);
@@ -153,17 +175,28 @@ static int meson_wdt_probe(struct platform_device *pdev)
 	if (IS_ERR(meson_wdt->wdt_base))
 		return PTR_ERR(meson_wdt->wdt_base);
 
+	of_id = of_match_device(meson_wdt_dt_ids, &pdev->dev);
+	if (!of_id) {
+		dev_err(&pdev->dev, "Unable to initialize WDT data\n");
+		return -ENODEV;
+	}
+	meson_wdt->data = of_id->data;
+
 	meson_wdt->wdt_dev.parent = &pdev->dev;
 	meson_wdt->wdt_dev.info = &meson_wdt_info;
 	meson_wdt->wdt_dev.ops = &meson_wdt_ops;
-	meson_wdt->wdt_dev.timeout = MESON_WDT_TIMEOUT;
-	meson_wdt->wdt_dev.max_timeout = MESON_WDT_MAX_TIMEOUT;
+	meson_wdt->wdt_dev.max_timeout =
+		meson_wdt->data->terminal_count_mask / meson_wdt->data->count_unit;
 	meson_wdt->wdt_dev.min_timeout = MESON_WDT_MIN_TIMEOUT;
+	meson_wdt->wdt_dev.timeout = min_t(unsigned int,
+					   MESON_WDT_TIMEOUT,
+					   meson_wdt->wdt_dev.max_timeout);
 
 	watchdog_set_drvdata(&meson_wdt->wdt_dev, meson_wdt);
 
 	watchdog_init_timeout(&meson_wdt->wdt_dev, timeout, &pdev->dev);
 	watchdog_set_nowayout(&meson_wdt->wdt_dev, nowayout);
+	watchdog_set_restart_priority(&meson_wdt->wdt_dev, 128);
 
 	meson_wdt_stop(&meson_wdt->wdt_dev);
 
@@ -173,13 +206,6 @@ static int meson_wdt_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, meson_wdt);
 
-	meson_wdt->restart_handler.notifier_call = meson_restart_handle;
-	meson_wdt->restart_handler.priority = 128;
-	err = register_restart_handler(&meson_wdt->restart_handler);
-	if (err)
-		dev_err(&pdev->dev,
-			"cannot register restart handler (err=%d)\n", err);
-
 	dev_info(&pdev->dev, "Watchdog enabled (timeout=%d sec, nowayout=%d)",
 		 meson_wdt->wdt_dev.timeout, nowayout);
 
@@ -190,8 +216,6 @@ static int meson_wdt_remove(struct platform_device *pdev)
 {
 	struct meson_wdt_dev *meson_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&meson_wdt->restart_handler);
-
 	watchdog_unregister_device(&meson_wdt->wdt_dev);
 
 	return 0;
@@ -204,12 +228,6 @@ static void meson_wdt_shutdown(struct platform_device *pdev)
 	meson_wdt_stop(&meson_wdt->wdt_dev);
 }
 
-static const struct of_device_id meson_wdt_dt_ids[] = {
-	{ .compatible = "amlogic,meson6-wdt" },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, meson_wdt_dt_ids);
-
 static struct platform_driver meson_wdt_driver = {
 	.probe		= meson_wdt_probe,
 	.remove		= meson_wdt_remove,
diff --git a/drivers/watchdog/moxart_wdt.c b/drivers/watchdog/moxart_wdt.c
index 60b0605bd7e6..885c81bc4210 100644
--- a/drivers/watchdog/moxart_wdt.c
+++ b/drivers/watchdog/moxart_wdt.c
@@ -15,9 +15,7 @@
 #include <linux/module.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
-#include <linux/notifier.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/watchdog.h>
 #include <linux/moduleparam.h>
 
@@ -29,22 +27,19 @@ struct moxart_wdt_dev {
 	struct watchdog_device dev;
 	void __iomem *base;
 	unsigned int clock_frequency;
-	struct notifier_block restart_handler;
 };
 
 static int heartbeat;
 
-static int moxart_restart_handle(struct notifier_block *this,
-				 unsigned long mode, void *cmd)
+static int moxart_wdt_restart(struct watchdog_device *wdt_dev)
 {
-	struct moxart_wdt_dev *moxart_wdt = container_of(this,
-							 struct moxart_wdt_dev,
-							 restart_handler);
+	struct moxart_wdt_dev *moxart_wdt = watchdog_get_drvdata(wdt_dev);
+
 	writel(1, moxart_wdt->base + REG_COUNT);
 	writel(0x5ab9, moxart_wdt->base + REG_MODE);
 	writel(0x03, moxart_wdt->base + REG_ENABLE);
 
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static int moxart_wdt_stop(struct watchdog_device *wdt_dev)
@@ -87,6 +82,7 @@ static const struct watchdog_ops moxart_wdt_ops = {
 	.start          = moxart_wdt_start,
 	.stop           = moxart_wdt_stop,
 	.set_timeout    = moxart_wdt_set_timeout,
+	.restart        = moxart_wdt_restart,
 };
 
 static int moxart_wdt_probe(struct platform_device *pdev)
@@ -134,6 +130,7 @@ static int moxart_wdt_probe(struct platform_device *pdev)
 
 	watchdog_init_timeout(&moxart_wdt->dev, heartbeat, dev);
 	watchdog_set_nowayout(&moxart_wdt->dev, nowayout);
+	watchdog_set_restart_priority(&moxart_wdt->dev, 128);
 
 	watchdog_set_drvdata(&moxart_wdt->dev, moxart_wdt);
 
@@ -141,13 +138,6 @@ static int moxart_wdt_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	moxart_wdt->restart_handler.notifier_call = moxart_restart_handle;
-	moxart_wdt->restart_handler.priority = 128;
-	err = register_restart_handler(&moxart_wdt->restart_handler);
-	if (err)
-		dev_err(dev, "cannot register restart notifier (err=%d)\n",
-			err);
-
 	dev_dbg(dev, "Watchdog enabled (heartbeat=%d sec, nowayout=%d)\n",
 		moxart_wdt->dev.timeout, nowayout);
 
@@ -158,7 +148,6 @@ static int moxart_wdt_remove(struct platform_device *pdev)
 {
 	struct moxart_wdt_dev *moxart_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&moxart_wdt->restart_handler);
 	moxart_wdt_stop(&moxart_wdt->dev);
 
 	return 0;
diff --git a/drivers/watchdog/mt7621_wdt.c b/drivers/watchdog/mt7621_wdt.c
new file mode 100644
index 000000000000..4a2290f900a8
--- /dev/null
+++ b/drivers/watchdog/mt7621_wdt.c
@@ -0,0 +1,186 @@
+/*
+ * Ralink MT7621/MT7628 built-in hardware watchdog timer
+ *
+ * Copyright (C) 2014 John Crispin <blogic@openwrt.org>
+ *
+ * This driver was based on: drivers/watchdog/rt2880_wdt.c
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/watchdog.h>
+#include <linux/moduleparam.h>
+#include <linux/platform_device.h>
+
+#include <asm/mach-ralink/ralink_regs.h>
+
+#define SYSC_RSTSTAT			0x38
+#define WDT_RST_CAUSE			BIT(1)
+
+#define RALINK_WDT_TIMEOUT		30
+
+#define TIMER_REG_TMRSTAT		0x00
+#define TIMER_REG_TMR1LOAD		0x24
+#define TIMER_REG_TMR1CTL		0x20
+
+#define TMR1CTL_ENABLE			BIT(7)
+#define TMR1CTL_RESTART			BIT(9)
+#define TMR1CTL_PRESCALE_SHIFT		16
+
+static void __iomem *mt7621_wdt_base;
+static struct reset_control *mt7621_wdt_reset;
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout,
+		 "Watchdog cannot be stopped once started (default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+static inline void rt_wdt_w32(unsigned reg, u32 val)
+{
+	iowrite32(val, mt7621_wdt_base + reg);
+}
+
+static inline u32 rt_wdt_r32(unsigned reg)
+{
+	return ioread32(mt7621_wdt_base + reg);
+}
+
+static int mt7621_wdt_ping(struct watchdog_device *w)
+{
+	rt_wdt_w32(TIMER_REG_TMRSTAT, TMR1CTL_RESTART);
+
+	return 0;
+}
+
+static int mt7621_wdt_set_timeout(struct watchdog_device *w, unsigned int t)
+{
+	w->timeout = t;
+	rt_wdt_w32(TIMER_REG_TMR1LOAD, t * 1000);
+	mt7621_wdt_ping(w);
+
+	return 0;
+}
+
+static int mt7621_wdt_start(struct watchdog_device *w)
+{
+	u32 t;
+
+	/* set the prescaler to 1ms == 1000us */
+	rt_wdt_w32(TIMER_REG_TMR1CTL, 1000 << TMR1CTL_PRESCALE_SHIFT);
+
+	mt7621_wdt_set_timeout(w, w->timeout);
+
+	t = rt_wdt_r32(TIMER_REG_TMR1CTL);
+	t |= TMR1CTL_ENABLE;
+	rt_wdt_w32(TIMER_REG_TMR1CTL, t);
+
+	return 0;
+}
+
+static int mt7621_wdt_stop(struct watchdog_device *w)
+{
+	u32 t;
+
+	mt7621_wdt_ping(w);
+
+	t = rt_wdt_r32(TIMER_REG_TMR1CTL);
+	t &= ~TMR1CTL_ENABLE;
+	rt_wdt_w32(TIMER_REG_TMR1CTL, t);
+
+	return 0;
+}
+
+static int mt7621_wdt_bootcause(void)
+{
+	if (rt_sysc_r32(SYSC_RSTSTAT) & WDT_RST_CAUSE)
+		return WDIOF_CARDRESET;
+
+	return 0;
+}
+
+static struct watchdog_info mt7621_wdt_info = {
+	.identity = "Mediatek Watchdog",
+	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
+};
+
+static struct watchdog_ops mt7621_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = mt7621_wdt_start,
+	.stop = mt7621_wdt_stop,
+	.ping = mt7621_wdt_ping,
+	.set_timeout = mt7621_wdt_set_timeout,
+};
+
+static struct watchdog_device mt7621_wdt_dev = {
+	.info = &mt7621_wdt_info,
+	.ops = &mt7621_wdt_ops,
+	.min_timeout = 1,
+	.max_timeout = 0xfffful / 1000,
+};
+
+static int mt7621_wdt_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mt7621_wdt_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mt7621_wdt_base))
+		return PTR_ERR(mt7621_wdt_base);
+
+	mt7621_wdt_reset = devm_reset_control_get(&pdev->dev, NULL);
+	if (!IS_ERR(mt7621_wdt_reset))
+		reset_control_deassert(mt7621_wdt_reset);
+
+	mt7621_wdt_dev.dev = &pdev->dev;
+	mt7621_wdt_dev.bootstatus = mt7621_wdt_bootcause();
+
+	watchdog_init_timeout(&mt7621_wdt_dev, mt7621_wdt_dev.max_timeout,
+			      &pdev->dev);
+	watchdog_set_nowayout(&mt7621_wdt_dev, nowayout);
+
+	ret = watchdog_register_device(&mt7621_wdt_dev);
+
+	return 0;
+}
+
+static int mt7621_wdt_remove(struct platform_device *pdev)
+{
+	watchdog_unregister_device(&mt7621_wdt_dev);
+
+	return 0;
+}
+
+static void mt7621_wdt_shutdown(struct platform_device *pdev)
+{
+	mt7621_wdt_stop(&mt7621_wdt_dev);
+}
+
+static const struct of_device_id mt7621_wdt_match[] = {
+	{ .compatible = "mediatek,mt7621-wdt" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mt7621_wdt_match);
+
+static struct platform_driver mt7621_wdt_driver = {
+	.probe		= mt7621_wdt_probe,
+	.remove		= mt7621_wdt_remove,
+	.shutdown	= mt7621_wdt_shutdown,
+	.driver		= {
+		.name		= KBUILD_MODNAME,
+		.of_match_table	= mt7621_wdt_match,
+	},
+};
+
+module_platform_driver(mt7621_wdt_driver);
+
+MODULE_DESCRIPTION("MediaTek MT762x hardware watchdog driver");
+MODULE_AUTHOR("John Crispin <blogic@openwrt.org");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/watchdog/mtk_wdt.c b/drivers/watchdog/mtk_wdt.c
index b751f43d76ed..b78776c05554 100644
--- a/drivers/watchdog/mtk_wdt.c
+++ b/drivers/watchdog/mtk_wdt.c
@@ -28,8 +28,6 @@
 #include <linux/platform_device.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
 #include <linux/delay.h>
 
 #define WDT_MAX_TIMEOUT		31
@@ -64,16 +62,13 @@ static unsigned int timeout = WDT_MAX_TIMEOUT;
 struct mtk_wdt_dev {
 	struct watchdog_device wdt_dev;
 	void __iomem *wdt_base;
-	struct notifier_block restart_handler;
 };
 
-static int mtk_reset_handler(struct notifier_block *this, unsigned long mode,
-				void *cmd)
+static int mtk_wdt_restart(struct watchdog_device *wdt_dev)
 {
-	struct mtk_wdt_dev *mtk_wdt;
+	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdt_dev);
 	void __iomem *wdt_base;
 
-	mtk_wdt = container_of(this, struct mtk_wdt_dev, restart_handler);
 	wdt_base = mtk_wdt->wdt_base;
 
 	while (1) {
@@ -81,7 +76,7 @@ static int mtk_reset_handler(struct notifier_block *this, unsigned long mode,
 		mdelay(5);
 	}
 
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static int mtk_wdt_ping(struct watchdog_device *wdt_dev)
@@ -161,6 +156,7 @@ static const struct watchdog_ops mtk_wdt_ops = {
 	.stop		= mtk_wdt_stop,
 	.ping		= mtk_wdt_ping,
 	.set_timeout	= mtk_wdt_set_timeout,
+	.restart	= mtk_wdt_restart,
 };
 
 static int mtk_wdt_probe(struct platform_device *pdev)
@@ -189,6 +185,7 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 
 	watchdog_init_timeout(&mtk_wdt->wdt_dev, timeout, &pdev->dev);
 	watchdog_set_nowayout(&mtk_wdt->wdt_dev, nowayout);
+	watchdog_set_restart_priority(&mtk_wdt->wdt_dev, 128);
 
 	watchdog_set_drvdata(&mtk_wdt->wdt_dev, mtk_wdt);
 
@@ -198,13 +195,6 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 	if (unlikely(err))
 		return err;
 
-	mtk_wdt->restart_handler.notifier_call = mtk_reset_handler;
-	mtk_wdt->restart_handler.priority = 128;
-	err = register_restart_handler(&mtk_wdt->restart_handler);
-	if (err)
-		dev_warn(&pdev->dev,
-			"cannot register restart handler (err=%d)\n", err);
-
 	dev_info(&pdev->dev, "Watchdog enabled (timeout=%d sec, nowayout=%d)\n",
 			mtk_wdt->wdt_dev.timeout, nowayout);
 
@@ -223,8 +213,6 @@ static int mtk_wdt_remove(struct platform_device *pdev)
 {
 	struct mtk_wdt_dev *mtk_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&mtk_wdt->restart_handler);
-
 	watchdog_unregister_device(&mtk_wdt->wdt_dev);
 
 	return 0;
diff --git a/drivers/watchdog/omap_wdt.c b/drivers/watchdog/omap_wdt.c
index 6f17c935a6cf..1b02bfa81b29 100644
--- a/drivers/watchdog/omap_wdt.c
+++ b/drivers/watchdog/omap_wdt.c
@@ -271,7 +271,8 @@ static int omap_wdt_probe(struct platform_device *pdev)
 			wdev->wdog.bootstatus = WDIOF_CARDRESET;
 	}
 
-	omap_wdt_disable(wdev);
+	if (!early_enable)
+		omap_wdt_disable(wdev);
 
 	ret = watchdog_register_device(&wdev->wdog);
 	if (ret) {
@@ -283,11 +284,11 @@ static int omap_wdt_probe(struct platform_device *pdev)
 		readl_relaxed(wdev->base + OMAP_WATCHDOG_REV) & 0xFF,
 		wdev->wdog.timeout);
 
-	pm_runtime_put_sync(wdev->dev);
-
 	if (early_enable)
 		omap_wdt_start(&wdev->wdog);
 
+	pm_runtime_put(wdev->dev);
+
 	return 0;
 }
 
diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c
index 773dcfaee7b2..424f9a952fee 100644
--- a/drivers/watchdog/qcom-wdt.c
+++ b/drivers/watchdog/qcom-wdt.c
@@ -17,7 +17,6 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/watchdog.h>
 
 #define WDT_RST		0x38
@@ -28,7 +27,6 @@ struct qcom_wdt {
 	struct watchdog_device	wdd;
 	struct clk		*clk;
 	unsigned long		rate;
-	struct notifier_block	restart_nb;
 	void __iomem		*base;
 };
 
@@ -72,25 +70,9 @@ static int qcom_wdt_set_timeout(struct watchdog_device *wdd,
 	return qcom_wdt_start(wdd);
 }
 
-static const struct watchdog_ops qcom_wdt_ops = {
-	.start		= qcom_wdt_start,
-	.stop		= qcom_wdt_stop,
-	.ping		= qcom_wdt_ping,
-	.set_timeout	= qcom_wdt_set_timeout,
-	.owner		= THIS_MODULE,
-};
-
-static const struct watchdog_info qcom_wdt_info = {
-	.options	= WDIOF_KEEPALIVEPING
-			| WDIOF_MAGICCLOSE
-			| WDIOF_SETTIMEOUT,
-	.identity	= KBUILD_MODNAME,
-};
-
-static int qcom_wdt_restart(struct notifier_block *nb, unsigned long action,
-			    void *data)
+static int qcom_wdt_restart(struct watchdog_device *wdd)
 {
-	struct qcom_wdt *wdt = container_of(nb, struct qcom_wdt, restart_nb);
+	struct qcom_wdt *wdt = to_qcom_wdt(wdd);
 	u32 timeout;
 
 	/*
@@ -110,9 +92,25 @@ static int qcom_wdt_restart(struct notifier_block *nb, unsigned long action,
 	wmb();
 
 	msleep(150);
-	return NOTIFY_DONE;
+	return 0;
 }
 
+static const struct watchdog_ops qcom_wdt_ops = {
+	.start		= qcom_wdt_start,
+	.stop		= qcom_wdt_stop,
+	.ping		= qcom_wdt_ping,
+	.set_timeout	= qcom_wdt_set_timeout,
+	.restart        = qcom_wdt_restart,
+	.owner		= THIS_MODULE,
+};
+
+static const struct watchdog_info qcom_wdt_info = {
+	.options	= WDIOF_KEEPALIVEPING
+			| WDIOF_MAGICCLOSE
+			| WDIOF_SETTIMEOUT,
+	.identity	= KBUILD_MODNAME,
+};
+
 static int qcom_wdt_probe(struct platform_device *pdev)
 {
 	struct qcom_wdt *wdt;
@@ -166,7 +164,6 @@ static int qcom_wdt_probe(struct platform_device *pdev)
 		goto err_clk_unprepare;
 	}
 
-	wdt->wdd.dev = &pdev->dev;
 	wdt->wdd.info = &qcom_wdt_info;
 	wdt->wdd.ops = &qcom_wdt_ops;
 	wdt->wdd.min_timeout = 1;
@@ -187,14 +184,6 @@ static int qcom_wdt_probe(struct platform_device *pdev)
 		goto err_clk_unprepare;
 	}
 
-	/*
-	 * WDT restart notifier has priority 0 (use as a last resort)
-	 */
-	wdt->restart_nb.notifier_call = qcom_wdt_restart;
-	ret = register_restart_handler(&wdt->restart_nb);
-	if (ret)
-		dev_err(&pdev->dev, "failed to setup restart handler\n");
-
 	platform_set_drvdata(pdev, wdt);
 	return 0;
 
@@ -207,7 +196,6 @@ static int qcom_wdt_remove(struct platform_device *pdev)
 {
 	struct qcom_wdt *wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&wdt->restart_nb);
 	watchdog_unregister_device(&wdt->wdd);
 	clk_disable_unprepare(wdt->clk);
 	return 0;
diff --git a/drivers/watchdog/s3c2410_wdt.c b/drivers/watchdog/s3c2410_wdt.c
index d781000c7825..0093450441fe 100644
--- a/drivers/watchdog/s3c2410_wdt.c
+++ b/drivers/watchdog/s3c2410_wdt.c
@@ -41,7 +41,6 @@
 #include <linux/of.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
-#include <linux/reboot.h>
 #include <linux/delay.h>
 
 #define S3C2410_WTCON		0x00
@@ -130,7 +129,6 @@ struct s3c2410_wdt {
 	unsigned long		wtdat_save;
 	struct watchdog_device	wdt_device;
 	struct notifier_block	freq_transition;
-	struct notifier_block	restart_handler;
 	struct s3c2410_wdt_variant *drv_data;
 	struct regmap *pmureg;
 };
@@ -351,6 +349,29 @@ static int s3c2410wdt_set_heartbeat(struct watchdog_device *wdd, unsigned timeou
 	return 0;
 }
 
+static int s3c2410wdt_restart(struct watchdog_device *wdd)
+{
+	struct s3c2410_wdt *wdt = watchdog_get_drvdata(wdd);
+	void __iomem *wdt_base = wdt->reg_base;
+
+	/* disable watchdog, to be safe  */
+	writel(0, wdt_base + S3C2410_WTCON);
+
+	/* put initial values into count and data */
+	writel(0x80, wdt_base + S3C2410_WTCNT);
+	writel(0x80, wdt_base + S3C2410_WTDAT);
+
+	/* set the watchdog to go and reset... */
+	writel(S3C2410_WTCON_ENABLE | S3C2410_WTCON_DIV16 |
+		S3C2410_WTCON_RSTEN | S3C2410_WTCON_PRESCALE(0x20),
+		wdt_base + S3C2410_WTCON);
+
+	/* wait for reset to assert... */
+	mdelay(500);
+
+	return 0;
+}
+
 #define OPTIONS (WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE)
 
 static const struct watchdog_info s3c2410_wdt_ident = {
@@ -365,6 +386,7 @@ static struct watchdog_ops s3c2410wdt_ops = {
 	.stop = s3c2410wdt_stop,
 	.ping = s3c2410wdt_keepalive,
 	.set_timeout = s3c2410wdt_set_heartbeat,
+	.restart = s3c2410wdt_restart,
 };
 
 static struct watchdog_device s3c2410_wdd = {
@@ -452,31 +474,6 @@ static inline void s3c2410wdt_cpufreq_deregister(struct s3c2410_wdt *wdt)
 }
 #endif
 
-static int s3c2410wdt_restart(struct notifier_block *this,
-			      unsigned long mode, void *cmd)
-{
-	struct s3c2410_wdt *wdt = container_of(this, struct s3c2410_wdt,
-					       restart_handler);
-	void __iomem *wdt_base = wdt->reg_base;
-
-	/* disable watchdog, to be safe  */
-	writel(0, wdt_base + S3C2410_WTCON);
-
-	/* put initial values into count and data */
-	writel(0x80, wdt_base + S3C2410_WTCNT);
-	writel(0x80, wdt_base + S3C2410_WTDAT);
-
-	/* set the watchdog to go and reset... */
-	writel(S3C2410_WTCON_ENABLE | S3C2410_WTCON_DIV16 |
-		S3C2410_WTCON_RSTEN | S3C2410_WTCON_PRESCALE(0x20),
-		wdt_base + S3C2410_WTCON);
-
-	/* wait for reset to assert... */
-	mdelay(500);
-
-	return NOTIFY_DONE;
-}
-
 static inline unsigned int s3c2410wdt_get_bootstatus(struct s3c2410_wdt *wdt)
 {
 	unsigned int rst_stat;
@@ -605,6 +602,7 @@ static int s3c2410wdt_probe(struct platform_device *pdev)
 	}
 
 	watchdog_set_nowayout(&wdt->wdt_device, nowayout);
+	watchdog_set_restart_priority(&wdt->wdt_device, 128);
 
 	wdt->wdt_device.bootstatus = s3c2410wdt_get_bootstatus(wdt);
 	wdt->wdt_device.parent = &pdev->dev;
@@ -632,12 +630,6 @@ static int s3c2410wdt_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, wdt);
 
-	wdt->restart_handler.notifier_call = s3c2410wdt_restart;
-	wdt->restart_handler.priority = 128;
-	ret = register_restart_handler(&wdt->restart_handler);
-	if (ret)
-		pr_err("cannot register restart handler, %d\n", ret);
-
 	/* print out a statement of readiness */
 
 	wtcon = readl(wdt->reg_base + S3C2410_WTCON);
@@ -667,8 +659,6 @@ static int s3c2410wdt_remove(struct platform_device *dev)
 	int ret;
 	struct s3c2410_wdt *wdt = platform_get_drvdata(dev);
 
-	unregister_restart_handler(&wdt->restart_handler);
-
 	ret = s3c2410wdt_mask_and_disable_reset(wdt, true);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/watchdog/softdog.c b/drivers/watchdog/softdog.c
index 0dc5e323d59d..99a06f9e3930 100644
--- a/drivers/watchdog/softdog.c
+++ b/drivers/watchdog/softdog.c
@@ -43,7 +43,6 @@
 #include <linux/types.h>
 #include <linux/timer.h>
 #include <linux/watchdog.h>
-#include <linux/notifier.h>
 #include <linux/reboot.h>
 #include <linux/init.h>
 #include <linux/jiffies.h>
@@ -87,6 +86,7 @@ static struct timer_list watchdog_ticktock =
 
 static void watchdog_fire(unsigned long data)
 {
+	module_put(THIS_MODULE);
 	if (soft_noboot)
 		pr_crit("Triggered - Reboot ignored\n");
 	else if (soft_panic) {
@@ -105,13 +105,16 @@ static void watchdog_fire(unsigned long data)
 
 static int softdog_ping(struct watchdog_device *w)
 {
-	mod_timer(&watchdog_ticktock, jiffies+(w->timeout*HZ));
+	if (!mod_timer(&watchdog_ticktock, jiffies+(w->timeout*HZ)))
+		__module_get(THIS_MODULE);
 	return 0;
 }
 
 static int softdog_stop(struct watchdog_device *w)
 {
-	del_timer(&watchdog_ticktock);
+	if (del_timer(&watchdog_ticktock))
+		module_put(THIS_MODULE);
+
 	return 0;
 }
 
@@ -122,26 +125,9 @@ static int softdog_set_timeout(struct watchdog_device *w, unsigned int t)
 }
 
 /*
- *	Notifier for system down
- */
-
-static int softdog_notify_sys(struct notifier_block *this, unsigned long code,
-	void *unused)
-{
-	if (code == SYS_DOWN || code == SYS_HALT)
-		/* Turn the WDT off */
-		softdog_stop(NULL);
-	return NOTIFY_DONE;
-}
-
-/*
  *	Kernel Interfaces
  */
 
-static struct notifier_block softdog_notifier = {
-	.notifier_call	= softdog_notify_sys,
-};
-
 static struct watchdog_info softdog_info = {
 	.identity = "Software Watchdog",
 	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
@@ -175,18 +161,11 @@ static int __init watchdog_init(void)
 	softdog_dev.timeout = soft_margin;
 
 	watchdog_set_nowayout(&softdog_dev, nowayout);
-
-	ret = register_reboot_notifier(&softdog_notifier);
-	if (ret) {
-		pr_err("cannot register reboot notifier (err=%d)\n", ret);
-		return ret;
-	}
+	watchdog_stop_on_reboot(&softdog_dev);
 
 	ret = watchdog_register_device(&softdog_dev);
-	if (ret) {
-		unregister_reboot_notifier(&softdog_notifier);
+	if (ret)
 		return ret;
-	}
 
 	pr_info("Software Watchdog Timer: 0.08 initialized. soft_noboot=%d soft_margin=%d sec soft_panic=%d (nowayout=%d)\n",
 		soft_noboot, soft_margin, soft_panic, nowayout);
@@ -197,7 +176,6 @@ static int __init watchdog_init(void)
 static void __exit watchdog_exit(void)
 {
 	watchdog_unregister_device(&softdog_dev);
-	unregister_reboot_notifier(&softdog_notifier);
 }
 
 module_init(watchdog_init);
diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c
index eb8044ef0ea0..6467b91f2245 100644
--- a/drivers/watchdog/sp5100_tco.c
+++ b/drivers/watchdog/sp5100_tco.c
@@ -306,6 +306,10 @@ static struct miscdevice sp5100_tco_miscdev = {
 static const struct pci_device_id sp5100_tco_pci_tbl[] = {
 	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_SBX00_SMBUS, PCI_ANY_ID,
 	  PCI_ANY_ID, },
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_HUDSON2_SMBUS, PCI_ANY_ID,
+	  PCI_ANY_ID, },
+	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_KERNCZ_SMBUS, PCI_ANY_ID,
+	  PCI_ANY_ID, },
 	{ 0, },			/* End of list */
 };
 MODULE_DEVICE_TABLE(pci, sp5100_tco_pci_tbl);
@@ -331,21 +335,24 @@ static unsigned char sp5100_tco_setupdevice(void)
 	if (!sp5100_tco_pci)
 		return 0;
 
-	pr_info("PCI Revision ID: 0x%x\n", sp5100_tco_pci->revision);
+	pr_info("PCI Vendor ID: 0x%x, Device ID: 0x%x, Revision ID: 0x%x\n",
+		sp5100_tco_pci->vendor, sp5100_tco_pci->device,
+		sp5100_tco_pci->revision);
 
 	/*
 	 * Determine type of southbridge chipset.
 	 */
-	if (sp5100_tco_pci->revision >= 0x40) {
-		dev_name = SB800_DEVNAME;
-		index_reg = SB800_IO_PM_INDEX_REG;
-		data_reg = SB800_IO_PM_DATA_REG;
-		base_addr = SB800_PM_WATCHDOG_BASE;
-	} else {
+	if (sp5100_tco_pci->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
+	    sp5100_tco_pci->revision < 0x40) {
 		dev_name = SP5100_DEVNAME;
 		index_reg = SP5100_IO_PM_INDEX_REG;
 		data_reg = SP5100_IO_PM_DATA_REG;
 		base_addr = SP5100_PM_WATCHDOG_BASE;
+	} else {
+		dev_name = SB800_DEVNAME;
+		index_reg = SB800_IO_PM_INDEX_REG;
+		data_reg = SB800_IO_PM_DATA_REG;
+		base_addr = SB800_PM_WATCHDOG_BASE;
 	}
 
 	/* Request the IO ports used by this driver */
@@ -381,7 +388,12 @@ static unsigned char sp5100_tco_setupdevice(void)
 	 * Secondly, Find the watchdog timer MMIO address
 	 * from SBResource_MMIO register.
 	 */
-	if (sp5100_tco_pci->revision >= 0x40) {
+	if (sp5100_tco_pci->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
+	    sp5100_tco_pci->revision < 0x40) {
+		/* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */
+		pci_read_config_dword(sp5100_tco_pci,
+				      SP5100_SB_RESOURCE_MMIO_BASE, &val);
+	} else {
 		/* Read SBResource_MMIO from AcpiMmioEn(PM_Reg: 24h) */
 		outb(SB800_PM_ACPI_MMIO_EN+3, SB800_IO_PM_INDEX_REG);
 		val = inb(SB800_IO_PM_DATA_REG);
@@ -391,10 +403,6 @@ static unsigned char sp5100_tco_setupdevice(void)
 		val = val << 8 | inb(SB800_IO_PM_DATA_REG);
 		outb(SB800_PM_ACPI_MMIO_EN+0, SB800_IO_PM_INDEX_REG);
 		val = val << 8 | inb(SB800_IO_PM_DATA_REG);
-	} else {
-		/* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */
-		pci_read_config_dword(sp5100_tco_pci,
-				      SP5100_SB_RESOURCE_MMIO_BASE, &val);
 	}
 
 	/* The SBResource_MMIO is enabled and mapped memory space? */
diff --git a/drivers/watchdog/stmp3xxx_rtc_wdt.c b/drivers/watchdog/stmp3xxx_rtc_wdt.c
index 3ee6128a540e..d8b11eb269ad 100644
--- a/drivers/watchdog/stmp3xxx_rtc_wdt.c
+++ b/drivers/watchdog/stmp3xxx_rtc_wdt.c
@@ -14,6 +14,8 @@
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
 #include <linux/stmp3xxx_rtc_wdt.h>
+#include <linux/notifier.h>
+#include <linux/reboot.h>
 
 #define WDOG_TICK_RATE 1000 /* 1 kHz clock */
 #define STMP3XXX_DEFAULT_TIMEOUT 19
@@ -69,6 +71,25 @@ static struct watchdog_device stmp3xxx_wdd = {
 	.status = WATCHDOG_NOWAYOUT_INIT_STATUS,
 };
 
+static int wdt_notify_sys(struct notifier_block *nb, unsigned long code,
+			  void *unused)
+{
+	switch (code) {
+	case SYS_DOWN:	/* keep enabled, system might crash while going down */
+		break;
+	case SYS_HALT:	/* allow the system to actually halt */
+	case SYS_POWER_OFF:
+		wdt_stop(&stmp3xxx_wdd);
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block wdt_notifier = {
+	.notifier_call = wdt_notify_sys,
+};
+
 static int stmp3xxx_wdt_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -84,6 +105,9 @@ static int stmp3xxx_wdt_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	if (register_reboot_notifier(&wdt_notifier))
+		dev_warn(&pdev->dev, "cannot register reboot notifier\n");
+
 	dev_info(&pdev->dev, "initialized watchdog with heartbeat %ds\n",
 			stmp3xxx_wdd.timeout);
 	return 0;
@@ -91,6 +115,7 @@ static int stmp3xxx_wdt_probe(struct platform_device *pdev)
 
 static int stmp3xxx_wdt_remove(struct platform_device *pdev)
 {
+	unregister_reboot_notifier(&wdt_notifier);
 	watchdog_unregister_device(&stmp3xxx_wdd);
 	return 0;
 }
diff --git a/drivers/watchdog/sunxi_wdt.c b/drivers/watchdog/sunxi_wdt.c
index 47bd8a14d01f..e027deb54740 100644
--- a/drivers/watchdog/sunxi_wdt.c
+++ b/drivers/watchdog/sunxi_wdt.c
@@ -21,11 +21,9 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <linux/notifier.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
-#include <linux/reboot.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -60,7 +58,6 @@ struct sunxi_wdt_dev {
 	struct watchdog_device wdt_dev;
 	void __iomem *wdt_base;
 	const struct sunxi_wdt_reg *wdt_regs;
-	struct notifier_block restart_handler;
 };
 
 /*
@@ -86,12 +83,9 @@ static const int wdt_timeout_map[] = {
 };
 
 
-static int sunxi_restart_handle(struct notifier_block *this, unsigned long mode,
-				void *cmd)
+static int sunxi_wdt_restart(struct watchdog_device *wdt_dev)
 {
-	struct sunxi_wdt_dev *sunxi_wdt = container_of(this,
-						       struct sunxi_wdt_dev,
-						       restart_handler);
+	struct sunxi_wdt_dev *sunxi_wdt = watchdog_get_drvdata(wdt_dev);
 	void __iomem *wdt_base = sunxi_wdt->wdt_base;
 	const struct sunxi_wdt_reg *regs = sunxi_wdt->wdt_regs;
 	u32 val;
@@ -120,7 +114,7 @@ static int sunxi_restart_handle(struct notifier_block *this, unsigned long mode,
 		val |= WDT_MODE_EN;
 		writel(val, wdt_base + regs->wdt_mode);
 	}
-	return NOTIFY_DONE;
+	return 0;
 }
 
 static int sunxi_wdt_ping(struct watchdog_device *wdt_dev)
@@ -208,6 +202,7 @@ static const struct watchdog_ops sunxi_wdt_ops = {
 	.stop		= sunxi_wdt_stop,
 	.ping		= sunxi_wdt_ping,
 	.set_timeout	= sunxi_wdt_set_timeout,
+	.restart	= sunxi_wdt_restart,
 };
 
 static const struct sunxi_wdt_reg sun4i_wdt_reg = {
@@ -268,6 +263,7 @@ static int sunxi_wdt_probe(struct platform_device *pdev)
 
 	watchdog_init_timeout(&sunxi_wdt->wdt_dev, timeout, &pdev->dev);
 	watchdog_set_nowayout(&sunxi_wdt->wdt_dev, nowayout);
+	watchdog_set_restart_priority(&sunxi_wdt->wdt_dev, 128);
 
 	watchdog_set_drvdata(&sunxi_wdt->wdt_dev, sunxi_wdt);
 
@@ -277,13 +273,6 @@ static int sunxi_wdt_probe(struct platform_device *pdev)
 	if (unlikely(err))
 		return err;
 
-	sunxi_wdt->restart_handler.notifier_call = sunxi_restart_handle;
-	sunxi_wdt->restart_handler.priority = 128;
-	err = register_restart_handler(&sunxi_wdt->restart_handler);
-	if (err)
-		dev_err(&pdev->dev,
-			"cannot register restart handler (err=%d)\n", err);
-
 	dev_info(&pdev->dev, "Watchdog enabled (timeout=%d sec, nowayout=%d)",
 			sunxi_wdt->wdt_dev.timeout, nowayout);
 
@@ -294,8 +283,6 @@ static int sunxi_wdt_remove(struct platform_device *pdev)
 {
 	struct sunxi_wdt_dev *sunxi_wdt = platform_get_drvdata(pdev);
 
-	unregister_restart_handler(&sunxi_wdt->restart_handler);
-
 	watchdog_unregister_device(&sunxi_wdt->wdt_dev);
 	watchdog_set_drvdata(&sunxi_wdt->wdt_dev, NULL);
 
diff --git a/drivers/watchdog/tangox_wdt.c b/drivers/watchdog/tangox_wdt.c
new file mode 100644
index 000000000000..709c1ed6fd79
--- /dev/null
+++ b/drivers/watchdog/tangox_wdt.c
@@ -0,0 +1,225 @@
+/*
+ *  Copyright (C) 2015 Mans Rullgard <mans@mansr.com>
+ *  SMP86xx/SMP87xx Watchdog driver
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under  the terms of the GNU General  Public License as published by the
+ *  Free Software Foundation;  either version 2 of the License, or (at your
+ *  option) any later version.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/notifier.h>
+#include <linux/platform_device.h>
+#include <linux/reboot.h>
+#include <linux/watchdog.h>
+
+#define DEFAULT_TIMEOUT 30
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout,
+		 "Watchdog cannot be stopped once started (default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+static unsigned int timeout;
+module_param(timeout, int, 0);
+MODULE_PARM_DESC(timeout, "Watchdog timeout");
+
+/*
+ * Counter counts down from programmed value.  Reset asserts when
+ * the counter reaches 1.
+ */
+#define WD_COUNTER		0
+
+#define WD_CONFIG		4
+#define WD_CONFIG_XTAL_IN	BIT(0)
+#define WD_CONFIG_DISABLE	BIT(31)
+
+struct tangox_wdt_device {
+	struct watchdog_device wdt;
+	void __iomem *base;
+	unsigned long clk_rate;
+	struct clk *clk;
+	struct notifier_block restart;
+};
+
+static int tangox_wdt_set_timeout(struct watchdog_device *wdt,
+				  unsigned int new_timeout)
+{
+	wdt->timeout = new_timeout;
+
+	return 0;
+}
+
+static int tangox_wdt_start(struct watchdog_device *wdt)
+{
+	struct tangox_wdt_device *dev = watchdog_get_drvdata(wdt);
+	u32 ticks;
+
+	ticks = 1 + wdt->timeout * dev->clk_rate;
+	writel(ticks, dev->base + WD_COUNTER);
+
+	return 0;
+}
+
+static int tangox_wdt_stop(struct watchdog_device *wdt)
+{
+	struct tangox_wdt_device *dev = watchdog_get_drvdata(wdt);
+
+	writel(0, dev->base + WD_COUNTER);
+
+	return 0;
+}
+
+static unsigned int tangox_wdt_get_timeleft(struct watchdog_device *wdt)
+{
+	struct tangox_wdt_device *dev = watchdog_get_drvdata(wdt);
+	u32 count;
+
+	count = readl(dev->base + WD_COUNTER);
+
+	if (!count)
+		return 0;
+
+	return (count - 1) / dev->clk_rate;
+}
+
+static const struct watchdog_info tangox_wdt_info = {
+	.options  = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
+	.identity = "tangox watchdog",
+};
+
+static const struct watchdog_ops tangox_wdt_ops = {
+	.start		= tangox_wdt_start,
+	.stop		= tangox_wdt_stop,
+	.set_timeout	= tangox_wdt_set_timeout,
+	.get_timeleft	= tangox_wdt_get_timeleft,
+};
+
+static int tangox_wdt_restart(struct notifier_block *nb, unsigned long action,
+			      void *data)
+{
+	struct tangox_wdt_device *dev =
+		container_of(nb, struct tangox_wdt_device, restart);
+
+	writel(1, dev->base + WD_COUNTER);
+
+	return NOTIFY_DONE;
+}
+
+static int tangox_wdt_probe(struct platform_device *pdev)
+{
+	struct tangox_wdt_device *dev;
+	struct resource *res;
+	u32 config;
+	int err;
+
+	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dev->base))
+		return PTR_ERR(dev->base);
+
+	dev->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(dev->clk))
+		return PTR_ERR(dev->clk);
+
+	err = clk_prepare_enable(dev->clk);
+	if (err)
+		return err;
+
+	dev->clk_rate = clk_get_rate(dev->clk);
+
+	dev->wdt.parent = &pdev->dev;
+	dev->wdt.info = &tangox_wdt_info;
+	dev->wdt.ops = &tangox_wdt_ops;
+	dev->wdt.timeout = DEFAULT_TIMEOUT;
+	dev->wdt.min_timeout = 1;
+	dev->wdt.max_timeout = (U32_MAX - 1) / dev->clk_rate;
+
+	watchdog_init_timeout(&dev->wdt, timeout, &pdev->dev);
+	watchdog_set_nowayout(&dev->wdt, nowayout);
+	watchdog_set_drvdata(&dev->wdt, dev);
+
+	/*
+	 * Deactivate counter if disable bit is set to avoid
+	 * accidental reset.
+	 */
+	config = readl(dev->base + WD_CONFIG);
+	if (config & WD_CONFIG_DISABLE)
+		writel(0, dev->base + WD_COUNTER);
+
+	writel(WD_CONFIG_XTAL_IN, dev->base + WD_CONFIG);
+
+	/*
+	 * Mark as active and restart with configured timeout if
+	 * already running.
+	 */
+	if (readl(dev->base + WD_COUNTER)) {
+		set_bit(WDOG_ACTIVE, &dev->wdt.status);
+		tangox_wdt_start(&dev->wdt);
+	}
+
+	err = watchdog_register_device(&dev->wdt);
+	if (err) {
+		clk_disable_unprepare(dev->clk);
+		return err;
+	}
+
+	platform_set_drvdata(pdev, dev);
+
+	dev->restart.notifier_call = tangox_wdt_restart;
+	dev->restart.priority = 128;
+	err = register_restart_handler(&dev->restart);
+	if (err)
+		dev_warn(&pdev->dev, "failed to register restart handler\n");
+
+	dev_info(&pdev->dev, "SMP86xx/SMP87xx watchdog registered\n");
+
+	return 0;
+}
+
+static int tangox_wdt_remove(struct platform_device *pdev)
+{
+	struct tangox_wdt_device *dev = platform_get_drvdata(pdev);
+
+	tangox_wdt_stop(&dev->wdt);
+	clk_disable_unprepare(dev->clk);
+
+	unregister_restart_handler(&dev->restart);
+	watchdog_unregister_device(&dev->wdt);
+
+	return 0;
+}
+
+static const struct of_device_id tangox_wdt_dt_ids[] = {
+	{ .compatible = "sigma,smp8642-wdt" },
+	{ .compatible = "sigma,smp8759-wdt" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tangox_wdt_dt_ids);
+
+static struct platform_driver tangox_wdt_driver = {
+	.probe	= tangox_wdt_probe,
+	.remove	= tangox_wdt_remove,
+	.driver	= {
+		.name		= "tangox-wdt",
+		.of_match_table	= tangox_wdt_dt_ids,
+	},
+};
+
+module_platform_driver(tangox_wdt_driver);
+
+MODULE_AUTHOR("Mans Rullgard <mans@mansr.com>");
+MODULE_DESCRIPTION("SMP86xx/SMP87xx Watchdog driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/ts4800_wdt.c b/drivers/watchdog/ts4800_wdt.c
new file mode 100644
index 000000000000..2b8de8602b67
--- /dev/null
+++ b/drivers/watchdog/ts4800_wdt.c
@@ -0,0 +1,215 @@
+/*
+ * Watchdog driver for TS-4800 based boards
+ *
+ * Copyright (c) 2015 - Savoir-faire Linux
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/watchdog.h>
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout,
+	"Watchdog cannot be stopped once started (default="
+	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+/* possible feed values */
+#define TS4800_WDT_FEED_2S       0x1
+#define TS4800_WDT_FEED_10S      0x2
+#define TS4800_WDT_DISABLE       0x3
+
+struct ts4800_wdt {
+	struct watchdog_device  wdd;
+	struct regmap           *regmap;
+	u32                     feed_offset;
+	u32                     feed_val;
+};
+
+/*
+ * TS-4800 supports the following timeout values:
+ *
+ *   value desc
+ *   ---------------------
+ *     0    feed for 338ms
+ *     1    feed for 2.706s
+ *     2    feed for 10.824s
+ *     3    disable watchdog
+ *
+ * Keep the regmap/timeout map ordered by timeout
+ */
+static const struct {
+	const int timeout;
+	const int regval;
+} ts4800_wdt_map[] = {
+	{ 2,  TS4800_WDT_FEED_2S },
+	{ 10, TS4800_WDT_FEED_10S },
+};
+
+#define MAX_TIMEOUT_INDEX       (ARRAY_SIZE(ts4800_wdt_map) - 1)
+
+static void ts4800_write_feed(struct ts4800_wdt *wdt, u32 val)
+{
+	regmap_write(wdt->regmap, wdt->feed_offset, val);
+}
+
+static int ts4800_wdt_start(struct watchdog_device *wdd)
+{
+	struct ts4800_wdt *wdt = watchdog_get_drvdata(wdd);
+
+	ts4800_write_feed(wdt, wdt->feed_val);
+	return 0;
+}
+
+static int ts4800_wdt_stop(struct watchdog_device *wdd)
+{
+	struct ts4800_wdt *wdt = watchdog_get_drvdata(wdd);
+
+	ts4800_write_feed(wdt, TS4800_WDT_DISABLE);
+	return 0;
+}
+
+static int ts4800_wdt_set_timeout(struct watchdog_device *wdd,
+				  unsigned int timeout)
+{
+	struct ts4800_wdt *wdt = watchdog_get_drvdata(wdd);
+	int i;
+
+	for (i = 0; i < MAX_TIMEOUT_INDEX; i++) {
+		if (ts4800_wdt_map[i].timeout >= timeout)
+			break;
+	}
+
+	wdd->timeout = ts4800_wdt_map[i].timeout;
+	wdt->feed_val = ts4800_wdt_map[i].regval;
+
+	return 0;
+}
+
+static const struct watchdog_ops ts4800_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = ts4800_wdt_start,
+	.stop = ts4800_wdt_stop,
+	.set_timeout = ts4800_wdt_set_timeout,
+};
+
+static const struct watchdog_info ts4800_wdt_info = {
+	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
+	.identity = "TS-4800 Watchdog",
+};
+
+static int ts4800_wdt_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *syscon_np;
+	struct watchdog_device *wdd;
+	struct ts4800_wdt *wdt;
+	u32 reg;
+	int ret;
+
+	syscon_np = of_parse_phandle(np, "syscon", 0);
+	if (!syscon_np) {
+		dev_err(&pdev->dev, "no syscon property\n");
+		return -ENODEV;
+	}
+
+	ret = of_property_read_u32_index(np, "syscon", 1, &reg);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "no offset in syscon\n");
+		return ret;
+	}
+
+	/* allocate memory for watchdog struct */
+	wdt = devm_kzalloc(&pdev->dev, sizeof(*wdt), GFP_KERNEL);
+	if (!wdt)
+		return -ENOMEM;
+
+	/* set regmap and offset to know where to write */
+	wdt->feed_offset = reg;
+	wdt->regmap = syscon_node_to_regmap(syscon_np);
+	if (IS_ERR(wdt->regmap)) {
+		dev_err(&pdev->dev, "cannot get parent's regmap\n");
+		return PTR_ERR(wdt->regmap);
+	}
+
+	/* Initialize struct watchdog_device */
+	wdd = &wdt->wdd;
+	wdd->parent = &pdev->dev;
+	wdd->info = &ts4800_wdt_info;
+	wdd->ops = &ts4800_wdt_ops;
+	wdd->min_timeout = ts4800_wdt_map[0].timeout;
+	wdd->max_timeout = ts4800_wdt_map[MAX_TIMEOUT_INDEX].timeout;
+
+	watchdog_set_drvdata(wdd, wdt);
+	watchdog_set_nowayout(wdd, nowayout);
+	watchdog_init_timeout(wdd, 0, &pdev->dev);
+
+	/*
+	 * As this watchdog supports only a few values, ts4800_wdt_set_timeout
+	 * must be called to initialize timeout and feed_val with valid values.
+	 * Default to maximum timeout if none, or an invalid one, is provided in
+	 * device tree.
+	 */
+	if (!wdd->timeout)
+		wdd->timeout = wdd->max_timeout;
+	ts4800_wdt_set_timeout(wdd, wdd->timeout);
+
+	/*
+	 * The feed register is write-only, so it is not possible to determine
+	 * watchdog's state. Disable it to be in a known state.
+	 */
+	ts4800_wdt_stop(wdd);
+
+	ret = watchdog_register_device(wdd);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"failed to register watchdog device\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, wdt);
+
+	dev_info(&pdev->dev,
+		 "initialized (timeout = %d sec, nowayout = %d)\n",
+		 wdd->timeout, nowayout);
+
+	return 0;
+}
+
+static int ts4800_wdt_remove(struct platform_device *pdev)
+{
+	struct ts4800_wdt *wdt = platform_get_drvdata(pdev);
+
+	watchdog_unregister_device(&wdt->wdd);
+
+	return 0;
+}
+
+static const struct of_device_id ts4800_wdt_of_match[] = {
+	{ .compatible = "technologic,ts4800-wdt", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ts4800_wdt_of_match);
+
+static struct platform_driver ts4800_wdt_driver = {
+	.probe		= ts4800_wdt_probe,
+	.remove		= ts4800_wdt_remove,
+	.driver		= {
+		.name	= "ts4800_wdt",
+		.of_match_table = ts4800_wdt_of_match,
+	},
+};
+
+module_platform_driver(ts4800_wdt_driver);
+
+MODULE_AUTHOR("Damien Riegel <damien.riegel@savoirfairelinux.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:ts4800_wdt");
diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c
index 5824e25eebbb..cab14bc9106c 100644
--- a/drivers/watchdog/w83627hf_wdt.c
+++ b/drivers/watchdog/w83627hf_wdt.c
@@ -36,8 +36,6 @@
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <linux/ioport.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
 #include <linux/init.h>
 #include <linux/io.h>
 
@@ -288,18 +286,6 @@ static unsigned int wdt_get_time(struct watchdog_device *wdog)
 }
 
 /*
- *	Notifier for system down
- */
-static int wdt_notify_sys(struct notifier_block *this, unsigned long code,
-	void *unused)
-{
-	if (code == SYS_DOWN || code == SYS_HALT)
-		wdt_set_time(0);	/* Turn the WDT off */
-
-	return NOTIFY_DONE;
-}
-
-/*
  *	Kernel Interfaces
  */
 
@@ -329,10 +315,6 @@ static struct watchdog_device wdt_dev = {
  *	turn the timebomb registers off.
  */
 
-static struct notifier_block wdt_notifier = {
-	.notifier_call = wdt_notify_sys,
-};
-
 static int wdt_find(int addr)
 {
 	u8 val;
@@ -456,6 +438,7 @@ static int __init wdt_init(void)
 
 	watchdog_init_timeout(&wdt_dev, timeout, NULL);
 	watchdog_set_nowayout(&wdt_dev, nowayout);
+	watchdog_stop_on_reboot(&wdt_dev);
 
 	ret = w83627hf_init(&wdt_dev, chip);
 	if (ret) {
@@ -463,30 +446,19 @@ static int __init wdt_init(void)
 		return ret;
 	}
 
-	ret = register_reboot_notifier(&wdt_notifier);
-	if (ret != 0) {
-		pr_err("cannot register reboot notifier (err=%d)\n", ret);
-		return ret;
-	}
-
 	ret = watchdog_register_device(&wdt_dev);
 	if (ret)
-		goto unreg_reboot;
+		return ret;
 
 	pr_info("initialized. timeout=%d sec (nowayout=%d)\n",
 		wdt_dev.timeout, nowayout);
 
 	return ret;
-
-unreg_reboot:
-	unregister_reboot_notifier(&wdt_notifier);
-	return ret;
 }
 
 static void __exit wdt_exit(void)
 {
 	watchdog_unregister_device(&wdt_dev);
-	unregister_reboot_notifier(&wdt_notifier);
 }
 
 module_init(wdt_init);
diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c
index 873f13972cf4..e600fd93b7de 100644
--- a/drivers/watchdog/watchdog_core.c
+++ b/drivers/watchdog/watchdog_core.c
@@ -32,6 +32,7 @@
 #include <linux/types.h>	/* For standard types */
 #include <linux/errno.h>	/* For the -ENODEV/... values */
 #include <linux/kernel.h>	/* For printk/panic/... */
+#include <linux/reboot.h>	/* For restart handler */
 #include <linux/watchdog.h>	/* For watchdog specific items */
 #include <linux/init.h>		/* For __init/__exit/... */
 #include <linux/idr.h>		/* For ida_* macros */
@@ -41,7 +42,6 @@
 #include "watchdog_core.h"	/* For watchdog_dev_register/... */
 
 static DEFINE_IDA(watchdog_ida);
-static struct class *watchdog_class;
 
 /*
  * Deferred Registration infrastructure.
@@ -137,9 +137,63 @@ int watchdog_init_timeout(struct watchdog_device *wdd,
 }
 EXPORT_SYMBOL_GPL(watchdog_init_timeout);
 
+static int watchdog_reboot_notifier(struct notifier_block *nb,
+				    unsigned long code, void *data)
+{
+	struct watchdog_device *wdd = container_of(nb, struct watchdog_device,
+						   reboot_nb);
+
+	if (code == SYS_DOWN || code == SYS_HALT) {
+		if (watchdog_active(wdd)) {
+			int ret;
+
+			ret = wdd->ops->stop(wdd);
+			if (ret)
+				return NOTIFY_BAD;
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int watchdog_restart_notifier(struct notifier_block *nb,
+				     unsigned long action, void *data)
+{
+	struct watchdog_device *wdd = container_of(nb, struct watchdog_device,
+						   restart_nb);
+
+	int ret;
+
+	ret = wdd->ops->restart(wdd);
+	if (ret)
+		return NOTIFY_BAD;
+
+	return NOTIFY_DONE;
+}
+
+/**
+ * watchdog_set_restart_priority - Change priority of restart handler
+ * @wdd: watchdog device
+ * @priority: priority of the restart handler, should follow these guidelines:
+ *   0:   use watchdog's restart function as last resort, has limited restart
+ *        capabilies
+ *   128: default restart handler, use if no other handler is expected to be
+ *        available and/or if restart is sufficient to restart the entire system
+ *   255: preempt all other handlers
+ *
+ * If a wdd->ops->restart function is provided when watchdog_register_device is
+ * called, it will be registered as a restart handler with the priority given
+ * here.
+ */
+void watchdog_set_restart_priority(struct watchdog_device *wdd, int priority)
+{
+	wdd->restart_nb.priority = priority;
+}
+EXPORT_SYMBOL_GPL(watchdog_set_restart_priority);
+
 static int __watchdog_register_device(struct watchdog_device *wdd)
 {
-	int ret, id = -1, devno;
+	int ret, id = -1;
 
 	if (wdd == NULL || wdd->info == NULL || wdd->ops == NULL)
 		return -EINVAL;
@@ -156,8 +210,6 @@ static int __watchdog_register_device(struct watchdog_device *wdd)
 	 * corrupted in a later stage then we expect a kernel panic!
 	 */
 
-	mutex_init(&wdd->lock);
-
 	/* Use alias for watchdog id if possible */
 	if (wdd->parent) {
 		ret = of_alias_get_id(wdd->parent->of_node, "watchdog");
@@ -192,14 +244,26 @@ static int __watchdog_register_device(struct watchdog_device *wdd)
 		}
 	}
 
-	devno = wdd->cdev.dev;
-	wdd->dev = device_create(watchdog_class, wdd->parent, devno,
-					NULL, "watchdog%d", wdd->id);
-	if (IS_ERR(wdd->dev)) {
-		watchdog_dev_unregister(wdd);
-		ida_simple_remove(&watchdog_ida, id);
-		ret = PTR_ERR(wdd->dev);
-		return ret;
+	if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) {
+		wdd->reboot_nb.notifier_call = watchdog_reboot_notifier;
+
+		ret = register_reboot_notifier(&wdd->reboot_nb);
+		if (ret) {
+			pr_err("watchdog%d: Cannot register reboot notifier (%d)\n",
+			       wdd->id, ret);
+			watchdog_dev_unregister(wdd);
+			ida_simple_remove(&watchdog_ida, wdd->id);
+			return ret;
+		}
+	}
+
+	if (wdd->ops->restart) {
+		wdd->restart_nb.notifier_call = watchdog_restart_notifier;
+
+		ret = register_restart_handler(&wdd->restart_nb);
+		if (ret)
+			pr_warn("watchog%d: Cannot register restart handler (%d)\n",
+				wdd->id, ret);
 	}
 
 	return 0;
@@ -232,19 +296,17 @@ EXPORT_SYMBOL_GPL(watchdog_register_device);
 
 static void __watchdog_unregister_device(struct watchdog_device *wdd)
 {
-	int ret;
-	int devno;
-
 	if (wdd == NULL)
 		return;
 
-	devno = wdd->cdev.dev;
-	ret = watchdog_dev_unregister(wdd);
-	if (ret)
-		pr_err("error unregistering /dev/watchdog (err=%d)\n", ret);
-	device_destroy(watchdog_class, devno);
+	if (wdd->ops->restart)
+		unregister_restart_handler(&wdd->restart_nb);
+
+	if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status))
+		unregister_reboot_notifier(&wdd->reboot_nb);
+
+	watchdog_dev_unregister(wdd);
 	ida_simple_remove(&watchdog_ida, wdd->id);
-	wdd->dev = NULL;
 }
 
 /**
@@ -287,17 +349,9 @@ static int __init watchdog_init(void)
 {
 	int err;
 
-	watchdog_class = class_create(THIS_MODULE, "watchdog");
-	if (IS_ERR(watchdog_class)) {
-		pr_err("couldn't create class\n");
-		return PTR_ERR(watchdog_class);
-	}
-
 	err = watchdog_dev_init();
-	if (err < 0) {
-		class_destroy(watchdog_class);
+	if (err < 0)
 		return err;
-	}
 
 	watchdog_deferred_registration();
 	return 0;
@@ -306,7 +360,6 @@ static int __init watchdog_init(void)
 static void __exit watchdog_exit(void)
 {
 	watchdog_dev_exit();
-	class_destroy(watchdog_class);
 	ida_destroy(&watchdog_ida);
 }
 
diff --git a/drivers/watchdog/watchdog_core.h b/drivers/watchdog/watchdog_core.h
index 6c951418fca7..86ff962d1e15 100644
--- a/drivers/watchdog/watchdog_core.h
+++ b/drivers/watchdog/watchdog_core.h
@@ -32,6 +32,6 @@
  *	Functions/procedures to be called by the core
  */
 extern int watchdog_dev_register(struct watchdog_device *);
-extern int watchdog_dev_unregister(struct watchdog_device *);
+extern void watchdog_dev_unregister(struct watchdog_device *);
 extern int __init watchdog_dev_init(void);
 extern void __exit watchdog_dev_exit(void);
diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c
index 56a649e66eb2..ba2ecce4aae6 100644
--- a/drivers/watchdog/watchdog_dev.c
+++ b/drivers/watchdog/watchdog_dev.c
@@ -32,27 +32,51 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/module.h>	/* For module stuff/... */
-#include <linux/types.h>	/* For standard types (like size_t) */
+#include <linux/cdev.h>		/* For character device */
 #include <linux/errno.h>	/* For the -ENODEV/... values */
-#include <linux/kernel.h>	/* For printk/panic/... */
 #include <linux/fs.h>		/* For file operations */
-#include <linux/watchdog.h>	/* For watchdog specific items */
-#include <linux/miscdevice.h>	/* For handling misc devices */
 #include <linux/init.h>		/* For __init/__exit/... */
+#include <linux/kernel.h>	/* For printk/panic/... */
+#include <linux/kref.h>		/* For data references */
+#include <linux/miscdevice.h>	/* For handling misc devices */
+#include <linux/module.h>	/* For module stuff/... */
+#include <linux/mutex.h>	/* For mutexes */
+#include <linux/slab.h>		/* For memory functions */
+#include <linux/types.h>	/* For standard types (like size_t) */
+#include <linux/watchdog.h>	/* For watchdog specific items */
 #include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
 
 #include "watchdog_core.h"
 
+/*
+ * struct watchdog_core_data - watchdog core internal data
+ * @kref:	Reference count.
+ * @cdev:	The watchdog's Character device.
+ * @wdd:	Pointer to watchdog device.
+ * @lock:	Lock for watchdog core.
+ * @status:	Watchdog core internal status bits.
+ */
+struct watchdog_core_data {
+	struct kref kref;
+	struct cdev cdev;
+	struct watchdog_device *wdd;
+	struct mutex lock;
+	unsigned long status;		/* Internal status bits */
+#define _WDOG_DEV_OPEN		0	/* Opened ? */
+#define _WDOG_ALLOW_RELEASE	1	/* Did we receive the magic char ? */
+};
+
 /* the dev_t structure to store the dynamically allocated watchdog devices */
 static dev_t watchdog_devt;
-/* the watchdog device behind /dev/watchdog */
-static struct watchdog_device *old_wdd;
+/* Reference to watchdog device behind /dev/watchdog */
+static struct watchdog_core_data *old_wd_data;
 
 /*
  *	watchdog_ping: ping the watchdog.
  *	@wdd: the watchdog device to ping
  *
+ *	The caller must hold wd_data->lock.
+ *
  *	If the watchdog has no own ping operation then it needs to be
  *	restarted via the start operation. This wrapper function does
  *	exactly that.
@@ -61,25 +85,16 @@ static struct watchdog_device *old_wdd;
 
 static int watchdog_ping(struct watchdog_device *wdd)
 {
-	int err = 0;
-
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_ping;
-	}
+	int err;
 
 	if (!watchdog_active(wdd))
-		goto out_ping;
+		return 0;
 
 	if (wdd->ops->ping)
 		err = wdd->ops->ping(wdd);	/* ping the watchdog */
 	else
 		err = wdd->ops->start(wdd);	/* restart watchdog */
 
-out_ping:
-	mutex_unlock(&wdd->lock);
 	return err;
 }
 
@@ -87,6 +102,8 @@ out_ping:
  *	watchdog_start: wrapper to start the watchdog.
  *	@wdd: the watchdog device to start
  *
+ *	The caller must hold wd_data->lock.
+ *
  *	Start the watchdog if it is not active and mark it active.
  *	This function returns zero on success or a negative errno code for
  *	failure.
@@ -94,24 +111,15 @@ out_ping:
 
 static int watchdog_start(struct watchdog_device *wdd)
 {
-	int err = 0;
-
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_start;
-	}
+	int err;
 
 	if (watchdog_active(wdd))
-		goto out_start;
+		return 0;
 
 	err = wdd->ops->start(wdd);
 	if (err == 0)
 		set_bit(WDOG_ACTIVE, &wdd->status);
 
-out_start:
-	mutex_unlock(&wdd->lock);
 	return err;
 }
 
@@ -119,6 +127,8 @@ out_start:
  *	watchdog_stop: wrapper to stop the watchdog.
  *	@wdd: the watchdog device to stop
  *
+ *	The caller must hold wd_data->lock.
+ *
  *	Stop the watchdog if it is still active and unmark it active.
  *	This function returns zero on success or a negative errno code for
  *	failure.
@@ -127,93 +137,59 @@ out_start:
 
 static int watchdog_stop(struct watchdog_device *wdd)
 {
-	int err = 0;
-
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_stop;
-	}
+	int err;
 
 	if (!watchdog_active(wdd))
-		goto out_stop;
+		return 0;
 
 	if (test_bit(WDOG_NO_WAY_OUT, &wdd->status)) {
-		dev_info(wdd->dev, "nowayout prevents watchdog being stopped!\n");
-		err = -EBUSY;
-		goto out_stop;
+		pr_info("watchdog%d: nowayout prevents watchdog being stopped!\n",
+			wdd->id);
+		return -EBUSY;
 	}
 
 	err = wdd->ops->stop(wdd);
 	if (err == 0)
 		clear_bit(WDOG_ACTIVE, &wdd->status);
 
-out_stop:
-	mutex_unlock(&wdd->lock);
 	return err;
 }
 
 /*
  *	watchdog_get_status: wrapper to get the watchdog status
  *	@wdd: the watchdog device to get the status from
- *	@status: the status of the watchdog device
+ *
+ *	The caller must hold wd_data->lock.
  *
  *	Get the watchdog's status flags.
  */
 
-static int watchdog_get_status(struct watchdog_device *wdd,
-							unsigned int *status)
+static unsigned int watchdog_get_status(struct watchdog_device *wdd)
 {
-	int err = 0;
-
-	*status = 0;
 	if (!wdd->ops->status)
-		return -EOPNOTSUPP;
-
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_status;
-	}
-
-	*status = wdd->ops->status(wdd);
+		return 0;
 
-out_status:
-	mutex_unlock(&wdd->lock);
-	return err;
+	return wdd->ops->status(wdd);
 }
 
 /*
  *	watchdog_set_timeout: set the watchdog timer timeout
  *	@wdd: the watchdog device to set the timeout for
  *	@timeout: timeout to set in seconds
+ *
+ *	The caller must hold wd_data->lock.
  */
 
 static int watchdog_set_timeout(struct watchdog_device *wdd,
 							unsigned int timeout)
 {
-	int err;
-
 	if (!wdd->ops->set_timeout || !(wdd->info->options & WDIOF_SETTIMEOUT))
 		return -EOPNOTSUPP;
 
 	if (watchdog_timeout_invalid(wdd, timeout))
 		return -EINVAL;
 
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_timeout;
-	}
-
-	err = wdd->ops->set_timeout(wdd, timeout);
-
-out_timeout:
-	mutex_unlock(&wdd->lock);
-	return err;
+	return wdd->ops->set_timeout(wdd, timeout);
 }
 
 /*
@@ -221,59 +197,156 @@ out_timeout:
  *	@wdd: the watchdog device to get the remaining time from
  *	@timeleft: the time that's left
  *
+ *	The caller must hold wd_data->lock.
+ *
  *	Get the time before a watchdog will reboot (if not pinged).
  */
 
 static int watchdog_get_timeleft(struct watchdog_device *wdd,
 							unsigned int *timeleft)
 {
-	int err = 0;
-
 	*timeleft = 0;
+
 	if (!wdd->ops->get_timeleft)
 		return -EOPNOTSUPP;
 
-	mutex_lock(&wdd->lock);
+	*timeleft = wdd->ops->get_timeleft(wdd);
 
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_timeleft;
-	}
+	return 0;
+}
 
-	*timeleft = wdd->ops->get_timeleft(wdd);
+#ifdef CONFIG_WATCHDOG_SYSFS
+static ssize_t nowayout_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
 
-out_timeleft:
-	mutex_unlock(&wdd->lock);
-	return err;
+	return sprintf(buf, "%d\n", !!test_bit(WDOG_NO_WAY_OUT, &wdd->status));
+}
+static DEVICE_ATTR_RO(nowayout);
+
+static ssize_t status_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+	struct watchdog_core_data *wd_data = wdd->wd_data;
+	unsigned int status;
+
+	mutex_lock(&wd_data->lock);
+	status = watchdog_get_status(wdd);
+	mutex_unlock(&wd_data->lock);
+
+	return sprintf(buf, "%u\n", status);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t bootstatus_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%u\n", wdd->bootstatus);
+}
+static DEVICE_ATTR_RO(bootstatus);
+
+static ssize_t timeleft_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+	struct watchdog_core_data *wd_data = wdd->wd_data;
+	ssize_t status;
+	unsigned int val;
+
+	mutex_lock(&wd_data->lock);
+	status = watchdog_get_timeleft(wdd, &val);
+	mutex_unlock(&wd_data->lock);
+	if (!status)
+		status = sprintf(buf, "%u\n", val);
+
+	return status;
 }
+static DEVICE_ATTR_RO(timeleft);
+
+static ssize_t timeout_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%u\n", wdd->timeout);
+}
+static DEVICE_ATTR_RO(timeout);
+
+static ssize_t identity_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%s\n", wdd->info->identity);
+}
+static DEVICE_ATTR_RO(identity);
+
+static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	if (watchdog_active(wdd))
+		return sprintf(buf, "active\n");
+
+	return sprintf(buf, "inactive\n");
+}
+static DEVICE_ATTR_RO(state);
+
+static umode_t wdt_is_visible(struct kobject *kobj, struct attribute *attr,
+				int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+	umode_t mode = attr->mode;
+
+	if (attr == &dev_attr_status.attr && !wdd->ops->status)
+		mode = 0;
+	else if (attr == &dev_attr_timeleft.attr && !wdd->ops->get_timeleft)
+		mode = 0;
+
+	return mode;
+}
+static struct attribute *wdt_attrs[] = {
+	&dev_attr_state.attr,
+	&dev_attr_identity.attr,
+	&dev_attr_timeout.attr,
+	&dev_attr_timeleft.attr,
+	&dev_attr_bootstatus.attr,
+	&dev_attr_status.attr,
+	&dev_attr_nowayout.attr,
+	NULL,
+};
+
+static const struct attribute_group wdt_group = {
+	.attrs = wdt_attrs,
+	.is_visible = wdt_is_visible,
+};
+__ATTRIBUTE_GROUPS(wdt);
+#else
+#define wdt_groups	NULL
+#endif
 
 /*
  *	watchdog_ioctl_op: call the watchdog drivers ioctl op if defined
  *	@wdd: the watchdog device to do the ioctl on
  *	@cmd: watchdog command
  *	@arg: argument pointer
+ *
+ *	The caller must hold wd_data->lock.
  */
 
 static int watchdog_ioctl_op(struct watchdog_device *wdd, unsigned int cmd,
 							unsigned long arg)
 {
-	int err;
-
 	if (!wdd->ops->ioctl)
 		return -ENOIOCTLCMD;
 
-	mutex_lock(&wdd->lock);
-
-	if (test_bit(WDOG_UNREGISTERED, &wdd->status)) {
-		err = -ENODEV;
-		goto out_ioctl;
-	}
-
-	err = wdd->ops->ioctl(wdd, cmd, arg);
-
-out_ioctl:
-	mutex_unlock(&wdd->lock);
-	return err;
+	return wdd->ops->ioctl(wdd, cmd, arg);
 }
 
 /*
@@ -291,10 +364,11 @@ out_ioctl:
 static ssize_t watchdog_write(struct file *file, const char __user *data,
 						size_t len, loff_t *ppos)
 {
-	struct watchdog_device *wdd = file->private_data;
+	struct watchdog_core_data *wd_data = file->private_data;
+	struct watchdog_device *wdd;
+	int err;
 	size_t i;
 	char c;
-	int err;
 
 	if (len == 0)
 		return 0;
@@ -303,18 +377,25 @@ static ssize_t watchdog_write(struct file *file, const char __user *data,
 	 * Note: just in case someone wrote the magic character
 	 * five months ago...
 	 */
-	clear_bit(WDOG_ALLOW_RELEASE, &wdd->status);
+	clear_bit(_WDOG_ALLOW_RELEASE, &wd_data->status);
 
 	/* scan to see whether or not we got the magic character */
 	for (i = 0; i != len; i++) {
 		if (get_user(c, data + i))
 			return -EFAULT;
 		if (c == 'V')
-			set_bit(WDOG_ALLOW_RELEASE, &wdd->status);
+			set_bit(_WDOG_ALLOW_RELEASE, &wd_data->status);
 	}
 
 	/* someone wrote to us, so we send the watchdog a keepalive ping */
-	err = watchdog_ping(wdd);
+
+	err = -ENODEV;
+	mutex_lock(&wd_data->lock);
+	wdd = wd_data->wdd;
+	if (wdd)
+		err = watchdog_ping(wdd);
+	mutex_unlock(&wd_data->lock);
+
 	if (err < 0)
 		return err;
 
@@ -334,71 +415,94 @@ static ssize_t watchdog_write(struct file *file, const char __user *data,
 static long watchdog_ioctl(struct file *file, unsigned int cmd,
 							unsigned long arg)
 {
-	struct watchdog_device *wdd = file->private_data;
+	struct watchdog_core_data *wd_data = file->private_data;
 	void __user *argp = (void __user *)arg;
+	struct watchdog_device *wdd;
 	int __user *p = argp;
 	unsigned int val;
 	int err;
 
+	mutex_lock(&wd_data->lock);
+
+	wdd = wd_data->wdd;
+	if (!wdd) {
+		err = -ENODEV;
+		goto out_ioctl;
+	}
+
 	err = watchdog_ioctl_op(wdd, cmd, arg);
 	if (err != -ENOIOCTLCMD)
-		return err;
+		goto out_ioctl;
 
 	switch (cmd) {
 	case WDIOC_GETSUPPORT:
-		return copy_to_user(argp, wdd->info,
+		err = copy_to_user(argp, wdd->info,
 			sizeof(struct watchdog_info)) ? -EFAULT : 0;
+		break;
 	case WDIOC_GETSTATUS:
-		err = watchdog_get_status(wdd, &val);
-		if (err == -ENODEV)
-			return err;
-		return put_user(val, p);
+		val = watchdog_get_status(wdd);
+		err = put_user(val, p);
+		break;
 	case WDIOC_GETBOOTSTATUS:
-		return put_user(wdd->bootstatus, p);
+		err = put_user(wdd->bootstatus, p);
+		break;
 	case WDIOC_SETOPTIONS:
-		if (get_user(val, p))
-			return -EFAULT;
+		if (get_user(val, p)) {
+			err = -EFAULT;
+			break;
+		}
 		if (val & WDIOS_DISABLECARD) {
 			err = watchdog_stop(wdd);
 			if (err < 0)
-				return err;
+				break;
 		}
-		if (val & WDIOS_ENABLECARD) {
+		if (val & WDIOS_ENABLECARD)
 			err = watchdog_start(wdd);
-			if (err < 0)
-				return err;
-		}
-		return 0;
+		break;
 	case WDIOC_KEEPALIVE:
-		if (!(wdd->info->options & WDIOF_KEEPALIVEPING))
-			return -EOPNOTSUPP;
-		return watchdog_ping(wdd);
+		if (!(wdd->info->options & WDIOF_KEEPALIVEPING)) {
+			err = -EOPNOTSUPP;
+			break;
+		}
+		err = watchdog_ping(wdd);
+		break;
 	case WDIOC_SETTIMEOUT:
-		if (get_user(val, p))
-			return -EFAULT;
+		if (get_user(val, p)) {
+			err = -EFAULT;
+			break;
+		}
 		err = watchdog_set_timeout(wdd, val);
 		if (err < 0)
-			return err;
+			break;
 		/* If the watchdog is active then we send a keepalive ping
 		 * to make sure that the watchdog keep's running (and if
 		 * possible that it takes the new timeout) */
 		err = watchdog_ping(wdd);
 		if (err < 0)
-			return err;
+			break;
 		/* Fall */
 	case WDIOC_GETTIMEOUT:
 		/* timeout == 0 means that we don't know the timeout */
-		if (wdd->timeout == 0)
-			return -EOPNOTSUPP;
-		return put_user(wdd->timeout, p);
+		if (wdd->timeout == 0) {
+			err = -EOPNOTSUPP;
+			break;
+		}
+		err = put_user(wdd->timeout, p);
+		break;
 	case WDIOC_GETTIMELEFT:
 		err = watchdog_get_timeleft(wdd, &val);
-		if (err)
-			return err;
-		return put_user(val, p);
+		if (err < 0)
+			break;
+		err = put_user(val, p);
+		break;
 	default:
-		return -ENOTTY;
+		err = -ENOTTY;
+		break;
 	}
+
+out_ioctl:
+	mutex_unlock(&wd_data->lock);
+	return err;
 }
 
 /*
@@ -413,45 +517,59 @@ static long watchdog_ioctl(struct file *file, unsigned int cmd,
 
 static int watchdog_open(struct inode *inode, struct file *file)
 {
-	int err = -EBUSY;
+	struct watchdog_core_data *wd_data;
 	struct watchdog_device *wdd;
+	int err;
 
 	/* Get the corresponding watchdog device */
 	if (imajor(inode) == MISC_MAJOR)
-		wdd = old_wdd;
+		wd_data = old_wd_data;
 	else
-		wdd = container_of(inode->i_cdev, struct watchdog_device, cdev);
+		wd_data = container_of(inode->i_cdev, struct watchdog_core_data,
+				       cdev);
 
 	/* the watchdog is single open! */
-	if (test_and_set_bit(WDOG_DEV_OPEN, &wdd->status))
+	if (test_and_set_bit(_WDOG_DEV_OPEN, &wd_data->status))
 		return -EBUSY;
 
+	wdd = wd_data->wdd;
+
 	/*
 	 * If the /dev/watchdog device is open, we don't want the module
 	 * to be unloaded.
 	 */
-	if (!try_module_get(wdd->ops->owner))
-		goto out;
+	if (!try_module_get(wdd->ops->owner)) {
+		err = -EBUSY;
+		goto out_clear;
+	}
 
 	err = watchdog_start(wdd);
 	if (err < 0)
 		goto out_mod;
 
-	file->private_data = wdd;
+	file->private_data = wd_data;
 
-	if (wdd->ops->ref)
-		wdd->ops->ref(wdd);
+	kref_get(&wd_data->kref);
 
 	/* dev/watchdog is a virtual (and thus non-seekable) filesystem */
 	return nonseekable_open(inode, file);
 
 out_mod:
-	module_put(wdd->ops->owner);
-out:
-	clear_bit(WDOG_DEV_OPEN, &wdd->status);
+	module_put(wd_data->wdd->ops->owner);
+out_clear:
+	clear_bit(_WDOG_DEV_OPEN, &wd_data->status);
 	return err;
 }
 
+static void watchdog_core_data_release(struct kref *kref)
+{
+	struct watchdog_core_data *wd_data;
+
+	wd_data = container_of(kref, struct watchdog_core_data, kref);
+
+	kfree(wd_data);
+}
+
 /*
  *	watchdog_release: release the watchdog device.
  *	@inode: inode of device
@@ -464,9 +582,16 @@ out:
 
 static int watchdog_release(struct inode *inode, struct file *file)
 {
-	struct watchdog_device *wdd = file->private_data;
+	struct watchdog_core_data *wd_data = file->private_data;
+	struct watchdog_device *wdd;
 	int err = -EBUSY;
 
+	mutex_lock(&wd_data->lock);
+
+	wdd = wd_data->wdd;
+	if (!wdd)
+		goto done;
+
 	/*
 	 * We only stop the watchdog if we received the magic character
 	 * or if WDIOF_MAGICCLOSE is not set. If nowayout was set then
@@ -474,29 +599,24 @@ static int watchdog_release(struct inode *inode, struct file *file)
 	 */
 	if (!test_bit(WDOG_ACTIVE, &wdd->status))
 		err = 0;
-	else if (test_and_clear_bit(WDOG_ALLOW_RELEASE, &wdd->status) ||
+	else if (test_and_clear_bit(_WDOG_ALLOW_RELEASE, &wd_data->status) ||
 		 !(wdd->info->options & WDIOF_MAGICCLOSE))
 		err = watchdog_stop(wdd);
 
 	/* If the watchdog was not stopped, send a keepalive ping */
 	if (err < 0) {
-		mutex_lock(&wdd->lock);
-		if (!test_bit(WDOG_UNREGISTERED, &wdd->status))
-			dev_crit(wdd->dev, "watchdog did not stop!\n");
-		mutex_unlock(&wdd->lock);
+		pr_crit("watchdog%d: watchdog did not stop!\n", wdd->id);
 		watchdog_ping(wdd);
 	}
 
-	/* Allow the owner module to be unloaded again */
-	module_put(wdd->ops->owner);
-
 	/* make sure that /dev/watchdog can be re-opened */
-	clear_bit(WDOG_DEV_OPEN, &wdd->status);
-
-	/* Note wdd may be gone after this, do not use after this! */
-	if (wdd->ops->unref)
-		wdd->ops->unref(wdd);
+	clear_bit(_WDOG_DEV_OPEN, &wd_data->status);
 
+done:
+	mutex_unlock(&wd_data->lock);
+	/* Allow the owner module to be unloaded again */
+	module_put(wd_data->cdev.owner);
+	kref_put(&wd_data->kref, watchdog_core_data_release);
 	return 0;
 }
 
@@ -515,20 +635,31 @@ static struct miscdevice watchdog_miscdev = {
 };
 
 /*
- *	watchdog_dev_register: register a watchdog device
+ *	watchdog_cdev_register: register watchdog character device
  *	@wdd: watchdog device
+ *	@devno: character device number
  *
- *	Register a watchdog device including handling the legacy
+ *	Register a watchdog character device including handling the legacy
  *	/dev/watchdog node. /dev/watchdog is actually a miscdevice and
  *	thus we set it up like that.
  */
 
-int watchdog_dev_register(struct watchdog_device *wdd)
+static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno)
 {
-	int err, devno;
+	struct watchdog_core_data *wd_data;
+	int err;
+
+	wd_data = kzalloc(sizeof(struct watchdog_core_data), GFP_KERNEL);
+	if (!wd_data)
+		return -ENOMEM;
+	kref_init(&wd_data->kref);
+	mutex_init(&wd_data->lock);
+
+	wd_data->wdd = wdd;
+	wdd->wd_data = wd_data;
 
 	if (wdd->id == 0) {
-		old_wdd = wdd;
+		old_wd_data = wd_data;
 		watchdog_miscdev.parent = wdd->parent;
 		err = misc_register(&watchdog_miscdev);
 		if (err != 0) {
@@ -537,48 +668,106 @@ int watchdog_dev_register(struct watchdog_device *wdd)
 			if (err == -EBUSY)
 				pr_err("%s: a legacy watchdog module is probably present.\n",
 					wdd->info->identity);
-			old_wdd = NULL;
+			old_wd_data = NULL;
+			kfree(wd_data);
 			return err;
 		}
 	}
 
 	/* Fill in the data structures */
-	devno = MKDEV(MAJOR(watchdog_devt), wdd->id);
-	cdev_init(&wdd->cdev, &watchdog_fops);
-	wdd->cdev.owner = wdd->ops->owner;
+	cdev_init(&wd_data->cdev, &watchdog_fops);
+	wd_data->cdev.owner = wdd->ops->owner;
 
 	/* Add the device */
-	err  = cdev_add(&wdd->cdev, devno, 1);
+	err = cdev_add(&wd_data->cdev, devno, 1);
 	if (err) {
 		pr_err("watchdog%d unable to add device %d:%d\n",
 			wdd->id,  MAJOR(watchdog_devt), wdd->id);
 		if (wdd->id == 0) {
 			misc_deregister(&watchdog_miscdev);
-			old_wdd = NULL;
+			old_wd_data = NULL;
+			kref_put(&wd_data->kref, watchdog_core_data_release);
 		}
 	}
 	return err;
 }
 
 /*
- *	watchdog_dev_unregister: unregister a watchdog device
+ *	watchdog_cdev_unregister: unregister watchdog character device
  *	@watchdog: watchdog device
  *
- *	Unregister the watchdog and if needed the legacy /dev/watchdog device.
+ *	Unregister watchdog character device and if needed the legacy
+ *	/dev/watchdog device.
  */
 
-int watchdog_dev_unregister(struct watchdog_device *wdd)
+static void watchdog_cdev_unregister(struct watchdog_device *wdd)
 {
-	mutex_lock(&wdd->lock);
-	set_bit(WDOG_UNREGISTERED, &wdd->status);
-	mutex_unlock(&wdd->lock);
+	struct watchdog_core_data *wd_data = wdd->wd_data;
 
-	cdev_del(&wdd->cdev);
+	cdev_del(&wd_data->cdev);
 	if (wdd->id == 0) {
 		misc_deregister(&watchdog_miscdev);
-		old_wdd = NULL;
+		old_wd_data = NULL;
 	}
-	return 0;
+
+	mutex_lock(&wd_data->lock);
+	wd_data->wdd = NULL;
+	wdd->wd_data = NULL;
+	mutex_unlock(&wd_data->lock);
+
+	kref_put(&wd_data->kref, watchdog_core_data_release);
+}
+
+static struct class watchdog_class = {
+	.name =		"watchdog",
+	.owner =	THIS_MODULE,
+	.dev_groups =	wdt_groups,
+};
+
+/*
+ *	watchdog_dev_register: register a watchdog device
+ *	@wdd: watchdog device
+ *
+ *	Register a watchdog device including handling the legacy
+ *	/dev/watchdog node. /dev/watchdog is actually a miscdevice and
+ *	thus we set it up like that.
+ */
+
+int watchdog_dev_register(struct watchdog_device *wdd)
+{
+	struct device *dev;
+	dev_t devno;
+	int ret;
+
+	devno = MKDEV(MAJOR(watchdog_devt), wdd->id);
+
+	ret = watchdog_cdev_register(wdd, devno);
+	if (ret)
+		return ret;
+
+	dev = device_create_with_groups(&watchdog_class, wdd->parent,
+					devno, wdd, wdd->groups,
+					"watchdog%d", wdd->id);
+	if (IS_ERR(dev)) {
+		watchdog_cdev_unregister(wdd);
+		return PTR_ERR(dev);
+	}
+
+	return ret;
+}
+
+/*
+ *	watchdog_dev_unregister: unregister a watchdog device
+ *	@watchdog: watchdog device
+ *
+ *	Unregister watchdog device and if needed the legacy
+ *	/dev/watchdog device.
+ */
+
+void watchdog_dev_unregister(struct watchdog_device *wdd)
+{
+	device_destroy(&watchdog_class, wdd->wd_data->cdev.dev);
+	watchdog_cdev_unregister(wdd);
 }
 
 /*
@@ -589,10 +778,22 @@ int watchdog_dev_unregister(struct watchdog_device *wdd)
 
 int __init watchdog_dev_init(void)
 {
-	int err = alloc_chrdev_region(&watchdog_devt, 0, MAX_DOGS, "watchdog");
-	if (err < 0)
+	int err;
+
+	err = class_register(&watchdog_class);
+	if (err < 0) {
+		pr_err("couldn't register class\n");
+		return err;
+	}
+
+	err = alloc_chrdev_region(&watchdog_devt, 0, MAX_DOGS, "watchdog");
+	if (err < 0) {
 		pr_err("watchdog: unable to allocate char dev region\n");
-	return err;
+		class_unregister(&watchdog_class);
+		return err;
+	}
+
+	return 0;
 }
 
 /*
@@ -604,4 +805,5 @@ int __init watchdog_dev_init(void)
 void __exit watchdog_dev_exit(void)
 {
 	unregister_chrdev_region(watchdog_devt, MAX_DOGS);
+	class_unregister(&watchdog_class);
 }
diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c
new file mode 100644
index 000000000000..0c7cb7302cf0
--- /dev/null
+++ b/drivers/watchdog/ziirave_wdt.c
@@ -0,0 +1,367 @@
+/*
+ * Copyright (C) 2015 Zodiac Inflight Innovations
+ *
+ * Author: Martyn Welch <martyn.welch@collabora.co.uk>
+ *
+ * Based on twl4030_wdt.c by Timo Kokkonen <timo.t.kokkonen at nokia.com>:
+ *
+ * Copyright (C) Nokia Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/version.h>
+#include <linux/watchdog.h>
+
+#define ZIIRAVE_TIMEOUT_MIN	3
+#define ZIIRAVE_TIMEOUT_MAX	255
+
+#define ZIIRAVE_PING_VALUE	0x0
+
+#define ZIIRAVE_STATE_INITIAL	0x0
+#define ZIIRAVE_STATE_OFF	0x1
+#define ZIIRAVE_STATE_ON	0x2
+
+static char *ziirave_reasons[] = {"power cycle", "triggered", NULL, NULL,
+				  "host request", NULL, "illegal configuration",
+				  "illegal instruction", "illegal trap",
+				  "unknown"};
+
+#define ZIIRAVE_WDT_FIRM_VER_MAJOR	0x1
+#define ZIIRAVE_WDT_BOOT_VER_MAJOR	0x3
+#define ZIIRAVE_WDT_RESET_REASON	0x5
+#define ZIIRAVE_WDT_STATE		0x6
+#define ZIIRAVE_WDT_TIMEOUT		0x7
+#define ZIIRAVE_WDT_TIME_LEFT		0x8
+#define ZIIRAVE_WDT_PING		0x9
+#define ZIIRAVE_WDT_RESET_DURATION	0xa
+
+struct ziirave_wdt_rev {
+	unsigned char major;
+	unsigned char minor;
+};
+
+struct ziirave_wdt_data {
+	struct watchdog_device wdd;
+	struct ziirave_wdt_rev bootloader_rev;
+	struct ziirave_wdt_rev firmware_rev;
+	int reset_reason;
+};
+
+static int wdt_timeout;
+module_param(wdt_timeout, int, 0);
+MODULE_PARM_DESC(wdt_timeout, "Watchdog timeout in seconds");
+
+static int reset_duration;
+module_param(reset_duration, int, 0);
+MODULE_PARM_DESC(reset_duration,
+		 "Watchdog reset pulse duration in milliseconds");
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+static int ziirave_wdt_revision(struct i2c_client *client,
+				struct ziirave_wdt_rev *rev, u8 command)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, command);
+	if (ret < 0)
+		return ret;
+
+	rev->major = ret;
+
+	ret = i2c_smbus_read_byte_data(client, command + 1);
+	if (ret < 0)
+		return ret;
+
+	rev->minor = ret;
+
+	return 0;
+}
+
+static int ziirave_wdt_set_state(struct watchdog_device *wdd, int state)
+{
+	struct i2c_client *client = to_i2c_client(wdd->parent);
+
+	return i2c_smbus_write_byte_data(client, ZIIRAVE_WDT_STATE, state);
+}
+
+static int ziirave_wdt_start(struct watchdog_device *wdd)
+{
+	return ziirave_wdt_set_state(wdd, ZIIRAVE_STATE_ON);
+}
+
+static int ziirave_wdt_stop(struct watchdog_device *wdd)
+{
+	return ziirave_wdt_set_state(wdd, ZIIRAVE_STATE_OFF);
+}
+
+static int ziirave_wdt_ping(struct watchdog_device *wdd)
+{
+	struct i2c_client *client = to_i2c_client(wdd->parent);
+
+	return i2c_smbus_write_byte_data(client, ZIIRAVE_WDT_PING,
+					 ZIIRAVE_PING_VALUE);
+}
+
+static int ziirave_wdt_set_timeout(struct watchdog_device *wdd,
+				   unsigned int timeout)
+{
+	struct i2c_client *client = to_i2c_client(wdd->parent);
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(client, ZIIRAVE_WDT_TIMEOUT, timeout);
+	if (!ret)
+		wdd->timeout = timeout;
+
+	return ret;
+}
+
+static unsigned int ziirave_wdt_get_timeleft(struct watchdog_device *wdd)
+{
+	struct i2c_client *client = to_i2c_client(wdd->parent);
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, ZIIRAVE_WDT_TIME_LEFT);
+	if (ret < 0)
+		ret = 0;
+
+	return ret;
+}
+
+static const struct watchdog_info ziirave_wdt_info = {
+	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
+	.identity = "Zodiac RAVE Watchdog",
+};
+
+static const struct watchdog_ops ziirave_wdt_ops = {
+	.owner		= THIS_MODULE,
+	.start		= ziirave_wdt_start,
+	.stop		= ziirave_wdt_stop,
+	.ping		= ziirave_wdt_ping,
+	.set_timeout	= ziirave_wdt_set_timeout,
+	.get_timeleft	= ziirave_wdt_get_timeleft,
+};
+
+static ssize_t ziirave_wdt_sysfs_show_firm(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client);
+
+	return sprintf(buf, "02.%02u.%02u", w_priv->firmware_rev.major,
+		       w_priv->firmware_rev.minor);
+}
+
+static DEVICE_ATTR(firmware_version, S_IRUGO, ziirave_wdt_sysfs_show_firm,
+		   NULL);
+
+static ssize_t ziirave_wdt_sysfs_show_boot(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client);
+
+	return sprintf(buf, "01.%02u.%02u", w_priv->bootloader_rev.major,
+		       w_priv->bootloader_rev.minor);
+}
+
+static DEVICE_ATTR(bootloader_version, S_IRUGO, ziirave_wdt_sysfs_show_boot,
+		   NULL);
+
+static ssize_t ziirave_wdt_sysfs_show_reason(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client);
+
+	return sprintf(buf, "%s", ziirave_reasons[w_priv->reset_reason]);
+}
+
+static DEVICE_ATTR(reset_reason, S_IRUGO, ziirave_wdt_sysfs_show_reason,
+		   NULL);
+
+static struct attribute *ziirave_wdt_attrs[] = {
+	&dev_attr_firmware_version.attr,
+	&dev_attr_bootloader_version.attr,
+	&dev_attr_reset_reason.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(ziirave_wdt);
+
+static int ziirave_wdt_init_duration(struct i2c_client *client)
+{
+	int ret;
+
+	if (!reset_duration) {
+		/* See if the reset pulse duration is provided in an of_node */
+		if (!client->dev.of_node)
+			ret = -ENODEV;
+		else
+			ret = of_property_read_u32(client->dev.of_node,
+						   "reset-duration-ms",
+						   &reset_duration);
+		if (ret) {
+			dev_info(&client->dev,
+				 "Unable to set reset pulse duration, using default\n");
+			return 0;
+		}
+	}
+
+	if (reset_duration < 1 || reset_duration > 255)
+		return -EINVAL;
+
+	dev_info(&client->dev, "Setting reset duration to %dms",
+		 reset_duration);
+
+	return i2c_smbus_write_byte_data(client, ZIIRAVE_WDT_RESET_DURATION,
+					 reset_duration);
+}
+
+static int ziirave_wdt_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	int ret;
+	struct ziirave_wdt_data *w_priv;
+	int val;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA))
+		return -ENODEV;
+
+	w_priv = devm_kzalloc(&client->dev, sizeof(*w_priv), GFP_KERNEL);
+	if (!w_priv)
+		return -ENOMEM;
+
+	w_priv->wdd.info = &ziirave_wdt_info;
+	w_priv->wdd.ops = &ziirave_wdt_ops;
+	w_priv->wdd.min_timeout = ZIIRAVE_TIMEOUT_MIN;
+	w_priv->wdd.max_timeout = ZIIRAVE_TIMEOUT_MAX;
+	w_priv->wdd.parent = &client->dev;
+	w_priv->wdd.groups = ziirave_wdt_groups;
+
+	ret = watchdog_init_timeout(&w_priv->wdd, wdt_timeout, &client->dev);
+	if (ret) {
+		dev_info(&client->dev,
+			 "Unable to select timeout value, using default\n");
+	}
+
+	/*
+	 * The default value set in the watchdog should be perfectly valid, so
+	 * pass that in if we haven't provided one via the module parameter or
+	 * of property.
+	 */
+	if (w_priv->wdd.timeout == 0) {
+		val = i2c_smbus_read_byte_data(client, ZIIRAVE_WDT_TIMEOUT);
+		if (val < 0)
+			return val;
+
+		if (val < ZIIRAVE_TIMEOUT_MIN)
+			return -ENODEV;
+
+		w_priv->wdd.timeout = val;
+	} else {
+		ret = ziirave_wdt_set_timeout(&w_priv->wdd,
+					      w_priv->wdd.timeout);
+		if (ret)
+			return ret;
+
+		dev_info(&client->dev, "Timeout set to %ds.",
+			 w_priv->wdd.timeout);
+	}
+
+	watchdog_set_nowayout(&w_priv->wdd, nowayout);
+
+	i2c_set_clientdata(client, w_priv);
+
+	/* If in unconfigured state, set to stopped */
+	val = i2c_smbus_read_byte_data(client, ZIIRAVE_WDT_STATE);
+	if (val < 0)
+		return val;
+
+	if (val == ZIIRAVE_STATE_INITIAL)
+		ziirave_wdt_stop(&w_priv->wdd);
+
+	ret = ziirave_wdt_init_duration(client);
+	if (ret)
+		return ret;
+
+	ret = ziirave_wdt_revision(client, &w_priv->firmware_rev,
+				   ZIIRAVE_WDT_FIRM_VER_MAJOR);
+	if (ret)
+		return ret;
+
+	ret = ziirave_wdt_revision(client, &w_priv->bootloader_rev,
+				   ZIIRAVE_WDT_BOOT_VER_MAJOR);
+	if (ret)
+		return ret;
+
+	w_priv->reset_reason = i2c_smbus_read_byte_data(client,
+						ZIIRAVE_WDT_RESET_REASON);
+	if (w_priv->reset_reason < 0)
+		return w_priv->reset_reason;
+
+	if (w_priv->reset_reason >= ARRAY_SIZE(ziirave_reasons) ||
+	    !ziirave_reasons[w_priv->reset_reason])
+		return -ENODEV;
+
+	ret = watchdog_register_device(&w_priv->wdd);
+
+	return ret;
+}
+
+static int ziirave_wdt_remove(struct i2c_client *client)
+{
+	struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client);
+
+	watchdog_unregister_device(&w_priv->wdd);
+
+	return 0;
+}
+
+static struct i2c_device_id ziirave_wdt_id[] = {
+	{ "ziirave-wdt", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ziirave_wdt_id);
+
+static const struct of_device_id zrv_wdt_of_match[] = {
+	{ .compatible = "zii,rave-wdt", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, zrv_wdt_of_match);
+
+static struct i2c_driver ziirave_wdt_driver = {
+	.driver = {
+		.name = "ziirave_wdt",
+		.of_match_table = zrv_wdt_of_match,
+	},
+	.probe = ziirave_wdt_probe,
+	.remove = ziirave_wdt_remove,
+	.id_table = ziirave_wdt_id,
+};
+
+module_i2c_driver(ziirave_wdt_driver);
+
+MODULE_AUTHOR("Martyn Welch <martyn.welch@collabora.co.uk");
+MODULE_DESCRIPTION("Zodiac Aerospace RAVE Switch Watchdog Processor Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 96a1b8da5371..eff2b88003d9 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -41,6 +41,7 @@
 #include <linux/percpu.h>
 #include <linux/cpu.h>
 
+#include <asm/barrier.h>
 #include <asm/sync_bitops.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/hypervisor.h>
@@ -296,7 +297,7 @@ static void consume_one_event(unsigned cpu,
 	 * control block.
 	 */
 	if (head == 0) {
-		rmb(); /* Ensure word is up-to-date before reading head. */
+		virt_rmb(); /* Ensure word is up-to-date before reading head. */
 		head = control_block->head[priority];
 	}
 
diff --git a/drivers/xen/tmem.c b/drivers/xen/tmem.c
index 945fc4327201..4ac2ca8a7656 100644
--- a/drivers/xen/tmem.c
+++ b/drivers/xen/tmem.c
@@ -242,7 +242,7 @@ static int tmem_cleancache_init_shared_fs(char *uuid, size_t pagesize)
 	return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
 }
 
-static struct cleancache_ops tmem_cleancache_ops = {
+static const struct cleancache_ops tmem_cleancache_ops = {
 	.put_page = tmem_cleancache_put_page,
 	.get_page = tmem_cleancache_get_page,
 	.invalidate_page = tmem_cleancache_flush_page,
diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c
index fdb0f339d0a7..ecdecce80a6c 100644
--- a/drivers/xen/xenbus/xenbus_comms.c
+++ b/drivers/xen/xenbus/xenbus_comms.c
@@ -123,14 +123,14 @@ int xb_write(const void *data, unsigned len)
 			avail = len;
 
 		/* Must write data /after/ reading the consumer index. */
-		mb();
+		virt_mb();
 
 		memcpy(dst, data, avail);
 		data += avail;
 		len -= avail;
 
 		/* Other side must not see new producer until data is there. */
-		wmb();
+		virt_wmb();
 		intf->req_prod += avail;
 
 		/* Implies mb(): other side will see the updated producer. */
@@ -180,14 +180,14 @@ int xb_read(void *data, unsigned len)
 			avail = len;
 
 		/* Must read data /after/ reading the producer index. */
-		rmb();
+		virt_rmb();
 
 		memcpy(data, src, avail);
 		data += avail;
 		len -= avail;
 
 		/* Other side must not see free space until we've copied out */
-		mb();
+		virt_mb();
 		intf->rsp_cons += avail;
 
 		pr_debug("Finished read of %i bytes (%i to go)\n", avail, len);